diff --git a/patches/5.2/0002-suspend.patch b/patches/5.2/0002-suspend.patch deleted file mode 100644 index e64161a3a..000000000 --- a/patches/5.2/0002-suspend.patch +++ /dev/null @@ -1,281 +0,0 @@ -From 3902051025f388147dea5c8ccec0d6b0a8b87bd2 Mon Sep 17 00:00:00 2001 -From: kitakar5525 <34676735+kitakar5525@users.noreply.github.com> -Date: Wed, 31 Jul 2019 08:41:30 +0900 -Subject: [PATCH 02/12] suspend - -Note: -NVMe part will be merged into Linux 5.3. Remove the part in this -patch when it arrives. ---- - drivers/nvme/host/core.c | 24 ++++++++-- - drivers/nvme/host/nvme.h | 6 +++ - drivers/nvme/host/pci.c | 95 ++++++++++++++++++++++++++++++++++++++-- - kernel/power/suspend.c | 11 +++++ - kernel/sysctl.c | 9 ++++ - 5 files changed, 139 insertions(+), 6 deletions(-) - -diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c -index 963b4c6309b9..4b8cf243c150 100644 ---- a/drivers/nvme/host/core.c -+++ b/drivers/nvme/host/core.c -@@ -1114,15 +1114,15 @@ static struct nvme_id_ns *nvme_identify_ns(struct nvme_ctrl *ctrl, - return id; - } - --static int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11, -- void *buffer, size_t buflen, u32 *result) -+static int nvme_features(struct nvme_ctrl *dev, u8 op, unsigned int fid, -+ unsigned int dword11, void *buffer, size_t buflen, u32 *result) - { - struct nvme_command c; - union nvme_result res; - int ret; - - memset(&c, 0, sizeof(c)); -- c.features.opcode = nvme_admin_set_features; -+ c.features.opcode = op; - c.features.fid = cpu_to_le32(fid); - c.features.dword11 = cpu_to_le32(dword11); - -@@ -1133,6 +1133,24 @@ static int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword - return ret; - } - -+int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid, -+ unsigned int dword11, void *buffer, size_t buflen, -+ u32 *result) -+{ -+ return nvme_features(dev, nvme_admin_set_features, fid, dword11, buffer, -+ buflen, result); -+} -+EXPORT_SYMBOL_GPL(nvme_set_features); -+ -+int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid, -+ unsigned int dword11, void *buffer, size_t buflen, -+ u32 *result) -+{ -+ return nvme_features(dev, nvme_admin_get_features, fid, dword11, buffer, -+ buflen, result); -+} -+EXPORT_SYMBOL_GPL(nvme_get_features); -+ - int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count) - { - u32 q_count = (*count - 1) | ((*count - 1) << 16); -diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h -index 81215ca32671..9285d5f6437b 100644 ---- a/drivers/nvme/host/nvme.h -+++ b/drivers/nvme/host/nvme.h -@@ -459,6 +459,12 @@ int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd, - union nvme_result *result, void *buffer, unsigned bufflen, - unsigned timeout, int qid, int at_head, - blk_mq_req_flags_t flags, bool poll); -+int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid, -+ unsigned int dword11, void *buffer, size_t buflen, -+ u32 *result); -+int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid, -+ unsigned int dword11, void *buffer, size_t buflen, -+ u32 *result); - int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count); - void nvme_stop_keep_alive(struct nvme_ctrl *ctrl); - int nvme_reset_ctrl(struct nvme_ctrl *ctrl); -diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c -index 09ffd21d1809..3e22d5f14e93 100644 ---- a/drivers/nvme/host/pci.c -+++ b/drivers/nvme/host/pci.c -@@ -18,6 +18,7 @@ - #include - #include - #include -+#include - #include - #include - #include -@@ -116,6 +117,7 @@ struct nvme_dev { - u32 cmbsz; - u32 cmbloc; - struct nvme_ctrl ctrl; -+ u32 last_ps; - - mempool_t *iod_mempool; - -@@ -2849,16 +2851,94 @@ static void nvme_remove(struct pci_dev *pdev) - } - - #ifdef CONFIG_PM_SLEEP -+static int nvme_get_power_state(struct nvme_ctrl *ctrl, u32 *ps) -+{ -+ return nvme_get_features(ctrl, NVME_FEAT_POWER_MGMT, 0, NULL, 0, ps); -+} -+ -+static int nvme_set_power_state(struct nvme_ctrl *ctrl, u32 ps) -+{ -+ return nvme_set_features(ctrl, NVME_FEAT_POWER_MGMT, ps, NULL, 0, NULL); -+} -+ -+static int nvme_resume(struct device *dev) -+{ -+ struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev)); -+ struct nvme_ctrl *ctrl = &ndev->ctrl; -+ -+ if (pm_resume_via_firmware() || !ctrl->npss || -+ nvme_set_power_state(ctrl, ndev->last_ps) != 0) -+ nvme_reset_ctrl(ctrl); -+ return 0; -+} -+ - static int nvme_suspend(struct device *dev) - { - struct pci_dev *pdev = to_pci_dev(dev); - struct nvme_dev *ndev = pci_get_drvdata(pdev); -+ struct nvme_ctrl *ctrl = &ndev->ctrl; -+ int ret = -EBUSY; -+ -+ /* -+ * The platform does not remove power for a kernel managed suspend so -+ * use host managed nvme power settings for lowest idle power if -+ * possible. This should have quicker resume latency than a full device -+ * shutdown. But if the firmware is involved after the suspend or the -+ * device does not support any non-default power states, shut down the -+ * device fully. -+ */ -+ if (pm_suspend_via_firmware() || !ctrl->npss) { -+ nvme_dev_disable(ndev, true); -+ return 0; -+ } -+ -+ nvme_start_freeze(ctrl); -+ nvme_wait_freeze(ctrl); -+ nvme_sync_queues(ctrl); -+ -+ if (ctrl->state != NVME_CTRL_LIVE && -+ ctrl->state != NVME_CTRL_ADMIN_ONLY) -+ goto unfreeze; -+ -+ ndev->last_ps = 0; -+ ret = nvme_get_power_state(ctrl, &ndev->last_ps); -+ if (ret < 0) -+ goto unfreeze; -+ -+ ret = nvme_set_power_state(ctrl, ctrl->npss); -+ if (ret < 0) -+ goto unfreeze; -+ -+ if (ret) { -+ /* -+ * Clearing npss forces a controller reset on resume. The -+ * correct value will be resdicovered then. -+ */ -+ nvme_dev_disable(ndev, true); -+ ctrl->npss = 0; -+ ret = 0; -+ goto unfreeze; -+ } -+ /* -+ * A saved state prevents pci pm from generically controlling the -+ * device's power. If we're using protocol specific settings, we don't -+ * want pci interfering. -+ */ -+ pci_save_state(pdev); -+unfreeze: -+ nvme_unfreeze(ctrl); -+ return ret; -+} -+ -+static int nvme_simple_suspend(struct device *dev) -+{ -+ struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev)); - - nvme_dev_disable(ndev, true); - return 0; - } - --static int nvme_resume(struct device *dev) -+static int nvme_simple_resume(struct device *dev) - { - struct pci_dev *pdev = to_pci_dev(dev); - struct nvme_dev *ndev = pci_get_drvdata(pdev); -@@ -2866,9 +2946,16 @@ static int nvme_resume(struct device *dev) - nvme_reset_ctrl(&ndev->ctrl); - return 0; - } --#endif - --static SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume); -+const struct dev_pm_ops nvme_dev_pm_ops = { -+ .suspend = nvme_suspend, -+ .resume = nvme_resume, -+ .freeze = nvme_simple_suspend, -+ .thaw = nvme_simple_resume, -+ .poweroff = nvme_simple_suspend, -+ .restore = nvme_simple_resume, -+}; -+#endif /* CONFIG_PM_SLEEP */ - - static pci_ers_result_t nvme_error_detected(struct pci_dev *pdev, - pci_channel_state_t state) -@@ -2975,9 +3062,11 @@ static struct pci_driver nvme_driver = { - .probe = nvme_probe, - .remove = nvme_remove, - .shutdown = nvme_shutdown, -+#ifdef CONFIG_PM_SLEEP - .driver = { - .pm = &nvme_dev_pm_ops, - }, -+#endif - .sriov_configure = pci_sriov_configure_simple, - .err_handler = &nvme_err_handler, - }; -diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c -index 096211299c07..0cb0fe170977 100644 ---- a/kernel/power/suspend.c -+++ b/kernel/power/suspend.c -@@ -533,6 +533,8 @@ int suspend_devices_and_enter(suspend_state_t state) - goto Resume_devices; - } - -+unsigned int resume_delay = 3000; -+ - /** - * suspend_finish - Clean up before finishing the suspend sequence. - * -@@ -541,6 +543,15 @@ int suspend_devices_and_enter(suspend_state_t state) - */ - static void suspend_finish(void) - { -+ if (resume_delay) { -+ /* Give kernel threads a head start, such that usb-storage -+ * can detect devices before syslog attempts to write log -+ * messages from the suspend code. -+ */ -+ thaw_kernel_threads(); -+ pr_debug("PM: Sleeping for %d milliseconds.\n", resume_delay); -+ msleep(resume_delay); -+ } - suspend_thaw_processes(); - pm_notifier_call_chain(PM_POST_SUSPEND); - pm_restore_console(); -diff --git a/kernel/sysctl.c b/kernel/sysctl.c -index 1beca96fb625..4b98db9bbc88 100644 ---- a/kernel/sysctl.c -+++ b/kernel/sysctl.c -@@ -318,7 +318,16 @@ static int min_extfrag_threshold; - static int max_extfrag_threshold = 1000; - #endif - -+extern unsigned int resume_delay; -+ - static struct ctl_table kern_table[] = { -+ { -+ .procname = "resume_delay", -+ .data = &resume_delay, -+ .maxlen = sizeof(unsigned int), -+ .mode = 0644, -+ .proc_handler = proc_dointvec, -+ }, - { - .procname = "sched_child_runs_first", - .data = &sysctl_sched_child_runs_first, --- -2.23.0 - diff --git a/patches/5.2/0004-cameras.patch b/patches/5.2/0004-cameras.patch deleted file mode 100644 index 591bc3101..000000000 --- a/patches/5.2/0004-cameras.patch +++ /dev/null @@ -1,2753 +0,0 @@ -From 1af16ec5d87924e40ff153cc83dff212c3b80a7c Mon Sep 17 00:00:00 2001 -From: Maximilian Luz -Date: Fri, 26 Jul 2019 04:45:19 +0200 -Subject: [PATCH 04/12] cameras - ---- - drivers/media/usb/uvc/uvc_driver.c | 40 + - drivers/staging/Makefile | 1 + - drivers/staging/ov5693/Kconfig | 10 + - drivers/staging/ov5693/Makefile | 5 + - drivers/staging/ov5693/ad5823.c | 218 +++++ - drivers/staging/ov5693/ad5823.h | 90 ++ - drivers/staging/ov5693/ov5693.c | 1461 ++++++++++++++++++++++++++++ - drivers/staging/ov5693/ov5693.h | 848 ++++++++++++++++ - 8 files changed, 2673 insertions(+) - create mode 100644 drivers/staging/ov5693/Kconfig - create mode 100644 drivers/staging/ov5693/Makefile - create mode 100644 drivers/staging/ov5693/ad5823.c - create mode 100644 drivers/staging/ov5693/ad5823.h - create mode 100644 drivers/staging/ov5693/ov5693.c - create mode 100644 drivers/staging/ov5693/ov5693.h - -diff --git a/drivers/media/usb/uvc/uvc_driver.c b/drivers/media/usb/uvc/uvc_driver.c -index 66ee168ddc7e..e4bccfab9da1 100644 ---- a/drivers/media/usb/uvc/uvc_driver.c -+++ b/drivers/media/usb/uvc/uvc_driver.c -@@ -2394,6 +2394,46 @@ static const struct uvc_device_info uvc_quirk_force_y8 = { - * though they are compliant. - */ - static const struct usb_device_id uvc_ids[] = { -+ /* Microsoft Surface Pro 3 Front */ -+ { .match_flags = USB_DEVICE_ID_MATCH_DEVICE -+ | USB_DEVICE_ID_MATCH_INT_INFO, -+ .idVendor = 0x045e, -+ .idProduct = 0x07be, -+ .bInterfaceClass = USB_CLASS_VIDEO, -+ .bInterfaceSubClass = 1, -+ .bInterfaceProtocol = 1 }, -+ /* Microsoft Surface Pro 3 Rear */ -+ { .match_flags = USB_DEVICE_ID_MATCH_DEVICE -+ | USB_DEVICE_ID_MATCH_INT_INFO, -+ .idVendor = 0x045e, -+ .idProduct = 0x07bf, -+ .bInterfaceClass = USB_CLASS_VIDEO, -+ .bInterfaceSubClass = 1, -+ .bInterfaceProtocol = 1 }, -+ /* Microsoft Surface Pro 4 Cam */ -+ { .match_flags = USB_DEVICE_ID_MATCH_DEVICE -+ | USB_DEVICE_ID_MATCH_INT_INFO, -+ .idVendor = 0x045e, -+ .idProduct = 0x090c, -+ .bInterfaceClass = USB_CLASS_VIDEO, -+ .bInterfaceSubClass = 1, -+ .bInterfaceProtocol = 1 }, -+ /* Microsoft Surface Book Cam 1 */ -+ { .match_flags = USB_DEVICE_ID_MATCH_DEVICE -+ | USB_DEVICE_ID_MATCH_INT_INFO, -+ .idVendor = 0x045e, -+ .idProduct = 0x090b, -+ .bInterfaceClass = USB_CLASS_VIDEO, -+ .bInterfaceSubClass = 1, -+ .bInterfaceProtocol = 1 }, -+ /* Microsoft Surface Book Cam 2 */ -+ { .match_flags = USB_DEVICE_ID_MATCH_DEVICE -+ | USB_DEVICE_ID_MATCH_INT_INFO, -+ .idVendor = 0x045e, -+ .idProduct = 0x091a, -+ .bInterfaceClass = USB_CLASS_VIDEO, -+ .bInterfaceSubClass = 1, -+ .bInterfaceProtocol = 1 }, - /* LogiLink Wireless Webcam */ - { .match_flags = USB_DEVICE_ID_MATCH_DEVICE - | USB_DEVICE_ID_MATCH_INT_INFO, -diff --git a/drivers/staging/Makefile b/drivers/staging/Makefile -index 0da0d3f0b5e4..1fffb6f5a3be 100644 ---- a/drivers/staging/Makefile -+++ b/drivers/staging/Makefile -@@ -49,3 +49,4 @@ obj-$(CONFIG_XIL_AXIS_FIFO) += axis-fifo/ - obj-$(CONFIG_EROFS_FS) += erofs/ - obj-$(CONFIG_FIELDBUS_DEV) += fieldbus/ - obj-$(CONFIG_KPC2000) += kpc2000/ -+obj-$(CONFIG_VIDEO_OV5693) += ov5693/ -diff --git a/drivers/staging/ov5693/Kconfig b/drivers/staging/ov5693/Kconfig -new file mode 100644 -index 000000000000..96000f112c4d ---- /dev/null -+++ b/drivers/staging/ov5693/Kconfig -@@ -0,0 +1,10 @@ -+config VIDEO_OV5693 -+ tristate "Omnivision ov5693 sensor support" -+ depends on I2C && VIDEO_V4L2 -+ ---help--- -+ This is a Video4Linux2 sensor-level driver for the Micron -+ ov5693 5 Mpixel camera. -+ -+ ov5693 is video camera sensor. -+ -+ It currently only works with the atomisp driver. -diff --git a/drivers/staging/ov5693/Makefile b/drivers/staging/ov5693/Makefile -new file mode 100644 -index 000000000000..d8a63faa591f ---- /dev/null -+++ b/drivers/staging/ov5693/Makefile -@@ -0,0 +1,5 @@ -+obj-$(CONFIG_VIDEO_OV5693) += ov569x.o -+ -+ov569x-objs := ov5693.o ad5823.o -+ -+ccflags-y += -Werror -diff --git a/drivers/staging/ov5693/ad5823.c b/drivers/staging/ov5693/ad5823.c -new file mode 100644 -index 000000000000..7c34c36e77e5 ---- /dev/null -+++ b/drivers/staging/ov5693/ad5823.c -@@ -0,0 +1,218 @@ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "ad5823.h" -+ -+static struct ad5823_device ad5823_dev; -+static int ad5823_i2c_write(struct i2c_client *client, u8 reg, u8 val) -+{ -+ struct i2c_msg msg; -+ u8 buf[2]; -+ buf[0] = reg; -+ buf[1] = val; -+ msg.addr = AD5823_VCM_ADDR; -+ msg.flags = 0; -+ msg.len = AD5823_8BIT; -+ msg.buf = &buf[0]; -+ -+ if (i2c_transfer(client->adapter, &msg, 1) != 1) -+ return -EIO; -+ return 0; -+} -+ -+static int ad5823_i2c_read(struct i2c_client *client, u8 reg, u8 *val) -+{ -+ struct i2c_msg msg[2]; -+ u8 buf[2]; -+ buf[0] = reg; -+ buf[1] = 0; -+ -+ msg[0].addr = AD5823_VCM_ADDR; -+ msg[0].flags = 0; -+ msg[0].len = AD5823_8BIT; -+ msg[0].buf = &buf[0]; -+ -+ msg[1].addr = AD5823_VCM_ADDR; -+ msg[1].flags = I2C_M_RD; -+ msg[1].len = AD5823_8BIT; -+ msg[1].buf = &buf[1]; -+ *val = 0; -+ if (i2c_transfer(client->adapter, msg, 2) != 2) -+ return -EIO; -+ *val = buf[1]; -+ return 0; -+} -+ -+int ad5823_vcm_power_up(struct v4l2_subdev *sd) -+{ -+ int ret = -ENODEV; -+ -+ /* Enable power */ -+ if (ad5823_dev.platform_data) -+ ret = ad5823_dev.platform_data->power_ctrl(sd, 1); -+ /* -+ * waiting time requested by AD5823(vcm) -+ */ -+ usleep_range(1000, 2000); -+ return ret; -+} -+ -+int ad5823_vcm_power_down(struct v4l2_subdev *sd) -+{ -+ int ret = -ENODEV; -+ -+ if (ad5823_dev.platform_data) -+ ret = ad5823_dev.platform_data->power_ctrl(sd, 0); -+ -+ return ret; -+} -+ -+ -+int ad5823_t_focus_vcm(struct v4l2_subdev *sd, u16 val) -+{ -+ struct i2c_client *client = v4l2_get_subdevdata(sd); -+ int ret = -EINVAL; -+ u8 vcm_code; -+ u8 vcm_mode_reg_val[4] = { -+ AD5823_ARC_RES0, -+ AD5823_ARC_RES1, -+ AD5823_ARC_RES2, -+ AD5823_ESRC -+ }; -+ -+ if (ad5823_dev.vcm_mode != AD5823_DIRECT) { -+ ret = ad5823_i2c_write(client, AD5823_REG_VCM_CODE_MSB, -+ AD5823_RING_CTRL_ENABLE); -+ if (ret) -+ return ret; -+ -+ ret = ad5823_i2c_write(client, AD5823_REG_MODE, -+ vcm_mode_reg_val[ad5823_dev.vcm_mode]); -+ if (ret) -+ return ret; -+ } else { -+ ret = ad5823_i2c_write(client, AD5823_REG_VCM_CODE_MSB, -+ AD5823_RING_CTRL_DISABLE); -+ if (ret) -+ return ret; -+ } -+ -+ ret = ad5823_i2c_read(client, AD5823_REG_VCM_CODE_MSB, &vcm_code); -+ if (ret) -+ return ret; -+ -+ /* set reg VCM_CODE_MSB Bit[1:0] */ -+ vcm_code = (vcm_code & VCM_CODE_MSB_MASK) | ((val >> 8) & ~VCM_CODE_MSB_MASK); -+ ret = ad5823_i2c_write(client, AD5823_REG_VCM_CODE_MSB, vcm_code); -+ if (ret) -+ return ret; -+ -+ /* set reg VCM_CODE_LSB Bit[7:0] */ -+ ret = ad5823_i2c_write(client, AD5823_REG_VCM_CODE_LSB, -+ (val & 0x0f)); -+ if (ret) -+ return ret; -+ -+ /* set required vcm move time */ -+ vcm_code = AD5823_RESONANCE_PERIOD / AD5823_RESONANCE_COEF -+ - AD5823_HIGH_FREQ_RANGE; -+ ret = ad5823_i2c_write(client, AD5823_REG_VCM_MOVE_TIME, vcm_code); -+ -+ return ret; -+} -+ -+int ad5823_t_focus_abs(struct v4l2_subdev *sd, s32 value) -+{ -+ int ret; -+ -+ value = min(value, AD5823_MAX_FOCUS_POS); -+ ret = ad5823_t_focus_vcm(sd, AD5823_MAX_FOCUS_POS - value); -+ if (ret == 0) { -+ ad5823_dev.number_of_steps = value - ad5823_dev.focus; -+ ad5823_dev.focus = value; -+ ktime_get_ts(&ad5823_dev.timestamp_t_focus_abs); -+ } -+ -+ return ret; -+} -+ -+int ad5823_t_focus_rel(struct v4l2_subdev *sd, s32 value) -+{ -+ return ad5823_t_focus_abs(sd, ad5823_dev.focus + value); -+} -+ -+int ad5823_q_focus_status(struct v4l2_subdev *sd, s32 *value) -+{ -+ u32 status = 0; -+ struct timespec temptime; -+ const struct timespec timedelay = { -+ 0, -+ min_t(u32, abs(ad5823_dev.number_of_steps)*DELAY_PER_STEP_NS, -+ DELAY_MAX_PER_STEP_NS), -+ }; -+ -+ ktime_get_ts(&temptime); -+ -+ temptime = timespec_sub(temptime, (ad5823_dev.timestamp_t_focus_abs)); -+ -+ if (timespec_compare(&temptime, &timedelay) <= 0) -+ status = ATOMISP_FOCUS_STATUS_MOVING -+ | ATOMISP_FOCUS_HP_IN_PROGRESS; -+ else -+ status = ATOMISP_FOCUS_STATUS_ACCEPTS_NEW_MOVE -+ | ATOMISP_FOCUS_HP_COMPLETE; -+ -+ *value = status; -+ -+ return 0; -+} -+ -+int ad5823_q_focus_abs(struct v4l2_subdev *sd, s32 *value) -+{ -+ s32 val; -+ -+ ad5823_q_focus_status(sd, &val); -+ -+ if (val & ATOMISP_FOCUS_STATUS_MOVING) -+ *value = ad5823_dev.focus - ad5823_dev.number_of_steps; -+ else -+ *value = ad5823_dev.focus ; -+ -+ return 0; -+} -+ -+int ad5823_t_vcm_slew(struct v4l2_subdev *sd, s32 value) -+{ -+ return 0; -+} -+ -+int ad5823_t_vcm_timing(struct v4l2_subdev *sd, s32 value) -+{ -+ return 0; -+} -+ -+int ad5823_vcm_init(struct v4l2_subdev *sd) -+{ -+ /* set vcm mode to ARC RES0.5 */ -+ ad5823_dev.vcm_mode = AD5823_ARC_RES1; -+ ad5823_dev.platform_data = camera_get_af_platform_data(); -+ return ad5823_dev.platform_data ? 0 : -ENODEV; -+} -diff --git a/drivers/staging/ov5693/ad5823.h b/drivers/staging/ov5693/ad5823.h -new file mode 100644 -index 000000000000..8b046c31f3af ---- /dev/null -+++ b/drivers/staging/ov5693/ad5823.h -@@ -0,0 +1,90 @@ -+/* -+ * Support for AD5823 VCM. -+ * -+ * Copyright (c) 2013 Intel Corporation. All Rights Reserved. -+ * -+ * This program is free software; you can redistribute it and/or -+ * modify it under the terms of the GNU General Public License version -+ * 2 as published by the Free Software Foundation. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, write to the Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA -+ * 02110-1301, USA. -+ * -+ */ -+ -+#ifndef __AD5823_H__ -+#define __AD5823_H__ -+ -+#include -+#include -+ -+ -+#define AD5823_VCM_ADDR 0x0c -+ -+#define AD5823_REG_RESET 0x01 -+#define AD5823_REG_MODE 0x02 -+#define AD5823_REG_VCM_MOVE_TIME 0x03 -+#define AD5823_REG_VCM_CODE_MSB 0x04 -+#define AD5823_REG_VCM_CODE_LSB 0x05 -+#define AD5823_REG_VCM_THRESHOLD_MSB 0x06 -+#define AD5823_REG_VCM_THRESHOLD_LSB 0x07 -+ -+#define AD5823_RING_CTRL_ENABLE 0x04 -+#define AD5823_RING_CTRL_DISABLE 0x00 -+ -+#define AD5823_RESONANCE_PERIOD 100000 -+#define AD5823_RESONANCE_COEF 512 -+#define AD5823_HIGH_FREQ_RANGE 0x80 -+ -+#define VCM_CODE_MSB_MASK 0xfc -+ -+enum ad5823_tok_type { -+ AD5823_8BIT = 0x0001, -+ AD5823_16BIT = 0x0002, -+}; -+ -+enum ad5823_vcm_mode { -+ AD5823_ARC_RES0 = 0x0, /* Actuator response control RES1 */ -+ AD5823_ARC_RES1 = 0x1, /* Actuator response control RES0.5 */ -+ AD5823_ARC_RES2 = 0x2, /* Actuator response control RES2 */ -+ AD5823_ESRC = 0x3, /* Enhanced slew rate control */ -+ AD5823_DIRECT = 0x4, /* Direct control */ -+}; -+ -+/* ad5823 device structure */ -+struct ad5823_device { -+ struct timespec timestamp_t_focus_abs; -+ enum ad5823_vcm_mode vcm_mode; -+ s16 number_of_steps; -+ bool initialized; /* true if ad5823 is detected */ -+ s32 focus; /* Current focus value */ -+ struct timespec focus_time; /* Time when focus was last time set */ -+ __u8 buffer[4]; /* Used for i2c transactions */ -+ const struct camera_af_platform_data *platform_data; -+}; -+ -+#define AD5823_INVALID_CONFIG 0xffffffff -+#define AD5823_MAX_FOCUS_POS 1023 -+ -+ -+#define DELAY_PER_STEP_NS 1000000 -+#define DELAY_MAX_PER_STEP_NS (1000000 * 1023) -+ -+int ad5823_vcm_power_up(struct v4l2_subdev *sd); -+int ad5823_vcm_power_down(struct v4l2_subdev *sd); -+int ad5823_vcm_init(struct v4l2_subdev *sd); -+ -+int ad5823_t_focus_vcm(struct v4l2_subdev *sd, u16 val); -+int ad5823_t_focus_abs(struct v4l2_subdev *sd, s32 value); -+int ad5823_t_focus_rel(struct v4l2_subdev *sd, s32 value); -+int ad5823_q_focus_status(struct v4l2_subdev *sd, s32 *value); -+int ad5823_q_focus_abs(struct v4l2_subdev *sd, s32 *value); -+ -+#endif -diff --git a/drivers/staging/ov5693/ov5693.c b/drivers/staging/ov5693/ov5693.c -new file mode 100644 -index 000000000000..51d218da3722 ---- /dev/null -+++ b/drivers/staging/ov5693/ov5693.c -@@ -0,0 +1,1461 @@ -+/* -+ * Support for OmniVision OV5693 5M HD camera sensor. -+ * -+ * Copyright (c) 2013 Intel Corporation. All Rights Reserved. -+ * -+ * This program is free software; you can redistribute it and/or -+ * modify it under the terms of the GNU General Public License version -+ * 2 as published by the Free Software Foundation. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, write to the Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA -+ * 02110-1301, USA. -+ * -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "ov5693.h" -+ -+/* i2c read/write stuff */ -+static int ov5693_read_reg(struct i2c_client *client, -+ u16 data_length, u16 reg, u16 *val) -+{ -+ int err; -+ struct i2c_msg msg[2]; -+ unsigned char data[6]; -+ -+ if (!client->adapter) { -+ dev_err(&client->dev, "%s error, no client->adapter\n", -+ __func__); -+ return -ENODEV; -+ } -+ -+ if (data_length != OV5693_8BIT && data_length != OV5693_16BIT -+ && data_length != OV5693_32BIT) { -+ dev_err(&client->dev, "%s error, invalid data length\n", -+ __func__); -+ return -EINVAL; -+ } -+ -+ memset(msg, 0 , sizeof(msg)); -+ -+ msg[0].addr = client->addr; -+ msg[0].flags = 0; -+ msg[0].len = I2C_MSG_LENGTH; -+ msg[0].buf = data; -+ -+ /* high byte goes out first */ -+ data[0] = (u8)(reg >> 8); -+ data[1] = (u8)(reg & 0xff); -+ -+ msg[1].addr = client->addr; -+ msg[1].len = data_length; -+ msg[1].flags = I2C_M_RD; -+ msg[1].buf = data; -+ -+ err = i2c_transfer(client->adapter, msg, 2); -+ if (err != 2) { -+ if (err >= 0) -+ err = -EIO; -+ dev_err(&client->dev, -+ "read from offset 0x%x error %d", reg, err); -+ return err; -+ } -+ -+ *val = 0; -+ /* high byte comes first */ -+ if (data_length == OV5693_8BIT) -+ *val = (u8)data[0]; -+ else if (data_length == OV5693_16BIT) -+ *val = be16_to_cpu(*(u16 *)&data[0]); -+ else -+ *val = be32_to_cpu(*(u32 *)&data[0]); -+ -+ return 0; -+} -+ -+static int ov5693_i2c_write(struct i2c_client *client, u16 len, u8 *data) -+{ -+ struct i2c_msg msg; -+ const int num_msg = 1; -+ int ret; -+ -+ msg.addr = client->addr; -+ msg.flags = 0; -+ msg.len = len; -+ msg.buf = data; -+ ret = i2c_transfer(client->adapter, &msg, 1); -+ -+ return ret == num_msg ? 0 : -EIO; -+} -+ -+static int ov5693_write_reg(struct i2c_client *client, u16 data_length, -+ u16 reg, u16 val) -+{ -+ int ret; -+ unsigned char data[4] = {0}; -+ u16 *wreg = (u16 *)data; -+ const u16 len = data_length + sizeof(u16); /* 16-bit address + data */ -+ -+ if (data_length != OV5693_8BIT && data_length != OV5693_16BIT) { -+ dev_err(&client->dev, -+ "%s error, invalid data_length\n", __func__); -+ return -EINVAL; -+ } -+ -+ /* high byte goes out first */ -+ *wreg = cpu_to_be16(reg); -+ -+ if (data_length == OV5693_8BIT) { -+ data[2] = (u8)(val); -+ } else { -+ /* OV5693_16BIT */ -+ u16 *wdata = (u16 *)&data[2]; -+ *wdata = cpu_to_be16(val); -+ } -+ -+ ret = ov5693_i2c_write(client, len, data); -+ if (ret) -+ dev_err(&client->dev, -+ "write error: wrote 0x%x to offset 0x%x error %d", -+ val, reg, ret); -+ -+ return ret; -+} -+ -+/* -+ * ov5693_write_reg_array - Initializes a list of OV5693 registers -+ * @client: i2c driver client structure -+ * @reglist: list of registers to be written -+ * -+ * This function initializes a list of registers. When consecutive addresses -+ * are found in a row on the list, this function creates a buffer and sends -+ * consecutive data in a single i2c_transfer(). -+ * -+ * __ov5693_flush_reg_array, __ov5693_buf_reg_array() and -+ * __ov5693_write_reg_is_consecutive() are internal functions to -+ * ov5693_write_reg_array_fast() and should be not used anywhere else. -+ * -+ */ -+static int __ov5693_flush_reg_array(struct i2c_client *client, -+ struct ov5693_write_ctrl *ctrl) -+{ -+ u16 size; -+ -+ if (ctrl->index == 0) -+ return 0; -+ -+ size = sizeof(u16) + ctrl->index; /* 16-bit address + data */ -+ ctrl->buffer.addr = cpu_to_be16(ctrl->buffer.addr); -+ ctrl->index = 0; -+ -+ return ov5693_i2c_write(client, size, (u8 *)&ctrl->buffer); -+} -+ -+static int __ov5693_buf_reg_array(struct i2c_client *client, -+ struct ov5693_write_ctrl *ctrl, -+ const struct ov5693_reg *next) -+{ -+ int size; -+ u16 *data16; -+ -+ switch (next->type) { -+ case OV5693_8BIT: -+ size = 1; -+ ctrl->buffer.data[ctrl->index] = (u8)next->val; -+ break; -+ case OV5693_16BIT: -+ size = 2; -+ data16 = (u16 *)&ctrl->buffer.data[ctrl->index]; -+ *data16 = cpu_to_be16((u16)next->val); -+ break; -+ default: -+ return -EINVAL; -+ } -+ -+ /* When first item is added, we need to store its starting address */ -+ if (ctrl->index == 0) -+ ctrl->buffer.addr = next->reg; -+ -+ ctrl->index += size; -+ -+ /* -+ * Buffer cannot guarantee free space for u32? Better flush it to avoid -+ * possible lack of memory for next item. -+ */ -+ if (ctrl->index + sizeof(u16) >= OV5693_MAX_WRITE_BUF_SIZE) -+ return __ov5693_flush_reg_array(client, ctrl); -+ -+ return 0; -+} -+ -+static int __ov5693_write_reg_is_consecutive(struct i2c_client *client, -+ struct ov5693_write_ctrl *ctrl, -+ const struct ov5693_reg *next) -+{ -+ if (ctrl->index == 0) -+ return 1; -+ -+ return ctrl->buffer.addr + ctrl->index == next->reg; -+} -+ -+static int ov5693_write_reg_array(struct i2c_client *client, -+ const struct ov5693_reg *reglist) -+{ -+ const struct ov5693_reg *next = reglist; -+ struct ov5693_write_ctrl ctrl; -+ int err; -+ -+ ctrl.index = 0; -+ for (; next->type != OV5693_TOK_TERM; next++) { -+ switch (next->type & OV5693_TOK_MASK) { -+ case OV5693_TOK_DELAY: -+ err = __ov5693_flush_reg_array(client, &ctrl); -+ if (err) -+ return err; -+ usleep_range(next->val * 1000, (next->val + 1) * 1000); -+ break; -+ default: -+ /* -+ * If next address is not consecutive, data needs to be -+ * flushed before proceed. -+ */ -+ if (!__ov5693_write_reg_is_consecutive(client, &ctrl, -+ next)) { -+ err = __ov5693_flush_reg_array(client, &ctrl); -+ if (err) -+ return err; -+ } -+ err = __ov5693_buf_reg_array(client, &ctrl, next); -+ if (err) { -+ dev_err(&client->dev, "%s: write error, aborted\n", -+ __func__); -+ return err; -+ } -+ break; -+ } -+ } -+ -+ return __ov5693_flush_reg_array(client, &ctrl); -+} -+static int ov5693_g_focal(struct v4l2_subdev *sd, s32 *val) -+{ -+ *val = (OV5693_FOCAL_LENGTH_NUM << 16) | OV5693_FOCAL_LENGTH_DEM; -+ return 0; -+} -+ -+static int ov5693_g_fnumber(struct v4l2_subdev *sd, s32 *val) -+{ -+ /*const f number for ov5693*/ -+ *val = (OV5693_F_NUMBER_DEFAULT_NUM << 16) | OV5693_F_NUMBER_DEM; -+ return 0; -+} -+ -+static int ov5693_g_fnumber_range(struct v4l2_subdev *sd, s32 *val) -+{ -+ *val = (OV5693_F_NUMBER_DEFAULT_NUM << 24) | -+ (OV5693_F_NUMBER_DEM << 16) | -+ (OV5693_F_NUMBER_DEFAULT_NUM << 8) | OV5693_F_NUMBER_DEM; -+ return 0; -+} -+ -+ -+static int ov5693_get_intg_factor(struct i2c_client *client, -+ struct camera_mipi_info *info, -+ const struct ov5693_resolution *res) -+{ -+ struct atomisp_sensor_mode_data *buf = &info->data; -+ unsigned int pix_clk_freq_hz; -+ u16 reg_val; -+ int ret; -+ -+ if (info == NULL) -+ return -EINVAL; -+ -+ /* pixel clock calculattion */ -+ pix_clk_freq_hz = res->pix_clk_freq * 1000000; -+ -+ buf->vt_pix_clk_freq_mhz = pix_clk_freq_hz; -+ -+ /* get integration time */ -+ buf->coarse_integration_time_min = OV5693_COARSE_INTG_TIME_MIN; -+ buf->coarse_integration_time_max_margin = -+ OV5693_COARSE_INTG_TIME_MAX_MARGIN; -+ -+ buf->fine_integration_time_min = OV5693_FINE_INTG_TIME_MIN; -+ buf->fine_integration_time_max_margin = -+ OV5693_FINE_INTG_TIME_MAX_MARGIN; -+ -+ buf->fine_integration_time_def = OV5693_FINE_INTG_TIME_MIN; -+ buf->frame_length_lines = res->lines_per_frame; -+ buf->line_length_pck = res->pixels_per_line; -+ buf->read_mode = res->bin_mode; -+ -+ /* get the cropping and output resolution to ISP for this mode. */ -+ ret = ov5693_read_reg(client, OV5693_16BIT, -+ OV5693_H_CROP_START_H, ®_val); -+ if (ret) -+ return ret; -+ buf->crop_horizontal_start = reg_val; -+ -+ ret = ov5693_read_reg(client, OV5693_16BIT, -+ OV5693_V_CROP_START_H, ®_val); -+ if (ret) -+ return ret; -+ buf->crop_vertical_start = reg_val; -+ -+ ret = ov5693_read_reg(client, OV5693_16BIT, -+ OV5693_H_CROP_END_H, ®_val); -+ if (ret) -+ return ret; -+ buf->crop_horizontal_end = reg_val; -+ -+ ret = ov5693_read_reg(client, OV5693_16BIT, -+ OV5693_V_CROP_END_H, ®_val); -+ if (ret) -+ return ret; -+ buf->crop_vertical_end = reg_val; -+ -+ ret = ov5693_read_reg(client, OV5693_16BIT, -+ OV5693_H_OUTSIZE_H, ®_val); -+ if (ret) -+ return ret; -+ buf->output_width = reg_val; -+ -+ ret = ov5693_read_reg(client, OV5693_16BIT, -+ OV5693_V_OUTSIZE_H, ®_val); -+ if (ret) -+ return ret; -+ buf->output_height = reg_val; -+ -+ /* -+ * we can't return 0 for bin_factor, this is because camera -+ * HAL will use them as denominator, bin_factor = 0 will -+ * cause camera HAL crash. So we return bin_factor as this -+ * rules: -+ * [1]. res->bin_factor = 0, return 1 for bin_factor. -+ * [2]. res->bin_factor > 0, return res->bin_factor. -+ */ -+ buf->binning_factor_x = res->bin_factor_x ? -+ res->bin_factor_x : 1; -+ buf->binning_factor_y = res->bin_factor_y ? -+ res->bin_factor_y : 1; -+ return 0; -+} -+ -+static long __ov5693_set_exposure(struct v4l2_subdev *sd, int coarse_itg, -+ int gain, int digitgain) -+ -+{ -+ struct i2c_client *client = v4l2_get_subdevdata(sd); -+ u16 vts; -+ int ret; -+ -+ /* -+ * According to spec, the low 4 bits of exposure/gain reg are -+ * fraction bits, so need to take 4 bits left shift to align -+ * reg integer bits. -+ */ -+ coarse_itg <<= 4; -+ gain <<= 4; -+ -+ ret = ov5693_read_reg(client, OV5693_16BIT, -+ OV5693_VTS_H, &vts); -+ if (ret) -+ return ret; -+ -+ if (coarse_itg + OV5693_INTEGRATION_TIME_MARGIN >= vts) -+ vts = coarse_itg + OV5693_INTEGRATION_TIME_MARGIN; -+ -+ ret = ov5693_write_reg(client, OV5693_16BIT, OV5693_VTS_H, vts); -+ if (ret) -+ return ret; -+ -+ /* group hold start */ -+ ret = ov5693_write_reg(client, OV5693_8BIT, OV5693_GROUP_ACCESS, 0); -+ if (ret) -+ return ret; -+ -+ /* set exposure */ -+ ret = ov5693_write_reg(client, OV5693_8BIT, -+ OV5693_AEC_PK_EXPO_L, -+ coarse_itg & 0xff); -+ if (ret) -+ return ret; -+ -+ ret = ov5693_write_reg(client, OV5693_16BIT, -+ OV5693_AEC_PK_EXPO_H, -+ (coarse_itg >> 8) & 0xfff); -+ if (ret) -+ return ret; -+ -+ /* set analog gain */ -+ ret = ov5693_write_reg(client, OV5693_16BIT, -+ OV5693_AGC_ADJ_H, gain); -+ if (ret) -+ return ret; -+ -+ /* set digital gain */ -+ ret = ov5693_write_reg(client, OV5693_16BIT, -+ OV5693_MWB_GAIN_R_H, digitgain); -+ if (ret) -+ return ret; -+ -+ ret = ov5693_write_reg(client, OV5693_16BIT, -+ OV5693_MWB_GAIN_G_H, digitgain); -+ if (ret) -+ return ret; -+ -+ ret = ov5693_write_reg(client, OV5693_16BIT, -+ OV5693_MWB_GAIN_B_H, digitgain); -+ if (ret) -+ return ret; -+ -+ /* group hold end */ -+ ret = ov5693_write_reg(client, OV5693_8BIT, -+ OV5693_GROUP_ACCESS, 0x10); -+ if (ret) -+ return ret; -+ -+ /* group hold launch */ -+ ret = ov5693_write_reg(client, OV5693_8BIT, -+ OV5693_GROUP_ACCESS, 0xa0); -+ -+ return ret; -+} -+ -+static int ov5693_set_exposure(struct v4l2_subdev *sd, int exposure, -+ int gain, int digitgain) -+{ -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ int ret; -+ -+ mutex_lock(&dev->input_lock); -+ ret = __ov5693_set_exposure(sd, exposure, gain, digitgain); -+ mutex_unlock(&dev->input_lock); -+ -+ return ret; -+} -+ -+static long ov5693_s_exposure(struct v4l2_subdev *sd, -+ struct atomisp_exposure *exposure) -+{ -+ struct i2c_client *client = v4l2_get_subdevdata(sd); -+ int exp = exposure->integration_time[0]; -+ int gain = exposure->gain[0]; -+ int digitgain = exposure->gain[1]; -+ -+ /* we should not accept the invalid value below. */ -+ if (gain == 0) { -+ dev_err(&client->dev, "%s: invalid value\n", __func__); -+ return -EINVAL; -+ } -+ -+ return ov5693_set_exposure(sd, exp, gain, digitgain); -+} -+ -+static long ov5693_ioctl(struct v4l2_subdev *sd, unsigned int cmd, void *arg) -+{ -+ -+ switch (cmd) { -+ case ATOMISP_IOC_S_EXPOSURE: -+ return ov5693_s_exposure(sd, arg); -+ default: -+ return -EINVAL; -+ } -+ return 0; -+} -+ -+/* This returns the exposure time being used. This should only be used -+ for filling in EXIF data, not for actual image processing. */ -+static int ov5693_q_exposure(struct v4l2_subdev *sd, s32 *value) -+{ -+ struct i2c_client *client = v4l2_get_subdevdata(sd); -+ u16 reg_v, reg_v2; -+ int ret; -+ -+ /* get exposure */ -+ ret = ov5693_read_reg(client, OV5693_8BIT, -+ OV5693_AEC_PK_EXPO_L, -+ ®_v); -+ if (ret) -+ goto err; -+ -+ ret = ov5693_read_reg(client, OV5693_8BIT, -+ OV5693_AEC_PK_EXPO_M, -+ ®_v2); -+ if (ret) -+ goto err; -+ -+ reg_v += reg_v2 << 8; -+ ret = ov5693_read_reg(client, OV5693_8BIT, -+ OV5693_AEC_PK_EXPO_H, -+ ®_v2); -+ if (ret) -+ goto err; -+ -+ *value = (reg_v + (((u32)reg_v2 << 16))) >> 4; -+err: -+ return ret; -+} -+ -+/* -+ * This below focus func don't need input_lock mutex_lock -+ * since they are just called in v4l2 s_ctrl/g_ctrl framework -+ * where mutex input_lock have been done. -+ */ -+int ov5693_t_focus_abs(struct v4l2_subdev *sd, s32 value) -+{ -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ int ret = 0; -+ -+ if (dev->vcm_driver && dev->vcm_driver->t_focus_abs) -+ ret = dev->vcm_driver->t_focus_abs(sd, value); -+ -+ return ret; -+} -+ -+int ov5693_t_focus_rel(struct v4l2_subdev *sd, s32 value) -+{ -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ int ret = 0; -+ -+ if (dev->vcm_driver && dev->vcm_driver->t_focus_rel) -+ ret = dev->vcm_driver->t_focus_rel(sd, value); -+ -+ return ret; -+} -+ -+int ov5693_q_focus_status(struct v4l2_subdev *sd, s32 *value) -+{ -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ int ret = 0; -+ -+ if (dev->vcm_driver && dev->vcm_driver->q_focus_status) -+ ret = dev->vcm_driver->q_focus_status(sd, value); -+ -+ return ret; -+} -+ -+int ov5693_q_focus_abs(struct v4l2_subdev *sd, s32 *value) -+{ -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ int ret = 0; -+ -+ if (dev->vcm_driver && dev->vcm_driver->q_focus_abs) -+ ret = dev->vcm_driver->q_focus_abs(sd, value); -+ -+ return ret; -+} -+ -+/* ov5693 control set/get */ -+static int ov5693_g_ctrl(struct v4l2_ctrl *ctrl) -+{ -+ struct ov5693_device *dev = container_of( -+ ctrl->handler, struct ov5693_device, ctrl_handler); -+ int ret = 0; -+ -+ switch (ctrl->id) { -+ case V4L2_CID_EXPOSURE_ABSOLUTE: -+ ret = ov5693_q_exposure(&dev->sd, &ctrl->val); -+ break; -+ case V4L2_CID_FOCUS_ABSOLUTE: -+ ret = ov5693_q_focus_abs(&dev->sd, &ctrl->val); -+ break; -+ case V4L2_CID_FOCUS_STATUS: -+ ret = ov5693_q_focus_status(&dev->sd, &ctrl->val); -+ break; -+ case V4L2_CID_FOCAL_ABSOLUTE: -+ ret = ov5693_g_focal(&dev->sd, &ctrl->val); -+ break; -+ case V4L2_CID_FNUMBER_ABSOLUTE: -+ ret = ov5693_g_fnumber(&dev->sd, &ctrl->val); -+ break; -+ case V4L2_CID_FNUMBER_RANGE: -+ ret = ov5693_g_fnumber_range(&dev->sd, &ctrl->val); -+ break; -+ case V4L2_CID_BIN_FACTOR_HORZ: -+ ctrl->val = dev->ov5693_res[dev->fmt_idx].bin_factor_x; -+ break; -+ case V4L2_CID_BIN_FACTOR_VERT: -+ ctrl->val = dev->ov5693_res[dev->fmt_idx].bin_factor_y; -+ break; -+ default: -+ ret = -EINVAL; -+ } -+ -+ return ret; -+} -+ -+static int ov5693_s_ctrl(struct v4l2_ctrl *ctrl) -+{ -+ struct ov5693_device *dev = container_of( -+ ctrl->handler, struct ov5693_device, ctrl_handler); -+ int ret = 0; -+ -+ if (!ctrl) -+ return -EINVAL; -+ -+ switch (ctrl->id) { -+ case V4L2_CID_RUN_MODE: -+ switch (ctrl->val) { -+ case ATOMISP_RUN_MODE_VIDEO: -+ dev->ov5693_res = ov5693_res_video; -+ dev->curr_res_num = N_RES_VIDEO; -+ break; -+ case ATOMISP_RUN_MODE_STILL_CAPTURE: -+ dev->ov5693_res = ov5693_res_still; -+ dev->curr_res_num = N_RES_STILL; -+ break; -+ default: -+ dev->ov5693_res = ov5693_res_preview; -+ dev->curr_res_num = N_RES_PREVIEW; -+ } -+ break; -+ case V4L2_CID_FOCUS_ABSOLUTE: -+ ret = ov5693_t_focus_abs(&dev->sd, ctrl->val); -+ break; -+ case V4L2_CID_FOCUS_RELATIVE: -+ ret = ov5693_t_focus_rel(&dev->sd, ctrl->val); -+ break; -+ default: -+ ret = -EINVAL; -+ } -+ -+ return ret; -+} -+ -+static int ov5693_init(struct v4l2_subdev *sd) -+{ -+ struct i2c_client *client = v4l2_get_subdevdata(sd); -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ int ret = 0; -+ -+ /* restore settings */ -+ dev->ov5693_res = ov5693_res_preview; -+ dev->curr_res_num = N_RES_PREVIEW; -+ -+ ret = ov5693_write_reg_array(client, ov5693_init_setting); -+ if (ret) -+ dev_err(&client->dev, "ov5693 write init setting reg err.\n"); -+ -+ return ret; -+} -+ -+ -+static int power_up(struct v4l2_subdev *sd) -+{ -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ struct i2c_client *client = v4l2_get_subdevdata(sd); -+ int ret; -+ -+ if (NULL == dev->platform_data) { -+ dev_err(&client->dev, -+ "no camera_sensor_platform_data"); -+ return -ENODEV; -+ } -+ -+ /* power control */ -+ ret = dev->platform_data->power_ctrl(sd, 1); -+ if (ret) -+ goto fail_power; -+ -+ /* gpio ctrl */ -+ ret = dev->platform_data->gpio_ctrl(sd, 1); -+ if (ret) -+ goto fail_power; -+ -+ /* flis clock control */ -+ ret = dev->platform_data->flisclk_ctrl(sd, 1); -+ if (ret) -+ goto fail_clk; -+ -+ /* according to DS, 20ms is needed between PWDN and i2c access */ -+ msleep(20); -+ -+ return 0; -+ -+fail_clk: -+ dev->platform_data->gpio_ctrl(sd, 0); -+fail_power: -+ dev->platform_data->power_ctrl(sd, 0); -+ dev_err(&client->dev, "sensor power-up failed\n"); -+ -+ return ret; -+} -+ -+static int power_down(struct v4l2_subdev *sd) -+{ -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ struct i2c_client *client = v4l2_get_subdevdata(sd); -+ int ret = 0; -+ -+ if (NULL == dev->platform_data) { -+ dev_err(&client->dev, -+ "no camera_sensor_platform_data"); -+ return -ENODEV; -+ } -+ -+ ret = dev->platform_data->flisclk_ctrl(sd, 0); -+ if (ret) -+ dev_err(&client->dev, "flisclk failed\n"); -+ -+ /* gpio ctrl */ -+ ret = dev->platform_data->gpio_ctrl(sd, 0); -+ if (ret) -+ dev_err(&client->dev, "gpio failed.\n"); -+ -+ /* power control */ -+ ret = dev->platform_data->power_ctrl(sd, 0); -+ if (ret) -+ dev_err(&client->dev, "vprog failed.\n"); -+ -+ return ret; -+} -+ -+static int ov5693_s_power(struct v4l2_subdev *sd, int on) -+{ -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ struct i2c_client *client = v4l2_get_subdevdata(sd); -+ int ret = 0; -+ -+ mutex_lock(&dev->input_lock); -+ if (on == 0) { -+ if (dev->vcm_driver && dev->vcm_driver->power_down) -+ ret = dev->vcm_driver->power_down(sd); -+ if (ret) -+ dev_err(&client->dev, "vcm power-down failed.\n"); -+ -+ ret = power_down(sd); -+ } else { -+ if (dev->vcm_driver && dev->vcm_driver->power_up) -+ ret = dev->vcm_driver->power_up(sd); -+ if (ret) -+ dev_err(&client->dev, "vcm power-up failed.\n"); -+ -+ ret = power_up(sd); -+ if (!ret) -+ ret = ov5693_init(sd); -+ } -+ mutex_unlock(&dev->input_lock); -+ return ret; -+} -+ -+/* -+ * distance - calculate the distance -+ * @res: resolution -+ * @w: width -+ * @h: height -+ * -+ * Get the gap between resolution and w/h. -+ * res->width/height smaller than w/h wouldn't be considered. -+ * Returns the value of gap or -1 if fail. -+ */ -+static int distance(struct ov5693_resolution *res, u32 w, u32 h) -+{ -+ unsigned int w_ratio = ((res->width << RATIO_SHIFT_BITS)/w); -+ unsigned int h_ratio; -+ int match; -+ -+ if (h == 0) -+ return -1; -+ h_ratio = ((res->height << RATIO_SHIFT_BITS) / h); -+ if (h_ratio == 0) -+ return -1; -+ match = abs(((w_ratio << RATIO_SHIFT_BITS) / h_ratio) -+ - ((int)(1 << RATIO_SHIFT_BITS))); -+ -+ if ((w_ratio < (int)(1 << RATIO_SHIFT_BITS)) -+ || (h_ratio < (int)(1 << RATIO_SHIFT_BITS)) || -+ (match > LARGEST_ALLOWED_RATIO_MISMATCH)) -+ return -1; -+ -+ return w_ratio + h_ratio; -+} -+ -+/* Return the nearest higher resolution index */ -+static int nearest_resolution_index(struct v4l2_subdev *sd, -+ int w, int h) -+{ -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ int i; -+ int idx = dev->curr_res_num-1; -+ int dist; -+ int min_dist = INT_MAX; -+ struct ov5693_resolution *tmp_res = NULL; -+ -+ for (i = 0; i < dev->curr_res_num; i++) { -+ tmp_res = &dev->ov5693_res[i]; -+ dist = distance(tmp_res, w, h); -+ if (dist == -1) -+ continue; -+ if (dist < min_dist) { -+ min_dist = dist; -+ idx = i; -+ } -+ } -+ -+ return idx; -+} -+ -+static int get_resolution_index(struct v4l2_subdev *sd, -+ int w, int h) -+{ -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ int i; -+ -+ for (i = 0; i < dev->curr_res_num; i++) { -+ if (w != dev->ov5693_res[i].width) -+ continue; -+ if (h != dev->ov5693_res[i].height) -+ continue; -+ -+ return i; -+ } -+ -+ return -1; -+} -+ -+static int __ov5693_try_mbus_fmt(struct v4l2_subdev *sd, -+ struct v4l2_mbus_framefmt *fmt) -+{ -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ int idx; -+ -+ if (!fmt) -+ return -EINVAL; -+ -+ idx = nearest_resolution_index(sd, fmt->width, fmt->height); -+ fmt->width = dev->ov5693_res[idx].width; -+ fmt->height = dev->ov5693_res[idx].height; -+ fmt->code = V4L2_MBUS_FMT_SGRBG10_1X10; -+ -+ return 0; -+} -+ -+static int ov5693_try_mbus_fmt(struct v4l2_subdev *sd, -+ struct v4l2_mbus_framefmt *fmt) -+{ -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ int ret; -+ -+ mutex_lock(&dev->input_lock); -+ ret = __ov5693_try_mbus_fmt(sd, fmt); -+ mutex_unlock(&dev->input_lock); -+ -+ return ret; -+} -+ -+static int ov5693_s_mbus_fmt(struct v4l2_subdev *sd, -+ struct v4l2_mbus_framefmt *fmt) -+{ -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ struct i2c_client *client = v4l2_get_subdevdata(sd); -+ struct camera_mipi_info *ov5693_info = NULL; -+ int ret = 0; -+ -+ ov5693_info = v4l2_get_subdev_hostdata(sd); -+ if (ov5693_info == NULL) -+ return -EINVAL; -+ -+ mutex_lock(&dev->input_lock); -+ ret = __ov5693_try_mbus_fmt(sd, fmt); -+ if (ret == -1) { -+ dev_err(&client->dev, "try fmt fail\n"); -+ goto done; -+ } -+ -+ dev->fmt_idx = get_resolution_index(sd, fmt->width, fmt->height); -+ if (dev->fmt_idx == -1) { -+ dev_err(&client->dev, "get resolution fail\n"); -+ goto done; -+ } -+ -+ ret = ov5693_write_reg_array(client, dev->ov5693_res[dev->fmt_idx].regs); -+ if (ret) { -+ dev_err(&client->dev, "ov5693 write fmt register err.\n"); -+ goto done; -+ } -+ -+ ret = ov5693_get_intg_factor(client, ov5693_info, -+ &dev->ov5693_res[dev->fmt_idx]); -+ if (ret) -+ dev_err(&client->dev, "failed to get integration_factor\n"); -+ -+done: -+ mutex_unlock(&dev->input_lock); -+ return ret; -+} -+static int ov5693_g_mbus_fmt(struct v4l2_subdev *sd, -+ struct v4l2_mbus_framefmt *fmt) -+{ -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ -+ mutex_lock(&dev->input_lock); -+ fmt->width = dev->ov5693_res[dev->fmt_idx].width; -+ fmt->height = dev->ov5693_res[dev->fmt_idx].height; -+ fmt->code = V4L2_MBUS_FMT_SBGGR10_1X10; -+ mutex_unlock(&dev->input_lock); -+ -+ return 0; -+} -+ -+static int ov5693_detect(struct i2c_client *client) -+{ -+ struct i2c_adapter *adapter = client->adapter; -+ int ret = 0; -+ u16 id; -+ u8 revision; -+ -+ if (!i2c_check_functionality(adapter, I2C_FUNC_I2C)) -+ return -ENODEV; -+ -+ ret = ov5693_read_reg(client, OV5693_16BIT, -+ OV5693_SC_CMMN_CHIP_ID, &id); -+ if (ret) { -+ dev_err(&client->dev, "read sensor_id err.\n"); -+ return -ENODEV; -+ } -+ -+ if (id != OV5693_ID) { -+ dev_err(&client->dev, "sensor ID error\n"); -+ return -ENODEV; -+ } -+ -+ ret = ov5693_read_reg(client, OV5693_8BIT, -+ OV5693_SC_CMMN_SUB_ID, &id); -+ revision = (u8)id & 0x0f; -+ -+ dev_dbg(&client->dev, "sensor_revision = 0x%x\n", revision); -+ dev_dbg(&client->dev, "detect ov5693 success\n"); -+ return ret; -+} -+ -+static int ov5693_s_stream(struct v4l2_subdev *sd, int enable) -+{ -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ struct i2c_client *client = v4l2_get_subdevdata(sd); -+ int ret; -+ -+ mutex_lock(&dev->input_lock); -+ -+ ret = ov5693_write_reg(client, OV5693_8BIT, OV5693_SW_STREAM, -+ enable ? OV5693_START_STREAMING : -+ OV5693_STOP_STREAMING); -+ -+ mutex_unlock(&dev->input_lock); -+ return ret; -+} -+ -+/* ov5693 enum frame size, frame intervals */ -+static int ov5693_enum_framesizes(struct v4l2_subdev *sd, -+ struct v4l2_frmsizeenum *fsize) -+{ -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ unsigned int index = fsize->index; -+ -+ mutex_lock(&dev->input_lock); -+ -+ if (index >= dev->curr_res_num) { -+ mutex_unlock(&dev->input_lock); -+ return -EINVAL; -+ } -+ -+ fsize->type = V4L2_FRMSIZE_TYPE_DISCRETE; -+ fsize->discrete.width = dev->ov5693_res[index].width; -+ fsize->discrete.height = dev->ov5693_res[index].height; -+ -+ mutex_unlock(&dev->input_lock); -+ return 0; -+} -+ -+static int ov5693_enum_frameintervals(struct v4l2_subdev *sd, -+ struct v4l2_frmivalenum *fival) -+{ -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ unsigned int index = fival->index; -+ -+ mutex_lock(&dev->input_lock); -+ -+ if (index >= dev->curr_res_num) { -+ mutex_unlock(&dev->input_lock); -+ return -EINVAL; -+ } -+ -+ fival->type = V4L2_FRMIVAL_TYPE_DISCRETE; -+ fival->width = dev->ov5693_res[index].width; -+ fival->height = dev->ov5693_res[index].height; -+ fival->discrete.numerator = 1; -+ fival->discrete.denominator = dev->ov5693_res[index].fps; -+ -+ mutex_unlock(&dev->input_lock); -+ -+ return 0; -+} -+ -+static int ov5693_enum_mbus_fmt(struct v4l2_subdev *sd, -+ unsigned int index, -+ enum v4l2_mbus_pixelcode *code) -+{ -+ *code = V4L2_MBUS_FMT_SBGGR10_1X10; -+ -+ return 0; -+} -+ -+static int ov5693_s_config(struct v4l2_subdev *sd, -+ int irq, void *platform_data) -+{ -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ struct i2c_client *client = v4l2_get_subdevdata(sd); -+ int ret = 0; -+ -+ if (platform_data == NULL) -+ return -ENODEV; -+ -+ mutex_lock(&dev->input_lock); -+ -+ dev->platform_data = platform_data; -+ -+ ret = power_up(sd); -+ if (ret) { -+ dev_err(&client->dev, "ov5693 power-up err.\n"); -+ goto fail_power_on; -+ } -+ -+ ret = dev->platform_data->csi_cfg(sd, 1); -+ if (ret) -+ goto fail_csi_cfg; -+ -+ /* config & detect sensor */ -+ ret = ov5693_detect(client); -+ if (ret) { -+ dev_err(&client->dev, "ov5693_detect err s_config.\n"); -+ goto fail_csi_cfg; -+ } -+ -+ /* turn off sensor, after probed */ -+ ret = power_down(sd); -+ if (ret) { -+ dev_err(&client->dev, "ov5693 power-off err.\n"); -+ goto fail_csi_cfg; -+ } -+ mutex_unlock(&dev->input_lock); -+ -+ return 0; -+ -+fail_csi_cfg: -+ dev->platform_data->csi_cfg(sd, 0); -+fail_power_on: -+ power_down(sd); -+ dev_err(&client->dev, "sensor power-gating failed\n"); -+ mutex_unlock(&dev->input_lock); -+ return ret; -+} -+ -+static int ov5693_g_parm(struct v4l2_subdev *sd, -+ struct v4l2_streamparm *param) -+{ -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ struct i2c_client *client = v4l2_get_subdevdata(sd); -+ -+ if (param->type != V4L2_BUF_TYPE_VIDEO_CAPTURE) { -+ dev_err(&client->dev, "unsupported buffer type.\n"); -+ return -EINVAL; -+ } -+ -+ memset(param, 0, sizeof(*param)); -+ param->type = V4L2_BUF_TYPE_VIDEO_CAPTURE; -+ -+ mutex_lock(&dev->input_lock); -+ if (dev->fmt_idx >= 0 && dev->fmt_idx < dev->curr_res_num) { -+ param->parm.capture.capability = V4L2_CAP_TIMEPERFRAME; -+ param->parm.capture.timeperframe.numerator = 1; -+ param->parm.capture.capturemode = dev->run_mode->val; -+ param->parm.capture.timeperframe.denominator = -+ dev->ov5693_res[dev->fmt_idx].fps; -+ } -+ mutex_unlock(&dev->input_lock); -+ return 0; -+} -+ -+static int ov5693_g_frame_interval(struct v4l2_subdev *sd, -+ struct v4l2_subdev_frame_interval *interval) -+{ -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ -+ mutex_lock(&dev->input_lock); -+ interval->interval.numerator = 1; -+ interval->interval.denominator = dev->ov5693_res[dev->fmt_idx].fps; -+ mutex_unlock(&dev->input_lock); -+ -+ return 0; -+} -+ -+static int ov5693_enum_mbus_code(struct v4l2_subdev *sd, -+ struct v4l2_subdev_fh *fh, -+ struct v4l2_subdev_mbus_code_enum *code) -+{ -+ code->code = V4L2_MBUS_FMT_SBGGR10_1X10; -+ return 0; -+} -+ -+static int ov5693_enum_frame_size(struct v4l2_subdev *sd, -+ struct v4l2_subdev_fh *fh, -+ struct v4l2_subdev_frame_size_enum *fse) -+{ -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ int index = fse->index; -+ -+ mutex_lock(&dev->input_lock); -+ -+ if (index >= dev->curr_res_num) { -+ mutex_unlock(&dev->input_lock); -+ return -EINVAL; -+ } -+ -+ fse->min_width = dev->ov5693_res[index].width; -+ fse->min_height = dev->ov5693_res[index].height; -+ fse->max_width = dev->ov5693_res[index].width; -+ fse->max_height = dev->ov5693_res[index].height; -+ -+ mutex_unlock(&dev->input_lock); -+ return 0; -+} -+ -+static int ov5693_get_pad_format(struct v4l2_subdev *sd, -+ struct v4l2_subdev_fh *fh, -+ struct v4l2_subdev_format *fmt) -+{ -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ struct v4l2_mbus_framefmt *format; -+ -+ mutex_lock(&dev->input_lock); -+ -+ switch (fmt->which) { -+ case V4L2_SUBDEV_FORMAT_TRY: -+ format = v4l2_subdev_get_try_format(fh, fmt->pad); -+ break; -+ case V4L2_SUBDEV_FORMAT_ACTIVE: -+ format = &dev->format; -+ break; -+ default: -+ format = NULL; -+ } -+ -+ mutex_unlock(&dev->input_lock); -+ -+ if (!format) -+ return -EINVAL; -+ -+ fmt->format = *format; -+ return 0; -+} -+ -+static int ov5693_set_pad_format(struct v4l2_subdev *sd, -+ struct v4l2_subdev_fh *fh, -+ struct v4l2_subdev_format *fmt) -+{ -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ -+ mutex_lock(&dev->input_lock); -+ -+ if (fmt->which == V4L2_SUBDEV_FORMAT_ACTIVE) -+ dev->format = fmt->format; -+ -+ mutex_unlock(&dev->input_lock); -+ return 0; -+} -+ -+static int ov5693_g_skip_frames(struct v4l2_subdev *sd, u32 *frames) -+{ -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ -+ mutex_lock(&dev->input_lock); -+ *frames = dev->ov5693_res[dev->fmt_idx].skip_frames; -+ mutex_unlock(&dev->input_lock); -+ -+ return 0; -+} -+ -+static const struct v4l2_ctrl_ops ctrl_ops = { -+ .s_ctrl = ov5693_s_ctrl, -+ .g_volatile_ctrl = ov5693_g_ctrl, -+}; -+ -+static const char * const ctrl_run_mode_menu[] = { -+ NULL, -+ "Video", -+ "Still capture", -+ "Continuous capture", -+ "Preview", -+}; -+ -+static const struct v4l2_ctrl_config ctrl_run_mode = { -+ .ops = &ctrl_ops, -+ .id = V4L2_CID_RUN_MODE, -+ .name = "run mode", -+ .type = V4L2_CTRL_TYPE_MENU, -+ .min = 1, -+ .def = 4, -+ .max = 4, -+ .qmenu = ctrl_run_mode_menu, -+}; -+ -+static const struct v4l2_ctrl_config ctrls[] = { -+ { -+ .ops = &ctrl_ops, -+ .id = V4L2_CID_EXPOSURE_ABSOLUTE, -+ .name = "absolute exposure", -+ .type = V4L2_CTRL_TYPE_INTEGER, -+ .min = 0x0, -+ .max = 0xffff, -+ .step = 0x01, -+ .def = 0x00, -+ .flags = 0, -+ }, { -+ .ops = &ctrl_ops, -+ .id = V4L2_CID_FOCUS_ABSOLUTE, -+ .type = V4L2_CTRL_TYPE_INTEGER, -+ .name = "focus move absolute", -+ .min = 0, -+ .max = OV5693_MAX_FOCUS_POS, -+ .step = 1, -+ .def = 0, -+ .flags = 0, -+ }, { -+ .ops = &ctrl_ops, -+ .id = V4L2_CID_FOCUS_RELATIVE, -+ .type = V4L2_CTRL_TYPE_INTEGER, -+ .name = "focus move relative", -+ .min = OV5693_MAX_FOCUS_NEG, -+ .max = OV5693_MAX_FOCUS_POS, -+ .step = 1, -+ .def = 0, -+ .flags = 0, -+ }, { -+ .ops = &ctrl_ops, -+ .id = V4L2_CID_FOCUS_STATUS, -+ .type = V4L2_CTRL_TYPE_INTEGER, -+ .name = "focus status", -+ .min = 0, -+ .max = 100, -+ .step = 1, -+ .def = 0, -+ .flags = 0, -+ }, { -+ .ops = &ctrl_ops, -+ .id = V4L2_CID_FOCAL_ABSOLUTE, -+ .type = V4L2_CTRL_TYPE_INTEGER, -+ .name = "focal length", -+ .min = OV5693_FOCAL_LENGTH_DEFAULT, -+ .max = OV5693_FOCAL_LENGTH_DEFAULT, -+ .step = 0x01, -+ .def = OV5693_FOCAL_LENGTH_DEFAULT, -+ .flags = 0, -+ }, { -+ .ops = &ctrl_ops, -+ .id = V4L2_CID_FNUMBER_ABSOLUTE, -+ .type = V4L2_CTRL_TYPE_INTEGER, -+ .name = "f-number", -+ .min = OV5693_F_NUMBER_DEFAULT, -+ .max = OV5693_F_NUMBER_DEFAULT, -+ .step = 0x01, -+ .def = OV5693_F_NUMBER_DEFAULT, -+ .flags = 0, -+ }, { -+ .ops = &ctrl_ops, -+ .id = V4L2_CID_FNUMBER_RANGE, -+ .type = V4L2_CTRL_TYPE_INTEGER, -+ .name = "f-number range", -+ .min = OV5693_F_NUMBER_RANGE, -+ .max = OV5693_F_NUMBER_RANGE, -+ .step = 0x01, -+ .def = OV5693_F_NUMBER_RANGE, -+ .flags = 0, -+ }, { -+ .ops = &ctrl_ops, -+ .id = V4L2_CID_BIN_FACTOR_HORZ, -+ .name = "horizontal binning factor", -+ .type = V4L2_CTRL_TYPE_INTEGER, -+ .max = OV5693_BIN_FACTOR_MAX, -+ .step = 2, -+ .flags = V4L2_CTRL_FLAG_READ_ONLY | V4L2_CTRL_FLAG_VOLATILE, -+ }, { -+ .ops = &ctrl_ops, -+ .id = V4L2_CID_BIN_FACTOR_VERT, -+ .name = "vertical binning factor", -+ .type = V4L2_CTRL_TYPE_INTEGER, -+ .max = OV5693_BIN_FACTOR_MAX, -+ .step = 2, -+ .flags = V4L2_CTRL_FLAG_READ_ONLY | V4L2_CTRL_FLAG_VOLATILE, -+ } -+}; -+ -+static const struct v4l2_subdev_sensor_ops ov5693_sensor_ops = { -+ .g_skip_frames = ov5693_g_skip_frames, -+}; -+ -+static const struct v4l2_subdev_video_ops ov5693_video_ops = { -+ .s_stream = ov5693_s_stream, -+ .g_parm = ov5693_g_parm, -+ .enum_framesizes = ov5693_enum_framesizes, -+ .enum_frameintervals = ov5693_enum_frameintervals, -+ .enum_mbus_fmt = ov5693_enum_mbus_fmt, -+ .try_mbus_fmt = ov5693_try_mbus_fmt, -+ .g_mbus_fmt = ov5693_g_mbus_fmt, -+ .s_mbus_fmt = ov5693_s_mbus_fmt, -+ .g_frame_interval = ov5693_g_frame_interval, -+}; -+ -+static const struct v4l2_subdev_core_ops ov5693_core_ops = { -+ .s_power = ov5693_s_power, -+ .queryctrl = v4l2_subdev_queryctrl, -+ .g_ctrl = v4l2_subdev_g_ctrl, -+ .s_ctrl = v4l2_subdev_s_ctrl, -+ .ioctl = ov5693_ioctl, -+}; -+ -+static const struct v4l2_subdev_pad_ops ov5693_pad_ops = { -+ .enum_mbus_code = ov5693_enum_mbus_code, -+ .enum_frame_size = ov5693_enum_frame_size, -+ .get_fmt = ov5693_get_pad_format, -+ .set_fmt = ov5693_set_pad_format, -+}; -+ -+static const struct v4l2_subdev_ops ov5693_ops = { -+ .core = &ov5693_core_ops, -+ .video = &ov5693_video_ops, -+ .pad = &ov5693_pad_ops, -+ .sensor = &ov5693_sensor_ops, -+}; -+ -+static int ov5693_remove(struct i2c_client *client) -+{ -+ struct v4l2_subdev *sd = i2c_get_clientdata(client); -+ struct ov5693_device *dev = to_ov5693_sensor(sd); -+ dev_dbg(&client->dev, "ov5693_remove...\n"); -+ -+ dev->platform_data->csi_cfg(sd, 0); -+ -+ v4l2_device_unregister_subdev(sd); -+ media_entity_cleanup(&dev->sd.entity); -+ devm_kfree(&client->dev, dev); -+ -+ return 0; -+} -+ -+static int ov5693_probe(struct i2c_client *client, -+ const struct i2c_device_id *id) -+{ -+ struct ov5693_device *dev; -+ int i; -+ int ret; -+ -+ dev = devm_kzalloc(&client->dev, sizeof(*dev), GFP_KERNEL); -+ if (!dev) { -+ dev_err(&client->dev, "out of memory\n"); -+ return -ENOMEM; -+ } -+ -+ mutex_init(&dev->input_lock); -+ -+ /* -+ * Initialize related res members of dev. -+ */ -+ dev->fmt_idx = 0; -+ dev->ov5693_res = ov5693_res_preview; -+ dev->curr_res_num = N_RES_PREVIEW; -+ -+ v4l2_i2c_subdev_init(&(dev->sd), client, &ov5693_ops); -+ -+ if (client->dev.platform_data) { -+ ret = ov5693_s_config(&dev->sd, client->irq, -+ client->dev.platform_data); -+ if (ret) -+ goto out_free; -+ } -+ -+ dev->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE; -+ dev->pad.flags = MEDIA_PAD_FL_SOURCE; -+ dev->format.code = V4L2_MBUS_FMT_SBGGR10_1X10; -+ dev->sd.entity.type = MEDIA_ENT_T_V4L2_SUBDEV_SENSOR; -+ dev->vcm_driver = &ov5693_vcm_ops; -+ -+ ret = v4l2_ctrl_handler_init(&dev->ctrl_handler, ARRAY_SIZE(ctrls) + 1); -+ if (ret) { -+ ov5693_remove(client); -+ return ret; -+ } -+ -+ dev->run_mode = v4l2_ctrl_new_custom(&dev->ctrl_handler, -+ &ctrl_run_mode, NULL); -+ -+ for (i = 0; i < ARRAY_SIZE(ctrls); i++) -+ v4l2_ctrl_new_custom(&dev->ctrl_handler, &ctrls[i], NULL); -+ -+ if (dev->ctrl_handler.error) { -+ ov5693_remove(client); -+ return dev->ctrl_handler.error; -+ } -+ -+ dev->ctrl_handler.lock = &dev->input_lock; -+ dev->sd.ctrl_handler = &dev->ctrl_handler; -+ v4l2_ctrl_handler_setup(&dev->ctrl_handler); -+ -+ ret = media_entity_init(&dev->sd.entity, 1, &dev->pad, 0); -+ if (ret) -+ ov5693_remove(client); -+ -+ /* vcm initialization */ -+ if (dev->vcm_driver && dev->vcm_driver->init) -+ ret = dev->vcm_driver->init(&dev->sd); -+ if (ret) { -+ dev_err(&client->dev, "vcm init failed.\n"); -+ ov5693_remove(client); -+ } -+ -+ return ret; -+out_free: -+ v4l2_device_unregister_subdev(&dev->sd); -+ devm_kfree(&client->dev, dev); -+ return ret; -+} -+ -+MODULE_DEVICE_TABLE(i2c, ov5693_id); -+static struct i2c_driver ov5693_driver = { -+ .driver = { -+ .owner = THIS_MODULE, -+ .name = OV5693_NAME, -+ }, -+ .probe = ov5693_probe, -+ .remove = ov5693_remove, -+ .id_table = ov5693_id, -+}; -+ -+module_i2c_driver(ov5693_driver); -+ -+MODULE_DESCRIPTION("A low-level driver for OmniVision 5693 sensors"); -+MODULE_LICENSE("GPL"); -diff --git a/drivers/staging/ov5693/ov5693.h b/drivers/staging/ov5693/ov5693.h -new file mode 100644 -index 000000000000..79aef69666e8 ---- /dev/null -+++ b/drivers/staging/ov5693/ov5693.h -@@ -0,0 +1,848 @@ -+/* -+ * Support for OmniVision OV5693 5M HD camera sensor. -+ * -+ * Copyright (c) 2013 Intel Corporation. All Rights Reserved. -+ * -+ * This program is free software; you can redistribute it and/or -+ * modify it under the terms of the GNU General Public License version -+ * 2 as published by the Free Software Foundation. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, write to the Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA -+ * 02110-1301, USA. -+ * -+ */ -+ -+#ifndef __OV5693_H__ -+#define __OV5693_H__ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+#include "ad5823.h" -+ -+#define OV5693_NAME "ov5693" -+ -+/* Defines for register writes and register array processing */ -+#define I2C_MSG_LENGTH 0x2 -+ -+#define OV5693_FOCAL_LENGTH_NUM 278 /*2.78mm*/ -+#define OV5693_FOCAL_LENGTH_DEM 100 -+#define OV5693_F_NUMBER_DEFAULT_NUM 26 -+#define OV5693_F_NUMBER_DEM 10 -+ -+#define OV5693_MAX_FOCUS_POS 1023 -+#define OV5693_MAX_FOCUS_POS 1023 -+#define OV5693_MAX_FOCUS_NEG (-1023) -+ -+#define LARGEST_ALLOWED_RATIO_MISMATCH 800 -+#define RATIO_SHIFT_BITS 13 -+ -+/* -+ * focal length bits definition: -+ * bits 31-16: numerator, bits 15-0: denominator -+ */ -+#define OV5693_FOCAL_LENGTH_DEFAULT 0x1160064 -+ -+/* -+ * current f-number bits definition: -+ * bits 31-16: numerator, bits 15-0: denominator -+ */ -+#define OV5693_F_NUMBER_DEFAULT 0x1a000a -+ -+/* -+ * f-number range bits definition: -+ * bits 31-24: max f-number numerator -+ * bits 23-16: max f-number denominator -+ * bits 15-8: min f-number numerator -+ * bits 7-0: min f-number denominator -+ */ -+#define OV5693_F_NUMBER_RANGE 0x1a0a1a0a -+#define OV5693_ID 0x5690 -+ -+#define OV5693_FINE_INTG_TIME_MIN 0 -+#define OV5693_FINE_INTG_TIME_MAX_MARGIN 0 -+#define OV5693_COARSE_INTG_TIME_MIN 1 -+#define OV5693_COARSE_INTG_TIME_MAX_MARGIN (0xffff - 6) -+#define OV5693_INTEGRATION_TIME_MARGIN 8 -+ -+#define OV5693_BIN_FACTOR_MAX 2 -+ -+/* -+ * OV5693 System control registers -+ */ -+#define OV5693_SW_RESET 0x0103 -+#define OV5693_SW_STREAM 0x0100 -+ -+#define OV5693_SC_CMMN_CHIP_ID 0x300a -+#define OV5693_SC_CMMN_SUB_ID 0x302a /* process, version*/ -+ -+#define OV5693_AEC_PK_EXPO_H 0x3500 -+#define OV5693_AEC_PK_EXPO_M 0x3501 -+#define OV5693_AEC_PK_EXPO_L 0x3502 -+#define OV5693_AGC_ADJ_H 0x350a -+#define OV5693_VTS_H 0x380e -+#define OV5693_GROUP_ACCESS 0x3208 -+ -+#define OV5693_MWB_GAIN_R_H 0x3400 -+#define OV5693_MWB_GAIN_G_H 0x3402 -+#define OV5693_MWB_GAIN_B_H 0x3404 -+ -+#define OV5693_H_CROP_START_H 0x3800 -+#define OV5693_V_CROP_START_H 0x3802 -+#define OV5693_H_CROP_END_H 0x3804 -+#define OV5693_V_CROP_END_H 0x3806 -+#define OV5693_H_OUTSIZE_H 0x3808 -+#define OV5693_V_OUTSIZE_H 0x380a -+ -+#define OV5693_START_STREAMING 0x01 -+#define OV5693_STOP_STREAMING 0x00 -+ -+struct ov5693_vcm { -+ int (*power_up)(struct v4l2_subdev *sd); -+ int (*power_down)(struct v4l2_subdev *sd); -+ int (*init)(struct v4l2_subdev *sd); -+ int (*t_focus_vcm)(struct v4l2_subdev *sd, u16 val); -+ int (*t_focus_abs)(struct v4l2_subdev *sd, s32 value); -+ int (*t_focus_rel)(struct v4l2_subdev *sd, s32 value); -+ int (*q_focus_status)(struct v4l2_subdev *sd, s32 *value); -+ int (*q_focus_abs)(struct v4l2_subdev *sd, s32 *value); -+}; -+ -+struct ov5693_resolution { -+ u8 *desc; -+ const struct ov5693_reg *regs; -+ int res; -+ int width; -+ int height; -+ int fps; -+ int pix_clk_freq; -+ u16 skip_frames; -+ u16 pixels_per_line; -+ u16 lines_per_frame; -+ u8 bin_factor_x; -+ u8 bin_factor_y; -+ u8 bin_mode; -+ bool used; -+}; -+ -+struct ov5693_control { -+ struct v4l2_queryctrl qc; -+ int (*query)(struct v4l2_subdev *sd, s32 *value); -+ int (*tweak)(struct v4l2_subdev *sd, s32 value); -+}; -+ -+/* -+ * ov5693 device structure. -+ */ -+struct ov5693_device { -+ struct v4l2_subdev sd; -+ struct media_pad pad; -+ struct v4l2_mbus_framefmt format; -+ struct mutex input_lock; -+ -+ struct camera_sensor_platform_data *platform_data; -+ struct ov5693_vcm *vcm_driver; -+ int fmt_idx; -+ u8 res; -+ u8 type; -+ -+ struct ov5693_resolution *ov5693_res; -+ int curr_res_num; -+ -+ struct v4l2_ctrl_handler ctrl_handler; -+ struct v4l2_ctrl *run_mode; -+}; -+ -+enum ov5693_tok_type { -+ OV5693_8BIT = 0x0001, -+ OV5693_16BIT = 0x0002, -+ OV5693_32BIT = 0x0004, -+ OV5693_TOK_TERM = 0xf000, /* terminating token for reg list */ -+ OV5693_TOK_DELAY = 0xfe00, /* delay token for reg list */ -+ OV5693_TOK_MASK = 0xfff0 -+}; -+ -+/** -+ * struct ov5693_reg - MI sensor register format -+ * @type: type of the register -+ * @reg: 16-bit offset to register -+ * @val: 8/16/32-bit register value -+ * -+ * Define a structure for sensor register initialization values -+ */ -+struct ov5693_reg { -+ enum ov5693_tok_type type; -+ u16 reg; -+ u32 val; /* @set value for read/mod/write, @mask */ -+}; -+ -+#define to_ov5693_sensor(x) container_of(x, struct ov5693_device, sd) -+ -+#define OV5693_MAX_WRITE_BUF_SIZE 30 -+ -+struct ov5693_write_buffer { -+ u16 addr; -+ u8 data[OV5693_MAX_WRITE_BUF_SIZE]; -+}; -+ -+struct ov5693_write_ctrl { -+ int index; -+ struct ov5693_write_buffer buffer; -+}; -+ -+static const struct i2c_device_id ov5693_id[] = { -+ {OV5693_NAME, 0}, -+ {} -+}; -+ -+/* ov5693 sensor initialization setting */ -+static struct ov5693_reg const ov5693_init_setting[] = { -+ {OV5693_8BIT, 0x0103, 0x01}, -+ {OV5693_8BIT, 0x3001, 0x0a}, -+ {OV5693_8BIT, 0x3002, 0x80}, -+ {OV5693_8BIT, 0x3006, 0x00}, -+ {OV5693_8BIT, 0x3011, 0x21}, -+ {OV5693_8BIT, 0x3012, 0x09}, -+ {OV5693_8BIT, 0x3013, 0x10}, -+ {OV5693_8BIT, 0x3014, 0x00}, -+ {OV5693_8BIT, 0x3015, 0x08}, -+ {OV5693_8BIT, 0x3016, 0xf0}, -+ {OV5693_8BIT, 0x3017, 0xf0}, -+ {OV5693_8BIT, 0x3018, 0xf0}, -+ {OV5693_8BIT, 0x301b, 0xb4}, -+ {OV5693_8BIT, 0x301d, 0x02}, -+ {OV5693_8BIT, 0x3021, 0x00}, -+ {OV5693_8BIT, 0x3022, 0x01}, -+ {OV5693_8BIT, 0x3028, 0x44}, -+ {OV5693_8BIT, 0x3098, 0x02}, -+ {OV5693_8BIT, 0x3099, 0x19}, -+ {OV5693_8BIT, 0x309a, 0x02}, -+ {OV5693_8BIT, 0x309b, 0x01}, -+ {OV5693_8BIT, 0x309c, 0x00}, -+ {OV5693_8BIT, 0x30a0, 0xd2}, -+ {OV5693_8BIT, 0x30a2, 0x01}, -+ {OV5693_8BIT, 0x30b2, 0x00}, -+ {OV5693_8BIT, 0x30b3, 0x7d}, -+ {OV5693_8BIT, 0x30b4, 0x03}, -+ {OV5693_8BIT, 0x30b5, 0x04}, -+ {OV5693_8BIT, 0x30b6, 0x01}, -+ {OV5693_8BIT, 0x3104, 0x21}, -+ {OV5693_8BIT, 0x3106, 0x00}, -+ -+ /* Manual white balance */ -+ {OV5693_8BIT, 0x3400, 0x04}, -+ {OV5693_8BIT, 0x3401, 0x00}, -+ {OV5693_8BIT, 0x3402, 0x04}, -+ {OV5693_8BIT, 0x3403, 0x00}, -+ {OV5693_8BIT, 0x3404, 0x04}, -+ {OV5693_8BIT, 0x3405, 0x00}, -+ {OV5693_8BIT, 0x3406, 0x01}, -+ -+ /* Manual exposure control */ -+ {OV5693_8BIT, 0x3500, 0x00}, -+ {OV5693_8BIT, 0x3503, 0x07}, -+ {OV5693_8BIT, 0x3504, 0x00}, -+ {OV5693_8BIT, 0x3505, 0x00}, -+ {OV5693_8BIT, 0x3506, 0x00}, -+ {OV5693_8BIT, 0x3507, 0x02}, -+ {OV5693_8BIT, 0x3508, 0x00}, -+ -+ /* Manual gain control */ -+ {OV5693_8BIT, 0x3509, 0x10}, -+ {OV5693_8BIT, 0x350a, 0x00}, -+ {OV5693_8BIT, 0x350b, 0x40}, -+ -+ {OV5693_8BIT, 0x3601, 0x0a}, -+ {OV5693_8BIT, 0x3602, 0x38}, -+ {OV5693_8BIT, 0x3612, 0x80}, -+ {OV5693_8BIT, 0x3620, 0x54}, -+ {OV5693_8BIT, 0x3621, 0xc7}, -+ {OV5693_8BIT, 0x3622, 0x0f}, -+ {OV5693_8BIT, 0x3625, 0x10}, -+ {OV5693_8BIT, 0x3630, 0x55}, -+ {OV5693_8BIT, 0x3631, 0xf4}, -+ {OV5693_8BIT, 0x3632, 0x00}, -+ {OV5693_8BIT, 0x3633, 0x34}, -+ {OV5693_8BIT, 0x3634, 0x02}, -+ {OV5693_8BIT, 0x364d, 0x0d}, -+ {OV5693_8BIT, 0x364f, 0xdd}, -+ {OV5693_8BIT, 0x3660, 0x04}, -+ {OV5693_8BIT, 0x3662, 0x10}, -+ {OV5693_8BIT, 0x3663, 0xf1}, -+ {OV5693_8BIT, 0x3665, 0x00}, -+ {OV5693_8BIT, 0x3666, 0x20}, -+ {OV5693_8BIT, 0x3667, 0x00}, -+ {OV5693_8BIT, 0x366a, 0x80}, -+ {OV5693_8BIT, 0x3680, 0xe0}, -+ {OV5693_8BIT, 0x3681, 0x00}, -+ {OV5693_8BIT, 0x3700, 0x42}, -+ {OV5693_8BIT, 0x3701, 0x14}, -+ {OV5693_8BIT, 0x3702, 0xa0}, -+ {OV5693_8BIT, 0x3703, 0xd8}, -+ {OV5693_8BIT, 0x3704, 0x78}, -+ {OV5693_8BIT, 0x3705, 0x02}, -+ {OV5693_8BIT, 0x370a, 0x00}, -+ {OV5693_8BIT, 0x370b, 0x20}, -+ {OV5693_8BIT, 0x370c, 0x0c}, -+ {OV5693_8BIT, 0x370d, 0x11}, -+ {OV5693_8BIT, 0x370e, 0x00}, -+ {OV5693_8BIT, 0x370f, 0x40}, -+ {OV5693_8BIT, 0x3710, 0x00}, -+ {OV5693_8BIT, 0x371a, 0x1c}, -+ {OV5693_8BIT, 0x371b, 0x05}, -+ {OV5693_8BIT, 0x371c, 0x01}, -+ {OV5693_8BIT, 0x371e, 0xa1}, -+ {OV5693_8BIT, 0x371f, 0x0c}, -+ {OV5693_8BIT, 0x3721, 0x00}, -+ {OV5693_8BIT, 0x3724, 0x10}, -+ {OV5693_8BIT, 0x3726, 0x00}, -+ {OV5693_8BIT, 0x372a, 0x01}, -+ {OV5693_8BIT, 0x3730, 0x10}, -+ {OV5693_8BIT, 0x3738, 0x22}, -+ {OV5693_8BIT, 0x3739, 0xe5}, -+ {OV5693_8BIT, 0x373a, 0x50}, -+ {OV5693_8BIT, 0x373b, 0x02}, -+ {OV5693_8BIT, 0x373c, 0x41}, -+ {OV5693_8BIT, 0x373f, 0x02}, -+ {OV5693_8BIT, 0x3740, 0x42}, -+ {OV5693_8BIT, 0x3741, 0x02}, -+ {OV5693_8BIT, 0x3742, 0x18}, -+ {OV5693_8BIT, 0x3743, 0x01}, -+ {OV5693_8BIT, 0x3744, 0x02}, -+ {OV5693_8BIT, 0x3747, 0x10}, -+ {OV5693_8BIT, 0x374c, 0x04}, -+ {OV5693_8BIT, 0x3751, 0xf0}, -+ {OV5693_8BIT, 0x3752, 0x00}, -+ {OV5693_8BIT, 0x3753, 0x00}, -+ {OV5693_8BIT, 0x3754, 0xc0}, -+ {OV5693_8BIT, 0x3755, 0x00}, -+ {OV5693_8BIT, 0x3756, 0x1a}, -+ {OV5693_8BIT, 0x3758, 0x00}, -+ {OV5693_8BIT, 0x3759, 0x0f}, -+ {OV5693_8BIT, 0x376b, 0x44}, -+ {OV5693_8BIT, 0x375c, 0x04}, -+ {OV5693_8BIT, 0x3774, 0x10}, -+ {OV5693_8BIT, 0x3776, 0x00}, -+ {OV5693_8BIT, 0x377f, 0x08}, -+ {OV5693_8BIT, 0x3780, 0x22}, -+ {OV5693_8BIT, 0x3781, 0x0c}, -+ {OV5693_8BIT, 0x3784, 0x2c}, -+ {OV5693_8BIT, 0x3785, 0x1e}, -+ {OV5693_8BIT, 0x378f, 0xf5}, -+ {OV5693_8BIT, 0x3791, 0xb0}, -+ {OV5693_8BIT, 0x3795, 0x00}, -+ {OV5693_8BIT, 0x3796, 0x64}, -+ {OV5693_8BIT, 0x3797, 0x11}, -+ {OV5693_8BIT, 0x3798, 0x30}, -+ {OV5693_8BIT, 0x3799, 0x41}, -+ {OV5693_8BIT, 0x379a, 0x07}, -+ {OV5693_8BIT, 0x379b, 0xb0}, -+ {OV5693_8BIT, 0x379c, 0x0c}, -+ {OV5693_8BIT, 0x37c5, 0x00}, -+ {OV5693_8BIT, 0x37c6, 0x00}, -+ {OV5693_8BIT, 0x37c7, 0x00}, -+ {OV5693_8BIT, 0x37c9, 0x00}, -+ {OV5693_8BIT, 0x37ca, 0x00}, -+ {OV5693_8BIT, 0x37cb, 0x00}, -+ {OV5693_8BIT, 0x37de, 0x00}, -+ {OV5693_8BIT, 0x37df, 0x00}, -+ {OV5693_8BIT, 0x3800, 0x00}, -+ {OV5693_8BIT, 0x3801, 0x00}, -+ {OV5693_8BIT, 0x3802, 0x00}, -+ {OV5693_8BIT, 0x3804, 0x0a}, -+ {OV5693_8BIT, 0x3805, 0x3f}, -+ {OV5693_8BIT, 0x3810, 0x00}, -+ {OV5693_8BIT, 0x3812, 0x00}, -+ {OV5693_8BIT, 0x3823, 0x00}, -+ {OV5693_8BIT, 0x3824, 0x00}, -+ {OV5693_8BIT, 0x3825, 0x00}, -+ {OV5693_8BIT, 0x3826, 0x00}, -+ {OV5693_8BIT, 0x3827, 0x00}, -+ {OV5693_8BIT, 0x382a, 0x04}, -+ {OV5693_8BIT, 0x3a04, 0x06}, -+ {OV5693_8BIT, 0x3a05, 0x14}, -+ {OV5693_8BIT, 0x3a06, 0x00}, -+ {OV5693_8BIT, 0x3a07, 0xfe}, -+ {OV5693_8BIT, 0x3b00, 0x00}, -+ {OV5693_8BIT, 0x3b02, 0x00}, -+ {OV5693_8BIT, 0x3b03, 0x00}, -+ {OV5693_8BIT, 0x3b04, 0x00}, -+ {OV5693_8BIT, 0x3b05, 0x00}, -+ {OV5693_8BIT, 0x3e07, 0x20}, -+ {OV5693_8BIT, 0x4000, 0x08}, -+ {OV5693_8BIT, 0x4001, 0x04}, -+ {OV5693_8BIT, 0x4002, 0x45}, -+ {OV5693_8BIT, 0x4004, 0x08}, -+ {OV5693_8BIT, 0x4005, 0x18}, -+ {OV5693_8BIT, 0x4006, 0x20}, -+ {OV5693_8BIT, 0x4008, 0x24}, -+ {OV5693_8BIT, 0x4009, 0x10}, -+ {OV5693_8BIT, 0x400c, 0x00}, -+ {OV5693_8BIT, 0x400d, 0x00}, -+ {OV5693_8BIT, 0x4058, 0x00}, -+ {OV5693_8BIT, 0x404e, 0x37}, -+ {OV5693_8BIT, 0x404f, 0x8f}, -+ {OV5693_8BIT, 0x4058, 0x00}, -+ {OV5693_8BIT, 0x4101, 0xb2}, -+ {OV5693_8BIT, 0x4303, 0x00}, -+ {OV5693_8BIT, 0x4304, 0x08}, -+ {OV5693_8BIT, 0x4307, 0x30}, -+ {OV5693_8BIT, 0x4311, 0x04}, -+ {OV5693_8BIT, 0x4315, 0x01}, -+ {OV5693_8BIT, 0x4511, 0x05}, -+ {OV5693_8BIT, 0x4512, 0x01}, -+ {OV5693_8BIT, 0x4806, 0x00}, -+ {OV5693_8BIT, 0x4816, 0x52}, -+ {OV5693_8BIT, 0x481f, 0x30}, -+ {OV5693_8BIT, 0x4826, 0x2c}, -+ {OV5693_8BIT, 0x4831, 0x64}, -+ {OV5693_8BIT, 0x4d00, 0x04}, -+ {OV5693_8BIT, 0x4d01, 0x71}, -+ {OV5693_8BIT, 0x4d02, 0xfd}, -+ {OV5693_8BIT, 0x4d03, 0xf5}, -+ {OV5693_8BIT, 0x4d04, 0x0c}, -+ {OV5693_8BIT, 0x4d05, 0xcc}, -+ {OV5693_8BIT, 0x4837, 0x0a}, -+ {OV5693_8BIT, 0x5000, 0x06}, -+ {OV5693_8BIT, 0x5001, 0x01}, -+ {OV5693_8BIT, 0x5003, 0x20}, -+ {OV5693_8BIT, 0x5046, 0x0a}, -+ {OV5693_8BIT, 0x5013, 0x00}, -+ {OV5693_8BIT, 0x5046, 0x0a}, -+ {OV5693_8BIT, 0x5780, 0x1c}, -+ {OV5693_8BIT, 0x5786, 0x20}, -+ {OV5693_8BIT, 0x5787, 0x10}, -+ {OV5693_8BIT, 0x5788, 0x18}, -+ {OV5693_8BIT, 0x578a, 0x04}, -+ {OV5693_8BIT, 0x578b, 0x02}, -+ {OV5693_8BIT, 0x578c, 0x02}, -+ {OV5693_8BIT, 0x578e, 0x06}, -+ {OV5693_8BIT, 0x578f, 0x02}, -+ {OV5693_8BIT, 0x5790, 0x02}, -+ {OV5693_8BIT, 0x5791, 0xff}, -+ {OV5693_8BIT, 0x5842, 0x01}, -+ {OV5693_8BIT, 0x5843, 0x2b}, -+ {OV5693_8BIT, 0x5844, 0x01}, -+ {OV5693_8BIT, 0x5845, 0x92}, -+ {OV5693_8BIT, 0x5846, 0x01}, -+ {OV5693_8BIT, 0x5847, 0x8f}, -+ {OV5693_8BIT, 0x5848, 0x01}, -+ {OV5693_8BIT, 0x5849, 0x0c}, -+ {OV5693_8BIT, 0x5e00, 0x00}, -+ {OV5693_8BIT, 0x5e10, 0x0c}, -+ {OV5693_8BIT, 0x0100, 0x00}, -+ {OV5693_TOK_TERM, 0, 0} -+}; -+ -+/* -+ * Register settings for various resolution -+ */ -+ -+/* -+------------------------------------ -+@@ FULL QSXGA (2592x1944) 15fps 33.33ms VBlank 2lane 10Bit -+100 99 2592 1944 -+100 98 1 0 -+102 3601 bb8 ;Pather tool use only -+c8 1 f2 ; New FPGA Board -+c8 20 22 ; New FPGA Board -+c8 10 42 ; MIPI DFGA CYCY3 Board use only -+; -+c8 f 32 ; input clock to 19.2MHz -+; -+; OV5690 setting version History -+; -+; -+; V18b -+*/ -+static struct ov5693_reg const ov5693_5M_15fps[] = { -+ {OV5693_8BIT, 0x3501, 0x7b}, -+ {OV5693_8BIT, 0x3502, 0x00}, -+ {OV5693_8BIT, 0x3708, 0xe2}, -+ {OV5693_8BIT, 0x3709, 0xc3}, -+ {OV5693_8BIT, 0x3800, 0x00}, /* x_addr_start: 0 */ -+ {OV5693_8BIT, 0x3801, 0x00}, -+ {OV5693_8BIT, 0x3802, 0x00}, /* y_addr_start: 0 */ -+ {OV5693_8BIT, 0x3803, 0x00}, -+ {OV5693_8BIT, 0x3804, 0x0a}, /* x_addr_end: 2623 */ -+ {OV5693_8BIT, 0x3805, 0x3f}, -+ {OV5693_8BIT, 0x3806, 0x07}, /* y_addr_end: 1955 */ -+ {OV5693_8BIT, 0x3807, 0xa3}, -+ {OV5693_8BIT, 0x3808, 0x0a}, /* x output size: 2592 */ -+ {OV5693_8BIT, 0x3809, 0x20}, -+ {OV5693_8BIT, 0x380a, 0x07}, /* y output size: 1944 */ -+ {OV5693_8BIT, 0x380b, 0x98}, -+ {OV5693_8BIT, 0x380c, 0x0e}, /* total x output size: 3688 */ -+ {OV5693_8BIT, 0x380d, 0x68}, -+ {OV5693_8BIT, 0x380e, 0x0f}, /* total y output size: 3968 */ -+ {OV5693_8BIT, 0x380f, 0x80}, -+ {OV5693_8BIT, 0x3810, 0x00}, /* x offset: 16 */ -+ {OV5693_8BIT, 0x3811, 0x10}, -+ {OV5693_8BIT, 0x3812, 0x00}, /* y offset: 6 */ -+ {OV5693_8BIT, 0x3813, 0x06}, -+ {OV5693_8BIT, 0x3814, 0x11}, -+ {OV5693_8BIT, 0x3815, 0x11}, -+ {OV5693_8BIT, 0x3820, 0x00}, -+ {OV5693_8BIT, 0x3821, 0x1e}, -+ {OV5693_8BIT, 0x5002, 0x00}, -+ {OV5693_TOK_TERM, 0, 0} -+}; -+ -+/* -+@@ OV5693 1940x1096 30fps 8.8ms VBlanking 2lane 10Bit(Scaling) -+100 99 1940 1096 -+100 98 1 0 -+102 3601 bb8 ;Pather tool use only -+102 40 0 ; HDR Mode off -+c8 1 f2 ; New FPGA Board -+c8 20 22 ; New FPGA Board -+c8 10 42 ; MIPI DFGA CYCY3 Board use only -+; -+c8 f 32 ; input clock to 19.2MHz -+; -+; OV5690 setting version History -+; -+; -+; V18b -+*/ -+static struct ov5693_reg const ov5693_1080p_30fps[] = { -+ {OV5693_8BIT, 0x3501, 0x7b}, -+ {OV5693_8BIT, 0x3502, 0x00}, -+ {OV5693_8BIT, 0x3708, 0xe2}, -+ {OV5693_8BIT, 0x3709, 0xc3}, -+ {OV5693_8BIT, 0x3800, 0x00}, /* x_addr_start: 0 */ -+ {OV5693_8BIT, 0x3801, 0x00}, -+ {OV5693_8BIT, 0x3802, 0x00}, /* y_addr_start: 240 */ -+ {OV5693_8BIT, 0x3803, 0xf0}, -+ {OV5693_8BIT, 0x3804, 0x0a}, /* x_addr_end: 2591 */ -+ {OV5693_8BIT, 0x3805, 0x1f}, -+ {OV5693_8BIT, 0x3806, 0x06}, /* y_addr_end: 1703 */ -+ {OV5693_8BIT, 0x3807, 0xa7}, -+ {OV5693_8BIT, 0x3808, 0x07}, /* x output size: 1940 */ -+ {OV5693_8BIT, 0x3809, 0x94}, -+ {OV5693_8BIT, 0x380a, 0x04}, /* y output size: 1096 */ -+ {OV5693_8BIT, 0x380b, 0x48}, -+ {OV5693_8BIT, 0x380c, 0x0e}, /* total x output size: 3688 */ -+ {OV5693_8BIT, 0x380d, 0x68}, -+ {OV5693_8BIT, 0x380e, 0x0b}, /* total y output size: 2984 */ -+ {OV5693_8BIT, 0x380f, 0xa8}, -+ {OV5693_8BIT, 0x3810, 0x00}, /* x offset: 2 */ -+ {OV5693_8BIT, 0x3811, 0x02}, -+ {OV5693_8BIT, 0x3812, 0x00}, /* y offset: 2 */ -+ {OV5693_8BIT, 0x3813, 0x02}, -+ {OV5693_8BIT, 0x3814, 0x11}, -+ {OV5693_8BIT, 0x3815, 0x11}, -+ {OV5693_8BIT, 0x3820, 0x00}, -+ {OV5693_8BIT, 0x3821, 0x1e}, -+ {OV5693_8BIT, 0x5002, 0x80}, -+ {OV5693_TOK_TERM, 0, 0} -+}; -+ -+/* -+ * 1296x736 30fps 8.8ms VBlanking 2lane 10Bit (Scaling) -+ */ -+static struct ov5693_reg const ov5693_720p_30fps[] = { -+ {OV5693_8BIT, 0x3501, 0x3d}, -+ {OV5693_8BIT, 0x3502, 0x00}, -+ {OV5693_8BIT, 0x3708, 0xe6}, -+ {OV5693_8BIT, 0x3709, 0xc7}, -+ {OV5693_8BIT, 0x3800, 0x00}, /* x_addr_start: 0 */ -+ {OV5693_8BIT, 0x3801, 0x00}, -+ {OV5693_8BIT, 0x3802, 0x00}, /* y_addr_start: 0 */ -+ {OV5693_8BIT, 0x3803, 0x00}, -+ {OV5693_8BIT, 0x3804, 0x0a}, /* x_addr_end: 2623 */ -+ {OV5693_8BIT, 0x3805, 0x3f}, -+ {OV5693_8BIT, 0x3806, 0x07}, /* y_addr_end: 1955 */ -+ {OV5693_8BIT, 0x3807, 0xa3}, -+ {OV5693_8BIT, 0x3808, 0x05}, /* x output size: 1296 */ -+ {OV5693_8BIT, 0x3809, 0x10}, -+ {OV5693_8BIT, 0x380a, 0x02}, /* y output size: 736 */ -+ {OV5693_8BIT, 0x380b, 0xe0}, -+ {OV5693_8BIT, 0x380c, 0x0a}, /* total x output size: 2688 */ -+ {OV5693_8BIT, 0x380d, 0x80}, -+ {OV5693_8BIT, 0x380e, 0x07}, /* total y output size: 1984 */ -+ {OV5693_8BIT, 0x380f, 0xc0}, -+ {OV5693_8BIT, 0x3810, 0x00}, /* x offset: 8 */ -+ {OV5693_8BIT, 0x3811, 0x08}, -+ {OV5693_8BIT, 0x3812, 0x00}, /* y offset: 2 */ -+ {OV5693_8BIT, 0x3813, 0x02}, -+ {OV5693_8BIT, 0x3814, 0x31}, -+ {OV5693_8BIT, 0x3815, 0x31}, -+ {OV5693_8BIT, 0x3820, 0x04}, -+ {OV5693_8BIT, 0x3821, 0x1f}, -+ {OV5693_8BIT, 0x5002, 0x80}, -+ {OV5693_TOK_TERM, 0, 0} -+}; -+ -+/* -+ * 736x496 30fps 8.8ms VBlanking 2lane 10Bit (Scaling) -+ */ -+static struct ov5693_reg const ov5693_480p_30fps[] = { -+ {OV5693_8BIT, 0x3501, 0x3d}, -+ {OV5693_8BIT, 0x3502, 0x00}, -+ {OV5693_8BIT, 0x3708, 0xe6}, -+ {OV5693_8BIT, 0x3709, 0xc7}, -+ {OV5693_8BIT, 0x3800, 0x00}, /* x_addr_start: 0 */ -+ {OV5693_8BIT, 0x3801, 0x00}, -+ {OV5693_8BIT, 0x3802, 0x00}, /* y_addr_start: 7 */ -+ {OV5693_8BIT, 0x3803, 0x07}, -+ {OV5693_8BIT, 0x3804, 0x0a}, /* x_addr_end: 2623 */ -+ {OV5693_8BIT, 0x3805, 0x3f}, -+ {OV5693_8BIT, 0x3806, 0x07}, /* y_addr_end: 1955 */ -+ {OV5693_8BIT, 0x3807, 0xa3}, -+ {OV5693_8BIT, 0x3808, 0x02}, /* x output size: 736 */ -+ {OV5693_8BIT, 0x3809, 0xe0}, -+ {OV5693_8BIT, 0x380a, 0x01}, /* y output size: 496 */ -+ {OV5693_8BIT, 0x380b, 0xf0}, -+ {OV5693_8BIT, 0x380c, 0x0a}, /* total x output size: 2688 */ -+ {OV5693_8BIT, 0x380d, 0x80}, -+ {OV5693_8BIT, 0x380e, 0x07}, /* total y output size: 1984 */ -+ {OV5693_8BIT, 0x380f, 0xc0}, -+ {OV5693_8BIT, 0x3810, 0x00}, /* x offset: 8 */ -+ {OV5693_8BIT, 0x3811, 0x08}, -+ {OV5693_8BIT, 0x3812, 0x00}, /* y offset: 2 */ -+ {OV5693_8BIT, 0x3813, 0x02}, -+ {OV5693_8BIT, 0x3814, 0x31}, -+ {OV5693_8BIT, 0x3815, 0x31}, -+ {OV5693_8BIT, 0x3820, 0x04}, -+ {OV5693_8BIT, 0x3821, 0x1f}, -+ {OV5693_8BIT, 0x5002, 0x80}, -+ {OV5693_TOK_TERM, 0, 0} -+}; -+ -+/* -+@@ OV5693 656x496 30fps 17ms VBlanking 2lane 10Bit(Scaling) -+100 99 656 496 -+100 98 1 0 -+102 3601 BB8 ;Pather tool use only -+c8 1 f2 ; New FPGA Board -+c8 20 22 ; New FPGA Board -+; OV5690 setting version History -+; -+c8 f 32 ; input clock to 19.2MHz -+; -+; V18b -+*/ -+static struct ov5693_reg const ov5693_VGA_30fps[] = { -+ {OV5693_8BIT, 0x3501, 0x3d}, -+ {OV5693_8BIT, 0x3502, 0x00}, -+ {OV5693_8BIT, 0x3708, 0xe6}, -+ {OV5693_8BIT, 0x3709, 0xc7}, -+ {OV5693_8BIT, 0x3800, 0x00}, /* x_addr_start: 0 */ -+ {OV5693_8BIT, 0x3801, 0x00}, -+ {OV5693_8BIT, 0x3802, 0x00}, /* y_addr_start: 0 */ -+ {OV5693_8BIT, 0x3803, 0x00}, -+ {OV5693_8BIT, 0x3804, 0x0a}, /* x_addr_end: 2623 */ -+ {OV5693_8BIT, 0x3805, 0x3f}, -+ {OV5693_8BIT, 0x3806, 0x07}, /* y_addr_end: 1955 */ -+ {OV5693_8BIT, 0x3807, 0xa3}, -+ {OV5693_8BIT, 0x3808, 0x02}, /* x output size: 656 */ -+ {OV5693_8BIT, 0x3809, 0x90}, -+ {OV5693_8BIT, 0x380a, 0x01}, /* y output size: 496 */ -+ {OV5693_8BIT, 0x380b, 0xf0}, -+ {OV5693_8BIT, 0x380c, 0x0a}, /* total x output size: 2688 */ -+ {OV5693_8BIT, 0x380d, 0x80}, -+ {OV5693_8BIT, 0x380e, 0x07}, /* total y output size: 1984 */ -+ {OV5693_8BIT, 0x380f, 0xc0}, -+ {OV5693_8BIT, 0x3810, 0x00}, /* x offset: 13 */ -+ {OV5693_8BIT, 0x3811, 0x0d}, -+ {OV5693_8BIT, 0x3812, 0x00}, /* y offset: 3 */ -+ {OV5693_8BIT, 0x3813, 0x03}, -+ {OV5693_8BIT, 0x3814, 0x31}, -+ {OV5693_8BIT, 0x3815, 0x31}, -+ {OV5693_8BIT, 0x3820, 0x04}, -+ {OV5693_8BIT, 0x3821, 0x1f}, -+ {OV5693_8BIT, 0x5002, 0x80}, -+ {OV5693_TOK_TERM, 0, 0} -+}; -+ -+struct ov5693_resolution ov5693_res_preview[] = { -+ { -+ .desc = "ov5693_VGA_30fps", -+ .width = 656, -+ .height = 496, -+ .fps = 30, -+ .pix_clk_freq = 81, -+ .used = 0, -+ .pixels_per_line = 2688, -+ .lines_per_frame = 1984, -+ .bin_factor_x = 2, -+ .bin_factor_y = 2, -+ .bin_mode = 0, -+ .skip_frames = 3, -+ .regs = ov5693_VGA_30fps, -+ }, -+ { -+ .desc = "ov5693_1080P_30fps", -+ .width = 1940, -+ .height = 1096, -+ .fps = 30, -+ .pix_clk_freq = 81, -+ .used = 0, -+ .pixels_per_line = 3688, -+ .lines_per_frame = 2984, -+ .bin_factor_x = 0, -+ .bin_factor_y = 0, -+ .bin_mode = 0, -+ .skip_frames = 3, -+ .regs = ov5693_1080p_30fps, -+ }, -+ { -+ .desc = "ov5693_5M_15fps", -+ .width = 2592, -+ .height = 1944, -+ .fps = 15, -+ .pix_clk_freq = 81, -+ .used = 0, -+ .pixels_per_line = 3688, -+ .lines_per_frame = 3968, -+ .bin_factor_x = 0, -+ .bin_factor_y = 0, -+ .bin_mode = 0, -+ .skip_frames = 3, -+ .regs = ov5693_5M_15fps, -+ }, -+}; -+#define N_RES_PREVIEW (ARRAY_SIZE(ov5693_res_preview)) -+ -+struct ov5693_resolution ov5693_res_still[] = { -+ { -+ .desc = "ov5693_VGA_30fps", -+ .width = 656, -+ .height = 496, -+ .fps = 30, -+ .pix_clk_freq = 81, -+ .used = 0, -+ .pixels_per_line = 2688, -+ .lines_per_frame = 1984, -+ .bin_factor_x = 2, -+ .bin_factor_y = 2, -+ .bin_mode = 0, -+ .skip_frames = 3, -+ .regs = ov5693_VGA_30fps, -+ }, -+ { -+ .desc = "ov5693_1080P_30fps", -+ .width = 1940, -+ .height = 1096, -+ .fps = 30, -+ .pix_clk_freq = 81, -+ .used = 0, -+ .pixels_per_line = 3688, -+ .lines_per_frame = 2984, -+ .bin_factor_x = 0, -+ .bin_factor_y = 0, -+ .bin_mode = 0, -+ .skip_frames = 3, -+ .regs = ov5693_1080p_30fps, -+ }, -+ { -+ .desc = "ov5693_5M_15fps", -+ .width = 2592, -+ .height = 1944, -+ .fps = 15, -+ .pix_clk_freq = 81, -+ .used = 0, -+ .pixels_per_line = 3688, -+ .lines_per_frame = 3968, -+ .bin_factor_x = 0, -+ .bin_factor_y = 0, -+ .bin_mode = 0, -+ .skip_frames = 3, -+ .regs = ov5693_5M_15fps, -+ }, -+}; -+#define N_RES_STILL (ARRAY_SIZE(ov5693_res_still)) -+ -+struct ov5693_resolution ov5693_res_video[] = { -+ { -+ .desc = "ov5693_VGA_30fps", -+ .width = 656, -+ .height = 496, -+ .fps = 30, -+ .pix_clk_freq = 81, -+ .used = 0, -+ .pixels_per_line = 2688, -+ .lines_per_frame = 1984, -+ .bin_factor_x = 2, -+ .bin_factor_y = 2, -+ .bin_mode = 0, -+ .skip_frames = 3, -+ .regs = ov5693_VGA_30fps, -+ }, -+ { -+ .desc = "ov5693_480P_30fps", -+ .width = 736, -+ .height = 496, -+ .fps = 30, -+ .pix_clk_freq = 81, -+ .used = 0, -+ .pixels_per_line = 2688, -+ .lines_per_frame = 1984, -+ .bin_factor_x = 2, -+ .bin_factor_y = 2, -+ .bin_mode = 0, -+ .skip_frames = 1, -+ .regs = ov5693_480p_30fps, -+ }, -+ { -+ .desc = "ov5693_720p_30fps", -+ .width = 1296, -+ .height = 736, -+ .fps = 30, -+ .pix_clk_freq = 81, -+ .used = 0, -+ .pixels_per_line = 2688, -+ .lines_per_frame = 1984, -+ .bin_factor_x = 2, -+ .bin_factor_y = 2, -+ .bin_mode = 0, -+ .skip_frames = 1, -+ .regs = ov5693_720p_30fps, -+ }, -+ { -+ .desc = "ov5693_1080P_30fps", -+ .width = 1940, -+ .height = 1096, -+ .fps = 30, -+ .pix_clk_freq = 81, -+ .used = 0, -+ .pixels_per_line = 3688, -+ .lines_per_frame = 2984, -+ .bin_factor_x = 0, -+ .bin_factor_y = 0, -+ .bin_mode = 0, -+ .skip_frames = 3, -+ .regs = ov5693_1080p_30fps, -+ }, -+}; -+#define N_RES_VIDEO (ARRAY_SIZE(ov5693_res_video)) -+ -+struct ov5693_vcm ov5693_vcm_ops = { -+ .power_up = ad5823_vcm_power_up, -+ .power_down = ad5823_vcm_power_down, -+ .init = ad5823_vcm_init, -+ .t_focus_vcm = ad5823_t_focus_vcm, -+ .t_focus_abs = ad5823_t_focus_abs, -+ .t_focus_rel = ad5823_t_focus_rel, -+ .q_focus_status = ad5823_q_focus_status, -+ .q_focus_abs = ad5823_q_focus_abs, -+}; -+#endif --- -2.23.0 - diff --git a/patches/5.2/0007-sdcard-reader.patch b/patches/5.2/0007-sdcard-reader.patch deleted file mode 100644 index c28aaa865..000000000 --- a/patches/5.2/0007-sdcard-reader.patch +++ /dev/null @@ -1,26 +0,0 @@ -From 2317a00a6f49dd9d0904515fe8867212c9cd187e Mon Sep 17 00:00:00 2001 -From: Maximilian Luz -Date: Fri, 26 Jul 2019 04:45:55 +0200 -Subject: [PATCH 07/12] sdcard-reader - ---- - drivers/usb/core/hub.c | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c -index 2844366dc173..989fabd6ab39 100644 ---- a/drivers/usb/core/hub.c -+++ b/drivers/usb/core/hub.c -@@ -4205,7 +4205,8 @@ void usb_enable_lpm(struct usb_device *udev) - if (!udev || !udev->parent || - udev->speed < USB_SPEED_SUPER || - !udev->lpm_capable || -- udev->state < USB_STATE_DEFAULT) -+ udev->state < USB_STATE_DEFAULT || -+ !udev->bos || !udev->bos->ss_cap) - return; - - udev->lpm_disable_count--; --- -2.23.0 - diff --git a/patches/5.2/0010-mwlwifi.patch b/patches/5.2/0010-mwlwifi.patch deleted file mode 100644 index cb80a4e24..000000000 --- a/patches/5.2/0010-mwlwifi.patch +++ /dev/null @@ -1,19755 +0,0 @@ -From 4697bc13237488ab48dc293bb2724ca8b7179301 Mon Sep 17 00:00:00 2001 -From: Maximilian Luz -Date: Fri, 26 Jul 2019 04:47:02 +0200 -Subject: [PATCH 10/12] mwlwifi - ---- - drivers/net/wireless/marvell/Kconfig | 1 + - drivers/net/wireless/marvell/Makefile | 1 + - drivers/net/wireless/marvell/mwlwifi/Kconfig | 23 + - drivers/net/wireless/marvell/mwlwifi/Makefile | 19 + - .../wireless/marvell/mwlwifi/Makefile.module | 28 + - .../net/wireless/marvell/mwlwifi/README.md | 142 + - drivers/net/wireless/marvell/mwlwifi/core.c | 1086 +++++ - drivers/net/wireless/marvell/mwlwifi/core.h | 517 +++ - .../net/wireless/marvell/mwlwifi/debugfs.c | 2201 ++++++++++ - .../net/wireless/marvell/mwlwifi/debugfs.h | 24 + - .../net/wireless/marvell/mwlwifi/hif/fwcmd.c | 3852 +++++++++++++++++ - .../net/wireless/marvell/mwlwifi/hif/fwcmd.h | 285 ++ - .../wireless/marvell/mwlwifi/hif/hif-ops.h | 297 ++ - .../net/wireless/marvell/mwlwifi/hif/hif.h | 81 + - .../wireless/marvell/mwlwifi/hif/hostcmd.h | 1285 ++++++ - .../wireless/marvell/mwlwifi/hif/pcie/dev.h | 1032 +++++ - .../wireless/marvell/mwlwifi/hif/pcie/fwdl.c | 274 ++ - .../wireless/marvell/mwlwifi/hif/pcie/fwdl.h | 24 + - .../wireless/marvell/mwlwifi/hif/pcie/pcie.c | 1645 +++++++ - .../wireless/marvell/mwlwifi/hif/pcie/rx.c | 540 +++ - .../wireless/marvell/mwlwifi/hif/pcie/rx.h | 25 + - .../marvell/mwlwifi/hif/pcie/rx_ndp.c | 612 +++ - .../marvell/mwlwifi/hif/pcie/rx_ndp.h | 26 + - .../marvell/mwlwifi/hif/pcie/sc4_ddr.h | 965 +++++ - .../wireless/marvell/mwlwifi/hif/pcie/tx.c | 1396 ++++++ - .../wireless/marvell/mwlwifi/hif/pcie/tx.h | 38 + - .../marvell/mwlwifi/hif/pcie/tx_ndp.c | 693 +++ - .../marvell/mwlwifi/hif/pcie/tx_ndp.h | 30 + - ...-workaround-for-80+80-and-160-MHz-channels | 32 + - .../wireless/marvell/mwlwifi/hostapd/README | 26 + - .../net/wireless/marvell/mwlwifi/mac80211.c | 933 ++++ - .../net/wireless/marvell/mwlwifi/mu_mimo.c | 21 + - .../net/wireless/marvell/mwlwifi/mu_mimo.h | 23 + - .../net/wireless/marvell/mwlwifi/sysadpt.h | 86 + - .../net/wireless/marvell/mwlwifi/thermal.c | 182 + - .../net/wireless/marvell/mwlwifi/thermal.h | 42 + - drivers/net/wireless/marvell/mwlwifi/utils.c | 576 +++ - drivers/net/wireless/marvell/mwlwifi/utils.h | 158 + - .../net/wireless/marvell/mwlwifi/vendor_cmd.c | 136 + - .../net/wireless/marvell/mwlwifi/vendor_cmd.h | 60 + - 40 files changed, 19417 insertions(+) - create mode 100644 drivers/net/wireless/marvell/mwlwifi/Kconfig - create mode 100644 drivers/net/wireless/marvell/mwlwifi/Makefile - create mode 100644 drivers/net/wireless/marvell/mwlwifi/Makefile.module - create mode 100644 drivers/net/wireless/marvell/mwlwifi/README.md - create mode 100644 drivers/net/wireless/marvell/mwlwifi/core.c - create mode 100644 drivers/net/wireless/marvell/mwlwifi/core.h - create mode 100644 drivers/net/wireless/marvell/mwlwifi/debugfs.c - create mode 100644 drivers/net/wireless/marvell/mwlwifi/debugfs.h - create mode 100644 drivers/net/wireless/marvell/mwlwifi/hif/fwcmd.c - create mode 100644 drivers/net/wireless/marvell/mwlwifi/hif/fwcmd.h - create mode 100644 drivers/net/wireless/marvell/mwlwifi/hif/hif-ops.h - create mode 100644 drivers/net/wireless/marvell/mwlwifi/hif/hif.h - create mode 100644 drivers/net/wireless/marvell/mwlwifi/hif/hostcmd.h - create mode 100644 drivers/net/wireless/marvell/mwlwifi/hif/pcie/dev.h - create mode 100644 drivers/net/wireless/marvell/mwlwifi/hif/pcie/fwdl.c - create mode 100644 drivers/net/wireless/marvell/mwlwifi/hif/pcie/fwdl.h - create mode 100644 drivers/net/wireless/marvell/mwlwifi/hif/pcie/pcie.c - create mode 100644 drivers/net/wireless/marvell/mwlwifi/hif/pcie/rx.c - create mode 100644 drivers/net/wireless/marvell/mwlwifi/hif/pcie/rx.h - create mode 100644 drivers/net/wireless/marvell/mwlwifi/hif/pcie/rx_ndp.c - create mode 100644 drivers/net/wireless/marvell/mwlwifi/hif/pcie/rx_ndp.h - create mode 100644 drivers/net/wireless/marvell/mwlwifi/hif/pcie/sc4_ddr.h - create mode 100644 drivers/net/wireless/marvell/mwlwifi/hif/pcie/tx.c - create mode 100644 drivers/net/wireless/marvell/mwlwifi/hif/pcie/tx.h - create mode 100644 drivers/net/wireless/marvell/mwlwifi/hif/pcie/tx_ndp.c - create mode 100644 drivers/net/wireless/marvell/mwlwifi/hif/pcie/tx_ndp.h - create mode 100644 drivers/net/wireless/marvell/mwlwifi/hostapd/700-interoperability-workaround-for-80+80-and-160-MHz-channels - create mode 100644 drivers/net/wireless/marvell/mwlwifi/hostapd/README - create mode 100644 drivers/net/wireless/marvell/mwlwifi/mac80211.c - create mode 100644 drivers/net/wireless/marvell/mwlwifi/mu_mimo.c - create mode 100644 drivers/net/wireless/marvell/mwlwifi/mu_mimo.h - create mode 100644 drivers/net/wireless/marvell/mwlwifi/sysadpt.h - create mode 100644 drivers/net/wireless/marvell/mwlwifi/thermal.c - create mode 100644 drivers/net/wireless/marvell/mwlwifi/thermal.h - create mode 100644 drivers/net/wireless/marvell/mwlwifi/utils.c - create mode 100644 drivers/net/wireless/marvell/mwlwifi/utils.h - create mode 100644 drivers/net/wireless/marvell/mwlwifi/vendor_cmd.c - create mode 100644 drivers/net/wireless/marvell/mwlwifi/vendor_cmd.h - -diff --git a/drivers/net/wireless/marvell/Kconfig b/drivers/net/wireless/marvell/Kconfig -index dff82fdbea78..c0790dbd52c0 100644 ---- a/drivers/net/wireless/marvell/Kconfig -+++ b/drivers/net/wireless/marvell/Kconfig -@@ -15,6 +15,7 @@ if WLAN_VENDOR_MARVELL - source "drivers/net/wireless/marvell/libertas/Kconfig" - source "drivers/net/wireless/marvell/libertas_tf/Kconfig" - source "drivers/net/wireless/marvell/mwifiex/Kconfig" -+source "drivers/net/wireless/marvell/mwlwifi/Kconfig" - - config MWL8K - tristate "Marvell 88W8xxx PCI/PCIe Wireless support" -diff --git a/drivers/net/wireless/marvell/Makefile b/drivers/net/wireless/marvell/Makefile -index 25f6d5d2fa0c..00fccce28cdd 100644 ---- a/drivers/net/wireless/marvell/Makefile -+++ b/drivers/net/wireless/marvell/Makefile -@@ -3,5 +3,6 @@ obj-$(CONFIG_LIBERTAS) += libertas/ - - obj-$(CONFIG_LIBERTAS_THINFIRM) += libertas_tf/ - obj-$(CONFIG_MWIFIEX) += mwifiex/ -+obj-$(CONFIG_MWLWIFI) += mwlwifi/ - - obj-$(CONFIG_MWL8K) += mwl8k.o -diff --git a/drivers/net/wireless/marvell/mwlwifi/Kconfig b/drivers/net/wireless/marvell/mwlwifi/Kconfig -new file mode 100644 -index 000000000000..a9bcb9cd4100 ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/Kconfig -@@ -0,0 +1,23 @@ -+config MWLWIFI -+ tristate "Marvell Avastar 88W8864/88W8897 PCIe driver (mac80211 compatible)" -+ depends on PCI && MAC80211 -+ select FW_LOADER -+ ---help--- -+ Select to build the driver supporting the: -+ -+ Marvell Wireless Wi-Fi 88W8864 modules -+ Marvell Wireless Wi-Fi 88W8897 modules -+ -+ This driver uses the kernel's mac80211 subsystem. -+ -+ If you want to compile the driver as a module (= code which can be -+ inserted in and removed from the running kernel whenever you want), -+ say M here and read . The -+ module will be called mwlwifi. -+ -+ NOTE: Selecting this driver may cause conflict with MWIFIEX driver -+ that also operates on the same part number 88W8897. Users should -+ select either MWIFIEX or MWLWIFI, not both. MWIFIEX is fullmac, -+ supporting more comprehensive client functions for laptops/embedded -+ devices. MWLWIFI is mac80211-based for full AP/Wireless Bridge. -+ -diff --git a/drivers/net/wireless/marvell/mwlwifi/Makefile b/drivers/net/wireless/marvell/mwlwifi/Makefile -new file mode 100644 -index 000000000000..061833703c7f ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/Makefile -@@ -0,0 +1,19 @@ -+obj-$(CONFIG_MWLWIFI) += mwlwifi.o -+ -+mwlwifi-objs += core.o -+mwlwifi-objs += mac80211.o -+mwlwifi-objs += mu_mimo.o -+mwlwifi-objs += vendor_cmd.o -+mwlwifi-objs += utils.o -+mwlwifi-$(CONFIG_THERMAL) += thermal.o -+mwlwifi-$(CONFIG_DEBUG_FS) += debugfs.o -+mwlwifi-objs += hif/fwcmd.o -+mwlwifi-objs += hif/pcie/pcie.o -+mwlwifi-objs += hif/pcie/fwdl.o -+mwlwifi-objs += hif/pcie/tx.o -+mwlwifi-objs += hif/pcie/rx.o -+mwlwifi-objs += hif/pcie/tx_ndp.o -+mwlwifi-objs += hif/pcie/rx_ndp.o -+ -+ccflags-y += -I$(src) -+ccflags-y += -D__CHECK_ENDIAN__ -diff --git a/drivers/net/wireless/marvell/mwlwifi/Makefile.module b/drivers/net/wireless/marvell/mwlwifi/Makefile.module -new file mode 100644 -index 000000000000..d11a1b88cab6 ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/Makefile.module -@@ -0,0 +1,28 @@ -+obj-m += mwlwifi.o -+ -+mwlwifi-objs += core.o -+mwlwifi-objs += mac80211.o -+mwlwifi-objs += mu_mimo.o -+mwlwifi-objs += vendor_cmd.o -+mwlwifi-objs += utils.o -+mwlwifi-$(CONFIG_THERMAL) += thermal.o -+mwlwifi-$(CONFIG_DEBUG_FS) += debugfs.o -+mwlwifi-objs += hif/fwcmd.o -+mwlwifi-objs += hif/pcie/pcie.o -+mwlwifi-objs += hif/pcie/fwdl.o -+mwlwifi-objs += hif/pcie/tx.o -+mwlwifi-objs += hif/pcie/rx.o -+mwlwifi-objs += hif/pcie/tx_ndp.o -+mwlwifi-objs += hif/pcie/rx_ndp.o -+ -+ccflags-y += -I$(src) -+ccflags-y += -O2 -funroll-loops -D__CHECK_ENDIAN__ -+ -+all: -+ $(MAKE) -C $(KDIR) M=$(PWD) -+ -+clean: -+ rm -f *.a *.s *.ko *.ko.cmd *.mod.* .mwlwifi.* modules.order Module.symvers -+ rm -rf .tmp_versions -+ find . -name ".*.o.cmd" -exec rm -f {} \; -+ find . -name "*.o" -exec rm -f {} \; -diff --git a/drivers/net/wireless/marvell/mwlwifi/README.md b/drivers/net/wireless/marvell/mwlwifi/README.md -new file mode 100644 -index 000000000000..788c5d4dc80d ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/README.md -@@ -0,0 +1,142 @@ -+# mwlwifi -+mac80211 driver for the Marvell 88W8x64 802.11ac chip -+ -+## Building mwlwifi With OpenWrt/LEDE -+1. Modify `package/kernel/mwlwifi/Makefile`: -+ ``` -+ PKG_VERSION:=10.3.0.17-20160601 -+ PKG_SOURCE_VERSION:=4bb95ba1aeccce506a95499b49b9b844ecfae8a1 -+ ``` -+ -+2. Rename `package/kernel/mwlwifi/patches` to `package/kernel/mwlwifi/patches.tmp`. -+3. Run the following commands: -+ ```sh -+ make package/kernel/mwlwifi/clean -+ make V=s (-jx) -+ ``` -+ -+### Special Considerations -+* After driver 10.3.0.17-20160603, [MAX-MPDU-7991] should be removed from vht_capab command of hostapd. -+ -+* Hostpad must include the following commit for 160 MHz operation: -+ ``` -+ commit 03a72eacda5d9a1837a74387081596a0d5466ec1 -+ Author: Jouni Malinen -+ Date: Thu Dec 17 18:39:19 2015 +0200 -+ -+ VHT: Add an interoperability workaround for 80+80 and 160 MHz channels -+ -+ Number of deployed 80 MHz capable VHT stations that do not support 80+80 -+ and 160 MHz bandwidths seem to misbehave when trying to connect to an AP -+ that advertises 80+80 or 160 MHz channel bandwidth in the VHT Operation -+ element. To avoid such issues with deployed devices, modify the design -+ based on newly proposed IEEE 802.11 standard changes. -+ -+ This allows poorly implemented VHT 80 MHz stations to connect with the -+ AP in 80 MHz mode. 80+80 and 160 MHz capable stations need to support -+ the new workaround mechanism to allow full bandwidth to be used. -+ However, there are more or less no impacted station with 80+80/160 -+ capability deployed. -+ -+ Signed-off-by: Jouni Malinen jouni@qca.qualcomm.com -+ -+ Note: After hostapd package 2016-06-15, this commit is already included. -+ ``` -+ -+* In order to let STA mode to support 160 MHz operation, mac80211 package should be 2016-10-08 or later. -+ -+* WiFi device does not use HT rates when using TKIP as the encryption cipher. If you want to have good performance, please use AES only. -+ -+* DTS parameters for mwlwifi driver (pcie@X,0): -+ ```sh -+ #Disable 2g band -+ marvell,2ghz = <0>; -+ -+ #Disable 5g band -+ marvell,5ghz = <0>; -+ -+ #Specify antenna number, default is 4x4. For WRT1200AC, you must set these values to 2x2. -+ marvell,chainmask = <4 4>; -+ -+ #Specify external power table. If your device needs external power table, you must provide the power table via this parameter, otherwise the Tx power will be pretty low. -+ marvell,powertable -+ ``` -+ -+ To see if your device needs/accepts an external power table or not, run the following: -+ ```sh -+ cat /sys/kernel/debug/ieee80211/phy0/mwlwifi/info -+ ``` -+ -+ You should see a line in the results which looks like the following: -+ ```sh -+ power table loaded from dts: no -+ ``` -+ -+ If it is "no", it does not allow you to load external power table (for newer devices due to FCC regulations). If it is "yes", you must provide power table in DTS file (for older devices). -+ -+* Changing interrupt to different CPU cores: -+ ```sh -+ #Use CPU0: -+ echo 1 > /proc/irq/irq number of phy0 or phy1/smp_affinity -+ -+ #Use CPU1: -+ echo 2 > /proc/irq/irq number of phy0 or phy1/smp_affinity -+ ``` -+ -+* Note for DFS of WRT3200ACM (88W8964): -+ -+ All WRT3200ACM devices are programmed with device power table. Mwlwifi driver will base on region code to set country code for your device and it will not allow you to change country code. There are another wifi (phy2) on WRT3200ACM which is not mwlwifi. It will allow you to change country code. Under this case, country code setting will be conflicted and it will let DFS can't work. -+ -+ There are two ways to resolve this problem: -+ * Please don't change country code and let mwlwifi set it for you. -+ * Remove phy2. Under this case, even though you change country code, mwlwifi will reject it. Because phy2 is not existed, country code setting won't be conflicted. To do this, run the following commands (for OpenWrt/LEDE): -+ -+ ```sh -+ opkg remove kmod-mwifiex-sdio -+ opkg remove mwifiex-sdio-firmware -+ reboot -+ ``` -+ -+ The better way is let mwlwifi set country code for you. -+ -+## Replacing mwlwifi on a Current OpenWrt/LEDE Build -+ -+1. Establish a symbolic link to your working mwlwifi directory with current mwlwifi package name under directory "dl": -+ ```sh -+ ls -l mwlwifi* -+ ``` -+ -+ You should see something like the following: -+ ```sh -+ lrwxrwxrwx 1 dlin dlin 48 mwlwifi-10.3.2.0-20170110 -> /home/dlin/home2/projects/github/mwlwifi -+ -+ -rw-r--r-- 1 dlin dlin 4175136 mwlwifi-10.3.2.0-20170110.tar.xz -+ ``` -+ -+2. Back up original mwlwifi package and tar your working mwlwifi to replace original mwlwifi package: -+ -+ ```sh -+ tar Jcvf mwlwifi-10.3.2.0-20170110.tar.xz mwlwifi-10.3.2.0-20170110/. -+ ``` -+ -+3. You can use `make V=s` to build the whole image or `make V=s package/kernel/mwlwifi/compile` to build mwlwifi package. The generated whole image or mwlwifi package can be found under directory "bin". -+ -+Due to package version being the same as previous one, you need to add option `--force-reinstall` when you use `opkg` to update mwlwifi package on your device. -+ -+## Monitor interface for debug -+ -+1. Create moinitor interface mon0: -+ ```sh -+ iw wlan0/wlan1 interface add mon0 type monitor -+ ifconfig mon0 up -+ ``` -+ -+2. Use tcpdump to dump dhcp packets: -+ ```sh -+ tcpdump -vvvi mon0 -n port 67 and port 68 -+ ``` -+ -+3. Use tcpdump to dump icmp packets: -+ ```sh -+ tcpdump -vvvi mon0 icmp -+ ``` -diff --git a/drivers/net/wireless/marvell/mwlwifi/core.c b/drivers/net/wireless/marvell/mwlwifi/core.c -new file mode 100644 -index 000000000000..9d2b5511607e ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/core.c -@@ -0,0 +1,1086 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file implements core layer related functions. */ -+ -+#include -+ -+#include "sysadpt.h" -+#include "core.h" -+#include "vendor_cmd.h" -+#include "thermal.h" -+#include "debugfs.h" -+#include "hif/fwcmd.h" -+#include "hif/hif-ops.h" -+ -+#define CMD_BUF_SIZE 0x4000 -+#define INVALID_WATCHDOG 0xAA -+ -+static const struct ieee80211_channel mwl_channels_24[] = { -+ { .band = NL80211_BAND_2GHZ, .center_freq = 2412, .hw_value = 1, }, -+ { .band = NL80211_BAND_2GHZ, .center_freq = 2417, .hw_value = 2, }, -+ { .band = NL80211_BAND_2GHZ, .center_freq = 2422, .hw_value = 3, }, -+ { .band = NL80211_BAND_2GHZ, .center_freq = 2427, .hw_value = 4, }, -+ { .band = NL80211_BAND_2GHZ, .center_freq = 2432, .hw_value = 5, }, -+ { .band = NL80211_BAND_2GHZ, .center_freq = 2437, .hw_value = 6, }, -+ { .band = NL80211_BAND_2GHZ, .center_freq = 2442, .hw_value = 7, }, -+ { .band = NL80211_BAND_2GHZ, .center_freq = 2447, .hw_value = 8, }, -+ { .band = NL80211_BAND_2GHZ, .center_freq = 2452, .hw_value = 9, }, -+ { .band = NL80211_BAND_2GHZ, .center_freq = 2457, .hw_value = 10, }, -+ { .band = NL80211_BAND_2GHZ, .center_freq = 2462, .hw_value = 11, }, -+ { .band = NL80211_BAND_2GHZ, .center_freq = 2467, .hw_value = 12, }, -+ { .band = NL80211_BAND_2GHZ, .center_freq = 2472, .hw_value = 13, }, -+ { .band = NL80211_BAND_2GHZ, .center_freq = 2484, .hw_value = 14, }, -+}; -+ -+static const struct ieee80211_rate mwl_rates_24[] = { -+ { .bitrate = 10, .hw_value = 2, }, -+ { .bitrate = 20, .hw_value = 4, }, -+ { .bitrate = 55, .hw_value = 11, }, -+ { .bitrate = 110, .hw_value = 22, }, -+ { .bitrate = 220, .hw_value = 44, }, -+ { .bitrate = 60, .hw_value = 12, }, -+ { .bitrate = 90, .hw_value = 18, }, -+ { .bitrate = 120, .hw_value = 24, }, -+ { .bitrate = 180, .hw_value = 36, }, -+ { .bitrate = 240, .hw_value = 48, }, -+ { .bitrate = 360, .hw_value = 72, }, -+ { .bitrate = 480, .hw_value = 96, }, -+ { .bitrate = 540, .hw_value = 108, }, -+}; -+ -+static const struct ieee80211_channel mwl_channels_50[] = { -+ { .band = NL80211_BAND_5GHZ, .center_freq = 5180, .hw_value = 36, }, -+ { .band = NL80211_BAND_5GHZ, .center_freq = 5200, .hw_value = 40, }, -+ { .band = NL80211_BAND_5GHZ, .center_freq = 5220, .hw_value = 44, }, -+ { .band = NL80211_BAND_5GHZ, .center_freq = 5240, .hw_value = 48, }, -+ { .band = NL80211_BAND_5GHZ, .center_freq = 5260, .hw_value = 52, }, -+ { .band = NL80211_BAND_5GHZ, .center_freq = 5280, .hw_value = 56, }, -+ { .band = NL80211_BAND_5GHZ, .center_freq = 5300, .hw_value = 60, }, -+ { .band = NL80211_BAND_5GHZ, .center_freq = 5320, .hw_value = 64, }, -+ { .band = NL80211_BAND_5GHZ, .center_freq = 5500, .hw_value = 100, }, -+ { .band = NL80211_BAND_5GHZ, .center_freq = 5520, .hw_value = 104, }, -+ { .band = NL80211_BAND_5GHZ, .center_freq = 5540, .hw_value = 108, }, -+ { .band = NL80211_BAND_5GHZ, .center_freq = 5560, .hw_value = 112, }, -+ { .band = NL80211_BAND_5GHZ, .center_freq = 5580, .hw_value = 116, }, -+ { .band = NL80211_BAND_5GHZ, .center_freq = 5600, .hw_value = 120, }, -+ { .band = NL80211_BAND_5GHZ, .center_freq = 5620, .hw_value = 124, }, -+ { .band = NL80211_BAND_5GHZ, .center_freq = 5640, .hw_value = 128, }, -+ { .band = NL80211_BAND_5GHZ, .center_freq = 5660, .hw_value = 132, }, -+ { .band = NL80211_BAND_5GHZ, .center_freq = 5680, .hw_value = 136, }, -+ { .band = NL80211_BAND_5GHZ, .center_freq = 5700, .hw_value = 140, }, -+ { .band = NL80211_BAND_5GHZ, .center_freq = 5720, .hw_value = 144, }, -+ { .band = NL80211_BAND_5GHZ, .center_freq = 5745, .hw_value = 149, }, -+ { .band = NL80211_BAND_5GHZ, .center_freq = 5765, .hw_value = 153, }, -+ { .band = NL80211_BAND_5GHZ, .center_freq = 5785, .hw_value = 157, }, -+ { .band = NL80211_BAND_5GHZ, .center_freq = 5805, .hw_value = 161, }, -+}; -+ -+static const struct ieee80211_rate mwl_rates_50[] = { -+ { .bitrate = 60, .hw_value = 12, }, -+ { .bitrate = 90, .hw_value = 18, }, -+ { .bitrate = 120, .hw_value = 24, }, -+ { .bitrate = 180, .hw_value = 36, }, -+ { .bitrate = 240, .hw_value = 48, }, -+ { .bitrate = 360, .hw_value = 72, }, -+ { .bitrate = 480, .hw_value = 96, }, -+ { .bitrate = 540, .hw_value = 108, }, -+}; -+ -+static const struct ieee80211_iface_limit ap_if_limits[] = { -+ { .max = SYSADPT_NUM_OF_AP, .types = BIT(NL80211_IFTYPE_AP) }, -+#if defined(CPTCFG_MAC80211_MESH) || defined(CONFIG_MAC80211_MESH) -+ { .max = SYSADPT_NUM_OF_MESH, .types = BIT(NL80211_IFTYPE_MESH_POINT) }, -+#endif -+ { .max = SYSADPT_NUM_OF_CLIENT, .types = BIT(NL80211_IFTYPE_STATION) }, -+}; -+ -+static const struct ieee80211_iface_combination ap_if_comb = { -+ .limits = ap_if_limits, -+ .n_limits = ARRAY_SIZE(ap_if_limits), -+ .max_interfaces = SYSADPT_NUM_OF_AP, -+ .num_different_channels = 1, -+ .radar_detect_widths = BIT(NL80211_CHAN_WIDTH_20_NOHT) | -+ BIT(NL80211_CHAN_WIDTH_20) | -+ BIT(NL80211_CHAN_WIDTH_40) | -+ BIT(NL80211_CHAN_WIDTH_80) | -+ BIT(NL80211_CHAN_WIDTH_160), -+}; -+ -+struct region_code_mapping { -+ const char *alpha2; -+ u32 region_code; -+}; -+ -+static const struct region_code_mapping regmap[] = { -+ {"US", 0x10}, /* US FCC */ -+ {"CA", 0x20}, /* Canada */ -+ {"FR", 0x30}, /* France */ -+ {"ES", 0x31}, /* Spain */ -+ {"FR", 0x32}, /* France */ -+ {"JP", 0x40}, /* Japan */ -+ {"TW", 0x80}, /* Taiwan */ -+ {"AU", 0x81}, /* Australia */ -+ {"CN", 0x90}, /* China (Asia) */ -+}; -+ -+static int mwl_prepare_cmd_buf(struct mwl_priv *priv) -+{ -+ priv->pcmd_buf = -+ (unsigned short *)dmam_alloc_coherent(priv->dev, -+ CMD_BUF_SIZE, -+ &priv->pphys_cmd_buf, -+ GFP_KERNEL); -+ if (!priv->pcmd_buf) { -+ wiphy_err(priv->hw->wiphy, -+ "cannot alloc memory for command buffer\n"); -+ goto err; -+ } -+ wiphy_debug(priv->hw->wiphy, -+ "priv->pcmd_buf = %p priv->pphys_cmd_buf = %p\n", -+ priv->pcmd_buf, -+ (void *)priv->pphys_cmd_buf); -+ memset(priv->pcmd_buf, 0x00, CMD_BUF_SIZE); -+ -+ return 0; -+ -+err: -+ wiphy_err(priv->hw->wiphy, "command buffer alloc fail\n"); -+ -+ return -EIO; -+} -+ -+static int mwl_init_firmware(struct mwl_priv *priv, const char *fw_name, -+ const char *cal_name, const char *txpwrlmt_name) -+{ -+ int rc = 0; -+ -+ rc = request_firmware((const struct firmware **)&priv->fw_ucode, -+ fw_name, priv->dev); -+ -+ if (rc) { -+ wiphy_err(priv->hw->wiphy, -+ "cannot find firmware image <%s>\n", fw_name); -+ goto err_load_fw; -+ } -+ -+ rc = mwl_hif_download_firmware(priv->hw); -+ if (rc) { -+ wiphy_err(priv->hw->wiphy, -+ "cannot download firmware image <%s>\n", fw_name); -+ goto err_download_fw; -+ } -+ -+ if (cal_name) { -+ if ((request_firmware((const struct firmware **)&priv->cal_data, -+ cal_name, priv->dev)) < 0) -+ wiphy_debug(priv->hw->wiphy, -+ "cannot find calibtration data\n"); -+ } -+ -+ if (txpwrlmt_name) { -+ if ((request_firmware( -+ (const struct firmware **)&priv->txpwrlmt_file, -+ txpwrlmt_name, priv->dev)) < 0) -+ wiphy_debug(priv->hw->wiphy, -+ "cannot find tx power limit data\n"); -+ } -+ -+ return rc; -+ -+err_download_fw: -+ -+ release_firmware(priv->fw_ucode); -+ -+err_load_fw: -+ -+ wiphy_err(priv->hw->wiphy, "firmware init fail\n"); -+ -+ return rc; -+} -+ -+static void mwl_process_of_dts(struct mwl_priv *priv) -+{ -+#ifdef CONFIG_OF -+ struct property *prop; -+ u32 prop_value; -+ -+ priv->dt_node = -+ of_find_node_by_name(mwl_hif_device_node(priv->hw), -+ "mwlwifi"); -+ if (!priv->dt_node) -+ return; -+ -+ /* look for all matching property names */ -+ for_each_property_of_node(priv->dt_node, prop) { -+ if (strcmp(prop->name, "marvell,2ghz") == 0) -+ priv->disable_2g = true; -+ if (strcmp(prop->name, "marvell,5ghz") == 0) -+ priv->disable_5g = true; -+ if (strcmp(prop->name, "marvell,chainmask") == 0) { -+ prop_value = be32_to_cpu(*((__be32 *)prop->value)); -+ if (prop_value == 2) -+ priv->antenna_tx = ANTENNA_TX_2; -+ else if (prop_value == 3) -+ priv->antenna_tx = ANTENNA_TX_3; -+ -+ prop_value = be32_to_cpu(*((__be32 *) -+ (prop->value + 4))); -+ if (prop_value == 2) -+ priv->antenna_rx = ANTENNA_RX_2; -+ else if (prop_value == 3) -+ priv->antenna_rx = ANTENNA_RX_3; -+ } -+ } -+ -+ priv->pwr_node = of_find_node_by_name(priv->dt_node, -+ "marvell,powertable"); -+#endif -+} -+ -+static void mwl_reg_notifier(struct wiphy *wiphy, -+ struct regulatory_request *request) -+{ -+ struct ieee80211_hw *hw; -+ struct mwl_priv *priv; -+#ifdef CONFIG_OF -+ struct property *prop; -+ struct property *fcc_prop = NULL; -+ struct property *etsi_prop = NULL; -+ struct property *specific_prop = NULL; -+ u32 prop_value; -+ int i, j, k; -+#endif -+ -+ hw = wiphy_to_ieee80211_hw(wiphy); -+ priv = hw->priv; -+ -+ if (priv->forbidden_setting) { -+ if (!priv->regulatory_set) { -+ regulatory_hint(wiphy, priv->fw_alpha2); -+ priv->regulatory_set = true; -+ } else { -+ if (memcmp(priv->fw_alpha2, request->alpha2, 2)) -+ regulatory_hint(wiphy, priv->fw_alpha2); -+ } -+ return; -+ } -+ -+ priv->dfs_region = request->dfs_region; -+ -+#ifdef CONFIG_OF -+ if ((priv->chip_type != MWL8997) && (priv->pwr_node)) { -+ for_each_property_of_node(priv->pwr_node, prop) { -+ if (strcmp(prop->name, "FCC") == 0) -+ fcc_prop = prop; -+ if (strcmp(prop->name, "ETSI") == 0) -+ etsi_prop = prop; -+ if ((prop->name[0] == request->alpha2[0]) && -+ (prop->name[1] == request->alpha2[1])) -+ specific_prop = prop; -+ } -+ -+ prop = NULL; -+ -+ if (specific_prop) { -+ prop = specific_prop; -+ } else { -+ if (priv->dfs_region == NL80211_DFS_ETSI) -+ prop = etsi_prop; -+ else -+ prop = fcc_prop; -+ } -+ -+ if (prop) { -+ /* Reset the whole table */ -+ for (i = 0; i < SYSADPT_MAX_NUM_CHANNELS; i++) -+ memset(&priv->tx_pwr_tbl[i], 0, -+ sizeof(struct mwl_tx_pwr_tbl)); -+ -+ /* Load related power table */ -+ i = 0; -+ j = 0; -+ while (i < prop->length) { -+ prop_value = -+ be32_to_cpu(*(__be32 *) -+ (prop->value + i)); -+ priv->tx_pwr_tbl[j].channel = prop_value; -+ i += 4; -+ prop_value = -+ be32_to_cpu(*(__be32 *) -+ (prop->value + i)); -+ priv->tx_pwr_tbl[j].setcap = prop_value; -+ i += 4; -+ for (k = 0; k < SYSADPT_TX_POWER_LEVEL_TOTAL; -+ k++) { -+ prop_value = -+ be32_to_cpu(*(__be32 *) -+ (prop->value + i)); -+ priv->tx_pwr_tbl[j].tx_power[k] = -+ prop_value; -+ i += 4; -+ } -+ prop_value = -+ be32_to_cpu(*(__be32 *) -+ (prop->value + i)); -+ priv->tx_pwr_tbl[j].cdd = -+ (prop_value == 0) ? false : true; -+ i += 4; -+ prop_value = -+ be32_to_cpu(*(__be32 *) -+ (prop->value + i)); -+ priv->tx_pwr_tbl[j].txantenna2 = prop_value; -+ i += 4; -+ j++; -+ } -+ -+ /* Dump loaded power tabel */ -+ wiphy_debug(hw->wiphy, "regdomain: %s\n", prop->name); -+ for (i = 0; i < SYSADPT_MAX_NUM_CHANNELS; i++) { -+ struct mwl_tx_pwr_tbl *pwr_tbl; -+ char disp_buf[64]; -+ char *disp_ptr; -+ -+ pwr_tbl = &priv->tx_pwr_tbl[i]; -+ if (pwr_tbl->channel == 0) -+ break; -+ wiphy_debug(hw->wiphy, -+ "Channel: %d: 0x%x 0x%x 0x%x\n", -+ pwr_tbl->channel, -+ pwr_tbl->setcap, -+ pwr_tbl->cdd, -+ pwr_tbl->txantenna2); -+ disp_ptr = disp_buf; -+ for (j = 0; j < SYSADPT_TX_POWER_LEVEL_TOTAL; -+ j++) { -+ disp_ptr += -+ sprintf(disp_ptr, "%x ", -+ pwr_tbl->tx_power[j]); -+ } -+ wiphy_debug(hw->wiphy, "%s\n", disp_buf); -+ } -+ } -+ } -+#endif -+} -+ -+static void mwl_regd_init(struct mwl_priv *priv) -+{ -+ u8 region_code; -+ int rc; -+ int i; -+ -+ /* hook regulatory domain change notification */ -+ priv->hw->wiphy->reg_notifier = mwl_reg_notifier; -+ -+ if (priv->chip_type == MWL8964) -+ rc = mwl_fwcmd_get_pwr_tbl_sc4(priv->hw, -+ &priv->device_pwr_tbl[0], -+ ®ion_code, -+ &priv->number_of_channels, -+ 0); -+ else -+ rc = mwl_fwcmd_get_device_pwr_tbl(priv->hw, -+ &priv->device_pwr_tbl[0], -+ ®ion_code, -+ &priv->number_of_channels, -+ 0); -+ if (rc) -+ return; -+ -+ priv->forbidden_setting = true; -+ -+ for (i = 1; i < priv->number_of_channels; i++) { -+ if (priv->chip_type == MWL8964) -+ mwl_fwcmd_get_pwr_tbl_sc4(priv->hw, -+ &priv->device_pwr_tbl[i], -+ ®ion_code, -+ &priv->number_of_channels, -+ i); -+ else -+ mwl_fwcmd_get_device_pwr_tbl(priv->hw, -+ &priv->device_pwr_tbl[i], -+ ®ion_code, -+ &priv->number_of_channels, -+ i); -+ } -+ -+ for (i = 0; i < ARRAY_SIZE(regmap); i++) -+ if (regmap[i].region_code == priv->fw_region_code) { -+ memcpy(priv->fw_alpha2, regmap[i].alpha2, 2); -+ break; -+ } -+} -+ -+static void mwl_set_ht_caps(struct mwl_priv *priv, -+ struct ieee80211_supported_band *band) -+{ -+ struct ieee80211_hw *hw; -+ const u8 ant_rx_no[ANTENNA_RX_MAX] = { 3, 1, 2, 3}; -+ int i; -+ -+ hw = priv->hw; -+ -+ band->ht_cap.ht_supported = 1; -+ if (priv->chip_type == MWL8964) -+ band->ht_cap.cap |= IEEE80211_HT_CAP_MAX_AMSDU; -+ band->ht_cap.cap |= IEEE80211_HT_CAP_LDPC_CODING; -+ band->ht_cap.cap |= IEEE80211_HT_CAP_SUP_WIDTH_20_40; -+ band->ht_cap.cap |= IEEE80211_HT_CAP_SM_PS; -+ band->ht_cap.cap |= IEEE80211_HT_CAP_SGI_20; -+ band->ht_cap.cap |= IEEE80211_HT_CAP_SGI_40; -+ band->ht_cap.cap |= IEEE80211_HT_CAP_DSSSCCK40; -+ -+ if ((priv->chip_type == MWL8997) && -+ (priv->antenna_tx != ANTENNA_TX_1)) { -+ band->ht_cap.cap |= IEEE80211_HT_CAP_TX_STBC; -+ band->ht_cap.cap |= (1 << IEEE80211_HT_CAP_RX_STBC_SHIFT); -+ } -+ -+ ieee80211_hw_set(hw, AMPDU_AGGREGATION); -+ ieee80211_hw_set(hw, SUPPORTS_AMSDU_IN_AMPDU); -+ band->ht_cap.ampdu_factor = IEEE80211_HT_MAX_AMPDU_64K; -+ band->ht_cap.ampdu_density = IEEE80211_HT_MPDU_DENSITY_4; -+ -+ for (i = 0; i < ant_rx_no[priv->antenna_rx]; i++) -+ band->ht_cap.mcs.rx_mask[i] = 0xff; -+ band->ht_cap.mcs.rx_mask[4] = 0x01; -+ -+ band->ht_cap.mcs.tx_params = IEEE80211_HT_MCS_TX_DEFINED; -+} -+ -+static void mwl_set_vht_caps(struct mwl_priv *priv, -+ struct ieee80211_supported_band *band) -+{ -+ u32 antenna_num = 4; -+ -+ band->vht_cap.vht_supported = 1; -+ -+ if (priv->chip_type == MWL8964) { -+ band->vht_cap.cap |= IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454; -+ band->vht_cap.cap |= IEEE80211_VHT_CAP_SHORT_GI_160; -+ band->vht_cap.cap |= IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160MHZ; -+ } else -+ band->vht_cap.cap |= IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_3895; -+ band->vht_cap.cap |= IEEE80211_VHT_CAP_RXLDPC; -+ band->vht_cap.cap |= IEEE80211_VHT_CAP_SHORT_GI_80; -+ band->vht_cap.cap |= IEEE80211_VHT_CAP_RXSTBC_1; -+ if (priv->antenna_tx != ANTENNA_TX_1) { -+ band->vht_cap.cap |= IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE; -+ if (priv->chip_type == MWL8964) -+ band->vht_cap.cap |= -+ IEEE80211_VHT_CAP_MU_BEAMFORMER_CAPABLE; -+ } -+ band->vht_cap.cap |= IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE; -+ if (priv->chip_type == MWL8964) -+ band->vht_cap.cap |= IEEE80211_VHT_CAP_MU_BEAMFORMEE_CAPABLE; -+ band->vht_cap.cap |= IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK; -+ band->vht_cap.cap |= IEEE80211_VHT_CAP_RX_ANTENNA_PATTERN; -+ band->vht_cap.cap |= IEEE80211_VHT_CAP_TX_ANTENNA_PATTERN; -+ if (priv->chip_type == MWL8997) { -+ if (priv->antenna_tx != ANTENNA_TX_1) -+ band->vht_cap.cap |= IEEE80211_VHT_CAP_TXSTBC; -+ } -+ -+ if (priv->antenna_rx == ANTENNA_RX_1) -+ band->vht_cap.vht_mcs.rx_mcs_map = cpu_to_le16(0xfffe); -+ else if (priv->antenna_rx == ANTENNA_RX_2) -+ band->vht_cap.vht_mcs.rx_mcs_map = cpu_to_le16(0xfffa); -+ else -+ band->vht_cap.vht_mcs.rx_mcs_map = cpu_to_le16(0xffea); -+ -+ if (priv->antenna_tx == ANTENNA_TX_1) { -+ band->vht_cap.vht_mcs.tx_mcs_map = cpu_to_le16(0xfffe); -+ antenna_num = 1; -+ } else if (priv->antenna_tx == ANTENNA_TX_2) { -+ band->vht_cap.vht_mcs.tx_mcs_map = cpu_to_le16(0xfffa); -+ antenna_num = 2; -+ } else -+ band->vht_cap.vht_mcs.tx_mcs_map = cpu_to_le16(0xffea); -+ -+ if (band->vht_cap.cap & (IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE | -+ IEEE80211_VHT_CAP_MU_BEAMFORMEE_CAPABLE)) { -+ band->vht_cap.cap |= -+ ((antenna_num - 1) << -+ IEEE80211_VHT_CAP_BEAMFORMEE_STS_SHIFT) & -+ IEEE80211_VHT_CAP_BEAMFORMEE_STS_MASK; -+ } -+ -+ if (band->vht_cap.cap & (IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE | -+ IEEE80211_VHT_CAP_MU_BEAMFORMER_CAPABLE)) { -+ band->vht_cap.cap |= -+ ((antenna_num - 1) << -+ IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_SHIFT) & -+ IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_MASK; -+ } -+} -+ -+static void mwl_set_caps(struct mwl_priv *priv) -+{ -+ struct ieee80211_hw *hw; -+ -+ hw = priv->hw; -+ -+ /* set up band information for 2.4G */ -+ if (!priv->disable_2g) { -+ BUILD_BUG_ON(sizeof(priv->channels_24) != -+ sizeof(mwl_channels_24)); -+ memcpy(priv->channels_24, mwl_channels_24, -+ sizeof(mwl_channels_24)); -+ -+ BUILD_BUG_ON(sizeof(priv->rates_24) != sizeof(mwl_rates_24)); -+ memcpy(priv->rates_24, mwl_rates_24, sizeof(mwl_rates_24)); -+ -+ priv->band_24.band = NL80211_BAND_2GHZ; -+ priv->band_24.channels = priv->channels_24; -+ priv->band_24.n_channels = ARRAY_SIZE(mwl_channels_24); -+ priv->band_24.bitrates = priv->rates_24; -+ priv->band_24.n_bitrates = ARRAY_SIZE(mwl_rates_24); -+ -+ mwl_set_ht_caps(priv, &priv->band_24); -+ mwl_set_vht_caps(priv, &priv->band_24); -+ -+ hw->wiphy->bands[NL80211_BAND_2GHZ] = &priv->band_24; -+ } -+ -+ /* set up band information for 5G */ -+ if (!priv->disable_5g) { -+ BUILD_BUG_ON(sizeof(priv->channels_50) != -+ sizeof(mwl_channels_50)); -+ memcpy(priv->channels_50, mwl_channels_50, -+ sizeof(mwl_channels_50)); -+ -+ BUILD_BUG_ON(sizeof(priv->rates_50) != sizeof(mwl_rates_50)); -+ memcpy(priv->rates_50, mwl_rates_50, sizeof(mwl_rates_50)); -+ -+ priv->band_50.band = NL80211_BAND_5GHZ; -+ priv->band_50.channels = priv->channels_50; -+ priv->band_50.n_channels = ARRAY_SIZE(mwl_channels_50); -+ priv->band_50.bitrates = priv->rates_50; -+ priv->band_50.n_bitrates = ARRAY_SIZE(mwl_rates_50); -+ -+ mwl_set_ht_caps(priv, &priv->band_50); -+ mwl_set_vht_caps(priv, &priv->band_50); -+ -+ hw->wiphy->bands[NL80211_BAND_5GHZ] = &priv->band_50; -+ } -+} -+ -+static void mwl_heartbeat_handle(struct work_struct *work) -+{ -+ struct mwl_priv *priv = -+ container_of(work, struct mwl_priv, heartbeat_handle); -+ u32 val; -+ -+ mwl_fwcmd_get_addr_value(priv->hw, 0, 1, &val, 0); -+ priv->heartbeating = false; -+} -+ -+static void mwl_watchdog_ba_events(struct work_struct *work) -+{ -+ int rc; -+ u8 bitmap = 0, stream_index; -+ struct mwl_ampdu_stream *streams; -+ struct mwl_priv *priv = -+ container_of(work, struct mwl_priv, watchdog_ba_handle); -+ -+ rc = mwl_fwcmd_get_watchdog_bitmap(priv->hw, &bitmap); -+ -+ if (rc) -+ return; -+ -+ spin_lock_bh(&priv->stream_lock); -+ -+ /* the bitmap is the hw queue number. Map it to the ampdu queue. */ -+ if (bitmap != INVALID_WATCHDOG) { -+ if (bitmap == priv->ampdu_num) -+ stream_index = 0; -+ else if (bitmap > priv->ampdu_num) -+ stream_index = bitmap - priv->ampdu_num; -+ else -+ stream_index = bitmap + 3; /** queue 0 is stream 3*/ -+ -+ if (bitmap != 0xFF) { -+ /* Check if the stream is in use before disabling it */ -+ streams = &priv->ampdu[stream_index]; -+ -+ if (streams->state == AMPDU_STREAM_ACTIVE) -+ ieee80211_stop_tx_ba_session(streams->sta, -+ streams->tid); -+ } else { -+ for (stream_index = 0; -+ stream_index < priv->ampdu_num; -+ stream_index++) { -+ streams = &priv->ampdu[stream_index]; -+ -+ if (streams->state != AMPDU_STREAM_ACTIVE) -+ continue; -+ -+ ieee80211_stop_tx_ba_session(streams->sta, -+ streams->tid); -+ } -+ } -+ } -+ -+ spin_unlock_bh(&priv->stream_lock); -+} -+ -+static void mwl_account_handle(struct work_struct *work) -+{ -+ struct mwl_priv *priv = -+ container_of(work, struct mwl_priv, account_handle); -+ -+ mwl_hif_process_account(priv->hw); -+} -+ -+static void mwl_wds_check_handle(struct work_struct *work) -+{ -+ struct mwl_priv *priv = -+ container_of(work, struct mwl_priv, wds_check_handle); -+ struct mwl_sta *sta_info; -+ struct ieee80211_sta *sta; -+ bool wds_sta = false; -+ -+ spin_lock_bh(&priv->sta_lock); -+ list_for_each_entry(sta_info, &priv->sta_list, list) { -+ if (sta_info->wds) -+ continue; -+ sta = container_of((void *)sta_info, struct ieee80211_sta, -+ drv_priv); -+ if (ether_addr_equal(sta->addr, priv->wds_check_sta)) { -+ wds_sta = true; -+ break; -+ } -+ } -+ spin_unlock_bh(&priv->sta_lock); -+ -+ if (wds_sta) { -+ mwl_fwcmd_set_new_stn_wds_sc4(priv->hw, sta->addr); -+ sta_info->wds = true; -+ } -+ -+ priv->wds_check = false; -+} -+ -+static void mwl_chnl_switch_event(struct work_struct *work) -+{ -+ struct mwl_priv *priv = -+ container_of(work, struct mwl_priv, chnl_switch_handle); -+ struct mwl_vif *mwl_vif; -+ struct ieee80211_vif *vif; -+ -+ if (!priv->csa_active) { -+ wiphy_err(priv->hw->wiphy, -+ "csa is not active (got channel switch event)\n"); -+ return; -+ } -+ -+ spin_lock_bh(&priv->vif_lock); -+ list_for_each_entry(mwl_vif, &priv->vif_list, list) { -+ vif = container_of((void *)mwl_vif, struct ieee80211_vif, -+ drv_priv); -+ -+ if (vif->csa_active) -+ ieee80211_csa_finish(vif); -+ } -+ spin_unlock_bh(&priv->vif_lock); -+ -+ wiphy_info(priv->hw->wiphy, "channel switch is done\n"); -+ -+ priv->csa_active = false; -+} -+ -+static irqreturn_t mwl_isr(int irq, void *dev_id) -+{ -+ struct ieee80211_hw *hw = dev_id; -+ -+ return mwl_hif_irq_handler(hw); -+} -+ -+#ifdef timer_setup -+static void timer_routine(struct timer_list *t) -+{ -+ struct mwl_priv *priv = from_timer(priv, t, period_timer); -+ struct ieee80211_hw *hw = priv->hw; -+#else -+static void timer_routine(unsigned long data) -+{ -+ struct ieee80211_hw *hw = (struct ieee80211_hw *)data; -+ struct mwl_priv *priv = hw->priv; -+#endif -+ if (priv->heartbeat) { -+ if ((jiffies - priv->pre_jiffies) >= -+ msecs_to_jiffies(priv->heartbeat * 1000)) { -+ if (!priv->heartbeating) { -+ priv->heartbeating = true; -+ ieee80211_queue_work(hw, -+ &priv->heartbeat_handle); -+ } -+ priv->pre_jiffies = jiffies; -+ } -+ } -+ -+ mwl_hif_timer_routine(hw); -+ -+ mod_timer(&priv->period_timer, jiffies + -+ msecs_to_jiffies(SYSADPT_TIMER_WAKEUP_TIME)); -+} -+ -+static int mwl_wl_init(struct mwl_priv *priv) -+{ -+ struct ieee80211_hw *hw = priv->hw; -+ int rc; -+ u16 addr_num; -+ struct mac_address *mac_addr; -+ u8 last_nibble; -+ -+ hw->extra_tx_headroom = mwl_hif_get_tx_head_room(hw); -+ hw->queues = SYSADPT_TX_WMM_QUEUES; -+ -+ /* Set rssi values to dBm */ -+ ieee80211_hw_set(hw, SIGNAL_DBM); -+ ieee80211_hw_set(hw, HAS_RATE_CONTROL); -+ -+ /* Ask mac80211 not to trigger PS mode -+ * based on PM bit of incoming frames. -+ */ -+ ieee80211_hw_set(hw, AP_LINK_PS); -+ -+ ieee80211_hw_set(hw, SUPPORTS_PER_STA_GTK); -+ ieee80211_hw_set(hw, MFP_CAPABLE); -+ -+ hw->wiphy->flags |= WIPHY_FLAG_IBSS_RSN; -+ hw->wiphy->flags |= WIPHY_FLAG_HAS_CHANNEL_SWITCH; -+ hw->wiphy->flags |= WIPHY_FLAG_SUPPORTS_TDLS; -+ hw->wiphy->flags |= WIPHY_FLAG_AP_UAPSD; -+ -+ hw->vif_data_size = sizeof(struct mwl_vif); -+ hw->sta_data_size = sizeof(struct mwl_sta); -+ -+ priv->ap_macids_supported = 0x0000ffff; -+ priv->sta_macids_supported = 0x00010000; -+ priv->macids_used = 0; -+ INIT_LIST_HEAD(&priv->vif_list); -+ INIT_LIST_HEAD(&priv->sta_list); -+ -+ /* Set default radio state, preamble and wmm */ -+ priv->noise = -104; -+ priv->radio_on = false; -+ priv->radio_short_preamble = false; -+ priv->wmm_enabled = false; -+ priv->powinited = 0; -+ priv->wds_check = false; -+ if (priv->chip_type == MWL8997) -+ priv->pwr_level = SYSADPT_TX_GRP_PWR_LEVEL_TOTAL; -+ else -+ priv->pwr_level = SYSADPT_TX_POWER_LEVEL_TOTAL; -+ priv->dfs_test = false; -+ priv->csa_active = false; -+ priv->dfs_chirp_count_min = 5; -+ priv->dfs_chirp_time_interval = 1000; -+ priv->dfs_pw_filter = 0; -+ priv->dfs_min_num_radar = 5; -+ priv->dfs_min_pri_count = 4; -+ priv->bf_type = TXBF_MODE_AUTO; -+ -+ /* Handle watchdog ba events */ -+ INIT_WORK(&priv->heartbeat_handle, mwl_heartbeat_handle); -+ INIT_WORK(&priv->watchdog_ba_handle, mwl_watchdog_ba_events); -+ INIT_WORK(&priv->account_handle, mwl_account_handle); -+ INIT_WORK(&priv->wds_check_handle, mwl_wds_check_handle); -+ INIT_WORK(&priv->chnl_switch_handle, mwl_chnl_switch_event); -+ -+ mutex_init(&priv->fwcmd_mutex); -+ spin_lock_init(&priv->vif_lock); -+ spin_lock_init(&priv->sta_lock); -+ spin_lock_init(&priv->stream_lock); -+ spin_lock_init(&priv->stnid_lock); -+ -+ rc = mwl_thermal_register(priv); -+ if (rc) { -+ wiphy_err(hw->wiphy, "fail to register thermal framework\n"); -+ goto err_thermal_register; -+ } -+ -+ rc = mwl_hif_init(hw); -+ if (rc) { -+ wiphy_err(hw->wiphy, "fail to initialize host interface\n"); -+ goto err_hif_init; -+ } -+ -+ SET_IEEE80211_PERM_ADDR(hw, priv->hw_data.mac_addr); -+ -+ if (priv->chip_type == MWL8964) { -+ addr_num = SYSADPT_NUM_OF_AP + SYSADPT_NUM_OF_CLIENT; -+ hw->wiphy->n_addresses = addr_num; -+ hw->wiphy->addresses = -+ kzalloc(addr_num * sizeof(*mac_addr), GFP_KERNEL); -+ -+ mac_addr = &hw->wiphy->addresses[0]; -+ ether_addr_copy(mac_addr->addr, priv->hw_data.mac_addr); -+ last_nibble = mac_addr->addr[5] & 0x0F; -+ for (addr_num = 0; addr_num < SYSADPT_NUM_OF_AP; addr_num++) { -+ mac_addr = &hw->wiphy->addresses[addr_num + 1]; -+ ether_addr_copy(mac_addr->addr, priv->hw_data.mac_addr); -+ if (!strcmp(wiphy_name(hw->wiphy), "phy0")) { -+ last_nibble++; -+ if (last_nibble == 0x10) -+ last_nibble = 0; -+ } else { -+ last_nibble--; -+ if (last_nibble == 0xFF) -+ last_nibble = 0x0F; -+ } -+ mac_addr->addr[5] = -+ (mac_addr->addr[5] & 0xF0) | last_nibble; -+ mac_addr->addr[0] |= 0x2; -+ } -+ } -+ -+ wiphy_info(hw->wiphy, -+ "firmware version: 0x%x\n", priv->hw_data.fw_release_num); -+ -+ if (priv->chip_type == MWL8997) { -+ mwl_fwcmd_set_cfg_data(hw, 2); -+ mwl_fwcmd_set_txpwrlmt_cfg_data(hw); -+ mwl_fwcmd_get_txpwrlmt_cfg_data(hw); -+ } -+ -+ if (priv->chip_type == MWL8964) -+ rc = mwl_fwcmd_get_fw_region_code_sc4(hw, -+ &priv->fw_region_code); -+ else -+ rc = mwl_fwcmd_get_fw_region_code(hw, &priv->fw_region_code); -+ if (!rc) { -+ priv->fw_device_pwrtbl = true; -+ mwl_regd_init(priv); -+ wiphy_info(hw->wiphy, -+ "firmware region code: %x\n", priv->fw_region_code); -+ } -+ -+ if (priv->chip_type == MWL8997) -+ mwl_fwcmd_dump_otp_data(hw); -+ -+ mwl_fwcmd_radio_disable(hw); -+ mwl_fwcmd_rf_antenna(hw, WL_ANTENNATYPE_TX, priv->antenna_tx); -+ mwl_fwcmd_rf_antenna(hw, WL_ANTENNATYPE_RX, priv->antenna_rx); -+ -+ hw->wiphy->interface_modes = 0; -+ hw->wiphy->interface_modes |= BIT(NL80211_IFTYPE_AP); -+#if defined(CPTCFG_MAC80211_MESH) || defined(CONFIG_MAC80211_MESH) -+ hw->wiphy->interface_modes |= BIT(NL80211_IFTYPE_MESH_POINT); -+#endif -+ hw->wiphy->interface_modes |= BIT(NL80211_IFTYPE_STATION); -+ hw->wiphy->iface_combinations = &ap_if_comb; -+ hw->wiphy->n_iface_combinations = 1; -+ -+ mwl_set_caps(priv); -+ -+ priv->led_blink_enable = 1; -+ priv->led_blink_rate = LED_BLINK_RATE_MID; -+ mwl_fwcmd_led_ctrl(hw, priv->led_blink_enable, priv->led_blink_rate); -+ -+ vendor_cmd_register(hw->wiphy); -+ -+ rc = ieee80211_register_hw(hw); -+ if (rc) { -+ wiphy_err(hw->wiphy, "fail to register device\n"); -+ goto err_register_hw; -+ } -+ -+ priv->irq = mwl_hif_get_irq_num(hw); -+ rc = request_irq(priv->irq, mwl_isr, IRQF_SHARED, -+ mwl_hif_get_driver_name(hw), hw); -+ if (rc) { -+ priv->irq = -1; -+ wiphy_err(hw->wiphy, "fail to register IRQ handler\n"); -+ goto err_register_irq; -+ } -+#ifdef timer_setup -+ timer_setup(&priv->period_timer, timer_routine, 0); -+#else -+ setup_timer(&priv->period_timer, timer_routine, (unsigned long)hw); -+#endif -+ mod_timer(&priv->period_timer, jiffies + -+ msecs_to_jiffies(SYSADPT_TIMER_WAKEUP_TIME)); -+ -+ return rc; -+ -+err_register_hw: -+err_register_irq: -+ mwl_hif_deinit(hw); -+ -+err_hif_init: -+err_thermal_register: -+ -+ wiphy_err(hw->wiphy, "init fail\n"); -+ -+ return rc; -+} -+ -+static void mwl_wl_deinit(struct mwl_priv *priv) -+{ -+ struct ieee80211_hw *hw = priv->hw; -+ -+ del_timer_sync(&priv->period_timer); -+ -+ if (priv->irq != -1) { -+ free_irq(priv->irq, hw); -+ priv->irq = -1; -+ } -+ -+ if (priv->chip_type == MWL8964) -+ kfree(hw->wiphy->addresses); -+ ieee80211_unregister_hw(hw); -+ mwl_thermal_unregister(priv); -+ cancel_work_sync(&priv->chnl_switch_handle); -+ cancel_work_sync(&priv->account_handle); -+ cancel_work_sync(&priv->wds_check_handle); -+ cancel_work_sync(&priv->watchdog_ba_handle); -+ cancel_work_sync(&priv->heartbeat_handle); -+ mwl_hif_deinit(hw); -+} -+ -+struct ieee80211_hw *mwl_alloc_hw(int bus_type, -+ int chip_type, -+ struct device *dev, -+ const struct mwl_hif_ops *ops, -+ size_t hif_data_len) -+{ -+ struct ieee80211_hw *hw; -+ struct mwl_priv *priv; -+ int priv_size; -+ -+ priv_size = ALIGN(sizeof(*priv), NETDEV_ALIGN) + hif_data_len; -+ -+ hw = ieee80211_alloc_hw(priv_size, &mwl_mac80211_ops); -+ if (!hw) { -+ pr_err("ieee80211 alloc hw failed\n"); -+ return NULL; -+ } -+ -+ priv = hw->priv; -+ priv->hw = hw; -+ priv->dev = dev; -+ priv->chip_type = chip_type; -+ priv->fw_device_pwrtbl = false; -+ priv->forbidden_setting = false; -+ priv->regulatory_set = false; -+ priv->use_short_slot = false; -+ priv->use_short_preamble = false; -+ priv->disable_2g = false; -+ priv->disable_5g = false; -+ priv->tx_amsdu = true; -+ priv->hif.bus = bus_type; -+ priv->hif.ops = ops; -+ priv->hif.priv = (char *)priv + ALIGN(sizeof(*priv), NETDEV_ALIGN); -+ priv->ampdu_num = mwl_hif_get_ampdu_num(hw); -+ priv->ampdu = -+ kzalloc(priv->ampdu_num * sizeof(*priv->ampdu), GFP_KERNEL); -+ if (!priv->ampdu) { -+ ieee80211_free_hw(hw); -+ pr_err("alloc ampdu stream failed\n"); -+ return NULL; -+ } -+ -+ if (chip_type == MWL8964) -+ priv->stnid_num = SYSADPT_MAX_STA_SC4; -+ else -+ priv->stnid_num = SYSADPT_MAX_STA; -+ priv->stnid = -+ kzalloc(priv->stnid_num * sizeof(struct mwl_stnid), GFP_KERNEL); -+ if (!priv->stnid) { -+ kfree(priv->ampdu); -+ ieee80211_free_hw(hw); -+ pr_err("alloc stnid failed\n"); -+ return NULL; -+ } -+ priv->available_stnid = 0; -+ -+ SET_IEEE80211_DEV(hw, dev); -+ -+ return hw; -+} -+ -+void mwl_free_hw(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ kfree(priv->stnid); -+ kfree(priv->ampdu); -+ ieee80211_free_hw(hw); -+} -+ -+int mwl_init_hw(struct ieee80211_hw *hw, const char *fw_name, -+ const char *cal_name, const char *txpwrlmt_name) -+{ -+ struct mwl_priv *priv = hw->priv; -+ int rc; -+ int tx_num = 4, rx_num = 4; -+ -+ -+ rc = mwl_prepare_cmd_buf(priv); -+ if (rc) { -+ wiphy_err(hw->wiphy, "fail to prepare command buffer\n"); -+ return -ENOMEM; -+ } -+ -+ rc = mwl_init_firmware(priv, fw_name, cal_name, txpwrlmt_name); -+ if (rc) { -+ wiphy_err(hw->wiphy, "fail to initialize firmware\n"); -+ return -EIO; -+ } -+ -+ /* firmware is loaded to H/W, it can be released now */ -+ release_firmware(priv->fw_ucode); -+ -+ mwl_process_of_dts(priv); -+ -+ rc = mwl_wl_init(priv); -+ if (rc) { -+ wiphy_err(hw->wiphy, "fail to initialize wireless lan\n"); -+ return -EIO; -+ } -+ -+ wiphy_info(priv->hw->wiphy, "2G %s, 5G %s\n", -+ priv->disable_2g ? "disabled" : "enabled", -+ priv->disable_5g ? "disabled" : "enabled"); -+ -+ if (priv->antenna_tx == ANTENNA_TX_2) -+ tx_num = 2; -+ else if (priv->antenna_tx == ANTENNA_TX_3) -+ tx_num = 3; -+ if (priv->antenna_rx == ANTENNA_RX_2) -+ rx_num = 2; -+ else if (priv->antenna_rx == ANTENNA_RX_3) -+ rx_num = 3; -+ wiphy_info(priv->hw->wiphy, "%d TX antennas, %d RX antennas\n", -+ tx_num, rx_num); -+ -+#ifdef CONFIG_DEBUG_FS -+ mwl_debugfs_init(hw); -+#endif -+ -+ return 0; -+} -+ -+void mwl_deinit_hw(struct ieee80211_hw *hw) -+{ -+#ifdef CONFIG_DEBUG_FS -+ mwl_debugfs_remove(hw); -+#endif -+ -+ mwl_wl_deinit(hw->priv); -+} -diff --git a/drivers/net/wireless/marvell/mwlwifi/core.h b/drivers/net/wireless/marvell/mwlwifi/core.h -new file mode 100644 -index 000000000000..00069c4f0b44 ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/core.h -@@ -0,0 +1,517 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file defines core layer related functions. */ -+ -+#ifndef _CORE_H_ -+#define _CORE_H_ -+ -+#include -+#include -+#include -+#include -+ -+#include "hif/hif.h" -+ -+/* antenna control */ -+#define ANTENNA_TX_4_AUTO 0 -+#define ANTENNA_TX_1 1 -+#define ANTENNA_TX_2 3 -+#define ANTENNA_TX_3 7 -+#define ANTENNA_RX_4_AUTO 0 -+#define ANTENNA_RX_1 1 -+#define ANTENNA_RX_2 2 -+#define ANTENNA_RX_3 3 -+#define ANTENNA_RX_MAX 4 -+ -+/* band related constants */ -+#define BAND_24_CHANNEL_NUM 14 -+#define BAND_24_RATE_NUM 13 -+#define BAND_50_CHANNEL_NUM 24 -+#define BAND_50_RATE_NUM 8 -+ -+#define NUM_WEP_KEYS 4 -+#define MWL_MAX_TID 8 -+#define MWL_AMSDU_SIZE_4K 1 -+#define MWL_AMSDU_SIZE_8K 2 -+#define MWL_AMSDU_SIZE_11K 3 -+ -+/* power init */ -+#define MWL_POWER_INIT_1 1 -+#define MWL_POWER_INIT_2 2 -+ -+/* tx rate information constants */ -+#define TX_RATE_FORMAT_LEGACY 0 -+#define TX_RATE_FORMAT_11N 1 -+#define TX_RATE_FORMAT_11AC 2 -+ -+#define TX_RATE_BANDWIDTH_20 0 -+#define TX_RATE_BANDWIDTH_40 1 -+#define TX_RATE_BANDWIDTH_80 2 -+#define TX_RATE_BANDWIDTH_160 3 -+ -+#define TX_RATE_INFO_STD_GI 0 -+#define TX_RATE_INFO_SHORT_GI 1 -+ -+/* tx rate information */ -+/* 0: legacy format 1: 11n format 2: 11ac format */ -+#define MWL_TX_RATE_FORMAT_MASK 0x00000003 -+#define MWL_TX_RATE_STBC_MASK 0x00000004 -+#define MWL_TX_RATE_STBC_SHIFT 2 -+/* 0: 20 MHz 1: 40 MHz 2: 80 MHz 3: 160 MHz */ -+#define MWL_TX_RATE_BANDWIDTH_MASK 0x00000030 -+#define MWL_TX_RATE_BANDWIDTH_SHIFT 4 -+/* 0: normal 1: short */ -+#define MWL_TX_RATE_SHORTGI_MASK 0x00000040 -+#define MWL_TX_RATE_SHORTGI_SHIFT 6 -+#define MWL_TX_RATE_RATEIDMCS_MASK 0x00007F00 -+#define MWL_TX_RATE_RATEIDMCS_SHIFT 8 -+/* 0: long 1: short */ -+#define MWL_TX_RATE_PREAMBLE_MASK 0x00008000 -+#define MWL_TX_RATE_PREAMBLE_SHIFT 15 -+#define MWL_TX_RATE_POWERID_MASK 0x003F0000 -+#define MWL_TX_RATE_POWERID_SHIFT 16 -+#define MWL_TX_RATE_ADVCODING_MASK 0x00400000 -+#define MWL_TX_RATE_ADVCODING_SHIFT 22 -+/* 0: beam forming off 1: beam forming on */ -+#define MWL_TX_RATE_BF_MASK 0x00800000 -+#define MWL_TX_RATE_BF_SHIFT 23 -+#define MWL_TX_RATE_ANTSELECT_MASK 0xFF000000 -+#define MWL_TX_RATE_ANTSELECT_SHIFT 24 -+ -+#define ACNT_BA_SIZE 1000 -+ -+/* Q stats */ -+#define QS_MAX_DATA_RATES_G 14 -+#define QS_NUM_SUPPORTED_11N_BW 2 -+#define QS_NUM_SUPPORTED_GI 2 -+#define QS_NUM_SUPPORTED_MCS 24 -+#define QS_NUM_SUPPORTED_11AC_NSS 3 -+#define QS_NUM_SUPPORTED_11AC_BW 4 -+#define QS_NUM_SUPPORTED_11AC_MCS 10 -+#define TX_RATE_HISTO_CUSTOM_CNT 1 -+#define TX_RATE_HISTO_PER_CNT 5 -+#define MAX_DATA_RATES_G 14 -+#define MAX_SUPPORTED_MCS 24 -+#define MAX_SUPPORTED_11AC_RATES 20 -+/* MAX_DATA_RATES_G + MAX_SUPPORTED_MCS + MAX_SUPPORTED_11AC_RATES */ -+#define MAX_SUPPORTED_RATES 58 -+#define SU_MIMO 0 -+#define MU_MIMO 1 -+#define SU_MU_TYPE_CNT 2 /* traffic type, SU and MU */ -+ -+/* BF operation mode */ -+#define TXBF_MODE_OFF 0x05 -+#define TXBF_MODE_AUTO 0x06 -+#define TXBF_MODE_BFMER_AUTO 0x07 -+ -+static const u8 TX_HISTO_PER_THRES[TX_RATE_HISTO_PER_CNT - 1] = {6, 12, 20, 30}; -+ -+enum { -+ MWL8864 = 0, -+ MWL8897, -+ MWL8964, -+ MWL8997, -+ MWLUNKNOWN, -+}; -+ -+enum mwl_bus { -+ MWL_BUS_PCIE, -+ MWL_BUS_SDIO, -+}; -+ -+enum { -+ AP_MODE_11AC = 0x10, /* generic 11ac indication mode */ -+ AP_MODE_2_4GHZ_11AC_MIXED = 0x17, -+}; -+ -+enum { -+ AMPDU_NO_STREAM = 0, -+ AMPDU_STREAM_NEW, -+ AMPDU_STREAM_IN_PROGRESS, -+ AMPDU_STREAM_ACTIVE, -+}; -+ -+enum { -+ LED_BLINK_RATE_LOW = 0x1, -+ LED_BLINK_RATE_MID, -+ LED_BLINK_RATE_HIGH, -+}; -+ -+struct mwl_chip_info { -+ const char *part_name; -+ const char *fw_image; -+ const char *cal_file; -+ const char *txpwrlmt_file; -+ int antenna_tx; -+ int antenna_rx; -+}; -+ -+struct mwl_device_pwr_tbl { -+ u8 channel; -+ u8 tx_pwr[SYSADPT_TX_PWR_LEVEL_TOTAL_SC4]; -+ u8 dfs_capable; -+ u8 ax_ant; -+ u8 cdd; -+}; -+ -+struct mwl_tx_pwr_tbl { -+ u8 channel; -+ u8 setcap; -+ u16 txantenna2; -+ u16 tx_power[SYSADPT_TX_POWER_LEVEL_TOTAL]; -+ bool cdd; -+}; -+ -+struct mwl_hw_data { -+ u32 fw_release_num; /* MajNbr:MinNbr:SubMin:PatchLevel */ -+ u8 hw_version; /* plain number indicating version */ -+ unsigned char mac_addr[ETH_ALEN]; /* well known -> AA:BB:CC:DD:EE:FF */ -+}; -+ -+struct mwl_ampdu_stream { -+ struct ieee80211_sta *sta; -+ u8 tid; -+ u8 state; -+ int idx; -+}; -+ -+struct mwl_stnid { -+ int macid; /* keep macid for related stnid */ -+ u16 aid; /* keep aid for related stnid */ -+}; -+ -+struct otp_data { -+ u8 buf[SYSADPT_OTP_BUF_SIZE]; -+ u32 len; /* Actual size of data in buf[] */ -+}; -+ -+struct txpwrlmt_cfg_data { -+ u8 buf[SYSADPT_TXPWRLMT_CFG_BUF_SIZE]; -+ u32 len; /* Actual size of data in buf[] */ -+}; -+ -+struct mwl_priv { -+ struct ieee80211_hw *hw; -+ struct device *dev; -+ struct firmware *fw_ucode; -+ struct firmware *cal_data; -+ struct firmware *txpwrlmt_file; -+ struct otp_data otp_data; -+ struct txpwrlmt_cfg_data txpwrlmt_data; -+ bool fw_device_pwrtbl; -+ bool forbidden_setting; -+ bool regulatory_set; -+ u32 fw_region_code; -+ char fw_alpha2[2]; -+ u8 number_of_channels; -+ struct mwl_device_pwr_tbl device_pwr_tbl[SYSADPT_MAX_NUM_CHANNELS]; -+ int chip_type; -+ -+ bool use_short_slot; -+ bool use_short_preamble; -+ -+ struct { -+ enum mwl_bus bus; -+ const struct mwl_hif_ops *ops; -+ void *priv; -+ } hif; -+ -+ struct device_node *dt_node; -+ struct device_node *pwr_node; -+ bool disable_2g; -+ bool disable_5g; -+ int antenna_tx; -+ int antenna_rx; -+ bool tx_amsdu; -+ bool dump_hostcmd; -+ bool dump_probe; -+ -+ struct mwl_tx_pwr_tbl tx_pwr_tbl[SYSADPT_MAX_NUM_CHANNELS]; -+ bool cdd; -+ u16 txantenna2; -+ u8 powinited; -+ u8 pwr_level; -+ u16 max_tx_pow[SYSADPT_TX_GRP_PWR_LEVEL_TOTAL]; /* max tx power (dBm) */ -+ u16 target_powers[SYSADPT_TX_GRP_PWR_LEVEL_TOTAL]; /* target powers */ -+ -+ struct mutex fwcmd_mutex; /* for firmware command */ -+ unsigned short *pcmd_buf; /* pointer to CmdBuf (virtual) */ -+ dma_addr_t pphys_cmd_buf; /* pointer to CmdBuf (physical) */ -+ bool in_send_cmd; -+ bool cmd_timeout; -+ bool rmmod; -+ int heartbeat; -+ u32 pre_jiffies; -+ bool heartbeating; -+ struct work_struct heartbeat_handle; -+ -+ int irq; -+ struct mwl_hw_data hw_data; /* Adapter HW specific info */ -+ -+ struct timer_list period_timer; -+ -+ /* keep survey information */ -+ bool sw_scanning; -+ int survey_info_idx; -+ struct mwl_survey_info survey_info[SYSADPT_MAX_NUM_CHANNELS]; -+ struct mwl_survey_info cur_survey_info; -+ -+ s8 noise; /* Most recently reported noise in dBm */ -+ -+ struct ieee80211_supported_band band_24; -+ struct ieee80211_channel channels_24[BAND_24_CHANNEL_NUM]; -+ struct ieee80211_rate rates_24[BAND_24_RATE_NUM]; -+ struct ieee80211_supported_band band_50; -+ struct ieee80211_channel channels_50[BAND_50_CHANNEL_NUM]; -+ struct ieee80211_rate rates_50[BAND_50_RATE_NUM]; -+ -+ u32 ap_macids_supported; -+ u32 sta_macids_supported; -+ u32 macids_used; -+ u32 running_bsses; /* bitmap of running BSSes */ -+ -+ struct { -+ spinlock_t vif_lock; /* for private interface info */ -+ struct list_head vif_list; /* List of interfaces. */ -+ } ____cacheline_aligned_in_smp; -+ -+ struct { -+ spinlock_t sta_lock; /* for private sta info */ -+ struct list_head sta_list; /* List of stations */ -+ } ____cacheline_aligned_in_smp; -+ -+ /* ampdu stream information */ -+ /* for ampdu stream */ -+ int ampdu_num; -+ struct { -+ spinlock_t stream_lock; /* for BA stream */ -+ struct mwl_ampdu_stream *ampdu; -+ } ____cacheline_aligned_in_smp; -+ struct work_struct watchdog_ba_handle; -+ -+ /* station id */ -+ int stnid_num; -+ struct { -+ spinlock_t stnid_lock; /* for station id */ -+ struct mwl_stnid *stnid; -+ u16 available_stnid; -+ } ____cacheline_aligned_in_smp; -+ -+ bool radio_on; -+ bool radio_short_preamble; -+ bool wmm_enabled; -+ struct ieee80211_tx_queue_params wmm_params[SYSADPT_TX_WMM_QUEUES]; -+ -+ struct work_struct account_handle; -+ -+ bool wds_check; -+ struct work_struct wds_check_handle; -+ u8 wds_check_sta[ETH_ALEN]; -+ -+ bool dfs_test; -+ bool csa_active; -+ struct work_struct chnl_switch_handle; -+ enum nl80211_dfs_regions dfs_region; -+ u16 dfs_chirp_count_min; -+ u16 dfs_chirp_time_interval; -+ u16 dfs_pw_filter; -+ u16 dfs_min_num_radar; -+ u16 dfs_min_pri_count; -+ -+ u8 bf_type; -+ -+ struct thermal_cooling_device *cdev; -+ u32 throttle_state; -+ u32 quiet_period; -+ int temperature; -+ -+ u8 led_blink_enable; -+ u8 led_blink_rate; -+ -+ struct dentry *debugfs_phy; -+ u32 reg_type; -+ u32 reg_offset; -+ u32 reg_value; -+ int ra_aid; -+ int ba_aid; -+ int fixed_rate; -+ bool coredump_text; -+ u32 ra_tx_attempt[2][6]; -+}; -+ -+struct beacon_info { -+ bool valid; -+ u16 cap_info; -+ u8 power_constraint; -+ u8 b_rate_set[SYSADPT_MAX_DATA_RATES_G]; -+ u8 op_rate_set[SYSADPT_MAX_DATA_RATES_G]; -+ u8 ie_list_ht[148]; -+ u8 ie_list_vht[24]; -+ u8 *ie_wmm_ptr; -+ u8 *ie_wsc_ptr; -+ u8 *ie_rsn_ptr; -+ u8 *ie_rsn48_ptr; -+ u8 *ie_mde_ptr; -+ u8 *ie_ht_ptr; -+ u8 *ie_vht_ptr; -+ u8 *ie_country_ptr; -+ u8 *ie_meshid_ptr; -+ u8 *ie_meshcfg_ptr; -+ u8 *ie_meshchsw_ptr; -+ u8 ie_wmm_len; -+ u8 ie_wsc_len; -+ u8 ie_rsn_len; -+ u8 ie_rsn48_len; -+ u8 ie_mde_len; -+ u8 ie_ht_len; -+ u8 ie_vht_len; -+ u8 ie_country_len; -+ u8 ie_meshid_len; -+ u8 ie_meshcfg_len; -+ u8 ie_meshchsw_len; -+}; -+ -+struct mwl_vif { -+ struct list_head list; -+ enum nl80211_iftype type; -+ int macid; /* Firmware macid for this vif. */ -+ u16 seqno; /* Non AMPDU sequence number assigned by driver. */ -+ struct { /* Saved WEP keys */ -+ u8 enabled; -+ u8 key[sizeof(struct ieee80211_key_conf) + WLAN_KEY_LEN_WEP104]; -+ } wep_key_conf[NUM_WEP_KEYS]; -+ u8 bssid[ETH_ALEN]; /* BSSID */ -+ u8 sta_mac[ETH_ALEN]; /* Station mac address */ -+ /* A flag to indicate is HW crypto is enabled for this bssid */ -+ bool is_hw_crypto_enabled; -+ /* Indicate if this is station mode */ -+ struct beacon_info beacon_info; -+ bool set_beacon; -+ int basic_rate_idx; -+ u8 broadcast_ssid; -+ u16 iv16; -+ u32 iv32; -+ s8 keyidx; -+}; -+ -+struct mwl_tx_info { -+ unsigned long start_time; -+ u32 pkts; -+}; -+ -+struct mwl_amsdu_frag { -+ struct sk_buff *skb; -+ u8 *cur_pos; -+ unsigned long jiffies; -+ u8 pad; -+ u8 num; -+}; -+ -+struct mwl_amsdu_ctrl { -+ struct mwl_amsdu_frag frag[SYSADPT_TX_WMM_QUEUES]; -+ u8 cap; -+}; -+ -+struct mwl_tx_ba_stats { -+ u8 ba_hole; /* Total pkt not acked in a BA bitmap */ -+ u8 ba_expected; /* Total Tx pkt expected to be acked */ -+ u8 no_ba; /* No BA is received */ -+ u8 pad; /* Unused */ -+}; -+ -+struct mwl_tx_ba_hist { -+ u16 index; /* Current buffer index */ -+ u8 type; /* 0:SU, 1: MU */ -+ bool enable; -+ struct mwl_tx_ba_stats *ba_stats; -+}; -+ -+struct mwl_tx_hist_data { -+ u32 rateinfo; -+ u32 cnt; -+ /* store according to TX_HISTO_PER_THRES threshold */ -+ u32 per[TX_RATE_HISTO_PER_CNT]; -+}; -+ -+struct mwl_tx_hist { -+ struct mwl_tx_hist_data su_rate[MAX_SUPPORTED_RATES]; -+ struct mwl_tx_hist_data mu_rate -+ [QS_NUM_SUPPORTED_11AC_NSS - 1][QS_NUM_SUPPORTED_11AC_BW] -+ [QS_NUM_SUPPORTED_GI][QS_NUM_SUPPORTED_11AC_MCS]; -+ struct mwl_tx_hist_data custom_rate[TX_RATE_HISTO_CUSTOM_CNT]; -+ /* Current rate for 0:SU, 1:MU */ -+ u32 cur_rate_info[SU_MU_TYPE_CNT]; -+ /* Total tx attempt cnt for 0:SU, 1:MU */ -+ u32 total_tx_cnt[SU_MU_TYPE_CNT]; -+}; -+ -+struct mwl_sta { -+ struct list_head list; -+ struct mwl_vif *mwl_vif; -+ u16 stnid; -+ u16 sta_stnid; -+ bool wds; -+ bool is_mesh_node; -+ bool is_ampdu_allowed; -+ struct mwl_tx_info tx_stats[MWL_MAX_TID]; -+ u32 check_ba_failed[MWL_MAX_TID]; -+ struct mwl_tx_ba_hist ba_hist; -+ bool is_amsdu_allowed; -+ bool is_key_set; -+ /* for amsdu aggregation */ -+ struct { -+ spinlock_t amsdu_lock; /* for amsdu */ -+ struct mwl_amsdu_ctrl amsdu_ctrl; -+ } ____cacheline_aligned_in_smp; -+ struct mwl_tx_hist tx_hist; -+ u32 tx_rate_info; -+ u16 rx_format; -+ u16 rx_nss; -+ u16 rx_bw; -+ u16 rx_gi; -+ u16 rx_rate_mcs; -+ u8 rx_signal; -+ u16 iv16; -+ u32 iv32; -+}; -+ -+static inline struct mwl_vif *mwl_dev_get_vif(const struct ieee80211_vif *vif) -+{ -+ return (struct mwl_vif *)&vif->drv_priv; -+} -+ -+static inline struct mwl_sta *mwl_dev_get_sta(const struct ieee80211_sta *sta) -+{ -+ return (struct mwl_sta *)&sta->drv_priv; -+} -+ -+struct ieee80211_hw *mwl_alloc_hw(int bus_type, -+ int chip_type, -+ struct device *dev, -+ const struct mwl_hif_ops *ops, -+ size_t hif_data_len); -+ -+void mwl_free_hw(struct ieee80211_hw *hw); -+ -+int mwl_init_hw(struct ieee80211_hw *hw, const char *fw_name, -+ const char *cal_name, const char *txpwrlmt_name); -+ -+void mwl_deinit_hw(struct ieee80211_hw *hw); -+ -+/* Defined in mac80211.c. */ -+extern const struct ieee80211_ops mwl_mac80211_ops; -+ -+#endif /* _CORE_H_ */ -diff --git a/drivers/net/wireless/marvell/mwlwifi/debugfs.c b/drivers/net/wireless/marvell/mwlwifi/debugfs.c -new file mode 100644 -index 000000000000..e375806a7111 ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/debugfs.c -@@ -0,0 +1,2201 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file implements debug fs related functions. */ -+ -+#include -+#include -+ -+#include "sysadpt.h" -+#include "core.h" -+#include "utils.h" -+#include "thermal.h" -+#include "hif/fwcmd.h" -+#include "hif/hif-ops.h" -+#include "debugfs.h" -+ -+#define MWLWIFI_DEBUGFS_ADD_FILE(name) do { \ -+ if (!debugfs_create_file(#name, 0644, priv->debugfs_phy, \ -+ priv, &mwl_debugfs_##name##_fops)) \ -+ return; \ -+} while (0) -+ -+#define MWLWIFI_DEBUGFS_FILE_OPS(name) \ -+static const struct file_operations mwl_debugfs_##name##_fops = { \ -+ .read = mwl_debugfs_##name##_read, \ -+ .write = mwl_debugfs_##name##_write, \ -+ .open = simple_open, \ -+} -+ -+#define MWLWIFI_DEBUGFS_FILE_READ_OPS(name) \ -+static const struct file_operations mwl_debugfs_##name##_fops = { \ -+ .read = mwl_debugfs_##name##_read, \ -+ .open = simple_open, \ -+} -+ -+#define MWLWIFI_DEBUGFS_FILE_WRITE_OPS(name) \ -+static const struct file_operations mwl_debugfs_##name##_fops = { \ -+ .write = mwl_debugfs_##name##_write, \ -+ .open = simple_open, \ -+} -+ -+static const char chipname[MWLUNKNOWN][8] = { -+ "88W8864", -+ "88W8897", -+ "88W8964", -+ "88W8997" -+}; -+ -+static void dump_data(char *p, int size, int *len, u8 *data, -+ int data_len, char *title) -+{ -+ int cur_byte = 0; -+ int i; -+ -+ *len += scnprintf(p + *len, size - *len, "%s\n", title); -+ -+ for (cur_byte = 0; cur_byte < data_len; cur_byte += 8) { -+ if ((cur_byte + 8) < data_len) { -+ for (i = 0; i < 8; i++) -+ *len += scnprintf(p + *len, size - *len, -+ "0x%02x ", -+ *(data + cur_byte + i)); -+ *len += scnprintf(p + *len, size - *len, "\n"); -+ } else { -+ for (i = 0; i < (data_len - cur_byte); i++) -+ *len += scnprintf(p + *len, size - *len, -+ "0x%02x ", -+ *(data + cur_byte + i)); -+ *len += scnprintf(p + *len, size - *len, "\n"); -+ break; -+ } -+ } -+} -+ -+static void _dump_tx_hist_mu(char *p, int size, int *len, bool *printed, -+ u32 *total, u8 nss, u8 bw, u8 mcs, u8 sgi, -+ struct mwl_sta *sta_info) -+{ -+ char *bw_str[4] = {"ht20", "ht40", "ht80", "ht160"}; -+ char *sgi_str[2] = {"lgi", "sgi"}; -+ struct mwl_tx_hist_data *tx_hist_data; -+ u32 cnt, rateinfo, per0, per1, per2, per3, per4, ratemask; -+ -+ tx_hist_data = &sta_info->tx_hist.mu_rate[nss][bw][sgi][mcs]; -+ cnt = le32_to_cpu(tx_hist_data->cnt); -+ rateinfo = le32_to_cpu(tx_hist_data->rateinfo); -+ if (cnt && (rateinfo > 0)) { -+ *total += cnt; -+ per4 = le32_to_cpu(tx_hist_data->per[4]); -+ per3 = le32_to_cpu(tx_hist_data->per[3]); -+ per2 = le32_to_cpu(tx_hist_data->per[2]); -+ per1 = le32_to_cpu(tx_hist_data->per[1]); -+ per0 = le32_to_cpu(tx_hist_data->per[0]); -+ if (!*printed) { -+ *len += scnprintf(p + *len, size - *len, -+ "%s %26s <%2d %8s%2d%8s%2d%8s%2d%8s%2d\n", -+ "MU_MIMO rate", " PER%", TX_HISTO_PER_THRES[0], -+ ">=", TX_HISTO_PER_THRES[0], -+ ">=", TX_HISTO_PER_THRES[1], -+ ">=", TX_HISTO_PER_THRES[2], -+ ">=", TX_HISTO_PER_THRES[3]); -+ *len += scnprintf(p + *len, size - *len, -+ "TOTAL MPDU tx pkt: %d\n", -+ sta_info->tx_hist.total_tx_cnt[MU_MIMO]); -+ *printed = true; -+ } -+ if ((rateinfo & 0x3) == 0) -+ ratemask = 0xfff; -+ else -+ ratemask = 0xffff; -+ if ((sta_info->tx_hist.cur_rate_info[MU_MIMO] & ratemask) == -+ (rateinfo & ratemask)) -+ /* mark as current rate */ -+ *len += scnprintf(p + *len, size - *len, "*"); -+ else -+ *len += scnprintf(p + *len, size - *len, " "); -+ *len += scnprintf(p + *len, size - *len, -+ "%5s_%3s_%1dSS_MCS%2d: %10u, %9d, %9d, %9d, %9d, %9d\n", -+ bw_str[bw], sgi_str[sgi], (nss + 1), mcs, cnt, per0, -+ per1, per2, per3, per4); -+ } -+} -+ -+static void dump_tx_hist_mu(char *p, int size, int *len, bool *printed, -+ u32 *total, struct mwl_sta *sta_info) -+{ -+ u8 nss, bw, mcs, sgi; -+ -+ for (nss = 0; nss < (QS_NUM_SUPPORTED_11AC_NSS - 1); nss++) { -+ for (bw = 0; bw < QS_NUM_SUPPORTED_11AC_BW; bw++) { -+ for (mcs = 0; mcs < QS_NUM_SUPPORTED_11AC_MCS; mcs++) { -+ for (sgi = 0; sgi < QS_NUM_SUPPORTED_GI; -+ sgi++) { -+ _dump_tx_hist_mu(p, size, len, printed, -+ total, nss, bw, mcs, -+ sgi, sta_info); -+ } -+ } -+ } -+ } -+} -+ -+ -+static void dump_tx_hist_su(char *p, int size, int *len, bool su, bool *printed, -+ u32 *total, struct mwl_sta *sta_info) -+{ -+ int g_rate[14] = {1, 2, 5, 11, 22, 6, 9, 12, 18, 24, 36, 48, 54, 72}; -+ char *bw_str[4] = {"ht20", "ht40", "ht80", "ht160"}; -+ char *sgi_str[2] = {"lgi", "sgi"}; -+ char title_str[32]; -+ struct mwl_tx_hist *tx_hist; -+ struct mwl_tx_hist_data *tx_hist_data; -+ u32 j, loopcnt; -+ u32 cnt, rateinfo, per0, per1, per2, per3, per4, ratemask; -+ u8 format, bw, sgi, mcs, nss; -+ -+ tx_hist = &sta_info->tx_hist; -+ if (su) { -+ loopcnt = MAX_SUPPORTED_RATES; -+ tx_hist_data = &tx_hist->su_rate[0]; -+ } else { -+ loopcnt = TX_RATE_HISTO_CUSTOM_CNT; -+ tx_hist_data = &tx_hist->custom_rate[0]; -+ } -+ -+ for (j = 0; j < loopcnt; j++) { -+ cnt = le32_to_cpu(tx_hist_data[j].cnt); -+ rateinfo = le32_to_cpu(tx_hist_data[j].rateinfo); -+ if (cnt && (rateinfo > 0)) { -+ *total += cnt; -+ per4 = le32_to_cpu(tx_hist_data[j].per[4]); -+ per3 = le32_to_cpu(tx_hist_data[j].per[3]); -+ per2 = le32_to_cpu(tx_hist_data[j].per[2]); -+ per1 = le32_to_cpu(tx_hist_data[j].per[1]); -+ per0 = le32_to_cpu(tx_hist_data[j].per[0]); -+ if (!*printed) { -+ *len += scnprintf(p + *len, size - *len, -+ "%s %26s <%2d %8s%2d%8s%2d%8s%2d%8s%2d\n", -+ su ? "SU_MIMO rate" : " Custom rate", -+ " PER%", TX_HISTO_PER_THRES[0], -+ ">=", TX_HISTO_PER_THRES[0], -+ ">=", TX_HISTO_PER_THRES[1], -+ ">=", TX_HISTO_PER_THRES[2], -+ ">=", TX_HISTO_PER_THRES[3]); -+ *len += scnprintf(p + *len, size - *len, -+ "TOTAL MPDU tx pkt: %d\n", -+ tx_hist->total_tx_cnt[SU_MIMO]); -+ *printed = true; -+ } -+ format = rateinfo & MWL_TX_RATE_FORMAT_MASK; -+ bw = (rateinfo & MWL_TX_RATE_BANDWIDTH_MASK) >> -+ MWL_TX_RATE_BANDWIDTH_SHIFT; -+ sgi = (rateinfo & MWL_TX_RATE_SHORTGI_MASK) >> -+ MWL_TX_RATE_SHORTGI_SHIFT; -+ mcs = (rateinfo & MWL_TX_RATE_RATEIDMCS_MASK) >> -+ MWL_TX_RATE_RATEIDMCS_SHIFT; -+ if (format == TX_RATE_FORMAT_LEGACY) -+ ratemask = 0xfff; -+ else -+ ratemask = 0xffff; -+ if ((tx_hist->cur_rate_info[SU_MIMO] & ratemask) == -+ (rateinfo & ratemask)) -+ /* mark as current rate */ -+ *len += scnprintf(p + *len, size - *len, "*"); -+ else -+ *len += scnprintf(p + *len, size - *len, " "); -+ if (format == TX_RATE_FORMAT_LEGACY) { -+ if (mcs == 2) { -+ *len += scnprintf(p + *len, size - *len, -+ "%s %10u, %9d, %9d, %9d, %9d, %9d\n", -+ "5.5Mbps :", cnt, per0, -+ per1, per2, per3, per4); -+ } else { -+ sprintf(title_str, -+ "%-3dMbps :", -+ g_rate[mcs]); -+ *len += scnprintf(p + *len, size - *len, -+ "%s %10u, %9d, %9d, %9d, %9d, %9d\n", -+ title_str, cnt, per0, per1, per2, per3, -+ per4); -+ } -+ } else if (format == TX_RATE_FORMAT_11N) { -+ sprintf(title_str, "%4s_%3s_MCS%2d :", -+ bw_str[bw], sgi_str[sgi], mcs); -+ *len += scnprintf(p + *len, size - *len, -+ "%s %10u, %9d, %9d, %9d, %9d, %9d\n", -+ title_str, cnt, per0, per1, per2, per3, -+ per4); -+ } else { -+ nss = (mcs >> 4); -+ sprintf(title_str, "%5s_%3s_%1dSS_MCS%2d :", -+ bw_str[bw], sgi_str[sgi], (nss+1), -+ (mcs & 0xf)); -+ *len += scnprintf(p + *len, size - *len, -+ "%s %10u, %9d, %9d, %9d, %9d, %9d\n", -+ title_str, cnt, per0, per1, per2, per3, -+ per4); -+ } -+ } -+ } -+} -+ -+static void dump_tx_hist(char *p, int size, int *len, struct mwl_sta *sta_info) -+{ -+ int type; -+ bool printed, su; -+ u32 total; -+ -+ for (type = 0; type <= SU_MU_TYPE_CNT; type++) { -+ printed = false; -+ total = 0; -+ if (type == MU_MIMO) { -+ dump_tx_hist_mu(p, size, len, &printed, -+ &total, sta_info); -+ } else { -+ su = (type == SU_MIMO) ? true : false; -+ dump_tx_hist_su(p, size, len, su, &printed, -+ &total, sta_info); -+ } -+ if (printed) -+ *len += scnprintf(p + *len, size - *len, -+ " TOTAL : %10u\n\n", -+ total); -+ } -+} -+ -+static void core_dump_file(u8 *valbuf, u32 length, u32 region, u32 address, -+ u32 append, u32 totallen, bool textmode) -+{ -+ struct file *filp_core = NULL; -+ char file_name[40]; -+ u8 *buf = kmalloc(length * 3, GFP_KERNEL); -+ u8 *data_p = buf; -+ u32 i, j = 0; -+ -+ if (!buf) -+ return; -+ -+ memset(file_name, 0, sizeof(file_name)); -+ sprintf(file_name, "/dev/shm/coredump-%x-%x", -+ region, (region + totallen)); -+ -+ if (append) -+ filp_core = filp_open(file_name, O_RDWR | O_APPEND, 0); -+ else -+ filp_core = filp_open(file_name, O_RDWR | O_CREAT | O_TRUNC, 0); -+ -+ if (!IS_ERR(filp_core)) { -+ if (textmode) { -+ for (i = 0; i < length; i += 4) { -+ u32 val = 0; -+ -+ val = le32_to_cpu(*(__le32 *)(&valbuf[i])); -+ -+ if (i % 16 == 0) { -+ sprintf(buf + j, "\n0x%08x", -+ (int)(address + i)); -+ j = strlen(buf); -+ } -+ sprintf(buf + j, " %08x", val); -+ j = strlen(buf); -+ } -+ data_p = buf + j; -+ data_p += sprintf(data_p, "\n"); -+ __kernel_write(filp_core, buf, strlen(buf), -+ &filp_core->f_pos); -+ } else -+ __kernel_write(filp_core, valbuf, length, -+ &filp_core->f_pos); -+ -+ filp_close(filp_core, current->files); -+ } -+ -+ kfree(buf); -+} -+ -+static ssize_t mwl_debugfs_info_read(struct file *file, char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long page = get_zeroed_page(GFP_KERNEL); -+ int tx_num = 4, rx_num = 4; -+ char *p = (char *)page; -+ int len = 0, size = PAGE_SIZE; -+ ssize_t ret; -+ -+ if (!p) -+ return -ENOMEM; -+ -+ len += scnprintf(p + len, size - len, "\n"); -+ len += scnprintf(p + len, size - len, -+ "driver name: %s\n", -+ mwl_hif_get_driver_name(priv->hw)); -+ len += scnprintf(p + len, size - len, "chip type: %s\n", -+ chipname[priv->chip_type]); -+ len += scnprintf(p + len, size - len, -+ "hw version: %X\n", priv->hw_data.hw_version); -+ len += scnprintf(p + len, size - len, -+ "driver version: %s\n", -+ mwl_hif_get_driver_version(priv->hw)); -+ len += scnprintf(p + len, size - len, "firmware version: 0x%08x\n", -+ priv->hw_data.fw_release_num); -+ len += scnprintf(p + len, size - len, -+ "power table loaded from dts: %s\n", -+ priv->forbidden_setting ? "no" : "yes"); -+ len += scnprintf(p + len, size - len, "firmware region code: 0x%x\n", -+ priv->fw_region_code); -+ len += scnprintf(p + len, size - len, -+ "mac address: %pM\n", priv->hw_data.mac_addr); -+ len += scnprintf(p + len, size - len, -+ "2g: %s\n", priv->disable_2g ? "disable" : "enable"); -+ len += scnprintf(p + len, size - len, -+ "5g: %s\n", priv->disable_5g ? "disable" : "enable"); -+ if (priv->antenna_tx == ANTENNA_TX_2) -+ tx_num = 2; -+ else if (priv->antenna_tx == ANTENNA_TX_3) -+ tx_num = 3; -+ if (priv->antenna_rx == ANTENNA_RX_2) -+ rx_num = 2; -+ else if (priv->antenna_rx == ANTENNA_RX_3) -+ rx_num = 3; -+ len += scnprintf(p + len, size - len, "antenna: %d %d\n", -+ tx_num, rx_num); -+ len += scnprintf(p + len, size - len, "irq number: %d\n", priv->irq); -+ len += scnprintf(p + len, size - len, "ap macid support: %08x\n", -+ priv->ap_macids_supported); -+ len += scnprintf(p + len, size - len, "sta macid support: %08x\n", -+ priv->sta_macids_supported); -+ len += scnprintf(p + len, size - len, -+ "macid used: %08x\n", priv->macids_used); -+ len += scnprintf(p + len, size - len, -+ "radio: %s\n", priv->radio_on ? "enable" : "disable"); -+ len += mwl_hif_get_info(priv->hw, p + len, size - len); -+ len += scnprintf(p + len, size - len, "\n"); -+ -+ ret = simple_read_from_buffer(ubuf, count, ppos, p, len); -+ free_page(page); -+ -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_tx_status_read(struct file *file, char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long page = get_zeroed_page(GFP_KERNEL); -+ char *p = (char *)page; -+ int len = 0, size = PAGE_SIZE; -+ ssize_t ret; -+ -+ if (!p) -+ return -ENOMEM; -+ -+ len += scnprintf(p + len, size - len, "\n"); -+ len += mwl_hif_get_tx_status(priv->hw, p + len, size - len); -+ len += scnprintf(p + len, size - len, "\n"); -+ -+ ret = simple_read_from_buffer(ubuf, count, ppos, p, len); -+ free_page(page); -+ -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_rx_status_read(struct file *file, char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long page = get_zeroed_page(GFP_KERNEL); -+ char *p = (char *)page; -+ int len = 0, size = PAGE_SIZE; -+ ssize_t ret; -+ -+ if (!p) -+ return -ENOMEM; -+ -+ len += scnprintf(p + len, size - len, "\n"); -+ len += mwl_hif_get_rx_status(priv->hw, p + len, size - len); -+ len += scnprintf(p + len, size - len, "\n"); -+ -+ ret = simple_read_from_buffer(ubuf, count, ppos, p, len); -+ free_page(page); -+ -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_vif_read(struct file *file, char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long page = get_zeroed_page(GFP_KERNEL); -+ char *p = (char *)page; -+ int len = 0, size = PAGE_SIZE; -+ struct mwl_vif *mwl_vif; -+ struct ieee80211_vif *vif; -+ char ssid[IEEE80211_MAX_SSID_LEN + 1]; -+ struct cfg80211_chan_def *chan_def; -+ struct beacon_info *beacon_info; -+ ssize_t ret; -+ -+ if (!p) -+ return -ENOMEM; -+ -+ len += scnprintf(p + len, size - len, "\n"); -+ spin_lock_bh(&priv->vif_lock); -+ list_for_each_entry(mwl_vif, &priv->vif_list, list) { -+ vif = container_of((void *)mwl_vif, struct ieee80211_vif, -+ drv_priv); -+ len += scnprintf(p + len, size - len, -+ "macid: %d\n", mwl_vif->macid); -+ switch (vif->type) { -+ case NL80211_IFTYPE_AP: -+ len += scnprintf(p + len, size - len, "type: ap\n"); -+ memcpy(ssid, vif->bss_conf.ssid, -+ vif->bss_conf.ssid_len); -+ ssid[vif->bss_conf.ssid_len] = 0; -+ len += scnprintf(p + len, size - len, -+ "ssid: %s\n", ssid); -+ len += scnprintf(p + len, size - len, -+ "mac address: %pM\n", mwl_vif->bssid); -+ break; -+ case NL80211_IFTYPE_MESH_POINT: -+ len += scnprintf(p + len, size - len, "type: mesh\n"); -+ len += scnprintf(p + len, size - len, -+ "mac address: %pM\n", mwl_vif->bssid); -+ break; -+ case NL80211_IFTYPE_STATION: -+ len += scnprintf(p + len, size - len, "type: sta\n"); -+ len += scnprintf(p + len, size - len, -+ "mac address: %pM\n", -+ mwl_vif->sta_mac); -+ break; -+ default: -+ len += scnprintf(p + len, size - len, -+ "type: unknown\n"); -+ break; -+ } -+ if (vif->chanctx_conf) { -+ chan_def = &vif->chanctx_conf->def; -+ len += scnprintf(p + len, size - len, -+ "channel: %d: width: %d\n", -+ chan_def->chan->hw_value, -+ chan_def->width); -+ len += scnprintf(p + len, size - len, -+ "freq: %d freq1: %d freq2: %d\n", -+ chan_def->chan->center_freq, -+ chan_def->center_freq1, -+ chan_def->center_freq2); -+ } -+ len += scnprintf(p + len, size - len, "hw_crypto_enabled: %s\n", -+ mwl_vif->is_hw_crypto_enabled ? -+ "true" : "false"); -+ len += scnprintf(p + len, size - len, -+ "key idx: %d\n", mwl_vif->keyidx); -+ len += scnprintf(p + len, size - len, -+ "IV: %08x%04x\n", mwl_vif->iv32, -+ mwl_vif->iv16); -+ beacon_info = &mwl_vif->beacon_info; -+ dump_data(p, size, &len, beacon_info->ie_wmm_ptr, -+ beacon_info->ie_wmm_len, "WMM:"); -+ dump_data(p, size, &len, beacon_info->ie_rsn_ptr, -+ beacon_info->ie_rsn_len, "RSN:"); -+ dump_data(p, size, &len, beacon_info->ie_rsn48_ptr, -+ beacon_info->ie_rsn48_len, "RSN48:"); -+ dump_data(p, size, &len, beacon_info->ie_mde_ptr, -+ beacon_info->ie_mde_len, "MDE:"); -+ dump_data(p, size, &len, beacon_info->ie_ht_ptr, -+ beacon_info->ie_ht_len, "HT:"); -+ dump_data(p, size, &len, beacon_info->ie_vht_ptr, -+ beacon_info->ie_vht_len, "VHT:"); -+ if (vif->type == NL80211_IFTYPE_MESH_POINT) { -+ dump_data(p, size, &len, beacon_info->ie_meshid_ptr, -+ beacon_info->ie_meshid_len, "MESHID:"); -+ dump_data(p, size, &len, beacon_info->ie_meshcfg_ptr, -+ beacon_info->ie_meshcfg_len, "MESHCFG:"); -+ dump_data(p, size, &len, beacon_info->ie_meshchsw_ptr, -+ beacon_info->ie_meshchsw_len, "MESHCHSW:"); -+ } -+ len += scnprintf(p + len, size - len, "\n"); -+ } -+ spin_unlock_bh(&priv->vif_lock); -+ -+ ret = simple_read_from_buffer(ubuf, count, ppos, p, len); -+ free_page(page); -+ -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_sta_read(struct file *file, char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long page = get_zeroed_page(GFP_KERNEL); -+ char *p = (char *)page; -+ int len = 0, size = PAGE_SIZE; -+ struct mwl_sta *sta_info; -+ struct ieee80211_sta *sta; -+ ssize_t ret; -+ -+ if (!p) -+ return -ENOMEM; -+ -+ len += scnprintf(p + len, size - len, "\n"); -+ spin_lock_bh(&priv->sta_lock); -+ list_for_each_entry(sta_info, &priv->sta_list, list) { -+ sta = container_of((void *)sta_info, struct ieee80211_sta, -+ drv_priv); -+ len += scnprintf(p + len, size - len, -+ "mac address: %pM\n", sta->addr); -+ len += scnprintf(p + len, size - len, "aid: %u\n", sta->aid); -+ len += scnprintf(p + len, size - len, "ampdu: %s\n", -+ sta_info->is_ampdu_allowed ? "true" : "false"); -+ len += scnprintf(p + len, size - len, "amsdu: %s\n", -+ sta_info->is_amsdu_allowed ? "true" : "false"); -+ len += scnprintf(p + len, size - len, "wds: %s\n", -+ sta_info->wds ? "true" : "false"); -+ len += scnprintf(p + len, size - len, "ba_hist: %s\n", -+ sta_info->ba_hist.enable ? -+ "enable" : "disable"); -+ if (sta_info->is_amsdu_allowed) { -+ len += scnprintf(p + len, size - len, -+ "amsdu cap: 0x%02x\n", -+ sta_info->amsdu_ctrl.cap); -+ } -+ if (sta->ht_cap.ht_supported) { -+ len += scnprintf(p + len, size - len, -+ "ht_cap: 0x%04x, ampdu: %02x, %02x\n", -+ sta->ht_cap.cap, -+ sta->ht_cap.ampdu_factor, -+ sta->ht_cap.ampdu_density); -+ len += scnprintf(p + len, size - len, -+ "rx_mask: 0x%02x, %02x, %02x, %02x\n", -+ sta->ht_cap.mcs.rx_mask[0], -+ sta->ht_cap.mcs.rx_mask[1], -+ sta->ht_cap.mcs.rx_mask[2], -+ sta->ht_cap.mcs.rx_mask[3]); -+ } -+ if (sta->vht_cap.vht_supported) { -+ len += scnprintf(p + len, size - len, -+ "vht_cap: 0x%08x, mcs: %02x, %02x\n", -+ sta->vht_cap.cap, -+ sta->vht_cap.vht_mcs.rx_mcs_map, -+ sta->vht_cap.vht_mcs.tx_mcs_map); -+ } -+ len += scnprintf(p + len, size - len, "rx_bw: %d, rx_nss: %d\n", -+ sta->bandwidth, sta->rx_nss); -+ len += scnprintf(p + len, size - len, -+ "tdls: %d, tdls_init: %d\n", -+ sta->tdls, sta->tdls_initiator); -+ len += scnprintf(p + len, size - len, "wme: %d, mfp: %d\n", -+ sta->wme, sta->mfp); -+ len += scnprintf(p + len, size - len, "IV: %08x%04x\n", -+ sta_info->iv32, sta_info->iv16); -+ len += scnprintf(p + len, size - len, "\n"); -+ } -+ spin_unlock_bh(&priv->sta_lock); -+ -+ ret = simple_read_from_buffer(ubuf, count, ppos, p, len); -+ free_page(page); -+ -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_ampdu_read(struct file *file, char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long page = get_zeroed_page(GFP_KERNEL); -+ char *p = (char *)page; -+ int len = 0, size = PAGE_SIZE; -+ struct mwl_ampdu_stream *stream; -+ int i; -+ struct mwl_sta *sta_info; -+ struct ieee80211_sta *sta; -+ ssize_t ret; -+ -+ if (!p) -+ return -ENOMEM; -+ -+ len += scnprintf(p + len, size - len, "\n"); -+ spin_lock_bh(&priv->stream_lock); -+ for (i = 0; i < priv->ampdu_num; i++) { -+ stream = &priv->ampdu[i]; -+ if (!stream->state) -+ continue; -+ len += scnprintf(p + len, size - len, "stream: %d\n", i); -+ len += scnprintf(p + len, size - len, "idx: %u\n", stream->idx); -+ len += scnprintf(p + len, size - len, -+ "state: %u\n", stream->state); -+ if (stream->sta) { -+ len += scnprintf(p + len, size - len, -+ "mac address: %pM\n", -+ stream->sta->addr); -+ len += scnprintf(p + len, size - len, -+ "tid: %u\n", stream->tid); -+ } -+ } -+ spin_unlock_bh(&priv->stream_lock); -+ spin_lock_bh(&priv->sta_lock); -+ list_for_each_entry(sta_info, &priv->sta_list, list) { -+ for (i = 0; i < MWL_MAX_TID; i++) { -+ if (sta_info->check_ba_failed[i]) { -+ sta = container_of((void *)sta_info, -+ struct ieee80211_sta, -+ drv_priv); -+ len += scnprintf(p + len, size - len, -+ "%pM(%d): %d\n", -+ sta->addr, i, -+ sta_info->check_ba_failed[i]); -+ } -+ } -+ } -+ spin_unlock_bh(&priv->sta_lock); -+ len += scnprintf(p + len, size - len, "\n"); -+ -+ ret = simple_read_from_buffer(ubuf, count, ppos, p, len); -+ free_page(page); -+ -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_stnid_read(struct file *file, char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long page = get_zeroed_page(GFP_KERNEL); -+ char *p = (char *)page; -+ int len = 0, size = PAGE_SIZE; -+ struct mwl_stnid *stnid; -+ int i; -+ ssize_t ret; -+ -+ if (!p) -+ return -ENOMEM; -+ -+ len += scnprintf(p + len, size - len, "\n"); -+ spin_lock_bh(&priv->stnid_lock); -+ for (i = 0; i < priv->stnid_num; i++) { -+ stnid = &priv->stnid[i]; -+ if (!stnid->aid) -+ continue; -+ len += scnprintf(p + len, size - len, -+ "stnid: %d macid: %d aid: %d\n", -+ i + 1, stnid->macid, stnid->aid); -+ } -+ spin_unlock_bh(&priv->stnid_lock); -+ len += scnprintf(p + len, size - len, "\n"); -+ -+ ret = simple_read_from_buffer(ubuf, count, ppos, p, len); -+ free_page(page); -+ -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_device_pwrtbl_read(struct file *file, -+ char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long page = get_zeroed_page(GFP_KERNEL); -+ char *p = (char *)page; -+ int len = 0, size = PAGE_SIZE; -+ int i, j; -+ ssize_t ret; -+ -+ if (!p) -+ return -ENOMEM; -+ -+ len += scnprintf(p + len, size - len, "\n"); -+ len += scnprintf(p + len, size - len, -+ "power table loaded from dts: %s\n", -+ priv->forbidden_setting ? "no" : "yes"); -+ len += scnprintf(p + len, size - len, "firmware region code: 0x%x\n", -+ priv->fw_region_code); -+ len += scnprintf(p + len, size - len, "number of channel: %d\n", -+ priv->number_of_channels); -+ for (i = 0; i < priv->number_of_channels; i++) { -+ len += scnprintf(p + len, size - len, "%3d ", -+ priv->device_pwr_tbl[i].channel); -+ for (j = 0; j < SYSADPT_TX_POWER_LEVEL_TOTAL; j++) -+ len += scnprintf(p + len, size - len, "%3d ", -+ priv->device_pwr_tbl[i].tx_pwr[j]); -+ len += scnprintf(p + len, size - len, "%3d ", -+ priv->device_pwr_tbl[i].dfs_capable); -+ len += scnprintf(p + len, size - len, "%3d ", -+ priv->device_pwr_tbl[i].ax_ant); -+ len += scnprintf(p + len, size - len, "%3d\n", -+ priv->device_pwr_tbl[i].cdd); -+ } -+ len += scnprintf(p + len, size - len, "\n"); -+ -+ ret = simple_read_from_buffer(ubuf, count, ppos, p, len); -+ free_page(page); -+ -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_txpwrlmt_read(struct file *file, -+ char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ -+ return simple_read_from_buffer(ubuf, count, ppos, -+ priv->txpwrlmt_data.buf, -+ priv->txpwrlmt_data.len); -+} -+ -+static ssize_t mwl_debugfs_tx_amsdu_read(struct file *file, -+ char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long page = get_zeroed_page(GFP_KERNEL); -+ char *p = (char *)page; -+ int len = 0, size = PAGE_SIZE; -+ ssize_t ret; -+ -+ if (!p) -+ return -ENOMEM; -+ -+ len += scnprintf(p + len, size - len, "\n"); -+ len += scnprintf(p + len, size - len, "tx amsdu: %s\n", -+ priv->tx_amsdu ? "enable" : "disable"); -+ len += scnprintf(p + len, size - len, "\n"); -+ -+ ret = simple_read_from_buffer(ubuf, count, ppos, p, len); -+ free_page(page); -+ -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_tx_amsdu_write(struct file *file, -+ const char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long addr = get_zeroed_page(GFP_KERNEL); -+ char *buf = (char *)addr; -+ size_t buf_size = min_t(size_t, count, PAGE_SIZE - 1); -+ int value; -+ ssize_t ret; -+ -+ if (!buf) -+ return -ENOMEM; -+ -+ if (copy_from_user(buf, ubuf, buf_size)) { -+ ret = -EFAULT; -+ goto err; -+ } -+ -+ if (kstrtoint(buf, 0, &value)) { -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ priv->tx_amsdu = value ? true : false; -+ -+ ret = count; -+ -+err: -+ free_page(addr); -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_dump_hostcmd_read(struct file *file, -+ char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long page = get_zeroed_page(GFP_KERNEL); -+ char *p = (char *)page; -+ int len = 0, size = PAGE_SIZE; -+ ssize_t ret; -+ -+ if (!p) -+ return -ENOMEM; -+ -+ len += scnprintf(p + len, size - len, "\n"); -+ len += scnprintf(p + len, size - len, "dump_hostcmd: %s\n", -+ priv->dump_hostcmd ? "enable" : "disable"); -+ len += scnprintf(p + len, size - len, "\n"); -+ -+ ret = simple_read_from_buffer(ubuf, count, ppos, p, len); -+ free_page(page); -+ -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_dump_hostcmd_write(struct file *file, -+ const char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long addr = get_zeroed_page(GFP_KERNEL); -+ char *buf = (char *)addr; -+ size_t buf_size = min_t(size_t, count, PAGE_SIZE - 1); -+ int value; -+ ssize_t ret; -+ -+ if (!buf) -+ return -ENOMEM; -+ -+ if (copy_from_user(buf, ubuf, buf_size)) { -+ ret = -EFAULT; -+ goto err; -+ } -+ -+ if (kstrtoint(buf, 0, &value)) { -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ priv->dump_hostcmd = value ? true : false; -+ -+ ret = count; -+ -+err: -+ free_page(addr); -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_dump_probe_read(struct file *file, -+ char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long page = get_zeroed_page(GFP_KERNEL); -+ char *p = (char *)page; -+ int len = 0, size = PAGE_SIZE; -+ ssize_t ret; -+ -+ if (!p) -+ return -ENOMEM; -+ -+ len += scnprintf(p + len, size - len, "\n"); -+ len += scnprintf(p + len, size - len, "dump_probe: %s\n", -+ priv->dump_probe ? "enable" : "disable"); -+ len += scnprintf(p + len, size - len, "\n"); -+ -+ ret = simple_read_from_buffer(ubuf, count, ppos, p, len); -+ free_page(page); -+ -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_dump_probe_write(struct file *file, -+ const char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long addr = get_zeroed_page(GFP_KERNEL); -+ char *buf = (char *)addr; -+ size_t buf_size = min_t(size_t, count, PAGE_SIZE - 1); -+ int value; -+ ssize_t ret; -+ -+ if (!buf) -+ return -ENOMEM; -+ -+ if (copy_from_user(buf, ubuf, buf_size)) { -+ ret = -EFAULT; -+ goto err; -+ } -+ -+ if (kstrtoint(buf, 0, &value)) { -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ priv->dump_probe = value ? true : false; -+ -+ ret = count; -+ -+err: -+ free_page(addr); -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_heartbeat_read(struct file *file, -+ char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long page = get_zeroed_page(GFP_KERNEL); -+ char *p = (char *)page; -+ int len = 0, size = PAGE_SIZE; -+ ssize_t ret; -+ -+ if (!p) -+ return -ENOMEM; -+ -+ len += scnprintf(p + len, size - len, "\n"); -+ len += scnprintf(p + len, size - len, "heartbeat: %d\n", -+ priv->heartbeat); -+ len += scnprintf(p + len, size - len, "\n"); -+ -+ ret = simple_read_from_buffer(ubuf, count, ppos, p, len); -+ free_page(page); -+ -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_heartbeat_write(struct file *file, -+ const char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long addr = get_zeroed_page(GFP_KERNEL); -+ char *buf = (char *)addr; -+ size_t buf_size = min_t(size_t, count, PAGE_SIZE - 1); -+ ssize_t ret; -+ -+ if (!buf) -+ return -ENOMEM; -+ -+ if (copy_from_user(buf, ubuf, buf_size)) { -+ ret = -EFAULT; -+ goto err; -+ } -+ -+ if (kstrtoint(buf, 0, &priv->heartbeat)) { -+ ret = -EINVAL; -+ goto err; -+ } -+ priv->pre_jiffies = jiffies; -+ -+ ret = count; -+ -+err: -+ free_page(addr); -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_dfs_test_read(struct file *file, -+ char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long page = get_zeroed_page(GFP_KERNEL); -+ char *p = (char *)page; -+ int len = 0, size = PAGE_SIZE; -+ ssize_t ret; -+ -+ if (!p) -+ return -ENOMEM; -+ -+ len += scnprintf(p + len, size - len, "\n"); -+ len += scnprintf(p + len, size - len, "dfs_test: %s\n", -+ priv->dfs_test ? "enable" : "disable"); -+ len += scnprintf(p + len, size - len, "\n"); -+ -+ ret = simple_read_from_buffer(ubuf, count, ppos, p, len); -+ free_page(page); -+ -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_dfs_test_write(struct file *file, -+ const char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long addr = get_zeroed_page(GFP_KERNEL); -+ char *buf = (char *)addr; -+ size_t buf_size = min_t(size_t, count, PAGE_SIZE - 1); -+ int value; -+ ssize_t ret; -+ -+ if (!buf) -+ return -ENOMEM; -+ -+ if (copy_from_user(buf, ubuf, buf_size)) { -+ ret = -EFAULT; -+ goto err; -+ } -+ -+ if (kstrtoint(buf, 0, &value)) { -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ priv->dfs_test = value ? true : false; -+ -+ ret = count; -+ -+err: -+ free_page(addr); -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_dfs_channel_read(struct file *file, -+ char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long page = get_zeroed_page(GFP_KERNEL); -+ char *p = (char *)page; -+ int len = 0, size = PAGE_SIZE; -+ struct ieee80211_supported_band *sband; -+ struct ieee80211_channel *channel; -+ int i; -+ ssize_t ret; -+ -+ if (!p) -+ return -ENOMEM; -+ -+ sband = priv->hw->wiphy->bands[NL80211_BAND_5GHZ]; -+ if (!sband) { -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ len += scnprintf(p + len, size - len, "\n"); -+ for (i = 0; i < sband->n_channels; i++) { -+ channel = &sband->channels[i]; -+ if (channel->flags & IEEE80211_CHAN_RADAR) { -+ len += scnprintf(p + len, size - len, -+ "%d(%d): flags: %08x dfs_state: %d\n", -+ channel->hw_value, -+ channel->center_freq, -+ channel->flags, channel->dfs_state); -+ len += scnprintf(p + len, size - len, -+ "cac timer: %d ms\n", -+ channel->dfs_cac_ms); -+ } -+ } -+ len += scnprintf(p + len, size - len, "\n"); -+ -+ ret = simple_read_from_buffer(ubuf, count, ppos, p, len); -+ -+err: -+ free_page(page); -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_dfs_channel_write(struct file *file, -+ const char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ struct ieee80211_supported_band *sband; -+ unsigned long addr = get_zeroed_page(GFP_KERNEL); -+ char *buf = (char *)addr; -+ size_t buf_size = min_t(size_t, count, PAGE_SIZE - 1); -+ int dfs_state = 0; -+ int cac_time = -1; -+ struct ieee80211_channel *channel; -+ int i; -+ ssize_t ret; -+ -+ if (!buf) -+ return -ENOMEM; -+ -+ sband = priv->hw->wiphy->bands[NL80211_BAND_5GHZ]; -+ if (!sband) { -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ if (copy_from_user(buf, ubuf, buf_size)) { -+ ret = -EFAULT; -+ goto err; -+ } -+ -+ ret = sscanf(buf, "%d %d", &dfs_state, &cac_time); -+ -+ if ((ret < 1) || (ret > 2)) { -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ for (i = 0; i < sband->n_channels; i++) { -+ channel = &sband->channels[i]; -+ if (channel->flags & IEEE80211_CHAN_RADAR) { -+ channel->dfs_state = dfs_state; -+ if (cac_time != -1) -+ channel->dfs_cac_ms = cac_time * 1000; -+ } -+ } -+ ret = count; -+ -+err: -+ free_page(addr); -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_dfs_radar_read(struct file *file, char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long page = get_zeroed_page(GFP_KERNEL); -+ char *p = (char *)page; -+ int len = 0, size = PAGE_SIZE; -+ ssize_t ret; -+ -+ if (!p) -+ return -ENOMEM; -+ -+ len += scnprintf(p + len, size - len, "\n"); -+ len += scnprintf(p + len, size - len, -+ "csa_active: %d\n", priv->csa_active); -+ len += scnprintf(p + len, size - len, -+ "dfs_region: %d\n", priv->dfs_region); -+ len += scnprintf(p + len, size - len, -+ "chirp_count_min: %d\n", priv->dfs_chirp_count_min); -+ len += scnprintf(p + len, size - len, "chirp_time_interval: %d\n", -+ priv->dfs_chirp_time_interval); -+ len += scnprintf(p + len, size - len, -+ "pw_filter: %d\n", priv->dfs_pw_filter); -+ len += scnprintf(p + len, size - len, -+ "min_num_radar: %d\n", priv->dfs_min_num_radar); -+ len += scnprintf(p + len, size - len, -+ "min_pri_count: %d\n", priv->dfs_min_pri_count); -+ len += scnprintf(p + len, size - len, "\n"); -+ -+ ret = simple_read_from_buffer(ubuf, count, ppos, p, len); -+ free_page(page); -+ -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_dfs_radar_write(struct file *file, -+ const char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ -+ wiphy_info(priv->hw->wiphy, "simulate radar detected\n"); -+ ieee80211_radar_detected(priv->hw); -+ -+ return count; -+} -+ -+static ssize_t mwl_debugfs_thermal_read(struct file *file, -+ char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long page = get_zeroed_page(GFP_KERNEL); -+ char *p = (char *)page; -+ int len = 0, size = PAGE_SIZE; -+ ssize_t ret; -+ -+ if (!p) -+ return -ENOMEM; -+ -+ mwl_fwcmd_get_temp(priv->hw, &priv->temperature); -+ -+ len += scnprintf(p + len, size - len, "\n"); -+ len += scnprintf(p + len, size - len, "quiet period: %d\n", -+ priv->quiet_period); -+ len += scnprintf(p + len, size - len, "throttle state: %d\n", -+ priv->throttle_state); -+ len += scnprintf(p + len, size - len, "temperature: %d\n", -+ priv->temperature); -+ len += scnprintf(p + len, size - len, "\n"); -+ -+ ret = simple_read_from_buffer(ubuf, count, ppos, p, len); -+ free_page(page); -+ -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_thermal_write(struct file *file, -+ const char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long addr = get_zeroed_page(GFP_KERNEL); -+ char *buf = (char *)addr; -+ size_t buf_size = min_t(size_t, count, PAGE_SIZE - 1); -+ int throttle_state; -+ ssize_t ret; -+ -+ if (!buf) -+ return -ENOMEM; -+ -+ if (copy_from_user(buf, ubuf, buf_size)) { -+ ret = -EFAULT; -+ goto err; -+ } -+ -+ if (kstrtoint(buf, 0, &throttle_state)) { -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ if (throttle_state > SYSADPT_THERMAL_THROTTLE_MAX) { -+ wiphy_warn(priv->hw->wiphy, -+ "throttle state %d is exceeding the limit %d\n", -+ throttle_state, SYSADPT_THERMAL_THROTTLE_MAX); -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ priv->throttle_state = throttle_state; -+ mwl_thermal_set_throttling(priv); -+ ret = count; -+ -+err: -+ free_page(addr); -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_led_ctrl_read(struct file *file, -+ char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long page = get_zeroed_page(GFP_KERNEL); -+ char *p = (char *)page; -+ int len = 0, size = PAGE_SIZE; -+ ssize_t ret; -+ -+ if (!p) -+ return -ENOMEM; -+ -+ len += scnprintf(p + len, size - len, "\n"); -+ len += scnprintf(p + len, size - len, "led blink %s\n", -+ priv->led_blink_enable ? "enable" : "disable"); -+ len += scnprintf(p + len, size - len, "led blink rate: %d\n", -+ priv->led_blink_rate); -+ len += scnprintf(p + len, size - len, "\n"); -+ -+ ret = simple_read_from_buffer(ubuf, count, ppos, p, len); -+ free_page(page); -+ -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_led_ctrl_write(struct file *file, -+ const char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long addr = get_zeroed_page(GFP_KERNEL); -+ char *buf = (char *)addr; -+ size_t buf_size = min_t(size_t, count, PAGE_SIZE - 1); -+ int enable, rate; -+ ssize_t ret; -+ -+ if (!buf) -+ return -ENOMEM; -+ -+ if (copy_from_user(buf, ubuf, buf_size)) { -+ ret = -EFAULT; -+ goto err; -+ } -+ -+ ret = sscanf(buf, "%x %x", &enable, &rate); -+ -+ if ((ret != 1) && (ret != 2)) { -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ if (enable && (ret != 2)) { -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ ret = mwl_fwcmd_led_ctrl(priv->hw, enable, rate); -+ -+ if (ret) -+ goto err; -+ -+ priv->led_blink_enable = enable; -+ if (enable) -+ priv->led_blink_rate = rate; -+ else -+ priv->led_blink_rate = 0; -+ -+ ret = count; -+ -+err: -+ free_page(addr); -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_regrdwr_read(struct file *file, char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long page = get_zeroed_page(GFP_KERNEL); -+ char *p = (char *)page; -+ int len = 0, size = PAGE_SIZE; -+ ssize_t ret; -+ -+ if (*ppos) -+ return len; -+ -+ if (!p) -+ return -ENOMEM; -+ -+ if (!priv->reg_type) { -+ /* No command has been given */ -+ len += scnprintf(p + len, size - len, "0"); -+ ret = -EINVAL; -+ goto none; -+ } -+ -+ /* Set command has been given */ -+ if (priv->reg_value != UINT_MAX) { -+ ret = mwl_hif_reg_access(priv->hw, true); -+ goto done; -+ } -+ /* Get command has been given */ -+ ret = mwl_hif_reg_access(priv->hw, false); -+ -+done: -+ if (!ret) -+ len += scnprintf(p + len, size - len, "%u 0x%08x 0x%08x\n", -+ priv->reg_type, priv->reg_offset, -+ priv->reg_value); -+ else -+ len += scnprintf(p + len, size - len, -+ "error: %d(%u 0x%08x 0x%08x)\n", -+ ret, priv->reg_type, priv->reg_offset, -+ priv->reg_value); -+ -+ ret = simple_read_from_buffer(ubuf, count, ppos, p, len); -+ -+none: -+ free_page(page); -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_regrdwr_write(struct file *file, -+ const char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long addr = get_zeroed_page(GFP_KERNEL); -+ char *buf = (char *)addr; -+ size_t buf_size = min_t(size_t, count, PAGE_SIZE - 1); -+ ssize_t ret; -+ u32 reg_type = 0, reg_offset = 0, reg_value = UINT_MAX; -+ -+ if (!buf) -+ return -ENOMEM; -+ -+ if (copy_from_user(buf, ubuf, buf_size)) { -+ ret = -EFAULT; -+ goto err; -+ } -+ -+ ret = sscanf(buf, "%u %x %x", ®_type, ®_offset, ®_value); -+ -+ if (!reg_type) { -+ ret = -EINVAL; -+ goto err; -+ } else { -+ priv->reg_type = reg_type; -+ priv->reg_offset = reg_offset; -+ priv->reg_value = reg_value; -+ ret = count; -+ } -+ -+err: -+ free_page(addr); -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_ratetable_read(struct file *file, char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long page = get_zeroed_page(GFP_KERNEL); -+ char *p = (char *)page; -+ int len = 0, size = PAGE_SIZE; -+ struct mwl_sta *sta_info; -+ struct ieee80211_sta *sta; -+ u8 addr[ETH_ALEN]; -+ int table_size = (sizeof(__le32) * 2 * SYSADPT_MAX_RATE_ADAPT_RATES); -+ u8 *rate_table, *rate_idx; -+ u32 rate_info; -+ u8 fmt, stbc, bw, sgi, mcs, preamble_gf, power_id, ldpc, bf, ant; -+ int idx, rate, nss; -+ ssize_t ret; -+ -+ if (!p) -+ return -ENOMEM; -+ -+ if (!priv->ra_aid) { -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ spin_lock_bh(&priv->sta_lock); -+ list_for_each_entry(sta_info, &priv->sta_list, list) { -+ sta = container_of((void *)sta_info, struct ieee80211_sta, -+ drv_priv); -+ if (priv->ra_aid == sta->aid) { -+ ether_addr_copy(addr, sta->addr); -+ break; -+ } -+ } -+ spin_unlock_bh(&priv->sta_lock); -+ -+ rate_table = kzalloc(size, GFP_KERNEL); -+ if (!rate_table) { -+ ret = -ENOMEM; -+ goto err; -+ } -+ -+ ret = mwl_fwcmd_get_ratetable(priv->hw, addr, rate_table, -+ table_size, 0); -+ if (ret) { -+ kfree(rate_table); -+ goto err; -+ } -+ -+ len += scnprintf(p + len, size - len, "\n"); -+ len += scnprintf(p + len, size - len, -+ "%3s %6s %5s %5s %5s %5s %5s %4s %2s %5s %4s %5s %5s\n", -+ "Num", "Fmt", "STBC", "BW", "SGI", "Nss", "RateId", -+ "GF/Pre", "PId", "LDPC", "BF", "TxAnt", "Rate"); -+ idx = 0; -+ rate_idx = rate_table; -+ rate_info = le32_to_cpu(*(__le32 *)rate_idx); -+ while (rate_info) { -+ fmt = rate_info & MWL_TX_RATE_FORMAT_MASK; -+ stbc = (rate_info & MWL_TX_RATE_STBC_MASK) >> -+ MWL_TX_RATE_STBC_SHIFT; -+ bw = (rate_info & MWL_TX_RATE_BANDWIDTH_MASK) >> -+ MWL_TX_RATE_BANDWIDTH_SHIFT; -+ sgi = (rate_info & MWL_TX_RATE_SHORTGI_MASK) >> -+ MWL_TX_RATE_SHORTGI_SHIFT; -+ mcs = (rate_info & MWL_TX_RATE_RATEIDMCS_MASK) >> -+ MWL_TX_RATE_RATEIDMCS_SHIFT; -+ preamble_gf = (rate_info & MWL_TX_RATE_PREAMBLE_MASK) >> -+ MWL_TX_RATE_PREAMBLE_SHIFT; -+ power_id = (rate_info & MWL_TX_RATE_POWERID_MASK) >> -+ MWL_TX_RATE_POWERID_SHIFT; -+ ldpc = (rate_info & MWL_TX_RATE_ADVCODING_MASK) >> -+ MWL_TX_RATE_ADVCODING_SHIFT; -+ bf = (rate_info & MWL_TX_RATE_BF_MASK) >> -+ MWL_TX_RATE_BF_SHIFT; -+ ant = (rate_info & MWL_TX_RATE_ANTSELECT_MASK) >> -+ MWL_TX_RATE_ANTSELECT_SHIFT; -+ -+ if (fmt == TX_RATE_FORMAT_11AC) { -+ rate = mcs & 0xf; /* 11ac, mcs[3:0]: rate */ -+ nss = mcs >> 4; /* 11ac, mcs[6:4] = nss code */ -+ nss++; /* ddd 1 to correct Nss representation */ -+ } else { -+ rate = mcs; -+ nss = 0; -+ if (fmt == TX_RATE_FORMAT_11N) { -+ if ((mcs >= 0) && (mcs < 8)) -+ nss = 1; -+ else if ((mcs >= 8) && (mcs < 16)) -+ nss = 2; -+ else if ((mcs >= 16) && (mcs < 24)) -+ nss = 3; -+ } -+ } -+ -+ len += scnprintf(p + len, size - len, -+ "%3d %5d %5d %5d %5d %5d %5d %5d %5d %5d %5d %5d %5d\n", -+ idx, (int)fmt, (int)stbc, (int)bw, (int)sgi, nss, rate, -+ (int)preamble_gf, (int)power_id, (int)ldpc, (int)bf, -+ (int)ant, -+ utils_get_phy_rate(fmt, bw, sgi, mcs)); -+ -+ idx++; -+ rate_idx += (2 * sizeof(__le32)); -+ rate_info = le32_to_cpu(*(__le32 *)rate_idx); -+ } -+ len += scnprintf(p + len, size - len, "\n"); -+ -+ kfree(rate_table); -+ ret = simple_read_from_buffer(ubuf, count, ppos, p, len); -+ -+err: -+ free_page(page); -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_ratetable_write(struct file *file, -+ const char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long addr = get_zeroed_page(GFP_KERNEL); -+ char *buf = (char *)addr; -+ size_t buf_size = min_t(size_t, count, PAGE_SIZE - 1); -+ int sta_aid; -+ ssize_t ret; -+ -+ if (!buf) -+ return -ENOMEM; -+ -+ if (copy_from_user(buf, ubuf, buf_size)) { -+ ret = -EFAULT; -+ goto err; -+ } -+ -+ if (kstrtoint(buf, 0, &sta_aid)) { -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ if ((sta_aid <= 0) || (sta_aid > SYSADPT_MAX_STA_SC4)) { -+ wiphy_warn(priv->hw->wiphy, -+ "station aid is exceeding the limit %d\n", sta_aid); -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ priv->ra_aid = sta_aid; -+ ret = count; -+ -+err: -+ free_page(addr); -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_tx_hist_read(struct file *file, char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long page = get_zeroed_page(GFP_KERNEL); -+ char *p = (char *)page; -+ int len = 0, size = PAGE_SIZE; -+ struct ieee80211_sta *sta; -+ struct mwl_sta *sta_info; -+ ssize_t ret; -+ -+ if (priv->chip_type != MWL8964) -+ return -EPERM; -+ -+ if (!p) -+ return -ENOMEM; -+ -+ len += scnprintf(p + len, size - len, "\n"); -+ len += scnprintf(p + len, size - len, -+ "SU: <4:%d >=4:%d >=15:%d >=50:%d >=100:%d >=250:%d\n", -+ priv->ra_tx_attempt[SU_MIMO][0], -+ priv->ra_tx_attempt[SU_MIMO][1], -+ priv->ra_tx_attempt[SU_MIMO][2], -+ priv->ra_tx_attempt[SU_MIMO][3], -+ priv->ra_tx_attempt[SU_MIMO][4], -+ priv->ra_tx_attempt[SU_MIMO][5]); -+ len += scnprintf(p + len, size - len, -+ "MU: <4:%d >=4:%d >=15:%d >=50:%d >=100:%d >=250:%d\n", -+ priv->ra_tx_attempt[MU_MIMO][0], -+ priv->ra_tx_attempt[MU_MIMO][1], -+ priv->ra_tx_attempt[MU_MIMO][2], -+ priv->ra_tx_attempt[MU_MIMO][3], -+ priv->ra_tx_attempt[MU_MIMO][4], -+ priv->ra_tx_attempt[MU_MIMO][5]); -+ spin_lock_bh(&priv->sta_lock); -+ list_for_each_entry(sta_info, &priv->sta_list, list) { -+ sta = container_of((void *)sta_info, struct ieee80211_sta, -+ drv_priv); -+ len += scnprintf(p + len, size - len, "\nSTA %pM\n", sta->addr); -+ len += scnprintf(p + len, size - len, -+ "============================\n"); -+ dump_tx_hist(p, size, &len, sta_info); -+ len += scnprintf(p + len, size - len, -+ "============================\n"); -+ } -+ spin_unlock_bh(&priv->sta_lock); -+ -+ ret = simple_read_from_buffer(ubuf, count, ppos, p, len); -+ free_page(page); -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_tx_hist_write(struct file *file, -+ const char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long addr = get_zeroed_page(GFP_KERNEL); -+ char *buf = (char *)addr; -+ size_t buf_size = min_t(size_t, count, PAGE_SIZE - 1); -+ int reset; -+ struct mwl_sta *sta_info; -+ ssize_t ret; -+ -+ if (!buf) -+ return -ENOMEM; -+ -+ if (copy_from_user(buf, ubuf, buf_size)) { -+ ret = -EFAULT; -+ goto err; -+ } -+ -+ if (kstrtoint(buf, 0, &reset)) { -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ if (!reset) { -+ memset(&priv->ra_tx_attempt, 0, 2 * 6 * sizeof(u32)); -+ spin_lock_bh(&priv->sta_lock); -+ list_for_each_entry(sta_info, &priv->sta_list, list) { -+ memset(&sta_info->tx_hist, 0, -+ sizeof(sta_info->tx_hist)); -+ } -+ spin_unlock_bh(&priv->sta_lock); -+ } -+ -+ ret = count; -+ -+err: -+ free_page(addr); -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_ba_hist_read(struct file *file, char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long page = get_zeroed_page(GFP_KERNEL); -+ char *p = (char *)page; -+ int len = 0, size = PAGE_SIZE; -+ struct mwl_sta *sta_info; -+ struct mwl_tx_ba_stats *ba_stats; -+ u32 i, data; -+ u32 baholecnt, baexpcnt, bmap0cnt, nobacnt; -+ u8 bmap0flag, nobaflag; -+ char buff[500], file_location[20]; -+ struct file *filp_bahisto; -+ u8 *data_p = buff; -+ ssize_t ret; -+ -+ if (!p) -+ return -ENOMEM; -+ -+ if (!priv->ba_aid) { -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ memset(buff, 0, sizeof(buff)); -+ memset(file_location, 0, sizeof(file_location)); -+ sprintf(file_location, "/tmp/ba_histo-%d", priv->ba_aid); -+ -+ filp_bahisto = filp_open(file_location, -+ O_RDWR | O_CREAT | O_TRUNC, 0); -+ -+ if (IS_ERR(filp_bahisto)) { -+ ret = -EIO; -+ goto err; -+ } -+ -+ sta_info = utils_find_sta_by_aid(priv, priv->ba_aid); -+ if (sta_info && sta_info->ba_hist.enable && -+ sta_info->ba_hist.ba_stats) { -+ ba_stats = sta_info->ba_hist.ba_stats; -+ len += scnprintf(p + len, size - len, -+ "BA histogram aid: %d, stnid: %d type: %s\n", -+ priv->ba_aid, sta_info->stnid, -+ sta_info->ba_hist.type ? "MU" : "SU"); -+ data_p += sprintf(data_p, -+ "BA histogram aid: %d, stnid: %d type: %s\n", -+ priv->ba_aid, sta_info->stnid, -+ sta_info->ba_hist.type ? "MU" : "SU"); -+ data_p += sprintf(data_p, "%8s,%8s,%8s,%8s\n", -+ "BAhole", "Expect", "Bmap0", "NoBA"); -+ data = *(u32 *)&ba_stats[0]; -+ baholecnt = 0; -+ baexpcnt = 0; -+ bmap0cnt = 0; -+ nobacnt = 0; -+ for (i = 0; i < ACNT_BA_SIZE && data; i++) { -+ data = *(u32 *)&ba_stats[i]; -+ if (data == 0) -+ break; -+ -+ /* If no BA event does not happen, check BA hole and BA -+ * expected to mark BA bitmap all 0 event -+ */ -+ if (!ba_stats[i].no_ba) -+ bmap0flag = (ba_stats[i].ba_hole == -+ ba_stats[i].ba_expected) ? 1 : 0; -+ else -+ bmap0flag = 0; -+ nobaflag = ba_stats[i].no_ba; -+ -+ /* Buffer is full. Write to file and reset buf */ -+ if ((strlen(buff) + 36) >= 500) { -+ __kernel_write(filp_bahisto, buff, strlen(buff), -+ &filp_bahisto->f_pos); -+ mdelay(2); -+ memset(buff, 0, sizeof(buff)); -+ data_p = buff; -+ } -+ -+ data_p += sprintf(data_p, "%8d,%8d,", -+ ba_stats[i].ba_hole, -+ ba_stats[i].ba_expected); -+ -+ baholecnt += ba_stats[i].ba_hole; -+ baexpcnt += ba_stats[i].ba_expected; -+ if (bmap0flag) { -+ data_p += sprintf(data_p, " #,"); -+ bmap0cnt++; -+ } else -+ data_p += sprintf(data_p, "%8d,", bmap0flag); -+ if (nobaflag) { -+ data_p += sprintf(data_p, " *\n"); -+ nobacnt++; -+ } else -+ data_p += sprintf(data_p, "%8d\n", nobaflag); -+ } -+ -+ __kernel_write(filp_bahisto, buff, strlen(buff), -+ &filp_bahisto->f_pos); -+ len += scnprintf(p + len, size - len, -+ "hole: %d, expect: %d, bmap0: %d, noba: %d\n", -+ baholecnt, baexpcnt, bmap0cnt, nobacnt); -+ len += scnprintf(p + len, size - len, -+ "BA histogram data written to %s\n", -+ file_location); -+ } else -+ len += scnprintf(p + len, size - len, -+ "No BA histogram for sta aid: %d\n", -+ priv->ba_aid); -+ -+ filp_close(filp_bahisto, current->files); -+ -+ ret = simple_read_from_buffer(ubuf, count, ppos, p, len); -+ -+err: -+ free_page(page); -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_ba_hist_write(struct file *file, -+ const char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long addr = get_zeroed_page(GFP_KERNEL); -+ char *buf = (char *)addr; -+ size_t buf_size = min_t(size_t, count, PAGE_SIZE - 1); -+ int sta_aid; -+ struct mwl_sta *sta_info; -+ int size; -+ ssize_t ret; -+ -+ if (!buf) -+ return -ENOMEM; -+ -+ if (copy_from_user(buf, ubuf, buf_size)) { -+ ret = -EFAULT; -+ goto err; -+ } -+ -+ if (kstrtoint(buf, 0, &sta_aid)) { -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ if ((sta_aid <= 0) || (sta_aid > SYSADPT_MAX_STA_SC4)) { -+ wiphy_warn(priv->hw->wiphy, -+ "station aid is exceeding the limit %d\n", sta_aid); -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ if (priv->ba_aid) { -+ sta_info = utils_find_sta_by_aid(priv, priv->ba_aid); -+ if (sta_info) { -+ sta_info->ba_hist.enable = false; -+ kfree(sta_info->ba_hist.ba_stats); -+ } -+ } -+ priv->ba_aid = 0; -+ sta_info = utils_find_sta_by_aid(priv, sta_aid); -+ if (sta_info) { -+ sta_info->ba_hist.enable = true; -+ sta_info->ba_hist.index = 0; -+ size = sizeof(struct mwl_tx_ba_stats) * ACNT_BA_SIZE; -+ sta_info->ba_hist.ba_stats = kmalloc(size, GFP_KERNEL); -+ if (sta_info->ba_hist.ba_stats) { -+ memset(sta_info->ba_hist.ba_stats, 0, size); -+ priv->ba_aid = sta_aid; -+ } -+ ret = count; -+ } else -+ ret = -EINVAL; -+ -+err: -+ free_page(addr); -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_fixed_rate_read(struct file *file, char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long page = get_zeroed_page(GFP_KERNEL); -+ char *p = (char *)page; -+ int len = 0, size = PAGE_SIZE; -+ ssize_t ret; -+ -+ if (!p) -+ return -ENOMEM; -+ -+ len += scnprintf(p + len, size - len, "\n"); -+ len += scnprintf(p + len, size - len, "fixed rate: 0x%08x\n", -+ priv->fixed_rate); -+ len += scnprintf(p + len, size - len, "\n"); -+ -+ ret = simple_read_from_buffer(ubuf, count, ppos, p, len); -+ free_page(page); -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_fixed_rate_write(struct file *file, -+ const char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long addr = get_zeroed_page(GFP_KERNEL); -+ char *buf = (char *)addr; -+ size_t buf_size = min_t(size_t, count, PAGE_SIZE - 1); -+ ssize_t ret; -+ int fixed_rate = 0, fwcmd_ret; -+ -+ if (!buf) -+ return -ENOMEM; -+ -+ if (copy_from_user(buf, ubuf, buf_size)) { -+ ret = -EFAULT; -+ goto err; -+ } -+ -+ ret = sscanf(buf, "%08x", &fixed_rate); -+ if (!ret) { -+ ret = -EIO; -+ goto err; -+ } -+ -+ priv->fixed_rate = fixed_rate; -+ -+ if (fixed_rate != 0) -+ fwcmd_ret = mwl_fwcmd_set_rate_drop(priv->hw, 3, -+ priv->fixed_rate, 0); -+ else -+ fwcmd_ret = mwl_fwcmd_set_rate_drop(priv->hw, 1, -+ priv->fixed_rate, 0); -+ if (fwcmd_ret) -+ ret = -EIO; -+ else -+ ret = count; -+ -+err: -+ free_page(addr); -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_core_dump_read(struct file *file, char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long page = get_zeroed_page(GFP_KERNEL); -+ char *p = (char *)page; -+ int len = 0, size = PAGE_SIZE; -+ struct coredump_cmd *core_dump = NULL; -+ struct coredump *cd = NULL; -+ char *buff = NULL; -+ u32 i, offset; -+ u32 address, length; -+ ssize_t ret; -+ -+ if (priv->chip_type != MWL8964) -+ return -EPERM; -+ -+ if (*ppos) -+ return len; -+ -+ if (!p) -+ return -ENOMEM; -+ -+ core_dump = kmalloc(sizeof(*core_dump), GFP_ATOMIC); -+ if (!core_dump) { -+ ret = -ENOMEM; -+ goto err; -+ } -+ -+ buff = kmalloc(MAX_CORE_DUMP_BUFFER, GFP_ATOMIC); -+ if (!buff) { -+ ret = -ENOMEM; -+ goto err; -+ } -+ memset((char *)buff, 0, MAX_CORE_DUMP_BUFFER); -+ -+ cd = kmalloc(sizeof(*cd), GFP_ATOMIC); -+ if (!cd) { -+ ret = -ENOMEM; -+ goto err; -+ } -+ -+ core_dump->context = 0; -+ core_dump->flags = 0; -+ core_dump->size_kb = 0; -+ if (mwl_fwcmd_get_fw_core_dump(priv->hw, core_dump, buff)) { -+ ret = -EIO; -+ goto err; -+ } -+ memcpy(cd, buff, sizeof(*cd)); -+ -+ len += scnprintf(p + len, size - len, "\n"); -+ len += scnprintf(p + len, size - len, "Major Version : %d\n", -+ cd->version_major); -+ len += scnprintf(p + len, size - len, "Minor Version : %d\n", -+ cd->version_minor); -+ len += scnprintf(p + len, size - len, "Patch Version : %d\n", -+ cd->version_patch); -+ len += scnprintf(p + len, size - len, "Num of Regions: %d\n", -+ cd->num_regions); -+ len += scnprintf(p + len, size - len, "Num of Symbols: %d\n", -+ cd->num_symbols); -+ -+ for (i = 0; i < cd->num_regions; i++) { -+ address = le32_to_cpu(cd->region[i].address); -+ length = le32_to_cpu(cd->region[i].length); -+ len += scnprintf(p + len, size - len, -+ "\ncd.region[%d]: address=%x, length=%x\n", -+ i, address, length); -+ -+ for (offset = 0; offset < length; -+ offset += MAX_CORE_DUMP_BUFFER) { -+ core_dump->context = cpu_to_le32((i << 28) | offset); -+ core_dump->flags = 0; -+ core_dump->size_kb = 0; -+ if (mwl_fwcmd_get_fw_core_dump(priv->hw, -+ core_dump, buff)) { -+ wiphy_info(priv->hw->wiphy, -+ "region:%d offset:%x\n", i, offset); -+ break; -+ } -+ core_dump_file(buff, MAX_CORE_DUMP_BUFFER, -+ address, address + offset, -+ offset, length, priv->coredump_text); -+ } -+ } -+ len += scnprintf(p + len, size - len, "\n"); -+ -+ ret = simple_read_from_buffer(ubuf, count, ppos, p, len); -+ -+err: -+ kfree(core_dump); -+ kfree(buff); -+ kfree(cd); -+ free_page(page); -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_core_dump_write(struct file *file, -+ const char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long addr = get_zeroed_page(GFP_KERNEL); -+ char *buf = (char *)addr; -+ size_t buf_size = min_t(size_t, count, PAGE_SIZE - 1); -+ int text_mode; -+ ssize_t ret; -+ -+ if (priv->chip_type != MWL8964) -+ return -EPERM; -+ -+ if (!buf) -+ return -ENOMEM; -+ -+ if (copy_from_user(buf, ubuf, buf_size)) { -+ ret = -EFAULT; -+ goto err; -+ } -+ -+ if (kstrtoint(buf, 0, &text_mode)) { -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ if ((text_mode < 0) || (text_mode > 1)) { -+ wiphy_warn(priv->hw->wiphy, -+ "text mode should be 0 (false) or 1 (true): %d\n", -+ text_mode); -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ mwl_fwcmd_core_dump_diag_mode(priv->hw, 1); -+ priv->coredump_text = text_mode ? true : false; -+ ret = count; -+ -+err: -+ free_page(addr); -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_mcast_cts_write(struct file *file, -+ const char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long addr = get_zeroed_page(GFP_KERNEL); -+ char *buf = (char *)addr; -+ size_t buf_size = min_t(size_t, count, PAGE_SIZE - 1); -+ int cts_enable = 0; -+ ssize_t ret; -+ -+ if (!buf) -+ return -ENOMEM; -+ -+ if (copy_from_user(buf, ubuf, buf_size)) { -+ ret = -EFAULT; -+ goto err; -+ } -+ -+ if (kstrtoint(buf, 0, &cts_enable)) { -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ ret = mwl_hif_mcast_cts(priv->hw, cts_enable ? true : false); -+ if (ret) -+ goto err; -+ -+ ret = count; -+ -+err: -+ free_page(addr); -+ return ret; -+} -+ -+static ssize_t mwl_debugfs_wmmedcaap_write(struct file *file, -+ const char __user *ubuf, -+ size_t count, loff_t *ppos) -+{ -+ struct mwl_priv *priv = (struct mwl_priv *)file->private_data; -+ unsigned long addr = get_zeroed_page(GFP_KERNEL); -+ char *buf = (char *)addr; -+ size_t buf_size = min_t(size_t, count, PAGE_SIZE - 1); -+ u32 index = 0, cw_min = 0, cw_max = 0, aifsn = 0, txop = 0; -+ ssize_t ret; -+ -+ if (!buf) -+ return -ENOMEM; -+ -+ if (copy_from_user(buf, ubuf, buf_size)) { -+ ret = -EFAULT; -+ goto err; -+ } -+ -+ ret = sscanf(buf, "%u %x %x %u %x", &index, &cw_min, -+ &cw_max, &aifsn, &txop); -+ if (ret != 5) { -+ ret = -EINVAL; -+ goto err; -+ } -+ wiphy_info(priv->hw->wiphy, "set TCQ%d wmm edca with:\n", index); -+ wiphy_info(priv->hw->wiphy, -+ "cw_min=0x%x, cw_max=0x%x, aifs_num=%d, txop=0x%x\n", -+ cw_min, cw_max, aifsn, txop); -+ -+ ret = mwl_fwcmd_set_edca_params(priv->hw, index, -+ cw_min, cw_max, aifsn, txop); -+ if (ret) -+ goto err; -+ -+ ret = count; -+ -+err: -+ free_page(addr); -+ return ret; -+} -+ -+MWLWIFI_DEBUGFS_FILE_READ_OPS(info); -+MWLWIFI_DEBUGFS_FILE_READ_OPS(tx_status); -+MWLWIFI_DEBUGFS_FILE_READ_OPS(rx_status); -+MWLWIFI_DEBUGFS_FILE_READ_OPS(vif); -+MWLWIFI_DEBUGFS_FILE_READ_OPS(sta); -+MWLWIFI_DEBUGFS_FILE_READ_OPS(ampdu); -+MWLWIFI_DEBUGFS_FILE_READ_OPS(stnid); -+MWLWIFI_DEBUGFS_FILE_READ_OPS(device_pwrtbl); -+MWLWIFI_DEBUGFS_FILE_READ_OPS(txpwrlmt); -+MWLWIFI_DEBUGFS_FILE_OPS(tx_amsdu); -+MWLWIFI_DEBUGFS_FILE_OPS(dump_hostcmd); -+MWLWIFI_DEBUGFS_FILE_OPS(dump_probe); -+MWLWIFI_DEBUGFS_FILE_OPS(heartbeat); -+MWLWIFI_DEBUGFS_FILE_OPS(dfs_test); -+MWLWIFI_DEBUGFS_FILE_OPS(dfs_channel); -+MWLWIFI_DEBUGFS_FILE_OPS(dfs_radar); -+MWLWIFI_DEBUGFS_FILE_OPS(thermal); -+MWLWIFI_DEBUGFS_FILE_OPS(led_ctrl); -+MWLWIFI_DEBUGFS_FILE_OPS(regrdwr); -+MWLWIFI_DEBUGFS_FILE_OPS(ratetable); -+MWLWIFI_DEBUGFS_FILE_OPS(tx_hist); -+MWLWIFI_DEBUGFS_FILE_OPS(ba_hist); -+MWLWIFI_DEBUGFS_FILE_OPS(fixed_rate); -+MWLWIFI_DEBUGFS_FILE_OPS(core_dump); -+MWLWIFI_DEBUGFS_FILE_WRITE_OPS(mcast_cts); -+MWLWIFI_DEBUGFS_FILE_WRITE_OPS(wmmedcaap); -+ -+void mwl_debugfs_init(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (!priv->debugfs_phy) -+ priv->debugfs_phy = debugfs_create_dir("mwlwifi", -+ hw->wiphy->debugfsdir); -+ -+ if (!priv->debugfs_phy) -+ return; -+ -+ MWLWIFI_DEBUGFS_ADD_FILE(info); -+ MWLWIFI_DEBUGFS_ADD_FILE(tx_status); -+ MWLWIFI_DEBUGFS_ADD_FILE(rx_status); -+ MWLWIFI_DEBUGFS_ADD_FILE(vif); -+ MWLWIFI_DEBUGFS_ADD_FILE(sta); -+ MWLWIFI_DEBUGFS_ADD_FILE(ampdu); -+ MWLWIFI_DEBUGFS_ADD_FILE(stnid); -+ MWLWIFI_DEBUGFS_ADD_FILE(device_pwrtbl); -+ MWLWIFI_DEBUGFS_ADD_FILE(txpwrlmt); -+ MWLWIFI_DEBUGFS_ADD_FILE(tx_amsdu); -+ MWLWIFI_DEBUGFS_ADD_FILE(dump_hostcmd); -+ MWLWIFI_DEBUGFS_ADD_FILE(dump_probe); -+ MWLWIFI_DEBUGFS_ADD_FILE(heartbeat); -+ MWLWIFI_DEBUGFS_ADD_FILE(dfs_test); -+ MWLWIFI_DEBUGFS_ADD_FILE(dfs_channel); -+ MWLWIFI_DEBUGFS_ADD_FILE(dfs_radar); -+ MWLWIFI_DEBUGFS_ADD_FILE(thermal); -+ MWLWIFI_DEBUGFS_ADD_FILE(led_ctrl); -+ MWLWIFI_DEBUGFS_ADD_FILE(regrdwr); -+ MWLWIFI_DEBUGFS_ADD_FILE(ratetable); -+ MWLWIFI_DEBUGFS_ADD_FILE(tx_hist); -+ MWLWIFI_DEBUGFS_ADD_FILE(ba_hist); -+ MWLWIFI_DEBUGFS_ADD_FILE(fixed_rate); -+ MWLWIFI_DEBUGFS_ADD_FILE(core_dump); -+ MWLWIFI_DEBUGFS_ADD_FILE(mcast_cts); -+ MWLWIFI_DEBUGFS_ADD_FILE(wmmedcaap); -+} -+ -+void mwl_debugfs_remove(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ debugfs_remove(priv->debugfs_phy); -+ priv->debugfs_phy = NULL; -+} -diff --git a/drivers/net/wireless/marvell/mwlwifi/debugfs.h b/drivers/net/wireless/marvell/mwlwifi/debugfs.h -new file mode 100644 -index 000000000000..e7595f563348 ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/debugfs.h -@@ -0,0 +1,24 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file defines debug fs related functions. */ -+ -+#ifndef _MWL_DEBUGFS_H_ -+#define _MWL_DEBUGFS_H_ -+ -+void mwl_debugfs_init(struct ieee80211_hw *hw); -+void mwl_debugfs_remove(struct ieee80211_hw *hw); -+ -+#endif /* _MWL_DEBUGFS_H_ */ -diff --git a/drivers/net/wireless/marvell/mwlwifi/hif/fwcmd.c b/drivers/net/wireless/marvell/mwlwifi/hif/fwcmd.c -new file mode 100644 -index 000000000000..ff943d6fe447 ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/hif/fwcmd.c -@@ -0,0 +1,3852 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file implements firmware host command related -+ * functions. -+ */ -+ -+#include -+#include -+ -+#include "sysadpt.h" -+#include "core.h" -+#include "utils.h" -+#include "hif/fwcmd.h" -+#include "hif/hif-ops.h" -+ -+#define MAX_WAIT_GET_HW_SPECS_ITERATONS 3 -+ -+struct cmd_header { -+ __le16 command; -+ __le16 len; -+} __packed; -+ -+char *mwl_fwcmd_get_cmd_string(unsigned short cmd) -+{ -+ int max_entries = 0; -+ int curr_cmd = 0; -+ -+ static const struct { -+ u16 cmd; -+ char *cmd_string; -+ } cmds[] = { -+ { HOSTCMD_CMD_GET_HW_SPEC, "GetHwSpecifications" }, -+ { HOSTCMD_CMD_SET_HW_SPEC, "SetHwSepcifications" }, -+ { HOSTCMD_CMD_802_11_GET_STAT, "80211GetStat" }, -+ { HOSTCMD_CMD_BBP_REG_ACCESS, "BBPRegAccess" }, -+ { HOSTCMD_CMD_RF_REG_ACCESS, "RFRegAccess" }, -+ { HOSTCMD_CMD_802_11_RADIO_CONTROL, "80211RadioControl" }, -+ { HOSTCMD_CMD_MEM_ADDR_ACCESS, "MEMAddrAccess" }, -+ { HOSTCMD_CMD_802_11_TX_POWER, "80211TxPower" }, -+ { HOSTCMD_CMD_802_11_RF_ANTENNA, "80211RfAntenna" }, -+ { HOSTCMD_CMD_BROADCAST_SSID_ENABLE, "BroadcastSsidEnable" }, -+ { HOSTCMD_CMD_SET_CFG, "SetCfg" }, -+ { HOSTCMD_CMD_SET_RF_CHANNEL, "SetRfChannel" }, -+ { HOSTCMD_CMD_SET_AID, "SetAid" }, -+ { HOSTCMD_CMD_SET_INFRA_MODE, "SetInfraMode" }, -+ { HOSTCMD_CMD_802_11_RTS_THSD, "80211RtsThreshold" }, -+ { HOSTCMD_CMD_SET_EDCA_PARAMS, "SetEDCAParams" }, -+ { HOSTCMD_CMD_802_11H_DETECT_RADAR, "80211hDetectRadar" }, -+ { HOSTCMD_CMD_SET_WMM_MODE, "SetWMMMode" }, -+ { HOSTCMD_CMD_HT_GUARD_INTERVAL, "HtGuardInterval" }, -+ { HOSTCMD_CMD_SET_FIXED_RATE, "SetFixedRate" }, -+ { HOSTCMD_CMD_SET_IES, "SetInformationElements" }, -+ { HOSTCMD_CMD_SET_LINKADAPT_CS_MODE, "LinkAdaptCsMode" }, -+ { HOSTCMD_CMD_DUMP_OTP_DATA, "DumpOtpData" }, -+ { HOSTCMD_CMD_SET_MAC_ADDR, "SetMacAddr" }, -+ { HOSTCMD_CMD_SET_RATE_ADAPT_MODE, "SetRateAdaptationMode" }, -+ { HOSTCMD_CMD_GET_WATCHDOG_BITMAP, "GetWatchdogBitMap" }, -+ { HOSTCMD_CMD_DEL_MAC_ADDR, "DelMacAddr" }, -+ { HOSTCMD_CMD_BSS_START, "BssStart" }, -+ { HOSTCMD_CMD_AP_BEACON, "SetApBeacon" }, -+ { HOSTCMD_CMD_SET_NEW_STN, "SetNewStation" }, -+ { HOSTCMD_CMD_SET_APMODE, "SetApMode" }, -+ { HOSTCMD_CMD_SET_SWITCH_CHANNEL, "SetSwitchChannel" }, -+ { HOSTCMD_CMD_UPDATE_ENCRYPTION, "UpdateEncryption" }, -+ { HOSTCMD_CMD_BASTREAM, "BAStream" }, -+ { HOSTCMD_CMD_SET_SPECTRUM_MGMT, "SetSpectrumMgmt" }, -+ { HOSTCMD_CMD_SET_POWER_CONSTRAINT, "SetPowerConstraint" }, -+ { HOSTCMD_CMD_SET_COUNTRY_CODE, "SetCountryCode" }, -+ { HOSTCMD_CMD_SET_OPTIMIZATION_LEVEL, "SetOptimizationLevel" }, -+ { HOSTCMD_CMD_SET_WSC_IE, "SetWscIE" }, -+ { HOSTCMD_CMD_GET_RATETABLE, "GetRateTable" }, -+ { HOSTCMD_CMD_GET_SEQNO, "GetSeqno" }, -+ { HOSTCMD_CMD_DWDS_ENABLE, "DwdsEnable" }, -+ { HOSTCMD_CMD_FW_FLUSH_TIMER, "FwFlushTimer" }, -+ { HOSTCMD_CMD_SET_CDD, "SetCDD" }, -+ { HOSTCMD_CMD_SET_BFTYPE, "SetBFType" }, -+ { HOSTCMD_CMD_CAU_REG_ACCESS, "CAURegAccess" }, -+ { HOSTCMD_CMD_GET_TEMP, "GetTemp" }, -+ { HOSTCMD_CMD_LED_CTRL, "LedCtrl" }, -+ { HOSTCMD_CMD_GET_FW_REGION_CODE, "GetFwRegionCode" }, -+ { HOSTCMD_CMD_GET_DEVICE_PWR_TBL, "GetDevicePwrTbl" }, -+ { HOSTCMD_CMD_SET_RATE_DROP, "SetRateDrop" }, -+ { HOSTCMD_CMD_NEWDP_DMATHREAD_START, "NewdpDMAThreadStart" }, -+ { HOSTCMD_CMD_GET_FW_REGION_CODE_SC4, "GetFwRegionCodeSC4" }, -+ { HOSTCMD_CMD_GET_DEVICE_PWR_TBL_SC4, "GetDevicePwrTblSC4" }, -+ { HOSTCMD_CMD_QUIET_MODE, "QuietMode" }, -+ { HOSTCMD_CMD_CORE_DUMP_DIAG_MODE, "CoreDumpDiagMode" }, -+ { HOSTCMD_CMD_802_11_SLOT_TIME, "80211SlotTime" }, -+ { HOSTCMD_CMD_GET_FW_CORE_DUMP, "GetFwCoreDump" }, -+ { HOSTCMD_CMD_EDMAC_CTRL, "EDMACCtrl" }, -+ { HOSTCMD_CMD_TXPWRLMT_CFG, "TxpwrlmtCfg" }, -+ { HOSTCMD_CMD_MCAST_CTS, "McastCts" }, -+ }; -+ -+ max_entries = ARRAY_SIZE(cmds); -+ -+ for (curr_cmd = 0; curr_cmd < max_entries; curr_cmd++) -+ if ((cmd & 0x7fff) == cmds[curr_cmd].cmd) -+ return cmds[curr_cmd].cmd_string; -+ -+ return "unknown"; -+} -+ -+static int mwl_fwcmd_802_11_radio_control(struct mwl_priv *priv, -+ bool enable, bool force) -+{ -+ struct hostcmd_cmd_802_11_radio_control *pcmd; -+ -+ if (enable == priv->radio_on && !force) -+ return 0; -+ -+ pcmd = (struct hostcmd_cmd_802_11_radio_control *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_802_11_RADIO_CONTROL); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->action = cpu_to_le16(WL_SET); -+ pcmd->control = cpu_to_le16(priv->radio_short_preamble ? -+ WL_AUTO_PREAMBLE : WL_LONG_PREAMBLE); -+ pcmd->radio_on = cpu_to_le16(enable ? WL_ENABLE : WL_DISABLE); -+ -+ if (mwl_hif_exec_cmd(priv->hw, HOSTCMD_CMD_802_11_RADIO_CONTROL)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ priv->radio_on = enable; -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+static int mwl_fwcmd_get_tx_powers(struct mwl_priv *priv, u16 *powlist, -+ u8 action, u16 ch, u16 band, -+ u16 width, u16 sub_ch) -+{ -+ struct hostcmd_cmd_802_11_tx_power *pcmd; -+ int i; -+ -+ pcmd = (struct hostcmd_cmd_802_11_tx_power *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ if (priv->chip_type == MWL8997) { -+ memset(pcmd, 0x00, -+ sizeof(struct hostcmd_cmd_802_11_tx_power_kf2)); -+ pcmd->cmd_hdr.len = cpu_to_le16( -+ sizeof(struct hostcmd_cmd_802_11_tx_power_kf2)); -+ } else { -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ } -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_802_11_TX_POWER); -+ pcmd->action = cpu_to_le16(action); -+ pcmd->ch = cpu_to_le16(ch); -+ pcmd->bw = cpu_to_le16(width); -+ pcmd->band = cpu_to_le16(band); -+ pcmd->sub_ch = cpu_to_le16(sub_ch); -+ -+ if (mwl_hif_exec_cmd(priv->hw, HOSTCMD_CMD_802_11_TX_POWER)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ for (i = 0; i < priv->pwr_level; i++) -+ powlist[i] = le16_to_cpu(pcmd->power_level_list[i]); -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+static int mwl_fwcmd_set_tx_powers(struct mwl_priv *priv, u16 txpow[], -+ u8 action, u16 ch, u16 band, -+ u16 width, u16 sub_ch) -+{ -+ struct hostcmd_cmd_802_11_tx_power *pcmd; -+ int i; -+ -+ pcmd = (struct hostcmd_cmd_802_11_tx_power *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ if (priv->chip_type == MWL8997) { -+ memset(pcmd, 0x00, -+ sizeof(struct hostcmd_cmd_802_11_tx_power_kf2)); -+ pcmd->cmd_hdr.len = cpu_to_le16( -+ sizeof(struct hostcmd_cmd_802_11_tx_power_kf2)); -+ } else { -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ } -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_802_11_TX_POWER); -+ pcmd->action = cpu_to_le16(action); -+ pcmd->ch = cpu_to_le16(ch); -+ pcmd->bw = cpu_to_le16(width); -+ pcmd->band = cpu_to_le16(band); -+ pcmd->sub_ch = cpu_to_le16(sub_ch); -+ -+ for (i = 0; i < priv->pwr_level; i++) -+ pcmd->power_level_list[i] = cpu_to_le16(txpow[i]); -+ -+ if (mwl_hif_exec_cmd(priv->hw, HOSTCMD_CMD_802_11_TX_POWER)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+static u8 mwl_fwcmd_get_80m_pri_chnl(u8 channel) -+{ -+ u8 act_primary = ACT_PRIMARY_CHAN_0; -+ -+ switch (channel) { -+ case 36: -+ act_primary = ACT_PRIMARY_CHAN_0; -+ break; -+ case 40: -+ act_primary = ACT_PRIMARY_CHAN_1; -+ break; -+ case 44: -+ act_primary = ACT_PRIMARY_CHAN_2; -+ break; -+ case 48: -+ act_primary = ACT_PRIMARY_CHAN_3; -+ break; -+ case 52: -+ act_primary = ACT_PRIMARY_CHAN_0; -+ break; -+ case 56: -+ act_primary = ACT_PRIMARY_CHAN_1; -+ break; -+ case 60: -+ act_primary = ACT_PRIMARY_CHAN_2; -+ break; -+ case 64: -+ act_primary = ACT_PRIMARY_CHAN_3; -+ break; -+ case 100: -+ act_primary = ACT_PRIMARY_CHAN_0; -+ break; -+ case 104: -+ act_primary = ACT_PRIMARY_CHAN_1; -+ break; -+ case 108: -+ act_primary = ACT_PRIMARY_CHAN_2; -+ break; -+ case 112: -+ act_primary = ACT_PRIMARY_CHAN_3; -+ break; -+ case 116: -+ act_primary = ACT_PRIMARY_CHAN_0; -+ break; -+ case 120: -+ act_primary = ACT_PRIMARY_CHAN_1; -+ break; -+ case 124: -+ act_primary = ACT_PRIMARY_CHAN_2; -+ break; -+ case 128: -+ act_primary = ACT_PRIMARY_CHAN_3; -+ break; -+ case 132: -+ act_primary = ACT_PRIMARY_CHAN_0; -+ break; -+ case 136: -+ act_primary = ACT_PRIMARY_CHAN_1; -+ break; -+ case 140: -+ act_primary = ACT_PRIMARY_CHAN_2; -+ break; -+ case 144: -+ act_primary = ACT_PRIMARY_CHAN_3; -+ break; -+ case 149: -+ act_primary = ACT_PRIMARY_CHAN_0; -+ break; -+ case 153: -+ act_primary = ACT_PRIMARY_CHAN_1; -+ break; -+ case 157: -+ act_primary = ACT_PRIMARY_CHAN_2; -+ break; -+ case 161: -+ act_primary = ACT_PRIMARY_CHAN_3; -+ break; -+ } -+ -+ return act_primary; -+} -+ -+static u8 mwl_fwcmd_get_160m_pri_chnl(u8 channel) -+{ -+ u8 act_primary = ACT_PRIMARY_CHAN_0; -+ -+ switch (channel) { -+ case 36: -+ act_primary = ACT_PRIMARY_CHAN_0; -+ break; -+ case 40: -+ act_primary = ACT_PRIMARY_CHAN_1; -+ break; -+ case 44: -+ act_primary = ACT_PRIMARY_CHAN_2; -+ break; -+ case 48: -+ act_primary = ACT_PRIMARY_CHAN_3; -+ break; -+ case 52: -+ act_primary = ACT_PRIMARY_CHAN_4; -+ break; -+ case 56: -+ act_primary = ACT_PRIMARY_CHAN_5; -+ break; -+ case 60: -+ act_primary = ACT_PRIMARY_CHAN_6; -+ break; -+ case 64: -+ act_primary = ACT_PRIMARY_CHAN_7; -+ break; -+ case 100: -+ act_primary = ACT_PRIMARY_CHAN_0; -+ break; -+ case 104: -+ act_primary = ACT_PRIMARY_CHAN_1; -+ break; -+ case 108: -+ act_primary = ACT_PRIMARY_CHAN_2; -+ break; -+ case 112: -+ act_primary = ACT_PRIMARY_CHAN_3; -+ break; -+ case 116: -+ act_primary = ACT_PRIMARY_CHAN_4; -+ break; -+ case 120: -+ act_primary = ACT_PRIMARY_CHAN_5; -+ break; -+ case 124: -+ act_primary = ACT_PRIMARY_CHAN_6; -+ break; -+ case 128: -+ act_primary = ACT_PRIMARY_CHAN_7; -+ break; -+ case 149: -+ act_primary = ACT_PRIMARY_CHAN_0; -+ break; -+ case 153: -+ act_primary = ACT_PRIMARY_CHAN_1; -+ break; -+ case 157: -+ act_primary = ACT_PRIMARY_CHAN_2; -+ break; -+ case 161: -+ act_primary = ACT_PRIMARY_CHAN_3; -+ break; -+ case 165: -+ act_primary = ACT_PRIMARY_CHAN_4; -+ break; -+ case 169: -+ act_primary = ACT_PRIMARY_CHAN_5; -+ break; -+ case 173: -+ act_primary = ACT_PRIMARY_CHAN_6; -+ break; -+ case 177: -+ act_primary = ACT_PRIMARY_CHAN_7; -+ break; -+ } -+ -+ return act_primary; -+} -+ -+static void mwl_fwcmd_parse_beacon(struct mwl_priv *priv, -+ struct mwl_vif *vif, u8 *beacon, int len) -+{ -+ struct ieee80211_mgmt *mgmt; -+ struct beacon_info *beacon_info; -+ int baselen; -+ u8 *pos; -+ size_t left; -+ bool elem_parse_failed; -+ -+ mgmt = (struct ieee80211_mgmt *)beacon; -+ -+ baselen = (u8 *)mgmt->u.beacon.variable - (u8 *)mgmt; -+ if (baselen > len) -+ return; -+ -+ beacon_info = &vif->beacon_info; -+ memset(beacon_info, 0, sizeof(struct beacon_info)); -+ beacon_info->valid = false; -+ beacon_info->ie_ht_ptr = &beacon_info->ie_list_ht[0]; -+ beacon_info->ie_vht_ptr = &beacon_info->ie_list_vht[0]; -+ -+ beacon_info->cap_info = le16_to_cpu(mgmt->u.beacon.capab_info); -+ beacon_info->power_constraint = 0; -+ -+ pos = (u8 *)mgmt->u.beacon.variable; -+ left = len - baselen; -+ -+ elem_parse_failed = false; -+ -+ while (left >= 2) { -+ u8 id, elen; -+ -+ id = *pos++; -+ elen = *pos++; -+ left -= 2; -+ -+ if (elen > left) { -+ elem_parse_failed = true; -+ break; -+ } -+ -+ switch (id) { -+ case WLAN_EID_COUNTRY: -+ beacon_info->ie_country_len = (elen + 2); -+ beacon_info->ie_country_ptr = (pos - 2); -+ break; -+ case WLAN_EID_SUPP_RATES: -+ case WLAN_EID_EXT_SUPP_RATES: -+ { -+ int idx, bi, oi; -+ u8 rate; -+ -+ for (bi = 0; bi < SYSADPT_MAX_DATA_RATES_G; -+ bi++) { -+ if (beacon_info->b_rate_set[bi] == 0) -+ break; -+ } -+ -+ for (oi = 0; oi < SYSADPT_MAX_DATA_RATES_G; -+ oi++) { -+ if (beacon_info->op_rate_set[oi] == 0) -+ break; -+ } -+ -+ for (idx = 0; idx < elen; idx++) { -+ rate = pos[idx]; -+ if ((rate & 0x80) != 0) { -+ if (bi < SYSADPT_MAX_DATA_RATES_G) -+ beacon_info->b_rate_set[bi++] -+ = rate & 0x7f; -+ else { -+ elem_parse_failed = true; -+ break; -+ } -+ } -+ if (oi < SYSADPT_MAX_DATA_RATES_G) -+ beacon_info->op_rate_set[oi++] = -+ rate & 0x7f; -+ else { -+ elem_parse_failed = true; -+ break; -+ } -+ } -+ } -+ break; -+ case WLAN_EID_PWR_CONSTRAINT: -+ if (elen == 1) -+ beacon_info->power_constraint = *pos; -+ break; -+ case WLAN_EID_RSN: -+ beacon_info->ie_rsn48_len = (elen + 2); -+ beacon_info->ie_rsn48_ptr = (pos - 2); -+ break; -+ case WLAN_EID_MOBILITY_DOMAIN: -+ beacon_info->ie_mde_len = (elen + 2); -+ beacon_info->ie_mde_ptr = (pos - 2); -+ break; -+ case WLAN_EID_HT_CAPABILITY: -+ case WLAN_EID_HT_OPERATION: -+ case WLAN_EID_OVERLAP_BSS_SCAN_PARAM: -+ case WLAN_EID_EXT_CAPABILITY: -+ beacon_info->ie_ht_len += (elen + 2); -+ if (beacon_info->ie_ht_len > -+ sizeof(beacon_info->ie_list_ht)) { -+ elem_parse_failed = true; -+ } else { -+ *beacon_info->ie_ht_ptr++ = id; -+ *beacon_info->ie_ht_ptr++ = elen; -+ memcpy(beacon_info->ie_ht_ptr, pos, elen); -+ beacon_info->ie_ht_ptr += elen; -+ } -+ break; -+ case WLAN_EID_MESH_CONFIG: -+ beacon_info->ie_meshcfg_len = (elen + 2); -+ beacon_info->ie_meshcfg_ptr = (pos - 2); -+ break; -+ case WLAN_EID_MESH_ID: -+ beacon_info->ie_meshid_len = (elen + 2); -+ beacon_info->ie_meshid_ptr = (pos - 2); -+ break; -+ case WLAN_EID_CHAN_SWITCH_PARAM: -+ beacon_info->ie_meshchsw_len = (elen + 2); -+ beacon_info->ie_meshchsw_ptr = (pos - 2); -+ break; -+ case WLAN_EID_VHT_CAPABILITY: -+ case WLAN_EID_VHT_OPERATION: -+ case WLAN_EID_OPMODE_NOTIF: -+ beacon_info->ie_vht_len += (elen + 2); -+ if (beacon_info->ie_vht_len > -+ sizeof(beacon_info->ie_list_vht)) { -+ elem_parse_failed = true; -+ } else { -+ *beacon_info->ie_vht_ptr++ = id; -+ *beacon_info->ie_vht_ptr++ = elen; -+ memcpy(beacon_info->ie_vht_ptr, pos, elen); -+ beacon_info->ie_vht_ptr += elen; -+ } -+ break; -+ case WLAN_EID_VENDOR_SPECIFIC: -+ if ((pos[0] == 0x00) && (pos[1] == 0x50) && -+ (pos[2] == 0xf2)) { -+ if (pos[3] == 0x01) { -+ beacon_info->ie_rsn_len = (elen + 2); -+ beacon_info->ie_rsn_ptr = (pos - 2); -+ } -+ -+ if (pos[3] == 0x02) { -+ beacon_info->ie_wmm_len = (elen + 2); -+ beacon_info->ie_wmm_ptr = (pos - 2); -+ } -+ -+ if (pos[3] == 0x04) { -+ beacon_info->ie_wsc_len = (elen + 2); -+ beacon_info->ie_wsc_ptr = (pos - 2); -+ } -+ } -+ break; -+ default: -+ break; -+ } -+ -+ left -= elen; -+ pos += elen; -+ } -+ -+ if (!elem_parse_failed) { -+ beacon_info->ie_ht_ptr = &beacon_info->ie_list_ht[0]; -+ beacon_info->ie_vht_ptr = &beacon_info->ie_list_vht[0]; -+ beacon_info->valid = true; -+ } -+} -+ -+static int mwl_fwcmd_set_ies(struct mwl_priv *priv, struct mwl_vif *mwl_vif) -+{ -+ struct hostcmd_cmd_set_ies *pcmd; -+ struct beacon_info *beacon = &mwl_vif->beacon_info; -+ u16 ie_list_len_proprietary = 0; -+ -+ if (beacon->ie_ht_len > sizeof(pcmd->ie_list_ht)) -+ goto einval; -+ -+ if (beacon->ie_vht_len > sizeof(pcmd->ie_list_vht)) -+ goto einval; -+ -+ pcmd = (struct hostcmd_cmd_set_ies *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_IES); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->cmd_hdr.macid = mwl_vif->macid; -+ -+ pcmd->action = cpu_to_le16(HOSTCMD_ACT_GEN_SET); -+ -+ memcpy(pcmd->ie_list_ht, beacon->ie_ht_ptr, beacon->ie_ht_len); -+ pcmd->ie_list_len_ht = cpu_to_le16(beacon->ie_ht_len); -+ -+ memcpy(pcmd->ie_list_vht, beacon->ie_vht_ptr, beacon->ie_vht_len); -+ pcmd->ie_list_len_vht = cpu_to_le16(beacon->ie_vht_len); -+ -+ memcpy(pcmd->ie_list_proprietary, beacon->ie_meshid_ptr, -+ beacon->ie_meshid_len); -+ ie_list_len_proprietary = beacon->ie_meshid_len; -+ -+ memcpy(pcmd->ie_list_proprietary + ie_list_len_proprietary, -+ beacon->ie_meshcfg_ptr, beacon->ie_meshcfg_len); -+ ie_list_len_proprietary += beacon->ie_meshcfg_len; -+ -+ memcpy(pcmd->ie_list_proprietary + ie_list_len_proprietary, -+ beacon->ie_meshchsw_ptr, beacon->ie_meshchsw_len); -+ ie_list_len_proprietary += beacon->ie_meshchsw_len; -+ -+ if (priv->chip_type == MWL8897) { -+ memcpy(pcmd->ie_list_proprietary + ie_list_len_proprietary, -+ beacon->ie_wmm_ptr, beacon->ie_wmm_len); -+ ie_list_len_proprietary += mwl_vif->beacon_info.ie_wmm_len; -+ } -+ -+ memcpy(pcmd->ie_list_proprietary + ie_list_len_proprietary, -+ beacon->ie_mde_ptr, beacon->ie_mde_len); -+ ie_list_len_proprietary += mwl_vif->beacon_info.ie_mde_len; -+ -+ pcmd->ie_list_len_proprietary = cpu_to_le16(ie_list_len_proprietary); -+ -+ if (mwl_hif_exec_cmd(priv->hw, HOSTCMD_CMD_SET_IES)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+ -+einval: -+ -+ wiphy_err(priv->hw->wiphy, "length of IE is too long\n"); -+ -+ return -EINVAL; -+} -+ -+static int mwl_fwcmd_set_ap_beacon(struct mwl_priv *priv, -+ struct mwl_vif *mwl_vif, -+ struct ieee80211_bss_conf *bss_conf) -+{ -+ struct hostcmd_cmd_ap_beacon *pcmd; -+ struct ds_params *phy_ds_param_set; -+ -+ /* wmm structure of start command is defined less one byte, -+ * due to following field country is not used, add byte one -+ * to bypass the check. -+ */ -+ if (mwl_vif->beacon_info.ie_wmm_len > -+ (sizeof(pcmd->start_cmd.wmm_param) + 1)) -+ goto ielenerr; -+ -+ if (mwl_vif->beacon_info.ie_rsn_len > sizeof(pcmd->start_cmd.rsn_ie)) -+ goto ielenerr; -+ -+ if (mwl_vif->beacon_info.ie_rsn48_len > -+ sizeof(pcmd->start_cmd.rsn48_ie)) -+ goto ielenerr; -+ -+ pcmd = (struct hostcmd_cmd_ap_beacon *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_AP_BEACON); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->cmd_hdr.macid = mwl_vif->macid; -+ -+ ether_addr_copy(pcmd->start_cmd.sta_mac_addr, mwl_vif->bssid); -+ memcpy(pcmd->start_cmd.ssid, bss_conf->ssid, bss_conf->ssid_len); -+ if (priv->chip_type == MWL8997) -+ ether_addr_copy(pcmd->start_cmd.bssid, mwl_vif->bssid); -+ pcmd->start_cmd.bss_type = 1; -+ pcmd->start_cmd.bcn_period = cpu_to_le16(bss_conf->beacon_int); -+ pcmd->start_cmd.dtim_period = bss_conf->dtim_period; /* 8bit */ -+ -+ phy_ds_param_set = &pcmd->start_cmd.phy_param_set.ds_param_set; -+ phy_ds_param_set->elem_id = WLAN_EID_DS_PARAMS; -+ phy_ds_param_set->len = sizeof(phy_ds_param_set->current_chnl); -+ phy_ds_param_set->current_chnl = bss_conf->chandef.chan->hw_value; -+ -+ pcmd->start_cmd.probe_delay = cpu_to_le16(10); -+ pcmd->start_cmd.cap_info = cpu_to_le16(mwl_vif->beacon_info.cap_info); -+ -+ memcpy(&pcmd->start_cmd.wmm_param, mwl_vif->beacon_info.ie_wmm_ptr, -+ mwl_vif->beacon_info.ie_wmm_len); -+ -+ memcpy(&pcmd->start_cmd.rsn_ie, mwl_vif->beacon_info.ie_rsn_ptr, -+ mwl_vif->beacon_info.ie_rsn_len); -+ -+ memcpy(&pcmd->start_cmd.rsn48_ie, mwl_vif->beacon_info.ie_rsn48_ptr, -+ mwl_vif->beacon_info.ie_rsn48_len); -+ -+ memcpy(pcmd->start_cmd.b_rate_set, mwl_vif->beacon_info.b_rate_set, -+ SYSADPT_MAX_DATA_RATES_G); -+ -+ memcpy(pcmd->start_cmd.op_rate_set, mwl_vif->beacon_info.op_rate_set, -+ SYSADPT_MAX_DATA_RATES_G); -+ -+ if (mwl_hif_exec_cmd(priv->hw, HOSTCMD_CMD_AP_BEACON)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+ -+ielenerr: -+ -+ wiphy_err(priv->hw->wiphy, "length of IE is too long\n"); -+ -+ return -EINVAL; -+} -+ -+static int mwl_fwcmd_set_spectrum_mgmt(struct mwl_priv *priv, bool enable) -+{ -+ struct hostcmd_cmd_set_spectrum_mgmt *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_set_spectrum_mgmt *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_SPECTRUM_MGMT); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->spectrum_mgmt = cpu_to_le32(enable); -+ -+ if (mwl_hif_exec_cmd(priv->hw, HOSTCMD_CMD_SET_SPECTRUM_MGMT)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+static int mwl_fwcmd_set_power_constraint(struct mwl_priv *priv, -+ u32 power_constraint) -+{ -+ struct hostcmd_cmd_set_power_constraint *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_set_power_constraint *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_POWER_CONSTRAINT); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->power_constraint = cpu_to_le32(power_constraint); -+ -+ if (mwl_hif_exec_cmd(priv->hw, HOSTCMD_CMD_SET_POWER_CONSTRAINT)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+static int mwl_fwcmd_set_country_code(struct mwl_priv *priv, -+ struct mwl_vif *mwl_vif, -+ struct ieee80211_bss_conf *bss_conf) -+{ -+ struct hostcmd_cmd_set_country_code *pcmd; -+ struct beacon_info *b_inf = &mwl_vif->beacon_info; -+ u8 chnl_len; -+ bool a_band; -+ bool enable = false; -+ -+ if (b_inf->ie_country_ptr) { -+ if (bss_conf->chandef.chan->band == NL80211_BAND_2GHZ) -+ a_band = false; -+ else if (bss_conf->chandef.chan->band == NL80211_BAND_5GHZ) -+ a_band = true; -+ else -+ return -EINVAL; -+ -+ chnl_len = b_inf->ie_country_len - 5; -+ if (a_band) { -+ if (chnl_len > sizeof(pcmd->domain_info.domain_entry_a)) -+ return -EINVAL; -+ } else { -+ if (chnl_len > sizeof(pcmd->domain_info.domain_entry_g)) -+ return -EINVAL; -+ } -+ -+ enable = true; -+ } -+ -+ pcmd = (struct hostcmd_cmd_set_country_code *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_COUNTRY_CODE); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->action = cpu_to_le32(enable); -+ if (enable) { -+ memcpy(pcmd->domain_info.country_string, -+ b_inf->ie_country_ptr + 2, 3); -+ if (a_band) { -+ pcmd->domain_info.g_chnl_len = 0; -+ pcmd->domain_info.a_chnl_len = chnl_len; -+ memcpy(pcmd->domain_info.domain_entry_a, -+ b_inf->ie_country_ptr + 5, chnl_len); -+ } else { -+ pcmd->domain_info.a_chnl_len = 0; -+ pcmd->domain_info.g_chnl_len = chnl_len; -+ memcpy(pcmd->domain_info.domain_entry_g, -+ b_inf->ie_country_ptr + 5, chnl_len); -+ } -+ } -+ -+ if (mwl_hif_exec_cmd(priv->hw, HOSTCMD_CMD_SET_COUNTRY_CODE)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+static int mwl_fwcmd_encryption_set_cmd_info(struct hostcmd_cmd_set_key *cmd, -+ u8 *addr, -+ struct ieee80211_key_conf *key) -+{ -+ cmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_UPDATE_ENCRYPTION); -+ cmd->cmd_hdr.len = cpu_to_le16(sizeof(*cmd)); -+ cmd->key_param.length = cpu_to_le16(sizeof(*cmd) - -+ offsetof(struct hostcmd_cmd_set_key, key_param)); -+ cmd->key_param.key_index = cpu_to_le32(key->keyidx); -+ cmd->key_param.key_len = cpu_to_le16(key->keylen); -+ ether_addr_copy(cmd->key_param.mac_addr, addr); -+ -+ switch (key->cipher) { -+ case WLAN_CIPHER_SUITE_WEP40: -+ case WLAN_CIPHER_SUITE_WEP104: -+ cmd->key_param.key_type_id = cpu_to_le16(KEY_TYPE_ID_WEP); -+ if (key->keyidx == 0) -+ cmd->key_param.key_info = -+ cpu_to_le32(ENCR_KEY_FLAG_WEP_TXKEY); -+ break; -+ case WLAN_CIPHER_SUITE_TKIP: -+ cmd->key_param.key_type_id = cpu_to_le16(KEY_TYPE_ID_TKIP); -+ cmd->key_param.key_info = -+ (key->flags & IEEE80211_KEY_FLAG_PAIRWISE) ? -+ cpu_to_le32(ENCR_KEY_FLAG_PAIRWISE) : -+ cpu_to_le32(ENCR_KEY_FLAG_TXGROUPKEY); -+ cmd->key_param.key_info |= -+ cpu_to_le32(ENCR_KEY_FLAG_MICKEY_VALID | -+ ENCR_KEY_FLAG_TSC_VALID); -+ break; -+ case WLAN_CIPHER_SUITE_CCMP: -+ cmd->key_param.key_type_id = cpu_to_le16(KEY_TYPE_ID_AES); -+ cmd->key_param.key_info = -+ (key->flags & IEEE80211_KEY_FLAG_PAIRWISE) ? -+ cpu_to_le32(ENCR_KEY_FLAG_PAIRWISE) : -+ cpu_to_le32(ENCR_KEY_FLAG_TXGROUPKEY); -+ break; -+ case WLAN_CIPHER_SUITE_AES_CMAC: -+ return 1; -+ default: -+ return -ENOTSUPP; -+ } -+ -+ return 0; -+} -+ -+static __le16 mwl_fwcmd_parse_cal_cfg(const u8 *src, size_t len, u8 *dst) -+{ -+ const u8 *ptr; -+ u8 *dptr; -+ char byte_str[3]; -+ long res; -+ -+ ptr = src; -+ dptr = dst; -+ byte_str[2] = '\0'; -+ -+ while (ptr - src < len) { -+ if (*ptr && (isspace(*ptr) || iscntrl(*ptr))) { -+ ptr++; -+ continue; -+ } -+ -+ if (isxdigit(*ptr)) { -+ byte_str[0] = *ptr++; -+ byte_str[1] = *ptr++; -+ kstrtol(byte_str, 16, &res); -+ *dptr++ = res; -+ } else { -+ ptr++; -+ } -+ } -+ -+ return cpu_to_le16(dptr - dst); -+} -+ -+static u16 mwl_fwcmd_parse_txpwrlmt_cfg(const u8 *src, size_t len, -+ u16 parse_len, u8 *dst) -+{ -+ const u8 *ptr; -+ u8 *dptr; -+ char byte_str[3]; -+ long res; -+ -+ ptr = src; -+ dptr = dst; -+ byte_str[2] = '\0'; -+ -+ while ((ptr - src < len) && (dptr - dst < parse_len)) { -+ if (*ptr && (isspace(*ptr) || iscntrl(*ptr))) { -+ ptr++; -+ continue; -+ } -+ -+ if (isxdigit(*ptr)) { -+ byte_str[0] = *ptr++; -+ byte_str[1] = *ptr++; -+ kstrtol(byte_str, 16, &res); -+ *dptr++ = res; -+ } else { -+ ptr++; -+ } -+ } -+ -+ return (ptr - src); -+} -+ -+const struct hostcmd_get_hw_spec -+*mwl_fwcmd_get_hw_specs(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_get_hw_spec *pcmd; -+ int retry; -+ -+ pcmd = (struct hostcmd_cmd_get_hw_spec *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ wiphy_debug(hw->wiphy, "pcmd = %p\n", pcmd); -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ eth_broadcast_addr(pcmd->hw_spec.permanent_addr); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_GET_HW_SPEC); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->hw_spec.fw_awake_cookie = cpu_to_le32(priv->pphys_cmd_buf + 2048); -+ -+ retry = 0; -+ while (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_GET_HW_SPEC)) { -+ if (retry++ > MAX_WAIT_GET_HW_SPECS_ITERATONS) { -+ wiphy_err(hw->wiphy, "can't get hw specs\n"); -+ mutex_unlock(&priv->fwcmd_mutex); -+ return NULL; -+ } -+ -+ msleep(1000); -+ wiphy_debug(hw->wiphy, -+ "repeat command = %p\n", pcmd); -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return &pcmd->hw_spec; -+} -+ -+int mwl_fwcmd_set_hw_specs(struct ieee80211_hw *hw, -+ struct hostcmd_set_hw_spec *spec) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_set_hw_spec *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_set_hw_spec *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_HW_SPEC); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ memcpy(&pcmd->hw_spec, spec, sizeof(*spec)); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_HW_SPEC)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_get_stat(struct ieee80211_hw *hw, -+ struct ieee80211_low_level_stats *stats) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_802_11_get_stat *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_802_11_get_stat *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_802_11_GET_STAT); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_802_11_GET_STAT)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ stats->dot11ACKFailureCount = -+ le32_to_cpu(pcmd->ack_failures); -+ stats->dot11RTSFailureCount = -+ le32_to_cpu(pcmd->rts_failures); -+ stats->dot11FCSErrorCount = -+ le32_to_cpu(pcmd->rx_fcs_errors); -+ stats->dot11RTSSuccessCount = -+ le32_to_cpu(pcmd->rts_successes); -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_reg_bb(struct ieee80211_hw *hw, u8 flag, u32 reg, u32 *val) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_bbp_reg_access *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_bbp_reg_access *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_BBP_REG_ACCESS); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->offset = cpu_to_le16(reg); -+ pcmd->action = cpu_to_le16(flag); -+ pcmd->value = *val; -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_BBP_REG_ACCESS)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ *val = pcmd->value; -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_reg_rf(struct ieee80211_hw *hw, u8 flag, u32 reg, u32 *val) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_rf_reg_access *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_rf_reg_access *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_RF_REG_ACCESS); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->offset = cpu_to_le16(reg); -+ pcmd->action = cpu_to_le16(flag); -+ pcmd->value = *val; -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_RF_REG_ACCESS)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ *val = pcmd->value; -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_radio_enable(struct ieee80211_hw *hw) -+{ -+ return mwl_fwcmd_802_11_radio_control(hw->priv, true, false); -+} -+ -+int mwl_fwcmd_radio_disable(struct ieee80211_hw *hw) -+{ -+ return mwl_fwcmd_802_11_radio_control(hw->priv, false, false); -+} -+ -+int mwl_fwcmd_set_radio_preamble(struct ieee80211_hw *hw, bool short_preamble) -+{ -+ struct mwl_priv *priv = hw->priv; -+ int rc; -+ -+ priv->radio_short_preamble = short_preamble; -+ rc = mwl_fwcmd_802_11_radio_control(priv, true, true); -+ -+ return rc; -+} -+ -+int mwl_fwcmd_get_addr_value(struct ieee80211_hw *hw, u32 addr, u32 len, -+ u32 *val, u16 set) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_mem_addr_access *pcmd; -+ int i; -+ -+ pcmd = (struct hostcmd_cmd_mem_addr_access *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_MEM_ADDR_ACCESS); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->address = cpu_to_le32(addr); -+ pcmd->length = cpu_to_le16(len); -+ pcmd->value[0] = cpu_to_le32(*val); -+ pcmd->reserved = cpu_to_le16(set); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_MEM_ADDR_ACCESS)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ for (i = 0; i < len; i++) -+ val[i] = le32_to_cpu(pcmd->value[i]); -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_max_tx_power(struct ieee80211_hw *hw, -+ struct ieee80211_conf *conf, u8 fraction) -+{ -+ struct ieee80211_channel *channel = conf->chandef.chan; -+ struct mwl_priv *priv = hw->priv; -+ int reduce_val = 0; -+ u16 band = 0, width = 0, sub_ch = 0; -+ u16 maxtxpow[SYSADPT_TX_GRP_PWR_LEVEL_TOTAL]; -+ int i, tmp; -+ int rc = 0; -+ -+ if ((priv->chip_type != MWL8997) && (priv->forbidden_setting)) -+ return rc; -+ -+ switch (fraction) { -+ case 0: -+ reduce_val = 0; /* Max */ -+ break; -+ case 1: -+ reduce_val = 2; /* 75% -1.25db */ -+ break; -+ case 2: -+ reduce_val = 3; /* 50% -3db */ -+ break; -+ case 3: -+ reduce_val = 6; /* 25% -6db */ -+ break; -+ default: -+ /* larger than case 3, pCmd->MaxPowerLevel is min */ -+ reduce_val = 0xff; -+ break; -+ } -+ -+ if (channel->band == NL80211_BAND_2GHZ) -+ band = FREQ_BAND_2DOT4GHZ; -+ else if (channel->band == NL80211_BAND_5GHZ) -+ band = FREQ_BAND_5GHZ; -+ -+ switch (conf->chandef.width) { -+ case NL80211_CHAN_WIDTH_20_NOHT: -+ case NL80211_CHAN_WIDTH_20: -+ width = CH_20_MHZ_WIDTH; -+ sub_ch = NO_EXT_CHANNEL; -+ break; -+ case NL80211_CHAN_WIDTH_40: -+ width = CH_40_MHZ_WIDTH; -+ if (conf->chandef.center_freq1 > channel->center_freq) -+ sub_ch = EXT_CH_ABOVE_CTRL_CH; -+ else -+ sub_ch = EXT_CH_BELOW_CTRL_CH; -+ break; -+ case NL80211_CHAN_WIDTH_80: -+ width = CH_80_MHZ_WIDTH; -+ if (conf->chandef.center_freq1 > channel->center_freq) -+ sub_ch = EXT_CH_ABOVE_CTRL_CH; -+ else -+ sub_ch = EXT_CH_BELOW_CTRL_CH; -+ break; -+ default: -+ return -EINVAL; -+ } -+ -+ if (priv->chip_type == MWL8997) { -+ mwl_fwcmd_get_tx_powers(priv, priv->max_tx_pow, -+ HOSTCMD_ACT_GET_MAX_TX_PWR, -+ channel->hw_value, band, width, sub_ch); -+ -+ for (i = 0; i < priv->pwr_level; i++) { -+ tmp = priv->max_tx_pow[i]; -+ maxtxpow[i] = ((tmp - reduce_val) > 0) ? -+ (tmp - reduce_val) : 0; -+ } -+ -+ rc = mwl_fwcmd_set_tx_powers(priv, maxtxpow, -+ HOSTCMD_ACT_SET_MAX_TX_PWR, -+ channel->hw_value, band, -+ width, sub_ch); -+ return rc; -+ } -+ -+ if ((priv->powinited & MWL_POWER_INIT_2) == 0) { -+ mwl_fwcmd_get_tx_powers(priv, priv->max_tx_pow, -+ HOSTCMD_ACT_GEN_GET_LIST, -+ channel->hw_value, band, width, sub_ch); -+ priv->powinited |= MWL_POWER_INIT_2; -+ } -+ -+ if ((priv->powinited & MWL_POWER_INIT_1) == 0) { -+ mwl_fwcmd_get_tx_powers(priv, priv->target_powers, -+ HOSTCMD_ACT_GEN_GET_LIST, -+ channel->hw_value, band, width, sub_ch); -+ priv->powinited |= MWL_POWER_INIT_1; -+ } -+ -+ for (i = 0; i < priv->pwr_level; i++) { -+ if (priv->target_powers[i] > priv->max_tx_pow[i]) -+ tmp = priv->max_tx_pow[i]; -+ else -+ tmp = priv->target_powers[i]; -+ maxtxpow[i] = ((tmp - reduce_val) > 0) ? (tmp - reduce_val) : 0; -+ } -+ -+ rc = mwl_fwcmd_set_tx_powers(priv, maxtxpow, HOSTCMD_ACT_GEN_SET, -+ channel->hw_value, band, width, sub_ch); -+ -+ return rc; -+} -+ -+int mwl_fwcmd_tx_power(struct ieee80211_hw *hw, -+ struct ieee80211_conf *conf, u8 fraction) -+{ -+ struct ieee80211_channel *channel = conf->chandef.chan; -+ struct mwl_priv *priv = hw->priv; -+ int reduce_val = 0; -+ u16 band = 0, width = 0, sub_ch = 0; -+ u16 txpow[SYSADPT_TX_GRP_PWR_LEVEL_TOTAL]; -+ int index, found = 0; -+ int i, tmp; -+ int rc = 0; -+ -+ if ((priv->chip_type != MWL8997) && (priv->forbidden_setting)) -+ return rc; -+ -+ switch (fraction) { -+ case 0: -+ reduce_val = 0; /* Max */ -+ break; -+ case 1: -+ reduce_val = 2; /* 75% -1.25db */ -+ break; -+ case 2: -+ reduce_val = 3; /* 50% -3db */ -+ break; -+ case 3: -+ reduce_val = 6; /* 25% -6db */ -+ break; -+ default: -+ /* larger than case 3, pCmd->MaxPowerLevel is min */ -+ reduce_val = 0xff; -+ break; -+ } -+ -+ if (channel->band == NL80211_BAND_2GHZ) -+ band = FREQ_BAND_2DOT4GHZ; -+ else if (channel->band == NL80211_BAND_5GHZ) -+ band = FREQ_BAND_5GHZ; -+ -+ switch (conf->chandef.width) { -+ case NL80211_CHAN_WIDTH_20_NOHT: -+ case NL80211_CHAN_WIDTH_20: -+ width = CH_20_MHZ_WIDTH; -+ sub_ch = NO_EXT_CHANNEL; -+ break; -+ case NL80211_CHAN_WIDTH_40: -+ width = CH_40_MHZ_WIDTH; -+ if (conf->chandef.center_freq1 > channel->center_freq) -+ sub_ch = EXT_CH_ABOVE_CTRL_CH; -+ else -+ sub_ch = EXT_CH_BELOW_CTRL_CH; -+ break; -+ case NL80211_CHAN_WIDTH_80: -+ width = CH_80_MHZ_WIDTH; -+ if (conf->chandef.center_freq1 > channel->center_freq) -+ sub_ch = EXT_CH_ABOVE_CTRL_CH; -+ else -+ sub_ch = EXT_CH_BELOW_CTRL_CH; -+ break; -+ default: -+ return -EINVAL; -+ } -+ -+ if (priv->chip_type == MWL8997) { -+ mwl_fwcmd_get_tx_powers(priv, priv->target_powers, -+ HOSTCMD_ACT_GET_TARGET_TX_PWR, -+ channel->hw_value, band, width, sub_ch); -+ -+ for (i = 0; i < priv->pwr_level; i++) { -+ tmp = priv->target_powers[i]; -+ txpow[i] = ((tmp - reduce_val) > 0) ? -+ (tmp - reduce_val) : 0; -+ } -+ -+ rc = mwl_fwcmd_set_tx_powers(priv, txpow, -+ HOSTCMD_ACT_SET_TARGET_TX_PWR, -+ channel->hw_value, band, -+ width, sub_ch); -+ -+ return rc; -+ } -+ -+ /* search tx power table if exist */ -+ for (index = 0; index < SYSADPT_MAX_NUM_CHANNELS; index++) { -+ struct mwl_tx_pwr_tbl *tx_pwr; -+ -+ tx_pwr = &priv->tx_pwr_tbl[index]; -+ -+ /* do nothing if table is not loaded */ -+ if (tx_pwr->channel == 0) -+ break; -+ -+ if (tx_pwr->channel == channel->hw_value) { -+ priv->cdd = tx_pwr->cdd; -+ priv->txantenna2 = tx_pwr->txantenna2; -+ -+ if (tx_pwr->setcap) -+ priv->powinited = MWL_POWER_INIT_1; -+ else -+ priv->powinited = MWL_POWER_INIT_2; -+ -+ for (i = 0; i < priv->pwr_level; i++) { -+ if (tx_pwr->setcap) -+ priv->max_tx_pow[i] = -+ tx_pwr->tx_power[i]; -+ else -+ priv->target_powers[i] = -+ tx_pwr->tx_power[i]; -+ } -+ -+ found = 1; -+ break; -+ } -+ } -+ -+ if ((priv->powinited & MWL_POWER_INIT_2) == 0) { -+ mwl_fwcmd_get_tx_powers(priv, priv->max_tx_pow, -+ HOSTCMD_ACT_GEN_GET_LIST, -+ channel->hw_value, band, width, sub_ch); -+ -+ priv->powinited |= MWL_POWER_INIT_2; -+ } -+ -+ if ((priv->powinited & MWL_POWER_INIT_1) == 0) { -+ mwl_fwcmd_get_tx_powers(priv, priv->target_powers, -+ HOSTCMD_ACT_GEN_GET_LIST, -+ channel->hw_value, band, width, sub_ch); -+ -+ priv->powinited |= MWL_POWER_INIT_1; -+ } -+ -+ for (i = 0; i < priv->pwr_level; i++) { -+ if (found) { -+ if ((priv->tx_pwr_tbl[index].setcap) && -+ (priv->tx_pwr_tbl[index].tx_power[i] > -+ priv->max_tx_pow[i])) -+ tmp = priv->max_tx_pow[i]; -+ else -+ tmp = priv->tx_pwr_tbl[index].tx_power[i]; -+ } else { -+ if (priv->target_powers[i] > priv->max_tx_pow[i]) -+ tmp = priv->max_tx_pow[i]; -+ else -+ tmp = priv->target_powers[i]; -+ } -+ -+ txpow[i] = ((tmp - reduce_val) > 0) ? (tmp - reduce_val) : 0; -+ } -+ -+ rc = mwl_fwcmd_set_tx_powers(priv, txpow, HOSTCMD_ACT_GEN_SET_LIST, -+ channel->hw_value, band, width, sub_ch); -+ -+ return rc; -+} -+ -+int mwl_fwcmd_rf_antenna(struct ieee80211_hw *hw, int dir, int antenna) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_802_11_rf_antenna *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_802_11_rf_antenna *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_802_11_RF_ANTENNA); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ -+ pcmd->action = cpu_to_le16(dir); -+ -+ if (dir == WL_ANTENNATYPE_RX) { -+ u8 rx_antenna; -+ -+ if (priv->chip_type == MWL8964) { -+ if (antenna == ANTENNA_RX_4_AUTO) -+ rx_antenna = 0xf; -+ else if (antenna == ANTENNA_RX_3) -+ rx_antenna = 7; -+ else if (antenna == ANTENNA_RX_2) -+ rx_antenna = 4; -+ else -+ rx_antenna = 1; -+ -+ pcmd->antenna_mode = cpu_to_le16(rx_antenna); -+ } else { -+ rx_antenna = 4; -+ -+ if (antenna != 0) -+ pcmd->antenna_mode = cpu_to_le16(antenna); -+ else -+ pcmd->antenna_mode = cpu_to_le16(rx_antenna); -+ } -+ } else { -+ u8 tx_antenna = 0xf; -+ -+ if (antenna != 0) -+ pcmd->antenna_mode = cpu_to_le16(antenna); -+ else -+ pcmd->antenna_mode = cpu_to_le16(tx_antenna); -+ } -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_802_11_RF_ANTENNA)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_broadcast_ssid_enable(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, bool enable) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct mwl_vif *mwl_vif; -+ struct hostcmd_cmd_broadcast_ssid_enable *pcmd; -+ -+ mwl_vif = mwl_dev_get_vif(vif); -+ -+ pcmd = (struct hostcmd_cmd_broadcast_ssid_enable *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_BROADCAST_SSID_ENABLE); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->cmd_hdr.macid = mwl_vif->macid; -+ pcmd->enable = cpu_to_le32(enable); -+ if (priv->chip_type == MWL8997) -+ pcmd->hidden_ssid_info = enable ? 0 : 2; -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_BROADCAST_SSID_ENABLE)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_set_cfg_data(struct ieee80211_hw *hw, u16 type) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_set_cfg *pcmd; -+ -+ if (!priv->cal_data) -+ return 0; -+ -+ pcmd = (struct hostcmd_cmd_set_cfg *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->data_len = mwl_fwcmd_parse_cal_cfg(priv->cal_data->data, -+ priv->cal_data->size, -+ pcmd->data); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_CFG); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd) + -+ le16_to_cpu(pcmd->data_len) - sizeof(pcmd->data)); -+ pcmd->action = cpu_to_le16(HOSTCMD_ACT_GEN_SET); -+ pcmd->type = cpu_to_le16(type); -+ -+ utils_dump_data_debug("CalData:", pcmd->data, -+ le16_to_cpu(pcmd->data_len)); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_CFG)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ release_firmware(priv->cal_data); -+ priv->cal_data = NULL; -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ release_firmware(priv->cal_data); -+ priv->cal_data = NULL; -+ -+ return 0; -+} -+ -+int mwl_fwcmd_set_rf_channel(struct ieee80211_hw *hw, -+ struct ieee80211_conf *conf) -+{ -+ struct ieee80211_channel *channel = conf->chandef.chan; -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_set_rf_channel *pcmd; -+ u32 chnl_flags, freq_band, chnl_width, act_primary; -+ -+ pcmd = (struct hostcmd_cmd_set_rf_channel *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ if (priv->chip_type == MWL8997) { -+ memset(pcmd, 0x00, -+ sizeof(struct hostcmd_cmd_set_rf_channel_kf2)); -+ pcmd->cmd_hdr.len = cpu_to_le16( -+ sizeof(struct hostcmd_cmd_set_rf_channel_kf2)); -+ } else { -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ } -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_RF_CHANNEL); -+ pcmd->action = cpu_to_le16(WL_SET); -+ pcmd->curr_chnl = channel->hw_value; -+ -+ if (channel->band == NL80211_BAND_2GHZ) { -+ freq_band = FREQ_BAND_2DOT4GHZ; -+ } else if (channel->band == NL80211_BAND_5GHZ) { -+ freq_band = FREQ_BAND_5GHZ; -+ } else { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EINVAL; -+ } -+ -+ switch (conf->chandef.width) { -+ case NL80211_CHAN_WIDTH_20_NOHT: -+ case NL80211_CHAN_WIDTH_20: -+ chnl_width = CH_20_MHZ_WIDTH; -+ act_primary = ACT_PRIMARY_CHAN_0; -+ break; -+ case NL80211_CHAN_WIDTH_40: -+ chnl_width = CH_40_MHZ_WIDTH; -+ if (conf->chandef.center_freq1 > channel->center_freq) -+ act_primary = ACT_PRIMARY_CHAN_0; -+ else -+ act_primary = ACT_PRIMARY_CHAN_1; -+ break; -+ case NL80211_CHAN_WIDTH_80: -+ chnl_width = CH_80_MHZ_WIDTH; -+ act_primary = -+ mwl_fwcmd_get_80m_pri_chnl(pcmd->curr_chnl); -+ break; -+ case NL80211_CHAN_WIDTH_160: -+ chnl_width = CH_160_MHZ_WIDTH; -+ act_primary = -+ mwl_fwcmd_get_160m_pri_chnl(pcmd->curr_chnl); -+ break; -+ default: -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EINVAL; -+ } -+ -+ chnl_flags = (freq_band & FREQ_BAND_MASK) | -+ ((chnl_width << CHNL_WIDTH_SHIFT) & CHNL_WIDTH_MASK) | -+ ((act_primary << ACT_PRIMARY_SHIFT) & ACT_PRIMARY_MASK); -+ -+ pcmd->chnl_flags = cpu_to_le32(chnl_flags); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_RF_CHANNEL)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ if (pcmd->cmd_hdr.result != 0) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EINVAL; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ if (priv->sw_scanning) { -+ priv->survey_info_idx++; -+ mwl_fwcmd_get_survey(hw, priv->survey_info_idx); -+ } else { -+ mwl_fwcmd_get_survey(hw, 0); -+ memset(&priv->cur_survey_info, 0, -+ sizeof(struct mwl_survey_info)); -+ } -+ -+ return 0; -+} -+ -+int mwl_fwcmd_set_aid(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, u8 *bssid, u16 aid) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct mwl_vif *mwl_vif; -+ struct hostcmd_cmd_set_aid *pcmd; -+ -+ mwl_vif = mwl_dev_get_vif(vif); -+ -+ pcmd = (struct hostcmd_cmd_set_aid *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_AID); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->cmd_hdr.macid = mwl_vif->macid; -+ pcmd->aid = cpu_to_le16(aid); -+ ether_addr_copy(pcmd->mac_addr, bssid); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_AID)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_set_infra_mode(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct mwl_vif *mwl_vif; -+ struct hostcmd_cmd_set_infra_mode *pcmd; -+ -+ mwl_vif = mwl_dev_get_vif(vif); -+ -+ pcmd = (struct hostcmd_cmd_set_infra_mode *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_INFRA_MODE); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->cmd_hdr.macid = mwl_vif->macid; -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_INFRA_MODE)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_set_rts_threshold(struct ieee80211_hw *hw, int threshold) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_802_11_rts_thsd *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_802_11_rts_thsd *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_802_11_RTS_THSD); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->action = cpu_to_le16(WL_SET); -+ pcmd->threshold = cpu_to_le16(threshold); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_802_11_RTS_THSD)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_set_edca_params(struct ieee80211_hw *hw, u8 index, -+ u16 cw_min, u16 cw_max, u8 aifs, u16 txop) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_set_edca_params *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_set_edca_params *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_EDCA_PARAMS); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ -+ pcmd->action = cpu_to_le16(0xffff); -+ pcmd->txop = cpu_to_le16(txop); -+ pcmd->cw_max = cpu_to_le32(cw_max); -+ pcmd->cw_min = cpu_to_le32(cw_min); -+ pcmd->aifsn = aifs; -+ pcmd->txq_num = index; -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_EDCA_PARAMS)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_set_radar_detect(struct ieee80211_hw *hw, u16 action) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_802_11h_detect_radar *pcmd; -+ u16 radar_type = RADAR_TYPE_CODE_0; -+ u8 channel = hw->conf.chandef.chan->hw_value; -+ -+ pcmd = (struct hostcmd_cmd_802_11h_detect_radar *)&priv->pcmd_buf[0]; -+ -+ if (priv->dfs_region == NL80211_DFS_JP) { -+ if (channel >= 52 && channel <= 64) -+ radar_type = RADAR_TYPE_CODE_53; -+ else if (channel >= 100 && channel <= 140) -+ radar_type = RADAR_TYPE_CODE_56; -+ else -+ radar_type = RADAR_TYPE_CODE_0; -+ } else if (priv->dfs_region == NL80211_DFS_ETSI) { -+ radar_type = RADAR_TYPE_CODE_ETSI; -+ } -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_802_11H_DETECT_RADAR); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->action = cpu_to_le16(action); -+ pcmd->radar_type_code = cpu_to_le16(radar_type); -+ pcmd->min_chirp_cnt = cpu_to_le16(priv->dfs_chirp_count_min); -+ pcmd->chirp_time_intvl = cpu_to_le16(priv->dfs_chirp_time_interval); -+ pcmd->pw_filter = cpu_to_le16(priv->dfs_pw_filter); -+ pcmd->min_num_radar = cpu_to_le16(priv->dfs_min_num_radar); -+ pcmd->pri_min_num = cpu_to_le16(priv->dfs_min_pri_count); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_802_11H_DETECT_RADAR)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_set_wmm_mode(struct ieee80211_hw *hw, bool enable) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_set_wmm_mode *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_set_wmm_mode *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_WMM_MODE); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->action = cpu_to_le16(enable ? WL_ENABLE : WL_DISABLE); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_WMM_MODE)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_ht_guard_interval(struct ieee80211_hw *hw, u32 gi_type) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_ht_guard_interval *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_ht_guard_interval *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_HT_GUARD_INTERVAL); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->action = cpu_to_le32(WL_SET); -+ pcmd->gi_type = cpu_to_le32(gi_type); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_HT_GUARD_INTERVAL)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_use_fixed_rate(struct ieee80211_hw *hw, int mcast, int mgmt) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_set_fixed_rate *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_set_fixed_rate *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_FIXED_RATE); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ -+ pcmd->action = cpu_to_le32(HOSTCMD_ACT_NOT_USE_FIXED_RATE); -+ pcmd->multicast_rate = mcast; -+ pcmd->management_rate = mgmt; -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_FIXED_RATE)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_set_linkadapt_cs_mode(struct ieee80211_hw *hw, u16 cs_mode) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_set_linkadapt_cs_mode *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_set_linkadapt_cs_mode *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_LINKADAPT_CS_MODE); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->action = cpu_to_le16(HOSTCMD_ACT_GEN_SET); -+ pcmd->cs_mode = cpu_to_le16(cs_mode); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_LINKADAPT_CS_MODE)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_dump_otp_data(struct ieee80211_hw *hw) -+{ -+ int otp_data_len; -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_dump_otp_data *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_dump_otp_data *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_DUMP_OTP_DATA); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_DUMP_OTP_DATA)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ otp_data_len = pcmd->cmd_hdr.len - cpu_to_le16(sizeof(*pcmd)); -+ -+ if (otp_data_len <= SYSADPT_OTP_BUF_SIZE) { -+ wiphy_info(hw->wiphy, "OTP data len = %d\n", otp_data_len); -+ priv->otp_data.len = otp_data_len; -+ memcpy(priv->otp_data.buf, pcmd->pload, otp_data_len); -+ } else { -+ wiphy_err(hw->wiphy, "Driver OTP buf size is less\n"); -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_set_rate_adapt_mode(struct ieee80211_hw *hw, u16 mode) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_set_rate_adapt_mode *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_set_rate_adapt_mode *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_RATE_ADAPT_MODE); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->action = cpu_to_le16(WL_SET); -+ pcmd->rate_adapt_mode = cpu_to_le16(mode); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_RATE_ADAPT_MODE)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_set_mac_addr_client(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, u8 *mac_addr) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct mwl_vif *mwl_vif; -+ struct hostcmd_cmd_set_mac_addr *pcmd; -+ -+ mwl_vif = mwl_dev_get_vif(vif); -+ -+ pcmd = (struct hostcmd_cmd_set_mac_addr *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_MAC_ADDR); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->cmd_hdr.macid = mwl_vif->macid; -+ pcmd->mac_type = cpu_to_le16(WL_MAC_TYPE_SECONDARY_CLIENT); -+ ether_addr_copy(pcmd->mac_addr, mac_addr); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_MAC_ADDR)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_get_watchdog_bitmap(struct ieee80211_hw *hw, u8 *bitmap) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_get_watchdog_bitmap *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_get_watchdog_bitmap *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_GET_WATCHDOG_BITMAP); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_GET_WATCHDOG_BITMAP)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ *bitmap = pcmd->watchdog_bitmap; -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_remove_mac_addr(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, u8 *mac_addr) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct mwl_vif *mwl_vif; -+ struct hostcmd_cmd_set_mac_addr *pcmd; -+ -+ mwl_vif = mwl_dev_get_vif(vif); -+ -+ pcmd = (struct hostcmd_cmd_set_mac_addr *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_DEL_MAC_ADDR); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->cmd_hdr.macid = mwl_vif->macid; -+ ether_addr_copy(pcmd->mac_addr, mac_addr); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_DEL_MAC_ADDR)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_bss_start(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, bool enable) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct mwl_vif *mwl_vif; -+ struct hostcmd_cmd_bss_start *pcmd; -+ -+ mwl_vif = mwl_dev_get_vif(vif); -+ -+ if (enable && (priv->running_bsses & (1 << mwl_vif->macid))) -+ return 0; -+ -+ if (!enable && !(priv->running_bsses & (1 << mwl_vif->macid))) -+ return 0; -+ -+ pcmd = (struct hostcmd_cmd_bss_start *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_BSS_START); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->cmd_hdr.macid = mwl_vif->macid; -+ -+ if (enable) { -+ pcmd->enable = cpu_to_le32(WL_ENABLE); -+ } else { -+ if (mwl_vif->macid == 0) -+ pcmd->enable = cpu_to_le32(WL_DISABLE); -+ else -+ pcmd->enable = cpu_to_le32(WL_DISABLE_VMAC); -+ } -+ if (priv->chip_type == MWL8964) -+ pcmd->amsdu = MWL_AMSDU_SIZE_11K; -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_BSS_START)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ if (enable) -+ priv->running_bsses |= (1 << mwl_vif->macid); -+ else -+ priv->running_bsses &= ~(1 << mwl_vif->macid); -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_set_beacon(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, u8 *beacon, int len) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct mwl_vif *mwl_vif; -+ struct beacon_info *b_inf; -+ int rc; -+ -+ mwl_vif = mwl_dev_get_vif(vif); -+ b_inf = &mwl_vif->beacon_info; -+ -+ mwl_fwcmd_parse_beacon(priv, mwl_vif, beacon, len); -+ -+ if (!b_inf->valid) -+ goto err; -+ -+ if (mwl_fwcmd_set_ies(priv, mwl_vif)) -+ goto err; -+ -+ if (mwl_fwcmd_set_wsc_ie(hw, b_inf->ie_wsc_len, b_inf->ie_wsc_ptr)) -+ goto err; -+ -+ if (mwl_fwcmd_set_ap_beacon(priv, mwl_vif, &vif->bss_conf)) -+ goto err; -+ -+ if (b_inf->cap_info & WLAN_CAPABILITY_SPECTRUM_MGMT) -+ rc = mwl_fwcmd_set_spectrum_mgmt(priv, true); -+ else -+ rc = mwl_fwcmd_set_spectrum_mgmt(priv, false); -+ if (rc) -+ goto err; -+ -+ if (b_inf->power_constraint) -+ rc = mwl_fwcmd_set_power_constraint(priv, -+ b_inf->power_constraint); -+ if (rc) -+ goto err; -+ -+ if (mwl_fwcmd_set_country_code(priv, mwl_vif, &vif->bss_conf)) -+ goto err; -+ -+ b_inf->valid = false; -+ -+ return 0; -+ -+err: -+ -+ b_inf->valid = false; -+ -+ return -EIO; -+} -+ -+int mwl_fwcmd_set_new_stn_add(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, -+ struct ieee80211_sta *sta) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct mwl_vif *mwl_vif; -+ struct mwl_sta *sta_info; -+ struct hostcmd_cmd_set_new_stn *pcmd; -+ u32 rates; -+ -+ mwl_vif = mwl_dev_get_vif(vif); -+ sta_info = mwl_dev_get_sta(sta); -+ -+ pcmd = (struct hostcmd_cmd_set_new_stn *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_NEW_STN); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->cmd_hdr.macid = mwl_vif->macid; -+ -+ pcmd->action = cpu_to_le16(HOSTCMD_ACT_STA_ACTION_ADD); -+ pcmd->aid = cpu_to_le16(sta->aid); -+ pcmd->stn_id = cpu_to_le16(sta_info->stnid); -+ if (priv->chip_type == MWL8997) -+ pcmd->if_type = cpu_to_le16(vif->type); -+ else -+ pcmd->if_type = cpu_to_le16(1); -+ ether_addr_copy(pcmd->mac_addr, sta->addr); -+ -+ if (hw->conf.chandef.chan->band == NL80211_BAND_2GHZ) -+ rates = sta->supp_rates[NL80211_BAND_2GHZ]; -+ else -+ rates = sta->supp_rates[NL80211_BAND_5GHZ] << 5; -+ pcmd->peer_info.legacy_rate_bitmap = cpu_to_le32(rates); -+ -+ if (sta->ht_cap.ht_supported) { -+ int i; -+ -+ for (i = 0; i < 4; i++) { -+ if (i < sta->rx_nss) { -+ pcmd->peer_info.ht_rates[i] = -+ sta->ht_cap.mcs.rx_mask[i]; -+ } else { -+ pcmd->peer_info.ht_rates[i] = 0; -+ } -+ } -+ pcmd->peer_info.ht_cap_info = cpu_to_le16(sta->ht_cap.cap); -+ pcmd->peer_info.mac_ht_param_info = -+ (sta->ht_cap.ampdu_factor & 3) | -+ ((sta->ht_cap.ampdu_density & 7) << 2); -+ } -+ -+ if (sta->vht_cap.vht_supported) { -+ u32 rx_mcs_map_mask = 0; -+ -+ rx_mcs_map_mask = ((0x0000FFFF) >> (sta->rx_nss * 2)) -+ << (sta->rx_nss * 2); -+ pcmd->peer_info.vht_max_rx_mcs = -+ cpu_to_le32((*((u32 *) -+ &sta->vht_cap.vht_mcs.rx_mcs_map)) | rx_mcs_map_mask); -+ pcmd->peer_info.vht_cap = cpu_to_le32(sta->vht_cap.cap); -+ pcmd->peer_info.vht_rx_channel_width = sta->bandwidth; -+ } -+ -+ pcmd->is_qos_sta = sta->wme; -+ pcmd->qos_info = ((sta->uapsd_queues << 4) | (sta->max_sp << 1)); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_NEW_STN)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ if (vif->type == NL80211_IFTYPE_STATION) { -+ ether_addr_copy(pcmd->mac_addr, mwl_vif->sta_mac); -+ pcmd->aid = cpu_to_le16(sta->aid + 1); -+ pcmd->stn_id = cpu_to_le16(sta_info->sta_stnid); -+ pcmd->if_type = cpu_to_le16(0); -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_NEW_STN)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_set_new_stn_add_sc4(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, -+ struct ieee80211_sta *sta, -+ u32 wds) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct mwl_vif *mwl_vif; -+ struct mwl_sta *sta_info; -+ struct hostcmd_cmd_set_new_stn_sc4 *pcmd; -+ u32 rates; -+ -+ mwl_vif = mwl_dev_get_vif(vif); -+ sta_info = mwl_dev_get_sta(sta); -+ -+ pcmd = (struct hostcmd_cmd_set_new_stn_sc4 *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_NEW_STN); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->cmd_hdr.macid = mwl_vif->macid; -+ -+ pcmd->action = cpu_to_le16(HOSTCMD_ACT_STA_ACTION_ADD); -+ pcmd->aid = cpu_to_le16(sta->aid); -+ pcmd->stn_id = cpu_to_le16(sta_info->stnid); -+ ether_addr_copy(pcmd->mac_addr, sta->addr); -+ -+ if (hw->conf.chandef.chan->band == NL80211_BAND_2GHZ) -+ rates = sta->supp_rates[NL80211_BAND_2GHZ]; -+ else -+ rates = sta->supp_rates[NL80211_BAND_5GHZ] << 5; -+ pcmd->peer_info.legacy_rate_bitmap = cpu_to_le32(rates); -+ -+ if (sta->ht_cap.ht_supported) { -+ int i; -+ -+ for (i = 0; i < 4; i++) { -+ if (i < sta->rx_nss) { -+ pcmd->peer_info.ht_rates[i] = -+ sta->ht_cap.mcs.rx_mask[i]; -+ } else { -+ pcmd->peer_info.ht_rates[i] = 0; -+ } -+ } -+ pcmd->peer_info.ht_cap_info = cpu_to_le16(sta->ht_cap.cap); -+ pcmd->peer_info.mac_ht_param_info = -+ (sta->ht_cap.ampdu_factor & 3) | -+ ((sta->ht_cap.ampdu_density & 7) << 2); -+ } -+ -+ if (sta->vht_cap.vht_supported) { -+ u32 rx_mcs_map_mask = 0; -+ -+ rx_mcs_map_mask = ((0x0000FFFF) >> (sta->rx_nss * 2)) -+ << (sta->rx_nss * 2); -+ pcmd->peer_info.vht_max_rx_mcs = -+ cpu_to_le32((*((u32 *) -+ &sta->vht_cap.vht_mcs.rx_mcs_map)) | rx_mcs_map_mask); -+ pcmd->peer_info.vht_cap = cpu_to_le32(sta->vht_cap.cap); -+ pcmd->peer_info.vht_rx_channel_width = sta->bandwidth; -+ } -+ -+ pcmd->is_qos_sta = sta->wme; -+ pcmd->qos_info = ((sta->uapsd_queues << 4) | (sta->max_sp << 1)); -+ pcmd->wds = cpu_to_le32(wds); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_NEW_STN)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ if (vif->type == NL80211_IFTYPE_STATION) { -+ ether_addr_copy(pcmd->mac_addr, mwl_vif->sta_mac); -+ pcmd->aid = cpu_to_le16(sta->aid + 1); -+ pcmd->stn_id = cpu_to_le16(sta_info->sta_stnid); -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_NEW_STN)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_set_new_stn_wds_sc4(struct ieee80211_hw *hw, u8 *addr) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_set_new_stn_sc4 *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_set_new_stn_sc4 *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_NEW_STN); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ -+ pcmd->action = cpu_to_le16(HOSTCMD_ACT_STA_ACTION_MODIFY); -+ ether_addr_copy(pcmd->mac_addr, addr); -+ pcmd->wds = cpu_to_le32(WDS_MODE); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_NEW_STN)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_set_new_stn_add_self(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct mwl_vif *mwl_vif; -+ struct hostcmd_cmd_set_new_stn *pcmd; -+ -+ mwl_vif = mwl_dev_get_vif(vif); -+ -+ pcmd = (struct hostcmd_cmd_set_new_stn *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ if (priv->chip_type == MWL8964) { -+ memset(pcmd, 0x00, sizeof(struct hostcmd_cmd_set_new_stn_sc4)); -+ pcmd->cmd_hdr.len = -+ cpu_to_le16(sizeof(struct hostcmd_cmd_set_new_stn_sc4)); -+ } else { -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ } -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_NEW_STN); -+ pcmd->cmd_hdr.macid = mwl_vif->macid; -+ -+ pcmd->action = cpu_to_le16(HOSTCMD_ACT_STA_ACTION_ADD); -+ ether_addr_copy(pcmd->mac_addr, vif->addr); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_NEW_STN)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_set_new_stn_del(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, u8 *addr) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct mwl_vif *mwl_vif; -+ struct hostcmd_cmd_set_new_stn *pcmd; -+ -+ mwl_vif = mwl_dev_get_vif(vif); -+ -+ pcmd = (struct hostcmd_cmd_set_new_stn *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ if (priv->chip_type == MWL8964) { -+ memset(pcmd, 0x00, sizeof(struct hostcmd_cmd_set_new_stn_sc4)); -+ pcmd->cmd_hdr.len = -+ cpu_to_le16(sizeof(struct hostcmd_cmd_set_new_stn_sc4)); -+ } else { -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ } -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_NEW_STN); -+ pcmd->cmd_hdr.macid = mwl_vif->macid; -+ -+ pcmd->action = cpu_to_le16(HOSTCMD_ACT_STA_ACTION_REMOVE); -+ ether_addr_copy(pcmd->mac_addr, addr); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_NEW_STN)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ if (vif->type == NL80211_IFTYPE_STATION) { -+ ether_addr_copy(pcmd->mac_addr, mwl_vif->sta_mac); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_NEW_STN)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_set_apmode(struct ieee80211_hw *hw, u8 apmode) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_set_apmode *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_set_apmode *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_APMODE); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->apmode = apmode; -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_APMODE)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_set_switch_channel(struct ieee80211_hw *hw, -+ struct ieee80211_channel_switch *ch_switch) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_set_switch_channel *pcmd; -+ struct cfg80211_chan_def *chandef = &ch_switch->chandef; -+ struct ieee80211_channel *channel = chandef->chan; -+ u32 chnl_flags, freq_band, chnl_width, act_primary, sec_chnl_offset; -+ -+ if (priv->csa_active) -+ return 0; -+ -+ if (channel->band == NL80211_BAND_2GHZ) -+ freq_band = FREQ_BAND_2DOT4GHZ; -+ else if (channel->band == NL80211_BAND_5GHZ) -+ freq_band = FREQ_BAND_5GHZ; -+ else -+ return -EINVAL; -+ -+ switch (chandef->width) { -+ case NL80211_CHAN_WIDTH_20_NOHT: -+ case NL80211_CHAN_WIDTH_20: -+ chnl_width = CH_20_MHZ_WIDTH; -+ act_primary = ACT_PRIMARY_CHAN_0; -+ sec_chnl_offset = IEEE80211_HT_PARAM_CHA_SEC_NONE; -+ break; -+ case NL80211_CHAN_WIDTH_40: -+ chnl_width = CH_40_MHZ_WIDTH; -+ if (chandef->center_freq1 > channel->center_freq) { -+ act_primary = ACT_PRIMARY_CHAN_0; -+ sec_chnl_offset = IEEE80211_HT_PARAM_CHA_SEC_ABOVE; -+ } else { -+ act_primary = ACT_PRIMARY_CHAN_1; -+ sec_chnl_offset = IEEE80211_HT_PARAM_CHA_SEC_BELOW; -+ } -+ break; -+ case NL80211_CHAN_WIDTH_80: -+ chnl_width = CH_80_MHZ_WIDTH; -+ act_primary = -+ mwl_fwcmd_get_80m_pri_chnl(channel->hw_value); -+ if ((act_primary == ACT_PRIMARY_CHAN_0) || -+ (act_primary == ACT_PRIMARY_CHAN_2)) -+ sec_chnl_offset = IEEE80211_HT_PARAM_CHA_SEC_ABOVE; -+ else -+ sec_chnl_offset = IEEE80211_HT_PARAM_CHA_SEC_BELOW; -+ break; -+ default: -+ return -EINVAL; -+ } -+ -+ chnl_flags = (freq_band & FREQ_BAND_MASK) | -+ ((chnl_width << CHNL_WIDTH_SHIFT) & CHNL_WIDTH_MASK) | -+ ((act_primary << ACT_PRIMARY_SHIFT) & ACT_PRIMARY_MASK); -+ -+ pcmd = (struct hostcmd_cmd_set_switch_channel *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_SWITCH_CHANNEL); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->next_11h_chnl = cpu_to_le32(channel->hw_value); -+ pcmd->mode = cpu_to_le32(ch_switch->block_tx); -+ pcmd->init_count = cpu_to_le32(ch_switch->count + 1); -+ pcmd->chnl_flags = cpu_to_le32(chnl_flags); -+ pcmd->next_ht_extchnl_offset = cpu_to_le32(sec_chnl_offset); -+ pcmd->dfs_test_mode = cpu_to_le32(priv->dfs_test); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_SWITCH_CHANNEL)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ priv->csa_active = true; -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_update_encryption_enable(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, -+ u8 *addr, u8 encr_type) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct mwl_vif *mwl_vif; -+ struct hostcmd_cmd_update_encryption *pcmd; -+ -+ mwl_vif = mwl_dev_get_vif(vif); -+ -+ pcmd = (struct hostcmd_cmd_update_encryption *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_UPDATE_ENCRYPTION); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->cmd_hdr.macid = mwl_vif->macid; -+ -+ pcmd->action_type = cpu_to_le32(ENCR_ACTION_ENABLE_HW_ENCR); -+ ether_addr_copy(pcmd->mac_addr, addr); -+ pcmd->action_data[0] = encr_type; -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_UPDATE_ENCRYPTION)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ if ((vif->type == NL80211_IFTYPE_STATION) && -+ (priv->chip_type != MWL8964)) { -+ if (ether_addr_equal(mwl_vif->bssid, addr)) -+ ether_addr_copy(pcmd->mac_addr, mwl_vif->sta_mac); -+ else -+ ether_addr_copy(pcmd->mac_addr, mwl_vif->bssid); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_UPDATE_ENCRYPTION)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_encryption_set_key(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, u8 *addr, -+ struct ieee80211_key_conf *key) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct mwl_vif *mwl_vif; -+ struct hostcmd_cmd_set_key *pcmd; -+ int rc; -+ int keymlen; -+ u32 action; -+ u8 idx; -+ -+ mwl_vif = mwl_dev_get_vif(vif); -+ -+ pcmd = (struct hostcmd_cmd_set_key *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_UPDATE_ENCRYPTION); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->cmd_hdr.macid = mwl_vif->macid; -+ -+ rc = mwl_fwcmd_encryption_set_cmd_info(pcmd, addr, key); -+ if (rc) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ if (rc != 1) -+ wiphy_err(hw->wiphy, "encryption not support\n"); -+ return rc; -+ } -+ -+ idx = key->keyidx; -+ -+ if (key->flags & IEEE80211_KEY_FLAG_PAIRWISE) -+ action = ENCR_ACTION_TYPE_SET_KEY; -+ else { -+ action = ENCR_ACTION_TYPE_SET_GROUP_KEY; -+ if (vif->type == NL80211_IFTYPE_MESH_POINT && -+ !ether_addr_equal(mwl_vif->bssid, addr)) -+ pcmd->key_param.key_info |= -+ cpu_to_le32(ENCR_KEY_FLAG_RXGROUPKEY); -+ } -+ -+ switch (key->cipher) { -+ case WLAN_CIPHER_SUITE_WEP40: -+ case WLAN_CIPHER_SUITE_WEP104: -+ if (!mwl_vif->wep_key_conf[idx].enabled) { -+ memcpy(mwl_vif->wep_key_conf[idx].key, key, -+ sizeof(*key) + key->keylen); -+ mwl_vif->wep_key_conf[idx].enabled = 1; -+ } -+ -+ keymlen = key->keylen; -+ action = ENCR_ACTION_TYPE_SET_KEY; -+ break; -+ case WLAN_CIPHER_SUITE_TKIP: -+ keymlen = MAX_ENCR_KEY_LENGTH + 2 * MIC_KEY_LENGTH; -+ break; -+ case WLAN_CIPHER_SUITE_CCMP: -+ keymlen = key->keylen; -+ break; -+ default: -+ mutex_unlock(&priv->fwcmd_mutex); -+ wiphy_err(hw->wiphy, "encryption not support\n"); -+ return -ENOTSUPP; -+ } -+ -+ memcpy((void *)&pcmd->key_param.key, key->key, keymlen); -+ pcmd->action_type = cpu_to_le32(action); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_UPDATE_ENCRYPTION)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ if (vif->type == NL80211_IFTYPE_STATION) { -+ if (ether_addr_equal(mwl_vif->bssid, addr)) -+ ether_addr_copy(pcmd->key_param.mac_addr, -+ mwl_vif->sta_mac); -+ else -+ ether_addr_copy(pcmd->key_param.mac_addr, -+ mwl_vif->bssid); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_UPDATE_ENCRYPTION)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_encryption_remove_key(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, u8 *addr, -+ struct ieee80211_key_conf *key) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct mwl_vif *mwl_vif; -+ struct hostcmd_cmd_set_key *pcmd; -+ int rc; -+ -+ mwl_vif = mwl_dev_get_vif(vif); -+ -+ pcmd = (struct hostcmd_cmd_set_key *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_UPDATE_ENCRYPTION); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->cmd_hdr.macid = mwl_vif->macid; -+ -+ rc = mwl_fwcmd_encryption_set_cmd_info(pcmd, addr, key); -+ if (rc) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ if (rc != 1) -+ wiphy_err(hw->wiphy, "encryption not support\n"); -+ return rc; -+ } -+ -+ pcmd->action_type = cpu_to_le32(ENCR_ACTION_TYPE_REMOVE_KEY); -+ -+ if (key->cipher == WLAN_CIPHER_SUITE_WEP40 || -+ key->cipher == WLAN_CIPHER_SUITE_WEP104) -+ mwl_vif->wep_key_conf[key->keyidx].enabled = 0; -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_UPDATE_ENCRYPTION)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_check_ba(struct ieee80211_hw *hw, -+ struct mwl_ampdu_stream *stream, -+ struct ieee80211_vif *vif, -+ u32 direction) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct mwl_vif *mwl_vif; -+ struct hostcmd_cmd_bastream *pcmd; -+ u32 ba_flags, ba_type, ba_direction; -+ -+ mwl_vif = mwl_dev_get_vif(vif); -+ -+ pcmd = (struct hostcmd_cmd_bastream *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_BASTREAM); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->cmd_hdr.macid = mwl_vif->macid; -+ pcmd->cmd_hdr.result = cpu_to_le16(0xffff); -+ -+ pcmd->action_type = cpu_to_le32(BA_CHECK_STREAM); -+ ether_addr_copy(&pcmd->ba_info.create_params.peer_mac_addr[0], -+ stream->sta->addr); -+ pcmd->ba_info.create_params.tid = stream->tid; -+ ba_type = BA_FLAG_IMMEDIATE_TYPE; -+ ba_direction = direction; -+ ba_flags = (ba_type & BA_TYPE_MASK) | -+ ((ba_direction << BA_DIRECTION_SHIFT) & BA_DIRECTION_MASK); -+ pcmd->ba_info.create_params.flags = cpu_to_le32(ba_flags); -+ pcmd->ba_info.create_params.queue_id = stream->idx; -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_BASTREAM)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ if (pcmd->cmd_hdr.result != 0) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EINVAL; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_create_ba(struct ieee80211_hw *hw, -+ struct mwl_ampdu_stream *stream, -+ struct ieee80211_vif *vif, -+ u32 direction, u8 buf_size, u16 seqno, bool amsdu) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct mwl_vif *mwl_vif; -+ struct hostcmd_cmd_bastream *pcmd; -+ u32 ba_flags, ba_type, ba_direction; -+ -+ mwl_vif = mwl_dev_get_vif(vif); -+ -+ pcmd = (struct hostcmd_cmd_bastream *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_BASTREAM); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->cmd_hdr.macid = mwl_vif->macid; -+ pcmd->cmd_hdr.result = cpu_to_le16(0xffff); -+ -+ pcmd->action_type = cpu_to_le32(BA_CREATE_STREAM); -+ pcmd->ba_info.create_params.bar_thrs = cpu_to_le32(buf_size); -+ pcmd->ba_info.create_params.window_size = cpu_to_le32(buf_size); -+ pcmd->ba_info.create_params.idle_thrs = cpu_to_le32(0x22000); -+ ether_addr_copy(&pcmd->ba_info.create_params.peer_mac_addr[0], -+ stream->sta->addr); -+ pcmd->ba_info.create_params.tid = stream->tid; -+ ba_direction = direction; -+ if (priv->chip_type == MWL8964) { -+ ba_type = amsdu ? MWL_AMSDU_SIZE_11K : 0; -+ ba_flags = (ba_type & BA_TYPE_MASK_NDP) | -+ ((ba_direction << BA_DIRECTION_SHIFT_NDP) & -+ BA_DIRECTION_MASK_NDP); -+ } else { -+ ba_type = BA_FLAG_IMMEDIATE_TYPE; -+ ba_flags = (ba_type & BA_TYPE_MASK) | -+ ((ba_direction << BA_DIRECTION_SHIFT) & -+ BA_DIRECTION_MASK); -+ } -+ pcmd->ba_info.create_params.flags = cpu_to_le32(ba_flags); -+ pcmd->ba_info.create_params.queue_id = stream->idx; -+ pcmd->ba_info.create_params.param_info = -+ (stream->sta->ht_cap.ampdu_factor & -+ IEEE80211_HT_AMPDU_PARM_FACTOR) | -+ ((stream->sta->ht_cap.ampdu_density << 2) & -+ IEEE80211_HT_AMPDU_PARM_DENSITY); -+ if (direction == BA_FLAG_DIRECTION_UP) { -+ pcmd->ba_info.create_params.reset_seq_no = 0; -+ pcmd->ba_info.create_params.current_seq = cpu_to_le16(seqno); -+ } else { -+ pcmd->ba_info.create_params.reset_seq_no = 1; -+ pcmd->ba_info.create_params.current_seq = cpu_to_le16(0); -+ } -+ if (priv->chip_type == MWL8964 && -+ stream->sta->vht_cap.vht_supported) { -+ pcmd->ba_info.create_params.vht_rx_factor = -+ cpu_to_le32((stream->sta->vht_cap.cap & -+ IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK) >> -+ IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_SHIFT); -+ } -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_BASTREAM)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ if (pcmd->cmd_hdr.result != 0) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ wiphy_err(hw->wiphy, "create ba result error %d\n", -+ le16_to_cpu(pcmd->cmd_hdr.result)); -+ return -EINVAL; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_destroy_ba(struct ieee80211_hw *hw, -+ struct mwl_ampdu_stream *stream, -+ u32 direction) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_bastream *pcmd; -+ u32 ba_flags, ba_type, ba_direction; -+ -+ pcmd = (struct hostcmd_cmd_bastream *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_BASTREAM); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ -+ pcmd->action_type = cpu_to_le32(BA_DESTROY_STREAM); -+ ba_type = 0; -+ ba_direction = direction; -+ if (priv->chip_type == MWL8964) -+ ba_flags = (ba_type & BA_TYPE_MASK_NDP) | -+ ((ba_direction << BA_DIRECTION_SHIFT_NDP) & -+ BA_DIRECTION_MASK_NDP); -+ else -+ ba_flags = (ba_type & BA_TYPE_MASK) | -+ ((ba_direction << BA_DIRECTION_SHIFT) & -+ BA_DIRECTION_MASK); -+ pcmd->ba_info.destroy_params.flags = cpu_to_le32(ba_flags); -+ pcmd->ba_info.destroy_params.fw_ba_context.context = -+ cpu_to_le32(stream->idx); -+ pcmd->ba_info.destroy_params.tid = stream->tid; -+ ether_addr_copy(&pcmd->ba_info.destroy_params.peer_mac_addr[0], -+ stream->sta->addr); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_BASTREAM)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+/* caller must hold priv->stream_lock when calling the stream functions */ -+struct mwl_ampdu_stream *mwl_fwcmd_add_stream(struct ieee80211_hw *hw, -+ struct ieee80211_sta *sta, -+ u8 tid) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct mwl_ampdu_stream *stream; -+ struct mwl_sta *sta_info = mwl_dev_get_sta(sta); -+ int idx; -+ -+ if (priv->chip_type == MWL8964) { -+ idx = ((sta_info->stnid - 1) * SYSADPT_MAX_TID) + tid; -+ -+ if (idx < priv->ampdu_num) { -+ stream = &priv->ampdu[idx]; -+ stream->sta = sta; -+ stream->state = AMPDU_STREAM_NEW; -+ stream->tid = tid; -+ stream->idx = idx; -+ return stream; -+ } -+ } else { -+ for (idx = 0; idx < priv->ampdu_num; idx++) { -+ stream = &priv->ampdu[idx]; -+ -+ if (stream->state == AMPDU_NO_STREAM) { -+ stream->sta = sta; -+ stream->state = AMPDU_STREAM_NEW; -+ stream->tid = tid; -+ stream->idx = idx; -+ return stream; -+ } -+ } -+ } -+ -+ return NULL; -+} -+ -+void mwl_fwcmd_del_sta_streams(struct ieee80211_hw *hw, -+ struct ieee80211_sta *sta) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct mwl_ampdu_stream *stream; -+ struct mwl_sta *sta_info = mwl_dev_get_sta(sta); -+ int i, idx; -+ -+ spin_lock_bh(&priv->stream_lock); -+ if (priv->chip_type == MWL8964) { -+ idx = (sta_info->stnid - 1) * SYSADPT_MAX_TID; -+ for (i = 0; i < SYSADPT_MAX_TID; i++) { -+ stream = &priv->ampdu[idx + i]; -+ -+ if (stream->sta == sta) { -+ spin_unlock_bh(&priv->stream_lock); -+ mwl_fwcmd_destroy_ba(hw, stream, -+ BA_FLAG_DIRECTION_UP); -+ spin_lock_bh(&priv->stream_lock); -+ mwl_fwcmd_remove_stream(hw, stream); -+ } -+ } -+ } else { -+ for (idx = 0; idx < priv->ampdu_num; idx++) { -+ stream = &priv->ampdu[idx]; -+ -+ if (stream->sta == sta) { -+ spin_unlock_bh(&priv->stream_lock); -+ mwl_fwcmd_destroy_ba(hw, stream, -+ BA_FLAG_DIRECTION_UP); -+ spin_lock_bh(&priv->stream_lock); -+ mwl_fwcmd_remove_stream(hw, stream); -+ } -+ } -+ } -+ spin_unlock_bh(&priv->stream_lock); -+} -+ -+int mwl_fwcmd_start_stream(struct ieee80211_hw *hw, -+ struct mwl_ampdu_stream *stream) -+{ -+ /* if the stream has already been started, don't start it again */ -+ if (stream->state != AMPDU_STREAM_NEW) -+ return 0; -+ -+ return ieee80211_start_tx_ba_session(stream->sta, stream->tid, 0); -+} -+ -+void mwl_fwcmd_remove_stream(struct ieee80211_hw *hw, -+ struct mwl_ampdu_stream *stream) -+{ -+ memset(stream, 0, sizeof(*stream)); -+} -+ -+struct mwl_ampdu_stream *mwl_fwcmd_lookup_stream(struct ieee80211_hw *hw, -+ struct ieee80211_sta *sta, -+ u8 tid) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct mwl_ampdu_stream *stream; -+ struct mwl_sta *sta_info = mwl_dev_get_sta(sta); -+ int idx; -+ -+ if (priv->chip_type == MWL8964) { -+ idx = ((sta_info->stnid - 1) * SYSADPT_MAX_TID) + tid; -+ if (idx < priv->ampdu_num) -+ return &priv->ampdu[idx]; -+ } else { -+ for (idx = 0; idx < priv->ampdu_num; idx++) { -+ stream = &priv->ampdu[idx]; -+ if (stream->state == AMPDU_NO_STREAM) -+ continue; -+ -+ if ((stream->sta == sta) && (stream->tid == tid)) -+ return stream; -+ } -+ } -+ -+ return NULL; -+} -+ -+bool mwl_fwcmd_ampdu_allowed(struct ieee80211_sta *sta, u8 tid) -+{ -+ struct mwl_sta *sta_info; -+ struct mwl_tx_info *tx_stats; -+ -+ if (WARN_ON(tid >= SYSADPT_MAX_TID)) -+ return false; -+ -+ sta_info = mwl_dev_get_sta(sta); -+ -+ tx_stats = &sta_info->tx_stats[tid]; -+ -+ return (sta_info->is_ampdu_allowed && -+ tx_stats->pkts > SYSADPT_AMPDU_PACKET_THRESHOLD); -+} -+ -+int mwl_fwcmd_set_optimization_level(struct ieee80211_hw *hw, u8 opt_level) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_set_optimization_level *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_set_optimization_level *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_OPTIMIZATION_LEVEL); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->opt_level = opt_level; -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_OPTIMIZATION_LEVEL)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_set_wsc_ie(struct ieee80211_hw *hw, u8 len, u8 *data) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_set_wsc_ie *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_set_wsc_ie *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_WSC_IE); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->len = cpu_to_le16(len); -+ memcpy(pcmd->data, data, len); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_WSC_IE)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ pcmd->ie_type = cpu_to_le16(WSC_IE_SET_PROBE_RESPONSE); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_WSC_IE)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_get_ratetable(struct ieee80211_hw *hw, u8 *addr, u8 *rate_table, -+ u32 size, u8 type) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_get_ratetable *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_get_ratetable *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_GET_RATETABLE); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->type = type; -+ ether_addr_copy(pcmd->addr, addr); -+ memset(rate_table, 0x00, size); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_GET_RATETABLE)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ memcpy(rate_table, &pcmd->sorted_rates_idx_map, size); -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_get_seqno(struct ieee80211_hw *hw, -+ struct mwl_ampdu_stream *stream, u16 *start_seqno) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_get_seqno *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_get_seqno *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_GET_SEQNO); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ ether_addr_copy(pcmd->mac_addr, stream->sta->addr); -+ pcmd->tid = stream->tid; -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_GET_SEQNO)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ *start_seqno = le16_to_cpu(pcmd->seq_no); -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_set_dwds_stamode(struct ieee80211_hw *hw, bool enable) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_dwds_enable *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_dwds_enable *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_DWDS_ENABLE); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->enable = cpu_to_le32(enable); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_DWDS_ENABLE)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_set_fw_flush_timer(struct ieee80211_hw *hw, u32 value) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_fw_flush_timer *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_fw_flush_timer *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_FW_FLUSH_TIMER); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->value = cpu_to_le32(value); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_FW_FLUSH_TIMER)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_set_cdd(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_set_cdd *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_set_cdd *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_CDD); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->enable = cpu_to_le32(priv->cdd); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_CDD)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_set_bftype(struct ieee80211_hw *hw, int mode) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_set_bftype *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_set_bftype *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_BFTYPE); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->action = cpu_to_le32(WL_SET); -+ pcmd->mode = cpu_to_le32(mode); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_BFTYPE)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_reg_cau(struct ieee80211_hw *hw, u8 flag, u32 reg, u32 *val) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_bbp_reg_access *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_bbp_reg_access *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_CAU_REG_ACCESS); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->offset = cpu_to_le16(reg); -+ pcmd->action = cpu_to_le16(flag); -+ pcmd->value = *val; -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_CAU_REG_ACCESS)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ *val = pcmd->value; -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_get_temp(struct ieee80211_hw *hw, u32 *temp) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_get_temp *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_get_temp *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_GET_TEMP); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_GET_TEMP)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ *temp = le32_to_cpu(pcmd->celcius); -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_led_ctrl(struct ieee80211_hw *hw, u8 enable, u8 rate) -+{ -+ struct hostcmd_cmd_led_ctrl *pcmd; -+ struct mwl_priv *priv = hw->priv; -+ -+ pcmd = (struct hostcmd_cmd_led_ctrl *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_LED_CTRL); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->action = 1; /* 1: set */ -+ pcmd->led_enable = enable; -+ pcmd->led_control = 1; /* 1: SW */ -+ -+ switch (rate) { -+ case LED_BLINK_RATE_LOW: -+ case LED_BLINK_RATE_MID: -+ case LED_BLINK_RATE_HIGH: -+ pcmd->led_blink_rate = rate; -+ break; -+ default: -+ if (enable) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EINVAL; -+ } -+ break; -+ } -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_LED_CTRL)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_get_fw_region_code(struct ieee80211_hw *hw, -+ u32 *fw_region_code) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_get_fw_region_code *pcmd; -+ u16 cmd; -+ int status; -+ -+ pcmd = (struct hostcmd_cmd_get_fw_region_code *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ cmd = HOSTCMD_CMD_GET_FW_REGION_CODE; -+ pcmd->cmd_hdr.cmd = cpu_to_le16(cmd); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ -+ if (mwl_hif_exec_cmd(hw, cmd)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ if (pcmd->cmd_hdr.result != 0) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EINVAL; -+ } -+ -+ status = le32_to_cpu(pcmd->status); -+ -+ if (!status) -+ *fw_region_code = le32_to_cpu(pcmd->fw_region_code); -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_get_device_pwr_tbl(struct ieee80211_hw *hw, -+ struct mwl_device_pwr_tbl *device_ch_pwrtbl, -+ u8 *region_code, -+ u8 *number_of_channels, -+ u32 channel_index) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_get_device_pwr_tbl *pcmd; -+ int status; -+ u16 cmd; -+ -+ pcmd = (struct hostcmd_cmd_get_device_pwr_tbl *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ cmd = HOSTCMD_CMD_GET_DEVICE_PWR_TBL; -+ pcmd->cmd_hdr.cmd = cpu_to_le16(cmd); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->status = cpu_to_le16(cmd); -+ pcmd->current_channel_index = cpu_to_le32(channel_index); -+ -+ if (mwl_hif_exec_cmd(hw, cmd)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ device_ch_pwrtbl->channel = pcmd->channel_pwr_tbl.channel; -+ memcpy(device_ch_pwrtbl->tx_pwr, pcmd->channel_pwr_tbl.tx_pwr, -+ priv->pwr_level); -+ device_ch_pwrtbl->dfs_capable = pcmd->channel_pwr_tbl.dfs_capable; -+ device_ch_pwrtbl->ax_ant = pcmd->channel_pwr_tbl.ax_ant; -+ device_ch_pwrtbl->cdd = pcmd->channel_pwr_tbl.cdd; -+ *region_code = pcmd->region_code; -+ *number_of_channels = pcmd->number_of_channels; -+ status = le16_to_cpu(pcmd->status); -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return status; -+} -+ -+int mwl_fwcmd_set_rate_drop(struct ieee80211_hw *hw, int enable, -+ int value, int staid) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_set_rate_drop *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_set_rate_drop *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_SET_RATE_DROP); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->enable = cpu_to_le32(enable); -+ pcmd->rate_index = cpu_to_le32(value); -+ pcmd->sta_index = cpu_to_le32(staid); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_SET_RATE_DROP)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_newdp_dmathread_start(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_newdp_dmathread_start *pcmd; -+ u16 cmd; -+ -+ pcmd = (struct hostcmd_cmd_newdp_dmathread_start *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ cmd = HOSTCMD_CMD_NEWDP_DMATHREAD_START; -+ pcmd->cmd_hdr.cmd = cpu_to_le16(cmd); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ -+ if (mwl_hif_exec_cmd(hw, cmd)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+ -+int mwl_fwcmd_get_fw_region_code_sc4(struct ieee80211_hw *hw, -+ u32 *fw_region_code) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_get_fw_region_code_sc4 *pcmd; -+ u16 cmd; -+ -+ pcmd = (struct hostcmd_cmd_get_fw_region_code_sc4 *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ cmd = HOSTCMD_CMD_GET_FW_REGION_CODE_SC4; -+ pcmd->cmd_hdr.cmd = cpu_to_le16(cmd); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ -+ if (mwl_hif_exec_cmd(hw, cmd)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ if (pcmd->cmd_hdr.result != 0) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EINVAL; -+ } -+ -+ if (pcmd->status) -+ *fw_region_code = (pcmd->status == 1) ? 0 : pcmd->status; -+ else -+ *fw_region_code = le32_to_cpu(pcmd->fw_region_code); -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_get_pwr_tbl_sc4(struct ieee80211_hw *hw, -+ struct mwl_device_pwr_tbl *device_ch_pwrtbl, -+ u8 *region_code, -+ u8 *number_of_channels, -+ u32 channel_index) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_get_device_pwr_tbl_sc4 *pcmd; -+ int status; -+ u16 cmd; -+ -+ pcmd = (struct hostcmd_cmd_get_device_pwr_tbl_sc4 *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ cmd = HOSTCMD_CMD_GET_DEVICE_PWR_TBL_SC4; -+ pcmd->cmd_hdr.cmd = cpu_to_le16(cmd); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->status = cpu_to_le16(cmd); -+ pcmd->current_channel_index = cpu_to_le32(channel_index); -+ -+ if (mwl_hif_exec_cmd(hw, cmd)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ device_ch_pwrtbl->channel = pcmd->channel_pwr_tbl.channel; -+ memcpy(device_ch_pwrtbl->tx_pwr, pcmd->channel_pwr_tbl.tx_pwr, -+ SYSADPT_TX_PWR_LEVEL_TOTAL_SC4); -+ device_ch_pwrtbl->dfs_capable = pcmd->channel_pwr_tbl.dfs_capable; -+ device_ch_pwrtbl->ax_ant = pcmd->channel_pwr_tbl.ax_ant; -+ device_ch_pwrtbl->cdd = pcmd->channel_pwr_tbl.cdd; -+ *region_code = pcmd->region_code; -+ *number_of_channels = pcmd->number_of_channels; -+ status = le16_to_cpu(pcmd->status); -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return status; -+} -+ -+int mwl_fwcmd_quiet_mode(struct ieee80211_hw *hw, bool enable, u32 period, -+ u32 duration, u32 next_offset) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_quiet_mode *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_quiet_mode *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_QUIET_MODE); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->action = cpu_to_le16(WL_SET); -+ pcmd->enable = cpu_to_le32(enable); -+ if (enable) { -+ pcmd->period = cpu_to_le32(period); -+ pcmd->duration = cpu_to_le32(duration); -+ pcmd->next_offset = cpu_to_le32(next_offset); -+ } -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_QUIET_MODE)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_core_dump_diag_mode(struct ieee80211_hw *hw, u16 status) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_core_dump_diag_mode *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_core_dump_diag_mode *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_CORE_DUMP_DIAG_MODE); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->status = cpu_to_le16(status); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_CORE_DUMP_DIAG_MODE)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_get_fw_core_dump(struct ieee80211_hw *hw, -+ struct coredump_cmd *core_dump, char *buff) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_get_fw_core_dump *pcmd; -+ -+ if (priv->chip_type != MWL8964) -+ return -EPERM; -+ -+ pcmd = (struct hostcmd_cmd_get_fw_core_dump *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_GET_FW_CORE_DUMP); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->cmd_data.coredump.context = core_dump->context; -+ pcmd->cmd_data.coredump.buffer = cpu_to_le32(priv->pphys_cmd_buf + -+ sizeof(struct hostcmd_cmd_get_fw_core_dump) - -+ sizeof(struct hostcmd_cmd_get_fw_core_dump_)); -+ pcmd->cmd_data.coredump.buffer_len = cpu_to_le32(MAX_CORE_DUMP_BUFFER); -+ pcmd->cmd_data.coredump.size_kb = core_dump->size_kb; -+ pcmd->cmd_data.coredump.flags = core_dump->flags; -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_GET_FW_CORE_DUMP)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ /* update core dump buffer */ -+ core_dump->context = pcmd->cmd_data.coredump.context; -+ core_dump->size_kb = pcmd->cmd_data.coredump.size_kb; -+ core_dump->flags = pcmd->cmd_data.coredump.flags; -+ memcpy(buff, -+ (const void *)((u32)pcmd + -+ sizeof(struct hostcmd_cmd_get_fw_core_dump) - -+ sizeof(struct hostcmd_cmd_get_fw_core_dump_)), -+ MAX_CORE_DUMP_BUFFER); -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_set_slot_time(struct ieee80211_hw *hw, bool short_slot) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_802_11_slot_time *pcmd; -+ -+ wiphy_debug(priv->hw->wiphy, "%s(): short_slot_time=%d\n", -+ __func__, short_slot); -+ -+ pcmd = (struct hostcmd_cmd_802_11_slot_time *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_802_11_SLOT_TIME); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->action = cpu_to_le16(WL_SET); -+ pcmd->short_slot = cpu_to_le16(short_slot ? 1 : 0); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_802_11_SLOT_TIME)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_config_EDMACCtrl(struct ieee80211_hw *hw, int EDMAC_Ctrl) -+{ -+ struct hostcmd_cmd_edmac_ctrl *pcmd; -+ struct mwl_priv *priv = hw->priv; -+ -+ pcmd = (struct hostcmd_cmd_edmac_ctrl *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_EDMAC_CTRL); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->action = cpu_to_le16(WL_SET); -+ pcmd->ed_ctrl_2g = cpu_to_le16((EDMAC_Ctrl & EDMAC_2G_ENABLE_MASK) -+ >> EDMAC_2G_ENABLE_SHIFT); -+ pcmd->ed_ctrl_5g = cpu_to_le16((EDMAC_Ctrl & EDMAC_5G_ENABLE_MASK) -+ >> EDMAC_5G_ENABLE_SHIFT); -+ pcmd->ed_offset_2g = cpu_to_le16((EDMAC_Ctrl & -+ EDMAC_2G_THRESHOLD_OFFSET_MASK) -+ >> EDMAC_2G_THRESHOLD_OFFSET_SHIFT); -+ pcmd->ed_offset_5g = cpu_to_le16((EDMAC_Ctrl & -+ EDMAC_5G_THRESHOLD_OFFSET_MASK) -+ >> EDMAC_5G_THRESHOLD_OFFSET_SHIFT); -+ pcmd->ed_bitmap_txq_lock = cpu_to_le16((EDMAC_Ctrl & -+ EDMAC_QLOCK_BITMAP_MASK) -+ >> EDMAC_QLOCK_BITMAP_SHIFT); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_EDMAC_CTRL)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+int mwl_fwcmd_set_txpwrlmt_cfg_data(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_txpwrlmt_cfg *pcmd; -+ struct mwl_txpwrlmt_cfg_entry_hdr hdr; -+ u16 id, parsed_len, size; -+ __le32 txpwr_cfg_sig; -+ u8 version[TXPWRLMT_CFG_VERSION_INFO_LEN]; -+ const u8 *ptr; -+ -+ if (!priv->txpwrlmt_file) -+ return 0; -+ -+ ptr = priv->txpwrlmt_file->data; -+ size = priv->txpwrlmt_file->size; -+ -+ /* Parsing TxPwrLmit Conf file Signature */ -+ parsed_len = mwl_fwcmd_parse_txpwrlmt_cfg(ptr, size, -+ TXPWRLMT_CFG_SIG_LEN, -+ (u8 *)&txpwr_cfg_sig); -+ ptr += parsed_len; -+ size -= parsed_len; -+ -+ if (le32_to_cpu(txpwr_cfg_sig) != TXPWRLMT_CFG_SIGNATURE) { -+ wiphy_err(hw->wiphy, -+ "txpwrlmt config signature mismatch\n"); -+ release_firmware(priv->txpwrlmt_file); -+ priv->txpwrlmt_file = NULL; -+ return 0; -+ } -+ -+ /* Parsing TxPwrLmit Conf file Version */ -+ parsed_len = mwl_fwcmd_parse_txpwrlmt_cfg(ptr, size, -+ TXPWRLMT_CFG_VERSION_INFO_LEN, -+ version); -+ ptr += parsed_len; -+ size -= parsed_len; -+ -+ for (id = 0; id < TXPWRLMT_CFG_MAX_SUBBAND_INFO; id++) { -+ u16 data_len; -+ -+ /*Parsing tx pwr cfg subband header info*/ -+ parsed_len = sizeof(struct mwl_txpwrlmt_cfg_entry_hdr); -+ parsed_len = mwl_fwcmd_parse_txpwrlmt_cfg(ptr, size, -+ parsed_len, -+ (u8 *)&hdr); -+ ptr += parsed_len; -+ size -= parsed_len; -+ data_len = le16_to_cpu(hdr.len) - -+ sizeof(struct mwl_txpwrlmt_cfg_entry_hdr); -+ -+ pcmd = (struct hostcmd_cmd_txpwrlmt_cfg *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->action = cpu_to_le16(HOSTCMD_ACT_GEN_SET); -+ pcmd->subband_id = hdr.id; -+ pcmd->data_len = cpu_to_le16(data_len); -+ pcmd->num_entries = hdr.num_entries; -+ -+ /* Parsing tx pwr cfg subband header info */ -+ parsed_len = mwl_fwcmd_parse_txpwrlmt_cfg(ptr, size, -+ data_len, pcmd->data); -+ ptr += parsed_len; -+ size -= parsed_len; -+ -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_TXPWRLMT_CFG); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd) + -+ data_len - sizeof(pcmd->data)); -+ -+ if (size < sizeof(struct mwl_txpwrlmt_cfg_entry_hdr)) -+ pcmd->cfgComplete = 1; -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_TXPWRLMT_CFG)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ release_firmware(priv->txpwrlmt_file); -+ priv->txpwrlmt_file = NULL; -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ } -+ -+ release_firmware(priv->txpwrlmt_file); -+ priv->txpwrlmt_file = NULL; -+ -+ return 0; -+} -+ -+int mwl_fwcmd_get_txpwrlmt_cfg_data(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_txpwrlmt_cfg *pcmd; -+ u16 subband_len, total_len = 0; -+ u8 id; -+ -+ for (id = 0; id < TXPWRLMT_CFG_MAX_SUBBAND_INFO; id++) { -+ pcmd = (struct hostcmd_cmd_txpwrlmt_cfg *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->action = 0; -+ pcmd->subband_id = id; -+ pcmd->data_len = 0; -+ pcmd->num_entries = 0; -+ -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_TXPWRLMT_CFG); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_TXPWRLMT_CFG)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ subband_len = le16_to_cpu(pcmd->cmd_hdr.len) - -+ sizeof(struct hostcmd_header) - 2; -+ if (total_len <= SYSADPT_TXPWRLMT_CFG_BUF_SIZE) { -+ wiphy_debug(hw->wiphy, "Subband len = %d\n", -+ subband_len); -+ memcpy(priv->txpwrlmt_data.buf + total_len, -+ &pcmd->subband_id, subband_len); -+ total_len += subband_len; -+ priv->txpwrlmt_data.buf[total_len] = '\n'; -+ total_len++; -+ priv->txpwrlmt_data.len = total_len; -+ } else { -+ wiphy_err(hw->wiphy, -+ "TxPwrLmt cfg buf size is not enough\n"); -+ } -+ } -+ -+ return 0; -+} -+ -+int mwl_fwcmd_mcast_cts(struct ieee80211_hw *hw, u8 enable) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct hostcmd_cmd_mcast_cts *pcmd; -+ -+ pcmd = (struct hostcmd_cmd_mcast_cts *)&priv->pcmd_buf[0]; -+ -+ mutex_lock(&priv->fwcmd_mutex); -+ -+ memset(pcmd, 0x00, sizeof(*pcmd)); -+ pcmd->cmd_hdr.cmd = cpu_to_le16(HOSTCMD_CMD_MCAST_CTS); -+ pcmd->cmd_hdr.len = cpu_to_le16(sizeof(*pcmd)); -+ pcmd->enable = enable; -+ -+ if (mwl_hif_exec_cmd(hw, HOSTCMD_CMD_MCAST_CTS)) { -+ mutex_unlock(&priv->fwcmd_mutex); -+ return -EIO; -+ } -+ -+ mutex_unlock(&priv->fwcmd_mutex); -+ -+ return 0; -+} -+ -+void mwl_fwcmd_get_survey(struct ieee80211_hw *hw, int idx) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct ieee80211_conf *conf = &hw->conf; -+ struct mwl_survey_info *survey_info; -+ -+ if (idx) -+ survey_info = &priv->survey_info[idx - 1]; -+ else -+ survey_info = &priv->cur_survey_info; -+ -+ memcpy(&survey_info->channel, conf->chandef.chan, -+ sizeof(struct ieee80211_channel)); -+ mwl_hif_get_survey(hw, survey_info); -+} -diff --git a/drivers/net/wireless/marvell/mwlwifi/hif/fwcmd.h b/drivers/net/wireless/marvell/mwlwifi/hif/fwcmd.h -new file mode 100644 -index 000000000000..9565cc447dc6 ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/hif/fwcmd.h -@@ -0,0 +1,285 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file defines firmware host command related -+ * functions. -+ */ -+ -+#ifndef _FWCMD_H_ -+#define _FWCMD_H_ -+ -+#include "hif/hostcmd.h" -+ -+/* Define OpMode for SoftAP/Station mode -+ * -+ * The following mode signature has to be written to PCI scratch register#0 -+ * right after successfully downloading the last block of firmware and -+ * before waiting for firmware ready signature -+ */ -+ -+#define HOSTCMD_STA_MODE 0x5A -+#define HOSTCMD_SOFTAP_MODE 0xA5 -+ -+#define HOSTCMD_STA_FWRDY_SIGNATURE 0xF0F1F2F4 -+#define HOSTCMD_SOFTAP_FWRDY_SIGNATURE 0xF1F2F4A5 -+ -+#define GUARD_INTERVAL_STANDARD 1 -+#define GUARD_INTERVAL_SHORT 2 -+#define GUARD_INTERVAL_AUTO 3 -+ -+#define LINK_CS_STATE_CONSERV 0 -+#define LINK_CS_STATE_AGGR 1 -+#define LINK_CS_STATE_AUTO 2 -+#define LINK_CS_STATE_AUTO_DISABLED 3 -+ -+#define STOP_DETECT_RADAR 0 -+#define CAC_START 1 -+#define MONITOR_START 3 -+ -+#define WDS_MODE 4 -+ -+enum { -+ WL_ANTENNATYPE_RX = 1, -+ WL_ANTENNATYPE_TX = 2, -+}; -+ -+enum encr_type { -+ ENCR_TYPE_WEP = 0, -+ ENCR_TYPE_DISABLE = 1, -+ ENCR_TYPE_TKIP = 4, -+ ENCR_TYPE_AES = 6, -+ ENCR_TYPE_MIX = 7, -+}; -+ -+char *mwl_fwcmd_get_cmd_string(unsigned short cmd); -+ -+const struct hostcmd_get_hw_spec -+*mwl_fwcmd_get_hw_specs(struct ieee80211_hw *hw); -+ -+int mwl_fwcmd_set_hw_specs(struct ieee80211_hw *hw, -+ struct hostcmd_set_hw_spec *spec); -+ -+int mwl_fwcmd_get_stat(struct ieee80211_hw *hw, -+ struct ieee80211_low_level_stats *stats); -+ -+int mwl_fwcmd_reg_bb(struct ieee80211_hw *hw, u8 flag, u32 reg, u32 *val); -+ -+int mwl_fwcmd_reg_rf(struct ieee80211_hw *hw, u8 flag, u32 reg, u32 *val); -+ -+int mwl_fwcmd_radio_enable(struct ieee80211_hw *hw); -+ -+int mwl_fwcmd_radio_disable(struct ieee80211_hw *hw); -+ -+int mwl_fwcmd_set_radio_preamble(struct ieee80211_hw *hw, -+ bool short_preamble); -+ -+int mwl_fwcmd_get_addr_value(struct ieee80211_hw *hw, u32 addr, u32 len, -+ u32 *val, u16 set); -+ -+int mwl_fwcmd_max_tx_power(struct ieee80211_hw *hw, -+ struct ieee80211_conf *conf, u8 fraction); -+ -+int mwl_fwcmd_tx_power(struct ieee80211_hw *hw, -+ struct ieee80211_conf *conf, u8 fraction); -+ -+int mwl_fwcmd_rf_antenna(struct ieee80211_hw *hw, int dir, int antenna); -+ -+int mwl_fwcmd_broadcast_ssid_enable(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, bool enable); -+ -+int mwl_fwcmd_set_cfg_data(struct ieee80211_hw *hw, u16 type); -+ -+int mwl_fwcmd_set_rf_channel(struct ieee80211_hw *hw, -+ struct ieee80211_conf *conf); -+ -+int mwl_fwcmd_set_aid(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, u8 *bssid, u16 aid); -+ -+int mwl_fwcmd_set_infra_mode(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif); -+ -+int mwl_fwcmd_set_rts_threshold(struct ieee80211_hw *hw, -+ int threshold); -+ -+int mwl_fwcmd_set_edca_params(struct ieee80211_hw *hw, u8 index, -+ u16 cw_min, u16 cw_max, u8 aifs, u16 txop); -+ -+int mwl_fwcmd_set_radar_detect(struct ieee80211_hw *hw, u16 action); -+ -+int mwl_fwcmd_set_wmm_mode(struct ieee80211_hw *hw, bool enable); -+ -+int mwl_fwcmd_ht_guard_interval(struct ieee80211_hw *hw, u32 gi_type); -+ -+int mwl_fwcmd_use_fixed_rate(struct ieee80211_hw *hw, -+ int mcast, int mgmt); -+ -+int mwl_fwcmd_set_linkadapt_cs_mode(struct ieee80211_hw *hw, -+ u16 cs_mode); -+ -+int mwl_fwcmd_dump_otp_data(struct ieee80211_hw *hw); -+ -+int mwl_fwcmd_set_rate_adapt_mode(struct ieee80211_hw *hw, -+ u16 mode); -+ -+int mwl_fwcmd_set_mac_addr_client(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, u8 *mac_addr); -+ -+int mwl_fwcmd_get_watchdog_bitmap(struct ieee80211_hw *hw, -+ u8 *bitmap); -+ -+int mwl_fwcmd_remove_mac_addr(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, u8 *mac_addr); -+ -+int mwl_fwcmd_bss_start(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, bool enable); -+ -+int mwl_fwcmd_set_beacon(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, u8 *beacon, int len); -+ -+int mwl_fwcmd_set_new_stn_add(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, -+ struct ieee80211_sta *sta); -+ -+int mwl_fwcmd_set_new_stn_add_sc4(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, -+ struct ieee80211_sta *sta, -+ u32 wds); -+ -+int mwl_fwcmd_set_new_stn_wds_sc4(struct ieee80211_hw *hw, u8 *addr); -+ -+int mwl_fwcmd_set_new_stn_add_self(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif); -+ -+int mwl_fwcmd_set_new_stn_del(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, u8 *addr); -+ -+int mwl_fwcmd_set_apmode(struct ieee80211_hw *hw, u8 apmode); -+ -+int mwl_fwcmd_set_switch_channel(struct ieee80211_hw *hw, -+ struct ieee80211_channel_switch *ch_switch); -+ -+int mwl_fwcmd_update_encryption_enable(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, -+ u8 *addr, u8 encr_type); -+ -+int mwl_fwcmd_encryption_set_key(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, u8 *addr, -+ struct ieee80211_key_conf *key); -+ -+int mwl_fwcmd_encryption_remove_key(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, u8 *addr, -+ struct ieee80211_key_conf *key); -+ -+int mwl_fwcmd_check_ba(struct ieee80211_hw *hw, -+ struct mwl_ampdu_stream *stream, -+ struct ieee80211_vif *vif, -+ u32 direction); -+ -+int mwl_fwcmd_create_ba(struct ieee80211_hw *hw, -+ struct mwl_ampdu_stream *stream, -+ struct ieee80211_vif *vif, -+ u32 direction, u8 buf_size, u16 seqno, bool amsdu); -+ -+int mwl_fwcmd_destroy_ba(struct ieee80211_hw *hw, -+ struct mwl_ampdu_stream *stream, -+ u32 direction); -+ -+struct mwl_ampdu_stream *mwl_fwcmd_add_stream(struct ieee80211_hw *hw, -+ struct ieee80211_sta *sta, -+ u8 tid); -+ -+void mwl_fwcmd_del_sta_streams(struct ieee80211_hw *hw, -+ struct ieee80211_sta *sta); -+ -+int mwl_fwcmd_start_stream(struct ieee80211_hw *hw, -+ struct mwl_ampdu_stream *stream); -+ -+void mwl_fwcmd_remove_stream(struct ieee80211_hw *hw, -+ struct mwl_ampdu_stream *stream); -+ -+struct mwl_ampdu_stream *mwl_fwcmd_lookup_stream(struct ieee80211_hw *hw, -+ struct ieee80211_sta *sta, -+ u8 tid); -+ -+bool mwl_fwcmd_ampdu_allowed(struct ieee80211_sta *sta, u8 tid); -+ -+int mwl_fwcmd_set_optimization_level(struct ieee80211_hw *hw, u8 opt_level); -+ -+int mwl_fwcmd_set_wsc_ie(struct ieee80211_hw *hw, u8 len, u8 *data); -+ -+int mwl_fwcmd_get_ratetable(struct ieee80211_hw *hw, u8 *addr, u8 *rate_table, -+ u32 size, u8 type); -+ -+int mwl_fwcmd_get_seqno(struct ieee80211_hw *hw, -+ struct mwl_ampdu_stream *stream, u16 *start_seqno); -+ -+int mwl_fwcmd_set_dwds_stamode(struct ieee80211_hw *hw, bool enable); -+ -+int mwl_fwcmd_set_fw_flush_timer(struct ieee80211_hw *hw, u32 value); -+ -+int mwl_fwcmd_set_cdd(struct ieee80211_hw *hw); -+ -+int mwl_fwcmd_set_bftype(struct ieee80211_hw *hw, int mode); -+ -+int mwl_fwcmd_reg_cau(struct ieee80211_hw *hw, u8 flag, u32 reg, u32 *val); -+ -+int mwl_fwcmd_get_temp(struct ieee80211_hw *hw, u32 *temp); -+ -+int mwl_fwcmd_led_ctrl(struct ieee80211_hw *hw, u8 enable, u8 rate); -+ -+int mwl_fwcmd_get_fw_region_code(struct ieee80211_hw *hw, -+ u32 *fw_region_code); -+ -+int mwl_fwcmd_get_device_pwr_tbl(struct ieee80211_hw *hw, -+ struct mwl_device_pwr_tbl *device_ch_pwrtbl, -+ u8 *region_code, -+ u8 *number_of_channels, -+ u32 channel_index); -+ -+int mwl_fwcmd_set_rate_drop(struct ieee80211_hw *hw, int enable, -+ int value, int staid); -+ -+int mwl_fwcmd_newdp_dmathread_start(struct ieee80211_hw *hw); -+ -+int mwl_fwcmd_get_fw_region_code_sc4(struct ieee80211_hw *hw, -+ u32 *fw_region_code); -+ -+int mwl_fwcmd_get_pwr_tbl_sc4(struct ieee80211_hw *hw, -+ struct mwl_device_pwr_tbl *device_ch_pwrtbl, -+ u8 *region_code, -+ u8 *number_of_channels, -+ u32 channel_index); -+ -+int mwl_fwcmd_quiet_mode(struct ieee80211_hw *hw, bool enable, u32 period, -+ u32 duration, u32 next_offset); -+ -+int mwl_fwcmd_core_dump_diag_mode(struct ieee80211_hw *hw, u16 status); -+ -+int mwl_fwcmd_get_fw_core_dump(struct ieee80211_hw *hw, -+ struct coredump_cmd *core_dump, char *buff); -+ -+int mwl_fwcmd_set_slot_time(struct ieee80211_hw *hw, bool short_slot); -+ -+int mwl_fwcmd_config_EDMACCtrl(struct ieee80211_hw *hw, int EDMAC_Ctrl); -+ -+int mwl_fwcmd_set_txpwrlmt_cfg_data(struct ieee80211_hw *hw); -+ -+int mwl_fwcmd_get_txpwrlmt_cfg_data(struct ieee80211_hw *hw); -+ -+int mwl_fwcmd_mcast_cts(struct ieee80211_hw *hw, u8 enable); -+ -+void mwl_fwcmd_get_survey(struct ieee80211_hw *hw, int idx); -+ -+#endif /* _FWCMD_H_ */ -diff --git a/drivers/net/wireless/marvell/mwlwifi/hif/hif-ops.h b/drivers/net/wireless/marvell/mwlwifi/hif/hif-ops.h -new file mode 100644 -index 000000000000..f5c7144b3c1b ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/hif/hif-ops.h -@@ -0,0 +1,297 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file defines host interface related operations. */ -+ -+#ifndef _HIF_OPS_H_ -+#define _HIF_OPS_H_ -+static inline const char *mwl_hif_get_driver_name(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ return priv->hif.ops->driver_name; -+} -+ -+static inline const char *mwl_hif_get_driver_version(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ return priv->hif.ops->driver_version; -+} -+ -+static inline unsigned int mwl_hif_get_tx_head_room(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ return priv->hif.ops->tx_head_room; -+} -+ -+static inline unsigned int mwl_hif_get_ampdu_num(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ return priv->hif.ops->ampdu_num; -+} -+ -+static inline void mwl_hif_reset(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->reset) -+ priv->hif.ops->reset(hw); -+} -+ -+static inline int mwl_hif_init(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->init) -+ return priv->hif.ops->init(hw); -+ else -+ return -ENOTSUPP; -+} -+ -+static inline void mwl_hif_deinit(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->deinit) -+ priv->hif.ops->deinit(hw); -+} -+ -+static inline int mwl_hif_get_info(struct ieee80211_hw *hw, -+ char *buf, size_t size) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->get_info) -+ return priv->hif.ops->get_info(hw, buf, size); -+ else -+ return 0; -+} -+ -+static inline int mwl_hif_get_tx_status(struct ieee80211_hw *hw, -+ char *buf, size_t size) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->get_tx_status) -+ return priv->hif.ops->get_tx_status(hw, buf, size); -+ else -+ return 0; -+} -+ -+static inline int mwl_hif_get_rx_status(struct ieee80211_hw *hw, -+ char *buf, size_t size) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->get_rx_status) -+ return priv->hif.ops->get_rx_status(hw, buf, size); -+ else -+ return 0; -+} -+ -+static inline void mwl_hif_enable_data_tasks(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->enable_data_tasks) -+ priv->hif.ops->enable_data_tasks(hw); -+} -+ -+static inline void mwl_hif_disable_data_tasks(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->disable_data_tasks) -+ priv->hif.ops->disable_data_tasks(hw); -+} -+ -+static inline int mwl_hif_exec_cmd(struct ieee80211_hw *hw, unsigned short cmd) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->exec_cmd) -+ return priv->hif.ops->exec_cmd(hw, cmd); -+ else -+ return -ENOTSUPP; -+} -+ -+static inline int mwl_hif_get_irq_num(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->get_irq_num) -+ return priv->hif.ops->get_irq_num(hw); -+ else -+ return -ENOTSUPP; -+} -+ -+static inline irqreturn_t mwl_hif_irq_handler(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->irq_handler) -+ return priv->hif.ops->irq_handler(hw); -+ else -+ return -ENOTSUPP; -+} -+ -+static inline void mwl_hif_irq_enable(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->irq_enable) -+ priv->hif.ops->irq_enable(hw); -+} -+ -+static inline void mwl_hif_irq_disable(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->irq_disable) -+ priv->hif.ops->irq_disable(hw); -+} -+ -+static inline int mwl_hif_download_firmware(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->download_firmware) -+ return priv->hif.ops->download_firmware(hw); -+ else -+ return -ENOTSUPP; -+} -+ -+static inline void mwl_hif_timer_routine(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->timer_routine) -+ priv->hif.ops->timer_routine(hw); -+} -+ -+static inline void mwl_hif_tx_xmit(struct ieee80211_hw *hw, -+ struct ieee80211_tx_control *control, -+ struct sk_buff *skb) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->tx_xmit) -+ priv->hif.ops->tx_xmit(hw, control, skb); -+} -+ -+static inline void mwl_hif_tx_del_pkts_via_vif(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->tx_del_pkts_via_vif) -+ priv->hif.ops->tx_del_pkts_via_vif(hw, vif); -+} -+ -+static inline void mwl_hif_tx_del_pkts_via_sta(struct ieee80211_hw *hw, -+ struct ieee80211_sta *sta) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->tx_del_pkts_via_sta) -+ priv->hif.ops->tx_del_pkts_via_sta(hw, sta); -+} -+ -+static inline void mwl_hif_tx_del_ampdu_pkts(struct ieee80211_hw *hw, -+ struct ieee80211_sta *sta, u8 tid) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->tx_del_ampdu_pkts) -+ priv->hif.ops->tx_del_ampdu_pkts(hw, sta, tid); -+} -+ -+static inline void mwl_hif_tx_del_sta_amsdu_pkts(struct ieee80211_hw *hw, -+ struct ieee80211_sta *sta) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->tx_del_sta_amsdu_pkts) -+ priv->hif.ops->tx_del_sta_amsdu_pkts(hw, sta); -+} -+ -+static inline void mwl_hif_tx_return_pkts(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->tx_return_pkts) -+ priv->hif.ops->tx_return_pkts(hw); -+} -+ -+static inline struct device_node *mwl_hif_device_node(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->get_device_node) -+ return priv->hif.ops->get_device_node(hw); -+ else -+ return NULL; -+} -+ -+static inline void mwl_hif_get_survey(struct ieee80211_hw *hw, -+ struct mwl_survey_info *survey_info) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->get_survey) -+ priv->hif.ops->get_survey(hw, survey_info); -+} -+ -+static inline int mwl_hif_reg_access(struct ieee80211_hw *hw, bool write) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->reg_access) -+ return priv->hif.ops->reg_access(hw, write); -+ else -+ return -ENOTSUPP; -+} -+ -+static inline void mwl_hif_set_sta_id(struct ieee80211_hw *hw, -+ struct ieee80211_sta *sta, -+ bool sta_mode, bool set) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->set_sta_id) -+ priv->hif.ops->set_sta_id(hw, sta, sta_mode, set); -+} -+ -+static inline void mwl_hif_process_account(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->process_account) -+ priv->hif.ops->process_account(hw); -+} -+ -+static inline int mwl_hif_mcast_cts(struct ieee80211_hw *hw, bool enable) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->hif.ops->mcast_cts) -+ return priv->hif.ops->mcast_cts(hw, enable); -+ else -+ return -ENOTSUPP; -+} -+#endif /* _HIF_OPS_H_ */ -diff --git a/drivers/net/wireless/marvell/mwlwifi/hif/hif.h b/drivers/net/wireless/marvell/mwlwifi/hif/hif.h -new file mode 100644 -index 000000000000..6ea6192ac5e0 ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/hif/hif.h -@@ -0,0 +1,81 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file defines host interface data structure. */ -+ -+#ifndef _HIF_H_ -+#define _HIF_H_ -+ -+/* memory/register access */ -+#define MWL_ACCESS_MAC 1 -+#define MWL_ACCESS_RF 2 -+#define MWL_ACCESS_BBP 3 -+#define MWL_ACCESS_CAU 4 -+#define MWL_ACCESS_ADDR0 5 -+#define MWL_ACCESS_ADDR1 6 -+#define MWL_ACCESS_ADDR 7 -+ -+struct mwl_survey_info { -+ struct ieee80211_channel channel; -+ u32 filled; -+ u32 time_period; -+ u32 time_busy; -+ u32 time_tx; -+ s8 noise; -+}; -+ -+struct mwl_hif_ops { -+ const char *driver_name; -+ const char *driver_version; -+ unsigned int tx_head_room; -+ int ampdu_num; -+ void (*reset)(struct ieee80211_hw *hw); -+ int (*init)(struct ieee80211_hw *hw); -+ void (*deinit)(struct ieee80211_hw *hw); -+ int (*get_info)(struct ieee80211_hw *hw, char *buf, size_t size); -+ int (*get_tx_status)(struct ieee80211_hw *hw, char *buf, size_t size); -+ int (*get_rx_status)(struct ieee80211_hw *hw, char *buf, size_t size); -+ void (*enable_data_tasks)(struct ieee80211_hw *hw); -+ void (*disable_data_tasks)(struct ieee80211_hw *hw); -+ int (*exec_cmd)(struct ieee80211_hw *hw, unsigned short cmd); -+ int (*get_irq_num)(struct ieee80211_hw *hw); -+ irqreturn_t (*irq_handler)(struct ieee80211_hw *hw); -+ void (*irq_enable)(struct ieee80211_hw *hw); -+ void (*irq_disable)(struct ieee80211_hw *hw); -+ int (*download_firmware)(struct ieee80211_hw *hw); -+ void (*timer_routine)(struct ieee80211_hw *hw); -+ void (*tx_xmit)(struct ieee80211_hw *hw, -+ struct ieee80211_tx_control *control, -+ struct sk_buff *skb); -+ void (*tx_del_pkts_via_vif)(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif); -+ void (*tx_del_pkts_via_sta)(struct ieee80211_hw *hw, -+ struct ieee80211_sta *sta); -+ void (*tx_del_ampdu_pkts)(struct ieee80211_hw *hw, -+ struct ieee80211_sta *sta, u8 tid); -+ void (*tx_del_sta_amsdu_pkts)(struct ieee80211_hw *hw, -+ struct ieee80211_sta *sta); -+ void (*tx_return_pkts)(struct ieee80211_hw *hw); -+ struct device_node *(*get_device_node)(struct ieee80211_hw *hw); -+ void (*get_survey)(struct ieee80211_hw *hw, -+ struct mwl_survey_info *survey_info); -+ int (*reg_access)(struct ieee80211_hw *hw, bool write); -+ void (*set_sta_id)(struct ieee80211_hw *hw, -+ struct ieee80211_sta *sta, -+ bool sta_mode, bool set); -+ void (*process_account)(struct ieee80211_hw *hw); -+ int (*mcast_cts)(struct ieee80211_hw *hw, bool enable); -+}; -+#endif /* _HIF_H_ */ -diff --git a/drivers/net/wireless/marvell/mwlwifi/hif/hostcmd.h b/drivers/net/wireless/marvell/mwlwifi/hif/hostcmd.h -new file mode 100644 -index 000000000000..b14f161f1410 ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/hif/hostcmd.h -@@ -0,0 +1,1285 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file defines firmware host command related -+ * structure. -+ */ -+ -+#ifndef _HOSTCMD_H_ -+#define _HOSTCMD_H_ -+ -+/* 16 bit host command code */ -+#define HOSTCMD_CMD_GET_HW_SPEC 0x0003 -+#define HOSTCMD_CMD_SET_HW_SPEC 0x0004 -+#define HOSTCMD_CMD_802_11_GET_STAT 0x0014 -+#define HOSTCMD_CMD_BBP_REG_ACCESS 0x001a -+#define HOSTCMD_CMD_RF_REG_ACCESS 0x001b -+#define HOSTCMD_CMD_802_11_RADIO_CONTROL 0x001c -+#define HOSTCMD_CMD_MEM_ADDR_ACCESS 0x001d -+#define HOSTCMD_CMD_802_11_TX_POWER 0x001f -+#define HOSTCMD_CMD_802_11_RF_ANTENNA 0x0020 -+#define HOSTCMD_CMD_BROADCAST_SSID_ENABLE 0x0050 /* per-vif */ -+#define HOSTCMD_CMD_SET_CFG 0x008f -+#define HOSTCMD_CMD_SET_RF_CHANNEL 0x010a -+#define HOSTCMD_CMD_SET_AID 0x010d /* per-vif */ -+#define HOSTCMD_CMD_SET_INFRA_MODE 0x010e /* per-vif */ -+#define HOSTCMD_CMD_802_11_RTS_THSD 0x0113 -+#define HOSTCMD_CMD_SET_EDCA_PARAMS 0x0115 -+#define HOSTCMD_CMD_802_11H_DETECT_RADAR 0x0120 -+#define HOSTCMD_CMD_SET_WMM_MODE 0x0123 -+#define HOSTCMD_CMD_HT_GUARD_INTERVAL 0x0124 -+#define HOSTCMD_CMD_SET_FIXED_RATE 0x0126 -+#define HOSTCMD_CMD_SET_IES 0x0127 -+#define HOSTCMD_CMD_SET_LINKADAPT_CS_MODE 0x0129 -+#define HOSTCMD_CMD_DUMP_OTP_DATA 0x0142 -+#define HOSTCMD_CMD_SET_MAC_ADDR 0x0202 /* per-vif */ -+#define HOSTCMD_CMD_SET_RATE_ADAPT_MODE 0x0203 -+#define HOSTCMD_CMD_GET_WATCHDOG_BITMAP 0x0205 -+#define HOSTCMD_CMD_DEL_MAC_ADDR 0x0206 /* per-vif */ -+#define HOSTCMD_CMD_BSS_START 0x1100 /* per-vif */ -+#define HOSTCMD_CMD_AP_BEACON 0x1101 /* per-vif */ -+#define HOSTCMD_CMD_SET_NEW_STN 0x1111 /* per-vif */ -+#define HOSTCMD_CMD_SET_APMODE 0x1114 -+#define HOSTCMD_CMD_SET_SWITCH_CHANNEL 0x1121 -+#define HOSTCMD_CMD_UPDATE_ENCRYPTION 0x1122 /* per-vif */ -+#define HOSTCMD_CMD_BASTREAM 0x1125 -+#define HOSTCMD_CMD_SET_SPECTRUM_MGMT 0x1128 -+#define HOSTCMD_CMD_SET_POWER_CONSTRAINT 0x1129 -+#define HOSTCMD_CMD_SET_COUNTRY_CODE 0x1130 -+#define HOSTCMD_CMD_SET_OPTIMIZATION_LEVEL 0x1133 -+#define HOSTCMD_CMD_SET_WSC_IE 0x1136 /* per-vif */ -+#define HOSTCMD_CMD_GET_RATETABLE 0x1137 -+#define HOSTCMD_CMD_GET_SEQNO 0x1143 -+#define HOSTCMD_CMD_DWDS_ENABLE 0x1144 -+#define HOSTCMD_CMD_FW_FLUSH_TIMER 0x1148 -+#define HOSTCMD_CMD_SET_CDD 0x1150 -+#define HOSTCMD_CMD_SET_BFTYPE 0x1155 -+#define HOSTCMD_CMD_CAU_REG_ACCESS 0x1157 -+#define HOSTCMD_CMD_GET_TEMP 0x1159 -+#define HOSTCMD_CMD_LED_CTRL 0x1169 -+#define HOSTCMD_CMD_GET_FW_REGION_CODE 0x116A -+#define HOSTCMD_CMD_GET_DEVICE_PWR_TBL 0x116B -+#define HOSTCMD_CMD_SET_RATE_DROP 0x1172 -+#define HOSTCMD_CMD_NEWDP_DMATHREAD_START 0x1189 -+#define HOSTCMD_CMD_GET_FW_REGION_CODE_SC4 0x118A -+#define HOSTCMD_CMD_GET_DEVICE_PWR_TBL_SC4 0x118B -+#define HOSTCMD_CMD_QUIET_MODE 0x1201 -+#define HOSTCMD_CMD_CORE_DUMP_DIAG_MODE 0x1202 -+#define HOSTCMD_CMD_802_11_SLOT_TIME 0x1203 -+#define HOSTCMD_CMD_GET_FW_CORE_DUMP 0x1203 -+#define HOSTCMD_CMD_EDMAC_CTRL 0x1204 -+#define HOSTCMD_CMD_TXPWRLMT_CFG 0x1211 -+#define HOSTCMD_CMD_MCAST_CTS 0x4001 -+ -+/* Define general result code for each command */ -+#define HOSTCMD_RESULT_OK 0x0000 -+/* General error */ -+#define HOSTCMD_RESULT_ERROR 0x0001 -+/* Command is not valid */ -+#define HOSTCMD_RESULT_NOT_SUPPORT 0x0002 -+/* Command is pending (will be processed) */ -+#define HOSTCMD_RESULT_PENDING 0x0003 -+/* System is busy (command ignored) */ -+#define HOSTCMD_RESULT_BUSY 0x0004 -+/* Data buffer is not big enough */ -+#define HOSTCMD_RESULT_PARTIAL_DATA 0x0005 -+ -+/* Define channel related constants */ -+#define FREQ_BAND_2DOT4GHZ 0x1 -+#define FREQ_BAND_4DOT9GHZ 0x2 -+#define FREQ_BAND_5GHZ 0x4 -+#define FREQ_BAND_5DOT2GHZ 0x8 -+#define CH_AUTO_WIDTH 0 -+#define CH_10_MHZ_WIDTH 0x1 -+#define CH_20_MHZ_WIDTH 0x2 -+#define CH_40_MHZ_WIDTH 0x4 -+#define CH_80_MHZ_WIDTH 0x5 -+#define CH_160_MHZ_WIDTH 0x6 -+#define EXT_CH_ABOVE_CTRL_CH 0x1 -+#define EXT_CH_AUTO 0x2 -+#define EXT_CH_BELOW_CTRL_CH 0x3 -+#define NO_EXT_CHANNEL 0x0 -+ -+#define ACT_PRIMARY_CHAN_0 0 -+#define ACT_PRIMARY_CHAN_1 1 -+#define ACT_PRIMARY_CHAN_2 2 -+#define ACT_PRIMARY_CHAN_3 3 -+#define ACT_PRIMARY_CHAN_4 4 -+#define ACT_PRIMARY_CHAN_5 5 -+#define ACT_PRIMARY_CHAN_6 6 -+#define ACT_PRIMARY_CHAN_7 7 -+ -+/* Define rate related constants */ -+#define HOSTCMD_ACT_NOT_USE_FIXED_RATE 0x0002 -+ -+/* Define station related constants */ -+#define HOSTCMD_ACT_STA_ACTION_ADD 0 -+#define HOSTCMD_ACT_STA_ACTION_MODIFY 1 -+#define HOSTCMD_ACT_STA_ACTION_REMOVE 2 -+ -+/* Define key related constants */ -+#define MAX_ENCR_KEY_LENGTH 16 -+#define MIC_KEY_LENGTH 8 -+ -+#define KEY_TYPE_ID_WEP 0x00 -+#define KEY_TYPE_ID_TKIP 0x01 -+#define KEY_TYPE_ID_AES 0x02 -+ -+/* Group key for RX only */ -+#define ENCR_KEY_FLAG_RXGROUPKEY 0x00000002 -+#define ENCR_KEY_FLAG_TXGROUPKEY 0x00000004 -+#define ENCR_KEY_FLAG_PAIRWISE 0x00000008 -+#define ENCR_KEY_FLAG_TSC_VALID 0x00000040 -+#define ENCR_KEY_FLAG_WEP_TXKEY 0x01000000 -+#define ENCR_KEY_FLAG_MICKEY_VALID 0x02000000 -+ -+/* Define block ack related constants */ -+#define BA_FLAG_IMMEDIATE_TYPE 1 -+#define BA_FLAG_DIRECTION_UP 0 -+#define BA_FLAG_DIRECTION_DOWN 1 -+ -+/* Define general purpose action */ -+#define HOSTCMD_ACT_GEN_SET 0x0001 -+#define HOSTCMD_ACT_GEN_SET_LIST 0x0002 -+#define HOSTCMD_ACT_GEN_GET_LIST 0x0003 -+ -+/* Define TXPower control action*/ -+#define HOSTCMD_ACT_GET_TARGET_TX_PWR 0x0000 -+#define HOSTCMD_ACT_GET_MAX_TX_PWR 0x0001 -+#define HOSTCMD_ACT_SET_TARGET_TX_PWR 0x0002 -+#define HOSTCMD_ACT_SET_MAX_TX_PWR 0x0003 -+ -+/* Misc */ -+#define WSC_IE_MAX_LENGTH 251 -+#define WSC_IE_SET_BEACON 0 -+#define WSC_IE_SET_PROBE_RESPONSE 1 -+ -+#define HW_SET_PARMS_FEATURES_HOST_PROBE_RESP 0x00000020 -+ -+#define EDMAC_2G_ENABLE_MASK 0x00000001 -+#define EDMAC_2G_ENABLE_SHIFT 0x0 -+#define EDMAC_5G_ENABLE_MASK 0x00000002 -+#define EDMAC_5G_ENABLE_SHIFT 0x1 -+#define EDMAC_2G_THRESHOLD_OFFSET_MASK 0x00000FF0 -+#define EDMAC_2G_THRESHOLD_OFFSET_SHIFT 0x4 -+#define EDMAC_5G_THRESHOLD_OFFSET_MASK 0x000FF000 -+#define EDMAC_5G_THRESHOLD_OFFSET_SHIFT 0xC -+#define EDMAC_QLOCK_BITMAP_MASK 0x0FF00000 -+#define EDMAC_QLOCK_BITMAP_SHIFT 0x14 -+ -+enum { -+ WL_DISABLE = 0, -+ WL_ENABLE = 1, -+ WL_DISABLE_VMAC = 0x80, -+}; -+ -+enum { -+ WL_GET = 0, -+ WL_SET = 1, -+ WL_RESET = 2, -+}; -+ -+enum { -+ WL_LONG_PREAMBLE = 1, -+ WL_SHORT_PREAMBLE = 3, -+ WL_AUTO_PREAMBLE = 5, -+}; -+ -+enum encr_action_type { -+ /* request to enable/disable HW encryption */ -+ ENCR_ACTION_ENABLE_HW_ENCR, -+ /* request to set encryption key */ -+ ENCR_ACTION_TYPE_SET_KEY, -+ /* request to remove one or more keys */ -+ ENCR_ACTION_TYPE_REMOVE_KEY, -+ ENCR_ACTION_TYPE_SET_GROUP_KEY, -+}; -+ -+enum ba_action_type { -+ BA_CREATE_STREAM, -+ BA_UPDATE_STREAM, -+ BA_DESTROY_STREAM, -+ BA_FLUSH_STREAM, -+ BA_CHECK_STREAM, -+}; -+ -+enum mac_type { -+ WL_MAC_TYPE_PRIMARY_CLIENT, -+ WL_MAC_TYPE_SECONDARY_CLIENT, -+ WL_MAC_TYPE_PRIMARY_AP, -+ WL_MAC_TYPE_SECONDARY_AP, -+}; -+ -+/* General host command header */ -+struct hostcmd_header { -+ __le16 cmd; -+ __le16 len; -+ u8 seq_num; -+ u8 macid; -+ __le16 result; -+} __packed; -+ -+/* HOSTCMD_CMD_GET_HW_SPEC */ -+struct hostcmd_get_hw_spec { -+ u8 version; /* version of the HW */ -+ u8 host_if; /* host interface */ -+ __le16 num_wcb; /* Max. number of WCB FW can handle */ -+ __le16 num_mcast_addr; /* MaxNbr of MC addresses FW can handle */ -+ u8 permanent_addr[ETH_ALEN]; /* MAC address programmed in HW */ -+ __le16 region_code; -+ __le16 num_antenna; /* Number of antenna used */ -+ __le32 fw_release_num; /* 4 byte of FW release number */ -+ __le32 wcb_base0; -+ __le32 rxpd_wr_ptr; -+ __le32 rxpd_rd_ptr; -+ __le32 fw_awake_cookie; -+ __le32 wcb_base[SYSADPT_TOTAL_TX_QUEUES - 1]; -+} __packed; -+ -+struct hostcmd_cmd_get_hw_spec { -+ struct hostcmd_header cmd_hdr; -+ struct hostcmd_get_hw_spec hw_spec; -+} __packed; -+ -+/* HOSTCMD_CMD_SET_HW_SPEC */ -+struct hostcmd_set_hw_spec { -+ /* HW revision */ -+ u8 version; -+ /* Host interface */ -+ u8 host_if; -+ /* Max. number of Multicast address FW can handle */ -+ __le16 num_mcast_addr; -+ /* MAC address */ -+ u8 permanent_addr[ETH_ALEN]; -+ /* Region Code */ -+ __le16 region_code; -+ /* 4 byte of FW release number, example 0x1234=1.2.3.4 */ -+ __le32 fw_release_num; -+ /* Firmware awake cookie - used to ensure that the device -+ * is not in sleep mode -+ */ -+ __le32 fw_awake_cookie; -+ /* Device capabilities (see above) */ -+ __le32 device_caps; -+ /* Rx shared memory queue */ -+ __le32 rxpd_wr_ptr; -+ /* Actual number of TX queues in WcbBase array */ -+ __le32 num_tx_queues; -+ /* TX WCB Rings */ -+ __le32 wcb_base[4 + SYSADPT_NUM_OF_AP]; -+ /* Max AMSDU size (00 - AMSDU Disabled, -+ * 01 - 4K, 10 - 8K, 11 - not defined) -+ */ -+ __le32 features; -+ __le32 tx_wcb_num_per_queue; -+ __le32 total_rx_wcb; -+ __le32 acnt_buf_size; -+ __le32 acnt_base_addr; -+} __packed; -+ -+struct hostcmd_cmd_set_hw_spec { -+ struct hostcmd_header cmd_hdr; -+ struct hostcmd_set_hw_spec hw_spec; -+} __packed; -+ -+/* HOSTCMD_CMD_802_11_GET_STAT */ -+struct hostcmd_cmd_802_11_get_stat { -+ struct hostcmd_header cmd_hdr; -+ __le32 tx_retry_successes; -+ __le32 tx_multiple_retry_successes; -+ __le32 tx_failures; -+ __le32 rts_successes; -+ __le32 rts_failures; -+ __le32 ack_failures; -+ __le32 rx_duplicate_frames; -+ __le32 rx_fcs_errors; -+ __le32 tx_watchdog_timeouts; -+ __le32 rx_overflows; -+ __le32 rx_frag_errors; -+ __le32 rx_mem_errors; -+ __le32 pointer_errors; -+ __le32 tx_underflows; -+ __le32 tx_done; -+ __le32 tx_done_buf_try_put; -+ __le32 tx_done_buf_put; -+ /* Put size of requested buffer in here */ -+ __le32 wait_for_tx_buf; -+ __le32 tx_attempts; -+ __le32 tx_successes; -+ __le32 tx_fragments; -+ __le32 tx_multicasts; -+ __le32 rx_non_ctl_pkts; -+ __le32 rx_multicasts; -+ __le32 rx_undecryptable_frames; -+ __le32 rx_icv_errors; -+ __le32 rx_excluded_frames; -+ __le32 rx_weak_iv_count; -+ __le32 rx_unicasts; -+ __le32 rx_bytes; -+ __le32 rx_errors; -+ __le32 rx_rts_count; -+ __le32 tx_cts_count; -+} __packed; -+ -+/* HOSTCMD_CMD_BBP_REG_ACCESS */ -+struct hostcmd_cmd_bbp_reg_access { -+ struct hostcmd_header cmd_hdr; -+ __le16 action; -+ __le16 offset; -+ u8 value; -+ u8 reserverd[3]; -+} __packed; -+ -+/* HOSTCMD_CMD_RF_REG_ACCESS */ -+struct hostcmd_cmd_rf_reg_access { -+ struct hostcmd_header cmd_hdr; -+ __le16 action; -+ __le16 offset; -+ u8 value; -+ u8 reserverd[3]; -+} __packed; -+ -+/* HOSTCMD_CMD_802_11_RADIO_CONTROL */ -+struct hostcmd_cmd_802_11_radio_control { -+ struct hostcmd_header cmd_hdr; -+ __le16 action; -+ /* @bit0: 1/0,on/off, @bit1: 1/0, long/short @bit2: 1/0,auto/fix */ -+ __le16 control; -+ __le16 radio_on; -+} __packed; -+ -+/* HOSTCMD_CMD_MEM_ADDR_ACCESS */ -+struct hostcmd_cmd_mem_addr_access { -+ struct hostcmd_header cmd_hdr; -+ __le32 address; -+ __le16 length; -+ __le16 reserved; -+ __le32 value[64]; -+} __packed; -+ -+/* HOSTCMD_CMD_802_11_TX_POWER */ -+struct hostcmd_cmd_802_11_tx_power { -+ struct hostcmd_header cmd_hdr; -+ __le16 action; -+ __le16 band; -+ __le16 ch; -+ __le16 bw; -+ __le16 sub_ch; -+ __le16 power_level_list[SYSADPT_TX_POWER_LEVEL_TOTAL]; -+} __packed; -+ -+struct hostcmd_cmd_802_11_tx_power_kf2 { -+ struct hostcmd_header cmd_hdr; -+ __le16 action; -+ __le16 band; -+ __le16 ch; -+ __le16 bw; -+ __le16 sub_ch; -+ __le16 power_level_list[SYSADPT_TX_GRP_PWR_LEVEL_TOTAL]; -+} __packed; -+ -+/* HOSTCMD_CMD_802_11_RF_ANTENNA */ -+struct hostcmd_cmd_802_11_rf_antenna { -+ struct hostcmd_header cmd_hdr; -+ __le16 action; -+ __le16 antenna_mode; /* Number of antennas or 0xffff(diversity) */ -+} __packed; -+ -+/* HOSTCMD_CMD_BROADCAST_SSID_ENABLE */ -+struct hostcmd_cmd_broadcast_ssid_enable { -+ struct hostcmd_header cmd_hdr; -+ __le32 enable; -+ __le32 hidden_ssid_info; -+} __packed; -+ -+/* HOSTCMD_CMD_SET_CFG */ -+struct hostcmd_cmd_set_cfg { -+ struct hostcmd_header cmd_hdr; -+ /* Action */ -+ __le16 action; -+ /* Type */ -+ __le16 type; -+ /* Data length */ -+ __le16 data_len; -+ /* Data */ -+ u8 data[1]; -+} __packed; -+ -+/* HOSTCMD_CMD_SET_RF_CHANNEL */ -+#define FREQ_BAND_MASK 0x0000003f -+#define CHNL_WIDTH_MASK 0x000007c0 -+#define CHNL_WIDTH_SHIFT 6 -+#define ACT_PRIMARY_MASK 0x00003800 -+#define ACT_PRIMARY_SHIFT 11 -+ -+struct hostcmd_cmd_set_rf_channel { -+ struct hostcmd_header cmd_hdr; -+ __le16 action; -+ u8 curr_chnl; -+ __le32 chnl_flags; -+} __packed; -+ -+struct hostcmd_cmd_set_rf_channel_kf2 { -+ struct hostcmd_header cmd_hdr; -+ __le16 action; -+ u8 curr_chnl; -+ __le32 chnl_flags; -+ u8 remain_on_chan; -+} __packed; -+ -+/* HOSTCMD_CMD_SET_AID */ -+struct hostcmd_cmd_set_aid { -+ struct hostcmd_header cmd_hdr; -+ __le16 aid; -+ u8 mac_addr[ETH_ALEN]; /* AP's Mac Address(BSSID) */ -+ __le32 gprotect; -+ u8 ap_rates[SYSADPT_MAX_DATA_RATES_G]; -+} __packed; -+ -+/* HOSTCMD_CMD_SET_INFRA_MODE */ -+struct hostcmd_cmd_set_infra_mode { -+ struct hostcmd_header cmd_hdr; -+} __packed; -+ -+/* HOSTCMD_CMD_802_11_RTS_THSD */ -+struct hostcmd_cmd_802_11_rts_thsd { -+ struct hostcmd_header cmd_hdr; -+ __le16 action; -+ __le16 threshold; -+} __packed; -+ -+/* HOSTCMD_CMD_SET_EDCA_PARAMS */ -+struct hostcmd_cmd_set_edca_params { -+ struct hostcmd_header cmd_hdr; -+ /* 0 = get all, 0x1 =set CWMin/Max, 0x2 = set TXOP , 0x4 =set AIFSN */ -+ __le16 action; -+ __le16 txop; /* in unit of 32 us */ -+ __le32 cw_max; /* 0~15 */ -+ __le32 cw_min; /* 0~15 */ -+ u8 aifsn; -+ u8 txq_num; /* Tx Queue number. */ -+} __packed; -+ -+/* HOSTCMD_CMD_802_11H_DETECT_RADAR */ -+#define RADAR_TYPE_CODE_0 0 -+#define RADAR_TYPE_CODE_53 53 -+#define RADAR_TYPE_CODE_56 56 -+#define RADAR_TYPE_CODE_ETSI 151 -+ -+struct hostcmd_cmd_802_11h_detect_radar { -+ struct hostcmd_header cmd_hdr; -+ __le16 action; -+ __le16 radar_type_code; -+ __le16 min_chirp_cnt; -+ __le16 chirp_time_intvl; -+ __le16 pw_filter; -+ __le16 min_num_radar; -+ __le16 pri_min_num; -+} __packed; -+ -+/* HOSTCMD_CMD_SET_WMM_MODE */ -+struct hostcmd_cmd_set_wmm_mode { -+ struct hostcmd_header cmd_hdr; -+ __le16 action; /* 0->unset, 1->set */ -+} __packed; -+ -+/* HOSTCMD_CMD_HT_GUARD_INTERVAL */ -+struct hostcmd_cmd_ht_guard_interval { -+ struct hostcmd_header cmd_hdr; -+ __le32 action; -+ __le32 gi_type; -+} __packed; -+ -+/* HOSTCMD_CMD_SET_FIXED_RATE */ -+struct fix_rate_flag { /* lower rate after the retry count */ -+ /* 0: legacy, 1: HT */ -+ __le32 fix_rate_type; -+ /* 0: retry count is not valid, 1: use retry count specified */ -+ __le32 retry_count_valid; -+} __packed; -+ -+struct fix_rate_entry { -+ struct fix_rate_flag fix_rate_type_flags; -+ /* depending on the flags above, this can be either a legacy -+ * rate(not index) or an MCS code. -+ */ -+ __le32 fixed_rate; -+ __le32 retry_count; -+} __packed; -+ -+struct hostcmd_cmd_set_fixed_rate { -+ struct hostcmd_header cmd_hdr; -+ /* HOSTCMD_ACT_NOT_USE_FIXED_RATE 0x0002 */ -+ __le32 action; -+ /* use fixed rate specified but firmware can drop to */ -+ __le32 allow_rate_drop; -+ __le32 entry_count; -+ struct fix_rate_entry fixed_rate_table[4]; -+ u8 multicast_rate; -+ u8 multi_rate_tx_type; -+ u8 management_rate; -+} __packed; -+ -+/* HOSTCMD_CMD_SET_IES */ -+struct hostcmd_cmd_set_ies { -+ struct hostcmd_header cmd_hdr; -+ __le16 action; /* 0->unset, 1->set */ -+ __le16 ie_list_len_ht; -+ __le16 ie_list_len_vht; -+ __le16 ie_list_len_proprietary; -+ /*Buffer size same as Generic_Beacon*/ -+ u8 ie_list_ht[148]; -+ u8 ie_list_vht[24]; -+ u8 ie_list_proprietary[112]; -+} __packed; -+ -+/* HOSTCMD_CMD_SET_LINKADAPT_CS_MODE */ -+struct hostcmd_cmd_set_linkadapt_cs_mode { -+ struct hostcmd_header cmd_hdr; -+ __le16 action; -+ __le16 cs_mode; -+} __packed; -+ -+/* HOSTCMD_CMD_DUMP_OTP_DATA */ -+struct hostcmd_cmd_dump_otp_data { -+ struct hostcmd_header cmd_hdr; -+ u8 pload[0]; -+} __packed; -+ -+/* HOSTCMD_CMD_SET_MAC_ADDR, HOSTCMD_CMD_DEL_MAC_ADDR */ -+struct hostcmd_cmd_set_mac_addr { -+ struct hostcmd_header cmd_hdr; -+ __le16 mac_type; -+ u8 mac_addr[ETH_ALEN]; -+} __packed; -+ -+/* HOSTCMD_CMD_SET_RATE_ADAPT_MODE */ -+struct hostcmd_cmd_set_rate_adapt_mode { -+ struct hostcmd_header cmd_hdr; -+ __le16 action; -+ __le16 rate_adapt_mode; /* 0:Indoor, 1:Outdoor */ -+} __packed; -+ -+/* HOSTCMD_CMD_GET_WATCHDOG_BITMAP */ -+struct hostcmd_cmd_get_watchdog_bitmap { -+ struct hostcmd_header cmd_hdr; -+ u8 watchdog_bitmap; /* for SW/BA */ -+} __packed; -+ -+/* HOSTCMD_CMD_BSS_START */ -+struct hostcmd_cmd_bss_start { -+ struct hostcmd_header cmd_hdr; -+ __le32 enable; /* FALSE: Disable or TRUE: Enable */ -+ u8 amsdu; -+} __packed; -+ -+/* HOSTCMD_CMD_AP_BEACON */ -+struct cf_params { -+ u8 elem_id; -+ u8 len; -+ u8 cfp_cnt; -+ u8 cfp_period; -+ __le16 cfp_max_duration; -+ __le16 cfp_duration_remaining; -+} __packed; -+ -+struct ibss_params { -+ u8 elem_id; -+ u8 len; -+ __le16 atim_window; -+} __packed; -+ -+union ss_params { -+ struct cf_params cf_param_set; -+ struct ibss_params ibss_param_set; -+} __packed; -+ -+struct fh_params { -+ u8 elem_id; -+ u8 len; -+ __le16 dwell_time; -+ u8 hop_set; -+ u8 hop_pattern; -+ u8 hop_index; -+} __packed; -+ -+struct ds_params { -+ u8 elem_id; -+ u8 len; -+ u8 current_chnl; -+} __packed; -+ -+union phy_params { -+ struct fh_params fh_param_set; -+ struct ds_params ds_param_set; -+} __packed; -+ -+struct rsn_ie { -+ u8 elem_id; -+ u8 len; -+ u8 oui_type[4]; /* 00:50:f2:01 */ -+ u8 ver[2]; -+ u8 grp_key_cipher[4]; -+ u8 pws_key_cnt[2]; -+ u8 pws_key_cipher_list[4]; -+ u8 auth_key_cnt[2]; -+ u8 auth_key_list[4]; -+} __packed; -+ -+struct rsn48_ie { -+ u8 elem_id; -+ u8 len; -+ u8 ver[2]; -+ u8 grp_key_cipher[4]; -+ u8 pws_key_cnt[2]; -+ u8 pws_key_cipher_list[4]; -+ u8 auth_key_cnt[2]; -+ u8 auth_key_list[4]; -+ u8 rsn_cap[2]; -+ u8 pmk_id_cnt[2]; -+ u8 pmk_id_list[16]; /* Should modify to 16 * S */ -+ u8 reserved[8]; -+} __packed; -+ -+struct ac_param_rcd { -+ u8 aci_aifsn; -+ u8 ecw_min_max; -+ __le16 txop_lim; -+} __packed; -+ -+struct wmm_param_elem { -+ u8 elem_id; -+ u8 len; -+ u8 oui[3]; -+ u8 type; -+ u8 sub_type; -+ u8 version; -+ u8 qos_info; -+ u8 rsvd; -+ struct ac_param_rcd ac_be; -+ struct ac_param_rcd ac_bk; -+ struct ac_param_rcd ac_vi; -+ struct ac_param_rcd ac_vo; -+} __packed; -+ -+struct channel_info { -+ u8 first_channel_num; -+ u8 num_channels; -+ u8 max_tx_pwr_level; -+} __packed; -+ -+struct country { -+ u8 elem_id; -+ u8 len; -+ u8 country_str[3]; -+ struct channel_info channel_info[40]; -+} __packed; -+ -+struct start_cmd { -+ u8 sta_mac_addr[ETH_ALEN]; -+ u8 ssid[IEEE80211_MAX_SSID_LEN]; -+ u8 bss_type; -+ __le16 bcn_period; -+ u8 dtim_period; -+ union ss_params ss_param_set; -+ union phy_params phy_param_set; -+ __le16 probe_delay; -+ __le16 cap_info; -+ u8 b_rate_set[SYSADPT_MAX_DATA_RATES_G]; -+ u8 op_rate_set[SYSADPT_MAX_DATA_RATES_G]; -+ struct rsn_ie rsn_ie; -+ struct rsn48_ie rsn48_ie; -+ struct wmm_param_elem wmm_param; -+ struct country country; -+ __le32 ap_rf_type; /* 0->B, 1->G, 2->Mixed, 3->A, 4->11J */ -+ u8 rsvd[3]; -+ u8 bssid[ETH_ALEN]; /* only for 88W8997 */ -+} __packed; -+ -+struct hostcmd_cmd_ap_beacon { -+ struct hostcmd_header cmd_hdr; -+ struct start_cmd start_cmd; -+} __packed; -+ -+/* HOSTCMD_CMD_SET_NEW_STN */ -+struct add_ht_info { -+ u8 control_chnl; -+ u8 add_chnl; -+ __le16 op_mode; -+ __le16 stbc; -+} __packed; -+ -+struct peer_info { -+ __le32 legacy_rate_bitmap; -+ u8 ht_rates[4]; -+ __le16 cap_info; -+ __le16 ht_cap_info; -+ u8 mac_ht_param_info; -+ u8 mrvl_sta; -+ struct add_ht_info add_ht_info; -+ __le32 tx_bf_capabilities; /* EXBF_SUPPORT */ -+ __le32 vht_max_rx_mcs; -+ __le32 vht_cap; -+ /* 0:20Mhz, 1:40Mhz, 2:80Mhz, 3:160 or 80+80Mhz */ -+ u8 vht_rx_channel_width; -+} __packed; -+ -+struct hostcmd_cmd_set_new_stn { -+ struct hostcmd_header cmd_hdr; -+ __le16 aid; -+ u8 mac_addr[ETH_ALEN]; -+ __le16 stn_id; -+ __le16 action; -+ __le16 if_type; -+ struct peer_info peer_info; -+ /* UAPSD_SUPPORT */ -+ u8 qos_info; -+ u8 is_qos_sta; -+ __le32 fw_sta_ptr; -+} __packed; -+ -+struct retry_cnt_qos { -+ u8 retry_cfg_enable; -+ u8 retry_cnt_BK; -+ u8 retry_cnt_BE; -+ u8 retry_cnt_VI; -+ u8 retry_cnt_VO; -+} __packed; -+ -+struct peer_info_sc4 { -+ __le32 legacy_rate_bitmap; -+ u8 ht_rates[4]; -+ __le16 cap_info; -+ __le16 ht_cap_info; -+ u8 mac_ht_param_info; -+ u8 mrvl_sta; -+ struct add_ht_info add_ht_info; -+ __le32 tx_bf_capabilities; /* EXBF_SUPPORT */ -+ __le32 vht_max_rx_mcs; -+ __le32 vht_cap; -+ /* 0:20Mhz, 1:40Mhz, 2:80Mhz, 3:160 or 80+80Mhz */ -+ u8 vht_rx_channel_width; -+ struct retry_cnt_qos retry_cnt_qos; -+ u8 assoc_rssi; -+} __packed; -+ -+struct hostcmd_cmd_set_new_stn_sc4 { -+ struct hostcmd_header cmd_hdr; -+ __le16 aid; -+ u8 mac_addr[ETH_ALEN]; -+ __le16 stn_id; -+ __le16 action; -+ __le16 reserved; -+ struct peer_info_sc4 peer_info; -+ /* UAPSD_SUPPORT */ -+ u8 qos_info; -+ u8 is_qos_sta; -+ __le32 fw_sta_ptr; -+ __le32 wds; -+} __packed; -+ -+/* HOSTCMD_CMD_SET_APMODE */ -+struct hostcmd_cmd_set_apmode { -+ struct hostcmd_header cmd_hdr; -+ u8 apmode; -+} __packed; -+ -+/* HOSTCMD_CMD_SET_SWITCH_CHANNEL */ -+struct hostcmd_cmd_set_switch_channel { -+ struct hostcmd_header cmd_hdr; -+ __le32 next_11h_chnl; -+ __le32 mode; -+ __le32 init_count; -+ __le32 chnl_flags; -+ __le32 next_ht_extchnl_offset; -+ __le32 dfs_test_mode; -+} __packed; -+ -+/* HOSTCMD_CMD_UPDATE_ENCRYPTION */ -+struct hostcmd_cmd_update_encryption { -+ struct hostcmd_header cmd_hdr; -+ /* Action type - see encr_action_type */ -+ __le32 action_type; /* encr_action_type */ -+ /* size of the data buffer attached. */ -+ __le32 data_length; -+ u8 mac_addr[ETH_ALEN]; -+ u8 action_data[1]; -+} __packed; -+ -+struct wep_type_key { -+ /* WEP key material (max 128bit) */ -+ u8 key_material[MAX_ENCR_KEY_LENGTH]; -+} __packed; -+ -+struct encr_tkip_seqcnt { -+ __le16 low; -+ __le32 high; -+} __packed; -+ -+struct tkip_type_key { -+ /* TKIP Key material. Key type (group or pairwise key) is -+ * determined by flags -+ */ -+ /* in KEY_PARAM_SET structure. */ -+ u8 key_material[MAX_ENCR_KEY_LENGTH]; -+ /* MIC keys */ -+ u8 tkip_tx_mic_key[MIC_KEY_LENGTH]; -+ u8 tkip_rx_mic_key[MIC_KEY_LENGTH]; -+ struct encr_tkip_seqcnt tkip_rsc; -+ struct encr_tkip_seqcnt tkip_tsc; -+} __packed; -+ -+struct aes_type_key { -+ /* AES Key material */ -+ u8 key_material[MAX_ENCR_KEY_LENGTH]; -+} __packed; -+ -+union mwl_key_type { -+ struct wep_type_key wep_key; -+ struct tkip_type_key tkip_key; -+ struct aes_type_key aes_key; -+} __packed; -+ -+struct key_param_set { -+ /* Total length of this structure (Key is variable size array) */ -+ __le16 length; -+ /* Key type - WEP, TKIP or AES-CCMP. */ -+ /* See definitions above */ -+ __le16 key_type_id; -+ /* key flags (ENCR_KEY_FLAG_XXX_ */ -+ __le32 key_info; -+ /* For WEP only - actual key index */ -+ __le32 key_index; -+ /* Size of the key */ -+ __le16 key_len; -+ /* Key material (variable size array) */ -+ union mwl_key_type key; -+ u8 mac_addr[ETH_ALEN]; -+} __packed; -+ -+struct hostcmd_cmd_set_key { -+ struct hostcmd_header cmd_hdr; -+ /* Action type - see encr_action_type */ -+ __le32 action_type; /* encr_action_type */ -+ /* size of the data buffer attached. */ -+ __le32 data_length; -+ /* data buffer - maps to one KEY_PARAM_SET structure */ -+ struct key_param_set key_param; -+} __packed; -+ -+/* HOSTCMD_CMD_BASTREAM */ -+#define BA_TYPE_MASK 0x00000001 -+#define BA_DIRECTION_MASK 0x0000000e -+#define BA_DIRECTION_SHIFT 1 -+ -+#define BA_TYPE_MASK_NDP 0x00000003 -+#define BA_DIRECTION_MASK_NDP 0x0000001c -+#define BA_DIRECTION_SHIFT_NDP 2 -+ -+struct ba_context { -+ __le32 context; -+} __packed; -+ -+/* parameters for block ack creation */ -+struct create_ba_params { -+ /* BA Creation flags - see above */ -+ __le32 flags; -+ /* idle threshold */ -+ __le32 idle_thrs; -+ /* block ack transmit threshold (after how many pkts should we -+ * send BAR?) -+ */ -+ __le32 bar_thrs; -+ /* receiver window size */ -+ __le32 window_size; -+ /* MAC Address of the BA partner */ -+ u8 peer_mac_addr[ETH_ALEN]; -+ /* Dialog Token */ -+ u8 dialog_token; -+ /* TID for the traffic stream in this BA */ -+ u8 tid; -+ /* shared memory queue ID (not sure if this is required) */ -+ u8 queue_id; -+ u8 param_info; -+ /* returned by firmware - firmware context pointer. */ -+ /* this context pointer will be passed to firmware for all -+ * future commands. -+ */ -+ struct ba_context fw_ba_context; -+ u8 reset_seq_no; /** 0 or 1**/ -+ __le16 current_seq; -+ __le32 vht_rx_factor; -+ /* This is for virtual station in Sta proxy mode for V6FW */ -+ u8 sta_src_mac_addr[ETH_ALEN]; -+} __packed; -+ -+/* new transmit sequence number information */ -+struct ba_update_seq_num { -+ /* BA flags - see above */ -+ __le32 flags; -+ /* returned by firmware in the create ba stream response */ -+ struct ba_context fw_ba_context; -+ /* new sequence number for this block ack stream */ -+ __le16 ba_seq_num; -+} __packed; -+ -+struct ba_stream_context { -+ /* BA Stream flags */ -+ __le32 flags; -+ /* returned by firmware in the create ba stream response */ -+ struct ba_context fw_ba_context; -+ u8 tid; -+ u8 peer_mac_addr[ETH_ALEN]; -+} __packed; -+ -+union ba_info { -+ /* information required to create BA Stream... */ -+ struct create_ba_params create_params; -+ /* update starting/new sequence number etc. */ -+ struct ba_update_seq_num updt_seq_num; -+ /* destroy an existing stream... */ -+ struct ba_stream_context destroy_params; -+ /* destroy an existing stream... */ -+ struct ba_stream_context flush_params; -+} __packed; -+ -+struct hostcmd_cmd_bastream { -+ struct hostcmd_header cmd_hdr; -+ __le32 action_type; -+ union ba_info ba_info; -+} __packed; -+ -+/* HOSTCMD_CMD_SET_SPECTRUM_MGMT */ -+struct hostcmd_cmd_set_spectrum_mgmt { -+ struct hostcmd_header cmd_hdr; -+ __le32 spectrum_mgmt; -+} __packed; -+ -+/* HOSTCMD_CMD_SET_POWER_CONSTRAINT */ -+struct hostcmd_cmd_set_power_constraint { -+ struct hostcmd_header cmd_hdr; -+ __le32 power_constraint; -+} __packed; -+ -+/* HOSTCMD_CMD_SET_COUNTRY_CODE */ -+struct domain_chnl_entry { -+ u8 first_chnl_num; -+ u8 chnl_num; -+ u8 max_transmit_pw; -+} __packed; -+ -+struct domain_country_info { -+ u8 country_string[3]; -+ u8 g_chnl_len; -+ struct domain_chnl_entry domain_entry_g[1]; -+ u8 a_chnl_len; -+ struct domain_chnl_entry domain_entry_a[20]; -+} __packed; -+ -+struct hostcmd_cmd_set_country_code { -+ struct hostcmd_header cmd_hdr; -+ __le32 action ; /* 0 -> unset, 1 ->set */ -+ struct domain_country_info domain_info; -+} __packed; -+ -+/* HOSTCMD_CMD_SET_OPTIMIZATION_LEVEL */ -+struct hostcmd_cmd_set_optimization_level { -+ struct hostcmd_header cmd_hdr; -+ u8 opt_level; -+} __packed; -+ -+/* HOSTCMD_CMD_SET_WSC_IE */ -+struct hostcmd_cmd_set_wsc_ie { -+ struct hostcmd_header cmd_hdr; -+ __le16 ie_type; /* 0 -- beacon. or 1 -- probe response. */ -+ __le16 len; -+ u8 data[WSC_IE_MAX_LENGTH]; -+} __packed; -+ -+/* HOSTCMD_CMD_GET_RATETABLE */ -+struct hostcmd_cmd_get_ratetable { -+ struct hostcmd_header cmd_hdr; -+ u8 addr[ETH_ALEN]; -+ u8 type; /* 0: SU, 1: MU */ -+ /* multiply 2 because 2 DWORD in rate info */ -+ __le32 sorted_rates_idx_map[2 * SYSADPT_MAX_RATE_ADAPT_RATES]; -+} __packed; -+ -+/* HOSTCMD_CMD_GET_SEQNO */ -+struct hostcmd_cmd_get_seqno { -+ struct hostcmd_header cmd_hdr; -+ u8 mac_addr[ETH_ALEN]; -+ u8 tid; -+ __le16 seq_no; -+ u8 reserved; -+} __packed; -+ -+/* HOSTCMD_CMD_DWDS_ENABLE */ -+struct hostcmd_cmd_dwds_enable { -+ struct hostcmd_header cmd_hdr; -+ __le32 enable; /* 0 -- Disable. or 1 -- Enable. */ -+} __packed; -+ -+/* HOSTCMD_CMD_FW_FLUSH_TIMER */ -+struct hostcmd_cmd_fw_flush_timer { -+ struct hostcmd_header cmd_hdr; -+ /* 0 -- Disable. > 0 -- holds time value in usecs. */ -+ __le32 value; -+} __packed; -+ -+/* HOSTCMD_CMD_SET_CDD */ -+struct hostcmd_cmd_set_cdd { -+ struct hostcmd_header cmd_hdr; -+ __le32 enable; -+} __packed; -+ -+/* HOSTCMD_CMD_SET_BFTYPE */ -+struct hostcmd_cmd_set_bftype { -+ struct hostcmd_header cmd_hdr; -+ __le32 action; -+ __le32 mode; -+} __packed; -+ -+/* HOSTCMD_CMD_GET_TEMP */ -+struct hostcmd_cmd_get_temp { -+ struct hostcmd_header cmd_hdr; -+ __le32 celcius; -+ __le32 raw_data; -+} __packed; -+ -+/* HOSTCMD_CMD_LED_CTRL */ -+struct hostcmd_cmd_led_ctrl { -+ struct hostcmd_header cmd_hdr; -+ __le16 action; /* 0: GET, 1: SET (only SET is supported) */ -+ u8 led_enable; /* 0: Disable, 1: Enable */ -+ u8 led_control; /* 0: HW 1: SW (only SW is supported) */ -+ u8 led_blink_rate; /* 1: Slow, 2: Medium, 3: Fast blink */ -+ u8 reserved; -+} __packed; -+ -+/* HOSTCMD_CMD_GET_FW_REGION_CODE */ -+struct hostcmd_cmd_get_fw_region_code { -+ struct hostcmd_header cmd_hdr; -+ __le32 status; /* 0 = Found, 1 = Error */ -+ __le32 fw_region_code; -+} __packed; -+ -+/* HOSTCMD_CMD_GET_DEVICE_PWR_TBL */ -+#define HAL_TRPC_ID_MAX 16 -+ -+struct channel_power_tbl { -+ u8 channel; -+ u8 tx_pwr[HAL_TRPC_ID_MAX]; -+ u8 dfs_capable; -+ u8 ax_ant; -+ u8 cdd; -+} __packed; -+ -+struct hostcmd_cmd_get_device_pwr_tbl { -+ struct hostcmd_header cmd_hdr; -+ __le16 status; /* 0 = Found, 1 = Error */ -+ u8 region_code; -+ u8 number_of_channels; -+ __le32 current_channel_index; -+ /* Only for 1 channel, so, 1 channel at a time */ -+ struct channel_power_tbl channel_pwr_tbl; -+} __packed; -+ -+/* HOSTCMD_CMD_SET_RATE_DROP */ -+struct hostcmd_cmd_set_rate_drop { -+ struct hostcmd_header cmd_hdr; -+ __le32 enable; -+ __le32 rate_index; -+ __le32 sta_index; -+} __packed; -+ -+/* HOSTCMD_CMD_NEWDP_DMATHREAD_START */ -+struct hostcmd_cmd_newdp_dmathread_start { -+ struct hostcmd_header cmd_hdr; -+} __packed; -+ -+/* HOSTCMD_CMD_GET_FW_REGION_CODE_SC4 */ -+struct hostcmd_cmd_get_fw_region_code_sc4 { -+ struct hostcmd_header cmd_hdr; -+ __le32 status; /* 0 = Found, 1 = Error */ -+ __le32 fw_region_code; -+} __packed; -+ -+/* HOSTCMD_CMD_GET_DEVICE_PWR_TBL_SC4 */ -+#define HAL_TRPC_ID_MAX_SC4 32 -+#define MAX_GROUP_PER_CHANNEL_5G 39 -+#define MAX_GROUP_PER_CHANNEL_2G 21 -+#define MAX(a, b) (((a) > (b)) ? (a) : (b)) -+#define MAX_GROUP_PER_CHANNEL_RATE \ -+ MAX(MAX_GROUP_PER_CHANNEL_5G, MAX_GROUP_PER_CHANNEL_2G) -+ -+struct channel_power_tbl_sc4 { -+ u8 channel; -+ u8 grp_pwr[MAX_GROUP_PER_CHANNEL_RATE]; -+ u8 tx_pwr[HAL_TRPC_ID_MAX_SC4]; -+ u8 dfs_capable; -+ u8 ax_ant; -+ u8 cdd; -+ u8 rsvd; -+} __packed; -+ -+struct hostcmd_cmd_get_device_pwr_tbl_sc4 { -+ struct hostcmd_header cmd_hdr; -+ __le16 status; /* 0 = Found, 1 = Error */ -+ u8 region_code; -+ u8 number_of_channels; -+ __le32 current_channel_index; -+ /* Only for 1 channel, so, 1 channel at a time */ -+ struct channel_power_tbl_sc4 channel_pwr_tbl; -+} __packed; -+ -+/* HOSTCMD_CMD_QUIET_MODE */ -+struct hostcmd_cmd_quiet_mode { -+ struct hostcmd_header cmd_hdr; -+ __le16 action; -+ __le32 enable; -+ __le32 period; -+ __le32 duration; -+ __le32 next_offset; -+} __packed; -+ -+/* HOSTCMD_CMD_CORE_DUMP_DIAG_MODE */ -+struct hostcmd_cmd_core_dump_diag_mode { -+ struct hostcmd_header cmd_hdr; -+ __le16 status; -+} __packed; -+ -+/* HOSTCMD_CMD_GET_FW_CORE_DUMP */ -+#define MAX_CORE_REGIONS 20 -+#define MAX_CORE_SYMBOLS 30 -+#define MVL_COREDUMP_DIAG_MODE 0x00000001 -+#define MVL_COREDUMP_INCL_EXT 0x00000002 -+#define MAX_CORE_DUMP_BUFFER 2048 -+ -+struct core_region { -+ __le32 address; -+ __le32 length; -+} __packed; -+ -+struct core_symbol { -+ u8 name[16]; -+ __le32 address; -+ __le32 length; -+ __le16 entries; -+} __packed; -+ -+struct coredump { -+ u8 version_major; -+ u8 version_minor; -+ u8 version_patch; -+ u8 hdr_version; -+ u8 num_regions; -+ u8 num_symbols; -+ u8 fill[2]; -+ struct core_region region[MAX_CORE_REGIONS]; -+ struct core_symbol symbol[MAX_CORE_SYMBOLS]; -+ __le32 fill_end[40]; -+} __packed; -+ -+struct coredump_cmd { -+ __le32 context; -+ __le32 buffer; -+ __le32 buffer_len; -+ __le16 size_kb; -+ __le16 flags; -+} __packed; -+ -+struct debug_mem_cmd { -+ __le32 set; -+ __le32 type; -+ __le32 addr; -+ __le32 val; -+} __packed; -+ -+struct hostcmd_cmd_get_fw_core_dump { -+ struct hostcmd_header cmd_hdr; -+ union { -+ struct coredump_cmd coredump; -+ struct debug_mem_cmd debug_mem; -+ } cmd_data; -+ /*Buffer where F/W Copies the Core Dump*/ -+ char buffer[MAX_CORE_DUMP_BUFFER]; -+} __packed; -+ -+struct hostcmd_cmd_get_fw_core_dump_ { -+ struct hostcmd_header cmd_hdr; -+ union { -+ struct coredump_cmd coredump; -+ struct debug_mem_cmd debug_mem; -+ } cmd_data; -+} __packed; -+ -+/* HOSTCMD_CMD_802_11_SLOT_TIME */ -+struct hostcmd_cmd_802_11_slot_time { -+ struct hostcmd_header cmd_hdr; -+ __le16 action; -+ /* 0:long slot; 1:short slot */ -+ __le16 short_slot; -+} __packed; -+ -+/* HOSTCMD_CMD_EDMAC_CTRL */ -+struct hostcmd_cmd_edmac_ctrl { -+ struct hostcmd_header cmd_hdr; -+ __le16 action; -+ __le16 ed_ctrl_2g; -+ __le16 ed_offset_2g; -+ __le16 ed_ctrl_5g; -+ __le16 ed_offset_5g; -+ __le16 ed_bitmap_txq_lock; -+} __packed; -+ -+/* HOSTCMD_CMD_TXPWRLMT_CFG */ -+#define TXPWRLMT_CFG_VERSION_INFO_LEN 0x4 -+#define TXPWRLMT_CFG_MAX_SUBBAND_INFO 0x5 -+#define TXPWRLMT_CFG_SIG_LEN 0x4 -+#define TXPWRLMT_CFG_SIGNATURE 0xA1240E01 -+ -+struct hostcmd_cmd_txpwrlmt_cfg { -+ struct hostcmd_header cmd_hdr; -+ /* Action */ -+ __le16 action; -+ /*Sub band id*/ -+ u8 subband_id; -+ /* Cfg Complete Info*/ -+ u8 cfgComplete; -+ /* Data length */ -+ __le16 data_len; -+ /*number of entries*/ -+ __le16 num_entries; -+ /* Data */ -+ u8 data[1]; -+} __packed; -+ -+struct mwl_txpwrlmt_cfg_entry_hdr { -+ /* subband id */ -+ __le16 id; -+ /* length */ -+ __le16 len; -+ /* number of entries */ -+ __le16 num_entries; -+} __packed; -+ -+/* HOSTCMD_CMD_MCAST_CTS */ -+struct hostcmd_cmd_mcast_cts { -+ struct hostcmd_header cmd_hdr; -+ u8 enable; /* 1:enable, 0:disable */ -+} __packed; -+ -+#endif /* _HOSTCMD_H_ */ -diff --git a/drivers/net/wireless/marvell/mwlwifi/hif/pcie/dev.h b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/dev.h -new file mode 100644 -index 000000000000..2c26bff80683 ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/dev.h -@@ -0,0 +1,1032 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file defines device related information. */ -+ -+#ifndef _DEV_H_ -+#define _DEV_H_ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#define PCIE_DRV_NAME KBUILD_MODNAME -+#define PCIE_DRV_VERSION "10.3.8.0-20181210" -+ -+#define PCIE_MIN_BYTES_HEADROOM 64 -+#define PCIE_MIN_TX_HEADROOM_KF2 96 -+#define PCIE_NUM_OF_DESC_DATA SYSADPT_TOTAL_TX_QUEUES -+#define PCIE_AMPDU_QUEUES 4 -+#define PCIE_MAX_NUM_TX_DESC 256 -+#define PCIE_TX_QUEUE_LIMIT (3 * PCIE_MAX_NUM_TX_DESC) -+#define PCIE_TX_WAKE_Q_THRESHOLD (2 * PCIE_MAX_NUM_TX_DESC) -+#define PCIE_DELAY_FREE_Q_LIMIT PCIE_MAX_NUM_TX_DESC -+#define PCIE_MAX_NUM_RX_DESC 256 -+#define PCIE_RECEIVE_LIMIT 64 -+ -+enum { -+ IEEE_TYPE_MANAGEMENT = 0, -+ IEEE_TYPE_CONTROL, -+ IEEE_TYPE_DATA -+}; -+ -+#define MAC_REG_ADDR(offset) (offset) -+#define MAC_REG_ADDR_PCI(offset) ((pcie_priv->iobase1 + 0xA000) + offset) -+ -+#define MCU_CCA_CNT MAC_REG_ADDR(0x06A0) -+#define MCU_TXPE_CNT MAC_REG_ADDR(0x06A4) -+#define MCU_LAST_READ MAC_REG_ADDR(0x06A8) -+ -+/* Map to 0x80000000 (Bus control) on BAR0 */ -+#define MACREG_REG_H2A_INTERRUPT_EVENTS 0x00000C18 /* (From host to ARM) */ -+#define MACREG_REG_H2A_INTERRUPT_CAUSE 0x00000C1C /* (From host to ARM) */ -+#define MACREG_REG_H2A_INTERRUPT_MASK 0x00000C20 /* (From host to ARM) */ -+#define MACREG_REG_H2A_INTERRUPT_CLEAR_SEL 0x00000C24 /* (From host to ARM) */ -+#define MACREG_REG_H2A_INTERRUPT_STATUS_MASK 0x00000C28 /* (From host to ARM) */ -+ -+#define MACREG_REG_A2H_INTERRUPT_EVENTS 0x00000C2C /* (From ARM to host) */ -+#define MACREG_REG_A2H_INTERRUPT_CAUSE 0x00000C30 /* (From ARM to host) */ -+#define MACREG_REG_A2H_INTERRUPT_MASK 0x00000C34 /* (From ARM to host) */ -+#define MACREG_REG_A2H_INTERRUPT_CLEAR_SEL 0x00000C38 /* (From ARM to host) */ -+#define MACREG_REG_A2H_INTERRUPT_STATUS_MASK 0x00000C3C /* (From ARM to host) */ -+ -+/* Map to 0x80000000 on BAR1 */ -+#define MACREG_REG_GEN_PTR 0x00000C10 -+#define MACREG_REG_INT_CODE 0x00000C14 -+ -+/* Bit definition for MACREG_REG_A2H_INTERRUPT_CAUSE (A2HRIC) */ -+#define MACREG_A2HRIC_BIT_TX_DONE BIT(0) -+#define MACREG_A2HRIC_BIT_RX_RDY BIT(1) -+#define MACREG_A2HRIC_BIT_OPC_DONE BIT(2) -+#define MACREG_A2HRIC_BIT_MAC_EVENT BIT(3) -+#define MACREG_A2HRIC_BIT_RX_PROBLEM BIT(4) -+#define MACREG_A2HRIC_BIT_RADIO_OFF BIT(5) -+#define MACREG_A2HRIC_BIT_RADIO_ON BIT(6) -+#define MACREG_A2HRIC_BIT_RADAR_DETECT BIT(7) -+#define MACREG_A2HRIC_BIT_ICV_ERROR BIT(8) -+#define MACREG_A2HRIC_BIT_WEAKIV_ERROR BIT(9) -+#define MACREG_A2HRIC_BIT_QUE_EMPTY BIT(10) -+#define MACREG_A2HRIC_BIT_QUE_FULL BIT(11) -+#define MACREG_A2HRIC_BIT_CHAN_SWITCH BIT(12) -+#define MACREG_A2HRIC_BIT_TX_WATCHDOG BIT(13) -+#define MACREG_A2HRIC_BA_WATCHDOG BIT(14) -+/* 15 taken by ISR_TXACK */ -+#define MACREG_A2HRIC_BIT_SSU_DONE BIT(16) -+#define MACREG_A2HRIC_CONSEC_TXFAIL BIT(17) -+ -+#define ISR_SRC_BITS (MACREG_A2HRIC_BIT_RX_RDY | \ -+ MACREG_A2HRIC_BIT_TX_DONE | \ -+ MACREG_A2HRIC_BIT_OPC_DONE | \ -+ MACREG_A2HRIC_BIT_MAC_EVENT | \ -+ MACREG_A2HRIC_BIT_WEAKIV_ERROR | \ -+ MACREG_A2HRIC_BIT_ICV_ERROR | \ -+ MACREG_A2HRIC_BIT_SSU_DONE | \ -+ MACREG_A2HRIC_BIT_RADAR_DETECT | \ -+ MACREG_A2HRIC_BIT_CHAN_SWITCH | \ -+ MACREG_A2HRIC_BIT_TX_WATCHDOG | \ -+ MACREG_A2HRIC_BIT_QUE_EMPTY | \ -+ MACREG_A2HRIC_BA_WATCHDOG | \ -+ MACREG_A2HRIC_CONSEC_TXFAIL) -+ -+#define MACREG_A2HRIC_BIT_MASK ISR_SRC_BITS -+ -+/* Bit definition for MACREG_REG_H2A_INTERRUPT_CAUSE (H2ARIC) */ -+#define MACREG_H2ARIC_BIT_PPA_READY BIT(0) -+#define MACREG_H2ARIC_BIT_DOOR_BELL BIT(1) -+#define MACREG_H2ARIC_BIT_PS BIT(2) -+#define MACREG_H2ARIC_BIT_PSPOLL BIT(3) -+#define ISR_RESET BIT(15) -+#define ISR_RESET_AP33 BIT(26) -+ -+/* Data descriptor related constants */ -+#define EAGLE_RXD_CTRL_DRIVER_OWN 0x00 -+#define EAGLE_RXD_CTRL_DMA_OWN 0x80 -+ -+#define EAGLE_RXD_STATUS_OK 0x01 -+ -+#define EAGLE_TXD_STATUS_IDLE 0x00000000 -+#define EAGLE_TXD_STATUS_OK 0x00000001 -+#define EAGLE_TXD_STATUS_FW_OWNED 0x80000000 -+ -+struct pcie_tx_desc { -+ u8 data_rate; -+ u8 tx_priority; -+ __le16 qos_ctrl; -+ __le32 pkt_ptr; -+ __le16 pkt_len; -+ u8 dest_addr[ETH_ALEN]; -+ __le32 pphys_next; -+ __le32 sap_pkt_info; -+ __le32 rate_info; -+ u8 type; -+ u8 xmit_control; /* bit 0: use rateinfo, bit 1: disable ampdu */ -+ __le16 reserved; -+ __le32 tcpack_sn; -+ __le32 tcpack_src_dst; -+ __le32 reserved1; -+ __le32 reserved2; -+ u8 reserved3[2]; -+ u8 packet_info; -+ u8 packet_id; -+ __le16 packet_len_and_retry; -+ __le16 packet_rate_info; -+ __le32 flags; -+ __le32 status; -+} __packed; -+ -+struct pcie_tx_hndl { -+ struct sk_buff *psk_buff; -+ struct pcie_tx_desc *pdesc; -+ struct pcie_tx_hndl *pnext; -+}; -+ -+/* Receive rate information constants */ -+#define RX_RATE_INFO_FORMAT_11A 0 -+#define RX_RATE_INFO_FORMAT_11B 1 -+#define RX_RATE_INFO_FORMAT_11N 2 -+#define RX_RATE_INFO_FORMAT_11AC 4 -+ -+#define RX_RATE_INFO_HT20 0 -+#define RX_RATE_INFO_HT40 1 -+#define RX_RATE_INFO_HT80 2 -+#define RX_RATE_INFO_HT160 3 -+ -+#define RX_RATE_INFO_LONG_INTERVAL 0 -+#define RX_RATE_INFO_SHORT_INTERVAL 1 -+ -+#define MWL_RX_RATE_FORMAT_MASK 0x0007 -+#define MWL_RX_RATE_NSS_MASK 0x0018 -+#define MWL_RX_RATE_NSS_SHIFT 3 -+#define MWL_RX_RATE_BW_MASK 0x0060 -+#define MWL_RX_RATE_BW_SHIFT 5 -+#define MWL_RX_RATE_GI_MASK 0x0080 -+#define MWL_RX_RATE_GI_SHIFT 7 -+#define MWL_RX_RATE_RT_MASK 0xFF00 -+#define MWL_RX_RATE_RT_SHIFT 8 -+ -+struct pcie_rx_desc { -+ __le16 pkt_len; /* total length of received data */ -+ __le16 rate; /* receive rate information */ -+ __le32 pphys_buff_data; /* physical address of payload data */ -+ __le32 pphys_next; /* physical address of next RX desc */ -+ __le16 qos_ctrl; /* received QosCtrl field variable */ -+ __le16 ht_sig2; /* like name states */ -+ __le32 hw_rssi_info; -+ __le32 hw_noise_floor_info; -+ u8 noise_floor; -+ u8 reserved[3]; -+ u8 rssi; /* received signal strengt indication */ -+ u8 status; /* status field containing USED bit */ -+ u8 channel; /* channel this pkt was received on */ -+ u8 rx_control; /* the control element of the desc */ -+ __le32 reserved1[3]; -+} __packed; -+ -+struct pcie_rx_hndl { -+ struct sk_buff *psk_buff; /* associated sk_buff for Linux */ -+ struct pcie_rx_desc *pdesc; -+ struct pcie_rx_hndl *pnext; -+}; -+ -+struct pcie_desc_data { -+ dma_addr_t pphys_tx_ring; /* ptr to first TX desc (phys.) */ -+ struct pcie_tx_desc *ptx_ring; /* ptr to first TX desc (virt.) */ -+ struct pcie_tx_hndl *tx_hndl; -+ struct pcie_tx_hndl *pnext_tx_hndl;/* next TX handle that can be used */ -+ struct pcie_tx_hndl *pstale_tx_hndl;/* the staled TX handle */ -+ dma_addr_t pphys_rx_ring; /* ptr to first RX desc (phys.) */ -+ struct pcie_rx_desc *prx_ring; /* ptr to first RX desc (virt.) */ -+ struct pcie_rx_hndl *rx_hndl; -+ struct pcie_rx_hndl *pnext_rx_hndl;/* next RX handle that can be used */ -+ u32 wcb_base; /* FW base offset for registers */ -+ u32 rx_desc_write; /* FW descriptor write position */ -+ u32 rx_desc_read; /* FW descriptor read position */ -+ u32 rx_buf_size; /* length of the RX buffers */ -+}; -+ -+/* DMA header used by firmware and hardware. */ -+struct pcie_dma_data { -+ __le16 fwlen; -+ struct ieee80211_hdr wh; -+ char data[0]; -+} __packed; -+ -+/* New Data Path */ -+#define MACREG_REG_SCRATCH3 0x00000C44 -+#define MACREG_REG_TXSENDHEAD 0x00000CD0 -+#define MACREG_REG_TXSEDNTAIL 0x00000CD4 -+#define MACREG_REG_TXDONEHEAD 0x00000CD8 -+#define MACREG_REG_TXDONETAIL 0x00000CDC -+#define MACREG_REG_RXDESCHEAD 0x00000CE0 -+#define MACREG_REG_RXDESCTAIL 0x00000CE4 -+#define MACREG_REG_RXDONEHEAD 0x00000CE8 -+#define MACREG_REG_RXDONETAIL 0x00000CEC -+#define MACREG_REG_ACNTHEAD 0x00000CF0 -+#define MACREG_REG_ACNTTAIL 0x00000CF4 -+ -+/* Buff removed from Tx Send Ring */ -+#define MACREG_A2HRIC_TX_DESC_TAIL_RDY (1<<9) -+/* Buff added to Tx Done Ring */ -+#define MACREG_A2HRIC_TX_DONE_HEAD_RDY (1<<10) -+/* Records added to Accounting Ring */ -+#define MACREG_A2HRIC_ACNT_HEAD_RDY (1<<12) -+/* Buff removed from Rx Desc Ring */ -+#define MACREG_A2HRIC_RX_DESC_TAIL_RDY (1<<17) -+/* Buff added to Rx Done Ring */ -+#define MACREG_A2HRIC_RX_DONE_HEAD_RDY (1<<18) -+#define MACREG_A2HRIC_NEWDP_DFS (1<<19) -+#define MACREG_A2HRIC_NEWDP_CHANNEL_SWITCH (1<<20) -+ -+#define ISR_SRC_BITS_NDP ((MACREG_A2HRIC_ACNT_HEAD_RDY) | \ -+ (MACREG_A2HRIC_RX_DONE_HEAD_RDY) | \ -+ (MACREG_A2HRIC_NEWDP_DFS) | \ -+ (MACREG_A2HRIC_NEWDP_CHANNEL_SWITCH)) -+ -+#define MACREG_A2HRIC_BIT_MASK_NDP ISR_SRC_BITS_NDP -+ -+#define MIN_BYTES_RX_HEADROOM (64 + 2) -+#define AMPDU_QUEUES_NDP (SYSADPT_MAX_STA_SC4 * \ -+ SYSADPT_MAX_TID) -+#define MAX_NUM_TX_DESC 1024 -+#define MAX_NUM_RX_DESC (1024 * 16) -+#define MAX_TX_RING_SEND_SIZE (4 * MAX_NUM_TX_DESC) -+#define MAX_TX_RING_DONE_SIZE MAX_NUM_TX_DESC -+#define MAX_RX_RING_SEND_SIZE MAX_NUM_RX_DESC -+#define MAX_RX_RING_DONE_SIZE MAX_NUM_RX_DESC -+#define DEFAULT_ACNT_RING_SIZE 0x10000 -+#define MAX_AGGR_SIZE 1900 -+#define TX_QUEUE_LIMIT MAX_NUM_TX_DESC -+#define TX_WAKE_Q_THRESHOLD (MAX_NUM_TX_DESC - 256) -+ -+/* RateCode usage notes: -+ * * General -+ * * No error checking is provided on RateCodes, so usage of invalid values -+ * or rates not supported by HW can result in undefined operation. -+ * * Some values are not allowed by Std, but are included to sanitize the -+ * table; -+ * * MaxPwr should only be used for rates that can be sent using Max Power, -+ * such as for TxEVM limits or regulatory. It is only valid for Host -+ * Generated frames, and not for DRA, etc. -+ * * VHT -+ * * Need to reconsile MU. -+ * * HT -+ * * MCS and SS are made to mimic 11ac, so MCS=mcs[2:0] and SS=mcs[4:3]; -+ * * MCS32 is selected by providing MCS=10; -+ * * Legacy -+ * * MCS0..7 = 6/9/12/18/24/36/48/54; -+ * * MCS8..15 = 1S/1L/2S/2L/5.5S/5.5L/11S/11L; -+ * * BW is used to request legacy duplicate modes; -+ */ -+#define RATECODE_DEFAULT 0xFFFF /* Don't override the Rate */ -+#define RATECODE_TYPE_MASK 0xC000 /* Mask to extract Type */ -+#define RATECODE_TYPE_SHIFT 14 /* Shift to extract Type */ -+#define RATECODE_TYPE_VHT 0x8000 /* Use VHT rates */ -+#define RATECODE_TYPE_HT 0x4000 /* Use HT rates */ -+#define RATECODE_TYPE_LEGACY 0x0000 /* Use Legacy (a/b/g) rates */ -+#define RATECODE_MAXPWR 0x2000 /* Send at Max Power / Off Channel */ -+#define RATECODE_RSVD 0x1000 /* Unused */ -+#define RATECODE_STBC 0x0800 /* Use Space Time Block Codes */ -+#define RATECODE_BFMR 0x0400 /* Use Beamforming */ -+#define RATECODE_SS_MASK 0x0300 /* Mask to extract nSS-1 */ -+#define RATECODE_SS_SHIFT 8 /* Shift to extract nSS-1 */ -+#define RATECODE_MCS_MASK 0x00F0 /* Mask to extract MCS rate */ -+#define RATECODE_MCS_SHIFT 4 /* Shift to extract MCS rate */ -+#define RATECODE_BW_MASK 0x000C /* Mask to extract Channel BW */ -+#define RATECODE_BW_SHIFT 2 /* Shift to extract Channel BW */ -+#define RATECODE_BW_160MHZ 0x000C /* Send 160M wide packet (or 80+80) */ -+#define RATECODE_BW_80MHZ 0x0008 /* Send 80M wide packet */ -+#define RATECODE_BW_40MHZ 0x0004 /* Send 40M wide packet */ -+#define RATECODE_BW_20MHZ 0x0000 /* Send 20M wide packet */ -+#define RATECODE_LDPC 0x0002 /* Use Low Density Parity Codes */ -+#define RATECODE_SGI 0x0001 /* Use Short Guard Interval */ -+ -+#define TXRING_CTRL_LEN_SHIFT 0 /* PCIe Payload size (Starts w/ SNAP) */ -+#define TXRING_CTRL_LEN_MASK 0x3FFF /* PCIe Payload size (Starts w/ SNAP) */ -+#define TXRING_CTRL_QID_SHIFT 14 /* Queue ID (STA*UP, Mcast, MC2UC, etc)*/ -+#define TXRING_CTRL_QID_MASK 0xFFF /* Queue ID (STA*UP, Mcast, MC2UC, etc)*/ -+#define TXRING_CTRL_TAG_SHIFT 26 /* Tags for special Processing */ -+#define TXRING_CTRL_TAG_MASK 0x3F /* Tags for special Processing */ -+#define TXRING_CTRL_TAG_MGMT 0x01 /* Has Host generated dot11 Header */ -+#define TXRING_CTRL_TAG_EAP 0x02 /* Tag for EAPOL frames */ -+#define TXRING_CTRL_TAG_TCP_ACK 0x4 -+#define TXRING_CTRL_TAG_RSVD 0x3C /* Unused */ -+ -+struct tx_info { /* Tx INFO used by MAC HW */ -+ __le32 reserved0[10]; -+ __le32 rate_info; -+ __le32 reserved1[14]; -+} __packed; -+ -+struct pcie_tx_desc_ndp { -+ union { /* Union for Tx DA/SA or Mgmt Overrides */ -+ struct { /* Fields for Data frames */ -+ u8 da[ETH_ALEN]; /* L2 Destination Address */ -+ u8 sa[ETH_ALEN]; /* L2 Source Address */ -+ }; -+ struct { /* Fields when marked as Mgmt */ -+ __le16 rate_code; /* Rate Code: Table + Index */ -+ u8 max_retry; -+ u8 pad[5]; /* Unused */ -+ __le32 call_back; /* Used for Packet returned to FW */ -+ }; -+ } u; -+ __le32 ctrl; /* Bit fields (TXRING_CTRL_*) */ -+ __le32 data; /* PCIe Payload Pointer (Starts w/ SNAP) */ -+ __le32 user; /* Value returned to Host when done */ -+ __le32 tcp_dst_src; -+ __le32 tcp_sn; -+} __packed; -+ -+struct tx_ring_done { -+ __le32 user; -+} __packed; -+ -+#define RXRING_CTRL_CASE_SHIFT 0 /* What is in the buffer(RXRING_CASE_*) */ -+#define RXRING_CTRL_CASE_MASK 0x1F /* What is in the buffer(RXRING_CASE_*) */ -+#define RXRING_CTRL_STA_SHIFT 5 /* STA information (or Mcast group) */ -+#define RXRING_CTRL_STA_MASK 0x1FF /* STA information (or Mcast group) */ -+#define RXRING_CTRL_STA_UNKNOWN 0x1FF /* STA Idx for packets from Unknown STA */ -+#define RXRING_CTRL_STA_FROMDS 0x1FE /* STA Idx for packets from DS */ -+#define RXRING_CTRL_TID_SHIFT 14 /* TID/UP for QoS Data frames */ -+#define RXRING_CTRL_TID_MASK 0xF /* TID/UP for QoS Data frames */ -+#define RXRING_CTRL_KEY_SHIFT 18 /* Key Type used (KEY_TYPE_*) */ -+#define RXRING_CTRL_KEY_MASK 0xF /* Key Type used (KEY_TYPE_*) */ -+#define RXRING_CTRL_TRUNC (1UL<<31) /* Packet Truncated */ -+ -+/* Rx Buffer Formats -+ * Each Case listed above will indicate the format used, and each format will -+ * carry their length in the packet buffer. Should the packet be too big for -+ * the buffer, it will be truncated, but the full length will still be -+ * indicated. Currently only a single, fixed size Rx Pool is envisioned. -+ * -+ * Fmt0 is used for Slow path, when some processing of dot11 headers may still -+ * be required, or for promiscuous mode captures. It is in the HW RxINFO -+ * (rx_info_t) format including dot11_t followed by Payload. The Length field in -+ * the dot11_t is updated to only include Payload bytes, and is in Little Endian -+ * format. If the frame is too big, it is truncated to the buffer size, and -+ * promiscuous packets may also be configured for truncation to reduce load. The -+ * mark field is replaced with software status, and the RSSI will be updated to -+ * apply Rx calibration. -+ * -+ * Fmt1 is used for fast path Data packets in the run state, where all rx -+ * processing of dot11 headers is performed from radio FW. It has an AMSDU -+ * centric format of DA/SA/Len followed by SNAP, with the Length in Big Endian -+ * Format. In most cases conversion to Ethernet format is accomplished by -+ * copying 12 bytes to drop 8 bytes in the middle. -+ * -+ * Fmt2 is used for fast path AMSDU packets that are malformed. They just -+ * contain the dot11 header (dot11_t) containing the residual Len (Little -+ * Endian) after any valid MSDU have been extracted. The header is followed by -+ * the first invalid MSDU which will be truncated to 64 bytes. -+ */ -+enum { /* What is in Rx Buffer and why it was delivered */ -+ /* Data for Assoc Clients in Run State on Channel [Fmt1] */ -+ RXRING_CASE_FAST_DATA, -+ RXRING_CASE_FAST_BAD_AMSDU, /* Fast Data with bad AMSDU Header [Fmt2] */ -+ /* Data for Assoc Clients using unconfigured queue [Fmt0] */ -+ RXRING_CASE_SLOW_NOQUEUE, -+ /* Data for Assoc Clients not matching Run State [Fmt0] */ -+ RXRING_CASE_SLOW_NORUN, -+ /* Data for filtered Multicast groups [Fmt0] */ -+ RXRING_CASE_SLOW_MCAST, -+ RXRING_CASE_SLOW_BAD_STA, /* Data for Unassoc Clients [Fmt0] */ -+ RXRING_CASE_SLOW_BAD_MIC, /* Decrypt failure [Fmt0] */ -+ RXRING_CASE_SLOW_BAD_PN, /* Decrypt PN replay [Fmt0] */ -+ RXRING_CASE_SLOW_MGMT, /* Mgmt traffic to this AP or Bcast [Fmt0]*/ -+ RXRING_CASE_SLOW_PROMISC, /* Packets captured promiscuously [Fmt0] */ -+ RXRING_CASE_SLOW_DEL_DONE, /* Client has been deleted [N/A] */ -+ RXRING_CASE_DROP, /* Buffer returned to Host [N/A] */ -+}; -+ -+enum { /* Type of Key */ -+ KEY_TYPE_NONE, /* Bypass (never stored in real keys) */ -+ KEY_TYPE_WEP40, /* WEP with 40 bit key + 24 bit IV = 64 */ -+ KEY_TYPE_WEP104, /* WEP with 104 bit key + 24 bit IV = 128 */ -+ KEY_TYPE_TKIP, /* TKIP */ -+ KEY_TYPE_CCMP128, /* CCMP with 128 bit Key */ -+ KEY_TYPE_CCMP256, /* CCMP with 256 bit Key + 16 byte MIC */ -+ KEY_TYPE_WAPI, /* WAPI */ -+ KEY_TYPE_UNKNOWN, /* Not known what key was used (Rx Only) */ -+ KEY_TYPE_GCMP128, /* GCMP with 128 bit Key */ -+ KEY_TYPE_GCMP256, /* GCMP with 256 bit Key + 16 byte MIC */ -+}; -+ -+#define RXINFO_RSSI_X_SHIFT 24 -+#define RXINFO_RSSI_X_MASK 0xFF -+#define RXINFO_HT_SIG1_SHIFT 0 -+#define RXINFO_HT_SIG1_MASK 0xFFFFFF -+#define RXINFO_HT_SIG2_SHIFT 0 -+#define RXINFO_HT_SIG2_MASK 0x3FFFF -+#define RXINFO_RATE_SHIFT 24 -+#define RXINFO_RATE_MASK 0xFF -+#define RXINFO_NF_A_SHIFT 12 -+#define RXINFO_NF_A_MASK 0xFFF -+#define RXINFO_NF_B_SHIFT 0 -+#define RXINFO_NF_B_MASK 0xFFF -+#define RXINFO_NF_C_SHIFT 12 -+#define RXINFO_NF_C_MASK 0xFFF -+#define RXINFO_NF_D_SHIFT 0 -+#define RXINFO_NF_D_MASK 0xFFF -+#define RXINFO_PARAM_SHIFT 0 -+#define RXINFO_PARAM_MASK 0xFFFFFF -+ -+struct rx_info { /* HW Rx buffer */ -+ __le32 reserved0[2]; -+ __le32 rssi_x; -+ __le32 reserved1[2]; -+ __le32 ht_sig1; -+ __le32 ht_sig2_rate; -+ __le32 reserved2[6]; -+ __le32 nf_a_b; -+ __le32 nf_c_d; -+ __le32 reserved3[6]; -+ __le32 param; -+ __le32 reserved4[2]; -+ __le32 hdr[0]; /* Len from HW includes rx_info w/ hdr */ -+} __packed; -+ -+struct pcie_rx_desc_ndp { /* ToNIC Rx Empty Buffer Ring Entry */ -+ __le32 data; /* PCIe Payload Pointer */ -+ __le32 user; /* Value returned to Host when done */ -+} __packed; -+ -+struct rx_ring_done { /* FromNIC Rx Done Ring Entry */ -+ __le32 user; /* Value returned to Host when done */ -+ __le32 tsf; /* Rx Radio Timestamp from MAC */ -+ __le32 ctrl; /* Bit fields (RXRING_CTRL_*) */ -+} __packed; -+ -+struct pcie_desc_data_ndp { -+ dma_addr_t pphys_tx_ring; /* ptr to first TX desc (phys.) */ -+ struct pcie_tx_desc_ndp *ptx_ring;/* ptr to first TX desc (virt.) */ -+ dma_addr_t pphys_rx_ring; /* ptr to first RX desc (phys.) */ -+ struct pcie_rx_desc_ndp *prx_ring;/* ptr to first RX desc (virt.) */ -+ u32 wcb_base; /* FW base offset for registers */ -+ u32 rx_buf_size; /* length of the RX buffers */ -+ u32 tx_sent_tail; /* index to the TX desc FW used */ -+ u32 tx_sent_head; /* index to next TX desc to be used*/ -+ u32 tx_done_tail; /* index to Tx Done queue tail */ -+ /* keept the skb owned by fw */ -+ dma_addr_t pphys_tx_buflist[MAX_TX_RING_SEND_SIZE]; -+ struct sk_buff *tx_vbuflist[MAX_TX_RING_SEND_SIZE]; -+ u32 tx_vbuflist_idx; /* idx to empty slot in tx_vbuflist*/ -+ struct sk_buff *rx_vbuflist[MAX_NUM_RX_DESC]; -+ struct tx_ring_done *ptx_ring_done; -+ dma_addr_t pphys_tx_ring_done; /* ptr to first TX done desc (phys.) */ -+ struct rx_ring_done *prx_ring_done; -+ dma_addr_t pphys_rx_ring_done; /* ptr to first RX done desc (phys.) */ -+ dma_addr_t pphys_acnt_ring; /* ptr to first account record (phys.)*/ -+ u8 *pacnt_ring; /* ptr to first accounting record */ -+ u32 tx_desc_busy_cnt; -+ u8 *pacnt_buf; -+ u32 acnt_ring_size; -+}; -+ -+struct ndp_rx_counter { -+ u32 fast_data_cnt; -+ u32 fast_bad_amsdu_cnt; -+ u32 slow_noqueue_cnt; -+ u32 slow_norun_cnt; -+ u32 slow_mcast_cnt; -+ u32 slow_bad_sta_cnt; -+ u32 slow_bad_mic_cnt; -+ u32 slow_bad_pn_cnt; -+ u32 slow_mgmt_cnt; -+ u32 slow_promisc_cnt; -+ u32 drop_cnt; -+ u32 offch_promisc_cnt; -+ u32 mu_pkt_cnt; -+}; -+ -+/* KF2 - 88W8997 */ -+#define PCIE_MAX_TXRX_BD 0x20 -+/* PCIE read data pointer for queue 0 and 1 */ -+#define PCIE_RD_DATA_PTR_Q0_Q1 0xC1A4 /* 0x8000C1A4 */ -+/* PCIE read data pointer for queue 2 and 3 */ -+#define PCIE_RD_DATA_PTR_Q2_Q3 0xC1A8 /* 0x8000C1A8 */ -+/* PCIE write data pointer for queue 0 and 1 */ -+#define PCIE_WR_DATA_PTR_Q0_Q1 0xC174 /* 0x8000C174 */ -+/* PCIE write data pointer for queue 2 and 3 */ -+#define PCIE_WR_DATA_PTR_Q2_Q3 0xC178 /* 0x8000C178 */ -+ -+/* TX buffer description read pointer */ -+#define REG_TXBD_RDPTR PCIE_RD_DATA_PTR_Q0_Q1 -+/* TX buffer description write pointer */ -+#define REG_TXBD_WRPTR PCIE_WR_DATA_PTR_Q0_Q1 -+ -+#define PCIE_TX_START_PTR 16 -+ -+#define PCIE_TXBD_MASK 0x0FFF0000 -+#define PCIE_TXBD_WRAP_MASK 0x1FFF0000 -+ -+#define PCIE_BD_FLAG_RX_ROLLOVER_IND BIT(12) -+#define PCIE_BD_FLAG_TX_START_PTR BIT(16) -+#define PCIE_BD_FLAG_TX_ROLLOVER_IND BIT(28) -+#define PCIE_BD_FLAG_TX2_START_PTR BIT(0) -+#define PCIE_BD_FLAG_TX2_ROLLOVER_IND BIT(12) -+ -+#define PCIE_BD_FLAG_FIRST_DESC BIT(0) -+#define PCIE_BD_FLAG_LAST_DESC BIT(1) -+ -+#define PCIE_TX_WCB_FLAGS_DONT_ENCRYPT 0x00000001 -+#define PCIE_TX_WCB_FLAGS_NO_CCK_RATE 0x00000002 -+ -+#define PCIE_TXBD_NOT_FULL(wrptr, rdptr) \ -+ (((wrptr & PCIE_TXBD_MASK) != (rdptr & PCIE_TXBD_MASK)) \ -+ || ((wrptr & PCIE_BD_FLAG_TX_ROLLOVER_IND) == \ -+ (rdptr & PCIE_BD_FLAG_TX_ROLLOVER_IND))) -+ -+struct pcie_data_buf { -+ /* Buffer descriptor flags */ -+ __le16 flags; -+ /* Offset of fragment/pkt to start of ip header */ -+ __le16 offset; -+ /* Fragment length of the buffer */ -+ __le16 frag_len; -+ /* Length of the buffer */ -+ __le16 len; -+ /* Physical address of the buffer */ -+ __le64 paddr; -+ /* Reserved */ -+ __le32 reserved; -+} __packed; -+ -+struct pcie_pfu_dma_data { -+ struct pcie_tx_desc tx_desc; -+ struct pcie_dma_data dma_data; -+} __packed; -+ -+struct pcie_priv { -+ struct mwl_priv *mwl_priv; -+ struct pci_dev *pdev; -+ void __iomem *iobase0; /* MEM Base Address Register 0 */ -+ void __iomem *iobase1; /* MEM Base Address Register 1 */ -+ u32 next_bar_num; -+ -+ struct sk_buff_head txq[PCIE_NUM_OF_DESC_DATA]; -+ -+ spinlock_t int_mask_lock ____cacheline_aligned_in_smp; -+ struct tasklet_struct tx_task; -+ struct tasklet_struct tx_done_task; -+ struct tasklet_struct rx_task; -+ struct tasklet_struct qe_task; -+ unsigned int tx_head_room; -+ int txq_limit; -+ int txq_wake_threshold; -+ bool is_tx_schedule; -+ bool is_tx_done_schedule; -+ int recv_limit; -+ bool is_rx_schedule; -+ bool is_qe_schedule; -+ u32 qe_trig_num; -+ unsigned long qe_trig_time; -+ -+ /* various descriptor data */ -+ /* for tx descriptor data */ -+ spinlock_t tx_desc_lock ____cacheline_aligned_in_smp; -+ struct pcie_desc_data desc_data[PCIE_NUM_OF_DESC_DATA]; -+ int delay_q_idx; -+ struct sk_buff *delay_q[PCIE_DELAY_FREE_Q_LIMIT]; -+ /* number of descriptors owned by fw at any one time */ -+ int fw_desc_cnt[PCIE_NUM_OF_DESC_DATA]; -+ -+ /* new data path */ -+ struct pcie_desc_data_ndp desc_data_ndp; -+ int tx_done_cnt; -+ struct ieee80211_sta *sta_link[SYSADPT_MAX_STA_SC4 + 1]; -+ struct sk_buff_head rx_skb_trace; -+ struct ndp_rx_counter rx_cnts; -+ u32 rx_skb_unlink_err; -+ u32 signature_err; -+ u32 recheck_rxringdone; -+ u32 acnt_busy; -+ u32 acnt_wrap; -+ u32 acnt_drop; -+ -+ /* KF2 - 88W8997 */ -+ struct firmware *cal_data; -+ /* Write pointer for TXBD ring */ -+ u32 txbd_wrptr; -+ /* Shadow copy of TXBD read pointer */ -+ u32 txbd_rdptr; -+ /* TXBD ring size */ -+ u32 txbd_ring_size; -+ /* Virtual base address of txbd_ring */ -+ u8 *txbd_ring_vbase; -+ /* Physical base address of txbd_ring */ -+ dma_addr_t txbd_ring_pbase; -+ /* Ring of buffer descriptors for TX */ -+ struct pcie_data_buf *txbd_ring[PCIE_MAX_TXRX_BD]; -+ struct sk_buff *tx_buf_list[PCIE_MAX_TXRX_BD]; -+}; -+ -+enum { /* Definition of accounting record codes */ -+ ACNT_CODE_BUSY = 0, /* Marked busy until filled in */ -+ ACNT_CODE_WRAP, /* Used to pad when wrapping */ -+ ACNT_CODE_DROP, /* Count of dropped records */ -+ ACNT_CODE_TX_ENQUEUE, /* TXINFO when added to TCQ (acnt_tx_s) */ -+ ACNT_CODE_RX_PPDU, /* RXINFO for each PPDu (acnt_rx_s) */ -+ ACNT_CODE_TX_FLUSH, /* Flush Tx Queue */ -+ ACNT_CODE_RX_RESET, /* Channel Change / Rx Reset */ -+ ACNT_CODE_TX_RESET, /* TCQ reset */ -+ ACNT_CODE_QUOTE_LEVEL,/* Quota Level changes */ -+ ACNT_CODE_TX_DONE, /* Tx status when done */ -+ ACNT_CODE_RA_STATS, /* rateinfo PER (acnt_ra_s) */ -+ ACNT_CODE_BA_STATS, /* BA stats (acnt_ba_s) */ -+ ACNT_CODE_BF_MIMO_CTRL,/* BF Mimo Ctrl Field Log (acnt_bf_mimo_ctrl_s)*/ -+}; -+ -+struct acnt_s { /* Baseline Accounting Record format */ -+ __le16 code; /* Unique code for each type */ -+ u8 len; /* Length in DWORDS, including header */ -+ u8 pad; /* Alignment for generic, but specific can reuse*/ -+ __le32 tsf; /* Timestamp for Entry (when len>1) */ -+} __packed; -+ -+struct acnt_tx_s { /* Accounting Record For Tx (at Enqueue time) */ -+ __le16 code; /* Unique code for each type */ -+ u8 len; /* Length in DWORDS, including header */ -+ u8 tcq; /* Which TCQ was used */ -+ __le32 tsf; /* Timestamp for Entry (when len>1) */ -+ __le64 bitmap; /* Map of SeqNr when AMPDU */ -+ __le16 air_time; /* Air Time used by PPDU */ -+ __le16 npkts; /* Number of Descriptors sent (AMPDU&AMSDU) */ -+ __le16 qid; /* Transmit Queue ID */ -+ __le16 latency; /* Latency of oldest frame in AMPDU (128us) */ -+ __le16 rate1; /* Rate Code for sending data */ -+ __le16 rate2; /* Rate Code for sending RTS/CTS protection */ -+ u8 rate_tbl_index; /* Rate table index for this TxInfo rate */ -+ u8 type; /* SU:0 or MU:1 */ -+ u8 pad[1]; /* Unused */ -+ u8 retries; /* Number of retries of oldest frame in AMPDU */ -+ __le32 tx_cnt; /* No. of pkt sent */ -+ struct tx_info tx_info;/* Transmit parameters used for 1st MPDU/AMPDU */ -+ struct pcie_dma_data hdr;/* Dot11 header used for 1st MPDU in AMPDU */ -+ u8 payload[0]; /* Variable Payload by use case */ -+} __packed; -+ -+struct acnt_rx_s { /* Accounting Record for Rx PPDU */ -+ __le16 code; /* Unique code for each type */ -+ u8 len; /* Length in DWORDS, including header */ -+ u8 flags; /* Flags (ACNTRX_*) */ -+ __le32 tsf; /* Timestamp for Entry (when len>1) */ -+ __le64 bitmap; /* Map of SeqNr when AMPDU */ -+ __le16 air_time; /* Air Time used by PPDU (no CSMA overhead) */ -+ __le16 rate; /* Rate Code for receiving data */ -+ struct rx_info rx_info;/* Receive parameters from 1st valid MPDU/AMPDU*/ -+} __packed; -+ -+struct acnt_ra_s { /* Accounting Record w/ rateinfo PER */ -+ __le16 code; /* Unique code for each type */ -+ u8 len; /* Length in DWORDS, including header */ -+ u8 per; /* PER for this rateinfo */ -+ __le32 tsf; /* Timestamp for Entry (when len>1) */ -+ __le16 stn_id; /* sta index this rateinfo is tied to */ -+ u8 type; /* SU:0 or MU:1 */ -+ u8 rate_tbl_index; /* ratetbl index */ -+ __le32 rate_info; /* rateinfo for this ratetbl index */ -+ __le32 tx_attempt_cnt;/* Total tx pkt during rate adapt interval */ -+} __packed; -+ -+struct acnt_ba_s { /* Accounting Record w/ rateinfo PER */ -+ __le16 code; /* Unique code for each type */ -+ u8 len; /* Length in DWORDS, including header */ -+ u8 ba_hole; /* Total missing pkt in a BA */ -+ __le32 tsf; /* Timestamp for Entry (when len>1) */ -+ __le16 stnid; /* sta index for this BA */ -+ u8 no_ba; /* No BA received */ -+ u8 ba_expected; /* Total expected pkt to be BA'd */ -+ u8 type; /* SU:0 or MU:1 */ -+ u8 pad[3]; /* Unused */ -+} __packed; -+ -+struct acnt_bf_mimo_ctrl_s {/* Accounting Record w/ BF MIMO Control Field Data*/ -+ __le16 code; /* Unique code for each type */ -+ u8 len; /* Length in DWORDS, including header */ -+ u8 type; /* SU:0, MU:1 */ -+ __le32 tsf; /* Timestamp for Entry (when len>1) */ -+ u8 rec_mac[6]; /* Received Packet Source MAC Address */ -+ __le16 pad; /* Padding */ -+ __le32 mimo_ctrl; /* BF MIMO Control Field Data */ -+ __le64 comp_bf_rep; /* First 8 bytes of Compressed BF Report */ -+} __packed; -+ -+static inline void pcie_tx_add_dma_header(struct mwl_priv *priv, -+ struct sk_buff *skb, -+ int head_pad, -+ int tail_pad) -+{ -+ struct ieee80211_hdr *wh; -+ int dma_hdrlen; -+ int hdrlen; -+ int reqd_hdrlen; -+ int needed_room; -+ struct pcie_dma_data *dma_data; -+ -+ dma_hdrlen = (priv->chip_type == MWL8997) ? -+ sizeof(struct pcie_pfu_dma_data) : -+ sizeof(struct pcie_dma_data); -+ -+ /* Add a firmware DMA header; the firmware requires that we -+ * present a 2-byte payload length followed by a 4-address -+ * header (without QoS field), followed (optionally) by any -+ * WEP/ExtIV header (but only filled in for CCMP). -+ */ -+ wh = (struct ieee80211_hdr *)skb->data; -+ -+ hdrlen = ieee80211_hdrlen(wh->frame_control); -+ -+ reqd_hdrlen = dma_hdrlen + head_pad; -+ -+ if (hdrlen != reqd_hdrlen) { -+ needed_room = reqd_hdrlen - hdrlen; -+ if (skb_headroom(skb) < needed_room) { -+ wiphy_debug(priv->hw->wiphy, "headroom is short: %d %d", -+ skb_headroom(skb), needed_room); -+ skb_cow(skb, needed_room); -+ } -+ skb_push(skb, needed_room); -+ } -+ -+ if (ieee80211_is_data_qos(wh->frame_control)) -+ hdrlen -= IEEE80211_QOS_CTL_LEN; -+ -+ if (priv->chip_type == MWL8997) -+ dma_data = &((struct pcie_pfu_dma_data *)skb->data)->dma_data; -+ else -+ dma_data = (struct pcie_dma_data *)skb->data; -+ -+ if (wh != &dma_data->wh) -+ memmove(&dma_data->wh, wh, hdrlen); -+ -+ if (hdrlen != sizeof(dma_data->wh)) -+ memset(((void *)&dma_data->wh) + hdrlen, 0, -+ sizeof(dma_data->wh) - hdrlen); -+ -+ /* Firmware length is the length of the fully formed "802.11 -+ * payload". That is, everything except for the 802.11 header. -+ * This includes all crypto material including the MIC. -+ */ -+ dma_data->fwlen = -+ cpu_to_le16(skb->len - dma_hdrlen + tail_pad); -+} -+ -+static inline void pcie_tx_encapsulate_frame(struct mwl_priv *priv, -+ struct sk_buff *skb, -+ struct ieee80211_key_conf *k_conf, -+ bool *ccmp) -+{ -+ int head_pad = 0; -+ int data_pad = 0; -+ -+ /* Make sure the packet header is in the DMA header format (4-address -+ * without QoS), and add head & tail padding when HW crypto is enabled. -+ * -+ * We have the following trailer padding requirements: -+ * - WEP: 4 trailer bytes (ICV) -+ * - TKIP: 12 trailer bytes (8 MIC + 4 ICV) -+ * - CCMP: 8 trailer bytes (MIC) -+ */ -+ -+ if (k_conf) { -+ head_pad = k_conf->iv_len; -+ -+ switch (k_conf->cipher) { -+ case WLAN_CIPHER_SUITE_WEP40: -+ case WLAN_CIPHER_SUITE_WEP104: -+ data_pad = 4; -+ break; -+ case WLAN_CIPHER_SUITE_TKIP: -+ data_pad = 12; -+ break; -+ case WLAN_CIPHER_SUITE_CCMP: -+ data_pad = 8; -+ if (ccmp) -+ *ccmp = true; -+ break; -+ } -+ } -+ -+ pcie_tx_add_dma_header(priv, skb, head_pad, data_pad); -+} -+ -+static inline void pcie_tx_prepare_info(struct mwl_priv *priv, u32 rate, -+ struct ieee80211_tx_info *info) -+{ -+ u32 format, bandwidth, short_gi, rate_id; -+ -+ ieee80211_tx_info_clear_status(info); -+ -+ info->status.rates[0].idx = -1; -+ info->status.rates[0].count = 0; -+ info->status.rates[0].flags = 0; -+ info->flags &= ~IEEE80211_TX_CTL_AMPDU; -+ info->flags |= IEEE80211_TX_STAT_ACK; -+ -+ if (rate) { -+ /* Prepare rate information */ -+ format = rate & MWL_TX_RATE_FORMAT_MASK; -+ bandwidth = -+ (rate & MWL_TX_RATE_BANDWIDTH_MASK) >> -+ MWL_TX_RATE_BANDWIDTH_SHIFT; -+ short_gi = (rate & MWL_TX_RATE_SHORTGI_MASK) >> -+ MWL_TX_RATE_SHORTGI_SHIFT; -+ rate_id = (rate & MWL_TX_RATE_RATEIDMCS_MASK) >> -+ MWL_TX_RATE_RATEIDMCS_SHIFT; -+ -+ info->status.rates[0].idx = rate_id; -+ if (format == TX_RATE_FORMAT_LEGACY) { -+ if (priv->hw->conf.chandef.chan->hw_value > -+ BAND_24_CHANNEL_NUM) { -+ info->status.rates[0].idx -= 5; -+ } -+ } -+ if (format == TX_RATE_FORMAT_11N) -+ info->status.rates[0].flags |= -+ IEEE80211_TX_RC_MCS; -+ if (format == TX_RATE_FORMAT_11AC) -+ info->status.rates[0].flags |= -+ IEEE80211_TX_RC_VHT_MCS; -+ if (bandwidth == TX_RATE_BANDWIDTH_40) -+ info->status.rates[0].flags |= -+ IEEE80211_TX_RC_40_MHZ_WIDTH; -+ if (bandwidth == TX_RATE_BANDWIDTH_80) -+ info->status.rates[0].flags |= -+ IEEE80211_TX_RC_80_MHZ_WIDTH; -+ if (bandwidth == TX_RATE_BANDWIDTH_160) -+ info->status.rates[0].flags |= -+ IEEE80211_TX_RC_160_MHZ_WIDTH; -+ if (short_gi == TX_RATE_INFO_SHORT_GI) -+ info->status.rates[0].flags |= -+ IEEE80211_TX_RC_SHORT_GI; -+ info->status.rates[0].count = 1; -+ info->status.rates[1].idx = -1; -+ } -+} -+ -+static inline void pcie_tx_count_packet(struct ieee80211_sta *sta, u8 tid) -+{ -+ struct mwl_sta *sta_info; -+ struct mwl_tx_info *tx_stats; -+ -+ if (WARN_ON(tid >= SYSADPT_MAX_TID)) -+ return; -+ -+ sta_info = mwl_dev_get_sta(sta); -+ -+ tx_stats = &sta_info->tx_stats[tid]; -+ -+ if (tx_stats->start_time == 0) -+ tx_stats->start_time = jiffies; -+ -+ /* reset the packet count after each second elapses. If the number of -+ * packets ever exceeds the ampdu_min_traffic threshold, we will allow -+ * an ampdu stream to be started. -+ */ -+ if (jiffies - tx_stats->start_time > HZ) { -+ tx_stats->pkts = 0; -+ tx_stats->start_time = jiffies; -+ } else { -+ tx_stats->pkts++; -+ } -+} -+ -+static inline void pcie_rx_prepare_status(struct mwl_priv *priv, u16 format, -+ u16 nss, u16 bw, u16 gi, u16 rate, -+ struct ieee80211_rx_status *status) -+{ -+#ifdef RX_ENC_FLAG_STBC_SHIFT -+ switch (format) { -+ case RX_RATE_INFO_FORMAT_11N: -+ status->encoding = RX_ENC_HT; -+ status->bw = RATE_INFO_BW_20; -+ if (bw == RX_RATE_INFO_HT40) -+ status->bw = RATE_INFO_BW_40; -+ if (gi == RX_RATE_INFO_SHORT_INTERVAL) -+ status->enc_flags |= RX_ENC_FLAG_SHORT_GI; -+ break; -+ case RX_RATE_INFO_FORMAT_11AC: -+ status->encoding = RX_ENC_VHT; -+ status->bw = RATE_INFO_BW_20; -+ if (bw == RX_RATE_INFO_HT40) -+ status->bw = RATE_INFO_BW_40; -+ if (bw == RX_RATE_INFO_HT80) -+ status->bw = RATE_INFO_BW_80; -+ if (bw == RX_RATE_INFO_HT160) -+ status->bw = RATE_INFO_BW_160; -+ if (gi == RX_RATE_INFO_SHORT_INTERVAL) -+ status->enc_flags |= RX_ENC_FLAG_SHORT_GI; -+ status->nss = (nss + 1); -+ break; -+ } -+#else -+ switch (format) { -+ case RX_RATE_INFO_FORMAT_11N: -+ status->flag |= RX_FLAG_HT; -+ if (bw == RX_RATE_INFO_HT40) -+ status->flag |= RX_FLAG_40MHZ; -+ if (gi == RX_RATE_INFO_SHORT_INTERVAL) -+ status->flag |= RX_FLAG_SHORT_GI; -+ break; -+ case RX_RATE_INFO_FORMAT_11AC: -+ status->flag |= RX_FLAG_VHT; -+ if (bw == RX_RATE_INFO_HT40) -+ status->flag |= RX_FLAG_40MHZ; -+ if (bw == RX_RATE_INFO_HT80) -+ status->vht_flag |= RX_VHT_FLAG_80MHZ; -+ if (bw == RX_RATE_INFO_HT160) -+ status->vht_flag |= RX_VHT_FLAG_160MHZ; -+ if (gi == RX_RATE_INFO_SHORT_INTERVAL) -+ status->flag |= RX_FLAG_SHORT_GI; -+ status->vht_nss = (nss + 1); -+ break; -+ } -+#endif -+ status->rate_idx = rate; -+ -+ if (priv->hw->conf.chandef.chan->hw_value > -+ BAND_24_CHANNEL_NUM) { -+ status->band = NL80211_BAND_5GHZ; -+#ifdef RX_ENC_FLAG_STBC_SHIFT -+ if ((!(status->encoding == RX_ENC_HT)) && -+ (!(status->encoding == RX_ENC_VHT))) { -+#else -+ if ((!(status->flag & RX_FLAG_HT)) && -+ (!(status->flag & RX_FLAG_VHT))) { -+#endif -+ status->rate_idx -= 5; -+ if (status->rate_idx >= BAND_50_RATE_NUM) -+ status->rate_idx = BAND_50_RATE_NUM - 1; -+ } -+ } else { -+ status->band = NL80211_BAND_2GHZ; -+#ifdef RX_ENC_FLAG_STBC_SHIFT -+ if ((!(status->encoding == RX_ENC_HT)) && -+ (!(status->encoding == RX_ENC_VHT))) { -+#else -+ if ((!(status->flag & RX_FLAG_HT)) && -+ (!(status->flag & RX_FLAG_VHT))) { -+#endif -+ if (status->rate_idx >= BAND_24_RATE_NUM) -+ status->rate_idx = BAND_24_RATE_NUM - 1; -+ } -+ } -+} -+ -+static inline void pcie_rx_remove_dma_header(struct sk_buff *skb, __le16 qos) -+{ -+ struct pcie_dma_data *dma_data; -+ int hdrlen; -+ -+ dma_data = (struct pcie_dma_data *)skb->data; -+ hdrlen = ieee80211_hdrlen(dma_data->wh.frame_control); -+ -+ if (hdrlen != sizeof(dma_data->wh)) { -+ if (ieee80211_is_data_qos(dma_data->wh.frame_control)) { -+ memmove(dma_data->data - hdrlen, -+ &dma_data->wh, hdrlen - 2); -+ *((__le16 *)(dma_data->data - 2)) = qos; -+ } else { -+ memmove(dma_data->data - hdrlen, &dma_data->wh, hdrlen); -+ } -+ } -+ -+ if (hdrlen != sizeof(*dma_data)) -+ skb_pull(skb, sizeof(*dma_data) - hdrlen); -+} -+ -+static inline void pcie_mask_int(struct pcie_priv *pcie_priv, -+ u32 mask_bit, bool set) -+{ -+ unsigned long flags; -+ void __iomem *int_status_mask; -+ u32 status; -+ -+ spin_lock_irqsave(&pcie_priv->int_mask_lock, flags); -+ int_status_mask = pcie_priv->iobase1 + -+ MACREG_REG_A2H_INTERRUPT_STATUS_MASK; -+ status = readl(int_status_mask); -+ if (set) -+ writel((status | mask_bit), int_status_mask); -+ else -+ writel((status & ~mask_bit), int_status_mask); -+ spin_unlock_irqrestore(&pcie_priv->int_mask_lock, flags); -+} -+ -+#endif /* _DEV_H_ */ -diff --git a/drivers/net/wireless/marvell/mwlwifi/hif/pcie/fwdl.c b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/fwdl.c -new file mode 100644 -index 000000000000..939ed54133c7 ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/fwdl.c -@@ -0,0 +1,274 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file implements firmware download related -+ * functions. -+ */ -+ -+#include -+ -+#include "sysadpt.h" -+#include "core.h" -+#include "hif/fwcmd.h" -+#include "hif/pcie/dev.h" -+#include "hif/pcie/sc4_ddr.h" -+#include "hif/pcie/fwdl.h" -+ -+#define FW_DOWNLOAD_BLOCK_SIZE 256 -+#define FW_CHECK_MSECS 3 -+ -+#define FW_MAX_NUM_CHECKS 0xffff -+ -+static void pcie_trigger_pcicmd_bootcode(struct pcie_priv *pcie_priv) -+{ -+ writel(pcie_priv->mwl_priv->pphys_cmd_buf, -+ pcie_priv->iobase1 + MACREG_REG_GEN_PTR); -+ writel(0x00, pcie_priv->iobase1 + MACREG_REG_INT_CODE); -+ writel(MACREG_H2ARIC_BIT_DOOR_BELL, -+ pcie_priv->iobase1 + MACREG_REG_H2A_INTERRUPT_EVENTS); -+} -+ -+static bool pcie_download_ddr_init(struct mwl_priv *priv) -+{ -+ struct pcie_priv *pcie_priv = (struct pcie_priv *)priv->hif.priv; -+ u32 size_ddr_init = sizeof(sc4_ddr_init); -+ u8 *p = (u8 *)&sc4_ddr_init[0]; -+ u32 curr_iteration = 0; -+ u32 size_ddr_init_downloaded = 0; -+ u32 int_code = 0; -+ u32 len = 0; -+ -+ /* download ddr init code */ -+ wiphy_debug(priv->hw->wiphy, "ddr init: download start\n"); -+ -+ while (size_ddr_init_downloaded < size_ddr_init) { -+ len = readl(pcie_priv->iobase1 + 0xc40); -+ -+ if (!len) -+ break; -+ -+ /* this copies the next chunk of fw binary to be delivered */ -+ memcpy((char *)&priv->pcmd_buf[0], p, len); -+ /* this is arbitrary per your platform; we use 0xffff */ -+ curr_iteration = (FW_MAX_NUM_CHECKS * 500); -+ /* this function writes pdata to c10, then write 2 to c18 */ -+ pcie_trigger_pcicmd_bootcode(pcie_priv); -+ -+ /* NOTE: the following back to back checks on C1C is time -+ * sensitive, hence may need to be tweaked dependent on host -+ * processor. Time for SC2 to go from the write of event 2 to -+ * C1C == 2 is ~1300 nSec. Hence the checkings on host has to -+ * consider how efficient your code can be to meet this timing, -+ * or you can alternatively tweak this routines to fit your -+ * platform -+ */ -+ do { -+ int_code = readl(pcie_priv->iobase1 + 0xc1c); -+ if (int_code != 0) -+ break; -+ cond_resched(); -+ curr_iteration--; -+ } while (curr_iteration); -+ -+ do { -+ int_code = readl(pcie_priv->iobase1 + 0xc1c); -+ if ((int_code & MACREG_H2ARIC_BIT_DOOR_BELL) != -+ MACREG_H2ARIC_BIT_DOOR_BELL) -+ break; -+ cond_resched(); -+ curr_iteration--; -+ } while (curr_iteration); -+ -+ if (curr_iteration == 0) { -+ /* This limited loop check allows you to exit gracefully -+ * without locking up your entire system just because fw -+ * download failed -+ */ -+ wiphy_err(priv->hw->wiphy, -+ "Exhausted curr_iteration during download\n"); -+ return false; -+ } -+ -+ p += len; -+ size_ddr_init_downloaded += len; -+ } -+ -+ wiphy_debug(priv->hw->wiphy, "ddr init: download complete\n"); -+ -+ return true; -+} -+ -+void pcie_reset(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ u32 regval; -+ -+ regval = readl(pcie_priv->iobase1 + MACREG_REG_INT_CODE); -+ if (regval == 0xffffffff) { -+ wiphy_err(priv->hw->wiphy, "adapter does not exist\n"); -+ return; -+ } -+ -+ writel(ISR_RESET, pcie_priv->iobase1 + MACREG_REG_H2A_INTERRUPT_EVENTS); -+} -+ -+int pcie_download_firmware(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ const struct firmware *fw = priv->fw_ucode; -+ u32 curr_iteration = 0; -+ u32 size_fw_downloaded = 0; -+ u32 int_code = 0; -+ u32 len = 0; -+ u32 fwreadysignature = HOSTCMD_SOFTAP_FWRDY_SIGNATURE; -+ -+ pcie_reset(hw); -+ -+ /* FW before jumping to boot rom, it will enable PCIe transaction retry, -+ * wait for boot code to stop it. -+ */ -+ usleep_range(FW_CHECK_MSECS * 1000, FW_CHECK_MSECS * 2000); -+ -+ if (priv->chip_type == MWL8964) { -+ writel(MACREG_A2HRIC_BIT_MASK_NDP, -+ pcie_priv->iobase1 + MACREG_REG_A2H_INTERRUPT_CLEAR_SEL); -+ } else { -+ writel(MACREG_A2HRIC_BIT_MASK, -+ pcie_priv->iobase1 + MACREG_REG_A2H_INTERRUPT_CLEAR_SEL); -+ } -+ writel(0x00, pcie_priv->iobase1 + MACREG_REG_A2H_INTERRUPT_CAUSE); -+ writel(0x00, pcie_priv->iobase1 + MACREG_REG_A2H_INTERRUPT_MASK); -+ if (priv->chip_type == MWL8964) { -+ writel(MACREG_A2HRIC_BIT_MASK_NDP, -+ pcie_priv->iobase1 + -+ MACREG_REG_A2H_INTERRUPT_STATUS_MASK); -+ } else { -+ writel(MACREG_A2HRIC_BIT_MASK, -+ pcie_priv->iobase1 + -+ MACREG_REG_A2H_INTERRUPT_STATUS_MASK); -+ } -+ -+ /* this routine interacts with SC2 bootrom to download firmware binary -+ * to the device. After DMA'd to SC2, the firmware could be deflated to -+ * reside on its respective blocks such as ITCM, DTCM, SQRAM, -+ * (or even DDR, AFTER DDR is init'd before fw download -+ */ -+ wiphy_debug(hw->wiphy, "fw download start\n"); -+ -+ if (priv->chip_type != MWL8997) -+ /* Disable PFU before FWDL */ -+ writel(0x100, pcie_priv->iobase1 + 0xE0E4); -+ -+ /* make sure SCRATCH2 C40 is clear, in case we are too quick */ -+ while (readl(pcie_priv->iobase1 + 0xc40) == 0) -+ cond_resched(); -+ -+ if (priv->chip_type == MWL8964) { -+ if (!pcie_download_ddr_init(priv)) { -+ wiphy_err(hw->wiphy, -+ "ddr init: code download failed\n"); -+ goto err_download; -+ } -+ } -+ -+ while (size_fw_downloaded < fw->size) { -+ len = readl(pcie_priv->iobase1 + 0xc40); -+ -+ if (!len) -+ break; -+ -+ /* this copies the next chunk of fw binary to be delivered */ -+ memcpy((char *)&priv->pcmd_buf[0], -+ (fw->data + size_fw_downloaded), len); -+ -+ /* this function writes pdata to c10, then write 2 to c18 */ -+ pcie_trigger_pcicmd_bootcode(pcie_priv); -+ -+ /* this is arbitrary per your platform; we use 0xffff */ -+ curr_iteration = FW_MAX_NUM_CHECKS; -+ -+ /* NOTE: the following back to back checks on C1C is time -+ * sensitive, hence may need to be tweaked dependent on host -+ * processor. Time for SC2 to go from the write of event 2 to -+ * C1C == 2 is ~1300 nSec. Hence the checkings on host has to -+ * consider how efficient your code can be to meet this timing, -+ * or you can alternatively tweak this routines to fit your -+ * platform -+ */ -+ do { -+ int_code = readl(pcie_priv->iobase1 + 0xc1c); -+ if ((int_code & MACREG_H2ARIC_BIT_DOOR_BELL) != -+ MACREG_H2ARIC_BIT_DOOR_BELL) -+ break; -+ cond_resched(); -+ curr_iteration--; -+ } while (curr_iteration); -+ -+ if (curr_iteration == 0) { -+ /* This limited loop check allows you to exit gracefully -+ * without locking up your entire system just because fw -+ * download failed -+ */ -+ wiphy_err(hw->wiphy, -+ "Exhausted curr_iteration for fw download\n"); -+ goto err_download; -+ } -+ -+ size_fw_downloaded += len; -+ } -+ -+ wiphy_debug(hw->wiphy, -+ "FwSize = %d downloaded Size = %d curr_iteration %d\n", -+ (int)fw->size, size_fw_downloaded, curr_iteration); -+ -+ /* Now firware is downloaded successfully, so this part is to check -+ * whether fw can properly execute to an extent that write back -+ * signature to indicate its readiness to the host. NOTE: if your -+ * downloaded fw crashes, this signature checking will fail. This -+ * part is similar as SC1 -+ */ -+ *((u32 *)&priv->pcmd_buf[1]) = 0; -+ pcie_trigger_pcicmd_bootcode(pcie_priv); -+ curr_iteration = FW_MAX_NUM_CHECKS; -+ do { -+ curr_iteration--; -+ writel(HOSTCMD_SOFTAP_MODE, -+ pcie_priv->iobase1 + MACREG_REG_GEN_PTR); -+ usleep_range(FW_CHECK_MSECS * 1000, FW_CHECK_MSECS * 2000); -+ int_code = readl(pcie_priv->iobase1 + MACREG_REG_INT_CODE); -+ if (!(curr_iteration % 0xff) && (int_code != 0)) -+ wiphy_err(hw->wiphy, "%x;", int_code); -+ } while ((curr_iteration) && -+ (int_code != fwreadysignature)); -+ -+ if (curr_iteration == 0) { -+ wiphy_err(hw->wiphy, -+ "Exhausted curr_iteration for fw signature\n"); -+ goto err_download; -+ } -+ -+ wiphy_debug(hw->wiphy, "fw download complete\n"); -+ writel(0x00, pcie_priv->iobase1 + MACREG_REG_INT_CODE); -+ -+ return 0; -+ -+err_download: -+ -+ pcie_reset(hw); -+ -+ return -EIO; -+} -diff --git a/drivers/net/wireless/marvell/mwlwifi/hif/pcie/fwdl.h b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/fwdl.h -new file mode 100644 -index 000000000000..36a3311aa678 ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/fwdl.h -@@ -0,0 +1,24 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file defines firmware download related functions. */ -+ -+#ifndef _FWDL_H_ -+#define _FWDL_H_ -+ -+void pcie_reset(struct ieee80211_hw *hw); -+int pcie_download_firmware(struct ieee80211_hw *hw); -+ -+#endif /* _FWDL_H_ */ -diff --git a/drivers/net/wireless/marvell/mwlwifi/hif/pcie/pcie.c b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/pcie.c -new file mode 100644 -index 000000000000..da55913c0570 ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/pcie.c -@@ -0,0 +1,1645 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file implements functions needed for PCIe module. */ -+ -+#include -+#include -+ -+#include "sysadpt.h" -+#include "core.h" -+#include "utils.h" -+#include "vendor_cmd.h" -+#include "hif/fwcmd.h" -+#include "hif/pcie/dev.h" -+#include "hif/pcie/fwdl.h" -+#include "hif/pcie/tx.h" -+#include "hif/pcie/rx.h" -+#include "hif/pcie/tx_ndp.h" -+#include "hif/pcie/rx_ndp.h" -+ -+#define PCIE_DRV_DESC "Marvell Mac80211 Wireless PCIE Network Driver" -+#define PCIE_DEV_NAME "Marvell 802.11ac PCIE Adapter" -+ -+#define MAX_WAIT_FW_COMPLETE_ITERATIONS 2000 -+#define CHECK_BA_TRAFFIC_TIME 300 /* msec */ -+#define CHECK_TX_DONE_TIME 50 /* msec */ -+ -+static struct pci_device_id pcie_id_tbl[] = { -+ { PCI_VDEVICE(MARVELL, 0x2a55), .driver_data = MWL8864, }, -+ { PCI_VDEVICE(MARVELL, 0x2b38), .driver_data = MWL8897, }, -+ { PCI_VDEVICE(MARVELL, 0x2b40), .driver_data = MWL8964, }, -+ { PCI_VDEVICE(MARVELL_EXT, 0x2b42), .driver_data = MWL8997, }, -+ { }, -+}; -+ -+static struct mwl_chip_info pcie_chip_tbl[] = { -+ [MWL8864] = { -+ .part_name = "88W8864", -+ .fw_image = "mwlwifi/88W8864.bin", -+ .cal_file = NULL, -+ .txpwrlmt_file = NULL, -+ .antenna_tx = ANTENNA_TX_4_AUTO, -+ .antenna_rx = ANTENNA_RX_4_AUTO, -+ }, -+ [MWL8897] = { -+ .part_name = "88W8897", -+ .fw_image = "mwlwifi/88W8897.bin", -+ .cal_file = NULL, -+ .txpwrlmt_file = NULL, -+ .antenna_tx = ANTENNA_TX_2, -+ .antenna_rx = ANTENNA_RX_2, -+ }, -+ [MWL8964] = { -+ .part_name = "88W8964", -+ .fw_image = "mwlwifi/88W8964.bin", -+ .cal_file = NULL, -+ .txpwrlmt_file = NULL, -+ .antenna_tx = ANTENNA_TX_4_AUTO, -+ .antenna_rx = ANTENNA_RX_4_AUTO, -+ }, -+ [MWL8997] = { -+ .part_name = "88W8997", -+ .fw_image = "mwlwifi/88W8997.bin", -+ .cal_file = "mwlwifi/WlanCalData_ext.conf", -+ .txpwrlmt_file = "mwlwifi/txpwrlmt_cfg.conf", -+ .antenna_tx = ANTENNA_TX_2, -+ .antenna_rx = ANTENNA_RX_2, -+ }, -+}; -+ -+static int pcie_alloc_resource(struct pcie_priv *pcie_priv) -+{ -+ struct pci_dev *pdev = pcie_priv->pdev; -+ struct device *dev = &pdev->dev; -+ void __iomem *addr; -+ -+ pcie_priv->next_bar_num = 1; /* 32-bit */ -+ if (pci_resource_flags(pdev, 0) & 0x04) -+ pcie_priv->next_bar_num = 2; /* 64-bit */ -+ -+ addr = devm_ioremap_resource(dev, &pdev->resource[0]); -+ if (IS_ERR(addr)) { -+ pr_err("%s: cannot reserve PCI memory region 0\n", -+ PCIE_DRV_NAME); -+ goto err; -+ } -+ pcie_priv->iobase0 = addr; -+ pr_debug("iobase0 = %p\n", pcie_priv->iobase0); -+ -+ addr = devm_ioremap_resource(dev, -+ &pdev->resource[pcie_priv->next_bar_num]); -+ if (IS_ERR(addr)) { -+ pr_err("%s: cannot reserve PCI memory region 1\n", -+ PCIE_DRV_NAME); -+ goto err; -+ } -+ pcie_priv->iobase1 = addr; -+ pr_debug("iobase1 = %p\n", pcie_priv->iobase1); -+ -+ return 0; -+ -+err: -+ pr_err("pci alloc fail\n"); -+ -+ return -EIO; -+} -+ -+static u32 pcie_read_mac_reg(struct pcie_priv *pcie_priv, u32 offset) -+{ -+ struct mwl_priv *priv = pcie_priv->mwl_priv; -+ -+ if (priv->chip_type == MWL8964) { -+ u32 *addr_val = kmalloc(64 * sizeof(u32), GFP_ATOMIC); -+ u32 val; -+ -+ if (addr_val) { -+ mwl_fwcmd_get_addr_value(priv->hw, -+ 0x8000a000 + offset, 4, -+ addr_val, 0); -+ val = addr_val[0]; -+ kfree(addr_val); -+ return val; -+ } -+ return 0; -+ } else -+ return le32_to_cpu(*(__le32 *) -+ (MAC_REG_ADDR_PCI(offset))); -+} -+ -+static bool pcie_chk_adapter(struct pcie_priv *pcie_priv) -+{ -+ struct mwl_priv *priv = pcie_priv->mwl_priv; -+ u32 regval; -+ -+ regval = readl(pcie_priv->iobase1 + MACREG_REG_INT_CODE); -+ -+ if (regval == 0xffffffff) { -+ wiphy_err(priv->hw->wiphy, "adapter does not exist\n"); -+ return false; -+ } -+ -+ if (priv->cmd_timeout) -+ wiphy_debug(priv->hw->wiphy, "MACREG_REG_INT_CODE: 0x%04x\n", -+ regval); -+ -+ return true; -+} -+ -+static void pcie_send_cmd(struct pcie_priv *pcie_priv) -+{ -+ writel(pcie_priv->mwl_priv->pphys_cmd_buf, -+ pcie_priv->iobase1 + MACREG_REG_GEN_PTR); -+ writel(MACREG_H2ARIC_BIT_DOOR_BELL, -+ pcie_priv->iobase1 + MACREG_REG_H2A_INTERRUPT_EVENTS); -+} -+ -+static int pcie_wait_complete(struct mwl_priv *priv, unsigned short cmd) -+{ -+ unsigned int curr_iteration = MAX_WAIT_FW_COMPLETE_ITERATIONS; -+ unsigned short int_code = 0; -+ -+ do { -+ int_code = le16_to_cpu(*((__le16 *)&priv->pcmd_buf[0])); -+ usleep_range(1000, 2000); -+ } while ((int_code != cmd) && (--curr_iteration) && !priv->rmmod); -+ -+ if (curr_iteration == 0) { -+ wiphy_err(priv->hw->wiphy, "cmd 0x%04x=%s timed out\n", -+ cmd, mwl_fwcmd_get_cmd_string(cmd)); -+ wiphy_err(priv->hw->wiphy, "return code: 0x%04x\n", int_code); -+ return -EIO; -+ } -+ -+ if (priv->chip_type != MWL8997) -+ usleep_range(3000, 5000); -+ -+ return 0; -+} -+ -+static int pcie_init(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ const struct hostcmd_get_hw_spec *get_hw_spec; -+ struct hostcmd_set_hw_spec set_hw_spec; -+ int rc, i; -+ -+ spin_lock_init(&pcie_priv->int_mask_lock); -+ tasklet_init(&pcie_priv->tx_task, -+ (void *)pcie_tx_skbs, (unsigned long)hw); -+ tasklet_disable(&pcie_priv->tx_task); -+ tasklet_init(&pcie_priv->tx_done_task, -+ (void *)pcie_tx_done, (unsigned long)hw); -+ tasklet_disable(&pcie_priv->tx_done_task); -+ spin_lock_init(&pcie_priv->tx_desc_lock); -+ tasklet_init(&pcie_priv->rx_task, -+ (void *)pcie_rx_recv, (unsigned long)hw); -+ tasklet_disable(&pcie_priv->rx_task); -+ tasklet_init(&pcie_priv->qe_task, -+ (void *)pcie_tx_flush_amsdu, (unsigned long)hw); -+ tasklet_disable(&pcie_priv->qe_task); -+ pcie_priv->txq_limit = PCIE_TX_QUEUE_LIMIT; -+ pcie_priv->txq_wake_threshold = PCIE_TX_WAKE_Q_THRESHOLD; -+ pcie_priv->is_tx_done_schedule = false; -+ pcie_priv->recv_limit = PCIE_RECEIVE_LIMIT; -+ pcie_priv->is_rx_schedule = false; -+ pcie_priv->is_qe_schedule = false; -+ pcie_priv->qe_trig_num = 0; -+ pcie_priv->qe_trig_time = jiffies; -+ -+ rc = pcie_tx_init(hw); -+ if (rc) { -+ wiphy_err(hw->wiphy, "%s: fail to initialize TX\n", -+ PCIE_DRV_NAME); -+ goto err_mwl_tx_init; -+ } -+ -+ rc = pcie_rx_init(hw); -+ if (rc) { -+ wiphy_err(hw->wiphy, "%s: fail to initialize RX\n", -+ PCIE_DRV_NAME); -+ goto err_mwl_rx_init; -+ } -+ -+ /* get and prepare HW specifications */ -+ get_hw_spec = mwl_fwcmd_get_hw_specs(hw); -+ if (!get_hw_spec) { -+ wiphy_err(hw->wiphy, "%s: fail to get HW specifications\n", -+ PCIE_DRV_NAME); -+ goto err_get_hw_specs; -+ } -+ ether_addr_copy(&priv->hw_data.mac_addr[0], -+ get_hw_spec->permanent_addr); -+ pcie_priv->desc_data[0].wcb_base = -+ le32_to_cpu(get_hw_spec->wcb_base0) & 0x0000ffff; -+ for (i = 1; i < SYSADPT_TOTAL_TX_QUEUES; i++) -+ pcie_priv->desc_data[i].wcb_base = -+ le32_to_cpu(get_hw_spec->wcb_base[i - 1]) & 0x0000ffff; -+ pcie_priv->desc_data[0].rx_desc_read = -+ le32_to_cpu(get_hw_spec->rxpd_rd_ptr) & 0x0000ffff; -+ pcie_priv->desc_data[0].rx_desc_write = -+ le32_to_cpu(get_hw_spec->rxpd_wr_ptr) & 0x0000ffff; -+ priv->hw_data.fw_release_num = le32_to_cpu(get_hw_spec->fw_release_num); -+ priv->hw_data.hw_version = get_hw_spec->version; -+ if (priv->chip_type != MWL8997) { -+ writel(pcie_priv->desc_data[0].pphys_tx_ring, -+ pcie_priv->iobase0 + pcie_priv->desc_data[0].wcb_base); -+ for (i = 1; i < SYSADPT_TOTAL_TX_QUEUES; i++) -+ writel(pcie_priv->desc_data[i].pphys_tx_ring, -+ pcie_priv->iobase0 + -+ pcie_priv->desc_data[i].wcb_base); -+ } -+ writel(pcie_priv->desc_data[0].pphys_rx_ring, -+ pcie_priv->iobase0 + pcie_priv->desc_data[0].rx_desc_read); -+ writel(pcie_priv->desc_data[0].pphys_rx_ring, -+ pcie_priv->iobase0 + pcie_priv->desc_data[0].rx_desc_write); -+ -+ /* prepare and set HW specifications */ -+ memset(&set_hw_spec, 0, sizeof(set_hw_spec)); -+ if (priv->chip_type == MWL8997) { -+ set_hw_spec.wcb_base[0] = -+ cpu_to_le32(pcie_priv->txbd_ring_pbase); -+ set_hw_spec.tx_wcb_num_per_queue = -+ cpu_to_le32(PCIE_MAX_TXRX_BD); -+ set_hw_spec.num_tx_queues = cpu_to_le32(1); -+ set_hw_spec.features |= HW_SET_PARMS_FEATURES_HOST_PROBE_RESP; -+ } else { -+ set_hw_spec.wcb_base[0] = -+ cpu_to_le32(pcie_priv->desc_data[0].pphys_tx_ring); -+ for (i = 1; i < SYSADPT_TOTAL_TX_QUEUES; i++) -+ set_hw_spec.wcb_base[i] = cpu_to_le32( -+ pcie_priv->desc_data[i].pphys_tx_ring); -+ set_hw_spec.tx_wcb_num_per_queue = -+ cpu_to_le32(PCIE_MAX_NUM_TX_DESC); -+ set_hw_spec.num_tx_queues = cpu_to_le32(PCIE_NUM_OF_DESC_DATA); -+ } -+ set_hw_spec.total_rx_wcb = cpu_to_le32(PCIE_MAX_NUM_RX_DESC); -+ set_hw_spec.rxpd_wr_ptr = -+ cpu_to_le32(pcie_priv->desc_data[0].pphys_rx_ring); -+ rc = mwl_fwcmd_set_hw_specs(hw, &set_hw_spec); -+ if (rc) { -+ wiphy_err(hw->wiphy, "%s: fail to set HW specifications\n", -+ PCIE_DRV_NAME); -+ goto err_set_hw_specs; -+ } -+ -+ return rc; -+ -+err_set_hw_specs: -+err_get_hw_specs: -+ -+ pcie_rx_deinit(hw); -+ -+err_mwl_rx_init: -+ -+ pcie_tx_deinit(hw); -+ -+err_mwl_tx_init: -+ -+ wiphy_err(hw->wiphy, "%s: init fail\n", PCIE_DRV_NAME); -+ -+ return rc; -+} -+ -+static void pcie_deinit(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ -+ pcie_rx_deinit(hw); -+ pcie_tx_deinit(hw); -+ tasklet_kill(&pcie_priv->qe_task); -+ tasklet_kill(&pcie_priv->rx_task); -+ tasklet_kill(&pcie_priv->tx_done_task); -+ tasklet_kill(&pcie_priv->tx_task); -+ pcie_reset(hw); -+} -+ -+static int pcie_get_info(struct ieee80211_hw *hw, char *buf, size_t size) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ char *p = buf; -+ int len = 0; -+ -+ len += scnprintf(p + len, size - len, "iobase0: %p\n", -+ pcie_priv->iobase0); -+ len += scnprintf(p + len, size - len, "iobase1: %p\n", -+ pcie_priv->iobase1); -+ len += scnprintf(p + len, size - len, -+ "tx limit: %d\n", pcie_priv->txq_limit); -+ len += scnprintf(p + len, size - len, -+ "rx limit: %d\n", pcie_priv->recv_limit); -+ len += scnprintf(p + len, size - len, -+ "qe trigger number: %d\n", pcie_priv->qe_trig_num); -+ return len; -+} -+ -+static void pcie_enable_data_tasks(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ -+ tasklet_enable(&pcie_priv->tx_task); -+ tasklet_enable(&pcie_priv->tx_done_task); -+ tasklet_enable(&pcie_priv->rx_task); -+ tasklet_enable(&pcie_priv->qe_task); -+} -+ -+static void pcie_disable_data_tasks(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ -+ tasklet_disable(&pcie_priv->tx_task); -+ tasklet_disable(&pcie_priv->tx_done_task); -+ tasklet_disable(&pcie_priv->rx_task); -+ tasklet_disable(&pcie_priv->qe_task); -+} -+ -+static int pcie_exec_cmd(struct ieee80211_hw *hw, unsigned short cmd) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ bool busy = false; -+ -+ might_sleep(); -+ -+ if (!pcie_chk_adapter(pcie_priv)) { -+ wiphy_err(priv->hw->wiphy, "adapter does not exist\n"); -+ priv->in_send_cmd = false; -+ return -EIO; -+ } -+ -+ if (!priv->in_send_cmd && !priv->rmmod) { -+ priv->in_send_cmd = true; -+ if (priv->dump_hostcmd) -+ wiphy_debug(priv->hw->wiphy, "send cmd 0x%04x=%s\n", -+ cmd, mwl_fwcmd_get_cmd_string(cmd)); -+ pcie_send_cmd(pcie_priv); -+ if (pcie_wait_complete(priv, 0x8000 | cmd)) { -+ wiphy_err(priv->hw->wiphy, "timeout: 0x%04x\n", cmd); -+ priv->in_send_cmd = false; -+ priv->cmd_timeout = true; -+ if (priv->heartbeat) -+ vendor_cmd_basic_event( -+ hw->wiphy, -+ MWL_VENDOR_EVENT_CMD_TIMEOUT); -+ return -EIO; -+ } -+ } else { -+ wiphy_warn(priv->hw->wiphy, -+ "previous command is running or module removed\n"); -+ busy = true; -+ } -+ -+ if (!busy) -+ priv->in_send_cmd = false; -+ -+ return 0; -+} -+ -+static int pcie_get_irq_num(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ -+ return pcie_priv->pdev->irq; -+} -+ -+static irqreturn_t pcie_isr(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ u32 int_status; -+ -+ int_status = readl(pcie_priv->iobase1 + MACREG_REG_A2H_INTERRUPT_CAUSE); -+ -+ if (int_status == 0x00000000) -+ return IRQ_NONE; -+ -+ if (int_status == 0xffffffff) { -+ wiphy_warn(hw->wiphy, "card unplugged?\n"); -+ } else { -+ writel(~int_status, -+ pcie_priv->iobase1 + MACREG_REG_A2H_INTERRUPT_CAUSE); -+ -+ if (int_status & MACREG_A2HRIC_BIT_TX_DONE) { -+ if (!pcie_priv->is_tx_done_schedule) { -+ pcie_mask_int(pcie_priv, -+ MACREG_A2HRIC_BIT_TX_DONE, false); -+ tasklet_schedule(&pcie_priv->tx_done_task); -+ pcie_priv->is_tx_done_schedule = true; -+ } -+ } -+ -+ if (int_status & MACREG_A2HRIC_BIT_RX_RDY) { -+ if (!pcie_priv->is_rx_schedule) { -+ pcie_mask_int(pcie_priv, -+ MACREG_A2HRIC_BIT_RX_RDY, false); -+ tasklet_schedule(&pcie_priv->rx_task); -+ pcie_priv->is_rx_schedule = true; -+ } -+ } -+ -+ if (int_status & MACREG_A2HRIC_BIT_RADAR_DETECT) { -+ wiphy_info(hw->wiphy, "radar detected by firmware\n"); -+ ieee80211_radar_detected(hw); -+ } -+ -+ if (int_status & MACREG_A2HRIC_BIT_QUE_EMPTY) { -+ if (!pcie_priv->is_qe_schedule) { -+ if (time_after(jiffies, -+ (pcie_priv->qe_trig_time + 1))) { -+ pcie_mask_int(pcie_priv, -+ MACREG_A2HRIC_BIT_QUE_EMPTY, -+ false); -+ tasklet_schedule(&pcie_priv->qe_task); -+ pcie_priv->qe_trig_num++; -+ pcie_priv->is_qe_schedule = true; -+ pcie_priv->qe_trig_time = jiffies; -+ } -+ } -+ } -+ -+ if (int_status & MACREG_A2HRIC_BIT_CHAN_SWITCH) -+ ieee80211_queue_work(hw, &priv->chnl_switch_handle); -+ -+ if (int_status & MACREG_A2HRIC_BA_WATCHDOG) -+ ieee80211_queue_work(hw, &priv->watchdog_ba_handle); -+ } -+ -+ return IRQ_HANDLED; -+} -+ -+static void pcie_irq_enable(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ -+ if (pcie_chk_adapter(pcie_priv)) { -+ writel(0x00, -+ pcie_priv->iobase1 + MACREG_REG_A2H_INTERRUPT_MASK); -+ writel(MACREG_A2HRIC_BIT_MASK, -+ pcie_priv->iobase1 + MACREG_REG_A2H_INTERRUPT_MASK); -+ } -+} -+ -+static void pcie_irq_disable(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ -+ if (pcie_chk_adapter(pcie_priv)) -+ writel(0x00, -+ pcie_priv->iobase1 + MACREG_REG_A2H_INTERRUPT_MASK); -+} -+ -+static void pcie_timer_routine(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ static int cnt; -+ struct mwl_ampdu_stream *stream; -+ struct mwl_sta *sta_info; -+ struct mwl_tx_info *tx_stats; -+ struct mwl_ampdu_stream *rm_stream = NULL; -+ u32 rm_pkts = 0; -+ bool ba_full = true; -+ int i; -+ -+ if ((++cnt * SYSADPT_TIMER_WAKEUP_TIME) < CHECK_BA_TRAFFIC_TIME) -+ return; -+ cnt = 0; -+ spin_lock_bh(&priv->stream_lock); -+ for (i = 0; i < priv->ampdu_num; i++) { -+ stream = &priv->ampdu[i]; -+ -+ if (stream->state == AMPDU_STREAM_ACTIVE) { -+ sta_info = mwl_dev_get_sta(stream->sta); -+ tx_stats = &sta_info->tx_stats[stream->tid]; -+ -+ if ((jiffies - tx_stats->start_time > HZ) && -+ (tx_stats->pkts < SYSADPT_AMPDU_PACKET_THRESHOLD)) { -+ if (rm_pkts) { -+ if (tx_stats->pkts < rm_pkts) { -+ rm_stream = stream; -+ rm_pkts = tx_stats->pkts; -+ } -+ } else { -+ rm_stream = stream; -+ rm_pkts = tx_stats->pkts; -+ } -+ } -+ -+ if (jiffies - tx_stats->start_time > HZ) { -+ tx_stats->pkts = 0; -+ tx_stats->start_time = jiffies; -+ } -+ } else -+ ba_full = false; -+ } -+ if (ba_full && rm_stream) { -+ ieee80211_stop_tx_ba_session(rm_stream->sta, -+ rm_stream->tid); -+ wiphy_debug(hw->wiphy, "Stop BA %pM\n", rm_stream->sta->addr); -+ } -+ spin_unlock_bh(&priv->stream_lock); -+} -+ -+static void pcie_tx_return_pkts(struct ieee80211_hw *hw) -+{ -+ pcie_tx_done((unsigned long)hw); -+} -+ -+static struct device_node *pcie_get_device_node(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct device_node *dev_node; -+ -+ dev_node = pci_bus_to_OF_node(pcie_priv->pdev->bus); -+ wiphy_info(priv->hw->wiphy, "device node: %s\n", dev_node->full_name); -+ -+ return dev_node; -+} -+ -+static void pcie_get_survey(struct ieee80211_hw *hw, -+ struct mwl_survey_info *survey_info) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ -+ survey_info->filled = SURVEY_INFO_TIME | -+ SURVEY_INFO_TIME_BUSY | -+ SURVEY_INFO_TIME_TX | -+ SURVEY_INFO_NOISE_DBM; -+ survey_info->time_period += pcie_read_mac_reg(pcie_priv, MCU_LAST_READ); -+ survey_info->time_busy += pcie_read_mac_reg(pcie_priv, MCU_CCA_CNT); -+ survey_info->time_tx += pcie_read_mac_reg(pcie_priv, MCU_TXPE_CNT); -+ survey_info->noise = priv->noise; -+} -+ -+static int pcie_reg_access(struct ieee80211_hw *hw, bool write) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ u8 set; -+ u32 *addr_val; -+ int ret = 0; -+ -+ set = write ? WL_SET : WL_GET; -+ -+ switch (priv->reg_type) { -+ case MWL_ACCESS_RF: -+ ret = mwl_fwcmd_reg_rf(hw, set, priv->reg_offset, -+ &priv->reg_value); -+ break; -+ case MWL_ACCESS_BBP: -+ ret = mwl_fwcmd_reg_bb(hw, set, priv->reg_offset, -+ &priv->reg_value); -+ break; -+ case MWL_ACCESS_CAU: -+ ret = mwl_fwcmd_reg_cau(hw, set, priv->reg_offset, -+ &priv->reg_value); -+ break; -+ case MWL_ACCESS_ADDR0: -+ if (set == WL_GET) -+ priv->reg_value = -+ readl(pcie_priv->iobase0 + priv->reg_offset); -+ else -+ writel(priv->reg_value, -+ pcie_priv->iobase0 + priv->reg_offset); -+ break; -+ case MWL_ACCESS_ADDR1: -+ if (set == WL_GET) -+ priv->reg_value = -+ readl(pcie_priv->iobase1 + priv->reg_offset); -+ else -+ writel(priv->reg_value, -+ pcie_priv->iobase1 + priv->reg_offset); -+ break; -+ case MWL_ACCESS_ADDR: -+ addr_val = kzalloc(64 * sizeof(u32), GFP_KERNEL); -+ if (addr_val) { -+ addr_val[0] = priv->reg_value; -+ ret = mwl_fwcmd_get_addr_value(hw, priv->reg_offset, -+ 4, addr_val, set); -+ if ((!ret) && (set == WL_GET)) -+ priv->reg_value = addr_val[0]; -+ kfree(addr_val); -+ } else { -+ ret = -ENOMEM; -+ } -+ break; -+ default: -+ ret = -EINVAL; -+ break; -+ } -+ -+ return ret; -+} -+ -+static struct mwl_hif_ops pcie_hif_ops = { -+ .driver_name = PCIE_DRV_NAME, -+ .driver_version = PCIE_DRV_VERSION, -+ .tx_head_room = PCIE_MIN_BYTES_HEADROOM, -+ .ampdu_num = PCIE_AMPDU_QUEUES, -+ .reset = pcie_reset, -+ .init = pcie_init, -+ .deinit = pcie_deinit, -+ .get_info = pcie_get_info, -+ .enable_data_tasks = pcie_enable_data_tasks, -+ .disable_data_tasks = pcie_disable_data_tasks, -+ .exec_cmd = pcie_exec_cmd, -+ .get_irq_num = pcie_get_irq_num, -+ .irq_handler = pcie_isr, -+ .irq_enable = pcie_irq_enable, -+ .irq_disable = pcie_irq_disable, -+ .download_firmware = pcie_download_firmware, -+ .timer_routine = pcie_timer_routine, -+ .tx_xmit = pcie_tx_xmit, -+ .tx_del_pkts_via_vif = pcie_tx_del_pkts_via_vif, -+ .tx_del_pkts_via_sta = pcie_tx_del_pkts_via_sta, -+ .tx_del_ampdu_pkts = pcie_tx_del_ampdu_pkts, -+ .tx_del_sta_amsdu_pkts = pcie_tx_del_sta_amsdu_pkts, -+ .tx_return_pkts = pcie_tx_return_pkts, -+ .get_device_node = pcie_get_device_node, -+ .get_survey = pcie_get_survey, -+ .reg_access = pcie_reg_access, -+}; -+ -+static int pcie_init_ndp(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ const struct hostcmd_get_hw_spec *get_hw_spec; -+ struct hostcmd_set_hw_spec set_hw_spec; -+ int rc; -+ -+ spin_lock_init(&pcie_priv->int_mask_lock); -+ tasklet_init(&pcie_priv->tx_task, -+ (void *)pcie_tx_skbs_ndp, (unsigned long)hw); -+ tasklet_disable(&pcie_priv->tx_task); -+ spin_lock_init(&pcie_priv->tx_desc_lock); -+ tasklet_init(&pcie_priv->rx_task, -+ (void *)pcie_rx_recv_ndp, (unsigned long)hw); -+ tasklet_disable(&pcie_priv->rx_task); -+ pcie_priv->txq_limit = TX_QUEUE_LIMIT; -+ pcie_priv->txq_wake_threshold = TX_WAKE_Q_THRESHOLD; -+ pcie_priv->is_tx_schedule = false; -+ pcie_priv->recv_limit = MAX_NUM_RX_DESC; -+ pcie_priv->is_rx_schedule = false; -+ -+ rc = pcie_tx_init_ndp(hw); -+ if (rc) { -+ wiphy_err(hw->wiphy, "%s: fail to initialize TX\n", -+ PCIE_DRV_NAME); -+ goto err_mwl_tx_init; -+ } -+ -+ rc = pcie_rx_init_ndp(hw); -+ if (rc) { -+ wiphy_err(hw->wiphy, "%s: fail to initialize RX\n", -+ PCIE_DRV_NAME); -+ goto err_mwl_rx_init; -+ } -+ -+ /* get and prepare HW specifications */ -+ get_hw_spec = mwl_fwcmd_get_hw_specs(hw); -+ if (!get_hw_spec) { -+ wiphy_err(hw->wiphy, "%s: fail to get HW specifications\n", -+ PCIE_DRV_NAME); -+ goto err_get_hw_specs; -+ } -+ ether_addr_copy(&priv->hw_data.mac_addr[0], -+ get_hw_spec->permanent_addr); -+ priv->hw_data.fw_release_num = le32_to_cpu(get_hw_spec->fw_release_num); -+ priv->hw_data.hw_version = get_hw_spec->version; -+ -+ /* prepare and set HW specifications */ -+ memset(&set_hw_spec, 0, sizeof(set_hw_spec)); -+ set_hw_spec.wcb_base[0] = -+ cpu_to_le32(pcie_priv->desc_data_ndp.pphys_tx_ring); -+ set_hw_spec.wcb_base[1] = -+ cpu_to_le32(pcie_priv->desc_data_ndp.pphys_tx_ring_done); -+ set_hw_spec.wcb_base[2] = -+ cpu_to_le32(pcie_priv->desc_data_ndp.pphys_rx_ring); -+ set_hw_spec.wcb_base[3] = -+ cpu_to_le32(pcie_priv->desc_data_ndp.pphys_rx_ring_done); -+ set_hw_spec.acnt_base_addr = -+ cpu_to_le32(pcie_priv->desc_data_ndp.pphys_acnt_ring); -+ set_hw_spec.acnt_buf_size = -+ cpu_to_le32(pcie_priv->desc_data_ndp.acnt_ring_size); -+ rc = mwl_fwcmd_set_hw_specs(hw, &set_hw_spec); -+ if (rc) { -+ wiphy_err(hw->wiphy, "%s: fail to set HW specifications\n", -+ PCIE_DRV_NAME); -+ goto err_set_hw_specs; -+ } -+ -+ return rc; -+ -+err_set_hw_specs: -+err_get_hw_specs: -+ -+ pcie_rx_deinit_ndp(hw); -+ -+err_mwl_rx_init: -+ -+ pcie_tx_deinit_ndp(hw); -+ -+err_mwl_tx_init: -+ -+ wiphy_err(hw->wiphy, "%s: init fail\n", PCIE_DRV_NAME); -+ -+ return rc; -+} -+ -+static void pcie_deinit_ndp(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ -+ pcie_rx_deinit_ndp(hw); -+ pcie_tx_deinit_ndp(hw); -+ tasklet_kill(&pcie_priv->rx_task); -+ tasklet_kill(&pcie_priv->tx_task); -+ pcie_reset(hw); -+} -+ -+static int pcie_get_info_ndp(struct ieee80211_hw *hw, char *buf, size_t size) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ char *p = buf; -+ int len = 0; -+ -+ len += scnprintf(p + len, size - len, "iobase0: %p\n", -+ pcie_priv->iobase0); -+ len += scnprintf(p + len, size - len, "iobase1: %p\n", -+ pcie_priv->iobase1); -+ len += scnprintf(p + len, size - len, -+ "tx limit: %d\n", pcie_priv->txq_limit); -+ len += scnprintf(p + len, size - len, -+ "rx limit: %d\n", pcie_priv->recv_limit); -+ return len; -+} -+ -+static int pcie_get_tx_status_ndp(struct ieee80211_hw *hw, char *buf, -+ size_t size) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ char *p = buf; -+ int len = 0; -+ -+ len += scnprintf(p + len, size - len, "tx_done_cnt: %d\n", -+ pcie_priv->tx_done_cnt); -+ len += scnprintf(p + len, size - len, "tx_desc_busy_cnt: %d\n", -+ pcie_priv->desc_data_ndp.tx_desc_busy_cnt); -+ len += scnprintf(p + len, size - len, "tx_sent_head: %d\n", -+ pcie_priv->desc_data_ndp.tx_sent_head); -+ len += scnprintf(p + len, size - len, "tx_sent_tail: %d\n", -+ pcie_priv->desc_data_ndp.tx_sent_tail); -+ len += scnprintf(p + len, size - len, "tx_done_head: %d\n", -+ readl(pcie_priv->iobase1 + MACREG_REG_TXDONEHEAD)); -+ len += scnprintf(p + len, size - len, "tx_done_tail: %d\n", -+ pcie_priv->desc_data_ndp.tx_done_tail); -+ len += scnprintf(p + len, size - len, "tx_vbuflist_idx: %d\n", -+ pcie_priv->desc_data_ndp.tx_vbuflist_idx); -+ return len; -+} -+ -+static int pcie_get_rx_status_ndp(struct ieee80211_hw *hw, char *buf, -+ size_t size) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ char *p = buf; -+ int len = 0; -+ -+ len += scnprintf(p + len, size - len, "rx_done_head: %d\n", -+ readl(pcie_priv->iobase1 + MACREG_REG_RXDONEHEAD)); -+ len += scnprintf(p + len, size - len, "rx_done_tail: %d\n", -+ readl(pcie_priv->iobase1 + MACREG_REG_RXDONETAIL)); -+ len += scnprintf(p + len, size - len, "rx_desc_head: %d\n", -+ readl(pcie_priv->iobase1 + MACREG_REG_RXDESCHEAD)); -+ len += scnprintf(p + len, size - len, "rx_skb_trace: %d\n", -+ skb_queue_len(&pcie_priv->rx_skb_trace)); -+ len += scnprintf(p + len, size - len, "rx_skb_unlink_err: %d\n", -+ pcie_priv->rx_skb_unlink_err); -+ len += scnprintf(p + len, size - len, "signature_err: %d\n", -+ pcie_priv->signature_err); -+ len += scnprintf(p + len, size - len, "recheck_rxringdone: %d\n", -+ pcie_priv->recheck_rxringdone); -+ len += scnprintf(p + len, size - len, "fast_data_cnt: %d\n", -+ pcie_priv->rx_cnts.fast_data_cnt); -+ len += scnprintf(p + len, size - len, "fast_bad_amsdu_cnt: %d\n", -+ pcie_priv->rx_cnts.fast_bad_amsdu_cnt); -+ len += scnprintf(p + len, size - len, "slow_noqueue_cnt: %d\n", -+ pcie_priv->rx_cnts.slow_noqueue_cnt); -+ len += scnprintf(p + len, size - len, "slow_norun_cnt: %d\n", -+ pcie_priv->rx_cnts.slow_norun_cnt); -+ len += scnprintf(p + len, size - len, "slow_mcast_cnt: %d\n", -+ pcie_priv->rx_cnts.slow_mcast_cnt); -+ len += scnprintf(p + len, size - len, "slow_bad_sta_cnt: %d\n", -+ pcie_priv->rx_cnts.slow_bad_sta_cnt); -+ len += scnprintf(p + len, size - len, "slow_bad_mic_cnt: %d\n", -+ pcie_priv->rx_cnts.slow_bad_mic_cnt); -+ len += scnprintf(p + len, size - len, "slow_bad_pn_cnt: %d\n", -+ pcie_priv->rx_cnts.slow_bad_pn_cnt); -+ len += scnprintf(p + len, size - len, "slow_mgmt_cnt: %d\n", -+ pcie_priv->rx_cnts.slow_mgmt_cnt); -+ len += scnprintf(p + len, size - len, "slow_promisc_cnt: %d\n", -+ pcie_priv->rx_cnts.slow_promisc_cnt); -+ len += scnprintf(p + len, size - len, "drop_cnt: %d\n", -+ pcie_priv->rx_cnts.drop_cnt); -+ len += scnprintf(p + len, size - len, "offch_promisc_cnt: %d\n", -+ pcie_priv->rx_cnts.offch_promisc_cnt); -+ len += scnprintf(p + len, size - len, "mu_pkt_cnt: %d\n", -+ pcie_priv->rx_cnts.mu_pkt_cnt); -+ return len; -+} -+ -+static void pcie_enable_data_tasks_ndp(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ -+ tasklet_enable(&pcie_priv->tx_task); -+ tasklet_enable(&pcie_priv->rx_task); -+} -+ -+static void pcie_disable_data_tasks_ndp(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ -+ tasklet_disable(&pcie_priv->tx_task); -+ tasklet_disable(&pcie_priv->rx_task); -+} -+ -+static irqreturn_t pcie_isr_ndp(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ u32 int_status; -+ -+ int_status = readl(pcie_priv->iobase1 + MACREG_REG_A2H_INTERRUPT_CAUSE); -+ -+ if (int_status == 0x00000000) -+ return IRQ_NONE; -+ -+ if (int_status == 0xffffffff) { -+ wiphy_warn(hw->wiphy, "card unplugged?\n"); -+ } else { -+ writel(~int_status, -+ pcie_priv->iobase1 + MACREG_REG_A2H_INTERRUPT_CAUSE); -+ -+ if (int_status & MACREG_A2HRIC_ACNT_HEAD_RDY) -+ ieee80211_queue_work(hw, &priv->account_handle); -+ -+ if (int_status & MACREG_A2HRIC_RX_DONE_HEAD_RDY) { -+ if (!pcie_priv->is_rx_schedule) { -+ pcie_mask_int(pcie_priv, -+ MACREG_A2HRIC_RX_DONE_HEAD_RDY, -+ false); -+ tasklet_schedule(&pcie_priv->rx_task); -+ pcie_priv->is_rx_schedule = true; -+ } -+ } -+ -+ if (int_status & MACREG_A2HRIC_NEWDP_DFS) { -+ wiphy_info(hw->wiphy, "radar detected by firmware\n"); -+ ieee80211_radar_detected(hw); -+ } -+ -+ if (int_status & MACREG_A2HRIC_NEWDP_CHANNEL_SWITCH) -+ ieee80211_queue_work(hw, &priv->chnl_switch_handle); -+ } -+ -+ return IRQ_HANDLED; -+} -+ -+static void pcie_irq_enable_ndp(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ -+ if (pcie_chk_adapter(pcie_priv)) { -+ writel(0x00, -+ pcie_priv->iobase1 + MACREG_REG_A2H_INTERRUPT_MASK); -+ writel(MACREG_A2HRIC_BIT_MASK_NDP, -+ pcie_priv->iobase1 + MACREG_REG_A2H_INTERRUPT_MASK); -+ } -+} -+ -+static void pcie_timer_routine_ndp(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ int num = SYSADPT_TX_WMM_QUEUES; -+ static int cnt; -+ -+ if (!pcie_priv->is_tx_schedule) { -+ while (num--) { -+ if (skb_queue_len(&pcie_priv->txq[num]) > 0) { -+ tasklet_schedule(&pcie_priv->tx_task); -+ pcie_priv->is_tx_schedule = true; -+ break; -+ } -+ } -+ } -+ -+ if ((++cnt * SYSADPT_TIMER_WAKEUP_TIME) >= CHECK_TX_DONE_TIME) { -+ pcie_tx_done_ndp(hw); -+ cnt = 0; -+ } -+} -+ -+static void pcie_tx_return_pkts_ndp(struct ieee80211_hw *hw) -+{ -+ pcie_tx_done_ndp(hw); -+} -+ -+static void pcie_set_sta_id(struct ieee80211_hw *hw, -+ struct ieee80211_sta *sta, -+ bool sta_mode, bool set) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct mwl_sta *sta_info; -+ u16 stnid; -+ -+ sta_info = mwl_dev_get_sta(sta); -+ stnid = sta_mode ? 0 : sta_info->stnid; -+ pcie_priv->sta_link[stnid] = set ? sta : NULL; -+} -+ -+static void pcie_tx_account(struct mwl_priv *priv, -+ struct mwl_sta *sta_info, -+ struct acnt_tx_s *acnt_tx) -+{ -+ u32 rate_info, tx_cnt; -+ u8 index, type, rate_ac, format, bw, gi, mcs, nss; -+ u16 ratemask; -+ u8 i, found; -+ struct mwl_tx_hist *tx_hist; -+ struct mwl_tx_hist_data *tx_hist_data; -+ -+ rate_info = le32_to_cpu(acnt_tx->tx_info.rate_info); -+ tx_cnt = le32_to_cpu(acnt_tx->tx_cnt); -+ index = acnt_tx->rate_tbl_index; -+ type = acnt_tx->type; -+ -+ if (!rate_info) -+ return; -+ sta_info->tx_rate_info = rate_info; -+ -+ tx_hist = &sta_info->tx_hist; -+ if (!tx_hist || (type >= SU_MU_TYPE_CNT)) -+ return; -+ -+ format = rate_info & MWL_TX_RATE_FORMAT_MASK; -+ bw = (rate_info & MWL_TX_RATE_BANDWIDTH_MASK) >> -+ MWL_TX_RATE_BANDWIDTH_SHIFT; -+ gi = (rate_info & MWL_TX_RATE_SHORTGI_MASK) >> -+ MWL_TX_RATE_SHORTGI_SHIFT; -+ mcs = (rate_info & MWL_TX_RATE_RATEIDMCS_MASK) >> -+ MWL_TX_RATE_RATEIDMCS_SHIFT; -+ -+ tx_hist->cur_rate_info[type] = rate_info; -+ -+ /* Rate table index is valid */ -+ if (index != 0xff) { -+ if (type == MU_MIMO) { -+ rate_ac = mcs & 0xf; -+ nss = mcs >> 4; -+ if (nss < (QS_NUM_SUPPORTED_11AC_NSS - 1)) { -+ tx_hist_data = -+ &tx_hist->mu_rate[nss][bw][gi][rate_ac]; -+ tx_hist_data->rateinfo = rate_info; -+ tx_hist_data->cnt++; -+ tx_hist->total_tx_cnt[type] += tx_cnt; -+ } -+ } else { -+ /* If legacy, skip legacy preamble bit 15 */ -+ if (format == TX_RATE_FORMAT_LEGACY) -+ ratemask = 0xfff; -+ else -+ ratemask = 0xffff; -+ tx_hist_data = &tx_hist->su_rate[0]; -+ if ((tx_hist_data[index].rateinfo & ratemask) == -+ (rate_info & ratemask)) { -+ tx_hist_data[index].cnt++; -+ tx_hist->total_tx_cnt[type] += tx_cnt; -+ } -+ } -+ } else { -+ if (type == MU_MIMO) { -+ rate_ac = mcs & 0xf; -+ nss = mcs >> 4; -+ if (nss < (QS_NUM_SUPPORTED_11AC_NSS - 1)) { -+ tx_hist_data = -+ &tx_hist->mu_rate[nss][bw][gi][rate_ac]; -+ tx_hist_data->rateinfo = rate_info; -+ tx_hist_data->cnt++; -+ tx_hist->total_tx_cnt[type] += tx_cnt; -+ } -+ } else { -+ /* If legacy, skip legacy preamble bit 15 */ -+ if (format == TX_RATE_FORMAT_LEGACY) -+ ratemask = 0xfff; -+ else -+ ratemask = 0xffff; -+ tx_hist_data = &tx_hist->custom_rate[0]; -+ /* Go through non rate table buffer to see if any has -+ * been used. If all used up, recycle by using index 0 -+ */ -+ for (i = 0; i < TX_RATE_HISTO_CUSTOM_CNT; i++) { -+ if (!tx_hist_data[i].rateinfo || -+ ((tx_hist_data[i].rateinfo & ratemask) == -+ (rate_info & ratemask))) { -+ found = 1; -+ break; -+ } -+ } -+ if (found) -+ index = i; -+ else -+ index = 0; /* reuse index 0 buffer */ -+ tx_hist_data[index].rateinfo = rate_info; -+ tx_hist_data[index].cnt++; -+ tx_hist->total_tx_cnt[type] += tx_cnt; -+ } -+ } -+} -+ -+static void pcie_rx_account(struct mwl_priv *priv, -+ struct mwl_sta *sta_info, -+ struct acnt_rx_s *acnt_rx) -+{ -+ u32 sig1, sig2, rate, param; -+ u16 format, nss, bw, gi, rate_mcs; -+ -+ sig1 = (le32_to_cpu(acnt_rx->rx_info.ht_sig1) >> -+ RXINFO_HT_SIG1_SHIFT) & RXINFO_HT_SIG1_MASK; -+ sig2 = (le32_to_cpu(acnt_rx->rx_info.ht_sig2_rate) >> -+ RXINFO_HT_SIG2_SHIFT) & RXINFO_HT_SIG2_MASK; -+ rate = (le32_to_cpu(acnt_rx->rx_info.ht_sig2_rate) >> -+ RXINFO_RATE_SHIFT) & RXINFO_RATE_MASK; -+ param = (le32_to_cpu(acnt_rx->rx_info.param) >> -+ RXINFO_PARAM_SHIFT) & RXINFO_PARAM_MASK; -+ -+ format = (param >> 3) & 0x7; -+ nss = 0; -+ bw = RX_RATE_INFO_HT20; -+ switch (format) { -+ case RX_RATE_INFO_FORMAT_11A: -+ rate_mcs = rate & 0xF; -+ if (rate_mcs == 10) -+ rate_mcs = 7; /* 12 Mbps */ -+ else -+ rate_mcs = utils_get_rate_id(rate_mcs); -+ gi = RX_RATE_INFO_SHORT_INTERVAL; -+ if ((rate_mcs == 5) || (rate_mcs == 7) || (rate_mcs == 9)) -+ return; -+ break; -+ case RX_RATE_INFO_FORMAT_11B: -+ rate_mcs = utils_get_rate_id(rate & 0xF); -+ gi = RX_RATE_INFO_LONG_INTERVAL; -+ if ((rate_mcs == 0) || (rate_mcs == 1)) -+ return; -+ break; -+ case RX_RATE_INFO_FORMAT_11N: -+ if ((sig1 & 0x3f) >= 16) -+ return; -+ bw = (sig1 >> 7) & 0x1; -+ gi = (sig2 >> 7) & 0x1; -+ rate_mcs = sig1 & 0x3F; -+ if (rate_mcs > 76) -+ return; -+ break; -+ case RX_RATE_INFO_FORMAT_11AC: -+ if (((sig2 >> 4) & 0xf) >= 10) -+ return; -+ nss = (sig1 >> 10) & 0x3; -+ if (!nss) -+ return; -+ bw = sig1 & 0x3; -+ gi = sig2 & 0x1; -+ rate_mcs = (sig2 >> 4) & 0xF; -+ if (rate_mcs > 9) -+ return; -+ break; -+ default: -+ return; -+ } -+ -+ sta_info->rx_format = format; -+ sta_info->rx_nss = nss; -+ sta_info->rx_bw = bw; -+ sta_info->rx_gi = gi; -+ sta_info->rx_rate_mcs = rate_mcs; -+ sta_info->rx_signal = ((le32_to_cpu(acnt_rx->rx_info.rssi_x) >> -+ RXINFO_RSSI_X_SHIFT) & RXINFO_RSSI_X_MASK); -+} -+ -+static void pcie_tx_per(struct mwl_priv *priv, struct mwl_sta *sta_info, -+ struct acnt_ra_s *acnt_ra) -+{ -+ u32 rate_info; -+ u8 index, per, type, rate_ac, per_index, format, bw, gi, mcs, nss; -+ u16 ratemask; -+ u8 i, found; -+ struct mwl_tx_hist *tx_hist; -+ struct mwl_tx_hist_data *tx_hist_data; -+ -+ rate_info = le32_to_cpu(acnt_ra->rate_info); -+ index = acnt_ra->rate_tbl_index; -+ per = acnt_ra->per; -+ type = acnt_ra->type; -+ -+ tx_hist = &sta_info->tx_hist; -+ -+ if (!tx_hist || !rate_info || (type >= SU_MU_TYPE_CNT)) -+ return; -+ -+ if ((type == SU_MIMO) && (index >= MAX_SUPPORTED_RATES) && -+ (index != 0xFF)) -+ return; -+ -+ if (per >= TX_HISTO_PER_THRES[3]) -+ per_index = 4; -+ else if (per >= TX_HISTO_PER_THRES[2]) -+ per_index = 3; -+ else if (per >= TX_HISTO_PER_THRES[1]) -+ per_index = 2; -+ else if (per >= TX_HISTO_PER_THRES[0]) -+ per_index = 1; -+ else -+ per_index = 0; -+ -+ format = rate_info & MWL_TX_RATE_FORMAT_MASK; -+ bw = (rate_info & MWL_TX_RATE_BANDWIDTH_MASK) >> -+ MWL_TX_RATE_BANDWIDTH_SHIFT; -+ gi = (rate_info & MWL_TX_RATE_SHORTGI_MASK) >> -+ MWL_TX_RATE_SHORTGI_SHIFT; -+ mcs = (rate_info & MWL_TX_RATE_RATEIDMCS_MASK) >> -+ MWL_TX_RATE_RATEIDMCS_SHIFT; -+ -+ /* Rate table index is valid */ -+ if (index != 0xff) { -+ if (type == MU_MIMO) { -+ rate_ac = mcs & 0xf; -+ nss = mcs >> 4; -+ if (nss < (QS_NUM_SUPPORTED_11AC_NSS - 1)) { -+ tx_hist_data = -+ &tx_hist->mu_rate[nss][bw][gi][rate_ac]; -+ tx_hist_data->rateinfo = rate_info; -+ tx_hist_data->per[per_index]++; -+ } -+ } else { -+ /* If legacy, skip legacy preamble bit 15 */ -+ if (format == TX_RATE_FORMAT_LEGACY) -+ ratemask = 0xfff; -+ else -+ ratemask = 0xffff; -+ tx_hist_data = &tx_hist->su_rate[0]; -+ if ((tx_hist_data[index].rateinfo & ratemask) == -+ (rate_info & ratemask)) -+ tx_hist_data[index].per[per_index]++; -+ } -+ } else { -+ if (type == MU_MIMO) { -+ rate_ac = mcs & 0xf; -+ nss = mcs >> 4; -+ if (nss < (QS_NUM_SUPPORTED_11AC_NSS - 1)) { -+ tx_hist_data = -+ &tx_hist->mu_rate[nss][bw][gi][rate_ac]; -+ tx_hist_data->rateinfo = rate_info; -+ tx_hist_data->per[per_index]++; -+ } -+ } else { -+ /* If legacy, skip legacy preamble bit 15 */ -+ if (format == TX_RATE_FORMAT_LEGACY) -+ ratemask = 0xfff; -+ else -+ ratemask = 0xffff; -+ tx_hist_data = &tx_hist->custom_rate[0]; -+ /* Go through non rate table buffer to see if any has -+ * been used. If all used up, recycle by using index 0 -+ */ -+ for (i = 0; i < TX_RATE_HISTO_CUSTOM_CNT; i++) { -+ if (!tx_hist_data[i].rateinfo || -+ ((tx_hist_data[i].rateinfo & ratemask) == -+ (rate_info & ratemask))) { -+ found = 1; -+ break; -+ } -+ } -+ if (found) -+ index = i; -+ else -+ index = 0; /* reuse index 0 buffer */ -+ tx_hist_data[index].rateinfo = rate_info; -+ tx_hist_data[index].per[per_index]++; -+ } -+ } -+} -+ -+static void pcie_ba_account(struct mwl_priv *priv, -+ struct mwl_sta *sta_info, -+ struct acnt_ba_s *acnt_ba) -+{ -+ struct mwl_tx_ba_hist *ba_hist = &sta_info->ba_hist; -+ -+ if (sta_info->stnid != le16_to_cpu(acnt_ba->stnid)) -+ return; -+ -+ if (ba_hist->enable && ba_hist->ba_stats && -+ (ba_hist->index < ACNT_BA_SIZE)) { -+ ba_hist->type = acnt_ba->type; -+ ba_hist->ba_stats[ba_hist->index].ba_hole = acnt_ba->ba_hole; -+ ba_hist->ba_stats[ba_hist->index].ba_expected = -+ acnt_ba->ba_expected; -+ ba_hist->ba_stats[ba_hist->index].no_ba = acnt_ba->no_ba; -+ ba_hist->index++; -+ if (ba_hist->index == ACNT_BA_SIZE) -+ wiphy_info(priv->hw->wiphy, -+ "Aid:%d BA histo collection done\n", -+ priv->ba_aid); -+ } -+} -+ -+static void pcie_bf_mimo_ctrl_decode(struct mwl_priv *priv, -+ struct acnt_bf_mimo_ctrl_s *bf_mimo_ctrl) -+{ -+ struct file *fp_data = NULL; -+ const char filename[] = "/tmp/BF_MIMO_Ctrl_Field_Output.txt"; -+ char str_buf[256]; -+ char *buf = &str_buf[0]; -+ mm_segment_t oldfs; -+ -+ oldfs = get_fs(); -+ set_fs(KERNEL_DS); -+ -+ buf += sprintf(buf, "\nMAC: %pM\n", bf_mimo_ctrl->rec_mac); -+ buf += sprintf(buf, "SU_0_MU_1: %d\n", bf_mimo_ctrl->type); -+ buf += sprintf(buf, "MIMO_Ctrl_Field: 0x%x\n", -+ le32_to_cpu(bf_mimo_ctrl->mimo_ctrl)); -+ buf += sprintf(buf, "Comp_BF_Report_First_8Bytes: 0x%llx\n", -+ le64_to_cpu(bf_mimo_ctrl->comp_bf_rep)); -+ -+ fp_data = filp_open(filename, O_RDWR | O_CREAT | O_TRUNC, 0); -+ -+ if (!IS_ERR(fp_data)) { -+ __kernel_write(fp_data, str_buf, strlen(str_buf), -+ &fp_data->f_pos); -+ filp_close(fp_data, current->files); -+ } else { -+ wiphy_err(priv->hw->wiphy, "Error opening %s! %x\n", -+ filename, (unsigned int)fp_data); -+ } -+ -+ set_fs(oldfs); -+} -+ -+static void pcie_process_account(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct pcie_desc_data_ndp *desc = &pcie_priv->desc_data_ndp; -+ u32 acnt_head, acnt_tail; -+ u32 read_size; -+ u8 *acnt_recds; -+ u8 *pstart, *pend; -+ struct acnt_s *acnt; -+ struct acnt_tx_s *acnt_tx; -+ struct acnt_rx_s *acnt_rx; -+ struct acnt_ra_s *acnt_ra; -+ struct acnt_ba_s *acnt_ba; -+ struct acnt_bf_mimo_ctrl_s *acnt_bf_mimo_ctrl; -+ struct pcie_dma_data *dma_data; -+ struct mwl_sta *sta_info; -+ u16 nf_a, nf_b, nf_c, nf_d; -+ u16 stnid; -+ u8 type; -+ -+ acnt_head = readl(pcie_priv->iobase1 + MACREG_REG_ACNTHEAD); -+ acnt_tail = readl(pcie_priv->iobase1 + MACREG_REG_ACNTTAIL); -+ -+ if (acnt_tail == acnt_head) -+ return; -+ -+ if (acnt_tail > acnt_head) { -+ read_size = desc->acnt_ring_size - acnt_tail + acnt_head; -+ if (read_size > desc->acnt_ring_size) { -+ wiphy_err(hw->wiphy, -+ "account size overflow (%d %d %d)\n", -+ acnt_head, acnt_tail, read_size); -+ goto process_next; -+ } -+ memset(desc->pacnt_buf, 0, desc->acnt_ring_size); -+ memcpy(desc->pacnt_buf, desc->pacnt_ring + acnt_tail, -+ desc->acnt_ring_size - acnt_tail); -+ memcpy(desc->pacnt_buf + desc->acnt_ring_size - acnt_tail, -+ desc->pacnt_ring, acnt_head); -+ acnt_recds = desc->pacnt_buf; -+ } else { -+ read_size = acnt_head - acnt_tail; -+ if (read_size > desc->acnt_ring_size) { -+ wiphy_err(hw->wiphy, -+ "account size overflow (%d %d %d)\n", -+ acnt_head, acnt_tail, read_size); -+ goto process_next; -+ } -+ acnt_recds = desc->pacnt_ring + acnt_tail; -+ } -+ -+ pstart = acnt_recds; -+ pend = pstart + read_size; -+ while (pstart < pend) { -+ acnt = (struct acnt_s *)pstart; -+ -+ switch (le16_to_cpu(acnt->code)) { -+ case ACNT_CODE_BUSY: -+ pcie_priv->acnt_busy++; -+ break; -+ case ACNT_CODE_WRAP: -+ pcie_priv->acnt_wrap++; -+ break; -+ case ACNT_CODE_DROP: -+ pcie_priv->acnt_drop++; -+ break; -+ case ACNT_CODE_TX_ENQUEUE: -+ acnt_tx = (struct acnt_tx_s *)pstart; -+ sta_info = utils_find_sta(priv, acnt_tx->hdr.wh.addr1); -+ if (sta_info) { -+ spin_lock_bh(&priv->sta_lock); -+ pcie_tx_account(priv, sta_info, acnt_tx); -+ spin_unlock_bh(&priv->sta_lock); -+ } -+ break; -+ case ACNT_CODE_RX_PPDU: -+ acnt_rx = (struct acnt_rx_s *)pstart; -+ nf_a = (le32_to_cpu(acnt_rx->rx_info.nf_a_b) >> -+ RXINFO_NF_A_SHIFT) & RXINFO_NF_A_MASK; -+ nf_b = (le32_to_cpu(acnt_rx->rx_info.nf_a_b) >> -+ RXINFO_NF_B_SHIFT) & RXINFO_NF_B_MASK; -+ nf_c = (le32_to_cpu(acnt_rx->rx_info.nf_c_d) >> -+ RXINFO_NF_C_SHIFT) & RXINFO_NF_C_MASK; -+ nf_d = (le32_to_cpu(acnt_rx->rx_info.nf_c_d) >> -+ RXINFO_NF_D_SHIFT) & RXINFO_NF_D_MASK; -+ if ((nf_a >= 2048) && (nf_b >= 2048) && -+ (nf_c >= 2048) && (nf_d >= 2048)) { -+ nf_a = ((4096 - nf_a) >> 4); -+ nf_b = ((4096 - nf_b) >> 4); -+ nf_c = ((4096 - nf_c) >> 4); -+ nf_d = ((4096 - nf_d) >> 4); -+ priv->noise = -+ -((nf_a + nf_b + nf_c + nf_d) / 4); -+ } -+ dma_data = (struct pcie_dma_data *) -+ &acnt_rx->rx_info.hdr[0]; -+ sta_info = utils_find_sta(priv, dma_data->wh.addr2); -+ if (sta_info) { -+ spin_lock_bh(&priv->sta_lock); -+ pcie_rx_account(priv, sta_info, acnt_rx); -+ spin_unlock_bh(&priv->sta_lock); -+ } -+ break; -+ case ACNT_CODE_RA_STATS: -+ acnt_ra = (struct acnt_ra_s *)pstart; -+ stnid = le16_to_cpu(acnt_ra->stn_id); -+ if ((stnid > 0) && (stnid <= priv->stnid_num)) { -+ type = acnt_ra->type; -+ if (type < 2) { -+ if (acnt_ra->tx_attempt_cnt >= 250) -+ priv->ra_tx_attempt[type][5]++; -+ else if (acnt_ra->tx_attempt_cnt >= 100) -+ priv->ra_tx_attempt[type][4]++; -+ else if (acnt_ra->tx_attempt_cnt >= 50) -+ priv->ra_tx_attempt[type][3]++; -+ else if (acnt_ra->tx_attempt_cnt >= 15) -+ priv->ra_tx_attempt[type][2]++; -+ else if (acnt_ra->tx_attempt_cnt >= 4) -+ priv->ra_tx_attempt[type][1]++; -+ else -+ priv->ra_tx_attempt[type][0]++; -+ } -+ sta_info = utils_find_sta_by_id(priv, stnid); -+ if (sta_info) { -+ spin_lock_bh(&priv->sta_lock); -+ pcie_tx_per(priv, sta_info, acnt_ra); -+ spin_unlock_bh(&priv->sta_lock); -+ } -+ } -+ break; -+ case ACNT_CODE_BA_STATS: -+ acnt_ba = (struct acnt_ba_s *)pstart; -+ if (priv->ba_aid) { -+ sta_info = utils_find_sta_by_aid(priv, -+ priv->ba_aid); -+ if (sta_info) { -+ spin_lock_bh(&priv->sta_lock); -+ pcie_ba_account(priv, sta_info, -+ acnt_ba); -+ spin_unlock_bh(&priv->sta_lock); -+ } -+ } -+ break; -+ case ACNT_CODE_BF_MIMO_CTRL: -+ acnt_bf_mimo_ctrl = -+ (struct acnt_bf_mimo_ctrl_s *)pstart; -+ pcie_bf_mimo_ctrl_decode(priv, acnt_bf_mimo_ctrl); -+ break; -+ default: -+ break; -+ } -+ -+ if (acnt->len) -+ pstart += acnt->len * 4; -+ else -+ goto process_next; -+ } -+process_next: -+ acnt_tail = acnt_head; -+ writel(acnt_tail, pcie_priv->iobase1 + MACREG_REG_ACNTTAIL); -+} -+ -+static int pcie_mcast_cts(struct ieee80211_hw *hw, bool enable) -+{ -+ return mwl_fwcmd_mcast_cts(hw, enable ? 1 : 0); -+} -+ -+static const struct mwl_hif_ops pcie_hif_ops_ndp = { -+ .driver_name = PCIE_DRV_NAME, -+ .driver_version = PCIE_DRV_VERSION, -+ .tx_head_room = PCIE_MIN_BYTES_HEADROOM, -+ .ampdu_num = AMPDU_QUEUES_NDP, -+ .reset = pcie_reset, -+ .init = pcie_init_ndp, -+ .deinit = pcie_deinit_ndp, -+ .get_info = pcie_get_info_ndp, -+ .get_tx_status = pcie_get_tx_status_ndp, -+ .get_rx_status = pcie_get_rx_status_ndp, -+ .enable_data_tasks = pcie_enable_data_tasks_ndp, -+ .disable_data_tasks = pcie_disable_data_tasks_ndp, -+ .exec_cmd = pcie_exec_cmd, -+ .get_irq_num = pcie_get_irq_num, -+ .irq_handler = pcie_isr_ndp, -+ .irq_enable = pcie_irq_enable_ndp, -+ .irq_disable = pcie_irq_disable, -+ .download_firmware = pcie_download_firmware, -+ .timer_routine = pcie_timer_routine_ndp, -+ .tx_xmit = pcie_tx_xmit_ndp, -+ .tx_return_pkts = pcie_tx_return_pkts_ndp, -+ .get_device_node = pcie_get_device_node, -+ .get_survey = pcie_get_survey, -+ .reg_access = pcie_reg_access, -+ .set_sta_id = pcie_set_sta_id, -+ .process_account = pcie_process_account, -+ .mcast_cts = pcie_mcast_cts, -+}; -+ -+static int pcie_probe(struct pci_dev *pdev, const struct pci_device_id *id) -+{ -+ static bool printed_version; -+ struct ieee80211_hw *hw; -+ struct mwl_priv *priv; -+ struct pcie_priv *pcie_priv; -+ const struct mwl_hif_ops *hif_ops; -+ int rc = 0; -+ -+ if (id->driver_data >= MWLUNKNOWN) -+ return -ENODEV; -+ -+ if (!printed_version) { -+ pr_info("<<%s version %s>>\n", -+ PCIE_DRV_DESC, PCIE_DRV_VERSION); -+ printed_version = true; -+ } -+ -+ rc = pci_enable_device(pdev); -+ if (rc) { -+ pr_err("%s: cannot enable new PCI device\n", -+ PCIE_DRV_NAME); -+ return rc; -+ } -+ -+ rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); -+ if (rc) { -+ pr_err("%s: 32-bit PCI DMA not supported\n", -+ PCIE_DRV_NAME); -+ goto err_pci_disable_device; -+ } -+ -+ pci_set_master(pdev); -+ -+ if (id->driver_data == MWL8964) -+ hif_ops = &pcie_hif_ops_ndp; -+ else -+ hif_ops = &pcie_hif_ops; -+ hw = mwl_alloc_hw(MWL_BUS_PCIE, id->driver_data, &pdev->dev, -+ hif_ops, sizeof(*pcie_priv)); -+ if (!hw) { -+ pr_err("%s: mwlwifi hw alloc failed\n", -+ PCIE_DRV_NAME); -+ rc = -ENOMEM; -+ goto err_pci_disable_device; -+ } -+ -+ pci_set_drvdata(pdev, hw); -+ -+ priv = hw->priv; -+ priv->antenna_tx = pcie_chip_tbl[priv->chip_type].antenna_tx; -+ priv->antenna_rx = pcie_chip_tbl[priv->chip_type].antenna_rx; -+ pcie_priv = priv->hif.priv; -+ pcie_priv->mwl_priv = priv; -+ pcie_priv->pdev = pdev; -+ if (id->driver_data != MWL8964) { -+ pcie_priv->tx_head_room = PCIE_MIN_BYTES_HEADROOM; -+ if (id->driver_data == MWL8997) { -+ if (NET_SKB_PAD < PCIE_MIN_TX_HEADROOM_KF2) { -+ pcie_priv->tx_head_room = -+ PCIE_MIN_TX_HEADROOM_KF2; -+ pcie_hif_ops.tx_head_room = -+ PCIE_MIN_TX_HEADROOM_KF2; -+ } -+ } -+ } -+ -+ rc = pcie_alloc_resource(pcie_priv); -+ if (rc) -+ goto err_alloc_pci_resource; -+ -+ rc = mwl_init_hw(hw, pcie_chip_tbl[priv->chip_type].fw_image, -+ pcie_chip_tbl[priv->chip_type].cal_file, -+ pcie_chip_tbl[priv->chip_type].txpwrlmt_file); -+ if (rc) -+ goto err_wl_init; -+ -+ vendor_cmd_basic_event(hw->wiphy, MWL_VENDOR_EVENT_DRIVER_READY); -+ -+ return rc; -+ -+err_wl_init: -+ -+ pcie_reset(hw); -+ -+err_alloc_pci_resource: -+ -+ pci_set_drvdata(pdev, NULL); -+ mwl_free_hw(hw); -+ -+err_pci_disable_device: -+ -+ pci_disable_device(pdev); -+ -+ return rc; -+} -+ -+static void pcie_remove(struct pci_dev *pdev) -+{ -+ struct ieee80211_hw *hw = pci_get_drvdata(pdev); -+ struct mwl_priv *priv = hw->priv; -+ -+ priv->rmmod = true; -+ while (priv->in_send_cmd) -+ usleep_range(1000, 2000); -+ vendor_cmd_basic_event(hw->wiphy, MWL_VENDOR_EVENT_DRIVER_START_REMOVE); -+ mwl_deinit_hw(hw); -+ pci_set_drvdata(pdev, NULL); -+ mwl_free_hw(hw); -+ pci_disable_device(pdev); -+} -+ -+static struct pci_driver mwl_pcie_driver = { -+ .name = PCIE_DRV_NAME, -+ .id_table = pcie_id_tbl, -+ .probe = pcie_probe, -+ .remove = pcie_remove -+}; -+ -+module_pci_driver(mwl_pcie_driver); -+ -+MODULE_DESCRIPTION(PCIE_DRV_DESC); -+MODULE_VERSION(PCIE_DRV_VERSION); -+MODULE_AUTHOR("Marvell Semiconductor, Inc."); -+MODULE_LICENSE("GPL v2"); -+MODULE_SUPPORTED_DEVICE(PCIE_DEV_NAME); -+MODULE_DEVICE_TABLE(pci, pcie_id_tbl); -diff --git a/drivers/net/wireless/marvell/mwlwifi/hif/pcie/rx.c b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/rx.c -new file mode 100644 -index 000000000000..25076c6d66df ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/rx.c -@@ -0,0 +1,540 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file implements receive related functions. */ -+ -+#include -+#include -+ -+#include "sysadpt.h" -+#include "core.h" -+#include "utils.h" -+#include "hif/pcie/dev.h" -+#include "hif/pcie/rx.h" -+ -+#define MAX_NUM_RX_RING_BYTES (PCIE_MAX_NUM_RX_DESC * \ -+ sizeof(struct pcie_rx_desc)) -+ -+#define MAX_NUM_RX_HNDL_BYTES (PCIE_MAX_NUM_RX_DESC * \ -+ sizeof(struct pcie_rx_hndl)) -+ -+#define DECRYPT_ERR_MASK 0x80 -+#define GENERAL_DECRYPT_ERR 0xFF -+#define TKIP_DECRYPT_MIC_ERR 0x02 -+#define WEP_DECRYPT_ICV_ERR 0x04 -+#define TKIP_DECRYPT_ICV_ERR 0x08 -+ -+#define W836X_RSSI_OFFSET 8 -+ -+static int pcie_rx_ring_alloc(struct mwl_priv *priv) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct pcie_desc_data *desc; -+ -+ desc = &pcie_priv->desc_data[0]; -+ -+ desc->prx_ring = (struct pcie_rx_desc *) -+ dma_alloc_coherent(priv->dev, -+ MAX_NUM_RX_RING_BYTES, -+ &desc->pphys_rx_ring, -+ GFP_KERNEL); -+ -+ if (!desc->prx_ring) { -+ wiphy_err(priv->hw->wiphy, "cannot alloc mem\n"); -+ return -ENOMEM; -+ } -+ -+ memset(desc->prx_ring, 0x00, MAX_NUM_RX_RING_BYTES); -+ -+ desc->rx_hndl = kzalloc(MAX_NUM_RX_HNDL_BYTES, GFP_KERNEL); -+ -+ if (!desc->rx_hndl) { -+ dma_free_coherent(priv->dev, -+ MAX_NUM_RX_RING_BYTES, -+ desc->prx_ring, -+ desc->pphys_rx_ring); -+ return -ENOMEM; -+ } -+ -+ return 0; -+} -+ -+static int pcie_rx_ring_init(struct mwl_priv *priv) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct pcie_desc_data *desc; -+ int i; -+ struct pcie_rx_hndl *rx_hndl; -+ dma_addr_t dma; -+ u32 val; -+ -+ desc = &pcie_priv->desc_data[0]; -+ -+ if (desc->prx_ring) { -+ desc->rx_buf_size = SYSADPT_MAX_AGGR_SIZE; -+ -+ for (i = 0; i < PCIE_MAX_NUM_RX_DESC; i++) { -+ rx_hndl = &desc->rx_hndl[i]; -+ rx_hndl->psk_buff = -+ dev_alloc_skb(desc->rx_buf_size); -+ -+ if (!rx_hndl->psk_buff) { -+ wiphy_err(priv->hw->wiphy, -+ "rxdesc %i: no skbuff available\n", -+ i); -+ return -ENOMEM; -+ } -+ -+ skb_reserve(rx_hndl->psk_buff, -+ PCIE_MIN_BYTES_HEADROOM); -+ desc->prx_ring[i].rx_control = -+ EAGLE_RXD_CTRL_DRIVER_OWN; -+ desc->prx_ring[i].status = EAGLE_RXD_STATUS_OK; -+ desc->prx_ring[i].qos_ctrl = 0x0000; -+ desc->prx_ring[i].channel = 0x00; -+ desc->prx_ring[i].rssi = 0x00; -+ desc->prx_ring[i].pkt_len = -+ cpu_to_le16(SYSADPT_MAX_AGGR_SIZE); -+ dma = pci_map_single(pcie_priv->pdev, -+ rx_hndl->psk_buff->data, -+ desc->rx_buf_size, -+ PCI_DMA_FROMDEVICE); -+ if (pci_dma_mapping_error(pcie_priv->pdev, dma)) { -+ wiphy_err(priv->hw->wiphy, -+ "failed to map pci memory!\n"); -+ return -ENOMEM; -+ } -+ desc->prx_ring[i].pphys_buff_data = cpu_to_le32(dma); -+ val = (u32)desc->pphys_rx_ring + -+ ((i + 1) * sizeof(struct pcie_rx_desc)); -+ desc->prx_ring[i].pphys_next = cpu_to_le32(val); -+ rx_hndl->pdesc = &desc->prx_ring[i]; -+ if (i < (PCIE_MAX_NUM_RX_DESC - 1)) -+ rx_hndl->pnext = &desc->rx_hndl[i + 1]; -+ } -+ desc->prx_ring[PCIE_MAX_NUM_RX_DESC - 1].pphys_next = -+ cpu_to_le32((u32)desc->pphys_rx_ring); -+ desc->rx_hndl[PCIE_MAX_NUM_RX_DESC - 1].pnext = -+ &desc->rx_hndl[0]; -+ desc->pnext_rx_hndl = &desc->rx_hndl[0]; -+ -+ return 0; -+ } -+ -+ wiphy_err(priv->hw->wiphy, "no valid RX mem\n"); -+ -+ return -ENOMEM; -+} -+ -+static void pcie_rx_ring_cleanup(struct mwl_priv *priv) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct pcie_desc_data *desc; -+ int i; -+ struct pcie_rx_hndl *rx_hndl; -+ -+ desc = &pcie_priv->desc_data[0]; -+ -+ if (desc->prx_ring) { -+ for (i = 0; i < PCIE_MAX_NUM_RX_DESC; i++) { -+ rx_hndl = &desc->rx_hndl[i]; -+ if (!rx_hndl->psk_buff) -+ continue; -+ -+ pci_unmap_single(pcie_priv->pdev, -+ le32_to_cpu -+ (rx_hndl->pdesc->pphys_buff_data), -+ desc->rx_buf_size, -+ PCI_DMA_FROMDEVICE); -+ -+ dev_kfree_skb_any(rx_hndl->psk_buff); -+ -+ wiphy_debug(priv->hw->wiphy, -+ "unmapped+free'd %i 0x%p 0x%x %i\n", -+ i, rx_hndl->psk_buff->data, -+ le32_to_cpu( -+ rx_hndl->pdesc->pphys_buff_data), -+ desc->rx_buf_size); -+ -+ rx_hndl->psk_buff = NULL; -+ } -+ } -+} -+ -+static void pcie_rx_ring_free(struct mwl_priv *priv) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct pcie_desc_data *desc; -+ -+ desc = &pcie_priv->desc_data[0]; -+ -+ if (desc->prx_ring) { -+ pcie_rx_ring_cleanup(priv); -+ -+ dma_free_coherent(priv->dev, -+ MAX_NUM_RX_RING_BYTES, -+ desc->prx_ring, -+ desc->pphys_rx_ring); -+ -+ desc->prx_ring = NULL; -+ } -+ -+ kfree(desc->rx_hndl); -+ -+ desc->pnext_rx_hndl = NULL; -+} -+ -+static inline void pcie_rx_status(struct mwl_priv *priv, -+ struct pcie_rx_desc *pdesc, -+ struct ieee80211_rx_status *status) -+{ -+ u16 rx_rate; -+ -+ memset(status, 0, sizeof(*status)); -+ -+ if (priv->chip_type == MWL8997) -+ status->signal = (s8)pdesc->rssi; -+ else -+ status->signal = -(pdesc->rssi + W836X_RSSI_OFFSET); -+ -+ rx_rate = le16_to_cpu(pdesc->rate); -+ pcie_rx_prepare_status(priv, -+ rx_rate & MWL_RX_RATE_FORMAT_MASK, -+ (rx_rate & MWL_RX_RATE_NSS_MASK) >> -+ MWL_RX_RATE_NSS_SHIFT, -+ (rx_rate & MWL_RX_RATE_BW_MASK) >> -+ MWL_RX_RATE_BW_SHIFT, -+ (rx_rate & MWL_RX_RATE_GI_MASK) >> -+ MWL_RX_RATE_GI_SHIFT, -+ (rx_rate & MWL_RX_RATE_RT_MASK) >> -+ MWL_RX_RATE_RT_SHIFT, -+ status); -+ -+ status->freq = ieee80211_channel_to_frequency(pdesc->channel, -+ status->band); -+ -+ /* check if status has a specific error bit (bit 7) set or indicates -+ * a general decrypt error -+ */ -+ if ((pdesc->status == GENERAL_DECRYPT_ERR) || -+ (pdesc->status & DECRYPT_ERR_MASK)) { -+ /* check if status is not equal to 0xFF -+ * the 0xFF check is for backward compatibility -+ */ -+ if (pdesc->status != GENERAL_DECRYPT_ERR) { -+ if (((pdesc->status & (~DECRYPT_ERR_MASK)) & -+ TKIP_DECRYPT_MIC_ERR) && !((pdesc->status & -+ (WEP_DECRYPT_ICV_ERR | TKIP_DECRYPT_ICV_ERR)))) { -+ status->flag |= RX_FLAG_MMIC_ERROR; -+ } -+ } -+ } -+} -+ -+static inline bool pcie_rx_process_mesh_amsdu(struct mwl_priv *priv, -+ struct sk_buff *skb, -+ struct ieee80211_rx_status *status) -+{ -+ struct ieee80211_hdr *wh; -+ struct mwl_sta *sta_info; -+ struct ieee80211_sta *sta; -+ u8 *qc; -+ int wh_len; -+ int len; -+ u8 pad; -+ u8 *data; -+ u16 frame_len; -+ struct sk_buff *newskb; -+ -+ wh = (struct ieee80211_hdr *)skb->data; -+ -+ spin_lock_bh(&priv->sta_lock); -+ list_for_each_entry(sta_info, &priv->sta_list, list) { -+ sta = container_of((void *)sta_info, struct ieee80211_sta, -+ drv_priv[0]); -+ if (ether_addr_equal(sta->addr, wh->addr2)) { -+ if (!sta_info->is_mesh_node) { -+ spin_unlock_bh(&priv->sta_lock); -+ return false; -+ } -+ } -+ } -+ spin_unlock_bh(&priv->sta_lock); -+ -+ qc = ieee80211_get_qos_ctl(wh); -+ *qc &= ~IEEE80211_QOS_CTL_A_MSDU_PRESENT; -+ -+ wh_len = ieee80211_hdrlen(wh->frame_control); -+ len = wh_len; -+ data = skb->data; -+ -+ while (len < skb->len) { -+ frame_len = *(u8 *)(data + len + ETH_HLEN - 1) | -+ (*(u8 *)(data + len + ETH_HLEN - 2) << 8); -+ -+ if ((len + ETH_HLEN + frame_len) > skb->len) -+ break; -+ -+ newskb = dev_alloc_skb(wh_len + frame_len); -+ if (!newskb) -+ break; -+ -+ ether_addr_copy(wh->addr3, data + len); -+ ether_addr_copy(wh->addr4, data + len + ETH_ALEN); -+ memcpy(newskb->data, wh, wh_len); -+ memcpy(newskb->data + wh_len, data + len + ETH_HLEN, frame_len); -+ skb_put(newskb, wh_len + frame_len); -+ -+ pad = ((ETH_HLEN + frame_len) % 4) ? -+ (4 - (ETH_HLEN + frame_len) % 4) : 0; -+ len += (ETH_HLEN + frame_len + pad); -+ if (len < skb->len) -+ status->flag |= RX_FLAG_AMSDU_MORE; -+ else -+ status->flag &= ~RX_FLAG_AMSDU_MORE; -+ memcpy(IEEE80211_SKB_RXCB(newskb), status, sizeof(*status)); -+ ieee80211_rx(priv->hw, newskb); -+ } -+ -+ dev_kfree_skb_any(skb); -+ -+ return true; -+} -+ -+static inline int pcie_rx_refill(struct mwl_priv *priv, -+ struct pcie_rx_hndl *rx_hndl) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct pcie_desc_data *desc; -+ dma_addr_t dma; -+ -+ desc = &pcie_priv->desc_data[0]; -+ -+ rx_hndl->psk_buff = dev_alloc_skb(desc->rx_buf_size); -+ -+ if (!rx_hndl->psk_buff) -+ return -ENOMEM; -+ -+ skb_reserve(rx_hndl->psk_buff, PCIE_MIN_BYTES_HEADROOM); -+ -+ rx_hndl->pdesc->status = EAGLE_RXD_STATUS_OK; -+ rx_hndl->pdesc->qos_ctrl = 0x0000; -+ rx_hndl->pdesc->channel = 0x00; -+ rx_hndl->pdesc->rssi = 0x00; -+ rx_hndl->pdesc->pkt_len = cpu_to_le16(desc->rx_buf_size); -+ -+ dma = pci_map_single(pcie_priv->pdev, -+ rx_hndl->psk_buff->data, -+ desc->rx_buf_size, -+ PCI_DMA_FROMDEVICE); -+ if (pci_dma_mapping_error(pcie_priv->pdev, dma)) { -+ dev_kfree_skb_any(rx_hndl->psk_buff); -+ wiphy_err(priv->hw->wiphy, -+ "failed to map pci memory!\n"); -+ return -ENOMEM; -+ } -+ -+ rx_hndl->pdesc->pphys_buff_data = cpu_to_le32(dma); -+ -+ return 0; -+} -+ -+int pcie_rx_init(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ int rc; -+ -+ rc = pcie_rx_ring_alloc(priv); -+ if (rc) { -+ wiphy_err(hw->wiphy, "allocating RX ring failed\n"); -+ return rc; -+ } -+ -+ rc = pcie_rx_ring_init(priv); -+ if (rc) { -+ pcie_rx_ring_free(priv); -+ wiphy_err(hw->wiphy, -+ "initializing RX ring failed\n"); -+ return rc; -+ } -+ -+ return 0; -+} -+ -+void pcie_rx_deinit(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ pcie_rx_ring_cleanup(priv); -+ pcie_rx_ring_free(priv); -+} -+ -+void pcie_rx_recv(unsigned long data) -+{ -+ struct ieee80211_hw *hw = (struct ieee80211_hw *)data; -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct pcie_desc_data *desc; -+ struct pcie_rx_hndl *curr_hndl; -+ int work_done = 0; -+ struct sk_buff *prx_skb = NULL; -+ int pkt_len; -+ struct ieee80211_rx_status *status; -+ struct mwl_vif *mwl_vif = NULL; -+ struct ieee80211_hdr *wh; -+ -+ desc = &pcie_priv->desc_data[0]; -+ curr_hndl = desc->pnext_rx_hndl; -+ -+ if (!curr_hndl) { -+ pcie_mask_int(pcie_priv, MACREG_A2HRIC_BIT_RX_RDY, true); -+ pcie_priv->is_rx_schedule = false; -+ wiphy_warn(hw->wiphy, "busy or no receiving packets\n"); -+ return; -+ } -+ -+ while ((curr_hndl->pdesc->rx_control == EAGLE_RXD_CTRL_DMA_OWN) && -+ (work_done < pcie_priv->recv_limit)) { -+ prx_skb = curr_hndl->psk_buff; -+ if (!prx_skb) -+ goto out; -+ pci_unmap_single(pcie_priv->pdev, -+ le32_to_cpu(curr_hndl->pdesc->pphys_buff_data), -+ desc->rx_buf_size, -+ PCI_DMA_FROMDEVICE); -+ pkt_len = le16_to_cpu(curr_hndl->pdesc->pkt_len); -+ -+ if (skb_tailroom(prx_skb) < pkt_len) { -+ dev_kfree_skb_any(prx_skb); -+ goto out; -+ } -+ -+ if (curr_hndl->pdesc->channel != -+ hw->conf.chandef.chan->hw_value) { -+ dev_kfree_skb_any(prx_skb); -+ goto out; -+ } -+ -+ status = IEEE80211_SKB_RXCB(prx_skb); -+ pcie_rx_status(priv, curr_hndl->pdesc, status); -+ -+ if (priv->chip_type == MWL8997) { -+ priv->noise = (s8)curr_hndl->pdesc->noise_floor; -+ if (priv->noise > 0) -+ priv->noise = -priv->noise; -+ } else -+ priv->noise = -curr_hndl->pdesc->noise_floor; -+ -+ wh = &((struct pcie_dma_data *)prx_skb->data)->wh; -+ -+ if (ieee80211_has_protected(wh->frame_control)) { -+ /* Check if hw crypto has been enabled for -+ * this bss. If yes, set the status flags -+ * accordingly -+ */ -+ if (ieee80211_has_tods(wh->frame_control)) { -+ mwl_vif = utils_find_vif_bss(priv, wh->addr1); -+ if (!mwl_vif && -+ ieee80211_has_a4(wh->frame_control)) -+ mwl_vif = -+ utils_find_vif_bss(priv, -+ wh->addr2); -+ } else { -+ mwl_vif = utils_find_vif_bss(priv, wh->addr2); -+ } -+ -+ if ((mwl_vif && mwl_vif->is_hw_crypto_enabled) || -+ is_multicast_ether_addr(wh->addr1) || -+ (ieee80211_is_mgmt(wh->frame_control) && -+ !is_multicast_ether_addr(wh->addr1))) { -+ /* When MMIC ERROR is encountered -+ * by the firmware, payload is -+ * dropped and only 32 bytes of -+ * mwlwifi Firmware header is sent -+ * to the host. -+ * -+ * We need to add four bytes of -+ * key information. In it -+ * MAC80211 expects keyidx set to -+ * 0 for triggering Counter -+ * Measure of MMIC failure. -+ */ -+ if (status->flag & RX_FLAG_MMIC_ERROR) { -+ struct pcie_dma_data *dma_data; -+ -+ dma_data = (struct pcie_dma_data *) -+ prx_skb->data; -+ memset((void *)&dma_data->data, 0, 4); -+ pkt_len += 4; -+ } -+ -+ if (!ieee80211_is_auth(wh->frame_control)) { -+ if (priv->chip_type != MWL8997) -+ status->flag |= -+ RX_FLAG_IV_STRIPPED | -+ RX_FLAG_DECRYPTED | -+ RX_FLAG_MMIC_STRIPPED; -+ else -+ status->flag |= -+ RX_FLAG_DECRYPTED | -+ RX_FLAG_MMIC_STRIPPED; -+ } -+ } -+ } -+ -+ skb_put(prx_skb, pkt_len); -+ pcie_rx_remove_dma_header(prx_skb, curr_hndl->pdesc->qos_ctrl); -+ -+ wh = (struct ieee80211_hdr *)prx_skb->data; -+ -+ if (ieee80211_is_data_qos(wh->frame_control)) { -+ const u8 eapol[] = {0x88, 0x8e}; -+ u8 *qc = ieee80211_get_qos_ctl(wh); -+ u8 *data; -+ -+ data = prx_skb->data + -+ ieee80211_hdrlen(wh->frame_control) + 6; -+ -+ if (!memcmp(data, eapol, sizeof(eapol))) -+ *qc |= 7; -+ } -+ -+ if (ieee80211_is_data_qos(wh->frame_control) && -+ ieee80211_has_a4(wh->frame_control)) { -+ u8 *qc = ieee80211_get_qos_ctl(wh); -+ -+ if (*qc & IEEE80211_QOS_CTL_A_MSDU_PRESENT) -+ if (pcie_rx_process_mesh_amsdu(priv, prx_skb, -+ status)) -+ goto out; -+ } -+ -+ if (ieee80211_is_probe_req(wh->frame_control) && -+ priv->dump_probe) -+ wiphy_info(hw->wiphy, "Probe Req: %pM\n", wh->addr2); -+ -+ ieee80211_rx(hw, prx_skb); -+out: -+ pcie_rx_refill(priv, curr_hndl); -+ curr_hndl->pdesc->rx_control = EAGLE_RXD_CTRL_DRIVER_OWN; -+ curr_hndl->pdesc->qos_ctrl = 0; -+ curr_hndl = curr_hndl->pnext; -+ work_done++; -+ } -+ -+ desc->pnext_rx_hndl = curr_hndl; -+ pcie_mask_int(pcie_priv, MACREG_A2HRIC_BIT_RX_RDY, true); -+ pcie_priv->is_rx_schedule = false; -+} -diff --git a/drivers/net/wireless/marvell/mwlwifi/hif/pcie/rx.h b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/rx.h -new file mode 100644 -index 000000000000..d2b580fceb0a ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/rx.h -@@ -0,0 +1,25 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file defines receive related functions. */ -+ -+#ifndef _RX_H_ -+#define _RX_H_ -+ -+int pcie_rx_init(struct ieee80211_hw *hw); -+void pcie_rx_deinit(struct ieee80211_hw *hw); -+void pcie_rx_recv(unsigned long data); -+ -+#endif /* _RX_H_ */ -diff --git a/drivers/net/wireless/marvell/mwlwifi/hif/pcie/rx_ndp.c b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/rx_ndp.c -new file mode 100644 -index 000000000000..d1ede588b4c1 ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/rx_ndp.c -@@ -0,0 +1,612 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file implements receive related functions for new data -+ * path. -+ */ -+ -+#include -+#include -+ -+#include "sysadpt.h" -+#include "core.h" -+#include "utils.h" -+#include "hif/pcie/dev.h" -+#include "hif/pcie/rx_ndp.h" -+ -+#define MAX_NUM_RX_RING_BYTES (MAX_NUM_RX_DESC * \ -+ sizeof(struct pcie_rx_desc_ndp)) -+#define MAX_NUM_RX_RING_DONE_BYTES (MAX_NUM_RX_DESC * \ -+ sizeof(struct rx_ring_done)) -+ -+static int pcie_rx_ring_alloc_ndp(struct mwl_priv *priv) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct pcie_desc_data_ndp *desc = &pcie_priv->desc_data_ndp; -+ -+ desc->prx_ring = (struct pcie_rx_desc_ndp *) -+ dma_alloc_coherent(priv->dev, -+ MAX_NUM_RX_RING_BYTES, -+ &desc->pphys_rx_ring, -+ GFP_KERNEL); -+ if (!desc->prx_ring) -+ goto err_no_mem; -+ memset(desc->prx_ring, 0x00, MAX_NUM_RX_RING_BYTES); -+ -+ desc->prx_ring_done = (struct rx_ring_done *) -+ dma_alloc_coherent(priv->dev, -+ MAX_NUM_RX_RING_DONE_BYTES, -+ &desc->pphys_rx_ring_done, -+ GFP_KERNEL); -+ if (!desc->prx_ring_done) -+ goto err_no_mem; -+ memset(desc->prx_ring_done, 0x00, MAX_NUM_RX_RING_DONE_BYTES); -+ return 0; -+ -+err_no_mem: -+ -+ wiphy_err(priv->hw->wiphy, "cannot alloc mem\n"); -+ return -ENOMEM; -+} -+ -+static int pcie_rx_ring_init_ndp(struct mwl_priv *priv) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct pcie_desc_data_ndp *desc = &pcie_priv->desc_data_ndp; -+ int i; -+ struct sk_buff *psk_buff; -+ dma_addr_t dma; -+ -+ skb_queue_head_init(&pcie_priv->rx_skb_trace); -+ if (desc->prx_ring) { -+ desc->rx_buf_size = MAX_AGGR_SIZE; -+ -+ for (i = 0; i < MAX_NUM_RX_DESC; i++) { -+ psk_buff = __alloc_skb(desc->rx_buf_size + NET_SKB_PAD, -+ GFP_ATOMIC, SKB_ALLOC_RX, -+ NUMA_NO_NODE); -+ skb_reserve(psk_buff, NET_SKB_PAD); -+ if (!psk_buff) { -+ wiphy_err(priv->hw->wiphy, -+ "rxdesc %i: no skbuff available\n", -+ i); -+ return -ENOMEM; -+ } -+ skb_reserve(psk_buff, MIN_BYTES_RX_HEADROOM); -+ -+ dma = pci_map_single(pcie_priv->pdev, -+ psk_buff->data, -+ desc->rx_buf_size, -+ PCI_DMA_FROMDEVICE); -+ if (pci_dma_mapping_error(pcie_priv->pdev, dma)) { -+ wiphy_err(priv->hw->wiphy, -+ "failed to map pci memory!\n"); -+ return -ENOMEM; -+ } -+ -+ desc->rx_vbuflist[i] = psk_buff; -+ desc->prx_ring[i].user = cpu_to_le32(i); -+ desc->prx_ring[i].data = cpu_to_le32(dma); -+ *((u32 *)&psk_buff->cb[16]) = 0xdeadbeef; -+ skb_queue_tail(&pcie_priv->rx_skb_trace, psk_buff); -+ } -+ -+ writel(1023, pcie_priv->iobase1 + MACREG_REG_RXDESCHEAD); -+ return 0; -+ } -+ -+ wiphy_err(priv->hw->wiphy, "no valid RX mem\n"); -+ return -ENOMEM; -+} -+ -+static void pcie_rx_ring_cleanup_ndp(struct mwl_priv *priv) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct pcie_desc_data_ndp *desc = &pcie_priv->desc_data_ndp; -+ int i; -+ -+ if (desc->prx_ring) { -+ for (i = 0; i < MAX_NUM_RX_DESC; i++) { -+ if (desc->rx_vbuflist[i]) { -+ pci_unmap_single(pcie_priv->pdev, -+ le32_to_cpu( -+ desc->prx_ring[i].data), -+ desc->rx_buf_size, -+ PCI_DMA_FROMDEVICE); -+ desc->rx_vbuflist[i] = NULL; -+ } -+ } -+ skb_queue_purge(&pcie_priv->rx_skb_trace); -+ } -+} -+ -+static void pcie_rx_ring_free_ndp(struct mwl_priv *priv) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct pcie_desc_data_ndp *desc = &pcie_priv->desc_data_ndp; -+ -+ if (desc->prx_ring) { -+ pcie_rx_ring_cleanup_ndp(priv); -+ dma_free_coherent(priv->dev, -+ MAX_NUM_RX_RING_BYTES, -+ desc->prx_ring, -+ desc->pphys_rx_ring); -+ desc->prx_ring = NULL; -+ } -+ -+ if (desc->prx_ring_done) { -+ dma_free_coherent(priv->dev, -+ MAX_NUM_RX_RING_DONE_BYTES, -+ desc->prx_ring_done, -+ desc->pphys_rx_ring_done); -+ desc->prx_ring_done = NULL; -+ } -+} -+ -+static inline void pcie_rx_update_ndp_cnts(struct mwl_priv *priv, u32 ctrl) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ -+ switch (ctrl) { -+ case RXRING_CASE_DROP: -+ pcie_priv->rx_cnts.drop_cnt++; -+ break; -+ case RXRING_CASE_FAST_BAD_AMSDU: -+ pcie_priv->rx_cnts.fast_bad_amsdu_cnt++; -+ break; -+ case RXRING_CASE_FAST_DATA: -+ pcie_priv->rx_cnts.fast_data_cnt++; -+ break; -+ case RXRING_CASE_SLOW_BAD_MIC: -+ pcie_priv->rx_cnts.slow_bad_mic_cnt++; -+ break; -+ case RXRING_CASE_SLOW_BAD_PN: -+ pcie_priv->rx_cnts.slow_bad_pn_cnt++; -+ break; -+ case RXRING_CASE_SLOW_BAD_STA: -+ pcie_priv->rx_cnts.slow_bad_sta_cnt++; -+ break; -+ case RXRING_CASE_SLOW_MCAST: -+ pcie_priv->rx_cnts.slow_mcast_cnt++; -+ break; -+ case RXRING_CASE_SLOW_MGMT: -+ pcie_priv->rx_cnts.slow_mgmt_cnt++; -+ break; -+ case RXRING_CASE_SLOW_NOQUEUE: -+ pcie_priv->rx_cnts.slow_noqueue_cnt++; -+ break; -+ case RXRING_CASE_SLOW_NORUN: -+ pcie_priv->rx_cnts.slow_norun_cnt++; -+ break; -+ case RXRING_CASE_SLOW_PROMISC: -+ pcie_priv->rx_cnts.slow_promisc_cnt++; -+ break; -+ } -+} -+ -+static void pcie_rx_status_ndp(struct mwl_priv *priv, -+ struct mwl_sta *sta_info, -+ struct ieee80211_rx_status *status) -+{ -+ memset(status, 0, sizeof(*status)); -+ pcie_rx_prepare_status(priv, -+ sta_info->rx_format, -+ sta_info->rx_nss, -+ sta_info->rx_bw, -+ sta_info->rx_gi, -+ sta_info->rx_rate_mcs, -+ status); -+ status->signal = -sta_info->rx_signal; -+ status->band = priv->hw->conf.chandef.chan->band; -+ status->freq = ieee80211_channel_to_frequency( -+ priv->hw->conf.chandef.chan->hw_value, status->band); -+} -+ -+static inline void pcie_rx_process_fast_data(struct mwl_priv *priv, -+ struct sk_buff *skb, -+ u16 stnid) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct ieee80211_sta *sta; -+ struct mwl_sta *sta_info; -+ struct mwl_vif *mwl_vif; -+ struct ieee80211_hdr hdr; -+ u16 hdrlen, ethertype; -+ __le16 fc; -+ struct ieee80211_rx_status *status; -+ -+ if (stnid == RXRING_CTRL_STA_FROMDS) -+ stnid = 0; -+ -+ if (stnid > SYSADPT_MAX_STA_SC4) -+ goto drop_packet; -+ -+ sta = pcie_priv->sta_link[stnid]; -+ if (!sta) -+ goto drop_packet; -+ -+ sta_info = mwl_dev_get_sta(sta); -+ mwl_vif = sta_info->mwl_vif; -+ if (!mwl_vif) -+ goto drop_packet; -+ -+ ethertype = (skb->data[20] << 8) | skb->data[21]; -+ fc = cpu_to_le16(IEEE80211_FTYPE_DATA | IEEE80211_STYPE_DATA); -+ -+ memset(&hdr, 0, sizeof(hdr)); -+ switch (mwl_vif->type) { -+ case NL80211_IFTYPE_AP: -+ if (sta_info->wds) { -+ fc |= (cpu_to_le16(IEEE80211_FCTL_TODS) | -+ cpu_to_le16(IEEE80211_FCTL_FROMDS)); -+ /* RA TA DA SA */ -+ ether_addr_copy(hdr.addr1, mwl_vif->bssid); -+ ether_addr_copy(hdr.addr2, sta->addr); -+ ether_addr_copy(hdr.addr3, skb->data); -+ ether_addr_copy(hdr.addr4, skb->data + ETH_ALEN); -+ hdrlen = 30; -+ } else { -+ fc |= cpu_to_le16(IEEE80211_FCTL_TODS); -+ /* BSSID SA DA */ -+ ether_addr_copy(hdr.addr1, mwl_vif->bssid); -+ ether_addr_copy(hdr.addr2, skb->data + ETH_ALEN); -+ ether_addr_copy(hdr.addr3, skb->data); -+ hdrlen = 24; -+ } -+ break; -+ case NL80211_IFTYPE_STATION: -+ if (sta_info->wds) { -+ fc |= (cpu_to_le16(IEEE80211_FCTL_TODS) | -+ cpu_to_le16(IEEE80211_FCTL_FROMDS)); -+ /* RA TA DA SA */ -+ ether_addr_copy(hdr.addr1, mwl_vif->sta_mac); -+ ether_addr_copy(hdr.addr2, mwl_vif->bssid); -+ ether_addr_copy(hdr.addr3, skb->data); -+ ether_addr_copy(hdr.addr4, skb->data + ETH_ALEN); -+ hdrlen = 30; -+ } else { -+ fc |= cpu_to_le16(IEEE80211_FCTL_FROMDS); -+ /* DA BSSID SA */ -+ ether_addr_copy(hdr.addr1, skb->data); -+ ether_addr_copy(hdr.addr2, mwl_vif->bssid); -+ ether_addr_copy(hdr.addr3, skb->data + ETH_ALEN); -+ hdrlen = 24; -+ } -+ break; -+ default: -+ goto drop_packet; -+ } -+ -+ if (sta->wme) { -+ fc |= cpu_to_le16(IEEE80211_STYPE_QOS_DATA); -+ hdrlen += 2; -+ } -+ -+ status = IEEE80211_SKB_RXCB(skb); -+ pcie_rx_status_ndp(priv, sta_info, status); -+ if (mwl_vif->is_hw_crypto_enabled) { -+ fc |= cpu_to_le16(IEEE80211_FCTL_PROTECTED); -+ status->flag |= RX_FLAG_IV_STRIPPED | -+ RX_FLAG_DECRYPTED | -+ RX_FLAG_MMIC_STRIPPED; -+ } -+ -+ hdr.frame_control = fc; -+ hdr.duration_id = 0; -+ -+ skb_pull(skb, ETH_HLEN); -+ -+ if (ieee80211_is_data_qos(fc)) { -+ __le16 *qos_control; -+ -+ qos_control = (__le16 *)skb_push(skb, 2); -+ memcpy(skb_push(skb, hdrlen - 2), &hdr, hdrlen - 2); -+ if (ethertype == ETH_P_PAE) -+ *qos_control = cpu_to_le16( -+ IEEE80211_QOS_CTL_ACK_POLICY_NOACK | 7); -+ else -+ *qos_control = cpu_to_le16( -+ IEEE80211_QOS_CTL_ACK_POLICY_NOACK); -+ } else -+ memcpy(skb_push(skb, hdrlen), &hdr, hdrlen); -+ -+ status->flag |= RX_FLAG_DUP_VALIDATED; -+ ieee80211_rx(priv->hw, skb); -+ -+ return; -+drop_packet: -+ -+ dev_kfree_skb_any(skb); -+} -+ -+static inline void pcie_rx_process_slow_data(struct mwl_priv *priv, -+ struct sk_buff *skb, -+ bool bad_mic, u8 signal) -+{ -+ struct ieee80211_rx_status *status; -+ struct ieee80211_hdr *wh; -+ struct mwl_vif *mwl_vif = NULL; -+ -+ pcie_rx_remove_dma_header(skb, 0); -+ status = IEEE80211_SKB_RXCB(skb); -+ memset(status, 0, sizeof(*status)); -+ status->signal = -signal; -+ status->band = priv->hw->conf.chandef.chan->band; -+ status->freq = ieee80211_channel_to_frequency( -+ priv->hw->conf.chandef.chan->hw_value, status->band); -+ -+ if (bad_mic) -+ status->flag |= RX_FLAG_MMIC_ERROR; -+ else { -+ wh = (struct ieee80211_hdr *)skb->data; -+ -+ if (ieee80211_has_protected(wh->frame_control)) { -+ if (ieee80211_has_tods(wh->frame_control)) { -+ mwl_vif = utils_find_vif_bss(priv, wh->addr1); -+ if (!mwl_vif && -+ ieee80211_has_a4(wh->frame_control)) -+ mwl_vif = -+ utils_find_vif_bss(priv, -+ wh->addr2); -+ } else { -+ mwl_vif = utils_find_vif_bss(priv, wh->addr2); -+ } -+ -+ if ((mwl_vif && mwl_vif->is_hw_crypto_enabled) || -+ is_multicast_ether_addr(wh->addr1) || -+ (ieee80211_is_mgmt(wh->frame_control) && -+ !is_multicast_ether_addr(wh->addr1))) { -+ if (!ieee80211_is_auth(wh->frame_control)) -+ status->flag |= RX_FLAG_IV_STRIPPED | -+ RX_FLAG_DECRYPTED | -+ RX_FLAG_MMIC_STRIPPED; -+ } -+ } -+ -+ if (ieee80211_has_a4(wh->frame_control) && !priv->wds_check) { -+ ether_addr_copy(priv->wds_check_sta, wh->addr2); -+ ieee80211_queue_work(priv->hw, &priv->wds_check_handle); -+ priv->wds_check = true; -+ } -+ } -+ -+ status->flag |= RX_FLAG_DUP_VALIDATED; -+ ieee80211_rx(priv->hw, skb); -+} -+ -+static inline int pcie_rx_refill_ndp(struct mwl_priv *priv, u32 buf_idx) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct pcie_desc_data_ndp *desc = &pcie_priv->desc_data_ndp; -+ struct sk_buff *psk_buff; -+ dma_addr_t dma; -+ -+ psk_buff = __alloc_skb(desc->rx_buf_size + NET_SKB_PAD, GFP_ATOMIC, -+ SKB_ALLOC_RX, NUMA_NO_NODE); -+ skb_reserve(psk_buff, NET_SKB_PAD); -+ if (!psk_buff) -+ return -ENOMEM; -+ skb_reserve(psk_buff, MIN_BYTES_RX_HEADROOM); -+ -+ dma = pci_map_single(pcie_priv->pdev, -+ psk_buff->data, -+ desc->rx_buf_size, -+ PCI_DMA_FROMDEVICE); -+ if (pci_dma_mapping_error(pcie_priv->pdev, dma)) { -+ wiphy_err(priv->hw->wiphy, -+ "refill: failed to map pci memory!\n"); -+ return -ENOMEM; -+ } -+ -+ desc->rx_vbuflist[buf_idx] = psk_buff; -+ desc->prx_ring[buf_idx].data = cpu_to_le32(dma); -+ *((u32 *)&psk_buff->cb[16]) = 0xdeadbeef; -+ skb_queue_tail(&pcie_priv->rx_skb_trace, psk_buff); -+ -+ return 0; -+} -+ -+int pcie_rx_init_ndp(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ int rc; -+ -+ rc = pcie_rx_ring_alloc_ndp(priv); -+ if (rc) { -+ pcie_rx_ring_free_ndp(priv); -+ wiphy_err(hw->wiphy, "allocating RX ring failed\n"); -+ return rc; -+ } -+ -+ rc = pcie_rx_ring_init_ndp(priv); -+ if (rc) { -+ pcie_rx_ring_free_ndp(priv); -+ wiphy_err(hw->wiphy, -+ "initializing RX ring failed\n"); -+ return rc; -+ } -+ -+ return 0; -+} -+ -+void pcie_rx_deinit_ndp(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ pcie_rx_ring_cleanup_ndp(priv); -+ pcie_rx_ring_free_ndp(priv); -+} -+ -+void pcie_rx_recv_ndp(unsigned long data) -+{ -+ struct ieee80211_hw *hw = (struct ieee80211_hw *)data; -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct pcie_desc_data_ndp *desc = &pcie_priv->desc_data_ndp; -+ struct rx_ring_done *prx_ring_done; -+ struct pcie_rx_desc_ndp *prx_desc; -+ u32 rx_done_head; -+ u32 rx_done_tail; -+ u32 rx_desc_head; -+ struct sk_buff *psk_buff; -+ u32 buf_idx; -+ u32 rx_cnt; -+ u32 ctrl, ctrl_case; -+ bool bad_mic; -+ u16 stnid; -+ u16 pktlen; -+ struct rx_info *rx_info; -+ struct pcie_dma_data *dma_data; -+ u8 signal; -+ -+ rx_done_head = readl(pcie_priv->iobase1 + MACREG_REG_RXDONEHEAD); -+ rx_done_tail = readl(pcie_priv->iobase1 + MACREG_REG_RXDONETAIL); -+ rx_desc_head = readl(pcie_priv->iobase1 + MACREG_REG_RXDESCHEAD); -+ rx_cnt = 0; -+ -+ while ((rx_done_tail != rx_done_head) && -+ (rx_cnt < pcie_priv->recv_limit)) { -+recheck: -+ prx_ring_done = &desc->prx_ring_done[rx_done_tail]; -+ wmb(); /*Data Memory Barrier*/ -+ if (le32_to_cpu(prx_ring_done->user) == 0xdeadbeef) { -+ pcie_priv->recheck_rxringdone++; -+ udelay(1); -+ goto recheck; -+ } -+ buf_idx = le32_to_cpu(prx_ring_done->user) & 0x3fff; -+ prx_ring_done->user = cpu_to_le32(0xdeadbeef); -+ rx_done_tail++; -+ prx_desc = &desc->prx_ring[buf_idx]; -+ if (!prx_desc->data) -+ wiphy_err(hw->wiphy, "RX desc data is NULL\n"); -+ psk_buff = desc->rx_vbuflist[buf_idx]; -+ if (!psk_buff) { -+ wiphy_err(hw->wiphy, "RX socket buffer is NULL\n"); -+ goto out; -+ } -+ if (*((u32 *)&psk_buff->cb[16]) != 0xdeadbeef) { -+ pcie_priv->signature_err++; -+ break; -+ } -+ if (psk_buff->next && psk_buff->prev) { -+ skb_unlink(psk_buff, &pcie_priv->rx_skb_trace); -+ *((u32 *)&psk_buff->cb[16]) = 0xbeefdead; -+ } else { -+ pcie_priv->rx_skb_unlink_err++; -+ break; -+ } -+ -+ pci_unmap_single(pcie_priv->pdev, -+ le32_to_cpu(prx_desc->data), -+ desc->rx_buf_size, -+ PCI_DMA_FROMDEVICE); -+ -+ bad_mic = false; -+ ctrl = le32_to_cpu(prx_ring_done->ctrl); -+ ctrl_case = ctrl & RXRING_CTRL_CASE_MASK; -+ stnid = (ctrl >> RXRING_CTRL_STA_SHIFT) & RXRING_CTRL_STA_MASK; -+ pcie_rx_update_ndp_cnts(priv, ctrl_case); -+ -+ switch (ctrl_case) { -+ case RXRING_CASE_FAST_DATA: -+ if (stnid == RXRING_CTRL_STA_UNKNOWN) { -+ dev_kfree_skb_any(psk_buff); -+ break; -+ } -+ pktlen = psk_buff->data[12] << 8 | psk_buff->data[13]; -+ pktlen += ETH_HLEN; -+ -+ if (skb_tailroom(psk_buff) >= pktlen) { -+ skb_put(psk_buff, pktlen); -+ pcie_rx_process_fast_data(priv, psk_buff, -+ stnid); -+ } else { -+ wiphy_err(hw->wiphy, -+ "fast: space %d(%d) is not enough\n", -+ skb_tailroom(psk_buff), pktlen); -+ dev_kfree_skb_any(psk_buff); -+ } -+ break; -+ case RXRING_CASE_FAST_BAD_AMSDU: -+ case RXRING_CASE_SLOW_BAD_STA: -+ case RXRING_CASE_SLOW_DEL_DONE: -+ case RXRING_CASE_DROP: -+ case RXRING_CASE_SLOW_BAD_PN: -+ if (ctrl_case == RXRING_CASE_SLOW_DEL_DONE) { -+ wiphy_debug(hw->wiphy, -+ "staid %d deleted\n", -+ stnid); -+ utils_free_stnid(priv, stnid); -+ } -+ dev_kfree_skb_any(psk_buff); -+ break; -+ case RXRING_CASE_SLOW_BAD_MIC: -+ bad_mic = true; -+ case RXRING_CASE_SLOW_NOQUEUE: -+ case RXRING_CASE_SLOW_NORUN: -+ case RXRING_CASE_SLOW_MGMT: -+ case RXRING_CASE_SLOW_MCAST: -+ case RXRING_CASE_SLOW_PROMISC: -+ rx_info = (struct rx_info *)psk_buff->data; -+ dma_data = (struct pcie_dma_data *)&rx_info->hdr[0]; -+ pktlen = le16_to_cpu(dma_data->fwlen); -+ pktlen += sizeof(*rx_info); -+ pktlen += sizeof(struct pcie_dma_data); -+ if (bad_mic) { -+ memset((void *)&dma_data->data, 0, 4); -+ pktlen += 4; -+ } -+ if (skb_tailroom(psk_buff) >= pktlen) { -+ skb_put(psk_buff, pktlen); -+ skb_pull(psk_buff, sizeof(*rx_info)); -+ signal = ((le32_to_cpu(rx_info->rssi_x) >> -+ RXINFO_RSSI_X_SHIFT) & -+ RXINFO_RSSI_X_MASK); -+ pcie_rx_process_slow_data(priv, psk_buff, -+ bad_mic, signal); -+ } else { -+ wiphy_err(hw->wiphy, -+ "slow: space %d(%d) is not enough\n", -+ skb_tailroom(psk_buff), pktlen); -+ dev_kfree_skb_any(psk_buff); -+ } -+ break; -+ default: -+ wiphy_err(hw->wiphy, "unknown control case: %d\n", -+ ctrl_case); -+ dev_kfree_skb_any(psk_buff); -+ break; -+ } -+out: -+ pcie_rx_refill_ndp(priv, buf_idx); -+ -+ if (rx_done_tail >= MAX_RX_RING_DONE_SIZE) -+ rx_done_tail = 0; -+ -+ rx_done_head = -+ readl(pcie_priv->iobase1 + MACREG_REG_RXDONEHEAD); -+ rx_cnt++; -+ } -+ -+ rx_desc_head += rx_cnt; -+ if (rx_desc_head >= MAX_RX_RING_SEND_SIZE) -+ rx_desc_head = rx_desc_head - MAX_RX_RING_SEND_SIZE; -+ writel(rx_done_tail, pcie_priv->iobase1 + MACREG_REG_RXDONETAIL); -+ writel(rx_desc_head, pcie_priv->iobase1 + MACREG_REG_RXDESCHEAD); -+ -+ pcie_mask_int(pcie_priv, MACREG_A2HRIC_RX_DONE_HEAD_RDY, true); -+ pcie_priv->is_rx_schedule = false; -+} -diff --git a/drivers/net/wireless/marvell/mwlwifi/hif/pcie/rx_ndp.h b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/rx_ndp.h -new file mode 100644 -index 000000000000..7e83cedf4351 ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/rx_ndp.h -@@ -0,0 +1,26 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file defines receive related functions for new data path. -+ */ -+ -+#ifndef _RX_NDP_H_ -+#define _RX_NDP_H_ -+ -+int pcie_rx_init_ndp(struct ieee80211_hw *hw); -+void pcie_rx_deinit_ndp(struct ieee80211_hw *hw); -+void pcie_rx_recv_ndp(unsigned long data); -+ -+#endif /* _RX_NDP_H_ */ -diff --git a/drivers/net/wireless/marvell/mwlwifi/hif/pcie/sc4_ddr.h b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/sc4_ddr.h -new file mode 100644 -index 000000000000..2da0257accba ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/sc4_ddr.h -@@ -0,0 +1,965 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+static unsigned char sc4_ddr_init[] = { -+0x05, -+0x00, -+0x00, -+0x00, -+0x00, -+0x00, -+0x00, -+0x00, -+0xa4, -+0x03, -+0x00, -+0x00, -+0x2a, -+0xbe, -+0xad, -+0x7e, -+0x00, -+0x00, -+0x00, -+0xa8, -+0x01, -+0x00, -+0x00, -+0x00, -+0x00, -+0x00, -+0x00, -+0x00, -+0x5c, -+0x48, -+0x5b, -+0x49, -+0x04, -+0x00, -+0x00, -+0x00, -+0x10, -+0xb5, -+0xc0, -+0xf8, -+0x08, -+0x00, -+0x00, -+0x00, -+0xe0, -+0x12, -+0xef, -+0x21, -+0x0c, -+0x00, -+0x00, -+0x00, -+0xc0, -+0xf8, -+0x20, -+0x13, -+0x10, -+0x00, -+0x00, -+0x00, -+0x02, -+0x21, -+0xc0, -+0xf8, -+0x14, -+0x00, -+0x00, -+0x00, -+0x24, -+0x13, -+0x02, -+0x01, -+0x18, -+0x00, -+0x00, -+0x00, -+0x57, -+0x49, -+0x0a, -+0x60, -+0x1c, -+0x00, -+0x00, -+0x00, -+0x00, -+0x21, -+0x56, -+0x4b, -+0x20, -+0x00, -+0x00, -+0x00, -+0xc0, -+0x3b, -+0x19, -+0x60, -+0x24, -+0x00, -+0x00, -+0x00, -+0x54, -+0x4b, -+0x1b, -+0x1d, -+0x28, -+0x00, -+0x00, -+0x00, -+0x1a, -+0x60, -+0x53, -+0x4b, -+0x2c, -+0x00, -+0x00, -+0x00, -+0xbc, -+0x3b, -+0x19, -+0x60, -+0x30, -+0x00, -+0x00, -+0x00, -+0x51, -+0x4b, -+0x08, -+0x33, -+0x34, -+0x00, -+0x00, -+0x00, -+0x1a, -+0x60, -+0x50, -+0x4b, -+0x38, -+0x00, -+0x00, -+0x00, -+0xb8, -+0x3b, -+0x19, -+0x60, -+0x3c, -+0x00, -+0x00, -+0x00, -+0x4e, -+0x4b, -+0x0c, -+0x33, -+0x40, -+0x00, -+0x00, -+0x00, -+0x1a, -+0x60, -+0x4d, -+0x4b, -+0x44, -+0x00, -+0x00, -+0x00, -+0xb4, -+0x3b, -+0x19, -+0x60, -+0x48, -+0x00, -+0x00, -+0x00, -+0x4b, -+0x4b, -+0x10, -+0x33, -+0x4c, -+0x00, -+0x00, -+0x00, -+0x1a, -+0x60, -+0x4a, -+0x4b, -+0x50, -+0x00, -+0x00, -+0x00, -+0xb0, -+0x3b, -+0x19, -+0x60, -+0x54, -+0x00, -+0x00, -+0x00, -+0x48, -+0x4b, -+0x80, -+0x33, -+0x58, -+0x00, -+0x00, -+0x00, -+0x1a, -+0x60, -+0x47, -+0x4b, -+0x5c, -+0x00, -+0x00, -+0x00, -+0x40, -+0x3b, -+0x19, -+0x60, -+0x60, -+0x00, -+0x00, -+0x00, -+0x47, -+0x4c, -+0x46, -+0x4b, -+0x64, -+0x00, -+0x00, -+0x00, -+0x23, -+0x60, -+0x24, -+0x1d, -+0x68, -+0x00, -+0x00, -+0x00, -+0x23, -+0x60, -+0x24, -+0x1d, -+0x6c, -+0x00, -+0x00, -+0x00, -+0x23, -+0x60, -+0x44, -+0x4b, -+0x70, -+0x00, -+0x00, -+0x00, -+0x40, -+0x33, -+0x19, -+0x60, -+0x74, -+0x00, -+0x00, -+0x00, -+0x1b, -+0x1d, -+0x19, -+0x60, -+0x78, -+0x00, -+0x00, -+0x00, -+0x1b, -+0x1d, -+0x19, -+0x60, -+0x7c, -+0x00, -+0x00, -+0x00, -+0x41, -+0x4b, -+0xc0, -+0xf8, -+0x80, -+0x00, -+0x00, -+0x00, -+0xe0, -+0x31, -+0x41, -+0x4b, -+0x84, -+0x00, -+0x00, -+0x00, -+0xc0, -+0xf8, -+0xf0, -+0x31, -+0x88, -+0x00, -+0x00, -+0x00, -+0x03, -+0x04, -+0xc0, -+0xf8, -+0x8c, -+0x00, -+0x00, -+0x00, -+0xf0, -+0x32, -+0x40, -+0xf2, -+0x90, -+0x00, -+0x00, -+0x00, -+0x55, -+0x13, -+0xc0, -+0xf8, -+0x94, -+0x00, -+0x00, -+0x00, -+0x60, -+0x33, -+0x3d, -+0x4b, -+0x98, -+0x00, -+0x00, -+0x00, -+0xc0, -+0xf8, -+0x64, -+0x33, -+0x9c, -+0x00, -+0x00, -+0x00, -+0x13, -+0x1d, -+0xc0, -+0xf8, -+0xa0, -+0x00, -+0x00, -+0x00, -+0x68, -+0x33, -+0x3b, -+0x4b, -+0xa4, -+0x00, -+0x00, -+0x00, -+0xc0, -+0xf8, -+0x6c, -+0x33, -+0xa8, -+0x00, -+0x00, -+0x00, -+0x3a, -+0x4b, -+0xc0, -+0xf8, -+0xac, -+0x00, -+0x00, -+0x00, -+0x70, -+0x33, -+0x3a, -+0x4b, -+0xb0, -+0x00, -+0x00, -+0x00, -+0xc0, -+0xf8, -+0x74, -+0x33, -+0xb4, -+0x00, -+0x00, -+0x00, -+0x39, -+0x4b, -+0xc0, -+0xf8, -+0xb8, -+0x00, -+0x00, -+0x00, -+0x78, -+0x33, -+0xc4, -+0x23, -+0xbc, -+0x00, -+0x00, -+0x00, -+0xc0, -+0xf8, -+0x7c, -+0x33, -+0xc0, -+0x00, -+0x00, -+0x00, -+0x37, -+0x4b, -+0xc0, -+0xf8, -+0xc4, -+0x00, -+0x00, -+0x00, -+0x80, -+0x33, -+0x37, -+0x4b, -+0xc8, -+0x00, -+0x00, -+0x00, -+0xc0, -+0xf8, -+0x84, -+0x33, -+0xcc, -+0x00, -+0x00, -+0x00, -+0x36, -+0x4b, -+0xc0, -+0xf8, -+0xd0, -+0x00, -+0x00, -+0x00, -+0x88, -+0x33, -+0x42, -+0xf2, -+0xd4, -+0x00, -+0x00, -+0x00, -+0x44, -+0x23, -+0xc0, -+0xf8, -+0xd8, -+0x00, -+0x00, -+0x00, -+0x8c, -+0x33, -+0xc0, -+0xf8, -+0xdc, -+0x00, -+0x00, -+0x00, -+0x90, -+0x13, -+0x4f, -+0xf4, -+0xe0, -+0x00, -+0x00, -+0x00, -+0x60, -+0x43, -+0xc0, -+0xf8, -+0xe4, -+0x00, -+0x00, -+0x00, -+0xa0, -+0x32, -+0xc0, -+0xf8, -+0xe8, -+0x00, -+0x00, -+0x00, -+0xa4, -+0x22, -+0xc0, -+0xf8, -+0xec, -+0x00, -+0x00, -+0x00, -+0xa8, -+0x12, -+0x44, -+0xf2, -+0xf0, -+0x00, -+0x00, -+0x00, -+0x40, -+0x02, -+0x2e, -+0x4b, -+0xf4, -+0x00, -+0x00, -+0x00, -+0x1a, -+0x60, -+0x1b, -+0x1d, -+0xf8, -+0x00, -+0x00, -+0x00, -+0x2d, -+0x4a, -+0x1a, -+0x60, -+0xfc, -+0x00, -+0x00, -+0x00, -+0x1b, -+0x1d, -+0x2d, -+0x4a, -+0x00, -+0x01, -+0x00, -+0x00, -+0x1a, -+0x60, -+0x1b, -+0x1d, -+0x04, -+0x01, -+0x00, -+0x00, -+0x2c, -+0x4a, -+0x1a, -+0x60, -+0x08, -+0x01, -+0x00, -+0x00, -+0x1b, -+0x1d, -+0x2c, -+0x4a, -+0x0c, -+0x01, -+0x00, -+0x00, -+0x1a, -+0x60, -+0x1b, -+0x1d, -+0x10, -+0x01, -+0x00, -+0x00, -+0x4f, -+0xf4, -+0x60, -+0x12, -+0x14, -+0x01, -+0x00, -+0x00, -+0x1a, -+0x60, -+0x25, -+0x4a, -+0x18, -+0x01, -+0x00, -+0x00, -+0x28, -+0x32, -+0x11, -+0x60, -+0x1c, -+0x01, -+0x00, -+0x00, -+0x23, -+0x4a, -+0x30, -+0x32, -+0x20, -+0x01, -+0x00, -+0x00, -+0x11, -+0x60, -+0x12, -+0x1d, -+0x24, -+0x01, -+0x00, -+0x00, -+0x11, -+0x60, -+0x03, -+0x22, -+0x28, -+0x01, -+0x00, -+0x00, -+0x20, -+0x4b, -+0x38, -+0x33, -+0x2c, -+0x01, -+0x00, -+0x00, -+0x1a, -+0x60, -+0x20, -+0x22, -+0x30, -+0x01, -+0x00, -+0x00, -+0x1b, -+0x1d, -+0x1a, -+0x60, -+0x34, -+0x01, -+0x00, -+0x00, -+0x1b, -+0x1d, -+0x4f, -+0xf0, -+0x38, -+0x01, -+0x00, -+0x00, -+0x04, -+0x22, -+0x1a, -+0x60, -+0x3c, -+0x01, -+0x00, -+0x00, -+0x03, -+0x06, -+0x1b, -+0x4a, -+0x40, -+0x01, -+0x00, -+0x00, -+0x20, -+0x32, -+0x13, -+0x60, -+0x44, -+0x01, -+0x00, -+0x00, -+0x43, -+0x06, -+0x13, -+0x60, -+0x48, -+0x01, -+0x00, -+0x00, -+0x83, -+0x06, -+0x13, -+0x60, -+0x4c, -+0x01, -+0x00, -+0x00, -+0x0c, -+0x4b, -+0x1c, -+0x4a, -+0x50, -+0x01, -+0x00, -+0x00, -+0x70, -+0x33, -+0x1a, -+0x60, -+0x54, -+0x01, -+0x00, -+0x00, -+0x41, -+0x62, -+0x4f, -+0xf4, -+0x58, -+0x01, -+0x00, -+0x00, -+0x7f, -+0x22, -+0x82, -+0x62, -+0x5c, -+0x01, -+0x00, -+0x00, -+0x19, -+0x4a, -+0x82, -+0x66, -+0x60, -+0x01, -+0x00, -+0x00, -+0xc1, -+0x66, -+0x40, -+0xf2, -+0x64, -+0x01, -+0x00, -+0x00, -+0x63, -+0x42, -+0x42, -+0x63, -+0x68, -+0x01, -+0x00, -+0x00, -+0x01, -+0x64, -+0x17, -+0x49, -+0x6c, -+0x01, -+0x00, -+0x00, -+0x01, -+0x60, -+0x10, -+0xbd, -+0x70, -+0x01, -+0x00, -+0x00, -+0x22, -+0x76, -+0x30, -+0x00, -+0x74, -+0x01, -+0x00, -+0x00, -+0x20, -+0x00, -+0x00, -+0xf0, -+0x78, -+0x01, -+0x00, -+0x00, -+0x40, -+0x06, -+0x00, -+0xf0, -+0x7c, -+0x01, -+0x00, -+0x00, -+0x04, -+0x00, -+0x15, -+0x15, -+0x80, -+0x01, -+0x00, -+0x00, -+0x00, -+0x05, -+0x00, -+0xf0, -+0x84, -+0x01, -+0x00, -+0x00, -+0x01, -+0x00, -+0x0d, -+0x00, -+0x88, -+0x01, -+0x00, -+0x00, -+0x32, -+0x05, -+0x00, -+0x04, -+0x8c, -+0x01, -+0x00, -+0x00, -+0xa9, -+0x02, -+0xb8, -+0x00, -+0x90, -+0x01, -+0x00, -+0x00, -+0x00, -+0x01, -+0x40, -+0x00, -+0x94, -+0x01, -+0x00, -+0x00, -+0xeb, -+0x06, -+0x77, -+0x00, -+0x98, -+0x01, -+0x00, -+0x00, -+0x00, -+0x52, -+0x7b, -+0x50, -+0x9c, -+0x01, -+0x00, -+0x00, -+0x0b, -+0x06, -+0x04, -+0x10, -+0xa0, -+0x01, -+0x00, -+0x00, -+0x10, -+0x07, -+0x17, -+0x13, -+0xa4, -+0x01, -+0x00, -+0x00, -+0x07, -+0x74, -+0x70, -+0x00, -+0xa8, -+0x01, -+0x00, -+0x00, -+0x40, -+0x00, -+0x70, -+0x50, -+0xac, -+0x01, -+0x00, -+0x00, -+0x00, -+0x04, -+0x00, -+0xf0, -+0xb0, -+0x01, -+0x00, -+0x00, -+0x79, -+0x07, -+0x70, -+0x17, -+0xb4, -+0x01, -+0x00, -+0x00, -+0x70, -+0x07, -+0xf0, -+0x0f, -+0xb8, -+0x01, -+0x00, -+0x00, -+0x77, -+0xfc, -+0x03, -+0x3f, -+0xbc, -+0x01, -+0x00, -+0x00, -+0x00, -+0x31, -+0x10, -+0x00, -+0xc0, -+0x01, -+0x00, -+0x00, -+0x01, -+0x00, -+0x00, -+0xc0, -+0xc4, -+0x01, -+0x00, -+0x00, -+0x66, -+0x66, -+0x66, -+0x00, -+0xc8, -+0x01, -+0x00, -+0x00, -+0x01, -+0x00, -+0x00, -+0x11, -+0x37, -+0x3e, -+0xfc, -+0xdc, -+}; -diff --git a/drivers/net/wireless/marvell/mwlwifi/hif/pcie/tx.c b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/tx.c -new file mode 100644 -index 000000000000..dd77589ef5a6 ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/tx.c -@@ -0,0 +1,1396 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file implements transmit related functions. */ -+ -+#include -+#include -+ -+#include "sysadpt.h" -+#include "core.h" -+#include "utils.h" -+#include "hif/fwcmd.h" -+#include "hif/pcie/dev.h" -+#include "hif/pcie/tx.h" -+ -+#define MAX_NUM_TX_RING_BYTES (PCIE_MAX_NUM_TX_DESC * \ -+ sizeof(struct pcie_tx_desc)) -+ -+#define MAX_NUM_TX_HNDL_BYTES (PCIE_MAX_NUM_TX_DESC * \ -+ sizeof(struct pcie_tx_hndl)) -+ -+#define TOTAL_HW_QUEUES (SYSADPT_TX_WMM_QUEUES + \ -+ PCIE_AMPDU_QUEUES) -+ -+#define EAGLE_TXD_XMITCTRL_USE_MC_RATE 0x8 /* Use multicast data rate */ -+ -+#define MWL_QOS_ACK_POLICY_MASK 0x0060 -+#define MWL_QOS_ACK_POLICY_NORMAL 0x0000 -+#define MWL_QOS_ACK_POLICY_BLOCKACK 0x0060 -+ -+#define EXT_IV 0x20 -+#define INCREASE_IV(iv16, iv32) \ -+{ \ -+ (iv16)++; \ -+ if ((iv16) == 0) \ -+ (iv32)++; \ -+} -+ -+/* Transmission information to transmit a socket buffer. */ -+struct pcie_tx_ctrl { -+ void *vif; -+ void *sta; -+ void *k_conf; -+ void *amsdu_pkts; -+ u8 tx_priority; -+ u8 type; -+ u16 qos_ctrl; -+ u8 xmit_control; -+}; -+ -+struct ccmp_hdr { -+ __le16 iv16; -+ u8 rsvd; -+ u8 key_id; -+ __le32 iv32; -+} __packed; -+ -+static int pcie_tx_ring_alloc(struct mwl_priv *priv) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct pcie_desc_data *desc; -+ int num; -+ u8 *mem; -+ -+ desc = &pcie_priv->desc_data[0]; -+ -+ mem = dma_alloc_coherent(priv->dev, -+ MAX_NUM_TX_RING_BYTES * -+ PCIE_NUM_OF_DESC_DATA, -+ &desc->pphys_tx_ring, -+ GFP_KERNEL); -+ -+ if (!mem) { -+ wiphy_err(priv->hw->wiphy, "cannot alloc mem\n"); -+ return -ENOMEM; -+ } -+ -+ for (num = 0; num < PCIE_NUM_OF_DESC_DATA; num++) { -+ desc = &pcie_priv->desc_data[num]; -+ -+ desc->ptx_ring = (struct pcie_tx_desc *) -+ (mem + num * MAX_NUM_TX_RING_BYTES); -+ -+ desc->pphys_tx_ring = (dma_addr_t) -+ ((u32)pcie_priv->desc_data[0].pphys_tx_ring + -+ num * MAX_NUM_TX_RING_BYTES); -+ -+ memset(desc->ptx_ring, 0x00, -+ MAX_NUM_TX_RING_BYTES); -+ } -+ -+ mem = kzalloc(MAX_NUM_TX_HNDL_BYTES * PCIE_NUM_OF_DESC_DATA, -+ GFP_KERNEL); -+ -+ if (!mem) { -+ wiphy_err(priv->hw->wiphy, "cannot alloc mem\n"); -+ dma_free_coherent(priv->dev, -+ MAX_NUM_TX_RING_BYTES * -+ PCIE_NUM_OF_DESC_DATA, -+ pcie_priv->desc_data[0].ptx_ring, -+ pcie_priv->desc_data[0].pphys_tx_ring); -+ return -ENOMEM; -+ } -+ -+ for (num = 0; num < PCIE_NUM_OF_DESC_DATA; num++) { -+ desc = &pcie_priv->desc_data[num]; -+ -+ desc->tx_hndl = (struct pcie_tx_hndl *) -+ (mem + num * MAX_NUM_TX_HNDL_BYTES); -+ } -+ -+ return 0; -+} -+ -+static int pcie_txbd_ring_create(struct mwl_priv *priv) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ int num; -+ u8 *mem; -+ -+ /* driver maintaines the write pointer and firmware maintaines the read -+ * pointer. -+ */ -+ pcie_priv->txbd_wrptr = 0; -+ pcie_priv->txbd_rdptr = 0; -+ -+ /* allocate shared memory for the BD ring and divide the same in to -+ * several descriptors -+ */ -+ pcie_priv->txbd_ring_size = -+ sizeof(struct pcie_data_buf) * PCIE_MAX_TXRX_BD; -+ wiphy_info(priv->hw->wiphy, "TX ring: allocating %d bytes\n", -+ pcie_priv->txbd_ring_size); -+ -+ mem = dma_alloc_coherent(priv->dev, -+ pcie_priv->txbd_ring_size, -+ &pcie_priv->txbd_ring_pbase, -+ GFP_KERNEL); -+ -+ if (!mem) { -+ wiphy_err(priv->hw->wiphy, "cannot alloc mem\n"); -+ return -ENOMEM; -+ } -+ pcie_priv->txbd_ring_vbase = mem; -+ wiphy_info(priv->hw->wiphy, -+ "TX ring: - base: %p, pbase: 0x%x, len: %d\n", -+ pcie_priv->txbd_ring_vbase, -+ pcie_priv->txbd_ring_pbase, -+ pcie_priv->txbd_ring_size); -+ -+ for (num = 0; num < PCIE_MAX_TXRX_BD; num++) { -+ pcie_priv->txbd_ring[num] = -+ (struct pcie_data_buf *)(pcie_priv->txbd_ring_vbase + -+ (sizeof(struct pcie_data_buf) * num)); -+ pcie_priv->txbd_ring[num]->flags = 0; -+ pcie_priv->txbd_ring[num]->offset = 0; -+ pcie_priv->txbd_ring[num]->frag_len = 0; -+ pcie_priv->txbd_ring[num]->len = 0; -+ pcie_priv->txbd_ring[num]->paddr = 0; -+ pcie_priv->tx_buf_list[num] = NULL; -+ } -+ -+ return 0; -+} -+ -+static int pcie_tx_ring_init(struct mwl_priv *priv) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ int num, i; -+ struct pcie_desc_data *desc; -+ -+ for (num = 0; num < PCIE_NUM_OF_DESC_DATA; num++) { -+ skb_queue_head_init(&pcie_priv->txq[num]); -+ pcie_priv->fw_desc_cnt[num] = 0; -+ -+ if (priv->chip_type == MWL8997) -+ continue; -+ -+ desc = &pcie_priv->desc_data[num]; -+ -+ if (desc->ptx_ring) { -+ for (i = 0; i < PCIE_MAX_NUM_TX_DESC; i++) { -+ desc->ptx_ring[i].status = -+ cpu_to_le32(EAGLE_TXD_STATUS_IDLE); -+ desc->ptx_ring[i].pphys_next = -+ cpu_to_le32((u32)desc->pphys_tx_ring + -+ ((i + 1) * -+ sizeof(struct pcie_tx_desc))); -+ desc->tx_hndl[i].pdesc = -+ &desc->ptx_ring[i]; -+ if (i < PCIE_MAX_NUM_TX_DESC - 1) -+ desc->tx_hndl[i].pnext = -+ &desc->tx_hndl[i + 1]; -+ } -+ desc->ptx_ring[PCIE_MAX_NUM_TX_DESC - 1].pphys_next = -+ cpu_to_le32((u32)desc->pphys_tx_ring); -+ desc->tx_hndl[PCIE_MAX_NUM_TX_DESC - 1].pnext = -+ &desc->tx_hndl[0]; -+ -+ desc->pstale_tx_hndl = &desc->tx_hndl[0]; -+ desc->pnext_tx_hndl = &desc->tx_hndl[0]; -+ } else { -+ wiphy_err(priv->hw->wiphy, "no valid TX mem\n"); -+ return -ENOMEM; -+ } -+ } -+ -+ return 0; -+} -+ -+static void pcie_tx_ring_cleanup(struct mwl_priv *priv) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ int cleaned_tx_desc = 0; -+ int num, i; -+ struct pcie_desc_data *desc; -+ -+ for (num = 0; num < PCIE_NUM_OF_DESC_DATA; num++) { -+ skb_queue_purge(&pcie_priv->txq[num]); -+ pcie_priv->fw_desc_cnt[num] = 0; -+ -+ if (priv->chip_type == MWL8997) -+ continue; -+ -+ desc = &pcie_priv->desc_data[num]; -+ -+ if (desc->ptx_ring) { -+ for (i = 0; i < PCIE_MAX_NUM_TX_DESC; i++) { -+ if (!desc->tx_hndl[i].psk_buff) -+ continue; -+ -+ wiphy_debug(priv->hw->wiphy, -+ "unmapped and free'd %i %p %x\n", -+ i, -+ desc->tx_hndl[i].psk_buff->data, -+ le32_to_cpu( -+ desc->ptx_ring[i].pkt_ptr)); -+ pci_unmap_single(pcie_priv->pdev, -+ le32_to_cpu( -+ desc->ptx_ring[i].pkt_ptr), -+ desc->tx_hndl[i].psk_buff->len, -+ PCI_DMA_TODEVICE); -+ dev_kfree_skb_any(desc->tx_hndl[i].psk_buff); -+ desc->ptx_ring[i].status = -+ cpu_to_le32(EAGLE_TXD_STATUS_IDLE); -+ desc->ptx_ring[i].pkt_ptr = 0; -+ desc->ptx_ring[i].pkt_len = 0; -+ desc->tx_hndl[i].psk_buff = NULL; -+ cleaned_tx_desc++; -+ } -+ } -+ } -+ -+ wiphy_info(priv->hw->wiphy, "cleaned %i TX descr\n", cleaned_tx_desc); -+} -+ -+static void pcie_tx_ring_free(struct mwl_priv *priv) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ int num; -+ -+ if (pcie_priv->desc_data[0].ptx_ring) { -+ dma_free_coherent(priv->dev, -+ MAX_NUM_TX_RING_BYTES * -+ PCIE_NUM_OF_DESC_DATA, -+ pcie_priv->desc_data[0].ptx_ring, -+ pcie_priv->desc_data[0].pphys_tx_ring); -+ } -+ -+ for (num = 0; num < PCIE_NUM_OF_DESC_DATA; num++) { -+ if (pcie_priv->desc_data[num].ptx_ring) -+ pcie_priv->desc_data[num].ptx_ring = NULL; -+ pcie_priv->desc_data[num].pstale_tx_hndl = NULL; -+ pcie_priv->desc_data[num].pnext_tx_hndl = NULL; -+ } -+ -+ kfree(pcie_priv->desc_data[0].tx_hndl); -+} -+ -+static void pcie_txbd_ring_delete(struct mwl_priv *priv) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct sk_buff *skb; -+ struct pcie_tx_desc *tx_desc; -+ int num; -+ -+ if (pcie_priv->txbd_ring_vbase) { -+ dma_free_coherent(priv->dev, -+ pcie_priv->txbd_ring_size, -+ pcie_priv->txbd_ring_vbase, -+ pcie_priv->txbd_ring_pbase); -+ } -+ -+ for (num = 0; num < PCIE_MAX_TXRX_BD; num++) { -+ pcie_priv->txbd_ring[num] = NULL; -+ if (pcie_priv->tx_buf_list[num]) { -+ skb = pcie_priv->tx_buf_list[num]; -+ tx_desc = (struct pcie_tx_desc *)skb->data; -+ -+ pci_unmap_single(pcie_priv->pdev, -+ le32_to_cpu(tx_desc->pkt_ptr), -+ skb->len, -+ PCI_DMA_TODEVICE); -+ dev_kfree_skb_any(skb); -+ } -+ pcie_priv->tx_buf_list[num] = NULL; -+ } -+ -+ pcie_priv->txbd_wrptr = 0; -+ pcie_priv->txbd_rdptr = 0; -+ pcie_priv->txbd_ring_size = 0; -+ pcie_priv->txbd_ring_vbase = NULL; -+ pcie_priv->txbd_ring_pbase = 0; -+} -+ -+static inline void pcie_tx_add_ccmp_hdr(u8 *pccmp_hdr, -+ u8 key_id, u16 iv16, u32 iv32) -+{ -+ struct ccmp_hdr *ccmp_h = (struct ccmp_hdr *)pccmp_hdr; -+ -+ ccmp_h->iv16 = cpu_to_le16(iv16); -+ ccmp_h->rsvd = 0; -+ ccmp_h->key_id = EXT_IV | (key_id << 6); -+ ccmp_h->iv32 = cpu_to_le32(iv32); -+} -+ -+static inline bool pcie_tx_available(struct mwl_priv *priv, int desc_num) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct pcie_tx_hndl *tx_hndl; -+ -+ if (priv->chip_type == MWL8997) -+ return PCIE_TXBD_NOT_FULL(pcie_priv->txbd_wrptr, -+ pcie_priv->txbd_rdptr); -+ -+ tx_hndl = pcie_priv->desc_data[desc_num].pnext_tx_hndl; -+ -+ if (!tx_hndl->pdesc) -+ return false; -+ -+ if (tx_hndl->pdesc->status != EAGLE_TXD_STATUS_IDLE) { -+ /* Interrupt F/W anyway */ -+ if (tx_hndl->pdesc->status & -+ cpu_to_le32(EAGLE_TXD_STATUS_FW_OWNED)) -+ writel(MACREG_H2ARIC_BIT_PPA_READY, -+ pcie_priv->iobase1 + -+ MACREG_REG_H2A_INTERRUPT_EVENTS); -+ return false; -+ } -+ -+ return true; -+} -+ -+static inline void pcie_tx_skb(struct mwl_priv *priv, int desc_num, -+ struct sk_buff *tx_skb) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct ieee80211_tx_info *tx_info; -+ struct pcie_tx_ctrl *tx_ctrl; -+ struct pcie_tx_hndl *tx_hndl = NULL; -+ struct pcie_tx_desc *tx_desc; -+ struct ieee80211_sta *sta; -+ struct ieee80211_vif *vif; -+ struct mwl_vif *mwl_vif; -+ struct ieee80211_key_conf *k_conf; -+ bool ccmp = false; -+ struct pcie_pfu_dma_data *pfu_dma_data; -+ struct pcie_dma_data *dma_data; -+ struct ieee80211_hdr *wh; -+ dma_addr_t dma; -+ -+ if (WARN_ON(!tx_skb)) -+ return; -+ -+ tx_info = IEEE80211_SKB_CB(tx_skb); -+ tx_ctrl = (struct pcie_tx_ctrl *)&tx_info->status; -+ sta = (struct ieee80211_sta *)tx_ctrl->sta; -+ vif = (struct ieee80211_vif *)tx_ctrl->vif; -+ mwl_vif = mwl_dev_get_vif(vif); -+ k_conf = (struct ieee80211_key_conf *)tx_ctrl->k_conf; -+ -+ pcie_tx_encapsulate_frame(priv, tx_skb, k_conf, &ccmp); -+ -+ if (priv->chip_type == MWL8997) { -+ pfu_dma_data = (struct pcie_pfu_dma_data *)tx_skb->data; -+ tx_desc = &pfu_dma_data->tx_desc; -+ dma_data = &pfu_dma_data->dma_data; -+ } else { -+ tx_hndl = pcie_priv->desc_data[desc_num].pnext_tx_hndl; -+ tx_hndl->psk_buff = tx_skb; -+ tx_desc = tx_hndl->pdesc; -+ dma_data = (struct pcie_dma_data *)tx_skb->data; -+ } -+ wh = &dma_data->wh; -+ -+ if (ieee80211_is_probe_resp(wh->frame_control) && -+ priv->dump_probe) -+ wiphy_info(priv->hw->wiphy, -+ "Probe Resp: %pM\n", wh->addr1); -+ -+ if (ieee80211_is_data(wh->frame_control) || -+ (ieee80211_is_mgmt(wh->frame_control) && -+ ieee80211_has_protected(wh->frame_control) && -+ !is_multicast_ether_addr(wh->addr1))) { -+ if (is_multicast_ether_addr(wh->addr1)) { -+ if (ccmp) { -+ pcie_tx_add_ccmp_hdr(dma_data->data, -+ mwl_vif->keyidx, -+ mwl_vif->iv16, -+ mwl_vif->iv32); -+ INCREASE_IV(mwl_vif->iv16, mwl_vif->iv32); -+ } -+ } else { -+ if (ccmp) { -+ if (vif->type == NL80211_IFTYPE_STATION) { -+ pcie_tx_add_ccmp_hdr(dma_data->data, -+ mwl_vif->keyidx, -+ mwl_vif->iv16, -+ mwl_vif->iv32); -+ INCREASE_IV(mwl_vif->iv16, -+ mwl_vif->iv32); -+ } else { -+ struct mwl_sta *sta_info; -+ -+ sta_info = mwl_dev_get_sta(sta); -+ -+ pcie_tx_add_ccmp_hdr(dma_data->data, -+ 0, -+ sta_info->iv16, -+ sta_info->iv32); -+ INCREASE_IV(sta_info->iv16, -+ sta_info->iv32); -+ } -+ } -+ } -+ } -+ -+ if (tx_info->flags & IEEE80211_TX_INTFL_DONT_ENCRYPT) -+ tx_desc->flags |= PCIE_TX_WCB_FLAGS_DONT_ENCRYPT; -+ if (tx_info->flags & IEEE80211_TX_CTL_NO_CCK_RATE) -+ tx_desc->flags |= PCIE_TX_WCB_FLAGS_NO_CCK_RATE; -+ tx_desc->tx_priority = tx_ctrl->tx_priority; -+ tx_desc->qos_ctrl = cpu_to_le16(tx_ctrl->qos_ctrl); -+ tx_desc->pkt_len = cpu_to_le16(tx_skb->len); -+ tx_desc->packet_info = 0; -+ tx_desc->data_rate = 0; -+ tx_desc->type = tx_ctrl->type; -+ tx_desc->xmit_control = tx_ctrl->xmit_control; -+ tx_desc->sap_pkt_info = 0; -+ dma = pci_map_single(pcie_priv->pdev, tx_skb->data, -+ tx_skb->len, PCI_DMA_TODEVICE); -+ if (pci_dma_mapping_error(pcie_priv->pdev, dma)) { -+ dev_kfree_skb_any(tx_skb); -+ wiphy_err(priv->hw->wiphy, -+ "failed to map pci memory!\n"); -+ return; -+ } -+ if (priv->chip_type == MWL8997) -+ tx_desc->pkt_ptr = cpu_to_le32(sizeof(struct pcie_tx_desc)); -+ else -+ tx_desc->pkt_ptr = cpu_to_le32(dma); -+ tx_desc->status = cpu_to_le32(EAGLE_TXD_STATUS_FW_OWNED); -+ /* make sure all the memory transactions done by cpu were completed */ -+ wmb(); /*Data Memory Barrier*/ -+ -+ if (priv->chip_type == MWL8997) { -+ u32 wrindx; -+ struct pcie_data_buf *data_buf; -+ const u32 num_tx_buffs = PCIE_MAX_TXRX_BD << PCIE_TX_START_PTR; -+ -+ wrindx = (pcie_priv->txbd_wrptr & PCIE_TXBD_MASK) >> -+ PCIE_TX_START_PTR; -+ pcie_priv->tx_buf_list[wrindx] = tx_skb; -+ data_buf = pcie_priv->txbd_ring[wrindx]; -+ data_buf->paddr = cpu_to_le64(dma); -+ data_buf->len = cpu_to_le16(tx_skb->len); -+ data_buf->flags = cpu_to_le16(PCIE_BD_FLAG_FIRST_DESC | -+ PCIE_BD_FLAG_LAST_DESC); -+ data_buf->frag_len = cpu_to_le16(tx_skb->len); -+ data_buf->offset = 0; -+ pcie_priv->txbd_wrptr += PCIE_BD_FLAG_TX_START_PTR; -+ -+ if ((pcie_priv->txbd_wrptr & PCIE_TXBD_MASK) == num_tx_buffs) -+ pcie_priv->txbd_wrptr = ((pcie_priv->txbd_wrptr & -+ PCIE_BD_FLAG_TX_ROLLOVER_IND) ^ -+ PCIE_BD_FLAG_TX_ROLLOVER_IND); -+ -+ /* Write the TX ring write pointer in to REG_TXBD_WRPTR */ -+ writel(pcie_priv->txbd_wrptr, -+ pcie_priv->iobase1 + REG_TXBD_WRPTR); -+ } else { -+ writel(MACREG_H2ARIC_BIT_PPA_READY, -+ pcie_priv->iobase1 + MACREG_REG_H2A_INTERRUPT_EVENTS); -+ pcie_priv->desc_data[desc_num].pnext_tx_hndl = tx_hndl->pnext; -+ pcie_priv->fw_desc_cnt[desc_num]++; -+ } -+} -+ -+static inline -+struct sk_buff *pcie_tx_do_amsdu(struct mwl_priv *priv, -+ int desc_num, -+ struct sk_buff *tx_skb, -+ struct ieee80211_tx_info *tx_info) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct ieee80211_sta *sta; -+ struct mwl_sta *sta_info; -+ struct pcie_tx_ctrl *tx_ctrl = (struct pcie_tx_ctrl *)&tx_info->status; -+ struct ieee80211_tx_info *amsdu_info; -+ struct sk_buff_head *amsdu_pkts; -+ struct mwl_amsdu_frag *amsdu; -+ int amsdu_allow_size; -+ struct ieee80211_hdr *wh; -+ int wh_len; -+ u16 len; -+ u8 *data; -+ -+ sta = (struct ieee80211_sta *)tx_ctrl->sta; -+ sta_info = mwl_dev_get_sta(sta); -+ -+ if (!sta_info->is_amsdu_allowed) -+ return tx_skb; -+ -+ wh = (struct ieee80211_hdr *)tx_skb->data; -+ if (sta_info->is_mesh_node && is_multicast_ether_addr(wh->addr3)) -+ return tx_skb; -+ -+ if (ieee80211_is_qos_nullfunc(wh->frame_control)) -+ return tx_skb; -+ -+ if (sta_info->amsdu_ctrl.cap == MWL_AMSDU_SIZE_4K) -+ amsdu_allow_size = SYSADPT_AMSDU_4K_MAX_SIZE; -+ else if (sta_info->amsdu_ctrl.cap == MWL_AMSDU_SIZE_8K) -+ amsdu_allow_size = SYSADPT_AMSDU_8K_MAX_SIZE; -+ else -+ return tx_skb; -+ -+ spin_lock_bh(&sta_info->amsdu_lock); -+ amsdu = &sta_info->amsdu_ctrl.frag[desc_num]; -+ -+ if ((tx_skb->len > SYSADPT_AMSDU_ALLOW_SIZE) || -+ utils_is_non_amsdu_packet(tx_skb->data, true)) { -+ if (amsdu->num) { -+ pcie_tx_skb(priv, desc_num, amsdu->skb); -+ amsdu->num = 0; -+ amsdu->cur_pos = NULL; -+ } -+ spin_unlock_bh(&sta_info->amsdu_lock); -+ return tx_skb; -+ } -+ -+ /* potential amsdu size, should add amsdu header 14 bytes + -+ * maximum padding 3. -+ */ -+ wh_len = ieee80211_hdrlen(wh->frame_control); -+ len = tx_skb->len - wh_len + 17; -+ -+ if (amsdu->num) { -+ if ((amsdu->skb->len + len) > amsdu_allow_size) { -+ pcie_tx_skb(priv, desc_num, amsdu->skb); -+ amsdu->num = 0; -+ amsdu->cur_pos = NULL; -+ } -+ } -+ -+ amsdu->jiffies = jiffies; -+ len = tx_skb->len - wh_len; -+ -+ if (amsdu->num == 0) { -+ struct sk_buff *newskb; -+ int headroom; -+ -+ amsdu_pkts = (struct sk_buff_head *) -+ kmalloc(sizeof(*amsdu_pkts), GFP_ATOMIC); -+ if (!amsdu_pkts) { -+ spin_unlock_bh(&sta_info->amsdu_lock); -+ return tx_skb; -+ } -+ newskb = dev_alloc_skb(amsdu_allow_size + -+ pcie_priv->tx_head_room); -+ if (!newskb) { -+ spin_unlock_bh(&sta_info->amsdu_lock); -+ kfree(amsdu_pkts); -+ return tx_skb; -+ } -+ -+ headroom = skb_headroom(newskb); -+ if (headroom < pcie_priv->tx_head_room) -+ skb_reserve(newskb, -+ (pcie_priv->tx_head_room - headroom)); -+ -+ data = newskb->data; -+ memcpy(data, tx_skb->data, wh_len); -+ if (sta_info->is_mesh_node) { -+ ether_addr_copy(data + wh_len, wh->addr3); -+ ether_addr_copy(data + wh_len + ETH_ALEN, wh->addr4); -+ } else { -+ ether_addr_copy(data + wh_len, -+ ieee80211_get_DA(wh)); -+ ether_addr_copy(data + wh_len + ETH_ALEN, -+ ieee80211_get_SA(wh)); -+ } -+ *(u8 *)(data + wh_len + ETH_HLEN - 1) = len & 0xff; -+ *(u8 *)(data + wh_len + ETH_HLEN - 2) = (len >> 8) & 0xff; -+ memcpy(data + wh_len + ETH_HLEN, tx_skb->data + wh_len, len); -+ -+ skb_put(newskb, tx_skb->len + ETH_HLEN); -+ tx_ctrl->qos_ctrl |= IEEE80211_QOS_CTL_A_MSDU_PRESENT; -+ amsdu_info = IEEE80211_SKB_CB(newskb); -+ memcpy(amsdu_info, tx_info, sizeof(*tx_info)); -+ skb_queue_head_init(amsdu_pkts); -+ ((struct pcie_tx_ctrl *)&amsdu_info->status)->amsdu_pkts = -+ (void *)amsdu_pkts; -+ amsdu->skb = newskb; -+ } else { -+ amsdu->cur_pos += amsdu->pad; -+ data = amsdu->cur_pos; -+ -+ if (sta_info->is_mesh_node) { -+ ether_addr_copy(data, wh->addr3); -+ ether_addr_copy(data + ETH_ALEN, wh->addr4); -+ } else { -+ ether_addr_copy(data, ieee80211_get_DA(wh)); -+ ether_addr_copy(data + ETH_ALEN, ieee80211_get_SA(wh)); -+ } -+ *(u8 *)(data + ETH_HLEN - 1) = len & 0xff; -+ *(u8 *)(data + ETH_HLEN - 2) = (len >> 8) & 0xff; -+ memcpy(data + ETH_HLEN, tx_skb->data + wh_len, len); -+ -+ skb_put(amsdu->skb, len + ETH_HLEN + amsdu->pad); -+ amsdu_info = IEEE80211_SKB_CB(amsdu->skb); -+ amsdu_pkts = (struct sk_buff_head *) -+ ((struct pcie_tx_ctrl *) -+ &amsdu_info->status)->amsdu_pkts; -+ } -+ -+ amsdu->num++; -+ amsdu->pad = ((len + ETH_HLEN) % 4) ? (4 - (len + ETH_HLEN) % 4) : 0; -+ amsdu->cur_pos = amsdu->skb->data + amsdu->skb->len; -+ skb_queue_tail(amsdu_pkts, tx_skb); -+ -+ if (amsdu->num > SYSADPT_AMSDU_PACKET_THRESHOLD) { -+ amsdu->num = 0; -+ amsdu->cur_pos = NULL; -+ spin_unlock_bh(&sta_info->amsdu_lock); -+ return amsdu->skb; -+ } -+ -+ spin_unlock_bh(&sta_info->amsdu_lock); -+ return NULL; -+} -+ -+static inline void pcie_tx_ack_amsdu_pkts(struct ieee80211_hw *hw, u32 rate, -+ struct sk_buff_head *amsdu_pkts) -+{ -+ struct sk_buff *amsdu_pkt; -+ struct ieee80211_tx_info *info; -+ -+ while (skb_queue_len(amsdu_pkts) > 0) { -+ amsdu_pkt = skb_dequeue(amsdu_pkts); -+ info = IEEE80211_SKB_CB(amsdu_pkt); -+ pcie_tx_prepare_info(hw->priv, rate, info); -+ ieee80211_tx_status(hw, amsdu_pkt); -+ } -+ -+ kfree(amsdu_pkts); -+} -+ -+static void pcie_pfu_tx_done(struct mwl_priv *priv) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ u32 wrdoneidx, rdptr; -+ const u32 num_tx_buffs = PCIE_MAX_TXRX_BD << PCIE_TX_START_PTR; -+ struct pcie_data_buf *data_buf; -+ struct sk_buff *done_skb; -+ struct pcie_pfu_dma_data *pfu_dma; -+ struct pcie_tx_desc *tx_desc; -+ struct pcie_dma_data *dma_data; -+ struct ieee80211_hdr *wh; -+ struct ieee80211_tx_info *info; -+ struct pcie_tx_ctrl *tx_ctrl; -+ struct ieee80211_sta *sta; -+ struct mwl_sta *sta_info; -+ u32 rate = 0; -+ struct sk_buff_head *amsdu_pkts; -+ int hdrlen; -+ -+ spin_lock_bh(&pcie_priv->tx_desc_lock); -+ /* Read the TX ring read pointer set by firmware */ -+ rdptr = readl(pcie_priv->iobase1 + REG_TXBD_RDPTR); -+ /* free from previous txbd_rdptr to current txbd_rdptr */ -+ while (((pcie_priv->txbd_rdptr & PCIE_TXBD_MASK) != -+ (rdptr & PCIE_TXBD_MASK)) || -+ ((pcie_priv->txbd_rdptr & PCIE_BD_FLAG_TX_ROLLOVER_IND) != -+ (rdptr & PCIE_BD_FLAG_TX_ROLLOVER_IND))) { -+ wrdoneidx = pcie_priv->txbd_rdptr & PCIE_TXBD_MASK; -+ wrdoneidx >>= PCIE_TX_START_PTR; -+ -+ data_buf = pcie_priv->txbd_ring[wrdoneidx]; -+ done_skb = pcie_priv->tx_buf_list[wrdoneidx]; -+ if (done_skb) { -+ pfu_dma = (struct pcie_pfu_dma_data *)done_skb->data; -+ tx_desc = &pfu_dma->tx_desc; -+ dma_data = &pfu_dma->dma_data; -+ pci_unmap_single(pcie_priv->pdev, -+ le32_to_cpu(data_buf->paddr), -+ le16_to_cpu(data_buf->len), -+ PCI_DMA_TODEVICE); -+ tx_desc->pkt_ptr = 0; -+ tx_desc->pkt_len = 0; -+ tx_desc->status = cpu_to_le32(EAGLE_TXD_STATUS_IDLE); -+ wmb(); /* memory barrier */ -+ -+ wh = &dma_data->wh; -+ if (ieee80211_is_nullfunc(wh->frame_control) || -+ ieee80211_is_qos_nullfunc(wh->frame_control)) { -+ dev_kfree_skb_any(done_skb); -+ done_skb = NULL; -+ goto next; -+ } -+ -+ info = IEEE80211_SKB_CB(done_skb); -+ tx_ctrl = (struct pcie_tx_ctrl *)&info->status; -+ sta = (struct ieee80211_sta *)tx_ctrl->sta; -+ if (sta) { -+ sta_info = mwl_dev_get_sta(sta); -+ rate = sta_info->tx_rate_info; -+ } -+ -+ if (ieee80211_is_data(wh->frame_control) || -+ ieee80211_is_data_qos(wh->frame_control)) { -+ amsdu_pkts = (struct sk_buff_head *) -+ tx_ctrl->amsdu_pkts; -+ if (amsdu_pkts) { -+ pcie_tx_ack_amsdu_pkts(priv->hw, rate, -+ amsdu_pkts); -+ dev_kfree_skb_any(done_skb); -+ done_skb = NULL; -+ } else { -+ pcie_tx_prepare_info(priv, rate, info); -+ } -+ } else { -+ pcie_tx_prepare_info(priv, 0, info); -+ } -+ -+ if (done_skb) { -+ /* Remove H/W dma header */ -+ hdrlen = ieee80211_hdrlen( -+ dma_data->wh.frame_control); -+ memmove(dma_data->data - hdrlen, -+ &dma_data->wh, hdrlen); -+ skb_pull(done_skb, sizeof(*pfu_dma) - hdrlen); -+ ieee80211_tx_status(priv->hw, done_skb); -+ } -+ } -+next: -+ memset(data_buf, 0, sizeof(*data_buf)); -+ pcie_priv->tx_buf_list[wrdoneidx] = NULL; -+ -+ pcie_priv->txbd_rdptr += PCIE_BD_FLAG_TX_START_PTR; -+ if ((pcie_priv->txbd_rdptr & PCIE_TXBD_MASK) == num_tx_buffs) -+ pcie_priv->txbd_rdptr = ((pcie_priv->txbd_rdptr & -+ PCIE_BD_FLAG_TX_ROLLOVER_IND) ^ -+ PCIE_BD_FLAG_TX_ROLLOVER_IND); -+ } -+ spin_unlock_bh(&pcie_priv->tx_desc_lock); -+ -+ if (pcie_priv->is_tx_done_schedule) { -+ pcie_mask_int(pcie_priv, MACREG_A2HRIC_BIT_TX_DONE, true); -+ tasklet_schedule(&pcie_priv->tx_task); -+ pcie_priv->is_tx_done_schedule = false; -+ } -+} -+ -+static void pcie_non_pfu_tx_done(struct mwl_priv *priv) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ int num; -+ struct pcie_desc_data *desc; -+ struct pcie_tx_hndl *tx_hndl; -+ struct pcie_tx_desc *tx_desc; -+ struct sk_buff *done_skb; -+ int idx; -+ u32 rate; -+ struct pcie_dma_data *dma_data; -+ struct ieee80211_hdr *wh; -+ struct ieee80211_tx_info *info; -+ struct pcie_tx_ctrl *tx_ctrl; -+ struct sk_buff_head *amsdu_pkts; -+ int hdrlen; -+ -+ spin_lock_bh(&pcie_priv->tx_desc_lock); -+ for (num = 0; num < SYSADPT_TX_WMM_QUEUES; num++) { -+ desc = &pcie_priv->desc_data[num]; -+ tx_hndl = desc->pstale_tx_hndl; -+ tx_desc = tx_hndl->pdesc; -+ -+ if ((tx_desc->status & -+ cpu_to_le32(EAGLE_TXD_STATUS_FW_OWNED)) && -+ (tx_hndl->pnext->pdesc->status & -+ cpu_to_le32(EAGLE_TXD_STATUS_OK))) -+ tx_desc->status = cpu_to_le32(EAGLE_TXD_STATUS_OK); -+ -+ while (tx_hndl && -+ (tx_desc->status & cpu_to_le32(EAGLE_TXD_STATUS_OK)) && -+ (!(tx_desc->status & -+ cpu_to_le32(EAGLE_TXD_STATUS_FW_OWNED)))) { -+ pci_unmap_single(pcie_priv->pdev, -+ le32_to_cpu(tx_desc->pkt_ptr), -+ le16_to_cpu(tx_desc->pkt_len), -+ PCI_DMA_TODEVICE); -+ done_skb = tx_hndl->psk_buff; -+ rate = le32_to_cpu(tx_desc->rate_info); -+ tx_desc->pkt_ptr = 0; -+ tx_desc->pkt_len = 0; -+ tx_desc->status = -+ cpu_to_le32(EAGLE_TXD_STATUS_IDLE); -+ tx_hndl->psk_buff = NULL; -+ wmb(); /*Data Memory Barrier*/ -+ -+ skb_get(done_skb); -+ idx = pcie_priv->delay_q_idx; -+ if (pcie_priv->delay_q[idx]) -+ dev_kfree_skb_any(pcie_priv->delay_q[idx]); -+ pcie_priv->delay_q[idx] = done_skb; -+ idx++; -+ if (idx >= PCIE_DELAY_FREE_Q_LIMIT) -+ idx = 0; -+ pcie_priv->delay_q_idx = idx; -+ -+ dma_data = (struct pcie_dma_data *)done_skb->data; -+ wh = &dma_data->wh; -+ if (ieee80211_is_nullfunc(wh->frame_control) || -+ ieee80211_is_qos_nullfunc(wh->frame_control)) { -+ dev_kfree_skb_any(done_skb); -+ done_skb = NULL; -+ goto next; -+ } -+ -+ info = IEEE80211_SKB_CB(done_skb); -+ if (ieee80211_is_data(wh->frame_control) || -+ ieee80211_is_data_qos(wh->frame_control)) { -+ tx_ctrl = (struct pcie_tx_ctrl *)&info->status; -+ amsdu_pkts = (struct sk_buff_head *) -+ tx_ctrl->amsdu_pkts; -+ if (amsdu_pkts) { -+ pcie_tx_ack_amsdu_pkts(priv->hw, rate, -+ amsdu_pkts); -+ dev_kfree_skb_any(done_skb); -+ done_skb = NULL; -+ } else { -+ pcie_tx_prepare_info(priv, rate, info); -+ } -+ } else { -+ pcie_tx_prepare_info(priv, 0, info); -+ } -+ -+ if (done_skb) { -+ /* Remove H/W dma header */ -+ hdrlen = ieee80211_hdrlen( -+ dma_data->wh.frame_control); -+ memmove(dma_data->data - hdrlen, -+ &dma_data->wh, hdrlen); -+ skb_pull(done_skb, sizeof(*dma_data) - hdrlen); -+ ieee80211_tx_status(priv->hw, done_skb); -+ } -+next: -+ tx_hndl = tx_hndl->pnext; -+ tx_desc = tx_hndl->pdesc; -+ pcie_priv->fw_desc_cnt[num]--; -+ } -+ -+ desc->pstale_tx_hndl = tx_hndl; -+ } -+ spin_unlock_bh(&pcie_priv->tx_desc_lock); -+ -+ if (pcie_priv->is_tx_done_schedule) { -+ pcie_mask_int(pcie_priv, MACREG_A2HRIC_BIT_TX_DONE, true); -+ tasklet_schedule(&pcie_priv->tx_task); -+ pcie_priv->is_tx_done_schedule = false; -+ } -+} -+ -+int pcie_tx_init(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ int rc; -+ int i; -+ -+ if (priv->chip_type == MWL8997) -+ rc = pcie_txbd_ring_create(priv); -+ else -+ rc = pcie_tx_ring_alloc(priv); -+ -+ if (rc) { -+ wiphy_err(hw->wiphy, "allocating TX ring failed\n"); -+ return rc; -+ } -+ -+ rc = pcie_tx_ring_init(priv); -+ if (rc) { -+ pcie_tx_ring_free(priv); -+ wiphy_err(hw->wiphy, "initializing TX ring failed\n"); -+ return rc; -+ } -+ -+ pcie_priv->delay_q_idx = 0; -+ for (i = 0; i < PCIE_DELAY_FREE_Q_LIMIT; i++) -+ pcie_priv->delay_q[i] = NULL; -+ -+ return 0; -+} -+ -+void pcie_tx_deinit(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ int i; -+ -+ for (i = 0; i < PCIE_DELAY_FREE_Q_LIMIT; i++) -+ if (pcie_priv->delay_q[i]) -+ dev_kfree_skb_any(pcie_priv->delay_q[i]); -+ -+ pcie_tx_ring_cleanup(priv); -+ -+ if (priv->chip_type == MWL8997) -+ pcie_txbd_ring_delete(priv); -+ else -+ pcie_tx_ring_free(priv); -+} -+ -+void pcie_tx_skbs(unsigned long data) -+{ -+ struct ieee80211_hw *hw = (struct ieee80211_hw *)data; -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ int num = SYSADPT_TX_WMM_QUEUES; -+ struct sk_buff *tx_skb; -+ -+ spin_lock_bh(&pcie_priv->tx_desc_lock); -+ while (num--) { -+ while (skb_queue_len(&pcie_priv->txq[num]) > 0) { -+ struct ieee80211_tx_info *tx_info; -+ struct pcie_tx_ctrl *tx_ctrl; -+ -+ if (!pcie_tx_available(priv, num)) -+ break; -+ -+ tx_skb = skb_dequeue(&pcie_priv->txq[num]); -+ if (!tx_skb) -+ continue; -+ tx_info = IEEE80211_SKB_CB(tx_skb); -+ tx_ctrl = (struct pcie_tx_ctrl *)&tx_info->status; -+ -+ if (tx_ctrl->tx_priority >= SYSADPT_TX_WMM_QUEUES) -+ tx_skb = pcie_tx_do_amsdu(priv, num, -+ tx_skb, tx_info); -+ -+ if (tx_skb) { -+ if (pcie_tx_available(priv, num)) -+ pcie_tx_skb(priv, num, tx_skb); -+ else -+ skb_queue_head(&pcie_priv->txq[num], -+ tx_skb); -+ } -+ } -+ -+ if (skb_queue_len(&pcie_priv->txq[num]) < -+ pcie_priv->txq_wake_threshold) { -+ int queue; -+ -+ queue = SYSADPT_TX_WMM_QUEUES - num - 1; -+ if (ieee80211_queue_stopped(hw, queue)) -+ ieee80211_wake_queue(hw, queue); -+ } -+ } -+ spin_unlock_bh(&pcie_priv->tx_desc_lock); -+} -+ -+void pcie_tx_flush_amsdu(unsigned long data) -+{ -+ struct ieee80211_hw *hw = (struct ieee80211_hw *)data; -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct mwl_sta *sta_info; -+ int i; -+ struct mwl_amsdu_frag *amsdu_frag; -+ -+ spin_lock(&priv->sta_lock); -+ list_for_each_entry(sta_info, &priv->sta_list, list) { -+ spin_lock(&pcie_priv->tx_desc_lock); -+ spin_lock(&sta_info->amsdu_lock); -+ for (i = 0; i < SYSADPT_TX_WMM_QUEUES; i++) { -+ amsdu_frag = &sta_info->amsdu_ctrl.frag[i]; -+ if (amsdu_frag->num) { -+ if (time_after(jiffies, -+ (amsdu_frag->jiffies + 1))) { -+ if (pcie_tx_available(priv, i)) { -+ pcie_tx_skb(priv, i, -+ amsdu_frag->skb); -+ amsdu_frag->num = 0; -+ amsdu_frag->cur_pos = NULL; -+ } -+ } -+ } -+ } -+ spin_unlock(&sta_info->amsdu_lock); -+ spin_unlock(&pcie_priv->tx_desc_lock); -+ } -+ spin_unlock(&priv->sta_lock); -+ -+ pcie_mask_int(pcie_priv, MACREG_A2HRIC_BIT_QUE_EMPTY, true); -+ pcie_priv->is_qe_schedule = false; -+} -+ -+void pcie_tx_done(unsigned long data) -+{ -+ struct ieee80211_hw *hw = (struct ieee80211_hw *)data; -+ struct mwl_priv *priv = hw->priv; -+ -+ if (priv->chip_type == MWL8997) -+ pcie_pfu_tx_done(priv); -+ else -+ pcie_non_pfu_tx_done(priv); -+} -+ -+void pcie_tx_xmit(struct ieee80211_hw *hw, -+ struct ieee80211_tx_control *control, -+ struct sk_buff *skb) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ int index; -+ struct ieee80211_sta *sta; -+ struct ieee80211_tx_info *tx_info; -+ struct mwl_vif *mwl_vif; -+ struct ieee80211_hdr *wh; -+ u8 xmitcontrol; -+ u16 qos; -+ int txpriority; -+ u8 tid = 0; -+ struct mwl_ampdu_stream *stream = NULL; -+ bool start_ba_session = false; -+ bool mgmtframe = false; -+ struct ieee80211_mgmt *mgmt; -+ bool eapol_frame = false; -+ struct pcie_tx_ctrl *tx_ctrl; -+ struct ieee80211_key_conf *k_conf = NULL; -+ int rc; -+ -+ index = skb_get_queue_mapping(skb); -+ sta = control->sta; -+ -+ wh = (struct ieee80211_hdr *)skb->data; -+ tx_info = IEEE80211_SKB_CB(skb); -+ mwl_vif = mwl_dev_get_vif(tx_info->control.vif); -+ -+ if (ieee80211_is_data_qos(wh->frame_control)) -+ qos = le16_to_cpu(*((__le16 *)ieee80211_get_qos_ctl(wh))); -+ else -+ qos = 0; -+ -+ if (ieee80211_is_mgmt(wh->frame_control)) { -+ mgmtframe = true; -+ mgmt = (struct ieee80211_mgmt *)skb->data; -+ } else { -+ u16 pkt_type; -+ struct mwl_sta *sta_info; -+ -+ pkt_type = be16_to_cpu(*((__be16 *) -+ &skb->data[ieee80211_hdrlen(wh->frame_control) + 6])); -+ if (pkt_type == ETH_P_PAE) { -+ index = IEEE80211_AC_VO; -+ eapol_frame = true; -+ } -+ if (sta) { -+ if (mwl_vif->is_hw_crypto_enabled) { -+ sta_info = mwl_dev_get_sta(sta); -+ if (!sta_info->is_key_set && !eapol_frame) { -+ dev_kfree_skb_any(skb); -+ return; -+ } -+ } -+ } -+ } -+ -+ if (tx_info->flags & IEEE80211_TX_CTL_ASSIGN_SEQ) { -+ wh->seq_ctrl &= cpu_to_le16(IEEE80211_SCTL_FRAG); -+ wh->seq_ctrl |= cpu_to_le16(mwl_vif->seqno); -+ mwl_vif->seqno += 0x10; -+ } -+ -+ /* Setup firmware control bit fields for each frame type. */ -+ xmitcontrol = 0; -+ -+ if (mgmtframe || ieee80211_is_ctl(wh->frame_control)) { -+ qos = 0; -+ } else if (ieee80211_is_data(wh->frame_control)) { -+ qos &= ~MWL_QOS_ACK_POLICY_MASK; -+ -+ if (tx_info->flags & IEEE80211_TX_CTL_AMPDU) { -+ xmitcontrol &= 0xfb; -+ qos |= MWL_QOS_ACK_POLICY_BLOCKACK; -+ } else { -+ xmitcontrol |= 0x4; -+ qos |= MWL_QOS_ACK_POLICY_NORMAL; -+ } -+ -+ if (is_multicast_ether_addr(wh->addr1) || eapol_frame) -+ xmitcontrol |= EAGLE_TXD_XMITCTRL_USE_MC_RATE; -+ } -+ -+ k_conf = tx_info->control.hw_key; -+ -+ /* Queue ADDBA request in the respective data queue. While setting up -+ * the ampdu stream, mac80211 queues further packets for that -+ * particular ra/tid pair. However, packets piled up in the hardware -+ * for that ra/tid pair will still go out. ADDBA request and the -+ * related data packets going out from different queues asynchronously -+ * will cause a shift in the receiver window which might result in -+ * ampdu packets getting dropped at the receiver after the stream has -+ * been setup. -+ */ -+ if (mgmtframe) { -+ u16 capab; -+ -+ if (unlikely(ieee80211_is_action(wh->frame_control) && -+ mgmt->u.action.category == WLAN_CATEGORY_BACK && -+ mgmt->u.action.u.addba_req.action_code == -+ WLAN_ACTION_ADDBA_REQ)) { -+ capab = le16_to_cpu(mgmt->u.action.u.addba_req.capab); -+ tid = (capab & IEEE80211_ADDBA_PARAM_TID_MASK) >> 2; -+ index = utils_tid_to_ac(tid); -+ } -+ -+ if (unlikely(ieee80211_is_assoc_req(wh->frame_control))) -+ utils_add_basic_rates(hw->conf.chandef.chan->band, skb); -+ } -+ -+ index = SYSADPT_TX_WMM_QUEUES - index - 1; -+ txpriority = index; -+ -+ if (sta && sta->ht_cap.ht_supported && !eapol_frame && -+ ieee80211_is_data_qos(wh->frame_control)) { -+ tid = qos & 0xf; -+ pcie_tx_count_packet(sta, tid); -+ -+ spin_lock_bh(&priv->stream_lock); -+ stream = mwl_fwcmd_lookup_stream(hw, sta, tid); -+ -+ if (stream) { -+ if (stream->state == AMPDU_STREAM_ACTIVE) { -+ if (WARN_ON(!(qos & -+ MWL_QOS_ACK_POLICY_BLOCKACK))) { -+ spin_unlock_bh(&priv->stream_lock); -+ dev_kfree_skb_any(skb); -+ return; -+ } -+ -+ txpriority = -+ (SYSADPT_TX_WMM_QUEUES + stream->idx) % -+ TOTAL_HW_QUEUES; -+ } else if (stream->state == AMPDU_STREAM_NEW) { -+ /* We get here if the driver sends us packets -+ * after we've initiated a stream, but before -+ * our ampdu_action routine has been called -+ * with IEEE80211_AMPDU_TX_START to get the SSN -+ * for the ADDBA request. So this packet can -+ * go out with no risk of sequence number -+ * mismatch. No special handling is required. -+ */ -+ } else { -+ /* Drop packets that would go out after the -+ * ADDBA request was sent but before the ADDBA -+ * response is received. If we don't do this, -+ * the recipient would probably receive it -+ * after the ADDBA request with SSN 0. This -+ * will cause the recipient's BA receive window -+ * to shift, which would cause the subsequent -+ * packets in the BA stream to be discarded. -+ * mac80211 queues our packets for us in this -+ * case, so this is really just a safety check. -+ */ -+ wiphy_warn(hw->wiphy, -+ "can't send packet during ADDBA\n"); -+ spin_unlock_bh(&priv->stream_lock); -+ dev_kfree_skb_any(skb); -+ return; -+ } -+ } else { -+ if (mwl_fwcmd_ampdu_allowed(sta, tid)) { -+ stream = mwl_fwcmd_add_stream(hw, sta, tid); -+ -+ if (stream) -+ start_ba_session = true; -+ } -+ } -+ -+ spin_unlock_bh(&priv->stream_lock); -+ } else { -+ qos &= ~MWL_QOS_ACK_POLICY_MASK; -+ qos |= MWL_QOS_ACK_POLICY_NORMAL; -+ } -+ -+ tx_ctrl = (struct pcie_tx_ctrl *)&tx_info->status; -+ tx_ctrl->vif = (void *)tx_info->control.vif; -+ tx_ctrl->sta = (void *)sta; -+ tx_ctrl->k_conf = (void *)k_conf; -+ tx_ctrl->amsdu_pkts = NULL; -+ tx_ctrl->tx_priority = txpriority; -+ tx_ctrl->type = (mgmtframe ? IEEE_TYPE_MANAGEMENT : IEEE_TYPE_DATA); -+ tx_ctrl->qos_ctrl = qos; -+ tx_ctrl->xmit_control = xmitcontrol; -+ -+ if (skb_queue_len(&pcie_priv->txq[index]) > pcie_priv->txq_limit) -+ ieee80211_stop_queue(hw, SYSADPT_TX_WMM_QUEUES - index - 1); -+ -+ skb_queue_tail(&pcie_priv->txq[index], skb); -+ -+ tasklet_schedule(&pcie_priv->tx_task); -+ -+ /* Initiate the ampdu session here */ -+ if (start_ba_session) { -+ spin_lock_bh(&priv->stream_lock); -+ rc = mwl_fwcmd_start_stream(hw, stream); -+ if (rc) -+ mwl_fwcmd_remove_stream(hw, stream); -+ else -+ wiphy_debug(hw->wiphy, "Mac80211 start BA %pM\n", -+ stream->sta->addr); -+ spin_unlock_bh(&priv->stream_lock); -+ } -+} -+ -+void pcie_tx_del_pkts_via_vif(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ int num; -+ struct sk_buff *skb, *tmp; -+ struct ieee80211_tx_info *tx_info; -+ struct pcie_tx_ctrl *tx_ctrl; -+ struct sk_buff_head *amsdu_pkts; -+ unsigned long flags; -+ -+ for (num = 1; num < PCIE_NUM_OF_DESC_DATA; num++) { -+ spin_lock_irqsave(&pcie_priv->txq[num].lock, flags); -+ skb_queue_walk_safe(&pcie_priv->txq[num], skb, tmp) { -+ tx_info = IEEE80211_SKB_CB(skb); -+ tx_ctrl = (struct pcie_tx_ctrl *)&tx_info->status; -+ if (tx_ctrl->vif == vif) { -+ amsdu_pkts = (struct sk_buff_head *) -+ tx_ctrl->amsdu_pkts; -+ if (amsdu_pkts) { -+ skb_queue_purge(amsdu_pkts); -+ kfree(amsdu_pkts); -+ } -+ __skb_unlink(skb, &pcie_priv->txq[num]); -+ dev_kfree_skb_any(skb); -+ } -+ } -+ spin_unlock_irqrestore(&pcie_priv->txq[num].lock, flags); -+ } -+} -+ -+void pcie_tx_del_pkts_via_sta(struct ieee80211_hw *hw, -+ struct ieee80211_sta *sta) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ int num; -+ struct sk_buff *skb, *tmp; -+ struct ieee80211_tx_info *tx_info; -+ struct pcie_tx_ctrl *tx_ctrl; -+ struct sk_buff_head *amsdu_pkts; -+ unsigned long flags; -+ -+ for (num = 1; num < PCIE_NUM_OF_DESC_DATA; num++) { -+ spin_lock_irqsave(&pcie_priv->txq[num].lock, flags); -+ skb_queue_walk_safe(&pcie_priv->txq[num], skb, tmp) { -+ tx_info = IEEE80211_SKB_CB(skb); -+ tx_ctrl = (struct pcie_tx_ctrl *)&tx_info->status; -+ if (tx_ctrl->sta == sta) { -+ amsdu_pkts = (struct sk_buff_head *) -+ tx_ctrl->amsdu_pkts; -+ if (amsdu_pkts) { -+ skb_queue_purge(amsdu_pkts); -+ kfree(amsdu_pkts); -+ } -+ __skb_unlink(skb, &pcie_priv->txq[num]); -+ dev_kfree_skb_any(skb); -+ } -+ } -+ spin_unlock_irqrestore(&pcie_priv->txq[num].lock, flags); -+ } -+} -+ -+void pcie_tx_del_ampdu_pkts(struct ieee80211_hw *hw, -+ struct ieee80211_sta *sta, u8 tid) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct mwl_sta *sta_info = mwl_dev_get_sta(sta); -+ int ac, desc_num; -+ struct mwl_amsdu_frag *amsdu_frag; -+ struct sk_buff *skb, *tmp; -+ struct ieee80211_tx_info *tx_info; -+ struct pcie_tx_ctrl *tx_ctrl; -+ struct sk_buff_head *amsdu_pkts; -+ unsigned long flags; -+ -+ ac = utils_tid_to_ac(tid); -+ desc_num = SYSADPT_TX_WMM_QUEUES - ac - 1; -+ spin_lock_irqsave(&pcie_priv->txq[desc_num].lock, flags); -+ skb_queue_walk_safe(&pcie_priv->txq[desc_num], skb, tmp) { -+ tx_info = IEEE80211_SKB_CB(skb); -+ tx_ctrl = (struct pcie_tx_ctrl *)&tx_info->status; -+ if (tx_ctrl->sta == sta) { -+ amsdu_pkts = (struct sk_buff_head *) -+ tx_ctrl->amsdu_pkts; -+ if (amsdu_pkts) { -+ skb_queue_purge(amsdu_pkts); -+ kfree(amsdu_pkts); -+ } -+ __skb_unlink(skb, &pcie_priv->txq[desc_num]); -+ dev_kfree_skb_any(skb); -+ } -+ } -+ spin_unlock_irqrestore(&pcie_priv->txq[desc_num].lock, flags); -+ -+ spin_lock_bh(&sta_info->amsdu_lock); -+ amsdu_frag = &sta_info->amsdu_ctrl.frag[desc_num]; -+ if (amsdu_frag->num) { -+ amsdu_frag->num = 0; -+ amsdu_frag->cur_pos = NULL; -+ if (amsdu_frag->skb) { -+ tx_info = IEEE80211_SKB_CB(amsdu_frag->skb); -+ tx_ctrl = (struct pcie_tx_ctrl *)&tx_info->status; -+ amsdu_pkts = (struct sk_buff_head *) -+ tx_ctrl->amsdu_pkts; -+ if (amsdu_pkts) { -+ skb_queue_purge(amsdu_pkts); -+ kfree(amsdu_pkts); -+ } -+ dev_kfree_skb_any(amsdu_frag->skb); -+ } -+ } -+ spin_unlock_bh(&sta_info->amsdu_lock); -+} -+ -+void pcie_tx_del_sta_amsdu_pkts(struct ieee80211_hw *hw, -+ struct ieee80211_sta *sta) -+{ -+ struct mwl_sta *sta_info = mwl_dev_get_sta(sta); -+ int num; -+ struct mwl_amsdu_frag *amsdu_frag; -+ struct ieee80211_tx_info *tx_info; -+ struct pcie_tx_ctrl *tx_ctrl; -+ struct sk_buff_head *amsdu_pkts; -+ -+ spin_lock_bh(&sta_info->amsdu_lock); -+ for (num = 0; num < SYSADPT_TX_WMM_QUEUES; num++) { -+ amsdu_frag = &sta_info->amsdu_ctrl.frag[num]; -+ if (amsdu_frag->num) { -+ amsdu_frag->num = 0; -+ amsdu_frag->cur_pos = NULL; -+ if (amsdu_frag->skb) { -+ tx_info = IEEE80211_SKB_CB(amsdu_frag->skb); -+ tx_ctrl = (struct pcie_tx_ctrl *) -+ &tx_info->status; -+ amsdu_pkts = (struct sk_buff_head *) -+ tx_ctrl->amsdu_pkts; -+ if (amsdu_pkts) { -+ skb_queue_purge(amsdu_pkts); -+ kfree(amsdu_pkts); -+ } -+ dev_kfree_skb_any(amsdu_frag->skb); -+ } -+ } -+ } -+ spin_unlock_bh(&sta_info->amsdu_lock); -+} -diff --git a/drivers/net/wireless/marvell/mwlwifi/hif/pcie/tx.h b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/tx.h -new file mode 100644 -index 000000000000..c233ba1aaa9d ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/tx.h -@@ -0,0 +1,38 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file defines transmit related functions. */ -+ -+#ifndef _TX_H_ -+#define _TX_H_ -+ -+int pcie_tx_init(struct ieee80211_hw *hw); -+void pcie_tx_deinit(struct ieee80211_hw *hw); -+void pcie_tx_skbs(unsigned long data); -+void pcie_tx_done(unsigned long data); -+void pcie_tx_flush_amsdu(unsigned long data); -+void pcie_tx_xmit(struct ieee80211_hw *hw, -+ struct ieee80211_tx_control *control, -+ struct sk_buff *skb); -+void pcie_tx_del_pkts_via_vif(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif); -+void pcie_tx_del_pkts_via_sta(struct ieee80211_hw *hw, -+ struct ieee80211_sta *sta); -+void pcie_tx_del_ampdu_pkts(struct ieee80211_hw *hw, -+ struct ieee80211_sta *sta, u8 tid); -+void pcie_tx_del_sta_amsdu_pkts(struct ieee80211_hw *hw, -+ struct ieee80211_sta *sta); -+ -+#endif /* _TX_H_ */ -diff --git a/drivers/net/wireless/marvell/mwlwifi/hif/pcie/tx_ndp.c b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/tx_ndp.c -new file mode 100644 -index 000000000000..6758cde363c6 ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/tx_ndp.c -@@ -0,0 +1,693 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file implements transmit related functions for new data -+ * path. -+ */ -+ -+#include -+#include -+ -+#include "sysadpt.h" -+#include "core.h" -+#include "utils.h" -+#include "hif/fwcmd.h" -+#include "hif/pcie/dev.h" -+#include "hif/pcie/tx_ndp.h" -+ -+#define MAX_NUM_TX_RING_BYTES (MAX_NUM_TX_DESC * \ -+ sizeof(struct pcie_tx_desc_ndp)) -+#define MAX_NUM_TX_RING_DONE_BYTES (MAX_NUM_TX_DESC * \ -+ sizeof(struct tx_ring_done)) -+#define QUEUE_STAOFFSET ((SYSADPT_NUM_OF_AP - 1) + \ -+ SYSADPT_NUM_OF_CLIENT) -+#define PROBE_RESPONSE_TXQNUM ((SYSADPT_MAX_STA_SC4 + SYSADPT_NUM_OF_AP + \ -+ SYSADPT_NUM_OF_CLIENT) * SYSADPT_MAX_TID) -+#define MGMT_TXQNUM ((PROBE_RESPONSE_TXQNUM + 1)) -+#define TXDONE_THRESHOLD 4 -+ -+#define TX_CTRL_TYPE_DATA BIT(0) -+#define TX_CTRL_EAPOL BIT(1) -+#define TX_CTRL_TCP_ACK BIT(2) -+ -+/* Transmission information to transmit a socket buffer. -+ */ -+struct pcie_tx_ctrl_ndp { -+ u16 tx_que_priority; -+ u8 hdrlen; -+ u8 flags; -+ u32 rate; -+ u32 tcp_dst_src; -+ u32 tcp_sn; -+} __packed; -+ -+static int pcie_tx_ring_alloc_ndp(struct mwl_priv *priv) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct pcie_desc_data_ndp *desc = &pcie_priv->desc_data_ndp; -+ u8 *mem; -+ -+ mem = dma_alloc_coherent(priv->dev, -+ MAX_NUM_TX_RING_BYTES, -+ &desc->pphys_tx_ring, -+ GFP_KERNEL); -+ if (!mem) -+ goto err_no_mem; -+ desc->ptx_ring = (struct pcie_tx_desc_ndp *)mem; -+ memset(desc->ptx_ring, 0x00, MAX_NUM_TX_RING_BYTES); -+ -+ mem = dma_alloc_coherent(priv->dev, -+ MAX_NUM_TX_RING_DONE_BYTES, -+ &desc->pphys_tx_ring_done, -+ GFP_KERNEL); -+ if (!mem) -+ goto err_no_mem; -+ desc->ptx_ring_done = (struct tx_ring_done *)mem; -+ memset(desc->ptx_ring_done, 0x00, MAX_NUM_TX_RING_DONE_BYTES); -+ -+ mem = dma_alloc_coherent(priv->dev, -+ DEFAULT_ACNT_RING_SIZE, -+ &desc->pphys_acnt_ring, -+ GFP_KERNEL); -+ if (!mem) -+ goto err_no_mem; -+ desc->pacnt_ring = (u8 *)mem; -+ memset(desc->pacnt_ring, 0x00, DEFAULT_ACNT_RING_SIZE); -+ -+ desc->pacnt_buf = kzalloc(DEFAULT_ACNT_RING_SIZE, GFP_KERNEL); -+ if (!desc->pacnt_buf) -+ goto err_no_mem; -+ desc->acnt_ring_size = DEFAULT_ACNT_RING_SIZE; -+ -+ return 0; -+ -+err_no_mem: -+ -+ wiphy_err(priv->hw->wiphy, "cannot alloc mem\n"); -+ -+ return -ENOMEM; -+} -+ -+static int pcie_tx_ring_init_ndp(struct mwl_priv *priv) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct pcie_desc_data_ndp *desc = &pcie_priv->desc_data_ndp; -+ int i; -+ -+ for (i = 0; i < PCIE_NUM_OF_DESC_DATA; i++) -+ skb_queue_head_init(&pcie_priv->txq[i]); -+ -+ if (!desc->ptx_ring) { -+ for (i = 0; i < MAX_NUM_TX_DESC; i++) -+ desc->ptx_ring[i].user = cpu_to_le32(i); -+ desc->tx_desc_busy_cnt = 0; -+ } -+ -+ return 0; -+} -+ -+static void pcie_tx_ring_cleanup_ndp(struct mwl_priv *priv) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct pcie_desc_data_ndp *desc = &pcie_priv->desc_data_ndp; -+ struct sk_buff *tx_skb; -+ int i; -+ -+ for (i = 0; i < PCIE_NUM_OF_DESC_DATA; i++) -+ skb_queue_purge(&pcie_priv->txq[i]); -+ -+ for (i = 0; i < MAX_TX_RING_SEND_SIZE; i++) { -+ tx_skb = desc->tx_vbuflist[i]; -+ if (tx_skb) { -+ pci_unmap_single(pcie_priv->pdev, -+ desc->pphys_tx_buflist[i], -+ tx_skb->len, -+ PCI_DMA_TODEVICE); -+ dev_kfree_skb_any(tx_skb); -+ desc->pphys_tx_buflist[i] = 0; -+ desc->tx_vbuflist[i] = NULL; -+ } -+ } -+ desc->tx_sent_tail = 0; -+ desc->tx_sent_head = 0; -+ desc->tx_done_tail = 0; -+ desc->tx_vbuflist_idx = 0; -+ desc->tx_desc_busy_cnt = 0; -+} -+ -+static void pcie_tx_ring_free_ndp(struct mwl_priv *priv) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct pcie_desc_data_ndp *desc = &pcie_priv->desc_data_ndp; -+ -+ if (desc->ptx_ring) { -+ dma_free_coherent(priv->dev, -+ MAX_NUM_TX_RING_BYTES, -+ desc->ptx_ring, -+ desc->pphys_tx_ring); -+ desc->ptx_ring = NULL; -+ } -+ -+ if (desc->ptx_ring_done) { -+ dma_free_coherent(priv->dev, -+ MAX_NUM_TX_RING_DONE_BYTES, -+ desc->ptx_ring_done, -+ desc->pphys_tx_ring_done); -+ desc->prx_ring_done = NULL; -+ } -+ -+ if (desc->pacnt_ring) { -+ dma_free_coherent(priv->dev, -+ DEFAULT_ACNT_RING_SIZE, -+ desc->pacnt_ring, -+ desc->pphys_acnt_ring); -+ desc->pacnt_ring = NULL; -+ } -+ -+ kfree(desc->pacnt_buf); -+} -+ -+static inline u32 pcie_tx_set_skb(struct mwl_priv *priv, struct sk_buff *skb, -+ dma_addr_t dma) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct pcie_desc_data_ndp *desc = &pcie_priv->desc_data_ndp; -+ u32 index = desc->tx_vbuflist_idx; -+ -+ while (desc->tx_vbuflist[index]) -+ index = (index + 1) % MAX_TX_RING_SEND_SIZE; -+ -+ desc->tx_vbuflist_idx = (index + 1) % MAX_TX_RING_SEND_SIZE; -+ desc->pphys_tx_buflist[index] = dma; -+ desc->tx_vbuflist[index] = skb; -+ -+ return index; -+} -+ -+static inline int pcie_tx_skb_ndp(struct mwl_priv *priv, -+ struct sk_buff *tx_skb) -+{ -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct pcie_desc_data_ndp *desc = &pcie_priv->desc_data_ndp; -+ u32 tx_send_tail; -+ u32 tx_send_head_new; -+ struct ieee80211_tx_info *tx_info; -+ struct pcie_tx_ctrl_ndp *tx_ctrl; -+ struct pcie_tx_desc_ndp *pnext_tx_desc; -+ struct ieee80211_hdr *wh; -+ u32 ctrl = 0; -+ dma_addr_t dma; -+ -+ spin_lock_bh(&pcie_priv->tx_desc_lock); -+ -+ tx_send_tail = desc->tx_sent_tail; -+ tx_send_head_new = desc->tx_sent_head; -+ -+ if (((tx_send_head_new + 1) & (MAX_NUM_TX_DESC-1)) == tx_send_tail) { -+ /* Update the tx_send_tail */ -+ tx_send_tail = readl(pcie_priv->iobase1 + -+ MACREG_REG_TXSEDNTAIL); -+ desc->tx_sent_tail = tx_send_tail; -+ -+ if (((tx_send_head_new + 1) & (MAX_NUM_TX_DESC-1)) == -+ tx_send_tail) { -+ spin_unlock_bh(&pcie_priv->tx_desc_lock); -+ return -EAGAIN; -+ } -+ } -+ -+ tx_info = IEEE80211_SKB_CB(tx_skb); -+ tx_ctrl = (struct pcie_tx_ctrl_ndp *)tx_info->status.status_driver_data; -+ pnext_tx_desc = &desc->ptx_ring[tx_send_head_new]; -+ -+ if (tx_ctrl->flags & TX_CTRL_TYPE_DATA) { -+ wh = (struct ieee80211_hdr *)tx_skb->data; -+ -+ skb_pull(tx_skb, tx_ctrl->hdrlen); -+ ether_addr_copy(pnext_tx_desc->u.sa, -+ ieee80211_get_SA(wh)); -+ ether_addr_copy(pnext_tx_desc->u.da, -+ ieee80211_get_DA(wh)); -+ -+ if (tx_ctrl->flags & TX_CTRL_EAPOL) -+ ctrl = TXRING_CTRL_TAG_EAP << TXRING_CTRL_TAG_SHIFT; -+ if (tx_ctrl->flags & TX_CTRL_TCP_ACK) { -+ pnext_tx_desc->tcp_dst_src = -+ cpu_to_le32(tx_ctrl->tcp_dst_src); -+ pnext_tx_desc->tcp_sn = cpu_to_le32(tx_ctrl->tcp_sn); -+ ctrl = TXRING_CTRL_TAG_TCP_ACK << TXRING_CTRL_TAG_SHIFT; -+ } -+ ctrl |= (((tx_ctrl->tx_que_priority & TXRING_CTRL_QID_MASK) << -+ TXRING_CTRL_QID_SHIFT) | -+ ((tx_skb->len & TXRING_CTRL_LEN_MASK) << -+ TXRING_CTRL_LEN_SHIFT)); -+ } else { -+ /* Assigning rate code; use legacy 6mbps rate. */ -+ pnext_tx_desc->u.rate_code = cpu_to_le16(RATECODE_TYPE_LEGACY + -+ (0 << RATECODE_MCS_SHIFT) + RATECODE_BW_20MHZ); -+ pnext_tx_desc->u.max_retry = 5; -+ -+ ctrl = (((tx_ctrl->tx_que_priority & TXRING_CTRL_QID_MASK) << -+ TXRING_CTRL_QID_SHIFT) | -+ (((tx_skb->len - sizeof(struct pcie_dma_data)) & -+ TXRING_CTRL_LEN_MASK) << TXRING_CTRL_LEN_SHIFT) | -+ (TXRING_CTRL_TAG_MGMT << TXRING_CTRL_TAG_SHIFT)); -+ } -+ -+ dma = pci_map_single(pcie_priv->pdev, tx_skb->data, -+ tx_skb->len, PCI_DMA_TODEVICE); -+ if (pci_dma_mapping_error(pcie_priv->pdev, dma)) { -+ dev_kfree_skb_any(tx_skb); -+ wiphy_err(priv->hw->wiphy, -+ "failed to map pci memory!\n"); -+ spin_unlock_bh(&pcie_priv->tx_desc_lock); -+ return -EIO; -+ } -+ -+ pnext_tx_desc->data = cpu_to_le32(dma); -+ pnext_tx_desc->ctrl = cpu_to_le32(ctrl); -+ pnext_tx_desc->user = cpu_to_le32(pcie_tx_set_skb(priv, tx_skb, dma)); -+ -+ if ((tx_ctrl->flags & TX_CTRL_TYPE_DATA) && -+ (tx_ctrl->rate != 0)) { -+ skb_push(tx_skb, tx_ctrl->hdrlen); -+ skb_get(tx_skb); -+ pcie_tx_prepare_info(priv, tx_ctrl->rate, tx_info); -+ tx_ctrl->flags |= TX_CTRL_TYPE_DATA; -+ ieee80211_tx_status(priv->hw, tx_skb); -+ } -+ -+ if (++tx_send_head_new >= MAX_NUM_TX_DESC) -+ tx_send_head_new = 0; -+ desc->tx_sent_head = tx_send_head_new; -+ wmb(); /*Data Memory Barrier*/ -+ writel(tx_send_head_new, pcie_priv->iobase1 + MACREG_REG_TXSENDHEAD); -+ desc->tx_desc_busy_cnt++; -+ -+ spin_unlock_bh(&pcie_priv->tx_desc_lock); -+ -+ return 0; -+} -+ -+static inline void pcie_tx_check_tcp_ack(struct sk_buff *tx_skb, -+ struct pcie_tx_ctrl_ndp *tx_ctrl) -+{ -+ struct iphdr *iph; -+ struct tcphdr *tcph; -+ -+ if (tx_ctrl->flags & TX_CTRL_TYPE_DATA) { -+ iph = (struct iphdr *)(tx_skb->data + tx_ctrl->hdrlen + 8); -+ tcph = (struct tcphdr *)((u8 *)iph + (iph->ihl * 4)); -+ if ((iph->protocol == IPPROTO_TCP) && -+ (tx_skb->protocol == htons(ETH_P_IP))) { -+ if ((tcph->ack == 1) && (ntohs(iph->tot_len) == -+ (iph->ihl * 4 + tcph->doff * 4))) { -+ if (tcph->syn || tcph->fin) -+ return; -+ -+ tx_ctrl->flags |= TX_CTRL_TCP_ACK; -+ tx_ctrl->tcp_dst_src = ntohs(tcph->source) | -+ (ntohs(tcph->dest) << 16); -+ tx_ctrl->tcp_sn = ntohl(tcph->ack_seq); -+ } -+ } -+ } -+} -+ -+int pcie_tx_init_ndp(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct sk_buff skb; -+ struct ieee80211_tx_info *tx_info = IEEE80211_SKB_CB(&skb); -+ int rc; -+ -+ if (sizeof(struct pcie_tx_ctrl_ndp) > -+ sizeof(tx_info->status.status_driver_data)) { -+ wiphy_err(hw->wiphy, "driver data is not enough: %d (%d)\n", -+ sizeof(struct pcie_tx_ctrl_ndp), -+ sizeof(tx_info->status.status_driver_data)); -+ return -ENOMEM; -+ } -+ -+ rc = pcie_tx_ring_alloc_ndp(priv); -+ if (rc) { -+ pcie_tx_ring_free_ndp(priv); -+ wiphy_err(hw->wiphy, "allocating TX ring failed\n"); -+ return rc; -+ } -+ -+ rc = pcie_tx_ring_init_ndp(priv); -+ if (rc) { -+ pcie_tx_ring_free_ndp(priv); -+ wiphy_err(hw->wiphy, "initializing TX ring failed\n"); -+ return rc; -+ } -+ -+ return 0; -+} -+ -+void pcie_tx_deinit_ndp(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ pcie_tx_ring_cleanup_ndp(priv); -+ pcie_tx_ring_free_ndp(priv); -+} -+ -+void pcie_tx_skbs_ndp(unsigned long data) -+{ -+ struct ieee80211_hw *hw = (struct ieee80211_hw *)data; -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ int num = SYSADPT_TX_WMM_QUEUES; -+ struct sk_buff *tx_skb; -+ int rc; -+ -+ while (num--) { -+ while (skb_queue_len(&pcie_priv->txq[num]) > 0) { -+ if (pcie_priv->desc_data_ndp.tx_desc_busy_cnt >= -+ (MAX_TX_RING_SEND_SIZE - 1)) { -+ pcie_tx_done_ndp(hw); -+ break; -+ } -+ -+ tx_skb = skb_dequeue(&pcie_priv->txq[num]); -+ -+ rc = pcie_tx_skb_ndp(priv, tx_skb); -+ if (rc) { -+ pcie_tx_done_ndp(hw); -+ if (rc == -EAGAIN) -+ skb_queue_head(&pcie_priv->txq[num], -+ tx_skb); -+ break; -+ } -+ -+ if (++pcie_priv->tx_done_cnt > TXDONE_THRESHOLD) { -+ pcie_tx_done_ndp(hw); -+ pcie_priv->tx_done_cnt = 0; -+ } -+ } -+ -+ if (skb_queue_len(&pcie_priv->txq[num]) < -+ pcie_priv->txq_wake_threshold) { -+ int queue; -+ -+ queue = SYSADPT_TX_WMM_QUEUES - num - 1; -+ if (ieee80211_queue_stopped(hw, queue)) -+ ieee80211_wake_queue(hw, queue); -+ } -+ } -+ -+ pcie_priv->is_tx_schedule = false; -+} -+ -+void pcie_tx_done_ndp(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct pcie_desc_data_ndp *desc = &pcie_priv->desc_data_ndp; -+ u32 tx_done_head, tx_done_tail; -+ struct tx_ring_done *ptx_ring_done; -+ u32 index; -+ struct sk_buff *skb; -+ struct ieee80211_tx_info *tx_info; -+ struct pcie_tx_ctrl_ndp *tx_ctrl; -+ struct pcie_dma_data *dma_data; -+ u16 hdrlen; -+ -+ spin_lock_bh(&pcie_priv->tx_desc_lock); -+ -+ tx_done_head = readl(pcie_priv->iobase1 + -+ MACREG_REG_TXDONEHEAD); -+ tx_done_tail = desc->tx_done_tail & (MAX_TX_RING_DONE_SIZE - 1); -+ tx_done_head &= (MAX_TX_RING_DONE_SIZE - 1); -+ -+ while (tx_done_head != tx_done_tail) { -+ ptx_ring_done = &desc->ptx_ring_done[tx_done_tail]; -+ -+ index = le32_to_cpu(ptx_ring_done->user); -+ ptx_ring_done->user = 0; -+ if (index >= MAX_TX_RING_SEND_SIZE) { -+ wiphy_err(hw->wiphy, -+ "corruption for index of buffer\n"); -+ break; -+ } -+ skb = desc->tx_vbuflist[index]; -+ if (!skb) { -+ wiphy_err(hw->wiphy, -+ "buffer is NULL for tx done ring\n"); -+ break; -+ } -+ pci_unmap_single(pcie_priv->pdev, -+ desc->pphys_tx_buflist[index], -+ skb->len, -+ PCI_DMA_TODEVICE); -+ desc->pphys_tx_buflist[index] = 0; -+ desc->tx_vbuflist[index] = NULL; -+ -+ tx_info = IEEE80211_SKB_CB(skb); -+ tx_ctrl = (struct pcie_tx_ctrl_ndp *) -+ tx_info->status.status_driver_data; -+ -+ if (tx_ctrl->flags & TX_CTRL_TYPE_DATA) { -+ dev_kfree_skb_any(skb); -+ goto bypass_ack; -+ } else { -+ /* Remove H/W dma header */ -+ dma_data = (struct pcie_dma_data *)skb->data; -+ -+ if (ieee80211_is_assoc_resp( -+ dma_data->wh.frame_control) || -+ ieee80211_is_reassoc_resp( -+ dma_data->wh.frame_control)) { -+ dev_kfree_skb_any(skb); -+ goto bypass_ack; -+ } -+ hdrlen = ieee80211_hdrlen( -+ dma_data->wh.frame_control); -+ memmove(dma_data->data - hdrlen, -+ &dma_data->wh, hdrlen); -+ skb_pull(skb, sizeof(*dma_data) - hdrlen); -+ } -+ -+ pcie_tx_prepare_info(priv, 0, tx_info); -+ ieee80211_tx_status(hw, skb); -+ -+bypass_ack: -+ if (++tx_done_tail >= MAX_TX_RING_DONE_SIZE) -+ tx_done_tail = 0; -+ desc->tx_desc_busy_cnt--; -+ } -+ -+ writel(tx_done_tail, pcie_priv->iobase1 + -+ MACREG_REG_TXDONETAIL); -+ desc->tx_done_tail = tx_done_tail; -+ -+ spin_unlock_bh(&pcie_priv->tx_desc_lock); -+} -+ -+void pcie_tx_xmit_ndp(struct ieee80211_hw *hw, -+ struct ieee80211_tx_control *control, -+ struct sk_buff *skb) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct pcie_priv *pcie_priv = priv->hif.priv; -+ struct ieee80211_tx_info *tx_info; -+ struct ieee80211_key_conf *k_conf; -+ struct mwl_vif *mwl_vif; -+ int index; -+ struct ieee80211_sta *sta; -+ struct mwl_sta *sta_info; -+ struct ieee80211_hdr *wh; -+ u8 *da; -+ u16 qos; -+ u8 tid = 0; -+ struct mwl_ampdu_stream *stream = NULL; -+ u16 tx_que_priority; -+ bool mgmtframe = false; -+ struct ieee80211_mgmt *mgmt; -+ bool eapol_frame = false; -+ bool start_ba_session = false; -+ struct pcie_tx_ctrl_ndp *tx_ctrl; -+ -+ tx_info = IEEE80211_SKB_CB(skb); -+ k_conf = tx_info->control.hw_key; -+ mwl_vif = mwl_dev_get_vif(tx_info->control.vif); -+ index = skb_get_queue_mapping(skb); -+ sta = control->sta; -+ sta_info = sta ? mwl_dev_get_sta(sta) : NULL; -+ -+ wh = (struct ieee80211_hdr *)skb->data; -+ -+ if (ieee80211_is_data_qos(wh->frame_control)) -+ qos = le16_to_cpu(*((__le16 *)ieee80211_get_qos_ctl(wh))); -+ else -+ qos = 0xFFFF; -+ -+ if (skb->protocol == cpu_to_be16(ETH_P_PAE)) { -+ index = IEEE80211_AC_VO; -+ eapol_frame = true; -+ } -+ -+ if (ieee80211_is_mgmt(wh->frame_control)) { -+ mgmtframe = true; -+ mgmt = (struct ieee80211_mgmt *)skb->data; -+ } -+ -+ if (mgmtframe) { -+ u16 capab; -+ -+ if (unlikely(ieee80211_is_action(wh->frame_control) && -+ mgmt->u.action.category == WLAN_CATEGORY_BACK && -+ mgmt->u.action.u.addba_req.action_code == -+ WLAN_ACTION_ADDBA_REQ)) { -+ capab = le16_to_cpu(mgmt->u.action.u.addba_req.capab); -+ tid = (capab & IEEE80211_ADDBA_PARAM_TID_MASK) >> 2; -+ index = utils_tid_to_ac(tid); -+ } -+ -+ if (unlikely(ieee80211_is_assoc_req(wh->frame_control))) -+ utils_add_basic_rates(hw->conf.chandef.chan->band, skb); -+ -+ if (ieee80211_is_probe_req(wh->frame_control) || -+ ieee80211_is_probe_resp(wh->frame_control)) -+ tx_que_priority = PROBE_RESPONSE_TXQNUM; -+ else { -+ if (( -+ (mwl_vif->macid == SYSADPT_NUM_OF_AP) && -+ (!ieee80211_has_protected(wh->frame_control) || -+ (ieee80211_has_protected(wh->frame_control) && -+ ieee80211_is_auth(wh->frame_control))) -+ ) || -+ !sta || -+ ieee80211_is_auth(wh->frame_control) || -+ ieee80211_is_assoc_req(wh->frame_control) || -+ ieee80211_is_assoc_resp(wh->frame_control)) -+ tx_que_priority = MGMT_TXQNUM; -+ else { -+ if (is_multicast_ether_addr(wh->addr1) && -+ (mwl_vif->macid != SYSADPT_NUM_OF_AP)) -+ tx_que_priority = mwl_vif->macid * -+ SYSADPT_MAX_TID; -+ else -+ tx_que_priority = SYSADPT_MAX_TID * -+ (sta_info->stnid + -+ QUEUE_STAOFFSET) + 6; -+ } -+ } -+ -+ if (ieee80211_is_assoc_resp(wh->frame_control) || -+ ieee80211_is_reassoc_resp(wh->frame_control)) { -+ struct sk_buff *ack_skb; -+ struct ieee80211_tx_info *ack_info; -+ -+ ack_skb = skb_copy(skb, GFP_ATOMIC); -+ ack_info = IEEE80211_SKB_CB(ack_skb); -+ pcie_tx_prepare_info(priv, 0, ack_info); -+ ieee80211_tx_status(hw, ack_skb); -+ } -+ -+ pcie_tx_encapsulate_frame(priv, skb, k_conf, NULL); -+ } else { -+ tid = qos & 0x7; -+ if (sta && sta->ht_cap.ht_supported && !eapol_frame && -+ qos != 0xFFFF) { -+ pcie_tx_count_packet(sta, tid); -+ spin_lock_bh(&priv->stream_lock); -+ stream = mwl_fwcmd_lookup_stream(hw, sta, tid); -+ if (!stream || -+ stream->state == AMPDU_STREAM_IN_PROGRESS) { -+ wiphy_warn(hw->wiphy, -+ "can't send packet during ADDBA\n"); -+ spin_unlock_bh(&priv->stream_lock); -+ dev_kfree_skb_any(skb); -+ return; -+ } -+ if ((stream->state == AMPDU_NO_STREAM) && -+ mwl_fwcmd_ampdu_allowed(sta, tid)) { -+ stream = mwl_fwcmd_add_stream(hw, sta, tid); -+ if (stream) -+ start_ba_session = true; -+ } -+ spin_unlock_bh(&priv->stream_lock); -+ } -+ -+ da = ieee80211_get_DA(wh); -+ -+ if (is_multicast_ether_addr(da) -+ && (mwl_vif->macid != SYSADPT_NUM_OF_AP)) { -+ -+ tx_que_priority = mwl_vif->macid * SYSADPT_MAX_TID; -+ -+ if (da[ETH_ALEN - 1] == 0xff) -+ tx_que_priority += 7; -+ -+ if (ieee80211_has_a4(wh->frame_control)) { -+ if (sta && sta_info->wds) -+ tx_que_priority = SYSADPT_MAX_TID * -+ (sta_info->stnid + -+ QUEUE_STAOFFSET) + 6; -+ } -+ } else { -+ if (sta) { -+ if (!eapol_frame) -+ tx_que_priority = SYSADPT_MAX_TID * -+ (sta_info->stnid + -+ QUEUE_STAOFFSET) + -+ ((qos == 0xFFFF) ? 0 : tid); -+ else -+ tx_que_priority = SYSADPT_MAX_TID * -+ (sta_info->stnid + -+ QUEUE_STAOFFSET) + -+ ((qos == 0xFFFF) ? 0 : 6); -+ } else -+ tx_que_priority = 0; -+ } -+ } -+ -+ index = SYSADPT_TX_WMM_QUEUES - index - 1; -+ -+ tx_ctrl = (struct pcie_tx_ctrl_ndp *)tx_info->status.status_driver_data; -+ tx_ctrl->tx_que_priority = tx_que_priority; -+ tx_ctrl->hdrlen = ieee80211_hdrlen(wh->frame_control); -+ tx_ctrl->flags = 0; -+ if (!mgmtframe) -+ tx_ctrl->flags |= TX_CTRL_TYPE_DATA; -+ if (eapol_frame) -+ tx_ctrl->flags |= TX_CTRL_EAPOL; -+ tx_ctrl->rate = sta ? sta_info->tx_rate_info : 0; -+ if (ieee80211_is_nullfunc(wh->frame_control) || -+ ieee80211_is_qos_nullfunc(wh->frame_control)) -+ tx_ctrl->rate = 0; -+ pcie_tx_check_tcp_ack(skb, tx_ctrl); -+ -+ if (skb_queue_len(&pcie_priv->txq[index]) > pcie_priv->txq_limit) -+ ieee80211_stop_queue(hw, SYSADPT_TX_WMM_QUEUES - index - 1); -+ -+ skb_queue_tail(&pcie_priv->txq[index], skb); -+ -+ if (!pcie_priv->is_tx_schedule) { -+ tasklet_schedule(&pcie_priv->tx_task); -+ pcie_priv->is_tx_schedule = true; -+ } -+ -+ /* Initiate the ampdu session here */ -+ if (start_ba_session) { -+ spin_lock_bh(&priv->stream_lock); -+ if (mwl_fwcmd_start_stream(hw, stream)) -+ mwl_fwcmd_remove_stream(hw, stream); -+ spin_unlock_bh(&priv->stream_lock); -+ } -+} -diff --git a/drivers/net/wireless/marvell/mwlwifi/hif/pcie/tx_ndp.h b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/tx_ndp.h -new file mode 100644 -index 000000000000..2ad5f381b9ee ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/hif/pcie/tx_ndp.h -@@ -0,0 +1,30 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file defines transmit related functions for new data path. -+ */ -+ -+#ifndef _TX_NDP_H_ -+#define _TX_NDP_H_ -+ -+int pcie_tx_init_ndp(struct ieee80211_hw *hw); -+void pcie_tx_deinit_ndp(struct ieee80211_hw *hw); -+void pcie_tx_skbs_ndp(unsigned long data); -+void pcie_tx_done_ndp(struct ieee80211_hw *hw); -+void pcie_tx_xmit_ndp(struct ieee80211_hw *hw, -+ struct ieee80211_tx_control *control, -+ struct sk_buff *skb); -+ -+#endif /* _TX_NDP_H_ */ -diff --git a/drivers/net/wireless/marvell/mwlwifi/hostapd/700-interoperability-workaround-for-80+80-and-160-MHz-channels b/drivers/net/wireless/marvell/mwlwifi/hostapd/700-interoperability-workaround-for-80+80-and-160-MHz-channels -new file mode 100644 -index 000000000000..adadd2e4d8d4 ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/hostapd/700-interoperability-workaround-for-80+80-and-160-MHz-channels -@@ -0,0 +1,32 @@ -+diff --git a/src/ap/ieee802_11_vht.c b/src/ap/ieee802_11_vht.c -+index 3236016..e923094 100644 -+--- a/src/ap/ieee802_11_vht.c -++++ b/src/ap/ieee802_11_vht.c -+@@ -82,6 +82,27 @@ u8 * hostapd_eid_vht_operation(struct hostapd_data *hapd, u8 *eid) -+ -+ oper->vht_op_info_chwidth = hapd->iconf->vht_oper_chwidth; -+ -++ if (hapd->iconf->vht_oper_chwidth == 2) { -++ /* -++ * Convert 160 MHz channel width to new style as interop -++ * workaround. -++ */ -++ oper->vht_op_info_chwidth = 1; -++ oper->vht_op_info_chan_center_freq_seg1_idx = -++ oper->vht_op_info_chan_center_freq_seg0_idx; -++ if (hapd->iconf->channel < -++ hapd->iconf->vht_oper_centr_freq_seg0_idx) -++ oper->vht_op_info_chan_center_freq_seg0_idx -= 8; -++ else -++ oper->vht_op_info_chan_center_freq_seg0_idx += 8; -++ } else if (hapd->iconf->vht_oper_chwidth == 3) { -++ /* -++ * Convert 80+80 MHz channel width to new style as interop -++ * workaround. -++ */ -++ oper->vht_op_info_chwidth = 1; -++ } -++ -+ /* VHT Basic MCS set comes from hw */ -+ /* Hard code 1 stream, MCS0-7 is a min Basic VHT MCS rates */ -+ oper->vht_basic_mcs_set = host_to_le16(0xfffc); -diff --git a/drivers/net/wireless/marvell/mwlwifi/hostapd/README b/drivers/net/wireless/marvell/mwlwifi/hostapd/README -new file mode 100644 -index 000000000000..a5fb2b68d3d3 ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/hostapd/README -@@ -0,0 +1,26 @@ -+700-interoperability-workaround-for-80+80-and-160-MHz-channels: -+ -+patch for OpenWrt hostapd package 2016-01-15 for following commit -+(move it to package/network/services/hostapd/patches). -+ -+Note: After hostapd package 2016-06-15, this commit is already included. -+ -+commit 03a72eacda5d9a1837a74387081596a0d5466ec1 -+Author: Jouni Malinen -+Date: Thu Dec 17 18:39:19 2015 +0200 -+ -+ VHT: Add an interoperability workaround for 80+80 and 160 MHz channels -+ -+ Number of deployed 80 MHz capable VHT stations that do not support 80+80 -+ and 160 MHz bandwidths seem to misbehave when trying to connect to an AP -+ that advertises 80+80 or 160 MHz channel bandwidth in the VHT Operation -+ element. To avoid such issues with deployed devices, modify the design -+ based on newly proposed IEEE 802.11 standard changes. -+ -+ This allows poorly implemented VHT 80 MHz stations to connect with the -+ AP in 80 MHz mode. 80+80 and 160 MHz capable stations need to support -+ the new workaround mechanism to allow full bandwidth to be used. -+ However, there are more or less no impacted station with 80+80/160 -+ capability deployed. -+ -+ Signed-off-by: Jouni Malinen jouni@qca.qualcomm.com -diff --git a/drivers/net/wireless/marvell/mwlwifi/mac80211.c b/drivers/net/wireless/marvell/mwlwifi/mac80211.c -new file mode 100644 -index 000000000000..725dec0f604b ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/mac80211.c -@@ -0,0 +1,933 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file implements mac80211 related functions. */ -+ -+#include -+ -+#include "sysadpt.h" -+#include "core.h" -+#include "utils.h" -+#include "hif/fwcmd.h" -+#include "hif/hif-ops.h" -+ -+#define MAX_AMPDU_ATTEMPTS 5 -+ -+static const struct ieee80211_rate mwl_rates_24[] = { -+ { .bitrate = 10, .hw_value = 2, }, -+ { .bitrate = 20, .hw_value = 4, }, -+ { .bitrate = 55, .hw_value = 11, }, -+ { .bitrate = 110, .hw_value = 22, }, -+ { .bitrate = 220, .hw_value = 44, }, -+ { .bitrate = 60, .hw_value = 12, }, -+ { .bitrate = 90, .hw_value = 18, }, -+ { .bitrate = 120, .hw_value = 24, }, -+ { .bitrate = 180, .hw_value = 36, }, -+ { .bitrate = 240, .hw_value = 48, }, -+ { .bitrate = 360, .hw_value = 72, }, -+ { .bitrate = 480, .hw_value = 96, }, -+ { .bitrate = 540, .hw_value = 108, }, -+}; -+ -+static const struct ieee80211_rate mwl_rates_50[] = { -+ { .bitrate = 60, .hw_value = 12, }, -+ { .bitrate = 90, .hw_value = 18, }, -+ { .bitrate = 120, .hw_value = 24, }, -+ { .bitrate = 180, .hw_value = 36, }, -+ { .bitrate = 240, .hw_value = 48, }, -+ { .bitrate = 360, .hw_value = 72, }, -+ { .bitrate = 480, .hw_value = 96, }, -+ { .bitrate = 540, .hw_value = 108, }, -+}; -+ -+static void mwl_get_rateinfo(struct mwl_priv *priv, u8 *addr, -+ struct mwl_sta *sta_info) -+{ -+ int table_size = (sizeof(__le32) * 2 * SYSADPT_MAX_RATE_ADAPT_RATES); -+ u8 *rate_table, *rate_idx; -+ u32 rate_info; -+ struct mwl_tx_hist_data *tx_hist_data; -+ int ret, idx; -+ -+ rate_table = kzalloc(table_size, GFP_KERNEL); -+ if (!rate_table) -+ return; -+ -+ ret = mwl_fwcmd_get_ratetable(priv->hw, addr, rate_table, -+ table_size, 0); -+ if (ret) { -+ kfree(rate_table); -+ return; -+ } -+ -+ idx = 0; -+ rate_idx = rate_table; -+ rate_info = le32_to_cpu(*(__le32 *)rate_idx); -+ tx_hist_data = &sta_info->tx_hist.su_rate[0]; -+ while (rate_info) { -+ if (idx < SYSADPT_MAX_RATE_ADAPT_RATES) -+ tx_hist_data[idx].rateinfo = rate_info; -+ idx++; -+ rate_idx += (2 * sizeof(__le32)); -+ rate_info = le32_to_cpu(*(__le32 *)rate_idx); -+ } -+ -+ kfree(rate_table); -+} -+ -+static void mwl_mac80211_tx(struct ieee80211_hw *hw, -+ struct ieee80211_tx_control *control, -+ struct sk_buff *skb) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if (!priv->radio_on) { -+ wiphy_warn(hw->wiphy, -+ "dropped TX frame since radio is disabled\n"); -+ dev_kfree_skb_any(skb); -+ return; -+ } -+ -+ mwl_hif_tx_xmit(hw, control, skb); -+} -+ -+static int mwl_mac80211_start(struct ieee80211_hw *hw) -+{ -+ struct mwl_priv *priv = hw->priv; -+ int rc; -+ -+ /* Enable TX and RX tasklets. */ -+ mwl_hif_enable_data_tasks(hw); -+ -+ /* Enable interrupts */ -+ mwl_hif_irq_enable(hw); -+ -+ rc = mwl_fwcmd_radio_enable(hw); -+ if (rc) -+ goto fwcmd_fail; -+ rc = mwl_fwcmd_set_rate_adapt_mode(hw, 0); -+ if (rc) -+ goto fwcmd_fail; -+ rc = mwl_fwcmd_set_wmm_mode(hw, true); -+ if (rc) -+ goto fwcmd_fail; -+ rc = mwl_fwcmd_ht_guard_interval(hw, GUARD_INTERVAL_AUTO); -+ if (rc) -+ goto fwcmd_fail; -+ rc = mwl_fwcmd_set_dwds_stamode(hw, true); -+ if (rc) -+ goto fwcmd_fail; -+ rc = mwl_fwcmd_set_fw_flush_timer(hw, SYSADPT_AMSDU_FLUSH_TIME); -+ if (rc) -+ goto fwcmd_fail; -+ rc = mwl_fwcmd_set_optimization_level(hw, 1); -+ if (rc) -+ goto fwcmd_fail; -+ if (priv->chip_type == MWL8997) { -+ rc = mwl_fwcmd_config_EDMACCtrl(hw, 0); -+ if (rc) -+ goto fwcmd_fail; -+ } -+ if (priv->chip_type == MWL8964) { -+ rc = mwl_fwcmd_newdp_dmathread_start(hw); -+ if (rc) -+ goto fwcmd_fail; -+ rc = mwl_fwcmd_set_bftype(hw, priv->bf_type); -+ if (rc) -+ goto fwcmd_fail; -+ } -+ -+ ieee80211_wake_queues(hw); -+ return 0; -+ -+fwcmd_fail: -+ mwl_hif_irq_disable(hw); -+ mwl_hif_disable_data_tasks(hw); -+ -+ return rc; -+} -+ -+static void mwl_mac80211_stop(struct ieee80211_hw *hw) -+{ -+ mwl_fwcmd_radio_disable(hw); -+ -+ ieee80211_stop_queues(hw); -+ -+ /* Disable interrupts */ -+ mwl_hif_irq_disable(hw); -+ -+ /* Disable TX and RX tasklets. */ -+ mwl_hif_disable_data_tasks(hw); -+ -+ /* Return all skbs to mac80211 */ -+ mwl_hif_tx_return_pkts(hw); -+} -+ -+static int mwl_mac80211_add_interface(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct mwl_vif *mwl_vif; -+ u32 macids_supported; -+ int macid; -+ -+ switch (vif->type) { -+ case NL80211_IFTYPE_AP: -+ case NL80211_IFTYPE_MESH_POINT: -+ if (vif->type == NL80211_IFTYPE_MESH_POINT) -+ if (priv->chip_type != MWL8997) -+ return -EPERM; -+ macids_supported = priv->ap_macids_supported; -+ break; -+ case NL80211_IFTYPE_STATION: -+ macids_supported = priv->sta_macids_supported; -+ break; -+ default: -+ return -EINVAL; -+ } -+ -+ macid = ffs(macids_supported & ~priv->macids_used); -+ -+ if (!macid) { -+ wiphy_warn(hw->wiphy, "no macid can be allocated\n"); -+ return -EBUSY; -+ } -+ macid--; -+ -+ /* Setup driver private area. */ -+ mwl_vif = mwl_dev_get_vif(vif); -+ memset(mwl_vif, 0, sizeof(*mwl_vif)); -+ mwl_vif->type = vif->type; -+ mwl_vif->macid = macid; -+ mwl_vif->seqno = 0; -+ mwl_vif->is_hw_crypto_enabled = false; -+ mwl_vif->beacon_info.valid = false; -+ mwl_vif->set_beacon = false; -+ mwl_vif->basic_rate_idx = 0; -+ mwl_vif->broadcast_ssid = 0xFF; -+ mwl_vif->iv16 = 1; -+ mwl_vif->iv32 = 0; -+ mwl_vif->keyidx = 0; -+ -+ switch (vif->type) { -+ case NL80211_IFTYPE_AP: -+ ether_addr_copy(mwl_vif->bssid, vif->addr); -+ mwl_fwcmd_set_new_stn_add_self(hw, vif); -+ if (priv->chip_type == MWL8964) { -+ /* allow firmware to really set channel */ -+ mwl_fwcmd_bss_start(hw, vif, true); -+ mwl_fwcmd_bss_start(hw, vif, false); -+ } -+ break; -+ case NL80211_IFTYPE_MESH_POINT: -+ ether_addr_copy(mwl_vif->bssid, vif->addr); -+ mwl_fwcmd_set_new_stn_add_self(hw, vif); -+ break; -+ case NL80211_IFTYPE_STATION: -+ ether_addr_copy(mwl_vif->sta_mac, vif->addr); -+ mwl_fwcmd_bss_start(hw, vif, true); -+ mwl_fwcmd_set_infra_mode(hw, vif); -+ mwl_fwcmd_set_mac_addr_client(hw, vif, vif->addr); -+ break; -+ default: -+ return -EINVAL; -+ } -+ -+ priv->macids_used |= 1 << mwl_vif->macid; -+ spin_lock_bh(&priv->vif_lock); -+ list_add_tail(&mwl_vif->list, &priv->vif_list); -+ spin_unlock_bh(&priv->vif_lock); -+ -+ return 0; -+} -+ -+static void mwl_mac80211_remove_vif(struct mwl_priv *priv, -+ struct ieee80211_vif *vif) -+{ -+ struct mwl_vif *mwl_vif = mwl_dev_get_vif(vif); -+ -+ if (!priv->macids_used) -+ return; -+ -+ mwl_hif_tx_del_pkts_via_vif(priv->hw, vif); -+ -+ priv->macids_used &= ~(1 << mwl_vif->macid); -+ spin_lock_bh(&priv->vif_lock); -+ list_del(&mwl_vif->list); -+ spin_unlock_bh(&priv->vif_lock); -+} -+ -+static void mwl_mac80211_remove_interface(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ switch (vif->type) { -+ case NL80211_IFTYPE_AP: -+ case NL80211_IFTYPE_MESH_POINT: -+ mwl_fwcmd_set_new_stn_del(hw, vif, vif->addr); -+ break; -+ case NL80211_IFTYPE_STATION: -+ mwl_fwcmd_remove_mac_addr(hw, vif, vif->addr); -+ break; -+ default: -+ break; -+ } -+ -+ mwl_mac80211_remove_vif(priv, vif); -+} -+ -+static int mwl_mac80211_config(struct ieee80211_hw *hw, -+ u32 changed) -+{ -+ struct ieee80211_conf *conf = &hw->conf; -+ int rc; -+ -+ wiphy_debug(hw->wiphy, "change: 0x%x\n", changed); -+ -+ if (conf->flags & IEEE80211_CONF_IDLE) -+ rc = mwl_fwcmd_radio_disable(hw); -+ else -+ rc = mwl_fwcmd_radio_enable(hw); -+ -+ if (rc) -+ goto out; -+ -+ if (changed & IEEE80211_CONF_CHANGE_CHANNEL) { -+ int rate = 0; -+ -+ if (conf->chandef.chan->band == NL80211_BAND_2GHZ) { -+ mwl_fwcmd_set_apmode(hw, AP_MODE_2_4GHZ_11AC_MIXED); -+ mwl_fwcmd_set_linkadapt_cs_mode(hw, -+ LINK_CS_STATE_CONSERV); -+ rate = mwl_rates_24[0].hw_value; -+ } else if (conf->chandef.chan->band == NL80211_BAND_5GHZ) { -+ mwl_fwcmd_set_apmode(hw, AP_MODE_11AC); -+ mwl_fwcmd_set_linkadapt_cs_mode(hw, -+ LINK_CS_STATE_AUTO); -+ rate = mwl_rates_50[0].hw_value; -+ -+ if (conf->radar_enabled) -+ mwl_fwcmd_set_radar_detect(hw, MONITOR_START); -+ else -+ mwl_fwcmd_set_radar_detect(hw, -+ STOP_DETECT_RADAR); -+ } -+ -+ rc = mwl_fwcmd_set_rf_channel(hw, conf); -+ if (rc) -+ goto out; -+ rc = mwl_fwcmd_use_fixed_rate(hw, rate, rate); -+ if (rc) -+ goto out; -+ rc = mwl_fwcmd_max_tx_power(hw, conf, 0); -+ if (rc) -+ goto out; -+ rc = mwl_fwcmd_tx_power(hw, conf, 0); -+ if (rc) -+ goto out; -+ rc = mwl_fwcmd_set_cdd(hw); -+ } -+ -+out: -+ -+ return rc; -+} -+ -+static void mwl_mac80211_bss_info_changed_sta(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, -+ struct ieee80211_bss_conf *info, -+ u32 changed) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ if ((changed & BSS_CHANGED_ERP_SLOT) && (priv->chip_type == MWL8997)) { -+ if (priv->use_short_slot != vif->bss_conf.use_short_slot) { -+ mwl_fwcmd_set_slot_time(hw, -+ vif->bss_conf.use_short_slot); -+ priv->use_short_slot = vif->bss_conf.use_short_slot; -+ } -+ } -+ -+ if (changed & BSS_CHANGED_ERP_PREAMBLE) { -+ if (priv->use_short_preamble != -+ vif->bss_conf.use_short_preamble) { -+ mwl_fwcmd_set_radio_preamble( -+ hw, vif->bss_conf.use_short_preamble); -+ priv->use_short_preamble = -+ vif->bss_conf.use_short_preamble; -+ } -+ } -+ -+ if ((changed & BSS_CHANGED_ASSOC) && vif->bss_conf.assoc) -+ mwl_fwcmd_set_aid(hw, vif, (u8 *)vif->bss_conf.bssid, -+ vif->bss_conf.aid); -+} -+ -+static void mwl_mac80211_bss_info_changed_ap(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, -+ struct ieee80211_bss_conf *info, -+ u32 changed) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct mwl_vif *mwl_vif; -+ -+ mwl_vif = mwl_dev_get_vif(vif); -+ -+ if ((changed & BSS_CHANGED_ERP_SLOT) && (priv->chip_type == MWL8997)) { -+ if (priv->use_short_slot != vif->bss_conf.use_short_slot) { -+ mwl_fwcmd_set_slot_time(hw, -+ vif->bss_conf.use_short_slot); -+ priv->use_short_slot = vif->bss_conf.use_short_slot; -+ } -+ } -+ -+ if (changed & BSS_CHANGED_ERP_PREAMBLE) { -+ if (priv->use_short_preamble != -+ vif->bss_conf.use_short_preamble) { -+ mwl_fwcmd_set_radio_preamble( -+ hw, vif->bss_conf.use_short_preamble); -+ priv->use_short_preamble = -+ vif->bss_conf.use_short_preamble; -+ } -+ } -+ -+ if (changed & BSS_CHANGED_BASIC_RATES) { -+ int idx; -+ int rate; -+ -+ /* Use lowest supported basic rate for multicasts -+ * and management frames (such as probe responses -- -+ * beacons will always go out at 1 Mb/s). -+ */ -+ idx = ffs(vif->bss_conf.basic_rates); -+ if (idx) -+ idx--; -+ if (mwl_vif->basic_rate_idx != idx) { -+ if (hw->conf.chandef.chan->band == NL80211_BAND_2GHZ) -+ rate = mwl_rates_24[idx].hw_value; -+ else -+ rate = mwl_rates_50[idx].hw_value; -+ -+ mwl_fwcmd_use_fixed_rate(hw, rate, rate); -+ mwl_vif->basic_rate_idx = idx; -+ } -+ } -+ -+ if (changed & (BSS_CHANGED_BEACON_INT | BSS_CHANGED_BEACON)) { -+ struct sk_buff *skb; -+ -+ if ((info->ssid[0] != '\0') && -+ (info->ssid_len != 0) && -+ (!info->hidden_ssid)) { -+ if (mwl_vif->broadcast_ssid != true) { -+ mwl_fwcmd_broadcast_ssid_enable(hw, vif, true); -+ mwl_vif->broadcast_ssid = true; -+ } -+ } else { -+ if (mwl_vif->broadcast_ssid != false) { -+ mwl_fwcmd_broadcast_ssid_enable(hw, vif, false); -+ mwl_vif->broadcast_ssid = false; -+ } -+ } -+ -+ if (!mwl_vif->set_beacon) { -+ skb = ieee80211_beacon_get(hw, vif); -+ -+ if (skb) { -+ mwl_fwcmd_set_beacon(hw, vif, skb->data, skb->len); -+ dev_kfree_skb_any(skb); -+ } -+ mwl_vif->set_beacon = true; -+ } -+ } -+ -+ if (changed & BSS_CHANGED_BEACON_ENABLED) -+ mwl_fwcmd_bss_start(hw, vif, info->enable_beacon); -+} -+ -+static void mwl_mac80211_bss_info_changed(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, -+ struct ieee80211_bss_conf *info, -+ u32 changed) -+{ -+ switch (vif->type) { -+ case NL80211_IFTYPE_AP: -+ case NL80211_IFTYPE_MESH_POINT: -+ mwl_mac80211_bss_info_changed_ap(hw, vif, info, changed); -+ break; -+ case NL80211_IFTYPE_STATION: -+ mwl_mac80211_bss_info_changed_sta(hw, vif, info, changed); -+ break; -+ default: -+ break; -+ } -+} -+ -+static void mwl_mac80211_configure_filter(struct ieee80211_hw *hw, -+ unsigned int changed_flags, -+ unsigned int *total_flags, -+ u64 multicast) -+{ -+ /* AP firmware doesn't allow fine-grained control over -+ * the receive filter. -+ */ -+ *total_flags &= FIF_ALLMULTI | FIF_BCN_PRBRESP_PROMISC; -+} -+ -+static int mwl_mac80211_set_key(struct ieee80211_hw *hw, -+ enum set_key_cmd cmd_param, -+ struct ieee80211_vif *vif, -+ struct ieee80211_sta *sta, -+ struct ieee80211_key_conf *key) -+{ -+ struct mwl_vif *mwl_vif; -+ struct mwl_sta *sta_info; -+ int rc = 0; -+ u8 encr_type; -+ u8 *addr; -+ -+ mwl_vif = mwl_dev_get_vif(vif); -+ addr = sta ? sta->addr : vif->addr; -+ -+ if (cmd_param == SET_KEY) { -+ if ((key->cipher == WLAN_CIPHER_SUITE_WEP40) || -+ (key->cipher == WLAN_CIPHER_SUITE_WEP104)) { -+ encr_type = ENCR_TYPE_WEP; -+ } else if (key->cipher == WLAN_CIPHER_SUITE_CCMP) { -+ encr_type = ENCR_TYPE_AES; -+ if ((key->flags & IEEE80211_KEY_FLAG_PAIRWISE) == 0) { -+ if (vif->type != NL80211_IFTYPE_STATION) -+ mwl_vif->keyidx = key->keyidx; -+ } -+ } else if (key->cipher == WLAN_CIPHER_SUITE_TKIP) { -+ encr_type = ENCR_TYPE_TKIP; -+ } else { -+ encr_type = ENCR_TYPE_DISABLE; -+ } -+ -+ rc = mwl_fwcmd_update_encryption_enable(hw, vif, addr, -+ encr_type); -+ if (rc) -+ goto out; -+ rc = mwl_fwcmd_encryption_set_key(hw, vif, addr, key); -+ if (rc) -+ goto out; -+ -+ mwl_vif->is_hw_crypto_enabled = true; -+ if (sta) { -+ sta_info = mwl_dev_get_sta(sta); -+ sta_info->is_key_set = true; -+ } -+ } else { -+ rc = mwl_fwcmd_encryption_remove_key(hw, vif, addr, key); -+ if (rc) -+ goto out; -+ } -+ -+out: -+ -+ return rc; -+} -+ -+static int mwl_mac80211_set_rts_threshold(struct ieee80211_hw *hw, -+ u32 value) -+{ -+ return mwl_fwcmd_set_rts_threshold(hw, value); -+} -+ -+static int mwl_mac80211_sta_add(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, -+ struct ieee80211_sta *sta) -+{ -+ struct mwl_priv *priv = hw->priv; -+ u16 stnid, sta_stnid = 0; -+ struct mwl_vif *mwl_vif; -+ struct wireless_dev *wdev = ieee80211_vif_to_wdev(vif); -+ bool use_4addr = wdev->use_4addr; -+ struct mwl_sta *sta_info; -+ struct ieee80211_key_conf *key; -+ int rc; -+ int i; -+ -+ if (vif->type == NL80211_IFTYPE_STATION) -+ sta->aid = 1; -+ mwl_vif = mwl_dev_get_vif(vif); -+ stnid = utils_assign_stnid(priv, mwl_vif->macid, sta->aid); -+ if (!stnid) -+ return -EPERM; -+ if (vif->type == NL80211_IFTYPE_STATION) { -+ sta_stnid = utils_assign_stnid(priv, mwl_vif->macid, -+ sta->aid + 1); -+ if (!sta_stnid) { -+ utils_free_stnid(priv, stnid); -+ return -EPERM; -+ } -+ ether_addr_copy(mwl_vif->bssid, sta->addr); -+ } -+ sta_info = mwl_dev_get_sta(sta); -+ memset(sta_info, 0, sizeof(*sta_info)); -+ -+ if (vif->type == NL80211_IFTYPE_MESH_POINT) -+ sta_info->is_mesh_node = true; -+ -+ if (sta->ht_cap.ht_supported) { -+ sta_info->is_ampdu_allowed = true; -+ sta_info->is_amsdu_allowed = false; -+ if (sta->ht_cap.cap & IEEE80211_HT_CAP_MAX_AMSDU) -+ sta_info->amsdu_ctrl.cap = MWL_AMSDU_SIZE_8K; -+ else -+ sta_info->amsdu_ctrl.cap = MWL_AMSDU_SIZE_4K; -+ if ((sta->tdls) && (!sta->wme)) -+ sta->wme = true; -+ } -+ sta_info->mwl_vif = mwl_vif; -+ sta_info->stnid = stnid; -+ if (vif->type == NL80211_IFTYPE_STATION) -+ sta_info->sta_stnid = sta_stnid; -+ sta_info->tx_rate_info = utils_get_init_tx_rate(priv, &hw->conf, sta); -+ sta_info->iv16 = 1; -+ sta_info->iv32 = 0; -+ spin_lock_init(&sta_info->amsdu_lock); -+ spin_lock_bh(&priv->sta_lock); -+ list_add_tail(&sta_info->list, &priv->sta_list); -+ spin_unlock_bh(&priv->sta_lock); -+ -+ if (vif->type == NL80211_IFTYPE_STATION) -+ mwl_fwcmd_set_new_stn_del(hw, vif, sta->addr); -+ -+ if (priv->chip_type == MWL8964) { -+ if (use_4addr) { -+ sta_info->wds = true; -+ rc = mwl_fwcmd_set_new_stn_add_sc4(hw, vif, sta, -+ WDS_MODE); -+ } else -+ rc = mwl_fwcmd_set_new_stn_add_sc4(hw, vif, sta, 0); -+ } else -+ rc = mwl_fwcmd_set_new_stn_add(hw, vif, sta); -+ -+ if ((vif->type == NL80211_IFTYPE_STATION) && !use_4addr) -+ mwl_hif_set_sta_id(hw, sta, true, true); -+ else -+ mwl_hif_set_sta_id(hw, sta, false, true); -+ -+ for (i = 0; i < NUM_WEP_KEYS; i++) { -+ key = (struct ieee80211_key_conf *)mwl_vif->wep_key_conf[i].key; -+ -+ if (mwl_vif->wep_key_conf[i].enabled) -+ mwl_mac80211_set_key(hw, SET_KEY, vif, sta, key); -+ } -+ -+ mwl_get_rateinfo(priv, sta->addr, sta_info); -+ -+ return rc; -+} -+ -+static int mwl_mac80211_sta_remove(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, -+ struct ieee80211_sta *sta) -+{ -+ struct mwl_priv *priv = hw->priv; -+ int rc; -+ struct mwl_sta *sta_info = mwl_dev_get_sta(sta); -+ -+ mwl_hif_tx_del_sta_amsdu_pkts(hw, sta); -+ mwl_fwcmd_del_sta_streams(hw, sta); -+ mwl_hif_tx_del_pkts_via_sta(hw, sta); -+ -+ rc = mwl_fwcmd_set_new_stn_del(hw, vif, sta->addr); -+ -+ if (vif->type == NL80211_IFTYPE_STATION) -+ mwl_hif_set_sta_id(hw, sta, true, false); -+ else -+ mwl_hif_set_sta_id(hw, sta, false, false); -+ -+ if (priv->chip_type != MWL8964) -+ utils_free_stnid(priv, sta_info->stnid); -+ if (vif->type == NL80211_IFTYPE_STATION) -+ utils_free_stnid(priv, sta_info->sta_stnid); -+ -+ spin_lock_bh(&priv->sta_lock); -+ list_del(&sta_info->list); -+ spin_unlock_bh(&priv->sta_lock); -+ -+ return rc; -+} -+ -+static int mwl_mac80211_conf_tx(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, -+ u16 queue, -+ const struct ieee80211_tx_queue_params *params) -+{ -+ struct mwl_priv *priv = hw->priv; -+ int rc = 0; -+ -+ if (WARN_ON(queue > SYSADPT_TX_WMM_QUEUES - 1)) -+ return -EINVAL; -+ -+ memcpy(&priv->wmm_params[queue], params, sizeof(*params)); -+ -+ if (!priv->wmm_enabled) { -+ rc = mwl_fwcmd_set_wmm_mode(hw, true); -+ priv->wmm_enabled = true; -+ } -+ -+ if (!rc) { -+ int q = SYSADPT_TX_WMM_QUEUES - 1 - queue; -+ -+ rc = mwl_fwcmd_set_edca_params(hw, q, -+ params->cw_min, params->cw_max, -+ params->aifs, params->txop); -+ } -+ -+ return rc; -+} -+ -+static int mwl_mac80211_get_stats(struct ieee80211_hw *hw, -+ struct ieee80211_low_level_stats *stats) -+{ -+ return mwl_fwcmd_get_stat(hw, stats); -+} -+ -+static int mwl_mac80211_get_survey(struct ieee80211_hw *hw, -+ int idx, -+ struct survey_info *survey) -+{ -+ struct mwl_priv *priv = hw->priv; -+ struct mwl_survey_info *survey_info; -+ -+ if (priv->survey_info_idx) { -+ if (idx >= priv->survey_info_idx) { -+ priv->survey_info_idx = 0; -+ return -ENOENT; -+ } -+ survey_info = &priv->survey_info[idx]; -+ } else { -+ if (idx != 0) -+ return -ENOENT; -+ mwl_fwcmd_get_survey(hw, 0); -+ survey_info = &priv->cur_survey_info; -+ if (!(hw->conf.flags & IEEE80211_CONF_OFFCHANNEL)) -+ survey->filled |= SURVEY_INFO_IN_USE; -+ } -+ -+ survey->channel = &survey_info->channel; -+ survey->filled |= survey_info->filled; -+ survey->time = survey_info->time_period / 1000; -+ survey->time_busy = survey_info->time_busy / 1000; -+ survey->time_tx = survey_info->time_tx / 1000; -+ survey->noise = survey_info->noise; -+ -+ return 0; -+} -+ -+static int mwl_mac80211_ampdu_action(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, -+ struct ieee80211_ampdu_params *params) -+{ -+ int rc = 0; -+ struct mwl_priv *priv = hw->priv; -+ struct mwl_ampdu_stream *stream; -+ enum ieee80211_ampdu_mlme_action action = params->action; -+ struct ieee80211_sta *sta = params->sta; -+ u16 tid = params->tid; -+ u8 buf_size = params->buf_size; -+ u8 *addr = sta->addr; -+ struct mwl_sta *sta_info; -+ -+ sta_info = mwl_dev_get_sta(sta); -+ -+ spin_lock_bh(&priv->stream_lock); -+ -+ stream = mwl_fwcmd_lookup_stream(hw, sta, tid); -+ -+ switch (action) { -+ case IEEE80211_AMPDU_RX_START: -+ if (priv->chip_type == MWL8964) { -+ struct mwl_ampdu_stream tmp; -+ -+ tmp.sta = sta; -+ tmp.tid = tid; -+ spin_unlock_bh(&priv->stream_lock); -+ mwl_fwcmd_create_ba(hw, &tmp, vif, -+ BA_FLAG_DIRECTION_DOWN, -+ buf_size, params->ssn, -+ params->amsdu); -+ spin_lock_bh(&priv->stream_lock); -+ break; -+ } -+ case IEEE80211_AMPDU_RX_STOP: -+ if (priv->chip_type == MWL8964) { -+ struct mwl_ampdu_stream tmp; -+ -+ tmp.sta = sta; -+ tmp.tid = tid; -+ spin_unlock_bh(&priv->stream_lock); -+ mwl_fwcmd_destroy_ba(hw, &tmp, -+ BA_FLAG_DIRECTION_DOWN); -+ spin_lock_bh(&priv->stream_lock); -+ } -+ break; -+ case IEEE80211_AMPDU_TX_START: -+ if (!sta_info->is_ampdu_allowed) { -+ wiphy_warn(hw->wiphy, "ampdu not allowed\n"); -+ rc = -EPERM; -+ break; -+ } -+ -+ if (!stream) { -+ stream = mwl_fwcmd_add_stream(hw, sta, tid); -+ if (!stream) { -+ wiphy_warn(hw->wiphy, "no stream found\n"); -+ rc = -EPERM; -+ break; -+ } -+ } -+ -+ if (priv->chip_type != MWL8964) { -+ spin_unlock_bh(&priv->stream_lock); -+ rc = mwl_fwcmd_check_ba(hw, stream, vif, -+ BA_FLAG_DIRECTION_UP); -+ spin_lock_bh(&priv->stream_lock); -+ if (rc) { -+ mwl_fwcmd_remove_stream(hw, stream); -+ sta_info->check_ba_failed[tid]++; -+ break; -+ } -+ } -+ stream->state = AMPDU_STREAM_IN_PROGRESS; -+ spin_unlock_bh(&priv->stream_lock); -+ rc = mwl_fwcmd_get_seqno(hw, stream, ¶ms->ssn); -+ spin_lock_bh(&priv->stream_lock); -+ if (rc) -+ break; -+ ieee80211_start_tx_ba_cb_irqsafe(vif, addr, tid); -+ break; -+ case IEEE80211_AMPDU_TX_STOP_CONT: -+ case IEEE80211_AMPDU_TX_STOP_FLUSH: -+ case IEEE80211_AMPDU_TX_STOP_FLUSH_CONT: -+ if (stream) { -+ if (stream->state == AMPDU_STREAM_ACTIVE) { -+ stream->state = AMPDU_STREAM_IN_PROGRESS; -+ mwl_hif_tx_del_ampdu_pkts(hw, sta, tid); -+ spin_unlock_bh(&priv->stream_lock); -+ mwl_fwcmd_destroy_ba(hw, stream, -+ BA_FLAG_DIRECTION_UP); -+ spin_lock_bh(&priv->stream_lock); -+ sta_info->is_amsdu_allowed = false; -+ } -+ -+ mwl_fwcmd_remove_stream(hw, stream); -+ ieee80211_stop_tx_ba_cb_irqsafe(vif, addr, tid); -+ } else { -+ rc = -EPERM; -+ } -+ break; -+ case IEEE80211_AMPDU_TX_OPERATIONAL: -+ if (stream) { -+ if (WARN_ON(stream->state != -+ AMPDU_STREAM_IN_PROGRESS)) { -+ rc = -EPERM; -+ break; -+ } -+ spin_unlock_bh(&priv->stream_lock); -+ rc = mwl_fwcmd_create_ba(hw, stream, vif, -+ BA_FLAG_DIRECTION_UP, -+ buf_size, params->ssn, -+ params->amsdu); -+ spin_lock_bh(&priv->stream_lock); -+ -+ if (!rc) { -+ stream->state = AMPDU_STREAM_ACTIVE; -+ sta_info->check_ba_failed[tid] = 0; -+ if (priv->tx_amsdu) -+ sta_info->is_amsdu_allowed = -+ params->amsdu; -+ else -+ sta_info->is_amsdu_allowed = false; -+ } else { -+ spin_unlock_bh(&priv->stream_lock); -+ mwl_fwcmd_destroy_ba(hw, stream, -+ BA_FLAG_DIRECTION_UP); -+ spin_lock_bh(&priv->stream_lock); -+ mwl_fwcmd_remove_stream(hw, stream); -+ wiphy_err(hw->wiphy, -+ "ampdu operation error code: %d\n", -+ rc); -+ } -+ } else { -+ rc = -EPERM; -+ } -+ break; -+ default: -+ rc = -ENOTSUPP; -+ break; -+ } -+ -+ spin_unlock_bh(&priv->stream_lock); -+ -+ return rc; -+} -+ -+static int mwl_mac80211_chnl_switch(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, -+ struct ieee80211_channel_switch *ch_switch) -+{ -+ int rc = 0; -+ -+ rc = mwl_fwcmd_set_switch_channel(hw, ch_switch); -+ -+ return rc; -+} -+ -+static void mwl_mac80211_sw_scan_start(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif, -+ const u8 *mac_addr) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ priv->sw_scanning = true; -+ priv->survey_info_idx = 0; -+} -+ -+static void mwl_mac80211_sw_scan_complete(struct ieee80211_hw *hw, -+ struct ieee80211_vif *vif) -+{ -+ struct mwl_priv *priv = hw->priv; -+ -+ priv->sw_scanning = false; -+} -+ -+const struct ieee80211_ops mwl_mac80211_ops = { -+ .tx = mwl_mac80211_tx, -+ .start = mwl_mac80211_start, -+ .stop = mwl_mac80211_stop, -+ .add_interface = mwl_mac80211_add_interface, -+ .remove_interface = mwl_mac80211_remove_interface, -+ .config = mwl_mac80211_config, -+ .bss_info_changed = mwl_mac80211_bss_info_changed, -+ .configure_filter = mwl_mac80211_configure_filter, -+ .set_key = mwl_mac80211_set_key, -+ .set_rts_threshold = mwl_mac80211_set_rts_threshold, -+ .sta_add = mwl_mac80211_sta_add, -+ .sta_remove = mwl_mac80211_sta_remove, -+ .conf_tx = mwl_mac80211_conf_tx, -+ .get_stats = mwl_mac80211_get_stats, -+ .get_survey = mwl_mac80211_get_survey, -+ .ampdu_action = mwl_mac80211_ampdu_action, -+ .pre_channel_switch = mwl_mac80211_chnl_switch, -+ .sw_scan_start = mwl_mac80211_sw_scan_start, -+ .sw_scan_complete = mwl_mac80211_sw_scan_complete, -+}; -diff --git a/drivers/net/wireless/marvell/mwlwifi/mu_mimo.c b/drivers/net/wireless/marvell/mwlwifi/mu_mimo.c -new file mode 100644 -index 000000000000..74ab054f947e ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/mu_mimo.c -@@ -0,0 +1,21 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file implements MU-MIMO functions. */ -+ -+#include "sysadpt.h" -+#include "core.h" -+#include "mu_mimo.h" -+ -diff --git a/drivers/net/wireless/marvell/mwlwifi/mu_mimo.h b/drivers/net/wireless/marvell/mwlwifi/mu_mimo.h -new file mode 100644 -index 000000000000..24179f404774 ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/mu_mimo.h -@@ -0,0 +1,23 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file defines MU-MIMO functions. */ -+ -+#ifndef _MU_MIMO_H_ -+#define _MU_MIMO_H_ -+ -+ -+ -+#endif /* _MU_MIMO_H_ */ -diff --git a/drivers/net/wireless/marvell/mwlwifi/sysadpt.h b/drivers/net/wireless/marvell/mwlwifi/sysadpt.h -new file mode 100644 -index 000000000000..1194e5271870 ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/sysadpt.h -@@ -0,0 +1,86 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file defines system adaptation related information. */ -+ -+#ifndef _SYSADPT_H_ -+#define _SYSADPT_H_ -+ -+#define SYSADPT_MAX_STA 64 -+ -+#define SYSADPT_MAX_STA_SC4 300 -+ -+#define SYSADPT_MAX_NUM_CHANNELS 64 -+ -+#define SYSADPT_MAX_DATA_RATES_G 14 -+ -+#define SYSADPT_MAX_MCS_RATES 24 -+ -+#define SYSADPT_MAX_11AC_RATES 20 -+ -+#define SYSADPT_MAX_RATE_ADAPT_RATES (SYSADPT_MAX_DATA_RATES_G + \ -+ SYSADPT_MAX_MCS_RATES + \ -+ SYSADPT_MAX_11AC_RATES) -+ -+#define SYSADPT_TX_POWER_LEVEL_TOTAL 16 /* SC3 */ -+ -+#define SYSADPT_TX_GRP_PWR_LEVEL_TOTAL 28 /* KF2 */ -+ -+#define SYSADPT_TX_PWR_LEVEL_TOTAL_SC4 32 /* SC4 */ -+ -+#define SYSADPT_TX_WMM_QUEUES 4 -+ -+#define SYSADPT_NUM_OF_CLIENT 1 -+ -+#define SYSADPT_NUM_OF_AP 16 -+ -+#define SYSADPT_NUM_OF_MESH 1 -+ -+#define SYSADPT_TOTAL_TX_QUEUES (SYSADPT_TX_WMM_QUEUES + \ -+ SYSADPT_NUM_OF_AP) -+ -+#define SYSADPT_MAX_AGGR_SIZE 4096 -+ -+#define SYSADPT_AMPDU_PACKET_THRESHOLD 64 -+ -+#define SYSADPT_AMSDU_FW_MAX_SIZE 3300 -+ -+#define SYSADPT_AMSDU_4K_MAX_SIZE SYSADPT_AMSDU_FW_MAX_SIZE -+ -+#define SYSADPT_AMSDU_8K_MAX_SIZE SYSADPT_AMSDU_FW_MAX_SIZE -+ -+#define SYSADPT_AMSDU_ALLOW_SIZE 1600 -+ -+#define SYSADPT_AMSDU_FLUSH_TIME 500 -+ -+#define SYSADPT_AMSDU_PACKET_THRESHOLD 10 -+ -+#define SYSADPT_MAX_TID 8 -+ -+#define SYSADPT_QUIET_PERIOD_DEFAULT 100 -+ -+#define SYSADPT_QUIET_PERIOD_MIN 25 -+ -+#define SYSADPT_QUIET_START_OFFSET 10 -+ -+#define SYSADPT_THERMAL_THROTTLE_MAX 100 -+ -+#define SYSADPT_TIMER_WAKEUP_TIME 10 /* ms */ -+ -+#define SYSADPT_OTP_BUF_SIZE (256*8) /* 258 lines * 8 bytes */ -+ -+#define SYSADPT_TXPWRLMT_CFG_BUF_SIZE (3650) -+ -+#endif /* _SYSADPT_H_ */ -diff --git a/drivers/net/wireless/marvell/mwlwifi/thermal.c b/drivers/net/wireless/marvell/mwlwifi/thermal.c -new file mode 100644 -index 000000000000..7c59def51e7f ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/thermal.c -@@ -0,0 +1,182 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file implements thermal framework related functions. */ -+ -+#include -+#include -+#include -+#include -+#include -+ -+#include "sysadpt.h" -+#include "core.h" -+#include "hif/fwcmd.h" -+#include "thermal.h" -+ -+static int -+mwl_thermal_get_max_throttle_state(struct thermal_cooling_device *cdev, -+ unsigned long *state) -+{ -+ *state = SYSADPT_THERMAL_THROTTLE_MAX; -+ -+ return 0; -+} -+ -+static int -+mwl_thermal_get_cur_throttle_state(struct thermal_cooling_device *cdev, -+ unsigned long *state) -+{ -+ struct mwl_priv *priv = cdev->devdata; -+ -+ *state = priv->throttle_state; -+ -+ return 0; -+} -+ -+static int -+mwl_thermal_set_cur_throttle_state(struct thermal_cooling_device *cdev, -+ unsigned long throttle_state) -+{ -+ struct mwl_priv *priv = cdev->devdata; -+ -+ if (throttle_state > SYSADPT_THERMAL_THROTTLE_MAX) { -+ wiphy_warn(priv->hw->wiphy, -+ "throttle state %ld is exceeding the limit %d\n", -+ throttle_state, SYSADPT_THERMAL_THROTTLE_MAX); -+ return -EINVAL; -+ } -+ priv->throttle_state = throttle_state; -+ mwl_thermal_set_throttling(priv); -+ -+ return 0; -+} -+ -+static struct thermal_cooling_device_ops mwl_thermal_ops = { -+ .get_max_state = mwl_thermal_get_max_throttle_state, -+ .get_cur_state = mwl_thermal_get_cur_throttle_state, -+ .set_cur_state = mwl_thermal_set_cur_throttle_state, -+}; -+ -+static ssize_t mwl_thermal_show_temp(struct device *dev, -+ struct device_attribute *attr, -+ char *buf) -+{ -+ struct mwl_priv *priv = dev_get_drvdata(dev); -+ int ret, temperature; -+ -+ ret = mwl_fwcmd_get_temp(priv->hw, &priv->temperature); -+ if (ret) { -+ wiphy_warn(priv->hw->wiphy, "failed: can't get temperature\n"); -+ goto out; -+ } -+ -+ temperature = priv->temperature; -+ -+ /* display in millidegree celcius */ -+ ret = snprintf(buf, PAGE_SIZE, "%d\n", temperature * 1000); -+out: -+ return ret; -+} -+ -+static SENSOR_DEVICE_ATTR(temp1_input, 0444, mwl_thermal_show_temp, -+ NULL, 0); -+ -+static struct attribute *mwl_hwmon_attrs[] = { -+ &sensor_dev_attr_temp1_input.dev_attr.attr, -+ NULL, -+}; -+ATTRIBUTE_GROUPS(mwl_hwmon); -+ -+void mwl_thermal_set_throttling(struct mwl_priv *priv) -+{ -+ u32 period, duration, enabled; -+ int ret; -+ -+ period = priv->quiet_period; -+ duration = (period * priv->throttle_state) / 100; -+ enabled = duration ? 1 : 0; -+ -+ ret = mwl_fwcmd_quiet_mode(priv->hw, enabled, period, -+ duration, SYSADPT_QUIET_START_OFFSET); -+ if (ret) { -+ wiphy_warn(priv->hw->wiphy, -+ "failed: period %u duarion %u enabled %u ret %d\n", -+ period, duration, enabled, ret); -+ } -+} -+ -+int mwl_thermal_register(struct mwl_priv *priv) -+{ -+ struct thermal_cooling_device *cdev; -+ struct device *hwmon_dev; -+ int ret; -+ -+ if (priv->chip_type != MWL8897) -+ return 0; -+ -+ cdev = thermal_cooling_device_register("mwlwifi_thermal", priv, -+ &mwl_thermal_ops); -+ if (IS_ERR(cdev)) { -+ wiphy_err(priv->hw->wiphy, -+ "failed to setup thermal device result: %ld\n", -+ PTR_ERR(cdev)); -+ return -EINVAL; -+ } -+ -+ ret = sysfs_create_link(&priv->dev->kobj, &cdev->device.kobj, -+ "cooling_device"); -+ if (ret) { -+ wiphy_err(priv->hw->wiphy, -+ "failed to create cooling device symlink\n"); -+ goto err_cooling_destroy; -+ } -+ -+ priv->cdev = cdev; -+ priv->quiet_period = SYSADPT_QUIET_PERIOD_DEFAULT; -+ -+ if (!IS_ENABLED(CONFIG_HWMON)) -+ return 0; -+ -+ hwmon_dev = -+ devm_hwmon_device_register_with_groups(priv->dev, -+ "mwlwifi_hwmon", priv, -+ mwl_hwmon_groups); -+ if (IS_ERR(hwmon_dev)) { -+ wiphy_err(priv->hw->wiphy, -+ "failed to register hwmon device: %ld\n", -+ PTR_ERR(hwmon_dev)); -+ ret = -EINVAL; -+ goto err_remove_link; -+ } -+ -+ return 0; -+ -+err_remove_link: -+ sysfs_remove_link(&priv->dev->kobj, "cooling_device"); -+err_cooling_destroy: -+ thermal_cooling_device_unregister(cdev); -+ -+ return ret; -+} -+ -+void mwl_thermal_unregister(struct mwl_priv *priv) -+{ -+ if (priv->chip_type != MWL8897) -+ return; -+ -+ sysfs_remove_link(&priv->dev->kobj, "cooling_device"); -+ thermal_cooling_device_unregister(priv->cdev); -+} -diff --git a/drivers/net/wireless/marvell/mwlwifi/thermal.h b/drivers/net/wireless/marvell/mwlwifi/thermal.h -new file mode 100644 -index 000000000000..c7f0ad2b87eb ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/thermal.h -@@ -0,0 +1,42 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file defines Linux thermal framework related functions. */ -+ -+#ifndef _THERMAL_H_ -+#define _THERMAL_H_ -+ -+#include -+ -+#if IS_ENABLED(CONFIG_THERMAL) -+int mwl_thermal_register(struct mwl_priv *priv); -+void mwl_thermal_unregister(struct mwl_priv *priv); -+void mwl_thermal_set_throttling(struct mwl_priv *priv); -+#else -+static inline int mwl_thermal_register(struct mwl_priv *priv) -+{ -+ return 0; -+} -+ -+static inline void mwl_thermal_unregister(struct mwl_priv *priv) -+{ -+} -+ -+static inline void mwl_thermal_set_throttling(struct mwl_priv *priv) -+{ -+} -+#endif -+ -+#endif /* _THERMAL_H_ */ -diff --git a/drivers/net/wireless/marvell/mwlwifi/utils.c b/drivers/net/wireless/marvell/mwlwifi/utils.c -new file mode 100644 -index 000000000000..b73054a3f55e ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/utils.c -@@ -0,0 +1,576 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file implements common utility functions. */ -+ -+#include -+ -+#include "sysadpt.h" -+#include "core.h" -+#include "utils.h" -+ -+static unsigned short phy_rate[][5] = { -+ {2, 13, 15, 27, 30}, /* 0 */ -+ {4, 26, 29, 54, 60}, /* 1 */ -+ {11, 39, 43, 81, 90}, /* 2 */ -+ {22, 52, 58, 108, 120}, /* 3 */ -+ {44, 78, 87, 162, 180}, /* 4 */ -+ {12, 104, 115, 216, 240}, /* 5 */ -+ {18, 117, 130, 243, 270}, /* 6 */ -+ {24, 130, 144, 270, 300}, /* 7 */ -+ {36, 26, 29, 54, 60}, /* 8 */ -+ {48, 52, 58, 108, 120}, /* 9 */ -+ {72, 78, 87, 162, 180}, /* 10 */ -+ {96, 104, 116, 216, 240}, /* 11 */ -+ {108, 156, 173, 324, 360}, /* 12 */ -+ {0, 208, 231, 432, 480}, /* 13 */ -+ {0, 234, 260, 486, 540}, /* 14 */ -+ {0, 260, 289, 540, 600}, /* 15 */ -+ {0, 39, 43, 81, 90}, /* 16 */ -+ {0, 78, 87, 162, 180}, /* 17 */ -+ {0, 117, 130, 243, 270}, /* 18 */ -+ {0, 156, 173, 324, 360}, /* 19 */ -+ {0, 234, 260, 486, 540}, /* 20 */ -+ {0, 312, 347, 648, 720}, /* 21 */ -+ {0, 351, 390, 729, 810}, /* 22 */ -+ {0, 390, 433, 810, 900}, /* 23 */ -+}; -+ -+/* 20Mhz: Nss1_LGI, Nss1_SGI, Nss2_LGI, Nss2_SGI, Nss3_LGI, Nss3_SGI */ -+static unsigned short phy_rate_11ac20M[][6] = { -+ {13, 15, 26, 29, 39, 44}, /* 0 */ -+ {26, 29, 52, 58, 78, 87}, /* 1 */ -+ {39, 44, 78, 87, 117, 130}, /* 2 */ -+ {52, 58, 104, 116, 156, 174}, /* 3 */ -+ {78, 87, 156, 174, 234, 260}, /* 4 */ -+ {104, 116, 208, 231, 312, 347}, /* 5 */ -+ {117, 130, 234, 260, 351, 390}, /* 6 */ -+ {130, 145, 260, 289, 390, 434}, /* 7 */ -+ {156, 174, 312, 347, 468, 520}, /* 8 */ -+ /* Nss 1 and Nss 2 mcs9 not valid */ -+ {2, 2, 2, 2, 520, 578}, /* 9 */ -+}; -+ -+/* 40Mhz: Nss1_LGI, Nss1_SGI, Nss2_LGI, Nss2_SGI, Nss3_LGI, Nss3_SGI */ -+static unsigned short phy_rate_11ac40M[][6] = { -+ {27, 30, 54, 60, 81, 90}, /* 0 */ -+ {54, 60, 108, 120, 162, 180}, /* 1 */ -+ {81, 90, 162, 180, 243, 270}, /* 2 */ -+ {108, 120, 216, 240, 324, 360}, /* 3 */ -+ {162, 180, 324, 360, 486, 540}, /* 4 */ -+ {216, 240, 432, 480, 648, 720}, /* 5 */ -+ {243, 270, 486, 540, 729, 810}, /* 6 */ -+ {270, 300, 540, 600, 810, 900}, /* 7 */ -+ {324, 360, 648, 720, 972, 1080}, /* 8 */ -+ {360, 400, 720, 800, 1080, 1200}, /* 9 */ -+}; -+ -+/* 80Mhz: Nss1_LGI, Nss1_SGI, Nss2_LGI, Nss2_SGI, Nss3_LGI, Nss3_SGI */ -+static unsigned short phy_rate_11ac80M[][6] = { -+ {59, 65, 117, 130, 175, 195}, /* 0 */ -+ {117, 130, 234, 260, 351, 390}, /* 1 */ -+ {175, 195, 351, 390, 527, 585}, /* 2 */ -+ {234, 260, 468, 520, 702, 780}, /* 3 */ -+ {351, 390, 702, 780, 1053, 1170}, /* 4 */ -+ {468, 520, 936, 1040, 1404, 1560}, /* 5 */ -+ {527, 585, 1053, 1170, 2, 2}, /* 6, Nss 3 mcs6 not valid */ -+ {585, 650, 1170, 1300, 1755, 1950}, /* 7 */ -+ {702, 780, 1404, 1560, 2106, 2340}, /* 8 */ -+ {780, 867, 1560, 1733, 2340, 2600}, /* 9 */ -+}; -+ -+/* 160Mhz: Nss1_LGI, Nss1_SGI, Nss2_LGI, Nss2_SGI, Nss3_LGI, Nss3_SGI */ -+static unsigned short phy_rate_11ac160M[][6] = { -+ {117, 130, 234, 260, 351, 390}, /* 0 */ -+ {234, 260, 468, 520, 702, 780}, /* 1 */ -+ {351, 390, 702, 780, 1053, 1170}, /* 2 */ -+ {468, 520, 936, 1040, 1404, 1560}, /* 3 */ -+ {702, 780, 1404, 1560, 2106, 2340}, /* 4 */ -+ {936, 1040, 1872, 2080, 2808, 3120}, /* 5 */ -+ {1053, 1170, 2106, 2340, 3159, 3510}, /* 6 */ -+ {1170, 1300, 2340, 2600, 3510, 3900}, /* 7 */ -+ {1404, 1560, 2808, 3120, 4212, 4680}, /* 8 */ -+ {1560, 1733, 2130, 3467, 4680, 5200}, /* 9 */ -+}; -+ -+int utils_get_phy_rate(u8 format, u8 bandwidth, u8 short_gi, u8 mcs_id) -+{ -+ u8 index = 0; -+ u8 nss_11ac = 0; -+ u8 rate_11ac = 0; -+ -+ if (format == TX_RATE_FORMAT_11N) { -+ index = (bandwidth << 1) | short_gi; -+ index++; -+ } else if (format == TX_RATE_FORMAT_11AC) { -+ rate_11ac = mcs_id & 0xf; /* 11ac, mcs_id[3:0]: rate */ -+ nss_11ac = mcs_id >> 4; /* 11ac, mcs_id[6:4]: nss code */ -+ index = (nss_11ac << 1) | short_gi; -+ } -+ -+ if (format != TX_RATE_FORMAT_11AC) -+ return (phy_rate[mcs_id][index] / 2); -+ -+ if (bandwidth == TX_RATE_BANDWIDTH_20) -+ return (phy_rate_11ac20M[rate_11ac][index] / 2); -+ else if (bandwidth == TX_RATE_BANDWIDTH_40) -+ return (phy_rate_11ac40M[rate_11ac][index] / 2); -+ else if (bandwidth == TX_RATE_BANDWIDTH_80) -+ return (phy_rate_11ac80M[rate_11ac][index] / 2); -+ else -+ return (phy_rate_11ac160M[rate_11ac][index] / 2); -+} -+ -+u8 utils_get_rate_id(u8 rate) -+{ -+ switch (rate) { -+ case 10: /* 1 Mbit/s or 12 Mbit/s */ -+ return 0; -+ case 20: /* 2 Mbit/s */ -+ return 1; -+ case 55: /* 5.5 Mbit/s */ -+ return 2; -+ case 110: /* 11 Mbit/s */ -+ return 3; -+ case 220: /* 22 Mbit/s */ -+ return 4; -+ case 0xb: /* 6 Mbit/s */ -+ return 5; -+ case 0xf: /* 9 Mbit/s */ -+ return 6; -+ case 0xe: /* 18 Mbit/s */ -+ return 8; -+ case 0x9: /* 24 Mbit/s */ -+ return 9; -+ case 0xd: /* 36 Mbit/s */ -+ return 10; -+ case 0x8: /* 48 Mbit/s */ -+ return 11; -+ case 0xc: /* 54 Mbit/s */ -+ return 12; -+ case 0x7: /* 72 Mbit/s */ -+ return 13; -+ } -+ -+ return 0; -+} -+ -+u32 utils_get_init_tx_rate(struct mwl_priv *priv, struct ieee80211_conf *conf, -+ struct ieee80211_sta *sta) -+{ -+ u32 tx_rate; -+ u16 format, nss, bw, rate_mcs; -+ -+ if (sta->vht_cap.vht_supported) -+ format = TX_RATE_FORMAT_11AC; -+ else if (sta->ht_cap.ht_supported) -+ format = TX_RATE_FORMAT_11N; -+ else -+ format = TX_RATE_FORMAT_LEGACY; -+ -+ switch (priv->antenna_tx) { -+ case ANTENNA_TX_1: -+ nss = 1; -+ break; -+ case ANTENNA_TX_2: -+ nss = 2; -+ break; -+ case ANTENNA_TX_3: -+ case ANTENNA_TX_4_AUTO: -+ nss = 3; -+ break; -+ default: -+ nss = sta->rx_nss; -+ break; -+ } -+ if (nss > sta->rx_nss) -+ nss = sta->rx_nss; -+ -+ switch (conf->chandef.width) { -+ case NL80211_CHAN_WIDTH_20_NOHT: -+ case NL80211_CHAN_WIDTH_20: -+ bw = TX_RATE_BANDWIDTH_20; -+ break; -+ case NL80211_CHAN_WIDTH_40: -+ bw = TX_RATE_BANDWIDTH_40; -+ break; -+ case NL80211_CHAN_WIDTH_80: -+ bw = TX_RATE_BANDWIDTH_80; -+ break; -+ case NL80211_CHAN_WIDTH_160: -+ bw = TX_RATE_BANDWIDTH_160; -+ break; -+ default: -+ bw = sta->bandwidth; -+ break; -+ } -+ if (bw > sta->bandwidth) -+ bw = sta->bandwidth; -+ -+ switch (format) { -+ case TX_RATE_FORMAT_LEGACY: -+ rate_mcs = 12; /* ignore 11b */ -+ break; -+ case TX_RATE_FORMAT_11N: -+ rate_mcs = (nss * 8) - 1; -+ break; -+ default: -+ rate_mcs = ((nss - 1) << 4) | 8; -+ break; -+ } -+ -+ tx_rate = (format | (bw << MWL_TX_RATE_BANDWIDTH_SHIFT) | -+ (TX_RATE_INFO_SHORT_GI << MWL_TX_RATE_SHORTGI_SHIFT) | -+ (rate_mcs << MWL_TX_RATE_RATEIDMCS_SHIFT)); -+ -+ return tx_rate; -+} -+ -+struct mwl_vif *utils_find_vif_bss(struct mwl_priv *priv, u8 *bssid) -+{ -+ struct mwl_vif *mwl_vif; -+ -+ spin_lock_bh(&priv->vif_lock); -+ list_for_each_entry(mwl_vif, &priv->vif_list, list) { -+ if (ether_addr_equal(bssid, mwl_vif->bssid)) { -+ spin_unlock_bh(&priv->vif_lock); -+ return mwl_vif; -+ } -+ } -+ spin_unlock_bh(&priv->vif_lock); -+ -+ return NULL; -+} -+ -+struct mwl_sta *utils_find_sta(struct mwl_priv *priv, u8 *addr) -+{ -+ struct mwl_sta *sta_info; -+ struct ieee80211_sta *sta; -+ -+ spin_lock_bh(&priv->sta_lock); -+ list_for_each_entry(sta_info, &priv->sta_list, list) { -+ sta = container_of((void *)sta_info, struct ieee80211_sta, -+ drv_priv); -+ if (ether_addr_equal(addr, sta->addr)) { -+ spin_unlock_bh(&priv->sta_lock); -+ return sta_info; -+ } -+ } -+ spin_unlock_bh(&priv->sta_lock); -+ -+ return NULL; -+} -+ -+struct mwl_sta *utils_find_sta_by_aid(struct mwl_priv *priv, u16 aid) -+{ -+ struct mwl_sta *sta_info; -+ struct ieee80211_sta *sta; -+ -+ spin_lock_bh(&priv->sta_lock); -+ list_for_each_entry(sta_info, &priv->sta_list, list) { -+ sta = container_of((void *)sta_info, struct ieee80211_sta, -+ drv_priv); -+ if (sta->aid == aid) { -+ spin_unlock_bh(&priv->sta_lock); -+ return sta_info; -+ } -+ } -+ spin_unlock_bh(&priv->sta_lock); -+ -+ return NULL; -+} -+ -+struct mwl_sta *utils_find_sta_by_id(struct mwl_priv *priv, u16 stnid) -+{ -+ struct mwl_sta *sta_info; -+ -+ spin_lock_bh(&priv->sta_lock); -+ list_for_each_entry(sta_info, &priv->sta_list, list) { -+ if (sta_info->stnid == stnid) { -+ spin_unlock_bh(&priv->sta_lock); -+ return sta_info; -+ } -+ } -+ spin_unlock_bh(&priv->sta_lock); -+ -+ return NULL; -+} -+ -+void utils_dump_data_info(const char *prefix_str, const void *buf, size_t len) -+{ -+ print_hex_dump(KERN_INFO, prefix_str, DUMP_PREFIX_OFFSET, -+ 16, 1, buf, len, true); -+} -+ -+void utils_dump_data_debug(const char *prefix_str, const void *buf, size_t len) -+{ -+ print_hex_dump(KERN_DEBUG, prefix_str, DUMP_PREFIX_OFFSET, -+ 16, 1, buf, len, true); -+} -+ -+bool utils_is_non_amsdu_packet(const void *packet, bool mac80211) -+{ -+ const u8 *data = packet; -+ struct ieee80211_hdr *wh; -+ __be16 *protocol; -+ struct iphdr *iph; -+ struct udphdr *udph; -+ -+ if (mac80211) { -+ /* mac80211 packet */ -+ wh = (struct ieee80211_hdr *)data; -+ data += ieee80211_hdrlen(wh->frame_control) + 6; -+ protocol = (__be16 *)data; -+ } else { -+ /* mac802.3 packet */ -+ data += (2 * ETH_ALEN); -+ protocol = (__be16 *)data; -+ } -+ -+ if (*protocol == cpu_to_be16(ETH_P_PAE)) -+ return true; -+ -+ if (*protocol == htons(ETH_P_ARP)) -+ return true; -+ -+ if (*protocol == htons(ETH_P_IP)) { -+ data += sizeof(__be16); -+ iph = (struct iphdr *)data; -+ if (iph->protocol == IPPROTO_ICMP) -+ return true; -+ if (iph->protocol == IPPROTO_UDP) { -+ data += (iph->ihl * 4); -+ udph = (struct udphdr *)data; -+ if (((udph->source == htons(68)) && -+ (udph->dest == htons(67))) || -+ ((udph->source == htons(67)) && -+ (udph->dest == htons(68)))) -+ return true; -+ } -+ } -+ -+ return false; -+} -+ -+bool utils_is_arp(const void *packet, bool mac80211, u16 *arp_op) -+{ -+ const u8 *data = packet; -+ struct ieee80211_hdr *wh; -+ __be16 *protocol; -+ struct arphdr *arph; -+ -+ if (mac80211) { -+ /* mac80211 packet */ -+ wh = (struct ieee80211_hdr *)data; -+ data += ieee80211_hdrlen(wh->frame_control) + 6; -+ protocol = (__be16 *)data; -+ } else { -+ /* mac802.3 packet */ -+ data += (2 * ETH_ALEN); -+ protocol = (__be16 *)data; -+ } -+ -+ if (*protocol == htons(ETH_P_ARP)) { -+ data += sizeof(__be16); -+ arph = (struct arphdr *)data; -+ *arp_op = ntohs(arph->ar_op); -+ return true; -+ } -+ -+ return false; -+} -+ -+bool utils_is_icmp_echo(const void *packet, bool mac80211, u8 *type) -+{ -+ const u8 *data = packet; -+ struct ieee80211_hdr *wh; -+ __be16 *protocol; -+ struct iphdr *iph; -+ struct icmphdr *icmph; -+ -+ if (mac80211) { -+ /* mac80211 packet */ -+ wh = (struct ieee80211_hdr *)data; -+ data += ieee80211_hdrlen(wh->frame_control) + 6; -+ protocol = (__be16 *)data; -+ } else { -+ /* mac802.3 packet */ -+ data += (2 * ETH_ALEN); -+ protocol = (__be16 *)data; -+ } -+ -+ if (*protocol == htons(ETH_P_IP)) { -+ data += sizeof(__be16); -+ iph = (struct iphdr *)data; -+ if (iph->protocol == IPPROTO_ICMP) { -+ data += (iph->ihl * 4); -+ icmph = (struct icmphdr *)data; -+ *type = icmph->type; -+ return true; -+ } -+ } -+ -+ return false; -+} -+ -+bool utils_is_dhcp(const void *packet, bool mac80211, u8 *op, u8 *dhcp_client) -+{ -+ const u8 *data = packet; -+ struct ieee80211_hdr *wh; -+ __be16 *protocol; -+ struct iphdr *iph; -+ struct udphdr *udph; -+ -+ if (mac80211) { -+ /* mac80211 packet */ -+ wh = (struct ieee80211_hdr *)data; -+ data += ieee80211_hdrlen(wh->frame_control) + 6; -+ protocol = (__be16 *)data; -+ } else { -+ /* mac802.3 packet */ -+ data += (2 * ETH_ALEN); -+ protocol = (__be16 *)data; -+ } -+ -+ if (*protocol == htons(ETH_P_IP)) { -+ data += sizeof(__be16); -+ iph = (struct iphdr *)data; -+ if (iph->protocol == IPPROTO_UDP) { -+ data += (iph->ihl * 4); -+ udph = (struct udphdr *)data; -+ if (((udph->source == htons(68)) && -+ (udph->dest == htons(67))) || -+ ((udph->source == htons(67)) && -+ (udph->dest == htons(68)))) { -+ data += sizeof(struct udphdr); -+ *op = *data; -+ ether_addr_copy(dhcp_client, data + 28); -+ return true; -+ } -+ } -+ } -+ -+ return false; -+} -+ -+void utils_dump_arp(const void *packet, bool mac80211, size_t len) -+{ -+ const u8 *data = packet; -+ struct ieee80211_hdr *wh; -+ __be16 *protocol; -+ struct arphdr *arph; -+ -+ if (mac80211) { -+ /* mac80211 packet */ -+ wh = (struct ieee80211_hdr *)data; -+ data += ieee80211_hdrlen(wh->frame_control) + 6; -+ protocol = (__be16 *)data; -+ } else { -+ /* mac802.3 packet */ -+ data += (2 * ETH_ALEN); -+ protocol = (__be16 *)data; -+ } -+ -+ if (*protocol == htons(ETH_P_ARP)) { -+ data += sizeof(__be16); -+ arph = (struct arphdr *)data; -+ if (arph->ar_op == htons(ARPOP_REQUEST)) -+ utils_dump_data_info("ARP REQUEST: ", packet, len); -+ else if (arph->ar_op == htons(ARPOP_REPLY)) -+ utils_dump_data_info("ARP REPLY: ", packet, len); -+ } -+} -+ -+void utils_dump_icmp_echo(const void *packet, bool mac80211, size_t len) -+{ -+ const u8 *data = packet; -+ struct ieee80211_hdr *wh; -+ __be16 *protocol; -+ struct iphdr *iph; -+ struct icmphdr *icmph; -+ -+ if (mac80211) { -+ /* mac80211 packet */ -+ wh = (struct ieee80211_hdr *)data; -+ data += ieee80211_hdrlen(wh->frame_control) + 6; -+ protocol = (__be16 *)data; -+ } else { -+ /* mac802.3 packet */ -+ data += (2 * ETH_ALEN); -+ protocol = (__be16 *)data; -+ } -+ -+ if (*protocol == htons(ETH_P_IP)) { -+ data += sizeof(__be16); -+ iph = (struct iphdr *)data; -+ if (iph->protocol == IPPROTO_ICMP) { -+ data += (iph->ihl * 4); -+ icmph = (struct icmphdr *)data; -+ if (icmph->type == ICMP_ECHO) -+ utils_dump_data_info("ECHO REQUEST: ", -+ packet, len); -+ else if (icmph->type == ICMP_ECHOREPLY) -+ utils_dump_data_info("ECHO REPLY: ", -+ packet, len); -+ } -+ } -+} -+ -+void utils_dump_dhcp(const void *packet, bool mac80211, size_t len) -+{ -+ const u8 *data = packet; -+ struct ieee80211_hdr *wh; -+ __be16 *protocol; -+ struct iphdr *iph; -+ struct udphdr *udph; -+ const char *dhcp_op[8] = { -+ "DHCPDISCOVER", -+ "DHCPOFFER", -+ "DHCPREQUEST", -+ "DHCPDECLINE", -+ "DHCPACK", -+ "DHCPNAK", -+ "DHCPRELEASE", -+ "DHCPINFORM" -+ }; -+ -+ if (mac80211) { -+ /* mac80211 packet */ -+ wh = (struct ieee80211_hdr *)data; -+ data += ieee80211_hdrlen(wh->frame_control) + 6; -+ protocol = (__be16 *)data; -+ } else { -+ /* mac802.3 packet */ -+ data += (2 * ETH_ALEN); -+ protocol = (__be16 *)data; -+ } -+ -+ if (*protocol == htons(ETH_P_IP)) { -+ data += sizeof(__be16); -+ iph = (struct iphdr *)data; -+ if (iph->protocol == IPPROTO_UDP) { -+ data += (iph->ihl * 4); -+ udph = (struct udphdr *)data; -+ if (((udph->source == htons(68)) && -+ (udph->dest == htons(67))) || -+ ((udph->source == htons(67)) && -+ (udph->dest == htons(68)))) { -+ data += sizeof(struct udphdr); -+ utils_dump_data_info(dhcp_op[*data - 1], -+ packet, len); -+ } -+ } -+ } -+} -diff --git a/drivers/net/wireless/marvell/mwlwifi/utils.h b/drivers/net/wireless/marvell/mwlwifi/utils.h -new file mode 100644 -index 000000000000..4a292e990412 ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/utils.h -@@ -0,0 +1,158 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file defines common utility functions. */ -+ -+#ifndef _UTILS_H_ -+#define _UTILS_H_ -+ -+#include -+#include -+#include -+#include -+ -+/* DHCP message types */ -+#define DHCPDISCOVER 1 -+#define DHCPOFFER 2 -+#define DHCPREQUEST 3 -+#define DHCPDECLINE 4 -+#define DHCPACK 5 -+#define DHCPNAK 6 -+#define DHCPRELEASE 7 -+#define DHCPINFORM 8 -+ -+static inline int utils_tid_to_ac(u8 tid) -+{ -+ switch (tid) { -+ case 0: -+ case 3: -+ return IEEE80211_AC_BE; -+ case 1: -+ case 2: -+ return IEEE80211_AC_BK; -+ case 4: -+ case 5: -+ return IEEE80211_AC_VI; -+ case 6: -+ case 7: -+ return IEEE80211_AC_VO; -+ default: -+ break; -+ } -+ -+ return -1; -+} -+ -+static inline void utils_add_basic_rates(int band, struct sk_buff *skb) -+{ -+ struct ieee80211_mgmt *mgmt; -+ int len; -+ u8 *pos; -+ -+ mgmt = (struct ieee80211_mgmt *)skb->data; -+ len = skb->len - ieee80211_hdrlen(mgmt->frame_control); -+ len -= 4; -+ pos = (u8 *)cfg80211_find_ie(WLAN_EID_SUPP_RATES, -+ mgmt->u.assoc_req.variable, -+ len); -+ if (pos) { -+ pos++; -+ len = *pos++; -+ while (len) { -+ if (band == NL80211_BAND_2GHZ) { -+ if ((*pos == 2) || (*pos == 4) || -+ (*pos == 11) || (*pos == 22)) -+ *pos |= 0x80; -+ } else { -+ if ((*pos == 12) || (*pos == 24) || -+ (*pos == 48)) -+ *pos |= 0x80; -+ } -+ pos++; -+ len--; -+ } -+ } -+} -+ -+static inline int utils_assign_stnid(struct mwl_priv *priv, int macid, u16 aid) -+{ -+ int stnid; -+ int i; -+ -+ spin_lock_bh(&priv->stnid_lock); -+ stnid = priv->available_stnid; -+ if (stnid >= priv->stnid_num) { -+ spin_unlock_bh(&priv->stnid_lock); -+ return 0; -+ } -+ priv->stnid[stnid].macid = macid; -+ priv->stnid[stnid].aid = aid; -+ stnid++; -+ for (i = stnid; i < priv->stnid_num; i++) { -+ if (!priv->stnid[i].aid) -+ break; -+ } -+ priv->available_stnid = i; -+ spin_unlock_bh(&priv->stnid_lock); -+ return stnid; -+} -+ -+static inline void utils_free_stnid(struct mwl_priv *priv, u16 stnid) -+{ -+ spin_lock_bh(&priv->stnid_lock); -+ if (stnid && (stnid <= priv->stnid_num)) { -+ stnid--; -+ priv->stnid[stnid].macid = 0; -+ priv->stnid[stnid].aid = 0; -+ if (priv->available_stnid > stnid) -+ priv->available_stnid = stnid; -+ } -+ spin_unlock_bh(&priv->stnid_lock); -+} -+ -+int utils_get_phy_rate(u8 format, u8 bandwidth, u8 short_gi, u8 mcs_id); -+ -+u8 utils_get_rate_id(u8 rate); -+ -+u32 utils_get_init_tx_rate(struct mwl_priv *priv, struct ieee80211_conf *conf, -+ struct ieee80211_sta *sta); -+ -+struct mwl_vif *utils_find_vif_bss(struct mwl_priv *priv, u8 *bssid); -+ -+struct mwl_sta *utils_find_sta(struct mwl_priv *priv, u8 *addr); -+ -+struct mwl_sta *utils_find_sta_by_aid(struct mwl_priv *priv, u16 aid); -+ -+struct mwl_sta *utils_find_sta_by_id(struct mwl_priv *priv, u16 stnid); -+ -+void utils_dump_data_info(const char *prefix_str, const void *buf, size_t len); -+ -+void utils_dump_data_debug(const char *prefix_str, const void *buf, size_t len); -+ -+bool utils_is_non_amsdu_packet(const void *packet, bool mac80211); -+ -+bool utils_is_arp(const void *packet, bool mac80211, u16 *arp_op); -+ -+bool utils_is_icmp_echo(const void *packet, bool mac80211, u8 *type); -+ -+bool utils_is_dhcp(const void *packet, bool mac80211, u8 *op, u8 *dhcp_client); -+ -+void utils_dump_arp(const void *packet, bool mac80211, size_t len); -+ -+void utils_dump_icmp_echo(const void *packet, bool mac80211, size_t len); -+ -+void utils_dump_dhcp(const void *packet, bool mac80211, size_t len); -+ -+#endif /* _UTILS_H_ */ -diff --git a/drivers/net/wireless/marvell/mwlwifi/vendor_cmd.c b/drivers/net/wireless/marvell/mwlwifi/vendor_cmd.c -new file mode 100644 -index 000000000000..3e26fc42c225 ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/vendor_cmd.c -@@ -0,0 +1,136 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file implements vendor spcific functions. */ -+ -+#include -+#include -+#include -+ -+#include "sysadpt.h" -+#include "core.h" -+#include "utils.h" -+#include "hif/fwcmd.h" -+#include "vendor_cmd.h" -+ -+static const struct nla_policy mwl_vendor_attr_policy[NUM_MWL_VENDOR_ATTR] = { -+ [MWL_VENDOR_ATTR_BF_TYPE] = { .type = NLA_U8 }, -+}; -+ -+static int mwl_vendor_cmd_set_bf_type(struct wiphy *wiphy, -+ struct wireless_dev *wdev, -+ const void *data, int data_len) -+{ -+ struct ieee80211_hw *hw = wiphy_to_ieee80211_hw(wiphy); -+ struct mwl_priv *priv = hw->priv; -+ struct nlattr *tb[NUM_MWL_VENDOR_ATTR]; -+ int rc; -+ u8 val; -+ -+ if (priv->chip_type != MWL8964) -+ return -EPERM; -+ -+ rc = nla_parse(tb, MWL_VENDOR_ATTR_MAX, data, data_len, -+ mwl_vendor_attr_policy -+#if (defined(LINUX_BACKPORT) || (LINUX_VERSION_CODE >=KERNEL_VERSION(4,12,0))) -+ , NULL -+#endif -+ ); -+ if (rc) -+ return rc; -+ -+ if (!tb[MWL_VENDOR_ATTR_BF_TYPE]) -+ return -EINVAL; -+ -+ val = nla_get_u8(tb[MWL_VENDOR_ATTR_BF_TYPE]); -+ if ((val < TXBF_MODE_OFF) || (val > TXBF_MODE_BFMER_AUTO)) -+ return -EINVAL; -+ wiphy_debug(wiphy, "set bf_type: 0x%x\n", val); -+ -+ rc = mwl_fwcmd_set_bftype(hw, val); -+ if (!rc) -+ priv->bf_type = val; -+ -+ return rc; -+} -+ -+static int mwl_vendor_cmd_get_bf_type(struct wiphy *wiphy, -+ struct wireless_dev *wdev, -+ const void *data, int data_len) -+{ -+ struct ieee80211_hw *hw = wiphy_to_ieee80211_hw(wiphy); -+ struct mwl_priv *priv = hw->priv; -+ struct sk_buff *skb; -+ -+ if (priv->chip_type != MWL8964) -+ return -EPERM; -+ -+ skb = cfg80211_vendor_cmd_alloc_reply_skb(wiphy, 8); -+ if (!skb) -+ return -ENOMEM; -+ -+ nla_put_u8(skb, MWL_VENDOR_ATTR_BF_TYPE, priv->bf_type); -+ -+ return cfg80211_vendor_cmd_reply(skb); -+} -+ -+static const struct wiphy_vendor_command mwl_vendor_commands[] = { -+ { -+ .info = { .vendor_id = MRVL_OUI, -+ .subcmd = MWL_VENDOR_CMD_SET_BF_TYPE}, -+ .flags = WIPHY_VENDOR_CMD_NEED_NETDEV, -+ .doit = mwl_vendor_cmd_set_bf_type, -+ }, -+ { -+ .info = { .vendor_id = MRVL_OUI, -+ .subcmd = MWL_VENDOR_CMD_GET_BF_TYPE}, -+ .flags = WIPHY_VENDOR_CMD_NEED_NETDEV, -+ .doit = mwl_vendor_cmd_get_bf_type, -+ } -+}; -+ -+static const struct nl80211_vendor_cmd_info mwl_vendor_events[] = { -+ { -+ .vendor_id = MRVL_OUI, -+ .subcmd = MWL_VENDOR_EVENT_DRIVER_READY, -+ }, -+ { -+ .vendor_id = MRVL_OUI, -+ .subcmd = MWL_VENDOR_EVENT_DRIVER_START_REMOVE, -+ }, -+ { -+ .vendor_id = MRVL_OUI, -+ .subcmd = MWL_VENDOR_EVENT_CMD_TIMEOUT, -+ } -+}; -+ -+void vendor_cmd_register(struct wiphy *wiphy) -+{ -+ wiphy->vendor_commands = mwl_vendor_commands; -+ wiphy->n_vendor_commands = ARRAY_SIZE(mwl_vendor_commands); -+ wiphy->vendor_events = mwl_vendor_events; -+ wiphy->n_vendor_events = ARRAY_SIZE(mwl_vendor_events); -+} -+ -+void vendor_cmd_basic_event(struct wiphy *wiphy, int event_idx) -+{ -+ struct sk_buff *skb; -+ -+ skb = cfg80211_vendor_event_alloc(wiphy, NULL, 0, -+ event_idx, GFP_KERNEL); -+ -+ if (skb) -+ cfg80211_vendor_event(skb, GFP_KERNEL); -+} -diff --git a/drivers/net/wireless/marvell/mwlwifi/vendor_cmd.h b/drivers/net/wireless/marvell/mwlwifi/vendor_cmd.h -new file mode 100644 -index 000000000000..b6fdf70c22fb ---- /dev/null -+++ b/drivers/net/wireless/marvell/mwlwifi/vendor_cmd.h -@@ -0,0 +1,60 @@ -+/* -+ * Copyright (C) 2006-2018, Marvell International Ltd. -+ * -+ * This software file (the "File") is distributed by Marvell International -+ * Ltd. under the terms of the GNU General Public License Version 2, June 1991 -+ * (the "License"). You may use, redistribute and/or modify this File in -+ * accordance with the terms and conditions of the License, a copy of which -+ * is available by writing to the Free Software Foundation, Inc. -+ * -+ * THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE -+ * ARE EXPRESSLY DISCLAIMED. The License provides additional details about -+ * this warranty disclaimer. -+ */ -+ -+/* Description: This file defines vendor constants and register function. */ -+ -+#ifndef _VENDOR_CMD_H_ -+#define _VENDOR_CMD_H_ -+ -+#ifdef __KERNEL__ -+void vendor_cmd_register(struct wiphy *wiphy); -+void vendor_cmd_basic_event(struct wiphy *wiphy, int event_idx); -+#endif -+ -+#define MRVL_OUI 0x005043 -+ -+enum mwl_vendor_commands { -+ MWL_VENDOR_CMD_SET_BF_TYPE, -+ MWL_VENDOR_CMD_GET_BF_TYPE, -+ -+ /* add commands here, update the command in vendor_cmd.c */ -+ -+ __MWL_VENDOR_CMD_AFTER_LAST, -+ NUM_MWL_VENDOR_CMD = __MWL_VENDOR_CMD_AFTER_LAST, -+ MWL_VENDOR_CMD_MAX = __MWL_VENDOR_CMD_AFTER_LAST - 1 -+}; -+ -+enum mwl_vendor_attributes { -+ MWL_VENDOR_ATTR_NOT_USE, -+ MWL_VENDOR_ATTR_BF_TYPE, -+ -+ /* add attributes here, update the policy in vendor_cmd.c */ -+ -+ __MWL_VENDOR_ATTR_AFTER_LAST, -+ NUM_MWL_VENDOR_ATTR = __MWL_VENDOR_ATTR_AFTER_LAST, -+ MWL_VENDOR_ATTR_MAX = __MWL_VENDOR_ATTR_AFTER_LAST - 1 -+}; -+ -+enum mwl_vendor_events { -+ MWL_VENDOR_EVENT_DRIVER_READY, -+ MWL_VENDOR_EVENT_DRIVER_START_REMOVE, -+ MWL_VENDOR_EVENT_CMD_TIMEOUT, -+ -+ __MWL_VENDOR_EVENT_AFTER_LAST, -+ NUM_MWL_VENDOR_EVENT = __MWL_VENDOR_EVENT_AFTER_LAST, -+ MWL_VENDOR_EVENT_MAX = __MWL_VENDOR_EVENT_AFTER_LAST - 1 -+}; -+ -+#endif /* _VENDOR_CMD_H_ */ --- -2.23.0 - diff --git a/patches/5.2/0001-surface-acpi.patch b/patches/5.3/0001-surface-acpi.patch similarity index 99% rename from patches/5.2/0001-surface-acpi.patch rename to patches/5.3/0001-surface-acpi.patch index 1e9cb7401..e773dba8a 100644 --- a/patches/5.2/0001-surface-acpi.patch +++ b/patches/5.3/0001-surface-acpi.patch @@ -1,7 +1,7 @@ -From d4d386abdc4817e58e10db9971ee6af3e6da2e07 Mon Sep 17 00:00:00 2001 -From: Maximilian Luz -Date: Fri, 26 Jul 2019 04:44:14 +0200 -Subject: [PATCH 01/12] surface-acpi +From 227d9d1cbb636b51b99b049a600668cc7653cbf0 Mon Sep 17 00:00:00 2001 +From: qzed +Date: Mon, 26 Aug 2019 01:11:08 +0200 +Subject: [PATCH 1/9] surface-acpi --- drivers/acpi/acpica/dsopcode.c | 2 +- @@ -59,10 +59,10 @@ index d3d2dbfba680..0b7f617a6e9b 100644 buffer_desc = acpi_ut_create_buffer_object(buffer_length); if (!buffer_desc) { diff --git a/drivers/platform/x86/Kconfig b/drivers/platform/x86/Kconfig -index 7c2fd1d72e18..c00cb830914a 100644 +index 1b67bb578f9f..0c1fc3655b5c 100644 --- a/drivers/platform/x86/Kconfig +++ b/drivers/platform/x86/Kconfig -@@ -623,6 +623,103 @@ config THINKPAD_ACPI_HOTKEY_POLL +@@ -620,6 +620,103 @@ config THINKPAD_ACPI_HOTKEY_POLL If you are not sure, say Y here. The driver enables polling only if it is strictly necessary to do so. @@ -167,7 +167,7 @@ index 7c2fd1d72e18..c00cb830914a 100644 tristate "Thinkpad Hard Drive Active Protection System (hdaps)" depends on INPUT diff --git a/drivers/platform/x86/Makefile b/drivers/platform/x86/Makefile -index 87b0069bd781..8b12c19dc165 100644 +index 415104033060..662e595ae13f 100644 --- a/drivers/platform/x86/Makefile +++ b/drivers/platform/x86/Makefile @@ -39,6 +39,7 @@ obj-$(CONFIG_TC1100_WMI) += tc1100-wmi.o @@ -180,7 +180,7 @@ index 87b0069bd781..8b12c19dc165 100644 obj-$(CONFIG_FUJITSU_TABLET) += fujitsu-tablet.o diff --git a/drivers/platform/x86/surface_acpi.c b/drivers/platform/x86/surface_acpi.c new file mode 100644 -index 000000000000..f62aecb0df69 +index 000000000000..ea417dbf4793 --- /dev/null +++ b/drivers/platform/x86/surface_acpi.c @@ -0,0 +1,3913 @@ @@ -1801,7 +1801,7 @@ index 000000000000..f62aecb0df69 +{ + struct surfacegen5_ec *ec; + unsigned long flags; -+ int status; ++ //int status; + + ec = surfacegen5_ec_acquire_init(); + if (!ec) { @@ -1811,10 +1811,10 @@ index 000000000000..f62aecb0df69 + surfacegen5_ssh_sysfs_unregister(&serdev->dev); + + // suspend EC and disable events -+ status = surfacegen5_ssh_ec_suspend(ec); -+ if (status) { -+ dev_err(&serdev->dev, "failed to suspend EC: %d\n", status); -+ } ++ //status = surfacegen5_ssh_ec_suspend(ec); ++ //if (status) { ++ // dev_err(&serdev->dev, "failed to suspend EC: %d\n", status); ++ //} + + // make sure all events (received up to now) have been properly handled + flush_workqueue(ec->events.queue_ack); diff --git a/patches/5.2/0003-buttons.patch b/patches/5.3/0002-buttons.patch similarity index 98% rename from patches/5.2/0003-buttons.patch rename to patches/5.3/0002-buttons.patch index 59c41885d..b349e99c5 100644 --- a/patches/5.2/0003-buttons.patch +++ b/patches/5.3/0002-buttons.patch @@ -1,7 +1,7 @@ -From 28bc715183ef72470ec7a0d5df3b572814a18500 Mon Sep 17 00:00:00 2001 +From 89c677637073842ed8a61000ac4cccaf9cee4f00 Mon Sep 17 00:00:00 2001 From: Maximilian Luz -Date: Fri, 26 Jul 2019 04:45:10 +0200 -Subject: [PATCH 03/12] buttons +Date: Sat, 27 Jul 2019 17:51:37 +0200 +Subject: [PATCH 2/9] buttons --- drivers/input/misc/Kconfig | 6 +- diff --git a/patches/5.2/0012-surfacebook2-dgpu.patch b/patches/5.3/0003-surfacebook2-dgpu.patch similarity index 97% rename from patches/5.2/0012-surfacebook2-dgpu.patch rename to patches/5.3/0003-surfacebook2-dgpu.patch index 1ddd2a576..39cadde73 100644 --- a/patches/5.2/0012-surfacebook2-dgpu.patch +++ b/patches/5.3/0003-surfacebook2-dgpu.patch @@ -1,7 +1,7 @@ -From 687a19de3665bf1408b8917796987399cd4bd4f1 Mon Sep 17 00:00:00 2001 +From 3dd82efff79f1291a81dba88cb66421a2c5630d0 Mon Sep 17 00:00:00 2001 From: Maximilian Luz -Date: Fri, 26 Jul 2019 04:47:27 +0200 -Subject: [PATCH 12/12] surfacebook2-dgpu +Date: Tue, 2 Jul 2019 22:17:46 +0200 +Subject: [PATCH 3/9] surfacebook2 dgpu --- drivers/platform/x86/Kconfig | 9 + @@ -11,10 +11,10 @@ Subject: [PATCH 12/12] surfacebook2-dgpu create mode 100644 drivers/platform/x86/surfacebook2_dgpu_hps.c diff --git a/drivers/platform/x86/Kconfig b/drivers/platform/x86/Kconfig -index 04421fe566ba..cb0a53da4de1 100644 +index 0c1fc3655b5c..c8abd3d70461 100644 --- a/drivers/platform/x86/Kconfig +++ b/drivers/platform/x86/Kconfig -@@ -484,6 +484,15 @@ config SURFACE3_WMI +@@ -481,6 +481,15 @@ config SURFACE3_WMI To compile this driver as a module, choose M here: the module will be called surface3-wmi. @@ -31,7 +31,7 @@ index 04421fe566ba..cb0a53da4de1 100644 tristate "ThinkPad ACPI Laptop Extras" depends on ACPI diff --git a/drivers/platform/x86/Makefile b/drivers/platform/x86/Makefile -index 58b07217c3cf..d18d3dcc5749 100644 +index 662e595ae13f..c33d76577b5a 100644 --- a/drivers/platform/x86/Makefile +++ b/drivers/platform/x86/Makefile @@ -49,6 +49,7 @@ obj-$(CONFIG_ACPI_WMI) += wmi.o diff --git a/patches/5.2/0006-hid.patch b/patches/5.3/0004-hid.patch similarity index 93% rename from patches/5.2/0006-hid.patch rename to patches/5.3/0004-hid.patch index 5fb7bf8dd..d10a799aa 100644 --- a/patches/5.2/0006-hid.patch +++ b/patches/5.3/0004-hid.patch @@ -1,7 +1,7 @@ -From 6f167c509db37afa78770af611d6e2fb2e764607 Mon Sep 17 00:00:00 2001 -From: Maximilian Luz -Date: Fri, 26 Jul 2019 04:45:42 +0200 -Subject: [PATCH 06/12] hid +From c54d91c901e7d6f42ea883f3f2d36d7b8c7b3698 Mon Sep 17 00:00:00 2001 +From: qzed +Date: Tue, 17 Sep 2019 17:16:23 +0200 +Subject: [PATCH 4/9] hid --- drivers/hid/hid-ids.h | 21 +++++++++---- @@ -11,10 +11,10 @@ Subject: [PATCH 06/12] hid 4 files changed, 86 insertions(+), 6 deletions(-) diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h -index 76aa474e92c1..2ce782095c63 100644 +index 0a00be19f7a0..7a898dca9c36 100644 --- a/drivers/hid/hid-ids.h +++ b/drivers/hid/hid-ids.h -@@ -822,11 +822,22 @@ +@@ -823,11 +823,22 @@ #define USB_DEVICE_ID_MS_DIGITAL_MEDIA_3KV1 0x0732 #define USB_DEVICE_ID_MS_DIGITAL_MEDIA_600 0x0750 #define USB_DEVICE_ID_MS_COMFORT_MOUSE_4500 0x076c @@ -57,10 +57,10 @@ index 8b3a922bdad3..0290a16881e5 100644 .driver_data = MS_PRESENTER }, { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_MICROSOFT, 0x091B), diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c -index 03448d3a29f2..09ca4b1f2797 100644 +index b603c14d043b..008e6707f467 100644 --- a/drivers/hid/hid-multitouch.c +++ b/drivers/hid/hid-multitouch.c -@@ -1983,6 +1983,63 @@ static const struct hid_device_id mt_devices[] = { +@@ -1967,6 +1967,63 @@ static const struct hid_device_id mt_devices[] = { HID_USB_DEVICE(USB_VENDOR_ID_LG, USB_DEVICE_ID_LG_MELFAS_MT) }, @@ -125,10 +125,10 @@ index 03448d3a29f2..09ca4b1f2797 100644 { .driver_data = MT_CLS_CONFIDENCE_MINUS_ONE, MT_USB_DEVICE(USB_VENDOR_ID_ASUS, diff --git a/drivers/hid/hid-quirks.c b/drivers/hid/hid-quirks.c -index 4fe2c3ab76f9..c9feb3f76e78 100644 +index 166f41f3173b..d8a595a97dc1 100644 --- a/drivers/hid/hid-quirks.c +++ b/drivers/hid/hid-quirks.c -@@ -112,6 +112,17 @@ static const struct hid_device_id hid_quirks[] = { +@@ -113,6 +113,17 @@ static const struct hid_device_id hid_quirks[] = { { HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_SURFACE_PRO_2), HID_QUIRK_NO_INIT_REPORTS }, { HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TOUCH_COVER_2), HID_QUIRK_NO_INIT_REPORTS }, { HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_2), HID_QUIRK_NO_INIT_REPORTS }, diff --git a/patches/5.2/0009-surface3-power.patch b/patches/5.3/0005-surface3-power.patch similarity index 97% rename from patches/5.2/0009-surface3-power.patch rename to patches/5.3/0005-surface3-power.patch index f4cdb620a..4c697bd4e 100644 --- a/patches/5.2/0009-surface3-power.patch +++ b/patches/5.3/0005-surface3-power.patch @@ -1,7 +1,7 @@ -From 8dcbe757ce810b137ddaa746598b261de98bd6f3 Mon Sep 17 00:00:00 2001 -From: Maximilian Luz -Date: Fri, 26 Jul 2019 04:46:48 +0200 -Subject: [PATCH 09/12] surface3-power +From af0373ecee9130cd0ac50a4f9c04f5e7fb357b93 Mon Sep 17 00:00:00 2001 +From: qzed +Date: Tue, 17 Sep 2019 17:17:56 +0200 +Subject: [PATCH 5/9] surface3 power --- drivers/platform/x86/Kconfig | 7 + @@ -11,10 +11,10 @@ Subject: [PATCH 09/12] surface3-power create mode 100644 drivers/platform/x86/surface3_power.c diff --git a/drivers/platform/x86/Kconfig b/drivers/platform/x86/Kconfig -index c00cb830914a..04421fe566ba 100644 +index c8abd3d70461..c381d14dea20 100644 --- a/drivers/platform/x86/Kconfig +++ b/drivers/platform/x86/Kconfig -@@ -1301,6 +1301,13 @@ config SURFACE_3_BUTTON +@@ -1315,6 +1315,13 @@ config SURFACE_3_BUTTON ---help--- This driver handles the power/home/volume buttons on the Microsoft Surface 3 tablet. @@ -29,10 +29,10 @@ index c00cb830914a..04421fe566ba 100644 tristate "Intel P-Unit IPC Driver" ---help--- diff --git a/drivers/platform/x86/Makefile b/drivers/platform/x86/Makefile -index 8b12c19dc165..58b07217c3cf 100644 +index c33d76577b5a..5c88172c0649 100644 --- a/drivers/platform/x86/Makefile +++ b/drivers/platform/x86/Makefile -@@ -85,6 +85,7 @@ obj-$(CONFIG_INTEL_PMC_IPC) += intel_pmc_ipc.o +@@ -87,6 +87,7 @@ obj-$(CONFIG_INTEL_PMC_IPC) += intel_pmc_ipc.o obj-$(CONFIG_TOUCHSCREEN_DMI) += touchscreen_dmi.o obj-$(CONFIG_SURFACE_PRO3_BUTTON) += surfacepro3_button.o obj-$(CONFIG_SURFACE_3_BUTTON) += surface3_button.o diff --git a/patches/5.2/0011-surface-lte.patch b/patches/5.3/0006-surface-lte.patch similarity index 79% rename from patches/5.2/0011-surface-lte.patch rename to patches/5.3/0006-surface-lte.patch index e4aaec3b6..e5c95689a 100644 --- a/patches/5.2/0011-surface-lte.patch +++ b/patches/5.3/0006-surface-lte.patch @@ -1,7 +1,7 @@ -From 332cb8daac70868fcec9e6c8c698e6a9318cfba6 Mon Sep 17 00:00:00 2001 -From: Maximilian Luz -Date: Fri, 26 Jul 2019 04:47:13 +0200 -Subject: [PATCH 11/12] surface-lte +From 28dd93a9ee6ed705ccd1c316a45ceef7c2f31580 Mon Sep 17 00:00:00 2001 +From: qzed +Date: Tue, 17 Sep 2019 17:21:43 +0200 +Subject: [PATCH 6/9] surface lte --- drivers/usb/serial/qcserial.c | 1 + diff --git a/patches/5.2/0008-wifi.patch b/patches/5.3/0007-wifi.patch similarity index 81% rename from patches/5.2/0008-wifi.patch rename to patches/5.3/0007-wifi.patch index 315889110..94c0b2df9 100644 --- a/patches/5.2/0008-wifi.patch +++ b/patches/5.3/0007-wifi.patch @@ -1,7 +1,7 @@ -From a8ef09ff10b30d17cff5057692bfdd60021a1a49 Mon Sep 17 00:00:00 2001 -From: Maximilian Luz -Date: Fri, 26 Jul 2019 04:46:16 +0200 -Subject: [PATCH 08/12] wifi +From 876823d3e53e97c8a5bc69ca30675fbc776d2a10 Mon Sep 17 00:00:00 2001 +From: qzed +Date: Wed, 18 Sep 2019 03:18:25 +0200 +Subject: [PATCH 7/9] wifi --- drivers/net/wireless/marvell/mwifiex/11n_aggr.c | 3 +-- @@ -12,17 +12,17 @@ Subject: [PATCH 08/12] wifi drivers/net/wireless/marvell/mwifiex/main.h | 2 ++ drivers/net/wireless/marvell/mwifiex/pcie.c | 9 +++++++++ drivers/net/wireless/marvell/mwifiex/sta_cmd.c | 4 ++-- - .../net/wireless/marvell/mwifiex/sta_cmdresp.c | 11 ++++++++--- + .../net/wireless/marvell/mwifiex/sta_cmdresp.c | 10 +++++++--- drivers/net/wireless/marvell/mwifiex/usb.c | 2 ++ scripts/leaking_addresses.pl | 0 - 11 files changed, 48 insertions(+), 16 deletions(-) + 11 files changed, 47 insertions(+), 16 deletions(-) mode change 100755 => 100644 scripts/leaking_addresses.pl diff --git a/drivers/net/wireless/marvell/mwifiex/11n_aggr.c b/drivers/net/wireless/marvell/mwifiex/11n_aggr.c -index 042a1d07f686..fc9041f58e9f 100644 +index 088612438530..4386e657dfdb 100644 --- a/drivers/net/wireless/marvell/mwifiex/11n_aggr.c +++ b/drivers/net/wireless/marvell/mwifiex/11n_aggr.c -@@ -200,8 +200,7 @@ mwifiex_11n_aggregate_pkt(struct mwifiex_private *priv, +@@ -198,8 +198,7 @@ mwifiex_11n_aggregate_pkt(struct mwifiex_private *priv, do { /* Check if AMSDU can accommodate this MSDU */ @@ -33,7 +33,7 @@ index 042a1d07f686..fc9041f58e9f 100644 skb_src = skb_dequeue(&pra_list->skb_head); diff --git a/drivers/net/wireless/marvell/mwifiex/cfg80211.c b/drivers/net/wireless/marvell/mwifiex/cfg80211.c -index e11a4bb67172..c3461a203deb 100644 +index d89684168500..1545bae9d6cf 100644 --- a/drivers/net/wireless/marvell/mwifiex/cfg80211.c +++ b/drivers/net/wireless/marvell/mwifiex/cfg80211.c @@ -437,7 +437,10 @@ mwifiex_cfg80211_set_power_mgmt(struct wiphy *wiphy, @@ -49,10 +49,10 @@ index e11a4bb67172..c3461a203deb 100644 return mwifiex_drv_set_power(priv, &ps_mode); } diff --git a/drivers/net/wireless/marvell/mwifiex/cmdevt.c b/drivers/net/wireless/marvell/mwifiex/cmdevt.c -index 8c35441fd9b7..71872139931e 100644 +index e8788c35a453..82d25b3ca914 100644 --- a/drivers/net/wireless/marvell/mwifiex/cmdevt.c +++ b/drivers/net/wireless/marvell/mwifiex/cmdevt.c -@@ -1006,6 +1006,7 @@ mwifiex_cmd_timeout_func(struct timer_list *t) +@@ -1004,6 +1004,7 @@ mwifiex_cmd_timeout_func(struct timer_list *t) if (cmd_node->wait_q_enabled) { adapter->cmd_wait_q.status = -ETIMEDOUT; mwifiex_cancel_pending_ioctl(adapter); @@ -60,7 +60,7 @@ index 8c35441fd9b7..71872139931e 100644 } } if (adapter->hw_status == MWIFIEX_HW_STATUS_INITIALIZING) { -@@ -1013,11 +1014,11 @@ mwifiex_cmd_timeout_func(struct timer_list *t) +@@ -1011,11 +1012,11 @@ mwifiex_cmd_timeout_func(struct timer_list *t) return; } @@ -76,7 +76,7 @@ index 8c35441fd9b7..71872139931e 100644 } void -@@ -1583,6 +1584,7 @@ int mwifiex_ret_get_hw_spec(struct mwifiex_private *priv, +@@ -1578,6 +1579,7 @@ int mwifiex_ret_get_hw_spec(struct mwifiex_private *priv, adapter->key_api_minor_ver); break; case FW_API_VER_ID: @@ -97,7 +97,7 @@ index 1fb76d2f5d3f..fb32379da99d 100644 struct hw_spec_api_rev { diff --git a/drivers/net/wireless/marvell/mwifiex/main.c b/drivers/net/wireless/marvell/mwifiex/main.c -index f6da8edab7f1..51a65f26206b 100644 +index a9657ae6d782..ba99d84a31ef 100644 --- a/drivers/net/wireless/marvell/mwifiex/main.c +++ b/drivers/net/wireless/marvell/mwifiex/main.c @@ -163,6 +163,7 @@ void mwifiex_queue_main_work(struct mwifiex_adapter *adapter) @@ -108,21 +108,19 @@ index f6da8edab7f1..51a65f26206b 100644 spin_unlock_irqrestore(&adapter->main_proc_lock, flags); } else { spin_unlock_irqrestore(&adapter->main_proc_lock, flags); -@@ -171,18 +172,20 @@ void mwifiex_queue_main_work(struct mwifiex_adapter *adapter) +@@ -171,16 +172,18 @@ void mwifiex_queue_main_work(struct mwifiex_adapter *adapter) } EXPORT_SYMBOL_GPL(mwifiex_queue_main_work); -static void mwifiex_queue_rx_work(struct mwifiex_adapter *adapter) +void mwifiex_queue_rx_work(struct mwifiex_adapter *adapter) { - unsigned long flags; - - spin_lock_irqsave(&adapter->rx_proc_lock, flags); + spin_lock_bh(&adapter->rx_proc_lock); if (adapter->rx_processing) { + adapter->more_rx_task_flag = true; - spin_unlock_irqrestore(&adapter->rx_proc_lock, flags); + spin_unlock_bh(&adapter->rx_proc_lock); } else { - spin_unlock_irqrestore(&adapter->rx_proc_lock, flags); + spin_unlock_bh(&adapter->rx_proc_lock); queue_work(adapter->rx_workqueue, &adapter->rx_work); } } @@ -130,35 +128,35 @@ index f6da8edab7f1..51a65f26206b 100644 static int mwifiex_process_rx(struct mwifiex_adapter *adapter) { -@@ -192,6 +195,7 @@ static int mwifiex_process_rx(struct mwifiex_adapter *adapter) +@@ -189,6 +192,7 @@ static int mwifiex_process_rx(struct mwifiex_adapter *adapter) - spin_lock_irqsave(&adapter->rx_proc_lock, flags); + spin_lock_bh(&adapter->rx_proc_lock); if (adapter->rx_processing || adapter->rx_locked) { + adapter->more_rx_task_flag = true; - spin_unlock_irqrestore(&adapter->rx_proc_lock, flags); + spin_unlock_bh(&adapter->rx_proc_lock); goto exit_rx_proc; } else { -@@ -199,6 +203,7 @@ static int mwifiex_process_rx(struct mwifiex_adapter *adapter) - spin_unlock_irqrestore(&adapter->rx_proc_lock, flags); +@@ -196,6 +200,7 @@ static int mwifiex_process_rx(struct mwifiex_adapter *adapter) + spin_unlock_bh(&adapter->rx_proc_lock); } +rx_process_start: /* Check for Rx data */ while ((skb = skb_dequeue(&adapter->rx_data_q))) { atomic_dec(&adapter->rx_pending); -@@ -220,6 +225,11 @@ static int mwifiex_process_rx(struct mwifiex_adapter *adapter) +@@ -217,6 +222,11 @@ static int mwifiex_process_rx(struct mwifiex_adapter *adapter) } } - spin_lock_irqsave(&adapter->rx_proc_lock, flags); + spin_lock_bh(&adapter->rx_proc_lock); + if (adapter->more_rx_task_flag) { + adapter->more_rx_task_flag = false; -+ spin_unlock_irqrestore(&adapter->rx_proc_lock, flags); ++ spin_unlock_bh(&adapter->rx_proc_lock); + goto rx_process_start; + } adapter->rx_processing = false; - spin_unlock_irqrestore(&adapter->rx_proc_lock, flags); + spin_unlock_bh(&adapter->rx_proc_lock); -@@ -283,11 +293,10 @@ int mwifiex_main_process(struct mwifiex_adapter *adapter) +@@ -280,11 +290,10 @@ int mwifiex_main_process(struct mwifiex_adapter *adapter) mwifiex_process_hs_config(adapter); if (adapter->if_ops.process_int_status) adapter->if_ops.process_int_status(adapter); @@ -173,7 +171,7 @@ index f6da8edab7f1..51a65f26206b 100644 if ((adapter->ps_state == PS_STATE_SLEEP) && (adapter->pm_wakeup_card_req && diff --git a/drivers/net/wireless/marvell/mwifiex/main.h b/drivers/net/wireless/marvell/mwifiex/main.h -index e39bb5c42c9a..8ec3275dad6d 100644 +index 095837fba300..5dca5c25c601 100644 --- a/drivers/net/wireless/marvell/mwifiex/main.h +++ b/drivers/net/wireless/marvell/mwifiex/main.h @@ -909,6 +909,7 @@ struct mwifiex_adapter { @@ -193,7 +191,7 @@ index e39bb5c42c9a..8ec3275dad6d 100644 int cmd_type, struct mwifiex_ds_wakeup_reason *wakeup_reason); diff --git a/drivers/net/wireless/marvell/mwifiex/pcie.c b/drivers/net/wireless/marvell/mwifiex/pcie.c -index 3fe81b2a929a..6e734a83e6bf 100644 +index b54f73e3d508..32502f443b9a 100644 --- a/drivers/net/wireless/marvell/mwifiex/pcie.c +++ b/drivers/net/wireless/marvell/mwifiex/pcie.c @@ -1743,6 +1743,15 @@ static int mwifiex_pcie_process_cmd_complete(struct mwifiex_adapter *adapter) @@ -228,29 +226,28 @@ index 4ed10cf82f9a..485360e8534b 100644 "deepsleep enabled=0(default), deepsleep disabled=1"); /* diff --git a/drivers/net/wireless/marvell/mwifiex/sta_cmdresp.c b/drivers/net/wireless/marvell/mwifiex/sta_cmdresp.c -index 24b33e20e7a9..51d0f34625e1 100644 +index 20c206da0631..0e58da83417c 100644 --- a/drivers/net/wireless/marvell/mwifiex/sta_cmdresp.c +++ b/drivers/net/wireless/marvell/mwifiex/sta_cmdresp.c -@@ -48,9 +48,14 @@ mwifiex_process_cmdresp_error(struct mwifiex_private *priv, +@@ -47,9 +47,13 @@ mwifiex_process_cmdresp_error(struct mwifiex_private *priv, + struct mwifiex_adapter *adapter = priv->adapter; struct host_cmd_ds_802_11_ps_mode_enh *pm; - unsigned long flags; - mwifiex_dbg(adapter, ERROR, - "CMD_RESP: cmd %#x error, result=%#x\n", - resp->command, resp->result); -+ if (resp->command == 271 && resp->result == 2){ ++ if (resp->command == 271 && resp->result == 2) { + // ignore this command as the firmware does not support it -+ } -+ else { ++ } else { + mwifiex_dbg(adapter, ERROR, -+ "CMD_RESP: cmd %#x error, result=%#x\n", -+ resp->command, resp->result); ++ "CMD_RESP: cmd %#x error, result=%#x\n", ++ resp->command, resp->result); + } if (adapter->curr_cmd->wait_q_enabled) adapter->cmd_wait_q.status = -1; diff --git a/drivers/net/wireless/marvell/mwifiex/usb.c b/drivers/net/wireless/marvell/mwifiex/usb.c -index d445acc4786b..ae8e60cc17cb 100644 +index c2365eeb7016..0a219ba378dd 100644 --- a/drivers/net/wireless/marvell/mwifiex/usb.c +++ b/drivers/net/wireless/marvell/mwifiex/usb.c @@ -144,6 +144,8 @@ static int mwifiex_usb_recv(struct mwifiex_adapter *adapter, diff --git a/patches/5.3/0008-use-legacy-i915-driver.patch b/patches/5.3/0008-use-legacy-i915-driver.patch new file mode 100644 index 000000000..a4df9186e --- /dev/null +++ b/patches/5.3/0008-use-legacy-i915-driver.patch @@ -0,0 +1,247070 @@ +From 697b2681d367cb0258d4e4b272bbc3e64b587b09 Mon Sep 17 00:00:00 2001 +From: Dorian Stoll +Date: Mon, 16 Sep 2019 04:10:51 +0200 +Subject: [PATCH 8/9] use legacy i915 driver + +--- + drivers/gpu/drm/Kconfig | 2 +- + drivers/gpu/drm/Makefile | 2 +- + drivers/gpu/drm/i915_legacy/.gitignore | 1 + + drivers/gpu/drm/i915_legacy/Kconfig | 135 + + drivers/gpu/drm/i915_legacy/Kconfig.debug | 183 + + drivers/gpu/drm/i915_legacy/Makefile | 212 + + .../gpu/drm/i915_legacy/Makefile.header-test | 47 + + drivers/gpu/drm/i915_legacy/dvo.h | 138 + + drivers/gpu/drm/i915_legacy/dvo_ch7017.c | 414 + + drivers/gpu/drm/i915_legacy/dvo_ch7xxx.c | 366 + + drivers/gpu/drm/i915_legacy/dvo_ivch.c | 502 + + drivers/gpu/drm/i915_legacy/dvo_ns2501.c | 709 + + drivers/gpu/drm/i915_legacy/dvo_sil164.c | 279 + + drivers/gpu/drm/i915_legacy/dvo_tfp410.c | 318 + + drivers/gpu/drm/i915_legacy/gvt/Makefile | 9 + + drivers/gpu/drm/i915_legacy/gvt/aperture_gm.c | 359 + + drivers/gpu/drm/i915_legacy/gvt/cfg_space.c | 424 + + drivers/gpu/drm/i915_legacy/gvt/cmd_parser.c | 2998 +++ + drivers/gpu/drm/i915_legacy/gvt/cmd_parser.h | 49 + + drivers/gpu/drm/i915_legacy/gvt/debug.h | 65 + + drivers/gpu/drm/i915_legacy/gvt/debugfs.c | 268 + + drivers/gpu/drm/i915_legacy/gvt/display.c | 531 + + drivers/gpu/drm/i915_legacy/gvt/display.h | 209 + + drivers/gpu/drm/i915_legacy/gvt/dmabuf.c | 561 + + drivers/gpu/drm/i915_legacy/gvt/dmabuf.h | 67 + + drivers/gpu/drm/i915_legacy/gvt/edid.c | 576 + + drivers/gpu/drm/i915_legacy/gvt/edid.h | 150 + + drivers/gpu/drm/i915_legacy/gvt/execlist.c | 567 + + drivers/gpu/drm/i915_legacy/gvt/execlist.h | 185 + + drivers/gpu/drm/i915_legacy/gvt/fb_decoder.c | 507 + + drivers/gpu/drm/i915_legacy/gvt/fb_decoder.h | 169 + + drivers/gpu/drm/i915_legacy/gvt/firmware.c | 276 + + drivers/gpu/drm/i915_legacy/gvt/gtt.c | 2818 +++ + drivers/gpu/drm/i915_legacy/gvt/gtt.h | 280 + + drivers/gpu/drm/i915_legacy/gvt/gvt.c | 453 + + drivers/gpu/drm/i915_legacy/gvt/gvt.h | 694 + + drivers/gpu/drm/i915_legacy/gvt/handlers.c | 3588 ++++ + drivers/gpu/drm/i915_legacy/gvt/hypercall.h | 78 + + drivers/gpu/drm/i915_legacy/gvt/interrupt.c | 710 + + drivers/gpu/drm/i915_legacy/gvt/interrupt.h | 233 + + drivers/gpu/drm/i915_legacy/gvt/kvmgt.c | 2075 ++ + drivers/gpu/drm/i915_legacy/gvt/mmio.c | 315 + + drivers/gpu/drm/i915_legacy/gvt/mmio.h | 105 + + .../gpu/drm/i915_legacy/gvt/mmio_context.c | 580 + + .../gpu/drm/i915_legacy/gvt/mmio_context.h | 60 + + drivers/gpu/drm/i915_legacy/gvt/mpt.h | 383 + + drivers/gpu/drm/i915_legacy/gvt/opregion.c | 570 + + drivers/gpu/drm/i915_legacy/gvt/page_track.c | 185 + + drivers/gpu/drm/i915_legacy/gvt/page_track.h | 56 + + drivers/gpu/drm/i915_legacy/gvt/reg.h | 131 + + .../gpu/drm/i915_legacy/gvt/sched_policy.c | 479 + + .../gpu/drm/i915_legacy/gvt/sched_policy.h | 62 + + drivers/gpu/drm/i915_legacy/gvt/scheduler.c | 1550 ++ + drivers/gpu/drm/i915_legacy/gvt/scheduler.h | 166 + + drivers/gpu/drm/i915_legacy/gvt/trace.h | 383 + + .../gpu/drm/i915_legacy/gvt/trace_points.c | 36 + + drivers/gpu/drm/i915_legacy/gvt/vgpu.c | 592 + + drivers/gpu/drm/i915_legacy/i915_active.c | 313 + + drivers/gpu/drm/i915_legacy/i915_active.h | 409 + + .../gpu/drm/i915_legacy/i915_active_types.h | 36 + + drivers/gpu/drm/i915_legacy/i915_cmd_parser.c | 1387 ++ + drivers/gpu/drm/i915_legacy/i915_debugfs.c | 4926 +++++ + drivers/gpu/drm/i915_legacy/i915_drv.c | 3195 +++ + drivers/gpu/drm/i915_legacy/i915_drv.h | 3693 ++++ + drivers/gpu/drm/i915_legacy/i915_fixed.h | 143 + + drivers/gpu/drm/i915_legacy/i915_gem.c | 5545 +++++ + drivers/gpu/drm/i915_legacy/i915_gem.h | 97 + + .../gpu/drm/i915_legacy/i915_gem_batch_pool.c | 140 + + .../gpu/drm/i915_legacy/i915_gem_batch_pool.h | 25 + + .../gpu/drm/i915_legacy/i915_gem_clflush.c | 178 + + .../gpu/drm/i915_legacy/i915_gem_clflush.h | 36 + + .../gpu/drm/i915_legacy/i915_gem_context.c | 1829 ++ + .../gpu/drm/i915_legacy/i915_gem_context.h | 185 + + .../drm/i915_legacy/i915_gem_context_types.h | 175 + + drivers/gpu/drm/i915_legacy/i915_gem_dmabuf.c | 337 + + drivers/gpu/drm/i915_legacy/i915_gem_evict.c | 444 + + .../gpu/drm/i915_legacy/i915_gem_execbuffer.c | 2722 +++ + .../gpu/drm/i915_legacy/i915_gem_fence_reg.c | 785 + + .../gpu/drm/i915_legacy/i915_gem_fence_reg.h | 52 + + drivers/gpu/drm/i915_legacy/i915_gem_gtt.c | 3922 ++++ + drivers/gpu/drm/i915_legacy/i915_gem_gtt.h | 664 + + .../gpu/drm/i915_legacy/i915_gem_internal.c | 210 + + drivers/gpu/drm/i915_legacy/i915_gem_object.c | 90 + + drivers/gpu/drm/i915_legacy/i915_gem_object.h | 509 + + .../drm/i915_legacy/i915_gem_render_state.c | 233 + + .../drm/i915_legacy/i915_gem_render_state.h | 31 + + .../gpu/drm/i915_legacy/i915_gem_shrinker.c | 556 + + drivers/gpu/drm/i915_legacy/i915_gem_stolen.c | 721 + + drivers/gpu/drm/i915_legacy/i915_gem_tiling.c | 457 + + .../gpu/drm/i915_legacy/i915_gem_userptr.c | 847 + + drivers/gpu/drm/i915_legacy/i915_gemfs.c | 75 + + drivers/gpu/drm/i915_legacy/i915_gemfs.h | 34 + + drivers/gpu/drm/i915_legacy/i915_globals.c | 125 + + drivers/gpu/drm/i915_legacy/i915_globals.h | 35 + + drivers/gpu/drm/i915_legacy/i915_gpu_error.c | 1876 ++ + drivers/gpu/drm/i915_legacy/i915_gpu_error.h | 315 + + drivers/gpu/drm/i915_legacy/i915_ioc32.c | 96 + + drivers/gpu/drm/i915_legacy/i915_irq.c | 4925 +++++ + drivers/gpu/drm/i915_legacy/i915_memcpy.c | 106 + + drivers/gpu/drm/i915_legacy/i915_mm.c | 83 + + drivers/gpu/drm/i915_legacy/i915_oa_bdw.c | 91 + + drivers/gpu/drm/i915_legacy/i915_oa_bdw.h | 15 + + drivers/gpu/drm/i915_legacy/i915_oa_bxt.c | 89 + + drivers/gpu/drm/i915_legacy/i915_oa_bxt.h | 15 + + drivers/gpu/drm/i915_legacy/i915_oa_cflgt2.c | 90 + + drivers/gpu/drm/i915_legacy/i915_oa_cflgt2.h | 15 + + drivers/gpu/drm/i915_legacy/i915_oa_cflgt3.c | 90 + + drivers/gpu/drm/i915_legacy/i915_oa_cflgt3.h | 15 + + drivers/gpu/drm/i915_legacy/i915_oa_chv.c | 90 + + drivers/gpu/drm/i915_legacy/i915_oa_chv.h | 15 + + drivers/gpu/drm/i915_legacy/i915_oa_cnl.c | 102 + + drivers/gpu/drm/i915_legacy/i915_oa_cnl.h | 15 + + drivers/gpu/drm/i915_legacy/i915_oa_glk.c | 89 + + drivers/gpu/drm/i915_legacy/i915_oa_glk.h | 15 + + drivers/gpu/drm/i915_legacy/i915_oa_hsw.c | 119 + + drivers/gpu/drm/i915_legacy/i915_oa_hsw.h | 15 + + drivers/gpu/drm/i915_legacy/i915_oa_icl.c | 99 + + drivers/gpu/drm/i915_legacy/i915_oa_icl.h | 15 + + drivers/gpu/drm/i915_legacy/i915_oa_kblgt2.c | 90 + + drivers/gpu/drm/i915_legacy/i915_oa_kblgt2.h | 15 + + drivers/gpu/drm/i915_legacy/i915_oa_kblgt3.c | 90 + + drivers/gpu/drm/i915_legacy/i915_oa_kblgt3.h | 15 + + drivers/gpu/drm/i915_legacy/i915_oa_sklgt2.c | 89 + + drivers/gpu/drm/i915_legacy/i915_oa_sklgt2.h | 15 + + drivers/gpu/drm/i915_legacy/i915_oa_sklgt3.c | 90 + + drivers/gpu/drm/i915_legacy/i915_oa_sklgt3.h | 15 + + drivers/gpu/drm/i915_legacy/i915_oa_sklgt4.c | 90 + + drivers/gpu/drm/i915_legacy/i915_oa_sklgt4.h | 15 + + drivers/gpu/drm/i915_legacy/i915_params.c | 237 + + drivers/gpu/drm/i915_legacy/i915_params.h | 94 + + drivers/gpu/drm/i915_legacy/i915_pci.c | 957 + + drivers/gpu/drm/i915_legacy/i915_perf.c | 3519 ++++ + drivers/gpu/drm/i915_legacy/i915_pmu.c | 1096 + + drivers/gpu/drm/i915_legacy/i915_pmu.h | 125 + + .../gpu/drm/i915_legacy/i915_priolist_types.h | 41 + + drivers/gpu/drm/i915_legacy/i915_pvinfo.h | 120 + + drivers/gpu/drm/i915_legacy/i915_query.c | 144 + + drivers/gpu/drm/i915_legacy/i915_query.h | 15 + + drivers/gpu/drm/i915_legacy/i915_reg.h | 11424 +++++++++++ + drivers/gpu/drm/i915_legacy/i915_request.c | 1511 ++ + drivers/gpu/drm/i915_legacy/i915_request.h | 423 + + drivers/gpu/drm/i915_legacy/i915_reset.c | 1474 ++ + drivers/gpu/drm/i915_legacy/i915_reset.h | 69 + + drivers/gpu/drm/i915_legacy/i915_scheduler.c | 492 + + drivers/gpu/drm/i915_legacy/i915_scheduler.h | 55 + + .../drm/i915_legacy/i915_scheduler_types.h | 73 + + drivers/gpu/drm/i915_legacy/i915_selftest.h | 105 + + drivers/gpu/drm/i915_legacy/i915_suspend.c | 150 + + drivers/gpu/drm/i915_legacy/i915_sw_fence.c | 576 + + drivers/gpu/drm/i915_legacy/i915_sw_fence.h | 109 + + drivers/gpu/drm/i915_legacy/i915_syncmap.c | 412 + + drivers/gpu/drm/i915_legacy/i915_syncmap.h | 38 + + drivers/gpu/drm/i915_legacy/i915_sysfs.c | 644 + + drivers/gpu/drm/i915_legacy/i915_timeline.c | 579 + + drivers/gpu/drm/i915_legacy/i915_timeline.h | 113 + + .../gpu/drm/i915_legacy/i915_timeline_types.h | 70 + + drivers/gpu/drm/i915_legacy/i915_trace.h | 1000 + + .../gpu/drm/i915_legacy/i915_trace_points.c | 14 + + .../drm/i915_legacy/i915_user_extensions.c | 61 + + .../drm/i915_legacy/i915_user_extensions.h | 20 + + drivers/gpu/drm/i915_legacy/i915_utils.h | 192 + + drivers/gpu/drm/i915_legacy/i915_vgpu.c | 279 + + drivers/gpu/drm/i915_legacy/i915_vgpu.h | 48 + + drivers/gpu/drm/i915_legacy/i915_vma.c | 1079 + + drivers/gpu/drm/i915_legacy/i915_vma.h | 446 + + drivers/gpu/drm/i915_legacy/icl_dsi.c | 1464 ++ + drivers/gpu/drm/i915_legacy/intel_acpi.c | 155 + + drivers/gpu/drm/i915_legacy/intel_atomic.c | 428 + + .../gpu/drm/i915_legacy/intel_atomic_plane.c | 373 + + .../gpu/drm/i915_legacy/intel_atomic_plane.h | 40 + + drivers/gpu/drm/i915_legacy/intel_audio.c | 1105 + + drivers/gpu/drm/i915_legacy/intel_audio.h | 24 + + drivers/gpu/drm/i915_legacy/intel_bios.c | 2298 +++ + drivers/gpu/drm/i915_legacy/intel_bios.h | 223 + + .../gpu/drm/i915_legacy/intel_breadcrumbs.c | 373 + + drivers/gpu/drm/i915_legacy/intel_cdclk.c | 2904 +++ + drivers/gpu/drm/i915_legacy/intel_cdclk.h | 46 + + drivers/gpu/drm/i915_legacy/intel_color.c | 1278 ++ + drivers/gpu/drm/i915_legacy/intel_color.h | 17 + + drivers/gpu/drm/i915_legacy/intel_combo_phy.c | 255 + + drivers/gpu/drm/i915_legacy/intel_connector.c | 282 + + drivers/gpu/drm/i915_legacy/intel_connector.h | 35 + + drivers/gpu/drm/i915_legacy/intel_context.c | 269 + + drivers/gpu/drm/i915_legacy/intel_context.h | 87 + + .../gpu/drm/i915_legacy/intel_context_types.h | 75 + + drivers/gpu/drm/i915_legacy/intel_crt.c | 1061 + + drivers/gpu/drm/i915_legacy/intel_crt.h | 21 + + drivers/gpu/drm/i915_legacy/intel_csr.c | 615 + + drivers/gpu/drm/i915_legacy/intel_csr.h | 17 + + drivers/gpu/drm/i915_legacy/intel_ddi.c | 4286 ++++ + drivers/gpu/drm/i915_legacy/intel_ddi.h | 53 + + .../gpu/drm/i915_legacy/intel_device_info.c | 1019 + + .../gpu/drm/i915_legacy/intel_device_info.h | 307 + + drivers/gpu/drm/i915_legacy/intel_display.c | 16814 ++++++++++++++++ + drivers/gpu/drm/i915_legacy/intel_display.h | 435 + + drivers/gpu/drm/i915_legacy/intel_dp.c | 7405 +++++++ + drivers/gpu/drm/i915_legacy/intel_dp.h | 122 + + .../drm/i915_legacy/intel_dp_aux_backlight.c | 280 + + .../drm/i915_legacy/intel_dp_link_training.c | 381 + + drivers/gpu/drm/i915_legacy/intel_dp_mst.c | 678 + + drivers/gpu/drm/i915_legacy/intel_dpio_phy.c | 1082 + + drivers/gpu/drm/i915_legacy/intel_dpll_mgr.c | 3382 ++++ + drivers/gpu/drm/i915_legacy/intel_dpll_mgr.h | 347 + + drivers/gpu/drm/i915_legacy/intel_drv.h | 2045 ++ + drivers/gpu/drm/i915_legacy/intel_dsi.c | 128 + + drivers/gpu/drm/i915_legacy/intel_dsi.h | 196 + + .../drm/i915_legacy/intel_dsi_dcs_backlight.c | 177 + + drivers/gpu/drm/i915_legacy/intel_dsi_vbt.c | 941 + + drivers/gpu/drm/i915_legacy/intel_dvo.c | 549 + + drivers/gpu/drm/i915_legacy/intel_dvo.h | 13 + + drivers/gpu/drm/i915_legacy/intel_engine_cs.c | 1758 ++ + .../gpu/drm/i915_legacy/intel_engine_types.h | 548 + + drivers/gpu/drm/i915_legacy/intel_fbc.c | 1341 ++ + drivers/gpu/drm/i915_legacy/intel_fbc.h | 42 + + drivers/gpu/drm/i915_legacy/intel_fbdev.c | 640 + + drivers/gpu/drm/i915_legacy/intel_fbdev.h | 53 + + .../gpu/drm/i915_legacy/intel_fifo_underrun.c | 457 + + .../gpu/drm/i915_legacy/intel_frontbuffer.c | 204 + + .../gpu/drm/i915_legacy/intel_frontbuffer.h | 98 + + .../gpu/drm/i915_legacy/intel_gpu_commands.h | 278 + + drivers/gpu/drm/i915_legacy/intel_guc.c | 723 + + drivers/gpu/drm/i915_legacy/intel_guc.h | 200 + + drivers/gpu/drm/i915_legacy/intel_guc_ads.c | 151 + + drivers/gpu/drm/i915_legacy/intel_guc_ads.h | 33 + + drivers/gpu/drm/i915_legacy/intel_guc_ct.c | 943 + + drivers/gpu/drm/i915_legacy/intel_guc_ct.h | 99 + + drivers/gpu/drm/i915_legacy/intel_guc_fw.c | 277 + + drivers/gpu/drm/i915_legacy/intel_guc_fw.h | 33 + + drivers/gpu/drm/i915_legacy/intel_guc_fwif.h | 705 + + drivers/gpu/drm/i915_legacy/intel_guc_log.c | 641 + + drivers/gpu/drm/i915_legacy/intel_guc_log.h | 100 + + drivers/gpu/drm/i915_legacy/intel_guc_reg.h | 130 + + .../drm/i915_legacy/intel_guc_submission.c | 1454 ++ + .../drm/i915_legacy/intel_guc_submission.h | 88 + + drivers/gpu/drm/i915_legacy/intel_gvt.c | 137 + + drivers/gpu/drm/i915_legacy/intel_gvt.h | 50 + + drivers/gpu/drm/i915_legacy/intel_hangcheck.c | 334 + + drivers/gpu/drm/i915_legacy/intel_hdcp.c | 1947 ++ + drivers/gpu/drm/i915_legacy/intel_hdcp.h | 33 + + drivers/gpu/drm/i915_legacy/intel_hdmi.c | 3111 +++ + drivers/gpu/drm/i915_legacy/intel_hdmi.h | 51 + + drivers/gpu/drm/i915_legacy/intel_hotplug.c | 686 + + drivers/gpu/drm/i915_legacy/intel_huc.c | 128 + + drivers/gpu/drm/i915_legacy/intel_huc.h | 54 + + drivers/gpu/drm/i915_legacy/intel_huc_fw.c | 168 + + drivers/gpu/drm/i915_legacy/intel_huc_fw.h | 15 + + drivers/gpu/drm/i915_legacy/intel_i2c.c | 933 + + drivers/gpu/drm/i915_legacy/intel_lpe_audio.c | 361 + + drivers/gpu/drm/i915_legacy/intel_lrc.c | 3041 +++ + drivers/gpu/drm/i915_legacy/intel_lrc.h | 120 + + drivers/gpu/drm/i915_legacy/intel_lrc_reg.h | 68 + + drivers/gpu/drm/i915_legacy/intel_lspcon.c | 588 + + drivers/gpu/drm/i915_legacy/intel_lspcon.h | 38 + + drivers/gpu/drm/i915_legacy/intel_lvds.c | 1006 + + drivers/gpu/drm/i915_legacy/intel_lvds.h | 22 + + drivers/gpu/drm/i915_legacy/intel_mocs.c | 564 + + drivers/gpu/drm/i915_legacy/intel_mocs.h | 58 + + drivers/gpu/drm/i915_legacy/intel_opregion.c | 1175 ++ + drivers/gpu/drm/i915_legacy/intel_opregion.h | 122 + + drivers/gpu/drm/i915_legacy/intel_overlay.c | 1495 ++ + drivers/gpu/drm/i915_legacy/intel_panel.c | 2049 ++ + drivers/gpu/drm/i915_legacy/intel_panel.h | 65 + + drivers/gpu/drm/i915_legacy/intel_pipe_crc.c | 679 + + drivers/gpu/drm/i915_legacy/intel_pipe_crc.h | 35 + + drivers/gpu/drm/i915_legacy/intel_pm.c | 10123 ++++++++++ + drivers/gpu/drm/i915_legacy/intel_pm.h | 71 + + drivers/gpu/drm/i915_legacy/intel_psr.c | 1312 ++ + drivers/gpu/drm/i915_legacy/intel_psr.h | 40 + + drivers/gpu/drm/i915_legacy/intel_quirks.c | 169 + + .../gpu/drm/i915_legacy/intel_renderstate.h | 47 + + .../drm/i915_legacy/intel_renderstate_gen6.c | 315 + + .../drm/i915_legacy/intel_renderstate_gen7.c | 279 + + .../drm/i915_legacy/intel_renderstate_gen8.c | 983 + + .../drm/i915_legacy/intel_renderstate_gen9.c | 999 + + .../gpu/drm/i915_legacy/intel_ringbuffer.c | 2345 +++ + .../gpu/drm/i915_legacy/intel_ringbuffer.h | 583 + + .../gpu/drm/i915_legacy/intel_runtime_pm.c | 4520 +++++ + drivers/gpu/drm/i915_legacy/intel_sdvo.c | 3325 +++ + drivers/gpu/drm/i915_legacy/intel_sdvo.h | 23 + + drivers/gpu/drm/i915_legacy/intel_sdvo_regs.h | 733 + + drivers/gpu/drm/i915_legacy/intel_sideband.c | 295 + + drivers/gpu/drm/i915_legacy/intel_sprite.c | 2462 +++ + drivers/gpu/drm/i915_legacy/intel_sprite.h | 55 + + drivers/gpu/drm/i915_legacy/intel_tv.c | 1992 ++ + drivers/gpu/drm/i915_legacy/intel_tv.h | 13 + + drivers/gpu/drm/i915_legacy/intel_uc.c | 511 + + drivers/gpu/drm/i915_legacy/intel_uc.h | 63 + + drivers/gpu/drm/i915_legacy/intel_uc_fw.c | 317 + + drivers/gpu/drm/i915_legacy/intel_uc_fw.h | 153 + + drivers/gpu/drm/i915_legacy/intel_uncore.c | 1958 ++ + drivers/gpu/drm/i915_legacy/intel_uncore.h | 399 + + drivers/gpu/drm/i915_legacy/intel_vbt_defs.h | 936 + + drivers/gpu/drm/i915_legacy/intel_vdsc.c | 964 + + drivers/gpu/drm/i915_legacy/intel_wopcm.c | 281 + + drivers/gpu/drm/i915_legacy/intel_wopcm.h | 31 + + .../gpu/drm/i915_legacy/intel_workarounds.c | 1344 ++ + .../gpu/drm/i915_legacy/intel_workarounds.h | 34 + + .../drm/i915_legacy/intel_workarounds_types.h | 27 + + .../i915_legacy/selftests/huge_gem_object.c | 139 + + .../i915_legacy/selftests/huge_gem_object.h | 45 + + .../drm/i915_legacy/selftests/huge_pages.c | 1792 ++ + .../drm/i915_legacy/selftests/i915_active.c | 157 + + .../gpu/drm/i915_legacy/selftests/i915_gem.c | 222 + + .../selftests/i915_gem_coherency.c | 397 + + .../i915_legacy/selftests/i915_gem_context.c | 1859 ++ + .../i915_legacy/selftests/i915_gem_dmabuf.c | 404 + + .../i915_legacy/selftests/i915_gem_evict.c | 554 + + .../drm/i915_legacy/selftests/i915_gem_gtt.c | 1733 ++ + .../i915_legacy/selftests/i915_gem_object.c | 659 + + .../selftests/i915_live_selftests.h | 28 + + .../selftests/i915_mock_selftests.h | 26 + + .../drm/i915_legacy/selftests/i915_random.c | 89 + + .../drm/i915_legacy/selftests/i915_random.h | 60 + + .../drm/i915_legacy/selftests/i915_request.c | 1246 ++ + .../drm/i915_legacy/selftests/i915_selftest.c | 299 + + .../drm/i915_legacy/selftests/i915_sw_fence.c | 757 + + .../drm/i915_legacy/selftests/i915_syncmap.c | 616 + + .../drm/i915_legacy/selftests/i915_timeline.c | 845 + + .../gpu/drm/i915_legacy/selftests/i915_vma.c | 754 + + .../i915_legacy/selftests/igt_flush_test.c | 33 + + .../i915_legacy/selftests/igt_flush_test.h | 14 + + .../drm/i915_legacy/selftests/igt_live_test.c | 78 + + .../drm/i915_legacy/selftests/igt_live_test.h | 35 + + .../gpu/drm/i915_legacy/selftests/igt_reset.c | 44 + + .../gpu/drm/i915_legacy/selftests/igt_reset.h | 15 + + .../drm/i915_legacy/selftests/igt_spinner.c | 201 + + .../drm/i915_legacy/selftests/igt_spinner.h | 37 + + .../drm/i915_legacy/selftests/igt_wedge_me.h | 58 + + .../i915_legacy/selftests/intel_engine_cs.c | 58 + + .../gpu/drm/i915_legacy/selftests/intel_guc.c | 358 + + .../i915_legacy/selftests/intel_hangcheck.c | 1919 ++ + .../gpu/drm/i915_legacy/selftests/intel_lrc.c | 1330 ++ + .../drm/i915_legacy/selftests/intel_uncore.c | 330 + + .../i915_legacy/selftests/intel_workarounds.c | 901 + + .../drm/i915_legacy/selftests/lib_sw_fence.c | 132 + + .../drm/i915_legacy/selftests/lib_sw_fence.h | 43 + + .../drm/i915_legacy/selftests/mock_context.c | 124 + + .../drm/i915_legacy/selftests/mock_context.h | 42 + + .../drm/i915_legacy/selftests/mock_dmabuf.c | 162 + + .../drm/i915_legacy/selftests/mock_dmabuf.h | 41 + + .../gpu/drm/i915_legacy/selftests/mock_drm.c | 73 + + .../gpu/drm/i915_legacy/selftests/mock_drm.h | 31 + + .../drm/i915_legacy/selftests/mock_engine.c | 321 + + .../drm/i915_legacy/selftests/mock_engine.h | 49 + + .../i915_legacy/selftests/mock_gem_device.c | 241 + + .../i915_legacy/selftests/mock_gem_device.h | 10 + + .../i915_legacy/selftests/mock_gem_object.h | 9 + + .../gpu/drm/i915_legacy/selftests/mock_gtt.c | 127 + + .../gpu/drm/i915_legacy/selftests/mock_gtt.h | 35 + + .../drm/i915_legacy/selftests/mock_request.c | 59 + + .../drm/i915_legacy/selftests/mock_request.h | 39 + + .../drm/i915_legacy/selftests/mock_timeline.c | 30 + + .../drm/i915_legacy/selftests/mock_timeline.h | 15 + + .../drm/i915_legacy/selftests/mock_uncore.c | 46 + + .../drm/i915_legacy/selftests/mock_uncore.h | 30 + + .../drm/i915_legacy/selftests/scatterlist.c | 379 + + drivers/gpu/drm/i915_legacy/vlv_dsi.c | 1830 ++ + drivers/gpu/drm/i915_legacy/vlv_dsi_pll.c | 567 + + 358 files changed, 244185 insertions(+), 2 deletions(-) + create mode 100644 drivers/gpu/drm/i915_legacy/.gitignore + create mode 100644 drivers/gpu/drm/i915_legacy/Kconfig + create mode 100644 drivers/gpu/drm/i915_legacy/Kconfig.debug + create mode 100644 drivers/gpu/drm/i915_legacy/Makefile + create mode 100644 drivers/gpu/drm/i915_legacy/Makefile.header-test + create mode 100644 drivers/gpu/drm/i915_legacy/dvo.h + create mode 100644 drivers/gpu/drm/i915_legacy/dvo_ch7017.c + create mode 100644 drivers/gpu/drm/i915_legacy/dvo_ch7xxx.c + create mode 100644 drivers/gpu/drm/i915_legacy/dvo_ivch.c + create mode 100644 drivers/gpu/drm/i915_legacy/dvo_ns2501.c + create mode 100644 drivers/gpu/drm/i915_legacy/dvo_sil164.c + create mode 100644 drivers/gpu/drm/i915_legacy/dvo_tfp410.c + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/Makefile + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/aperture_gm.c + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/cfg_space.c + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/cmd_parser.c + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/cmd_parser.h + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/debug.h + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/debugfs.c + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/display.c + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/display.h + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/dmabuf.c + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/dmabuf.h + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/edid.c + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/edid.h + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/execlist.c + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/execlist.h + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/fb_decoder.c + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/fb_decoder.h + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/firmware.c + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/gtt.c + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/gtt.h + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/gvt.c + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/gvt.h + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/handlers.c + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/hypercall.h + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/interrupt.c + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/interrupt.h + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/kvmgt.c + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/mmio.c + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/mmio.h + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/mmio_context.c + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/mmio_context.h + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/mpt.h + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/opregion.c + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/page_track.c + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/page_track.h + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/reg.h + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/sched_policy.c + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/sched_policy.h + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/scheduler.c + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/scheduler.h + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/trace.h + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/trace_points.c + create mode 100644 drivers/gpu/drm/i915_legacy/gvt/vgpu.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_active.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_active.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_active_types.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_cmd_parser.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_debugfs.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_drv.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_drv.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_fixed.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gem.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gem.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gem_batch_pool.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gem_batch_pool.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gem_clflush.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gem_clflush.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gem_context.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gem_context.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gem_context_types.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gem_dmabuf.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gem_evict.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gem_execbuffer.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gem_fence_reg.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gem_fence_reg.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gem_gtt.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gem_gtt.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gem_internal.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gem_object.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gem_object.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gem_render_state.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gem_render_state.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gem_shrinker.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gem_stolen.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gem_tiling.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gem_userptr.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gemfs.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gemfs.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_globals.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_globals.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gpu_error.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_gpu_error.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_ioc32.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_irq.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_memcpy.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_mm.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_bdw.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_bdw.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_bxt.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_bxt.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_cflgt2.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_cflgt2.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_cflgt3.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_cflgt3.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_chv.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_chv.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_cnl.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_cnl.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_glk.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_glk.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_hsw.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_hsw.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_icl.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_icl.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_kblgt2.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_kblgt2.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_kblgt3.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_kblgt3.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_sklgt2.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_sklgt2.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_sklgt3.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_sklgt3.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_sklgt4.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_oa_sklgt4.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_params.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_params.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_pci.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_perf.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_pmu.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_pmu.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_priolist_types.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_pvinfo.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_query.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_query.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_reg.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_request.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_request.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_reset.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_reset.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_scheduler.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_scheduler.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_scheduler_types.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_selftest.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_suspend.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_sw_fence.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_sw_fence.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_syncmap.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_syncmap.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_sysfs.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_timeline.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_timeline.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_timeline_types.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_trace.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_trace_points.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_user_extensions.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_user_extensions.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_utils.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_vgpu.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_vgpu.h + create mode 100644 drivers/gpu/drm/i915_legacy/i915_vma.c + create mode 100644 drivers/gpu/drm/i915_legacy/i915_vma.h + create mode 100644 drivers/gpu/drm/i915_legacy/icl_dsi.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_acpi.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_atomic.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_atomic_plane.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_atomic_plane.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_audio.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_audio.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_bios.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_bios.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_breadcrumbs.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_cdclk.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_cdclk.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_color.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_color.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_combo_phy.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_connector.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_connector.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_context.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_context.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_context_types.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_crt.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_crt.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_csr.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_csr.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_ddi.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_ddi.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_device_info.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_device_info.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_display.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_display.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_dp.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_dp.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_dp_aux_backlight.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_dp_link_training.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_dp_mst.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_dpio_phy.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_dpll_mgr.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_dpll_mgr.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_drv.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_dsi.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_dsi.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_dsi_dcs_backlight.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_dsi_vbt.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_dvo.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_dvo.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_engine_cs.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_engine_types.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_fbc.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_fbc.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_fbdev.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_fbdev.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_fifo_underrun.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_frontbuffer.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_frontbuffer.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_gpu_commands.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_guc.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_guc.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_guc_ads.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_guc_ads.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_guc_ct.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_guc_ct.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_guc_fw.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_guc_fw.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_guc_fwif.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_guc_log.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_guc_log.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_guc_reg.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_guc_submission.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_guc_submission.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_gvt.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_gvt.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_hangcheck.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_hdcp.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_hdcp.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_hdmi.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_hdmi.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_hotplug.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_huc.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_huc.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_huc_fw.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_huc_fw.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_i2c.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_lpe_audio.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_lrc.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_lrc.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_lrc_reg.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_lspcon.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_lspcon.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_lvds.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_lvds.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_mocs.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_mocs.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_opregion.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_opregion.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_overlay.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_panel.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_panel.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_pipe_crc.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_pipe_crc.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_pm.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_pm.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_psr.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_psr.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_quirks.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_renderstate.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_renderstate_gen6.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_renderstate_gen7.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_renderstate_gen8.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_renderstate_gen9.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_ringbuffer.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_ringbuffer.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_runtime_pm.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_sdvo.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_sdvo.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_sdvo_regs.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_sideband.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_sprite.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_sprite.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_tv.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_tv.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_uc.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_uc.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_uc_fw.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_uc_fw.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_uncore.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_uncore.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_vbt_defs.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_vdsc.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_wopcm.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_wopcm.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_workarounds.c + create mode 100644 drivers/gpu/drm/i915_legacy/intel_workarounds.h + create mode 100644 drivers/gpu/drm/i915_legacy/intel_workarounds_types.h + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/huge_gem_object.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/huge_gem_object.h + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/huge_pages.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/i915_active.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/i915_gem.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/i915_gem_coherency.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/i915_gem_context.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/i915_gem_dmabuf.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/i915_gem_evict.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/i915_gem_gtt.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/i915_gem_object.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/i915_live_selftests.h + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/i915_mock_selftests.h + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/i915_random.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/i915_random.h + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/i915_request.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/i915_selftest.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/i915_sw_fence.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/i915_syncmap.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/i915_timeline.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/i915_vma.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/igt_flush_test.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/igt_flush_test.h + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/igt_live_test.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/igt_live_test.h + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/igt_reset.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/igt_reset.h + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/igt_spinner.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/igt_spinner.h + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/igt_wedge_me.h + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/intel_engine_cs.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/intel_guc.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/intel_hangcheck.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/intel_lrc.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/intel_uncore.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/intel_workarounds.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/lib_sw_fence.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/lib_sw_fence.h + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/mock_context.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/mock_context.h + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/mock_dmabuf.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/mock_dmabuf.h + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/mock_drm.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/mock_drm.h + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/mock_engine.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/mock_engine.h + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/mock_gem_device.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/mock_gem_device.h + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/mock_gem_object.h + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/mock_gtt.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/mock_gtt.h + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/mock_request.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/mock_request.h + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/mock_timeline.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/mock_timeline.h + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/mock_uncore.c + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/mock_uncore.h + create mode 100644 drivers/gpu/drm/i915_legacy/selftests/scatterlist.c + create mode 100644 drivers/gpu/drm/i915_legacy/vlv_dsi.c + create mode 100644 drivers/gpu/drm/i915_legacy/vlv_dsi_pll.c + +diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig +index 3c88420e3497..d95b242622b3 100644 +--- a/drivers/gpu/drm/Kconfig ++++ b/drivers/gpu/drm/Kconfig +@@ -239,7 +239,7 @@ source "drivers/gpu/drm/amd/amdgpu/Kconfig" + + source "drivers/gpu/drm/nouveau/Kconfig" + +-source "drivers/gpu/drm/i915/Kconfig" ++source "drivers/gpu/drm/i915_legacy/Kconfig" + + config DRM_VGEM + tristate "Virtual GEM provider" +diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile +index 9f0d2ee35794..532f50012a65 100644 +--- a/drivers/gpu/drm/Makefile ++++ b/drivers/gpu/drm/Makefile +@@ -67,7 +67,7 @@ obj-$(CONFIG_DRM_RADEON)+= radeon/ + obj-$(CONFIG_DRM_AMDGPU)+= amd/amdgpu/ + obj-$(CONFIG_DRM_MGA) += mga/ + obj-$(CONFIG_DRM_I810) += i810/ +-obj-$(CONFIG_DRM_I915) += i915/ ++obj-$(CONFIG_DRM_I915) += i915_legacy/ + obj-$(CONFIG_DRM_MGAG200) += mgag200/ + obj-$(CONFIG_DRM_V3D) += v3d/ + obj-$(CONFIG_DRM_VC4) += vc4/ +diff --git a/drivers/gpu/drm/i915_legacy/.gitignore b/drivers/gpu/drm/i915_legacy/.gitignore +new file mode 100644 +index 000000000000..cff45d81f42f +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/.gitignore +@@ -0,0 +1 @@ ++header_test_*.c +diff --git a/drivers/gpu/drm/i915_legacy/Kconfig b/drivers/gpu/drm/i915_legacy/Kconfig +new file mode 100644 +index 000000000000..255f224db64b +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/Kconfig +@@ -0,0 +1,135 @@ ++# SPDX-License-Identifier: GPL-2.0-only ++config DRM_I915 ++ tristate "Intel 8xx/9xx/G3x/G4x/HD Graphics" ++ depends on DRM ++ depends on X86 && PCI ++ select INTEL_GTT ++ select INTERVAL_TREE ++ # we need shmfs for the swappable backing store, and in particular ++ # the shmem_readpage() which depends upon tmpfs ++ select SHMEM ++ select TMPFS ++ select DRM_KMS_HELPER ++ select DRM_PANEL ++ select DRM_MIPI_DSI ++ select RELAY ++ select IRQ_WORK ++ # i915 depends on ACPI_VIDEO when ACPI is enabled ++ # but for select to work, need to select ACPI_VIDEO's dependencies, ick ++ select BACKLIGHT_CLASS_DEVICE if ACPI ++ select INPUT if ACPI ++ select ACPI_VIDEO if ACPI ++ select ACPI_BUTTON if ACPI ++ select SYNC_FILE ++ select IOSF_MBI ++ select CRC32 ++ select SND_HDA_I915 if SND_HDA_CORE ++ select CEC_CORE if CEC_NOTIFIER ++ help ++ Choose this option if you have a system that has "Intel Graphics ++ Media Accelerator" or "HD Graphics" integrated graphics, ++ including 830M, 845G, 852GM, 855GM, 865G, 915G, 945G, 965G, ++ G35, G41, G43, G45 chipsets and Celeron, Pentium, Core i3, ++ Core i5, Core i7 as well as Atom CPUs with integrated graphics. ++ ++ This driver is used by the Intel driver in X.org 6.8 and ++ XFree86 4.4 and above. It replaces the older i830 module that ++ supported a subset of the hardware in older X.org releases. ++ ++ Note that the older i810/i815 chipsets require the use of the ++ i810 driver instead, and the Atom z5xx series has an entirely ++ different implementation. ++ ++ If "M" is selected, the module will be called i915. ++ ++config DRM_I915_ALPHA_SUPPORT ++ bool "Enable alpha quality support for new Intel hardware by default" ++ depends on DRM_I915 ++ default n ++ help ++ Choose this option if you have new Intel hardware and want to enable ++ the alpha quality i915 driver support for the hardware in this kernel ++ version. You can also enable the support at runtime using the module ++ parameter i915.alpha_support=1; this option changes the default for ++ that module parameter. ++ ++ It is recommended to upgrade to a kernel version with proper support ++ as soon as it is available. Generally fixes for platforms with alpha ++ support are not backported to older kernels. ++ ++ If in doubt, say "N". ++ ++config DRM_I915_CAPTURE_ERROR ++ bool "Enable capturing GPU state following a hang" ++ depends on DRM_I915 ++ default y ++ help ++ This option enables capturing the GPU state when a hang is detected. ++ This information is vital for triaging hangs and assists in debugging. ++ Please report any hang to ++ https://bugs.freedesktop.org/enter_bug.cgi?product=DRI ++ for triaging. ++ ++ If in doubt, say "Y". ++ ++config DRM_I915_COMPRESS_ERROR ++ bool "Compress GPU error state" ++ depends on DRM_I915_CAPTURE_ERROR ++ select ZLIB_DEFLATE ++ default y ++ help ++ This option selects ZLIB_DEFLATE if it isn't already ++ selected and causes any error state captured upon a GPU hang ++ to be compressed using zlib. ++ ++ If in doubt, say "Y". ++ ++config DRM_I915_USERPTR ++ bool "Always enable userptr support" ++ depends on DRM_I915 ++ select MMU_NOTIFIER ++ default y ++ help ++ This option selects CONFIG_MMU_NOTIFIER if it isn't already ++ selected to enabled full userptr support. ++ ++ If in doubt, say "Y". ++ ++config DRM_I915_GVT ++ bool "Enable Intel GVT-g graphics virtualization host support" ++ depends on DRM_I915 ++ depends on 64BIT ++ default n ++ help ++ Choose this option if you want to enable Intel GVT-g graphics ++ virtualization technology host support with integrated graphics. ++ With GVT-g, it's possible to have one integrated graphics ++ device shared by multiple VMs under different hypervisors. ++ ++ Note that at least one hypervisor like Xen or KVM is required for ++ this driver to work, and it only supports newer device from ++ Broadwell+. For further information and setup guide, you can ++ visit: http://01.org/igvt-g. ++ ++ Now it's just a stub to support the modifications of i915 for ++ GVT device model. It requires at least one MPT modules for Xen/KVM ++ and other components of GVT device model to work. Use it under ++ you own risk. ++ ++ If in doubt, say "N". ++ ++config DRM_I915_GVT_KVMGT ++ tristate "Enable KVM/VFIO support for Intel GVT-g" ++ depends on DRM_I915_GVT ++ depends on KVM ++ depends on VFIO_MDEV && VFIO_MDEV_DEVICE ++ default n ++ help ++ Choose this option if you want to enable KVMGT support for ++ Intel GVT-g. ++ ++menu "drm/i915 Debugging" ++depends on DRM_I915 ++depends on EXPERT ++source "drivers/gpu/drm/i915/Kconfig.debug" ++endmenu +diff --git a/drivers/gpu/drm/i915_legacy/Kconfig.debug b/drivers/gpu/drm/i915_legacy/Kconfig.debug +new file mode 100644 +index 000000000000..04b686d2c2d0 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/Kconfig.debug +@@ -0,0 +1,183 @@ ++# SPDX-License-Identifier: GPL-2.0-only ++config DRM_I915_WERROR ++ bool "Force GCC to throw an error instead of a warning when compiling" ++ # As this may inadvertently break the build, only allow the user ++ # to shoot oneself in the foot iff they aim really hard ++ depends on EXPERT ++ # We use the dependency on !COMPILE_TEST to not be enabled in ++ # allmodconfig or allyesconfig configurations ++ depends on !COMPILE_TEST ++ default n ++ help ++ Add -Werror to the build flags for (and only for) i915.ko. ++ Do not enable this unless you are writing code for the i915.ko module. ++ ++ Recommended for driver developers only. ++ ++ If in doubt, say "N". ++ ++config DRM_I915_DEBUG ++ bool "Enable additional driver debugging" ++ depends on DRM_I915 ++ select DEBUG_FS ++ select PREEMPT_COUNT ++ select I2C_CHARDEV ++ select STACKDEPOT ++ select DRM_DP_AUX_CHARDEV ++ select X86_MSR # used by igt/pm_rpm ++ select DRM_VGEM # used by igt/prime_vgem (dmabuf interop checks) ++ select DRM_DEBUG_MM if DRM=y ++ select DRM_DEBUG_SELFTEST ++ select SW_SYNC # signaling validation framework (igt/syncobj*) ++ select DRM_I915_SW_FENCE_DEBUG_OBJECTS ++ select DRM_I915_SELFTEST ++ select DRM_I915_DEBUG_RUNTIME_PM ++ default n ++ help ++ Choose this option to turn on extra driver debugging that may affect ++ performance but will catch some internal issues. ++ ++ Recommended for driver developers only. ++ ++ If in doubt, say "N". ++ ++config DRM_I915_DEBUG_GEM ++ bool "Insert extra checks into the GEM internals" ++ default n ++ depends on DRM_I915_WERROR ++ help ++ Enable extra sanity checks (including BUGs) along the GEM driver ++ paths that may slow the system down and if hit hang the machine. ++ ++ Recommended for driver developers only. ++ ++ If in doubt, say "N". ++ ++config DRM_I915_ERRLOG_GEM ++ bool "Insert extra logging (very verbose) for common GEM errors" ++ default n ++ depends on DRM_I915_DEBUG_GEM ++ help ++ Enable additional logging that may help track down the cause of ++ principally userspace errors. ++ ++ Recommended for driver developers only. ++ ++ If in doubt, say "N". ++ ++config DRM_I915_TRACE_GEM ++ bool "Insert extra ftrace output from the GEM internals" ++ depends on DRM_I915_DEBUG_GEM ++ select TRACING ++ default n ++ help ++ Enable additional and verbose debugging output that will spam ++ ordinary tests, but may be vital for post-mortem debugging when ++ used with /proc/sys/kernel/ftrace_dump_on_oops ++ ++ Recommended for driver developers only. ++ ++ If in doubt, say "N". ++ ++config DRM_I915_SW_FENCE_DEBUG_OBJECTS ++ bool "Enable additional driver debugging for fence objects" ++ depends on DRM_I915 ++ select DEBUG_OBJECTS ++ default n ++ help ++ Choose this option to turn on extra driver debugging that may affect ++ performance but will catch some internal issues. ++ ++ Recommended for driver developers only. ++ ++ If in doubt, say "N". ++ ++config DRM_I915_SW_FENCE_CHECK_DAG ++ bool "Enable additional driver debugging for detecting dependency cycles" ++ depends on DRM_I915 ++ default n ++ help ++ Choose this option to turn on extra driver debugging that may affect ++ performance but will catch some internal issues. ++ ++ Recommended for driver developers only. ++ ++ If in doubt, say "N". ++ ++config DRM_I915_DEBUG_GUC ++ bool "Enable additional driver debugging for GuC" ++ depends on DRM_I915 ++ default n ++ help ++ Choose this option to turn on extra driver debugging that may affect ++ performance but will help resolve GuC related issues. ++ ++ Recommended for driver developers only. ++ ++ If in doubt, say "N". ++ ++config DRM_I915_SELFTEST ++ bool "Enable selftests upon driver load" ++ depends on DRM_I915 ++ default n ++ select FAULT_INJECTION ++ select PRIME_NUMBERS ++ help ++ Choose this option to allow the driver to perform selftests upon ++ loading; also requires the i915.selftest=1 module parameter. To ++ exit the module after running the selftests (i.e. to prevent normal ++ module initialisation afterwards) use i915.selftest=-1. ++ ++ Recommended for driver developers only. ++ ++ If in doubt, say "N". ++ ++config DRM_I915_SELFTEST_BROKEN ++ bool "Enable broken and dangerous selftests" ++ depends on DRM_I915_SELFTEST ++ depends on BROKEN ++ default n ++ help ++ This option enables the execution of selftests that are "dangerous" ++ and may trigger unintended HW side-effects as they break strict ++ rules given in the HW specification. For science. ++ ++ Recommended for masochistic driver developers only. ++ ++ If in doubt, say "N". ++ ++config DRM_I915_LOW_LEVEL_TRACEPOINTS ++ bool "Enable low level request tracing events" ++ depends on DRM_I915 ++ default n ++ help ++ Choose this option to turn on low level request tracing events. ++ This provides the ability to precisely monitor engine utilisation ++ and also analyze the request dependency resolving timeline. ++ ++ If in doubt, say "N". ++ ++config DRM_I915_DEBUG_VBLANK_EVADE ++ bool "Enable extra debug warnings for vblank evasion" ++ depends on DRM_I915 ++ default n ++ help ++ Choose this option to turn on extra debug warnings for the ++ vblank evade mechanism. This gives a warning every time the ++ the deadline allotted for the vblank evade critical section ++ is exceeded, even if there isn't an actual risk of missing ++ the vblank. ++ ++ If in doubt, say "N". ++ ++config DRM_I915_DEBUG_RUNTIME_PM ++ bool "Enable extra state checking for runtime PM" ++ depends on DRM_I915 ++ default n ++ select STACKDEPOT ++ help ++ Choose this option to turn on extra state checking for the ++ runtime PM functionality. This may introduce overhead during ++ driver loading, suspend and resume operations. ++ ++ If in doubt, say "N" +diff --git a/drivers/gpu/drm/i915_legacy/Makefile b/drivers/gpu/drm/i915_legacy/Makefile +new file mode 100644 +index 000000000000..fbcb0904f4a8 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/Makefile +@@ -0,0 +1,212 @@ ++# SPDX-License-Identifier: GPL-2.0 ++# ++# Makefile for the drm device driver. This driver provides support for the ++# Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher. ++ ++# Add a set of useful warning flags and enable -Werror for CI to prevent ++# trivial mistakes from creeping in. We have to do this piecemeal as we reject ++# any patch that isn't warning clean, so turning on -Wall -Wextra (or W=1) we ++# need to filter out dubious warnings. Still it is our interest ++# to keep running locally with W=1 C=1 until we are completely clean. ++# ++# Note the danger in using -Wall -Wextra is that when CI updates gcc we ++# will most likely get a sudden build breakage... Hopefully we will fix ++# new warnings before CI updates! ++subdir-ccflags-y := -Wall -Wextra ++subdir-ccflags-y += $(call cc-disable-warning, unused-parameter) ++subdir-ccflags-y += $(call cc-disable-warning, type-limits) ++subdir-ccflags-y += $(call cc-disable-warning, missing-field-initializers) ++subdir-ccflags-y += $(call cc-disable-warning, implicit-fallthrough) ++subdir-ccflags-y += $(call cc-disable-warning, unused-but-set-variable) ++# clang warnings ++subdir-ccflags-y += $(call cc-disable-warning, sign-compare) ++subdir-ccflags-y += $(call cc-disable-warning, sometimes-uninitialized) ++subdir-ccflags-y += $(call cc-disable-warning, initializer-overrides) ++subdir-ccflags-y += $(call cc-disable-warning, uninitialized) ++subdir-ccflags-$(CONFIG_DRM_I915_WERROR) += -Werror ++ ++# Fine grained warnings disable ++CFLAGS_i915_pci.o = $(call cc-disable-warning, override-init) ++CFLAGS_intel_fbdev.o = $(call cc-disable-warning, override-init) ++ ++subdir-ccflags-y += \ ++ $(call as-instr,movntdqa (%eax)$(comma)%xmm0,-DCONFIG_AS_MOVNTDQA) ++ ++# Extra header tests ++include $(src)/Makefile.header-test ++ ++# Please keep these build lists sorted! ++ ++# core driver code ++i915-y += i915_drv.o \ ++ i915_irq.o \ ++ i915_memcpy.o \ ++ i915_mm.o \ ++ i915_params.o \ ++ i915_pci.o \ ++ i915_reset.o \ ++ i915_suspend.o \ ++ i915_sw_fence.o \ ++ i915_syncmap.o \ ++ i915_sysfs.o \ ++ i915_user_extensions.o \ ++ intel_csr.o \ ++ intel_device_info.o \ ++ intel_pm.o \ ++ intel_runtime_pm.o \ ++ intel_workarounds.o ++ ++i915-$(CONFIG_COMPAT) += i915_ioc32.o ++i915-$(CONFIG_DEBUG_FS) += i915_debugfs.o intel_pipe_crc.o ++i915-$(CONFIG_PERF_EVENTS) += i915_pmu.o ++ ++# GEM code ++i915-y += \ ++ i915_active.o \ ++ i915_cmd_parser.o \ ++ i915_gem_batch_pool.o \ ++ i915_gem_clflush.o \ ++ i915_gem_context.o \ ++ i915_gem_dmabuf.o \ ++ i915_gem_evict.o \ ++ i915_gem_execbuffer.o \ ++ i915_gem_fence_reg.o \ ++ i915_gem_gtt.o \ ++ i915_gem_internal.o \ ++ i915_gem.o \ ++ i915_gem_object.o \ ++ i915_gem_render_state.o \ ++ i915_gem_shrinker.o \ ++ i915_gem_stolen.o \ ++ i915_gem_tiling.o \ ++ i915_gem_userptr.o \ ++ i915_gemfs.o \ ++ i915_globals.o \ ++ i915_query.o \ ++ i915_request.o \ ++ i915_scheduler.o \ ++ i915_timeline.o \ ++ i915_trace_points.o \ ++ i915_vma.o \ ++ intel_breadcrumbs.o \ ++ intel_context.o \ ++ intel_engine_cs.o \ ++ intel_hangcheck.o \ ++ intel_lrc.o \ ++ intel_mocs.o \ ++ intel_ringbuffer.o \ ++ intel_uncore.o \ ++ intel_wopcm.o ++ ++# general-purpose microcontroller (GuC) support ++i915-y += intel_uc.o \ ++ intel_uc_fw.o \ ++ intel_guc.o \ ++ intel_guc_ads.o \ ++ intel_guc_ct.o \ ++ intel_guc_fw.o \ ++ intel_guc_log.o \ ++ intel_guc_submission.o \ ++ intel_huc.o \ ++ intel_huc_fw.o ++ ++# autogenerated null render state ++i915-y += intel_renderstate_gen6.o \ ++ intel_renderstate_gen7.o \ ++ intel_renderstate_gen8.o \ ++ intel_renderstate_gen9.o ++ ++# modesetting core code ++i915-y += intel_audio.o \ ++ intel_atomic.o \ ++ intel_atomic_plane.o \ ++ intel_bios.o \ ++ intel_cdclk.o \ ++ intel_color.o \ ++ intel_combo_phy.o \ ++ intel_connector.o \ ++ intel_display.o \ ++ intel_dpio_phy.o \ ++ intel_dpll_mgr.o \ ++ intel_fbc.o \ ++ intel_fifo_underrun.o \ ++ intel_frontbuffer.o \ ++ intel_hdcp.o \ ++ intel_hotplug.o \ ++ intel_overlay.o \ ++ intel_psr.o \ ++ intel_quirks.o \ ++ intel_sideband.o \ ++ intel_sprite.o ++i915-$(CONFIG_ACPI) += intel_acpi.o intel_opregion.o ++i915-$(CONFIG_DRM_FBDEV_EMULATION) += intel_fbdev.o ++ ++# modesetting output/encoder code ++i915-y += dvo_ch7017.o \ ++ dvo_ch7xxx.o \ ++ dvo_ivch.o \ ++ dvo_ns2501.o \ ++ dvo_sil164.o \ ++ dvo_tfp410.o \ ++ icl_dsi.o \ ++ intel_crt.o \ ++ intel_ddi.o \ ++ intel_dp_aux_backlight.o \ ++ intel_dp_link_training.o \ ++ intel_dp_mst.o \ ++ intel_dp.o \ ++ intel_dsi.o \ ++ intel_dsi_dcs_backlight.o \ ++ intel_dsi_vbt.o \ ++ intel_dvo.o \ ++ intel_hdmi.o \ ++ intel_i2c.o \ ++ intel_lspcon.o \ ++ intel_lvds.o \ ++ intel_panel.o \ ++ intel_sdvo.o \ ++ intel_tv.o \ ++ vlv_dsi.o \ ++ vlv_dsi_pll.o \ ++ intel_vdsc.o ++ ++# Post-mortem debug and GPU hang state capture ++i915-$(CONFIG_DRM_I915_CAPTURE_ERROR) += i915_gpu_error.o ++i915-$(CONFIG_DRM_I915_SELFTEST) += \ ++ selftests/i915_random.o \ ++ selftests/i915_selftest.o \ ++ selftests/igt_flush_test.o \ ++ selftests/igt_live_test.o \ ++ selftests/igt_reset.o \ ++ selftests/igt_spinner.o ++ ++# virtual gpu code ++i915-y += i915_vgpu.o ++ ++# perf code ++i915-y += i915_perf.o \ ++ i915_oa_hsw.o \ ++ i915_oa_bdw.o \ ++ i915_oa_chv.o \ ++ i915_oa_sklgt2.o \ ++ i915_oa_sklgt3.o \ ++ i915_oa_sklgt4.o \ ++ i915_oa_bxt.o \ ++ i915_oa_kblgt2.o \ ++ i915_oa_kblgt3.o \ ++ i915_oa_glk.o \ ++ i915_oa_cflgt2.o \ ++ i915_oa_cflgt3.o \ ++ i915_oa_cnl.o \ ++ i915_oa_icl.o ++ ++ifeq ($(CONFIG_DRM_I915_GVT),y) ++i915-y += intel_gvt.o ++include $(src)/gvt/Makefile ++endif ++ ++# LPE Audio for VLV and CHT ++i915-y += intel_lpe_audio.o ++ ++obj-$(CONFIG_DRM_I915) += i915.o ++obj-$(CONFIG_DRM_I915_GVT_KVMGT) += gvt/kvmgt.o +diff --git a/drivers/gpu/drm/i915_legacy/Makefile.header-test b/drivers/gpu/drm/i915_legacy/Makefile.header-test +new file mode 100644 +index 000000000000..c1c391816fa7 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/Makefile.header-test +@@ -0,0 +1,47 @@ ++# SPDX-License-Identifier: MIT ++# Copyright © 2019 Intel Corporation ++ ++# Test the headers are compilable as standalone units ++header_test := \ ++ i915_active_types.h \ ++ i915_gem_context_types.h \ ++ i915_priolist_types.h \ ++ i915_scheduler_types.h \ ++ i915_timeline_types.h \ ++ intel_atomic_plane.h \ ++ intel_audio.h \ ++ intel_cdclk.h \ ++ intel_color.h \ ++ intel_connector.h \ ++ intel_context_types.h \ ++ intel_crt.h \ ++ intel_csr.h \ ++ intel_ddi.h \ ++ intel_dp.h \ ++ intel_dvo.h \ ++ intel_engine_types.h \ ++ intel_fbc.h \ ++ intel_fbdev.h \ ++ intel_frontbuffer.h \ ++ intel_hdcp.h \ ++ intel_hdmi.h \ ++ intel_lspcon.h \ ++ intel_lvds.h \ ++ intel_panel.h \ ++ intel_pipe_crc.h \ ++ intel_pm.h \ ++ intel_psr.h \ ++ intel_sdvo.h \ ++ intel_sprite.h \ ++ intel_tv.h \ ++ intel_workarounds_types.h ++ ++quiet_cmd_header_test = HDRTEST $@ ++ cmd_header_test = echo "\#include \"$( $@ ++ ++header_test_%.c: %.h ++ $(call cmd,header_test) ++ ++i915-$(CONFIG_DRM_I915_WERROR) += $(foreach h,$(header_test),$(patsubst %.h,header_test_%.o,$(h))) ++ ++clean-files += $(foreach h,$(header_test),$(patsubst %.h,header_test_%.c,$(h))) +diff --git a/drivers/gpu/drm/i915_legacy/dvo.h b/drivers/gpu/drm/i915_legacy/dvo.h +new file mode 100644 +index 000000000000..16e0345b711f +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/dvo.h +@@ -0,0 +1,138 @@ ++/* ++ * Copyright © 2006 Eric Anholt ++ * ++ * Permission to use, copy, modify, distribute, and sell this software and its ++ * documentation for any purpose is hereby granted without fee, provided that ++ * the above copyright notice appear in all copies and that both that copyright ++ * notice and this permission notice appear in supporting documentation, and ++ * that the name of the copyright holders not be used in advertising or ++ * publicity pertaining to distribution of the software without specific, ++ * written prior permission. The copyright holders make no representations ++ * about the suitability of this software for any purpose. It is provided "as ++ * is" without express or implied warranty. ++ * ++ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, ++ * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO ++ * EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR ++ * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, ++ * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER ++ * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE ++ * OF THIS SOFTWARE. ++ */ ++ ++#ifndef _INTEL_DVO_H ++#define _INTEL_DVO_H ++ ++#include ++#include ++#include "intel_drv.h" ++ ++struct intel_dvo_device { ++ const char *name; ++ int type; ++ /* DVOA/B/C output register */ ++ i915_reg_t dvo_reg; ++ i915_reg_t dvo_srcdim_reg; ++ /* GPIO register used for i2c bus to control this device */ ++ u32 gpio; ++ int slave_addr; ++ ++ const struct intel_dvo_dev_ops *dev_ops; ++ void *dev_priv; ++ struct i2c_adapter *i2c_bus; ++}; ++ ++struct intel_dvo_dev_ops { ++ /* ++ * Initialize the device at startup time. ++ * Returns NULL if the device does not exist. ++ */ ++ bool (*init)(struct intel_dvo_device *dvo, ++ struct i2c_adapter *i2cbus); ++ ++ /* ++ * Called to allow the output a chance to create properties after the ++ * RandR objects have been created. ++ */ ++ void (*create_resources)(struct intel_dvo_device *dvo); ++ ++ /* ++ * Turn on/off output. ++ * ++ * Because none of our dvo drivers support an intermediate power levels, ++ * we don't expose this in the interfac. ++ */ ++ void (*dpms)(struct intel_dvo_device *dvo, bool enable); ++ ++ /* ++ * Callback for testing a video mode for a given output. ++ * ++ * This function should only check for cases where a mode can't ++ * be supported on the output specifically, and not represent ++ * generic CRTC limitations. ++ * ++ * \return MODE_OK if the mode is valid, or another MODE_* otherwise. ++ */ ++ int (*mode_valid)(struct intel_dvo_device *dvo, ++ struct drm_display_mode *mode); ++ ++ /* ++ * Callback for preparing mode changes on an output ++ */ ++ void (*prepare)(struct intel_dvo_device *dvo); ++ ++ /* ++ * Callback for committing mode changes on an output ++ */ ++ void (*commit)(struct intel_dvo_device *dvo); ++ ++ /* ++ * Callback for setting up a video mode after fixups have been made. ++ * ++ * This is only called while the output is disabled. The dpms callback ++ * must be all that's necessary for the output, to turn the output on ++ * after this function is called. ++ */ ++ void (*mode_set)(struct intel_dvo_device *dvo, ++ const struct drm_display_mode *mode, ++ const struct drm_display_mode *adjusted_mode); ++ ++ /* ++ * Probe for a connected output, and return detect_status. ++ */ ++ enum drm_connector_status (*detect)(struct intel_dvo_device *dvo); ++ ++ /* ++ * Probe the current hw status, returning true if the connected output ++ * is active. ++ */ ++ bool (*get_hw_state)(struct intel_dvo_device *dev); ++ ++ /** ++ * Query the device for the modes it provides. ++ * ++ * This function may also update MonInfo, mm_width, and mm_height. ++ * ++ * \return singly-linked list of modes or NULL if no modes found. ++ */ ++ struct drm_display_mode *(*get_modes)(struct intel_dvo_device *dvo); ++ ++ /** ++ * Clean up driver-specific bits of the output ++ */ ++ void (*destroy) (struct intel_dvo_device *dvo); ++ ++ /** ++ * Debugging hook to dump device registers to log file ++ */ ++ void (*dump_regs)(struct intel_dvo_device *dvo); ++}; ++ ++extern const struct intel_dvo_dev_ops sil164_ops; ++extern const struct intel_dvo_dev_ops ch7xxx_ops; ++extern const struct intel_dvo_dev_ops ivch_ops; ++extern const struct intel_dvo_dev_ops tfp410_ops; ++extern const struct intel_dvo_dev_ops ch7017_ops; ++extern const struct intel_dvo_dev_ops ns2501_ops; ++ ++#endif /* _INTEL_DVO_H */ +diff --git a/drivers/gpu/drm/i915_legacy/dvo_ch7017.c b/drivers/gpu/drm/i915_legacy/dvo_ch7017.c +new file mode 100644 +index 000000000000..caac9942e1e3 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/dvo_ch7017.c +@@ -0,0 +1,414 @@ ++/* ++ * Copyright © 2006 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER ++ * DEALINGS IN THE SOFTWARE. ++ * ++ * Authors: ++ * Eric Anholt ++ * ++ */ ++ ++#include "dvo.h" ++ ++#define CH7017_TV_DISPLAY_MODE 0x00 ++#define CH7017_FLICKER_FILTER 0x01 ++#define CH7017_VIDEO_BANDWIDTH 0x02 ++#define CH7017_TEXT_ENHANCEMENT 0x03 ++#define CH7017_START_ACTIVE_VIDEO 0x04 ++#define CH7017_HORIZONTAL_POSITION 0x05 ++#define CH7017_VERTICAL_POSITION 0x06 ++#define CH7017_BLACK_LEVEL 0x07 ++#define CH7017_CONTRAST_ENHANCEMENT 0x08 ++#define CH7017_TV_PLL 0x09 ++#define CH7017_TV_PLL_M 0x0a ++#define CH7017_TV_PLL_N 0x0b ++#define CH7017_SUB_CARRIER_0 0x0c ++#define CH7017_CIV_CONTROL 0x10 ++#define CH7017_CIV_0 0x11 ++#define CH7017_CHROMA_BOOST 0x14 ++#define CH7017_CLOCK_MODE 0x1c ++#define CH7017_INPUT_CLOCK 0x1d ++#define CH7017_GPIO_CONTROL 0x1e ++#define CH7017_INPUT_DATA_FORMAT 0x1f ++#define CH7017_CONNECTION_DETECT 0x20 ++#define CH7017_DAC_CONTROL 0x21 ++#define CH7017_BUFFERED_CLOCK_OUTPUT 0x22 ++#define CH7017_DEFEAT_VSYNC 0x47 ++#define CH7017_TEST_PATTERN 0x48 ++ ++#define CH7017_POWER_MANAGEMENT 0x49 ++/** Enables the TV output path. */ ++#define CH7017_TV_EN (1 << 0) ++#define CH7017_DAC0_POWER_DOWN (1 << 1) ++#define CH7017_DAC1_POWER_DOWN (1 << 2) ++#define CH7017_DAC2_POWER_DOWN (1 << 3) ++#define CH7017_DAC3_POWER_DOWN (1 << 4) ++/** Powers down the TV out block, and DAC0-3 */ ++#define CH7017_TV_POWER_DOWN_EN (1 << 5) ++ ++#define CH7017_VERSION_ID 0x4a ++ ++#define CH7017_DEVICE_ID 0x4b ++#define CH7017_DEVICE_ID_VALUE 0x1b ++#define CH7018_DEVICE_ID_VALUE 0x1a ++#define CH7019_DEVICE_ID_VALUE 0x19 ++ ++#define CH7017_XCLK_D2_ADJUST 0x53 ++#define CH7017_UP_SCALER_COEFF_0 0x55 ++#define CH7017_UP_SCALER_COEFF_1 0x56 ++#define CH7017_UP_SCALER_COEFF_2 0x57 ++#define CH7017_UP_SCALER_COEFF_3 0x58 ++#define CH7017_UP_SCALER_COEFF_4 0x59 ++#define CH7017_UP_SCALER_VERTICAL_INC_0 0x5a ++#define CH7017_UP_SCALER_VERTICAL_INC_1 0x5b ++#define CH7017_GPIO_INVERT 0x5c ++#define CH7017_UP_SCALER_HORIZONTAL_INC_0 0x5d ++#define CH7017_UP_SCALER_HORIZONTAL_INC_1 0x5e ++ ++#define CH7017_HORIZONTAL_ACTIVE_PIXEL_INPUT 0x5f ++/**< Low bits of horizontal active pixel input */ ++ ++#define CH7017_ACTIVE_INPUT_LINE_OUTPUT 0x60 ++/** High bits of horizontal active pixel input */ ++#define CH7017_LVDS_HAP_INPUT_MASK (0x7 << 0) ++/** High bits of vertical active line output */ ++#define CH7017_LVDS_VAL_HIGH_MASK (0x7 << 3) ++ ++#define CH7017_VERTICAL_ACTIVE_LINE_OUTPUT 0x61 ++/**< Low bits of vertical active line output */ ++ ++#define CH7017_HORIZONTAL_ACTIVE_PIXEL_OUTPUT 0x62 ++/**< Low bits of horizontal active pixel output */ ++ ++#define CH7017_LVDS_POWER_DOWN 0x63 ++/** High bits of horizontal active pixel output */ ++#define CH7017_LVDS_HAP_HIGH_MASK (0x7 << 0) ++/** Enables the LVDS power down state transition */ ++#define CH7017_LVDS_POWER_DOWN_EN (1 << 6) ++/** Enables the LVDS upscaler */ ++#define CH7017_LVDS_UPSCALER_EN (1 << 7) ++#define CH7017_LVDS_POWER_DOWN_DEFAULT_RESERVED 0x08 ++ ++#define CH7017_LVDS_ENCODING 0x64 ++#define CH7017_LVDS_DITHER_2D (1 << 2) ++#define CH7017_LVDS_DITHER_DIS (1 << 3) ++#define CH7017_LVDS_DUAL_CHANNEL_EN (1 << 4) ++#define CH7017_LVDS_24_BIT (1 << 5) ++ ++#define CH7017_LVDS_ENCODING_2 0x65 ++ ++#define CH7017_LVDS_PLL_CONTROL 0x66 ++/** Enables the LVDS panel output path */ ++#define CH7017_LVDS_PANEN (1 << 0) ++/** Enables the LVDS panel backlight */ ++#define CH7017_LVDS_BKLEN (1 << 3) ++ ++#define CH7017_POWER_SEQUENCING_T1 0x67 ++#define CH7017_POWER_SEQUENCING_T2 0x68 ++#define CH7017_POWER_SEQUENCING_T3 0x69 ++#define CH7017_POWER_SEQUENCING_T4 0x6a ++#define CH7017_POWER_SEQUENCING_T5 0x6b ++#define CH7017_GPIO_DRIVER_TYPE 0x6c ++#define CH7017_GPIO_DATA 0x6d ++#define CH7017_GPIO_DIRECTION_CONTROL 0x6e ++ ++#define CH7017_LVDS_PLL_FEEDBACK_DIV 0x71 ++# define CH7017_LVDS_PLL_FEED_BACK_DIVIDER_SHIFT 4 ++# define CH7017_LVDS_PLL_FEED_FORWARD_DIVIDER_SHIFT 0 ++# define CH7017_LVDS_PLL_FEEDBACK_DEFAULT_RESERVED 0x80 ++ ++#define CH7017_LVDS_PLL_VCO_CONTROL 0x72 ++# define CH7017_LVDS_PLL_VCO_DEFAULT_RESERVED 0x80 ++# define CH7017_LVDS_PLL_VCO_SHIFT 4 ++# define CH7017_LVDS_PLL_POST_SCALE_DIV_SHIFT 0 ++ ++#define CH7017_OUTPUTS_ENABLE 0x73 ++# define CH7017_CHARGE_PUMP_LOW 0x0 ++# define CH7017_CHARGE_PUMP_HIGH 0x3 ++# define CH7017_LVDS_CHANNEL_A (1 << 3) ++# define CH7017_LVDS_CHANNEL_B (1 << 4) ++# define CH7017_TV_DAC_A (1 << 5) ++# define CH7017_TV_DAC_B (1 << 6) ++# define CH7017_DDC_SELECT_DC2 (1 << 7) ++ ++#define CH7017_LVDS_OUTPUT_AMPLITUDE 0x74 ++#define CH7017_LVDS_PLL_EMI_REDUCTION 0x75 ++#define CH7017_LVDS_POWER_DOWN_FLICKER 0x76 ++ ++#define CH7017_LVDS_CONTROL_2 0x78 ++# define CH7017_LOOP_FILTER_SHIFT 5 ++# define CH7017_PHASE_DETECTOR_SHIFT 0 ++ ++#define CH7017_BANG_LIMIT_CONTROL 0x7f ++ ++struct ch7017_priv { ++ u8 dummy; ++}; ++ ++static void ch7017_dump_regs(struct intel_dvo_device *dvo); ++static void ch7017_dpms(struct intel_dvo_device *dvo, bool enable); ++ ++static bool ch7017_read(struct intel_dvo_device *dvo, u8 addr, u8 *val) ++{ ++ struct i2c_msg msgs[] = { ++ { ++ .addr = dvo->slave_addr, ++ .flags = 0, ++ .len = 1, ++ .buf = &addr, ++ }, ++ { ++ .addr = dvo->slave_addr, ++ .flags = I2C_M_RD, ++ .len = 1, ++ .buf = val, ++ } ++ }; ++ return i2c_transfer(dvo->i2c_bus, msgs, 2) == 2; ++} ++ ++static bool ch7017_write(struct intel_dvo_device *dvo, u8 addr, u8 val) ++{ ++ u8 buf[2] = { addr, val }; ++ struct i2c_msg msg = { ++ .addr = dvo->slave_addr, ++ .flags = 0, ++ .len = 2, ++ .buf = buf, ++ }; ++ return i2c_transfer(dvo->i2c_bus, &msg, 1) == 1; ++} ++ ++/** Probes for a CH7017 on the given bus and slave address. */ ++static bool ch7017_init(struct intel_dvo_device *dvo, ++ struct i2c_adapter *adapter) ++{ ++ struct ch7017_priv *priv; ++ const char *str; ++ u8 val; ++ ++ priv = kzalloc(sizeof(struct ch7017_priv), GFP_KERNEL); ++ if (priv == NULL) ++ return false; ++ ++ dvo->i2c_bus = adapter; ++ dvo->dev_priv = priv; ++ ++ if (!ch7017_read(dvo, CH7017_DEVICE_ID, &val)) ++ goto fail; ++ ++ switch (val) { ++ case CH7017_DEVICE_ID_VALUE: ++ str = "ch7017"; ++ break; ++ case CH7018_DEVICE_ID_VALUE: ++ str = "ch7018"; ++ break; ++ case CH7019_DEVICE_ID_VALUE: ++ str = "ch7019"; ++ break; ++ default: ++ DRM_DEBUG_KMS("ch701x not detected, got %d: from %s " ++ "slave %d.\n", ++ val, adapter->name, dvo->slave_addr); ++ goto fail; ++ } ++ ++ DRM_DEBUG_KMS("%s detected on %s, addr %d\n", ++ str, adapter->name, dvo->slave_addr); ++ return true; ++ ++fail: ++ kfree(priv); ++ return false; ++} ++ ++static enum drm_connector_status ch7017_detect(struct intel_dvo_device *dvo) ++{ ++ return connector_status_connected; ++} ++ ++static enum drm_mode_status ch7017_mode_valid(struct intel_dvo_device *dvo, ++ struct drm_display_mode *mode) ++{ ++ if (mode->clock > 160000) ++ return MODE_CLOCK_HIGH; ++ ++ return MODE_OK; ++} ++ ++static void ch7017_mode_set(struct intel_dvo_device *dvo, ++ const struct drm_display_mode *mode, ++ const struct drm_display_mode *adjusted_mode) ++{ ++ u8 lvds_pll_feedback_div, lvds_pll_vco_control; ++ u8 outputs_enable, lvds_control_2, lvds_power_down; ++ u8 horizontal_active_pixel_input; ++ u8 horizontal_active_pixel_output, vertical_active_line_output; ++ u8 active_input_line_output; ++ ++ DRM_DEBUG_KMS("Registers before mode setting\n"); ++ ch7017_dump_regs(dvo); ++ ++ /* LVDS PLL settings from page 75 of 7017-7017ds.pdf*/ ++ if (mode->clock < 100000) { ++ outputs_enable = CH7017_LVDS_CHANNEL_A | CH7017_CHARGE_PUMP_LOW; ++ lvds_pll_feedback_div = CH7017_LVDS_PLL_FEEDBACK_DEFAULT_RESERVED | ++ (2 << CH7017_LVDS_PLL_FEED_BACK_DIVIDER_SHIFT) | ++ (13 << CH7017_LVDS_PLL_FEED_FORWARD_DIVIDER_SHIFT); ++ lvds_pll_vco_control = CH7017_LVDS_PLL_VCO_DEFAULT_RESERVED | ++ (2 << CH7017_LVDS_PLL_VCO_SHIFT) | ++ (3 << CH7017_LVDS_PLL_POST_SCALE_DIV_SHIFT); ++ lvds_control_2 = (1 << CH7017_LOOP_FILTER_SHIFT) | ++ (0 << CH7017_PHASE_DETECTOR_SHIFT); ++ } else { ++ outputs_enable = CH7017_LVDS_CHANNEL_A | CH7017_CHARGE_PUMP_HIGH; ++ lvds_pll_feedback_div = ++ CH7017_LVDS_PLL_FEEDBACK_DEFAULT_RESERVED | ++ (2 << CH7017_LVDS_PLL_FEED_BACK_DIVIDER_SHIFT) | ++ (3 << CH7017_LVDS_PLL_FEED_FORWARD_DIVIDER_SHIFT); ++ lvds_control_2 = (3 << CH7017_LOOP_FILTER_SHIFT) | ++ (0 << CH7017_PHASE_DETECTOR_SHIFT); ++ if (1) { /* XXX: dual channel panel detection. Assume yes for now. */ ++ outputs_enable |= CH7017_LVDS_CHANNEL_B; ++ lvds_pll_vco_control = CH7017_LVDS_PLL_VCO_DEFAULT_RESERVED | ++ (2 << CH7017_LVDS_PLL_VCO_SHIFT) | ++ (13 << CH7017_LVDS_PLL_POST_SCALE_DIV_SHIFT); ++ } else { ++ lvds_pll_vco_control = CH7017_LVDS_PLL_VCO_DEFAULT_RESERVED | ++ (1 << CH7017_LVDS_PLL_VCO_SHIFT) | ++ (13 << CH7017_LVDS_PLL_POST_SCALE_DIV_SHIFT); ++ } ++ } ++ ++ horizontal_active_pixel_input = mode->hdisplay & 0x00ff; ++ ++ vertical_active_line_output = mode->vdisplay & 0x00ff; ++ horizontal_active_pixel_output = mode->hdisplay & 0x00ff; ++ ++ active_input_line_output = ((mode->hdisplay & 0x0700) >> 8) | ++ (((mode->vdisplay & 0x0700) >> 8) << 3); ++ ++ lvds_power_down = CH7017_LVDS_POWER_DOWN_DEFAULT_RESERVED | ++ (mode->hdisplay & 0x0700) >> 8; ++ ++ ch7017_dpms(dvo, false); ++ ch7017_write(dvo, CH7017_HORIZONTAL_ACTIVE_PIXEL_INPUT, ++ horizontal_active_pixel_input); ++ ch7017_write(dvo, CH7017_HORIZONTAL_ACTIVE_PIXEL_OUTPUT, ++ horizontal_active_pixel_output); ++ ch7017_write(dvo, CH7017_VERTICAL_ACTIVE_LINE_OUTPUT, ++ vertical_active_line_output); ++ ch7017_write(dvo, CH7017_ACTIVE_INPUT_LINE_OUTPUT, ++ active_input_line_output); ++ ch7017_write(dvo, CH7017_LVDS_PLL_VCO_CONTROL, lvds_pll_vco_control); ++ ch7017_write(dvo, CH7017_LVDS_PLL_FEEDBACK_DIV, lvds_pll_feedback_div); ++ ch7017_write(dvo, CH7017_LVDS_CONTROL_2, lvds_control_2); ++ ch7017_write(dvo, CH7017_OUTPUTS_ENABLE, outputs_enable); ++ ++ /* Turn the LVDS back on with new settings. */ ++ ch7017_write(dvo, CH7017_LVDS_POWER_DOWN, lvds_power_down); ++ ++ DRM_DEBUG_KMS("Registers after mode setting\n"); ++ ch7017_dump_regs(dvo); ++} ++ ++/* set the CH7017 power state */ ++static void ch7017_dpms(struct intel_dvo_device *dvo, bool enable) ++{ ++ u8 val; ++ ++ ch7017_read(dvo, CH7017_LVDS_POWER_DOWN, &val); ++ ++ /* Turn off TV/VGA, and never turn it on since we don't support it. */ ++ ch7017_write(dvo, CH7017_POWER_MANAGEMENT, ++ CH7017_DAC0_POWER_DOWN | ++ CH7017_DAC1_POWER_DOWN | ++ CH7017_DAC2_POWER_DOWN | ++ CH7017_DAC3_POWER_DOWN | ++ CH7017_TV_POWER_DOWN_EN); ++ ++ if (enable) { ++ /* Turn on the LVDS */ ++ ch7017_write(dvo, CH7017_LVDS_POWER_DOWN, ++ val & ~CH7017_LVDS_POWER_DOWN_EN); ++ } else { ++ /* Turn off the LVDS */ ++ ch7017_write(dvo, CH7017_LVDS_POWER_DOWN, ++ val | CH7017_LVDS_POWER_DOWN_EN); ++ } ++ ++ /* XXX: Should actually wait for update power status somehow */ ++ msleep(20); ++} ++ ++static bool ch7017_get_hw_state(struct intel_dvo_device *dvo) ++{ ++ u8 val; ++ ++ ch7017_read(dvo, CH7017_LVDS_POWER_DOWN, &val); ++ ++ if (val & CH7017_LVDS_POWER_DOWN_EN) ++ return false; ++ else ++ return true; ++} ++ ++static void ch7017_dump_regs(struct intel_dvo_device *dvo) ++{ ++ u8 val; ++ ++#define DUMP(reg) \ ++do { \ ++ ch7017_read(dvo, reg, &val); \ ++ DRM_DEBUG_KMS(#reg ": %02x\n", val); \ ++} while (0) ++ ++ DUMP(CH7017_HORIZONTAL_ACTIVE_PIXEL_INPUT); ++ DUMP(CH7017_HORIZONTAL_ACTIVE_PIXEL_OUTPUT); ++ DUMP(CH7017_VERTICAL_ACTIVE_LINE_OUTPUT); ++ DUMP(CH7017_ACTIVE_INPUT_LINE_OUTPUT); ++ DUMP(CH7017_LVDS_PLL_VCO_CONTROL); ++ DUMP(CH7017_LVDS_PLL_FEEDBACK_DIV); ++ DUMP(CH7017_LVDS_CONTROL_2); ++ DUMP(CH7017_OUTPUTS_ENABLE); ++ DUMP(CH7017_LVDS_POWER_DOWN); ++} ++ ++static void ch7017_destroy(struct intel_dvo_device *dvo) ++{ ++ struct ch7017_priv *priv = dvo->dev_priv; ++ ++ if (priv) { ++ kfree(priv); ++ dvo->dev_priv = NULL; ++ } ++} ++ ++const struct intel_dvo_dev_ops ch7017_ops = { ++ .init = ch7017_init, ++ .detect = ch7017_detect, ++ .mode_valid = ch7017_mode_valid, ++ .mode_set = ch7017_mode_set, ++ .dpms = ch7017_dpms, ++ .get_hw_state = ch7017_get_hw_state, ++ .dump_regs = ch7017_dump_regs, ++ .destroy = ch7017_destroy, ++}; +diff --git a/drivers/gpu/drm/i915_legacy/dvo_ch7xxx.c b/drivers/gpu/drm/i915_legacy/dvo_ch7xxx.c +new file mode 100644 +index 000000000000..397ac5233726 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/dvo_ch7xxx.c +@@ -0,0 +1,366 @@ ++/************************************************************************** ++ ++Copyright © 2006 Dave Airlie ++ ++All Rights Reserved. ++ ++Permission is hereby granted, free of charge, to any person obtaining a ++copy of this software and associated documentation files (the ++"Software"), to deal in the Software without restriction, including ++without limitation the rights to use, copy, modify, merge, publish, ++distribute, sub license, and/or sell copies of the Software, and to ++permit persons to whom the Software is furnished to do so, subject to ++the following conditions: ++ ++The above copyright notice and this permission notice (including the ++next paragraph) shall be included in all copies or substantial portions ++of the Software. ++ ++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS ++OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF ++MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. ++IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ++ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, ++TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE ++SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ++ ++**************************************************************************/ ++ ++#include "dvo.h" ++ ++#define CH7xxx_REG_VID 0x4a ++#define CH7xxx_REG_DID 0x4b ++ ++#define CH7011_VID 0x83 /* 7010 as well */ ++#define CH7010B_VID 0x05 ++#define CH7009A_VID 0x84 ++#define CH7009B_VID 0x85 ++#define CH7301_VID 0x95 ++ ++#define CH7xxx_VID 0x84 ++#define CH7xxx_DID 0x17 ++#define CH7010_DID 0x16 ++ ++#define CH7xxx_NUM_REGS 0x4c ++ ++#define CH7xxx_CM 0x1c ++#define CH7xxx_CM_XCM (1<<0) ++#define CH7xxx_CM_MCP (1<<2) ++#define CH7xxx_INPUT_CLOCK 0x1d ++#define CH7xxx_GPIO 0x1e ++#define CH7xxx_GPIO_HPIR (1<<3) ++#define CH7xxx_IDF 0x1f ++ ++#define CH7xxx_IDF_HSP (1<<3) ++#define CH7xxx_IDF_VSP (1<<4) ++ ++#define CH7xxx_CONNECTION_DETECT 0x20 ++#define CH7xxx_CDET_DVI (1<<5) ++ ++#define CH7301_DAC_CNTL 0x21 ++#define CH7301_HOTPLUG 0x23 ++#define CH7xxx_TCTL 0x31 ++#define CH7xxx_TVCO 0x32 ++#define CH7xxx_TPCP 0x33 ++#define CH7xxx_TPD 0x34 ++#define CH7xxx_TPVT 0x35 ++#define CH7xxx_TLPF 0x36 ++#define CH7xxx_TCT 0x37 ++#define CH7301_TEST_PATTERN 0x48 ++ ++#define CH7xxx_PM 0x49 ++#define CH7xxx_PM_FPD (1<<0) ++#define CH7301_PM_DACPD0 (1<<1) ++#define CH7301_PM_DACPD1 (1<<2) ++#define CH7301_PM_DACPD2 (1<<3) ++#define CH7xxx_PM_DVIL (1<<6) ++#define CH7xxx_PM_DVIP (1<<7) ++ ++#define CH7301_SYNC_POLARITY 0x56 ++#define CH7301_SYNC_RGB_YUV (1<<0) ++#define CH7301_SYNC_POL_DVI (1<<5) ++ ++/** @file ++ * driver for the Chrontel 7xxx DVI chip over DVO. ++ */ ++ ++static struct ch7xxx_id_struct { ++ u8 vid; ++ char *name; ++} ch7xxx_ids[] = { ++ { CH7011_VID, "CH7011" }, ++ { CH7010B_VID, "CH7010B" }, ++ { CH7009A_VID, "CH7009A" }, ++ { CH7009B_VID, "CH7009B" }, ++ { CH7301_VID, "CH7301" }, ++}; ++ ++static struct ch7xxx_did_struct { ++ u8 did; ++ char *name; ++} ch7xxx_dids[] = { ++ { CH7xxx_DID, "CH7XXX" }, ++ { CH7010_DID, "CH7010B" }, ++}; ++ ++struct ch7xxx_priv { ++ bool quiet; ++}; ++ ++static char *ch7xxx_get_id(u8 vid) ++{ ++ int i; ++ ++ for (i = 0; i < ARRAY_SIZE(ch7xxx_ids); i++) { ++ if (ch7xxx_ids[i].vid == vid) ++ return ch7xxx_ids[i].name; ++ } ++ ++ return NULL; ++} ++ ++static char *ch7xxx_get_did(u8 did) ++{ ++ int i; ++ ++ for (i = 0; i < ARRAY_SIZE(ch7xxx_dids); i++) { ++ if (ch7xxx_dids[i].did == did) ++ return ch7xxx_dids[i].name; ++ } ++ ++ return NULL; ++} ++ ++/** Reads an 8 bit register */ ++static bool ch7xxx_readb(struct intel_dvo_device *dvo, int addr, u8 *ch) ++{ ++ struct ch7xxx_priv *ch7xxx = dvo->dev_priv; ++ struct i2c_adapter *adapter = dvo->i2c_bus; ++ u8 out_buf[2]; ++ u8 in_buf[2]; ++ ++ struct i2c_msg msgs[] = { ++ { ++ .addr = dvo->slave_addr, ++ .flags = 0, ++ .len = 1, ++ .buf = out_buf, ++ }, ++ { ++ .addr = dvo->slave_addr, ++ .flags = I2C_M_RD, ++ .len = 1, ++ .buf = in_buf, ++ } ++ }; ++ ++ out_buf[0] = addr; ++ out_buf[1] = 0; ++ ++ if (i2c_transfer(adapter, msgs, 2) == 2) { ++ *ch = in_buf[0]; ++ return true; ++ } ++ ++ if (!ch7xxx->quiet) { ++ DRM_DEBUG_KMS("Unable to read register 0x%02x from %s:%02x.\n", ++ addr, adapter->name, dvo->slave_addr); ++ } ++ return false; ++} ++ ++/** Writes an 8 bit register */ ++static bool ch7xxx_writeb(struct intel_dvo_device *dvo, int addr, u8 ch) ++{ ++ struct ch7xxx_priv *ch7xxx = dvo->dev_priv; ++ struct i2c_adapter *adapter = dvo->i2c_bus; ++ u8 out_buf[2]; ++ struct i2c_msg msg = { ++ .addr = dvo->slave_addr, ++ .flags = 0, ++ .len = 2, ++ .buf = out_buf, ++ }; ++ ++ out_buf[0] = addr; ++ out_buf[1] = ch; ++ ++ if (i2c_transfer(adapter, &msg, 1) == 1) ++ return true; ++ ++ if (!ch7xxx->quiet) { ++ DRM_DEBUG_KMS("Unable to write register 0x%02x to %s:%d.\n", ++ addr, adapter->name, dvo->slave_addr); ++ } ++ ++ return false; ++} ++ ++static bool ch7xxx_init(struct intel_dvo_device *dvo, ++ struct i2c_adapter *adapter) ++{ ++ /* this will detect the CH7xxx chip on the specified i2c bus */ ++ struct ch7xxx_priv *ch7xxx; ++ u8 vendor, device; ++ char *name, *devid; ++ ++ ch7xxx = kzalloc(sizeof(struct ch7xxx_priv), GFP_KERNEL); ++ if (ch7xxx == NULL) ++ return false; ++ ++ dvo->i2c_bus = adapter; ++ dvo->dev_priv = ch7xxx; ++ ch7xxx->quiet = true; ++ ++ if (!ch7xxx_readb(dvo, CH7xxx_REG_VID, &vendor)) ++ goto out; ++ ++ name = ch7xxx_get_id(vendor); ++ if (!name) { ++ DRM_DEBUG_KMS("ch7xxx not detected; got VID 0x%02x from %s slave %d.\n", ++ vendor, adapter->name, dvo->slave_addr); ++ goto out; ++ } ++ ++ ++ if (!ch7xxx_readb(dvo, CH7xxx_REG_DID, &device)) ++ goto out; ++ ++ devid = ch7xxx_get_did(device); ++ if (!devid) { ++ DRM_DEBUG_KMS("ch7xxx not detected; got DID 0x%02x from %s slave %d.\n", ++ device, adapter->name, dvo->slave_addr); ++ goto out; ++ } ++ ++ ch7xxx->quiet = false; ++ DRM_DEBUG_KMS("Detected %s chipset, vendor/device ID 0x%02x/0x%02x\n", ++ name, vendor, device); ++ return true; ++out: ++ kfree(ch7xxx); ++ return false; ++} ++ ++static enum drm_connector_status ch7xxx_detect(struct intel_dvo_device *dvo) ++{ ++ u8 cdet, orig_pm, pm; ++ ++ ch7xxx_readb(dvo, CH7xxx_PM, &orig_pm); ++ ++ pm = orig_pm; ++ pm &= ~CH7xxx_PM_FPD; ++ pm |= CH7xxx_PM_DVIL | CH7xxx_PM_DVIP; ++ ++ ch7xxx_writeb(dvo, CH7xxx_PM, pm); ++ ++ ch7xxx_readb(dvo, CH7xxx_CONNECTION_DETECT, &cdet); ++ ++ ch7xxx_writeb(dvo, CH7xxx_PM, orig_pm); ++ ++ if (cdet & CH7xxx_CDET_DVI) ++ return connector_status_connected; ++ return connector_status_disconnected; ++} ++ ++static enum drm_mode_status ch7xxx_mode_valid(struct intel_dvo_device *dvo, ++ struct drm_display_mode *mode) ++{ ++ if (mode->clock > 165000) ++ return MODE_CLOCK_HIGH; ++ ++ return MODE_OK; ++} ++ ++static void ch7xxx_mode_set(struct intel_dvo_device *dvo, ++ const struct drm_display_mode *mode, ++ const struct drm_display_mode *adjusted_mode) ++{ ++ u8 tvco, tpcp, tpd, tlpf, idf; ++ ++ if (mode->clock <= 65000) { ++ tvco = 0x23; ++ tpcp = 0x08; ++ tpd = 0x16; ++ tlpf = 0x60; ++ } else { ++ tvco = 0x2d; ++ tpcp = 0x06; ++ tpd = 0x26; ++ tlpf = 0xa0; ++ } ++ ++ ch7xxx_writeb(dvo, CH7xxx_TCTL, 0x00); ++ ch7xxx_writeb(dvo, CH7xxx_TVCO, tvco); ++ ch7xxx_writeb(dvo, CH7xxx_TPCP, tpcp); ++ ch7xxx_writeb(dvo, CH7xxx_TPD, tpd); ++ ch7xxx_writeb(dvo, CH7xxx_TPVT, 0x30); ++ ch7xxx_writeb(dvo, CH7xxx_TLPF, tlpf); ++ ch7xxx_writeb(dvo, CH7xxx_TCT, 0x00); ++ ++ ch7xxx_readb(dvo, CH7xxx_IDF, &idf); ++ ++ idf &= ~(CH7xxx_IDF_HSP | CH7xxx_IDF_VSP); ++ if (mode->flags & DRM_MODE_FLAG_PHSYNC) ++ idf |= CH7xxx_IDF_HSP; ++ ++ if (mode->flags & DRM_MODE_FLAG_PVSYNC) ++ idf |= CH7xxx_IDF_VSP; ++ ++ ch7xxx_writeb(dvo, CH7xxx_IDF, idf); ++} ++ ++/* set the CH7xxx power state */ ++static void ch7xxx_dpms(struct intel_dvo_device *dvo, bool enable) ++{ ++ if (enable) ++ ch7xxx_writeb(dvo, CH7xxx_PM, CH7xxx_PM_DVIL | CH7xxx_PM_DVIP); ++ else ++ ch7xxx_writeb(dvo, CH7xxx_PM, CH7xxx_PM_FPD); ++} ++ ++static bool ch7xxx_get_hw_state(struct intel_dvo_device *dvo) ++{ ++ u8 val; ++ ++ ch7xxx_readb(dvo, CH7xxx_PM, &val); ++ ++ if (val & (CH7xxx_PM_DVIL | CH7xxx_PM_DVIP)) ++ return true; ++ else ++ return false; ++} ++ ++static void ch7xxx_dump_regs(struct intel_dvo_device *dvo) ++{ ++ int i; ++ ++ for (i = 0; i < CH7xxx_NUM_REGS; i++) { ++ u8 val; ++ if ((i % 8) == 0) ++ DRM_DEBUG_KMS("\n %02X: ", i); ++ ch7xxx_readb(dvo, i, &val); ++ DRM_DEBUG_KMS("%02X ", val); ++ } ++} ++ ++static void ch7xxx_destroy(struct intel_dvo_device *dvo) ++{ ++ struct ch7xxx_priv *ch7xxx = dvo->dev_priv; ++ ++ if (ch7xxx) { ++ kfree(ch7xxx); ++ dvo->dev_priv = NULL; ++ } ++} ++ ++const struct intel_dvo_dev_ops ch7xxx_ops = { ++ .init = ch7xxx_init, ++ .detect = ch7xxx_detect, ++ .mode_valid = ch7xxx_mode_valid, ++ .mode_set = ch7xxx_mode_set, ++ .dpms = ch7xxx_dpms, ++ .get_hw_state = ch7xxx_get_hw_state, ++ .dump_regs = ch7xxx_dump_regs, ++ .destroy = ch7xxx_destroy, ++}; +diff --git a/drivers/gpu/drm/i915_legacy/dvo_ivch.c b/drivers/gpu/drm/i915_legacy/dvo_ivch.c +new file mode 100644 +index 000000000000..24278cc49090 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/dvo_ivch.c +@@ -0,0 +1,502 @@ ++/* ++ * Copyright © 2006 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER ++ * DEALINGS IN THE SOFTWARE. ++ * ++ * Authors: ++ * Eric Anholt ++ * Thomas Richter ++ * ++ * Minor modifications (Dithering enable): ++ * Thomas Richter ++ * ++ */ ++ ++#include "dvo.h" ++ ++/* ++ * register definitions for the i82807aa. ++ * ++ * Documentation on this chipset can be found in datasheet #29069001 at ++ * intel.com. ++ */ ++ ++/* ++ * VCH Revision & GMBus Base Addr ++ */ ++#define VR00 0x00 ++# define VR00_BASE_ADDRESS_MASK 0x007f ++ ++/* ++ * Functionality Enable ++ */ ++#define VR01 0x01 ++ ++/* ++ * Enable the panel fitter ++ */ ++# define VR01_PANEL_FIT_ENABLE (1 << 3) ++/* ++ * Enables the LCD display. ++ * ++ * This must not be set while VR01_DVO_BYPASS_ENABLE is set. ++ */ ++# define VR01_LCD_ENABLE (1 << 2) ++/* Enables the DVO repeater. */ ++# define VR01_DVO_BYPASS_ENABLE (1 << 1) ++/* Enables the DVO clock */ ++# define VR01_DVO_ENABLE (1 << 0) ++/* Enable dithering for 18bpp panels. Not documented. */ ++# define VR01_DITHER_ENABLE (1 << 4) ++ ++/* ++ * LCD Interface Format ++ */ ++#define VR10 0x10 ++/* Enables LVDS output instead of CMOS */ ++# define VR10_LVDS_ENABLE (1 << 4) ++/* Enables 18-bit LVDS output. */ ++# define VR10_INTERFACE_1X18 (0 << 2) ++/* Enables 24-bit LVDS or CMOS output */ ++# define VR10_INTERFACE_1X24 (1 << 2) ++/* Enables 2x18-bit LVDS or CMOS output. */ ++# define VR10_INTERFACE_2X18 (2 << 2) ++/* Enables 2x24-bit LVDS output */ ++# define VR10_INTERFACE_2X24 (3 << 2) ++/* Mask that defines the depth of the pipeline */ ++# define VR10_INTERFACE_DEPTH_MASK (3 << 2) ++ ++/* ++ * VR20 LCD Horizontal Display Size ++ */ ++#define VR20 0x20 ++ ++/* ++ * LCD Vertical Display Size ++ */ ++#define VR21 0x21 ++ ++/* ++ * Panel power down status ++ */ ++#define VR30 0x30 ++/* Read only bit indicating that the panel is not in a safe poweroff state. */ ++# define VR30_PANEL_ON (1 << 15) ++ ++#define VR40 0x40 ++# define VR40_STALL_ENABLE (1 << 13) ++# define VR40_VERTICAL_INTERP_ENABLE (1 << 12) ++# define VR40_ENHANCED_PANEL_FITTING (1 << 11) ++# define VR40_HORIZONTAL_INTERP_ENABLE (1 << 10) ++# define VR40_AUTO_RATIO_ENABLE (1 << 9) ++# define VR40_CLOCK_GATING_ENABLE (1 << 8) ++ ++/* ++ * Panel Fitting Vertical Ratio ++ * (((image_height - 1) << 16) / ((panel_height - 1))) >> 2 ++ */ ++#define VR41 0x41 ++ ++/* ++ * Panel Fitting Horizontal Ratio ++ * (((image_width - 1) << 16) / ((panel_width - 1))) >> 2 ++ */ ++#define VR42 0x42 ++ ++/* ++ * Horizontal Image Size ++ */ ++#define VR43 0x43 ++ ++/* VR80 GPIO 0 ++ */ ++#define VR80 0x80 ++#define VR81 0x81 ++#define VR82 0x82 ++#define VR83 0x83 ++#define VR84 0x84 ++#define VR85 0x85 ++#define VR86 0x86 ++#define VR87 0x87 ++ ++/* VR88 GPIO 8 ++ */ ++#define VR88 0x88 ++ ++/* Graphics BIOS scratch 0 ++ */ ++#define VR8E 0x8E ++# define VR8E_PANEL_TYPE_MASK (0xf << 0) ++# define VR8E_PANEL_INTERFACE_CMOS (0 << 4) ++# define VR8E_PANEL_INTERFACE_LVDS (1 << 4) ++# define VR8E_FORCE_DEFAULT_PANEL (1 << 5) ++ ++/* Graphics BIOS scratch 1 ++ */ ++#define VR8F 0x8F ++# define VR8F_VCH_PRESENT (1 << 0) ++# define VR8F_DISPLAY_CONN (1 << 1) ++# define VR8F_POWER_MASK (0x3c) ++# define VR8F_POWER_POS (2) ++ ++/* Some Bios implementations do not restore the DVO state upon ++ * resume from standby. Thus, this driver has to handle it ++ * instead. The following list contains all registers that ++ * require saving. ++ */ ++static const u16 backup_addresses[] = { ++ 0x11, 0x12, ++ 0x18, 0x19, 0x1a, 0x1f, ++ 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, ++ 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, ++ 0x8e, 0x8f, ++ 0x10 /* this must come last */ ++}; ++ ++ ++struct ivch_priv { ++ bool quiet; ++ ++ u16 width, height; ++ ++ /* Register backup */ ++ ++ u16 reg_backup[ARRAY_SIZE(backup_addresses)]; ++}; ++ ++ ++static void ivch_dump_regs(struct intel_dvo_device *dvo); ++/* ++ * Reads a register on the ivch. ++ * ++ * Each of the 256 registers are 16 bits long. ++ */ ++static bool ivch_read(struct intel_dvo_device *dvo, int addr, u16 *data) ++{ ++ struct ivch_priv *priv = dvo->dev_priv; ++ struct i2c_adapter *adapter = dvo->i2c_bus; ++ u8 out_buf[1]; ++ u8 in_buf[2]; ++ ++ struct i2c_msg msgs[] = { ++ { ++ .addr = dvo->slave_addr, ++ .flags = I2C_M_RD, ++ .len = 0, ++ }, ++ { ++ .addr = 0, ++ .flags = I2C_M_NOSTART, ++ .len = 1, ++ .buf = out_buf, ++ }, ++ { ++ .addr = dvo->slave_addr, ++ .flags = I2C_M_RD | I2C_M_NOSTART, ++ .len = 2, ++ .buf = in_buf, ++ } ++ }; ++ ++ out_buf[0] = addr; ++ ++ if (i2c_transfer(adapter, msgs, 3) == 3) { ++ *data = (in_buf[1] << 8) | in_buf[0]; ++ return true; ++ } ++ ++ if (!priv->quiet) { ++ DRM_DEBUG_KMS("Unable to read register 0x%02x from " ++ "%s:%02x.\n", ++ addr, adapter->name, dvo->slave_addr); ++ } ++ return false; ++} ++ ++/* Writes a 16-bit register on the ivch */ ++static bool ivch_write(struct intel_dvo_device *dvo, int addr, u16 data) ++{ ++ struct ivch_priv *priv = dvo->dev_priv; ++ struct i2c_adapter *adapter = dvo->i2c_bus; ++ u8 out_buf[3]; ++ struct i2c_msg msg = { ++ .addr = dvo->slave_addr, ++ .flags = 0, ++ .len = 3, ++ .buf = out_buf, ++ }; ++ ++ out_buf[0] = addr; ++ out_buf[1] = data & 0xff; ++ out_buf[2] = data >> 8; ++ ++ if (i2c_transfer(adapter, &msg, 1) == 1) ++ return true; ++ ++ if (!priv->quiet) { ++ DRM_DEBUG_KMS("Unable to write register 0x%02x to %s:%d.\n", ++ addr, adapter->name, dvo->slave_addr); ++ } ++ ++ return false; ++} ++ ++/* Probes the given bus and slave address for an ivch */ ++static bool ivch_init(struct intel_dvo_device *dvo, ++ struct i2c_adapter *adapter) ++{ ++ struct ivch_priv *priv; ++ u16 temp; ++ int i; ++ ++ priv = kzalloc(sizeof(struct ivch_priv), GFP_KERNEL); ++ if (priv == NULL) ++ return false; ++ ++ dvo->i2c_bus = adapter; ++ dvo->dev_priv = priv; ++ priv->quiet = true; ++ ++ if (!ivch_read(dvo, VR00, &temp)) ++ goto out; ++ priv->quiet = false; ++ ++ /* Since the identification bits are probably zeroes, which doesn't seem ++ * very unique, check that the value in the base address field matches ++ * the address it's responding on. ++ */ ++ if ((temp & VR00_BASE_ADDRESS_MASK) != dvo->slave_addr) { ++ DRM_DEBUG_KMS("ivch detect failed due to address mismatch " ++ "(%d vs %d)\n", ++ (temp & VR00_BASE_ADDRESS_MASK), dvo->slave_addr); ++ goto out; ++ } ++ ++ ivch_read(dvo, VR20, &priv->width); ++ ivch_read(dvo, VR21, &priv->height); ++ ++ /* Make a backup of the registers to be able to restore them ++ * upon suspend. ++ */ ++ for (i = 0; i < ARRAY_SIZE(backup_addresses); i++) ++ ivch_read(dvo, backup_addresses[i], priv->reg_backup + i); ++ ++ ivch_dump_regs(dvo); ++ ++ return true; ++ ++out: ++ kfree(priv); ++ return false; ++} ++ ++static enum drm_connector_status ivch_detect(struct intel_dvo_device *dvo) ++{ ++ return connector_status_connected; ++} ++ ++static enum drm_mode_status ivch_mode_valid(struct intel_dvo_device *dvo, ++ struct drm_display_mode *mode) ++{ ++ if (mode->clock > 112000) ++ return MODE_CLOCK_HIGH; ++ ++ return MODE_OK; ++} ++ ++/* Restore the DVO registers after a resume ++ * from RAM. Registers have been saved during ++ * the initialization. ++ */ ++static void ivch_reset(struct intel_dvo_device *dvo) ++{ ++ struct ivch_priv *priv = dvo->dev_priv; ++ int i; ++ ++ DRM_DEBUG_KMS("Resetting the IVCH registers\n"); ++ ++ ivch_write(dvo, VR10, 0x0000); ++ ++ for (i = 0; i < ARRAY_SIZE(backup_addresses); i++) ++ ivch_write(dvo, backup_addresses[i], priv->reg_backup[i]); ++} ++ ++/* Sets the power state of the panel connected to the ivch */ ++static void ivch_dpms(struct intel_dvo_device *dvo, bool enable) ++{ ++ int i; ++ u16 vr01, vr30, backlight; ++ ++ ivch_reset(dvo); ++ ++ /* Set the new power state of the panel. */ ++ if (!ivch_read(dvo, VR01, &vr01)) ++ return; ++ ++ if (enable) ++ backlight = 1; ++ else ++ backlight = 0; ++ ++ ivch_write(dvo, VR80, backlight); ++ ++ if (enable) ++ vr01 |= VR01_LCD_ENABLE | VR01_DVO_ENABLE; ++ else ++ vr01 &= ~(VR01_LCD_ENABLE | VR01_DVO_ENABLE); ++ ++ ivch_write(dvo, VR01, vr01); ++ ++ /* Wait for the panel to make its state transition */ ++ for (i = 0; i < 100; i++) { ++ if (!ivch_read(dvo, VR30, &vr30)) ++ break; ++ ++ if (((vr30 & VR30_PANEL_ON) != 0) == enable) ++ break; ++ udelay(1000); ++ } ++ /* wait some more; vch may fail to resync sometimes without this */ ++ udelay(16 * 1000); ++} ++ ++static bool ivch_get_hw_state(struct intel_dvo_device *dvo) ++{ ++ u16 vr01; ++ ++ ivch_reset(dvo); ++ ++ /* Set the new power state of the panel. */ ++ if (!ivch_read(dvo, VR01, &vr01)) ++ return false; ++ ++ if (vr01 & VR01_LCD_ENABLE) ++ return true; ++ else ++ return false; ++} ++ ++static void ivch_mode_set(struct intel_dvo_device *dvo, ++ const struct drm_display_mode *mode, ++ const struct drm_display_mode *adjusted_mode) ++{ ++ struct ivch_priv *priv = dvo->dev_priv; ++ u16 vr40 = 0; ++ u16 vr01 = 0; ++ u16 vr10; ++ ++ ivch_reset(dvo); ++ ++ vr10 = priv->reg_backup[ARRAY_SIZE(backup_addresses) - 1]; ++ ++ /* Enable dithering for 18 bpp pipelines */ ++ vr10 &= VR10_INTERFACE_DEPTH_MASK; ++ if (vr10 == VR10_INTERFACE_2X18 || vr10 == VR10_INTERFACE_1X18) ++ vr01 = VR01_DITHER_ENABLE; ++ ++ vr40 = (VR40_STALL_ENABLE | VR40_VERTICAL_INTERP_ENABLE | ++ VR40_HORIZONTAL_INTERP_ENABLE); ++ ++ if (mode->hdisplay != adjusted_mode->crtc_hdisplay || ++ mode->vdisplay != adjusted_mode->crtc_vdisplay) { ++ u16 x_ratio, y_ratio; ++ ++ vr01 |= VR01_PANEL_FIT_ENABLE; ++ vr40 |= VR40_CLOCK_GATING_ENABLE; ++ x_ratio = (((mode->hdisplay - 1) << 16) / ++ (adjusted_mode->crtc_hdisplay - 1)) >> 2; ++ y_ratio = (((mode->vdisplay - 1) << 16) / ++ (adjusted_mode->crtc_vdisplay - 1)) >> 2; ++ ivch_write(dvo, VR42, x_ratio); ++ ivch_write(dvo, VR41, y_ratio); ++ } else { ++ vr01 &= ~VR01_PANEL_FIT_ENABLE; ++ vr40 &= ~VR40_CLOCK_GATING_ENABLE; ++ } ++ vr40 &= ~VR40_AUTO_RATIO_ENABLE; ++ ++ ivch_write(dvo, VR01, vr01); ++ ivch_write(dvo, VR40, vr40); ++} ++ ++static void ivch_dump_regs(struct intel_dvo_device *dvo) ++{ ++ u16 val; ++ ++ ivch_read(dvo, VR00, &val); ++ DRM_DEBUG_KMS("VR00: 0x%04x\n", val); ++ ivch_read(dvo, VR01, &val); ++ DRM_DEBUG_KMS("VR01: 0x%04x\n", val); ++ ivch_read(dvo, VR10, &val); ++ DRM_DEBUG_KMS("VR10: 0x%04x\n", val); ++ ivch_read(dvo, VR30, &val); ++ DRM_DEBUG_KMS("VR30: 0x%04x\n", val); ++ ivch_read(dvo, VR40, &val); ++ DRM_DEBUG_KMS("VR40: 0x%04x\n", val); ++ ++ /* GPIO registers */ ++ ivch_read(dvo, VR80, &val); ++ DRM_DEBUG_KMS("VR80: 0x%04x\n", val); ++ ivch_read(dvo, VR81, &val); ++ DRM_DEBUG_KMS("VR81: 0x%04x\n", val); ++ ivch_read(dvo, VR82, &val); ++ DRM_DEBUG_KMS("VR82: 0x%04x\n", val); ++ ivch_read(dvo, VR83, &val); ++ DRM_DEBUG_KMS("VR83: 0x%04x\n", val); ++ ivch_read(dvo, VR84, &val); ++ DRM_DEBUG_KMS("VR84: 0x%04x\n", val); ++ ivch_read(dvo, VR85, &val); ++ DRM_DEBUG_KMS("VR85: 0x%04x\n", val); ++ ivch_read(dvo, VR86, &val); ++ DRM_DEBUG_KMS("VR86: 0x%04x\n", val); ++ ivch_read(dvo, VR87, &val); ++ DRM_DEBUG_KMS("VR87: 0x%04x\n", val); ++ ivch_read(dvo, VR88, &val); ++ DRM_DEBUG_KMS("VR88: 0x%04x\n", val); ++ ++ /* Scratch register 0 - AIM Panel type */ ++ ivch_read(dvo, VR8E, &val); ++ DRM_DEBUG_KMS("VR8E: 0x%04x\n", val); ++ ++ /* Scratch register 1 - Status register */ ++ ivch_read(dvo, VR8F, &val); ++ DRM_DEBUG_KMS("VR8F: 0x%04x\n", val); ++} ++ ++static void ivch_destroy(struct intel_dvo_device *dvo) ++{ ++ struct ivch_priv *priv = dvo->dev_priv; ++ ++ if (priv) { ++ kfree(priv); ++ dvo->dev_priv = NULL; ++ } ++} ++ ++const struct intel_dvo_dev_ops ivch_ops = { ++ .init = ivch_init, ++ .dpms = ivch_dpms, ++ .get_hw_state = ivch_get_hw_state, ++ .mode_valid = ivch_mode_valid, ++ .mode_set = ivch_mode_set, ++ .detect = ivch_detect, ++ .dump_regs = ivch_dump_regs, ++ .destroy = ivch_destroy, ++}; +diff --git a/drivers/gpu/drm/i915_legacy/dvo_ns2501.c b/drivers/gpu/drm/i915_legacy/dvo_ns2501.c +new file mode 100644 +index 000000000000..c584e01dc8dc +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/dvo_ns2501.c +@@ -0,0 +1,709 @@ ++/* ++ * ++ * Copyright (c) 2012 Gilles Dartiguelongue, Thomas Richter ++ * ++ * All Rights Reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the ++ * "Software"), to deal in the Software without restriction, including ++ * without limitation the rights to use, copy, modify, merge, publish, ++ * distribute, sub license, and/or sell copies of the Software, and to ++ * permit persons to whom the Software is furnished to do so, subject to ++ * the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the ++ * next paragraph) shall be included in all copies or substantial portions ++ * of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS ++ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF ++ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. ++ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ++ * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, ++ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE ++ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ++ * ++ */ ++ ++#include "dvo.h" ++#include "i915_reg.h" ++#include "i915_drv.h" ++ ++#define NS2501_VID 0x1305 ++#define NS2501_DID 0x6726 ++ ++#define NS2501_VID_LO 0x00 ++#define NS2501_VID_HI 0x01 ++#define NS2501_DID_LO 0x02 ++#define NS2501_DID_HI 0x03 ++#define NS2501_REV 0x04 ++#define NS2501_RSVD 0x05 ++#define NS2501_FREQ_LO 0x06 ++#define NS2501_FREQ_HI 0x07 ++ ++#define NS2501_REG8 0x08 ++#define NS2501_8_VEN (1<<5) ++#define NS2501_8_HEN (1<<4) ++#define NS2501_8_DSEL (1<<3) ++#define NS2501_8_BPAS (1<<2) ++#define NS2501_8_RSVD (1<<1) ++#define NS2501_8_PD (1<<0) ++ ++#define NS2501_REG9 0x09 ++#define NS2501_9_VLOW (1<<7) ++#define NS2501_9_MSEL_MASK (0x7<<4) ++#define NS2501_9_TSEL (1<<3) ++#define NS2501_9_RSEN (1<<2) ++#define NS2501_9_RSVD (1<<1) ++#define NS2501_9_MDI (1<<0) ++ ++#define NS2501_REGC 0x0c ++ ++/* ++ * The following registers are not part of the official datasheet ++ * and are the result of reverse engineering. ++ */ ++ ++/* ++ * Register c0 controls how the DVO synchronizes with ++ * its input. ++ */ ++#define NS2501_REGC0 0xc0 ++#define NS2501_C0_ENABLE (1<<0) /* enable the DVO sync in general */ ++#define NS2501_C0_HSYNC (1<<1) /* synchronize horizontal with input */ ++#define NS2501_C0_VSYNC (1<<2) /* synchronize vertical with input */ ++#define NS2501_C0_RESET (1<<7) /* reset the synchronization flip/flops */ ++ ++/* ++ * Register 41 is somehow related to the sync register and sync ++ * configuration. It should be 0x32 whenever regC0 is 0x05 (hsync off) ++ * and 0x00 otherwise. ++ */ ++#define NS2501_REG41 0x41 ++ ++/* ++ * this register controls the dithering of the DVO ++ * One bit enables it, the other define the dithering depth. ++ * The higher the value, the lower the dithering depth. ++ */ ++#define NS2501_F9_REG 0xf9 ++#define NS2501_F9_ENABLE (1<<0) /* if set, dithering is enabled */ ++#define NS2501_F9_DITHER_MASK (0x7f<<1) /* controls the dither depth */ ++#define NS2501_F9_DITHER_SHIFT 1 /* shifts the dither mask */ ++ ++/* ++ * PLL configuration register. This is a pair of registers, ++ * one single byte register at 1B, and a pair at 1C,1D. ++ * These registers are counters/dividers. ++ */ ++#define NS2501_REG1B 0x1b /* one byte PLL control register */ ++#define NS2501_REG1C 0x1c /* low-part of the second register */ ++#define NS2501_REG1D 0x1d /* high-part of the second register */ ++ ++/* ++ * Scaler control registers. Horizontal at b8,b9, ++ * vertical at 10,11. The scale factor is computed as ++ * 2^16/control-value. The low-byte comes first. ++ */ ++#define NS2501_REG10 0x10 /* low-byte vertical scaler */ ++#define NS2501_REG11 0x11 /* high-byte vertical scaler */ ++#define NS2501_REGB8 0xb8 /* low-byte horizontal scaler */ ++#define NS2501_REGB9 0xb9 /* high-byte horizontal scaler */ ++ ++/* ++ * Display window definition. This consists of four registers ++ * per dimension. One register pair defines the start of the ++ * display, one the end. ++ * As far as I understand, this defines the window within which ++ * the scaler samples the input. ++ */ ++#define NS2501_REGC1 0xc1 /* low-byte horizontal display start */ ++#define NS2501_REGC2 0xc2 /* high-byte horizontal display start */ ++#define NS2501_REGC3 0xc3 /* low-byte horizontal display stop */ ++#define NS2501_REGC4 0xc4 /* high-byte horizontal display stop */ ++#define NS2501_REGC5 0xc5 /* low-byte vertical display start */ ++#define NS2501_REGC6 0xc6 /* high-byte vertical display start */ ++#define NS2501_REGC7 0xc7 /* low-byte vertical display stop */ ++#define NS2501_REGC8 0xc8 /* high-byte vertical display stop */ ++ ++/* ++ * The following register pair seems to define the start of ++ * the vertical sync. If automatic syncing is enabled, and the ++ * register value defines a sync pulse that is later than the ++ * incoming sync, then the register value is ignored and the ++ * external hsync triggers the synchronization. ++ */ ++#define NS2501_REG80 0x80 /* low-byte vsync-start */ ++#define NS2501_REG81 0x81 /* high-byte vsync-start */ ++ ++/* ++ * The following register pair seems to define the total number ++ * of lines created at the output side of the scaler. ++ * This is again a low-high register pair. ++ */ ++#define NS2501_REG82 0x82 /* output display height, low byte */ ++#define NS2501_REG83 0x83 /* output display height, high byte */ ++ ++/* ++ * The following registers define the end of the front-porch ++ * in horizontal and vertical position and hence allow to shift ++ * the image left/right or up/down. ++ */ ++#define NS2501_REG98 0x98 /* horizontal start of display + 256, low */ ++#define NS2501_REG99 0x99 /* horizontal start of display + 256, high */ ++#define NS2501_REG8E 0x8e /* vertical start of the display, low byte */ ++#define NS2501_REG8F 0x8f /* vertical start of the display, high byte */ ++ ++/* ++ * The following register pair control the function of the ++ * backlight and the DVO output. To enable the corresponding ++ * function, the corresponding bit must be set in both registers. ++ */ ++#define NS2501_REG34 0x34 /* DVO enable functions, first register */ ++#define NS2501_REG35 0x35 /* DVO enable functions, second register */ ++#define NS2501_34_ENABLE_OUTPUT (1<<0) /* enable DVO output */ ++#define NS2501_34_ENABLE_BACKLIGHT (1<<1) /* enable backlight */ ++ ++/* ++ * Registers 9C and 9D define the vertical output offset ++ * of the visible region. ++ */ ++#define NS2501_REG9C 0x9c ++#define NS2501_REG9D 0x9d ++ ++/* ++ * The register 9F defines the dithering. This requires the ++ * scaler to be ON. Bit 0 enables dithering, the remaining ++ * bits control the depth of the dither. The higher the value, ++ * the LOWER the dithering amplitude. A good value seems to be ++ * 15 (total register value). ++ */ ++#define NS2501_REGF9 0xf9 ++#define NS2501_F9_ENABLE_DITHER (1<<0) /* enable dithering */ ++#define NS2501_F9_DITHER_MASK (0x7f<<1) /* dither masking */ ++#define NS2501_F9_DITHER_SHIFT 1 /* upshift of the dither mask */ ++ ++enum { ++ MODE_640x480, ++ MODE_800x600, ++ MODE_1024x768, ++}; ++ ++struct ns2501_reg { ++ u8 offset; ++ u8 value; ++}; ++ ++/* ++ * The following structure keeps the complete configuration of ++ * the DVO, given a specific output configuration. ++ * This is pretty much guess-work from reverse-engineering, so ++ * read all this with a grain of salt. ++ */ ++struct ns2501_configuration { ++ u8 sync; /* configuration of the C0 register */ ++ u8 conf; /* configuration register 8 */ ++ u8 syncb; /* configuration register 41 */ ++ u8 dither; /* configuration of the dithering */ ++ u8 pll_a; /* PLL configuration, register A, 1B */ ++ u16 pll_b; /* PLL configuration, register B, 1C/1D */ ++ u16 hstart; /* horizontal start, registers C1/C2 */ ++ u16 hstop; /* horizontal total, registers C3/C4 */ ++ u16 vstart; /* vertical start, registers C5/C6 */ ++ u16 vstop; /* vertical total, registers C7/C8 */ ++ u16 vsync; /* manual vertical sync start, 80/81 */ ++ u16 vtotal; /* number of lines generated, 82/83 */ ++ u16 hpos; /* horizontal position + 256, 98/99 */ ++ u16 vpos; /* vertical position, 8e/8f */ ++ u16 voffs; /* vertical output offset, 9c/9d */ ++ u16 hscale; /* horizontal scaling factor, b8/b9 */ ++ u16 vscale; /* vertical scaling factor, 10/11 */ ++}; ++ ++/* ++ * DVO configuration values, partially based on what the BIOS ++ * of the Fujitsu Lifebook S6010 writes into registers, ++ * partially found by manual tweaking. These configurations assume ++ * a 1024x768 panel. ++ */ ++static const struct ns2501_configuration ns2501_modes[] = { ++ [MODE_640x480] = { ++ .sync = NS2501_C0_ENABLE | NS2501_C0_VSYNC, ++ .conf = NS2501_8_VEN | NS2501_8_HEN | NS2501_8_PD, ++ .syncb = 0x32, ++ .dither = 0x0f, ++ .pll_a = 17, ++ .pll_b = 852, ++ .hstart = 144, ++ .hstop = 783, ++ .vstart = 22, ++ .vstop = 514, ++ .vsync = 2047, /* actually, ignored with this config */ ++ .vtotal = 1341, ++ .hpos = 0, ++ .vpos = 16, ++ .voffs = 36, ++ .hscale = 40960, ++ .vscale = 40960 ++ }, ++ [MODE_800x600] = { ++ .sync = NS2501_C0_ENABLE | ++ NS2501_C0_HSYNC | NS2501_C0_VSYNC, ++ .conf = NS2501_8_VEN | NS2501_8_HEN | NS2501_8_PD, ++ .syncb = 0x00, ++ .dither = 0x0f, ++ .pll_a = 25, ++ .pll_b = 612, ++ .hstart = 215, ++ .hstop = 1016, ++ .vstart = 26, ++ .vstop = 627, ++ .vsync = 807, ++ .vtotal = 1341, ++ .hpos = 0, ++ .vpos = 4, ++ .voffs = 35, ++ .hscale = 51248, ++ .vscale = 51232 ++ }, ++ [MODE_1024x768] = { ++ .sync = NS2501_C0_ENABLE | NS2501_C0_VSYNC, ++ .conf = NS2501_8_VEN | NS2501_8_HEN | NS2501_8_PD, ++ .syncb = 0x32, ++ .dither = 0x0f, ++ .pll_a = 11, ++ .pll_b = 1350, ++ .hstart = 276, ++ .hstop = 1299, ++ .vstart = 15, ++ .vstop = 1056, ++ .vsync = 2047, ++ .vtotal = 1341, ++ .hpos = 0, ++ .vpos = 7, ++ .voffs = 27, ++ .hscale = 65535, ++ .vscale = 65535 ++ } ++}; ++ ++/* ++ * Other configuration values left by the BIOS of the ++ * Fujitsu S6010 in the DVO control registers. Their ++ * value does not depend on the BIOS and their meaning ++ * is unknown. ++ */ ++ ++static const struct ns2501_reg mode_agnostic_values[] = { ++ /* 08 is mode specific */ ++ [0] = { .offset = 0x0a, .value = 0x81, }, ++ /* 10,11 are part of the mode specific configuration */ ++ [1] = { .offset = 0x12, .value = 0x02, }, ++ [2] = { .offset = 0x18, .value = 0x07, }, ++ [3] = { .offset = 0x19, .value = 0x00, }, ++ [4] = { .offset = 0x1a, .value = 0x00, }, /* PLL?, ignored */ ++ /* 1b,1c,1d are part of the mode specific configuration */ ++ [5] = { .offset = 0x1e, .value = 0x02, }, ++ [6] = { .offset = 0x1f, .value = 0x40, }, ++ [7] = { .offset = 0x20, .value = 0x00, }, ++ [8] = { .offset = 0x21, .value = 0x00, }, ++ [9] = { .offset = 0x22, .value = 0x00, }, ++ [10] = { .offset = 0x23, .value = 0x00, }, ++ [11] = { .offset = 0x24, .value = 0x00, }, ++ [12] = { .offset = 0x25, .value = 0x00, }, ++ [13] = { .offset = 0x26, .value = 0x00, }, ++ [14] = { .offset = 0x27, .value = 0x00, }, ++ [15] = { .offset = 0x7e, .value = 0x18, }, ++ /* 80-84 are part of the mode-specific configuration */ ++ [16] = { .offset = 0x84, .value = 0x00, }, ++ [17] = { .offset = 0x85, .value = 0x00, }, ++ [18] = { .offset = 0x86, .value = 0x00, }, ++ [19] = { .offset = 0x87, .value = 0x00, }, ++ [20] = { .offset = 0x88, .value = 0x00, }, ++ [21] = { .offset = 0x89, .value = 0x00, }, ++ [22] = { .offset = 0x8a, .value = 0x00, }, ++ [23] = { .offset = 0x8b, .value = 0x00, }, ++ [24] = { .offset = 0x8c, .value = 0x10, }, ++ [25] = { .offset = 0x8d, .value = 0x02, }, ++ /* 8e,8f are part of the mode-specific configuration */ ++ [26] = { .offset = 0x90, .value = 0xff, }, ++ [27] = { .offset = 0x91, .value = 0x07, }, ++ [28] = { .offset = 0x92, .value = 0xa0, }, ++ [29] = { .offset = 0x93, .value = 0x02, }, ++ [30] = { .offset = 0x94, .value = 0x00, }, ++ [31] = { .offset = 0x95, .value = 0x00, }, ++ [32] = { .offset = 0x96, .value = 0x05, }, ++ [33] = { .offset = 0x97, .value = 0x00, }, ++ /* 98,99 are part of the mode-specific configuration */ ++ [34] = { .offset = 0x9a, .value = 0x88, }, ++ [35] = { .offset = 0x9b, .value = 0x00, }, ++ /* 9c,9d are part of the mode-specific configuration */ ++ [36] = { .offset = 0x9e, .value = 0x25, }, ++ [37] = { .offset = 0x9f, .value = 0x03, }, ++ [38] = { .offset = 0xa0, .value = 0x28, }, ++ [39] = { .offset = 0xa1, .value = 0x01, }, ++ [40] = { .offset = 0xa2, .value = 0x28, }, ++ [41] = { .offset = 0xa3, .value = 0x05, }, ++ /* register 0xa4 is mode specific, but 0x80..0x84 works always */ ++ [42] = { .offset = 0xa4, .value = 0x84, }, ++ [43] = { .offset = 0xa5, .value = 0x00, }, ++ [44] = { .offset = 0xa6, .value = 0x00, }, ++ [45] = { .offset = 0xa7, .value = 0x00, }, ++ [46] = { .offset = 0xa8, .value = 0x00, }, ++ /* 0xa9 to 0xab are mode specific, but have no visible effect */ ++ [47] = { .offset = 0xa9, .value = 0x04, }, ++ [48] = { .offset = 0xaa, .value = 0x70, }, ++ [49] = { .offset = 0xab, .value = 0x4f, }, ++ [50] = { .offset = 0xac, .value = 0x00, }, ++ [51] = { .offset = 0xad, .value = 0x00, }, ++ [52] = { .offset = 0xb6, .value = 0x09, }, ++ [53] = { .offset = 0xb7, .value = 0x03, }, ++ /* b8,b9 are part of the mode-specific configuration */ ++ [54] = { .offset = 0xba, .value = 0x00, }, ++ [55] = { .offset = 0xbb, .value = 0x20, }, ++ [56] = { .offset = 0xf3, .value = 0x90, }, ++ [57] = { .offset = 0xf4, .value = 0x00, }, ++ [58] = { .offset = 0xf7, .value = 0x88, }, ++ /* f8 is mode specific, but the value does not matter */ ++ [59] = { .offset = 0xf8, .value = 0x0a, }, ++ [60] = { .offset = 0xf9, .value = 0x00, } ++}; ++ ++static const struct ns2501_reg regs_init[] = { ++ [0] = { .offset = 0x35, .value = 0xff, }, ++ [1] = { .offset = 0x34, .value = 0x00, }, ++ [2] = { .offset = 0x08, .value = 0x30, }, ++}; ++ ++struct ns2501_priv { ++ bool quiet; ++ const struct ns2501_configuration *conf; ++}; ++ ++#define NSPTR(d) ((NS2501Ptr)(d->DriverPrivate.ptr)) ++ ++/* ++** Read a register from the ns2501. ++** Returns true if successful, false otherwise. ++** If it returns false, it might be wise to enable the ++** DVO with the above function. ++*/ ++static bool ns2501_readb(struct intel_dvo_device *dvo, int addr, u8 *ch) ++{ ++ struct ns2501_priv *ns = dvo->dev_priv; ++ struct i2c_adapter *adapter = dvo->i2c_bus; ++ u8 out_buf[2]; ++ u8 in_buf[2]; ++ ++ struct i2c_msg msgs[] = { ++ { ++ .addr = dvo->slave_addr, ++ .flags = 0, ++ .len = 1, ++ .buf = out_buf, ++ }, ++ { ++ .addr = dvo->slave_addr, ++ .flags = I2C_M_RD, ++ .len = 1, ++ .buf = in_buf, ++ } ++ }; ++ ++ out_buf[0] = addr; ++ out_buf[1] = 0; ++ ++ if (i2c_transfer(adapter, msgs, 2) == 2) { ++ *ch = in_buf[0]; ++ return true; ++ } ++ ++ if (!ns->quiet) { ++ DRM_DEBUG_KMS ++ ("Unable to read register 0x%02x from %s:0x%02x.\n", addr, ++ adapter->name, dvo->slave_addr); ++ } ++ ++ return false; ++} ++ ++/* ++** Write a register to the ns2501. ++** Returns true if successful, false otherwise. ++** If it returns false, it might be wise to enable the ++** DVO with the above function. ++*/ ++static bool ns2501_writeb(struct intel_dvo_device *dvo, int addr, u8 ch) ++{ ++ struct ns2501_priv *ns = dvo->dev_priv; ++ struct i2c_adapter *adapter = dvo->i2c_bus; ++ u8 out_buf[2]; ++ ++ struct i2c_msg msg = { ++ .addr = dvo->slave_addr, ++ .flags = 0, ++ .len = 2, ++ .buf = out_buf, ++ }; ++ ++ out_buf[0] = addr; ++ out_buf[1] = ch; ++ ++ if (i2c_transfer(adapter, &msg, 1) == 1) { ++ return true; ++ } ++ ++ if (!ns->quiet) { ++ DRM_DEBUG_KMS("Unable to write register 0x%02x to %s:%d\n", ++ addr, adapter->name, dvo->slave_addr); ++ } ++ ++ return false; ++} ++ ++/* National Semiconductor 2501 driver for chip on i2c bus ++ * scan for the chip on the bus. ++ * Hope the VBIOS initialized the PLL correctly so we can ++ * talk to it. If not, it will not be seen and not detected. ++ * Bummer! ++ */ ++static bool ns2501_init(struct intel_dvo_device *dvo, ++ struct i2c_adapter *adapter) ++{ ++ /* this will detect the NS2501 chip on the specified i2c bus */ ++ struct ns2501_priv *ns; ++ unsigned char ch; ++ ++ ns = kzalloc(sizeof(struct ns2501_priv), GFP_KERNEL); ++ if (ns == NULL) ++ return false; ++ ++ dvo->i2c_bus = adapter; ++ dvo->dev_priv = ns; ++ ns->quiet = true; ++ ++ if (!ns2501_readb(dvo, NS2501_VID_LO, &ch)) ++ goto out; ++ ++ if (ch != (NS2501_VID & 0xff)) { ++ DRM_DEBUG_KMS("ns2501 not detected got %d: from %s Slave %d.\n", ++ ch, adapter->name, dvo->slave_addr); ++ goto out; ++ } ++ ++ if (!ns2501_readb(dvo, NS2501_DID_LO, &ch)) ++ goto out; ++ ++ if (ch != (NS2501_DID & 0xff)) { ++ DRM_DEBUG_KMS("ns2501 not detected got %d: from %s Slave %d.\n", ++ ch, adapter->name, dvo->slave_addr); ++ goto out; ++ } ++ ns->quiet = false; ++ ++ DRM_DEBUG_KMS("init ns2501 dvo controller successfully!\n"); ++ ++ return true; ++ ++out: ++ kfree(ns); ++ return false; ++} ++ ++static enum drm_connector_status ns2501_detect(struct intel_dvo_device *dvo) ++{ ++ /* ++ * This is a Laptop display, it doesn't have hotplugging. ++ * Even if not, the detection bit of the 2501 is unreliable as ++ * it only works for some display types. ++ * It is even more unreliable as the PLL must be active for ++ * allowing reading from the chiop. ++ */ ++ return connector_status_connected; ++} ++ ++static enum drm_mode_status ns2501_mode_valid(struct intel_dvo_device *dvo, ++ struct drm_display_mode *mode) ++{ ++ DRM_DEBUG_KMS ++ ("is mode valid (hdisplay=%d,htotal=%d,vdisplay=%d,vtotal=%d)\n", ++ mode->hdisplay, mode->htotal, mode->vdisplay, mode->vtotal); ++ ++ /* ++ * Currently, these are all the modes I have data from. ++ * More might exist. Unclear how to find the native resolution ++ * of the panel in here so we could always accept it ++ * by disabling the scaler. ++ */ ++ if ((mode->hdisplay == 640 && mode->vdisplay == 480 && mode->clock == 25175) || ++ (mode->hdisplay == 800 && mode->vdisplay == 600 && mode->clock == 40000) || ++ (mode->hdisplay == 1024 && mode->vdisplay == 768 && mode->clock == 65000)) { ++ return MODE_OK; ++ } else { ++ return MODE_ONE_SIZE; /* Is this a reasonable error? */ ++ } ++} ++ ++static void ns2501_mode_set(struct intel_dvo_device *dvo, ++ const struct drm_display_mode *mode, ++ const struct drm_display_mode *adjusted_mode) ++{ ++ const struct ns2501_configuration *conf; ++ struct ns2501_priv *ns = (struct ns2501_priv *)(dvo->dev_priv); ++ int mode_idx, i; ++ ++ DRM_DEBUG_KMS ++ ("set mode (hdisplay=%d,htotal=%d,vdisplay=%d,vtotal=%d).\n", ++ mode->hdisplay, mode->htotal, mode->vdisplay, mode->vtotal); ++ ++ DRM_DEBUG_KMS("Detailed requested mode settings are:\n" ++ "clock : %d kHz\n" ++ "hdisplay : %d\n" ++ "hblank start : %d\n" ++ "hblank end : %d\n" ++ "hsync start : %d\n" ++ "hsync end : %d\n" ++ "htotal : %d\n" ++ "hskew : %d\n" ++ "vdisplay : %d\n" ++ "vblank start : %d\n" ++ "hblank end : %d\n" ++ "vsync start : %d\n" ++ "vsync end : %d\n" ++ "vtotal : %d\n", ++ adjusted_mode->crtc_clock, ++ adjusted_mode->crtc_hdisplay, ++ adjusted_mode->crtc_hblank_start, ++ adjusted_mode->crtc_hblank_end, ++ adjusted_mode->crtc_hsync_start, ++ adjusted_mode->crtc_hsync_end, ++ adjusted_mode->crtc_htotal, ++ adjusted_mode->crtc_hskew, ++ adjusted_mode->crtc_vdisplay, ++ adjusted_mode->crtc_vblank_start, ++ adjusted_mode->crtc_vblank_end, ++ adjusted_mode->crtc_vsync_start, ++ adjusted_mode->crtc_vsync_end, ++ adjusted_mode->crtc_vtotal); ++ ++ if (mode->hdisplay == 640 && mode->vdisplay == 480) ++ mode_idx = MODE_640x480; ++ else if (mode->hdisplay == 800 && mode->vdisplay == 600) ++ mode_idx = MODE_800x600; ++ else if (mode->hdisplay == 1024 && mode->vdisplay == 768) ++ mode_idx = MODE_1024x768; ++ else ++ return; ++ ++ /* Hopefully doing it every time won't hurt... */ ++ for (i = 0; i < ARRAY_SIZE(regs_init); i++) ++ ns2501_writeb(dvo, regs_init[i].offset, regs_init[i].value); ++ ++ /* Write the mode-agnostic values */ ++ for (i = 0; i < ARRAY_SIZE(mode_agnostic_values); i++) ++ ns2501_writeb(dvo, mode_agnostic_values[i].offset, ++ mode_agnostic_values[i].value); ++ ++ /* Write now the mode-specific configuration */ ++ conf = ns2501_modes + mode_idx; ++ ns->conf = conf; ++ ++ ns2501_writeb(dvo, NS2501_REG8, conf->conf); ++ ns2501_writeb(dvo, NS2501_REG1B, conf->pll_a); ++ ns2501_writeb(dvo, NS2501_REG1C, conf->pll_b & 0xff); ++ ns2501_writeb(dvo, NS2501_REG1D, conf->pll_b >> 8); ++ ns2501_writeb(dvo, NS2501_REGC1, conf->hstart & 0xff); ++ ns2501_writeb(dvo, NS2501_REGC2, conf->hstart >> 8); ++ ns2501_writeb(dvo, NS2501_REGC3, conf->hstop & 0xff); ++ ns2501_writeb(dvo, NS2501_REGC4, conf->hstop >> 8); ++ ns2501_writeb(dvo, NS2501_REGC5, conf->vstart & 0xff); ++ ns2501_writeb(dvo, NS2501_REGC6, conf->vstart >> 8); ++ ns2501_writeb(dvo, NS2501_REGC7, conf->vstop & 0xff); ++ ns2501_writeb(dvo, NS2501_REGC8, conf->vstop >> 8); ++ ns2501_writeb(dvo, NS2501_REG80, conf->vsync & 0xff); ++ ns2501_writeb(dvo, NS2501_REG81, conf->vsync >> 8); ++ ns2501_writeb(dvo, NS2501_REG82, conf->vtotal & 0xff); ++ ns2501_writeb(dvo, NS2501_REG83, conf->vtotal >> 8); ++ ns2501_writeb(dvo, NS2501_REG98, conf->hpos & 0xff); ++ ns2501_writeb(dvo, NS2501_REG99, conf->hpos >> 8); ++ ns2501_writeb(dvo, NS2501_REG8E, conf->vpos & 0xff); ++ ns2501_writeb(dvo, NS2501_REG8F, conf->vpos >> 8); ++ ns2501_writeb(dvo, NS2501_REG9C, conf->voffs & 0xff); ++ ns2501_writeb(dvo, NS2501_REG9D, conf->voffs >> 8); ++ ns2501_writeb(dvo, NS2501_REGB8, conf->hscale & 0xff); ++ ns2501_writeb(dvo, NS2501_REGB9, conf->hscale >> 8); ++ ns2501_writeb(dvo, NS2501_REG10, conf->vscale & 0xff); ++ ns2501_writeb(dvo, NS2501_REG11, conf->vscale >> 8); ++ ns2501_writeb(dvo, NS2501_REGF9, conf->dither); ++ ns2501_writeb(dvo, NS2501_REG41, conf->syncb); ++ ns2501_writeb(dvo, NS2501_REGC0, conf->sync); ++} ++ ++/* set the NS2501 power state */ ++static bool ns2501_get_hw_state(struct intel_dvo_device *dvo) ++{ ++ unsigned char ch; ++ ++ if (!ns2501_readb(dvo, NS2501_REG8, &ch)) ++ return false; ++ ++ return ch & NS2501_8_PD; ++} ++ ++/* set the NS2501 power state */ ++static void ns2501_dpms(struct intel_dvo_device *dvo, bool enable) ++{ ++ struct ns2501_priv *ns = (struct ns2501_priv *)(dvo->dev_priv); ++ ++ DRM_DEBUG_KMS("Trying set the dpms of the DVO to %i\n", enable); ++ ++ if (enable) { ++ ns2501_writeb(dvo, NS2501_REGC0, ns->conf->sync | 0x08); ++ ++ ns2501_writeb(dvo, NS2501_REG41, ns->conf->syncb); ++ ++ ns2501_writeb(dvo, NS2501_REG34, NS2501_34_ENABLE_OUTPUT); ++ msleep(15); ++ ++ ns2501_writeb(dvo, NS2501_REG8, ++ ns->conf->conf | NS2501_8_BPAS); ++ if (!(ns->conf->conf & NS2501_8_BPAS)) ++ ns2501_writeb(dvo, NS2501_REG8, ns->conf->conf); ++ msleep(200); ++ ++ ns2501_writeb(dvo, NS2501_REG34, ++ NS2501_34_ENABLE_OUTPUT | NS2501_34_ENABLE_BACKLIGHT); ++ ++ ns2501_writeb(dvo, NS2501_REGC0, ns->conf->sync); ++ } else { ++ ns2501_writeb(dvo, NS2501_REG34, NS2501_34_ENABLE_OUTPUT); ++ msleep(200); ++ ++ ns2501_writeb(dvo, NS2501_REG8, NS2501_8_VEN | NS2501_8_HEN | ++ NS2501_8_BPAS); ++ msleep(15); ++ ++ ns2501_writeb(dvo, NS2501_REG34, 0x00); ++ } ++} ++ ++static void ns2501_destroy(struct intel_dvo_device *dvo) ++{ ++ struct ns2501_priv *ns = dvo->dev_priv; ++ ++ if (ns) { ++ kfree(ns); ++ dvo->dev_priv = NULL; ++ } ++} ++ ++const struct intel_dvo_dev_ops ns2501_ops = { ++ .init = ns2501_init, ++ .detect = ns2501_detect, ++ .mode_valid = ns2501_mode_valid, ++ .mode_set = ns2501_mode_set, ++ .dpms = ns2501_dpms, ++ .get_hw_state = ns2501_get_hw_state, ++ .destroy = ns2501_destroy, ++}; +diff --git a/drivers/gpu/drm/i915_legacy/dvo_sil164.c b/drivers/gpu/drm/i915_legacy/dvo_sil164.c +new file mode 100644 +index 000000000000..4ae5d8fd9ff0 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/dvo_sil164.c +@@ -0,0 +1,279 @@ ++/************************************************************************** ++ ++Copyright © 2006 Dave Airlie ++ ++All Rights Reserved. ++ ++Permission is hereby granted, free of charge, to any person obtaining a ++copy of this software and associated documentation files (the ++"Software"), to deal in the Software without restriction, including ++without limitation the rights to use, copy, modify, merge, publish, ++distribute, sub license, and/or sell copies of the Software, and to ++permit persons to whom the Software is furnished to do so, subject to ++the following conditions: ++ ++The above copyright notice and this permission notice (including the ++next paragraph) shall be included in all copies or substantial portions ++of the Software. ++ ++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS ++OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF ++MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. ++IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ++ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, ++TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE ++SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ++ ++**************************************************************************/ ++ ++#include "dvo.h" ++ ++#define SIL164_VID 0x0001 ++#define SIL164_DID 0x0006 ++ ++#define SIL164_VID_LO 0x00 ++#define SIL164_VID_HI 0x01 ++#define SIL164_DID_LO 0x02 ++#define SIL164_DID_HI 0x03 ++#define SIL164_REV 0x04 ++#define SIL164_RSVD 0x05 ++#define SIL164_FREQ_LO 0x06 ++#define SIL164_FREQ_HI 0x07 ++ ++#define SIL164_REG8 0x08 ++#define SIL164_8_VEN (1<<5) ++#define SIL164_8_HEN (1<<4) ++#define SIL164_8_DSEL (1<<3) ++#define SIL164_8_BSEL (1<<2) ++#define SIL164_8_EDGE (1<<1) ++#define SIL164_8_PD (1<<0) ++ ++#define SIL164_REG9 0x09 ++#define SIL164_9_VLOW (1<<7) ++#define SIL164_9_MSEL_MASK (0x7<<4) ++#define SIL164_9_TSEL (1<<3) ++#define SIL164_9_RSEN (1<<2) ++#define SIL164_9_HTPLG (1<<1) ++#define SIL164_9_MDI (1<<0) ++ ++#define SIL164_REGC 0x0c ++ ++struct sil164_priv { ++ //I2CDevRec d; ++ bool quiet; ++}; ++ ++#define SILPTR(d) ((SIL164Ptr)(d->DriverPrivate.ptr)) ++ ++static bool sil164_readb(struct intel_dvo_device *dvo, int addr, u8 *ch) ++{ ++ struct sil164_priv *sil = dvo->dev_priv; ++ struct i2c_adapter *adapter = dvo->i2c_bus; ++ u8 out_buf[2]; ++ u8 in_buf[2]; ++ ++ struct i2c_msg msgs[] = { ++ { ++ .addr = dvo->slave_addr, ++ .flags = 0, ++ .len = 1, ++ .buf = out_buf, ++ }, ++ { ++ .addr = dvo->slave_addr, ++ .flags = I2C_M_RD, ++ .len = 1, ++ .buf = in_buf, ++ } ++ }; ++ ++ out_buf[0] = addr; ++ out_buf[1] = 0; ++ ++ if (i2c_transfer(adapter, msgs, 2) == 2) { ++ *ch = in_buf[0]; ++ return true; ++ } ++ ++ if (!sil->quiet) { ++ DRM_DEBUG_KMS("Unable to read register 0x%02x from %s:%02x.\n", ++ addr, adapter->name, dvo->slave_addr); ++ } ++ return false; ++} ++ ++static bool sil164_writeb(struct intel_dvo_device *dvo, int addr, u8 ch) ++{ ++ struct sil164_priv *sil = dvo->dev_priv; ++ struct i2c_adapter *adapter = dvo->i2c_bus; ++ u8 out_buf[2]; ++ struct i2c_msg msg = { ++ .addr = dvo->slave_addr, ++ .flags = 0, ++ .len = 2, ++ .buf = out_buf, ++ }; ++ ++ out_buf[0] = addr; ++ out_buf[1] = ch; ++ ++ if (i2c_transfer(adapter, &msg, 1) == 1) ++ return true; ++ ++ if (!sil->quiet) { ++ DRM_DEBUG_KMS("Unable to write register 0x%02x to %s:%d.\n", ++ addr, adapter->name, dvo->slave_addr); ++ } ++ ++ return false; ++} ++ ++/* Silicon Image 164 driver for chip on i2c bus */ ++static bool sil164_init(struct intel_dvo_device *dvo, ++ struct i2c_adapter *adapter) ++{ ++ /* this will detect the SIL164 chip on the specified i2c bus */ ++ struct sil164_priv *sil; ++ unsigned char ch; ++ ++ sil = kzalloc(sizeof(struct sil164_priv), GFP_KERNEL); ++ if (sil == NULL) ++ return false; ++ ++ dvo->i2c_bus = adapter; ++ dvo->dev_priv = sil; ++ sil->quiet = true; ++ ++ if (!sil164_readb(dvo, SIL164_VID_LO, &ch)) ++ goto out; ++ ++ if (ch != (SIL164_VID & 0xff)) { ++ DRM_DEBUG_KMS("sil164 not detected got %d: from %s Slave %d.\n", ++ ch, adapter->name, dvo->slave_addr); ++ goto out; ++ } ++ ++ if (!sil164_readb(dvo, SIL164_DID_LO, &ch)) ++ goto out; ++ ++ if (ch != (SIL164_DID & 0xff)) { ++ DRM_DEBUG_KMS("sil164 not detected got %d: from %s Slave %d.\n", ++ ch, adapter->name, dvo->slave_addr); ++ goto out; ++ } ++ sil->quiet = false; ++ ++ DRM_DEBUG_KMS("init sil164 dvo controller successfully!\n"); ++ return true; ++ ++out: ++ kfree(sil); ++ return false; ++} ++ ++static enum drm_connector_status sil164_detect(struct intel_dvo_device *dvo) ++{ ++ u8 reg9; ++ ++ sil164_readb(dvo, SIL164_REG9, ®9); ++ ++ if (reg9 & SIL164_9_HTPLG) ++ return connector_status_connected; ++ else ++ return connector_status_disconnected; ++} ++ ++static enum drm_mode_status sil164_mode_valid(struct intel_dvo_device *dvo, ++ struct drm_display_mode *mode) ++{ ++ return MODE_OK; ++} ++ ++static void sil164_mode_set(struct intel_dvo_device *dvo, ++ const struct drm_display_mode *mode, ++ const struct drm_display_mode *adjusted_mode) ++{ ++ /* As long as the basics are set up, since we don't have clock ++ * dependencies in the mode setup, we can just leave the ++ * registers alone and everything will work fine. ++ */ ++ /* recommended programming sequence from doc */ ++ /*sil164_writeb(sil, 0x08, 0x30); ++ sil164_writeb(sil, 0x09, 0x00); ++ sil164_writeb(sil, 0x0a, 0x90); ++ sil164_writeb(sil, 0x0c, 0x89); ++ sil164_writeb(sil, 0x08, 0x31);*/ ++ /* don't do much */ ++ return; ++} ++ ++/* set the SIL164 power state */ ++static void sil164_dpms(struct intel_dvo_device *dvo, bool enable) ++{ ++ int ret; ++ unsigned char ch; ++ ++ ret = sil164_readb(dvo, SIL164_REG8, &ch); ++ if (ret == false) ++ return; ++ ++ if (enable) ++ ch |= SIL164_8_PD; ++ else ++ ch &= ~SIL164_8_PD; ++ ++ sil164_writeb(dvo, SIL164_REG8, ch); ++ return; ++} ++ ++static bool sil164_get_hw_state(struct intel_dvo_device *dvo) ++{ ++ int ret; ++ unsigned char ch; ++ ++ ret = sil164_readb(dvo, SIL164_REG8, &ch); ++ if (ret == false) ++ return false; ++ ++ if (ch & SIL164_8_PD) ++ return true; ++ else ++ return false; ++} ++ ++static void sil164_dump_regs(struct intel_dvo_device *dvo) ++{ ++ u8 val; ++ ++ sil164_readb(dvo, SIL164_FREQ_LO, &val); ++ DRM_DEBUG_KMS("SIL164_FREQ_LO: 0x%02x\n", val); ++ sil164_readb(dvo, SIL164_FREQ_HI, &val); ++ DRM_DEBUG_KMS("SIL164_FREQ_HI: 0x%02x\n", val); ++ sil164_readb(dvo, SIL164_REG8, &val); ++ DRM_DEBUG_KMS("SIL164_REG8: 0x%02x\n", val); ++ sil164_readb(dvo, SIL164_REG9, &val); ++ DRM_DEBUG_KMS("SIL164_REG9: 0x%02x\n", val); ++ sil164_readb(dvo, SIL164_REGC, &val); ++ DRM_DEBUG_KMS("SIL164_REGC: 0x%02x\n", val); ++} ++ ++static void sil164_destroy(struct intel_dvo_device *dvo) ++{ ++ struct sil164_priv *sil = dvo->dev_priv; ++ ++ if (sil) { ++ kfree(sil); ++ dvo->dev_priv = NULL; ++ } ++} ++ ++const struct intel_dvo_dev_ops sil164_ops = { ++ .init = sil164_init, ++ .detect = sil164_detect, ++ .mode_valid = sil164_mode_valid, ++ .mode_set = sil164_mode_set, ++ .dpms = sil164_dpms, ++ .get_hw_state = sil164_get_hw_state, ++ .dump_regs = sil164_dump_regs, ++ .destroy = sil164_destroy, ++}; +diff --git a/drivers/gpu/drm/i915_legacy/dvo_tfp410.c b/drivers/gpu/drm/i915_legacy/dvo_tfp410.c +new file mode 100644 +index 000000000000..d603bc2f2506 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/dvo_tfp410.c +@@ -0,0 +1,318 @@ ++/* ++ * Copyright © 2007 Dave Mueller ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ * Authors: ++ * Dave Mueller ++ * ++ */ ++ ++#include "dvo.h" ++ ++/* register definitions according to the TFP410 data sheet */ ++#define TFP410_VID 0x014C ++#define TFP410_DID 0x0410 ++ ++#define TFP410_VID_LO 0x00 ++#define TFP410_VID_HI 0x01 ++#define TFP410_DID_LO 0x02 ++#define TFP410_DID_HI 0x03 ++#define TFP410_REV 0x04 ++ ++#define TFP410_CTL_1 0x08 ++#define TFP410_CTL_1_TDIS (1<<6) ++#define TFP410_CTL_1_VEN (1<<5) ++#define TFP410_CTL_1_HEN (1<<4) ++#define TFP410_CTL_1_DSEL (1<<3) ++#define TFP410_CTL_1_BSEL (1<<2) ++#define TFP410_CTL_1_EDGE (1<<1) ++#define TFP410_CTL_1_PD (1<<0) ++ ++#define TFP410_CTL_2 0x09 ++#define TFP410_CTL_2_VLOW (1<<7) ++#define TFP410_CTL_2_MSEL_MASK (0x7<<4) ++#define TFP410_CTL_2_MSEL (1<<4) ++#define TFP410_CTL_2_TSEL (1<<3) ++#define TFP410_CTL_2_RSEN (1<<2) ++#define TFP410_CTL_2_HTPLG (1<<1) ++#define TFP410_CTL_2_MDI (1<<0) ++ ++#define TFP410_CTL_3 0x0A ++#define TFP410_CTL_3_DK_MASK (0x7<<5) ++#define TFP410_CTL_3_DK (1<<5) ++#define TFP410_CTL_3_DKEN (1<<4) ++#define TFP410_CTL_3_CTL_MASK (0x7<<1) ++#define TFP410_CTL_3_CTL (1<<1) ++ ++#define TFP410_USERCFG 0x0B ++ ++#define TFP410_DE_DLY 0x32 ++ ++#define TFP410_DE_CTL 0x33 ++#define TFP410_DE_CTL_DEGEN (1<<6) ++#define TFP410_DE_CTL_VSPOL (1<<5) ++#define TFP410_DE_CTL_HSPOL (1<<4) ++#define TFP410_DE_CTL_DEDLY8 (1<<0) ++ ++#define TFP410_DE_TOP 0x34 ++ ++#define TFP410_DE_CNT_LO 0x36 ++#define TFP410_DE_CNT_HI 0x37 ++ ++#define TFP410_DE_LIN_LO 0x38 ++#define TFP410_DE_LIN_HI 0x39 ++ ++#define TFP410_H_RES_LO 0x3A ++#define TFP410_H_RES_HI 0x3B ++ ++#define TFP410_V_RES_LO 0x3C ++#define TFP410_V_RES_HI 0x3D ++ ++struct tfp410_priv { ++ bool quiet; ++}; ++ ++static bool tfp410_readb(struct intel_dvo_device *dvo, int addr, u8 *ch) ++{ ++ struct tfp410_priv *tfp = dvo->dev_priv; ++ struct i2c_adapter *adapter = dvo->i2c_bus; ++ u8 out_buf[2]; ++ u8 in_buf[2]; ++ ++ struct i2c_msg msgs[] = { ++ { ++ .addr = dvo->slave_addr, ++ .flags = 0, ++ .len = 1, ++ .buf = out_buf, ++ }, ++ { ++ .addr = dvo->slave_addr, ++ .flags = I2C_M_RD, ++ .len = 1, ++ .buf = in_buf, ++ } ++ }; ++ ++ out_buf[0] = addr; ++ out_buf[1] = 0; ++ ++ if (i2c_transfer(adapter, msgs, 2) == 2) { ++ *ch = in_buf[0]; ++ return true; ++ } ++ ++ if (!tfp->quiet) { ++ DRM_DEBUG_KMS("Unable to read register 0x%02x from %s:%02x.\n", ++ addr, adapter->name, dvo->slave_addr); ++ } ++ return false; ++} ++ ++static bool tfp410_writeb(struct intel_dvo_device *dvo, int addr, u8 ch) ++{ ++ struct tfp410_priv *tfp = dvo->dev_priv; ++ struct i2c_adapter *adapter = dvo->i2c_bus; ++ u8 out_buf[2]; ++ struct i2c_msg msg = { ++ .addr = dvo->slave_addr, ++ .flags = 0, ++ .len = 2, ++ .buf = out_buf, ++ }; ++ ++ out_buf[0] = addr; ++ out_buf[1] = ch; ++ ++ if (i2c_transfer(adapter, &msg, 1) == 1) ++ return true; ++ ++ if (!tfp->quiet) { ++ DRM_DEBUG_KMS("Unable to write register 0x%02x to %s:%d.\n", ++ addr, adapter->name, dvo->slave_addr); ++ } ++ ++ return false; ++} ++ ++static int tfp410_getid(struct intel_dvo_device *dvo, int addr) ++{ ++ u8 ch1, ch2; ++ ++ if (tfp410_readb(dvo, addr+0, &ch1) && ++ tfp410_readb(dvo, addr+1, &ch2)) ++ return ((ch2 << 8) & 0xFF00) | (ch1 & 0x00FF); ++ ++ return -1; ++} ++ ++/* Ti TFP410 driver for chip on i2c bus */ ++static bool tfp410_init(struct intel_dvo_device *dvo, ++ struct i2c_adapter *adapter) ++{ ++ /* this will detect the tfp410 chip on the specified i2c bus */ ++ struct tfp410_priv *tfp; ++ int id; ++ ++ tfp = kzalloc(sizeof(struct tfp410_priv), GFP_KERNEL); ++ if (tfp == NULL) ++ return false; ++ ++ dvo->i2c_bus = adapter; ++ dvo->dev_priv = tfp; ++ tfp->quiet = true; ++ ++ if ((id = tfp410_getid(dvo, TFP410_VID_LO)) != TFP410_VID) { ++ DRM_DEBUG_KMS("tfp410 not detected got VID %X: from %s " ++ "Slave %d.\n", ++ id, adapter->name, dvo->slave_addr); ++ goto out; ++ } ++ ++ if ((id = tfp410_getid(dvo, TFP410_DID_LO)) != TFP410_DID) { ++ DRM_DEBUG_KMS("tfp410 not detected got DID %X: from %s " ++ "Slave %d.\n", ++ id, adapter->name, dvo->slave_addr); ++ goto out; ++ } ++ tfp->quiet = false; ++ return true; ++out: ++ kfree(tfp); ++ return false; ++} ++ ++static enum drm_connector_status tfp410_detect(struct intel_dvo_device *dvo) ++{ ++ enum drm_connector_status ret = connector_status_disconnected; ++ u8 ctl2; ++ ++ if (tfp410_readb(dvo, TFP410_CTL_2, &ctl2)) { ++ if (ctl2 & TFP410_CTL_2_RSEN) ++ ret = connector_status_connected; ++ else ++ ret = connector_status_disconnected; ++ } ++ ++ return ret; ++} ++ ++static enum drm_mode_status tfp410_mode_valid(struct intel_dvo_device *dvo, ++ struct drm_display_mode *mode) ++{ ++ return MODE_OK; ++} ++ ++static void tfp410_mode_set(struct intel_dvo_device *dvo, ++ const struct drm_display_mode *mode, ++ const struct drm_display_mode *adjusted_mode) ++{ ++ /* As long as the basics are set up, since we don't have clock dependencies ++ * in the mode setup, we can just leave the registers alone and everything ++ * will work fine. ++ */ ++ /* don't do much */ ++ return; ++} ++ ++/* set the tfp410 power state */ ++static void tfp410_dpms(struct intel_dvo_device *dvo, bool enable) ++{ ++ u8 ctl1; ++ ++ if (!tfp410_readb(dvo, TFP410_CTL_1, &ctl1)) ++ return; ++ ++ if (enable) ++ ctl1 |= TFP410_CTL_1_PD; ++ else ++ ctl1 &= ~TFP410_CTL_1_PD; ++ ++ tfp410_writeb(dvo, TFP410_CTL_1, ctl1); ++} ++ ++static bool tfp410_get_hw_state(struct intel_dvo_device *dvo) ++{ ++ u8 ctl1; ++ ++ if (!tfp410_readb(dvo, TFP410_CTL_1, &ctl1)) ++ return false; ++ ++ if (ctl1 & TFP410_CTL_1_PD) ++ return true; ++ else ++ return false; ++} ++ ++static void tfp410_dump_regs(struct intel_dvo_device *dvo) ++{ ++ u8 val, val2; ++ ++ tfp410_readb(dvo, TFP410_REV, &val); ++ DRM_DEBUG_KMS("TFP410_REV: 0x%02X\n", val); ++ tfp410_readb(dvo, TFP410_CTL_1, &val); ++ DRM_DEBUG_KMS("TFP410_CTL1: 0x%02X\n", val); ++ tfp410_readb(dvo, TFP410_CTL_2, &val); ++ DRM_DEBUG_KMS("TFP410_CTL2: 0x%02X\n", val); ++ tfp410_readb(dvo, TFP410_CTL_3, &val); ++ DRM_DEBUG_KMS("TFP410_CTL3: 0x%02X\n", val); ++ tfp410_readb(dvo, TFP410_USERCFG, &val); ++ DRM_DEBUG_KMS("TFP410_USERCFG: 0x%02X\n", val); ++ tfp410_readb(dvo, TFP410_DE_DLY, &val); ++ DRM_DEBUG_KMS("TFP410_DE_DLY: 0x%02X\n", val); ++ tfp410_readb(dvo, TFP410_DE_CTL, &val); ++ DRM_DEBUG_KMS("TFP410_DE_CTL: 0x%02X\n", val); ++ tfp410_readb(dvo, TFP410_DE_TOP, &val); ++ DRM_DEBUG_KMS("TFP410_DE_TOP: 0x%02X\n", val); ++ tfp410_readb(dvo, TFP410_DE_CNT_LO, &val); ++ tfp410_readb(dvo, TFP410_DE_CNT_HI, &val2); ++ DRM_DEBUG_KMS("TFP410_DE_CNT: 0x%02X%02X\n", val2, val); ++ tfp410_readb(dvo, TFP410_DE_LIN_LO, &val); ++ tfp410_readb(dvo, TFP410_DE_LIN_HI, &val2); ++ DRM_DEBUG_KMS("TFP410_DE_LIN: 0x%02X%02X\n", val2, val); ++ tfp410_readb(dvo, TFP410_H_RES_LO, &val); ++ tfp410_readb(dvo, TFP410_H_RES_HI, &val2); ++ DRM_DEBUG_KMS("TFP410_H_RES: 0x%02X%02X\n", val2, val); ++ tfp410_readb(dvo, TFP410_V_RES_LO, &val); ++ tfp410_readb(dvo, TFP410_V_RES_HI, &val2); ++ DRM_DEBUG_KMS("TFP410_V_RES: 0x%02X%02X\n", val2, val); ++} ++ ++static void tfp410_destroy(struct intel_dvo_device *dvo) ++{ ++ struct tfp410_priv *tfp = dvo->dev_priv; ++ ++ if (tfp) { ++ kfree(tfp); ++ dvo->dev_priv = NULL; ++ } ++} ++ ++const struct intel_dvo_dev_ops tfp410_ops = { ++ .init = tfp410_init, ++ .detect = tfp410_detect, ++ .mode_valid = tfp410_mode_valid, ++ .mode_set = tfp410_mode_set, ++ .dpms = tfp410_dpms, ++ .get_hw_state = tfp410_get_hw_state, ++ .dump_regs = tfp410_dump_regs, ++ .destroy = tfp410_destroy, ++}; +diff --git a/drivers/gpu/drm/i915_legacy/gvt/Makefile b/drivers/gpu/drm/i915_legacy/gvt/Makefile +new file mode 100644 +index 000000000000..ea8324abc784 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/Makefile +@@ -0,0 +1,9 @@ ++# SPDX-License-Identifier: GPL-2.0 ++GVT_DIR := gvt ++GVT_SOURCE := gvt.o aperture_gm.o handlers.o vgpu.o trace_points.o firmware.o \ ++ interrupt.o gtt.o cfg_space.o opregion.o mmio.o display.o edid.o \ ++ execlist.o scheduler.o sched_policy.o mmio_context.o cmd_parser.o debugfs.o \ ++ fb_decoder.o dmabuf.o page_track.o ++ ++ccflags-y += -I $(srctree)/$(src) -I $(srctree)/$(src)/$(GVT_DIR)/ ++i915-y += $(addprefix $(GVT_DIR)/, $(GVT_SOURCE)) +diff --git a/drivers/gpu/drm/i915_legacy/gvt/aperture_gm.c b/drivers/gpu/drm/i915_legacy/gvt/aperture_gm.c +new file mode 100644 +index 000000000000..1fa2f65c3cd1 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/aperture_gm.c +@@ -0,0 +1,359 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Kevin Tian ++ * Dexuan Cui ++ * ++ * Contributors: ++ * Pei Zhang ++ * Min He ++ * Niu Bing ++ * Yulei Zhang ++ * Zhenyu Wang ++ * Zhi Wang ++ * ++ */ ++ ++#include "i915_drv.h" ++#include "gvt.h" ++ ++static int alloc_gm(struct intel_vgpu *vgpu, bool high_gm) ++{ ++ struct intel_gvt *gvt = vgpu->gvt; ++ struct drm_i915_private *dev_priv = gvt->dev_priv; ++ unsigned int flags; ++ u64 start, end, size; ++ struct drm_mm_node *node; ++ int ret; ++ ++ if (high_gm) { ++ node = &vgpu->gm.high_gm_node; ++ size = vgpu_hidden_sz(vgpu); ++ start = ALIGN(gvt_hidden_gmadr_base(gvt), I915_GTT_PAGE_SIZE); ++ end = ALIGN(gvt_hidden_gmadr_end(gvt), I915_GTT_PAGE_SIZE); ++ flags = PIN_HIGH; ++ } else { ++ node = &vgpu->gm.low_gm_node; ++ size = vgpu_aperture_sz(vgpu); ++ start = ALIGN(gvt_aperture_gmadr_base(gvt), I915_GTT_PAGE_SIZE); ++ end = ALIGN(gvt_aperture_gmadr_end(gvt), I915_GTT_PAGE_SIZE); ++ flags = PIN_MAPPABLE; ++ } ++ ++ mutex_lock(&dev_priv->drm.struct_mutex); ++ mmio_hw_access_pre(dev_priv); ++ ret = i915_gem_gtt_insert(&dev_priv->ggtt.vm, node, ++ size, I915_GTT_PAGE_SIZE, ++ I915_COLOR_UNEVICTABLE, ++ start, end, flags); ++ mmio_hw_access_post(dev_priv); ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++ if (ret) ++ gvt_err("fail to alloc %s gm space from host\n", ++ high_gm ? "high" : "low"); ++ ++ return ret; ++} ++ ++static int alloc_vgpu_gm(struct intel_vgpu *vgpu) ++{ ++ struct intel_gvt *gvt = vgpu->gvt; ++ struct drm_i915_private *dev_priv = gvt->dev_priv; ++ int ret; ++ ++ ret = alloc_gm(vgpu, false); ++ if (ret) ++ return ret; ++ ++ ret = alloc_gm(vgpu, true); ++ if (ret) ++ goto out_free_aperture; ++ ++ gvt_dbg_core("vgpu%d: alloc low GM start %llx size %llx\n", vgpu->id, ++ vgpu_aperture_offset(vgpu), vgpu_aperture_sz(vgpu)); ++ ++ gvt_dbg_core("vgpu%d: alloc high GM start %llx size %llx\n", vgpu->id, ++ vgpu_hidden_offset(vgpu), vgpu_hidden_sz(vgpu)); ++ ++ return 0; ++out_free_aperture: ++ mutex_lock(&dev_priv->drm.struct_mutex); ++ drm_mm_remove_node(&vgpu->gm.low_gm_node); ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++ return ret; ++} ++ ++static void free_vgpu_gm(struct intel_vgpu *vgpu) ++{ ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ ++ mutex_lock(&dev_priv->drm.struct_mutex); ++ drm_mm_remove_node(&vgpu->gm.low_gm_node); ++ drm_mm_remove_node(&vgpu->gm.high_gm_node); ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++} ++ ++/** ++ * intel_vgpu_write_fence - write fence registers owned by a vGPU ++ * @vgpu: vGPU instance ++ * @fence: vGPU fence register number ++ * @value: Fence register value to be written ++ * ++ * This function is used to write fence registers owned by a vGPU. The vGPU ++ * fence register number will be translated into HW fence register number. ++ * ++ */ ++void intel_vgpu_write_fence(struct intel_vgpu *vgpu, ++ u32 fence, u64 value) ++{ ++ struct intel_gvt *gvt = vgpu->gvt; ++ struct drm_i915_private *dev_priv = gvt->dev_priv; ++ struct drm_i915_fence_reg *reg; ++ i915_reg_t fence_reg_lo, fence_reg_hi; ++ ++ assert_rpm_wakelock_held(dev_priv); ++ ++ if (WARN_ON(fence >= vgpu_fence_sz(vgpu))) ++ return; ++ ++ reg = vgpu->fence.regs[fence]; ++ if (WARN_ON(!reg)) ++ return; ++ ++ fence_reg_lo = FENCE_REG_GEN6_LO(reg->id); ++ fence_reg_hi = FENCE_REG_GEN6_HI(reg->id); ++ ++ I915_WRITE(fence_reg_lo, 0); ++ POSTING_READ(fence_reg_lo); ++ ++ I915_WRITE(fence_reg_hi, upper_32_bits(value)); ++ I915_WRITE(fence_reg_lo, lower_32_bits(value)); ++ POSTING_READ(fence_reg_lo); ++} ++ ++static void _clear_vgpu_fence(struct intel_vgpu *vgpu) ++{ ++ int i; ++ ++ for (i = 0; i < vgpu_fence_sz(vgpu); i++) ++ intel_vgpu_write_fence(vgpu, i, 0); ++} ++ ++static void free_vgpu_fence(struct intel_vgpu *vgpu) ++{ ++ struct intel_gvt *gvt = vgpu->gvt; ++ struct drm_i915_private *dev_priv = gvt->dev_priv; ++ struct drm_i915_fence_reg *reg; ++ u32 i; ++ ++ if (WARN_ON(!vgpu_fence_sz(vgpu))) ++ return; ++ ++ intel_runtime_pm_get(dev_priv); ++ ++ mutex_lock(&dev_priv->drm.struct_mutex); ++ _clear_vgpu_fence(vgpu); ++ for (i = 0; i < vgpu_fence_sz(vgpu); i++) { ++ reg = vgpu->fence.regs[i]; ++ i915_unreserve_fence(reg); ++ vgpu->fence.regs[i] = NULL; ++ } ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++ ++ intel_runtime_pm_put_unchecked(dev_priv); ++} ++ ++static int alloc_vgpu_fence(struct intel_vgpu *vgpu) ++{ ++ struct intel_gvt *gvt = vgpu->gvt; ++ struct drm_i915_private *dev_priv = gvt->dev_priv; ++ struct drm_i915_fence_reg *reg; ++ int i; ++ ++ intel_runtime_pm_get(dev_priv); ++ ++ /* Request fences from host */ ++ mutex_lock(&dev_priv->drm.struct_mutex); ++ ++ for (i = 0; i < vgpu_fence_sz(vgpu); i++) { ++ reg = i915_reserve_fence(dev_priv); ++ if (IS_ERR(reg)) ++ goto out_free_fence; ++ ++ vgpu->fence.regs[i] = reg; ++ } ++ ++ _clear_vgpu_fence(vgpu); ++ ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++ intel_runtime_pm_put_unchecked(dev_priv); ++ return 0; ++out_free_fence: ++ gvt_vgpu_err("Failed to alloc fences\n"); ++ /* Return fences to host, if fail */ ++ for (i = 0; i < vgpu_fence_sz(vgpu); i++) { ++ reg = vgpu->fence.regs[i]; ++ if (!reg) ++ continue; ++ i915_unreserve_fence(reg); ++ vgpu->fence.regs[i] = NULL; ++ } ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++ intel_runtime_pm_put_unchecked(dev_priv); ++ return -ENOSPC; ++} ++ ++static void free_resource(struct intel_vgpu *vgpu) ++{ ++ struct intel_gvt *gvt = vgpu->gvt; ++ ++ gvt->gm.vgpu_allocated_low_gm_size -= vgpu_aperture_sz(vgpu); ++ gvt->gm.vgpu_allocated_high_gm_size -= vgpu_hidden_sz(vgpu); ++ gvt->fence.vgpu_allocated_fence_num -= vgpu_fence_sz(vgpu); ++} ++ ++static int alloc_resource(struct intel_vgpu *vgpu, ++ struct intel_vgpu_creation_params *param) ++{ ++ struct intel_gvt *gvt = vgpu->gvt; ++ unsigned long request, avail, max, taken; ++ const char *item; ++ ++ if (!param->low_gm_sz || !param->high_gm_sz || !param->fence_sz) { ++ gvt_vgpu_err("Invalid vGPU creation params\n"); ++ return -EINVAL; ++ } ++ ++ item = "low GM space"; ++ max = gvt_aperture_sz(gvt) - HOST_LOW_GM_SIZE; ++ taken = gvt->gm.vgpu_allocated_low_gm_size; ++ avail = max - taken; ++ request = MB_TO_BYTES(param->low_gm_sz); ++ ++ if (request > avail) ++ goto no_enough_resource; ++ ++ vgpu_aperture_sz(vgpu) = ALIGN(request, I915_GTT_PAGE_SIZE); ++ ++ item = "high GM space"; ++ max = gvt_hidden_sz(gvt) - HOST_HIGH_GM_SIZE; ++ taken = gvt->gm.vgpu_allocated_high_gm_size; ++ avail = max - taken; ++ request = MB_TO_BYTES(param->high_gm_sz); ++ ++ if (request > avail) ++ goto no_enough_resource; ++ ++ vgpu_hidden_sz(vgpu) = ALIGN(request, I915_GTT_PAGE_SIZE); ++ ++ item = "fence"; ++ max = gvt_fence_sz(gvt) - HOST_FENCE; ++ taken = gvt->fence.vgpu_allocated_fence_num; ++ avail = max - taken; ++ request = param->fence_sz; ++ ++ if (request > avail) ++ goto no_enough_resource; ++ ++ vgpu_fence_sz(vgpu) = request; ++ ++ gvt->gm.vgpu_allocated_low_gm_size += MB_TO_BYTES(param->low_gm_sz); ++ gvt->gm.vgpu_allocated_high_gm_size += MB_TO_BYTES(param->high_gm_sz); ++ gvt->fence.vgpu_allocated_fence_num += param->fence_sz; ++ return 0; ++ ++no_enough_resource: ++ gvt_err("fail to allocate resource %s\n", item); ++ gvt_err("request %luMB avail %luMB max %luMB taken %luMB\n", ++ BYTES_TO_MB(request), BYTES_TO_MB(avail), ++ BYTES_TO_MB(max), BYTES_TO_MB(taken)); ++ return -ENOSPC; ++} ++ ++/** ++ * inte_gvt_free_vgpu_resource - free HW resource owned by a vGPU ++ * @vgpu: a vGPU ++ * ++ * This function is used to free the HW resource owned by a vGPU. ++ * ++ */ ++void intel_vgpu_free_resource(struct intel_vgpu *vgpu) ++{ ++ free_vgpu_gm(vgpu); ++ free_vgpu_fence(vgpu); ++ free_resource(vgpu); ++} ++ ++/** ++ * intel_vgpu_reset_resource - reset resource state owned by a vGPU ++ * @vgpu: a vGPU ++ * ++ * This function is used to reset resource state owned by a vGPU. ++ * ++ */ ++void intel_vgpu_reset_resource(struct intel_vgpu *vgpu) ++{ ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ ++ intel_runtime_pm_get(dev_priv); ++ _clear_vgpu_fence(vgpu); ++ intel_runtime_pm_put_unchecked(dev_priv); ++} ++ ++/** ++ * intel_alloc_vgpu_resource - allocate HW resource for a vGPU ++ * @vgpu: vGPU ++ * @param: vGPU creation params ++ * ++ * This function is used to allocate HW resource for a vGPU. User specifies ++ * the resource configuration through the creation params. ++ * ++ * Returns: ++ * zero on success, negative error code if failed. ++ * ++ */ ++int intel_vgpu_alloc_resource(struct intel_vgpu *vgpu, ++ struct intel_vgpu_creation_params *param) ++{ ++ int ret; ++ ++ ret = alloc_resource(vgpu, param); ++ if (ret) ++ return ret; ++ ++ ret = alloc_vgpu_gm(vgpu); ++ if (ret) ++ goto out_free_resource; ++ ++ ret = alloc_vgpu_fence(vgpu); ++ if (ret) ++ goto out_free_vgpu_gm; ++ ++ return 0; ++ ++out_free_vgpu_gm: ++ free_vgpu_gm(vgpu); ++out_free_resource: ++ free_resource(vgpu); ++ return ret; ++} +diff --git a/drivers/gpu/drm/i915_legacy/gvt/cfg_space.c b/drivers/gpu/drm/i915_legacy/gvt/cfg_space.c +new file mode 100644 +index 000000000000..19cf1bbe059d +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/cfg_space.c +@@ -0,0 +1,424 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Eddie Dong ++ * Jike Song ++ * ++ * Contributors: ++ * Zhi Wang ++ * Min He ++ * Bing Niu ++ * ++ */ ++ ++#include "i915_drv.h" ++#include "gvt.h" ++ ++enum { ++ INTEL_GVT_PCI_BAR_GTTMMIO = 0, ++ INTEL_GVT_PCI_BAR_APERTURE, ++ INTEL_GVT_PCI_BAR_PIO, ++ INTEL_GVT_PCI_BAR_MAX, ++}; ++ ++/* bitmap for writable bits (RW or RW1C bits, but cannot co-exist in one ++ * byte) byte by byte in standard pci configuration space. (not the full ++ * 256 bytes.) ++ */ ++static const u8 pci_cfg_space_rw_bmp[PCI_INTERRUPT_LINE + 4] = { ++ [PCI_COMMAND] = 0xff, 0x07, ++ [PCI_STATUS] = 0x00, 0xf9, /* the only one RW1C byte */ ++ [PCI_CACHE_LINE_SIZE] = 0xff, ++ [PCI_BASE_ADDRESS_0 ... PCI_CARDBUS_CIS - 1] = 0xff, ++ [PCI_ROM_ADDRESS] = 0x01, 0xf8, 0xff, 0xff, ++ [PCI_INTERRUPT_LINE] = 0xff, ++}; ++ ++/** ++ * vgpu_pci_cfg_mem_write - write virtual cfg space memory ++ * @vgpu: target vgpu ++ * @off: offset ++ * @src: src ptr to write ++ * @bytes: number of bytes ++ * ++ * Use this function to write virtual cfg space memory. ++ * For standard cfg space, only RW bits can be changed, ++ * and we emulates the RW1C behavior of PCI_STATUS register. ++ */ ++static void vgpu_pci_cfg_mem_write(struct intel_vgpu *vgpu, unsigned int off, ++ u8 *src, unsigned int bytes) ++{ ++ u8 *cfg_base = vgpu_cfg_space(vgpu); ++ u8 mask, new, old; ++ int i = 0; ++ ++ for (; i < bytes && (off + i < sizeof(pci_cfg_space_rw_bmp)); i++) { ++ mask = pci_cfg_space_rw_bmp[off + i]; ++ old = cfg_base[off + i]; ++ new = src[i] & mask; ++ ++ /** ++ * The PCI_STATUS high byte has RW1C bits, here ++ * emulates clear by writing 1 for these bits. ++ * Writing a 0b to RW1C bits has no effect. ++ */ ++ if (off + i == PCI_STATUS + 1) ++ new = (~new & old) & mask; ++ ++ cfg_base[off + i] = (old & ~mask) | new; ++ } ++ ++ /* For other configuration space directly copy as it is. */ ++ if (i < bytes) ++ memcpy(cfg_base + off + i, src + i, bytes - i); ++} ++ ++/** ++ * intel_vgpu_emulate_cfg_read - emulate vGPU configuration space read ++ * @vgpu: target vgpu ++ * @offset: offset ++ * @p_data: return data ptr ++ * @bytes: number of bytes to read ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++int intel_vgpu_emulate_cfg_read(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ if (WARN_ON(bytes > 4)) ++ return -EINVAL; ++ ++ if (WARN_ON(offset + bytes > vgpu->gvt->device_info.cfg_space_size)) ++ return -EINVAL; ++ ++ memcpy(p_data, vgpu_cfg_space(vgpu) + offset, bytes); ++ return 0; ++} ++ ++static int map_aperture(struct intel_vgpu *vgpu, bool map) ++{ ++ phys_addr_t aperture_pa = vgpu_aperture_pa_base(vgpu); ++ unsigned long aperture_sz = vgpu_aperture_sz(vgpu); ++ u64 first_gfn; ++ u64 val; ++ int ret; ++ ++ if (map == vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_APERTURE].tracked) ++ return 0; ++ ++ val = vgpu_cfg_space(vgpu)[PCI_BASE_ADDRESS_2]; ++ if (val & PCI_BASE_ADDRESS_MEM_TYPE_64) ++ val = *(u64 *)(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_2); ++ else ++ val = *(u32 *)(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_2); ++ ++ first_gfn = (val + vgpu_aperture_offset(vgpu)) >> PAGE_SHIFT; ++ ++ ret = intel_gvt_hypervisor_map_gfn_to_mfn(vgpu, first_gfn, ++ aperture_pa >> PAGE_SHIFT, ++ aperture_sz >> PAGE_SHIFT, ++ map); ++ if (ret) ++ return ret; ++ ++ vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_APERTURE].tracked = map; ++ return 0; ++} ++ ++static int trap_gttmmio(struct intel_vgpu *vgpu, bool trap) ++{ ++ u64 start, end; ++ u64 val; ++ int ret; ++ ++ if (trap == vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_GTTMMIO].tracked) ++ return 0; ++ ++ val = vgpu_cfg_space(vgpu)[PCI_BASE_ADDRESS_0]; ++ if (val & PCI_BASE_ADDRESS_MEM_TYPE_64) ++ start = *(u64 *)(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_0); ++ else ++ start = *(u32 *)(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_0); ++ ++ start &= ~GENMASK(3, 0); ++ end = start + vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_GTTMMIO].size - 1; ++ ++ ret = intel_gvt_hypervisor_set_trap_area(vgpu, start, end, trap); ++ if (ret) ++ return ret; ++ ++ vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_GTTMMIO].tracked = trap; ++ return 0; ++} ++ ++static int emulate_pci_command_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ u8 old = vgpu_cfg_space(vgpu)[offset]; ++ u8 new = *(u8 *)p_data; ++ u8 changed = old ^ new; ++ int ret; ++ ++ vgpu_pci_cfg_mem_write(vgpu, offset, p_data, bytes); ++ if (!(changed & PCI_COMMAND_MEMORY)) ++ return 0; ++ ++ if (old & PCI_COMMAND_MEMORY) { ++ ret = trap_gttmmio(vgpu, false); ++ if (ret) ++ return ret; ++ ret = map_aperture(vgpu, false); ++ if (ret) ++ return ret; ++ } else { ++ ret = trap_gttmmio(vgpu, true); ++ if (ret) ++ return ret; ++ ret = map_aperture(vgpu, true); ++ if (ret) ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static int emulate_pci_rom_bar_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ u32 *pval = (u32 *)(vgpu_cfg_space(vgpu) + offset); ++ u32 new = *(u32 *)(p_data); ++ ++ if ((new & PCI_ROM_ADDRESS_MASK) == PCI_ROM_ADDRESS_MASK) ++ /* We don't have rom, return size of 0. */ ++ *pval = 0; ++ else ++ vgpu_pci_cfg_mem_write(vgpu, offset, p_data, bytes); ++ return 0; ++} ++ ++static int emulate_pci_bar_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ u32 new = *(u32 *)(p_data); ++ bool lo = IS_ALIGNED(offset, 8); ++ u64 size; ++ int ret = 0; ++ bool mmio_enabled = ++ vgpu_cfg_space(vgpu)[PCI_COMMAND] & PCI_COMMAND_MEMORY; ++ struct intel_vgpu_pci_bar *bars = vgpu->cfg_space.bar; ++ ++ /* ++ * Power-up software can determine how much address ++ * space the device requires by writing a value of ++ * all 1's to the register and then reading the value ++ * back. The device will return 0's in all don't-care ++ * address bits. ++ */ ++ if (new == 0xffffffff) { ++ switch (offset) { ++ case PCI_BASE_ADDRESS_0: ++ case PCI_BASE_ADDRESS_1: ++ size = ~(bars[INTEL_GVT_PCI_BAR_GTTMMIO].size -1); ++ intel_vgpu_write_pci_bar(vgpu, offset, ++ size >> (lo ? 0 : 32), lo); ++ /* ++ * Untrap the BAR, since guest hasn't configured a ++ * valid GPA ++ */ ++ ret = trap_gttmmio(vgpu, false); ++ break; ++ case PCI_BASE_ADDRESS_2: ++ case PCI_BASE_ADDRESS_3: ++ size = ~(bars[INTEL_GVT_PCI_BAR_APERTURE].size -1); ++ intel_vgpu_write_pci_bar(vgpu, offset, ++ size >> (lo ? 0 : 32), lo); ++ ret = map_aperture(vgpu, false); ++ break; ++ default: ++ /* Unimplemented BARs */ ++ intel_vgpu_write_pci_bar(vgpu, offset, 0x0, false); ++ } ++ } else { ++ switch (offset) { ++ case PCI_BASE_ADDRESS_0: ++ case PCI_BASE_ADDRESS_1: ++ /* ++ * Untrap the old BAR first, since guest has ++ * re-configured the BAR ++ */ ++ trap_gttmmio(vgpu, false); ++ intel_vgpu_write_pci_bar(vgpu, offset, new, lo); ++ ret = trap_gttmmio(vgpu, mmio_enabled); ++ break; ++ case PCI_BASE_ADDRESS_2: ++ case PCI_BASE_ADDRESS_3: ++ map_aperture(vgpu, false); ++ intel_vgpu_write_pci_bar(vgpu, offset, new, lo); ++ ret = map_aperture(vgpu, mmio_enabled); ++ break; ++ default: ++ intel_vgpu_write_pci_bar(vgpu, offset, new, lo); ++ } ++ } ++ return ret; ++} ++ ++/** ++ * intel_vgpu_emulate_cfg_read - emulate vGPU configuration space write ++ * @vgpu: target vgpu ++ * @offset: offset ++ * @p_data: write data ptr ++ * @bytes: number of bytes to write ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++int intel_vgpu_emulate_cfg_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ int ret; ++ ++ if (WARN_ON(bytes > 4)) ++ return -EINVAL; ++ ++ if (WARN_ON(offset + bytes > vgpu->gvt->device_info.cfg_space_size)) ++ return -EINVAL; ++ ++ /* First check if it's PCI_COMMAND */ ++ if (IS_ALIGNED(offset, 2) && offset == PCI_COMMAND) { ++ if (WARN_ON(bytes > 2)) ++ return -EINVAL; ++ return emulate_pci_command_write(vgpu, offset, p_data, bytes); ++ } ++ ++ switch (rounddown(offset, 4)) { ++ case PCI_ROM_ADDRESS: ++ if (WARN_ON(!IS_ALIGNED(offset, 4))) ++ return -EINVAL; ++ return emulate_pci_rom_bar_write(vgpu, offset, p_data, bytes); ++ ++ case PCI_BASE_ADDRESS_0 ... PCI_BASE_ADDRESS_5: ++ if (WARN_ON(!IS_ALIGNED(offset, 4))) ++ return -EINVAL; ++ return emulate_pci_bar_write(vgpu, offset, p_data, bytes); ++ ++ case INTEL_GVT_PCI_SWSCI: ++ if (WARN_ON(!IS_ALIGNED(offset, 4))) ++ return -EINVAL; ++ ret = intel_vgpu_emulate_opregion_request(vgpu, *(u32 *)p_data); ++ if (ret) ++ return ret; ++ break; ++ ++ case INTEL_GVT_PCI_OPREGION: ++ if (WARN_ON(!IS_ALIGNED(offset, 4))) ++ return -EINVAL; ++ ret = intel_vgpu_opregion_base_write_handler(vgpu, ++ *(u32 *)p_data); ++ if (ret) ++ return ret; ++ ++ vgpu_pci_cfg_mem_write(vgpu, offset, p_data, bytes); ++ break; ++ default: ++ vgpu_pci_cfg_mem_write(vgpu, offset, p_data, bytes); ++ break; ++ } ++ return 0; ++} ++ ++/** ++ * intel_vgpu_init_cfg_space - init vGPU configuration space when create vGPU ++ * ++ * @vgpu: a vGPU ++ * @primary: is the vGPU presented as primary ++ * ++ */ ++void intel_vgpu_init_cfg_space(struct intel_vgpu *vgpu, ++ bool primary) ++{ ++ struct intel_gvt *gvt = vgpu->gvt; ++ const struct intel_gvt_device_info *info = &gvt->device_info; ++ u16 *gmch_ctl; ++ ++ memcpy(vgpu_cfg_space(vgpu), gvt->firmware.cfg_space, ++ info->cfg_space_size); ++ ++ if (!primary) { ++ vgpu_cfg_space(vgpu)[PCI_CLASS_DEVICE] = ++ INTEL_GVT_PCI_CLASS_VGA_OTHER; ++ vgpu_cfg_space(vgpu)[PCI_CLASS_PROG] = ++ INTEL_GVT_PCI_CLASS_VGA_OTHER; ++ } ++ ++ /* Show guest that there isn't any stolen memory.*/ ++ gmch_ctl = (u16 *)(vgpu_cfg_space(vgpu) + INTEL_GVT_PCI_GMCH_CONTROL); ++ *gmch_ctl &= ~(BDW_GMCH_GMS_MASK << BDW_GMCH_GMS_SHIFT); ++ ++ intel_vgpu_write_pci_bar(vgpu, PCI_BASE_ADDRESS_2, ++ gvt_aperture_pa_base(gvt), true); ++ ++ vgpu_cfg_space(vgpu)[PCI_COMMAND] &= ~(PCI_COMMAND_IO ++ | PCI_COMMAND_MEMORY ++ | PCI_COMMAND_MASTER); ++ /* ++ * Clear the bar upper 32bit and let guest to assign the new value ++ */ ++ memset(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_1, 0, 4); ++ memset(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_3, 0, 4); ++ memset(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_4, 0, 8); ++ memset(vgpu_cfg_space(vgpu) + INTEL_GVT_PCI_OPREGION, 0, 4); ++ ++ vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_GTTMMIO].size = ++ pci_resource_len(gvt->dev_priv->drm.pdev, 0); ++ vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_APERTURE].size = ++ pci_resource_len(gvt->dev_priv->drm.pdev, 2); ++ ++ memset(vgpu_cfg_space(vgpu) + PCI_ROM_ADDRESS, 0, 4); ++} ++ ++/** ++ * intel_vgpu_reset_cfg_space - reset vGPU configuration space ++ * ++ * @vgpu: a vGPU ++ * ++ */ ++void intel_vgpu_reset_cfg_space(struct intel_vgpu *vgpu) ++{ ++ u8 cmd = vgpu_cfg_space(vgpu)[PCI_COMMAND]; ++ bool primary = vgpu_cfg_space(vgpu)[PCI_CLASS_DEVICE] != ++ INTEL_GVT_PCI_CLASS_VGA_OTHER; ++ ++ if (cmd & PCI_COMMAND_MEMORY) { ++ trap_gttmmio(vgpu, false); ++ map_aperture(vgpu, false); ++ } ++ ++ /** ++ * Currently we only do such reset when vGPU is not ++ * owned by any VM, so we simply restore entire cfg ++ * space to default value. ++ */ ++ intel_vgpu_init_cfg_space(vgpu, primary); ++} +diff --git a/drivers/gpu/drm/i915_legacy/gvt/cmd_parser.c b/drivers/gpu/drm/i915_legacy/gvt/cmd_parser.c +new file mode 100644 +index 000000000000..de5347725564 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/cmd_parser.c +@@ -0,0 +1,2998 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Ke Yu ++ * Kevin Tian ++ * Zhiyuan Lv ++ * ++ * Contributors: ++ * Min He ++ * Ping Gao ++ * Tina Zhang ++ * Yulei Zhang ++ * Zhi Wang ++ * ++ */ ++ ++#include ++#include "i915_drv.h" ++#include "gvt.h" ++#include "i915_pvinfo.h" ++#include "trace.h" ++ ++#define INVALID_OP (~0U) ++ ++#define OP_LEN_MI 9 ++#define OP_LEN_2D 10 ++#define OP_LEN_3D_MEDIA 16 ++#define OP_LEN_MFX_VC 16 ++#define OP_LEN_VEBOX 16 ++ ++#define CMD_TYPE(cmd) (((cmd) >> 29) & 7) ++ ++struct sub_op_bits { ++ int hi; ++ int low; ++}; ++struct decode_info { ++ const char *name; ++ int op_len; ++ int nr_sub_op; ++ const struct sub_op_bits *sub_op; ++}; ++ ++#define MAX_CMD_BUDGET 0x7fffffff ++#define MI_WAIT_FOR_PLANE_C_FLIP_PENDING (1<<15) ++#define MI_WAIT_FOR_PLANE_B_FLIP_PENDING (1<<9) ++#define MI_WAIT_FOR_PLANE_A_FLIP_PENDING (1<<1) ++ ++#define MI_WAIT_FOR_SPRITE_C_FLIP_PENDING (1<<20) ++#define MI_WAIT_FOR_SPRITE_B_FLIP_PENDING (1<<10) ++#define MI_WAIT_FOR_SPRITE_A_FLIP_PENDING (1<<2) ++ ++/* Render Command Map */ ++ ++/* MI_* command Opcode (28:23) */ ++#define OP_MI_NOOP 0x0 ++#define OP_MI_SET_PREDICATE 0x1 /* HSW+ */ ++#define OP_MI_USER_INTERRUPT 0x2 ++#define OP_MI_WAIT_FOR_EVENT 0x3 ++#define OP_MI_FLUSH 0x4 ++#define OP_MI_ARB_CHECK 0x5 ++#define OP_MI_RS_CONTROL 0x6 /* HSW+ */ ++#define OP_MI_REPORT_HEAD 0x7 ++#define OP_MI_ARB_ON_OFF 0x8 ++#define OP_MI_URB_ATOMIC_ALLOC 0x9 /* HSW+ */ ++#define OP_MI_BATCH_BUFFER_END 0xA ++#define OP_MI_SUSPEND_FLUSH 0xB ++#define OP_MI_PREDICATE 0xC /* IVB+ */ ++#define OP_MI_TOPOLOGY_FILTER 0xD /* IVB+ */ ++#define OP_MI_SET_APPID 0xE /* IVB+ */ ++#define OP_MI_RS_CONTEXT 0xF /* HSW+ */ ++#define OP_MI_LOAD_SCAN_LINES_INCL 0x12 /* HSW+ */ ++#define OP_MI_DISPLAY_FLIP 0x14 ++#define OP_MI_SEMAPHORE_MBOX 0x16 ++#define OP_MI_SET_CONTEXT 0x18 ++#define OP_MI_MATH 0x1A ++#define OP_MI_URB_CLEAR 0x19 ++#define OP_MI_SEMAPHORE_SIGNAL 0x1B /* BDW+ */ ++#define OP_MI_SEMAPHORE_WAIT 0x1C /* BDW+ */ ++ ++#define OP_MI_STORE_DATA_IMM 0x20 ++#define OP_MI_STORE_DATA_INDEX 0x21 ++#define OP_MI_LOAD_REGISTER_IMM 0x22 ++#define OP_MI_UPDATE_GTT 0x23 ++#define OP_MI_STORE_REGISTER_MEM 0x24 ++#define OP_MI_FLUSH_DW 0x26 ++#define OP_MI_CLFLUSH 0x27 ++#define OP_MI_REPORT_PERF_COUNT 0x28 ++#define OP_MI_LOAD_REGISTER_MEM 0x29 /* HSW+ */ ++#define OP_MI_LOAD_REGISTER_REG 0x2A /* HSW+ */ ++#define OP_MI_RS_STORE_DATA_IMM 0x2B /* HSW+ */ ++#define OP_MI_LOAD_URB_MEM 0x2C /* HSW+ */ ++#define OP_MI_STORE_URM_MEM 0x2D /* HSW+ */ ++#define OP_MI_2E 0x2E /* BDW+ */ ++#define OP_MI_2F 0x2F /* BDW+ */ ++#define OP_MI_BATCH_BUFFER_START 0x31 ++ ++/* Bit definition for dword 0 */ ++#define _CMDBIT_BB_START_IN_PPGTT (1UL << 8) ++ ++#define OP_MI_CONDITIONAL_BATCH_BUFFER_END 0x36 ++ ++#define BATCH_BUFFER_ADDR_MASK ((1UL << 32) - (1U << 2)) ++#define BATCH_BUFFER_ADDR_HIGH_MASK ((1UL << 16) - (1U)) ++#define BATCH_BUFFER_ADR_SPACE_BIT(x) (((x) >> 8) & 1U) ++#define BATCH_BUFFER_2ND_LEVEL_BIT(x) ((x) >> 22 & 1U) ++ ++/* 2D command: Opcode (28:22) */ ++#define OP_2D(x) ((2<<7) | x) ++ ++#define OP_XY_SETUP_BLT OP_2D(0x1) ++#define OP_XY_SETUP_CLIP_BLT OP_2D(0x3) ++#define OP_XY_SETUP_MONO_PATTERN_SL_BLT OP_2D(0x11) ++#define OP_XY_PIXEL_BLT OP_2D(0x24) ++#define OP_XY_SCANLINES_BLT OP_2D(0x25) ++#define OP_XY_TEXT_BLT OP_2D(0x26) ++#define OP_XY_TEXT_IMMEDIATE_BLT OP_2D(0x31) ++#define OP_XY_COLOR_BLT OP_2D(0x50) ++#define OP_XY_PAT_BLT OP_2D(0x51) ++#define OP_XY_MONO_PAT_BLT OP_2D(0x52) ++#define OP_XY_SRC_COPY_BLT OP_2D(0x53) ++#define OP_XY_MONO_SRC_COPY_BLT OP_2D(0x54) ++#define OP_XY_FULL_BLT OP_2D(0x55) ++#define OP_XY_FULL_MONO_SRC_BLT OP_2D(0x56) ++#define OP_XY_FULL_MONO_PATTERN_BLT OP_2D(0x57) ++#define OP_XY_FULL_MONO_PATTERN_MONO_SRC_BLT OP_2D(0x58) ++#define OP_XY_MONO_PAT_FIXED_BLT OP_2D(0x59) ++#define OP_XY_MONO_SRC_COPY_IMMEDIATE_BLT OP_2D(0x71) ++#define OP_XY_PAT_BLT_IMMEDIATE OP_2D(0x72) ++#define OP_XY_SRC_COPY_CHROMA_BLT OP_2D(0x73) ++#define OP_XY_FULL_IMMEDIATE_PATTERN_BLT OP_2D(0x74) ++#define OP_XY_FULL_MONO_SRC_IMMEDIATE_PATTERN_BLT OP_2D(0x75) ++#define OP_XY_PAT_CHROMA_BLT OP_2D(0x76) ++#define OP_XY_PAT_CHROMA_BLT_IMMEDIATE OP_2D(0x77) ++ ++/* 3D/Media Command: Pipeline Type(28:27) Opcode(26:24) Sub Opcode(23:16) */ ++#define OP_3D_MEDIA(sub_type, opcode, sub_opcode) \ ++ ((3 << 13) | ((sub_type) << 11) | ((opcode) << 8) | (sub_opcode)) ++ ++#define OP_STATE_PREFETCH OP_3D_MEDIA(0x0, 0x0, 0x03) ++ ++#define OP_STATE_BASE_ADDRESS OP_3D_MEDIA(0x0, 0x1, 0x01) ++#define OP_STATE_SIP OP_3D_MEDIA(0x0, 0x1, 0x02) ++#define OP_3D_MEDIA_0_1_4 OP_3D_MEDIA(0x0, 0x1, 0x04) ++ ++#define OP_3DSTATE_VF_STATISTICS_GM45 OP_3D_MEDIA(0x1, 0x0, 0x0B) ++ ++#define OP_PIPELINE_SELECT OP_3D_MEDIA(0x1, 0x1, 0x04) ++ ++#define OP_MEDIA_VFE_STATE OP_3D_MEDIA(0x2, 0x0, 0x0) ++#define OP_MEDIA_CURBE_LOAD OP_3D_MEDIA(0x2, 0x0, 0x1) ++#define OP_MEDIA_INTERFACE_DESCRIPTOR_LOAD OP_3D_MEDIA(0x2, 0x0, 0x2) ++#define OP_MEDIA_GATEWAY_STATE OP_3D_MEDIA(0x2, 0x0, 0x3) ++#define OP_MEDIA_STATE_FLUSH OP_3D_MEDIA(0x2, 0x0, 0x4) ++#define OP_MEDIA_POOL_STATE OP_3D_MEDIA(0x2, 0x0, 0x5) ++ ++#define OP_MEDIA_OBJECT OP_3D_MEDIA(0x2, 0x1, 0x0) ++#define OP_MEDIA_OBJECT_PRT OP_3D_MEDIA(0x2, 0x1, 0x2) ++#define OP_MEDIA_OBJECT_WALKER OP_3D_MEDIA(0x2, 0x1, 0x3) ++#define OP_GPGPU_WALKER OP_3D_MEDIA(0x2, 0x1, 0x5) ++ ++#define OP_3DSTATE_CLEAR_PARAMS OP_3D_MEDIA(0x3, 0x0, 0x04) /* IVB+ */ ++#define OP_3DSTATE_DEPTH_BUFFER OP_3D_MEDIA(0x3, 0x0, 0x05) /* IVB+ */ ++#define OP_3DSTATE_STENCIL_BUFFER OP_3D_MEDIA(0x3, 0x0, 0x06) /* IVB+ */ ++#define OP_3DSTATE_HIER_DEPTH_BUFFER OP_3D_MEDIA(0x3, 0x0, 0x07) /* IVB+ */ ++#define OP_3DSTATE_VERTEX_BUFFERS OP_3D_MEDIA(0x3, 0x0, 0x08) ++#define OP_3DSTATE_VERTEX_ELEMENTS OP_3D_MEDIA(0x3, 0x0, 0x09) ++#define OP_3DSTATE_INDEX_BUFFER OP_3D_MEDIA(0x3, 0x0, 0x0A) ++#define OP_3DSTATE_VF_STATISTICS OP_3D_MEDIA(0x3, 0x0, 0x0B) ++#define OP_3DSTATE_VF OP_3D_MEDIA(0x3, 0x0, 0x0C) /* HSW+ */ ++#define OP_3DSTATE_CC_STATE_POINTERS OP_3D_MEDIA(0x3, 0x0, 0x0E) ++#define OP_3DSTATE_SCISSOR_STATE_POINTERS OP_3D_MEDIA(0x3, 0x0, 0x0F) ++#define OP_3DSTATE_VS OP_3D_MEDIA(0x3, 0x0, 0x10) ++#define OP_3DSTATE_GS OP_3D_MEDIA(0x3, 0x0, 0x11) ++#define OP_3DSTATE_CLIP OP_3D_MEDIA(0x3, 0x0, 0x12) ++#define OP_3DSTATE_SF OP_3D_MEDIA(0x3, 0x0, 0x13) ++#define OP_3DSTATE_WM OP_3D_MEDIA(0x3, 0x0, 0x14) ++#define OP_3DSTATE_CONSTANT_VS OP_3D_MEDIA(0x3, 0x0, 0x15) ++#define OP_3DSTATE_CONSTANT_GS OP_3D_MEDIA(0x3, 0x0, 0x16) ++#define OP_3DSTATE_CONSTANT_PS OP_3D_MEDIA(0x3, 0x0, 0x17) ++#define OP_3DSTATE_SAMPLE_MASK OP_3D_MEDIA(0x3, 0x0, 0x18) ++#define OP_3DSTATE_CONSTANT_HS OP_3D_MEDIA(0x3, 0x0, 0x19) /* IVB+ */ ++#define OP_3DSTATE_CONSTANT_DS OP_3D_MEDIA(0x3, 0x0, 0x1A) /* IVB+ */ ++#define OP_3DSTATE_HS OP_3D_MEDIA(0x3, 0x0, 0x1B) /* IVB+ */ ++#define OP_3DSTATE_TE OP_3D_MEDIA(0x3, 0x0, 0x1C) /* IVB+ */ ++#define OP_3DSTATE_DS OP_3D_MEDIA(0x3, 0x0, 0x1D) /* IVB+ */ ++#define OP_3DSTATE_STREAMOUT OP_3D_MEDIA(0x3, 0x0, 0x1E) /* IVB+ */ ++#define OP_3DSTATE_SBE OP_3D_MEDIA(0x3, 0x0, 0x1F) /* IVB+ */ ++#define OP_3DSTATE_PS OP_3D_MEDIA(0x3, 0x0, 0x20) /* IVB+ */ ++#define OP_3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP OP_3D_MEDIA(0x3, 0x0, 0x21) /* IVB+ */ ++#define OP_3DSTATE_VIEWPORT_STATE_POINTERS_CC OP_3D_MEDIA(0x3, 0x0, 0x23) /* IVB+ */ ++#define OP_3DSTATE_BLEND_STATE_POINTERS OP_3D_MEDIA(0x3, 0x0, 0x24) /* IVB+ */ ++#define OP_3DSTATE_DEPTH_STENCIL_STATE_POINTERS OP_3D_MEDIA(0x3, 0x0, 0x25) /* IVB+ */ ++#define OP_3DSTATE_BINDING_TABLE_POINTERS_VS OP_3D_MEDIA(0x3, 0x0, 0x26) /* IVB+ */ ++#define OP_3DSTATE_BINDING_TABLE_POINTERS_HS OP_3D_MEDIA(0x3, 0x0, 0x27) /* IVB+ */ ++#define OP_3DSTATE_BINDING_TABLE_POINTERS_DS OP_3D_MEDIA(0x3, 0x0, 0x28) /* IVB+ */ ++#define OP_3DSTATE_BINDING_TABLE_POINTERS_GS OP_3D_MEDIA(0x3, 0x0, 0x29) /* IVB+ */ ++#define OP_3DSTATE_BINDING_TABLE_POINTERS_PS OP_3D_MEDIA(0x3, 0x0, 0x2A) /* IVB+ */ ++#define OP_3DSTATE_SAMPLER_STATE_POINTERS_VS OP_3D_MEDIA(0x3, 0x0, 0x2B) /* IVB+ */ ++#define OP_3DSTATE_SAMPLER_STATE_POINTERS_HS OP_3D_MEDIA(0x3, 0x0, 0x2C) /* IVB+ */ ++#define OP_3DSTATE_SAMPLER_STATE_POINTERS_DS OP_3D_MEDIA(0x3, 0x0, 0x2D) /* IVB+ */ ++#define OP_3DSTATE_SAMPLER_STATE_POINTERS_GS OP_3D_MEDIA(0x3, 0x0, 0x2E) /* IVB+ */ ++#define OP_3DSTATE_SAMPLER_STATE_POINTERS_PS OP_3D_MEDIA(0x3, 0x0, 0x2F) /* IVB+ */ ++#define OP_3DSTATE_URB_VS OP_3D_MEDIA(0x3, 0x0, 0x30) /* IVB+ */ ++#define OP_3DSTATE_URB_HS OP_3D_MEDIA(0x3, 0x0, 0x31) /* IVB+ */ ++#define OP_3DSTATE_URB_DS OP_3D_MEDIA(0x3, 0x0, 0x32) /* IVB+ */ ++#define OP_3DSTATE_URB_GS OP_3D_MEDIA(0x3, 0x0, 0x33) /* IVB+ */ ++#define OP_3DSTATE_GATHER_CONSTANT_VS OP_3D_MEDIA(0x3, 0x0, 0x34) /* HSW+ */ ++#define OP_3DSTATE_GATHER_CONSTANT_GS OP_3D_MEDIA(0x3, 0x0, 0x35) /* HSW+ */ ++#define OP_3DSTATE_GATHER_CONSTANT_HS OP_3D_MEDIA(0x3, 0x0, 0x36) /* HSW+ */ ++#define OP_3DSTATE_GATHER_CONSTANT_DS OP_3D_MEDIA(0x3, 0x0, 0x37) /* HSW+ */ ++#define OP_3DSTATE_GATHER_CONSTANT_PS OP_3D_MEDIA(0x3, 0x0, 0x38) /* HSW+ */ ++#define OP_3DSTATE_DX9_CONSTANTF_VS OP_3D_MEDIA(0x3, 0x0, 0x39) /* HSW+ */ ++#define OP_3DSTATE_DX9_CONSTANTF_PS OP_3D_MEDIA(0x3, 0x0, 0x3A) /* HSW+ */ ++#define OP_3DSTATE_DX9_CONSTANTI_VS OP_3D_MEDIA(0x3, 0x0, 0x3B) /* HSW+ */ ++#define OP_3DSTATE_DX9_CONSTANTI_PS OP_3D_MEDIA(0x3, 0x0, 0x3C) /* HSW+ */ ++#define OP_3DSTATE_DX9_CONSTANTB_VS OP_3D_MEDIA(0x3, 0x0, 0x3D) /* HSW+ */ ++#define OP_3DSTATE_DX9_CONSTANTB_PS OP_3D_MEDIA(0x3, 0x0, 0x3E) /* HSW+ */ ++#define OP_3DSTATE_DX9_LOCAL_VALID_VS OP_3D_MEDIA(0x3, 0x0, 0x3F) /* HSW+ */ ++#define OP_3DSTATE_DX9_LOCAL_VALID_PS OP_3D_MEDIA(0x3, 0x0, 0x40) /* HSW+ */ ++#define OP_3DSTATE_DX9_GENERATE_ACTIVE_VS OP_3D_MEDIA(0x3, 0x0, 0x41) /* HSW+ */ ++#define OP_3DSTATE_DX9_GENERATE_ACTIVE_PS OP_3D_MEDIA(0x3, 0x0, 0x42) /* HSW+ */ ++#define OP_3DSTATE_BINDING_TABLE_EDIT_VS OP_3D_MEDIA(0x3, 0x0, 0x43) /* HSW+ */ ++#define OP_3DSTATE_BINDING_TABLE_EDIT_GS OP_3D_MEDIA(0x3, 0x0, 0x44) /* HSW+ */ ++#define OP_3DSTATE_BINDING_TABLE_EDIT_HS OP_3D_MEDIA(0x3, 0x0, 0x45) /* HSW+ */ ++#define OP_3DSTATE_BINDING_TABLE_EDIT_DS OP_3D_MEDIA(0x3, 0x0, 0x46) /* HSW+ */ ++#define OP_3DSTATE_BINDING_TABLE_EDIT_PS OP_3D_MEDIA(0x3, 0x0, 0x47) /* HSW+ */ ++ ++#define OP_3DSTATE_VF_INSTANCING OP_3D_MEDIA(0x3, 0x0, 0x49) /* BDW+ */ ++#define OP_3DSTATE_VF_SGVS OP_3D_MEDIA(0x3, 0x0, 0x4A) /* BDW+ */ ++#define OP_3DSTATE_VF_TOPOLOGY OP_3D_MEDIA(0x3, 0x0, 0x4B) /* BDW+ */ ++#define OP_3DSTATE_WM_CHROMAKEY OP_3D_MEDIA(0x3, 0x0, 0x4C) /* BDW+ */ ++#define OP_3DSTATE_PS_BLEND OP_3D_MEDIA(0x3, 0x0, 0x4D) /* BDW+ */ ++#define OP_3DSTATE_WM_DEPTH_STENCIL OP_3D_MEDIA(0x3, 0x0, 0x4E) /* BDW+ */ ++#define OP_3DSTATE_PS_EXTRA OP_3D_MEDIA(0x3, 0x0, 0x4F) /* BDW+ */ ++#define OP_3DSTATE_RASTER OP_3D_MEDIA(0x3, 0x0, 0x50) /* BDW+ */ ++#define OP_3DSTATE_SBE_SWIZ OP_3D_MEDIA(0x3, 0x0, 0x51) /* BDW+ */ ++#define OP_3DSTATE_WM_HZ_OP OP_3D_MEDIA(0x3, 0x0, 0x52) /* BDW+ */ ++#define OP_3DSTATE_COMPONENT_PACKING OP_3D_MEDIA(0x3, 0x0, 0x55) /* SKL+ */ ++ ++#define OP_3DSTATE_DRAWING_RECTANGLE OP_3D_MEDIA(0x3, 0x1, 0x00) ++#define OP_3DSTATE_SAMPLER_PALETTE_LOAD0 OP_3D_MEDIA(0x3, 0x1, 0x02) ++#define OP_3DSTATE_CHROMA_KEY OP_3D_MEDIA(0x3, 0x1, 0x04) ++#define OP_SNB_3DSTATE_DEPTH_BUFFER OP_3D_MEDIA(0x3, 0x1, 0x05) ++#define OP_3DSTATE_POLY_STIPPLE_OFFSET OP_3D_MEDIA(0x3, 0x1, 0x06) ++#define OP_3DSTATE_POLY_STIPPLE_PATTERN OP_3D_MEDIA(0x3, 0x1, 0x07) ++#define OP_3DSTATE_LINE_STIPPLE OP_3D_MEDIA(0x3, 0x1, 0x08) ++#define OP_3DSTATE_AA_LINE_PARAMS OP_3D_MEDIA(0x3, 0x1, 0x0A) ++#define OP_3DSTATE_GS_SVB_INDEX OP_3D_MEDIA(0x3, 0x1, 0x0B) ++#define OP_3DSTATE_SAMPLER_PALETTE_LOAD1 OP_3D_MEDIA(0x3, 0x1, 0x0C) ++#define OP_3DSTATE_MULTISAMPLE_BDW OP_3D_MEDIA(0x3, 0x0, 0x0D) ++#define OP_SNB_3DSTATE_STENCIL_BUFFER OP_3D_MEDIA(0x3, 0x1, 0x0E) ++#define OP_SNB_3DSTATE_HIER_DEPTH_BUFFER OP_3D_MEDIA(0x3, 0x1, 0x0F) ++#define OP_SNB_3DSTATE_CLEAR_PARAMS OP_3D_MEDIA(0x3, 0x1, 0x10) ++#define OP_3DSTATE_MONOFILTER_SIZE OP_3D_MEDIA(0x3, 0x1, 0x11) ++#define OP_3DSTATE_PUSH_CONSTANT_ALLOC_VS OP_3D_MEDIA(0x3, 0x1, 0x12) /* IVB+ */ ++#define OP_3DSTATE_PUSH_CONSTANT_ALLOC_HS OP_3D_MEDIA(0x3, 0x1, 0x13) /* IVB+ */ ++#define OP_3DSTATE_PUSH_CONSTANT_ALLOC_DS OP_3D_MEDIA(0x3, 0x1, 0x14) /* IVB+ */ ++#define OP_3DSTATE_PUSH_CONSTANT_ALLOC_GS OP_3D_MEDIA(0x3, 0x1, 0x15) /* IVB+ */ ++#define OP_3DSTATE_PUSH_CONSTANT_ALLOC_PS OP_3D_MEDIA(0x3, 0x1, 0x16) /* IVB+ */ ++#define OP_3DSTATE_SO_DECL_LIST OP_3D_MEDIA(0x3, 0x1, 0x17) ++#define OP_3DSTATE_SO_BUFFER OP_3D_MEDIA(0x3, 0x1, 0x18) ++#define OP_3DSTATE_BINDING_TABLE_POOL_ALLOC OP_3D_MEDIA(0x3, 0x1, 0x19) /* HSW+ */ ++#define OP_3DSTATE_GATHER_POOL_ALLOC OP_3D_MEDIA(0x3, 0x1, 0x1A) /* HSW+ */ ++#define OP_3DSTATE_DX9_CONSTANT_BUFFER_POOL_ALLOC OP_3D_MEDIA(0x3, 0x1, 0x1B) /* HSW+ */ ++#define OP_3DSTATE_SAMPLE_PATTERN OP_3D_MEDIA(0x3, 0x1, 0x1C) ++#define OP_PIPE_CONTROL OP_3D_MEDIA(0x3, 0x2, 0x00) ++#define OP_3DPRIMITIVE OP_3D_MEDIA(0x3, 0x3, 0x00) ++ ++/* VCCP Command Parser */ ++ ++/* ++ * Below MFX and VBE cmd definition is from vaapi intel driver project (BSD License) ++ * git://anongit.freedesktop.org/vaapi/intel-driver ++ * src/i965_defines.h ++ * ++ */ ++ ++#define OP_MFX(pipeline, op, sub_opa, sub_opb) \ ++ (3 << 13 | \ ++ (pipeline) << 11 | \ ++ (op) << 8 | \ ++ (sub_opa) << 5 | \ ++ (sub_opb)) ++ ++#define OP_MFX_PIPE_MODE_SELECT OP_MFX(2, 0, 0, 0) /* ALL */ ++#define OP_MFX_SURFACE_STATE OP_MFX(2, 0, 0, 1) /* ALL */ ++#define OP_MFX_PIPE_BUF_ADDR_STATE OP_MFX(2, 0, 0, 2) /* ALL */ ++#define OP_MFX_IND_OBJ_BASE_ADDR_STATE OP_MFX(2, 0, 0, 3) /* ALL */ ++#define OP_MFX_BSP_BUF_BASE_ADDR_STATE OP_MFX(2, 0, 0, 4) /* ALL */ ++#define OP_2_0_0_5 OP_MFX(2, 0, 0, 5) /* ALL */ ++#define OP_MFX_STATE_POINTER OP_MFX(2, 0, 0, 6) /* ALL */ ++#define OP_MFX_QM_STATE OP_MFX(2, 0, 0, 7) /* IVB+ */ ++#define OP_MFX_FQM_STATE OP_MFX(2, 0, 0, 8) /* IVB+ */ ++#define OP_MFX_PAK_INSERT_OBJECT OP_MFX(2, 0, 2, 8) /* IVB+ */ ++#define OP_MFX_STITCH_OBJECT OP_MFX(2, 0, 2, 0xA) /* IVB+ */ ++ ++#define OP_MFD_IT_OBJECT OP_MFX(2, 0, 1, 9) /* ALL */ ++ ++#define OP_MFX_WAIT OP_MFX(1, 0, 0, 0) /* IVB+ */ ++#define OP_MFX_AVC_IMG_STATE OP_MFX(2, 1, 0, 0) /* ALL */ ++#define OP_MFX_AVC_QM_STATE OP_MFX(2, 1, 0, 1) /* ALL */ ++#define OP_MFX_AVC_DIRECTMODE_STATE OP_MFX(2, 1, 0, 2) /* ALL */ ++#define OP_MFX_AVC_SLICE_STATE OP_MFX(2, 1, 0, 3) /* ALL */ ++#define OP_MFX_AVC_REF_IDX_STATE OP_MFX(2, 1, 0, 4) /* ALL */ ++#define OP_MFX_AVC_WEIGHTOFFSET_STATE OP_MFX(2, 1, 0, 5) /* ALL */ ++#define OP_MFD_AVC_PICID_STATE OP_MFX(2, 1, 1, 5) /* HSW+ */ ++#define OP_MFD_AVC_DPB_STATE OP_MFX(2, 1, 1, 6) /* IVB+ */ ++#define OP_MFD_AVC_SLICEADDR OP_MFX(2, 1, 1, 7) /* IVB+ */ ++#define OP_MFD_AVC_BSD_OBJECT OP_MFX(2, 1, 1, 8) /* ALL */ ++#define OP_MFC_AVC_PAK_OBJECT OP_MFX(2, 1, 2, 9) /* ALL */ ++ ++#define OP_MFX_VC1_PRED_PIPE_STATE OP_MFX(2, 2, 0, 1) /* ALL */ ++#define OP_MFX_VC1_DIRECTMODE_STATE OP_MFX(2, 2, 0, 2) /* ALL */ ++#define OP_MFD_VC1_SHORT_PIC_STATE OP_MFX(2, 2, 1, 0) /* IVB+ */ ++#define OP_MFD_VC1_LONG_PIC_STATE OP_MFX(2, 2, 1, 1) /* IVB+ */ ++#define OP_MFD_VC1_BSD_OBJECT OP_MFX(2, 2, 1, 8) /* ALL */ ++ ++#define OP_MFX_MPEG2_PIC_STATE OP_MFX(2, 3, 0, 0) /* ALL */ ++#define OP_MFX_MPEG2_QM_STATE OP_MFX(2, 3, 0, 1) /* ALL */ ++#define OP_MFD_MPEG2_BSD_OBJECT OP_MFX(2, 3, 1, 8) /* ALL */ ++#define OP_MFC_MPEG2_SLICEGROUP_STATE OP_MFX(2, 3, 2, 3) /* ALL */ ++#define OP_MFC_MPEG2_PAK_OBJECT OP_MFX(2, 3, 2, 9) /* ALL */ ++ ++#define OP_MFX_2_6_0_0 OP_MFX(2, 6, 0, 0) /* IVB+ */ ++#define OP_MFX_2_6_0_8 OP_MFX(2, 6, 0, 8) /* IVB+ */ ++#define OP_MFX_2_6_0_9 OP_MFX(2, 6, 0, 9) /* IVB+ */ ++ ++#define OP_MFX_JPEG_PIC_STATE OP_MFX(2, 7, 0, 0) ++#define OP_MFX_JPEG_HUFF_TABLE_STATE OP_MFX(2, 7, 0, 2) ++#define OP_MFD_JPEG_BSD_OBJECT OP_MFX(2, 7, 1, 8) ++ ++#define OP_VEB(pipeline, op, sub_opa, sub_opb) \ ++ (3 << 13 | \ ++ (pipeline) << 11 | \ ++ (op) << 8 | \ ++ (sub_opa) << 5 | \ ++ (sub_opb)) ++ ++#define OP_VEB_SURFACE_STATE OP_VEB(2, 4, 0, 0) ++#define OP_VEB_STATE OP_VEB(2, 4, 0, 2) ++#define OP_VEB_DNDI_IECP_STATE OP_VEB(2, 4, 0, 3) ++ ++struct parser_exec_state; ++ ++typedef int (*parser_cmd_handler)(struct parser_exec_state *s); ++ ++#define GVT_CMD_HASH_BITS 7 ++ ++/* which DWords need address fix */ ++#define ADDR_FIX_1(x1) (1 << (x1)) ++#define ADDR_FIX_2(x1, x2) (ADDR_FIX_1(x1) | ADDR_FIX_1(x2)) ++#define ADDR_FIX_3(x1, x2, x3) (ADDR_FIX_1(x1) | ADDR_FIX_2(x2, x3)) ++#define ADDR_FIX_4(x1, x2, x3, x4) (ADDR_FIX_1(x1) | ADDR_FIX_3(x2, x3, x4)) ++#define ADDR_FIX_5(x1, x2, x3, x4, x5) (ADDR_FIX_1(x1) | ADDR_FIX_4(x2, x3, x4, x5)) ++ ++struct cmd_info { ++ const char *name; ++ u32 opcode; ++ ++#define F_LEN_MASK (1U<<0) ++#define F_LEN_CONST 1U ++#define F_LEN_VAR 0U ++ ++/* ++ * command has its own ip advance logic ++ * e.g. MI_BATCH_START, MI_BATCH_END ++ */ ++#define F_IP_ADVANCE_CUSTOM (1<<1) ++ ++#define F_POST_HANDLE (1<<2) ++ u32 flag; ++ ++#define R_RCS BIT(RCS0) ++#define R_VCS1 BIT(VCS0) ++#define R_VCS2 BIT(VCS1) ++#define R_VCS (R_VCS1 | R_VCS2) ++#define R_BCS BIT(BCS0) ++#define R_VECS BIT(VECS0) ++#define R_ALL (R_RCS | R_VCS | R_BCS | R_VECS) ++ /* rings that support this cmd: BLT/RCS/VCS/VECS */ ++ u16 rings; ++ ++ /* devices that support this cmd: SNB/IVB/HSW/... */ ++ u16 devices; ++ ++ /* which DWords are address that need fix up. ++ * bit 0 means a 32-bit non address operand in command ++ * bit 1 means address operand, which could be 32-bit ++ * or 64-bit depending on different architectures.( ++ * defined by "gmadr_bytes_in_cmd" in intel_gvt. ++ * No matter the address length, each address only takes ++ * one bit in the bitmap. ++ */ ++ u16 addr_bitmap; ++ ++ /* flag == F_LEN_CONST : command length ++ * flag == F_LEN_VAR : length bias bits ++ * Note: length is in DWord ++ */ ++ u8 len; ++ ++ parser_cmd_handler handler; ++}; ++ ++struct cmd_entry { ++ struct hlist_node hlist; ++ const struct cmd_info *info; ++}; ++ ++enum { ++ RING_BUFFER_INSTRUCTION, ++ BATCH_BUFFER_INSTRUCTION, ++ BATCH_BUFFER_2ND_LEVEL, ++}; ++ ++enum { ++ GTT_BUFFER, ++ PPGTT_BUFFER ++}; ++ ++struct parser_exec_state { ++ struct intel_vgpu *vgpu; ++ int ring_id; ++ ++ int buf_type; ++ ++ /* batch buffer address type */ ++ int buf_addr_type; ++ ++ /* graphics memory address of ring buffer start */ ++ unsigned long ring_start; ++ unsigned long ring_size; ++ unsigned long ring_head; ++ unsigned long ring_tail; ++ ++ /* instruction graphics memory address */ ++ unsigned long ip_gma; ++ ++ /* mapped va of the instr_gma */ ++ void *ip_va; ++ void *rb_va; ++ ++ void *ret_bb_va; ++ /* next instruction when return from batch buffer to ring buffer */ ++ unsigned long ret_ip_gma_ring; ++ ++ /* next instruction when return from 2nd batch buffer to batch buffer */ ++ unsigned long ret_ip_gma_bb; ++ ++ /* batch buffer address type (GTT or PPGTT) ++ * used when ret from 2nd level batch buffer ++ */ ++ int saved_buf_addr_type; ++ bool is_ctx_wa; ++ ++ const struct cmd_info *info; ++ ++ struct intel_vgpu_workload *workload; ++}; ++ ++#define gmadr_dw_number(s) \ ++ (s->vgpu->gvt->device_info.gmadr_bytes_in_cmd >> 2) ++ ++static unsigned long bypass_scan_mask = 0; ++ ++/* ring ALL, type = 0 */ ++static const struct sub_op_bits sub_op_mi[] = { ++ {31, 29}, ++ {28, 23}, ++}; ++ ++static const struct decode_info decode_info_mi = { ++ "MI", ++ OP_LEN_MI, ++ ARRAY_SIZE(sub_op_mi), ++ sub_op_mi, ++}; ++ ++/* ring RCS, command type 2 */ ++static const struct sub_op_bits sub_op_2d[] = { ++ {31, 29}, ++ {28, 22}, ++}; ++ ++static const struct decode_info decode_info_2d = { ++ "2D", ++ OP_LEN_2D, ++ ARRAY_SIZE(sub_op_2d), ++ sub_op_2d, ++}; ++ ++/* ring RCS, command type 3 */ ++static const struct sub_op_bits sub_op_3d_media[] = { ++ {31, 29}, ++ {28, 27}, ++ {26, 24}, ++ {23, 16}, ++}; ++ ++static const struct decode_info decode_info_3d_media = { ++ "3D_Media", ++ OP_LEN_3D_MEDIA, ++ ARRAY_SIZE(sub_op_3d_media), ++ sub_op_3d_media, ++}; ++ ++/* ring VCS, command type 3 */ ++static const struct sub_op_bits sub_op_mfx_vc[] = { ++ {31, 29}, ++ {28, 27}, ++ {26, 24}, ++ {23, 21}, ++ {20, 16}, ++}; ++ ++static const struct decode_info decode_info_mfx_vc = { ++ "MFX_VC", ++ OP_LEN_MFX_VC, ++ ARRAY_SIZE(sub_op_mfx_vc), ++ sub_op_mfx_vc, ++}; ++ ++/* ring VECS, command type 3 */ ++static const struct sub_op_bits sub_op_vebox[] = { ++ {31, 29}, ++ {28, 27}, ++ {26, 24}, ++ {23, 21}, ++ {20, 16}, ++}; ++ ++static const struct decode_info decode_info_vebox = { ++ "VEBOX", ++ OP_LEN_VEBOX, ++ ARRAY_SIZE(sub_op_vebox), ++ sub_op_vebox, ++}; ++ ++static const struct decode_info *ring_decode_info[I915_NUM_ENGINES][8] = { ++ [RCS0] = { ++ &decode_info_mi, ++ NULL, ++ NULL, ++ &decode_info_3d_media, ++ NULL, ++ NULL, ++ NULL, ++ NULL, ++ }, ++ ++ [VCS0] = { ++ &decode_info_mi, ++ NULL, ++ NULL, ++ &decode_info_mfx_vc, ++ NULL, ++ NULL, ++ NULL, ++ NULL, ++ }, ++ ++ [BCS0] = { ++ &decode_info_mi, ++ NULL, ++ &decode_info_2d, ++ NULL, ++ NULL, ++ NULL, ++ NULL, ++ NULL, ++ }, ++ ++ [VECS0] = { ++ &decode_info_mi, ++ NULL, ++ NULL, ++ &decode_info_vebox, ++ NULL, ++ NULL, ++ NULL, ++ NULL, ++ }, ++ ++ [VCS1] = { ++ &decode_info_mi, ++ NULL, ++ NULL, ++ &decode_info_mfx_vc, ++ NULL, ++ NULL, ++ NULL, ++ NULL, ++ }, ++}; ++ ++static inline u32 get_opcode(u32 cmd, int ring_id) ++{ ++ const struct decode_info *d_info; ++ ++ d_info = ring_decode_info[ring_id][CMD_TYPE(cmd)]; ++ if (d_info == NULL) ++ return INVALID_OP; ++ ++ return cmd >> (32 - d_info->op_len); ++} ++ ++static inline const struct cmd_info *find_cmd_entry(struct intel_gvt *gvt, ++ unsigned int opcode, int ring_id) ++{ ++ struct cmd_entry *e; ++ ++ hash_for_each_possible(gvt->cmd_table, e, hlist, opcode) { ++ if (opcode == e->info->opcode && e->info->rings & BIT(ring_id)) ++ return e->info; ++ } ++ return NULL; ++} ++ ++static inline const struct cmd_info *get_cmd_info(struct intel_gvt *gvt, ++ u32 cmd, int ring_id) ++{ ++ u32 opcode; ++ ++ opcode = get_opcode(cmd, ring_id); ++ if (opcode == INVALID_OP) ++ return NULL; ++ ++ return find_cmd_entry(gvt, opcode, ring_id); ++} ++ ++static inline u32 sub_op_val(u32 cmd, u32 hi, u32 low) ++{ ++ return (cmd >> low) & ((1U << (hi - low + 1)) - 1); ++} ++ ++static inline void print_opcode(u32 cmd, int ring_id) ++{ ++ const struct decode_info *d_info; ++ int i; ++ ++ d_info = ring_decode_info[ring_id][CMD_TYPE(cmd)]; ++ if (d_info == NULL) ++ return; ++ ++ gvt_dbg_cmd("opcode=0x%x %s sub_ops:", ++ cmd >> (32 - d_info->op_len), d_info->name); ++ ++ for (i = 0; i < d_info->nr_sub_op; i++) ++ pr_err("0x%x ", sub_op_val(cmd, d_info->sub_op[i].hi, ++ d_info->sub_op[i].low)); ++ ++ pr_err("\n"); ++} ++ ++static inline u32 *cmd_ptr(struct parser_exec_state *s, int index) ++{ ++ return s->ip_va + (index << 2); ++} ++ ++static inline u32 cmd_val(struct parser_exec_state *s, int index) ++{ ++ return *cmd_ptr(s, index); ++} ++ ++static void parser_exec_state_dump(struct parser_exec_state *s) ++{ ++ int cnt = 0; ++ int i; ++ ++ gvt_dbg_cmd(" vgpu%d RING%d: ring_start(%08lx) ring_end(%08lx)" ++ " ring_head(%08lx) ring_tail(%08lx)\n", s->vgpu->id, ++ s->ring_id, s->ring_start, s->ring_start + s->ring_size, ++ s->ring_head, s->ring_tail); ++ ++ gvt_dbg_cmd(" %s %s ip_gma(%08lx) ", ++ s->buf_type == RING_BUFFER_INSTRUCTION ? ++ "RING_BUFFER" : "BATCH_BUFFER", ++ s->buf_addr_type == GTT_BUFFER ? ++ "GTT" : "PPGTT", s->ip_gma); ++ ++ if (s->ip_va == NULL) { ++ gvt_dbg_cmd(" ip_va(NULL)"); ++ return; ++ } ++ ++ gvt_dbg_cmd(" ip_va=%p: %08x %08x %08x %08x\n", ++ s->ip_va, cmd_val(s, 0), cmd_val(s, 1), ++ cmd_val(s, 2), cmd_val(s, 3)); ++ ++ print_opcode(cmd_val(s, 0), s->ring_id); ++ ++ s->ip_va = (u32 *)((((u64)s->ip_va) >> 12) << 12); ++ ++ while (cnt < 1024) { ++ gvt_dbg_cmd("ip_va=%p: ", s->ip_va); ++ for (i = 0; i < 8; i++) ++ gvt_dbg_cmd("%08x ", cmd_val(s, i)); ++ gvt_dbg_cmd("\n"); ++ ++ s->ip_va += 8 * sizeof(u32); ++ cnt += 8; ++ } ++} ++ ++static inline void update_ip_va(struct parser_exec_state *s) ++{ ++ unsigned long len = 0; ++ ++ if (WARN_ON(s->ring_head == s->ring_tail)) ++ return; ++ ++ if (s->buf_type == RING_BUFFER_INSTRUCTION) { ++ unsigned long ring_top = s->ring_start + s->ring_size; ++ ++ if (s->ring_head > s->ring_tail) { ++ if (s->ip_gma >= s->ring_head && s->ip_gma < ring_top) ++ len = (s->ip_gma - s->ring_head); ++ else if (s->ip_gma >= s->ring_start && ++ s->ip_gma <= s->ring_tail) ++ len = (ring_top - s->ring_head) + ++ (s->ip_gma - s->ring_start); ++ } else ++ len = (s->ip_gma - s->ring_head); ++ ++ s->ip_va = s->rb_va + len; ++ } else {/* shadow batch buffer */ ++ s->ip_va = s->ret_bb_va; ++ } ++} ++ ++static inline int ip_gma_set(struct parser_exec_state *s, ++ unsigned long ip_gma) ++{ ++ WARN_ON(!IS_ALIGNED(ip_gma, 4)); ++ ++ s->ip_gma = ip_gma; ++ update_ip_va(s); ++ return 0; ++} ++ ++static inline int ip_gma_advance(struct parser_exec_state *s, ++ unsigned int dw_len) ++{ ++ s->ip_gma += (dw_len << 2); ++ ++ if (s->buf_type == RING_BUFFER_INSTRUCTION) { ++ if (s->ip_gma >= s->ring_start + s->ring_size) ++ s->ip_gma -= s->ring_size; ++ update_ip_va(s); ++ } else { ++ s->ip_va += (dw_len << 2); ++ } ++ ++ return 0; ++} ++ ++static inline int get_cmd_length(const struct cmd_info *info, u32 cmd) ++{ ++ if ((info->flag & F_LEN_MASK) == F_LEN_CONST) ++ return info->len; ++ else ++ return (cmd & ((1U << info->len) - 1)) + 2; ++ return 0; ++} ++ ++static inline int cmd_length(struct parser_exec_state *s) ++{ ++ return get_cmd_length(s->info, cmd_val(s, 0)); ++} ++ ++/* do not remove this, some platform may need clflush here */ ++#define patch_value(s, addr, val) do { \ ++ *addr = val; \ ++} while (0) ++ ++static bool is_shadowed_mmio(unsigned int offset) ++{ ++ bool ret = false; ++ ++ if ((offset == 0x2168) || /*BB current head register UDW */ ++ (offset == 0x2140) || /*BB current header register */ ++ (offset == 0x211c) || /*second BB header register UDW */ ++ (offset == 0x2114)) { /*second BB header register UDW */ ++ ret = true; ++ } ++ return ret; ++} ++ ++static inline bool is_force_nonpriv_mmio(unsigned int offset) ++{ ++ return (offset >= 0x24d0 && offset < 0x2500); ++} ++ ++static int force_nonpriv_reg_handler(struct parser_exec_state *s, ++ unsigned int offset, unsigned int index, char *cmd) ++{ ++ struct intel_gvt *gvt = s->vgpu->gvt; ++ unsigned int data; ++ u32 ring_base; ++ u32 nopid; ++ struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv; ++ ++ if (!strcmp(cmd, "lri")) ++ data = cmd_val(s, index + 1); ++ else { ++ gvt_err("Unexpected forcenonpriv 0x%x write from cmd %s\n", ++ offset, cmd); ++ return -EINVAL; ++ } ++ ++ ring_base = dev_priv->engine[s->ring_id]->mmio_base; ++ nopid = i915_mmio_reg_offset(RING_NOPID(ring_base)); ++ ++ if (!intel_gvt_in_force_nonpriv_whitelist(gvt, data) && ++ data != nopid) { ++ gvt_err("Unexpected forcenonpriv 0x%x LRI write, value=0x%x\n", ++ offset, data); ++ patch_value(s, cmd_ptr(s, index), nopid); ++ return 0; ++ } ++ return 0; ++} ++ ++static inline bool is_mocs_mmio(unsigned int offset) ++{ ++ return ((offset >= 0xc800) && (offset <= 0xcff8)) || ++ ((offset >= 0xb020) && (offset <= 0xb0a0)); ++} ++ ++static int mocs_cmd_reg_handler(struct parser_exec_state *s, ++ unsigned int offset, unsigned int index) ++{ ++ if (!is_mocs_mmio(offset)) ++ return -EINVAL; ++ vgpu_vreg(s->vgpu, offset) = cmd_val(s, index + 1); ++ return 0; ++} ++ ++static int cmd_reg_handler(struct parser_exec_state *s, ++ unsigned int offset, unsigned int index, char *cmd) ++{ ++ struct intel_vgpu *vgpu = s->vgpu; ++ struct intel_gvt *gvt = vgpu->gvt; ++ u32 ctx_sr_ctl; ++ ++ if (offset + 4 > gvt->device_info.mmio_size) { ++ gvt_vgpu_err("%s access to (%x) outside of MMIO range\n", ++ cmd, offset); ++ return -EFAULT; ++ } ++ ++ if (!intel_gvt_mmio_is_cmd_access(gvt, offset)) { ++ gvt_vgpu_err("%s access to non-render register (%x)\n", ++ cmd, offset); ++ return -EBADRQC; ++ } ++ ++ if (is_shadowed_mmio(offset)) { ++ gvt_vgpu_err("found access of shadowed MMIO %x\n", offset); ++ return 0; ++ } ++ ++ if (is_mocs_mmio(offset) && ++ mocs_cmd_reg_handler(s, offset, index)) ++ return -EINVAL; ++ ++ if (is_force_nonpriv_mmio(offset) && ++ force_nonpriv_reg_handler(s, offset, index, cmd)) ++ return -EPERM; ++ ++ if (offset == i915_mmio_reg_offset(DERRMR) || ++ offset == i915_mmio_reg_offset(FORCEWAKE_MT)) { ++ /* Writing to HW VGT_PVINFO_PAGE offset will be discarded */ ++ patch_value(s, cmd_ptr(s, index), VGT_PVINFO_PAGE); ++ } ++ ++ /* TODO ++ * In order to let workload with inhibit context to generate ++ * correct image data into memory, vregs values will be loaded to ++ * hw via LRIs in the workload with inhibit context. But as ++ * indirect context is loaded prior to LRIs in workload, we don't ++ * want reg values specified in indirect context overwritten by ++ * LRIs in workloads. So, when scanning an indirect context, we ++ * update reg values in it into vregs, so LRIs in workload with ++ * inhibit context will restore with correct values ++ */ ++ if (IS_GEN(gvt->dev_priv, 9) && ++ intel_gvt_mmio_is_in_ctx(gvt, offset) && ++ !strncmp(cmd, "lri", 3)) { ++ intel_gvt_hypervisor_read_gpa(s->vgpu, ++ s->workload->ring_context_gpa + 12, &ctx_sr_ctl, 4); ++ /* check inhibit context */ ++ if (ctx_sr_ctl & 1) { ++ u32 data = cmd_val(s, index + 1); ++ ++ if (intel_gvt_mmio_has_mode_mask(s->vgpu->gvt, offset)) ++ intel_vgpu_mask_mmio_write(vgpu, ++ offset, &data, 4); ++ else ++ vgpu_vreg(vgpu, offset) = data; ++ } ++ } ++ ++ /* TODO: Update the global mask if this MMIO is a masked-MMIO */ ++ intel_gvt_mmio_set_cmd_accessed(gvt, offset); ++ return 0; ++} ++ ++#define cmd_reg(s, i) \ ++ (cmd_val(s, i) & GENMASK(22, 2)) ++ ++#define cmd_reg_inhibit(s, i) \ ++ (cmd_val(s, i) & GENMASK(22, 18)) ++ ++#define cmd_gma(s, i) \ ++ (cmd_val(s, i) & GENMASK(31, 2)) ++ ++#define cmd_gma_hi(s, i) \ ++ (cmd_val(s, i) & GENMASK(15, 0)) ++ ++static int cmd_handler_lri(struct parser_exec_state *s) ++{ ++ int i, ret = 0; ++ int cmd_len = cmd_length(s); ++ struct intel_gvt *gvt = s->vgpu->gvt; ++ ++ for (i = 1; i < cmd_len; i += 2) { ++ if (IS_BROADWELL(gvt->dev_priv) && s->ring_id != RCS0) { ++ if (s->ring_id == BCS0 && ++ cmd_reg(s, i) == i915_mmio_reg_offset(DERRMR)) ++ ret |= 0; ++ else ++ ret |= cmd_reg_inhibit(s, i) ? -EBADRQC : 0; ++ } ++ if (ret) ++ break; ++ ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lri"); ++ if (ret) ++ break; ++ } ++ return ret; ++} ++ ++static int cmd_handler_lrr(struct parser_exec_state *s) ++{ ++ int i, ret = 0; ++ int cmd_len = cmd_length(s); ++ ++ for (i = 1; i < cmd_len; i += 2) { ++ if (IS_BROADWELL(s->vgpu->gvt->dev_priv)) ++ ret |= ((cmd_reg_inhibit(s, i) || ++ (cmd_reg_inhibit(s, i + 1)))) ? ++ -EBADRQC : 0; ++ if (ret) ++ break; ++ ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lrr-src"); ++ if (ret) ++ break; ++ ret |= cmd_reg_handler(s, cmd_reg(s, i + 1), i, "lrr-dst"); ++ if (ret) ++ break; ++ } ++ return ret; ++} ++ ++static inline int cmd_address_audit(struct parser_exec_state *s, ++ unsigned long guest_gma, int op_size, bool index_mode); ++ ++static int cmd_handler_lrm(struct parser_exec_state *s) ++{ ++ struct intel_gvt *gvt = s->vgpu->gvt; ++ int gmadr_bytes = gvt->device_info.gmadr_bytes_in_cmd; ++ unsigned long gma; ++ int i, ret = 0; ++ int cmd_len = cmd_length(s); ++ ++ for (i = 1; i < cmd_len;) { ++ if (IS_BROADWELL(gvt->dev_priv)) ++ ret |= (cmd_reg_inhibit(s, i)) ? -EBADRQC : 0; ++ if (ret) ++ break; ++ ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lrm"); ++ if (ret) ++ break; ++ if (cmd_val(s, 0) & (1 << 22)) { ++ gma = cmd_gma(s, i + 1); ++ if (gmadr_bytes == 8) ++ gma |= (cmd_gma_hi(s, i + 2)) << 32; ++ ret |= cmd_address_audit(s, gma, sizeof(u32), false); ++ if (ret) ++ break; ++ } ++ i += gmadr_dw_number(s) + 1; ++ } ++ return ret; ++} ++ ++static int cmd_handler_srm(struct parser_exec_state *s) ++{ ++ int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd; ++ unsigned long gma; ++ int i, ret = 0; ++ int cmd_len = cmd_length(s); ++ ++ for (i = 1; i < cmd_len;) { ++ ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "srm"); ++ if (ret) ++ break; ++ if (cmd_val(s, 0) & (1 << 22)) { ++ gma = cmd_gma(s, i + 1); ++ if (gmadr_bytes == 8) ++ gma |= (cmd_gma_hi(s, i + 2)) << 32; ++ ret |= cmd_address_audit(s, gma, sizeof(u32), false); ++ if (ret) ++ break; ++ } ++ i += gmadr_dw_number(s) + 1; ++ } ++ return ret; ++} ++ ++struct cmd_interrupt_event { ++ int pipe_control_notify; ++ int mi_flush_dw; ++ int mi_user_interrupt; ++}; ++ ++static struct cmd_interrupt_event cmd_interrupt_events[] = { ++ [RCS0] = { ++ .pipe_control_notify = RCS_PIPE_CONTROL, ++ .mi_flush_dw = INTEL_GVT_EVENT_RESERVED, ++ .mi_user_interrupt = RCS_MI_USER_INTERRUPT, ++ }, ++ [BCS0] = { ++ .pipe_control_notify = INTEL_GVT_EVENT_RESERVED, ++ .mi_flush_dw = BCS_MI_FLUSH_DW, ++ .mi_user_interrupt = BCS_MI_USER_INTERRUPT, ++ }, ++ [VCS0] = { ++ .pipe_control_notify = INTEL_GVT_EVENT_RESERVED, ++ .mi_flush_dw = VCS_MI_FLUSH_DW, ++ .mi_user_interrupt = VCS_MI_USER_INTERRUPT, ++ }, ++ [VCS1] = { ++ .pipe_control_notify = INTEL_GVT_EVENT_RESERVED, ++ .mi_flush_dw = VCS2_MI_FLUSH_DW, ++ .mi_user_interrupt = VCS2_MI_USER_INTERRUPT, ++ }, ++ [VECS0] = { ++ .pipe_control_notify = INTEL_GVT_EVENT_RESERVED, ++ .mi_flush_dw = VECS_MI_FLUSH_DW, ++ .mi_user_interrupt = VECS_MI_USER_INTERRUPT, ++ }, ++}; ++ ++static int cmd_handler_pipe_control(struct parser_exec_state *s) ++{ ++ int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd; ++ unsigned long gma; ++ bool index_mode = false; ++ unsigned int post_sync; ++ int ret = 0; ++ u32 hws_pga, val; ++ ++ post_sync = (cmd_val(s, 1) & PIPE_CONTROL_POST_SYNC_OP_MASK) >> 14; ++ ++ /* LRI post sync */ ++ if (cmd_val(s, 1) & PIPE_CONTROL_MMIO_WRITE) ++ ret = cmd_reg_handler(s, cmd_reg(s, 2), 1, "pipe_ctrl"); ++ /* post sync */ ++ else if (post_sync) { ++ if (post_sync == 2) ++ ret = cmd_reg_handler(s, 0x2350, 1, "pipe_ctrl"); ++ else if (post_sync == 3) ++ ret = cmd_reg_handler(s, 0x2358, 1, "pipe_ctrl"); ++ else if (post_sync == 1) { ++ /* check ggtt*/ ++ if ((cmd_val(s, 1) & PIPE_CONTROL_GLOBAL_GTT_IVB)) { ++ gma = cmd_val(s, 2) & GENMASK(31, 3); ++ if (gmadr_bytes == 8) ++ gma |= (cmd_gma_hi(s, 3)) << 32; ++ /* Store Data Index */ ++ if (cmd_val(s, 1) & (1 << 21)) ++ index_mode = true; ++ ret |= cmd_address_audit(s, gma, sizeof(u64), ++ index_mode); ++ if (ret) ++ return ret; ++ if (index_mode) { ++ hws_pga = s->vgpu->hws_pga[s->ring_id]; ++ gma = hws_pga + gma; ++ patch_value(s, cmd_ptr(s, 2), gma); ++ val = cmd_val(s, 1) & (~(1 << 21)); ++ patch_value(s, cmd_ptr(s, 1), val); ++ } ++ } ++ } ++ } ++ ++ if (ret) ++ return ret; ++ ++ if (cmd_val(s, 1) & PIPE_CONTROL_NOTIFY) ++ set_bit(cmd_interrupt_events[s->ring_id].pipe_control_notify, ++ s->workload->pending_events); ++ return 0; ++} ++ ++static int cmd_handler_mi_user_interrupt(struct parser_exec_state *s) ++{ ++ set_bit(cmd_interrupt_events[s->ring_id].mi_user_interrupt, ++ s->workload->pending_events); ++ patch_value(s, cmd_ptr(s, 0), MI_NOOP); ++ return 0; ++} ++ ++static int cmd_advance_default(struct parser_exec_state *s) ++{ ++ return ip_gma_advance(s, cmd_length(s)); ++} ++ ++static int cmd_handler_mi_batch_buffer_end(struct parser_exec_state *s) ++{ ++ int ret; ++ ++ if (s->buf_type == BATCH_BUFFER_2ND_LEVEL) { ++ s->buf_type = BATCH_BUFFER_INSTRUCTION; ++ ret = ip_gma_set(s, s->ret_ip_gma_bb); ++ s->buf_addr_type = s->saved_buf_addr_type; ++ } else { ++ s->buf_type = RING_BUFFER_INSTRUCTION; ++ s->buf_addr_type = GTT_BUFFER; ++ if (s->ret_ip_gma_ring >= s->ring_start + s->ring_size) ++ s->ret_ip_gma_ring -= s->ring_size; ++ ret = ip_gma_set(s, s->ret_ip_gma_ring); ++ } ++ return ret; ++} ++ ++struct mi_display_flip_command_info { ++ int pipe; ++ int plane; ++ int event; ++ i915_reg_t stride_reg; ++ i915_reg_t ctrl_reg; ++ i915_reg_t surf_reg; ++ u64 stride_val; ++ u64 tile_val; ++ u64 surf_val; ++ bool async_flip; ++}; ++ ++struct plane_code_mapping { ++ int pipe; ++ int plane; ++ int event; ++}; ++ ++static int gen8_decode_mi_display_flip(struct parser_exec_state *s, ++ struct mi_display_flip_command_info *info) ++{ ++ struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv; ++ struct plane_code_mapping gen8_plane_code[] = { ++ [0] = {PIPE_A, PLANE_A, PRIMARY_A_FLIP_DONE}, ++ [1] = {PIPE_B, PLANE_A, PRIMARY_B_FLIP_DONE}, ++ [2] = {PIPE_A, PLANE_B, SPRITE_A_FLIP_DONE}, ++ [3] = {PIPE_B, PLANE_B, SPRITE_B_FLIP_DONE}, ++ [4] = {PIPE_C, PLANE_A, PRIMARY_C_FLIP_DONE}, ++ [5] = {PIPE_C, PLANE_B, SPRITE_C_FLIP_DONE}, ++ }; ++ u32 dword0, dword1, dword2; ++ u32 v; ++ ++ dword0 = cmd_val(s, 0); ++ dword1 = cmd_val(s, 1); ++ dword2 = cmd_val(s, 2); ++ ++ v = (dword0 & GENMASK(21, 19)) >> 19; ++ if (WARN_ON(v >= ARRAY_SIZE(gen8_plane_code))) ++ return -EBADRQC; ++ ++ info->pipe = gen8_plane_code[v].pipe; ++ info->plane = gen8_plane_code[v].plane; ++ info->event = gen8_plane_code[v].event; ++ info->stride_val = (dword1 & GENMASK(15, 6)) >> 6; ++ info->tile_val = (dword1 & 0x1); ++ info->surf_val = (dword2 & GENMASK(31, 12)) >> 12; ++ info->async_flip = ((dword2 & GENMASK(1, 0)) == 0x1); ++ ++ if (info->plane == PLANE_A) { ++ info->ctrl_reg = DSPCNTR(info->pipe); ++ info->stride_reg = DSPSTRIDE(info->pipe); ++ info->surf_reg = DSPSURF(info->pipe); ++ } else if (info->plane == PLANE_B) { ++ info->ctrl_reg = SPRCTL(info->pipe); ++ info->stride_reg = SPRSTRIDE(info->pipe); ++ info->surf_reg = SPRSURF(info->pipe); ++ } else { ++ WARN_ON(1); ++ return -EBADRQC; ++ } ++ return 0; ++} ++ ++static int skl_decode_mi_display_flip(struct parser_exec_state *s, ++ struct mi_display_flip_command_info *info) ++{ ++ struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv; ++ struct intel_vgpu *vgpu = s->vgpu; ++ u32 dword0 = cmd_val(s, 0); ++ u32 dword1 = cmd_val(s, 1); ++ u32 dword2 = cmd_val(s, 2); ++ u32 plane = (dword0 & GENMASK(12, 8)) >> 8; ++ ++ info->plane = PRIMARY_PLANE; ++ ++ switch (plane) { ++ case MI_DISPLAY_FLIP_SKL_PLANE_1_A: ++ info->pipe = PIPE_A; ++ info->event = PRIMARY_A_FLIP_DONE; ++ break; ++ case MI_DISPLAY_FLIP_SKL_PLANE_1_B: ++ info->pipe = PIPE_B; ++ info->event = PRIMARY_B_FLIP_DONE; ++ break; ++ case MI_DISPLAY_FLIP_SKL_PLANE_1_C: ++ info->pipe = PIPE_C; ++ info->event = PRIMARY_C_FLIP_DONE; ++ break; ++ ++ case MI_DISPLAY_FLIP_SKL_PLANE_2_A: ++ info->pipe = PIPE_A; ++ info->event = SPRITE_A_FLIP_DONE; ++ info->plane = SPRITE_PLANE; ++ break; ++ case MI_DISPLAY_FLIP_SKL_PLANE_2_B: ++ info->pipe = PIPE_B; ++ info->event = SPRITE_B_FLIP_DONE; ++ info->plane = SPRITE_PLANE; ++ break; ++ case MI_DISPLAY_FLIP_SKL_PLANE_2_C: ++ info->pipe = PIPE_C; ++ info->event = SPRITE_C_FLIP_DONE; ++ info->plane = SPRITE_PLANE; ++ break; ++ ++ default: ++ gvt_vgpu_err("unknown plane code %d\n", plane); ++ return -EBADRQC; ++ } ++ ++ info->stride_val = (dword1 & GENMASK(15, 6)) >> 6; ++ info->tile_val = (dword1 & GENMASK(2, 0)); ++ info->surf_val = (dword2 & GENMASK(31, 12)) >> 12; ++ info->async_flip = ((dword2 & GENMASK(1, 0)) == 0x1); ++ ++ info->ctrl_reg = DSPCNTR(info->pipe); ++ info->stride_reg = DSPSTRIDE(info->pipe); ++ info->surf_reg = DSPSURF(info->pipe); ++ ++ return 0; ++} ++ ++static int gen8_check_mi_display_flip(struct parser_exec_state *s, ++ struct mi_display_flip_command_info *info) ++{ ++ struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv; ++ u32 stride, tile; ++ ++ if (!info->async_flip) ++ return 0; ++ ++ if (INTEL_GEN(dev_priv) >= 9) { ++ stride = vgpu_vreg_t(s->vgpu, info->stride_reg) & GENMASK(9, 0); ++ tile = (vgpu_vreg_t(s->vgpu, info->ctrl_reg) & ++ GENMASK(12, 10)) >> 10; ++ } else { ++ stride = (vgpu_vreg_t(s->vgpu, info->stride_reg) & ++ GENMASK(15, 6)) >> 6; ++ tile = (vgpu_vreg_t(s->vgpu, info->ctrl_reg) & (1 << 10)) >> 10; ++ } ++ ++ if (stride != info->stride_val) ++ gvt_dbg_cmd("cannot change stride during async flip\n"); ++ ++ if (tile != info->tile_val) ++ gvt_dbg_cmd("cannot change tile during async flip\n"); ++ ++ return 0; ++} ++ ++static int gen8_update_plane_mmio_from_mi_display_flip( ++ struct parser_exec_state *s, ++ struct mi_display_flip_command_info *info) ++{ ++ struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv; ++ struct intel_vgpu *vgpu = s->vgpu; ++ ++ set_mask_bits(&vgpu_vreg_t(vgpu, info->surf_reg), GENMASK(31, 12), ++ info->surf_val << 12); ++ if (INTEL_GEN(dev_priv) >= 9) { ++ set_mask_bits(&vgpu_vreg_t(vgpu, info->stride_reg), GENMASK(9, 0), ++ info->stride_val); ++ set_mask_bits(&vgpu_vreg_t(vgpu, info->ctrl_reg), GENMASK(12, 10), ++ info->tile_val << 10); ++ } else { ++ set_mask_bits(&vgpu_vreg_t(vgpu, info->stride_reg), GENMASK(15, 6), ++ info->stride_val << 6); ++ set_mask_bits(&vgpu_vreg_t(vgpu, info->ctrl_reg), GENMASK(10, 10), ++ info->tile_val << 10); ++ } ++ ++ if (info->plane == PLANE_PRIMARY) ++ vgpu_vreg_t(vgpu, PIPE_FLIPCOUNT_G4X(info->pipe))++; ++ ++ if (info->async_flip) ++ intel_vgpu_trigger_virtual_event(vgpu, info->event); ++ else ++ set_bit(info->event, vgpu->irq.flip_done_event[info->pipe]); ++ ++ return 0; ++} ++ ++static int decode_mi_display_flip(struct parser_exec_state *s, ++ struct mi_display_flip_command_info *info) ++{ ++ struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv; ++ ++ if (IS_BROADWELL(dev_priv)) ++ return gen8_decode_mi_display_flip(s, info); ++ if (INTEL_GEN(dev_priv) >= 9) ++ return skl_decode_mi_display_flip(s, info); ++ ++ return -ENODEV; ++} ++ ++static int check_mi_display_flip(struct parser_exec_state *s, ++ struct mi_display_flip_command_info *info) ++{ ++ return gen8_check_mi_display_flip(s, info); ++} ++ ++static int update_plane_mmio_from_mi_display_flip( ++ struct parser_exec_state *s, ++ struct mi_display_flip_command_info *info) ++{ ++ return gen8_update_plane_mmio_from_mi_display_flip(s, info); ++} ++ ++static int cmd_handler_mi_display_flip(struct parser_exec_state *s) ++{ ++ struct mi_display_flip_command_info info; ++ struct intel_vgpu *vgpu = s->vgpu; ++ int ret; ++ int i; ++ int len = cmd_length(s); ++ ++ ret = decode_mi_display_flip(s, &info); ++ if (ret) { ++ gvt_vgpu_err("fail to decode MI display flip command\n"); ++ return ret; ++ } ++ ++ ret = check_mi_display_flip(s, &info); ++ if (ret) { ++ gvt_vgpu_err("invalid MI display flip command\n"); ++ return ret; ++ } ++ ++ ret = update_plane_mmio_from_mi_display_flip(s, &info); ++ if (ret) { ++ gvt_vgpu_err("fail to update plane mmio\n"); ++ return ret; ++ } ++ ++ for (i = 0; i < len; i++) ++ patch_value(s, cmd_ptr(s, i), MI_NOOP); ++ return 0; ++} ++ ++static bool is_wait_for_flip_pending(u32 cmd) ++{ ++ return cmd & (MI_WAIT_FOR_PLANE_A_FLIP_PENDING | ++ MI_WAIT_FOR_PLANE_B_FLIP_PENDING | ++ MI_WAIT_FOR_PLANE_C_FLIP_PENDING | ++ MI_WAIT_FOR_SPRITE_A_FLIP_PENDING | ++ MI_WAIT_FOR_SPRITE_B_FLIP_PENDING | ++ MI_WAIT_FOR_SPRITE_C_FLIP_PENDING); ++} ++ ++static int cmd_handler_mi_wait_for_event(struct parser_exec_state *s) ++{ ++ u32 cmd = cmd_val(s, 0); ++ ++ if (!is_wait_for_flip_pending(cmd)) ++ return 0; ++ ++ patch_value(s, cmd_ptr(s, 0), MI_NOOP); ++ return 0; ++} ++ ++static unsigned long get_gma_bb_from_cmd(struct parser_exec_state *s, int index) ++{ ++ unsigned long addr; ++ unsigned long gma_high, gma_low; ++ struct intel_vgpu *vgpu = s->vgpu; ++ int gmadr_bytes = vgpu->gvt->device_info.gmadr_bytes_in_cmd; ++ ++ if (WARN_ON(gmadr_bytes != 4 && gmadr_bytes != 8)) { ++ gvt_vgpu_err("invalid gma bytes %d\n", gmadr_bytes); ++ return INTEL_GVT_INVALID_ADDR; ++ } ++ ++ gma_low = cmd_val(s, index) & BATCH_BUFFER_ADDR_MASK; ++ if (gmadr_bytes == 4) { ++ addr = gma_low; ++ } else { ++ gma_high = cmd_val(s, index + 1) & BATCH_BUFFER_ADDR_HIGH_MASK; ++ addr = (((unsigned long)gma_high) << 32) | gma_low; ++ } ++ return addr; ++} ++ ++static inline int cmd_address_audit(struct parser_exec_state *s, ++ unsigned long guest_gma, int op_size, bool index_mode) ++{ ++ struct intel_vgpu *vgpu = s->vgpu; ++ u32 max_surface_size = vgpu->gvt->device_info.max_surface_size; ++ int i; ++ int ret; ++ ++ if (op_size > max_surface_size) { ++ gvt_vgpu_err("command address audit fail name %s\n", ++ s->info->name); ++ return -EFAULT; ++ } ++ ++ if (index_mode) { ++ if (guest_gma >= I915_GTT_PAGE_SIZE) { ++ ret = -EFAULT; ++ goto err; ++ } ++ } else if (!intel_gvt_ggtt_validate_range(vgpu, guest_gma, op_size)) { ++ ret = -EFAULT; ++ goto err; ++ } ++ ++ return 0; ++ ++err: ++ gvt_vgpu_err("cmd_parser: Malicious %s detected, addr=0x%lx, len=%d!\n", ++ s->info->name, guest_gma, op_size); ++ ++ pr_err("cmd dump: "); ++ for (i = 0; i < cmd_length(s); i++) { ++ if (!(i % 4)) ++ pr_err("\n%08x ", cmd_val(s, i)); ++ else ++ pr_err("%08x ", cmd_val(s, i)); ++ } ++ pr_err("\nvgpu%d: aperture 0x%llx - 0x%llx, hidden 0x%llx - 0x%llx\n", ++ vgpu->id, ++ vgpu_aperture_gmadr_base(vgpu), ++ vgpu_aperture_gmadr_end(vgpu), ++ vgpu_hidden_gmadr_base(vgpu), ++ vgpu_hidden_gmadr_end(vgpu)); ++ return ret; ++} ++ ++static int cmd_handler_mi_store_data_imm(struct parser_exec_state *s) ++{ ++ int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd; ++ int op_size = (cmd_length(s) - 3) * sizeof(u32); ++ int core_id = (cmd_val(s, 2) & (1 << 0)) ? 1 : 0; ++ unsigned long gma, gma_low, gma_high; ++ int ret = 0; ++ ++ /* check ppggt */ ++ if (!(cmd_val(s, 0) & (1 << 22))) ++ return 0; ++ ++ gma = cmd_val(s, 2) & GENMASK(31, 2); ++ ++ if (gmadr_bytes == 8) { ++ gma_low = cmd_val(s, 1) & GENMASK(31, 2); ++ gma_high = cmd_val(s, 2) & GENMASK(15, 0); ++ gma = (gma_high << 32) | gma_low; ++ core_id = (cmd_val(s, 1) & (1 << 0)) ? 1 : 0; ++ } ++ ret = cmd_address_audit(s, gma + op_size * core_id, op_size, false); ++ return ret; ++} ++ ++static inline int unexpected_cmd(struct parser_exec_state *s) ++{ ++ struct intel_vgpu *vgpu = s->vgpu; ++ ++ gvt_vgpu_err("Unexpected %s in command buffer!\n", s->info->name); ++ ++ return -EBADRQC; ++} ++ ++static int cmd_handler_mi_semaphore_wait(struct parser_exec_state *s) ++{ ++ return unexpected_cmd(s); ++} ++ ++static int cmd_handler_mi_report_perf_count(struct parser_exec_state *s) ++{ ++ return unexpected_cmd(s); ++} ++ ++static int cmd_handler_mi_op_2e(struct parser_exec_state *s) ++{ ++ return unexpected_cmd(s); ++} ++ ++static int cmd_handler_mi_op_2f(struct parser_exec_state *s) ++{ ++ int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd; ++ int op_size = (1 << ((cmd_val(s, 0) & GENMASK(20, 19)) >> 19)) * ++ sizeof(u32); ++ unsigned long gma, gma_high; ++ int ret = 0; ++ ++ if (!(cmd_val(s, 0) & (1 << 22))) ++ return ret; ++ ++ gma = cmd_val(s, 1) & GENMASK(31, 2); ++ if (gmadr_bytes == 8) { ++ gma_high = cmd_val(s, 2) & GENMASK(15, 0); ++ gma = (gma_high << 32) | gma; ++ } ++ ret = cmd_address_audit(s, gma, op_size, false); ++ return ret; ++} ++ ++static int cmd_handler_mi_store_data_index(struct parser_exec_state *s) ++{ ++ return unexpected_cmd(s); ++} ++ ++static int cmd_handler_mi_clflush(struct parser_exec_state *s) ++{ ++ return unexpected_cmd(s); ++} ++ ++static int cmd_handler_mi_conditional_batch_buffer_end( ++ struct parser_exec_state *s) ++{ ++ return unexpected_cmd(s); ++} ++ ++static int cmd_handler_mi_update_gtt(struct parser_exec_state *s) ++{ ++ return unexpected_cmd(s); ++} ++ ++static int cmd_handler_mi_flush_dw(struct parser_exec_state *s) ++{ ++ int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd; ++ unsigned long gma; ++ bool index_mode = false; ++ int ret = 0; ++ u32 hws_pga, val; ++ ++ /* Check post-sync and ppgtt bit */ ++ if (((cmd_val(s, 0) >> 14) & 0x3) && (cmd_val(s, 1) & (1 << 2))) { ++ gma = cmd_val(s, 1) & GENMASK(31, 3); ++ if (gmadr_bytes == 8) ++ gma |= (cmd_val(s, 2) & GENMASK(15, 0)) << 32; ++ /* Store Data Index */ ++ if (cmd_val(s, 0) & (1 << 21)) ++ index_mode = true; ++ ret = cmd_address_audit(s, gma, sizeof(u64), index_mode); ++ if (ret) ++ return ret; ++ if (index_mode) { ++ hws_pga = s->vgpu->hws_pga[s->ring_id]; ++ gma = hws_pga + gma; ++ patch_value(s, cmd_ptr(s, 1), gma); ++ val = cmd_val(s, 0) & (~(1 << 21)); ++ patch_value(s, cmd_ptr(s, 0), val); ++ } ++ } ++ /* Check notify bit */ ++ if ((cmd_val(s, 0) & (1 << 8))) ++ set_bit(cmd_interrupt_events[s->ring_id].mi_flush_dw, ++ s->workload->pending_events); ++ return ret; ++} ++ ++static void addr_type_update_snb(struct parser_exec_state *s) ++{ ++ if ((s->buf_type == RING_BUFFER_INSTRUCTION) && ++ (BATCH_BUFFER_ADR_SPACE_BIT(cmd_val(s, 0)) == 1)) { ++ s->buf_addr_type = PPGTT_BUFFER; ++ } ++} ++ ++ ++static int copy_gma_to_hva(struct intel_vgpu *vgpu, struct intel_vgpu_mm *mm, ++ unsigned long gma, unsigned long end_gma, void *va) ++{ ++ unsigned long copy_len, offset; ++ unsigned long len = 0; ++ unsigned long gpa; ++ ++ while (gma != end_gma) { ++ gpa = intel_vgpu_gma_to_gpa(mm, gma); ++ if (gpa == INTEL_GVT_INVALID_ADDR) { ++ gvt_vgpu_err("invalid gma address: %lx\n", gma); ++ return -EFAULT; ++ } ++ ++ offset = gma & (I915_GTT_PAGE_SIZE - 1); ++ ++ copy_len = (end_gma - gma) >= (I915_GTT_PAGE_SIZE - offset) ? ++ I915_GTT_PAGE_SIZE - offset : end_gma - gma; ++ ++ intel_gvt_hypervisor_read_gpa(vgpu, gpa, va + len, copy_len); ++ ++ len += copy_len; ++ gma += copy_len; ++ } ++ return len; ++} ++ ++ ++/* ++ * Check whether a batch buffer needs to be scanned. Currently ++ * the only criteria is based on privilege. ++ */ ++static int batch_buffer_needs_scan(struct parser_exec_state *s) ++{ ++ /* Decide privilege based on address space */ ++ if (cmd_val(s, 0) & (1 << 8) && ++ !(s->vgpu->scan_nonprivbb & (1 << s->ring_id))) ++ return 0; ++ return 1; ++} ++ ++static int find_bb_size(struct parser_exec_state *s, unsigned long *bb_size) ++{ ++ unsigned long gma = 0; ++ const struct cmd_info *info; ++ u32 cmd_len = 0; ++ bool bb_end = false; ++ struct intel_vgpu *vgpu = s->vgpu; ++ u32 cmd; ++ struct intel_vgpu_mm *mm = (s->buf_addr_type == GTT_BUFFER) ? ++ s->vgpu->gtt.ggtt_mm : s->workload->shadow_mm; ++ ++ *bb_size = 0; ++ ++ /* get the start gm address of the batch buffer */ ++ gma = get_gma_bb_from_cmd(s, 1); ++ if (gma == INTEL_GVT_INVALID_ADDR) ++ return -EFAULT; ++ ++ cmd = cmd_val(s, 0); ++ info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id); ++ if (info == NULL) { ++ gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x, addr_type=%s, ring %d, workload=%p\n", ++ cmd, get_opcode(cmd, s->ring_id), ++ (s->buf_addr_type == PPGTT_BUFFER) ? ++ "ppgtt" : "ggtt", s->ring_id, s->workload); ++ return -EBADRQC; ++ } ++ do { ++ if (copy_gma_to_hva(s->vgpu, mm, ++ gma, gma + 4, &cmd) < 0) ++ return -EFAULT; ++ info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id); ++ if (info == NULL) { ++ gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x, addr_type=%s, ring %d, workload=%p\n", ++ cmd, get_opcode(cmd, s->ring_id), ++ (s->buf_addr_type == PPGTT_BUFFER) ? ++ "ppgtt" : "ggtt", s->ring_id, s->workload); ++ return -EBADRQC; ++ } ++ ++ if (info->opcode == OP_MI_BATCH_BUFFER_END) { ++ bb_end = true; ++ } else if (info->opcode == OP_MI_BATCH_BUFFER_START) { ++ if (BATCH_BUFFER_2ND_LEVEL_BIT(cmd) == 0) ++ /* chained batch buffer */ ++ bb_end = true; ++ } ++ cmd_len = get_cmd_length(info, cmd) << 2; ++ *bb_size += cmd_len; ++ gma += cmd_len; ++ } while (!bb_end); ++ ++ return 0; ++} ++ ++static int perform_bb_shadow(struct parser_exec_state *s) ++{ ++ struct intel_vgpu *vgpu = s->vgpu; ++ struct intel_vgpu_shadow_bb *bb; ++ unsigned long gma = 0; ++ unsigned long bb_size; ++ int ret = 0; ++ struct intel_vgpu_mm *mm = (s->buf_addr_type == GTT_BUFFER) ? ++ s->vgpu->gtt.ggtt_mm : s->workload->shadow_mm; ++ unsigned long gma_start_offset = 0; ++ ++ /* get the start gm address of the batch buffer */ ++ gma = get_gma_bb_from_cmd(s, 1); ++ if (gma == INTEL_GVT_INVALID_ADDR) ++ return -EFAULT; ++ ++ ret = find_bb_size(s, &bb_size); ++ if (ret) ++ return ret; ++ ++ bb = kzalloc(sizeof(*bb), GFP_KERNEL); ++ if (!bb) ++ return -ENOMEM; ++ ++ bb->ppgtt = (s->buf_addr_type == GTT_BUFFER) ? false : true; ++ ++ /* the gma_start_offset stores the batch buffer's start gma's ++ * offset relative to page boundary. so for non-privileged batch ++ * buffer, the shadowed gem object holds exactly the same page ++ * layout as original gem object. This is for the convience of ++ * replacing the whole non-privilged batch buffer page to this ++ * shadowed one in PPGTT at the same gma address. (this replacing ++ * action is not implemented yet now, but may be necessary in ++ * future). ++ * for prileged batch buffer, we just change start gma address to ++ * that of shadowed page. ++ */ ++ if (bb->ppgtt) ++ gma_start_offset = gma & ~I915_GTT_PAGE_MASK; ++ ++ bb->obj = i915_gem_object_create(s->vgpu->gvt->dev_priv, ++ roundup(bb_size + gma_start_offset, PAGE_SIZE)); ++ if (IS_ERR(bb->obj)) { ++ ret = PTR_ERR(bb->obj); ++ goto err_free_bb; ++ } ++ ++ ret = i915_gem_obj_prepare_shmem_write(bb->obj, &bb->clflush); ++ if (ret) ++ goto err_free_obj; ++ ++ bb->va = i915_gem_object_pin_map(bb->obj, I915_MAP_WB); ++ if (IS_ERR(bb->va)) { ++ ret = PTR_ERR(bb->va); ++ goto err_finish_shmem_access; ++ } ++ ++ if (bb->clflush & CLFLUSH_BEFORE) { ++ drm_clflush_virt_range(bb->va, bb->obj->base.size); ++ bb->clflush &= ~CLFLUSH_BEFORE; ++ } ++ ++ ret = copy_gma_to_hva(s->vgpu, mm, ++ gma, gma + bb_size, ++ bb->va + gma_start_offset); ++ if (ret < 0) { ++ gvt_vgpu_err("fail to copy guest ring buffer\n"); ++ ret = -EFAULT; ++ goto err_unmap; ++ } ++ ++ INIT_LIST_HEAD(&bb->list); ++ list_add(&bb->list, &s->workload->shadow_bb); ++ ++ bb->accessing = true; ++ bb->bb_start_cmd_va = s->ip_va; ++ ++ if ((s->buf_type == BATCH_BUFFER_INSTRUCTION) && (!s->is_ctx_wa)) ++ bb->bb_offset = s->ip_va - s->rb_va; ++ else ++ bb->bb_offset = 0; ++ ++ /* ++ * ip_va saves the virtual address of the shadow batch buffer, while ++ * ip_gma saves the graphics address of the original batch buffer. ++ * As the shadow batch buffer is just a copy from the originial one, ++ * it should be right to use shadow batch buffer'va and original batch ++ * buffer's gma in pair. After all, we don't want to pin the shadow ++ * buffer here (too early). ++ */ ++ s->ip_va = bb->va + gma_start_offset; ++ s->ip_gma = gma; ++ return 0; ++err_unmap: ++ i915_gem_object_unpin_map(bb->obj); ++err_finish_shmem_access: ++ i915_gem_obj_finish_shmem_access(bb->obj); ++err_free_obj: ++ i915_gem_object_put(bb->obj); ++err_free_bb: ++ kfree(bb); ++ return ret; ++} ++ ++static int cmd_handler_mi_batch_buffer_start(struct parser_exec_state *s) ++{ ++ bool second_level; ++ int ret = 0; ++ struct intel_vgpu *vgpu = s->vgpu; ++ ++ if (s->buf_type == BATCH_BUFFER_2ND_LEVEL) { ++ gvt_vgpu_err("Found MI_BATCH_BUFFER_START in 2nd level BB\n"); ++ return -EFAULT; ++ } ++ ++ second_level = BATCH_BUFFER_2ND_LEVEL_BIT(cmd_val(s, 0)) == 1; ++ if (second_level && (s->buf_type != BATCH_BUFFER_INSTRUCTION)) { ++ gvt_vgpu_err("Jumping to 2nd level BB from RB is not allowed\n"); ++ return -EFAULT; ++ } ++ ++ s->saved_buf_addr_type = s->buf_addr_type; ++ addr_type_update_snb(s); ++ if (s->buf_type == RING_BUFFER_INSTRUCTION) { ++ s->ret_ip_gma_ring = s->ip_gma + cmd_length(s) * sizeof(u32); ++ s->buf_type = BATCH_BUFFER_INSTRUCTION; ++ } else if (second_level) { ++ s->buf_type = BATCH_BUFFER_2ND_LEVEL; ++ s->ret_ip_gma_bb = s->ip_gma + cmd_length(s) * sizeof(u32); ++ s->ret_bb_va = s->ip_va + cmd_length(s) * sizeof(u32); ++ } ++ ++ if (batch_buffer_needs_scan(s)) { ++ ret = perform_bb_shadow(s); ++ if (ret < 0) ++ gvt_vgpu_err("invalid shadow batch buffer\n"); ++ } else { ++ /* emulate a batch buffer end to do return right */ ++ ret = cmd_handler_mi_batch_buffer_end(s); ++ if (ret < 0) ++ return ret; ++ } ++ return ret; ++} ++ ++static int mi_noop_index; ++ ++static const struct cmd_info cmd_info[] = { ++ {"MI_NOOP", OP_MI_NOOP, F_LEN_CONST, R_ALL, D_ALL, 0, 1, NULL}, ++ ++ {"MI_SET_PREDICATE", OP_MI_SET_PREDICATE, F_LEN_CONST, R_ALL, D_ALL, ++ 0, 1, NULL}, ++ ++ {"MI_USER_INTERRUPT", OP_MI_USER_INTERRUPT, F_LEN_CONST, R_ALL, D_ALL, ++ 0, 1, cmd_handler_mi_user_interrupt}, ++ ++ {"MI_WAIT_FOR_EVENT", OP_MI_WAIT_FOR_EVENT, F_LEN_CONST, R_RCS | R_BCS, ++ D_ALL, 0, 1, cmd_handler_mi_wait_for_event}, ++ ++ {"MI_FLUSH", OP_MI_FLUSH, F_LEN_CONST, R_ALL, D_ALL, 0, 1, NULL}, ++ ++ {"MI_ARB_CHECK", OP_MI_ARB_CHECK, F_LEN_CONST, R_ALL, D_ALL, 0, 1, ++ NULL}, ++ ++ {"MI_RS_CONTROL", OP_MI_RS_CONTROL, F_LEN_CONST, R_RCS, D_ALL, 0, 1, ++ NULL}, ++ ++ {"MI_REPORT_HEAD", OP_MI_REPORT_HEAD, F_LEN_CONST, R_ALL, D_ALL, 0, 1, ++ NULL}, ++ ++ {"MI_ARB_ON_OFF", OP_MI_ARB_ON_OFF, F_LEN_CONST, R_ALL, D_ALL, 0, 1, ++ NULL}, ++ ++ {"MI_URB_ATOMIC_ALLOC", OP_MI_URB_ATOMIC_ALLOC, F_LEN_CONST, R_RCS, ++ D_ALL, 0, 1, NULL}, ++ ++ {"MI_BATCH_BUFFER_END", OP_MI_BATCH_BUFFER_END, ++ F_IP_ADVANCE_CUSTOM | F_LEN_CONST, R_ALL, D_ALL, 0, 1, ++ cmd_handler_mi_batch_buffer_end}, ++ ++ {"MI_SUSPEND_FLUSH", OP_MI_SUSPEND_FLUSH, F_LEN_CONST, R_ALL, D_ALL, ++ 0, 1, NULL}, ++ ++ {"MI_PREDICATE", OP_MI_PREDICATE, F_LEN_CONST, R_RCS, D_ALL, 0, 1, ++ NULL}, ++ ++ {"MI_TOPOLOGY_FILTER", OP_MI_TOPOLOGY_FILTER, F_LEN_CONST, R_ALL, ++ D_ALL, 0, 1, NULL}, ++ ++ {"MI_SET_APPID", OP_MI_SET_APPID, F_LEN_CONST, R_ALL, D_ALL, 0, 1, ++ NULL}, ++ ++ {"MI_RS_CONTEXT", OP_MI_RS_CONTEXT, F_LEN_CONST, R_RCS, D_ALL, 0, 1, ++ NULL}, ++ ++ {"MI_DISPLAY_FLIP", OP_MI_DISPLAY_FLIP, F_LEN_VAR | F_POST_HANDLE, ++ R_RCS | R_BCS, D_ALL, 0, 8, cmd_handler_mi_display_flip}, ++ ++ {"MI_SEMAPHORE_MBOX", OP_MI_SEMAPHORE_MBOX, F_LEN_VAR, R_ALL, D_ALL, ++ 0, 8, NULL}, ++ ++ {"MI_MATH", OP_MI_MATH, F_LEN_VAR, R_ALL, D_ALL, 0, 8, NULL}, ++ ++ {"MI_URB_CLEAR", OP_MI_URB_CLEAR, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"MI_SEMAPHORE_SIGNAL", OP_MI_SEMAPHORE_SIGNAL, F_LEN_VAR, R_ALL, ++ D_BDW_PLUS, 0, 8, NULL}, ++ ++ {"MI_SEMAPHORE_WAIT", OP_MI_SEMAPHORE_WAIT, F_LEN_VAR, R_ALL, ++ D_BDW_PLUS, ADDR_FIX_1(2), 8, cmd_handler_mi_semaphore_wait}, ++ ++ {"MI_STORE_DATA_IMM", OP_MI_STORE_DATA_IMM, F_LEN_VAR, R_ALL, D_BDW_PLUS, ++ ADDR_FIX_1(1), 10, cmd_handler_mi_store_data_imm}, ++ ++ {"MI_STORE_DATA_INDEX", OP_MI_STORE_DATA_INDEX, F_LEN_VAR, R_ALL, D_ALL, ++ 0, 8, cmd_handler_mi_store_data_index}, ++ ++ {"MI_LOAD_REGISTER_IMM", OP_MI_LOAD_REGISTER_IMM, F_LEN_VAR, R_ALL, ++ D_ALL, 0, 8, cmd_handler_lri}, ++ ++ {"MI_UPDATE_GTT", OP_MI_UPDATE_GTT, F_LEN_VAR, R_ALL, D_BDW_PLUS, 0, 10, ++ cmd_handler_mi_update_gtt}, ++ ++ {"MI_STORE_REGISTER_MEM", OP_MI_STORE_REGISTER_MEM, F_LEN_VAR, R_ALL, ++ D_ALL, ADDR_FIX_1(2), 8, cmd_handler_srm}, ++ ++ {"MI_FLUSH_DW", OP_MI_FLUSH_DW, F_LEN_VAR, R_ALL, D_ALL, 0, 6, ++ cmd_handler_mi_flush_dw}, ++ ++ {"MI_CLFLUSH", OP_MI_CLFLUSH, F_LEN_VAR, R_ALL, D_ALL, ADDR_FIX_1(1), ++ 10, cmd_handler_mi_clflush}, ++ ++ {"MI_REPORT_PERF_COUNT", OP_MI_REPORT_PERF_COUNT, F_LEN_VAR, R_ALL, ++ D_ALL, ADDR_FIX_1(1), 6, cmd_handler_mi_report_perf_count}, ++ ++ {"MI_LOAD_REGISTER_MEM", OP_MI_LOAD_REGISTER_MEM, F_LEN_VAR, R_ALL, ++ D_ALL, ADDR_FIX_1(2), 8, cmd_handler_lrm}, ++ ++ {"MI_LOAD_REGISTER_REG", OP_MI_LOAD_REGISTER_REG, F_LEN_VAR, R_ALL, ++ D_ALL, 0, 8, cmd_handler_lrr}, ++ ++ {"MI_RS_STORE_DATA_IMM", OP_MI_RS_STORE_DATA_IMM, F_LEN_VAR, R_RCS, ++ D_ALL, 0, 8, NULL}, ++ ++ {"MI_LOAD_URB_MEM", OP_MI_LOAD_URB_MEM, F_LEN_VAR, R_RCS, D_ALL, ++ ADDR_FIX_1(2), 8, NULL}, ++ ++ {"MI_STORE_URM_MEM", OP_MI_STORE_URM_MEM, F_LEN_VAR, R_RCS, D_ALL, ++ ADDR_FIX_1(2), 8, NULL}, ++ ++ {"MI_OP_2E", OP_MI_2E, F_LEN_VAR, R_ALL, D_BDW_PLUS, ADDR_FIX_2(1, 2), ++ 8, cmd_handler_mi_op_2e}, ++ ++ {"MI_OP_2F", OP_MI_2F, F_LEN_VAR, R_ALL, D_BDW_PLUS, ADDR_FIX_1(1), ++ 8, cmd_handler_mi_op_2f}, ++ ++ {"MI_BATCH_BUFFER_START", OP_MI_BATCH_BUFFER_START, ++ F_IP_ADVANCE_CUSTOM, R_ALL, D_ALL, 0, 8, ++ cmd_handler_mi_batch_buffer_start}, ++ ++ {"MI_CONDITIONAL_BATCH_BUFFER_END", OP_MI_CONDITIONAL_BATCH_BUFFER_END, ++ F_LEN_VAR, R_ALL, D_ALL, ADDR_FIX_1(2), 8, ++ cmd_handler_mi_conditional_batch_buffer_end}, ++ ++ {"MI_LOAD_SCAN_LINES_INCL", OP_MI_LOAD_SCAN_LINES_INCL, F_LEN_CONST, ++ R_RCS | R_BCS, D_ALL, 0, 2, NULL}, ++ ++ {"XY_SETUP_BLT", OP_XY_SETUP_BLT, F_LEN_VAR, R_BCS, D_ALL, ++ ADDR_FIX_2(4, 7), 8, NULL}, ++ ++ {"XY_SETUP_CLIP_BLT", OP_XY_SETUP_CLIP_BLT, F_LEN_VAR, R_BCS, D_ALL, ++ 0, 8, NULL}, ++ ++ {"XY_SETUP_MONO_PATTERN_SL_BLT", OP_XY_SETUP_MONO_PATTERN_SL_BLT, ++ F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_1(4), 8, NULL}, ++ ++ {"XY_PIXEL_BLT", OP_XY_PIXEL_BLT, F_LEN_VAR, R_BCS, D_ALL, 0, 8, NULL}, ++ ++ {"XY_SCANLINES_BLT", OP_XY_SCANLINES_BLT, F_LEN_VAR, R_BCS, D_ALL, ++ 0, 8, NULL}, ++ ++ {"XY_TEXT_BLT", OP_XY_TEXT_BLT, F_LEN_VAR, R_BCS, D_ALL, ++ ADDR_FIX_1(3), 8, NULL}, ++ ++ {"XY_TEXT_IMMEDIATE_BLT", OP_XY_TEXT_IMMEDIATE_BLT, F_LEN_VAR, R_BCS, ++ D_ALL, 0, 8, NULL}, ++ ++ {"XY_COLOR_BLT", OP_XY_COLOR_BLT, F_LEN_VAR, R_BCS, D_ALL, ++ ADDR_FIX_1(4), 8, NULL}, ++ ++ {"XY_PAT_BLT", OP_XY_PAT_BLT, F_LEN_VAR, R_BCS, D_ALL, ++ ADDR_FIX_2(4, 5), 8, NULL}, ++ ++ {"XY_MONO_PAT_BLT", OP_XY_MONO_PAT_BLT, F_LEN_VAR, R_BCS, D_ALL, ++ ADDR_FIX_1(4), 8, NULL}, ++ ++ {"XY_SRC_COPY_BLT", OP_XY_SRC_COPY_BLT, F_LEN_VAR, R_BCS, D_ALL, ++ ADDR_FIX_2(4, 7), 8, NULL}, ++ ++ {"XY_MONO_SRC_COPY_BLT", OP_XY_MONO_SRC_COPY_BLT, F_LEN_VAR, R_BCS, ++ D_ALL, ADDR_FIX_2(4, 5), 8, NULL}, ++ ++ {"XY_FULL_BLT", OP_XY_FULL_BLT, F_LEN_VAR, R_BCS, D_ALL, 0, 8, NULL}, ++ ++ {"XY_FULL_MONO_SRC_BLT", OP_XY_FULL_MONO_SRC_BLT, F_LEN_VAR, R_BCS, ++ D_ALL, ADDR_FIX_3(4, 5, 8), 8, NULL}, ++ ++ {"XY_FULL_MONO_PATTERN_BLT", OP_XY_FULL_MONO_PATTERN_BLT, F_LEN_VAR, ++ R_BCS, D_ALL, ADDR_FIX_2(4, 7), 8, NULL}, ++ ++ {"XY_FULL_MONO_PATTERN_MONO_SRC_BLT", ++ OP_XY_FULL_MONO_PATTERN_MONO_SRC_BLT, ++ F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_2(4, 5), 8, NULL}, ++ ++ {"XY_MONO_PAT_FIXED_BLT", OP_XY_MONO_PAT_FIXED_BLT, F_LEN_VAR, R_BCS, ++ D_ALL, ADDR_FIX_1(4), 8, NULL}, ++ ++ {"XY_MONO_SRC_COPY_IMMEDIATE_BLT", OP_XY_MONO_SRC_COPY_IMMEDIATE_BLT, ++ F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_1(4), 8, NULL}, ++ ++ {"XY_PAT_BLT_IMMEDIATE", OP_XY_PAT_BLT_IMMEDIATE, F_LEN_VAR, R_BCS, ++ D_ALL, ADDR_FIX_1(4), 8, NULL}, ++ ++ {"XY_SRC_COPY_CHROMA_BLT", OP_XY_SRC_COPY_CHROMA_BLT, F_LEN_VAR, R_BCS, ++ D_ALL, ADDR_FIX_2(4, 7), 8, NULL}, ++ ++ {"XY_FULL_IMMEDIATE_PATTERN_BLT", OP_XY_FULL_IMMEDIATE_PATTERN_BLT, ++ F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_2(4, 7), 8, NULL}, ++ ++ {"XY_FULL_MONO_SRC_IMMEDIATE_PATTERN_BLT", ++ OP_XY_FULL_MONO_SRC_IMMEDIATE_PATTERN_BLT, ++ F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_2(4, 5), 8, NULL}, ++ ++ {"XY_PAT_CHROMA_BLT", OP_XY_PAT_CHROMA_BLT, F_LEN_VAR, R_BCS, D_ALL, ++ ADDR_FIX_2(4, 5), 8, NULL}, ++ ++ {"XY_PAT_CHROMA_BLT_IMMEDIATE", OP_XY_PAT_CHROMA_BLT_IMMEDIATE, ++ F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_1(4), 8, NULL}, ++ ++ {"3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP", ++ OP_3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_VIEWPORT_STATE_POINTERS_CC", ++ OP_3DSTATE_VIEWPORT_STATE_POINTERS_CC, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_BLEND_STATE_POINTERS", ++ OP_3DSTATE_BLEND_STATE_POINTERS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_DEPTH_STENCIL_STATE_POINTERS", ++ OP_3DSTATE_DEPTH_STENCIL_STATE_POINTERS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_BINDING_TABLE_POINTERS_VS", ++ OP_3DSTATE_BINDING_TABLE_POINTERS_VS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_BINDING_TABLE_POINTERS_HS", ++ OP_3DSTATE_BINDING_TABLE_POINTERS_HS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_BINDING_TABLE_POINTERS_DS", ++ OP_3DSTATE_BINDING_TABLE_POINTERS_DS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_BINDING_TABLE_POINTERS_GS", ++ OP_3DSTATE_BINDING_TABLE_POINTERS_GS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_BINDING_TABLE_POINTERS_PS", ++ OP_3DSTATE_BINDING_TABLE_POINTERS_PS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_SAMPLER_STATE_POINTERS_VS", ++ OP_3DSTATE_SAMPLER_STATE_POINTERS_VS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_SAMPLER_STATE_POINTERS_HS", ++ OP_3DSTATE_SAMPLER_STATE_POINTERS_HS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_SAMPLER_STATE_POINTERS_DS", ++ OP_3DSTATE_SAMPLER_STATE_POINTERS_DS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_SAMPLER_STATE_POINTERS_GS", ++ OP_3DSTATE_SAMPLER_STATE_POINTERS_GS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_SAMPLER_STATE_POINTERS_PS", ++ OP_3DSTATE_SAMPLER_STATE_POINTERS_PS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_URB_VS", OP_3DSTATE_URB_VS, F_LEN_VAR, R_RCS, D_ALL, ++ 0, 8, NULL}, ++ ++ {"3DSTATE_URB_HS", OP_3DSTATE_URB_HS, F_LEN_VAR, R_RCS, D_ALL, ++ 0, 8, NULL}, ++ ++ {"3DSTATE_URB_DS", OP_3DSTATE_URB_DS, F_LEN_VAR, R_RCS, D_ALL, ++ 0, 8, NULL}, ++ ++ {"3DSTATE_URB_GS", OP_3DSTATE_URB_GS, F_LEN_VAR, R_RCS, D_ALL, ++ 0, 8, NULL}, ++ ++ {"3DSTATE_GATHER_CONSTANT_VS", OP_3DSTATE_GATHER_CONSTANT_VS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_GATHER_CONSTANT_GS", OP_3DSTATE_GATHER_CONSTANT_GS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_GATHER_CONSTANT_HS", OP_3DSTATE_GATHER_CONSTANT_HS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_GATHER_CONSTANT_DS", OP_3DSTATE_GATHER_CONSTANT_DS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_GATHER_CONSTANT_PS", OP_3DSTATE_GATHER_CONSTANT_PS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_DX9_CONSTANTF_VS", OP_3DSTATE_DX9_CONSTANTF_VS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 11, NULL}, ++ ++ {"3DSTATE_DX9_CONSTANTF_PS", OP_3DSTATE_DX9_CONSTANTF_PS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 11, NULL}, ++ ++ {"3DSTATE_DX9_CONSTANTI_VS", OP_3DSTATE_DX9_CONSTANTI_VS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_DX9_CONSTANTI_PS", OP_3DSTATE_DX9_CONSTANTI_PS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_DX9_CONSTANTB_VS", OP_3DSTATE_DX9_CONSTANTB_VS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_DX9_CONSTANTB_PS", OP_3DSTATE_DX9_CONSTANTB_PS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_DX9_LOCAL_VALID_VS", OP_3DSTATE_DX9_LOCAL_VALID_VS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_DX9_LOCAL_VALID_PS", OP_3DSTATE_DX9_LOCAL_VALID_PS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_DX9_GENERATE_ACTIVE_VS", OP_3DSTATE_DX9_GENERATE_ACTIVE_VS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_DX9_GENERATE_ACTIVE_PS", OP_3DSTATE_DX9_GENERATE_ACTIVE_PS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_BINDING_TABLE_EDIT_VS", OP_3DSTATE_BINDING_TABLE_EDIT_VS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL}, ++ ++ {"3DSTATE_BINDING_TABLE_EDIT_GS", OP_3DSTATE_BINDING_TABLE_EDIT_GS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL}, ++ ++ {"3DSTATE_BINDING_TABLE_EDIT_HS", OP_3DSTATE_BINDING_TABLE_EDIT_HS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL}, ++ ++ {"3DSTATE_BINDING_TABLE_EDIT_DS", OP_3DSTATE_BINDING_TABLE_EDIT_DS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL}, ++ ++ {"3DSTATE_BINDING_TABLE_EDIT_PS", OP_3DSTATE_BINDING_TABLE_EDIT_PS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL}, ++ ++ {"3DSTATE_VF_INSTANCING", OP_3DSTATE_VF_INSTANCING, F_LEN_VAR, R_RCS, ++ D_BDW_PLUS, 0, 8, NULL}, ++ ++ {"3DSTATE_VF_SGVS", OP_3DSTATE_VF_SGVS, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8, ++ NULL}, ++ ++ {"3DSTATE_VF_TOPOLOGY", OP_3DSTATE_VF_TOPOLOGY, F_LEN_VAR, R_RCS, ++ D_BDW_PLUS, 0, 8, NULL}, ++ ++ {"3DSTATE_WM_CHROMAKEY", OP_3DSTATE_WM_CHROMAKEY, F_LEN_VAR, R_RCS, ++ D_BDW_PLUS, 0, 8, NULL}, ++ ++ {"3DSTATE_PS_BLEND", OP_3DSTATE_PS_BLEND, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, ++ 8, NULL}, ++ ++ {"3DSTATE_WM_DEPTH_STENCIL", OP_3DSTATE_WM_DEPTH_STENCIL, F_LEN_VAR, ++ R_RCS, D_BDW_PLUS, 0, 8, NULL}, ++ ++ {"3DSTATE_PS_EXTRA", OP_3DSTATE_PS_EXTRA, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, ++ 8, NULL}, ++ ++ {"3DSTATE_RASTER", OP_3DSTATE_RASTER, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8, ++ NULL}, ++ ++ {"3DSTATE_SBE_SWIZ", OP_3DSTATE_SBE_SWIZ, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8, ++ NULL}, ++ ++ {"3DSTATE_WM_HZ_OP", OP_3DSTATE_WM_HZ_OP, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8, ++ NULL}, ++ ++ {"3DSTATE_VERTEX_BUFFERS", OP_3DSTATE_VERTEX_BUFFERS, F_LEN_VAR, R_RCS, ++ D_BDW_PLUS, 0, 8, NULL}, ++ ++ {"3DSTATE_VERTEX_ELEMENTS", OP_3DSTATE_VERTEX_ELEMENTS, F_LEN_VAR, ++ R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_INDEX_BUFFER", OP_3DSTATE_INDEX_BUFFER, F_LEN_VAR, R_RCS, ++ D_BDW_PLUS, ADDR_FIX_1(2), 8, NULL}, ++ ++ {"3DSTATE_VF_STATISTICS", OP_3DSTATE_VF_STATISTICS, F_LEN_CONST, ++ R_RCS, D_ALL, 0, 1, NULL}, ++ ++ {"3DSTATE_VF", OP_3DSTATE_VF, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_CC_STATE_POINTERS", OP_3DSTATE_CC_STATE_POINTERS, F_LEN_VAR, ++ R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_SCISSOR_STATE_POINTERS", OP_3DSTATE_SCISSOR_STATE_POINTERS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_GS", OP_3DSTATE_GS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_CLIP", OP_3DSTATE_CLIP, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_WM", OP_3DSTATE_WM, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_CONSTANT_GS", OP_3DSTATE_CONSTANT_GS, F_LEN_VAR, R_RCS, ++ D_BDW_PLUS, 0, 8, NULL}, ++ ++ {"3DSTATE_CONSTANT_PS", OP_3DSTATE_CONSTANT_PS, F_LEN_VAR, R_RCS, ++ D_BDW_PLUS, 0, 8, NULL}, ++ ++ {"3DSTATE_SAMPLE_MASK", OP_3DSTATE_SAMPLE_MASK, F_LEN_VAR, R_RCS, ++ D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_CONSTANT_HS", OP_3DSTATE_CONSTANT_HS, F_LEN_VAR, R_RCS, ++ D_BDW_PLUS, 0, 8, NULL}, ++ ++ {"3DSTATE_CONSTANT_DS", OP_3DSTATE_CONSTANT_DS, F_LEN_VAR, R_RCS, ++ D_BDW_PLUS, 0, 8, NULL}, ++ ++ {"3DSTATE_HS", OP_3DSTATE_HS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_TE", OP_3DSTATE_TE, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_DS", OP_3DSTATE_DS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_STREAMOUT", OP_3DSTATE_STREAMOUT, F_LEN_VAR, R_RCS, ++ D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_SBE", OP_3DSTATE_SBE, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_PS", OP_3DSTATE_PS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_DRAWING_RECTANGLE", OP_3DSTATE_DRAWING_RECTANGLE, F_LEN_VAR, ++ R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_SAMPLER_PALETTE_LOAD0", OP_3DSTATE_SAMPLER_PALETTE_LOAD0, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_CHROMA_KEY", OP_3DSTATE_CHROMA_KEY, F_LEN_VAR, R_RCS, D_ALL, ++ 0, 8, NULL}, ++ ++ {"3DSTATE_DEPTH_BUFFER", OP_3DSTATE_DEPTH_BUFFER, F_LEN_VAR, R_RCS, ++ D_ALL, ADDR_FIX_1(2), 8, NULL}, ++ ++ {"3DSTATE_POLY_STIPPLE_OFFSET", OP_3DSTATE_POLY_STIPPLE_OFFSET, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_POLY_STIPPLE_PATTERN", OP_3DSTATE_POLY_STIPPLE_PATTERN, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_LINE_STIPPLE", OP_3DSTATE_LINE_STIPPLE, F_LEN_VAR, R_RCS, ++ D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_AA_LINE_PARAMS", OP_3DSTATE_AA_LINE_PARAMS, F_LEN_VAR, R_RCS, ++ D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_GS_SVB_INDEX", OP_3DSTATE_GS_SVB_INDEX, F_LEN_VAR, R_RCS, ++ D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_SAMPLER_PALETTE_LOAD1", OP_3DSTATE_SAMPLER_PALETTE_LOAD1, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_MULTISAMPLE", OP_3DSTATE_MULTISAMPLE_BDW, F_LEN_VAR, R_RCS, ++ D_BDW_PLUS, 0, 8, NULL}, ++ ++ {"3DSTATE_STENCIL_BUFFER", OP_3DSTATE_STENCIL_BUFFER, F_LEN_VAR, R_RCS, ++ D_ALL, ADDR_FIX_1(2), 8, NULL}, ++ ++ {"3DSTATE_HIER_DEPTH_BUFFER", OP_3DSTATE_HIER_DEPTH_BUFFER, F_LEN_VAR, ++ R_RCS, D_ALL, ADDR_FIX_1(2), 8, NULL}, ++ ++ {"3DSTATE_CLEAR_PARAMS", OP_3DSTATE_CLEAR_PARAMS, F_LEN_VAR, ++ R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_PUSH_CONSTANT_ALLOC_VS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_VS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_PUSH_CONSTANT_ALLOC_HS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_HS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_PUSH_CONSTANT_ALLOC_DS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_DS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_PUSH_CONSTANT_ALLOC_GS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_GS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_PUSH_CONSTANT_ALLOC_PS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_PS, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_MONOFILTER_SIZE", OP_3DSTATE_MONOFILTER_SIZE, F_LEN_VAR, ++ R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_SO_DECL_LIST", OP_3DSTATE_SO_DECL_LIST, F_LEN_VAR, R_RCS, ++ D_ALL, 0, 9, NULL}, ++ ++ {"3DSTATE_SO_BUFFER", OP_3DSTATE_SO_BUFFER, F_LEN_VAR, R_RCS, D_BDW_PLUS, ++ ADDR_FIX_2(2, 4), 8, NULL}, ++ ++ {"3DSTATE_BINDING_TABLE_POOL_ALLOC", ++ OP_3DSTATE_BINDING_TABLE_POOL_ALLOC, ++ F_LEN_VAR, R_RCS, D_BDW_PLUS, ADDR_FIX_1(1), 8, NULL}, ++ ++ {"3DSTATE_GATHER_POOL_ALLOC", OP_3DSTATE_GATHER_POOL_ALLOC, ++ F_LEN_VAR, R_RCS, D_BDW_PLUS, ADDR_FIX_1(1), 8, NULL}, ++ ++ {"3DSTATE_DX9_CONSTANT_BUFFER_POOL_ALLOC", ++ OP_3DSTATE_DX9_CONSTANT_BUFFER_POOL_ALLOC, ++ F_LEN_VAR, R_RCS, D_BDW_PLUS, ADDR_FIX_1(1), 8, NULL}, ++ ++ {"3DSTATE_SAMPLE_PATTERN", OP_3DSTATE_SAMPLE_PATTERN, F_LEN_VAR, R_RCS, ++ D_BDW_PLUS, 0, 8, NULL}, ++ ++ {"PIPE_CONTROL", OP_PIPE_CONTROL, F_LEN_VAR, R_RCS, D_ALL, ++ ADDR_FIX_1(2), 8, cmd_handler_pipe_control}, ++ ++ {"3DPRIMITIVE", OP_3DPRIMITIVE, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"PIPELINE_SELECT", OP_PIPELINE_SELECT, F_LEN_CONST, R_RCS, D_ALL, 0, ++ 1, NULL}, ++ ++ {"STATE_PREFETCH", OP_STATE_PREFETCH, F_LEN_VAR, R_RCS, D_ALL, ++ ADDR_FIX_1(1), 8, NULL}, ++ ++ {"STATE_SIP", OP_STATE_SIP, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"STATE_BASE_ADDRESS", OP_STATE_BASE_ADDRESS, F_LEN_VAR, R_RCS, D_BDW_PLUS, ++ ADDR_FIX_5(1, 3, 4, 5, 6), 8, NULL}, ++ ++ {"OP_3D_MEDIA_0_1_4", OP_3D_MEDIA_0_1_4, F_LEN_VAR, R_RCS, D_ALL, ++ ADDR_FIX_1(1), 8, NULL}, ++ ++ {"3DSTATE_VS", OP_3DSTATE_VS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_SF", OP_3DSTATE_SF, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, ++ ++ {"3DSTATE_CONSTANT_VS", OP_3DSTATE_CONSTANT_VS, F_LEN_VAR, R_RCS, D_BDW_PLUS, ++ 0, 8, NULL}, ++ ++ {"3DSTATE_COMPONENT_PACKING", OP_3DSTATE_COMPONENT_PACKING, F_LEN_VAR, R_RCS, ++ D_SKL_PLUS, 0, 8, NULL}, ++ ++ {"MEDIA_INTERFACE_DESCRIPTOR_LOAD", OP_MEDIA_INTERFACE_DESCRIPTOR_LOAD, ++ F_LEN_VAR, R_RCS, D_ALL, 0, 16, NULL}, ++ ++ {"MEDIA_GATEWAY_STATE", OP_MEDIA_GATEWAY_STATE, F_LEN_VAR, R_RCS, D_ALL, ++ 0, 16, NULL}, ++ ++ {"MEDIA_STATE_FLUSH", OP_MEDIA_STATE_FLUSH, F_LEN_VAR, R_RCS, D_ALL, ++ 0, 16, NULL}, ++ ++ {"MEDIA_POOL_STATE", OP_MEDIA_POOL_STATE, F_LEN_VAR, R_RCS, D_ALL, ++ 0, 16, NULL}, ++ ++ {"MEDIA_OBJECT", OP_MEDIA_OBJECT, F_LEN_VAR, R_RCS, D_ALL, 0, 16, NULL}, ++ ++ {"MEDIA_CURBE_LOAD", OP_MEDIA_CURBE_LOAD, F_LEN_VAR, R_RCS, D_ALL, ++ 0, 16, NULL}, ++ ++ {"MEDIA_OBJECT_PRT", OP_MEDIA_OBJECT_PRT, F_LEN_VAR, R_RCS, D_ALL, ++ 0, 16, NULL}, ++ ++ {"MEDIA_OBJECT_WALKER", OP_MEDIA_OBJECT_WALKER, F_LEN_VAR, R_RCS, D_ALL, ++ 0, 16, NULL}, ++ ++ {"GPGPU_WALKER", OP_GPGPU_WALKER, F_LEN_VAR, R_RCS, D_ALL, ++ 0, 8, NULL}, ++ ++ {"MEDIA_VFE_STATE", OP_MEDIA_VFE_STATE, F_LEN_VAR, R_RCS, D_ALL, 0, 16, ++ NULL}, ++ ++ {"3DSTATE_VF_STATISTICS_GM45", OP_3DSTATE_VF_STATISTICS_GM45, ++ F_LEN_CONST, R_ALL, D_ALL, 0, 1, NULL}, ++ ++ {"MFX_PIPE_MODE_SELECT", OP_MFX_PIPE_MODE_SELECT, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFX_SURFACE_STATE", OP_MFX_SURFACE_STATE, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFX_PIPE_BUF_ADDR_STATE", OP_MFX_PIPE_BUF_ADDR_STATE, F_LEN_VAR, ++ R_VCS, D_BDW_PLUS, 0, 12, NULL}, ++ ++ {"MFX_IND_OBJ_BASE_ADDR_STATE", OP_MFX_IND_OBJ_BASE_ADDR_STATE, ++ F_LEN_VAR, R_VCS, D_BDW_PLUS, 0, 12, NULL}, ++ ++ {"MFX_BSP_BUF_BASE_ADDR_STATE", OP_MFX_BSP_BUF_BASE_ADDR_STATE, ++ F_LEN_VAR, R_VCS, D_BDW_PLUS, ADDR_FIX_3(1, 3, 5), 12, NULL}, ++ ++ {"OP_2_0_0_5", OP_2_0_0_5, F_LEN_VAR, R_VCS, D_BDW_PLUS, 0, 12, NULL}, ++ ++ {"MFX_STATE_POINTER", OP_MFX_STATE_POINTER, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFX_QM_STATE", OP_MFX_QM_STATE, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFX_FQM_STATE", OP_MFX_FQM_STATE, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFX_PAK_INSERT_OBJECT", OP_MFX_PAK_INSERT_OBJECT, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFX_STITCH_OBJECT", OP_MFX_STITCH_OBJECT, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFD_IT_OBJECT", OP_MFD_IT_OBJECT, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFX_WAIT", OP_MFX_WAIT, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 6, NULL}, ++ ++ {"MFX_AVC_IMG_STATE", OP_MFX_AVC_IMG_STATE, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFX_AVC_QM_STATE", OP_MFX_AVC_QM_STATE, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFX_AVC_DIRECTMODE_STATE", OP_MFX_AVC_DIRECTMODE_STATE, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFX_AVC_SLICE_STATE", OP_MFX_AVC_SLICE_STATE, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFX_AVC_REF_IDX_STATE", OP_MFX_AVC_REF_IDX_STATE, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFX_AVC_WEIGHTOFFSET_STATE", OP_MFX_AVC_WEIGHTOFFSET_STATE, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFD_AVC_PICID_STATE", OP_MFD_AVC_PICID_STATE, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ {"MFD_AVC_DPB_STATE", OP_MFD_AVC_DPB_STATE, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFD_AVC_BSD_OBJECT", OP_MFD_AVC_BSD_OBJECT, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFD_AVC_SLICEADDR", OP_MFD_AVC_SLICEADDR, F_LEN_VAR, ++ R_VCS, D_ALL, ADDR_FIX_1(2), 12, NULL}, ++ ++ {"MFC_AVC_PAK_OBJECT", OP_MFC_AVC_PAK_OBJECT, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFX_VC1_PRED_PIPE_STATE", OP_MFX_VC1_PRED_PIPE_STATE, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFX_VC1_DIRECTMODE_STATE", OP_MFX_VC1_DIRECTMODE_STATE, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFD_VC1_SHORT_PIC_STATE", OP_MFD_VC1_SHORT_PIC_STATE, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFD_VC1_LONG_PIC_STATE", OP_MFD_VC1_LONG_PIC_STATE, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFD_VC1_BSD_OBJECT", OP_MFD_VC1_BSD_OBJECT, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFC_MPEG2_SLICEGROUP_STATE", OP_MFC_MPEG2_SLICEGROUP_STATE, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFC_MPEG2_PAK_OBJECT", OP_MFC_MPEG2_PAK_OBJECT, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFX_MPEG2_PIC_STATE", OP_MFX_MPEG2_PIC_STATE, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFX_MPEG2_QM_STATE", OP_MFX_MPEG2_QM_STATE, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFD_MPEG2_BSD_OBJECT", OP_MFD_MPEG2_BSD_OBJECT, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFX_2_6_0_0", OP_MFX_2_6_0_0, F_LEN_VAR, R_VCS, D_ALL, ++ 0, 16, NULL}, ++ ++ {"MFX_2_6_0_9", OP_MFX_2_6_0_9, F_LEN_VAR, R_VCS, D_ALL, 0, 16, NULL}, ++ ++ {"MFX_2_6_0_8", OP_MFX_2_6_0_8, F_LEN_VAR, R_VCS, D_ALL, 0, 16, NULL}, ++ ++ {"MFX_JPEG_PIC_STATE", OP_MFX_JPEG_PIC_STATE, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFX_JPEG_HUFF_TABLE_STATE", OP_MFX_JPEG_HUFF_TABLE_STATE, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"MFD_JPEG_BSD_OBJECT", OP_MFD_JPEG_BSD_OBJECT, F_LEN_VAR, ++ R_VCS, D_ALL, 0, 12, NULL}, ++ ++ {"VEBOX_STATE", OP_VEB_STATE, F_LEN_VAR, R_VECS, D_ALL, 0, 12, NULL}, ++ ++ {"VEBOX_SURFACE_STATE", OP_VEB_SURFACE_STATE, F_LEN_VAR, R_VECS, D_ALL, ++ 0, 12, NULL}, ++ ++ {"VEB_DI_IECP", OP_VEB_DNDI_IECP_STATE, F_LEN_VAR, R_VECS, D_BDW_PLUS, ++ 0, 12, NULL}, ++}; ++ ++static void add_cmd_entry(struct intel_gvt *gvt, struct cmd_entry *e) ++{ ++ hash_add(gvt->cmd_table, &e->hlist, e->info->opcode); ++} ++ ++/* call the cmd handler, and advance ip */ ++static int cmd_parser_exec(struct parser_exec_state *s) ++{ ++ struct intel_vgpu *vgpu = s->vgpu; ++ const struct cmd_info *info; ++ u32 cmd; ++ int ret = 0; ++ ++ cmd = cmd_val(s, 0); ++ ++ /* fastpath for MI_NOOP */ ++ if (cmd == MI_NOOP) ++ info = &cmd_info[mi_noop_index]; ++ else ++ info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id); ++ ++ if (info == NULL) { ++ gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x, addr_type=%s, ring %d, workload=%p\n", ++ cmd, get_opcode(cmd, s->ring_id), ++ (s->buf_addr_type == PPGTT_BUFFER) ? ++ "ppgtt" : "ggtt", s->ring_id, s->workload); ++ return -EBADRQC; ++ } ++ ++ s->info = info; ++ ++ trace_gvt_command(vgpu->id, s->ring_id, s->ip_gma, s->ip_va, ++ cmd_length(s), s->buf_type, s->buf_addr_type, ++ s->workload, info->name); ++ ++ if (info->handler) { ++ ret = info->handler(s); ++ if (ret < 0) { ++ gvt_vgpu_err("%s handler error\n", info->name); ++ return ret; ++ } ++ } ++ ++ if (!(info->flag & F_IP_ADVANCE_CUSTOM)) { ++ ret = cmd_advance_default(s); ++ if (ret) { ++ gvt_vgpu_err("%s IP advance error\n", info->name); ++ return ret; ++ } ++ } ++ return 0; ++} ++ ++static inline bool gma_out_of_range(unsigned long gma, ++ unsigned long gma_head, unsigned int gma_tail) ++{ ++ if (gma_tail >= gma_head) ++ return (gma < gma_head) || (gma > gma_tail); ++ else ++ return (gma > gma_tail) && (gma < gma_head); ++} ++ ++/* Keep the consistent return type, e.g EBADRQC for unknown ++ * cmd, EFAULT for invalid address, EPERM for nonpriv. later ++ * works as the input of VM healthy status. ++ */ ++static int command_scan(struct parser_exec_state *s, ++ unsigned long rb_head, unsigned long rb_tail, ++ unsigned long rb_start, unsigned long rb_len) ++{ ++ ++ unsigned long gma_head, gma_tail, gma_bottom; ++ int ret = 0; ++ struct intel_vgpu *vgpu = s->vgpu; ++ ++ gma_head = rb_start + rb_head; ++ gma_tail = rb_start + rb_tail; ++ gma_bottom = rb_start + rb_len; ++ ++ while (s->ip_gma != gma_tail) { ++ if (s->buf_type == RING_BUFFER_INSTRUCTION) { ++ if (!(s->ip_gma >= rb_start) || ++ !(s->ip_gma < gma_bottom)) { ++ gvt_vgpu_err("ip_gma %lx out of ring scope." ++ "(base:0x%lx, bottom: 0x%lx)\n", ++ s->ip_gma, rb_start, ++ gma_bottom); ++ parser_exec_state_dump(s); ++ return -EFAULT; ++ } ++ if (gma_out_of_range(s->ip_gma, gma_head, gma_tail)) { ++ gvt_vgpu_err("ip_gma %lx out of range." ++ "base 0x%lx head 0x%lx tail 0x%lx\n", ++ s->ip_gma, rb_start, ++ rb_head, rb_tail); ++ parser_exec_state_dump(s); ++ break; ++ } ++ } ++ ret = cmd_parser_exec(s); ++ if (ret) { ++ gvt_vgpu_err("cmd parser error\n"); ++ parser_exec_state_dump(s); ++ break; ++ } ++ } ++ ++ return ret; ++} ++ ++static int scan_workload(struct intel_vgpu_workload *workload) ++{ ++ unsigned long gma_head, gma_tail, gma_bottom; ++ struct parser_exec_state s; ++ int ret = 0; ++ ++ /* ring base is page aligned */ ++ if (WARN_ON(!IS_ALIGNED(workload->rb_start, I915_GTT_PAGE_SIZE))) ++ return -EINVAL; ++ ++ gma_head = workload->rb_start + workload->rb_head; ++ gma_tail = workload->rb_start + workload->rb_tail; ++ gma_bottom = workload->rb_start + _RING_CTL_BUF_SIZE(workload->rb_ctl); ++ ++ s.buf_type = RING_BUFFER_INSTRUCTION; ++ s.buf_addr_type = GTT_BUFFER; ++ s.vgpu = workload->vgpu; ++ s.ring_id = workload->ring_id; ++ s.ring_start = workload->rb_start; ++ s.ring_size = _RING_CTL_BUF_SIZE(workload->rb_ctl); ++ s.ring_head = gma_head; ++ s.ring_tail = gma_tail; ++ s.rb_va = workload->shadow_ring_buffer_va; ++ s.workload = workload; ++ s.is_ctx_wa = false; ++ ++ if ((bypass_scan_mask & (1 << workload->ring_id)) || ++ gma_head == gma_tail) ++ return 0; ++ ++ if (!intel_gvt_ggtt_validate_range(s.vgpu, s.ring_start, s.ring_size)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ ret = ip_gma_set(&s, gma_head); ++ if (ret) ++ goto out; ++ ++ ret = command_scan(&s, workload->rb_head, workload->rb_tail, ++ workload->rb_start, _RING_CTL_BUF_SIZE(workload->rb_ctl)); ++ ++out: ++ return ret; ++} ++ ++static int scan_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) ++{ ++ ++ unsigned long gma_head, gma_tail, gma_bottom, ring_size, ring_tail; ++ struct parser_exec_state s; ++ int ret = 0; ++ struct intel_vgpu_workload *workload = container_of(wa_ctx, ++ struct intel_vgpu_workload, ++ wa_ctx); ++ ++ /* ring base is page aligned */ ++ if (WARN_ON(!IS_ALIGNED(wa_ctx->indirect_ctx.guest_gma, ++ I915_GTT_PAGE_SIZE))) ++ return -EINVAL; ++ ++ ring_tail = wa_ctx->indirect_ctx.size + 3 * sizeof(u32); ++ ring_size = round_up(wa_ctx->indirect_ctx.size + CACHELINE_BYTES, ++ PAGE_SIZE); ++ gma_head = wa_ctx->indirect_ctx.guest_gma; ++ gma_tail = wa_ctx->indirect_ctx.guest_gma + ring_tail; ++ gma_bottom = wa_ctx->indirect_ctx.guest_gma + ring_size; ++ ++ s.buf_type = RING_BUFFER_INSTRUCTION; ++ s.buf_addr_type = GTT_BUFFER; ++ s.vgpu = workload->vgpu; ++ s.ring_id = workload->ring_id; ++ s.ring_start = wa_ctx->indirect_ctx.guest_gma; ++ s.ring_size = ring_size; ++ s.ring_head = gma_head; ++ s.ring_tail = gma_tail; ++ s.rb_va = wa_ctx->indirect_ctx.shadow_va; ++ s.workload = workload; ++ s.is_ctx_wa = true; ++ ++ if (!intel_gvt_ggtt_validate_range(s.vgpu, s.ring_start, s.ring_size)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ ret = ip_gma_set(&s, gma_head); ++ if (ret) ++ goto out; ++ ++ ret = command_scan(&s, 0, ring_tail, ++ wa_ctx->indirect_ctx.guest_gma, ring_size); ++out: ++ return ret; ++} ++ ++static int shadow_workload_ring_buffer(struct intel_vgpu_workload *workload) ++{ ++ struct intel_vgpu *vgpu = workload->vgpu; ++ struct intel_vgpu_submission *s = &vgpu->submission; ++ unsigned long gma_head, gma_tail, gma_top, guest_rb_size; ++ void *shadow_ring_buffer_va; ++ int ring_id = workload->ring_id; ++ int ret; ++ ++ guest_rb_size = _RING_CTL_BUF_SIZE(workload->rb_ctl); ++ ++ /* calculate workload ring buffer size */ ++ workload->rb_len = (workload->rb_tail + guest_rb_size - ++ workload->rb_head) % guest_rb_size; ++ ++ gma_head = workload->rb_start + workload->rb_head; ++ gma_tail = workload->rb_start + workload->rb_tail; ++ gma_top = workload->rb_start + guest_rb_size; ++ ++ if (workload->rb_len > s->ring_scan_buffer_size[ring_id]) { ++ void *p; ++ ++ /* realloc the new ring buffer if needed */ ++ p = krealloc(s->ring_scan_buffer[ring_id], workload->rb_len, ++ GFP_KERNEL); ++ if (!p) { ++ gvt_vgpu_err("fail to re-alloc ring scan buffer\n"); ++ return -ENOMEM; ++ } ++ s->ring_scan_buffer[ring_id] = p; ++ s->ring_scan_buffer_size[ring_id] = workload->rb_len; ++ } ++ ++ shadow_ring_buffer_va = s->ring_scan_buffer[ring_id]; ++ ++ /* get shadow ring buffer va */ ++ workload->shadow_ring_buffer_va = shadow_ring_buffer_va; ++ ++ /* head > tail --> copy head <-> top */ ++ if (gma_head > gma_tail) { ++ ret = copy_gma_to_hva(vgpu, vgpu->gtt.ggtt_mm, ++ gma_head, gma_top, shadow_ring_buffer_va); ++ if (ret < 0) { ++ gvt_vgpu_err("fail to copy guest ring buffer\n"); ++ return ret; ++ } ++ shadow_ring_buffer_va += ret; ++ gma_head = workload->rb_start; ++ } ++ ++ /* copy head or start <-> tail */ ++ ret = copy_gma_to_hva(vgpu, vgpu->gtt.ggtt_mm, gma_head, gma_tail, ++ shadow_ring_buffer_va); ++ if (ret < 0) { ++ gvt_vgpu_err("fail to copy guest ring buffer\n"); ++ return ret; ++ } ++ return 0; ++} ++ ++int intel_gvt_scan_and_shadow_ringbuffer(struct intel_vgpu_workload *workload) ++{ ++ int ret; ++ struct intel_vgpu *vgpu = workload->vgpu; ++ ++ ret = shadow_workload_ring_buffer(workload); ++ if (ret) { ++ gvt_vgpu_err("fail to shadow workload ring_buffer\n"); ++ return ret; ++ } ++ ++ ret = scan_workload(workload); ++ if (ret) { ++ gvt_vgpu_err("scan workload error\n"); ++ return ret; ++ } ++ return 0; ++} ++ ++static int shadow_indirect_ctx(struct intel_shadow_wa_ctx *wa_ctx) ++{ ++ int ctx_size = wa_ctx->indirect_ctx.size; ++ unsigned long guest_gma = wa_ctx->indirect_ctx.guest_gma; ++ struct intel_vgpu_workload *workload = container_of(wa_ctx, ++ struct intel_vgpu_workload, ++ wa_ctx); ++ struct intel_vgpu *vgpu = workload->vgpu; ++ struct drm_i915_gem_object *obj; ++ int ret = 0; ++ void *map; ++ ++ obj = i915_gem_object_create(workload->vgpu->gvt->dev_priv, ++ roundup(ctx_size + CACHELINE_BYTES, ++ PAGE_SIZE)); ++ if (IS_ERR(obj)) ++ return PTR_ERR(obj); ++ ++ /* get the va of the shadow batch buffer */ ++ map = i915_gem_object_pin_map(obj, I915_MAP_WB); ++ if (IS_ERR(map)) { ++ gvt_vgpu_err("failed to vmap shadow indirect ctx\n"); ++ ret = PTR_ERR(map); ++ goto put_obj; ++ } ++ ++ ret = i915_gem_object_set_to_cpu_domain(obj, false); ++ if (ret) { ++ gvt_vgpu_err("failed to set shadow indirect ctx to CPU\n"); ++ goto unmap_src; ++ } ++ ++ ret = copy_gma_to_hva(workload->vgpu, ++ workload->vgpu->gtt.ggtt_mm, ++ guest_gma, guest_gma + ctx_size, ++ map); ++ if (ret < 0) { ++ gvt_vgpu_err("fail to copy guest indirect ctx\n"); ++ goto unmap_src; ++ } ++ ++ wa_ctx->indirect_ctx.obj = obj; ++ wa_ctx->indirect_ctx.shadow_va = map; ++ return 0; ++ ++unmap_src: ++ i915_gem_object_unpin_map(obj); ++put_obj: ++ i915_gem_object_put(obj); ++ return ret; ++} ++ ++static int combine_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) ++{ ++ u32 per_ctx_start[CACHELINE_DWORDS] = {0}; ++ unsigned char *bb_start_sva; ++ ++ if (!wa_ctx->per_ctx.valid) ++ return 0; ++ ++ per_ctx_start[0] = 0x18800001; ++ per_ctx_start[1] = wa_ctx->per_ctx.guest_gma; ++ ++ bb_start_sva = (unsigned char *)wa_ctx->indirect_ctx.shadow_va + ++ wa_ctx->indirect_ctx.size; ++ ++ memcpy(bb_start_sva, per_ctx_start, CACHELINE_BYTES); ++ ++ return 0; ++} ++ ++int intel_gvt_scan_and_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) ++{ ++ int ret; ++ struct intel_vgpu_workload *workload = container_of(wa_ctx, ++ struct intel_vgpu_workload, ++ wa_ctx); ++ struct intel_vgpu *vgpu = workload->vgpu; ++ ++ if (wa_ctx->indirect_ctx.size == 0) ++ return 0; ++ ++ ret = shadow_indirect_ctx(wa_ctx); ++ if (ret) { ++ gvt_vgpu_err("fail to shadow indirect ctx\n"); ++ return ret; ++ } ++ ++ combine_wa_ctx(wa_ctx); ++ ++ ret = scan_wa_ctx(wa_ctx); ++ if (ret) { ++ gvt_vgpu_err("scan wa ctx error\n"); ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static const struct cmd_info *find_cmd_entry_any_ring(struct intel_gvt *gvt, ++ unsigned int opcode, unsigned long rings) ++{ ++ const struct cmd_info *info = NULL; ++ unsigned int ring; ++ ++ for_each_set_bit(ring, &rings, I915_NUM_ENGINES) { ++ info = find_cmd_entry(gvt, opcode, ring); ++ if (info) ++ break; ++ } ++ return info; ++} ++ ++static int init_cmd_table(struct intel_gvt *gvt) ++{ ++ int i; ++ struct cmd_entry *e; ++ const struct cmd_info *info; ++ unsigned int gen_type; ++ ++ gen_type = intel_gvt_get_device_type(gvt); ++ ++ for (i = 0; i < ARRAY_SIZE(cmd_info); i++) { ++ if (!(cmd_info[i].devices & gen_type)) ++ continue; ++ ++ e = kzalloc(sizeof(*e), GFP_KERNEL); ++ if (!e) ++ return -ENOMEM; ++ ++ e->info = &cmd_info[i]; ++ info = find_cmd_entry_any_ring(gvt, ++ e->info->opcode, e->info->rings); ++ if (info) { ++ gvt_err("%s %s duplicated\n", e->info->name, ++ info->name); ++ kfree(e); ++ return -EEXIST; ++ } ++ if (cmd_info[i].opcode == OP_MI_NOOP) ++ mi_noop_index = i; ++ ++ INIT_HLIST_NODE(&e->hlist); ++ add_cmd_entry(gvt, e); ++ gvt_dbg_cmd("add %-30s op %04x flag %x devs %02x rings %02x\n", ++ e->info->name, e->info->opcode, e->info->flag, ++ e->info->devices, e->info->rings); ++ } ++ return 0; ++} ++ ++static void clean_cmd_table(struct intel_gvt *gvt) ++{ ++ struct hlist_node *tmp; ++ struct cmd_entry *e; ++ int i; ++ ++ hash_for_each_safe(gvt->cmd_table, i, tmp, e, hlist) ++ kfree(e); ++ ++ hash_init(gvt->cmd_table); ++} ++ ++void intel_gvt_clean_cmd_parser(struct intel_gvt *gvt) ++{ ++ clean_cmd_table(gvt); ++} ++ ++int intel_gvt_init_cmd_parser(struct intel_gvt *gvt) ++{ ++ int ret; ++ ++ ret = init_cmd_table(gvt); ++ if (ret) { ++ intel_gvt_clean_cmd_parser(gvt); ++ return ret; ++ } ++ return 0; ++} +diff --git a/drivers/gpu/drm/i915_legacy/gvt/cmd_parser.h b/drivers/gpu/drm/i915_legacy/gvt/cmd_parser.h +new file mode 100644 +index 000000000000..286703643002 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/cmd_parser.h +@@ -0,0 +1,49 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Ke Yu ++ * Kevin Tian ++ * Zhiyuan Lv ++ * ++ * Contributors: ++ * Min He ++ * Ping Gao ++ * Tina Zhang ++ * Yulei Zhang ++ * Zhi Wang ++ * ++ */ ++#ifndef _GVT_CMD_PARSER_H_ ++#define _GVT_CMD_PARSER_H_ ++ ++#define GVT_CMD_HASH_BITS 7 ++ ++void intel_gvt_clean_cmd_parser(struct intel_gvt *gvt); ++ ++int intel_gvt_init_cmd_parser(struct intel_gvt *gvt); ++ ++int intel_gvt_scan_and_shadow_ringbuffer(struct intel_vgpu_workload *workload); ++ ++int intel_gvt_scan_and_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/gvt/debug.h b/drivers/gpu/drm/i915_legacy/gvt/debug.h +new file mode 100644 +index 000000000000..c6027125c1ec +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/debug.h +@@ -0,0 +1,65 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ */ ++ ++#ifndef __GVT_DEBUG_H__ ++#define __GVT_DEBUG_H__ ++ ++#define gvt_err(fmt, args...) \ ++ pr_err("gvt: "fmt, ##args) ++ ++#define gvt_vgpu_err(fmt, args...) \ ++do { \ ++ if (IS_ERR_OR_NULL(vgpu)) \ ++ pr_err("gvt: "fmt, ##args); \ ++ else \ ++ pr_err("gvt: vgpu %d: "fmt, vgpu->id, ##args);\ ++} while (0) ++ ++#define gvt_dbg_core(fmt, args...) \ ++ pr_debug("gvt: core: "fmt, ##args) ++ ++#define gvt_dbg_irq(fmt, args...) \ ++ pr_debug("gvt: irq: "fmt, ##args) ++ ++#define gvt_dbg_mm(fmt, args...) \ ++ pr_debug("gvt: mm: "fmt, ##args) ++ ++#define gvt_dbg_mmio(fmt, args...) \ ++ pr_debug("gvt: mmio: "fmt, ##args) ++ ++#define gvt_dbg_dpy(fmt, args...) \ ++ pr_debug("gvt: dpy: "fmt, ##args) ++ ++#define gvt_dbg_el(fmt, args...) \ ++ pr_debug("gvt: el: "fmt, ##args) ++ ++#define gvt_dbg_sched(fmt, args...) \ ++ pr_debug("gvt: sched: "fmt, ##args) ++ ++#define gvt_dbg_render(fmt, args...) \ ++ pr_debug("gvt: render: "fmt, ##args) ++ ++#define gvt_dbg_cmd(fmt, args...) \ ++ pr_debug("gvt: cmd: "fmt, ##args) ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/gvt/debugfs.c b/drivers/gpu/drm/i915_legacy/gvt/debugfs.c +new file mode 100644 +index 000000000000..8a9606f91e68 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/debugfs.c +@@ -0,0 +1,268 @@ ++/* ++ * Copyright(c) 2011-2017 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ */ ++#include ++#include ++#include "i915_drv.h" ++#include "gvt.h" ++ ++struct mmio_diff_param { ++ struct intel_vgpu *vgpu; ++ int total; ++ int diff; ++ struct list_head diff_mmio_list; ++}; ++ ++struct diff_mmio { ++ struct list_head node; ++ u32 offset; ++ u32 preg; ++ u32 vreg; ++}; ++ ++/* Compare two diff_mmio items. */ ++static int mmio_offset_compare(void *priv, ++ struct list_head *a, struct list_head *b) ++{ ++ struct diff_mmio *ma; ++ struct diff_mmio *mb; ++ ++ ma = container_of(a, struct diff_mmio, node); ++ mb = container_of(b, struct diff_mmio, node); ++ if (ma->offset < mb->offset) ++ return -1; ++ else if (ma->offset > mb->offset) ++ return 1; ++ return 0; ++} ++ ++static inline int mmio_diff_handler(struct intel_gvt *gvt, ++ u32 offset, void *data) ++{ ++ struct drm_i915_private *dev_priv = gvt->dev_priv; ++ struct mmio_diff_param *param = data; ++ struct diff_mmio *node; ++ u32 preg, vreg; ++ ++ preg = I915_READ_NOTRACE(_MMIO(offset)); ++ vreg = vgpu_vreg(param->vgpu, offset); ++ ++ if (preg != vreg) { ++ node = kmalloc(sizeof(*node), GFP_KERNEL); ++ if (!node) ++ return -ENOMEM; ++ ++ node->offset = offset; ++ node->preg = preg; ++ node->vreg = vreg; ++ list_add(&node->node, ¶m->diff_mmio_list); ++ param->diff++; ++ } ++ param->total++; ++ return 0; ++} ++ ++/* Show the all the different values of tracked mmio. */ ++static int vgpu_mmio_diff_show(struct seq_file *s, void *unused) ++{ ++ struct intel_vgpu *vgpu = s->private; ++ struct intel_gvt *gvt = vgpu->gvt; ++ struct mmio_diff_param param = { ++ .vgpu = vgpu, ++ .total = 0, ++ .diff = 0, ++ }; ++ struct diff_mmio *node, *next; ++ ++ INIT_LIST_HEAD(¶m.diff_mmio_list); ++ ++ mutex_lock(&gvt->lock); ++ spin_lock_bh(&gvt->scheduler.mmio_context_lock); ++ ++ mmio_hw_access_pre(gvt->dev_priv); ++ /* Recognize all the diff mmios to list. */ ++ intel_gvt_for_each_tracked_mmio(gvt, mmio_diff_handler, ¶m); ++ mmio_hw_access_post(gvt->dev_priv); ++ ++ spin_unlock_bh(&gvt->scheduler.mmio_context_lock); ++ mutex_unlock(&gvt->lock); ++ ++ /* In an ascending order by mmio offset. */ ++ list_sort(NULL, ¶m.diff_mmio_list, mmio_offset_compare); ++ ++ seq_printf(s, "%-8s %-8s %-8s %-8s\n", "Offset", "HW", "vGPU", "Diff"); ++ list_for_each_entry_safe(node, next, ¶m.diff_mmio_list, node) { ++ u32 diff = node->preg ^ node->vreg; ++ ++ seq_printf(s, "%08x %08x %08x %*pbl\n", ++ node->offset, node->preg, node->vreg, ++ 32, &diff); ++ list_del(&node->node); ++ kfree(node); ++ } ++ seq_printf(s, "Total: %d, Diff: %d\n", param.total, param.diff); ++ return 0; ++} ++DEFINE_SHOW_ATTRIBUTE(vgpu_mmio_diff); ++ ++static int ++vgpu_scan_nonprivbb_get(void *data, u64 *val) ++{ ++ struct intel_vgpu *vgpu = (struct intel_vgpu *)data; ++ *val = vgpu->scan_nonprivbb; ++ return 0; ++} ++ ++/* ++ * set/unset bit engine_id of vgpu->scan_nonprivbb to turn on/off scanning ++ * of non-privileged batch buffer. e.g. ++ * if vgpu->scan_nonprivbb=3, then it will scan non-privileged batch buffer ++ * on engine 0 and 1. ++ */ ++static int ++vgpu_scan_nonprivbb_set(void *data, u64 val) ++{ ++ struct intel_vgpu *vgpu = (struct intel_vgpu *)data; ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ enum intel_engine_id id; ++ char buf[128], *s; ++ int len; ++ ++ val &= (1 << I915_NUM_ENGINES) - 1; ++ ++ if (vgpu->scan_nonprivbb == val) ++ return 0; ++ ++ if (!val) ++ goto done; ++ ++ len = sprintf(buf, ++ "gvt: vgpu %d turns on non-privileged batch buffers scanning on Engines:", ++ vgpu->id); ++ ++ s = buf + len; ++ ++ for (id = 0; id < I915_NUM_ENGINES; id++) { ++ struct intel_engine_cs *engine; ++ ++ engine = dev_priv->engine[id]; ++ if (engine && (val & (1 << id))) { ++ len = snprintf(s, 4, "%d, ", engine->id); ++ s += len; ++ } else ++ val &= ~(1 << id); ++ } ++ ++ if (val) ++ sprintf(s, "low performance expected."); ++ ++ pr_warn("%s\n", buf); ++ ++done: ++ vgpu->scan_nonprivbb = val; ++ return 0; ++} ++ ++DEFINE_SIMPLE_ATTRIBUTE(vgpu_scan_nonprivbb_fops, ++ vgpu_scan_nonprivbb_get, vgpu_scan_nonprivbb_set, ++ "0x%llx\n"); ++ ++/** ++ * intel_gvt_debugfs_add_vgpu - register debugfs entries for a vGPU ++ * @vgpu: a vGPU ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++int intel_gvt_debugfs_add_vgpu(struct intel_vgpu *vgpu) ++{ ++ struct dentry *ent; ++ char name[16] = ""; ++ ++ snprintf(name, 16, "vgpu%d", vgpu->id); ++ vgpu->debugfs = debugfs_create_dir(name, vgpu->gvt->debugfs_root); ++ if (!vgpu->debugfs) ++ return -ENOMEM; ++ ++ ent = debugfs_create_bool("active", 0444, vgpu->debugfs, ++ &vgpu->active); ++ if (!ent) ++ return -ENOMEM; ++ ++ ent = debugfs_create_file("mmio_diff", 0444, vgpu->debugfs, ++ vgpu, &vgpu_mmio_diff_fops); ++ if (!ent) ++ return -ENOMEM; ++ ++ ent = debugfs_create_file("scan_nonprivbb", 0644, vgpu->debugfs, ++ vgpu, &vgpu_scan_nonprivbb_fops); ++ if (!ent) ++ return -ENOMEM; ++ ++ return 0; ++} ++ ++/** ++ * intel_gvt_debugfs_remove_vgpu - remove debugfs entries of a vGPU ++ * @vgpu: a vGPU ++ */ ++void intel_gvt_debugfs_remove_vgpu(struct intel_vgpu *vgpu) ++{ ++ debugfs_remove_recursive(vgpu->debugfs); ++ vgpu->debugfs = NULL; ++} ++ ++/** ++ * intel_gvt_debugfs_init - register gvt debugfs root entry ++ * @gvt: GVT device ++ * ++ * Returns: ++ * zero on success, negative if failed. ++ */ ++int intel_gvt_debugfs_init(struct intel_gvt *gvt) ++{ ++ struct drm_minor *minor = gvt->dev_priv->drm.primary; ++ struct dentry *ent; ++ ++ gvt->debugfs_root = debugfs_create_dir("gvt", minor->debugfs_root); ++ if (!gvt->debugfs_root) { ++ gvt_err("Cannot create debugfs dir\n"); ++ return -ENOMEM; ++ } ++ ++ ent = debugfs_create_ulong("num_tracked_mmio", 0444, gvt->debugfs_root, ++ &gvt->mmio.num_tracked_mmio); ++ if (!ent) ++ return -ENOMEM; ++ ++ return 0; ++} ++ ++/** ++ * intel_gvt_debugfs_clean - remove debugfs entries ++ * @gvt: GVT device ++ */ ++void intel_gvt_debugfs_clean(struct intel_gvt *gvt) ++{ ++ debugfs_remove_recursive(gvt->debugfs_root); ++ gvt->debugfs_root = NULL; ++} +diff --git a/drivers/gpu/drm/i915_legacy/gvt/display.c b/drivers/gpu/drm/i915_legacy/gvt/display.c +new file mode 100644 +index 000000000000..e1c313da6c00 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/display.c +@@ -0,0 +1,531 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Ke Yu ++ * Zhiyuan Lv ++ * ++ * Contributors: ++ * Terrence Xu ++ * Changbin Du ++ * Bing Niu ++ * Zhi Wang ++ * ++ */ ++ ++#include "i915_drv.h" ++#include "gvt.h" ++ ++static int get_edp_pipe(struct intel_vgpu *vgpu) ++{ ++ u32 data = vgpu_vreg(vgpu, _TRANS_DDI_FUNC_CTL_EDP); ++ int pipe = -1; ++ ++ switch (data & TRANS_DDI_EDP_INPUT_MASK) { ++ case TRANS_DDI_EDP_INPUT_A_ON: ++ case TRANS_DDI_EDP_INPUT_A_ONOFF: ++ pipe = PIPE_A; ++ break; ++ case TRANS_DDI_EDP_INPUT_B_ONOFF: ++ pipe = PIPE_B; ++ break; ++ case TRANS_DDI_EDP_INPUT_C_ONOFF: ++ pipe = PIPE_C; ++ break; ++ } ++ return pipe; ++} ++ ++static int edp_pipe_is_enabled(struct intel_vgpu *vgpu) ++{ ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ ++ if (!(vgpu_vreg_t(vgpu, PIPECONF(_PIPE_EDP)) & PIPECONF_ENABLE)) ++ return 0; ++ ++ if (!(vgpu_vreg(vgpu, _TRANS_DDI_FUNC_CTL_EDP) & TRANS_DDI_FUNC_ENABLE)) ++ return 0; ++ return 1; ++} ++ ++int pipe_is_enabled(struct intel_vgpu *vgpu, int pipe) ++{ ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ ++ if (WARN_ON(pipe < PIPE_A || pipe >= I915_MAX_PIPES)) ++ return -EINVAL; ++ ++ if (vgpu_vreg_t(vgpu, PIPECONF(pipe)) & PIPECONF_ENABLE) ++ return 1; ++ ++ if (edp_pipe_is_enabled(vgpu) && ++ get_edp_pipe(vgpu) == pipe) ++ return 1; ++ return 0; ++} ++ ++static unsigned char virtual_dp_monitor_edid[GVT_EDID_NUM][EDID_SIZE] = { ++ { ++/* EDID with 1024x768 as its resolution */ ++ /*Header*/ ++ 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, ++ /* Vendor & Product Identification */ ++ 0x22, 0xf0, 0x54, 0x29, 0x00, 0x00, 0x00, 0x00, 0x04, 0x17, ++ /* Version & Revision */ ++ 0x01, 0x04, ++ /* Basic Display Parameters & Features */ ++ 0xa5, 0x34, 0x20, 0x78, 0x23, ++ /* Color Characteristics */ ++ 0xfc, 0x81, 0xa4, 0x55, 0x4d, 0x9d, 0x25, 0x12, 0x50, 0x54, ++ /* Established Timings: maximum resolution is 1024x768 */ ++ 0x21, 0x08, 0x00, ++ /* Standard Timings. All invalid */ ++ 0x00, 0xc0, 0x00, 0xc0, 0x00, 0x40, 0x00, 0x80, 0x00, 0x00, ++ 0x00, 0x40, 0x00, 0x00, 0x00, 0x01, ++ /* 18 Byte Data Blocks 1: invalid */ ++ 0x00, 0x00, 0x80, 0xa0, 0x70, 0xb0, ++ 0x23, 0x40, 0x30, 0x20, 0x36, 0x00, 0x06, 0x44, 0x21, 0x00, 0x00, 0x1a, ++ /* 18 Byte Data Blocks 2: invalid */ ++ 0x00, 0x00, 0x00, 0xfd, 0x00, 0x18, 0x3c, 0x18, 0x50, 0x11, 0x00, 0x0a, ++ 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, ++ /* 18 Byte Data Blocks 3: invalid */ ++ 0x00, 0x00, 0x00, 0xfc, 0x00, 0x48, ++ 0x50, 0x20, 0x5a, 0x52, 0x32, 0x34, 0x34, 0x30, 0x77, 0x0a, 0x20, 0x20, ++ /* 18 Byte Data Blocks 4: invalid */ ++ 0x00, 0x00, 0x00, 0xff, 0x00, 0x43, 0x4e, 0x34, 0x33, 0x30, 0x34, 0x30, ++ 0x44, 0x58, 0x51, 0x0a, 0x20, 0x20, ++ /* Extension Block Count */ ++ 0x00, ++ /* Checksum */ ++ 0xef, ++ }, ++ { ++/* EDID with 1920x1200 as its resolution */ ++ /*Header*/ ++ 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, ++ /* Vendor & Product Identification */ ++ 0x22, 0xf0, 0x54, 0x29, 0x00, 0x00, 0x00, 0x00, 0x04, 0x17, ++ /* Version & Revision */ ++ 0x01, 0x04, ++ /* Basic Display Parameters & Features */ ++ 0xa5, 0x34, 0x20, 0x78, 0x23, ++ /* Color Characteristics */ ++ 0xfc, 0x81, 0xa4, 0x55, 0x4d, 0x9d, 0x25, 0x12, 0x50, 0x54, ++ /* Established Timings: maximum resolution is 1024x768 */ ++ 0x21, 0x08, 0x00, ++ /* ++ * Standard Timings. ++ * below new resolutions can be supported: ++ * 1920x1080, 1280x720, 1280x960, 1280x1024, ++ * 1440x900, 1600x1200, 1680x1050 ++ */ ++ 0xd1, 0xc0, 0x81, 0xc0, 0x81, 0x40, 0x81, 0x80, 0x95, 0x00, ++ 0xa9, 0x40, 0xb3, 0x00, 0x01, 0x01, ++ /* 18 Byte Data Blocks 1: max resolution is 1920x1200 */ ++ 0x28, 0x3c, 0x80, 0xa0, 0x70, 0xb0, ++ 0x23, 0x40, 0x30, 0x20, 0x36, 0x00, 0x06, 0x44, 0x21, 0x00, 0x00, 0x1a, ++ /* 18 Byte Data Blocks 2: invalid */ ++ 0x00, 0x00, 0x00, 0xfd, 0x00, 0x18, 0x3c, 0x18, 0x50, 0x11, 0x00, 0x0a, ++ 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, ++ /* 18 Byte Data Blocks 3: invalid */ ++ 0x00, 0x00, 0x00, 0xfc, 0x00, 0x48, ++ 0x50, 0x20, 0x5a, 0x52, 0x32, 0x34, 0x34, 0x30, 0x77, 0x0a, 0x20, 0x20, ++ /* 18 Byte Data Blocks 4: invalid */ ++ 0x00, 0x00, 0x00, 0xff, 0x00, 0x43, 0x4e, 0x34, 0x33, 0x30, 0x34, 0x30, ++ 0x44, 0x58, 0x51, 0x0a, 0x20, 0x20, ++ /* Extension Block Count */ ++ 0x00, ++ /* Checksum */ ++ 0x45, ++ }, ++}; ++ ++#define DPCD_HEADER_SIZE 0xb ++ ++/* let the virtual display supports DP1.2 */ ++static u8 dpcd_fix_data[DPCD_HEADER_SIZE] = { ++ 0x12, 0x014, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 ++}; ++ ++static void emulate_monitor_status_change(struct intel_vgpu *vgpu) ++{ ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ int pipe; ++ ++ if (IS_BROXTON(dev_priv)) { ++ vgpu_vreg_t(vgpu, GEN8_DE_PORT_ISR) &= ~(BXT_DE_PORT_HP_DDIA | ++ BXT_DE_PORT_HP_DDIB | ++ BXT_DE_PORT_HP_DDIC); ++ ++ if (intel_vgpu_has_monitor_on_port(vgpu, PORT_A)) { ++ vgpu_vreg_t(vgpu, GEN8_DE_PORT_ISR) |= ++ BXT_DE_PORT_HP_DDIA; ++ } ++ ++ if (intel_vgpu_has_monitor_on_port(vgpu, PORT_B)) { ++ vgpu_vreg_t(vgpu, GEN8_DE_PORT_ISR) |= ++ BXT_DE_PORT_HP_DDIB; ++ } ++ ++ if (intel_vgpu_has_monitor_on_port(vgpu, PORT_C)) { ++ vgpu_vreg_t(vgpu, GEN8_DE_PORT_ISR) |= ++ BXT_DE_PORT_HP_DDIC; ++ } ++ ++ return; ++ } ++ ++ vgpu_vreg_t(vgpu, SDEISR) &= ~(SDE_PORTB_HOTPLUG_CPT | ++ SDE_PORTC_HOTPLUG_CPT | ++ SDE_PORTD_HOTPLUG_CPT); ++ ++ if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv) || ++ IS_COFFEELAKE(dev_priv)) { ++ vgpu_vreg_t(vgpu, SDEISR) &= ~(SDE_PORTA_HOTPLUG_SPT | ++ SDE_PORTE_HOTPLUG_SPT); ++ vgpu_vreg_t(vgpu, SKL_FUSE_STATUS) |= ++ SKL_FUSE_DOWNLOAD_STATUS | ++ SKL_FUSE_PG_DIST_STATUS(SKL_PG0) | ++ SKL_FUSE_PG_DIST_STATUS(SKL_PG1) | ++ SKL_FUSE_PG_DIST_STATUS(SKL_PG2); ++ vgpu_vreg_t(vgpu, LCPLL1_CTL) |= ++ LCPLL_PLL_ENABLE | ++ LCPLL_PLL_LOCK; ++ vgpu_vreg_t(vgpu, LCPLL2_CTL) |= LCPLL_PLL_ENABLE; ++ ++ } ++ ++ if (intel_vgpu_has_monitor_on_port(vgpu, PORT_B)) { ++ vgpu_vreg_t(vgpu, SFUSE_STRAP) |= SFUSE_STRAP_DDIB_DETECTED; ++ vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) &= ++ ~(TRANS_DDI_BPC_MASK | TRANS_DDI_MODE_SELECT_MASK | ++ TRANS_DDI_PORT_MASK); ++ vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |= ++ (TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DVI | ++ (PORT_B << TRANS_DDI_PORT_SHIFT) | ++ TRANS_DDI_FUNC_ENABLE); ++ if (IS_BROADWELL(dev_priv)) { ++ vgpu_vreg_t(vgpu, PORT_CLK_SEL(PORT_B)) &= ++ ~PORT_CLK_SEL_MASK; ++ vgpu_vreg_t(vgpu, PORT_CLK_SEL(PORT_B)) |= ++ PORT_CLK_SEL_LCPLL_810; ++ } ++ vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_B)) |= DDI_BUF_CTL_ENABLE; ++ vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_B)) &= ~DDI_BUF_IS_IDLE; ++ vgpu_vreg_t(vgpu, SDEISR) |= SDE_PORTB_HOTPLUG_CPT; ++ } ++ ++ if (intel_vgpu_has_monitor_on_port(vgpu, PORT_C)) { ++ vgpu_vreg_t(vgpu, SDEISR) |= SDE_PORTC_HOTPLUG_CPT; ++ vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) &= ++ ~(TRANS_DDI_BPC_MASK | TRANS_DDI_MODE_SELECT_MASK | ++ TRANS_DDI_PORT_MASK); ++ vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |= ++ (TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DVI | ++ (PORT_C << TRANS_DDI_PORT_SHIFT) | ++ TRANS_DDI_FUNC_ENABLE); ++ if (IS_BROADWELL(dev_priv)) { ++ vgpu_vreg_t(vgpu, PORT_CLK_SEL(PORT_C)) &= ++ ~PORT_CLK_SEL_MASK; ++ vgpu_vreg_t(vgpu, PORT_CLK_SEL(PORT_C)) |= ++ PORT_CLK_SEL_LCPLL_810; ++ } ++ vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_C)) |= DDI_BUF_CTL_ENABLE; ++ vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_C)) &= ~DDI_BUF_IS_IDLE; ++ vgpu_vreg_t(vgpu, SFUSE_STRAP) |= SFUSE_STRAP_DDIC_DETECTED; ++ } ++ ++ if (intel_vgpu_has_monitor_on_port(vgpu, PORT_D)) { ++ vgpu_vreg_t(vgpu, SDEISR) |= SDE_PORTD_HOTPLUG_CPT; ++ vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) &= ++ ~(TRANS_DDI_BPC_MASK | TRANS_DDI_MODE_SELECT_MASK | ++ TRANS_DDI_PORT_MASK); ++ vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |= ++ (TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DVI | ++ (PORT_D << TRANS_DDI_PORT_SHIFT) | ++ TRANS_DDI_FUNC_ENABLE); ++ if (IS_BROADWELL(dev_priv)) { ++ vgpu_vreg_t(vgpu, PORT_CLK_SEL(PORT_D)) &= ++ ~PORT_CLK_SEL_MASK; ++ vgpu_vreg_t(vgpu, PORT_CLK_SEL(PORT_D)) |= ++ PORT_CLK_SEL_LCPLL_810; ++ } ++ vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_D)) |= DDI_BUF_CTL_ENABLE; ++ vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_D)) &= ~DDI_BUF_IS_IDLE; ++ vgpu_vreg_t(vgpu, SFUSE_STRAP) |= SFUSE_STRAP_DDID_DETECTED; ++ } ++ ++ if ((IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv) || ++ IS_COFFEELAKE(dev_priv)) && ++ intel_vgpu_has_monitor_on_port(vgpu, PORT_E)) { ++ vgpu_vreg_t(vgpu, SDEISR) |= SDE_PORTE_HOTPLUG_SPT; ++ } ++ ++ if (intel_vgpu_has_monitor_on_port(vgpu, PORT_A)) { ++ if (IS_BROADWELL(dev_priv)) ++ vgpu_vreg_t(vgpu, GEN8_DE_PORT_ISR) |= ++ GEN8_PORT_DP_A_HOTPLUG; ++ else ++ vgpu_vreg_t(vgpu, SDEISR) |= SDE_PORTA_HOTPLUG_SPT; ++ ++ vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_A)) |= DDI_INIT_DISPLAY_DETECTED; ++ } ++ ++ /* Clear host CRT status, so guest couldn't detect this host CRT. */ ++ if (IS_BROADWELL(dev_priv)) ++ vgpu_vreg_t(vgpu, PCH_ADPA) &= ~ADPA_CRT_HOTPLUG_MONITOR_MASK; ++ ++ /* Disable Primary/Sprite/Cursor plane */ ++ for_each_pipe(dev_priv, pipe) { ++ vgpu_vreg_t(vgpu, DSPCNTR(pipe)) &= ~DISPLAY_PLANE_ENABLE; ++ vgpu_vreg_t(vgpu, SPRCTL(pipe)) &= ~SPRITE_ENABLE; ++ vgpu_vreg_t(vgpu, CURCNTR(pipe)) &= ~MCURSOR_MODE; ++ vgpu_vreg_t(vgpu, CURCNTR(pipe)) |= MCURSOR_MODE_DISABLE; ++ } ++ ++ vgpu_vreg_t(vgpu, PIPECONF(PIPE_A)) |= PIPECONF_ENABLE; ++} ++ ++static void clean_virtual_dp_monitor(struct intel_vgpu *vgpu, int port_num) ++{ ++ struct intel_vgpu_port *port = intel_vgpu_port(vgpu, port_num); ++ ++ kfree(port->edid); ++ port->edid = NULL; ++ ++ kfree(port->dpcd); ++ port->dpcd = NULL; ++} ++ ++static int setup_virtual_dp_monitor(struct intel_vgpu *vgpu, int port_num, ++ int type, unsigned int resolution) ++{ ++ struct intel_vgpu_port *port = intel_vgpu_port(vgpu, port_num); ++ ++ if (WARN_ON(resolution >= GVT_EDID_NUM)) ++ return -EINVAL; ++ ++ port->edid = kzalloc(sizeof(*(port->edid)), GFP_KERNEL); ++ if (!port->edid) ++ return -ENOMEM; ++ ++ port->dpcd = kzalloc(sizeof(*(port->dpcd)), GFP_KERNEL); ++ if (!port->dpcd) { ++ kfree(port->edid); ++ return -ENOMEM; ++ } ++ ++ memcpy(port->edid->edid_block, virtual_dp_monitor_edid[resolution], ++ EDID_SIZE); ++ port->edid->data_valid = true; ++ ++ memcpy(port->dpcd->data, dpcd_fix_data, DPCD_HEADER_SIZE); ++ port->dpcd->data_valid = true; ++ port->dpcd->data[DPCD_SINK_COUNT] = 0x1; ++ port->type = type; ++ port->id = resolution; ++ ++ emulate_monitor_status_change(vgpu); ++ ++ return 0; ++} ++ ++/** ++ * intel_gvt_check_vblank_emulation - check if vblank emulation timer should ++ * be turned on/off when a virtual pipe is enabled/disabled. ++ * @gvt: a GVT device ++ * ++ * This function is used to turn on/off vblank timer according to currently ++ * enabled/disabled virtual pipes. ++ * ++ */ ++void intel_gvt_check_vblank_emulation(struct intel_gvt *gvt) ++{ ++ struct intel_gvt_irq *irq = &gvt->irq; ++ struct intel_vgpu *vgpu; ++ int pipe, id; ++ int found = false; ++ ++ mutex_lock(&gvt->lock); ++ for_each_active_vgpu(gvt, vgpu, id) { ++ for (pipe = 0; pipe < I915_MAX_PIPES; pipe++) { ++ if (pipe_is_enabled(vgpu, pipe)) { ++ found = true; ++ break; ++ } ++ } ++ if (found) ++ break; ++ } ++ ++ /* all the pipes are disabled */ ++ if (!found) ++ hrtimer_cancel(&irq->vblank_timer.timer); ++ else ++ hrtimer_start(&irq->vblank_timer.timer, ++ ktime_add_ns(ktime_get(), irq->vblank_timer.period), ++ HRTIMER_MODE_ABS); ++ mutex_unlock(&gvt->lock); ++} ++ ++static void emulate_vblank_on_pipe(struct intel_vgpu *vgpu, int pipe) ++{ ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ struct intel_vgpu_irq *irq = &vgpu->irq; ++ int vblank_event[] = { ++ [PIPE_A] = PIPE_A_VBLANK, ++ [PIPE_B] = PIPE_B_VBLANK, ++ [PIPE_C] = PIPE_C_VBLANK, ++ }; ++ int event; ++ ++ if (pipe < PIPE_A || pipe > PIPE_C) ++ return; ++ ++ for_each_set_bit(event, irq->flip_done_event[pipe], ++ INTEL_GVT_EVENT_MAX) { ++ clear_bit(event, irq->flip_done_event[pipe]); ++ if (!pipe_is_enabled(vgpu, pipe)) ++ continue; ++ ++ intel_vgpu_trigger_virtual_event(vgpu, event); ++ } ++ ++ if (pipe_is_enabled(vgpu, pipe)) { ++ vgpu_vreg_t(vgpu, PIPE_FRMCOUNT_G4X(pipe))++; ++ intel_vgpu_trigger_virtual_event(vgpu, vblank_event[pipe]); ++ } ++} ++ ++static void emulate_vblank(struct intel_vgpu *vgpu) ++{ ++ int pipe; ++ ++ mutex_lock(&vgpu->vgpu_lock); ++ for_each_pipe(vgpu->gvt->dev_priv, pipe) ++ emulate_vblank_on_pipe(vgpu, pipe); ++ mutex_unlock(&vgpu->vgpu_lock); ++} ++ ++/** ++ * intel_gvt_emulate_vblank - trigger vblank events for vGPUs on GVT device ++ * @gvt: a GVT device ++ * ++ * This function is used to trigger vblank interrupts for vGPUs on GVT device ++ * ++ */ ++void intel_gvt_emulate_vblank(struct intel_gvt *gvt) ++{ ++ struct intel_vgpu *vgpu; ++ int id; ++ ++ mutex_lock(&gvt->lock); ++ for_each_active_vgpu(gvt, vgpu, id) ++ emulate_vblank(vgpu); ++ mutex_unlock(&gvt->lock); ++} ++ ++/** ++ * intel_vgpu_emulate_hotplug - trigger hotplug event for vGPU ++ * @vgpu: a vGPU ++ * @connected: link state ++ * ++ * This function is used to trigger hotplug interrupt for vGPU ++ * ++ */ ++void intel_vgpu_emulate_hotplug(struct intel_vgpu *vgpu, bool connected) ++{ ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ ++ /* TODO: add more platforms support */ ++ if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { ++ if (connected) { ++ vgpu_vreg_t(vgpu, SFUSE_STRAP) |= ++ SFUSE_STRAP_DDID_DETECTED; ++ vgpu_vreg_t(vgpu, SDEISR) |= SDE_PORTD_HOTPLUG_CPT; ++ } else { ++ vgpu_vreg_t(vgpu, SFUSE_STRAP) &= ++ ~SFUSE_STRAP_DDID_DETECTED; ++ vgpu_vreg_t(vgpu, SDEISR) &= ~SDE_PORTD_HOTPLUG_CPT; ++ } ++ vgpu_vreg_t(vgpu, SDEIIR) |= SDE_PORTD_HOTPLUG_CPT; ++ vgpu_vreg_t(vgpu, PCH_PORT_HOTPLUG) |= ++ PORTD_HOTPLUG_STATUS_MASK; ++ intel_vgpu_trigger_virtual_event(vgpu, DP_D_HOTPLUG); ++ } ++} ++ ++/** ++ * intel_vgpu_clean_display - clean vGPU virtual display emulation ++ * @vgpu: a vGPU ++ * ++ * This function is used to clean vGPU virtual display emulation stuffs ++ * ++ */ ++void intel_vgpu_clean_display(struct intel_vgpu *vgpu) ++{ ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ ++ if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv) || ++ IS_COFFEELAKE(dev_priv)) ++ clean_virtual_dp_monitor(vgpu, PORT_D); ++ else ++ clean_virtual_dp_monitor(vgpu, PORT_B); ++} ++ ++/** ++ * intel_vgpu_init_display- initialize vGPU virtual display emulation ++ * @vgpu: a vGPU ++ * @resolution: resolution index for intel_vgpu_edid ++ * ++ * This function is used to initialize vGPU virtual display emulation stuffs ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ * ++ */ ++int intel_vgpu_init_display(struct intel_vgpu *vgpu, u64 resolution) ++{ ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ ++ intel_vgpu_init_i2c_edid(vgpu); ++ ++ if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv) || ++ IS_COFFEELAKE(dev_priv)) ++ return setup_virtual_dp_monitor(vgpu, PORT_D, GVT_DP_D, ++ resolution); ++ else ++ return setup_virtual_dp_monitor(vgpu, PORT_B, GVT_DP_B, ++ resolution); ++} ++ ++/** ++ * intel_vgpu_reset_display- reset vGPU virtual display emulation ++ * @vgpu: a vGPU ++ * ++ * This function is used to reset vGPU virtual display emulation stuffs ++ * ++ */ ++void intel_vgpu_reset_display(struct intel_vgpu *vgpu) ++{ ++ emulate_monitor_status_change(vgpu); ++} +diff --git a/drivers/gpu/drm/i915_legacy/gvt/display.h b/drivers/gpu/drm/i915_legacy/gvt/display.h +new file mode 100644 +index 000000000000..a87f33e6a23c +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/display.h +@@ -0,0 +1,209 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Ke Yu ++ * Zhiyuan Lv ++ * ++ * Contributors: ++ * Terrence Xu ++ * Changbin Du ++ * Bing Niu ++ * Zhi Wang ++ * ++ */ ++ ++#ifndef _GVT_DISPLAY_H_ ++#define _GVT_DISPLAY_H_ ++ ++#define SBI_REG_MAX 20 ++#define DPCD_SIZE 0x700 ++ ++#define intel_vgpu_port(vgpu, port) \ ++ (&(vgpu->display.ports[port])) ++ ++#define intel_vgpu_has_monitor_on_port(vgpu, port) \ ++ (intel_vgpu_port(vgpu, port)->edid && \ ++ intel_vgpu_port(vgpu, port)->edid->data_valid) ++ ++#define intel_vgpu_port_is_dp(vgpu, port) \ ++ ((intel_vgpu_port(vgpu, port)->type == GVT_DP_A) || \ ++ (intel_vgpu_port(vgpu, port)->type == GVT_DP_B) || \ ++ (intel_vgpu_port(vgpu, port)->type == GVT_DP_C) || \ ++ (intel_vgpu_port(vgpu, port)->type == GVT_DP_D)) ++ ++#define INTEL_GVT_MAX_UEVENT_VARS 3 ++ ++/* DPCD start */ ++#define DPCD_SIZE 0x700 ++ ++/* DPCD */ ++#define DP_SET_POWER 0x600 ++#define DP_SET_POWER_D0 0x1 ++#define AUX_NATIVE_WRITE 0x8 ++#define AUX_NATIVE_READ 0x9 ++ ++#define AUX_NATIVE_REPLY_MASK (0x3 << 4) ++#define AUX_NATIVE_REPLY_ACK (0x0 << 4) ++#define AUX_NATIVE_REPLY_NAK (0x1 << 4) ++#define AUX_NATIVE_REPLY_DEFER (0x2 << 4) ++ ++#define AUX_BURST_SIZE 20 ++ ++/* DPCD addresses */ ++#define DPCD_REV 0x000 ++#define DPCD_MAX_LINK_RATE 0x001 ++#define DPCD_MAX_LANE_COUNT 0x002 ++ ++#define DPCD_TRAINING_PATTERN_SET 0x102 ++#define DPCD_SINK_COUNT 0x200 ++#define DPCD_LANE0_1_STATUS 0x202 ++#define DPCD_LANE2_3_STATUS 0x203 ++#define DPCD_LANE_ALIGN_STATUS_UPDATED 0x204 ++#define DPCD_SINK_STATUS 0x205 ++ ++/* link training */ ++#define DPCD_TRAINING_PATTERN_SET_MASK 0x03 ++#define DPCD_LINK_TRAINING_DISABLED 0x00 ++#define DPCD_TRAINING_PATTERN_1 0x01 ++#define DPCD_TRAINING_PATTERN_2 0x02 ++ ++#define DPCD_CP_READY_MASK (1 << 6) ++ ++/* lane status */ ++#define DPCD_LANES_CR_DONE 0x11 ++#define DPCD_LANES_EQ_DONE 0x22 ++#define DPCD_SYMBOL_LOCKED 0x44 ++ ++#define DPCD_INTERLANE_ALIGN_DONE 0x01 ++ ++#define DPCD_SINK_IN_SYNC 0x03 ++/* DPCD end */ ++ ++#define SBI_RESPONSE_MASK 0x3 ++#define SBI_RESPONSE_SHIFT 0x1 ++#define SBI_STAT_MASK 0x1 ++#define SBI_STAT_SHIFT 0x0 ++#define SBI_OPCODE_SHIFT 8 ++#define SBI_OPCODE_MASK (0xff << SBI_OPCODE_SHIFT) ++#define SBI_CMD_IORD 2 ++#define SBI_CMD_IOWR 3 ++#define SBI_CMD_CRRD 6 ++#define SBI_CMD_CRWR 7 ++#define SBI_ADDR_OFFSET_SHIFT 16 ++#define SBI_ADDR_OFFSET_MASK (0xffff << SBI_ADDR_OFFSET_SHIFT) ++ ++struct intel_vgpu_sbi_register { ++ unsigned int offset; ++ u32 value; ++}; ++ ++struct intel_vgpu_sbi { ++ int number; ++ struct intel_vgpu_sbi_register registers[SBI_REG_MAX]; ++}; ++ ++enum intel_gvt_plane_type { ++ PRIMARY_PLANE = 0, ++ CURSOR_PLANE, ++ SPRITE_PLANE, ++ MAX_PLANE ++}; ++ ++struct intel_vgpu_dpcd_data { ++ bool data_valid; ++ u8 data[DPCD_SIZE]; ++}; ++ ++enum intel_vgpu_port_type { ++ GVT_CRT = 0, ++ GVT_DP_A, ++ GVT_DP_B, ++ GVT_DP_C, ++ GVT_DP_D, ++ GVT_HDMI_B, ++ GVT_HDMI_C, ++ GVT_HDMI_D, ++ GVT_PORT_MAX ++}; ++ ++enum intel_vgpu_edid { ++ GVT_EDID_1024_768, ++ GVT_EDID_1920_1200, ++ GVT_EDID_NUM, ++}; ++ ++struct intel_vgpu_port { ++ /* per display EDID information */ ++ struct intel_vgpu_edid_data *edid; ++ /* per display DPCD information */ ++ struct intel_vgpu_dpcd_data *dpcd; ++ int type; ++ enum intel_vgpu_edid id; ++}; ++ ++static inline char *vgpu_edid_str(enum intel_vgpu_edid id) ++{ ++ switch (id) { ++ case GVT_EDID_1024_768: ++ return "1024x768"; ++ case GVT_EDID_1920_1200: ++ return "1920x1200"; ++ default: ++ return ""; ++ } ++} ++ ++static inline unsigned int vgpu_edid_xres(enum intel_vgpu_edid id) ++{ ++ switch (id) { ++ case GVT_EDID_1024_768: ++ return 1024; ++ case GVT_EDID_1920_1200: ++ return 1920; ++ default: ++ return 0; ++ } ++} ++ ++static inline unsigned int vgpu_edid_yres(enum intel_vgpu_edid id) ++{ ++ switch (id) { ++ case GVT_EDID_1024_768: ++ return 768; ++ case GVT_EDID_1920_1200: ++ return 1200; ++ default: ++ return 0; ++ } ++} ++ ++void intel_gvt_emulate_vblank(struct intel_gvt *gvt); ++void intel_gvt_check_vblank_emulation(struct intel_gvt *gvt); ++ ++int intel_vgpu_init_display(struct intel_vgpu *vgpu, u64 resolution); ++void intel_vgpu_reset_display(struct intel_vgpu *vgpu); ++void intel_vgpu_clean_display(struct intel_vgpu *vgpu); ++ ++int pipe_is_enabled(struct intel_vgpu *vgpu, int pipe); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/gvt/dmabuf.c b/drivers/gpu/drm/i915_legacy/gvt/dmabuf.c +new file mode 100644 +index 000000000000..41c8ebc60c63 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/dmabuf.c +@@ -0,0 +1,561 @@ ++/* ++ * Copyright 2017 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER ++ * DEALINGS IN THE SOFTWARE. ++ * ++ * Authors: ++ * Zhiyuan Lv ++ * ++ * Contributors: ++ * Xiaoguang Chen ++ * Tina Zhang ++ */ ++ ++#include ++#include ++ ++#include "i915_drv.h" ++#include "gvt.h" ++ ++#define GEN8_DECODE_PTE(pte) (pte & GENMASK_ULL(63, 12)) ++ ++static int vgpu_gem_get_pages( ++ struct drm_i915_gem_object *obj) ++{ ++ struct drm_i915_private *dev_priv = to_i915(obj->base.dev); ++ struct sg_table *st; ++ struct scatterlist *sg; ++ int i, ret; ++ gen8_pte_t __iomem *gtt_entries; ++ struct intel_vgpu_fb_info *fb_info; ++ u32 page_num; ++ ++ fb_info = (struct intel_vgpu_fb_info *)obj->gvt_info; ++ if (WARN_ON(!fb_info)) ++ return -ENODEV; ++ ++ st = kmalloc(sizeof(*st), GFP_KERNEL); ++ if (unlikely(!st)) ++ return -ENOMEM; ++ ++ page_num = obj->base.size >> PAGE_SHIFT; ++ ret = sg_alloc_table(st, page_num, GFP_KERNEL); ++ if (ret) { ++ kfree(st); ++ return ret; ++ } ++ gtt_entries = (gen8_pte_t __iomem *)dev_priv->ggtt.gsm + ++ (fb_info->start >> PAGE_SHIFT); ++ for_each_sg(st->sgl, sg, page_num, i) { ++ sg->offset = 0; ++ sg->length = PAGE_SIZE; ++ sg_dma_address(sg) = ++ GEN8_DECODE_PTE(readq(>t_entries[i])); ++ sg_dma_len(sg) = PAGE_SIZE; ++ } ++ ++ __i915_gem_object_set_pages(obj, st, PAGE_SIZE); ++ ++ return 0; ++} ++ ++static void vgpu_gem_put_pages(struct drm_i915_gem_object *obj, ++ struct sg_table *pages) ++{ ++ sg_free_table(pages); ++ kfree(pages); ++} ++ ++static void dmabuf_gem_object_free(struct kref *kref) ++{ ++ struct intel_vgpu_dmabuf_obj *obj = ++ container_of(kref, struct intel_vgpu_dmabuf_obj, kref); ++ struct intel_vgpu *vgpu = obj->vgpu; ++ struct list_head *pos; ++ struct intel_vgpu_dmabuf_obj *dmabuf_obj; ++ ++ if (vgpu && vgpu->active && !list_empty(&vgpu->dmabuf_obj_list_head)) { ++ list_for_each(pos, &vgpu->dmabuf_obj_list_head) { ++ dmabuf_obj = container_of(pos, ++ struct intel_vgpu_dmabuf_obj, list); ++ if (dmabuf_obj == obj) { ++ intel_gvt_hypervisor_put_vfio_device(vgpu); ++ idr_remove(&vgpu->object_idr, ++ dmabuf_obj->dmabuf_id); ++ kfree(dmabuf_obj->info); ++ kfree(dmabuf_obj); ++ list_del(pos); ++ break; ++ } ++ } ++ } else { ++ /* Free the orphan dmabuf_objs here */ ++ kfree(obj->info); ++ kfree(obj); ++ } ++} ++ ++ ++static inline void dmabuf_obj_get(struct intel_vgpu_dmabuf_obj *obj) ++{ ++ kref_get(&obj->kref); ++} ++ ++static inline void dmabuf_obj_put(struct intel_vgpu_dmabuf_obj *obj) ++{ ++ kref_put(&obj->kref, dmabuf_gem_object_free); ++} ++ ++static void vgpu_gem_release(struct drm_i915_gem_object *gem_obj) ++{ ++ ++ struct intel_vgpu_fb_info *fb_info = gem_obj->gvt_info; ++ struct intel_vgpu_dmabuf_obj *obj = fb_info->obj; ++ struct intel_vgpu *vgpu = obj->vgpu; ++ ++ if (vgpu) { ++ mutex_lock(&vgpu->dmabuf_lock); ++ gem_obj->base.dma_buf = NULL; ++ dmabuf_obj_put(obj); ++ mutex_unlock(&vgpu->dmabuf_lock); ++ } else { ++ /* vgpu is NULL, as it has been removed already */ ++ gem_obj->base.dma_buf = NULL; ++ dmabuf_obj_put(obj); ++ } ++} ++ ++static const struct drm_i915_gem_object_ops intel_vgpu_gem_ops = { ++ .flags = I915_GEM_OBJECT_IS_PROXY, ++ .get_pages = vgpu_gem_get_pages, ++ .put_pages = vgpu_gem_put_pages, ++ .release = vgpu_gem_release, ++}; ++ ++static struct drm_i915_gem_object *vgpu_create_gem(struct drm_device *dev, ++ struct intel_vgpu_fb_info *info) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct drm_i915_gem_object *obj; ++ ++ obj = i915_gem_object_alloc(); ++ if (obj == NULL) ++ return NULL; ++ ++ drm_gem_private_object_init(dev, &obj->base, ++ roundup(info->size, PAGE_SIZE)); ++ i915_gem_object_init(obj, &intel_vgpu_gem_ops); ++ ++ obj->read_domains = I915_GEM_DOMAIN_GTT; ++ obj->write_domain = 0; ++ if (INTEL_GEN(dev_priv) >= 9) { ++ unsigned int tiling_mode = 0; ++ unsigned int stride = 0; ++ ++ switch (info->drm_format_mod) { ++ case DRM_FORMAT_MOD_LINEAR: ++ tiling_mode = I915_TILING_NONE; ++ break; ++ case I915_FORMAT_MOD_X_TILED: ++ tiling_mode = I915_TILING_X; ++ stride = info->stride; ++ break; ++ case I915_FORMAT_MOD_Y_TILED: ++ case I915_FORMAT_MOD_Yf_TILED: ++ tiling_mode = I915_TILING_Y; ++ stride = info->stride; ++ break; ++ default: ++ gvt_dbg_core("invalid drm_format_mod %llx for tiling\n", ++ info->drm_format_mod); ++ } ++ obj->tiling_and_stride = tiling_mode | stride; ++ } else { ++ obj->tiling_and_stride = info->drm_format_mod ? ++ I915_TILING_X : 0; ++ } ++ ++ return obj; ++} ++ ++static bool validate_hotspot(struct intel_vgpu_cursor_plane_format *c) ++{ ++ if (c && c->x_hot <= c->width && c->y_hot <= c->height) ++ return true; ++ else ++ return false; ++} ++ ++static int vgpu_get_plane_info(struct drm_device *dev, ++ struct intel_vgpu *vgpu, ++ struct intel_vgpu_fb_info *info, ++ int plane_id) ++{ ++ struct intel_vgpu_primary_plane_format p; ++ struct intel_vgpu_cursor_plane_format c; ++ int ret, tile_height = 1; ++ ++ memset(info, 0, sizeof(*info)); ++ ++ if (plane_id == DRM_PLANE_TYPE_PRIMARY) { ++ ret = intel_vgpu_decode_primary_plane(vgpu, &p); ++ if (ret) ++ return ret; ++ info->start = p.base; ++ info->start_gpa = p.base_gpa; ++ info->width = p.width; ++ info->height = p.height; ++ info->stride = p.stride; ++ info->drm_format = p.drm_format; ++ ++ switch (p.tiled) { ++ case PLANE_CTL_TILED_LINEAR: ++ info->drm_format_mod = DRM_FORMAT_MOD_LINEAR; ++ break; ++ case PLANE_CTL_TILED_X: ++ info->drm_format_mod = I915_FORMAT_MOD_X_TILED; ++ tile_height = 8; ++ break; ++ case PLANE_CTL_TILED_Y: ++ info->drm_format_mod = I915_FORMAT_MOD_Y_TILED; ++ tile_height = 32; ++ break; ++ case PLANE_CTL_TILED_YF: ++ info->drm_format_mod = I915_FORMAT_MOD_Yf_TILED; ++ tile_height = 32; ++ break; ++ default: ++ gvt_vgpu_err("invalid tiling mode: %x\n", p.tiled); ++ } ++ } else if (plane_id == DRM_PLANE_TYPE_CURSOR) { ++ ret = intel_vgpu_decode_cursor_plane(vgpu, &c); ++ if (ret) ++ return ret; ++ info->start = c.base; ++ info->start_gpa = c.base_gpa; ++ info->width = c.width; ++ info->height = c.height; ++ info->stride = c.width * (c.bpp / 8); ++ info->drm_format = c.drm_format; ++ info->drm_format_mod = 0; ++ info->x_pos = c.x_pos; ++ info->y_pos = c.y_pos; ++ ++ if (validate_hotspot(&c)) { ++ info->x_hot = c.x_hot; ++ info->y_hot = c.y_hot; ++ } else { ++ info->x_hot = UINT_MAX; ++ info->y_hot = UINT_MAX; ++ } ++ } else { ++ gvt_vgpu_err("invalid plane id:%d\n", plane_id); ++ return -EINVAL; ++ } ++ ++ info->size = info->stride * roundup(info->height, tile_height); ++ if (info->size == 0) { ++ gvt_vgpu_err("fb size is zero\n"); ++ return -EINVAL; ++ } ++ ++ if (info->start & (PAGE_SIZE - 1)) { ++ gvt_vgpu_err("Not aligned fb address:0x%llx\n", info->start); ++ return -EFAULT; ++ } ++ ++ if (!intel_gvt_ggtt_validate_range(vgpu, info->start, info->size)) { ++ gvt_vgpu_err("invalid gma addr\n"); ++ return -EFAULT; ++ } ++ ++ return 0; ++} ++ ++static struct intel_vgpu_dmabuf_obj * ++pick_dmabuf_by_info(struct intel_vgpu *vgpu, ++ struct intel_vgpu_fb_info *latest_info) ++{ ++ struct list_head *pos; ++ struct intel_vgpu_fb_info *fb_info; ++ struct intel_vgpu_dmabuf_obj *dmabuf_obj = NULL; ++ struct intel_vgpu_dmabuf_obj *ret = NULL; ++ ++ list_for_each(pos, &vgpu->dmabuf_obj_list_head) { ++ dmabuf_obj = container_of(pos, struct intel_vgpu_dmabuf_obj, ++ list); ++ if ((dmabuf_obj == NULL) || ++ (dmabuf_obj->info == NULL)) ++ continue; ++ ++ fb_info = (struct intel_vgpu_fb_info *)dmabuf_obj->info; ++ if ((fb_info->start == latest_info->start) && ++ (fb_info->start_gpa == latest_info->start_gpa) && ++ (fb_info->size == latest_info->size) && ++ (fb_info->drm_format_mod == latest_info->drm_format_mod) && ++ (fb_info->drm_format == latest_info->drm_format) && ++ (fb_info->width == latest_info->width) && ++ (fb_info->height == latest_info->height)) { ++ ret = dmabuf_obj; ++ break; ++ } ++ } ++ ++ return ret; ++} ++ ++static struct intel_vgpu_dmabuf_obj * ++pick_dmabuf_by_num(struct intel_vgpu *vgpu, u32 id) ++{ ++ struct list_head *pos; ++ struct intel_vgpu_dmabuf_obj *dmabuf_obj = NULL; ++ struct intel_vgpu_dmabuf_obj *ret = NULL; ++ ++ list_for_each(pos, &vgpu->dmabuf_obj_list_head) { ++ dmabuf_obj = container_of(pos, struct intel_vgpu_dmabuf_obj, ++ list); ++ if (!dmabuf_obj) ++ continue; ++ ++ if (dmabuf_obj->dmabuf_id == id) { ++ ret = dmabuf_obj; ++ break; ++ } ++ } ++ ++ return ret; ++} ++ ++static void update_fb_info(struct vfio_device_gfx_plane_info *gvt_dmabuf, ++ struct intel_vgpu_fb_info *fb_info) ++{ ++ gvt_dmabuf->drm_format = fb_info->drm_format; ++ gvt_dmabuf->drm_format_mod = fb_info->drm_format_mod; ++ gvt_dmabuf->width = fb_info->width; ++ gvt_dmabuf->height = fb_info->height; ++ gvt_dmabuf->stride = fb_info->stride; ++ gvt_dmabuf->size = fb_info->size; ++ gvt_dmabuf->x_pos = fb_info->x_pos; ++ gvt_dmabuf->y_pos = fb_info->y_pos; ++ gvt_dmabuf->x_hot = fb_info->x_hot; ++ gvt_dmabuf->y_hot = fb_info->y_hot; ++} ++ ++int intel_vgpu_query_plane(struct intel_vgpu *vgpu, void *args) ++{ ++ struct drm_device *dev = &vgpu->gvt->dev_priv->drm; ++ struct vfio_device_gfx_plane_info *gfx_plane_info = args; ++ struct intel_vgpu_dmabuf_obj *dmabuf_obj; ++ struct intel_vgpu_fb_info fb_info; ++ int ret = 0; ++ ++ if (gfx_plane_info->flags == (VFIO_GFX_PLANE_TYPE_DMABUF | ++ VFIO_GFX_PLANE_TYPE_PROBE)) ++ return ret; ++ else if ((gfx_plane_info->flags & ~VFIO_GFX_PLANE_TYPE_DMABUF) || ++ (!gfx_plane_info->flags)) ++ return -EINVAL; ++ ++ ret = vgpu_get_plane_info(dev, vgpu, &fb_info, ++ gfx_plane_info->drm_plane_type); ++ if (ret != 0) ++ goto out; ++ ++ mutex_lock(&vgpu->dmabuf_lock); ++ /* If exists, pick up the exposed dmabuf_obj */ ++ dmabuf_obj = pick_dmabuf_by_info(vgpu, &fb_info); ++ if (dmabuf_obj) { ++ update_fb_info(gfx_plane_info, &fb_info); ++ gfx_plane_info->dmabuf_id = dmabuf_obj->dmabuf_id; ++ ++ /* This buffer may be released between query_plane ioctl and ++ * get_dmabuf ioctl. Add the refcount to make sure it won't ++ * be released between the two ioctls. ++ */ ++ if (!dmabuf_obj->initref) { ++ dmabuf_obj->initref = true; ++ dmabuf_obj_get(dmabuf_obj); ++ } ++ ret = 0; ++ gvt_dbg_dpy("vgpu%d: re-use dmabuf_obj ref %d, id %d\n", ++ vgpu->id, kref_read(&dmabuf_obj->kref), ++ gfx_plane_info->dmabuf_id); ++ mutex_unlock(&vgpu->dmabuf_lock); ++ goto out; ++ } ++ ++ mutex_unlock(&vgpu->dmabuf_lock); ++ ++ /* Need to allocate a new one*/ ++ dmabuf_obj = kmalloc(sizeof(struct intel_vgpu_dmabuf_obj), GFP_KERNEL); ++ if (unlikely(!dmabuf_obj)) { ++ gvt_vgpu_err("alloc dmabuf_obj failed\n"); ++ ret = -ENOMEM; ++ goto out; ++ } ++ ++ dmabuf_obj->info = kmalloc(sizeof(struct intel_vgpu_fb_info), ++ GFP_KERNEL); ++ if (unlikely(!dmabuf_obj->info)) { ++ gvt_vgpu_err("allocate intel vgpu fb info failed\n"); ++ ret = -ENOMEM; ++ goto out_free_dmabuf; ++ } ++ memcpy(dmabuf_obj->info, &fb_info, sizeof(struct intel_vgpu_fb_info)); ++ ++ ((struct intel_vgpu_fb_info *)dmabuf_obj->info)->obj = dmabuf_obj; ++ ++ dmabuf_obj->vgpu = vgpu; ++ ++ ret = idr_alloc(&vgpu->object_idr, dmabuf_obj, 1, 0, GFP_NOWAIT); ++ if (ret < 0) ++ goto out_free_info; ++ gfx_plane_info->dmabuf_id = ret; ++ dmabuf_obj->dmabuf_id = ret; ++ ++ dmabuf_obj->initref = true; ++ ++ kref_init(&dmabuf_obj->kref); ++ ++ mutex_lock(&vgpu->dmabuf_lock); ++ if (intel_gvt_hypervisor_get_vfio_device(vgpu)) { ++ gvt_vgpu_err("get vfio device failed\n"); ++ mutex_unlock(&vgpu->dmabuf_lock); ++ goto out_free_info; ++ } ++ mutex_unlock(&vgpu->dmabuf_lock); ++ ++ update_fb_info(gfx_plane_info, &fb_info); ++ ++ INIT_LIST_HEAD(&dmabuf_obj->list); ++ mutex_lock(&vgpu->dmabuf_lock); ++ list_add_tail(&dmabuf_obj->list, &vgpu->dmabuf_obj_list_head); ++ mutex_unlock(&vgpu->dmabuf_lock); ++ ++ gvt_dbg_dpy("vgpu%d: %s new dmabuf_obj ref %d, id %d\n", vgpu->id, ++ __func__, kref_read(&dmabuf_obj->kref), ret); ++ ++ return 0; ++ ++out_free_info: ++ kfree(dmabuf_obj->info); ++out_free_dmabuf: ++ kfree(dmabuf_obj); ++out: ++ /* ENODEV means plane isn't ready, which might be a normal case. */ ++ return (ret == -ENODEV) ? 0 : ret; ++} ++ ++/* To associate an exposed dmabuf with the dmabuf_obj */ ++int intel_vgpu_get_dmabuf(struct intel_vgpu *vgpu, unsigned int dmabuf_id) ++{ ++ struct drm_device *dev = &vgpu->gvt->dev_priv->drm; ++ struct intel_vgpu_dmabuf_obj *dmabuf_obj; ++ struct drm_i915_gem_object *obj; ++ struct dma_buf *dmabuf; ++ int dmabuf_fd; ++ int ret = 0; ++ ++ mutex_lock(&vgpu->dmabuf_lock); ++ ++ dmabuf_obj = pick_dmabuf_by_num(vgpu, dmabuf_id); ++ if (dmabuf_obj == NULL) { ++ gvt_vgpu_err("invalid dmabuf id:%d\n", dmabuf_id); ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ obj = vgpu_create_gem(dev, dmabuf_obj->info); ++ if (obj == NULL) { ++ gvt_vgpu_err("create gvt gem obj failed\n"); ++ ret = -ENOMEM; ++ goto out; ++ } ++ ++ obj->gvt_info = dmabuf_obj->info; ++ ++ dmabuf = i915_gem_prime_export(dev, &obj->base, DRM_CLOEXEC | DRM_RDWR); ++ if (IS_ERR(dmabuf)) { ++ gvt_vgpu_err("export dma-buf failed\n"); ++ ret = PTR_ERR(dmabuf); ++ goto out_free_gem; ++ } ++ ++ i915_gem_object_put(obj); ++ ++ ret = dma_buf_fd(dmabuf, DRM_CLOEXEC | DRM_RDWR); ++ if (ret < 0) { ++ gvt_vgpu_err("create dma-buf fd failed ret:%d\n", ret); ++ goto out_free_dmabuf; ++ } ++ dmabuf_fd = ret; ++ ++ dmabuf_obj_get(dmabuf_obj); ++ ++ if (dmabuf_obj->initref) { ++ dmabuf_obj->initref = false; ++ dmabuf_obj_put(dmabuf_obj); ++ } ++ ++ mutex_unlock(&vgpu->dmabuf_lock); ++ ++ gvt_dbg_dpy("vgpu%d: dmabuf:%d, dmabuf ref %d, fd:%d\n" ++ " file count: %ld, GEM ref: %d\n", ++ vgpu->id, dmabuf_obj->dmabuf_id, ++ kref_read(&dmabuf_obj->kref), ++ dmabuf_fd, ++ file_count(dmabuf->file), ++ kref_read(&obj->base.refcount)); ++ ++ return dmabuf_fd; ++ ++out_free_dmabuf: ++ dma_buf_put(dmabuf); ++out_free_gem: ++ i915_gem_object_put(obj); ++out: ++ mutex_unlock(&vgpu->dmabuf_lock); ++ return ret; ++} ++ ++void intel_vgpu_dmabuf_cleanup(struct intel_vgpu *vgpu) ++{ ++ struct list_head *pos, *n; ++ struct intel_vgpu_dmabuf_obj *dmabuf_obj; ++ ++ mutex_lock(&vgpu->dmabuf_lock); ++ list_for_each_safe(pos, n, &vgpu->dmabuf_obj_list_head) { ++ dmabuf_obj = container_of(pos, struct intel_vgpu_dmabuf_obj, ++ list); ++ dmabuf_obj->vgpu = NULL; ++ ++ idr_remove(&vgpu->object_idr, dmabuf_obj->dmabuf_id); ++ intel_gvt_hypervisor_put_vfio_device(vgpu); ++ list_del(pos); ++ ++ /* dmabuf_obj might be freed in dmabuf_obj_put */ ++ if (dmabuf_obj->initref) { ++ dmabuf_obj->initref = false; ++ dmabuf_obj_put(dmabuf_obj); ++ } ++ ++ } ++ mutex_unlock(&vgpu->dmabuf_lock); ++} +diff --git a/drivers/gpu/drm/i915_legacy/gvt/dmabuf.h b/drivers/gpu/drm/i915_legacy/gvt/dmabuf.h +new file mode 100644 +index 000000000000..5f8f03fb1d1b +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/dmabuf.h +@@ -0,0 +1,67 @@ ++/* ++ * Copyright(c) 2017 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Zhiyuan Lv ++ * ++ * Contributors: ++ * Xiaoguang Chen ++ * Tina Zhang ++ */ ++ ++#ifndef _GVT_DMABUF_H_ ++#define _GVT_DMABUF_H_ ++#include ++ ++struct intel_vgpu_fb_info { ++ __u64 start; ++ __u64 start_gpa; ++ __u64 drm_format_mod; ++ __u32 drm_format; /* drm format of plane */ ++ __u32 width; /* width of plane */ ++ __u32 height; /* height of plane */ ++ __u32 stride; /* stride of plane */ ++ __u32 size; /* size of plane in bytes, align on page */ ++ __u32 x_pos; /* horizontal position of cursor plane */ ++ __u32 y_pos; /* vertical position of cursor plane */ ++ __u32 x_hot; /* horizontal position of cursor hotspot */ ++ __u32 y_hot; /* vertical position of cursor hotspot */ ++ struct intel_vgpu_dmabuf_obj *obj; ++}; ++ ++/** ++ * struct intel_vgpu_dmabuf_obj- Intel vGPU device buffer object ++ */ ++struct intel_vgpu_dmabuf_obj { ++ struct intel_vgpu *vgpu; ++ struct intel_vgpu_fb_info *info; ++ __u32 dmabuf_id; ++ struct kref kref; ++ bool initref; ++ struct list_head list; ++}; ++ ++int intel_vgpu_query_plane(struct intel_vgpu *vgpu, void *args); ++int intel_vgpu_get_dmabuf(struct intel_vgpu *vgpu, unsigned int dmabuf_id); ++void intel_vgpu_dmabuf_cleanup(struct intel_vgpu *vgpu); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/gvt/edid.c b/drivers/gpu/drm/i915_legacy/gvt/edid.c +new file mode 100644 +index 000000000000..1fe6124918f1 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/edid.c +@@ -0,0 +1,576 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Ke Yu ++ * Zhiyuan Lv ++ * ++ * Contributors: ++ * Terrence Xu ++ * Changbin Du ++ * Bing Niu ++ * Zhi Wang ++ * ++ */ ++ ++#include "i915_drv.h" ++#include "gvt.h" ++ ++#define GMBUS1_TOTAL_BYTES_SHIFT 16 ++#define GMBUS1_TOTAL_BYTES_MASK 0x1ff ++#define gmbus1_total_byte_count(v) (((v) >> \ ++ GMBUS1_TOTAL_BYTES_SHIFT) & GMBUS1_TOTAL_BYTES_MASK) ++#define gmbus1_slave_addr(v) (((v) & 0xff) >> 1) ++#define gmbus1_slave_index(v) (((v) >> 8) & 0xff) ++#define gmbus1_bus_cycle(v) (((v) >> 25) & 0x7) ++ ++/* GMBUS0 bits definitions */ ++#define _GMBUS_PIN_SEL_MASK (0x7) ++ ++static unsigned char edid_get_byte(struct intel_vgpu *vgpu) ++{ ++ struct intel_vgpu_i2c_edid *edid = &vgpu->display.i2c_edid; ++ unsigned char chr = 0; ++ ++ if (edid->state == I2C_NOT_SPECIFIED || !edid->slave_selected) { ++ gvt_vgpu_err("Driver tries to read EDID without proper sequence!\n"); ++ return 0; ++ } ++ if (edid->current_edid_read >= EDID_SIZE) { ++ gvt_vgpu_err("edid_get_byte() exceeds the size of EDID!\n"); ++ return 0; ++ } ++ ++ if (!edid->edid_available) { ++ gvt_vgpu_err("Reading EDID but EDID is not available!\n"); ++ return 0; ++ } ++ ++ if (intel_vgpu_has_monitor_on_port(vgpu, edid->port)) { ++ struct intel_vgpu_edid_data *edid_data = ++ intel_vgpu_port(vgpu, edid->port)->edid; ++ ++ chr = edid_data->edid_block[edid->current_edid_read]; ++ edid->current_edid_read++; ++ } else { ++ gvt_vgpu_err("No EDID available during the reading?\n"); ++ } ++ return chr; ++} ++ ++static inline int cnp_get_port_from_gmbus0(u32 gmbus0) ++{ ++ int port_select = gmbus0 & _GMBUS_PIN_SEL_MASK; ++ int port = -EINVAL; ++ ++ if (port_select == GMBUS_PIN_1_BXT) ++ port = PORT_B; ++ else if (port_select == GMBUS_PIN_2_BXT) ++ port = PORT_C; ++ else if (port_select == GMBUS_PIN_3_BXT) ++ port = PORT_D; ++ else if (port_select == GMBUS_PIN_4_CNP) ++ port = PORT_E; ++ return port; ++} ++ ++static inline int bxt_get_port_from_gmbus0(u32 gmbus0) ++{ ++ int port_select = gmbus0 & _GMBUS_PIN_SEL_MASK; ++ int port = -EINVAL; ++ ++ if (port_select == GMBUS_PIN_1_BXT) ++ port = PORT_B; ++ else if (port_select == GMBUS_PIN_2_BXT) ++ port = PORT_C; ++ else if (port_select == GMBUS_PIN_3_BXT) ++ port = PORT_D; ++ return port; ++} ++ ++static inline int get_port_from_gmbus0(u32 gmbus0) ++{ ++ int port_select = gmbus0 & _GMBUS_PIN_SEL_MASK; ++ int port = -EINVAL; ++ ++ if (port_select == GMBUS_PIN_VGADDC) ++ port = PORT_E; ++ else if (port_select == GMBUS_PIN_DPC) ++ port = PORT_C; ++ else if (port_select == GMBUS_PIN_DPB) ++ port = PORT_B; ++ else if (port_select == GMBUS_PIN_DPD) ++ port = PORT_D; ++ return port; ++} ++ ++static void reset_gmbus_controller(struct intel_vgpu *vgpu) ++{ ++ vgpu_vreg_t(vgpu, PCH_GMBUS2) = GMBUS_HW_RDY; ++ if (!vgpu->display.i2c_edid.edid_available) ++ vgpu_vreg_t(vgpu, PCH_GMBUS2) |= GMBUS_SATOER; ++ vgpu->display.i2c_edid.gmbus.phase = GMBUS_IDLE_PHASE; ++} ++ ++/* GMBUS0 */ ++static int gmbus0_mmio_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ int port, pin_select; ++ ++ memcpy(&vgpu_vreg(vgpu, offset), p_data, bytes); ++ ++ pin_select = vgpu_vreg(vgpu, offset) & _GMBUS_PIN_SEL_MASK; ++ ++ intel_vgpu_init_i2c_edid(vgpu); ++ ++ if (pin_select == 0) ++ return 0; ++ ++ if (IS_BROXTON(dev_priv)) ++ port = bxt_get_port_from_gmbus0(pin_select); ++ else if (IS_COFFEELAKE(dev_priv)) ++ port = cnp_get_port_from_gmbus0(pin_select); ++ else ++ port = get_port_from_gmbus0(pin_select); ++ if (WARN_ON(port < 0)) ++ return 0; ++ ++ vgpu->display.i2c_edid.state = I2C_GMBUS; ++ vgpu->display.i2c_edid.gmbus.phase = GMBUS_IDLE_PHASE; ++ ++ vgpu_vreg_t(vgpu, PCH_GMBUS2) &= ~GMBUS_ACTIVE; ++ vgpu_vreg_t(vgpu, PCH_GMBUS2) |= GMBUS_HW_RDY | GMBUS_HW_WAIT_PHASE; ++ ++ if (intel_vgpu_has_monitor_on_port(vgpu, port) && ++ !intel_vgpu_port_is_dp(vgpu, port)) { ++ vgpu->display.i2c_edid.port = port; ++ vgpu->display.i2c_edid.edid_available = true; ++ vgpu_vreg_t(vgpu, PCH_GMBUS2) &= ~GMBUS_SATOER; ++ } else ++ vgpu_vreg_t(vgpu, PCH_GMBUS2) |= GMBUS_SATOER; ++ return 0; ++} ++ ++static int gmbus1_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ struct intel_vgpu_i2c_edid *i2c_edid = &vgpu->display.i2c_edid; ++ u32 slave_addr; ++ u32 wvalue = *(u32 *)p_data; ++ ++ if (vgpu_vreg(vgpu, offset) & GMBUS_SW_CLR_INT) { ++ if (!(wvalue & GMBUS_SW_CLR_INT)) { ++ vgpu_vreg(vgpu, offset) &= ~GMBUS_SW_CLR_INT; ++ reset_gmbus_controller(vgpu); ++ } ++ /* ++ * TODO: "This bit is cleared to zero when an event ++ * causes the HW_RDY bit transition to occur " ++ */ ++ } else { ++ /* ++ * per bspec setting this bit can cause: ++ * 1) INT status bit cleared ++ * 2) HW_RDY bit asserted ++ */ ++ if (wvalue & GMBUS_SW_CLR_INT) { ++ vgpu_vreg_t(vgpu, PCH_GMBUS2) &= ~GMBUS_INT; ++ vgpu_vreg_t(vgpu, PCH_GMBUS2) |= GMBUS_HW_RDY; ++ } ++ ++ /* For virtualization, we suppose that HW is always ready, ++ * so GMBUS_SW_RDY should always be cleared ++ */ ++ if (wvalue & GMBUS_SW_RDY) ++ wvalue &= ~GMBUS_SW_RDY; ++ ++ i2c_edid->gmbus.total_byte_count = ++ gmbus1_total_byte_count(wvalue); ++ slave_addr = gmbus1_slave_addr(wvalue); ++ ++ /* vgpu gmbus only support EDID */ ++ if (slave_addr == EDID_ADDR) { ++ i2c_edid->slave_selected = true; ++ } else if (slave_addr != 0) { ++ gvt_dbg_dpy( ++ "vgpu%d: unsupported gmbus slave addr(0x%x)\n" ++ " gmbus operations will be ignored.\n", ++ vgpu->id, slave_addr); ++ } ++ ++ if (wvalue & GMBUS_CYCLE_INDEX) ++ i2c_edid->current_edid_read = ++ gmbus1_slave_index(wvalue); ++ ++ i2c_edid->gmbus.cycle_type = gmbus1_bus_cycle(wvalue); ++ switch (gmbus1_bus_cycle(wvalue)) { ++ case GMBUS_NOCYCLE: ++ break; ++ case GMBUS_STOP: ++ /* From spec: ++ * This can only cause a STOP to be generated ++ * if a GMBUS cycle is generated, the GMBUS is ++ * currently in a data/wait/idle phase, or it is in a ++ * WAIT phase ++ */ ++ if (gmbus1_bus_cycle(vgpu_vreg(vgpu, offset)) ++ != GMBUS_NOCYCLE) { ++ intel_vgpu_init_i2c_edid(vgpu); ++ /* After the 'stop' cycle, hw state would become ++ * 'stop phase' and then 'idle phase' after a ++ * few milliseconds. In emulation, we just set ++ * it as 'idle phase' ('stop phase' is not ++ * visible in gmbus interface) ++ */ ++ i2c_edid->gmbus.phase = GMBUS_IDLE_PHASE; ++ vgpu_vreg_t(vgpu, PCH_GMBUS2) &= ~GMBUS_ACTIVE; ++ } ++ break; ++ case NIDX_NS_W: ++ case IDX_NS_W: ++ case NIDX_STOP: ++ case IDX_STOP: ++ /* From hw spec the GMBUS phase ++ * transition like this: ++ * START (-->INDEX) -->DATA ++ */ ++ i2c_edid->gmbus.phase = GMBUS_DATA_PHASE; ++ vgpu_vreg_t(vgpu, PCH_GMBUS2) |= GMBUS_ACTIVE; ++ break; ++ default: ++ gvt_vgpu_err("Unknown/reserved GMBUS cycle detected!\n"); ++ break; ++ } ++ /* ++ * From hw spec the WAIT state will be ++ * cleared: ++ * (1) in a new GMBUS cycle ++ * (2) by generating a stop ++ */ ++ vgpu_vreg(vgpu, offset) = wvalue; ++ } ++ return 0; ++} ++ ++static int gmbus3_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ WARN_ON(1); ++ return 0; ++} ++ ++static int gmbus3_mmio_read(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ int i; ++ unsigned char byte_data; ++ struct intel_vgpu_i2c_edid *i2c_edid = &vgpu->display.i2c_edid; ++ int byte_left = i2c_edid->gmbus.total_byte_count - ++ i2c_edid->current_edid_read; ++ int byte_count = byte_left; ++ u32 reg_data = 0; ++ ++ /* Data can only be recevied if previous settings correct */ ++ if (vgpu_vreg_t(vgpu, PCH_GMBUS1) & GMBUS_SLAVE_READ) { ++ if (byte_left <= 0) { ++ memcpy(p_data, &vgpu_vreg(vgpu, offset), bytes); ++ return 0; ++ } ++ ++ if (byte_count > 4) ++ byte_count = 4; ++ for (i = 0; i < byte_count; i++) { ++ byte_data = edid_get_byte(vgpu); ++ reg_data |= (byte_data << (i << 3)); ++ } ++ ++ memcpy(&vgpu_vreg(vgpu, offset), ®_data, byte_count); ++ memcpy(p_data, &vgpu_vreg(vgpu, offset), bytes); ++ ++ if (byte_left <= 4) { ++ switch (i2c_edid->gmbus.cycle_type) { ++ case NIDX_STOP: ++ case IDX_STOP: ++ i2c_edid->gmbus.phase = GMBUS_IDLE_PHASE; ++ break; ++ case NIDX_NS_W: ++ case IDX_NS_W: ++ default: ++ i2c_edid->gmbus.phase = GMBUS_WAIT_PHASE; ++ break; ++ } ++ intel_vgpu_init_i2c_edid(vgpu); ++ } ++ /* ++ * Read GMBUS3 during send operation, ++ * return the latest written value ++ */ ++ } else { ++ memcpy(p_data, &vgpu_vreg(vgpu, offset), bytes); ++ gvt_vgpu_err("warning: gmbus3 read with nothing returned\n"); ++ } ++ return 0; ++} ++ ++static int gmbus2_mmio_read(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ u32 value = vgpu_vreg(vgpu, offset); ++ ++ if (!(vgpu_vreg(vgpu, offset) & GMBUS_INUSE)) ++ vgpu_vreg(vgpu, offset) |= GMBUS_INUSE; ++ memcpy(p_data, (void *)&value, bytes); ++ return 0; ++} ++ ++static int gmbus2_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ u32 wvalue = *(u32 *)p_data; ++ ++ if (wvalue & GMBUS_INUSE) ++ vgpu_vreg(vgpu, offset) &= ~GMBUS_INUSE; ++ /* All other bits are read-only */ ++ return 0; ++} ++ ++/** ++ * intel_gvt_i2c_handle_gmbus_read - emulate gmbus register mmio read ++ * @vgpu: a vGPU ++ * @offset: reg offset ++ * @p_data: data return buffer ++ * @bytes: access data length ++ * ++ * This function is used to emulate gmbus register mmio read ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ * ++ */ ++int intel_gvt_i2c_handle_gmbus_read(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ if (WARN_ON(bytes > 8 && (offset & (bytes - 1)))) ++ return -EINVAL; ++ ++ if (offset == i915_mmio_reg_offset(PCH_GMBUS2)) ++ return gmbus2_mmio_read(vgpu, offset, p_data, bytes); ++ else if (offset == i915_mmio_reg_offset(PCH_GMBUS3)) ++ return gmbus3_mmio_read(vgpu, offset, p_data, bytes); ++ ++ memcpy(p_data, &vgpu_vreg(vgpu, offset), bytes); ++ return 0; ++} ++ ++/** ++ * intel_gvt_i2c_handle_gmbus_write - emulate gmbus register mmio write ++ * @vgpu: a vGPU ++ * @offset: reg offset ++ * @p_data: data return buffer ++ * @bytes: access data length ++ * ++ * This function is used to emulate gmbus register mmio write ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ * ++ */ ++int intel_gvt_i2c_handle_gmbus_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ if (WARN_ON(bytes > 8 && (offset & (bytes - 1)))) ++ return -EINVAL; ++ ++ if (offset == i915_mmio_reg_offset(PCH_GMBUS0)) ++ return gmbus0_mmio_write(vgpu, offset, p_data, bytes); ++ else if (offset == i915_mmio_reg_offset(PCH_GMBUS1)) ++ return gmbus1_mmio_write(vgpu, offset, p_data, bytes); ++ else if (offset == i915_mmio_reg_offset(PCH_GMBUS2)) ++ return gmbus2_mmio_write(vgpu, offset, p_data, bytes); ++ else if (offset == i915_mmio_reg_offset(PCH_GMBUS3)) ++ return gmbus3_mmio_write(vgpu, offset, p_data, bytes); ++ ++ memcpy(&vgpu_vreg(vgpu, offset), p_data, bytes); ++ return 0; ++} ++ ++enum { ++ AUX_CH_CTL = 0, ++ AUX_CH_DATA1, ++ AUX_CH_DATA2, ++ AUX_CH_DATA3, ++ AUX_CH_DATA4, ++ AUX_CH_DATA5 ++}; ++ ++static inline int get_aux_ch_reg(unsigned int offset) ++{ ++ int reg; ++ ++ switch (offset & 0xff) { ++ case 0x10: ++ reg = AUX_CH_CTL; ++ break; ++ case 0x14: ++ reg = AUX_CH_DATA1; ++ break; ++ case 0x18: ++ reg = AUX_CH_DATA2; ++ break; ++ case 0x1c: ++ reg = AUX_CH_DATA3; ++ break; ++ case 0x20: ++ reg = AUX_CH_DATA4; ++ break; ++ case 0x24: ++ reg = AUX_CH_DATA5; ++ break; ++ default: ++ reg = -1; ++ break; ++ } ++ return reg; ++} ++ ++#define AUX_CTL_MSG_LENGTH(reg) \ ++ ((reg & DP_AUX_CH_CTL_MESSAGE_SIZE_MASK) >> \ ++ DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT) ++ ++/** ++ * intel_gvt_i2c_handle_aux_ch_write - emulate AUX channel register write ++ * @vgpu: a vGPU ++ * @port_idx: port index ++ * @offset: reg offset ++ * @p_data: write ptr ++ * ++ * This function is used to emulate AUX channel register write ++ * ++ */ ++void intel_gvt_i2c_handle_aux_ch_write(struct intel_vgpu *vgpu, ++ int port_idx, ++ unsigned int offset, ++ void *p_data) ++{ ++ struct intel_vgpu_i2c_edid *i2c_edid = &vgpu->display.i2c_edid; ++ int msg_length, ret_msg_size; ++ int msg, addr, ctrl, op; ++ u32 value = *(u32 *)p_data; ++ int aux_data_for_write = 0; ++ int reg = get_aux_ch_reg(offset); ++ ++ if (reg != AUX_CH_CTL) { ++ vgpu_vreg(vgpu, offset) = value; ++ return; ++ } ++ ++ msg_length = AUX_CTL_MSG_LENGTH(value); ++ // check the msg in DATA register. ++ msg = vgpu_vreg(vgpu, offset + 4); ++ addr = (msg >> 8) & 0xffff; ++ ctrl = (msg >> 24) & 0xff; ++ op = ctrl >> 4; ++ if (!(value & DP_AUX_CH_CTL_SEND_BUSY)) { ++ /* The ctl write to clear some states */ ++ return; ++ } ++ ++ /* Always set the wanted value for vms. */ ++ ret_msg_size = (((op & 0x1) == GVT_AUX_I2C_READ) ? 2 : 1); ++ vgpu_vreg(vgpu, offset) = ++ DP_AUX_CH_CTL_DONE | ++ ((ret_msg_size << DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT) & ++ DP_AUX_CH_CTL_MESSAGE_SIZE_MASK); ++ ++ if (msg_length == 3) { ++ if (!(op & GVT_AUX_I2C_MOT)) { ++ /* stop */ ++ intel_vgpu_init_i2c_edid(vgpu); ++ } else { ++ /* start or restart */ ++ i2c_edid->aux_ch.i2c_over_aux_ch = true; ++ i2c_edid->aux_ch.aux_ch_mot = true; ++ if (addr == 0) { ++ /* reset the address */ ++ intel_vgpu_init_i2c_edid(vgpu); ++ } else if (addr == EDID_ADDR) { ++ i2c_edid->state = I2C_AUX_CH; ++ i2c_edid->port = port_idx; ++ i2c_edid->slave_selected = true; ++ if (intel_vgpu_has_monitor_on_port(vgpu, ++ port_idx) && ++ intel_vgpu_port_is_dp(vgpu, port_idx)) ++ i2c_edid->edid_available = true; ++ } ++ } ++ } else if ((op & 0x1) == GVT_AUX_I2C_WRITE) { ++ /* TODO ++ * We only support EDID reading from I2C_over_AUX. And ++ * we do not expect the index mode to be used. Right now ++ * the WRITE operation is ignored. It is good enough to ++ * support the gfx driver to do EDID access. ++ */ ++ } else { ++ if (WARN_ON((op & 0x1) != GVT_AUX_I2C_READ)) ++ return; ++ if (WARN_ON(msg_length != 4)) ++ return; ++ if (i2c_edid->edid_available && i2c_edid->slave_selected) { ++ unsigned char val = edid_get_byte(vgpu); ++ ++ aux_data_for_write = (val << 16); ++ } else ++ aux_data_for_write = (0xff << 16); ++ } ++ /* write the return value in AUX_CH_DATA reg which includes: ++ * ACK of I2C_WRITE ++ * returned byte if it is READ ++ */ ++ aux_data_for_write |= GVT_AUX_I2C_REPLY_ACK << 24; ++ vgpu_vreg(vgpu, offset + 4) = aux_data_for_write; ++} ++ ++/** ++ * intel_vgpu_init_i2c_edid - initialize vGPU i2c edid emulation ++ * @vgpu: a vGPU ++ * ++ * This function is used to initialize vGPU i2c edid emulation stuffs ++ * ++ */ ++void intel_vgpu_init_i2c_edid(struct intel_vgpu *vgpu) ++{ ++ struct intel_vgpu_i2c_edid *edid = &vgpu->display.i2c_edid; ++ ++ edid->state = I2C_NOT_SPECIFIED; ++ ++ edid->port = -1; ++ edid->slave_selected = false; ++ edid->edid_available = false; ++ edid->current_edid_read = 0; ++ ++ memset(&edid->gmbus, 0, sizeof(struct intel_vgpu_i2c_gmbus)); ++ ++ edid->aux_ch.i2c_over_aux_ch = false; ++ edid->aux_ch.aux_ch_mot = false; ++} +diff --git a/drivers/gpu/drm/i915_legacy/gvt/edid.h b/drivers/gpu/drm/i915_legacy/gvt/edid.h +new file mode 100644 +index 000000000000..f6dfc8b795ec +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/edid.h +@@ -0,0 +1,150 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Ke Yu ++ * Zhiyuan Lv ++ * ++ * Contributors: ++ * Terrence Xu ++ * Changbin Du ++ * Bing Niu ++ * Zhi Wang ++ * ++ */ ++ ++#ifndef _GVT_EDID_H_ ++#define _GVT_EDID_H_ ++ ++#define EDID_SIZE 128 ++#define EDID_ADDR 0x50 /* Linux hvm EDID addr */ ++ ++#define GVT_AUX_NATIVE_WRITE 0x8 ++#define GVT_AUX_NATIVE_READ 0x9 ++#define GVT_AUX_I2C_WRITE 0x0 ++#define GVT_AUX_I2C_READ 0x1 ++#define GVT_AUX_I2C_STATUS 0x2 ++#define GVT_AUX_I2C_MOT 0x4 ++#define GVT_AUX_I2C_REPLY_ACK 0x0 ++ ++struct intel_vgpu_edid_data { ++ bool data_valid; ++ unsigned char edid_block[EDID_SIZE]; ++}; ++ ++enum gmbus_cycle_type { ++ GMBUS_NOCYCLE = 0x0, ++ NIDX_NS_W = 0x1, ++ IDX_NS_W = 0x3, ++ GMBUS_STOP = 0x4, ++ NIDX_STOP = 0x5, ++ IDX_STOP = 0x7 ++}; ++ ++/* ++ * States of GMBUS ++ * ++ * GMBUS0-3 could be related to the EDID virtualization. Another two GMBUS ++ * registers, GMBUS4 (interrupt mask) and GMBUS5 (2 byte indes register), are ++ * not considered here. Below describes the usage of GMBUS registers that are ++ * cared by the EDID virtualization ++ * ++ * GMBUS0: ++ * R/W ++ * port selection. value of bit0 - bit2 corresponds to the GPIO registers. ++ * ++ * GMBUS1: ++ * R/W Protect ++ * Command and Status. ++ * bit0 is the direction bit: 1 is read; 0 is write. ++ * bit1 - bit7 is slave 7-bit address. ++ * bit16 - bit24 total byte count (ignore?) ++ * ++ * GMBUS2: ++ * Most of bits are read only except bit 15 (IN_USE) ++ * Status register ++ * bit0 - bit8 current byte count ++ * bit 11: hardware ready; ++ * ++ * GMBUS3: ++ * Read/Write ++ * Data for transfer ++ */ ++ ++/* From hw specs, Other phases like START, ADDRESS, INDEX ++ * are invisible to GMBUS MMIO interface. So no definitions ++ * in below enum types ++ */ ++enum gvt_gmbus_phase { ++ GMBUS_IDLE_PHASE = 0, ++ GMBUS_DATA_PHASE, ++ GMBUS_WAIT_PHASE, ++ //GMBUS_STOP_PHASE, ++ GMBUS_MAX_PHASE ++}; ++ ++struct intel_vgpu_i2c_gmbus { ++ unsigned int total_byte_count; /* from GMBUS1 */ ++ enum gmbus_cycle_type cycle_type; ++ enum gvt_gmbus_phase phase; ++}; ++ ++struct intel_vgpu_i2c_aux_ch { ++ bool i2c_over_aux_ch; ++ bool aux_ch_mot; ++}; ++ ++enum i2c_state { ++ I2C_NOT_SPECIFIED = 0, ++ I2C_GMBUS = 1, ++ I2C_AUX_CH = 2 ++}; ++ ++/* I2C sequences cannot interleave. ++ * GMBUS and AUX_CH sequences cannot interleave. ++ */ ++struct intel_vgpu_i2c_edid { ++ enum i2c_state state; ++ ++ unsigned int port; ++ bool slave_selected; ++ bool edid_available; ++ unsigned int current_edid_read; ++ ++ struct intel_vgpu_i2c_gmbus gmbus; ++ struct intel_vgpu_i2c_aux_ch aux_ch; ++}; ++ ++void intel_vgpu_init_i2c_edid(struct intel_vgpu *vgpu); ++ ++int intel_gvt_i2c_handle_gmbus_read(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes); ++ ++int intel_gvt_i2c_handle_gmbus_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes); ++ ++void intel_gvt_i2c_handle_aux_ch_write(struct intel_vgpu *vgpu, ++ int port_idx, ++ unsigned int offset, ++ void *p_data); ++ ++#endif /*_GVT_EDID_H_*/ +diff --git a/drivers/gpu/drm/i915_legacy/gvt/execlist.c b/drivers/gpu/drm/i915_legacy/gvt/execlist.c +new file mode 100644 +index 000000000000..f21b8fb5b37e +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/execlist.c +@@ -0,0 +1,567 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Zhiyuan Lv ++ * Zhi Wang ++ * ++ * Contributors: ++ * Min He ++ * Bing Niu ++ * Ping Gao ++ * Tina Zhang ++ * ++ */ ++ ++#include "i915_drv.h" ++#include "gvt.h" ++ ++#define _EL_OFFSET_STATUS 0x234 ++#define _EL_OFFSET_STATUS_BUF 0x370 ++#define _EL_OFFSET_STATUS_PTR 0x3A0 ++ ++#define execlist_ring_mmio(gvt, ring_id, offset) \ ++ (gvt->dev_priv->engine[ring_id]->mmio_base + (offset)) ++ ++#define valid_context(ctx) ((ctx)->valid) ++#define same_context(a, b) (((a)->context_id == (b)->context_id) && \ ++ ((a)->lrca == (b)->lrca)) ++ ++static int context_switch_events[] = { ++ [RCS0] = RCS_AS_CONTEXT_SWITCH, ++ [BCS0] = BCS_AS_CONTEXT_SWITCH, ++ [VCS0] = VCS_AS_CONTEXT_SWITCH, ++ [VCS1] = VCS2_AS_CONTEXT_SWITCH, ++ [VECS0] = VECS_AS_CONTEXT_SWITCH, ++}; ++ ++static int ring_id_to_context_switch_event(unsigned int ring_id) ++{ ++ if (WARN_ON(ring_id >= ARRAY_SIZE(context_switch_events))) ++ return -EINVAL; ++ ++ return context_switch_events[ring_id]; ++} ++ ++static void switch_virtual_execlist_slot(struct intel_vgpu_execlist *execlist) ++{ ++ gvt_dbg_el("[before] running slot %d/context %x pending slot %d\n", ++ execlist->running_slot ? ++ execlist->running_slot->index : -1, ++ execlist->running_context ? ++ execlist->running_context->context_id : 0, ++ execlist->pending_slot ? ++ execlist->pending_slot->index : -1); ++ ++ execlist->running_slot = execlist->pending_slot; ++ execlist->pending_slot = NULL; ++ execlist->running_context = execlist->running_context ? ++ &execlist->running_slot->ctx[0] : NULL; ++ ++ gvt_dbg_el("[after] running slot %d/context %x pending slot %d\n", ++ execlist->running_slot ? ++ execlist->running_slot->index : -1, ++ execlist->running_context ? ++ execlist->running_context->context_id : 0, ++ execlist->pending_slot ? ++ execlist->pending_slot->index : -1); ++} ++ ++static void emulate_execlist_status(struct intel_vgpu_execlist *execlist) ++{ ++ struct intel_vgpu_execlist_slot *running = execlist->running_slot; ++ struct intel_vgpu_execlist_slot *pending = execlist->pending_slot; ++ struct execlist_ctx_descriptor_format *desc = execlist->running_context; ++ struct intel_vgpu *vgpu = execlist->vgpu; ++ struct execlist_status_format status; ++ int ring_id = execlist->ring_id; ++ u32 status_reg = execlist_ring_mmio(vgpu->gvt, ++ ring_id, _EL_OFFSET_STATUS); ++ ++ status.ldw = vgpu_vreg(vgpu, status_reg); ++ status.udw = vgpu_vreg(vgpu, status_reg + 4); ++ ++ if (running) { ++ status.current_execlist_pointer = !!running->index; ++ status.execlist_write_pointer = !!!running->index; ++ status.execlist_0_active = status.execlist_0_valid = ++ !!!(running->index); ++ status.execlist_1_active = status.execlist_1_valid = ++ !!(running->index); ++ } else { ++ status.context_id = 0; ++ status.execlist_0_active = status.execlist_0_valid = 0; ++ status.execlist_1_active = status.execlist_1_valid = 0; ++ } ++ ++ status.context_id = desc ? desc->context_id : 0; ++ status.execlist_queue_full = !!(pending); ++ ++ vgpu_vreg(vgpu, status_reg) = status.ldw; ++ vgpu_vreg(vgpu, status_reg + 4) = status.udw; ++ ++ gvt_dbg_el("vgpu%d: status reg offset %x ldw %x udw %x\n", ++ vgpu->id, status_reg, status.ldw, status.udw); ++} ++ ++static void emulate_csb_update(struct intel_vgpu_execlist *execlist, ++ struct execlist_context_status_format *status, ++ bool trigger_interrupt_later) ++{ ++ struct intel_vgpu *vgpu = execlist->vgpu; ++ int ring_id = execlist->ring_id; ++ struct execlist_context_status_pointer_format ctx_status_ptr; ++ u32 write_pointer; ++ u32 ctx_status_ptr_reg, ctx_status_buf_reg, offset; ++ unsigned long hwsp_gpa; ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ ++ ctx_status_ptr_reg = execlist_ring_mmio(vgpu->gvt, ring_id, ++ _EL_OFFSET_STATUS_PTR); ++ ctx_status_buf_reg = execlist_ring_mmio(vgpu->gvt, ring_id, ++ _EL_OFFSET_STATUS_BUF); ++ ++ ctx_status_ptr.dw = vgpu_vreg(vgpu, ctx_status_ptr_reg); ++ ++ write_pointer = ctx_status_ptr.write_ptr; ++ ++ if (write_pointer == 0x7) ++ write_pointer = 0; ++ else { ++ ++write_pointer; ++ write_pointer %= 0x6; ++ } ++ ++ offset = ctx_status_buf_reg + write_pointer * 8; ++ ++ vgpu_vreg(vgpu, offset) = status->ldw; ++ vgpu_vreg(vgpu, offset + 4) = status->udw; ++ ++ ctx_status_ptr.write_ptr = write_pointer; ++ vgpu_vreg(vgpu, ctx_status_ptr_reg) = ctx_status_ptr.dw; ++ ++ /* Update the CSB and CSB write pointer in HWSP */ ++ hwsp_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, ++ vgpu->hws_pga[ring_id]); ++ if (hwsp_gpa != INTEL_GVT_INVALID_ADDR) { ++ intel_gvt_hypervisor_write_gpa(vgpu, ++ hwsp_gpa + I915_HWS_CSB_BUF0_INDEX * 4 + ++ write_pointer * 8, ++ status, 8); ++ intel_gvt_hypervisor_write_gpa(vgpu, ++ hwsp_gpa + ++ intel_hws_csb_write_index(dev_priv) * 4, ++ &write_pointer, 4); ++ } ++ ++ gvt_dbg_el("vgpu%d: w pointer %u reg %x csb l %x csb h %x\n", ++ vgpu->id, write_pointer, offset, status->ldw, status->udw); ++ ++ if (trigger_interrupt_later) ++ return; ++ ++ intel_vgpu_trigger_virtual_event(vgpu, ++ ring_id_to_context_switch_event(execlist->ring_id)); ++} ++ ++static int emulate_execlist_ctx_schedule_out( ++ struct intel_vgpu_execlist *execlist, ++ struct execlist_ctx_descriptor_format *ctx) ++{ ++ struct intel_vgpu *vgpu = execlist->vgpu; ++ struct intel_vgpu_execlist_slot *running = execlist->running_slot; ++ struct intel_vgpu_execlist_slot *pending = execlist->pending_slot; ++ struct execlist_ctx_descriptor_format *ctx0 = &running->ctx[0]; ++ struct execlist_ctx_descriptor_format *ctx1 = &running->ctx[1]; ++ struct execlist_context_status_format status; ++ ++ memset(&status, 0, sizeof(status)); ++ ++ gvt_dbg_el("schedule out context id %x\n", ctx->context_id); ++ ++ if (WARN_ON(!same_context(ctx, execlist->running_context))) { ++ gvt_vgpu_err("schedule out context is not running context," ++ "ctx id %x running ctx id %x\n", ++ ctx->context_id, ++ execlist->running_context->context_id); ++ return -EINVAL; ++ } ++ ++ /* ctx1 is valid, ctx0/ctx is scheduled-out -> element switch */ ++ if (valid_context(ctx1) && same_context(ctx0, ctx)) { ++ gvt_dbg_el("ctx 1 valid, ctx/ctx 0 is scheduled-out\n"); ++ ++ execlist->running_context = ctx1; ++ ++ emulate_execlist_status(execlist); ++ ++ status.context_complete = status.element_switch = 1; ++ status.context_id = ctx->context_id; ++ ++ emulate_csb_update(execlist, &status, false); ++ /* ++ * ctx1 is not valid, ctx == ctx0 ++ * ctx1 is valid, ctx1 == ctx ++ * --> last element is finished ++ * emulate: ++ * active-to-idle if there is *no* pending execlist ++ * context-complete if there *is* pending execlist ++ */ ++ } else if ((!valid_context(ctx1) && same_context(ctx0, ctx)) ++ || (valid_context(ctx1) && same_context(ctx1, ctx))) { ++ gvt_dbg_el("need to switch virtual execlist slot\n"); ++ ++ switch_virtual_execlist_slot(execlist); ++ ++ emulate_execlist_status(execlist); ++ ++ status.context_complete = status.active_to_idle = 1; ++ status.context_id = ctx->context_id; ++ ++ if (!pending) { ++ emulate_csb_update(execlist, &status, false); ++ } else { ++ emulate_csb_update(execlist, &status, true); ++ ++ memset(&status, 0, sizeof(status)); ++ ++ status.idle_to_active = 1; ++ status.context_id = 0; ++ ++ emulate_csb_update(execlist, &status, false); ++ } ++ } else { ++ WARN_ON(1); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static struct intel_vgpu_execlist_slot *get_next_execlist_slot( ++ struct intel_vgpu_execlist *execlist) ++{ ++ struct intel_vgpu *vgpu = execlist->vgpu; ++ int ring_id = execlist->ring_id; ++ u32 status_reg = execlist_ring_mmio(vgpu->gvt, ring_id, ++ _EL_OFFSET_STATUS); ++ struct execlist_status_format status; ++ ++ status.ldw = vgpu_vreg(vgpu, status_reg); ++ status.udw = vgpu_vreg(vgpu, status_reg + 4); ++ ++ if (status.execlist_queue_full) { ++ gvt_vgpu_err("virtual execlist slots are full\n"); ++ return NULL; ++ } ++ ++ return &execlist->slot[status.execlist_write_pointer]; ++} ++ ++static int emulate_execlist_schedule_in(struct intel_vgpu_execlist *execlist, ++ struct execlist_ctx_descriptor_format ctx[2]) ++{ ++ struct intel_vgpu_execlist_slot *running = execlist->running_slot; ++ struct intel_vgpu_execlist_slot *slot = ++ get_next_execlist_slot(execlist); ++ ++ struct execlist_ctx_descriptor_format *ctx0, *ctx1; ++ struct execlist_context_status_format status; ++ struct intel_vgpu *vgpu = execlist->vgpu; ++ ++ gvt_dbg_el("emulate schedule-in\n"); ++ ++ if (!slot) { ++ gvt_vgpu_err("no available execlist slot\n"); ++ return -EINVAL; ++ } ++ ++ memset(&status, 0, sizeof(status)); ++ memset(slot->ctx, 0, sizeof(slot->ctx)); ++ ++ slot->ctx[0] = ctx[0]; ++ slot->ctx[1] = ctx[1]; ++ ++ gvt_dbg_el("alloc slot index %d ctx 0 %x ctx 1 %x\n", ++ slot->index, ctx[0].context_id, ++ ctx[1].context_id); ++ ++ /* ++ * no running execlist, make this write bundle as running execlist ++ * -> idle-to-active ++ */ ++ if (!running) { ++ gvt_dbg_el("no current running execlist\n"); ++ ++ execlist->running_slot = slot; ++ execlist->pending_slot = NULL; ++ execlist->running_context = &slot->ctx[0]; ++ ++ gvt_dbg_el("running slot index %d running context %x\n", ++ execlist->running_slot->index, ++ execlist->running_context->context_id); ++ ++ emulate_execlist_status(execlist); ++ ++ status.idle_to_active = 1; ++ status.context_id = 0; ++ ++ emulate_csb_update(execlist, &status, false); ++ return 0; ++ } ++ ++ ctx0 = &running->ctx[0]; ++ ctx1 = &running->ctx[1]; ++ ++ gvt_dbg_el("current running slot index %d ctx 0 %x ctx 1 %x\n", ++ running->index, ctx0->context_id, ctx1->context_id); ++ ++ /* ++ * already has an running execlist ++ * a. running ctx1 is valid, ++ * ctx0 is finished, and running ctx1 == new execlist ctx[0] ++ * b. running ctx1 is not valid, ++ * ctx0 == new execlist ctx[0] ++ * ----> lite-restore + preempted ++ */ ++ if ((valid_context(ctx1) && same_context(ctx1, &slot->ctx[0]) && ++ /* condition a */ ++ (!same_context(ctx0, execlist->running_context))) || ++ (!valid_context(ctx1) && ++ same_context(ctx0, &slot->ctx[0]))) { /* condition b */ ++ gvt_dbg_el("need to switch virtual execlist slot\n"); ++ ++ execlist->pending_slot = slot; ++ switch_virtual_execlist_slot(execlist); ++ ++ emulate_execlist_status(execlist); ++ ++ status.lite_restore = status.preempted = 1; ++ status.context_id = ctx[0].context_id; ++ ++ emulate_csb_update(execlist, &status, false); ++ } else { ++ gvt_dbg_el("emulate as pending slot\n"); ++ /* ++ * otherwise ++ * --> emulate pending execlist exist + but no preemption case ++ */ ++ execlist->pending_slot = slot; ++ emulate_execlist_status(execlist); ++ } ++ return 0; ++} ++ ++#define get_desc_from_elsp_dwords(ed, i) \ ++ ((struct execlist_ctx_descriptor_format *)&((ed)->data[i * 2])) ++ ++static int prepare_execlist_workload(struct intel_vgpu_workload *workload) ++{ ++ struct intel_vgpu *vgpu = workload->vgpu; ++ struct intel_vgpu_submission *s = &vgpu->submission; ++ struct execlist_ctx_descriptor_format ctx[2]; ++ int ring_id = workload->ring_id; ++ int ret; ++ ++ if (!workload->emulate_schedule_in) ++ return 0; ++ ++ ctx[0] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 0); ++ ctx[1] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 1); ++ ++ ret = emulate_execlist_schedule_in(&s->execlist[ring_id], ctx); ++ if (ret) { ++ gvt_vgpu_err("fail to emulate execlist schedule in\n"); ++ return ret; ++ } ++ return 0; ++} ++ ++static int complete_execlist_workload(struct intel_vgpu_workload *workload) ++{ ++ struct intel_vgpu *vgpu = workload->vgpu; ++ int ring_id = workload->ring_id; ++ struct intel_vgpu_submission *s = &vgpu->submission; ++ struct intel_vgpu_execlist *execlist = &s->execlist[ring_id]; ++ struct intel_vgpu_workload *next_workload; ++ struct list_head *next = workload_q_head(vgpu, ring_id)->next; ++ bool lite_restore = false; ++ int ret = 0; ++ ++ gvt_dbg_el("complete workload %p status %d\n", workload, ++ workload->status); ++ ++ if (workload->status || (vgpu->resetting_eng & BIT(ring_id))) ++ goto out; ++ ++ if (!list_empty(workload_q_head(vgpu, ring_id))) { ++ struct execlist_ctx_descriptor_format *this_desc, *next_desc; ++ ++ next_workload = container_of(next, ++ struct intel_vgpu_workload, list); ++ this_desc = &workload->ctx_desc; ++ next_desc = &next_workload->ctx_desc; ++ ++ lite_restore = same_context(this_desc, next_desc); ++ } ++ ++ if (lite_restore) { ++ gvt_dbg_el("next context == current - no schedule-out\n"); ++ goto out; ++ } ++ ++ ret = emulate_execlist_ctx_schedule_out(execlist, &workload->ctx_desc); ++out: ++ intel_vgpu_unpin_mm(workload->shadow_mm); ++ intel_vgpu_destroy_workload(workload); ++ return ret; ++} ++ ++static int submit_context(struct intel_vgpu *vgpu, int ring_id, ++ struct execlist_ctx_descriptor_format *desc, ++ bool emulate_schedule_in) ++{ ++ struct intel_vgpu_submission *s = &vgpu->submission; ++ struct intel_vgpu_workload *workload = NULL; ++ ++ workload = intel_vgpu_create_workload(vgpu, ring_id, desc); ++ if (IS_ERR(workload)) ++ return PTR_ERR(workload); ++ ++ workload->prepare = prepare_execlist_workload; ++ workload->complete = complete_execlist_workload; ++ workload->emulate_schedule_in = emulate_schedule_in; ++ ++ if (emulate_schedule_in) ++ workload->elsp_dwords = s->execlist[ring_id].elsp_dwords; ++ ++ gvt_dbg_el("workload %p emulate schedule_in %d\n", workload, ++ emulate_schedule_in); ++ ++ intel_vgpu_queue_workload(workload); ++ return 0; ++} ++ ++int intel_vgpu_submit_execlist(struct intel_vgpu *vgpu, int ring_id) ++{ ++ struct intel_vgpu_submission *s = &vgpu->submission; ++ struct intel_vgpu_execlist *execlist = &s->execlist[ring_id]; ++ struct execlist_ctx_descriptor_format *desc[2]; ++ int i, ret; ++ ++ desc[0] = get_desc_from_elsp_dwords(&execlist->elsp_dwords, 0); ++ desc[1] = get_desc_from_elsp_dwords(&execlist->elsp_dwords, 1); ++ ++ if (!desc[0]->valid) { ++ gvt_vgpu_err("invalid elsp submission, desc0 is invalid\n"); ++ goto inv_desc; ++ } ++ ++ for (i = 0; i < ARRAY_SIZE(desc); i++) { ++ if (!desc[i]->valid) ++ continue; ++ if (!desc[i]->privilege_access) { ++ gvt_vgpu_err("unexpected GGTT elsp submission\n"); ++ goto inv_desc; ++ } ++ } ++ ++ /* submit workload */ ++ for (i = 0; i < ARRAY_SIZE(desc); i++) { ++ if (!desc[i]->valid) ++ continue; ++ ret = submit_context(vgpu, ring_id, desc[i], i == 0); ++ if (ret) { ++ gvt_vgpu_err("failed to submit desc %d\n", i); ++ return ret; ++ } ++ } ++ ++ return 0; ++ ++inv_desc: ++ gvt_vgpu_err("descriptors content: desc0 %08x %08x desc1 %08x %08x\n", ++ desc[0]->udw, desc[0]->ldw, desc[1]->udw, desc[1]->ldw); ++ return -EINVAL; ++} ++ ++static void init_vgpu_execlist(struct intel_vgpu *vgpu, int ring_id) ++{ ++ struct intel_vgpu_submission *s = &vgpu->submission; ++ struct intel_vgpu_execlist *execlist = &s->execlist[ring_id]; ++ struct execlist_context_status_pointer_format ctx_status_ptr; ++ u32 ctx_status_ptr_reg; ++ ++ memset(execlist, 0, sizeof(*execlist)); ++ ++ execlist->vgpu = vgpu; ++ execlist->ring_id = ring_id; ++ execlist->slot[0].index = 0; ++ execlist->slot[1].index = 1; ++ ++ ctx_status_ptr_reg = execlist_ring_mmio(vgpu->gvt, ring_id, ++ _EL_OFFSET_STATUS_PTR); ++ ctx_status_ptr.dw = vgpu_vreg(vgpu, ctx_status_ptr_reg); ++ ctx_status_ptr.read_ptr = 0; ++ ctx_status_ptr.write_ptr = 0x7; ++ vgpu_vreg(vgpu, ctx_status_ptr_reg) = ctx_status_ptr.dw; ++} ++ ++static void clean_execlist(struct intel_vgpu *vgpu, ++ intel_engine_mask_t engine_mask) ++{ ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ struct intel_engine_cs *engine; ++ struct intel_vgpu_submission *s = &vgpu->submission; ++ intel_engine_mask_t tmp; ++ ++ for_each_engine_masked(engine, dev_priv, engine_mask, tmp) { ++ kfree(s->ring_scan_buffer[engine->id]); ++ s->ring_scan_buffer[engine->id] = NULL; ++ s->ring_scan_buffer_size[engine->id] = 0; ++ } ++} ++ ++static void reset_execlist(struct intel_vgpu *vgpu, ++ intel_engine_mask_t engine_mask) ++{ ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ struct intel_engine_cs *engine; ++ intel_engine_mask_t tmp; ++ ++ for_each_engine_masked(engine, dev_priv, engine_mask, tmp) ++ init_vgpu_execlist(vgpu, engine->id); ++} ++ ++static int init_execlist(struct intel_vgpu *vgpu, ++ intel_engine_mask_t engine_mask) ++{ ++ reset_execlist(vgpu, engine_mask); ++ return 0; ++} ++ ++const struct intel_vgpu_submission_ops intel_vgpu_execlist_submission_ops = { ++ .name = "execlist", ++ .init = init_execlist, ++ .reset = reset_execlist, ++ .clean = clean_execlist, ++}; +diff --git a/drivers/gpu/drm/i915_legacy/gvt/execlist.h b/drivers/gpu/drm/i915_legacy/gvt/execlist.h +new file mode 100644 +index 000000000000..5ccc2c695848 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/execlist.h +@@ -0,0 +1,185 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Zhiyuan Lv ++ * Zhi Wang ++ * ++ * Contributors: ++ * Min He ++ * Bing Niu ++ * Ping Gao ++ * Tina Zhang ++ * ++ */ ++ ++#ifndef _GVT_EXECLIST_H_ ++#define _GVT_EXECLIST_H_ ++ ++struct execlist_ctx_descriptor_format { ++ union { ++ u32 ldw; ++ struct { ++ u32 valid : 1; ++ u32 force_pd_restore : 1; ++ u32 force_restore : 1; ++ u32 addressing_mode : 2; ++ u32 llc_coherency : 1; ++ u32 fault_handling : 2; ++ u32 privilege_access : 1; ++ u32 reserved : 3; ++ u32 lrca : 20; ++ }; ++ }; ++ union { ++ u32 udw; ++ u32 context_id; ++ }; ++}; ++ ++struct execlist_status_format { ++ union { ++ u32 ldw; ++ struct { ++ u32 current_execlist_pointer :1; ++ u32 execlist_write_pointer :1; ++ u32 execlist_queue_full :1; ++ u32 execlist_1_valid :1; ++ u32 execlist_0_valid :1; ++ u32 last_ctx_switch_reason :9; ++ u32 current_active_elm_status :2; ++ u32 arbitration_enable :1; ++ u32 execlist_1_active :1; ++ u32 execlist_0_active :1; ++ u32 reserved :13; ++ }; ++ }; ++ union { ++ u32 udw; ++ u32 context_id; ++ }; ++}; ++ ++struct execlist_context_status_pointer_format { ++ union { ++ u32 dw; ++ struct { ++ u32 write_ptr :3; ++ u32 reserved :5; ++ u32 read_ptr :3; ++ u32 reserved2 :5; ++ u32 mask :16; ++ }; ++ }; ++}; ++ ++struct execlist_context_status_format { ++ union { ++ u32 ldw; ++ struct { ++ u32 idle_to_active :1; ++ u32 preempted :1; ++ u32 element_switch :1; ++ u32 active_to_idle :1; ++ u32 context_complete :1; ++ u32 wait_on_sync_flip :1; ++ u32 wait_on_vblank :1; ++ u32 wait_on_semaphore :1; ++ u32 wait_on_scanline :1; ++ u32 reserved :2; ++ u32 semaphore_wait_mode :1; ++ u32 display_plane :3; ++ u32 lite_restore :1; ++ u32 reserved_2 :16; ++ }; ++ }; ++ union { ++ u32 udw; ++ u32 context_id; ++ }; ++}; ++ ++struct execlist_mmio_pair { ++ u32 addr; ++ u32 val; ++}; ++ ++/* The first 52 dwords in register state context */ ++struct execlist_ring_context { ++ u32 nop1; ++ u32 lri_cmd_1; ++ struct execlist_mmio_pair ctx_ctrl; ++ struct execlist_mmio_pair ring_header; ++ struct execlist_mmio_pair ring_tail; ++ struct execlist_mmio_pair rb_start; ++ struct execlist_mmio_pair rb_ctrl; ++ struct execlist_mmio_pair bb_cur_head_UDW; ++ struct execlist_mmio_pair bb_cur_head_LDW; ++ struct execlist_mmio_pair bb_state; ++ struct execlist_mmio_pair second_bb_addr_UDW; ++ struct execlist_mmio_pair second_bb_addr_LDW; ++ struct execlist_mmio_pair second_bb_state; ++ struct execlist_mmio_pair bb_per_ctx_ptr; ++ struct execlist_mmio_pair rcs_indirect_ctx; ++ struct execlist_mmio_pair rcs_indirect_ctx_offset; ++ u32 nop2; ++ u32 nop3; ++ u32 nop4; ++ u32 lri_cmd_2; ++ struct execlist_mmio_pair ctx_timestamp; ++ /* ++ * pdps[8]={ pdp3_UDW, pdp3_LDW, pdp2_UDW, pdp2_LDW, ++ * pdp1_UDW, pdp1_LDW, pdp0_UDW, pdp0_LDW} ++ */ ++ struct execlist_mmio_pair pdps[8]; ++}; ++ ++struct intel_vgpu_elsp_dwords { ++ u32 data[4]; ++ u32 index; ++}; ++ ++struct intel_vgpu_execlist_slot { ++ struct execlist_ctx_descriptor_format ctx[2]; ++ u32 index; ++}; ++ ++struct intel_vgpu_execlist { ++ struct intel_vgpu_execlist_slot slot[2]; ++ struct intel_vgpu_execlist_slot *running_slot; ++ struct intel_vgpu_execlist_slot *pending_slot; ++ struct execlist_ctx_descriptor_format *running_context; ++ int ring_id; ++ struct intel_vgpu *vgpu; ++ struct intel_vgpu_elsp_dwords elsp_dwords; ++}; ++ ++void intel_vgpu_clean_execlist(struct intel_vgpu *vgpu); ++ ++int intel_vgpu_init_execlist(struct intel_vgpu *vgpu); ++ ++int intel_vgpu_submit_execlist(struct intel_vgpu *vgpu, int ring_id); ++ ++void intel_vgpu_reset_execlist(struct intel_vgpu *vgpu, ++ intel_engine_mask_t engine_mask); ++ ++#endif /*_GVT_EXECLIST_H_*/ +diff --git a/drivers/gpu/drm/i915_legacy/gvt/fb_decoder.c b/drivers/gpu/drm/i915_legacy/gvt/fb_decoder.c +new file mode 100644 +index 000000000000..65e847392aea +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/fb_decoder.c +@@ -0,0 +1,507 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Kevin Tian ++ * ++ * Contributors: ++ * Bing Niu ++ * Xu Han ++ * Ping Gao ++ * Xiaoguang Chen ++ * Yang Liu ++ * Tina Zhang ++ * ++ */ ++ ++#include ++#include "i915_drv.h" ++#include "gvt.h" ++#include "i915_pvinfo.h" ++ ++#define PRIMARY_FORMAT_NUM 16 ++struct pixel_format { ++ int drm_format; /* Pixel format in DRM definition */ ++ int bpp; /* Bits per pixel, 0 indicates invalid */ ++ char *desc; /* The description */ ++}; ++ ++static struct pixel_format bdw_pixel_formats[] = { ++ {DRM_FORMAT_C8, 8, "8-bit Indexed"}, ++ {DRM_FORMAT_RGB565, 16, "16-bit BGRX (5:6:5 MSB-R:G:B)"}, ++ {DRM_FORMAT_XRGB8888, 32, "32-bit BGRX (8:8:8:8 MSB-X:R:G:B)"}, ++ {DRM_FORMAT_XBGR2101010, 32, "32-bit RGBX (2:10:10:10 MSB-X:B:G:R)"}, ++ ++ {DRM_FORMAT_XRGB2101010, 32, "32-bit BGRX (2:10:10:10 MSB-X:R:G:B)"}, ++ {DRM_FORMAT_XBGR8888, 32, "32-bit RGBX (8:8:8:8 MSB-X:B:G:R)"}, ++ ++ /* non-supported format has bpp default to 0 */ ++ {0, 0, NULL}, ++}; ++ ++static struct pixel_format skl_pixel_formats[] = { ++ {DRM_FORMAT_YUYV, 16, "16-bit packed YUYV (8:8:8:8 MSB-V:Y2:U:Y1)"}, ++ {DRM_FORMAT_UYVY, 16, "16-bit packed UYVY (8:8:8:8 MSB-Y2:V:Y1:U)"}, ++ {DRM_FORMAT_YVYU, 16, "16-bit packed YVYU (8:8:8:8 MSB-U:Y2:V:Y1)"}, ++ {DRM_FORMAT_VYUY, 16, "16-bit packed VYUY (8:8:8:8 MSB-Y2:U:Y1:V)"}, ++ ++ {DRM_FORMAT_C8, 8, "8-bit Indexed"}, ++ {DRM_FORMAT_RGB565, 16, "16-bit BGRX (5:6:5 MSB-R:G:B)"}, ++ {DRM_FORMAT_ABGR8888, 32, "32-bit RGBA (8:8:8:8 MSB-A:B:G:R)"}, ++ {DRM_FORMAT_XBGR8888, 32, "32-bit RGBX (8:8:8:8 MSB-X:B:G:R)"}, ++ ++ {DRM_FORMAT_ARGB8888, 32, "32-bit BGRA (8:8:8:8 MSB-A:R:G:B)"}, ++ {DRM_FORMAT_XRGB8888, 32, "32-bit BGRX (8:8:8:8 MSB-X:R:G:B)"}, ++ {DRM_FORMAT_XBGR2101010, 32, "32-bit RGBX (2:10:10:10 MSB-X:B:G:R)"}, ++ {DRM_FORMAT_XRGB2101010, 32, "32-bit BGRX (2:10:10:10 MSB-X:R:G:B)"}, ++ ++ /* non-supported format has bpp default to 0 */ ++ {0, 0, NULL}, ++}; ++ ++static int bdw_format_to_drm(int format) ++{ ++ int bdw_pixel_formats_index = 6; ++ ++ switch (format) { ++ case DISPPLANE_8BPP: ++ bdw_pixel_formats_index = 0; ++ break; ++ case DISPPLANE_BGRX565: ++ bdw_pixel_formats_index = 1; ++ break; ++ case DISPPLANE_BGRX888: ++ bdw_pixel_formats_index = 2; ++ break; ++ case DISPPLANE_RGBX101010: ++ bdw_pixel_formats_index = 3; ++ break; ++ case DISPPLANE_BGRX101010: ++ bdw_pixel_formats_index = 4; ++ break; ++ case DISPPLANE_RGBX888: ++ bdw_pixel_formats_index = 5; ++ break; ++ ++ default: ++ break; ++ } ++ ++ return bdw_pixel_formats_index; ++} ++ ++static int skl_format_to_drm(int format, bool rgb_order, bool alpha, ++ int yuv_order) ++{ ++ int skl_pixel_formats_index = 12; ++ ++ switch (format) { ++ case PLANE_CTL_FORMAT_INDEXED: ++ skl_pixel_formats_index = 4; ++ break; ++ case PLANE_CTL_FORMAT_RGB_565: ++ skl_pixel_formats_index = 5; ++ break; ++ case PLANE_CTL_FORMAT_XRGB_8888: ++ if (rgb_order) ++ skl_pixel_formats_index = alpha ? 6 : 7; ++ else ++ skl_pixel_formats_index = alpha ? 8 : 9; ++ break; ++ case PLANE_CTL_FORMAT_XRGB_2101010: ++ skl_pixel_formats_index = rgb_order ? 10 : 11; ++ break; ++ case PLANE_CTL_FORMAT_YUV422: ++ skl_pixel_formats_index = yuv_order >> 16; ++ if (skl_pixel_formats_index > 3) ++ return -EINVAL; ++ break; ++ ++ default: ++ break; ++ } ++ ++ return skl_pixel_formats_index; ++} ++ ++static u32 intel_vgpu_get_stride(struct intel_vgpu *vgpu, int pipe, ++ u32 tiled, int stride_mask, int bpp) ++{ ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ ++ u32 stride_reg = vgpu_vreg_t(vgpu, DSPSTRIDE(pipe)) & stride_mask; ++ u32 stride = stride_reg; ++ ++ if (INTEL_GEN(dev_priv) >= 9) { ++ switch (tiled) { ++ case PLANE_CTL_TILED_LINEAR: ++ stride = stride_reg * 64; ++ break; ++ case PLANE_CTL_TILED_X: ++ stride = stride_reg * 512; ++ break; ++ case PLANE_CTL_TILED_Y: ++ stride = stride_reg * 128; ++ break; ++ case PLANE_CTL_TILED_YF: ++ if (bpp == 8) ++ stride = stride_reg * 64; ++ else if (bpp == 16 || bpp == 32 || bpp == 64) ++ stride = stride_reg * 128; ++ else ++ gvt_dbg_core("skl: unsupported bpp:%d\n", bpp); ++ break; ++ default: ++ gvt_dbg_core("skl: unsupported tile format:%x\n", ++ tiled); ++ } ++ } ++ ++ return stride; ++} ++ ++static int get_active_pipe(struct intel_vgpu *vgpu) ++{ ++ int i; ++ ++ for (i = 0; i < I915_MAX_PIPES; i++) ++ if (pipe_is_enabled(vgpu, i)) ++ break; ++ ++ return i; ++} ++ ++/** ++ * intel_vgpu_decode_primary_plane - Decode primary plane ++ * @vgpu: input vgpu ++ * @plane: primary plane to save decoded info ++ * This function is called for decoding plane ++ * ++ * Returns: ++ * 0 on success, non-zero if failed. ++ */ ++int intel_vgpu_decode_primary_plane(struct intel_vgpu *vgpu, ++ struct intel_vgpu_primary_plane_format *plane) ++{ ++ u32 val, fmt; ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ int pipe; ++ ++ pipe = get_active_pipe(vgpu); ++ if (pipe >= I915_MAX_PIPES) ++ return -ENODEV; ++ ++ val = vgpu_vreg_t(vgpu, DSPCNTR(pipe)); ++ plane->enabled = !!(val & DISPLAY_PLANE_ENABLE); ++ if (!plane->enabled) ++ return -ENODEV; ++ ++ if (INTEL_GEN(dev_priv) >= 9) { ++ plane->tiled = val & PLANE_CTL_TILED_MASK; ++ fmt = skl_format_to_drm( ++ val & PLANE_CTL_FORMAT_MASK, ++ val & PLANE_CTL_ORDER_RGBX, ++ val & PLANE_CTL_ALPHA_MASK, ++ val & PLANE_CTL_YUV422_ORDER_MASK); ++ ++ if (fmt >= ARRAY_SIZE(skl_pixel_formats)) { ++ gvt_vgpu_err("Out-of-bounds pixel format index\n"); ++ return -EINVAL; ++ } ++ ++ plane->bpp = skl_pixel_formats[fmt].bpp; ++ plane->drm_format = skl_pixel_formats[fmt].drm_format; ++ } else { ++ plane->tiled = val & DISPPLANE_TILED; ++ fmt = bdw_format_to_drm(val & DISPPLANE_PIXFORMAT_MASK); ++ plane->bpp = bdw_pixel_formats[fmt].bpp; ++ plane->drm_format = bdw_pixel_formats[fmt].drm_format; ++ } ++ ++ if (!plane->bpp) { ++ gvt_vgpu_err("Non-supported pixel format (0x%x)\n", fmt); ++ return -EINVAL; ++ } ++ ++ plane->hw_format = fmt; ++ ++ plane->base = vgpu_vreg_t(vgpu, DSPSURF(pipe)) & I915_GTT_PAGE_MASK; ++ if (!intel_gvt_ggtt_validate_range(vgpu, plane->base, 0)) ++ return -EINVAL; ++ ++ plane->base_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, plane->base); ++ if (plane->base_gpa == INTEL_GVT_INVALID_ADDR) { ++ gvt_vgpu_err("Translate primary plane gma 0x%x to gpa fail\n", ++ plane->base); ++ return -EINVAL; ++ } ++ ++ plane->stride = intel_vgpu_get_stride(vgpu, pipe, plane->tiled, ++ (INTEL_GEN(dev_priv) >= 9) ? ++ (_PRI_PLANE_STRIDE_MASK >> 6) : ++ _PRI_PLANE_STRIDE_MASK, plane->bpp); ++ ++ plane->width = (vgpu_vreg_t(vgpu, PIPESRC(pipe)) & _PIPE_H_SRCSZ_MASK) >> ++ _PIPE_H_SRCSZ_SHIFT; ++ plane->width += 1; ++ plane->height = (vgpu_vreg_t(vgpu, PIPESRC(pipe)) & ++ _PIPE_V_SRCSZ_MASK) >> _PIPE_V_SRCSZ_SHIFT; ++ plane->height += 1; /* raw height is one minus the real value */ ++ ++ val = vgpu_vreg_t(vgpu, DSPTILEOFF(pipe)); ++ plane->x_offset = (val & _PRI_PLANE_X_OFF_MASK) >> ++ _PRI_PLANE_X_OFF_SHIFT; ++ plane->y_offset = (val & _PRI_PLANE_Y_OFF_MASK) >> ++ _PRI_PLANE_Y_OFF_SHIFT; ++ ++ return 0; ++} ++ ++#define CURSOR_FORMAT_NUM (1 << 6) ++struct cursor_mode_format { ++ int drm_format; /* Pixel format in DRM definition */ ++ u8 bpp; /* Bits per pixel; 0 indicates invalid */ ++ u32 width; /* In pixel */ ++ u32 height; /* In lines */ ++ char *desc; /* The description */ ++}; ++ ++static struct cursor_mode_format cursor_pixel_formats[] = { ++ {DRM_FORMAT_ARGB8888, 32, 128, 128, "128x128 32bpp ARGB"}, ++ {DRM_FORMAT_ARGB8888, 32, 256, 256, "256x256 32bpp ARGB"}, ++ {DRM_FORMAT_ARGB8888, 32, 64, 64, "64x64 32bpp ARGB"}, ++ {DRM_FORMAT_ARGB8888, 32, 64, 64, "64x64 32bpp ARGB"}, ++ ++ /* non-supported format has bpp default to 0 */ ++ {0, 0, 0, 0, NULL}, ++}; ++ ++static int cursor_mode_to_drm(int mode) ++{ ++ int cursor_pixel_formats_index = 4; ++ ++ switch (mode) { ++ case MCURSOR_MODE_128_ARGB_AX: ++ cursor_pixel_formats_index = 0; ++ break; ++ case MCURSOR_MODE_256_ARGB_AX: ++ cursor_pixel_formats_index = 1; ++ break; ++ case MCURSOR_MODE_64_ARGB_AX: ++ cursor_pixel_formats_index = 2; ++ break; ++ case MCURSOR_MODE_64_32B_AX: ++ cursor_pixel_formats_index = 3; ++ break; ++ ++ default: ++ break; ++ } ++ ++ return cursor_pixel_formats_index; ++} ++ ++/** ++ * intel_vgpu_decode_cursor_plane - Decode sprite plane ++ * @vgpu: input vgpu ++ * @plane: cursor plane to save decoded info ++ * This function is called for decoding plane ++ * ++ * Returns: ++ * 0 on success, non-zero if failed. ++ */ ++int intel_vgpu_decode_cursor_plane(struct intel_vgpu *vgpu, ++ struct intel_vgpu_cursor_plane_format *plane) ++{ ++ u32 val, mode, index; ++ u32 alpha_plane, alpha_force; ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ int pipe; ++ ++ pipe = get_active_pipe(vgpu); ++ if (pipe >= I915_MAX_PIPES) ++ return -ENODEV; ++ ++ val = vgpu_vreg_t(vgpu, CURCNTR(pipe)); ++ mode = val & MCURSOR_MODE; ++ plane->enabled = (mode != MCURSOR_MODE_DISABLE); ++ if (!plane->enabled) ++ return -ENODEV; ++ ++ index = cursor_mode_to_drm(mode); ++ ++ if (!cursor_pixel_formats[index].bpp) { ++ gvt_vgpu_err("Non-supported cursor mode (0x%x)\n", mode); ++ return -EINVAL; ++ } ++ plane->mode = mode; ++ plane->bpp = cursor_pixel_formats[index].bpp; ++ plane->drm_format = cursor_pixel_formats[index].drm_format; ++ plane->width = cursor_pixel_formats[index].width; ++ plane->height = cursor_pixel_formats[index].height; ++ ++ alpha_plane = (val & _CURSOR_ALPHA_PLANE_MASK) >> ++ _CURSOR_ALPHA_PLANE_SHIFT; ++ alpha_force = (val & _CURSOR_ALPHA_FORCE_MASK) >> ++ _CURSOR_ALPHA_FORCE_SHIFT; ++ if (alpha_plane || alpha_force) ++ gvt_dbg_core("alpha_plane=0x%x, alpha_force=0x%x\n", ++ alpha_plane, alpha_force); ++ ++ plane->base = vgpu_vreg_t(vgpu, CURBASE(pipe)) & I915_GTT_PAGE_MASK; ++ if (!intel_gvt_ggtt_validate_range(vgpu, plane->base, 0)) ++ return -EINVAL; ++ ++ plane->base_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, plane->base); ++ if (plane->base_gpa == INTEL_GVT_INVALID_ADDR) { ++ gvt_vgpu_err("Translate cursor plane gma 0x%x to gpa fail\n", ++ plane->base); ++ return -EINVAL; ++ } ++ ++ val = vgpu_vreg_t(vgpu, CURPOS(pipe)); ++ plane->x_pos = (val & _CURSOR_POS_X_MASK) >> _CURSOR_POS_X_SHIFT; ++ plane->x_sign = (val & _CURSOR_SIGN_X_MASK) >> _CURSOR_SIGN_X_SHIFT; ++ plane->y_pos = (val & _CURSOR_POS_Y_MASK) >> _CURSOR_POS_Y_SHIFT; ++ plane->y_sign = (val & _CURSOR_SIGN_Y_MASK) >> _CURSOR_SIGN_Y_SHIFT; ++ ++ plane->x_hot = vgpu_vreg_t(vgpu, vgtif_reg(cursor_x_hot)); ++ plane->y_hot = vgpu_vreg_t(vgpu, vgtif_reg(cursor_y_hot)); ++ return 0; ++} ++ ++#define SPRITE_FORMAT_NUM (1 << 3) ++ ++static struct pixel_format sprite_pixel_formats[SPRITE_FORMAT_NUM] = { ++ [0x0] = {DRM_FORMAT_YUV422, 16, "YUV 16-bit 4:2:2 packed"}, ++ [0x1] = {DRM_FORMAT_XRGB2101010, 32, "RGB 32-bit 2:10:10:10"}, ++ [0x2] = {DRM_FORMAT_XRGB8888, 32, "RGB 32-bit 8:8:8:8"}, ++ [0x4] = {DRM_FORMAT_AYUV, 32, ++ "YUV 32-bit 4:4:4 packed (8:8:8:8 MSB-X:Y:U:V)"}, ++}; ++ ++/** ++ * intel_vgpu_decode_sprite_plane - Decode sprite plane ++ * @vgpu: input vgpu ++ * @plane: sprite plane to save decoded info ++ * This function is called for decoding plane ++ * ++ * Returns: ++ * 0 on success, non-zero if failed. ++ */ ++int intel_vgpu_decode_sprite_plane(struct intel_vgpu *vgpu, ++ struct intel_vgpu_sprite_plane_format *plane) ++{ ++ u32 val, fmt; ++ u32 color_order, yuv_order; ++ int drm_format; ++ int pipe; ++ ++ pipe = get_active_pipe(vgpu); ++ if (pipe >= I915_MAX_PIPES) ++ return -ENODEV; ++ ++ val = vgpu_vreg_t(vgpu, SPRCTL(pipe)); ++ plane->enabled = !!(val & SPRITE_ENABLE); ++ if (!plane->enabled) ++ return -ENODEV; ++ ++ plane->tiled = !!(val & SPRITE_TILED); ++ color_order = !!(val & SPRITE_RGB_ORDER_RGBX); ++ yuv_order = (val & SPRITE_YUV_BYTE_ORDER_MASK) >> ++ _SPRITE_YUV_ORDER_SHIFT; ++ ++ fmt = (val & SPRITE_PIXFORMAT_MASK) >> _SPRITE_FMT_SHIFT; ++ if (!sprite_pixel_formats[fmt].bpp) { ++ gvt_vgpu_err("Non-supported pixel format (0x%x)\n", fmt); ++ return -EINVAL; ++ } ++ plane->hw_format = fmt; ++ plane->bpp = sprite_pixel_formats[fmt].bpp; ++ drm_format = sprite_pixel_formats[fmt].drm_format; ++ ++ /* Order of RGB values in an RGBxxx buffer may be ordered RGB or ++ * BGR depending on the state of the color_order field ++ */ ++ if (!color_order) { ++ if (drm_format == DRM_FORMAT_XRGB2101010) ++ drm_format = DRM_FORMAT_XBGR2101010; ++ else if (drm_format == DRM_FORMAT_XRGB8888) ++ drm_format = DRM_FORMAT_XBGR8888; ++ } ++ ++ if (drm_format == DRM_FORMAT_YUV422) { ++ switch (yuv_order) { ++ case 0: ++ drm_format = DRM_FORMAT_YUYV; ++ break; ++ case 1: ++ drm_format = DRM_FORMAT_UYVY; ++ break; ++ case 2: ++ drm_format = DRM_FORMAT_YVYU; ++ break; ++ case 3: ++ drm_format = DRM_FORMAT_VYUY; ++ break; ++ default: ++ /* yuv_order has only 2 bits */ ++ break; ++ } ++ } ++ ++ plane->drm_format = drm_format; ++ ++ plane->base = vgpu_vreg_t(vgpu, SPRSURF(pipe)) & I915_GTT_PAGE_MASK; ++ if (!intel_gvt_ggtt_validate_range(vgpu, plane->base, 0)) ++ return -EINVAL; ++ ++ plane->base_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, plane->base); ++ if (plane->base_gpa == INTEL_GVT_INVALID_ADDR) { ++ gvt_vgpu_err("Translate sprite plane gma 0x%x to gpa fail\n", ++ plane->base); ++ return -EINVAL; ++ } ++ ++ plane->stride = vgpu_vreg_t(vgpu, SPRSTRIDE(pipe)) & ++ _SPRITE_STRIDE_MASK; ++ ++ val = vgpu_vreg_t(vgpu, SPRSIZE(pipe)); ++ plane->height = (val & _SPRITE_SIZE_HEIGHT_MASK) >> ++ _SPRITE_SIZE_HEIGHT_SHIFT; ++ plane->width = (val & _SPRITE_SIZE_WIDTH_MASK) >> ++ _SPRITE_SIZE_WIDTH_SHIFT; ++ plane->height += 1; /* raw height is one minus the real value */ ++ plane->width += 1; /* raw width is one minus the real value */ ++ ++ val = vgpu_vreg_t(vgpu, SPRPOS(pipe)); ++ plane->x_pos = (val & _SPRITE_POS_X_MASK) >> _SPRITE_POS_X_SHIFT; ++ plane->y_pos = (val & _SPRITE_POS_Y_MASK) >> _SPRITE_POS_Y_SHIFT; ++ ++ val = vgpu_vreg_t(vgpu, SPROFFSET(pipe)); ++ plane->x_offset = (val & _SPRITE_OFFSET_START_X_MASK) >> ++ _SPRITE_OFFSET_START_X_SHIFT; ++ plane->y_offset = (val & _SPRITE_OFFSET_START_Y_MASK) >> ++ _SPRITE_OFFSET_START_Y_SHIFT; ++ ++ return 0; ++} +diff --git a/drivers/gpu/drm/i915_legacy/gvt/fb_decoder.h b/drivers/gpu/drm/i915_legacy/gvt/fb_decoder.h +new file mode 100644 +index 000000000000..60c155085029 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/fb_decoder.h +@@ -0,0 +1,169 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Kevin Tian ++ * ++ * Contributors: ++ * Bing Niu ++ * Xu Han ++ * Ping Gao ++ * Xiaoguang Chen ++ * Yang Liu ++ * Tina Zhang ++ * ++ */ ++ ++#ifndef _GVT_FB_DECODER_H_ ++#define _GVT_FB_DECODER_H_ ++ ++#define _PLANE_CTL_FORMAT_SHIFT 24 ++#define _PLANE_CTL_TILED_SHIFT 10 ++#define _PIPE_V_SRCSZ_SHIFT 0 ++#define _PIPE_V_SRCSZ_MASK (0xfff << _PIPE_V_SRCSZ_SHIFT) ++#define _PIPE_H_SRCSZ_SHIFT 16 ++#define _PIPE_H_SRCSZ_MASK (0x1fff << _PIPE_H_SRCSZ_SHIFT) ++ ++#define _PRI_PLANE_FMT_SHIFT 26 ++#define _PRI_PLANE_STRIDE_MASK (0x3ff << 6) ++#define _PRI_PLANE_X_OFF_SHIFT 0 ++#define _PRI_PLANE_X_OFF_MASK (0x1fff << _PRI_PLANE_X_OFF_SHIFT) ++#define _PRI_PLANE_Y_OFF_SHIFT 16 ++#define _PRI_PLANE_Y_OFF_MASK (0xfff << _PRI_PLANE_Y_OFF_SHIFT) ++ ++#define _CURSOR_MODE 0x3f ++#define _CURSOR_ALPHA_FORCE_SHIFT 8 ++#define _CURSOR_ALPHA_FORCE_MASK (0x3 << _CURSOR_ALPHA_FORCE_SHIFT) ++#define _CURSOR_ALPHA_PLANE_SHIFT 10 ++#define _CURSOR_ALPHA_PLANE_MASK (0x3 << _CURSOR_ALPHA_PLANE_SHIFT) ++#define _CURSOR_POS_X_SHIFT 0 ++#define _CURSOR_POS_X_MASK (0x1fff << _CURSOR_POS_X_SHIFT) ++#define _CURSOR_SIGN_X_SHIFT 15 ++#define _CURSOR_SIGN_X_MASK (1 << _CURSOR_SIGN_X_SHIFT) ++#define _CURSOR_POS_Y_SHIFT 16 ++#define _CURSOR_POS_Y_MASK (0xfff << _CURSOR_POS_Y_SHIFT) ++#define _CURSOR_SIGN_Y_SHIFT 31 ++#define _CURSOR_SIGN_Y_MASK (1 << _CURSOR_SIGN_Y_SHIFT) ++ ++#define _SPRITE_FMT_SHIFT 25 ++#define _SPRITE_COLOR_ORDER_SHIFT 20 ++#define _SPRITE_YUV_ORDER_SHIFT 16 ++#define _SPRITE_STRIDE_SHIFT 6 ++#define _SPRITE_STRIDE_MASK (0x1ff << _SPRITE_STRIDE_SHIFT) ++#define _SPRITE_SIZE_WIDTH_SHIFT 0 ++#define _SPRITE_SIZE_HEIGHT_SHIFT 16 ++#define _SPRITE_SIZE_WIDTH_MASK (0x1fff << _SPRITE_SIZE_WIDTH_SHIFT) ++#define _SPRITE_SIZE_HEIGHT_MASK (0xfff << _SPRITE_SIZE_HEIGHT_SHIFT) ++#define _SPRITE_POS_X_SHIFT 0 ++#define _SPRITE_POS_Y_SHIFT 16 ++#define _SPRITE_POS_X_MASK (0x1fff << _SPRITE_POS_X_SHIFT) ++#define _SPRITE_POS_Y_MASK (0xfff << _SPRITE_POS_Y_SHIFT) ++#define _SPRITE_OFFSET_START_X_SHIFT 0 ++#define _SPRITE_OFFSET_START_Y_SHIFT 16 ++#define _SPRITE_OFFSET_START_X_MASK (0x1fff << _SPRITE_OFFSET_START_X_SHIFT) ++#define _SPRITE_OFFSET_START_Y_MASK (0xfff << _SPRITE_OFFSET_START_Y_SHIFT) ++ ++enum GVT_FB_EVENT { ++ FB_MODE_SET_START = 1, ++ FB_MODE_SET_END, ++ FB_DISPLAY_FLIP, ++}; ++ ++enum DDI_PORT { ++ DDI_PORT_NONE = 0, ++ DDI_PORT_B = 1, ++ DDI_PORT_C = 2, ++ DDI_PORT_D = 3, ++ DDI_PORT_E = 4 ++}; ++ ++struct intel_gvt; ++ ++/* color space conversion and gamma correction are not included */ ++struct intel_vgpu_primary_plane_format { ++ u8 enabled; /* plane is enabled */ ++ u32 tiled; /* tiling mode: linear, X-tiled, Y tiled, etc */ ++ u8 bpp; /* bits per pixel */ ++ u32 hw_format; /* format field in the PRI_CTL register */ ++ u32 drm_format; /* format in DRM definition */ ++ u32 base; /* framebuffer base in graphics memory */ ++ u64 base_gpa; ++ u32 x_offset; /* in pixels */ ++ u32 y_offset; /* in lines */ ++ u32 width; /* in pixels */ ++ u32 height; /* in lines */ ++ u32 stride; /* in bytes */ ++}; ++ ++struct intel_vgpu_sprite_plane_format { ++ u8 enabled; /* plane is enabled */ ++ u8 tiled; /* X-tiled */ ++ u8 bpp; /* bits per pixel */ ++ u32 hw_format; /* format field in the SPR_CTL register */ ++ u32 drm_format; /* format in DRM definition */ ++ u32 base; /* sprite base in graphics memory */ ++ u64 base_gpa; ++ u32 x_pos; /* in pixels */ ++ u32 y_pos; /* in lines */ ++ u32 x_offset; /* in pixels */ ++ u32 y_offset; /* in lines */ ++ u32 width; /* in pixels */ ++ u32 height; /* in lines */ ++ u32 stride; /* in bytes */ ++}; ++ ++struct intel_vgpu_cursor_plane_format { ++ u8 enabled; ++ u8 mode; /* cursor mode select */ ++ u8 bpp; /* bits per pixel */ ++ u32 drm_format; /* format in DRM definition */ ++ u32 base; /* cursor base in graphics memory */ ++ u64 base_gpa; ++ u32 x_pos; /* in pixels */ ++ u32 y_pos; /* in lines */ ++ u8 x_sign; /* X Position Sign */ ++ u8 y_sign; /* Y Position Sign */ ++ u32 width; /* in pixels */ ++ u32 height; /* in lines */ ++ u32 x_hot; /* in pixels */ ++ u32 y_hot; /* in pixels */ ++}; ++ ++struct intel_vgpu_pipe_format { ++ struct intel_vgpu_primary_plane_format primary; ++ struct intel_vgpu_sprite_plane_format sprite; ++ struct intel_vgpu_cursor_plane_format cursor; ++ enum DDI_PORT ddi_port; /* the DDI port that pipe is connected to */ ++}; ++ ++struct intel_vgpu_fb_format { ++ struct intel_vgpu_pipe_format pipes[I915_MAX_PIPES]; ++}; ++ ++int intel_vgpu_decode_primary_plane(struct intel_vgpu *vgpu, ++ struct intel_vgpu_primary_plane_format *plane); ++int intel_vgpu_decode_cursor_plane(struct intel_vgpu *vgpu, ++ struct intel_vgpu_cursor_plane_format *plane); ++int intel_vgpu_decode_sprite_plane(struct intel_vgpu *vgpu, ++ struct intel_vgpu_sprite_plane_format *plane); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/gvt/firmware.c b/drivers/gpu/drm/i915_legacy/gvt/firmware.c +new file mode 100644 +index 000000000000..4ac18b447247 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/firmware.c +@@ -0,0 +1,276 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Zhi Wang ++ * ++ * Contributors: ++ * Changbin Du ++ * ++ */ ++ ++#include ++#include ++ ++#include "i915_drv.h" ++#include "gvt.h" ++#include "i915_pvinfo.h" ++ ++#define FIRMWARE_VERSION (0x0) ++ ++struct gvt_firmware_header { ++ u64 magic; ++ u32 crc32; /* protect the data after this field */ ++ u32 version; ++ u64 cfg_space_size; ++ u64 cfg_space_offset; /* offset in the file */ ++ u64 mmio_size; ++ u64 mmio_offset; /* offset in the file */ ++ unsigned char data[1]; ++}; ++ ++#define dev_to_drm_minor(d) dev_get_drvdata((d)) ++ ++static ssize_t ++gvt_firmware_read(struct file *filp, struct kobject *kobj, ++ struct bin_attribute *attr, char *buf, ++ loff_t offset, size_t count) ++{ ++ memcpy(buf, attr->private + offset, count); ++ return count; ++} ++ ++static struct bin_attribute firmware_attr = { ++ .attr = {.name = "gvt_firmware", .mode = (S_IRUSR)}, ++ .read = gvt_firmware_read, ++ .write = NULL, ++ .mmap = NULL, ++}; ++ ++static int mmio_snapshot_handler(struct intel_gvt *gvt, u32 offset, void *data) ++{ ++ struct drm_i915_private *dev_priv = gvt->dev_priv; ++ ++ *(u32 *)(data + offset) = I915_READ_NOTRACE(_MMIO(offset)); ++ return 0; ++} ++ ++static int expose_firmware_sysfs(struct intel_gvt *gvt) ++{ ++ struct intel_gvt_device_info *info = &gvt->device_info; ++ struct pci_dev *pdev = gvt->dev_priv->drm.pdev; ++ struct gvt_firmware_header *h; ++ void *firmware; ++ void *p; ++ unsigned long size, crc32_start; ++ int i, ret; ++ ++ size = sizeof(*h) + info->mmio_size + info->cfg_space_size; ++ firmware = vzalloc(size); ++ if (!firmware) ++ return -ENOMEM; ++ ++ h = firmware; ++ ++ h->magic = VGT_MAGIC; ++ h->version = FIRMWARE_VERSION; ++ h->cfg_space_size = info->cfg_space_size; ++ h->cfg_space_offset = offsetof(struct gvt_firmware_header, data); ++ h->mmio_size = info->mmio_size; ++ h->mmio_offset = h->cfg_space_offset + h->cfg_space_size; ++ ++ p = firmware + h->cfg_space_offset; ++ ++ for (i = 0; i < h->cfg_space_size; i += 4) ++ pci_read_config_dword(pdev, i, p + i); ++ ++ memcpy(gvt->firmware.cfg_space, p, info->cfg_space_size); ++ ++ p = firmware + h->mmio_offset; ++ ++ /* Take a snapshot of hw mmio registers. */ ++ intel_gvt_for_each_tracked_mmio(gvt, mmio_snapshot_handler, p); ++ ++ memcpy(gvt->firmware.mmio, p, info->mmio_size); ++ ++ crc32_start = offsetof(struct gvt_firmware_header, crc32) + 4; ++ h->crc32 = crc32_le(0, firmware + crc32_start, size - crc32_start); ++ ++ firmware_attr.size = size; ++ firmware_attr.private = firmware; ++ ++ ret = device_create_bin_file(&pdev->dev, &firmware_attr); ++ if (ret) { ++ vfree(firmware); ++ return ret; ++ } ++ return 0; ++} ++ ++static void clean_firmware_sysfs(struct intel_gvt *gvt) ++{ ++ struct pci_dev *pdev = gvt->dev_priv->drm.pdev; ++ ++ device_remove_bin_file(&pdev->dev, &firmware_attr); ++ vfree(firmware_attr.private); ++} ++ ++/** ++ * intel_gvt_free_firmware - free GVT firmware ++ * @gvt: intel gvt device ++ * ++ */ ++void intel_gvt_free_firmware(struct intel_gvt *gvt) ++{ ++ if (!gvt->firmware.firmware_loaded) ++ clean_firmware_sysfs(gvt); ++ ++ kfree(gvt->firmware.cfg_space); ++ kfree(gvt->firmware.mmio); ++} ++ ++static int verify_firmware(struct intel_gvt *gvt, ++ const struct firmware *fw) ++{ ++ struct intel_gvt_device_info *info = &gvt->device_info; ++ struct drm_i915_private *dev_priv = gvt->dev_priv; ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ struct gvt_firmware_header *h; ++ unsigned long id, crc32_start; ++ const void *mem; ++ const char *item; ++ u64 file, request; ++ ++ h = (struct gvt_firmware_header *)fw->data; ++ ++ crc32_start = offsetofend(struct gvt_firmware_header, crc32); ++ mem = fw->data + crc32_start; ++ ++#define VERIFY(s, a, b) do { \ ++ item = (s); file = (u64)(a); request = (u64)(b); \ ++ if ((a) != (b)) \ ++ goto invalid_firmware; \ ++} while (0) ++ ++ VERIFY("magic number", h->magic, VGT_MAGIC); ++ VERIFY("version", h->version, FIRMWARE_VERSION); ++ VERIFY("crc32", h->crc32, crc32_le(0, mem, fw->size - crc32_start)); ++ VERIFY("cfg space size", h->cfg_space_size, info->cfg_space_size); ++ VERIFY("mmio size", h->mmio_size, info->mmio_size); ++ ++ mem = (fw->data + h->cfg_space_offset); ++ ++ id = *(u16 *)(mem + PCI_VENDOR_ID); ++ VERIFY("vender id", id, pdev->vendor); ++ ++ id = *(u16 *)(mem + PCI_DEVICE_ID); ++ VERIFY("device id", id, pdev->device); ++ ++ id = *(u8 *)(mem + PCI_REVISION_ID); ++ VERIFY("revision id", id, pdev->revision); ++ ++#undef VERIFY ++ return 0; ++ ++invalid_firmware: ++ gvt_dbg_core("Invalid firmware: %s [file] 0x%llx [request] 0x%llx\n", ++ item, file, request); ++ return -EINVAL; ++} ++ ++#define GVT_FIRMWARE_PATH "i915/gvt" ++ ++/** ++ * intel_gvt_load_firmware - load GVT firmware ++ * @gvt: intel gvt device ++ * ++ */ ++int intel_gvt_load_firmware(struct intel_gvt *gvt) ++{ ++ struct intel_gvt_device_info *info = &gvt->device_info; ++ struct drm_i915_private *dev_priv = gvt->dev_priv; ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ struct intel_gvt_firmware *firmware = &gvt->firmware; ++ struct gvt_firmware_header *h; ++ const struct firmware *fw; ++ char *path; ++ void *mem; ++ int ret; ++ ++ path = kmalloc(PATH_MAX, GFP_KERNEL); ++ if (!path) ++ return -ENOMEM; ++ ++ mem = kmalloc(info->cfg_space_size, GFP_KERNEL); ++ if (!mem) { ++ kfree(path); ++ return -ENOMEM; ++ } ++ ++ firmware->cfg_space = mem; ++ ++ mem = kmalloc(info->mmio_size, GFP_KERNEL); ++ if (!mem) { ++ kfree(path); ++ kfree(firmware->cfg_space); ++ return -ENOMEM; ++ } ++ ++ firmware->mmio = mem; ++ ++ sprintf(path, "%s/vid_0x%04x_did_0x%04x_rid_0x%02x.golden_hw_state", ++ GVT_FIRMWARE_PATH, pdev->vendor, pdev->device, ++ pdev->revision); ++ ++ gvt_dbg_core("request hw state firmware %s...\n", path); ++ ++ ret = request_firmware(&fw, path, &dev_priv->drm.pdev->dev); ++ kfree(path); ++ ++ if (ret) ++ goto expose_firmware; ++ ++ gvt_dbg_core("success.\n"); ++ ++ ret = verify_firmware(gvt, fw); ++ if (ret) ++ goto out_free_fw; ++ ++ gvt_dbg_core("verified.\n"); ++ ++ h = (struct gvt_firmware_header *)fw->data; ++ ++ memcpy(firmware->cfg_space, fw->data + h->cfg_space_offset, ++ h->cfg_space_size); ++ memcpy(firmware->mmio, fw->data + h->mmio_offset, ++ h->mmio_size); ++ ++ release_firmware(fw); ++ firmware->firmware_loaded = true; ++ return 0; ++ ++out_free_fw: ++ release_firmware(fw); ++expose_firmware: ++ expose_firmware_sysfs(gvt); ++ return 0; ++} +diff --git a/drivers/gpu/drm/i915_legacy/gvt/gtt.c b/drivers/gpu/drm/i915_legacy/gvt/gtt.c +new file mode 100644 +index 000000000000..53115bdae12b +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/gtt.c +@@ -0,0 +1,2818 @@ ++/* ++ * GTT virtualization ++ * ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Zhi Wang ++ * Zhenyu Wang ++ * Xiao Zheng ++ * ++ * Contributors: ++ * Min He ++ * Bing Niu ++ * ++ */ ++ ++#include "i915_drv.h" ++#include "gvt.h" ++#include "i915_pvinfo.h" ++#include "trace.h" ++ ++#if defined(VERBOSE_DEBUG) ++#define gvt_vdbg_mm(fmt, args...) gvt_dbg_mm(fmt, ##args) ++#else ++#define gvt_vdbg_mm(fmt, args...) ++#endif ++ ++static bool enable_out_of_sync = false; ++static int preallocated_oos_pages = 8192; ++ ++/* ++ * validate a gm address and related range size, ++ * translate it to host gm address ++ */ ++bool intel_gvt_ggtt_validate_range(struct intel_vgpu *vgpu, u64 addr, u32 size) ++{ ++ if (size == 0) ++ return vgpu_gmadr_is_valid(vgpu, addr); ++ ++ if (vgpu_gmadr_is_aperture(vgpu, addr) && ++ vgpu_gmadr_is_aperture(vgpu, addr + size - 1)) ++ return true; ++ else if (vgpu_gmadr_is_hidden(vgpu, addr) && ++ vgpu_gmadr_is_hidden(vgpu, addr + size - 1)) ++ return true; ++ ++ gvt_dbg_mm("Invalid ggtt range at 0x%llx, size: 0x%x\n", ++ addr, size); ++ return false; ++} ++ ++/* translate a guest gmadr to host gmadr */ ++int intel_gvt_ggtt_gmadr_g2h(struct intel_vgpu *vgpu, u64 g_addr, u64 *h_addr) ++{ ++ if (WARN(!vgpu_gmadr_is_valid(vgpu, g_addr), ++ "invalid guest gmadr %llx\n", g_addr)) ++ return -EACCES; ++ ++ if (vgpu_gmadr_is_aperture(vgpu, g_addr)) ++ *h_addr = vgpu_aperture_gmadr_base(vgpu) ++ + (g_addr - vgpu_aperture_offset(vgpu)); ++ else ++ *h_addr = vgpu_hidden_gmadr_base(vgpu) ++ + (g_addr - vgpu_hidden_offset(vgpu)); ++ return 0; ++} ++ ++/* translate a host gmadr to guest gmadr */ ++int intel_gvt_ggtt_gmadr_h2g(struct intel_vgpu *vgpu, u64 h_addr, u64 *g_addr) ++{ ++ if (WARN(!gvt_gmadr_is_valid(vgpu->gvt, h_addr), ++ "invalid host gmadr %llx\n", h_addr)) ++ return -EACCES; ++ ++ if (gvt_gmadr_is_aperture(vgpu->gvt, h_addr)) ++ *g_addr = vgpu_aperture_gmadr_base(vgpu) ++ + (h_addr - gvt_aperture_gmadr_base(vgpu->gvt)); ++ else ++ *g_addr = vgpu_hidden_gmadr_base(vgpu) ++ + (h_addr - gvt_hidden_gmadr_base(vgpu->gvt)); ++ return 0; ++} ++ ++int intel_gvt_ggtt_index_g2h(struct intel_vgpu *vgpu, unsigned long g_index, ++ unsigned long *h_index) ++{ ++ u64 h_addr; ++ int ret; ++ ++ ret = intel_gvt_ggtt_gmadr_g2h(vgpu, g_index << I915_GTT_PAGE_SHIFT, ++ &h_addr); ++ if (ret) ++ return ret; ++ ++ *h_index = h_addr >> I915_GTT_PAGE_SHIFT; ++ return 0; ++} ++ ++int intel_gvt_ggtt_h2g_index(struct intel_vgpu *vgpu, unsigned long h_index, ++ unsigned long *g_index) ++{ ++ u64 g_addr; ++ int ret; ++ ++ ret = intel_gvt_ggtt_gmadr_h2g(vgpu, h_index << I915_GTT_PAGE_SHIFT, ++ &g_addr); ++ if (ret) ++ return ret; ++ ++ *g_index = g_addr >> I915_GTT_PAGE_SHIFT; ++ return 0; ++} ++ ++#define gtt_type_is_entry(type) \ ++ (type > GTT_TYPE_INVALID && type < GTT_TYPE_PPGTT_ENTRY \ ++ && type != GTT_TYPE_PPGTT_PTE_ENTRY \ ++ && type != GTT_TYPE_PPGTT_ROOT_ENTRY) ++ ++#define gtt_type_is_pt(type) \ ++ (type >= GTT_TYPE_PPGTT_PTE_PT && type < GTT_TYPE_MAX) ++ ++#define gtt_type_is_pte_pt(type) \ ++ (type == GTT_TYPE_PPGTT_PTE_PT) ++ ++#define gtt_type_is_root_pointer(type) \ ++ (gtt_type_is_entry(type) && type > GTT_TYPE_PPGTT_ROOT_ENTRY) ++ ++#define gtt_init_entry(e, t, p, v) do { \ ++ (e)->type = t; \ ++ (e)->pdev = p; \ ++ memcpy(&(e)->val64, &v, sizeof(v)); \ ++} while (0) ++ ++/* ++ * Mappings between GTT_TYPE* enumerations. ++ * Following information can be found according to the given type: ++ * - type of next level page table ++ * - type of entry inside this level page table ++ * - type of entry with PSE set ++ * ++ * If the given type doesn't have such a kind of information, ++ * e.g. give a l4 root entry type, then request to get its PSE type, ++ * give a PTE page table type, then request to get its next level page ++ * table type, as we know l4 root entry doesn't have a PSE bit, ++ * and a PTE page table doesn't have a next level page table type, ++ * GTT_TYPE_INVALID will be returned. This is useful when traversing a ++ * page table. ++ */ ++ ++struct gtt_type_table_entry { ++ int entry_type; ++ int pt_type; ++ int next_pt_type; ++ int pse_entry_type; ++}; ++ ++#define GTT_TYPE_TABLE_ENTRY(type, e_type, cpt_type, npt_type, pse_type) \ ++ [type] = { \ ++ .entry_type = e_type, \ ++ .pt_type = cpt_type, \ ++ .next_pt_type = npt_type, \ ++ .pse_entry_type = pse_type, \ ++ } ++ ++static struct gtt_type_table_entry gtt_type_table[] = { ++ GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_ROOT_L4_ENTRY, ++ GTT_TYPE_PPGTT_ROOT_L4_ENTRY, ++ GTT_TYPE_INVALID, ++ GTT_TYPE_PPGTT_PML4_PT, ++ GTT_TYPE_INVALID), ++ GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PML4_PT, ++ GTT_TYPE_PPGTT_PML4_ENTRY, ++ GTT_TYPE_PPGTT_PML4_PT, ++ GTT_TYPE_PPGTT_PDP_PT, ++ GTT_TYPE_INVALID), ++ GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PML4_ENTRY, ++ GTT_TYPE_PPGTT_PML4_ENTRY, ++ GTT_TYPE_PPGTT_PML4_PT, ++ GTT_TYPE_PPGTT_PDP_PT, ++ GTT_TYPE_INVALID), ++ GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDP_PT, ++ GTT_TYPE_PPGTT_PDP_ENTRY, ++ GTT_TYPE_PPGTT_PDP_PT, ++ GTT_TYPE_PPGTT_PDE_PT, ++ GTT_TYPE_PPGTT_PTE_1G_ENTRY), ++ GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_ROOT_L3_ENTRY, ++ GTT_TYPE_PPGTT_ROOT_L3_ENTRY, ++ GTT_TYPE_INVALID, ++ GTT_TYPE_PPGTT_PDE_PT, ++ GTT_TYPE_PPGTT_PTE_1G_ENTRY), ++ GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDP_ENTRY, ++ GTT_TYPE_PPGTT_PDP_ENTRY, ++ GTT_TYPE_PPGTT_PDP_PT, ++ GTT_TYPE_PPGTT_PDE_PT, ++ GTT_TYPE_PPGTT_PTE_1G_ENTRY), ++ GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDE_PT, ++ GTT_TYPE_PPGTT_PDE_ENTRY, ++ GTT_TYPE_PPGTT_PDE_PT, ++ GTT_TYPE_PPGTT_PTE_PT, ++ GTT_TYPE_PPGTT_PTE_2M_ENTRY), ++ GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDE_ENTRY, ++ GTT_TYPE_PPGTT_PDE_ENTRY, ++ GTT_TYPE_PPGTT_PDE_PT, ++ GTT_TYPE_PPGTT_PTE_PT, ++ GTT_TYPE_PPGTT_PTE_2M_ENTRY), ++ /* We take IPS bit as 'PSE' for PTE level. */ ++ GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_PT, ++ GTT_TYPE_PPGTT_PTE_4K_ENTRY, ++ GTT_TYPE_PPGTT_PTE_PT, ++ GTT_TYPE_INVALID, ++ GTT_TYPE_PPGTT_PTE_64K_ENTRY), ++ GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_4K_ENTRY, ++ GTT_TYPE_PPGTT_PTE_4K_ENTRY, ++ GTT_TYPE_PPGTT_PTE_PT, ++ GTT_TYPE_INVALID, ++ GTT_TYPE_PPGTT_PTE_64K_ENTRY), ++ GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_64K_ENTRY, ++ GTT_TYPE_PPGTT_PTE_4K_ENTRY, ++ GTT_TYPE_PPGTT_PTE_PT, ++ GTT_TYPE_INVALID, ++ GTT_TYPE_PPGTT_PTE_64K_ENTRY), ++ GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_2M_ENTRY, ++ GTT_TYPE_PPGTT_PDE_ENTRY, ++ GTT_TYPE_PPGTT_PDE_PT, ++ GTT_TYPE_INVALID, ++ GTT_TYPE_PPGTT_PTE_2M_ENTRY), ++ GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_1G_ENTRY, ++ GTT_TYPE_PPGTT_PDP_ENTRY, ++ GTT_TYPE_PPGTT_PDP_PT, ++ GTT_TYPE_INVALID, ++ GTT_TYPE_PPGTT_PTE_1G_ENTRY), ++ GTT_TYPE_TABLE_ENTRY(GTT_TYPE_GGTT_PTE, ++ GTT_TYPE_GGTT_PTE, ++ GTT_TYPE_INVALID, ++ GTT_TYPE_INVALID, ++ GTT_TYPE_INVALID), ++}; ++ ++static inline int get_next_pt_type(int type) ++{ ++ return gtt_type_table[type].next_pt_type; ++} ++ ++static inline int get_pt_type(int type) ++{ ++ return gtt_type_table[type].pt_type; ++} ++ ++static inline int get_entry_type(int type) ++{ ++ return gtt_type_table[type].entry_type; ++} ++ ++static inline int get_pse_type(int type) ++{ ++ return gtt_type_table[type].pse_entry_type; ++} ++ ++static u64 read_pte64(struct drm_i915_private *dev_priv, unsigned long index) ++{ ++ void __iomem *addr = (gen8_pte_t __iomem *)dev_priv->ggtt.gsm + index; ++ ++ return readq(addr); ++} ++ ++static void ggtt_invalidate(struct drm_i915_private *dev_priv) ++{ ++ mmio_hw_access_pre(dev_priv); ++ I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); ++ mmio_hw_access_post(dev_priv); ++} ++ ++static void write_pte64(struct drm_i915_private *dev_priv, ++ unsigned long index, u64 pte) ++{ ++ void __iomem *addr = (gen8_pte_t __iomem *)dev_priv->ggtt.gsm + index; ++ ++ writeq(pte, addr); ++} ++ ++static inline int gtt_get_entry64(void *pt, ++ struct intel_gvt_gtt_entry *e, ++ unsigned long index, bool hypervisor_access, unsigned long gpa, ++ struct intel_vgpu *vgpu) ++{ ++ const struct intel_gvt_device_info *info = &vgpu->gvt->device_info; ++ int ret; ++ ++ if (WARN_ON(info->gtt_entry_size != 8)) ++ return -EINVAL; ++ ++ if (hypervisor_access) { ++ ret = intel_gvt_hypervisor_read_gpa(vgpu, gpa + ++ (index << info->gtt_entry_size_shift), ++ &e->val64, 8); ++ if (WARN_ON(ret)) ++ return ret; ++ } else if (!pt) { ++ e->val64 = read_pte64(vgpu->gvt->dev_priv, index); ++ } else { ++ e->val64 = *((u64 *)pt + index); ++ } ++ return 0; ++} ++ ++static inline int gtt_set_entry64(void *pt, ++ struct intel_gvt_gtt_entry *e, ++ unsigned long index, bool hypervisor_access, unsigned long gpa, ++ struct intel_vgpu *vgpu) ++{ ++ const struct intel_gvt_device_info *info = &vgpu->gvt->device_info; ++ int ret; ++ ++ if (WARN_ON(info->gtt_entry_size != 8)) ++ return -EINVAL; ++ ++ if (hypervisor_access) { ++ ret = intel_gvt_hypervisor_write_gpa(vgpu, gpa + ++ (index << info->gtt_entry_size_shift), ++ &e->val64, 8); ++ if (WARN_ON(ret)) ++ return ret; ++ } else if (!pt) { ++ write_pte64(vgpu->gvt->dev_priv, index, e->val64); ++ } else { ++ *((u64 *)pt + index) = e->val64; ++ } ++ return 0; ++} ++ ++#define GTT_HAW 46 ++ ++#define ADDR_1G_MASK GENMASK_ULL(GTT_HAW - 1, 30) ++#define ADDR_2M_MASK GENMASK_ULL(GTT_HAW - 1, 21) ++#define ADDR_64K_MASK GENMASK_ULL(GTT_HAW - 1, 16) ++#define ADDR_4K_MASK GENMASK_ULL(GTT_HAW - 1, 12) ++ ++#define GTT_SPTE_FLAG_MASK GENMASK_ULL(62, 52) ++#define GTT_SPTE_FLAG_64K_SPLITED BIT(52) /* splited 64K gtt entry */ ++ ++#define GTT_64K_PTE_STRIDE 16 ++ ++static unsigned long gen8_gtt_get_pfn(struct intel_gvt_gtt_entry *e) ++{ ++ unsigned long pfn; ++ ++ if (e->type == GTT_TYPE_PPGTT_PTE_1G_ENTRY) ++ pfn = (e->val64 & ADDR_1G_MASK) >> PAGE_SHIFT; ++ else if (e->type == GTT_TYPE_PPGTT_PTE_2M_ENTRY) ++ pfn = (e->val64 & ADDR_2M_MASK) >> PAGE_SHIFT; ++ else if (e->type == GTT_TYPE_PPGTT_PTE_64K_ENTRY) ++ pfn = (e->val64 & ADDR_64K_MASK) >> PAGE_SHIFT; ++ else ++ pfn = (e->val64 & ADDR_4K_MASK) >> PAGE_SHIFT; ++ return pfn; ++} ++ ++static void gen8_gtt_set_pfn(struct intel_gvt_gtt_entry *e, unsigned long pfn) ++{ ++ if (e->type == GTT_TYPE_PPGTT_PTE_1G_ENTRY) { ++ e->val64 &= ~ADDR_1G_MASK; ++ pfn &= (ADDR_1G_MASK >> PAGE_SHIFT); ++ } else if (e->type == GTT_TYPE_PPGTT_PTE_2M_ENTRY) { ++ e->val64 &= ~ADDR_2M_MASK; ++ pfn &= (ADDR_2M_MASK >> PAGE_SHIFT); ++ } else if (e->type == GTT_TYPE_PPGTT_PTE_64K_ENTRY) { ++ e->val64 &= ~ADDR_64K_MASK; ++ pfn &= (ADDR_64K_MASK >> PAGE_SHIFT); ++ } else { ++ e->val64 &= ~ADDR_4K_MASK; ++ pfn &= (ADDR_4K_MASK >> PAGE_SHIFT); ++ } ++ ++ e->val64 |= (pfn << PAGE_SHIFT); ++} ++ ++static bool gen8_gtt_test_pse(struct intel_gvt_gtt_entry *e) ++{ ++ return !!(e->val64 & _PAGE_PSE); ++} ++ ++static void gen8_gtt_clear_pse(struct intel_gvt_gtt_entry *e) ++{ ++ if (gen8_gtt_test_pse(e)) { ++ switch (e->type) { ++ case GTT_TYPE_PPGTT_PTE_2M_ENTRY: ++ e->val64 &= ~_PAGE_PSE; ++ e->type = GTT_TYPE_PPGTT_PDE_ENTRY; ++ break; ++ case GTT_TYPE_PPGTT_PTE_1G_ENTRY: ++ e->type = GTT_TYPE_PPGTT_PDP_ENTRY; ++ e->val64 &= ~_PAGE_PSE; ++ break; ++ default: ++ WARN_ON(1); ++ } ++ } ++} ++ ++static bool gen8_gtt_test_ips(struct intel_gvt_gtt_entry *e) ++{ ++ if (GEM_WARN_ON(e->type != GTT_TYPE_PPGTT_PDE_ENTRY)) ++ return false; ++ ++ return !!(e->val64 & GEN8_PDE_IPS_64K); ++} ++ ++static void gen8_gtt_clear_ips(struct intel_gvt_gtt_entry *e) ++{ ++ if (GEM_WARN_ON(e->type != GTT_TYPE_PPGTT_PDE_ENTRY)) ++ return; ++ ++ e->val64 &= ~GEN8_PDE_IPS_64K; ++} ++ ++static bool gen8_gtt_test_present(struct intel_gvt_gtt_entry *e) ++{ ++ /* ++ * i915 writes PDP root pointer registers without present bit, ++ * it also works, so we need to treat root pointer entry ++ * specifically. ++ */ ++ if (e->type == GTT_TYPE_PPGTT_ROOT_L3_ENTRY ++ || e->type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY) ++ return (e->val64 != 0); ++ else ++ return (e->val64 & _PAGE_PRESENT); ++} ++ ++static void gtt_entry_clear_present(struct intel_gvt_gtt_entry *e) ++{ ++ e->val64 &= ~_PAGE_PRESENT; ++} ++ ++static void gtt_entry_set_present(struct intel_gvt_gtt_entry *e) ++{ ++ e->val64 |= _PAGE_PRESENT; ++} ++ ++static bool gen8_gtt_test_64k_splited(struct intel_gvt_gtt_entry *e) ++{ ++ return !!(e->val64 & GTT_SPTE_FLAG_64K_SPLITED); ++} ++ ++static void gen8_gtt_set_64k_splited(struct intel_gvt_gtt_entry *e) ++{ ++ e->val64 |= GTT_SPTE_FLAG_64K_SPLITED; ++} ++ ++static void gen8_gtt_clear_64k_splited(struct intel_gvt_gtt_entry *e) ++{ ++ e->val64 &= ~GTT_SPTE_FLAG_64K_SPLITED; ++} ++ ++/* ++ * Per-platform GMA routines. ++ */ ++static unsigned long gma_to_ggtt_pte_index(unsigned long gma) ++{ ++ unsigned long x = (gma >> I915_GTT_PAGE_SHIFT); ++ ++ trace_gma_index(__func__, gma, x); ++ return x; ++} ++ ++#define DEFINE_PPGTT_GMA_TO_INDEX(prefix, ename, exp) \ ++static unsigned long prefix##_gma_to_##ename##_index(unsigned long gma) \ ++{ \ ++ unsigned long x = (exp); \ ++ trace_gma_index(__func__, gma, x); \ ++ return x; \ ++} ++ ++DEFINE_PPGTT_GMA_TO_INDEX(gen8, pte, (gma >> 12 & 0x1ff)); ++DEFINE_PPGTT_GMA_TO_INDEX(gen8, pde, (gma >> 21 & 0x1ff)); ++DEFINE_PPGTT_GMA_TO_INDEX(gen8, l3_pdp, (gma >> 30 & 0x3)); ++DEFINE_PPGTT_GMA_TO_INDEX(gen8, l4_pdp, (gma >> 30 & 0x1ff)); ++DEFINE_PPGTT_GMA_TO_INDEX(gen8, pml4, (gma >> 39 & 0x1ff)); ++ ++static struct intel_gvt_gtt_pte_ops gen8_gtt_pte_ops = { ++ .get_entry = gtt_get_entry64, ++ .set_entry = gtt_set_entry64, ++ .clear_present = gtt_entry_clear_present, ++ .set_present = gtt_entry_set_present, ++ .test_present = gen8_gtt_test_present, ++ .test_pse = gen8_gtt_test_pse, ++ .clear_pse = gen8_gtt_clear_pse, ++ .clear_ips = gen8_gtt_clear_ips, ++ .test_ips = gen8_gtt_test_ips, ++ .clear_64k_splited = gen8_gtt_clear_64k_splited, ++ .set_64k_splited = gen8_gtt_set_64k_splited, ++ .test_64k_splited = gen8_gtt_test_64k_splited, ++ .get_pfn = gen8_gtt_get_pfn, ++ .set_pfn = gen8_gtt_set_pfn, ++}; ++ ++static struct intel_gvt_gtt_gma_ops gen8_gtt_gma_ops = { ++ .gma_to_ggtt_pte_index = gma_to_ggtt_pte_index, ++ .gma_to_pte_index = gen8_gma_to_pte_index, ++ .gma_to_pde_index = gen8_gma_to_pde_index, ++ .gma_to_l3_pdp_index = gen8_gma_to_l3_pdp_index, ++ .gma_to_l4_pdp_index = gen8_gma_to_l4_pdp_index, ++ .gma_to_pml4_index = gen8_gma_to_pml4_index, ++}; ++ ++/* Update entry type per pse and ips bit. */ ++static void update_entry_type_for_real(struct intel_gvt_gtt_pte_ops *pte_ops, ++ struct intel_gvt_gtt_entry *entry, bool ips) ++{ ++ switch (entry->type) { ++ case GTT_TYPE_PPGTT_PDE_ENTRY: ++ case GTT_TYPE_PPGTT_PDP_ENTRY: ++ if (pte_ops->test_pse(entry)) ++ entry->type = get_pse_type(entry->type); ++ break; ++ case GTT_TYPE_PPGTT_PTE_4K_ENTRY: ++ if (ips) ++ entry->type = get_pse_type(entry->type); ++ break; ++ default: ++ GEM_BUG_ON(!gtt_type_is_entry(entry->type)); ++ } ++ ++ GEM_BUG_ON(entry->type == GTT_TYPE_INVALID); ++} ++ ++/* ++ * MM helpers. ++ */ ++static void _ppgtt_get_root_entry(struct intel_vgpu_mm *mm, ++ struct intel_gvt_gtt_entry *entry, unsigned long index, ++ bool guest) ++{ ++ struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops; ++ ++ GEM_BUG_ON(mm->type != INTEL_GVT_MM_PPGTT); ++ ++ entry->type = mm->ppgtt_mm.root_entry_type; ++ pte_ops->get_entry(guest ? mm->ppgtt_mm.guest_pdps : ++ mm->ppgtt_mm.shadow_pdps, ++ entry, index, false, 0, mm->vgpu); ++ update_entry_type_for_real(pte_ops, entry, false); ++} ++ ++static inline void ppgtt_get_guest_root_entry(struct intel_vgpu_mm *mm, ++ struct intel_gvt_gtt_entry *entry, unsigned long index) ++{ ++ _ppgtt_get_root_entry(mm, entry, index, true); ++} ++ ++static inline void ppgtt_get_shadow_root_entry(struct intel_vgpu_mm *mm, ++ struct intel_gvt_gtt_entry *entry, unsigned long index) ++{ ++ _ppgtt_get_root_entry(mm, entry, index, false); ++} ++ ++static void _ppgtt_set_root_entry(struct intel_vgpu_mm *mm, ++ struct intel_gvt_gtt_entry *entry, unsigned long index, ++ bool guest) ++{ ++ struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops; ++ ++ pte_ops->set_entry(guest ? mm->ppgtt_mm.guest_pdps : ++ mm->ppgtt_mm.shadow_pdps, ++ entry, index, false, 0, mm->vgpu); ++} ++ ++static inline void ppgtt_set_guest_root_entry(struct intel_vgpu_mm *mm, ++ struct intel_gvt_gtt_entry *entry, unsigned long index) ++{ ++ _ppgtt_set_root_entry(mm, entry, index, true); ++} ++ ++static inline void ppgtt_set_shadow_root_entry(struct intel_vgpu_mm *mm, ++ struct intel_gvt_gtt_entry *entry, unsigned long index) ++{ ++ _ppgtt_set_root_entry(mm, entry, index, false); ++} ++ ++static void ggtt_get_guest_entry(struct intel_vgpu_mm *mm, ++ struct intel_gvt_gtt_entry *entry, unsigned long index) ++{ ++ struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops; ++ ++ GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT); ++ ++ entry->type = GTT_TYPE_GGTT_PTE; ++ pte_ops->get_entry(mm->ggtt_mm.virtual_ggtt, entry, index, ++ false, 0, mm->vgpu); ++} ++ ++static void ggtt_set_guest_entry(struct intel_vgpu_mm *mm, ++ struct intel_gvt_gtt_entry *entry, unsigned long index) ++{ ++ struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops; ++ ++ GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT); ++ ++ pte_ops->set_entry(mm->ggtt_mm.virtual_ggtt, entry, index, ++ false, 0, mm->vgpu); ++} ++ ++static void ggtt_get_host_entry(struct intel_vgpu_mm *mm, ++ struct intel_gvt_gtt_entry *entry, unsigned long index) ++{ ++ struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops; ++ ++ GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT); ++ ++ pte_ops->get_entry(NULL, entry, index, false, 0, mm->vgpu); ++} ++ ++static void ggtt_set_host_entry(struct intel_vgpu_mm *mm, ++ struct intel_gvt_gtt_entry *entry, unsigned long index) ++{ ++ struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops; ++ ++ GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT); ++ ++ pte_ops->set_entry(NULL, entry, index, false, 0, mm->vgpu); ++} ++ ++/* ++ * PPGTT shadow page table helpers. ++ */ ++static inline int ppgtt_spt_get_entry( ++ struct intel_vgpu_ppgtt_spt *spt, ++ void *page_table, int type, ++ struct intel_gvt_gtt_entry *e, unsigned long index, ++ bool guest) ++{ ++ struct intel_gvt *gvt = spt->vgpu->gvt; ++ struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops; ++ int ret; ++ ++ e->type = get_entry_type(type); ++ ++ if (WARN(!gtt_type_is_entry(e->type), "invalid entry type\n")) ++ return -EINVAL; ++ ++ ret = ops->get_entry(page_table, e, index, guest, ++ spt->guest_page.gfn << I915_GTT_PAGE_SHIFT, ++ spt->vgpu); ++ if (ret) ++ return ret; ++ ++ update_entry_type_for_real(ops, e, guest ? ++ spt->guest_page.pde_ips : false); ++ ++ gvt_vdbg_mm("read ppgtt entry, spt type %d, entry type %d, index %lu, value %llx\n", ++ type, e->type, index, e->val64); ++ return 0; ++} ++ ++static inline int ppgtt_spt_set_entry( ++ struct intel_vgpu_ppgtt_spt *spt, ++ void *page_table, int type, ++ struct intel_gvt_gtt_entry *e, unsigned long index, ++ bool guest) ++{ ++ struct intel_gvt *gvt = spt->vgpu->gvt; ++ struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops; ++ ++ if (WARN(!gtt_type_is_entry(e->type), "invalid entry type\n")) ++ return -EINVAL; ++ ++ gvt_vdbg_mm("set ppgtt entry, spt type %d, entry type %d, index %lu, value %llx\n", ++ type, e->type, index, e->val64); ++ ++ return ops->set_entry(page_table, e, index, guest, ++ spt->guest_page.gfn << I915_GTT_PAGE_SHIFT, ++ spt->vgpu); ++} ++ ++#define ppgtt_get_guest_entry(spt, e, index) \ ++ ppgtt_spt_get_entry(spt, NULL, \ ++ spt->guest_page.type, e, index, true) ++ ++#define ppgtt_set_guest_entry(spt, e, index) \ ++ ppgtt_spt_set_entry(spt, NULL, \ ++ spt->guest_page.type, e, index, true) ++ ++#define ppgtt_get_shadow_entry(spt, e, index) \ ++ ppgtt_spt_get_entry(spt, spt->shadow_page.vaddr, \ ++ spt->shadow_page.type, e, index, false) ++ ++#define ppgtt_set_shadow_entry(spt, e, index) \ ++ ppgtt_spt_set_entry(spt, spt->shadow_page.vaddr, \ ++ spt->shadow_page.type, e, index, false) ++ ++static void *alloc_spt(gfp_t gfp_mask) ++{ ++ struct intel_vgpu_ppgtt_spt *spt; ++ ++ spt = kzalloc(sizeof(*spt), gfp_mask); ++ if (!spt) ++ return NULL; ++ ++ spt->shadow_page.page = alloc_page(gfp_mask); ++ if (!spt->shadow_page.page) { ++ kfree(spt); ++ return NULL; ++ } ++ return spt; ++} ++ ++static void free_spt(struct intel_vgpu_ppgtt_spt *spt) ++{ ++ __free_page(spt->shadow_page.page); ++ kfree(spt); ++} ++ ++static int detach_oos_page(struct intel_vgpu *vgpu, ++ struct intel_vgpu_oos_page *oos_page); ++ ++static void ppgtt_free_spt(struct intel_vgpu_ppgtt_spt *spt) ++{ ++ struct device *kdev = &spt->vgpu->gvt->dev_priv->drm.pdev->dev; ++ ++ trace_spt_free(spt->vgpu->id, spt, spt->guest_page.type); ++ ++ dma_unmap_page(kdev, spt->shadow_page.mfn << I915_GTT_PAGE_SHIFT, 4096, ++ PCI_DMA_BIDIRECTIONAL); ++ ++ radix_tree_delete(&spt->vgpu->gtt.spt_tree, spt->shadow_page.mfn); ++ ++ if (spt->guest_page.gfn) { ++ if (spt->guest_page.oos_page) ++ detach_oos_page(spt->vgpu, spt->guest_page.oos_page); ++ ++ intel_vgpu_unregister_page_track(spt->vgpu, spt->guest_page.gfn); ++ } ++ ++ list_del_init(&spt->post_shadow_list); ++ free_spt(spt); ++} ++ ++static void ppgtt_free_all_spt(struct intel_vgpu *vgpu) ++{ ++ struct intel_vgpu_ppgtt_spt *spt, *spn; ++ struct radix_tree_iter iter; ++ LIST_HEAD(all_spt); ++ void __rcu **slot; ++ ++ rcu_read_lock(); ++ radix_tree_for_each_slot(slot, &vgpu->gtt.spt_tree, &iter, 0) { ++ spt = radix_tree_deref_slot(slot); ++ list_move(&spt->post_shadow_list, &all_spt); ++ } ++ rcu_read_unlock(); ++ ++ list_for_each_entry_safe(spt, spn, &all_spt, post_shadow_list) ++ ppgtt_free_spt(spt); ++} ++ ++static int ppgtt_handle_guest_write_page_table_bytes( ++ struct intel_vgpu_ppgtt_spt *spt, ++ u64 pa, void *p_data, int bytes); ++ ++static int ppgtt_write_protection_handler( ++ struct intel_vgpu_page_track *page_track, ++ u64 gpa, void *data, int bytes) ++{ ++ struct intel_vgpu_ppgtt_spt *spt = page_track->priv_data; ++ ++ int ret; ++ ++ if (bytes != 4 && bytes != 8) ++ return -EINVAL; ++ ++ ret = ppgtt_handle_guest_write_page_table_bytes(spt, gpa, data, bytes); ++ if (ret) ++ return ret; ++ return ret; ++} ++ ++/* Find a spt by guest gfn. */ ++static struct intel_vgpu_ppgtt_spt *intel_vgpu_find_spt_by_gfn( ++ struct intel_vgpu *vgpu, unsigned long gfn) ++{ ++ struct intel_vgpu_page_track *track; ++ ++ track = intel_vgpu_find_page_track(vgpu, gfn); ++ if (track && track->handler == ppgtt_write_protection_handler) ++ return track->priv_data; ++ ++ return NULL; ++} ++ ++/* Find the spt by shadow page mfn. */ ++static inline struct intel_vgpu_ppgtt_spt *intel_vgpu_find_spt_by_mfn( ++ struct intel_vgpu *vgpu, unsigned long mfn) ++{ ++ return radix_tree_lookup(&vgpu->gtt.spt_tree, mfn); ++} ++ ++static int reclaim_one_ppgtt_mm(struct intel_gvt *gvt); ++ ++/* Allocate shadow page table without guest page. */ ++static struct intel_vgpu_ppgtt_spt *ppgtt_alloc_spt( ++ struct intel_vgpu *vgpu, enum intel_gvt_gtt_type type) ++{ ++ struct device *kdev = &vgpu->gvt->dev_priv->drm.pdev->dev; ++ struct intel_vgpu_ppgtt_spt *spt = NULL; ++ dma_addr_t daddr; ++ int ret; ++ ++retry: ++ spt = alloc_spt(GFP_KERNEL | __GFP_ZERO); ++ if (!spt) { ++ if (reclaim_one_ppgtt_mm(vgpu->gvt)) ++ goto retry; ++ ++ gvt_vgpu_err("fail to allocate ppgtt shadow page\n"); ++ return ERR_PTR(-ENOMEM); ++ } ++ ++ spt->vgpu = vgpu; ++ atomic_set(&spt->refcount, 1); ++ INIT_LIST_HEAD(&spt->post_shadow_list); ++ ++ /* ++ * Init shadow_page. ++ */ ++ spt->shadow_page.type = type; ++ daddr = dma_map_page(kdev, spt->shadow_page.page, ++ 0, 4096, PCI_DMA_BIDIRECTIONAL); ++ if (dma_mapping_error(kdev, daddr)) { ++ gvt_vgpu_err("fail to map dma addr\n"); ++ ret = -EINVAL; ++ goto err_free_spt; ++ } ++ spt->shadow_page.vaddr = page_address(spt->shadow_page.page); ++ spt->shadow_page.mfn = daddr >> I915_GTT_PAGE_SHIFT; ++ ++ ret = radix_tree_insert(&vgpu->gtt.spt_tree, spt->shadow_page.mfn, spt); ++ if (ret) ++ goto err_unmap_dma; ++ ++ return spt; ++ ++err_unmap_dma: ++ dma_unmap_page(kdev, daddr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); ++err_free_spt: ++ free_spt(spt); ++ return ERR_PTR(ret); ++} ++ ++/* Allocate shadow page table associated with specific gfn. */ ++static struct intel_vgpu_ppgtt_spt *ppgtt_alloc_spt_gfn( ++ struct intel_vgpu *vgpu, enum intel_gvt_gtt_type type, ++ unsigned long gfn, bool guest_pde_ips) ++{ ++ struct intel_vgpu_ppgtt_spt *spt; ++ int ret; ++ ++ spt = ppgtt_alloc_spt(vgpu, type); ++ if (IS_ERR(spt)) ++ return spt; ++ ++ /* ++ * Init guest_page. ++ */ ++ ret = intel_vgpu_register_page_track(vgpu, gfn, ++ ppgtt_write_protection_handler, spt); ++ if (ret) { ++ ppgtt_free_spt(spt); ++ return ERR_PTR(ret); ++ } ++ ++ spt->guest_page.type = type; ++ spt->guest_page.gfn = gfn; ++ spt->guest_page.pde_ips = guest_pde_ips; ++ ++ trace_spt_alloc(vgpu->id, spt, type, spt->shadow_page.mfn, gfn); ++ ++ return spt; ++} ++ ++#define pt_entry_size_shift(spt) \ ++ ((spt)->vgpu->gvt->device_info.gtt_entry_size_shift) ++ ++#define pt_entries(spt) \ ++ (I915_GTT_PAGE_SIZE >> pt_entry_size_shift(spt)) ++ ++#define for_each_present_guest_entry(spt, e, i) \ ++ for (i = 0; i < pt_entries(spt); \ ++ i += spt->guest_page.pde_ips ? GTT_64K_PTE_STRIDE : 1) \ ++ if (!ppgtt_get_guest_entry(spt, e, i) && \ ++ spt->vgpu->gvt->gtt.pte_ops->test_present(e)) ++ ++#define for_each_present_shadow_entry(spt, e, i) \ ++ for (i = 0; i < pt_entries(spt); \ ++ i += spt->shadow_page.pde_ips ? GTT_64K_PTE_STRIDE : 1) \ ++ if (!ppgtt_get_shadow_entry(spt, e, i) && \ ++ spt->vgpu->gvt->gtt.pte_ops->test_present(e)) ++ ++#define for_each_shadow_entry(spt, e, i) \ ++ for (i = 0; i < pt_entries(spt); \ ++ i += (spt->shadow_page.pde_ips ? GTT_64K_PTE_STRIDE : 1)) \ ++ if (!ppgtt_get_shadow_entry(spt, e, i)) ++ ++static inline void ppgtt_get_spt(struct intel_vgpu_ppgtt_spt *spt) ++{ ++ int v = atomic_read(&spt->refcount); ++ ++ trace_spt_refcount(spt->vgpu->id, "inc", spt, v, (v + 1)); ++ atomic_inc(&spt->refcount); ++} ++ ++static inline int ppgtt_put_spt(struct intel_vgpu_ppgtt_spt *spt) ++{ ++ int v = atomic_read(&spt->refcount); ++ ++ trace_spt_refcount(spt->vgpu->id, "dec", spt, v, (v - 1)); ++ return atomic_dec_return(&spt->refcount); ++} ++ ++static int ppgtt_invalidate_spt(struct intel_vgpu_ppgtt_spt *spt); ++ ++static int ppgtt_invalidate_spt_by_shadow_entry(struct intel_vgpu *vgpu, ++ struct intel_gvt_gtt_entry *e) ++{ ++ struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; ++ struct intel_vgpu_ppgtt_spt *s; ++ enum intel_gvt_gtt_type cur_pt_type; ++ ++ GEM_BUG_ON(!gtt_type_is_pt(get_next_pt_type(e->type))); ++ ++ if (e->type != GTT_TYPE_PPGTT_ROOT_L3_ENTRY ++ && e->type != GTT_TYPE_PPGTT_ROOT_L4_ENTRY) { ++ cur_pt_type = get_next_pt_type(e->type); ++ ++ if (!gtt_type_is_pt(cur_pt_type) || ++ !gtt_type_is_pt(cur_pt_type + 1)) { ++ WARN(1, "Invalid page table type, cur_pt_type is: %d\n", cur_pt_type); ++ return -EINVAL; ++ } ++ ++ cur_pt_type += 1; ++ ++ if (ops->get_pfn(e) == ++ vgpu->gtt.scratch_pt[cur_pt_type].page_mfn) ++ return 0; ++ } ++ s = intel_vgpu_find_spt_by_mfn(vgpu, ops->get_pfn(e)); ++ if (!s) { ++ gvt_vgpu_err("fail to find shadow page: mfn: 0x%lx\n", ++ ops->get_pfn(e)); ++ return -ENXIO; ++ } ++ return ppgtt_invalidate_spt(s); ++} ++ ++static inline void ppgtt_invalidate_pte(struct intel_vgpu_ppgtt_spt *spt, ++ struct intel_gvt_gtt_entry *entry) ++{ ++ struct intel_vgpu *vgpu = spt->vgpu; ++ struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; ++ unsigned long pfn; ++ int type; ++ ++ pfn = ops->get_pfn(entry); ++ type = spt->shadow_page.type; ++ ++ /* Uninitialized spte or unshadowed spte. */ ++ if (!pfn || pfn == vgpu->gtt.scratch_pt[type].page_mfn) ++ return; ++ ++ intel_gvt_hypervisor_dma_unmap_guest_page(vgpu, pfn << PAGE_SHIFT); ++} ++ ++static int ppgtt_invalidate_spt(struct intel_vgpu_ppgtt_spt *spt) ++{ ++ struct intel_vgpu *vgpu = spt->vgpu; ++ struct intel_gvt_gtt_entry e; ++ unsigned long index; ++ int ret; ++ ++ trace_spt_change(spt->vgpu->id, "die", spt, ++ spt->guest_page.gfn, spt->shadow_page.type); ++ ++ if (ppgtt_put_spt(spt) > 0) ++ return 0; ++ ++ for_each_present_shadow_entry(spt, &e, index) { ++ switch (e.type) { ++ case GTT_TYPE_PPGTT_PTE_4K_ENTRY: ++ gvt_vdbg_mm("invalidate 4K entry\n"); ++ ppgtt_invalidate_pte(spt, &e); ++ break; ++ case GTT_TYPE_PPGTT_PTE_64K_ENTRY: ++ /* We don't setup 64K shadow entry so far. */ ++ WARN(1, "suspicious 64K gtt entry\n"); ++ continue; ++ case GTT_TYPE_PPGTT_PTE_2M_ENTRY: ++ gvt_vdbg_mm("invalidate 2M entry\n"); ++ continue; ++ case GTT_TYPE_PPGTT_PTE_1G_ENTRY: ++ WARN(1, "GVT doesn't support 1GB page\n"); ++ continue; ++ case GTT_TYPE_PPGTT_PML4_ENTRY: ++ case GTT_TYPE_PPGTT_PDP_ENTRY: ++ case GTT_TYPE_PPGTT_PDE_ENTRY: ++ gvt_vdbg_mm("invalidate PMUL4/PDP/PDE entry\n"); ++ ret = ppgtt_invalidate_spt_by_shadow_entry( ++ spt->vgpu, &e); ++ if (ret) ++ goto fail; ++ break; ++ default: ++ GEM_BUG_ON(1); ++ } ++ } ++ ++ trace_spt_change(spt->vgpu->id, "release", spt, ++ spt->guest_page.gfn, spt->shadow_page.type); ++ ppgtt_free_spt(spt); ++ return 0; ++fail: ++ gvt_vgpu_err("fail: shadow page %p shadow entry 0x%llx type %d\n", ++ spt, e.val64, e.type); ++ return ret; ++} ++ ++static bool vgpu_ips_enabled(struct intel_vgpu *vgpu) ++{ ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ ++ if (INTEL_GEN(dev_priv) == 9 || INTEL_GEN(dev_priv) == 10) { ++ u32 ips = vgpu_vreg_t(vgpu, GEN8_GAMW_ECO_DEV_RW_IA) & ++ GAMW_ECO_ENABLE_64K_IPS_FIELD; ++ ++ return ips == GAMW_ECO_ENABLE_64K_IPS_FIELD; ++ } else if (INTEL_GEN(dev_priv) >= 11) { ++ /* 64K paging only controlled by IPS bit in PTE now. */ ++ return true; ++ } else ++ return false; ++} ++ ++static int ppgtt_populate_spt(struct intel_vgpu_ppgtt_spt *spt); ++ ++static struct intel_vgpu_ppgtt_spt *ppgtt_populate_spt_by_guest_entry( ++ struct intel_vgpu *vgpu, struct intel_gvt_gtt_entry *we) ++{ ++ struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; ++ struct intel_vgpu_ppgtt_spt *spt = NULL; ++ bool ips = false; ++ int ret; ++ ++ GEM_BUG_ON(!gtt_type_is_pt(get_next_pt_type(we->type))); ++ ++ if (we->type == GTT_TYPE_PPGTT_PDE_ENTRY) ++ ips = vgpu_ips_enabled(vgpu) && ops->test_ips(we); ++ ++ spt = intel_vgpu_find_spt_by_gfn(vgpu, ops->get_pfn(we)); ++ if (spt) { ++ ppgtt_get_spt(spt); ++ ++ if (ips != spt->guest_page.pde_ips) { ++ spt->guest_page.pde_ips = ips; ++ ++ gvt_dbg_mm("reshadow PDE since ips changed\n"); ++ clear_page(spt->shadow_page.vaddr); ++ ret = ppgtt_populate_spt(spt); ++ if (ret) { ++ ppgtt_put_spt(spt); ++ goto err; ++ } ++ } ++ } else { ++ int type = get_next_pt_type(we->type); ++ ++ if (!gtt_type_is_pt(type)) { ++ ret = -EINVAL; ++ goto err; ++ } ++ ++ spt = ppgtt_alloc_spt_gfn(vgpu, type, ops->get_pfn(we), ips); ++ if (IS_ERR(spt)) { ++ ret = PTR_ERR(spt); ++ goto err; ++ } ++ ++ ret = intel_vgpu_enable_page_track(vgpu, spt->guest_page.gfn); ++ if (ret) ++ goto err_free_spt; ++ ++ ret = ppgtt_populate_spt(spt); ++ if (ret) ++ goto err_free_spt; ++ ++ trace_spt_change(vgpu->id, "new", spt, spt->guest_page.gfn, ++ spt->shadow_page.type); ++ } ++ return spt; ++ ++err_free_spt: ++ ppgtt_free_spt(spt); ++ spt = NULL; ++err: ++ gvt_vgpu_err("fail: shadow page %p guest entry 0x%llx type %d\n", ++ spt, we->val64, we->type); ++ return ERR_PTR(ret); ++} ++ ++static inline void ppgtt_generate_shadow_entry(struct intel_gvt_gtt_entry *se, ++ struct intel_vgpu_ppgtt_spt *s, struct intel_gvt_gtt_entry *ge) ++{ ++ struct intel_gvt_gtt_pte_ops *ops = s->vgpu->gvt->gtt.pte_ops; ++ ++ se->type = ge->type; ++ se->val64 = ge->val64; ++ ++ /* Because we always split 64KB pages, so clear IPS in shadow PDE. */ ++ if (se->type == GTT_TYPE_PPGTT_PDE_ENTRY) ++ ops->clear_ips(se); ++ ++ ops->set_pfn(se, s->shadow_page.mfn); ++} ++ ++/** ++ * Check if can do 2M page ++ * @vgpu: target vgpu ++ * @entry: target pfn's gtt entry ++ * ++ * Return 1 if 2MB huge gtt shadowing is possilbe, 0 if miscondition, ++ * negtive if found err. ++ */ ++static int is_2MB_gtt_possible(struct intel_vgpu *vgpu, ++ struct intel_gvt_gtt_entry *entry) ++{ ++ struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; ++ unsigned long pfn; ++ ++ if (!HAS_PAGE_SIZES(vgpu->gvt->dev_priv, I915_GTT_PAGE_SIZE_2M)) ++ return 0; ++ ++ pfn = intel_gvt_hypervisor_gfn_to_mfn(vgpu, ops->get_pfn(entry)); ++ if (pfn == INTEL_GVT_INVALID_ADDR) ++ return -EINVAL; ++ ++ return PageTransHuge(pfn_to_page(pfn)); ++} ++ ++static int split_2MB_gtt_entry(struct intel_vgpu *vgpu, ++ struct intel_vgpu_ppgtt_spt *spt, unsigned long index, ++ struct intel_gvt_gtt_entry *se) ++{ ++ struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; ++ struct intel_vgpu_ppgtt_spt *sub_spt; ++ struct intel_gvt_gtt_entry sub_se; ++ unsigned long start_gfn; ++ dma_addr_t dma_addr; ++ unsigned long sub_index; ++ int ret; ++ ++ gvt_dbg_mm("Split 2M gtt entry, index %lu\n", index); ++ ++ start_gfn = ops->get_pfn(se); ++ ++ sub_spt = ppgtt_alloc_spt(vgpu, GTT_TYPE_PPGTT_PTE_PT); ++ if (IS_ERR(sub_spt)) ++ return PTR_ERR(sub_spt); ++ ++ for_each_shadow_entry(sub_spt, &sub_se, sub_index) { ++ ret = intel_gvt_hypervisor_dma_map_guest_page(vgpu, ++ start_gfn + sub_index, PAGE_SIZE, &dma_addr); ++ if (ret) { ++ ppgtt_invalidate_spt(spt); ++ return ret; ++ } ++ sub_se.val64 = se->val64; ++ ++ /* Copy the PAT field from PDE. */ ++ sub_se.val64 &= ~_PAGE_PAT; ++ sub_se.val64 |= (se->val64 & _PAGE_PAT_LARGE) >> 5; ++ ++ ops->set_pfn(&sub_se, dma_addr >> PAGE_SHIFT); ++ ppgtt_set_shadow_entry(sub_spt, &sub_se, sub_index); ++ } ++ ++ /* Clear dirty field. */ ++ se->val64 &= ~_PAGE_DIRTY; ++ ++ ops->clear_pse(se); ++ ops->clear_ips(se); ++ ops->set_pfn(se, sub_spt->shadow_page.mfn); ++ ppgtt_set_shadow_entry(spt, se, index); ++ return 0; ++} ++ ++static int split_64KB_gtt_entry(struct intel_vgpu *vgpu, ++ struct intel_vgpu_ppgtt_spt *spt, unsigned long index, ++ struct intel_gvt_gtt_entry *se) ++{ ++ struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; ++ struct intel_gvt_gtt_entry entry = *se; ++ unsigned long start_gfn; ++ dma_addr_t dma_addr; ++ int i, ret; ++ ++ gvt_vdbg_mm("Split 64K gtt entry, index %lu\n", index); ++ ++ GEM_BUG_ON(index % GTT_64K_PTE_STRIDE); ++ ++ start_gfn = ops->get_pfn(se); ++ ++ entry.type = GTT_TYPE_PPGTT_PTE_4K_ENTRY; ++ ops->set_64k_splited(&entry); ++ ++ for (i = 0; i < GTT_64K_PTE_STRIDE; i++) { ++ ret = intel_gvt_hypervisor_dma_map_guest_page(vgpu, ++ start_gfn + i, PAGE_SIZE, &dma_addr); ++ if (ret) ++ return ret; ++ ++ ops->set_pfn(&entry, dma_addr >> PAGE_SHIFT); ++ ppgtt_set_shadow_entry(spt, &entry, index + i); ++ } ++ return 0; ++} ++ ++static int ppgtt_populate_shadow_entry(struct intel_vgpu *vgpu, ++ struct intel_vgpu_ppgtt_spt *spt, unsigned long index, ++ struct intel_gvt_gtt_entry *ge) ++{ ++ struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops; ++ struct intel_gvt_gtt_entry se = *ge; ++ unsigned long gfn, page_size = PAGE_SIZE; ++ dma_addr_t dma_addr; ++ int ret; ++ ++ if (!pte_ops->test_present(ge)) ++ return 0; ++ ++ gfn = pte_ops->get_pfn(ge); ++ ++ switch (ge->type) { ++ case GTT_TYPE_PPGTT_PTE_4K_ENTRY: ++ gvt_vdbg_mm("shadow 4K gtt entry\n"); ++ break; ++ case GTT_TYPE_PPGTT_PTE_64K_ENTRY: ++ gvt_vdbg_mm("shadow 64K gtt entry\n"); ++ /* ++ * The layout of 64K page is special, the page size is ++ * controlled by uper PDE. To be simple, we always split ++ * 64K page to smaller 4K pages in shadow PT. ++ */ ++ return split_64KB_gtt_entry(vgpu, spt, index, &se); ++ case GTT_TYPE_PPGTT_PTE_2M_ENTRY: ++ gvt_vdbg_mm("shadow 2M gtt entry\n"); ++ ret = is_2MB_gtt_possible(vgpu, ge); ++ if (ret == 0) ++ return split_2MB_gtt_entry(vgpu, spt, index, &se); ++ else if (ret < 0) ++ return ret; ++ page_size = I915_GTT_PAGE_SIZE_2M; ++ break; ++ case GTT_TYPE_PPGTT_PTE_1G_ENTRY: ++ gvt_vgpu_err("GVT doesn't support 1GB entry\n"); ++ return -EINVAL; ++ default: ++ GEM_BUG_ON(1); ++ }; ++ ++ /* direct shadow */ ++ ret = intel_gvt_hypervisor_dma_map_guest_page(vgpu, gfn, page_size, ++ &dma_addr); ++ if (ret) ++ return -ENXIO; ++ ++ pte_ops->set_pfn(&se, dma_addr >> PAGE_SHIFT); ++ ppgtt_set_shadow_entry(spt, &se, index); ++ return 0; ++} ++ ++static int ppgtt_populate_spt(struct intel_vgpu_ppgtt_spt *spt) ++{ ++ struct intel_vgpu *vgpu = spt->vgpu; ++ struct intel_gvt *gvt = vgpu->gvt; ++ struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops; ++ struct intel_vgpu_ppgtt_spt *s; ++ struct intel_gvt_gtt_entry se, ge; ++ unsigned long gfn, i; ++ int ret; ++ ++ trace_spt_change(spt->vgpu->id, "born", spt, ++ spt->guest_page.gfn, spt->shadow_page.type); ++ ++ for_each_present_guest_entry(spt, &ge, i) { ++ if (gtt_type_is_pt(get_next_pt_type(ge.type))) { ++ s = ppgtt_populate_spt_by_guest_entry(vgpu, &ge); ++ if (IS_ERR(s)) { ++ ret = PTR_ERR(s); ++ goto fail; ++ } ++ ppgtt_get_shadow_entry(spt, &se, i); ++ ppgtt_generate_shadow_entry(&se, s, &ge); ++ ppgtt_set_shadow_entry(spt, &se, i); ++ } else { ++ gfn = ops->get_pfn(&ge); ++ if (!intel_gvt_hypervisor_is_valid_gfn(vgpu, gfn)) { ++ ops->set_pfn(&se, gvt->gtt.scratch_mfn); ++ ppgtt_set_shadow_entry(spt, &se, i); ++ continue; ++ } ++ ++ ret = ppgtt_populate_shadow_entry(vgpu, spt, i, &ge); ++ if (ret) ++ goto fail; ++ } ++ } ++ return 0; ++fail: ++ gvt_vgpu_err("fail: shadow page %p guest entry 0x%llx type %d\n", ++ spt, ge.val64, ge.type); ++ return ret; ++} ++ ++static int ppgtt_handle_guest_entry_removal(struct intel_vgpu_ppgtt_spt *spt, ++ struct intel_gvt_gtt_entry *se, unsigned long index) ++{ ++ struct intel_vgpu *vgpu = spt->vgpu; ++ struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; ++ int ret; ++ ++ trace_spt_guest_change(spt->vgpu->id, "remove", spt, ++ spt->shadow_page.type, se->val64, index); ++ ++ gvt_vdbg_mm("destroy old shadow entry, type %d, index %lu, value %llx\n", ++ se->type, index, se->val64); ++ ++ if (!ops->test_present(se)) ++ return 0; ++ ++ if (ops->get_pfn(se) == ++ vgpu->gtt.scratch_pt[spt->shadow_page.type].page_mfn) ++ return 0; ++ ++ if (gtt_type_is_pt(get_next_pt_type(se->type))) { ++ struct intel_vgpu_ppgtt_spt *s = ++ intel_vgpu_find_spt_by_mfn(vgpu, ops->get_pfn(se)); ++ if (!s) { ++ gvt_vgpu_err("fail to find guest page\n"); ++ ret = -ENXIO; ++ goto fail; ++ } ++ ret = ppgtt_invalidate_spt(s); ++ if (ret) ++ goto fail; ++ } else { ++ /* We don't setup 64K shadow entry so far. */ ++ WARN(se->type == GTT_TYPE_PPGTT_PTE_64K_ENTRY, ++ "suspicious 64K entry\n"); ++ ppgtt_invalidate_pte(spt, se); ++ } ++ ++ return 0; ++fail: ++ gvt_vgpu_err("fail: shadow page %p guest entry 0x%llx type %d\n", ++ spt, se->val64, se->type); ++ return ret; ++} ++ ++static int ppgtt_handle_guest_entry_add(struct intel_vgpu_ppgtt_spt *spt, ++ struct intel_gvt_gtt_entry *we, unsigned long index) ++{ ++ struct intel_vgpu *vgpu = spt->vgpu; ++ struct intel_gvt_gtt_entry m; ++ struct intel_vgpu_ppgtt_spt *s; ++ int ret; ++ ++ trace_spt_guest_change(spt->vgpu->id, "add", spt, spt->shadow_page.type, ++ we->val64, index); ++ ++ gvt_vdbg_mm("add shadow entry: type %d, index %lu, value %llx\n", ++ we->type, index, we->val64); ++ ++ if (gtt_type_is_pt(get_next_pt_type(we->type))) { ++ s = ppgtt_populate_spt_by_guest_entry(vgpu, we); ++ if (IS_ERR(s)) { ++ ret = PTR_ERR(s); ++ goto fail; ++ } ++ ppgtt_get_shadow_entry(spt, &m, index); ++ ppgtt_generate_shadow_entry(&m, s, we); ++ ppgtt_set_shadow_entry(spt, &m, index); ++ } else { ++ ret = ppgtt_populate_shadow_entry(vgpu, spt, index, we); ++ if (ret) ++ goto fail; ++ } ++ return 0; ++fail: ++ gvt_vgpu_err("fail: spt %p guest entry 0x%llx type %d\n", ++ spt, we->val64, we->type); ++ return ret; ++} ++ ++static int sync_oos_page(struct intel_vgpu *vgpu, ++ struct intel_vgpu_oos_page *oos_page) ++{ ++ const struct intel_gvt_device_info *info = &vgpu->gvt->device_info; ++ struct intel_gvt *gvt = vgpu->gvt; ++ struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops; ++ struct intel_vgpu_ppgtt_spt *spt = oos_page->spt; ++ struct intel_gvt_gtt_entry old, new; ++ int index; ++ int ret; ++ ++ trace_oos_change(vgpu->id, "sync", oos_page->id, ++ spt, spt->guest_page.type); ++ ++ old.type = new.type = get_entry_type(spt->guest_page.type); ++ old.val64 = new.val64 = 0; ++ ++ for (index = 0; index < (I915_GTT_PAGE_SIZE >> ++ info->gtt_entry_size_shift); index++) { ++ ops->get_entry(oos_page->mem, &old, index, false, 0, vgpu); ++ ops->get_entry(NULL, &new, index, true, ++ spt->guest_page.gfn << PAGE_SHIFT, vgpu); ++ ++ if (old.val64 == new.val64 ++ && !test_and_clear_bit(index, spt->post_shadow_bitmap)) ++ continue; ++ ++ trace_oos_sync(vgpu->id, oos_page->id, ++ spt, spt->guest_page.type, ++ new.val64, index); ++ ++ ret = ppgtt_populate_shadow_entry(vgpu, spt, index, &new); ++ if (ret) ++ return ret; ++ ++ ops->set_entry(oos_page->mem, &new, index, false, 0, vgpu); ++ } ++ ++ spt->guest_page.write_cnt = 0; ++ list_del_init(&spt->post_shadow_list); ++ return 0; ++} ++ ++static int detach_oos_page(struct intel_vgpu *vgpu, ++ struct intel_vgpu_oos_page *oos_page) ++{ ++ struct intel_gvt *gvt = vgpu->gvt; ++ struct intel_vgpu_ppgtt_spt *spt = oos_page->spt; ++ ++ trace_oos_change(vgpu->id, "detach", oos_page->id, ++ spt, spt->guest_page.type); ++ ++ spt->guest_page.write_cnt = 0; ++ spt->guest_page.oos_page = NULL; ++ oos_page->spt = NULL; ++ ++ list_del_init(&oos_page->vm_list); ++ list_move_tail(&oos_page->list, &gvt->gtt.oos_page_free_list_head); ++ ++ return 0; ++} ++ ++static int attach_oos_page(struct intel_vgpu_oos_page *oos_page, ++ struct intel_vgpu_ppgtt_spt *spt) ++{ ++ struct intel_gvt *gvt = spt->vgpu->gvt; ++ int ret; ++ ++ ret = intel_gvt_hypervisor_read_gpa(spt->vgpu, ++ spt->guest_page.gfn << I915_GTT_PAGE_SHIFT, ++ oos_page->mem, I915_GTT_PAGE_SIZE); ++ if (ret) ++ return ret; ++ ++ oos_page->spt = spt; ++ spt->guest_page.oos_page = oos_page; ++ ++ list_move_tail(&oos_page->list, &gvt->gtt.oos_page_use_list_head); ++ ++ trace_oos_change(spt->vgpu->id, "attach", oos_page->id, ++ spt, spt->guest_page.type); ++ return 0; ++} ++ ++static int ppgtt_set_guest_page_sync(struct intel_vgpu_ppgtt_spt *spt) ++{ ++ struct intel_vgpu_oos_page *oos_page = spt->guest_page.oos_page; ++ int ret; ++ ++ ret = intel_vgpu_enable_page_track(spt->vgpu, spt->guest_page.gfn); ++ if (ret) ++ return ret; ++ ++ trace_oos_change(spt->vgpu->id, "set page sync", oos_page->id, ++ spt, spt->guest_page.type); ++ ++ list_del_init(&oos_page->vm_list); ++ return sync_oos_page(spt->vgpu, oos_page); ++} ++ ++static int ppgtt_allocate_oos_page(struct intel_vgpu_ppgtt_spt *spt) ++{ ++ struct intel_gvt *gvt = spt->vgpu->gvt; ++ struct intel_gvt_gtt *gtt = &gvt->gtt; ++ struct intel_vgpu_oos_page *oos_page = spt->guest_page.oos_page; ++ int ret; ++ ++ WARN(oos_page, "shadow PPGTT page has already has a oos page\n"); ++ ++ if (list_empty(>t->oos_page_free_list_head)) { ++ oos_page = container_of(gtt->oos_page_use_list_head.next, ++ struct intel_vgpu_oos_page, list); ++ ret = ppgtt_set_guest_page_sync(oos_page->spt); ++ if (ret) ++ return ret; ++ ret = detach_oos_page(spt->vgpu, oos_page); ++ if (ret) ++ return ret; ++ } else ++ oos_page = container_of(gtt->oos_page_free_list_head.next, ++ struct intel_vgpu_oos_page, list); ++ return attach_oos_page(oos_page, spt); ++} ++ ++static int ppgtt_set_guest_page_oos(struct intel_vgpu_ppgtt_spt *spt) ++{ ++ struct intel_vgpu_oos_page *oos_page = spt->guest_page.oos_page; ++ ++ if (WARN(!oos_page, "shadow PPGTT page should have a oos page\n")) ++ return -EINVAL; ++ ++ trace_oos_change(spt->vgpu->id, "set page out of sync", oos_page->id, ++ spt, spt->guest_page.type); ++ ++ list_add_tail(&oos_page->vm_list, &spt->vgpu->gtt.oos_page_list_head); ++ return intel_vgpu_disable_page_track(spt->vgpu, spt->guest_page.gfn); ++} ++ ++/** ++ * intel_vgpu_sync_oos_pages - sync all the out-of-synced shadow for vGPU ++ * @vgpu: a vGPU ++ * ++ * This function is called before submitting a guest workload to host, ++ * to sync all the out-of-synced shadow for vGPU ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++int intel_vgpu_sync_oos_pages(struct intel_vgpu *vgpu) ++{ ++ struct list_head *pos, *n; ++ struct intel_vgpu_oos_page *oos_page; ++ int ret; ++ ++ if (!enable_out_of_sync) ++ return 0; ++ ++ list_for_each_safe(pos, n, &vgpu->gtt.oos_page_list_head) { ++ oos_page = container_of(pos, ++ struct intel_vgpu_oos_page, vm_list); ++ ret = ppgtt_set_guest_page_sync(oos_page->spt); ++ if (ret) ++ return ret; ++ } ++ return 0; ++} ++ ++/* ++ * The heart of PPGTT shadow page table. ++ */ ++static int ppgtt_handle_guest_write_page_table( ++ struct intel_vgpu_ppgtt_spt *spt, ++ struct intel_gvt_gtt_entry *we, unsigned long index) ++{ ++ struct intel_vgpu *vgpu = spt->vgpu; ++ int type = spt->shadow_page.type; ++ struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; ++ struct intel_gvt_gtt_entry old_se; ++ int new_present; ++ int i, ret; ++ ++ new_present = ops->test_present(we); ++ ++ /* ++ * Adding the new entry first and then removing the old one, that can ++ * guarantee the ppgtt table is validated during the window between ++ * adding and removal. ++ */ ++ ppgtt_get_shadow_entry(spt, &old_se, index); ++ ++ if (new_present) { ++ ret = ppgtt_handle_guest_entry_add(spt, we, index); ++ if (ret) ++ goto fail; ++ } ++ ++ ret = ppgtt_handle_guest_entry_removal(spt, &old_se, index); ++ if (ret) ++ goto fail; ++ ++ if (!new_present) { ++ /* For 64KB splited entries, we need clear them all. */ ++ if (ops->test_64k_splited(&old_se) && ++ !(index % GTT_64K_PTE_STRIDE)) { ++ gvt_vdbg_mm("remove splited 64K shadow entries\n"); ++ for (i = 0; i < GTT_64K_PTE_STRIDE; i++) { ++ ops->clear_64k_splited(&old_se); ++ ops->set_pfn(&old_se, ++ vgpu->gtt.scratch_pt[type].page_mfn); ++ ppgtt_set_shadow_entry(spt, &old_se, index + i); ++ } ++ } else if (old_se.type == GTT_TYPE_PPGTT_PTE_2M_ENTRY || ++ old_se.type == GTT_TYPE_PPGTT_PTE_1G_ENTRY) { ++ ops->clear_pse(&old_se); ++ ops->set_pfn(&old_se, ++ vgpu->gtt.scratch_pt[type].page_mfn); ++ ppgtt_set_shadow_entry(spt, &old_se, index); ++ } else { ++ ops->set_pfn(&old_se, ++ vgpu->gtt.scratch_pt[type].page_mfn); ++ ppgtt_set_shadow_entry(spt, &old_se, index); ++ } ++ } ++ ++ return 0; ++fail: ++ gvt_vgpu_err("fail: shadow page %p guest entry 0x%llx type %d.\n", ++ spt, we->val64, we->type); ++ return ret; ++} ++ ++ ++ ++static inline bool can_do_out_of_sync(struct intel_vgpu_ppgtt_spt *spt) ++{ ++ return enable_out_of_sync ++ && gtt_type_is_pte_pt(spt->guest_page.type) ++ && spt->guest_page.write_cnt >= 2; ++} ++ ++static void ppgtt_set_post_shadow(struct intel_vgpu_ppgtt_spt *spt, ++ unsigned long index) ++{ ++ set_bit(index, spt->post_shadow_bitmap); ++ if (!list_empty(&spt->post_shadow_list)) ++ return; ++ ++ list_add_tail(&spt->post_shadow_list, ++ &spt->vgpu->gtt.post_shadow_list_head); ++} ++ ++/** ++ * intel_vgpu_flush_post_shadow - flush the post shadow transactions ++ * @vgpu: a vGPU ++ * ++ * This function is called before submitting a guest workload to host, ++ * to flush all the post shadows for a vGPU. ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++int intel_vgpu_flush_post_shadow(struct intel_vgpu *vgpu) ++{ ++ struct list_head *pos, *n; ++ struct intel_vgpu_ppgtt_spt *spt; ++ struct intel_gvt_gtt_entry ge; ++ unsigned long index; ++ int ret; ++ ++ list_for_each_safe(pos, n, &vgpu->gtt.post_shadow_list_head) { ++ spt = container_of(pos, struct intel_vgpu_ppgtt_spt, ++ post_shadow_list); ++ ++ for_each_set_bit(index, spt->post_shadow_bitmap, ++ GTT_ENTRY_NUM_IN_ONE_PAGE) { ++ ppgtt_get_guest_entry(spt, &ge, index); ++ ++ ret = ppgtt_handle_guest_write_page_table(spt, ++ &ge, index); ++ if (ret) ++ return ret; ++ clear_bit(index, spt->post_shadow_bitmap); ++ } ++ list_del_init(&spt->post_shadow_list); ++ } ++ return 0; ++} ++ ++static int ppgtt_handle_guest_write_page_table_bytes( ++ struct intel_vgpu_ppgtt_spt *spt, ++ u64 pa, void *p_data, int bytes) ++{ ++ struct intel_vgpu *vgpu = spt->vgpu; ++ struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; ++ const struct intel_gvt_device_info *info = &vgpu->gvt->device_info; ++ struct intel_gvt_gtt_entry we, se; ++ unsigned long index; ++ int ret; ++ ++ index = (pa & (PAGE_SIZE - 1)) >> info->gtt_entry_size_shift; ++ ++ ppgtt_get_guest_entry(spt, &we, index); ++ ++ /* ++ * For page table which has 64K gtt entry, only PTE#0, PTE#16, ++ * PTE#32, ... PTE#496 are used. Unused PTEs update should be ++ * ignored. ++ */ ++ if (we.type == GTT_TYPE_PPGTT_PTE_64K_ENTRY && ++ (index % GTT_64K_PTE_STRIDE)) { ++ gvt_vdbg_mm("Ignore write to unused PTE entry, index %lu\n", ++ index); ++ return 0; ++ } ++ ++ if (bytes == info->gtt_entry_size) { ++ ret = ppgtt_handle_guest_write_page_table(spt, &we, index); ++ if (ret) ++ return ret; ++ } else { ++ if (!test_bit(index, spt->post_shadow_bitmap)) { ++ int type = spt->shadow_page.type; ++ ++ ppgtt_get_shadow_entry(spt, &se, index); ++ ret = ppgtt_handle_guest_entry_removal(spt, &se, index); ++ if (ret) ++ return ret; ++ ops->set_pfn(&se, vgpu->gtt.scratch_pt[type].page_mfn); ++ ppgtt_set_shadow_entry(spt, &se, index); ++ } ++ ppgtt_set_post_shadow(spt, index); ++ } ++ ++ if (!enable_out_of_sync) ++ return 0; ++ ++ spt->guest_page.write_cnt++; ++ ++ if (spt->guest_page.oos_page) ++ ops->set_entry(spt->guest_page.oos_page->mem, &we, index, ++ false, 0, vgpu); ++ ++ if (can_do_out_of_sync(spt)) { ++ if (!spt->guest_page.oos_page) ++ ppgtt_allocate_oos_page(spt); ++ ++ ret = ppgtt_set_guest_page_oos(spt); ++ if (ret < 0) ++ return ret; ++ } ++ return 0; ++} ++ ++static void invalidate_ppgtt_mm(struct intel_vgpu_mm *mm) ++{ ++ struct intel_vgpu *vgpu = mm->vgpu; ++ struct intel_gvt *gvt = vgpu->gvt; ++ struct intel_gvt_gtt *gtt = &gvt->gtt; ++ struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops; ++ struct intel_gvt_gtt_entry se; ++ int index; ++ ++ if (!mm->ppgtt_mm.shadowed) ++ return; ++ ++ for (index = 0; index < ARRAY_SIZE(mm->ppgtt_mm.shadow_pdps); index++) { ++ ppgtt_get_shadow_root_entry(mm, &se, index); ++ ++ if (!ops->test_present(&se)) ++ continue; ++ ++ ppgtt_invalidate_spt_by_shadow_entry(vgpu, &se); ++ se.val64 = 0; ++ ppgtt_set_shadow_root_entry(mm, &se, index); ++ ++ trace_spt_guest_change(vgpu->id, "destroy root pointer", ++ NULL, se.type, se.val64, index); ++ } ++ ++ mm->ppgtt_mm.shadowed = false; ++} ++ ++ ++static int shadow_ppgtt_mm(struct intel_vgpu_mm *mm) ++{ ++ struct intel_vgpu *vgpu = mm->vgpu; ++ struct intel_gvt *gvt = vgpu->gvt; ++ struct intel_gvt_gtt *gtt = &gvt->gtt; ++ struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops; ++ struct intel_vgpu_ppgtt_spt *spt; ++ struct intel_gvt_gtt_entry ge, se; ++ int index, ret; ++ ++ if (mm->ppgtt_mm.shadowed) ++ return 0; ++ ++ mm->ppgtt_mm.shadowed = true; ++ ++ for (index = 0; index < ARRAY_SIZE(mm->ppgtt_mm.guest_pdps); index++) { ++ ppgtt_get_guest_root_entry(mm, &ge, index); ++ ++ if (!ops->test_present(&ge)) ++ continue; ++ ++ trace_spt_guest_change(vgpu->id, __func__, NULL, ++ ge.type, ge.val64, index); ++ ++ spt = ppgtt_populate_spt_by_guest_entry(vgpu, &ge); ++ if (IS_ERR(spt)) { ++ gvt_vgpu_err("fail to populate guest root pointer\n"); ++ ret = PTR_ERR(spt); ++ goto fail; ++ } ++ ppgtt_generate_shadow_entry(&se, spt, &ge); ++ ppgtt_set_shadow_root_entry(mm, &se, index); ++ ++ trace_spt_guest_change(vgpu->id, "populate root pointer", ++ NULL, se.type, se.val64, index); ++ } ++ ++ return 0; ++fail: ++ invalidate_ppgtt_mm(mm); ++ return ret; ++} ++ ++static struct intel_vgpu_mm *vgpu_alloc_mm(struct intel_vgpu *vgpu) ++{ ++ struct intel_vgpu_mm *mm; ++ ++ mm = kzalloc(sizeof(*mm), GFP_KERNEL); ++ if (!mm) ++ return NULL; ++ ++ mm->vgpu = vgpu; ++ kref_init(&mm->ref); ++ atomic_set(&mm->pincount, 0); ++ ++ return mm; ++} ++ ++static void vgpu_free_mm(struct intel_vgpu_mm *mm) ++{ ++ kfree(mm); ++} ++ ++/** ++ * intel_vgpu_create_ppgtt_mm - create a ppgtt mm object for a vGPU ++ * @vgpu: a vGPU ++ * @root_entry_type: ppgtt root entry type ++ * @pdps: guest pdps. ++ * ++ * This function is used to create a ppgtt mm object for a vGPU. ++ * ++ * Returns: ++ * Zero on success, negative error code in pointer if failed. ++ */ ++struct intel_vgpu_mm *intel_vgpu_create_ppgtt_mm(struct intel_vgpu *vgpu, ++ enum intel_gvt_gtt_type root_entry_type, u64 pdps[]) ++{ ++ struct intel_gvt *gvt = vgpu->gvt; ++ struct intel_vgpu_mm *mm; ++ int ret; ++ ++ mm = vgpu_alloc_mm(vgpu); ++ if (!mm) ++ return ERR_PTR(-ENOMEM); ++ ++ mm->type = INTEL_GVT_MM_PPGTT; ++ ++ GEM_BUG_ON(root_entry_type != GTT_TYPE_PPGTT_ROOT_L3_ENTRY && ++ root_entry_type != GTT_TYPE_PPGTT_ROOT_L4_ENTRY); ++ mm->ppgtt_mm.root_entry_type = root_entry_type; ++ ++ INIT_LIST_HEAD(&mm->ppgtt_mm.list); ++ INIT_LIST_HEAD(&mm->ppgtt_mm.lru_list); ++ ++ if (root_entry_type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY) ++ mm->ppgtt_mm.guest_pdps[0] = pdps[0]; ++ else ++ memcpy(mm->ppgtt_mm.guest_pdps, pdps, ++ sizeof(mm->ppgtt_mm.guest_pdps)); ++ ++ ret = shadow_ppgtt_mm(mm); ++ if (ret) { ++ gvt_vgpu_err("failed to shadow ppgtt mm\n"); ++ vgpu_free_mm(mm); ++ return ERR_PTR(ret); ++ } ++ ++ list_add_tail(&mm->ppgtt_mm.list, &vgpu->gtt.ppgtt_mm_list_head); ++ ++ mutex_lock(&gvt->gtt.ppgtt_mm_lock); ++ list_add_tail(&mm->ppgtt_mm.lru_list, &gvt->gtt.ppgtt_mm_lru_list_head); ++ mutex_unlock(&gvt->gtt.ppgtt_mm_lock); ++ ++ return mm; ++} ++ ++static struct intel_vgpu_mm *intel_vgpu_create_ggtt_mm(struct intel_vgpu *vgpu) ++{ ++ struct intel_vgpu_mm *mm; ++ unsigned long nr_entries; ++ ++ mm = vgpu_alloc_mm(vgpu); ++ if (!mm) ++ return ERR_PTR(-ENOMEM); ++ ++ mm->type = INTEL_GVT_MM_GGTT; ++ ++ nr_entries = gvt_ggtt_gm_sz(vgpu->gvt) >> I915_GTT_PAGE_SHIFT; ++ mm->ggtt_mm.virtual_ggtt = ++ vzalloc(array_size(nr_entries, ++ vgpu->gvt->device_info.gtt_entry_size)); ++ if (!mm->ggtt_mm.virtual_ggtt) { ++ vgpu_free_mm(mm); ++ return ERR_PTR(-ENOMEM); ++ } ++ ++ return mm; ++} ++ ++/** ++ * _intel_vgpu_mm_release - destroy a mm object ++ * @mm_ref: a kref object ++ * ++ * This function is used to destroy a mm object for vGPU ++ * ++ */ ++void _intel_vgpu_mm_release(struct kref *mm_ref) ++{ ++ struct intel_vgpu_mm *mm = container_of(mm_ref, typeof(*mm), ref); ++ ++ if (GEM_WARN_ON(atomic_read(&mm->pincount))) ++ gvt_err("vgpu mm pin count bug detected\n"); ++ ++ if (mm->type == INTEL_GVT_MM_PPGTT) { ++ list_del(&mm->ppgtt_mm.list); ++ list_del(&mm->ppgtt_mm.lru_list); ++ invalidate_ppgtt_mm(mm); ++ } else { ++ vfree(mm->ggtt_mm.virtual_ggtt); ++ } ++ ++ vgpu_free_mm(mm); ++} ++ ++/** ++ * intel_vgpu_unpin_mm - decrease the pin count of a vGPU mm object ++ * @mm: a vGPU mm object ++ * ++ * This function is called when user doesn't want to use a vGPU mm object ++ */ ++void intel_vgpu_unpin_mm(struct intel_vgpu_mm *mm) ++{ ++ atomic_dec_if_positive(&mm->pincount); ++} ++ ++/** ++ * intel_vgpu_pin_mm - increase the pin count of a vGPU mm object ++ * @mm: target vgpu mm ++ * ++ * This function is called when user wants to use a vGPU mm object. If this ++ * mm object hasn't been shadowed yet, the shadow will be populated at this ++ * time. ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++int intel_vgpu_pin_mm(struct intel_vgpu_mm *mm) ++{ ++ int ret; ++ ++ atomic_inc(&mm->pincount); ++ ++ if (mm->type == INTEL_GVT_MM_PPGTT) { ++ ret = shadow_ppgtt_mm(mm); ++ if (ret) ++ return ret; ++ ++ mutex_lock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock); ++ list_move_tail(&mm->ppgtt_mm.lru_list, ++ &mm->vgpu->gvt->gtt.ppgtt_mm_lru_list_head); ++ mutex_unlock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock); ++ } ++ ++ return 0; ++} ++ ++static int reclaim_one_ppgtt_mm(struct intel_gvt *gvt) ++{ ++ struct intel_vgpu_mm *mm; ++ struct list_head *pos, *n; ++ ++ mutex_lock(&gvt->gtt.ppgtt_mm_lock); ++ ++ list_for_each_safe(pos, n, &gvt->gtt.ppgtt_mm_lru_list_head) { ++ mm = container_of(pos, struct intel_vgpu_mm, ppgtt_mm.lru_list); ++ ++ if (atomic_read(&mm->pincount)) ++ continue; ++ ++ list_del_init(&mm->ppgtt_mm.lru_list); ++ mutex_unlock(&gvt->gtt.ppgtt_mm_lock); ++ invalidate_ppgtt_mm(mm); ++ return 1; ++ } ++ mutex_unlock(&gvt->gtt.ppgtt_mm_lock); ++ return 0; ++} ++ ++/* ++ * GMA translation APIs. ++ */ ++static inline int ppgtt_get_next_level_entry(struct intel_vgpu_mm *mm, ++ struct intel_gvt_gtt_entry *e, unsigned long index, bool guest) ++{ ++ struct intel_vgpu *vgpu = mm->vgpu; ++ struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; ++ struct intel_vgpu_ppgtt_spt *s; ++ ++ s = intel_vgpu_find_spt_by_mfn(vgpu, ops->get_pfn(e)); ++ if (!s) ++ return -ENXIO; ++ ++ if (!guest) ++ ppgtt_get_shadow_entry(s, e, index); ++ else ++ ppgtt_get_guest_entry(s, e, index); ++ return 0; ++} ++ ++/** ++ * intel_vgpu_gma_to_gpa - translate a gma to GPA ++ * @mm: mm object. could be a PPGTT or GGTT mm object ++ * @gma: graphics memory address in this mm object ++ * ++ * This function is used to translate a graphics memory address in specific ++ * graphics memory space to guest physical address. ++ * ++ * Returns: ++ * Guest physical address on success, INTEL_GVT_INVALID_ADDR if failed. ++ */ ++unsigned long intel_vgpu_gma_to_gpa(struct intel_vgpu_mm *mm, unsigned long gma) ++{ ++ struct intel_vgpu *vgpu = mm->vgpu; ++ struct intel_gvt *gvt = vgpu->gvt; ++ struct intel_gvt_gtt_pte_ops *pte_ops = gvt->gtt.pte_ops; ++ struct intel_gvt_gtt_gma_ops *gma_ops = gvt->gtt.gma_ops; ++ unsigned long gpa = INTEL_GVT_INVALID_ADDR; ++ unsigned long gma_index[4]; ++ struct intel_gvt_gtt_entry e; ++ int i, levels = 0; ++ int ret; ++ ++ GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT && ++ mm->type != INTEL_GVT_MM_PPGTT); ++ ++ if (mm->type == INTEL_GVT_MM_GGTT) { ++ if (!vgpu_gmadr_is_valid(vgpu, gma)) ++ goto err; ++ ++ ggtt_get_guest_entry(mm, &e, ++ gma_ops->gma_to_ggtt_pte_index(gma)); ++ ++ gpa = (pte_ops->get_pfn(&e) << I915_GTT_PAGE_SHIFT) ++ + (gma & ~I915_GTT_PAGE_MASK); ++ ++ trace_gma_translate(vgpu->id, "ggtt", 0, 0, gma, gpa); ++ } else { ++ switch (mm->ppgtt_mm.root_entry_type) { ++ case GTT_TYPE_PPGTT_ROOT_L4_ENTRY: ++ ppgtt_get_shadow_root_entry(mm, &e, 0); ++ ++ gma_index[0] = gma_ops->gma_to_pml4_index(gma); ++ gma_index[1] = gma_ops->gma_to_l4_pdp_index(gma); ++ gma_index[2] = gma_ops->gma_to_pde_index(gma); ++ gma_index[3] = gma_ops->gma_to_pte_index(gma); ++ levels = 4; ++ break; ++ case GTT_TYPE_PPGTT_ROOT_L3_ENTRY: ++ ppgtt_get_shadow_root_entry(mm, &e, ++ gma_ops->gma_to_l3_pdp_index(gma)); ++ ++ gma_index[0] = gma_ops->gma_to_pde_index(gma); ++ gma_index[1] = gma_ops->gma_to_pte_index(gma); ++ levels = 2; ++ break; ++ default: ++ GEM_BUG_ON(1); ++ } ++ ++ /* walk the shadow page table and get gpa from guest entry */ ++ for (i = 0; i < levels; i++) { ++ ret = ppgtt_get_next_level_entry(mm, &e, gma_index[i], ++ (i == levels - 1)); ++ if (ret) ++ goto err; ++ ++ if (!pte_ops->test_present(&e)) { ++ gvt_dbg_core("GMA 0x%lx is not present\n", gma); ++ goto err; ++ } ++ } ++ ++ gpa = (pte_ops->get_pfn(&e) << I915_GTT_PAGE_SHIFT) + ++ (gma & ~I915_GTT_PAGE_MASK); ++ trace_gma_translate(vgpu->id, "ppgtt", 0, ++ mm->ppgtt_mm.root_entry_type, gma, gpa); ++ } ++ ++ return gpa; ++err: ++ gvt_vgpu_err("invalid mm type: %d gma %lx\n", mm->type, gma); ++ return INTEL_GVT_INVALID_ADDR; ++} ++ ++static int emulate_ggtt_mmio_read(struct intel_vgpu *vgpu, ++ unsigned int off, void *p_data, unsigned int bytes) ++{ ++ struct intel_vgpu_mm *ggtt_mm = vgpu->gtt.ggtt_mm; ++ const struct intel_gvt_device_info *info = &vgpu->gvt->device_info; ++ unsigned long index = off >> info->gtt_entry_size_shift; ++ struct intel_gvt_gtt_entry e; ++ ++ if (bytes != 4 && bytes != 8) ++ return -EINVAL; ++ ++ ggtt_get_guest_entry(ggtt_mm, &e, index); ++ memcpy(p_data, (void *)&e.val64 + (off & (info->gtt_entry_size - 1)), ++ bytes); ++ return 0; ++} ++ ++/** ++ * intel_vgpu_emulate_gtt_mmio_read - emulate GTT MMIO register read ++ * @vgpu: a vGPU ++ * @off: register offset ++ * @p_data: data will be returned to guest ++ * @bytes: data length ++ * ++ * This function is used to emulate the GTT MMIO register read ++ * ++ * Returns: ++ * Zero on success, error code if failed. ++ */ ++int intel_vgpu_emulate_ggtt_mmio_read(struct intel_vgpu *vgpu, unsigned int off, ++ void *p_data, unsigned int bytes) ++{ ++ const struct intel_gvt_device_info *info = &vgpu->gvt->device_info; ++ int ret; ++ ++ if (bytes != 4 && bytes != 8) ++ return -EINVAL; ++ ++ off -= info->gtt_start_offset; ++ ret = emulate_ggtt_mmio_read(vgpu, off, p_data, bytes); ++ return ret; ++} ++ ++static void ggtt_invalidate_pte(struct intel_vgpu *vgpu, ++ struct intel_gvt_gtt_entry *entry) ++{ ++ struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops; ++ unsigned long pfn; ++ ++ pfn = pte_ops->get_pfn(entry); ++ if (pfn != vgpu->gvt->gtt.scratch_mfn) ++ intel_gvt_hypervisor_dma_unmap_guest_page(vgpu, ++ pfn << PAGE_SHIFT); ++} ++ ++static int emulate_ggtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off, ++ void *p_data, unsigned int bytes) ++{ ++ struct intel_gvt *gvt = vgpu->gvt; ++ const struct intel_gvt_device_info *info = &gvt->device_info; ++ struct intel_vgpu_mm *ggtt_mm = vgpu->gtt.ggtt_mm; ++ struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops; ++ unsigned long g_gtt_index = off >> info->gtt_entry_size_shift; ++ unsigned long gma, gfn; ++ struct intel_gvt_gtt_entry e = {.val64 = 0, .type = GTT_TYPE_GGTT_PTE}; ++ struct intel_gvt_gtt_entry m = {.val64 = 0, .type = GTT_TYPE_GGTT_PTE}; ++ dma_addr_t dma_addr; ++ int ret; ++ struct intel_gvt_partial_pte *partial_pte, *pos, *n; ++ bool partial_update = false; ++ ++ if (bytes != 4 && bytes != 8) ++ return -EINVAL; ++ ++ gma = g_gtt_index << I915_GTT_PAGE_SHIFT; ++ ++ /* the VM may configure the whole GM space when ballooning is used */ ++ if (!vgpu_gmadr_is_valid(vgpu, gma)) ++ return 0; ++ ++ e.type = GTT_TYPE_GGTT_PTE; ++ memcpy((void *)&e.val64 + (off & (info->gtt_entry_size - 1)), p_data, ++ bytes); ++ ++ /* If ggtt entry size is 8 bytes, and it's split into two 4 bytes ++ * write, save the first 4 bytes in a list and update virtual ++ * PTE. Only update shadow PTE when the second 4 bytes comes. ++ */ ++ if (bytes < info->gtt_entry_size) { ++ bool found = false; ++ ++ list_for_each_entry_safe(pos, n, ++ &ggtt_mm->ggtt_mm.partial_pte_list, list) { ++ if (g_gtt_index == pos->offset >> ++ info->gtt_entry_size_shift) { ++ if (off != pos->offset) { ++ /* the second partial part*/ ++ int last_off = pos->offset & ++ (info->gtt_entry_size - 1); ++ ++ memcpy((void *)&e.val64 + last_off, ++ (void *)&pos->data + last_off, ++ bytes); ++ ++ list_del(&pos->list); ++ kfree(pos); ++ found = true; ++ break; ++ } ++ ++ /* update of the first partial part */ ++ pos->data = e.val64; ++ ggtt_set_guest_entry(ggtt_mm, &e, g_gtt_index); ++ return 0; ++ } ++ } ++ ++ if (!found) { ++ /* the first partial part */ ++ partial_pte = kzalloc(sizeof(*partial_pte), GFP_KERNEL); ++ if (!partial_pte) ++ return -ENOMEM; ++ partial_pte->offset = off; ++ partial_pte->data = e.val64; ++ list_add_tail(&partial_pte->list, ++ &ggtt_mm->ggtt_mm.partial_pte_list); ++ partial_update = true; ++ } ++ } ++ ++ if (!partial_update && (ops->test_present(&e))) { ++ gfn = ops->get_pfn(&e); ++ m.val64 = e.val64; ++ m.type = e.type; ++ ++ /* one PTE update may be issued in multiple writes and the ++ * first write may not construct a valid gfn ++ */ ++ if (!intel_gvt_hypervisor_is_valid_gfn(vgpu, gfn)) { ++ ops->set_pfn(&m, gvt->gtt.scratch_mfn); ++ goto out; ++ } ++ ++ ret = intel_gvt_hypervisor_dma_map_guest_page(vgpu, gfn, ++ PAGE_SIZE, &dma_addr); ++ if (ret) { ++ gvt_vgpu_err("fail to populate guest ggtt entry\n"); ++ /* guest driver may read/write the entry when partial ++ * update the entry in this situation p2m will fail ++ * settting the shadow entry to point to a scratch page ++ */ ++ ops->set_pfn(&m, gvt->gtt.scratch_mfn); ++ } else ++ ops->set_pfn(&m, dma_addr >> PAGE_SHIFT); ++ } else { ++ ops->set_pfn(&m, gvt->gtt.scratch_mfn); ++ ops->clear_present(&m); ++ } ++ ++out: ++ ggtt_set_guest_entry(ggtt_mm, &e, g_gtt_index); ++ ++ ggtt_get_host_entry(ggtt_mm, &e, g_gtt_index); ++ ggtt_invalidate_pte(vgpu, &e); ++ ++ ggtt_set_host_entry(ggtt_mm, &m, g_gtt_index); ++ ggtt_invalidate(gvt->dev_priv); ++ return 0; ++} ++ ++/* ++ * intel_vgpu_emulate_ggtt_mmio_write - emulate GTT MMIO register write ++ * @vgpu: a vGPU ++ * @off: register offset ++ * @p_data: data from guest write ++ * @bytes: data length ++ * ++ * This function is used to emulate the GTT MMIO register write ++ * ++ * Returns: ++ * Zero on success, error code if failed. ++ */ ++int intel_vgpu_emulate_ggtt_mmio_write(struct intel_vgpu *vgpu, ++ unsigned int off, void *p_data, unsigned int bytes) ++{ ++ const struct intel_gvt_device_info *info = &vgpu->gvt->device_info; ++ int ret; ++ ++ if (bytes != 4 && bytes != 8) ++ return -EINVAL; ++ ++ off -= info->gtt_start_offset; ++ ret = emulate_ggtt_mmio_write(vgpu, off, p_data, bytes); ++ return ret; ++} ++ ++static int alloc_scratch_pages(struct intel_vgpu *vgpu, ++ enum intel_gvt_gtt_type type) ++{ ++ struct intel_vgpu_gtt *gtt = &vgpu->gtt; ++ struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; ++ int page_entry_num = I915_GTT_PAGE_SIZE >> ++ vgpu->gvt->device_info.gtt_entry_size_shift; ++ void *scratch_pt; ++ int i; ++ struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev; ++ dma_addr_t daddr; ++ ++ if (WARN_ON(type < GTT_TYPE_PPGTT_PTE_PT || type >= GTT_TYPE_MAX)) ++ return -EINVAL; ++ ++ scratch_pt = (void *)get_zeroed_page(GFP_KERNEL); ++ if (!scratch_pt) { ++ gvt_vgpu_err("fail to allocate scratch page\n"); ++ return -ENOMEM; ++ } ++ ++ daddr = dma_map_page(dev, virt_to_page(scratch_pt), 0, ++ 4096, PCI_DMA_BIDIRECTIONAL); ++ if (dma_mapping_error(dev, daddr)) { ++ gvt_vgpu_err("fail to dmamap scratch_pt\n"); ++ __free_page(virt_to_page(scratch_pt)); ++ return -ENOMEM; ++ } ++ gtt->scratch_pt[type].page_mfn = ++ (unsigned long)(daddr >> I915_GTT_PAGE_SHIFT); ++ gtt->scratch_pt[type].page = virt_to_page(scratch_pt); ++ gvt_dbg_mm("vgpu%d create scratch_pt: type %d mfn=0x%lx\n", ++ vgpu->id, type, gtt->scratch_pt[type].page_mfn); ++ ++ /* Build the tree by full filled the scratch pt with the entries which ++ * point to the next level scratch pt or scratch page. The ++ * scratch_pt[type] indicate the scratch pt/scratch page used by the ++ * 'type' pt. ++ * e.g. scratch_pt[GTT_TYPE_PPGTT_PDE_PT] is used by ++ * GTT_TYPE_PPGTT_PDE_PT level pt, that means this scratch_pt it self ++ * is GTT_TYPE_PPGTT_PTE_PT, and full filled by scratch page mfn. ++ */ ++ if (type > GTT_TYPE_PPGTT_PTE_PT) { ++ struct intel_gvt_gtt_entry se; ++ ++ memset(&se, 0, sizeof(struct intel_gvt_gtt_entry)); ++ se.type = get_entry_type(type - 1); ++ ops->set_pfn(&se, gtt->scratch_pt[type - 1].page_mfn); ++ ++ /* The entry parameters like present/writeable/cache type ++ * set to the same as i915's scratch page tree. ++ */ ++ se.val64 |= _PAGE_PRESENT | _PAGE_RW; ++ if (type == GTT_TYPE_PPGTT_PDE_PT) ++ se.val64 |= PPAT_CACHED; ++ ++ for (i = 0; i < page_entry_num; i++) ++ ops->set_entry(scratch_pt, &se, i, false, 0, vgpu); ++ } ++ ++ return 0; ++} ++ ++static int release_scratch_page_tree(struct intel_vgpu *vgpu) ++{ ++ int i; ++ struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev; ++ dma_addr_t daddr; ++ ++ for (i = GTT_TYPE_PPGTT_PTE_PT; i < GTT_TYPE_MAX; i++) { ++ if (vgpu->gtt.scratch_pt[i].page != NULL) { ++ daddr = (dma_addr_t)(vgpu->gtt.scratch_pt[i].page_mfn << ++ I915_GTT_PAGE_SHIFT); ++ dma_unmap_page(dev, daddr, 4096, PCI_DMA_BIDIRECTIONAL); ++ __free_page(vgpu->gtt.scratch_pt[i].page); ++ vgpu->gtt.scratch_pt[i].page = NULL; ++ vgpu->gtt.scratch_pt[i].page_mfn = 0; ++ } ++ } ++ ++ return 0; ++} ++ ++static int create_scratch_page_tree(struct intel_vgpu *vgpu) ++{ ++ int i, ret; ++ ++ for (i = GTT_TYPE_PPGTT_PTE_PT; i < GTT_TYPE_MAX; i++) { ++ ret = alloc_scratch_pages(vgpu, i); ++ if (ret) ++ goto err; ++ } ++ ++ return 0; ++ ++err: ++ release_scratch_page_tree(vgpu); ++ return ret; ++} ++ ++/** ++ * intel_vgpu_init_gtt - initialize per-vGPU graphics memory virulization ++ * @vgpu: a vGPU ++ * ++ * This function is used to initialize per-vGPU graphics memory virtualization ++ * components. ++ * ++ * Returns: ++ * Zero on success, error code if failed. ++ */ ++int intel_vgpu_init_gtt(struct intel_vgpu *vgpu) ++{ ++ struct intel_vgpu_gtt *gtt = &vgpu->gtt; ++ ++ INIT_RADIX_TREE(>t->spt_tree, GFP_KERNEL); ++ ++ INIT_LIST_HEAD(>t->ppgtt_mm_list_head); ++ INIT_LIST_HEAD(>t->oos_page_list_head); ++ INIT_LIST_HEAD(>t->post_shadow_list_head); ++ ++ gtt->ggtt_mm = intel_vgpu_create_ggtt_mm(vgpu); ++ if (IS_ERR(gtt->ggtt_mm)) { ++ gvt_vgpu_err("fail to create mm for ggtt.\n"); ++ return PTR_ERR(gtt->ggtt_mm); ++ } ++ ++ intel_vgpu_reset_ggtt(vgpu, false); ++ ++ INIT_LIST_HEAD(>t->ggtt_mm->ggtt_mm.partial_pte_list); ++ ++ return create_scratch_page_tree(vgpu); ++} ++ ++static void intel_vgpu_destroy_all_ppgtt_mm(struct intel_vgpu *vgpu) ++{ ++ struct list_head *pos, *n; ++ struct intel_vgpu_mm *mm; ++ ++ list_for_each_safe(pos, n, &vgpu->gtt.ppgtt_mm_list_head) { ++ mm = container_of(pos, struct intel_vgpu_mm, ppgtt_mm.list); ++ intel_vgpu_destroy_mm(mm); ++ } ++ ++ if (GEM_WARN_ON(!list_empty(&vgpu->gtt.ppgtt_mm_list_head))) ++ gvt_err("vgpu ppgtt mm is not fully destroyed\n"); ++ ++ if (GEM_WARN_ON(!radix_tree_empty(&vgpu->gtt.spt_tree))) { ++ gvt_err("Why we still has spt not freed?\n"); ++ ppgtt_free_all_spt(vgpu); ++ } ++} ++ ++static void intel_vgpu_destroy_ggtt_mm(struct intel_vgpu *vgpu) ++{ ++ struct intel_gvt_partial_pte *pos, *next; ++ ++ list_for_each_entry_safe(pos, next, ++ &vgpu->gtt.ggtt_mm->ggtt_mm.partial_pte_list, ++ list) { ++ gvt_dbg_mm("partial PTE update on hold 0x%lx : 0x%llx\n", ++ pos->offset, pos->data); ++ kfree(pos); ++ } ++ intel_vgpu_destroy_mm(vgpu->gtt.ggtt_mm); ++ vgpu->gtt.ggtt_mm = NULL; ++} ++ ++/** ++ * intel_vgpu_clean_gtt - clean up per-vGPU graphics memory virulization ++ * @vgpu: a vGPU ++ * ++ * This function is used to clean up per-vGPU graphics memory virtualization ++ * components. ++ * ++ * Returns: ++ * Zero on success, error code if failed. ++ */ ++void intel_vgpu_clean_gtt(struct intel_vgpu *vgpu) ++{ ++ intel_vgpu_destroy_all_ppgtt_mm(vgpu); ++ intel_vgpu_destroy_ggtt_mm(vgpu); ++ release_scratch_page_tree(vgpu); ++} ++ ++static void clean_spt_oos(struct intel_gvt *gvt) ++{ ++ struct intel_gvt_gtt *gtt = &gvt->gtt; ++ struct list_head *pos, *n; ++ struct intel_vgpu_oos_page *oos_page; ++ ++ WARN(!list_empty(>t->oos_page_use_list_head), ++ "someone is still using oos page\n"); ++ ++ list_for_each_safe(pos, n, >t->oos_page_free_list_head) { ++ oos_page = container_of(pos, struct intel_vgpu_oos_page, list); ++ list_del(&oos_page->list); ++ free_page((unsigned long)oos_page->mem); ++ kfree(oos_page); ++ } ++} ++ ++static int setup_spt_oos(struct intel_gvt *gvt) ++{ ++ struct intel_gvt_gtt *gtt = &gvt->gtt; ++ struct intel_vgpu_oos_page *oos_page; ++ int i; ++ int ret; ++ ++ INIT_LIST_HEAD(>t->oos_page_free_list_head); ++ INIT_LIST_HEAD(>t->oos_page_use_list_head); ++ ++ for (i = 0; i < preallocated_oos_pages; i++) { ++ oos_page = kzalloc(sizeof(*oos_page), GFP_KERNEL); ++ if (!oos_page) { ++ ret = -ENOMEM; ++ goto fail; ++ } ++ oos_page->mem = (void *)__get_free_pages(GFP_KERNEL, 0); ++ if (!oos_page->mem) { ++ ret = -ENOMEM; ++ kfree(oos_page); ++ goto fail; ++ } ++ ++ INIT_LIST_HEAD(&oos_page->list); ++ INIT_LIST_HEAD(&oos_page->vm_list); ++ oos_page->id = i; ++ list_add_tail(&oos_page->list, >t->oos_page_free_list_head); ++ } ++ ++ gvt_dbg_mm("%d oos pages preallocated\n", i); ++ ++ return 0; ++fail: ++ clean_spt_oos(gvt); ++ return ret; ++} ++ ++/** ++ * intel_vgpu_find_ppgtt_mm - find a PPGTT mm object ++ * @vgpu: a vGPU ++ * @pdps: pdp root array ++ * ++ * This function is used to find a PPGTT mm object from mm object pool ++ * ++ * Returns: ++ * pointer to mm object on success, NULL if failed. ++ */ ++struct intel_vgpu_mm *intel_vgpu_find_ppgtt_mm(struct intel_vgpu *vgpu, ++ u64 pdps[]) ++{ ++ struct intel_vgpu_mm *mm; ++ struct list_head *pos; ++ ++ list_for_each(pos, &vgpu->gtt.ppgtt_mm_list_head) { ++ mm = container_of(pos, struct intel_vgpu_mm, ppgtt_mm.list); ++ ++ switch (mm->ppgtt_mm.root_entry_type) { ++ case GTT_TYPE_PPGTT_ROOT_L4_ENTRY: ++ if (pdps[0] == mm->ppgtt_mm.guest_pdps[0]) ++ return mm; ++ break; ++ case GTT_TYPE_PPGTT_ROOT_L3_ENTRY: ++ if (!memcmp(pdps, mm->ppgtt_mm.guest_pdps, ++ sizeof(mm->ppgtt_mm.guest_pdps))) ++ return mm; ++ break; ++ default: ++ GEM_BUG_ON(1); ++ } ++ } ++ return NULL; ++} ++ ++/** ++ * intel_vgpu_get_ppgtt_mm - get or create a PPGTT mm object. ++ * @vgpu: a vGPU ++ * @root_entry_type: ppgtt root entry type ++ * @pdps: guest pdps ++ * ++ * This function is used to find or create a PPGTT mm object from a guest. ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++struct intel_vgpu_mm *intel_vgpu_get_ppgtt_mm(struct intel_vgpu *vgpu, ++ enum intel_gvt_gtt_type root_entry_type, u64 pdps[]) ++{ ++ struct intel_vgpu_mm *mm; ++ ++ mm = intel_vgpu_find_ppgtt_mm(vgpu, pdps); ++ if (mm) { ++ intel_vgpu_mm_get(mm); ++ } else { ++ mm = intel_vgpu_create_ppgtt_mm(vgpu, root_entry_type, pdps); ++ if (IS_ERR(mm)) ++ gvt_vgpu_err("fail to create mm\n"); ++ } ++ return mm; ++} ++ ++/** ++ * intel_vgpu_put_ppgtt_mm - find and put a PPGTT mm object. ++ * @vgpu: a vGPU ++ * @pdps: guest pdps ++ * ++ * This function is used to find a PPGTT mm object from a guest and destroy it. ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++int intel_vgpu_put_ppgtt_mm(struct intel_vgpu *vgpu, u64 pdps[]) ++{ ++ struct intel_vgpu_mm *mm; ++ ++ mm = intel_vgpu_find_ppgtt_mm(vgpu, pdps); ++ if (!mm) { ++ gvt_vgpu_err("fail to find ppgtt instance.\n"); ++ return -EINVAL; ++ } ++ intel_vgpu_mm_put(mm); ++ return 0; ++} ++ ++/** ++ * intel_gvt_init_gtt - initialize mm components of a GVT device ++ * @gvt: GVT device ++ * ++ * This function is called at the initialization stage, to initialize ++ * the mm components of a GVT device. ++ * ++ * Returns: ++ * zero on success, negative error code if failed. ++ */ ++int intel_gvt_init_gtt(struct intel_gvt *gvt) ++{ ++ int ret; ++ void *page; ++ struct device *dev = &gvt->dev_priv->drm.pdev->dev; ++ dma_addr_t daddr; ++ ++ gvt_dbg_core("init gtt\n"); ++ ++ gvt->gtt.pte_ops = &gen8_gtt_pte_ops; ++ gvt->gtt.gma_ops = &gen8_gtt_gma_ops; ++ ++ page = (void *)get_zeroed_page(GFP_KERNEL); ++ if (!page) { ++ gvt_err("fail to allocate scratch ggtt page\n"); ++ return -ENOMEM; ++ } ++ ++ daddr = dma_map_page(dev, virt_to_page(page), 0, ++ 4096, PCI_DMA_BIDIRECTIONAL); ++ if (dma_mapping_error(dev, daddr)) { ++ gvt_err("fail to dmamap scratch ggtt page\n"); ++ __free_page(virt_to_page(page)); ++ return -ENOMEM; ++ } ++ ++ gvt->gtt.scratch_page = virt_to_page(page); ++ gvt->gtt.scratch_mfn = (unsigned long)(daddr >> I915_GTT_PAGE_SHIFT); ++ ++ if (enable_out_of_sync) { ++ ret = setup_spt_oos(gvt); ++ if (ret) { ++ gvt_err("fail to initialize SPT oos\n"); ++ dma_unmap_page(dev, daddr, 4096, PCI_DMA_BIDIRECTIONAL); ++ __free_page(gvt->gtt.scratch_page); ++ return ret; ++ } ++ } ++ INIT_LIST_HEAD(&gvt->gtt.ppgtt_mm_lru_list_head); ++ mutex_init(&gvt->gtt.ppgtt_mm_lock); ++ return 0; ++} ++ ++/** ++ * intel_gvt_clean_gtt - clean up mm components of a GVT device ++ * @gvt: GVT device ++ * ++ * This function is called at the driver unloading stage, to clean up the ++ * the mm components of a GVT device. ++ * ++ */ ++void intel_gvt_clean_gtt(struct intel_gvt *gvt) ++{ ++ struct device *dev = &gvt->dev_priv->drm.pdev->dev; ++ dma_addr_t daddr = (dma_addr_t)(gvt->gtt.scratch_mfn << ++ I915_GTT_PAGE_SHIFT); ++ ++ dma_unmap_page(dev, daddr, 4096, PCI_DMA_BIDIRECTIONAL); ++ ++ __free_page(gvt->gtt.scratch_page); ++ ++ if (enable_out_of_sync) ++ clean_spt_oos(gvt); ++} ++ ++/** ++ * intel_vgpu_invalidate_ppgtt - invalidate PPGTT instances ++ * @vgpu: a vGPU ++ * ++ * This function is called when invalidate all PPGTT instances of a vGPU. ++ * ++ */ ++void intel_vgpu_invalidate_ppgtt(struct intel_vgpu *vgpu) ++{ ++ struct list_head *pos, *n; ++ struct intel_vgpu_mm *mm; ++ ++ list_for_each_safe(pos, n, &vgpu->gtt.ppgtt_mm_list_head) { ++ mm = container_of(pos, struct intel_vgpu_mm, ppgtt_mm.list); ++ if (mm->type == INTEL_GVT_MM_PPGTT) { ++ mutex_lock(&vgpu->gvt->gtt.ppgtt_mm_lock); ++ list_del_init(&mm->ppgtt_mm.lru_list); ++ mutex_unlock(&vgpu->gvt->gtt.ppgtt_mm_lock); ++ if (mm->ppgtt_mm.shadowed) ++ invalidate_ppgtt_mm(mm); ++ } ++ } ++} ++ ++/** ++ * intel_vgpu_reset_ggtt - reset the GGTT entry ++ * @vgpu: a vGPU ++ * @invalidate_old: invalidate old entries ++ * ++ * This function is called at the vGPU create stage ++ * to reset all the GGTT entries. ++ * ++ */ ++void intel_vgpu_reset_ggtt(struct intel_vgpu *vgpu, bool invalidate_old) ++{ ++ struct intel_gvt *gvt = vgpu->gvt; ++ struct drm_i915_private *dev_priv = gvt->dev_priv; ++ struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops; ++ struct intel_gvt_gtt_entry entry = {.type = GTT_TYPE_GGTT_PTE}; ++ struct intel_gvt_gtt_entry old_entry; ++ u32 index; ++ u32 num_entries; ++ ++ pte_ops->set_pfn(&entry, gvt->gtt.scratch_mfn); ++ pte_ops->set_present(&entry); ++ ++ index = vgpu_aperture_gmadr_base(vgpu) >> PAGE_SHIFT; ++ num_entries = vgpu_aperture_sz(vgpu) >> PAGE_SHIFT; ++ while (num_entries--) { ++ if (invalidate_old) { ++ ggtt_get_host_entry(vgpu->gtt.ggtt_mm, &old_entry, index); ++ ggtt_invalidate_pte(vgpu, &old_entry); ++ } ++ ggtt_set_host_entry(vgpu->gtt.ggtt_mm, &entry, index++); ++ } ++ ++ index = vgpu_hidden_gmadr_base(vgpu) >> PAGE_SHIFT; ++ num_entries = vgpu_hidden_sz(vgpu) >> PAGE_SHIFT; ++ while (num_entries--) { ++ if (invalidate_old) { ++ ggtt_get_host_entry(vgpu->gtt.ggtt_mm, &old_entry, index); ++ ggtt_invalidate_pte(vgpu, &old_entry); ++ } ++ ggtt_set_host_entry(vgpu->gtt.ggtt_mm, &entry, index++); ++ } ++ ++ ggtt_invalidate(dev_priv); ++} ++ ++/** ++ * intel_vgpu_reset_gtt - reset the all GTT related status ++ * @vgpu: a vGPU ++ * ++ * This function is called from vfio core to reset reset all ++ * GTT related status, including GGTT, PPGTT, scratch page. ++ * ++ */ ++void intel_vgpu_reset_gtt(struct intel_vgpu *vgpu) ++{ ++ /* Shadow pages are only created when there is no page ++ * table tracking data, so remove page tracking data after ++ * removing the shadow pages. ++ */ ++ intel_vgpu_destroy_all_ppgtt_mm(vgpu); ++ intel_vgpu_reset_ggtt(vgpu, true); ++} +diff --git a/drivers/gpu/drm/i915_legacy/gvt/gtt.h b/drivers/gpu/drm/i915_legacy/gvt/gtt.h +new file mode 100644 +index 000000000000..42d0394f0de2 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/gtt.h +@@ -0,0 +1,280 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Zhi Wang ++ * Zhenyu Wang ++ * Xiao Zheng ++ * ++ * Contributors: ++ * Min He ++ * Bing Niu ++ * ++ */ ++ ++#ifndef _GVT_GTT_H_ ++#define _GVT_GTT_H_ ++ ++#define I915_GTT_PAGE_SHIFT 12 ++ ++struct intel_vgpu_mm; ++ ++#define INTEL_GVT_INVALID_ADDR (~0UL) ++ ++struct intel_gvt_gtt_entry { ++ u64 val64; ++ int type; ++}; ++ ++struct intel_gvt_gtt_pte_ops { ++ int (*get_entry)(void *pt, ++ struct intel_gvt_gtt_entry *e, ++ unsigned long index, ++ bool hypervisor_access, ++ unsigned long gpa, ++ struct intel_vgpu *vgpu); ++ int (*set_entry)(void *pt, ++ struct intel_gvt_gtt_entry *e, ++ unsigned long index, ++ bool hypervisor_access, ++ unsigned long gpa, ++ struct intel_vgpu *vgpu); ++ bool (*test_present)(struct intel_gvt_gtt_entry *e); ++ void (*clear_present)(struct intel_gvt_gtt_entry *e); ++ void (*set_present)(struct intel_gvt_gtt_entry *e); ++ bool (*test_pse)(struct intel_gvt_gtt_entry *e); ++ void (*clear_pse)(struct intel_gvt_gtt_entry *e); ++ bool (*test_ips)(struct intel_gvt_gtt_entry *e); ++ void (*clear_ips)(struct intel_gvt_gtt_entry *e); ++ bool (*test_64k_splited)(struct intel_gvt_gtt_entry *e); ++ void (*clear_64k_splited)(struct intel_gvt_gtt_entry *e); ++ void (*set_64k_splited)(struct intel_gvt_gtt_entry *e); ++ void (*set_pfn)(struct intel_gvt_gtt_entry *e, unsigned long pfn); ++ unsigned long (*get_pfn)(struct intel_gvt_gtt_entry *e); ++}; ++ ++struct intel_gvt_gtt_gma_ops { ++ unsigned long (*gma_to_ggtt_pte_index)(unsigned long gma); ++ unsigned long (*gma_to_pte_index)(unsigned long gma); ++ unsigned long (*gma_to_pde_index)(unsigned long gma); ++ unsigned long (*gma_to_l3_pdp_index)(unsigned long gma); ++ unsigned long (*gma_to_l4_pdp_index)(unsigned long gma); ++ unsigned long (*gma_to_pml4_index)(unsigned long gma); ++}; ++ ++struct intel_gvt_gtt { ++ struct intel_gvt_gtt_pte_ops *pte_ops; ++ struct intel_gvt_gtt_gma_ops *gma_ops; ++ int (*mm_alloc_page_table)(struct intel_vgpu_mm *mm); ++ void (*mm_free_page_table)(struct intel_vgpu_mm *mm); ++ struct list_head oos_page_use_list_head; ++ struct list_head oos_page_free_list_head; ++ struct mutex ppgtt_mm_lock; ++ struct list_head ppgtt_mm_lru_list_head; ++ ++ struct page *scratch_page; ++ unsigned long scratch_mfn; ++}; ++ ++enum intel_gvt_gtt_type { ++ GTT_TYPE_INVALID = 0, ++ ++ GTT_TYPE_GGTT_PTE, ++ ++ GTT_TYPE_PPGTT_PTE_4K_ENTRY, ++ GTT_TYPE_PPGTT_PTE_64K_ENTRY, ++ GTT_TYPE_PPGTT_PTE_2M_ENTRY, ++ GTT_TYPE_PPGTT_PTE_1G_ENTRY, ++ ++ GTT_TYPE_PPGTT_PTE_ENTRY, ++ ++ GTT_TYPE_PPGTT_PDE_ENTRY, ++ GTT_TYPE_PPGTT_PDP_ENTRY, ++ GTT_TYPE_PPGTT_PML4_ENTRY, ++ ++ GTT_TYPE_PPGTT_ROOT_ENTRY, ++ ++ GTT_TYPE_PPGTT_ROOT_L3_ENTRY, ++ GTT_TYPE_PPGTT_ROOT_L4_ENTRY, ++ ++ GTT_TYPE_PPGTT_ENTRY, ++ ++ GTT_TYPE_PPGTT_PTE_PT, ++ GTT_TYPE_PPGTT_PDE_PT, ++ GTT_TYPE_PPGTT_PDP_PT, ++ GTT_TYPE_PPGTT_PML4_PT, ++ ++ GTT_TYPE_MAX, ++}; ++ ++enum intel_gvt_mm_type { ++ INTEL_GVT_MM_GGTT, ++ INTEL_GVT_MM_PPGTT, ++}; ++ ++#define GVT_RING_CTX_NR_PDPS GEN8_3LVL_PDPES ++ ++struct intel_gvt_partial_pte { ++ unsigned long offset; ++ u64 data; ++ struct list_head list; ++}; ++ ++struct intel_vgpu_mm { ++ enum intel_gvt_mm_type type; ++ struct intel_vgpu *vgpu; ++ ++ struct kref ref; ++ atomic_t pincount; ++ ++ union { ++ struct { ++ enum intel_gvt_gtt_type root_entry_type; ++ /* ++ * The 4 PDPs in ring context. For 48bit addressing, ++ * only PDP0 is valid and point to PML4. For 32it ++ * addressing, all 4 are used as true PDPs. ++ */ ++ u64 guest_pdps[GVT_RING_CTX_NR_PDPS]; ++ u64 shadow_pdps[GVT_RING_CTX_NR_PDPS]; ++ bool shadowed; ++ ++ struct list_head list; ++ struct list_head lru_list; ++ } ppgtt_mm; ++ struct { ++ void *virtual_ggtt; ++ struct list_head partial_pte_list; ++ } ggtt_mm; ++ }; ++}; ++ ++struct intel_vgpu_mm *intel_vgpu_create_ppgtt_mm(struct intel_vgpu *vgpu, ++ enum intel_gvt_gtt_type root_entry_type, u64 pdps[]); ++ ++static inline void intel_vgpu_mm_get(struct intel_vgpu_mm *mm) ++{ ++ kref_get(&mm->ref); ++} ++ ++void _intel_vgpu_mm_release(struct kref *mm_ref); ++ ++static inline void intel_vgpu_mm_put(struct intel_vgpu_mm *mm) ++{ ++ kref_put(&mm->ref, _intel_vgpu_mm_release); ++} ++ ++static inline void intel_vgpu_destroy_mm(struct intel_vgpu_mm *mm) ++{ ++ intel_vgpu_mm_put(mm); ++} ++ ++struct intel_vgpu_guest_page; ++ ++struct intel_vgpu_scratch_pt { ++ struct page *page; ++ unsigned long page_mfn; ++}; ++ ++struct intel_vgpu_gtt { ++ struct intel_vgpu_mm *ggtt_mm; ++ unsigned long active_ppgtt_mm_bitmap; ++ struct list_head ppgtt_mm_list_head; ++ struct radix_tree_root spt_tree; ++ struct list_head oos_page_list_head; ++ struct list_head post_shadow_list_head; ++ struct intel_vgpu_scratch_pt scratch_pt[GTT_TYPE_MAX]; ++}; ++ ++extern int intel_vgpu_init_gtt(struct intel_vgpu *vgpu); ++extern void intel_vgpu_clean_gtt(struct intel_vgpu *vgpu); ++void intel_vgpu_reset_ggtt(struct intel_vgpu *vgpu, bool invalidate_old); ++void intel_vgpu_invalidate_ppgtt(struct intel_vgpu *vgpu); ++ ++extern int intel_gvt_init_gtt(struct intel_gvt *gvt); ++void intel_vgpu_reset_gtt(struct intel_vgpu *vgpu); ++extern void intel_gvt_clean_gtt(struct intel_gvt *gvt); ++ ++extern struct intel_vgpu_mm *intel_gvt_find_ppgtt_mm(struct intel_vgpu *vgpu, ++ int page_table_level, void *root_entry); ++ ++struct intel_vgpu_oos_page { ++ struct intel_vgpu_ppgtt_spt *spt; ++ struct list_head list; ++ struct list_head vm_list; ++ int id; ++ void *mem; ++}; ++ ++#define GTT_ENTRY_NUM_IN_ONE_PAGE 512 ++ ++/* Represent a vgpu shadow page table. */ ++struct intel_vgpu_ppgtt_spt { ++ atomic_t refcount; ++ struct intel_vgpu *vgpu; ++ ++ struct { ++ enum intel_gvt_gtt_type type; ++ bool pde_ips; /* for 64KB PTEs */ ++ void *vaddr; ++ struct page *page; ++ unsigned long mfn; ++ } shadow_page; ++ ++ struct { ++ enum intel_gvt_gtt_type type; ++ bool pde_ips; /* for 64KB PTEs */ ++ unsigned long gfn; ++ unsigned long write_cnt; ++ struct intel_vgpu_oos_page *oos_page; ++ } guest_page; ++ ++ DECLARE_BITMAP(post_shadow_bitmap, GTT_ENTRY_NUM_IN_ONE_PAGE); ++ struct list_head post_shadow_list; ++}; ++ ++int intel_vgpu_sync_oos_pages(struct intel_vgpu *vgpu); ++ ++int intel_vgpu_flush_post_shadow(struct intel_vgpu *vgpu); ++ ++int intel_vgpu_pin_mm(struct intel_vgpu_mm *mm); ++ ++void intel_vgpu_unpin_mm(struct intel_vgpu_mm *mm); ++ ++unsigned long intel_vgpu_gma_to_gpa(struct intel_vgpu_mm *mm, ++ unsigned long gma); ++ ++struct intel_vgpu_mm *intel_vgpu_find_ppgtt_mm(struct intel_vgpu *vgpu, ++ u64 pdps[]); ++ ++struct intel_vgpu_mm *intel_vgpu_get_ppgtt_mm(struct intel_vgpu *vgpu, ++ enum intel_gvt_gtt_type root_entry_type, u64 pdps[]); ++ ++int intel_vgpu_put_ppgtt_mm(struct intel_vgpu *vgpu, u64 pdps[]); ++ ++int intel_vgpu_emulate_ggtt_mmio_read(struct intel_vgpu *vgpu, ++ unsigned int off, void *p_data, unsigned int bytes); ++ ++int intel_vgpu_emulate_ggtt_mmio_write(struct intel_vgpu *vgpu, ++ unsigned int off, void *p_data, unsigned int bytes); ++ ++#endif /* _GVT_GTT_H_ */ +diff --git a/drivers/gpu/drm/i915_legacy/gvt/gvt.c b/drivers/gpu/drm/i915_legacy/gvt/gvt.c +new file mode 100644 +index 000000000000..43f4242062dd +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/gvt.c +@@ -0,0 +1,453 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Kevin Tian ++ * Eddie Dong ++ * ++ * Contributors: ++ * Niu Bing ++ * Zhi Wang ++ * ++ */ ++ ++#include ++#include ++#include ++ ++#include "i915_drv.h" ++#include "gvt.h" ++#include ++#include ++ ++struct intel_gvt_host intel_gvt_host; ++ ++static const char * const supported_hypervisors[] = { ++ [INTEL_GVT_HYPERVISOR_XEN] = "XEN", ++ [INTEL_GVT_HYPERVISOR_KVM] = "KVM", ++}; ++ ++static struct intel_vgpu_type *intel_gvt_find_vgpu_type(struct intel_gvt *gvt, ++ const char *name) ++{ ++ int i; ++ struct intel_vgpu_type *t; ++ const char *driver_name = dev_driver_string( ++ &gvt->dev_priv->drm.pdev->dev); ++ ++ for (i = 0; i < gvt->num_types; i++) { ++ t = &gvt->types[i]; ++ if (!strncmp(t->name, name + strlen(driver_name) + 1, ++ sizeof(t->name))) ++ return t; ++ } ++ ++ return NULL; ++} ++ ++static ssize_t available_instances_show(struct kobject *kobj, ++ struct device *dev, char *buf) ++{ ++ struct intel_vgpu_type *type; ++ unsigned int num = 0; ++ void *gvt = kdev_to_i915(dev)->gvt; ++ ++ type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj)); ++ if (!type) ++ num = 0; ++ else ++ num = type->avail_instance; ++ ++ return sprintf(buf, "%u\n", num); ++} ++ ++static ssize_t device_api_show(struct kobject *kobj, struct device *dev, ++ char *buf) ++{ ++ return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING); ++} ++ ++static ssize_t description_show(struct kobject *kobj, struct device *dev, ++ char *buf) ++{ ++ struct intel_vgpu_type *type; ++ void *gvt = kdev_to_i915(dev)->gvt; ++ ++ type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj)); ++ if (!type) ++ return 0; ++ ++ return sprintf(buf, "low_gm_size: %dMB\nhigh_gm_size: %dMB\n" ++ "fence: %d\nresolution: %s\n" ++ "weight: %d\n", ++ BYTES_TO_MB(type->low_gm_size), ++ BYTES_TO_MB(type->high_gm_size), ++ type->fence, vgpu_edid_str(type->resolution), ++ type->weight); ++} ++ ++static MDEV_TYPE_ATTR_RO(available_instances); ++static MDEV_TYPE_ATTR_RO(device_api); ++static MDEV_TYPE_ATTR_RO(description); ++ ++static struct attribute *gvt_type_attrs[] = { ++ &mdev_type_attr_available_instances.attr, ++ &mdev_type_attr_device_api.attr, ++ &mdev_type_attr_description.attr, ++ NULL, ++}; ++ ++static struct attribute_group *gvt_vgpu_type_groups[] = { ++ [0 ... NR_MAX_INTEL_VGPU_TYPES - 1] = NULL, ++}; ++ ++static bool intel_get_gvt_attrs(struct attribute ***type_attrs, ++ struct attribute_group ***intel_vgpu_type_groups) ++{ ++ *type_attrs = gvt_type_attrs; ++ *intel_vgpu_type_groups = gvt_vgpu_type_groups; ++ return true; ++} ++ ++static bool intel_gvt_init_vgpu_type_groups(struct intel_gvt *gvt) ++{ ++ int i, j; ++ struct intel_vgpu_type *type; ++ struct attribute_group *group; ++ ++ for (i = 0; i < gvt->num_types; i++) { ++ type = &gvt->types[i]; ++ ++ group = kzalloc(sizeof(struct attribute_group), GFP_KERNEL); ++ if (WARN_ON(!group)) ++ goto unwind; ++ ++ group->name = type->name; ++ group->attrs = gvt_type_attrs; ++ gvt_vgpu_type_groups[i] = group; ++ } ++ ++ return true; ++ ++unwind: ++ for (j = 0; j < i; j++) { ++ group = gvt_vgpu_type_groups[j]; ++ kfree(group); ++ } ++ ++ return false; ++} ++ ++static void intel_gvt_cleanup_vgpu_type_groups(struct intel_gvt *gvt) ++{ ++ int i; ++ struct attribute_group *group; ++ ++ for (i = 0; i < gvt->num_types; i++) { ++ group = gvt_vgpu_type_groups[i]; ++ gvt_vgpu_type_groups[i] = NULL; ++ kfree(group); ++ } ++} ++ ++static const struct intel_gvt_ops intel_gvt_ops = { ++ .emulate_cfg_read = intel_vgpu_emulate_cfg_read, ++ .emulate_cfg_write = intel_vgpu_emulate_cfg_write, ++ .emulate_mmio_read = intel_vgpu_emulate_mmio_read, ++ .emulate_mmio_write = intel_vgpu_emulate_mmio_write, ++ .vgpu_create = intel_gvt_create_vgpu, ++ .vgpu_destroy = intel_gvt_destroy_vgpu, ++ .vgpu_release = intel_gvt_release_vgpu, ++ .vgpu_reset = intel_gvt_reset_vgpu, ++ .vgpu_activate = intel_gvt_activate_vgpu, ++ .vgpu_deactivate = intel_gvt_deactivate_vgpu, ++ .gvt_find_vgpu_type = intel_gvt_find_vgpu_type, ++ .get_gvt_attrs = intel_get_gvt_attrs, ++ .vgpu_query_plane = intel_vgpu_query_plane, ++ .vgpu_get_dmabuf = intel_vgpu_get_dmabuf, ++ .write_protect_handler = intel_vgpu_page_track_handler, ++ .emulate_hotplug = intel_vgpu_emulate_hotplug, ++}; ++ ++static void init_device_info(struct intel_gvt *gvt) ++{ ++ struct intel_gvt_device_info *info = &gvt->device_info; ++ struct pci_dev *pdev = gvt->dev_priv->drm.pdev; ++ ++ info->max_support_vgpus = 8; ++ info->cfg_space_size = PCI_CFG_SPACE_EXP_SIZE; ++ info->mmio_size = 2 * 1024 * 1024; ++ info->mmio_bar = 0; ++ info->gtt_start_offset = 8 * 1024 * 1024; ++ info->gtt_entry_size = 8; ++ info->gtt_entry_size_shift = 3; ++ info->gmadr_bytes_in_cmd = 8; ++ info->max_surface_size = 36 * 1024 * 1024; ++ info->msi_cap_offset = pdev->msi_cap; ++} ++ ++static int gvt_service_thread(void *data) ++{ ++ struct intel_gvt *gvt = (struct intel_gvt *)data; ++ int ret; ++ ++ gvt_dbg_core("service thread start\n"); ++ ++ while (!kthread_should_stop()) { ++ ret = wait_event_interruptible(gvt->service_thread_wq, ++ kthread_should_stop() || gvt->service_request); ++ ++ if (kthread_should_stop()) ++ break; ++ ++ if (WARN_ONCE(ret, "service thread is waken up by signal.\n")) ++ continue; ++ ++ if (test_and_clear_bit(INTEL_GVT_REQUEST_EMULATE_VBLANK, ++ (void *)&gvt->service_request)) ++ intel_gvt_emulate_vblank(gvt); ++ ++ if (test_bit(INTEL_GVT_REQUEST_SCHED, ++ (void *)&gvt->service_request) || ++ test_bit(INTEL_GVT_REQUEST_EVENT_SCHED, ++ (void *)&gvt->service_request)) { ++ intel_gvt_schedule(gvt); ++ } ++ } ++ ++ return 0; ++} ++ ++static void clean_service_thread(struct intel_gvt *gvt) ++{ ++ kthread_stop(gvt->service_thread); ++} ++ ++static int init_service_thread(struct intel_gvt *gvt) ++{ ++ init_waitqueue_head(&gvt->service_thread_wq); ++ ++ gvt->service_thread = kthread_run(gvt_service_thread, ++ gvt, "gvt_service_thread"); ++ if (IS_ERR(gvt->service_thread)) { ++ gvt_err("fail to start service thread.\n"); ++ return PTR_ERR(gvt->service_thread); ++ } ++ return 0; ++} ++ ++/** ++ * intel_gvt_clean_device - clean a GVT device ++ * @dev_priv: i915 private ++ * ++ * This function is called at the driver unloading stage, to free the ++ * resources owned by a GVT device. ++ * ++ */ ++void intel_gvt_clean_device(struct drm_i915_private *dev_priv) ++{ ++ struct intel_gvt *gvt = to_gvt(dev_priv); ++ ++ if (WARN_ON(!gvt)) ++ return; ++ ++ intel_gvt_destroy_idle_vgpu(gvt->idle_vgpu); ++ intel_gvt_cleanup_vgpu_type_groups(gvt); ++ intel_gvt_clean_vgpu_types(gvt); ++ ++ intel_gvt_debugfs_clean(gvt); ++ clean_service_thread(gvt); ++ intel_gvt_clean_cmd_parser(gvt); ++ intel_gvt_clean_sched_policy(gvt); ++ intel_gvt_clean_workload_scheduler(gvt); ++ intel_gvt_clean_gtt(gvt); ++ intel_gvt_clean_irq(gvt); ++ intel_gvt_free_firmware(gvt); ++ intel_gvt_clean_mmio_info(gvt); ++ idr_destroy(&gvt->vgpu_idr); ++ ++ kfree(dev_priv->gvt); ++ dev_priv->gvt = NULL; ++} ++ ++/** ++ * intel_gvt_init_device - initialize a GVT device ++ * @dev_priv: drm i915 private data ++ * ++ * This function is called at the initialization stage, to initialize ++ * necessary GVT components. ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ * ++ */ ++int intel_gvt_init_device(struct drm_i915_private *dev_priv) ++{ ++ struct intel_gvt *gvt; ++ struct intel_vgpu *vgpu; ++ int ret; ++ ++ if (WARN_ON(dev_priv->gvt)) ++ return -EEXIST; ++ ++ gvt = kzalloc(sizeof(struct intel_gvt), GFP_KERNEL); ++ if (!gvt) ++ return -ENOMEM; ++ ++ gvt_dbg_core("init gvt device\n"); ++ ++ idr_init(&gvt->vgpu_idr); ++ spin_lock_init(&gvt->scheduler.mmio_context_lock); ++ mutex_init(&gvt->lock); ++ mutex_init(&gvt->sched_lock); ++ gvt->dev_priv = dev_priv; ++ ++ init_device_info(gvt); ++ ++ ret = intel_gvt_setup_mmio_info(gvt); ++ if (ret) ++ goto out_clean_idr; ++ ++ intel_gvt_init_engine_mmio_context(gvt); ++ ++ ret = intel_gvt_load_firmware(gvt); ++ if (ret) ++ goto out_clean_mmio_info; ++ ++ ret = intel_gvt_init_irq(gvt); ++ if (ret) ++ goto out_free_firmware; ++ ++ ret = intel_gvt_init_gtt(gvt); ++ if (ret) ++ goto out_clean_irq; ++ ++ ret = intel_gvt_init_workload_scheduler(gvt); ++ if (ret) ++ goto out_clean_gtt; ++ ++ ret = intel_gvt_init_sched_policy(gvt); ++ if (ret) ++ goto out_clean_workload_scheduler; ++ ++ ret = intel_gvt_init_cmd_parser(gvt); ++ if (ret) ++ goto out_clean_sched_policy; ++ ++ ret = init_service_thread(gvt); ++ if (ret) ++ goto out_clean_cmd_parser; ++ ++ ret = intel_gvt_init_vgpu_types(gvt); ++ if (ret) ++ goto out_clean_thread; ++ ++ ret = intel_gvt_init_vgpu_type_groups(gvt); ++ if (ret == false) { ++ gvt_err("failed to init vgpu type groups: %d\n", ret); ++ goto out_clean_types; ++ } ++ ++ vgpu = intel_gvt_create_idle_vgpu(gvt); ++ if (IS_ERR(vgpu)) { ++ ret = PTR_ERR(vgpu); ++ gvt_err("failed to create idle vgpu\n"); ++ goto out_clean_types; ++ } ++ gvt->idle_vgpu = vgpu; ++ ++ ret = intel_gvt_debugfs_init(gvt); ++ if (ret) ++ gvt_err("debugfs registration failed, go on.\n"); ++ ++ gvt_dbg_core("gvt device initialization is done\n"); ++ dev_priv->gvt = gvt; ++ intel_gvt_host.dev = &dev_priv->drm.pdev->dev; ++ intel_gvt_host.initialized = true; ++ return 0; ++ ++out_clean_types: ++ intel_gvt_clean_vgpu_types(gvt); ++out_clean_thread: ++ clean_service_thread(gvt); ++out_clean_cmd_parser: ++ intel_gvt_clean_cmd_parser(gvt); ++out_clean_sched_policy: ++ intel_gvt_clean_sched_policy(gvt); ++out_clean_workload_scheduler: ++ intel_gvt_clean_workload_scheduler(gvt); ++out_clean_gtt: ++ intel_gvt_clean_gtt(gvt); ++out_clean_irq: ++ intel_gvt_clean_irq(gvt); ++out_free_firmware: ++ intel_gvt_free_firmware(gvt); ++out_clean_mmio_info: ++ intel_gvt_clean_mmio_info(gvt); ++out_clean_idr: ++ idr_destroy(&gvt->vgpu_idr); ++ kfree(gvt); ++ return ret; ++} ++ ++int ++intel_gvt_register_hypervisor(struct intel_gvt_mpt *m) ++{ ++ int ret; ++ void *gvt; ++ ++ if (!intel_gvt_host.initialized) ++ return -ENODEV; ++ ++ if (m->type != INTEL_GVT_HYPERVISOR_KVM && ++ m->type != INTEL_GVT_HYPERVISOR_XEN) ++ return -EINVAL; ++ ++ /* Get a reference for device model module */ ++ if (!try_module_get(THIS_MODULE)) ++ return -ENODEV; ++ ++ intel_gvt_host.mpt = m; ++ intel_gvt_host.hypervisor_type = m->type; ++ gvt = (void *)kdev_to_i915(intel_gvt_host.dev)->gvt; ++ ++ ret = intel_gvt_hypervisor_host_init(intel_gvt_host.dev, gvt, ++ &intel_gvt_ops); ++ if (ret < 0) { ++ gvt_err("Failed to init %s hypervisor module\n", ++ supported_hypervisors[intel_gvt_host.hypervisor_type]); ++ module_put(THIS_MODULE); ++ return -ENODEV; ++ } ++ gvt_dbg_core("Running with hypervisor %s in host mode\n", ++ supported_hypervisors[intel_gvt_host.hypervisor_type]); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(intel_gvt_register_hypervisor); ++ ++void ++intel_gvt_unregister_hypervisor(void) ++{ ++ intel_gvt_hypervisor_host_exit(intel_gvt_host.dev); ++ module_put(THIS_MODULE); ++} ++EXPORT_SYMBOL_GPL(intel_gvt_unregister_hypervisor); +diff --git a/drivers/gpu/drm/i915_legacy/gvt/gvt.h b/drivers/gpu/drm/i915_legacy/gvt/gvt.h +new file mode 100644 +index 000000000000..f5a328b5290a +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/gvt.h +@@ -0,0 +1,694 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Kevin Tian ++ * Eddie Dong ++ * ++ * Contributors: ++ * Niu Bing ++ * Zhi Wang ++ * ++ */ ++ ++#ifndef _GVT_H_ ++#define _GVT_H_ ++ ++#include "debug.h" ++#include "hypercall.h" ++#include "mmio.h" ++#include "reg.h" ++#include "interrupt.h" ++#include "gtt.h" ++#include "display.h" ++#include "edid.h" ++#include "execlist.h" ++#include "scheduler.h" ++#include "sched_policy.h" ++#include "mmio_context.h" ++#include "cmd_parser.h" ++#include "fb_decoder.h" ++#include "dmabuf.h" ++#include "page_track.h" ++ ++#define GVT_MAX_VGPU 8 ++ ++struct intel_gvt_host { ++ struct device *dev; ++ bool initialized; ++ int hypervisor_type; ++ struct intel_gvt_mpt *mpt; ++}; ++ ++extern struct intel_gvt_host intel_gvt_host; ++ ++/* Describe per-platform limitations. */ ++struct intel_gvt_device_info { ++ u32 max_support_vgpus; ++ u32 cfg_space_size; ++ u32 mmio_size; ++ u32 mmio_bar; ++ unsigned long msi_cap_offset; ++ u32 gtt_start_offset; ++ u32 gtt_entry_size; ++ u32 gtt_entry_size_shift; ++ int gmadr_bytes_in_cmd; ++ u32 max_surface_size; ++}; ++ ++/* GM resources owned by a vGPU */ ++struct intel_vgpu_gm { ++ u64 aperture_sz; ++ u64 hidden_sz; ++ struct drm_mm_node low_gm_node; ++ struct drm_mm_node high_gm_node; ++}; ++ ++#define INTEL_GVT_MAX_NUM_FENCES 32 ++ ++/* Fences owned by a vGPU */ ++struct intel_vgpu_fence { ++ struct drm_i915_fence_reg *regs[INTEL_GVT_MAX_NUM_FENCES]; ++ u32 base; ++ u32 size; ++}; ++ ++struct intel_vgpu_mmio { ++ void *vreg; ++}; ++ ++#define INTEL_GVT_MAX_BAR_NUM 4 ++ ++struct intel_vgpu_pci_bar { ++ u64 size; ++ bool tracked; ++}; ++ ++struct intel_vgpu_cfg_space { ++ unsigned char virtual_cfg_space[PCI_CFG_SPACE_EXP_SIZE]; ++ struct intel_vgpu_pci_bar bar[INTEL_GVT_MAX_BAR_NUM]; ++}; ++ ++#define vgpu_cfg_space(vgpu) ((vgpu)->cfg_space.virtual_cfg_space) ++ ++struct intel_vgpu_irq { ++ bool irq_warn_once[INTEL_GVT_EVENT_MAX]; ++ DECLARE_BITMAP(flip_done_event[I915_MAX_PIPES], ++ INTEL_GVT_EVENT_MAX); ++}; ++ ++struct intel_vgpu_opregion { ++ bool mapped; ++ void *va; ++ u32 gfn[INTEL_GVT_OPREGION_PAGES]; ++}; ++ ++#define vgpu_opregion(vgpu) (&(vgpu->opregion)) ++ ++struct intel_vgpu_display { ++ struct intel_vgpu_i2c_edid i2c_edid; ++ struct intel_vgpu_port ports[I915_MAX_PORTS]; ++ struct intel_vgpu_sbi sbi; ++}; ++ ++struct vgpu_sched_ctl { ++ int weight; ++}; ++ ++enum { ++ INTEL_VGPU_EXECLIST_SUBMISSION = 1, ++ INTEL_VGPU_GUC_SUBMISSION, ++}; ++ ++struct intel_vgpu_submission_ops { ++ const char *name; ++ int (*init)(struct intel_vgpu *vgpu, intel_engine_mask_t engine_mask); ++ void (*clean)(struct intel_vgpu *vgpu, intel_engine_mask_t engine_mask); ++ void (*reset)(struct intel_vgpu *vgpu, intel_engine_mask_t engine_mask); ++}; ++ ++struct intel_vgpu_submission { ++ struct intel_vgpu_execlist execlist[I915_NUM_ENGINES]; ++ struct list_head workload_q_head[I915_NUM_ENGINES]; ++ struct kmem_cache *workloads; ++ atomic_t running_workload_num; ++ struct i915_gem_context *shadow_ctx; ++ union { ++ u64 i915_context_pml4; ++ u64 i915_context_pdps[GEN8_3LVL_PDPES]; ++ }; ++ DECLARE_BITMAP(shadow_ctx_desc_updated, I915_NUM_ENGINES); ++ DECLARE_BITMAP(tlb_handle_pending, I915_NUM_ENGINES); ++ void *ring_scan_buffer[I915_NUM_ENGINES]; ++ int ring_scan_buffer_size[I915_NUM_ENGINES]; ++ const struct intel_vgpu_submission_ops *ops; ++ int virtual_submission_interface; ++ bool active; ++}; ++ ++struct intel_vgpu { ++ struct intel_gvt *gvt; ++ struct mutex vgpu_lock; ++ int id; ++ unsigned long handle; /* vGPU handle used by hypervisor MPT modules */ ++ bool active; ++ bool pv_notified; ++ bool failsafe; ++ unsigned int resetting_eng; ++ ++ /* Both sched_data and sched_ctl can be seen a part of the global gvt ++ * scheduler structure. So below 2 vgpu data are protected ++ * by sched_lock, not vgpu_lock. ++ */ ++ void *sched_data; ++ struct vgpu_sched_ctl sched_ctl; ++ ++ struct intel_vgpu_fence fence; ++ struct intel_vgpu_gm gm; ++ struct intel_vgpu_cfg_space cfg_space; ++ struct intel_vgpu_mmio mmio; ++ struct intel_vgpu_irq irq; ++ struct intel_vgpu_gtt gtt; ++ struct intel_vgpu_opregion opregion; ++ struct intel_vgpu_display display; ++ struct intel_vgpu_submission submission; ++ struct radix_tree_root page_track_tree; ++ u32 hws_pga[I915_NUM_ENGINES]; ++ ++ struct dentry *debugfs; ++ ++#if IS_ENABLED(CONFIG_DRM_I915_GVT_KVMGT) ++ struct { ++ struct mdev_device *mdev; ++ struct vfio_region *region; ++ int num_regions; ++ struct eventfd_ctx *intx_trigger; ++ struct eventfd_ctx *msi_trigger; ++ ++ /* ++ * Two caches are used to avoid mapping duplicated pages (eg. ++ * scratch pages). This help to reduce dma setup overhead. ++ */ ++ struct rb_root gfn_cache; ++ struct rb_root dma_addr_cache; ++ unsigned long nr_cache_entries; ++ struct mutex cache_lock; ++ ++ struct notifier_block iommu_notifier; ++ struct notifier_block group_notifier; ++ struct kvm *kvm; ++ struct work_struct release_work; ++ atomic_t released; ++ struct vfio_device *vfio_device; ++ } vdev; ++#endif ++ ++ struct list_head dmabuf_obj_list_head; ++ struct mutex dmabuf_lock; ++ struct idr object_idr; ++ ++ struct completion vblank_done; ++ ++ u32 scan_nonprivbb; ++}; ++ ++/* validating GM healthy status*/ ++#define vgpu_is_vm_unhealthy(ret_val) \ ++ (((ret_val) == -EBADRQC) || ((ret_val) == -EFAULT)) ++ ++struct intel_gvt_gm { ++ unsigned long vgpu_allocated_low_gm_size; ++ unsigned long vgpu_allocated_high_gm_size; ++}; ++ ++struct intel_gvt_fence { ++ unsigned long vgpu_allocated_fence_num; ++}; ++ ++/* Special MMIO blocks. */ ++struct gvt_mmio_block { ++ unsigned int device; ++ i915_reg_t offset; ++ unsigned int size; ++ gvt_mmio_func read; ++ gvt_mmio_func write; ++}; ++ ++#define INTEL_GVT_MMIO_HASH_BITS 11 ++ ++struct intel_gvt_mmio { ++ u8 *mmio_attribute; ++/* Register contains RO bits */ ++#define F_RO (1 << 0) ++/* Register contains graphics address */ ++#define F_GMADR (1 << 1) ++/* Mode mask registers with high 16 bits as the mask bits */ ++#define F_MODE_MASK (1 << 2) ++/* This reg can be accessed by GPU commands */ ++#define F_CMD_ACCESS (1 << 3) ++/* This reg has been accessed by a VM */ ++#define F_ACCESSED (1 << 4) ++/* This reg has been accessed through GPU commands */ ++#define F_CMD_ACCESSED (1 << 5) ++/* This reg could be accessed by unaligned address */ ++#define F_UNALIGN (1 << 6) ++/* This reg is saved/restored in context */ ++#define F_IN_CTX (1 << 7) ++ ++ struct gvt_mmio_block *mmio_block; ++ unsigned int num_mmio_block; ++ ++ DECLARE_HASHTABLE(mmio_info_table, INTEL_GVT_MMIO_HASH_BITS); ++ unsigned long num_tracked_mmio; ++}; ++ ++struct intel_gvt_firmware { ++ void *cfg_space; ++ void *mmio; ++ bool firmware_loaded; ++}; ++ ++#define NR_MAX_INTEL_VGPU_TYPES 20 ++struct intel_vgpu_type { ++ char name[16]; ++ unsigned int avail_instance; ++ unsigned int low_gm_size; ++ unsigned int high_gm_size; ++ unsigned int fence; ++ unsigned int weight; ++ enum intel_vgpu_edid resolution; ++}; ++ ++struct intel_gvt { ++ /* GVT scope lock, protect GVT itself, and all resource currently ++ * not yet protected by special locks(vgpu and scheduler lock). ++ */ ++ struct mutex lock; ++ /* scheduler scope lock, protect gvt and vgpu schedule related data */ ++ struct mutex sched_lock; ++ ++ struct drm_i915_private *dev_priv; ++ struct idr vgpu_idr; /* vGPU IDR pool */ ++ ++ struct intel_gvt_device_info device_info; ++ struct intel_gvt_gm gm; ++ struct intel_gvt_fence fence; ++ struct intel_gvt_mmio mmio; ++ struct intel_gvt_firmware firmware; ++ struct intel_gvt_irq irq; ++ struct intel_gvt_gtt gtt; ++ struct intel_gvt_workload_scheduler scheduler; ++ struct notifier_block shadow_ctx_notifier_block[I915_NUM_ENGINES]; ++ DECLARE_HASHTABLE(cmd_table, GVT_CMD_HASH_BITS); ++ struct intel_vgpu_type *types; ++ unsigned int num_types; ++ struct intel_vgpu *idle_vgpu; ++ ++ struct task_struct *service_thread; ++ wait_queue_head_t service_thread_wq; ++ ++ /* service_request is always used in bit operation, we should always ++ * use it with atomic bit ops so that no need to use gvt big lock. ++ */ ++ unsigned long service_request; ++ ++ struct { ++ struct engine_mmio *mmio; ++ int ctx_mmio_count[I915_NUM_ENGINES]; ++ } engine_mmio_list; ++ ++ struct dentry *debugfs_root; ++}; ++ ++static inline struct intel_gvt *to_gvt(struct drm_i915_private *i915) ++{ ++ return i915->gvt; ++} ++ ++enum { ++ INTEL_GVT_REQUEST_EMULATE_VBLANK = 0, ++ ++ /* Scheduling trigger by timer */ ++ INTEL_GVT_REQUEST_SCHED = 1, ++ ++ /* Scheduling trigger by event */ ++ INTEL_GVT_REQUEST_EVENT_SCHED = 2, ++}; ++ ++static inline void intel_gvt_request_service(struct intel_gvt *gvt, ++ int service) ++{ ++ set_bit(service, (void *)&gvt->service_request); ++ wake_up(&gvt->service_thread_wq); ++} ++ ++void intel_gvt_free_firmware(struct intel_gvt *gvt); ++int intel_gvt_load_firmware(struct intel_gvt *gvt); ++ ++/* Aperture/GM space definitions for GVT device */ ++#define MB_TO_BYTES(mb) ((mb) << 20ULL) ++#define BYTES_TO_MB(b) ((b) >> 20ULL) ++ ++#define HOST_LOW_GM_SIZE MB_TO_BYTES(128) ++#define HOST_HIGH_GM_SIZE MB_TO_BYTES(384) ++#define HOST_FENCE 4 ++ ++/* Aperture/GM space definitions for GVT device */ ++#define gvt_aperture_sz(gvt) (gvt->dev_priv->ggtt.mappable_end) ++#define gvt_aperture_pa_base(gvt) (gvt->dev_priv->ggtt.gmadr.start) ++ ++#define gvt_ggtt_gm_sz(gvt) (gvt->dev_priv->ggtt.vm.total) ++#define gvt_ggtt_sz(gvt) \ ++ ((gvt->dev_priv->ggtt.vm.total >> PAGE_SHIFT) << 3) ++#define gvt_hidden_sz(gvt) (gvt_ggtt_gm_sz(gvt) - gvt_aperture_sz(gvt)) ++ ++#define gvt_aperture_gmadr_base(gvt) (0) ++#define gvt_aperture_gmadr_end(gvt) (gvt_aperture_gmadr_base(gvt) \ ++ + gvt_aperture_sz(gvt) - 1) ++ ++#define gvt_hidden_gmadr_base(gvt) (gvt_aperture_gmadr_base(gvt) \ ++ + gvt_aperture_sz(gvt)) ++#define gvt_hidden_gmadr_end(gvt) (gvt_hidden_gmadr_base(gvt) \ ++ + gvt_hidden_sz(gvt) - 1) ++ ++#define gvt_fence_sz(gvt) (gvt->dev_priv->num_fence_regs) ++ ++/* Aperture/GM space definitions for vGPU */ ++#define vgpu_aperture_offset(vgpu) ((vgpu)->gm.low_gm_node.start) ++#define vgpu_hidden_offset(vgpu) ((vgpu)->gm.high_gm_node.start) ++#define vgpu_aperture_sz(vgpu) ((vgpu)->gm.aperture_sz) ++#define vgpu_hidden_sz(vgpu) ((vgpu)->gm.hidden_sz) ++ ++#define vgpu_aperture_pa_base(vgpu) \ ++ (gvt_aperture_pa_base(vgpu->gvt) + vgpu_aperture_offset(vgpu)) ++ ++#define vgpu_ggtt_gm_sz(vgpu) ((vgpu)->gm.aperture_sz + (vgpu)->gm.hidden_sz) ++ ++#define vgpu_aperture_pa_end(vgpu) \ ++ (vgpu_aperture_pa_base(vgpu) + vgpu_aperture_sz(vgpu) - 1) ++ ++#define vgpu_aperture_gmadr_base(vgpu) (vgpu_aperture_offset(vgpu)) ++#define vgpu_aperture_gmadr_end(vgpu) \ ++ (vgpu_aperture_gmadr_base(vgpu) + vgpu_aperture_sz(vgpu) - 1) ++ ++#define vgpu_hidden_gmadr_base(vgpu) (vgpu_hidden_offset(vgpu)) ++#define vgpu_hidden_gmadr_end(vgpu) \ ++ (vgpu_hidden_gmadr_base(vgpu) + vgpu_hidden_sz(vgpu) - 1) ++ ++#define vgpu_fence_base(vgpu) (vgpu->fence.base) ++#define vgpu_fence_sz(vgpu) (vgpu->fence.size) ++ ++struct intel_vgpu_creation_params { ++ __u64 handle; ++ __u64 low_gm_sz; /* in MB */ ++ __u64 high_gm_sz; /* in MB */ ++ __u64 fence_sz; ++ __u64 resolution; ++ __s32 primary; ++ __u64 vgpu_id; ++ ++ __u32 weight; ++}; ++ ++int intel_vgpu_alloc_resource(struct intel_vgpu *vgpu, ++ struct intel_vgpu_creation_params *param); ++void intel_vgpu_reset_resource(struct intel_vgpu *vgpu); ++void intel_vgpu_free_resource(struct intel_vgpu *vgpu); ++void intel_vgpu_write_fence(struct intel_vgpu *vgpu, ++ u32 fence, u64 value); ++ ++/* Macros for easily accessing vGPU virtual/shadow register. ++ Explicitly seperate use for typed MMIO reg or real offset.*/ ++#define vgpu_vreg_t(vgpu, reg) \ ++ (*(u32 *)(vgpu->mmio.vreg + i915_mmio_reg_offset(reg))) ++#define vgpu_vreg(vgpu, offset) \ ++ (*(u32 *)(vgpu->mmio.vreg + (offset))) ++#define vgpu_vreg64_t(vgpu, reg) \ ++ (*(u64 *)(vgpu->mmio.vreg + i915_mmio_reg_offset(reg))) ++#define vgpu_vreg64(vgpu, offset) \ ++ (*(u64 *)(vgpu->mmio.vreg + (offset))) ++ ++#define for_each_active_vgpu(gvt, vgpu, id) \ ++ idr_for_each_entry((&(gvt)->vgpu_idr), (vgpu), (id)) \ ++ for_each_if(vgpu->active) ++ ++static inline void intel_vgpu_write_pci_bar(struct intel_vgpu *vgpu, ++ u32 offset, u32 val, bool low) ++{ ++ u32 *pval; ++ ++ /* BAR offset should be 32 bits algiend */ ++ offset = rounddown(offset, 4); ++ pval = (u32 *)(vgpu_cfg_space(vgpu) + offset); ++ ++ if (low) { ++ /* ++ * only update bit 31 - bit 4, ++ * leave the bit 3 - bit 0 unchanged. ++ */ ++ *pval = (val & GENMASK(31, 4)) | (*pval & GENMASK(3, 0)); ++ } else { ++ *pval = val; ++ } ++} ++ ++int intel_gvt_init_vgpu_types(struct intel_gvt *gvt); ++void intel_gvt_clean_vgpu_types(struct intel_gvt *gvt); ++ ++struct intel_vgpu *intel_gvt_create_idle_vgpu(struct intel_gvt *gvt); ++void intel_gvt_destroy_idle_vgpu(struct intel_vgpu *vgpu); ++struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt, ++ struct intel_vgpu_type *type); ++void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu); ++void intel_gvt_release_vgpu(struct intel_vgpu *vgpu); ++void intel_gvt_reset_vgpu_locked(struct intel_vgpu *vgpu, bool dmlr, ++ intel_engine_mask_t engine_mask); ++void intel_gvt_reset_vgpu(struct intel_vgpu *vgpu); ++void intel_gvt_activate_vgpu(struct intel_vgpu *vgpu); ++void intel_gvt_deactivate_vgpu(struct intel_vgpu *vgpu); ++ ++/* validating GM functions */ ++#define vgpu_gmadr_is_aperture(vgpu, gmadr) \ ++ ((gmadr >= vgpu_aperture_gmadr_base(vgpu)) && \ ++ (gmadr <= vgpu_aperture_gmadr_end(vgpu))) ++ ++#define vgpu_gmadr_is_hidden(vgpu, gmadr) \ ++ ((gmadr >= vgpu_hidden_gmadr_base(vgpu)) && \ ++ (gmadr <= vgpu_hidden_gmadr_end(vgpu))) ++ ++#define vgpu_gmadr_is_valid(vgpu, gmadr) \ ++ ((vgpu_gmadr_is_aperture(vgpu, gmadr) || \ ++ (vgpu_gmadr_is_hidden(vgpu, gmadr)))) ++ ++#define gvt_gmadr_is_aperture(gvt, gmadr) \ ++ ((gmadr >= gvt_aperture_gmadr_base(gvt)) && \ ++ (gmadr <= gvt_aperture_gmadr_end(gvt))) ++ ++#define gvt_gmadr_is_hidden(gvt, gmadr) \ ++ ((gmadr >= gvt_hidden_gmadr_base(gvt)) && \ ++ (gmadr <= gvt_hidden_gmadr_end(gvt))) ++ ++#define gvt_gmadr_is_valid(gvt, gmadr) \ ++ (gvt_gmadr_is_aperture(gvt, gmadr) || \ ++ gvt_gmadr_is_hidden(gvt, gmadr)) ++ ++bool intel_gvt_ggtt_validate_range(struct intel_vgpu *vgpu, u64 addr, u32 size); ++int intel_gvt_ggtt_gmadr_g2h(struct intel_vgpu *vgpu, u64 g_addr, u64 *h_addr); ++int intel_gvt_ggtt_gmadr_h2g(struct intel_vgpu *vgpu, u64 h_addr, u64 *g_addr); ++int intel_gvt_ggtt_index_g2h(struct intel_vgpu *vgpu, unsigned long g_index, ++ unsigned long *h_index); ++int intel_gvt_ggtt_h2g_index(struct intel_vgpu *vgpu, unsigned long h_index, ++ unsigned long *g_index); ++ ++void intel_vgpu_init_cfg_space(struct intel_vgpu *vgpu, ++ bool primary); ++void intel_vgpu_reset_cfg_space(struct intel_vgpu *vgpu); ++ ++int intel_vgpu_emulate_cfg_read(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes); ++ ++int intel_vgpu_emulate_cfg_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes); ++ ++void intel_vgpu_emulate_hotplug(struct intel_vgpu *vgpu, bool connected); ++ ++static inline u64 intel_vgpu_get_bar_gpa(struct intel_vgpu *vgpu, int bar) ++{ ++ /* We are 64bit bar. */ ++ return (*(u64 *)(vgpu->cfg_space.virtual_cfg_space + bar)) & ++ PCI_BASE_ADDRESS_MEM_MASK; ++} ++ ++void intel_vgpu_clean_opregion(struct intel_vgpu *vgpu); ++int intel_vgpu_init_opregion(struct intel_vgpu *vgpu); ++int intel_vgpu_opregion_base_write_handler(struct intel_vgpu *vgpu, u32 gpa); ++ ++int intel_vgpu_emulate_opregion_request(struct intel_vgpu *vgpu, u32 swsci); ++void populate_pvinfo_page(struct intel_vgpu *vgpu); ++ ++int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload); ++void enter_failsafe_mode(struct intel_vgpu *vgpu, int reason); ++ ++struct intel_gvt_ops { ++ int (*emulate_cfg_read)(struct intel_vgpu *, unsigned int, void *, ++ unsigned int); ++ int (*emulate_cfg_write)(struct intel_vgpu *, unsigned int, void *, ++ unsigned int); ++ int (*emulate_mmio_read)(struct intel_vgpu *, u64, void *, ++ unsigned int); ++ int (*emulate_mmio_write)(struct intel_vgpu *, u64, void *, ++ unsigned int); ++ struct intel_vgpu *(*vgpu_create)(struct intel_gvt *, ++ struct intel_vgpu_type *); ++ void (*vgpu_destroy)(struct intel_vgpu *vgpu); ++ void (*vgpu_release)(struct intel_vgpu *vgpu); ++ void (*vgpu_reset)(struct intel_vgpu *); ++ void (*vgpu_activate)(struct intel_vgpu *); ++ void (*vgpu_deactivate)(struct intel_vgpu *); ++ struct intel_vgpu_type *(*gvt_find_vgpu_type)(struct intel_gvt *gvt, ++ const char *name); ++ bool (*get_gvt_attrs)(struct attribute ***type_attrs, ++ struct attribute_group ***intel_vgpu_type_groups); ++ int (*vgpu_query_plane)(struct intel_vgpu *vgpu, void *); ++ int (*vgpu_get_dmabuf)(struct intel_vgpu *vgpu, unsigned int); ++ int (*write_protect_handler)(struct intel_vgpu *, u64, void *, ++ unsigned int); ++ void (*emulate_hotplug)(struct intel_vgpu *vgpu, bool connected); ++}; ++ ++ ++enum { ++ GVT_FAILSAFE_UNSUPPORTED_GUEST, ++ GVT_FAILSAFE_INSUFFICIENT_RESOURCE, ++ GVT_FAILSAFE_GUEST_ERR, ++}; ++ ++static inline void mmio_hw_access_pre(struct drm_i915_private *dev_priv) ++{ ++ intel_runtime_pm_get(dev_priv); ++} ++ ++static inline void mmio_hw_access_post(struct drm_i915_private *dev_priv) ++{ ++ intel_runtime_pm_put_unchecked(dev_priv); ++} ++ ++/** ++ * intel_gvt_mmio_set_accessed - mark a MMIO has been accessed ++ * @gvt: a GVT device ++ * @offset: register offset ++ * ++ */ ++static inline void intel_gvt_mmio_set_accessed( ++ struct intel_gvt *gvt, unsigned int offset) ++{ ++ gvt->mmio.mmio_attribute[offset >> 2] |= F_ACCESSED; ++} ++ ++/** ++ * intel_gvt_mmio_is_cmd_accessed - mark a MMIO could be accessed by command ++ * @gvt: a GVT device ++ * @offset: register offset ++ * ++ */ ++static inline bool intel_gvt_mmio_is_cmd_access( ++ struct intel_gvt *gvt, unsigned int offset) ++{ ++ return gvt->mmio.mmio_attribute[offset >> 2] & F_CMD_ACCESS; ++} ++ ++/** ++ * intel_gvt_mmio_is_unalign - mark a MMIO could be accessed unaligned ++ * @gvt: a GVT device ++ * @offset: register offset ++ * ++ */ ++static inline bool intel_gvt_mmio_is_unalign( ++ struct intel_gvt *gvt, unsigned int offset) ++{ ++ return gvt->mmio.mmio_attribute[offset >> 2] & F_UNALIGN; ++} ++ ++/** ++ * intel_gvt_mmio_set_cmd_accessed - mark a MMIO has been accessed by command ++ * @gvt: a GVT device ++ * @offset: register offset ++ * ++ */ ++static inline void intel_gvt_mmio_set_cmd_accessed( ++ struct intel_gvt *gvt, unsigned int offset) ++{ ++ gvt->mmio.mmio_attribute[offset >> 2] |= F_CMD_ACCESSED; ++} ++ ++/** ++ * intel_gvt_mmio_has_mode_mask - if a MMIO has a mode mask ++ * @gvt: a GVT device ++ * @offset: register offset ++ * ++ * Returns: ++ * True if a MMIO has a mode mask in its higher 16 bits, false if it isn't. ++ * ++ */ ++static inline bool intel_gvt_mmio_has_mode_mask( ++ struct intel_gvt *gvt, unsigned int offset) ++{ ++ return gvt->mmio.mmio_attribute[offset >> 2] & F_MODE_MASK; ++} ++ ++/** ++ * intel_gvt_mmio_is_in_ctx - check if a MMIO has in-ctx mask ++ * @gvt: a GVT device ++ * @offset: register offset ++ * ++ * Returns: ++ * True if a MMIO has a in-context mask, false if it isn't. ++ * ++ */ ++static inline bool intel_gvt_mmio_is_in_ctx( ++ struct intel_gvt *gvt, unsigned int offset) ++{ ++ return gvt->mmio.mmio_attribute[offset >> 2] & F_IN_CTX; ++} ++ ++/** ++ * intel_gvt_mmio_set_in_ctx - mask a MMIO in logical context ++ * @gvt: a GVT device ++ * @offset: register offset ++ * ++ */ ++static inline void intel_gvt_mmio_set_in_ctx( ++ struct intel_gvt *gvt, unsigned int offset) ++{ ++ gvt->mmio.mmio_attribute[offset >> 2] |= F_IN_CTX; ++} ++ ++int intel_gvt_debugfs_add_vgpu(struct intel_vgpu *vgpu); ++void intel_gvt_debugfs_remove_vgpu(struct intel_vgpu *vgpu); ++int intel_gvt_debugfs_init(struct intel_gvt *gvt); ++void intel_gvt_debugfs_clean(struct intel_gvt *gvt); ++ ++ ++#include "trace.h" ++#include "mpt.h" ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/gvt/handlers.c b/drivers/gpu/drm/i915_legacy/gvt/handlers.c +new file mode 100644 +index 000000000000..25f78196b964 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/handlers.c +@@ -0,0 +1,3588 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Kevin Tian ++ * Eddie Dong ++ * Zhiyuan Lv ++ * ++ * Contributors: ++ * Min He ++ * Tina Zhang ++ * Pei Zhang ++ * Niu Bing ++ * Ping Gao ++ * Zhi Wang ++ * ++ ++ */ ++ ++#include "i915_drv.h" ++#include "gvt.h" ++#include "i915_pvinfo.h" ++ ++/* XXX FIXME i915 has changed PP_XXX definition */ ++#define PCH_PP_STATUS _MMIO(0xc7200) ++#define PCH_PP_CONTROL _MMIO(0xc7204) ++#define PCH_PP_ON_DELAYS _MMIO(0xc7208) ++#define PCH_PP_OFF_DELAYS _MMIO(0xc720c) ++#define PCH_PP_DIVISOR _MMIO(0xc7210) ++ ++unsigned long intel_gvt_get_device_type(struct intel_gvt *gvt) ++{ ++ if (IS_BROADWELL(gvt->dev_priv)) ++ return D_BDW; ++ else if (IS_SKYLAKE(gvt->dev_priv)) ++ return D_SKL; ++ else if (IS_KABYLAKE(gvt->dev_priv)) ++ return D_KBL; ++ else if (IS_BROXTON(gvt->dev_priv)) ++ return D_BXT; ++ else if (IS_COFFEELAKE(gvt->dev_priv)) ++ return D_CFL; ++ ++ return 0; ++} ++ ++bool intel_gvt_match_device(struct intel_gvt *gvt, ++ unsigned long device) ++{ ++ return intel_gvt_get_device_type(gvt) & device; ++} ++ ++static void read_vreg(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ memcpy(p_data, &vgpu_vreg(vgpu, offset), bytes); ++} ++ ++static void write_vreg(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ memcpy(&vgpu_vreg(vgpu, offset), p_data, bytes); ++} ++ ++static struct intel_gvt_mmio_info *find_mmio_info(struct intel_gvt *gvt, ++ unsigned int offset) ++{ ++ struct intel_gvt_mmio_info *e; ++ ++ hash_for_each_possible(gvt->mmio.mmio_info_table, e, node, offset) { ++ if (e->offset == offset) ++ return e; ++ } ++ return NULL; ++} ++ ++static int new_mmio_info(struct intel_gvt *gvt, ++ u32 offset, u8 flags, u32 size, ++ u32 addr_mask, u32 ro_mask, u32 device, ++ gvt_mmio_func read, gvt_mmio_func write) ++{ ++ struct intel_gvt_mmio_info *info, *p; ++ u32 start, end, i; ++ ++ if (!intel_gvt_match_device(gvt, device)) ++ return 0; ++ ++ if (WARN_ON(!IS_ALIGNED(offset, 4))) ++ return -EINVAL; ++ ++ start = offset; ++ end = offset + size; ++ ++ for (i = start; i < end; i += 4) { ++ info = kzalloc(sizeof(*info), GFP_KERNEL); ++ if (!info) ++ return -ENOMEM; ++ ++ info->offset = i; ++ p = find_mmio_info(gvt, info->offset); ++ if (p) { ++ WARN(1, "dup mmio definition offset %x\n", ++ info->offset); ++ kfree(info); ++ ++ /* We return -EEXIST here to make GVT-g load fail. ++ * So duplicated MMIO can be found as soon as ++ * possible. ++ */ ++ return -EEXIST; ++ } ++ ++ info->ro_mask = ro_mask; ++ info->device = device; ++ info->read = read ? read : intel_vgpu_default_mmio_read; ++ info->write = write ? write : intel_vgpu_default_mmio_write; ++ gvt->mmio.mmio_attribute[info->offset / 4] = flags; ++ INIT_HLIST_NODE(&info->node); ++ hash_add(gvt->mmio.mmio_info_table, &info->node, info->offset); ++ gvt->mmio.num_tracked_mmio++; ++ } ++ return 0; ++} ++ ++/** ++ * intel_gvt_render_mmio_to_ring_id - convert a mmio offset into ring id ++ * @gvt: a GVT device ++ * @offset: register offset ++ * ++ * Returns: ++ * Ring ID on success, negative error code if failed. ++ */ ++int intel_gvt_render_mmio_to_ring_id(struct intel_gvt *gvt, ++ unsigned int offset) ++{ ++ enum intel_engine_id id; ++ struct intel_engine_cs *engine; ++ ++ offset &= ~GENMASK(11, 0); ++ for_each_engine(engine, gvt->dev_priv, id) { ++ if (engine->mmio_base == offset) ++ return id; ++ } ++ return -ENODEV; ++} ++ ++#define offset_to_fence_num(offset) \ ++ ((offset - i915_mmio_reg_offset(FENCE_REG_GEN6_LO(0))) >> 3) ++ ++#define fence_num_to_offset(num) \ ++ (num * 8 + i915_mmio_reg_offset(FENCE_REG_GEN6_LO(0))) ++ ++ ++void enter_failsafe_mode(struct intel_vgpu *vgpu, int reason) ++{ ++ switch (reason) { ++ case GVT_FAILSAFE_UNSUPPORTED_GUEST: ++ pr_err("Detected your guest driver doesn't support GVT-g.\n"); ++ break; ++ case GVT_FAILSAFE_INSUFFICIENT_RESOURCE: ++ pr_err("Graphics resource is not enough for the guest\n"); ++ break; ++ case GVT_FAILSAFE_GUEST_ERR: ++ pr_err("GVT Internal error for the guest\n"); ++ break; ++ default: ++ break; ++ } ++ pr_err("Now vgpu %d will enter failsafe mode.\n", vgpu->id); ++ vgpu->failsafe = true; ++} ++ ++static int sanitize_fence_mmio_access(struct intel_vgpu *vgpu, ++ unsigned int fence_num, void *p_data, unsigned int bytes) ++{ ++ unsigned int max_fence = vgpu_fence_sz(vgpu); ++ ++ if (fence_num >= max_fence) { ++ gvt_vgpu_err("access oob fence reg %d/%d\n", ++ fence_num, max_fence); ++ ++ /* When guest access oob fence regs without access ++ * pv_info first, we treat guest not supporting GVT, ++ * and we will let vgpu enter failsafe mode. ++ */ ++ if (!vgpu->pv_notified) ++ enter_failsafe_mode(vgpu, ++ GVT_FAILSAFE_UNSUPPORTED_GUEST); ++ ++ memset(p_data, 0, bytes); ++ return -EINVAL; ++ } ++ return 0; ++} ++ ++static int gamw_echo_dev_rw_ia_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ u32 ips = (*(u32 *)p_data) & GAMW_ECO_ENABLE_64K_IPS_FIELD; ++ ++ if (INTEL_GEN(vgpu->gvt->dev_priv) <= 10) { ++ if (ips == GAMW_ECO_ENABLE_64K_IPS_FIELD) ++ gvt_dbg_core("vgpu%d: ips enabled\n", vgpu->id); ++ else if (!ips) ++ gvt_dbg_core("vgpu%d: ips disabled\n", vgpu->id); ++ else { ++ /* All engines must be enabled together for vGPU, ++ * since we don't know which engine the ppgtt will ++ * bind to when shadowing. ++ */ ++ gvt_vgpu_err("Unsupported IPS setting %x, cannot enable 64K gtt.\n", ++ ips); ++ return -EINVAL; ++ } ++ } ++ ++ write_vreg(vgpu, offset, p_data, bytes); ++ return 0; ++} ++ ++static int fence_mmio_read(struct intel_vgpu *vgpu, unsigned int off, ++ void *p_data, unsigned int bytes) ++{ ++ int ret; ++ ++ ret = sanitize_fence_mmio_access(vgpu, offset_to_fence_num(off), ++ p_data, bytes); ++ if (ret) ++ return ret; ++ read_vreg(vgpu, off, p_data, bytes); ++ return 0; ++} ++ ++static int fence_mmio_write(struct intel_vgpu *vgpu, unsigned int off, ++ void *p_data, unsigned int bytes) ++{ ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ unsigned int fence_num = offset_to_fence_num(off); ++ int ret; ++ ++ ret = sanitize_fence_mmio_access(vgpu, fence_num, p_data, bytes); ++ if (ret) ++ return ret; ++ write_vreg(vgpu, off, p_data, bytes); ++ ++ mmio_hw_access_pre(dev_priv); ++ intel_vgpu_write_fence(vgpu, fence_num, ++ vgpu_vreg64(vgpu, fence_num_to_offset(fence_num))); ++ mmio_hw_access_post(dev_priv); ++ return 0; ++} ++ ++#define CALC_MODE_MASK_REG(old, new) \ ++ (((new) & GENMASK(31, 16)) \ ++ | ((((old) & GENMASK(15, 0)) & ~((new) >> 16)) \ ++ | ((new) & ((new) >> 16)))) ++ ++static int mul_force_wake_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ u32 old, new; ++ u32 ack_reg_offset; ++ ++ old = vgpu_vreg(vgpu, offset); ++ new = CALC_MODE_MASK_REG(old, *(u32 *)p_data); ++ ++ if (INTEL_GEN(vgpu->gvt->dev_priv) >= 9) { ++ switch (offset) { ++ case FORCEWAKE_RENDER_GEN9_REG: ++ ack_reg_offset = FORCEWAKE_ACK_RENDER_GEN9_REG; ++ break; ++ case FORCEWAKE_BLITTER_GEN9_REG: ++ ack_reg_offset = FORCEWAKE_ACK_BLITTER_GEN9_REG; ++ break; ++ case FORCEWAKE_MEDIA_GEN9_REG: ++ ack_reg_offset = FORCEWAKE_ACK_MEDIA_GEN9_REG; ++ break; ++ default: ++ /*should not hit here*/ ++ gvt_vgpu_err("invalid forcewake offset 0x%x\n", offset); ++ return -EINVAL; ++ } ++ } else { ++ ack_reg_offset = FORCEWAKE_ACK_HSW_REG; ++ } ++ ++ vgpu_vreg(vgpu, offset) = new; ++ vgpu_vreg(vgpu, ack_reg_offset) = (new & GENMASK(15, 0)); ++ return 0; ++} ++ ++static int gdrst_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ intel_engine_mask_t engine_mask = 0; ++ u32 data; ++ ++ write_vreg(vgpu, offset, p_data, bytes); ++ data = vgpu_vreg(vgpu, offset); ++ ++ if (data & GEN6_GRDOM_FULL) { ++ gvt_dbg_mmio("vgpu%d: request full GPU reset\n", vgpu->id); ++ engine_mask = ALL_ENGINES; ++ } else { ++ if (data & GEN6_GRDOM_RENDER) { ++ gvt_dbg_mmio("vgpu%d: request RCS reset\n", vgpu->id); ++ engine_mask |= BIT(RCS0); ++ } ++ if (data & GEN6_GRDOM_MEDIA) { ++ gvt_dbg_mmio("vgpu%d: request VCS reset\n", vgpu->id); ++ engine_mask |= BIT(VCS0); ++ } ++ if (data & GEN6_GRDOM_BLT) { ++ gvt_dbg_mmio("vgpu%d: request BCS Reset\n", vgpu->id); ++ engine_mask |= BIT(BCS0); ++ } ++ if (data & GEN6_GRDOM_VECS) { ++ gvt_dbg_mmio("vgpu%d: request VECS Reset\n", vgpu->id); ++ engine_mask |= BIT(VECS0); ++ } ++ if (data & GEN8_GRDOM_MEDIA2) { ++ gvt_dbg_mmio("vgpu%d: request VCS2 Reset\n", vgpu->id); ++ engine_mask |= BIT(VCS1); ++ } ++ engine_mask &= INTEL_INFO(vgpu->gvt->dev_priv)->engine_mask; ++ } ++ ++ /* vgpu_lock already hold by emulate mmio r/w */ ++ intel_gvt_reset_vgpu_locked(vgpu, false, engine_mask); ++ ++ /* sw will wait for the device to ack the reset request */ ++ vgpu_vreg(vgpu, offset) = 0; ++ ++ return 0; ++} ++ ++static int gmbus_mmio_read(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ return intel_gvt_i2c_handle_gmbus_read(vgpu, offset, p_data, bytes); ++} ++ ++static int gmbus_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ return intel_gvt_i2c_handle_gmbus_write(vgpu, offset, p_data, bytes); ++} ++ ++static int pch_pp_control_mmio_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ write_vreg(vgpu, offset, p_data, bytes); ++ ++ if (vgpu_vreg(vgpu, offset) & PANEL_POWER_ON) { ++ vgpu_vreg_t(vgpu, PCH_PP_STATUS) |= PP_ON; ++ vgpu_vreg_t(vgpu, PCH_PP_STATUS) |= PP_SEQUENCE_STATE_ON_IDLE; ++ vgpu_vreg_t(vgpu, PCH_PP_STATUS) &= ~PP_SEQUENCE_POWER_DOWN; ++ vgpu_vreg_t(vgpu, PCH_PP_STATUS) &= ~PP_CYCLE_DELAY_ACTIVE; ++ ++ } else ++ vgpu_vreg_t(vgpu, PCH_PP_STATUS) &= ++ ~(PP_ON | PP_SEQUENCE_POWER_DOWN ++ | PP_CYCLE_DELAY_ACTIVE); ++ return 0; ++} ++ ++static int transconf_mmio_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ write_vreg(vgpu, offset, p_data, bytes); ++ ++ if (vgpu_vreg(vgpu, offset) & TRANS_ENABLE) ++ vgpu_vreg(vgpu, offset) |= TRANS_STATE_ENABLE; ++ else ++ vgpu_vreg(vgpu, offset) &= ~TRANS_STATE_ENABLE; ++ return 0; ++} ++ ++static int lcpll_ctl_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ write_vreg(vgpu, offset, p_data, bytes); ++ ++ if (vgpu_vreg(vgpu, offset) & LCPLL_PLL_DISABLE) ++ vgpu_vreg(vgpu, offset) &= ~LCPLL_PLL_LOCK; ++ else ++ vgpu_vreg(vgpu, offset) |= LCPLL_PLL_LOCK; ++ ++ if (vgpu_vreg(vgpu, offset) & LCPLL_CD_SOURCE_FCLK) ++ vgpu_vreg(vgpu, offset) |= LCPLL_CD_SOURCE_FCLK_DONE; ++ else ++ vgpu_vreg(vgpu, offset) &= ~LCPLL_CD_SOURCE_FCLK_DONE; ++ ++ return 0; ++} ++ ++static int dpy_reg_mmio_read(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ switch (offset) { ++ case 0xe651c: ++ case 0xe661c: ++ case 0xe671c: ++ case 0xe681c: ++ vgpu_vreg(vgpu, offset) = 1 << 17; ++ break; ++ case 0xe6c04: ++ vgpu_vreg(vgpu, offset) = 0x3; ++ break; ++ case 0xe6e1c: ++ vgpu_vreg(vgpu, offset) = 0x2f << 16; ++ break; ++ default: ++ return -EINVAL; ++ } ++ ++ read_vreg(vgpu, offset, p_data, bytes); ++ return 0; ++} ++ ++static int pipeconf_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ u32 data; ++ ++ write_vreg(vgpu, offset, p_data, bytes); ++ data = vgpu_vreg(vgpu, offset); ++ ++ if (data & PIPECONF_ENABLE) ++ vgpu_vreg(vgpu, offset) |= I965_PIPECONF_ACTIVE; ++ else ++ vgpu_vreg(vgpu, offset) &= ~I965_PIPECONF_ACTIVE; ++ /* vgpu_lock already hold by emulate mmio r/w */ ++ mutex_unlock(&vgpu->vgpu_lock); ++ intel_gvt_check_vblank_emulation(vgpu->gvt); ++ mutex_lock(&vgpu->vgpu_lock); ++ return 0; ++} ++ ++/* ascendingly sorted */ ++static i915_reg_t force_nonpriv_white_list[] = { ++ GEN9_CS_DEBUG_MODE1, //_MMIO(0x20ec) ++ GEN9_CTX_PREEMPT_REG,//_MMIO(0x2248) ++ GEN8_CS_CHICKEN1,//_MMIO(0x2580) ++ _MMIO(0x2690), ++ _MMIO(0x2694), ++ _MMIO(0x2698), ++ _MMIO(0x2754), ++ _MMIO(0x28a0), ++ _MMIO(0x4de0), ++ _MMIO(0x4de4), ++ _MMIO(0x4dfc), ++ GEN7_COMMON_SLICE_CHICKEN1,//_MMIO(0x7010) ++ _MMIO(0x7014), ++ HDC_CHICKEN0,//_MMIO(0x7300) ++ GEN8_HDC_CHICKEN1,//_MMIO(0x7304) ++ _MMIO(0x7700), ++ _MMIO(0x7704), ++ _MMIO(0x7708), ++ _MMIO(0x770c), ++ _MMIO(0x83a8), ++ _MMIO(0xb110), ++ GEN8_L3SQCREG4,//_MMIO(0xb118) ++ _MMIO(0xe100), ++ _MMIO(0xe18c), ++ _MMIO(0xe48c), ++ _MMIO(0xe5f4), ++}; ++ ++/* a simple bsearch */ ++static inline bool in_whitelist(unsigned int reg) ++{ ++ int left = 0, right = ARRAY_SIZE(force_nonpriv_white_list); ++ i915_reg_t *array = force_nonpriv_white_list; ++ ++ while (left < right) { ++ int mid = (left + right)/2; ++ ++ if (reg > array[mid].reg) ++ left = mid + 1; ++ else if (reg < array[mid].reg) ++ right = mid; ++ else ++ return true; ++ } ++ return false; ++} ++ ++static int force_nonpriv_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ u32 reg_nonpriv = *(u32 *)p_data; ++ int ring_id = intel_gvt_render_mmio_to_ring_id(vgpu->gvt, offset); ++ u32 ring_base; ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ int ret = -EINVAL; ++ ++ if ((bytes != 4) || ((offset & (bytes - 1)) != 0) || ring_id < 0) { ++ gvt_err("vgpu(%d) ring %d Invalid FORCE_NONPRIV offset %x(%dB)\n", ++ vgpu->id, ring_id, offset, bytes); ++ return ret; ++ } ++ ++ ring_base = dev_priv->engine[ring_id]->mmio_base; ++ ++ if (in_whitelist(reg_nonpriv) || ++ reg_nonpriv == i915_mmio_reg_offset(RING_NOPID(ring_base))) { ++ ret = intel_vgpu_default_mmio_write(vgpu, offset, p_data, ++ bytes); ++ } else ++ gvt_err("vgpu(%d) Invalid FORCE_NONPRIV write %x at offset %x\n", ++ vgpu->id, reg_nonpriv, offset); ++ ++ return 0; ++} ++ ++static int ddi_buf_ctl_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ write_vreg(vgpu, offset, p_data, bytes); ++ ++ if (vgpu_vreg(vgpu, offset) & DDI_BUF_CTL_ENABLE) { ++ vgpu_vreg(vgpu, offset) &= ~DDI_BUF_IS_IDLE; ++ } else { ++ vgpu_vreg(vgpu, offset) |= DDI_BUF_IS_IDLE; ++ if (offset == i915_mmio_reg_offset(DDI_BUF_CTL(PORT_E))) ++ vgpu_vreg_t(vgpu, DP_TP_STATUS(PORT_E)) ++ &= ~DP_TP_STATUS_AUTOTRAIN_DONE; ++ } ++ return 0; ++} ++ ++static int fdi_rx_iir_mmio_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ vgpu_vreg(vgpu, offset) &= ~*(u32 *)p_data; ++ return 0; ++} ++ ++#define FDI_LINK_TRAIN_PATTERN1 0 ++#define FDI_LINK_TRAIN_PATTERN2 1 ++ ++static int fdi_auto_training_started(struct intel_vgpu *vgpu) ++{ ++ u32 ddi_buf_ctl = vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_E)); ++ u32 rx_ctl = vgpu_vreg(vgpu, _FDI_RXA_CTL); ++ u32 tx_ctl = vgpu_vreg_t(vgpu, DP_TP_CTL(PORT_E)); ++ ++ if ((ddi_buf_ctl & DDI_BUF_CTL_ENABLE) && ++ (rx_ctl & FDI_RX_ENABLE) && ++ (rx_ctl & FDI_AUTO_TRAINING) && ++ (tx_ctl & DP_TP_CTL_ENABLE) && ++ (tx_ctl & DP_TP_CTL_FDI_AUTOTRAIN)) ++ return 1; ++ else ++ return 0; ++} ++ ++static int check_fdi_rx_train_status(struct intel_vgpu *vgpu, ++ enum pipe pipe, unsigned int train_pattern) ++{ ++ i915_reg_t fdi_rx_imr, fdi_tx_ctl, fdi_rx_ctl; ++ unsigned int fdi_rx_check_bits, fdi_tx_check_bits; ++ unsigned int fdi_rx_train_bits, fdi_tx_train_bits; ++ unsigned int fdi_iir_check_bits; ++ ++ fdi_rx_imr = FDI_RX_IMR(pipe); ++ fdi_tx_ctl = FDI_TX_CTL(pipe); ++ fdi_rx_ctl = FDI_RX_CTL(pipe); ++ ++ if (train_pattern == FDI_LINK_TRAIN_PATTERN1) { ++ fdi_rx_train_bits = FDI_LINK_TRAIN_PATTERN_1_CPT; ++ fdi_tx_train_bits = FDI_LINK_TRAIN_PATTERN_1; ++ fdi_iir_check_bits = FDI_RX_BIT_LOCK; ++ } else if (train_pattern == FDI_LINK_TRAIN_PATTERN2) { ++ fdi_rx_train_bits = FDI_LINK_TRAIN_PATTERN_2_CPT; ++ fdi_tx_train_bits = FDI_LINK_TRAIN_PATTERN_2; ++ fdi_iir_check_bits = FDI_RX_SYMBOL_LOCK; ++ } else { ++ gvt_vgpu_err("Invalid train pattern %d\n", train_pattern); ++ return -EINVAL; ++ } ++ ++ fdi_rx_check_bits = FDI_RX_ENABLE | fdi_rx_train_bits; ++ fdi_tx_check_bits = FDI_TX_ENABLE | fdi_tx_train_bits; ++ ++ /* If imr bit has been masked */ ++ if (vgpu_vreg_t(vgpu, fdi_rx_imr) & fdi_iir_check_bits) ++ return 0; ++ ++ if (((vgpu_vreg_t(vgpu, fdi_tx_ctl) & fdi_tx_check_bits) ++ == fdi_tx_check_bits) ++ && ((vgpu_vreg_t(vgpu, fdi_rx_ctl) & fdi_rx_check_bits) ++ == fdi_rx_check_bits)) ++ return 1; ++ else ++ return 0; ++} ++ ++#define INVALID_INDEX (~0U) ++ ++static unsigned int calc_index(unsigned int offset, unsigned int start, ++ unsigned int next, unsigned int end, i915_reg_t i915_end) ++{ ++ unsigned int range = next - start; ++ ++ if (!end) ++ end = i915_mmio_reg_offset(i915_end); ++ if (offset < start || offset > end) ++ return INVALID_INDEX; ++ offset -= start; ++ return offset / range; ++} ++ ++#define FDI_RX_CTL_TO_PIPE(offset) \ ++ calc_index(offset, _FDI_RXA_CTL, _FDI_RXB_CTL, 0, FDI_RX_CTL(PIPE_C)) ++ ++#define FDI_TX_CTL_TO_PIPE(offset) \ ++ calc_index(offset, _FDI_TXA_CTL, _FDI_TXB_CTL, 0, FDI_TX_CTL(PIPE_C)) ++ ++#define FDI_RX_IMR_TO_PIPE(offset) \ ++ calc_index(offset, _FDI_RXA_IMR, _FDI_RXB_IMR, 0, FDI_RX_IMR(PIPE_C)) ++ ++static int update_fdi_rx_iir_status(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ i915_reg_t fdi_rx_iir; ++ unsigned int index; ++ int ret; ++ ++ if (FDI_RX_CTL_TO_PIPE(offset) != INVALID_INDEX) ++ index = FDI_RX_CTL_TO_PIPE(offset); ++ else if (FDI_TX_CTL_TO_PIPE(offset) != INVALID_INDEX) ++ index = FDI_TX_CTL_TO_PIPE(offset); ++ else if (FDI_RX_IMR_TO_PIPE(offset) != INVALID_INDEX) ++ index = FDI_RX_IMR_TO_PIPE(offset); ++ else { ++ gvt_vgpu_err("Unsupport registers %x\n", offset); ++ return -EINVAL; ++ } ++ ++ write_vreg(vgpu, offset, p_data, bytes); ++ ++ fdi_rx_iir = FDI_RX_IIR(index); ++ ++ ret = check_fdi_rx_train_status(vgpu, index, FDI_LINK_TRAIN_PATTERN1); ++ if (ret < 0) ++ return ret; ++ if (ret) ++ vgpu_vreg_t(vgpu, fdi_rx_iir) |= FDI_RX_BIT_LOCK; ++ ++ ret = check_fdi_rx_train_status(vgpu, index, FDI_LINK_TRAIN_PATTERN2); ++ if (ret < 0) ++ return ret; ++ if (ret) ++ vgpu_vreg_t(vgpu, fdi_rx_iir) |= FDI_RX_SYMBOL_LOCK; ++ ++ if (offset == _FDI_RXA_CTL) ++ if (fdi_auto_training_started(vgpu)) ++ vgpu_vreg_t(vgpu, DP_TP_STATUS(PORT_E)) |= ++ DP_TP_STATUS_AUTOTRAIN_DONE; ++ return 0; ++} ++ ++#define DP_TP_CTL_TO_PORT(offset) \ ++ calc_index(offset, _DP_TP_CTL_A, _DP_TP_CTL_B, 0, DP_TP_CTL(PORT_E)) ++ ++static int dp_tp_ctl_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ i915_reg_t status_reg; ++ unsigned int index; ++ u32 data; ++ ++ write_vreg(vgpu, offset, p_data, bytes); ++ ++ index = DP_TP_CTL_TO_PORT(offset); ++ data = (vgpu_vreg(vgpu, offset) & GENMASK(10, 8)) >> 8; ++ if (data == 0x2) { ++ status_reg = DP_TP_STATUS(index); ++ vgpu_vreg_t(vgpu, status_reg) |= (1 << 25); ++ } ++ return 0; ++} ++ ++static int dp_tp_status_mmio_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ u32 reg_val; ++ u32 sticky_mask; ++ ++ reg_val = *((u32 *)p_data); ++ sticky_mask = GENMASK(27, 26) | (1 << 24); ++ ++ vgpu_vreg(vgpu, offset) = (reg_val & ~sticky_mask) | ++ (vgpu_vreg(vgpu, offset) & sticky_mask); ++ vgpu_vreg(vgpu, offset) &= ~(reg_val & sticky_mask); ++ return 0; ++} ++ ++static int pch_adpa_mmio_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ u32 data; ++ ++ write_vreg(vgpu, offset, p_data, bytes); ++ data = vgpu_vreg(vgpu, offset); ++ ++ if (data & ADPA_CRT_HOTPLUG_FORCE_TRIGGER) ++ vgpu_vreg(vgpu, offset) &= ~ADPA_CRT_HOTPLUG_FORCE_TRIGGER; ++ return 0; ++} ++ ++static int south_chicken2_mmio_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ u32 data; ++ ++ write_vreg(vgpu, offset, p_data, bytes); ++ data = vgpu_vreg(vgpu, offset); ++ ++ if (data & FDI_MPHY_IOSFSB_RESET_CTL) ++ vgpu_vreg(vgpu, offset) |= FDI_MPHY_IOSFSB_RESET_STATUS; ++ else ++ vgpu_vreg(vgpu, offset) &= ~FDI_MPHY_IOSFSB_RESET_STATUS; ++ return 0; ++} ++ ++#define DSPSURF_TO_PIPE(offset) \ ++ calc_index(offset, _DSPASURF, _DSPBSURF, 0, DSPSURF(PIPE_C)) ++ ++static int pri_surf_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ u32 pipe = DSPSURF_TO_PIPE(offset); ++ int event = SKL_FLIP_EVENT(pipe, PLANE_PRIMARY); ++ ++ write_vreg(vgpu, offset, p_data, bytes); ++ vgpu_vreg_t(vgpu, DSPSURFLIVE(pipe)) = vgpu_vreg(vgpu, offset); ++ ++ vgpu_vreg_t(vgpu, PIPE_FLIPCOUNT_G4X(pipe))++; ++ ++ if (vgpu_vreg_t(vgpu, DSPCNTR(pipe)) & PLANE_CTL_ASYNC_FLIP) ++ intel_vgpu_trigger_virtual_event(vgpu, event); ++ else ++ set_bit(event, vgpu->irq.flip_done_event[pipe]); ++ ++ return 0; ++} ++ ++#define SPRSURF_TO_PIPE(offset) \ ++ calc_index(offset, _SPRA_SURF, _SPRB_SURF, 0, SPRSURF(PIPE_C)) ++ ++static int spr_surf_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ u32 pipe = SPRSURF_TO_PIPE(offset); ++ int event = SKL_FLIP_EVENT(pipe, PLANE_SPRITE0); ++ ++ write_vreg(vgpu, offset, p_data, bytes); ++ vgpu_vreg_t(vgpu, SPRSURFLIVE(pipe)) = vgpu_vreg(vgpu, offset); ++ ++ if (vgpu_vreg_t(vgpu, SPRCTL(pipe)) & PLANE_CTL_ASYNC_FLIP) ++ intel_vgpu_trigger_virtual_event(vgpu, event); ++ else ++ set_bit(event, vgpu->irq.flip_done_event[pipe]); ++ ++ return 0; ++} ++ ++static int reg50080_mmio_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, ++ unsigned int bytes) ++{ ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ enum pipe pipe = REG_50080_TO_PIPE(offset); ++ enum plane_id plane = REG_50080_TO_PLANE(offset); ++ int event = SKL_FLIP_EVENT(pipe, plane); ++ ++ write_vreg(vgpu, offset, p_data, bytes); ++ if (plane == PLANE_PRIMARY) { ++ vgpu_vreg_t(vgpu, DSPSURFLIVE(pipe)) = vgpu_vreg(vgpu, offset); ++ vgpu_vreg_t(vgpu, PIPE_FLIPCOUNT_G4X(pipe))++; ++ } else { ++ vgpu_vreg_t(vgpu, SPRSURFLIVE(pipe)) = vgpu_vreg(vgpu, offset); ++ } ++ ++ if ((vgpu_vreg(vgpu, offset) & REG50080_FLIP_TYPE_MASK) == REG50080_FLIP_TYPE_ASYNC) ++ intel_vgpu_trigger_virtual_event(vgpu, event); ++ else ++ set_bit(event, vgpu->irq.flip_done_event[pipe]); ++ ++ return 0; ++} ++ ++static int trigger_aux_channel_interrupt(struct intel_vgpu *vgpu, ++ unsigned int reg) ++{ ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ enum intel_gvt_event_type event; ++ ++ if (reg == _DPA_AUX_CH_CTL) ++ event = AUX_CHANNEL_A; ++ else if (reg == _PCH_DPB_AUX_CH_CTL || reg == _DPB_AUX_CH_CTL) ++ event = AUX_CHANNEL_B; ++ else if (reg == _PCH_DPC_AUX_CH_CTL || reg == _DPC_AUX_CH_CTL) ++ event = AUX_CHANNEL_C; ++ else if (reg == _PCH_DPD_AUX_CH_CTL || reg == _DPD_AUX_CH_CTL) ++ event = AUX_CHANNEL_D; ++ else { ++ WARN_ON(true); ++ return -EINVAL; ++ } ++ ++ intel_vgpu_trigger_virtual_event(vgpu, event); ++ return 0; ++} ++ ++static int dp_aux_ch_ctl_trans_done(struct intel_vgpu *vgpu, u32 value, ++ unsigned int reg, int len, bool data_valid) ++{ ++ /* mark transaction done */ ++ value |= DP_AUX_CH_CTL_DONE; ++ value &= ~DP_AUX_CH_CTL_SEND_BUSY; ++ value &= ~DP_AUX_CH_CTL_RECEIVE_ERROR; ++ ++ if (data_valid) ++ value &= ~DP_AUX_CH_CTL_TIME_OUT_ERROR; ++ else ++ value |= DP_AUX_CH_CTL_TIME_OUT_ERROR; ++ ++ /* message size */ ++ value &= ~(0xf << 20); ++ value |= (len << 20); ++ vgpu_vreg(vgpu, reg) = value; ++ ++ if (value & DP_AUX_CH_CTL_INTERRUPT) ++ return trigger_aux_channel_interrupt(vgpu, reg); ++ return 0; ++} ++ ++static void dp_aux_ch_ctl_link_training(struct intel_vgpu_dpcd_data *dpcd, ++ u8 t) ++{ ++ if ((t & DPCD_TRAINING_PATTERN_SET_MASK) == DPCD_TRAINING_PATTERN_1) { ++ /* training pattern 1 for CR */ ++ /* set LANE0_CR_DONE, LANE1_CR_DONE */ ++ dpcd->data[DPCD_LANE0_1_STATUS] |= DPCD_LANES_CR_DONE; ++ /* set LANE2_CR_DONE, LANE3_CR_DONE */ ++ dpcd->data[DPCD_LANE2_3_STATUS] |= DPCD_LANES_CR_DONE; ++ } else if ((t & DPCD_TRAINING_PATTERN_SET_MASK) == ++ DPCD_TRAINING_PATTERN_2) { ++ /* training pattern 2 for EQ */ ++ /* Set CHANNEL_EQ_DONE and SYMBOL_LOCKED for Lane0_1 */ ++ dpcd->data[DPCD_LANE0_1_STATUS] |= DPCD_LANES_EQ_DONE; ++ dpcd->data[DPCD_LANE0_1_STATUS] |= DPCD_SYMBOL_LOCKED; ++ /* Set CHANNEL_EQ_DONE and SYMBOL_LOCKED for Lane2_3 */ ++ dpcd->data[DPCD_LANE2_3_STATUS] |= DPCD_LANES_EQ_DONE; ++ dpcd->data[DPCD_LANE2_3_STATUS] |= DPCD_SYMBOL_LOCKED; ++ /* set INTERLANE_ALIGN_DONE */ ++ dpcd->data[DPCD_LANE_ALIGN_STATUS_UPDATED] |= ++ DPCD_INTERLANE_ALIGN_DONE; ++ } else if ((t & DPCD_TRAINING_PATTERN_SET_MASK) == ++ DPCD_LINK_TRAINING_DISABLED) { ++ /* finish link training */ ++ /* set sink status as synchronized */ ++ dpcd->data[DPCD_SINK_STATUS] = DPCD_SINK_IN_SYNC; ++ } ++} ++ ++#define _REG_HSW_DP_AUX_CH_CTL(dp) \ ++ ((dp) ? (_PCH_DPB_AUX_CH_CTL + ((dp)-1)*0x100) : 0x64010) ++ ++#define _REG_SKL_DP_AUX_CH_CTL(dp) (0x64010 + (dp) * 0x100) ++ ++#define OFFSET_TO_DP_AUX_PORT(offset) (((offset) & 0xF00) >> 8) ++ ++#define dpy_is_valid_port(port) \ ++ (((port) >= PORT_A) && ((port) < I915_MAX_PORTS)) ++ ++static int dp_aux_ch_ctl_mmio_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ struct intel_vgpu_display *display = &vgpu->display; ++ int msg, addr, ctrl, op, len; ++ int port_index = OFFSET_TO_DP_AUX_PORT(offset); ++ struct intel_vgpu_dpcd_data *dpcd = NULL; ++ struct intel_vgpu_port *port = NULL; ++ u32 data; ++ ++ if (!dpy_is_valid_port(port_index)) { ++ gvt_vgpu_err("Unsupported DP port access!\n"); ++ return 0; ++ } ++ ++ write_vreg(vgpu, offset, p_data, bytes); ++ data = vgpu_vreg(vgpu, offset); ++ ++ if ((INTEL_GEN(vgpu->gvt->dev_priv) >= 9) ++ && offset != _REG_SKL_DP_AUX_CH_CTL(port_index)) { ++ /* SKL DPB/C/D aux ctl register changed */ ++ return 0; ++ } else if (IS_BROADWELL(vgpu->gvt->dev_priv) && ++ offset != _REG_HSW_DP_AUX_CH_CTL(port_index)) { ++ /* write to the data registers */ ++ return 0; ++ } ++ ++ if (!(data & DP_AUX_CH_CTL_SEND_BUSY)) { ++ /* just want to clear the sticky bits */ ++ vgpu_vreg(vgpu, offset) = 0; ++ return 0; ++ } ++ ++ port = &display->ports[port_index]; ++ dpcd = port->dpcd; ++ ++ /* read out message from DATA1 register */ ++ msg = vgpu_vreg(vgpu, offset + 4); ++ addr = (msg >> 8) & 0xffff; ++ ctrl = (msg >> 24) & 0xff; ++ len = msg & 0xff; ++ op = ctrl >> 4; ++ ++ if (op == GVT_AUX_NATIVE_WRITE) { ++ int t; ++ u8 buf[16]; ++ ++ if ((addr + len + 1) >= DPCD_SIZE) { ++ /* ++ * Write request exceeds what we supported, ++ * DCPD spec: When a Source Device is writing a DPCD ++ * address not supported by the Sink Device, the Sink ++ * Device shall reply with AUX NACK and “M” equal to ++ * zero. ++ */ ++ ++ /* NAK the write */ ++ vgpu_vreg(vgpu, offset + 4) = AUX_NATIVE_REPLY_NAK; ++ dp_aux_ch_ctl_trans_done(vgpu, data, offset, 2, true); ++ return 0; ++ } ++ ++ /* ++ * Write request format: Headr (command + address + size) occupies ++ * 4 bytes, followed by (len + 1) bytes of data. See details at ++ * intel_dp_aux_transfer(). ++ */ ++ if ((len + 1 + 4) > AUX_BURST_SIZE) { ++ gvt_vgpu_err("dp_aux_header: len %d is too large\n", len); ++ return -EINVAL; ++ } ++ ++ /* unpack data from vreg to buf */ ++ for (t = 0; t < 4; t++) { ++ u32 r = vgpu_vreg(vgpu, offset + 8 + t * 4); ++ ++ buf[t * 4] = (r >> 24) & 0xff; ++ buf[t * 4 + 1] = (r >> 16) & 0xff; ++ buf[t * 4 + 2] = (r >> 8) & 0xff; ++ buf[t * 4 + 3] = r & 0xff; ++ } ++ ++ /* write to virtual DPCD */ ++ if (dpcd && dpcd->data_valid) { ++ for (t = 0; t <= len; t++) { ++ int p = addr + t; ++ ++ dpcd->data[p] = buf[t]; ++ /* check for link training */ ++ if (p == DPCD_TRAINING_PATTERN_SET) ++ dp_aux_ch_ctl_link_training(dpcd, ++ buf[t]); ++ } ++ } ++ ++ /* ACK the write */ ++ vgpu_vreg(vgpu, offset + 4) = 0; ++ dp_aux_ch_ctl_trans_done(vgpu, data, offset, 1, ++ dpcd && dpcd->data_valid); ++ return 0; ++ } ++ ++ if (op == GVT_AUX_NATIVE_READ) { ++ int idx, i, ret = 0; ++ ++ if ((addr + len + 1) >= DPCD_SIZE) { ++ /* ++ * read request exceeds what we supported ++ * DPCD spec: A Sink Device receiving a Native AUX CH ++ * read request for an unsupported DPCD address must ++ * reply with an AUX ACK and read data set equal to ++ * zero instead of replying with AUX NACK. ++ */ ++ ++ /* ACK the READ*/ ++ vgpu_vreg(vgpu, offset + 4) = 0; ++ vgpu_vreg(vgpu, offset + 8) = 0; ++ vgpu_vreg(vgpu, offset + 12) = 0; ++ vgpu_vreg(vgpu, offset + 16) = 0; ++ vgpu_vreg(vgpu, offset + 20) = 0; ++ ++ dp_aux_ch_ctl_trans_done(vgpu, data, offset, len + 2, ++ true); ++ return 0; ++ } ++ ++ for (idx = 1; idx <= 5; idx++) { ++ /* clear the data registers */ ++ vgpu_vreg(vgpu, offset + 4 * idx) = 0; ++ } ++ ++ /* ++ * Read reply format: ACK (1 byte) plus (len + 1) bytes of data. ++ */ ++ if ((len + 2) > AUX_BURST_SIZE) { ++ gvt_vgpu_err("dp_aux_header: len %d is too large\n", len); ++ return -EINVAL; ++ } ++ ++ /* read from virtual DPCD to vreg */ ++ /* first 4 bytes: [ACK][addr][addr+1][addr+2] */ ++ if (dpcd && dpcd->data_valid) { ++ for (i = 1; i <= (len + 1); i++) { ++ int t; ++ ++ t = dpcd->data[addr + i - 1]; ++ t <<= (24 - 8 * (i % 4)); ++ ret |= t; ++ ++ if ((i % 4 == 3) || (i == (len + 1))) { ++ vgpu_vreg(vgpu, offset + ++ (i / 4 + 1) * 4) = ret; ++ ret = 0; ++ } ++ } ++ } ++ dp_aux_ch_ctl_trans_done(vgpu, data, offset, len + 2, ++ dpcd && dpcd->data_valid); ++ return 0; ++ } ++ ++ /* i2c transaction starts */ ++ intel_gvt_i2c_handle_aux_ch_write(vgpu, port_index, offset, p_data); ++ ++ if (data & DP_AUX_CH_CTL_INTERRUPT) ++ trigger_aux_channel_interrupt(vgpu, offset); ++ return 0; ++} ++ ++static int mbctl_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ *(u32 *)p_data &= (~GEN6_MBCTL_ENABLE_BOOT_FETCH); ++ write_vreg(vgpu, offset, p_data, bytes); ++ return 0; ++} ++ ++static int vga_control_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ bool vga_disable; ++ ++ write_vreg(vgpu, offset, p_data, bytes); ++ vga_disable = vgpu_vreg(vgpu, offset) & VGA_DISP_DISABLE; ++ ++ gvt_dbg_core("vgpu%d: %s VGA mode\n", vgpu->id, ++ vga_disable ? "Disable" : "Enable"); ++ return 0; ++} ++ ++static u32 read_virtual_sbi_register(struct intel_vgpu *vgpu, ++ unsigned int sbi_offset) ++{ ++ struct intel_vgpu_display *display = &vgpu->display; ++ int num = display->sbi.number; ++ int i; ++ ++ for (i = 0; i < num; ++i) ++ if (display->sbi.registers[i].offset == sbi_offset) ++ break; ++ ++ if (i == num) ++ return 0; ++ ++ return display->sbi.registers[i].value; ++} ++ ++static void write_virtual_sbi_register(struct intel_vgpu *vgpu, ++ unsigned int offset, u32 value) ++{ ++ struct intel_vgpu_display *display = &vgpu->display; ++ int num = display->sbi.number; ++ int i; ++ ++ for (i = 0; i < num; ++i) { ++ if (display->sbi.registers[i].offset == offset) ++ break; ++ } ++ ++ if (i == num) { ++ if (num == SBI_REG_MAX) { ++ gvt_vgpu_err("SBI caching meets maximum limits\n"); ++ return; ++ } ++ display->sbi.number++; ++ } ++ ++ display->sbi.registers[i].offset = offset; ++ display->sbi.registers[i].value = value; ++} ++ ++static int sbi_data_mmio_read(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ if (((vgpu_vreg_t(vgpu, SBI_CTL_STAT) & SBI_OPCODE_MASK) >> ++ SBI_OPCODE_SHIFT) == SBI_CMD_CRRD) { ++ unsigned int sbi_offset = (vgpu_vreg_t(vgpu, SBI_ADDR) & ++ SBI_ADDR_OFFSET_MASK) >> SBI_ADDR_OFFSET_SHIFT; ++ vgpu_vreg(vgpu, offset) = read_virtual_sbi_register(vgpu, ++ sbi_offset); ++ } ++ read_vreg(vgpu, offset, p_data, bytes); ++ return 0; ++} ++ ++static int sbi_ctl_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ u32 data; ++ ++ write_vreg(vgpu, offset, p_data, bytes); ++ data = vgpu_vreg(vgpu, offset); ++ ++ data &= ~(SBI_STAT_MASK << SBI_STAT_SHIFT); ++ data |= SBI_READY; ++ ++ data &= ~(SBI_RESPONSE_MASK << SBI_RESPONSE_SHIFT); ++ data |= SBI_RESPONSE_SUCCESS; ++ ++ vgpu_vreg(vgpu, offset) = data; ++ ++ if (((vgpu_vreg_t(vgpu, SBI_CTL_STAT) & SBI_OPCODE_MASK) >> ++ SBI_OPCODE_SHIFT) == SBI_CMD_CRWR) { ++ unsigned int sbi_offset = (vgpu_vreg_t(vgpu, SBI_ADDR) & ++ SBI_ADDR_OFFSET_MASK) >> SBI_ADDR_OFFSET_SHIFT; ++ ++ write_virtual_sbi_register(vgpu, sbi_offset, ++ vgpu_vreg_t(vgpu, SBI_DATA)); ++ } ++ return 0; ++} ++ ++#define _vgtif_reg(x) \ ++ (VGT_PVINFO_PAGE + offsetof(struct vgt_if, x)) ++ ++static int pvinfo_mmio_read(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ bool invalid_read = false; ++ ++ read_vreg(vgpu, offset, p_data, bytes); ++ ++ switch (offset) { ++ case _vgtif_reg(magic) ... _vgtif_reg(vgt_id): ++ if (offset + bytes > _vgtif_reg(vgt_id) + 4) ++ invalid_read = true; ++ break; ++ case _vgtif_reg(avail_rs.mappable_gmadr.base) ... ++ _vgtif_reg(avail_rs.fence_num): ++ if (offset + bytes > ++ _vgtif_reg(avail_rs.fence_num) + 4) ++ invalid_read = true; ++ break; ++ case 0x78010: /* vgt_caps */ ++ case 0x7881c: ++ break; ++ default: ++ invalid_read = true; ++ break; ++ } ++ if (invalid_read) ++ gvt_vgpu_err("invalid pvinfo read: [%x:%x] = %x\n", ++ offset, bytes, *(u32 *)p_data); ++ vgpu->pv_notified = true; ++ return 0; ++} ++ ++static int handle_g2v_notification(struct intel_vgpu *vgpu, int notification) ++{ ++ enum intel_gvt_gtt_type root_entry_type = GTT_TYPE_PPGTT_ROOT_L4_ENTRY; ++ struct intel_vgpu_mm *mm; ++ u64 *pdps; ++ ++ pdps = (u64 *)&vgpu_vreg64_t(vgpu, vgtif_reg(pdp[0])); ++ ++ switch (notification) { ++ case VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE: ++ root_entry_type = GTT_TYPE_PPGTT_ROOT_L3_ENTRY; ++ /* fall through */ ++ case VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE: ++ mm = intel_vgpu_get_ppgtt_mm(vgpu, root_entry_type, pdps); ++ return PTR_ERR_OR_ZERO(mm); ++ case VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY: ++ case VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY: ++ return intel_vgpu_put_ppgtt_mm(vgpu, pdps); ++ case VGT_G2V_EXECLIST_CONTEXT_CREATE: ++ case VGT_G2V_EXECLIST_CONTEXT_DESTROY: ++ case 1: /* Remove this in guest driver. */ ++ break; ++ default: ++ gvt_vgpu_err("Invalid PV notification %d\n", notification); ++ } ++ return 0; ++} ++ ++static int send_display_ready_uevent(struct intel_vgpu *vgpu, int ready) ++{ ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ struct kobject *kobj = &dev_priv->drm.primary->kdev->kobj; ++ char *env[3] = {NULL, NULL, NULL}; ++ char vmid_str[20]; ++ char display_ready_str[20]; ++ ++ snprintf(display_ready_str, 20, "GVT_DISPLAY_READY=%d", ready); ++ env[0] = display_ready_str; ++ ++ snprintf(vmid_str, 20, "VMID=%d", vgpu->id); ++ env[1] = vmid_str; ++ ++ return kobject_uevent_env(kobj, KOBJ_ADD, env); ++} ++ ++static int pvinfo_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ u32 data = *(u32 *)p_data; ++ bool invalid_write = false; ++ ++ switch (offset) { ++ case _vgtif_reg(display_ready): ++ send_display_ready_uevent(vgpu, data ? 1 : 0); ++ break; ++ case _vgtif_reg(g2v_notify): ++ handle_g2v_notification(vgpu, data); ++ break; ++ /* add xhot and yhot to handled list to avoid error log */ ++ case _vgtif_reg(cursor_x_hot): ++ case _vgtif_reg(cursor_y_hot): ++ case _vgtif_reg(pdp[0].lo): ++ case _vgtif_reg(pdp[0].hi): ++ case _vgtif_reg(pdp[1].lo): ++ case _vgtif_reg(pdp[1].hi): ++ case _vgtif_reg(pdp[2].lo): ++ case _vgtif_reg(pdp[2].hi): ++ case _vgtif_reg(pdp[3].lo): ++ case _vgtif_reg(pdp[3].hi): ++ case _vgtif_reg(execlist_context_descriptor_lo): ++ case _vgtif_reg(execlist_context_descriptor_hi): ++ break; ++ case _vgtif_reg(rsv5[0])..._vgtif_reg(rsv5[3]): ++ invalid_write = true; ++ enter_failsafe_mode(vgpu, GVT_FAILSAFE_INSUFFICIENT_RESOURCE); ++ break; ++ default: ++ invalid_write = true; ++ gvt_vgpu_err("invalid pvinfo write offset %x bytes %x data %x\n", ++ offset, bytes, data); ++ break; ++ } ++ ++ if (!invalid_write) ++ write_vreg(vgpu, offset, p_data, bytes); ++ ++ return 0; ++} ++ ++static int pf_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ u32 val = *(u32 *)p_data; ++ ++ if ((offset == _PS_1A_CTRL || offset == _PS_2A_CTRL || ++ offset == _PS_1B_CTRL || offset == _PS_2B_CTRL || ++ offset == _PS_1C_CTRL) && (val & PS_PLANE_SEL_MASK) != 0) { ++ WARN_ONCE(true, "VM(%d): guest is trying to scaling a plane\n", ++ vgpu->id); ++ return 0; ++ } ++ ++ return intel_vgpu_default_mmio_write(vgpu, offset, p_data, bytes); ++} ++ ++static int power_well_ctl_mmio_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ write_vreg(vgpu, offset, p_data, bytes); ++ ++ if (vgpu_vreg(vgpu, offset) & ++ HSW_PWR_WELL_CTL_REQ(HSW_PW_CTL_IDX_GLOBAL)) ++ vgpu_vreg(vgpu, offset) |= ++ HSW_PWR_WELL_CTL_STATE(HSW_PW_CTL_IDX_GLOBAL); ++ else ++ vgpu_vreg(vgpu, offset) &= ++ ~HSW_PWR_WELL_CTL_STATE(HSW_PW_CTL_IDX_GLOBAL); ++ return 0; ++} ++ ++static int gen9_dbuf_ctl_mmio_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ write_vreg(vgpu, offset, p_data, bytes); ++ ++ if (vgpu_vreg(vgpu, offset) & DBUF_POWER_REQUEST) ++ vgpu_vreg(vgpu, offset) |= DBUF_POWER_STATE; ++ else ++ vgpu_vreg(vgpu, offset) &= ~DBUF_POWER_STATE; ++ ++ return 0; ++} ++ ++static int fpga_dbg_mmio_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ write_vreg(vgpu, offset, p_data, bytes); ++ ++ if (vgpu_vreg(vgpu, offset) & FPGA_DBG_RM_NOCLAIM) ++ vgpu_vreg(vgpu, offset) &= ~FPGA_DBG_RM_NOCLAIM; ++ return 0; ++} ++ ++static int dma_ctrl_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ u32 mode; ++ ++ write_vreg(vgpu, offset, p_data, bytes); ++ mode = vgpu_vreg(vgpu, offset); ++ ++ if (GFX_MODE_BIT_SET_IN_MASK(mode, START_DMA)) { ++ WARN_ONCE(1, "VM(%d): iGVT-g doesn't support GuC\n", ++ vgpu->id); ++ return 0; ++ } ++ ++ return 0; ++} ++ ++static int gen9_trtte_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ u32 trtte = *(u32 *)p_data; ++ ++ if ((trtte & 1) && (trtte & (1 << 1)) == 0) { ++ WARN(1, "VM(%d): Use physical address for TRTT!\n", ++ vgpu->id); ++ return -EINVAL; ++ } ++ write_vreg(vgpu, offset, p_data, bytes); ++ ++ return 0; ++} ++ ++static int gen9_trtt_chicken_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ write_vreg(vgpu, offset, p_data, bytes); ++ return 0; ++} ++ ++static int dpll_status_read(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ u32 v = 0; ++ ++ if (vgpu_vreg(vgpu, 0x46010) & (1 << 31)) ++ v |= (1 << 0); ++ ++ if (vgpu_vreg(vgpu, 0x46014) & (1 << 31)) ++ v |= (1 << 8); ++ ++ if (vgpu_vreg(vgpu, 0x46040) & (1 << 31)) ++ v |= (1 << 16); ++ ++ if (vgpu_vreg(vgpu, 0x46060) & (1 << 31)) ++ v |= (1 << 24); ++ ++ vgpu_vreg(vgpu, offset) = v; ++ ++ return intel_vgpu_default_mmio_read(vgpu, offset, p_data, bytes); ++} ++ ++static int mailbox_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ u32 value = *(u32 *)p_data; ++ u32 cmd = value & 0xff; ++ u32 *data0 = &vgpu_vreg_t(vgpu, GEN6_PCODE_DATA); ++ ++ switch (cmd) { ++ case GEN9_PCODE_READ_MEM_LATENCY: ++ if (IS_SKYLAKE(vgpu->gvt->dev_priv) ++ || IS_KABYLAKE(vgpu->gvt->dev_priv) ++ || IS_COFFEELAKE(vgpu->gvt->dev_priv)) { ++ /** ++ * "Read memory latency" command on gen9. ++ * Below memory latency values are read ++ * from skylake platform. ++ */ ++ if (!*data0) ++ *data0 = 0x1e1a1100; ++ else ++ *data0 = 0x61514b3d; ++ } else if (IS_BROXTON(vgpu->gvt->dev_priv)) { ++ /** ++ * "Read memory latency" command on gen9. ++ * Below memory latency values are read ++ * from Broxton MRB. ++ */ ++ if (!*data0) ++ *data0 = 0x16080707; ++ else ++ *data0 = 0x16161616; ++ } ++ break; ++ case SKL_PCODE_CDCLK_CONTROL: ++ if (IS_SKYLAKE(vgpu->gvt->dev_priv) ++ || IS_KABYLAKE(vgpu->gvt->dev_priv) ++ || IS_COFFEELAKE(vgpu->gvt->dev_priv)) ++ *data0 = SKL_CDCLK_READY_FOR_CHANGE; ++ break; ++ case GEN6_PCODE_READ_RC6VIDS: ++ *data0 |= 0x1; ++ break; ++ } ++ ++ gvt_dbg_core("VM(%d) write %x to mailbox, return data0 %x\n", ++ vgpu->id, value, *data0); ++ /** ++ * PCODE_READY clear means ready for pcode read/write, ++ * PCODE_ERROR_MASK clear means no error happened. In GVT-g we ++ * always emulate as pcode read/write success and ready for access ++ * anytime, since we don't touch real physical registers here. ++ */ ++ value &= ~(GEN6_PCODE_READY | GEN6_PCODE_ERROR_MASK); ++ return intel_vgpu_default_mmio_write(vgpu, offset, &value, bytes); ++} ++ ++static int hws_pga_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ u32 value = *(u32 *)p_data; ++ int ring_id = intel_gvt_render_mmio_to_ring_id(vgpu->gvt, offset); ++ ++ if (!intel_gvt_ggtt_validate_range(vgpu, value, I915_GTT_PAGE_SIZE)) { ++ gvt_vgpu_err("write invalid HWSP address, reg:0x%x, value:0x%x\n", ++ offset, value); ++ return -EINVAL; ++ } ++ /* ++ * Need to emulate all the HWSP register write to ensure host can ++ * update the VM CSB status correctly. Here listed registers can ++ * support BDW, SKL or other platforms with same HWSP registers. ++ */ ++ if (unlikely(ring_id < 0 || ring_id >= I915_NUM_ENGINES)) { ++ gvt_vgpu_err("access unknown hardware status page register:0x%x\n", ++ offset); ++ return -EINVAL; ++ } ++ vgpu->hws_pga[ring_id] = value; ++ gvt_dbg_mmio("VM(%d) write: 0x%x to HWSP: 0x%x\n", ++ vgpu->id, value, offset); ++ ++ return intel_vgpu_default_mmio_write(vgpu, offset, &value, bytes); ++} ++ ++static int skl_power_well_ctl_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ u32 v = *(u32 *)p_data; ++ ++ if (IS_BROXTON(vgpu->gvt->dev_priv)) ++ v &= (1 << 31) | (1 << 29); ++ else ++ v &= (1 << 31) | (1 << 29) | (1 << 9) | ++ (1 << 7) | (1 << 5) | (1 << 3) | (1 << 1); ++ v |= (v >> 1); ++ ++ return intel_vgpu_default_mmio_write(vgpu, offset, &v, bytes); ++} ++ ++static int skl_lcpll_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ u32 v = *(u32 *)p_data; ++ ++ /* other bits are MBZ. */ ++ v &= (1 << 31) | (1 << 30); ++ v & (1 << 31) ? (v |= (1 << 30)) : (v &= ~(1 << 30)); ++ ++ vgpu_vreg(vgpu, offset) = v; ++ ++ return 0; ++} ++ ++static int bxt_de_pll_enable_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ u32 v = *(u32 *)p_data; ++ ++ if (v & BXT_DE_PLL_PLL_ENABLE) ++ v |= BXT_DE_PLL_LOCK; ++ ++ vgpu_vreg(vgpu, offset) = v; ++ ++ return 0; ++} ++ ++static int bxt_port_pll_enable_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ u32 v = *(u32 *)p_data; ++ ++ if (v & PORT_PLL_ENABLE) ++ v |= PORT_PLL_LOCK; ++ ++ vgpu_vreg(vgpu, offset) = v; ++ ++ return 0; ++} ++ ++static int bxt_phy_ctl_family_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ u32 v = *(u32 *)p_data; ++ u32 data = v & COMMON_RESET_DIS ? BXT_PHY_LANE_ENABLED : 0; ++ ++ switch (offset) { ++ case _PHY_CTL_FAMILY_EDP: ++ vgpu_vreg(vgpu, _BXT_PHY_CTL_DDI_A) = data; ++ break; ++ case _PHY_CTL_FAMILY_DDI: ++ vgpu_vreg(vgpu, _BXT_PHY_CTL_DDI_B) = data; ++ vgpu_vreg(vgpu, _BXT_PHY_CTL_DDI_C) = data; ++ break; ++ } ++ ++ vgpu_vreg(vgpu, offset) = v; ++ ++ return 0; ++} ++ ++static int bxt_port_tx_dw3_read(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ u32 v = vgpu_vreg(vgpu, offset); ++ ++ v &= ~UNIQUE_TRANGE_EN_METHOD; ++ ++ vgpu_vreg(vgpu, offset) = v; ++ ++ return intel_vgpu_default_mmio_read(vgpu, offset, p_data, bytes); ++} ++ ++static int bxt_pcs_dw12_grp_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ u32 v = *(u32 *)p_data; ++ ++ if (offset == _PORT_PCS_DW12_GRP_A || offset == _PORT_PCS_DW12_GRP_B) { ++ vgpu_vreg(vgpu, offset - 0x600) = v; ++ vgpu_vreg(vgpu, offset - 0x800) = v; ++ } else { ++ vgpu_vreg(vgpu, offset - 0x400) = v; ++ vgpu_vreg(vgpu, offset - 0x600) = v; ++ } ++ ++ vgpu_vreg(vgpu, offset) = v; ++ ++ return 0; ++} ++ ++static int bxt_gt_disp_pwron_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ u32 v = *(u32 *)p_data; ++ ++ if (v & BIT(0)) { ++ vgpu_vreg_t(vgpu, BXT_PORT_CL1CM_DW0(DPIO_PHY0)) &= ++ ~PHY_RESERVED; ++ vgpu_vreg_t(vgpu, BXT_PORT_CL1CM_DW0(DPIO_PHY0)) |= ++ PHY_POWER_GOOD; ++ } ++ ++ if (v & BIT(1)) { ++ vgpu_vreg_t(vgpu, BXT_PORT_CL1CM_DW0(DPIO_PHY1)) &= ++ ~PHY_RESERVED; ++ vgpu_vreg_t(vgpu, BXT_PORT_CL1CM_DW0(DPIO_PHY1)) |= ++ PHY_POWER_GOOD; ++ } ++ ++ ++ vgpu_vreg(vgpu, offset) = v; ++ ++ return 0; ++} ++ ++static int edp_psr_imr_iir_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ vgpu_vreg(vgpu, offset) = 0; ++ return 0; ++} ++ ++static int mmio_read_from_hw(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ struct intel_gvt *gvt = vgpu->gvt; ++ struct drm_i915_private *dev_priv = gvt->dev_priv; ++ int ring_id; ++ u32 ring_base; ++ ++ ring_id = intel_gvt_render_mmio_to_ring_id(gvt, offset); ++ /** ++ * Read HW reg in following case ++ * a. the offset isn't a ring mmio ++ * b. the offset's ring is running on hw. ++ * c. the offset is ring time stamp mmio ++ */ ++ if (ring_id >= 0) ++ ring_base = dev_priv->engine[ring_id]->mmio_base; ++ ++ if (ring_id < 0 || vgpu == gvt->scheduler.engine_owner[ring_id] || ++ offset == i915_mmio_reg_offset(RING_TIMESTAMP(ring_base)) || ++ offset == i915_mmio_reg_offset(RING_TIMESTAMP_UDW(ring_base))) { ++ mmio_hw_access_pre(dev_priv); ++ vgpu_vreg(vgpu, offset) = I915_READ(_MMIO(offset)); ++ mmio_hw_access_post(dev_priv); ++ } ++ ++ return intel_vgpu_default_mmio_read(vgpu, offset, p_data, bytes); ++} ++ ++static int elsp_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ int ring_id = intel_gvt_render_mmio_to_ring_id(vgpu->gvt, offset); ++ struct intel_vgpu_execlist *execlist; ++ u32 data = *(u32 *)p_data; ++ int ret = 0; ++ ++ if (WARN_ON(ring_id < 0 || ring_id >= I915_NUM_ENGINES)) ++ return -EINVAL; ++ ++ execlist = &vgpu->submission.execlist[ring_id]; ++ ++ execlist->elsp_dwords.data[3 - execlist->elsp_dwords.index] = data; ++ if (execlist->elsp_dwords.index == 3) { ++ ret = intel_vgpu_submit_execlist(vgpu, ring_id); ++ if(ret) ++ gvt_vgpu_err("fail submit workload on ring %d\n", ++ ring_id); ++ } ++ ++ ++execlist->elsp_dwords.index; ++ execlist->elsp_dwords.index &= 0x3; ++ return ret; ++} ++ ++static int ring_mode_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ u32 data = *(u32 *)p_data; ++ int ring_id = intel_gvt_render_mmio_to_ring_id(vgpu->gvt, offset); ++ bool enable_execlist; ++ int ret; ++ ++ (*(u32 *)p_data) &= ~_MASKED_BIT_ENABLE(1); ++ if (IS_COFFEELAKE(vgpu->gvt->dev_priv)) ++ (*(u32 *)p_data) &= ~_MASKED_BIT_ENABLE(2); ++ write_vreg(vgpu, offset, p_data, bytes); ++ ++ if (data & _MASKED_BIT_ENABLE(1)) { ++ enter_failsafe_mode(vgpu, GVT_FAILSAFE_UNSUPPORTED_GUEST); ++ return 0; ++ } ++ ++ if (IS_COFFEELAKE(vgpu->gvt->dev_priv) && ++ data & _MASKED_BIT_ENABLE(2)) { ++ enter_failsafe_mode(vgpu, GVT_FAILSAFE_UNSUPPORTED_GUEST); ++ return 0; ++ } ++ ++ /* when PPGTT mode enabled, we will check if guest has called ++ * pvinfo, if not, we will treat this guest as non-gvtg-aware ++ * guest, and stop emulating its cfg space, mmio, gtt, etc. ++ */ ++ if (((data & _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)) || ++ (data & _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE))) ++ && !vgpu->pv_notified) { ++ enter_failsafe_mode(vgpu, GVT_FAILSAFE_UNSUPPORTED_GUEST); ++ return 0; ++ } ++ if ((data & _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE)) ++ || (data & _MASKED_BIT_DISABLE(GFX_RUN_LIST_ENABLE))) { ++ enable_execlist = !!(data & GFX_RUN_LIST_ENABLE); ++ ++ gvt_dbg_core("EXECLIST %s on ring %d\n", ++ (enable_execlist ? "enabling" : "disabling"), ++ ring_id); ++ ++ if (!enable_execlist) ++ return 0; ++ ++ ret = intel_vgpu_select_submission_ops(vgpu, ++ BIT(ring_id), ++ INTEL_VGPU_EXECLIST_SUBMISSION); ++ if (ret) ++ return ret; ++ ++ intel_vgpu_start_schedule(vgpu); ++ } ++ return 0; ++} ++ ++static int gvt_reg_tlb_control_handler(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ unsigned int id = 0; ++ ++ write_vreg(vgpu, offset, p_data, bytes); ++ vgpu_vreg(vgpu, offset) = 0; ++ ++ switch (offset) { ++ case 0x4260: ++ id = RCS0; ++ break; ++ case 0x4264: ++ id = VCS0; ++ break; ++ case 0x4268: ++ id = VCS1; ++ break; ++ case 0x426c: ++ id = BCS0; ++ break; ++ case 0x4270: ++ id = VECS0; ++ break; ++ default: ++ return -EINVAL; ++ } ++ set_bit(id, (void *)vgpu->submission.tlb_handle_pending); ++ ++ return 0; ++} ++ ++static int ring_reset_ctl_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, unsigned int bytes) ++{ ++ u32 data; ++ ++ write_vreg(vgpu, offset, p_data, bytes); ++ data = vgpu_vreg(vgpu, offset); ++ ++ if (data & _MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET)) ++ data |= RESET_CTL_READY_TO_RESET; ++ else if (data & _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET)) ++ data &= ~RESET_CTL_READY_TO_RESET; ++ ++ vgpu_vreg(vgpu, offset) = data; ++ return 0; ++} ++ ++static int csfe_chicken1_mmio_write(struct intel_vgpu *vgpu, ++ unsigned int offset, void *p_data, ++ unsigned int bytes) ++{ ++ u32 data = *(u32 *)p_data; ++ ++ (*(u32 *)p_data) &= ~_MASKED_BIT_ENABLE(0x18); ++ write_vreg(vgpu, offset, p_data, bytes); ++ ++ if (data & _MASKED_BIT_ENABLE(0x10) || data & _MASKED_BIT_ENABLE(0x8)) ++ enter_failsafe_mode(vgpu, GVT_FAILSAFE_UNSUPPORTED_GUEST); ++ ++ return 0; ++} ++ ++#define MMIO_F(reg, s, f, am, rm, d, r, w) do { \ ++ ret = new_mmio_info(gvt, i915_mmio_reg_offset(reg), \ ++ f, s, am, rm, d, r, w); \ ++ if (ret) \ ++ return ret; \ ++} while (0) ++ ++#define MMIO_D(reg, d) \ ++ MMIO_F(reg, 4, 0, 0, 0, d, NULL, NULL) ++ ++#define MMIO_DH(reg, d, r, w) \ ++ MMIO_F(reg, 4, 0, 0, 0, d, r, w) ++ ++#define MMIO_DFH(reg, d, f, r, w) \ ++ MMIO_F(reg, 4, f, 0, 0, d, r, w) ++ ++#define MMIO_GM(reg, d, r, w) \ ++ MMIO_F(reg, 4, F_GMADR, 0xFFFFF000, 0, d, r, w) ++ ++#define MMIO_GM_RDR(reg, d, r, w) \ ++ MMIO_F(reg, 4, F_GMADR | F_CMD_ACCESS, 0xFFFFF000, 0, d, r, w) ++ ++#define MMIO_RO(reg, d, f, rm, r, w) \ ++ MMIO_F(reg, 4, F_RO | f, 0, rm, d, r, w) ++ ++#define MMIO_RING_F(prefix, s, f, am, rm, d, r, w) do { \ ++ MMIO_F(prefix(RENDER_RING_BASE), s, f, am, rm, d, r, w); \ ++ MMIO_F(prefix(BLT_RING_BASE), s, f, am, rm, d, r, w); \ ++ MMIO_F(prefix(GEN6_BSD_RING_BASE), s, f, am, rm, d, r, w); \ ++ MMIO_F(prefix(VEBOX_RING_BASE), s, f, am, rm, d, r, w); \ ++ if (HAS_ENGINE(dev_priv, VCS1)) \ ++ MMIO_F(prefix(GEN8_BSD2_RING_BASE), s, f, am, rm, d, r, w); \ ++} while (0) ++ ++#define MMIO_RING_D(prefix, d) \ ++ MMIO_RING_F(prefix, 4, 0, 0, 0, d, NULL, NULL) ++ ++#define MMIO_RING_DFH(prefix, d, f, r, w) \ ++ MMIO_RING_F(prefix, 4, f, 0, 0, d, r, w) ++ ++#define MMIO_RING_GM(prefix, d, r, w) \ ++ MMIO_RING_F(prefix, 4, F_GMADR, 0xFFFF0000, 0, d, r, w) ++ ++#define MMIO_RING_GM_RDR(prefix, d, r, w) \ ++ MMIO_RING_F(prefix, 4, F_GMADR | F_CMD_ACCESS, 0xFFFF0000, 0, d, r, w) ++ ++#define MMIO_RING_RO(prefix, d, f, rm, r, w) \ ++ MMIO_RING_F(prefix, 4, F_RO | f, 0, rm, d, r, w) ++ ++static int init_generic_mmio_info(struct intel_gvt *gvt) ++{ ++ struct drm_i915_private *dev_priv = gvt->dev_priv; ++ int ret; ++ ++ MMIO_RING_DFH(RING_IMR, D_ALL, F_CMD_ACCESS, NULL, ++ intel_vgpu_reg_imr_handler); ++ ++ MMIO_DFH(SDEIMR, D_ALL, 0, NULL, intel_vgpu_reg_imr_handler); ++ MMIO_DFH(SDEIER, D_ALL, 0, NULL, intel_vgpu_reg_ier_handler); ++ MMIO_DFH(SDEIIR, D_ALL, 0, NULL, intel_vgpu_reg_iir_handler); ++ MMIO_D(SDEISR, D_ALL); ++ ++ MMIO_RING_DFH(RING_HWSTAM, D_ALL, F_CMD_ACCESS, NULL, NULL); ++ ++ MMIO_DH(GEN8_GAMW_ECO_DEV_RW_IA, D_BDW_PLUS, NULL, ++ gamw_echo_dev_rw_ia_write); ++ ++ MMIO_GM_RDR(BSD_HWS_PGA_GEN7, D_ALL, NULL, NULL); ++ MMIO_GM_RDR(BLT_HWS_PGA_GEN7, D_ALL, NULL, NULL); ++ MMIO_GM_RDR(VEBOX_HWS_PGA_GEN7, D_ALL, NULL, NULL); ++ ++#define RING_REG(base) _MMIO((base) + 0x28) ++ MMIO_RING_DFH(RING_REG, D_ALL, F_CMD_ACCESS, NULL, NULL); ++#undef RING_REG ++ ++#define RING_REG(base) _MMIO((base) + 0x134) ++ MMIO_RING_DFH(RING_REG, D_ALL, F_CMD_ACCESS, NULL, NULL); ++#undef RING_REG ++ ++#define RING_REG(base) _MMIO((base) + 0x6c) ++ MMIO_RING_DFH(RING_REG, D_ALL, 0, mmio_read_from_hw, NULL); ++#undef RING_REG ++ MMIO_DH(GEN7_SC_INSTDONE, D_BDW_PLUS, mmio_read_from_hw, NULL); ++ ++ MMIO_GM_RDR(_MMIO(0x2148), D_ALL, NULL, NULL); ++ MMIO_GM_RDR(CCID(RENDER_RING_BASE), D_ALL, NULL, NULL); ++ MMIO_GM_RDR(_MMIO(0x12198), D_ALL, NULL, NULL); ++ MMIO_D(GEN7_CXT_SIZE, D_ALL); ++ ++ MMIO_RING_DFH(RING_TAIL, D_ALL, F_CMD_ACCESS, NULL, NULL); ++ MMIO_RING_DFH(RING_HEAD, D_ALL, F_CMD_ACCESS, NULL, NULL); ++ MMIO_RING_DFH(RING_CTL, D_ALL, F_CMD_ACCESS, NULL, NULL); ++ MMIO_RING_DFH(RING_ACTHD, D_ALL, F_CMD_ACCESS, mmio_read_from_hw, NULL); ++ MMIO_RING_GM_RDR(RING_START, D_ALL, NULL, NULL); ++ ++ /* RING MODE */ ++#define RING_REG(base) _MMIO((base) + 0x29c) ++ MMIO_RING_DFH(RING_REG, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, ++ ring_mode_mmio_write); ++#undef RING_REG ++ ++ MMIO_RING_DFH(RING_MI_MODE, D_ALL, F_MODE_MASK | F_CMD_ACCESS, ++ NULL, NULL); ++ MMIO_RING_DFH(RING_INSTPM, D_ALL, F_MODE_MASK | F_CMD_ACCESS, ++ NULL, NULL); ++ MMIO_RING_DFH(RING_TIMESTAMP, D_ALL, F_CMD_ACCESS, ++ mmio_read_from_hw, NULL); ++ MMIO_RING_DFH(RING_TIMESTAMP_UDW, D_ALL, F_CMD_ACCESS, ++ mmio_read_from_hw, NULL); ++ ++ MMIO_DFH(GEN7_GT_MODE, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(CACHE_MODE_0_GEN7, D_ALL, F_MODE_MASK | F_CMD_ACCESS, ++ NULL, NULL); ++ MMIO_DFH(CACHE_MODE_1, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(CACHE_MODE_0, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0x2124), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); ++ ++ MMIO_DFH(_MMIO(0x20dc), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_3D_CHICKEN3, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0x2088), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(FF_SLICE_CS_CHICKEN2, D_ALL, ++ F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0x2470), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(GAM_ECOCHK, D_ALL, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(GEN7_COMMON_SLICE_CHICKEN1, D_ALL, F_MODE_MASK | F_CMD_ACCESS, ++ NULL, NULL); ++ MMIO_DFH(COMMON_SLICE_CHICKEN2, D_ALL, F_MODE_MASK | F_CMD_ACCESS, ++ NULL, NULL); ++ MMIO_DFH(_MMIO(0x9030), D_ALL, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0x20a0), D_ALL, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0x2420), D_ALL, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0x2430), D_ALL, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0x2434), D_ALL, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0x2438), D_ALL, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0x243c), D_ALL, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0x7018), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(HALF_SLICE_CHICKEN3, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(GEN7_HALF_SLICE_CHICKEN1, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); ++ ++ /* display */ ++ MMIO_F(_MMIO(0x60220), 0x20, 0, 0, 0, D_ALL, NULL, NULL); ++ MMIO_D(_MMIO(0x602a0), D_ALL); ++ ++ MMIO_D(_MMIO(0x65050), D_ALL); ++ MMIO_D(_MMIO(0x650b4), D_ALL); ++ ++ MMIO_D(_MMIO(0xc4040), D_ALL); ++ MMIO_D(DERRMR, D_ALL); ++ ++ MMIO_D(PIPEDSL(PIPE_A), D_ALL); ++ MMIO_D(PIPEDSL(PIPE_B), D_ALL); ++ MMIO_D(PIPEDSL(PIPE_C), D_ALL); ++ MMIO_D(PIPEDSL(_PIPE_EDP), D_ALL); ++ ++ MMIO_DH(PIPECONF(PIPE_A), D_ALL, NULL, pipeconf_mmio_write); ++ MMIO_DH(PIPECONF(PIPE_B), D_ALL, NULL, pipeconf_mmio_write); ++ MMIO_DH(PIPECONF(PIPE_C), D_ALL, NULL, pipeconf_mmio_write); ++ MMIO_DH(PIPECONF(_PIPE_EDP), D_ALL, NULL, pipeconf_mmio_write); ++ ++ MMIO_D(PIPESTAT(PIPE_A), D_ALL); ++ MMIO_D(PIPESTAT(PIPE_B), D_ALL); ++ MMIO_D(PIPESTAT(PIPE_C), D_ALL); ++ MMIO_D(PIPESTAT(_PIPE_EDP), D_ALL); ++ ++ MMIO_D(PIPE_FLIPCOUNT_G4X(PIPE_A), D_ALL); ++ MMIO_D(PIPE_FLIPCOUNT_G4X(PIPE_B), D_ALL); ++ MMIO_D(PIPE_FLIPCOUNT_G4X(PIPE_C), D_ALL); ++ MMIO_D(PIPE_FLIPCOUNT_G4X(_PIPE_EDP), D_ALL); ++ ++ MMIO_D(PIPE_FRMCOUNT_G4X(PIPE_A), D_ALL); ++ MMIO_D(PIPE_FRMCOUNT_G4X(PIPE_B), D_ALL); ++ MMIO_D(PIPE_FRMCOUNT_G4X(PIPE_C), D_ALL); ++ MMIO_D(PIPE_FRMCOUNT_G4X(_PIPE_EDP), D_ALL); ++ ++ MMIO_D(CURCNTR(PIPE_A), D_ALL); ++ MMIO_D(CURCNTR(PIPE_B), D_ALL); ++ MMIO_D(CURCNTR(PIPE_C), D_ALL); ++ ++ MMIO_D(CURPOS(PIPE_A), D_ALL); ++ MMIO_D(CURPOS(PIPE_B), D_ALL); ++ MMIO_D(CURPOS(PIPE_C), D_ALL); ++ ++ MMIO_D(CURBASE(PIPE_A), D_ALL); ++ MMIO_D(CURBASE(PIPE_B), D_ALL); ++ MMIO_D(CURBASE(PIPE_C), D_ALL); ++ ++ MMIO_D(CUR_FBC_CTL(PIPE_A), D_ALL); ++ MMIO_D(CUR_FBC_CTL(PIPE_B), D_ALL); ++ MMIO_D(CUR_FBC_CTL(PIPE_C), D_ALL); ++ ++ MMIO_D(_MMIO(0x700ac), D_ALL); ++ MMIO_D(_MMIO(0x710ac), D_ALL); ++ MMIO_D(_MMIO(0x720ac), D_ALL); ++ ++ MMIO_D(_MMIO(0x70090), D_ALL); ++ MMIO_D(_MMIO(0x70094), D_ALL); ++ MMIO_D(_MMIO(0x70098), D_ALL); ++ MMIO_D(_MMIO(0x7009c), D_ALL); ++ ++ MMIO_D(DSPCNTR(PIPE_A), D_ALL); ++ MMIO_D(DSPADDR(PIPE_A), D_ALL); ++ MMIO_D(DSPSTRIDE(PIPE_A), D_ALL); ++ MMIO_D(DSPPOS(PIPE_A), D_ALL); ++ MMIO_D(DSPSIZE(PIPE_A), D_ALL); ++ MMIO_DH(DSPSURF(PIPE_A), D_ALL, NULL, pri_surf_mmio_write); ++ MMIO_D(DSPOFFSET(PIPE_A), D_ALL); ++ MMIO_D(DSPSURFLIVE(PIPE_A), D_ALL); ++ MMIO_DH(REG_50080(PIPE_A, PLANE_PRIMARY), D_ALL, NULL, ++ reg50080_mmio_write); ++ ++ MMIO_D(DSPCNTR(PIPE_B), D_ALL); ++ MMIO_D(DSPADDR(PIPE_B), D_ALL); ++ MMIO_D(DSPSTRIDE(PIPE_B), D_ALL); ++ MMIO_D(DSPPOS(PIPE_B), D_ALL); ++ MMIO_D(DSPSIZE(PIPE_B), D_ALL); ++ MMIO_DH(DSPSURF(PIPE_B), D_ALL, NULL, pri_surf_mmio_write); ++ MMIO_D(DSPOFFSET(PIPE_B), D_ALL); ++ MMIO_D(DSPSURFLIVE(PIPE_B), D_ALL); ++ MMIO_DH(REG_50080(PIPE_B, PLANE_PRIMARY), D_ALL, NULL, ++ reg50080_mmio_write); ++ ++ MMIO_D(DSPCNTR(PIPE_C), D_ALL); ++ MMIO_D(DSPADDR(PIPE_C), D_ALL); ++ MMIO_D(DSPSTRIDE(PIPE_C), D_ALL); ++ MMIO_D(DSPPOS(PIPE_C), D_ALL); ++ MMIO_D(DSPSIZE(PIPE_C), D_ALL); ++ MMIO_DH(DSPSURF(PIPE_C), D_ALL, NULL, pri_surf_mmio_write); ++ MMIO_D(DSPOFFSET(PIPE_C), D_ALL); ++ MMIO_D(DSPSURFLIVE(PIPE_C), D_ALL); ++ MMIO_DH(REG_50080(PIPE_C, PLANE_PRIMARY), D_ALL, NULL, ++ reg50080_mmio_write); ++ ++ MMIO_D(SPRCTL(PIPE_A), D_ALL); ++ MMIO_D(SPRLINOFF(PIPE_A), D_ALL); ++ MMIO_D(SPRSTRIDE(PIPE_A), D_ALL); ++ MMIO_D(SPRPOS(PIPE_A), D_ALL); ++ MMIO_D(SPRSIZE(PIPE_A), D_ALL); ++ MMIO_D(SPRKEYVAL(PIPE_A), D_ALL); ++ MMIO_D(SPRKEYMSK(PIPE_A), D_ALL); ++ MMIO_DH(SPRSURF(PIPE_A), D_ALL, NULL, spr_surf_mmio_write); ++ MMIO_D(SPRKEYMAX(PIPE_A), D_ALL); ++ MMIO_D(SPROFFSET(PIPE_A), D_ALL); ++ MMIO_D(SPRSCALE(PIPE_A), D_ALL); ++ MMIO_D(SPRSURFLIVE(PIPE_A), D_ALL); ++ MMIO_DH(REG_50080(PIPE_A, PLANE_SPRITE0), D_ALL, NULL, ++ reg50080_mmio_write); ++ ++ MMIO_D(SPRCTL(PIPE_B), D_ALL); ++ MMIO_D(SPRLINOFF(PIPE_B), D_ALL); ++ MMIO_D(SPRSTRIDE(PIPE_B), D_ALL); ++ MMIO_D(SPRPOS(PIPE_B), D_ALL); ++ MMIO_D(SPRSIZE(PIPE_B), D_ALL); ++ MMIO_D(SPRKEYVAL(PIPE_B), D_ALL); ++ MMIO_D(SPRKEYMSK(PIPE_B), D_ALL); ++ MMIO_DH(SPRSURF(PIPE_B), D_ALL, NULL, spr_surf_mmio_write); ++ MMIO_D(SPRKEYMAX(PIPE_B), D_ALL); ++ MMIO_D(SPROFFSET(PIPE_B), D_ALL); ++ MMIO_D(SPRSCALE(PIPE_B), D_ALL); ++ MMIO_D(SPRSURFLIVE(PIPE_B), D_ALL); ++ MMIO_DH(REG_50080(PIPE_B, PLANE_SPRITE0), D_ALL, NULL, ++ reg50080_mmio_write); ++ ++ MMIO_D(SPRCTL(PIPE_C), D_ALL); ++ MMIO_D(SPRLINOFF(PIPE_C), D_ALL); ++ MMIO_D(SPRSTRIDE(PIPE_C), D_ALL); ++ MMIO_D(SPRPOS(PIPE_C), D_ALL); ++ MMIO_D(SPRSIZE(PIPE_C), D_ALL); ++ MMIO_D(SPRKEYVAL(PIPE_C), D_ALL); ++ MMIO_D(SPRKEYMSK(PIPE_C), D_ALL); ++ MMIO_DH(SPRSURF(PIPE_C), D_ALL, NULL, spr_surf_mmio_write); ++ MMIO_D(SPRKEYMAX(PIPE_C), D_ALL); ++ MMIO_D(SPROFFSET(PIPE_C), D_ALL); ++ MMIO_D(SPRSCALE(PIPE_C), D_ALL); ++ MMIO_D(SPRSURFLIVE(PIPE_C), D_ALL); ++ MMIO_DH(REG_50080(PIPE_C, PLANE_SPRITE0), D_ALL, NULL, ++ reg50080_mmio_write); ++ ++ MMIO_D(HTOTAL(TRANSCODER_A), D_ALL); ++ MMIO_D(HBLANK(TRANSCODER_A), D_ALL); ++ MMIO_D(HSYNC(TRANSCODER_A), D_ALL); ++ MMIO_D(VTOTAL(TRANSCODER_A), D_ALL); ++ MMIO_D(VBLANK(TRANSCODER_A), D_ALL); ++ MMIO_D(VSYNC(TRANSCODER_A), D_ALL); ++ MMIO_D(BCLRPAT(TRANSCODER_A), D_ALL); ++ MMIO_D(VSYNCSHIFT(TRANSCODER_A), D_ALL); ++ MMIO_D(PIPESRC(TRANSCODER_A), D_ALL); ++ ++ MMIO_D(HTOTAL(TRANSCODER_B), D_ALL); ++ MMIO_D(HBLANK(TRANSCODER_B), D_ALL); ++ MMIO_D(HSYNC(TRANSCODER_B), D_ALL); ++ MMIO_D(VTOTAL(TRANSCODER_B), D_ALL); ++ MMIO_D(VBLANK(TRANSCODER_B), D_ALL); ++ MMIO_D(VSYNC(TRANSCODER_B), D_ALL); ++ MMIO_D(BCLRPAT(TRANSCODER_B), D_ALL); ++ MMIO_D(VSYNCSHIFT(TRANSCODER_B), D_ALL); ++ MMIO_D(PIPESRC(TRANSCODER_B), D_ALL); ++ ++ MMIO_D(HTOTAL(TRANSCODER_C), D_ALL); ++ MMIO_D(HBLANK(TRANSCODER_C), D_ALL); ++ MMIO_D(HSYNC(TRANSCODER_C), D_ALL); ++ MMIO_D(VTOTAL(TRANSCODER_C), D_ALL); ++ MMIO_D(VBLANK(TRANSCODER_C), D_ALL); ++ MMIO_D(VSYNC(TRANSCODER_C), D_ALL); ++ MMIO_D(BCLRPAT(TRANSCODER_C), D_ALL); ++ MMIO_D(VSYNCSHIFT(TRANSCODER_C), D_ALL); ++ MMIO_D(PIPESRC(TRANSCODER_C), D_ALL); ++ ++ MMIO_D(HTOTAL(TRANSCODER_EDP), D_ALL); ++ MMIO_D(HBLANK(TRANSCODER_EDP), D_ALL); ++ MMIO_D(HSYNC(TRANSCODER_EDP), D_ALL); ++ MMIO_D(VTOTAL(TRANSCODER_EDP), D_ALL); ++ MMIO_D(VBLANK(TRANSCODER_EDP), D_ALL); ++ MMIO_D(VSYNC(TRANSCODER_EDP), D_ALL); ++ MMIO_D(BCLRPAT(TRANSCODER_EDP), D_ALL); ++ MMIO_D(VSYNCSHIFT(TRANSCODER_EDP), D_ALL); ++ ++ MMIO_D(PIPE_DATA_M1(TRANSCODER_A), D_ALL); ++ MMIO_D(PIPE_DATA_N1(TRANSCODER_A), D_ALL); ++ MMIO_D(PIPE_DATA_M2(TRANSCODER_A), D_ALL); ++ MMIO_D(PIPE_DATA_N2(TRANSCODER_A), D_ALL); ++ MMIO_D(PIPE_LINK_M1(TRANSCODER_A), D_ALL); ++ MMIO_D(PIPE_LINK_N1(TRANSCODER_A), D_ALL); ++ MMIO_D(PIPE_LINK_M2(TRANSCODER_A), D_ALL); ++ MMIO_D(PIPE_LINK_N2(TRANSCODER_A), D_ALL); ++ ++ MMIO_D(PIPE_DATA_M1(TRANSCODER_B), D_ALL); ++ MMIO_D(PIPE_DATA_N1(TRANSCODER_B), D_ALL); ++ MMIO_D(PIPE_DATA_M2(TRANSCODER_B), D_ALL); ++ MMIO_D(PIPE_DATA_N2(TRANSCODER_B), D_ALL); ++ MMIO_D(PIPE_LINK_M1(TRANSCODER_B), D_ALL); ++ MMIO_D(PIPE_LINK_N1(TRANSCODER_B), D_ALL); ++ MMIO_D(PIPE_LINK_M2(TRANSCODER_B), D_ALL); ++ MMIO_D(PIPE_LINK_N2(TRANSCODER_B), D_ALL); ++ ++ MMIO_D(PIPE_DATA_M1(TRANSCODER_C), D_ALL); ++ MMIO_D(PIPE_DATA_N1(TRANSCODER_C), D_ALL); ++ MMIO_D(PIPE_DATA_M2(TRANSCODER_C), D_ALL); ++ MMIO_D(PIPE_DATA_N2(TRANSCODER_C), D_ALL); ++ MMIO_D(PIPE_LINK_M1(TRANSCODER_C), D_ALL); ++ MMIO_D(PIPE_LINK_N1(TRANSCODER_C), D_ALL); ++ MMIO_D(PIPE_LINK_M2(TRANSCODER_C), D_ALL); ++ MMIO_D(PIPE_LINK_N2(TRANSCODER_C), D_ALL); ++ ++ MMIO_D(PIPE_DATA_M1(TRANSCODER_EDP), D_ALL); ++ MMIO_D(PIPE_DATA_N1(TRANSCODER_EDP), D_ALL); ++ MMIO_D(PIPE_DATA_M2(TRANSCODER_EDP), D_ALL); ++ MMIO_D(PIPE_DATA_N2(TRANSCODER_EDP), D_ALL); ++ MMIO_D(PIPE_LINK_M1(TRANSCODER_EDP), D_ALL); ++ MMIO_D(PIPE_LINK_N1(TRANSCODER_EDP), D_ALL); ++ MMIO_D(PIPE_LINK_M2(TRANSCODER_EDP), D_ALL); ++ MMIO_D(PIPE_LINK_N2(TRANSCODER_EDP), D_ALL); ++ ++ MMIO_D(PF_CTL(PIPE_A), D_ALL); ++ MMIO_D(PF_WIN_SZ(PIPE_A), D_ALL); ++ MMIO_D(PF_WIN_POS(PIPE_A), D_ALL); ++ MMIO_D(PF_VSCALE(PIPE_A), D_ALL); ++ MMIO_D(PF_HSCALE(PIPE_A), D_ALL); ++ ++ MMIO_D(PF_CTL(PIPE_B), D_ALL); ++ MMIO_D(PF_WIN_SZ(PIPE_B), D_ALL); ++ MMIO_D(PF_WIN_POS(PIPE_B), D_ALL); ++ MMIO_D(PF_VSCALE(PIPE_B), D_ALL); ++ MMIO_D(PF_HSCALE(PIPE_B), D_ALL); ++ ++ MMIO_D(PF_CTL(PIPE_C), D_ALL); ++ MMIO_D(PF_WIN_SZ(PIPE_C), D_ALL); ++ MMIO_D(PF_WIN_POS(PIPE_C), D_ALL); ++ MMIO_D(PF_VSCALE(PIPE_C), D_ALL); ++ MMIO_D(PF_HSCALE(PIPE_C), D_ALL); ++ ++ MMIO_D(WM0_PIPEA_ILK, D_ALL); ++ MMIO_D(WM0_PIPEB_ILK, D_ALL); ++ MMIO_D(WM0_PIPEC_IVB, D_ALL); ++ MMIO_D(WM1_LP_ILK, D_ALL); ++ MMIO_D(WM2_LP_ILK, D_ALL); ++ MMIO_D(WM3_LP_ILK, D_ALL); ++ MMIO_D(WM1S_LP_ILK, D_ALL); ++ MMIO_D(WM2S_LP_IVB, D_ALL); ++ MMIO_D(WM3S_LP_IVB, D_ALL); ++ ++ MMIO_D(BLC_PWM_CPU_CTL2, D_ALL); ++ MMIO_D(BLC_PWM_CPU_CTL, D_ALL); ++ MMIO_D(BLC_PWM_PCH_CTL1, D_ALL); ++ MMIO_D(BLC_PWM_PCH_CTL2, D_ALL); ++ ++ MMIO_D(_MMIO(0x48268), D_ALL); ++ ++ MMIO_F(PCH_GMBUS0, 4 * 4, 0, 0, 0, D_ALL, gmbus_mmio_read, ++ gmbus_mmio_write); ++ MMIO_F(PCH_GPIO_BASE, 6 * 4, F_UNALIGN, 0, 0, D_ALL, NULL, NULL); ++ MMIO_F(_MMIO(0xe4f00), 0x28, 0, 0, 0, D_ALL, NULL, NULL); ++ ++ MMIO_F(_MMIO(_PCH_DPB_AUX_CH_CTL), 6 * 4, 0, 0, 0, D_PRE_SKL, NULL, ++ dp_aux_ch_ctl_mmio_write); ++ MMIO_F(_MMIO(_PCH_DPC_AUX_CH_CTL), 6 * 4, 0, 0, 0, D_PRE_SKL, NULL, ++ dp_aux_ch_ctl_mmio_write); ++ MMIO_F(_MMIO(_PCH_DPD_AUX_CH_CTL), 6 * 4, 0, 0, 0, D_PRE_SKL, NULL, ++ dp_aux_ch_ctl_mmio_write); ++ ++ MMIO_DH(PCH_ADPA, D_PRE_SKL, NULL, pch_adpa_mmio_write); ++ ++ MMIO_DH(_MMIO(_PCH_TRANSACONF), D_ALL, NULL, transconf_mmio_write); ++ MMIO_DH(_MMIO(_PCH_TRANSBCONF), D_ALL, NULL, transconf_mmio_write); ++ ++ MMIO_DH(FDI_RX_IIR(PIPE_A), D_ALL, NULL, fdi_rx_iir_mmio_write); ++ MMIO_DH(FDI_RX_IIR(PIPE_B), D_ALL, NULL, fdi_rx_iir_mmio_write); ++ MMIO_DH(FDI_RX_IIR(PIPE_C), D_ALL, NULL, fdi_rx_iir_mmio_write); ++ MMIO_DH(FDI_RX_IMR(PIPE_A), D_ALL, NULL, update_fdi_rx_iir_status); ++ MMIO_DH(FDI_RX_IMR(PIPE_B), D_ALL, NULL, update_fdi_rx_iir_status); ++ MMIO_DH(FDI_RX_IMR(PIPE_C), D_ALL, NULL, update_fdi_rx_iir_status); ++ MMIO_DH(FDI_RX_CTL(PIPE_A), D_ALL, NULL, update_fdi_rx_iir_status); ++ MMIO_DH(FDI_RX_CTL(PIPE_B), D_ALL, NULL, update_fdi_rx_iir_status); ++ MMIO_DH(FDI_RX_CTL(PIPE_C), D_ALL, NULL, update_fdi_rx_iir_status); ++ ++ MMIO_D(_MMIO(_PCH_TRANS_HTOTAL_A), D_ALL); ++ MMIO_D(_MMIO(_PCH_TRANS_HBLANK_A), D_ALL); ++ MMIO_D(_MMIO(_PCH_TRANS_HSYNC_A), D_ALL); ++ MMIO_D(_MMIO(_PCH_TRANS_VTOTAL_A), D_ALL); ++ MMIO_D(_MMIO(_PCH_TRANS_VBLANK_A), D_ALL); ++ MMIO_D(_MMIO(_PCH_TRANS_VSYNC_A), D_ALL); ++ MMIO_D(_MMIO(_PCH_TRANS_VSYNCSHIFT_A), D_ALL); ++ ++ MMIO_D(_MMIO(_PCH_TRANS_HTOTAL_B), D_ALL); ++ MMIO_D(_MMIO(_PCH_TRANS_HBLANK_B), D_ALL); ++ MMIO_D(_MMIO(_PCH_TRANS_HSYNC_B), D_ALL); ++ MMIO_D(_MMIO(_PCH_TRANS_VTOTAL_B), D_ALL); ++ MMIO_D(_MMIO(_PCH_TRANS_VBLANK_B), D_ALL); ++ MMIO_D(_MMIO(_PCH_TRANS_VSYNC_B), D_ALL); ++ MMIO_D(_MMIO(_PCH_TRANS_VSYNCSHIFT_B), D_ALL); ++ ++ MMIO_D(_MMIO(_PCH_TRANSA_DATA_M1), D_ALL); ++ MMIO_D(_MMIO(_PCH_TRANSA_DATA_N1), D_ALL); ++ MMIO_D(_MMIO(_PCH_TRANSA_DATA_M2), D_ALL); ++ MMIO_D(_MMIO(_PCH_TRANSA_DATA_N2), D_ALL); ++ MMIO_D(_MMIO(_PCH_TRANSA_LINK_M1), D_ALL); ++ MMIO_D(_MMIO(_PCH_TRANSA_LINK_N1), D_ALL); ++ MMIO_D(_MMIO(_PCH_TRANSA_LINK_M2), D_ALL); ++ MMIO_D(_MMIO(_PCH_TRANSA_LINK_N2), D_ALL); ++ ++ MMIO_D(TRANS_DP_CTL(PIPE_A), D_ALL); ++ MMIO_D(TRANS_DP_CTL(PIPE_B), D_ALL); ++ MMIO_D(TRANS_DP_CTL(PIPE_C), D_ALL); ++ ++ MMIO_D(TVIDEO_DIP_CTL(PIPE_A), D_ALL); ++ MMIO_D(TVIDEO_DIP_DATA(PIPE_A), D_ALL); ++ MMIO_D(TVIDEO_DIP_GCP(PIPE_A), D_ALL); ++ ++ MMIO_D(TVIDEO_DIP_CTL(PIPE_B), D_ALL); ++ MMIO_D(TVIDEO_DIP_DATA(PIPE_B), D_ALL); ++ MMIO_D(TVIDEO_DIP_GCP(PIPE_B), D_ALL); ++ ++ MMIO_D(TVIDEO_DIP_CTL(PIPE_C), D_ALL); ++ MMIO_D(TVIDEO_DIP_DATA(PIPE_C), D_ALL); ++ MMIO_D(TVIDEO_DIP_GCP(PIPE_C), D_ALL); ++ ++ MMIO_D(_MMIO(_FDI_RXA_MISC), D_ALL); ++ MMIO_D(_MMIO(_FDI_RXB_MISC), D_ALL); ++ MMIO_D(_MMIO(_FDI_RXA_TUSIZE1), D_ALL); ++ MMIO_D(_MMIO(_FDI_RXA_TUSIZE2), D_ALL); ++ MMIO_D(_MMIO(_FDI_RXB_TUSIZE1), D_ALL); ++ MMIO_D(_MMIO(_FDI_RXB_TUSIZE2), D_ALL); ++ ++ MMIO_DH(PCH_PP_CONTROL, D_ALL, NULL, pch_pp_control_mmio_write); ++ MMIO_D(PCH_PP_DIVISOR, D_ALL); ++ MMIO_D(PCH_PP_STATUS, D_ALL); ++ MMIO_D(PCH_LVDS, D_ALL); ++ MMIO_D(_MMIO(_PCH_DPLL_A), D_ALL); ++ MMIO_D(_MMIO(_PCH_DPLL_B), D_ALL); ++ MMIO_D(_MMIO(_PCH_FPA0), D_ALL); ++ MMIO_D(_MMIO(_PCH_FPA1), D_ALL); ++ MMIO_D(_MMIO(_PCH_FPB0), D_ALL); ++ MMIO_D(_MMIO(_PCH_FPB1), D_ALL); ++ MMIO_D(PCH_DREF_CONTROL, D_ALL); ++ MMIO_D(PCH_RAWCLK_FREQ, D_ALL); ++ MMIO_D(PCH_DPLL_SEL, D_ALL); ++ ++ MMIO_D(_MMIO(0x61208), D_ALL); ++ MMIO_D(_MMIO(0x6120c), D_ALL); ++ MMIO_D(PCH_PP_ON_DELAYS, D_ALL); ++ MMIO_D(PCH_PP_OFF_DELAYS, D_ALL); ++ ++ MMIO_DH(_MMIO(0xe651c), D_ALL, dpy_reg_mmio_read, NULL); ++ MMIO_DH(_MMIO(0xe661c), D_ALL, dpy_reg_mmio_read, NULL); ++ MMIO_DH(_MMIO(0xe671c), D_ALL, dpy_reg_mmio_read, NULL); ++ MMIO_DH(_MMIO(0xe681c), D_ALL, dpy_reg_mmio_read, NULL); ++ MMIO_DH(_MMIO(0xe6c04), D_ALL, dpy_reg_mmio_read, NULL); ++ MMIO_DH(_MMIO(0xe6e1c), D_ALL, dpy_reg_mmio_read, NULL); ++ ++ MMIO_RO(PCH_PORT_HOTPLUG, D_ALL, 0, ++ PORTA_HOTPLUG_STATUS_MASK ++ | PORTB_HOTPLUG_STATUS_MASK ++ | PORTC_HOTPLUG_STATUS_MASK ++ | PORTD_HOTPLUG_STATUS_MASK, ++ NULL, NULL); ++ ++ MMIO_DH(LCPLL_CTL, D_ALL, NULL, lcpll_ctl_mmio_write); ++ MMIO_D(FUSE_STRAP, D_ALL); ++ MMIO_D(DIGITAL_PORT_HOTPLUG_CNTRL, D_ALL); ++ ++ MMIO_D(DISP_ARB_CTL, D_ALL); ++ MMIO_D(DISP_ARB_CTL2, D_ALL); ++ ++ MMIO_D(ILK_DISPLAY_CHICKEN1, D_ALL); ++ MMIO_D(ILK_DISPLAY_CHICKEN2, D_ALL); ++ MMIO_D(ILK_DSPCLK_GATE_D, D_ALL); ++ ++ MMIO_D(SOUTH_CHICKEN1, D_ALL); ++ MMIO_DH(SOUTH_CHICKEN2, D_ALL, NULL, south_chicken2_mmio_write); ++ MMIO_D(_MMIO(_TRANSA_CHICKEN1), D_ALL); ++ MMIO_D(_MMIO(_TRANSB_CHICKEN1), D_ALL); ++ MMIO_D(SOUTH_DSPCLK_GATE_D, D_ALL); ++ MMIO_D(_MMIO(_TRANSA_CHICKEN2), D_ALL); ++ MMIO_D(_MMIO(_TRANSB_CHICKEN2), D_ALL); ++ ++ MMIO_D(ILK_DPFC_CB_BASE, D_ALL); ++ MMIO_D(ILK_DPFC_CONTROL, D_ALL); ++ MMIO_D(ILK_DPFC_RECOMP_CTL, D_ALL); ++ MMIO_D(ILK_DPFC_STATUS, D_ALL); ++ MMIO_D(ILK_DPFC_FENCE_YOFF, D_ALL); ++ MMIO_D(ILK_DPFC_CHICKEN, D_ALL); ++ MMIO_D(ILK_FBC_RT_BASE, D_ALL); ++ ++ MMIO_D(IPS_CTL, D_ALL); ++ ++ MMIO_D(PIPE_CSC_COEFF_RY_GY(PIPE_A), D_ALL); ++ MMIO_D(PIPE_CSC_COEFF_BY(PIPE_A), D_ALL); ++ MMIO_D(PIPE_CSC_COEFF_RU_GU(PIPE_A), D_ALL); ++ MMIO_D(PIPE_CSC_COEFF_BU(PIPE_A), D_ALL); ++ MMIO_D(PIPE_CSC_COEFF_RV_GV(PIPE_A), D_ALL); ++ MMIO_D(PIPE_CSC_COEFF_BV(PIPE_A), D_ALL); ++ MMIO_D(PIPE_CSC_MODE(PIPE_A), D_ALL); ++ MMIO_D(PIPE_CSC_PREOFF_HI(PIPE_A), D_ALL); ++ MMIO_D(PIPE_CSC_PREOFF_ME(PIPE_A), D_ALL); ++ MMIO_D(PIPE_CSC_PREOFF_LO(PIPE_A), D_ALL); ++ MMIO_D(PIPE_CSC_POSTOFF_HI(PIPE_A), D_ALL); ++ MMIO_D(PIPE_CSC_POSTOFF_ME(PIPE_A), D_ALL); ++ MMIO_D(PIPE_CSC_POSTOFF_LO(PIPE_A), D_ALL); ++ ++ MMIO_D(PIPE_CSC_COEFF_RY_GY(PIPE_B), D_ALL); ++ MMIO_D(PIPE_CSC_COEFF_BY(PIPE_B), D_ALL); ++ MMIO_D(PIPE_CSC_COEFF_RU_GU(PIPE_B), D_ALL); ++ MMIO_D(PIPE_CSC_COEFF_BU(PIPE_B), D_ALL); ++ MMIO_D(PIPE_CSC_COEFF_RV_GV(PIPE_B), D_ALL); ++ MMIO_D(PIPE_CSC_COEFF_BV(PIPE_B), D_ALL); ++ MMIO_D(PIPE_CSC_MODE(PIPE_B), D_ALL); ++ MMIO_D(PIPE_CSC_PREOFF_HI(PIPE_B), D_ALL); ++ MMIO_D(PIPE_CSC_PREOFF_ME(PIPE_B), D_ALL); ++ MMIO_D(PIPE_CSC_PREOFF_LO(PIPE_B), D_ALL); ++ MMIO_D(PIPE_CSC_POSTOFF_HI(PIPE_B), D_ALL); ++ MMIO_D(PIPE_CSC_POSTOFF_ME(PIPE_B), D_ALL); ++ MMIO_D(PIPE_CSC_POSTOFF_LO(PIPE_B), D_ALL); ++ ++ MMIO_D(PIPE_CSC_COEFF_RY_GY(PIPE_C), D_ALL); ++ MMIO_D(PIPE_CSC_COEFF_BY(PIPE_C), D_ALL); ++ MMIO_D(PIPE_CSC_COEFF_RU_GU(PIPE_C), D_ALL); ++ MMIO_D(PIPE_CSC_COEFF_BU(PIPE_C), D_ALL); ++ MMIO_D(PIPE_CSC_COEFF_RV_GV(PIPE_C), D_ALL); ++ MMIO_D(PIPE_CSC_COEFF_BV(PIPE_C), D_ALL); ++ MMIO_D(PIPE_CSC_MODE(PIPE_C), D_ALL); ++ MMIO_D(PIPE_CSC_PREOFF_HI(PIPE_C), D_ALL); ++ MMIO_D(PIPE_CSC_PREOFF_ME(PIPE_C), D_ALL); ++ MMIO_D(PIPE_CSC_PREOFF_LO(PIPE_C), D_ALL); ++ MMIO_D(PIPE_CSC_POSTOFF_HI(PIPE_C), D_ALL); ++ MMIO_D(PIPE_CSC_POSTOFF_ME(PIPE_C), D_ALL); ++ MMIO_D(PIPE_CSC_POSTOFF_LO(PIPE_C), D_ALL); ++ ++ MMIO_D(PREC_PAL_INDEX(PIPE_A), D_ALL); ++ MMIO_D(PREC_PAL_DATA(PIPE_A), D_ALL); ++ MMIO_F(PREC_PAL_GC_MAX(PIPE_A, 0), 4 * 3, 0, 0, 0, D_ALL, NULL, NULL); ++ ++ MMIO_D(PREC_PAL_INDEX(PIPE_B), D_ALL); ++ MMIO_D(PREC_PAL_DATA(PIPE_B), D_ALL); ++ MMIO_F(PREC_PAL_GC_MAX(PIPE_B, 0), 4 * 3, 0, 0, 0, D_ALL, NULL, NULL); ++ ++ MMIO_D(PREC_PAL_INDEX(PIPE_C), D_ALL); ++ MMIO_D(PREC_PAL_DATA(PIPE_C), D_ALL); ++ MMIO_F(PREC_PAL_GC_MAX(PIPE_C, 0), 4 * 3, 0, 0, 0, D_ALL, NULL, NULL); ++ ++ MMIO_D(_MMIO(0x60110), D_ALL); ++ MMIO_D(_MMIO(0x61110), D_ALL); ++ MMIO_F(_MMIO(0x70400), 0x40, 0, 0, 0, D_ALL, NULL, NULL); ++ MMIO_F(_MMIO(0x71400), 0x40, 0, 0, 0, D_ALL, NULL, NULL); ++ MMIO_F(_MMIO(0x72400), 0x40, 0, 0, 0, D_ALL, NULL, NULL); ++ MMIO_F(_MMIO(0x70440), 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL); ++ MMIO_F(_MMIO(0x71440), 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL); ++ MMIO_F(_MMIO(0x72440), 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL); ++ MMIO_F(_MMIO(0x7044c), 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL); ++ MMIO_F(_MMIO(0x7144c), 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL); ++ MMIO_F(_MMIO(0x7244c), 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL); ++ ++ MMIO_D(PIPE_WM_LINETIME(PIPE_A), D_ALL); ++ MMIO_D(PIPE_WM_LINETIME(PIPE_B), D_ALL); ++ MMIO_D(PIPE_WM_LINETIME(PIPE_C), D_ALL); ++ MMIO_D(SPLL_CTL, D_ALL); ++ MMIO_D(_MMIO(_WRPLL_CTL1), D_ALL); ++ MMIO_D(_MMIO(_WRPLL_CTL2), D_ALL); ++ MMIO_D(PORT_CLK_SEL(PORT_A), D_ALL); ++ MMIO_D(PORT_CLK_SEL(PORT_B), D_ALL); ++ MMIO_D(PORT_CLK_SEL(PORT_C), D_ALL); ++ MMIO_D(PORT_CLK_SEL(PORT_D), D_ALL); ++ MMIO_D(PORT_CLK_SEL(PORT_E), D_ALL); ++ MMIO_D(TRANS_CLK_SEL(TRANSCODER_A), D_ALL); ++ MMIO_D(TRANS_CLK_SEL(TRANSCODER_B), D_ALL); ++ MMIO_D(TRANS_CLK_SEL(TRANSCODER_C), D_ALL); ++ ++ MMIO_D(HSW_NDE_RSTWRN_OPT, D_ALL); ++ MMIO_D(_MMIO(0x46508), D_ALL); ++ ++ MMIO_D(_MMIO(0x49080), D_ALL); ++ MMIO_D(_MMIO(0x49180), D_ALL); ++ MMIO_D(_MMIO(0x49280), D_ALL); ++ ++ MMIO_F(_MMIO(0x49090), 0x14, 0, 0, 0, D_ALL, NULL, NULL); ++ MMIO_F(_MMIO(0x49190), 0x14, 0, 0, 0, D_ALL, NULL, NULL); ++ MMIO_F(_MMIO(0x49290), 0x14, 0, 0, 0, D_ALL, NULL, NULL); ++ ++ MMIO_D(GAMMA_MODE(PIPE_A), D_ALL); ++ MMIO_D(GAMMA_MODE(PIPE_B), D_ALL); ++ MMIO_D(GAMMA_MODE(PIPE_C), D_ALL); ++ ++ MMIO_D(PIPE_MULT(PIPE_A), D_ALL); ++ MMIO_D(PIPE_MULT(PIPE_B), D_ALL); ++ MMIO_D(PIPE_MULT(PIPE_C), D_ALL); ++ ++ MMIO_D(HSW_TVIDEO_DIP_CTL(TRANSCODER_A), D_ALL); ++ MMIO_D(HSW_TVIDEO_DIP_CTL(TRANSCODER_B), D_ALL); ++ MMIO_D(HSW_TVIDEO_DIP_CTL(TRANSCODER_C), D_ALL); ++ ++ MMIO_DH(SFUSE_STRAP, D_ALL, NULL, NULL); ++ MMIO_D(SBI_ADDR, D_ALL); ++ MMIO_DH(SBI_DATA, D_ALL, sbi_data_mmio_read, NULL); ++ MMIO_DH(SBI_CTL_STAT, D_ALL, NULL, sbi_ctl_mmio_write); ++ MMIO_D(PIXCLK_GATE, D_ALL); ++ ++ MMIO_F(_MMIO(_DPA_AUX_CH_CTL), 6 * 4, 0, 0, 0, D_ALL, NULL, ++ dp_aux_ch_ctl_mmio_write); ++ ++ MMIO_DH(DDI_BUF_CTL(PORT_A), D_ALL, NULL, ddi_buf_ctl_mmio_write); ++ MMIO_DH(DDI_BUF_CTL(PORT_B), D_ALL, NULL, ddi_buf_ctl_mmio_write); ++ MMIO_DH(DDI_BUF_CTL(PORT_C), D_ALL, NULL, ddi_buf_ctl_mmio_write); ++ MMIO_DH(DDI_BUF_CTL(PORT_D), D_ALL, NULL, ddi_buf_ctl_mmio_write); ++ MMIO_DH(DDI_BUF_CTL(PORT_E), D_ALL, NULL, ddi_buf_ctl_mmio_write); ++ ++ MMIO_DH(DP_TP_CTL(PORT_A), D_ALL, NULL, dp_tp_ctl_mmio_write); ++ MMIO_DH(DP_TP_CTL(PORT_B), D_ALL, NULL, dp_tp_ctl_mmio_write); ++ MMIO_DH(DP_TP_CTL(PORT_C), D_ALL, NULL, dp_tp_ctl_mmio_write); ++ MMIO_DH(DP_TP_CTL(PORT_D), D_ALL, NULL, dp_tp_ctl_mmio_write); ++ MMIO_DH(DP_TP_CTL(PORT_E), D_ALL, NULL, dp_tp_ctl_mmio_write); ++ ++ MMIO_DH(DP_TP_STATUS(PORT_A), D_ALL, NULL, dp_tp_status_mmio_write); ++ MMIO_DH(DP_TP_STATUS(PORT_B), D_ALL, NULL, dp_tp_status_mmio_write); ++ MMIO_DH(DP_TP_STATUS(PORT_C), D_ALL, NULL, dp_tp_status_mmio_write); ++ MMIO_DH(DP_TP_STATUS(PORT_D), D_ALL, NULL, dp_tp_status_mmio_write); ++ MMIO_DH(DP_TP_STATUS(PORT_E), D_ALL, NULL, NULL); ++ ++ MMIO_F(_MMIO(_DDI_BUF_TRANS_A), 0x50, 0, 0, 0, D_ALL, NULL, NULL); ++ MMIO_F(_MMIO(0x64e60), 0x50, 0, 0, 0, D_ALL, NULL, NULL); ++ MMIO_F(_MMIO(0x64eC0), 0x50, 0, 0, 0, D_ALL, NULL, NULL); ++ MMIO_F(_MMIO(0x64f20), 0x50, 0, 0, 0, D_ALL, NULL, NULL); ++ MMIO_F(_MMIO(0x64f80), 0x50, 0, 0, 0, D_ALL, NULL, NULL); ++ ++ MMIO_D(HSW_AUD_CFG(PIPE_A), D_ALL); ++ MMIO_D(HSW_AUD_PIN_ELD_CP_VLD, D_ALL); ++ MMIO_D(HSW_AUD_MISC_CTRL(PIPE_A), D_ALL); ++ ++ MMIO_DH(_MMIO(_TRANS_DDI_FUNC_CTL_A), D_ALL, NULL, NULL); ++ MMIO_DH(_MMIO(_TRANS_DDI_FUNC_CTL_B), D_ALL, NULL, NULL); ++ MMIO_DH(_MMIO(_TRANS_DDI_FUNC_CTL_C), D_ALL, NULL, NULL); ++ MMIO_DH(_MMIO(_TRANS_DDI_FUNC_CTL_EDP), D_ALL, NULL, NULL); ++ ++ MMIO_D(_MMIO(_TRANSA_MSA_MISC), D_ALL); ++ MMIO_D(_MMIO(_TRANSB_MSA_MISC), D_ALL); ++ MMIO_D(_MMIO(_TRANSC_MSA_MISC), D_ALL); ++ MMIO_D(_MMIO(_TRANS_EDP_MSA_MISC), D_ALL); ++ ++ MMIO_DH(FORCEWAKE, D_ALL, NULL, NULL); ++ MMIO_D(FORCEWAKE_ACK, D_ALL); ++ MMIO_D(GEN6_GT_CORE_STATUS, D_ALL); ++ MMIO_D(GEN6_GT_THREAD_STATUS_REG, D_ALL); ++ MMIO_DFH(GTFIFODBG, D_ALL, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(GTFIFOCTL, D_ALL, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DH(FORCEWAKE_MT, D_PRE_SKL, NULL, mul_force_wake_write); ++ MMIO_DH(FORCEWAKE_ACK_HSW, D_BDW, NULL, NULL); ++ MMIO_D(ECOBUS, D_ALL); ++ MMIO_DH(GEN6_RC_CONTROL, D_ALL, NULL, NULL); ++ MMIO_DH(GEN6_RC_STATE, D_ALL, NULL, NULL); ++ MMIO_D(GEN6_RPNSWREQ, D_ALL); ++ MMIO_D(GEN6_RC_VIDEO_FREQ, D_ALL); ++ MMIO_D(GEN6_RP_DOWN_TIMEOUT, D_ALL); ++ MMIO_D(GEN6_RP_INTERRUPT_LIMITS, D_ALL); ++ MMIO_D(GEN6_RPSTAT1, D_ALL); ++ MMIO_D(GEN6_RP_CONTROL, D_ALL); ++ MMIO_D(GEN6_RP_UP_THRESHOLD, D_ALL); ++ MMIO_D(GEN6_RP_DOWN_THRESHOLD, D_ALL); ++ MMIO_D(GEN6_RP_CUR_UP_EI, D_ALL); ++ MMIO_D(GEN6_RP_CUR_UP, D_ALL); ++ MMIO_D(GEN6_RP_PREV_UP, D_ALL); ++ MMIO_D(GEN6_RP_CUR_DOWN_EI, D_ALL); ++ MMIO_D(GEN6_RP_CUR_DOWN, D_ALL); ++ MMIO_D(GEN6_RP_PREV_DOWN, D_ALL); ++ MMIO_D(GEN6_RP_UP_EI, D_ALL); ++ MMIO_D(GEN6_RP_DOWN_EI, D_ALL); ++ MMIO_D(GEN6_RP_IDLE_HYSTERSIS, D_ALL); ++ MMIO_D(GEN6_RC1_WAKE_RATE_LIMIT, D_ALL); ++ MMIO_D(GEN6_RC6_WAKE_RATE_LIMIT, D_ALL); ++ MMIO_D(GEN6_RC6pp_WAKE_RATE_LIMIT, D_ALL); ++ MMIO_D(GEN6_RC_EVALUATION_INTERVAL, D_ALL); ++ MMIO_D(GEN6_RC_IDLE_HYSTERSIS, D_ALL); ++ MMIO_D(GEN6_RC_SLEEP, D_ALL); ++ MMIO_D(GEN6_RC1e_THRESHOLD, D_ALL); ++ MMIO_D(GEN6_RC6_THRESHOLD, D_ALL); ++ MMIO_D(GEN6_RC6p_THRESHOLD, D_ALL); ++ MMIO_D(GEN6_RC6pp_THRESHOLD, D_ALL); ++ MMIO_D(GEN6_PMINTRMSK, D_ALL); ++ MMIO_DH(HSW_PWR_WELL_CTL1, D_BDW, NULL, power_well_ctl_mmio_write); ++ MMIO_DH(HSW_PWR_WELL_CTL2, D_BDW, NULL, power_well_ctl_mmio_write); ++ MMIO_DH(HSW_PWR_WELL_CTL3, D_BDW, NULL, power_well_ctl_mmio_write); ++ MMIO_DH(HSW_PWR_WELL_CTL4, D_BDW, NULL, power_well_ctl_mmio_write); ++ MMIO_DH(HSW_PWR_WELL_CTL5, D_BDW, NULL, power_well_ctl_mmio_write); ++ MMIO_DH(HSW_PWR_WELL_CTL6, D_BDW, NULL, power_well_ctl_mmio_write); ++ ++ MMIO_D(RSTDBYCTL, D_ALL); ++ ++ MMIO_DH(GEN6_GDRST, D_ALL, NULL, gdrst_mmio_write); ++ MMIO_F(FENCE_REG_GEN6_LO(0), 0x80, 0, 0, 0, D_ALL, fence_mmio_read, fence_mmio_write); ++ MMIO_DH(CPU_VGACNTRL, D_ALL, NULL, vga_control_mmio_write); ++ ++ MMIO_D(TILECTL, D_ALL); ++ ++ MMIO_D(GEN6_UCGCTL1, D_ALL); ++ MMIO_D(GEN6_UCGCTL2, D_ALL); ++ ++ MMIO_F(_MMIO(0x4f000), 0x90, 0, 0, 0, D_ALL, NULL, NULL); ++ ++ MMIO_D(GEN6_PCODE_DATA, D_ALL); ++ MMIO_D(_MMIO(0x13812c), D_ALL); ++ MMIO_DH(GEN7_ERR_INT, D_ALL, NULL, NULL); ++ MMIO_D(HSW_EDRAM_CAP, D_ALL); ++ MMIO_D(HSW_IDICR, D_ALL); ++ MMIO_DH(GFX_FLSH_CNTL_GEN6, D_ALL, NULL, NULL); ++ ++ MMIO_D(_MMIO(0x3c), D_ALL); ++ MMIO_D(_MMIO(0x860), D_ALL); ++ MMIO_D(ECOSKPD, D_ALL); ++ MMIO_D(_MMIO(0x121d0), D_ALL); ++ MMIO_D(GEN6_BLITTER_ECOSKPD, D_ALL); ++ MMIO_D(_MMIO(0x41d0), D_ALL); ++ MMIO_D(GAC_ECO_BITS, D_ALL); ++ MMIO_D(_MMIO(0x6200), D_ALL); ++ MMIO_D(_MMIO(0x6204), D_ALL); ++ MMIO_D(_MMIO(0x6208), D_ALL); ++ MMIO_D(_MMIO(0x7118), D_ALL); ++ MMIO_D(_MMIO(0x7180), D_ALL); ++ MMIO_D(_MMIO(0x7408), D_ALL); ++ MMIO_D(_MMIO(0x7c00), D_ALL); ++ MMIO_DH(GEN6_MBCTL, D_ALL, NULL, mbctl_write); ++ MMIO_D(_MMIO(0x911c), D_ALL); ++ MMIO_D(_MMIO(0x9120), D_ALL); ++ MMIO_DFH(GEN7_UCGCTL4, D_ALL, F_CMD_ACCESS, NULL, NULL); ++ ++ MMIO_D(GAB_CTL, D_ALL); ++ MMIO_D(_MMIO(0x48800), D_ALL); ++ MMIO_D(_MMIO(0xce044), D_ALL); ++ MMIO_D(_MMIO(0xe6500), D_ALL); ++ MMIO_D(_MMIO(0xe6504), D_ALL); ++ MMIO_D(_MMIO(0xe6600), D_ALL); ++ MMIO_D(_MMIO(0xe6604), D_ALL); ++ MMIO_D(_MMIO(0xe6700), D_ALL); ++ MMIO_D(_MMIO(0xe6704), D_ALL); ++ MMIO_D(_MMIO(0xe6800), D_ALL); ++ MMIO_D(_MMIO(0xe6804), D_ALL); ++ MMIO_D(PCH_GMBUS4, D_ALL); ++ MMIO_D(PCH_GMBUS5, D_ALL); ++ ++ MMIO_D(_MMIO(0x902c), D_ALL); ++ MMIO_D(_MMIO(0xec008), D_ALL); ++ MMIO_D(_MMIO(0xec00c), D_ALL); ++ MMIO_D(_MMIO(0xec008 + 0x18), D_ALL); ++ MMIO_D(_MMIO(0xec00c + 0x18), D_ALL); ++ MMIO_D(_MMIO(0xec008 + 0x18 * 2), D_ALL); ++ MMIO_D(_MMIO(0xec00c + 0x18 * 2), D_ALL); ++ MMIO_D(_MMIO(0xec008 + 0x18 * 3), D_ALL); ++ MMIO_D(_MMIO(0xec00c + 0x18 * 3), D_ALL); ++ MMIO_D(_MMIO(0xec408), D_ALL); ++ MMIO_D(_MMIO(0xec40c), D_ALL); ++ MMIO_D(_MMIO(0xec408 + 0x18), D_ALL); ++ MMIO_D(_MMIO(0xec40c + 0x18), D_ALL); ++ MMIO_D(_MMIO(0xec408 + 0x18 * 2), D_ALL); ++ MMIO_D(_MMIO(0xec40c + 0x18 * 2), D_ALL); ++ MMIO_D(_MMIO(0xec408 + 0x18 * 3), D_ALL); ++ MMIO_D(_MMIO(0xec40c + 0x18 * 3), D_ALL); ++ MMIO_D(_MMIO(0xfc810), D_ALL); ++ MMIO_D(_MMIO(0xfc81c), D_ALL); ++ MMIO_D(_MMIO(0xfc828), D_ALL); ++ MMIO_D(_MMIO(0xfc834), D_ALL); ++ MMIO_D(_MMIO(0xfcc00), D_ALL); ++ MMIO_D(_MMIO(0xfcc0c), D_ALL); ++ MMIO_D(_MMIO(0xfcc18), D_ALL); ++ MMIO_D(_MMIO(0xfcc24), D_ALL); ++ MMIO_D(_MMIO(0xfd000), D_ALL); ++ MMIO_D(_MMIO(0xfd00c), D_ALL); ++ MMIO_D(_MMIO(0xfd018), D_ALL); ++ MMIO_D(_MMIO(0xfd024), D_ALL); ++ MMIO_D(_MMIO(0xfd034), D_ALL); ++ ++ MMIO_DH(FPGA_DBG, D_ALL, NULL, fpga_dbg_mmio_write); ++ MMIO_D(_MMIO(0x2054), D_ALL); ++ MMIO_D(_MMIO(0x12054), D_ALL); ++ MMIO_D(_MMIO(0x22054), D_ALL); ++ MMIO_D(_MMIO(0x1a054), D_ALL); ++ ++ MMIO_D(_MMIO(0x44070), D_ALL); ++ MMIO_DFH(_MMIO(0x215c), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0x2178), D_ALL, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0x217c), D_ALL, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0x12178), D_ALL, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0x1217c), D_ALL, F_CMD_ACCESS, NULL, NULL); ++ ++ MMIO_F(_MMIO(0x2290), 8, F_CMD_ACCESS, 0, 0, D_BDW_PLUS, NULL, NULL); ++ MMIO_D(_MMIO(0x2b00), D_BDW_PLUS); ++ MMIO_D(_MMIO(0x2360), D_BDW_PLUS); ++ MMIO_F(_MMIO(0x5200), 32, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); ++ MMIO_F(_MMIO(0x5240), 32, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); ++ MMIO_F(_MMIO(0x5280), 16, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); ++ ++ MMIO_DFH(_MMIO(0x1c17c), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0x1c178), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(BCS_SWCTRL, D_ALL, F_CMD_ACCESS, NULL, NULL); ++ ++ MMIO_F(HS_INVOCATION_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); ++ MMIO_F(DS_INVOCATION_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); ++ MMIO_F(IA_VERTICES_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); ++ MMIO_F(IA_PRIMITIVES_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); ++ MMIO_F(VS_INVOCATION_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); ++ MMIO_F(GS_INVOCATION_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); ++ MMIO_F(GS_PRIMITIVES_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); ++ MMIO_F(CL_INVOCATION_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); ++ MMIO_F(CL_PRIMITIVES_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); ++ MMIO_F(PS_INVOCATION_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); ++ MMIO_F(PS_DEPTH_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); ++ MMIO_DH(_MMIO(0x4260), D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler); ++ MMIO_DH(_MMIO(0x4264), D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler); ++ MMIO_DH(_MMIO(0x4268), D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler); ++ MMIO_DH(_MMIO(0x426c), D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler); ++ MMIO_DH(_MMIO(0x4270), D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler); ++ MMIO_DFH(_MMIO(0x4094), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); ++ ++ MMIO_DFH(ARB_MODE, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); ++ MMIO_RING_GM_RDR(RING_BBADDR, D_ALL, NULL, NULL); ++ MMIO_DFH(_MMIO(0x2220), D_ALL, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0x12220), D_ALL, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0x22220), D_ALL, F_CMD_ACCESS, NULL, NULL); ++ MMIO_RING_DFH(RING_SYNC_1, D_ALL, F_CMD_ACCESS, NULL, NULL); ++ MMIO_RING_DFH(RING_SYNC_0, D_ALL, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0x22178), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0x1a178), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0x1a17c), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0x2217c), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); ++ ++ MMIO_DH(EDP_PSR_IMR, D_BDW_PLUS, NULL, edp_psr_imr_iir_write); ++ MMIO_DH(EDP_PSR_IIR, D_BDW_PLUS, NULL, edp_psr_imr_iir_write); ++ return 0; ++} ++ ++static int init_broadwell_mmio_info(struct intel_gvt *gvt) ++{ ++ struct drm_i915_private *dev_priv = gvt->dev_priv; ++ int ret; ++ ++ MMIO_DH(GEN8_GT_IMR(0), D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler); ++ MMIO_DH(GEN8_GT_IER(0), D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler); ++ MMIO_DH(GEN8_GT_IIR(0), D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler); ++ MMIO_D(GEN8_GT_ISR(0), D_BDW_PLUS); ++ ++ MMIO_DH(GEN8_GT_IMR(1), D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler); ++ MMIO_DH(GEN8_GT_IER(1), D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler); ++ MMIO_DH(GEN8_GT_IIR(1), D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler); ++ MMIO_D(GEN8_GT_ISR(1), D_BDW_PLUS); ++ ++ MMIO_DH(GEN8_GT_IMR(2), D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler); ++ MMIO_DH(GEN8_GT_IER(2), D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler); ++ MMIO_DH(GEN8_GT_IIR(2), D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler); ++ MMIO_D(GEN8_GT_ISR(2), D_BDW_PLUS); ++ ++ MMIO_DH(GEN8_GT_IMR(3), D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler); ++ MMIO_DH(GEN8_GT_IER(3), D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler); ++ MMIO_DH(GEN8_GT_IIR(3), D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler); ++ MMIO_D(GEN8_GT_ISR(3), D_BDW_PLUS); ++ ++ MMIO_DH(GEN8_DE_PIPE_IMR(PIPE_A), D_BDW_PLUS, NULL, ++ intel_vgpu_reg_imr_handler); ++ MMIO_DH(GEN8_DE_PIPE_IER(PIPE_A), D_BDW_PLUS, NULL, ++ intel_vgpu_reg_ier_handler); ++ MMIO_DH(GEN8_DE_PIPE_IIR(PIPE_A), D_BDW_PLUS, NULL, ++ intel_vgpu_reg_iir_handler); ++ MMIO_D(GEN8_DE_PIPE_ISR(PIPE_A), D_BDW_PLUS); ++ ++ MMIO_DH(GEN8_DE_PIPE_IMR(PIPE_B), D_BDW_PLUS, NULL, ++ intel_vgpu_reg_imr_handler); ++ MMIO_DH(GEN8_DE_PIPE_IER(PIPE_B), D_BDW_PLUS, NULL, ++ intel_vgpu_reg_ier_handler); ++ MMIO_DH(GEN8_DE_PIPE_IIR(PIPE_B), D_BDW_PLUS, NULL, ++ intel_vgpu_reg_iir_handler); ++ MMIO_D(GEN8_DE_PIPE_ISR(PIPE_B), D_BDW_PLUS); ++ ++ MMIO_DH(GEN8_DE_PIPE_IMR(PIPE_C), D_BDW_PLUS, NULL, ++ intel_vgpu_reg_imr_handler); ++ MMIO_DH(GEN8_DE_PIPE_IER(PIPE_C), D_BDW_PLUS, NULL, ++ intel_vgpu_reg_ier_handler); ++ MMIO_DH(GEN8_DE_PIPE_IIR(PIPE_C), D_BDW_PLUS, NULL, ++ intel_vgpu_reg_iir_handler); ++ MMIO_D(GEN8_DE_PIPE_ISR(PIPE_C), D_BDW_PLUS); ++ ++ MMIO_DH(GEN8_DE_PORT_IMR, D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler); ++ MMIO_DH(GEN8_DE_PORT_IER, D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler); ++ MMIO_DH(GEN8_DE_PORT_IIR, D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler); ++ MMIO_D(GEN8_DE_PORT_ISR, D_BDW_PLUS); ++ ++ MMIO_DH(GEN8_DE_MISC_IMR, D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler); ++ MMIO_DH(GEN8_DE_MISC_IER, D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler); ++ MMIO_DH(GEN8_DE_MISC_IIR, D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler); ++ MMIO_D(GEN8_DE_MISC_ISR, D_BDW_PLUS); ++ ++ MMIO_DH(GEN8_PCU_IMR, D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler); ++ MMIO_DH(GEN8_PCU_IER, D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler); ++ MMIO_DH(GEN8_PCU_IIR, D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler); ++ MMIO_D(GEN8_PCU_ISR, D_BDW_PLUS); ++ ++ MMIO_DH(GEN8_MASTER_IRQ, D_BDW_PLUS, NULL, ++ intel_vgpu_reg_master_irq_handler); ++ ++ MMIO_RING_DFH(RING_ACTHD_UDW, D_BDW_PLUS, F_CMD_ACCESS, ++ mmio_read_from_hw, NULL); ++ ++#define RING_REG(base) _MMIO((base) + 0xd0) ++ MMIO_RING_F(RING_REG, 4, F_RO, 0, ++ ~_MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET), D_BDW_PLUS, NULL, ++ ring_reset_ctl_write); ++#undef RING_REG ++ ++#define RING_REG(base) _MMIO((base) + 0x230) ++ MMIO_RING_DFH(RING_REG, D_BDW_PLUS, 0, NULL, elsp_mmio_write); ++#undef RING_REG ++ ++#define RING_REG(base) _MMIO((base) + 0x234) ++ MMIO_RING_F(RING_REG, 8, F_RO | F_CMD_ACCESS, 0, ~0, D_BDW_PLUS, ++ NULL, NULL); ++#undef RING_REG ++ ++#define RING_REG(base) _MMIO((base) + 0x244) ++ MMIO_RING_DFH(RING_REG, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); ++#undef RING_REG ++ ++#define RING_REG(base) _MMIO((base) + 0x370) ++ MMIO_RING_F(RING_REG, 48, F_RO, 0, ~0, D_BDW_PLUS, NULL, NULL); ++#undef RING_REG ++ ++#define RING_REG(base) _MMIO((base) + 0x3a0) ++ MMIO_RING_DFH(RING_REG, D_BDW_PLUS, F_MODE_MASK, NULL, NULL); ++#undef RING_REG ++ ++ MMIO_D(PIPEMISC(PIPE_A), D_BDW_PLUS); ++ MMIO_D(PIPEMISC(PIPE_B), D_BDW_PLUS); ++ MMIO_D(PIPEMISC(PIPE_C), D_BDW_PLUS); ++ MMIO_D(_MMIO(0x1c1d0), D_BDW_PLUS); ++ MMIO_D(GEN6_MBCUNIT_SNPCR, D_BDW_PLUS); ++ MMIO_D(GEN7_MISCCPCTL, D_BDW_PLUS); ++ MMIO_D(_MMIO(0x1c054), D_BDW_PLUS); ++ ++ MMIO_DH(GEN6_PCODE_MAILBOX, D_BDW_PLUS, NULL, mailbox_write); ++ ++ MMIO_D(GEN8_PRIVATE_PAT_LO, D_BDW_PLUS); ++ MMIO_D(GEN8_PRIVATE_PAT_HI, D_BDW_PLUS); ++ ++ MMIO_D(GAMTARBMODE, D_BDW_PLUS); ++ ++#define RING_REG(base) _MMIO((base) + 0x270) ++ MMIO_RING_F(RING_REG, 32, 0, 0, 0, D_BDW_PLUS, NULL, NULL); ++#undef RING_REG ++ ++ MMIO_RING_GM_RDR(RING_HWS_PGA, D_BDW_PLUS, NULL, hws_pga_write); ++ ++ MMIO_DFH(HDC_CHICKEN0, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); ++ ++ MMIO_D(CHICKEN_PIPESL_1(PIPE_A), D_BDW_PLUS); ++ MMIO_D(CHICKEN_PIPESL_1(PIPE_B), D_BDW_PLUS); ++ MMIO_D(CHICKEN_PIPESL_1(PIPE_C), D_BDW_PLUS); ++ ++ MMIO_D(WM_MISC, D_BDW); ++ MMIO_D(_MMIO(BDW_EDP_PSR_BASE), D_BDW); ++ ++ MMIO_D(_MMIO(0x6671c), D_BDW_PLUS); ++ MMIO_D(_MMIO(0x66c00), D_BDW_PLUS); ++ MMIO_D(_MMIO(0x66c04), D_BDW_PLUS); ++ ++ MMIO_D(HSW_GTT_CACHE_EN, D_BDW_PLUS); ++ ++ MMIO_D(GEN8_EU_DISABLE0, D_BDW_PLUS); ++ MMIO_D(GEN8_EU_DISABLE1, D_BDW_PLUS); ++ MMIO_D(GEN8_EU_DISABLE2, D_BDW_PLUS); ++ ++ MMIO_D(_MMIO(0xfdc), D_BDW_PLUS); ++ MMIO_DFH(GEN8_ROW_CHICKEN, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, ++ NULL, NULL); ++ MMIO_DFH(GEN7_ROW_CHICKEN2, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, ++ NULL, NULL); ++ MMIO_DFH(GEN8_UCGCTL6, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); ++ ++ MMIO_DFH(_MMIO(0xb1f0), D_BDW, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0xb1c0), D_BDW, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(GEN8_L3SQCREG4, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0xb100), D_BDW, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0xb10c), D_BDW, F_CMD_ACCESS, NULL, NULL); ++ MMIO_D(_MMIO(0xb110), D_BDW); ++ ++ MMIO_F(_MMIO(0x24d0), 48, F_CMD_ACCESS, 0, 0, D_BDW_PLUS, ++ NULL, force_nonpriv_write); ++ ++ MMIO_D(_MMIO(0x44484), D_BDW_PLUS); ++ MMIO_D(_MMIO(0x4448c), D_BDW_PLUS); ++ ++ MMIO_DFH(_MMIO(0x83a4), D_BDW, F_CMD_ACCESS, NULL, NULL); ++ MMIO_D(GEN8_L3_LRA_1_GPGPU, D_BDW_PLUS); ++ ++ MMIO_DFH(_MMIO(0x8430), D_BDW, F_CMD_ACCESS, NULL, NULL); ++ ++ MMIO_D(_MMIO(0x110000), D_BDW_PLUS); ++ ++ MMIO_D(_MMIO(0x48400), D_BDW_PLUS); ++ ++ MMIO_D(_MMIO(0x6e570), D_BDW_PLUS); ++ MMIO_D(_MMIO(0x65f10), D_BDW_PLUS); ++ ++ MMIO_DFH(_MMIO(0xe194), D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0xe188), D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(HALF_SLICE_CHICKEN2, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0x2580), D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); ++ ++ MMIO_DFH(_MMIO(0x2248), D_BDW, F_CMD_ACCESS, NULL, NULL); ++ ++ MMIO_DFH(_MMIO(0xe220), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0xe230), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0xe240), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0xe260), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0xe270), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0xe280), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0xe2a0), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0xe2b0), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0xe2c0), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(_MMIO(0x21f0), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); ++ return 0; ++} ++ ++static int init_skl_mmio_info(struct intel_gvt *gvt) ++{ ++ struct drm_i915_private *dev_priv = gvt->dev_priv; ++ int ret; ++ ++ MMIO_DH(FORCEWAKE_RENDER_GEN9, D_SKL_PLUS, NULL, mul_force_wake_write); ++ MMIO_DH(FORCEWAKE_ACK_RENDER_GEN9, D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(FORCEWAKE_BLITTER_GEN9, D_SKL_PLUS, NULL, mul_force_wake_write); ++ MMIO_DH(FORCEWAKE_ACK_BLITTER_GEN9, D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(FORCEWAKE_MEDIA_GEN9, D_SKL_PLUS, NULL, mul_force_wake_write); ++ MMIO_DH(FORCEWAKE_ACK_MEDIA_GEN9, D_SKL_PLUS, NULL, NULL); ++ ++ MMIO_F(_MMIO(_DPB_AUX_CH_CTL), 6 * 4, 0, 0, 0, D_SKL_PLUS, NULL, ++ dp_aux_ch_ctl_mmio_write); ++ MMIO_F(_MMIO(_DPC_AUX_CH_CTL), 6 * 4, 0, 0, 0, D_SKL_PLUS, NULL, ++ dp_aux_ch_ctl_mmio_write); ++ MMIO_F(_MMIO(_DPD_AUX_CH_CTL), 6 * 4, 0, 0, 0, D_SKL_PLUS, NULL, ++ dp_aux_ch_ctl_mmio_write); ++ ++ MMIO_D(HSW_PWR_WELL_CTL1, D_SKL_PLUS); ++ MMIO_DH(HSW_PWR_WELL_CTL2, D_SKL_PLUS, NULL, skl_power_well_ctl_write); ++ ++ MMIO_DH(DBUF_CTL, D_SKL_PLUS, NULL, gen9_dbuf_ctl_mmio_write); ++ ++ MMIO_D(GEN9_PG_ENABLE, D_SKL_PLUS); ++ MMIO_D(GEN9_MEDIA_PG_IDLE_HYSTERESIS, D_SKL_PLUS); ++ MMIO_D(GEN9_RENDER_PG_IDLE_HYSTERESIS, D_SKL_PLUS); ++ MMIO_DFH(GEN9_GAMT_ECO_REG_RW_IA, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DH(MMCD_MISC_CTRL, D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(CHICKEN_PAR1_1, D_SKL_PLUS, NULL, NULL); ++ MMIO_D(DC_STATE_EN, D_SKL_PLUS); ++ MMIO_D(DC_STATE_DEBUG, D_SKL_PLUS); ++ MMIO_D(CDCLK_CTL, D_SKL_PLUS); ++ MMIO_DH(LCPLL1_CTL, D_SKL_PLUS, NULL, skl_lcpll_write); ++ MMIO_DH(LCPLL2_CTL, D_SKL_PLUS, NULL, skl_lcpll_write); ++ MMIO_D(_MMIO(_DPLL1_CFGCR1), D_SKL_PLUS); ++ MMIO_D(_MMIO(_DPLL2_CFGCR1), D_SKL_PLUS); ++ MMIO_D(_MMIO(_DPLL3_CFGCR1), D_SKL_PLUS); ++ MMIO_D(_MMIO(_DPLL1_CFGCR2), D_SKL_PLUS); ++ MMIO_D(_MMIO(_DPLL2_CFGCR2), D_SKL_PLUS); ++ MMIO_D(_MMIO(_DPLL3_CFGCR2), D_SKL_PLUS); ++ MMIO_D(DPLL_CTRL1, D_SKL_PLUS); ++ MMIO_D(DPLL_CTRL2, D_SKL_PLUS); ++ MMIO_DH(DPLL_STATUS, D_SKL_PLUS, dpll_status_read, NULL); ++ ++ MMIO_DH(SKL_PS_WIN_POS(PIPE_A, 0), D_SKL_PLUS, NULL, pf_write); ++ MMIO_DH(SKL_PS_WIN_POS(PIPE_A, 1), D_SKL_PLUS, NULL, pf_write); ++ MMIO_DH(SKL_PS_WIN_POS(PIPE_B, 0), D_SKL_PLUS, NULL, pf_write); ++ MMIO_DH(SKL_PS_WIN_POS(PIPE_B, 1), D_SKL_PLUS, NULL, pf_write); ++ MMIO_DH(SKL_PS_WIN_POS(PIPE_C, 0), D_SKL_PLUS, NULL, pf_write); ++ MMIO_DH(SKL_PS_WIN_POS(PIPE_C, 1), D_SKL_PLUS, NULL, pf_write); ++ ++ MMIO_DH(SKL_PS_WIN_SZ(PIPE_A, 0), D_SKL_PLUS, NULL, pf_write); ++ MMIO_DH(SKL_PS_WIN_SZ(PIPE_A, 1), D_SKL_PLUS, NULL, pf_write); ++ MMIO_DH(SKL_PS_WIN_SZ(PIPE_B, 0), D_SKL_PLUS, NULL, pf_write); ++ MMIO_DH(SKL_PS_WIN_SZ(PIPE_B, 1), D_SKL_PLUS, NULL, pf_write); ++ MMIO_DH(SKL_PS_WIN_SZ(PIPE_C, 0), D_SKL_PLUS, NULL, pf_write); ++ MMIO_DH(SKL_PS_WIN_SZ(PIPE_C, 1), D_SKL_PLUS, NULL, pf_write); ++ ++ MMIO_DH(SKL_PS_CTRL(PIPE_A, 0), D_SKL_PLUS, NULL, pf_write); ++ MMIO_DH(SKL_PS_CTRL(PIPE_A, 1), D_SKL_PLUS, NULL, pf_write); ++ MMIO_DH(SKL_PS_CTRL(PIPE_B, 0), D_SKL_PLUS, NULL, pf_write); ++ MMIO_DH(SKL_PS_CTRL(PIPE_B, 1), D_SKL_PLUS, NULL, pf_write); ++ MMIO_DH(SKL_PS_CTRL(PIPE_C, 0), D_SKL_PLUS, NULL, pf_write); ++ MMIO_DH(SKL_PS_CTRL(PIPE_C, 1), D_SKL_PLUS, NULL, pf_write); ++ ++ MMIO_DH(PLANE_BUF_CFG(PIPE_A, 0), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(PLANE_BUF_CFG(PIPE_A, 1), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(PLANE_BUF_CFG(PIPE_A, 2), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(PLANE_BUF_CFG(PIPE_A, 3), D_SKL_PLUS, NULL, NULL); ++ ++ MMIO_DH(PLANE_BUF_CFG(PIPE_B, 0), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(PLANE_BUF_CFG(PIPE_B, 1), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(PLANE_BUF_CFG(PIPE_B, 2), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(PLANE_BUF_CFG(PIPE_B, 3), D_SKL_PLUS, NULL, NULL); ++ ++ MMIO_DH(PLANE_BUF_CFG(PIPE_C, 0), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(PLANE_BUF_CFG(PIPE_C, 1), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(PLANE_BUF_CFG(PIPE_C, 2), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(PLANE_BUF_CFG(PIPE_C, 3), D_SKL_PLUS, NULL, NULL); ++ ++ MMIO_DH(CUR_BUF_CFG(PIPE_A), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(CUR_BUF_CFG(PIPE_B), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(CUR_BUF_CFG(PIPE_C), D_SKL_PLUS, NULL, NULL); ++ ++ MMIO_F(PLANE_WM(PIPE_A, 0, 0), 4 * 8, 0, 0, 0, D_SKL_PLUS, NULL, NULL); ++ MMIO_F(PLANE_WM(PIPE_A, 1, 0), 4 * 8, 0, 0, 0, D_SKL_PLUS, NULL, NULL); ++ MMIO_F(PLANE_WM(PIPE_A, 2, 0), 4 * 8, 0, 0, 0, D_SKL_PLUS, NULL, NULL); ++ ++ MMIO_F(PLANE_WM(PIPE_B, 0, 0), 4 * 8, 0, 0, 0, D_SKL_PLUS, NULL, NULL); ++ MMIO_F(PLANE_WM(PIPE_B, 1, 0), 4 * 8, 0, 0, 0, D_SKL_PLUS, NULL, NULL); ++ MMIO_F(PLANE_WM(PIPE_B, 2, 0), 4 * 8, 0, 0, 0, D_SKL_PLUS, NULL, NULL); ++ ++ MMIO_F(PLANE_WM(PIPE_C, 0, 0), 4 * 8, 0, 0, 0, D_SKL_PLUS, NULL, NULL); ++ MMIO_F(PLANE_WM(PIPE_C, 1, 0), 4 * 8, 0, 0, 0, D_SKL_PLUS, NULL, NULL); ++ MMIO_F(PLANE_WM(PIPE_C, 2, 0), 4 * 8, 0, 0, 0, D_SKL_PLUS, NULL, NULL); ++ ++ MMIO_F(CUR_WM(PIPE_A, 0), 4 * 8, 0, 0, 0, D_SKL_PLUS, NULL, NULL); ++ MMIO_F(CUR_WM(PIPE_B, 0), 4 * 8, 0, 0, 0, D_SKL_PLUS, NULL, NULL); ++ MMIO_F(CUR_WM(PIPE_C, 0), 4 * 8, 0, 0, 0, D_SKL_PLUS, NULL, NULL); ++ ++ MMIO_DH(PLANE_WM_TRANS(PIPE_A, 0), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(PLANE_WM_TRANS(PIPE_A, 1), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(PLANE_WM_TRANS(PIPE_A, 2), D_SKL_PLUS, NULL, NULL); ++ ++ MMIO_DH(PLANE_WM_TRANS(PIPE_B, 0), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(PLANE_WM_TRANS(PIPE_B, 1), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(PLANE_WM_TRANS(PIPE_B, 2), D_SKL_PLUS, NULL, NULL); ++ ++ MMIO_DH(PLANE_WM_TRANS(PIPE_C, 0), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(PLANE_WM_TRANS(PIPE_C, 1), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(PLANE_WM_TRANS(PIPE_C, 2), D_SKL_PLUS, NULL, NULL); ++ ++ MMIO_DH(CUR_WM_TRANS(PIPE_A), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(CUR_WM_TRANS(PIPE_B), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(CUR_WM_TRANS(PIPE_C), D_SKL_PLUS, NULL, NULL); ++ ++ MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_A, 0), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_A, 1), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_A, 2), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_A, 3), D_SKL_PLUS, NULL, NULL); ++ ++ MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_B, 0), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_B, 1), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_B, 2), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_B, 3), D_SKL_PLUS, NULL, NULL); ++ ++ MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_C, 0), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_C, 1), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_C, 2), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_C, 3), D_SKL_PLUS, NULL, NULL); ++ ++ MMIO_DH(_MMIO(_REG_701C0(PIPE_A, 1)), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(_MMIO(_REG_701C0(PIPE_A, 2)), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(_MMIO(_REG_701C0(PIPE_A, 3)), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(_MMIO(_REG_701C0(PIPE_A, 4)), D_SKL_PLUS, NULL, NULL); ++ ++ MMIO_DH(_MMIO(_REG_701C0(PIPE_B, 1)), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(_MMIO(_REG_701C0(PIPE_B, 2)), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(_MMIO(_REG_701C0(PIPE_B, 3)), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(_MMIO(_REG_701C0(PIPE_B, 4)), D_SKL_PLUS, NULL, NULL); ++ ++ MMIO_DH(_MMIO(_REG_701C0(PIPE_C, 1)), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(_MMIO(_REG_701C0(PIPE_C, 2)), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(_MMIO(_REG_701C0(PIPE_C, 3)), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(_MMIO(_REG_701C0(PIPE_C, 4)), D_SKL_PLUS, NULL, NULL); ++ ++ MMIO_DH(_MMIO(_REG_701C4(PIPE_A, 1)), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(_MMIO(_REG_701C4(PIPE_A, 2)), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(_MMIO(_REG_701C4(PIPE_A, 3)), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(_MMIO(_REG_701C4(PIPE_A, 4)), D_SKL_PLUS, NULL, NULL); ++ ++ MMIO_DH(_MMIO(_REG_701C4(PIPE_B, 1)), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(_MMIO(_REG_701C4(PIPE_B, 2)), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(_MMIO(_REG_701C4(PIPE_B, 3)), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(_MMIO(_REG_701C4(PIPE_B, 4)), D_SKL_PLUS, NULL, NULL); ++ ++ MMIO_DH(_MMIO(_REG_701C4(PIPE_C, 1)), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(_MMIO(_REG_701C4(PIPE_C, 2)), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(_MMIO(_REG_701C4(PIPE_C, 3)), D_SKL_PLUS, NULL, NULL); ++ MMIO_DH(_MMIO(_REG_701C4(PIPE_C, 4)), D_SKL_PLUS, NULL, NULL); ++ ++ MMIO_D(_MMIO(_PLANE_CTL_3_A), D_SKL_PLUS); ++ MMIO_D(_MMIO(_PLANE_CTL_3_B), D_SKL_PLUS); ++ MMIO_D(_MMIO(0x72380), D_SKL_PLUS); ++ MMIO_D(_MMIO(0x7239c), D_SKL_PLUS); ++ MMIO_D(_MMIO(_PLANE_SURF_3_A), D_SKL_PLUS); ++ ++ MMIO_D(CSR_SSP_BASE, D_SKL_PLUS); ++ MMIO_D(CSR_HTP_SKL, D_SKL_PLUS); ++ MMIO_D(CSR_LAST_WRITE, D_SKL_PLUS); ++ ++ MMIO_DFH(BDW_SCRATCH1, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); ++ ++ MMIO_D(SKL_DFSM, D_SKL_PLUS); ++ MMIO_D(DISPIO_CR_TX_BMU_CR0, D_SKL_PLUS); ++ ++ MMIO_F(GEN9_GFX_MOCS(0), 0x7f8, F_CMD_ACCESS, 0, 0, D_SKL_PLUS, ++ NULL, NULL); ++ MMIO_F(GEN7_L3CNTLREG2, 0x80, F_CMD_ACCESS, 0, 0, D_SKL_PLUS, ++ NULL, NULL); ++ ++ MMIO_D(RPM_CONFIG0, D_SKL_PLUS); ++ MMIO_D(_MMIO(0xd08), D_SKL_PLUS); ++ MMIO_D(RC6_LOCATION, D_SKL_PLUS); ++ MMIO_DFH(GEN7_FF_SLICE_CS_CHICKEN1, D_SKL_PLUS, ++ F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(GEN9_CS_DEBUG_MODE1, D_SKL_PLUS, F_MODE_MASK | F_CMD_ACCESS, ++ NULL, NULL); ++ ++ /* TRTT */ ++ MMIO_DFH(TRVATTL3PTRDW(0), D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(TRVATTL3PTRDW(1), D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(TRVATTL3PTRDW(2), D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(TRVATTL3PTRDW(3), D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(TRVADR, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DFH(TRTTE, D_SKL_PLUS, F_CMD_ACCESS, ++ NULL, gen9_trtte_write); ++ MMIO_DH(_MMIO(0x4dfc), D_SKL_PLUS, NULL, gen9_trtt_chicken_write); ++ ++ MMIO_D(_MMIO(0x46430), D_SKL_PLUS); ++ ++ MMIO_D(_MMIO(0x46520), D_SKL_PLUS); ++ ++ MMIO_D(_MMIO(0xc403c), D_SKL_PLUS); ++ MMIO_DFH(GEN8_GARBCNTL, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); ++ MMIO_DH(DMA_CTRL, D_SKL_PLUS, NULL, dma_ctrl_write); ++ ++ MMIO_D(_MMIO(0x65900), D_SKL_PLUS); ++ MMIO_D(GEN6_STOLEN_RESERVED, D_SKL_PLUS); ++ MMIO_D(_MMIO(0x4068), D_SKL_PLUS); ++ MMIO_D(_MMIO(0x67054), D_SKL_PLUS); ++ MMIO_D(_MMIO(0x6e560), D_SKL_PLUS); ++ MMIO_D(_MMIO(0x6e554), D_SKL_PLUS); ++ MMIO_D(_MMIO(0x2b20), D_SKL_PLUS); ++ MMIO_D(_MMIO(0x65f00), D_SKL_PLUS); ++ MMIO_D(_MMIO(0x65f08), D_SKL_PLUS); ++ MMIO_D(_MMIO(0x320f0), D_SKL_PLUS); ++ ++ MMIO_D(_MMIO(0x70034), D_SKL_PLUS); ++ MMIO_D(_MMIO(0x71034), D_SKL_PLUS); ++ MMIO_D(_MMIO(0x72034), D_SKL_PLUS); ++ ++ MMIO_D(_MMIO(_PLANE_KEYVAL_1(PIPE_A)), D_SKL_PLUS); ++ MMIO_D(_MMIO(_PLANE_KEYVAL_1(PIPE_B)), D_SKL_PLUS); ++ MMIO_D(_MMIO(_PLANE_KEYVAL_1(PIPE_C)), D_SKL_PLUS); ++ MMIO_D(_MMIO(_PLANE_KEYMAX_1(PIPE_A)), D_SKL_PLUS); ++ MMIO_D(_MMIO(_PLANE_KEYMAX_1(PIPE_B)), D_SKL_PLUS); ++ MMIO_D(_MMIO(_PLANE_KEYMAX_1(PIPE_C)), D_SKL_PLUS); ++ MMIO_D(_MMIO(_PLANE_KEYMSK_1(PIPE_A)), D_SKL_PLUS); ++ MMIO_D(_MMIO(_PLANE_KEYMSK_1(PIPE_B)), D_SKL_PLUS); ++ MMIO_D(_MMIO(_PLANE_KEYMSK_1(PIPE_C)), D_SKL_PLUS); ++ ++ MMIO_D(_MMIO(0x44500), D_SKL_PLUS); ++#define CSFE_CHICKEN1_REG(base) _MMIO((base) + 0xD4) ++ MMIO_RING_DFH(CSFE_CHICKEN1_REG, D_SKL_PLUS, F_MODE_MASK | F_CMD_ACCESS, ++ NULL, csfe_chicken1_mmio_write); ++#undef CSFE_CHICKEN1_REG ++ MMIO_DFH(GEN8_HDC_CHICKEN1, D_SKL_PLUS, F_MODE_MASK | F_CMD_ACCESS, ++ NULL, NULL); ++ MMIO_DFH(GEN9_WM_CHICKEN3, D_SKL_PLUS, F_MODE_MASK | F_CMD_ACCESS, ++ NULL, NULL); ++ ++ MMIO_D(GAMT_CHKN_BIT_REG, D_KBL); ++ MMIO_D(GEN9_CTX_PREEMPT_REG, D_KBL | D_SKL); ++ ++ return 0; ++} ++ ++static int init_bxt_mmio_info(struct intel_gvt *gvt) ++{ ++ struct drm_i915_private *dev_priv = gvt->dev_priv; ++ int ret; ++ ++ MMIO_F(_MMIO(0x80000), 0x3000, 0, 0, 0, D_BXT, NULL, NULL); ++ ++ MMIO_D(GEN7_SAMPLER_INSTDONE, D_BXT); ++ MMIO_D(GEN7_ROW_INSTDONE, D_BXT); ++ MMIO_D(GEN8_FAULT_TLB_DATA0, D_BXT); ++ MMIO_D(GEN8_FAULT_TLB_DATA1, D_BXT); ++ MMIO_D(ERROR_GEN6, D_BXT); ++ MMIO_D(DONE_REG, D_BXT); ++ MMIO_D(EIR, D_BXT); ++ MMIO_D(PGTBL_ER, D_BXT); ++ MMIO_D(_MMIO(0x4194), D_BXT); ++ MMIO_D(_MMIO(0x4294), D_BXT); ++ MMIO_D(_MMIO(0x4494), D_BXT); ++ ++ MMIO_RING_D(RING_PSMI_CTL, D_BXT); ++ MMIO_RING_D(RING_DMA_FADD, D_BXT); ++ MMIO_RING_D(RING_DMA_FADD_UDW, D_BXT); ++ MMIO_RING_D(RING_IPEHR, D_BXT); ++ MMIO_RING_D(RING_INSTPS, D_BXT); ++ MMIO_RING_D(RING_BBADDR_UDW, D_BXT); ++ MMIO_RING_D(RING_BBSTATE, D_BXT); ++ MMIO_RING_D(RING_IPEIR, D_BXT); ++ ++ MMIO_F(SOFT_SCRATCH(0), 16 * 4, 0, 0, 0, D_BXT, NULL, NULL); ++ ++ MMIO_DH(BXT_P_CR_GT_DISP_PWRON, D_BXT, NULL, bxt_gt_disp_pwron_write); ++ MMIO_D(BXT_RP_STATE_CAP, D_BXT); ++ MMIO_DH(BXT_PHY_CTL_FAMILY(DPIO_PHY0), D_BXT, ++ NULL, bxt_phy_ctl_family_write); ++ MMIO_DH(BXT_PHY_CTL_FAMILY(DPIO_PHY1), D_BXT, ++ NULL, bxt_phy_ctl_family_write); ++ MMIO_D(BXT_PHY_CTL(PORT_A), D_BXT); ++ MMIO_D(BXT_PHY_CTL(PORT_B), D_BXT); ++ MMIO_D(BXT_PHY_CTL(PORT_C), D_BXT); ++ MMIO_DH(BXT_PORT_PLL_ENABLE(PORT_A), D_BXT, ++ NULL, bxt_port_pll_enable_write); ++ MMIO_DH(BXT_PORT_PLL_ENABLE(PORT_B), D_BXT, ++ NULL, bxt_port_pll_enable_write); ++ MMIO_DH(BXT_PORT_PLL_ENABLE(PORT_C), D_BXT, NULL, ++ bxt_port_pll_enable_write); ++ ++ MMIO_D(BXT_PORT_CL1CM_DW0(DPIO_PHY0), D_BXT); ++ MMIO_D(BXT_PORT_CL1CM_DW9(DPIO_PHY0), D_BXT); ++ MMIO_D(BXT_PORT_CL1CM_DW10(DPIO_PHY0), D_BXT); ++ MMIO_D(BXT_PORT_CL1CM_DW28(DPIO_PHY0), D_BXT); ++ MMIO_D(BXT_PORT_CL1CM_DW30(DPIO_PHY0), D_BXT); ++ MMIO_D(BXT_PORT_CL2CM_DW6(DPIO_PHY0), D_BXT); ++ MMIO_D(BXT_PORT_REF_DW3(DPIO_PHY0), D_BXT); ++ MMIO_D(BXT_PORT_REF_DW6(DPIO_PHY0), D_BXT); ++ MMIO_D(BXT_PORT_REF_DW8(DPIO_PHY0), D_BXT); ++ ++ MMIO_D(BXT_PORT_CL1CM_DW0(DPIO_PHY1), D_BXT); ++ MMIO_D(BXT_PORT_CL1CM_DW9(DPIO_PHY1), D_BXT); ++ MMIO_D(BXT_PORT_CL1CM_DW10(DPIO_PHY1), D_BXT); ++ MMIO_D(BXT_PORT_CL1CM_DW28(DPIO_PHY1), D_BXT); ++ MMIO_D(BXT_PORT_CL1CM_DW30(DPIO_PHY1), D_BXT); ++ MMIO_D(BXT_PORT_CL2CM_DW6(DPIO_PHY1), D_BXT); ++ MMIO_D(BXT_PORT_REF_DW3(DPIO_PHY1), D_BXT); ++ MMIO_D(BXT_PORT_REF_DW6(DPIO_PHY1), D_BXT); ++ MMIO_D(BXT_PORT_REF_DW8(DPIO_PHY1), D_BXT); ++ ++ MMIO_D(BXT_PORT_PLL_EBB_0(DPIO_PHY0, DPIO_CH0), D_BXT); ++ MMIO_D(BXT_PORT_PLL_EBB_4(DPIO_PHY0, DPIO_CH0), D_BXT); ++ MMIO_D(BXT_PORT_PCS_DW10_LN01(DPIO_PHY0, DPIO_CH0), D_BXT); ++ MMIO_D(BXT_PORT_PCS_DW10_GRP(DPIO_PHY0, DPIO_CH0), D_BXT); ++ MMIO_D(BXT_PORT_PCS_DW12_LN01(DPIO_PHY0, DPIO_CH0), D_BXT); ++ MMIO_D(BXT_PORT_PCS_DW12_LN23(DPIO_PHY0, DPIO_CH0), D_BXT); ++ MMIO_DH(BXT_PORT_PCS_DW12_GRP(DPIO_PHY0, DPIO_CH0), D_BXT, ++ NULL, bxt_pcs_dw12_grp_write); ++ MMIO_D(BXT_PORT_TX_DW2_LN0(DPIO_PHY0, DPIO_CH0), D_BXT); ++ MMIO_D(BXT_PORT_TX_DW2_GRP(DPIO_PHY0, DPIO_CH0), D_BXT); ++ MMIO_DH(BXT_PORT_TX_DW3_LN0(DPIO_PHY0, DPIO_CH0), D_BXT, ++ bxt_port_tx_dw3_read, NULL); ++ MMIO_D(BXT_PORT_TX_DW3_GRP(DPIO_PHY0, DPIO_CH0), D_BXT); ++ MMIO_D(BXT_PORT_TX_DW4_LN0(DPIO_PHY0, DPIO_CH0), D_BXT); ++ MMIO_D(BXT_PORT_TX_DW4_GRP(DPIO_PHY0, DPIO_CH0), D_BXT); ++ MMIO_D(BXT_PORT_TX_DW14_LN(DPIO_PHY0, DPIO_CH0, 0), D_BXT); ++ MMIO_D(BXT_PORT_TX_DW14_LN(DPIO_PHY0, DPIO_CH0, 1), D_BXT); ++ MMIO_D(BXT_PORT_TX_DW14_LN(DPIO_PHY0, DPIO_CH0, 2), D_BXT); ++ MMIO_D(BXT_PORT_TX_DW14_LN(DPIO_PHY0, DPIO_CH0, 3), D_BXT); ++ MMIO_D(BXT_PORT_PLL(DPIO_PHY0, DPIO_CH0, 0), D_BXT); ++ MMIO_D(BXT_PORT_PLL(DPIO_PHY0, DPIO_CH0, 1), D_BXT); ++ MMIO_D(BXT_PORT_PLL(DPIO_PHY0, DPIO_CH0, 2), D_BXT); ++ MMIO_D(BXT_PORT_PLL(DPIO_PHY0, DPIO_CH0, 3), D_BXT); ++ MMIO_D(BXT_PORT_PLL(DPIO_PHY0, DPIO_CH0, 6), D_BXT); ++ MMIO_D(BXT_PORT_PLL(DPIO_PHY0, DPIO_CH0, 8), D_BXT); ++ MMIO_D(BXT_PORT_PLL(DPIO_PHY0, DPIO_CH0, 9), D_BXT); ++ MMIO_D(BXT_PORT_PLL(DPIO_PHY0, DPIO_CH0, 10), D_BXT); ++ ++ MMIO_D(BXT_PORT_PLL_EBB_0(DPIO_PHY0, DPIO_CH1), D_BXT); ++ MMIO_D(BXT_PORT_PLL_EBB_4(DPIO_PHY0, DPIO_CH1), D_BXT); ++ MMIO_D(BXT_PORT_PCS_DW10_LN01(DPIO_PHY0, DPIO_CH1), D_BXT); ++ MMIO_D(BXT_PORT_PCS_DW10_GRP(DPIO_PHY0, DPIO_CH1), D_BXT); ++ MMIO_D(BXT_PORT_PCS_DW12_LN01(DPIO_PHY0, DPIO_CH1), D_BXT); ++ MMIO_D(BXT_PORT_PCS_DW12_LN23(DPIO_PHY0, DPIO_CH1), D_BXT); ++ MMIO_DH(BXT_PORT_PCS_DW12_GRP(DPIO_PHY0, DPIO_CH1), D_BXT, ++ NULL, bxt_pcs_dw12_grp_write); ++ MMIO_D(BXT_PORT_TX_DW2_LN0(DPIO_PHY0, DPIO_CH1), D_BXT); ++ MMIO_D(BXT_PORT_TX_DW2_GRP(DPIO_PHY0, DPIO_CH1), D_BXT); ++ MMIO_DH(BXT_PORT_TX_DW3_LN0(DPIO_PHY0, DPIO_CH1), D_BXT, ++ bxt_port_tx_dw3_read, NULL); ++ MMIO_D(BXT_PORT_TX_DW3_GRP(DPIO_PHY0, DPIO_CH1), D_BXT); ++ MMIO_D(BXT_PORT_TX_DW4_LN0(DPIO_PHY0, DPIO_CH1), D_BXT); ++ MMIO_D(BXT_PORT_TX_DW4_GRP(DPIO_PHY0, DPIO_CH1), D_BXT); ++ MMIO_D(BXT_PORT_TX_DW14_LN(DPIO_PHY0, DPIO_CH1, 0), D_BXT); ++ MMIO_D(BXT_PORT_TX_DW14_LN(DPIO_PHY0, DPIO_CH1, 1), D_BXT); ++ MMIO_D(BXT_PORT_TX_DW14_LN(DPIO_PHY0, DPIO_CH1, 2), D_BXT); ++ MMIO_D(BXT_PORT_TX_DW14_LN(DPIO_PHY0, DPIO_CH1, 3), D_BXT); ++ MMIO_D(BXT_PORT_PLL(DPIO_PHY0, DPIO_CH1, 0), D_BXT); ++ MMIO_D(BXT_PORT_PLL(DPIO_PHY0, DPIO_CH1, 1), D_BXT); ++ MMIO_D(BXT_PORT_PLL(DPIO_PHY0, DPIO_CH1, 2), D_BXT); ++ MMIO_D(BXT_PORT_PLL(DPIO_PHY0, DPIO_CH1, 3), D_BXT); ++ MMIO_D(BXT_PORT_PLL(DPIO_PHY0, DPIO_CH1, 6), D_BXT); ++ MMIO_D(BXT_PORT_PLL(DPIO_PHY0, DPIO_CH1, 8), D_BXT); ++ MMIO_D(BXT_PORT_PLL(DPIO_PHY0, DPIO_CH1, 9), D_BXT); ++ MMIO_D(BXT_PORT_PLL(DPIO_PHY0, DPIO_CH1, 10), D_BXT); ++ ++ MMIO_D(BXT_PORT_PLL_EBB_0(DPIO_PHY1, DPIO_CH0), D_BXT); ++ MMIO_D(BXT_PORT_PLL_EBB_4(DPIO_PHY1, DPIO_CH0), D_BXT); ++ MMIO_D(BXT_PORT_PCS_DW10_LN01(DPIO_PHY1, DPIO_CH0), D_BXT); ++ MMIO_D(BXT_PORT_PCS_DW10_GRP(DPIO_PHY1, DPIO_CH0), D_BXT); ++ MMIO_D(BXT_PORT_PCS_DW12_LN01(DPIO_PHY1, DPIO_CH0), D_BXT); ++ MMIO_D(BXT_PORT_PCS_DW12_LN23(DPIO_PHY1, DPIO_CH0), D_BXT); ++ MMIO_DH(BXT_PORT_PCS_DW12_GRP(DPIO_PHY1, DPIO_CH0), D_BXT, ++ NULL, bxt_pcs_dw12_grp_write); ++ MMIO_D(BXT_PORT_TX_DW2_LN0(DPIO_PHY1, DPIO_CH0), D_BXT); ++ MMIO_D(BXT_PORT_TX_DW2_GRP(DPIO_PHY1, DPIO_CH0), D_BXT); ++ MMIO_DH(BXT_PORT_TX_DW3_LN0(DPIO_PHY1, DPIO_CH0), D_BXT, ++ bxt_port_tx_dw3_read, NULL); ++ MMIO_D(BXT_PORT_TX_DW3_GRP(DPIO_PHY1, DPIO_CH0), D_BXT); ++ MMIO_D(BXT_PORT_TX_DW4_LN0(DPIO_PHY1, DPIO_CH0), D_BXT); ++ MMIO_D(BXT_PORT_TX_DW4_GRP(DPIO_PHY1, DPIO_CH0), D_BXT); ++ MMIO_D(BXT_PORT_TX_DW14_LN(DPIO_PHY1, DPIO_CH0, 0), D_BXT); ++ MMIO_D(BXT_PORT_TX_DW14_LN(DPIO_PHY1, DPIO_CH0, 1), D_BXT); ++ MMIO_D(BXT_PORT_TX_DW14_LN(DPIO_PHY1, DPIO_CH0, 2), D_BXT); ++ MMIO_D(BXT_PORT_TX_DW14_LN(DPIO_PHY1, DPIO_CH0, 3), D_BXT); ++ MMIO_D(BXT_PORT_PLL(DPIO_PHY1, DPIO_CH0, 0), D_BXT); ++ MMIO_D(BXT_PORT_PLL(DPIO_PHY1, DPIO_CH0, 1), D_BXT); ++ MMIO_D(BXT_PORT_PLL(DPIO_PHY1, DPIO_CH0, 2), D_BXT); ++ MMIO_D(BXT_PORT_PLL(DPIO_PHY1, DPIO_CH0, 3), D_BXT); ++ MMIO_D(BXT_PORT_PLL(DPIO_PHY1, DPIO_CH0, 6), D_BXT); ++ MMIO_D(BXT_PORT_PLL(DPIO_PHY1, DPIO_CH0, 8), D_BXT); ++ MMIO_D(BXT_PORT_PLL(DPIO_PHY1, DPIO_CH0, 9), D_BXT); ++ MMIO_D(BXT_PORT_PLL(DPIO_PHY1, DPIO_CH0, 10), D_BXT); ++ ++ MMIO_D(BXT_DE_PLL_CTL, D_BXT); ++ MMIO_DH(BXT_DE_PLL_ENABLE, D_BXT, NULL, bxt_de_pll_enable_write); ++ MMIO_D(BXT_DSI_PLL_CTL, D_BXT); ++ MMIO_D(BXT_DSI_PLL_ENABLE, D_BXT); ++ ++ MMIO_D(GEN9_CLKGATE_DIS_0, D_BXT); ++ MMIO_D(GEN9_CLKGATE_DIS_4, D_BXT); ++ ++ MMIO_D(HSW_TVIDEO_DIP_GCP(TRANSCODER_A), D_BXT); ++ MMIO_D(HSW_TVIDEO_DIP_GCP(TRANSCODER_B), D_BXT); ++ MMIO_D(HSW_TVIDEO_DIP_GCP(TRANSCODER_C), D_BXT); ++ ++ MMIO_D(RC6_CTX_BASE, D_BXT); ++ ++ MMIO_D(GEN8_PUSHBUS_CONTROL, D_BXT); ++ MMIO_D(GEN8_PUSHBUS_ENABLE, D_BXT); ++ MMIO_D(GEN8_PUSHBUS_SHIFT, D_BXT); ++ MMIO_D(GEN6_GFXPAUSE, D_BXT); ++ MMIO_DFH(GEN8_L3SQCREG1, D_BXT, F_CMD_ACCESS, NULL, NULL); ++ ++ MMIO_DFH(GEN9_CTX_PREEMPT_REG, D_BXT, F_CMD_ACCESS, NULL, NULL); ++ ++ return 0; ++} ++ ++static struct gvt_mmio_block *find_mmio_block(struct intel_gvt *gvt, ++ unsigned int offset) ++{ ++ unsigned long device = intel_gvt_get_device_type(gvt); ++ struct gvt_mmio_block *block = gvt->mmio.mmio_block; ++ int num = gvt->mmio.num_mmio_block; ++ int i; ++ ++ for (i = 0; i < num; i++, block++) { ++ if (!(device & block->device)) ++ continue; ++ if (offset >= i915_mmio_reg_offset(block->offset) && ++ offset < i915_mmio_reg_offset(block->offset) + block->size) ++ return block; ++ } ++ return NULL; ++} ++ ++/** ++ * intel_gvt_clean_mmio_info - clean up MMIO information table for GVT device ++ * @gvt: GVT device ++ * ++ * This function is called at the driver unloading stage, to clean up the MMIO ++ * information table of GVT device ++ * ++ */ ++void intel_gvt_clean_mmio_info(struct intel_gvt *gvt) ++{ ++ struct hlist_node *tmp; ++ struct intel_gvt_mmio_info *e; ++ int i; ++ ++ hash_for_each_safe(gvt->mmio.mmio_info_table, i, tmp, e, node) ++ kfree(e); ++ ++ vfree(gvt->mmio.mmio_attribute); ++ gvt->mmio.mmio_attribute = NULL; ++} ++ ++/* Special MMIO blocks. */ ++static struct gvt_mmio_block mmio_blocks[] = { ++ {D_SKL_PLUS, _MMIO(CSR_MMIO_START_RANGE), 0x3000, NULL, NULL}, ++ {D_ALL, _MMIO(MCHBAR_MIRROR_BASE_SNB), 0x40000, NULL, NULL}, ++ {D_ALL, _MMIO(VGT_PVINFO_PAGE), VGT_PVINFO_SIZE, ++ pvinfo_mmio_read, pvinfo_mmio_write}, ++ {D_ALL, LGC_PALETTE(PIPE_A, 0), 1024, NULL, NULL}, ++ {D_ALL, LGC_PALETTE(PIPE_B, 0), 1024, NULL, NULL}, ++ {D_ALL, LGC_PALETTE(PIPE_C, 0), 1024, NULL, NULL}, ++}; ++ ++/** ++ * intel_gvt_setup_mmio_info - setup MMIO information table for GVT device ++ * @gvt: GVT device ++ * ++ * This function is called at the initialization stage, to setup the MMIO ++ * information table for GVT device ++ * ++ * Returns: ++ * zero on success, negative if failed. ++ */ ++int intel_gvt_setup_mmio_info(struct intel_gvt *gvt) ++{ ++ struct intel_gvt_device_info *info = &gvt->device_info; ++ struct drm_i915_private *dev_priv = gvt->dev_priv; ++ int size = info->mmio_size / 4 * sizeof(*gvt->mmio.mmio_attribute); ++ int ret; ++ ++ gvt->mmio.mmio_attribute = vzalloc(size); ++ if (!gvt->mmio.mmio_attribute) ++ return -ENOMEM; ++ ++ ret = init_generic_mmio_info(gvt); ++ if (ret) ++ goto err; ++ ++ if (IS_BROADWELL(dev_priv)) { ++ ret = init_broadwell_mmio_info(gvt); ++ if (ret) ++ goto err; ++ } else if (IS_SKYLAKE(dev_priv) ++ || IS_KABYLAKE(dev_priv) ++ || IS_COFFEELAKE(dev_priv)) { ++ ret = init_broadwell_mmio_info(gvt); ++ if (ret) ++ goto err; ++ ret = init_skl_mmio_info(gvt); ++ if (ret) ++ goto err; ++ } else if (IS_BROXTON(dev_priv)) { ++ ret = init_broadwell_mmio_info(gvt); ++ if (ret) ++ goto err; ++ ret = init_skl_mmio_info(gvt); ++ if (ret) ++ goto err; ++ ret = init_bxt_mmio_info(gvt); ++ if (ret) ++ goto err; ++ } ++ ++ gvt->mmio.mmio_block = mmio_blocks; ++ gvt->mmio.num_mmio_block = ARRAY_SIZE(mmio_blocks); ++ ++ return 0; ++err: ++ intel_gvt_clean_mmio_info(gvt); ++ return ret; ++} ++ ++/** ++ * intel_gvt_for_each_tracked_mmio - iterate each tracked mmio ++ * @gvt: a GVT device ++ * @handler: the handler ++ * @data: private data given to handler ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++int intel_gvt_for_each_tracked_mmio(struct intel_gvt *gvt, ++ int (*handler)(struct intel_gvt *gvt, u32 offset, void *data), ++ void *data) ++{ ++ struct gvt_mmio_block *block = gvt->mmio.mmio_block; ++ struct intel_gvt_mmio_info *e; ++ int i, j, ret; ++ ++ hash_for_each(gvt->mmio.mmio_info_table, i, e, node) { ++ ret = handler(gvt, e->offset, data); ++ if (ret) ++ return ret; ++ } ++ ++ for (i = 0; i < gvt->mmio.num_mmio_block; i++, block++) { ++ for (j = 0; j < block->size; j += 4) { ++ ret = handler(gvt, ++ i915_mmio_reg_offset(block->offset) + j, ++ data); ++ if (ret) ++ return ret; ++ } ++ } ++ return 0; ++} ++ ++/** ++ * intel_vgpu_default_mmio_read - default MMIO read handler ++ * @vgpu: a vGPU ++ * @offset: access offset ++ * @p_data: data return buffer ++ * @bytes: access data length ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++int intel_vgpu_default_mmio_read(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ read_vreg(vgpu, offset, p_data, bytes); ++ return 0; ++} ++ ++/** ++ * intel_t_default_mmio_write - default MMIO write handler ++ * @vgpu: a vGPU ++ * @offset: access offset ++ * @p_data: write data buffer ++ * @bytes: access data length ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++int intel_vgpu_default_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ write_vreg(vgpu, offset, p_data, bytes); ++ return 0; ++} ++ ++/** ++ * intel_vgpu_mask_mmio_write - write mask register ++ * @vgpu: a vGPU ++ * @offset: access offset ++ * @p_data: write data buffer ++ * @bytes: access data length ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++int intel_vgpu_mask_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes) ++{ ++ u32 mask, old_vreg; ++ ++ old_vreg = vgpu_vreg(vgpu, offset); ++ write_vreg(vgpu, offset, p_data, bytes); ++ mask = vgpu_vreg(vgpu, offset) >> 16; ++ vgpu_vreg(vgpu, offset) = (old_vreg & ~mask) | ++ (vgpu_vreg(vgpu, offset) & mask); ++ ++ return 0; ++} ++ ++/** ++ * intel_gvt_in_force_nonpriv_whitelist - if a mmio is in whitelist to be ++ * force-nopriv register ++ * ++ * @gvt: a GVT device ++ * @offset: register offset ++ * ++ * Returns: ++ * True if the register is in force-nonpriv whitelist; ++ * False if outside; ++ */ ++bool intel_gvt_in_force_nonpriv_whitelist(struct intel_gvt *gvt, ++ unsigned int offset) ++{ ++ return in_whitelist(offset); ++} ++ ++/** ++ * intel_vgpu_mmio_reg_rw - emulate tracked mmio registers ++ * @vgpu: a vGPU ++ * @offset: register offset ++ * @pdata: data buffer ++ * @bytes: data length ++ * @is_read: read or write ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++int intel_vgpu_mmio_reg_rw(struct intel_vgpu *vgpu, unsigned int offset, ++ void *pdata, unsigned int bytes, bool is_read) ++{ ++ struct intel_gvt *gvt = vgpu->gvt; ++ struct intel_gvt_mmio_info *mmio_info; ++ struct gvt_mmio_block *mmio_block; ++ gvt_mmio_func func; ++ int ret; ++ ++ if (WARN_ON(bytes > 8)) ++ return -EINVAL; ++ ++ /* ++ * Handle special MMIO blocks. ++ */ ++ mmio_block = find_mmio_block(gvt, offset); ++ if (mmio_block) { ++ func = is_read ? mmio_block->read : mmio_block->write; ++ if (func) ++ return func(vgpu, offset, pdata, bytes); ++ goto default_rw; ++ } ++ ++ /* ++ * Normal tracked MMIOs. ++ */ ++ mmio_info = find_mmio_info(gvt, offset); ++ if (!mmio_info) { ++ gvt_dbg_mmio("untracked MMIO %08x len %d\n", offset, bytes); ++ goto default_rw; ++ } ++ ++ if (is_read) ++ return mmio_info->read(vgpu, offset, pdata, bytes); ++ else { ++ u64 ro_mask = mmio_info->ro_mask; ++ u32 old_vreg = 0; ++ u64 data = 0; ++ ++ if (intel_gvt_mmio_has_mode_mask(gvt, mmio_info->offset)) { ++ old_vreg = vgpu_vreg(vgpu, offset); ++ } ++ ++ if (likely(!ro_mask)) ++ ret = mmio_info->write(vgpu, offset, pdata, bytes); ++ else if (!~ro_mask) { ++ gvt_vgpu_err("try to write RO reg %x\n", offset); ++ return 0; ++ } else { ++ /* keep the RO bits in the virtual register */ ++ memcpy(&data, pdata, bytes); ++ data &= ~ro_mask; ++ data |= vgpu_vreg(vgpu, offset) & ro_mask; ++ ret = mmio_info->write(vgpu, offset, &data, bytes); ++ } ++ ++ /* higher 16bits of mode ctl regs are mask bits for change */ ++ if (intel_gvt_mmio_has_mode_mask(gvt, mmio_info->offset)) { ++ u32 mask = vgpu_vreg(vgpu, offset) >> 16; ++ ++ vgpu_vreg(vgpu, offset) = (old_vreg & ~mask) ++ | (vgpu_vreg(vgpu, offset) & mask); ++ } ++ } ++ ++ return ret; ++ ++default_rw: ++ return is_read ? ++ intel_vgpu_default_mmio_read(vgpu, offset, pdata, bytes) : ++ intel_vgpu_default_mmio_write(vgpu, offset, pdata, bytes); ++} +diff --git a/drivers/gpu/drm/i915_legacy/gvt/hypercall.h b/drivers/gpu/drm/i915_legacy/gvt/hypercall.h +new file mode 100644 +index 000000000000..4862fb12778e +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/hypercall.h +@@ -0,0 +1,78 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Eddie Dong ++ * Dexuan Cui ++ * Jike Song ++ * ++ * Contributors: ++ * Zhi Wang ++ * ++ */ ++ ++#ifndef _GVT_HYPERCALL_H_ ++#define _GVT_HYPERCALL_H_ ++ ++enum hypervisor_type { ++ INTEL_GVT_HYPERVISOR_XEN = 0, ++ INTEL_GVT_HYPERVISOR_KVM, ++}; ++ ++/* ++ * Specific GVT-g MPT modules function collections. Currently GVT-g supports ++ * both Xen and KVM by providing dedicated hypervisor-related MPT modules. ++ */ ++struct intel_gvt_mpt { ++ enum hypervisor_type type; ++ int (*host_init)(struct device *dev, void *gvt, const void *ops); ++ void (*host_exit)(struct device *dev); ++ int (*attach_vgpu)(void *vgpu, unsigned long *handle); ++ void (*detach_vgpu)(void *vgpu); ++ int (*inject_msi)(unsigned long handle, u32 addr, u16 data); ++ unsigned long (*from_virt_to_mfn)(void *p); ++ int (*enable_page_track)(unsigned long handle, u64 gfn); ++ int (*disable_page_track)(unsigned long handle, u64 gfn); ++ int (*read_gpa)(unsigned long handle, unsigned long gpa, void *buf, ++ unsigned long len); ++ int (*write_gpa)(unsigned long handle, unsigned long gpa, void *buf, ++ unsigned long len); ++ unsigned long (*gfn_to_mfn)(unsigned long handle, unsigned long gfn); ++ ++ int (*dma_map_guest_page)(unsigned long handle, unsigned long gfn, ++ unsigned long size, dma_addr_t *dma_addr); ++ void (*dma_unmap_guest_page)(unsigned long handle, dma_addr_t dma_addr); ++ ++ int (*map_gfn_to_mfn)(unsigned long handle, unsigned long gfn, ++ unsigned long mfn, unsigned int nr, bool map); ++ int (*set_trap_area)(unsigned long handle, u64 start, u64 end, ++ bool map); ++ int (*set_opregion)(void *vgpu); ++ int (*set_edid)(void *vgpu, int port_num); ++ int (*get_vfio_device)(void *vgpu); ++ void (*put_vfio_device)(void *vgpu); ++ bool (*is_valid_gfn)(unsigned long handle, unsigned long gfn); ++}; ++ ++extern struct intel_gvt_mpt xengt_mpt; ++ ++#endif /* _GVT_HYPERCALL_H_ */ +diff --git a/drivers/gpu/drm/i915_legacy/gvt/interrupt.c b/drivers/gpu/drm/i915_legacy/gvt/interrupt.c +new file mode 100644 +index 000000000000..951681813230 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/interrupt.c +@@ -0,0 +1,710 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Kevin Tian ++ * Zhi Wang ++ * ++ * Contributors: ++ * Min he ++ * ++ */ ++ ++#include "i915_drv.h" ++#include "gvt.h" ++#include "trace.h" ++ ++/* common offset among interrupt control registers */ ++#define regbase_to_isr(base) (base) ++#define regbase_to_imr(base) (base + 0x4) ++#define regbase_to_iir(base) (base + 0x8) ++#define regbase_to_ier(base) (base + 0xC) ++ ++#define iir_to_regbase(iir) (iir - 0x8) ++#define ier_to_regbase(ier) (ier - 0xC) ++ ++#define get_event_virt_handler(irq, e) (irq->events[e].v_handler) ++#define get_irq_info(irq, e) (irq->events[e].info) ++ ++#define irq_to_gvt(irq) \ ++ container_of(irq, struct intel_gvt, irq) ++ ++static void update_upstream_irq(struct intel_vgpu *vgpu, ++ struct intel_gvt_irq_info *info); ++ ++static const char * const irq_name[INTEL_GVT_EVENT_MAX] = { ++ [RCS_MI_USER_INTERRUPT] = "Render CS MI USER INTERRUPT", ++ [RCS_DEBUG] = "Render EU debug from SVG", ++ [RCS_MMIO_SYNC_FLUSH] = "Render MMIO sync flush status", ++ [RCS_CMD_STREAMER_ERR] = "Render CS error interrupt", ++ [RCS_PIPE_CONTROL] = "Render PIPE CONTROL notify", ++ [RCS_WATCHDOG_EXCEEDED] = "Render CS Watchdog counter exceeded", ++ [RCS_PAGE_DIRECTORY_FAULT] = "Render page directory faults", ++ [RCS_AS_CONTEXT_SWITCH] = "Render AS Context Switch Interrupt", ++ ++ [VCS_MI_USER_INTERRUPT] = "Video CS MI USER INTERRUPT", ++ [VCS_MMIO_SYNC_FLUSH] = "Video MMIO sync flush status", ++ [VCS_CMD_STREAMER_ERR] = "Video CS error interrupt", ++ [VCS_MI_FLUSH_DW] = "Video MI FLUSH DW notify", ++ [VCS_WATCHDOG_EXCEEDED] = "Video CS Watchdog counter exceeded", ++ [VCS_PAGE_DIRECTORY_FAULT] = "Video page directory faults", ++ [VCS_AS_CONTEXT_SWITCH] = "Video AS Context Switch Interrupt", ++ [VCS2_MI_USER_INTERRUPT] = "VCS2 Video CS MI USER INTERRUPT", ++ [VCS2_MI_FLUSH_DW] = "VCS2 Video MI FLUSH DW notify", ++ [VCS2_AS_CONTEXT_SWITCH] = "VCS2 Context Switch Interrupt", ++ ++ [BCS_MI_USER_INTERRUPT] = "Blitter CS MI USER INTERRUPT", ++ [BCS_MMIO_SYNC_FLUSH] = "Billter MMIO sync flush status", ++ [BCS_CMD_STREAMER_ERR] = "Blitter CS error interrupt", ++ [BCS_MI_FLUSH_DW] = "Blitter MI FLUSH DW notify", ++ [BCS_PAGE_DIRECTORY_FAULT] = "Blitter page directory faults", ++ [BCS_AS_CONTEXT_SWITCH] = "Blitter AS Context Switch Interrupt", ++ ++ [VECS_MI_FLUSH_DW] = "Video Enhanced Streamer MI FLUSH DW notify", ++ [VECS_AS_CONTEXT_SWITCH] = "VECS Context Switch Interrupt", ++ ++ [PIPE_A_FIFO_UNDERRUN] = "Pipe A FIFO underrun", ++ [PIPE_A_CRC_ERR] = "Pipe A CRC error", ++ [PIPE_A_CRC_DONE] = "Pipe A CRC done", ++ [PIPE_A_VSYNC] = "Pipe A vsync", ++ [PIPE_A_LINE_COMPARE] = "Pipe A line compare", ++ [PIPE_A_ODD_FIELD] = "Pipe A odd field", ++ [PIPE_A_EVEN_FIELD] = "Pipe A even field", ++ [PIPE_A_VBLANK] = "Pipe A vblank", ++ [PIPE_B_FIFO_UNDERRUN] = "Pipe B FIFO underrun", ++ [PIPE_B_CRC_ERR] = "Pipe B CRC error", ++ [PIPE_B_CRC_DONE] = "Pipe B CRC done", ++ [PIPE_B_VSYNC] = "Pipe B vsync", ++ [PIPE_B_LINE_COMPARE] = "Pipe B line compare", ++ [PIPE_B_ODD_FIELD] = "Pipe B odd field", ++ [PIPE_B_EVEN_FIELD] = "Pipe B even field", ++ [PIPE_B_VBLANK] = "Pipe B vblank", ++ [PIPE_C_VBLANK] = "Pipe C vblank", ++ [DPST_PHASE_IN] = "DPST phase in event", ++ [DPST_HISTOGRAM] = "DPST histogram event", ++ [GSE] = "GSE", ++ [DP_A_HOTPLUG] = "DP A Hotplug", ++ [AUX_CHANNEL_A] = "AUX Channel A", ++ [PERF_COUNTER] = "Performance counter", ++ [POISON] = "Poison", ++ [GTT_FAULT] = "GTT fault", ++ [PRIMARY_A_FLIP_DONE] = "Primary Plane A flip done", ++ [PRIMARY_B_FLIP_DONE] = "Primary Plane B flip done", ++ [PRIMARY_C_FLIP_DONE] = "Primary Plane C flip done", ++ [SPRITE_A_FLIP_DONE] = "Sprite Plane A flip done", ++ [SPRITE_B_FLIP_DONE] = "Sprite Plane B flip done", ++ [SPRITE_C_FLIP_DONE] = "Sprite Plane C flip done", ++ ++ [PCU_THERMAL] = "PCU Thermal Event", ++ [PCU_PCODE2DRIVER_MAILBOX] = "PCU pcode2driver mailbox event", ++ ++ [FDI_RX_INTERRUPTS_TRANSCODER_A] = "FDI RX Interrupts Combined A", ++ [AUDIO_CP_CHANGE_TRANSCODER_A] = "Audio CP Change Transcoder A", ++ [AUDIO_CP_REQUEST_TRANSCODER_A] = "Audio CP Request Transcoder A", ++ [FDI_RX_INTERRUPTS_TRANSCODER_B] = "FDI RX Interrupts Combined B", ++ [AUDIO_CP_CHANGE_TRANSCODER_B] = "Audio CP Change Transcoder B", ++ [AUDIO_CP_REQUEST_TRANSCODER_B] = "Audio CP Request Transcoder B", ++ [FDI_RX_INTERRUPTS_TRANSCODER_C] = "FDI RX Interrupts Combined C", ++ [AUDIO_CP_CHANGE_TRANSCODER_C] = "Audio CP Change Transcoder C", ++ [AUDIO_CP_REQUEST_TRANSCODER_C] = "Audio CP Request Transcoder C", ++ [ERR_AND_DBG] = "South Error and Debug Interrupts Combined", ++ [GMBUS] = "Gmbus", ++ [SDVO_B_HOTPLUG] = "SDVO B hotplug", ++ [CRT_HOTPLUG] = "CRT Hotplug", ++ [DP_B_HOTPLUG] = "DisplayPort/HDMI/DVI B Hotplug", ++ [DP_C_HOTPLUG] = "DisplayPort/HDMI/DVI C Hotplug", ++ [DP_D_HOTPLUG] = "DisplayPort/HDMI/DVI D Hotplug", ++ [AUX_CHANNEL_B] = "AUX Channel B", ++ [AUX_CHANNEL_C] = "AUX Channel C", ++ [AUX_CHANNEL_D] = "AUX Channel D", ++ [AUDIO_POWER_STATE_CHANGE_B] = "Audio Power State change Port B", ++ [AUDIO_POWER_STATE_CHANGE_C] = "Audio Power State change Port C", ++ [AUDIO_POWER_STATE_CHANGE_D] = "Audio Power State change Port D", ++ ++ [INTEL_GVT_EVENT_RESERVED] = "RESERVED EVENTS!!!", ++}; ++ ++static inline struct intel_gvt_irq_info *regbase_to_irq_info( ++ struct intel_gvt *gvt, ++ unsigned int reg) ++{ ++ struct intel_gvt_irq *irq = &gvt->irq; ++ int i; ++ ++ for_each_set_bit(i, irq->irq_info_bitmap, INTEL_GVT_IRQ_INFO_MAX) { ++ if (i915_mmio_reg_offset(irq->info[i]->reg_base) == reg) ++ return irq->info[i]; ++ } ++ ++ return NULL; ++} ++ ++/** ++ * intel_vgpu_reg_imr_handler - Generic IMR register emulation write handler ++ * @vgpu: a vGPU ++ * @reg: register offset written by guest ++ * @p_data: register data written by guest ++ * @bytes: register data length ++ * ++ * This function is used to emulate the generic IMR register bit change ++ * behavior. ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ * ++ */ ++int intel_vgpu_reg_imr_handler(struct intel_vgpu *vgpu, ++ unsigned int reg, void *p_data, unsigned int bytes) ++{ ++ struct intel_gvt *gvt = vgpu->gvt; ++ struct intel_gvt_irq_ops *ops = gvt->irq.ops; ++ u32 imr = *(u32 *)p_data; ++ ++ trace_write_ir(vgpu->id, "IMR", reg, imr, vgpu_vreg(vgpu, reg), ++ (vgpu_vreg(vgpu, reg) ^ imr)); ++ ++ vgpu_vreg(vgpu, reg) = imr; ++ ++ ops->check_pending_irq(vgpu); ++ ++ return 0; ++} ++ ++/** ++ * intel_vgpu_reg_master_irq_handler - master IRQ write emulation handler ++ * @vgpu: a vGPU ++ * @reg: register offset written by guest ++ * @p_data: register data written by guest ++ * @bytes: register data length ++ * ++ * This function is used to emulate the master IRQ register on gen8+. ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ * ++ */ ++int intel_vgpu_reg_master_irq_handler(struct intel_vgpu *vgpu, ++ unsigned int reg, void *p_data, unsigned int bytes) ++{ ++ struct intel_gvt *gvt = vgpu->gvt; ++ struct intel_gvt_irq_ops *ops = gvt->irq.ops; ++ u32 ier = *(u32 *)p_data; ++ u32 virtual_ier = vgpu_vreg(vgpu, reg); ++ ++ trace_write_ir(vgpu->id, "MASTER_IRQ", reg, ier, virtual_ier, ++ (virtual_ier ^ ier)); ++ ++ /* ++ * GEN8_MASTER_IRQ is a special irq register, ++ * only bit 31 is allowed to be modified ++ * and treated as an IER bit. ++ */ ++ ier &= GEN8_MASTER_IRQ_CONTROL; ++ virtual_ier &= GEN8_MASTER_IRQ_CONTROL; ++ vgpu_vreg(vgpu, reg) &= ~GEN8_MASTER_IRQ_CONTROL; ++ vgpu_vreg(vgpu, reg) |= ier; ++ ++ ops->check_pending_irq(vgpu); ++ ++ return 0; ++} ++ ++/** ++ * intel_vgpu_reg_ier_handler - Generic IER write emulation handler ++ * @vgpu: a vGPU ++ * @reg: register offset written by guest ++ * @p_data: register data written by guest ++ * @bytes: register data length ++ * ++ * This function is used to emulate the generic IER register behavior. ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ * ++ */ ++int intel_vgpu_reg_ier_handler(struct intel_vgpu *vgpu, ++ unsigned int reg, void *p_data, unsigned int bytes) ++{ ++ struct intel_gvt *gvt = vgpu->gvt; ++ struct intel_gvt_irq_ops *ops = gvt->irq.ops; ++ struct intel_gvt_irq_info *info; ++ u32 ier = *(u32 *)p_data; ++ ++ trace_write_ir(vgpu->id, "IER", reg, ier, vgpu_vreg(vgpu, reg), ++ (vgpu_vreg(vgpu, reg) ^ ier)); ++ ++ vgpu_vreg(vgpu, reg) = ier; ++ ++ info = regbase_to_irq_info(gvt, ier_to_regbase(reg)); ++ if (WARN_ON(!info)) ++ return -EINVAL; ++ ++ if (info->has_upstream_irq) ++ update_upstream_irq(vgpu, info); ++ ++ ops->check_pending_irq(vgpu); ++ ++ return 0; ++} ++ ++/** ++ * intel_vgpu_reg_iir_handler - Generic IIR write emulation handler ++ * @vgpu: a vGPU ++ * @reg: register offset written by guest ++ * @p_data: register data written by guest ++ * @bytes: register data length ++ * ++ * This function is used to emulate the generic IIR register behavior. ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ * ++ */ ++int intel_vgpu_reg_iir_handler(struct intel_vgpu *vgpu, unsigned int reg, ++ void *p_data, unsigned int bytes) ++{ ++ struct intel_gvt_irq_info *info = regbase_to_irq_info(vgpu->gvt, ++ iir_to_regbase(reg)); ++ u32 iir = *(u32 *)p_data; ++ ++ trace_write_ir(vgpu->id, "IIR", reg, iir, vgpu_vreg(vgpu, reg), ++ (vgpu_vreg(vgpu, reg) ^ iir)); ++ ++ if (WARN_ON(!info)) ++ return -EINVAL; ++ ++ vgpu_vreg(vgpu, reg) &= ~iir; ++ ++ if (info->has_upstream_irq) ++ update_upstream_irq(vgpu, info); ++ return 0; ++} ++ ++static struct intel_gvt_irq_map gen8_irq_map[] = { ++ { INTEL_GVT_IRQ_INFO_MASTER, 0, INTEL_GVT_IRQ_INFO_GT0, 0xffff }, ++ { INTEL_GVT_IRQ_INFO_MASTER, 1, INTEL_GVT_IRQ_INFO_GT0, 0xffff0000 }, ++ { INTEL_GVT_IRQ_INFO_MASTER, 2, INTEL_GVT_IRQ_INFO_GT1, 0xffff }, ++ { INTEL_GVT_IRQ_INFO_MASTER, 3, INTEL_GVT_IRQ_INFO_GT1, 0xffff0000 }, ++ { INTEL_GVT_IRQ_INFO_MASTER, 4, INTEL_GVT_IRQ_INFO_GT2, 0xffff }, ++ { INTEL_GVT_IRQ_INFO_MASTER, 6, INTEL_GVT_IRQ_INFO_GT3, 0xffff }, ++ { INTEL_GVT_IRQ_INFO_MASTER, 16, INTEL_GVT_IRQ_INFO_DE_PIPE_A, ~0 }, ++ { INTEL_GVT_IRQ_INFO_MASTER, 17, INTEL_GVT_IRQ_INFO_DE_PIPE_B, ~0 }, ++ { INTEL_GVT_IRQ_INFO_MASTER, 18, INTEL_GVT_IRQ_INFO_DE_PIPE_C, ~0 }, ++ { INTEL_GVT_IRQ_INFO_MASTER, 20, INTEL_GVT_IRQ_INFO_DE_PORT, ~0 }, ++ { INTEL_GVT_IRQ_INFO_MASTER, 22, INTEL_GVT_IRQ_INFO_DE_MISC, ~0 }, ++ { INTEL_GVT_IRQ_INFO_MASTER, 23, INTEL_GVT_IRQ_INFO_PCH, ~0 }, ++ { INTEL_GVT_IRQ_INFO_MASTER, 30, INTEL_GVT_IRQ_INFO_PCU, ~0 }, ++ { -1, -1, ~0 }, ++}; ++ ++static void update_upstream_irq(struct intel_vgpu *vgpu, ++ struct intel_gvt_irq_info *info) ++{ ++ struct intel_gvt_irq *irq = &vgpu->gvt->irq; ++ struct intel_gvt_irq_map *map = irq->irq_map; ++ struct intel_gvt_irq_info *up_irq_info = NULL; ++ u32 set_bits = 0; ++ u32 clear_bits = 0; ++ int bit; ++ u32 val = vgpu_vreg(vgpu, ++ regbase_to_iir(i915_mmio_reg_offset(info->reg_base))) ++ & vgpu_vreg(vgpu, ++ regbase_to_ier(i915_mmio_reg_offset(info->reg_base))); ++ ++ if (!info->has_upstream_irq) ++ return; ++ ++ for (map = irq->irq_map; map->up_irq_bit != -1; map++) { ++ if (info->group != map->down_irq_group) ++ continue; ++ ++ if (!up_irq_info) ++ up_irq_info = irq->info[map->up_irq_group]; ++ else ++ WARN_ON(up_irq_info != irq->info[map->up_irq_group]); ++ ++ bit = map->up_irq_bit; ++ ++ if (val & map->down_irq_bitmask) ++ set_bits |= (1 << bit); ++ else ++ clear_bits |= (1 << bit); ++ } ++ ++ if (WARN_ON(!up_irq_info)) ++ return; ++ ++ if (up_irq_info->group == INTEL_GVT_IRQ_INFO_MASTER) { ++ u32 isr = i915_mmio_reg_offset(up_irq_info->reg_base); ++ ++ vgpu_vreg(vgpu, isr) &= ~clear_bits; ++ vgpu_vreg(vgpu, isr) |= set_bits; ++ } else { ++ u32 iir = regbase_to_iir( ++ i915_mmio_reg_offset(up_irq_info->reg_base)); ++ u32 imr = regbase_to_imr( ++ i915_mmio_reg_offset(up_irq_info->reg_base)); ++ ++ vgpu_vreg(vgpu, iir) |= (set_bits & ~vgpu_vreg(vgpu, imr)); ++ } ++ ++ if (up_irq_info->has_upstream_irq) ++ update_upstream_irq(vgpu, up_irq_info); ++} ++ ++static void init_irq_map(struct intel_gvt_irq *irq) ++{ ++ struct intel_gvt_irq_map *map; ++ struct intel_gvt_irq_info *up_info, *down_info; ++ int up_bit; ++ ++ for (map = irq->irq_map; map->up_irq_bit != -1; map++) { ++ up_info = irq->info[map->up_irq_group]; ++ up_bit = map->up_irq_bit; ++ down_info = irq->info[map->down_irq_group]; ++ ++ set_bit(up_bit, up_info->downstream_irq_bitmap); ++ down_info->has_upstream_irq = true; ++ ++ gvt_dbg_irq("[up] grp %d bit %d -> [down] grp %d bitmask %x\n", ++ up_info->group, up_bit, ++ down_info->group, map->down_irq_bitmask); ++ } ++} ++ ++/* =======================vEvent injection===================== */ ++static int inject_virtual_interrupt(struct intel_vgpu *vgpu) ++{ ++ return intel_gvt_hypervisor_inject_msi(vgpu); ++} ++ ++static void propagate_event(struct intel_gvt_irq *irq, ++ enum intel_gvt_event_type event, struct intel_vgpu *vgpu) ++{ ++ struct intel_gvt_irq_info *info; ++ unsigned int reg_base; ++ int bit; ++ ++ info = get_irq_info(irq, event); ++ if (WARN_ON(!info)) ++ return; ++ ++ reg_base = i915_mmio_reg_offset(info->reg_base); ++ bit = irq->events[event].bit; ++ ++ if (!test_bit(bit, (void *)&vgpu_vreg(vgpu, ++ regbase_to_imr(reg_base)))) { ++ trace_propagate_event(vgpu->id, irq_name[event], bit); ++ set_bit(bit, (void *)&vgpu_vreg(vgpu, ++ regbase_to_iir(reg_base))); ++ } ++} ++ ++/* =======================vEvent Handlers===================== */ ++static void handle_default_event_virt(struct intel_gvt_irq *irq, ++ enum intel_gvt_event_type event, struct intel_vgpu *vgpu) ++{ ++ if (!vgpu->irq.irq_warn_once[event]) { ++ gvt_dbg_core("vgpu%d: IRQ receive event %d (%s)\n", ++ vgpu->id, event, irq_name[event]); ++ vgpu->irq.irq_warn_once[event] = true; ++ } ++ propagate_event(irq, event, vgpu); ++} ++ ++/* =====================GEN specific logic======================= */ ++/* GEN8 interrupt routines. */ ++ ++#define DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(regname, regbase) \ ++static struct intel_gvt_irq_info gen8_##regname##_info = { \ ++ .name = #regname"-IRQ", \ ++ .reg_base = (regbase), \ ++ .bit_to_event = {[0 ... INTEL_GVT_IRQ_BITWIDTH-1] = \ ++ INTEL_GVT_EVENT_RESERVED}, \ ++} ++ ++DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(gt0, GEN8_GT_ISR(0)); ++DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(gt1, GEN8_GT_ISR(1)); ++DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(gt2, GEN8_GT_ISR(2)); ++DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(gt3, GEN8_GT_ISR(3)); ++DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(de_pipe_a, GEN8_DE_PIPE_ISR(PIPE_A)); ++DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(de_pipe_b, GEN8_DE_PIPE_ISR(PIPE_B)); ++DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(de_pipe_c, GEN8_DE_PIPE_ISR(PIPE_C)); ++DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(de_port, GEN8_DE_PORT_ISR); ++DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(de_misc, GEN8_DE_MISC_ISR); ++DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(pcu, GEN8_PCU_ISR); ++DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(master, GEN8_MASTER_IRQ); ++ ++static struct intel_gvt_irq_info gvt_base_pch_info = { ++ .name = "PCH-IRQ", ++ .reg_base = SDEISR, ++ .bit_to_event = {[0 ... INTEL_GVT_IRQ_BITWIDTH-1] = ++ INTEL_GVT_EVENT_RESERVED}, ++}; ++ ++static void gen8_check_pending_irq(struct intel_vgpu *vgpu) ++{ ++ struct intel_gvt_irq *irq = &vgpu->gvt->irq; ++ int i; ++ ++ if (!(vgpu_vreg(vgpu, i915_mmio_reg_offset(GEN8_MASTER_IRQ)) & ++ GEN8_MASTER_IRQ_CONTROL)) ++ return; ++ ++ for_each_set_bit(i, irq->irq_info_bitmap, INTEL_GVT_IRQ_INFO_MAX) { ++ struct intel_gvt_irq_info *info = irq->info[i]; ++ u32 reg_base; ++ ++ if (!info->has_upstream_irq) ++ continue; ++ ++ reg_base = i915_mmio_reg_offset(info->reg_base); ++ if ((vgpu_vreg(vgpu, regbase_to_iir(reg_base)) ++ & vgpu_vreg(vgpu, regbase_to_ier(reg_base)))) ++ update_upstream_irq(vgpu, info); ++ } ++ ++ if (vgpu_vreg(vgpu, i915_mmio_reg_offset(GEN8_MASTER_IRQ)) ++ & ~GEN8_MASTER_IRQ_CONTROL) ++ inject_virtual_interrupt(vgpu); ++} ++ ++static void gen8_init_irq( ++ struct intel_gvt_irq *irq) ++{ ++ struct intel_gvt *gvt = irq_to_gvt(irq); ++ ++#define SET_BIT_INFO(s, b, e, i) \ ++ do { \ ++ s->events[e].bit = b; \ ++ s->events[e].info = s->info[i]; \ ++ s->info[i]->bit_to_event[b] = e;\ ++ } while (0) ++ ++#define SET_IRQ_GROUP(s, g, i) \ ++ do { \ ++ s->info[g] = i; \ ++ (i)->group = g; \ ++ set_bit(g, s->irq_info_bitmap); \ ++ } while (0) ++ ++ SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_MASTER, &gen8_master_info); ++ SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_GT0, &gen8_gt0_info); ++ SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_GT1, &gen8_gt1_info); ++ SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_GT2, &gen8_gt2_info); ++ SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_GT3, &gen8_gt3_info); ++ SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_DE_PIPE_A, &gen8_de_pipe_a_info); ++ SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_DE_PIPE_B, &gen8_de_pipe_b_info); ++ SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_DE_PIPE_C, &gen8_de_pipe_c_info); ++ SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_DE_PORT, &gen8_de_port_info); ++ SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_DE_MISC, &gen8_de_misc_info); ++ SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_PCU, &gen8_pcu_info); ++ SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_PCH, &gvt_base_pch_info); ++ ++ /* GEN8 level 2 interrupts. */ ++ ++ /* GEN8 interrupt GT0 events */ ++ SET_BIT_INFO(irq, 0, RCS_MI_USER_INTERRUPT, INTEL_GVT_IRQ_INFO_GT0); ++ SET_BIT_INFO(irq, 4, RCS_PIPE_CONTROL, INTEL_GVT_IRQ_INFO_GT0); ++ SET_BIT_INFO(irq, 8, RCS_AS_CONTEXT_SWITCH, INTEL_GVT_IRQ_INFO_GT0); ++ ++ SET_BIT_INFO(irq, 16, BCS_MI_USER_INTERRUPT, INTEL_GVT_IRQ_INFO_GT0); ++ SET_BIT_INFO(irq, 20, BCS_MI_FLUSH_DW, INTEL_GVT_IRQ_INFO_GT0); ++ SET_BIT_INFO(irq, 24, BCS_AS_CONTEXT_SWITCH, INTEL_GVT_IRQ_INFO_GT0); ++ ++ /* GEN8 interrupt GT1 events */ ++ SET_BIT_INFO(irq, 0, VCS_MI_USER_INTERRUPT, INTEL_GVT_IRQ_INFO_GT1); ++ SET_BIT_INFO(irq, 4, VCS_MI_FLUSH_DW, INTEL_GVT_IRQ_INFO_GT1); ++ SET_BIT_INFO(irq, 8, VCS_AS_CONTEXT_SWITCH, INTEL_GVT_IRQ_INFO_GT1); ++ ++ if (HAS_ENGINE(gvt->dev_priv, VCS1)) { ++ SET_BIT_INFO(irq, 16, VCS2_MI_USER_INTERRUPT, ++ INTEL_GVT_IRQ_INFO_GT1); ++ SET_BIT_INFO(irq, 20, VCS2_MI_FLUSH_DW, ++ INTEL_GVT_IRQ_INFO_GT1); ++ SET_BIT_INFO(irq, 24, VCS2_AS_CONTEXT_SWITCH, ++ INTEL_GVT_IRQ_INFO_GT1); ++ } ++ ++ /* GEN8 interrupt GT3 events */ ++ SET_BIT_INFO(irq, 0, VECS_MI_USER_INTERRUPT, INTEL_GVT_IRQ_INFO_GT3); ++ SET_BIT_INFO(irq, 4, VECS_MI_FLUSH_DW, INTEL_GVT_IRQ_INFO_GT3); ++ SET_BIT_INFO(irq, 8, VECS_AS_CONTEXT_SWITCH, INTEL_GVT_IRQ_INFO_GT3); ++ ++ SET_BIT_INFO(irq, 0, PIPE_A_VBLANK, INTEL_GVT_IRQ_INFO_DE_PIPE_A); ++ SET_BIT_INFO(irq, 0, PIPE_B_VBLANK, INTEL_GVT_IRQ_INFO_DE_PIPE_B); ++ SET_BIT_INFO(irq, 0, PIPE_C_VBLANK, INTEL_GVT_IRQ_INFO_DE_PIPE_C); ++ ++ /* GEN8 interrupt DE PORT events */ ++ SET_BIT_INFO(irq, 0, AUX_CHANNEL_A, INTEL_GVT_IRQ_INFO_DE_PORT); ++ SET_BIT_INFO(irq, 3, DP_A_HOTPLUG, INTEL_GVT_IRQ_INFO_DE_PORT); ++ ++ /* GEN8 interrupt DE MISC events */ ++ SET_BIT_INFO(irq, 0, GSE, INTEL_GVT_IRQ_INFO_DE_MISC); ++ ++ /* PCH events */ ++ SET_BIT_INFO(irq, 17, GMBUS, INTEL_GVT_IRQ_INFO_PCH); ++ SET_BIT_INFO(irq, 19, CRT_HOTPLUG, INTEL_GVT_IRQ_INFO_PCH); ++ SET_BIT_INFO(irq, 21, DP_B_HOTPLUG, INTEL_GVT_IRQ_INFO_PCH); ++ SET_BIT_INFO(irq, 22, DP_C_HOTPLUG, INTEL_GVT_IRQ_INFO_PCH); ++ SET_BIT_INFO(irq, 23, DP_D_HOTPLUG, INTEL_GVT_IRQ_INFO_PCH); ++ ++ if (IS_BROADWELL(gvt->dev_priv)) { ++ SET_BIT_INFO(irq, 25, AUX_CHANNEL_B, INTEL_GVT_IRQ_INFO_PCH); ++ SET_BIT_INFO(irq, 26, AUX_CHANNEL_C, INTEL_GVT_IRQ_INFO_PCH); ++ SET_BIT_INFO(irq, 27, AUX_CHANNEL_D, INTEL_GVT_IRQ_INFO_PCH); ++ ++ SET_BIT_INFO(irq, 4, PRIMARY_A_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_A); ++ SET_BIT_INFO(irq, 5, SPRITE_A_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_A); ++ ++ SET_BIT_INFO(irq, 4, PRIMARY_B_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_B); ++ SET_BIT_INFO(irq, 5, SPRITE_B_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_B); ++ ++ SET_BIT_INFO(irq, 4, PRIMARY_C_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_C); ++ SET_BIT_INFO(irq, 5, SPRITE_C_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_C); ++ } else if (INTEL_GEN(gvt->dev_priv) >= 9) { ++ SET_BIT_INFO(irq, 25, AUX_CHANNEL_B, INTEL_GVT_IRQ_INFO_DE_PORT); ++ SET_BIT_INFO(irq, 26, AUX_CHANNEL_C, INTEL_GVT_IRQ_INFO_DE_PORT); ++ SET_BIT_INFO(irq, 27, AUX_CHANNEL_D, INTEL_GVT_IRQ_INFO_DE_PORT); ++ ++ SET_BIT_INFO(irq, 3, PRIMARY_A_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_A); ++ SET_BIT_INFO(irq, 3, PRIMARY_B_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_B); ++ SET_BIT_INFO(irq, 3, PRIMARY_C_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_C); ++ ++ SET_BIT_INFO(irq, 4, SPRITE_A_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_A); ++ SET_BIT_INFO(irq, 4, SPRITE_B_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_B); ++ SET_BIT_INFO(irq, 4, SPRITE_C_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_C); ++ } ++ ++ /* GEN8 interrupt PCU events */ ++ SET_BIT_INFO(irq, 24, PCU_THERMAL, INTEL_GVT_IRQ_INFO_PCU); ++ SET_BIT_INFO(irq, 25, PCU_PCODE2DRIVER_MAILBOX, INTEL_GVT_IRQ_INFO_PCU); ++} ++ ++static struct intel_gvt_irq_ops gen8_irq_ops = { ++ .init_irq = gen8_init_irq, ++ .check_pending_irq = gen8_check_pending_irq, ++}; ++ ++/** ++ * intel_vgpu_trigger_virtual_event - Trigger a virtual event for a vGPU ++ * @vgpu: a vGPU ++ * @event: interrupt event ++ * ++ * This function is used to trigger a virtual interrupt event for vGPU. ++ * The caller provides the event to be triggered, the framework itself ++ * will emulate the IRQ register bit change. ++ * ++ */ ++void intel_vgpu_trigger_virtual_event(struct intel_vgpu *vgpu, ++ enum intel_gvt_event_type event) ++{ ++ struct intel_gvt *gvt = vgpu->gvt; ++ struct intel_gvt_irq *irq = &gvt->irq; ++ gvt_event_virt_handler_t handler; ++ struct intel_gvt_irq_ops *ops = gvt->irq.ops; ++ ++ handler = get_event_virt_handler(irq, event); ++ WARN_ON(!handler); ++ ++ handler(irq, event, vgpu); ++ ++ ops->check_pending_irq(vgpu); ++} ++ ++static void init_events( ++ struct intel_gvt_irq *irq) ++{ ++ int i; ++ ++ for (i = 0; i < INTEL_GVT_EVENT_MAX; i++) { ++ irq->events[i].info = NULL; ++ irq->events[i].v_handler = handle_default_event_virt; ++ } ++} ++ ++static enum hrtimer_restart vblank_timer_fn(struct hrtimer *data) ++{ ++ struct intel_gvt_vblank_timer *vblank_timer; ++ struct intel_gvt_irq *irq; ++ struct intel_gvt *gvt; ++ ++ vblank_timer = container_of(data, struct intel_gvt_vblank_timer, timer); ++ irq = container_of(vblank_timer, struct intel_gvt_irq, vblank_timer); ++ gvt = container_of(irq, struct intel_gvt, irq); ++ ++ intel_gvt_request_service(gvt, INTEL_GVT_REQUEST_EMULATE_VBLANK); ++ hrtimer_add_expires_ns(&vblank_timer->timer, vblank_timer->period); ++ return HRTIMER_RESTART; ++} ++ ++/** ++ * intel_gvt_clean_irq - clean up GVT-g IRQ emulation subsystem ++ * @gvt: a GVT device ++ * ++ * This function is called at driver unloading stage, to clean up GVT-g IRQ ++ * emulation subsystem. ++ * ++ */ ++void intel_gvt_clean_irq(struct intel_gvt *gvt) ++{ ++ struct intel_gvt_irq *irq = &gvt->irq; ++ ++ hrtimer_cancel(&irq->vblank_timer.timer); ++} ++ ++#define VBLNAK_TIMER_PERIOD 16000000 ++ ++/** ++ * intel_gvt_init_irq - initialize GVT-g IRQ emulation subsystem ++ * @gvt: a GVT device ++ * ++ * This function is called at driver loading stage, to initialize the GVT-g IRQ ++ * emulation subsystem. ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++int intel_gvt_init_irq(struct intel_gvt *gvt) ++{ ++ struct intel_gvt_irq *irq = &gvt->irq; ++ struct intel_gvt_vblank_timer *vblank_timer = &irq->vblank_timer; ++ ++ gvt_dbg_core("init irq framework\n"); ++ ++ irq->ops = &gen8_irq_ops; ++ irq->irq_map = gen8_irq_map; ++ ++ /* common event initialization */ ++ init_events(irq); ++ ++ /* gen specific initialization */ ++ irq->ops->init_irq(irq); ++ ++ init_irq_map(irq); ++ ++ hrtimer_init(&vblank_timer->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); ++ vblank_timer->timer.function = vblank_timer_fn; ++ vblank_timer->period = VBLNAK_TIMER_PERIOD; ++ ++ return 0; ++} +diff --git a/drivers/gpu/drm/i915_legacy/gvt/interrupt.h b/drivers/gpu/drm/i915_legacy/gvt/interrupt.h +new file mode 100644 +index 000000000000..5313fb1b33e1 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/interrupt.h +@@ -0,0 +1,233 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Kevin Tian ++ * Zhi Wang ++ * ++ * Contributors: ++ * Min he ++ * ++ */ ++ ++#ifndef _GVT_INTERRUPT_H_ ++#define _GVT_INTERRUPT_H_ ++ ++enum intel_gvt_event_type { ++ RCS_MI_USER_INTERRUPT = 0, ++ RCS_DEBUG, ++ RCS_MMIO_SYNC_FLUSH, ++ RCS_CMD_STREAMER_ERR, ++ RCS_PIPE_CONTROL, ++ RCS_L3_PARITY_ERR, ++ RCS_WATCHDOG_EXCEEDED, ++ RCS_PAGE_DIRECTORY_FAULT, ++ RCS_AS_CONTEXT_SWITCH, ++ RCS_MONITOR_BUFF_HALF_FULL, ++ ++ VCS_MI_USER_INTERRUPT, ++ VCS_MMIO_SYNC_FLUSH, ++ VCS_CMD_STREAMER_ERR, ++ VCS_MI_FLUSH_DW, ++ VCS_WATCHDOG_EXCEEDED, ++ VCS_PAGE_DIRECTORY_FAULT, ++ VCS_AS_CONTEXT_SWITCH, ++ ++ VCS2_MI_USER_INTERRUPT, ++ VCS2_MI_FLUSH_DW, ++ VCS2_AS_CONTEXT_SWITCH, ++ ++ BCS_MI_USER_INTERRUPT, ++ BCS_MMIO_SYNC_FLUSH, ++ BCS_CMD_STREAMER_ERR, ++ BCS_MI_FLUSH_DW, ++ BCS_PAGE_DIRECTORY_FAULT, ++ BCS_AS_CONTEXT_SWITCH, ++ ++ VECS_MI_USER_INTERRUPT, ++ VECS_MI_FLUSH_DW, ++ VECS_AS_CONTEXT_SWITCH, ++ ++ PIPE_A_FIFO_UNDERRUN, ++ PIPE_B_FIFO_UNDERRUN, ++ PIPE_A_CRC_ERR, ++ PIPE_B_CRC_ERR, ++ PIPE_A_CRC_DONE, ++ PIPE_B_CRC_DONE, ++ PIPE_A_ODD_FIELD, ++ PIPE_B_ODD_FIELD, ++ PIPE_A_EVEN_FIELD, ++ PIPE_B_EVEN_FIELD, ++ PIPE_A_LINE_COMPARE, ++ PIPE_B_LINE_COMPARE, ++ PIPE_C_LINE_COMPARE, ++ PIPE_A_VBLANK, ++ PIPE_B_VBLANK, ++ PIPE_C_VBLANK, ++ PIPE_A_VSYNC, ++ PIPE_B_VSYNC, ++ PIPE_C_VSYNC, ++ PRIMARY_A_FLIP_DONE, ++ PRIMARY_B_FLIP_DONE, ++ PRIMARY_C_FLIP_DONE, ++ SPRITE_A_FLIP_DONE, ++ SPRITE_B_FLIP_DONE, ++ SPRITE_C_FLIP_DONE, ++ ++ PCU_THERMAL, ++ PCU_PCODE2DRIVER_MAILBOX, ++ ++ DPST_PHASE_IN, ++ DPST_HISTOGRAM, ++ GSE, ++ DP_A_HOTPLUG, ++ AUX_CHANNEL_A, ++ PERF_COUNTER, ++ POISON, ++ GTT_FAULT, ++ ERROR_INTERRUPT_COMBINED, ++ ++ FDI_RX_INTERRUPTS_TRANSCODER_A, ++ AUDIO_CP_CHANGE_TRANSCODER_A, ++ AUDIO_CP_REQUEST_TRANSCODER_A, ++ FDI_RX_INTERRUPTS_TRANSCODER_B, ++ AUDIO_CP_CHANGE_TRANSCODER_B, ++ AUDIO_CP_REQUEST_TRANSCODER_B, ++ FDI_RX_INTERRUPTS_TRANSCODER_C, ++ AUDIO_CP_CHANGE_TRANSCODER_C, ++ AUDIO_CP_REQUEST_TRANSCODER_C, ++ ERR_AND_DBG, ++ GMBUS, ++ SDVO_B_HOTPLUG, ++ CRT_HOTPLUG, ++ DP_B_HOTPLUG, ++ DP_C_HOTPLUG, ++ DP_D_HOTPLUG, ++ AUX_CHANNEL_B, ++ AUX_CHANNEL_C, ++ AUX_CHANNEL_D, ++ AUDIO_POWER_STATE_CHANGE_B, ++ AUDIO_POWER_STATE_CHANGE_C, ++ AUDIO_POWER_STATE_CHANGE_D, ++ ++ INTEL_GVT_EVENT_RESERVED, ++ INTEL_GVT_EVENT_MAX, ++}; ++ ++struct intel_gvt_irq; ++struct intel_gvt; ++ ++typedef void (*gvt_event_virt_handler_t)(struct intel_gvt_irq *irq, ++ enum intel_gvt_event_type event, struct intel_vgpu *vgpu); ++ ++struct intel_gvt_irq_ops { ++ void (*init_irq)(struct intel_gvt_irq *irq); ++ void (*check_pending_irq)(struct intel_vgpu *vgpu); ++}; ++ ++/* the list of physical interrupt control register groups */ ++enum intel_gvt_irq_type { ++ INTEL_GVT_IRQ_INFO_GT, ++ INTEL_GVT_IRQ_INFO_DPY, ++ INTEL_GVT_IRQ_INFO_PCH, ++ INTEL_GVT_IRQ_INFO_PM, ++ ++ INTEL_GVT_IRQ_INFO_MASTER, ++ INTEL_GVT_IRQ_INFO_GT0, ++ INTEL_GVT_IRQ_INFO_GT1, ++ INTEL_GVT_IRQ_INFO_GT2, ++ INTEL_GVT_IRQ_INFO_GT3, ++ INTEL_GVT_IRQ_INFO_DE_PIPE_A, ++ INTEL_GVT_IRQ_INFO_DE_PIPE_B, ++ INTEL_GVT_IRQ_INFO_DE_PIPE_C, ++ INTEL_GVT_IRQ_INFO_DE_PORT, ++ INTEL_GVT_IRQ_INFO_DE_MISC, ++ INTEL_GVT_IRQ_INFO_AUD, ++ INTEL_GVT_IRQ_INFO_PCU, ++ ++ INTEL_GVT_IRQ_INFO_MAX, ++}; ++ ++#define INTEL_GVT_IRQ_BITWIDTH 32 ++ ++/* device specific interrupt bit definitions */ ++struct intel_gvt_irq_info { ++ char *name; ++ i915_reg_t reg_base; ++ enum intel_gvt_event_type bit_to_event[INTEL_GVT_IRQ_BITWIDTH]; ++ unsigned long warned; ++ int group; ++ DECLARE_BITMAP(downstream_irq_bitmap, INTEL_GVT_IRQ_BITWIDTH); ++ bool has_upstream_irq; ++}; ++ ++/* per-event information */ ++struct intel_gvt_event_info { ++ int bit; /* map to register bit */ ++ int policy; /* forwarding policy */ ++ struct intel_gvt_irq_info *info; /* register info */ ++ gvt_event_virt_handler_t v_handler; /* for v_event */ ++}; ++ ++struct intel_gvt_irq_map { ++ int up_irq_group; ++ int up_irq_bit; ++ int down_irq_group; ++ u32 down_irq_bitmask; ++}; ++ ++struct intel_gvt_vblank_timer { ++ struct hrtimer timer; ++ u64 period; ++}; ++ ++/* structure containing device specific IRQ state */ ++struct intel_gvt_irq { ++ struct intel_gvt_irq_ops *ops; ++ struct intel_gvt_irq_info *info[INTEL_GVT_IRQ_INFO_MAX]; ++ DECLARE_BITMAP(irq_info_bitmap, INTEL_GVT_IRQ_INFO_MAX); ++ struct intel_gvt_event_info events[INTEL_GVT_EVENT_MAX]; ++ DECLARE_BITMAP(pending_events, INTEL_GVT_EVENT_MAX); ++ struct intel_gvt_irq_map *irq_map; ++ struct intel_gvt_vblank_timer vblank_timer; ++}; ++ ++int intel_gvt_init_irq(struct intel_gvt *gvt); ++void intel_gvt_clean_irq(struct intel_gvt *gvt); ++ ++void intel_vgpu_trigger_virtual_event(struct intel_vgpu *vgpu, ++ enum intel_gvt_event_type event); ++ ++int intel_vgpu_reg_iir_handler(struct intel_vgpu *vgpu, unsigned int reg, ++ void *p_data, unsigned int bytes); ++int intel_vgpu_reg_ier_handler(struct intel_vgpu *vgpu, ++ unsigned int reg, void *p_data, unsigned int bytes); ++int intel_vgpu_reg_master_irq_handler(struct intel_vgpu *vgpu, ++ unsigned int reg, void *p_data, unsigned int bytes); ++int intel_vgpu_reg_imr_handler(struct intel_vgpu *vgpu, ++ unsigned int reg, void *p_data, unsigned int bytes); ++ ++int gvt_ring_id_to_pipe_control_notify_event(int ring_id); ++int gvt_ring_id_to_mi_flush_dw_event(int ring_id); ++int gvt_ring_id_to_mi_user_interrupt_event(int ring_id); ++ ++#endif /* _GVT_INTERRUPT_H_ */ +diff --git a/drivers/gpu/drm/i915_legacy/gvt/kvmgt.c b/drivers/gpu/drm/i915_legacy/gvt/kvmgt.c +new file mode 100644 +index 000000000000..4a7cf8646b0d +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/kvmgt.c +@@ -0,0 +1,2075 @@ ++/* ++ * KVMGT - the implementation of Intel mediated pass-through framework for KVM ++ * ++ * Copyright(c) 2014-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Kevin Tian ++ * Jike Song ++ * Xiaoguang Chen ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include "i915_drv.h" ++#include "gvt.h" ++ ++static const struct intel_gvt_ops *intel_gvt_ops; ++ ++/* helper macros copied from vfio-pci */ ++#define VFIO_PCI_OFFSET_SHIFT 40 ++#define VFIO_PCI_OFFSET_TO_INDEX(off) (off >> VFIO_PCI_OFFSET_SHIFT) ++#define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT) ++#define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1) ++ ++#define EDID_BLOB_OFFSET (PAGE_SIZE/2) ++ ++#define OPREGION_SIGNATURE "IntelGraphicsMem" ++ ++struct vfio_region; ++struct intel_vgpu_regops { ++ size_t (*rw)(struct intel_vgpu *vgpu, char *buf, ++ size_t count, loff_t *ppos, bool iswrite); ++ void (*release)(struct intel_vgpu *vgpu, ++ struct vfio_region *region); ++}; ++ ++struct vfio_region { ++ u32 type; ++ u32 subtype; ++ size_t size; ++ u32 flags; ++ const struct intel_vgpu_regops *ops; ++ void *data; ++}; ++ ++struct vfio_edid_region { ++ struct vfio_region_gfx_edid vfio_edid_regs; ++ void *edid_blob; ++}; ++ ++struct kvmgt_pgfn { ++ gfn_t gfn; ++ struct hlist_node hnode; ++}; ++ ++struct kvmgt_guest_info { ++ struct kvm *kvm; ++ struct intel_vgpu *vgpu; ++ struct kvm_page_track_notifier_node track_node; ++#define NR_BKT (1 << 18) ++ struct hlist_head ptable[NR_BKT]; ++#undef NR_BKT ++ struct dentry *debugfs_cache_entries; ++}; ++ ++struct gvt_dma { ++ struct intel_vgpu *vgpu; ++ struct rb_node gfn_node; ++ struct rb_node dma_addr_node; ++ gfn_t gfn; ++ dma_addr_t dma_addr; ++ unsigned long size; ++ struct kref ref; ++}; ++ ++static inline bool handle_valid(unsigned long handle) ++{ ++ return !!(handle & ~0xff); ++} ++ ++static int kvmgt_guest_init(struct mdev_device *mdev); ++static void intel_vgpu_release_work(struct work_struct *work); ++static bool kvmgt_guest_exit(struct kvmgt_guest_info *info); ++ ++static void gvt_unpin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn, ++ unsigned long size) ++{ ++ int total_pages; ++ int npage; ++ int ret; ++ ++ total_pages = roundup(size, PAGE_SIZE) / PAGE_SIZE; ++ ++ for (npage = 0; npage < total_pages; npage++) { ++ unsigned long cur_gfn = gfn + npage; ++ ++ ret = vfio_unpin_pages(mdev_dev(vgpu->vdev.mdev), &cur_gfn, 1); ++ WARN_ON(ret != 1); ++ } ++} ++ ++/* Pin a normal or compound guest page for dma. */ ++static int gvt_pin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn, ++ unsigned long size, struct page **page) ++{ ++ unsigned long base_pfn = 0; ++ int total_pages; ++ int npage; ++ int ret; ++ ++ total_pages = roundup(size, PAGE_SIZE) / PAGE_SIZE; ++ /* ++ * We pin the pages one-by-one to avoid allocating a big arrary ++ * on stack to hold pfns. ++ */ ++ for (npage = 0; npage < total_pages; npage++) { ++ unsigned long cur_gfn = gfn + npage; ++ unsigned long pfn; ++ ++ ret = vfio_pin_pages(mdev_dev(vgpu->vdev.mdev), &cur_gfn, 1, ++ IOMMU_READ | IOMMU_WRITE, &pfn); ++ if (ret != 1) { ++ gvt_vgpu_err("vfio_pin_pages failed for gfn 0x%lx, ret %d\n", ++ cur_gfn, ret); ++ goto err; ++ } ++ ++ if (!pfn_valid(pfn)) { ++ gvt_vgpu_err("pfn 0x%lx is not mem backed\n", pfn); ++ npage++; ++ ret = -EFAULT; ++ goto err; ++ } ++ ++ if (npage == 0) ++ base_pfn = pfn; ++ else if (base_pfn + npage != pfn) { ++ gvt_vgpu_err("The pages are not continuous\n"); ++ ret = -EINVAL; ++ npage++; ++ goto err; ++ } ++ } ++ ++ *page = pfn_to_page(base_pfn); ++ return 0; ++err: ++ gvt_unpin_guest_page(vgpu, gfn, npage * PAGE_SIZE); ++ return ret; ++} ++ ++static int gvt_dma_map_page(struct intel_vgpu *vgpu, unsigned long gfn, ++ dma_addr_t *dma_addr, unsigned long size) ++{ ++ struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev; ++ struct page *page = NULL; ++ int ret; ++ ++ ret = gvt_pin_guest_page(vgpu, gfn, size, &page); ++ if (ret) ++ return ret; ++ ++ /* Setup DMA mapping. */ ++ *dma_addr = dma_map_page(dev, page, 0, size, PCI_DMA_BIDIRECTIONAL); ++ if (dma_mapping_error(dev, *dma_addr)) { ++ gvt_vgpu_err("DMA mapping failed for pfn 0x%lx, ret %d\n", ++ page_to_pfn(page), ret); ++ gvt_unpin_guest_page(vgpu, gfn, size); ++ return -ENOMEM; ++ } ++ ++ return 0; ++} ++ ++static void gvt_dma_unmap_page(struct intel_vgpu *vgpu, unsigned long gfn, ++ dma_addr_t dma_addr, unsigned long size) ++{ ++ struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev; ++ ++ dma_unmap_page(dev, dma_addr, size, PCI_DMA_BIDIRECTIONAL); ++ gvt_unpin_guest_page(vgpu, gfn, size); ++} ++ ++static struct gvt_dma *__gvt_cache_find_dma_addr(struct intel_vgpu *vgpu, ++ dma_addr_t dma_addr) ++{ ++ struct rb_node *node = vgpu->vdev.dma_addr_cache.rb_node; ++ struct gvt_dma *itr; ++ ++ while (node) { ++ itr = rb_entry(node, struct gvt_dma, dma_addr_node); ++ ++ if (dma_addr < itr->dma_addr) ++ node = node->rb_left; ++ else if (dma_addr > itr->dma_addr) ++ node = node->rb_right; ++ else ++ return itr; ++ } ++ return NULL; ++} ++ ++static struct gvt_dma *__gvt_cache_find_gfn(struct intel_vgpu *vgpu, gfn_t gfn) ++{ ++ struct rb_node *node = vgpu->vdev.gfn_cache.rb_node; ++ struct gvt_dma *itr; ++ ++ while (node) { ++ itr = rb_entry(node, struct gvt_dma, gfn_node); ++ ++ if (gfn < itr->gfn) ++ node = node->rb_left; ++ else if (gfn > itr->gfn) ++ node = node->rb_right; ++ else ++ return itr; ++ } ++ return NULL; ++} ++ ++static int __gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn, ++ dma_addr_t dma_addr, unsigned long size) ++{ ++ struct gvt_dma *new, *itr; ++ struct rb_node **link, *parent = NULL; ++ ++ new = kzalloc(sizeof(struct gvt_dma), GFP_KERNEL); ++ if (!new) ++ return -ENOMEM; ++ ++ new->vgpu = vgpu; ++ new->gfn = gfn; ++ new->dma_addr = dma_addr; ++ new->size = size; ++ kref_init(&new->ref); ++ ++ /* gfn_cache maps gfn to struct gvt_dma. */ ++ link = &vgpu->vdev.gfn_cache.rb_node; ++ while (*link) { ++ parent = *link; ++ itr = rb_entry(parent, struct gvt_dma, gfn_node); ++ ++ if (gfn < itr->gfn) ++ link = &parent->rb_left; ++ else ++ link = &parent->rb_right; ++ } ++ rb_link_node(&new->gfn_node, parent, link); ++ rb_insert_color(&new->gfn_node, &vgpu->vdev.gfn_cache); ++ ++ /* dma_addr_cache maps dma addr to struct gvt_dma. */ ++ parent = NULL; ++ link = &vgpu->vdev.dma_addr_cache.rb_node; ++ while (*link) { ++ parent = *link; ++ itr = rb_entry(parent, struct gvt_dma, dma_addr_node); ++ ++ if (dma_addr < itr->dma_addr) ++ link = &parent->rb_left; ++ else ++ link = &parent->rb_right; ++ } ++ rb_link_node(&new->dma_addr_node, parent, link); ++ rb_insert_color(&new->dma_addr_node, &vgpu->vdev.dma_addr_cache); ++ ++ vgpu->vdev.nr_cache_entries++; ++ return 0; ++} ++ ++static void __gvt_cache_remove_entry(struct intel_vgpu *vgpu, ++ struct gvt_dma *entry) ++{ ++ rb_erase(&entry->gfn_node, &vgpu->vdev.gfn_cache); ++ rb_erase(&entry->dma_addr_node, &vgpu->vdev.dma_addr_cache); ++ kfree(entry); ++ vgpu->vdev.nr_cache_entries--; ++} ++ ++static void gvt_cache_destroy(struct intel_vgpu *vgpu) ++{ ++ struct gvt_dma *dma; ++ struct rb_node *node = NULL; ++ ++ for (;;) { ++ mutex_lock(&vgpu->vdev.cache_lock); ++ node = rb_first(&vgpu->vdev.gfn_cache); ++ if (!node) { ++ mutex_unlock(&vgpu->vdev.cache_lock); ++ break; ++ } ++ dma = rb_entry(node, struct gvt_dma, gfn_node); ++ gvt_dma_unmap_page(vgpu, dma->gfn, dma->dma_addr, dma->size); ++ __gvt_cache_remove_entry(vgpu, dma); ++ mutex_unlock(&vgpu->vdev.cache_lock); ++ } ++} ++ ++static void gvt_cache_init(struct intel_vgpu *vgpu) ++{ ++ vgpu->vdev.gfn_cache = RB_ROOT; ++ vgpu->vdev.dma_addr_cache = RB_ROOT; ++ vgpu->vdev.nr_cache_entries = 0; ++ mutex_init(&vgpu->vdev.cache_lock); ++} ++ ++static void kvmgt_protect_table_init(struct kvmgt_guest_info *info) ++{ ++ hash_init(info->ptable); ++} ++ ++static void kvmgt_protect_table_destroy(struct kvmgt_guest_info *info) ++{ ++ struct kvmgt_pgfn *p; ++ struct hlist_node *tmp; ++ int i; ++ ++ hash_for_each_safe(info->ptable, i, tmp, p, hnode) { ++ hash_del(&p->hnode); ++ kfree(p); ++ } ++} ++ ++static struct kvmgt_pgfn * ++__kvmgt_protect_table_find(struct kvmgt_guest_info *info, gfn_t gfn) ++{ ++ struct kvmgt_pgfn *p, *res = NULL; ++ ++ hash_for_each_possible(info->ptable, p, hnode, gfn) { ++ if (gfn == p->gfn) { ++ res = p; ++ break; ++ } ++ } ++ ++ return res; ++} ++ ++static bool kvmgt_gfn_is_write_protected(struct kvmgt_guest_info *info, ++ gfn_t gfn) ++{ ++ struct kvmgt_pgfn *p; ++ ++ p = __kvmgt_protect_table_find(info, gfn); ++ return !!p; ++} ++ ++static void kvmgt_protect_table_add(struct kvmgt_guest_info *info, gfn_t gfn) ++{ ++ struct kvmgt_pgfn *p; ++ ++ if (kvmgt_gfn_is_write_protected(info, gfn)) ++ return; ++ ++ p = kzalloc(sizeof(struct kvmgt_pgfn), GFP_ATOMIC); ++ if (WARN(!p, "gfn: 0x%llx\n", gfn)) ++ return; ++ ++ p->gfn = gfn; ++ hash_add(info->ptable, &p->hnode, gfn); ++} ++ ++static void kvmgt_protect_table_del(struct kvmgt_guest_info *info, ++ gfn_t gfn) ++{ ++ struct kvmgt_pgfn *p; ++ ++ p = __kvmgt_protect_table_find(info, gfn); ++ if (p) { ++ hash_del(&p->hnode); ++ kfree(p); ++ } ++} ++ ++static size_t intel_vgpu_reg_rw_opregion(struct intel_vgpu *vgpu, char *buf, ++ size_t count, loff_t *ppos, bool iswrite) ++{ ++ unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) - ++ VFIO_PCI_NUM_REGIONS; ++ void *base = vgpu->vdev.region[i].data; ++ loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK; ++ ++ if (pos >= vgpu->vdev.region[i].size || iswrite) { ++ gvt_vgpu_err("invalid op or offset for Intel vgpu OpRegion\n"); ++ return -EINVAL; ++ } ++ count = min(count, (size_t)(vgpu->vdev.region[i].size - pos)); ++ memcpy(buf, base + pos, count); ++ ++ return count; ++} ++ ++static void intel_vgpu_reg_release_opregion(struct intel_vgpu *vgpu, ++ struct vfio_region *region) ++{ ++} ++ ++static const struct intel_vgpu_regops intel_vgpu_regops_opregion = { ++ .rw = intel_vgpu_reg_rw_opregion, ++ .release = intel_vgpu_reg_release_opregion, ++}; ++ ++static int handle_edid_regs(struct intel_vgpu *vgpu, ++ struct vfio_edid_region *region, char *buf, ++ size_t count, u16 offset, bool is_write) ++{ ++ struct vfio_region_gfx_edid *regs = ®ion->vfio_edid_regs; ++ unsigned int data; ++ ++ if (offset + count > sizeof(*regs)) ++ return -EINVAL; ++ ++ if (count != 4) ++ return -EINVAL; ++ ++ if (is_write) { ++ data = *((unsigned int *)buf); ++ switch (offset) { ++ case offsetof(struct vfio_region_gfx_edid, link_state): ++ if (data == VFIO_DEVICE_GFX_LINK_STATE_UP) { ++ if (!drm_edid_block_valid( ++ (u8 *)region->edid_blob, ++ 0, ++ true, ++ NULL)) { ++ gvt_vgpu_err("invalid EDID blob\n"); ++ return -EINVAL; ++ } ++ intel_gvt_ops->emulate_hotplug(vgpu, true); ++ } else if (data == VFIO_DEVICE_GFX_LINK_STATE_DOWN) ++ intel_gvt_ops->emulate_hotplug(vgpu, false); ++ else { ++ gvt_vgpu_err("invalid EDID link state %d\n", ++ regs->link_state); ++ return -EINVAL; ++ } ++ regs->link_state = data; ++ break; ++ case offsetof(struct vfio_region_gfx_edid, edid_size): ++ if (data > regs->edid_max_size) { ++ gvt_vgpu_err("EDID size is bigger than %d!\n", ++ regs->edid_max_size); ++ return -EINVAL; ++ } ++ regs->edid_size = data; ++ break; ++ default: ++ /* read-only regs */ ++ gvt_vgpu_err("write read-only EDID region at offset %d\n", ++ offset); ++ return -EPERM; ++ } ++ } else { ++ memcpy(buf, (char *)regs + offset, count); ++ } ++ ++ return count; ++} ++ ++static int handle_edid_blob(struct vfio_edid_region *region, char *buf, ++ size_t count, u16 offset, bool is_write) ++{ ++ if (offset + count > region->vfio_edid_regs.edid_size) ++ return -EINVAL; ++ ++ if (is_write) ++ memcpy(region->edid_blob + offset, buf, count); ++ else ++ memcpy(buf, region->edid_blob + offset, count); ++ ++ return count; ++} ++ ++static size_t intel_vgpu_reg_rw_edid(struct intel_vgpu *vgpu, char *buf, ++ size_t count, loff_t *ppos, bool iswrite) ++{ ++ int ret; ++ unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) - ++ VFIO_PCI_NUM_REGIONS; ++ struct vfio_edid_region *region = ++ (struct vfio_edid_region *)vgpu->vdev.region[i].data; ++ loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK; ++ ++ if (pos < region->vfio_edid_regs.edid_offset) { ++ ret = handle_edid_regs(vgpu, region, buf, count, pos, iswrite); ++ } else { ++ pos -= EDID_BLOB_OFFSET; ++ ret = handle_edid_blob(region, buf, count, pos, iswrite); ++ } ++ ++ if (ret < 0) ++ gvt_vgpu_err("failed to access EDID region\n"); ++ ++ return ret; ++} ++ ++static void intel_vgpu_reg_release_edid(struct intel_vgpu *vgpu, ++ struct vfio_region *region) ++{ ++ kfree(region->data); ++} ++ ++static const struct intel_vgpu_regops intel_vgpu_regops_edid = { ++ .rw = intel_vgpu_reg_rw_edid, ++ .release = intel_vgpu_reg_release_edid, ++}; ++ ++static int intel_vgpu_register_reg(struct intel_vgpu *vgpu, ++ unsigned int type, unsigned int subtype, ++ const struct intel_vgpu_regops *ops, ++ size_t size, u32 flags, void *data) ++{ ++ struct vfio_region *region; ++ ++ region = krealloc(vgpu->vdev.region, ++ (vgpu->vdev.num_regions + 1) * sizeof(*region), ++ GFP_KERNEL); ++ if (!region) ++ return -ENOMEM; ++ ++ vgpu->vdev.region = region; ++ vgpu->vdev.region[vgpu->vdev.num_regions].type = type; ++ vgpu->vdev.region[vgpu->vdev.num_regions].subtype = subtype; ++ vgpu->vdev.region[vgpu->vdev.num_regions].ops = ops; ++ vgpu->vdev.region[vgpu->vdev.num_regions].size = size; ++ vgpu->vdev.region[vgpu->vdev.num_regions].flags = flags; ++ vgpu->vdev.region[vgpu->vdev.num_regions].data = data; ++ vgpu->vdev.num_regions++; ++ return 0; ++} ++ ++static int kvmgt_get_vfio_device(void *p_vgpu) ++{ ++ struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu; ++ ++ vgpu->vdev.vfio_device = vfio_device_get_from_dev( ++ mdev_dev(vgpu->vdev.mdev)); ++ if (!vgpu->vdev.vfio_device) { ++ gvt_vgpu_err("failed to get vfio device\n"); ++ return -ENODEV; ++ } ++ return 0; ++} ++ ++ ++static int kvmgt_set_opregion(void *p_vgpu) ++{ ++ struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu; ++ void *base; ++ int ret; ++ ++ /* Each vgpu has its own opregion, although VFIO would create another ++ * one later. This one is used to expose opregion to VFIO. And the ++ * other one created by VFIO later, is used by guest actually. ++ */ ++ base = vgpu_opregion(vgpu)->va; ++ if (!base) ++ return -ENOMEM; ++ ++ if (memcmp(base, OPREGION_SIGNATURE, 16)) { ++ memunmap(base); ++ return -EINVAL; ++ } ++ ++ ret = intel_vgpu_register_reg(vgpu, ++ PCI_VENDOR_ID_INTEL | VFIO_REGION_TYPE_PCI_VENDOR_TYPE, ++ VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION, ++ &intel_vgpu_regops_opregion, OPREGION_SIZE, ++ VFIO_REGION_INFO_FLAG_READ, base); ++ ++ return ret; ++} ++ ++static int kvmgt_set_edid(void *p_vgpu, int port_num) ++{ ++ struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu; ++ struct intel_vgpu_port *port = intel_vgpu_port(vgpu, port_num); ++ struct vfio_edid_region *base; ++ int ret; ++ ++ base = kzalloc(sizeof(*base), GFP_KERNEL); ++ if (!base) ++ return -ENOMEM; ++ ++ /* TODO: Add multi-port and EDID extension block support */ ++ base->vfio_edid_regs.edid_offset = EDID_BLOB_OFFSET; ++ base->vfio_edid_regs.edid_max_size = EDID_SIZE; ++ base->vfio_edid_regs.edid_size = EDID_SIZE; ++ base->vfio_edid_regs.max_xres = vgpu_edid_xres(port->id); ++ base->vfio_edid_regs.max_yres = vgpu_edid_yres(port->id); ++ base->edid_blob = port->edid->edid_block; ++ ++ ret = intel_vgpu_register_reg(vgpu, ++ VFIO_REGION_TYPE_GFX, ++ VFIO_REGION_SUBTYPE_GFX_EDID, ++ &intel_vgpu_regops_edid, EDID_SIZE, ++ VFIO_REGION_INFO_FLAG_READ | ++ VFIO_REGION_INFO_FLAG_WRITE | ++ VFIO_REGION_INFO_FLAG_CAPS, base); ++ ++ return ret; ++} ++ ++static void kvmgt_put_vfio_device(void *vgpu) ++{ ++ if (WARN_ON(!((struct intel_vgpu *)vgpu)->vdev.vfio_device)) ++ return; ++ ++ vfio_device_put(((struct intel_vgpu *)vgpu)->vdev.vfio_device); ++} ++ ++static int intel_vgpu_create(struct kobject *kobj, struct mdev_device *mdev) ++{ ++ struct intel_vgpu *vgpu = NULL; ++ struct intel_vgpu_type *type; ++ struct device *pdev; ++ void *gvt; ++ int ret; ++ ++ pdev = mdev_parent_dev(mdev); ++ gvt = kdev_to_i915(pdev)->gvt; ++ ++ type = intel_gvt_ops->gvt_find_vgpu_type(gvt, kobject_name(kobj)); ++ if (!type) { ++ gvt_vgpu_err("failed to find type %s to create\n", ++ kobject_name(kobj)); ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ vgpu = intel_gvt_ops->vgpu_create(gvt, type); ++ if (IS_ERR_OR_NULL(vgpu)) { ++ ret = vgpu == NULL ? -EFAULT : PTR_ERR(vgpu); ++ gvt_err("failed to create intel vgpu: %d\n", ret); ++ goto out; ++ } ++ ++ INIT_WORK(&vgpu->vdev.release_work, intel_vgpu_release_work); ++ ++ vgpu->vdev.mdev = mdev; ++ mdev_set_drvdata(mdev, vgpu); ++ ++ gvt_dbg_core("intel_vgpu_create succeeded for mdev: %s\n", ++ dev_name(mdev_dev(mdev))); ++ ret = 0; ++ ++out: ++ return ret; ++} ++ ++static int intel_vgpu_remove(struct mdev_device *mdev) ++{ ++ struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); ++ ++ if (handle_valid(vgpu->handle)) ++ return -EBUSY; ++ ++ intel_gvt_ops->vgpu_destroy(vgpu); ++ return 0; ++} ++ ++static int intel_vgpu_iommu_notifier(struct notifier_block *nb, ++ unsigned long action, void *data) ++{ ++ struct intel_vgpu *vgpu = container_of(nb, ++ struct intel_vgpu, ++ vdev.iommu_notifier); ++ ++ if (action == VFIO_IOMMU_NOTIFY_DMA_UNMAP) { ++ struct vfio_iommu_type1_dma_unmap *unmap = data; ++ struct gvt_dma *entry; ++ unsigned long iov_pfn, end_iov_pfn; ++ ++ iov_pfn = unmap->iova >> PAGE_SHIFT; ++ end_iov_pfn = iov_pfn + unmap->size / PAGE_SIZE; ++ ++ mutex_lock(&vgpu->vdev.cache_lock); ++ for (; iov_pfn < end_iov_pfn; iov_pfn++) { ++ entry = __gvt_cache_find_gfn(vgpu, iov_pfn); ++ if (!entry) ++ continue; ++ ++ gvt_dma_unmap_page(vgpu, entry->gfn, entry->dma_addr, ++ entry->size); ++ __gvt_cache_remove_entry(vgpu, entry); ++ } ++ mutex_unlock(&vgpu->vdev.cache_lock); ++ } ++ ++ return NOTIFY_OK; ++} ++ ++static int intel_vgpu_group_notifier(struct notifier_block *nb, ++ unsigned long action, void *data) ++{ ++ struct intel_vgpu *vgpu = container_of(nb, ++ struct intel_vgpu, ++ vdev.group_notifier); ++ ++ /* the only action we care about */ ++ if (action == VFIO_GROUP_NOTIFY_SET_KVM) { ++ vgpu->vdev.kvm = data; ++ ++ if (!data) ++ schedule_work(&vgpu->vdev.release_work); ++ } ++ ++ return NOTIFY_OK; ++} ++ ++static int intel_vgpu_open(struct mdev_device *mdev) ++{ ++ struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); ++ unsigned long events; ++ int ret; ++ ++ vgpu->vdev.iommu_notifier.notifier_call = intel_vgpu_iommu_notifier; ++ vgpu->vdev.group_notifier.notifier_call = intel_vgpu_group_notifier; ++ ++ events = VFIO_IOMMU_NOTIFY_DMA_UNMAP; ++ ret = vfio_register_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, &events, ++ &vgpu->vdev.iommu_notifier); ++ if (ret != 0) { ++ gvt_vgpu_err("vfio_register_notifier for iommu failed: %d\n", ++ ret); ++ goto out; ++ } ++ ++ events = VFIO_GROUP_NOTIFY_SET_KVM; ++ ret = vfio_register_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY, &events, ++ &vgpu->vdev.group_notifier); ++ if (ret != 0) { ++ gvt_vgpu_err("vfio_register_notifier for group failed: %d\n", ++ ret); ++ goto undo_iommu; ++ } ++ ++ /* Take a module reference as mdev core doesn't take ++ * a reference for vendor driver. ++ */ ++ if (!try_module_get(THIS_MODULE)) ++ goto undo_group; ++ ++ ret = kvmgt_guest_init(mdev); ++ if (ret) ++ goto undo_group; ++ ++ intel_gvt_ops->vgpu_activate(vgpu); ++ ++ atomic_set(&vgpu->vdev.released, 0); ++ return ret; ++ ++undo_group: ++ vfio_unregister_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY, ++ &vgpu->vdev.group_notifier); ++ ++undo_iommu: ++ vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, ++ &vgpu->vdev.iommu_notifier); ++out: ++ return ret; ++} ++ ++static void intel_vgpu_release_msi_eventfd_ctx(struct intel_vgpu *vgpu) ++{ ++ struct eventfd_ctx *trigger; ++ ++ trigger = vgpu->vdev.msi_trigger; ++ if (trigger) { ++ eventfd_ctx_put(trigger); ++ vgpu->vdev.msi_trigger = NULL; ++ } ++} ++ ++static void __intel_vgpu_release(struct intel_vgpu *vgpu) ++{ ++ struct kvmgt_guest_info *info; ++ int ret; ++ ++ if (!handle_valid(vgpu->handle)) ++ return; ++ ++ if (atomic_cmpxchg(&vgpu->vdev.released, 0, 1)) ++ return; ++ ++ intel_gvt_ops->vgpu_release(vgpu); ++ ++ ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_IOMMU_NOTIFY, ++ &vgpu->vdev.iommu_notifier); ++ WARN(ret, "vfio_unregister_notifier for iommu failed: %d\n", ret); ++ ++ ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_GROUP_NOTIFY, ++ &vgpu->vdev.group_notifier); ++ WARN(ret, "vfio_unregister_notifier for group failed: %d\n", ret); ++ ++ /* dereference module reference taken at open */ ++ module_put(THIS_MODULE); ++ ++ info = (struct kvmgt_guest_info *)vgpu->handle; ++ kvmgt_guest_exit(info); ++ ++ intel_vgpu_release_msi_eventfd_ctx(vgpu); ++ ++ vgpu->vdev.kvm = NULL; ++ vgpu->handle = 0; ++} ++ ++static void intel_vgpu_release(struct mdev_device *mdev) ++{ ++ struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); ++ ++ __intel_vgpu_release(vgpu); ++} ++ ++static void intel_vgpu_release_work(struct work_struct *work) ++{ ++ struct intel_vgpu *vgpu = container_of(work, struct intel_vgpu, ++ vdev.release_work); ++ ++ __intel_vgpu_release(vgpu); ++} ++ ++static u64 intel_vgpu_get_bar_addr(struct intel_vgpu *vgpu, int bar) ++{ ++ u32 start_lo, start_hi; ++ u32 mem_type; ++ ++ start_lo = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) & ++ PCI_BASE_ADDRESS_MEM_MASK; ++ mem_type = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) & ++ PCI_BASE_ADDRESS_MEM_TYPE_MASK; ++ ++ switch (mem_type) { ++ case PCI_BASE_ADDRESS_MEM_TYPE_64: ++ start_hi = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space ++ + bar + 4)); ++ break; ++ case PCI_BASE_ADDRESS_MEM_TYPE_32: ++ case PCI_BASE_ADDRESS_MEM_TYPE_1M: ++ /* 1M mem BAR treated as 32-bit BAR */ ++ default: ++ /* mem unknown type treated as 32-bit BAR */ ++ start_hi = 0; ++ break; ++ } ++ ++ return ((u64)start_hi << 32) | start_lo; ++} ++ ++static int intel_vgpu_bar_rw(struct intel_vgpu *vgpu, int bar, u64 off, ++ void *buf, unsigned int count, bool is_write) ++{ ++ u64 bar_start = intel_vgpu_get_bar_addr(vgpu, bar); ++ int ret; ++ ++ if (is_write) ++ ret = intel_gvt_ops->emulate_mmio_write(vgpu, ++ bar_start + off, buf, count); ++ else ++ ret = intel_gvt_ops->emulate_mmio_read(vgpu, ++ bar_start + off, buf, count); ++ return ret; ++} ++ ++static inline bool intel_vgpu_in_aperture(struct intel_vgpu *vgpu, u64 off) ++{ ++ return off >= vgpu_aperture_offset(vgpu) && ++ off < vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu); ++} ++ ++static int intel_vgpu_aperture_rw(struct intel_vgpu *vgpu, u64 off, ++ void *buf, unsigned long count, bool is_write) ++{ ++ void __iomem *aperture_va; ++ ++ if (!intel_vgpu_in_aperture(vgpu, off) || ++ !intel_vgpu_in_aperture(vgpu, off + count)) { ++ gvt_vgpu_err("Invalid aperture offset %llu\n", off); ++ return -EINVAL; ++ } ++ ++ aperture_va = io_mapping_map_wc(&vgpu->gvt->dev_priv->ggtt.iomap, ++ ALIGN_DOWN(off, PAGE_SIZE), ++ count + offset_in_page(off)); ++ if (!aperture_va) ++ return -EIO; ++ ++ if (is_write) ++ memcpy_toio(aperture_va + offset_in_page(off), buf, count); ++ else ++ memcpy_fromio(buf, aperture_va + offset_in_page(off), count); ++ ++ io_mapping_unmap(aperture_va); ++ ++ return 0; ++} ++ ++static ssize_t intel_vgpu_rw(struct mdev_device *mdev, char *buf, ++ size_t count, loff_t *ppos, bool is_write) ++{ ++ struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); ++ unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); ++ u64 pos = *ppos & VFIO_PCI_OFFSET_MASK; ++ int ret = -EINVAL; ++ ++ ++ if (index >= VFIO_PCI_NUM_REGIONS + vgpu->vdev.num_regions) { ++ gvt_vgpu_err("invalid index: %u\n", index); ++ return -EINVAL; ++ } ++ ++ switch (index) { ++ case VFIO_PCI_CONFIG_REGION_INDEX: ++ if (is_write) ++ ret = intel_gvt_ops->emulate_cfg_write(vgpu, pos, ++ buf, count); ++ else ++ ret = intel_gvt_ops->emulate_cfg_read(vgpu, pos, ++ buf, count); ++ break; ++ case VFIO_PCI_BAR0_REGION_INDEX: ++ ret = intel_vgpu_bar_rw(vgpu, PCI_BASE_ADDRESS_0, pos, ++ buf, count, is_write); ++ break; ++ case VFIO_PCI_BAR2_REGION_INDEX: ++ ret = intel_vgpu_aperture_rw(vgpu, pos, buf, count, is_write); ++ break; ++ case VFIO_PCI_BAR1_REGION_INDEX: ++ case VFIO_PCI_BAR3_REGION_INDEX: ++ case VFIO_PCI_BAR4_REGION_INDEX: ++ case VFIO_PCI_BAR5_REGION_INDEX: ++ case VFIO_PCI_VGA_REGION_INDEX: ++ case VFIO_PCI_ROM_REGION_INDEX: ++ break; ++ default: ++ if (index >= VFIO_PCI_NUM_REGIONS + vgpu->vdev.num_regions) ++ return -EINVAL; ++ ++ index -= VFIO_PCI_NUM_REGIONS; ++ return vgpu->vdev.region[index].ops->rw(vgpu, buf, count, ++ ppos, is_write); ++ } ++ ++ return ret == 0 ? count : ret; ++} ++ ++static bool gtt_entry(struct mdev_device *mdev, loff_t *ppos) ++{ ++ struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); ++ unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); ++ struct intel_gvt *gvt = vgpu->gvt; ++ int offset; ++ ++ /* Only allow MMIO GGTT entry access */ ++ if (index != PCI_BASE_ADDRESS_0) ++ return false; ++ ++ offset = (u64)(*ppos & VFIO_PCI_OFFSET_MASK) - ++ intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_0); ++ ++ return (offset >= gvt->device_info.gtt_start_offset && ++ offset < gvt->device_info.gtt_start_offset + gvt_ggtt_sz(gvt)) ? ++ true : false; ++} ++ ++static ssize_t intel_vgpu_read(struct mdev_device *mdev, char __user *buf, ++ size_t count, loff_t *ppos) ++{ ++ unsigned int done = 0; ++ int ret; ++ ++ while (count) { ++ size_t filled; ++ ++ /* Only support GGTT entry 8 bytes read */ ++ if (count >= 8 && !(*ppos % 8) && ++ gtt_entry(mdev, ppos)) { ++ u64 val; ++ ++ ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), ++ ppos, false); ++ if (ret <= 0) ++ goto read_err; ++ ++ if (copy_to_user(buf, &val, sizeof(val))) ++ goto read_err; ++ ++ filled = 8; ++ } else if (count >= 4 && !(*ppos % 4)) { ++ u32 val; ++ ++ ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), ++ ppos, false); ++ if (ret <= 0) ++ goto read_err; ++ ++ if (copy_to_user(buf, &val, sizeof(val))) ++ goto read_err; ++ ++ filled = 4; ++ } else if (count >= 2 && !(*ppos % 2)) { ++ u16 val; ++ ++ ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), ++ ppos, false); ++ if (ret <= 0) ++ goto read_err; ++ ++ if (copy_to_user(buf, &val, sizeof(val))) ++ goto read_err; ++ ++ filled = 2; ++ } else { ++ u8 val; ++ ++ ret = intel_vgpu_rw(mdev, &val, sizeof(val), ppos, ++ false); ++ if (ret <= 0) ++ goto read_err; ++ ++ if (copy_to_user(buf, &val, sizeof(val))) ++ goto read_err; ++ ++ filled = 1; ++ } ++ ++ count -= filled; ++ done += filled; ++ *ppos += filled; ++ buf += filled; ++ } ++ ++ return done; ++ ++read_err: ++ return -EFAULT; ++} ++ ++static ssize_t intel_vgpu_write(struct mdev_device *mdev, ++ const char __user *buf, ++ size_t count, loff_t *ppos) ++{ ++ unsigned int done = 0; ++ int ret; ++ ++ while (count) { ++ size_t filled; ++ ++ /* Only support GGTT entry 8 bytes write */ ++ if (count >= 8 && !(*ppos % 8) && ++ gtt_entry(mdev, ppos)) { ++ u64 val; ++ ++ if (copy_from_user(&val, buf, sizeof(val))) ++ goto write_err; ++ ++ ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), ++ ppos, true); ++ if (ret <= 0) ++ goto write_err; ++ ++ filled = 8; ++ } else if (count >= 4 && !(*ppos % 4)) { ++ u32 val; ++ ++ if (copy_from_user(&val, buf, sizeof(val))) ++ goto write_err; ++ ++ ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), ++ ppos, true); ++ if (ret <= 0) ++ goto write_err; ++ ++ filled = 4; ++ } else if (count >= 2 && !(*ppos % 2)) { ++ u16 val; ++ ++ if (copy_from_user(&val, buf, sizeof(val))) ++ goto write_err; ++ ++ ret = intel_vgpu_rw(mdev, (char *)&val, ++ sizeof(val), ppos, true); ++ if (ret <= 0) ++ goto write_err; ++ ++ filled = 2; ++ } else { ++ u8 val; ++ ++ if (copy_from_user(&val, buf, sizeof(val))) ++ goto write_err; ++ ++ ret = intel_vgpu_rw(mdev, &val, sizeof(val), ++ ppos, true); ++ if (ret <= 0) ++ goto write_err; ++ ++ filled = 1; ++ } ++ ++ count -= filled; ++ done += filled; ++ *ppos += filled; ++ buf += filled; ++ } ++ ++ return done; ++write_err: ++ return -EFAULT; ++} ++ ++static int intel_vgpu_mmap(struct mdev_device *mdev, struct vm_area_struct *vma) ++{ ++ unsigned int index; ++ u64 virtaddr; ++ unsigned long req_size, pgoff, req_start; ++ pgprot_t pg_prot; ++ struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); ++ ++ index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT); ++ if (index >= VFIO_PCI_ROM_REGION_INDEX) ++ return -EINVAL; ++ ++ if (vma->vm_end < vma->vm_start) ++ return -EINVAL; ++ if ((vma->vm_flags & VM_SHARED) == 0) ++ return -EINVAL; ++ if (index != VFIO_PCI_BAR2_REGION_INDEX) ++ return -EINVAL; ++ ++ pg_prot = vma->vm_page_prot; ++ virtaddr = vma->vm_start; ++ req_size = vma->vm_end - vma->vm_start; ++ pgoff = vma->vm_pgoff & ++ ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1); ++ req_start = pgoff << PAGE_SHIFT; ++ ++ if (!intel_vgpu_in_aperture(vgpu, req_start)) ++ return -EINVAL; ++ if (req_start + req_size > ++ vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu)) ++ return -EINVAL; ++ ++ pgoff = (gvt_aperture_pa_base(vgpu->gvt) >> PAGE_SHIFT) + pgoff; ++ ++ return remap_pfn_range(vma, virtaddr, pgoff, req_size, pg_prot); ++} ++ ++static int intel_vgpu_get_irq_count(struct intel_vgpu *vgpu, int type) ++{ ++ if (type == VFIO_PCI_INTX_IRQ_INDEX || type == VFIO_PCI_MSI_IRQ_INDEX) ++ return 1; ++ ++ return 0; ++} ++ ++static int intel_vgpu_set_intx_mask(struct intel_vgpu *vgpu, ++ unsigned int index, unsigned int start, ++ unsigned int count, u32 flags, ++ void *data) ++{ ++ return 0; ++} ++ ++static int intel_vgpu_set_intx_unmask(struct intel_vgpu *vgpu, ++ unsigned int index, unsigned int start, ++ unsigned int count, u32 flags, void *data) ++{ ++ return 0; ++} ++ ++static int intel_vgpu_set_intx_trigger(struct intel_vgpu *vgpu, ++ unsigned int index, unsigned int start, unsigned int count, ++ u32 flags, void *data) ++{ ++ return 0; ++} ++ ++static int intel_vgpu_set_msi_trigger(struct intel_vgpu *vgpu, ++ unsigned int index, unsigned int start, unsigned int count, ++ u32 flags, void *data) ++{ ++ struct eventfd_ctx *trigger; ++ ++ if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { ++ int fd = *(int *)data; ++ ++ trigger = eventfd_ctx_fdget(fd); ++ if (IS_ERR(trigger)) { ++ gvt_vgpu_err("eventfd_ctx_fdget failed\n"); ++ return PTR_ERR(trigger); ++ } ++ vgpu->vdev.msi_trigger = trigger; ++ } else if ((flags & VFIO_IRQ_SET_DATA_NONE) && !count) ++ intel_vgpu_release_msi_eventfd_ctx(vgpu); ++ ++ return 0; ++} ++ ++static int intel_vgpu_set_irqs(struct intel_vgpu *vgpu, u32 flags, ++ unsigned int index, unsigned int start, unsigned int count, ++ void *data) ++{ ++ int (*func)(struct intel_vgpu *vgpu, unsigned int index, ++ unsigned int start, unsigned int count, u32 flags, ++ void *data) = NULL; ++ ++ switch (index) { ++ case VFIO_PCI_INTX_IRQ_INDEX: ++ switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { ++ case VFIO_IRQ_SET_ACTION_MASK: ++ func = intel_vgpu_set_intx_mask; ++ break; ++ case VFIO_IRQ_SET_ACTION_UNMASK: ++ func = intel_vgpu_set_intx_unmask; ++ break; ++ case VFIO_IRQ_SET_ACTION_TRIGGER: ++ func = intel_vgpu_set_intx_trigger; ++ break; ++ } ++ break; ++ case VFIO_PCI_MSI_IRQ_INDEX: ++ switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { ++ case VFIO_IRQ_SET_ACTION_MASK: ++ case VFIO_IRQ_SET_ACTION_UNMASK: ++ /* XXX Need masking support exported */ ++ break; ++ case VFIO_IRQ_SET_ACTION_TRIGGER: ++ func = intel_vgpu_set_msi_trigger; ++ break; ++ } ++ break; ++ } ++ ++ if (!func) ++ return -ENOTTY; ++ ++ return func(vgpu, index, start, count, flags, data); ++} ++ ++static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd, ++ unsigned long arg) ++{ ++ struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); ++ unsigned long minsz; ++ ++ gvt_dbg_core("vgpu%d ioctl, cmd: %d\n", vgpu->id, cmd); ++ ++ if (cmd == VFIO_DEVICE_GET_INFO) { ++ struct vfio_device_info info; ++ ++ minsz = offsetofend(struct vfio_device_info, num_irqs); ++ ++ if (copy_from_user(&info, (void __user *)arg, minsz)) ++ return -EFAULT; ++ ++ if (info.argsz < minsz) ++ return -EINVAL; ++ ++ info.flags = VFIO_DEVICE_FLAGS_PCI; ++ info.flags |= VFIO_DEVICE_FLAGS_RESET; ++ info.num_regions = VFIO_PCI_NUM_REGIONS + ++ vgpu->vdev.num_regions; ++ info.num_irqs = VFIO_PCI_NUM_IRQS; ++ ++ return copy_to_user((void __user *)arg, &info, minsz) ? ++ -EFAULT : 0; ++ ++ } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) { ++ struct vfio_region_info info; ++ struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; ++ unsigned int i; ++ int ret; ++ struct vfio_region_info_cap_sparse_mmap *sparse = NULL; ++ size_t size; ++ int nr_areas = 1; ++ int cap_type_id; ++ ++ minsz = offsetofend(struct vfio_region_info, offset); ++ ++ if (copy_from_user(&info, (void __user *)arg, minsz)) ++ return -EFAULT; ++ ++ if (info.argsz < minsz) ++ return -EINVAL; ++ ++ switch (info.index) { ++ case VFIO_PCI_CONFIG_REGION_INDEX: ++ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); ++ info.size = vgpu->gvt->device_info.cfg_space_size; ++ info.flags = VFIO_REGION_INFO_FLAG_READ | ++ VFIO_REGION_INFO_FLAG_WRITE; ++ break; ++ case VFIO_PCI_BAR0_REGION_INDEX: ++ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); ++ info.size = vgpu->cfg_space.bar[info.index].size; ++ if (!info.size) { ++ info.flags = 0; ++ break; ++ } ++ ++ info.flags = VFIO_REGION_INFO_FLAG_READ | ++ VFIO_REGION_INFO_FLAG_WRITE; ++ break; ++ case VFIO_PCI_BAR1_REGION_INDEX: ++ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); ++ info.size = 0; ++ info.flags = 0; ++ break; ++ case VFIO_PCI_BAR2_REGION_INDEX: ++ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); ++ info.flags = VFIO_REGION_INFO_FLAG_CAPS | ++ VFIO_REGION_INFO_FLAG_MMAP | ++ VFIO_REGION_INFO_FLAG_READ | ++ VFIO_REGION_INFO_FLAG_WRITE; ++ info.size = gvt_aperture_sz(vgpu->gvt); ++ ++ size = sizeof(*sparse) + ++ (nr_areas * sizeof(*sparse->areas)); ++ sparse = kzalloc(size, GFP_KERNEL); ++ if (!sparse) ++ return -ENOMEM; ++ ++ sparse->header.id = VFIO_REGION_INFO_CAP_SPARSE_MMAP; ++ sparse->header.version = 1; ++ sparse->nr_areas = nr_areas; ++ cap_type_id = VFIO_REGION_INFO_CAP_SPARSE_MMAP; ++ sparse->areas[0].offset = ++ PAGE_ALIGN(vgpu_aperture_offset(vgpu)); ++ sparse->areas[0].size = vgpu_aperture_sz(vgpu); ++ break; ++ ++ case VFIO_PCI_BAR3_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: ++ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); ++ info.size = 0; ++ info.flags = 0; ++ ++ gvt_dbg_core("get region info bar:%d\n", info.index); ++ break; ++ ++ case VFIO_PCI_ROM_REGION_INDEX: ++ case VFIO_PCI_VGA_REGION_INDEX: ++ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); ++ info.size = 0; ++ info.flags = 0; ++ ++ gvt_dbg_core("get region info index:%d\n", info.index); ++ break; ++ default: ++ { ++ struct vfio_region_info_cap_type cap_type = { ++ .header.id = VFIO_REGION_INFO_CAP_TYPE, ++ .header.version = 1 }; ++ ++ if (info.index >= VFIO_PCI_NUM_REGIONS + ++ vgpu->vdev.num_regions) ++ return -EINVAL; ++ info.index = ++ array_index_nospec(info.index, ++ VFIO_PCI_NUM_REGIONS + ++ vgpu->vdev.num_regions); ++ ++ i = info.index - VFIO_PCI_NUM_REGIONS; ++ ++ info.offset = ++ VFIO_PCI_INDEX_TO_OFFSET(info.index); ++ info.size = vgpu->vdev.region[i].size; ++ info.flags = vgpu->vdev.region[i].flags; ++ ++ cap_type.type = vgpu->vdev.region[i].type; ++ cap_type.subtype = vgpu->vdev.region[i].subtype; ++ ++ ret = vfio_info_add_capability(&caps, ++ &cap_type.header, ++ sizeof(cap_type)); ++ if (ret) ++ return ret; ++ } ++ } ++ ++ if ((info.flags & VFIO_REGION_INFO_FLAG_CAPS) && sparse) { ++ switch (cap_type_id) { ++ case VFIO_REGION_INFO_CAP_SPARSE_MMAP: ++ ret = vfio_info_add_capability(&caps, ++ &sparse->header, sizeof(*sparse) + ++ (sparse->nr_areas * ++ sizeof(*sparse->areas))); ++ if (ret) { ++ kfree(sparse); ++ return ret; ++ } ++ break; ++ default: ++ kfree(sparse); ++ return -EINVAL; ++ } ++ } ++ ++ if (caps.size) { ++ info.flags |= VFIO_REGION_INFO_FLAG_CAPS; ++ if (info.argsz < sizeof(info) + caps.size) { ++ info.argsz = sizeof(info) + caps.size; ++ info.cap_offset = 0; ++ } else { ++ vfio_info_cap_shift(&caps, sizeof(info)); ++ if (copy_to_user((void __user *)arg + ++ sizeof(info), caps.buf, ++ caps.size)) { ++ kfree(caps.buf); ++ kfree(sparse); ++ return -EFAULT; ++ } ++ info.cap_offset = sizeof(info); ++ } ++ ++ kfree(caps.buf); ++ } ++ ++ kfree(sparse); ++ return copy_to_user((void __user *)arg, &info, minsz) ? ++ -EFAULT : 0; ++ } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) { ++ struct vfio_irq_info info; ++ ++ minsz = offsetofend(struct vfio_irq_info, count); ++ ++ if (copy_from_user(&info, (void __user *)arg, minsz)) ++ return -EFAULT; ++ ++ if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS) ++ return -EINVAL; ++ ++ switch (info.index) { ++ case VFIO_PCI_INTX_IRQ_INDEX: ++ case VFIO_PCI_MSI_IRQ_INDEX: ++ break; ++ default: ++ return -EINVAL; ++ } ++ ++ info.flags = VFIO_IRQ_INFO_EVENTFD; ++ ++ info.count = intel_vgpu_get_irq_count(vgpu, info.index); ++ ++ if (info.index == VFIO_PCI_INTX_IRQ_INDEX) ++ info.flags |= (VFIO_IRQ_INFO_MASKABLE | ++ VFIO_IRQ_INFO_AUTOMASKED); ++ else ++ info.flags |= VFIO_IRQ_INFO_NORESIZE; ++ ++ return copy_to_user((void __user *)arg, &info, minsz) ? ++ -EFAULT : 0; ++ } else if (cmd == VFIO_DEVICE_SET_IRQS) { ++ struct vfio_irq_set hdr; ++ u8 *data = NULL; ++ int ret = 0; ++ size_t data_size = 0; ++ ++ minsz = offsetofend(struct vfio_irq_set, count); ++ ++ if (copy_from_user(&hdr, (void __user *)arg, minsz)) ++ return -EFAULT; ++ ++ if (!(hdr.flags & VFIO_IRQ_SET_DATA_NONE)) { ++ int max = intel_vgpu_get_irq_count(vgpu, hdr.index); ++ ++ ret = vfio_set_irqs_validate_and_prepare(&hdr, max, ++ VFIO_PCI_NUM_IRQS, &data_size); ++ if (ret) { ++ gvt_vgpu_err("intel:vfio_set_irqs_validate_and_prepare failed\n"); ++ return -EINVAL; ++ } ++ if (data_size) { ++ data = memdup_user((void __user *)(arg + minsz), ++ data_size); ++ if (IS_ERR(data)) ++ return PTR_ERR(data); ++ } ++ } ++ ++ ret = intel_vgpu_set_irqs(vgpu, hdr.flags, hdr.index, ++ hdr.start, hdr.count, data); ++ kfree(data); ++ ++ return ret; ++ } else if (cmd == VFIO_DEVICE_RESET) { ++ intel_gvt_ops->vgpu_reset(vgpu); ++ return 0; ++ } else if (cmd == VFIO_DEVICE_QUERY_GFX_PLANE) { ++ struct vfio_device_gfx_plane_info dmabuf; ++ int ret = 0; ++ ++ minsz = offsetofend(struct vfio_device_gfx_plane_info, ++ dmabuf_id); ++ if (copy_from_user(&dmabuf, (void __user *)arg, minsz)) ++ return -EFAULT; ++ if (dmabuf.argsz < minsz) ++ return -EINVAL; ++ ++ ret = intel_gvt_ops->vgpu_query_plane(vgpu, &dmabuf); ++ if (ret != 0) ++ return ret; ++ ++ return copy_to_user((void __user *)arg, &dmabuf, minsz) ? ++ -EFAULT : 0; ++ } else if (cmd == VFIO_DEVICE_GET_GFX_DMABUF) { ++ __u32 dmabuf_id; ++ __s32 dmabuf_fd; ++ ++ if (get_user(dmabuf_id, (__u32 __user *)arg)) ++ return -EFAULT; ++ ++ dmabuf_fd = intel_gvt_ops->vgpu_get_dmabuf(vgpu, dmabuf_id); ++ return dmabuf_fd; ++ ++ } ++ ++ return -ENOTTY; ++} ++ ++static ssize_t ++vgpu_id_show(struct device *dev, struct device_attribute *attr, ++ char *buf) ++{ ++ struct mdev_device *mdev = mdev_from_dev(dev); ++ ++ if (mdev) { ++ struct intel_vgpu *vgpu = (struct intel_vgpu *) ++ mdev_get_drvdata(mdev); ++ return sprintf(buf, "%d\n", vgpu->id); ++ } ++ return sprintf(buf, "\n"); ++} ++ ++static ssize_t ++hw_id_show(struct device *dev, struct device_attribute *attr, ++ char *buf) ++{ ++ struct mdev_device *mdev = mdev_from_dev(dev); ++ ++ if (mdev) { ++ struct intel_vgpu *vgpu = (struct intel_vgpu *) ++ mdev_get_drvdata(mdev); ++ return sprintf(buf, "%u\n", ++ vgpu->submission.shadow_ctx->hw_id); ++ } ++ return sprintf(buf, "\n"); ++} ++ ++static DEVICE_ATTR_RO(vgpu_id); ++static DEVICE_ATTR_RO(hw_id); ++ ++static struct attribute *intel_vgpu_attrs[] = { ++ &dev_attr_vgpu_id.attr, ++ &dev_attr_hw_id.attr, ++ NULL ++}; ++ ++static const struct attribute_group intel_vgpu_group = { ++ .name = "intel_vgpu", ++ .attrs = intel_vgpu_attrs, ++}; ++ ++static const struct attribute_group *intel_vgpu_groups[] = { ++ &intel_vgpu_group, ++ NULL, ++}; ++ ++static struct mdev_parent_ops intel_vgpu_ops = { ++ .mdev_attr_groups = intel_vgpu_groups, ++ .create = intel_vgpu_create, ++ .remove = intel_vgpu_remove, ++ ++ .open = intel_vgpu_open, ++ .release = intel_vgpu_release, ++ ++ .read = intel_vgpu_read, ++ .write = intel_vgpu_write, ++ .mmap = intel_vgpu_mmap, ++ .ioctl = intel_vgpu_ioctl, ++}; ++ ++static int kvmgt_host_init(struct device *dev, void *gvt, const void *ops) ++{ ++ struct attribute **kvm_type_attrs; ++ struct attribute_group **kvm_vgpu_type_groups; ++ ++ intel_gvt_ops = ops; ++ if (!intel_gvt_ops->get_gvt_attrs(&kvm_type_attrs, ++ &kvm_vgpu_type_groups)) ++ return -EFAULT; ++ intel_vgpu_ops.supported_type_groups = kvm_vgpu_type_groups; ++ ++ return mdev_register_device(dev, &intel_vgpu_ops); ++} ++ ++static void kvmgt_host_exit(struct device *dev) ++{ ++ mdev_unregister_device(dev); ++} ++ ++static int kvmgt_page_track_add(unsigned long handle, u64 gfn) ++{ ++ struct kvmgt_guest_info *info; ++ struct kvm *kvm; ++ struct kvm_memory_slot *slot; ++ int idx; ++ ++ if (!handle_valid(handle)) ++ return -ESRCH; ++ ++ info = (struct kvmgt_guest_info *)handle; ++ kvm = info->kvm; ++ ++ idx = srcu_read_lock(&kvm->srcu); ++ slot = gfn_to_memslot(kvm, gfn); ++ if (!slot) { ++ srcu_read_unlock(&kvm->srcu, idx); ++ return -EINVAL; ++ } ++ ++ spin_lock(&kvm->mmu_lock); ++ ++ if (kvmgt_gfn_is_write_protected(info, gfn)) ++ goto out; ++ ++ kvm_slot_page_track_add_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE); ++ kvmgt_protect_table_add(info, gfn); ++ ++out: ++ spin_unlock(&kvm->mmu_lock); ++ srcu_read_unlock(&kvm->srcu, idx); ++ return 0; ++} ++ ++static int kvmgt_page_track_remove(unsigned long handle, u64 gfn) ++{ ++ struct kvmgt_guest_info *info; ++ struct kvm *kvm; ++ struct kvm_memory_slot *slot; ++ int idx; ++ ++ if (!handle_valid(handle)) ++ return 0; ++ ++ info = (struct kvmgt_guest_info *)handle; ++ kvm = info->kvm; ++ ++ idx = srcu_read_lock(&kvm->srcu); ++ slot = gfn_to_memslot(kvm, gfn); ++ if (!slot) { ++ srcu_read_unlock(&kvm->srcu, idx); ++ return -EINVAL; ++ } ++ ++ spin_lock(&kvm->mmu_lock); ++ ++ if (!kvmgt_gfn_is_write_protected(info, gfn)) ++ goto out; ++ ++ kvm_slot_page_track_remove_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE); ++ kvmgt_protect_table_del(info, gfn); ++ ++out: ++ spin_unlock(&kvm->mmu_lock); ++ srcu_read_unlock(&kvm->srcu, idx); ++ return 0; ++} ++ ++static void kvmgt_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, ++ const u8 *val, int len, ++ struct kvm_page_track_notifier_node *node) ++{ ++ struct kvmgt_guest_info *info = container_of(node, ++ struct kvmgt_guest_info, track_node); ++ ++ if (kvmgt_gfn_is_write_protected(info, gpa_to_gfn(gpa))) ++ intel_gvt_ops->write_protect_handler(info->vgpu, gpa, ++ (void *)val, len); ++} ++ ++static void kvmgt_page_track_flush_slot(struct kvm *kvm, ++ struct kvm_memory_slot *slot, ++ struct kvm_page_track_notifier_node *node) ++{ ++ int i; ++ gfn_t gfn; ++ struct kvmgt_guest_info *info = container_of(node, ++ struct kvmgt_guest_info, track_node); ++ ++ spin_lock(&kvm->mmu_lock); ++ for (i = 0; i < slot->npages; i++) { ++ gfn = slot->base_gfn + i; ++ if (kvmgt_gfn_is_write_protected(info, gfn)) { ++ kvm_slot_page_track_remove_page(kvm, slot, gfn, ++ KVM_PAGE_TRACK_WRITE); ++ kvmgt_protect_table_del(info, gfn); ++ } ++ } ++ spin_unlock(&kvm->mmu_lock); ++} ++ ++static bool __kvmgt_vgpu_exist(struct intel_vgpu *vgpu, struct kvm *kvm) ++{ ++ struct intel_vgpu *itr; ++ struct kvmgt_guest_info *info; ++ int id; ++ bool ret = false; ++ ++ mutex_lock(&vgpu->gvt->lock); ++ for_each_active_vgpu(vgpu->gvt, itr, id) { ++ if (!handle_valid(itr->handle)) ++ continue; ++ ++ info = (struct kvmgt_guest_info *)itr->handle; ++ if (kvm && kvm == info->kvm) { ++ ret = true; ++ goto out; ++ } ++ } ++out: ++ mutex_unlock(&vgpu->gvt->lock); ++ return ret; ++} ++ ++static int kvmgt_guest_init(struct mdev_device *mdev) ++{ ++ struct kvmgt_guest_info *info; ++ struct intel_vgpu *vgpu; ++ struct kvm *kvm; ++ ++ vgpu = mdev_get_drvdata(mdev); ++ if (handle_valid(vgpu->handle)) ++ return -EEXIST; ++ ++ kvm = vgpu->vdev.kvm; ++ if (!kvm || kvm->mm != current->mm) { ++ gvt_vgpu_err("KVM is required to use Intel vGPU\n"); ++ return -ESRCH; ++ } ++ ++ if (__kvmgt_vgpu_exist(vgpu, kvm)) ++ return -EEXIST; ++ ++ info = vzalloc(sizeof(struct kvmgt_guest_info)); ++ if (!info) ++ return -ENOMEM; ++ ++ vgpu->handle = (unsigned long)info; ++ info->vgpu = vgpu; ++ info->kvm = kvm; ++ kvm_get_kvm(info->kvm); ++ ++ kvmgt_protect_table_init(info); ++ gvt_cache_init(vgpu); ++ ++ init_completion(&vgpu->vblank_done); ++ ++ info->track_node.track_write = kvmgt_page_track_write; ++ info->track_node.track_flush_slot = kvmgt_page_track_flush_slot; ++ kvm_page_track_register_notifier(kvm, &info->track_node); ++ ++ info->debugfs_cache_entries = debugfs_create_ulong( ++ "kvmgt_nr_cache_entries", ++ 0444, vgpu->debugfs, ++ &vgpu->vdev.nr_cache_entries); ++ if (!info->debugfs_cache_entries) ++ gvt_vgpu_err("Cannot create kvmgt debugfs entry\n"); ++ ++ return 0; ++} ++ ++static bool kvmgt_guest_exit(struct kvmgt_guest_info *info) ++{ ++ debugfs_remove(info->debugfs_cache_entries); ++ ++ kvm_page_track_unregister_notifier(info->kvm, &info->track_node); ++ kvm_put_kvm(info->kvm); ++ kvmgt_protect_table_destroy(info); ++ gvt_cache_destroy(info->vgpu); ++ vfree(info); ++ ++ return true; ++} ++ ++static int kvmgt_attach_vgpu(void *vgpu, unsigned long *handle) ++{ ++ /* nothing to do here */ ++ return 0; ++} ++ ++static void kvmgt_detach_vgpu(void *p_vgpu) ++{ ++ int i; ++ struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu; ++ ++ if (!vgpu->vdev.region) ++ return; ++ ++ for (i = 0; i < vgpu->vdev.num_regions; i++) ++ if (vgpu->vdev.region[i].ops->release) ++ vgpu->vdev.region[i].ops->release(vgpu, ++ &vgpu->vdev.region[i]); ++ vgpu->vdev.num_regions = 0; ++ kfree(vgpu->vdev.region); ++ vgpu->vdev.region = NULL; ++} ++ ++static int kvmgt_inject_msi(unsigned long handle, u32 addr, u16 data) ++{ ++ struct kvmgt_guest_info *info; ++ struct intel_vgpu *vgpu; ++ ++ if (!handle_valid(handle)) ++ return -ESRCH; ++ ++ info = (struct kvmgt_guest_info *)handle; ++ vgpu = info->vgpu; ++ ++ /* ++ * When guest is poweroff, msi_trigger is set to NULL, but vgpu's ++ * config and mmio register isn't restored to default during guest ++ * poweroff. If this vgpu is still used in next vm, this vgpu's pipe ++ * may be enabled, then once this vgpu is active, it will get inject ++ * vblank interrupt request. But msi_trigger is null until msi is ++ * enabled by guest. so if msi_trigger is null, success is still ++ * returned and don't inject interrupt into guest. ++ */ ++ if (vgpu->vdev.msi_trigger == NULL) ++ return 0; ++ ++ if (eventfd_signal(vgpu->vdev.msi_trigger, 1) == 1) ++ return 0; ++ ++ return -EFAULT; ++} ++ ++static unsigned long kvmgt_gfn_to_pfn(unsigned long handle, unsigned long gfn) ++{ ++ struct kvmgt_guest_info *info; ++ kvm_pfn_t pfn; ++ ++ if (!handle_valid(handle)) ++ return INTEL_GVT_INVALID_ADDR; ++ ++ info = (struct kvmgt_guest_info *)handle; ++ ++ pfn = gfn_to_pfn(info->kvm, gfn); ++ if (is_error_noslot_pfn(pfn)) ++ return INTEL_GVT_INVALID_ADDR; ++ ++ return pfn; ++} ++ ++static int kvmgt_dma_map_guest_page(unsigned long handle, unsigned long gfn, ++ unsigned long size, dma_addr_t *dma_addr) ++{ ++ struct kvmgt_guest_info *info; ++ struct intel_vgpu *vgpu; ++ struct gvt_dma *entry; ++ int ret; ++ ++ if (!handle_valid(handle)) ++ return -EINVAL; ++ ++ info = (struct kvmgt_guest_info *)handle; ++ vgpu = info->vgpu; ++ ++ mutex_lock(&info->vgpu->vdev.cache_lock); ++ ++ entry = __gvt_cache_find_gfn(info->vgpu, gfn); ++ if (!entry) { ++ ret = gvt_dma_map_page(vgpu, gfn, dma_addr, size); ++ if (ret) ++ goto err_unlock; ++ ++ ret = __gvt_cache_add(info->vgpu, gfn, *dma_addr, size); ++ if (ret) ++ goto err_unmap; ++ } else if (entry->size != size) { ++ /* the same gfn with different size: unmap and re-map */ ++ gvt_dma_unmap_page(vgpu, gfn, entry->dma_addr, entry->size); ++ __gvt_cache_remove_entry(vgpu, entry); ++ ++ ret = gvt_dma_map_page(vgpu, gfn, dma_addr, size); ++ if (ret) ++ goto err_unlock; ++ ++ ret = __gvt_cache_add(info->vgpu, gfn, *dma_addr, size); ++ if (ret) ++ goto err_unmap; ++ } else { ++ kref_get(&entry->ref); ++ *dma_addr = entry->dma_addr; ++ } ++ ++ mutex_unlock(&info->vgpu->vdev.cache_lock); ++ return 0; ++ ++err_unmap: ++ gvt_dma_unmap_page(vgpu, gfn, *dma_addr, size); ++err_unlock: ++ mutex_unlock(&info->vgpu->vdev.cache_lock); ++ return ret; ++} ++ ++static void __gvt_dma_release(struct kref *ref) ++{ ++ struct gvt_dma *entry = container_of(ref, typeof(*entry), ref); ++ ++ gvt_dma_unmap_page(entry->vgpu, entry->gfn, entry->dma_addr, ++ entry->size); ++ __gvt_cache_remove_entry(entry->vgpu, entry); ++} ++ ++static void kvmgt_dma_unmap_guest_page(unsigned long handle, dma_addr_t dma_addr) ++{ ++ struct kvmgt_guest_info *info; ++ struct gvt_dma *entry; ++ ++ if (!handle_valid(handle)) ++ return; ++ ++ info = (struct kvmgt_guest_info *)handle; ++ ++ mutex_lock(&info->vgpu->vdev.cache_lock); ++ entry = __gvt_cache_find_dma_addr(info->vgpu, dma_addr); ++ if (entry) ++ kref_put(&entry->ref, __gvt_dma_release); ++ mutex_unlock(&info->vgpu->vdev.cache_lock); ++} ++ ++static int kvmgt_rw_gpa(unsigned long handle, unsigned long gpa, ++ void *buf, unsigned long len, bool write) ++{ ++ struct kvmgt_guest_info *info; ++ struct kvm *kvm; ++ int idx, ret; ++ bool kthread = current->mm == NULL; ++ ++ if (!handle_valid(handle)) ++ return -ESRCH; ++ ++ info = (struct kvmgt_guest_info *)handle; ++ kvm = info->kvm; ++ ++ if (kthread) { ++ if (!mmget_not_zero(kvm->mm)) ++ return -EFAULT; ++ use_mm(kvm->mm); ++ } ++ ++ idx = srcu_read_lock(&kvm->srcu); ++ ret = write ? kvm_write_guest(kvm, gpa, buf, len) : ++ kvm_read_guest(kvm, gpa, buf, len); ++ srcu_read_unlock(&kvm->srcu, idx); ++ ++ if (kthread) { ++ unuse_mm(kvm->mm); ++ mmput(kvm->mm); ++ } ++ ++ return ret; ++} ++ ++static int kvmgt_read_gpa(unsigned long handle, unsigned long gpa, ++ void *buf, unsigned long len) ++{ ++ return kvmgt_rw_gpa(handle, gpa, buf, len, false); ++} ++ ++static int kvmgt_write_gpa(unsigned long handle, unsigned long gpa, ++ void *buf, unsigned long len) ++{ ++ return kvmgt_rw_gpa(handle, gpa, buf, len, true); ++} ++ ++static unsigned long kvmgt_virt_to_pfn(void *addr) ++{ ++ return PFN_DOWN(__pa(addr)); ++} ++ ++static bool kvmgt_is_valid_gfn(unsigned long handle, unsigned long gfn) ++{ ++ struct kvmgt_guest_info *info; ++ struct kvm *kvm; ++ int idx; ++ bool ret; ++ ++ if (!handle_valid(handle)) ++ return false; ++ ++ info = (struct kvmgt_guest_info *)handle; ++ kvm = info->kvm; ++ ++ idx = srcu_read_lock(&kvm->srcu); ++ ret = kvm_is_visible_gfn(kvm, gfn); ++ srcu_read_unlock(&kvm->srcu, idx); ++ ++ return ret; ++} ++ ++static struct intel_gvt_mpt kvmgt_mpt = { ++ .type = INTEL_GVT_HYPERVISOR_KVM, ++ .host_init = kvmgt_host_init, ++ .host_exit = kvmgt_host_exit, ++ .attach_vgpu = kvmgt_attach_vgpu, ++ .detach_vgpu = kvmgt_detach_vgpu, ++ .inject_msi = kvmgt_inject_msi, ++ .from_virt_to_mfn = kvmgt_virt_to_pfn, ++ .enable_page_track = kvmgt_page_track_add, ++ .disable_page_track = kvmgt_page_track_remove, ++ .read_gpa = kvmgt_read_gpa, ++ .write_gpa = kvmgt_write_gpa, ++ .gfn_to_mfn = kvmgt_gfn_to_pfn, ++ .dma_map_guest_page = kvmgt_dma_map_guest_page, ++ .dma_unmap_guest_page = kvmgt_dma_unmap_guest_page, ++ .set_opregion = kvmgt_set_opregion, ++ .set_edid = kvmgt_set_edid, ++ .get_vfio_device = kvmgt_get_vfio_device, ++ .put_vfio_device = kvmgt_put_vfio_device, ++ .is_valid_gfn = kvmgt_is_valid_gfn, ++}; ++ ++static int __init kvmgt_init(void) ++{ ++ if (intel_gvt_register_hypervisor(&kvmgt_mpt) < 0) ++ return -ENODEV; ++ return 0; ++} ++ ++static void __exit kvmgt_exit(void) ++{ ++ intel_gvt_unregister_hypervisor(); ++} ++ ++module_init(kvmgt_init); ++module_exit(kvmgt_exit); ++ ++MODULE_LICENSE("GPL and additional rights"); ++MODULE_AUTHOR("Intel Corporation"); +diff --git a/drivers/gpu/drm/i915_legacy/gvt/mmio.c b/drivers/gpu/drm/i915_legacy/gvt/mmio.c +new file mode 100644 +index 000000000000..a55178884d67 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/mmio.c +@@ -0,0 +1,315 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Ke Yu ++ * Kevin Tian ++ * Dexuan Cui ++ * ++ * Contributors: ++ * Tina Zhang ++ * Min He ++ * Niu Bing ++ * Zhi Wang ++ * ++ */ ++ ++#include "i915_drv.h" ++#include "gvt.h" ++ ++/** ++ * intel_vgpu_gpa_to_mmio_offset - translate a GPA to MMIO offset ++ * @vgpu: a vGPU ++ * @gpa: guest physical address ++ * ++ * Returns: ++ * Zero on success, negative error code if failed ++ */ ++int intel_vgpu_gpa_to_mmio_offset(struct intel_vgpu *vgpu, u64 gpa) ++{ ++ u64 gttmmio_gpa = intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_0); ++ return gpa - gttmmio_gpa; ++} ++ ++#define reg_is_mmio(gvt, reg) \ ++ (reg >= 0 && reg < gvt->device_info.mmio_size) ++ ++#define reg_is_gtt(gvt, reg) \ ++ (reg >= gvt->device_info.gtt_start_offset \ ++ && reg < gvt->device_info.gtt_start_offset + gvt_ggtt_sz(gvt)) ++ ++static void failsafe_emulate_mmio_rw(struct intel_vgpu *vgpu, u64 pa, ++ void *p_data, unsigned int bytes, bool read) ++{ ++ struct intel_gvt *gvt = NULL; ++ void *pt = NULL; ++ unsigned int offset = 0; ++ ++ if (!vgpu || !p_data) ++ return; ++ ++ gvt = vgpu->gvt; ++ mutex_lock(&vgpu->vgpu_lock); ++ offset = intel_vgpu_gpa_to_mmio_offset(vgpu, pa); ++ if (reg_is_mmio(gvt, offset)) { ++ if (read) ++ intel_vgpu_default_mmio_read(vgpu, offset, p_data, ++ bytes); ++ else ++ intel_vgpu_default_mmio_write(vgpu, offset, p_data, ++ bytes); ++ } else if (reg_is_gtt(gvt, offset)) { ++ offset -= gvt->device_info.gtt_start_offset; ++ pt = vgpu->gtt.ggtt_mm->ggtt_mm.virtual_ggtt + offset; ++ if (read) ++ memcpy(p_data, pt, bytes); ++ else ++ memcpy(pt, p_data, bytes); ++ ++ } ++ mutex_unlock(&vgpu->vgpu_lock); ++} ++ ++/** ++ * intel_vgpu_emulate_mmio_read - emulate MMIO read ++ * @vgpu: a vGPU ++ * @pa: guest physical address ++ * @p_data: data return buffer ++ * @bytes: access data length ++ * ++ * Returns: ++ * Zero on success, negative error code if failed ++ */ ++int intel_vgpu_emulate_mmio_read(struct intel_vgpu *vgpu, u64 pa, ++ void *p_data, unsigned int bytes) ++{ ++ struct intel_gvt *gvt = vgpu->gvt; ++ unsigned int offset = 0; ++ int ret = -EINVAL; ++ ++ if (vgpu->failsafe) { ++ failsafe_emulate_mmio_rw(vgpu, pa, p_data, bytes, true); ++ return 0; ++ } ++ mutex_lock(&vgpu->vgpu_lock); ++ ++ offset = intel_vgpu_gpa_to_mmio_offset(vgpu, pa); ++ ++ if (WARN_ON(bytes > 8)) ++ goto err; ++ ++ if (reg_is_gtt(gvt, offset)) { ++ if (WARN_ON(!IS_ALIGNED(offset, 4) && !IS_ALIGNED(offset, 8))) ++ goto err; ++ if (WARN_ON(bytes != 4 && bytes != 8)) ++ goto err; ++ if (WARN_ON(!reg_is_gtt(gvt, offset + bytes - 1))) ++ goto err; ++ ++ ret = intel_vgpu_emulate_ggtt_mmio_read(vgpu, offset, ++ p_data, bytes); ++ if (ret) ++ goto err; ++ goto out; ++ } ++ ++ if (WARN_ON_ONCE(!reg_is_mmio(gvt, offset))) { ++ ret = intel_gvt_hypervisor_read_gpa(vgpu, pa, p_data, bytes); ++ goto out; ++ } ++ ++ if (WARN_ON(!reg_is_mmio(gvt, offset + bytes - 1))) ++ goto err; ++ ++ if (!intel_gvt_mmio_is_unalign(gvt, offset)) { ++ if (WARN_ON(!IS_ALIGNED(offset, bytes))) ++ goto err; ++ } ++ ++ ret = intel_vgpu_mmio_reg_rw(vgpu, offset, p_data, bytes, true); ++ if (ret < 0) ++ goto err; ++ ++ intel_gvt_mmio_set_accessed(gvt, offset); ++ ret = 0; ++ goto out; ++ ++err: ++ gvt_vgpu_err("fail to emulate MMIO read %08x len %d\n", ++ offset, bytes); ++out: ++ mutex_unlock(&vgpu->vgpu_lock); ++ return ret; ++} ++ ++/** ++ * intel_vgpu_emulate_mmio_write - emulate MMIO write ++ * @vgpu: a vGPU ++ * @pa: guest physical address ++ * @p_data: write data buffer ++ * @bytes: access data length ++ * ++ * Returns: ++ * Zero on success, negative error code if failed ++ */ ++int intel_vgpu_emulate_mmio_write(struct intel_vgpu *vgpu, u64 pa, ++ void *p_data, unsigned int bytes) ++{ ++ struct intel_gvt *gvt = vgpu->gvt; ++ unsigned int offset = 0; ++ int ret = -EINVAL; ++ ++ if (vgpu->failsafe) { ++ failsafe_emulate_mmio_rw(vgpu, pa, p_data, bytes, false); ++ return 0; ++ } ++ ++ mutex_lock(&vgpu->vgpu_lock); ++ ++ offset = intel_vgpu_gpa_to_mmio_offset(vgpu, pa); ++ ++ if (WARN_ON(bytes > 8)) ++ goto err; ++ ++ if (reg_is_gtt(gvt, offset)) { ++ if (WARN_ON(!IS_ALIGNED(offset, 4) && !IS_ALIGNED(offset, 8))) ++ goto err; ++ if (WARN_ON(bytes != 4 && bytes != 8)) ++ goto err; ++ if (WARN_ON(!reg_is_gtt(gvt, offset + bytes - 1))) ++ goto err; ++ ++ ret = intel_vgpu_emulate_ggtt_mmio_write(vgpu, offset, ++ p_data, bytes); ++ if (ret) ++ goto err; ++ goto out; ++ } ++ ++ if (WARN_ON_ONCE(!reg_is_mmio(gvt, offset))) { ++ ret = intel_gvt_hypervisor_write_gpa(vgpu, pa, p_data, bytes); ++ goto out; ++ } ++ ++ ret = intel_vgpu_mmio_reg_rw(vgpu, offset, p_data, bytes, false); ++ if (ret < 0) ++ goto err; ++ ++ intel_gvt_mmio_set_accessed(gvt, offset); ++ ret = 0; ++ goto out; ++err: ++ gvt_vgpu_err("fail to emulate MMIO write %08x len %d\n", offset, ++ bytes); ++out: ++ mutex_unlock(&vgpu->vgpu_lock); ++ return ret; ++} ++ ++ ++/** ++ * intel_vgpu_reset_mmio - reset virtual MMIO space ++ * @vgpu: a vGPU ++ * @dmlr: whether this is device model level reset ++ */ ++void intel_vgpu_reset_mmio(struct intel_vgpu *vgpu, bool dmlr) ++{ ++ struct intel_gvt *gvt = vgpu->gvt; ++ const struct intel_gvt_device_info *info = &gvt->device_info; ++ void *mmio = gvt->firmware.mmio; ++ ++ if (dmlr) { ++ memcpy(vgpu->mmio.vreg, mmio, info->mmio_size); ++ ++ vgpu_vreg_t(vgpu, GEN6_GT_THREAD_STATUS_REG) = 0; ++ ++ /* set the bit 0:2(Core C-State ) to C0 */ ++ vgpu_vreg_t(vgpu, GEN6_GT_CORE_STATUS) = 0; ++ ++ if (IS_BROXTON(vgpu->gvt->dev_priv)) { ++ vgpu_vreg_t(vgpu, BXT_P_CR_GT_DISP_PWRON) &= ++ ~(BIT(0) | BIT(1)); ++ vgpu_vreg_t(vgpu, BXT_PORT_CL1CM_DW0(DPIO_PHY0)) &= ++ ~PHY_POWER_GOOD; ++ vgpu_vreg_t(vgpu, BXT_PORT_CL1CM_DW0(DPIO_PHY1)) &= ++ ~PHY_POWER_GOOD; ++ vgpu_vreg_t(vgpu, BXT_PHY_CTL_FAMILY(DPIO_PHY0)) &= ++ ~BIT(30); ++ vgpu_vreg_t(vgpu, BXT_PHY_CTL_FAMILY(DPIO_PHY1)) &= ++ ~BIT(30); ++ vgpu_vreg_t(vgpu, BXT_PHY_CTL(PORT_A)) &= ++ ~BXT_PHY_LANE_ENABLED; ++ vgpu_vreg_t(vgpu, BXT_PHY_CTL(PORT_A)) |= ++ BXT_PHY_CMNLANE_POWERDOWN_ACK | ++ BXT_PHY_LANE_POWERDOWN_ACK; ++ vgpu_vreg_t(vgpu, BXT_PHY_CTL(PORT_B)) &= ++ ~BXT_PHY_LANE_ENABLED; ++ vgpu_vreg_t(vgpu, BXT_PHY_CTL(PORT_B)) |= ++ BXT_PHY_CMNLANE_POWERDOWN_ACK | ++ BXT_PHY_LANE_POWERDOWN_ACK; ++ vgpu_vreg_t(vgpu, BXT_PHY_CTL(PORT_C)) &= ++ ~BXT_PHY_LANE_ENABLED; ++ vgpu_vreg_t(vgpu, BXT_PHY_CTL(PORT_C)) |= ++ BXT_PHY_CMNLANE_POWERDOWN_ACK | ++ BXT_PHY_LANE_POWERDOWN_ACK; ++ } ++ } else { ++#define GVT_GEN8_MMIO_RESET_OFFSET (0x44200) ++ /* only reset the engine related, so starting with 0x44200 ++ * interrupt include DE,display mmio related will not be ++ * touched ++ */ ++ memcpy(vgpu->mmio.vreg, mmio, GVT_GEN8_MMIO_RESET_OFFSET); ++ } ++ ++} ++ ++/** ++ * intel_vgpu_init_mmio - init MMIO space ++ * @vgpu: a vGPU ++ * ++ * Returns: ++ * Zero on success, negative error code if failed ++ */ ++int intel_vgpu_init_mmio(struct intel_vgpu *vgpu) ++{ ++ const struct intel_gvt_device_info *info = &vgpu->gvt->device_info; ++ ++ vgpu->mmio.vreg = vzalloc(info->mmio_size); ++ if (!vgpu->mmio.vreg) ++ return -ENOMEM; ++ ++ intel_vgpu_reset_mmio(vgpu, true); ++ ++ return 0; ++} ++ ++/** ++ * intel_vgpu_clean_mmio - clean MMIO space ++ * @vgpu: a vGPU ++ * ++ */ ++void intel_vgpu_clean_mmio(struct intel_vgpu *vgpu) ++{ ++ vfree(vgpu->mmio.vreg); ++ vgpu->mmio.vreg = NULL; ++} +diff --git a/drivers/gpu/drm/i915_legacy/gvt/mmio.h b/drivers/gpu/drm/i915_legacy/gvt/mmio.h +new file mode 100644 +index 000000000000..5874f1cb4306 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/mmio.h +@@ -0,0 +1,105 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Ke Yu ++ * Kevin Tian ++ * Dexuan Cui ++ * ++ * Contributors: ++ * Tina Zhang ++ * Min He ++ * Niu Bing ++ * Zhi Wang ++ * ++ */ ++ ++#ifndef _GVT_MMIO_H_ ++#define _GVT_MMIO_H_ ++ ++struct intel_gvt; ++struct intel_vgpu; ++ ++#define D_BDW (1 << 0) ++#define D_SKL (1 << 1) ++#define D_KBL (1 << 2) ++#define D_BXT (1 << 3) ++#define D_CFL (1 << 4) ++ ++#define D_GEN9PLUS (D_SKL | D_KBL | D_BXT | D_CFL) ++#define D_GEN8PLUS (D_BDW | D_SKL | D_KBL | D_BXT | D_CFL) ++ ++#define D_SKL_PLUS (D_SKL | D_KBL | D_BXT | D_CFL) ++#define D_BDW_PLUS (D_BDW | D_SKL | D_KBL | D_BXT | D_CFL) ++ ++#define D_PRE_SKL (D_BDW) ++#define D_ALL (D_BDW | D_SKL | D_KBL | D_BXT | D_CFL) ++ ++typedef int (*gvt_mmio_func)(struct intel_vgpu *, unsigned int, void *, ++ unsigned int); ++ ++struct intel_gvt_mmio_info { ++ u32 offset; ++ u64 ro_mask; ++ u32 device; ++ gvt_mmio_func read; ++ gvt_mmio_func write; ++ u32 addr_range; ++ struct hlist_node node; ++}; ++ ++int intel_gvt_render_mmio_to_ring_id(struct intel_gvt *gvt, ++ unsigned int reg); ++unsigned long intel_gvt_get_device_type(struct intel_gvt *gvt); ++bool intel_gvt_match_device(struct intel_gvt *gvt, unsigned long device); ++ ++int intel_gvt_setup_mmio_info(struct intel_gvt *gvt); ++void intel_gvt_clean_mmio_info(struct intel_gvt *gvt); ++int intel_gvt_for_each_tracked_mmio(struct intel_gvt *gvt, ++ int (*handler)(struct intel_gvt *gvt, u32 offset, void *data), ++ void *data); ++ ++int intel_vgpu_init_mmio(struct intel_vgpu *vgpu); ++void intel_vgpu_reset_mmio(struct intel_vgpu *vgpu, bool dmlr); ++void intel_vgpu_clean_mmio(struct intel_vgpu *vgpu); ++ ++int intel_vgpu_gpa_to_mmio_offset(struct intel_vgpu *vgpu, u64 gpa); ++ ++int intel_vgpu_emulate_mmio_read(struct intel_vgpu *vgpu, u64 pa, ++ void *p_data, unsigned int bytes); ++int intel_vgpu_emulate_mmio_write(struct intel_vgpu *vgpu, u64 pa, ++ void *p_data, unsigned int bytes); ++ ++int intel_vgpu_default_mmio_read(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes); ++int intel_vgpu_default_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes); ++ ++bool intel_gvt_in_force_nonpriv_whitelist(struct intel_gvt *gvt, ++ unsigned int offset); ++ ++int intel_vgpu_mmio_reg_rw(struct intel_vgpu *vgpu, unsigned int offset, ++ void *pdata, unsigned int bytes, bool is_read); ++ ++int intel_vgpu_mask_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, ++ void *p_data, unsigned int bytes); ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/gvt/mmio_context.c b/drivers/gpu/drm/i915_legacy/gvt/mmio_context.c +new file mode 100644 +index 000000000000..90bb3df0db50 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/mmio_context.c +@@ -0,0 +1,580 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Eddie Dong ++ * Kevin Tian ++ * ++ * Contributors: ++ * Zhi Wang ++ * Changbin Du ++ * Zhenyu Wang ++ * Tina Zhang ++ * Bing Niu ++ * ++ */ ++ ++#include "i915_drv.h" ++#include "gvt.h" ++#include "trace.h" ++ ++#define GEN9_MOCS_SIZE 64 ++ ++/* Raw offset is appened to each line for convenience. */ ++static struct engine_mmio gen8_engine_mmio_list[] __cacheline_aligned = { ++ {RCS0, GFX_MODE_GEN7, 0xffff, false}, /* 0x229c */ ++ {RCS0, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */ ++ {RCS0, HWSTAM, 0x0, false}, /* 0x2098 */ ++ {RCS0, INSTPM, 0xffff, true}, /* 0x20c0 */ ++ {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 0), 0, false}, /* 0x24d0 */ ++ {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 1), 0, false}, /* 0x24d4 */ ++ {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 2), 0, false}, /* 0x24d8 */ ++ {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 3), 0, false}, /* 0x24dc */ ++ {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 4), 0, false}, /* 0x24e0 */ ++ {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 5), 0, false}, /* 0x24e4 */ ++ {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 6), 0, false}, /* 0x24e8 */ ++ {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 7), 0, false}, /* 0x24ec */ ++ {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 8), 0, false}, /* 0x24f0 */ ++ {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 9), 0, false}, /* 0x24f4 */ ++ {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 10), 0, false}, /* 0x24f8 */ ++ {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 11), 0, false}, /* 0x24fc */ ++ {RCS0, CACHE_MODE_1, 0xffff, true}, /* 0x7004 */ ++ {RCS0, GEN7_GT_MODE, 0xffff, true}, /* 0x7008 */ ++ {RCS0, CACHE_MODE_0_GEN7, 0xffff, true}, /* 0x7000 */ ++ {RCS0, GEN7_COMMON_SLICE_CHICKEN1, 0xffff, true}, /* 0x7010 */ ++ {RCS0, HDC_CHICKEN0, 0xffff, true}, /* 0x7300 */ ++ {RCS0, VF_GUARDBAND, 0xffff, true}, /* 0x83a4 */ ++ ++ {BCS0, RING_GFX_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2229c */ ++ {BCS0, RING_MI_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2209c */ ++ {BCS0, RING_INSTPM(BLT_RING_BASE), 0xffff, false}, /* 0x220c0 */ ++ {BCS0, RING_HWSTAM(BLT_RING_BASE), 0x0, false}, /* 0x22098 */ ++ {BCS0, RING_EXCC(BLT_RING_BASE), 0xffff, false}, /* 0x22028 */ ++ {RCS0, INVALID_MMIO_REG, 0, false } /* Terminated */ ++}; ++ ++static struct engine_mmio gen9_engine_mmio_list[] __cacheline_aligned = { ++ {RCS0, GFX_MODE_GEN7, 0xffff, false}, /* 0x229c */ ++ {RCS0, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */ ++ {RCS0, HWSTAM, 0x0, false}, /* 0x2098 */ ++ {RCS0, INSTPM, 0xffff, true}, /* 0x20c0 */ ++ {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 0), 0, false}, /* 0x24d0 */ ++ {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 1), 0, false}, /* 0x24d4 */ ++ {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 2), 0, false}, /* 0x24d8 */ ++ {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 3), 0, false}, /* 0x24dc */ ++ {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 4), 0, false}, /* 0x24e0 */ ++ {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 5), 0, false}, /* 0x24e4 */ ++ {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 6), 0, false}, /* 0x24e8 */ ++ {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 7), 0, false}, /* 0x24ec */ ++ {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 8), 0, false}, /* 0x24f0 */ ++ {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 9), 0, false}, /* 0x24f4 */ ++ {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 10), 0, false}, /* 0x24f8 */ ++ {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 11), 0, false}, /* 0x24fc */ ++ {RCS0, CACHE_MODE_1, 0xffff, true}, /* 0x7004 */ ++ {RCS0, GEN7_GT_MODE, 0xffff, true}, /* 0x7008 */ ++ {RCS0, CACHE_MODE_0_GEN7, 0xffff, true}, /* 0x7000 */ ++ {RCS0, GEN7_COMMON_SLICE_CHICKEN1, 0xffff, true}, /* 0x7010 */ ++ {RCS0, HDC_CHICKEN0, 0xffff, true}, /* 0x7300 */ ++ {RCS0, VF_GUARDBAND, 0xffff, true}, /* 0x83a4 */ ++ ++ {RCS0, GEN8_PRIVATE_PAT_LO, 0, false}, /* 0x40e0 */ ++ {RCS0, GEN8_PRIVATE_PAT_HI, 0, false}, /* 0x40e4 */ ++ {RCS0, GEN8_CS_CHICKEN1, 0xffff, true}, /* 0x2580 */ ++ {RCS0, COMMON_SLICE_CHICKEN2, 0xffff, true}, /* 0x7014 */ ++ {RCS0, GEN9_CS_DEBUG_MODE1, 0xffff, false}, /* 0x20ec */ ++ {RCS0, GEN8_L3SQCREG4, 0, false}, /* 0xb118 */ ++ {RCS0, GEN7_HALF_SLICE_CHICKEN1, 0xffff, true}, /* 0xe100 */ ++ {RCS0, HALF_SLICE_CHICKEN2, 0xffff, true}, /* 0xe180 */ ++ {RCS0, HALF_SLICE_CHICKEN3, 0xffff, true}, /* 0xe184 */ ++ {RCS0, GEN9_HALF_SLICE_CHICKEN5, 0xffff, true}, /* 0xe188 */ ++ {RCS0, GEN9_HALF_SLICE_CHICKEN7, 0xffff, true}, /* 0xe194 */ ++ {RCS0, GEN8_ROW_CHICKEN, 0xffff, true}, /* 0xe4f0 */ ++ {RCS0, TRVATTL3PTRDW(0), 0, true}, /* 0x4de0 */ ++ {RCS0, TRVATTL3PTRDW(1), 0, true}, /* 0x4de4 */ ++ {RCS0, TRNULLDETCT, 0, true}, /* 0x4de8 */ ++ {RCS0, TRINVTILEDETCT, 0, true}, /* 0x4dec */ ++ {RCS0, TRVADR, 0, true}, /* 0x4df0 */ ++ {RCS0, TRTTE, 0, true}, /* 0x4df4 */ ++ {RCS0, _MMIO(0x4dfc), 0, true}, ++ ++ {BCS0, RING_GFX_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2229c */ ++ {BCS0, RING_MI_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2209c */ ++ {BCS0, RING_INSTPM(BLT_RING_BASE), 0xffff, false}, /* 0x220c0 */ ++ {BCS0, RING_HWSTAM(BLT_RING_BASE), 0x0, false}, /* 0x22098 */ ++ {BCS0, RING_EXCC(BLT_RING_BASE), 0xffff, false}, /* 0x22028 */ ++ ++ {VCS1, RING_EXCC(GEN8_BSD2_RING_BASE), 0xffff, false}, /* 0x1c028 */ ++ ++ {VECS0, RING_EXCC(VEBOX_RING_BASE), 0xffff, false}, /* 0x1a028 */ ++ ++ {RCS0, GEN8_HDC_CHICKEN1, 0xffff, true}, /* 0x7304 */ ++ {RCS0, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */ ++ {RCS0, GEN7_UCGCTL4, 0x0, false}, /* 0x940c */ ++ {RCS0, GAMT_CHKN_BIT_REG, 0x0, false}, /* 0x4ab8 */ ++ ++ {RCS0, GEN9_GAMT_ECO_REG_RW_IA, 0x0, false}, /* 0x4ab0 */ ++ {RCS0, GEN9_CSFE_CHICKEN1_RCS, 0xffff, false}, /* 0x20d4 */ ++ {RCS0, _MMIO(0x20D8), 0xffff, true}, /* 0x20d8 */ ++ ++ {RCS0, GEN8_GARBCNTL, 0x0, false}, /* 0xb004 */ ++ {RCS0, GEN7_FF_THREAD_MODE, 0x0, false}, /* 0x20a0 */ ++ {RCS0, FF_SLICE_CS_CHICKEN2, 0xffff, false}, /* 0x20e4 */ ++ {RCS0, INVALID_MMIO_REG, 0, false } /* Terminated */ ++}; ++ ++static struct { ++ bool initialized; ++ u32 control_table[I915_NUM_ENGINES][GEN9_MOCS_SIZE]; ++ u32 l3cc_table[GEN9_MOCS_SIZE / 2]; ++} gen9_render_mocs; ++ ++static void load_render_mocs(struct drm_i915_private *dev_priv) ++{ ++ i915_reg_t offset; ++ u32 regs[] = { ++ [RCS0] = 0xc800, ++ [VCS0] = 0xc900, ++ [VCS1] = 0xca00, ++ [BCS0] = 0xcc00, ++ [VECS0] = 0xcb00, ++ }; ++ int ring_id, i; ++ ++ for (ring_id = 0; ring_id < ARRAY_SIZE(regs); ring_id++) { ++ if (!HAS_ENGINE(dev_priv, ring_id)) ++ continue; ++ offset.reg = regs[ring_id]; ++ for (i = 0; i < GEN9_MOCS_SIZE; i++) { ++ gen9_render_mocs.control_table[ring_id][i] = ++ I915_READ_FW(offset); ++ offset.reg += 4; ++ } ++ } ++ ++ offset.reg = 0xb020; ++ for (i = 0; i < GEN9_MOCS_SIZE / 2; i++) { ++ gen9_render_mocs.l3cc_table[i] = ++ I915_READ_FW(offset); ++ offset.reg += 4; ++ } ++ gen9_render_mocs.initialized = true; ++} ++ ++static int ++restore_context_mmio_for_inhibit(struct intel_vgpu *vgpu, ++ struct i915_request *req) ++{ ++ u32 *cs; ++ int ret; ++ struct engine_mmio *mmio; ++ struct intel_gvt *gvt = vgpu->gvt; ++ int ring_id = req->engine->id; ++ int count = gvt->engine_mmio_list.ctx_mmio_count[ring_id]; ++ ++ if (count == 0) ++ return 0; ++ ++ ret = req->engine->emit_flush(req, EMIT_BARRIER); ++ if (ret) ++ return ret; ++ ++ cs = intel_ring_begin(req, count * 2 + 2); ++ if (IS_ERR(cs)) ++ return PTR_ERR(cs); ++ ++ *cs++ = MI_LOAD_REGISTER_IMM(count); ++ for (mmio = gvt->engine_mmio_list.mmio; ++ i915_mmio_reg_valid(mmio->reg); mmio++) { ++ if (mmio->ring_id != ring_id || ++ !mmio->in_context) ++ continue; ++ ++ *cs++ = i915_mmio_reg_offset(mmio->reg); ++ *cs++ = vgpu_vreg_t(vgpu, mmio->reg) | ++ (mmio->mask << 16); ++ gvt_dbg_core("add lri reg pair 0x%x:0x%x in inhibit ctx, vgpu:%d, rind_id:%d\n", ++ *(cs-2), *(cs-1), vgpu->id, ring_id); ++ } ++ ++ *cs++ = MI_NOOP; ++ intel_ring_advance(req, cs); ++ ++ ret = req->engine->emit_flush(req, EMIT_BARRIER); ++ if (ret) ++ return ret; ++ ++ return 0; ++} ++ ++static int ++restore_render_mocs_control_for_inhibit(struct intel_vgpu *vgpu, ++ struct i915_request *req) ++{ ++ unsigned int index; ++ u32 *cs; ++ ++ cs = intel_ring_begin(req, 2 * GEN9_MOCS_SIZE + 2); ++ if (IS_ERR(cs)) ++ return PTR_ERR(cs); ++ ++ *cs++ = MI_LOAD_REGISTER_IMM(GEN9_MOCS_SIZE); ++ ++ for (index = 0; index < GEN9_MOCS_SIZE; index++) { ++ *cs++ = i915_mmio_reg_offset(GEN9_GFX_MOCS(index)); ++ *cs++ = vgpu_vreg_t(vgpu, GEN9_GFX_MOCS(index)); ++ gvt_dbg_core("add lri reg pair 0x%x:0x%x in inhibit ctx, vgpu:%d, rind_id:%d\n", ++ *(cs-2), *(cs-1), vgpu->id, req->engine->id); ++ ++ } ++ ++ *cs++ = MI_NOOP; ++ intel_ring_advance(req, cs); ++ ++ return 0; ++} ++ ++static int ++restore_render_mocs_l3cc_for_inhibit(struct intel_vgpu *vgpu, ++ struct i915_request *req) ++{ ++ unsigned int index; ++ u32 *cs; ++ ++ cs = intel_ring_begin(req, 2 * GEN9_MOCS_SIZE / 2 + 2); ++ if (IS_ERR(cs)) ++ return PTR_ERR(cs); ++ ++ *cs++ = MI_LOAD_REGISTER_IMM(GEN9_MOCS_SIZE / 2); ++ ++ for (index = 0; index < GEN9_MOCS_SIZE / 2; index++) { ++ *cs++ = i915_mmio_reg_offset(GEN9_LNCFCMOCS(index)); ++ *cs++ = vgpu_vreg_t(vgpu, GEN9_LNCFCMOCS(index)); ++ gvt_dbg_core("add lri reg pair 0x%x:0x%x in inhibit ctx, vgpu:%d, rind_id:%d\n", ++ *(cs-2), *(cs-1), vgpu->id, req->engine->id); ++ ++ } ++ ++ *cs++ = MI_NOOP; ++ intel_ring_advance(req, cs); ++ ++ return 0; ++} ++ ++/* ++ * Use lri command to initialize the mmio which is in context state image for ++ * inhibit context, it contains tracked engine mmio, render_mocs and ++ * render_mocs_l3cc. ++ */ ++int intel_vgpu_restore_inhibit_context(struct intel_vgpu *vgpu, ++ struct i915_request *req) ++{ ++ int ret; ++ u32 *cs; ++ ++ cs = intel_ring_begin(req, 2); ++ if (IS_ERR(cs)) ++ return PTR_ERR(cs); ++ ++ *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; ++ *cs++ = MI_NOOP; ++ intel_ring_advance(req, cs); ++ ++ ret = restore_context_mmio_for_inhibit(vgpu, req); ++ if (ret) ++ goto out; ++ ++ /* no MOCS register in context except render engine */ ++ if (req->engine->id != RCS0) ++ goto out; ++ ++ ret = restore_render_mocs_control_for_inhibit(vgpu, req); ++ if (ret) ++ goto out; ++ ++ ret = restore_render_mocs_l3cc_for_inhibit(vgpu, req); ++ if (ret) ++ goto out; ++ ++out: ++ cs = intel_ring_begin(req, 2); ++ if (IS_ERR(cs)) ++ return PTR_ERR(cs); ++ ++ *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; ++ *cs++ = MI_NOOP; ++ intel_ring_advance(req, cs); ++ ++ return ret; ++} ++ ++static void handle_tlb_pending_event(struct intel_vgpu *vgpu, int ring_id) ++{ ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ struct intel_uncore *uncore = &dev_priv->uncore; ++ struct intel_vgpu_submission *s = &vgpu->submission; ++ enum forcewake_domains fw; ++ i915_reg_t reg; ++ u32 regs[] = { ++ [RCS0] = 0x4260, ++ [VCS0] = 0x4264, ++ [VCS1] = 0x4268, ++ [BCS0] = 0x426c, ++ [VECS0] = 0x4270, ++ }; ++ ++ if (WARN_ON(ring_id >= ARRAY_SIZE(regs))) ++ return; ++ ++ if (!test_and_clear_bit(ring_id, (void *)s->tlb_handle_pending)) ++ return; ++ ++ reg = _MMIO(regs[ring_id]); ++ ++ /* WaForceWakeRenderDuringMmioTLBInvalidate:skl ++ * we need to put a forcewake when invalidating RCS TLB caches, ++ * otherwise device can go to RC6 state and interrupt invalidation ++ * process ++ */ ++ fw = intel_uncore_forcewake_for_reg(uncore, reg, ++ FW_REG_READ | FW_REG_WRITE); ++ if (ring_id == RCS0 && INTEL_GEN(dev_priv) >= 9) ++ fw |= FORCEWAKE_RENDER; ++ ++ intel_uncore_forcewake_get(uncore, fw); ++ ++ intel_uncore_write_fw(uncore, reg, 0x1); ++ ++ if (wait_for_atomic((intel_uncore_read_fw(uncore, reg) == 0), 50)) ++ gvt_vgpu_err("timeout in invalidate ring (%d) tlb\n", ring_id); ++ else ++ vgpu_vreg_t(vgpu, reg) = 0; ++ ++ intel_uncore_forcewake_put(uncore, fw); ++ ++ gvt_dbg_core("invalidate TLB for ring %d\n", ring_id); ++} ++ ++static void switch_mocs(struct intel_vgpu *pre, struct intel_vgpu *next, ++ int ring_id) ++{ ++ struct drm_i915_private *dev_priv; ++ i915_reg_t offset, l3_offset; ++ u32 old_v, new_v; ++ ++ u32 regs[] = { ++ [RCS0] = 0xc800, ++ [VCS0] = 0xc900, ++ [VCS1] = 0xca00, ++ [BCS0] = 0xcc00, ++ [VECS0] = 0xcb00, ++ }; ++ int i; ++ ++ dev_priv = pre ? pre->gvt->dev_priv : next->gvt->dev_priv; ++ if (WARN_ON(ring_id >= ARRAY_SIZE(regs))) ++ return; ++ ++ if (ring_id == RCS0 && IS_GEN(dev_priv, 9)) ++ return; ++ ++ if (!pre && !gen9_render_mocs.initialized) ++ load_render_mocs(dev_priv); ++ ++ offset.reg = regs[ring_id]; ++ for (i = 0; i < GEN9_MOCS_SIZE; i++) { ++ if (pre) ++ old_v = vgpu_vreg_t(pre, offset); ++ else ++ old_v = gen9_render_mocs.control_table[ring_id][i]; ++ if (next) ++ new_v = vgpu_vreg_t(next, offset); ++ else ++ new_v = gen9_render_mocs.control_table[ring_id][i]; ++ ++ if (old_v != new_v) ++ I915_WRITE_FW(offset, new_v); ++ ++ offset.reg += 4; ++ } ++ ++ if (ring_id == RCS0) { ++ l3_offset.reg = 0xb020; ++ for (i = 0; i < GEN9_MOCS_SIZE / 2; i++) { ++ if (pre) ++ old_v = vgpu_vreg_t(pre, l3_offset); ++ else ++ old_v = gen9_render_mocs.l3cc_table[i]; ++ if (next) ++ new_v = vgpu_vreg_t(next, l3_offset); ++ else ++ new_v = gen9_render_mocs.l3cc_table[i]; ++ ++ if (old_v != new_v) ++ I915_WRITE_FW(l3_offset, new_v); ++ ++ l3_offset.reg += 4; ++ } ++ } ++} ++ ++#define CTX_CONTEXT_CONTROL_VAL 0x03 ++ ++bool is_inhibit_context(struct intel_context *ce) ++{ ++ const u32 *reg_state = ce->lrc_reg_state; ++ u32 inhibit_mask = ++ _MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT); ++ ++ return inhibit_mask == ++ (reg_state[CTX_CONTEXT_CONTROL_VAL] & inhibit_mask); ++} ++ ++/* Switch ring mmio values (context). */ ++static void switch_mmio(struct intel_vgpu *pre, ++ struct intel_vgpu *next, ++ int ring_id) ++{ ++ struct drm_i915_private *dev_priv; ++ struct intel_vgpu_submission *s; ++ struct engine_mmio *mmio; ++ u32 old_v, new_v; ++ ++ dev_priv = pre ? pre->gvt->dev_priv : next->gvt->dev_priv; ++ if (INTEL_GEN(dev_priv) >= 9) ++ switch_mocs(pre, next, ring_id); ++ ++ for (mmio = dev_priv->gvt->engine_mmio_list.mmio; ++ i915_mmio_reg_valid(mmio->reg); mmio++) { ++ if (mmio->ring_id != ring_id) ++ continue; ++ /* ++ * No need to do save or restore of the mmio which is in context ++ * state image on gen9, it's initialized by lri command and ++ * save or restore with context together. ++ */ ++ if (IS_GEN(dev_priv, 9) && mmio->in_context) ++ continue; ++ ++ // save ++ if (pre) { ++ vgpu_vreg_t(pre, mmio->reg) = I915_READ_FW(mmio->reg); ++ if (mmio->mask) ++ vgpu_vreg_t(pre, mmio->reg) &= ++ ~(mmio->mask << 16); ++ old_v = vgpu_vreg_t(pre, mmio->reg); ++ } else ++ old_v = mmio->value = I915_READ_FW(mmio->reg); ++ ++ // restore ++ if (next) { ++ s = &next->submission; ++ /* ++ * No need to restore the mmio which is in context state ++ * image if it's not inhibit context, it will restore ++ * itself. ++ */ ++ if (mmio->in_context && ++ !is_inhibit_context(intel_context_lookup(s->shadow_ctx, ++ dev_priv->engine[ring_id]))) ++ continue; ++ ++ if (mmio->mask) ++ new_v = vgpu_vreg_t(next, mmio->reg) | ++ (mmio->mask << 16); ++ else ++ new_v = vgpu_vreg_t(next, mmio->reg); ++ } else { ++ if (mmio->in_context) ++ continue; ++ if (mmio->mask) ++ new_v = mmio->value | (mmio->mask << 16); ++ else ++ new_v = mmio->value; ++ } ++ ++ I915_WRITE_FW(mmio->reg, new_v); ++ ++ trace_render_mmio(pre ? pre->id : 0, ++ next ? next->id : 0, ++ "switch", ++ i915_mmio_reg_offset(mmio->reg), ++ old_v, new_v); ++ } ++ ++ if (next) ++ handle_tlb_pending_event(next, ring_id); ++} ++ ++/** ++ * intel_gvt_switch_render_mmio - switch mmio context of specific engine ++ * @pre: the last vGPU that own the engine ++ * @next: the vGPU to switch to ++ * @ring_id: specify the engine ++ * ++ * If pre is null indicates that host own the engine. If next is null ++ * indicates that we are switching to host workload. ++ */ ++void intel_gvt_switch_mmio(struct intel_vgpu *pre, ++ struct intel_vgpu *next, int ring_id) ++{ ++ struct drm_i915_private *dev_priv; ++ ++ if (WARN_ON(!pre && !next)) ++ return; ++ ++ gvt_dbg_render("switch ring %d from %s to %s\n", ring_id, ++ pre ? "vGPU" : "host", next ? "vGPU" : "HOST"); ++ ++ dev_priv = pre ? pre->gvt->dev_priv : next->gvt->dev_priv; ++ ++ /** ++ * We are using raw mmio access wrapper to improve the ++ * performace for batch mmio read/write, so we need ++ * handle forcewake mannually. ++ */ ++ intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL); ++ switch_mmio(pre, next, ring_id); ++ intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); ++} ++ ++/** ++ * intel_gvt_init_engine_mmio_context - Initiate the engine mmio list ++ * @gvt: GVT device ++ * ++ */ ++void intel_gvt_init_engine_mmio_context(struct intel_gvt *gvt) ++{ ++ struct engine_mmio *mmio; ++ ++ if (INTEL_GEN(gvt->dev_priv) >= 9) ++ gvt->engine_mmio_list.mmio = gen9_engine_mmio_list; ++ else ++ gvt->engine_mmio_list.mmio = gen8_engine_mmio_list; ++ ++ for (mmio = gvt->engine_mmio_list.mmio; ++ i915_mmio_reg_valid(mmio->reg); mmio++) { ++ if (mmio->in_context) { ++ gvt->engine_mmio_list.ctx_mmio_count[mmio->ring_id]++; ++ intel_gvt_mmio_set_in_ctx(gvt, mmio->reg.reg); ++ } ++ } ++} +diff --git a/drivers/gpu/drm/i915_legacy/gvt/mmio_context.h b/drivers/gpu/drm/i915_legacy/gvt/mmio_context.h +new file mode 100644 +index 000000000000..f7eaa442403f +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/mmio_context.h +@@ -0,0 +1,60 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Eddie Dong ++ * Kevin Tian ++ * ++ * Contributors: ++ * Zhi Wang ++ * Changbin Du ++ * Zhenyu Wang ++ * Tina Zhang ++ * Bing Niu ++ * ++ */ ++ ++#ifndef __GVT_RENDER_H__ ++#define __GVT_RENDER_H__ ++ ++struct engine_mmio { ++ int ring_id; ++ i915_reg_t reg; ++ u32 mask; ++ bool in_context; ++ u32 value; ++}; ++ ++void intel_gvt_switch_mmio(struct intel_vgpu *pre, ++ struct intel_vgpu *next, int ring_id); ++ ++void intel_gvt_init_engine_mmio_context(struct intel_gvt *gvt); ++ ++bool is_inhibit_context(struct intel_context *ce); ++ ++int intel_vgpu_restore_inhibit_context(struct intel_vgpu *vgpu, ++ struct i915_request *req); ++#define IS_RESTORE_INHIBIT(a) \ ++ (_MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT) == \ ++ ((a) & _MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT))) ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/gvt/mpt.h b/drivers/gpu/drm/i915_legacy/gvt/mpt.h +new file mode 100644 +index 000000000000..0f9440128123 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/mpt.h +@@ -0,0 +1,383 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Eddie Dong ++ * Dexuan Cui ++ * Jike Song ++ * ++ * Contributors: ++ * Zhi Wang ++ * ++ */ ++ ++#ifndef _GVT_MPT_H_ ++#define _GVT_MPT_H_ ++ ++/** ++ * DOC: Hypervisor Service APIs for GVT-g Core Logic ++ * ++ * This is the glue layer between specific hypervisor MPT modules and GVT-g core ++ * logic. Each kind of hypervisor MPT module provides a collection of function ++ * callbacks and will be attached to GVT host when the driver is loading. ++ * GVT-g core logic will call these APIs to request specific services from ++ * hypervisor. ++ */ ++ ++/** ++ * intel_gvt_hypervisor_host_init - init GVT-g host side ++ * ++ * Returns: ++ * Zero on success, negative error code if failed ++ */ ++static inline int intel_gvt_hypervisor_host_init(struct device *dev, ++ void *gvt, const void *ops) ++{ ++ if (!intel_gvt_host.mpt->host_init) ++ return -ENODEV; ++ ++ return intel_gvt_host.mpt->host_init(dev, gvt, ops); ++} ++ ++/** ++ * intel_gvt_hypervisor_host_exit - exit GVT-g host side ++ */ ++static inline void intel_gvt_hypervisor_host_exit(struct device *dev) ++{ ++ /* optional to provide */ ++ if (!intel_gvt_host.mpt->host_exit) ++ return; ++ ++ intel_gvt_host.mpt->host_exit(dev); ++} ++ ++/** ++ * intel_gvt_hypervisor_attach_vgpu - call hypervisor to initialize vGPU ++ * related stuffs inside hypervisor. ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++static inline int intel_gvt_hypervisor_attach_vgpu(struct intel_vgpu *vgpu) ++{ ++ /* optional to provide */ ++ if (!intel_gvt_host.mpt->attach_vgpu) ++ return 0; ++ ++ return intel_gvt_host.mpt->attach_vgpu(vgpu, &vgpu->handle); ++} ++ ++/** ++ * intel_gvt_hypervisor_detach_vgpu - call hypervisor to release vGPU ++ * related stuffs inside hypervisor. ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++static inline void intel_gvt_hypervisor_detach_vgpu(struct intel_vgpu *vgpu) ++{ ++ /* optional to provide */ ++ if (!intel_gvt_host.mpt->detach_vgpu) ++ return; ++ ++ intel_gvt_host.mpt->detach_vgpu(vgpu); ++} ++ ++#define MSI_CAP_CONTROL(offset) (offset + 2) ++#define MSI_CAP_ADDRESS(offset) (offset + 4) ++#define MSI_CAP_DATA(offset) (offset + 8) ++#define MSI_CAP_EN 0x1 ++ ++/** ++ * intel_gvt_hypervisor_inject_msi - inject a MSI interrupt into vGPU ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++static inline int intel_gvt_hypervisor_inject_msi(struct intel_vgpu *vgpu) ++{ ++ unsigned long offset = vgpu->gvt->device_info.msi_cap_offset; ++ u16 control, data; ++ u32 addr; ++ int ret; ++ ++ control = *(u16 *)(vgpu_cfg_space(vgpu) + MSI_CAP_CONTROL(offset)); ++ addr = *(u32 *)(vgpu_cfg_space(vgpu) + MSI_CAP_ADDRESS(offset)); ++ data = *(u16 *)(vgpu_cfg_space(vgpu) + MSI_CAP_DATA(offset)); ++ ++ /* Do not generate MSI if MSIEN is disable */ ++ if (!(control & MSI_CAP_EN)) ++ return 0; ++ ++ if (WARN(control & GENMASK(15, 1), "only support one MSI format\n")) ++ return -EINVAL; ++ ++ trace_inject_msi(vgpu->id, addr, data); ++ ++ ret = intel_gvt_host.mpt->inject_msi(vgpu->handle, addr, data); ++ if (ret) ++ return ret; ++ return 0; ++} ++ ++/** ++ * intel_gvt_hypervisor_set_wp_page - translate a host VA into MFN ++ * @p: host kernel virtual address ++ * ++ * Returns: ++ * MFN on success, INTEL_GVT_INVALID_ADDR if failed. ++ */ ++static inline unsigned long intel_gvt_hypervisor_virt_to_mfn(void *p) ++{ ++ return intel_gvt_host.mpt->from_virt_to_mfn(p); ++} ++ ++/** ++ * intel_gvt_hypervisor_enable_page_track - track a guest page ++ * @vgpu: a vGPU ++ * @gfn: the gfn of guest ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++static inline int intel_gvt_hypervisor_enable_page_track( ++ struct intel_vgpu *vgpu, unsigned long gfn) ++{ ++ return intel_gvt_host.mpt->enable_page_track(vgpu->handle, gfn); ++} ++ ++/** ++ * intel_gvt_hypervisor_disable_page_track - untrack a guest page ++ * @vgpu: a vGPU ++ * @gfn: the gfn of guest ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++static inline int intel_gvt_hypervisor_disable_page_track( ++ struct intel_vgpu *vgpu, unsigned long gfn) ++{ ++ return intel_gvt_host.mpt->disable_page_track(vgpu->handle, gfn); ++} ++ ++/** ++ * intel_gvt_hypervisor_read_gpa - copy data from GPA to host data buffer ++ * @vgpu: a vGPU ++ * @gpa: guest physical address ++ * @buf: host data buffer ++ * @len: data length ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++static inline int intel_gvt_hypervisor_read_gpa(struct intel_vgpu *vgpu, ++ unsigned long gpa, void *buf, unsigned long len) ++{ ++ return intel_gvt_host.mpt->read_gpa(vgpu->handle, gpa, buf, len); ++} ++ ++/** ++ * intel_gvt_hypervisor_write_gpa - copy data from host data buffer to GPA ++ * @vgpu: a vGPU ++ * @gpa: guest physical address ++ * @buf: host data buffer ++ * @len: data length ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++static inline int intel_gvt_hypervisor_write_gpa(struct intel_vgpu *vgpu, ++ unsigned long gpa, void *buf, unsigned long len) ++{ ++ return intel_gvt_host.mpt->write_gpa(vgpu->handle, gpa, buf, len); ++} ++ ++/** ++ * intel_gvt_hypervisor_gfn_to_mfn - translate a GFN to MFN ++ * @vgpu: a vGPU ++ * @gpfn: guest pfn ++ * ++ * Returns: ++ * MFN on success, INTEL_GVT_INVALID_ADDR if failed. ++ */ ++static inline unsigned long intel_gvt_hypervisor_gfn_to_mfn( ++ struct intel_vgpu *vgpu, unsigned long gfn) ++{ ++ return intel_gvt_host.mpt->gfn_to_mfn(vgpu->handle, gfn); ++} ++ ++/** ++ * intel_gvt_hypervisor_dma_map_guest_page - setup dma map for guest page ++ * @vgpu: a vGPU ++ * @gfn: guest pfn ++ * @size: page size ++ * @dma_addr: retrieve allocated dma addr ++ * ++ * Returns: ++ * 0 on success, negative error code if failed. ++ */ ++static inline int intel_gvt_hypervisor_dma_map_guest_page( ++ struct intel_vgpu *vgpu, unsigned long gfn, unsigned long size, ++ dma_addr_t *dma_addr) ++{ ++ return intel_gvt_host.mpt->dma_map_guest_page(vgpu->handle, gfn, size, ++ dma_addr); ++} ++ ++/** ++ * intel_gvt_hypervisor_dma_unmap_guest_page - cancel dma map for guest page ++ * @vgpu: a vGPU ++ * @dma_addr: the mapped dma addr ++ */ ++static inline void intel_gvt_hypervisor_dma_unmap_guest_page( ++ struct intel_vgpu *vgpu, dma_addr_t dma_addr) ++{ ++ intel_gvt_host.mpt->dma_unmap_guest_page(vgpu->handle, dma_addr); ++} ++ ++/** ++ * intel_gvt_hypervisor_map_gfn_to_mfn - map a GFN region to MFN ++ * @vgpu: a vGPU ++ * @gfn: guest PFN ++ * @mfn: host PFN ++ * @nr: amount of PFNs ++ * @map: map or unmap ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++static inline int intel_gvt_hypervisor_map_gfn_to_mfn( ++ struct intel_vgpu *vgpu, unsigned long gfn, ++ unsigned long mfn, unsigned int nr, ++ bool map) ++{ ++ /* a MPT implementation could have MMIO mapped elsewhere */ ++ if (!intel_gvt_host.mpt->map_gfn_to_mfn) ++ return 0; ++ ++ return intel_gvt_host.mpt->map_gfn_to_mfn(vgpu->handle, gfn, mfn, nr, ++ map); ++} ++ ++/** ++ * intel_gvt_hypervisor_set_trap_area - Trap a guest PA region ++ * @vgpu: a vGPU ++ * @start: the beginning of the guest physical address region ++ * @end: the end of the guest physical address region ++ * @map: map or unmap ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++static inline int intel_gvt_hypervisor_set_trap_area( ++ struct intel_vgpu *vgpu, u64 start, u64 end, bool map) ++{ ++ /* a MPT implementation could have MMIO trapped elsewhere */ ++ if (!intel_gvt_host.mpt->set_trap_area) ++ return 0; ++ ++ return intel_gvt_host.mpt->set_trap_area(vgpu->handle, start, end, map); ++} ++ ++/** ++ * intel_gvt_hypervisor_set_opregion - Set opregion for guest ++ * @vgpu: a vGPU ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++static inline int intel_gvt_hypervisor_set_opregion(struct intel_vgpu *vgpu) ++{ ++ if (!intel_gvt_host.mpt->set_opregion) ++ return 0; ++ ++ return intel_gvt_host.mpt->set_opregion(vgpu); ++} ++ ++/** ++ * intel_gvt_hypervisor_set_edid - Set EDID region for guest ++ * @vgpu: a vGPU ++ * @port_num: display port number ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++static inline int intel_gvt_hypervisor_set_edid(struct intel_vgpu *vgpu, ++ int port_num) ++{ ++ if (!intel_gvt_host.mpt->set_edid) ++ return 0; ++ ++ return intel_gvt_host.mpt->set_edid(vgpu, port_num); ++} ++ ++/** ++ * intel_gvt_hypervisor_get_vfio_device - increase vfio device ref count ++ * @vgpu: a vGPU ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++static inline int intel_gvt_hypervisor_get_vfio_device(struct intel_vgpu *vgpu) ++{ ++ if (!intel_gvt_host.mpt->get_vfio_device) ++ return 0; ++ ++ return intel_gvt_host.mpt->get_vfio_device(vgpu); ++} ++ ++/** ++ * intel_gvt_hypervisor_put_vfio_device - decrease vfio device ref count ++ * @vgpu: a vGPU ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++static inline void intel_gvt_hypervisor_put_vfio_device(struct intel_vgpu *vgpu) ++{ ++ if (!intel_gvt_host.mpt->put_vfio_device) ++ return; ++ ++ intel_gvt_host.mpt->put_vfio_device(vgpu); ++} ++ ++/** ++ * intel_gvt_hypervisor_is_valid_gfn - check if a visible gfn ++ * @vgpu: a vGPU ++ * @gfn: guest PFN ++ * ++ * Returns: ++ * true on valid gfn, false on not. ++ */ ++static inline bool intel_gvt_hypervisor_is_valid_gfn( ++ struct intel_vgpu *vgpu, unsigned long gfn) ++{ ++ if (!intel_gvt_host.mpt->is_valid_gfn) ++ return true; ++ ++ return intel_gvt_host.mpt->is_valid_gfn(vgpu->handle, gfn); ++} ++ ++int intel_gvt_register_hypervisor(struct intel_gvt_mpt *); ++void intel_gvt_unregister_hypervisor(void); ++ ++#endif /* _GVT_MPT_H_ */ +diff --git a/drivers/gpu/drm/i915_legacy/gvt/opregion.c b/drivers/gpu/drm/i915_legacy/gvt/opregion.c +new file mode 100644 +index 000000000000..276db53f1bf1 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/opregion.c +@@ -0,0 +1,570 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ */ ++ ++#include ++#include "i915_drv.h" ++#include "gvt.h" ++ ++/* ++ * Note: Only for GVT-g virtual VBT generation, other usage must ++ * not do like this. ++ */ ++#define _INTEL_BIOS_PRIVATE ++#include "intel_vbt_defs.h" ++ ++#define OPREGION_SIGNATURE "IntelGraphicsMem" ++#define MBOX_VBT (1<<3) ++ ++/* device handle */ ++#define DEVICE_TYPE_CRT 0x01 ++#define DEVICE_TYPE_EFP1 0x04 ++#define DEVICE_TYPE_EFP2 0x40 ++#define DEVICE_TYPE_EFP3 0x20 ++#define DEVICE_TYPE_EFP4 0x10 ++ ++struct opregion_header { ++ u8 signature[16]; ++ u32 size; ++ u32 opregion_ver; ++ u8 bios_ver[32]; ++ u8 vbios_ver[16]; ++ u8 driver_ver[16]; ++ u32 mboxes; ++ u32 driver_model; ++ u32 pcon; ++ u8 dver[32]; ++ u8 rsvd[124]; ++} __packed; ++ ++struct bdb_data_header { ++ u8 id; ++ u16 size; /* data size */ ++} __packed; ++ ++/* For supporting windows guest with opregion, here hardcode the emulated ++ * bdb header version as '186', and the corresponding child_device_config ++ * length should be '33' but not '38'. ++ */ ++struct efp_child_device_config { ++ u16 handle; ++ u16 device_type; ++ u16 device_class; ++ u8 i2c_speed; ++ u8 dp_onboard_redriver; /* 158 */ ++ u8 dp_ondock_redriver; /* 158 */ ++ u8 hdmi_level_shifter_value:4; /* 169 */ ++ u8 hdmi_max_data_rate:4; /* 204 */ ++ u16 dtd_buf_ptr; /* 161 */ ++ u8 edidless_efp:1; /* 161 */ ++ u8 compression_enable:1; /* 198 */ ++ u8 compression_method:1; /* 198 */ ++ u8 ganged_edp:1; /* 202 */ ++ u8 skip0:4; ++ u8 compression_structure_index:4; /* 198 */ ++ u8 skip1:4; ++ u8 slave_port; /* 202 */ ++ u8 skip2; ++ u8 dvo_port; ++ u8 i2c_pin; /* for add-in card */ ++ u8 slave_addr; /* for add-in card */ ++ u8 ddc_pin; ++ u16 edid_ptr; ++ u8 dvo_config; ++ u8 efp_docked_port:1; /* 158 */ ++ u8 lane_reversal:1; /* 184 */ ++ u8 onboard_lspcon:1; /* 192 */ ++ u8 iboost_enable:1; /* 196 */ ++ u8 hpd_invert:1; /* BXT 196 */ ++ u8 slip3:3; ++ u8 hdmi_compat:1; ++ u8 dp_compat:1; ++ u8 tmds_compat:1; ++ u8 skip4:5; ++ u8 aux_channel; ++ u8 dongle_detect; ++ u8 pipe_cap:2; ++ u8 sdvo_stall:1; /* 158 */ ++ u8 hpd_status:2; ++ u8 integrated_encoder:1; ++ u8 skip5:2; ++ u8 dvo_wiring; ++ u8 mipi_bridge_type; /* 171 */ ++ u16 device_class_ext; ++ u8 dvo_function; ++} __packed; ++ ++struct vbt { ++ /* header->bdb_offset point to bdb_header offset */ ++ struct vbt_header header; ++ struct bdb_header bdb_header; ++ ++ struct bdb_data_header general_features_header; ++ struct bdb_general_features general_features; ++ ++ struct bdb_data_header general_definitions_header; ++ struct bdb_general_definitions general_definitions; ++ ++ struct efp_child_device_config child0; ++ struct efp_child_device_config child1; ++ struct efp_child_device_config child2; ++ struct efp_child_device_config child3; ++ ++ struct bdb_data_header driver_features_header; ++ struct bdb_driver_features driver_features; ++}; ++ ++static void virt_vbt_generation(struct vbt *v) ++{ ++ int num_child; ++ ++ memset(v, 0, sizeof(struct vbt)); ++ ++ v->header.signature[0] = '$'; ++ v->header.signature[1] = 'V'; ++ v->header.signature[2] = 'B'; ++ v->header.signature[3] = 'T'; ++ ++ /* there's features depending on version! */ ++ v->header.version = 155; ++ v->header.header_size = sizeof(v->header); ++ v->header.vbt_size = sizeof(struct vbt) - sizeof(v->header); ++ v->header.bdb_offset = offsetof(struct vbt, bdb_header); ++ ++ strcpy(&v->bdb_header.signature[0], "BIOS_DATA_BLOCK"); ++ v->bdb_header.version = 186; /* child_dev_size = 33 */ ++ v->bdb_header.header_size = sizeof(v->bdb_header); ++ ++ v->bdb_header.bdb_size = sizeof(struct vbt) - sizeof(struct vbt_header) ++ - sizeof(struct bdb_header); ++ ++ /* general features */ ++ v->general_features_header.id = BDB_GENERAL_FEATURES; ++ v->general_features_header.size = sizeof(struct bdb_general_features); ++ v->general_features.int_crt_support = 0; ++ v->general_features.int_tv_support = 0; ++ ++ /* child device */ ++ num_child = 4; /* each port has one child */ ++ v->general_definitions.child_dev_size = ++ sizeof(struct efp_child_device_config); ++ v->general_definitions_header.id = BDB_GENERAL_DEFINITIONS; ++ /* size will include child devices */ ++ v->general_definitions_header.size = ++ sizeof(struct bdb_general_definitions) + ++ num_child * v->general_definitions.child_dev_size; ++ ++ /* portA */ ++ v->child0.handle = DEVICE_TYPE_EFP1; ++ v->child0.device_type = DEVICE_TYPE_DP; ++ v->child0.dvo_port = DVO_PORT_DPA; ++ v->child0.aux_channel = DP_AUX_A; ++ v->child0.dp_compat = true; ++ v->child0.integrated_encoder = true; ++ ++ /* portB */ ++ v->child1.handle = DEVICE_TYPE_EFP2; ++ v->child1.device_type = DEVICE_TYPE_DP; ++ v->child1.dvo_port = DVO_PORT_DPB; ++ v->child1.aux_channel = DP_AUX_B; ++ v->child1.dp_compat = true; ++ v->child1.integrated_encoder = true; ++ ++ /* portC */ ++ v->child2.handle = DEVICE_TYPE_EFP3; ++ v->child2.device_type = DEVICE_TYPE_DP; ++ v->child2.dvo_port = DVO_PORT_DPC; ++ v->child2.aux_channel = DP_AUX_C; ++ v->child2.dp_compat = true; ++ v->child2.integrated_encoder = true; ++ ++ /* portD */ ++ v->child3.handle = DEVICE_TYPE_EFP4; ++ v->child3.device_type = DEVICE_TYPE_DP; ++ v->child3.dvo_port = DVO_PORT_DPD; ++ v->child3.aux_channel = DP_AUX_D; ++ v->child3.dp_compat = true; ++ v->child3.integrated_encoder = true; ++ ++ /* driver features */ ++ v->driver_features_header.id = BDB_DRIVER_FEATURES; ++ v->driver_features_header.size = sizeof(struct bdb_driver_features); ++ v->driver_features.lvds_config = BDB_DRIVER_FEATURE_NO_LVDS; ++} ++ ++/** ++ * intel_vgpu_init_opregion - initialize the stuff used to emulate opregion ++ * @vgpu: a vGPU ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++int intel_vgpu_init_opregion(struct intel_vgpu *vgpu) ++{ ++ u8 *buf; ++ struct opregion_header *header; ++ struct vbt v; ++ const char opregion_signature[16] = OPREGION_SIGNATURE; ++ ++ gvt_dbg_core("init vgpu%d opregion\n", vgpu->id); ++ vgpu_opregion(vgpu)->va = (void *)__get_free_pages(GFP_KERNEL | ++ __GFP_ZERO, ++ get_order(INTEL_GVT_OPREGION_SIZE)); ++ if (!vgpu_opregion(vgpu)->va) { ++ gvt_err("fail to get memory for vgpu virt opregion\n"); ++ return -ENOMEM; ++ } ++ ++ /* emulated opregion with VBT mailbox only */ ++ buf = (u8 *)vgpu_opregion(vgpu)->va; ++ header = (struct opregion_header *)buf; ++ memcpy(header->signature, opregion_signature, ++ sizeof(opregion_signature)); ++ header->size = 0x8; ++ header->opregion_ver = 0x02000000; ++ header->mboxes = MBOX_VBT; ++ ++ /* for unknown reason, the value in LID field is incorrect ++ * which block the windows guest, so workaround it by force ++ * setting it to "OPEN" ++ */ ++ buf[INTEL_GVT_OPREGION_CLID] = 0x3; ++ ++ /* emulated vbt from virt vbt generation */ ++ virt_vbt_generation(&v); ++ memcpy(buf + INTEL_GVT_OPREGION_VBT_OFFSET, &v, sizeof(struct vbt)); ++ ++ return 0; ++} ++ ++static int map_vgpu_opregion(struct intel_vgpu *vgpu, bool map) ++{ ++ u64 mfn; ++ int i, ret; ++ ++ for (i = 0; i < INTEL_GVT_OPREGION_PAGES; i++) { ++ mfn = intel_gvt_hypervisor_virt_to_mfn(vgpu_opregion(vgpu)->va ++ + i * PAGE_SIZE); ++ if (mfn == INTEL_GVT_INVALID_ADDR) { ++ gvt_vgpu_err("fail to get MFN from VA\n"); ++ return -EINVAL; ++ } ++ ret = intel_gvt_hypervisor_map_gfn_to_mfn(vgpu, ++ vgpu_opregion(vgpu)->gfn[i], ++ mfn, 1, map); ++ if (ret) { ++ gvt_vgpu_err("fail to map GFN to MFN, errno: %d\n", ++ ret); ++ return ret; ++ } ++ } ++ ++ vgpu_opregion(vgpu)->mapped = map; ++ ++ return 0; ++} ++ ++/** ++ * intel_vgpu_opregion_base_write_handler - Opregion base register write handler ++ * ++ * @vgpu: a vGPU ++ * @gpa: guest physical address of opregion ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ */ ++int intel_vgpu_opregion_base_write_handler(struct intel_vgpu *vgpu, u32 gpa) ++{ ++ ++ int i, ret = 0; ++ ++ gvt_dbg_core("emulate opregion from kernel\n"); ++ ++ switch (intel_gvt_host.hypervisor_type) { ++ case INTEL_GVT_HYPERVISOR_KVM: ++ for (i = 0; i < INTEL_GVT_OPREGION_PAGES; i++) ++ vgpu_opregion(vgpu)->gfn[i] = (gpa >> PAGE_SHIFT) + i; ++ break; ++ case INTEL_GVT_HYPERVISOR_XEN: ++ /** ++ * Wins guest on Xengt will write this register twice: xen ++ * hvmloader and windows graphic driver. ++ */ ++ if (vgpu_opregion(vgpu)->mapped) ++ map_vgpu_opregion(vgpu, false); ++ ++ for (i = 0; i < INTEL_GVT_OPREGION_PAGES; i++) ++ vgpu_opregion(vgpu)->gfn[i] = (gpa >> PAGE_SHIFT) + i; ++ ++ ret = map_vgpu_opregion(vgpu, true); ++ break; ++ default: ++ ret = -EINVAL; ++ gvt_vgpu_err("not supported hypervisor\n"); ++ } ++ ++ return ret; ++} ++ ++/** ++ * intel_vgpu_clean_opregion - clean the stuff used to emulate opregion ++ * @vgpu: a vGPU ++ * ++ */ ++void intel_vgpu_clean_opregion(struct intel_vgpu *vgpu) ++{ ++ gvt_dbg_core("vgpu%d: clean vgpu opregion\n", vgpu->id); ++ ++ if (!vgpu_opregion(vgpu)->va) ++ return; ++ ++ if (intel_gvt_host.hypervisor_type == INTEL_GVT_HYPERVISOR_XEN) { ++ if (vgpu_opregion(vgpu)->mapped) ++ map_vgpu_opregion(vgpu, false); ++ } else if (intel_gvt_host.hypervisor_type == INTEL_GVT_HYPERVISOR_KVM) { ++ /* Guest opregion is released by VFIO */ ++ } ++ free_pages((unsigned long)vgpu_opregion(vgpu)->va, ++ get_order(INTEL_GVT_OPREGION_SIZE)); ++ ++ vgpu_opregion(vgpu)->va = NULL; ++ ++} ++ ++ ++#define GVT_OPREGION_FUNC(scic) \ ++ ({ \ ++ u32 __ret; \ ++ __ret = (scic & OPREGION_SCIC_FUNC_MASK) >> \ ++ OPREGION_SCIC_FUNC_SHIFT; \ ++ __ret; \ ++ }) ++ ++#define GVT_OPREGION_SUBFUNC(scic) \ ++ ({ \ ++ u32 __ret; \ ++ __ret = (scic & OPREGION_SCIC_SUBFUNC_MASK) >> \ ++ OPREGION_SCIC_SUBFUNC_SHIFT; \ ++ __ret; \ ++ }) ++ ++static const char *opregion_func_name(u32 func) ++{ ++ const char *name = NULL; ++ ++ switch (func) { ++ case 0 ... 3: ++ case 5: ++ case 7 ... 15: ++ name = "Reserved"; ++ break; ++ ++ case 4: ++ name = "Get BIOS Data"; ++ break; ++ ++ case 6: ++ name = "System BIOS Callbacks"; ++ break; ++ ++ default: ++ name = "Unknown"; ++ break; ++ } ++ return name; ++} ++ ++static const char *opregion_subfunc_name(u32 subfunc) ++{ ++ const char *name = NULL; ++ ++ switch (subfunc) { ++ case 0: ++ name = "Supported Calls"; ++ break; ++ ++ case 1: ++ name = "Requested Callbacks"; ++ break; ++ ++ case 2 ... 3: ++ case 8 ... 9: ++ name = "Reserved"; ++ break; ++ ++ case 5: ++ name = "Boot Display"; ++ break; ++ ++ case 6: ++ name = "TV-Standard/Video-Connector"; ++ break; ++ ++ case 7: ++ name = "Internal Graphics"; ++ break; ++ ++ case 10: ++ name = "Spread Spectrum Clocks"; ++ break; ++ ++ case 11: ++ name = "Get AKSV"; ++ break; ++ ++ default: ++ name = "Unknown"; ++ break; ++ } ++ return name; ++}; ++ ++static bool querying_capabilities(u32 scic) ++{ ++ u32 func, subfunc; ++ ++ func = GVT_OPREGION_FUNC(scic); ++ subfunc = GVT_OPREGION_SUBFUNC(scic); ++ ++ if ((func == INTEL_GVT_OPREGION_SCIC_F_GETBIOSDATA && ++ subfunc == INTEL_GVT_OPREGION_SCIC_SF_SUPPRTEDCALLS) ++ || (func == INTEL_GVT_OPREGION_SCIC_F_GETBIOSDATA && ++ subfunc == INTEL_GVT_OPREGION_SCIC_SF_REQEUSTEDCALLBACKS) ++ || (func == INTEL_GVT_OPREGION_SCIC_F_GETBIOSCALLBACKS && ++ subfunc == INTEL_GVT_OPREGION_SCIC_SF_SUPPRTEDCALLS)) { ++ return true; ++ } ++ return false; ++} ++ ++/** ++ * intel_vgpu_emulate_opregion_request - emulating OpRegion request ++ * @vgpu: a vGPU ++ * @swsci: SWSCI request ++ * ++ * Returns: ++ * Zero on success, negative error code if failed ++ */ ++int intel_vgpu_emulate_opregion_request(struct intel_vgpu *vgpu, u32 swsci) ++{ ++ u32 scic, parm; ++ u32 func, subfunc; ++ u64 scic_pa = 0, parm_pa = 0; ++ int ret; ++ ++ switch (intel_gvt_host.hypervisor_type) { ++ case INTEL_GVT_HYPERVISOR_XEN: ++ scic = *((u32 *)vgpu_opregion(vgpu)->va + ++ INTEL_GVT_OPREGION_SCIC); ++ parm = *((u32 *)vgpu_opregion(vgpu)->va + ++ INTEL_GVT_OPREGION_PARM); ++ break; ++ case INTEL_GVT_HYPERVISOR_KVM: ++ scic_pa = (vgpu_opregion(vgpu)->gfn[0] << PAGE_SHIFT) + ++ INTEL_GVT_OPREGION_SCIC; ++ parm_pa = (vgpu_opregion(vgpu)->gfn[0] << PAGE_SHIFT) + ++ INTEL_GVT_OPREGION_PARM; ++ ++ ret = intel_gvt_hypervisor_read_gpa(vgpu, scic_pa, ++ &scic, sizeof(scic)); ++ if (ret) { ++ gvt_vgpu_err("guest opregion read error %d, gpa 0x%llx, len %lu\n", ++ ret, scic_pa, sizeof(scic)); ++ return ret; ++ } ++ ++ ret = intel_gvt_hypervisor_read_gpa(vgpu, parm_pa, ++ &parm, sizeof(parm)); ++ if (ret) { ++ gvt_vgpu_err("guest opregion read error %d, gpa 0x%llx, len %lu\n", ++ ret, scic_pa, sizeof(scic)); ++ return ret; ++ } ++ ++ break; ++ default: ++ gvt_vgpu_err("not supported hypervisor\n"); ++ return -EINVAL; ++ } ++ ++ if (!(swsci & SWSCI_SCI_SELECT)) { ++ gvt_vgpu_err("requesting SMI service\n"); ++ return 0; ++ } ++ /* ignore non 0->1 trasitions */ ++ if ((vgpu_cfg_space(vgpu)[INTEL_GVT_PCI_SWSCI] ++ & SWSCI_SCI_TRIGGER) || ++ !(swsci & SWSCI_SCI_TRIGGER)) { ++ return 0; ++ } ++ ++ func = GVT_OPREGION_FUNC(scic); ++ subfunc = GVT_OPREGION_SUBFUNC(scic); ++ if (!querying_capabilities(scic)) { ++ gvt_vgpu_err("requesting runtime service: func \"%s\"," ++ " subfunc \"%s\"\n", ++ opregion_func_name(func), ++ opregion_subfunc_name(subfunc)); ++ /* ++ * emulate exit status of function call, '0' means ++ * "failure, generic, unsupported or unknown cause" ++ */ ++ scic &= ~OPREGION_SCIC_EXIT_MASK; ++ goto out; ++ } ++ ++ scic = 0; ++ parm = 0; ++ ++out: ++ switch (intel_gvt_host.hypervisor_type) { ++ case INTEL_GVT_HYPERVISOR_XEN: ++ *((u32 *)vgpu_opregion(vgpu)->va + ++ INTEL_GVT_OPREGION_SCIC) = scic; ++ *((u32 *)vgpu_opregion(vgpu)->va + ++ INTEL_GVT_OPREGION_PARM) = parm; ++ break; ++ case INTEL_GVT_HYPERVISOR_KVM: ++ ret = intel_gvt_hypervisor_write_gpa(vgpu, scic_pa, ++ &scic, sizeof(scic)); ++ if (ret) { ++ gvt_vgpu_err("guest opregion write error %d, gpa 0x%llx, len %lu\n", ++ ret, scic_pa, sizeof(scic)); ++ return ret; ++ } ++ ++ ret = intel_gvt_hypervisor_write_gpa(vgpu, parm_pa, ++ &parm, sizeof(parm)); ++ if (ret) { ++ gvt_vgpu_err("guest opregion write error %d, gpa 0x%llx, len %lu\n", ++ ret, scic_pa, sizeof(scic)); ++ return ret; ++ } ++ ++ break; ++ default: ++ gvt_vgpu_err("not supported hypervisor\n"); ++ return -EINVAL; ++ } ++ ++ return 0; ++} +diff --git a/drivers/gpu/drm/i915_legacy/gvt/page_track.c b/drivers/gpu/drm/i915_legacy/gvt/page_track.c +new file mode 100644 +index 000000000000..84856022528e +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/page_track.c +@@ -0,0 +1,185 @@ ++/* ++ * Copyright(c) 2011-2017 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ */ ++#include "i915_drv.h" ++#include "gvt.h" ++ ++/** ++ * intel_vgpu_find_page_track - find page track rcord of guest page ++ * @vgpu: a vGPU ++ * @gfn: the gfn of guest page ++ * ++ * Returns: ++ * A pointer to struct intel_vgpu_page_track if found, else NULL returned. ++ */ ++struct intel_vgpu_page_track *intel_vgpu_find_page_track( ++ struct intel_vgpu *vgpu, unsigned long gfn) ++{ ++ return radix_tree_lookup(&vgpu->page_track_tree, gfn); ++} ++ ++/** ++ * intel_vgpu_register_page_track - register a guest page to be tacked ++ * @vgpu: a vGPU ++ * @gfn: the gfn of guest page ++ * @handler: page track handler ++ * @priv: tracker private ++ * ++ * Returns: ++ * zero on success, negative error code if failed. ++ */ ++int intel_vgpu_register_page_track(struct intel_vgpu *vgpu, unsigned long gfn, ++ gvt_page_track_handler_t handler, void *priv) ++{ ++ struct intel_vgpu_page_track *track; ++ int ret; ++ ++ track = intel_vgpu_find_page_track(vgpu, gfn); ++ if (track) ++ return -EEXIST; ++ ++ track = kzalloc(sizeof(*track), GFP_KERNEL); ++ if (!track) ++ return -ENOMEM; ++ ++ track->handler = handler; ++ track->priv_data = priv; ++ ++ ret = radix_tree_insert(&vgpu->page_track_tree, gfn, track); ++ if (ret) { ++ kfree(track); ++ return ret; ++ } ++ ++ return 0; ++} ++ ++/** ++ * intel_vgpu_unregister_page_track - unregister the tracked guest page ++ * @vgpu: a vGPU ++ * @gfn: the gfn of guest page ++ * ++ */ ++void intel_vgpu_unregister_page_track(struct intel_vgpu *vgpu, ++ unsigned long gfn) ++{ ++ struct intel_vgpu_page_track *track; ++ ++ track = radix_tree_delete(&vgpu->page_track_tree, gfn); ++ if (track) { ++ if (track->tracked) ++ intel_gvt_hypervisor_disable_page_track(vgpu, gfn); ++ kfree(track); ++ } ++} ++ ++/** ++ * intel_vgpu_enable_page_track - set write-protection on guest page ++ * @vgpu: a vGPU ++ * @gfn: the gfn of guest page ++ * ++ * Returns: ++ * zero on success, negative error code if failed. ++ */ ++int intel_vgpu_enable_page_track(struct intel_vgpu *vgpu, unsigned long gfn) ++{ ++ struct intel_vgpu_page_track *track; ++ int ret; ++ ++ track = intel_vgpu_find_page_track(vgpu, gfn); ++ if (!track) ++ return -ENXIO; ++ ++ if (track->tracked) ++ return 0; ++ ++ ret = intel_gvt_hypervisor_enable_page_track(vgpu, gfn); ++ if (ret) ++ return ret; ++ track->tracked = true; ++ return 0; ++} ++ ++/** ++ * intel_vgpu_enable_page_track - cancel write-protection on guest page ++ * @vgpu: a vGPU ++ * @gfn: the gfn of guest page ++ * ++ * Returns: ++ * zero on success, negative error code if failed. ++ */ ++int intel_vgpu_disable_page_track(struct intel_vgpu *vgpu, unsigned long gfn) ++{ ++ struct intel_vgpu_page_track *track; ++ int ret; ++ ++ track = intel_vgpu_find_page_track(vgpu, gfn); ++ if (!track) ++ return -ENXIO; ++ ++ if (!track->tracked) ++ return 0; ++ ++ ret = intel_gvt_hypervisor_disable_page_track(vgpu, gfn); ++ if (ret) ++ return ret; ++ track->tracked = false; ++ return 0; ++} ++ ++/** ++ * intel_vgpu_page_track_handler - called when write to write-protected page ++ * @vgpu: a vGPU ++ * @gpa: the gpa of this write ++ * @data: the writed data ++ * @bytes: the length of this write ++ * ++ * Returns: ++ * zero on success, negative error code if failed. ++ */ ++int intel_vgpu_page_track_handler(struct intel_vgpu *vgpu, u64 gpa, ++ void *data, unsigned int bytes) ++{ ++ struct intel_vgpu_page_track *page_track; ++ int ret = 0; ++ ++ mutex_lock(&vgpu->vgpu_lock); ++ ++ page_track = intel_vgpu_find_page_track(vgpu, gpa >> PAGE_SHIFT); ++ if (!page_track) { ++ ret = -ENXIO; ++ goto out; ++ } ++ ++ if (unlikely(vgpu->failsafe)) { ++ /* Remove write protection to prevent furture traps. */ ++ intel_vgpu_disable_page_track(vgpu, gpa >> PAGE_SHIFT); ++ } else { ++ ret = page_track->handler(page_track, gpa, data, bytes); ++ if (ret) ++ gvt_err("guest page write error, gpa %llx\n", gpa); ++ } ++ ++out: ++ mutex_unlock(&vgpu->vgpu_lock); ++ return ret; ++} +diff --git a/drivers/gpu/drm/i915_legacy/gvt/page_track.h b/drivers/gpu/drm/i915_legacy/gvt/page_track.h +new file mode 100644 +index 000000000000..fa607a71c3c0 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/page_track.h +@@ -0,0 +1,56 @@ ++/* ++ * Copyright(c) 2011-2017 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ */ ++ ++#ifndef _GVT_PAGE_TRACK_H_ ++#define _GVT_PAGE_TRACK_H_ ++ ++struct intel_vgpu_page_track; ++ ++typedef int (*gvt_page_track_handler_t)( ++ struct intel_vgpu_page_track *page_track, ++ u64 gpa, void *data, int bytes); ++ ++/* Track record for a write-protected guest page. */ ++struct intel_vgpu_page_track { ++ gvt_page_track_handler_t handler; ++ bool tracked; ++ void *priv_data; ++}; ++ ++struct intel_vgpu_page_track *intel_vgpu_find_page_track( ++ struct intel_vgpu *vgpu, unsigned long gfn); ++ ++int intel_vgpu_register_page_track(struct intel_vgpu *vgpu, ++ unsigned long gfn, gvt_page_track_handler_t handler, ++ void *priv); ++void intel_vgpu_unregister_page_track(struct intel_vgpu *vgpu, ++ unsigned long gfn); ++ ++int intel_vgpu_enable_page_track(struct intel_vgpu *vgpu, unsigned long gfn); ++int intel_vgpu_disable_page_track(struct intel_vgpu *vgpu, unsigned long gfn); ++ ++int intel_vgpu_page_track_handler(struct intel_vgpu *vgpu, u64 gpa, ++ void *data, unsigned int bytes); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/gvt/reg.h b/drivers/gpu/drm/i915_legacy/gvt/reg.h +new file mode 100644 +index 000000000000..5b66e14c5b7b +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/reg.h +@@ -0,0 +1,131 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ */ ++ ++#ifndef _GVT_REG_H ++#define _GVT_REG_H ++ ++#define INTEL_GVT_PCI_CLASS_VGA_OTHER 0x80 ++ ++#define INTEL_GVT_PCI_GMCH_CONTROL 0x50 ++#define BDW_GMCH_GMS_SHIFT 8 ++#define BDW_GMCH_GMS_MASK 0xff ++ ++#define INTEL_GVT_PCI_SWSCI 0xe8 ++#define SWSCI_SCI_SELECT (1 << 15) ++#define SWSCI_SCI_TRIGGER 1 ++ ++#define INTEL_GVT_PCI_OPREGION 0xfc ++ ++#define INTEL_GVT_OPREGION_CLID 0x1AC ++#define INTEL_GVT_OPREGION_SCIC 0x200 ++#define OPREGION_SCIC_FUNC_MASK 0x1E ++#define OPREGION_SCIC_FUNC_SHIFT 1 ++#define OPREGION_SCIC_SUBFUNC_MASK 0xFF00 ++#define OPREGION_SCIC_SUBFUNC_SHIFT 8 ++#define OPREGION_SCIC_EXIT_MASK 0xE0 ++#define INTEL_GVT_OPREGION_SCIC_F_GETBIOSDATA 4 ++#define INTEL_GVT_OPREGION_SCIC_F_GETBIOSCALLBACKS 6 ++#define INTEL_GVT_OPREGION_SCIC_SF_SUPPRTEDCALLS 0 ++#define INTEL_GVT_OPREGION_SCIC_SF_REQEUSTEDCALLBACKS 1 ++#define INTEL_GVT_OPREGION_PARM 0x204 ++ ++#define INTEL_GVT_OPREGION_PAGES 2 ++#define INTEL_GVT_OPREGION_SIZE (INTEL_GVT_OPREGION_PAGES * PAGE_SIZE) ++#define INTEL_GVT_OPREGION_VBT_OFFSET 0x400 ++#define INTEL_GVT_OPREGION_VBT_SIZE \ ++ (INTEL_GVT_OPREGION_SIZE - INTEL_GVT_OPREGION_VBT_OFFSET) ++ ++#define VGT_SPRSTRIDE(pipe) _PIPE(pipe, _SPRA_STRIDE, _PLANE_STRIDE_2_B) ++ ++#define _REG_701C0(pipe, plane) (0x701c0 + pipe * 0x1000 + (plane - 1) * 0x100) ++#define _REG_701C4(pipe, plane) (0x701c4 + pipe * 0x1000 + (plane - 1) * 0x100) ++ ++#define SKL_FLIP_EVENT(pipe, plane) (PRIMARY_A_FLIP_DONE + (plane) * 3 + (pipe)) ++ ++#define PLANE_CTL_ASYNC_FLIP (1 << 9) ++#define REG50080_FLIP_TYPE_MASK 0x3 ++#define REG50080_FLIP_TYPE_ASYNC 0x1 ++ ++#define REG_50080(_pipe, _plane) ({ \ ++ typeof(_pipe) (p) = (_pipe); \ ++ typeof(_plane) (q) = (_plane); \ ++ (((p) == PIPE_A) ? (((q) == PLANE_PRIMARY) ? (_MMIO(0x50080)) : \ ++ (_MMIO(0x50090))) : \ ++ (((p) == PIPE_B) ? (((q) == PLANE_PRIMARY) ? (_MMIO(0x50088)) : \ ++ (_MMIO(0x50098))) : \ ++ (((p) == PIPE_C) ? (((q) == PLANE_PRIMARY) ? (_MMIO(0x5008C)) : \ ++ (_MMIO(0x5009C))) : \ ++ (_MMIO(0x50080))))); }) ++ ++#define REG_50080_TO_PIPE(_reg) ({ \ ++ typeof(_reg) (reg) = (_reg); \ ++ (((reg) == 0x50080 || (reg) == 0x50090) ? (PIPE_A) : \ ++ (((reg) == 0x50088 || (reg) == 0x50098) ? (PIPE_B) : \ ++ (((reg) == 0x5008C || (reg) == 0x5009C) ? (PIPE_C) : \ ++ (INVALID_PIPE)))); }) ++ ++#define REG_50080_TO_PLANE(_reg) ({ \ ++ typeof(_reg) (reg) = (_reg); \ ++ (((reg) == 0x50080 || (reg) == 0x50088 || (reg) == 0x5008C) ? \ ++ (PLANE_PRIMARY) : \ ++ (((reg) == 0x50090 || (reg) == 0x50098 || (reg) == 0x5009C) ? \ ++ (PLANE_SPRITE0) : (I915_MAX_PLANES))); }) ++ ++#define GFX_MODE_BIT_SET_IN_MASK(val, bit) \ ++ ((((bit) & 0xffff0000) == 0) && !!((val) & (((bit) << 16)))) ++ ++#define FORCEWAKE_RENDER_GEN9_REG 0xa278 ++#define FORCEWAKE_ACK_RENDER_GEN9_REG 0x0D84 ++#define FORCEWAKE_BLITTER_GEN9_REG 0xa188 ++#define FORCEWAKE_ACK_BLITTER_GEN9_REG 0x130044 ++#define FORCEWAKE_MEDIA_GEN9_REG 0xa270 ++#define FORCEWAKE_ACK_MEDIA_GEN9_REG 0x0D88 ++#define FORCEWAKE_ACK_HSW_REG 0x130044 ++ ++#define RB_HEAD_WRAP_CNT_MAX ((1 << 11) - 1) ++#define RB_HEAD_WRAP_CNT_OFF 21 ++#define RB_HEAD_OFF_MASK ((1U << 21) - (1U << 2)) ++#define RB_TAIL_OFF_MASK ((1U << 21) - (1U << 3)) ++#define RB_TAIL_SIZE_MASK ((1U << 21) - (1U << 12)) ++#define _RING_CTL_BUF_SIZE(ctl) (((ctl) & RB_TAIL_SIZE_MASK) + \ ++ I915_GTT_PAGE_SIZE) ++ ++#define PCH_GPIO_BASE _MMIO(0xc5010) ++ ++#define PCH_GMBUS0 _MMIO(0xc5100) ++#define PCH_GMBUS1 _MMIO(0xc5104) ++#define PCH_GMBUS2 _MMIO(0xc5108) ++#define PCH_GMBUS3 _MMIO(0xc510c) ++#define PCH_GMBUS4 _MMIO(0xc5110) ++#define PCH_GMBUS5 _MMIO(0xc5120) ++ ++#define TRVATTL3PTRDW(i) _MMIO(0x4de0 + (i) * 4) ++#define TRNULLDETCT _MMIO(0x4de8) ++#define TRINVTILEDETCT _MMIO(0x4dec) ++#define TRVADR _MMIO(0x4df0) ++#define TRTTE _MMIO(0x4df4) ++#define RING_EXCC(base) _MMIO((base) + 0x28) ++#define RING_GFX_MODE(base) _MMIO((base) + 0x29c) ++#define VF_GUARDBAND _MMIO(0x83a4) ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/gvt/sched_policy.c b/drivers/gpu/drm/i915_legacy/gvt/sched_policy.c +new file mode 100644 +index 000000000000..1c763a27a412 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/sched_policy.c +@@ -0,0 +1,479 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Anhua Xu ++ * Kevin Tian ++ * ++ * Contributors: ++ * Min He ++ * Bing Niu ++ * Zhi Wang ++ * ++ */ ++ ++#include "i915_drv.h" ++#include "gvt.h" ++ ++static bool vgpu_has_pending_workload(struct intel_vgpu *vgpu) ++{ ++ enum intel_engine_id i; ++ struct intel_engine_cs *engine; ++ ++ for_each_engine(engine, vgpu->gvt->dev_priv, i) { ++ if (!list_empty(workload_q_head(vgpu, i))) ++ return true; ++ } ++ ++ return false; ++} ++ ++/* We give 2 seconds higher prio for vGPU during start */ ++#define GVT_SCHED_VGPU_PRI_TIME 2 ++ ++struct vgpu_sched_data { ++ struct list_head lru_list; ++ struct intel_vgpu *vgpu; ++ bool active; ++ bool pri_sched; ++ ktime_t pri_time; ++ ktime_t sched_in_time; ++ ktime_t sched_time; ++ ktime_t left_ts; ++ ktime_t allocated_ts; ++ ++ struct vgpu_sched_ctl sched_ctl; ++}; ++ ++struct gvt_sched_data { ++ struct intel_gvt *gvt; ++ struct hrtimer timer; ++ unsigned long period; ++ struct list_head lru_runq_head; ++ ktime_t expire_time; ++}; ++ ++static void vgpu_update_timeslice(struct intel_vgpu *vgpu, ktime_t cur_time) ++{ ++ ktime_t delta_ts; ++ struct vgpu_sched_data *vgpu_data; ++ ++ if (!vgpu || vgpu == vgpu->gvt->idle_vgpu) ++ return; ++ ++ vgpu_data = vgpu->sched_data; ++ delta_ts = ktime_sub(cur_time, vgpu_data->sched_in_time); ++ vgpu_data->sched_time = ktime_add(vgpu_data->sched_time, delta_ts); ++ vgpu_data->left_ts = ktime_sub(vgpu_data->left_ts, delta_ts); ++ vgpu_data->sched_in_time = cur_time; ++} ++ ++#define GVT_TS_BALANCE_PERIOD_MS 100 ++#define GVT_TS_BALANCE_STAGE_NUM 10 ++ ++static void gvt_balance_timeslice(struct gvt_sched_data *sched_data) ++{ ++ struct vgpu_sched_data *vgpu_data; ++ struct list_head *pos; ++ static u64 stage_check; ++ int stage = stage_check++ % GVT_TS_BALANCE_STAGE_NUM; ++ ++ /* The timeslice accumulation reset at stage 0, which is ++ * allocated again without adding previous debt. ++ */ ++ if (stage == 0) { ++ int total_weight = 0; ++ ktime_t fair_timeslice; ++ ++ list_for_each(pos, &sched_data->lru_runq_head) { ++ vgpu_data = container_of(pos, struct vgpu_sched_data, lru_list); ++ total_weight += vgpu_data->sched_ctl.weight; ++ } ++ ++ list_for_each(pos, &sched_data->lru_runq_head) { ++ vgpu_data = container_of(pos, struct vgpu_sched_data, lru_list); ++ fair_timeslice = ktime_divns(ms_to_ktime(GVT_TS_BALANCE_PERIOD_MS), ++ total_weight) * vgpu_data->sched_ctl.weight; ++ ++ vgpu_data->allocated_ts = fair_timeslice; ++ vgpu_data->left_ts = vgpu_data->allocated_ts; ++ } ++ } else { ++ list_for_each(pos, &sched_data->lru_runq_head) { ++ vgpu_data = container_of(pos, struct vgpu_sched_data, lru_list); ++ ++ /* timeslice for next 100ms should add the left/debt ++ * slice of previous stages. ++ */ ++ vgpu_data->left_ts += vgpu_data->allocated_ts; ++ } ++ } ++} ++ ++static void try_to_schedule_next_vgpu(struct intel_gvt *gvt) ++{ ++ struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; ++ enum intel_engine_id i; ++ struct intel_engine_cs *engine; ++ struct vgpu_sched_data *vgpu_data; ++ ktime_t cur_time; ++ ++ /* no need to schedule if next_vgpu is the same with current_vgpu, ++ * let scheduler chose next_vgpu again by setting it to NULL. ++ */ ++ if (scheduler->next_vgpu == scheduler->current_vgpu) { ++ scheduler->next_vgpu = NULL; ++ return; ++ } ++ ++ /* ++ * after the flag is set, workload dispatch thread will ++ * stop dispatching workload for current vgpu ++ */ ++ scheduler->need_reschedule = true; ++ ++ /* still have uncompleted workload? */ ++ for_each_engine(engine, gvt->dev_priv, i) { ++ if (scheduler->current_workload[i]) ++ return; ++ } ++ ++ cur_time = ktime_get(); ++ vgpu_update_timeslice(scheduler->current_vgpu, cur_time); ++ vgpu_data = scheduler->next_vgpu->sched_data; ++ vgpu_data->sched_in_time = cur_time; ++ ++ /* switch current vgpu */ ++ scheduler->current_vgpu = scheduler->next_vgpu; ++ scheduler->next_vgpu = NULL; ++ ++ scheduler->need_reschedule = false; ++ ++ /* wake up workload dispatch thread */ ++ for_each_engine(engine, gvt->dev_priv, i) ++ wake_up(&scheduler->waitq[i]); ++} ++ ++static struct intel_vgpu *find_busy_vgpu(struct gvt_sched_data *sched_data) ++{ ++ struct vgpu_sched_data *vgpu_data; ++ struct intel_vgpu *vgpu = NULL; ++ struct list_head *head = &sched_data->lru_runq_head; ++ struct list_head *pos; ++ ++ /* search a vgpu with pending workload */ ++ list_for_each(pos, head) { ++ ++ vgpu_data = container_of(pos, struct vgpu_sched_data, lru_list); ++ if (!vgpu_has_pending_workload(vgpu_data->vgpu)) ++ continue; ++ ++ if (vgpu_data->pri_sched) { ++ if (ktime_before(ktime_get(), vgpu_data->pri_time)) { ++ vgpu = vgpu_data->vgpu; ++ break; ++ } else ++ vgpu_data->pri_sched = false; ++ } ++ ++ /* Return the vGPU only if it has time slice left */ ++ if (vgpu_data->left_ts > 0) { ++ vgpu = vgpu_data->vgpu; ++ break; ++ } ++ } ++ ++ return vgpu; ++} ++ ++/* in nanosecond */ ++#define GVT_DEFAULT_TIME_SLICE 1000000 ++ ++static void tbs_sched_func(struct gvt_sched_data *sched_data) ++{ ++ struct intel_gvt *gvt = sched_data->gvt; ++ struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; ++ struct vgpu_sched_data *vgpu_data; ++ struct intel_vgpu *vgpu = NULL; ++ ++ /* no active vgpu or has already had a target */ ++ if (list_empty(&sched_data->lru_runq_head) || scheduler->next_vgpu) ++ goto out; ++ ++ vgpu = find_busy_vgpu(sched_data); ++ if (vgpu) { ++ scheduler->next_vgpu = vgpu; ++ vgpu_data = vgpu->sched_data; ++ if (!vgpu_data->pri_sched) { ++ /* Move the last used vGPU to the tail of lru_list */ ++ list_del_init(&vgpu_data->lru_list); ++ list_add_tail(&vgpu_data->lru_list, ++ &sched_data->lru_runq_head); ++ } ++ } else { ++ scheduler->next_vgpu = gvt->idle_vgpu; ++ } ++out: ++ if (scheduler->next_vgpu) ++ try_to_schedule_next_vgpu(gvt); ++} ++ ++void intel_gvt_schedule(struct intel_gvt *gvt) ++{ ++ struct gvt_sched_data *sched_data = gvt->scheduler.sched_data; ++ ktime_t cur_time; ++ ++ mutex_lock(&gvt->sched_lock); ++ cur_time = ktime_get(); ++ ++ if (test_and_clear_bit(INTEL_GVT_REQUEST_SCHED, ++ (void *)&gvt->service_request)) { ++ if (cur_time >= sched_data->expire_time) { ++ gvt_balance_timeslice(sched_data); ++ sched_data->expire_time = ktime_add_ms( ++ cur_time, GVT_TS_BALANCE_PERIOD_MS); ++ } ++ } ++ clear_bit(INTEL_GVT_REQUEST_EVENT_SCHED, (void *)&gvt->service_request); ++ ++ vgpu_update_timeslice(gvt->scheduler.current_vgpu, cur_time); ++ tbs_sched_func(sched_data); ++ ++ mutex_unlock(&gvt->sched_lock); ++} ++ ++static enum hrtimer_restart tbs_timer_fn(struct hrtimer *timer_data) ++{ ++ struct gvt_sched_data *data; ++ ++ data = container_of(timer_data, struct gvt_sched_data, timer); ++ ++ intel_gvt_request_service(data->gvt, INTEL_GVT_REQUEST_SCHED); ++ ++ hrtimer_add_expires_ns(&data->timer, data->period); ++ ++ return HRTIMER_RESTART; ++} ++ ++static int tbs_sched_init(struct intel_gvt *gvt) ++{ ++ struct intel_gvt_workload_scheduler *scheduler = ++ &gvt->scheduler; ++ ++ struct gvt_sched_data *data; ++ ++ data = kzalloc(sizeof(*data), GFP_KERNEL); ++ if (!data) ++ return -ENOMEM; ++ ++ INIT_LIST_HEAD(&data->lru_runq_head); ++ hrtimer_init(&data->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); ++ data->timer.function = tbs_timer_fn; ++ data->period = GVT_DEFAULT_TIME_SLICE; ++ data->gvt = gvt; ++ ++ scheduler->sched_data = data; ++ ++ return 0; ++} ++ ++static void tbs_sched_clean(struct intel_gvt *gvt) ++{ ++ struct intel_gvt_workload_scheduler *scheduler = ++ &gvt->scheduler; ++ struct gvt_sched_data *data = scheduler->sched_data; ++ ++ hrtimer_cancel(&data->timer); ++ ++ kfree(data); ++ scheduler->sched_data = NULL; ++} ++ ++static int tbs_sched_init_vgpu(struct intel_vgpu *vgpu) ++{ ++ struct vgpu_sched_data *data; ++ ++ data = kzalloc(sizeof(*data), GFP_KERNEL); ++ if (!data) ++ return -ENOMEM; ++ ++ data->sched_ctl.weight = vgpu->sched_ctl.weight; ++ data->vgpu = vgpu; ++ INIT_LIST_HEAD(&data->lru_list); ++ ++ vgpu->sched_data = data; ++ ++ return 0; ++} ++ ++static void tbs_sched_clean_vgpu(struct intel_vgpu *vgpu) ++{ ++ struct intel_gvt *gvt = vgpu->gvt; ++ struct gvt_sched_data *sched_data = gvt->scheduler.sched_data; ++ ++ kfree(vgpu->sched_data); ++ vgpu->sched_data = NULL; ++ ++ /* this vgpu id has been removed */ ++ if (idr_is_empty(&gvt->vgpu_idr)) ++ hrtimer_cancel(&sched_data->timer); ++} ++ ++static void tbs_sched_start_schedule(struct intel_vgpu *vgpu) ++{ ++ struct gvt_sched_data *sched_data = vgpu->gvt->scheduler.sched_data; ++ struct vgpu_sched_data *vgpu_data = vgpu->sched_data; ++ ktime_t now; ++ ++ if (!list_empty(&vgpu_data->lru_list)) ++ return; ++ ++ now = ktime_get(); ++ vgpu_data->pri_time = ktime_add(now, ++ ktime_set(GVT_SCHED_VGPU_PRI_TIME, 0)); ++ vgpu_data->pri_sched = true; ++ ++ list_add(&vgpu_data->lru_list, &sched_data->lru_runq_head); ++ ++ if (!hrtimer_active(&sched_data->timer)) ++ hrtimer_start(&sched_data->timer, ktime_add_ns(ktime_get(), ++ sched_data->period), HRTIMER_MODE_ABS); ++ vgpu_data->active = true; ++} ++ ++static void tbs_sched_stop_schedule(struct intel_vgpu *vgpu) ++{ ++ struct vgpu_sched_data *vgpu_data = vgpu->sched_data; ++ ++ list_del_init(&vgpu_data->lru_list); ++ vgpu_data->active = false; ++} ++ ++static struct intel_gvt_sched_policy_ops tbs_schedule_ops = { ++ .init = tbs_sched_init, ++ .clean = tbs_sched_clean, ++ .init_vgpu = tbs_sched_init_vgpu, ++ .clean_vgpu = tbs_sched_clean_vgpu, ++ .start_schedule = tbs_sched_start_schedule, ++ .stop_schedule = tbs_sched_stop_schedule, ++}; ++ ++int intel_gvt_init_sched_policy(struct intel_gvt *gvt) ++{ ++ int ret; ++ ++ mutex_lock(&gvt->sched_lock); ++ gvt->scheduler.sched_ops = &tbs_schedule_ops; ++ ret = gvt->scheduler.sched_ops->init(gvt); ++ mutex_unlock(&gvt->sched_lock); ++ ++ return ret; ++} ++ ++void intel_gvt_clean_sched_policy(struct intel_gvt *gvt) ++{ ++ mutex_lock(&gvt->sched_lock); ++ gvt->scheduler.sched_ops->clean(gvt); ++ mutex_unlock(&gvt->sched_lock); ++} ++ ++/* for per-vgpu scheduler policy, there are 2 per-vgpu data: ++ * sched_data, and sched_ctl. We see these 2 data as part of ++ * the global scheduler which are proteced by gvt->sched_lock. ++ * Caller should make their decision if the vgpu_lock should ++ * be hold outside. ++ */ ++ ++int intel_vgpu_init_sched_policy(struct intel_vgpu *vgpu) ++{ ++ int ret; ++ ++ mutex_lock(&vgpu->gvt->sched_lock); ++ ret = vgpu->gvt->scheduler.sched_ops->init_vgpu(vgpu); ++ mutex_unlock(&vgpu->gvt->sched_lock); ++ ++ return ret; ++} ++ ++void intel_vgpu_clean_sched_policy(struct intel_vgpu *vgpu) ++{ ++ mutex_lock(&vgpu->gvt->sched_lock); ++ vgpu->gvt->scheduler.sched_ops->clean_vgpu(vgpu); ++ mutex_unlock(&vgpu->gvt->sched_lock); ++} ++ ++void intel_vgpu_start_schedule(struct intel_vgpu *vgpu) ++{ ++ struct vgpu_sched_data *vgpu_data = vgpu->sched_data; ++ ++ mutex_lock(&vgpu->gvt->sched_lock); ++ if (!vgpu_data->active) { ++ gvt_dbg_core("vgpu%d: start schedule\n", vgpu->id); ++ vgpu->gvt->scheduler.sched_ops->start_schedule(vgpu); ++ } ++ mutex_unlock(&vgpu->gvt->sched_lock); ++} ++ ++void intel_gvt_kick_schedule(struct intel_gvt *gvt) ++{ ++ mutex_lock(&gvt->sched_lock); ++ intel_gvt_request_service(gvt, INTEL_GVT_REQUEST_EVENT_SCHED); ++ mutex_unlock(&gvt->sched_lock); ++} ++ ++void intel_vgpu_stop_schedule(struct intel_vgpu *vgpu) ++{ ++ struct intel_gvt_workload_scheduler *scheduler = ++ &vgpu->gvt->scheduler; ++ int ring_id; ++ struct vgpu_sched_data *vgpu_data = vgpu->sched_data; ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ ++ if (!vgpu_data->active) ++ return; ++ ++ gvt_dbg_core("vgpu%d: stop schedule\n", vgpu->id); ++ ++ mutex_lock(&vgpu->gvt->sched_lock); ++ scheduler->sched_ops->stop_schedule(vgpu); ++ ++ if (scheduler->next_vgpu == vgpu) ++ scheduler->next_vgpu = NULL; ++ ++ if (scheduler->current_vgpu == vgpu) { ++ /* stop workload dispatching */ ++ scheduler->need_reschedule = true; ++ scheduler->current_vgpu = NULL; ++ } ++ ++ intel_runtime_pm_get(dev_priv); ++ spin_lock_bh(&scheduler->mmio_context_lock); ++ for (ring_id = 0; ring_id < I915_NUM_ENGINES; ring_id++) { ++ if (scheduler->engine_owner[ring_id] == vgpu) { ++ intel_gvt_switch_mmio(vgpu, NULL, ring_id); ++ scheduler->engine_owner[ring_id] = NULL; ++ } ++ } ++ spin_unlock_bh(&scheduler->mmio_context_lock); ++ intel_runtime_pm_put_unchecked(dev_priv); ++ mutex_unlock(&vgpu->gvt->sched_lock); ++} +diff --git a/drivers/gpu/drm/i915_legacy/gvt/sched_policy.h b/drivers/gpu/drm/i915_legacy/gvt/sched_policy.h +new file mode 100644 +index 000000000000..7b59e3e88b8b +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/sched_policy.h +@@ -0,0 +1,62 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Anhua Xu ++ * Kevin Tian ++ * ++ * Contributors: ++ * Min He ++ * Bing Niu ++ * Zhi Wang ++ * ++ */ ++ ++#ifndef __GVT_SCHED_POLICY__ ++#define __GVT_SCHED_POLICY__ ++ ++struct intel_gvt_sched_policy_ops { ++ int (*init)(struct intel_gvt *gvt); ++ void (*clean)(struct intel_gvt *gvt); ++ int (*init_vgpu)(struct intel_vgpu *vgpu); ++ void (*clean_vgpu)(struct intel_vgpu *vgpu); ++ void (*start_schedule)(struct intel_vgpu *vgpu); ++ void (*stop_schedule)(struct intel_vgpu *vgpu); ++}; ++ ++void intel_gvt_schedule(struct intel_gvt *gvt); ++ ++int intel_gvt_init_sched_policy(struct intel_gvt *gvt); ++ ++void intel_gvt_clean_sched_policy(struct intel_gvt *gvt); ++ ++int intel_vgpu_init_sched_policy(struct intel_vgpu *vgpu); ++ ++void intel_vgpu_clean_sched_policy(struct intel_vgpu *vgpu); ++ ++void intel_vgpu_start_schedule(struct intel_vgpu *vgpu); ++ ++void intel_vgpu_stop_schedule(struct intel_vgpu *vgpu); ++ ++void intel_gvt_kick_schedule(struct intel_gvt *gvt); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/gvt/scheduler.c b/drivers/gpu/drm/i915_legacy/gvt/scheduler.c +new file mode 100644 +index 000000000000..0f919f0a43d4 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/scheduler.c +@@ -0,0 +1,1550 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Zhi Wang ++ * ++ * Contributors: ++ * Ping Gao ++ * Tina Zhang ++ * Chanbin Du ++ * Min He ++ * Bing Niu ++ * Zhenyu Wang ++ * ++ */ ++ ++#include ++ ++#include "i915_drv.h" ++#include "gvt.h" ++ ++#define RING_CTX_OFF(x) \ ++ offsetof(struct execlist_ring_context, x) ++ ++static void set_context_pdp_root_pointer( ++ struct execlist_ring_context *ring_context, ++ u32 pdp[8]) ++{ ++ int i; ++ ++ for (i = 0; i < 8; i++) ++ ring_context->pdps[i].val = pdp[7 - i]; ++} ++ ++static void update_shadow_pdps(struct intel_vgpu_workload *workload) ++{ ++ struct drm_i915_gem_object *ctx_obj = ++ workload->req->hw_context->state->obj; ++ struct execlist_ring_context *shadow_ring_context; ++ struct page *page; ++ ++ if (WARN_ON(!workload->shadow_mm)) ++ return; ++ ++ if (WARN_ON(!atomic_read(&workload->shadow_mm->pincount))) ++ return; ++ ++ page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN); ++ shadow_ring_context = kmap(page); ++ set_context_pdp_root_pointer(shadow_ring_context, ++ (void *)workload->shadow_mm->ppgtt_mm.shadow_pdps); ++ kunmap(page); ++} ++ ++/* ++ * when populating shadow ctx from guest, we should not overrride oa related ++ * registers, so that they will not be overlapped by guest oa configs. Thus ++ * made it possible to capture oa data from host for both host and guests. ++ */ ++static void sr_oa_regs(struct intel_vgpu_workload *workload, ++ u32 *reg_state, bool save) ++{ ++ struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv; ++ u32 ctx_oactxctrl = dev_priv->perf.oa.ctx_oactxctrl_offset; ++ u32 ctx_flexeu0 = dev_priv->perf.oa.ctx_flexeu0_offset; ++ int i = 0; ++ u32 flex_mmio[] = { ++ i915_mmio_reg_offset(EU_PERF_CNTL0), ++ i915_mmio_reg_offset(EU_PERF_CNTL1), ++ i915_mmio_reg_offset(EU_PERF_CNTL2), ++ i915_mmio_reg_offset(EU_PERF_CNTL3), ++ i915_mmio_reg_offset(EU_PERF_CNTL4), ++ i915_mmio_reg_offset(EU_PERF_CNTL5), ++ i915_mmio_reg_offset(EU_PERF_CNTL6), ++ }; ++ ++ if (workload->ring_id != RCS0) ++ return; ++ ++ if (save) { ++ workload->oactxctrl = reg_state[ctx_oactxctrl + 1]; ++ ++ for (i = 0; i < ARRAY_SIZE(workload->flex_mmio); i++) { ++ u32 state_offset = ctx_flexeu0 + i * 2; ++ ++ workload->flex_mmio[i] = reg_state[state_offset + 1]; ++ } ++ } else { ++ reg_state[ctx_oactxctrl] = ++ i915_mmio_reg_offset(GEN8_OACTXCONTROL); ++ reg_state[ctx_oactxctrl + 1] = workload->oactxctrl; ++ ++ for (i = 0; i < ARRAY_SIZE(workload->flex_mmio); i++) { ++ u32 state_offset = ctx_flexeu0 + i * 2; ++ u32 mmio = flex_mmio[i]; ++ ++ reg_state[state_offset] = mmio; ++ reg_state[state_offset + 1] = workload->flex_mmio[i]; ++ } ++ } ++} ++ ++static int populate_shadow_context(struct intel_vgpu_workload *workload) ++{ ++ struct intel_vgpu *vgpu = workload->vgpu; ++ struct intel_gvt *gvt = vgpu->gvt; ++ int ring_id = workload->ring_id; ++ struct drm_i915_gem_object *ctx_obj = ++ workload->req->hw_context->state->obj; ++ struct execlist_ring_context *shadow_ring_context; ++ struct page *page; ++ void *dst; ++ unsigned long context_gpa, context_page_num; ++ int i; ++ ++ page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN); ++ shadow_ring_context = kmap(page); ++ ++ sr_oa_regs(workload, (u32 *)shadow_ring_context, true); ++#define COPY_REG(name) \ ++ intel_gvt_hypervisor_read_gpa(vgpu, workload->ring_context_gpa \ ++ + RING_CTX_OFF(name.val), &shadow_ring_context->name.val, 4) ++#define COPY_REG_MASKED(name) {\ ++ intel_gvt_hypervisor_read_gpa(vgpu, workload->ring_context_gpa \ ++ + RING_CTX_OFF(name.val),\ ++ &shadow_ring_context->name.val, 4);\ ++ shadow_ring_context->name.val |= 0xffff << 16;\ ++ } ++ ++ COPY_REG_MASKED(ctx_ctrl); ++ COPY_REG(ctx_timestamp); ++ ++ if (ring_id == RCS0) { ++ COPY_REG(bb_per_ctx_ptr); ++ COPY_REG(rcs_indirect_ctx); ++ COPY_REG(rcs_indirect_ctx_offset); ++ } ++#undef COPY_REG ++#undef COPY_REG_MASKED ++ ++ intel_gvt_hypervisor_read_gpa(vgpu, ++ workload->ring_context_gpa + ++ sizeof(*shadow_ring_context), ++ (void *)shadow_ring_context + ++ sizeof(*shadow_ring_context), ++ I915_GTT_PAGE_SIZE - sizeof(*shadow_ring_context)); ++ ++ sr_oa_regs(workload, (u32 *)shadow_ring_context, false); ++ kunmap(page); ++ ++ if (IS_RESTORE_INHIBIT(shadow_ring_context->ctx_ctrl.val)) ++ return 0; ++ ++ gvt_dbg_sched("ring id %d workload lrca %x", ring_id, ++ workload->ctx_desc.lrca); ++ ++ context_page_num = gvt->dev_priv->engine[ring_id]->context_size; ++ ++ context_page_num = context_page_num >> PAGE_SHIFT; ++ ++ if (IS_BROADWELL(gvt->dev_priv) && ring_id == RCS0) ++ context_page_num = 19; ++ ++ i = 2; ++ while (i < context_page_num) { ++ context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, ++ (u32)((workload->ctx_desc.lrca + i) << ++ I915_GTT_PAGE_SHIFT)); ++ if (context_gpa == INTEL_GVT_INVALID_ADDR) { ++ gvt_vgpu_err("Invalid guest context descriptor\n"); ++ return -EFAULT; ++ } ++ ++ page = i915_gem_object_get_page(ctx_obj, LRC_HEADER_PAGES + i); ++ dst = kmap(page); ++ intel_gvt_hypervisor_read_gpa(vgpu, context_gpa, dst, ++ I915_GTT_PAGE_SIZE); ++ kunmap(page); ++ i++; ++ } ++ return 0; ++} ++ ++static inline bool is_gvt_request(struct i915_request *req) ++{ ++ return i915_gem_context_force_single_submission(req->gem_context); ++} ++ ++static void save_ring_hw_state(struct intel_vgpu *vgpu, int ring_id) ++{ ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ u32 ring_base = dev_priv->engine[ring_id]->mmio_base; ++ i915_reg_t reg; ++ ++ reg = RING_INSTDONE(ring_base); ++ vgpu_vreg(vgpu, i915_mmio_reg_offset(reg)) = I915_READ_FW(reg); ++ reg = RING_ACTHD(ring_base); ++ vgpu_vreg(vgpu, i915_mmio_reg_offset(reg)) = I915_READ_FW(reg); ++ reg = RING_ACTHD_UDW(ring_base); ++ vgpu_vreg(vgpu, i915_mmio_reg_offset(reg)) = I915_READ_FW(reg); ++} ++ ++static int shadow_context_status_change(struct notifier_block *nb, ++ unsigned long action, void *data) ++{ ++ struct i915_request *req = data; ++ struct intel_gvt *gvt = container_of(nb, struct intel_gvt, ++ shadow_ctx_notifier_block[req->engine->id]); ++ struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; ++ enum intel_engine_id ring_id = req->engine->id; ++ struct intel_vgpu_workload *workload; ++ unsigned long flags; ++ ++ if (!is_gvt_request(req)) { ++ spin_lock_irqsave(&scheduler->mmio_context_lock, flags); ++ if (action == INTEL_CONTEXT_SCHEDULE_IN && ++ scheduler->engine_owner[ring_id]) { ++ /* Switch ring from vGPU to host. */ ++ intel_gvt_switch_mmio(scheduler->engine_owner[ring_id], ++ NULL, ring_id); ++ scheduler->engine_owner[ring_id] = NULL; ++ } ++ spin_unlock_irqrestore(&scheduler->mmio_context_lock, flags); ++ ++ return NOTIFY_OK; ++ } ++ ++ workload = scheduler->current_workload[ring_id]; ++ if (unlikely(!workload)) ++ return NOTIFY_OK; ++ ++ switch (action) { ++ case INTEL_CONTEXT_SCHEDULE_IN: ++ spin_lock_irqsave(&scheduler->mmio_context_lock, flags); ++ if (workload->vgpu != scheduler->engine_owner[ring_id]) { ++ /* Switch ring from host to vGPU or vGPU to vGPU. */ ++ intel_gvt_switch_mmio(scheduler->engine_owner[ring_id], ++ workload->vgpu, ring_id); ++ scheduler->engine_owner[ring_id] = workload->vgpu; ++ } else ++ gvt_dbg_sched("skip ring %d mmio switch for vgpu%d\n", ++ ring_id, workload->vgpu->id); ++ spin_unlock_irqrestore(&scheduler->mmio_context_lock, flags); ++ atomic_set(&workload->shadow_ctx_active, 1); ++ break; ++ case INTEL_CONTEXT_SCHEDULE_OUT: ++ save_ring_hw_state(workload->vgpu, ring_id); ++ atomic_set(&workload->shadow_ctx_active, 0); ++ break; ++ case INTEL_CONTEXT_SCHEDULE_PREEMPTED: ++ save_ring_hw_state(workload->vgpu, ring_id); ++ break; ++ default: ++ WARN_ON(1); ++ return NOTIFY_OK; ++ } ++ wake_up(&workload->shadow_ctx_status_wq); ++ return NOTIFY_OK; ++} ++ ++static void shadow_context_descriptor_update(struct intel_context *ce) ++{ ++ u64 desc = 0; ++ ++ desc = ce->lrc_desc; ++ ++ /* Update bits 0-11 of the context descriptor which includes flags ++ * like GEN8_CTX_* cached in desc_template ++ */ ++ desc &= U64_MAX << 12; ++ desc |= ce->gem_context->desc_template & ((1ULL << 12) - 1); ++ ++ ce->lrc_desc = desc; ++} ++ ++static int copy_workload_to_ring_buffer(struct intel_vgpu_workload *workload) ++{ ++ struct intel_vgpu *vgpu = workload->vgpu; ++ struct i915_request *req = workload->req; ++ void *shadow_ring_buffer_va; ++ u32 *cs; ++ int err; ++ ++ if (IS_GEN(req->i915, 9) && is_inhibit_context(req->hw_context)) ++ intel_vgpu_restore_inhibit_context(vgpu, req); ++ ++ /* ++ * To track whether a request has started on HW, we can emit a ++ * breadcrumb at the beginning of the request and check its ++ * timeline's HWSP to see if the breadcrumb has advanced past the ++ * start of this request. Actually, the request must have the ++ * init_breadcrumb if its timeline set has_init_bread_crumb, or the ++ * scheduler might get a wrong state of it during reset. Since the ++ * requests from gvt always set the has_init_breadcrumb flag, here ++ * need to do the emit_init_breadcrumb for all the requests. ++ */ ++ if (req->engine->emit_init_breadcrumb) { ++ err = req->engine->emit_init_breadcrumb(req); ++ if (err) { ++ gvt_vgpu_err("fail to emit init breadcrumb\n"); ++ return err; ++ } ++ } ++ ++ /* allocate shadow ring buffer */ ++ cs = intel_ring_begin(workload->req, workload->rb_len / sizeof(u32)); ++ if (IS_ERR(cs)) { ++ gvt_vgpu_err("fail to alloc size =%ld shadow ring buffer\n", ++ workload->rb_len); ++ return PTR_ERR(cs); ++ } ++ ++ shadow_ring_buffer_va = workload->shadow_ring_buffer_va; ++ ++ /* get shadow ring buffer va */ ++ workload->shadow_ring_buffer_va = cs; ++ ++ memcpy(cs, shadow_ring_buffer_va, ++ workload->rb_len); ++ ++ cs += workload->rb_len / sizeof(u32); ++ intel_ring_advance(workload->req, cs); ++ ++ return 0; ++} ++ ++static void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) ++{ ++ if (!wa_ctx->indirect_ctx.obj) ++ return; ++ ++ i915_gem_object_unpin_map(wa_ctx->indirect_ctx.obj); ++ i915_gem_object_put(wa_ctx->indirect_ctx.obj); ++ ++ wa_ctx->indirect_ctx.obj = NULL; ++ wa_ctx->indirect_ctx.shadow_va = NULL; ++} ++ ++static int set_context_ppgtt_from_shadow(struct intel_vgpu_workload *workload, ++ struct i915_gem_context *ctx) ++{ ++ struct intel_vgpu_mm *mm = workload->shadow_mm; ++ struct i915_hw_ppgtt *ppgtt = ctx->ppgtt; ++ int i = 0; ++ ++ if (mm->type != INTEL_GVT_MM_PPGTT || !mm->ppgtt_mm.shadowed) ++ return -EINVAL; ++ ++ if (mm->ppgtt_mm.root_entry_type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY) { ++ px_dma(&ppgtt->pml4) = mm->ppgtt_mm.shadow_pdps[0]; ++ } else { ++ for (i = 0; i < GVT_RING_CTX_NR_PDPS; i++) { ++ px_dma(ppgtt->pdp.page_directory[i]) = ++ mm->ppgtt_mm.shadow_pdps[i]; ++ } ++ } ++ ++ return 0; ++} ++ ++static int ++intel_gvt_workload_req_alloc(struct intel_vgpu_workload *workload) ++{ ++ struct intel_vgpu *vgpu = workload->vgpu; ++ struct intel_vgpu_submission *s = &vgpu->submission; ++ struct i915_gem_context *shadow_ctx = s->shadow_ctx; ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ struct intel_engine_cs *engine = dev_priv->engine[workload->ring_id]; ++ struct i915_request *rq; ++ int ret = 0; ++ ++ lockdep_assert_held(&dev_priv->drm.struct_mutex); ++ ++ if (workload->req) ++ goto out; ++ ++ rq = i915_request_alloc(engine, shadow_ctx); ++ if (IS_ERR(rq)) { ++ gvt_vgpu_err("fail to allocate gem request\n"); ++ ret = PTR_ERR(rq); ++ goto out; ++ } ++ workload->req = i915_request_get(rq); ++out: ++ return ret; ++} ++ ++/** ++ * intel_gvt_scan_and_shadow_workload - audit the workload by scanning and ++ * shadow it as well, include ringbuffer,wa_ctx and ctx. ++ * @workload: an abstract entity for each execlist submission. ++ * ++ * This function is called before the workload submitting to i915, to make ++ * sure the content of the workload is valid. ++ */ ++int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload) ++{ ++ struct intel_vgpu *vgpu = workload->vgpu; ++ struct intel_vgpu_submission *s = &vgpu->submission; ++ struct i915_gem_context *shadow_ctx = s->shadow_ctx; ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ struct intel_engine_cs *engine = dev_priv->engine[workload->ring_id]; ++ struct intel_context *ce; ++ int ret; ++ ++ lockdep_assert_held(&dev_priv->drm.struct_mutex); ++ ++ if (workload->shadow) ++ return 0; ++ ++ /* pin shadow context by gvt even the shadow context will be pinned ++ * when i915 alloc request. That is because gvt will update the guest ++ * context from shadow context when workload is completed, and at that ++ * moment, i915 may already unpined the shadow context to make the ++ * shadow_ctx pages invalid. So gvt need to pin itself. After update ++ * the guest context, gvt can unpin the shadow_ctx safely. ++ */ ++ ce = intel_context_pin(shadow_ctx, engine); ++ if (IS_ERR(ce)) { ++ gvt_vgpu_err("fail to pin shadow context\n"); ++ return PTR_ERR(ce); ++ } ++ ++ shadow_ctx->desc_template &= ~(0x3 << GEN8_CTX_ADDRESSING_MODE_SHIFT); ++ shadow_ctx->desc_template |= workload->ctx_desc.addressing_mode << ++ GEN8_CTX_ADDRESSING_MODE_SHIFT; ++ ++ if (!test_and_set_bit(workload->ring_id, s->shadow_ctx_desc_updated)) ++ shadow_context_descriptor_update(ce); ++ ++ ret = intel_gvt_scan_and_shadow_ringbuffer(workload); ++ if (ret) ++ goto err_unpin; ++ ++ if (workload->ring_id == RCS0 && workload->wa_ctx.indirect_ctx.size) { ++ ret = intel_gvt_scan_and_shadow_wa_ctx(&workload->wa_ctx); ++ if (ret) ++ goto err_shadow; ++ } ++ ++ workload->shadow = true; ++ return 0; ++err_shadow: ++ release_shadow_wa_ctx(&workload->wa_ctx); ++err_unpin: ++ intel_context_unpin(ce); ++ return ret; ++} ++ ++static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload); ++ ++static int prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload) ++{ ++ struct intel_gvt *gvt = workload->vgpu->gvt; ++ const int gmadr_bytes = gvt->device_info.gmadr_bytes_in_cmd; ++ struct intel_vgpu_shadow_bb *bb; ++ int ret; ++ ++ list_for_each_entry(bb, &workload->shadow_bb, list) { ++ /* For privilge batch buffer and not wa_ctx, the bb_start_cmd_va ++ * is only updated into ring_scan_buffer, not real ring address ++ * allocated in later copy_workload_to_ring_buffer. pls be noted ++ * shadow_ring_buffer_va is now pointed to real ring buffer va ++ * in copy_workload_to_ring_buffer. ++ */ ++ ++ if (bb->bb_offset) ++ bb->bb_start_cmd_va = workload->shadow_ring_buffer_va ++ + bb->bb_offset; ++ ++ if (bb->ppgtt) { ++ /* for non-priv bb, scan&shadow is only for ++ * debugging purpose, so the content of shadow bb ++ * is the same as original bb. Therefore, ++ * here, rather than switch to shadow bb's gma ++ * address, we directly use original batch buffer's ++ * gma address, and send original bb to hardware ++ * directly ++ */ ++ if (bb->clflush & CLFLUSH_AFTER) { ++ drm_clflush_virt_range(bb->va, ++ bb->obj->base.size); ++ bb->clflush &= ~CLFLUSH_AFTER; ++ } ++ i915_gem_obj_finish_shmem_access(bb->obj); ++ bb->accessing = false; ++ ++ } else { ++ bb->vma = i915_gem_object_ggtt_pin(bb->obj, ++ NULL, 0, 0, 0); ++ if (IS_ERR(bb->vma)) { ++ ret = PTR_ERR(bb->vma); ++ goto err; ++ } ++ ++ /* relocate shadow batch buffer */ ++ bb->bb_start_cmd_va[1] = i915_ggtt_offset(bb->vma); ++ if (gmadr_bytes == 8) ++ bb->bb_start_cmd_va[2] = 0; ++ ++ /* No one is going to touch shadow bb from now on. */ ++ if (bb->clflush & CLFLUSH_AFTER) { ++ drm_clflush_virt_range(bb->va, ++ bb->obj->base.size); ++ bb->clflush &= ~CLFLUSH_AFTER; ++ } ++ ++ ret = i915_gem_object_set_to_gtt_domain(bb->obj, ++ false); ++ if (ret) ++ goto err; ++ ++ i915_gem_obj_finish_shmem_access(bb->obj); ++ bb->accessing = false; ++ ++ ret = i915_vma_move_to_active(bb->vma, ++ workload->req, ++ 0); ++ if (ret) ++ goto err; ++ } ++ } ++ return 0; ++err: ++ release_shadow_batch_buffer(workload); ++ return ret; ++} ++ ++static void update_wa_ctx_2_shadow_ctx(struct intel_shadow_wa_ctx *wa_ctx) ++{ ++ struct intel_vgpu_workload *workload = ++ container_of(wa_ctx, struct intel_vgpu_workload, wa_ctx); ++ struct i915_request *rq = workload->req; ++ struct execlist_ring_context *shadow_ring_context = ++ (struct execlist_ring_context *)rq->hw_context->lrc_reg_state; ++ ++ shadow_ring_context->bb_per_ctx_ptr.val = ++ (shadow_ring_context->bb_per_ctx_ptr.val & ++ (~PER_CTX_ADDR_MASK)) | wa_ctx->per_ctx.shadow_gma; ++ shadow_ring_context->rcs_indirect_ctx.val = ++ (shadow_ring_context->rcs_indirect_ctx.val & ++ (~INDIRECT_CTX_ADDR_MASK)) | wa_ctx->indirect_ctx.shadow_gma; ++} ++ ++static int prepare_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) ++{ ++ struct i915_vma *vma; ++ unsigned char *per_ctx_va = ++ (unsigned char *)wa_ctx->indirect_ctx.shadow_va + ++ wa_ctx->indirect_ctx.size; ++ ++ if (wa_ctx->indirect_ctx.size == 0) ++ return 0; ++ ++ vma = i915_gem_object_ggtt_pin(wa_ctx->indirect_ctx.obj, NULL, ++ 0, CACHELINE_BYTES, 0); ++ if (IS_ERR(vma)) ++ return PTR_ERR(vma); ++ ++ /* FIXME: we are not tracking our pinned VMA leaving it ++ * up to the core to fix up the stray pin_count upon ++ * free. ++ */ ++ ++ wa_ctx->indirect_ctx.shadow_gma = i915_ggtt_offset(vma); ++ ++ wa_ctx->per_ctx.shadow_gma = *((unsigned int *)per_ctx_va + 1); ++ memset(per_ctx_va, 0, CACHELINE_BYTES); ++ ++ update_wa_ctx_2_shadow_ctx(wa_ctx); ++ return 0; ++} ++ ++static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload) ++{ ++ struct intel_vgpu *vgpu = workload->vgpu; ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ struct intel_vgpu_shadow_bb *bb, *pos; ++ ++ if (list_empty(&workload->shadow_bb)) ++ return; ++ ++ bb = list_first_entry(&workload->shadow_bb, ++ struct intel_vgpu_shadow_bb, list); ++ ++ mutex_lock(&dev_priv->drm.struct_mutex); ++ ++ list_for_each_entry_safe(bb, pos, &workload->shadow_bb, list) { ++ if (bb->obj) { ++ if (bb->accessing) ++ i915_gem_obj_finish_shmem_access(bb->obj); ++ ++ if (bb->va && !IS_ERR(bb->va)) ++ i915_gem_object_unpin_map(bb->obj); ++ ++ if (bb->vma && !IS_ERR(bb->vma)) { ++ i915_vma_unpin(bb->vma); ++ i915_vma_close(bb->vma); ++ } ++ __i915_gem_object_release_unless_active(bb->obj); ++ } ++ list_del(&bb->list); ++ kfree(bb); ++ } ++ ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++} ++ ++static int prepare_workload(struct intel_vgpu_workload *workload) ++{ ++ struct intel_vgpu *vgpu = workload->vgpu; ++ int ret = 0; ++ ++ ret = intel_vgpu_pin_mm(workload->shadow_mm); ++ if (ret) { ++ gvt_vgpu_err("fail to vgpu pin mm\n"); ++ return ret; ++ } ++ ++ update_shadow_pdps(workload); ++ ++ ret = intel_vgpu_sync_oos_pages(workload->vgpu); ++ if (ret) { ++ gvt_vgpu_err("fail to vgpu sync oos pages\n"); ++ goto err_unpin_mm; ++ } ++ ++ ret = intel_vgpu_flush_post_shadow(workload->vgpu); ++ if (ret) { ++ gvt_vgpu_err("fail to flush post shadow\n"); ++ goto err_unpin_mm; ++ } ++ ++ ret = copy_workload_to_ring_buffer(workload); ++ if (ret) { ++ gvt_vgpu_err("fail to generate request\n"); ++ goto err_unpin_mm; ++ } ++ ++ ret = prepare_shadow_batch_buffer(workload); ++ if (ret) { ++ gvt_vgpu_err("fail to prepare_shadow_batch_buffer\n"); ++ goto err_unpin_mm; ++ } ++ ++ ret = prepare_shadow_wa_ctx(&workload->wa_ctx); ++ if (ret) { ++ gvt_vgpu_err("fail to prepare_shadow_wa_ctx\n"); ++ goto err_shadow_batch; ++ } ++ ++ if (workload->prepare) { ++ ret = workload->prepare(workload); ++ if (ret) ++ goto err_shadow_wa_ctx; ++ } ++ ++ return 0; ++err_shadow_wa_ctx: ++ release_shadow_wa_ctx(&workload->wa_ctx); ++err_shadow_batch: ++ release_shadow_batch_buffer(workload); ++err_unpin_mm: ++ intel_vgpu_unpin_mm(workload->shadow_mm); ++ return ret; ++} ++ ++static int dispatch_workload(struct intel_vgpu_workload *workload) ++{ ++ struct intel_vgpu *vgpu = workload->vgpu; ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ struct intel_vgpu_submission *s = &vgpu->submission; ++ struct i915_gem_context *shadow_ctx = s->shadow_ctx; ++ struct i915_request *rq; ++ int ring_id = workload->ring_id; ++ int ret; ++ ++ gvt_dbg_sched("ring id %d prepare to dispatch workload %p\n", ++ ring_id, workload); ++ ++ mutex_lock(&vgpu->vgpu_lock); ++ mutex_lock(&dev_priv->drm.struct_mutex); ++ ++ ret = set_context_ppgtt_from_shadow(workload, shadow_ctx); ++ if (ret < 0) { ++ gvt_vgpu_err("workload shadow ppgtt isn't ready\n"); ++ goto err_req; ++ } ++ ++ ret = intel_gvt_workload_req_alloc(workload); ++ if (ret) ++ goto err_req; ++ ++ ret = intel_gvt_scan_and_shadow_workload(workload); ++ if (ret) ++ goto out; ++ ++ ret = populate_shadow_context(workload); ++ if (ret) { ++ release_shadow_wa_ctx(&workload->wa_ctx); ++ goto out; ++ } ++ ++ ret = prepare_workload(workload); ++out: ++ if (ret) { ++ /* We might still need to add request with ++ * clean ctx to retire it properly.. ++ */ ++ rq = fetch_and_zero(&workload->req); ++ i915_request_put(rq); ++ } ++ ++ if (!IS_ERR_OR_NULL(workload->req)) { ++ gvt_dbg_sched("ring id %d submit workload to i915 %p\n", ++ ring_id, workload->req); ++ i915_request_add(workload->req); ++ workload->dispatched = true; ++ } ++err_req: ++ if (ret) ++ workload->status = ret; ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++ mutex_unlock(&vgpu->vgpu_lock); ++ return ret; ++} ++ ++static struct intel_vgpu_workload *pick_next_workload( ++ struct intel_gvt *gvt, int ring_id) ++{ ++ struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; ++ struct intel_vgpu_workload *workload = NULL; ++ ++ mutex_lock(&gvt->sched_lock); ++ ++ /* ++ * no current vgpu / will be scheduled out / no workload ++ * bail out ++ */ ++ if (!scheduler->current_vgpu) { ++ gvt_dbg_sched("ring id %d stop - no current vgpu\n", ring_id); ++ goto out; ++ } ++ ++ if (scheduler->need_reschedule) { ++ gvt_dbg_sched("ring id %d stop - will reschedule\n", ring_id); ++ goto out; ++ } ++ ++ if (!scheduler->current_vgpu->active || ++ list_empty(workload_q_head(scheduler->current_vgpu, ring_id))) ++ goto out; ++ ++ /* ++ * still have current workload, maybe the workload disptacher ++ * fail to submit it for some reason, resubmit it. ++ */ ++ if (scheduler->current_workload[ring_id]) { ++ workload = scheduler->current_workload[ring_id]; ++ gvt_dbg_sched("ring id %d still have current workload %p\n", ++ ring_id, workload); ++ goto out; ++ } ++ ++ /* ++ * pick a workload as current workload ++ * once current workload is set, schedule policy routines ++ * will wait the current workload is finished when trying to ++ * schedule out a vgpu. ++ */ ++ scheduler->current_workload[ring_id] = container_of( ++ workload_q_head(scheduler->current_vgpu, ring_id)->next, ++ struct intel_vgpu_workload, list); ++ ++ workload = scheduler->current_workload[ring_id]; ++ ++ gvt_dbg_sched("ring id %d pick new workload %p\n", ring_id, workload); ++ ++ atomic_inc(&workload->vgpu->submission.running_workload_num); ++out: ++ mutex_unlock(&gvt->sched_lock); ++ return workload; ++} ++ ++static void update_guest_context(struct intel_vgpu_workload *workload) ++{ ++ struct i915_request *rq = workload->req; ++ struct intel_vgpu *vgpu = workload->vgpu; ++ struct intel_gvt *gvt = vgpu->gvt; ++ struct drm_i915_gem_object *ctx_obj = rq->hw_context->state->obj; ++ struct execlist_ring_context *shadow_ring_context; ++ struct page *page; ++ void *src; ++ unsigned long context_gpa, context_page_num; ++ int i; ++ struct drm_i915_private *dev_priv = gvt->dev_priv; ++ u32 ring_base; ++ u32 head, tail; ++ u16 wrap_count; ++ ++ gvt_dbg_sched("ring id %d workload lrca %x\n", rq->engine->id, ++ workload->ctx_desc.lrca); ++ ++ head = workload->rb_head; ++ tail = workload->rb_tail; ++ wrap_count = workload->guest_rb_head >> RB_HEAD_WRAP_CNT_OFF; ++ ++ if (tail < head) { ++ if (wrap_count == RB_HEAD_WRAP_CNT_MAX) ++ wrap_count = 0; ++ else ++ wrap_count += 1; ++ } ++ ++ head = (wrap_count << RB_HEAD_WRAP_CNT_OFF) | tail; ++ ++ ring_base = dev_priv->engine[workload->ring_id]->mmio_base; ++ vgpu_vreg_t(vgpu, RING_TAIL(ring_base)) = tail; ++ vgpu_vreg_t(vgpu, RING_HEAD(ring_base)) = head; ++ ++ context_page_num = rq->engine->context_size; ++ context_page_num = context_page_num >> PAGE_SHIFT; ++ ++ if (IS_BROADWELL(gvt->dev_priv) && rq->engine->id == RCS0) ++ context_page_num = 19; ++ ++ i = 2; ++ ++ while (i < context_page_num) { ++ context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, ++ (u32)((workload->ctx_desc.lrca + i) << ++ I915_GTT_PAGE_SHIFT)); ++ if (context_gpa == INTEL_GVT_INVALID_ADDR) { ++ gvt_vgpu_err("invalid guest context descriptor\n"); ++ return; ++ } ++ ++ page = i915_gem_object_get_page(ctx_obj, LRC_HEADER_PAGES + i); ++ src = kmap(page); ++ intel_gvt_hypervisor_write_gpa(vgpu, context_gpa, src, ++ I915_GTT_PAGE_SIZE); ++ kunmap(page); ++ i++; ++ } ++ ++ intel_gvt_hypervisor_write_gpa(vgpu, workload->ring_context_gpa + ++ RING_CTX_OFF(ring_header.val), &workload->rb_tail, 4); ++ ++ page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN); ++ shadow_ring_context = kmap(page); ++ ++#define COPY_REG(name) \ ++ intel_gvt_hypervisor_write_gpa(vgpu, workload->ring_context_gpa + \ ++ RING_CTX_OFF(name.val), &shadow_ring_context->name.val, 4) ++ ++ COPY_REG(ctx_ctrl); ++ COPY_REG(ctx_timestamp); ++ ++#undef COPY_REG ++ ++ intel_gvt_hypervisor_write_gpa(vgpu, ++ workload->ring_context_gpa + ++ sizeof(*shadow_ring_context), ++ (void *)shadow_ring_context + ++ sizeof(*shadow_ring_context), ++ I915_GTT_PAGE_SIZE - sizeof(*shadow_ring_context)); ++ ++ kunmap(page); ++} ++ ++void intel_vgpu_clean_workloads(struct intel_vgpu *vgpu, ++ intel_engine_mask_t engine_mask) ++{ ++ struct intel_vgpu_submission *s = &vgpu->submission; ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ struct intel_engine_cs *engine; ++ struct intel_vgpu_workload *pos, *n; ++ intel_engine_mask_t tmp; ++ ++ /* free the unsubmited workloads in the queues. */ ++ for_each_engine_masked(engine, dev_priv, engine_mask, tmp) { ++ list_for_each_entry_safe(pos, n, ++ &s->workload_q_head[engine->id], list) { ++ list_del_init(&pos->list); ++ intel_vgpu_destroy_workload(pos); ++ } ++ clear_bit(engine->id, s->shadow_ctx_desc_updated); ++ } ++} ++ ++static void complete_current_workload(struct intel_gvt *gvt, int ring_id) ++{ ++ struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; ++ struct intel_vgpu_workload *workload = ++ scheduler->current_workload[ring_id]; ++ struct intel_vgpu *vgpu = workload->vgpu; ++ struct intel_vgpu_submission *s = &vgpu->submission; ++ struct i915_request *rq = workload->req; ++ int event; ++ ++ mutex_lock(&vgpu->vgpu_lock); ++ mutex_lock(&gvt->sched_lock); ++ ++ /* For the workload w/ request, needs to wait for the context ++ * switch to make sure request is completed. ++ * For the workload w/o request, directly complete the workload. ++ */ ++ if (rq) { ++ wait_event(workload->shadow_ctx_status_wq, ++ !atomic_read(&workload->shadow_ctx_active)); ++ ++ /* If this request caused GPU hang, req->fence.error will ++ * be set to -EIO. Use -EIO to set workload status so ++ * that when this request caused GPU hang, didn't trigger ++ * context switch interrupt to guest. ++ */ ++ if (likely(workload->status == -EINPROGRESS)) { ++ if (workload->req->fence.error == -EIO) ++ workload->status = -EIO; ++ else ++ workload->status = 0; ++ } ++ ++ if (!workload->status && ++ !(vgpu->resetting_eng & BIT(ring_id))) { ++ update_guest_context(workload); ++ ++ for_each_set_bit(event, workload->pending_events, ++ INTEL_GVT_EVENT_MAX) ++ intel_vgpu_trigger_virtual_event(vgpu, event); ++ } ++ ++ /* unpin shadow ctx as the shadow_ctx update is done */ ++ mutex_lock(&rq->i915->drm.struct_mutex); ++ intel_context_unpin(rq->hw_context); ++ mutex_unlock(&rq->i915->drm.struct_mutex); ++ ++ i915_request_put(fetch_and_zero(&workload->req)); ++ } ++ ++ gvt_dbg_sched("ring id %d complete workload %p status %d\n", ++ ring_id, workload, workload->status); ++ ++ scheduler->current_workload[ring_id] = NULL; ++ ++ list_del_init(&workload->list); ++ ++ if (workload->status || vgpu->resetting_eng & BIT(ring_id)) { ++ /* if workload->status is not successful means HW GPU ++ * has occurred GPU hang or something wrong with i915/GVT, ++ * and GVT won't inject context switch interrupt to guest. ++ * So this error is a vGPU hang actually to the guest. ++ * According to this we should emunlate a vGPU hang. If ++ * there are pending workloads which are already submitted ++ * from guest, we should clean them up like HW GPU does. ++ * ++ * if it is in middle of engine resetting, the pending ++ * workloads won't be submitted to HW GPU and will be ++ * cleaned up during the resetting process later, so doing ++ * the workload clean up here doesn't have any impact. ++ **/ ++ intel_vgpu_clean_workloads(vgpu, BIT(ring_id)); ++ } ++ ++ workload->complete(workload); ++ ++ atomic_dec(&s->running_workload_num); ++ wake_up(&scheduler->workload_complete_wq); ++ ++ if (gvt->scheduler.need_reschedule) ++ intel_gvt_request_service(gvt, INTEL_GVT_REQUEST_EVENT_SCHED); ++ ++ mutex_unlock(&gvt->sched_lock); ++ mutex_unlock(&vgpu->vgpu_lock); ++} ++ ++struct workload_thread_param { ++ struct intel_gvt *gvt; ++ int ring_id; ++}; ++ ++static int workload_thread(void *priv) ++{ ++ struct workload_thread_param *p = (struct workload_thread_param *)priv; ++ struct intel_gvt *gvt = p->gvt; ++ int ring_id = p->ring_id; ++ struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; ++ struct intel_vgpu_workload *workload = NULL; ++ struct intel_vgpu *vgpu = NULL; ++ int ret; ++ bool need_force_wake = (INTEL_GEN(gvt->dev_priv) >= 9); ++ DEFINE_WAIT_FUNC(wait, woken_wake_function); ++ ++ kfree(p); ++ ++ gvt_dbg_core("workload thread for ring %d started\n", ring_id); ++ ++ while (!kthread_should_stop()) { ++ add_wait_queue(&scheduler->waitq[ring_id], &wait); ++ do { ++ workload = pick_next_workload(gvt, ring_id); ++ if (workload) ++ break; ++ wait_woken(&wait, TASK_INTERRUPTIBLE, ++ MAX_SCHEDULE_TIMEOUT); ++ } while (!kthread_should_stop()); ++ remove_wait_queue(&scheduler->waitq[ring_id], &wait); ++ ++ if (!workload) ++ break; ++ ++ gvt_dbg_sched("ring id %d next workload %p vgpu %d\n", ++ workload->ring_id, workload, ++ workload->vgpu->id); ++ ++ intel_runtime_pm_get(gvt->dev_priv); ++ ++ gvt_dbg_sched("ring id %d will dispatch workload %p\n", ++ workload->ring_id, workload); ++ ++ if (need_force_wake) ++ intel_uncore_forcewake_get(&gvt->dev_priv->uncore, ++ FORCEWAKE_ALL); ++ ++ ret = dispatch_workload(workload); ++ ++ if (ret) { ++ vgpu = workload->vgpu; ++ gvt_vgpu_err("fail to dispatch workload, skip\n"); ++ goto complete; ++ } ++ ++ gvt_dbg_sched("ring id %d wait workload %p\n", ++ workload->ring_id, workload); ++ i915_request_wait(workload->req, 0, MAX_SCHEDULE_TIMEOUT); ++ ++complete: ++ gvt_dbg_sched("will complete workload %p, status: %d\n", ++ workload, workload->status); ++ ++ complete_current_workload(gvt, ring_id); ++ ++ if (need_force_wake) ++ intel_uncore_forcewake_put(&gvt->dev_priv->uncore, ++ FORCEWAKE_ALL); ++ ++ intel_runtime_pm_put_unchecked(gvt->dev_priv); ++ if (ret && (vgpu_is_vm_unhealthy(ret))) ++ enter_failsafe_mode(vgpu, GVT_FAILSAFE_GUEST_ERR); ++ } ++ return 0; ++} ++ ++void intel_gvt_wait_vgpu_idle(struct intel_vgpu *vgpu) ++{ ++ struct intel_vgpu_submission *s = &vgpu->submission; ++ struct intel_gvt *gvt = vgpu->gvt; ++ struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; ++ ++ if (atomic_read(&s->running_workload_num)) { ++ gvt_dbg_sched("wait vgpu idle\n"); ++ ++ wait_event(scheduler->workload_complete_wq, ++ !atomic_read(&s->running_workload_num)); ++ } ++} ++ ++void intel_gvt_clean_workload_scheduler(struct intel_gvt *gvt) ++{ ++ struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; ++ struct intel_engine_cs *engine; ++ enum intel_engine_id i; ++ ++ gvt_dbg_core("clean workload scheduler\n"); ++ ++ for_each_engine(engine, gvt->dev_priv, i) { ++ atomic_notifier_chain_unregister( ++ &engine->context_status_notifier, ++ &gvt->shadow_ctx_notifier_block[i]); ++ kthread_stop(scheduler->thread[i]); ++ } ++} ++ ++int intel_gvt_init_workload_scheduler(struct intel_gvt *gvt) ++{ ++ struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; ++ struct workload_thread_param *param = NULL; ++ struct intel_engine_cs *engine; ++ enum intel_engine_id i; ++ int ret; ++ ++ gvt_dbg_core("init workload scheduler\n"); ++ ++ init_waitqueue_head(&scheduler->workload_complete_wq); ++ ++ for_each_engine(engine, gvt->dev_priv, i) { ++ init_waitqueue_head(&scheduler->waitq[i]); ++ ++ param = kzalloc(sizeof(*param), GFP_KERNEL); ++ if (!param) { ++ ret = -ENOMEM; ++ goto err; ++ } ++ ++ param->gvt = gvt; ++ param->ring_id = i; ++ ++ scheduler->thread[i] = kthread_run(workload_thread, param, ++ "gvt workload %d", i); ++ if (IS_ERR(scheduler->thread[i])) { ++ gvt_err("fail to create workload thread\n"); ++ ret = PTR_ERR(scheduler->thread[i]); ++ goto err; ++ } ++ ++ gvt->shadow_ctx_notifier_block[i].notifier_call = ++ shadow_context_status_change; ++ atomic_notifier_chain_register(&engine->context_status_notifier, ++ &gvt->shadow_ctx_notifier_block[i]); ++ } ++ return 0; ++err: ++ intel_gvt_clean_workload_scheduler(gvt); ++ kfree(param); ++ param = NULL; ++ return ret; ++} ++ ++static void ++i915_context_ppgtt_root_restore(struct intel_vgpu_submission *s) ++{ ++ struct i915_hw_ppgtt *i915_ppgtt = s->shadow_ctx->ppgtt; ++ int i; ++ ++ if (i915_vm_is_4lvl(&i915_ppgtt->vm)) { ++ px_dma(&i915_ppgtt->pml4) = s->i915_context_pml4; ++ } else { ++ for (i = 0; i < GEN8_3LVL_PDPES; i++) ++ px_dma(i915_ppgtt->pdp.page_directory[i]) = ++ s->i915_context_pdps[i]; ++ } ++} ++ ++/** ++ * intel_vgpu_clean_submission - free submission-related resource for vGPU ++ * @vgpu: a vGPU ++ * ++ * This function is called when a vGPU is being destroyed. ++ * ++ */ ++void intel_vgpu_clean_submission(struct intel_vgpu *vgpu) ++{ ++ struct intel_vgpu_submission *s = &vgpu->submission; ++ ++ intel_vgpu_select_submission_ops(vgpu, ALL_ENGINES, 0); ++ i915_context_ppgtt_root_restore(s); ++ i915_gem_context_put(s->shadow_ctx); ++ kmem_cache_destroy(s->workloads); ++} ++ ++ ++/** ++ * intel_vgpu_reset_submission - reset submission-related resource for vGPU ++ * @vgpu: a vGPU ++ * @engine_mask: engines expected to be reset ++ * ++ * This function is called when a vGPU is being destroyed. ++ * ++ */ ++void intel_vgpu_reset_submission(struct intel_vgpu *vgpu, ++ intel_engine_mask_t engine_mask) ++{ ++ struct intel_vgpu_submission *s = &vgpu->submission; ++ ++ if (!s->active) ++ return; ++ ++ intel_vgpu_clean_workloads(vgpu, engine_mask); ++ s->ops->reset(vgpu, engine_mask); ++} ++ ++static void ++i915_context_ppgtt_root_save(struct intel_vgpu_submission *s) ++{ ++ struct i915_hw_ppgtt *i915_ppgtt = s->shadow_ctx->ppgtt; ++ int i; ++ ++ if (i915_vm_is_4lvl(&i915_ppgtt->vm)) ++ s->i915_context_pml4 = px_dma(&i915_ppgtt->pml4); ++ else { ++ for (i = 0; i < GEN8_3LVL_PDPES; i++) ++ s->i915_context_pdps[i] = ++ px_dma(i915_ppgtt->pdp.page_directory[i]); ++ } ++} ++ ++/** ++ * intel_vgpu_setup_submission - setup submission-related resource for vGPU ++ * @vgpu: a vGPU ++ * ++ * This function is called when a vGPU is being created. ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ * ++ */ ++int intel_vgpu_setup_submission(struct intel_vgpu *vgpu) ++{ ++ struct intel_vgpu_submission *s = &vgpu->submission; ++ enum intel_engine_id i; ++ struct intel_engine_cs *engine; ++ int ret; ++ ++ s->shadow_ctx = i915_gem_context_create_gvt( ++ &vgpu->gvt->dev_priv->drm); ++ if (IS_ERR(s->shadow_ctx)) ++ return PTR_ERR(s->shadow_ctx); ++ ++ i915_context_ppgtt_root_save(s); ++ ++ bitmap_zero(s->shadow_ctx_desc_updated, I915_NUM_ENGINES); ++ ++ s->workloads = kmem_cache_create_usercopy("gvt-g_vgpu_workload", ++ sizeof(struct intel_vgpu_workload), 0, ++ SLAB_HWCACHE_ALIGN, ++ offsetof(struct intel_vgpu_workload, rb_tail), ++ sizeof_field(struct intel_vgpu_workload, rb_tail), ++ NULL); ++ ++ if (!s->workloads) { ++ ret = -ENOMEM; ++ goto out_shadow_ctx; ++ } ++ ++ for_each_engine(engine, vgpu->gvt->dev_priv, i) ++ INIT_LIST_HEAD(&s->workload_q_head[i]); ++ ++ atomic_set(&s->running_workload_num, 0); ++ bitmap_zero(s->tlb_handle_pending, I915_NUM_ENGINES); ++ ++ return 0; ++ ++out_shadow_ctx: ++ i915_gem_context_put(s->shadow_ctx); ++ return ret; ++} ++ ++/** ++ * intel_vgpu_select_submission_ops - select virtual submission interface ++ * @vgpu: a vGPU ++ * @engine_mask: either ALL_ENGINES or target engine mask ++ * @interface: expected vGPU virtual submission interface ++ * ++ * This function is called when guest configures submission interface. ++ * ++ * Returns: ++ * Zero on success, negative error code if failed. ++ * ++ */ ++int intel_vgpu_select_submission_ops(struct intel_vgpu *vgpu, ++ intel_engine_mask_t engine_mask, ++ unsigned int interface) ++{ ++ struct intel_vgpu_submission *s = &vgpu->submission; ++ const struct intel_vgpu_submission_ops *ops[] = { ++ [INTEL_VGPU_EXECLIST_SUBMISSION] = ++ &intel_vgpu_execlist_submission_ops, ++ }; ++ int ret; ++ ++ if (WARN_ON(interface >= ARRAY_SIZE(ops))) ++ return -EINVAL; ++ ++ if (WARN_ON(interface == 0 && engine_mask != ALL_ENGINES)) ++ return -EINVAL; ++ ++ if (s->active) ++ s->ops->clean(vgpu, engine_mask); ++ ++ if (interface == 0) { ++ s->ops = NULL; ++ s->virtual_submission_interface = 0; ++ s->active = false; ++ gvt_dbg_core("vgpu%d: remove submission ops\n", vgpu->id); ++ return 0; ++ } ++ ++ ret = ops[interface]->init(vgpu, engine_mask); ++ if (ret) ++ return ret; ++ ++ s->ops = ops[interface]; ++ s->virtual_submission_interface = interface; ++ s->active = true; ++ ++ gvt_dbg_core("vgpu%d: activate ops [ %s ]\n", ++ vgpu->id, s->ops->name); ++ ++ return 0; ++} ++ ++/** ++ * intel_vgpu_destroy_workload - destroy a vGPU workload ++ * @workload: workload to destroy ++ * ++ * This function is called when destroy a vGPU workload. ++ * ++ */ ++void intel_vgpu_destroy_workload(struct intel_vgpu_workload *workload) ++{ ++ struct intel_vgpu_submission *s = &workload->vgpu->submission; ++ ++ release_shadow_batch_buffer(workload); ++ release_shadow_wa_ctx(&workload->wa_ctx); ++ ++ if (workload->shadow_mm) ++ intel_vgpu_mm_put(workload->shadow_mm); ++ ++ kmem_cache_free(s->workloads, workload); ++} ++ ++static struct intel_vgpu_workload * ++alloc_workload(struct intel_vgpu *vgpu) ++{ ++ struct intel_vgpu_submission *s = &vgpu->submission; ++ struct intel_vgpu_workload *workload; ++ ++ workload = kmem_cache_zalloc(s->workloads, GFP_KERNEL); ++ if (!workload) ++ return ERR_PTR(-ENOMEM); ++ ++ INIT_LIST_HEAD(&workload->list); ++ INIT_LIST_HEAD(&workload->shadow_bb); ++ ++ init_waitqueue_head(&workload->shadow_ctx_status_wq); ++ atomic_set(&workload->shadow_ctx_active, 0); ++ ++ workload->status = -EINPROGRESS; ++ workload->vgpu = vgpu; ++ ++ return workload; ++} ++ ++#define RING_CTX_OFF(x) \ ++ offsetof(struct execlist_ring_context, x) ++ ++static void read_guest_pdps(struct intel_vgpu *vgpu, ++ u64 ring_context_gpa, u32 pdp[8]) ++{ ++ u64 gpa; ++ int i; ++ ++ gpa = ring_context_gpa + RING_CTX_OFF(pdps[0].val); ++ ++ for (i = 0; i < 8; i++) ++ intel_gvt_hypervisor_read_gpa(vgpu, ++ gpa + i * 8, &pdp[7 - i], 4); ++} ++ ++static int prepare_mm(struct intel_vgpu_workload *workload) ++{ ++ struct execlist_ctx_descriptor_format *desc = &workload->ctx_desc; ++ struct intel_vgpu_mm *mm; ++ struct intel_vgpu *vgpu = workload->vgpu; ++ enum intel_gvt_gtt_type root_entry_type; ++ u64 pdps[GVT_RING_CTX_NR_PDPS]; ++ ++ switch (desc->addressing_mode) { ++ case 1: /* legacy 32-bit */ ++ root_entry_type = GTT_TYPE_PPGTT_ROOT_L3_ENTRY; ++ break; ++ case 3: /* legacy 64-bit */ ++ root_entry_type = GTT_TYPE_PPGTT_ROOT_L4_ENTRY; ++ break; ++ default: ++ gvt_vgpu_err("Advanced Context mode(SVM) is not supported!\n"); ++ return -EINVAL; ++ } ++ ++ read_guest_pdps(workload->vgpu, workload->ring_context_gpa, (void *)pdps); ++ ++ mm = intel_vgpu_get_ppgtt_mm(workload->vgpu, root_entry_type, pdps); ++ if (IS_ERR(mm)) ++ return PTR_ERR(mm); ++ ++ workload->shadow_mm = mm; ++ return 0; ++} ++ ++#define same_context(a, b) (((a)->context_id == (b)->context_id) && \ ++ ((a)->lrca == (b)->lrca)) ++ ++#define get_last_workload(q) \ ++ (list_empty(q) ? NULL : container_of(q->prev, \ ++ struct intel_vgpu_workload, list)) ++/** ++ * intel_vgpu_create_workload - create a vGPU workload ++ * @vgpu: a vGPU ++ * @ring_id: ring index ++ * @desc: a guest context descriptor ++ * ++ * This function is called when creating a vGPU workload. ++ * ++ * Returns: ++ * struct intel_vgpu_workload * on success, negative error code in ++ * pointer if failed. ++ * ++ */ ++struct intel_vgpu_workload * ++intel_vgpu_create_workload(struct intel_vgpu *vgpu, int ring_id, ++ struct execlist_ctx_descriptor_format *desc) ++{ ++ struct intel_vgpu_submission *s = &vgpu->submission; ++ struct list_head *q = workload_q_head(vgpu, ring_id); ++ struct intel_vgpu_workload *last_workload = get_last_workload(q); ++ struct intel_vgpu_workload *workload = NULL; ++ struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ++ u64 ring_context_gpa; ++ u32 head, tail, start, ctl, ctx_ctl, per_ctx, indirect_ctx; ++ u32 guest_head; ++ int ret; ++ ++ ring_context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, ++ (u32)((desc->lrca + 1) << I915_GTT_PAGE_SHIFT)); ++ if (ring_context_gpa == INTEL_GVT_INVALID_ADDR) { ++ gvt_vgpu_err("invalid guest context LRCA: %x\n", desc->lrca); ++ return ERR_PTR(-EINVAL); ++ } ++ ++ intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + ++ RING_CTX_OFF(ring_header.val), &head, 4); ++ ++ intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + ++ RING_CTX_OFF(ring_tail.val), &tail, 4); ++ ++ guest_head = head; ++ ++ head &= RB_HEAD_OFF_MASK; ++ tail &= RB_TAIL_OFF_MASK; ++ ++ if (last_workload && same_context(&last_workload->ctx_desc, desc)) { ++ gvt_dbg_el("ring id %d cur workload == last\n", ring_id); ++ gvt_dbg_el("ctx head %x real head %lx\n", head, ++ last_workload->rb_tail); ++ /* ++ * cannot use guest context head pointer here, ++ * as it might not be updated at this time ++ */ ++ head = last_workload->rb_tail; ++ } ++ ++ gvt_dbg_el("ring id %d begin a new workload\n", ring_id); ++ ++ /* record some ring buffer register values for scan and shadow */ ++ intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + ++ RING_CTX_OFF(rb_start.val), &start, 4); ++ intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + ++ RING_CTX_OFF(rb_ctrl.val), &ctl, 4); ++ intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + ++ RING_CTX_OFF(ctx_ctrl.val), &ctx_ctl, 4); ++ ++ workload = alloc_workload(vgpu); ++ if (IS_ERR(workload)) ++ return workload; ++ ++ workload->ring_id = ring_id; ++ workload->ctx_desc = *desc; ++ workload->ring_context_gpa = ring_context_gpa; ++ workload->rb_head = head; ++ workload->guest_rb_head = guest_head; ++ workload->rb_tail = tail; ++ workload->rb_start = start; ++ workload->rb_ctl = ctl; ++ ++ if (ring_id == RCS0) { ++ intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + ++ RING_CTX_OFF(bb_per_ctx_ptr.val), &per_ctx, 4); ++ intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + ++ RING_CTX_OFF(rcs_indirect_ctx.val), &indirect_ctx, 4); ++ ++ workload->wa_ctx.indirect_ctx.guest_gma = ++ indirect_ctx & INDIRECT_CTX_ADDR_MASK; ++ workload->wa_ctx.indirect_ctx.size = ++ (indirect_ctx & INDIRECT_CTX_SIZE_MASK) * ++ CACHELINE_BYTES; ++ workload->wa_ctx.per_ctx.guest_gma = ++ per_ctx & PER_CTX_ADDR_MASK; ++ workload->wa_ctx.per_ctx.valid = per_ctx & 1; ++ } ++ ++ gvt_dbg_el("workload %p ring id %d head %x tail %x start %x ctl %x\n", ++ workload, ring_id, head, tail, start, ctl); ++ ++ ret = prepare_mm(workload); ++ if (ret) { ++ kmem_cache_free(s->workloads, workload); ++ return ERR_PTR(ret); ++ } ++ ++ /* Only scan and shadow the first workload in the queue ++ * as there is only one pre-allocated buf-obj for shadow. ++ */ ++ if (list_empty(workload_q_head(vgpu, ring_id))) { ++ intel_runtime_pm_get(dev_priv); ++ mutex_lock(&dev_priv->drm.struct_mutex); ++ ret = intel_gvt_scan_and_shadow_workload(workload); ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++ intel_runtime_pm_put_unchecked(dev_priv); ++ } ++ ++ if (ret) { ++ if (vgpu_is_vm_unhealthy(ret)) ++ enter_failsafe_mode(vgpu, GVT_FAILSAFE_GUEST_ERR); ++ intel_vgpu_destroy_workload(workload); ++ return ERR_PTR(ret); ++ } ++ ++ return workload; ++} ++ ++/** ++ * intel_vgpu_queue_workload - Qeue a vGPU workload ++ * @workload: the workload to queue in ++ */ ++void intel_vgpu_queue_workload(struct intel_vgpu_workload *workload) ++{ ++ list_add_tail(&workload->list, ++ workload_q_head(workload->vgpu, workload->ring_id)); ++ intel_gvt_kick_schedule(workload->vgpu->gvt); ++ wake_up(&workload->vgpu->gvt->scheduler.waitq[workload->ring_id]); ++} +diff --git a/drivers/gpu/drm/i915_legacy/gvt/scheduler.h b/drivers/gpu/drm/i915_legacy/gvt/scheduler.h +new file mode 100644 +index 000000000000..c50d14a9ce85 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/scheduler.h +@@ -0,0 +1,166 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Zhi Wang ++ * ++ * Contributors: ++ * Ping Gao ++ * Tina Zhang ++ * Chanbin Du ++ * Min He ++ * Bing Niu ++ * Zhenyu Wang ++ * ++ */ ++ ++#ifndef _GVT_SCHEDULER_H_ ++#define _GVT_SCHEDULER_H_ ++ ++struct intel_gvt_workload_scheduler { ++ struct intel_vgpu *current_vgpu; ++ struct intel_vgpu *next_vgpu; ++ struct intel_vgpu_workload *current_workload[I915_NUM_ENGINES]; ++ bool need_reschedule; ++ ++ spinlock_t mmio_context_lock; ++ /* can be null when owner is host */ ++ struct intel_vgpu *engine_owner[I915_NUM_ENGINES]; ++ ++ wait_queue_head_t workload_complete_wq; ++ struct task_struct *thread[I915_NUM_ENGINES]; ++ wait_queue_head_t waitq[I915_NUM_ENGINES]; ++ ++ void *sched_data; ++ struct intel_gvt_sched_policy_ops *sched_ops; ++}; ++ ++#define INDIRECT_CTX_ADDR_MASK 0xffffffc0 ++#define INDIRECT_CTX_SIZE_MASK 0x3f ++struct shadow_indirect_ctx { ++ struct drm_i915_gem_object *obj; ++ unsigned long guest_gma; ++ unsigned long shadow_gma; ++ void *shadow_va; ++ u32 size; ++}; ++ ++#define PER_CTX_ADDR_MASK 0xfffff000 ++struct shadow_per_ctx { ++ unsigned long guest_gma; ++ unsigned long shadow_gma; ++ unsigned valid; ++}; ++ ++struct intel_shadow_wa_ctx { ++ struct shadow_indirect_ctx indirect_ctx; ++ struct shadow_per_ctx per_ctx; ++ ++}; ++ ++struct intel_vgpu_workload { ++ struct intel_vgpu *vgpu; ++ int ring_id; ++ struct i915_request *req; ++ /* if this workload has been dispatched to i915? */ ++ bool dispatched; ++ bool shadow; /* if workload has done shadow of guest request */ ++ int status; ++ ++ struct intel_vgpu_mm *shadow_mm; ++ ++ /* different submission model may need different handler */ ++ int (*prepare)(struct intel_vgpu_workload *); ++ int (*complete)(struct intel_vgpu_workload *); ++ struct list_head list; ++ ++ DECLARE_BITMAP(pending_events, INTEL_GVT_EVENT_MAX); ++ void *shadow_ring_buffer_va; ++ ++ /* execlist context information */ ++ struct execlist_ctx_descriptor_format ctx_desc; ++ struct execlist_ring_context *ring_context; ++ unsigned long rb_head, rb_tail, rb_ctl, rb_start, rb_len; ++ unsigned long guest_rb_head; ++ bool restore_inhibit; ++ struct intel_vgpu_elsp_dwords elsp_dwords; ++ bool emulate_schedule_in; ++ atomic_t shadow_ctx_active; ++ wait_queue_head_t shadow_ctx_status_wq; ++ u64 ring_context_gpa; ++ ++ /* shadow batch buffer */ ++ struct list_head shadow_bb; ++ struct intel_shadow_wa_ctx wa_ctx; ++ ++ /* oa registers */ ++ u32 oactxctrl; ++ u32 flex_mmio[7]; ++}; ++ ++struct intel_vgpu_shadow_bb { ++ struct list_head list; ++ struct drm_i915_gem_object *obj; ++ struct i915_vma *vma; ++ void *va; ++ u32 *bb_start_cmd_va; ++ unsigned int clflush; ++ bool accessing; ++ unsigned long bb_offset; ++ bool ppgtt; ++}; ++ ++#define workload_q_head(vgpu, ring_id) \ ++ (&(vgpu->submission.workload_q_head[ring_id])) ++ ++void intel_vgpu_queue_workload(struct intel_vgpu_workload *workload); ++ ++int intel_gvt_init_workload_scheduler(struct intel_gvt *gvt); ++ ++void intel_gvt_clean_workload_scheduler(struct intel_gvt *gvt); ++ ++void intel_gvt_wait_vgpu_idle(struct intel_vgpu *vgpu); ++ ++int intel_vgpu_setup_submission(struct intel_vgpu *vgpu); ++ ++void intel_vgpu_reset_submission(struct intel_vgpu *vgpu, ++ intel_engine_mask_t engine_mask); ++ ++void intel_vgpu_clean_submission(struct intel_vgpu *vgpu); ++ ++int intel_vgpu_select_submission_ops(struct intel_vgpu *vgpu, ++ intel_engine_mask_t engine_mask, ++ unsigned int interface); ++ ++extern const struct intel_vgpu_submission_ops ++intel_vgpu_execlist_submission_ops; ++ ++struct intel_vgpu_workload * ++intel_vgpu_create_workload(struct intel_vgpu *vgpu, int ring_id, ++ struct execlist_ctx_descriptor_format *desc); ++ ++void intel_vgpu_destroy_workload(struct intel_vgpu_workload *workload); ++ ++void intel_vgpu_clean_workloads(struct intel_vgpu *vgpu, ++ intel_engine_mask_t engine_mask); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/gvt/trace.h b/drivers/gpu/drm/i915_legacy/gvt/trace.h +new file mode 100644 +index 000000000000..6d787750d279 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/trace.h +@@ -0,0 +1,383 @@ ++/* ++ * Copyright © 2011-2016 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ * Authors: ++ * Jike Song ++ * ++ * Contributors: ++ * Zhi Wang ++ * ++ */ ++ ++#if !defined(_GVT_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ) ++#define _GVT_TRACE_H_ ++ ++#include ++#include ++#include ++#include ++ ++#undef TRACE_SYSTEM ++#define TRACE_SYSTEM gvt ++ ++TRACE_EVENT(spt_alloc, ++ TP_PROTO(int id, void *spt, int type, unsigned long mfn, ++ unsigned long gpt_gfn), ++ ++ TP_ARGS(id, spt, type, mfn, gpt_gfn), ++ ++ TP_STRUCT__entry( ++ __field(int, id) ++ __field(void *, spt) ++ __field(int, type) ++ __field(unsigned long, mfn) ++ __field(unsigned long, gpt_gfn) ++ ), ++ ++ TP_fast_assign( ++ __entry->id = id; ++ __entry->spt = spt; ++ __entry->type = type; ++ __entry->mfn = mfn; ++ __entry->gpt_gfn = gpt_gfn; ++ ), ++ ++ TP_printk("VM%d [alloc] spt %p type %d mfn 0x%lx gfn 0x%lx\n", ++ __entry->id, ++ __entry->spt, ++ __entry->type, ++ __entry->mfn, ++ __entry->gpt_gfn) ++); ++ ++TRACE_EVENT(spt_free, ++ TP_PROTO(int id, void *spt, int type), ++ ++ TP_ARGS(id, spt, type), ++ ++ TP_STRUCT__entry( ++ __field(int, id) ++ __field(void *, spt) ++ __field(int, type) ++ ), ++ ++ TP_fast_assign( ++ __entry->id = id; ++ __entry->spt = spt; ++ __entry->type = type; ++ ), ++ ++ TP_printk("VM%u [free] spt %p type %d\n", ++ __entry->id, ++ __entry->spt, ++ __entry->type) ++); ++ ++#define MAX_BUF_LEN 256 ++ ++TRACE_EVENT(gma_index, ++ TP_PROTO(const char *prefix, unsigned long gma, ++ unsigned long index), ++ ++ TP_ARGS(prefix, gma, index), ++ ++ TP_STRUCT__entry( ++ __array(char, buf, MAX_BUF_LEN) ++ ), ++ ++ TP_fast_assign( ++ snprintf(__entry->buf, MAX_BUF_LEN, ++ "%s gma 0x%lx index 0x%lx\n", prefix, gma, index); ++ ), ++ ++ TP_printk("%s", __entry->buf) ++); ++ ++TRACE_EVENT(gma_translate, ++ TP_PROTO(int id, char *type, int ring_id, int root_entry_type, ++ unsigned long gma, unsigned long gpa), ++ ++ TP_ARGS(id, type, ring_id, root_entry_type, gma, gpa), ++ ++ TP_STRUCT__entry( ++ __array(char, buf, MAX_BUF_LEN) ++ ), ++ ++ TP_fast_assign( ++ snprintf(__entry->buf, MAX_BUF_LEN, ++ "VM%d %s ring %d root_entry_type %d gma 0x%lx -> gpa 0x%lx\n", ++ id, type, ring_id, root_entry_type, gma, gpa); ++ ), ++ ++ TP_printk("%s", __entry->buf) ++); ++ ++TRACE_EVENT(spt_refcount, ++ TP_PROTO(int id, char *action, void *spt, int before, int after), ++ ++ TP_ARGS(id, action, spt, before, after), ++ ++ TP_STRUCT__entry( ++ __array(char, buf, MAX_BUF_LEN) ++ ), ++ ++ TP_fast_assign( ++ snprintf(__entry->buf, MAX_BUF_LEN, ++ "VM%d [%s] spt %p before %d -> after %d\n", ++ id, action, spt, before, after); ++ ), ++ ++ TP_printk("%s", __entry->buf) ++); ++ ++TRACE_EVENT(spt_change, ++ TP_PROTO(int id, char *action, void *spt, unsigned long gfn, ++ int type), ++ ++ TP_ARGS(id, action, spt, gfn, type), ++ ++ TP_STRUCT__entry( ++ __array(char, buf, MAX_BUF_LEN) ++ ), ++ ++ TP_fast_assign( ++ snprintf(__entry->buf, MAX_BUF_LEN, ++ "VM%d [%s] spt %p gfn 0x%lx type %d\n", ++ id, action, spt, gfn, type); ++ ), ++ ++ TP_printk("%s", __entry->buf) ++); ++ ++TRACE_EVENT(spt_guest_change, ++ TP_PROTO(int id, const char *tag, void *spt, int type, u64 v, ++ unsigned long index), ++ ++ TP_ARGS(id, tag, spt, type, v, index), ++ ++ TP_STRUCT__entry( ++ __array(char, buf, MAX_BUF_LEN) ++ ), ++ ++ TP_fast_assign( ++ snprintf(__entry->buf, MAX_BUF_LEN, ++ "VM%d [%s] spt %p type %d entry 0x%llx index 0x%lx\n", ++ id, tag, spt, type, v, index); ++ ), ++ ++ TP_printk("%s", __entry->buf) ++); ++ ++TRACE_EVENT(oos_change, ++ TP_PROTO(int id, const char *tag, int page_id, void *gpt, int type), ++ ++ TP_ARGS(id, tag, page_id, gpt, type), ++ ++ TP_STRUCT__entry( ++ __array(char, buf, MAX_BUF_LEN) ++ ), ++ ++ TP_fast_assign( ++ snprintf(__entry->buf, MAX_BUF_LEN, ++ "VM%d [oos %s] page id %d gpt %p type %d\n", ++ id, tag, page_id, gpt, type); ++ ), ++ ++ TP_printk("%s", __entry->buf) ++); ++ ++TRACE_EVENT(oos_sync, ++ TP_PROTO(int id, int page_id, void *gpt, int type, u64 v, ++ unsigned long index), ++ ++ TP_ARGS(id, page_id, gpt, type, v, index), ++ ++ TP_STRUCT__entry( ++ __array(char, buf, MAX_BUF_LEN) ++ ), ++ ++ TP_fast_assign( ++ snprintf(__entry->buf, MAX_BUF_LEN, ++ "VM%d [oos sync] page id %d gpt %p type %d entry 0x%llx index 0x%lx\n", ++ id, page_id, gpt, type, v, index); ++ ), ++ ++ TP_printk("%s", __entry->buf) ++); ++ ++#define GVT_CMD_STR_LEN 40 ++TRACE_EVENT(gvt_command, ++ TP_PROTO(u8 vgpu_id, u8 ring_id, u32 ip_gma, u32 *cmd_va, ++ u32 cmd_len, u32 buf_type, u32 buf_addr_type, ++ void *workload, const char *cmd_name), ++ ++ TP_ARGS(vgpu_id, ring_id, ip_gma, cmd_va, cmd_len, buf_type, ++ buf_addr_type, workload, cmd_name), ++ ++ TP_STRUCT__entry( ++ __field(u8, vgpu_id) ++ __field(u8, ring_id) ++ __field(u32, ip_gma) ++ __field(u32, buf_type) ++ __field(u32, buf_addr_type) ++ __field(u32, cmd_len) ++ __field(void*, workload) ++ __dynamic_array(u32, raw_cmd, cmd_len) ++ __array(char, cmd_name, GVT_CMD_STR_LEN) ++ ), ++ ++ TP_fast_assign( ++ __entry->vgpu_id = vgpu_id; ++ __entry->ring_id = ring_id; ++ __entry->ip_gma = ip_gma; ++ __entry->buf_type = buf_type; ++ __entry->buf_addr_type = buf_addr_type; ++ __entry->cmd_len = cmd_len; ++ __entry->workload = workload; ++ snprintf(__entry->cmd_name, GVT_CMD_STR_LEN, "%s", cmd_name); ++ memcpy(__get_dynamic_array(raw_cmd), cmd_va, cmd_len * sizeof(*cmd_va)); ++ ), ++ ++ ++ TP_printk("vgpu%d ring %d: address_type %u, buf_type %u, ip_gma %08x,cmd (name=%s,len=%u,raw cmd=%s), workload=%p\n", ++ __entry->vgpu_id, ++ __entry->ring_id, ++ __entry->buf_addr_type, ++ __entry->buf_type, ++ __entry->ip_gma, ++ __entry->cmd_name, ++ __entry->cmd_len, ++ __print_array(__get_dynamic_array(raw_cmd), ++ __entry->cmd_len, 4), ++ __entry->workload) ++); ++ ++#define GVT_TEMP_STR_LEN 10 ++TRACE_EVENT(write_ir, ++ TP_PROTO(int id, char *reg_name, unsigned int reg, unsigned int new_val, ++ unsigned int old_val, bool changed), ++ ++ TP_ARGS(id, reg_name, reg, new_val, old_val, changed), ++ ++ TP_STRUCT__entry( ++ __field(int, id) ++ __array(char, buf, GVT_TEMP_STR_LEN) ++ __field(unsigned int, reg) ++ __field(unsigned int, new_val) ++ __field(unsigned int, old_val) ++ __field(bool, changed) ++ ), ++ ++ TP_fast_assign( ++ __entry->id = id; ++ snprintf(__entry->buf, GVT_TEMP_STR_LEN, "%s", reg_name); ++ __entry->reg = reg; ++ __entry->new_val = new_val; ++ __entry->old_val = old_val; ++ __entry->changed = changed; ++ ), ++ ++ TP_printk("VM%u write [%s] %x, new %08x, old %08x, changed %08x\n", ++ __entry->id, __entry->buf, __entry->reg, __entry->new_val, ++ __entry->old_val, __entry->changed) ++); ++ ++TRACE_EVENT(propagate_event, ++ TP_PROTO(int id, const char *irq_name, int bit), ++ ++ TP_ARGS(id, irq_name, bit), ++ ++ TP_STRUCT__entry( ++ __field(int, id) ++ __array(char, buf, GVT_TEMP_STR_LEN) ++ __field(int, bit) ++ ), ++ ++ TP_fast_assign( ++ __entry->id = id; ++ snprintf(__entry->buf, GVT_TEMP_STR_LEN, "%s", irq_name); ++ __entry->bit = bit; ++ ), ++ ++ TP_printk("Set bit (%d) for (%s) for vgpu (%d)\n", ++ __entry->bit, __entry->buf, __entry->id) ++); ++ ++TRACE_EVENT(inject_msi, ++ TP_PROTO(int id, unsigned int address, unsigned int data), ++ ++ TP_ARGS(id, address, data), ++ ++ TP_STRUCT__entry( ++ __field(int, id) ++ __field(unsigned int, address) ++ __field(unsigned int, data) ++ ), ++ ++ TP_fast_assign( ++ __entry->id = id; ++ __entry->address = address; ++ __entry->data = data; ++ ), ++ ++ TP_printk("vgpu%d:inject msi address %x data %x\n", ++ __entry->id, __entry->address, __entry->data) ++); ++ ++TRACE_EVENT(render_mmio, ++ TP_PROTO(int old_id, int new_id, char *action, unsigned int reg, ++ unsigned int old_val, unsigned int new_val), ++ ++ TP_ARGS(old_id, new_id, action, reg, old_val, new_val), ++ ++ TP_STRUCT__entry( ++ __field(int, old_id) ++ __field(int, new_id) ++ __array(char, buf, GVT_TEMP_STR_LEN) ++ __field(unsigned int, reg) ++ __field(unsigned int, old_val) ++ __field(unsigned int, new_val) ++ ), ++ ++ TP_fast_assign( ++ __entry->old_id = old_id; ++ __entry->new_id = new_id; ++ snprintf(__entry->buf, GVT_TEMP_STR_LEN, "%s", action); ++ __entry->reg = reg; ++ __entry->old_val = old_val; ++ __entry->new_val = new_val; ++ ), ++ ++ TP_printk("VM%u -> VM%u %s reg %x, old %08x new %08x\n", ++ __entry->old_id, __entry->new_id, ++ __entry->buf, __entry->reg, ++ __entry->old_val, __entry->new_val) ++); ++ ++#endif /* _GVT_TRACE_H_ */ ++ ++/* This part must be out of protection */ ++#undef TRACE_INCLUDE_PATH ++#define TRACE_INCLUDE_PATH . ++#undef TRACE_INCLUDE_FILE ++#define TRACE_INCLUDE_FILE trace ++#include +diff --git a/drivers/gpu/drm/i915_legacy/gvt/trace_points.c b/drivers/gpu/drm/i915_legacy/gvt/trace_points.c +new file mode 100644 +index 000000000000..a3deed692b9c +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/trace_points.c +@@ -0,0 +1,36 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Jike Song ++ * ++ * Contributors: ++ * Zhi Wang ++ * ++ */ ++ ++#include "trace.h" ++ ++#ifndef __CHECKER__ ++#define CREATE_TRACE_POINTS ++#include "trace.h" ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/gvt/vgpu.c b/drivers/gpu/drm/i915_legacy/gvt/vgpu.c +new file mode 100644 +index 000000000000..44ce3c2b9ac1 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/gvt/vgpu.c +@@ -0,0 +1,592 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Eddie Dong ++ * Kevin Tian ++ * ++ * Contributors: ++ * Ping Gao ++ * Zhi Wang ++ * Bing Niu ++ * ++ */ ++ ++#include "i915_drv.h" ++#include "gvt.h" ++#include "i915_pvinfo.h" ++ ++void populate_pvinfo_page(struct intel_vgpu *vgpu) ++{ ++ /* setup the ballooning information */ ++ vgpu_vreg64_t(vgpu, vgtif_reg(magic)) = VGT_MAGIC; ++ vgpu_vreg_t(vgpu, vgtif_reg(version_major)) = 1; ++ vgpu_vreg_t(vgpu, vgtif_reg(version_minor)) = 0; ++ vgpu_vreg_t(vgpu, vgtif_reg(display_ready)) = 0; ++ vgpu_vreg_t(vgpu, vgtif_reg(vgt_id)) = vgpu->id; ++ ++ vgpu_vreg_t(vgpu, vgtif_reg(vgt_caps)) = VGT_CAPS_FULL_PPGTT; ++ vgpu_vreg_t(vgpu, vgtif_reg(vgt_caps)) |= VGT_CAPS_HWSP_EMULATION; ++ vgpu_vreg_t(vgpu, vgtif_reg(vgt_caps)) |= VGT_CAPS_HUGE_GTT; ++ ++ vgpu_vreg_t(vgpu, vgtif_reg(avail_rs.mappable_gmadr.base)) = ++ vgpu_aperture_gmadr_base(vgpu); ++ vgpu_vreg_t(vgpu, vgtif_reg(avail_rs.mappable_gmadr.size)) = ++ vgpu_aperture_sz(vgpu); ++ vgpu_vreg_t(vgpu, vgtif_reg(avail_rs.nonmappable_gmadr.base)) = ++ vgpu_hidden_gmadr_base(vgpu); ++ vgpu_vreg_t(vgpu, vgtif_reg(avail_rs.nonmappable_gmadr.size)) = ++ vgpu_hidden_sz(vgpu); ++ ++ vgpu_vreg_t(vgpu, vgtif_reg(avail_rs.fence_num)) = vgpu_fence_sz(vgpu); ++ ++ vgpu_vreg_t(vgpu, vgtif_reg(cursor_x_hot)) = UINT_MAX; ++ vgpu_vreg_t(vgpu, vgtif_reg(cursor_y_hot)) = UINT_MAX; ++ ++ gvt_dbg_core("Populate PVINFO PAGE for vGPU %d\n", vgpu->id); ++ gvt_dbg_core("aperture base [GMADR] 0x%llx size 0x%llx\n", ++ vgpu_aperture_gmadr_base(vgpu), vgpu_aperture_sz(vgpu)); ++ gvt_dbg_core("hidden base [GMADR] 0x%llx size=0x%llx\n", ++ vgpu_hidden_gmadr_base(vgpu), vgpu_hidden_sz(vgpu)); ++ gvt_dbg_core("fence size %d\n", vgpu_fence_sz(vgpu)); ++ ++ WARN_ON(sizeof(struct vgt_if) != VGT_PVINFO_SIZE); ++} ++ ++#define VGPU_MAX_WEIGHT 16 ++#define VGPU_WEIGHT(vgpu_num) \ ++ (VGPU_MAX_WEIGHT / (vgpu_num)) ++ ++static struct { ++ unsigned int low_mm; ++ unsigned int high_mm; ++ unsigned int fence; ++ ++ /* A vGPU with a weight of 8 will get twice as much GPU as a vGPU ++ * with a weight of 4 on a contended host, different vGPU type has ++ * different weight set. Legal weights range from 1 to 16. ++ */ ++ unsigned int weight; ++ enum intel_vgpu_edid edid; ++ char *name; ++} vgpu_types[] = { ++/* Fixed vGPU type table */ ++ { MB_TO_BYTES(64), MB_TO_BYTES(384), 4, VGPU_WEIGHT(8), GVT_EDID_1024_768, "8" }, ++ { MB_TO_BYTES(128), MB_TO_BYTES(512), 4, VGPU_WEIGHT(4), GVT_EDID_1920_1200, "4" }, ++ { MB_TO_BYTES(256), MB_TO_BYTES(1024), 4, VGPU_WEIGHT(2), GVT_EDID_1920_1200, "2" }, ++ { MB_TO_BYTES(512), MB_TO_BYTES(2048), 4, VGPU_WEIGHT(1), GVT_EDID_1920_1200, "1" }, ++}; ++ ++/** ++ * intel_gvt_init_vgpu_types - initialize vGPU type list ++ * @gvt : GVT device ++ * ++ * Initialize vGPU type list based on available resource. ++ * ++ */ ++int intel_gvt_init_vgpu_types(struct intel_gvt *gvt) ++{ ++ unsigned int num_types; ++ unsigned int i, low_avail, high_avail; ++ unsigned int min_low; ++ ++ /* vGPU type name is defined as GVTg_Vx_y which contains ++ * physical GPU generation type (e.g V4 as BDW server, V5 as ++ * SKL server). ++ * ++ * Depend on physical SKU resource, might see vGPU types like ++ * GVTg_V4_8, GVTg_V4_4, GVTg_V4_2, etc. We can create ++ * different types of vGPU on same physical GPU depending on ++ * available resource. Each vGPU type will have "avail_instance" ++ * to indicate how many vGPU instance can be created for this ++ * type. ++ * ++ */ ++ low_avail = gvt_aperture_sz(gvt) - HOST_LOW_GM_SIZE; ++ high_avail = gvt_hidden_sz(gvt) - HOST_HIGH_GM_SIZE; ++ num_types = sizeof(vgpu_types) / sizeof(vgpu_types[0]); ++ ++ gvt->types = kcalloc(num_types, sizeof(struct intel_vgpu_type), ++ GFP_KERNEL); ++ if (!gvt->types) ++ return -ENOMEM; ++ ++ min_low = MB_TO_BYTES(32); ++ for (i = 0; i < num_types; ++i) { ++ if (low_avail / vgpu_types[i].low_mm == 0) ++ break; ++ ++ gvt->types[i].low_gm_size = vgpu_types[i].low_mm; ++ gvt->types[i].high_gm_size = vgpu_types[i].high_mm; ++ gvt->types[i].fence = vgpu_types[i].fence; ++ ++ if (vgpu_types[i].weight < 1 || ++ vgpu_types[i].weight > VGPU_MAX_WEIGHT) ++ return -EINVAL; ++ ++ gvt->types[i].weight = vgpu_types[i].weight; ++ gvt->types[i].resolution = vgpu_types[i].edid; ++ gvt->types[i].avail_instance = min(low_avail / vgpu_types[i].low_mm, ++ high_avail / vgpu_types[i].high_mm); ++ ++ if (IS_GEN(gvt->dev_priv, 8)) ++ sprintf(gvt->types[i].name, "GVTg_V4_%s", ++ vgpu_types[i].name); ++ else if (IS_GEN(gvt->dev_priv, 9)) ++ sprintf(gvt->types[i].name, "GVTg_V5_%s", ++ vgpu_types[i].name); ++ ++ gvt_dbg_core("type[%d]: %s avail %u low %u high %u fence %u weight %u res %s\n", ++ i, gvt->types[i].name, ++ gvt->types[i].avail_instance, ++ gvt->types[i].low_gm_size, ++ gvt->types[i].high_gm_size, gvt->types[i].fence, ++ gvt->types[i].weight, ++ vgpu_edid_str(gvt->types[i].resolution)); ++ } ++ ++ gvt->num_types = i; ++ return 0; ++} ++ ++void intel_gvt_clean_vgpu_types(struct intel_gvt *gvt) ++{ ++ kfree(gvt->types); ++} ++ ++static void intel_gvt_update_vgpu_types(struct intel_gvt *gvt) ++{ ++ int i; ++ unsigned int low_gm_avail, high_gm_avail, fence_avail; ++ unsigned int low_gm_min, high_gm_min, fence_min; ++ ++ /* Need to depend on maxium hw resource size but keep on ++ * static config for now. ++ */ ++ low_gm_avail = gvt_aperture_sz(gvt) - HOST_LOW_GM_SIZE - ++ gvt->gm.vgpu_allocated_low_gm_size; ++ high_gm_avail = gvt_hidden_sz(gvt) - HOST_HIGH_GM_SIZE - ++ gvt->gm.vgpu_allocated_high_gm_size; ++ fence_avail = gvt_fence_sz(gvt) - HOST_FENCE - ++ gvt->fence.vgpu_allocated_fence_num; ++ ++ for (i = 0; i < gvt->num_types; i++) { ++ low_gm_min = low_gm_avail / gvt->types[i].low_gm_size; ++ high_gm_min = high_gm_avail / gvt->types[i].high_gm_size; ++ fence_min = fence_avail / gvt->types[i].fence; ++ gvt->types[i].avail_instance = min(min(low_gm_min, high_gm_min), ++ fence_min); ++ ++ gvt_dbg_core("update type[%d]: %s avail %u low %u high %u fence %u\n", ++ i, gvt->types[i].name, ++ gvt->types[i].avail_instance, gvt->types[i].low_gm_size, ++ gvt->types[i].high_gm_size, gvt->types[i].fence); ++ } ++} ++ ++/** ++ * intel_gvt_active_vgpu - activate a virtual GPU ++ * @vgpu: virtual GPU ++ * ++ * This function is called when user wants to activate a virtual GPU. ++ * ++ */ ++void intel_gvt_activate_vgpu(struct intel_vgpu *vgpu) ++{ ++ mutex_lock(&vgpu->gvt->lock); ++ vgpu->active = true; ++ mutex_unlock(&vgpu->gvt->lock); ++} ++ ++/** ++ * intel_gvt_deactive_vgpu - deactivate a virtual GPU ++ * @vgpu: virtual GPU ++ * ++ * This function is called when user wants to deactivate a virtual GPU. ++ * The virtual GPU will be stopped. ++ * ++ */ ++void intel_gvt_deactivate_vgpu(struct intel_vgpu *vgpu) ++{ ++ mutex_lock(&vgpu->vgpu_lock); ++ ++ vgpu->active = false; ++ ++ if (atomic_read(&vgpu->submission.running_workload_num)) { ++ mutex_unlock(&vgpu->vgpu_lock); ++ intel_gvt_wait_vgpu_idle(vgpu); ++ mutex_lock(&vgpu->vgpu_lock); ++ } ++ ++ intel_vgpu_stop_schedule(vgpu); ++ ++ mutex_unlock(&vgpu->vgpu_lock); ++} ++ ++/** ++ * intel_gvt_release_vgpu - release a virtual GPU ++ * @vgpu: virtual GPU ++ * ++ * This function is called when user wants to release a virtual GPU. ++ * The virtual GPU will be stopped and all runtime information will be ++ * destroyed. ++ * ++ */ ++void intel_gvt_release_vgpu(struct intel_vgpu *vgpu) ++{ ++ intel_gvt_deactivate_vgpu(vgpu); ++ ++ mutex_lock(&vgpu->vgpu_lock); ++ intel_vgpu_clean_workloads(vgpu, ALL_ENGINES); ++ intel_vgpu_dmabuf_cleanup(vgpu); ++ mutex_unlock(&vgpu->vgpu_lock); ++} ++ ++/** ++ * intel_gvt_destroy_vgpu - destroy a virtual GPU ++ * @vgpu: virtual GPU ++ * ++ * This function is called when user wants to destroy a virtual GPU. ++ * ++ */ ++void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu) ++{ ++ struct intel_gvt *gvt = vgpu->gvt; ++ ++ mutex_lock(&vgpu->vgpu_lock); ++ ++ WARN(vgpu->active, "vGPU is still active!\n"); ++ ++ intel_gvt_debugfs_remove_vgpu(vgpu); ++ intel_vgpu_clean_sched_policy(vgpu); ++ intel_vgpu_clean_submission(vgpu); ++ intel_vgpu_clean_display(vgpu); ++ intel_vgpu_clean_opregion(vgpu); ++ intel_vgpu_reset_ggtt(vgpu, true); ++ intel_vgpu_clean_gtt(vgpu); ++ intel_gvt_hypervisor_detach_vgpu(vgpu); ++ intel_vgpu_free_resource(vgpu); ++ intel_vgpu_clean_mmio(vgpu); ++ intel_vgpu_dmabuf_cleanup(vgpu); ++ mutex_unlock(&vgpu->vgpu_lock); ++ ++ mutex_lock(&gvt->lock); ++ idr_remove(&gvt->vgpu_idr, vgpu->id); ++ if (idr_is_empty(&gvt->vgpu_idr)) ++ intel_gvt_clean_irq(gvt); ++ intel_gvt_update_vgpu_types(gvt); ++ mutex_unlock(&gvt->lock); ++ ++ vfree(vgpu); ++} ++ ++#define IDLE_VGPU_IDR 0 ++ ++/** ++ * intel_gvt_create_idle_vgpu - create an idle virtual GPU ++ * @gvt: GVT device ++ * ++ * This function is called when user wants to create an idle virtual GPU. ++ * ++ * Returns: ++ * pointer to intel_vgpu, error pointer if failed. ++ */ ++struct intel_vgpu *intel_gvt_create_idle_vgpu(struct intel_gvt *gvt) ++{ ++ struct intel_vgpu *vgpu; ++ enum intel_engine_id i; ++ int ret; ++ ++ vgpu = vzalloc(sizeof(*vgpu)); ++ if (!vgpu) ++ return ERR_PTR(-ENOMEM); ++ ++ vgpu->id = IDLE_VGPU_IDR; ++ vgpu->gvt = gvt; ++ mutex_init(&vgpu->vgpu_lock); ++ ++ for (i = 0; i < I915_NUM_ENGINES; i++) ++ INIT_LIST_HEAD(&vgpu->submission.workload_q_head[i]); ++ ++ ret = intel_vgpu_init_sched_policy(vgpu); ++ if (ret) ++ goto out_free_vgpu; ++ ++ vgpu->active = false; ++ ++ return vgpu; ++ ++out_free_vgpu: ++ vfree(vgpu); ++ return ERR_PTR(ret); ++} ++ ++/** ++ * intel_gvt_destroy_vgpu - destroy an idle virtual GPU ++ * @vgpu: virtual GPU ++ * ++ * This function is called when user wants to destroy an idle virtual GPU. ++ * ++ */ ++void intel_gvt_destroy_idle_vgpu(struct intel_vgpu *vgpu) ++{ ++ mutex_lock(&vgpu->vgpu_lock); ++ intel_vgpu_clean_sched_policy(vgpu); ++ mutex_unlock(&vgpu->vgpu_lock); ++ ++ vfree(vgpu); ++} ++ ++static struct intel_vgpu *__intel_gvt_create_vgpu(struct intel_gvt *gvt, ++ struct intel_vgpu_creation_params *param) ++{ ++ struct intel_vgpu *vgpu; ++ int ret; ++ ++ gvt_dbg_core("handle %llu low %llu MB high %llu MB fence %llu\n", ++ param->handle, param->low_gm_sz, param->high_gm_sz, ++ param->fence_sz); ++ ++ vgpu = vzalloc(sizeof(*vgpu)); ++ if (!vgpu) ++ return ERR_PTR(-ENOMEM); ++ ++ ret = idr_alloc(&gvt->vgpu_idr, vgpu, IDLE_VGPU_IDR + 1, GVT_MAX_VGPU, ++ GFP_KERNEL); ++ if (ret < 0) ++ goto out_free_vgpu; ++ ++ vgpu->id = ret; ++ vgpu->handle = param->handle; ++ vgpu->gvt = gvt; ++ vgpu->sched_ctl.weight = param->weight; ++ mutex_init(&vgpu->vgpu_lock); ++ mutex_init(&vgpu->dmabuf_lock); ++ INIT_LIST_HEAD(&vgpu->dmabuf_obj_list_head); ++ INIT_RADIX_TREE(&vgpu->page_track_tree, GFP_KERNEL); ++ idr_init(&vgpu->object_idr); ++ intel_vgpu_init_cfg_space(vgpu, param->primary); ++ ++ ret = intel_vgpu_init_mmio(vgpu); ++ if (ret) ++ goto out_clean_idr; ++ ++ ret = intel_vgpu_alloc_resource(vgpu, param); ++ if (ret) ++ goto out_clean_vgpu_mmio; ++ ++ populate_pvinfo_page(vgpu); ++ ++ ret = intel_gvt_hypervisor_attach_vgpu(vgpu); ++ if (ret) ++ goto out_clean_vgpu_resource; ++ ++ ret = intel_vgpu_init_gtt(vgpu); ++ if (ret) ++ goto out_detach_hypervisor_vgpu; ++ ++ ret = intel_vgpu_init_opregion(vgpu); ++ if (ret) ++ goto out_clean_gtt; ++ ++ ret = intel_vgpu_init_display(vgpu, param->resolution); ++ if (ret) ++ goto out_clean_opregion; ++ ++ ret = intel_vgpu_setup_submission(vgpu); ++ if (ret) ++ goto out_clean_display; ++ ++ ret = intel_vgpu_init_sched_policy(vgpu); ++ if (ret) ++ goto out_clean_submission; ++ ++ ret = intel_gvt_debugfs_add_vgpu(vgpu); ++ if (ret) ++ goto out_clean_sched_policy; ++ ++ ret = intel_gvt_hypervisor_set_opregion(vgpu); ++ if (ret) ++ goto out_clean_sched_policy; ++ ++ /*TODO: add more platforms support */ ++ if (IS_SKYLAKE(gvt->dev_priv) || IS_KABYLAKE(gvt->dev_priv)) ++ ret = intel_gvt_hypervisor_set_edid(vgpu, PORT_D); ++ if (ret) ++ goto out_clean_sched_policy; ++ ++ return vgpu; ++ ++out_clean_sched_policy: ++ intel_vgpu_clean_sched_policy(vgpu); ++out_clean_submission: ++ intel_vgpu_clean_submission(vgpu); ++out_clean_display: ++ intel_vgpu_clean_display(vgpu); ++out_clean_opregion: ++ intel_vgpu_clean_opregion(vgpu); ++out_clean_gtt: ++ intel_vgpu_clean_gtt(vgpu); ++out_detach_hypervisor_vgpu: ++ intel_gvt_hypervisor_detach_vgpu(vgpu); ++out_clean_vgpu_resource: ++ intel_vgpu_free_resource(vgpu); ++out_clean_vgpu_mmio: ++ intel_vgpu_clean_mmio(vgpu); ++out_clean_idr: ++ idr_remove(&gvt->vgpu_idr, vgpu->id); ++out_free_vgpu: ++ vfree(vgpu); ++ return ERR_PTR(ret); ++} ++ ++/** ++ * intel_gvt_create_vgpu - create a virtual GPU ++ * @gvt: GVT device ++ * @type: type of the vGPU to create ++ * ++ * This function is called when user wants to create a virtual GPU. ++ * ++ * Returns: ++ * pointer to intel_vgpu, error pointer if failed. ++ */ ++struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt, ++ struct intel_vgpu_type *type) ++{ ++ struct intel_vgpu_creation_params param; ++ struct intel_vgpu *vgpu; ++ ++ param.handle = 0; ++ param.primary = 1; ++ param.low_gm_sz = type->low_gm_size; ++ param.high_gm_sz = type->high_gm_size; ++ param.fence_sz = type->fence; ++ param.weight = type->weight; ++ param.resolution = type->resolution; ++ ++ /* XXX current param based on MB */ ++ param.low_gm_sz = BYTES_TO_MB(param.low_gm_sz); ++ param.high_gm_sz = BYTES_TO_MB(param.high_gm_sz); ++ ++ mutex_lock(&gvt->lock); ++ vgpu = __intel_gvt_create_vgpu(gvt, ¶m); ++ if (!IS_ERR(vgpu)) ++ /* calculate left instance change for types */ ++ intel_gvt_update_vgpu_types(gvt); ++ mutex_unlock(&gvt->lock); ++ ++ return vgpu; ++} ++ ++/** ++ * intel_gvt_reset_vgpu_locked - reset a virtual GPU by DMLR or GT reset ++ * @vgpu: virtual GPU ++ * @dmlr: vGPU Device Model Level Reset or GT Reset ++ * @engine_mask: engines to reset for GT reset ++ * ++ * This function is called when user wants to reset a virtual GPU through ++ * device model reset or GT reset. The caller should hold the vgpu lock. ++ * ++ * vGPU Device Model Level Reset (DMLR) simulates the PCI level reset to reset ++ * the whole vGPU to default state as when it is created. This vGPU function ++ * is required both for functionary and security concerns.The ultimate goal ++ * of vGPU FLR is that reuse a vGPU instance by virtual machines. When we ++ * assign a vGPU to a virtual machine we must isse such reset first. ++ * ++ * Full GT Reset and Per-Engine GT Reset are soft reset flow for GPU engines ++ * (Render, Blitter, Video, Video Enhancement). It is defined by GPU Spec. ++ * Unlike the FLR, GT reset only reset particular resource of a vGPU per ++ * the reset request. Guest driver can issue a GT reset by programming the ++ * virtual GDRST register to reset specific virtual GPU engine or all ++ * engines. ++ * ++ * The parameter dev_level is to identify if we will do DMLR or GT reset. ++ * The parameter engine_mask is to specific the engines that need to be ++ * resetted. If value ALL_ENGINES is given for engine_mask, it means ++ * the caller requests a full GT reset that we will reset all virtual ++ * GPU engines. For FLR, engine_mask is ignored. ++ */ ++void intel_gvt_reset_vgpu_locked(struct intel_vgpu *vgpu, bool dmlr, ++ intel_engine_mask_t engine_mask) ++{ ++ struct intel_gvt *gvt = vgpu->gvt; ++ struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; ++ intel_engine_mask_t resetting_eng = dmlr ? ALL_ENGINES : engine_mask; ++ ++ gvt_dbg_core("------------------------------------------\n"); ++ gvt_dbg_core("resseting vgpu%d, dmlr %d, engine_mask %08x\n", ++ vgpu->id, dmlr, engine_mask); ++ ++ vgpu->resetting_eng = resetting_eng; ++ ++ intel_vgpu_stop_schedule(vgpu); ++ /* ++ * The current_vgpu will set to NULL after stopping the ++ * scheduler when the reset is triggered by current vgpu. ++ */ ++ if (scheduler->current_vgpu == NULL) { ++ mutex_unlock(&vgpu->vgpu_lock); ++ intel_gvt_wait_vgpu_idle(vgpu); ++ mutex_lock(&vgpu->vgpu_lock); ++ } ++ ++ intel_vgpu_reset_submission(vgpu, resetting_eng); ++ /* full GPU reset or device model level reset */ ++ if (engine_mask == ALL_ENGINES || dmlr) { ++ intel_vgpu_select_submission_ops(vgpu, ALL_ENGINES, 0); ++ intel_vgpu_invalidate_ppgtt(vgpu); ++ /*fence will not be reset during virtual reset */ ++ if (dmlr) { ++ intel_vgpu_reset_gtt(vgpu); ++ intel_vgpu_reset_resource(vgpu); ++ } ++ ++ intel_vgpu_reset_mmio(vgpu, dmlr); ++ populate_pvinfo_page(vgpu); ++ intel_vgpu_reset_display(vgpu); ++ ++ if (dmlr) { ++ intel_vgpu_reset_cfg_space(vgpu); ++ /* only reset the failsafe mode when dmlr reset */ ++ vgpu->failsafe = false; ++ vgpu->pv_notified = false; ++ } ++ } ++ ++ vgpu->resetting_eng = 0; ++ gvt_dbg_core("reset vgpu%d done\n", vgpu->id); ++ gvt_dbg_core("------------------------------------------\n"); ++} ++ ++/** ++ * intel_gvt_reset_vgpu - reset a virtual GPU (Function Level) ++ * @vgpu: virtual GPU ++ * ++ * This function is called when user wants to reset a virtual GPU. ++ * ++ */ ++void intel_gvt_reset_vgpu(struct intel_vgpu *vgpu) ++{ ++ mutex_lock(&vgpu->vgpu_lock); ++ intel_gvt_reset_vgpu_locked(vgpu, true, 0); ++ mutex_unlock(&vgpu->vgpu_lock); ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_active.c b/drivers/gpu/drm/i915_legacy/i915_active.c +new file mode 100644 +index 000000000000..863ae12707ba +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_active.c +@@ -0,0 +1,313 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2019 Intel Corporation ++ */ ++ ++#include "i915_drv.h" ++#include "i915_active.h" ++#include "i915_globals.h" ++ ++#define BKL(ref) (&(ref)->i915->drm.struct_mutex) ++ ++/* ++ * Active refs memory management ++ * ++ * To be more economical with memory, we reap all the i915_active trees as ++ * they idle (when we know the active requests are inactive) and allocate the ++ * nodes from a local slab cache to hopefully reduce the fragmentation. ++ */ ++static struct i915_global_active { ++ struct i915_global base; ++ struct kmem_cache *slab_cache; ++} global; ++ ++struct active_node { ++ struct i915_active_request base; ++ struct i915_active *ref; ++ struct rb_node node; ++ u64 timeline; ++}; ++ ++static void ++__active_park(struct i915_active *ref) ++{ ++ struct active_node *it, *n; ++ ++ rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) { ++ GEM_BUG_ON(i915_active_request_isset(&it->base)); ++ kmem_cache_free(global.slab_cache, it); ++ } ++ ref->tree = RB_ROOT; ++} ++ ++static void ++__active_retire(struct i915_active *ref) ++{ ++ GEM_BUG_ON(!ref->count); ++ if (--ref->count) ++ return; ++ ++ /* return the unused nodes to our slabcache */ ++ __active_park(ref); ++ ++ ref->retire(ref); ++} ++ ++static void ++node_retire(struct i915_active_request *base, struct i915_request *rq) ++{ ++ __active_retire(container_of(base, struct active_node, base)->ref); ++} ++ ++static void ++last_retire(struct i915_active_request *base, struct i915_request *rq) ++{ ++ __active_retire(container_of(base, struct i915_active, last)); ++} ++ ++static struct i915_active_request * ++active_instance(struct i915_active *ref, u64 idx) ++{ ++ struct active_node *node; ++ struct rb_node **p, *parent; ++ struct i915_request *old; ++ ++ /* ++ * We track the most recently used timeline to skip a rbtree search ++ * for the common case, under typical loads we never need the rbtree ++ * at all. We can reuse the last slot if it is empty, that is ++ * after the previous activity has been retired, or if it matches the ++ * current timeline. ++ * ++ * Note that we allow the timeline to be active simultaneously in ++ * the rbtree and the last cache. We do this to avoid having ++ * to search and replace the rbtree element for a new timeline, with ++ * the cost being that we must be aware that the ref may be retired ++ * twice for the same timeline (as the older rbtree element will be ++ * retired before the new request added to last). ++ */ ++ old = i915_active_request_raw(&ref->last, BKL(ref)); ++ if (!old || old->fence.context == idx) ++ goto out; ++ ++ /* Move the currently active fence into the rbtree */ ++ idx = old->fence.context; ++ ++ parent = NULL; ++ p = &ref->tree.rb_node; ++ while (*p) { ++ parent = *p; ++ ++ node = rb_entry(parent, struct active_node, node); ++ if (node->timeline == idx) ++ goto replace; ++ ++ if (node->timeline < idx) ++ p = &parent->rb_right; ++ else ++ p = &parent->rb_left; ++ } ++ ++ node = kmem_cache_alloc(global.slab_cache, GFP_KERNEL); ++ ++ /* kmalloc may retire the ref->last (thanks shrinker)! */ ++ if (unlikely(!i915_active_request_raw(&ref->last, BKL(ref)))) { ++ kmem_cache_free(global.slab_cache, node); ++ goto out; ++ } ++ ++ if (unlikely(!node)) ++ return ERR_PTR(-ENOMEM); ++ ++ i915_active_request_init(&node->base, NULL, node_retire); ++ node->ref = ref; ++ node->timeline = idx; ++ ++ rb_link_node(&node->node, parent, p); ++ rb_insert_color(&node->node, &ref->tree); ++ ++replace: ++ /* ++ * Overwrite the previous active slot in the rbtree with last, ++ * leaving last zeroed. If the previous slot is still active, ++ * we must be careful as we now only expect to receive one retire ++ * callback not two, and so much undo the active counting for the ++ * overwritten slot. ++ */ ++ if (i915_active_request_isset(&node->base)) { ++ /* Retire ourselves from the old rq->active_list */ ++ __list_del_entry(&node->base.link); ++ ref->count--; ++ GEM_BUG_ON(!ref->count); ++ } ++ GEM_BUG_ON(list_empty(&ref->last.link)); ++ list_replace_init(&ref->last.link, &node->base.link); ++ node->base.request = fetch_and_zero(&ref->last.request); ++ ++out: ++ return &ref->last; ++} ++ ++void i915_active_init(struct drm_i915_private *i915, ++ struct i915_active *ref, ++ void (*retire)(struct i915_active *ref)) ++{ ++ ref->i915 = i915; ++ ref->retire = retire; ++ ref->tree = RB_ROOT; ++ i915_active_request_init(&ref->last, NULL, last_retire); ++ ref->count = 0; ++} ++ ++int i915_active_ref(struct i915_active *ref, ++ u64 timeline, ++ struct i915_request *rq) ++{ ++ struct i915_active_request *active; ++ int err = 0; ++ ++ /* Prevent reaping in case we malloc/wait while building the tree */ ++ i915_active_acquire(ref); ++ ++ active = active_instance(ref, timeline); ++ if (IS_ERR(active)) { ++ err = PTR_ERR(active); ++ goto out; ++ } ++ ++ if (!i915_active_request_isset(active)) ++ ref->count++; ++ __i915_active_request_set(active, rq); ++ ++ GEM_BUG_ON(!ref->count); ++out: ++ i915_active_release(ref); ++ return err; ++} ++ ++bool i915_active_acquire(struct i915_active *ref) ++{ ++ lockdep_assert_held(BKL(ref)); ++ return !ref->count++; ++} ++ ++void i915_active_release(struct i915_active *ref) ++{ ++ lockdep_assert_held(BKL(ref)); ++ __active_retire(ref); ++} ++ ++int i915_active_wait(struct i915_active *ref) ++{ ++ struct active_node *it, *n; ++ int ret = 0; ++ ++ if (i915_active_acquire(ref)) ++ goto out_release; ++ ++ ret = i915_active_request_retire(&ref->last, BKL(ref)); ++ if (ret) ++ goto out_release; ++ ++ rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) { ++ ret = i915_active_request_retire(&it->base, BKL(ref)); ++ if (ret) ++ break; ++ } ++ ++out_release: ++ i915_active_release(ref); ++ return ret; ++} ++ ++int i915_request_await_active_request(struct i915_request *rq, ++ struct i915_active_request *active) ++{ ++ struct i915_request *barrier = ++ i915_active_request_raw(active, &rq->i915->drm.struct_mutex); ++ ++ return barrier ? i915_request_await_dma_fence(rq, &barrier->fence) : 0; ++} ++ ++int i915_request_await_active(struct i915_request *rq, struct i915_active *ref) ++{ ++ struct active_node *it, *n; ++ int err = 0; ++ ++ /* await allocates and so we need to avoid hitting the shrinker */ ++ if (i915_active_acquire(ref)) ++ goto out; /* was idle */ ++ ++ err = i915_request_await_active_request(rq, &ref->last); ++ if (err) ++ goto out; ++ ++ rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) { ++ err = i915_request_await_active_request(rq, &it->base); ++ if (err) ++ goto out; ++ } ++ ++out: ++ i915_active_release(ref); ++ return err; ++} ++ ++#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) ++void i915_active_fini(struct i915_active *ref) ++{ ++ GEM_BUG_ON(i915_active_request_isset(&ref->last)); ++ GEM_BUG_ON(!RB_EMPTY_ROOT(&ref->tree)); ++ GEM_BUG_ON(ref->count); ++} ++#endif ++ ++int i915_active_request_set(struct i915_active_request *active, ++ struct i915_request *rq) ++{ ++ int err; ++ ++ /* Must maintain ordering wrt previous active requests */ ++ err = i915_request_await_active_request(rq, active); ++ if (err) ++ return err; ++ ++ __i915_active_request_set(active, rq); ++ return 0; ++} ++ ++void i915_active_retire_noop(struct i915_active_request *active, ++ struct i915_request *request) ++{ ++ /* Space left intentionally blank */ ++} ++ ++#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) ++#include "selftests/i915_active.c" ++#endif ++ ++static void i915_global_active_shrink(void) ++{ ++ kmem_cache_shrink(global.slab_cache); ++} ++ ++static void i915_global_active_exit(void) ++{ ++ kmem_cache_destroy(global.slab_cache); ++} ++ ++static struct i915_global_active global = { { ++ .shrink = i915_global_active_shrink, ++ .exit = i915_global_active_exit, ++} }; ++ ++int __init i915_global_active_init(void) ++{ ++ global.slab_cache = KMEM_CACHE(active_node, SLAB_HWCACHE_ALIGN); ++ if (!global.slab_cache) ++ return -ENOMEM; ++ ++ i915_global_register(&global.base); ++ return 0; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_active.h b/drivers/gpu/drm/i915_legacy/i915_active.h +new file mode 100644 +index 000000000000..7d758719ce39 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_active.h +@@ -0,0 +1,409 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2019 Intel Corporation ++ */ ++ ++#ifndef _I915_ACTIVE_H_ ++#define _I915_ACTIVE_H_ ++ ++#include ++ ++#include "i915_active_types.h" ++#include "i915_request.h" ++ ++/* ++ * We treat requests as fences. This is not be to confused with our ++ * "fence registers" but pipeline synchronisation objects ala GL_ARB_sync. ++ * We use the fences to synchronize access from the CPU with activity on the ++ * GPU, for example, we should not rewrite an object's PTE whilst the GPU ++ * is reading them. We also track fences at a higher level to provide ++ * implicit synchronisation around GEM objects, e.g. set-domain will wait ++ * for outstanding GPU rendering before marking the object ready for CPU ++ * access, or a pageflip will wait until the GPU is complete before showing ++ * the frame on the scanout. ++ * ++ * In order to use a fence, the object must track the fence it needs to ++ * serialise with. For example, GEM objects want to track both read and ++ * write access so that we can perform concurrent read operations between ++ * the CPU and GPU engines, as well as waiting for all rendering to ++ * complete, or waiting for the last GPU user of a "fence register". The ++ * object then embeds a #i915_active_request to track the most recent (in ++ * retirement order) request relevant for the desired mode of access. ++ * The #i915_active_request is updated with i915_active_request_set() to ++ * track the most recent fence request, typically this is done as part of ++ * i915_vma_move_to_active(). ++ * ++ * When the #i915_active_request completes (is retired), it will ++ * signal its completion to the owner through a callback as well as mark ++ * itself as idle (i915_active_request.request == NULL). The owner ++ * can then perform any action, such as delayed freeing of an active ++ * resource including itself. ++ */ ++ ++void i915_active_retire_noop(struct i915_active_request *active, ++ struct i915_request *request); ++ ++/** ++ * i915_active_request_init - prepares the activity tracker for use ++ * @active - the active tracker ++ * @rq - initial request to track, can be NULL ++ * @func - a callback when then the tracker is retired (becomes idle), ++ * can be NULL ++ * ++ * i915_active_request_init() prepares the embedded @active struct for use as ++ * an activity tracker, that is for tracking the last known active request ++ * associated with it. When the last request becomes idle, when it is retired ++ * after completion, the optional callback @func is invoked. ++ */ ++static inline void ++i915_active_request_init(struct i915_active_request *active, ++ struct i915_request *rq, ++ i915_active_retire_fn retire) ++{ ++ RCU_INIT_POINTER(active->request, rq); ++ INIT_LIST_HEAD(&active->link); ++ active->retire = retire ?: i915_active_retire_noop; ++} ++ ++#define INIT_ACTIVE_REQUEST(name) i915_active_request_init((name), NULL, NULL) ++ ++/** ++ * i915_active_request_set - updates the tracker to watch the current request ++ * @active - the active tracker ++ * @request - the request to watch ++ * ++ * __i915_active_request_set() watches the given @request for completion. Whilst ++ * that @request is busy, the @active reports busy. When that @request is ++ * retired, the @active tracker is updated to report idle. ++ */ ++static inline void ++__i915_active_request_set(struct i915_active_request *active, ++ struct i915_request *request) ++{ ++ list_move(&active->link, &request->active_list); ++ rcu_assign_pointer(active->request, request); ++} ++ ++int __must_check ++i915_active_request_set(struct i915_active_request *active, ++ struct i915_request *rq); ++ ++/** ++ * i915_active_request_set_retire_fn - updates the retirement callback ++ * @active - the active tracker ++ * @fn - the routine called when the request is retired ++ * @mutex - struct_mutex used to guard retirements ++ * ++ * i915_active_request_set_retire_fn() updates the function pointer that ++ * is called when the final request associated with the @active tracker ++ * is retired. ++ */ ++static inline void ++i915_active_request_set_retire_fn(struct i915_active_request *active, ++ i915_active_retire_fn fn, ++ struct mutex *mutex) ++{ ++ lockdep_assert_held(mutex); ++ active->retire = fn ?: i915_active_retire_noop; ++} ++ ++/** ++ * i915_active_request_raw - return the active request ++ * @active - the active tracker ++ * ++ * i915_active_request_raw() returns the current request being tracked, or NULL. ++ * It does not obtain a reference on the request for the caller, so the caller ++ * must hold struct_mutex. ++ */ ++static inline struct i915_request * ++i915_active_request_raw(const struct i915_active_request *active, ++ struct mutex *mutex) ++{ ++ return rcu_dereference_protected(active->request, ++ lockdep_is_held(mutex)); ++} ++ ++/** ++ * i915_active_request_peek - report the active request being monitored ++ * @active - the active tracker ++ * ++ * i915_active_request_peek() returns the current request being tracked if ++ * still active, or NULL. It does not obtain a reference on the request ++ * for the caller, so the caller must hold struct_mutex. ++ */ ++static inline struct i915_request * ++i915_active_request_peek(const struct i915_active_request *active, ++ struct mutex *mutex) ++{ ++ struct i915_request *request; ++ ++ request = i915_active_request_raw(active, mutex); ++ if (!request || i915_request_completed(request)) ++ return NULL; ++ ++ return request; ++} ++ ++/** ++ * i915_active_request_get - return a reference to the active request ++ * @active - the active tracker ++ * ++ * i915_active_request_get() returns a reference to the active request, or NULL ++ * if the active tracker is idle. The caller must hold struct_mutex. ++ */ ++static inline struct i915_request * ++i915_active_request_get(const struct i915_active_request *active, ++ struct mutex *mutex) ++{ ++ return i915_request_get(i915_active_request_peek(active, mutex)); ++} ++ ++/** ++ * __i915_active_request_get_rcu - return a reference to the active request ++ * @active - the active tracker ++ * ++ * __i915_active_request_get() returns a reference to the active request, ++ * or NULL if the active tracker is idle. The caller must hold the RCU read ++ * lock, but the returned pointer is safe to use outside of RCU. ++ */ ++static inline struct i915_request * ++__i915_active_request_get_rcu(const struct i915_active_request *active) ++{ ++ /* ++ * Performing a lockless retrieval of the active request is super ++ * tricky. SLAB_TYPESAFE_BY_RCU merely guarantees that the backing ++ * slab of request objects will not be freed whilst we hold the ++ * RCU read lock. It does not guarantee that the request itself ++ * will not be freed and then *reused*. Viz, ++ * ++ * Thread A Thread B ++ * ++ * rq = active.request ++ * retire(rq) -> free(rq); ++ * (rq is now first on the slab freelist) ++ * active.request = NULL ++ * ++ * rq = new submission on a new object ++ * ref(rq) ++ * ++ * To prevent the request from being reused whilst the caller ++ * uses it, we take a reference like normal. Whilst acquiring ++ * the reference we check that it is not in a destroyed state ++ * (refcnt == 0). That prevents the request being reallocated ++ * whilst the caller holds on to it. To check that the request ++ * was not reallocated as we acquired the reference we have to ++ * check that our request remains the active request across ++ * the lookup, in the same manner as a seqlock. The visibility ++ * of the pointer versus the reference counting is controlled ++ * by using RCU barriers (rcu_dereference and rcu_assign_pointer). ++ * ++ * In the middle of all that, we inspect whether the request is ++ * complete. Retiring is lazy so the request may be completed long ++ * before the active tracker is updated. Querying whether the ++ * request is complete is far cheaper (as it involves no locked ++ * instructions setting cachelines to exclusive) than acquiring ++ * the reference, so we do it first. The RCU read lock ensures the ++ * pointer dereference is valid, but does not ensure that the ++ * seqno nor HWS is the right one! However, if the request was ++ * reallocated, that means the active tracker's request was complete. ++ * If the new request is also complete, then both are and we can ++ * just report the active tracker is idle. If the new request is ++ * incomplete, then we acquire a reference on it and check that ++ * it remained the active request. ++ * ++ * It is then imperative that we do not zero the request on ++ * reallocation, so that we can chase the dangling pointers! ++ * See i915_request_alloc(). ++ */ ++ do { ++ struct i915_request *request; ++ ++ request = rcu_dereference(active->request); ++ if (!request || i915_request_completed(request)) ++ return NULL; ++ ++ /* ++ * An especially silly compiler could decide to recompute the ++ * result of i915_request_completed, more specifically ++ * re-emit the load for request->fence.seqno. A race would catch ++ * a later seqno value, which could flip the result from true to ++ * false. Which means part of the instructions below might not ++ * be executed, while later on instructions are executed. Due to ++ * barriers within the refcounting the inconsistency can't reach ++ * past the call to i915_request_get_rcu, but not executing ++ * that while still executing i915_request_put() creates ++ * havoc enough. Prevent this with a compiler barrier. ++ */ ++ barrier(); ++ ++ request = i915_request_get_rcu(request); ++ ++ /* ++ * What stops the following rcu_access_pointer() from occurring ++ * before the above i915_request_get_rcu()? If we were ++ * to read the value before pausing to get the reference to ++ * the request, we may not notice a change in the active ++ * tracker. ++ * ++ * The rcu_access_pointer() is a mere compiler barrier, which ++ * means both the CPU and compiler are free to perform the ++ * memory read without constraint. The compiler only has to ++ * ensure that any operations after the rcu_access_pointer() ++ * occur afterwards in program order. This means the read may ++ * be performed earlier by an out-of-order CPU, or adventurous ++ * compiler. ++ * ++ * The atomic operation at the heart of ++ * i915_request_get_rcu(), see dma_fence_get_rcu(), is ++ * atomic_inc_not_zero() which is only a full memory barrier ++ * when successful. That is, if i915_request_get_rcu() ++ * returns the request (and so with the reference counted ++ * incremented) then the following read for rcu_access_pointer() ++ * must occur after the atomic operation and so confirm ++ * that this request is the one currently being tracked. ++ * ++ * The corresponding write barrier is part of ++ * rcu_assign_pointer(). ++ */ ++ if (!request || request == rcu_access_pointer(active->request)) ++ return rcu_pointer_handoff(request); ++ ++ i915_request_put(request); ++ } while (1); ++} ++ ++/** ++ * i915_active_request_get_unlocked - return a reference to the active request ++ * @active - the active tracker ++ * ++ * i915_active_request_get_unlocked() returns a reference to the active request, ++ * or NULL if the active tracker is idle. The reference is obtained under RCU, ++ * so no locking is required by the caller. ++ * ++ * The reference should be freed with i915_request_put(). ++ */ ++static inline struct i915_request * ++i915_active_request_get_unlocked(const struct i915_active_request *active) ++{ ++ struct i915_request *request; ++ ++ rcu_read_lock(); ++ request = __i915_active_request_get_rcu(active); ++ rcu_read_unlock(); ++ ++ return request; ++} ++ ++/** ++ * i915_active_request_isset - report whether the active tracker is assigned ++ * @active - the active tracker ++ * ++ * i915_active_request_isset() returns true if the active tracker is currently ++ * assigned to a request. Due to the lazy retiring, that request may be idle ++ * and this may report stale information. ++ */ ++static inline bool ++i915_active_request_isset(const struct i915_active_request *active) ++{ ++ return rcu_access_pointer(active->request); ++} ++ ++/** ++ * i915_active_request_retire - waits until the request is retired ++ * @active - the active request on which to wait ++ * ++ * i915_active_request_retire() waits until the request is completed, ++ * and then ensures that at least the retirement handler for this ++ * @active tracker is called before returning. If the @active ++ * tracker is idle, the function returns immediately. ++ */ ++static inline int __must_check ++i915_active_request_retire(struct i915_active_request *active, ++ struct mutex *mutex) ++{ ++ struct i915_request *request; ++ long ret; ++ ++ request = i915_active_request_raw(active, mutex); ++ if (!request) ++ return 0; ++ ++ ret = i915_request_wait(request, ++ I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED, ++ MAX_SCHEDULE_TIMEOUT); ++ if (ret < 0) ++ return ret; ++ ++ list_del_init(&active->link); ++ RCU_INIT_POINTER(active->request, NULL); ++ ++ active->retire(active, request); ++ ++ return 0; ++} ++ ++/* ++ * GPU activity tracking ++ * ++ * Each set of commands submitted to the GPU compromises a single request that ++ * signals a fence upon completion. struct i915_request combines the ++ * command submission, scheduling and fence signaling roles. If we want to see ++ * if a particular task is complete, we need to grab the fence (struct ++ * i915_request) for that task and check or wait for it to be signaled. More ++ * often though we want to track the status of a bunch of tasks, for example ++ * to wait for the GPU to finish accessing some memory across a variety of ++ * different command pipelines from different clients. We could choose to ++ * track every single request associated with the task, but knowing that ++ * each request belongs to an ordered timeline (later requests within a ++ * timeline must wait for earlier requests), we need only track the ++ * latest request in each timeline to determine the overall status of the ++ * task. ++ * ++ * struct i915_active provides this tracking across timelines. It builds a ++ * composite shared-fence, and is updated as new work is submitted to the task, ++ * forming a snapshot of the current status. It should be embedded into the ++ * different resources that need to track their associated GPU activity to ++ * provide a callback when that GPU activity has ceased, or otherwise to ++ * provide a serialisation point either for request submission or for CPU ++ * synchronisation. ++ */ ++ ++void i915_active_init(struct drm_i915_private *i915, ++ struct i915_active *ref, ++ void (*retire)(struct i915_active *ref)); ++ ++int i915_active_ref(struct i915_active *ref, ++ u64 timeline, ++ struct i915_request *rq); ++ ++int i915_active_wait(struct i915_active *ref); ++ ++int i915_request_await_active(struct i915_request *rq, ++ struct i915_active *ref); ++int i915_request_await_active_request(struct i915_request *rq, ++ struct i915_active_request *active); ++ ++bool i915_active_acquire(struct i915_active *ref); ++ ++static inline void i915_active_cancel(struct i915_active *ref) ++{ ++ GEM_BUG_ON(ref->count != 1); ++ ref->count = 0; ++} ++ ++void i915_active_release(struct i915_active *ref); ++ ++static inline bool ++i915_active_is_idle(const struct i915_active *ref) ++{ ++ return !ref->count; ++} ++ ++#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) ++void i915_active_fini(struct i915_active *ref); ++#else ++static inline void i915_active_fini(struct i915_active *ref) { } ++#endif ++ ++#endif /* _I915_ACTIVE_H_ */ +diff --git a/drivers/gpu/drm/i915_legacy/i915_active_types.h b/drivers/gpu/drm/i915_legacy/i915_active_types.h +new file mode 100644 +index 000000000000..b679253b53a5 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_active_types.h +@@ -0,0 +1,36 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2019 Intel Corporation ++ */ ++ ++#ifndef _I915_ACTIVE_TYPES_H_ ++#define _I915_ACTIVE_TYPES_H_ ++ ++#include ++#include ++ ++struct drm_i915_private; ++struct i915_active_request; ++struct i915_request; ++ ++typedef void (*i915_active_retire_fn)(struct i915_active_request *, ++ struct i915_request *); ++ ++struct i915_active_request { ++ struct i915_request __rcu *request; ++ struct list_head link; ++ i915_active_retire_fn retire; ++}; ++ ++struct i915_active { ++ struct drm_i915_private *i915; ++ ++ struct rb_root tree; ++ struct i915_active_request last; ++ unsigned int count; ++ ++ void (*retire)(struct i915_active *ref); ++}; ++ ++#endif /* _I915_ACTIVE_TYPES_H_ */ +diff --git a/drivers/gpu/drm/i915_legacy/i915_cmd_parser.c b/drivers/gpu/drm/i915_legacy/i915_cmd_parser.c +new file mode 100644 +index 000000000000..503d548a55f7 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_cmd_parser.c +@@ -0,0 +1,1387 @@ ++/* ++ * Copyright © 2013 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ * Authors: ++ * Brad Volkin ++ * ++ */ ++ ++#include "i915_drv.h" ++#include "intel_ringbuffer.h" ++ ++/** ++ * DOC: batch buffer command parser ++ * ++ * Motivation: ++ * Certain OpenGL features (e.g. transform feedback, performance monitoring) ++ * require userspace code to submit batches containing commands such as ++ * MI_LOAD_REGISTER_IMM to access various registers. Unfortunately, some ++ * generations of the hardware will noop these commands in "unsecure" batches ++ * (which includes all userspace batches submitted via i915) even though the ++ * commands may be safe and represent the intended programming model of the ++ * device. ++ * ++ * The software command parser is similar in operation to the command parsing ++ * done in hardware for unsecure batches. However, the software parser allows ++ * some operations that would be noop'd by hardware, if the parser determines ++ * the operation is safe, and submits the batch as "secure" to prevent hardware ++ * parsing. ++ * ++ * Threats: ++ * At a high level, the hardware (and software) checks attempt to prevent ++ * granting userspace undue privileges. There are three categories of privilege. ++ * ++ * First, commands which are explicitly defined as privileged or which should ++ * only be used by the kernel driver. The parser generally rejects such ++ * commands, though it may allow some from the drm master process. ++ * ++ * Second, commands which access registers. To support correct/enhanced ++ * userspace functionality, particularly certain OpenGL extensions, the parser ++ * provides a whitelist of registers which userspace may safely access (for both ++ * normal and drm master processes). ++ * ++ * Third, commands which access privileged memory (i.e. GGTT, HWS page, etc). ++ * The parser always rejects such commands. ++ * ++ * The majority of the problematic commands fall in the MI_* range, with only a ++ * few specific commands on each engine (e.g. PIPE_CONTROL and MI_FLUSH_DW). ++ * ++ * Implementation: ++ * Each engine maintains tables of commands and registers which the parser ++ * uses in scanning batch buffers submitted to that engine. ++ * ++ * Since the set of commands that the parser must check for is significantly ++ * smaller than the number of commands supported, the parser tables contain only ++ * those commands required by the parser. This generally works because command ++ * opcode ranges have standard command length encodings. So for commands that ++ * the parser does not need to check, it can easily skip them. This is ++ * implemented via a per-engine length decoding vfunc. ++ * ++ * Unfortunately, there are a number of commands that do not follow the standard ++ * length encoding for their opcode range, primarily amongst the MI_* commands. ++ * To handle this, the parser provides a way to define explicit "skip" entries ++ * in the per-engine command tables. ++ * ++ * Other command table entries map fairly directly to high level categories ++ * mentioned above: rejected, master-only, register whitelist. The parser ++ * implements a number of checks, including the privileged memory checks, via a ++ * general bitmasking mechanism. ++ */ ++ ++/* ++ * A command that requires special handling by the command parser. ++ */ ++struct drm_i915_cmd_descriptor { ++ /* ++ * Flags describing how the command parser processes the command. ++ * ++ * CMD_DESC_FIXED: The command has a fixed length if this is set, ++ * a length mask if not set ++ * CMD_DESC_SKIP: The command is allowed but does not follow the ++ * standard length encoding for the opcode range in ++ * which it falls ++ * CMD_DESC_REJECT: The command is never allowed ++ * CMD_DESC_REGISTER: The command should be checked against the ++ * register whitelist for the appropriate ring ++ * CMD_DESC_MASTER: The command is allowed if the submitting process ++ * is the DRM master ++ */ ++ u32 flags; ++#define CMD_DESC_FIXED (1<<0) ++#define CMD_DESC_SKIP (1<<1) ++#define CMD_DESC_REJECT (1<<2) ++#define CMD_DESC_REGISTER (1<<3) ++#define CMD_DESC_BITMASK (1<<4) ++#define CMD_DESC_MASTER (1<<5) ++ ++ /* ++ * The command's unique identification bits and the bitmask to get them. ++ * This isn't strictly the opcode field as defined in the spec and may ++ * also include type, subtype, and/or subop fields. ++ */ ++ struct { ++ u32 value; ++ u32 mask; ++ } cmd; ++ ++ /* ++ * The command's length. The command is either fixed length (i.e. does ++ * not include a length field) or has a length field mask. The flag ++ * CMD_DESC_FIXED indicates a fixed length. Otherwise, the command has ++ * a length mask. All command entries in a command table must include ++ * length information. ++ */ ++ union { ++ u32 fixed; ++ u32 mask; ++ } length; ++ ++ /* ++ * Describes where to find a register address in the command to check ++ * against the ring's register whitelist. Only valid if flags has the ++ * CMD_DESC_REGISTER bit set. ++ * ++ * A non-zero step value implies that the command may access multiple ++ * registers in sequence (e.g. LRI), in that case step gives the ++ * distance in dwords between individual offset fields. ++ */ ++ struct { ++ u32 offset; ++ u32 mask; ++ u32 step; ++ } reg; ++ ++#define MAX_CMD_DESC_BITMASKS 3 ++ /* ++ * Describes command checks where a particular dword is masked and ++ * compared against an expected value. If the command does not match ++ * the expected value, the parser rejects it. Only valid if flags has ++ * the CMD_DESC_BITMASK bit set. Only entries where mask is non-zero ++ * are valid. ++ * ++ * If the check specifies a non-zero condition_mask then the parser ++ * only performs the check when the bits specified by condition_mask ++ * are non-zero. ++ */ ++ struct { ++ u32 offset; ++ u32 mask; ++ u32 expected; ++ u32 condition_offset; ++ u32 condition_mask; ++ } bits[MAX_CMD_DESC_BITMASKS]; ++}; ++ ++/* ++ * A table of commands requiring special handling by the command parser. ++ * ++ * Each engine has an array of tables. Each table consists of an array of ++ * command descriptors, which must be sorted with command opcodes in ++ * ascending order. ++ */ ++struct drm_i915_cmd_table { ++ const struct drm_i915_cmd_descriptor *table; ++ int count; ++}; ++ ++#define STD_MI_OPCODE_SHIFT (32 - 9) ++#define STD_3D_OPCODE_SHIFT (32 - 16) ++#define STD_2D_OPCODE_SHIFT (32 - 10) ++#define STD_MFX_OPCODE_SHIFT (32 - 16) ++#define MIN_OPCODE_SHIFT 16 ++ ++#define CMD(op, opm, f, lm, fl, ...) \ ++ { \ ++ .flags = (fl) | ((f) ? CMD_DESC_FIXED : 0), \ ++ .cmd = { (op), ~0u << (opm) }, \ ++ .length = { (lm) }, \ ++ __VA_ARGS__ \ ++ } ++ ++/* Convenience macros to compress the tables */ ++#define SMI STD_MI_OPCODE_SHIFT ++#define S3D STD_3D_OPCODE_SHIFT ++#define S2D STD_2D_OPCODE_SHIFT ++#define SMFX STD_MFX_OPCODE_SHIFT ++#define F true ++#define S CMD_DESC_SKIP ++#define R CMD_DESC_REJECT ++#define W CMD_DESC_REGISTER ++#define B CMD_DESC_BITMASK ++#define M CMD_DESC_MASTER ++ ++/* Command Mask Fixed Len Action ++ ---------------------------------------------------------- */ ++static const struct drm_i915_cmd_descriptor common_cmds[] = { ++ CMD( MI_NOOP, SMI, F, 1, S ), ++ CMD( MI_USER_INTERRUPT, SMI, F, 1, R ), ++ CMD( MI_WAIT_FOR_EVENT, SMI, F, 1, M ), ++ CMD( MI_ARB_CHECK, SMI, F, 1, S ), ++ CMD( MI_REPORT_HEAD, SMI, F, 1, S ), ++ CMD( MI_SUSPEND_FLUSH, SMI, F, 1, S ), ++ CMD( MI_SEMAPHORE_MBOX, SMI, !F, 0xFF, R ), ++ CMD( MI_STORE_DWORD_INDEX, SMI, !F, 0xFF, R ), ++ CMD( MI_LOAD_REGISTER_IMM(1), SMI, !F, 0xFF, W, ++ .reg = { .offset = 1, .mask = 0x007FFFFC, .step = 2 } ), ++ CMD( MI_STORE_REGISTER_MEM, SMI, F, 3, W | B, ++ .reg = { .offset = 1, .mask = 0x007FFFFC }, ++ .bits = {{ ++ .offset = 0, ++ .mask = MI_GLOBAL_GTT, ++ .expected = 0, ++ }}, ), ++ CMD( MI_LOAD_REGISTER_MEM, SMI, F, 3, W | B, ++ .reg = { .offset = 1, .mask = 0x007FFFFC }, ++ .bits = {{ ++ .offset = 0, ++ .mask = MI_GLOBAL_GTT, ++ .expected = 0, ++ }}, ), ++ /* ++ * MI_BATCH_BUFFER_START requires some special handling. It's not ++ * really a 'skip' action but it doesn't seem like it's worth adding ++ * a new action. See i915_parse_cmds(). ++ */ ++ CMD( MI_BATCH_BUFFER_START, SMI, !F, 0xFF, S ), ++}; ++ ++static const struct drm_i915_cmd_descriptor render_cmds[] = { ++ CMD( MI_FLUSH, SMI, F, 1, S ), ++ CMD( MI_ARB_ON_OFF, SMI, F, 1, R ), ++ CMD( MI_PREDICATE, SMI, F, 1, S ), ++ CMD( MI_TOPOLOGY_FILTER, SMI, F, 1, S ), ++ CMD( MI_SET_APPID, SMI, F, 1, S ), ++ CMD( MI_DISPLAY_FLIP, SMI, !F, 0xFF, R ), ++ CMD( MI_SET_CONTEXT, SMI, !F, 0xFF, R ), ++ CMD( MI_URB_CLEAR, SMI, !F, 0xFF, S ), ++ CMD( MI_STORE_DWORD_IMM, SMI, !F, 0x3F, B, ++ .bits = {{ ++ .offset = 0, ++ .mask = MI_GLOBAL_GTT, ++ .expected = 0, ++ }}, ), ++ CMD( MI_UPDATE_GTT, SMI, !F, 0xFF, R ), ++ CMD( MI_CLFLUSH, SMI, !F, 0x3FF, B, ++ .bits = {{ ++ .offset = 0, ++ .mask = MI_GLOBAL_GTT, ++ .expected = 0, ++ }}, ), ++ CMD( MI_REPORT_PERF_COUNT, SMI, !F, 0x3F, B, ++ .bits = {{ ++ .offset = 1, ++ .mask = MI_REPORT_PERF_COUNT_GGTT, ++ .expected = 0, ++ }}, ), ++ CMD( MI_CONDITIONAL_BATCH_BUFFER_END, SMI, !F, 0xFF, B, ++ .bits = {{ ++ .offset = 0, ++ .mask = MI_GLOBAL_GTT, ++ .expected = 0, ++ }}, ), ++ CMD( GFX_OP_3DSTATE_VF_STATISTICS, S3D, F, 1, S ), ++ CMD( PIPELINE_SELECT, S3D, F, 1, S ), ++ CMD( MEDIA_VFE_STATE, S3D, !F, 0xFFFF, B, ++ .bits = {{ ++ .offset = 2, ++ .mask = MEDIA_VFE_STATE_MMIO_ACCESS_MASK, ++ .expected = 0, ++ }}, ), ++ CMD( GPGPU_OBJECT, S3D, !F, 0xFF, S ), ++ CMD( GPGPU_WALKER, S3D, !F, 0xFF, S ), ++ CMD( GFX_OP_3DSTATE_SO_DECL_LIST, S3D, !F, 0x1FF, S ), ++ CMD( GFX_OP_PIPE_CONTROL(5), S3D, !F, 0xFF, B, ++ .bits = {{ ++ .offset = 1, ++ .mask = (PIPE_CONTROL_MMIO_WRITE | PIPE_CONTROL_NOTIFY), ++ .expected = 0, ++ }, ++ { ++ .offset = 1, ++ .mask = (PIPE_CONTROL_GLOBAL_GTT_IVB | ++ PIPE_CONTROL_STORE_DATA_INDEX), ++ .expected = 0, ++ .condition_offset = 1, ++ .condition_mask = PIPE_CONTROL_POST_SYNC_OP_MASK, ++ }}, ), ++}; ++ ++static const struct drm_i915_cmd_descriptor hsw_render_cmds[] = { ++ CMD( MI_SET_PREDICATE, SMI, F, 1, S ), ++ CMD( MI_RS_CONTROL, SMI, F, 1, S ), ++ CMD( MI_URB_ATOMIC_ALLOC, SMI, F, 1, S ), ++ CMD( MI_SET_APPID, SMI, F, 1, S ), ++ CMD( MI_RS_CONTEXT, SMI, F, 1, S ), ++ CMD( MI_LOAD_SCAN_LINES_INCL, SMI, !F, 0x3F, M ), ++ CMD( MI_LOAD_SCAN_LINES_EXCL, SMI, !F, 0x3F, R ), ++ CMD( MI_LOAD_REGISTER_REG, SMI, !F, 0xFF, W, ++ .reg = { .offset = 1, .mask = 0x007FFFFC, .step = 1 } ), ++ CMD( MI_RS_STORE_DATA_IMM, SMI, !F, 0xFF, S ), ++ CMD( MI_LOAD_URB_MEM, SMI, !F, 0xFF, S ), ++ CMD( MI_STORE_URB_MEM, SMI, !F, 0xFF, S ), ++ CMD( GFX_OP_3DSTATE_DX9_CONSTANTF_VS, S3D, !F, 0x7FF, S ), ++ CMD( GFX_OP_3DSTATE_DX9_CONSTANTF_PS, S3D, !F, 0x7FF, S ), ++ ++ CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_VS, S3D, !F, 0x1FF, S ), ++ CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_GS, S3D, !F, 0x1FF, S ), ++ CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_HS, S3D, !F, 0x1FF, S ), ++ CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_DS, S3D, !F, 0x1FF, S ), ++ CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_PS, S3D, !F, 0x1FF, S ), ++}; ++ ++static const struct drm_i915_cmd_descriptor video_cmds[] = { ++ CMD( MI_ARB_ON_OFF, SMI, F, 1, R ), ++ CMD( MI_SET_APPID, SMI, F, 1, S ), ++ CMD( MI_STORE_DWORD_IMM, SMI, !F, 0xFF, B, ++ .bits = {{ ++ .offset = 0, ++ .mask = MI_GLOBAL_GTT, ++ .expected = 0, ++ }}, ), ++ CMD( MI_UPDATE_GTT, SMI, !F, 0x3F, R ), ++ CMD( MI_FLUSH_DW, SMI, !F, 0x3F, B, ++ .bits = {{ ++ .offset = 0, ++ .mask = MI_FLUSH_DW_NOTIFY, ++ .expected = 0, ++ }, ++ { ++ .offset = 1, ++ .mask = MI_FLUSH_DW_USE_GTT, ++ .expected = 0, ++ .condition_offset = 0, ++ .condition_mask = MI_FLUSH_DW_OP_MASK, ++ }, ++ { ++ .offset = 0, ++ .mask = MI_FLUSH_DW_STORE_INDEX, ++ .expected = 0, ++ .condition_offset = 0, ++ .condition_mask = MI_FLUSH_DW_OP_MASK, ++ }}, ), ++ CMD( MI_CONDITIONAL_BATCH_BUFFER_END, SMI, !F, 0xFF, B, ++ .bits = {{ ++ .offset = 0, ++ .mask = MI_GLOBAL_GTT, ++ .expected = 0, ++ }}, ), ++ /* ++ * MFX_WAIT doesn't fit the way we handle length for most commands. ++ * It has a length field but it uses a non-standard length bias. ++ * It is always 1 dword though, so just treat it as fixed length. ++ */ ++ CMD( MFX_WAIT, SMFX, F, 1, S ), ++}; ++ ++static const struct drm_i915_cmd_descriptor vecs_cmds[] = { ++ CMD( MI_ARB_ON_OFF, SMI, F, 1, R ), ++ CMD( MI_SET_APPID, SMI, F, 1, S ), ++ CMD( MI_STORE_DWORD_IMM, SMI, !F, 0xFF, B, ++ .bits = {{ ++ .offset = 0, ++ .mask = MI_GLOBAL_GTT, ++ .expected = 0, ++ }}, ), ++ CMD( MI_UPDATE_GTT, SMI, !F, 0x3F, R ), ++ CMD( MI_FLUSH_DW, SMI, !F, 0x3F, B, ++ .bits = {{ ++ .offset = 0, ++ .mask = MI_FLUSH_DW_NOTIFY, ++ .expected = 0, ++ }, ++ { ++ .offset = 1, ++ .mask = MI_FLUSH_DW_USE_GTT, ++ .expected = 0, ++ .condition_offset = 0, ++ .condition_mask = MI_FLUSH_DW_OP_MASK, ++ }, ++ { ++ .offset = 0, ++ .mask = MI_FLUSH_DW_STORE_INDEX, ++ .expected = 0, ++ .condition_offset = 0, ++ .condition_mask = MI_FLUSH_DW_OP_MASK, ++ }}, ), ++ CMD( MI_CONDITIONAL_BATCH_BUFFER_END, SMI, !F, 0xFF, B, ++ .bits = {{ ++ .offset = 0, ++ .mask = MI_GLOBAL_GTT, ++ .expected = 0, ++ }}, ), ++}; ++ ++static const struct drm_i915_cmd_descriptor blt_cmds[] = { ++ CMD( MI_DISPLAY_FLIP, SMI, !F, 0xFF, R ), ++ CMD( MI_STORE_DWORD_IMM, SMI, !F, 0x3FF, B, ++ .bits = {{ ++ .offset = 0, ++ .mask = MI_GLOBAL_GTT, ++ .expected = 0, ++ }}, ), ++ CMD( MI_UPDATE_GTT, SMI, !F, 0x3F, R ), ++ CMD( MI_FLUSH_DW, SMI, !F, 0x3F, B, ++ .bits = {{ ++ .offset = 0, ++ .mask = MI_FLUSH_DW_NOTIFY, ++ .expected = 0, ++ }, ++ { ++ .offset = 1, ++ .mask = MI_FLUSH_DW_USE_GTT, ++ .expected = 0, ++ .condition_offset = 0, ++ .condition_mask = MI_FLUSH_DW_OP_MASK, ++ }, ++ { ++ .offset = 0, ++ .mask = MI_FLUSH_DW_STORE_INDEX, ++ .expected = 0, ++ .condition_offset = 0, ++ .condition_mask = MI_FLUSH_DW_OP_MASK, ++ }}, ), ++ CMD( COLOR_BLT, S2D, !F, 0x3F, S ), ++ CMD( SRC_COPY_BLT, S2D, !F, 0x3F, S ), ++}; ++ ++static const struct drm_i915_cmd_descriptor hsw_blt_cmds[] = { ++ CMD( MI_LOAD_SCAN_LINES_INCL, SMI, !F, 0x3F, M ), ++ CMD( MI_LOAD_SCAN_LINES_EXCL, SMI, !F, 0x3F, R ), ++}; ++ ++static const struct drm_i915_cmd_descriptor noop_desc = ++ CMD(MI_NOOP, SMI, F, 1, S); ++ ++#undef CMD ++#undef SMI ++#undef S3D ++#undef S2D ++#undef SMFX ++#undef F ++#undef S ++#undef R ++#undef W ++#undef B ++#undef M ++ ++static const struct drm_i915_cmd_table gen7_render_cmds[] = { ++ { common_cmds, ARRAY_SIZE(common_cmds) }, ++ { render_cmds, ARRAY_SIZE(render_cmds) }, ++}; ++ ++static const struct drm_i915_cmd_table hsw_render_ring_cmds[] = { ++ { common_cmds, ARRAY_SIZE(common_cmds) }, ++ { render_cmds, ARRAY_SIZE(render_cmds) }, ++ { hsw_render_cmds, ARRAY_SIZE(hsw_render_cmds) }, ++}; ++ ++static const struct drm_i915_cmd_table gen7_video_cmds[] = { ++ { common_cmds, ARRAY_SIZE(common_cmds) }, ++ { video_cmds, ARRAY_SIZE(video_cmds) }, ++}; ++ ++static const struct drm_i915_cmd_table hsw_vebox_cmds[] = { ++ { common_cmds, ARRAY_SIZE(common_cmds) }, ++ { vecs_cmds, ARRAY_SIZE(vecs_cmds) }, ++}; ++ ++static const struct drm_i915_cmd_table gen7_blt_cmds[] = { ++ { common_cmds, ARRAY_SIZE(common_cmds) }, ++ { blt_cmds, ARRAY_SIZE(blt_cmds) }, ++}; ++ ++static const struct drm_i915_cmd_table hsw_blt_ring_cmds[] = { ++ { common_cmds, ARRAY_SIZE(common_cmds) }, ++ { blt_cmds, ARRAY_SIZE(blt_cmds) }, ++ { hsw_blt_cmds, ARRAY_SIZE(hsw_blt_cmds) }, ++}; ++ ++/* ++ * Register whitelists, sorted by increasing register offset. ++ */ ++ ++/* ++ * An individual whitelist entry granting access to register addr. If ++ * mask is non-zero the argument of immediate register writes will be ++ * AND-ed with mask, and the command will be rejected if the result ++ * doesn't match value. ++ * ++ * Registers with non-zero mask are only allowed to be written using ++ * LRI. ++ */ ++struct drm_i915_reg_descriptor { ++ i915_reg_t addr; ++ u32 mask; ++ u32 value; ++}; ++ ++/* Convenience macro for adding 32-bit registers. */ ++#define REG32(_reg, ...) \ ++ { .addr = (_reg), __VA_ARGS__ } ++ ++/* ++ * Convenience macro for adding 64-bit registers. ++ * ++ * Some registers that userspace accesses are 64 bits. The register ++ * access commands only allow 32-bit accesses. Hence, we have to include ++ * entries for both halves of the 64-bit registers. ++ */ ++#define REG64(_reg) \ ++ { .addr = _reg }, \ ++ { .addr = _reg ## _UDW } ++ ++#define REG64_IDX(_reg, idx) \ ++ { .addr = _reg(idx) }, \ ++ { .addr = _reg ## _UDW(idx) } ++ ++static const struct drm_i915_reg_descriptor gen7_render_regs[] = { ++ REG64(GPGPU_THREADS_DISPATCHED), ++ REG64(HS_INVOCATION_COUNT), ++ REG64(DS_INVOCATION_COUNT), ++ REG64(IA_VERTICES_COUNT), ++ REG64(IA_PRIMITIVES_COUNT), ++ REG64(VS_INVOCATION_COUNT), ++ REG64(GS_INVOCATION_COUNT), ++ REG64(GS_PRIMITIVES_COUNT), ++ REG64(CL_INVOCATION_COUNT), ++ REG64(CL_PRIMITIVES_COUNT), ++ REG64(PS_INVOCATION_COUNT), ++ REG64(PS_DEPTH_COUNT), ++ REG64_IDX(RING_TIMESTAMP, RENDER_RING_BASE), ++ REG64(MI_PREDICATE_SRC0), ++ REG64(MI_PREDICATE_SRC1), ++ REG32(GEN7_3DPRIM_END_OFFSET), ++ REG32(GEN7_3DPRIM_START_VERTEX), ++ REG32(GEN7_3DPRIM_VERTEX_COUNT), ++ REG32(GEN7_3DPRIM_INSTANCE_COUNT), ++ REG32(GEN7_3DPRIM_START_INSTANCE), ++ REG32(GEN7_3DPRIM_BASE_VERTEX), ++ REG32(GEN7_GPGPU_DISPATCHDIMX), ++ REG32(GEN7_GPGPU_DISPATCHDIMY), ++ REG32(GEN7_GPGPU_DISPATCHDIMZ), ++ REG64_IDX(RING_TIMESTAMP, BSD_RING_BASE), ++ REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 0), ++ REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 1), ++ REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 2), ++ REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 3), ++ REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 0), ++ REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 1), ++ REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 2), ++ REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 3), ++ REG32(GEN7_SO_WRITE_OFFSET(0)), ++ REG32(GEN7_SO_WRITE_OFFSET(1)), ++ REG32(GEN7_SO_WRITE_OFFSET(2)), ++ REG32(GEN7_SO_WRITE_OFFSET(3)), ++ REG32(GEN7_L3SQCREG1), ++ REG32(GEN7_L3CNTLREG2), ++ REG32(GEN7_L3CNTLREG3), ++ REG64_IDX(RING_TIMESTAMP, BLT_RING_BASE), ++}; ++ ++static const struct drm_i915_reg_descriptor hsw_render_regs[] = { ++ REG64_IDX(HSW_CS_GPR, 0), ++ REG64_IDX(HSW_CS_GPR, 1), ++ REG64_IDX(HSW_CS_GPR, 2), ++ REG64_IDX(HSW_CS_GPR, 3), ++ REG64_IDX(HSW_CS_GPR, 4), ++ REG64_IDX(HSW_CS_GPR, 5), ++ REG64_IDX(HSW_CS_GPR, 6), ++ REG64_IDX(HSW_CS_GPR, 7), ++ REG64_IDX(HSW_CS_GPR, 8), ++ REG64_IDX(HSW_CS_GPR, 9), ++ REG64_IDX(HSW_CS_GPR, 10), ++ REG64_IDX(HSW_CS_GPR, 11), ++ REG64_IDX(HSW_CS_GPR, 12), ++ REG64_IDX(HSW_CS_GPR, 13), ++ REG64_IDX(HSW_CS_GPR, 14), ++ REG64_IDX(HSW_CS_GPR, 15), ++ REG32(HSW_SCRATCH1, ++ .mask = ~HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE, ++ .value = 0), ++ REG32(HSW_ROW_CHICKEN3, ++ .mask = ~(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE << 16 | ++ HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE), ++ .value = 0), ++}; ++ ++static const struct drm_i915_reg_descriptor gen7_blt_regs[] = { ++ REG64_IDX(RING_TIMESTAMP, RENDER_RING_BASE), ++ REG64_IDX(RING_TIMESTAMP, BSD_RING_BASE), ++ REG32(BCS_SWCTRL), ++ REG64_IDX(RING_TIMESTAMP, BLT_RING_BASE), ++}; ++ ++static const struct drm_i915_reg_descriptor ivb_master_regs[] = { ++ REG32(FORCEWAKE_MT), ++ REG32(DERRMR), ++ REG32(GEN7_PIPE_DE_LOAD_SL(PIPE_A)), ++ REG32(GEN7_PIPE_DE_LOAD_SL(PIPE_B)), ++ REG32(GEN7_PIPE_DE_LOAD_SL(PIPE_C)), ++}; ++ ++static const struct drm_i915_reg_descriptor hsw_master_regs[] = { ++ REG32(FORCEWAKE_MT), ++ REG32(DERRMR), ++}; ++ ++#undef REG64 ++#undef REG32 ++ ++struct drm_i915_reg_table { ++ const struct drm_i915_reg_descriptor *regs; ++ int num_regs; ++ bool master; ++}; ++ ++static const struct drm_i915_reg_table ivb_render_reg_tables[] = { ++ { gen7_render_regs, ARRAY_SIZE(gen7_render_regs), false }, ++ { ivb_master_regs, ARRAY_SIZE(ivb_master_regs), true }, ++}; ++ ++static const struct drm_i915_reg_table ivb_blt_reg_tables[] = { ++ { gen7_blt_regs, ARRAY_SIZE(gen7_blt_regs), false }, ++ { ivb_master_regs, ARRAY_SIZE(ivb_master_regs), true }, ++}; ++ ++static const struct drm_i915_reg_table hsw_render_reg_tables[] = { ++ { gen7_render_regs, ARRAY_SIZE(gen7_render_regs), false }, ++ { hsw_render_regs, ARRAY_SIZE(hsw_render_regs), false }, ++ { hsw_master_regs, ARRAY_SIZE(hsw_master_regs), true }, ++}; ++ ++static const struct drm_i915_reg_table hsw_blt_reg_tables[] = { ++ { gen7_blt_regs, ARRAY_SIZE(gen7_blt_regs), false }, ++ { hsw_master_regs, ARRAY_SIZE(hsw_master_regs), true }, ++}; ++ ++static u32 gen7_render_get_cmd_length_mask(u32 cmd_header) ++{ ++ u32 client = cmd_header >> INSTR_CLIENT_SHIFT; ++ u32 subclient = ++ (cmd_header & INSTR_SUBCLIENT_MASK) >> INSTR_SUBCLIENT_SHIFT; ++ ++ if (client == INSTR_MI_CLIENT) ++ return 0x3F; ++ else if (client == INSTR_RC_CLIENT) { ++ if (subclient == INSTR_MEDIA_SUBCLIENT) ++ return 0xFFFF; ++ else ++ return 0xFF; ++ } ++ ++ DRM_DEBUG_DRIVER("CMD: Abnormal rcs cmd length! 0x%08X\n", cmd_header); ++ return 0; ++} ++ ++static u32 gen7_bsd_get_cmd_length_mask(u32 cmd_header) ++{ ++ u32 client = cmd_header >> INSTR_CLIENT_SHIFT; ++ u32 subclient = ++ (cmd_header & INSTR_SUBCLIENT_MASK) >> INSTR_SUBCLIENT_SHIFT; ++ u32 op = (cmd_header & INSTR_26_TO_24_MASK) >> INSTR_26_TO_24_SHIFT; ++ ++ if (client == INSTR_MI_CLIENT) ++ return 0x3F; ++ else if (client == INSTR_RC_CLIENT) { ++ if (subclient == INSTR_MEDIA_SUBCLIENT) { ++ if (op == 6) ++ return 0xFFFF; ++ else ++ return 0xFFF; ++ } else ++ return 0xFF; ++ } ++ ++ DRM_DEBUG_DRIVER("CMD: Abnormal bsd cmd length! 0x%08X\n", cmd_header); ++ return 0; ++} ++ ++static u32 gen7_blt_get_cmd_length_mask(u32 cmd_header) ++{ ++ u32 client = cmd_header >> INSTR_CLIENT_SHIFT; ++ ++ if (client == INSTR_MI_CLIENT) ++ return 0x3F; ++ else if (client == INSTR_BC_CLIENT) ++ return 0xFF; ++ ++ DRM_DEBUG_DRIVER("CMD: Abnormal blt cmd length! 0x%08X\n", cmd_header); ++ return 0; ++} ++ ++static bool validate_cmds_sorted(const struct intel_engine_cs *engine, ++ const struct drm_i915_cmd_table *cmd_tables, ++ int cmd_table_count) ++{ ++ int i; ++ bool ret = true; ++ ++ if (!cmd_tables || cmd_table_count == 0) ++ return true; ++ ++ for (i = 0; i < cmd_table_count; i++) { ++ const struct drm_i915_cmd_table *table = &cmd_tables[i]; ++ u32 previous = 0; ++ int j; ++ ++ for (j = 0; j < table->count; j++) { ++ const struct drm_i915_cmd_descriptor *desc = ++ &table->table[j]; ++ u32 curr = desc->cmd.value & desc->cmd.mask; ++ ++ if (curr < previous) { ++ DRM_ERROR("CMD: %s [%d] command table not sorted: " ++ "table=%d entry=%d cmd=0x%08X prev=0x%08X\n", ++ engine->name, engine->id, ++ i, j, curr, previous); ++ ret = false; ++ } ++ ++ previous = curr; ++ } ++ } ++ ++ return ret; ++} ++ ++static bool check_sorted(const struct intel_engine_cs *engine, ++ const struct drm_i915_reg_descriptor *reg_table, ++ int reg_count) ++{ ++ int i; ++ u32 previous = 0; ++ bool ret = true; ++ ++ for (i = 0; i < reg_count; i++) { ++ u32 curr = i915_mmio_reg_offset(reg_table[i].addr); ++ ++ if (curr < previous) { ++ DRM_ERROR("CMD: %s [%d] register table not sorted: " ++ "entry=%d reg=0x%08X prev=0x%08X\n", ++ engine->name, engine->id, ++ i, curr, previous); ++ ret = false; ++ } ++ ++ previous = curr; ++ } ++ ++ return ret; ++} ++ ++static bool validate_regs_sorted(struct intel_engine_cs *engine) ++{ ++ int i; ++ const struct drm_i915_reg_table *table; ++ ++ for (i = 0; i < engine->reg_table_count; i++) { ++ table = &engine->reg_tables[i]; ++ if (!check_sorted(engine, table->regs, table->num_regs)) ++ return false; ++ } ++ ++ return true; ++} ++ ++struct cmd_node { ++ const struct drm_i915_cmd_descriptor *desc; ++ struct hlist_node node; ++}; ++ ++/* ++ * Different command ranges have different numbers of bits for the opcode. For ++ * example, MI commands use bits 31:23 while 3D commands use bits 31:16. The ++ * problem is that, for example, MI commands use bits 22:16 for other fields ++ * such as GGTT vs PPGTT bits. If we include those bits in the mask then when ++ * we mask a command from a batch it could hash to the wrong bucket due to ++ * non-opcode bits being set. But if we don't include those bits, some 3D ++ * commands may hash to the same bucket due to not including opcode bits that ++ * make the command unique. For now, we will risk hashing to the same bucket. ++ */ ++static inline u32 cmd_header_key(u32 x) ++{ ++ switch (x >> INSTR_CLIENT_SHIFT) { ++ default: ++ case INSTR_MI_CLIENT: ++ return x >> STD_MI_OPCODE_SHIFT; ++ case INSTR_RC_CLIENT: ++ return x >> STD_3D_OPCODE_SHIFT; ++ case INSTR_BC_CLIENT: ++ return x >> STD_2D_OPCODE_SHIFT; ++ } ++} ++ ++static int init_hash_table(struct intel_engine_cs *engine, ++ const struct drm_i915_cmd_table *cmd_tables, ++ int cmd_table_count) ++{ ++ int i, j; ++ ++ hash_init(engine->cmd_hash); ++ ++ for (i = 0; i < cmd_table_count; i++) { ++ const struct drm_i915_cmd_table *table = &cmd_tables[i]; ++ ++ for (j = 0; j < table->count; j++) { ++ const struct drm_i915_cmd_descriptor *desc = ++ &table->table[j]; ++ struct cmd_node *desc_node = ++ kmalloc(sizeof(*desc_node), GFP_KERNEL); ++ ++ if (!desc_node) ++ return -ENOMEM; ++ ++ desc_node->desc = desc; ++ hash_add(engine->cmd_hash, &desc_node->node, ++ cmd_header_key(desc->cmd.value)); ++ } ++ } ++ ++ return 0; ++} ++ ++static void fini_hash_table(struct intel_engine_cs *engine) ++{ ++ struct hlist_node *tmp; ++ struct cmd_node *desc_node; ++ int i; ++ ++ hash_for_each_safe(engine->cmd_hash, i, tmp, desc_node, node) { ++ hash_del(&desc_node->node); ++ kfree(desc_node); ++ } ++} ++ ++/** ++ * intel_engine_init_cmd_parser() - set cmd parser related fields for an engine ++ * @engine: the engine to initialize ++ * ++ * Optionally initializes fields related to batch buffer command parsing in the ++ * struct intel_engine_cs based on whether the platform requires software ++ * command parsing. ++ */ ++void intel_engine_init_cmd_parser(struct intel_engine_cs *engine) ++{ ++ const struct drm_i915_cmd_table *cmd_tables; ++ int cmd_table_count; ++ int ret; ++ ++ if (!IS_GEN(engine->i915, 7)) ++ return; ++ ++ switch (engine->class) { ++ case RENDER_CLASS: ++ if (IS_HASWELL(engine->i915)) { ++ cmd_tables = hsw_render_ring_cmds; ++ cmd_table_count = ++ ARRAY_SIZE(hsw_render_ring_cmds); ++ } else { ++ cmd_tables = gen7_render_cmds; ++ cmd_table_count = ARRAY_SIZE(gen7_render_cmds); ++ } ++ ++ if (IS_HASWELL(engine->i915)) { ++ engine->reg_tables = hsw_render_reg_tables; ++ engine->reg_table_count = ARRAY_SIZE(hsw_render_reg_tables); ++ } else { ++ engine->reg_tables = ivb_render_reg_tables; ++ engine->reg_table_count = ARRAY_SIZE(ivb_render_reg_tables); ++ } ++ ++ engine->get_cmd_length_mask = gen7_render_get_cmd_length_mask; ++ break; ++ case VIDEO_DECODE_CLASS: ++ cmd_tables = gen7_video_cmds; ++ cmd_table_count = ARRAY_SIZE(gen7_video_cmds); ++ engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask; ++ break; ++ case COPY_ENGINE_CLASS: ++ if (IS_HASWELL(engine->i915)) { ++ cmd_tables = hsw_blt_ring_cmds; ++ cmd_table_count = ARRAY_SIZE(hsw_blt_ring_cmds); ++ } else { ++ cmd_tables = gen7_blt_cmds; ++ cmd_table_count = ARRAY_SIZE(gen7_blt_cmds); ++ } ++ ++ if (IS_HASWELL(engine->i915)) { ++ engine->reg_tables = hsw_blt_reg_tables; ++ engine->reg_table_count = ARRAY_SIZE(hsw_blt_reg_tables); ++ } else { ++ engine->reg_tables = ivb_blt_reg_tables; ++ engine->reg_table_count = ARRAY_SIZE(ivb_blt_reg_tables); ++ } ++ ++ engine->get_cmd_length_mask = gen7_blt_get_cmd_length_mask; ++ break; ++ case VIDEO_ENHANCEMENT_CLASS: ++ cmd_tables = hsw_vebox_cmds; ++ cmd_table_count = ARRAY_SIZE(hsw_vebox_cmds); ++ /* VECS can use the same length_mask function as VCS */ ++ engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask; ++ break; ++ default: ++ MISSING_CASE(engine->class); ++ return; ++ } ++ ++ if (!validate_cmds_sorted(engine, cmd_tables, cmd_table_count)) { ++ DRM_ERROR("%s: command descriptions are not sorted\n", ++ engine->name); ++ return; ++ } ++ if (!validate_regs_sorted(engine)) { ++ DRM_ERROR("%s: registers are not sorted\n", engine->name); ++ return; ++ } ++ ++ ret = init_hash_table(engine, cmd_tables, cmd_table_count); ++ if (ret) { ++ DRM_ERROR("%s: initialised failed!\n", engine->name); ++ fini_hash_table(engine); ++ return; ++ } ++ ++ engine->flags |= I915_ENGINE_NEEDS_CMD_PARSER; ++} ++ ++/** ++ * intel_engine_cleanup_cmd_parser() - clean up cmd parser related fields ++ * @engine: the engine to clean up ++ * ++ * Releases any resources related to command parsing that may have been ++ * initialized for the specified engine. ++ */ ++void intel_engine_cleanup_cmd_parser(struct intel_engine_cs *engine) ++{ ++ if (!intel_engine_needs_cmd_parser(engine)) ++ return; ++ ++ fini_hash_table(engine); ++} ++ ++static const struct drm_i915_cmd_descriptor* ++find_cmd_in_table(struct intel_engine_cs *engine, ++ u32 cmd_header) ++{ ++ struct cmd_node *desc_node; ++ ++ hash_for_each_possible(engine->cmd_hash, desc_node, node, ++ cmd_header_key(cmd_header)) { ++ const struct drm_i915_cmd_descriptor *desc = desc_node->desc; ++ if (((cmd_header ^ desc->cmd.value) & desc->cmd.mask) == 0) ++ return desc; ++ } ++ ++ return NULL; ++} ++ ++/* ++ * Returns a pointer to a descriptor for the command specified by cmd_header. ++ * ++ * The caller must supply space for a default descriptor via the default_desc ++ * parameter. If no descriptor for the specified command exists in the engine's ++ * command parser tables, this function fills in default_desc based on the ++ * engine's default length encoding and returns default_desc. ++ */ ++static const struct drm_i915_cmd_descriptor* ++find_cmd(struct intel_engine_cs *engine, ++ u32 cmd_header, ++ const struct drm_i915_cmd_descriptor *desc, ++ struct drm_i915_cmd_descriptor *default_desc) ++{ ++ u32 mask; ++ ++ if (((cmd_header ^ desc->cmd.value) & desc->cmd.mask) == 0) ++ return desc; ++ ++ desc = find_cmd_in_table(engine, cmd_header); ++ if (desc) ++ return desc; ++ ++ mask = engine->get_cmd_length_mask(cmd_header); ++ if (!mask) ++ return NULL; ++ ++ default_desc->cmd.value = cmd_header; ++ default_desc->cmd.mask = ~0u << MIN_OPCODE_SHIFT; ++ default_desc->length.mask = mask; ++ default_desc->flags = CMD_DESC_SKIP; ++ return default_desc; ++} ++ ++static const struct drm_i915_reg_descriptor * ++__find_reg(const struct drm_i915_reg_descriptor *table, int count, u32 addr) ++{ ++ int start = 0, end = count; ++ while (start < end) { ++ int mid = start + (end - start) / 2; ++ int ret = addr - i915_mmio_reg_offset(table[mid].addr); ++ if (ret < 0) ++ end = mid; ++ else if (ret > 0) ++ start = mid + 1; ++ else ++ return &table[mid]; ++ } ++ return NULL; ++} ++ ++static const struct drm_i915_reg_descriptor * ++find_reg(const struct intel_engine_cs *engine, bool is_master, u32 addr) ++{ ++ const struct drm_i915_reg_table *table = engine->reg_tables; ++ int count = engine->reg_table_count; ++ ++ for (; count > 0; ++table, --count) { ++ if (!table->master || is_master) { ++ const struct drm_i915_reg_descriptor *reg; ++ ++ reg = __find_reg(table->regs, table->num_regs, addr); ++ if (reg != NULL) ++ return reg; ++ } ++ } ++ ++ return NULL; ++} ++ ++/* Returns a vmap'd pointer to dst_obj, which the caller must unmap */ ++static u32 *copy_batch(struct drm_i915_gem_object *dst_obj, ++ struct drm_i915_gem_object *src_obj, ++ u32 batch_start_offset, ++ u32 batch_len, ++ bool *needs_clflush_after) ++{ ++ unsigned int src_needs_clflush; ++ unsigned int dst_needs_clflush; ++ void *dst, *src; ++ int ret; ++ ++ ret = i915_gem_obj_prepare_shmem_read(src_obj, &src_needs_clflush); ++ if (ret) ++ return ERR_PTR(ret); ++ ++ ret = i915_gem_obj_prepare_shmem_write(dst_obj, &dst_needs_clflush); ++ if (ret) { ++ dst = ERR_PTR(ret); ++ goto unpin_src; ++ } ++ ++ dst = i915_gem_object_pin_map(dst_obj, I915_MAP_FORCE_WB); ++ if (IS_ERR(dst)) ++ goto unpin_dst; ++ ++ src = ERR_PTR(-ENODEV); ++ if (src_needs_clflush && ++ i915_can_memcpy_from_wc(NULL, batch_start_offset, 0)) { ++ src = i915_gem_object_pin_map(src_obj, I915_MAP_WC); ++ if (!IS_ERR(src)) { ++ i915_memcpy_from_wc(dst, ++ src + batch_start_offset, ++ ALIGN(batch_len, 16)); ++ i915_gem_object_unpin_map(src_obj); ++ } ++ } ++ if (IS_ERR(src)) { ++ void *ptr; ++ int offset, n; ++ ++ offset = offset_in_page(batch_start_offset); ++ ++ /* We can avoid clflushing partial cachelines before the write ++ * if we only every write full cache-lines. Since we know that ++ * both the source and destination are in multiples of ++ * PAGE_SIZE, we can simply round up to the next cacheline. ++ * We don't care about copying too much here as we only ++ * validate up to the end of the batch. ++ */ ++ if (dst_needs_clflush & CLFLUSH_BEFORE) ++ batch_len = roundup(batch_len, ++ boot_cpu_data.x86_clflush_size); ++ ++ ptr = dst; ++ for (n = batch_start_offset >> PAGE_SHIFT; batch_len; n++) { ++ int len = min_t(int, batch_len, PAGE_SIZE - offset); ++ ++ src = kmap_atomic(i915_gem_object_get_page(src_obj, n)); ++ if (src_needs_clflush) ++ drm_clflush_virt_range(src + offset, len); ++ memcpy(ptr, src + offset, len); ++ kunmap_atomic(src); ++ ++ ptr += len; ++ batch_len -= len; ++ offset = 0; ++ } ++ } ++ ++ /* dst_obj is returned with vmap pinned */ ++ *needs_clflush_after = dst_needs_clflush & CLFLUSH_AFTER; ++ ++unpin_dst: ++ i915_gem_obj_finish_shmem_access(dst_obj); ++unpin_src: ++ i915_gem_obj_finish_shmem_access(src_obj); ++ return dst; ++} ++ ++static bool check_cmd(const struct intel_engine_cs *engine, ++ const struct drm_i915_cmd_descriptor *desc, ++ const u32 *cmd, u32 length, ++ const bool is_master) ++{ ++ if (desc->flags & CMD_DESC_SKIP) ++ return true; ++ ++ if (desc->flags & CMD_DESC_REJECT) { ++ DRM_DEBUG_DRIVER("CMD: Rejected command: 0x%08X\n", *cmd); ++ return false; ++ } ++ ++ if ((desc->flags & CMD_DESC_MASTER) && !is_master) { ++ DRM_DEBUG_DRIVER("CMD: Rejected master-only command: 0x%08X\n", ++ *cmd); ++ return false; ++ } ++ ++ if (desc->flags & CMD_DESC_REGISTER) { ++ /* ++ * Get the distance between individual register offset ++ * fields if the command can perform more than one ++ * access at a time. ++ */ ++ const u32 step = desc->reg.step ? desc->reg.step : length; ++ u32 offset; ++ ++ for (offset = desc->reg.offset; offset < length; ++ offset += step) { ++ const u32 reg_addr = cmd[offset] & desc->reg.mask; ++ const struct drm_i915_reg_descriptor *reg = ++ find_reg(engine, is_master, reg_addr); ++ ++ if (!reg) { ++ DRM_DEBUG_DRIVER("CMD: Rejected register 0x%08X in command: 0x%08X (%s)\n", ++ reg_addr, *cmd, engine->name); ++ return false; ++ } ++ ++ /* ++ * Check the value written to the register against the ++ * allowed mask/value pair given in the whitelist entry. ++ */ ++ if (reg->mask) { ++ if (desc->cmd.value == MI_LOAD_REGISTER_MEM) { ++ DRM_DEBUG_DRIVER("CMD: Rejected LRM to masked register 0x%08X\n", ++ reg_addr); ++ return false; ++ } ++ ++ if (desc->cmd.value == MI_LOAD_REGISTER_REG) { ++ DRM_DEBUG_DRIVER("CMD: Rejected LRR to masked register 0x%08X\n", ++ reg_addr); ++ return false; ++ } ++ ++ if (desc->cmd.value == MI_LOAD_REGISTER_IMM(1) && ++ (offset + 2 > length || ++ (cmd[offset + 1] & reg->mask) != reg->value)) { ++ DRM_DEBUG_DRIVER("CMD: Rejected LRI to masked register 0x%08X\n", ++ reg_addr); ++ return false; ++ } ++ } ++ } ++ } ++ ++ if (desc->flags & CMD_DESC_BITMASK) { ++ int i; ++ ++ for (i = 0; i < MAX_CMD_DESC_BITMASKS; i++) { ++ u32 dword; ++ ++ if (desc->bits[i].mask == 0) ++ break; ++ ++ if (desc->bits[i].condition_mask != 0) { ++ u32 offset = ++ desc->bits[i].condition_offset; ++ u32 condition = cmd[offset] & ++ desc->bits[i].condition_mask; ++ ++ if (condition == 0) ++ continue; ++ } ++ ++ if (desc->bits[i].offset >= length) { ++ DRM_DEBUG_DRIVER("CMD: Rejected command 0x%08X, too short to check bitmask (%s)\n", ++ *cmd, engine->name); ++ return false; ++ } ++ ++ dword = cmd[desc->bits[i].offset] & ++ desc->bits[i].mask; ++ ++ if (dword != desc->bits[i].expected) { ++ DRM_DEBUG_DRIVER("CMD: Rejected command 0x%08X for bitmask 0x%08X (exp=0x%08X act=0x%08X) (%s)\n", ++ *cmd, ++ desc->bits[i].mask, ++ desc->bits[i].expected, ++ dword, engine->name); ++ return false; ++ } ++ } ++ } ++ ++ return true; ++} ++ ++#define LENGTH_BIAS 2 ++ ++/** ++ * i915_parse_cmds() - parse a submitted batch buffer for privilege violations ++ * @engine: the engine on which the batch is to execute ++ * @batch_obj: the batch buffer in question ++ * @shadow_batch_obj: copy of the batch buffer in question ++ * @batch_start_offset: byte offset in the batch at which execution starts ++ * @batch_len: length of the commands in batch_obj ++ * @is_master: is the submitting process the drm master? ++ * ++ * Parses the specified batch buffer looking for privilege violations as ++ * described in the overview. ++ * ++ * Return: non-zero if the parser finds violations or otherwise fails; -EACCES ++ * if the batch appears legal but should use hardware parsing ++ */ ++int intel_engine_cmd_parser(struct intel_engine_cs *engine, ++ struct drm_i915_gem_object *batch_obj, ++ struct drm_i915_gem_object *shadow_batch_obj, ++ u32 batch_start_offset, ++ u32 batch_len, ++ bool is_master) ++{ ++ u32 *cmd, *batch_end; ++ struct drm_i915_cmd_descriptor default_desc = noop_desc; ++ const struct drm_i915_cmd_descriptor *desc = &default_desc; ++ bool needs_clflush_after = false; ++ int ret = 0; ++ ++ cmd = copy_batch(shadow_batch_obj, batch_obj, ++ batch_start_offset, batch_len, ++ &needs_clflush_after); ++ if (IS_ERR(cmd)) { ++ DRM_DEBUG_DRIVER("CMD: Failed to copy batch\n"); ++ return PTR_ERR(cmd); ++ } ++ ++ /* ++ * We use the batch length as size because the shadow object is as ++ * large or larger and copy_batch() will write MI_NOPs to the extra ++ * space. Parsing should be faster in some cases this way. ++ */ ++ batch_end = cmd + (batch_len / sizeof(*batch_end)); ++ do { ++ u32 length; ++ ++ if (*cmd == MI_BATCH_BUFFER_END) { ++ if (needs_clflush_after) { ++ void *ptr = page_mask_bits(shadow_batch_obj->mm.mapping); ++ drm_clflush_virt_range(ptr, ++ (void *)(cmd + 1) - ptr); ++ } ++ break; ++ } ++ ++ desc = find_cmd(engine, *cmd, desc, &default_desc); ++ if (!desc) { ++ DRM_DEBUG_DRIVER("CMD: Unrecognized command: 0x%08X\n", ++ *cmd); ++ ret = -EINVAL; ++ break; ++ } ++ ++ /* ++ * If the batch buffer contains a chained batch, return an ++ * error that tells the caller to abort and dispatch the ++ * workload as a non-secure batch. ++ */ ++ if (desc->cmd.value == MI_BATCH_BUFFER_START) { ++ ret = -EACCES; ++ break; ++ } ++ ++ if (desc->flags & CMD_DESC_FIXED) ++ length = desc->length.fixed; ++ else ++ length = ((*cmd & desc->length.mask) + LENGTH_BIAS); ++ ++ if ((batch_end - cmd) < length) { ++ DRM_DEBUG_DRIVER("CMD: Command length exceeds batch length: 0x%08X length=%u batchlen=%td\n", ++ *cmd, ++ length, ++ batch_end - cmd); ++ ret = -EINVAL; ++ break; ++ } ++ ++ if (!check_cmd(engine, desc, cmd, length, is_master)) { ++ ret = -EACCES; ++ break; ++ } ++ ++ cmd += length; ++ if (cmd >= batch_end) { ++ DRM_DEBUG_DRIVER("CMD: Got to the end of the buffer w/o a BBE cmd!\n"); ++ ret = -EINVAL; ++ break; ++ } ++ } while (1); ++ ++ i915_gem_object_unpin_map(shadow_batch_obj); ++ return ret; ++} ++ ++/** ++ * i915_cmd_parser_get_version() - get the cmd parser version number ++ * @dev_priv: i915 device private ++ * ++ * The cmd parser maintains a simple increasing integer version number suitable ++ * for passing to userspace clients to determine what operations are permitted. ++ * ++ * Return: the current version number of the cmd parser ++ */ ++int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv) ++{ ++ struct intel_engine_cs *engine; ++ enum intel_engine_id id; ++ bool active = false; ++ ++ /* If the command parser is not enabled, report 0 - unsupported */ ++ for_each_engine(engine, dev_priv, id) { ++ if (intel_engine_needs_cmd_parser(engine)) { ++ active = true; ++ break; ++ } ++ } ++ if (!active) ++ return 0; ++ ++ /* ++ * Command parser version history ++ * ++ * 1. Initial version. Checks batches and reports violations, but leaves ++ * hardware parsing enabled (so does not allow new use cases). ++ * 2. Allow access to the MI_PREDICATE_SRC0 and ++ * MI_PREDICATE_SRC1 registers. ++ * 3. Allow access to the GPGPU_THREADS_DISPATCHED register. ++ * 4. L3 atomic chicken bits of HSW_SCRATCH1 and HSW_ROW_CHICKEN3. ++ * 5. GPGPU dispatch compute indirect registers. ++ * 6. TIMESTAMP register and Haswell CS GPR registers ++ * 7. Allow MI_LOAD_REGISTER_REG between whitelisted registers. ++ * 8. Don't report cmd_check() failures as EINVAL errors to userspace; ++ * rely on the HW to NOOP disallowed commands as it would without ++ * the parser enabled. ++ * 9. Don't whitelist or handle oacontrol specially, as ownership ++ * for oacontrol state is moving to i915-perf. ++ */ ++ return 9; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_debugfs.c b/drivers/gpu/drm/i915_legacy/i915_debugfs.c +new file mode 100644 +index 000000000000..5823ffb17821 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_debugfs.c +@@ -0,0 +1,4926 @@ ++/* ++ * Copyright © 2008 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ * Authors: ++ * Eric Anholt ++ * Keith Packard ++ * ++ */ ++ ++#include ++#include ++ ++#include ++#include ++ ++#include "i915_reset.h" ++#include "intel_dp.h" ++#include "intel_drv.h" ++#include "intel_fbc.h" ++#include "intel_guc_submission.h" ++#include "intel_hdcp.h" ++#include "intel_hdmi.h" ++#include "intel_pm.h" ++#include "intel_psr.h" ++ ++static inline struct drm_i915_private *node_to_i915(struct drm_info_node *node) ++{ ++ return to_i915(node->minor->dev); ++} ++ ++static int i915_capabilities(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ const struct intel_device_info *info = INTEL_INFO(dev_priv); ++ struct drm_printer p = drm_seq_file_printer(m); ++ ++ seq_printf(m, "gen: %d\n", INTEL_GEN(dev_priv)); ++ seq_printf(m, "platform: %s\n", intel_platform_name(info->platform)); ++ seq_printf(m, "pch: %d\n", INTEL_PCH_TYPE(dev_priv)); ++ ++ intel_device_info_dump_flags(info, &p); ++ intel_device_info_dump_runtime(RUNTIME_INFO(dev_priv), &p); ++ intel_driver_caps_print(&dev_priv->caps, &p); ++ ++ kernel_param_lock(THIS_MODULE); ++ i915_params_dump(&i915_modparams, &p); ++ kernel_param_unlock(THIS_MODULE); ++ ++ return 0; ++} ++ ++static char get_active_flag(struct drm_i915_gem_object *obj) ++{ ++ return i915_gem_object_is_active(obj) ? '*' : ' '; ++} ++ ++static char get_pin_flag(struct drm_i915_gem_object *obj) ++{ ++ return obj->pin_global ? 'p' : ' '; ++} ++ ++static char get_tiling_flag(struct drm_i915_gem_object *obj) ++{ ++ switch (i915_gem_object_get_tiling(obj)) { ++ default: ++ case I915_TILING_NONE: return ' '; ++ case I915_TILING_X: return 'X'; ++ case I915_TILING_Y: return 'Y'; ++ } ++} ++ ++static char get_global_flag(struct drm_i915_gem_object *obj) ++{ ++ return obj->userfault_count ? 'g' : ' '; ++} ++ ++static char get_pin_mapped_flag(struct drm_i915_gem_object *obj) ++{ ++ return obj->mm.mapping ? 'M' : ' '; ++} ++ ++static u64 i915_gem_obj_total_ggtt_size(struct drm_i915_gem_object *obj) ++{ ++ u64 size = 0; ++ struct i915_vma *vma; ++ ++ for_each_ggtt_vma(vma, obj) { ++ if (drm_mm_node_allocated(&vma->node)) ++ size += vma->node.size; ++ } ++ ++ return size; ++} ++ ++static const char * ++stringify_page_sizes(unsigned int page_sizes, char *buf, size_t len) ++{ ++ size_t x = 0; ++ ++ switch (page_sizes) { ++ case 0: ++ return ""; ++ case I915_GTT_PAGE_SIZE_4K: ++ return "4K"; ++ case I915_GTT_PAGE_SIZE_64K: ++ return "64K"; ++ case I915_GTT_PAGE_SIZE_2M: ++ return "2M"; ++ default: ++ if (!buf) ++ return "M"; ++ ++ if (page_sizes & I915_GTT_PAGE_SIZE_2M) ++ x += snprintf(buf + x, len - x, "2M, "); ++ if (page_sizes & I915_GTT_PAGE_SIZE_64K) ++ x += snprintf(buf + x, len - x, "64K, "); ++ if (page_sizes & I915_GTT_PAGE_SIZE_4K) ++ x += snprintf(buf + x, len - x, "4K, "); ++ buf[x-2] = '\0'; ++ ++ return buf; ++ } ++} ++ ++static void ++describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) ++{ ++ struct drm_i915_private *dev_priv = to_i915(obj->base.dev); ++ struct intel_engine_cs *engine; ++ struct i915_vma *vma; ++ unsigned int frontbuffer_bits; ++ int pin_count = 0; ++ ++ lockdep_assert_held(&obj->base.dev->struct_mutex); ++ ++ seq_printf(m, "%pK: %c%c%c%c%c %8zdKiB %02x %02x %s%s%s", ++ &obj->base, ++ get_active_flag(obj), ++ get_pin_flag(obj), ++ get_tiling_flag(obj), ++ get_global_flag(obj), ++ get_pin_mapped_flag(obj), ++ obj->base.size / 1024, ++ obj->read_domains, ++ obj->write_domain, ++ i915_cache_level_str(dev_priv, obj->cache_level), ++ obj->mm.dirty ? " dirty" : "", ++ obj->mm.madv == I915_MADV_DONTNEED ? " purgeable" : ""); ++ if (obj->base.name) ++ seq_printf(m, " (name: %d)", obj->base.name); ++ list_for_each_entry(vma, &obj->vma.list, obj_link) { ++ if (i915_vma_is_pinned(vma)) ++ pin_count++; ++ } ++ seq_printf(m, " (pinned x %d)", pin_count); ++ if (obj->pin_global) ++ seq_printf(m, " (global)"); ++ list_for_each_entry(vma, &obj->vma.list, obj_link) { ++ if (!drm_mm_node_allocated(&vma->node)) ++ continue; ++ ++ seq_printf(m, " (%sgtt offset: %08llx, size: %08llx, pages: %s", ++ i915_vma_is_ggtt(vma) ? "g" : "pp", ++ vma->node.start, vma->node.size, ++ stringify_page_sizes(vma->page_sizes.gtt, NULL, 0)); ++ if (i915_vma_is_ggtt(vma)) { ++ switch (vma->ggtt_view.type) { ++ case I915_GGTT_VIEW_NORMAL: ++ seq_puts(m, ", normal"); ++ break; ++ ++ case I915_GGTT_VIEW_PARTIAL: ++ seq_printf(m, ", partial [%08llx+%x]", ++ vma->ggtt_view.partial.offset << PAGE_SHIFT, ++ vma->ggtt_view.partial.size << PAGE_SHIFT); ++ break; ++ ++ case I915_GGTT_VIEW_ROTATED: ++ seq_printf(m, ", rotated [(%ux%u, stride=%u, offset=%u), (%ux%u, stride=%u, offset=%u)]", ++ vma->ggtt_view.rotated.plane[0].width, ++ vma->ggtt_view.rotated.plane[0].height, ++ vma->ggtt_view.rotated.plane[0].stride, ++ vma->ggtt_view.rotated.plane[0].offset, ++ vma->ggtt_view.rotated.plane[1].width, ++ vma->ggtt_view.rotated.plane[1].height, ++ vma->ggtt_view.rotated.plane[1].stride, ++ vma->ggtt_view.rotated.plane[1].offset); ++ break; ++ ++ default: ++ MISSING_CASE(vma->ggtt_view.type); ++ break; ++ } ++ } ++ if (vma->fence) ++ seq_printf(m, " , fence: %d%s", ++ vma->fence->id, ++ i915_active_request_isset(&vma->last_fence) ? "*" : ""); ++ seq_puts(m, ")"); ++ } ++ if (obj->stolen) ++ seq_printf(m, " (stolen: %08llx)", obj->stolen->start); ++ ++ engine = i915_gem_object_last_write_engine(obj); ++ if (engine) ++ seq_printf(m, " (%s)", engine->name); ++ ++ frontbuffer_bits = atomic_read(&obj->frontbuffer_bits); ++ if (frontbuffer_bits) ++ seq_printf(m, " (frontbuffer: 0x%03x)", frontbuffer_bits); ++} ++ ++static int obj_rank_by_stolen(const void *A, const void *B) ++{ ++ const struct drm_i915_gem_object *a = ++ *(const struct drm_i915_gem_object **)A; ++ const struct drm_i915_gem_object *b = ++ *(const struct drm_i915_gem_object **)B; ++ ++ if (a->stolen->start < b->stolen->start) ++ return -1; ++ if (a->stolen->start > b->stolen->start) ++ return 1; ++ return 0; ++} ++ ++static int i915_gem_stolen_list_info(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct drm_device *dev = &dev_priv->drm; ++ struct drm_i915_gem_object **objects; ++ struct drm_i915_gem_object *obj; ++ u64 total_obj_size, total_gtt_size; ++ unsigned long total, count, n; ++ int ret; ++ ++ total = READ_ONCE(dev_priv->mm.object_count); ++ objects = kvmalloc_array(total, sizeof(*objects), GFP_KERNEL); ++ if (!objects) ++ return -ENOMEM; ++ ++ ret = mutex_lock_interruptible(&dev->struct_mutex); ++ if (ret) ++ goto out; ++ ++ total_obj_size = total_gtt_size = count = 0; ++ ++ spin_lock(&dev_priv->mm.obj_lock); ++ list_for_each_entry(obj, &dev_priv->mm.bound_list, mm.link) { ++ if (count == total) ++ break; ++ ++ if (obj->stolen == NULL) ++ continue; ++ ++ objects[count++] = obj; ++ total_obj_size += obj->base.size; ++ total_gtt_size += i915_gem_obj_total_ggtt_size(obj); ++ ++ } ++ list_for_each_entry(obj, &dev_priv->mm.unbound_list, mm.link) { ++ if (count == total) ++ break; ++ ++ if (obj->stolen == NULL) ++ continue; ++ ++ objects[count++] = obj; ++ total_obj_size += obj->base.size; ++ } ++ spin_unlock(&dev_priv->mm.obj_lock); ++ ++ sort(objects, count, sizeof(*objects), obj_rank_by_stolen, NULL); ++ ++ seq_puts(m, "Stolen:\n"); ++ for (n = 0; n < count; n++) { ++ seq_puts(m, " "); ++ describe_obj(m, objects[n]); ++ seq_putc(m, '\n'); ++ } ++ seq_printf(m, "Total %lu objects, %llu bytes, %llu GTT size\n", ++ count, total_obj_size, total_gtt_size); ++ ++ mutex_unlock(&dev->struct_mutex); ++out: ++ kvfree(objects); ++ return ret; ++} ++ ++struct file_stats { ++ struct i915_address_space *vm; ++ unsigned long count; ++ u64 total, unbound; ++ u64 global, shared; ++ u64 active, inactive; ++ u64 closed; ++}; ++ ++static int per_file_stats(int id, void *ptr, void *data) ++{ ++ struct drm_i915_gem_object *obj = ptr; ++ struct file_stats *stats = data; ++ struct i915_vma *vma; ++ ++ lockdep_assert_held(&obj->base.dev->struct_mutex); ++ ++ stats->count++; ++ stats->total += obj->base.size; ++ if (!obj->bind_count) ++ stats->unbound += obj->base.size; ++ if (obj->base.name || obj->base.dma_buf) ++ stats->shared += obj->base.size; ++ ++ list_for_each_entry(vma, &obj->vma.list, obj_link) { ++ if (!drm_mm_node_allocated(&vma->node)) ++ continue; ++ ++ if (i915_vma_is_ggtt(vma)) { ++ stats->global += vma->node.size; ++ } else { ++ if (vma->vm != stats->vm) ++ continue; ++ } ++ ++ if (i915_vma_is_active(vma)) ++ stats->active += vma->node.size; ++ else ++ stats->inactive += vma->node.size; ++ ++ if (i915_vma_is_closed(vma)) ++ stats->closed += vma->node.size; ++ } ++ ++ return 0; ++} ++ ++#define print_file_stats(m, name, stats) do { \ ++ if (stats.count) \ ++ seq_printf(m, "%s: %lu objects, %llu bytes (%llu active, %llu inactive, %llu global, %llu shared, %llu unbound, %llu closed)\n", \ ++ name, \ ++ stats.count, \ ++ stats.total, \ ++ stats.active, \ ++ stats.inactive, \ ++ stats.global, \ ++ stats.shared, \ ++ stats.unbound, \ ++ stats.closed); \ ++} while (0) ++ ++static void print_batch_pool_stats(struct seq_file *m, ++ struct drm_i915_private *dev_priv) ++{ ++ struct drm_i915_gem_object *obj; ++ struct intel_engine_cs *engine; ++ struct file_stats stats = {}; ++ enum intel_engine_id id; ++ int j; ++ ++ for_each_engine(engine, dev_priv, id) { ++ for (j = 0; j < ARRAY_SIZE(engine->batch_pool.cache_list); j++) { ++ list_for_each_entry(obj, ++ &engine->batch_pool.cache_list[j], ++ batch_pool_link) ++ per_file_stats(0, obj, &stats); ++ } ++ } ++ ++ print_file_stats(m, "[k]batch pool", stats); ++} ++ ++static void print_context_stats(struct seq_file *m, ++ struct drm_i915_private *i915) ++{ ++ struct file_stats kstats = {}; ++ struct i915_gem_context *ctx; ++ ++ list_for_each_entry(ctx, &i915->contexts.list, link) { ++ struct intel_context *ce; ++ ++ list_for_each_entry(ce, &ctx->active_engines, active_link) { ++ if (ce->state) ++ per_file_stats(0, ce->state->obj, &kstats); ++ if (ce->ring) ++ per_file_stats(0, ce->ring->vma->obj, &kstats); ++ } ++ ++ if (!IS_ERR_OR_NULL(ctx->file_priv)) { ++ struct file_stats stats = { .vm = &ctx->ppgtt->vm, }; ++ struct drm_file *file = ctx->file_priv->file; ++ struct task_struct *task; ++ char name[80]; ++ ++ spin_lock(&file->table_lock); ++ idr_for_each(&file->object_idr, per_file_stats, &stats); ++ spin_unlock(&file->table_lock); ++ ++ rcu_read_lock(); ++ task = pid_task(ctx->pid ?: file->pid, PIDTYPE_PID); ++ snprintf(name, sizeof(name), "%s", ++ task ? task->comm : ""); ++ rcu_read_unlock(); ++ ++ print_file_stats(m, name, stats); ++ } ++ } ++ ++ print_file_stats(m, "[k]contexts", kstats); ++} ++ ++static int i915_gem_object_info(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct drm_device *dev = &dev_priv->drm; ++ struct i915_ggtt *ggtt = &dev_priv->ggtt; ++ u32 count, mapped_count, purgeable_count, dpy_count, huge_count; ++ u64 size, mapped_size, purgeable_size, dpy_size, huge_size; ++ struct drm_i915_gem_object *obj; ++ unsigned int page_sizes = 0; ++ char buf[80]; ++ int ret; ++ ++ seq_printf(m, "%u objects, %llu bytes\n", ++ dev_priv->mm.object_count, ++ dev_priv->mm.object_memory); ++ ++ size = count = 0; ++ mapped_size = mapped_count = 0; ++ purgeable_size = purgeable_count = 0; ++ huge_size = huge_count = 0; ++ ++ spin_lock(&dev_priv->mm.obj_lock); ++ list_for_each_entry(obj, &dev_priv->mm.unbound_list, mm.link) { ++ size += obj->base.size; ++ ++count; ++ ++ if (obj->mm.madv == I915_MADV_DONTNEED) { ++ purgeable_size += obj->base.size; ++ ++purgeable_count; ++ } ++ ++ if (obj->mm.mapping) { ++ mapped_count++; ++ mapped_size += obj->base.size; ++ } ++ ++ if (obj->mm.page_sizes.sg > I915_GTT_PAGE_SIZE) { ++ huge_count++; ++ huge_size += obj->base.size; ++ page_sizes |= obj->mm.page_sizes.sg; ++ } ++ } ++ seq_printf(m, "%u unbound objects, %llu bytes\n", count, size); ++ ++ size = count = dpy_size = dpy_count = 0; ++ list_for_each_entry(obj, &dev_priv->mm.bound_list, mm.link) { ++ size += obj->base.size; ++ ++count; ++ ++ if (obj->pin_global) { ++ dpy_size += obj->base.size; ++ ++dpy_count; ++ } ++ ++ if (obj->mm.madv == I915_MADV_DONTNEED) { ++ purgeable_size += obj->base.size; ++ ++purgeable_count; ++ } ++ ++ if (obj->mm.mapping) { ++ mapped_count++; ++ mapped_size += obj->base.size; ++ } ++ ++ if (obj->mm.page_sizes.sg > I915_GTT_PAGE_SIZE) { ++ huge_count++; ++ huge_size += obj->base.size; ++ page_sizes |= obj->mm.page_sizes.sg; ++ } ++ } ++ spin_unlock(&dev_priv->mm.obj_lock); ++ ++ seq_printf(m, "%u bound objects, %llu bytes\n", ++ count, size); ++ seq_printf(m, "%u purgeable objects, %llu bytes\n", ++ purgeable_count, purgeable_size); ++ seq_printf(m, "%u mapped objects, %llu bytes\n", ++ mapped_count, mapped_size); ++ seq_printf(m, "%u huge-paged objects (%s) %llu bytes\n", ++ huge_count, ++ stringify_page_sizes(page_sizes, buf, sizeof(buf)), ++ huge_size); ++ seq_printf(m, "%u display objects (globally pinned), %llu bytes\n", ++ dpy_count, dpy_size); ++ ++ seq_printf(m, "%llu [%pa] gtt total\n", ++ ggtt->vm.total, &ggtt->mappable_end); ++ seq_printf(m, "Supported page sizes: %s\n", ++ stringify_page_sizes(INTEL_INFO(dev_priv)->page_sizes, ++ buf, sizeof(buf))); ++ ++ seq_putc(m, '\n'); ++ ++ ret = mutex_lock_interruptible(&dev->struct_mutex); ++ if (ret) ++ return ret; ++ ++ print_batch_pool_stats(m, dev_priv); ++ print_context_stats(m, dev_priv); ++ mutex_unlock(&dev->struct_mutex); ++ ++ return 0; ++} ++ ++static int i915_gem_gtt_info(struct seq_file *m, void *data) ++{ ++ struct drm_info_node *node = m->private; ++ struct drm_i915_private *dev_priv = node_to_i915(node); ++ struct drm_device *dev = &dev_priv->drm; ++ struct drm_i915_gem_object **objects; ++ struct drm_i915_gem_object *obj; ++ u64 total_obj_size, total_gtt_size; ++ unsigned long nobject, n; ++ int count, ret; ++ ++ nobject = READ_ONCE(dev_priv->mm.object_count); ++ objects = kvmalloc_array(nobject, sizeof(*objects), GFP_KERNEL); ++ if (!objects) ++ return -ENOMEM; ++ ++ ret = mutex_lock_interruptible(&dev->struct_mutex); ++ if (ret) ++ return ret; ++ ++ count = 0; ++ spin_lock(&dev_priv->mm.obj_lock); ++ list_for_each_entry(obj, &dev_priv->mm.bound_list, mm.link) { ++ objects[count++] = obj; ++ if (count == nobject) ++ break; ++ } ++ spin_unlock(&dev_priv->mm.obj_lock); ++ ++ total_obj_size = total_gtt_size = 0; ++ for (n = 0; n < count; n++) { ++ obj = objects[n]; ++ ++ seq_puts(m, " "); ++ describe_obj(m, obj); ++ seq_putc(m, '\n'); ++ total_obj_size += obj->base.size; ++ total_gtt_size += i915_gem_obj_total_ggtt_size(obj); ++ } ++ ++ mutex_unlock(&dev->struct_mutex); ++ ++ seq_printf(m, "Total %d objects, %llu bytes, %llu GTT size\n", ++ count, total_obj_size, total_gtt_size); ++ kvfree(objects); ++ ++ return 0; ++} ++ ++static int i915_gem_batch_pool_info(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct drm_device *dev = &dev_priv->drm; ++ struct drm_i915_gem_object *obj; ++ struct intel_engine_cs *engine; ++ enum intel_engine_id id; ++ int total = 0; ++ int ret, j; ++ ++ ret = mutex_lock_interruptible(&dev->struct_mutex); ++ if (ret) ++ return ret; ++ ++ for_each_engine(engine, dev_priv, id) { ++ for (j = 0; j < ARRAY_SIZE(engine->batch_pool.cache_list); j++) { ++ int count; ++ ++ count = 0; ++ list_for_each_entry(obj, ++ &engine->batch_pool.cache_list[j], ++ batch_pool_link) ++ count++; ++ seq_printf(m, "%s cache[%d]: %d objects\n", ++ engine->name, j, count); ++ ++ list_for_each_entry(obj, ++ &engine->batch_pool.cache_list[j], ++ batch_pool_link) { ++ seq_puts(m, " "); ++ describe_obj(m, obj); ++ seq_putc(m, '\n'); ++ } ++ ++ total += count; ++ } ++ } ++ ++ seq_printf(m, "total: %d\n", total); ++ ++ mutex_unlock(&dev->struct_mutex); ++ ++ return 0; ++} ++ ++static void gen8_display_interrupt_info(struct seq_file *m) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ int pipe; ++ ++ for_each_pipe(dev_priv, pipe) { ++ enum intel_display_power_domain power_domain; ++ intel_wakeref_t wakeref; ++ ++ power_domain = POWER_DOMAIN_PIPE(pipe); ++ wakeref = intel_display_power_get_if_enabled(dev_priv, ++ power_domain); ++ if (!wakeref) { ++ seq_printf(m, "Pipe %c power disabled\n", ++ pipe_name(pipe)); ++ continue; ++ } ++ seq_printf(m, "Pipe %c IMR:\t%08x\n", ++ pipe_name(pipe), ++ I915_READ(GEN8_DE_PIPE_IMR(pipe))); ++ seq_printf(m, "Pipe %c IIR:\t%08x\n", ++ pipe_name(pipe), ++ I915_READ(GEN8_DE_PIPE_IIR(pipe))); ++ seq_printf(m, "Pipe %c IER:\t%08x\n", ++ pipe_name(pipe), ++ I915_READ(GEN8_DE_PIPE_IER(pipe))); ++ ++ intel_display_power_put(dev_priv, power_domain, wakeref); ++ } ++ ++ seq_printf(m, "Display Engine port interrupt mask:\t%08x\n", ++ I915_READ(GEN8_DE_PORT_IMR)); ++ seq_printf(m, "Display Engine port interrupt identity:\t%08x\n", ++ I915_READ(GEN8_DE_PORT_IIR)); ++ seq_printf(m, "Display Engine port interrupt enable:\t%08x\n", ++ I915_READ(GEN8_DE_PORT_IER)); ++ ++ seq_printf(m, "Display Engine misc interrupt mask:\t%08x\n", ++ I915_READ(GEN8_DE_MISC_IMR)); ++ seq_printf(m, "Display Engine misc interrupt identity:\t%08x\n", ++ I915_READ(GEN8_DE_MISC_IIR)); ++ seq_printf(m, "Display Engine misc interrupt enable:\t%08x\n", ++ I915_READ(GEN8_DE_MISC_IER)); ++ ++ seq_printf(m, "PCU interrupt mask:\t%08x\n", ++ I915_READ(GEN8_PCU_IMR)); ++ seq_printf(m, "PCU interrupt identity:\t%08x\n", ++ I915_READ(GEN8_PCU_IIR)); ++ seq_printf(m, "PCU interrupt enable:\t%08x\n", ++ I915_READ(GEN8_PCU_IER)); ++} ++ ++static int i915_interrupt_info(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct intel_engine_cs *engine; ++ enum intel_engine_id id; ++ intel_wakeref_t wakeref; ++ int i, pipe; ++ ++ wakeref = intel_runtime_pm_get(dev_priv); ++ ++ if (IS_CHERRYVIEW(dev_priv)) { ++ intel_wakeref_t pref; ++ ++ seq_printf(m, "Master Interrupt Control:\t%08x\n", ++ I915_READ(GEN8_MASTER_IRQ)); ++ ++ seq_printf(m, "Display IER:\t%08x\n", ++ I915_READ(VLV_IER)); ++ seq_printf(m, "Display IIR:\t%08x\n", ++ I915_READ(VLV_IIR)); ++ seq_printf(m, "Display IIR_RW:\t%08x\n", ++ I915_READ(VLV_IIR_RW)); ++ seq_printf(m, "Display IMR:\t%08x\n", ++ I915_READ(VLV_IMR)); ++ for_each_pipe(dev_priv, pipe) { ++ enum intel_display_power_domain power_domain; ++ ++ power_domain = POWER_DOMAIN_PIPE(pipe); ++ pref = intel_display_power_get_if_enabled(dev_priv, ++ power_domain); ++ if (!pref) { ++ seq_printf(m, "Pipe %c power disabled\n", ++ pipe_name(pipe)); ++ continue; ++ } ++ ++ seq_printf(m, "Pipe %c stat:\t%08x\n", ++ pipe_name(pipe), ++ I915_READ(PIPESTAT(pipe))); ++ ++ intel_display_power_put(dev_priv, power_domain, pref); ++ } ++ ++ pref = intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); ++ seq_printf(m, "Port hotplug:\t%08x\n", ++ I915_READ(PORT_HOTPLUG_EN)); ++ seq_printf(m, "DPFLIPSTAT:\t%08x\n", ++ I915_READ(VLV_DPFLIPSTAT)); ++ seq_printf(m, "DPINVGTT:\t%08x\n", ++ I915_READ(DPINVGTT)); ++ intel_display_power_put(dev_priv, POWER_DOMAIN_INIT, pref); ++ ++ for (i = 0; i < 4; i++) { ++ seq_printf(m, "GT Interrupt IMR %d:\t%08x\n", ++ i, I915_READ(GEN8_GT_IMR(i))); ++ seq_printf(m, "GT Interrupt IIR %d:\t%08x\n", ++ i, I915_READ(GEN8_GT_IIR(i))); ++ seq_printf(m, "GT Interrupt IER %d:\t%08x\n", ++ i, I915_READ(GEN8_GT_IER(i))); ++ } ++ ++ seq_printf(m, "PCU interrupt mask:\t%08x\n", ++ I915_READ(GEN8_PCU_IMR)); ++ seq_printf(m, "PCU interrupt identity:\t%08x\n", ++ I915_READ(GEN8_PCU_IIR)); ++ seq_printf(m, "PCU interrupt enable:\t%08x\n", ++ I915_READ(GEN8_PCU_IER)); ++ } else if (INTEL_GEN(dev_priv) >= 11) { ++ seq_printf(m, "Master Interrupt Control: %08x\n", ++ I915_READ(GEN11_GFX_MSTR_IRQ)); ++ ++ seq_printf(m, "Render/Copy Intr Enable: %08x\n", ++ I915_READ(GEN11_RENDER_COPY_INTR_ENABLE)); ++ seq_printf(m, "VCS/VECS Intr Enable: %08x\n", ++ I915_READ(GEN11_VCS_VECS_INTR_ENABLE)); ++ seq_printf(m, "GUC/SG Intr Enable:\t %08x\n", ++ I915_READ(GEN11_GUC_SG_INTR_ENABLE)); ++ seq_printf(m, "GPM/WGBOXPERF Intr Enable: %08x\n", ++ I915_READ(GEN11_GPM_WGBOXPERF_INTR_ENABLE)); ++ seq_printf(m, "Crypto Intr Enable:\t %08x\n", ++ I915_READ(GEN11_CRYPTO_RSVD_INTR_ENABLE)); ++ seq_printf(m, "GUnit/CSME Intr Enable:\t %08x\n", ++ I915_READ(GEN11_GUNIT_CSME_INTR_ENABLE)); ++ ++ seq_printf(m, "Display Interrupt Control:\t%08x\n", ++ I915_READ(GEN11_DISPLAY_INT_CTL)); ++ ++ gen8_display_interrupt_info(m); ++ } else if (INTEL_GEN(dev_priv) >= 8) { ++ seq_printf(m, "Master Interrupt Control:\t%08x\n", ++ I915_READ(GEN8_MASTER_IRQ)); ++ ++ for (i = 0; i < 4; i++) { ++ seq_printf(m, "GT Interrupt IMR %d:\t%08x\n", ++ i, I915_READ(GEN8_GT_IMR(i))); ++ seq_printf(m, "GT Interrupt IIR %d:\t%08x\n", ++ i, I915_READ(GEN8_GT_IIR(i))); ++ seq_printf(m, "GT Interrupt IER %d:\t%08x\n", ++ i, I915_READ(GEN8_GT_IER(i))); ++ } ++ ++ gen8_display_interrupt_info(m); ++ } else if (IS_VALLEYVIEW(dev_priv)) { ++ seq_printf(m, "Display IER:\t%08x\n", ++ I915_READ(VLV_IER)); ++ seq_printf(m, "Display IIR:\t%08x\n", ++ I915_READ(VLV_IIR)); ++ seq_printf(m, "Display IIR_RW:\t%08x\n", ++ I915_READ(VLV_IIR_RW)); ++ seq_printf(m, "Display IMR:\t%08x\n", ++ I915_READ(VLV_IMR)); ++ for_each_pipe(dev_priv, pipe) { ++ enum intel_display_power_domain power_domain; ++ intel_wakeref_t pref; ++ ++ power_domain = POWER_DOMAIN_PIPE(pipe); ++ pref = intel_display_power_get_if_enabled(dev_priv, ++ power_domain); ++ if (!pref) { ++ seq_printf(m, "Pipe %c power disabled\n", ++ pipe_name(pipe)); ++ continue; ++ } ++ ++ seq_printf(m, "Pipe %c stat:\t%08x\n", ++ pipe_name(pipe), ++ I915_READ(PIPESTAT(pipe))); ++ intel_display_power_put(dev_priv, power_domain, pref); ++ } ++ ++ seq_printf(m, "Master IER:\t%08x\n", ++ I915_READ(VLV_MASTER_IER)); ++ ++ seq_printf(m, "Render IER:\t%08x\n", ++ I915_READ(GTIER)); ++ seq_printf(m, "Render IIR:\t%08x\n", ++ I915_READ(GTIIR)); ++ seq_printf(m, "Render IMR:\t%08x\n", ++ I915_READ(GTIMR)); ++ ++ seq_printf(m, "PM IER:\t\t%08x\n", ++ I915_READ(GEN6_PMIER)); ++ seq_printf(m, "PM IIR:\t\t%08x\n", ++ I915_READ(GEN6_PMIIR)); ++ seq_printf(m, "PM IMR:\t\t%08x\n", ++ I915_READ(GEN6_PMIMR)); ++ ++ seq_printf(m, "Port hotplug:\t%08x\n", ++ I915_READ(PORT_HOTPLUG_EN)); ++ seq_printf(m, "DPFLIPSTAT:\t%08x\n", ++ I915_READ(VLV_DPFLIPSTAT)); ++ seq_printf(m, "DPINVGTT:\t%08x\n", ++ I915_READ(DPINVGTT)); ++ ++ } else if (!HAS_PCH_SPLIT(dev_priv)) { ++ seq_printf(m, "Interrupt enable: %08x\n", ++ I915_READ(GEN2_IER)); ++ seq_printf(m, "Interrupt identity: %08x\n", ++ I915_READ(GEN2_IIR)); ++ seq_printf(m, "Interrupt mask: %08x\n", ++ I915_READ(GEN2_IMR)); ++ for_each_pipe(dev_priv, pipe) ++ seq_printf(m, "Pipe %c stat: %08x\n", ++ pipe_name(pipe), ++ I915_READ(PIPESTAT(pipe))); ++ } else { ++ seq_printf(m, "North Display Interrupt enable: %08x\n", ++ I915_READ(DEIER)); ++ seq_printf(m, "North Display Interrupt identity: %08x\n", ++ I915_READ(DEIIR)); ++ seq_printf(m, "North Display Interrupt mask: %08x\n", ++ I915_READ(DEIMR)); ++ seq_printf(m, "South Display Interrupt enable: %08x\n", ++ I915_READ(SDEIER)); ++ seq_printf(m, "South Display Interrupt identity: %08x\n", ++ I915_READ(SDEIIR)); ++ seq_printf(m, "South Display Interrupt mask: %08x\n", ++ I915_READ(SDEIMR)); ++ seq_printf(m, "Graphics Interrupt enable: %08x\n", ++ I915_READ(GTIER)); ++ seq_printf(m, "Graphics Interrupt identity: %08x\n", ++ I915_READ(GTIIR)); ++ seq_printf(m, "Graphics Interrupt mask: %08x\n", ++ I915_READ(GTIMR)); ++ } ++ ++ if (INTEL_GEN(dev_priv) >= 11) { ++ seq_printf(m, "RCS Intr Mask:\t %08x\n", ++ I915_READ(GEN11_RCS0_RSVD_INTR_MASK)); ++ seq_printf(m, "BCS Intr Mask:\t %08x\n", ++ I915_READ(GEN11_BCS_RSVD_INTR_MASK)); ++ seq_printf(m, "VCS0/VCS1 Intr Mask:\t %08x\n", ++ I915_READ(GEN11_VCS0_VCS1_INTR_MASK)); ++ seq_printf(m, "VCS2/VCS3 Intr Mask:\t %08x\n", ++ I915_READ(GEN11_VCS2_VCS3_INTR_MASK)); ++ seq_printf(m, "VECS0/VECS1 Intr Mask:\t %08x\n", ++ I915_READ(GEN11_VECS0_VECS1_INTR_MASK)); ++ seq_printf(m, "GUC/SG Intr Mask:\t %08x\n", ++ I915_READ(GEN11_GUC_SG_INTR_MASK)); ++ seq_printf(m, "GPM/WGBOXPERF Intr Mask: %08x\n", ++ I915_READ(GEN11_GPM_WGBOXPERF_INTR_MASK)); ++ seq_printf(m, "Crypto Intr Mask:\t %08x\n", ++ I915_READ(GEN11_CRYPTO_RSVD_INTR_MASK)); ++ seq_printf(m, "Gunit/CSME Intr Mask:\t %08x\n", ++ I915_READ(GEN11_GUNIT_CSME_INTR_MASK)); ++ ++ } else if (INTEL_GEN(dev_priv) >= 6) { ++ for_each_engine(engine, dev_priv, id) { ++ seq_printf(m, ++ "Graphics Interrupt mask (%s): %08x\n", ++ engine->name, ENGINE_READ(engine, RING_IMR)); ++ } ++ } ++ ++ intel_runtime_pm_put(dev_priv, wakeref); ++ ++ return 0; ++} ++ ++static int i915_gem_fence_regs_info(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct drm_device *dev = &dev_priv->drm; ++ int i, ret; ++ ++ ret = mutex_lock_interruptible(&dev->struct_mutex); ++ if (ret) ++ return ret; ++ ++ seq_printf(m, "Total fences = %d\n", dev_priv->num_fence_regs); ++ for (i = 0; i < dev_priv->num_fence_regs; i++) { ++ struct i915_vma *vma = dev_priv->fence_regs[i].vma; ++ ++ seq_printf(m, "Fence %d, pin count = %d, object = ", ++ i, dev_priv->fence_regs[i].pin_count); ++ if (!vma) ++ seq_puts(m, "unused"); ++ else ++ describe_obj(m, vma->obj); ++ seq_putc(m, '\n'); ++ } ++ ++ mutex_unlock(&dev->struct_mutex); ++ return 0; ++} ++ ++#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) ++static ssize_t gpu_state_read(struct file *file, char __user *ubuf, ++ size_t count, loff_t *pos) ++{ ++ struct i915_gpu_state *error; ++ ssize_t ret; ++ void *buf; ++ ++ error = file->private_data; ++ if (!error) ++ return 0; ++ ++ /* Bounce buffer required because of kernfs __user API convenience. */ ++ buf = kmalloc(count, GFP_KERNEL); ++ if (!buf) ++ return -ENOMEM; ++ ++ ret = i915_gpu_state_copy_to_buffer(error, buf, *pos, count); ++ if (ret <= 0) ++ goto out; ++ ++ if (!copy_to_user(ubuf, buf, ret)) ++ *pos += ret; ++ else ++ ret = -EFAULT; ++ ++out: ++ kfree(buf); ++ return ret; ++} ++ ++static int gpu_state_release(struct inode *inode, struct file *file) ++{ ++ i915_gpu_state_put(file->private_data); ++ return 0; ++} ++ ++static int i915_gpu_info_open(struct inode *inode, struct file *file) ++{ ++ struct drm_i915_private *i915 = inode->i_private; ++ struct i915_gpu_state *gpu; ++ intel_wakeref_t wakeref; ++ ++ gpu = NULL; ++ with_intel_runtime_pm(i915, wakeref) ++ gpu = i915_capture_gpu_state(i915); ++ if (IS_ERR(gpu)) ++ return PTR_ERR(gpu); ++ ++ file->private_data = gpu; ++ return 0; ++} ++ ++static const struct file_operations i915_gpu_info_fops = { ++ .owner = THIS_MODULE, ++ .open = i915_gpu_info_open, ++ .read = gpu_state_read, ++ .llseek = default_llseek, ++ .release = gpu_state_release, ++}; ++ ++static ssize_t ++i915_error_state_write(struct file *filp, ++ const char __user *ubuf, ++ size_t cnt, ++ loff_t *ppos) ++{ ++ struct i915_gpu_state *error = filp->private_data; ++ ++ if (!error) ++ return 0; ++ ++ DRM_DEBUG_DRIVER("Resetting error state\n"); ++ i915_reset_error_state(error->i915); ++ ++ return cnt; ++} ++ ++static int i915_error_state_open(struct inode *inode, struct file *file) ++{ ++ struct i915_gpu_state *error; ++ ++ error = i915_first_error_state(inode->i_private); ++ if (IS_ERR(error)) ++ return PTR_ERR(error); ++ ++ file->private_data = error; ++ return 0; ++} ++ ++static const struct file_operations i915_error_state_fops = { ++ .owner = THIS_MODULE, ++ .open = i915_error_state_open, ++ .read = gpu_state_read, ++ .write = i915_error_state_write, ++ .llseek = default_llseek, ++ .release = gpu_state_release, ++}; ++#endif ++ ++static int i915_frequency_info(struct seq_file *m, void *unused) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct intel_rps *rps = &dev_priv->gt_pm.rps; ++ intel_wakeref_t wakeref; ++ int ret = 0; ++ ++ wakeref = intel_runtime_pm_get(dev_priv); ++ ++ if (IS_GEN(dev_priv, 5)) { ++ u16 rgvswctl = I915_READ16(MEMSWCTL); ++ u16 rgvstat = I915_READ16(MEMSTAT_ILK); ++ ++ seq_printf(m, "Requested P-state: %d\n", (rgvswctl >> 8) & 0xf); ++ seq_printf(m, "Requested VID: %d\n", rgvswctl & 0x3f); ++ seq_printf(m, "Current VID: %d\n", (rgvstat & MEMSTAT_VID_MASK) >> ++ MEMSTAT_VID_SHIFT); ++ seq_printf(m, "Current P-state: %d\n", ++ (rgvstat & MEMSTAT_PSTATE_MASK) >> MEMSTAT_PSTATE_SHIFT); ++ } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { ++ u32 rpmodectl, freq_sts; ++ ++ mutex_lock(&dev_priv->pcu_lock); ++ ++ rpmodectl = I915_READ(GEN6_RP_CONTROL); ++ seq_printf(m, "Video Turbo Mode: %s\n", ++ yesno(rpmodectl & GEN6_RP_MEDIA_TURBO)); ++ seq_printf(m, "HW control enabled: %s\n", ++ yesno(rpmodectl & GEN6_RP_ENABLE)); ++ seq_printf(m, "SW control enabled: %s\n", ++ yesno((rpmodectl & GEN6_RP_MEDIA_MODE_MASK) == ++ GEN6_RP_MEDIA_SW_MODE)); ++ ++ freq_sts = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); ++ seq_printf(m, "PUNIT_REG_GPU_FREQ_STS: 0x%08x\n", freq_sts); ++ seq_printf(m, "DDR freq: %d MHz\n", dev_priv->mem_freq); ++ ++ seq_printf(m, "actual GPU freq: %d MHz\n", ++ intel_gpu_freq(dev_priv, (freq_sts >> 8) & 0xff)); ++ ++ seq_printf(m, "current GPU freq: %d MHz\n", ++ intel_gpu_freq(dev_priv, rps->cur_freq)); ++ ++ seq_printf(m, "max GPU freq: %d MHz\n", ++ intel_gpu_freq(dev_priv, rps->max_freq)); ++ ++ seq_printf(m, "min GPU freq: %d MHz\n", ++ intel_gpu_freq(dev_priv, rps->min_freq)); ++ ++ seq_printf(m, "idle GPU freq: %d MHz\n", ++ intel_gpu_freq(dev_priv, rps->idle_freq)); ++ ++ seq_printf(m, ++ "efficient (RPe) frequency: %d MHz\n", ++ intel_gpu_freq(dev_priv, rps->efficient_freq)); ++ mutex_unlock(&dev_priv->pcu_lock); ++ } else if (INTEL_GEN(dev_priv) >= 6) { ++ u32 rp_state_limits; ++ u32 gt_perf_status; ++ u32 rp_state_cap; ++ u32 rpmodectl, rpinclimit, rpdeclimit; ++ u32 rpstat, cagf, reqf; ++ u32 rpupei, rpcurup, rpprevup; ++ u32 rpdownei, rpcurdown, rpprevdown; ++ u32 pm_ier, pm_imr, pm_isr, pm_iir, pm_mask; ++ int max_freq; ++ ++ rp_state_limits = I915_READ(GEN6_RP_STATE_LIMITS); ++ if (IS_GEN9_LP(dev_priv)) { ++ rp_state_cap = I915_READ(BXT_RP_STATE_CAP); ++ gt_perf_status = I915_READ(BXT_GT_PERF_STATUS); ++ } else { ++ rp_state_cap = I915_READ(GEN6_RP_STATE_CAP); ++ gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS); ++ } ++ ++ /* RPSTAT1 is in the GT power well */ ++ intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL); ++ ++ reqf = I915_READ(GEN6_RPNSWREQ); ++ if (INTEL_GEN(dev_priv) >= 9) ++ reqf >>= 23; ++ else { ++ reqf &= ~GEN6_TURBO_DISABLE; ++ if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) ++ reqf >>= 24; ++ else ++ reqf >>= 25; ++ } ++ reqf = intel_gpu_freq(dev_priv, reqf); ++ ++ rpmodectl = I915_READ(GEN6_RP_CONTROL); ++ rpinclimit = I915_READ(GEN6_RP_UP_THRESHOLD); ++ rpdeclimit = I915_READ(GEN6_RP_DOWN_THRESHOLD); ++ ++ rpstat = I915_READ(GEN6_RPSTAT1); ++ rpupei = I915_READ(GEN6_RP_CUR_UP_EI) & GEN6_CURICONT_MASK; ++ rpcurup = I915_READ(GEN6_RP_CUR_UP) & GEN6_CURBSYTAVG_MASK; ++ rpprevup = I915_READ(GEN6_RP_PREV_UP) & GEN6_CURBSYTAVG_MASK; ++ rpdownei = I915_READ(GEN6_RP_CUR_DOWN_EI) & GEN6_CURIAVG_MASK; ++ rpcurdown = I915_READ(GEN6_RP_CUR_DOWN) & GEN6_CURBSYTAVG_MASK; ++ rpprevdown = I915_READ(GEN6_RP_PREV_DOWN) & GEN6_CURBSYTAVG_MASK; ++ cagf = intel_gpu_freq(dev_priv, ++ intel_get_cagf(dev_priv, rpstat)); ++ ++ intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); ++ ++ if (INTEL_GEN(dev_priv) >= 11) { ++ pm_ier = I915_READ(GEN11_GPM_WGBOXPERF_INTR_ENABLE); ++ pm_imr = I915_READ(GEN11_GPM_WGBOXPERF_INTR_MASK); ++ /* ++ * The equivalent to the PM ISR & IIR cannot be read ++ * without affecting the current state of the system ++ */ ++ pm_isr = 0; ++ pm_iir = 0; ++ } else if (INTEL_GEN(dev_priv) >= 8) { ++ pm_ier = I915_READ(GEN8_GT_IER(2)); ++ pm_imr = I915_READ(GEN8_GT_IMR(2)); ++ pm_isr = I915_READ(GEN8_GT_ISR(2)); ++ pm_iir = I915_READ(GEN8_GT_IIR(2)); ++ } else { ++ pm_ier = I915_READ(GEN6_PMIER); ++ pm_imr = I915_READ(GEN6_PMIMR); ++ pm_isr = I915_READ(GEN6_PMISR); ++ pm_iir = I915_READ(GEN6_PMIIR); ++ } ++ pm_mask = I915_READ(GEN6_PMINTRMSK); ++ ++ seq_printf(m, "Video Turbo Mode: %s\n", ++ yesno(rpmodectl & GEN6_RP_MEDIA_TURBO)); ++ seq_printf(m, "HW control enabled: %s\n", ++ yesno(rpmodectl & GEN6_RP_ENABLE)); ++ seq_printf(m, "SW control enabled: %s\n", ++ yesno((rpmodectl & GEN6_RP_MEDIA_MODE_MASK) == ++ GEN6_RP_MEDIA_SW_MODE)); ++ ++ seq_printf(m, "PM IER=0x%08x IMR=0x%08x, MASK=0x%08x\n", ++ pm_ier, pm_imr, pm_mask); ++ if (INTEL_GEN(dev_priv) <= 10) ++ seq_printf(m, "PM ISR=0x%08x IIR=0x%08x\n", ++ pm_isr, pm_iir); ++ seq_printf(m, "pm_intrmsk_mbz: 0x%08x\n", ++ rps->pm_intrmsk_mbz); ++ seq_printf(m, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status); ++ seq_printf(m, "Render p-state ratio: %d\n", ++ (gt_perf_status & (INTEL_GEN(dev_priv) >= 9 ? 0x1ff00 : 0xff00)) >> 8); ++ seq_printf(m, "Render p-state VID: %d\n", ++ gt_perf_status & 0xff); ++ seq_printf(m, "Render p-state limit: %d\n", ++ rp_state_limits & 0xff); ++ seq_printf(m, "RPSTAT1: 0x%08x\n", rpstat); ++ seq_printf(m, "RPMODECTL: 0x%08x\n", rpmodectl); ++ seq_printf(m, "RPINCLIMIT: 0x%08x\n", rpinclimit); ++ seq_printf(m, "RPDECLIMIT: 0x%08x\n", rpdeclimit); ++ seq_printf(m, "RPNSWREQ: %dMHz\n", reqf); ++ seq_printf(m, "CAGF: %dMHz\n", cagf); ++ seq_printf(m, "RP CUR UP EI: %d (%dus)\n", ++ rpupei, GT_PM_INTERVAL_TO_US(dev_priv, rpupei)); ++ seq_printf(m, "RP CUR UP: %d (%dus)\n", ++ rpcurup, GT_PM_INTERVAL_TO_US(dev_priv, rpcurup)); ++ seq_printf(m, "RP PREV UP: %d (%dus)\n", ++ rpprevup, GT_PM_INTERVAL_TO_US(dev_priv, rpprevup)); ++ seq_printf(m, "Up threshold: %d%%\n", ++ rps->power.up_threshold); ++ ++ seq_printf(m, "RP CUR DOWN EI: %d (%dus)\n", ++ rpdownei, GT_PM_INTERVAL_TO_US(dev_priv, rpdownei)); ++ seq_printf(m, "RP CUR DOWN: %d (%dus)\n", ++ rpcurdown, GT_PM_INTERVAL_TO_US(dev_priv, rpcurdown)); ++ seq_printf(m, "RP PREV DOWN: %d (%dus)\n", ++ rpprevdown, GT_PM_INTERVAL_TO_US(dev_priv, rpprevdown)); ++ seq_printf(m, "Down threshold: %d%%\n", ++ rps->power.down_threshold); ++ ++ max_freq = (IS_GEN9_LP(dev_priv) ? rp_state_cap >> 0 : ++ rp_state_cap >> 16) & 0xff; ++ max_freq *= (IS_GEN9_BC(dev_priv) || ++ INTEL_GEN(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1); ++ seq_printf(m, "Lowest (RPN) frequency: %dMHz\n", ++ intel_gpu_freq(dev_priv, max_freq)); ++ ++ max_freq = (rp_state_cap & 0xff00) >> 8; ++ max_freq *= (IS_GEN9_BC(dev_priv) || ++ INTEL_GEN(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1); ++ seq_printf(m, "Nominal (RP1) frequency: %dMHz\n", ++ intel_gpu_freq(dev_priv, max_freq)); ++ ++ max_freq = (IS_GEN9_LP(dev_priv) ? rp_state_cap >> 16 : ++ rp_state_cap >> 0) & 0xff; ++ max_freq *= (IS_GEN9_BC(dev_priv) || ++ INTEL_GEN(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1); ++ seq_printf(m, "Max non-overclocked (RP0) frequency: %dMHz\n", ++ intel_gpu_freq(dev_priv, max_freq)); ++ seq_printf(m, "Max overclocked frequency: %dMHz\n", ++ intel_gpu_freq(dev_priv, rps->max_freq)); ++ ++ seq_printf(m, "Current freq: %d MHz\n", ++ intel_gpu_freq(dev_priv, rps->cur_freq)); ++ seq_printf(m, "Actual freq: %d MHz\n", cagf); ++ seq_printf(m, "Idle freq: %d MHz\n", ++ intel_gpu_freq(dev_priv, rps->idle_freq)); ++ seq_printf(m, "Min freq: %d MHz\n", ++ intel_gpu_freq(dev_priv, rps->min_freq)); ++ seq_printf(m, "Boost freq: %d MHz\n", ++ intel_gpu_freq(dev_priv, rps->boost_freq)); ++ seq_printf(m, "Max freq: %d MHz\n", ++ intel_gpu_freq(dev_priv, rps->max_freq)); ++ seq_printf(m, ++ "efficient (RPe) frequency: %d MHz\n", ++ intel_gpu_freq(dev_priv, rps->efficient_freq)); ++ } else { ++ seq_puts(m, "no P-state info available\n"); ++ } ++ ++ seq_printf(m, "Current CD clock frequency: %d kHz\n", dev_priv->cdclk.hw.cdclk); ++ seq_printf(m, "Max CD clock frequency: %d kHz\n", dev_priv->max_cdclk_freq); ++ seq_printf(m, "Max pixel clock frequency: %d kHz\n", dev_priv->max_dotclk_freq); ++ ++ intel_runtime_pm_put(dev_priv, wakeref); ++ return ret; ++} ++ ++static void i915_instdone_info(struct drm_i915_private *dev_priv, ++ struct seq_file *m, ++ struct intel_instdone *instdone) ++{ ++ int slice; ++ int subslice; ++ ++ seq_printf(m, "\t\tINSTDONE: 0x%08x\n", ++ instdone->instdone); ++ ++ if (INTEL_GEN(dev_priv) <= 3) ++ return; ++ ++ seq_printf(m, "\t\tSC_INSTDONE: 0x%08x\n", ++ instdone->slice_common); ++ ++ if (INTEL_GEN(dev_priv) <= 6) ++ return; ++ ++ for_each_instdone_slice_subslice(dev_priv, slice, subslice) ++ seq_printf(m, "\t\tSAMPLER_INSTDONE[%d][%d]: 0x%08x\n", ++ slice, subslice, instdone->sampler[slice][subslice]); ++ ++ for_each_instdone_slice_subslice(dev_priv, slice, subslice) ++ seq_printf(m, "\t\tROW_INSTDONE[%d][%d]: 0x%08x\n", ++ slice, subslice, instdone->row[slice][subslice]); ++} ++ ++static int i915_hangcheck_info(struct seq_file *m, void *unused) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct intel_engine_cs *engine; ++ u64 acthd[I915_NUM_ENGINES]; ++ u32 seqno[I915_NUM_ENGINES]; ++ struct intel_instdone instdone; ++ intel_wakeref_t wakeref; ++ enum intel_engine_id id; ++ ++ seq_printf(m, "Reset flags: %lx\n", dev_priv->gpu_error.flags); ++ if (test_bit(I915_WEDGED, &dev_priv->gpu_error.flags)) ++ seq_puts(m, "\tWedged\n"); ++ if (test_bit(I915_RESET_BACKOFF, &dev_priv->gpu_error.flags)) ++ seq_puts(m, "\tDevice (global) reset in progress\n"); ++ ++ if (!i915_modparams.enable_hangcheck) { ++ seq_puts(m, "Hangcheck disabled\n"); ++ return 0; ++ } ++ ++ with_intel_runtime_pm(dev_priv, wakeref) { ++ for_each_engine(engine, dev_priv, id) { ++ acthd[id] = intel_engine_get_active_head(engine); ++ seqno[id] = intel_engine_get_hangcheck_seqno(engine); ++ } ++ ++ intel_engine_get_instdone(dev_priv->engine[RCS0], &instdone); ++ } ++ ++ if (timer_pending(&dev_priv->gpu_error.hangcheck_work.timer)) ++ seq_printf(m, "Hangcheck active, timer fires in %dms\n", ++ jiffies_to_msecs(dev_priv->gpu_error.hangcheck_work.timer.expires - ++ jiffies)); ++ else if (delayed_work_pending(&dev_priv->gpu_error.hangcheck_work)) ++ seq_puts(m, "Hangcheck active, work pending\n"); ++ else ++ seq_puts(m, "Hangcheck inactive\n"); ++ ++ seq_printf(m, "GT active? %s\n", yesno(dev_priv->gt.awake)); ++ ++ for_each_engine(engine, dev_priv, id) { ++ seq_printf(m, "%s:\n", engine->name); ++ seq_printf(m, "\tseqno = %x [current %x, last %x], %dms ago\n", ++ engine->hangcheck.last_seqno, ++ seqno[id], ++ engine->hangcheck.next_seqno, ++ jiffies_to_msecs(jiffies - ++ engine->hangcheck.action_timestamp)); ++ ++ seq_printf(m, "\tACTHD = 0x%08llx [current 0x%08llx]\n", ++ (long long)engine->hangcheck.acthd, ++ (long long)acthd[id]); ++ ++ if (engine->id == RCS0) { ++ seq_puts(m, "\tinstdone read =\n"); ++ ++ i915_instdone_info(dev_priv, m, &instdone); ++ ++ seq_puts(m, "\tinstdone accu =\n"); ++ ++ i915_instdone_info(dev_priv, m, ++ &engine->hangcheck.instdone); ++ } ++ } ++ ++ return 0; ++} ++ ++static int i915_reset_info(struct seq_file *m, void *unused) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct i915_gpu_error *error = &dev_priv->gpu_error; ++ struct intel_engine_cs *engine; ++ enum intel_engine_id id; ++ ++ seq_printf(m, "full gpu reset = %u\n", i915_reset_count(error)); ++ ++ for_each_engine(engine, dev_priv, id) { ++ seq_printf(m, "%s = %u\n", engine->name, ++ i915_reset_engine_count(error, engine)); ++ } ++ ++ return 0; ++} ++ ++static int ironlake_drpc_info(struct seq_file *m) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ u32 rgvmodectl, rstdbyctl; ++ u16 crstandvid; ++ ++ rgvmodectl = I915_READ(MEMMODECTL); ++ rstdbyctl = I915_READ(RSTDBYCTL); ++ crstandvid = I915_READ16(CRSTANDVID); ++ ++ seq_printf(m, "HD boost: %s\n", yesno(rgvmodectl & MEMMODE_BOOST_EN)); ++ seq_printf(m, "Boost freq: %d\n", ++ (rgvmodectl & MEMMODE_BOOST_FREQ_MASK) >> ++ MEMMODE_BOOST_FREQ_SHIFT); ++ seq_printf(m, "HW control enabled: %s\n", ++ yesno(rgvmodectl & MEMMODE_HWIDLE_EN)); ++ seq_printf(m, "SW control enabled: %s\n", ++ yesno(rgvmodectl & MEMMODE_SWMODE_EN)); ++ seq_printf(m, "Gated voltage change: %s\n", ++ yesno(rgvmodectl & MEMMODE_RCLK_GATE)); ++ seq_printf(m, "Starting frequency: P%d\n", ++ (rgvmodectl & MEMMODE_FSTART_MASK) >> MEMMODE_FSTART_SHIFT); ++ seq_printf(m, "Max P-state: P%d\n", ++ (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT); ++ seq_printf(m, "Min P-state: P%d\n", (rgvmodectl & MEMMODE_FMIN_MASK)); ++ seq_printf(m, "RS1 VID: %d\n", (crstandvid & 0x3f)); ++ seq_printf(m, "RS2 VID: %d\n", ((crstandvid >> 8) & 0x3f)); ++ seq_printf(m, "Render standby enabled: %s\n", ++ yesno(!(rstdbyctl & RCX_SW_EXIT))); ++ seq_puts(m, "Current RS state: "); ++ switch (rstdbyctl & RSX_STATUS_MASK) { ++ case RSX_STATUS_ON: ++ seq_puts(m, "on\n"); ++ break; ++ case RSX_STATUS_RC1: ++ seq_puts(m, "RC1\n"); ++ break; ++ case RSX_STATUS_RC1E: ++ seq_puts(m, "RC1E\n"); ++ break; ++ case RSX_STATUS_RS1: ++ seq_puts(m, "RS1\n"); ++ break; ++ case RSX_STATUS_RS2: ++ seq_puts(m, "RS2 (RC6)\n"); ++ break; ++ case RSX_STATUS_RS3: ++ seq_puts(m, "RC3 (RC6+)\n"); ++ break; ++ default: ++ seq_puts(m, "unknown\n"); ++ break; ++ } ++ ++ return 0; ++} ++ ++static int i915_forcewake_domains(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *i915 = node_to_i915(m->private); ++ struct intel_uncore *uncore = &i915->uncore; ++ struct intel_uncore_forcewake_domain *fw_domain; ++ unsigned int tmp; ++ ++ seq_printf(m, "user.bypass_count = %u\n", ++ uncore->user_forcewake.count); ++ ++ for_each_fw_domain(fw_domain, uncore, tmp) ++ seq_printf(m, "%s.wake_count = %u\n", ++ intel_uncore_forcewake_domain_to_str(fw_domain->id), ++ READ_ONCE(fw_domain->wake_count)); ++ ++ return 0; ++} ++ ++static void print_rc6_res(struct seq_file *m, ++ const char *title, ++ const i915_reg_t reg) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ ++ seq_printf(m, "%s %u (%llu us)\n", ++ title, I915_READ(reg), ++ intel_rc6_residency_us(dev_priv, reg)); ++} ++ ++static int vlv_drpc_info(struct seq_file *m) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ u32 rcctl1, pw_status; ++ ++ pw_status = I915_READ(VLV_GTLC_PW_STATUS); ++ rcctl1 = I915_READ(GEN6_RC_CONTROL); ++ ++ seq_printf(m, "RC6 Enabled: %s\n", ++ yesno(rcctl1 & (GEN7_RC_CTL_TO_MODE | ++ GEN6_RC_CTL_EI_MODE(1)))); ++ seq_printf(m, "Render Power Well: %s\n", ++ (pw_status & VLV_GTLC_PW_RENDER_STATUS_MASK) ? "Up" : "Down"); ++ seq_printf(m, "Media Power Well: %s\n", ++ (pw_status & VLV_GTLC_PW_MEDIA_STATUS_MASK) ? "Up" : "Down"); ++ ++ print_rc6_res(m, "Render RC6 residency since boot:", VLV_GT_RENDER_RC6); ++ print_rc6_res(m, "Media RC6 residency since boot:", VLV_GT_MEDIA_RC6); ++ ++ return i915_forcewake_domains(m, NULL); ++} ++ ++static int gen6_drpc_info(struct seq_file *m) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ u32 gt_core_status, rcctl1, rc6vids = 0; ++ u32 gen9_powergate_enable = 0, gen9_powergate_status = 0; ++ ++ gt_core_status = I915_READ_FW(GEN6_GT_CORE_STATUS); ++ trace_i915_reg_rw(false, GEN6_GT_CORE_STATUS, gt_core_status, 4, true); ++ ++ rcctl1 = I915_READ(GEN6_RC_CONTROL); ++ if (INTEL_GEN(dev_priv) >= 9) { ++ gen9_powergate_enable = I915_READ(GEN9_PG_ENABLE); ++ gen9_powergate_status = I915_READ(GEN9_PWRGT_DOMAIN_STATUS); ++ } ++ ++ if (INTEL_GEN(dev_priv) <= 7) { ++ mutex_lock(&dev_priv->pcu_lock); ++ sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, ++ &rc6vids); ++ mutex_unlock(&dev_priv->pcu_lock); ++ } ++ ++ seq_printf(m, "RC1e Enabled: %s\n", ++ yesno(rcctl1 & GEN6_RC_CTL_RC1e_ENABLE)); ++ seq_printf(m, "RC6 Enabled: %s\n", ++ yesno(rcctl1 & GEN6_RC_CTL_RC6_ENABLE)); ++ if (INTEL_GEN(dev_priv) >= 9) { ++ seq_printf(m, "Render Well Gating Enabled: %s\n", ++ yesno(gen9_powergate_enable & GEN9_RENDER_PG_ENABLE)); ++ seq_printf(m, "Media Well Gating Enabled: %s\n", ++ yesno(gen9_powergate_enable & GEN9_MEDIA_PG_ENABLE)); ++ } ++ seq_printf(m, "Deep RC6 Enabled: %s\n", ++ yesno(rcctl1 & GEN6_RC_CTL_RC6p_ENABLE)); ++ seq_printf(m, "Deepest RC6 Enabled: %s\n", ++ yesno(rcctl1 & GEN6_RC_CTL_RC6pp_ENABLE)); ++ seq_puts(m, "Current RC state: "); ++ switch (gt_core_status & GEN6_RCn_MASK) { ++ case GEN6_RC0: ++ if (gt_core_status & GEN6_CORE_CPD_STATE_MASK) ++ seq_puts(m, "Core Power Down\n"); ++ else ++ seq_puts(m, "on\n"); ++ break; ++ case GEN6_RC3: ++ seq_puts(m, "RC3\n"); ++ break; ++ case GEN6_RC6: ++ seq_puts(m, "RC6\n"); ++ break; ++ case GEN6_RC7: ++ seq_puts(m, "RC7\n"); ++ break; ++ default: ++ seq_puts(m, "Unknown\n"); ++ break; ++ } ++ ++ seq_printf(m, "Core Power Down: %s\n", ++ yesno(gt_core_status & GEN6_CORE_CPD_STATE_MASK)); ++ if (INTEL_GEN(dev_priv) >= 9) { ++ seq_printf(m, "Render Power Well: %s\n", ++ (gen9_powergate_status & ++ GEN9_PWRGT_RENDER_STATUS_MASK) ? "Up" : "Down"); ++ seq_printf(m, "Media Power Well: %s\n", ++ (gen9_powergate_status & ++ GEN9_PWRGT_MEDIA_STATUS_MASK) ? "Up" : "Down"); ++ } ++ ++ /* Not exactly sure what this is */ ++ print_rc6_res(m, "RC6 \"Locked to RPn\" residency since boot:", ++ GEN6_GT_GFX_RC6_LOCKED); ++ print_rc6_res(m, "RC6 residency since boot:", GEN6_GT_GFX_RC6); ++ print_rc6_res(m, "RC6+ residency since boot:", GEN6_GT_GFX_RC6p); ++ print_rc6_res(m, "RC6++ residency since boot:", GEN6_GT_GFX_RC6pp); ++ ++ if (INTEL_GEN(dev_priv) <= 7) { ++ seq_printf(m, "RC6 voltage: %dmV\n", ++ GEN6_DECODE_RC6_VID(((rc6vids >> 0) & 0xff))); ++ seq_printf(m, "RC6+ voltage: %dmV\n", ++ GEN6_DECODE_RC6_VID(((rc6vids >> 8) & 0xff))); ++ seq_printf(m, "RC6++ voltage: %dmV\n", ++ GEN6_DECODE_RC6_VID(((rc6vids >> 16) & 0xff))); ++ } ++ ++ return i915_forcewake_domains(m, NULL); ++} ++ ++static int i915_drpc_info(struct seq_file *m, void *unused) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ intel_wakeref_t wakeref; ++ int err = -ENODEV; ++ ++ with_intel_runtime_pm(dev_priv, wakeref) { ++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ err = vlv_drpc_info(m); ++ else if (INTEL_GEN(dev_priv) >= 6) ++ err = gen6_drpc_info(m); ++ else ++ err = ironlake_drpc_info(m); ++ } ++ ++ return err; ++} ++ ++static int i915_frontbuffer_tracking(struct seq_file *m, void *unused) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ ++ seq_printf(m, "FB tracking busy bits: 0x%08x\n", ++ dev_priv->fb_tracking.busy_bits); ++ ++ seq_printf(m, "FB tracking flip bits: 0x%08x\n", ++ dev_priv->fb_tracking.flip_bits); ++ ++ return 0; ++} ++ ++static int i915_fbc_status(struct seq_file *m, void *unused) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct intel_fbc *fbc = &dev_priv->fbc; ++ intel_wakeref_t wakeref; ++ ++ if (!HAS_FBC(dev_priv)) ++ return -ENODEV; ++ ++ wakeref = intel_runtime_pm_get(dev_priv); ++ mutex_lock(&fbc->lock); ++ ++ if (intel_fbc_is_active(dev_priv)) ++ seq_puts(m, "FBC enabled\n"); ++ else ++ seq_printf(m, "FBC disabled: %s\n", fbc->no_fbc_reason); ++ ++ if (intel_fbc_is_active(dev_priv)) { ++ u32 mask; ++ ++ if (INTEL_GEN(dev_priv) >= 8) ++ mask = I915_READ(IVB_FBC_STATUS2) & BDW_FBC_COMP_SEG_MASK; ++ else if (INTEL_GEN(dev_priv) >= 7) ++ mask = I915_READ(IVB_FBC_STATUS2) & IVB_FBC_COMP_SEG_MASK; ++ else if (INTEL_GEN(dev_priv) >= 5) ++ mask = I915_READ(ILK_DPFC_STATUS) & ILK_DPFC_COMP_SEG_MASK; ++ else if (IS_G4X(dev_priv)) ++ mask = I915_READ(DPFC_STATUS) & DPFC_COMP_SEG_MASK; ++ else ++ mask = I915_READ(FBC_STATUS) & (FBC_STAT_COMPRESSING | ++ FBC_STAT_COMPRESSED); ++ ++ seq_printf(m, "Compressing: %s\n", yesno(mask)); ++ } ++ ++ mutex_unlock(&fbc->lock); ++ intel_runtime_pm_put(dev_priv, wakeref); ++ ++ return 0; ++} ++ ++static int i915_fbc_false_color_get(void *data, u64 *val) ++{ ++ struct drm_i915_private *dev_priv = data; ++ ++ if (INTEL_GEN(dev_priv) < 7 || !HAS_FBC(dev_priv)) ++ return -ENODEV; ++ ++ *val = dev_priv->fbc.false_color; ++ ++ return 0; ++} ++ ++static int i915_fbc_false_color_set(void *data, u64 val) ++{ ++ struct drm_i915_private *dev_priv = data; ++ u32 reg; ++ ++ if (INTEL_GEN(dev_priv) < 7 || !HAS_FBC(dev_priv)) ++ return -ENODEV; ++ ++ mutex_lock(&dev_priv->fbc.lock); ++ ++ reg = I915_READ(ILK_DPFC_CONTROL); ++ dev_priv->fbc.false_color = val; ++ ++ I915_WRITE(ILK_DPFC_CONTROL, val ? ++ (reg | FBC_CTL_FALSE_COLOR) : ++ (reg & ~FBC_CTL_FALSE_COLOR)); ++ ++ mutex_unlock(&dev_priv->fbc.lock); ++ return 0; ++} ++ ++DEFINE_SIMPLE_ATTRIBUTE(i915_fbc_false_color_fops, ++ i915_fbc_false_color_get, i915_fbc_false_color_set, ++ "%llu\n"); ++ ++static int i915_ips_status(struct seq_file *m, void *unused) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ intel_wakeref_t wakeref; ++ ++ if (!HAS_IPS(dev_priv)) ++ return -ENODEV; ++ ++ wakeref = intel_runtime_pm_get(dev_priv); ++ ++ seq_printf(m, "Enabled by kernel parameter: %s\n", ++ yesno(i915_modparams.enable_ips)); ++ ++ if (INTEL_GEN(dev_priv) >= 8) { ++ seq_puts(m, "Currently: unknown\n"); ++ } else { ++ if (I915_READ(IPS_CTL) & IPS_ENABLE) ++ seq_puts(m, "Currently: enabled\n"); ++ else ++ seq_puts(m, "Currently: disabled\n"); ++ } ++ ++ intel_runtime_pm_put(dev_priv, wakeref); ++ ++ return 0; ++} ++ ++static int i915_sr_status(struct seq_file *m, void *unused) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ intel_wakeref_t wakeref; ++ bool sr_enabled = false; ++ ++ wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); ++ ++ if (INTEL_GEN(dev_priv) >= 9) ++ /* no global SR status; inspect per-plane WM */; ++ else if (HAS_PCH_SPLIT(dev_priv)) ++ sr_enabled = I915_READ(WM1_LP_ILK) & WM1_LP_SR_EN; ++ else if (IS_I965GM(dev_priv) || IS_G4X(dev_priv) || ++ IS_I945G(dev_priv) || IS_I945GM(dev_priv)) ++ sr_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN; ++ else if (IS_I915GM(dev_priv)) ++ sr_enabled = I915_READ(INSTPM) & INSTPM_SELF_EN; ++ else if (IS_PINEVIEW(dev_priv)) ++ sr_enabled = I915_READ(DSPFW3) & PINEVIEW_SELF_REFRESH_EN; ++ else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ sr_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN; ++ ++ intel_display_power_put(dev_priv, POWER_DOMAIN_INIT, wakeref); ++ ++ seq_printf(m, "self-refresh: %s\n", enableddisabled(sr_enabled)); ++ ++ return 0; ++} ++ ++static int i915_emon_status(struct seq_file *m, void *unused) ++{ ++ struct drm_i915_private *i915 = node_to_i915(m->private); ++ intel_wakeref_t wakeref; ++ ++ if (!IS_GEN(i915, 5)) ++ return -ENODEV; ++ ++ with_intel_runtime_pm(i915, wakeref) { ++ unsigned long temp, chipset, gfx; ++ ++ temp = i915_mch_val(i915); ++ chipset = i915_chipset_val(i915); ++ gfx = i915_gfx_val(i915); ++ ++ seq_printf(m, "GMCH temp: %ld\n", temp); ++ seq_printf(m, "Chipset power: %ld\n", chipset); ++ seq_printf(m, "GFX power: %ld\n", gfx); ++ seq_printf(m, "Total power: %ld\n", chipset + gfx); ++ } ++ ++ return 0; ++} ++ ++static int i915_ring_freq_table(struct seq_file *m, void *unused) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct intel_rps *rps = &dev_priv->gt_pm.rps; ++ unsigned int max_gpu_freq, min_gpu_freq; ++ intel_wakeref_t wakeref; ++ int gpu_freq, ia_freq; ++ int ret; ++ ++ if (!HAS_LLC(dev_priv)) ++ return -ENODEV; ++ ++ wakeref = intel_runtime_pm_get(dev_priv); ++ ++ ret = mutex_lock_interruptible(&dev_priv->pcu_lock); ++ if (ret) ++ goto out; ++ ++ min_gpu_freq = rps->min_freq; ++ max_gpu_freq = rps->max_freq; ++ if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) { ++ /* Convert GT frequency to 50 HZ units */ ++ min_gpu_freq /= GEN9_FREQ_SCALER; ++ max_gpu_freq /= GEN9_FREQ_SCALER; ++ } ++ ++ seq_puts(m, "GPU freq (MHz)\tEffective CPU freq (MHz)\tEffective Ring freq (MHz)\n"); ++ ++ for (gpu_freq = min_gpu_freq; gpu_freq <= max_gpu_freq; gpu_freq++) { ++ ia_freq = gpu_freq; ++ sandybridge_pcode_read(dev_priv, ++ GEN6_PCODE_READ_MIN_FREQ_TABLE, ++ &ia_freq); ++ seq_printf(m, "%d\t\t%d\t\t\t\t%d\n", ++ intel_gpu_freq(dev_priv, (gpu_freq * ++ (IS_GEN9_BC(dev_priv) || ++ INTEL_GEN(dev_priv) >= 10 ? ++ GEN9_FREQ_SCALER : 1))), ++ ((ia_freq >> 0) & 0xff) * 100, ++ ((ia_freq >> 8) & 0xff) * 100); ++ } ++ ++ mutex_unlock(&dev_priv->pcu_lock); ++ ++out: ++ intel_runtime_pm_put(dev_priv, wakeref); ++ return ret; ++} ++ ++static int i915_opregion(struct seq_file *m, void *unused) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct drm_device *dev = &dev_priv->drm; ++ struct intel_opregion *opregion = &dev_priv->opregion; ++ int ret; ++ ++ ret = mutex_lock_interruptible(&dev->struct_mutex); ++ if (ret) ++ goto out; ++ ++ if (opregion->header) ++ seq_write(m, opregion->header, OPREGION_SIZE); ++ ++ mutex_unlock(&dev->struct_mutex); ++ ++out: ++ return 0; ++} ++ ++static int i915_vbt(struct seq_file *m, void *unused) ++{ ++ struct intel_opregion *opregion = &node_to_i915(m->private)->opregion; ++ ++ if (opregion->vbt) ++ seq_write(m, opregion->vbt, opregion->vbt_size); ++ ++ return 0; ++} ++ ++static int i915_gem_framebuffer_info(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct drm_device *dev = &dev_priv->drm; ++ struct intel_framebuffer *fbdev_fb = NULL; ++ struct drm_framebuffer *drm_fb; ++ int ret; ++ ++ ret = mutex_lock_interruptible(&dev->struct_mutex); ++ if (ret) ++ return ret; ++ ++#ifdef CONFIG_DRM_FBDEV_EMULATION ++ if (dev_priv->fbdev && dev_priv->fbdev->helper.fb) { ++ fbdev_fb = to_intel_framebuffer(dev_priv->fbdev->helper.fb); ++ ++ seq_printf(m, "fbcon size: %d x %d, depth %d, %d bpp, modifier 0x%llx, refcount %d, obj ", ++ fbdev_fb->base.width, ++ fbdev_fb->base.height, ++ fbdev_fb->base.format->depth, ++ fbdev_fb->base.format->cpp[0] * 8, ++ fbdev_fb->base.modifier, ++ drm_framebuffer_read_refcount(&fbdev_fb->base)); ++ describe_obj(m, intel_fb_obj(&fbdev_fb->base)); ++ seq_putc(m, '\n'); ++ } ++#endif ++ ++ mutex_lock(&dev->mode_config.fb_lock); ++ drm_for_each_fb(drm_fb, dev) { ++ struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb); ++ if (fb == fbdev_fb) ++ continue; ++ ++ seq_printf(m, "user size: %d x %d, depth %d, %d bpp, modifier 0x%llx, refcount %d, obj ", ++ fb->base.width, ++ fb->base.height, ++ fb->base.format->depth, ++ fb->base.format->cpp[0] * 8, ++ fb->base.modifier, ++ drm_framebuffer_read_refcount(&fb->base)); ++ describe_obj(m, intel_fb_obj(&fb->base)); ++ seq_putc(m, '\n'); ++ } ++ mutex_unlock(&dev->mode_config.fb_lock); ++ mutex_unlock(&dev->struct_mutex); ++ ++ return 0; ++} ++ ++static void describe_ctx_ring(struct seq_file *m, struct intel_ring *ring) ++{ ++ seq_printf(m, " (ringbuffer, space: %d, head: %u, tail: %u, emit: %u)", ++ ring->space, ring->head, ring->tail, ring->emit); ++} ++ ++static int i915_context_status(struct seq_file *m, void *unused) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct drm_device *dev = &dev_priv->drm; ++ struct i915_gem_context *ctx; ++ int ret; ++ ++ ret = mutex_lock_interruptible(&dev->struct_mutex); ++ if (ret) ++ return ret; ++ ++ list_for_each_entry(ctx, &dev_priv->contexts.list, link) { ++ struct intel_context *ce; ++ ++ seq_puts(m, "HW context "); ++ if (!list_empty(&ctx->hw_id_link)) ++ seq_printf(m, "%x [pin %u]", ctx->hw_id, ++ atomic_read(&ctx->hw_id_pin_count)); ++ if (ctx->pid) { ++ struct task_struct *task; ++ ++ task = get_pid_task(ctx->pid, PIDTYPE_PID); ++ if (task) { ++ seq_printf(m, "(%s [%d]) ", ++ task->comm, task->pid); ++ put_task_struct(task); ++ } ++ } else if (IS_ERR(ctx->file_priv)) { ++ seq_puts(m, "(deleted) "); ++ } else { ++ seq_puts(m, "(kernel) "); ++ } ++ ++ seq_putc(m, ctx->remap_slice ? 'R' : 'r'); ++ seq_putc(m, '\n'); ++ ++ list_for_each_entry(ce, &ctx->active_engines, active_link) { ++ seq_printf(m, "%s: ", ce->engine->name); ++ if (ce->state) ++ describe_obj(m, ce->state->obj); ++ if (ce->ring) ++ describe_ctx_ring(m, ce->ring); ++ seq_putc(m, '\n'); ++ } ++ ++ seq_putc(m, '\n'); ++ } ++ ++ mutex_unlock(&dev->struct_mutex); ++ ++ return 0; ++} ++ ++static const char *swizzle_string(unsigned swizzle) ++{ ++ switch (swizzle) { ++ case I915_BIT_6_SWIZZLE_NONE: ++ return "none"; ++ case I915_BIT_6_SWIZZLE_9: ++ return "bit9"; ++ case I915_BIT_6_SWIZZLE_9_10: ++ return "bit9/bit10"; ++ case I915_BIT_6_SWIZZLE_9_11: ++ return "bit9/bit11"; ++ case I915_BIT_6_SWIZZLE_9_10_11: ++ return "bit9/bit10/bit11"; ++ case I915_BIT_6_SWIZZLE_9_17: ++ return "bit9/bit17"; ++ case I915_BIT_6_SWIZZLE_9_10_17: ++ return "bit9/bit10/bit17"; ++ case I915_BIT_6_SWIZZLE_UNKNOWN: ++ return "unknown"; ++ } ++ ++ return "bug"; ++} ++ ++static int i915_swizzle_info(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ intel_wakeref_t wakeref; ++ ++ wakeref = intel_runtime_pm_get(dev_priv); ++ ++ seq_printf(m, "bit6 swizzle for X-tiling = %s\n", ++ swizzle_string(dev_priv->mm.bit_6_swizzle_x)); ++ seq_printf(m, "bit6 swizzle for Y-tiling = %s\n", ++ swizzle_string(dev_priv->mm.bit_6_swizzle_y)); ++ ++ if (IS_GEN_RANGE(dev_priv, 3, 4)) { ++ seq_printf(m, "DDC = 0x%08x\n", ++ I915_READ(DCC)); ++ seq_printf(m, "DDC2 = 0x%08x\n", ++ I915_READ(DCC2)); ++ seq_printf(m, "C0DRB3 = 0x%04x\n", ++ I915_READ16(C0DRB3)); ++ seq_printf(m, "C1DRB3 = 0x%04x\n", ++ I915_READ16(C1DRB3)); ++ } else if (INTEL_GEN(dev_priv) >= 6) { ++ seq_printf(m, "MAD_DIMM_C0 = 0x%08x\n", ++ I915_READ(MAD_DIMM_C0)); ++ seq_printf(m, "MAD_DIMM_C1 = 0x%08x\n", ++ I915_READ(MAD_DIMM_C1)); ++ seq_printf(m, "MAD_DIMM_C2 = 0x%08x\n", ++ I915_READ(MAD_DIMM_C2)); ++ seq_printf(m, "TILECTL = 0x%08x\n", ++ I915_READ(TILECTL)); ++ if (INTEL_GEN(dev_priv) >= 8) ++ seq_printf(m, "GAMTARBMODE = 0x%08x\n", ++ I915_READ(GAMTARBMODE)); ++ else ++ seq_printf(m, "ARB_MODE = 0x%08x\n", ++ I915_READ(ARB_MODE)); ++ seq_printf(m, "DISP_ARB_CTL = 0x%08x\n", ++ I915_READ(DISP_ARB_CTL)); ++ } ++ ++ if (dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) ++ seq_puts(m, "L-shaped memory detected\n"); ++ ++ intel_runtime_pm_put(dev_priv, wakeref); ++ ++ return 0; ++} ++ ++static const char *rps_power_to_str(unsigned int power) ++{ ++ static const char * const strings[] = { ++ [LOW_POWER] = "low power", ++ [BETWEEN] = "mixed", ++ [HIGH_POWER] = "high power", ++ }; ++ ++ if (power >= ARRAY_SIZE(strings) || !strings[power]) ++ return "unknown"; ++ ++ return strings[power]; ++} ++ ++static int i915_rps_boost_info(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct intel_rps *rps = &dev_priv->gt_pm.rps; ++ u32 act_freq = rps->cur_freq; ++ intel_wakeref_t wakeref; ++ ++ with_intel_runtime_pm_if_in_use(dev_priv, wakeref) { ++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { ++ mutex_lock(&dev_priv->pcu_lock); ++ act_freq = vlv_punit_read(dev_priv, ++ PUNIT_REG_GPU_FREQ_STS); ++ act_freq = (act_freq >> 8) & 0xff; ++ mutex_unlock(&dev_priv->pcu_lock); ++ } else { ++ act_freq = intel_get_cagf(dev_priv, ++ I915_READ(GEN6_RPSTAT1)); ++ } ++ } ++ ++ seq_printf(m, "RPS enabled? %d\n", rps->enabled); ++ seq_printf(m, "GPU busy? %s [%d requests]\n", ++ yesno(dev_priv->gt.awake), dev_priv->gt.active_requests); ++ seq_printf(m, "Boosts outstanding? %d\n", ++ atomic_read(&rps->num_waiters)); ++ seq_printf(m, "Interactive? %d\n", READ_ONCE(rps->power.interactive)); ++ seq_printf(m, "Frequency requested %d, actual %d\n", ++ intel_gpu_freq(dev_priv, rps->cur_freq), ++ intel_gpu_freq(dev_priv, act_freq)); ++ seq_printf(m, " min hard:%d, soft:%d; max soft:%d, hard:%d\n", ++ intel_gpu_freq(dev_priv, rps->min_freq), ++ intel_gpu_freq(dev_priv, rps->min_freq_softlimit), ++ intel_gpu_freq(dev_priv, rps->max_freq_softlimit), ++ intel_gpu_freq(dev_priv, rps->max_freq)); ++ seq_printf(m, " idle:%d, efficient:%d, boost:%d\n", ++ intel_gpu_freq(dev_priv, rps->idle_freq), ++ intel_gpu_freq(dev_priv, rps->efficient_freq), ++ intel_gpu_freq(dev_priv, rps->boost_freq)); ++ ++ seq_printf(m, "Wait boosts: %d\n", atomic_read(&rps->boosts)); ++ ++ if (INTEL_GEN(dev_priv) >= 6 && ++ rps->enabled && ++ dev_priv->gt.active_requests) { ++ u32 rpup, rpupei; ++ u32 rpdown, rpdownei; ++ ++ intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL); ++ rpup = I915_READ_FW(GEN6_RP_CUR_UP) & GEN6_RP_EI_MASK; ++ rpupei = I915_READ_FW(GEN6_RP_CUR_UP_EI) & GEN6_RP_EI_MASK; ++ rpdown = I915_READ_FW(GEN6_RP_CUR_DOWN) & GEN6_RP_EI_MASK; ++ rpdownei = I915_READ_FW(GEN6_RP_CUR_DOWN_EI) & GEN6_RP_EI_MASK; ++ intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); ++ ++ seq_printf(m, "\nRPS Autotuning (current \"%s\" window):\n", ++ rps_power_to_str(rps->power.mode)); ++ seq_printf(m, " Avg. up: %d%% [above threshold? %d%%]\n", ++ rpup && rpupei ? 100 * rpup / rpupei : 0, ++ rps->power.up_threshold); ++ seq_printf(m, " Avg. down: %d%% [below threshold? %d%%]\n", ++ rpdown && rpdownei ? 100 * rpdown / rpdownei : 0, ++ rps->power.down_threshold); ++ } else { ++ seq_puts(m, "\nRPS Autotuning inactive\n"); ++ } ++ ++ return 0; ++} ++ ++static int i915_llc(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ const bool edram = INTEL_GEN(dev_priv) > 8; ++ ++ seq_printf(m, "LLC: %s\n", yesno(HAS_LLC(dev_priv))); ++ seq_printf(m, "%s: %uMB\n", edram ? "eDRAM" : "eLLC", ++ dev_priv->edram_size_mb); ++ ++ return 0; ++} ++ ++static int i915_huc_load_status_info(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ intel_wakeref_t wakeref; ++ struct drm_printer p; ++ ++ if (!HAS_HUC(dev_priv)) ++ return -ENODEV; ++ ++ p = drm_seq_file_printer(m); ++ intel_uc_fw_dump(&dev_priv->huc.fw, &p); ++ ++ with_intel_runtime_pm(dev_priv, wakeref) ++ seq_printf(m, "\nHuC status 0x%08x:\n", I915_READ(HUC_STATUS2)); ++ ++ return 0; ++} ++ ++static int i915_guc_load_status_info(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ intel_wakeref_t wakeref; ++ struct drm_printer p; ++ ++ if (!HAS_GUC(dev_priv)) ++ return -ENODEV; ++ ++ p = drm_seq_file_printer(m); ++ intel_uc_fw_dump(&dev_priv->guc.fw, &p); ++ ++ with_intel_runtime_pm(dev_priv, wakeref) { ++ u32 tmp = I915_READ(GUC_STATUS); ++ u32 i; ++ ++ seq_printf(m, "\nGuC status 0x%08x:\n", tmp); ++ seq_printf(m, "\tBootrom status = 0x%x\n", ++ (tmp & GS_BOOTROM_MASK) >> GS_BOOTROM_SHIFT); ++ seq_printf(m, "\tuKernel status = 0x%x\n", ++ (tmp & GS_UKERNEL_MASK) >> GS_UKERNEL_SHIFT); ++ seq_printf(m, "\tMIA Core status = 0x%x\n", ++ (tmp & GS_MIA_MASK) >> GS_MIA_SHIFT); ++ seq_puts(m, "\nScratch registers:\n"); ++ for (i = 0; i < 16; i++) { ++ seq_printf(m, "\t%2d: \t0x%x\n", ++ i, I915_READ(SOFT_SCRATCH(i))); ++ } ++ } ++ ++ return 0; ++} ++ ++static const char * ++stringify_guc_log_type(enum guc_log_buffer_type type) ++{ ++ switch (type) { ++ case GUC_ISR_LOG_BUFFER: ++ return "ISR"; ++ case GUC_DPC_LOG_BUFFER: ++ return "DPC"; ++ case GUC_CRASH_DUMP_LOG_BUFFER: ++ return "CRASH"; ++ default: ++ MISSING_CASE(type); ++ } ++ ++ return ""; ++} ++ ++static void i915_guc_log_info(struct seq_file *m, ++ struct drm_i915_private *dev_priv) ++{ ++ struct intel_guc_log *log = &dev_priv->guc.log; ++ enum guc_log_buffer_type type; ++ ++ if (!intel_guc_log_relay_enabled(log)) { ++ seq_puts(m, "GuC log relay disabled\n"); ++ return; ++ } ++ ++ seq_puts(m, "GuC logging stats:\n"); ++ ++ seq_printf(m, "\tRelay full count: %u\n", ++ log->relay.full_count); ++ ++ for (type = GUC_ISR_LOG_BUFFER; type < GUC_MAX_LOG_BUFFER; type++) { ++ seq_printf(m, "\t%s:\tflush count %10u, overflow count %10u\n", ++ stringify_guc_log_type(type), ++ log->stats[type].flush, ++ log->stats[type].sampled_overflow); ++ } ++} ++ ++static void i915_guc_client_info(struct seq_file *m, ++ struct drm_i915_private *dev_priv, ++ struct intel_guc_client *client) ++{ ++ struct intel_engine_cs *engine; ++ enum intel_engine_id id; ++ u64 tot = 0; ++ ++ seq_printf(m, "\tPriority %d, GuC stage index: %u, PD offset 0x%x\n", ++ client->priority, client->stage_id, client->proc_desc_offset); ++ seq_printf(m, "\tDoorbell id %d, offset: 0x%lx\n", ++ client->doorbell_id, client->doorbell_offset); ++ ++ for_each_engine(engine, dev_priv, id) { ++ u64 submissions = client->submissions[id]; ++ tot += submissions; ++ seq_printf(m, "\tSubmissions: %llu %s\n", ++ submissions, engine->name); ++ } ++ seq_printf(m, "\tTotal: %llu\n", tot); ++} ++ ++static int i915_guc_info(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ const struct intel_guc *guc = &dev_priv->guc; ++ ++ if (!USES_GUC(dev_priv)) ++ return -ENODEV; ++ ++ i915_guc_log_info(m, dev_priv); ++ ++ if (!USES_GUC_SUBMISSION(dev_priv)) ++ return 0; ++ ++ GEM_BUG_ON(!guc->execbuf_client); ++ ++ seq_printf(m, "\nDoorbell map:\n"); ++ seq_printf(m, "\t%*pb\n", GUC_NUM_DOORBELLS, guc->doorbell_bitmap); ++ seq_printf(m, "Doorbell next cacheline: 0x%x\n", guc->db_cacheline); ++ ++ seq_printf(m, "\nGuC execbuf client @ %p:\n", guc->execbuf_client); ++ i915_guc_client_info(m, dev_priv, guc->execbuf_client); ++ if (guc->preempt_client) { ++ seq_printf(m, "\nGuC preempt client @ %p:\n", ++ guc->preempt_client); ++ i915_guc_client_info(m, dev_priv, guc->preempt_client); ++ } ++ ++ /* Add more as required ... */ ++ ++ return 0; ++} ++ ++static int i915_guc_stage_pool(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ const struct intel_guc *guc = &dev_priv->guc; ++ struct guc_stage_desc *desc = guc->stage_desc_pool_vaddr; ++ struct intel_guc_client *client = guc->execbuf_client; ++ intel_engine_mask_t tmp; ++ int index; ++ ++ if (!USES_GUC_SUBMISSION(dev_priv)) ++ return -ENODEV; ++ ++ for (index = 0; index < GUC_MAX_STAGE_DESCRIPTORS; index++, desc++) { ++ struct intel_engine_cs *engine; ++ ++ if (!(desc->attribute & GUC_STAGE_DESC_ATTR_ACTIVE)) ++ continue; ++ ++ seq_printf(m, "GuC stage descriptor %u:\n", index); ++ seq_printf(m, "\tIndex: %u\n", desc->stage_id); ++ seq_printf(m, "\tAttribute: 0x%x\n", desc->attribute); ++ seq_printf(m, "\tPriority: %d\n", desc->priority); ++ seq_printf(m, "\tDoorbell id: %d\n", desc->db_id); ++ seq_printf(m, "\tEngines used: 0x%x\n", ++ desc->engines_used); ++ seq_printf(m, "\tDoorbell trigger phy: 0x%llx, cpu: 0x%llx, uK: 0x%x\n", ++ desc->db_trigger_phy, ++ desc->db_trigger_cpu, ++ desc->db_trigger_uk); ++ seq_printf(m, "\tProcess descriptor: 0x%x\n", ++ desc->process_desc); ++ seq_printf(m, "\tWorkqueue address: 0x%x, size: 0x%x\n", ++ desc->wq_addr, desc->wq_size); ++ seq_putc(m, '\n'); ++ ++ for_each_engine_masked(engine, dev_priv, client->engines, tmp) { ++ u32 guc_engine_id = engine->guc_id; ++ struct guc_execlist_context *lrc = ++ &desc->lrc[guc_engine_id]; ++ ++ seq_printf(m, "\t%s LRC:\n", engine->name); ++ seq_printf(m, "\t\tContext desc: 0x%x\n", ++ lrc->context_desc); ++ seq_printf(m, "\t\tContext id: 0x%x\n", lrc->context_id); ++ seq_printf(m, "\t\tLRCA: 0x%x\n", lrc->ring_lrca); ++ seq_printf(m, "\t\tRing begin: 0x%x\n", lrc->ring_begin); ++ seq_printf(m, "\t\tRing end: 0x%x\n", lrc->ring_end); ++ seq_putc(m, '\n'); ++ } ++ } ++ ++ return 0; ++} ++ ++static int i915_guc_log_dump(struct seq_file *m, void *data) ++{ ++ struct drm_info_node *node = m->private; ++ struct drm_i915_private *dev_priv = node_to_i915(node); ++ bool dump_load_err = !!node->info_ent->data; ++ struct drm_i915_gem_object *obj = NULL; ++ u32 *log; ++ int i = 0; ++ ++ if (!HAS_GUC(dev_priv)) ++ return -ENODEV; ++ ++ if (dump_load_err) ++ obj = dev_priv->guc.load_err_log; ++ else if (dev_priv->guc.log.vma) ++ obj = dev_priv->guc.log.vma->obj; ++ ++ if (!obj) ++ return 0; ++ ++ log = i915_gem_object_pin_map(obj, I915_MAP_WC); ++ if (IS_ERR(log)) { ++ DRM_DEBUG("Failed to pin object\n"); ++ seq_puts(m, "(log data unaccessible)\n"); ++ return PTR_ERR(log); ++ } ++ ++ for (i = 0; i < obj->base.size / sizeof(u32); i += 4) ++ seq_printf(m, "0x%08x 0x%08x 0x%08x 0x%08x\n", ++ *(log + i), *(log + i + 1), ++ *(log + i + 2), *(log + i + 3)); ++ ++ seq_putc(m, '\n'); ++ ++ i915_gem_object_unpin_map(obj); ++ ++ return 0; ++} ++ ++static int i915_guc_log_level_get(void *data, u64 *val) ++{ ++ struct drm_i915_private *dev_priv = data; ++ ++ if (!USES_GUC(dev_priv)) ++ return -ENODEV; ++ ++ *val = intel_guc_log_get_level(&dev_priv->guc.log); ++ ++ return 0; ++} ++ ++static int i915_guc_log_level_set(void *data, u64 val) ++{ ++ struct drm_i915_private *dev_priv = data; ++ ++ if (!USES_GUC(dev_priv)) ++ return -ENODEV; ++ ++ return intel_guc_log_set_level(&dev_priv->guc.log, val); ++} ++ ++DEFINE_SIMPLE_ATTRIBUTE(i915_guc_log_level_fops, ++ i915_guc_log_level_get, i915_guc_log_level_set, ++ "%lld\n"); ++ ++static int i915_guc_log_relay_open(struct inode *inode, struct file *file) ++{ ++ struct drm_i915_private *dev_priv = inode->i_private; ++ ++ if (!USES_GUC(dev_priv)) ++ return -ENODEV; ++ ++ file->private_data = &dev_priv->guc.log; ++ ++ return intel_guc_log_relay_open(&dev_priv->guc.log); ++} ++ ++static ssize_t ++i915_guc_log_relay_write(struct file *filp, ++ const char __user *ubuf, ++ size_t cnt, ++ loff_t *ppos) ++{ ++ struct intel_guc_log *log = filp->private_data; ++ ++ intel_guc_log_relay_flush(log); ++ ++ return cnt; ++} ++ ++static int i915_guc_log_relay_release(struct inode *inode, struct file *file) ++{ ++ struct drm_i915_private *dev_priv = inode->i_private; ++ ++ intel_guc_log_relay_close(&dev_priv->guc.log); ++ ++ return 0; ++} ++ ++static const struct file_operations i915_guc_log_relay_fops = { ++ .owner = THIS_MODULE, ++ .open = i915_guc_log_relay_open, ++ .write = i915_guc_log_relay_write, ++ .release = i915_guc_log_relay_release, ++}; ++ ++static int i915_psr_sink_status_show(struct seq_file *m, void *data) ++{ ++ u8 val; ++ static const char * const sink_status[] = { ++ "inactive", ++ "transition to active, capture and display", ++ "active, display from RFB", ++ "active, capture and display on sink device timings", ++ "transition to inactive, capture and display, timing re-sync", ++ "reserved", ++ "reserved", ++ "sink internal error", ++ }; ++ struct drm_connector *connector = m->private; ++ struct drm_i915_private *dev_priv = to_i915(connector->dev); ++ struct intel_dp *intel_dp = ++ enc_to_intel_dp(&intel_attached_encoder(connector)->base); ++ int ret; ++ ++ if (!CAN_PSR(dev_priv)) { ++ seq_puts(m, "PSR Unsupported\n"); ++ return -ENODEV; ++ } ++ ++ if (connector->status != connector_status_connected) ++ return -ENODEV; ++ ++ ret = drm_dp_dpcd_readb(&intel_dp->aux, DP_PSR_STATUS, &val); ++ ++ if (ret == 1) { ++ const char *str = "unknown"; ++ ++ val &= DP_PSR_SINK_STATE_MASK; ++ if (val < ARRAY_SIZE(sink_status)) ++ str = sink_status[val]; ++ seq_printf(m, "Sink PSR status: 0x%x [%s]\n", val, str); ++ } else { ++ return ret; ++ } ++ ++ return 0; ++} ++DEFINE_SHOW_ATTRIBUTE(i915_psr_sink_status); ++ ++static void ++psr_source_status(struct drm_i915_private *dev_priv, struct seq_file *m) ++{ ++ u32 val, status_val; ++ const char *status = "unknown"; ++ ++ if (dev_priv->psr.psr2_enabled) { ++ static const char * const live_status[] = { ++ "IDLE", ++ "CAPTURE", ++ "CAPTURE_FS", ++ "SLEEP", ++ "BUFON_FW", ++ "ML_UP", ++ "SU_STANDBY", ++ "FAST_SLEEP", ++ "DEEP_SLEEP", ++ "BUF_ON", ++ "TG_ON" ++ }; ++ val = I915_READ(EDP_PSR2_STATUS); ++ status_val = (val & EDP_PSR2_STATUS_STATE_MASK) >> ++ EDP_PSR2_STATUS_STATE_SHIFT; ++ if (status_val < ARRAY_SIZE(live_status)) ++ status = live_status[status_val]; ++ } else { ++ static const char * const live_status[] = { ++ "IDLE", ++ "SRDONACK", ++ "SRDENT", ++ "BUFOFF", ++ "BUFON", ++ "AUXACK", ++ "SRDOFFACK", ++ "SRDENT_ON", ++ }; ++ val = I915_READ(EDP_PSR_STATUS); ++ status_val = (val & EDP_PSR_STATUS_STATE_MASK) >> ++ EDP_PSR_STATUS_STATE_SHIFT; ++ if (status_val < ARRAY_SIZE(live_status)) ++ status = live_status[status_val]; ++ } ++ ++ seq_printf(m, "Source PSR status: %s [0x%08x]\n", status, val); ++} ++ ++static int i915_edp_psr_status(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct i915_psr *psr = &dev_priv->psr; ++ intel_wakeref_t wakeref; ++ const char *status; ++ bool enabled; ++ u32 val; ++ ++ if (!HAS_PSR(dev_priv)) ++ return -ENODEV; ++ ++ seq_printf(m, "Sink support: %s", yesno(psr->sink_support)); ++ if (psr->dp) ++ seq_printf(m, " [0x%02x]", psr->dp->psr_dpcd[0]); ++ seq_puts(m, "\n"); ++ ++ if (!psr->sink_support) ++ return 0; ++ ++ wakeref = intel_runtime_pm_get(dev_priv); ++ mutex_lock(&psr->lock); ++ ++ if (psr->enabled) ++ status = psr->psr2_enabled ? "PSR2 enabled" : "PSR1 enabled"; ++ else ++ status = "disabled"; ++ seq_printf(m, "PSR mode: %s\n", status); ++ ++ if (!psr->enabled) ++ goto unlock; ++ ++ if (psr->psr2_enabled) { ++ val = I915_READ(EDP_PSR2_CTL); ++ enabled = val & EDP_PSR2_ENABLE; ++ } else { ++ val = I915_READ(EDP_PSR_CTL); ++ enabled = val & EDP_PSR_ENABLE; ++ } ++ seq_printf(m, "Source PSR ctl: %s [0x%08x]\n", ++ enableddisabled(enabled), val); ++ psr_source_status(dev_priv, m); ++ seq_printf(m, "Busy frontbuffer bits: 0x%08x\n", ++ psr->busy_frontbuffer_bits); ++ ++ /* ++ * SKL+ Perf counter is reset to 0 everytime DC state is entered ++ */ ++ if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { ++ val = I915_READ(EDP_PSR_PERF_CNT) & EDP_PSR_PERF_CNT_MASK; ++ seq_printf(m, "Performance counter: %u\n", val); ++ } ++ ++ if (psr->debug & I915_PSR_DEBUG_IRQ) { ++ seq_printf(m, "Last attempted entry at: %lld\n", ++ psr->last_entry_attempt); ++ seq_printf(m, "Last exit at: %lld\n", psr->last_exit); ++ } ++ ++ if (psr->psr2_enabled) { ++ u32 su_frames_val[3]; ++ int frame; ++ ++ /* ++ * Reading all 3 registers before hand to minimize crossing a ++ * frame boundary between register reads ++ */ ++ for (frame = 0; frame < PSR2_SU_STATUS_FRAMES; frame += 3) ++ su_frames_val[frame / 3] = I915_READ(PSR2_SU_STATUS(frame)); ++ ++ seq_puts(m, "Frame:\tPSR2 SU blocks:\n"); ++ ++ for (frame = 0; frame < PSR2_SU_STATUS_FRAMES; frame++) { ++ u32 su_blocks; ++ ++ su_blocks = su_frames_val[frame / 3] & ++ PSR2_SU_STATUS_MASK(frame); ++ su_blocks = su_blocks >> PSR2_SU_STATUS_SHIFT(frame); ++ seq_printf(m, "%d\t%d\n", frame, su_blocks); ++ } ++ } ++ ++unlock: ++ mutex_unlock(&psr->lock); ++ intel_runtime_pm_put(dev_priv, wakeref); ++ ++ return 0; ++} ++ ++static int ++i915_edp_psr_debug_set(void *data, u64 val) ++{ ++ struct drm_i915_private *dev_priv = data; ++ intel_wakeref_t wakeref; ++ int ret; ++ ++ if (!CAN_PSR(dev_priv)) ++ return -ENODEV; ++ ++ DRM_DEBUG_KMS("Setting PSR debug to %llx\n", val); ++ ++ wakeref = intel_runtime_pm_get(dev_priv); ++ ++ ret = intel_psr_debug_set(dev_priv, val); ++ ++ intel_runtime_pm_put(dev_priv, wakeref); ++ ++ return ret; ++} ++ ++static int ++i915_edp_psr_debug_get(void *data, u64 *val) ++{ ++ struct drm_i915_private *dev_priv = data; ++ ++ if (!CAN_PSR(dev_priv)) ++ return -ENODEV; ++ ++ *val = READ_ONCE(dev_priv->psr.debug); ++ return 0; ++} ++ ++DEFINE_SIMPLE_ATTRIBUTE(i915_edp_psr_debug_fops, ++ i915_edp_psr_debug_get, i915_edp_psr_debug_set, ++ "%llu\n"); ++ ++static int i915_energy_uJ(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ unsigned long long power; ++ intel_wakeref_t wakeref; ++ u32 units; ++ ++ if (INTEL_GEN(dev_priv) < 6) ++ return -ENODEV; ++ ++ if (rdmsrl_safe(MSR_RAPL_POWER_UNIT, &power)) ++ return -ENODEV; ++ ++ units = (power & 0x1f00) >> 8; ++ with_intel_runtime_pm(dev_priv, wakeref) ++ power = I915_READ(MCH_SECP_NRG_STTS); ++ ++ power = (1000000 * power) >> units; /* convert to uJ */ ++ seq_printf(m, "%llu", power); ++ ++ return 0; ++} ++ ++static int i915_runtime_pm_status(struct seq_file *m, void *unused) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ ++ if (!HAS_RUNTIME_PM(dev_priv)) ++ seq_puts(m, "Runtime power management not supported\n"); ++ ++ seq_printf(m, "Runtime power status: %s\n", ++ enableddisabled(!dev_priv->power_domains.wakeref)); ++ ++ seq_printf(m, "GPU idle: %s\n", yesno(!dev_priv->gt.awake)); ++ seq_printf(m, "IRQs disabled: %s\n", ++ yesno(!intel_irqs_enabled(dev_priv))); ++#ifdef CONFIG_PM ++ seq_printf(m, "Usage count: %d\n", ++ atomic_read(&dev_priv->drm.dev->power.usage_count)); ++#else ++ seq_printf(m, "Device Power Management (CONFIG_PM) disabled\n"); ++#endif ++ seq_printf(m, "PCI device power state: %s [%d]\n", ++ pci_power_name(pdev->current_state), ++ pdev->current_state); ++ ++ if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)) { ++ struct drm_printer p = drm_seq_file_printer(m); ++ ++ print_intel_runtime_pm_wakeref(dev_priv, &p); ++ } ++ ++ return 0; ++} ++ ++static int i915_power_domain_info(struct seq_file *m, void *unused) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct i915_power_domains *power_domains = &dev_priv->power_domains; ++ int i; ++ ++ mutex_lock(&power_domains->lock); ++ ++ seq_printf(m, "%-25s %s\n", "Power well/domain", "Use count"); ++ for (i = 0; i < power_domains->power_well_count; i++) { ++ struct i915_power_well *power_well; ++ enum intel_display_power_domain power_domain; ++ ++ power_well = &power_domains->power_wells[i]; ++ seq_printf(m, "%-25s %d\n", power_well->desc->name, ++ power_well->count); ++ ++ for_each_power_domain(power_domain, power_well->desc->domains) ++ seq_printf(m, " %-23s %d\n", ++ intel_display_power_domain_str(power_domain), ++ power_domains->domain_use_count[power_domain]); ++ } ++ ++ mutex_unlock(&power_domains->lock); ++ ++ return 0; ++} ++ ++static int i915_dmc_info(struct seq_file *m, void *unused) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ intel_wakeref_t wakeref; ++ struct intel_csr *csr; ++ ++ if (!HAS_CSR(dev_priv)) ++ return -ENODEV; ++ ++ csr = &dev_priv->csr; ++ ++ wakeref = intel_runtime_pm_get(dev_priv); ++ ++ seq_printf(m, "fw loaded: %s\n", yesno(csr->dmc_payload != NULL)); ++ seq_printf(m, "path: %s\n", csr->fw_path); ++ ++ if (!csr->dmc_payload) ++ goto out; ++ ++ seq_printf(m, "version: %d.%d\n", CSR_VERSION_MAJOR(csr->version), ++ CSR_VERSION_MINOR(csr->version)); ++ ++ if (WARN_ON(INTEL_GEN(dev_priv) > 11)) ++ goto out; ++ ++ seq_printf(m, "DC3 -> DC5 count: %d\n", ++ I915_READ(IS_BROXTON(dev_priv) ? BXT_CSR_DC3_DC5_COUNT : ++ SKL_CSR_DC3_DC5_COUNT)); ++ if (!IS_GEN9_LP(dev_priv)) ++ seq_printf(m, "DC5 -> DC6 count: %d\n", ++ I915_READ(SKL_CSR_DC5_DC6_COUNT)); ++ ++out: ++ seq_printf(m, "program base: 0x%08x\n", I915_READ(CSR_PROGRAM(0))); ++ seq_printf(m, "ssp base: 0x%08x\n", I915_READ(CSR_SSP_BASE)); ++ seq_printf(m, "htp: 0x%08x\n", I915_READ(CSR_HTP_SKL)); ++ ++ intel_runtime_pm_put(dev_priv, wakeref); ++ ++ return 0; ++} ++ ++static void intel_seq_print_mode(struct seq_file *m, int tabs, ++ struct drm_display_mode *mode) ++{ ++ int i; ++ ++ for (i = 0; i < tabs; i++) ++ seq_putc(m, '\t'); ++ ++ seq_printf(m, DRM_MODE_FMT "\n", DRM_MODE_ARG(mode)); ++} ++ ++static void intel_encoder_info(struct seq_file *m, ++ struct intel_crtc *intel_crtc, ++ struct intel_encoder *intel_encoder) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct drm_device *dev = &dev_priv->drm; ++ struct drm_crtc *crtc = &intel_crtc->base; ++ struct intel_connector *intel_connector; ++ struct drm_encoder *encoder; ++ ++ encoder = &intel_encoder->base; ++ seq_printf(m, "\tencoder %d: type: %s, connectors:\n", ++ encoder->base.id, encoder->name); ++ for_each_connector_on_encoder(dev, encoder, intel_connector) { ++ struct drm_connector *connector = &intel_connector->base; ++ seq_printf(m, "\t\tconnector %d: type: %s, status: %s", ++ connector->base.id, ++ connector->name, ++ drm_get_connector_status_name(connector->status)); ++ if (connector->status == connector_status_connected) { ++ struct drm_display_mode *mode = &crtc->mode; ++ seq_printf(m, ", mode:\n"); ++ intel_seq_print_mode(m, 2, mode); ++ } else { ++ seq_putc(m, '\n'); ++ } ++ } ++} ++ ++static void intel_crtc_info(struct seq_file *m, struct intel_crtc *intel_crtc) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct drm_device *dev = &dev_priv->drm; ++ struct drm_crtc *crtc = &intel_crtc->base; ++ struct intel_encoder *intel_encoder; ++ struct drm_plane_state *plane_state = crtc->primary->state; ++ struct drm_framebuffer *fb = plane_state->fb; ++ ++ if (fb) ++ seq_printf(m, "\tfb: %d, pos: %dx%d, size: %dx%d\n", ++ fb->base.id, plane_state->src_x >> 16, ++ plane_state->src_y >> 16, fb->width, fb->height); ++ else ++ seq_puts(m, "\tprimary plane disabled\n"); ++ for_each_encoder_on_crtc(dev, crtc, intel_encoder) ++ intel_encoder_info(m, intel_crtc, intel_encoder); ++} ++ ++static void intel_panel_info(struct seq_file *m, struct intel_panel *panel) ++{ ++ struct drm_display_mode *mode = panel->fixed_mode; ++ ++ seq_printf(m, "\tfixed mode:\n"); ++ intel_seq_print_mode(m, 2, mode); ++} ++ ++static void intel_dp_info(struct seq_file *m, ++ struct intel_connector *intel_connector) ++{ ++ struct intel_encoder *intel_encoder = intel_connector->encoder; ++ struct intel_dp *intel_dp = enc_to_intel_dp(&intel_encoder->base); ++ ++ seq_printf(m, "\tDPCD rev: %x\n", intel_dp->dpcd[DP_DPCD_REV]); ++ seq_printf(m, "\taudio support: %s\n", yesno(intel_dp->has_audio)); ++ if (intel_connector->base.connector_type == DRM_MODE_CONNECTOR_eDP) ++ intel_panel_info(m, &intel_connector->panel); ++ ++ drm_dp_downstream_debug(m, intel_dp->dpcd, intel_dp->downstream_ports, ++ &intel_dp->aux); ++} ++ ++static void intel_dp_mst_info(struct seq_file *m, ++ struct intel_connector *intel_connector) ++{ ++ struct intel_encoder *intel_encoder = intel_connector->encoder; ++ struct intel_dp_mst_encoder *intel_mst = ++ enc_to_mst(&intel_encoder->base); ++ struct intel_digital_port *intel_dig_port = intel_mst->primary; ++ struct intel_dp *intel_dp = &intel_dig_port->dp; ++ bool has_audio = drm_dp_mst_port_has_audio(&intel_dp->mst_mgr, ++ intel_connector->port); ++ ++ seq_printf(m, "\taudio support: %s\n", yesno(has_audio)); ++} ++ ++static void intel_hdmi_info(struct seq_file *m, ++ struct intel_connector *intel_connector) ++{ ++ struct intel_encoder *intel_encoder = intel_connector->encoder; ++ struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&intel_encoder->base); ++ ++ seq_printf(m, "\taudio support: %s\n", yesno(intel_hdmi->has_audio)); ++} ++ ++static void intel_lvds_info(struct seq_file *m, ++ struct intel_connector *intel_connector) ++{ ++ intel_panel_info(m, &intel_connector->panel); ++} ++ ++static void intel_connector_info(struct seq_file *m, ++ struct drm_connector *connector) ++{ ++ struct intel_connector *intel_connector = to_intel_connector(connector); ++ struct intel_encoder *intel_encoder = intel_connector->encoder; ++ struct drm_display_mode *mode; ++ ++ seq_printf(m, "connector %d: type %s, status: %s\n", ++ connector->base.id, connector->name, ++ drm_get_connector_status_name(connector->status)); ++ ++ if (connector->status == connector_status_disconnected) ++ return; ++ ++ seq_printf(m, "\tphysical dimensions: %dx%dmm\n", ++ connector->display_info.width_mm, ++ connector->display_info.height_mm); ++ seq_printf(m, "\tsubpixel order: %s\n", ++ drm_get_subpixel_order_name(connector->display_info.subpixel_order)); ++ seq_printf(m, "\tCEA rev: %d\n", connector->display_info.cea_rev); ++ ++ if (!intel_encoder) ++ return; ++ ++ switch (connector->connector_type) { ++ case DRM_MODE_CONNECTOR_DisplayPort: ++ case DRM_MODE_CONNECTOR_eDP: ++ if (intel_encoder->type == INTEL_OUTPUT_DP_MST) ++ intel_dp_mst_info(m, intel_connector); ++ else ++ intel_dp_info(m, intel_connector); ++ break; ++ case DRM_MODE_CONNECTOR_LVDS: ++ if (intel_encoder->type == INTEL_OUTPUT_LVDS) ++ intel_lvds_info(m, intel_connector); ++ break; ++ case DRM_MODE_CONNECTOR_HDMIA: ++ if (intel_encoder->type == INTEL_OUTPUT_HDMI || ++ intel_encoder->type == INTEL_OUTPUT_DDI) ++ intel_hdmi_info(m, intel_connector); ++ break; ++ default: ++ break; ++ } ++ ++ seq_printf(m, "\tmodes:\n"); ++ list_for_each_entry(mode, &connector->modes, head) ++ intel_seq_print_mode(m, 2, mode); ++} ++ ++static const char *plane_type(enum drm_plane_type type) ++{ ++ switch (type) { ++ case DRM_PLANE_TYPE_OVERLAY: ++ return "OVL"; ++ case DRM_PLANE_TYPE_PRIMARY: ++ return "PRI"; ++ case DRM_PLANE_TYPE_CURSOR: ++ return "CUR"; ++ /* ++ * Deliberately omitting default: to generate compiler warnings ++ * when a new drm_plane_type gets added. ++ */ ++ } ++ ++ return "unknown"; ++} ++ ++static void plane_rotation(char *buf, size_t bufsize, unsigned int rotation) ++{ ++ /* ++ * According to doc only one DRM_MODE_ROTATE_ is allowed but this ++ * will print them all to visualize if the values are misused ++ */ ++ snprintf(buf, bufsize, ++ "%s%s%s%s%s%s(0x%08x)", ++ (rotation & DRM_MODE_ROTATE_0) ? "0 " : "", ++ (rotation & DRM_MODE_ROTATE_90) ? "90 " : "", ++ (rotation & DRM_MODE_ROTATE_180) ? "180 " : "", ++ (rotation & DRM_MODE_ROTATE_270) ? "270 " : "", ++ (rotation & DRM_MODE_REFLECT_X) ? "FLIPX " : "", ++ (rotation & DRM_MODE_REFLECT_Y) ? "FLIPY " : "", ++ rotation); ++} ++ ++static void intel_plane_info(struct seq_file *m, struct intel_crtc *intel_crtc) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct drm_device *dev = &dev_priv->drm; ++ struct intel_plane *intel_plane; ++ ++ for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { ++ struct drm_plane_state *state; ++ struct drm_plane *plane = &intel_plane->base; ++ struct drm_format_name_buf format_name; ++ char rot_str[48]; ++ ++ if (!plane->state) { ++ seq_puts(m, "plane->state is NULL!\n"); ++ continue; ++ } ++ ++ state = plane->state; ++ ++ if (state->fb) { ++ drm_get_format_name(state->fb->format->format, ++ &format_name); ++ } else { ++ sprintf(format_name.str, "N/A"); ++ } ++ ++ plane_rotation(rot_str, sizeof(rot_str), state->rotation); ++ ++ seq_printf(m, "\t--Plane id %d: type=%s, crtc_pos=%4dx%4d, crtc_size=%4dx%4d, src_pos=%d.%04ux%d.%04u, src_size=%d.%04ux%d.%04u, format=%s, rotation=%s\n", ++ plane->base.id, ++ plane_type(intel_plane->base.type), ++ state->crtc_x, state->crtc_y, ++ state->crtc_w, state->crtc_h, ++ (state->src_x >> 16), ++ ((state->src_x & 0xffff) * 15625) >> 10, ++ (state->src_y >> 16), ++ ((state->src_y & 0xffff) * 15625) >> 10, ++ (state->src_w >> 16), ++ ((state->src_w & 0xffff) * 15625) >> 10, ++ (state->src_h >> 16), ++ ((state->src_h & 0xffff) * 15625) >> 10, ++ format_name.str, ++ rot_str); ++ } ++} ++ ++static void intel_scaler_info(struct seq_file *m, struct intel_crtc *intel_crtc) ++{ ++ struct intel_crtc_state *pipe_config; ++ int num_scalers = intel_crtc->num_scalers; ++ int i; ++ ++ pipe_config = to_intel_crtc_state(intel_crtc->base.state); ++ ++ /* Not all platformas have a scaler */ ++ if (num_scalers) { ++ seq_printf(m, "\tnum_scalers=%d, scaler_users=%x scaler_id=%d", ++ num_scalers, ++ pipe_config->scaler_state.scaler_users, ++ pipe_config->scaler_state.scaler_id); ++ ++ for (i = 0; i < num_scalers; i++) { ++ struct intel_scaler *sc = ++ &pipe_config->scaler_state.scalers[i]; ++ ++ seq_printf(m, ", scalers[%d]: use=%s, mode=%x", ++ i, yesno(sc->in_use), sc->mode); ++ } ++ seq_puts(m, "\n"); ++ } else { ++ seq_puts(m, "\tNo scalers available on this platform\n"); ++ } ++} ++ ++static int i915_display_info(struct seq_file *m, void *unused) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct drm_device *dev = &dev_priv->drm; ++ struct intel_crtc *crtc; ++ struct drm_connector *connector; ++ struct drm_connector_list_iter conn_iter; ++ intel_wakeref_t wakeref; ++ ++ wakeref = intel_runtime_pm_get(dev_priv); ++ ++ seq_printf(m, "CRTC info\n"); ++ seq_printf(m, "---------\n"); ++ for_each_intel_crtc(dev, crtc) { ++ struct intel_crtc_state *pipe_config; ++ ++ drm_modeset_lock(&crtc->base.mutex, NULL); ++ pipe_config = to_intel_crtc_state(crtc->base.state); ++ ++ seq_printf(m, "CRTC %d: pipe: %c, active=%s, (size=%dx%d), dither=%s, bpp=%d\n", ++ crtc->base.base.id, pipe_name(crtc->pipe), ++ yesno(pipe_config->base.active), ++ pipe_config->pipe_src_w, pipe_config->pipe_src_h, ++ yesno(pipe_config->dither), pipe_config->pipe_bpp); ++ ++ if (pipe_config->base.active) { ++ struct intel_plane *cursor = ++ to_intel_plane(crtc->base.cursor); ++ ++ intel_crtc_info(m, crtc); ++ ++ seq_printf(m, "\tcursor visible? %s, position (%d, %d), size %dx%d, addr 0x%08x\n", ++ yesno(cursor->base.state->visible), ++ cursor->base.state->crtc_x, ++ cursor->base.state->crtc_y, ++ cursor->base.state->crtc_w, ++ cursor->base.state->crtc_h, ++ cursor->cursor.base); ++ intel_scaler_info(m, crtc); ++ intel_plane_info(m, crtc); ++ } ++ ++ seq_printf(m, "\tunderrun reporting: cpu=%s pch=%s \n", ++ yesno(!crtc->cpu_fifo_underrun_disabled), ++ yesno(!crtc->pch_fifo_underrun_disabled)); ++ drm_modeset_unlock(&crtc->base.mutex); ++ } ++ ++ seq_printf(m, "\n"); ++ seq_printf(m, "Connector info\n"); ++ seq_printf(m, "--------------\n"); ++ mutex_lock(&dev->mode_config.mutex); ++ drm_connector_list_iter_begin(dev, &conn_iter); ++ drm_for_each_connector_iter(connector, &conn_iter) ++ intel_connector_info(m, connector); ++ drm_connector_list_iter_end(&conn_iter); ++ mutex_unlock(&dev->mode_config.mutex); ++ ++ intel_runtime_pm_put(dev_priv, wakeref); ++ ++ return 0; ++} ++ ++static int i915_engine_info(struct seq_file *m, void *unused) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct intel_engine_cs *engine; ++ intel_wakeref_t wakeref; ++ enum intel_engine_id id; ++ struct drm_printer p; ++ ++ wakeref = intel_runtime_pm_get(dev_priv); ++ ++ seq_printf(m, "GT awake? %s\n", yesno(dev_priv->gt.awake)); ++ seq_printf(m, "Global active requests: %d\n", ++ dev_priv->gt.active_requests); ++ seq_printf(m, "CS timestamp frequency: %u kHz\n", ++ RUNTIME_INFO(dev_priv)->cs_timestamp_frequency_khz); ++ ++ p = drm_seq_file_printer(m); ++ for_each_engine(engine, dev_priv, id) ++ intel_engine_dump(engine, &p, "%s\n", engine->name); ++ ++ intel_runtime_pm_put(dev_priv, wakeref); ++ ++ return 0; ++} ++ ++static int i915_rcs_topology(struct seq_file *m, void *unused) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct drm_printer p = drm_seq_file_printer(m); ++ ++ intel_device_info_dump_topology(&RUNTIME_INFO(dev_priv)->sseu, &p); ++ ++ return 0; ++} ++ ++static int i915_shrinker_info(struct seq_file *m, void *unused) ++{ ++ struct drm_i915_private *i915 = node_to_i915(m->private); ++ ++ seq_printf(m, "seeks = %d\n", i915->mm.shrinker.seeks); ++ seq_printf(m, "batch = %lu\n", i915->mm.shrinker.batch); ++ ++ return 0; ++} ++ ++static int i915_shared_dplls_info(struct seq_file *m, void *unused) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct drm_device *dev = &dev_priv->drm; ++ int i; ++ ++ drm_modeset_lock_all(dev); ++ for (i = 0; i < dev_priv->num_shared_dpll; i++) { ++ struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i]; ++ ++ seq_printf(m, "DPLL%i: %s, id: %i\n", i, pll->info->name, ++ pll->info->id); ++ seq_printf(m, " crtc_mask: 0x%08x, active: 0x%x, on: %s\n", ++ pll->state.crtc_mask, pll->active_mask, yesno(pll->on)); ++ seq_printf(m, " tracked hardware state:\n"); ++ seq_printf(m, " dpll: 0x%08x\n", pll->state.hw_state.dpll); ++ seq_printf(m, " dpll_md: 0x%08x\n", ++ pll->state.hw_state.dpll_md); ++ seq_printf(m, " fp0: 0x%08x\n", pll->state.hw_state.fp0); ++ seq_printf(m, " fp1: 0x%08x\n", pll->state.hw_state.fp1); ++ seq_printf(m, " wrpll: 0x%08x\n", pll->state.hw_state.wrpll); ++ seq_printf(m, " cfgcr0: 0x%08x\n", pll->state.hw_state.cfgcr0); ++ seq_printf(m, " cfgcr1: 0x%08x\n", pll->state.hw_state.cfgcr1); ++ seq_printf(m, " mg_refclkin_ctl: 0x%08x\n", ++ pll->state.hw_state.mg_refclkin_ctl); ++ seq_printf(m, " mg_clktop2_coreclkctl1: 0x%08x\n", ++ pll->state.hw_state.mg_clktop2_coreclkctl1); ++ seq_printf(m, " mg_clktop2_hsclkctl: 0x%08x\n", ++ pll->state.hw_state.mg_clktop2_hsclkctl); ++ seq_printf(m, " mg_pll_div0: 0x%08x\n", ++ pll->state.hw_state.mg_pll_div0); ++ seq_printf(m, " mg_pll_div1: 0x%08x\n", ++ pll->state.hw_state.mg_pll_div1); ++ seq_printf(m, " mg_pll_lf: 0x%08x\n", ++ pll->state.hw_state.mg_pll_lf); ++ seq_printf(m, " mg_pll_frac_lock: 0x%08x\n", ++ pll->state.hw_state.mg_pll_frac_lock); ++ seq_printf(m, " mg_pll_ssc: 0x%08x\n", ++ pll->state.hw_state.mg_pll_ssc); ++ seq_printf(m, " mg_pll_bias: 0x%08x\n", ++ pll->state.hw_state.mg_pll_bias); ++ seq_printf(m, " mg_pll_tdc_coldst_bias: 0x%08x\n", ++ pll->state.hw_state.mg_pll_tdc_coldst_bias); ++ } ++ drm_modeset_unlock_all(dev); ++ ++ return 0; ++} ++ ++static int i915_wa_registers(struct seq_file *m, void *unused) ++{ ++ struct drm_i915_private *i915 = node_to_i915(m->private); ++ const struct i915_wa_list *wal = &i915->engine[RCS0]->ctx_wa_list; ++ struct i915_wa *wa; ++ unsigned int i; ++ ++ seq_printf(m, "Workarounds applied: %u\n", wal->count); ++ for (i = 0, wa = wal->list; i < wal->count; i++, wa++) ++ seq_printf(m, "0x%X: 0x%08X, mask: 0x%08X\n", ++ i915_mmio_reg_offset(wa->reg), wa->val, wa->mask); ++ ++ return 0; ++} ++ ++static int i915_ipc_status_show(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *dev_priv = m->private; ++ ++ seq_printf(m, "Isochronous Priority Control: %s\n", ++ yesno(dev_priv->ipc_enabled)); ++ return 0; ++} ++ ++static int i915_ipc_status_open(struct inode *inode, struct file *file) ++{ ++ struct drm_i915_private *dev_priv = inode->i_private; ++ ++ if (!HAS_IPC(dev_priv)) ++ return -ENODEV; ++ ++ return single_open(file, i915_ipc_status_show, dev_priv); ++} ++ ++static ssize_t i915_ipc_status_write(struct file *file, const char __user *ubuf, ++ size_t len, loff_t *offp) ++{ ++ struct seq_file *m = file->private_data; ++ struct drm_i915_private *dev_priv = m->private; ++ intel_wakeref_t wakeref; ++ bool enable; ++ int ret; ++ ++ ret = kstrtobool_from_user(ubuf, len, &enable); ++ if (ret < 0) ++ return ret; ++ ++ with_intel_runtime_pm(dev_priv, wakeref) { ++ if (!dev_priv->ipc_enabled && enable) ++ DRM_INFO("Enabling IPC: WM will be proper only after next commit\n"); ++ dev_priv->wm.distrust_bios_wm = true; ++ dev_priv->ipc_enabled = enable; ++ intel_enable_ipc(dev_priv); ++ } ++ ++ return len; ++} ++ ++static const struct file_operations i915_ipc_status_fops = { ++ .owner = THIS_MODULE, ++ .open = i915_ipc_status_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++ .write = i915_ipc_status_write ++}; ++ ++static int i915_ddb_info(struct seq_file *m, void *unused) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct drm_device *dev = &dev_priv->drm; ++ struct skl_ddb_entry *entry; ++ struct intel_crtc *crtc; ++ ++ if (INTEL_GEN(dev_priv) < 9) ++ return -ENODEV; ++ ++ drm_modeset_lock_all(dev); ++ ++ seq_printf(m, "%-15s%8s%8s%8s\n", "", "Start", "End", "Size"); ++ ++ for_each_intel_crtc(&dev_priv->drm, crtc) { ++ struct intel_crtc_state *crtc_state = ++ to_intel_crtc_state(crtc->base.state); ++ enum pipe pipe = crtc->pipe; ++ enum plane_id plane_id; ++ ++ seq_printf(m, "Pipe %c\n", pipe_name(pipe)); ++ ++ for_each_plane_id_on_crtc(crtc, plane_id) { ++ entry = &crtc_state->wm.skl.plane_ddb_y[plane_id]; ++ seq_printf(m, " Plane%-8d%8u%8u%8u\n", plane_id + 1, ++ entry->start, entry->end, ++ skl_ddb_entry_size(entry)); ++ } ++ ++ entry = &crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR]; ++ seq_printf(m, " %-13s%8u%8u%8u\n", "Cursor", entry->start, ++ entry->end, skl_ddb_entry_size(entry)); ++ } ++ ++ drm_modeset_unlock_all(dev); ++ ++ return 0; ++} ++ ++static void drrs_status_per_crtc(struct seq_file *m, ++ struct drm_device *dev, ++ struct intel_crtc *intel_crtc) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct i915_drrs *drrs = &dev_priv->drrs; ++ int vrefresh = 0; ++ struct drm_connector *connector; ++ struct drm_connector_list_iter conn_iter; ++ ++ drm_connector_list_iter_begin(dev, &conn_iter); ++ drm_for_each_connector_iter(connector, &conn_iter) { ++ if (connector->state->crtc != &intel_crtc->base) ++ continue; ++ ++ seq_printf(m, "%s:\n", connector->name); ++ } ++ drm_connector_list_iter_end(&conn_iter); ++ ++ if (dev_priv->vbt.drrs_type == STATIC_DRRS_SUPPORT) ++ seq_puts(m, "\tVBT: DRRS_type: Static"); ++ else if (dev_priv->vbt.drrs_type == SEAMLESS_DRRS_SUPPORT) ++ seq_puts(m, "\tVBT: DRRS_type: Seamless"); ++ else if (dev_priv->vbt.drrs_type == DRRS_NOT_SUPPORTED) ++ seq_puts(m, "\tVBT: DRRS_type: None"); ++ else ++ seq_puts(m, "\tVBT: DRRS_type: FIXME: Unrecognized Value"); ++ ++ seq_puts(m, "\n\n"); ++ ++ if (to_intel_crtc_state(intel_crtc->base.state)->has_drrs) { ++ struct intel_panel *panel; ++ ++ mutex_lock(&drrs->mutex); ++ /* DRRS Supported */ ++ seq_puts(m, "\tDRRS Supported: Yes\n"); ++ ++ /* disable_drrs() will make drrs->dp NULL */ ++ if (!drrs->dp) { ++ seq_puts(m, "Idleness DRRS: Disabled\n"); ++ if (dev_priv->psr.enabled) ++ seq_puts(m, ++ "\tAs PSR is enabled, DRRS is not enabled\n"); ++ mutex_unlock(&drrs->mutex); ++ return; ++ } ++ ++ panel = &drrs->dp->attached_connector->panel; ++ seq_printf(m, "\t\tBusy_frontbuffer_bits: 0x%X", ++ drrs->busy_frontbuffer_bits); ++ ++ seq_puts(m, "\n\t\t"); ++ if (drrs->refresh_rate_type == DRRS_HIGH_RR) { ++ seq_puts(m, "DRRS_State: DRRS_HIGH_RR\n"); ++ vrefresh = panel->fixed_mode->vrefresh; ++ } else if (drrs->refresh_rate_type == DRRS_LOW_RR) { ++ seq_puts(m, "DRRS_State: DRRS_LOW_RR\n"); ++ vrefresh = panel->downclock_mode->vrefresh; ++ } else { ++ seq_printf(m, "DRRS_State: Unknown(%d)\n", ++ drrs->refresh_rate_type); ++ mutex_unlock(&drrs->mutex); ++ return; ++ } ++ seq_printf(m, "\t\tVrefresh: %d", vrefresh); ++ ++ seq_puts(m, "\n\t\t"); ++ mutex_unlock(&drrs->mutex); ++ } else { ++ /* DRRS not supported. Print the VBT parameter*/ ++ seq_puts(m, "\tDRRS Supported : No"); ++ } ++ seq_puts(m, "\n"); ++} ++ ++static int i915_drrs_status(struct seq_file *m, void *unused) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct drm_device *dev = &dev_priv->drm; ++ struct intel_crtc *intel_crtc; ++ int active_crtc_cnt = 0; ++ ++ drm_modeset_lock_all(dev); ++ for_each_intel_crtc(dev, intel_crtc) { ++ if (intel_crtc->base.state->active) { ++ active_crtc_cnt++; ++ seq_printf(m, "\nCRTC %d: ", active_crtc_cnt); ++ ++ drrs_status_per_crtc(m, dev, intel_crtc); ++ } ++ } ++ drm_modeset_unlock_all(dev); ++ ++ if (!active_crtc_cnt) ++ seq_puts(m, "No active crtc found\n"); ++ ++ return 0; ++} ++ ++static int i915_dp_mst_info(struct seq_file *m, void *unused) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct drm_device *dev = &dev_priv->drm; ++ struct intel_encoder *intel_encoder; ++ struct intel_digital_port *intel_dig_port; ++ struct drm_connector *connector; ++ struct drm_connector_list_iter conn_iter; ++ ++ drm_connector_list_iter_begin(dev, &conn_iter); ++ drm_for_each_connector_iter(connector, &conn_iter) { ++ if (connector->connector_type != DRM_MODE_CONNECTOR_DisplayPort) ++ continue; ++ ++ intel_encoder = intel_attached_encoder(connector); ++ if (!intel_encoder || intel_encoder->type == INTEL_OUTPUT_DP_MST) ++ continue; ++ ++ intel_dig_port = enc_to_dig_port(&intel_encoder->base); ++ if (!intel_dig_port->dp.can_mst) ++ continue; ++ ++ seq_printf(m, "MST Source Port %c\n", ++ port_name(intel_dig_port->base.port)); ++ drm_dp_mst_dump_topology(m, &intel_dig_port->dp.mst_mgr); ++ } ++ drm_connector_list_iter_end(&conn_iter); ++ ++ return 0; ++} ++ ++static ssize_t i915_displayport_test_active_write(struct file *file, ++ const char __user *ubuf, ++ size_t len, loff_t *offp) ++{ ++ char *input_buffer; ++ int status = 0; ++ struct drm_device *dev; ++ struct drm_connector *connector; ++ struct drm_connector_list_iter conn_iter; ++ struct intel_dp *intel_dp; ++ int val = 0; ++ ++ dev = ((struct seq_file *)file->private_data)->private; ++ ++ if (len == 0) ++ return 0; ++ ++ input_buffer = memdup_user_nul(ubuf, len); ++ if (IS_ERR(input_buffer)) ++ return PTR_ERR(input_buffer); ++ ++ DRM_DEBUG_DRIVER("Copied %d bytes from user\n", (unsigned int)len); ++ ++ drm_connector_list_iter_begin(dev, &conn_iter); ++ drm_for_each_connector_iter(connector, &conn_iter) { ++ struct intel_encoder *encoder; ++ ++ if (connector->connector_type != ++ DRM_MODE_CONNECTOR_DisplayPort) ++ continue; ++ ++ encoder = to_intel_encoder(connector->encoder); ++ if (encoder && encoder->type == INTEL_OUTPUT_DP_MST) ++ continue; ++ ++ if (encoder && connector->status == connector_status_connected) { ++ intel_dp = enc_to_intel_dp(&encoder->base); ++ status = kstrtoint(input_buffer, 10, &val); ++ if (status < 0) ++ break; ++ DRM_DEBUG_DRIVER("Got %d for test active\n", val); ++ /* To prevent erroneous activation of the compliance ++ * testing code, only accept an actual value of 1 here ++ */ ++ if (val == 1) ++ intel_dp->compliance.test_active = 1; ++ else ++ intel_dp->compliance.test_active = 0; ++ } ++ } ++ drm_connector_list_iter_end(&conn_iter); ++ kfree(input_buffer); ++ if (status < 0) ++ return status; ++ ++ *offp += len; ++ return len; ++} ++ ++static int i915_displayport_test_active_show(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *dev_priv = m->private; ++ struct drm_device *dev = &dev_priv->drm; ++ struct drm_connector *connector; ++ struct drm_connector_list_iter conn_iter; ++ struct intel_dp *intel_dp; ++ ++ drm_connector_list_iter_begin(dev, &conn_iter); ++ drm_for_each_connector_iter(connector, &conn_iter) { ++ struct intel_encoder *encoder; ++ ++ if (connector->connector_type != ++ DRM_MODE_CONNECTOR_DisplayPort) ++ continue; ++ ++ encoder = to_intel_encoder(connector->encoder); ++ if (encoder && encoder->type == INTEL_OUTPUT_DP_MST) ++ continue; ++ ++ if (encoder && connector->status == connector_status_connected) { ++ intel_dp = enc_to_intel_dp(&encoder->base); ++ if (intel_dp->compliance.test_active) ++ seq_puts(m, "1"); ++ else ++ seq_puts(m, "0"); ++ } else ++ seq_puts(m, "0"); ++ } ++ drm_connector_list_iter_end(&conn_iter); ++ ++ return 0; ++} ++ ++static int i915_displayport_test_active_open(struct inode *inode, ++ struct file *file) ++{ ++ return single_open(file, i915_displayport_test_active_show, ++ inode->i_private); ++} ++ ++static const struct file_operations i915_displayport_test_active_fops = { ++ .owner = THIS_MODULE, ++ .open = i915_displayport_test_active_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++ .write = i915_displayport_test_active_write ++}; ++ ++static int i915_displayport_test_data_show(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *dev_priv = m->private; ++ struct drm_device *dev = &dev_priv->drm; ++ struct drm_connector *connector; ++ struct drm_connector_list_iter conn_iter; ++ struct intel_dp *intel_dp; ++ ++ drm_connector_list_iter_begin(dev, &conn_iter); ++ drm_for_each_connector_iter(connector, &conn_iter) { ++ struct intel_encoder *encoder; ++ ++ if (connector->connector_type != ++ DRM_MODE_CONNECTOR_DisplayPort) ++ continue; ++ ++ encoder = to_intel_encoder(connector->encoder); ++ if (encoder && encoder->type == INTEL_OUTPUT_DP_MST) ++ continue; ++ ++ if (encoder && connector->status == connector_status_connected) { ++ intel_dp = enc_to_intel_dp(&encoder->base); ++ if (intel_dp->compliance.test_type == ++ DP_TEST_LINK_EDID_READ) ++ seq_printf(m, "%lx", ++ intel_dp->compliance.test_data.edid); ++ else if (intel_dp->compliance.test_type == ++ DP_TEST_LINK_VIDEO_PATTERN) { ++ seq_printf(m, "hdisplay: %d\n", ++ intel_dp->compliance.test_data.hdisplay); ++ seq_printf(m, "vdisplay: %d\n", ++ intel_dp->compliance.test_data.vdisplay); ++ seq_printf(m, "bpc: %u\n", ++ intel_dp->compliance.test_data.bpc); ++ } ++ } else ++ seq_puts(m, "0"); ++ } ++ drm_connector_list_iter_end(&conn_iter); ++ ++ return 0; ++} ++DEFINE_SHOW_ATTRIBUTE(i915_displayport_test_data); ++ ++static int i915_displayport_test_type_show(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *dev_priv = m->private; ++ struct drm_device *dev = &dev_priv->drm; ++ struct drm_connector *connector; ++ struct drm_connector_list_iter conn_iter; ++ struct intel_dp *intel_dp; ++ ++ drm_connector_list_iter_begin(dev, &conn_iter); ++ drm_for_each_connector_iter(connector, &conn_iter) { ++ struct intel_encoder *encoder; ++ ++ if (connector->connector_type != ++ DRM_MODE_CONNECTOR_DisplayPort) ++ continue; ++ ++ encoder = to_intel_encoder(connector->encoder); ++ if (encoder && encoder->type == INTEL_OUTPUT_DP_MST) ++ continue; ++ ++ if (encoder && connector->status == connector_status_connected) { ++ intel_dp = enc_to_intel_dp(&encoder->base); ++ seq_printf(m, "%02lx", intel_dp->compliance.test_type); ++ } else ++ seq_puts(m, "0"); ++ } ++ drm_connector_list_iter_end(&conn_iter); ++ ++ return 0; ++} ++DEFINE_SHOW_ATTRIBUTE(i915_displayport_test_type); ++ ++static void wm_latency_show(struct seq_file *m, const u16 wm[8]) ++{ ++ struct drm_i915_private *dev_priv = m->private; ++ struct drm_device *dev = &dev_priv->drm; ++ int level; ++ int num_levels; ++ ++ if (IS_CHERRYVIEW(dev_priv)) ++ num_levels = 3; ++ else if (IS_VALLEYVIEW(dev_priv)) ++ num_levels = 1; ++ else if (IS_G4X(dev_priv)) ++ num_levels = 3; ++ else ++ num_levels = ilk_wm_max_level(dev_priv) + 1; ++ ++ drm_modeset_lock_all(dev); ++ ++ for (level = 0; level < num_levels; level++) { ++ unsigned int latency = wm[level]; ++ ++ /* ++ * - WM1+ latency values in 0.5us units ++ * - latencies are in us on gen9/vlv/chv ++ */ ++ if (INTEL_GEN(dev_priv) >= 9 || ++ IS_VALLEYVIEW(dev_priv) || ++ IS_CHERRYVIEW(dev_priv) || ++ IS_G4X(dev_priv)) ++ latency *= 10; ++ else if (level > 0) ++ latency *= 5; ++ ++ seq_printf(m, "WM%d %u (%u.%u usec)\n", ++ level, wm[level], latency / 10, latency % 10); ++ } ++ ++ drm_modeset_unlock_all(dev); ++} ++ ++static int pri_wm_latency_show(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *dev_priv = m->private; ++ const u16 *latencies; ++ ++ if (INTEL_GEN(dev_priv) >= 9) ++ latencies = dev_priv->wm.skl_latency; ++ else ++ latencies = dev_priv->wm.pri_latency; ++ ++ wm_latency_show(m, latencies); ++ ++ return 0; ++} ++ ++static int spr_wm_latency_show(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *dev_priv = m->private; ++ const u16 *latencies; ++ ++ if (INTEL_GEN(dev_priv) >= 9) ++ latencies = dev_priv->wm.skl_latency; ++ else ++ latencies = dev_priv->wm.spr_latency; ++ ++ wm_latency_show(m, latencies); ++ ++ return 0; ++} ++ ++static int cur_wm_latency_show(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *dev_priv = m->private; ++ const u16 *latencies; ++ ++ if (INTEL_GEN(dev_priv) >= 9) ++ latencies = dev_priv->wm.skl_latency; ++ else ++ latencies = dev_priv->wm.cur_latency; ++ ++ wm_latency_show(m, latencies); ++ ++ return 0; ++} ++ ++static int pri_wm_latency_open(struct inode *inode, struct file *file) ++{ ++ struct drm_i915_private *dev_priv = inode->i_private; ++ ++ if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) ++ return -ENODEV; ++ ++ return single_open(file, pri_wm_latency_show, dev_priv); ++} ++ ++static int spr_wm_latency_open(struct inode *inode, struct file *file) ++{ ++ struct drm_i915_private *dev_priv = inode->i_private; ++ ++ if (HAS_GMCH(dev_priv)) ++ return -ENODEV; ++ ++ return single_open(file, spr_wm_latency_show, dev_priv); ++} ++ ++static int cur_wm_latency_open(struct inode *inode, struct file *file) ++{ ++ struct drm_i915_private *dev_priv = inode->i_private; ++ ++ if (HAS_GMCH(dev_priv)) ++ return -ENODEV; ++ ++ return single_open(file, cur_wm_latency_show, dev_priv); ++} ++ ++static ssize_t wm_latency_write(struct file *file, const char __user *ubuf, ++ size_t len, loff_t *offp, u16 wm[8]) ++{ ++ struct seq_file *m = file->private_data; ++ struct drm_i915_private *dev_priv = m->private; ++ struct drm_device *dev = &dev_priv->drm; ++ u16 new[8] = { 0 }; ++ int num_levels; ++ int level; ++ int ret; ++ char tmp[32]; ++ ++ if (IS_CHERRYVIEW(dev_priv)) ++ num_levels = 3; ++ else if (IS_VALLEYVIEW(dev_priv)) ++ num_levels = 1; ++ else if (IS_G4X(dev_priv)) ++ num_levels = 3; ++ else ++ num_levels = ilk_wm_max_level(dev_priv) + 1; ++ ++ if (len >= sizeof(tmp)) ++ return -EINVAL; ++ ++ if (copy_from_user(tmp, ubuf, len)) ++ return -EFAULT; ++ ++ tmp[len] = '\0'; ++ ++ ret = sscanf(tmp, "%hu %hu %hu %hu %hu %hu %hu %hu", ++ &new[0], &new[1], &new[2], &new[3], ++ &new[4], &new[5], &new[6], &new[7]); ++ if (ret != num_levels) ++ return -EINVAL; ++ ++ drm_modeset_lock_all(dev); ++ ++ for (level = 0; level < num_levels; level++) ++ wm[level] = new[level]; ++ ++ drm_modeset_unlock_all(dev); ++ ++ return len; ++} ++ ++ ++static ssize_t pri_wm_latency_write(struct file *file, const char __user *ubuf, ++ size_t len, loff_t *offp) ++{ ++ struct seq_file *m = file->private_data; ++ struct drm_i915_private *dev_priv = m->private; ++ u16 *latencies; ++ ++ if (INTEL_GEN(dev_priv) >= 9) ++ latencies = dev_priv->wm.skl_latency; ++ else ++ latencies = dev_priv->wm.pri_latency; ++ ++ return wm_latency_write(file, ubuf, len, offp, latencies); ++} ++ ++static ssize_t spr_wm_latency_write(struct file *file, const char __user *ubuf, ++ size_t len, loff_t *offp) ++{ ++ struct seq_file *m = file->private_data; ++ struct drm_i915_private *dev_priv = m->private; ++ u16 *latencies; ++ ++ if (INTEL_GEN(dev_priv) >= 9) ++ latencies = dev_priv->wm.skl_latency; ++ else ++ latencies = dev_priv->wm.spr_latency; ++ ++ return wm_latency_write(file, ubuf, len, offp, latencies); ++} ++ ++static ssize_t cur_wm_latency_write(struct file *file, const char __user *ubuf, ++ size_t len, loff_t *offp) ++{ ++ struct seq_file *m = file->private_data; ++ struct drm_i915_private *dev_priv = m->private; ++ u16 *latencies; ++ ++ if (INTEL_GEN(dev_priv) >= 9) ++ latencies = dev_priv->wm.skl_latency; ++ else ++ latencies = dev_priv->wm.cur_latency; ++ ++ return wm_latency_write(file, ubuf, len, offp, latencies); ++} ++ ++static const struct file_operations i915_pri_wm_latency_fops = { ++ .owner = THIS_MODULE, ++ .open = pri_wm_latency_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++ .write = pri_wm_latency_write ++}; ++ ++static const struct file_operations i915_spr_wm_latency_fops = { ++ .owner = THIS_MODULE, ++ .open = spr_wm_latency_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++ .write = spr_wm_latency_write ++}; ++ ++static const struct file_operations i915_cur_wm_latency_fops = { ++ .owner = THIS_MODULE, ++ .open = cur_wm_latency_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++ .write = cur_wm_latency_write ++}; ++ ++static int ++i915_wedged_get(void *data, u64 *val) ++{ ++ int ret = i915_terminally_wedged(data); ++ ++ switch (ret) { ++ case -EIO: ++ *val = 1; ++ return 0; ++ case 0: ++ *val = 0; ++ return 0; ++ default: ++ return ret; ++ } ++} ++ ++static int ++i915_wedged_set(void *data, u64 val) ++{ ++ struct drm_i915_private *i915 = data; ++ ++ /* Flush any previous reset before applying for a new one */ ++ wait_event(i915->gpu_error.reset_queue, ++ !test_bit(I915_RESET_BACKOFF, &i915->gpu_error.flags)); ++ ++ i915_handle_error(i915, val, I915_ERROR_CAPTURE, ++ "Manually set wedged engine mask = %llx", val); ++ return 0; ++} ++ ++DEFINE_SIMPLE_ATTRIBUTE(i915_wedged_fops, ++ i915_wedged_get, i915_wedged_set, ++ "%llu\n"); ++ ++#define DROP_UNBOUND BIT(0) ++#define DROP_BOUND BIT(1) ++#define DROP_RETIRE BIT(2) ++#define DROP_ACTIVE BIT(3) ++#define DROP_FREED BIT(4) ++#define DROP_SHRINK_ALL BIT(5) ++#define DROP_IDLE BIT(6) ++#define DROP_RESET_ACTIVE BIT(7) ++#define DROP_RESET_SEQNO BIT(8) ++#define DROP_ALL (DROP_UNBOUND | \ ++ DROP_BOUND | \ ++ DROP_RETIRE | \ ++ DROP_ACTIVE | \ ++ DROP_FREED | \ ++ DROP_SHRINK_ALL |\ ++ DROP_IDLE | \ ++ DROP_RESET_ACTIVE | \ ++ DROP_RESET_SEQNO) ++static int ++i915_drop_caches_get(void *data, u64 *val) ++{ ++ *val = DROP_ALL; ++ ++ return 0; ++} ++ ++static int ++i915_drop_caches_set(void *data, u64 val) ++{ ++ struct drm_i915_private *i915 = data; ++ ++ DRM_DEBUG("Dropping caches: 0x%08llx [0x%08llx]\n", ++ val, val & DROP_ALL); ++ ++ if (val & DROP_RESET_ACTIVE && ++ wait_for(intel_engines_are_idle(i915), I915_IDLE_ENGINES_TIMEOUT)) ++ i915_gem_set_wedged(i915); ++ ++ /* No need to check and wait for gpu resets, only libdrm auto-restarts ++ * on ioctls on -EAGAIN. */ ++ if (val & (DROP_ACTIVE | DROP_RETIRE | DROP_RESET_SEQNO)) { ++ int ret; ++ ++ ret = mutex_lock_interruptible(&i915->drm.struct_mutex); ++ if (ret) ++ return ret; ++ ++ if (val & DROP_ACTIVE) ++ ret = i915_gem_wait_for_idle(i915, ++ I915_WAIT_INTERRUPTIBLE | ++ I915_WAIT_LOCKED, ++ MAX_SCHEDULE_TIMEOUT); ++ ++ if (val & DROP_RETIRE) ++ i915_retire_requests(i915); ++ ++ mutex_unlock(&i915->drm.struct_mutex); ++ } ++ ++ if (val & DROP_RESET_ACTIVE && i915_terminally_wedged(i915)) ++ i915_handle_error(i915, ALL_ENGINES, 0, NULL); ++ ++ fs_reclaim_acquire(GFP_KERNEL); ++ if (val & DROP_BOUND) ++ i915_gem_shrink(i915, LONG_MAX, NULL, I915_SHRINK_BOUND); ++ ++ if (val & DROP_UNBOUND) ++ i915_gem_shrink(i915, LONG_MAX, NULL, I915_SHRINK_UNBOUND); ++ ++ if (val & DROP_SHRINK_ALL) ++ i915_gem_shrink_all(i915); ++ fs_reclaim_release(GFP_KERNEL); ++ ++ if (val & DROP_IDLE) { ++ do { ++ if (READ_ONCE(i915->gt.active_requests)) ++ flush_delayed_work(&i915->gt.retire_work); ++ drain_delayed_work(&i915->gt.idle_work); ++ } while (READ_ONCE(i915->gt.awake)); ++ } ++ ++ if (val & DROP_FREED) ++ i915_gem_drain_freed_objects(i915); ++ ++ return 0; ++} ++ ++DEFINE_SIMPLE_ATTRIBUTE(i915_drop_caches_fops, ++ i915_drop_caches_get, i915_drop_caches_set, ++ "0x%08llx\n"); ++ ++static int ++i915_cache_sharing_get(void *data, u64 *val) ++{ ++ struct drm_i915_private *dev_priv = data; ++ intel_wakeref_t wakeref; ++ u32 snpcr = 0; ++ ++ if (!(IS_GEN_RANGE(dev_priv, 6, 7))) ++ return -ENODEV; ++ ++ with_intel_runtime_pm(dev_priv, wakeref) ++ snpcr = I915_READ(GEN6_MBCUNIT_SNPCR); ++ ++ *val = (snpcr & GEN6_MBC_SNPCR_MASK) >> GEN6_MBC_SNPCR_SHIFT; ++ ++ return 0; ++} ++ ++static int ++i915_cache_sharing_set(void *data, u64 val) ++{ ++ struct drm_i915_private *dev_priv = data; ++ intel_wakeref_t wakeref; ++ ++ if (!(IS_GEN_RANGE(dev_priv, 6, 7))) ++ return -ENODEV; ++ ++ if (val > 3) ++ return -EINVAL; ++ ++ DRM_DEBUG_DRIVER("Manually setting uncore sharing to %llu\n", val); ++ with_intel_runtime_pm(dev_priv, wakeref) { ++ u32 snpcr; ++ ++ /* Update the cache sharing policy here as well */ ++ snpcr = I915_READ(GEN6_MBCUNIT_SNPCR); ++ snpcr &= ~GEN6_MBC_SNPCR_MASK; ++ snpcr |= val << GEN6_MBC_SNPCR_SHIFT; ++ I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr); ++ } ++ ++ return 0; ++} ++ ++DEFINE_SIMPLE_ATTRIBUTE(i915_cache_sharing_fops, ++ i915_cache_sharing_get, i915_cache_sharing_set, ++ "%llu\n"); ++ ++static void cherryview_sseu_device_status(struct drm_i915_private *dev_priv, ++ struct sseu_dev_info *sseu) ++{ ++#define SS_MAX 2 ++ const int ss_max = SS_MAX; ++ u32 sig1[SS_MAX], sig2[SS_MAX]; ++ int ss; ++ ++ sig1[0] = I915_READ(CHV_POWER_SS0_SIG1); ++ sig1[1] = I915_READ(CHV_POWER_SS1_SIG1); ++ sig2[0] = I915_READ(CHV_POWER_SS0_SIG2); ++ sig2[1] = I915_READ(CHV_POWER_SS1_SIG2); ++ ++ for (ss = 0; ss < ss_max; ss++) { ++ unsigned int eu_cnt; ++ ++ if (sig1[ss] & CHV_SS_PG_ENABLE) ++ /* skip disabled subslice */ ++ continue; ++ ++ sseu->slice_mask = BIT(0); ++ sseu->subslice_mask[0] |= BIT(ss); ++ eu_cnt = ((sig1[ss] & CHV_EU08_PG_ENABLE) ? 0 : 2) + ++ ((sig1[ss] & CHV_EU19_PG_ENABLE) ? 0 : 2) + ++ ((sig1[ss] & CHV_EU210_PG_ENABLE) ? 0 : 2) + ++ ((sig2[ss] & CHV_EU311_PG_ENABLE) ? 0 : 2); ++ sseu->eu_total += eu_cnt; ++ sseu->eu_per_subslice = max_t(unsigned int, ++ sseu->eu_per_subslice, eu_cnt); ++ } ++#undef SS_MAX ++} ++ ++static void gen10_sseu_device_status(struct drm_i915_private *dev_priv, ++ struct sseu_dev_info *sseu) ++{ ++#define SS_MAX 6 ++ const struct intel_runtime_info *info = RUNTIME_INFO(dev_priv); ++ u32 s_reg[SS_MAX], eu_reg[2 * SS_MAX], eu_mask[2]; ++ int s, ss; ++ ++ for (s = 0; s < info->sseu.max_slices; s++) { ++ /* ++ * FIXME: Valid SS Mask respects the spec and read ++ * only valid bits for those registers, excluding reserved ++ * although this seems wrong because it would leave many ++ * subslices without ACK. ++ */ ++ s_reg[s] = I915_READ(GEN10_SLICE_PGCTL_ACK(s)) & ++ GEN10_PGCTL_VALID_SS_MASK(s); ++ eu_reg[2 * s] = I915_READ(GEN10_SS01_EU_PGCTL_ACK(s)); ++ eu_reg[2 * s + 1] = I915_READ(GEN10_SS23_EU_PGCTL_ACK(s)); ++ } ++ ++ eu_mask[0] = GEN9_PGCTL_SSA_EU08_ACK | ++ GEN9_PGCTL_SSA_EU19_ACK | ++ GEN9_PGCTL_SSA_EU210_ACK | ++ GEN9_PGCTL_SSA_EU311_ACK; ++ eu_mask[1] = GEN9_PGCTL_SSB_EU08_ACK | ++ GEN9_PGCTL_SSB_EU19_ACK | ++ GEN9_PGCTL_SSB_EU210_ACK | ++ GEN9_PGCTL_SSB_EU311_ACK; ++ ++ for (s = 0; s < info->sseu.max_slices; s++) { ++ if ((s_reg[s] & GEN9_PGCTL_SLICE_ACK) == 0) ++ /* skip disabled slice */ ++ continue; ++ ++ sseu->slice_mask |= BIT(s); ++ sseu->subslice_mask[s] = info->sseu.subslice_mask[s]; ++ ++ for (ss = 0; ss < info->sseu.max_subslices; ss++) { ++ unsigned int eu_cnt; ++ ++ if (!(s_reg[s] & (GEN9_PGCTL_SS_ACK(ss)))) ++ /* skip disabled subslice */ ++ continue; ++ ++ eu_cnt = 2 * hweight32(eu_reg[2 * s + ss / 2] & ++ eu_mask[ss % 2]); ++ sseu->eu_total += eu_cnt; ++ sseu->eu_per_subslice = max_t(unsigned int, ++ sseu->eu_per_subslice, ++ eu_cnt); ++ } ++ } ++#undef SS_MAX ++} ++ ++static void gen9_sseu_device_status(struct drm_i915_private *dev_priv, ++ struct sseu_dev_info *sseu) ++{ ++#define SS_MAX 3 ++ const struct intel_runtime_info *info = RUNTIME_INFO(dev_priv); ++ u32 s_reg[SS_MAX], eu_reg[2 * SS_MAX], eu_mask[2]; ++ int s, ss; ++ ++ for (s = 0; s < info->sseu.max_slices; s++) { ++ s_reg[s] = I915_READ(GEN9_SLICE_PGCTL_ACK(s)); ++ eu_reg[2*s] = I915_READ(GEN9_SS01_EU_PGCTL_ACK(s)); ++ eu_reg[2*s + 1] = I915_READ(GEN9_SS23_EU_PGCTL_ACK(s)); ++ } ++ ++ eu_mask[0] = GEN9_PGCTL_SSA_EU08_ACK | ++ GEN9_PGCTL_SSA_EU19_ACK | ++ GEN9_PGCTL_SSA_EU210_ACK | ++ GEN9_PGCTL_SSA_EU311_ACK; ++ eu_mask[1] = GEN9_PGCTL_SSB_EU08_ACK | ++ GEN9_PGCTL_SSB_EU19_ACK | ++ GEN9_PGCTL_SSB_EU210_ACK | ++ GEN9_PGCTL_SSB_EU311_ACK; ++ ++ for (s = 0; s < info->sseu.max_slices; s++) { ++ if ((s_reg[s] & GEN9_PGCTL_SLICE_ACK) == 0) ++ /* skip disabled slice */ ++ continue; ++ ++ sseu->slice_mask |= BIT(s); ++ ++ if (IS_GEN9_BC(dev_priv)) ++ sseu->subslice_mask[s] = ++ RUNTIME_INFO(dev_priv)->sseu.subslice_mask[s]; ++ ++ for (ss = 0; ss < info->sseu.max_subslices; ss++) { ++ unsigned int eu_cnt; ++ ++ if (IS_GEN9_LP(dev_priv)) { ++ if (!(s_reg[s] & (GEN9_PGCTL_SS_ACK(ss)))) ++ /* skip disabled subslice */ ++ continue; ++ ++ sseu->subslice_mask[s] |= BIT(ss); ++ } ++ ++ eu_cnt = 2 * hweight32(eu_reg[2*s + ss/2] & ++ eu_mask[ss%2]); ++ sseu->eu_total += eu_cnt; ++ sseu->eu_per_subslice = max_t(unsigned int, ++ sseu->eu_per_subslice, ++ eu_cnt); ++ } ++ } ++#undef SS_MAX ++} ++ ++static void broadwell_sseu_device_status(struct drm_i915_private *dev_priv, ++ struct sseu_dev_info *sseu) ++{ ++ u32 slice_info = I915_READ(GEN8_GT_SLICE_INFO); ++ int s; ++ ++ sseu->slice_mask = slice_info & GEN8_LSLICESTAT_MASK; ++ ++ if (sseu->slice_mask) { ++ sseu->eu_per_subslice = ++ RUNTIME_INFO(dev_priv)->sseu.eu_per_subslice; ++ for (s = 0; s < fls(sseu->slice_mask); s++) { ++ sseu->subslice_mask[s] = ++ RUNTIME_INFO(dev_priv)->sseu.subslice_mask[s]; ++ } ++ sseu->eu_total = sseu->eu_per_subslice * ++ sseu_subslice_total(sseu); ++ ++ /* subtract fused off EU(s) from enabled slice(s) */ ++ for (s = 0; s < fls(sseu->slice_mask); s++) { ++ u8 subslice_7eu = ++ RUNTIME_INFO(dev_priv)->sseu.subslice_7eu[s]; ++ ++ sseu->eu_total -= hweight8(subslice_7eu); ++ } ++ } ++} ++ ++static void i915_print_sseu_info(struct seq_file *m, bool is_available_info, ++ const struct sseu_dev_info *sseu) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ const char *type = is_available_info ? "Available" : "Enabled"; ++ int s; ++ ++ seq_printf(m, " %s Slice Mask: %04x\n", type, ++ sseu->slice_mask); ++ seq_printf(m, " %s Slice Total: %u\n", type, ++ hweight8(sseu->slice_mask)); ++ seq_printf(m, " %s Subslice Total: %u\n", type, ++ sseu_subslice_total(sseu)); ++ for (s = 0; s < fls(sseu->slice_mask); s++) { ++ seq_printf(m, " %s Slice%i subslices: %u\n", type, ++ s, hweight8(sseu->subslice_mask[s])); ++ } ++ seq_printf(m, " %s EU Total: %u\n", type, ++ sseu->eu_total); ++ seq_printf(m, " %s EU Per Subslice: %u\n", type, ++ sseu->eu_per_subslice); ++ ++ if (!is_available_info) ++ return; ++ ++ seq_printf(m, " Has Pooled EU: %s\n", yesno(HAS_POOLED_EU(dev_priv))); ++ if (HAS_POOLED_EU(dev_priv)) ++ seq_printf(m, " Min EU in pool: %u\n", sseu->min_eu_in_pool); ++ ++ seq_printf(m, " Has Slice Power Gating: %s\n", ++ yesno(sseu->has_slice_pg)); ++ seq_printf(m, " Has Subslice Power Gating: %s\n", ++ yesno(sseu->has_subslice_pg)); ++ seq_printf(m, " Has EU Power Gating: %s\n", ++ yesno(sseu->has_eu_pg)); ++} ++ ++static int i915_sseu_status(struct seq_file *m, void *unused) ++{ ++ struct drm_i915_private *dev_priv = node_to_i915(m->private); ++ struct sseu_dev_info sseu; ++ intel_wakeref_t wakeref; ++ ++ if (INTEL_GEN(dev_priv) < 8) ++ return -ENODEV; ++ ++ seq_puts(m, "SSEU Device Info\n"); ++ i915_print_sseu_info(m, true, &RUNTIME_INFO(dev_priv)->sseu); ++ ++ seq_puts(m, "SSEU Device Status\n"); ++ memset(&sseu, 0, sizeof(sseu)); ++ sseu.max_slices = RUNTIME_INFO(dev_priv)->sseu.max_slices; ++ sseu.max_subslices = RUNTIME_INFO(dev_priv)->sseu.max_subslices; ++ sseu.max_eus_per_subslice = ++ RUNTIME_INFO(dev_priv)->sseu.max_eus_per_subslice; ++ ++ with_intel_runtime_pm(dev_priv, wakeref) { ++ if (IS_CHERRYVIEW(dev_priv)) ++ cherryview_sseu_device_status(dev_priv, &sseu); ++ else if (IS_BROADWELL(dev_priv)) ++ broadwell_sseu_device_status(dev_priv, &sseu); ++ else if (IS_GEN(dev_priv, 9)) ++ gen9_sseu_device_status(dev_priv, &sseu); ++ else if (INTEL_GEN(dev_priv) >= 10) ++ gen10_sseu_device_status(dev_priv, &sseu); ++ } ++ ++ i915_print_sseu_info(m, false, &sseu); ++ ++ return 0; ++} ++ ++static int i915_forcewake_open(struct inode *inode, struct file *file) ++{ ++ struct drm_i915_private *i915 = inode->i_private; ++ ++ if (INTEL_GEN(i915) < 6) ++ return 0; ++ ++ file->private_data = (void *)(uintptr_t)intel_runtime_pm_get(i915); ++ intel_uncore_forcewake_user_get(&i915->uncore); ++ ++ return 0; ++} ++ ++static int i915_forcewake_release(struct inode *inode, struct file *file) ++{ ++ struct drm_i915_private *i915 = inode->i_private; ++ ++ if (INTEL_GEN(i915) < 6) ++ return 0; ++ ++ intel_uncore_forcewake_user_put(&i915->uncore); ++ intel_runtime_pm_put(i915, ++ (intel_wakeref_t)(uintptr_t)file->private_data); ++ ++ return 0; ++} ++ ++static const struct file_operations i915_forcewake_fops = { ++ .owner = THIS_MODULE, ++ .open = i915_forcewake_open, ++ .release = i915_forcewake_release, ++}; ++ ++static int i915_hpd_storm_ctl_show(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *dev_priv = m->private; ++ struct i915_hotplug *hotplug = &dev_priv->hotplug; ++ ++ /* Synchronize with everything first in case there's been an HPD ++ * storm, but we haven't finished handling it in the kernel yet ++ */ ++ synchronize_irq(dev_priv->drm.irq); ++ flush_work(&dev_priv->hotplug.dig_port_work); ++ flush_work(&dev_priv->hotplug.hotplug_work); ++ ++ seq_printf(m, "Threshold: %d\n", hotplug->hpd_storm_threshold); ++ seq_printf(m, "Detected: %s\n", ++ yesno(delayed_work_pending(&hotplug->reenable_work))); ++ ++ return 0; ++} ++ ++static ssize_t i915_hpd_storm_ctl_write(struct file *file, ++ const char __user *ubuf, size_t len, ++ loff_t *offp) ++{ ++ struct seq_file *m = file->private_data; ++ struct drm_i915_private *dev_priv = m->private; ++ struct i915_hotplug *hotplug = &dev_priv->hotplug; ++ unsigned int new_threshold; ++ int i; ++ char *newline; ++ char tmp[16]; ++ ++ if (len >= sizeof(tmp)) ++ return -EINVAL; ++ ++ if (copy_from_user(tmp, ubuf, len)) ++ return -EFAULT; ++ ++ tmp[len] = '\0'; ++ ++ /* Strip newline, if any */ ++ newline = strchr(tmp, '\n'); ++ if (newline) ++ *newline = '\0'; ++ ++ if (strcmp(tmp, "reset") == 0) ++ new_threshold = HPD_STORM_DEFAULT_THRESHOLD; ++ else if (kstrtouint(tmp, 10, &new_threshold) != 0) ++ return -EINVAL; ++ ++ if (new_threshold > 0) ++ DRM_DEBUG_KMS("Setting HPD storm detection threshold to %d\n", ++ new_threshold); ++ else ++ DRM_DEBUG_KMS("Disabling HPD storm detection\n"); ++ ++ spin_lock_irq(&dev_priv->irq_lock); ++ hotplug->hpd_storm_threshold = new_threshold; ++ /* Reset the HPD storm stats so we don't accidentally trigger a storm */ ++ for_each_hpd_pin(i) ++ hotplug->stats[i].count = 0; ++ spin_unlock_irq(&dev_priv->irq_lock); ++ ++ /* Re-enable hpd immediately if we were in an irq storm */ ++ flush_delayed_work(&dev_priv->hotplug.reenable_work); ++ ++ return len; ++} ++ ++static int i915_hpd_storm_ctl_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, i915_hpd_storm_ctl_show, inode->i_private); ++} ++ ++static const struct file_operations i915_hpd_storm_ctl_fops = { ++ .owner = THIS_MODULE, ++ .open = i915_hpd_storm_ctl_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++ .write = i915_hpd_storm_ctl_write ++}; ++ ++static int i915_hpd_short_storm_ctl_show(struct seq_file *m, void *data) ++{ ++ struct drm_i915_private *dev_priv = m->private; ++ ++ seq_printf(m, "Enabled: %s\n", ++ yesno(dev_priv->hotplug.hpd_short_storm_enabled)); ++ ++ return 0; ++} ++ ++static int ++i915_hpd_short_storm_ctl_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, i915_hpd_short_storm_ctl_show, ++ inode->i_private); ++} ++ ++static ssize_t i915_hpd_short_storm_ctl_write(struct file *file, ++ const char __user *ubuf, ++ size_t len, loff_t *offp) ++{ ++ struct seq_file *m = file->private_data; ++ struct drm_i915_private *dev_priv = m->private; ++ struct i915_hotplug *hotplug = &dev_priv->hotplug; ++ char *newline; ++ char tmp[16]; ++ int i; ++ bool new_state; ++ ++ if (len >= sizeof(tmp)) ++ return -EINVAL; ++ ++ if (copy_from_user(tmp, ubuf, len)) ++ return -EFAULT; ++ ++ tmp[len] = '\0'; ++ ++ /* Strip newline, if any */ ++ newline = strchr(tmp, '\n'); ++ if (newline) ++ *newline = '\0'; ++ ++ /* Reset to the "default" state for this system */ ++ if (strcmp(tmp, "reset") == 0) ++ new_state = !HAS_DP_MST(dev_priv); ++ else if (kstrtobool(tmp, &new_state) != 0) ++ return -EINVAL; ++ ++ DRM_DEBUG_KMS("%sabling HPD short storm detection\n", ++ new_state ? "En" : "Dis"); ++ ++ spin_lock_irq(&dev_priv->irq_lock); ++ hotplug->hpd_short_storm_enabled = new_state; ++ /* Reset the HPD storm stats so we don't accidentally trigger a storm */ ++ for_each_hpd_pin(i) ++ hotplug->stats[i].count = 0; ++ spin_unlock_irq(&dev_priv->irq_lock); ++ ++ /* Re-enable hpd immediately if we were in an irq storm */ ++ flush_delayed_work(&dev_priv->hotplug.reenable_work); ++ ++ return len; ++} ++ ++static const struct file_operations i915_hpd_short_storm_ctl_fops = { ++ .owner = THIS_MODULE, ++ .open = i915_hpd_short_storm_ctl_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++ .write = i915_hpd_short_storm_ctl_write, ++}; ++ ++static int i915_drrs_ctl_set(void *data, u64 val) ++{ ++ struct drm_i915_private *dev_priv = data; ++ struct drm_device *dev = &dev_priv->drm; ++ struct intel_crtc *crtc; ++ ++ if (INTEL_GEN(dev_priv) < 7) ++ return -ENODEV; ++ ++ for_each_intel_crtc(dev, crtc) { ++ struct drm_connector_list_iter conn_iter; ++ struct intel_crtc_state *crtc_state; ++ struct drm_connector *connector; ++ struct drm_crtc_commit *commit; ++ int ret; ++ ++ ret = drm_modeset_lock_single_interruptible(&crtc->base.mutex); ++ if (ret) ++ return ret; ++ ++ crtc_state = to_intel_crtc_state(crtc->base.state); ++ ++ if (!crtc_state->base.active || ++ !crtc_state->has_drrs) ++ goto out; ++ ++ commit = crtc_state->base.commit; ++ if (commit) { ++ ret = wait_for_completion_interruptible(&commit->hw_done); ++ if (ret) ++ goto out; ++ } ++ ++ drm_connector_list_iter_begin(dev, &conn_iter); ++ drm_for_each_connector_iter(connector, &conn_iter) { ++ struct intel_encoder *encoder; ++ struct intel_dp *intel_dp; ++ ++ if (!(crtc_state->base.connector_mask & ++ drm_connector_mask(connector))) ++ continue; ++ ++ encoder = intel_attached_encoder(connector); ++ if (encoder->type != INTEL_OUTPUT_EDP) ++ continue; ++ ++ DRM_DEBUG_DRIVER("Manually %sabling DRRS. %llu\n", ++ val ? "en" : "dis", val); ++ ++ intel_dp = enc_to_intel_dp(&encoder->base); ++ if (val) ++ intel_edp_drrs_enable(intel_dp, ++ crtc_state); ++ else ++ intel_edp_drrs_disable(intel_dp, ++ crtc_state); ++ } ++ drm_connector_list_iter_end(&conn_iter); ++ ++out: ++ drm_modeset_unlock(&crtc->base.mutex); ++ if (ret) ++ return ret; ++ } ++ ++ return 0; ++} ++ ++DEFINE_SIMPLE_ATTRIBUTE(i915_drrs_ctl_fops, NULL, i915_drrs_ctl_set, "%llu\n"); ++ ++static ssize_t ++i915_fifo_underrun_reset_write(struct file *filp, ++ const char __user *ubuf, ++ size_t cnt, loff_t *ppos) ++{ ++ struct drm_i915_private *dev_priv = filp->private_data; ++ struct intel_crtc *intel_crtc; ++ struct drm_device *dev = &dev_priv->drm; ++ int ret; ++ bool reset; ++ ++ ret = kstrtobool_from_user(ubuf, cnt, &reset); ++ if (ret) ++ return ret; ++ ++ if (!reset) ++ return cnt; ++ ++ for_each_intel_crtc(dev, intel_crtc) { ++ struct drm_crtc_commit *commit; ++ struct intel_crtc_state *crtc_state; ++ ++ ret = drm_modeset_lock_single_interruptible(&intel_crtc->base.mutex); ++ if (ret) ++ return ret; ++ ++ crtc_state = to_intel_crtc_state(intel_crtc->base.state); ++ commit = crtc_state->base.commit; ++ if (commit) { ++ ret = wait_for_completion_interruptible(&commit->hw_done); ++ if (!ret) ++ ret = wait_for_completion_interruptible(&commit->flip_done); ++ } ++ ++ if (!ret && crtc_state->base.active) { ++ DRM_DEBUG_KMS("Re-arming FIFO underruns on pipe %c\n", ++ pipe_name(intel_crtc->pipe)); ++ ++ intel_crtc_arm_fifo_underrun(intel_crtc, crtc_state); ++ } ++ ++ drm_modeset_unlock(&intel_crtc->base.mutex); ++ ++ if (ret) ++ return ret; ++ } ++ ++ ret = intel_fbc_reset_underrun(dev_priv); ++ if (ret) ++ return ret; ++ ++ return cnt; ++} ++ ++static const struct file_operations i915_fifo_underrun_reset_ops = { ++ .owner = THIS_MODULE, ++ .open = simple_open, ++ .write = i915_fifo_underrun_reset_write, ++ .llseek = default_llseek, ++}; ++ ++static const struct drm_info_list i915_debugfs_list[] = { ++ {"i915_capabilities", i915_capabilities, 0}, ++ {"i915_gem_objects", i915_gem_object_info, 0}, ++ {"i915_gem_gtt", i915_gem_gtt_info, 0}, ++ {"i915_gem_stolen", i915_gem_stolen_list_info }, ++ {"i915_gem_fence_regs", i915_gem_fence_regs_info, 0}, ++ {"i915_gem_interrupt", i915_interrupt_info, 0}, ++ {"i915_gem_batch_pool", i915_gem_batch_pool_info, 0}, ++ {"i915_guc_info", i915_guc_info, 0}, ++ {"i915_guc_load_status", i915_guc_load_status_info, 0}, ++ {"i915_guc_log_dump", i915_guc_log_dump, 0}, ++ {"i915_guc_load_err_log_dump", i915_guc_log_dump, 0, (void *)1}, ++ {"i915_guc_stage_pool", i915_guc_stage_pool, 0}, ++ {"i915_huc_load_status", i915_huc_load_status_info, 0}, ++ {"i915_frequency_info", i915_frequency_info, 0}, ++ {"i915_hangcheck_info", i915_hangcheck_info, 0}, ++ {"i915_reset_info", i915_reset_info, 0}, ++ {"i915_drpc_info", i915_drpc_info, 0}, ++ {"i915_emon_status", i915_emon_status, 0}, ++ {"i915_ring_freq_table", i915_ring_freq_table, 0}, ++ {"i915_frontbuffer_tracking", i915_frontbuffer_tracking, 0}, ++ {"i915_fbc_status", i915_fbc_status, 0}, ++ {"i915_ips_status", i915_ips_status, 0}, ++ {"i915_sr_status", i915_sr_status, 0}, ++ {"i915_opregion", i915_opregion, 0}, ++ {"i915_vbt", i915_vbt, 0}, ++ {"i915_gem_framebuffer", i915_gem_framebuffer_info, 0}, ++ {"i915_context_status", i915_context_status, 0}, ++ {"i915_forcewake_domains", i915_forcewake_domains, 0}, ++ {"i915_swizzle_info", i915_swizzle_info, 0}, ++ {"i915_llc", i915_llc, 0}, ++ {"i915_edp_psr_status", i915_edp_psr_status, 0}, ++ {"i915_energy_uJ", i915_energy_uJ, 0}, ++ {"i915_runtime_pm_status", i915_runtime_pm_status, 0}, ++ {"i915_power_domain_info", i915_power_domain_info, 0}, ++ {"i915_dmc_info", i915_dmc_info, 0}, ++ {"i915_display_info", i915_display_info, 0}, ++ {"i915_engine_info", i915_engine_info, 0}, ++ {"i915_rcs_topology", i915_rcs_topology, 0}, ++ {"i915_shrinker_info", i915_shrinker_info, 0}, ++ {"i915_shared_dplls_info", i915_shared_dplls_info, 0}, ++ {"i915_dp_mst_info", i915_dp_mst_info, 0}, ++ {"i915_wa_registers", i915_wa_registers, 0}, ++ {"i915_ddb_info", i915_ddb_info, 0}, ++ {"i915_sseu_status", i915_sseu_status, 0}, ++ {"i915_drrs_status", i915_drrs_status, 0}, ++ {"i915_rps_boost_info", i915_rps_boost_info, 0}, ++}; ++#define I915_DEBUGFS_ENTRIES ARRAY_SIZE(i915_debugfs_list) ++ ++static const struct i915_debugfs_files { ++ const char *name; ++ const struct file_operations *fops; ++} i915_debugfs_files[] = { ++ {"i915_wedged", &i915_wedged_fops}, ++ {"i915_cache_sharing", &i915_cache_sharing_fops}, ++ {"i915_gem_drop_caches", &i915_drop_caches_fops}, ++#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) ++ {"i915_error_state", &i915_error_state_fops}, ++ {"i915_gpu_info", &i915_gpu_info_fops}, ++#endif ++ {"i915_fifo_underrun_reset", &i915_fifo_underrun_reset_ops}, ++ {"i915_pri_wm_latency", &i915_pri_wm_latency_fops}, ++ {"i915_spr_wm_latency", &i915_spr_wm_latency_fops}, ++ {"i915_cur_wm_latency", &i915_cur_wm_latency_fops}, ++ {"i915_fbc_false_color", &i915_fbc_false_color_fops}, ++ {"i915_dp_test_data", &i915_displayport_test_data_fops}, ++ {"i915_dp_test_type", &i915_displayport_test_type_fops}, ++ {"i915_dp_test_active", &i915_displayport_test_active_fops}, ++ {"i915_guc_log_level", &i915_guc_log_level_fops}, ++ {"i915_guc_log_relay", &i915_guc_log_relay_fops}, ++ {"i915_hpd_storm_ctl", &i915_hpd_storm_ctl_fops}, ++ {"i915_hpd_short_storm_ctl", &i915_hpd_short_storm_ctl_fops}, ++ {"i915_ipc_status", &i915_ipc_status_fops}, ++ {"i915_drrs_ctl", &i915_drrs_ctl_fops}, ++ {"i915_edp_psr_debug", &i915_edp_psr_debug_fops} ++}; ++ ++int i915_debugfs_register(struct drm_i915_private *dev_priv) ++{ ++ struct drm_minor *minor = dev_priv->drm.primary; ++ struct dentry *ent; ++ int i; ++ ++ ent = debugfs_create_file("i915_forcewake_user", S_IRUSR, ++ minor->debugfs_root, to_i915(minor->dev), ++ &i915_forcewake_fops); ++ if (!ent) ++ return -ENOMEM; ++ ++ for (i = 0; i < ARRAY_SIZE(i915_debugfs_files); i++) { ++ ent = debugfs_create_file(i915_debugfs_files[i].name, ++ S_IRUGO | S_IWUSR, ++ minor->debugfs_root, ++ to_i915(minor->dev), ++ i915_debugfs_files[i].fops); ++ if (!ent) ++ return -ENOMEM; ++ } ++ ++ return drm_debugfs_create_files(i915_debugfs_list, ++ I915_DEBUGFS_ENTRIES, ++ minor->debugfs_root, minor); ++} ++ ++struct dpcd_block { ++ /* DPCD dump start address. */ ++ unsigned int offset; ++ /* DPCD dump end address, inclusive. If unset, .size will be used. */ ++ unsigned int end; ++ /* DPCD dump size. Used if .end is unset. If unset, defaults to 1. */ ++ size_t size; ++ /* Only valid for eDP. */ ++ bool edp; ++}; ++ ++static const struct dpcd_block i915_dpcd_debug[] = { ++ { .offset = DP_DPCD_REV, .size = DP_RECEIVER_CAP_SIZE }, ++ { .offset = DP_PSR_SUPPORT, .end = DP_PSR_CAPS }, ++ { .offset = DP_DOWNSTREAM_PORT_0, .size = 16 }, ++ { .offset = DP_LINK_BW_SET, .end = DP_EDP_CONFIGURATION_SET }, ++ { .offset = DP_SINK_COUNT, .end = DP_ADJUST_REQUEST_LANE2_3 }, ++ { .offset = DP_SET_POWER }, ++ { .offset = DP_EDP_DPCD_REV }, ++ { .offset = DP_EDP_GENERAL_CAP_1, .end = DP_EDP_GENERAL_CAP_3 }, ++ { .offset = DP_EDP_DISPLAY_CONTROL_REGISTER, .end = DP_EDP_BACKLIGHT_FREQ_CAP_MAX_LSB }, ++ { .offset = DP_EDP_DBC_MINIMUM_BRIGHTNESS_SET, .end = DP_EDP_DBC_MAXIMUM_BRIGHTNESS_SET }, ++}; ++ ++static int i915_dpcd_show(struct seq_file *m, void *data) ++{ ++ struct drm_connector *connector = m->private; ++ struct intel_dp *intel_dp = ++ enc_to_intel_dp(&intel_attached_encoder(connector)->base); ++ u8 buf[16]; ++ ssize_t err; ++ int i; ++ ++ if (connector->status != connector_status_connected) ++ return -ENODEV; ++ ++ for (i = 0; i < ARRAY_SIZE(i915_dpcd_debug); i++) { ++ const struct dpcd_block *b = &i915_dpcd_debug[i]; ++ size_t size = b->end ? b->end - b->offset + 1 : (b->size ?: 1); ++ ++ if (b->edp && ++ connector->connector_type != DRM_MODE_CONNECTOR_eDP) ++ continue; ++ ++ /* low tech for now */ ++ if (WARN_ON(size > sizeof(buf))) ++ continue; ++ ++ err = drm_dp_dpcd_read(&intel_dp->aux, b->offset, buf, size); ++ if (err < 0) ++ seq_printf(m, "%04x: ERROR %d\n", b->offset, (int)err); ++ else ++ seq_printf(m, "%04x: %*ph\n", b->offset, (int)err, buf); ++ } ++ ++ return 0; ++} ++DEFINE_SHOW_ATTRIBUTE(i915_dpcd); ++ ++static int i915_panel_show(struct seq_file *m, void *data) ++{ ++ struct drm_connector *connector = m->private; ++ struct intel_dp *intel_dp = ++ enc_to_intel_dp(&intel_attached_encoder(connector)->base); ++ ++ if (connector->status != connector_status_connected) ++ return -ENODEV; ++ ++ seq_printf(m, "Panel power up delay: %d\n", ++ intel_dp->panel_power_up_delay); ++ seq_printf(m, "Panel power down delay: %d\n", ++ intel_dp->panel_power_down_delay); ++ seq_printf(m, "Backlight on delay: %d\n", ++ intel_dp->backlight_on_delay); ++ seq_printf(m, "Backlight off delay: %d\n", ++ intel_dp->backlight_off_delay); ++ ++ return 0; ++} ++DEFINE_SHOW_ATTRIBUTE(i915_panel); ++ ++static int i915_hdcp_sink_capability_show(struct seq_file *m, void *data) ++{ ++ struct drm_connector *connector = m->private; ++ struct intel_connector *intel_connector = to_intel_connector(connector); ++ ++ if (connector->status != connector_status_connected) ++ return -ENODEV; ++ ++ /* HDCP is supported by connector */ ++ if (!intel_connector->hdcp.shim) ++ return -EINVAL; ++ ++ seq_printf(m, "%s:%d HDCP version: ", connector->name, ++ connector->base.id); ++ seq_printf(m, "%s ", !intel_hdcp_capable(intel_connector) ? ++ "None" : "HDCP1.4"); ++ seq_puts(m, "\n"); ++ ++ return 0; ++} ++DEFINE_SHOW_ATTRIBUTE(i915_hdcp_sink_capability); ++ ++static int i915_dsc_fec_support_show(struct seq_file *m, void *data) ++{ ++ struct drm_connector *connector = m->private; ++ struct drm_device *dev = connector->dev; ++ struct drm_crtc *crtc; ++ struct intel_dp *intel_dp; ++ struct drm_modeset_acquire_ctx ctx; ++ struct intel_crtc_state *crtc_state = NULL; ++ int ret = 0; ++ bool try_again = false; ++ ++ drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE); ++ ++ do { ++ try_again = false; ++ ret = drm_modeset_lock(&dev->mode_config.connection_mutex, ++ &ctx); ++ if (ret) { ++ if (ret == -EDEADLK && !drm_modeset_backoff(&ctx)) { ++ try_again = true; ++ continue; ++ } ++ break; ++ } ++ crtc = connector->state->crtc; ++ if (connector->status != connector_status_connected || !crtc) { ++ ret = -ENODEV; ++ break; ++ } ++ ret = drm_modeset_lock(&crtc->mutex, &ctx); ++ if (ret == -EDEADLK) { ++ ret = drm_modeset_backoff(&ctx); ++ if (!ret) { ++ try_again = true; ++ continue; ++ } ++ break; ++ } else if (ret) { ++ break; ++ } ++ intel_dp = enc_to_intel_dp(&intel_attached_encoder(connector)->base); ++ crtc_state = to_intel_crtc_state(crtc->state); ++ seq_printf(m, "DSC_Enabled: %s\n", ++ yesno(crtc_state->dsc_params.compression_enable)); ++ seq_printf(m, "DSC_Sink_Support: %s\n", ++ yesno(drm_dp_sink_supports_dsc(intel_dp->dsc_dpcd))); ++ seq_printf(m, "Force_DSC_Enable: %s\n", ++ yesno(intel_dp->force_dsc_en)); ++ if (!intel_dp_is_edp(intel_dp)) ++ seq_printf(m, "FEC_Sink_Support: %s\n", ++ yesno(drm_dp_sink_supports_fec(intel_dp->fec_capable))); ++ } while (try_again); ++ ++ drm_modeset_drop_locks(&ctx); ++ drm_modeset_acquire_fini(&ctx); ++ ++ return ret; ++} ++ ++static ssize_t i915_dsc_fec_support_write(struct file *file, ++ const char __user *ubuf, ++ size_t len, loff_t *offp) ++{ ++ bool dsc_enable = false; ++ int ret; ++ struct drm_connector *connector = ++ ((struct seq_file *)file->private_data)->private; ++ struct intel_encoder *encoder = intel_attached_encoder(connector); ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ ++ if (len == 0) ++ return 0; ++ ++ DRM_DEBUG_DRIVER("Copied %zu bytes from user to force DSC\n", ++ len); ++ ++ ret = kstrtobool_from_user(ubuf, len, &dsc_enable); ++ if (ret < 0) ++ return ret; ++ ++ DRM_DEBUG_DRIVER("Got %s for DSC Enable\n", ++ (dsc_enable) ? "true" : "false"); ++ intel_dp->force_dsc_en = dsc_enable; ++ ++ *offp += len; ++ return len; ++} ++ ++static int i915_dsc_fec_support_open(struct inode *inode, ++ struct file *file) ++{ ++ return single_open(file, i915_dsc_fec_support_show, ++ inode->i_private); ++} ++ ++static const struct file_operations i915_dsc_fec_support_fops = { ++ .owner = THIS_MODULE, ++ .open = i915_dsc_fec_support_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++ .write = i915_dsc_fec_support_write ++}; ++ ++/** ++ * i915_debugfs_connector_add - add i915 specific connector debugfs files ++ * @connector: pointer to a registered drm_connector ++ * ++ * Cleanup will be done by drm_connector_unregister() through a call to ++ * drm_debugfs_connector_remove(). ++ * ++ * Returns 0 on success, negative error codes on error. ++ */ ++int i915_debugfs_connector_add(struct drm_connector *connector) ++{ ++ struct dentry *root = connector->debugfs_entry; ++ struct drm_i915_private *dev_priv = to_i915(connector->dev); ++ ++ /* The connector must have been registered beforehands. */ ++ if (!root) ++ return -ENODEV; ++ ++ if (connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort || ++ connector->connector_type == DRM_MODE_CONNECTOR_eDP) ++ debugfs_create_file("i915_dpcd", S_IRUGO, root, ++ connector, &i915_dpcd_fops); ++ ++ if (connector->connector_type == DRM_MODE_CONNECTOR_eDP) { ++ debugfs_create_file("i915_panel_timings", S_IRUGO, root, ++ connector, &i915_panel_fops); ++ debugfs_create_file("i915_psr_sink_status", S_IRUGO, root, ++ connector, &i915_psr_sink_status_fops); ++ } ++ ++ if (connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort || ++ connector->connector_type == DRM_MODE_CONNECTOR_HDMIA || ++ connector->connector_type == DRM_MODE_CONNECTOR_HDMIB) { ++ debugfs_create_file("i915_hdcp_sink_capability", S_IRUGO, root, ++ connector, &i915_hdcp_sink_capability_fops); ++ } ++ ++ if (INTEL_GEN(dev_priv) >= 10 && ++ (connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort || ++ connector->connector_type == DRM_MODE_CONNECTOR_eDP)) ++ debugfs_create_file("i915_dsc_fec_support", S_IRUGO, root, ++ connector, &i915_dsc_fec_support_fops); ++ ++ return 0; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_drv.c b/drivers/gpu/drm/i915_legacy/i915_drv.c +new file mode 100644 +index 000000000000..d485d49c473b +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_drv.c +@@ -0,0 +1,3195 @@ ++/* i915_drv.c -- i830,i845,i855,i865,i915 driver -*- linux-c -*- ++ */ ++/* ++ * ++ * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. ++ * All Rights Reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the ++ * "Software"), to deal in the Software without restriction, including ++ * without limitation the rights to use, copy, modify, merge, publish, ++ * distribute, sub license, and/or sell copies of the Software, and to ++ * permit persons to whom the Software is furnished to do so, subject to ++ * the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the ++ * next paragraph) shall be included in all copies or substantial portions ++ * of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS ++ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF ++ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. ++ * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR ++ * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, ++ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE ++ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "i915_drv.h" ++#include "i915_pmu.h" ++#include "i915_query.h" ++#include "i915_reset.h" ++#include "i915_trace.h" ++#include "i915_vgpu.h" ++#include "intel_audio.h" ++#include "intel_cdclk.h" ++#include "intel_csr.h" ++#include "intel_dp.h" ++#include "intel_drv.h" ++#include "intel_fbdev.h" ++#include "intel_pm.h" ++#include "intel_sprite.h" ++#include "intel_uc.h" ++#include "intel_workarounds.h" ++ ++static struct drm_driver driver; ++ ++#if IS_ENABLED(CONFIG_DRM_I915_DEBUG) ++static unsigned int i915_load_fail_count; ++ ++bool __i915_inject_load_failure(const char *func, int line) ++{ ++ if (i915_load_fail_count >= i915_modparams.inject_load_failure) ++ return false; ++ ++ if (++i915_load_fail_count == i915_modparams.inject_load_failure) { ++ DRM_INFO("Injecting failure at checkpoint %u [%s:%d]\n", ++ i915_modparams.inject_load_failure, func, line); ++ i915_modparams.inject_load_failure = 0; ++ return true; ++ } ++ ++ return false; ++} ++ ++bool i915_error_injected(void) ++{ ++ return i915_load_fail_count && !i915_modparams.inject_load_failure; ++} ++ ++#endif ++ ++#define FDO_BUG_URL "https://bugs.freedesktop.org/enter_bug.cgi?product=DRI" ++#define FDO_BUG_MSG "Please file a bug at " FDO_BUG_URL " against DRM/Intel " \ ++ "providing the dmesg log by booting with drm.debug=0xf" ++ ++void ++__i915_printk(struct drm_i915_private *dev_priv, const char *level, ++ const char *fmt, ...) ++{ ++ static bool shown_bug_once; ++ struct device *kdev = dev_priv->drm.dev; ++ bool is_error = level[1] <= KERN_ERR[1]; ++ bool is_debug = level[1] == KERN_DEBUG[1]; ++ struct va_format vaf; ++ va_list args; ++ ++ if (is_debug && !(drm_debug & DRM_UT_DRIVER)) ++ return; ++ ++ va_start(args, fmt); ++ ++ vaf.fmt = fmt; ++ vaf.va = &args; ++ ++ if (is_error) ++ dev_printk(level, kdev, "%pV", &vaf); ++ else ++ dev_printk(level, kdev, "[" DRM_NAME ":%ps] %pV", ++ __builtin_return_address(0), &vaf); ++ ++ va_end(args); ++ ++ if (is_error && !shown_bug_once) { ++ /* ++ * Ask the user to file a bug report for the error, except ++ * if they may have caused the bug by fiddling with unsafe ++ * module parameters. ++ */ ++ if (!test_taint(TAINT_USER)) ++ dev_notice(kdev, "%s", FDO_BUG_MSG); ++ shown_bug_once = true; ++ } ++} ++ ++/* Map PCH device id to PCH type, or PCH_NONE if unknown. */ ++static enum intel_pch ++intel_pch_type(const struct drm_i915_private *dev_priv, unsigned short id) ++{ ++ switch (id) { ++ case INTEL_PCH_IBX_DEVICE_ID_TYPE: ++ DRM_DEBUG_KMS("Found Ibex Peak PCH\n"); ++ WARN_ON(!IS_GEN(dev_priv, 5)); ++ return PCH_IBX; ++ case INTEL_PCH_CPT_DEVICE_ID_TYPE: ++ DRM_DEBUG_KMS("Found CougarPoint PCH\n"); ++ WARN_ON(!IS_GEN(dev_priv, 6) && !IS_IVYBRIDGE(dev_priv)); ++ return PCH_CPT; ++ case INTEL_PCH_PPT_DEVICE_ID_TYPE: ++ DRM_DEBUG_KMS("Found PantherPoint PCH\n"); ++ WARN_ON(!IS_GEN(dev_priv, 6) && !IS_IVYBRIDGE(dev_priv)); ++ /* PantherPoint is CPT compatible */ ++ return PCH_CPT; ++ case INTEL_PCH_LPT_DEVICE_ID_TYPE: ++ DRM_DEBUG_KMS("Found LynxPoint PCH\n"); ++ WARN_ON(!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)); ++ WARN_ON(IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv)); ++ return PCH_LPT; ++ case INTEL_PCH_LPT_LP_DEVICE_ID_TYPE: ++ DRM_DEBUG_KMS("Found LynxPoint LP PCH\n"); ++ WARN_ON(!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)); ++ WARN_ON(!IS_HSW_ULT(dev_priv) && !IS_BDW_ULT(dev_priv)); ++ return PCH_LPT; ++ case INTEL_PCH_WPT_DEVICE_ID_TYPE: ++ DRM_DEBUG_KMS("Found WildcatPoint PCH\n"); ++ WARN_ON(!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)); ++ WARN_ON(IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv)); ++ /* WildcatPoint is LPT compatible */ ++ return PCH_LPT; ++ case INTEL_PCH_WPT_LP_DEVICE_ID_TYPE: ++ DRM_DEBUG_KMS("Found WildcatPoint LP PCH\n"); ++ WARN_ON(!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)); ++ WARN_ON(!IS_HSW_ULT(dev_priv) && !IS_BDW_ULT(dev_priv)); ++ /* WildcatPoint is LPT compatible */ ++ return PCH_LPT; ++ case INTEL_PCH_SPT_DEVICE_ID_TYPE: ++ DRM_DEBUG_KMS("Found SunrisePoint PCH\n"); ++ WARN_ON(!IS_SKYLAKE(dev_priv) && !IS_KABYLAKE(dev_priv)); ++ return PCH_SPT; ++ case INTEL_PCH_SPT_LP_DEVICE_ID_TYPE: ++ DRM_DEBUG_KMS("Found SunrisePoint LP PCH\n"); ++ WARN_ON(!IS_SKYLAKE(dev_priv) && !IS_KABYLAKE(dev_priv)); ++ return PCH_SPT; ++ case INTEL_PCH_KBP_DEVICE_ID_TYPE: ++ DRM_DEBUG_KMS("Found Kaby Lake PCH (KBP)\n"); ++ WARN_ON(!IS_SKYLAKE(dev_priv) && !IS_KABYLAKE(dev_priv) && ++ !IS_COFFEELAKE(dev_priv)); ++ return PCH_KBP; ++ case INTEL_PCH_CNP_DEVICE_ID_TYPE: ++ DRM_DEBUG_KMS("Found Cannon Lake PCH (CNP)\n"); ++ WARN_ON(!IS_CANNONLAKE(dev_priv) && !IS_COFFEELAKE(dev_priv)); ++ return PCH_CNP; ++ case INTEL_PCH_CNP_LP_DEVICE_ID_TYPE: ++ DRM_DEBUG_KMS("Found Cannon Lake LP PCH (CNP-LP)\n"); ++ WARN_ON(!IS_CANNONLAKE(dev_priv) && !IS_COFFEELAKE(dev_priv)); ++ return PCH_CNP; ++ case INTEL_PCH_CMP_DEVICE_ID_TYPE: ++ DRM_DEBUG_KMS("Found Comet Lake PCH (CMP)\n"); ++ WARN_ON(!IS_COFFEELAKE(dev_priv)); ++ /* CometPoint is CNP Compatible */ ++ return PCH_CNP; ++ case INTEL_PCH_ICP_DEVICE_ID_TYPE: ++ DRM_DEBUG_KMS("Found Ice Lake PCH\n"); ++ WARN_ON(!IS_ICELAKE(dev_priv)); ++ return PCH_ICP; ++ default: ++ return PCH_NONE; ++ } ++} ++ ++static bool intel_is_virt_pch(unsigned short id, ++ unsigned short svendor, unsigned short sdevice) ++{ ++ return (id == INTEL_PCH_P2X_DEVICE_ID_TYPE || ++ id == INTEL_PCH_P3X_DEVICE_ID_TYPE || ++ (id == INTEL_PCH_QEMU_DEVICE_ID_TYPE && ++ svendor == PCI_SUBVENDOR_ID_REDHAT_QUMRANET && ++ sdevice == PCI_SUBDEVICE_ID_QEMU)); ++} ++ ++static unsigned short ++intel_virt_detect_pch(const struct drm_i915_private *dev_priv) ++{ ++ unsigned short id = 0; ++ ++ /* ++ * In a virtualized passthrough environment we can be in a ++ * setup where the ISA bridge is not able to be passed through. ++ * In this case, a south bridge can be emulated and we have to ++ * make an educated guess as to which PCH is really there. ++ */ ++ ++ if (IS_ICELAKE(dev_priv)) ++ id = INTEL_PCH_ICP_DEVICE_ID_TYPE; ++ else if (IS_CANNONLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) ++ id = INTEL_PCH_CNP_DEVICE_ID_TYPE; ++ else if (IS_KABYLAKE(dev_priv) || IS_SKYLAKE(dev_priv)) ++ id = INTEL_PCH_SPT_DEVICE_ID_TYPE; ++ else if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv)) ++ id = INTEL_PCH_LPT_LP_DEVICE_ID_TYPE; ++ else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) ++ id = INTEL_PCH_LPT_DEVICE_ID_TYPE; ++ else if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv)) ++ id = INTEL_PCH_CPT_DEVICE_ID_TYPE; ++ else if (IS_GEN(dev_priv, 5)) ++ id = INTEL_PCH_IBX_DEVICE_ID_TYPE; ++ ++ if (id) ++ DRM_DEBUG_KMS("Assuming PCH ID %04x\n", id); ++ else ++ DRM_DEBUG_KMS("Assuming no PCH\n"); ++ ++ return id; ++} ++ ++static void intel_detect_pch(struct drm_i915_private *dev_priv) ++{ ++ struct pci_dev *pch = NULL; ++ ++ /* ++ * The reason to probe ISA bridge instead of Dev31:Fun0 is to ++ * make graphics device passthrough work easy for VMM, that only ++ * need to expose ISA bridge to let driver know the real hardware ++ * underneath. This is a requirement from virtualization team. ++ * ++ * In some virtualized environments (e.g. XEN), there is irrelevant ++ * ISA bridge in the system. To work reliably, we should scan trhough ++ * all the ISA bridge devices and check for the first match, instead ++ * of only checking the first one. ++ */ ++ while ((pch = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, pch))) { ++ unsigned short id; ++ enum intel_pch pch_type; ++ ++ if (pch->vendor != PCI_VENDOR_ID_INTEL) ++ continue; ++ ++ id = pch->device & INTEL_PCH_DEVICE_ID_MASK; ++ ++ pch_type = intel_pch_type(dev_priv, id); ++ if (pch_type != PCH_NONE) { ++ dev_priv->pch_type = pch_type; ++ dev_priv->pch_id = id; ++ break; ++ } else if (intel_is_virt_pch(id, pch->subsystem_vendor, ++ pch->subsystem_device)) { ++ id = intel_virt_detect_pch(dev_priv); ++ pch_type = intel_pch_type(dev_priv, id); ++ ++ /* Sanity check virtual PCH id */ ++ if (WARN_ON(id && pch_type == PCH_NONE)) ++ id = 0; ++ ++ dev_priv->pch_type = pch_type; ++ dev_priv->pch_id = id; ++ break; ++ } ++ } ++ ++ /* ++ * Use PCH_NOP (PCH but no South Display) for PCH platforms without ++ * display. ++ */ ++ if (pch && !HAS_DISPLAY(dev_priv)) { ++ DRM_DEBUG_KMS("Display disabled, reverting to NOP PCH\n"); ++ dev_priv->pch_type = PCH_NOP; ++ dev_priv->pch_id = 0; ++ } ++ ++ if (!pch) ++ DRM_DEBUG_KMS("No PCH found.\n"); ++ ++ pci_dev_put(pch); ++} ++ ++static int i915_getparam_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file_priv) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ drm_i915_getparam_t *param = data; ++ int value; ++ ++ switch (param->param) { ++ case I915_PARAM_IRQ_ACTIVE: ++ case I915_PARAM_ALLOW_BATCHBUFFER: ++ case I915_PARAM_LAST_DISPATCH: ++ case I915_PARAM_HAS_EXEC_CONSTANTS: ++ /* Reject all old ums/dri params. */ ++ return -ENODEV; ++ case I915_PARAM_CHIPSET_ID: ++ value = pdev->device; ++ break; ++ case I915_PARAM_REVISION: ++ value = pdev->revision; ++ break; ++ case I915_PARAM_NUM_FENCES_AVAIL: ++ value = dev_priv->num_fence_regs; ++ break; ++ case I915_PARAM_HAS_OVERLAY: ++ value = dev_priv->overlay ? 1 : 0; ++ break; ++ case I915_PARAM_HAS_BSD: ++ value = !!dev_priv->engine[VCS0]; ++ break; ++ case I915_PARAM_HAS_BLT: ++ value = !!dev_priv->engine[BCS0]; ++ break; ++ case I915_PARAM_HAS_VEBOX: ++ value = !!dev_priv->engine[VECS0]; ++ break; ++ case I915_PARAM_HAS_BSD2: ++ value = !!dev_priv->engine[VCS1]; ++ break; ++ case I915_PARAM_HAS_LLC: ++ value = HAS_LLC(dev_priv); ++ break; ++ case I915_PARAM_HAS_WT: ++ value = HAS_WT(dev_priv); ++ break; ++ case I915_PARAM_HAS_ALIASING_PPGTT: ++ value = INTEL_PPGTT(dev_priv); ++ break; ++ case I915_PARAM_HAS_SEMAPHORES: ++ value = !!(dev_priv->caps.scheduler & I915_SCHEDULER_CAP_SEMAPHORES); ++ break; ++ case I915_PARAM_HAS_SECURE_BATCHES: ++ value = capable(CAP_SYS_ADMIN); ++ break; ++ case I915_PARAM_CMD_PARSER_VERSION: ++ value = i915_cmd_parser_get_version(dev_priv); ++ break; ++ case I915_PARAM_SUBSLICE_TOTAL: ++ value = sseu_subslice_total(&RUNTIME_INFO(dev_priv)->sseu); ++ if (!value) ++ return -ENODEV; ++ break; ++ case I915_PARAM_EU_TOTAL: ++ value = RUNTIME_INFO(dev_priv)->sseu.eu_total; ++ if (!value) ++ return -ENODEV; ++ break; ++ case I915_PARAM_HAS_GPU_RESET: ++ value = i915_modparams.enable_hangcheck && ++ intel_has_gpu_reset(dev_priv); ++ if (value && intel_has_reset_engine(dev_priv)) ++ value = 2; ++ break; ++ case I915_PARAM_HAS_RESOURCE_STREAMER: ++ value = 0; ++ break; ++ case I915_PARAM_HAS_POOLED_EU: ++ value = HAS_POOLED_EU(dev_priv); ++ break; ++ case I915_PARAM_MIN_EU_IN_POOL: ++ value = RUNTIME_INFO(dev_priv)->sseu.min_eu_in_pool; ++ break; ++ case I915_PARAM_HUC_STATUS: ++ value = intel_huc_check_status(&dev_priv->huc); ++ if (value < 0) ++ return value; ++ break; ++ case I915_PARAM_MMAP_GTT_VERSION: ++ /* Though we've started our numbering from 1, and so class all ++ * earlier versions as 0, in effect their value is undefined as ++ * the ioctl will report EINVAL for the unknown param! ++ */ ++ value = i915_gem_mmap_gtt_version(); ++ break; ++ case I915_PARAM_HAS_SCHEDULER: ++ value = dev_priv->caps.scheduler; ++ break; ++ ++ case I915_PARAM_MMAP_VERSION: ++ /* Remember to bump this if the version changes! */ ++ case I915_PARAM_HAS_GEM: ++ case I915_PARAM_HAS_PAGEFLIPPING: ++ case I915_PARAM_HAS_EXECBUF2: /* depends on GEM */ ++ case I915_PARAM_HAS_RELAXED_FENCING: ++ case I915_PARAM_HAS_COHERENT_RINGS: ++ case I915_PARAM_HAS_RELAXED_DELTA: ++ case I915_PARAM_HAS_GEN7_SOL_RESET: ++ case I915_PARAM_HAS_WAIT_TIMEOUT: ++ case I915_PARAM_HAS_PRIME_VMAP_FLUSH: ++ case I915_PARAM_HAS_PINNED_BATCHES: ++ case I915_PARAM_HAS_EXEC_NO_RELOC: ++ case I915_PARAM_HAS_EXEC_HANDLE_LUT: ++ case I915_PARAM_HAS_COHERENT_PHYS_GTT: ++ case I915_PARAM_HAS_EXEC_SOFTPIN: ++ case I915_PARAM_HAS_EXEC_ASYNC: ++ case I915_PARAM_HAS_EXEC_FENCE: ++ case I915_PARAM_HAS_EXEC_CAPTURE: ++ case I915_PARAM_HAS_EXEC_BATCH_FIRST: ++ case I915_PARAM_HAS_EXEC_FENCE_ARRAY: ++ /* For the time being all of these are always true; ++ * if some supported hardware does not have one of these ++ * features this value needs to be provided from ++ * INTEL_INFO(), a feature macro, or similar. ++ */ ++ value = 1; ++ break; ++ case I915_PARAM_HAS_CONTEXT_ISOLATION: ++ value = intel_engines_has_context_isolation(dev_priv); ++ break; ++ case I915_PARAM_SLICE_MASK: ++ value = RUNTIME_INFO(dev_priv)->sseu.slice_mask; ++ if (!value) ++ return -ENODEV; ++ break; ++ case I915_PARAM_SUBSLICE_MASK: ++ value = RUNTIME_INFO(dev_priv)->sseu.subslice_mask[0]; ++ if (!value) ++ return -ENODEV; ++ break; ++ case I915_PARAM_CS_TIMESTAMP_FREQUENCY: ++ value = 1000 * RUNTIME_INFO(dev_priv)->cs_timestamp_frequency_khz; ++ break; ++ case I915_PARAM_MMAP_GTT_COHERENT: ++ value = INTEL_INFO(dev_priv)->has_coherent_ggtt; ++ break; ++ default: ++ DRM_DEBUG("Unknown parameter %d\n", param->param); ++ return -EINVAL; ++ } ++ ++ if (put_user(value, param->value)) ++ return -EFAULT; ++ ++ return 0; ++} ++ ++static int i915_get_bridge_dev(struct drm_i915_private *dev_priv) ++{ ++ int domain = pci_domain_nr(dev_priv->drm.pdev->bus); ++ ++ dev_priv->bridge_dev = ++ pci_get_domain_bus_and_slot(domain, 0, PCI_DEVFN(0, 0)); ++ if (!dev_priv->bridge_dev) { ++ DRM_ERROR("bridge device not found\n"); ++ return -1; ++ } ++ return 0; ++} ++ ++/* Allocate space for the MCH regs if needed, return nonzero on error */ ++static int ++intel_alloc_mchbar_resource(struct drm_i915_private *dev_priv) ++{ ++ int reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915; ++ u32 temp_lo, temp_hi = 0; ++ u64 mchbar_addr; ++ int ret; ++ ++ if (INTEL_GEN(dev_priv) >= 4) ++ pci_read_config_dword(dev_priv->bridge_dev, reg + 4, &temp_hi); ++ pci_read_config_dword(dev_priv->bridge_dev, reg, &temp_lo); ++ mchbar_addr = ((u64)temp_hi << 32) | temp_lo; ++ ++ /* If ACPI doesn't have it, assume we need to allocate it ourselves */ ++#ifdef CONFIG_PNP ++ if (mchbar_addr && ++ pnp_range_reserved(mchbar_addr, mchbar_addr + MCHBAR_SIZE)) ++ return 0; ++#endif ++ ++ /* Get some space for it */ ++ dev_priv->mch_res.name = "i915 MCHBAR"; ++ dev_priv->mch_res.flags = IORESOURCE_MEM; ++ ret = pci_bus_alloc_resource(dev_priv->bridge_dev->bus, ++ &dev_priv->mch_res, ++ MCHBAR_SIZE, MCHBAR_SIZE, ++ PCIBIOS_MIN_MEM, ++ 0, pcibios_align_resource, ++ dev_priv->bridge_dev); ++ if (ret) { ++ DRM_DEBUG_DRIVER("failed bus alloc: %d\n", ret); ++ dev_priv->mch_res.start = 0; ++ return ret; ++ } ++ ++ if (INTEL_GEN(dev_priv) >= 4) ++ pci_write_config_dword(dev_priv->bridge_dev, reg + 4, ++ upper_32_bits(dev_priv->mch_res.start)); ++ ++ pci_write_config_dword(dev_priv->bridge_dev, reg, ++ lower_32_bits(dev_priv->mch_res.start)); ++ return 0; ++} ++ ++/* Setup MCHBAR if possible, return true if we should disable it again */ ++static void ++intel_setup_mchbar(struct drm_i915_private *dev_priv) ++{ ++ int mchbar_reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915; ++ u32 temp; ++ bool enabled; ++ ++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ return; ++ ++ dev_priv->mchbar_need_disable = false; ++ ++ if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) { ++ pci_read_config_dword(dev_priv->bridge_dev, DEVEN, &temp); ++ enabled = !!(temp & DEVEN_MCHBAR_EN); ++ } else { ++ pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp); ++ enabled = temp & 1; ++ } ++ ++ /* If it's already enabled, don't have to do anything */ ++ if (enabled) ++ return; ++ ++ if (intel_alloc_mchbar_resource(dev_priv)) ++ return; ++ ++ dev_priv->mchbar_need_disable = true; ++ ++ /* Space is allocated or reserved, so enable it. */ ++ if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) { ++ pci_write_config_dword(dev_priv->bridge_dev, DEVEN, ++ temp | DEVEN_MCHBAR_EN); ++ } else { ++ pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp); ++ pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, temp | 1); ++ } ++} ++ ++static void ++intel_teardown_mchbar(struct drm_i915_private *dev_priv) ++{ ++ int mchbar_reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915; ++ ++ if (dev_priv->mchbar_need_disable) { ++ if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) { ++ u32 deven_val; ++ ++ pci_read_config_dword(dev_priv->bridge_dev, DEVEN, ++ &deven_val); ++ deven_val &= ~DEVEN_MCHBAR_EN; ++ pci_write_config_dword(dev_priv->bridge_dev, DEVEN, ++ deven_val); ++ } else { ++ u32 mchbar_val; ++ ++ pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, ++ &mchbar_val); ++ mchbar_val &= ~1; ++ pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, ++ mchbar_val); ++ } ++ } ++ ++ if (dev_priv->mch_res.start) ++ release_resource(&dev_priv->mch_res); ++} ++ ++/* true = enable decode, false = disable decoder */ ++static unsigned int i915_vga_set_decode(void *cookie, bool state) ++{ ++ struct drm_i915_private *dev_priv = cookie; ++ ++ intel_modeset_vga_set_state(dev_priv, state); ++ if (state) ++ return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM | ++ VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; ++ else ++ return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; ++} ++ ++static int i915_resume_switcheroo(struct drm_device *dev); ++static int i915_suspend_switcheroo(struct drm_device *dev, pm_message_t state); ++ ++static void i915_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state) ++{ ++ struct drm_device *dev = pci_get_drvdata(pdev); ++ pm_message_t pmm = { .event = PM_EVENT_SUSPEND }; ++ ++ if (state == VGA_SWITCHEROO_ON) { ++ pr_info("switched on\n"); ++ dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; ++ /* i915 resume handler doesn't set to D0 */ ++ pci_set_power_state(pdev, PCI_D0); ++ i915_resume_switcheroo(dev); ++ dev->switch_power_state = DRM_SWITCH_POWER_ON; ++ } else { ++ pr_info("switched off\n"); ++ dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; ++ i915_suspend_switcheroo(dev, pmm); ++ dev->switch_power_state = DRM_SWITCH_POWER_OFF; ++ } ++} ++ ++static bool i915_switcheroo_can_switch(struct pci_dev *pdev) ++{ ++ struct drm_device *dev = pci_get_drvdata(pdev); ++ ++ /* ++ * FIXME: open_count is protected by drm_global_mutex but that would lead to ++ * locking inversion with the driver load path. And the access here is ++ * completely racy anyway. So don't bother with locking for now. ++ */ ++ return dev->open_count == 0; ++} ++ ++static const struct vga_switcheroo_client_ops i915_switcheroo_ops = { ++ .set_gpu_state = i915_switcheroo_set_state, ++ .reprobe = NULL, ++ .can_switch = i915_switcheroo_can_switch, ++}; ++ ++static int i915_load_modeset_init(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ int ret; ++ ++ if (i915_inject_load_failure()) ++ return -ENODEV; ++ ++ if (HAS_DISPLAY(dev_priv)) { ++ ret = drm_vblank_init(&dev_priv->drm, ++ INTEL_INFO(dev_priv)->num_pipes); ++ if (ret) ++ goto out; ++ } ++ ++ intel_bios_init(dev_priv); ++ ++ /* If we have > 1 VGA cards, then we need to arbitrate access ++ * to the common VGA resources. ++ * ++ * If we are a secondary display controller (!PCI_DISPLAY_CLASS_VGA), ++ * then we do not take part in VGA arbitration and the ++ * vga_client_register() fails with -ENODEV. ++ */ ++ ret = vga_client_register(pdev, dev_priv, NULL, i915_vga_set_decode); ++ if (ret && ret != -ENODEV) ++ goto out; ++ ++ intel_register_dsm_handler(); ++ ++ ret = vga_switcheroo_register_client(pdev, &i915_switcheroo_ops, false); ++ if (ret) ++ goto cleanup_vga_client; ++ ++ /* must happen before intel_power_domains_init_hw() on VLV/CHV */ ++ intel_update_rawclk(dev_priv); ++ ++ intel_power_domains_init_hw(dev_priv, false); ++ ++ intel_csr_ucode_init(dev_priv); ++ ++ ret = intel_irq_install(dev_priv); ++ if (ret) ++ goto cleanup_csr; ++ ++ intel_setup_gmbus(dev_priv); ++ ++ /* Important: The output setup functions called by modeset_init need ++ * working irqs for e.g. gmbus and dp aux transfers. */ ++ ret = intel_modeset_init(dev); ++ if (ret) ++ goto cleanup_irq; ++ ++ ret = i915_gem_init(dev_priv); ++ if (ret) ++ goto cleanup_modeset; ++ ++ intel_overlay_setup(dev_priv); ++ ++ if (!HAS_DISPLAY(dev_priv)) ++ return 0; ++ ++ ret = intel_fbdev_init(dev); ++ if (ret) ++ goto cleanup_gem; ++ ++ /* Only enable hotplug handling once the fbdev is fully set up. */ ++ intel_hpd_init(dev_priv); ++ ++ intel_init_ipc(dev_priv); ++ ++ return 0; ++ ++cleanup_gem: ++ i915_gem_suspend(dev_priv); ++ i915_gem_fini(dev_priv); ++cleanup_modeset: ++ intel_modeset_cleanup(dev); ++cleanup_irq: ++ drm_irq_uninstall(dev); ++ intel_teardown_gmbus(dev_priv); ++cleanup_csr: ++ intel_csr_ucode_fini(dev_priv); ++ intel_power_domains_fini_hw(dev_priv); ++ vga_switcheroo_unregister_client(pdev); ++cleanup_vga_client: ++ vga_client_register(pdev, NULL, NULL, NULL); ++out: ++ return ret; ++} ++ ++static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv) ++{ ++ struct apertures_struct *ap; ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ struct i915_ggtt *ggtt = &dev_priv->ggtt; ++ bool primary; ++ int ret; ++ ++ ap = alloc_apertures(1); ++ if (!ap) ++ return -ENOMEM; ++ ++ ap->ranges[0].base = ggtt->gmadr.start; ++ ap->ranges[0].size = ggtt->mappable_end; ++ ++ primary = ++ pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW; ++ ++ ret = drm_fb_helper_remove_conflicting_framebuffers(ap, "inteldrmfb", primary); ++ ++ kfree(ap); ++ ++ return ret; ++} ++ ++static void intel_init_dpio(struct drm_i915_private *dev_priv) ++{ ++ /* ++ * IOSF_PORT_DPIO is used for VLV x2 PHY (DP/HDMI B and C), ++ * CHV x1 PHY (DP/HDMI D) ++ * IOSF_PORT_DPIO_2 is used for CHV x2 PHY (DP/HDMI B and C) ++ */ ++ if (IS_CHERRYVIEW(dev_priv)) { ++ DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO_2; ++ DPIO_PHY_IOSF_PORT(DPIO_PHY1) = IOSF_PORT_DPIO; ++ } else if (IS_VALLEYVIEW(dev_priv)) { ++ DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO; ++ } ++} ++ ++static int i915_workqueues_init(struct drm_i915_private *dev_priv) ++{ ++ /* ++ * The i915 workqueue is primarily used for batched retirement of ++ * requests (and thus managing bo) once the task has been completed ++ * by the GPU. i915_retire_requests() is called directly when we ++ * need high-priority retirement, such as waiting for an explicit ++ * bo. ++ * ++ * It is also used for periodic low-priority events, such as ++ * idle-timers and recording error state. ++ * ++ * All tasks on the workqueue are expected to acquire the dev mutex ++ * so there is no point in running more than one instance of the ++ * workqueue at any time. Use an ordered one. ++ */ ++ dev_priv->wq = alloc_ordered_workqueue("i915", 0); ++ if (dev_priv->wq == NULL) ++ goto out_err; ++ ++ dev_priv->hotplug.dp_wq = alloc_ordered_workqueue("i915-dp", 0); ++ if (dev_priv->hotplug.dp_wq == NULL) ++ goto out_free_wq; ++ ++ return 0; ++ ++out_free_wq: ++ destroy_workqueue(dev_priv->wq); ++out_err: ++ DRM_ERROR("Failed to allocate workqueues.\n"); ++ ++ return -ENOMEM; ++} ++ ++static void i915_engines_cleanup(struct drm_i915_private *i915) ++{ ++ struct intel_engine_cs *engine; ++ enum intel_engine_id id; ++ ++ for_each_engine(engine, i915, id) ++ kfree(engine); ++} ++ ++static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv) ++{ ++ destroy_workqueue(dev_priv->hotplug.dp_wq); ++ destroy_workqueue(dev_priv->wq); ++} ++ ++/* ++ * We don't keep the workarounds for pre-production hardware, so we expect our ++ * driver to fail on these machines in one way or another. A little warning on ++ * dmesg may help both the user and the bug triagers. ++ * ++ * Our policy for removing pre-production workarounds is to keep the ++ * current gen workarounds as a guide to the bring-up of the next gen ++ * (workarounds have a habit of persisting!). Anything older than that ++ * should be removed along with the complications they introduce. ++ */ ++static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv) ++{ ++ bool pre = false; ++ ++ pre |= IS_HSW_EARLY_SDV(dev_priv); ++ pre |= IS_SKL_REVID(dev_priv, 0, SKL_REVID_F0); ++ pre |= IS_BXT_REVID(dev_priv, 0, BXT_REVID_B_LAST); ++ pre |= IS_KBL_REVID(dev_priv, 0, KBL_REVID_A0); ++ ++ if (pre) { ++ DRM_ERROR("This is a pre-production stepping. " ++ "It may not be fully functional.\n"); ++ add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK); ++ } ++} ++ ++/** ++ * i915_driver_init_early - setup state not requiring device access ++ * @dev_priv: device private ++ * ++ * Initialize everything that is a "SW-only" state, that is state not ++ * requiring accessing the device or exposing the driver via kernel internal ++ * or userspace interfaces. Example steps belonging here: lock initialization, ++ * system memory allocation, setting up device specific attributes and ++ * function hooks not requiring accessing the device. ++ */ ++static int i915_driver_init_early(struct drm_i915_private *dev_priv) ++{ ++ int ret = 0; ++ ++ if (i915_inject_load_failure()) ++ return -ENODEV; ++ ++ intel_device_info_subplatform_init(dev_priv); ++ ++ intel_uncore_init_early(&dev_priv->uncore); ++ ++ spin_lock_init(&dev_priv->irq_lock); ++ spin_lock_init(&dev_priv->gpu_error.lock); ++ mutex_init(&dev_priv->backlight_lock); ++ ++ mutex_init(&dev_priv->sb_lock); ++ mutex_init(&dev_priv->av_mutex); ++ mutex_init(&dev_priv->wm.wm_mutex); ++ mutex_init(&dev_priv->pps_mutex); ++ mutex_init(&dev_priv->hdcp_comp_mutex); ++ ++ i915_memcpy_init_early(dev_priv); ++ intel_runtime_pm_init_early(dev_priv); ++ ++ ret = i915_workqueues_init(dev_priv); ++ if (ret < 0) ++ goto err_engines; ++ ++ ret = i915_gem_init_early(dev_priv); ++ if (ret < 0) ++ goto err_workqueues; ++ ++ /* This must be called before any calls to HAS_PCH_* */ ++ intel_detect_pch(dev_priv); ++ ++ intel_wopcm_init_early(&dev_priv->wopcm); ++ intel_uc_init_early(dev_priv); ++ intel_pm_setup(dev_priv); ++ intel_init_dpio(dev_priv); ++ ret = intel_power_domains_init(dev_priv); ++ if (ret < 0) ++ goto err_uc; ++ intel_irq_init(dev_priv); ++ intel_hangcheck_init(dev_priv); ++ intel_init_display_hooks(dev_priv); ++ intel_init_clock_gating_hooks(dev_priv); ++ intel_init_audio_hooks(dev_priv); ++ intel_display_crc_init(dev_priv); ++ ++ intel_detect_preproduction_hw(dev_priv); ++ ++ return 0; ++ ++err_uc: ++ intel_uc_cleanup_early(dev_priv); ++ i915_gem_cleanup_early(dev_priv); ++err_workqueues: ++ i915_workqueues_cleanup(dev_priv); ++err_engines: ++ i915_engines_cleanup(dev_priv); ++ return ret; ++} ++ ++/** ++ * i915_driver_cleanup_early - cleanup the setup done in i915_driver_init_early() ++ * @dev_priv: device private ++ */ ++static void i915_driver_cleanup_early(struct drm_i915_private *dev_priv) ++{ ++ intel_irq_fini(dev_priv); ++ intel_power_domains_cleanup(dev_priv); ++ intel_uc_cleanup_early(dev_priv); ++ i915_gem_cleanup_early(dev_priv); ++ i915_workqueues_cleanup(dev_priv); ++ i915_engines_cleanup(dev_priv); ++} ++ ++/** ++ * i915_driver_init_mmio - setup device MMIO ++ * @dev_priv: device private ++ * ++ * Setup minimal device state necessary for MMIO accesses later in the ++ * initialization sequence. The setup here should avoid any other device-wide ++ * side effects or exposing the driver via kernel internal or user space ++ * interfaces. ++ */ ++static int i915_driver_init_mmio(struct drm_i915_private *dev_priv) ++{ ++ int ret; ++ ++ if (i915_inject_load_failure()) ++ return -ENODEV; ++ ++ if (i915_get_bridge_dev(dev_priv)) ++ return -EIO; ++ ++ ret = intel_uncore_init_mmio(&dev_priv->uncore); ++ if (ret < 0) ++ goto err_bridge; ++ ++ /* Try to make sure MCHBAR is enabled before poking at it */ ++ intel_setup_mchbar(dev_priv); ++ ++ intel_device_info_init_mmio(dev_priv); ++ ++ intel_uncore_prune_mmio_domains(&dev_priv->uncore); ++ ++ intel_uc_init_mmio(dev_priv); ++ ++ ret = intel_engines_init_mmio(dev_priv); ++ if (ret) ++ goto err_uncore; ++ ++ i915_gem_init_mmio(dev_priv); ++ ++ return 0; ++ ++err_uncore: ++ intel_teardown_mchbar(dev_priv); ++ intel_uncore_fini_mmio(&dev_priv->uncore); ++err_bridge: ++ pci_dev_put(dev_priv->bridge_dev); ++ ++ return ret; ++} ++ ++/** ++ * i915_driver_cleanup_mmio - cleanup the setup done in i915_driver_init_mmio() ++ * @dev_priv: device private ++ */ ++static void i915_driver_cleanup_mmio(struct drm_i915_private *dev_priv) ++{ ++ intel_teardown_mchbar(dev_priv); ++ intel_uncore_fini_mmio(&dev_priv->uncore); ++ pci_dev_put(dev_priv->bridge_dev); ++} ++ ++static void intel_sanitize_options(struct drm_i915_private *dev_priv) ++{ ++ intel_gvt_sanitize_options(dev_priv); ++} ++ ++#define DRAM_TYPE_STR(type) [INTEL_DRAM_ ## type] = #type ++ ++static const char *intel_dram_type_str(enum intel_dram_type type) ++{ ++ static const char * const str[] = { ++ DRAM_TYPE_STR(UNKNOWN), ++ DRAM_TYPE_STR(DDR3), ++ DRAM_TYPE_STR(DDR4), ++ DRAM_TYPE_STR(LPDDR3), ++ DRAM_TYPE_STR(LPDDR4), ++ }; ++ ++ if (type >= ARRAY_SIZE(str)) ++ type = INTEL_DRAM_UNKNOWN; ++ ++ return str[type]; ++} ++ ++#undef DRAM_TYPE_STR ++ ++static int intel_dimm_num_devices(const struct dram_dimm_info *dimm) ++{ ++ return dimm->ranks * 64 / (dimm->width ?: 1); ++} ++ ++/* Returns total GB for the whole DIMM */ ++static int skl_get_dimm_size(u16 val) ++{ ++ return val & SKL_DRAM_SIZE_MASK; ++} ++ ++static int skl_get_dimm_width(u16 val) ++{ ++ if (skl_get_dimm_size(val) == 0) ++ return 0; ++ ++ switch (val & SKL_DRAM_WIDTH_MASK) { ++ case SKL_DRAM_WIDTH_X8: ++ case SKL_DRAM_WIDTH_X16: ++ case SKL_DRAM_WIDTH_X32: ++ val = (val & SKL_DRAM_WIDTH_MASK) >> SKL_DRAM_WIDTH_SHIFT; ++ return 8 << val; ++ default: ++ MISSING_CASE(val); ++ return 0; ++ } ++} ++ ++static int skl_get_dimm_ranks(u16 val) ++{ ++ if (skl_get_dimm_size(val) == 0) ++ return 0; ++ ++ val = (val & SKL_DRAM_RANK_MASK) >> SKL_DRAM_RANK_SHIFT; ++ ++ return val + 1; ++} ++ ++/* Returns total GB for the whole DIMM */ ++static int cnl_get_dimm_size(u16 val) ++{ ++ return (val & CNL_DRAM_SIZE_MASK) / 2; ++} ++ ++static int cnl_get_dimm_width(u16 val) ++{ ++ if (cnl_get_dimm_size(val) == 0) ++ return 0; ++ ++ switch (val & CNL_DRAM_WIDTH_MASK) { ++ case CNL_DRAM_WIDTH_X8: ++ case CNL_DRAM_WIDTH_X16: ++ case CNL_DRAM_WIDTH_X32: ++ val = (val & CNL_DRAM_WIDTH_MASK) >> CNL_DRAM_WIDTH_SHIFT; ++ return 8 << val; ++ default: ++ MISSING_CASE(val); ++ return 0; ++ } ++} ++ ++static int cnl_get_dimm_ranks(u16 val) ++{ ++ if (cnl_get_dimm_size(val) == 0) ++ return 0; ++ ++ val = (val & CNL_DRAM_RANK_MASK) >> CNL_DRAM_RANK_SHIFT; ++ ++ return val + 1; ++} ++ ++static bool ++skl_is_16gb_dimm(const struct dram_dimm_info *dimm) ++{ ++ /* Convert total GB to Gb per DRAM device */ ++ return 8 * dimm->size / (intel_dimm_num_devices(dimm) ?: 1) == 16; ++} ++ ++static void ++skl_dram_get_dimm_info(struct drm_i915_private *dev_priv, ++ struct dram_dimm_info *dimm, ++ int channel, char dimm_name, u16 val) ++{ ++ if (INTEL_GEN(dev_priv) >= 10) { ++ dimm->size = cnl_get_dimm_size(val); ++ dimm->width = cnl_get_dimm_width(val); ++ dimm->ranks = cnl_get_dimm_ranks(val); ++ } else { ++ dimm->size = skl_get_dimm_size(val); ++ dimm->width = skl_get_dimm_width(val); ++ dimm->ranks = skl_get_dimm_ranks(val); ++ } ++ ++ DRM_DEBUG_KMS("CH%u DIMM %c size: %u GB, width: X%u, ranks: %u, 16Gb DIMMs: %s\n", ++ channel, dimm_name, dimm->size, dimm->width, dimm->ranks, ++ yesno(skl_is_16gb_dimm(dimm))); ++} ++ ++static int ++skl_dram_get_channel_info(struct drm_i915_private *dev_priv, ++ struct dram_channel_info *ch, ++ int channel, u32 val) ++{ ++ skl_dram_get_dimm_info(dev_priv, &ch->dimm_l, ++ channel, 'L', val & 0xffff); ++ skl_dram_get_dimm_info(dev_priv, &ch->dimm_s, ++ channel, 'S', val >> 16); ++ ++ if (ch->dimm_l.size == 0 && ch->dimm_s.size == 0) { ++ DRM_DEBUG_KMS("CH%u not populated\n", channel); ++ return -EINVAL; ++ } ++ ++ if (ch->dimm_l.ranks == 2 || ch->dimm_s.ranks == 2) ++ ch->ranks = 2; ++ else if (ch->dimm_l.ranks == 1 && ch->dimm_s.ranks == 1) ++ ch->ranks = 2; ++ else ++ ch->ranks = 1; ++ ++ ch->is_16gb_dimm = ++ skl_is_16gb_dimm(&ch->dimm_l) || ++ skl_is_16gb_dimm(&ch->dimm_s); ++ ++ DRM_DEBUG_KMS("CH%u ranks: %u, 16Gb DIMMs: %s\n", ++ channel, ch->ranks, yesno(ch->is_16gb_dimm)); ++ ++ return 0; ++} ++ ++static bool ++intel_is_dram_symmetric(const struct dram_channel_info *ch0, ++ const struct dram_channel_info *ch1) ++{ ++ return !memcmp(ch0, ch1, sizeof(*ch0)) && ++ (ch0->dimm_s.size == 0 || ++ !memcmp(&ch0->dimm_l, &ch0->dimm_s, sizeof(ch0->dimm_l))); ++} ++ ++static int ++skl_dram_get_channels_info(struct drm_i915_private *dev_priv) ++{ ++ struct dram_info *dram_info = &dev_priv->dram_info; ++ struct dram_channel_info ch0 = {}, ch1 = {}; ++ u32 val; ++ int ret; ++ ++ val = I915_READ(SKL_MAD_DIMM_CH0_0_0_0_MCHBAR_MCMAIN); ++ ret = skl_dram_get_channel_info(dev_priv, &ch0, 0, val); ++ if (ret == 0) ++ dram_info->num_channels++; ++ ++ val = I915_READ(SKL_MAD_DIMM_CH1_0_0_0_MCHBAR_MCMAIN); ++ ret = skl_dram_get_channel_info(dev_priv, &ch1, 1, val); ++ if (ret == 0) ++ dram_info->num_channels++; ++ ++ if (dram_info->num_channels == 0) { ++ DRM_INFO("Number of memory channels is zero\n"); ++ return -EINVAL; ++ } ++ ++ /* ++ * If any of the channel is single rank channel, worst case output ++ * will be same as if single rank memory, so consider single rank ++ * memory. ++ */ ++ if (ch0.ranks == 1 || ch1.ranks == 1) ++ dram_info->ranks = 1; ++ else ++ dram_info->ranks = max(ch0.ranks, ch1.ranks); ++ ++ if (dram_info->ranks == 0) { ++ DRM_INFO("couldn't get memory rank information\n"); ++ return -EINVAL; ++ } ++ ++ dram_info->is_16gb_dimm = ch0.is_16gb_dimm || ch1.is_16gb_dimm; ++ ++ dram_info->symmetric_memory = intel_is_dram_symmetric(&ch0, &ch1); ++ ++ DRM_DEBUG_KMS("Memory configuration is symmetric? %s\n", ++ yesno(dram_info->symmetric_memory)); ++ return 0; ++} ++ ++static enum intel_dram_type ++skl_get_dram_type(struct drm_i915_private *dev_priv) ++{ ++ u32 val; ++ ++ val = I915_READ(SKL_MAD_INTER_CHANNEL_0_0_0_MCHBAR_MCMAIN); ++ ++ switch (val & SKL_DRAM_DDR_TYPE_MASK) { ++ case SKL_DRAM_DDR_TYPE_DDR3: ++ return INTEL_DRAM_DDR3; ++ case SKL_DRAM_DDR_TYPE_DDR4: ++ return INTEL_DRAM_DDR4; ++ case SKL_DRAM_DDR_TYPE_LPDDR3: ++ return INTEL_DRAM_LPDDR3; ++ case SKL_DRAM_DDR_TYPE_LPDDR4: ++ return INTEL_DRAM_LPDDR4; ++ default: ++ MISSING_CASE(val); ++ return INTEL_DRAM_UNKNOWN; ++ } ++} ++ ++static int ++skl_get_dram_info(struct drm_i915_private *dev_priv) ++{ ++ struct dram_info *dram_info = &dev_priv->dram_info; ++ u32 mem_freq_khz, val; ++ int ret; ++ ++ dram_info->type = skl_get_dram_type(dev_priv); ++ DRM_DEBUG_KMS("DRAM type: %s\n", intel_dram_type_str(dram_info->type)); ++ ++ ret = skl_dram_get_channels_info(dev_priv); ++ if (ret) ++ return ret; ++ ++ val = I915_READ(SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU); ++ mem_freq_khz = DIV_ROUND_UP((val & SKL_REQ_DATA_MASK) * ++ SKL_MEMORY_FREQ_MULTIPLIER_HZ, 1000); ++ ++ dram_info->bandwidth_kbps = dram_info->num_channels * ++ mem_freq_khz * 8; ++ ++ if (dram_info->bandwidth_kbps == 0) { ++ DRM_INFO("Couldn't get system memory bandwidth\n"); ++ return -EINVAL; ++ } ++ ++ dram_info->valid = true; ++ return 0; ++} ++ ++/* Returns Gb per DRAM device */ ++static int bxt_get_dimm_size(u32 val) ++{ ++ switch (val & BXT_DRAM_SIZE_MASK) { ++ case BXT_DRAM_SIZE_4GBIT: ++ return 4; ++ case BXT_DRAM_SIZE_6GBIT: ++ return 6; ++ case BXT_DRAM_SIZE_8GBIT: ++ return 8; ++ case BXT_DRAM_SIZE_12GBIT: ++ return 12; ++ case BXT_DRAM_SIZE_16GBIT: ++ return 16; ++ default: ++ MISSING_CASE(val); ++ return 0; ++ } ++} ++ ++static int bxt_get_dimm_width(u32 val) ++{ ++ if (!bxt_get_dimm_size(val)) ++ return 0; ++ ++ val = (val & BXT_DRAM_WIDTH_MASK) >> BXT_DRAM_WIDTH_SHIFT; ++ ++ return 8 << val; ++} ++ ++static int bxt_get_dimm_ranks(u32 val) ++{ ++ if (!bxt_get_dimm_size(val)) ++ return 0; ++ ++ switch (val & BXT_DRAM_RANK_MASK) { ++ case BXT_DRAM_RANK_SINGLE: ++ return 1; ++ case BXT_DRAM_RANK_DUAL: ++ return 2; ++ default: ++ MISSING_CASE(val); ++ return 0; ++ } ++} ++ ++static enum intel_dram_type bxt_get_dimm_type(u32 val) ++{ ++ if (!bxt_get_dimm_size(val)) ++ return INTEL_DRAM_UNKNOWN; ++ ++ switch (val & BXT_DRAM_TYPE_MASK) { ++ case BXT_DRAM_TYPE_DDR3: ++ return INTEL_DRAM_DDR3; ++ case BXT_DRAM_TYPE_LPDDR3: ++ return INTEL_DRAM_LPDDR3; ++ case BXT_DRAM_TYPE_DDR4: ++ return INTEL_DRAM_DDR4; ++ case BXT_DRAM_TYPE_LPDDR4: ++ return INTEL_DRAM_LPDDR4; ++ default: ++ MISSING_CASE(val); ++ return INTEL_DRAM_UNKNOWN; ++ } ++} ++ ++static void bxt_get_dimm_info(struct dram_dimm_info *dimm, ++ u32 val) ++{ ++ dimm->width = bxt_get_dimm_width(val); ++ dimm->ranks = bxt_get_dimm_ranks(val); ++ ++ /* ++ * Size in register is Gb per DRAM device. Convert to total ++ * GB to match the way we report this for non-LP platforms. ++ */ ++ dimm->size = bxt_get_dimm_size(val) * intel_dimm_num_devices(dimm) / 8; ++} ++ ++static int ++bxt_get_dram_info(struct drm_i915_private *dev_priv) ++{ ++ struct dram_info *dram_info = &dev_priv->dram_info; ++ u32 dram_channels; ++ u32 mem_freq_khz, val; ++ u8 num_active_channels; ++ int i; ++ ++ val = I915_READ(BXT_P_CR_MC_BIOS_REQ_0_0_0); ++ mem_freq_khz = DIV_ROUND_UP((val & BXT_REQ_DATA_MASK) * ++ BXT_MEMORY_FREQ_MULTIPLIER_HZ, 1000); ++ ++ dram_channels = val & BXT_DRAM_CHANNEL_ACTIVE_MASK; ++ num_active_channels = hweight32(dram_channels); ++ ++ /* Each active bit represents 4-byte channel */ ++ dram_info->bandwidth_kbps = (mem_freq_khz * num_active_channels * 4); ++ ++ if (dram_info->bandwidth_kbps == 0) { ++ DRM_INFO("Couldn't get system memory bandwidth\n"); ++ return -EINVAL; ++ } ++ ++ /* ++ * Now read each DUNIT8/9/10/11 to check the rank of each dimms. ++ */ ++ for (i = BXT_D_CR_DRP0_DUNIT_START; i <= BXT_D_CR_DRP0_DUNIT_END; i++) { ++ struct dram_dimm_info dimm; ++ enum intel_dram_type type; ++ ++ val = I915_READ(BXT_D_CR_DRP0_DUNIT(i)); ++ if (val == 0xFFFFFFFF) ++ continue; ++ ++ dram_info->num_channels++; ++ ++ bxt_get_dimm_info(&dimm, val); ++ type = bxt_get_dimm_type(val); ++ ++ WARN_ON(type != INTEL_DRAM_UNKNOWN && ++ dram_info->type != INTEL_DRAM_UNKNOWN && ++ dram_info->type != type); ++ ++ DRM_DEBUG_KMS("CH%u DIMM size: %u GB, width: X%u, ranks: %u, type: %s\n", ++ i - BXT_D_CR_DRP0_DUNIT_START, ++ dimm.size, dimm.width, dimm.ranks, ++ intel_dram_type_str(type)); ++ ++ /* ++ * If any of the channel is single rank channel, ++ * worst case output will be same as if single rank ++ * memory, so consider single rank memory. ++ */ ++ if (dram_info->ranks == 0) ++ dram_info->ranks = dimm.ranks; ++ else if (dimm.ranks == 1) ++ dram_info->ranks = 1; ++ ++ if (type != INTEL_DRAM_UNKNOWN) ++ dram_info->type = type; ++ } ++ ++ if (dram_info->type == INTEL_DRAM_UNKNOWN || ++ dram_info->ranks == 0) { ++ DRM_INFO("couldn't get memory information\n"); ++ return -EINVAL; ++ } ++ ++ dram_info->valid = true; ++ return 0; ++} ++ ++static void ++intel_get_dram_info(struct drm_i915_private *dev_priv) ++{ ++ struct dram_info *dram_info = &dev_priv->dram_info; ++ int ret; ++ ++ /* ++ * Assume 16Gb DIMMs are present until proven otherwise. ++ * This is only used for the level 0 watermark latency ++ * w/a which does not apply to bxt/glk. ++ */ ++ dram_info->is_16gb_dimm = !IS_GEN9_LP(dev_priv); ++ ++ if (INTEL_GEN(dev_priv) < 9) ++ return; ++ ++ if (IS_GEN9_LP(dev_priv)) ++ ret = bxt_get_dram_info(dev_priv); ++ else ++ ret = skl_get_dram_info(dev_priv); ++ if (ret) ++ return; ++ ++ DRM_DEBUG_KMS("DRAM bandwidth: %u kBps, channels: %u\n", ++ dram_info->bandwidth_kbps, ++ dram_info->num_channels); ++ ++ DRM_DEBUG_KMS("DRAM ranks: %u, 16Gb DIMMs: %s\n", ++ dram_info->ranks, yesno(dram_info->is_16gb_dimm)); ++} ++ ++static u32 gen9_edram_size_mb(struct drm_i915_private *dev_priv, u32 cap) ++{ ++ const unsigned int ways[8] = { 4, 8, 12, 16, 16, 16, 16, 16 }; ++ const unsigned int sets[4] = { 1, 1, 2, 2 }; ++ ++ return EDRAM_NUM_BANKS(cap) * ++ ways[EDRAM_WAYS_IDX(cap)] * ++ sets[EDRAM_SETS_IDX(cap)]; ++} ++ ++static void edram_detect(struct drm_i915_private *dev_priv) ++{ ++ u32 edram_cap = 0; ++ ++ if (!(IS_HASWELL(dev_priv) || ++ IS_BROADWELL(dev_priv) || ++ INTEL_GEN(dev_priv) >= 9)) ++ return; ++ ++ edram_cap = __raw_uncore_read32(&dev_priv->uncore, HSW_EDRAM_CAP); ++ ++ /* NB: We can't write IDICR yet because we don't have gt funcs set up */ ++ ++ if (!(edram_cap & EDRAM_ENABLED)) ++ return; ++ ++ /* ++ * The needed capability bits for size calculation are not there with ++ * pre gen9 so return 128MB always. ++ */ ++ if (INTEL_GEN(dev_priv) < 9) ++ dev_priv->edram_size_mb = 128; ++ else ++ dev_priv->edram_size_mb = ++ gen9_edram_size_mb(dev_priv, edram_cap); ++ ++ DRM_INFO("Found %uMB of eDRAM\n", dev_priv->edram_size_mb); ++} ++ ++/** ++ * i915_driver_init_hw - setup state requiring device access ++ * @dev_priv: device private ++ * ++ * Setup state that requires accessing the device, but doesn't require ++ * exposing the driver via kernel internal or userspace interfaces. ++ */ ++static int i915_driver_init_hw(struct drm_i915_private *dev_priv) ++{ ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ int ret; ++ ++ if (i915_inject_load_failure()) ++ return -ENODEV; ++ ++ intel_device_info_runtime_init(dev_priv); ++ ++ if (HAS_PPGTT(dev_priv)) { ++ if (intel_vgpu_active(dev_priv) && ++ !intel_vgpu_has_full_ppgtt(dev_priv)) { ++ i915_report_error(dev_priv, ++ "incompatible vGPU found, support for isolated ppGTT required\n"); ++ return -ENXIO; ++ } ++ } ++ ++ if (HAS_EXECLISTS(dev_priv)) { ++ /* ++ * Older GVT emulation depends upon intercepting CSB mmio, ++ * which we no longer use, preferring to use the HWSP cache ++ * instead. ++ */ ++ if (intel_vgpu_active(dev_priv) && ++ !intel_vgpu_has_hwsp_emulation(dev_priv)) { ++ i915_report_error(dev_priv, ++ "old vGPU host found, support for HWSP emulation required\n"); ++ return -ENXIO; ++ } ++ } ++ ++ intel_sanitize_options(dev_priv); ++ ++ /* needs to be done before ggtt probe */ ++ edram_detect(dev_priv); ++ ++ i915_perf_init(dev_priv); ++ ++ ret = i915_ggtt_probe_hw(dev_priv); ++ if (ret) ++ goto err_perf; ++ ++ /* ++ * WARNING: Apparently we must kick fbdev drivers before vgacon, ++ * otherwise the vga fbdev driver falls over. ++ */ ++ ret = i915_kick_out_firmware_fb(dev_priv); ++ if (ret) { ++ DRM_ERROR("failed to remove conflicting framebuffer drivers\n"); ++ goto err_ggtt; ++ } ++ ++ ret = vga_remove_vgacon(pdev); ++ if (ret) { ++ DRM_ERROR("failed to remove conflicting VGA console\n"); ++ goto err_ggtt; ++ } ++ ++ ret = i915_ggtt_init_hw(dev_priv); ++ if (ret) ++ goto err_ggtt; ++ ++ ret = i915_ggtt_enable_hw(dev_priv); ++ if (ret) { ++ DRM_ERROR("failed to enable GGTT\n"); ++ goto err_ggtt; ++ } ++ ++ pci_set_master(pdev); ++ ++ /* ++ * We don't have a max segment size, so set it to the max so sg's ++ * debugging layer doesn't complain ++ */ ++ dma_set_max_seg_size(&pdev->dev, UINT_MAX); ++ ++ /* overlay on gen2 is broken and can't address above 1G */ ++ if (IS_GEN(dev_priv, 2)) { ++ ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(30)); ++ if (ret) { ++ DRM_ERROR("failed to set DMA mask\n"); ++ ++ goto err_ggtt; ++ } ++ } ++ ++ /* 965GM sometimes incorrectly writes to hardware status page (HWS) ++ * using 32bit addressing, overwriting memory if HWS is located ++ * above 4GB. ++ * ++ * The documentation also mentions an issue with undefined ++ * behaviour if any general state is accessed within a page above 4GB, ++ * which also needs to be handled carefully. ++ */ ++ if (IS_I965G(dev_priv) || IS_I965GM(dev_priv)) { ++ ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32)); ++ ++ if (ret) { ++ DRM_ERROR("failed to set DMA mask\n"); ++ ++ goto err_ggtt; ++ } ++ } ++ ++ pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, ++ PM_QOS_DEFAULT_VALUE); ++ ++ intel_uncore_sanitize(dev_priv); ++ ++ intel_gt_init_workarounds(dev_priv); ++ i915_gem_load_init_fences(dev_priv); ++ ++ /* On the 945G/GM, the chipset reports the MSI capability on the ++ * integrated graphics even though the support isn't actually there ++ * according to the published specs. It doesn't appear to function ++ * correctly in testing on 945G. ++ * This may be a side effect of MSI having been made available for PEG ++ * and the registers being closely associated. ++ * ++ * According to chipset errata, on the 965GM, MSI interrupts may ++ * be lost or delayed, and was defeatured. MSI interrupts seem to ++ * get lost on g4x as well, and interrupt delivery seems to stay ++ * properly dead afterwards. So we'll just disable them for all ++ * pre-gen5 chipsets. ++ * ++ * dp aux and gmbus irq on gen4 seems to be able to generate legacy ++ * interrupts even when in MSI mode. This results in spurious ++ * interrupt warnings if the legacy irq no. is shared with another ++ * device. The kernel then disables that interrupt source and so ++ * prevents the other device from working properly. ++ */ ++ if (INTEL_GEN(dev_priv) >= 5) { ++ if (pci_enable_msi(pdev) < 0) ++ DRM_DEBUG_DRIVER("can't enable MSI"); ++ } ++ ++ ret = intel_gvt_init(dev_priv); ++ if (ret) ++ goto err_msi; ++ ++ intel_opregion_setup(dev_priv); ++ /* ++ * Fill the dram structure to get the system raw bandwidth and ++ * dram info. This will be used for memory latency calculation. ++ */ ++ intel_get_dram_info(dev_priv); ++ ++ ++ return 0; ++ ++err_msi: ++ if (pdev->msi_enabled) ++ pci_disable_msi(pdev); ++ pm_qos_remove_request(&dev_priv->pm_qos); ++err_ggtt: ++ i915_ggtt_cleanup_hw(dev_priv); ++err_perf: ++ i915_perf_fini(dev_priv); ++ return ret; ++} ++ ++/** ++ * i915_driver_cleanup_hw - cleanup the setup done in i915_driver_init_hw() ++ * @dev_priv: device private ++ */ ++static void i915_driver_cleanup_hw(struct drm_i915_private *dev_priv) ++{ ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ ++ i915_perf_fini(dev_priv); ++ ++ if (pdev->msi_enabled) ++ pci_disable_msi(pdev); ++ ++ pm_qos_remove_request(&dev_priv->pm_qos); ++ i915_ggtt_cleanup_hw(dev_priv); ++} ++ ++/** ++ * i915_driver_register - register the driver with the rest of the system ++ * @dev_priv: device private ++ * ++ * Perform any steps necessary to make the driver available via kernel ++ * internal or userspace interfaces. ++ */ ++static void i915_driver_register(struct drm_i915_private *dev_priv) ++{ ++ struct drm_device *dev = &dev_priv->drm; ++ ++ i915_gem_shrinker_register(dev_priv); ++ i915_pmu_register(dev_priv); ++ ++ /* ++ * Notify a valid surface after modesetting, ++ * when running inside a VM. ++ */ ++ if (intel_vgpu_active(dev_priv)) ++ I915_WRITE(vgtif_reg(display_ready), VGT_DRV_DISPLAY_READY); ++ ++ /* Reveal our presence to userspace */ ++ if (drm_dev_register(dev, 0) == 0) { ++ i915_debugfs_register(dev_priv); ++ i915_setup_sysfs(dev_priv); ++ ++ /* Depends on sysfs having been initialized */ ++ i915_perf_register(dev_priv); ++ } else ++ DRM_ERROR("Failed to register driver for userspace access!\n"); ++ ++ if (HAS_DISPLAY(dev_priv)) { ++ /* Must be done after probing outputs */ ++ intel_opregion_register(dev_priv); ++ acpi_video_register(); ++ } ++ ++ if (IS_GEN(dev_priv, 5)) ++ intel_gpu_ips_init(dev_priv); ++ ++ intel_audio_init(dev_priv); ++ ++ /* ++ * Some ports require correctly set-up hpd registers for detection to ++ * work properly (leading to ghost connected connector status), e.g. VGA ++ * on gm45. Hence we can only set up the initial fbdev config after hpd ++ * irqs are fully enabled. We do it last so that the async config ++ * cannot run before the connectors are registered. ++ */ ++ intel_fbdev_initial_config_async(dev); ++ ++ /* ++ * We need to coordinate the hotplugs with the asynchronous fbdev ++ * configuration, for which we use the fbdev->async_cookie. ++ */ ++ if (HAS_DISPLAY(dev_priv)) ++ drm_kms_helper_poll_init(dev); ++ ++ intel_power_domains_enable(dev_priv); ++ intel_runtime_pm_enable(dev_priv); ++} ++ ++/** ++ * i915_driver_unregister - cleanup the registration done in i915_driver_regiser() ++ * @dev_priv: device private ++ */ ++static void i915_driver_unregister(struct drm_i915_private *dev_priv) ++{ ++ intel_runtime_pm_disable(dev_priv); ++ intel_power_domains_disable(dev_priv); ++ ++ intel_fbdev_unregister(dev_priv); ++ intel_audio_deinit(dev_priv); ++ ++ /* ++ * After flushing the fbdev (incl. a late async config which will ++ * have delayed queuing of a hotplug event), then flush the hotplug ++ * events. ++ */ ++ drm_kms_helper_poll_fini(&dev_priv->drm); ++ ++ intel_gpu_ips_teardown(); ++ acpi_video_unregister(); ++ intel_opregion_unregister(dev_priv); ++ ++ i915_perf_unregister(dev_priv); ++ i915_pmu_unregister(dev_priv); ++ ++ i915_teardown_sysfs(dev_priv); ++ drm_dev_unregister(&dev_priv->drm); ++ ++ i915_gem_shrinker_unregister(dev_priv); ++} ++ ++static void i915_welcome_messages(struct drm_i915_private *dev_priv) ++{ ++ if (drm_debug & DRM_UT_DRIVER) { ++ struct drm_printer p = drm_debug_printer("i915 device info:"); ++ ++ drm_printf(&p, "pciid=0x%04x rev=0x%02x platform=%s (subplatform=0x%x) gen=%i\n", ++ INTEL_DEVID(dev_priv), ++ INTEL_REVID(dev_priv), ++ intel_platform_name(INTEL_INFO(dev_priv)->platform), ++ intel_subplatform(RUNTIME_INFO(dev_priv), ++ INTEL_INFO(dev_priv)->platform), ++ INTEL_GEN(dev_priv)); ++ ++ intel_device_info_dump_flags(INTEL_INFO(dev_priv), &p); ++ intel_device_info_dump_runtime(RUNTIME_INFO(dev_priv), &p); ++ } ++ ++ if (IS_ENABLED(CONFIG_DRM_I915_DEBUG)) ++ DRM_INFO("DRM_I915_DEBUG enabled\n"); ++ if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) ++ DRM_INFO("DRM_I915_DEBUG_GEM enabled\n"); ++ if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)) ++ DRM_INFO("DRM_I915_DEBUG_RUNTIME_PM enabled\n"); ++} ++ ++static struct drm_i915_private * ++i915_driver_create(struct pci_dev *pdev, const struct pci_device_id *ent) ++{ ++ const struct intel_device_info *match_info = ++ (struct intel_device_info *)ent->driver_data; ++ struct intel_device_info *device_info; ++ struct drm_i915_private *i915; ++ int err; ++ ++ i915 = kzalloc(sizeof(*i915), GFP_KERNEL); ++ if (!i915) ++ return ERR_PTR(-ENOMEM); ++ ++ err = drm_dev_init(&i915->drm, &driver, &pdev->dev); ++ if (err) { ++ kfree(i915); ++ return ERR_PTR(err); ++ } ++ ++ i915->drm.pdev = pdev; ++ i915->drm.dev_private = i915; ++ pci_set_drvdata(pdev, &i915->drm); ++ ++ /* Setup the write-once "constant" device info */ ++ device_info = mkwrite_device_info(i915); ++ memcpy(device_info, match_info, sizeof(*device_info)); ++ RUNTIME_INFO(i915)->device_id = pdev->device; ++ ++ BUG_ON(device_info->gen > BITS_PER_TYPE(device_info->gen_mask)); ++ ++ return i915; ++} ++ ++static void i915_driver_destroy(struct drm_i915_private *i915) ++{ ++ struct pci_dev *pdev = i915->drm.pdev; ++ ++ drm_dev_fini(&i915->drm); ++ kfree(i915); ++ ++ /* And make sure we never chase our dangling pointer from pci_dev */ ++ pci_set_drvdata(pdev, NULL); ++} ++ ++/** ++ * i915_driver_load - setup chip and create an initial config ++ * @pdev: PCI device ++ * @ent: matching PCI ID entry ++ * ++ * The driver load routine has to do several things: ++ * - drive output discovery via intel_modeset_init() ++ * - initialize the memory manager ++ * - allocate initial config memory ++ * - setup the DRM framebuffer with the allocated memory ++ */ ++int i915_driver_load(struct pci_dev *pdev, const struct pci_device_id *ent) ++{ ++ const struct intel_device_info *match_info = ++ (struct intel_device_info *)ent->driver_data; ++ struct drm_i915_private *dev_priv; ++ int ret; ++ ++ dev_priv = i915_driver_create(pdev, ent); ++ if (IS_ERR(dev_priv)) ++ return PTR_ERR(dev_priv); ++ ++ /* Disable nuclear pageflip by default on pre-ILK */ ++ if (!i915_modparams.nuclear_pageflip && match_info->gen < 5) ++ dev_priv->drm.driver_features &= ~DRIVER_ATOMIC; ++ ++ ret = pci_enable_device(pdev); ++ if (ret) ++ goto out_fini; ++ ++ ret = i915_driver_init_early(dev_priv); ++ if (ret < 0) ++ goto out_pci_disable; ++ ++ disable_rpm_wakeref_asserts(dev_priv); ++ ++ ret = i915_driver_init_mmio(dev_priv); ++ if (ret < 0) ++ goto out_runtime_pm_put; ++ ++ ret = i915_driver_init_hw(dev_priv); ++ if (ret < 0) ++ goto out_cleanup_mmio; ++ ++ ret = i915_load_modeset_init(&dev_priv->drm); ++ if (ret < 0) ++ goto out_cleanup_hw; ++ ++ i915_driver_register(dev_priv); ++ ++ enable_rpm_wakeref_asserts(dev_priv); ++ ++ i915_welcome_messages(dev_priv); ++ ++ return 0; ++ ++out_cleanup_hw: ++ i915_driver_cleanup_hw(dev_priv); ++out_cleanup_mmio: ++ i915_driver_cleanup_mmio(dev_priv); ++out_runtime_pm_put: ++ enable_rpm_wakeref_asserts(dev_priv); ++ i915_driver_cleanup_early(dev_priv); ++out_pci_disable: ++ pci_disable_device(pdev); ++out_fini: ++ i915_load_error(dev_priv, "Device initialization failed (%d)\n", ret); ++ i915_driver_destroy(dev_priv); ++ return ret; ++} ++ ++void i915_driver_unload(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ ++ disable_rpm_wakeref_asserts(dev_priv); ++ ++ i915_driver_unregister(dev_priv); ++ ++ /* ++ * After unregistering the device to prevent any new users, cancel ++ * all in-flight requests so that we can quickly unbind the active ++ * resources. ++ */ ++ i915_gem_set_wedged(dev_priv); ++ ++ /* Flush any external code that still may be under the RCU lock */ ++ synchronize_rcu(); ++ ++ i915_gem_suspend(dev_priv); ++ ++ drm_atomic_helper_shutdown(dev); ++ ++ intel_gvt_cleanup(dev_priv); ++ ++ intel_modeset_cleanup(dev); ++ ++ intel_bios_cleanup(dev_priv); ++ ++ vga_switcheroo_unregister_client(pdev); ++ vga_client_register(pdev, NULL, NULL, NULL); ++ ++ intel_csr_ucode_fini(dev_priv); ++ ++ /* Free error state after interrupts are fully disabled. */ ++ cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work); ++ i915_reset_error_state(dev_priv); ++ ++ i915_gem_fini(dev_priv); ++ ++ intel_power_domains_fini_hw(dev_priv); ++ ++ i915_driver_cleanup_hw(dev_priv); ++ i915_driver_cleanup_mmio(dev_priv); ++ ++ enable_rpm_wakeref_asserts(dev_priv); ++ intel_runtime_pm_cleanup(dev_priv); ++} ++ ++static void i915_driver_release(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ ++ i915_driver_cleanup_early(dev_priv); ++ i915_driver_destroy(dev_priv); ++} ++ ++static int i915_driver_open(struct drm_device *dev, struct drm_file *file) ++{ ++ struct drm_i915_private *i915 = to_i915(dev); ++ int ret; ++ ++ ret = i915_gem_open(i915, file); ++ if (ret) ++ return ret; ++ ++ return 0; ++} ++ ++/** ++ * i915_driver_lastclose - clean up after all DRM clients have exited ++ * @dev: DRM device ++ * ++ * Take care of cleaning up after all DRM clients have exited. In the ++ * mode setting case, we want to restore the kernel's initial mode (just ++ * in case the last client left us in a bad state). ++ * ++ * Additionally, in the non-mode setting case, we'll tear down the GTT ++ * and DMA structures, since the kernel won't be using them, and clea ++ * up any GEM state. ++ */ ++static void i915_driver_lastclose(struct drm_device *dev) ++{ ++ intel_fbdev_restore_mode(dev); ++ vga_switcheroo_process_delayed_switch(); ++} ++ ++static void i915_driver_postclose(struct drm_device *dev, struct drm_file *file) ++{ ++ struct drm_i915_file_private *file_priv = file->driver_priv; ++ ++ mutex_lock(&dev->struct_mutex); ++ i915_gem_context_close(file); ++ i915_gem_release(dev, file); ++ mutex_unlock(&dev->struct_mutex); ++ ++ kfree(file_priv); ++} ++ ++static void intel_suspend_encoders(struct drm_i915_private *dev_priv) ++{ ++ struct drm_device *dev = &dev_priv->drm; ++ struct intel_encoder *encoder; ++ ++ drm_modeset_lock_all(dev); ++ for_each_intel_encoder(dev, encoder) ++ if (encoder->suspend) ++ encoder->suspend(encoder); ++ drm_modeset_unlock_all(dev); ++} ++ ++static int vlv_resume_prepare(struct drm_i915_private *dev_priv, ++ bool rpm_resume); ++static int vlv_suspend_complete(struct drm_i915_private *dev_priv); ++ ++static bool suspend_to_idle(struct drm_i915_private *dev_priv) ++{ ++#if IS_ENABLED(CONFIG_ACPI_SLEEP) ++ if (acpi_target_system_state() < ACPI_STATE_S3) ++ return true; ++#endif ++ return false; ++} ++ ++static int i915_drm_prepare(struct drm_device *dev) ++{ ++ struct drm_i915_private *i915 = to_i915(dev); ++ ++ /* ++ * NB intel_display_suspend() may issue new requests after we've ++ * ostensibly marked the GPU as ready-to-sleep here. We need to ++ * split out that work and pull it forward so that after point, ++ * the GPU is not woken again. ++ */ ++ i915_gem_suspend(i915); ++ ++ return 0; ++} ++ ++static int i915_drm_suspend(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ pci_power_t opregion_target_state; ++ ++ disable_rpm_wakeref_asserts(dev_priv); ++ ++ /* We do a lot of poking in a lot of registers, make sure they work ++ * properly. */ ++ intel_power_domains_disable(dev_priv); ++ ++ drm_kms_helper_poll_disable(dev); ++ ++ pci_save_state(pdev); ++ ++ intel_display_suspend(dev); ++ ++ intel_dp_mst_suspend(dev_priv); ++ ++ intel_runtime_pm_disable_interrupts(dev_priv); ++ intel_hpd_cancel_work(dev_priv); ++ ++ intel_suspend_encoders(dev_priv); ++ ++ intel_suspend_hw(dev_priv); ++ ++ i915_gem_suspend_gtt_mappings(dev_priv); ++ ++ i915_save_state(dev_priv); ++ ++ opregion_target_state = suspend_to_idle(dev_priv) ? PCI_D1 : PCI_D3cold; ++ intel_opregion_suspend(dev_priv, opregion_target_state); ++ ++ intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED, true); ++ ++ dev_priv->suspend_count++; ++ ++ intel_csr_ucode_suspend(dev_priv); ++ ++ enable_rpm_wakeref_asserts(dev_priv); ++ ++ return 0; ++} ++ ++static enum i915_drm_suspend_mode ++get_suspend_mode(struct drm_i915_private *dev_priv, bool hibernate) ++{ ++ if (hibernate) ++ return I915_DRM_SUSPEND_HIBERNATE; ++ ++ if (suspend_to_idle(dev_priv)) ++ return I915_DRM_SUSPEND_IDLE; ++ ++ return I915_DRM_SUSPEND_MEM; ++} ++ ++static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ int ret; ++ ++ disable_rpm_wakeref_asserts(dev_priv); ++ ++ i915_gem_suspend_late(dev_priv); ++ ++ intel_uncore_suspend(&dev_priv->uncore); ++ ++ intel_power_domains_suspend(dev_priv, ++ get_suspend_mode(dev_priv, hibernation)); ++ ++ ret = 0; ++ if (INTEL_GEN(dev_priv) >= 11 || IS_GEN9_LP(dev_priv)) ++ bxt_enable_dc9(dev_priv); ++ else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) ++ hsw_enable_pc8(dev_priv); ++ else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ ret = vlv_suspend_complete(dev_priv); ++ ++ if (ret) { ++ DRM_ERROR("Suspend complete failed: %d\n", ret); ++ intel_power_domains_resume(dev_priv); ++ ++ goto out; ++ } ++ ++ pci_disable_device(pdev); ++ /* ++ * During hibernation on some platforms the BIOS may try to access ++ * the device even though it's already in D3 and hang the machine. So ++ * leave the device in D0 on those platforms and hope the BIOS will ++ * power down the device properly. The issue was seen on multiple old ++ * GENs with different BIOS vendors, so having an explicit blacklist ++ * is inpractical; apply the workaround on everything pre GEN6. The ++ * platforms where the issue was seen: ++ * Lenovo Thinkpad X301, X61s, X60, T60, X41 ++ * Fujitsu FSC S7110 ++ * Acer Aspire 1830T ++ */ ++ if (!(hibernation && INTEL_GEN(dev_priv) < 6)) ++ pci_set_power_state(pdev, PCI_D3hot); ++ ++out: ++ enable_rpm_wakeref_asserts(dev_priv); ++ if (!dev_priv->uncore.user_forcewake.count) ++ intel_runtime_pm_cleanup(dev_priv); ++ ++ return ret; ++} ++ ++static int i915_suspend_switcheroo(struct drm_device *dev, pm_message_t state) ++{ ++ int error; ++ ++ if (!dev) { ++ DRM_ERROR("dev: %p\n", dev); ++ DRM_ERROR("DRM not initialized, aborting suspend.\n"); ++ return -ENODEV; ++ } ++ ++ if (WARN_ON_ONCE(state.event != PM_EVENT_SUSPEND && ++ state.event != PM_EVENT_FREEZE)) ++ return -EINVAL; ++ ++ if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) ++ return 0; ++ ++ error = i915_drm_suspend(dev); ++ if (error) ++ return error; ++ ++ return i915_drm_suspend_late(dev, false); ++} ++ ++static int i915_drm_resume(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ int ret; ++ ++ disable_rpm_wakeref_asserts(dev_priv); ++ intel_sanitize_gt_powersave(dev_priv); ++ ++ i915_gem_sanitize(dev_priv); ++ ++ ret = i915_ggtt_enable_hw(dev_priv); ++ if (ret) ++ DRM_ERROR("failed to re-enable GGTT\n"); ++ ++ intel_csr_ucode_resume(dev_priv); ++ ++ i915_restore_state(dev_priv); ++ intel_pps_unlock_regs_wa(dev_priv); ++ ++ intel_init_pch_refclk(dev_priv); ++ ++ /* ++ * Interrupts have to be enabled before any batches are run. If not the ++ * GPU will hang. i915_gem_init_hw() will initiate batches to ++ * update/restore the context. ++ * ++ * drm_mode_config_reset() needs AUX interrupts. ++ * ++ * Modeset enabling in intel_modeset_init_hw() also needs working ++ * interrupts. ++ */ ++ intel_runtime_pm_enable_interrupts(dev_priv); ++ ++ drm_mode_config_reset(dev); ++ ++ i915_gem_resume(dev_priv); ++ ++ intel_modeset_init_hw(dev); ++ intel_init_clock_gating(dev_priv); ++ ++ spin_lock_irq(&dev_priv->irq_lock); ++ if (dev_priv->display.hpd_irq_setup) ++ dev_priv->display.hpd_irq_setup(dev_priv); ++ spin_unlock_irq(&dev_priv->irq_lock); ++ ++ intel_dp_mst_resume(dev_priv); ++ ++ intel_display_resume(dev); ++ ++ drm_kms_helper_poll_enable(dev); ++ ++ /* ++ * ... but also need to make sure that hotplug processing ++ * doesn't cause havoc. Like in the driver load code we don't ++ * bother with the tiny race here where we might lose hotplug ++ * notifications. ++ * */ ++ intel_hpd_init(dev_priv); ++ ++ intel_opregion_resume(dev_priv); ++ ++ intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING, false); ++ ++ intel_power_domains_enable(dev_priv); ++ ++ enable_rpm_wakeref_asserts(dev_priv); ++ ++ return 0; ++} ++ ++static int i915_drm_resume_early(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ int ret; ++ ++ /* ++ * We have a resume ordering issue with the snd-hda driver also ++ * requiring our device to be power up. Due to the lack of a ++ * parent/child relationship we currently solve this with an early ++ * resume hook. ++ * ++ * FIXME: This should be solved with a special hdmi sink device or ++ * similar so that power domains can be employed. ++ */ ++ ++ /* ++ * Note that we need to set the power state explicitly, since we ++ * powered off the device during freeze and the PCI core won't power ++ * it back up for us during thaw. Powering off the device during ++ * freeze is not a hard requirement though, and during the ++ * suspend/resume phases the PCI core makes sure we get here with the ++ * device powered on. So in case we change our freeze logic and keep ++ * the device powered we can also remove the following set power state ++ * call. ++ */ ++ ret = pci_set_power_state(pdev, PCI_D0); ++ if (ret) { ++ DRM_ERROR("failed to set PCI D0 power state (%d)\n", ret); ++ return ret; ++ } ++ ++ /* ++ * Note that pci_enable_device() first enables any parent bridge ++ * device and only then sets the power state for this device. The ++ * bridge enabling is a nop though, since bridge devices are resumed ++ * first. The order of enabling power and enabling the device is ++ * imposed by the PCI core as described above, so here we preserve the ++ * same order for the freeze/thaw phases. ++ * ++ * TODO: eventually we should remove pci_disable_device() / ++ * pci_enable_enable_device() from suspend/resume. Due to how they ++ * depend on the device enable refcount we can't anyway depend on them ++ * disabling/enabling the device. ++ */ ++ if (pci_enable_device(pdev)) ++ return -EIO; ++ ++ pci_set_master(pdev); ++ ++ disable_rpm_wakeref_asserts(dev_priv); ++ ++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ ret = vlv_resume_prepare(dev_priv, false); ++ if (ret) ++ DRM_ERROR("Resume prepare failed: %d, continuing anyway\n", ++ ret); ++ ++ intel_uncore_resume_early(&dev_priv->uncore); ++ ++ i915_check_and_clear_faults(dev_priv); ++ ++ if (INTEL_GEN(dev_priv) >= 11 || IS_GEN9_LP(dev_priv)) { ++ gen9_sanitize_dc_state(dev_priv); ++ bxt_disable_dc9(dev_priv); ++ } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { ++ hsw_disable_pc8(dev_priv); ++ } ++ ++ intel_uncore_sanitize(dev_priv); ++ ++ intel_power_domains_resume(dev_priv); ++ ++ intel_engines_sanitize(dev_priv, true); ++ ++ enable_rpm_wakeref_asserts(dev_priv); ++ ++ return ret; ++} ++ ++static int i915_resume_switcheroo(struct drm_device *dev) ++{ ++ int ret; ++ ++ if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) ++ return 0; ++ ++ ret = i915_drm_resume_early(dev); ++ if (ret) ++ return ret; ++ ++ return i915_drm_resume(dev); ++} ++ ++static int i915_pm_prepare(struct device *kdev) ++{ ++ struct pci_dev *pdev = to_pci_dev(kdev); ++ struct drm_device *dev = pci_get_drvdata(pdev); ++ ++ if (!dev) { ++ dev_err(kdev, "DRM not initialized, aborting suspend.\n"); ++ return -ENODEV; ++ } ++ ++ if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) ++ return 0; ++ ++ return i915_drm_prepare(dev); ++} ++ ++static int i915_pm_suspend(struct device *kdev) ++{ ++ struct pci_dev *pdev = to_pci_dev(kdev); ++ struct drm_device *dev = pci_get_drvdata(pdev); ++ ++ if (!dev) { ++ dev_err(kdev, "DRM not initialized, aborting suspend.\n"); ++ return -ENODEV; ++ } ++ ++ if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) ++ return 0; ++ ++ return i915_drm_suspend(dev); ++} ++ ++static int i915_pm_suspend_late(struct device *kdev) ++{ ++ struct drm_device *dev = &kdev_to_i915(kdev)->drm; ++ ++ /* ++ * We have a suspend ordering issue with the snd-hda driver also ++ * requiring our device to be power up. Due to the lack of a ++ * parent/child relationship we currently solve this with an late ++ * suspend hook. ++ * ++ * FIXME: This should be solved with a special hdmi sink device or ++ * similar so that power domains can be employed. ++ */ ++ if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) ++ return 0; ++ ++ return i915_drm_suspend_late(dev, false); ++} ++ ++static int i915_pm_poweroff_late(struct device *kdev) ++{ ++ struct drm_device *dev = &kdev_to_i915(kdev)->drm; ++ ++ if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) ++ return 0; ++ ++ return i915_drm_suspend_late(dev, true); ++} ++ ++static int i915_pm_resume_early(struct device *kdev) ++{ ++ struct drm_device *dev = &kdev_to_i915(kdev)->drm; ++ ++ if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) ++ return 0; ++ ++ return i915_drm_resume_early(dev); ++} ++ ++static int i915_pm_resume(struct device *kdev) ++{ ++ struct drm_device *dev = &kdev_to_i915(kdev)->drm; ++ ++ if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) ++ return 0; ++ ++ return i915_drm_resume(dev); ++} ++ ++/* freeze: before creating the hibernation_image */ ++static int i915_pm_freeze(struct device *kdev) ++{ ++ struct drm_device *dev = &kdev_to_i915(kdev)->drm; ++ int ret; ++ ++ if (dev->switch_power_state != DRM_SWITCH_POWER_OFF) { ++ ret = i915_drm_suspend(dev); ++ if (ret) ++ return ret; ++ } ++ ++ ret = i915_gem_freeze(kdev_to_i915(kdev)); ++ if (ret) ++ return ret; ++ ++ return 0; ++} ++ ++static int i915_pm_freeze_late(struct device *kdev) ++{ ++ struct drm_device *dev = &kdev_to_i915(kdev)->drm; ++ int ret; ++ ++ if (dev->switch_power_state != DRM_SWITCH_POWER_OFF) { ++ ret = i915_drm_suspend_late(dev, true); ++ if (ret) ++ return ret; ++ } ++ ++ ret = i915_gem_freeze_late(kdev_to_i915(kdev)); ++ if (ret) ++ return ret; ++ ++ return 0; ++} ++ ++/* thaw: called after creating the hibernation image, but before turning off. */ ++static int i915_pm_thaw_early(struct device *kdev) ++{ ++ return i915_pm_resume_early(kdev); ++} ++ ++static int i915_pm_thaw(struct device *kdev) ++{ ++ return i915_pm_resume(kdev); ++} ++ ++/* restore: called after loading the hibernation image. */ ++static int i915_pm_restore_early(struct device *kdev) ++{ ++ return i915_pm_resume_early(kdev); ++} ++ ++static int i915_pm_restore(struct device *kdev) ++{ ++ return i915_pm_resume(kdev); ++} ++ ++/* ++ * Save all Gunit registers that may be lost after a D3 and a subsequent ++ * S0i[R123] transition. The list of registers needing a save/restore is ++ * defined in the VLV2_S0IXRegs document. This documents marks all Gunit ++ * registers in the following way: ++ * - Driver: saved/restored by the driver ++ * - Punit : saved/restored by the Punit firmware ++ * - No, w/o marking: no need to save/restore, since the register is R/O or ++ * used internally by the HW in a way that doesn't depend ++ * keeping the content across a suspend/resume. ++ * - Debug : used for debugging ++ * ++ * We save/restore all registers marked with 'Driver', with the following ++ * exceptions: ++ * - Registers out of use, including also registers marked with 'Debug'. ++ * These have no effect on the driver's operation, so we don't save/restore ++ * them to reduce the overhead. ++ * - Registers that are fully setup by an initialization function called from ++ * the resume path. For example many clock gating and RPS/RC6 registers. ++ * - Registers that provide the right functionality with their reset defaults. ++ * ++ * TODO: Except for registers that based on the above 3 criteria can be safely ++ * ignored, we save/restore all others, practically treating the HW context as ++ * a black-box for the driver. Further investigation is needed to reduce the ++ * saved/restored registers even further, by following the same 3 criteria. ++ */ ++static void vlv_save_gunit_s0ix_state(struct drm_i915_private *dev_priv) ++{ ++ struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state; ++ int i; ++ ++ /* GAM 0x4000-0x4770 */ ++ s->wr_watermark = I915_READ(GEN7_WR_WATERMARK); ++ s->gfx_prio_ctrl = I915_READ(GEN7_GFX_PRIO_CTRL); ++ s->arb_mode = I915_READ(ARB_MODE); ++ s->gfx_pend_tlb0 = I915_READ(GEN7_GFX_PEND_TLB0); ++ s->gfx_pend_tlb1 = I915_READ(GEN7_GFX_PEND_TLB1); ++ ++ for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++) ++ s->lra_limits[i] = I915_READ(GEN7_LRA_LIMITS(i)); ++ ++ s->media_max_req_count = I915_READ(GEN7_MEDIA_MAX_REQ_COUNT); ++ s->gfx_max_req_count = I915_READ(GEN7_GFX_MAX_REQ_COUNT); ++ ++ s->render_hwsp = I915_READ(RENDER_HWS_PGA_GEN7); ++ s->ecochk = I915_READ(GAM_ECOCHK); ++ s->bsd_hwsp = I915_READ(BSD_HWS_PGA_GEN7); ++ s->blt_hwsp = I915_READ(BLT_HWS_PGA_GEN7); ++ ++ s->tlb_rd_addr = I915_READ(GEN7_TLB_RD_ADDR); ++ ++ /* MBC 0x9024-0x91D0, 0x8500 */ ++ s->g3dctl = I915_READ(VLV_G3DCTL); ++ s->gsckgctl = I915_READ(VLV_GSCKGCTL); ++ s->mbctl = I915_READ(GEN6_MBCTL); ++ ++ /* GCP 0x9400-0x9424, 0x8100-0x810C */ ++ s->ucgctl1 = I915_READ(GEN6_UCGCTL1); ++ s->ucgctl3 = I915_READ(GEN6_UCGCTL3); ++ s->rcgctl1 = I915_READ(GEN6_RCGCTL1); ++ s->rcgctl2 = I915_READ(GEN6_RCGCTL2); ++ s->rstctl = I915_READ(GEN6_RSTCTL); ++ s->misccpctl = I915_READ(GEN7_MISCCPCTL); ++ ++ /* GPM 0xA000-0xAA84, 0x8000-0x80FC */ ++ s->gfxpause = I915_READ(GEN6_GFXPAUSE); ++ s->rpdeuhwtc = I915_READ(GEN6_RPDEUHWTC); ++ s->rpdeuc = I915_READ(GEN6_RPDEUC); ++ s->ecobus = I915_READ(ECOBUS); ++ s->pwrdwnupctl = I915_READ(VLV_PWRDWNUPCTL); ++ s->rp_down_timeout = I915_READ(GEN6_RP_DOWN_TIMEOUT); ++ s->rp_deucsw = I915_READ(GEN6_RPDEUCSW); ++ s->rcubmabdtmr = I915_READ(GEN6_RCUBMABDTMR); ++ s->rcedata = I915_READ(VLV_RCEDATA); ++ s->spare2gh = I915_READ(VLV_SPAREG2H); ++ ++ /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */ ++ s->gt_imr = I915_READ(GTIMR); ++ s->gt_ier = I915_READ(GTIER); ++ s->pm_imr = I915_READ(GEN6_PMIMR); ++ s->pm_ier = I915_READ(GEN6_PMIER); ++ ++ for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++) ++ s->gt_scratch[i] = I915_READ(GEN7_GT_SCRATCH(i)); ++ ++ /* GT SA CZ domain, 0x100000-0x138124 */ ++ s->tilectl = I915_READ(TILECTL); ++ s->gt_fifoctl = I915_READ(GTFIFOCTL); ++ s->gtlc_wake_ctrl = I915_READ(VLV_GTLC_WAKE_CTRL); ++ s->gtlc_survive = I915_READ(VLV_GTLC_SURVIVABILITY_REG); ++ s->pmwgicz = I915_READ(VLV_PMWGICZ); ++ ++ /* Gunit-Display CZ domain, 0x182028-0x1821CF */ ++ s->gu_ctl0 = I915_READ(VLV_GU_CTL0); ++ s->gu_ctl1 = I915_READ(VLV_GU_CTL1); ++ s->pcbr = I915_READ(VLV_PCBR); ++ s->clock_gate_dis2 = I915_READ(VLV_GUNIT_CLOCK_GATE2); ++ ++ /* ++ * Not saving any of: ++ * DFT, 0x9800-0x9EC0 ++ * SARB, 0xB000-0xB1FC ++ * GAC, 0x5208-0x524C, 0x14000-0x14C000 ++ * PCI CFG ++ */ ++} ++ ++static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *dev_priv) ++{ ++ struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state; ++ u32 val; ++ int i; ++ ++ /* GAM 0x4000-0x4770 */ ++ I915_WRITE(GEN7_WR_WATERMARK, s->wr_watermark); ++ I915_WRITE(GEN7_GFX_PRIO_CTRL, s->gfx_prio_ctrl); ++ I915_WRITE(ARB_MODE, s->arb_mode | (0xffff << 16)); ++ I915_WRITE(GEN7_GFX_PEND_TLB0, s->gfx_pend_tlb0); ++ I915_WRITE(GEN7_GFX_PEND_TLB1, s->gfx_pend_tlb1); ++ ++ for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++) ++ I915_WRITE(GEN7_LRA_LIMITS(i), s->lra_limits[i]); ++ ++ I915_WRITE(GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count); ++ I915_WRITE(GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count); ++ ++ I915_WRITE(RENDER_HWS_PGA_GEN7, s->render_hwsp); ++ I915_WRITE(GAM_ECOCHK, s->ecochk); ++ I915_WRITE(BSD_HWS_PGA_GEN7, s->bsd_hwsp); ++ I915_WRITE(BLT_HWS_PGA_GEN7, s->blt_hwsp); ++ ++ I915_WRITE(GEN7_TLB_RD_ADDR, s->tlb_rd_addr); ++ ++ /* MBC 0x9024-0x91D0, 0x8500 */ ++ I915_WRITE(VLV_G3DCTL, s->g3dctl); ++ I915_WRITE(VLV_GSCKGCTL, s->gsckgctl); ++ I915_WRITE(GEN6_MBCTL, s->mbctl); ++ ++ /* GCP 0x9400-0x9424, 0x8100-0x810C */ ++ I915_WRITE(GEN6_UCGCTL1, s->ucgctl1); ++ I915_WRITE(GEN6_UCGCTL3, s->ucgctl3); ++ I915_WRITE(GEN6_RCGCTL1, s->rcgctl1); ++ I915_WRITE(GEN6_RCGCTL2, s->rcgctl2); ++ I915_WRITE(GEN6_RSTCTL, s->rstctl); ++ I915_WRITE(GEN7_MISCCPCTL, s->misccpctl); ++ ++ /* GPM 0xA000-0xAA84, 0x8000-0x80FC */ ++ I915_WRITE(GEN6_GFXPAUSE, s->gfxpause); ++ I915_WRITE(GEN6_RPDEUHWTC, s->rpdeuhwtc); ++ I915_WRITE(GEN6_RPDEUC, s->rpdeuc); ++ I915_WRITE(ECOBUS, s->ecobus); ++ I915_WRITE(VLV_PWRDWNUPCTL, s->pwrdwnupctl); ++ I915_WRITE(GEN6_RP_DOWN_TIMEOUT,s->rp_down_timeout); ++ I915_WRITE(GEN6_RPDEUCSW, s->rp_deucsw); ++ I915_WRITE(GEN6_RCUBMABDTMR, s->rcubmabdtmr); ++ I915_WRITE(VLV_RCEDATA, s->rcedata); ++ I915_WRITE(VLV_SPAREG2H, s->spare2gh); ++ ++ /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */ ++ I915_WRITE(GTIMR, s->gt_imr); ++ I915_WRITE(GTIER, s->gt_ier); ++ I915_WRITE(GEN6_PMIMR, s->pm_imr); ++ I915_WRITE(GEN6_PMIER, s->pm_ier); ++ ++ for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++) ++ I915_WRITE(GEN7_GT_SCRATCH(i), s->gt_scratch[i]); ++ ++ /* GT SA CZ domain, 0x100000-0x138124 */ ++ I915_WRITE(TILECTL, s->tilectl); ++ I915_WRITE(GTFIFOCTL, s->gt_fifoctl); ++ /* ++ * Preserve the GT allow wake and GFX force clock bit, they are not ++ * be restored, as they are used to control the s0ix suspend/resume ++ * sequence by the caller. ++ */ ++ val = I915_READ(VLV_GTLC_WAKE_CTRL); ++ val &= VLV_GTLC_ALLOWWAKEREQ; ++ val |= s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ; ++ I915_WRITE(VLV_GTLC_WAKE_CTRL, val); ++ ++ val = I915_READ(VLV_GTLC_SURVIVABILITY_REG); ++ val &= VLV_GFX_CLK_FORCE_ON_BIT; ++ val |= s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT; ++ I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val); ++ ++ I915_WRITE(VLV_PMWGICZ, s->pmwgicz); ++ ++ /* Gunit-Display CZ domain, 0x182028-0x1821CF */ ++ I915_WRITE(VLV_GU_CTL0, s->gu_ctl0); ++ I915_WRITE(VLV_GU_CTL1, s->gu_ctl1); ++ I915_WRITE(VLV_PCBR, s->pcbr); ++ I915_WRITE(VLV_GUNIT_CLOCK_GATE2, s->clock_gate_dis2); ++} ++ ++static int vlv_wait_for_pw_status(struct drm_i915_private *dev_priv, ++ u32 mask, u32 val) ++{ ++ i915_reg_t reg = VLV_GTLC_PW_STATUS; ++ u32 reg_value; ++ int ret; ++ ++ /* The HW does not like us polling for PW_STATUS frequently, so ++ * use the sleeping loop rather than risk the busy spin within ++ * intel_wait_for_register(). ++ * ++ * Transitioning between RC6 states should be at most 2ms (see ++ * valleyview_enable_rps) so use a 3ms timeout. ++ */ ++ ret = wait_for(((reg_value = I915_READ_NOTRACE(reg)) & mask) == val, 3); ++ ++ /* just trace the final value */ ++ trace_i915_reg_rw(false, reg, reg_value, sizeof(reg_value), true); ++ ++ return ret; ++} ++ ++int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on) ++{ ++ u32 val; ++ int err; ++ ++ val = I915_READ(VLV_GTLC_SURVIVABILITY_REG); ++ val &= ~VLV_GFX_CLK_FORCE_ON_BIT; ++ if (force_on) ++ val |= VLV_GFX_CLK_FORCE_ON_BIT; ++ I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val); ++ ++ if (!force_on) ++ return 0; ++ ++ err = intel_wait_for_register(&dev_priv->uncore, ++ VLV_GTLC_SURVIVABILITY_REG, ++ VLV_GFX_CLK_STATUS_BIT, ++ VLV_GFX_CLK_STATUS_BIT, ++ 20); ++ if (err) ++ DRM_ERROR("timeout waiting for GFX clock force-on (%08x)\n", ++ I915_READ(VLV_GTLC_SURVIVABILITY_REG)); ++ ++ return err; ++} ++ ++static int vlv_allow_gt_wake(struct drm_i915_private *dev_priv, bool allow) ++{ ++ u32 mask; ++ u32 val; ++ int err; ++ ++ val = I915_READ(VLV_GTLC_WAKE_CTRL); ++ val &= ~VLV_GTLC_ALLOWWAKEREQ; ++ if (allow) ++ val |= VLV_GTLC_ALLOWWAKEREQ; ++ I915_WRITE(VLV_GTLC_WAKE_CTRL, val); ++ POSTING_READ(VLV_GTLC_WAKE_CTRL); ++ ++ mask = VLV_GTLC_ALLOWWAKEACK; ++ val = allow ? mask : 0; ++ ++ err = vlv_wait_for_pw_status(dev_priv, mask, val); ++ if (err) ++ DRM_ERROR("timeout disabling GT waking\n"); ++ ++ return err; ++} ++ ++static void vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv, ++ bool wait_for_on) ++{ ++ u32 mask; ++ u32 val; ++ ++ mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK; ++ val = wait_for_on ? mask : 0; ++ ++ /* ++ * RC6 transitioning can be delayed up to 2 msec (see ++ * valleyview_enable_rps), use 3 msec for safety. ++ * ++ * This can fail to turn off the rc6 if the GPU is stuck after a failed ++ * reset and we are trying to force the machine to sleep. ++ */ ++ if (vlv_wait_for_pw_status(dev_priv, mask, val)) ++ DRM_DEBUG_DRIVER("timeout waiting for GT wells to go %s\n", ++ onoff(wait_for_on)); ++} ++ ++static void vlv_check_no_gt_access(struct drm_i915_private *dev_priv) ++{ ++ if (!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR)) ++ return; ++ ++ DRM_DEBUG_DRIVER("GT register access while GT waking disabled\n"); ++ I915_WRITE(VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR); ++} ++ ++static int vlv_suspend_complete(struct drm_i915_private *dev_priv) ++{ ++ u32 mask; ++ int err; ++ ++ /* ++ * Bspec defines the following GT well on flags as debug only, so ++ * don't treat them as hard failures. ++ */ ++ vlv_wait_for_gt_wells(dev_priv, false); ++ ++ mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS; ++ WARN_ON((I915_READ(VLV_GTLC_WAKE_CTRL) & mask) != mask); ++ ++ vlv_check_no_gt_access(dev_priv); ++ ++ err = vlv_force_gfx_clock(dev_priv, true); ++ if (err) ++ goto err1; ++ ++ err = vlv_allow_gt_wake(dev_priv, false); ++ if (err) ++ goto err2; ++ ++ if (!IS_CHERRYVIEW(dev_priv)) ++ vlv_save_gunit_s0ix_state(dev_priv); ++ ++ err = vlv_force_gfx_clock(dev_priv, false); ++ if (err) ++ goto err2; ++ ++ return 0; ++ ++err2: ++ /* For safety always re-enable waking and disable gfx clock forcing */ ++ vlv_allow_gt_wake(dev_priv, true); ++err1: ++ vlv_force_gfx_clock(dev_priv, false); ++ ++ return err; ++} ++ ++static int vlv_resume_prepare(struct drm_i915_private *dev_priv, ++ bool rpm_resume) ++{ ++ int err; ++ int ret; ++ ++ /* ++ * If any of the steps fail just try to continue, that's the best we ++ * can do at this point. Return the first error code (which will also ++ * leave RPM permanently disabled). ++ */ ++ ret = vlv_force_gfx_clock(dev_priv, true); ++ ++ if (!IS_CHERRYVIEW(dev_priv)) ++ vlv_restore_gunit_s0ix_state(dev_priv); ++ ++ err = vlv_allow_gt_wake(dev_priv, true); ++ if (!ret) ++ ret = err; ++ ++ err = vlv_force_gfx_clock(dev_priv, false); ++ if (!ret) ++ ret = err; ++ ++ vlv_check_no_gt_access(dev_priv); ++ ++ if (rpm_resume) ++ intel_init_clock_gating(dev_priv); ++ ++ return ret; ++} ++ ++static int intel_runtime_suspend(struct device *kdev) ++{ ++ struct pci_dev *pdev = to_pci_dev(kdev); ++ struct drm_device *dev = pci_get_drvdata(pdev); ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ int ret; ++ ++ if (WARN_ON_ONCE(!(dev_priv->gt_pm.rc6.enabled && HAS_RC6(dev_priv)))) ++ return -ENODEV; ++ ++ if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev_priv))) ++ return -ENODEV; ++ ++ DRM_DEBUG_KMS("Suspending device\n"); ++ ++ disable_rpm_wakeref_asserts(dev_priv); ++ ++ /* ++ * We are safe here against re-faults, since the fault handler takes ++ * an RPM reference. ++ */ ++ i915_gem_runtime_suspend(dev_priv); ++ ++ intel_uc_suspend(dev_priv); ++ ++ intel_runtime_pm_disable_interrupts(dev_priv); ++ ++ intel_uncore_suspend(&dev_priv->uncore); ++ ++ ret = 0; ++ if (INTEL_GEN(dev_priv) >= 11) { ++ icl_display_core_uninit(dev_priv); ++ bxt_enable_dc9(dev_priv); ++ } else if (IS_GEN9_LP(dev_priv)) { ++ bxt_display_core_uninit(dev_priv); ++ bxt_enable_dc9(dev_priv); ++ } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { ++ hsw_enable_pc8(dev_priv); ++ } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { ++ ret = vlv_suspend_complete(dev_priv); ++ } ++ ++ if (ret) { ++ DRM_ERROR("Runtime suspend failed, disabling it (%d)\n", ret); ++ intel_uncore_runtime_resume(&dev_priv->uncore); ++ ++ intel_runtime_pm_enable_interrupts(dev_priv); ++ ++ intel_uc_resume(dev_priv); ++ ++ i915_gem_init_swizzling(dev_priv); ++ i915_gem_restore_fences(dev_priv); ++ ++ enable_rpm_wakeref_asserts(dev_priv); ++ ++ return ret; ++ } ++ ++ enable_rpm_wakeref_asserts(dev_priv); ++ intel_runtime_pm_cleanup(dev_priv); ++ ++ if (intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore)) ++ DRM_ERROR("Unclaimed access detected prior to suspending\n"); ++ ++ dev_priv->runtime_pm.suspended = true; ++ ++ /* ++ * FIXME: We really should find a document that references the arguments ++ * used below! ++ */ ++ if (IS_BROADWELL(dev_priv)) { ++ /* ++ * On Broadwell, if we use PCI_D1 the PCH DDI ports will stop ++ * being detected, and the call we do at intel_runtime_resume() ++ * won't be able to restore them. Since PCI_D3hot matches the ++ * actual specification and appears to be working, use it. ++ */ ++ intel_opregion_notify_adapter(dev_priv, PCI_D3hot); ++ } else { ++ /* ++ * current versions of firmware which depend on this opregion ++ * notification have repurposed the D1 definition to mean ++ * "runtime suspended" vs. what you would normally expect (D3) ++ * to distinguish it from notifications that might be sent via ++ * the suspend path. ++ */ ++ intel_opregion_notify_adapter(dev_priv, PCI_D1); ++ } ++ ++ assert_forcewakes_inactive(&dev_priv->uncore); ++ ++ if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) ++ intel_hpd_poll_init(dev_priv); ++ ++ DRM_DEBUG_KMS("Device suspended\n"); ++ return 0; ++} ++ ++static int intel_runtime_resume(struct device *kdev) ++{ ++ struct pci_dev *pdev = to_pci_dev(kdev); ++ struct drm_device *dev = pci_get_drvdata(pdev); ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ int ret = 0; ++ ++ if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev_priv))) ++ return -ENODEV; ++ ++ DRM_DEBUG_KMS("Resuming device\n"); ++ ++ WARN_ON_ONCE(atomic_read(&dev_priv->runtime_pm.wakeref_count)); ++ disable_rpm_wakeref_asserts(dev_priv); ++ ++ intel_opregion_notify_adapter(dev_priv, PCI_D0); ++ dev_priv->runtime_pm.suspended = false; ++ if (intel_uncore_unclaimed_mmio(&dev_priv->uncore)) ++ DRM_DEBUG_DRIVER("Unclaimed access during suspend, bios?\n"); ++ ++ if (INTEL_GEN(dev_priv) >= 11) { ++ bxt_disable_dc9(dev_priv); ++ icl_display_core_init(dev_priv, true); ++ if (dev_priv->csr.dmc_payload) { ++ if (dev_priv->csr.allowed_dc_mask & ++ DC_STATE_EN_UPTO_DC6) ++ skl_enable_dc6(dev_priv); ++ else if (dev_priv->csr.allowed_dc_mask & ++ DC_STATE_EN_UPTO_DC5) ++ gen9_enable_dc5(dev_priv); ++ } ++ } else if (IS_GEN9_LP(dev_priv)) { ++ bxt_disable_dc9(dev_priv); ++ bxt_display_core_init(dev_priv, true); ++ if (dev_priv->csr.dmc_payload && ++ (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC5)) ++ gen9_enable_dc5(dev_priv); ++ } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { ++ hsw_disable_pc8(dev_priv); ++ } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { ++ ret = vlv_resume_prepare(dev_priv, true); ++ } ++ ++ intel_uncore_runtime_resume(&dev_priv->uncore); ++ ++ intel_runtime_pm_enable_interrupts(dev_priv); ++ ++ intel_uc_resume(dev_priv); ++ ++ /* ++ * No point of rolling back things in case of an error, as the best ++ * we can do is to hope that things will still work (and disable RPM). ++ */ ++ i915_gem_init_swizzling(dev_priv); ++ i915_gem_restore_fences(dev_priv); ++ ++ /* ++ * On VLV/CHV display interrupts are part of the display ++ * power well, so hpd is reinitialized from there. For ++ * everyone else do it here. ++ */ ++ if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) ++ intel_hpd_init(dev_priv); ++ ++ intel_enable_ipc(dev_priv); ++ ++ enable_rpm_wakeref_asserts(dev_priv); ++ ++ if (ret) ++ DRM_ERROR("Runtime resume failed, disabling it (%d)\n", ret); ++ else ++ DRM_DEBUG_KMS("Device resumed\n"); ++ ++ return ret; ++} ++ ++const struct dev_pm_ops i915_pm_ops = { ++ /* ++ * S0ix (via system suspend) and S3 event handlers [PMSG_SUSPEND, ++ * PMSG_RESUME] ++ */ ++ .prepare = i915_pm_prepare, ++ .suspend = i915_pm_suspend, ++ .suspend_late = i915_pm_suspend_late, ++ .resume_early = i915_pm_resume_early, ++ .resume = i915_pm_resume, ++ ++ /* ++ * S4 event handlers ++ * @freeze, @freeze_late : called (1) before creating the ++ * hibernation image [PMSG_FREEZE] and ++ * (2) after rebooting, before restoring ++ * the image [PMSG_QUIESCE] ++ * @thaw, @thaw_early : called (1) after creating the hibernation ++ * image, before writing it [PMSG_THAW] ++ * and (2) after failing to create or ++ * restore the image [PMSG_RECOVER] ++ * @poweroff, @poweroff_late: called after writing the hibernation ++ * image, before rebooting [PMSG_HIBERNATE] ++ * @restore, @restore_early : called after rebooting and restoring the ++ * hibernation image [PMSG_RESTORE] ++ */ ++ .freeze = i915_pm_freeze, ++ .freeze_late = i915_pm_freeze_late, ++ .thaw_early = i915_pm_thaw_early, ++ .thaw = i915_pm_thaw, ++ .poweroff = i915_pm_suspend, ++ .poweroff_late = i915_pm_poweroff_late, ++ .restore_early = i915_pm_restore_early, ++ .restore = i915_pm_restore, ++ ++ /* S0ix (via runtime suspend) event handlers */ ++ .runtime_suspend = intel_runtime_suspend, ++ .runtime_resume = intel_runtime_resume, ++}; ++ ++static const struct vm_operations_struct i915_gem_vm_ops = { ++ .fault = i915_gem_fault, ++ .open = drm_gem_vm_open, ++ .close = drm_gem_vm_close, ++}; ++ ++static const struct file_operations i915_driver_fops = { ++ .owner = THIS_MODULE, ++ .open = drm_open, ++ .release = drm_release, ++ .unlocked_ioctl = drm_ioctl, ++ .mmap = drm_gem_mmap, ++ .poll = drm_poll, ++ .read = drm_read, ++ .compat_ioctl = i915_compat_ioctl, ++ .llseek = noop_llseek, ++}; ++ ++static int ++i915_gem_reject_pin_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ return -ENODEV; ++} ++ ++static const struct drm_ioctl_desc i915_ioctls[] = { ++ DRM_IOCTL_DEF_DRV(I915_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), ++ DRM_IOCTL_DEF_DRV(I915_FLUSH, drm_noop, DRM_AUTH), ++ DRM_IOCTL_DEF_DRV(I915_FLIP, drm_noop, DRM_AUTH), ++ DRM_IOCTL_DEF_DRV(I915_BATCHBUFFER, drm_noop, DRM_AUTH), ++ DRM_IOCTL_DEF_DRV(I915_IRQ_EMIT, drm_noop, DRM_AUTH), ++ DRM_IOCTL_DEF_DRV(I915_IRQ_WAIT, drm_noop, DRM_AUTH), ++ DRM_IOCTL_DEF_DRV(I915_GETPARAM, i915_getparam_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_SETPARAM, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), ++ DRM_IOCTL_DEF_DRV(I915_ALLOC, drm_noop, DRM_AUTH), ++ DRM_IOCTL_DEF_DRV(I915_FREE, drm_noop, DRM_AUTH), ++ DRM_IOCTL_DEF_DRV(I915_INIT_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), ++ DRM_IOCTL_DEF_DRV(I915_CMDBUFFER, drm_noop, DRM_AUTH), ++ DRM_IOCTL_DEF_DRV(I915_DESTROY_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), ++ DRM_IOCTL_DEF_DRV(I915_SET_VBLANK_PIPE, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), ++ DRM_IOCTL_DEF_DRV(I915_GET_VBLANK_PIPE, drm_noop, DRM_AUTH), ++ DRM_IOCTL_DEF_DRV(I915_VBLANK_SWAP, drm_noop, DRM_AUTH), ++ DRM_IOCTL_DEF_DRV(I915_HWS_ADDR, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), ++ DRM_IOCTL_DEF_DRV(I915_GEM_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), ++ DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, i915_gem_execbuffer_ioctl, DRM_AUTH), ++ DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2_WR, i915_gem_execbuffer2_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY), ++ DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY), ++ DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_GEM_SET_CACHING, i915_gem_set_caching_ioctl, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_GEM_GET_CACHING, i915_gem_get_caching_ioctl, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), ++ DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), ++ DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_GEM_MMAP_GTT, i915_gem_mmap_gtt_ioctl, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_GEM_SET_TILING, i915_gem_set_tiling_ioctl, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_GEM_GET_TILING, i915_gem_get_tiling_ioctl, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id_ioctl, 0), ++ DRM_IOCTL_DEF_DRV(I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image_ioctl, DRM_MASTER), ++ DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs_ioctl, DRM_MASTER), ++ DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey_ioctl, DRM_MASTER), ++ DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, drm_noop, DRM_MASTER), ++ DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE_EXT, i915_gem_context_create_ioctl, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_GET_RESET_STATS, i915_gem_context_reset_stats_ioctl, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_GEM_USERPTR, i915_gem_userptr_ioctl, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_GETPARAM, i915_gem_context_getparam_ioctl, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_SETPARAM, i915_gem_context_setparam_ioctl, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_PERF_OPEN, i915_perf_open_ioctl, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_PERF_ADD_CONFIG, i915_perf_add_config_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_PERF_REMOVE_CONFIG, i915_perf_remove_config_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(I915_QUERY, i915_query_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), ++}; ++ ++static struct drm_driver driver = { ++ /* Don't use MTRRs here; the Xserver or userspace app should ++ * deal with them for Intel hardware. ++ */ ++ .driver_features = ++ DRIVER_GEM | DRIVER_PRIME | ++ DRIVER_RENDER | DRIVER_MODESET | DRIVER_ATOMIC | DRIVER_SYNCOBJ, ++ .release = i915_driver_release, ++ .open = i915_driver_open, ++ .lastclose = i915_driver_lastclose, ++ .postclose = i915_driver_postclose, ++ ++ .gem_close_object = i915_gem_close_object, ++ .gem_free_object_unlocked = i915_gem_free_object, ++ .gem_vm_ops = &i915_gem_vm_ops, ++ ++ .prime_handle_to_fd = drm_gem_prime_handle_to_fd, ++ .prime_fd_to_handle = drm_gem_prime_fd_to_handle, ++ .gem_prime_export = i915_gem_prime_export, ++ .gem_prime_import = i915_gem_prime_import, ++ ++ .dumb_create = i915_gem_dumb_create, ++ .dumb_map_offset = i915_gem_mmap_gtt, ++ .ioctls = i915_ioctls, ++ .num_ioctls = ARRAY_SIZE(i915_ioctls), ++ .fops = &i915_driver_fops, ++ .name = DRIVER_NAME, ++ .desc = DRIVER_DESC, ++ .date = DRIVER_DATE, ++ .major = DRIVER_MAJOR, ++ .minor = DRIVER_MINOR, ++ .patchlevel = DRIVER_PATCHLEVEL, ++}; ++ ++#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) ++#include "selftests/mock_drm.c" ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_drv.h b/drivers/gpu/drm/i915_legacy/i915_drv.h +new file mode 100644 +index 000000000000..066fd2a12851 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_drv.h +@@ -0,0 +1,3693 @@ ++/* i915_drv.h -- Private header for the I915 driver -*- linux-c -*- ++ */ ++/* ++ * ++ * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. ++ * All Rights Reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the ++ * "Software"), to deal in the Software without restriction, including ++ * without limitation the rights to use, copy, modify, merge, publish, ++ * distribute, sub license, and/or sell copies of the Software, and to ++ * permit persons to whom the Software is furnished to do so, subject to ++ * the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the ++ * next paragraph) shall be included in all copies or substantial portions ++ * of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS ++ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF ++ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. ++ * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR ++ * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, ++ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE ++ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ++ * ++ */ ++ ++#ifndef _I915_DRV_H_ ++#define _I915_DRV_H_ ++ ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include /* for struct drm_dma_handle */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "i915_fixed.h" ++#include "i915_params.h" ++#include "i915_reg.h" ++#include "i915_utils.h" ++ ++#include "intel_bios.h" ++#include "intel_device_info.h" ++#include "intel_display.h" ++#include "intel_dpll_mgr.h" ++#include "intel_frontbuffer.h" ++#include "intel_lrc.h" ++#include "intel_opregion.h" ++#include "intel_ringbuffer.h" ++#include "intel_uc.h" ++#include "intel_uncore.h" ++#include "intel_wopcm.h" ++#include "intel_workarounds.h" ++ ++#include "i915_gem.h" ++#include "i915_gem_context.h" ++#include "i915_gem_fence_reg.h" ++#include "i915_gem_object.h" ++#include "i915_gem_gtt.h" ++#include "i915_gpu_error.h" ++#include "i915_request.h" ++#include "i915_scheduler.h" ++#include "i915_timeline.h" ++#include "i915_vma.h" ++ ++#include "intel_gvt.h" ++ ++/* General customization: ++ */ ++ ++#define DRIVER_NAME "i915" ++#define DRIVER_DESC "Intel Graphics" ++#define DRIVER_DATE "20190417" ++#define DRIVER_TIMESTAMP 1555492067 ++ ++/* Use I915_STATE_WARN(x) and I915_STATE_WARN_ON() (rather than WARN() and ++ * WARN_ON()) for hw state sanity checks to check for unexpected conditions ++ * which may not necessarily be a user visible problem. This will either ++ * WARN() or DRM_ERROR() depending on the verbose_checks moduleparam, to ++ * enable distros and users to tailor their preferred amount of i915 abrt ++ * spam. ++ */ ++#define I915_STATE_WARN(condition, format...) ({ \ ++ int __ret_warn_on = !!(condition); \ ++ if (unlikely(__ret_warn_on)) \ ++ if (!WARN(i915_modparams.verbose_state_checks, format)) \ ++ DRM_ERROR(format); \ ++ unlikely(__ret_warn_on); \ ++}) ++ ++#define I915_STATE_WARN_ON(x) \ ++ I915_STATE_WARN((x), "%s", "WARN_ON(" __stringify(x) ")") ++ ++#if IS_ENABLED(CONFIG_DRM_I915_DEBUG) ++ ++bool __i915_inject_load_failure(const char *func, int line); ++#define i915_inject_load_failure() \ ++ __i915_inject_load_failure(__func__, __LINE__) ++ ++bool i915_error_injected(void); ++ ++#else ++ ++#define i915_inject_load_failure() false ++#define i915_error_injected() false ++ ++#endif ++ ++#define i915_load_error(i915, fmt, ...) \ ++ __i915_printk(i915, i915_error_injected() ? KERN_DEBUG : KERN_ERR, \ ++ fmt, ##__VA_ARGS__) ++ ++typedef depot_stack_handle_t intel_wakeref_t; ++ ++enum hpd_pin { ++ HPD_NONE = 0, ++ HPD_TV = HPD_NONE, /* TV is known to be unreliable */ ++ HPD_CRT, ++ HPD_SDVO_B, ++ HPD_SDVO_C, ++ HPD_PORT_A, ++ HPD_PORT_B, ++ HPD_PORT_C, ++ HPD_PORT_D, ++ HPD_PORT_E, ++ HPD_PORT_F, ++ HPD_NUM_PINS ++}; ++ ++#define for_each_hpd_pin(__pin) \ ++ for ((__pin) = (HPD_NONE + 1); (__pin) < HPD_NUM_PINS; (__pin)++) ++ ++/* Threshold == 5 for long IRQs, 50 for short */ ++#define HPD_STORM_DEFAULT_THRESHOLD 50 ++ ++struct i915_hotplug { ++ struct work_struct hotplug_work; ++ ++ struct { ++ unsigned long last_jiffies; ++ int count; ++ enum { ++ HPD_ENABLED = 0, ++ HPD_DISABLED = 1, ++ HPD_MARK_DISABLED = 2 ++ } state; ++ } stats[HPD_NUM_PINS]; ++ u32 event_bits; ++ struct delayed_work reenable_work; ++ ++ u32 long_port_mask; ++ u32 short_port_mask; ++ struct work_struct dig_port_work; ++ ++ struct work_struct poll_init_work; ++ bool poll_enabled; ++ ++ unsigned int hpd_storm_threshold; ++ /* Whether or not to count short HPD IRQs in HPD storms */ ++ u8 hpd_short_storm_enabled; ++ ++ /* ++ * if we get a HPD irq from DP and a HPD irq from non-DP ++ * the non-DP HPD could block the workqueue on a mode config ++ * mutex getting, that userspace may have taken. However ++ * userspace is waiting on the DP workqueue to run which is ++ * blocked behind the non-DP one. ++ */ ++ struct workqueue_struct *dp_wq; ++}; ++ ++#define I915_GEM_GPU_DOMAINS \ ++ (I915_GEM_DOMAIN_RENDER | \ ++ I915_GEM_DOMAIN_SAMPLER | \ ++ I915_GEM_DOMAIN_COMMAND | \ ++ I915_GEM_DOMAIN_INSTRUCTION | \ ++ I915_GEM_DOMAIN_VERTEX) ++ ++struct drm_i915_private; ++struct i915_mm_struct; ++struct i915_mmu_object; ++ ++struct drm_i915_file_private { ++ struct drm_i915_private *dev_priv; ++ struct drm_file *file; ++ ++ struct { ++ spinlock_t lock; ++ struct list_head request_list; ++/* 20ms is a fairly arbitrary limit (greater than the average frame time) ++ * chosen to prevent the CPU getting more than a frame ahead of the GPU ++ * (when using lax throttling for the frontbuffer). We also use it to ++ * offer free GPU waitboosts for severely congested workloads. ++ */ ++#define DRM_I915_THROTTLE_JIFFIES msecs_to_jiffies(20) ++ } mm; ++ ++ struct idr context_idr; ++ struct mutex context_idr_lock; /* guards context_idr */ ++ ++ struct idr vm_idr; ++ struct mutex vm_idr_lock; /* guards vm_idr */ ++ ++ unsigned int bsd_engine; ++ ++/* ++ * Every context ban increments per client ban score. Also ++ * hangs in short succession increments ban score. If ban threshold ++ * is reached, client is considered banned and submitting more work ++ * will fail. This is a stop gap measure to limit the badly behaving ++ * clients access to gpu. Note that unbannable contexts never increment ++ * the client ban score. ++ */ ++#define I915_CLIENT_SCORE_HANG_FAST 1 ++#define I915_CLIENT_FAST_HANG_JIFFIES (60 * HZ) ++#define I915_CLIENT_SCORE_CONTEXT_BAN 3 ++#define I915_CLIENT_SCORE_BANNED 9 ++ /** ban_score: Accumulated score of all ctx bans and fast hangs. */ ++ atomic_t ban_score; ++ unsigned long hang_timestamp; ++}; ++ ++/* Interface history: ++ * ++ * 1.1: Original. ++ * 1.2: Add Power Management ++ * 1.3: Add vblank support ++ * 1.4: Fix cmdbuffer path, add heap destroy ++ * 1.5: Add vblank pipe configuration ++ * 1.6: - New ioctl for scheduling buffer swaps on vertical blank ++ * - Support vertical blank on secondary display pipe ++ */ ++#define DRIVER_MAJOR 1 ++#define DRIVER_MINOR 6 ++#define DRIVER_PATCHLEVEL 0 ++ ++struct intel_overlay; ++struct intel_overlay_error_state; ++ ++struct sdvo_device_mapping { ++ u8 initialized; ++ u8 dvo_port; ++ u8 slave_addr; ++ u8 dvo_wiring; ++ u8 i2c_pin; ++ u8 ddc_pin; ++}; ++ ++struct intel_connector; ++struct intel_encoder; ++struct intel_atomic_state; ++struct intel_crtc_state; ++struct intel_initial_plane_config; ++struct intel_crtc; ++struct intel_limit; ++struct dpll; ++struct intel_cdclk_state; ++ ++struct drm_i915_display_funcs { ++ void (*get_cdclk)(struct drm_i915_private *dev_priv, ++ struct intel_cdclk_state *cdclk_state); ++ void (*set_cdclk)(struct drm_i915_private *dev_priv, ++ const struct intel_cdclk_state *cdclk_state, ++ enum pipe pipe); ++ int (*get_fifo_size)(struct drm_i915_private *dev_priv, ++ enum i9xx_plane_id i9xx_plane); ++ int (*compute_pipe_wm)(struct intel_crtc_state *cstate); ++ int (*compute_intermediate_wm)(struct intel_crtc_state *newstate); ++ void (*initial_watermarks)(struct intel_atomic_state *state, ++ struct intel_crtc_state *cstate); ++ void (*atomic_update_watermarks)(struct intel_atomic_state *state, ++ struct intel_crtc_state *cstate); ++ void (*optimize_watermarks)(struct intel_atomic_state *state, ++ struct intel_crtc_state *cstate); ++ int (*compute_global_watermarks)(struct intel_atomic_state *state); ++ void (*update_wm)(struct intel_crtc *crtc); ++ int (*modeset_calc_cdclk)(struct drm_atomic_state *state); ++ /* Returns the active state of the crtc, and if the crtc is active, ++ * fills out the pipe-config with the hw state. */ ++ bool (*get_pipe_config)(struct intel_crtc *, ++ struct intel_crtc_state *); ++ void (*get_initial_plane_config)(struct intel_crtc *, ++ struct intel_initial_plane_config *); ++ int (*crtc_compute_clock)(struct intel_crtc *crtc, ++ struct intel_crtc_state *crtc_state); ++ void (*crtc_enable)(struct intel_crtc_state *pipe_config, ++ struct drm_atomic_state *old_state); ++ void (*crtc_disable)(struct intel_crtc_state *old_crtc_state, ++ struct drm_atomic_state *old_state); ++ void (*update_crtcs)(struct drm_atomic_state *state); ++ void (*audio_codec_enable)(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state); ++ void (*audio_codec_disable)(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state); ++ void (*fdi_link_train)(struct intel_crtc *crtc, ++ const struct intel_crtc_state *crtc_state); ++ void (*init_clock_gating)(struct drm_i915_private *dev_priv); ++ void (*hpd_irq_setup)(struct drm_i915_private *dev_priv); ++ /* clock updates for mode set */ ++ /* cursor updates */ ++ /* render clock increase/decrease */ ++ /* display clock increase/decrease */ ++ /* pll clock increase/decrease */ ++ ++ int (*color_check)(struct intel_crtc_state *crtc_state); ++ /* ++ * Program double buffered color management registers during ++ * vblank evasion. The registers should then latch during the ++ * next vblank start, alongside any other double buffered registers ++ * involved with the same commit. ++ */ ++ void (*color_commit)(const struct intel_crtc_state *crtc_state); ++ /* ++ * Load LUTs (and other single buffered color management ++ * registers). Will (hopefully) be called during the vblank ++ * following the latching of any double buffered registers ++ * involved with the same commit. ++ */ ++ void (*load_luts)(const struct intel_crtc_state *crtc_state); ++}; ++ ++#define CSR_VERSION(major, minor) ((major) << 16 | (minor)) ++#define CSR_VERSION_MAJOR(version) ((version) >> 16) ++#define CSR_VERSION_MINOR(version) ((version) & 0xffff) ++ ++struct intel_csr { ++ struct work_struct work; ++ const char *fw_path; ++ u32 required_version; ++ u32 max_fw_size; /* bytes */ ++ u32 *dmc_payload; ++ u32 dmc_fw_size; /* dwords */ ++ u32 version; ++ u32 mmio_count; ++ i915_reg_t mmioaddr[8]; ++ u32 mmiodata[8]; ++ u32 dc_state; ++ u32 allowed_dc_mask; ++ intel_wakeref_t wakeref; ++}; ++ ++enum i915_cache_level { ++ I915_CACHE_NONE = 0, ++ I915_CACHE_LLC, /* also used for snoopable memory on non-LLC */ ++ I915_CACHE_L3_LLC, /* gen7+, L3 sits between the domain specifc ++ caches, eg sampler/render caches, and the ++ large Last-Level-Cache. LLC is coherent with ++ the CPU, but L3 is only visible to the GPU. */ ++ I915_CACHE_WT, /* hsw:gt3e WriteThrough for scanouts */ ++}; ++ ++#define I915_COLOR_UNEVICTABLE (-1) /* a non-vma sharing the address space */ ++ ++struct intel_fbc { ++ /* This is always the inner lock when overlapping with struct_mutex and ++ * it's the outer lock when overlapping with stolen_lock. */ ++ struct mutex lock; ++ unsigned threshold; ++ unsigned int possible_framebuffer_bits; ++ unsigned int busy_bits; ++ unsigned int visible_pipes_mask; ++ struct intel_crtc *crtc; ++ ++ struct drm_mm_node compressed_fb; ++ struct drm_mm_node *compressed_llb; ++ ++ bool false_color; ++ ++ bool enabled; ++ bool active; ++ bool flip_pending; ++ ++ bool underrun_detected; ++ struct work_struct underrun_work; ++ ++ /* ++ * Due to the atomic rules we can't access some structures without the ++ * appropriate locking, so we cache information here in order to avoid ++ * these problems. ++ */ ++ struct intel_fbc_state_cache { ++ struct i915_vma *vma; ++ unsigned long flags; ++ ++ struct { ++ unsigned int mode_flags; ++ u32 hsw_bdw_pixel_rate; ++ } crtc; ++ ++ struct { ++ unsigned int rotation; ++ int src_w; ++ int src_h; ++ bool visible; ++ /* ++ * Display surface base address adjustement for ++ * pageflips. Note that on gen4+ this only adjusts up ++ * to a tile, offsets within a tile are handled in ++ * the hw itself (with the TILEOFF register). ++ */ ++ int adjusted_x; ++ int adjusted_y; ++ ++ int y; ++ ++ u16 pixel_blend_mode; ++ } plane; ++ ++ struct { ++ const struct drm_format_info *format; ++ unsigned int stride; ++ } fb; ++ } state_cache; ++ ++ /* ++ * This structure contains everything that's relevant to program the ++ * hardware registers. When we want to figure out if we need to disable ++ * and re-enable FBC for a new configuration we just check if there's ++ * something different in the struct. The genx_fbc_activate functions ++ * are supposed to read from it in order to program the registers. ++ */ ++ struct intel_fbc_reg_params { ++ struct i915_vma *vma; ++ unsigned long flags; ++ ++ struct { ++ enum pipe pipe; ++ enum i9xx_plane_id i9xx_plane; ++ unsigned int fence_y_offset; ++ } crtc; ++ ++ struct { ++ const struct drm_format_info *format; ++ unsigned int stride; ++ } fb; ++ ++ int cfb_size; ++ unsigned int gen9_wa_cfb_stride; ++ } params; ++ ++ const char *no_fbc_reason; ++}; ++ ++/* ++ * HIGH_RR is the highest eDP panel refresh rate read from EDID ++ * LOW_RR is the lowest eDP panel refresh rate found from EDID ++ * parsing for same resolution. ++ */ ++enum drrs_refresh_rate_type { ++ DRRS_HIGH_RR, ++ DRRS_LOW_RR, ++ DRRS_MAX_RR, /* RR count */ ++}; ++ ++enum drrs_support_type { ++ DRRS_NOT_SUPPORTED = 0, ++ STATIC_DRRS_SUPPORT = 1, ++ SEAMLESS_DRRS_SUPPORT = 2 ++}; ++ ++struct intel_dp; ++struct i915_drrs { ++ struct mutex mutex; ++ struct delayed_work work; ++ struct intel_dp *dp; ++ unsigned busy_frontbuffer_bits; ++ enum drrs_refresh_rate_type refresh_rate_type; ++ enum drrs_support_type type; ++}; ++ ++struct i915_psr { ++ struct mutex lock; ++ ++#define I915_PSR_DEBUG_MODE_MASK 0x0f ++#define I915_PSR_DEBUG_DEFAULT 0x00 ++#define I915_PSR_DEBUG_DISABLE 0x01 ++#define I915_PSR_DEBUG_ENABLE 0x02 ++#define I915_PSR_DEBUG_FORCE_PSR1 0x03 ++#define I915_PSR_DEBUG_IRQ 0x10 ++ ++ u32 debug; ++ bool sink_support; ++ bool enabled; ++ struct intel_dp *dp; ++ enum pipe pipe; ++ bool active; ++ struct work_struct work; ++ unsigned busy_frontbuffer_bits; ++ bool sink_psr2_support; ++ bool link_standby; ++ bool colorimetry_support; ++ bool psr2_enabled; ++ u8 sink_sync_latency; ++ ktime_t last_entry_attempt; ++ ktime_t last_exit; ++ bool sink_not_reliable; ++ bool irq_aux_error; ++ u16 su_x_granularity; ++}; ++ ++/* ++ * Sorted by south display engine compatibility. ++ * If the new PCH comes with a south display engine that is not ++ * inherited from the latest item, please do not add it to the ++ * end. Instead, add it right after its "parent" PCH. ++ */ ++enum intel_pch { ++ PCH_NOP = -1, /* PCH without south display */ ++ PCH_NONE = 0, /* No PCH present */ ++ PCH_IBX, /* Ibexpeak PCH */ ++ PCH_CPT, /* Cougarpoint/Pantherpoint PCH */ ++ PCH_LPT, /* Lynxpoint/Wildcatpoint PCH */ ++ PCH_SPT, /* Sunrisepoint PCH */ ++ PCH_KBP, /* Kaby Lake PCH */ ++ PCH_CNP, /* Cannon/Comet Lake PCH */ ++ PCH_ICP, /* Ice Lake PCH */ ++}; ++ ++enum intel_sbi_destination { ++ SBI_ICLK, ++ SBI_MPHY, ++}; ++ ++#define QUIRK_LVDS_SSC_DISABLE (1<<1) ++#define QUIRK_INVERT_BRIGHTNESS (1<<2) ++#define QUIRK_BACKLIGHT_PRESENT (1<<3) ++#define QUIRK_PIN_SWIZZLED_PAGES (1<<5) ++#define QUIRK_INCREASE_T12_DELAY (1<<6) ++#define QUIRK_INCREASE_DDI_DISABLED_TIME (1<<7) ++ ++struct intel_fbdev; ++struct intel_fbc_work; ++ ++struct intel_gmbus { ++ struct i2c_adapter adapter; ++#define GMBUS_FORCE_BIT_RETRY (1U << 31) ++ u32 force_bit; ++ u32 reg0; ++ i915_reg_t gpio_reg; ++ struct i2c_algo_bit_data bit_algo; ++ struct drm_i915_private *dev_priv; ++}; ++ ++struct i915_suspend_saved_registers { ++ u32 saveDSPARB; ++ u32 saveFBC_CONTROL; ++ u32 saveCACHE_MODE_0; ++ u32 saveMI_ARB_STATE; ++ u32 saveSWF0[16]; ++ u32 saveSWF1[16]; ++ u32 saveSWF3[3]; ++ u64 saveFENCE[I915_MAX_NUM_FENCES]; ++ u32 savePCH_PORT_HOTPLUG; ++ u16 saveGCDGMBUS; ++}; ++ ++struct vlv_s0ix_state { ++ /* GAM */ ++ u32 wr_watermark; ++ u32 gfx_prio_ctrl; ++ u32 arb_mode; ++ u32 gfx_pend_tlb0; ++ u32 gfx_pend_tlb1; ++ u32 lra_limits[GEN7_LRA_LIMITS_REG_NUM]; ++ u32 media_max_req_count; ++ u32 gfx_max_req_count; ++ u32 render_hwsp; ++ u32 ecochk; ++ u32 bsd_hwsp; ++ u32 blt_hwsp; ++ u32 tlb_rd_addr; ++ ++ /* MBC */ ++ u32 g3dctl; ++ u32 gsckgctl; ++ u32 mbctl; ++ ++ /* GCP */ ++ u32 ucgctl1; ++ u32 ucgctl3; ++ u32 rcgctl1; ++ u32 rcgctl2; ++ u32 rstctl; ++ u32 misccpctl; ++ ++ /* GPM */ ++ u32 gfxpause; ++ u32 rpdeuhwtc; ++ u32 rpdeuc; ++ u32 ecobus; ++ u32 pwrdwnupctl; ++ u32 rp_down_timeout; ++ u32 rp_deucsw; ++ u32 rcubmabdtmr; ++ u32 rcedata; ++ u32 spare2gh; ++ ++ /* Display 1 CZ domain */ ++ u32 gt_imr; ++ u32 gt_ier; ++ u32 pm_imr; ++ u32 pm_ier; ++ u32 gt_scratch[GEN7_GT_SCRATCH_REG_NUM]; ++ ++ /* GT SA CZ domain */ ++ u32 tilectl; ++ u32 gt_fifoctl; ++ u32 gtlc_wake_ctrl; ++ u32 gtlc_survive; ++ u32 pmwgicz; ++ ++ /* Display 2 CZ domain */ ++ u32 gu_ctl0; ++ u32 gu_ctl1; ++ u32 pcbr; ++ u32 clock_gate_dis2; ++}; ++ ++struct intel_rps_ei { ++ ktime_t ktime; ++ u32 render_c0; ++ u32 media_c0; ++}; ++ ++struct intel_rps { ++ /* ++ * work, interrupts_enabled and pm_iir are protected by ++ * dev_priv->irq_lock ++ */ ++ struct work_struct work; ++ bool interrupts_enabled; ++ u32 pm_iir; ++ ++ /* PM interrupt bits that should never be masked */ ++ u32 pm_intrmsk_mbz; ++ ++ /* Frequencies are stored in potentially platform dependent multiples. ++ * In other words, *_freq needs to be multiplied by X to be interesting. ++ * Soft limits are those which are used for the dynamic reclocking done ++ * by the driver (raise frequencies under heavy loads, and lower for ++ * lighter loads). Hard limits are those imposed by the hardware. ++ * ++ * A distinction is made for overclocking, which is never enabled by ++ * default, and is considered to be above the hard limit if it's ++ * possible at all. ++ */ ++ u8 cur_freq; /* Current frequency (cached, may not == HW) */ ++ u8 min_freq_softlimit; /* Minimum frequency permitted by the driver */ ++ u8 max_freq_softlimit; /* Max frequency permitted by the driver */ ++ u8 max_freq; /* Maximum frequency, RP0 if not overclocking */ ++ u8 min_freq; /* AKA RPn. Minimum frequency */ ++ u8 boost_freq; /* Frequency to request when wait boosting */ ++ u8 idle_freq; /* Frequency to request when we are idle */ ++ u8 efficient_freq; /* AKA RPe. Pre-determined balanced frequency */ ++ u8 rp1_freq; /* "less than" RP0 power/freqency */ ++ u8 rp0_freq; /* Non-overclocked max frequency. */ ++ u16 gpll_ref_freq; /* vlv/chv GPLL reference frequency */ ++ ++ int last_adj; ++ ++ struct { ++ struct mutex mutex; ++ ++ enum { LOW_POWER, BETWEEN, HIGH_POWER } mode; ++ unsigned int interactive; ++ ++ u8 up_threshold; /* Current %busy required to uplock */ ++ u8 down_threshold; /* Current %busy required to downclock */ ++ } power; ++ ++ bool enabled; ++ atomic_t num_waiters; ++ atomic_t boosts; ++ ++ /* manual wa residency calculations */ ++ struct intel_rps_ei ei; ++}; ++ ++struct intel_rc6 { ++ bool enabled; ++ u64 prev_hw_residency[4]; ++ u64 cur_residency[4]; ++}; ++ ++struct intel_llc_pstate { ++ bool enabled; ++}; ++ ++struct intel_gen6_power_mgmt { ++ struct intel_rps rps; ++ struct intel_rc6 rc6; ++ struct intel_llc_pstate llc_pstate; ++}; ++ ++/* defined intel_pm.c */ ++extern spinlock_t mchdev_lock; ++ ++struct intel_ilk_power_mgmt { ++ u8 cur_delay; ++ u8 min_delay; ++ u8 max_delay; ++ u8 fmax; ++ u8 fstart; ++ ++ u64 last_count1; ++ unsigned long last_time1; ++ unsigned long chipset_power; ++ u64 last_count2; ++ u64 last_time2; ++ unsigned long gfx_power; ++ u8 corr; ++ ++ int c_m; ++ int r_t; ++}; ++ ++struct drm_i915_private; ++struct i915_power_well; ++ ++struct i915_power_well_ops { ++ /* ++ * Synchronize the well's hw state to match the current sw state, for ++ * example enable/disable it based on the current refcount. Called ++ * during driver init and resume time, possibly after first calling ++ * the enable/disable handlers. ++ */ ++ void (*sync_hw)(struct drm_i915_private *dev_priv, ++ struct i915_power_well *power_well); ++ /* ++ * Enable the well and resources that depend on it (for example ++ * interrupts located on the well). Called after the 0->1 refcount ++ * transition. ++ */ ++ void (*enable)(struct drm_i915_private *dev_priv, ++ struct i915_power_well *power_well); ++ /* ++ * Disable the well and resources that depend on it. Called after ++ * the 1->0 refcount transition. ++ */ ++ void (*disable)(struct drm_i915_private *dev_priv, ++ struct i915_power_well *power_well); ++ /* Returns the hw enabled state. */ ++ bool (*is_enabled)(struct drm_i915_private *dev_priv, ++ struct i915_power_well *power_well); ++}; ++ ++struct i915_power_well_regs { ++ i915_reg_t bios; ++ i915_reg_t driver; ++ i915_reg_t kvmr; ++ i915_reg_t debug; ++}; ++ ++/* Power well structure for haswell */ ++struct i915_power_well_desc { ++ const char *name; ++ bool always_on; ++ u64 domains; ++ /* unique identifier for this power well */ ++ enum i915_power_well_id id; ++ /* ++ * Arbitraty data associated with this power well. Platform and power ++ * well specific. ++ */ ++ union { ++ struct { ++ /* ++ * request/status flag index in the PUNIT power well ++ * control/status registers. ++ */ ++ u8 idx; ++ } vlv; ++ struct { ++ enum dpio_phy phy; ++ } bxt; ++ struct { ++ const struct i915_power_well_regs *regs; ++ /* ++ * request/status flag index in the power well ++ * constrol/status registers. ++ */ ++ u8 idx; ++ /* Mask of pipes whose IRQ logic is backed by the pw */ ++ u8 irq_pipe_mask; ++ /* The pw is backing the VGA functionality */ ++ bool has_vga:1; ++ bool has_fuses:1; ++ /* ++ * The pw is for an ICL+ TypeC PHY port in ++ * Thunderbolt mode. ++ */ ++ bool is_tc_tbt:1; ++ } hsw; ++ }; ++ const struct i915_power_well_ops *ops; ++}; ++ ++struct i915_power_well { ++ const struct i915_power_well_desc *desc; ++ /* power well enable/disable usage count */ ++ int count; ++ /* cached hw enabled state */ ++ bool hw_enabled; ++}; ++ ++struct i915_power_domains { ++ /* ++ * Power wells needed for initialization at driver init and suspend ++ * time are on. They are kept on until after the first modeset. ++ */ ++ bool initializing; ++ bool display_core_suspended; ++ int power_well_count; ++ ++ intel_wakeref_t wakeref; ++ ++ struct mutex lock; ++ int domain_use_count[POWER_DOMAIN_NUM]; ++ struct i915_power_well *power_wells; ++}; ++ ++#define MAX_L3_SLICES 2 ++struct intel_l3_parity { ++ u32 *remap_info[MAX_L3_SLICES]; ++ struct work_struct error_work; ++ int which_slice; ++}; ++ ++struct i915_gem_mm { ++ /** Memory allocator for GTT stolen memory */ ++ struct drm_mm stolen; ++ /** Protects the usage of the GTT stolen memory allocator. This is ++ * always the inner lock when overlapping with struct_mutex. */ ++ struct mutex stolen_lock; ++ ++ /* Protects bound_list/unbound_list and #drm_i915_gem_object.mm.link */ ++ spinlock_t obj_lock; ++ ++ /** List of all objects in gtt_space. Used to restore gtt ++ * mappings on resume */ ++ struct list_head bound_list; ++ /** ++ * List of objects which are not bound to the GTT (thus ++ * are idle and not used by the GPU). These objects may or may ++ * not actually have any pages attached. ++ */ ++ struct list_head unbound_list; ++ ++ /** List of all objects in gtt_space, currently mmaped by userspace. ++ * All objects within this list must also be on bound_list. ++ */ ++ struct list_head userfault_list; ++ ++ /** ++ * List of objects which are pending destruction. ++ */ ++ struct llist_head free_list; ++ struct work_struct free_work; ++ spinlock_t free_lock; ++ /** ++ * Count of objects pending destructions. Used to skip needlessly ++ * waiting on an RCU barrier if no objects are waiting to be freed. ++ */ ++ atomic_t free_count; ++ ++ /** ++ * Small stash of WC pages ++ */ ++ struct pagestash wc_stash; ++ ++ /** ++ * tmpfs instance used for shmem backed objects ++ */ ++ struct vfsmount *gemfs; ++ ++ /** PPGTT used for aliasing the PPGTT with the GTT */ ++ struct i915_hw_ppgtt *aliasing_ppgtt; ++ ++ struct notifier_block oom_notifier; ++ struct notifier_block vmap_notifier; ++ struct shrinker shrinker; ++ ++ /** LRU list of objects with fence regs on them. */ ++ struct list_head fence_list; ++ ++ /** ++ * Workqueue to fault in userptr pages, flushed by the execbuf ++ * when required but otherwise left to userspace to try again ++ * on EAGAIN. ++ */ ++ struct workqueue_struct *userptr_wq; ++ ++ u64 unordered_timeline; ++ ++ /* the indicator for dispatch video commands on two BSD rings */ ++ atomic_t bsd_engine_dispatch_index; ++ ++ /** Bit 6 swizzling required for X tiling */ ++ u32 bit_6_swizzle_x; ++ /** Bit 6 swizzling required for Y tiling */ ++ u32 bit_6_swizzle_y; ++ ++ /* accounting, useful for userland debugging */ ++ spinlock_t object_stat_lock; ++ u64 object_memory; ++ u32 object_count; ++}; ++ ++#define I915_IDLE_ENGINES_TIMEOUT (200) /* in ms */ ++ ++#define I915_RESET_TIMEOUT (10 * HZ) /* 10s */ ++#define I915_FENCE_TIMEOUT (10 * HZ) /* 10s */ ++ ++#define I915_ENGINE_DEAD_TIMEOUT (4 * HZ) /* Seqno, head and subunits dead */ ++#define I915_SEQNO_DEAD_TIMEOUT (12 * HZ) /* Seqno dead with active head */ ++ ++#define I915_ENGINE_WEDGED_TIMEOUT (60 * HZ) /* Reset but no recovery? */ ++ ++struct ddi_vbt_port_info { ++ int max_tmds_clock; ++ ++ /* ++ * This is an index in the HDMI/DVI DDI buffer translation table. ++ * The special value HDMI_LEVEL_SHIFT_UNKNOWN means the VBT didn't ++ * populate this field. ++ */ ++#define HDMI_LEVEL_SHIFT_UNKNOWN 0xff ++ u8 hdmi_level_shift; ++ ++ u8 present:1; ++ u8 supports_dvi:1; ++ u8 supports_hdmi:1; ++ u8 supports_dp:1; ++ u8 supports_edp:1; ++ u8 supports_typec_usb:1; ++ u8 supports_tbt:1; ++ ++ u8 alternate_aux_channel; ++ u8 alternate_ddc_pin; ++ ++ u8 dp_boost_level; ++ u8 hdmi_boost_level; ++ int dp_max_link_rate; /* 0 for not limited by VBT */ ++}; ++ ++enum psr_lines_to_wait { ++ PSR_0_LINES_TO_WAIT = 0, ++ PSR_1_LINE_TO_WAIT, ++ PSR_4_LINES_TO_WAIT, ++ PSR_8_LINES_TO_WAIT ++}; ++ ++struct intel_vbt_data { ++ struct drm_display_mode *lfp_lvds_vbt_mode; /* if any */ ++ struct drm_display_mode *sdvo_lvds_vbt_mode; /* if any */ ++ ++ /* Feature bits */ ++ unsigned int int_tv_support:1; ++ unsigned int lvds_dither:1; ++ unsigned int int_crt_support:1; ++ unsigned int lvds_use_ssc:1; ++ unsigned int int_lvds_support:1; ++ unsigned int display_clock_mode:1; ++ unsigned int fdi_rx_polarity_inverted:1; ++ unsigned int panel_type:4; ++ int lvds_ssc_freq; ++ unsigned int bios_lvds_val; /* initial [PCH_]LVDS reg val in VBIOS */ ++ enum drm_panel_orientation orientation; ++ ++ enum drrs_support_type drrs_type; ++ ++ struct { ++ int rate; ++ int lanes; ++ int preemphasis; ++ int vswing; ++ bool low_vswing; ++ bool initialized; ++ int bpp; ++ struct edp_power_seq pps; ++ } edp; ++ ++ struct { ++ bool enable; ++ bool full_link; ++ bool require_aux_wakeup; ++ int idle_frames; ++ enum psr_lines_to_wait lines_to_wait; ++ int tp1_wakeup_time_us; ++ int tp2_tp3_wakeup_time_us; ++ int psr2_tp2_tp3_wakeup_time_us; ++ } psr; ++ ++ struct { ++ u16 pwm_freq_hz; ++ bool present; ++ bool active_low_pwm; ++ u8 min_brightness; /* min_brightness/255 of max */ ++ u8 controller; /* brightness controller number */ ++ enum intel_backlight_type type; ++ } backlight; ++ ++ /* MIPI DSI */ ++ struct { ++ u16 panel_id; ++ struct mipi_config *config; ++ struct mipi_pps_data *pps; ++ u16 bl_ports; ++ u16 cabc_ports; ++ u8 seq_version; ++ u32 size; ++ u8 *data; ++ const u8 *sequence[MIPI_SEQ_MAX]; ++ u8 *deassert_seq; /* Used by fixup_mipi_sequences() */ ++ enum drm_panel_orientation orientation; ++ } dsi; ++ ++ int crt_ddc_pin; ++ ++ int child_dev_num; ++ struct child_device_config *child_dev; ++ ++ struct ddi_vbt_port_info ddi_port_info[I915_MAX_PORTS]; ++ struct sdvo_device_mapping sdvo_mappings[2]; ++}; ++ ++enum intel_ddb_partitioning { ++ INTEL_DDB_PART_1_2, ++ INTEL_DDB_PART_5_6, /* IVB+ */ ++}; ++ ++struct intel_wm_level { ++ bool enable; ++ u32 pri_val; ++ u32 spr_val; ++ u32 cur_val; ++ u32 fbc_val; ++}; ++ ++struct ilk_wm_values { ++ u32 wm_pipe[3]; ++ u32 wm_lp[3]; ++ u32 wm_lp_spr[3]; ++ u32 wm_linetime[3]; ++ bool enable_fbc_wm; ++ enum intel_ddb_partitioning partitioning; ++}; ++ ++struct g4x_pipe_wm { ++ u16 plane[I915_MAX_PLANES]; ++ u16 fbc; ++}; ++ ++struct g4x_sr_wm { ++ u16 plane; ++ u16 cursor; ++ u16 fbc; ++}; ++ ++struct vlv_wm_ddl_values { ++ u8 plane[I915_MAX_PLANES]; ++}; ++ ++struct vlv_wm_values { ++ struct g4x_pipe_wm pipe[3]; ++ struct g4x_sr_wm sr; ++ struct vlv_wm_ddl_values ddl[3]; ++ u8 level; ++ bool cxsr; ++}; ++ ++struct g4x_wm_values { ++ struct g4x_pipe_wm pipe[2]; ++ struct g4x_sr_wm sr; ++ struct g4x_sr_wm hpll; ++ bool cxsr; ++ bool hpll_en; ++ bool fbc_en; ++}; ++ ++struct skl_ddb_entry { ++ u16 start, end; /* in number of blocks, 'end' is exclusive */ ++}; ++ ++static inline u16 skl_ddb_entry_size(const struct skl_ddb_entry *entry) ++{ ++ return entry->end - entry->start; ++} ++ ++static inline bool skl_ddb_entry_equal(const struct skl_ddb_entry *e1, ++ const struct skl_ddb_entry *e2) ++{ ++ if (e1->start == e2->start && e1->end == e2->end) ++ return true; ++ ++ return false; ++} ++ ++struct skl_ddb_allocation { ++ u8 enabled_slices; /* GEN11 has configurable 2 slices */ ++}; ++ ++struct skl_ddb_values { ++ unsigned dirty_pipes; ++ struct skl_ddb_allocation ddb; ++}; ++ ++struct skl_wm_level { ++ u16 min_ddb_alloc; ++ u16 plane_res_b; ++ u8 plane_res_l; ++ bool plane_en; ++ bool ignore_lines; ++}; ++ ++/* Stores plane specific WM parameters */ ++struct skl_wm_params { ++ bool x_tiled, y_tiled; ++ bool rc_surface; ++ bool is_planar; ++ u32 width; ++ u8 cpp; ++ u32 plane_pixel_rate; ++ u32 y_min_scanlines; ++ u32 plane_bytes_per_line; ++ uint_fixed_16_16_t plane_blocks_per_line; ++ uint_fixed_16_16_t y_tile_minimum; ++ u32 linetime_us; ++ u32 dbuf_block_size; ++}; ++ ++/* ++ * This struct helps tracking the state needed for runtime PM, which puts the ++ * device in PCI D3 state. Notice that when this happens, nothing on the ++ * graphics device works, even register access, so we don't get interrupts nor ++ * anything else. ++ * ++ * Every piece of our code that needs to actually touch the hardware needs to ++ * either call intel_runtime_pm_get or call intel_display_power_get with the ++ * appropriate power domain. ++ * ++ * Our driver uses the autosuspend delay feature, which means we'll only really ++ * suspend if we stay with zero refcount for a certain amount of time. The ++ * default value is currently very conservative (see intel_runtime_pm_enable), but ++ * it can be changed with the standard runtime PM files from sysfs. ++ * ++ * The irqs_disabled variable becomes true exactly after we disable the IRQs and ++ * goes back to false exactly before we reenable the IRQs. We use this variable ++ * to check if someone is trying to enable/disable IRQs while they're supposed ++ * to be disabled. This shouldn't happen and we'll print some error messages in ++ * case it happens. ++ * ++ * For more, read the Documentation/power/runtime_pm.txt. ++ */ ++struct i915_runtime_pm { ++ atomic_t wakeref_count; ++ bool suspended; ++ bool irqs_enabled; ++ ++#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM) ++ /* ++ * To aide detection of wakeref leaks and general misuse, we ++ * track all wakeref holders. With manual markup (i.e. returning ++ * a cookie to each rpm_get caller which they then supply to their ++ * paired rpm_put) we can remove corresponding pairs of and keep ++ * the array trimmed to active wakerefs. ++ */ ++ struct intel_runtime_pm_debug { ++ spinlock_t lock; ++ ++ depot_stack_handle_t last_acquire; ++ depot_stack_handle_t last_release; ++ ++ depot_stack_handle_t *owners; ++ unsigned long count; ++ } debug; ++#endif ++}; ++ ++enum intel_pipe_crc_source { ++ INTEL_PIPE_CRC_SOURCE_NONE, ++ INTEL_PIPE_CRC_SOURCE_PLANE1, ++ INTEL_PIPE_CRC_SOURCE_PLANE2, ++ INTEL_PIPE_CRC_SOURCE_PLANE3, ++ INTEL_PIPE_CRC_SOURCE_PLANE4, ++ INTEL_PIPE_CRC_SOURCE_PLANE5, ++ INTEL_PIPE_CRC_SOURCE_PLANE6, ++ INTEL_PIPE_CRC_SOURCE_PLANE7, ++ INTEL_PIPE_CRC_SOURCE_PIPE, ++ /* TV/DP on pre-gen5/vlv can't use the pipe source. */ ++ INTEL_PIPE_CRC_SOURCE_TV, ++ INTEL_PIPE_CRC_SOURCE_DP_B, ++ INTEL_PIPE_CRC_SOURCE_DP_C, ++ INTEL_PIPE_CRC_SOURCE_DP_D, ++ INTEL_PIPE_CRC_SOURCE_AUTO, ++ INTEL_PIPE_CRC_SOURCE_MAX, ++}; ++ ++#define INTEL_PIPE_CRC_ENTRIES_NR 128 ++struct intel_pipe_crc { ++ spinlock_t lock; ++ int skipped; ++ enum intel_pipe_crc_source source; ++}; ++ ++struct i915_frontbuffer_tracking { ++ spinlock_t lock; ++ ++ /* ++ * Tracking bits for delayed frontbuffer flushing du to gpu activity or ++ * scheduled flips. ++ */ ++ unsigned busy_bits; ++ unsigned flip_bits; ++}; ++ ++struct i915_virtual_gpu { ++ bool active; ++ u32 caps; ++}; ++ ++/* used in computing the new watermarks state */ ++struct intel_wm_config { ++ unsigned int num_pipes_active; ++ bool sprites_enabled; ++ bool sprites_scaled; ++}; ++ ++struct i915_oa_format { ++ u32 format; ++ int size; ++}; ++ ++struct i915_oa_reg { ++ i915_reg_t addr; ++ u32 value; ++}; ++ ++struct i915_oa_config { ++ char uuid[UUID_STRING_LEN + 1]; ++ int id; ++ ++ const struct i915_oa_reg *mux_regs; ++ u32 mux_regs_len; ++ const struct i915_oa_reg *b_counter_regs; ++ u32 b_counter_regs_len; ++ const struct i915_oa_reg *flex_regs; ++ u32 flex_regs_len; ++ ++ struct attribute_group sysfs_metric; ++ struct attribute *attrs[2]; ++ struct device_attribute sysfs_metric_id; ++ ++ atomic_t ref_count; ++}; ++ ++struct i915_perf_stream; ++ ++/** ++ * struct i915_perf_stream_ops - the OPs to support a specific stream type ++ */ ++struct i915_perf_stream_ops { ++ /** ++ * @enable: Enables the collection of HW samples, either in response to ++ * `I915_PERF_IOCTL_ENABLE` or implicitly called when stream is opened ++ * without `I915_PERF_FLAG_DISABLED`. ++ */ ++ void (*enable)(struct i915_perf_stream *stream); ++ ++ /** ++ * @disable: Disables the collection of HW samples, either in response ++ * to `I915_PERF_IOCTL_DISABLE` or implicitly called before destroying ++ * the stream. ++ */ ++ void (*disable)(struct i915_perf_stream *stream); ++ ++ /** ++ * @poll_wait: Call poll_wait, passing a wait queue that will be woken ++ * once there is something ready to read() for the stream ++ */ ++ void (*poll_wait)(struct i915_perf_stream *stream, ++ struct file *file, ++ poll_table *wait); ++ ++ /** ++ * @wait_unlocked: For handling a blocking read, wait until there is ++ * something to ready to read() for the stream. E.g. wait on the same ++ * wait queue that would be passed to poll_wait(). ++ */ ++ int (*wait_unlocked)(struct i915_perf_stream *stream); ++ ++ /** ++ * @read: Copy buffered metrics as records to userspace ++ * **buf**: the userspace, destination buffer ++ * **count**: the number of bytes to copy, requested by userspace ++ * **offset**: zero at the start of the read, updated as the read ++ * proceeds, it represents how many bytes have been copied so far and ++ * the buffer offset for copying the next record. ++ * ++ * Copy as many buffered i915 perf samples and records for this stream ++ * to userspace as will fit in the given buffer. ++ * ++ * Only write complete records; returning -%ENOSPC if there isn't room ++ * for a complete record. ++ * ++ * Return any error condition that results in a short read such as ++ * -%ENOSPC or -%EFAULT, even though these may be squashed before ++ * returning to userspace. ++ */ ++ int (*read)(struct i915_perf_stream *stream, ++ char __user *buf, ++ size_t count, ++ size_t *offset); ++ ++ /** ++ * @destroy: Cleanup any stream specific resources. ++ * ++ * The stream will always be disabled before this is called. ++ */ ++ void (*destroy)(struct i915_perf_stream *stream); ++}; ++ ++/** ++ * struct i915_perf_stream - state for a single open stream FD ++ */ ++struct i915_perf_stream { ++ /** ++ * @dev_priv: i915 drm device ++ */ ++ struct drm_i915_private *dev_priv; ++ ++ /** ++ * @link: Links the stream into ``&drm_i915_private->streams`` ++ */ ++ struct list_head link; ++ ++ /** ++ * @wakeref: As we keep the device awake while the perf stream is ++ * active, we track our runtime pm reference for later release. ++ */ ++ intel_wakeref_t wakeref; ++ ++ /** ++ * @sample_flags: Flags representing the `DRM_I915_PERF_PROP_SAMPLE_*` ++ * properties given when opening a stream, representing the contents ++ * of a single sample as read() by userspace. ++ */ ++ u32 sample_flags; ++ ++ /** ++ * @sample_size: Considering the configured contents of a sample ++ * combined with the required header size, this is the total size ++ * of a single sample record. ++ */ ++ int sample_size; ++ ++ /** ++ * @ctx: %NULL if measuring system-wide across all contexts or a ++ * specific context that is being monitored. ++ */ ++ struct i915_gem_context *ctx; ++ ++ /** ++ * @enabled: Whether the stream is currently enabled, considering ++ * whether the stream was opened in a disabled state and based ++ * on `I915_PERF_IOCTL_ENABLE` and `I915_PERF_IOCTL_DISABLE` calls. ++ */ ++ bool enabled; ++ ++ /** ++ * @ops: The callbacks providing the implementation of this specific ++ * type of configured stream. ++ */ ++ const struct i915_perf_stream_ops *ops; ++ ++ /** ++ * @oa_config: The OA configuration used by the stream. ++ */ ++ struct i915_oa_config *oa_config; ++}; ++ ++/** ++ * struct i915_oa_ops - Gen specific implementation of an OA unit stream ++ */ ++struct i915_oa_ops { ++ /** ++ * @is_valid_b_counter_reg: Validates register's address for ++ * programming boolean counters for a particular platform. ++ */ ++ bool (*is_valid_b_counter_reg)(struct drm_i915_private *dev_priv, ++ u32 addr); ++ ++ /** ++ * @is_valid_mux_reg: Validates register's address for programming mux ++ * for a particular platform. ++ */ ++ bool (*is_valid_mux_reg)(struct drm_i915_private *dev_priv, u32 addr); ++ ++ /** ++ * @is_valid_flex_reg: Validates register's address for programming ++ * flex EU filtering for a particular platform. ++ */ ++ bool (*is_valid_flex_reg)(struct drm_i915_private *dev_priv, u32 addr); ++ ++ /** ++ * @enable_metric_set: Selects and applies any MUX configuration to set ++ * up the Boolean and Custom (B/C) counters that are part of the ++ * counter reports being sampled. May apply system constraints such as ++ * disabling EU clock gating as required. ++ */ ++ int (*enable_metric_set)(struct i915_perf_stream *stream); ++ ++ /** ++ * @disable_metric_set: Remove system constraints associated with using ++ * the OA unit. ++ */ ++ void (*disable_metric_set)(struct drm_i915_private *dev_priv); ++ ++ /** ++ * @oa_enable: Enable periodic sampling ++ */ ++ void (*oa_enable)(struct i915_perf_stream *stream); ++ ++ /** ++ * @oa_disable: Disable periodic sampling ++ */ ++ void (*oa_disable)(struct i915_perf_stream *stream); ++ ++ /** ++ * @read: Copy data from the circular OA buffer into a given userspace ++ * buffer. ++ */ ++ int (*read)(struct i915_perf_stream *stream, ++ char __user *buf, ++ size_t count, ++ size_t *offset); ++ ++ /** ++ * @oa_hw_tail_read: read the OA tail pointer register ++ * ++ * In particular this enables us to share all the fiddly code for ++ * handling the OA unit tail pointer race that affects multiple ++ * generations. ++ */ ++ u32 (*oa_hw_tail_read)(struct drm_i915_private *dev_priv); ++}; ++ ++struct intel_cdclk_state { ++ unsigned int cdclk, vco, ref, bypass; ++ u8 voltage_level; ++}; ++ ++struct drm_i915_private { ++ struct drm_device drm; ++ ++ const struct intel_device_info __info; /* Use INTEL_INFO() to access. */ ++ struct intel_runtime_info __runtime; /* Use RUNTIME_INFO() to access. */ ++ struct intel_driver_caps caps; ++ ++ /** ++ * Data Stolen Memory - aka "i915 stolen memory" gives us the start and ++ * end of stolen which we can optionally use to create GEM objects ++ * backed by stolen memory. Note that stolen_usable_size tells us ++ * exactly how much of this we are actually allowed to use, given that ++ * some portion of it is in fact reserved for use by hardware functions. ++ */ ++ struct resource dsm; ++ /** ++ * Reseved portion of Data Stolen Memory ++ */ ++ struct resource dsm_reserved; ++ ++ /* ++ * Stolen memory is segmented in hardware with different portions ++ * offlimits to certain functions. ++ * ++ * The drm_mm is initialised to the total accessible range, as found ++ * from the PCI config. On Broadwell+, this is further restricted to ++ * avoid the first page! The upper end of stolen memory is reserved for ++ * hardware functions and similarly removed from the accessible range. ++ */ ++ resource_size_t stolen_usable_size; /* Total size minus reserved ranges */ ++ ++ struct intel_uncore uncore; ++ ++ struct i915_virtual_gpu vgpu; ++ ++ struct intel_gvt *gvt; ++ ++ struct intel_wopcm wopcm; ++ ++ struct intel_huc huc; ++ struct intel_guc guc; ++ ++ struct intel_csr csr; ++ ++ struct intel_gmbus gmbus[GMBUS_NUM_PINS]; ++ ++ /** gmbus_mutex protects against concurrent usage of the single hw gmbus ++ * controller on different i2c buses. */ ++ struct mutex gmbus_mutex; ++ ++ /** ++ * Base address of where the gmbus and gpio blocks are located (either ++ * on PCH or on SoC for platforms without PCH). ++ */ ++ u32 gpio_mmio_base; ++ ++ /* MMIO base address for MIPI regs */ ++ u32 mipi_mmio_base; ++ ++ u32 psr_mmio_base; ++ ++ u32 pps_mmio_base; ++ ++ wait_queue_head_t gmbus_wait_queue; ++ ++ struct pci_dev *bridge_dev; ++ struct intel_engine_cs *engine[I915_NUM_ENGINES]; ++ /* Context used internally to idle the GPU and setup initial state */ ++ struct i915_gem_context *kernel_context; ++ /* Context only to be used for injecting preemption commands */ ++ struct i915_gem_context *preempt_context; ++ struct intel_engine_cs *engine_class[MAX_ENGINE_CLASS + 1] ++ [MAX_ENGINE_INSTANCE + 1]; ++ ++ struct resource mch_res; ++ ++ /* protects the irq masks */ ++ spinlock_t irq_lock; ++ ++ bool display_irqs_enabled; ++ ++ /* To control wakeup latency, e.g. for irq-driven dp aux transfers. */ ++ struct pm_qos_request pm_qos; ++ ++ /* Sideband mailbox protection */ ++ struct mutex sb_lock; ++ ++ /** Cached value of IMR to avoid reads in updating the bitfield */ ++ union { ++ u32 irq_mask; ++ u32 de_irq_mask[I915_MAX_PIPES]; ++ }; ++ u32 gt_irq_mask; ++ u32 pm_imr; ++ u32 pm_ier; ++ u32 pm_rps_events; ++ u32 pm_guc_events; ++ u32 pipestat_irq_mask[I915_MAX_PIPES]; ++ ++ struct i915_hotplug hotplug; ++ struct intel_fbc fbc; ++ struct i915_drrs drrs; ++ struct intel_opregion opregion; ++ struct intel_vbt_data vbt; ++ ++ bool preserve_bios_swizzle; ++ ++ /* overlay */ ++ struct intel_overlay *overlay; ++ ++ /* backlight registers and fields in struct intel_panel */ ++ struct mutex backlight_lock; ++ ++ /* LVDS info */ ++ bool no_aux_handshake; ++ ++ /* protects panel power sequencer state */ ++ struct mutex pps_mutex; ++ ++ struct drm_i915_fence_reg fence_regs[I915_MAX_NUM_FENCES]; /* assume 965 */ ++ int num_fence_regs; /* 8 on pre-965, 16 otherwise */ ++ ++ unsigned int fsb_freq, mem_freq, is_ddr3; ++ unsigned int skl_preferred_vco_freq; ++ unsigned int max_cdclk_freq; ++ ++ unsigned int max_dotclk_freq; ++ unsigned int rawclk_freq; ++ unsigned int hpll_freq; ++ unsigned int fdi_pll_freq; ++ unsigned int czclk_freq; ++ ++ struct { ++ /* ++ * The current logical cdclk state. ++ * See intel_atomic_state.cdclk.logical ++ * ++ * For reading holding any crtc lock is sufficient, ++ * for writing must hold all of them. ++ */ ++ struct intel_cdclk_state logical; ++ /* ++ * The current actual cdclk state. ++ * See intel_atomic_state.cdclk.actual ++ */ ++ struct intel_cdclk_state actual; ++ /* The current hardware cdclk state */ ++ struct intel_cdclk_state hw; ++ ++ int force_min_cdclk; ++ } cdclk; ++ ++ /** ++ * wq - Driver workqueue for GEM. ++ * ++ * NOTE: Work items scheduled here are not allowed to grab any modeset ++ * locks, for otherwise the flushing done in the pageflip code will ++ * result in deadlocks. ++ */ ++ struct workqueue_struct *wq; ++ ++ /* ordered wq for modesets */ ++ struct workqueue_struct *modeset_wq; ++ ++ /* Display functions */ ++ struct drm_i915_display_funcs display; ++ ++ /* PCH chipset type */ ++ enum intel_pch pch_type; ++ unsigned short pch_id; ++ ++ unsigned long quirks; ++ ++ struct drm_atomic_state *modeset_restore_state; ++ struct drm_modeset_acquire_ctx reset_ctx; ++ ++ struct i915_ggtt ggtt; /* VM representing the global address space */ ++ ++ struct i915_gem_mm mm; ++ DECLARE_HASHTABLE(mm_structs, 7); ++ struct mutex mm_lock; ++ ++ struct intel_ppat ppat; ++ ++ /* Kernel Modesetting */ ++ ++ struct intel_crtc *plane_to_crtc_mapping[I915_MAX_PIPES]; ++ struct intel_crtc *pipe_to_crtc_mapping[I915_MAX_PIPES]; ++ ++#ifdef CONFIG_DEBUG_FS ++ struct intel_pipe_crc pipe_crc[I915_MAX_PIPES]; ++#endif ++ ++ /* dpll and cdclk state is protected by connection_mutex */ ++ int num_shared_dpll; ++ struct intel_shared_dpll shared_dplls[I915_NUM_PLLS]; ++ const struct intel_dpll_mgr *dpll_mgr; ++ ++ /* ++ * dpll_lock serializes intel_{prepare,enable,disable}_shared_dpll. ++ * Must be global rather than per dpll, because on some platforms ++ * plls share registers. ++ */ ++ struct mutex dpll_lock; ++ ++ unsigned int active_crtcs; ++ /* minimum acceptable cdclk for each pipe */ ++ int min_cdclk[I915_MAX_PIPES]; ++ /* minimum acceptable voltage level for each pipe */ ++ u8 min_voltage_level[I915_MAX_PIPES]; ++ ++ int dpio_phy_iosf_port[I915_NUM_PHYS_VLV]; ++ ++ struct i915_wa_list gt_wa_list; ++ ++ struct i915_frontbuffer_tracking fb_tracking; ++ ++ struct intel_atomic_helper { ++ struct llist_head free_list; ++ struct work_struct free_work; ++ } atomic_helper; ++ ++ u16 orig_clock; ++ ++ bool mchbar_need_disable; ++ ++ struct intel_l3_parity l3_parity; ++ ++ /* ++ * edram size in MB. ++ * Cannot be determined by PCIID. You must always read a register. ++ */ ++ u32 edram_size_mb; ++ ++ /* ++ * Protects RPS/RC6 register access and PCU communication. ++ * Must be taken after struct_mutex if nested. Note that ++ * this lock may be held for long periods of time when ++ * talking to hw - so only take it when talking to hw! ++ */ ++ struct mutex pcu_lock; ++ ++ /* gen6+ GT PM state */ ++ struct intel_gen6_power_mgmt gt_pm; ++ ++ /* ilk-only ips/rps state. Everything in here is protected by the global ++ * mchdev_lock in intel_pm.c */ ++ struct intel_ilk_power_mgmt ips; ++ ++ struct i915_power_domains power_domains; ++ ++ struct i915_psr psr; ++ ++ struct i915_gpu_error gpu_error; ++ ++ struct drm_i915_gem_object *vlv_pctx; ++ ++ /* list of fbdev register on this device */ ++ struct intel_fbdev *fbdev; ++ struct work_struct fbdev_suspend_work; ++ ++ struct drm_property *broadcast_rgb_property; ++ struct drm_property *force_audio_property; ++ ++ /* hda/i915 audio component */ ++ struct i915_audio_component *audio_component; ++ bool audio_component_registered; ++ /** ++ * av_mutex - mutex for audio/video sync ++ * ++ */ ++ struct mutex av_mutex; ++ int audio_power_refcount; ++ ++ struct { ++ struct mutex mutex; ++ struct list_head list; ++ struct llist_head free_list; ++ struct work_struct free_work; ++ ++ /* The hw wants to have a stable context identifier for the ++ * lifetime of the context (for OA, PASID, faults, etc). ++ * This is limited in execlists to 21 bits. ++ */ ++ struct ida hw_ida; ++#define MAX_CONTEXT_HW_ID (1<<21) /* exclusive */ ++#define MAX_GUC_CONTEXT_HW_ID (1 << 20) /* exclusive */ ++#define GEN11_MAX_CONTEXT_HW_ID (1<<11) /* exclusive */ ++ struct list_head hw_id_list; ++ } contexts; ++ ++ u32 fdi_rx_config; ++ ++ /* Shadow for DISPLAY_PHY_CONTROL which can't be safely read */ ++ u32 chv_phy_control; ++ /* ++ * Shadows for CHV DPLL_MD regs to keep the state ++ * checker somewhat working in the presence hardware ++ * crappiness (can't read out DPLL_MD for pipes B & C). ++ */ ++ u32 chv_dpll_md[I915_MAX_PIPES]; ++ u32 bxt_phy_grc; ++ ++ u32 suspend_count; ++ bool power_domains_suspended; ++ struct i915_suspend_saved_registers regfile; ++ struct vlv_s0ix_state vlv_s0ix_state; ++ ++ enum { ++ I915_SAGV_UNKNOWN = 0, ++ I915_SAGV_DISABLED, ++ I915_SAGV_ENABLED, ++ I915_SAGV_NOT_CONTROLLED ++ } sagv_status; ++ ++ struct { ++ /* ++ * Raw watermark latency values: ++ * in 0.1us units for WM0, ++ * in 0.5us units for WM1+. ++ */ ++ /* primary */ ++ u16 pri_latency[5]; ++ /* sprite */ ++ u16 spr_latency[5]; ++ /* cursor */ ++ u16 cur_latency[5]; ++ /* ++ * Raw watermark memory latency values ++ * for SKL for all 8 levels ++ * in 1us units. ++ */ ++ u16 skl_latency[8]; ++ ++ /* current hardware state */ ++ union { ++ struct ilk_wm_values hw; ++ struct skl_ddb_values skl_hw; ++ struct vlv_wm_values vlv; ++ struct g4x_wm_values g4x; ++ }; ++ ++ u8 max_level; ++ ++ /* ++ * Should be held around atomic WM register writing; also ++ * protects * intel_crtc->wm.active and ++ * cstate->wm.need_postvbl_update. ++ */ ++ struct mutex wm_mutex; ++ ++ /* ++ * Set during HW readout of watermarks/DDB. Some platforms ++ * need to know when we're still using BIOS-provided values ++ * (which we don't fully trust). ++ */ ++ bool distrust_bios_wm; ++ } wm; ++ ++ struct dram_info { ++ bool valid; ++ bool is_16gb_dimm; ++ u8 num_channels; ++ u8 ranks; ++ u32 bandwidth_kbps; ++ bool symmetric_memory; ++ enum intel_dram_type { ++ INTEL_DRAM_UNKNOWN, ++ INTEL_DRAM_DDR3, ++ INTEL_DRAM_DDR4, ++ INTEL_DRAM_LPDDR3, ++ INTEL_DRAM_LPDDR4 ++ } type; ++ } dram_info; ++ ++ struct i915_runtime_pm runtime_pm; ++ ++ struct { ++ bool initialized; ++ ++ struct kobject *metrics_kobj; ++ struct ctl_table_header *sysctl_header; ++ ++ /* ++ * Lock associated with adding/modifying/removing OA configs ++ * in dev_priv->perf.metrics_idr. ++ */ ++ struct mutex metrics_lock; ++ ++ /* ++ * List of dynamic configurations, you need to hold ++ * dev_priv->perf.metrics_lock to access it. ++ */ ++ struct idr metrics_idr; ++ ++ /* ++ * Lock associated with anything below within this structure ++ * except exclusive_stream. ++ */ ++ struct mutex lock; ++ struct list_head streams; ++ ++ struct { ++ /* ++ * The stream currently using the OA unit. If accessed ++ * outside a syscall associated to its file ++ * descriptor, you need to hold ++ * dev_priv->drm.struct_mutex. ++ */ ++ struct i915_perf_stream *exclusive_stream; ++ ++ struct intel_context *pinned_ctx; ++ u32 specific_ctx_id; ++ u32 specific_ctx_id_mask; ++ ++ struct hrtimer poll_check_timer; ++ wait_queue_head_t poll_wq; ++ bool pollin; ++ ++ /** ++ * For rate limiting any notifications of spurious ++ * invalid OA reports ++ */ ++ struct ratelimit_state spurious_report_rs; ++ ++ bool periodic; ++ int period_exponent; ++ ++ struct i915_oa_config test_config; ++ ++ struct { ++ struct i915_vma *vma; ++ u8 *vaddr; ++ u32 last_ctx_id; ++ int format; ++ int format_size; ++ ++ /** ++ * Locks reads and writes to all head/tail state ++ * ++ * Consider: the head and tail pointer state ++ * needs to be read consistently from a hrtimer ++ * callback (atomic context) and read() fop ++ * (user context) with tail pointer updates ++ * happening in atomic context and head updates ++ * in user context and the (unlikely) ++ * possibility of read() errors needing to ++ * reset all head/tail state. ++ * ++ * Note: Contention or performance aren't ++ * currently a significant concern here ++ * considering the relatively low frequency of ++ * hrtimer callbacks (5ms period) and that ++ * reads typically only happen in response to a ++ * hrtimer event and likely complete before the ++ * next callback. ++ * ++ * Note: This lock is not held *while* reading ++ * and copying data to userspace so the value ++ * of head observed in htrimer callbacks won't ++ * represent any partial consumption of data. ++ */ ++ spinlock_t ptr_lock; ++ ++ /** ++ * One 'aging' tail pointer and one 'aged' ++ * tail pointer ready to used for reading. ++ * ++ * Initial values of 0xffffffff are invalid ++ * and imply that an update is required ++ * (and should be ignored by an attempted ++ * read) ++ */ ++ struct { ++ u32 offset; ++ } tails[2]; ++ ++ /** ++ * Index for the aged tail ready to read() ++ * data up to. ++ */ ++ unsigned int aged_tail_idx; ++ ++ /** ++ * A monotonic timestamp for when the current ++ * aging tail pointer was read; used to ++ * determine when it is old enough to trust. ++ */ ++ u64 aging_timestamp; ++ ++ /** ++ * Although we can always read back the head ++ * pointer register, we prefer to avoid ++ * trusting the HW state, just to avoid any ++ * risk that some hardware condition could ++ * somehow bump the head pointer unpredictably ++ * and cause us to forward the wrong OA buffer ++ * data to userspace. ++ */ ++ u32 head; ++ } oa_buffer; ++ ++ u32 gen7_latched_oastatus1; ++ u32 ctx_oactxctrl_offset; ++ u32 ctx_flexeu0_offset; ++ ++ /** ++ * The RPT_ID/reason field for Gen8+ includes a bit ++ * to determine if the CTX ID in the report is valid ++ * but the specific bit differs between Gen 8 and 9 ++ */ ++ u32 gen8_valid_ctx_bit; ++ ++ struct i915_oa_ops ops; ++ const struct i915_oa_format *oa_formats; ++ } oa; ++ } perf; ++ ++ /* Abstract the submission mechanism (legacy ringbuffer or execlists) away */ ++ struct { ++ void (*cleanup_engine)(struct intel_engine_cs *engine); ++ ++ struct i915_gt_timelines { ++ struct mutex mutex; /* protects list, tainted by GPU */ ++ struct list_head active_list; ++ ++ /* Pack multiple timelines' seqnos into the same page */ ++ spinlock_t hwsp_lock; ++ struct list_head hwsp_free_list; ++ } timelines; ++ ++ intel_engine_mask_t active_engines; ++ struct list_head active_rings; ++ struct list_head closed_vma; ++ u32 active_requests; ++ ++ /** ++ * Is the GPU currently considered idle, or busy executing ++ * userspace requests? Whilst idle, we allow runtime power ++ * management to power down the hardware and display clocks. ++ * In order to reduce the effect on performance, there ++ * is a slight delay before we do so. ++ */ ++ intel_wakeref_t awake; ++ ++ /** ++ * We leave the user IRQ off as much as possible, ++ * but this means that requests will finish and never ++ * be retired once the system goes idle. Set a timer to ++ * fire periodically while the ring is running. When it ++ * fires, go retire requests. ++ */ ++ struct delayed_work retire_work; ++ ++ /** ++ * When we detect an idle GPU, we want to turn on ++ * powersaving features. So once we see that there ++ * are no more requests outstanding and no more ++ * arrive within a small period of time, we fire ++ * off the idle_work. ++ */ ++ struct delayed_work idle_work; ++ ++ ktime_t last_init_time; ++ ++ struct i915_vma *scratch; ++ } gt; ++ ++ /* For i945gm vblank irq vs. C3 workaround */ ++ struct { ++ struct work_struct work; ++ struct pm_qos_request pm_qos; ++ u8 c3_disable_latency; ++ u8 enabled; ++ } i945gm_vblank; ++ ++ /* perform PHY state sanity checks? */ ++ bool chv_phy_assert[2]; ++ ++ bool ipc_enabled; ++ ++ /* Used to save the pipe-to-encoder mapping for audio */ ++ struct intel_encoder *av_enc_map[I915_MAX_PIPES]; ++ ++ /* necessary resource sharing with HDMI LPE audio driver. */ ++ struct { ++ struct platform_device *platdev; ++ int irq; ++ } lpe_audio; ++ ++ struct i915_pmu pmu; ++ ++ struct i915_hdcp_comp_master *hdcp_master; ++ bool hdcp_comp_added; ++ ++ /* Mutex to protect the above hdcp component related values. */ ++ struct mutex hdcp_comp_mutex; ++ ++ /* ++ * NOTE: This is the dri1/ums dungeon, don't add stuff here. Your patch ++ * will be rejected. Instead look for a better place. ++ */ ++}; ++ ++struct dram_dimm_info { ++ u8 size, width, ranks; ++}; ++ ++struct dram_channel_info { ++ struct dram_dimm_info dimm_l, dimm_s; ++ u8 ranks; ++ bool is_16gb_dimm; ++}; ++ ++static inline struct drm_i915_private *to_i915(const struct drm_device *dev) ++{ ++ return container_of(dev, struct drm_i915_private, drm); ++} ++ ++static inline struct drm_i915_private *kdev_to_i915(struct device *kdev) ++{ ++ return to_i915(dev_get_drvdata(kdev)); ++} ++ ++static inline struct drm_i915_private *wopcm_to_i915(struct intel_wopcm *wopcm) ++{ ++ return container_of(wopcm, struct drm_i915_private, wopcm); ++} ++ ++static inline struct drm_i915_private *guc_to_i915(struct intel_guc *guc) ++{ ++ return container_of(guc, struct drm_i915_private, guc); ++} ++ ++static inline struct drm_i915_private *huc_to_i915(struct intel_huc *huc) ++{ ++ return container_of(huc, struct drm_i915_private, huc); ++} ++ ++static inline struct drm_i915_private *uncore_to_i915(struct intel_uncore *uncore) ++{ ++ return container_of(uncore, struct drm_i915_private, uncore); ++} ++ ++/* Simple iterator over all initialised engines */ ++#define for_each_engine(engine__, dev_priv__, id__) \ ++ for ((id__) = 0; \ ++ (id__) < I915_NUM_ENGINES; \ ++ (id__)++) \ ++ for_each_if ((engine__) = (dev_priv__)->engine[(id__)]) ++ ++/* Iterator over subset of engines selected by mask */ ++#define for_each_engine_masked(engine__, dev_priv__, mask__, tmp__) \ ++ for ((tmp__) = (mask__) & INTEL_INFO(dev_priv__)->engine_mask; \ ++ (tmp__) ? \ ++ ((engine__) = (dev_priv__)->engine[__mask_next_bit(tmp__)]), 1 : \ ++ 0;) ++ ++enum hdmi_force_audio { ++ HDMI_AUDIO_OFF_DVI = -2, /* no aux data for HDMI-DVI converter */ ++ HDMI_AUDIO_OFF, /* force turn off HDMI audio */ ++ HDMI_AUDIO_AUTO, /* trust EDID */ ++ HDMI_AUDIO_ON, /* force turn on HDMI audio */ ++}; ++ ++#define I915_GTT_OFFSET_NONE ((u32)-1) ++ ++/* ++ * Frontbuffer tracking bits. Set in obj->frontbuffer_bits while a gem bo is ++ * considered to be the frontbuffer for the given plane interface-wise. This ++ * doesn't mean that the hw necessarily already scans it out, but that any ++ * rendering (by the cpu or gpu) will land in the frontbuffer eventually. ++ * ++ * We have one bit per pipe and per scanout plane type. ++ */ ++#define INTEL_FRONTBUFFER_BITS_PER_PIPE 8 ++#define INTEL_FRONTBUFFER(pipe, plane_id) ({ \ ++ BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES > 32); \ ++ BUILD_BUG_ON(I915_MAX_PLANES > INTEL_FRONTBUFFER_BITS_PER_PIPE); \ ++ BIT((plane_id) + INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)); \ ++}) ++#define INTEL_FRONTBUFFER_OVERLAY(pipe) \ ++ BIT(INTEL_FRONTBUFFER_BITS_PER_PIPE - 1 + INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)) ++#define INTEL_FRONTBUFFER_ALL_MASK(pipe) \ ++ GENMASK(INTEL_FRONTBUFFER_BITS_PER_PIPE * ((pipe) + 1) - 1, \ ++ INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)) ++ ++/* ++ * Optimised SGL iterator for GEM objects ++ */ ++static __always_inline struct sgt_iter { ++ struct scatterlist *sgp; ++ union { ++ unsigned long pfn; ++ dma_addr_t dma; ++ }; ++ unsigned int curr; ++ unsigned int max; ++} __sgt_iter(struct scatterlist *sgl, bool dma) { ++ struct sgt_iter s = { .sgp = sgl }; ++ ++ if (s.sgp) { ++ s.max = s.curr = s.sgp->offset; ++ s.max += s.sgp->length; ++ if (dma) ++ s.dma = sg_dma_address(s.sgp); ++ else ++ s.pfn = page_to_pfn(sg_page(s.sgp)); ++ } ++ ++ return s; ++} ++ ++static inline struct scatterlist *____sg_next(struct scatterlist *sg) ++{ ++ ++sg; ++ if (unlikely(sg_is_chain(sg))) ++ sg = sg_chain_ptr(sg); ++ return sg; ++} ++ ++/** ++ * __sg_next - return the next scatterlist entry in a list ++ * @sg: The current sg entry ++ * ++ * Description: ++ * If the entry is the last, return NULL; otherwise, step to the next ++ * element in the array (@sg@+1). If that's a chain pointer, follow it; ++ * otherwise just return the pointer to the current element. ++ **/ ++static inline struct scatterlist *__sg_next(struct scatterlist *sg) ++{ ++ return sg_is_last(sg) ? NULL : ____sg_next(sg); ++} ++ ++/** ++ * for_each_sgt_dma - iterate over the DMA addresses of the given sg_table ++ * @__dmap: DMA address (output) ++ * @__iter: 'struct sgt_iter' (iterator state, internal) ++ * @__sgt: sg_table to iterate over (input) ++ */ ++#define for_each_sgt_dma(__dmap, __iter, __sgt) \ ++ for ((__iter) = __sgt_iter((__sgt)->sgl, true); \ ++ ((__dmap) = (__iter).dma + (__iter).curr); \ ++ (((__iter).curr += I915_GTT_PAGE_SIZE) >= (__iter).max) ? \ ++ (__iter) = __sgt_iter(__sg_next((__iter).sgp), true), 0 : 0) ++ ++/** ++ * for_each_sgt_page - iterate over the pages of the given sg_table ++ * @__pp: page pointer (output) ++ * @__iter: 'struct sgt_iter' (iterator state, internal) ++ * @__sgt: sg_table to iterate over (input) ++ */ ++#define for_each_sgt_page(__pp, __iter, __sgt) \ ++ for ((__iter) = __sgt_iter((__sgt)->sgl, false); \ ++ ((__pp) = (__iter).pfn == 0 ? NULL : \ ++ pfn_to_page((__iter).pfn + ((__iter).curr >> PAGE_SHIFT))); \ ++ (((__iter).curr += PAGE_SIZE) >= (__iter).max) ? \ ++ (__iter) = __sgt_iter(__sg_next((__iter).sgp), false), 0 : 0) ++ ++bool i915_sg_trim(struct sg_table *orig_st); ++ ++static inline unsigned int i915_sg_page_sizes(struct scatterlist *sg) ++{ ++ unsigned int page_sizes; ++ ++ page_sizes = 0; ++ while (sg) { ++ GEM_BUG_ON(sg->offset); ++ GEM_BUG_ON(!IS_ALIGNED(sg->length, PAGE_SIZE)); ++ page_sizes |= sg->length; ++ sg = __sg_next(sg); ++ } ++ ++ return page_sizes; ++} ++ ++static inline unsigned int i915_sg_segment_size(void) ++{ ++ unsigned int size = swiotlb_max_segment(); ++ ++ if (size == 0) ++ return SCATTERLIST_MAX_SEGMENT; ++ ++ size = rounddown(size, PAGE_SIZE); ++ /* swiotlb_max_segment_size can return 1 byte when it means one page. */ ++ if (size < PAGE_SIZE) ++ size = PAGE_SIZE; ++ ++ return size; ++} ++ ++#define INTEL_INFO(dev_priv) (&(dev_priv)->__info) ++#define RUNTIME_INFO(dev_priv) (&(dev_priv)->__runtime) ++#define DRIVER_CAPS(dev_priv) (&(dev_priv)->caps) ++ ++#define INTEL_GEN(dev_priv) (INTEL_INFO(dev_priv)->gen) ++#define INTEL_DEVID(dev_priv) (RUNTIME_INFO(dev_priv)->device_id) ++ ++#define REVID_FOREVER 0xff ++#define INTEL_REVID(dev_priv) ((dev_priv)->drm.pdev->revision) ++ ++#define INTEL_GEN_MASK(s, e) ( \ ++ BUILD_BUG_ON_ZERO(!__builtin_constant_p(s)) + \ ++ BUILD_BUG_ON_ZERO(!__builtin_constant_p(e)) + \ ++ GENMASK((e) - 1, (s) - 1)) ++ ++/* Returns true if Gen is in inclusive range [Start, End] */ ++#define IS_GEN_RANGE(dev_priv, s, e) \ ++ (!!(INTEL_INFO(dev_priv)->gen_mask & INTEL_GEN_MASK((s), (e)))) ++ ++#define IS_GEN(dev_priv, n) \ ++ (BUILD_BUG_ON_ZERO(!__builtin_constant_p(n)) + \ ++ INTEL_INFO(dev_priv)->gen == (n)) ++ ++/* ++ * Return true if revision is in range [since,until] inclusive. ++ * ++ * Use 0 for open-ended since, and REVID_FOREVER for open-ended until. ++ */ ++#define IS_REVID(p, since, until) \ ++ (INTEL_REVID(p) >= (since) && INTEL_REVID(p) <= (until)) ++ ++static __always_inline unsigned int ++__platform_mask_index(const struct intel_runtime_info *info, ++ enum intel_platform p) ++{ ++ const unsigned int pbits = ++ BITS_PER_TYPE(info->platform_mask[0]) - INTEL_SUBPLATFORM_BITS; ++ ++ /* Expand the platform_mask array if this fails. */ ++ BUILD_BUG_ON(INTEL_MAX_PLATFORMS > ++ pbits * ARRAY_SIZE(info->platform_mask)); ++ ++ return p / pbits; ++} ++ ++static __always_inline unsigned int ++__platform_mask_bit(const struct intel_runtime_info *info, ++ enum intel_platform p) ++{ ++ const unsigned int pbits = ++ BITS_PER_TYPE(info->platform_mask[0]) - INTEL_SUBPLATFORM_BITS; ++ ++ return p % pbits + INTEL_SUBPLATFORM_BITS; ++} ++ ++static inline u32 ++intel_subplatform(const struct intel_runtime_info *info, enum intel_platform p) ++{ ++ const unsigned int pi = __platform_mask_index(info, p); ++ ++ return info->platform_mask[pi] & INTEL_SUBPLATFORM_BITS; ++} ++ ++static __always_inline bool ++IS_PLATFORM(const struct drm_i915_private *i915, enum intel_platform p) ++{ ++ const struct intel_runtime_info *info = RUNTIME_INFO(i915); ++ const unsigned int pi = __platform_mask_index(info, p); ++ const unsigned int pb = __platform_mask_bit(info, p); ++ ++ BUILD_BUG_ON(!__builtin_constant_p(p)); ++ ++ return info->platform_mask[pi] & BIT(pb); ++} ++ ++static __always_inline bool ++IS_SUBPLATFORM(const struct drm_i915_private *i915, ++ enum intel_platform p, unsigned int s) ++{ ++ const struct intel_runtime_info *info = RUNTIME_INFO(i915); ++ const unsigned int pi = __platform_mask_index(info, p); ++ const unsigned int pb = __platform_mask_bit(info, p); ++ const unsigned int msb = BITS_PER_TYPE(info->platform_mask[0]) - 1; ++ const u32 mask = info->platform_mask[pi]; ++ ++ BUILD_BUG_ON(!__builtin_constant_p(p)); ++ BUILD_BUG_ON(!__builtin_constant_p(s)); ++ BUILD_BUG_ON((s) >= INTEL_SUBPLATFORM_BITS); ++ ++ /* Shift and test on the MSB position so sign flag can be used. */ ++ return ((mask << (msb - pb)) & (mask << (msb - s))) & BIT(msb); ++} ++ ++#define IS_MOBILE(dev_priv) (INTEL_INFO(dev_priv)->is_mobile) ++ ++#define IS_I830(dev_priv) IS_PLATFORM(dev_priv, INTEL_I830) ++#define IS_I845G(dev_priv) IS_PLATFORM(dev_priv, INTEL_I845G) ++#define IS_I85X(dev_priv) IS_PLATFORM(dev_priv, INTEL_I85X) ++#define IS_I865G(dev_priv) IS_PLATFORM(dev_priv, INTEL_I865G) ++#define IS_I915G(dev_priv) IS_PLATFORM(dev_priv, INTEL_I915G) ++#define IS_I915GM(dev_priv) IS_PLATFORM(dev_priv, INTEL_I915GM) ++#define IS_I945G(dev_priv) IS_PLATFORM(dev_priv, INTEL_I945G) ++#define IS_I945GM(dev_priv) IS_PLATFORM(dev_priv, INTEL_I945GM) ++#define IS_I965G(dev_priv) IS_PLATFORM(dev_priv, INTEL_I965G) ++#define IS_I965GM(dev_priv) IS_PLATFORM(dev_priv, INTEL_I965GM) ++#define IS_G45(dev_priv) IS_PLATFORM(dev_priv, INTEL_G45) ++#define IS_GM45(dev_priv) IS_PLATFORM(dev_priv, INTEL_GM45) ++#define IS_G4X(dev_priv) (IS_G45(dev_priv) || IS_GM45(dev_priv)) ++#define IS_PINEVIEW(dev_priv) IS_PLATFORM(dev_priv, INTEL_PINEVIEW) ++#define IS_G33(dev_priv) IS_PLATFORM(dev_priv, INTEL_G33) ++#define IS_IRONLAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_IRONLAKE) ++#define IS_IRONLAKE_M(dev_priv) \ ++ (IS_PLATFORM(dev_priv, INTEL_IRONLAKE) && IS_MOBILE(dev_priv)) ++#define IS_IVYBRIDGE(dev_priv) IS_PLATFORM(dev_priv, INTEL_IVYBRIDGE) ++#define IS_IVB_GT1(dev_priv) (IS_IVYBRIDGE(dev_priv) && \ ++ INTEL_INFO(dev_priv)->gt == 1) ++#define IS_VALLEYVIEW(dev_priv) IS_PLATFORM(dev_priv, INTEL_VALLEYVIEW) ++#define IS_CHERRYVIEW(dev_priv) IS_PLATFORM(dev_priv, INTEL_CHERRYVIEW) ++#define IS_HASWELL(dev_priv) IS_PLATFORM(dev_priv, INTEL_HASWELL) ++#define IS_BROADWELL(dev_priv) IS_PLATFORM(dev_priv, INTEL_BROADWELL) ++#define IS_SKYLAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_SKYLAKE) ++#define IS_BROXTON(dev_priv) IS_PLATFORM(dev_priv, INTEL_BROXTON) ++#define IS_KABYLAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_KABYLAKE) ++#define IS_GEMINILAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_GEMINILAKE) ++#define IS_COFFEELAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_COFFEELAKE) ++#define IS_CANNONLAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_CANNONLAKE) ++#define IS_ICELAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_ICELAKE) ++#define IS_ELKHARTLAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_ELKHARTLAKE) ++#define IS_HSW_EARLY_SDV(dev_priv) (IS_HASWELL(dev_priv) && \ ++ (INTEL_DEVID(dev_priv) & 0xFF00) == 0x0C00) ++#define IS_BDW_ULT(dev_priv) \ ++ IS_SUBPLATFORM(dev_priv, INTEL_BROADWELL, INTEL_SUBPLATFORM_ULT) ++#define IS_BDW_ULX(dev_priv) \ ++ IS_SUBPLATFORM(dev_priv, INTEL_BROADWELL, INTEL_SUBPLATFORM_ULX) ++#define IS_BDW_GT3(dev_priv) (IS_BROADWELL(dev_priv) && \ ++ INTEL_INFO(dev_priv)->gt == 3) ++#define IS_HSW_ULT(dev_priv) \ ++ IS_SUBPLATFORM(dev_priv, INTEL_HASWELL, INTEL_SUBPLATFORM_ULT) ++#define IS_HSW_GT3(dev_priv) (IS_HASWELL(dev_priv) && \ ++ INTEL_INFO(dev_priv)->gt == 3) ++#define IS_HSW_GT1(dev_priv) (IS_HASWELL(dev_priv) && \ ++ INTEL_INFO(dev_priv)->gt == 1) ++/* ULX machines are also considered ULT. */ ++#define IS_HSW_ULX(dev_priv) \ ++ IS_SUBPLATFORM(dev_priv, INTEL_HASWELL, INTEL_SUBPLATFORM_ULX) ++#define IS_SKL_ULT(dev_priv) \ ++ IS_SUBPLATFORM(dev_priv, INTEL_SKYLAKE, INTEL_SUBPLATFORM_ULT) ++#define IS_SKL_ULX(dev_priv) \ ++ IS_SUBPLATFORM(dev_priv, INTEL_SKYLAKE, INTEL_SUBPLATFORM_ULX) ++#define IS_KBL_ULT(dev_priv) \ ++ IS_SUBPLATFORM(dev_priv, INTEL_KABYLAKE, INTEL_SUBPLATFORM_ULT) ++#define IS_KBL_ULX(dev_priv) \ ++ IS_SUBPLATFORM(dev_priv, INTEL_KABYLAKE, INTEL_SUBPLATFORM_ULX) ++#define IS_AML_ULX(dev_priv) \ ++ (IS_SUBPLATFORM(dev_priv, INTEL_KABYLAKE, INTEL_SUBPLATFORM_AML) || \ ++ IS_SUBPLATFORM(dev_priv, INTEL_COFFEELAKE, INTEL_SUBPLATFORM_AML)) ++#define IS_SKL_GT2(dev_priv) (IS_SKYLAKE(dev_priv) && \ ++ INTEL_INFO(dev_priv)->gt == 2) ++#define IS_SKL_GT3(dev_priv) (IS_SKYLAKE(dev_priv) && \ ++ INTEL_INFO(dev_priv)->gt == 3) ++#define IS_SKL_GT4(dev_priv) (IS_SKYLAKE(dev_priv) && \ ++ INTEL_INFO(dev_priv)->gt == 4) ++#define IS_KBL_GT2(dev_priv) (IS_KABYLAKE(dev_priv) && \ ++ INTEL_INFO(dev_priv)->gt == 2) ++#define IS_KBL_GT3(dev_priv) (IS_KABYLAKE(dev_priv) && \ ++ INTEL_INFO(dev_priv)->gt == 3) ++#define IS_CFL_ULT(dev_priv) \ ++ IS_SUBPLATFORM(dev_priv, INTEL_COFFEELAKE, INTEL_SUBPLATFORM_ULT) ++#define IS_CFL_GT2(dev_priv) (IS_COFFEELAKE(dev_priv) && \ ++ INTEL_INFO(dev_priv)->gt == 2) ++#define IS_CFL_GT3(dev_priv) (IS_COFFEELAKE(dev_priv) && \ ++ INTEL_INFO(dev_priv)->gt == 3) ++#define IS_CNL_WITH_PORT_F(dev_priv) \ ++ IS_SUBPLATFORM(dev_priv, INTEL_CANNONLAKE, INTEL_SUBPLATFORM_PORTF) ++#define IS_ICL_WITH_PORT_F(dev_priv) \ ++ IS_SUBPLATFORM(dev_priv, INTEL_ICELAKE, INTEL_SUBPLATFORM_PORTF) ++ ++#define IS_ALPHA_SUPPORT(intel_info) ((intel_info)->is_alpha_support) ++ ++#define SKL_REVID_A0 0x0 ++#define SKL_REVID_B0 0x1 ++#define SKL_REVID_C0 0x2 ++#define SKL_REVID_D0 0x3 ++#define SKL_REVID_E0 0x4 ++#define SKL_REVID_F0 0x5 ++#define SKL_REVID_G0 0x6 ++#define SKL_REVID_H0 0x7 ++ ++#define IS_SKL_REVID(p, since, until) (IS_SKYLAKE(p) && IS_REVID(p, since, until)) ++ ++#define BXT_REVID_A0 0x0 ++#define BXT_REVID_A1 0x1 ++#define BXT_REVID_B0 0x3 ++#define BXT_REVID_B_LAST 0x8 ++#define BXT_REVID_C0 0x9 ++ ++#define IS_BXT_REVID(dev_priv, since, until) \ ++ (IS_BROXTON(dev_priv) && IS_REVID(dev_priv, since, until)) ++ ++#define KBL_REVID_A0 0x0 ++#define KBL_REVID_B0 0x1 ++#define KBL_REVID_C0 0x2 ++#define KBL_REVID_D0 0x3 ++#define KBL_REVID_E0 0x4 ++ ++#define IS_KBL_REVID(dev_priv, since, until) \ ++ (IS_KABYLAKE(dev_priv) && IS_REVID(dev_priv, since, until)) ++ ++#define GLK_REVID_A0 0x0 ++#define GLK_REVID_A1 0x1 ++ ++#define IS_GLK_REVID(dev_priv, since, until) \ ++ (IS_GEMINILAKE(dev_priv) && IS_REVID(dev_priv, since, until)) ++ ++#define CNL_REVID_A0 0x0 ++#define CNL_REVID_B0 0x1 ++#define CNL_REVID_C0 0x2 ++ ++#define IS_CNL_REVID(p, since, until) \ ++ (IS_CANNONLAKE(p) && IS_REVID(p, since, until)) ++ ++#define ICL_REVID_A0 0x0 ++#define ICL_REVID_A2 0x1 ++#define ICL_REVID_B0 0x3 ++#define ICL_REVID_B2 0x4 ++#define ICL_REVID_C0 0x5 ++ ++#define IS_ICL_REVID(p, since, until) \ ++ (IS_ICELAKE(p) && IS_REVID(p, since, until)) ++ ++#define IS_LP(dev_priv) (INTEL_INFO(dev_priv)->is_lp) ++#define IS_GEN9_LP(dev_priv) (IS_GEN(dev_priv, 9) && IS_LP(dev_priv)) ++#define IS_GEN9_BC(dev_priv) (IS_GEN(dev_priv, 9) && !IS_LP(dev_priv)) ++ ++#define HAS_ENGINE(dev_priv, id) (INTEL_INFO(dev_priv)->engine_mask & BIT(id)) ++ ++#define ENGINE_INSTANCES_MASK(dev_priv, first, count) ({ \ ++ unsigned int first__ = (first); \ ++ unsigned int count__ = (count); \ ++ (INTEL_INFO(dev_priv)->engine_mask & \ ++ GENMASK(first__ + count__ - 1, first__)) >> first__; \ ++}) ++#define VDBOX_MASK(dev_priv) \ ++ ENGINE_INSTANCES_MASK(dev_priv, VCS0, I915_MAX_VCS) ++#define VEBOX_MASK(dev_priv) \ ++ ENGINE_INSTANCES_MASK(dev_priv, VECS0, I915_MAX_VECS) ++ ++#define HAS_LLC(dev_priv) (INTEL_INFO(dev_priv)->has_llc) ++#define HAS_SNOOP(dev_priv) (INTEL_INFO(dev_priv)->has_snoop) ++#define HAS_EDRAM(dev_priv) ((dev_priv)->edram_size_mb) ++#define HAS_WT(dev_priv) ((IS_HASWELL(dev_priv) || \ ++ IS_BROADWELL(dev_priv)) && HAS_EDRAM(dev_priv)) ++ ++#define HWS_NEEDS_PHYSICAL(dev_priv) (INTEL_INFO(dev_priv)->hws_needs_physical) ++ ++#define HAS_LOGICAL_RING_CONTEXTS(dev_priv) \ ++ (INTEL_INFO(dev_priv)->has_logical_ring_contexts) ++#define HAS_LOGICAL_RING_ELSQ(dev_priv) \ ++ (INTEL_INFO(dev_priv)->has_logical_ring_elsq) ++#define HAS_LOGICAL_RING_PREEMPTION(dev_priv) \ ++ (INTEL_INFO(dev_priv)->has_logical_ring_preemption) ++ ++#define HAS_EXECLISTS(dev_priv) HAS_LOGICAL_RING_CONTEXTS(dev_priv) ++ ++#define INTEL_PPGTT(dev_priv) (INTEL_INFO(dev_priv)->ppgtt_type) ++#define HAS_PPGTT(dev_priv) \ ++ (INTEL_PPGTT(dev_priv) != INTEL_PPGTT_NONE) ++#define HAS_FULL_PPGTT(dev_priv) \ ++ (INTEL_PPGTT(dev_priv) >= INTEL_PPGTT_FULL) ++ ++#define HAS_PAGE_SIZES(dev_priv, sizes) ({ \ ++ GEM_BUG_ON((sizes) == 0); \ ++ ((sizes) & ~INTEL_INFO(dev_priv)->page_sizes) == 0; \ ++}) ++ ++#define HAS_OVERLAY(dev_priv) (INTEL_INFO(dev_priv)->display.has_overlay) ++#define OVERLAY_NEEDS_PHYSICAL(dev_priv) \ ++ (INTEL_INFO(dev_priv)->display.overlay_needs_physical) ++ ++/* Early gen2 have a totally busted CS tlb and require pinned batches. */ ++#define HAS_BROKEN_CS_TLB(dev_priv) (IS_I830(dev_priv) || IS_I845G(dev_priv)) ++ ++/* WaRsDisableCoarsePowerGating:skl,cnl */ ++#define NEEDS_WaRsDisableCoarsePowerGating(dev_priv) \ ++ (IS_CANNONLAKE(dev_priv) || \ ++ IS_SKL_GT3(dev_priv) || IS_SKL_GT4(dev_priv)) ++ ++#define HAS_GMBUS_IRQ(dev_priv) (INTEL_GEN(dev_priv) >= 4) ++#define HAS_GMBUS_BURST_READ(dev_priv) (INTEL_GEN(dev_priv) >= 10 || \ ++ IS_GEMINILAKE(dev_priv) || \ ++ IS_KABYLAKE(dev_priv)) ++ ++/* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte ++ * rows, which changed the alignment requirements and fence programming. ++ */ ++#define HAS_128_BYTE_Y_TILING(dev_priv) (!IS_GEN(dev_priv, 2) && \ ++ !(IS_I915G(dev_priv) || \ ++ IS_I915GM(dev_priv))) ++#define SUPPORTS_TV(dev_priv) (INTEL_INFO(dev_priv)->display.supports_tv) ++#define I915_HAS_HOTPLUG(dev_priv) (INTEL_INFO(dev_priv)->display.has_hotplug) ++ ++#define HAS_FW_BLC(dev_priv) (INTEL_GEN(dev_priv) > 2) ++#define HAS_FBC(dev_priv) (INTEL_INFO(dev_priv)->display.has_fbc) ++#define HAS_CUR_FBC(dev_priv) (!HAS_GMCH(dev_priv) && INTEL_GEN(dev_priv) >= 7) ++ ++#define HAS_IPS(dev_priv) (IS_HSW_ULT(dev_priv) || IS_BROADWELL(dev_priv)) ++ ++#define HAS_DP_MST(dev_priv) (INTEL_INFO(dev_priv)->display.has_dp_mst) ++ ++#define HAS_DDI(dev_priv) (INTEL_INFO(dev_priv)->display.has_ddi) ++#define HAS_FPGA_DBG_UNCLAIMED(dev_priv) (INTEL_INFO(dev_priv)->has_fpga_dbg) ++#define HAS_PSR(dev_priv) (INTEL_INFO(dev_priv)->display.has_psr) ++#define HAS_TRANSCODER_EDP(dev_priv) (INTEL_INFO(dev_priv)->trans_offsets[TRANSCODER_EDP] != 0) ++ ++#define HAS_RC6(dev_priv) (INTEL_INFO(dev_priv)->has_rc6) ++#define HAS_RC6p(dev_priv) (INTEL_INFO(dev_priv)->has_rc6p) ++#define HAS_RC6pp(dev_priv) (false) /* HW was never validated */ ++ ++#define HAS_CSR(dev_priv) (INTEL_INFO(dev_priv)->display.has_csr) ++ ++#define HAS_RUNTIME_PM(dev_priv) (INTEL_INFO(dev_priv)->has_runtime_pm) ++#define HAS_64BIT_RELOC(dev_priv) (INTEL_INFO(dev_priv)->has_64bit_reloc) ++ ++#define HAS_IPC(dev_priv) (INTEL_INFO(dev_priv)->display.has_ipc) ++ ++/* ++ * For now, anything with a GuC requires uCode loading, and then supports ++ * command submission once loaded. But these are logically independent ++ * properties, so we have separate macros to test them. ++ */ ++#define HAS_GUC(dev_priv) (INTEL_INFO(dev_priv)->has_guc) ++#define HAS_GUC_CT(dev_priv) (INTEL_INFO(dev_priv)->has_guc_ct) ++#define HAS_GUC_UCODE(dev_priv) (HAS_GUC(dev_priv)) ++#define HAS_GUC_SCHED(dev_priv) (HAS_GUC(dev_priv)) ++ ++/* For now, anything with a GuC has also HuC */ ++#define HAS_HUC(dev_priv) (HAS_GUC(dev_priv)) ++#define HAS_HUC_UCODE(dev_priv) (HAS_GUC(dev_priv)) ++ ++/* Having a GuC is not the same as using a GuC */ ++#define USES_GUC(dev_priv) intel_uc_is_using_guc(dev_priv) ++#define USES_GUC_SUBMISSION(dev_priv) intel_uc_is_using_guc_submission(dev_priv) ++#define USES_HUC(dev_priv) intel_uc_is_using_huc(dev_priv) ++ ++#define HAS_POOLED_EU(dev_priv) (INTEL_INFO(dev_priv)->has_pooled_eu) ++ ++#define INTEL_PCH_DEVICE_ID_MASK 0xff80 ++#define INTEL_PCH_IBX_DEVICE_ID_TYPE 0x3b00 ++#define INTEL_PCH_CPT_DEVICE_ID_TYPE 0x1c00 ++#define INTEL_PCH_PPT_DEVICE_ID_TYPE 0x1e00 ++#define INTEL_PCH_LPT_DEVICE_ID_TYPE 0x8c00 ++#define INTEL_PCH_LPT_LP_DEVICE_ID_TYPE 0x9c00 ++#define INTEL_PCH_WPT_DEVICE_ID_TYPE 0x8c80 ++#define INTEL_PCH_WPT_LP_DEVICE_ID_TYPE 0x9c80 ++#define INTEL_PCH_SPT_DEVICE_ID_TYPE 0xA100 ++#define INTEL_PCH_SPT_LP_DEVICE_ID_TYPE 0x9D00 ++#define INTEL_PCH_KBP_DEVICE_ID_TYPE 0xA280 ++#define INTEL_PCH_CNP_DEVICE_ID_TYPE 0xA300 ++#define INTEL_PCH_CNP_LP_DEVICE_ID_TYPE 0x9D80 ++#define INTEL_PCH_CMP_DEVICE_ID_TYPE 0x0280 ++#define INTEL_PCH_ICP_DEVICE_ID_TYPE 0x3480 ++#define INTEL_PCH_P2X_DEVICE_ID_TYPE 0x7100 ++#define INTEL_PCH_P3X_DEVICE_ID_TYPE 0x7000 ++#define INTEL_PCH_QEMU_DEVICE_ID_TYPE 0x2900 /* qemu q35 has 2918 */ ++ ++#define INTEL_PCH_TYPE(dev_priv) ((dev_priv)->pch_type) ++#define INTEL_PCH_ID(dev_priv) ((dev_priv)->pch_id) ++#define HAS_PCH_ICP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_ICP) ++#define HAS_PCH_CNP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_CNP) ++#define HAS_PCH_KBP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_KBP) ++#define HAS_PCH_SPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_SPT) ++#define HAS_PCH_LPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_LPT) ++#define HAS_PCH_LPT_LP(dev_priv) \ ++ (INTEL_PCH_ID(dev_priv) == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE || \ ++ INTEL_PCH_ID(dev_priv) == INTEL_PCH_WPT_LP_DEVICE_ID_TYPE) ++#define HAS_PCH_LPT_H(dev_priv) \ ++ (INTEL_PCH_ID(dev_priv) == INTEL_PCH_LPT_DEVICE_ID_TYPE || \ ++ INTEL_PCH_ID(dev_priv) == INTEL_PCH_WPT_DEVICE_ID_TYPE) ++#define HAS_PCH_CPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_CPT) ++#define HAS_PCH_IBX(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_IBX) ++#define HAS_PCH_NOP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_NOP) ++#define HAS_PCH_SPLIT(dev_priv) (INTEL_PCH_TYPE(dev_priv) != PCH_NONE) ++ ++#define HAS_GMCH(dev_priv) (INTEL_INFO(dev_priv)->display.has_gmch) ++ ++#define HAS_LSPCON(dev_priv) (INTEL_GEN(dev_priv) >= 9) ++ ++/* DPF == dynamic parity feature */ ++#define HAS_L3_DPF(dev_priv) (INTEL_INFO(dev_priv)->has_l3_dpf) ++#define NUM_L3_SLICES(dev_priv) (IS_HSW_GT3(dev_priv) ? \ ++ 2 : HAS_L3_DPF(dev_priv)) ++ ++#define GT_FREQUENCY_MULTIPLIER 50 ++#define GEN9_FREQ_SCALER 3 ++ ++#define HAS_DISPLAY(dev_priv) (INTEL_INFO(dev_priv)->num_pipes > 0) ++ ++#include "i915_trace.h" ++ ++static inline bool intel_vtd_active(void) ++{ ++#ifdef CONFIG_INTEL_IOMMU ++ if (intel_iommu_gfx_mapped) ++ return true; ++#endif ++ return false; ++} ++ ++static inline bool intel_scanout_needs_vtd_wa(struct drm_i915_private *dev_priv) ++{ ++ return INTEL_GEN(dev_priv) >= 6 && intel_vtd_active(); ++} ++ ++static inline bool ++intel_ggtt_update_needs_vtd_wa(struct drm_i915_private *dev_priv) ++{ ++ return IS_BROXTON(dev_priv) && intel_vtd_active(); ++} ++ ++/* i915_drv.c */ ++void __printf(3, 4) ++__i915_printk(struct drm_i915_private *dev_priv, const char *level, ++ const char *fmt, ...); ++ ++#define i915_report_error(dev_priv, fmt, ...) \ ++ __i915_printk(dev_priv, KERN_ERR, fmt, ##__VA_ARGS__) ++ ++#ifdef CONFIG_COMPAT ++extern long i915_compat_ioctl(struct file *filp, unsigned int cmd, ++ unsigned long arg); ++#else ++#define i915_compat_ioctl NULL ++#endif ++extern const struct dev_pm_ops i915_pm_ops; ++ ++extern int i915_driver_load(struct pci_dev *pdev, ++ const struct pci_device_id *ent); ++extern void i915_driver_unload(struct drm_device *dev); ++ ++extern void intel_engine_init_hangcheck(struct intel_engine_cs *engine); ++extern void intel_hangcheck_init(struct drm_i915_private *dev_priv); ++extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv); ++extern unsigned long i915_mch_val(struct drm_i915_private *dev_priv); ++extern unsigned long i915_gfx_val(struct drm_i915_private *dev_priv); ++extern void i915_update_gfx_val(struct drm_i915_private *dev_priv); ++int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool on); ++ ++int intel_engines_init_mmio(struct drm_i915_private *dev_priv); ++int intel_engines_init(struct drm_i915_private *dev_priv); ++ ++u32 intel_calculate_mcr_s_ss_select(struct drm_i915_private *dev_priv); ++ ++/* intel_hotplug.c */ ++void intel_hpd_irq_handler(struct drm_i915_private *dev_priv, ++ u32 pin_mask, u32 long_mask); ++void intel_hpd_init(struct drm_i915_private *dev_priv); ++void intel_hpd_init_work(struct drm_i915_private *dev_priv); ++void intel_hpd_cancel_work(struct drm_i915_private *dev_priv); ++enum hpd_pin intel_hpd_pin_default(struct drm_i915_private *dev_priv, ++ enum port port); ++bool intel_hpd_disable(struct drm_i915_private *dev_priv, enum hpd_pin pin); ++void intel_hpd_enable(struct drm_i915_private *dev_priv, enum hpd_pin pin); ++ ++/* i915_irq.c */ ++static inline void i915_queue_hangcheck(struct drm_i915_private *dev_priv) ++{ ++ unsigned long delay; ++ ++ if (unlikely(!i915_modparams.enable_hangcheck)) ++ return; ++ ++ /* Don't continually defer the hangcheck so that it is always run at ++ * least once after work has been scheduled on any ring. Otherwise, ++ * we will ignore a hung ring if a second ring is kept busy. ++ */ ++ ++ delay = round_jiffies_up_relative(DRM_I915_HANGCHECK_JIFFIES); ++ queue_delayed_work(system_long_wq, ++ &dev_priv->gpu_error.hangcheck_work, delay); ++} ++ ++extern void intel_irq_init(struct drm_i915_private *dev_priv); ++extern void intel_irq_fini(struct drm_i915_private *dev_priv); ++int intel_irq_install(struct drm_i915_private *dev_priv); ++void intel_irq_uninstall(struct drm_i915_private *dev_priv); ++ ++static inline bool intel_gvt_active(struct drm_i915_private *dev_priv) ++{ ++ return dev_priv->gvt; ++} ++ ++static inline bool intel_vgpu_active(struct drm_i915_private *dev_priv) ++{ ++ return dev_priv->vgpu.active; ++} ++ ++u32 i915_pipestat_enable_mask(struct drm_i915_private *dev_priv, ++ enum pipe pipe); ++void ++i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, ++ u32 status_mask); ++ ++void ++i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, ++ u32 status_mask); ++ ++void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv); ++void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv); ++void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv, ++ u32 mask, ++ u32 bits); ++void ilk_update_display_irq(struct drm_i915_private *dev_priv, ++ u32 interrupt_mask, ++ u32 enabled_irq_mask); ++static inline void ++ilk_enable_display_irq(struct drm_i915_private *dev_priv, u32 bits) ++{ ++ ilk_update_display_irq(dev_priv, bits, bits); ++} ++static inline void ++ilk_disable_display_irq(struct drm_i915_private *dev_priv, u32 bits) ++{ ++ ilk_update_display_irq(dev_priv, bits, 0); ++} ++void bdw_update_pipe_irq(struct drm_i915_private *dev_priv, ++ enum pipe pipe, ++ u32 interrupt_mask, ++ u32 enabled_irq_mask); ++static inline void bdw_enable_pipe_irq(struct drm_i915_private *dev_priv, ++ enum pipe pipe, u32 bits) ++{ ++ bdw_update_pipe_irq(dev_priv, pipe, bits, bits); ++} ++static inline void bdw_disable_pipe_irq(struct drm_i915_private *dev_priv, ++ enum pipe pipe, u32 bits) ++{ ++ bdw_update_pipe_irq(dev_priv, pipe, bits, 0); ++} ++void ibx_display_interrupt_update(struct drm_i915_private *dev_priv, ++ u32 interrupt_mask, ++ u32 enabled_irq_mask); ++static inline void ++ibx_enable_display_interrupt(struct drm_i915_private *dev_priv, u32 bits) ++{ ++ ibx_display_interrupt_update(dev_priv, bits, bits); ++} ++static inline void ++ibx_disable_display_interrupt(struct drm_i915_private *dev_priv, u32 bits) ++{ ++ ibx_display_interrupt_update(dev_priv, bits, 0); ++} ++ ++/* i915_gem.c */ ++int i915_gem_create_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file_priv); ++int i915_gem_pread_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file_priv); ++int i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file_priv); ++int i915_gem_mmap_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file_priv); ++int i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file_priv); ++int i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file_priv); ++int i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file_priv); ++int i915_gem_execbuffer_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file_priv); ++int i915_gem_execbuffer2_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file_priv); ++int i915_gem_busy_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file_priv); ++int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file); ++int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file); ++int i915_gem_throttle_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file_priv); ++int i915_gem_madvise_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file_priv); ++int i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file_priv); ++int i915_gem_get_tiling_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file_priv); ++int i915_gem_init_userptr(struct drm_i915_private *dev_priv); ++void i915_gem_cleanup_userptr(struct drm_i915_private *dev_priv); ++int i915_gem_userptr_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file); ++int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file_priv); ++int i915_gem_wait_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file_priv); ++void i915_gem_sanitize(struct drm_i915_private *i915); ++int i915_gem_init_early(struct drm_i915_private *dev_priv); ++void i915_gem_cleanup_early(struct drm_i915_private *dev_priv); ++void i915_gem_load_init_fences(struct drm_i915_private *dev_priv); ++int i915_gem_freeze(struct drm_i915_private *dev_priv); ++int i915_gem_freeze_late(struct drm_i915_private *dev_priv); ++ ++void i915_gem_object_init(struct drm_i915_gem_object *obj, ++ const struct drm_i915_gem_object_ops *ops); ++struct drm_i915_gem_object * ++i915_gem_object_create(struct drm_i915_private *dev_priv, u64 size); ++struct drm_i915_gem_object * ++i915_gem_object_create_from_data(struct drm_i915_private *dev_priv, ++ const void *data, size_t size); ++void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file); ++void i915_gem_free_object(struct drm_gem_object *obj); ++ ++static inline void i915_gem_drain_freed_objects(struct drm_i915_private *i915) ++{ ++ if (!atomic_read(&i915->mm.free_count)) ++ return; ++ ++ /* A single pass should suffice to release all the freed objects (along ++ * most call paths) , but be a little more paranoid in that freeing ++ * the objects does take a little amount of time, during which the rcu ++ * callbacks could have added new objects into the freed list, and ++ * armed the work again. ++ */ ++ do { ++ rcu_barrier(); ++ } while (flush_work(&i915->mm.free_work)); ++} ++ ++static inline void i915_gem_drain_workqueue(struct drm_i915_private *i915) ++{ ++ /* ++ * Similar to objects above (see i915_gem_drain_freed-objects), in ++ * general we have workers that are armed by RCU and then rearm ++ * themselves in their callbacks. To be paranoid, we need to ++ * drain the workqueue a second time after waiting for the RCU ++ * grace period so that we catch work queued via RCU from the first ++ * pass. As neither drain_workqueue() nor flush_workqueue() report ++ * a result, we make an assumption that we only don't require more ++ * than 2 passes to catch all recursive RCU delayed work. ++ * ++ */ ++ int pass = 2; ++ do { ++ rcu_barrier(); ++ i915_gem_drain_freed_objects(i915); ++ drain_workqueue(i915->wq); ++ } while (--pass); ++} ++ ++struct i915_vma * __must_check ++i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, ++ const struct i915_ggtt_view *view, ++ u64 size, ++ u64 alignment, ++ u64 flags); ++ ++int i915_gem_object_unbind(struct drm_i915_gem_object *obj); ++void i915_gem_release_mmap(struct drm_i915_gem_object *obj); ++ ++void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv); ++ ++static inline int __sg_page_count(const struct scatterlist *sg) ++{ ++ return sg->length >> PAGE_SHIFT; ++} ++ ++struct scatterlist * ++i915_gem_object_get_sg(struct drm_i915_gem_object *obj, ++ unsigned int n, unsigned int *offset); ++ ++struct page * ++i915_gem_object_get_page(struct drm_i915_gem_object *obj, ++ unsigned int n); ++ ++struct page * ++i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, ++ unsigned int n); ++ ++dma_addr_t ++i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, ++ unsigned long n); ++ ++void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, ++ struct sg_table *pages, ++ unsigned int sg_page_sizes); ++int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj); ++ ++static inline int __must_check ++i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) ++{ ++ might_lock(&obj->mm.lock); ++ ++ if (atomic_inc_not_zero(&obj->mm.pages_pin_count)) ++ return 0; ++ ++ return __i915_gem_object_get_pages(obj); ++} ++ ++static inline bool ++i915_gem_object_has_pages(struct drm_i915_gem_object *obj) ++{ ++ return !IS_ERR_OR_NULL(READ_ONCE(obj->mm.pages)); ++} ++ ++static inline void ++__i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) ++{ ++ GEM_BUG_ON(!i915_gem_object_has_pages(obj)); ++ ++ atomic_inc(&obj->mm.pages_pin_count); ++} ++ ++static inline bool ++i915_gem_object_has_pinned_pages(struct drm_i915_gem_object *obj) ++{ ++ return atomic_read(&obj->mm.pages_pin_count); ++} ++ ++static inline void ++__i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) ++{ ++ GEM_BUG_ON(!i915_gem_object_has_pages(obj)); ++ GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); ++ ++ atomic_dec(&obj->mm.pages_pin_count); ++} ++ ++static inline void ++i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) ++{ ++ __i915_gem_object_unpin_pages(obj); ++} ++ ++enum i915_mm_subclass { /* lockdep subclass for obj->mm.lock/struct_mutex */ ++ I915_MM_NORMAL = 0, ++ I915_MM_SHRINKER /* called "recursively" from direct-reclaim-esque */ ++}; ++ ++int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj, ++ enum i915_mm_subclass subclass); ++void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj); ++ ++enum i915_map_type { ++ I915_MAP_WB = 0, ++ I915_MAP_WC, ++#define I915_MAP_OVERRIDE BIT(31) ++ I915_MAP_FORCE_WB = I915_MAP_WB | I915_MAP_OVERRIDE, ++ I915_MAP_FORCE_WC = I915_MAP_WC | I915_MAP_OVERRIDE, ++}; ++ ++static inline enum i915_map_type ++i915_coherent_map_type(struct drm_i915_private *i915) ++{ ++ return HAS_LLC(i915) ? I915_MAP_WB : I915_MAP_WC; ++} ++ ++/** ++ * i915_gem_object_pin_map - return a contiguous mapping of the entire object ++ * @obj: the object to map into kernel address space ++ * @type: the type of mapping, used to select pgprot_t ++ * ++ * Calls i915_gem_object_pin_pages() to prevent reaping of the object's ++ * pages and then returns a contiguous mapping of the backing storage into ++ * the kernel address space. Based on the @type of mapping, the PTE will be ++ * set to either WriteBack or WriteCombine (via pgprot_t). ++ * ++ * The caller is responsible for calling i915_gem_object_unpin_map() when the ++ * mapping is no longer required. ++ * ++ * Returns the pointer through which to access the mapped object, or an ++ * ERR_PTR() on error. ++ */ ++void *__must_check i915_gem_object_pin_map(struct drm_i915_gem_object *obj, ++ enum i915_map_type type); ++ ++void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj, ++ unsigned long offset, ++ unsigned long size); ++static inline void i915_gem_object_flush_map(struct drm_i915_gem_object *obj) ++{ ++ __i915_gem_object_flush_map(obj, 0, obj->base.size); ++} ++ ++/** ++ * i915_gem_object_unpin_map - releases an earlier mapping ++ * @obj: the object to unmap ++ * ++ * After pinning the object and mapping its pages, once you are finished ++ * with your access, call i915_gem_object_unpin_map() to release the pin ++ * upon the mapping. Once the pin count reaches zero, that mapping may be ++ * removed. ++ */ ++static inline void i915_gem_object_unpin_map(struct drm_i915_gem_object *obj) ++{ ++ i915_gem_object_unpin_pages(obj); ++} ++ ++int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj, ++ unsigned int *needs_clflush); ++int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj, ++ unsigned int *needs_clflush); ++#define CLFLUSH_BEFORE BIT(0) ++#define CLFLUSH_AFTER BIT(1) ++#define CLFLUSH_FLAGS (CLFLUSH_BEFORE | CLFLUSH_AFTER) ++ ++static inline void ++i915_gem_obj_finish_shmem_access(struct drm_i915_gem_object *obj) ++{ ++ i915_gem_object_unpin_pages(obj); ++} ++ ++static inline int __must_check ++i915_mutex_lock_interruptible(struct drm_device *dev) ++{ ++ return mutex_lock_interruptible(&dev->struct_mutex); ++} ++ ++int i915_gem_dumb_create(struct drm_file *file_priv, ++ struct drm_device *dev, ++ struct drm_mode_create_dumb *args); ++int i915_gem_mmap_gtt(struct drm_file *file_priv, struct drm_device *dev, ++ u32 handle, u64 *offset); ++int i915_gem_mmap_gtt_version(void); ++ ++void i915_gem_track_fb(struct drm_i915_gem_object *old, ++ struct drm_i915_gem_object *new, ++ unsigned frontbuffer_bits); ++ ++int __must_check i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno); ++ ++static inline bool __i915_wedged(struct i915_gpu_error *error) ++{ ++ return unlikely(test_bit(I915_WEDGED, &error->flags)); ++} ++ ++static inline bool i915_reset_failed(struct drm_i915_private *i915) ++{ ++ return __i915_wedged(&i915->gpu_error); ++} ++ ++static inline u32 i915_reset_count(struct i915_gpu_error *error) ++{ ++ return READ_ONCE(error->reset_count); ++} ++ ++static inline u32 i915_reset_engine_count(struct i915_gpu_error *error, ++ struct intel_engine_cs *engine) ++{ ++ return READ_ONCE(error->reset_engine_count[engine->id]); ++} ++ ++void i915_gem_set_wedged(struct drm_i915_private *dev_priv); ++bool i915_gem_unset_wedged(struct drm_i915_private *dev_priv); ++ ++void i915_gem_init_mmio(struct drm_i915_private *i915); ++int __must_check i915_gem_init(struct drm_i915_private *dev_priv); ++int __must_check i915_gem_init_hw(struct drm_i915_private *dev_priv); ++void i915_gem_init_swizzling(struct drm_i915_private *dev_priv); ++void i915_gem_fini(struct drm_i915_private *dev_priv); ++void i915_gem_cleanup_engines(struct drm_i915_private *dev_priv); ++int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv, ++ unsigned int flags, long timeout); ++void i915_gem_suspend(struct drm_i915_private *dev_priv); ++void i915_gem_suspend_late(struct drm_i915_private *dev_priv); ++void i915_gem_resume(struct drm_i915_private *dev_priv); ++vm_fault_t i915_gem_fault(struct vm_fault *vmf); ++int i915_gem_object_wait(struct drm_i915_gem_object *obj, ++ unsigned int flags, ++ long timeout); ++int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj, ++ unsigned int flags, ++ const struct i915_sched_attr *attr); ++#define I915_PRIORITY_DISPLAY I915_USER_PRIORITY(I915_PRIORITY_MAX) ++ ++int __must_check ++i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write); ++int __must_check ++i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write); ++int __must_check ++i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write); ++struct i915_vma * __must_check ++i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, ++ u32 alignment, ++ const struct i915_ggtt_view *view, ++ unsigned int flags); ++void i915_gem_object_unpin_from_display_plane(struct i915_vma *vma); ++int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, ++ int align); ++int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file); ++void i915_gem_release(struct drm_device *dev, struct drm_file *file); ++ ++int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, ++ enum i915_cache_level cache_level); ++ ++struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev, ++ struct dma_buf *dma_buf); ++ ++struct dma_buf *i915_gem_prime_export(struct drm_device *dev, ++ struct drm_gem_object *gem_obj, int flags); ++ ++static inline struct i915_hw_ppgtt * ++i915_vm_to_ppgtt(struct i915_address_space *vm) ++{ ++ return container_of(vm, struct i915_hw_ppgtt, vm); ++} ++ ++/* i915_gem_fence_reg.c */ ++struct drm_i915_fence_reg * ++i915_reserve_fence(struct drm_i915_private *dev_priv); ++void i915_unreserve_fence(struct drm_i915_fence_reg *fence); ++ ++void i915_gem_restore_fences(struct drm_i915_private *dev_priv); ++ ++void i915_gem_detect_bit_6_swizzle(struct drm_i915_private *dev_priv); ++void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj, ++ struct sg_table *pages); ++void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj, ++ struct sg_table *pages); ++ ++static inline struct i915_gem_context * ++__i915_gem_context_lookup_rcu(struct drm_i915_file_private *file_priv, u32 id) ++{ ++ return idr_find(&file_priv->context_idr, id); ++} ++ ++static inline struct i915_gem_context * ++i915_gem_context_lookup(struct drm_i915_file_private *file_priv, u32 id) ++{ ++ struct i915_gem_context *ctx; ++ ++ rcu_read_lock(); ++ ctx = __i915_gem_context_lookup_rcu(file_priv, id); ++ if (ctx && !kref_get_unless_zero(&ctx->ref)) ++ ctx = NULL; ++ rcu_read_unlock(); ++ ++ return ctx; ++} ++ ++int i915_perf_open_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file); ++int i915_perf_add_config_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file); ++int i915_perf_remove_config_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file); ++void i915_oa_init_reg_state(struct intel_engine_cs *engine, ++ struct intel_context *ce, ++ u32 *reg_state); ++ ++/* i915_gem_evict.c */ ++int __must_check i915_gem_evict_something(struct i915_address_space *vm, ++ u64 min_size, u64 alignment, ++ unsigned cache_level, ++ u64 start, u64 end, ++ unsigned flags); ++int __must_check i915_gem_evict_for_node(struct i915_address_space *vm, ++ struct drm_mm_node *node, ++ unsigned int flags); ++int i915_gem_evict_vm(struct i915_address_space *vm); ++ ++void i915_gem_flush_ggtt_writes(struct drm_i915_private *dev_priv); ++ ++/* belongs in i915_gem_gtt.h */ ++static inline void i915_gem_chipset_flush(struct drm_i915_private *dev_priv) ++{ ++ wmb(); ++ if (INTEL_GEN(dev_priv) < 6) ++ intel_gtt_chipset_flush(); ++} ++ ++/* i915_gem_stolen.c */ ++int i915_gem_stolen_insert_node(struct drm_i915_private *dev_priv, ++ struct drm_mm_node *node, u64 size, ++ unsigned alignment); ++int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *dev_priv, ++ struct drm_mm_node *node, u64 size, ++ unsigned alignment, u64 start, ++ u64 end); ++void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv, ++ struct drm_mm_node *node); ++int i915_gem_init_stolen(struct drm_i915_private *dev_priv); ++void i915_gem_cleanup_stolen(struct drm_i915_private *dev_priv); ++struct drm_i915_gem_object * ++i915_gem_object_create_stolen(struct drm_i915_private *dev_priv, ++ resource_size_t size); ++struct drm_i915_gem_object * ++i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv, ++ resource_size_t stolen_offset, ++ resource_size_t gtt_offset, ++ resource_size_t size); ++ ++/* i915_gem_internal.c */ ++struct drm_i915_gem_object * ++i915_gem_object_create_internal(struct drm_i915_private *dev_priv, ++ phys_addr_t size); ++ ++/* i915_gem_shrinker.c */ ++unsigned long i915_gem_shrink(struct drm_i915_private *i915, ++ unsigned long target, ++ unsigned long *nr_scanned, ++ unsigned flags); ++#define I915_SHRINK_PURGEABLE 0x1 ++#define I915_SHRINK_UNBOUND 0x2 ++#define I915_SHRINK_BOUND 0x4 ++#define I915_SHRINK_ACTIVE 0x8 ++#define I915_SHRINK_VMAPS 0x10 ++unsigned long i915_gem_shrink_all(struct drm_i915_private *i915); ++void i915_gem_shrinker_register(struct drm_i915_private *i915); ++void i915_gem_shrinker_unregister(struct drm_i915_private *i915); ++void i915_gem_shrinker_taints_mutex(struct drm_i915_private *i915, ++ struct mutex *mutex); ++ ++/* i915_gem_tiling.c */ ++static inline bool i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) ++{ ++ struct drm_i915_private *dev_priv = to_i915(obj->base.dev); ++ ++ return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && ++ i915_gem_object_is_tiled(obj); ++} ++ ++u32 i915_gem_fence_size(struct drm_i915_private *dev_priv, u32 size, ++ unsigned int tiling, unsigned int stride); ++u32 i915_gem_fence_alignment(struct drm_i915_private *dev_priv, u32 size, ++ unsigned int tiling, unsigned int stride); ++ ++/* i915_debugfs.c */ ++#ifdef CONFIG_DEBUG_FS ++int i915_debugfs_register(struct drm_i915_private *dev_priv); ++int i915_debugfs_connector_add(struct drm_connector *connector); ++void intel_display_crc_init(struct drm_i915_private *dev_priv); ++#else ++static inline int i915_debugfs_register(struct drm_i915_private *dev_priv) {return 0;} ++static inline int i915_debugfs_connector_add(struct drm_connector *connector) ++{ return 0; } ++static inline void intel_display_crc_init(struct drm_i915_private *dev_priv) {} ++#endif ++ ++const char *i915_cache_level_str(struct drm_i915_private *i915, int type); ++ ++/* i915_cmd_parser.c */ ++int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv); ++void intel_engine_init_cmd_parser(struct intel_engine_cs *engine); ++void intel_engine_cleanup_cmd_parser(struct intel_engine_cs *engine); ++int intel_engine_cmd_parser(struct intel_engine_cs *engine, ++ struct drm_i915_gem_object *batch_obj, ++ struct drm_i915_gem_object *shadow_batch_obj, ++ u32 batch_start_offset, ++ u32 batch_len, ++ bool is_master); ++ ++/* i915_perf.c */ ++extern void i915_perf_init(struct drm_i915_private *dev_priv); ++extern void i915_perf_fini(struct drm_i915_private *dev_priv); ++extern void i915_perf_register(struct drm_i915_private *dev_priv); ++extern void i915_perf_unregister(struct drm_i915_private *dev_priv); ++ ++/* i915_suspend.c */ ++extern int i915_save_state(struct drm_i915_private *dev_priv); ++extern int i915_restore_state(struct drm_i915_private *dev_priv); ++ ++/* i915_sysfs.c */ ++void i915_setup_sysfs(struct drm_i915_private *dev_priv); ++void i915_teardown_sysfs(struct drm_i915_private *dev_priv); ++ ++/* intel_lpe_audio.c */ ++int intel_lpe_audio_init(struct drm_i915_private *dev_priv); ++void intel_lpe_audio_teardown(struct drm_i915_private *dev_priv); ++void intel_lpe_audio_irq_handler(struct drm_i915_private *dev_priv); ++void intel_lpe_audio_notify(struct drm_i915_private *dev_priv, ++ enum pipe pipe, enum port port, ++ const void *eld, int ls_clock, bool dp_output); ++ ++/* intel_i2c.c */ ++extern int intel_setup_gmbus(struct drm_i915_private *dev_priv); ++extern void intel_teardown_gmbus(struct drm_i915_private *dev_priv); ++extern bool intel_gmbus_is_valid_pin(struct drm_i915_private *dev_priv, ++ unsigned int pin); ++extern int intel_gmbus_output_aksv(struct i2c_adapter *adapter); ++ ++extern struct i2c_adapter * ++intel_gmbus_get_adapter(struct drm_i915_private *dev_priv, unsigned int pin); ++extern void intel_gmbus_set_speed(struct i2c_adapter *adapter, int speed); ++extern void intel_gmbus_force_bit(struct i2c_adapter *adapter, bool force_bit); ++static inline bool intel_gmbus_is_forced_bit(struct i2c_adapter *adapter) ++{ ++ return container_of(adapter, struct intel_gmbus, adapter)->force_bit; ++} ++extern void intel_i2c_reset(struct drm_i915_private *dev_priv); ++ ++/* intel_bios.c */ ++void intel_bios_init(struct drm_i915_private *dev_priv); ++void intel_bios_cleanup(struct drm_i915_private *dev_priv); ++bool intel_bios_is_valid_vbt(const void *buf, size_t size); ++bool intel_bios_is_tv_present(struct drm_i915_private *dev_priv); ++bool intel_bios_is_lvds_present(struct drm_i915_private *dev_priv, u8 *i2c_pin); ++bool intel_bios_is_port_present(struct drm_i915_private *dev_priv, enum port port); ++bool intel_bios_is_port_edp(struct drm_i915_private *dev_priv, enum port port); ++bool intel_bios_is_port_dp_dual_mode(struct drm_i915_private *dev_priv, enum port port); ++bool intel_bios_is_dsi_present(struct drm_i915_private *dev_priv, enum port *port); ++bool intel_bios_is_port_hpd_inverted(struct drm_i915_private *dev_priv, ++ enum port port); ++bool intel_bios_is_lspcon_present(struct drm_i915_private *dev_priv, ++ enum port port); ++enum aux_ch intel_bios_port_aux_ch(struct drm_i915_private *dev_priv, enum port port); ++ ++/* intel_acpi.c */ ++#ifdef CONFIG_ACPI ++extern void intel_register_dsm_handler(void); ++extern void intel_unregister_dsm_handler(void); ++#else ++static inline void intel_register_dsm_handler(void) { return; } ++static inline void intel_unregister_dsm_handler(void) { return; } ++#endif /* CONFIG_ACPI */ ++ ++/* intel_device_info.c */ ++static inline struct intel_device_info * ++mkwrite_device_info(struct drm_i915_private *dev_priv) ++{ ++ return (struct intel_device_info *)INTEL_INFO(dev_priv); ++} ++ ++static inline struct intel_sseu ++intel_device_default_sseu(struct drm_i915_private *i915) ++{ ++ const struct sseu_dev_info *sseu = &RUNTIME_INFO(i915)->sseu; ++ struct intel_sseu value = { ++ .slice_mask = sseu->slice_mask, ++ .subslice_mask = sseu->subslice_mask[0], ++ .min_eus_per_subslice = sseu->max_eus_per_subslice, ++ .max_eus_per_subslice = sseu->max_eus_per_subslice, ++ }; ++ ++ return value; ++} ++ ++/* modesetting */ ++extern void intel_modeset_init_hw(struct drm_device *dev); ++extern int intel_modeset_init(struct drm_device *dev); ++extern void intel_modeset_cleanup(struct drm_device *dev); ++extern int intel_modeset_vga_set_state(struct drm_i915_private *dev_priv, ++ bool state); ++extern void intel_display_resume(struct drm_device *dev); ++extern void i915_redisable_vga(struct drm_i915_private *dev_priv); ++extern void i915_redisable_vga_power_on(struct drm_i915_private *dev_priv); ++extern bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val); ++extern void intel_init_pch_refclk(struct drm_i915_private *dev_priv); ++extern int intel_set_rps(struct drm_i915_private *dev_priv, u8 val); ++extern void intel_rps_mark_interactive(struct drm_i915_private *i915, ++ bool interactive); ++extern bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, ++ bool enable); ++void intel_dsc_enable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state); ++void intel_dsc_disable(const struct intel_crtc_state *crtc_state); ++ ++int i915_reg_read_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file); ++ ++/* overlay */ ++extern struct intel_overlay_error_state * ++intel_overlay_capture_error_state(struct drm_i915_private *dev_priv); ++extern void intel_overlay_print_error_state(struct drm_i915_error_state_buf *e, ++ struct intel_overlay_error_state *error); ++ ++extern struct intel_display_error_state * ++intel_display_capture_error_state(struct drm_i915_private *dev_priv); ++extern void intel_display_print_error_state(struct drm_i915_error_state_buf *e, ++ struct intel_display_error_state *error); ++ ++int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val); ++int sandybridge_pcode_write_timeout(struct drm_i915_private *dev_priv, u32 mbox, ++ u32 val, int fast_timeout_us, ++ int slow_timeout_ms); ++#define sandybridge_pcode_write(dev_priv, mbox, val) \ ++ sandybridge_pcode_write_timeout(dev_priv, mbox, val, 500, 0) ++ ++int skl_pcode_request(struct drm_i915_private *dev_priv, u32 mbox, u32 request, ++ u32 reply_mask, u32 reply, int timeout_base_ms); ++ ++/* intel_sideband.c */ ++u32 vlv_punit_read(struct drm_i915_private *dev_priv, u32 addr); ++int vlv_punit_write(struct drm_i915_private *dev_priv, u32 addr, u32 val); ++u32 vlv_nc_read(struct drm_i915_private *dev_priv, u8 addr); ++u32 vlv_iosf_sb_read(struct drm_i915_private *dev_priv, u8 port, u32 reg); ++void vlv_iosf_sb_write(struct drm_i915_private *dev_priv, u8 port, u32 reg, u32 val); ++u32 vlv_cck_read(struct drm_i915_private *dev_priv, u32 reg); ++void vlv_cck_write(struct drm_i915_private *dev_priv, u32 reg, u32 val); ++u32 vlv_ccu_read(struct drm_i915_private *dev_priv, u32 reg); ++void vlv_ccu_write(struct drm_i915_private *dev_priv, u32 reg, u32 val); ++u32 vlv_bunit_read(struct drm_i915_private *dev_priv, u32 reg); ++void vlv_bunit_write(struct drm_i915_private *dev_priv, u32 reg, u32 val); ++u32 vlv_dpio_read(struct drm_i915_private *dev_priv, enum pipe pipe, int reg); ++void vlv_dpio_write(struct drm_i915_private *dev_priv, enum pipe pipe, int reg, u32 val); ++u32 intel_sbi_read(struct drm_i915_private *dev_priv, u16 reg, ++ enum intel_sbi_destination destination); ++void intel_sbi_write(struct drm_i915_private *dev_priv, u16 reg, u32 value, ++ enum intel_sbi_destination destination); ++u32 vlv_flisdsi_read(struct drm_i915_private *dev_priv, u32 reg); ++void vlv_flisdsi_write(struct drm_i915_private *dev_priv, u32 reg, u32 val); ++ ++/* intel_dpio_phy.c */ ++void bxt_port_to_phy_channel(struct drm_i915_private *dev_priv, enum port port, ++ enum dpio_phy *phy, enum dpio_channel *ch); ++void bxt_ddi_phy_set_signal_level(struct drm_i915_private *dev_priv, ++ enum port port, u32 margin, u32 scale, ++ u32 enable, u32 deemphasis); ++void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy); ++void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy); ++bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv, ++ enum dpio_phy phy); ++bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv, ++ enum dpio_phy phy); ++u8 bxt_ddi_phy_calc_lane_lat_optim_mask(u8 lane_count); ++void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder, ++ u8 lane_lat_optim_mask); ++u8 bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder); ++ ++void chv_set_phy_signal_level(struct intel_encoder *encoder, ++ u32 deemph_reg_value, u32 margin_reg_value, ++ bool uniq_trans_scale); ++void chv_data_lane_soft_reset(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state, ++ bool reset); ++void chv_phy_pre_pll_enable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state); ++void chv_phy_pre_encoder_enable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state); ++void chv_phy_release_cl2_override(struct intel_encoder *encoder); ++void chv_phy_post_pll_disable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state); ++ ++void vlv_set_phy_signal_level(struct intel_encoder *encoder, ++ u32 demph_reg_value, u32 preemph_reg_value, ++ u32 uniqtranscale_reg_value, u32 tx3_demph); ++void vlv_phy_pre_pll_enable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state); ++void vlv_phy_pre_encoder_enable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state); ++void vlv_phy_reset_lanes(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state); ++ ++/* intel_combo_phy.c */ ++void icl_combo_phys_init(struct drm_i915_private *dev_priv); ++void icl_combo_phys_uninit(struct drm_i915_private *dev_priv); ++void cnl_combo_phys_init(struct drm_i915_private *dev_priv); ++void cnl_combo_phys_uninit(struct drm_i915_private *dev_priv); ++ ++int intel_gpu_freq(struct drm_i915_private *dev_priv, int val); ++int intel_freq_opcode(struct drm_i915_private *dev_priv, int val); ++u64 intel_rc6_residency_ns(struct drm_i915_private *dev_priv, ++ const i915_reg_t reg); ++ ++u32 intel_get_cagf(struct drm_i915_private *dev_priv, u32 rpstat1); ++ ++static inline u64 intel_rc6_residency_us(struct drm_i915_private *dev_priv, ++ const i915_reg_t reg) ++{ ++ return DIV_ROUND_UP_ULL(intel_rc6_residency_ns(dev_priv, reg), 1000); ++} ++ ++#define __I915_REG_OP(op__, dev_priv__, ...) \ ++ intel_uncore_##op__(&(dev_priv__)->uncore, __VA_ARGS__) ++ ++#define I915_READ8(reg__) __I915_REG_OP(read8, dev_priv, (reg__)) ++#define I915_WRITE8(reg__, val__) __I915_REG_OP(write8, dev_priv, (reg__), (val__)) ++ ++#define I915_READ16(reg__) __I915_REG_OP(read16, dev_priv, (reg__)) ++#define I915_WRITE16(reg__, val__) __I915_REG_OP(write16, dev_priv, (reg__), (val__)) ++#define I915_READ16_NOTRACE(reg__) __I915_REG_OP(read16_notrace, dev_priv, (reg__)) ++#define I915_WRITE16_NOTRACE(reg__, val__) __I915_REG_OP(write16_notrace, dev_priv, (reg__), (val__)) ++ ++#define I915_READ(reg__) __I915_REG_OP(read, dev_priv, (reg__)) ++#define I915_WRITE(reg__, val__) __I915_REG_OP(write, dev_priv, (reg__), (val__)) ++#define I915_READ_NOTRACE(reg__) __I915_REG_OP(read_notrace, dev_priv, (reg__)) ++#define I915_WRITE_NOTRACE(reg__, val__) __I915_REG_OP(write_notrace, dev_priv, (reg__), (val__)) ++ ++/* Be very careful with read/write 64-bit values. On 32-bit machines, they ++ * will be implemented using 2 32-bit writes in an arbitrary order with ++ * an arbitrary delay between them. This can cause the hardware to ++ * act upon the intermediate value, possibly leading to corruption and ++ * machine death. For this reason we do not support I915_WRITE64, or ++ * dev_priv->uncore.funcs.mmio_writeq. ++ * ++ * When reading a 64-bit value as two 32-bit values, the delay may cause ++ * the two reads to mismatch, e.g. a timestamp overflowing. Also note that ++ * occasionally a 64-bit register does not actualy support a full readq ++ * and must be read using two 32-bit reads. ++ * ++ * You have been warned. ++ */ ++#define I915_READ64(reg__) __I915_REG_OP(read64, dev_priv, (reg__)) ++#define I915_READ64_2x32(lower_reg__, upper_reg__) \ ++ __I915_REG_OP(read64_2x32, dev_priv, (lower_reg__), (upper_reg__)) ++ ++#define POSTING_READ(reg__) __I915_REG_OP(posting_read, dev_priv, (reg__)) ++#define POSTING_READ16(reg__) __I915_REG_OP(posting_read16, dev_priv, (reg__)) ++ ++/* These are untraced mmio-accessors that are only valid to be used inside ++ * critical sections, such as inside IRQ handlers, where forcewake is explicitly ++ * controlled. ++ * ++ * Think twice, and think again, before using these. ++ * ++ * As an example, these accessors can possibly be used between: ++ * ++ * spin_lock_irq(&dev_priv->uncore.lock); ++ * intel_uncore_forcewake_get__locked(); ++ * ++ * and ++ * ++ * intel_uncore_forcewake_put__locked(); ++ * spin_unlock_irq(&dev_priv->uncore.lock); ++ * ++ * ++ * Note: some registers may not need forcewake held, so ++ * intel_uncore_forcewake_{get,put} can be omitted, see ++ * intel_uncore_forcewake_for_reg(). ++ * ++ * Certain architectures will die if the same cacheline is concurrently accessed ++ * by different clients (e.g. on Ivybridge). Access to registers should ++ * therefore generally be serialised, by either the dev_priv->uncore.lock or ++ * a more localised lock guarding all access to that bank of registers. ++ */ ++#define I915_READ_FW(reg__) __I915_REG_OP(read_fw, dev_priv, (reg__)) ++#define I915_WRITE_FW(reg__, val__) __I915_REG_OP(write_fw, dev_priv, (reg__), (val__)) ++#define I915_WRITE64_FW(reg__, val__) __I915_REG_OP(write64_fw, dev_priv, (reg__), (val__)) ++#define POSTING_READ_FW(reg__) __I915_REG_OP(posting_read_fw, dev_priv, (reg__)) ++ ++/* "Broadcast RGB" property */ ++#define INTEL_BROADCAST_RGB_AUTO 0 ++#define INTEL_BROADCAST_RGB_FULL 1 ++#define INTEL_BROADCAST_RGB_LIMITED 2 ++ ++static inline i915_reg_t i915_vgacntrl_reg(struct drm_i915_private *dev_priv) ++{ ++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ return VLV_VGACNTRL; ++ else if (INTEL_GEN(dev_priv) >= 5) ++ return CPU_VGACNTRL; ++ else ++ return VGACNTRL; ++} ++ ++static inline unsigned long msecs_to_jiffies_timeout(const unsigned int m) ++{ ++ unsigned long j = msecs_to_jiffies(m); ++ ++ return min_t(unsigned long, MAX_JIFFY_OFFSET, j + 1); ++} ++ ++static inline unsigned long nsecs_to_jiffies_timeout(const u64 n) ++{ ++ /* nsecs_to_jiffies64() does not guard against overflow */ ++ if (NSEC_PER_SEC % HZ && ++ div_u64(n, NSEC_PER_SEC) >= MAX_JIFFY_OFFSET / HZ) ++ return MAX_JIFFY_OFFSET; ++ ++ return min_t(u64, MAX_JIFFY_OFFSET, nsecs_to_jiffies64(n) + 1); ++} ++ ++/* ++ * If you need to wait X milliseconds between events A and B, but event B ++ * doesn't happen exactly after event A, you record the timestamp (jiffies) of ++ * when event A happened, then just before event B you call this function and ++ * pass the timestamp as the first argument, and X as the second argument. ++ */ ++static inline void ++wait_remaining_ms_from_jiffies(unsigned long timestamp_jiffies, int to_wait_ms) ++{ ++ unsigned long target_jiffies, tmp_jiffies, remaining_jiffies; ++ ++ /* ++ * Don't re-read the value of "jiffies" every time since it may change ++ * behind our back and break the math. ++ */ ++ tmp_jiffies = jiffies; ++ target_jiffies = timestamp_jiffies + ++ msecs_to_jiffies_timeout(to_wait_ms); ++ ++ if (time_after(target_jiffies, tmp_jiffies)) { ++ remaining_jiffies = target_jiffies - tmp_jiffies; ++ while (remaining_jiffies) ++ remaining_jiffies = ++ schedule_timeout_uninterruptible(remaining_jiffies); ++ } ++} ++ ++void i915_memcpy_init_early(struct drm_i915_private *dev_priv); ++bool i915_memcpy_from_wc(void *dst, const void *src, unsigned long len); ++ ++/* The movntdqa instructions used for memcpy-from-wc require 16-byte alignment, ++ * as well as SSE4.1 support. i915_memcpy_from_wc() will report if it cannot ++ * perform the operation. To check beforehand, pass in the parameters to ++ * to i915_can_memcpy_from_wc() - since we only care about the low 4 bits, ++ * you only need to pass in the minor offsets, page-aligned pointers are ++ * always valid. ++ * ++ * For just checking for SSE4.1, in the foreknowledge that the future use ++ * will be correctly aligned, just use i915_has_memcpy_from_wc(). ++ */ ++#define i915_can_memcpy_from_wc(dst, src, len) \ ++ i915_memcpy_from_wc((void *)((unsigned long)(dst) | (unsigned long)(src) | (len)), NULL, 0) ++ ++#define i915_has_memcpy_from_wc() \ ++ i915_memcpy_from_wc(NULL, NULL, 0) ++ ++/* i915_mm.c */ ++int remap_io_mapping(struct vm_area_struct *vma, ++ unsigned long addr, unsigned long pfn, unsigned long size, ++ struct io_mapping *iomap); ++ ++static inline int intel_hws_csb_write_index(struct drm_i915_private *i915) ++{ ++ if (INTEL_GEN(i915) >= 10) ++ return CNL_HWS_CSB_WRITE_INDEX; ++ else ++ return I915_HWS_CSB_WRITE_INDEX; ++} ++ ++static inline u32 i915_scratch_offset(const struct drm_i915_private *i915) ++{ ++ return i915_ggtt_offset(i915->gt.scratch); ++} ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_fixed.h b/drivers/gpu/drm/i915_legacy/i915_fixed.h +new file mode 100644 +index 000000000000..591dd89ba7af +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_fixed.h +@@ -0,0 +1,143 @@ ++/* SPDX-License-Identifier: MIT */ ++/* ++ * Copyright © 2018 Intel Corporation ++ */ ++ ++#ifndef _I915_FIXED_H_ ++#define _I915_FIXED_H_ ++ ++typedef struct { ++ u32 val; ++} uint_fixed_16_16_t; ++ ++#define FP_16_16_MAX ((uint_fixed_16_16_t){ .val = UINT_MAX }) ++ ++static inline bool is_fixed16_zero(uint_fixed_16_16_t val) ++{ ++ return val.val == 0; ++} ++ ++static inline uint_fixed_16_16_t u32_to_fixed16(u32 val) ++{ ++ uint_fixed_16_16_t fp = { .val = val << 16 }; ++ ++ WARN_ON(val > U16_MAX); ++ ++ return fp; ++} ++ ++static inline u32 fixed16_to_u32_round_up(uint_fixed_16_16_t fp) ++{ ++ return DIV_ROUND_UP(fp.val, 1 << 16); ++} ++ ++static inline u32 fixed16_to_u32(uint_fixed_16_16_t fp) ++{ ++ return fp.val >> 16; ++} ++ ++static inline uint_fixed_16_16_t min_fixed16(uint_fixed_16_16_t min1, ++ uint_fixed_16_16_t min2) ++{ ++ uint_fixed_16_16_t min = { .val = min(min1.val, min2.val) }; ++ ++ return min; ++} ++ ++static inline uint_fixed_16_16_t max_fixed16(uint_fixed_16_16_t max1, ++ uint_fixed_16_16_t max2) ++{ ++ uint_fixed_16_16_t max = { .val = max(max1.val, max2.val) }; ++ ++ return max; ++} ++ ++static inline uint_fixed_16_16_t clamp_u64_to_fixed16(u64 val) ++{ ++ uint_fixed_16_16_t fp = { .val = (u32)val }; ++ ++ WARN_ON(val > U32_MAX); ++ ++ return fp; ++} ++ ++static inline u32 div_round_up_fixed16(uint_fixed_16_16_t val, ++ uint_fixed_16_16_t d) ++{ ++ return DIV_ROUND_UP(val.val, d.val); ++} ++ ++static inline u32 mul_round_up_u32_fixed16(u32 val, uint_fixed_16_16_t mul) ++{ ++ u64 tmp; ++ ++ tmp = (u64)val * mul.val; ++ tmp = DIV_ROUND_UP_ULL(tmp, 1 << 16); ++ WARN_ON(tmp > U32_MAX); ++ ++ return (u32)tmp; ++} ++ ++static inline uint_fixed_16_16_t mul_fixed16(uint_fixed_16_16_t val, ++ uint_fixed_16_16_t mul) ++{ ++ u64 tmp; ++ ++ tmp = (u64)val.val * mul.val; ++ tmp = tmp >> 16; ++ ++ return clamp_u64_to_fixed16(tmp); ++} ++ ++static inline uint_fixed_16_16_t div_fixed16(u32 val, u32 d) ++{ ++ u64 tmp; ++ ++ tmp = (u64)val << 16; ++ tmp = DIV_ROUND_UP_ULL(tmp, d); ++ ++ return clamp_u64_to_fixed16(tmp); ++} ++ ++static inline u32 div_round_up_u32_fixed16(u32 val, uint_fixed_16_16_t d) ++{ ++ u64 tmp; ++ ++ tmp = (u64)val << 16; ++ tmp = DIV_ROUND_UP_ULL(tmp, d.val); ++ WARN_ON(tmp > U32_MAX); ++ ++ return (u32)tmp; ++} ++ ++static inline uint_fixed_16_16_t mul_u32_fixed16(u32 val, uint_fixed_16_16_t mul) ++{ ++ u64 tmp; ++ ++ tmp = (u64)val * mul.val; ++ ++ return clamp_u64_to_fixed16(tmp); ++} ++ ++static inline uint_fixed_16_16_t add_fixed16(uint_fixed_16_16_t add1, ++ uint_fixed_16_16_t add2) ++{ ++ u64 tmp; ++ ++ tmp = (u64)add1.val + add2.val; ++ ++ return clamp_u64_to_fixed16(tmp); ++} ++ ++static inline uint_fixed_16_16_t add_fixed16_u32(uint_fixed_16_16_t add1, ++ u32 add2) ++{ ++ uint_fixed_16_16_t tmp_add2 = u32_to_fixed16(add2); ++ u64 tmp; ++ ++ tmp = (u64)add1.val + tmp_add2.val; ++ ++ return clamp_u64_to_fixed16(tmp); ++} ++ ++#endif /* _I915_FIXED_H_ */ +diff --git a/drivers/gpu/drm/i915_legacy/i915_gem.c b/drivers/gpu/drm/i915_legacy/i915_gem.c +new file mode 100644 +index 000000000000..ad01c92aaf74 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gem.c +@@ -0,0 +1,5545 @@ ++/* ++ * Copyright © 2008-2015 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ * Authors: ++ * Eric Anholt ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "i915_drv.h" ++#include "i915_gem_clflush.h" ++#include "i915_gemfs.h" ++#include "i915_globals.h" ++#include "i915_reset.h" ++#include "i915_trace.h" ++#include "i915_vgpu.h" ++ ++#include "intel_drv.h" ++#include "intel_frontbuffer.h" ++#include "intel_mocs.h" ++#include "intel_pm.h" ++#include "intel_workarounds.h" ++ ++static void i915_gem_flush_free_objects(struct drm_i915_private *i915); ++ ++static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj) ++{ ++ if (obj->cache_dirty) ++ return false; ++ ++ if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE)) ++ return true; ++ ++ return obj->pin_global; /* currently in use by HW, keep flushed */ ++} ++ ++static int ++insert_mappable_node(struct i915_ggtt *ggtt, ++ struct drm_mm_node *node, u32 size) ++{ ++ memset(node, 0, sizeof(*node)); ++ return drm_mm_insert_node_in_range(&ggtt->vm.mm, node, ++ size, 0, I915_COLOR_UNEVICTABLE, ++ 0, ggtt->mappable_end, ++ DRM_MM_INSERT_LOW); ++} ++ ++static void ++remove_mappable_node(struct drm_mm_node *node) ++{ ++ drm_mm_remove_node(node); ++} ++ ++/* some bookkeeping */ ++static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, ++ u64 size) ++{ ++ spin_lock(&dev_priv->mm.object_stat_lock); ++ dev_priv->mm.object_count++; ++ dev_priv->mm.object_memory += size; ++ spin_unlock(&dev_priv->mm.object_stat_lock); ++} ++ ++static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, ++ u64 size) ++{ ++ spin_lock(&dev_priv->mm.object_stat_lock); ++ dev_priv->mm.object_count--; ++ dev_priv->mm.object_memory -= size; ++ spin_unlock(&dev_priv->mm.object_stat_lock); ++} ++ ++static void __i915_gem_park(struct drm_i915_private *i915) ++{ ++ intel_wakeref_t wakeref; ++ ++ GEM_TRACE("\n"); ++ ++ lockdep_assert_held(&i915->drm.struct_mutex); ++ GEM_BUG_ON(i915->gt.active_requests); ++ GEM_BUG_ON(!list_empty(&i915->gt.active_rings)); ++ ++ if (!i915->gt.awake) ++ return; ++ ++ /* ++ * Be paranoid and flush a concurrent interrupt to make sure ++ * we don't reactivate any irq tasklets after parking. ++ * ++ * FIXME: Note that even though we have waited for execlists to be idle, ++ * there may still be an in-flight interrupt even though the CSB ++ * is now empty. synchronize_irq() makes sure that a residual interrupt ++ * is completed before we continue, but it doesn't prevent the HW from ++ * raising a spurious interrupt later. To complete the shield we should ++ * coordinate disabling the CS irq with flushing the interrupts. ++ */ ++ synchronize_irq(i915->drm.irq); ++ ++ intel_engines_park(i915); ++ i915_timelines_park(i915); ++ ++ i915_pmu_gt_parked(i915); ++ i915_vma_parked(i915); ++ ++ wakeref = fetch_and_zero(&i915->gt.awake); ++ GEM_BUG_ON(!wakeref); ++ ++ if (INTEL_GEN(i915) >= 6) ++ gen6_rps_idle(i915); ++ ++ intel_display_power_put(i915, POWER_DOMAIN_GT_IRQ, wakeref); ++ ++ i915_globals_park(); ++} ++ ++void i915_gem_park(struct drm_i915_private *i915) ++{ ++ GEM_TRACE("\n"); ++ ++ lockdep_assert_held(&i915->drm.struct_mutex); ++ GEM_BUG_ON(i915->gt.active_requests); ++ ++ if (!i915->gt.awake) ++ return; ++ ++ /* Defer the actual call to __i915_gem_park() to prevent ping-pongs */ ++ mod_delayed_work(i915->wq, &i915->gt.idle_work, msecs_to_jiffies(100)); ++} ++ ++void i915_gem_unpark(struct drm_i915_private *i915) ++{ ++ GEM_TRACE("\n"); ++ ++ lockdep_assert_held(&i915->drm.struct_mutex); ++ GEM_BUG_ON(!i915->gt.active_requests); ++ assert_rpm_wakelock_held(i915); ++ ++ if (i915->gt.awake) ++ return; ++ ++ /* ++ * It seems that the DMC likes to transition between the DC states a lot ++ * when there are no connected displays (no active power domains) during ++ * command submission. ++ * ++ * This activity has negative impact on the performance of the chip with ++ * huge latencies observed in the interrupt handler and elsewhere. ++ * ++ * Work around it by grabbing a GT IRQ power domain whilst there is any ++ * GT activity, preventing any DC state transitions. ++ */ ++ i915->gt.awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ); ++ GEM_BUG_ON(!i915->gt.awake); ++ ++ i915_globals_unpark(); ++ ++ intel_enable_gt_powersave(i915); ++ i915_update_gfx_val(i915); ++ if (INTEL_GEN(i915) >= 6) ++ gen6_rps_busy(i915); ++ i915_pmu_gt_unparked(i915); ++ ++ intel_engines_unpark(i915); ++ ++ i915_queue_hangcheck(i915); ++ ++ queue_delayed_work(i915->wq, ++ &i915->gt.retire_work, ++ round_jiffies_up_relative(HZ)); ++} ++ ++int ++i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ struct i915_ggtt *ggtt = &to_i915(dev)->ggtt; ++ struct drm_i915_gem_get_aperture *args = data; ++ struct i915_vma *vma; ++ u64 pinned; ++ ++ mutex_lock(&ggtt->vm.mutex); ++ ++ pinned = ggtt->vm.reserved; ++ list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link) ++ if (i915_vma_is_pinned(vma)) ++ pinned += vma->node.size; ++ ++ mutex_unlock(&ggtt->vm.mutex); ++ ++ args->aper_size = ggtt->vm.total; ++ args->aper_available_size = args->aper_size - pinned; ++ ++ return 0; ++} ++ ++static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) ++{ ++ struct address_space *mapping = obj->base.filp->f_mapping; ++ drm_dma_handle_t *phys; ++ struct sg_table *st; ++ struct scatterlist *sg; ++ char *vaddr; ++ int i; ++ int err; ++ ++ if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj))) ++ return -EINVAL; ++ ++ /* Always aligning to the object size, allows a single allocation ++ * to handle all possible callers, and given typical object sizes, ++ * the alignment of the buddy allocation will naturally match. ++ */ ++ phys = drm_pci_alloc(obj->base.dev, ++ roundup_pow_of_two(obj->base.size), ++ roundup_pow_of_two(obj->base.size)); ++ if (!phys) ++ return -ENOMEM; ++ ++ vaddr = phys->vaddr; ++ for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { ++ struct page *page; ++ char *src; ++ ++ page = shmem_read_mapping_page(mapping, i); ++ if (IS_ERR(page)) { ++ err = PTR_ERR(page); ++ goto err_phys; ++ } ++ ++ src = kmap_atomic(page); ++ memcpy(vaddr, src, PAGE_SIZE); ++ drm_clflush_virt_range(vaddr, PAGE_SIZE); ++ kunmap_atomic(src); ++ ++ put_page(page); ++ vaddr += PAGE_SIZE; ++ } ++ ++ i915_gem_chipset_flush(to_i915(obj->base.dev)); ++ ++ st = kmalloc(sizeof(*st), GFP_KERNEL); ++ if (!st) { ++ err = -ENOMEM; ++ goto err_phys; ++ } ++ ++ if (sg_alloc_table(st, 1, GFP_KERNEL)) { ++ kfree(st); ++ err = -ENOMEM; ++ goto err_phys; ++ } ++ ++ sg = st->sgl; ++ sg->offset = 0; ++ sg->length = obj->base.size; ++ ++ sg_dma_address(sg) = phys->busaddr; ++ sg_dma_len(sg) = obj->base.size; ++ ++ obj->phys_handle = phys; ++ ++ __i915_gem_object_set_pages(obj, st, sg->length); ++ ++ return 0; ++ ++err_phys: ++ drm_pci_free(obj->base.dev, phys); ++ ++ return err; ++} ++ ++static void __start_cpu_write(struct drm_i915_gem_object *obj) ++{ ++ obj->read_domains = I915_GEM_DOMAIN_CPU; ++ obj->write_domain = I915_GEM_DOMAIN_CPU; ++ if (cpu_write_needs_clflush(obj)) ++ obj->cache_dirty = true; ++} ++ ++void ++__i915_gem_object_release_shmem(struct drm_i915_gem_object *obj, ++ struct sg_table *pages, ++ bool needs_clflush) ++{ ++ GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED); ++ ++ if (obj->mm.madv == I915_MADV_DONTNEED) ++ obj->mm.dirty = false; ++ ++ if (needs_clflush && ++ (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0 && ++ !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)) ++ drm_clflush_sg(pages); ++ ++ __start_cpu_write(obj); ++} ++ ++static void ++i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj, ++ struct sg_table *pages) ++{ ++ __i915_gem_object_release_shmem(obj, pages, false); ++ ++ if (obj->mm.dirty) { ++ struct address_space *mapping = obj->base.filp->f_mapping; ++ char *vaddr = obj->phys_handle->vaddr; ++ int i; ++ ++ for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { ++ struct page *page; ++ char *dst; ++ ++ page = shmem_read_mapping_page(mapping, i); ++ if (IS_ERR(page)) ++ continue; ++ ++ dst = kmap_atomic(page); ++ drm_clflush_virt_range(vaddr, PAGE_SIZE); ++ memcpy(dst, vaddr, PAGE_SIZE); ++ kunmap_atomic(dst); ++ ++ set_page_dirty(page); ++ if (obj->mm.madv == I915_MADV_WILLNEED) ++ mark_page_accessed(page); ++ put_page(page); ++ vaddr += PAGE_SIZE; ++ } ++ obj->mm.dirty = false; ++ } ++ ++ sg_free_table(pages); ++ kfree(pages); ++ ++ drm_pci_free(obj->base.dev, obj->phys_handle); ++} ++ ++static void ++i915_gem_object_release_phys(struct drm_i915_gem_object *obj) ++{ ++ i915_gem_object_unpin_pages(obj); ++} ++ ++static const struct drm_i915_gem_object_ops i915_gem_phys_ops = { ++ .get_pages = i915_gem_object_get_pages_phys, ++ .put_pages = i915_gem_object_put_pages_phys, ++ .release = i915_gem_object_release_phys, ++}; ++ ++static const struct drm_i915_gem_object_ops i915_gem_object_ops; ++ ++int i915_gem_object_unbind(struct drm_i915_gem_object *obj) ++{ ++ struct i915_vma *vma; ++ LIST_HEAD(still_in_list); ++ int ret; ++ ++ lockdep_assert_held(&obj->base.dev->struct_mutex); ++ ++ /* Closed vma are removed from the obj->vma_list - but they may ++ * still have an active binding on the object. To remove those we ++ * must wait for all rendering to complete to the object (as unbinding ++ * must anyway), and retire the requests. ++ */ ++ ret = i915_gem_object_set_to_cpu_domain(obj, false); ++ if (ret) ++ return ret; ++ ++ spin_lock(&obj->vma.lock); ++ while (!ret && (vma = list_first_entry_or_null(&obj->vma.list, ++ struct i915_vma, ++ obj_link))) { ++ list_move_tail(&vma->obj_link, &still_in_list); ++ spin_unlock(&obj->vma.lock); ++ ++ ret = i915_vma_unbind(vma); ++ ++ spin_lock(&obj->vma.lock); ++ } ++ list_splice(&still_in_list, &obj->vma.list); ++ spin_unlock(&obj->vma.lock); ++ ++ return ret; ++} ++ ++static long ++i915_gem_object_wait_fence(struct dma_fence *fence, ++ unsigned int flags, ++ long timeout) ++{ ++ struct i915_request *rq; ++ ++ BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1); ++ ++ if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) ++ return timeout; ++ ++ if (!dma_fence_is_i915(fence)) ++ return dma_fence_wait_timeout(fence, ++ flags & I915_WAIT_INTERRUPTIBLE, ++ timeout); ++ ++ rq = to_request(fence); ++ if (i915_request_completed(rq)) ++ goto out; ++ ++ timeout = i915_request_wait(rq, flags, timeout); ++ ++out: ++ if (flags & I915_WAIT_LOCKED && i915_request_completed(rq)) ++ i915_request_retire_upto(rq); ++ ++ return timeout; ++} ++ ++static long ++i915_gem_object_wait_reservation(struct reservation_object *resv, ++ unsigned int flags, ++ long timeout) ++{ ++ unsigned int seq = __read_seqcount_begin(&resv->seq); ++ struct dma_fence *excl; ++ bool prune_fences = false; ++ ++ if (flags & I915_WAIT_ALL) { ++ struct dma_fence **shared; ++ unsigned int count, i; ++ int ret; ++ ++ ret = reservation_object_get_fences_rcu(resv, ++ &excl, &count, &shared); ++ if (ret) ++ return ret; ++ ++ for (i = 0; i < count; i++) { ++ timeout = i915_gem_object_wait_fence(shared[i], ++ flags, timeout); ++ if (timeout < 0) ++ break; ++ ++ dma_fence_put(shared[i]); ++ } ++ ++ for (; i < count; i++) ++ dma_fence_put(shared[i]); ++ kfree(shared); ++ ++ /* ++ * If both shared fences and an exclusive fence exist, ++ * then by construction the shared fences must be later ++ * than the exclusive fence. If we successfully wait for ++ * all the shared fences, we know that the exclusive fence ++ * must all be signaled. If all the shared fences are ++ * signaled, we can prune the array and recover the ++ * floating references on the fences/requests. ++ */ ++ prune_fences = count && timeout >= 0; ++ } else { ++ excl = reservation_object_get_excl_rcu(resv); ++ } ++ ++ if (excl && timeout >= 0) ++ timeout = i915_gem_object_wait_fence(excl, flags, timeout); ++ ++ dma_fence_put(excl); ++ ++ /* ++ * Opportunistically prune the fences iff we know they have *all* been ++ * signaled and that the reservation object has not been changed (i.e. ++ * no new fences have been added). ++ */ ++ if (prune_fences && !__read_seqcount_retry(&resv->seq, seq)) { ++ if (reservation_object_trylock(resv)) { ++ if (!__read_seqcount_retry(&resv->seq, seq)) ++ reservation_object_add_excl_fence(resv, NULL); ++ reservation_object_unlock(resv); ++ } ++ } ++ ++ return timeout; ++} ++ ++static void __fence_set_priority(struct dma_fence *fence, ++ const struct i915_sched_attr *attr) ++{ ++ struct i915_request *rq; ++ struct intel_engine_cs *engine; ++ ++ if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence)) ++ return; ++ ++ rq = to_request(fence); ++ engine = rq->engine; ++ ++ local_bh_disable(); ++ rcu_read_lock(); /* RCU serialisation for set-wedged protection */ ++ if (engine->schedule) ++ engine->schedule(rq, attr); ++ rcu_read_unlock(); ++ local_bh_enable(); /* kick the tasklets if queues were reprioritised */ ++} ++ ++static void fence_set_priority(struct dma_fence *fence, ++ const struct i915_sched_attr *attr) ++{ ++ /* Recurse once into a fence-array */ ++ if (dma_fence_is_array(fence)) { ++ struct dma_fence_array *array = to_dma_fence_array(fence); ++ int i; ++ ++ for (i = 0; i < array->num_fences; i++) ++ __fence_set_priority(array->fences[i], attr); ++ } else { ++ __fence_set_priority(fence, attr); ++ } ++} ++ ++int ++i915_gem_object_wait_priority(struct drm_i915_gem_object *obj, ++ unsigned int flags, ++ const struct i915_sched_attr *attr) ++{ ++ struct dma_fence *excl; ++ ++ if (flags & I915_WAIT_ALL) { ++ struct dma_fence **shared; ++ unsigned int count, i; ++ int ret; ++ ++ ret = reservation_object_get_fences_rcu(obj->resv, ++ &excl, &count, &shared); ++ if (ret) ++ return ret; ++ ++ for (i = 0; i < count; i++) { ++ fence_set_priority(shared[i], attr); ++ dma_fence_put(shared[i]); ++ } ++ ++ kfree(shared); ++ } else { ++ excl = reservation_object_get_excl_rcu(obj->resv); ++ } ++ ++ if (excl) { ++ fence_set_priority(excl, attr); ++ dma_fence_put(excl); ++ } ++ return 0; ++} ++ ++/** ++ * Waits for rendering to the object to be completed ++ * @obj: i915 gem object ++ * @flags: how to wait (under a lock, for all rendering or just for writes etc) ++ * @timeout: how long to wait ++ */ ++int ++i915_gem_object_wait(struct drm_i915_gem_object *obj, ++ unsigned int flags, ++ long timeout) ++{ ++ might_sleep(); ++ GEM_BUG_ON(timeout < 0); ++ ++ timeout = i915_gem_object_wait_reservation(obj->resv, flags, timeout); ++ return timeout < 0 ? timeout : 0; ++} ++ ++static int ++i915_gem_phys_pwrite(struct drm_i915_gem_object *obj, ++ struct drm_i915_gem_pwrite *args, ++ struct drm_file *file) ++{ ++ void *vaddr = obj->phys_handle->vaddr + args->offset; ++ char __user *user_data = u64_to_user_ptr(args->data_ptr); ++ ++ /* We manually control the domain here and pretend that it ++ * remains coherent i.e. in the GTT domain, like shmem_pwrite. ++ */ ++ intel_fb_obj_invalidate(obj, ORIGIN_CPU); ++ if (copy_from_user(vaddr, user_data, args->size)) ++ return -EFAULT; ++ ++ drm_clflush_virt_range(vaddr, args->size); ++ i915_gem_chipset_flush(to_i915(obj->base.dev)); ++ ++ intel_fb_obj_flush(obj, ORIGIN_CPU); ++ return 0; ++} ++ ++static int ++i915_gem_create(struct drm_file *file, ++ struct drm_i915_private *dev_priv, ++ u64 *size_p, ++ u32 *handle_p) ++{ ++ struct drm_i915_gem_object *obj; ++ u32 handle; ++ u64 size; ++ int ret; ++ ++ size = round_up(*size_p, PAGE_SIZE); ++ if (size == 0) ++ return -EINVAL; ++ ++ /* Allocate the new object */ ++ obj = i915_gem_object_create(dev_priv, size); ++ if (IS_ERR(obj)) ++ return PTR_ERR(obj); ++ ++ ret = drm_gem_handle_create(file, &obj->base, &handle); ++ /* drop reference from allocate - handle holds it now */ ++ i915_gem_object_put(obj); ++ if (ret) ++ return ret; ++ ++ *handle_p = handle; ++ *size_p = size; ++ return 0; ++} ++ ++int ++i915_gem_dumb_create(struct drm_file *file, ++ struct drm_device *dev, ++ struct drm_mode_create_dumb *args) ++{ ++ /* have to work out size/pitch and return them */ ++ args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64); ++ args->size = args->pitch * args->height; ++ return i915_gem_create(file, to_i915(dev), ++ &args->size, &args->handle); ++} ++ ++static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj) ++{ ++ return !(obj->cache_level == I915_CACHE_NONE || ++ obj->cache_level == I915_CACHE_WT); ++} ++ ++/** ++ * Creates a new mm object and returns a handle to it. ++ * @dev: drm device pointer ++ * @data: ioctl data blob ++ * @file: drm file pointer ++ */ ++int ++i915_gem_create_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct drm_i915_gem_create *args = data; ++ ++ i915_gem_flush_free_objects(dev_priv); ++ ++ return i915_gem_create(file, dev_priv, ++ &args->size, &args->handle); ++} ++ ++static inline enum fb_op_origin ++fb_write_origin(struct drm_i915_gem_object *obj, unsigned int domain) ++{ ++ return (domain == I915_GEM_DOMAIN_GTT ? ++ obj->frontbuffer_ggtt_origin : ORIGIN_CPU); ++} ++ ++void i915_gem_flush_ggtt_writes(struct drm_i915_private *dev_priv) ++{ ++ intel_wakeref_t wakeref; ++ ++ /* ++ * No actual flushing is required for the GTT write domain for reads ++ * from the GTT domain. Writes to it "immediately" go to main memory ++ * as far as we know, so there's no chipset flush. It also doesn't ++ * land in the GPU render cache. ++ * ++ * However, we do have to enforce the order so that all writes through ++ * the GTT land before any writes to the device, such as updates to ++ * the GATT itself. ++ * ++ * We also have to wait a bit for the writes to land from the GTT. ++ * An uncached read (i.e. mmio) seems to be ideal for the round-trip ++ * timing. This issue has only been observed when switching quickly ++ * between GTT writes and CPU reads from inside the kernel on recent hw, ++ * and it appears to only affect discrete GTT blocks (i.e. on LLC ++ * system agents we cannot reproduce this behaviour, until Cannonlake ++ * that was!). ++ */ ++ ++ wmb(); ++ ++ if (INTEL_INFO(dev_priv)->has_coherent_ggtt) ++ return; ++ ++ i915_gem_chipset_flush(dev_priv); ++ ++ with_intel_runtime_pm(dev_priv, wakeref) { ++ spin_lock_irq(&dev_priv->uncore.lock); ++ ++ POSTING_READ_FW(RING_HEAD(RENDER_RING_BASE)); ++ ++ spin_unlock_irq(&dev_priv->uncore.lock); ++ } ++} ++ ++static void ++flush_write_domain(struct drm_i915_gem_object *obj, unsigned int flush_domains) ++{ ++ struct drm_i915_private *dev_priv = to_i915(obj->base.dev); ++ struct i915_vma *vma; ++ ++ if (!(obj->write_domain & flush_domains)) ++ return; ++ ++ switch (obj->write_domain) { ++ case I915_GEM_DOMAIN_GTT: ++ i915_gem_flush_ggtt_writes(dev_priv); ++ ++ intel_fb_obj_flush(obj, ++ fb_write_origin(obj, I915_GEM_DOMAIN_GTT)); ++ ++ for_each_ggtt_vma(vma, obj) { ++ if (vma->iomap) ++ continue; ++ ++ i915_vma_unset_ggtt_write(vma); ++ } ++ break; ++ ++ case I915_GEM_DOMAIN_WC: ++ wmb(); ++ break; ++ ++ case I915_GEM_DOMAIN_CPU: ++ i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC); ++ break; ++ ++ case I915_GEM_DOMAIN_RENDER: ++ if (gpu_write_needs_clflush(obj)) ++ obj->cache_dirty = true; ++ break; ++ } ++ ++ obj->write_domain = 0; ++} ++ ++/* ++ * Pins the specified object's pages and synchronizes the object with ++ * GPU accesses. Sets needs_clflush to non-zero if the caller should ++ * flush the object from the CPU cache. ++ */ ++int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj, ++ unsigned int *needs_clflush) ++{ ++ int ret; ++ ++ lockdep_assert_held(&obj->base.dev->struct_mutex); ++ ++ *needs_clflush = 0; ++ if (!i915_gem_object_has_struct_page(obj)) ++ return -ENODEV; ++ ++ ret = i915_gem_object_wait(obj, ++ I915_WAIT_INTERRUPTIBLE | ++ I915_WAIT_LOCKED, ++ MAX_SCHEDULE_TIMEOUT); ++ if (ret) ++ return ret; ++ ++ ret = i915_gem_object_pin_pages(obj); ++ if (ret) ++ return ret; ++ ++ if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ || ++ !static_cpu_has(X86_FEATURE_CLFLUSH)) { ++ ret = i915_gem_object_set_to_cpu_domain(obj, false); ++ if (ret) ++ goto err_unpin; ++ else ++ goto out; ++ } ++ ++ flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU); ++ ++ /* If we're not in the cpu read domain, set ourself into the gtt ++ * read domain and manually flush cachelines (if required). This ++ * optimizes for the case when the gpu will dirty the data ++ * anyway again before the next pread happens. ++ */ ++ if (!obj->cache_dirty && ++ !(obj->read_domains & I915_GEM_DOMAIN_CPU)) ++ *needs_clflush = CLFLUSH_BEFORE; ++ ++out: ++ /* return with the pages pinned */ ++ return 0; ++ ++err_unpin: ++ i915_gem_object_unpin_pages(obj); ++ return ret; ++} ++ ++int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj, ++ unsigned int *needs_clflush) ++{ ++ int ret; ++ ++ lockdep_assert_held(&obj->base.dev->struct_mutex); ++ ++ *needs_clflush = 0; ++ if (!i915_gem_object_has_struct_page(obj)) ++ return -ENODEV; ++ ++ ret = i915_gem_object_wait(obj, ++ I915_WAIT_INTERRUPTIBLE | ++ I915_WAIT_LOCKED | ++ I915_WAIT_ALL, ++ MAX_SCHEDULE_TIMEOUT); ++ if (ret) ++ return ret; ++ ++ ret = i915_gem_object_pin_pages(obj); ++ if (ret) ++ return ret; ++ ++ if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE || ++ !static_cpu_has(X86_FEATURE_CLFLUSH)) { ++ ret = i915_gem_object_set_to_cpu_domain(obj, true); ++ if (ret) ++ goto err_unpin; ++ else ++ goto out; ++ } ++ ++ flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU); ++ ++ /* If we're not in the cpu write domain, set ourself into the ++ * gtt write domain and manually flush cachelines (as required). ++ * This optimizes for the case when the gpu will use the data ++ * right away and we therefore have to clflush anyway. ++ */ ++ if (!obj->cache_dirty) { ++ *needs_clflush |= CLFLUSH_AFTER; ++ ++ /* ++ * Same trick applies to invalidate partially written ++ * cachelines read before writing. ++ */ ++ if (!(obj->read_domains & I915_GEM_DOMAIN_CPU)) ++ *needs_clflush |= CLFLUSH_BEFORE; ++ } ++ ++out: ++ intel_fb_obj_invalidate(obj, ORIGIN_CPU); ++ obj->mm.dirty = true; ++ /* return with the pages pinned */ ++ return 0; ++ ++err_unpin: ++ i915_gem_object_unpin_pages(obj); ++ return ret; ++} ++ ++static int ++shmem_pread(struct page *page, int offset, int len, char __user *user_data, ++ bool needs_clflush) ++{ ++ char *vaddr; ++ int ret; ++ ++ vaddr = kmap(page); ++ ++ if (needs_clflush) ++ drm_clflush_virt_range(vaddr + offset, len); ++ ++ ret = __copy_to_user(user_data, vaddr + offset, len); ++ ++ kunmap(page); ++ ++ return ret ? -EFAULT : 0; ++} ++ ++static int ++i915_gem_shmem_pread(struct drm_i915_gem_object *obj, ++ struct drm_i915_gem_pread *args) ++{ ++ char __user *user_data; ++ u64 remain; ++ unsigned int needs_clflush; ++ unsigned int idx, offset; ++ int ret; ++ ++ ret = mutex_lock_interruptible(&obj->base.dev->struct_mutex); ++ if (ret) ++ return ret; ++ ++ ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush); ++ mutex_unlock(&obj->base.dev->struct_mutex); ++ if (ret) ++ return ret; ++ ++ remain = args->size; ++ user_data = u64_to_user_ptr(args->data_ptr); ++ offset = offset_in_page(args->offset); ++ for (idx = args->offset >> PAGE_SHIFT; remain; idx++) { ++ struct page *page = i915_gem_object_get_page(obj, idx); ++ unsigned int length = min_t(u64, remain, PAGE_SIZE - offset); ++ ++ ret = shmem_pread(page, offset, length, user_data, ++ needs_clflush); ++ if (ret) ++ break; ++ ++ remain -= length; ++ user_data += length; ++ offset = 0; ++ } ++ ++ i915_gem_obj_finish_shmem_access(obj); ++ return ret; ++} ++ ++static inline bool ++gtt_user_read(struct io_mapping *mapping, ++ loff_t base, int offset, ++ char __user *user_data, int length) ++{ ++ void __iomem *vaddr; ++ unsigned long unwritten; ++ ++ /* We can use the cpu mem copy function because this is X86. */ ++ vaddr = io_mapping_map_atomic_wc(mapping, base); ++ unwritten = __copy_to_user_inatomic(user_data, ++ (void __force *)vaddr + offset, ++ length); ++ io_mapping_unmap_atomic(vaddr); ++ if (unwritten) { ++ vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE); ++ unwritten = copy_to_user(user_data, ++ (void __force *)vaddr + offset, ++ length); ++ io_mapping_unmap(vaddr); ++ } ++ return unwritten; ++} ++ ++static int ++i915_gem_gtt_pread(struct drm_i915_gem_object *obj, ++ const struct drm_i915_gem_pread *args) ++{ ++ struct drm_i915_private *i915 = to_i915(obj->base.dev); ++ struct i915_ggtt *ggtt = &i915->ggtt; ++ intel_wakeref_t wakeref; ++ struct drm_mm_node node; ++ struct i915_vma *vma; ++ void __user *user_data; ++ u64 remain, offset; ++ int ret; ++ ++ ret = mutex_lock_interruptible(&i915->drm.struct_mutex); ++ if (ret) ++ return ret; ++ ++ wakeref = intel_runtime_pm_get(i915); ++ vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, ++ PIN_MAPPABLE | ++ PIN_NONFAULT | ++ PIN_NONBLOCK); ++ if (!IS_ERR(vma)) { ++ node.start = i915_ggtt_offset(vma); ++ node.allocated = false; ++ ret = i915_vma_put_fence(vma); ++ if (ret) { ++ i915_vma_unpin(vma); ++ vma = ERR_PTR(ret); ++ } ++ } ++ if (IS_ERR(vma)) { ++ ret = insert_mappable_node(ggtt, &node, PAGE_SIZE); ++ if (ret) ++ goto out_unlock; ++ GEM_BUG_ON(!node.allocated); ++ } ++ ++ ret = i915_gem_object_set_to_gtt_domain(obj, false); ++ if (ret) ++ goto out_unpin; ++ ++ mutex_unlock(&i915->drm.struct_mutex); ++ ++ user_data = u64_to_user_ptr(args->data_ptr); ++ remain = args->size; ++ offset = args->offset; ++ ++ while (remain > 0) { ++ /* Operation in this page ++ * ++ * page_base = page offset within aperture ++ * page_offset = offset within page ++ * page_length = bytes to copy for this page ++ */ ++ u32 page_base = node.start; ++ unsigned page_offset = offset_in_page(offset); ++ unsigned page_length = PAGE_SIZE - page_offset; ++ page_length = remain < page_length ? remain : page_length; ++ if (node.allocated) { ++ wmb(); ++ ggtt->vm.insert_page(&ggtt->vm, ++ i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT), ++ node.start, I915_CACHE_NONE, 0); ++ wmb(); ++ } else { ++ page_base += offset & PAGE_MASK; ++ } ++ ++ if (gtt_user_read(&ggtt->iomap, page_base, page_offset, ++ user_data, page_length)) { ++ ret = -EFAULT; ++ break; ++ } ++ ++ remain -= page_length; ++ user_data += page_length; ++ offset += page_length; ++ } ++ ++ mutex_lock(&i915->drm.struct_mutex); ++out_unpin: ++ if (node.allocated) { ++ wmb(); ++ ggtt->vm.clear_range(&ggtt->vm, node.start, node.size); ++ remove_mappable_node(&node); ++ } else { ++ i915_vma_unpin(vma); ++ } ++out_unlock: ++ intel_runtime_pm_put(i915, wakeref); ++ mutex_unlock(&i915->drm.struct_mutex); ++ ++ return ret; ++} ++ ++/** ++ * Reads data from the object referenced by handle. ++ * @dev: drm device pointer ++ * @data: ioctl data blob ++ * @file: drm file pointer ++ * ++ * On error, the contents of *data are undefined. ++ */ ++int ++i915_gem_pread_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_i915_gem_pread *args = data; ++ struct drm_i915_gem_object *obj; ++ int ret; ++ ++ if (args->size == 0) ++ return 0; ++ ++ if (!access_ok(u64_to_user_ptr(args->data_ptr), ++ args->size)) ++ return -EFAULT; ++ ++ obj = i915_gem_object_lookup(file, args->handle); ++ if (!obj) ++ return -ENOENT; ++ ++ /* Bounds check source. */ ++ if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ trace_i915_gem_object_pread(obj, args->offset, args->size); ++ ++ ret = i915_gem_object_wait(obj, ++ I915_WAIT_INTERRUPTIBLE, ++ MAX_SCHEDULE_TIMEOUT); ++ if (ret) ++ goto out; ++ ++ ret = i915_gem_object_pin_pages(obj); ++ if (ret) ++ goto out; ++ ++ ret = i915_gem_shmem_pread(obj, args); ++ if (ret == -EFAULT || ret == -ENODEV) ++ ret = i915_gem_gtt_pread(obj, args); ++ ++ i915_gem_object_unpin_pages(obj); ++out: ++ i915_gem_object_put(obj); ++ return ret; ++} ++ ++/* This is the fast write path which cannot handle ++ * page faults in the source data ++ */ ++ ++static inline bool ++ggtt_write(struct io_mapping *mapping, ++ loff_t base, int offset, ++ char __user *user_data, int length) ++{ ++ void __iomem *vaddr; ++ unsigned long unwritten; ++ ++ /* We can use the cpu mem copy function because this is X86. */ ++ vaddr = io_mapping_map_atomic_wc(mapping, base); ++ unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset, ++ user_data, length); ++ io_mapping_unmap_atomic(vaddr); ++ if (unwritten) { ++ vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE); ++ unwritten = copy_from_user((void __force *)vaddr + offset, ++ user_data, length); ++ io_mapping_unmap(vaddr); ++ } ++ ++ return unwritten; ++} ++ ++/** ++ * This is the fast pwrite path, where we copy the data directly from the ++ * user into the GTT, uncached. ++ * @obj: i915 GEM object ++ * @args: pwrite arguments structure ++ */ ++static int ++i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj, ++ const struct drm_i915_gem_pwrite *args) ++{ ++ struct drm_i915_private *i915 = to_i915(obj->base.dev); ++ struct i915_ggtt *ggtt = &i915->ggtt; ++ intel_wakeref_t wakeref; ++ struct drm_mm_node node; ++ struct i915_vma *vma; ++ u64 remain, offset; ++ void __user *user_data; ++ int ret; ++ ++ ret = mutex_lock_interruptible(&i915->drm.struct_mutex); ++ if (ret) ++ return ret; ++ ++ if (i915_gem_object_has_struct_page(obj)) { ++ /* ++ * Avoid waking the device up if we can fallback, as ++ * waking/resuming is very slow (worst-case 10-100 ms ++ * depending on PCI sleeps and our own resume time). ++ * This easily dwarfs any performance advantage from ++ * using the cache bypass of indirect GGTT access. ++ */ ++ wakeref = intel_runtime_pm_get_if_in_use(i915); ++ if (!wakeref) { ++ ret = -EFAULT; ++ goto out_unlock; ++ } ++ } else { ++ /* No backing pages, no fallback, we must force GGTT access */ ++ wakeref = intel_runtime_pm_get(i915); ++ } ++ ++ vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, ++ PIN_MAPPABLE | ++ PIN_NONFAULT | ++ PIN_NONBLOCK); ++ if (!IS_ERR(vma)) { ++ node.start = i915_ggtt_offset(vma); ++ node.allocated = false; ++ ret = i915_vma_put_fence(vma); ++ if (ret) { ++ i915_vma_unpin(vma); ++ vma = ERR_PTR(ret); ++ } ++ } ++ if (IS_ERR(vma)) { ++ ret = insert_mappable_node(ggtt, &node, PAGE_SIZE); ++ if (ret) ++ goto out_rpm; ++ GEM_BUG_ON(!node.allocated); ++ } ++ ++ ret = i915_gem_object_set_to_gtt_domain(obj, true); ++ if (ret) ++ goto out_unpin; ++ ++ mutex_unlock(&i915->drm.struct_mutex); ++ ++ intel_fb_obj_invalidate(obj, ORIGIN_CPU); ++ ++ user_data = u64_to_user_ptr(args->data_ptr); ++ offset = args->offset; ++ remain = args->size; ++ while (remain) { ++ /* Operation in this page ++ * ++ * page_base = page offset within aperture ++ * page_offset = offset within page ++ * page_length = bytes to copy for this page ++ */ ++ u32 page_base = node.start; ++ unsigned int page_offset = offset_in_page(offset); ++ unsigned int page_length = PAGE_SIZE - page_offset; ++ page_length = remain < page_length ? remain : page_length; ++ if (node.allocated) { ++ wmb(); /* flush the write before we modify the GGTT */ ++ ggtt->vm.insert_page(&ggtt->vm, ++ i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT), ++ node.start, I915_CACHE_NONE, 0); ++ wmb(); /* flush modifications to the GGTT (insert_page) */ ++ } else { ++ page_base += offset & PAGE_MASK; ++ } ++ /* If we get a fault while copying data, then (presumably) our ++ * source page isn't available. Return the error and we'll ++ * retry in the slow path. ++ * If the object is non-shmem backed, we retry again with the ++ * path that handles page fault. ++ */ ++ if (ggtt_write(&ggtt->iomap, page_base, page_offset, ++ user_data, page_length)) { ++ ret = -EFAULT; ++ break; ++ } ++ ++ remain -= page_length; ++ user_data += page_length; ++ offset += page_length; ++ } ++ intel_fb_obj_flush(obj, ORIGIN_CPU); ++ ++ mutex_lock(&i915->drm.struct_mutex); ++out_unpin: ++ if (node.allocated) { ++ wmb(); ++ ggtt->vm.clear_range(&ggtt->vm, node.start, node.size); ++ remove_mappable_node(&node); ++ } else { ++ i915_vma_unpin(vma); ++ } ++out_rpm: ++ intel_runtime_pm_put(i915, wakeref); ++out_unlock: ++ mutex_unlock(&i915->drm.struct_mutex); ++ return ret; ++} ++ ++/* Per-page copy function for the shmem pwrite fastpath. ++ * Flushes invalid cachelines before writing to the target if ++ * needs_clflush_before is set and flushes out any written cachelines after ++ * writing if needs_clflush is set. ++ */ ++static int ++shmem_pwrite(struct page *page, int offset, int len, char __user *user_data, ++ bool needs_clflush_before, ++ bool needs_clflush_after) ++{ ++ char *vaddr; ++ int ret; ++ ++ vaddr = kmap(page); ++ ++ if (needs_clflush_before) ++ drm_clflush_virt_range(vaddr + offset, len); ++ ++ ret = __copy_from_user(vaddr + offset, user_data, len); ++ if (!ret && needs_clflush_after) ++ drm_clflush_virt_range(vaddr + offset, len); ++ ++ kunmap(page); ++ ++ return ret ? -EFAULT : 0; ++} ++ ++static int ++i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj, ++ const struct drm_i915_gem_pwrite *args) ++{ ++ struct drm_i915_private *i915 = to_i915(obj->base.dev); ++ void __user *user_data; ++ u64 remain; ++ unsigned int partial_cacheline_write; ++ unsigned int needs_clflush; ++ unsigned int offset, idx; ++ int ret; ++ ++ ret = mutex_lock_interruptible(&i915->drm.struct_mutex); ++ if (ret) ++ return ret; ++ ++ ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush); ++ mutex_unlock(&i915->drm.struct_mutex); ++ if (ret) ++ return ret; ++ ++ /* If we don't overwrite a cacheline completely we need to be ++ * careful to have up-to-date data by first clflushing. Don't ++ * overcomplicate things and flush the entire patch. ++ */ ++ partial_cacheline_write = 0; ++ if (needs_clflush & CLFLUSH_BEFORE) ++ partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1; ++ ++ user_data = u64_to_user_ptr(args->data_ptr); ++ remain = args->size; ++ offset = offset_in_page(args->offset); ++ for (idx = args->offset >> PAGE_SHIFT; remain; idx++) { ++ struct page *page = i915_gem_object_get_page(obj, idx); ++ unsigned int length = min_t(u64, remain, PAGE_SIZE - offset); ++ ++ ret = shmem_pwrite(page, offset, length, user_data, ++ (offset | length) & partial_cacheline_write, ++ needs_clflush & CLFLUSH_AFTER); ++ if (ret) ++ break; ++ ++ remain -= length; ++ user_data += length; ++ offset = 0; ++ } ++ ++ intel_fb_obj_flush(obj, ORIGIN_CPU); ++ i915_gem_obj_finish_shmem_access(obj); ++ return ret; ++} ++ ++/** ++ * Writes data to the object referenced by handle. ++ * @dev: drm device ++ * @data: ioctl data blob ++ * @file: drm file ++ * ++ * On error, the contents of the buffer that were to be modified are undefined. ++ */ ++int ++i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_i915_gem_pwrite *args = data; ++ struct drm_i915_gem_object *obj; ++ int ret; ++ ++ if (args->size == 0) ++ return 0; ++ ++ if (!access_ok(u64_to_user_ptr(args->data_ptr), args->size)) ++ return -EFAULT; ++ ++ obj = i915_gem_object_lookup(file, args->handle); ++ if (!obj) ++ return -ENOENT; ++ ++ /* Bounds check destination. */ ++ if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) { ++ ret = -EINVAL; ++ goto err; ++ } ++ ++ /* Writes not allowed into this read-only object */ ++ if (i915_gem_object_is_readonly(obj)) { ++ ret = -EINVAL; ++ goto err; ++ } ++ ++ trace_i915_gem_object_pwrite(obj, args->offset, args->size); ++ ++ ret = -ENODEV; ++ if (obj->ops->pwrite) ++ ret = obj->ops->pwrite(obj, args); ++ if (ret != -ENODEV) ++ goto err; ++ ++ ret = i915_gem_object_wait(obj, ++ I915_WAIT_INTERRUPTIBLE | ++ I915_WAIT_ALL, ++ MAX_SCHEDULE_TIMEOUT); ++ if (ret) ++ goto err; ++ ++ ret = i915_gem_object_pin_pages(obj); ++ if (ret) ++ goto err; ++ ++ ret = -EFAULT; ++ /* We can only do the GTT pwrite on untiled buffers, as otherwise ++ * it would end up going through the fenced access, and we'll get ++ * different detiling behavior between reading and writing. ++ * pread/pwrite currently are reading and writing from the CPU ++ * perspective, requiring manual detiling by the client. ++ */ ++ if (!i915_gem_object_has_struct_page(obj) || ++ cpu_write_needs_clflush(obj)) ++ /* Note that the gtt paths might fail with non-page-backed user ++ * pointers (e.g. gtt mappings when moving data between ++ * textures). Fallback to the shmem path in that case. ++ */ ++ ret = i915_gem_gtt_pwrite_fast(obj, args); ++ ++ if (ret == -EFAULT || ret == -ENOSPC) { ++ if (obj->phys_handle) ++ ret = i915_gem_phys_pwrite(obj, args, file); ++ else ++ ret = i915_gem_shmem_pwrite(obj, args); ++ } ++ ++ i915_gem_object_unpin_pages(obj); ++err: ++ i915_gem_object_put(obj); ++ return ret; ++} ++ ++static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj) ++{ ++ struct drm_i915_private *i915 = to_i915(obj->base.dev); ++ struct list_head *list; ++ struct i915_vma *vma; ++ ++ GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); ++ ++ mutex_lock(&i915->ggtt.vm.mutex); ++ for_each_ggtt_vma(vma, obj) { ++ if (!drm_mm_node_allocated(&vma->node)) ++ continue; ++ ++ list_move_tail(&vma->vm_link, &vma->vm->bound_list); ++ } ++ mutex_unlock(&i915->ggtt.vm.mutex); ++ ++ spin_lock(&i915->mm.obj_lock); ++ list = obj->bind_count ? &i915->mm.bound_list : &i915->mm.unbound_list; ++ list_move_tail(&obj->mm.link, list); ++ spin_unlock(&i915->mm.obj_lock); ++} ++ ++/** ++ * Called when user space prepares to use an object with the CPU, either ++ * through the mmap ioctl's mapping or a GTT mapping. ++ * @dev: drm device ++ * @data: ioctl data blob ++ * @file: drm file ++ */ ++int ++i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_i915_gem_set_domain *args = data; ++ struct drm_i915_gem_object *obj; ++ u32 read_domains = args->read_domains; ++ u32 write_domain = args->write_domain; ++ int err; ++ ++ /* Only handle setting domains to types used by the CPU. */ ++ if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS) ++ return -EINVAL; ++ ++ /* ++ * Having something in the write domain implies it's in the read ++ * domain, and only that read domain. Enforce that in the request. ++ */ ++ if (write_domain && read_domains != write_domain) ++ return -EINVAL; ++ ++ if (!read_domains) ++ return 0; ++ ++ obj = i915_gem_object_lookup(file, args->handle); ++ if (!obj) ++ return -ENOENT; ++ ++ /* ++ * Already in the desired write domain? Nothing for us to do! ++ * ++ * We apply a little bit of cunning here to catch a broader set of ++ * no-ops. If obj->write_domain is set, we must be in the same ++ * obj->read_domains, and only that domain. Therefore, if that ++ * obj->write_domain matches the request read_domains, we are ++ * already in the same read/write domain and can skip the operation, ++ * without having to further check the requested write_domain. ++ */ ++ if (READ_ONCE(obj->write_domain) == read_domains) { ++ err = 0; ++ goto out; ++ } ++ ++ /* ++ * Try to flush the object off the GPU without holding the lock. ++ * We will repeat the flush holding the lock in the normal manner ++ * to catch cases where we are gazumped. ++ */ ++ err = i915_gem_object_wait(obj, ++ I915_WAIT_INTERRUPTIBLE | ++ I915_WAIT_PRIORITY | ++ (write_domain ? I915_WAIT_ALL : 0), ++ MAX_SCHEDULE_TIMEOUT); ++ if (err) ++ goto out; ++ ++ /* ++ * Proxy objects do not control access to the backing storage, ergo ++ * they cannot be used as a means to manipulate the cache domain ++ * tracking for that backing storage. The proxy object is always ++ * considered to be outside of any cache domain. ++ */ ++ if (i915_gem_object_is_proxy(obj)) { ++ err = -ENXIO; ++ goto out; ++ } ++ ++ /* ++ * Flush and acquire obj->pages so that we are coherent through ++ * direct access in memory with previous cached writes through ++ * shmemfs and that our cache domain tracking remains valid. ++ * For example, if the obj->filp was moved to swap without us ++ * being notified and releasing the pages, we would mistakenly ++ * continue to assume that the obj remained out of the CPU cached ++ * domain. ++ */ ++ err = i915_gem_object_pin_pages(obj); ++ if (err) ++ goto out; ++ ++ err = i915_mutex_lock_interruptible(dev); ++ if (err) ++ goto out_unpin; ++ ++ if (read_domains & I915_GEM_DOMAIN_WC) ++ err = i915_gem_object_set_to_wc_domain(obj, write_domain); ++ else if (read_domains & I915_GEM_DOMAIN_GTT) ++ err = i915_gem_object_set_to_gtt_domain(obj, write_domain); ++ else ++ err = i915_gem_object_set_to_cpu_domain(obj, write_domain); ++ ++ /* And bump the LRU for this access */ ++ i915_gem_object_bump_inactive_ggtt(obj); ++ ++ mutex_unlock(&dev->struct_mutex); ++ ++ if (write_domain != 0) ++ intel_fb_obj_invalidate(obj, ++ fb_write_origin(obj, write_domain)); ++ ++out_unpin: ++ i915_gem_object_unpin_pages(obj); ++out: ++ i915_gem_object_put(obj); ++ return err; ++} ++ ++/** ++ * Called when user space has done writes to this buffer ++ * @dev: drm device ++ * @data: ioctl data blob ++ * @file: drm file ++ */ ++int ++i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_i915_gem_sw_finish *args = data; ++ struct drm_i915_gem_object *obj; ++ ++ obj = i915_gem_object_lookup(file, args->handle); ++ if (!obj) ++ return -ENOENT; ++ ++ /* ++ * Proxy objects are barred from CPU access, so there is no ++ * need to ban sw_finish as it is a nop. ++ */ ++ ++ /* Pinned buffers may be scanout, so flush the cache */ ++ i915_gem_object_flush_if_display(obj); ++ i915_gem_object_put(obj); ++ ++ return 0; ++} ++ ++static inline bool ++__vma_matches(struct vm_area_struct *vma, struct file *filp, ++ unsigned long addr, unsigned long size) ++{ ++ if (vma->vm_file != filp) ++ return false; ++ ++ return vma->vm_start == addr && ++ (vma->vm_end - vma->vm_start) == PAGE_ALIGN(size); ++} ++ ++/** ++ * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address ++ * it is mapped to. ++ * @dev: drm device ++ * @data: ioctl data blob ++ * @file: drm file ++ * ++ * While the mapping holds a reference on the contents of the object, it doesn't ++ * imply a ref on the object itself. ++ * ++ * IMPORTANT: ++ * ++ * DRM driver writers who look a this function as an example for how to do GEM ++ * mmap support, please don't implement mmap support like here. The modern way ++ * to implement DRM mmap support is with an mmap offset ioctl (like ++ * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly. ++ * That way debug tooling like valgrind will understand what's going on, hiding ++ * the mmap call in a driver private ioctl will break that. The i915 driver only ++ * does cpu mmaps this way because we didn't know better. ++ */ ++int ++i915_gem_mmap_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_i915_gem_mmap *args = data; ++ struct drm_i915_gem_object *obj; ++ unsigned long addr; ++ ++ if (args->flags & ~(I915_MMAP_WC)) ++ return -EINVAL; ++ ++ if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT)) ++ return -ENODEV; ++ ++ obj = i915_gem_object_lookup(file, args->handle); ++ if (!obj) ++ return -ENOENT; ++ ++ /* prime objects have no backing filp to GEM mmap ++ * pages from. ++ */ ++ if (!obj->base.filp) { ++ addr = -ENXIO; ++ goto err; ++ } ++ ++ if (range_overflows(args->offset, args->size, (u64)obj->base.size)) { ++ addr = -EINVAL; ++ goto err; ++ } ++ ++ addr = vm_mmap(obj->base.filp, 0, args->size, ++ PROT_READ | PROT_WRITE, MAP_SHARED, ++ args->offset); ++ if (IS_ERR_VALUE(addr)) ++ goto err; ++ ++ if (args->flags & I915_MMAP_WC) { ++ struct mm_struct *mm = current->mm; ++ struct vm_area_struct *vma; ++ ++ if (down_write_killable(&mm->mmap_sem)) { ++ addr = -EINTR; ++ goto err; ++ } ++ vma = find_vma(mm, addr); ++ if (vma && __vma_matches(vma, obj->base.filp, addr, args->size)) ++ vma->vm_page_prot = ++ pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); ++ else ++ addr = -ENOMEM; ++ up_write(&mm->mmap_sem); ++ if (IS_ERR_VALUE(addr)) ++ goto err; ++ ++ /* This may race, but that's ok, it only gets set */ ++ WRITE_ONCE(obj->frontbuffer_ggtt_origin, ORIGIN_CPU); ++ } ++ i915_gem_object_put(obj); ++ ++ args->addr_ptr = (u64)addr; ++ return 0; ++ ++err: ++ i915_gem_object_put(obj); ++ return addr; ++} ++ ++static unsigned int tile_row_pages(const struct drm_i915_gem_object *obj) ++{ ++ return i915_gem_object_get_tile_row_size(obj) >> PAGE_SHIFT; ++} ++ ++/** ++ * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps ++ * ++ * A history of the GTT mmap interface: ++ * ++ * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to ++ * aligned and suitable for fencing, and still fit into the available ++ * mappable space left by the pinned display objects. A classic problem ++ * we called the page-fault-of-doom where we would ping-pong between ++ * two objects that could not fit inside the GTT and so the memcpy ++ * would page one object in at the expense of the other between every ++ * single byte. ++ * ++ * 1 - Objects can be any size, and have any compatible fencing (X Y, or none ++ * as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the ++ * object is too large for the available space (or simply too large ++ * for the mappable aperture!), a view is created instead and faulted ++ * into userspace. (This view is aligned and sized appropriately for ++ * fenced access.) ++ * ++ * 2 - Recognise WC as a separate cache domain so that we can flush the ++ * delayed writes via GTT before performing direct access via WC. ++ * ++ * 3 - Remove implicit set-domain(GTT) and synchronisation on initial ++ * pagefault; swapin remains transparent. ++ * ++ * Restrictions: ++ * ++ * * snoopable objects cannot be accessed via the GTT. It can cause machine ++ * hangs on some architectures, corruption on others. An attempt to service ++ * a GTT page fault from a snoopable object will generate a SIGBUS. ++ * ++ * * the object must be able to fit into RAM (physical memory, though no ++ * limited to the mappable aperture). ++ * ++ * ++ * Caveats: ++ * ++ * * a new GTT page fault will synchronize rendering from the GPU and flush ++ * all data to system memory. Subsequent access will not be synchronized. ++ * ++ * * all mappings are revoked on runtime device suspend. ++ * ++ * * there are only 8, 16 or 32 fence registers to share between all users ++ * (older machines require fence register for display and blitter access ++ * as well). Contention of the fence registers will cause the previous users ++ * to be unmapped and any new access will generate new page faults. ++ * ++ * * running out of memory while servicing a fault may generate a SIGBUS, ++ * rather than the expected SIGSEGV. ++ */ ++int i915_gem_mmap_gtt_version(void) ++{ ++ return 3; ++} ++ ++static inline struct i915_ggtt_view ++compute_partial_view(const struct drm_i915_gem_object *obj, ++ pgoff_t page_offset, ++ unsigned int chunk) ++{ ++ struct i915_ggtt_view view; ++ ++ if (i915_gem_object_is_tiled(obj)) ++ chunk = roundup(chunk, tile_row_pages(obj)); ++ ++ view.type = I915_GGTT_VIEW_PARTIAL; ++ view.partial.offset = rounddown(page_offset, chunk); ++ view.partial.size = ++ min_t(unsigned int, chunk, ++ (obj->base.size >> PAGE_SHIFT) - view.partial.offset); ++ ++ /* If the partial covers the entire object, just create a normal VMA. */ ++ if (chunk >= obj->base.size >> PAGE_SHIFT) ++ view.type = I915_GGTT_VIEW_NORMAL; ++ ++ return view; ++} ++ ++/** ++ * i915_gem_fault - fault a page into the GTT ++ * @vmf: fault info ++ * ++ * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped ++ * from userspace. The fault handler takes care of binding the object to ++ * the GTT (if needed), allocating and programming a fence register (again, ++ * only if needed based on whether the old reg is still valid or the object ++ * is tiled) and inserting a new PTE into the faulting process. ++ * ++ * Note that the faulting process may involve evicting existing objects ++ * from the GTT and/or fence registers to make room. So performance may ++ * suffer if the GTT working set is large or there are few fence registers ++ * left. ++ * ++ * The current feature set supported by i915_gem_fault() and thus GTT mmaps ++ * is exposed via I915_PARAM_MMAP_GTT_VERSION (see i915_gem_mmap_gtt_version). ++ */ ++vm_fault_t i915_gem_fault(struct vm_fault *vmf) ++{ ++#define MIN_CHUNK_PAGES (SZ_1M >> PAGE_SHIFT) ++ struct vm_area_struct *area = vmf->vma; ++ struct drm_i915_gem_object *obj = to_intel_bo(area->vm_private_data); ++ struct drm_device *dev = obj->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct i915_ggtt *ggtt = &dev_priv->ggtt; ++ bool write = area->vm_flags & VM_WRITE; ++ intel_wakeref_t wakeref; ++ struct i915_vma *vma; ++ pgoff_t page_offset; ++ int srcu; ++ int ret; ++ ++ /* Sanity check that we allow writing into this object */ ++ if (i915_gem_object_is_readonly(obj) && write) ++ return VM_FAULT_SIGBUS; ++ ++ /* We don't use vmf->pgoff since that has the fake offset */ ++ page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT; ++ ++ trace_i915_gem_object_fault(obj, page_offset, true, write); ++ ++ ret = i915_gem_object_pin_pages(obj); ++ if (ret) ++ goto err; ++ ++ wakeref = intel_runtime_pm_get(dev_priv); ++ ++ srcu = i915_reset_trylock(dev_priv); ++ if (srcu < 0) { ++ ret = srcu; ++ goto err_rpm; ++ } ++ ++ ret = i915_mutex_lock_interruptible(dev); ++ if (ret) ++ goto err_reset; ++ ++ /* Access to snoopable pages through the GTT is incoherent. */ ++ if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev_priv)) { ++ ret = -EFAULT; ++ goto err_unlock; ++ } ++ ++ /* Now pin it into the GTT as needed */ ++ vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, ++ PIN_MAPPABLE | ++ PIN_NONBLOCK | ++ PIN_NONFAULT); ++ if (IS_ERR(vma)) { ++ /* Use a partial view if it is bigger than available space */ ++ struct i915_ggtt_view view = ++ compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES); ++ unsigned int flags; ++ ++ flags = PIN_MAPPABLE; ++ if (view.type == I915_GGTT_VIEW_NORMAL) ++ flags |= PIN_NONBLOCK; /* avoid warnings for pinned */ ++ ++ /* ++ * Userspace is now writing through an untracked VMA, abandon ++ * all hope that the hardware is able to track future writes. ++ */ ++ obj->frontbuffer_ggtt_origin = ORIGIN_CPU; ++ ++ vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, flags); ++ if (IS_ERR(vma) && !view.type) { ++ flags = PIN_MAPPABLE; ++ view.type = I915_GGTT_VIEW_PARTIAL; ++ vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, flags); ++ } ++ } ++ if (IS_ERR(vma)) { ++ ret = PTR_ERR(vma); ++ goto err_unlock; ++ } ++ ++ ret = i915_vma_pin_fence(vma); ++ if (ret) ++ goto err_unpin; ++ ++ /* Finally, remap it using the new GTT offset */ ++ ret = remap_io_mapping(area, ++ area->vm_start + (vma->ggtt_view.partial.offset << PAGE_SHIFT), ++ (ggtt->gmadr.start + vma->node.start) >> PAGE_SHIFT, ++ min_t(u64, vma->size, area->vm_end - area->vm_start), ++ &ggtt->iomap); ++ if (ret) ++ goto err_fence; ++ ++ /* Mark as being mmapped into userspace for later revocation */ ++ assert_rpm_wakelock_held(dev_priv); ++ if (!i915_vma_set_userfault(vma) && !obj->userfault_count++) ++ list_add(&obj->userfault_link, &dev_priv->mm.userfault_list); ++ GEM_BUG_ON(!obj->userfault_count); ++ ++ i915_vma_set_ggtt_write(vma); ++ ++err_fence: ++ i915_vma_unpin_fence(vma); ++err_unpin: ++ __i915_vma_unpin(vma); ++err_unlock: ++ mutex_unlock(&dev->struct_mutex); ++err_reset: ++ i915_reset_unlock(dev_priv, srcu); ++err_rpm: ++ intel_runtime_pm_put(dev_priv, wakeref); ++ i915_gem_object_unpin_pages(obj); ++err: ++ switch (ret) { ++ case -EIO: ++ /* ++ * We eat errors when the gpu is terminally wedged to avoid ++ * userspace unduly crashing (gl has no provisions for mmaps to ++ * fail). But any other -EIO isn't ours (e.g. swap in failure) ++ * and so needs to be reported. ++ */ ++ if (!i915_terminally_wedged(dev_priv)) ++ return VM_FAULT_SIGBUS; ++ /* else: fall through */ ++ case -EAGAIN: ++ /* ++ * EAGAIN means the gpu is hung and we'll wait for the error ++ * handler to reset everything when re-faulting in ++ * i915_mutex_lock_interruptible. ++ */ ++ case 0: ++ case -ERESTARTSYS: ++ case -EINTR: ++ case -EBUSY: ++ /* ++ * EBUSY is ok: this just means that another thread ++ * already did the job. ++ */ ++ return VM_FAULT_NOPAGE; ++ case -ENOMEM: ++ return VM_FAULT_OOM; ++ case -ENOSPC: ++ case -EFAULT: ++ return VM_FAULT_SIGBUS; ++ default: ++ WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret); ++ return VM_FAULT_SIGBUS; ++ } ++} ++ ++static void __i915_gem_object_release_mmap(struct drm_i915_gem_object *obj) ++{ ++ struct i915_vma *vma; ++ ++ GEM_BUG_ON(!obj->userfault_count); ++ ++ obj->userfault_count = 0; ++ list_del(&obj->userfault_link); ++ drm_vma_node_unmap(&obj->base.vma_node, ++ obj->base.dev->anon_inode->i_mapping); ++ ++ for_each_ggtt_vma(vma, obj) ++ i915_vma_unset_userfault(vma); ++} ++ ++/** ++ * i915_gem_release_mmap - remove physical page mappings ++ * @obj: obj in question ++ * ++ * Preserve the reservation of the mmapping with the DRM core code, but ++ * relinquish ownership of the pages back to the system. ++ * ++ * It is vital that we remove the page mapping if we have mapped a tiled ++ * object through the GTT and then lose the fence register due to ++ * resource pressure. Similarly if the object has been moved out of the ++ * aperture, than pages mapped into userspace must be revoked. Removing the ++ * mapping will then trigger a page fault on the next user access, allowing ++ * fixup by i915_gem_fault(). ++ */ ++void ++i915_gem_release_mmap(struct drm_i915_gem_object *obj) ++{ ++ struct drm_i915_private *i915 = to_i915(obj->base.dev); ++ intel_wakeref_t wakeref; ++ ++ /* Serialisation between user GTT access and our code depends upon ++ * revoking the CPU's PTE whilst the mutex is held. The next user ++ * pagefault then has to wait until we release the mutex. ++ * ++ * Note that RPM complicates somewhat by adding an additional ++ * requirement that operations to the GGTT be made holding the RPM ++ * wakeref. ++ */ ++ lockdep_assert_held(&i915->drm.struct_mutex); ++ wakeref = intel_runtime_pm_get(i915); ++ ++ if (!obj->userfault_count) ++ goto out; ++ ++ __i915_gem_object_release_mmap(obj); ++ ++ /* Ensure that the CPU's PTE are revoked and there are not outstanding ++ * memory transactions from userspace before we return. The TLB ++ * flushing implied above by changing the PTE above *should* be ++ * sufficient, an extra barrier here just provides us with a bit ++ * of paranoid documentation about our requirement to serialise ++ * memory writes before touching registers / GSM. ++ */ ++ wmb(); ++ ++out: ++ intel_runtime_pm_put(i915, wakeref); ++} ++ ++void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv) ++{ ++ struct drm_i915_gem_object *obj, *on; ++ int i; ++ ++ /* ++ * Only called during RPM suspend. All users of the userfault_list ++ * must be holding an RPM wakeref to ensure that this can not ++ * run concurrently with themselves (and use the struct_mutex for ++ * protection between themselves). ++ */ ++ ++ list_for_each_entry_safe(obj, on, ++ &dev_priv->mm.userfault_list, userfault_link) ++ __i915_gem_object_release_mmap(obj); ++ ++ /* The fence will be lost when the device powers down. If any were ++ * in use by hardware (i.e. they are pinned), we should not be powering ++ * down! All other fences will be reacquired by the user upon waking. ++ */ ++ for (i = 0; i < dev_priv->num_fence_regs; i++) { ++ struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; ++ ++ /* Ideally we want to assert that the fence register is not ++ * live at this point (i.e. that no piece of code will be ++ * trying to write through fence + GTT, as that both violates ++ * our tracking of activity and associated locking/barriers, ++ * but also is illegal given that the hw is powered down). ++ * ++ * Previously we used reg->pin_count as a "liveness" indicator. ++ * That is not sufficient, and we need a more fine-grained ++ * tool if we want to have a sanity check here. ++ */ ++ ++ if (!reg->vma) ++ continue; ++ ++ GEM_BUG_ON(i915_vma_has_userfault(reg->vma)); ++ reg->dirty = true; ++ } ++} ++ ++static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj) ++{ ++ struct drm_i915_private *dev_priv = to_i915(obj->base.dev); ++ int err; ++ ++ err = drm_gem_create_mmap_offset(&obj->base); ++ if (likely(!err)) ++ return 0; ++ ++ /* Attempt to reap some mmap space from dead objects */ ++ do { ++ err = i915_gem_wait_for_idle(dev_priv, ++ I915_WAIT_INTERRUPTIBLE, ++ MAX_SCHEDULE_TIMEOUT); ++ if (err) ++ break; ++ ++ i915_gem_drain_freed_objects(dev_priv); ++ err = drm_gem_create_mmap_offset(&obj->base); ++ if (!err) ++ break; ++ ++ } while (flush_delayed_work(&dev_priv->gt.retire_work)); ++ ++ return err; ++} ++ ++static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj) ++{ ++ drm_gem_free_mmap_offset(&obj->base); ++} ++ ++int ++i915_gem_mmap_gtt(struct drm_file *file, ++ struct drm_device *dev, ++ u32 handle, ++ u64 *offset) ++{ ++ struct drm_i915_gem_object *obj; ++ int ret; ++ ++ obj = i915_gem_object_lookup(file, handle); ++ if (!obj) ++ return -ENOENT; ++ ++ ret = i915_gem_object_create_mmap_offset(obj); ++ if (ret == 0) ++ *offset = drm_vma_node_offset_addr(&obj->base.vma_node); ++ ++ i915_gem_object_put(obj); ++ return ret; ++} ++ ++/** ++ * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing ++ * @dev: DRM device ++ * @data: GTT mapping ioctl data ++ * @file: GEM object info ++ * ++ * Simply returns the fake offset to userspace so it can mmap it. ++ * The mmap call will end up in drm_gem_mmap(), which will set things ++ * up so we can get faults in the handler above. ++ * ++ * The fault handler will take care of binding the object into the GTT ++ * (since it may have been evicted to make room for something), allocating ++ * a fence register, and mapping the appropriate aperture address into ++ * userspace. ++ */ ++int ++i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_i915_gem_mmap_gtt *args = data; ++ ++ return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset); ++} ++ ++/* Immediately discard the backing storage */ ++static void ++i915_gem_object_truncate(struct drm_i915_gem_object *obj) ++{ ++ i915_gem_object_free_mmap_offset(obj); ++ ++ if (obj->base.filp == NULL) ++ return; ++ ++ /* Our goal here is to return as much of the memory as ++ * is possible back to the system as we are called from OOM. ++ * To do this we must instruct the shmfs to drop all of its ++ * backing pages, *now*. ++ */ ++ shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1); ++ obj->mm.madv = __I915_MADV_PURGED; ++ obj->mm.pages = ERR_PTR(-EFAULT); ++} ++ ++/* Try to discard unwanted pages */ ++void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj) ++{ ++ struct address_space *mapping; ++ ++ lockdep_assert_held(&obj->mm.lock); ++ GEM_BUG_ON(i915_gem_object_has_pages(obj)); ++ ++ switch (obj->mm.madv) { ++ case I915_MADV_DONTNEED: ++ i915_gem_object_truncate(obj); ++ case __I915_MADV_PURGED: ++ return; ++ } ++ ++ if (obj->base.filp == NULL) ++ return; ++ ++ mapping = obj->base.filp->f_mapping, ++ invalidate_mapping_pages(mapping, 0, (loff_t)-1); ++} ++ ++/* ++ * Move pages to appropriate lru and release the pagevec, decrementing the ++ * ref count of those pages. ++ */ ++static void check_release_pagevec(struct pagevec *pvec) ++{ ++ check_move_unevictable_pages(pvec); ++ __pagevec_release(pvec); ++ cond_resched(); ++} ++ ++static void ++i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj, ++ struct sg_table *pages) ++{ ++ struct sgt_iter sgt_iter; ++ struct pagevec pvec; ++ struct page *page; ++ ++ __i915_gem_object_release_shmem(obj, pages, true); ++ i915_gem_gtt_finish_pages(obj, pages); ++ ++ if (i915_gem_object_needs_bit17_swizzle(obj)) ++ i915_gem_object_save_bit_17_swizzle(obj, pages); ++ ++ mapping_clear_unevictable(file_inode(obj->base.filp)->i_mapping); ++ ++ pagevec_init(&pvec); ++ for_each_sgt_page(page, sgt_iter, pages) { ++ if (obj->mm.dirty) ++ set_page_dirty(page); ++ ++ if (obj->mm.madv == I915_MADV_WILLNEED) ++ mark_page_accessed(page); ++ ++ if (!pagevec_add(&pvec, page)) ++ check_release_pagevec(&pvec); ++ } ++ if (pagevec_count(&pvec)) ++ check_release_pagevec(&pvec); ++ obj->mm.dirty = false; ++ ++ sg_free_table(pages); ++ kfree(pages); ++} ++ ++static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj) ++{ ++ struct radix_tree_iter iter; ++ void __rcu **slot; ++ ++ rcu_read_lock(); ++ radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0) ++ radix_tree_delete(&obj->mm.get_page.radix, iter.index); ++ rcu_read_unlock(); ++} ++ ++static struct sg_table * ++__i915_gem_object_unset_pages(struct drm_i915_gem_object *obj) ++{ ++ struct drm_i915_private *i915 = to_i915(obj->base.dev); ++ struct sg_table *pages; ++ ++ pages = fetch_and_zero(&obj->mm.pages); ++ if (IS_ERR_OR_NULL(pages)) ++ return pages; ++ ++ spin_lock(&i915->mm.obj_lock); ++ list_del(&obj->mm.link); ++ spin_unlock(&i915->mm.obj_lock); ++ ++ if (obj->mm.mapping) { ++ void *ptr; ++ ++ ptr = page_mask_bits(obj->mm.mapping); ++ if (is_vmalloc_addr(ptr)) ++ vunmap(ptr); ++ else ++ kunmap(kmap_to_page(ptr)); ++ ++ obj->mm.mapping = NULL; ++ } ++ ++ __i915_gem_object_reset_page_iter(obj); ++ obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0; ++ ++ return pages; ++} ++ ++int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj, ++ enum i915_mm_subclass subclass) ++{ ++ struct sg_table *pages; ++ int ret; ++ ++ if (i915_gem_object_has_pinned_pages(obj)) ++ return -EBUSY; ++ ++ GEM_BUG_ON(obj->bind_count); ++ ++ /* May be called by shrinker from within get_pages() (on another bo) */ ++ mutex_lock_nested(&obj->mm.lock, subclass); ++ if (unlikely(atomic_read(&obj->mm.pages_pin_count))) { ++ ret = -EBUSY; ++ goto unlock; ++ } ++ ++ /* ++ * ->put_pages might need to allocate memory for the bit17 swizzle ++ * array, hence protect them from being reaped by removing them from gtt ++ * lists early. ++ */ ++ pages = __i915_gem_object_unset_pages(obj); ++ ++ /* ++ * XXX Temporary hijinx to avoid updating all backends to handle ++ * NULL pages. In the future, when we have more asynchronous ++ * get_pages backends we should be better able to handle the ++ * cancellation of the async task in a more uniform manner. ++ */ ++ if (!pages && !i915_gem_object_needs_async_cancel(obj)) ++ pages = ERR_PTR(-EINVAL); ++ ++ if (!IS_ERR(pages)) ++ obj->ops->put_pages(obj, pages); ++ ++ ret = 0; ++unlock: ++ mutex_unlock(&obj->mm.lock); ++ ++ return ret; ++} ++ ++bool i915_sg_trim(struct sg_table *orig_st) ++{ ++ struct sg_table new_st; ++ struct scatterlist *sg, *new_sg; ++ unsigned int i; ++ ++ if (orig_st->nents == orig_st->orig_nents) ++ return false; ++ ++ if (sg_alloc_table(&new_st, orig_st->nents, GFP_KERNEL | __GFP_NOWARN)) ++ return false; ++ ++ new_sg = new_st.sgl; ++ for_each_sg(orig_st->sgl, sg, orig_st->nents, i) { ++ sg_set_page(new_sg, sg_page(sg), sg->length, 0); ++ sg_dma_address(new_sg) = sg_dma_address(sg); ++ sg_dma_len(new_sg) = sg_dma_len(sg); ++ ++ new_sg = sg_next(new_sg); ++ } ++ GEM_BUG_ON(new_sg); /* Should walk exactly nents and hit the end */ ++ ++ sg_free_table(orig_st); ++ ++ *orig_st = new_st; ++ return true; ++} ++ ++static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) ++{ ++ struct drm_i915_private *dev_priv = to_i915(obj->base.dev); ++ const unsigned long page_count = obj->base.size / PAGE_SIZE; ++ unsigned long i; ++ struct address_space *mapping; ++ struct sg_table *st; ++ struct scatterlist *sg; ++ struct sgt_iter sgt_iter; ++ struct page *page; ++ unsigned long last_pfn = 0; /* suppress gcc warning */ ++ unsigned int max_segment = i915_sg_segment_size(); ++ unsigned int sg_page_sizes; ++ struct pagevec pvec; ++ gfp_t noreclaim; ++ int ret; ++ ++ /* ++ * Assert that the object is not currently in any GPU domain. As it ++ * wasn't in the GTT, there shouldn't be any way it could have been in ++ * a GPU cache ++ */ ++ GEM_BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS); ++ GEM_BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS); ++ ++ /* ++ * If there's no chance of allocating enough pages for the whole ++ * object, bail early. ++ */ ++ if (page_count > totalram_pages()) ++ return -ENOMEM; ++ ++ st = kmalloc(sizeof(*st), GFP_KERNEL); ++ if (st == NULL) ++ return -ENOMEM; ++ ++rebuild_st: ++ if (sg_alloc_table(st, page_count, GFP_KERNEL)) { ++ kfree(st); ++ return -ENOMEM; ++ } ++ ++ /* ++ * Get the list of pages out of our struct file. They'll be pinned ++ * at this point until we release them. ++ * ++ * Fail silently without starting the shrinker ++ */ ++ mapping = obj->base.filp->f_mapping; ++ mapping_set_unevictable(mapping); ++ noreclaim = mapping_gfp_constraint(mapping, ~__GFP_RECLAIM); ++ noreclaim |= __GFP_NORETRY | __GFP_NOWARN; ++ ++ sg = st->sgl; ++ st->nents = 0; ++ sg_page_sizes = 0; ++ for (i = 0; i < page_count; i++) { ++ const unsigned int shrink[] = { ++ I915_SHRINK_BOUND | I915_SHRINK_UNBOUND | I915_SHRINK_PURGEABLE, ++ 0, ++ }, *s = shrink; ++ gfp_t gfp = noreclaim; ++ ++ do { ++ cond_resched(); ++ page = shmem_read_mapping_page_gfp(mapping, i, gfp); ++ if (!IS_ERR(page)) ++ break; ++ ++ if (!*s) { ++ ret = PTR_ERR(page); ++ goto err_sg; ++ } ++ ++ i915_gem_shrink(dev_priv, 2 * page_count, NULL, *s++); ++ ++ /* ++ * We've tried hard to allocate the memory by reaping ++ * our own buffer, now let the real VM do its job and ++ * go down in flames if truly OOM. ++ * ++ * However, since graphics tend to be disposable, ++ * defer the oom here by reporting the ENOMEM back ++ * to userspace. ++ */ ++ if (!*s) { ++ /* reclaim and warn, but no oom */ ++ gfp = mapping_gfp_mask(mapping); ++ ++ /* ++ * Our bo are always dirty and so we require ++ * kswapd to reclaim our pages (direct reclaim ++ * does not effectively begin pageout of our ++ * buffers on its own). However, direct reclaim ++ * only waits for kswapd when under allocation ++ * congestion. So as a result __GFP_RECLAIM is ++ * unreliable and fails to actually reclaim our ++ * dirty pages -- unless you try over and over ++ * again with !__GFP_NORETRY. However, we still ++ * want to fail this allocation rather than ++ * trigger the out-of-memory killer and for ++ * this we want __GFP_RETRY_MAYFAIL. ++ */ ++ gfp |= __GFP_RETRY_MAYFAIL; ++ } ++ } while (1); ++ ++ if (!i || ++ sg->length >= max_segment || ++ page_to_pfn(page) != last_pfn + 1) { ++ if (i) { ++ sg_page_sizes |= sg->length; ++ sg = sg_next(sg); ++ } ++ st->nents++; ++ sg_set_page(sg, page, PAGE_SIZE, 0); ++ } else { ++ sg->length += PAGE_SIZE; ++ } ++ last_pfn = page_to_pfn(page); ++ ++ /* Check that the i965g/gm workaround works. */ ++ WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL)); ++ } ++ if (sg) { /* loop terminated early; short sg table */ ++ sg_page_sizes |= sg->length; ++ sg_mark_end(sg); ++ } ++ ++ /* Trim unused sg entries to avoid wasting memory. */ ++ i915_sg_trim(st); ++ ++ ret = i915_gem_gtt_prepare_pages(obj, st); ++ if (ret) { ++ /* ++ * DMA remapping failed? One possible cause is that ++ * it could not reserve enough large entries, asking ++ * for PAGE_SIZE chunks instead may be helpful. ++ */ ++ if (max_segment > PAGE_SIZE) { ++ for_each_sgt_page(page, sgt_iter, st) ++ put_page(page); ++ sg_free_table(st); ++ ++ max_segment = PAGE_SIZE; ++ goto rebuild_st; ++ } else { ++ dev_warn(&dev_priv->drm.pdev->dev, ++ "Failed to DMA remap %lu pages\n", ++ page_count); ++ goto err_pages; ++ } ++ } ++ ++ if (i915_gem_object_needs_bit17_swizzle(obj)) ++ i915_gem_object_do_bit_17_swizzle(obj, st); ++ ++ __i915_gem_object_set_pages(obj, st, sg_page_sizes); ++ ++ return 0; ++ ++err_sg: ++ sg_mark_end(sg); ++err_pages: ++ mapping_clear_unevictable(mapping); ++ pagevec_init(&pvec); ++ for_each_sgt_page(page, sgt_iter, st) { ++ if (!pagevec_add(&pvec, page)) ++ check_release_pagevec(&pvec); ++ } ++ if (pagevec_count(&pvec)) ++ check_release_pagevec(&pvec); ++ sg_free_table(st); ++ kfree(st); ++ ++ /* ++ * shmemfs first checks if there is enough memory to allocate the page ++ * and reports ENOSPC should there be insufficient, along with the usual ++ * ENOMEM for a genuine allocation failure. ++ * ++ * We use ENOSPC in our driver to mean that we have run out of aperture ++ * space and so want to translate the error from shmemfs back to our ++ * usual understanding of ENOMEM. ++ */ ++ if (ret == -ENOSPC) ++ ret = -ENOMEM; ++ ++ return ret; ++} ++ ++void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, ++ struct sg_table *pages, ++ unsigned int sg_page_sizes) ++{ ++ struct drm_i915_private *i915 = to_i915(obj->base.dev); ++ unsigned long supported = INTEL_INFO(i915)->page_sizes; ++ int i; ++ ++ lockdep_assert_held(&obj->mm.lock); ++ ++ /* Make the pages coherent with the GPU (flushing any swapin). */ ++ if (obj->cache_dirty) { ++ obj->write_domain = 0; ++ if (i915_gem_object_has_struct_page(obj)) ++ drm_clflush_sg(pages); ++ obj->cache_dirty = false; ++ } ++ ++ obj->mm.get_page.sg_pos = pages->sgl; ++ obj->mm.get_page.sg_idx = 0; ++ ++ obj->mm.pages = pages; ++ ++ if (i915_gem_object_is_tiled(obj) && ++ i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) { ++ GEM_BUG_ON(obj->mm.quirked); ++ __i915_gem_object_pin_pages(obj); ++ obj->mm.quirked = true; ++ } ++ ++ GEM_BUG_ON(!sg_page_sizes); ++ obj->mm.page_sizes.phys = sg_page_sizes; ++ ++ /* ++ * Calculate the supported page-sizes which fit into the given ++ * sg_page_sizes. This will give us the page-sizes which we may be able ++ * to use opportunistically when later inserting into the GTT. For ++ * example if phys=2G, then in theory we should be able to use 1G, 2M, ++ * 64K or 4K pages, although in practice this will depend on a number of ++ * other factors. ++ */ ++ obj->mm.page_sizes.sg = 0; ++ for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) { ++ if (obj->mm.page_sizes.phys & ~0u << i) ++ obj->mm.page_sizes.sg |= BIT(i); ++ } ++ GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg)); ++ ++ spin_lock(&i915->mm.obj_lock); ++ list_add(&obj->mm.link, &i915->mm.unbound_list); ++ spin_unlock(&i915->mm.obj_lock); ++} ++ ++static int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj) ++{ ++ int err; ++ ++ if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) { ++ DRM_DEBUG("Attempting to obtain a purgeable object\n"); ++ return -EFAULT; ++ } ++ ++ err = obj->ops->get_pages(obj); ++ GEM_BUG_ON(!err && !i915_gem_object_has_pages(obj)); ++ ++ return err; ++} ++ ++/* Ensure that the associated pages are gathered from the backing storage ++ * and pinned into our object. i915_gem_object_pin_pages() may be called ++ * multiple times before they are released by a single call to ++ * i915_gem_object_unpin_pages() - once the pages are no longer referenced ++ * either as a result of memory pressure (reaping pages under the shrinker) ++ * or as the object is itself released. ++ */ ++int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj) ++{ ++ int err; ++ ++ err = mutex_lock_interruptible(&obj->mm.lock); ++ if (err) ++ return err; ++ ++ if (unlikely(!i915_gem_object_has_pages(obj))) { ++ GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj)); ++ ++ err = ____i915_gem_object_get_pages(obj); ++ if (err) ++ goto unlock; ++ ++ smp_mb__before_atomic(); ++ } ++ atomic_inc(&obj->mm.pages_pin_count); ++ ++unlock: ++ mutex_unlock(&obj->mm.lock); ++ return err; ++} ++ ++/* The 'mapping' part of i915_gem_object_pin_map() below */ ++static void *i915_gem_object_map(const struct drm_i915_gem_object *obj, ++ enum i915_map_type type) ++{ ++ unsigned long n_pages = obj->base.size >> PAGE_SHIFT; ++ struct sg_table *sgt = obj->mm.pages; ++ struct sgt_iter sgt_iter; ++ struct page *page; ++ struct page *stack_pages[32]; ++ struct page **pages = stack_pages; ++ unsigned long i = 0; ++ pgprot_t pgprot; ++ void *addr; ++ ++ /* A single page can always be kmapped */ ++ if (n_pages == 1 && type == I915_MAP_WB) ++ return kmap(sg_page(sgt->sgl)); ++ ++ if (n_pages > ARRAY_SIZE(stack_pages)) { ++ /* Too big for stack -- allocate temporary array instead */ ++ pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL); ++ if (!pages) ++ return NULL; ++ } ++ ++ for_each_sgt_page(page, sgt_iter, sgt) ++ pages[i++] = page; ++ ++ /* Check that we have the expected number of pages */ ++ GEM_BUG_ON(i != n_pages); ++ ++ switch (type) { ++ default: ++ MISSING_CASE(type); ++ /* fallthrough to use PAGE_KERNEL anyway */ ++ case I915_MAP_WB: ++ pgprot = PAGE_KERNEL; ++ break; ++ case I915_MAP_WC: ++ pgprot = pgprot_writecombine(PAGE_KERNEL_IO); ++ break; ++ } ++ addr = vmap(pages, n_pages, 0, pgprot); ++ ++ if (pages != stack_pages) ++ kvfree(pages); ++ ++ return addr; ++} ++ ++/* get, pin, and map the pages of the object into kernel space */ ++void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj, ++ enum i915_map_type type) ++{ ++ enum i915_map_type has_type; ++ bool pinned; ++ void *ptr; ++ int ret; ++ ++ if (unlikely(!i915_gem_object_has_struct_page(obj))) ++ return ERR_PTR(-ENXIO); ++ ++ ret = mutex_lock_interruptible(&obj->mm.lock); ++ if (ret) ++ return ERR_PTR(ret); ++ ++ pinned = !(type & I915_MAP_OVERRIDE); ++ type &= ~I915_MAP_OVERRIDE; ++ ++ if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) { ++ if (unlikely(!i915_gem_object_has_pages(obj))) { ++ GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj)); ++ ++ ret = ____i915_gem_object_get_pages(obj); ++ if (ret) ++ goto err_unlock; ++ ++ smp_mb__before_atomic(); ++ } ++ atomic_inc(&obj->mm.pages_pin_count); ++ pinned = false; ++ } ++ GEM_BUG_ON(!i915_gem_object_has_pages(obj)); ++ ++ ptr = page_unpack_bits(obj->mm.mapping, &has_type); ++ if (ptr && has_type != type) { ++ if (pinned) { ++ ret = -EBUSY; ++ goto err_unpin; ++ } ++ ++ if (is_vmalloc_addr(ptr)) ++ vunmap(ptr); ++ else ++ kunmap(kmap_to_page(ptr)); ++ ++ ptr = obj->mm.mapping = NULL; ++ } ++ ++ if (!ptr) { ++ ptr = i915_gem_object_map(obj, type); ++ if (!ptr) { ++ ret = -ENOMEM; ++ goto err_unpin; ++ } ++ ++ obj->mm.mapping = page_pack_bits(ptr, type); ++ } ++ ++out_unlock: ++ mutex_unlock(&obj->mm.lock); ++ return ptr; ++ ++err_unpin: ++ atomic_dec(&obj->mm.pages_pin_count); ++err_unlock: ++ ptr = ERR_PTR(ret); ++ goto out_unlock; ++} ++ ++void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj, ++ unsigned long offset, ++ unsigned long size) ++{ ++ enum i915_map_type has_type; ++ void *ptr; ++ ++ GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); ++ GEM_BUG_ON(range_overflows_t(typeof(obj->base.size), ++ offset, size, obj->base.size)); ++ ++ obj->mm.dirty = true; ++ ++ if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE) ++ return; ++ ++ ptr = page_unpack_bits(obj->mm.mapping, &has_type); ++ if (has_type == I915_MAP_WC) ++ return; ++ ++ drm_clflush_virt_range(ptr + offset, size); ++ if (size == obj->base.size) { ++ obj->write_domain &= ~I915_GEM_DOMAIN_CPU; ++ obj->cache_dirty = false; ++ } ++} ++ ++static int ++i915_gem_object_pwrite_gtt(struct drm_i915_gem_object *obj, ++ const struct drm_i915_gem_pwrite *arg) ++{ ++ struct address_space *mapping = obj->base.filp->f_mapping; ++ char __user *user_data = u64_to_user_ptr(arg->data_ptr); ++ u64 remain, offset; ++ unsigned int pg; ++ ++ /* Caller already validated user args */ ++ GEM_BUG_ON(!access_ok(user_data, arg->size)); ++ ++ /* ++ * Before we instantiate/pin the backing store for our use, we ++ * can prepopulate the shmemfs filp efficiently using a write into ++ * the pagecache. We avoid the penalty of instantiating all the ++ * pages, important if the user is just writing to a few and never ++ * uses the object on the GPU, and using a direct write into shmemfs ++ * allows it to avoid the cost of retrieving a page (either swapin ++ * or clearing-before-use) before it is overwritten. ++ */ ++ if (i915_gem_object_has_pages(obj)) ++ return -ENODEV; ++ ++ if (obj->mm.madv != I915_MADV_WILLNEED) ++ return -EFAULT; ++ ++ /* ++ * Before the pages are instantiated the object is treated as being ++ * in the CPU domain. The pages will be clflushed as required before ++ * use, and we can freely write into the pages directly. If userspace ++ * races pwrite with any other operation; corruption will ensue - ++ * that is userspace's prerogative! ++ */ ++ ++ remain = arg->size; ++ offset = arg->offset; ++ pg = offset_in_page(offset); ++ ++ do { ++ unsigned int len, unwritten; ++ struct page *page; ++ void *data, *vaddr; ++ int err; ++ char c; ++ ++ len = PAGE_SIZE - pg; ++ if (len > remain) ++ len = remain; ++ ++ /* Prefault the user page to reduce potential recursion */ ++ err = __get_user(c, user_data); ++ if (err) ++ return err; ++ ++ err = __get_user(c, user_data + len - 1); ++ if (err) ++ return err; ++ ++ err = pagecache_write_begin(obj->base.filp, mapping, ++ offset, len, 0, ++ &page, &data); ++ if (err < 0) ++ return err; ++ ++ vaddr = kmap_atomic(page); ++ unwritten = __copy_from_user_inatomic(vaddr + pg, ++ user_data, ++ len); ++ kunmap_atomic(vaddr); ++ ++ err = pagecache_write_end(obj->base.filp, mapping, ++ offset, len, len - unwritten, ++ page, data); ++ if (err < 0) ++ return err; ++ ++ /* We don't handle -EFAULT, leave it to the caller to check */ ++ if (unwritten) ++ return -ENODEV; ++ ++ remain -= len; ++ user_data += len; ++ offset += len; ++ pg = 0; ++ } while (remain); ++ ++ return 0; ++} ++ ++static void ++i915_gem_retire_work_handler(struct work_struct *work) ++{ ++ struct drm_i915_private *dev_priv = ++ container_of(work, typeof(*dev_priv), gt.retire_work.work); ++ struct drm_device *dev = &dev_priv->drm; ++ ++ /* Come back later if the device is busy... */ ++ if (mutex_trylock(&dev->struct_mutex)) { ++ i915_retire_requests(dev_priv); ++ mutex_unlock(&dev->struct_mutex); ++ } ++ ++ /* ++ * Keep the retire handler running until we are finally idle. ++ * We do not need to do this test under locking as in the worst-case ++ * we queue the retire worker once too often. ++ */ ++ if (READ_ONCE(dev_priv->gt.awake)) ++ queue_delayed_work(dev_priv->wq, ++ &dev_priv->gt.retire_work, ++ round_jiffies_up_relative(HZ)); ++} ++ ++static bool switch_to_kernel_context_sync(struct drm_i915_private *i915, ++ unsigned long mask) ++{ ++ bool result = true; ++ ++ /* ++ * Even if we fail to switch, give whatever is running a small chance ++ * to save itself before we report the failure. Yes, this may be a ++ * false positive due to e.g. ENOMEM, caveat emptor! ++ */ ++ if (i915_gem_switch_to_kernel_context(i915, mask)) ++ result = false; ++ ++ if (i915_gem_wait_for_idle(i915, ++ I915_WAIT_LOCKED | ++ I915_WAIT_FOR_IDLE_BOOST, ++ I915_GEM_IDLE_TIMEOUT)) ++ result = false; ++ ++ if (!result) { ++ if (i915_modparams.reset) { /* XXX hide warning from gem_eio */ ++ dev_err(i915->drm.dev, ++ "Failed to idle engines, declaring wedged!\n"); ++ GEM_TRACE_DUMP(); ++ } ++ ++ /* Forcibly cancel outstanding work and leave the gpu quiet. */ ++ i915_gem_set_wedged(i915); ++ } ++ ++ i915_retire_requests(i915); /* ensure we flush after wedging */ ++ return result; ++} ++ ++static bool load_power_context(struct drm_i915_private *i915) ++{ ++ /* Force loading the kernel context on all engines */ ++ if (!switch_to_kernel_context_sync(i915, ALL_ENGINES)) ++ return false; ++ ++ /* ++ * Immediately park the GPU so that we enable powersaving and ++ * treat it as idle. The next time we issue a request, we will ++ * unpark and start using the engine->pinned_default_state, otherwise ++ * it is in limbo and an early reset may fail. ++ */ ++ __i915_gem_park(i915); ++ ++ return true; ++} ++ ++static void ++i915_gem_idle_work_handler(struct work_struct *work) ++{ ++ struct drm_i915_private *i915 = ++ container_of(work, typeof(*i915), gt.idle_work.work); ++ bool rearm_hangcheck; ++ ++ if (!READ_ONCE(i915->gt.awake)) ++ return; ++ ++ if (READ_ONCE(i915->gt.active_requests)) ++ return; ++ ++ rearm_hangcheck = ++ cancel_delayed_work_sync(&i915->gpu_error.hangcheck_work); ++ ++ if (!mutex_trylock(&i915->drm.struct_mutex)) { ++ /* Currently busy, come back later */ ++ mod_delayed_work(i915->wq, ++ &i915->gt.idle_work, ++ msecs_to_jiffies(50)); ++ goto out_rearm; ++ } ++ ++ /* ++ * Flush out the last user context, leaving only the pinned ++ * kernel context resident. Should anything unfortunate happen ++ * while we are idle (such as the GPU being power cycled), no users ++ * will be harmed. ++ */ ++ if (!work_pending(&i915->gt.idle_work.work) && ++ !i915->gt.active_requests) { ++ ++i915->gt.active_requests; /* don't requeue idle */ ++ ++ switch_to_kernel_context_sync(i915, i915->gt.active_engines); ++ ++ if (!--i915->gt.active_requests) { ++ __i915_gem_park(i915); ++ rearm_hangcheck = false; ++ } ++ } ++ ++ mutex_unlock(&i915->drm.struct_mutex); ++ ++out_rearm: ++ if (rearm_hangcheck) { ++ GEM_BUG_ON(!i915->gt.awake); ++ i915_queue_hangcheck(i915); ++ } ++} ++ ++void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file) ++{ ++ struct drm_i915_private *i915 = to_i915(gem->dev); ++ struct drm_i915_gem_object *obj = to_intel_bo(gem); ++ struct drm_i915_file_private *fpriv = file->driver_priv; ++ struct i915_lut_handle *lut, *ln; ++ ++ mutex_lock(&i915->drm.struct_mutex); ++ ++ list_for_each_entry_safe(lut, ln, &obj->lut_list, obj_link) { ++ struct i915_gem_context *ctx = lut->ctx; ++ struct i915_vma *vma; ++ ++ GEM_BUG_ON(ctx->file_priv == ERR_PTR(-EBADF)); ++ if (ctx->file_priv != fpriv) ++ continue; ++ ++ vma = radix_tree_delete(&ctx->handles_vma, lut->handle); ++ GEM_BUG_ON(vma->obj != obj); ++ ++ /* We allow the process to have multiple handles to the same ++ * vma, in the same fd namespace, by virtue of flink/open. ++ */ ++ GEM_BUG_ON(!vma->open_count); ++ if (!--vma->open_count && !i915_vma_is_ggtt(vma)) ++ i915_vma_close(vma); ++ ++ list_del(&lut->obj_link); ++ list_del(&lut->ctx_link); ++ ++ i915_lut_handle_free(lut); ++ __i915_gem_object_release_unless_active(obj); ++ } ++ ++ mutex_unlock(&i915->drm.struct_mutex); ++} ++ ++static unsigned long to_wait_timeout(s64 timeout_ns) ++{ ++ if (timeout_ns < 0) ++ return MAX_SCHEDULE_TIMEOUT; ++ ++ if (timeout_ns == 0) ++ return 0; ++ ++ return nsecs_to_jiffies_timeout(timeout_ns); ++} ++ ++/** ++ * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT ++ * @dev: drm device pointer ++ * @data: ioctl data blob ++ * @file: drm file pointer ++ * ++ * Returns 0 if successful, else an error is returned with the remaining time in ++ * the timeout parameter. ++ * -ETIME: object is still busy after timeout ++ * -ERESTARTSYS: signal interrupted the wait ++ * -ENONENT: object doesn't exist ++ * Also possible, but rare: ++ * -EAGAIN: incomplete, restart syscall ++ * -ENOMEM: damn ++ * -ENODEV: Internal IRQ fail ++ * -E?: The add request failed ++ * ++ * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any ++ * non-zero timeout parameter the wait ioctl will wait for the given number of ++ * nanoseconds on an object becoming unbusy. Since the wait itself does so ++ * without holding struct_mutex the object may become re-busied before this ++ * function completes. A similar but shorter * race condition exists in the busy ++ * ioctl ++ */ ++int ++i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) ++{ ++ struct drm_i915_gem_wait *args = data; ++ struct drm_i915_gem_object *obj; ++ ktime_t start; ++ long ret; ++ ++ if (args->flags != 0) ++ return -EINVAL; ++ ++ obj = i915_gem_object_lookup(file, args->bo_handle); ++ if (!obj) ++ return -ENOENT; ++ ++ start = ktime_get(); ++ ++ ret = i915_gem_object_wait(obj, ++ I915_WAIT_INTERRUPTIBLE | ++ I915_WAIT_PRIORITY | ++ I915_WAIT_ALL, ++ to_wait_timeout(args->timeout_ns)); ++ ++ if (args->timeout_ns > 0) { ++ args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start)); ++ if (args->timeout_ns < 0) ++ args->timeout_ns = 0; ++ ++ /* ++ * Apparently ktime isn't accurate enough and occasionally has a ++ * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch ++ * things up to make the test happy. We allow up to 1 jiffy. ++ * ++ * This is a regression from the timespec->ktime conversion. ++ */ ++ if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns)) ++ args->timeout_ns = 0; ++ ++ /* Asked to wait beyond the jiffie/scheduler precision? */ ++ if (ret == -ETIME && args->timeout_ns) ++ ret = -EAGAIN; ++ } ++ ++ i915_gem_object_put(obj); ++ return ret; ++} ++ ++static int wait_for_engines(struct drm_i915_private *i915) ++{ ++ if (wait_for(intel_engines_are_idle(i915), I915_IDLE_ENGINES_TIMEOUT)) { ++ dev_err(i915->drm.dev, ++ "Failed to idle engines, declaring wedged!\n"); ++ GEM_TRACE_DUMP(); ++ i915_gem_set_wedged(i915); ++ return -EIO; ++ } ++ ++ return 0; ++} ++ ++static long ++wait_for_timelines(struct drm_i915_private *i915, ++ unsigned int flags, long timeout) ++{ ++ struct i915_gt_timelines *gt = &i915->gt.timelines; ++ struct i915_timeline *tl; ++ ++ if (!READ_ONCE(i915->gt.active_requests)) ++ return timeout; ++ ++ mutex_lock(>->mutex); ++ list_for_each_entry(tl, >->active_list, link) { ++ struct i915_request *rq; ++ ++ rq = i915_active_request_get_unlocked(&tl->last_request); ++ if (!rq) ++ continue; ++ ++ mutex_unlock(>->mutex); ++ ++ /* ++ * "Race-to-idle". ++ * ++ * Switching to the kernel context is often used a synchronous ++ * step prior to idling, e.g. in suspend for flushing all ++ * current operations to memory before sleeping. These we ++ * want to complete as quickly as possible to avoid prolonged ++ * stalls, so allow the gpu to boost to maximum clocks. ++ */ ++ if (flags & I915_WAIT_FOR_IDLE_BOOST) ++ gen6_rps_boost(rq); ++ ++ timeout = i915_request_wait(rq, flags, timeout); ++ i915_request_put(rq); ++ if (timeout < 0) ++ return timeout; ++ ++ /* restart after reacquiring the lock */ ++ mutex_lock(>->mutex); ++ tl = list_entry(>->active_list, typeof(*tl), link); ++ } ++ mutex_unlock(>->mutex); ++ ++ return timeout; ++} ++ ++int i915_gem_wait_for_idle(struct drm_i915_private *i915, ++ unsigned int flags, long timeout) ++{ ++ GEM_TRACE("flags=%x (%s), timeout=%ld%s\n", ++ flags, flags & I915_WAIT_LOCKED ? "locked" : "unlocked", ++ timeout, timeout == MAX_SCHEDULE_TIMEOUT ? " (forever)" : ""); ++ ++ /* If the device is asleep, we have no requests outstanding */ ++ if (!READ_ONCE(i915->gt.awake)) ++ return 0; ++ ++ timeout = wait_for_timelines(i915, flags, timeout); ++ if (timeout < 0) ++ return timeout; ++ ++ if (flags & I915_WAIT_LOCKED) { ++ int err; ++ ++ lockdep_assert_held(&i915->drm.struct_mutex); ++ ++ err = wait_for_engines(i915); ++ if (err) ++ return err; ++ ++ i915_retire_requests(i915); ++ } ++ ++ return 0; ++} ++ ++static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj) ++{ ++ /* ++ * We manually flush the CPU domain so that we can override and ++ * force the flush for the display, and perform it asyncrhonously. ++ */ ++ flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU); ++ if (obj->cache_dirty) ++ i915_gem_clflush_object(obj, I915_CLFLUSH_FORCE); ++ obj->write_domain = 0; ++} ++ ++void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj) ++{ ++ if (!READ_ONCE(obj->pin_global)) ++ return; ++ ++ mutex_lock(&obj->base.dev->struct_mutex); ++ __i915_gem_object_flush_for_display(obj); ++ mutex_unlock(&obj->base.dev->struct_mutex); ++} ++ ++/** ++ * Moves a single object to the WC read, and possibly write domain. ++ * @obj: object to act on ++ * @write: ask for write access or read only ++ * ++ * This function returns when the move is complete, including waiting on ++ * flushes to occur. ++ */ ++int ++i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write) ++{ ++ int ret; ++ ++ lockdep_assert_held(&obj->base.dev->struct_mutex); ++ ++ ret = i915_gem_object_wait(obj, ++ I915_WAIT_INTERRUPTIBLE | ++ I915_WAIT_LOCKED | ++ (write ? I915_WAIT_ALL : 0), ++ MAX_SCHEDULE_TIMEOUT); ++ if (ret) ++ return ret; ++ ++ if (obj->write_domain == I915_GEM_DOMAIN_WC) ++ return 0; ++ ++ /* Flush and acquire obj->pages so that we are coherent through ++ * direct access in memory with previous cached writes through ++ * shmemfs and that our cache domain tracking remains valid. ++ * For example, if the obj->filp was moved to swap without us ++ * being notified and releasing the pages, we would mistakenly ++ * continue to assume that the obj remained out of the CPU cached ++ * domain. ++ */ ++ ret = i915_gem_object_pin_pages(obj); ++ if (ret) ++ return ret; ++ ++ flush_write_domain(obj, ~I915_GEM_DOMAIN_WC); ++ ++ /* Serialise direct access to this object with the barriers for ++ * coherent writes from the GPU, by effectively invalidating the ++ * WC domain upon first access. ++ */ ++ if ((obj->read_domains & I915_GEM_DOMAIN_WC) == 0) ++ mb(); ++ ++ /* It should now be out of any other write domains, and we can update ++ * the domain values for our changes. ++ */ ++ GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_WC) != 0); ++ obj->read_domains |= I915_GEM_DOMAIN_WC; ++ if (write) { ++ obj->read_domains = I915_GEM_DOMAIN_WC; ++ obj->write_domain = I915_GEM_DOMAIN_WC; ++ obj->mm.dirty = true; ++ } ++ ++ i915_gem_object_unpin_pages(obj); ++ return 0; ++} ++ ++/** ++ * Moves a single object to the GTT read, and possibly write domain. ++ * @obj: object to act on ++ * @write: ask for write access or read only ++ * ++ * This function returns when the move is complete, including waiting on ++ * flushes to occur. ++ */ ++int ++i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) ++{ ++ int ret; ++ ++ lockdep_assert_held(&obj->base.dev->struct_mutex); ++ ++ ret = i915_gem_object_wait(obj, ++ I915_WAIT_INTERRUPTIBLE | ++ I915_WAIT_LOCKED | ++ (write ? I915_WAIT_ALL : 0), ++ MAX_SCHEDULE_TIMEOUT); ++ if (ret) ++ return ret; ++ ++ if (obj->write_domain == I915_GEM_DOMAIN_GTT) ++ return 0; ++ ++ /* Flush and acquire obj->pages so that we are coherent through ++ * direct access in memory with previous cached writes through ++ * shmemfs and that our cache domain tracking remains valid. ++ * For example, if the obj->filp was moved to swap without us ++ * being notified and releasing the pages, we would mistakenly ++ * continue to assume that the obj remained out of the CPU cached ++ * domain. ++ */ ++ ret = i915_gem_object_pin_pages(obj); ++ if (ret) ++ return ret; ++ ++ flush_write_domain(obj, ~I915_GEM_DOMAIN_GTT); ++ ++ /* Serialise direct access to this object with the barriers for ++ * coherent writes from the GPU, by effectively invalidating the ++ * GTT domain upon first access. ++ */ ++ if ((obj->read_domains & I915_GEM_DOMAIN_GTT) == 0) ++ mb(); ++ ++ /* It should now be out of any other write domains, and we can update ++ * the domain values for our changes. ++ */ ++ GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0); ++ obj->read_domains |= I915_GEM_DOMAIN_GTT; ++ if (write) { ++ obj->read_domains = I915_GEM_DOMAIN_GTT; ++ obj->write_domain = I915_GEM_DOMAIN_GTT; ++ obj->mm.dirty = true; ++ } ++ ++ i915_gem_object_unpin_pages(obj); ++ return 0; ++} ++ ++/** ++ * Changes the cache-level of an object across all VMA. ++ * @obj: object to act on ++ * @cache_level: new cache level to set for the object ++ * ++ * After this function returns, the object will be in the new cache-level ++ * across all GTT and the contents of the backing storage will be coherent, ++ * with respect to the new cache-level. In order to keep the backing storage ++ * coherent for all users, we only allow a single cache level to be set ++ * globally on the object and prevent it from being changed whilst the ++ * hardware is reading from the object. That is if the object is currently ++ * on the scanout it will be set to uncached (or equivalent display ++ * cache coherency) and all non-MOCS GPU access will also be uncached so ++ * that all direct access to the scanout remains coherent. ++ */ ++int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, ++ enum i915_cache_level cache_level) ++{ ++ struct i915_vma *vma; ++ int ret; ++ ++ lockdep_assert_held(&obj->base.dev->struct_mutex); ++ ++ if (obj->cache_level == cache_level) ++ return 0; ++ ++ /* Inspect the list of currently bound VMA and unbind any that would ++ * be invalid given the new cache-level. This is principally to ++ * catch the issue of the CS prefetch crossing page boundaries and ++ * reading an invalid PTE on older architectures. ++ */ ++restart: ++ list_for_each_entry(vma, &obj->vma.list, obj_link) { ++ if (!drm_mm_node_allocated(&vma->node)) ++ continue; ++ ++ if (i915_vma_is_pinned(vma)) { ++ DRM_DEBUG("can not change the cache level of pinned objects\n"); ++ return -EBUSY; ++ } ++ ++ if (!i915_vma_is_closed(vma) && ++ i915_gem_valid_gtt_space(vma, cache_level)) ++ continue; ++ ++ ret = i915_vma_unbind(vma); ++ if (ret) ++ return ret; ++ ++ /* As unbinding may affect other elements in the ++ * obj->vma_list (due to side-effects from retiring ++ * an active vma), play safe and restart the iterator. ++ */ ++ goto restart; ++ } ++ ++ /* We can reuse the existing drm_mm nodes but need to change the ++ * cache-level on the PTE. We could simply unbind them all and ++ * rebind with the correct cache-level on next use. However since ++ * we already have a valid slot, dma mapping, pages etc, we may as ++ * rewrite the PTE in the belief that doing so tramples upon less ++ * state and so involves less work. ++ */ ++ if (obj->bind_count) { ++ /* Before we change the PTE, the GPU must not be accessing it. ++ * If we wait upon the object, we know that all the bound ++ * VMA are no longer active. ++ */ ++ ret = i915_gem_object_wait(obj, ++ I915_WAIT_INTERRUPTIBLE | ++ I915_WAIT_LOCKED | ++ I915_WAIT_ALL, ++ MAX_SCHEDULE_TIMEOUT); ++ if (ret) ++ return ret; ++ ++ if (!HAS_LLC(to_i915(obj->base.dev)) && ++ cache_level != I915_CACHE_NONE) { ++ /* Access to snoopable pages through the GTT is ++ * incoherent and on some machines causes a hard ++ * lockup. Relinquish the CPU mmaping to force ++ * userspace to refault in the pages and we can ++ * then double check if the GTT mapping is still ++ * valid for that pointer access. ++ */ ++ i915_gem_release_mmap(obj); ++ ++ /* As we no longer need a fence for GTT access, ++ * we can relinquish it now (and so prevent having ++ * to steal a fence from someone else on the next ++ * fence request). Note GPU activity would have ++ * dropped the fence as all snoopable access is ++ * supposed to be linear. ++ */ ++ for_each_ggtt_vma(vma, obj) { ++ ret = i915_vma_put_fence(vma); ++ if (ret) ++ return ret; ++ } ++ } else { ++ /* We either have incoherent backing store and ++ * so no GTT access or the architecture is fully ++ * coherent. In such cases, existing GTT mmaps ++ * ignore the cache bit in the PTE and we can ++ * rewrite it without confusing the GPU or having ++ * to force userspace to fault back in its mmaps. ++ */ ++ } ++ ++ list_for_each_entry(vma, &obj->vma.list, obj_link) { ++ if (!drm_mm_node_allocated(&vma->node)) ++ continue; ++ ++ ret = i915_vma_bind(vma, cache_level, PIN_UPDATE); ++ if (ret) ++ return ret; ++ } ++ } ++ ++ list_for_each_entry(vma, &obj->vma.list, obj_link) ++ vma->node.color = cache_level; ++ i915_gem_object_set_cache_coherency(obj, cache_level); ++ obj->cache_dirty = true; /* Always invalidate stale cachelines */ ++ ++ return 0; ++} ++ ++int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_i915_gem_caching *args = data; ++ struct drm_i915_gem_object *obj; ++ int err = 0; ++ ++ rcu_read_lock(); ++ obj = i915_gem_object_lookup_rcu(file, args->handle); ++ if (!obj) { ++ err = -ENOENT; ++ goto out; ++ } ++ ++ switch (obj->cache_level) { ++ case I915_CACHE_LLC: ++ case I915_CACHE_L3_LLC: ++ args->caching = I915_CACHING_CACHED; ++ break; ++ ++ case I915_CACHE_WT: ++ args->caching = I915_CACHING_DISPLAY; ++ break; ++ ++ default: ++ args->caching = I915_CACHING_NONE; ++ break; ++ } ++out: ++ rcu_read_unlock(); ++ return err; ++} ++ ++int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_i915_private *i915 = to_i915(dev); ++ struct drm_i915_gem_caching *args = data; ++ struct drm_i915_gem_object *obj; ++ enum i915_cache_level level; ++ int ret = 0; ++ ++ switch (args->caching) { ++ case I915_CACHING_NONE: ++ level = I915_CACHE_NONE; ++ break; ++ case I915_CACHING_CACHED: ++ /* ++ * Due to a HW issue on BXT A stepping, GPU stores via a ++ * snooped mapping may leave stale data in a corresponding CPU ++ * cacheline, whereas normally such cachelines would get ++ * invalidated. ++ */ ++ if (!HAS_LLC(i915) && !HAS_SNOOP(i915)) ++ return -ENODEV; ++ ++ level = I915_CACHE_LLC; ++ break; ++ case I915_CACHING_DISPLAY: ++ level = HAS_WT(i915) ? I915_CACHE_WT : I915_CACHE_NONE; ++ break; ++ default: ++ return -EINVAL; ++ } ++ ++ obj = i915_gem_object_lookup(file, args->handle); ++ if (!obj) ++ return -ENOENT; ++ ++ /* ++ * The caching mode of proxy object is handled by its generator, and ++ * not allowed to be changed by userspace. ++ */ ++ if (i915_gem_object_is_proxy(obj)) { ++ ret = -ENXIO; ++ goto out; ++ } ++ ++ if (obj->cache_level == level) ++ goto out; ++ ++ ret = i915_gem_object_wait(obj, ++ I915_WAIT_INTERRUPTIBLE, ++ MAX_SCHEDULE_TIMEOUT); ++ if (ret) ++ goto out; ++ ++ ret = i915_mutex_lock_interruptible(dev); ++ if (ret) ++ goto out; ++ ++ ret = i915_gem_object_set_cache_level(obj, level); ++ mutex_unlock(&dev->struct_mutex); ++ ++out: ++ i915_gem_object_put(obj); ++ return ret; ++} ++ ++/* ++ * Prepare buffer for display plane (scanout, cursors, etc). Can be called from ++ * an uninterruptible phase (modesetting) and allows any flushes to be pipelined ++ * (for pageflips). We only flush the caches while preparing the buffer for ++ * display, the callers are responsible for frontbuffer flush. ++ */ ++struct i915_vma * ++i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, ++ u32 alignment, ++ const struct i915_ggtt_view *view, ++ unsigned int flags) ++{ ++ struct i915_vma *vma; ++ int ret; ++ ++ lockdep_assert_held(&obj->base.dev->struct_mutex); ++ ++ /* Mark the global pin early so that we account for the ++ * display coherency whilst setting up the cache domains. ++ */ ++ obj->pin_global++; ++ ++ /* The display engine is not coherent with the LLC cache on gen6. As ++ * a result, we make sure that the pinning that is about to occur is ++ * done with uncached PTEs. This is lowest common denominator for all ++ * chipsets. ++ * ++ * However for gen6+, we could do better by using the GFDT bit instead ++ * of uncaching, which would allow us to flush all the LLC-cached data ++ * with that bit in the PTE to main memory with just one PIPE_CONTROL. ++ */ ++ ret = i915_gem_object_set_cache_level(obj, ++ HAS_WT(to_i915(obj->base.dev)) ? ++ I915_CACHE_WT : I915_CACHE_NONE); ++ if (ret) { ++ vma = ERR_PTR(ret); ++ goto err_unpin_global; ++ } ++ ++ /* As the user may map the buffer once pinned in the display plane ++ * (e.g. libkms for the bootup splash), we have to ensure that we ++ * always use map_and_fenceable for all scanout buffers. However, ++ * it may simply be too big to fit into mappable, in which case ++ * put it anyway and hope that userspace can cope (but always first ++ * try to preserve the existing ABI). ++ */ ++ vma = ERR_PTR(-ENOSPC); ++ if ((flags & PIN_MAPPABLE) == 0 && ++ (!view || view->type == I915_GGTT_VIEW_NORMAL)) ++ vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, ++ flags | ++ PIN_MAPPABLE | ++ PIN_NONBLOCK); ++ if (IS_ERR(vma)) ++ vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, flags); ++ if (IS_ERR(vma)) ++ goto err_unpin_global; ++ ++ vma->display_alignment = max_t(u64, vma->display_alignment, alignment); ++ ++ __i915_gem_object_flush_for_display(obj); ++ ++ /* It should now be out of any other write domains, and we can update ++ * the domain values for our changes. ++ */ ++ obj->read_domains |= I915_GEM_DOMAIN_GTT; ++ ++ return vma; ++ ++err_unpin_global: ++ obj->pin_global--; ++ return vma; ++} ++ ++void ++i915_gem_object_unpin_from_display_plane(struct i915_vma *vma) ++{ ++ lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); ++ ++ if (WARN_ON(vma->obj->pin_global == 0)) ++ return; ++ ++ if (--vma->obj->pin_global == 0) ++ vma->display_alignment = I915_GTT_MIN_ALIGNMENT; ++ ++ /* Bump the LRU to try and avoid premature eviction whilst flipping */ ++ i915_gem_object_bump_inactive_ggtt(vma->obj); ++ ++ i915_vma_unpin(vma); ++} ++ ++/** ++ * Moves a single object to the CPU read, and possibly write domain. ++ * @obj: object to act on ++ * @write: requesting write or read-only access ++ * ++ * This function returns when the move is complete, including waiting on ++ * flushes to occur. ++ */ ++int ++i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) ++{ ++ int ret; ++ ++ lockdep_assert_held(&obj->base.dev->struct_mutex); ++ ++ ret = i915_gem_object_wait(obj, ++ I915_WAIT_INTERRUPTIBLE | ++ I915_WAIT_LOCKED | ++ (write ? I915_WAIT_ALL : 0), ++ MAX_SCHEDULE_TIMEOUT); ++ if (ret) ++ return ret; ++ ++ flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU); ++ ++ /* Flush the CPU cache if it's still invalid. */ ++ if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) { ++ i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC); ++ obj->read_domains |= I915_GEM_DOMAIN_CPU; ++ } ++ ++ /* It should now be out of any other write domains, and we can update ++ * the domain values for our changes. ++ */ ++ GEM_BUG_ON(obj->write_domain & ~I915_GEM_DOMAIN_CPU); ++ ++ /* If we're writing through the CPU, then the GPU read domains will ++ * need to be invalidated at next use. ++ */ ++ if (write) ++ __start_cpu_write(obj); ++ ++ return 0; ++} ++ ++/* Throttle our rendering by waiting until the ring has completed our requests ++ * emitted over 20 msec ago. ++ * ++ * Note that if we were to use the current jiffies each time around the loop, ++ * we wouldn't escape the function with any frames outstanding if the time to ++ * render a frame was over 20ms. ++ * ++ * This should get us reasonable parallelism between CPU and GPU but also ++ * relatively low latency when blocking on a particular request to finish. ++ */ ++static int ++i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct drm_i915_file_private *file_priv = file->driver_priv; ++ unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES; ++ struct i915_request *request, *target = NULL; ++ long ret; ++ ++ /* ABI: return -EIO if already wedged */ ++ ret = i915_terminally_wedged(dev_priv); ++ if (ret) ++ return ret; ++ ++ spin_lock(&file_priv->mm.lock); ++ list_for_each_entry(request, &file_priv->mm.request_list, client_link) { ++ if (time_after_eq(request->emitted_jiffies, recent_enough)) ++ break; ++ ++ if (target) { ++ list_del(&target->client_link); ++ target->file_priv = NULL; ++ } ++ ++ target = request; ++ } ++ if (target) ++ i915_request_get(target); ++ spin_unlock(&file_priv->mm.lock); ++ ++ if (target == NULL) ++ return 0; ++ ++ ret = i915_request_wait(target, ++ I915_WAIT_INTERRUPTIBLE, ++ MAX_SCHEDULE_TIMEOUT); ++ i915_request_put(target); ++ ++ return ret < 0 ? ret : 0; ++} ++ ++struct i915_vma * ++i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, ++ const struct i915_ggtt_view *view, ++ u64 size, ++ u64 alignment, ++ u64 flags) ++{ ++ struct drm_i915_private *dev_priv = to_i915(obj->base.dev); ++ struct i915_address_space *vm = &dev_priv->ggtt.vm; ++ struct i915_vma *vma; ++ int ret; ++ ++ lockdep_assert_held(&obj->base.dev->struct_mutex); ++ ++ if (flags & PIN_MAPPABLE && ++ (!view || view->type == I915_GGTT_VIEW_NORMAL)) { ++ /* If the required space is larger than the available ++ * aperture, we will not able to find a slot for the ++ * object and unbinding the object now will be in ++ * vain. Worse, doing so may cause us to ping-pong ++ * the object in and out of the Global GTT and ++ * waste a lot of cycles under the mutex. ++ */ ++ if (obj->base.size > dev_priv->ggtt.mappable_end) ++ return ERR_PTR(-E2BIG); ++ ++ /* If NONBLOCK is set the caller is optimistically ++ * trying to cache the full object within the mappable ++ * aperture, and *must* have a fallback in place for ++ * situations where we cannot bind the object. We ++ * can be a little more lax here and use the fallback ++ * more often to avoid costly migrations of ourselves ++ * and other objects within the aperture. ++ * ++ * Half-the-aperture is used as a simple heuristic. ++ * More interesting would to do search for a free ++ * block prior to making the commitment to unbind. ++ * That caters for the self-harm case, and with a ++ * little more heuristics (e.g. NOFAULT, NOEVICT) ++ * we could try to minimise harm to others. ++ */ ++ if (flags & PIN_NONBLOCK && ++ obj->base.size > dev_priv->ggtt.mappable_end / 2) ++ return ERR_PTR(-ENOSPC); ++ } ++ ++ vma = i915_vma_instance(obj, vm, view); ++ if (IS_ERR(vma)) ++ return vma; ++ ++ if (i915_vma_misplaced(vma, size, alignment, flags)) { ++ if (flags & PIN_NONBLOCK) { ++ if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)) ++ return ERR_PTR(-ENOSPC); ++ ++ if (flags & PIN_MAPPABLE && ++ vma->fence_size > dev_priv->ggtt.mappable_end / 2) ++ return ERR_PTR(-ENOSPC); ++ } ++ ++ WARN(i915_vma_is_pinned(vma), ++ "bo is already pinned in ggtt with incorrect alignment:" ++ " offset=%08x, req.alignment=%llx," ++ " req.map_and_fenceable=%d, vma->map_and_fenceable=%d\n", ++ i915_ggtt_offset(vma), alignment, ++ !!(flags & PIN_MAPPABLE), ++ i915_vma_is_map_and_fenceable(vma)); ++ ret = i915_vma_unbind(vma); ++ if (ret) ++ return ERR_PTR(ret); ++ } ++ ++ ret = i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL); ++ if (ret) ++ return ERR_PTR(ret); ++ ++ return vma; ++} ++ ++static __always_inline u32 __busy_read_flag(u8 id) ++{ ++ if (id == (u8)I915_ENGINE_CLASS_INVALID) ++ return 0xffff0000u; ++ ++ GEM_BUG_ON(id >= 16); ++ return 0x10000u << id; ++} ++ ++static __always_inline u32 __busy_write_id(u8 id) ++{ ++ /* ++ * The uABI guarantees an active writer is also amongst the read ++ * engines. This would be true if we accessed the activity tracking ++ * under the lock, but as we perform the lookup of the object and ++ * its activity locklessly we can not guarantee that the last_write ++ * being active implies that we have set the same engine flag from ++ * last_read - hence we always set both read and write busy for ++ * last_write. ++ */ ++ if (id == (u8)I915_ENGINE_CLASS_INVALID) ++ return 0xffffffffu; ++ ++ return (id + 1) | __busy_read_flag(id); ++} ++ ++static __always_inline unsigned int ++__busy_set_if_active(const struct dma_fence *fence, u32 (*flag)(u8 id)) ++{ ++ const struct i915_request *rq; ++ ++ /* ++ * We have to check the current hw status of the fence as the uABI ++ * guarantees forward progress. We could rely on the idle worker ++ * to eventually flush us, but to minimise latency just ask the ++ * hardware. ++ * ++ * Note we only report on the status of native fences. ++ */ ++ if (!dma_fence_is_i915(fence)) ++ return 0; ++ ++ /* opencode to_request() in order to avoid const warnings */ ++ rq = container_of(fence, const struct i915_request, fence); ++ if (i915_request_completed(rq)) ++ return 0; ++ ++ /* Beware type-expansion follies! */ ++ BUILD_BUG_ON(!typecheck(u8, rq->engine->uabi_class)); ++ return flag(rq->engine->uabi_class); ++} ++ ++static __always_inline unsigned int ++busy_check_reader(const struct dma_fence *fence) ++{ ++ return __busy_set_if_active(fence, __busy_read_flag); ++} ++ ++static __always_inline unsigned int ++busy_check_writer(const struct dma_fence *fence) ++{ ++ if (!fence) ++ return 0; ++ ++ return __busy_set_if_active(fence, __busy_write_id); ++} ++ ++int ++i915_gem_busy_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_i915_gem_busy *args = data; ++ struct drm_i915_gem_object *obj; ++ struct reservation_object_list *list; ++ unsigned int seq; ++ int err; ++ ++ err = -ENOENT; ++ rcu_read_lock(); ++ obj = i915_gem_object_lookup_rcu(file, args->handle); ++ if (!obj) ++ goto out; ++ ++ /* ++ * A discrepancy here is that we do not report the status of ++ * non-i915 fences, i.e. even though we may report the object as idle, ++ * a call to set-domain may still stall waiting for foreign rendering. ++ * This also means that wait-ioctl may report an object as busy, ++ * where busy-ioctl considers it idle. ++ * ++ * We trade the ability to warn of foreign fences to report on which ++ * i915 engines are active for the object. ++ * ++ * Alternatively, we can trade that extra information on read/write ++ * activity with ++ * args->busy = ++ * !reservation_object_test_signaled_rcu(obj->resv, true); ++ * to report the overall busyness. This is what the wait-ioctl does. ++ * ++ */ ++retry: ++ seq = raw_read_seqcount(&obj->resv->seq); ++ ++ /* Translate the exclusive fence to the READ *and* WRITE engine */ ++ args->busy = busy_check_writer(rcu_dereference(obj->resv->fence_excl)); ++ ++ /* Translate shared fences to READ set of engines */ ++ list = rcu_dereference(obj->resv->fence); ++ if (list) { ++ unsigned int shared_count = list->shared_count, i; ++ ++ for (i = 0; i < shared_count; ++i) { ++ struct dma_fence *fence = ++ rcu_dereference(list->shared[i]); ++ ++ args->busy |= busy_check_reader(fence); ++ } ++ } ++ ++ if (args->busy && read_seqcount_retry(&obj->resv->seq, seq)) ++ goto retry; ++ ++ err = 0; ++out: ++ rcu_read_unlock(); ++ return err; ++} ++ ++int ++i915_gem_throttle_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file_priv) ++{ ++ return i915_gem_ring_throttle(dev, file_priv); ++} ++ ++int ++i915_gem_madvise_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file_priv) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct drm_i915_gem_madvise *args = data; ++ struct drm_i915_gem_object *obj; ++ int err; ++ ++ switch (args->madv) { ++ case I915_MADV_DONTNEED: ++ case I915_MADV_WILLNEED: ++ break; ++ default: ++ return -EINVAL; ++ } ++ ++ obj = i915_gem_object_lookup(file_priv, args->handle); ++ if (!obj) ++ return -ENOENT; ++ ++ err = mutex_lock_interruptible(&obj->mm.lock); ++ if (err) ++ goto out; ++ ++ if (i915_gem_object_has_pages(obj) && ++ i915_gem_object_is_tiled(obj) && ++ dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) { ++ if (obj->mm.madv == I915_MADV_WILLNEED) { ++ GEM_BUG_ON(!obj->mm.quirked); ++ __i915_gem_object_unpin_pages(obj); ++ obj->mm.quirked = false; ++ } ++ if (args->madv == I915_MADV_WILLNEED) { ++ GEM_BUG_ON(obj->mm.quirked); ++ __i915_gem_object_pin_pages(obj); ++ obj->mm.quirked = true; ++ } ++ } ++ ++ if (obj->mm.madv != __I915_MADV_PURGED) ++ obj->mm.madv = args->madv; ++ ++ /* if the object is no longer attached, discard its backing storage */ ++ if (obj->mm.madv == I915_MADV_DONTNEED && ++ !i915_gem_object_has_pages(obj)) ++ i915_gem_object_truncate(obj); ++ ++ args->retained = obj->mm.madv != __I915_MADV_PURGED; ++ mutex_unlock(&obj->mm.lock); ++ ++out: ++ i915_gem_object_put(obj); ++ return err; ++} ++ ++static void ++frontbuffer_retire(struct i915_active_request *active, ++ struct i915_request *request) ++{ ++ struct drm_i915_gem_object *obj = ++ container_of(active, typeof(*obj), frontbuffer_write); ++ ++ intel_fb_obj_flush(obj, ORIGIN_CS); ++} ++ ++void i915_gem_object_init(struct drm_i915_gem_object *obj, ++ const struct drm_i915_gem_object_ops *ops) ++{ ++ mutex_init(&obj->mm.lock); ++ ++ spin_lock_init(&obj->vma.lock); ++ INIT_LIST_HEAD(&obj->vma.list); ++ ++ INIT_LIST_HEAD(&obj->lut_list); ++ INIT_LIST_HEAD(&obj->batch_pool_link); ++ ++ init_rcu_head(&obj->rcu); ++ ++ obj->ops = ops; ++ ++ reservation_object_init(&obj->__builtin_resv); ++ obj->resv = &obj->__builtin_resv; ++ ++ obj->frontbuffer_ggtt_origin = ORIGIN_GTT; ++ i915_active_request_init(&obj->frontbuffer_write, ++ NULL, frontbuffer_retire); ++ ++ obj->mm.madv = I915_MADV_WILLNEED; ++ INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL | __GFP_NOWARN); ++ mutex_init(&obj->mm.get_page.lock); ++ ++ i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size); ++} ++ ++static const struct drm_i915_gem_object_ops i915_gem_object_ops = { ++ .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | ++ I915_GEM_OBJECT_IS_SHRINKABLE, ++ ++ .get_pages = i915_gem_object_get_pages_gtt, ++ .put_pages = i915_gem_object_put_pages_gtt, ++ ++ .pwrite = i915_gem_object_pwrite_gtt, ++}; ++ ++static int i915_gem_object_create_shmem(struct drm_device *dev, ++ struct drm_gem_object *obj, ++ size_t size) ++{ ++ struct drm_i915_private *i915 = to_i915(dev); ++ unsigned long flags = VM_NORESERVE; ++ struct file *filp; ++ ++ drm_gem_private_object_init(dev, obj, size); ++ ++ if (i915->mm.gemfs) ++ filp = shmem_file_setup_with_mnt(i915->mm.gemfs, "i915", size, ++ flags); ++ else ++ filp = shmem_file_setup("i915", size, flags); ++ ++ if (IS_ERR(filp)) ++ return PTR_ERR(filp); ++ ++ obj->filp = filp; ++ ++ return 0; ++} ++ ++struct drm_i915_gem_object * ++i915_gem_object_create(struct drm_i915_private *dev_priv, u64 size) ++{ ++ struct drm_i915_gem_object *obj; ++ struct address_space *mapping; ++ unsigned int cache_level; ++ gfp_t mask; ++ int ret; ++ ++ /* There is a prevalence of the assumption that we fit the object's ++ * page count inside a 32bit _signed_ variable. Let's document this and ++ * catch if we ever need to fix it. In the meantime, if you do spot ++ * such a local variable, please consider fixing! ++ */ ++ if (size >> PAGE_SHIFT > INT_MAX) ++ return ERR_PTR(-E2BIG); ++ ++ if (overflows_type(size, obj->base.size)) ++ return ERR_PTR(-E2BIG); ++ ++ obj = i915_gem_object_alloc(); ++ if (obj == NULL) ++ return ERR_PTR(-ENOMEM); ++ ++ ret = i915_gem_object_create_shmem(&dev_priv->drm, &obj->base, size); ++ if (ret) ++ goto fail; ++ ++ mask = GFP_HIGHUSER | __GFP_RECLAIMABLE; ++ if (IS_I965GM(dev_priv) || IS_I965G(dev_priv)) { ++ /* 965gm cannot relocate objects above 4GiB. */ ++ mask &= ~__GFP_HIGHMEM; ++ mask |= __GFP_DMA32; ++ } ++ ++ mapping = obj->base.filp->f_mapping; ++ mapping_set_gfp_mask(mapping, mask); ++ GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM)); ++ ++ i915_gem_object_init(obj, &i915_gem_object_ops); ++ ++ obj->write_domain = I915_GEM_DOMAIN_CPU; ++ obj->read_domains = I915_GEM_DOMAIN_CPU; ++ ++ if (HAS_LLC(dev_priv)) ++ /* On some devices, we can have the GPU use the LLC (the CPU ++ * cache) for about a 10% performance improvement ++ * compared to uncached. Graphics requests other than ++ * display scanout are coherent with the CPU in ++ * accessing this cache. This means in this mode we ++ * don't need to clflush on the CPU side, and on the ++ * GPU side we only need to flush internal caches to ++ * get data visible to the CPU. ++ * ++ * However, we maintain the display planes as UC, and so ++ * need to rebind when first used as such. ++ */ ++ cache_level = I915_CACHE_LLC; ++ else ++ cache_level = I915_CACHE_NONE; ++ ++ i915_gem_object_set_cache_coherency(obj, cache_level); ++ ++ trace_i915_gem_object_create(obj); ++ ++ return obj; ++ ++fail: ++ i915_gem_object_free(obj); ++ return ERR_PTR(ret); ++} ++ ++static bool discard_backing_storage(struct drm_i915_gem_object *obj) ++{ ++ /* If we are the last user of the backing storage (be it shmemfs ++ * pages or stolen etc), we know that the pages are going to be ++ * immediately released. In this case, we can then skip copying ++ * back the contents from the GPU. ++ */ ++ ++ if (obj->mm.madv != I915_MADV_WILLNEED) ++ return false; ++ ++ if (obj->base.filp == NULL) ++ return true; ++ ++ /* At first glance, this looks racy, but then again so would be ++ * userspace racing mmap against close. However, the first external ++ * reference to the filp can only be obtained through the ++ * i915_gem_mmap_ioctl() which safeguards us against the user ++ * acquiring such a reference whilst we are in the middle of ++ * freeing the object. ++ */ ++ return file_count(obj->base.filp) == 1; ++} ++ ++static void __i915_gem_free_objects(struct drm_i915_private *i915, ++ struct llist_node *freed) ++{ ++ struct drm_i915_gem_object *obj, *on; ++ intel_wakeref_t wakeref; ++ ++ wakeref = intel_runtime_pm_get(i915); ++ llist_for_each_entry_safe(obj, on, freed, freed) { ++ struct i915_vma *vma, *vn; ++ ++ trace_i915_gem_object_destroy(obj); ++ ++ mutex_lock(&i915->drm.struct_mutex); ++ ++ GEM_BUG_ON(i915_gem_object_is_active(obj)); ++ list_for_each_entry_safe(vma, vn, &obj->vma.list, obj_link) { ++ GEM_BUG_ON(i915_vma_is_active(vma)); ++ vma->flags &= ~I915_VMA_PIN_MASK; ++ i915_vma_destroy(vma); ++ } ++ GEM_BUG_ON(!list_empty(&obj->vma.list)); ++ GEM_BUG_ON(!RB_EMPTY_ROOT(&obj->vma.tree)); ++ ++ /* This serializes freeing with the shrinker. Since the free ++ * is delayed, first by RCU then by the workqueue, we want the ++ * shrinker to be able to free pages of unreferenced objects, ++ * or else we may oom whilst there are plenty of deferred ++ * freed objects. ++ */ ++ if (i915_gem_object_has_pages(obj)) { ++ spin_lock(&i915->mm.obj_lock); ++ list_del_init(&obj->mm.link); ++ spin_unlock(&i915->mm.obj_lock); ++ } ++ ++ mutex_unlock(&i915->drm.struct_mutex); ++ ++ GEM_BUG_ON(obj->bind_count); ++ GEM_BUG_ON(obj->userfault_count); ++ GEM_BUG_ON(atomic_read(&obj->frontbuffer_bits)); ++ GEM_BUG_ON(!list_empty(&obj->lut_list)); ++ ++ if (obj->ops->release) ++ obj->ops->release(obj); ++ ++ if (WARN_ON(i915_gem_object_has_pinned_pages(obj))) ++ atomic_set(&obj->mm.pages_pin_count, 0); ++ __i915_gem_object_put_pages(obj, I915_MM_NORMAL); ++ GEM_BUG_ON(i915_gem_object_has_pages(obj)); ++ ++ if (obj->base.import_attach) ++ drm_prime_gem_destroy(&obj->base, NULL); ++ ++ reservation_object_fini(&obj->__builtin_resv); ++ drm_gem_object_release(&obj->base); ++ i915_gem_info_remove_obj(i915, obj->base.size); ++ ++ bitmap_free(obj->bit_17); ++ i915_gem_object_free(obj); ++ ++ GEM_BUG_ON(!atomic_read(&i915->mm.free_count)); ++ atomic_dec(&i915->mm.free_count); ++ ++ if (on) ++ cond_resched(); ++ } ++ intel_runtime_pm_put(i915, wakeref); ++} ++ ++static void i915_gem_flush_free_objects(struct drm_i915_private *i915) ++{ ++ struct llist_node *freed; ++ ++ /* Free the oldest, most stale object to keep the free_list short */ ++ freed = NULL; ++ if (!llist_empty(&i915->mm.free_list)) { /* quick test for hotpath */ ++ /* Only one consumer of llist_del_first() allowed */ ++ spin_lock(&i915->mm.free_lock); ++ freed = llist_del_first(&i915->mm.free_list); ++ spin_unlock(&i915->mm.free_lock); ++ } ++ if (unlikely(freed)) { ++ freed->next = NULL; ++ __i915_gem_free_objects(i915, freed); ++ } ++} ++ ++static void __i915_gem_free_work(struct work_struct *work) ++{ ++ struct drm_i915_private *i915 = ++ container_of(work, struct drm_i915_private, mm.free_work); ++ struct llist_node *freed; ++ ++ /* ++ * All file-owned VMA should have been released by this point through ++ * i915_gem_close_object(), or earlier by i915_gem_context_close(). ++ * However, the object may also be bound into the global GTT (e.g. ++ * older GPUs without per-process support, or for direct access through ++ * the GTT either for the user or for scanout). Those VMA still need to ++ * unbound now. ++ */ ++ ++ spin_lock(&i915->mm.free_lock); ++ while ((freed = llist_del_all(&i915->mm.free_list))) { ++ spin_unlock(&i915->mm.free_lock); ++ ++ __i915_gem_free_objects(i915, freed); ++ if (need_resched()) ++ return; ++ ++ spin_lock(&i915->mm.free_lock); ++ } ++ spin_unlock(&i915->mm.free_lock); ++} ++ ++static void __i915_gem_free_object_rcu(struct rcu_head *head) ++{ ++ struct drm_i915_gem_object *obj = ++ container_of(head, typeof(*obj), rcu); ++ struct drm_i915_private *i915 = to_i915(obj->base.dev); ++ ++ /* ++ * We reuse obj->rcu for the freed list, so we had better not treat ++ * it like a rcu_head from this point forwards. And we expect all ++ * objects to be freed via this path. ++ */ ++ destroy_rcu_head(&obj->rcu); ++ ++ /* ++ * Since we require blocking on struct_mutex to unbind the freed ++ * object from the GPU before releasing resources back to the ++ * system, we can not do that directly from the RCU callback (which may ++ * be a softirq context), but must instead then defer that work onto a ++ * kthread. We use the RCU callback rather than move the freed object ++ * directly onto the work queue so that we can mix between using the ++ * worker and performing frees directly from subsequent allocations for ++ * crude but effective memory throttling. ++ */ ++ if (llist_add(&obj->freed, &i915->mm.free_list)) ++ queue_work(i915->wq, &i915->mm.free_work); ++} ++ ++void i915_gem_free_object(struct drm_gem_object *gem_obj) ++{ ++ struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); ++ ++ if (obj->mm.quirked) ++ __i915_gem_object_unpin_pages(obj); ++ ++ if (discard_backing_storage(obj)) ++ obj->mm.madv = I915_MADV_DONTNEED; ++ ++ /* ++ * Before we free the object, make sure any pure RCU-only ++ * read-side critical sections are complete, e.g. ++ * i915_gem_busy_ioctl(). For the corresponding synchronized ++ * lookup see i915_gem_object_lookup_rcu(). ++ */ ++ atomic_inc(&to_i915(obj->base.dev)->mm.free_count); ++ call_rcu(&obj->rcu, __i915_gem_free_object_rcu); ++} ++ ++void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj) ++{ ++ lockdep_assert_held(&obj->base.dev->struct_mutex); ++ ++ if (!i915_gem_object_has_active_reference(obj) && ++ i915_gem_object_is_active(obj)) ++ i915_gem_object_set_active_reference(obj); ++ else ++ i915_gem_object_put(obj); ++} ++ ++void i915_gem_sanitize(struct drm_i915_private *i915) ++{ ++ intel_wakeref_t wakeref; ++ ++ GEM_TRACE("\n"); ++ ++ wakeref = intel_runtime_pm_get(i915); ++ intel_uncore_forcewake_get(&i915->uncore, FORCEWAKE_ALL); ++ ++ /* ++ * As we have just resumed the machine and woken the device up from ++ * deep PCI sleep (presumably D3_cold), assume the HW has been reset ++ * back to defaults, recovering from whatever wedged state we left it ++ * in and so worth trying to use the device once more. ++ */ ++ if (i915_terminally_wedged(i915)) ++ i915_gem_unset_wedged(i915); ++ ++ /* ++ * If we inherit context state from the BIOS or earlier occupants ++ * of the GPU, the GPU may be in an inconsistent state when we ++ * try to take over. The only way to remove the earlier state ++ * is by resetting. However, resetting on earlier gen is tricky as ++ * it may impact the display and we are uncertain about the stability ++ * of the reset, so this could be applied to even earlier gen. ++ */ ++ intel_engines_sanitize(i915, false); ++ ++ intel_uncore_forcewake_put(&i915->uncore, FORCEWAKE_ALL); ++ intel_runtime_pm_put(i915, wakeref); ++ ++ mutex_lock(&i915->drm.struct_mutex); ++ i915_gem_contexts_lost(i915); ++ mutex_unlock(&i915->drm.struct_mutex); ++} ++ ++void i915_gem_suspend(struct drm_i915_private *i915) ++{ ++ intel_wakeref_t wakeref; ++ ++ GEM_TRACE("\n"); ++ ++ wakeref = intel_runtime_pm_get(i915); ++ ++ flush_workqueue(i915->wq); ++ ++ mutex_lock(&i915->drm.struct_mutex); ++ ++ /* ++ * We have to flush all the executing contexts to main memory so ++ * that they can saved in the hibernation image. To ensure the last ++ * context image is coherent, we have to switch away from it. That ++ * leaves the i915->kernel_context still active when ++ * we actually suspend, and its image in memory may not match the GPU ++ * state. Fortunately, the kernel_context is disposable and we do ++ * not rely on its state. ++ */ ++ switch_to_kernel_context_sync(i915, i915->gt.active_engines); ++ ++ mutex_unlock(&i915->drm.struct_mutex); ++ i915_reset_flush(i915); ++ ++ drain_delayed_work(&i915->gt.retire_work); ++ ++ /* ++ * As the idle_work is rearming if it detects a race, play safe and ++ * repeat the flush until it is definitely idle. ++ */ ++ drain_delayed_work(&i915->gt.idle_work); ++ ++ /* ++ * Assert that we successfully flushed all the work and ++ * reset the GPU back to its idle, low power state. ++ */ ++ GEM_BUG_ON(i915->gt.awake); ++ ++ intel_uc_suspend(i915); ++ ++ intel_runtime_pm_put(i915, wakeref); ++} ++ ++void i915_gem_suspend_late(struct drm_i915_private *i915) ++{ ++ struct drm_i915_gem_object *obj; ++ struct list_head *phases[] = { ++ &i915->mm.unbound_list, ++ &i915->mm.bound_list, ++ NULL ++ }, **phase; ++ ++ /* ++ * Neither the BIOS, ourselves or any other kernel ++ * expects the system to be in execlists mode on startup, ++ * so we need to reset the GPU back to legacy mode. And the only ++ * known way to disable logical contexts is through a GPU reset. ++ * ++ * So in order to leave the system in a known default configuration, ++ * always reset the GPU upon unload and suspend. Afterwards we then ++ * clean up the GEM state tracking, flushing off the requests and ++ * leaving the system in a known idle state. ++ * ++ * Note that is of the upmost importance that the GPU is idle and ++ * all stray writes are flushed *before* we dismantle the backing ++ * storage for the pinned objects. ++ * ++ * However, since we are uncertain that resetting the GPU on older ++ * machines is a good idea, we don't - just in case it leaves the ++ * machine in an unusable condition. ++ */ ++ ++ mutex_lock(&i915->drm.struct_mutex); ++ for (phase = phases; *phase; phase++) { ++ list_for_each_entry(obj, *phase, mm.link) ++ WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false)); ++ } ++ mutex_unlock(&i915->drm.struct_mutex); ++ ++ intel_uc_sanitize(i915); ++ i915_gem_sanitize(i915); ++} ++ ++void i915_gem_resume(struct drm_i915_private *i915) ++{ ++ GEM_TRACE("\n"); ++ ++ WARN_ON(i915->gt.awake); ++ ++ mutex_lock(&i915->drm.struct_mutex); ++ intel_uncore_forcewake_get(&i915->uncore, FORCEWAKE_ALL); ++ ++ i915_gem_restore_gtt_mappings(i915); ++ i915_gem_restore_fences(i915); ++ ++ /* ++ * As we didn't flush the kernel context before suspend, we cannot ++ * guarantee that the context image is complete. So let's just reset ++ * it and start again. ++ */ ++ intel_gt_resume(i915); ++ ++ if (i915_gem_init_hw(i915)) ++ goto err_wedged; ++ ++ intel_uc_resume(i915); ++ ++ /* Always reload a context for powersaving. */ ++ if (!load_power_context(i915)) ++ goto err_wedged; ++ ++out_unlock: ++ intel_uncore_forcewake_put(&i915->uncore, FORCEWAKE_ALL); ++ mutex_unlock(&i915->drm.struct_mutex); ++ return; ++ ++err_wedged: ++ if (!i915_reset_failed(i915)) { ++ dev_err(i915->drm.dev, ++ "Failed to re-initialize GPU, declaring it wedged!\n"); ++ i915_gem_set_wedged(i915); ++ } ++ goto out_unlock; ++} ++ ++void i915_gem_init_swizzling(struct drm_i915_private *dev_priv) ++{ ++ if (INTEL_GEN(dev_priv) < 5 || ++ dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) ++ return; ++ ++ I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | ++ DISP_TILE_SURFACE_SWIZZLING); ++ ++ if (IS_GEN(dev_priv, 5)) ++ return; ++ ++ I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); ++ if (IS_GEN(dev_priv, 6)) ++ I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB)); ++ else if (IS_GEN(dev_priv, 7)) ++ I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB)); ++ else if (IS_GEN(dev_priv, 8)) ++ I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW)); ++ else ++ BUG(); ++} ++ ++static void init_unused_ring(struct drm_i915_private *dev_priv, u32 base) ++{ ++ I915_WRITE(RING_CTL(base), 0); ++ I915_WRITE(RING_HEAD(base), 0); ++ I915_WRITE(RING_TAIL(base), 0); ++ I915_WRITE(RING_START(base), 0); ++} ++ ++static void init_unused_rings(struct drm_i915_private *dev_priv) ++{ ++ if (IS_I830(dev_priv)) { ++ init_unused_ring(dev_priv, PRB1_BASE); ++ init_unused_ring(dev_priv, SRB0_BASE); ++ init_unused_ring(dev_priv, SRB1_BASE); ++ init_unused_ring(dev_priv, SRB2_BASE); ++ init_unused_ring(dev_priv, SRB3_BASE); ++ } else if (IS_GEN(dev_priv, 2)) { ++ init_unused_ring(dev_priv, SRB0_BASE); ++ init_unused_ring(dev_priv, SRB1_BASE); ++ } else if (IS_GEN(dev_priv, 3)) { ++ init_unused_ring(dev_priv, PRB1_BASE); ++ init_unused_ring(dev_priv, PRB2_BASE); ++ } ++} ++ ++static int __i915_gem_restart_engines(void *data) ++{ ++ struct drm_i915_private *i915 = data; ++ struct intel_engine_cs *engine; ++ enum intel_engine_id id; ++ int err; ++ ++ for_each_engine(engine, i915, id) { ++ err = engine->init_hw(engine); ++ if (err) { ++ DRM_ERROR("Failed to restart %s (%d)\n", ++ engine->name, err); ++ return err; ++ } ++ } ++ ++ intel_engines_set_scheduler_caps(i915); ++ ++ return 0; ++} ++ ++int i915_gem_init_hw(struct drm_i915_private *dev_priv) ++{ ++ int ret; ++ ++ dev_priv->gt.last_init_time = ktime_get(); ++ ++ /* Double layer security blanket, see i915_gem_init() */ ++ intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL); ++ ++ if (HAS_EDRAM(dev_priv) && INTEL_GEN(dev_priv) < 9) ++ I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf)); ++ ++ if (IS_HASWELL(dev_priv)) ++ I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev_priv) ? ++ LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED); ++ ++ /* Apply the GT workarounds... */ ++ intel_gt_apply_workarounds(dev_priv); ++ /* ...and determine whether they are sticking. */ ++ intel_gt_verify_workarounds(dev_priv, "init"); ++ ++ i915_gem_init_swizzling(dev_priv); ++ ++ /* ++ * At least 830 can leave some of the unused rings ++ * "active" (ie. head != tail) after resume which ++ * will prevent c3 entry. Makes sure all unused rings ++ * are totally idle. ++ */ ++ init_unused_rings(dev_priv); ++ ++ BUG_ON(!dev_priv->kernel_context); ++ ret = i915_terminally_wedged(dev_priv); ++ if (ret) ++ goto out; ++ ++ ret = i915_ppgtt_init_hw(dev_priv); ++ if (ret) { ++ DRM_ERROR("Enabling PPGTT failed (%d)\n", ret); ++ goto out; ++ } ++ ++ ret = intel_wopcm_init_hw(&dev_priv->wopcm); ++ if (ret) { ++ DRM_ERROR("Enabling WOPCM failed (%d)\n", ret); ++ goto out; ++ } ++ ++ /* We can't enable contexts until all firmware is loaded */ ++ ret = intel_uc_init_hw(dev_priv); ++ if (ret) { ++ DRM_ERROR("Enabling uc failed (%d)\n", ret); ++ goto out; ++ } ++ ++ intel_mocs_init_l3cc_table(dev_priv); ++ ++ /* Only when the HW is re-initialised, can we replay the requests */ ++ ret = __i915_gem_restart_engines(dev_priv); ++ if (ret) ++ goto cleanup_uc; ++ ++ intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); ++ ++ return 0; ++ ++cleanup_uc: ++ intel_uc_fini_hw(dev_priv); ++out: ++ intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); ++ ++ return ret; ++} ++ ++static int __intel_engines_record_defaults(struct drm_i915_private *i915) ++{ ++ struct i915_gem_context *ctx; ++ struct intel_engine_cs *engine; ++ enum intel_engine_id id; ++ int err = 0; ++ ++ /* ++ * As we reset the gpu during very early sanitisation, the current ++ * register state on the GPU should reflect its defaults values. ++ * We load a context onto the hw (with restore-inhibit), then switch ++ * over to a second context to save that default register state. We ++ * can then prime every new context with that state so they all start ++ * from the same default HW values. ++ */ ++ ++ ctx = i915_gem_context_create_kernel(i915, 0); ++ if (IS_ERR(ctx)) ++ return PTR_ERR(ctx); ++ ++ for_each_engine(engine, i915, id) { ++ struct i915_request *rq; ++ ++ rq = i915_request_alloc(engine, ctx); ++ if (IS_ERR(rq)) { ++ err = PTR_ERR(rq); ++ goto out_ctx; ++ } ++ ++ err = 0; ++ if (engine->init_context) ++ err = engine->init_context(rq); ++ ++ i915_request_add(rq); ++ if (err) ++ goto err_active; ++ } ++ ++ /* Flush the default context image to memory, and enable powersaving. */ ++ if (!load_power_context(i915)) { ++ err = -EIO; ++ goto err_active; ++ } ++ ++ for_each_engine(engine, i915, id) { ++ struct intel_context *ce; ++ struct i915_vma *state; ++ void *vaddr; ++ ++ ce = intel_context_lookup(ctx, engine); ++ if (!ce) ++ continue; ++ ++ state = ce->state; ++ if (!state) ++ continue; ++ ++ GEM_BUG_ON(intel_context_is_pinned(ce)); ++ ++ /* ++ * As we will hold a reference to the logical state, it will ++ * not be torn down with the context, and importantly the ++ * object will hold onto its vma (making it possible for a ++ * stray GTT write to corrupt our defaults). Unmap the vma ++ * from the GTT to prevent such accidents and reclaim the ++ * space. ++ */ ++ err = i915_vma_unbind(state); ++ if (err) ++ goto err_active; ++ ++ err = i915_gem_object_set_to_cpu_domain(state->obj, false); ++ if (err) ++ goto err_active; ++ ++ engine->default_state = i915_gem_object_get(state->obj); ++ i915_gem_object_set_cache_coherency(engine->default_state, ++ I915_CACHE_LLC); ++ ++ /* Check we can acquire the image of the context state */ ++ vaddr = i915_gem_object_pin_map(engine->default_state, ++ I915_MAP_FORCE_WB); ++ if (IS_ERR(vaddr)) { ++ err = PTR_ERR(vaddr); ++ goto err_active; ++ } ++ ++ i915_gem_object_unpin_map(engine->default_state); ++ } ++ ++ if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) { ++ unsigned int found = intel_engines_has_context_isolation(i915); ++ ++ /* ++ * Make sure that classes with multiple engine instances all ++ * share the same basic configuration. ++ */ ++ for_each_engine(engine, i915, id) { ++ unsigned int bit = BIT(engine->uabi_class); ++ unsigned int expected = engine->default_state ? bit : 0; ++ ++ if ((found & bit) != expected) { ++ DRM_ERROR("mismatching default context state for class %d on engine %s\n", ++ engine->uabi_class, engine->name); ++ } ++ } ++ } ++ ++out_ctx: ++ i915_gem_context_set_closed(ctx); ++ i915_gem_context_put(ctx); ++ return err; ++ ++err_active: ++ /* ++ * If we have to abandon now, we expect the engines to be idle ++ * and ready to be torn-down. The quickest way we can accomplish ++ * this is by declaring ourselves wedged. ++ */ ++ i915_gem_set_wedged(i915); ++ goto out_ctx; ++} ++ ++static int ++i915_gem_init_scratch(struct drm_i915_private *i915, unsigned int size) ++{ ++ struct drm_i915_gem_object *obj; ++ struct i915_vma *vma; ++ int ret; ++ ++ obj = i915_gem_object_create_stolen(i915, size); ++ if (!obj) ++ obj = i915_gem_object_create_internal(i915, size); ++ if (IS_ERR(obj)) { ++ DRM_ERROR("Failed to allocate scratch page\n"); ++ return PTR_ERR(obj); ++ } ++ ++ vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); ++ if (IS_ERR(vma)) { ++ ret = PTR_ERR(vma); ++ goto err_unref; ++ } ++ ++ ret = i915_vma_pin(vma, 0, 0, PIN_GLOBAL | PIN_HIGH); ++ if (ret) ++ goto err_unref; ++ ++ i915->gt.scratch = vma; ++ return 0; ++ ++err_unref: ++ i915_gem_object_put(obj); ++ return ret; ++} ++ ++static void i915_gem_fini_scratch(struct drm_i915_private *i915) ++{ ++ i915_vma_unpin_and_release(&i915->gt.scratch, 0); ++} ++ ++int i915_gem_init(struct drm_i915_private *dev_priv) ++{ ++ int ret; ++ ++ /* We need to fallback to 4K pages if host doesn't support huge gtt. */ ++ if (intel_vgpu_active(dev_priv) && !intel_vgpu_has_huge_gtt(dev_priv)) ++ mkwrite_device_info(dev_priv)->page_sizes = ++ I915_GTT_PAGE_SIZE_4K; ++ ++ dev_priv->mm.unordered_timeline = dma_fence_context_alloc(1); ++ ++ if (HAS_LOGICAL_RING_CONTEXTS(dev_priv)) ++ dev_priv->gt.cleanup_engine = intel_logical_ring_cleanup; ++ else ++ dev_priv->gt.cleanup_engine = intel_engine_cleanup; ++ ++ i915_timelines_init(dev_priv); ++ ++ ret = i915_gem_init_userptr(dev_priv); ++ if (ret) ++ return ret; ++ ++ ret = intel_uc_init_misc(dev_priv); ++ if (ret) ++ return ret; ++ ++ ret = intel_wopcm_init(&dev_priv->wopcm); ++ if (ret) ++ goto err_uc_misc; ++ ++ /* This is just a security blanket to placate dragons. ++ * On some systems, we very sporadically observe that the first TLBs ++ * used by the CS may be stale, despite us poking the TLB reset. If ++ * we hold the forcewake during initialisation these problems ++ * just magically go away. ++ */ ++ mutex_lock(&dev_priv->drm.struct_mutex); ++ intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL); ++ ++ ret = i915_gem_init_ggtt(dev_priv); ++ if (ret) { ++ GEM_BUG_ON(ret == -EIO); ++ goto err_unlock; ++ } ++ ++ ret = i915_gem_init_scratch(dev_priv, ++ IS_GEN(dev_priv, 2) ? SZ_256K : PAGE_SIZE); ++ if (ret) { ++ GEM_BUG_ON(ret == -EIO); ++ goto err_ggtt; ++ } ++ ++ ret = i915_gem_contexts_init(dev_priv); ++ if (ret) { ++ GEM_BUG_ON(ret == -EIO); ++ goto err_scratch; ++ } ++ ++ ret = intel_engines_init(dev_priv); ++ if (ret) { ++ GEM_BUG_ON(ret == -EIO); ++ goto err_context; ++ } ++ ++ intel_init_gt_powersave(dev_priv); ++ ++ ret = intel_uc_init(dev_priv); ++ if (ret) ++ goto err_pm; ++ ++ ret = i915_gem_init_hw(dev_priv); ++ if (ret) ++ goto err_uc_init; ++ ++ /* ++ * Despite its name intel_init_clock_gating applies both display ++ * clock gating workarounds; GT mmio workarounds and the occasional ++ * GT power context workaround. Worse, sometimes it includes a context ++ * register workaround which we need to apply before we record the ++ * default HW state for all contexts. ++ * ++ * FIXME: break up the workarounds and apply them at the right time! ++ */ ++ intel_init_clock_gating(dev_priv); ++ ++ ret = __intel_engines_record_defaults(dev_priv); ++ if (ret) ++ goto err_init_hw; ++ ++ if (i915_inject_load_failure()) { ++ ret = -ENODEV; ++ goto err_init_hw; ++ } ++ ++ if (i915_inject_load_failure()) { ++ ret = -EIO; ++ goto err_init_hw; ++ } ++ ++ intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++ ++ return 0; ++ ++ /* ++ * Unwinding is complicated by that we want to handle -EIO to mean ++ * disable GPU submission but keep KMS alive. We want to mark the ++ * HW as irrevisibly wedged, but keep enough state around that the ++ * driver doesn't explode during runtime. ++ */ ++err_init_hw: ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++ ++ i915_gem_suspend(dev_priv); ++ i915_gem_suspend_late(dev_priv); ++ ++ i915_gem_drain_workqueue(dev_priv); ++ ++ mutex_lock(&dev_priv->drm.struct_mutex); ++ intel_uc_fini_hw(dev_priv); ++err_uc_init: ++ intel_uc_fini(dev_priv); ++err_pm: ++ if (ret != -EIO) { ++ intel_cleanup_gt_powersave(dev_priv); ++ i915_gem_cleanup_engines(dev_priv); ++ } ++err_context: ++ if (ret != -EIO) ++ i915_gem_contexts_fini(dev_priv); ++err_scratch: ++ i915_gem_fini_scratch(dev_priv); ++err_ggtt: ++err_unlock: ++ intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++ ++err_uc_misc: ++ intel_uc_fini_misc(dev_priv); ++ ++ if (ret != -EIO) { ++ i915_gem_cleanup_userptr(dev_priv); ++ i915_timelines_fini(dev_priv); ++ } ++ ++ if (ret == -EIO) { ++ mutex_lock(&dev_priv->drm.struct_mutex); ++ ++ /* ++ * Allow engine initialisation to fail by marking the GPU as ++ * wedged. But we only want to do this where the GPU is angry, ++ * for all other failure, such as an allocation failure, bail. ++ */ ++ if (!i915_reset_failed(dev_priv)) { ++ i915_load_error(dev_priv, ++ "Failed to initialize GPU, declaring it wedged!\n"); ++ i915_gem_set_wedged(dev_priv); ++ } ++ ++ /* Minimal basic recovery for KMS */ ++ ret = i915_ggtt_enable_hw(dev_priv); ++ i915_gem_restore_gtt_mappings(dev_priv); ++ i915_gem_restore_fences(dev_priv); ++ intel_init_clock_gating(dev_priv); ++ ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++ } ++ ++ i915_gem_drain_freed_objects(dev_priv); ++ return ret; ++} ++ ++void i915_gem_fini(struct drm_i915_private *dev_priv) ++{ ++ i915_gem_suspend_late(dev_priv); ++ intel_disable_gt_powersave(dev_priv); ++ ++ /* Flush any outstanding unpin_work. */ ++ i915_gem_drain_workqueue(dev_priv); ++ ++ mutex_lock(&dev_priv->drm.struct_mutex); ++ intel_uc_fini_hw(dev_priv); ++ intel_uc_fini(dev_priv); ++ i915_gem_cleanup_engines(dev_priv); ++ i915_gem_contexts_fini(dev_priv); ++ i915_gem_fini_scratch(dev_priv); ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++ ++ intel_wa_list_free(&dev_priv->gt_wa_list); ++ ++ intel_cleanup_gt_powersave(dev_priv); ++ ++ intel_uc_fini_misc(dev_priv); ++ i915_gem_cleanup_userptr(dev_priv); ++ i915_timelines_fini(dev_priv); ++ ++ i915_gem_drain_freed_objects(dev_priv); ++ ++ WARN_ON(!list_empty(&dev_priv->contexts.list)); ++} ++ ++void i915_gem_init_mmio(struct drm_i915_private *i915) ++{ ++ i915_gem_sanitize(i915); ++} ++ ++void ++i915_gem_cleanup_engines(struct drm_i915_private *dev_priv) ++{ ++ struct intel_engine_cs *engine; ++ enum intel_engine_id id; ++ ++ for_each_engine(engine, dev_priv, id) ++ dev_priv->gt.cleanup_engine(engine); ++} ++ ++void ++i915_gem_load_init_fences(struct drm_i915_private *dev_priv) ++{ ++ int i; ++ ++ if (INTEL_GEN(dev_priv) >= 7 && !IS_VALLEYVIEW(dev_priv) && ++ !IS_CHERRYVIEW(dev_priv)) ++ dev_priv->num_fence_regs = 32; ++ else if (INTEL_GEN(dev_priv) >= 4 || ++ IS_I945G(dev_priv) || IS_I945GM(dev_priv) || ++ IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) ++ dev_priv->num_fence_regs = 16; ++ else ++ dev_priv->num_fence_regs = 8; ++ ++ if (intel_vgpu_active(dev_priv)) ++ dev_priv->num_fence_regs = ++ I915_READ(vgtif_reg(avail_rs.fence_num)); ++ ++ /* Initialize fence registers to zero */ ++ for (i = 0; i < dev_priv->num_fence_regs; i++) { ++ struct drm_i915_fence_reg *fence = &dev_priv->fence_regs[i]; ++ ++ fence->i915 = dev_priv; ++ fence->id = i; ++ list_add_tail(&fence->link, &dev_priv->mm.fence_list); ++ } ++ i915_gem_restore_fences(dev_priv); ++ ++ i915_gem_detect_bit_6_swizzle(dev_priv); ++} ++ ++static void i915_gem_init__mm(struct drm_i915_private *i915) ++{ ++ spin_lock_init(&i915->mm.object_stat_lock); ++ spin_lock_init(&i915->mm.obj_lock); ++ spin_lock_init(&i915->mm.free_lock); ++ ++ init_llist_head(&i915->mm.free_list); ++ ++ INIT_LIST_HEAD(&i915->mm.unbound_list); ++ INIT_LIST_HEAD(&i915->mm.bound_list); ++ INIT_LIST_HEAD(&i915->mm.fence_list); ++ INIT_LIST_HEAD(&i915->mm.userfault_list); ++ ++ INIT_WORK(&i915->mm.free_work, __i915_gem_free_work); ++} ++ ++int i915_gem_init_early(struct drm_i915_private *dev_priv) ++{ ++ int err; ++ ++ INIT_LIST_HEAD(&dev_priv->gt.active_rings); ++ INIT_LIST_HEAD(&dev_priv->gt.closed_vma); ++ ++ i915_gem_init__mm(dev_priv); ++ ++ INIT_DELAYED_WORK(&dev_priv->gt.retire_work, ++ i915_gem_retire_work_handler); ++ INIT_DELAYED_WORK(&dev_priv->gt.idle_work, ++ i915_gem_idle_work_handler); ++ init_waitqueue_head(&dev_priv->gpu_error.wait_queue); ++ init_waitqueue_head(&dev_priv->gpu_error.reset_queue); ++ mutex_init(&dev_priv->gpu_error.wedge_mutex); ++ init_srcu_struct(&dev_priv->gpu_error.reset_backoff_srcu); ++ ++ atomic_set(&dev_priv->mm.bsd_engine_dispatch_index, 0); ++ ++ spin_lock_init(&dev_priv->fb_tracking.lock); ++ ++ err = i915_gemfs_init(dev_priv); ++ if (err) ++ DRM_NOTE("Unable to create a private tmpfs mount, hugepage support will be disabled(%d).\n", err); ++ ++ return 0; ++} ++ ++void i915_gem_cleanup_early(struct drm_i915_private *dev_priv) ++{ ++ i915_gem_drain_freed_objects(dev_priv); ++ GEM_BUG_ON(!llist_empty(&dev_priv->mm.free_list)); ++ GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count)); ++ WARN_ON(dev_priv->mm.object_count); ++ ++ cleanup_srcu_struct(&dev_priv->gpu_error.reset_backoff_srcu); ++ ++ i915_gemfs_fini(dev_priv); ++} ++ ++int i915_gem_freeze(struct drm_i915_private *dev_priv) ++{ ++ /* Discard all purgeable objects, let userspace recover those as ++ * required after resuming. ++ */ ++ i915_gem_shrink_all(dev_priv); ++ ++ return 0; ++} ++ ++int i915_gem_freeze_late(struct drm_i915_private *i915) ++{ ++ struct drm_i915_gem_object *obj; ++ struct list_head *phases[] = { ++ &i915->mm.unbound_list, ++ &i915->mm.bound_list, ++ NULL ++ }, **phase; ++ ++ /* ++ * Called just before we write the hibernation image. ++ * ++ * We need to update the domain tracking to reflect that the CPU ++ * will be accessing all the pages to create and restore from the ++ * hibernation, and so upon restoration those pages will be in the ++ * CPU domain. ++ * ++ * To make sure the hibernation image contains the latest state, ++ * we update that state just before writing out the image. ++ * ++ * To try and reduce the hibernation image, we manually shrink ++ * the objects as well, see i915_gem_freeze() ++ */ ++ ++ i915_gem_shrink(i915, -1UL, NULL, I915_SHRINK_UNBOUND); ++ i915_gem_drain_freed_objects(i915); ++ ++ mutex_lock(&i915->drm.struct_mutex); ++ for (phase = phases; *phase; phase++) { ++ list_for_each_entry(obj, *phase, mm.link) ++ WARN_ON(i915_gem_object_set_to_cpu_domain(obj, true)); ++ } ++ mutex_unlock(&i915->drm.struct_mutex); ++ ++ return 0; ++} ++ ++void i915_gem_release(struct drm_device *dev, struct drm_file *file) ++{ ++ struct drm_i915_file_private *file_priv = file->driver_priv; ++ struct i915_request *request; ++ ++ /* Clean up our request list when the client is going away, so that ++ * later retire_requests won't dereference our soon-to-be-gone ++ * file_priv. ++ */ ++ spin_lock(&file_priv->mm.lock); ++ list_for_each_entry(request, &file_priv->mm.request_list, client_link) ++ request->file_priv = NULL; ++ spin_unlock(&file_priv->mm.lock); ++} ++ ++int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file) ++{ ++ struct drm_i915_file_private *file_priv; ++ int ret; ++ ++ DRM_DEBUG("\n"); ++ ++ file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL); ++ if (!file_priv) ++ return -ENOMEM; ++ ++ file->driver_priv = file_priv; ++ file_priv->dev_priv = i915; ++ file_priv->file = file; ++ ++ spin_lock_init(&file_priv->mm.lock); ++ INIT_LIST_HEAD(&file_priv->mm.request_list); ++ ++ file_priv->bsd_engine = -1; ++ file_priv->hang_timestamp = jiffies; ++ ++ ret = i915_gem_context_open(i915, file); ++ if (ret) ++ kfree(file_priv); ++ ++ return ret; ++} ++ ++/** ++ * i915_gem_track_fb - update frontbuffer tracking ++ * @old: current GEM buffer for the frontbuffer slots ++ * @new: new GEM buffer for the frontbuffer slots ++ * @frontbuffer_bits: bitmask of frontbuffer slots ++ * ++ * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them ++ * from @old and setting them in @new. Both @old and @new can be NULL. ++ */ ++void i915_gem_track_fb(struct drm_i915_gem_object *old, ++ struct drm_i915_gem_object *new, ++ unsigned frontbuffer_bits) ++{ ++ /* Control of individual bits within the mask are guarded by ++ * the owning plane->mutex, i.e. we can never see concurrent ++ * manipulation of individual bits. But since the bitfield as a whole ++ * is updated using RMW, we need to use atomics in order to update ++ * the bits. ++ */ ++ BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES > ++ BITS_PER_TYPE(atomic_t)); ++ ++ if (old) { ++ WARN_ON(!(atomic_read(&old->frontbuffer_bits) & frontbuffer_bits)); ++ atomic_andnot(frontbuffer_bits, &old->frontbuffer_bits); ++ } ++ ++ if (new) { ++ WARN_ON(atomic_read(&new->frontbuffer_bits) & frontbuffer_bits); ++ atomic_or(frontbuffer_bits, &new->frontbuffer_bits); ++ } ++} ++ ++/* Allocate a new GEM object and fill it with the supplied data */ ++struct drm_i915_gem_object * ++i915_gem_object_create_from_data(struct drm_i915_private *dev_priv, ++ const void *data, size_t size) ++{ ++ struct drm_i915_gem_object *obj; ++ struct file *file; ++ size_t offset; ++ int err; ++ ++ obj = i915_gem_object_create(dev_priv, round_up(size, PAGE_SIZE)); ++ if (IS_ERR(obj)) ++ return obj; ++ ++ GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU); ++ ++ file = obj->base.filp; ++ offset = 0; ++ do { ++ unsigned int len = min_t(typeof(size), size, PAGE_SIZE); ++ struct page *page; ++ void *pgdata, *vaddr; ++ ++ err = pagecache_write_begin(file, file->f_mapping, ++ offset, len, 0, ++ &page, &pgdata); ++ if (err < 0) ++ goto fail; ++ ++ vaddr = kmap(page); ++ memcpy(vaddr, data, len); ++ kunmap(page); ++ ++ err = pagecache_write_end(file, file->f_mapping, ++ offset, len, len, ++ page, pgdata); ++ if (err < 0) ++ goto fail; ++ ++ size -= len; ++ data += len; ++ offset += len; ++ } while (size); ++ ++ return obj; ++ ++fail: ++ i915_gem_object_put(obj); ++ return ERR_PTR(err); ++} ++ ++struct scatterlist * ++i915_gem_object_get_sg(struct drm_i915_gem_object *obj, ++ unsigned int n, ++ unsigned int *offset) ++{ ++ struct i915_gem_object_page_iter *iter = &obj->mm.get_page; ++ struct scatterlist *sg; ++ unsigned int idx, count; ++ ++ might_sleep(); ++ GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT); ++ GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); ++ ++ /* As we iterate forward through the sg, we record each entry in a ++ * radixtree for quick repeated (backwards) lookups. If we have seen ++ * this index previously, we will have an entry for it. ++ * ++ * Initial lookup is O(N), but this is amortized to O(1) for ++ * sequential page access (where each new request is consecutive ++ * to the previous one). Repeated lookups are O(lg(obj->base.size)), ++ * i.e. O(1) with a large constant! ++ */ ++ if (n < READ_ONCE(iter->sg_idx)) ++ goto lookup; ++ ++ mutex_lock(&iter->lock); ++ ++ /* We prefer to reuse the last sg so that repeated lookup of this ++ * (or the subsequent) sg are fast - comparing against the last ++ * sg is faster than going through the radixtree. ++ */ ++ ++ sg = iter->sg_pos; ++ idx = iter->sg_idx; ++ count = __sg_page_count(sg); ++ ++ while (idx + count <= n) { ++ void *entry; ++ unsigned long i; ++ int ret; ++ ++ /* If we cannot allocate and insert this entry, or the ++ * individual pages from this range, cancel updating the ++ * sg_idx so that on this lookup we are forced to linearly ++ * scan onwards, but on future lookups we will try the ++ * insertion again (in which case we need to be careful of ++ * the error return reporting that we have already inserted ++ * this index). ++ */ ++ ret = radix_tree_insert(&iter->radix, idx, sg); ++ if (ret && ret != -EEXIST) ++ goto scan; ++ ++ entry = xa_mk_value(idx); ++ for (i = 1; i < count; i++) { ++ ret = radix_tree_insert(&iter->radix, idx + i, entry); ++ if (ret && ret != -EEXIST) ++ goto scan; ++ } ++ ++ idx += count; ++ sg = ____sg_next(sg); ++ count = __sg_page_count(sg); ++ } ++ ++scan: ++ iter->sg_pos = sg; ++ iter->sg_idx = idx; ++ ++ mutex_unlock(&iter->lock); ++ ++ if (unlikely(n < idx)) /* insertion completed by another thread */ ++ goto lookup; ++ ++ /* In case we failed to insert the entry into the radixtree, we need ++ * to look beyond the current sg. ++ */ ++ while (idx + count <= n) { ++ idx += count; ++ sg = ____sg_next(sg); ++ count = __sg_page_count(sg); ++ } ++ ++ *offset = n - idx; ++ return sg; ++ ++lookup: ++ rcu_read_lock(); ++ ++ sg = radix_tree_lookup(&iter->radix, n); ++ GEM_BUG_ON(!sg); ++ ++ /* If this index is in the middle of multi-page sg entry, ++ * the radix tree will contain a value entry that points ++ * to the start of that range. We will return the pointer to ++ * the base page and the offset of this page within the ++ * sg entry's range. ++ */ ++ *offset = 0; ++ if (unlikely(xa_is_value(sg))) { ++ unsigned long base = xa_to_value(sg); ++ ++ sg = radix_tree_lookup(&iter->radix, base); ++ GEM_BUG_ON(!sg); ++ ++ *offset = n - base; ++ } ++ ++ rcu_read_unlock(); ++ ++ return sg; ++} ++ ++struct page * ++i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n) ++{ ++ struct scatterlist *sg; ++ unsigned int offset; ++ ++ GEM_BUG_ON(!i915_gem_object_has_struct_page(obj)); ++ ++ sg = i915_gem_object_get_sg(obj, n, &offset); ++ return nth_page(sg_page(sg), offset); ++} ++ ++/* Like i915_gem_object_get_page(), but mark the returned page dirty */ ++struct page * ++i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, ++ unsigned int n) ++{ ++ struct page *page; ++ ++ page = i915_gem_object_get_page(obj, n); ++ if (!obj->mm.dirty) ++ set_page_dirty(page); ++ ++ return page; ++} ++ ++dma_addr_t ++i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, ++ unsigned long n) ++{ ++ struct scatterlist *sg; ++ unsigned int offset; ++ ++ sg = i915_gem_object_get_sg(obj, n, &offset); ++ return sg_dma_address(sg) + (offset << PAGE_SHIFT); ++} ++ ++int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align) ++{ ++ struct sg_table *pages; ++ int err; ++ ++ if (align > obj->base.size) ++ return -EINVAL; ++ ++ if (obj->ops == &i915_gem_phys_ops) ++ return 0; ++ ++ if (obj->ops != &i915_gem_object_ops) ++ return -EINVAL; ++ ++ err = i915_gem_object_unbind(obj); ++ if (err) ++ return err; ++ ++ mutex_lock(&obj->mm.lock); ++ ++ if (obj->mm.madv != I915_MADV_WILLNEED) { ++ err = -EFAULT; ++ goto err_unlock; ++ } ++ ++ if (obj->mm.quirked) { ++ err = -EFAULT; ++ goto err_unlock; ++ } ++ ++ if (obj->mm.mapping) { ++ err = -EBUSY; ++ goto err_unlock; ++ } ++ ++ pages = __i915_gem_object_unset_pages(obj); ++ ++ obj->ops = &i915_gem_phys_ops; ++ ++ err = ____i915_gem_object_get_pages(obj); ++ if (err) ++ goto err_xfer; ++ ++ /* Perma-pin (until release) the physical set of pages */ ++ __i915_gem_object_pin_pages(obj); ++ ++ if (!IS_ERR_OR_NULL(pages)) ++ i915_gem_object_ops.put_pages(obj, pages); ++ mutex_unlock(&obj->mm.lock); ++ return 0; ++ ++err_xfer: ++ obj->ops = &i915_gem_object_ops; ++ if (!IS_ERR_OR_NULL(pages)) { ++ unsigned int sg_page_sizes = i915_sg_page_sizes(pages->sgl); ++ ++ __i915_gem_object_set_pages(obj, pages, sg_page_sizes); ++ } ++err_unlock: ++ mutex_unlock(&obj->mm.lock); ++ return err; ++} ++ ++#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) ++#include "selftests/scatterlist.c" ++#include "selftests/mock_gem_device.c" ++#include "selftests/huge_gem_object.c" ++#include "selftests/huge_pages.c" ++#include "selftests/i915_gem_object.c" ++#include "selftests/i915_gem_coherency.c" ++#include "selftests/i915_gem.c" ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_gem.h b/drivers/gpu/drm/i915_legacy/i915_gem.h +new file mode 100644 +index 000000000000..9074eb1e843f +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gem.h +@@ -0,0 +1,97 @@ ++/* ++ * Copyright © 2016 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#ifndef __I915_GEM_H__ ++#define __I915_GEM_H__ ++ ++#include ++#include ++ ++struct drm_i915_private; ++ ++#ifdef CONFIG_DRM_I915_DEBUG_GEM ++ ++#define GEM_SHOW_DEBUG() (drm_debug & DRM_UT_DRIVER) ++ ++#define GEM_BUG_ON(condition) do { if (unlikely((condition))) { \ ++ pr_err("%s:%d GEM_BUG_ON(%s)\n", \ ++ __func__, __LINE__, __stringify(condition)); \ ++ GEM_TRACE("%s:%d GEM_BUG_ON(%s)\n", \ ++ __func__, __LINE__, __stringify(condition)); \ ++ BUG(); \ ++ } \ ++ } while(0) ++#define GEM_WARN_ON(expr) WARN_ON(expr) ++ ++#define GEM_DEBUG_DECL(var) var ++#define GEM_DEBUG_EXEC(expr) expr ++#define GEM_DEBUG_BUG_ON(expr) GEM_BUG_ON(expr) ++#define GEM_DEBUG_WARN_ON(expr) GEM_WARN_ON(expr) ++ ++#else ++ ++#define GEM_SHOW_DEBUG() (0) ++ ++#define GEM_BUG_ON(expr) BUILD_BUG_ON_INVALID(expr) ++#define GEM_WARN_ON(expr) ({ unlikely(!!(expr)); }) ++ ++#define GEM_DEBUG_DECL(var) ++#define GEM_DEBUG_EXEC(expr) do { } while (0) ++#define GEM_DEBUG_BUG_ON(expr) ++#define GEM_DEBUG_WARN_ON(expr) ({ BUILD_BUG_ON_INVALID(expr); 0; }) ++#endif ++ ++#if IS_ENABLED(CONFIG_DRM_I915_TRACE_GEM) ++#define GEM_TRACE(...) trace_printk(__VA_ARGS__) ++#define GEM_TRACE_DUMP() ftrace_dump(DUMP_ALL) ++#define GEM_TRACE_DUMP_ON(expr) \ ++ do { if (expr) ftrace_dump(DUMP_ALL); } while (0) ++#else ++#define GEM_TRACE(...) do { } while (0) ++#define GEM_TRACE_DUMP() do { } while (0) ++#define GEM_TRACE_DUMP_ON(expr) BUILD_BUG_ON_INVALID(expr) ++#endif ++ ++#define I915_GEM_IDLE_TIMEOUT (HZ / 5) ++ ++void i915_gem_park(struct drm_i915_private *i915); ++void i915_gem_unpark(struct drm_i915_private *i915); ++ ++static inline void __tasklet_disable_sync_once(struct tasklet_struct *t) ++{ ++ if (!atomic_fetch_inc(&t->count)) ++ tasklet_unlock_wait(t); ++} ++ ++static inline bool __tasklet_is_enabled(const struct tasklet_struct *t) ++{ ++ return !atomic_read(&t->count); ++} ++ ++static inline bool __tasklet_enable(struct tasklet_struct *t) ++{ ++ return atomic_dec_and_test(&t->count); ++} ++ ++#endif /* __I915_GEM_H__ */ +diff --git a/drivers/gpu/drm/i915_legacy/i915_gem_batch_pool.c b/drivers/gpu/drm/i915_legacy/i915_gem_batch_pool.c +new file mode 100644 +index 000000000000..f3890b664e3f +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gem_batch_pool.c +@@ -0,0 +1,140 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2014-2018 Intel Corporation ++ */ ++ ++#include "i915_gem_batch_pool.h" ++#include "i915_drv.h" ++ ++/** ++ * DOC: batch pool ++ * ++ * In order to submit batch buffers as 'secure', the software command parser ++ * must ensure that a batch buffer cannot be modified after parsing. It does ++ * this by copying the user provided batch buffer contents to a kernel owned ++ * buffer from which the hardware will actually execute, and by carefully ++ * managing the address space bindings for such buffers. ++ * ++ * The batch pool framework provides a mechanism for the driver to manage a ++ * set of scratch buffers to use for this purpose. The framework can be ++ * extended to support other uses cases should they arise. ++ */ ++ ++/** ++ * i915_gem_batch_pool_init() - initialize a batch buffer pool ++ * @pool: the batch buffer pool ++ * @engine: the associated request submission engine ++ */ ++void i915_gem_batch_pool_init(struct i915_gem_batch_pool *pool, ++ struct intel_engine_cs *engine) ++{ ++ int n; ++ ++ pool->engine = engine; ++ ++ for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) ++ INIT_LIST_HEAD(&pool->cache_list[n]); ++} ++ ++/** ++ * i915_gem_batch_pool_fini() - clean up a batch buffer pool ++ * @pool: the pool to clean up ++ * ++ * Note: Callers must hold the struct_mutex. ++ */ ++void i915_gem_batch_pool_fini(struct i915_gem_batch_pool *pool) ++{ ++ int n; ++ ++ lockdep_assert_held(&pool->engine->i915->drm.struct_mutex); ++ ++ for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) { ++ struct drm_i915_gem_object *obj, *next; ++ ++ list_for_each_entry_safe(obj, next, ++ &pool->cache_list[n], ++ batch_pool_link) ++ __i915_gem_object_release_unless_active(obj); ++ ++ INIT_LIST_HEAD(&pool->cache_list[n]); ++ } ++} ++ ++/** ++ * i915_gem_batch_pool_get() - allocate a buffer from the pool ++ * @pool: the batch buffer pool ++ * @size: the minimum desired size of the returned buffer ++ * ++ * Returns an inactive buffer from @pool with at least @size bytes, ++ * with the pages pinned. The caller must i915_gem_object_unpin_pages() ++ * on the returned object. ++ * ++ * Note: Callers must hold the struct_mutex ++ * ++ * Return: the buffer object or an error pointer ++ */ ++struct drm_i915_gem_object * ++i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, ++ size_t size) ++{ ++ struct drm_i915_gem_object *obj; ++ struct list_head *list; ++ int n, ret; ++ ++ lockdep_assert_held(&pool->engine->i915->drm.struct_mutex); ++ ++ /* Compute a power-of-two bucket, but throw everything greater than ++ * 16KiB into the same bucket: i.e. the the buckets hold objects of ++ * (1 page, 2 pages, 4 pages, 8+ pages). ++ */ ++ n = fls(size >> PAGE_SHIFT) - 1; ++ if (n >= ARRAY_SIZE(pool->cache_list)) ++ n = ARRAY_SIZE(pool->cache_list) - 1; ++ list = &pool->cache_list[n]; ++ ++ list_for_each_entry(obj, list, batch_pool_link) { ++ /* The batches are strictly LRU ordered */ ++ if (i915_gem_object_is_active(obj)) { ++ struct reservation_object *resv = obj->resv; ++ ++ if (!reservation_object_test_signaled_rcu(resv, true)) ++ break; ++ ++ i915_retire_requests(pool->engine->i915); ++ GEM_BUG_ON(i915_gem_object_is_active(obj)); ++ ++ /* ++ * The object is now idle, clear the array of shared ++ * fences before we add a new request. Although, we ++ * remain on the same engine, we may be on a different ++ * timeline and so may continually grow the array, ++ * trapping a reference to all the old fences, rather ++ * than replace the existing fence. ++ */ ++ if (rcu_access_pointer(resv->fence)) { ++ reservation_object_lock(resv, NULL); ++ reservation_object_add_excl_fence(resv, NULL); ++ reservation_object_unlock(resv); ++ } ++ } ++ ++ GEM_BUG_ON(!reservation_object_test_signaled_rcu(obj->resv, ++ true)); ++ ++ if (obj->base.size >= size) ++ goto found; ++ } ++ ++ obj = i915_gem_object_create_internal(pool->engine->i915, size); ++ if (IS_ERR(obj)) ++ return obj; ++ ++found: ++ ret = i915_gem_object_pin_pages(obj); ++ if (ret) ++ return ERR_PTR(ret); ++ ++ list_move_tail(&obj->batch_pool_link, list); ++ return obj; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_gem_batch_pool.h b/drivers/gpu/drm/i915_legacy/i915_gem_batch_pool.h +new file mode 100644 +index 000000000000..56947daaaf65 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gem_batch_pool.h +@@ -0,0 +1,25 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2014-2018 Intel Corporation ++ */ ++ ++#ifndef I915_GEM_BATCH_POOL_H ++#define I915_GEM_BATCH_POOL_H ++ ++#include ++ ++struct intel_engine_cs; ++ ++struct i915_gem_batch_pool { ++ struct intel_engine_cs *engine; ++ struct list_head cache_list[4]; ++}; ++ ++void i915_gem_batch_pool_init(struct i915_gem_batch_pool *pool, ++ struct intel_engine_cs *engine); ++void i915_gem_batch_pool_fini(struct i915_gem_batch_pool *pool); ++struct drm_i915_gem_object* ++i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, size_t size); ++ ++#endif /* I915_GEM_BATCH_POOL_H */ +diff --git a/drivers/gpu/drm/i915_legacy/i915_gem_clflush.c b/drivers/gpu/drm/i915_legacy/i915_gem_clflush.c +new file mode 100644 +index 000000000000..8e74c23cbd91 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gem_clflush.c +@@ -0,0 +1,178 @@ ++/* ++ * Copyright © 2016 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#include "i915_drv.h" ++#include "intel_frontbuffer.h" ++#include "i915_gem_clflush.h" ++ ++static DEFINE_SPINLOCK(clflush_lock); ++ ++struct clflush { ++ struct dma_fence dma; /* Must be first for dma_fence_free() */ ++ struct i915_sw_fence wait; ++ struct work_struct work; ++ struct drm_i915_gem_object *obj; ++}; ++ ++static const char *i915_clflush_get_driver_name(struct dma_fence *fence) ++{ ++ return DRIVER_NAME; ++} ++ ++static const char *i915_clflush_get_timeline_name(struct dma_fence *fence) ++{ ++ return "clflush"; ++} ++ ++static void i915_clflush_release(struct dma_fence *fence) ++{ ++ struct clflush *clflush = container_of(fence, typeof(*clflush), dma); ++ ++ i915_sw_fence_fini(&clflush->wait); ++ ++ BUILD_BUG_ON(offsetof(typeof(*clflush), dma)); ++ dma_fence_free(&clflush->dma); ++} ++ ++static const struct dma_fence_ops i915_clflush_ops = { ++ .get_driver_name = i915_clflush_get_driver_name, ++ .get_timeline_name = i915_clflush_get_timeline_name, ++ .release = i915_clflush_release, ++}; ++ ++static void __i915_do_clflush(struct drm_i915_gem_object *obj) ++{ ++ GEM_BUG_ON(!i915_gem_object_has_pages(obj)); ++ drm_clflush_sg(obj->mm.pages); ++ intel_fb_obj_flush(obj, ORIGIN_CPU); ++} ++ ++static void i915_clflush_work(struct work_struct *work) ++{ ++ struct clflush *clflush = container_of(work, typeof(*clflush), work); ++ struct drm_i915_gem_object *obj = clflush->obj; ++ ++ if (i915_gem_object_pin_pages(obj)) { ++ DRM_ERROR("Failed to acquire obj->pages for clflushing\n"); ++ goto out; ++ } ++ ++ __i915_do_clflush(obj); ++ ++ i915_gem_object_unpin_pages(obj); ++ ++out: ++ i915_gem_object_put(obj); ++ ++ dma_fence_signal(&clflush->dma); ++ dma_fence_put(&clflush->dma); ++} ++ ++static int __i915_sw_fence_call ++i915_clflush_notify(struct i915_sw_fence *fence, ++ enum i915_sw_fence_notify state) ++{ ++ struct clflush *clflush = container_of(fence, typeof(*clflush), wait); ++ ++ switch (state) { ++ case FENCE_COMPLETE: ++ schedule_work(&clflush->work); ++ break; ++ ++ case FENCE_FREE: ++ dma_fence_put(&clflush->dma); ++ break; ++ } ++ ++ return NOTIFY_DONE; ++} ++ ++bool i915_gem_clflush_object(struct drm_i915_gem_object *obj, ++ unsigned int flags) ++{ ++ struct clflush *clflush; ++ ++ /* ++ * Stolen memory is always coherent with the GPU as it is explicitly ++ * marked as wc by the system, or the system is cache-coherent. ++ * Similarly, we only access struct pages through the CPU cache, so ++ * anything not backed by physical memory we consider to be always ++ * coherent and not need clflushing. ++ */ ++ if (!i915_gem_object_has_struct_page(obj)) { ++ obj->cache_dirty = false; ++ return false; ++ } ++ ++ /* If the GPU is snooping the contents of the CPU cache, ++ * we do not need to manually clear the CPU cache lines. However, ++ * the caches are only snooped when the render cache is ++ * flushed/invalidated. As we always have to emit invalidations ++ * and flushes when moving into and out of the RENDER domain, correct ++ * snooping behaviour occurs naturally as the result of our domain ++ * tracking. ++ */ ++ if (!(flags & I915_CLFLUSH_FORCE) && ++ obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ) ++ return false; ++ ++ trace_i915_gem_object_clflush(obj); ++ ++ clflush = NULL; ++ if (!(flags & I915_CLFLUSH_SYNC)) ++ clflush = kmalloc(sizeof(*clflush), GFP_KERNEL); ++ if (clflush) { ++ GEM_BUG_ON(!obj->cache_dirty); ++ ++ dma_fence_init(&clflush->dma, ++ &i915_clflush_ops, ++ &clflush_lock, ++ to_i915(obj->base.dev)->mm.unordered_timeline, ++ 0); ++ i915_sw_fence_init(&clflush->wait, i915_clflush_notify); ++ ++ clflush->obj = i915_gem_object_get(obj); ++ INIT_WORK(&clflush->work, i915_clflush_work); ++ ++ dma_fence_get(&clflush->dma); ++ ++ i915_sw_fence_await_reservation(&clflush->wait, ++ obj->resv, NULL, ++ true, I915_FENCE_TIMEOUT, ++ I915_FENCE_GFP); ++ ++ reservation_object_lock(obj->resv, NULL); ++ reservation_object_add_excl_fence(obj->resv, &clflush->dma); ++ reservation_object_unlock(obj->resv); ++ ++ i915_sw_fence_commit(&clflush->wait); ++ } else if (obj->mm.pages) { ++ __i915_do_clflush(obj); ++ } else { ++ GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU); ++ } ++ ++ obj->cache_dirty = false; ++ return true; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_gem_clflush.h b/drivers/gpu/drm/i915_legacy/i915_gem_clflush.h +new file mode 100644 +index 000000000000..f390247561b3 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gem_clflush.h +@@ -0,0 +1,36 @@ ++/* ++ * Copyright © 2016 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#ifndef __I915_GEM_CLFLUSH_H__ ++#define __I915_GEM_CLFLUSH_H__ ++ ++struct drm_i915_private; ++struct drm_i915_gem_object; ++ ++bool i915_gem_clflush_object(struct drm_i915_gem_object *obj, ++ unsigned int flags); ++#define I915_CLFLUSH_FORCE BIT(0) ++#define I915_CLFLUSH_SYNC BIT(1) ++ ++#endif /* __I915_GEM_CLFLUSH_H__ */ +diff --git a/drivers/gpu/drm/i915_legacy/i915_gem_context.c b/drivers/gpu/drm/i915_legacy/i915_gem_context.c +new file mode 100644 +index 000000000000..fb5e2784d3c7 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gem_context.c +@@ -0,0 +1,1829 @@ ++/* ++ * Copyright © 2011-2012 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ * Authors: ++ * Ben Widawsky ++ * ++ */ ++ ++/* ++ * This file implements HW context support. On gen5+ a HW context consists of an ++ * opaque GPU object which is referenced at times of context saves and restores. ++ * With RC6 enabled, the context is also referenced as the GPU enters and exists ++ * from RC6 (GPU has it's own internal power context, except on gen5). Though ++ * something like a context does exist for the media ring, the code only ++ * supports contexts for the render ring. ++ * ++ * In software, there is a distinction between contexts created by the user, ++ * and the default HW context. The default HW context is used by GPU clients ++ * that do not request setup of their own hardware context. The default ++ * context's state is never restored to help prevent programming errors. This ++ * would happen if a client ran and piggy-backed off another clients GPU state. ++ * The default context only exists to give the GPU some offset to load as the ++ * current to invoke a save of the context we actually care about. In fact, the ++ * code could likely be constructed, albeit in a more complicated fashion, to ++ * never use the default context, though that limits the driver's ability to ++ * swap out, and/or destroy other contexts. ++ * ++ * All other contexts are created as a request by the GPU client. These contexts ++ * store GPU state, and thus allow GPU clients to not re-emit state (and ++ * potentially query certain state) at any time. The kernel driver makes ++ * certain that the appropriate commands are inserted. ++ * ++ * The context life cycle is semi-complicated in that context BOs may live ++ * longer than the context itself because of the way the hardware, and object ++ * tracking works. Below is a very crude representation of the state machine ++ * describing the context life. ++ * refcount pincount active ++ * S0: initial state 0 0 0 ++ * S1: context created 1 0 0 ++ * S2: context is currently running 2 1 X ++ * S3: GPU referenced, but not current 2 0 1 ++ * S4: context is current, but destroyed 1 1 0 ++ * S5: like S3, but destroyed 1 0 1 ++ * ++ * The most common (but not all) transitions: ++ * S0->S1: client creates a context ++ * S1->S2: client submits execbuf with context ++ * S2->S3: other clients submits execbuf with context ++ * S3->S1: context object was retired ++ * S3->S2: clients submits another execbuf ++ * S2->S4: context destroy called with current context ++ * S3->S5->S0: destroy path ++ * S4->S5->S0: destroy path on current context ++ * ++ * There are two confusing terms used above: ++ * The "current context" means the context which is currently running on the ++ * GPU. The GPU has loaded its state already and has stored away the gtt ++ * offset of the BO. The GPU is not actively referencing the data at this ++ * offset, but it will on the next context switch. The only way to avoid this ++ * is to do a GPU reset. ++ * ++ * An "active context' is one which was previously the "current context" and is ++ * on the active list waiting for the next context switch to occur. Until this ++ * happens, the object must remain at the same gtt offset. It is therefore ++ * possible to destroy a context, but it is still active. ++ * ++ */ ++ ++#include ++#include ++#include "i915_drv.h" ++#include "i915_globals.h" ++#include "i915_trace.h" ++#include "i915_user_extensions.h" ++#include "intel_lrc_reg.h" ++#include "intel_workarounds.h" ++ ++#define I915_CONTEXT_PARAM_VM 0x9 ++ ++#define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1 ++ ++static struct i915_global_gem_context { ++ struct i915_global base; ++ struct kmem_cache *slab_luts; ++} global; ++ ++struct i915_lut_handle *i915_lut_handle_alloc(void) ++{ ++ return kmem_cache_alloc(global.slab_luts, GFP_KERNEL); ++} ++ ++void i915_lut_handle_free(struct i915_lut_handle *lut) ++{ ++ return kmem_cache_free(global.slab_luts, lut); ++} ++ ++static void lut_close(struct i915_gem_context *ctx) ++{ ++ struct i915_lut_handle *lut, *ln; ++ struct radix_tree_iter iter; ++ void __rcu **slot; ++ ++ list_for_each_entry_safe(lut, ln, &ctx->handles_list, ctx_link) { ++ list_del(&lut->obj_link); ++ i915_lut_handle_free(lut); ++ } ++ INIT_LIST_HEAD(&ctx->handles_list); ++ ++ rcu_read_lock(); ++ radix_tree_for_each_slot(slot, &ctx->handles_vma, &iter, 0) { ++ struct i915_vma *vma = rcu_dereference_raw(*slot); ++ ++ radix_tree_iter_delete(&ctx->handles_vma, &iter, slot); ++ ++ vma->open_count--; ++ __i915_gem_object_release_unless_active(vma->obj); ++ } ++ rcu_read_unlock(); ++} ++ ++static inline int new_hw_id(struct drm_i915_private *i915, gfp_t gfp) ++{ ++ unsigned int max; ++ ++ lockdep_assert_held(&i915->contexts.mutex); ++ ++ if (INTEL_GEN(i915) >= 11) ++ max = GEN11_MAX_CONTEXT_HW_ID; ++ else if (USES_GUC_SUBMISSION(i915)) ++ /* ++ * When using GuC in proxy submission, GuC consumes the ++ * highest bit in the context id to indicate proxy submission. ++ */ ++ max = MAX_GUC_CONTEXT_HW_ID; ++ else ++ max = MAX_CONTEXT_HW_ID; ++ ++ return ida_simple_get(&i915->contexts.hw_ida, 0, max, gfp); ++} ++ ++static int steal_hw_id(struct drm_i915_private *i915) ++{ ++ struct i915_gem_context *ctx, *cn; ++ LIST_HEAD(pinned); ++ int id = -ENOSPC; ++ ++ lockdep_assert_held(&i915->contexts.mutex); ++ ++ list_for_each_entry_safe(ctx, cn, ++ &i915->contexts.hw_id_list, hw_id_link) { ++ if (atomic_read(&ctx->hw_id_pin_count)) { ++ list_move_tail(&ctx->hw_id_link, &pinned); ++ continue; ++ } ++ ++ GEM_BUG_ON(!ctx->hw_id); /* perma-pinned kernel context */ ++ list_del_init(&ctx->hw_id_link); ++ id = ctx->hw_id; ++ break; ++ } ++ ++ /* ++ * Remember how far we got up on the last repossesion scan, so the ++ * list is kept in a "least recently scanned" order. ++ */ ++ list_splice_tail(&pinned, &i915->contexts.hw_id_list); ++ return id; ++} ++ ++static int assign_hw_id(struct drm_i915_private *i915, unsigned int *out) ++{ ++ int ret; ++ ++ lockdep_assert_held(&i915->contexts.mutex); ++ ++ /* ++ * We prefer to steal/stall ourselves and our users over that of the ++ * entire system. That may be a little unfair to our users, and ++ * even hurt high priority clients. The choice is whether to oomkill ++ * something else, or steal a context id. ++ */ ++ ret = new_hw_id(i915, GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN); ++ if (unlikely(ret < 0)) { ++ ret = steal_hw_id(i915); ++ if (ret < 0) /* once again for the correct errno code */ ++ ret = new_hw_id(i915, GFP_KERNEL); ++ if (ret < 0) ++ return ret; ++ } ++ ++ *out = ret; ++ return 0; ++} ++ ++static void release_hw_id(struct i915_gem_context *ctx) ++{ ++ struct drm_i915_private *i915 = ctx->i915; ++ ++ if (list_empty(&ctx->hw_id_link)) ++ return; ++ ++ mutex_lock(&i915->contexts.mutex); ++ if (!list_empty(&ctx->hw_id_link)) { ++ ida_simple_remove(&i915->contexts.hw_ida, ctx->hw_id); ++ list_del_init(&ctx->hw_id_link); ++ } ++ mutex_unlock(&i915->contexts.mutex); ++} ++ ++static void i915_gem_context_free(struct i915_gem_context *ctx) ++{ ++ struct intel_context *it, *n; ++ ++ lockdep_assert_held(&ctx->i915->drm.struct_mutex); ++ GEM_BUG_ON(!i915_gem_context_is_closed(ctx)); ++ GEM_BUG_ON(!list_empty(&ctx->active_engines)); ++ ++ release_hw_id(ctx); ++ i915_ppgtt_put(ctx->ppgtt); ++ ++ rbtree_postorder_for_each_entry_safe(it, n, &ctx->hw_contexts, node) ++ intel_context_put(it); ++ ++ if (ctx->timeline) ++ i915_timeline_put(ctx->timeline); ++ ++ kfree(ctx->name); ++ put_pid(ctx->pid); ++ ++ list_del(&ctx->link); ++ mutex_destroy(&ctx->mutex); ++ ++ kfree_rcu(ctx, rcu); ++} ++ ++static void contexts_free(struct drm_i915_private *i915) ++{ ++ struct llist_node *freed = llist_del_all(&i915->contexts.free_list); ++ struct i915_gem_context *ctx, *cn; ++ ++ lockdep_assert_held(&i915->drm.struct_mutex); ++ ++ llist_for_each_entry_safe(ctx, cn, freed, free_link) ++ i915_gem_context_free(ctx); ++} ++ ++static void contexts_free_first(struct drm_i915_private *i915) ++{ ++ struct i915_gem_context *ctx; ++ struct llist_node *freed; ++ ++ lockdep_assert_held(&i915->drm.struct_mutex); ++ ++ freed = llist_del_first(&i915->contexts.free_list); ++ if (!freed) ++ return; ++ ++ ctx = container_of(freed, typeof(*ctx), free_link); ++ i915_gem_context_free(ctx); ++} ++ ++static void contexts_free_worker(struct work_struct *work) ++{ ++ struct drm_i915_private *i915 = ++ container_of(work, typeof(*i915), contexts.free_work); ++ ++ mutex_lock(&i915->drm.struct_mutex); ++ contexts_free(i915); ++ mutex_unlock(&i915->drm.struct_mutex); ++} ++ ++void i915_gem_context_release(struct kref *ref) ++{ ++ struct i915_gem_context *ctx = container_of(ref, typeof(*ctx), ref); ++ struct drm_i915_private *i915 = ctx->i915; ++ ++ trace_i915_context_free(ctx); ++ if (llist_add(&ctx->free_link, &i915->contexts.free_list)) ++ queue_work(i915->wq, &i915->contexts.free_work); ++} ++ ++static void context_close(struct i915_gem_context *ctx) ++{ ++ i915_gem_context_set_closed(ctx); ++ ++ /* ++ * This context will never again be assinged to HW, so we can ++ * reuse its ID for the next context. ++ */ ++ release_hw_id(ctx); ++ ++ /* ++ * The LUT uses the VMA as a backpointer to unref the object, ++ * so we need to clear the LUT before we close all the VMA (inside ++ * the ppgtt). ++ */ ++ lut_close(ctx); ++ ++ ctx->file_priv = ERR_PTR(-EBADF); ++ i915_gem_context_put(ctx); ++} ++ ++static u32 default_desc_template(const struct drm_i915_private *i915, ++ const struct i915_hw_ppgtt *ppgtt) ++{ ++ u32 address_mode; ++ u32 desc; ++ ++ desc = GEN8_CTX_VALID | GEN8_CTX_PRIVILEGE; ++ ++ address_mode = INTEL_LEGACY_32B_CONTEXT; ++ if (ppgtt && i915_vm_is_4lvl(&ppgtt->vm)) ++ address_mode = INTEL_LEGACY_64B_CONTEXT; ++ desc |= address_mode << GEN8_CTX_ADDRESSING_MODE_SHIFT; ++ ++ if (IS_GEN(i915, 8)) ++ desc |= GEN8_CTX_L3LLC_COHERENT; ++ ++ /* TODO: WaDisableLiteRestore when we start using semaphore ++ * signalling between Command Streamers ++ * ring->ctx_desc_template |= GEN8_CTX_FORCE_RESTORE; ++ */ ++ ++ return desc; ++} ++ ++static struct i915_gem_context * ++__create_context(struct drm_i915_private *dev_priv) ++{ ++ struct i915_gem_context *ctx; ++ int i; ++ ++ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); ++ if (!ctx) ++ return ERR_PTR(-ENOMEM); ++ ++ kref_init(&ctx->ref); ++ list_add_tail(&ctx->link, &dev_priv->contexts.list); ++ ctx->i915 = dev_priv; ++ ctx->sched.priority = I915_USER_PRIORITY(I915_PRIORITY_NORMAL); ++ INIT_LIST_HEAD(&ctx->active_engines); ++ mutex_init(&ctx->mutex); ++ ++ ctx->hw_contexts = RB_ROOT; ++ spin_lock_init(&ctx->hw_contexts_lock); ++ ++ INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL); ++ INIT_LIST_HEAD(&ctx->handles_list); ++ INIT_LIST_HEAD(&ctx->hw_id_link); ++ ++ /* NB: Mark all slices as needing a remap so that when the context first ++ * loads it will restore whatever remap state already exists. If there ++ * is no remap info, it will be a NOP. */ ++ ctx->remap_slice = ALL_L3_SLICES(dev_priv); ++ ++ i915_gem_context_set_bannable(ctx); ++ i915_gem_context_set_recoverable(ctx); ++ ++ ctx->ring_size = 4 * PAGE_SIZE; ++ ctx->desc_template = ++ default_desc_template(dev_priv, dev_priv->mm.aliasing_ppgtt); ++ ++ for (i = 0; i < ARRAY_SIZE(ctx->hang_timestamp); i++) ++ ctx->hang_timestamp[i] = jiffies - CONTEXT_FAST_HANG_JIFFIES; ++ ++ return ctx; ++} ++ ++static struct i915_hw_ppgtt * ++__set_ppgtt(struct i915_gem_context *ctx, struct i915_hw_ppgtt *ppgtt) ++{ ++ struct i915_hw_ppgtt *old = ctx->ppgtt; ++ ++ ctx->ppgtt = i915_ppgtt_get(ppgtt); ++ ctx->desc_template = default_desc_template(ctx->i915, ppgtt); ++ ++ return old; ++} ++ ++static void __assign_ppgtt(struct i915_gem_context *ctx, ++ struct i915_hw_ppgtt *ppgtt) ++{ ++ if (ppgtt == ctx->ppgtt) ++ return; ++ ++ ppgtt = __set_ppgtt(ctx, ppgtt); ++ if (ppgtt) ++ i915_ppgtt_put(ppgtt); ++} ++ ++static struct i915_gem_context * ++i915_gem_create_context(struct drm_i915_private *dev_priv, unsigned int flags) ++{ ++ struct i915_gem_context *ctx; ++ ++ lockdep_assert_held(&dev_priv->drm.struct_mutex); ++ ++ if (flags & I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE && ++ !HAS_EXECLISTS(dev_priv)) ++ return ERR_PTR(-EINVAL); ++ ++ /* Reap the most stale context */ ++ contexts_free_first(dev_priv); ++ ++ ctx = __create_context(dev_priv); ++ if (IS_ERR(ctx)) ++ return ctx; ++ ++ if (HAS_FULL_PPGTT(dev_priv)) { ++ struct i915_hw_ppgtt *ppgtt; ++ ++ ppgtt = i915_ppgtt_create(dev_priv); ++ if (IS_ERR(ppgtt)) { ++ DRM_DEBUG_DRIVER("PPGTT setup failed (%ld)\n", ++ PTR_ERR(ppgtt)); ++ context_close(ctx); ++ return ERR_CAST(ppgtt); ++ } ++ ++ __assign_ppgtt(ctx, ppgtt); ++ i915_ppgtt_put(ppgtt); ++ } ++ ++ if (flags & I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE) { ++ struct i915_timeline *timeline; ++ ++ timeline = i915_timeline_create(dev_priv, NULL); ++ if (IS_ERR(timeline)) { ++ context_close(ctx); ++ return ERR_CAST(timeline); ++ } ++ ++ ctx->timeline = timeline; ++ } ++ ++ trace_i915_context_create(ctx); ++ ++ return ctx; ++} ++ ++/** ++ * i915_gem_context_create_gvt - create a GVT GEM context ++ * @dev: drm device * ++ * ++ * This function is used to create a GVT specific GEM context. ++ * ++ * Returns: ++ * pointer to i915_gem_context on success, error pointer if failed ++ * ++ */ ++struct i915_gem_context * ++i915_gem_context_create_gvt(struct drm_device *dev) ++{ ++ struct i915_gem_context *ctx; ++ int ret; ++ ++ if (!IS_ENABLED(CONFIG_DRM_I915_GVT)) ++ return ERR_PTR(-ENODEV); ++ ++ ret = i915_mutex_lock_interruptible(dev); ++ if (ret) ++ return ERR_PTR(ret); ++ ++ ctx = i915_gem_create_context(to_i915(dev), 0); ++ if (IS_ERR(ctx)) ++ goto out; ++ ++ ret = i915_gem_context_pin_hw_id(ctx); ++ if (ret) { ++ context_close(ctx); ++ ctx = ERR_PTR(ret); ++ goto out; ++ } ++ ++ ctx->file_priv = ERR_PTR(-EBADF); ++ i915_gem_context_set_closed(ctx); /* not user accessible */ ++ i915_gem_context_clear_bannable(ctx); ++ i915_gem_context_set_force_single_submission(ctx); ++ if (!USES_GUC_SUBMISSION(to_i915(dev))) ++ ctx->ring_size = 512 * PAGE_SIZE; /* Max ring buffer size */ ++ ++ GEM_BUG_ON(i915_gem_context_is_kernel(ctx)); ++out: ++ mutex_unlock(&dev->struct_mutex); ++ return ctx; ++} ++ ++static void ++destroy_kernel_context(struct i915_gem_context **ctxp) ++{ ++ struct i915_gem_context *ctx; ++ ++ /* Keep the context ref so that we can free it immediately ourselves */ ++ ctx = i915_gem_context_get(fetch_and_zero(ctxp)); ++ GEM_BUG_ON(!i915_gem_context_is_kernel(ctx)); ++ ++ context_close(ctx); ++ i915_gem_context_free(ctx); ++} ++ ++struct i915_gem_context * ++i915_gem_context_create_kernel(struct drm_i915_private *i915, int prio) ++{ ++ struct i915_gem_context *ctx; ++ int err; ++ ++ ctx = i915_gem_create_context(i915, 0); ++ if (IS_ERR(ctx)) ++ return ctx; ++ ++ err = i915_gem_context_pin_hw_id(ctx); ++ if (err) { ++ destroy_kernel_context(&ctx); ++ return ERR_PTR(err); ++ } ++ ++ i915_gem_context_clear_bannable(ctx); ++ ctx->sched.priority = I915_USER_PRIORITY(prio); ++ ctx->ring_size = PAGE_SIZE; ++ ++ GEM_BUG_ON(!i915_gem_context_is_kernel(ctx)); ++ ++ return ctx; ++} ++ ++static void init_contexts(struct drm_i915_private *i915) ++{ ++ mutex_init(&i915->contexts.mutex); ++ INIT_LIST_HEAD(&i915->contexts.list); ++ ++ /* Using the simple ida interface, the max is limited by sizeof(int) */ ++ BUILD_BUG_ON(MAX_CONTEXT_HW_ID > INT_MAX); ++ BUILD_BUG_ON(GEN11_MAX_CONTEXT_HW_ID > INT_MAX); ++ ida_init(&i915->contexts.hw_ida); ++ INIT_LIST_HEAD(&i915->contexts.hw_id_list); ++ ++ INIT_WORK(&i915->contexts.free_work, contexts_free_worker); ++ init_llist_head(&i915->contexts.free_list); ++} ++ ++static bool needs_preempt_context(struct drm_i915_private *i915) ++{ ++ return HAS_EXECLISTS(i915); ++} ++ ++int i915_gem_contexts_init(struct drm_i915_private *dev_priv) ++{ ++ struct i915_gem_context *ctx; ++ ++ /* Reassure ourselves we are only called once */ ++ GEM_BUG_ON(dev_priv->kernel_context); ++ GEM_BUG_ON(dev_priv->preempt_context); ++ ++ intel_engine_init_ctx_wa(dev_priv->engine[RCS0]); ++ init_contexts(dev_priv); ++ ++ /* lowest priority; idle task */ ++ ctx = i915_gem_context_create_kernel(dev_priv, I915_PRIORITY_MIN); ++ if (IS_ERR(ctx)) { ++ DRM_ERROR("Failed to create default global context\n"); ++ return PTR_ERR(ctx); ++ } ++ /* ++ * For easy recognisablity, we want the kernel context to be 0 and then ++ * all user contexts will have non-zero hw_id. Kernel contexts are ++ * permanently pinned, so that we never suffer a stall and can ++ * use them from any allocation context (e.g. for evicting other ++ * contexts and from inside the shrinker). ++ */ ++ GEM_BUG_ON(ctx->hw_id); ++ GEM_BUG_ON(!atomic_read(&ctx->hw_id_pin_count)); ++ dev_priv->kernel_context = ctx; ++ ++ /* highest priority; preempting task */ ++ if (needs_preempt_context(dev_priv)) { ++ ctx = i915_gem_context_create_kernel(dev_priv, INT_MAX); ++ if (!IS_ERR(ctx)) ++ dev_priv->preempt_context = ctx; ++ else ++ DRM_ERROR("Failed to create preempt context; disabling preemption\n"); ++ } ++ ++ DRM_DEBUG_DRIVER("%s context support initialized\n", ++ DRIVER_CAPS(dev_priv)->has_logical_contexts ? ++ "logical" : "fake"); ++ return 0; ++} ++ ++void i915_gem_contexts_lost(struct drm_i915_private *dev_priv) ++{ ++ struct intel_engine_cs *engine; ++ enum intel_engine_id id; ++ ++ lockdep_assert_held(&dev_priv->drm.struct_mutex); ++ ++ for_each_engine(engine, dev_priv, id) ++ intel_engine_lost_context(engine); ++} ++ ++void i915_gem_contexts_fini(struct drm_i915_private *i915) ++{ ++ lockdep_assert_held(&i915->drm.struct_mutex); ++ ++ if (i915->preempt_context) ++ destroy_kernel_context(&i915->preempt_context); ++ destroy_kernel_context(&i915->kernel_context); ++ ++ /* Must free all deferred contexts (via flush_workqueue) first */ ++ GEM_BUG_ON(!list_empty(&i915->contexts.hw_id_list)); ++ ida_destroy(&i915->contexts.hw_ida); ++} ++ ++static int context_idr_cleanup(int id, void *p, void *data) ++{ ++ context_close(p); ++ return 0; ++} ++ ++static int vm_idr_cleanup(int id, void *p, void *data) ++{ ++ i915_ppgtt_put(p); ++ return 0; ++} ++ ++static int gem_context_register(struct i915_gem_context *ctx, ++ struct drm_i915_file_private *fpriv) ++{ ++ int ret; ++ ++ ctx->file_priv = fpriv; ++ if (ctx->ppgtt) ++ ctx->ppgtt->vm.file = fpriv; ++ ++ ctx->pid = get_task_pid(current, PIDTYPE_PID); ++ ctx->name = kasprintf(GFP_KERNEL, "%s[%d]", ++ current->comm, pid_nr(ctx->pid)); ++ if (!ctx->name) { ++ ret = -ENOMEM; ++ goto err_pid; ++ } ++ ++ /* And finally expose ourselves to userspace via the idr */ ++ mutex_lock(&fpriv->context_idr_lock); ++ ret = idr_alloc(&fpriv->context_idr, ctx, 0, 0, GFP_KERNEL); ++ mutex_unlock(&fpriv->context_idr_lock); ++ if (ret >= 0) ++ goto out; ++ ++ kfree(fetch_and_zero(&ctx->name)); ++err_pid: ++ put_pid(fetch_and_zero(&ctx->pid)); ++out: ++ return ret; ++} ++ ++int i915_gem_context_open(struct drm_i915_private *i915, ++ struct drm_file *file) ++{ ++ struct drm_i915_file_private *file_priv = file->driver_priv; ++ struct i915_gem_context *ctx; ++ int err; ++ ++ mutex_init(&file_priv->context_idr_lock); ++ mutex_init(&file_priv->vm_idr_lock); ++ ++ idr_init(&file_priv->context_idr); ++ idr_init_base(&file_priv->vm_idr, 1); ++ ++ mutex_lock(&i915->drm.struct_mutex); ++ ctx = i915_gem_create_context(i915, 0); ++ mutex_unlock(&i915->drm.struct_mutex); ++ if (IS_ERR(ctx)) { ++ err = PTR_ERR(ctx); ++ goto err; ++ } ++ ++ err = gem_context_register(ctx, file_priv); ++ if (err < 0) ++ goto err_ctx; ++ ++ GEM_BUG_ON(i915_gem_context_is_kernel(ctx)); ++ GEM_BUG_ON(err > 0); ++ ++ return 0; ++ ++err_ctx: ++ mutex_lock(&i915->drm.struct_mutex); ++ context_close(ctx); ++ mutex_unlock(&i915->drm.struct_mutex); ++err: ++ idr_destroy(&file_priv->vm_idr); ++ idr_destroy(&file_priv->context_idr); ++ mutex_destroy(&file_priv->vm_idr_lock); ++ mutex_destroy(&file_priv->context_idr_lock); ++ return err; ++} ++ ++void i915_gem_context_close(struct drm_file *file) ++{ ++ struct drm_i915_file_private *file_priv = file->driver_priv; ++ ++ lockdep_assert_held(&file_priv->dev_priv->drm.struct_mutex); ++ ++ idr_for_each(&file_priv->context_idr, context_idr_cleanup, NULL); ++ idr_destroy(&file_priv->context_idr); ++ mutex_destroy(&file_priv->context_idr_lock); ++ ++ idr_for_each(&file_priv->vm_idr, vm_idr_cleanup, NULL); ++ idr_destroy(&file_priv->vm_idr); ++ mutex_destroy(&file_priv->vm_idr_lock); ++} ++ ++int i915_gem_vm_create_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_i915_private *i915 = to_i915(dev); ++ struct drm_i915_gem_vm_control *args = data; ++ struct drm_i915_file_private *file_priv = file->driver_priv; ++ struct i915_hw_ppgtt *ppgtt; ++ int err; ++ ++ if (!HAS_FULL_PPGTT(i915)) ++ return -ENODEV; ++ ++ if (args->flags) ++ return -EINVAL; ++ ++ ppgtt = i915_ppgtt_create(i915); ++ if (IS_ERR(ppgtt)) ++ return PTR_ERR(ppgtt); ++ ++ ppgtt->vm.file = file_priv; ++ ++ if (args->extensions) { ++ err = i915_user_extensions(u64_to_user_ptr(args->extensions), ++ NULL, 0, ++ ppgtt); ++ if (err) ++ goto err_put; ++ } ++ ++ err = mutex_lock_interruptible(&file_priv->vm_idr_lock); ++ if (err) ++ goto err_put; ++ ++ err = idr_alloc(&file_priv->vm_idr, ppgtt, 0, 0, GFP_KERNEL); ++ if (err < 0) ++ goto err_unlock; ++ ++ GEM_BUG_ON(err == 0); /* reserved for default/unassigned ppgtt */ ++ ppgtt->user_handle = err; ++ ++ mutex_unlock(&file_priv->vm_idr_lock); ++ ++ args->vm_id = err; ++ return 0; ++ ++err_unlock: ++ mutex_unlock(&file_priv->vm_idr_lock); ++err_put: ++ i915_ppgtt_put(ppgtt); ++ return err; ++} ++ ++int i915_gem_vm_destroy_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_i915_file_private *file_priv = file->driver_priv; ++ struct drm_i915_gem_vm_control *args = data; ++ struct i915_hw_ppgtt *ppgtt; ++ int err; ++ u32 id; ++ ++ if (args->flags) ++ return -EINVAL; ++ ++ if (args->extensions) ++ return -EINVAL; ++ ++ id = args->vm_id; ++ if (!id) ++ return -ENOENT; ++ ++ err = mutex_lock_interruptible(&file_priv->vm_idr_lock); ++ if (err) ++ return err; ++ ++ ppgtt = idr_remove(&file_priv->vm_idr, id); ++ if (ppgtt) { ++ GEM_BUG_ON(ppgtt->user_handle != id); ++ ppgtt->user_handle = 0; ++ } ++ ++ mutex_unlock(&file_priv->vm_idr_lock); ++ if (!ppgtt) ++ return -ENOENT; ++ ++ i915_ppgtt_put(ppgtt); ++ return 0; ++} ++ ++static struct i915_request * ++last_request_on_engine(struct i915_timeline *timeline, ++ struct intel_engine_cs *engine) ++{ ++ struct i915_request *rq; ++ ++ GEM_BUG_ON(timeline == &engine->timeline); ++ ++ rq = i915_active_request_raw(&timeline->last_request, ++ &engine->i915->drm.struct_mutex); ++ if (rq && rq->engine->mask & engine->mask) { ++ GEM_TRACE("last request on engine %s: %llx:%llu\n", ++ engine->name, rq->fence.context, rq->fence.seqno); ++ GEM_BUG_ON(rq->timeline != timeline); ++ return rq; ++ } ++ ++ return NULL; ++} ++ ++struct context_barrier_task { ++ struct i915_active base; ++ void (*task)(void *data); ++ void *data; ++}; ++ ++static void cb_retire(struct i915_active *base) ++{ ++ struct context_barrier_task *cb = container_of(base, typeof(*cb), base); ++ ++ if (cb->task) ++ cb->task(cb->data); ++ ++ i915_active_fini(&cb->base); ++ kfree(cb); ++} ++ ++I915_SELFTEST_DECLARE(static intel_engine_mask_t context_barrier_inject_fault); ++static int context_barrier_task(struct i915_gem_context *ctx, ++ intel_engine_mask_t engines, ++ int (*emit)(struct i915_request *rq, void *data), ++ void (*task)(void *data), ++ void *data) ++{ ++ struct drm_i915_private *i915 = ctx->i915; ++ struct context_barrier_task *cb; ++ struct intel_context *ce, *next; ++ intel_wakeref_t wakeref; ++ int err = 0; ++ ++ lockdep_assert_held(&i915->drm.struct_mutex); ++ GEM_BUG_ON(!task); ++ ++ cb = kmalloc(sizeof(*cb), GFP_KERNEL); ++ if (!cb) ++ return -ENOMEM; ++ ++ i915_active_init(i915, &cb->base, cb_retire); ++ i915_active_acquire(&cb->base); ++ ++ wakeref = intel_runtime_pm_get(i915); ++ rbtree_postorder_for_each_entry_safe(ce, next, &ctx->hw_contexts, node) { ++ struct intel_engine_cs *engine = ce->engine; ++ struct i915_request *rq; ++ ++ if (!(engine->mask & engines)) ++ continue; ++ ++ if (I915_SELFTEST_ONLY(context_barrier_inject_fault & ++ engine->mask)) { ++ err = -ENXIO; ++ break; ++ } ++ ++ rq = i915_request_alloc(engine, ctx); ++ if (IS_ERR(rq)) { ++ err = PTR_ERR(rq); ++ break; ++ } ++ ++ err = 0; ++ if (emit) ++ err = emit(rq, data); ++ if (err == 0) ++ err = i915_active_ref(&cb->base, rq->fence.context, rq); ++ ++ i915_request_add(rq); ++ if (err) ++ break; ++ } ++ intel_runtime_pm_put(i915, wakeref); ++ ++ cb->task = err ? NULL : task; /* caller needs to unwind instead */ ++ cb->data = data; ++ ++ i915_active_release(&cb->base); ++ ++ return err; ++} ++ ++int i915_gem_switch_to_kernel_context(struct drm_i915_private *i915, ++ intel_engine_mask_t mask) ++{ ++ struct intel_engine_cs *engine; ++ ++ GEM_TRACE("awake?=%s\n", yesno(i915->gt.awake)); ++ ++ lockdep_assert_held(&i915->drm.struct_mutex); ++ GEM_BUG_ON(!i915->kernel_context); ++ ++ /* Inoperable, so presume the GPU is safely pointing into the void! */ ++ if (i915_terminally_wedged(i915)) ++ return 0; ++ ++ for_each_engine_masked(engine, i915, mask, mask) { ++ struct intel_ring *ring; ++ struct i915_request *rq; ++ ++ rq = i915_request_alloc(engine, i915->kernel_context); ++ if (IS_ERR(rq)) ++ return PTR_ERR(rq); ++ ++ /* Queue this switch after all other activity */ ++ list_for_each_entry(ring, &i915->gt.active_rings, active_link) { ++ struct i915_request *prev; ++ ++ prev = last_request_on_engine(ring->timeline, engine); ++ if (!prev) ++ continue; ++ ++ if (prev->gem_context == i915->kernel_context) ++ continue; ++ ++ GEM_TRACE("add barrier on %s for %llx:%lld\n", ++ engine->name, ++ prev->fence.context, ++ prev->fence.seqno); ++ i915_sw_fence_await_sw_fence_gfp(&rq->submit, ++ &prev->submit, ++ I915_FENCE_GFP); ++ } ++ ++ i915_request_add(rq); ++ } ++ ++ return 0; ++} ++ ++static int get_ppgtt(struct drm_i915_file_private *file_priv, ++ struct i915_gem_context *ctx, ++ struct drm_i915_gem_context_param *args) ++{ ++ struct i915_hw_ppgtt *ppgtt; ++ int ret; ++ ++ return -EINVAL; /* nothing to see here; please move along */ ++ ++ if (!ctx->ppgtt) ++ return -ENODEV; ++ ++ /* XXX rcu acquire? */ ++ ret = mutex_lock_interruptible(&ctx->i915->drm.struct_mutex); ++ if (ret) ++ return ret; ++ ++ ppgtt = i915_ppgtt_get(ctx->ppgtt); ++ mutex_unlock(&ctx->i915->drm.struct_mutex); ++ ++ ret = mutex_lock_interruptible(&file_priv->vm_idr_lock); ++ if (ret) ++ goto err_put; ++ ++ if (!ppgtt->user_handle) { ++ ret = idr_alloc(&file_priv->vm_idr, ppgtt, 0, 0, GFP_KERNEL); ++ GEM_BUG_ON(!ret); ++ if (ret < 0) ++ goto err_unlock; ++ ++ ppgtt->user_handle = ret; ++ i915_ppgtt_get(ppgtt); ++ } ++ ++ args->size = 0; ++ args->value = ppgtt->user_handle; ++ ++ ret = 0; ++err_unlock: ++ mutex_unlock(&file_priv->vm_idr_lock); ++err_put: ++ i915_ppgtt_put(ppgtt); ++ return ret; ++} ++ ++static void set_ppgtt_barrier(void *data) ++{ ++ struct i915_hw_ppgtt *old = data; ++ ++ if (INTEL_GEN(old->vm.i915) < 8) ++ gen6_ppgtt_unpin_all(old); ++ ++ i915_ppgtt_put(old); ++} ++ ++static int emit_ppgtt_update(struct i915_request *rq, void *data) ++{ ++ struct i915_hw_ppgtt *ppgtt = rq->gem_context->ppgtt; ++ struct intel_engine_cs *engine = rq->engine; ++ u32 base = engine->mmio_base; ++ u32 *cs; ++ int i; ++ ++ if (i915_vm_is_4lvl(&ppgtt->vm)) { ++ const dma_addr_t pd_daddr = px_dma(&ppgtt->pml4); ++ ++ cs = intel_ring_begin(rq, 6); ++ if (IS_ERR(cs)) ++ return PTR_ERR(cs); ++ ++ *cs++ = MI_LOAD_REGISTER_IMM(2); ++ ++ *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, 0)); ++ *cs++ = upper_32_bits(pd_daddr); ++ *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, 0)); ++ *cs++ = lower_32_bits(pd_daddr); ++ ++ *cs++ = MI_NOOP; ++ intel_ring_advance(rq, cs); ++ } else if (HAS_LOGICAL_RING_CONTEXTS(engine->i915)) { ++ cs = intel_ring_begin(rq, 4 * GEN8_3LVL_PDPES + 2); ++ if (IS_ERR(cs)) ++ return PTR_ERR(cs); ++ ++ *cs++ = MI_LOAD_REGISTER_IMM(2 * GEN8_3LVL_PDPES); ++ for (i = GEN8_3LVL_PDPES; i--; ) { ++ const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i); ++ ++ *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, i)); ++ *cs++ = upper_32_bits(pd_daddr); ++ *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, i)); ++ *cs++ = lower_32_bits(pd_daddr); ++ } ++ *cs++ = MI_NOOP; ++ intel_ring_advance(rq, cs); ++ } else { ++ /* ppGTT is not part of the legacy context image */ ++ gen6_ppgtt_pin(ppgtt); ++ } ++ ++ return 0; ++} ++ ++static int set_ppgtt(struct drm_i915_file_private *file_priv, ++ struct i915_gem_context *ctx, ++ struct drm_i915_gem_context_param *args) ++{ ++ struct i915_hw_ppgtt *ppgtt, *old; ++ int err; ++ ++ return -EINVAL; /* nothing to see here; please move along */ ++ ++ if (args->size) ++ return -EINVAL; ++ ++ if (!ctx->ppgtt) ++ return -ENODEV; ++ ++ if (upper_32_bits(args->value)) ++ return -ENOENT; ++ ++ err = mutex_lock_interruptible(&file_priv->vm_idr_lock); ++ if (err) ++ return err; ++ ++ ppgtt = idr_find(&file_priv->vm_idr, args->value); ++ if (ppgtt) { ++ GEM_BUG_ON(ppgtt->user_handle != args->value); ++ i915_ppgtt_get(ppgtt); ++ } ++ mutex_unlock(&file_priv->vm_idr_lock); ++ if (!ppgtt) ++ return -ENOENT; ++ ++ err = mutex_lock_interruptible(&ctx->i915->drm.struct_mutex); ++ if (err) ++ goto out; ++ ++ if (ppgtt == ctx->ppgtt) ++ goto unlock; ++ ++ /* Teardown the existing obj:vma cache, it will have to be rebuilt. */ ++ lut_close(ctx); ++ ++ old = __set_ppgtt(ctx, ppgtt); ++ ++ /* ++ * We need to flush any requests using the current ppgtt before ++ * we release it as the requests do not hold a reference themselves, ++ * only indirectly through the context. ++ */ ++ err = context_barrier_task(ctx, ALL_ENGINES, ++ emit_ppgtt_update, ++ set_ppgtt_barrier, ++ old); ++ if (err) { ++ ctx->ppgtt = old; ++ ctx->desc_template = default_desc_template(ctx->i915, old); ++ i915_ppgtt_put(ppgtt); ++ } ++ ++unlock: ++ mutex_unlock(&ctx->i915->drm.struct_mutex); ++ ++out: ++ i915_ppgtt_put(ppgtt); ++ return err; ++} ++ ++static int gen8_emit_rpcs_config(struct i915_request *rq, ++ struct intel_context *ce, ++ struct intel_sseu sseu) ++{ ++ u64 offset; ++ u32 *cs; ++ ++ cs = intel_ring_begin(rq, 4); ++ if (IS_ERR(cs)) ++ return PTR_ERR(cs); ++ ++ offset = i915_ggtt_offset(ce->state) + ++ LRC_STATE_PN * PAGE_SIZE + ++ (CTX_R_PWR_CLK_STATE + 1) * 4; ++ ++ *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; ++ *cs++ = lower_32_bits(offset); ++ *cs++ = upper_32_bits(offset); ++ *cs++ = gen8_make_rpcs(rq->i915, &sseu); ++ ++ intel_ring_advance(rq, cs); ++ ++ return 0; ++} ++ ++static int ++gen8_modify_rpcs(struct intel_context *ce, struct intel_sseu sseu) ++{ ++ struct drm_i915_private *i915 = ce->engine->i915; ++ struct i915_request *rq; ++ intel_wakeref_t wakeref; ++ int ret; ++ ++ lockdep_assert_held(&ce->pin_mutex); ++ ++ /* ++ * If the context is not idle, we have to submit an ordered request to ++ * modify its context image via the kernel context (writing to our own ++ * image, or into the registers directory, does not stick). Pristine ++ * and idle contexts will be configured on pinning. ++ */ ++ if (!intel_context_is_pinned(ce)) ++ return 0; ++ ++ /* Submitting requests etc needs the hw awake. */ ++ wakeref = intel_runtime_pm_get(i915); ++ ++ rq = i915_request_alloc(ce->engine, i915->kernel_context); ++ if (IS_ERR(rq)) { ++ ret = PTR_ERR(rq); ++ goto out_put; ++ } ++ ++ /* Queue this switch after all other activity by this context. */ ++ ret = i915_active_request_set(&ce->ring->timeline->last_request, rq); ++ if (ret) ++ goto out_add; ++ ++ ret = gen8_emit_rpcs_config(rq, ce, sseu); ++ if (ret) ++ goto out_add; ++ ++ /* ++ * Guarantee context image and the timeline remains pinned until the ++ * modifying request is retired by setting the ce activity tracker. ++ * ++ * But we only need to take one pin on the account of it. Or in other ++ * words transfer the pinned ce object to tracked active request. ++ */ ++ if (!i915_active_request_isset(&ce->active_tracker)) ++ __intel_context_pin(ce); ++ __i915_active_request_set(&ce->active_tracker, rq); ++ ++out_add: ++ i915_request_add(rq); ++out_put: ++ intel_runtime_pm_put(i915, wakeref); ++ ++ return ret; ++} ++ ++static int ++__i915_gem_context_reconfigure_sseu(struct i915_gem_context *ctx, ++ struct intel_engine_cs *engine, ++ struct intel_sseu sseu) ++{ ++ struct intel_context *ce; ++ int ret = 0; ++ ++ GEM_BUG_ON(INTEL_GEN(ctx->i915) < 8); ++ GEM_BUG_ON(engine->id != RCS0); ++ ++ ce = intel_context_pin_lock(ctx, engine); ++ if (IS_ERR(ce)) ++ return PTR_ERR(ce); ++ ++ /* Nothing to do if unmodified. */ ++ if (!memcmp(&ce->sseu, &sseu, sizeof(sseu))) ++ goto unlock; ++ ++ ret = gen8_modify_rpcs(ce, sseu); ++ if (!ret) ++ ce->sseu = sseu; ++ ++unlock: ++ intel_context_pin_unlock(ce); ++ return ret; ++} ++ ++static int ++i915_gem_context_reconfigure_sseu(struct i915_gem_context *ctx, ++ struct intel_engine_cs *engine, ++ struct intel_sseu sseu) ++{ ++ int ret; ++ ++ ret = mutex_lock_interruptible(&ctx->i915->drm.struct_mutex); ++ if (ret) ++ return ret; ++ ++ ret = __i915_gem_context_reconfigure_sseu(ctx, engine, sseu); ++ ++ mutex_unlock(&ctx->i915->drm.struct_mutex); ++ ++ return ret; ++} ++ ++static int ++user_to_context_sseu(struct drm_i915_private *i915, ++ const struct drm_i915_gem_context_param_sseu *user, ++ struct intel_sseu *context) ++{ ++ const struct sseu_dev_info *device = &RUNTIME_INFO(i915)->sseu; ++ ++ /* No zeros in any field. */ ++ if (!user->slice_mask || !user->subslice_mask || ++ !user->min_eus_per_subslice || !user->max_eus_per_subslice) ++ return -EINVAL; ++ ++ /* Max > min. */ ++ if (user->max_eus_per_subslice < user->min_eus_per_subslice) ++ return -EINVAL; ++ ++ /* ++ * Some future proofing on the types since the uAPI is wider than the ++ * current internal implementation. ++ */ ++ if (overflows_type(user->slice_mask, context->slice_mask) || ++ overflows_type(user->subslice_mask, context->subslice_mask) || ++ overflows_type(user->min_eus_per_subslice, ++ context->min_eus_per_subslice) || ++ overflows_type(user->max_eus_per_subslice, ++ context->max_eus_per_subslice)) ++ return -EINVAL; ++ ++ /* Check validity against hardware. */ ++ if (user->slice_mask & ~device->slice_mask) ++ return -EINVAL; ++ ++ if (user->subslice_mask & ~device->subslice_mask[0]) ++ return -EINVAL; ++ ++ if (user->max_eus_per_subslice > device->max_eus_per_subslice) ++ return -EINVAL; ++ ++ context->slice_mask = user->slice_mask; ++ context->subslice_mask = user->subslice_mask; ++ context->min_eus_per_subslice = user->min_eus_per_subslice; ++ context->max_eus_per_subslice = user->max_eus_per_subslice; ++ ++ /* Part specific restrictions. */ ++ if (IS_GEN(i915, 11)) { ++ unsigned int hw_s = hweight8(device->slice_mask); ++ unsigned int hw_ss_per_s = hweight8(device->subslice_mask[0]); ++ unsigned int req_s = hweight8(context->slice_mask); ++ unsigned int req_ss = hweight8(context->subslice_mask); ++ ++ /* ++ * Only full subslice enablement is possible if more than one ++ * slice is turned on. ++ */ ++ if (req_s > 1 && req_ss != hw_ss_per_s) ++ return -EINVAL; ++ ++ /* ++ * If more than four (SScount bitfield limit) subslices are ++ * requested then the number has to be even. ++ */ ++ if (req_ss > 4 && (req_ss & 1)) ++ return -EINVAL; ++ ++ /* ++ * If only one slice is enabled and subslice count is below the ++ * device full enablement, it must be at most half of the all ++ * available subslices. ++ */ ++ if (req_s == 1 && req_ss < hw_ss_per_s && ++ req_ss > (hw_ss_per_s / 2)) ++ return -EINVAL; ++ ++ /* ABI restriction - VME use case only. */ ++ ++ /* All slices or one slice only. */ ++ if (req_s != 1 && req_s != hw_s) ++ return -EINVAL; ++ ++ /* ++ * Half subslices or full enablement only when one slice is ++ * enabled. ++ */ ++ if (req_s == 1 && ++ (req_ss != hw_ss_per_s && req_ss != (hw_ss_per_s / 2))) ++ return -EINVAL; ++ ++ /* No EU configuration changes. */ ++ if ((user->min_eus_per_subslice != ++ device->max_eus_per_subslice) || ++ (user->max_eus_per_subslice != ++ device->max_eus_per_subslice)) ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static int set_sseu(struct i915_gem_context *ctx, ++ struct drm_i915_gem_context_param *args) ++{ ++ struct drm_i915_private *i915 = ctx->i915; ++ struct drm_i915_gem_context_param_sseu user_sseu; ++ struct intel_engine_cs *engine; ++ struct intel_sseu sseu; ++ int ret; ++ ++ if (args->size < sizeof(user_sseu)) ++ return -EINVAL; ++ ++ if (!IS_GEN(i915, 11)) ++ return -ENODEV; ++ ++ if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value), ++ sizeof(user_sseu))) ++ return -EFAULT; ++ ++ if (user_sseu.flags || user_sseu.rsvd) ++ return -EINVAL; ++ ++ engine = intel_engine_lookup_user(i915, ++ user_sseu.engine.engine_class, ++ user_sseu.engine.engine_instance); ++ if (!engine) ++ return -EINVAL; ++ ++ /* Only render engine supports RPCS configuration. */ ++ if (engine->class != RENDER_CLASS) ++ return -ENODEV; ++ ++ ret = user_to_context_sseu(i915, &user_sseu, &sseu); ++ if (ret) ++ return ret; ++ ++ ret = i915_gem_context_reconfigure_sseu(ctx, engine, sseu); ++ if (ret) ++ return ret; ++ ++ args->size = sizeof(user_sseu); ++ ++ return 0; ++} ++ ++static int ctx_setparam(struct drm_i915_file_private *fpriv, ++ struct i915_gem_context *ctx, ++ struct drm_i915_gem_context_param *args) ++{ ++ int ret = 0; ++ ++ switch (args->param) { ++ case I915_CONTEXT_PARAM_NO_ZEROMAP: ++ if (args->size) ++ ret = -EINVAL; ++ else if (args->value) ++ set_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags); ++ else ++ clear_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags); ++ break; ++ ++ case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: ++ if (args->size) ++ ret = -EINVAL; ++ else if (args->value) ++ i915_gem_context_set_no_error_capture(ctx); ++ else ++ i915_gem_context_clear_no_error_capture(ctx); ++ break; ++ ++ case I915_CONTEXT_PARAM_BANNABLE: ++ if (args->size) ++ ret = -EINVAL; ++ else if (!capable(CAP_SYS_ADMIN) && !args->value) ++ ret = -EPERM; ++ else if (args->value) ++ i915_gem_context_set_bannable(ctx); ++ else ++ i915_gem_context_clear_bannable(ctx); ++ break; ++ ++ case I915_CONTEXT_PARAM_RECOVERABLE: ++ if (args->size) ++ ret = -EINVAL; ++ else if (args->value) ++ i915_gem_context_set_recoverable(ctx); ++ else ++ i915_gem_context_clear_recoverable(ctx); ++ break; ++ ++ case I915_CONTEXT_PARAM_PRIORITY: ++ { ++ s64 priority = args->value; ++ ++ if (args->size) ++ ret = -EINVAL; ++ else if (!(ctx->i915->caps.scheduler & I915_SCHEDULER_CAP_PRIORITY)) ++ ret = -ENODEV; ++ else if (priority > I915_CONTEXT_MAX_USER_PRIORITY || ++ priority < I915_CONTEXT_MIN_USER_PRIORITY) ++ ret = -EINVAL; ++ else if (priority > I915_CONTEXT_DEFAULT_PRIORITY && ++ !capable(CAP_SYS_NICE)) ++ ret = -EPERM; ++ else ++ ctx->sched.priority = ++ I915_USER_PRIORITY(priority); ++ } ++ break; ++ ++ case I915_CONTEXT_PARAM_SSEU: ++ ret = set_sseu(ctx, args); ++ break; ++ ++ case I915_CONTEXT_PARAM_VM: ++ ret = set_ppgtt(fpriv, ctx, args); ++ break; ++ ++ case I915_CONTEXT_PARAM_BAN_PERIOD: ++ default: ++ ret = -EINVAL; ++ break; ++ } ++ ++ return ret; ++} ++ ++struct create_ext { ++ struct i915_gem_context *ctx; ++ struct drm_i915_file_private *fpriv; ++}; ++ ++static int create_setparam(struct i915_user_extension __user *ext, void *data) ++{ ++ struct drm_i915_gem_context_create_ext_setparam local; ++ const struct create_ext *arg = data; ++ ++ if (copy_from_user(&local, ext, sizeof(local))) ++ return -EFAULT; ++ ++ if (local.param.ctx_id) ++ return -EINVAL; ++ ++ return ctx_setparam(arg->fpriv, arg->ctx, &local.param); ++} ++ ++static const i915_user_extension_fn create_extensions[] = { ++ [I915_CONTEXT_CREATE_EXT_SETPARAM] = create_setparam, ++}; ++ ++static bool client_is_banned(struct drm_i915_file_private *file_priv) ++{ ++ return atomic_read(&file_priv->ban_score) >= I915_CLIENT_SCORE_BANNED; ++} ++ ++int i915_gem_context_create_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_i915_private *i915 = to_i915(dev); ++ struct drm_i915_gem_context_create_ext *args = data; ++ struct create_ext ext_data; ++ int ret; ++ ++ if (!DRIVER_CAPS(i915)->has_logical_contexts) ++ return -ENODEV; ++ ++ if (args->flags & I915_CONTEXT_CREATE_FLAGS_UNKNOWN) ++ return -EINVAL; ++ ++ ret = i915_terminally_wedged(i915); ++ if (ret) ++ return ret; ++ ++ ext_data.fpriv = file->driver_priv; ++ if (client_is_banned(ext_data.fpriv)) { ++ DRM_DEBUG("client %s[%d] banned from creating ctx\n", ++ current->comm, ++ pid_nr(get_task_pid(current, PIDTYPE_PID))); ++ return -EIO; ++ } ++ ++ ret = i915_mutex_lock_interruptible(dev); ++ if (ret) ++ return ret; ++ ++ ext_data.ctx = i915_gem_create_context(i915, args->flags); ++ mutex_unlock(&dev->struct_mutex); ++ if (IS_ERR(ext_data.ctx)) ++ return PTR_ERR(ext_data.ctx); ++ ++ if (args->flags & I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS) { ++ ret = i915_user_extensions(u64_to_user_ptr(args->extensions), ++ create_extensions, ++ ARRAY_SIZE(create_extensions), ++ &ext_data); ++ if (ret) ++ goto err_ctx; ++ } ++ ++ ret = gem_context_register(ext_data.ctx, ext_data.fpriv); ++ if (ret < 0) ++ goto err_ctx; ++ ++ args->ctx_id = ret; ++ DRM_DEBUG("HW context %d created\n", args->ctx_id); ++ ++ return 0; ++ ++err_ctx: ++ mutex_lock(&dev->struct_mutex); ++ context_close(ext_data.ctx); ++ mutex_unlock(&dev->struct_mutex); ++ return ret; ++} ++ ++int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_i915_gem_context_destroy *args = data; ++ struct drm_i915_file_private *file_priv = file->driver_priv; ++ struct i915_gem_context *ctx; ++ ++ if (args->pad != 0) ++ return -EINVAL; ++ ++ if (!args->ctx_id) ++ return -ENOENT; ++ ++ if (mutex_lock_interruptible(&file_priv->context_idr_lock)) ++ return -EINTR; ++ ++ ctx = idr_remove(&file_priv->context_idr, args->ctx_id); ++ mutex_unlock(&file_priv->context_idr_lock); ++ if (!ctx) ++ return -ENOENT; ++ ++ mutex_lock(&dev->struct_mutex); ++ context_close(ctx); ++ mutex_unlock(&dev->struct_mutex); ++ ++ return 0; ++} ++ ++static int get_sseu(struct i915_gem_context *ctx, ++ struct drm_i915_gem_context_param *args) ++{ ++ struct drm_i915_gem_context_param_sseu user_sseu; ++ struct intel_engine_cs *engine; ++ struct intel_context *ce; ++ ++ if (args->size == 0) ++ goto out; ++ else if (args->size < sizeof(user_sseu)) ++ return -EINVAL; ++ ++ if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value), ++ sizeof(user_sseu))) ++ return -EFAULT; ++ ++ if (user_sseu.flags || user_sseu.rsvd) ++ return -EINVAL; ++ ++ engine = intel_engine_lookup_user(ctx->i915, ++ user_sseu.engine.engine_class, ++ user_sseu.engine.engine_instance); ++ if (!engine) ++ return -EINVAL; ++ ++ ce = intel_context_pin_lock(ctx, engine); /* serialises with set_sseu */ ++ if (IS_ERR(ce)) ++ return PTR_ERR(ce); ++ ++ user_sseu.slice_mask = ce->sseu.slice_mask; ++ user_sseu.subslice_mask = ce->sseu.subslice_mask; ++ user_sseu.min_eus_per_subslice = ce->sseu.min_eus_per_subslice; ++ user_sseu.max_eus_per_subslice = ce->sseu.max_eus_per_subslice; ++ ++ intel_context_pin_unlock(ce); ++ ++ if (copy_to_user(u64_to_user_ptr(args->value), &user_sseu, ++ sizeof(user_sseu))) ++ return -EFAULT; ++ ++out: ++ args->size = sizeof(user_sseu); ++ ++ return 0; ++} ++ ++int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_i915_file_private *file_priv = file->driver_priv; ++ struct drm_i915_gem_context_param *args = data; ++ struct i915_gem_context *ctx; ++ int ret = 0; ++ ++ ctx = i915_gem_context_lookup(file_priv, args->ctx_id); ++ if (!ctx) ++ return -ENOENT; ++ ++ switch (args->param) { ++ case I915_CONTEXT_PARAM_NO_ZEROMAP: ++ args->size = 0; ++ args->value = test_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags); ++ break; ++ ++ case I915_CONTEXT_PARAM_GTT_SIZE: ++ args->size = 0; ++ if (ctx->ppgtt) ++ args->value = ctx->ppgtt->vm.total; ++ else if (to_i915(dev)->mm.aliasing_ppgtt) ++ args->value = to_i915(dev)->mm.aliasing_ppgtt->vm.total; ++ else ++ args->value = to_i915(dev)->ggtt.vm.total; ++ break; ++ ++ case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: ++ args->size = 0; ++ args->value = i915_gem_context_no_error_capture(ctx); ++ break; ++ ++ case I915_CONTEXT_PARAM_BANNABLE: ++ args->size = 0; ++ args->value = i915_gem_context_is_bannable(ctx); ++ break; ++ ++ case I915_CONTEXT_PARAM_RECOVERABLE: ++ args->size = 0; ++ args->value = i915_gem_context_is_recoverable(ctx); ++ break; ++ ++ case I915_CONTEXT_PARAM_PRIORITY: ++ args->size = 0; ++ args->value = ctx->sched.priority >> I915_USER_PRIORITY_SHIFT; ++ break; ++ ++ case I915_CONTEXT_PARAM_SSEU: ++ ret = get_sseu(ctx, args); ++ break; ++ ++ case I915_CONTEXT_PARAM_VM: ++ ret = get_ppgtt(file_priv, ctx, args); ++ break; ++ ++ case I915_CONTEXT_PARAM_BAN_PERIOD: ++ default: ++ ret = -EINVAL; ++ break; ++ } ++ ++ i915_gem_context_put(ctx); ++ return ret; ++} ++ ++int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_i915_file_private *file_priv = file->driver_priv; ++ struct drm_i915_gem_context_param *args = data; ++ struct i915_gem_context *ctx; ++ int ret; ++ ++ ctx = i915_gem_context_lookup(file_priv, args->ctx_id); ++ if (!ctx) ++ return -ENOENT; ++ ++ ret = ctx_setparam(file_priv, ctx, args); ++ ++ i915_gem_context_put(ctx); ++ return ret; ++} ++ ++int i915_gem_context_reset_stats_ioctl(struct drm_device *dev, ++ void *data, struct drm_file *file) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct drm_i915_reset_stats *args = data; ++ struct i915_gem_context *ctx; ++ int ret; ++ ++ if (args->flags || args->pad) ++ return -EINVAL; ++ ++ ret = -ENOENT; ++ rcu_read_lock(); ++ ctx = __i915_gem_context_lookup_rcu(file->driver_priv, args->ctx_id); ++ if (!ctx) ++ goto out; ++ ++ /* ++ * We opt for unserialised reads here. This may result in tearing ++ * in the extremely unlikely event of a GPU hang on this context ++ * as we are querying them. If we need that extra layer of protection, ++ * we should wrap the hangstats with a seqlock. ++ */ ++ ++ if (capable(CAP_SYS_ADMIN)) ++ args->reset_count = i915_reset_count(&dev_priv->gpu_error); ++ else ++ args->reset_count = 0; ++ ++ args->batch_active = atomic_read(&ctx->guilty_count); ++ args->batch_pending = atomic_read(&ctx->active_count); ++ ++ ret = 0; ++out: ++ rcu_read_unlock(); ++ return ret; ++} ++ ++int __i915_gem_context_pin_hw_id(struct i915_gem_context *ctx) ++{ ++ struct drm_i915_private *i915 = ctx->i915; ++ int err = 0; ++ ++ mutex_lock(&i915->contexts.mutex); ++ ++ GEM_BUG_ON(i915_gem_context_is_closed(ctx)); ++ ++ if (list_empty(&ctx->hw_id_link)) { ++ GEM_BUG_ON(atomic_read(&ctx->hw_id_pin_count)); ++ ++ err = assign_hw_id(i915, &ctx->hw_id); ++ if (err) ++ goto out_unlock; ++ ++ list_add_tail(&ctx->hw_id_link, &i915->contexts.hw_id_list); ++ } ++ ++ GEM_BUG_ON(atomic_read(&ctx->hw_id_pin_count) == ~0u); ++ atomic_inc(&ctx->hw_id_pin_count); ++ ++out_unlock: ++ mutex_unlock(&i915->contexts.mutex); ++ return err; ++} ++ ++#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) ++#include "selftests/mock_context.c" ++#include "selftests/i915_gem_context.c" ++#endif ++ ++static void i915_global_gem_context_shrink(void) ++{ ++ kmem_cache_shrink(global.slab_luts); ++} ++ ++static void i915_global_gem_context_exit(void) ++{ ++ kmem_cache_destroy(global.slab_luts); ++} ++ ++static struct i915_global_gem_context global = { { ++ .shrink = i915_global_gem_context_shrink, ++ .exit = i915_global_gem_context_exit, ++} }; ++ ++int __init i915_global_gem_context_init(void) ++{ ++ global.slab_luts = KMEM_CACHE(i915_lut_handle, 0); ++ if (!global.slab_luts) ++ return -ENOMEM; ++ ++ i915_global_register(&global.base); ++ return 0; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_gem_context.h b/drivers/gpu/drm/i915_legacy/i915_gem_context.h +new file mode 100644 +index 000000000000..23dcb01bfd82 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gem_context.h +@@ -0,0 +1,185 @@ ++/* ++ * Copyright © 2016 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#ifndef __I915_GEM_CONTEXT_H__ ++#define __I915_GEM_CONTEXT_H__ ++ ++#include "i915_gem_context_types.h" ++ ++#include "i915_gem.h" ++#include "i915_scheduler.h" ++#include "intel_context.h" ++#include "intel_device_info.h" ++ ++struct drm_device; ++struct drm_file; ++ ++static inline bool i915_gem_context_is_closed(const struct i915_gem_context *ctx) ++{ ++ return test_bit(CONTEXT_CLOSED, &ctx->flags); ++} ++ ++static inline void i915_gem_context_set_closed(struct i915_gem_context *ctx) ++{ ++ GEM_BUG_ON(i915_gem_context_is_closed(ctx)); ++ set_bit(CONTEXT_CLOSED, &ctx->flags); ++} ++ ++static inline bool i915_gem_context_no_error_capture(const struct i915_gem_context *ctx) ++{ ++ return test_bit(UCONTEXT_NO_ERROR_CAPTURE, &ctx->user_flags); ++} ++ ++static inline void i915_gem_context_set_no_error_capture(struct i915_gem_context *ctx) ++{ ++ set_bit(UCONTEXT_NO_ERROR_CAPTURE, &ctx->user_flags); ++} ++ ++static inline void i915_gem_context_clear_no_error_capture(struct i915_gem_context *ctx) ++{ ++ clear_bit(UCONTEXT_NO_ERROR_CAPTURE, &ctx->user_flags); ++} ++ ++static inline bool i915_gem_context_is_bannable(const struct i915_gem_context *ctx) ++{ ++ return test_bit(UCONTEXT_BANNABLE, &ctx->user_flags); ++} ++ ++static inline void i915_gem_context_set_bannable(struct i915_gem_context *ctx) ++{ ++ set_bit(UCONTEXT_BANNABLE, &ctx->user_flags); ++} ++ ++static inline void i915_gem_context_clear_bannable(struct i915_gem_context *ctx) ++{ ++ clear_bit(UCONTEXT_BANNABLE, &ctx->user_flags); ++} ++ ++static inline bool i915_gem_context_is_recoverable(const struct i915_gem_context *ctx) ++{ ++ return test_bit(UCONTEXT_RECOVERABLE, &ctx->user_flags); ++} ++ ++static inline void i915_gem_context_set_recoverable(struct i915_gem_context *ctx) ++{ ++ set_bit(UCONTEXT_RECOVERABLE, &ctx->user_flags); ++} ++ ++static inline void i915_gem_context_clear_recoverable(struct i915_gem_context *ctx) ++{ ++ clear_bit(UCONTEXT_RECOVERABLE, &ctx->user_flags); ++} ++ ++static inline bool i915_gem_context_is_banned(const struct i915_gem_context *ctx) ++{ ++ return test_bit(CONTEXT_BANNED, &ctx->flags); ++} ++ ++static inline void i915_gem_context_set_banned(struct i915_gem_context *ctx) ++{ ++ set_bit(CONTEXT_BANNED, &ctx->flags); ++} ++ ++static inline bool i915_gem_context_force_single_submission(const struct i915_gem_context *ctx) ++{ ++ return test_bit(CONTEXT_FORCE_SINGLE_SUBMISSION, &ctx->flags); ++} ++ ++static inline void i915_gem_context_set_force_single_submission(struct i915_gem_context *ctx) ++{ ++ __set_bit(CONTEXT_FORCE_SINGLE_SUBMISSION, &ctx->flags); ++} ++ ++int __i915_gem_context_pin_hw_id(struct i915_gem_context *ctx); ++static inline int i915_gem_context_pin_hw_id(struct i915_gem_context *ctx) ++{ ++ if (atomic_inc_not_zero(&ctx->hw_id_pin_count)) ++ return 0; ++ ++ return __i915_gem_context_pin_hw_id(ctx); ++} ++ ++static inline void i915_gem_context_unpin_hw_id(struct i915_gem_context *ctx) ++{ ++ GEM_BUG_ON(atomic_read(&ctx->hw_id_pin_count) == 0u); ++ atomic_dec(&ctx->hw_id_pin_count); ++} ++ ++static inline bool i915_gem_context_is_kernel(struct i915_gem_context *ctx) ++{ ++ return !ctx->file_priv; ++} ++ ++/* i915_gem_context.c */ ++int __must_check i915_gem_contexts_init(struct drm_i915_private *dev_priv); ++void i915_gem_contexts_lost(struct drm_i915_private *dev_priv); ++void i915_gem_contexts_fini(struct drm_i915_private *dev_priv); ++ ++int i915_gem_context_open(struct drm_i915_private *i915, ++ struct drm_file *file); ++void i915_gem_context_close(struct drm_file *file); ++ ++int i915_switch_context(struct i915_request *rq); ++int i915_gem_switch_to_kernel_context(struct drm_i915_private *i915, ++ intel_engine_mask_t engine_mask); ++ ++void i915_gem_context_release(struct kref *ctx_ref); ++struct i915_gem_context * ++i915_gem_context_create_gvt(struct drm_device *dev); ++ ++int i915_gem_vm_create_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file); ++int i915_gem_vm_destroy_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file); ++ ++int i915_gem_context_create_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file); ++int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file); ++int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file_priv); ++int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file_priv); ++int i915_gem_context_reset_stats_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file); ++ ++struct i915_gem_context * ++i915_gem_context_create_kernel(struct drm_i915_private *i915, int prio); ++ ++static inline struct i915_gem_context * ++i915_gem_context_get(struct i915_gem_context *ctx) ++{ ++ kref_get(&ctx->ref); ++ return ctx; ++} ++ ++static inline void i915_gem_context_put(struct i915_gem_context *ctx) ++{ ++ kref_put(&ctx->ref, i915_gem_context_release); ++} ++ ++struct i915_lut_handle *i915_lut_handle_alloc(void); ++void i915_lut_handle_free(struct i915_lut_handle *lut); ++ ++#endif /* !__I915_GEM_CONTEXT_H__ */ +diff --git a/drivers/gpu/drm/i915_legacy/i915_gem_context_types.h b/drivers/gpu/drm/i915_legacy/i915_gem_context_types.h +new file mode 100644 +index 000000000000..e2ec58b10fb2 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gem_context_types.h +@@ -0,0 +1,175 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2019 Intel Corporation ++ */ ++ ++#ifndef __I915_GEM_CONTEXT_TYPES_H__ ++#define __I915_GEM_CONTEXT_TYPES_H__ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "i915_scheduler.h" ++#include "intel_context_types.h" ++ ++struct pid; ++ ++struct drm_i915_private; ++struct drm_i915_file_private; ++struct i915_hw_ppgtt; ++struct i915_timeline; ++struct intel_ring; ++ ++/** ++ * struct i915_gem_context - client state ++ * ++ * The struct i915_gem_context represents the combined view of the driver and ++ * logical hardware state for a particular client. ++ */ ++struct i915_gem_context { ++ /** i915: i915 device backpointer */ ++ struct drm_i915_private *i915; ++ ++ /** file_priv: owning file descriptor */ ++ struct drm_i915_file_private *file_priv; ++ ++ struct i915_timeline *timeline; ++ ++ /** ++ * @ppgtt: unique address space (GTT) ++ * ++ * In full-ppgtt mode, each context has its own address space ensuring ++ * complete seperation of one client from all others. ++ * ++ * In other modes, this is a NULL pointer with the expectation that ++ * the caller uses the shared global GTT. ++ */ ++ struct i915_hw_ppgtt *ppgtt; ++ ++ /** ++ * @pid: process id of creator ++ * ++ * Note that who created the context may not be the principle user, ++ * as the context may be shared across a local socket. However, ++ * that should only affect the default context, all contexts created ++ * explicitly by the client are expected to be isolated. ++ */ ++ struct pid *pid; ++ ++ /** ++ * @name: arbitrary name ++ * ++ * A name is constructed for the context from the creator's process ++ * name, pid and user handle in order to uniquely identify the ++ * context in messages. ++ */ ++ const char *name; ++ ++ /** link: place with &drm_i915_private.context_list */ ++ struct list_head link; ++ struct llist_node free_link; ++ ++ /** ++ * @ref: reference count ++ * ++ * A reference to a context is held by both the client who created it ++ * and on each request submitted to the hardware using the request ++ * (to ensure the hardware has access to the state until it has ++ * finished all pending writes). See i915_gem_context_get() and ++ * i915_gem_context_put() for access. ++ */ ++ struct kref ref; ++ ++ /** ++ * @rcu: rcu_head for deferred freeing. ++ */ ++ struct rcu_head rcu; ++ ++ /** ++ * @user_flags: small set of booleans controlled by the user ++ */ ++ unsigned long user_flags; ++#define UCONTEXT_NO_ZEROMAP 0 ++#define UCONTEXT_NO_ERROR_CAPTURE 1 ++#define UCONTEXT_BANNABLE 2 ++#define UCONTEXT_RECOVERABLE 3 ++ ++ /** ++ * @flags: small set of booleans ++ */ ++ unsigned long flags; ++#define CONTEXT_BANNED 0 ++#define CONTEXT_CLOSED 1 ++#define CONTEXT_FORCE_SINGLE_SUBMISSION 2 ++ ++ /** ++ * @hw_id: - unique identifier for the context ++ * ++ * The hardware needs to uniquely identify the context for a few ++ * functions like fault reporting, PASID, scheduling. The ++ * &drm_i915_private.context_hw_ida is used to assign a unqiue ++ * id for the lifetime of the context. ++ * ++ * @hw_id_pin_count: - number of times this context had been pinned ++ * for use (should be, at most, once per engine). ++ * ++ * @hw_id_link: - all contexts with an assigned id are tracked ++ * for possible repossession. ++ */ ++ unsigned int hw_id; ++ atomic_t hw_id_pin_count; ++ struct list_head hw_id_link; ++ ++ struct list_head active_engines; ++ struct mutex mutex; ++ ++ struct i915_sched_attr sched; ++ ++ /** hw_contexts: per-engine logical HW state */ ++ struct rb_root hw_contexts; ++ spinlock_t hw_contexts_lock; ++ ++ /** ring_size: size for allocating the per-engine ring buffer */ ++ u32 ring_size; ++ /** desc_template: invariant fields for the HW context descriptor */ ++ u32 desc_template; ++ ++ /** guilty_count: How many times this context has caused a GPU hang. */ ++ atomic_t guilty_count; ++ /** ++ * @active_count: How many times this context was active during a GPU ++ * hang, but did not cause it. ++ */ ++ atomic_t active_count; ++ ++ /** ++ * @hang_timestamp: The last time(s) this context caused a GPU hang ++ */ ++ unsigned long hang_timestamp[2]; ++#define CONTEXT_FAST_HANG_JIFFIES (120 * HZ) /* 3 hangs within 120s? Banned! */ ++ ++ /** remap_slice: Bitmask of cache lines that need remapping */ ++ u8 remap_slice; ++ ++ /** handles_vma: rbtree to look up our context specific obj/vma for ++ * the user handle. (user handles are per fd, but the binding is ++ * per vm, which may be one per context or shared with the global GTT) ++ */ ++ struct radix_tree_root handles_vma; ++ ++ /** handles_list: reverse list of all the rbtree entries in use for ++ * this context, which allows us to free all the allocations on ++ * context close. ++ */ ++ struct list_head handles_list; ++}; ++ ++#endif /* __I915_GEM_CONTEXT_TYPES_H__ */ +diff --git a/drivers/gpu/drm/i915_legacy/i915_gem_dmabuf.c b/drivers/gpu/drm/i915_legacy/i915_gem_dmabuf.c +new file mode 100644 +index 000000000000..5a101a9462d8 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gem_dmabuf.c +@@ -0,0 +1,337 @@ ++/* ++ * Copyright 2012 Red Hat Inc ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER ++ * DEALINGS IN THE SOFTWARE. ++ * ++ * Authors: ++ * Dave Airlie ++ */ ++ ++#include ++#include ++ ++ ++#include "i915_drv.h" ++ ++static struct drm_i915_gem_object *dma_buf_to_obj(struct dma_buf *buf) ++{ ++ return to_intel_bo(buf->priv); ++} ++ ++static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachment, ++ enum dma_data_direction dir) ++{ ++ struct drm_i915_gem_object *obj = dma_buf_to_obj(attachment->dmabuf); ++ struct sg_table *st; ++ struct scatterlist *src, *dst; ++ int ret, i; ++ ++ ret = i915_gem_object_pin_pages(obj); ++ if (ret) ++ goto err; ++ ++ /* Copy sg so that we make an independent mapping */ ++ st = kmalloc(sizeof(struct sg_table), GFP_KERNEL); ++ if (st == NULL) { ++ ret = -ENOMEM; ++ goto err_unpin_pages; ++ } ++ ++ ret = sg_alloc_table(st, obj->mm.pages->nents, GFP_KERNEL); ++ if (ret) ++ goto err_free; ++ ++ src = obj->mm.pages->sgl; ++ dst = st->sgl; ++ for (i = 0; i < obj->mm.pages->nents; i++) { ++ sg_set_page(dst, sg_page(src), src->length, 0); ++ dst = sg_next(dst); ++ src = sg_next(src); ++ } ++ ++ if (!dma_map_sg(attachment->dev, st->sgl, st->nents, dir)) { ++ ret = -ENOMEM; ++ goto err_free_sg; ++ } ++ ++ return st; ++ ++err_free_sg: ++ sg_free_table(st); ++err_free: ++ kfree(st); ++err_unpin_pages: ++ i915_gem_object_unpin_pages(obj); ++err: ++ return ERR_PTR(ret); ++} ++ ++static void i915_gem_unmap_dma_buf(struct dma_buf_attachment *attachment, ++ struct sg_table *sg, ++ enum dma_data_direction dir) ++{ ++ struct drm_i915_gem_object *obj = dma_buf_to_obj(attachment->dmabuf); ++ ++ dma_unmap_sg(attachment->dev, sg->sgl, sg->nents, dir); ++ sg_free_table(sg); ++ kfree(sg); ++ ++ i915_gem_object_unpin_pages(obj); ++} ++ ++static void *i915_gem_dmabuf_vmap(struct dma_buf *dma_buf) ++{ ++ struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf); ++ ++ return i915_gem_object_pin_map(obj, I915_MAP_WB); ++} ++ ++static void i915_gem_dmabuf_vunmap(struct dma_buf *dma_buf, void *vaddr) ++{ ++ struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf); ++ ++ i915_gem_object_flush_map(obj); ++ i915_gem_object_unpin_map(obj); ++} ++ ++static void *i915_gem_dmabuf_kmap(struct dma_buf *dma_buf, unsigned long page_num) ++{ ++ struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf); ++ struct page *page; ++ ++ if (page_num >= obj->base.size >> PAGE_SHIFT) ++ return NULL; ++ ++ if (!i915_gem_object_has_struct_page(obj)) ++ return NULL; ++ ++ if (i915_gem_object_pin_pages(obj)) ++ return NULL; ++ ++ /* Synchronisation is left to the caller (via .begin_cpu_access()) */ ++ page = i915_gem_object_get_page(obj, page_num); ++ if (IS_ERR(page)) ++ goto err_unpin; ++ ++ return kmap(page); ++ ++err_unpin: ++ i915_gem_object_unpin_pages(obj); ++ return NULL; ++} ++ ++static void i915_gem_dmabuf_kunmap(struct dma_buf *dma_buf, unsigned long page_num, void *addr) ++{ ++ struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf); ++ ++ kunmap(virt_to_page(addr)); ++ i915_gem_object_unpin_pages(obj); ++} ++ ++static int i915_gem_dmabuf_mmap(struct dma_buf *dma_buf, struct vm_area_struct *vma) ++{ ++ struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf); ++ int ret; ++ ++ if (obj->base.size < vma->vm_end - vma->vm_start) ++ return -EINVAL; ++ ++ if (!obj->base.filp) ++ return -ENODEV; ++ ++ ret = call_mmap(obj->base.filp, vma); ++ if (ret) ++ return ret; ++ ++ fput(vma->vm_file); ++ vma->vm_file = get_file(obj->base.filp); ++ ++ return 0; ++} ++ ++static int i915_gem_begin_cpu_access(struct dma_buf *dma_buf, enum dma_data_direction direction) ++{ ++ struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf); ++ struct drm_device *dev = obj->base.dev; ++ bool write = (direction == DMA_BIDIRECTIONAL || direction == DMA_TO_DEVICE); ++ int err; ++ ++ err = i915_gem_object_pin_pages(obj); ++ if (err) ++ return err; ++ ++ err = i915_mutex_lock_interruptible(dev); ++ if (err) ++ goto out; ++ ++ err = i915_gem_object_set_to_cpu_domain(obj, write); ++ mutex_unlock(&dev->struct_mutex); ++ ++out: ++ i915_gem_object_unpin_pages(obj); ++ return err; ++} ++ ++static int i915_gem_end_cpu_access(struct dma_buf *dma_buf, enum dma_data_direction direction) ++{ ++ struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf); ++ struct drm_device *dev = obj->base.dev; ++ int err; ++ ++ err = i915_gem_object_pin_pages(obj); ++ if (err) ++ return err; ++ ++ err = i915_mutex_lock_interruptible(dev); ++ if (err) ++ goto out; ++ ++ err = i915_gem_object_set_to_gtt_domain(obj, false); ++ mutex_unlock(&dev->struct_mutex); ++ ++out: ++ i915_gem_object_unpin_pages(obj); ++ return err; ++} ++ ++static const struct dma_buf_ops i915_dmabuf_ops = { ++ .map_dma_buf = i915_gem_map_dma_buf, ++ .unmap_dma_buf = i915_gem_unmap_dma_buf, ++ .release = drm_gem_dmabuf_release, ++ .map = i915_gem_dmabuf_kmap, ++ .unmap = i915_gem_dmabuf_kunmap, ++ .mmap = i915_gem_dmabuf_mmap, ++ .vmap = i915_gem_dmabuf_vmap, ++ .vunmap = i915_gem_dmabuf_vunmap, ++ .begin_cpu_access = i915_gem_begin_cpu_access, ++ .end_cpu_access = i915_gem_end_cpu_access, ++}; ++ ++struct dma_buf *i915_gem_prime_export(struct drm_device *dev, ++ struct drm_gem_object *gem_obj, int flags) ++{ ++ struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); ++ DEFINE_DMA_BUF_EXPORT_INFO(exp_info); ++ ++ exp_info.ops = &i915_dmabuf_ops; ++ exp_info.size = gem_obj->size; ++ exp_info.flags = flags; ++ exp_info.priv = gem_obj; ++ exp_info.resv = obj->resv; ++ ++ if (obj->ops->dmabuf_export) { ++ int ret = obj->ops->dmabuf_export(obj); ++ if (ret) ++ return ERR_PTR(ret); ++ } ++ ++ return drm_gem_dmabuf_export(dev, &exp_info); ++} ++ ++static int i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj) ++{ ++ struct sg_table *pages; ++ unsigned int sg_page_sizes; ++ ++ pages = dma_buf_map_attachment(obj->base.import_attach, ++ DMA_BIDIRECTIONAL); ++ if (IS_ERR(pages)) ++ return PTR_ERR(pages); ++ ++ sg_page_sizes = i915_sg_page_sizes(pages->sgl); ++ ++ __i915_gem_object_set_pages(obj, pages, sg_page_sizes); ++ ++ return 0; ++} ++ ++static void i915_gem_object_put_pages_dmabuf(struct drm_i915_gem_object *obj, ++ struct sg_table *pages) ++{ ++ dma_buf_unmap_attachment(obj->base.import_attach, pages, ++ DMA_BIDIRECTIONAL); ++} ++ ++static const struct drm_i915_gem_object_ops i915_gem_object_dmabuf_ops = { ++ .get_pages = i915_gem_object_get_pages_dmabuf, ++ .put_pages = i915_gem_object_put_pages_dmabuf, ++}; ++ ++struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev, ++ struct dma_buf *dma_buf) ++{ ++ struct dma_buf_attachment *attach; ++ struct drm_i915_gem_object *obj; ++ int ret; ++ ++ /* is this one of own objects? */ ++ if (dma_buf->ops == &i915_dmabuf_ops) { ++ obj = dma_buf_to_obj(dma_buf); ++ /* is it from our device? */ ++ if (obj->base.dev == dev) { ++ /* ++ * Importing dmabuf exported from out own gem increases ++ * refcount on gem itself instead of f_count of dmabuf. ++ */ ++ return &i915_gem_object_get(obj)->base; ++ } ++ } ++ ++ /* need to attach */ ++ attach = dma_buf_attach(dma_buf, dev->dev); ++ if (IS_ERR(attach)) ++ return ERR_CAST(attach); ++ ++ get_dma_buf(dma_buf); ++ ++ obj = i915_gem_object_alloc(); ++ if (obj == NULL) { ++ ret = -ENOMEM; ++ goto fail_detach; ++ } ++ ++ drm_gem_private_object_init(dev, &obj->base, dma_buf->size); ++ i915_gem_object_init(obj, &i915_gem_object_dmabuf_ops); ++ obj->base.import_attach = attach; ++ obj->resv = dma_buf->resv; ++ ++ /* We use GTT as shorthand for a coherent domain, one that is ++ * neither in the GPU cache nor in the CPU cache, where all ++ * writes are immediately visible in memory. (That's not strictly ++ * true, but it's close! There are internal buffers such as the ++ * write-combined buffer or a delay through the chipset for GTT ++ * writes that do require us to treat GTT as a separate cache domain.) ++ */ ++ obj->read_domains = I915_GEM_DOMAIN_GTT; ++ obj->write_domain = 0; ++ ++ return &obj->base; ++ ++fail_detach: ++ dma_buf_detach(dma_buf, attach); ++ dma_buf_put(dma_buf); ++ ++ return ERR_PTR(ret); ++} ++ ++#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) ++#include "selftests/mock_dmabuf.c" ++#include "selftests/i915_gem_dmabuf.c" ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_gem_evict.c b/drivers/gpu/drm/i915_legacy/i915_gem_evict.c +new file mode 100644 +index 000000000000..060f5903544a +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gem_evict.c +@@ -0,0 +1,444 @@ ++/* ++ * Copyright © 2008-2010 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ * Authors: ++ * Eric Anholt ++ * Chris Wilson ++ * ++ */ ++ ++#include ++ ++#include "i915_drv.h" ++#include "intel_drv.h" ++#include "i915_trace.h" ++ ++I915_SELFTEST_DECLARE(static struct igt_evict_ctl { ++ bool fail_if_busy:1; ++} igt_evict_ctl;) ++ ++static bool ggtt_is_idle(struct drm_i915_private *i915) ++{ ++ return !i915->gt.active_requests; ++} ++ ++static int ggtt_flush(struct drm_i915_private *i915) ++{ ++ int err; ++ ++ /* ++ * Not everything in the GGTT is tracked via vma (otherwise we ++ * could evict as required with minimal stalling) so we are forced ++ * to idle the GPU and explicitly retire outstanding requests in ++ * the hopes that we can then remove contexts and the like only ++ * bound by their active reference. ++ */ ++ err = i915_gem_switch_to_kernel_context(i915, i915->gt.active_engines); ++ if (err) ++ return err; ++ ++ err = i915_gem_wait_for_idle(i915, ++ I915_WAIT_INTERRUPTIBLE | ++ I915_WAIT_LOCKED, ++ MAX_SCHEDULE_TIMEOUT); ++ if (err) ++ return err; ++ ++ GEM_BUG_ON(!ggtt_is_idle(i915)); ++ return 0; ++} ++ ++static bool ++mark_free(struct drm_mm_scan *scan, ++ struct i915_vma *vma, ++ unsigned int flags, ++ struct list_head *unwind) ++{ ++ if (i915_vma_is_pinned(vma)) ++ return false; ++ ++ if (flags & PIN_NONFAULT && i915_vma_has_userfault(vma)) ++ return false; ++ ++ list_add(&vma->evict_link, unwind); ++ return drm_mm_scan_add_block(scan, &vma->node); ++} ++ ++/** ++ * i915_gem_evict_something - Evict vmas to make room for binding a new one ++ * @vm: address space to evict from ++ * @min_size: size of the desired free space ++ * @alignment: alignment constraint of the desired free space ++ * @cache_level: cache_level for the desired space ++ * @start: start (inclusive) of the range from which to evict objects ++ * @end: end (exclusive) of the range from which to evict objects ++ * @flags: additional flags to control the eviction algorithm ++ * ++ * This function will try to evict vmas until a free space satisfying the ++ * requirements is found. Callers must check first whether any such hole exists ++ * already before calling this function. ++ * ++ * This function is used by the object/vma binding code. ++ * ++ * Since this function is only used to free up virtual address space it only ++ * ignores pinned vmas, and not object where the backing storage itself is ++ * pinned. Hence obj->pages_pin_count does not protect against eviction. ++ * ++ * To clarify: This is for freeing up virtual address space, not for freeing ++ * memory in e.g. the shrinker. ++ */ ++int ++i915_gem_evict_something(struct i915_address_space *vm, ++ u64 min_size, u64 alignment, ++ unsigned cache_level, ++ u64 start, u64 end, ++ unsigned flags) ++{ ++ struct drm_i915_private *dev_priv = vm->i915; ++ struct drm_mm_scan scan; ++ struct list_head eviction_list; ++ struct i915_vma *vma, *next; ++ struct drm_mm_node *node; ++ enum drm_mm_insert_mode mode; ++ struct i915_vma *active; ++ int ret; ++ ++ lockdep_assert_held(&vm->i915->drm.struct_mutex); ++ trace_i915_gem_evict(vm, min_size, alignment, flags); ++ ++ /* ++ * The goal is to evict objects and amalgamate space in rough LRU order. ++ * Since both active and inactive objects reside on the same list, ++ * in a mix of creation and last scanned order, as we process the list ++ * we sort it into inactive/active, which keeps the active portion ++ * in a rough MRU order. ++ * ++ * The retirement sequence is thus: ++ * 1. Inactive objects (already retired, random order) ++ * 2. Active objects (will stall on unbinding, oldest scanned first) ++ */ ++ mode = DRM_MM_INSERT_BEST; ++ if (flags & PIN_HIGH) ++ mode = DRM_MM_INSERT_HIGH; ++ if (flags & PIN_MAPPABLE) ++ mode = DRM_MM_INSERT_LOW; ++ drm_mm_scan_init_with_range(&scan, &vm->mm, ++ min_size, alignment, cache_level, ++ start, end, mode); ++ ++ /* ++ * Retire before we search the active list. Although we have ++ * reasonable accuracy in our retirement lists, we may have ++ * a stray pin (preventing eviction) that can only be resolved by ++ * retiring. ++ */ ++ if (!(flags & PIN_NONBLOCK)) ++ i915_retire_requests(dev_priv); ++ ++search_again: ++ active = NULL; ++ INIT_LIST_HEAD(&eviction_list); ++ list_for_each_entry_safe(vma, next, &vm->bound_list, vm_link) { ++ /* ++ * We keep this list in a rough least-recently scanned order ++ * of active elements (inactive elements are cheap to reap). ++ * New entries are added to the end, and we move anything we ++ * scan to the end. The assumption is that the working set ++ * of applications is either steady state (and thanks to the ++ * userspace bo cache it almost always is) or volatile and ++ * frequently replaced after a frame, which are self-evicting! ++ * Given that assumption, the MRU order of the scan list is ++ * fairly static, and keeping it in least-recently scan order ++ * is suitable. ++ * ++ * To notice when we complete one full cycle, we record the ++ * first active element seen, before moving it to the tail. ++ */ ++ if (i915_vma_is_active(vma)) { ++ if (vma == active) { ++ if (flags & PIN_NONBLOCK) ++ break; ++ ++ active = ERR_PTR(-EAGAIN); ++ } ++ ++ if (active != ERR_PTR(-EAGAIN)) { ++ if (!active) ++ active = vma; ++ ++ list_move_tail(&vma->vm_link, &vm->bound_list); ++ continue; ++ } ++ } ++ ++ if (mark_free(&scan, vma, flags, &eviction_list)) ++ goto found; ++ } ++ ++ /* Nothing found, clean up and bail out! */ ++ list_for_each_entry_safe(vma, next, &eviction_list, evict_link) { ++ ret = drm_mm_scan_remove_block(&scan, &vma->node); ++ BUG_ON(ret); ++ } ++ ++ /* ++ * Can we unpin some objects such as idle hw contents, ++ * or pending flips? But since only the GGTT has global entries ++ * such as scanouts, rinbuffers and contexts, we can skip the ++ * purge when inspecting per-process local address spaces. ++ */ ++ if (!i915_is_ggtt(vm) || flags & PIN_NONBLOCK) ++ return -ENOSPC; ++ ++ /* ++ * Not everything in the GGTT is tracked via VMA using ++ * i915_vma_move_to_active(), otherwise we could evict as required ++ * with minimal stalling. Instead we are forced to idle the GPU and ++ * explicitly retire outstanding requests which will then remove ++ * the pinning for active objects such as contexts and ring, ++ * enabling us to evict them on the next iteration. ++ * ++ * To ensure that all user contexts are evictable, we perform ++ * a switch to the perma-pinned kernel context. This all also gives ++ * us a termination condition, when the last retired context is ++ * the kernel's there is no more we can evict. ++ */ ++ if (!ggtt_is_idle(dev_priv)) { ++ if (I915_SELFTEST_ONLY(igt_evict_ctl.fail_if_busy)) ++ return -EBUSY; ++ ++ ret = ggtt_flush(dev_priv); ++ if (ret) ++ return ret; ++ ++ cond_resched(); ++ goto search_again; ++ } ++ ++ /* ++ * If we still have pending pageflip completions, drop ++ * back to userspace to give our workqueues time to ++ * acquire our locks and unpin the old scanouts. ++ */ ++ return intel_has_pending_fb_unpin(dev_priv) ? -EAGAIN : -ENOSPC; ++ ++found: ++ /* drm_mm doesn't allow any other other operations while ++ * scanning, therefore store to-be-evicted objects on a ++ * temporary list and take a reference for all before ++ * calling unbind (which may remove the active reference ++ * of any of our objects, thus corrupting the list). ++ */ ++ list_for_each_entry_safe(vma, next, &eviction_list, evict_link) { ++ if (drm_mm_scan_remove_block(&scan, &vma->node)) ++ __i915_vma_pin(vma); ++ else ++ list_del(&vma->evict_link); ++ } ++ ++ /* Unbinding will emit any required flushes */ ++ ret = 0; ++ list_for_each_entry_safe(vma, next, &eviction_list, evict_link) { ++ __i915_vma_unpin(vma); ++ if (ret == 0) ++ ret = i915_vma_unbind(vma); ++ } ++ ++ while (ret == 0 && (node = drm_mm_scan_color_evict(&scan))) { ++ vma = container_of(node, struct i915_vma, node); ++ ret = i915_vma_unbind(vma); ++ } ++ ++ return ret; ++} ++ ++/** ++ * i915_gem_evict_for_vma - Evict vmas to make room for binding a new one ++ * @vm: address space to evict from ++ * @target: range (and color) to evict for ++ * @flags: additional flags to control the eviction algorithm ++ * ++ * This function will try to evict vmas that overlap the target node. ++ * ++ * To clarify: This is for freeing up virtual address space, not for freeing ++ * memory in e.g. the shrinker. ++ */ ++int i915_gem_evict_for_node(struct i915_address_space *vm, ++ struct drm_mm_node *target, ++ unsigned int flags) ++{ ++ LIST_HEAD(eviction_list); ++ struct drm_mm_node *node; ++ u64 start = target->start; ++ u64 end = start + target->size; ++ struct i915_vma *vma, *next; ++ bool check_color; ++ int ret = 0; ++ ++ lockdep_assert_held(&vm->i915->drm.struct_mutex); ++ GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE)); ++ GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE)); ++ ++ trace_i915_gem_evict_node(vm, target, flags); ++ ++ /* Retire before we search the active list. Although we have ++ * reasonable accuracy in our retirement lists, we may have ++ * a stray pin (preventing eviction) that can only be resolved by ++ * retiring. ++ */ ++ if (!(flags & PIN_NONBLOCK)) ++ i915_retire_requests(vm->i915); ++ ++ check_color = vm->mm.color_adjust; ++ if (check_color) { ++ /* Expand search to cover neighbouring guard pages (or lack!) */ ++ if (start) ++ start -= I915_GTT_PAGE_SIZE; ++ ++ /* Always look at the page afterwards to avoid the end-of-GTT */ ++ end += I915_GTT_PAGE_SIZE; ++ } ++ GEM_BUG_ON(start >= end); ++ ++ drm_mm_for_each_node_in_range(node, &vm->mm, start, end) { ++ /* If we find any non-objects (!vma), we cannot evict them */ ++ if (node->color == I915_COLOR_UNEVICTABLE) { ++ ret = -ENOSPC; ++ break; ++ } ++ ++ GEM_BUG_ON(!node->allocated); ++ vma = container_of(node, typeof(*vma), node); ++ ++ /* If we are using coloring to insert guard pages between ++ * different cache domains within the address space, we have ++ * to check whether the objects on either side of our range ++ * abutt and conflict. If they are in conflict, then we evict ++ * those as well to make room for our guard pages. ++ */ ++ if (check_color) { ++ if (node->start + node->size == target->start) { ++ if (node->color == target->color) ++ continue; ++ } ++ if (node->start == target->start + target->size) { ++ if (node->color == target->color) ++ continue; ++ } ++ } ++ ++ if (flags & PIN_NONBLOCK && ++ (i915_vma_is_pinned(vma) || i915_vma_is_active(vma))) { ++ ret = -ENOSPC; ++ break; ++ } ++ ++ if (flags & PIN_NONFAULT && i915_vma_has_userfault(vma)) { ++ ret = -ENOSPC; ++ break; ++ } ++ ++ /* Overlap of objects in the same batch? */ ++ if (i915_vma_is_pinned(vma)) { ++ ret = -ENOSPC; ++ if (vma->exec_flags && ++ *vma->exec_flags & EXEC_OBJECT_PINNED) ++ ret = -EINVAL; ++ break; ++ } ++ ++ /* Never show fear in the face of dragons! ++ * ++ * We cannot directly remove this node from within this ++ * iterator and as with i915_gem_evict_something() we employ ++ * the vma pin_count in order to prevent the action of ++ * unbinding one vma from freeing (by dropping its active ++ * reference) another in our eviction list. ++ */ ++ __i915_vma_pin(vma); ++ list_add(&vma->evict_link, &eviction_list); ++ } ++ ++ list_for_each_entry_safe(vma, next, &eviction_list, evict_link) { ++ __i915_vma_unpin(vma); ++ if (ret == 0) ++ ret = i915_vma_unbind(vma); ++ } ++ ++ return ret; ++} ++ ++/** ++ * i915_gem_evict_vm - Evict all idle vmas from a vm ++ * @vm: Address space to cleanse ++ * ++ * This function evicts all vmas from a vm. ++ * ++ * This is used by the execbuf code as a last-ditch effort to defragment the ++ * address space. ++ * ++ * To clarify: This is for freeing up virtual address space, not for freeing ++ * memory in e.g. the shrinker. ++ */ ++int i915_gem_evict_vm(struct i915_address_space *vm) ++{ ++ struct list_head eviction_list; ++ struct i915_vma *vma, *next; ++ int ret; ++ ++ lockdep_assert_held(&vm->i915->drm.struct_mutex); ++ trace_i915_gem_evict_vm(vm); ++ ++ /* Switch back to the default context in order to unpin ++ * the existing context objects. However, such objects only ++ * pin themselves inside the global GTT and performing the ++ * switch otherwise is ineffective. ++ */ ++ if (i915_is_ggtt(vm)) { ++ ret = ggtt_flush(vm->i915); ++ if (ret) ++ return ret; ++ } ++ ++ INIT_LIST_HEAD(&eviction_list); ++ mutex_lock(&vm->mutex); ++ list_for_each_entry(vma, &vm->bound_list, vm_link) { ++ if (i915_vma_is_pinned(vma)) ++ continue; ++ ++ __i915_vma_pin(vma); ++ list_add(&vma->evict_link, &eviction_list); ++ } ++ mutex_unlock(&vm->mutex); ++ ++ ret = 0; ++ list_for_each_entry_safe(vma, next, &eviction_list, evict_link) { ++ __i915_vma_unpin(vma); ++ if (ret == 0) ++ ret = i915_vma_unbind(vma); ++ } ++ return ret; ++} ++ ++#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) ++#include "selftests/i915_gem_evict.c" ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_gem_execbuffer.c b/drivers/gpu/drm/i915_legacy/i915_gem_execbuffer.c +new file mode 100644 +index 000000000000..c83d2a195d15 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gem_execbuffer.c +@@ -0,0 +1,2722 @@ ++/* ++ * Copyright © 2008,2010 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ * Authors: ++ * Eric Anholt ++ * Chris Wilson ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++ ++#include ++#include ++ ++#include "i915_drv.h" ++#include "i915_gem_clflush.h" ++#include "i915_trace.h" ++#include "intel_drv.h" ++#include "intel_frontbuffer.h" ++ ++enum { ++ FORCE_CPU_RELOC = 1, ++ FORCE_GTT_RELOC, ++ FORCE_GPU_RELOC, ++#define DBG_FORCE_RELOC 0 /* choose one of the above! */ ++}; ++ ++#define __EXEC_OBJECT_HAS_REF BIT(31) ++#define __EXEC_OBJECT_HAS_PIN BIT(30) ++#define __EXEC_OBJECT_HAS_FENCE BIT(29) ++#define __EXEC_OBJECT_NEEDS_MAP BIT(28) ++#define __EXEC_OBJECT_NEEDS_BIAS BIT(27) ++#define __EXEC_OBJECT_INTERNAL_FLAGS (~0u << 27) /* all of the above */ ++#define __EXEC_OBJECT_RESERVED (__EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_FENCE) ++ ++#define __EXEC_HAS_RELOC BIT(31) ++#define __EXEC_VALIDATED BIT(30) ++#define __EXEC_INTERNAL_FLAGS (~0u << 30) ++#define UPDATE PIN_OFFSET_FIXED ++ ++#define BATCH_OFFSET_BIAS (256*1024) ++ ++#define __I915_EXEC_ILLEGAL_FLAGS \ ++ (__I915_EXEC_UNKNOWN_FLAGS | \ ++ I915_EXEC_CONSTANTS_MASK | \ ++ I915_EXEC_RESOURCE_STREAMER) ++ ++/* Catch emission of unexpected errors for CI! */ ++#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) ++#undef EINVAL ++#define EINVAL ({ \ ++ DRM_DEBUG_DRIVER("EINVAL at %s:%d\n", __func__, __LINE__); \ ++ 22; \ ++}) ++#endif ++ ++/** ++ * DOC: User command execution ++ * ++ * Userspace submits commands to be executed on the GPU as an instruction ++ * stream within a GEM object we call a batchbuffer. This instructions may ++ * refer to other GEM objects containing auxiliary state such as kernels, ++ * samplers, render targets and even secondary batchbuffers. Userspace does ++ * not know where in the GPU memory these objects reside and so before the ++ * batchbuffer is passed to the GPU for execution, those addresses in the ++ * batchbuffer and auxiliary objects are updated. This is known as relocation, ++ * or patching. To try and avoid having to relocate each object on the next ++ * execution, userspace is told the location of those objects in this pass, ++ * but this remains just a hint as the kernel may choose a new location for ++ * any object in the future. ++ * ++ * At the level of talking to the hardware, submitting a batchbuffer for the ++ * GPU to execute is to add content to a buffer from which the HW ++ * command streamer is reading. ++ * ++ * 1. Add a command to load the HW context. For Logical Ring Contexts, i.e. ++ * Execlists, this command is not placed on the same buffer as the ++ * remaining items. ++ * ++ * 2. Add a command to invalidate caches to the buffer. ++ * ++ * 3. Add a batchbuffer start command to the buffer; the start command is ++ * essentially a token together with the GPU address of the batchbuffer ++ * to be executed. ++ * ++ * 4. Add a pipeline flush to the buffer. ++ * ++ * 5. Add a memory write command to the buffer to record when the GPU ++ * is done executing the batchbuffer. The memory write writes the ++ * global sequence number of the request, ``i915_request::global_seqno``; ++ * the i915 driver uses the current value in the register to determine ++ * if the GPU has completed the batchbuffer. ++ * ++ * 6. Add a user interrupt command to the buffer. This command instructs ++ * the GPU to issue an interrupt when the command, pipeline flush and ++ * memory write are completed. ++ * ++ * 7. Inform the hardware of the additional commands added to the buffer ++ * (by updating the tail pointer). ++ * ++ * Processing an execbuf ioctl is conceptually split up into a few phases. ++ * ++ * 1. Validation - Ensure all the pointers, handles and flags are valid. ++ * 2. Reservation - Assign GPU address space for every object ++ * 3. Relocation - Update any addresses to point to the final locations ++ * 4. Serialisation - Order the request with respect to its dependencies ++ * 5. Construction - Construct a request to execute the batchbuffer ++ * 6. Submission (at some point in the future execution) ++ * ++ * Reserving resources for the execbuf is the most complicated phase. We ++ * neither want to have to migrate the object in the address space, nor do ++ * we want to have to update any relocations pointing to this object. Ideally, ++ * we want to leave the object where it is and for all the existing relocations ++ * to match. If the object is given a new address, or if userspace thinks the ++ * object is elsewhere, we have to parse all the relocation entries and update ++ * the addresses. Userspace can set the I915_EXEC_NORELOC flag to hint that ++ * all the target addresses in all of its objects match the value in the ++ * relocation entries and that they all match the presumed offsets given by the ++ * list of execbuffer objects. Using this knowledge, we know that if we haven't ++ * moved any buffers, all the relocation entries are valid and we can skip ++ * the update. (If userspace is wrong, the likely outcome is an impromptu GPU ++ * hang.) The requirement for using I915_EXEC_NO_RELOC are: ++ * ++ * The addresses written in the objects must match the corresponding ++ * reloc.presumed_offset which in turn must match the corresponding ++ * execobject.offset. ++ * ++ * Any render targets written to in the batch must be flagged with ++ * EXEC_OBJECT_WRITE. ++ * ++ * To avoid stalling, execobject.offset should match the current ++ * address of that object within the active context. ++ * ++ * The reservation is done is multiple phases. First we try and keep any ++ * object already bound in its current location - so as long as meets the ++ * constraints imposed by the new execbuffer. Any object left unbound after the ++ * first pass is then fitted into any available idle space. If an object does ++ * not fit, all objects are removed from the reservation and the process rerun ++ * after sorting the objects into a priority order (more difficult to fit ++ * objects are tried first). Failing that, the entire VM is cleared and we try ++ * to fit the execbuf once last time before concluding that it simply will not ++ * fit. ++ * ++ * A small complication to all of this is that we allow userspace not only to ++ * specify an alignment and a size for the object in the address space, but ++ * we also allow userspace to specify the exact offset. This objects are ++ * simpler to place (the location is known a priori) all we have to do is make ++ * sure the space is available. ++ * ++ * Once all the objects are in place, patching up the buried pointers to point ++ * to the final locations is a fairly simple job of walking over the relocation ++ * entry arrays, looking up the right address and rewriting the value into ++ * the object. Simple! ... The relocation entries are stored in user memory ++ * and so to access them we have to copy them into a local buffer. That copy ++ * has to avoid taking any pagefaults as they may lead back to a GEM object ++ * requiring the struct_mutex (i.e. recursive deadlock). So once again we split ++ * the relocation into multiple passes. First we try to do everything within an ++ * atomic context (avoid the pagefaults) which requires that we never wait. If ++ * we detect that we may wait, or if we need to fault, then we have to fallback ++ * to a slower path. The slowpath has to drop the mutex. (Can you hear alarm ++ * bells yet?) Dropping the mutex means that we lose all the state we have ++ * built up so far for the execbuf and we must reset any global data. However, ++ * we do leave the objects pinned in their final locations - which is a ++ * potential issue for concurrent execbufs. Once we have left the mutex, we can ++ * allocate and copy all the relocation entries into a large array at our ++ * leisure, reacquire the mutex, reclaim all the objects and other state and ++ * then proceed to update any incorrect addresses with the objects. ++ * ++ * As we process the relocation entries, we maintain a record of whether the ++ * object is being written to. Using NORELOC, we expect userspace to provide ++ * this information instead. We also check whether we can skip the relocation ++ * by comparing the expected value inside the relocation entry with the target's ++ * final address. If they differ, we have to map the current object and rewrite ++ * the 4 or 8 byte pointer within. ++ * ++ * Serialising an execbuf is quite simple according to the rules of the GEM ++ * ABI. Execution within each context is ordered by the order of submission. ++ * Writes to any GEM object are in order of submission and are exclusive. Reads ++ * from a GEM object are unordered with respect to other reads, but ordered by ++ * writes. A write submitted after a read cannot occur before the read, and ++ * similarly any read submitted after a write cannot occur before the write. ++ * Writes are ordered between engines such that only one write occurs at any ++ * time (completing any reads beforehand) - using semaphores where available ++ * and CPU serialisation otherwise. Other GEM access obey the same rules, any ++ * write (either via mmaps using set-domain, or via pwrite) must flush all GPU ++ * reads before starting, and any read (either using set-domain or pread) must ++ * flush all GPU writes before starting. (Note we only employ a barrier before, ++ * we currently rely on userspace not concurrently starting a new execution ++ * whilst reading or writing to an object. This may be an advantage or not ++ * depending on how much you trust userspace not to shoot themselves in the ++ * foot.) Serialisation may just result in the request being inserted into ++ * a DAG awaiting its turn, but most simple is to wait on the CPU until ++ * all dependencies are resolved. ++ * ++ * After all of that, is just a matter of closing the request and handing it to ++ * the hardware (well, leaving it in a queue to be executed). However, we also ++ * offer the ability for batchbuffers to be run with elevated privileges so ++ * that they access otherwise hidden registers. (Used to adjust L3 cache etc.) ++ * Before any batch is given extra privileges we first must check that it ++ * contains no nefarious instructions, we check that each instruction is from ++ * our whitelist and all registers are also from an allowed list. We first ++ * copy the user's batchbuffer to a shadow (so that the user doesn't have ++ * access to it, either by the CPU or GPU as we scan it) and then parse each ++ * instruction. If everything is ok, we set a flag telling the hardware to run ++ * the batchbuffer in trusted mode, otherwise the ioctl is rejected. ++ */ ++ ++struct i915_execbuffer { ++ struct drm_i915_private *i915; /** i915 backpointer */ ++ struct drm_file *file; /** per-file lookup tables and limits */ ++ struct drm_i915_gem_execbuffer2 *args; /** ioctl parameters */ ++ struct drm_i915_gem_exec_object2 *exec; /** ioctl execobj[] */ ++ struct i915_vma **vma; ++ unsigned int *flags; ++ ++ struct intel_engine_cs *engine; /** engine to queue the request to */ ++ struct i915_gem_context *ctx; /** context for building the request */ ++ struct i915_address_space *vm; /** GTT and vma for the request */ ++ ++ struct i915_request *request; /** our request to build */ ++ struct i915_vma *batch; /** identity of the batch obj/vma */ ++ ++ /** actual size of execobj[] as we may extend it for the cmdparser */ ++ unsigned int buffer_count; ++ ++ /** list of vma not yet bound during reservation phase */ ++ struct list_head unbound; ++ ++ /** list of vma that have execobj.relocation_count */ ++ struct list_head relocs; ++ ++ /** ++ * Track the most recently used object for relocations, as we ++ * frequently have to perform multiple relocations within the same ++ * obj/page ++ */ ++ struct reloc_cache { ++ struct drm_mm_node node; /** temporary GTT binding */ ++ unsigned long vaddr; /** Current kmap address */ ++ unsigned long page; /** Currently mapped page index */ ++ unsigned int gen; /** Cached value of INTEL_GEN */ ++ bool use_64bit_reloc : 1; ++ bool has_llc : 1; ++ bool has_fence : 1; ++ bool needs_unfenced : 1; ++ ++ struct i915_request *rq; ++ u32 *rq_cmd; ++ unsigned int rq_size; ++ } reloc_cache; ++ ++ u64 invalid_flags; /** Set of execobj.flags that are invalid */ ++ u32 context_flags; /** Set of execobj.flags to insert from the ctx */ ++ ++ u32 batch_start_offset; /** Location within object of batch */ ++ u32 batch_len; /** Length of batch within object */ ++ u32 batch_flags; /** Flags composed for emit_bb_start() */ ++ ++ /** ++ * Indicate either the size of the hastable used to resolve ++ * relocation handles, or if negative that we are using a direct ++ * index into the execobj[]. ++ */ ++ int lut_size; ++ struct hlist_head *buckets; /** ht for relocation handles */ ++}; ++ ++#define exec_entry(EB, VMA) (&(EB)->exec[(VMA)->exec_flags - (EB)->flags]) ++ ++/* ++ * Used to convert any address to canonical form. ++ * Starting from gen8, some commands (e.g. STATE_BASE_ADDRESS, ++ * MI_LOAD_REGISTER_MEM and others, see Broadwell PRM Vol2a) require the ++ * addresses to be in a canonical form: ++ * "GraphicsAddress[63:48] are ignored by the HW and assumed to be in correct ++ * canonical form [63:48] == [47]." ++ */ ++#define GEN8_HIGH_ADDRESS_BIT 47 ++static inline u64 gen8_canonical_addr(u64 address) ++{ ++ return sign_extend64(address, GEN8_HIGH_ADDRESS_BIT); ++} ++ ++static inline u64 gen8_noncanonical_addr(u64 address) ++{ ++ return address & GENMASK_ULL(GEN8_HIGH_ADDRESS_BIT, 0); ++} ++ ++static inline bool eb_use_cmdparser(const struct i915_execbuffer *eb) ++{ ++ return intel_engine_needs_cmd_parser(eb->engine) && eb->batch_len; ++} ++ ++static int eb_create(struct i915_execbuffer *eb) ++{ ++ if (!(eb->args->flags & I915_EXEC_HANDLE_LUT)) { ++ unsigned int size = 1 + ilog2(eb->buffer_count); ++ ++ /* ++ * Without a 1:1 association between relocation handles and ++ * the execobject[] index, we instead create a hashtable. ++ * We size it dynamically based on available memory, starting ++ * first with 1:1 assocative hash and scaling back until ++ * the allocation succeeds. ++ * ++ * Later on we use a positive lut_size to indicate we are ++ * using this hashtable, and a negative value to indicate a ++ * direct lookup. ++ */ ++ do { ++ gfp_t flags; ++ ++ /* While we can still reduce the allocation size, don't ++ * raise a warning and allow the allocation to fail. ++ * On the last pass though, we want to try as hard ++ * as possible to perform the allocation and warn ++ * if it fails. ++ */ ++ flags = GFP_KERNEL; ++ if (size > 1) ++ flags |= __GFP_NORETRY | __GFP_NOWARN; ++ ++ eb->buckets = kzalloc(sizeof(struct hlist_head) << size, ++ flags); ++ if (eb->buckets) ++ break; ++ } while (--size); ++ ++ if (unlikely(!size)) ++ return -ENOMEM; ++ ++ eb->lut_size = size; ++ } else { ++ eb->lut_size = -eb->buffer_count; ++ } ++ ++ return 0; ++} ++ ++static bool ++eb_vma_misplaced(const struct drm_i915_gem_exec_object2 *entry, ++ const struct i915_vma *vma, ++ unsigned int flags) ++{ ++ if (vma->node.size < entry->pad_to_size) ++ return true; ++ ++ if (entry->alignment && !IS_ALIGNED(vma->node.start, entry->alignment)) ++ return true; ++ ++ if (flags & EXEC_OBJECT_PINNED && ++ vma->node.start != entry->offset) ++ return true; ++ ++ if (flags & __EXEC_OBJECT_NEEDS_BIAS && ++ vma->node.start < BATCH_OFFSET_BIAS) ++ return true; ++ ++ if (!(flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) && ++ (vma->node.start + vma->node.size - 1) >> 32) ++ return true; ++ ++ if (flags & __EXEC_OBJECT_NEEDS_MAP && ++ !i915_vma_is_map_and_fenceable(vma)) ++ return true; ++ ++ return false; ++} ++ ++static inline bool ++eb_pin_vma(struct i915_execbuffer *eb, ++ const struct drm_i915_gem_exec_object2 *entry, ++ struct i915_vma *vma) ++{ ++ unsigned int exec_flags = *vma->exec_flags; ++ u64 pin_flags; ++ ++ if (vma->node.size) ++ pin_flags = vma->node.start; ++ else ++ pin_flags = entry->offset & PIN_OFFSET_MASK; ++ ++ pin_flags |= PIN_USER | PIN_NOEVICT | PIN_OFFSET_FIXED; ++ if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_GTT)) ++ pin_flags |= PIN_GLOBAL; ++ ++ if (unlikely(i915_vma_pin(vma, 0, 0, pin_flags))) ++ return false; ++ ++ if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_FENCE)) { ++ if (unlikely(i915_vma_pin_fence(vma))) { ++ i915_vma_unpin(vma); ++ return false; ++ } ++ ++ if (vma->fence) ++ exec_flags |= __EXEC_OBJECT_HAS_FENCE; ++ } ++ ++ *vma->exec_flags = exec_flags | __EXEC_OBJECT_HAS_PIN; ++ return !eb_vma_misplaced(entry, vma, exec_flags); ++} ++ ++static inline void __eb_unreserve_vma(struct i915_vma *vma, unsigned int flags) ++{ ++ GEM_BUG_ON(!(flags & __EXEC_OBJECT_HAS_PIN)); ++ ++ if (unlikely(flags & __EXEC_OBJECT_HAS_FENCE)) ++ __i915_vma_unpin_fence(vma); ++ ++ __i915_vma_unpin(vma); ++} ++ ++static inline void ++eb_unreserve_vma(struct i915_vma *vma, unsigned int *flags) ++{ ++ if (!(*flags & __EXEC_OBJECT_HAS_PIN)) ++ return; ++ ++ __eb_unreserve_vma(vma, *flags); ++ *flags &= ~__EXEC_OBJECT_RESERVED; ++} ++ ++static int ++eb_validate_vma(struct i915_execbuffer *eb, ++ struct drm_i915_gem_exec_object2 *entry, ++ struct i915_vma *vma) ++{ ++ if (unlikely(entry->flags & eb->invalid_flags)) ++ return -EINVAL; ++ ++ if (unlikely(entry->alignment && !is_power_of_2(entry->alignment))) ++ return -EINVAL; ++ ++ /* ++ * Offset can be used as input (EXEC_OBJECT_PINNED), reject ++ * any non-page-aligned or non-canonical addresses. ++ */ ++ if (unlikely(entry->flags & EXEC_OBJECT_PINNED && ++ entry->offset != gen8_canonical_addr(entry->offset & I915_GTT_PAGE_MASK))) ++ return -EINVAL; ++ ++ /* pad_to_size was once a reserved field, so sanitize it */ ++ if (entry->flags & EXEC_OBJECT_PAD_TO_SIZE) { ++ if (unlikely(offset_in_page(entry->pad_to_size))) ++ return -EINVAL; ++ } else { ++ entry->pad_to_size = 0; ++ } ++ ++ if (unlikely(vma->exec_flags)) { ++ DRM_DEBUG("Object [handle %d, index %d] appears more than once in object list\n", ++ entry->handle, (int)(entry - eb->exec)); ++ return -EINVAL; ++ } ++ ++ /* ++ * From drm_mm perspective address space is continuous, ++ * so from this point we're always using non-canonical ++ * form internally. ++ */ ++ entry->offset = gen8_noncanonical_addr(entry->offset); ++ ++ if (!eb->reloc_cache.has_fence) { ++ entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE; ++ } else { ++ if ((entry->flags & EXEC_OBJECT_NEEDS_FENCE || ++ eb->reloc_cache.needs_unfenced) && ++ i915_gem_object_is_tiled(vma->obj)) ++ entry->flags |= EXEC_OBJECT_NEEDS_GTT | __EXEC_OBJECT_NEEDS_MAP; ++ } ++ ++ if (!(entry->flags & EXEC_OBJECT_PINNED)) ++ entry->flags |= eb->context_flags; ++ ++ return 0; ++} ++ ++static int ++eb_add_vma(struct i915_execbuffer *eb, ++ unsigned int i, unsigned batch_idx, ++ struct i915_vma *vma) ++{ ++ struct drm_i915_gem_exec_object2 *entry = &eb->exec[i]; ++ int err; ++ ++ GEM_BUG_ON(i915_vma_is_closed(vma)); ++ ++ if (!(eb->args->flags & __EXEC_VALIDATED)) { ++ err = eb_validate_vma(eb, entry, vma); ++ if (unlikely(err)) ++ return err; ++ } ++ ++ if (eb->lut_size > 0) { ++ vma->exec_handle = entry->handle; ++ hlist_add_head(&vma->exec_node, ++ &eb->buckets[hash_32(entry->handle, ++ eb->lut_size)]); ++ } ++ ++ if (entry->relocation_count) ++ list_add_tail(&vma->reloc_link, &eb->relocs); ++ ++ /* ++ * Stash a pointer from the vma to execobj, so we can query its flags, ++ * size, alignment etc as provided by the user. Also we stash a pointer ++ * to the vma inside the execobj so that we can use a direct lookup ++ * to find the right target VMA when doing relocations. ++ */ ++ eb->vma[i] = vma; ++ eb->flags[i] = entry->flags; ++ vma->exec_flags = &eb->flags[i]; ++ ++ /* ++ * SNA is doing fancy tricks with compressing batch buffers, which leads ++ * to negative relocation deltas. Usually that works out ok since the ++ * relocate address is still positive, except when the batch is placed ++ * very low in the GTT. Ensure this doesn't happen. ++ * ++ * Note that actual hangs have only been observed on gen7, but for ++ * paranoia do it everywhere. ++ */ ++ if (i == batch_idx) { ++ if (entry->relocation_count && ++ !(eb->flags[i] & EXEC_OBJECT_PINNED)) ++ eb->flags[i] |= __EXEC_OBJECT_NEEDS_BIAS; ++ if (eb->reloc_cache.has_fence) ++ eb->flags[i] |= EXEC_OBJECT_NEEDS_FENCE; ++ ++ eb->batch = vma; ++ } ++ ++ err = 0; ++ if (eb_pin_vma(eb, entry, vma)) { ++ if (entry->offset != vma->node.start) { ++ entry->offset = vma->node.start | UPDATE; ++ eb->args->flags |= __EXEC_HAS_RELOC; ++ } ++ } else { ++ eb_unreserve_vma(vma, vma->exec_flags); ++ ++ list_add_tail(&vma->exec_link, &eb->unbound); ++ if (drm_mm_node_allocated(&vma->node)) ++ err = i915_vma_unbind(vma); ++ if (unlikely(err)) ++ vma->exec_flags = NULL; ++ } ++ return err; ++} ++ ++static inline int use_cpu_reloc(const struct reloc_cache *cache, ++ const struct drm_i915_gem_object *obj) ++{ ++ if (!i915_gem_object_has_struct_page(obj)) ++ return false; ++ ++ if (DBG_FORCE_RELOC == FORCE_CPU_RELOC) ++ return true; ++ ++ if (DBG_FORCE_RELOC == FORCE_GTT_RELOC) ++ return false; ++ ++ return (cache->has_llc || ++ obj->cache_dirty || ++ obj->cache_level != I915_CACHE_NONE); ++} ++ ++static int eb_reserve_vma(const struct i915_execbuffer *eb, ++ struct i915_vma *vma) ++{ ++ struct drm_i915_gem_exec_object2 *entry = exec_entry(eb, vma); ++ unsigned int exec_flags = *vma->exec_flags; ++ u64 pin_flags; ++ int err; ++ ++ pin_flags = PIN_USER | PIN_NONBLOCK; ++ if (exec_flags & EXEC_OBJECT_NEEDS_GTT) ++ pin_flags |= PIN_GLOBAL; ++ ++ /* ++ * Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset, ++ * limit address to the first 4GBs for unflagged objects. ++ */ ++ if (!(exec_flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS)) ++ pin_flags |= PIN_ZONE_4G; ++ ++ if (exec_flags & __EXEC_OBJECT_NEEDS_MAP) ++ pin_flags |= PIN_MAPPABLE; ++ ++ if (exec_flags & EXEC_OBJECT_PINNED) { ++ pin_flags |= entry->offset | PIN_OFFSET_FIXED; ++ pin_flags &= ~PIN_NONBLOCK; /* force overlapping checks */ ++ } else if (exec_flags & __EXEC_OBJECT_NEEDS_BIAS) { ++ pin_flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS; ++ } ++ ++ err = i915_vma_pin(vma, ++ entry->pad_to_size, entry->alignment, ++ pin_flags); ++ if (err) ++ return err; ++ ++ if (entry->offset != vma->node.start) { ++ entry->offset = vma->node.start | UPDATE; ++ eb->args->flags |= __EXEC_HAS_RELOC; ++ } ++ ++ if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_FENCE)) { ++ err = i915_vma_pin_fence(vma); ++ if (unlikely(err)) { ++ i915_vma_unpin(vma); ++ return err; ++ } ++ ++ if (vma->fence) ++ exec_flags |= __EXEC_OBJECT_HAS_FENCE; ++ } ++ ++ *vma->exec_flags = exec_flags | __EXEC_OBJECT_HAS_PIN; ++ GEM_BUG_ON(eb_vma_misplaced(entry, vma, exec_flags)); ++ ++ return 0; ++} ++ ++static int eb_reserve(struct i915_execbuffer *eb) ++{ ++ const unsigned int count = eb->buffer_count; ++ struct list_head last; ++ struct i915_vma *vma; ++ unsigned int i, pass; ++ int err; ++ ++ /* ++ * Attempt to pin all of the buffers into the GTT. ++ * This is done in 3 phases: ++ * ++ * 1a. Unbind all objects that do not match the GTT constraints for ++ * the execbuffer (fenceable, mappable, alignment etc). ++ * 1b. Increment pin count for already bound objects. ++ * 2. Bind new objects. ++ * 3. Decrement pin count. ++ * ++ * This avoid unnecessary unbinding of later objects in order to make ++ * room for the earlier objects *unless* we need to defragment. ++ */ ++ ++ pass = 0; ++ err = 0; ++ do { ++ list_for_each_entry(vma, &eb->unbound, exec_link) { ++ err = eb_reserve_vma(eb, vma); ++ if (err) ++ break; ++ } ++ if (err != -ENOSPC) ++ return err; ++ ++ /* Resort *all* the objects into priority order */ ++ INIT_LIST_HEAD(&eb->unbound); ++ INIT_LIST_HEAD(&last); ++ for (i = 0; i < count; i++) { ++ unsigned int flags = eb->flags[i]; ++ struct i915_vma *vma = eb->vma[i]; ++ ++ if (flags & EXEC_OBJECT_PINNED && ++ flags & __EXEC_OBJECT_HAS_PIN) ++ continue; ++ ++ eb_unreserve_vma(vma, &eb->flags[i]); ++ ++ if (flags & EXEC_OBJECT_PINNED) ++ /* Pinned must have their slot */ ++ list_add(&vma->exec_link, &eb->unbound); ++ else if (flags & __EXEC_OBJECT_NEEDS_MAP) ++ /* Map require the lowest 256MiB (aperture) */ ++ list_add_tail(&vma->exec_link, &eb->unbound); ++ else if (!(flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS)) ++ /* Prioritise 4GiB region for restricted bo */ ++ list_add(&vma->exec_link, &last); ++ else ++ list_add_tail(&vma->exec_link, &last); ++ } ++ list_splice_tail(&last, &eb->unbound); ++ ++ switch (pass++) { ++ case 0: ++ break; ++ ++ case 1: ++ /* Too fragmented, unbind everything and retry */ ++ err = i915_gem_evict_vm(eb->vm); ++ if (err) ++ return err; ++ break; ++ ++ default: ++ return -ENOSPC; ++ } ++ } while (1); ++} ++ ++static unsigned int eb_batch_index(const struct i915_execbuffer *eb) ++{ ++ if (eb->args->flags & I915_EXEC_BATCH_FIRST) ++ return 0; ++ else ++ return eb->buffer_count - 1; ++} ++ ++static int eb_select_context(struct i915_execbuffer *eb) ++{ ++ struct i915_gem_context *ctx; ++ ++ ctx = i915_gem_context_lookup(eb->file->driver_priv, eb->args->rsvd1); ++ if (unlikely(!ctx)) ++ return -ENOENT; ++ ++ eb->ctx = ctx; ++ if (ctx->ppgtt) { ++ eb->vm = &ctx->ppgtt->vm; ++ eb->invalid_flags |= EXEC_OBJECT_NEEDS_GTT; ++ } else { ++ eb->vm = &eb->i915->ggtt.vm; ++ } ++ ++ eb->context_flags = 0; ++ if (test_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags)) ++ eb->context_flags |= __EXEC_OBJECT_NEEDS_BIAS; ++ ++ return 0; ++} ++ ++static struct i915_request *__eb_wait_for_ring(struct intel_ring *ring) ++{ ++ struct i915_request *rq; ++ ++ /* ++ * Completely unscientific finger-in-the-air estimates for suitable ++ * maximum user request size (to avoid blocking) and then backoff. ++ */ ++ if (intel_ring_update_space(ring) >= PAGE_SIZE) ++ return NULL; ++ ++ /* ++ * Find a request that after waiting upon, there will be at least half ++ * the ring available. The hysteresis allows us to compete for the ++ * shared ring and should mean that we sleep less often prior to ++ * claiming our resources, but not so long that the ring completely ++ * drains before we can submit our next request. ++ */ ++ list_for_each_entry(rq, &ring->request_list, ring_link) { ++ if (__intel_ring_space(rq->postfix, ++ ring->emit, ring->size) > ring->size / 2) ++ break; ++ } ++ if (&rq->ring_link == &ring->request_list) ++ return NULL; /* weird, we will check again later for real */ ++ ++ return i915_request_get(rq); ++} ++ ++static int eb_wait_for_ring(const struct i915_execbuffer *eb) ++{ ++ const struct intel_context *ce; ++ struct i915_request *rq; ++ int ret = 0; ++ ++ /* ++ * Apply a light amount of backpressure to prevent excessive hogs ++ * from blocking waiting for space whilst holding struct_mutex and ++ * keeping all of their resources pinned. ++ */ ++ ++ ce = intel_context_lookup(eb->ctx, eb->engine); ++ if (!ce || !ce->ring) /* first use, assume empty! */ ++ return 0; ++ ++ rq = __eb_wait_for_ring(ce->ring); ++ if (rq) { ++ mutex_unlock(&eb->i915->drm.struct_mutex); ++ ++ if (i915_request_wait(rq, ++ I915_WAIT_INTERRUPTIBLE, ++ MAX_SCHEDULE_TIMEOUT) < 0) ++ ret = -EINTR; ++ ++ i915_request_put(rq); ++ ++ mutex_lock(&eb->i915->drm.struct_mutex); ++ } ++ ++ return ret; ++} ++ ++static int eb_lookup_vmas(struct i915_execbuffer *eb) ++{ ++ struct radix_tree_root *handles_vma = &eb->ctx->handles_vma; ++ struct drm_i915_gem_object *obj; ++ unsigned int i, batch; ++ int err; ++ ++ if (unlikely(i915_gem_context_is_closed(eb->ctx))) ++ return -ENOENT; ++ ++ if (unlikely(i915_gem_context_is_banned(eb->ctx))) ++ return -EIO; ++ ++ INIT_LIST_HEAD(&eb->relocs); ++ INIT_LIST_HEAD(&eb->unbound); ++ ++ batch = eb_batch_index(eb); ++ ++ for (i = 0; i < eb->buffer_count; i++) { ++ u32 handle = eb->exec[i].handle; ++ struct i915_lut_handle *lut; ++ struct i915_vma *vma; ++ ++ vma = radix_tree_lookup(handles_vma, handle); ++ if (likely(vma)) ++ goto add_vma; ++ ++ obj = i915_gem_object_lookup(eb->file, handle); ++ if (unlikely(!obj)) { ++ err = -ENOENT; ++ goto err_vma; ++ } ++ ++ vma = i915_vma_instance(obj, eb->vm, NULL); ++ if (IS_ERR(vma)) { ++ err = PTR_ERR(vma); ++ goto err_obj; ++ } ++ ++ lut = i915_lut_handle_alloc(); ++ if (unlikely(!lut)) { ++ err = -ENOMEM; ++ goto err_obj; ++ } ++ ++ err = radix_tree_insert(handles_vma, handle, vma); ++ if (unlikely(err)) { ++ i915_lut_handle_free(lut); ++ goto err_obj; ++ } ++ ++ /* transfer ref to ctx */ ++ if (!vma->open_count++) ++ i915_vma_reopen(vma); ++ list_add(&lut->obj_link, &obj->lut_list); ++ list_add(&lut->ctx_link, &eb->ctx->handles_list); ++ lut->ctx = eb->ctx; ++ lut->handle = handle; ++ ++add_vma: ++ err = eb_add_vma(eb, i, batch, vma); ++ if (unlikely(err)) ++ goto err_vma; ++ ++ GEM_BUG_ON(vma != eb->vma[i]); ++ GEM_BUG_ON(vma->exec_flags != &eb->flags[i]); ++ GEM_BUG_ON(drm_mm_node_allocated(&vma->node) && ++ eb_vma_misplaced(&eb->exec[i], vma, eb->flags[i])); ++ } ++ ++ eb->args->flags |= __EXEC_VALIDATED; ++ return eb_reserve(eb); ++ ++err_obj: ++ i915_gem_object_put(obj); ++err_vma: ++ eb->vma[i] = NULL; ++ return err; ++} ++ ++static struct i915_vma * ++eb_get_vma(const struct i915_execbuffer *eb, unsigned long handle) ++{ ++ if (eb->lut_size < 0) { ++ if (handle >= -eb->lut_size) ++ return NULL; ++ return eb->vma[handle]; ++ } else { ++ struct hlist_head *head; ++ struct i915_vma *vma; ++ ++ head = &eb->buckets[hash_32(handle, eb->lut_size)]; ++ hlist_for_each_entry(vma, head, exec_node) { ++ if (vma->exec_handle == handle) ++ return vma; ++ } ++ return NULL; ++ } ++} ++ ++static void eb_release_vmas(const struct i915_execbuffer *eb) ++{ ++ const unsigned int count = eb->buffer_count; ++ unsigned int i; ++ ++ for (i = 0; i < count; i++) { ++ struct i915_vma *vma = eb->vma[i]; ++ unsigned int flags = eb->flags[i]; ++ ++ if (!vma) ++ break; ++ ++ GEM_BUG_ON(vma->exec_flags != &eb->flags[i]); ++ vma->exec_flags = NULL; ++ eb->vma[i] = NULL; ++ ++ if (flags & __EXEC_OBJECT_HAS_PIN) ++ __eb_unreserve_vma(vma, flags); ++ ++ if (flags & __EXEC_OBJECT_HAS_REF) ++ i915_vma_put(vma); ++ } ++} ++ ++static void eb_reset_vmas(const struct i915_execbuffer *eb) ++{ ++ eb_release_vmas(eb); ++ if (eb->lut_size > 0) ++ memset(eb->buckets, 0, ++ sizeof(struct hlist_head) << eb->lut_size); ++} ++ ++static void eb_destroy(const struct i915_execbuffer *eb) ++{ ++ GEM_BUG_ON(eb->reloc_cache.rq); ++ ++ if (eb->lut_size > 0) ++ kfree(eb->buckets); ++} ++ ++static inline u64 ++relocation_target(const struct drm_i915_gem_relocation_entry *reloc, ++ const struct i915_vma *target) ++{ ++ return gen8_canonical_addr((int)reloc->delta + target->node.start); ++} ++ ++static void reloc_cache_init(struct reloc_cache *cache, ++ struct drm_i915_private *i915) ++{ ++ cache->page = -1; ++ cache->vaddr = 0; ++ /* Must be a variable in the struct to allow GCC to unroll. */ ++ cache->gen = INTEL_GEN(i915); ++ cache->has_llc = HAS_LLC(i915); ++ cache->use_64bit_reloc = HAS_64BIT_RELOC(i915); ++ cache->has_fence = cache->gen < 4; ++ cache->needs_unfenced = INTEL_INFO(i915)->unfenced_needs_alignment; ++ cache->node.allocated = false; ++ cache->rq = NULL; ++ cache->rq_size = 0; ++} ++ ++static inline void *unmask_page(unsigned long p) ++{ ++ return (void *)(uintptr_t)(p & PAGE_MASK); ++} ++ ++static inline unsigned int unmask_flags(unsigned long p) ++{ ++ return p & ~PAGE_MASK; ++} ++ ++#define KMAP 0x4 /* after CLFLUSH_FLAGS */ ++ ++static inline struct i915_ggtt *cache_to_ggtt(struct reloc_cache *cache) ++{ ++ struct drm_i915_private *i915 = ++ container_of(cache, struct i915_execbuffer, reloc_cache)->i915; ++ return &i915->ggtt; ++} ++ ++static void reloc_gpu_flush(struct reloc_cache *cache) ++{ ++ GEM_BUG_ON(cache->rq_size >= cache->rq->batch->obj->base.size / sizeof(u32)); ++ cache->rq_cmd[cache->rq_size] = MI_BATCH_BUFFER_END; ++ ++ __i915_gem_object_flush_map(cache->rq->batch->obj, 0, cache->rq_size); ++ i915_gem_object_unpin_map(cache->rq->batch->obj); ++ ++ i915_gem_chipset_flush(cache->rq->i915); ++ ++ i915_request_add(cache->rq); ++ cache->rq = NULL; ++} ++ ++static void reloc_cache_reset(struct reloc_cache *cache) ++{ ++ void *vaddr; ++ ++ if (cache->rq) ++ reloc_gpu_flush(cache); ++ ++ if (!cache->vaddr) ++ return; ++ ++ vaddr = unmask_page(cache->vaddr); ++ if (cache->vaddr & KMAP) { ++ if (cache->vaddr & CLFLUSH_AFTER) ++ mb(); ++ ++ kunmap_atomic(vaddr); ++ i915_gem_obj_finish_shmem_access((struct drm_i915_gem_object *)cache->node.mm); ++ } else { ++ wmb(); ++ io_mapping_unmap_atomic((void __iomem *)vaddr); ++ if (cache->node.allocated) { ++ struct i915_ggtt *ggtt = cache_to_ggtt(cache); ++ ++ ggtt->vm.clear_range(&ggtt->vm, ++ cache->node.start, ++ cache->node.size); ++ drm_mm_remove_node(&cache->node); ++ } else { ++ i915_vma_unpin((struct i915_vma *)cache->node.mm); ++ } ++ } ++ ++ cache->vaddr = 0; ++ cache->page = -1; ++} ++ ++static void *reloc_kmap(struct drm_i915_gem_object *obj, ++ struct reloc_cache *cache, ++ unsigned long page) ++{ ++ void *vaddr; ++ ++ if (cache->vaddr) { ++ kunmap_atomic(unmask_page(cache->vaddr)); ++ } else { ++ unsigned int flushes; ++ int err; ++ ++ err = i915_gem_obj_prepare_shmem_write(obj, &flushes); ++ if (err) ++ return ERR_PTR(err); ++ ++ BUILD_BUG_ON(KMAP & CLFLUSH_FLAGS); ++ BUILD_BUG_ON((KMAP | CLFLUSH_FLAGS) & PAGE_MASK); ++ ++ cache->vaddr = flushes | KMAP; ++ cache->node.mm = (void *)obj; ++ if (flushes) ++ mb(); ++ } ++ ++ vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj, page)); ++ cache->vaddr = unmask_flags(cache->vaddr) | (unsigned long)vaddr; ++ cache->page = page; ++ ++ return vaddr; ++} ++ ++static void *reloc_iomap(struct drm_i915_gem_object *obj, ++ struct reloc_cache *cache, ++ unsigned long page) ++{ ++ struct i915_ggtt *ggtt = cache_to_ggtt(cache); ++ unsigned long offset; ++ void *vaddr; ++ ++ if (cache->vaddr) { ++ io_mapping_unmap_atomic((void __force __iomem *) unmask_page(cache->vaddr)); ++ } else { ++ struct i915_vma *vma; ++ int err; ++ ++ if (use_cpu_reloc(cache, obj)) ++ return NULL; ++ ++ err = i915_gem_object_set_to_gtt_domain(obj, true); ++ if (err) ++ return ERR_PTR(err); ++ ++ vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, ++ PIN_MAPPABLE | ++ PIN_NONBLOCK | ++ PIN_NONFAULT); ++ if (IS_ERR(vma)) { ++ memset(&cache->node, 0, sizeof(cache->node)); ++ err = drm_mm_insert_node_in_range ++ (&ggtt->vm.mm, &cache->node, ++ PAGE_SIZE, 0, I915_COLOR_UNEVICTABLE, ++ 0, ggtt->mappable_end, ++ DRM_MM_INSERT_LOW); ++ if (err) /* no inactive aperture space, use cpu reloc */ ++ return NULL; ++ } else { ++ err = i915_vma_put_fence(vma); ++ if (err) { ++ i915_vma_unpin(vma); ++ return ERR_PTR(err); ++ } ++ ++ cache->node.start = vma->node.start; ++ cache->node.mm = (void *)vma; ++ } ++ } ++ ++ offset = cache->node.start; ++ if (cache->node.allocated) { ++ wmb(); ++ ggtt->vm.insert_page(&ggtt->vm, ++ i915_gem_object_get_dma_address(obj, page), ++ offset, I915_CACHE_NONE, 0); ++ } else { ++ offset += page << PAGE_SHIFT; ++ } ++ ++ vaddr = (void __force *)io_mapping_map_atomic_wc(&ggtt->iomap, ++ offset); ++ cache->page = page; ++ cache->vaddr = (unsigned long)vaddr; ++ ++ return vaddr; ++} ++ ++static void *reloc_vaddr(struct drm_i915_gem_object *obj, ++ struct reloc_cache *cache, ++ unsigned long page) ++{ ++ void *vaddr; ++ ++ if (cache->page == page) { ++ vaddr = unmask_page(cache->vaddr); ++ } else { ++ vaddr = NULL; ++ if ((cache->vaddr & KMAP) == 0) ++ vaddr = reloc_iomap(obj, cache, page); ++ if (!vaddr) ++ vaddr = reloc_kmap(obj, cache, page); ++ } ++ ++ return vaddr; ++} ++ ++static void clflush_write32(u32 *addr, u32 value, unsigned int flushes) ++{ ++ if (unlikely(flushes & (CLFLUSH_BEFORE | CLFLUSH_AFTER))) { ++ if (flushes & CLFLUSH_BEFORE) { ++ clflushopt(addr); ++ mb(); ++ } ++ ++ *addr = value; ++ ++ /* ++ * Writes to the same cacheline are serialised by the CPU ++ * (including clflush). On the write path, we only require ++ * that it hits memory in an orderly fashion and place ++ * mb barriers at the start and end of the relocation phase ++ * to ensure ordering of clflush wrt to the system. ++ */ ++ if (flushes & CLFLUSH_AFTER) ++ clflushopt(addr); ++ } else ++ *addr = value; ++} ++ ++static int __reloc_gpu_alloc(struct i915_execbuffer *eb, ++ struct i915_vma *vma, ++ unsigned int len) ++{ ++ struct reloc_cache *cache = &eb->reloc_cache; ++ struct drm_i915_gem_object *obj; ++ struct i915_request *rq; ++ struct i915_vma *batch; ++ u32 *cmd; ++ int err; ++ ++ if (DBG_FORCE_RELOC == FORCE_GPU_RELOC) { ++ obj = vma->obj; ++ if (obj->cache_dirty & ~obj->cache_coherent) ++ i915_gem_clflush_object(obj, 0); ++ obj->write_domain = 0; ++ } ++ ++ GEM_BUG_ON(vma->obj->write_domain & I915_GEM_DOMAIN_CPU); ++ ++ obj = i915_gem_batch_pool_get(&eb->engine->batch_pool, PAGE_SIZE); ++ if (IS_ERR(obj)) ++ return PTR_ERR(obj); ++ ++ cmd = i915_gem_object_pin_map(obj, ++ cache->has_llc ? ++ I915_MAP_FORCE_WB : ++ I915_MAP_FORCE_WC); ++ i915_gem_object_unpin_pages(obj); ++ if (IS_ERR(cmd)) ++ return PTR_ERR(cmd); ++ ++ batch = i915_vma_instance(obj, vma->vm, NULL); ++ if (IS_ERR(batch)) { ++ err = PTR_ERR(batch); ++ goto err_unmap; ++ } ++ ++ err = i915_vma_pin(batch, 0, 0, PIN_USER | PIN_NONBLOCK); ++ if (err) ++ goto err_unmap; ++ ++ rq = i915_request_alloc(eb->engine, eb->ctx); ++ if (IS_ERR(rq)) { ++ err = PTR_ERR(rq); ++ goto err_unpin; ++ } ++ ++ err = i915_request_await_object(rq, vma->obj, true); ++ if (err) ++ goto err_request; ++ ++ err = eb->engine->emit_bb_start(rq, ++ batch->node.start, PAGE_SIZE, ++ cache->gen > 5 ? 0 : I915_DISPATCH_SECURE); ++ if (err) ++ goto err_request; ++ ++ GEM_BUG_ON(!reservation_object_test_signaled_rcu(batch->resv, true)); ++ err = i915_vma_move_to_active(batch, rq, 0); ++ if (err) ++ goto skip_request; ++ ++ err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); ++ if (err) ++ goto skip_request; ++ ++ rq->batch = batch; ++ i915_vma_unpin(batch); ++ ++ cache->rq = rq; ++ cache->rq_cmd = cmd; ++ cache->rq_size = 0; ++ ++ /* Return with batch mapping (cmd) still pinned */ ++ return 0; ++ ++skip_request: ++ i915_request_skip(rq, err); ++err_request: ++ i915_request_add(rq); ++err_unpin: ++ i915_vma_unpin(batch); ++err_unmap: ++ i915_gem_object_unpin_map(obj); ++ return err; ++} ++ ++static u32 *reloc_gpu(struct i915_execbuffer *eb, ++ struct i915_vma *vma, ++ unsigned int len) ++{ ++ struct reloc_cache *cache = &eb->reloc_cache; ++ u32 *cmd; ++ ++ if (cache->rq_size > PAGE_SIZE/sizeof(u32) - (len + 1)) ++ reloc_gpu_flush(cache); ++ ++ if (unlikely(!cache->rq)) { ++ int err; ++ ++ /* If we need to copy for the cmdparser, we will stall anyway */ ++ if (eb_use_cmdparser(eb)) ++ return ERR_PTR(-EWOULDBLOCK); ++ ++ if (!intel_engine_can_store_dword(eb->engine)) ++ return ERR_PTR(-ENODEV); ++ ++ err = __reloc_gpu_alloc(eb, vma, len); ++ if (unlikely(err)) ++ return ERR_PTR(err); ++ } ++ ++ cmd = cache->rq_cmd + cache->rq_size; ++ cache->rq_size += len; ++ ++ return cmd; ++} ++ ++static u64 ++relocate_entry(struct i915_vma *vma, ++ const struct drm_i915_gem_relocation_entry *reloc, ++ struct i915_execbuffer *eb, ++ const struct i915_vma *target) ++{ ++ u64 offset = reloc->offset; ++ u64 target_offset = relocation_target(reloc, target); ++ bool wide = eb->reloc_cache.use_64bit_reloc; ++ void *vaddr; ++ ++ if (!eb->reloc_cache.vaddr && ++ (DBG_FORCE_RELOC == FORCE_GPU_RELOC || ++ !reservation_object_test_signaled_rcu(vma->resv, true))) { ++ const unsigned int gen = eb->reloc_cache.gen; ++ unsigned int len; ++ u32 *batch; ++ u64 addr; ++ ++ if (wide) ++ len = offset & 7 ? 8 : 5; ++ else if (gen >= 4) ++ len = 4; ++ else ++ len = 3; ++ ++ batch = reloc_gpu(eb, vma, len); ++ if (IS_ERR(batch)) ++ goto repeat; ++ ++ addr = gen8_canonical_addr(vma->node.start + offset); ++ if (wide) { ++ if (offset & 7) { ++ *batch++ = MI_STORE_DWORD_IMM_GEN4; ++ *batch++ = lower_32_bits(addr); ++ *batch++ = upper_32_bits(addr); ++ *batch++ = lower_32_bits(target_offset); ++ ++ addr = gen8_canonical_addr(addr + 4); ++ ++ *batch++ = MI_STORE_DWORD_IMM_GEN4; ++ *batch++ = lower_32_bits(addr); ++ *batch++ = upper_32_bits(addr); ++ *batch++ = upper_32_bits(target_offset); ++ } else { ++ *batch++ = (MI_STORE_DWORD_IMM_GEN4 | (1 << 21)) + 1; ++ *batch++ = lower_32_bits(addr); ++ *batch++ = upper_32_bits(addr); ++ *batch++ = lower_32_bits(target_offset); ++ *batch++ = upper_32_bits(target_offset); ++ } ++ } else if (gen >= 6) { ++ *batch++ = MI_STORE_DWORD_IMM_GEN4; ++ *batch++ = 0; ++ *batch++ = addr; ++ *batch++ = target_offset; ++ } else if (gen >= 4) { ++ *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; ++ *batch++ = 0; ++ *batch++ = addr; ++ *batch++ = target_offset; ++ } else { ++ *batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; ++ *batch++ = addr; ++ *batch++ = target_offset; ++ } ++ ++ goto out; ++ } ++ ++repeat: ++ vaddr = reloc_vaddr(vma->obj, &eb->reloc_cache, offset >> PAGE_SHIFT); ++ if (IS_ERR(vaddr)) ++ return PTR_ERR(vaddr); ++ ++ clflush_write32(vaddr + offset_in_page(offset), ++ lower_32_bits(target_offset), ++ eb->reloc_cache.vaddr); ++ ++ if (wide) { ++ offset += sizeof(u32); ++ target_offset >>= 32; ++ wide = false; ++ goto repeat; ++ } ++ ++out: ++ return target->node.start | UPDATE; ++} ++ ++static u64 ++eb_relocate_entry(struct i915_execbuffer *eb, ++ struct i915_vma *vma, ++ const struct drm_i915_gem_relocation_entry *reloc) ++{ ++ struct i915_vma *target; ++ int err; ++ ++ /* we've already hold a reference to all valid objects */ ++ target = eb_get_vma(eb, reloc->target_handle); ++ if (unlikely(!target)) ++ return -ENOENT; ++ ++ /* Validate that the target is in a valid r/w GPU domain */ ++ if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) { ++ DRM_DEBUG("reloc with multiple write domains: " ++ "target %d offset %d " ++ "read %08x write %08x", ++ reloc->target_handle, ++ (int) reloc->offset, ++ reloc->read_domains, ++ reloc->write_domain); ++ return -EINVAL; ++ } ++ if (unlikely((reloc->write_domain | reloc->read_domains) ++ & ~I915_GEM_GPU_DOMAINS)) { ++ DRM_DEBUG("reloc with read/write non-GPU domains: " ++ "target %d offset %d " ++ "read %08x write %08x", ++ reloc->target_handle, ++ (int) reloc->offset, ++ reloc->read_domains, ++ reloc->write_domain); ++ return -EINVAL; ++ } ++ ++ if (reloc->write_domain) { ++ *target->exec_flags |= EXEC_OBJECT_WRITE; ++ ++ /* ++ * Sandybridge PPGTT errata: We need a global gtt mapping ++ * for MI and pipe_control writes because the gpu doesn't ++ * properly redirect them through the ppgtt for non_secure ++ * batchbuffers. ++ */ ++ if (reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION && ++ IS_GEN(eb->i915, 6)) { ++ err = i915_vma_bind(target, target->obj->cache_level, ++ PIN_GLOBAL); ++ if (WARN_ONCE(err, ++ "Unexpected failure to bind target VMA!")) ++ return err; ++ } ++ } ++ ++ /* ++ * If the relocation already has the right value in it, no ++ * more work needs to be done. ++ */ ++ if (!DBG_FORCE_RELOC && ++ gen8_canonical_addr(target->node.start) == reloc->presumed_offset) ++ return 0; ++ ++ /* Check that the relocation address is valid... */ ++ if (unlikely(reloc->offset > ++ vma->size - (eb->reloc_cache.use_64bit_reloc ? 8 : 4))) { ++ DRM_DEBUG("Relocation beyond object bounds: " ++ "target %d offset %d size %d.\n", ++ reloc->target_handle, ++ (int)reloc->offset, ++ (int)vma->size); ++ return -EINVAL; ++ } ++ if (unlikely(reloc->offset & 3)) { ++ DRM_DEBUG("Relocation not 4-byte aligned: " ++ "target %d offset %d.\n", ++ reloc->target_handle, ++ (int)reloc->offset); ++ return -EINVAL; ++ } ++ ++ /* ++ * If we write into the object, we need to force the synchronisation ++ * barrier, either with an asynchronous clflush or if we executed the ++ * patching using the GPU (though that should be serialised by the ++ * timeline). To be completely sure, and since we are required to ++ * do relocations we are already stalling, disable the user's opt ++ * out of our synchronisation. ++ */ ++ *vma->exec_flags &= ~EXEC_OBJECT_ASYNC; ++ ++ /* and update the user's relocation entry */ ++ return relocate_entry(vma, reloc, eb, target); ++} ++ ++static int eb_relocate_vma(struct i915_execbuffer *eb, struct i915_vma *vma) ++{ ++#define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry)) ++ struct drm_i915_gem_relocation_entry stack[N_RELOC(512)]; ++ struct drm_i915_gem_relocation_entry __user *urelocs; ++ const struct drm_i915_gem_exec_object2 *entry = exec_entry(eb, vma); ++ unsigned int remain; ++ ++ urelocs = u64_to_user_ptr(entry->relocs_ptr); ++ remain = entry->relocation_count; ++ if (unlikely(remain > N_RELOC(ULONG_MAX))) ++ return -EINVAL; ++ ++ /* ++ * We must check that the entire relocation array is safe ++ * to read. However, if the array is not writable the user loses ++ * the updated relocation values. ++ */ ++ if (unlikely(!access_ok(urelocs, remain*sizeof(*urelocs)))) ++ return -EFAULT; ++ ++ do { ++ struct drm_i915_gem_relocation_entry *r = stack; ++ unsigned int count = ++ min_t(unsigned int, remain, ARRAY_SIZE(stack)); ++ unsigned int copied; ++ ++ /* ++ * This is the fast path and we cannot handle a pagefault ++ * whilst holding the struct mutex lest the user pass in the ++ * relocations contained within a mmaped bo. For in such a case ++ * we, the page fault handler would call i915_gem_fault() and ++ * we would try to acquire the struct mutex again. Obviously ++ * this is bad and so lockdep complains vehemently. ++ */ ++ pagefault_disable(); ++ copied = __copy_from_user_inatomic(r, urelocs, count * sizeof(r[0])); ++ pagefault_enable(); ++ if (unlikely(copied)) { ++ remain = -EFAULT; ++ goto out; ++ } ++ ++ remain -= count; ++ do { ++ u64 offset = eb_relocate_entry(eb, vma, r); ++ ++ if (likely(offset == 0)) { ++ } else if ((s64)offset < 0) { ++ remain = (int)offset; ++ goto out; ++ } else { ++ /* ++ * Note that reporting an error now ++ * leaves everything in an inconsistent ++ * state as we have *already* changed ++ * the relocation value inside the ++ * object. As we have not changed the ++ * reloc.presumed_offset or will not ++ * change the execobject.offset, on the ++ * call we may not rewrite the value ++ * inside the object, leaving it ++ * dangling and causing a GPU hang. Unless ++ * userspace dynamically rebuilds the ++ * relocations on each execbuf rather than ++ * presume a static tree. ++ * ++ * We did previously check if the relocations ++ * were writable (access_ok), an error now ++ * would be a strange race with mprotect, ++ * having already demonstrated that we ++ * can read from this userspace address. ++ */ ++ offset = gen8_canonical_addr(offset & ~UPDATE); ++ if (unlikely(__put_user(offset, &urelocs[r-stack].presumed_offset))) { ++ remain = -EFAULT; ++ goto out; ++ } ++ } ++ } while (r++, --count); ++ urelocs += ARRAY_SIZE(stack); ++ } while (remain); ++out: ++ reloc_cache_reset(&eb->reloc_cache); ++ return remain; ++} ++ ++static int ++eb_relocate_vma_slow(struct i915_execbuffer *eb, struct i915_vma *vma) ++{ ++ const struct drm_i915_gem_exec_object2 *entry = exec_entry(eb, vma); ++ struct drm_i915_gem_relocation_entry *relocs = ++ u64_to_ptr(typeof(*relocs), entry->relocs_ptr); ++ unsigned int i; ++ int err; ++ ++ for (i = 0; i < entry->relocation_count; i++) { ++ u64 offset = eb_relocate_entry(eb, vma, &relocs[i]); ++ ++ if ((s64)offset < 0) { ++ err = (int)offset; ++ goto err; ++ } ++ } ++ err = 0; ++err: ++ reloc_cache_reset(&eb->reloc_cache); ++ return err; ++} ++ ++static int check_relocations(const struct drm_i915_gem_exec_object2 *entry) ++{ ++ const char __user *addr, *end; ++ unsigned long size; ++ char __maybe_unused c; ++ ++ size = entry->relocation_count; ++ if (size == 0) ++ return 0; ++ ++ if (size > N_RELOC(ULONG_MAX)) ++ return -EINVAL; ++ ++ addr = u64_to_user_ptr(entry->relocs_ptr); ++ size *= sizeof(struct drm_i915_gem_relocation_entry); ++ if (!access_ok(addr, size)) ++ return -EFAULT; ++ ++ end = addr + size; ++ for (; addr < end; addr += PAGE_SIZE) { ++ int err = __get_user(c, addr); ++ if (err) ++ return err; ++ } ++ return __get_user(c, end - 1); ++} ++ ++static int eb_copy_relocations(const struct i915_execbuffer *eb) ++{ ++ const unsigned int count = eb->buffer_count; ++ unsigned int i; ++ int err; ++ ++ for (i = 0; i < count; i++) { ++ const unsigned int nreloc = eb->exec[i].relocation_count; ++ struct drm_i915_gem_relocation_entry __user *urelocs; ++ struct drm_i915_gem_relocation_entry *relocs; ++ unsigned long size; ++ unsigned long copied; ++ ++ if (nreloc == 0) ++ continue; ++ ++ err = check_relocations(&eb->exec[i]); ++ if (err) ++ goto err; ++ ++ urelocs = u64_to_user_ptr(eb->exec[i].relocs_ptr); ++ size = nreloc * sizeof(*relocs); ++ ++ relocs = kvmalloc_array(size, 1, GFP_KERNEL); ++ if (!relocs) { ++ err = -ENOMEM; ++ goto err; ++ } ++ ++ /* copy_from_user is limited to < 4GiB */ ++ copied = 0; ++ do { ++ unsigned int len = ++ min_t(u64, BIT_ULL(31), size - copied); ++ ++ if (__copy_from_user((char *)relocs + copied, ++ (char __user *)urelocs + copied, ++ len)) { ++end_user: ++ user_access_end(); ++end: ++ kvfree(relocs); ++ err = -EFAULT; ++ goto err; ++ } ++ ++ copied += len; ++ } while (copied < size); ++ ++ /* ++ * As we do not update the known relocation offsets after ++ * relocating (due to the complexities in lock handling), ++ * we need to mark them as invalid now so that we force the ++ * relocation processing next time. Just in case the target ++ * object is evicted and then rebound into its old ++ * presumed_offset before the next execbuffer - if that ++ * happened we would make the mistake of assuming that the ++ * relocations were valid. ++ */ ++ if (!user_access_begin(urelocs, size)) ++ goto end; ++ ++ for (copied = 0; copied < nreloc; copied++) ++ unsafe_put_user(-1, ++ &urelocs[copied].presumed_offset, ++ end_user); ++ user_access_end(); ++ ++ eb->exec[i].relocs_ptr = (uintptr_t)relocs; ++ } ++ ++ return 0; ++ ++err: ++ while (i--) { ++ struct drm_i915_gem_relocation_entry *relocs = ++ u64_to_ptr(typeof(*relocs), eb->exec[i].relocs_ptr); ++ if (eb->exec[i].relocation_count) ++ kvfree(relocs); ++ } ++ return err; ++} ++ ++static int eb_prefault_relocations(const struct i915_execbuffer *eb) ++{ ++ const unsigned int count = eb->buffer_count; ++ unsigned int i; ++ ++ if (unlikely(i915_modparams.prefault_disable)) ++ return 0; ++ ++ for (i = 0; i < count; i++) { ++ int err; ++ ++ err = check_relocations(&eb->exec[i]); ++ if (err) ++ return err; ++ } ++ ++ return 0; ++} ++ ++static noinline int eb_relocate_slow(struct i915_execbuffer *eb) ++{ ++ struct drm_device *dev = &eb->i915->drm; ++ bool have_copy = false; ++ struct i915_vma *vma; ++ int err = 0; ++ ++repeat: ++ if (signal_pending(current)) { ++ err = -ERESTARTSYS; ++ goto out; ++ } ++ ++ /* We may process another execbuffer during the unlock... */ ++ eb_reset_vmas(eb); ++ mutex_unlock(&dev->struct_mutex); ++ ++ /* ++ * We take 3 passes through the slowpatch. ++ * ++ * 1 - we try to just prefault all the user relocation entries and ++ * then attempt to reuse the atomic pagefault disabled fast path again. ++ * ++ * 2 - we copy the user entries to a local buffer here outside of the ++ * local and allow ourselves to wait upon any rendering before ++ * relocations ++ * ++ * 3 - we already have a local copy of the relocation entries, but ++ * were interrupted (EAGAIN) whilst waiting for the objects, try again. ++ */ ++ if (!err) { ++ err = eb_prefault_relocations(eb); ++ } else if (!have_copy) { ++ err = eb_copy_relocations(eb); ++ have_copy = err == 0; ++ } else { ++ cond_resched(); ++ err = 0; ++ } ++ if (err) { ++ mutex_lock(&dev->struct_mutex); ++ goto out; ++ } ++ ++ /* A frequent cause for EAGAIN are currently unavailable client pages */ ++ flush_workqueue(eb->i915->mm.userptr_wq); ++ ++ err = i915_mutex_lock_interruptible(dev); ++ if (err) { ++ mutex_lock(&dev->struct_mutex); ++ goto out; ++ } ++ ++ /* reacquire the objects */ ++ err = eb_lookup_vmas(eb); ++ if (err) ++ goto err; ++ ++ GEM_BUG_ON(!eb->batch); ++ ++ list_for_each_entry(vma, &eb->relocs, reloc_link) { ++ if (!have_copy) { ++ pagefault_disable(); ++ err = eb_relocate_vma(eb, vma); ++ pagefault_enable(); ++ if (err) ++ goto repeat; ++ } else { ++ err = eb_relocate_vma_slow(eb, vma); ++ if (err) ++ goto err; ++ } ++ } ++ ++ /* ++ * Leave the user relocations as are, this is the painfully slow path, ++ * and we want to avoid the complication of dropping the lock whilst ++ * having buffers reserved in the aperture and so causing spurious ++ * ENOSPC for random operations. ++ */ ++ ++err: ++ if (err == -EAGAIN) ++ goto repeat; ++ ++out: ++ if (have_copy) { ++ const unsigned int count = eb->buffer_count; ++ unsigned int i; ++ ++ for (i = 0; i < count; i++) { ++ const struct drm_i915_gem_exec_object2 *entry = ++ &eb->exec[i]; ++ struct drm_i915_gem_relocation_entry *relocs; ++ ++ if (!entry->relocation_count) ++ continue; ++ ++ relocs = u64_to_ptr(typeof(*relocs), entry->relocs_ptr); ++ kvfree(relocs); ++ } ++ } ++ ++ return err; ++} ++ ++static int eb_relocate(struct i915_execbuffer *eb) ++{ ++ if (eb_lookup_vmas(eb)) ++ goto slow; ++ ++ /* The objects are in their final locations, apply the relocations. */ ++ if (eb->args->flags & __EXEC_HAS_RELOC) { ++ struct i915_vma *vma; ++ ++ list_for_each_entry(vma, &eb->relocs, reloc_link) { ++ if (eb_relocate_vma(eb, vma)) ++ goto slow; ++ } ++ } ++ ++ return 0; ++ ++slow: ++ return eb_relocate_slow(eb); ++} ++ ++static int eb_move_to_gpu(struct i915_execbuffer *eb) ++{ ++ const unsigned int count = eb->buffer_count; ++ unsigned int i; ++ int err; ++ ++ for (i = 0; i < count; i++) { ++ unsigned int flags = eb->flags[i]; ++ struct i915_vma *vma = eb->vma[i]; ++ struct drm_i915_gem_object *obj = vma->obj; ++ ++ if (flags & EXEC_OBJECT_CAPTURE) { ++ struct i915_capture_list *capture; ++ ++ capture = kmalloc(sizeof(*capture), GFP_KERNEL); ++ if (unlikely(!capture)) ++ return -ENOMEM; ++ ++ capture->next = eb->request->capture_list; ++ capture->vma = eb->vma[i]; ++ eb->request->capture_list = capture; ++ } ++ ++ /* ++ * If the GPU is not _reading_ through the CPU cache, we need ++ * to make sure that any writes (both previous GPU writes from ++ * before a change in snooping levels and normal CPU writes) ++ * caught in that cache are flushed to main memory. ++ * ++ * We want to say ++ * obj->cache_dirty && ++ * !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ) ++ * but gcc's optimiser doesn't handle that as well and emits ++ * two jumps instead of one. Maybe one day... ++ */ ++ if (unlikely(obj->cache_dirty & ~obj->cache_coherent)) { ++ if (i915_gem_clflush_object(obj, 0)) ++ flags &= ~EXEC_OBJECT_ASYNC; ++ } ++ ++ if (flags & EXEC_OBJECT_ASYNC) ++ continue; ++ ++ err = i915_request_await_object ++ (eb->request, obj, flags & EXEC_OBJECT_WRITE); ++ if (err) ++ return err; ++ } ++ ++ for (i = 0; i < count; i++) { ++ unsigned int flags = eb->flags[i]; ++ struct i915_vma *vma = eb->vma[i]; ++ ++ err = i915_vma_move_to_active(vma, eb->request, flags); ++ if (unlikely(err)) { ++ i915_request_skip(eb->request, err); ++ return err; ++ } ++ ++ __eb_unreserve_vma(vma, flags); ++ vma->exec_flags = NULL; ++ ++ if (unlikely(flags & __EXEC_OBJECT_HAS_REF)) ++ i915_vma_put(vma); ++ } ++ eb->exec = NULL; ++ ++ /* Unconditionally flush any chipset caches (for streaming writes). */ ++ i915_gem_chipset_flush(eb->i915); ++ ++ return 0; ++} ++ ++static bool i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec) ++{ ++ if (exec->flags & __I915_EXEC_ILLEGAL_FLAGS) ++ return false; ++ ++ /* Kernel clipping was a DRI1 misfeature */ ++ if (!(exec->flags & I915_EXEC_FENCE_ARRAY)) { ++ if (exec->num_cliprects || exec->cliprects_ptr) ++ return false; ++ } ++ ++ if (exec->DR4 == 0xffffffff) { ++ DRM_DEBUG("UXA submitting garbage DR4, fixing up\n"); ++ exec->DR4 = 0; ++ } ++ if (exec->DR1 || exec->DR4) ++ return false; ++ ++ if ((exec->batch_start_offset | exec->batch_len) & 0x7) ++ return false; ++ ++ return true; ++} ++ ++static int i915_reset_gen7_sol_offsets(struct i915_request *rq) ++{ ++ u32 *cs; ++ int i; ++ ++ if (!IS_GEN(rq->i915, 7) || rq->engine->id != RCS0) { ++ DRM_DEBUG("sol reset is gen7/rcs only\n"); ++ return -EINVAL; ++ } ++ ++ cs = intel_ring_begin(rq, 4 * 2 + 2); ++ if (IS_ERR(cs)) ++ return PTR_ERR(cs); ++ ++ *cs++ = MI_LOAD_REGISTER_IMM(4); ++ for (i = 0; i < 4; i++) { ++ *cs++ = i915_mmio_reg_offset(GEN7_SO_WRITE_OFFSET(i)); ++ *cs++ = 0; ++ } ++ *cs++ = MI_NOOP; ++ intel_ring_advance(rq, cs); ++ ++ return 0; ++} ++ ++static struct i915_vma *eb_parse(struct i915_execbuffer *eb, bool is_master) ++{ ++ struct drm_i915_gem_object *shadow_batch_obj; ++ struct i915_vma *vma; ++ int err; ++ ++ shadow_batch_obj = i915_gem_batch_pool_get(&eb->engine->batch_pool, ++ PAGE_ALIGN(eb->batch_len)); ++ if (IS_ERR(shadow_batch_obj)) ++ return ERR_CAST(shadow_batch_obj); ++ ++ err = intel_engine_cmd_parser(eb->engine, ++ eb->batch->obj, ++ shadow_batch_obj, ++ eb->batch_start_offset, ++ eb->batch_len, ++ is_master); ++ if (err) { ++ if (err == -EACCES) /* unhandled chained batch */ ++ vma = NULL; ++ else ++ vma = ERR_PTR(err); ++ goto out; ++ } ++ ++ vma = i915_gem_object_ggtt_pin(shadow_batch_obj, NULL, 0, 0, 0); ++ if (IS_ERR(vma)) ++ goto out; ++ ++ eb->vma[eb->buffer_count] = i915_vma_get(vma); ++ eb->flags[eb->buffer_count] = ++ __EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_REF; ++ vma->exec_flags = &eb->flags[eb->buffer_count]; ++ eb->buffer_count++; ++ ++out: ++ i915_gem_object_unpin_pages(shadow_batch_obj); ++ return vma; ++} ++ ++static void ++add_to_client(struct i915_request *rq, struct drm_file *file) ++{ ++ rq->file_priv = file->driver_priv; ++ list_add_tail(&rq->client_link, &rq->file_priv->mm.request_list); ++} ++ ++static int eb_submit(struct i915_execbuffer *eb) ++{ ++ int err; ++ ++ err = eb_move_to_gpu(eb); ++ if (err) ++ return err; ++ ++ if (eb->args->flags & I915_EXEC_GEN7_SOL_RESET) { ++ err = i915_reset_gen7_sol_offsets(eb->request); ++ if (err) ++ return err; ++ } ++ ++ /* ++ * After we completed waiting for other engines (using HW semaphores) ++ * then we can signal that this request/batch is ready to run. This ++ * allows us to determine if the batch is still waiting on the GPU ++ * or actually running by checking the breadcrumb. ++ */ ++ if (eb->engine->emit_init_breadcrumb) { ++ err = eb->engine->emit_init_breadcrumb(eb->request); ++ if (err) ++ return err; ++ } ++ ++ err = eb->engine->emit_bb_start(eb->request, ++ eb->batch->node.start + ++ eb->batch_start_offset, ++ eb->batch_len, ++ eb->batch_flags); ++ if (err) ++ return err; ++ ++ return 0; ++} ++ ++/* ++ * Find one BSD ring to dispatch the corresponding BSD command. ++ * The engine index is returned. ++ */ ++static unsigned int ++gen8_dispatch_bsd_engine(struct drm_i915_private *dev_priv, ++ struct drm_file *file) ++{ ++ struct drm_i915_file_private *file_priv = file->driver_priv; ++ ++ /* Check whether the file_priv has already selected one ring. */ ++ if ((int)file_priv->bsd_engine < 0) ++ file_priv->bsd_engine = atomic_fetch_xor(1, ++ &dev_priv->mm.bsd_engine_dispatch_index); ++ ++ return file_priv->bsd_engine; ++} ++ ++#define I915_USER_RINGS (4) ++ ++static const enum intel_engine_id user_ring_map[I915_USER_RINGS + 1] = { ++ [I915_EXEC_DEFAULT] = RCS0, ++ [I915_EXEC_RENDER] = RCS0, ++ [I915_EXEC_BLT] = BCS0, ++ [I915_EXEC_BSD] = VCS0, ++ [I915_EXEC_VEBOX] = VECS0 ++}; ++ ++static struct intel_engine_cs * ++eb_select_engine(struct drm_i915_private *dev_priv, ++ struct drm_file *file, ++ struct drm_i915_gem_execbuffer2 *args) ++{ ++ unsigned int user_ring_id = args->flags & I915_EXEC_RING_MASK; ++ struct intel_engine_cs *engine; ++ ++ if (user_ring_id > I915_USER_RINGS) { ++ DRM_DEBUG("execbuf with unknown ring: %u\n", user_ring_id); ++ return NULL; ++ } ++ ++ if ((user_ring_id != I915_EXEC_BSD) && ++ ((args->flags & I915_EXEC_BSD_MASK) != 0)) { ++ DRM_DEBUG("execbuf with non bsd ring but with invalid " ++ "bsd dispatch flags: %d\n", (int)(args->flags)); ++ return NULL; ++ } ++ ++ if (user_ring_id == I915_EXEC_BSD && HAS_ENGINE(dev_priv, VCS1)) { ++ unsigned int bsd_idx = args->flags & I915_EXEC_BSD_MASK; ++ ++ if (bsd_idx == I915_EXEC_BSD_DEFAULT) { ++ bsd_idx = gen8_dispatch_bsd_engine(dev_priv, file); ++ } else if (bsd_idx >= I915_EXEC_BSD_RING1 && ++ bsd_idx <= I915_EXEC_BSD_RING2) { ++ bsd_idx >>= I915_EXEC_BSD_SHIFT; ++ bsd_idx--; ++ } else { ++ DRM_DEBUG("execbuf with unknown bsd ring: %u\n", ++ bsd_idx); ++ return NULL; ++ } ++ ++ engine = dev_priv->engine[_VCS(bsd_idx)]; ++ } else { ++ engine = dev_priv->engine[user_ring_map[user_ring_id]]; ++ } ++ ++ if (!engine) { ++ DRM_DEBUG("execbuf with invalid ring: %u\n", user_ring_id); ++ return NULL; ++ } ++ ++ return engine; ++} ++ ++static void ++__free_fence_array(struct drm_syncobj **fences, unsigned int n) ++{ ++ while (n--) ++ drm_syncobj_put(ptr_mask_bits(fences[n], 2)); ++ kvfree(fences); ++} ++ ++static struct drm_syncobj ** ++get_fence_array(struct drm_i915_gem_execbuffer2 *args, ++ struct drm_file *file) ++{ ++ const unsigned long nfences = args->num_cliprects; ++ struct drm_i915_gem_exec_fence __user *user; ++ struct drm_syncobj **fences; ++ unsigned long n; ++ int err; ++ ++ if (!(args->flags & I915_EXEC_FENCE_ARRAY)) ++ return NULL; ++ ++ /* Check multiplication overflow for access_ok() and kvmalloc_array() */ ++ BUILD_BUG_ON(sizeof(size_t) > sizeof(unsigned long)); ++ if (nfences > min_t(unsigned long, ++ ULONG_MAX / sizeof(*user), ++ SIZE_MAX / sizeof(*fences))) ++ return ERR_PTR(-EINVAL); ++ ++ user = u64_to_user_ptr(args->cliprects_ptr); ++ if (!access_ok(user, nfences * sizeof(*user))) ++ return ERR_PTR(-EFAULT); ++ ++ fences = kvmalloc_array(nfences, sizeof(*fences), ++ __GFP_NOWARN | GFP_KERNEL); ++ if (!fences) ++ return ERR_PTR(-ENOMEM); ++ ++ for (n = 0; n < nfences; n++) { ++ struct drm_i915_gem_exec_fence fence; ++ struct drm_syncobj *syncobj; ++ ++ if (__copy_from_user(&fence, user++, sizeof(fence))) { ++ err = -EFAULT; ++ goto err; ++ } ++ ++ if (fence.flags & __I915_EXEC_FENCE_UNKNOWN_FLAGS) { ++ err = -EINVAL; ++ goto err; ++ } ++ ++ syncobj = drm_syncobj_find(file, fence.handle); ++ if (!syncobj) { ++ DRM_DEBUG("Invalid syncobj handle provided\n"); ++ err = -ENOENT; ++ goto err; ++ } ++ ++ BUILD_BUG_ON(~(ARCH_KMALLOC_MINALIGN - 1) & ++ ~__I915_EXEC_FENCE_UNKNOWN_FLAGS); ++ ++ fences[n] = ptr_pack_bits(syncobj, fence.flags, 2); ++ } ++ ++ return fences; ++ ++err: ++ __free_fence_array(fences, n); ++ return ERR_PTR(err); ++} ++ ++static void ++put_fence_array(struct drm_i915_gem_execbuffer2 *args, ++ struct drm_syncobj **fences) ++{ ++ if (fences) ++ __free_fence_array(fences, args->num_cliprects); ++} ++ ++static int ++await_fence_array(struct i915_execbuffer *eb, ++ struct drm_syncobj **fences) ++{ ++ const unsigned int nfences = eb->args->num_cliprects; ++ unsigned int n; ++ int err; ++ ++ for (n = 0; n < nfences; n++) { ++ struct drm_syncobj *syncobj; ++ struct dma_fence *fence; ++ unsigned int flags; ++ ++ syncobj = ptr_unpack_bits(fences[n], &flags, 2); ++ if (!(flags & I915_EXEC_FENCE_WAIT)) ++ continue; ++ ++ fence = drm_syncobj_fence_get(syncobj); ++ if (!fence) ++ return -EINVAL; ++ ++ err = i915_request_await_dma_fence(eb->request, fence); ++ dma_fence_put(fence); ++ if (err < 0) ++ return err; ++ } ++ ++ return 0; ++} ++ ++static void ++signal_fence_array(struct i915_execbuffer *eb, ++ struct drm_syncobj **fences) ++{ ++ const unsigned int nfences = eb->args->num_cliprects; ++ struct dma_fence * const fence = &eb->request->fence; ++ unsigned int n; ++ ++ for (n = 0; n < nfences; n++) { ++ struct drm_syncobj *syncobj; ++ unsigned int flags; ++ ++ syncobj = ptr_unpack_bits(fences[n], &flags, 2); ++ if (!(flags & I915_EXEC_FENCE_SIGNAL)) ++ continue; ++ ++ drm_syncobj_replace_fence(syncobj, fence); ++ } ++} ++ ++static int ++i915_gem_do_execbuffer(struct drm_device *dev, ++ struct drm_file *file, ++ struct drm_i915_gem_execbuffer2 *args, ++ struct drm_i915_gem_exec_object2 *exec, ++ struct drm_syncobj **fences) ++{ ++ struct i915_execbuffer eb; ++ struct dma_fence *in_fence = NULL; ++ struct sync_file *out_fence = NULL; ++ intel_wakeref_t wakeref; ++ int out_fence_fd = -1; ++ int err; ++ ++ BUILD_BUG_ON(__EXEC_INTERNAL_FLAGS & ~__I915_EXEC_ILLEGAL_FLAGS); ++ BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS & ++ ~__EXEC_OBJECT_UNKNOWN_FLAGS); ++ ++ eb.i915 = to_i915(dev); ++ eb.file = file; ++ eb.args = args; ++ if (DBG_FORCE_RELOC || !(args->flags & I915_EXEC_NO_RELOC)) ++ args->flags |= __EXEC_HAS_RELOC; ++ ++ eb.exec = exec; ++ eb.vma = (struct i915_vma **)(exec + args->buffer_count + 1); ++ eb.vma[0] = NULL; ++ eb.flags = (unsigned int *)(eb.vma + args->buffer_count + 1); ++ ++ eb.invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS; ++ reloc_cache_init(&eb.reloc_cache, eb.i915); ++ ++ eb.buffer_count = args->buffer_count; ++ eb.batch_start_offset = args->batch_start_offset; ++ eb.batch_len = args->batch_len; ++ ++ eb.batch_flags = 0; ++ if (args->flags & I915_EXEC_SECURE) { ++ if (!drm_is_current_master(file) || !capable(CAP_SYS_ADMIN)) ++ return -EPERM; ++ ++ eb.batch_flags |= I915_DISPATCH_SECURE; ++ } ++ if (args->flags & I915_EXEC_IS_PINNED) ++ eb.batch_flags |= I915_DISPATCH_PINNED; ++ ++ if (args->flags & I915_EXEC_FENCE_IN) { ++ in_fence = sync_file_get_fence(lower_32_bits(args->rsvd2)); ++ if (!in_fence) ++ return -EINVAL; ++ } ++ ++ if (args->flags & I915_EXEC_FENCE_OUT) { ++ out_fence_fd = get_unused_fd_flags(O_CLOEXEC); ++ if (out_fence_fd < 0) { ++ err = out_fence_fd; ++ goto err_in_fence; ++ } ++ } ++ ++ err = eb_create(&eb); ++ if (err) ++ goto err_out_fence; ++ ++ GEM_BUG_ON(!eb.lut_size); ++ ++ err = eb_select_context(&eb); ++ if (unlikely(err)) ++ goto err_destroy; ++ ++ eb.engine = eb_select_engine(eb.i915, file, args); ++ if (!eb.engine) { ++ err = -EINVAL; ++ goto err_engine; ++ } ++ ++ /* ++ * Take a local wakeref for preparing to dispatch the execbuf as ++ * we expect to access the hardware fairly frequently in the ++ * process. Upon first dispatch, we acquire another prolonged ++ * wakeref that we hold until the GPU has been idle for at least ++ * 100ms. ++ */ ++ wakeref = intel_runtime_pm_get(eb.i915); ++ ++ err = i915_mutex_lock_interruptible(dev); ++ if (err) ++ goto err_rpm; ++ ++ err = eb_wait_for_ring(&eb); /* may temporarily drop struct_mutex */ ++ if (unlikely(err)) ++ goto err_unlock; ++ ++ err = eb_relocate(&eb); ++ if (err) { ++ /* ++ * If the user expects the execobject.offset and ++ * reloc.presumed_offset to be an exact match, ++ * as for using NO_RELOC, then we cannot update ++ * the execobject.offset until we have completed ++ * relocation. ++ */ ++ args->flags &= ~__EXEC_HAS_RELOC; ++ goto err_vma; ++ } ++ ++ if (unlikely(*eb.batch->exec_flags & EXEC_OBJECT_WRITE)) { ++ DRM_DEBUG("Attempting to use self-modifying batch buffer\n"); ++ err = -EINVAL; ++ goto err_vma; ++ } ++ if (eb.batch_start_offset > eb.batch->size || ++ eb.batch_len > eb.batch->size - eb.batch_start_offset) { ++ DRM_DEBUG("Attempting to use out-of-bounds batch\n"); ++ err = -EINVAL; ++ goto err_vma; ++ } ++ ++ if (eb_use_cmdparser(&eb)) { ++ struct i915_vma *vma; ++ ++ vma = eb_parse(&eb, drm_is_current_master(file)); ++ if (IS_ERR(vma)) { ++ err = PTR_ERR(vma); ++ goto err_vma; ++ } ++ ++ if (vma) { ++ /* ++ * Batch parsed and accepted: ++ * ++ * Set the DISPATCH_SECURE bit to remove the NON_SECURE ++ * bit from MI_BATCH_BUFFER_START commands issued in ++ * the dispatch_execbuffer implementations. We ++ * specifically don't want that set on batches the ++ * command parser has accepted. ++ */ ++ eb.batch_flags |= I915_DISPATCH_SECURE; ++ eb.batch_start_offset = 0; ++ eb.batch = vma; ++ } ++ } ++ ++ if (eb.batch_len == 0) ++ eb.batch_len = eb.batch->size - eb.batch_start_offset; ++ ++ /* ++ * snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure ++ * batch" bit. Hence we need to pin secure batches into the global gtt. ++ * hsw should have this fixed, but bdw mucks it up again. */ ++ if (eb.batch_flags & I915_DISPATCH_SECURE) { ++ struct i915_vma *vma; ++ ++ /* ++ * So on first glance it looks freaky that we pin the batch here ++ * outside of the reservation loop. But: ++ * - The batch is already pinned into the relevant ppgtt, so we ++ * already have the backing storage fully allocated. ++ * - No other BO uses the global gtt (well contexts, but meh), ++ * so we don't really have issues with multiple objects not ++ * fitting due to fragmentation. ++ * So this is actually safe. ++ */ ++ vma = i915_gem_object_ggtt_pin(eb.batch->obj, NULL, 0, 0, 0); ++ if (IS_ERR(vma)) { ++ err = PTR_ERR(vma); ++ goto err_vma; ++ } ++ ++ eb.batch = vma; ++ } ++ ++ /* All GPU relocation batches must be submitted prior to the user rq */ ++ GEM_BUG_ON(eb.reloc_cache.rq); ++ ++ /* Allocate a request for this batch buffer nice and early. */ ++ eb.request = i915_request_alloc(eb.engine, eb.ctx); ++ if (IS_ERR(eb.request)) { ++ err = PTR_ERR(eb.request); ++ goto err_batch_unpin; ++ } ++ ++ if (in_fence) { ++ err = i915_request_await_dma_fence(eb.request, in_fence); ++ if (err < 0) ++ goto err_request; ++ } ++ ++ if (fences) { ++ err = await_fence_array(&eb, fences); ++ if (err) ++ goto err_request; ++ } ++ ++ if (out_fence_fd != -1) { ++ out_fence = sync_file_create(&eb.request->fence); ++ if (!out_fence) { ++ err = -ENOMEM; ++ goto err_request; ++ } ++ } ++ ++ /* ++ * Whilst this request exists, batch_obj will be on the ++ * active_list, and so will hold the active reference. Only when this ++ * request is retired will the the batch_obj be moved onto the ++ * inactive_list and lose its active reference. Hence we do not need ++ * to explicitly hold another reference here. ++ */ ++ eb.request->batch = eb.batch; ++ ++ trace_i915_request_queue(eb.request, eb.batch_flags); ++ err = eb_submit(&eb); ++err_request: ++ i915_request_add(eb.request); ++ add_to_client(eb.request, file); ++ ++ if (fences) ++ signal_fence_array(&eb, fences); ++ ++ if (out_fence) { ++ if (err == 0) { ++ fd_install(out_fence_fd, out_fence->file); ++ args->rsvd2 &= GENMASK_ULL(31, 0); /* keep in-fence */ ++ args->rsvd2 |= (u64)out_fence_fd << 32; ++ out_fence_fd = -1; ++ } else { ++ fput(out_fence->file); ++ } ++ } ++ ++err_batch_unpin: ++ if (eb.batch_flags & I915_DISPATCH_SECURE) ++ i915_vma_unpin(eb.batch); ++err_vma: ++ if (eb.exec) ++ eb_release_vmas(&eb); ++err_unlock: ++ mutex_unlock(&dev->struct_mutex); ++err_rpm: ++ intel_runtime_pm_put(eb.i915, wakeref); ++err_engine: ++ i915_gem_context_put(eb.ctx); ++err_destroy: ++ eb_destroy(&eb); ++err_out_fence: ++ if (out_fence_fd != -1) ++ put_unused_fd(out_fence_fd); ++err_in_fence: ++ dma_fence_put(in_fence); ++ return err; ++} ++ ++static size_t eb_element_size(void) ++{ ++ return (sizeof(struct drm_i915_gem_exec_object2) + ++ sizeof(struct i915_vma *) + ++ sizeof(unsigned int)); ++} ++ ++static bool check_buffer_count(size_t count) ++{ ++ const size_t sz = eb_element_size(); ++ ++ /* ++ * When using LUT_HANDLE, we impose a limit of INT_MAX for the lookup ++ * array size (see eb_create()). Otherwise, we can accept an array as ++ * large as can be addressed (though use large arrays at your peril)! ++ */ ++ ++ return !(count < 1 || count > INT_MAX || count > SIZE_MAX / sz - 1); ++} ++ ++/* ++ * Legacy execbuffer just creates an exec2 list from the original exec object ++ * list array and passes it to the real function. ++ */ ++int ++i915_gem_execbuffer_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_i915_gem_execbuffer *args = data; ++ struct drm_i915_gem_execbuffer2 exec2; ++ struct drm_i915_gem_exec_object *exec_list = NULL; ++ struct drm_i915_gem_exec_object2 *exec2_list = NULL; ++ const size_t count = args->buffer_count; ++ unsigned int i; ++ int err; ++ ++ if (!check_buffer_count(count)) { ++ DRM_DEBUG("execbuf2 with %zd buffers\n", count); ++ return -EINVAL; ++ } ++ ++ exec2.buffers_ptr = args->buffers_ptr; ++ exec2.buffer_count = args->buffer_count; ++ exec2.batch_start_offset = args->batch_start_offset; ++ exec2.batch_len = args->batch_len; ++ exec2.DR1 = args->DR1; ++ exec2.DR4 = args->DR4; ++ exec2.num_cliprects = args->num_cliprects; ++ exec2.cliprects_ptr = args->cliprects_ptr; ++ exec2.flags = I915_EXEC_RENDER; ++ i915_execbuffer2_set_context_id(exec2, 0); ++ ++ if (!i915_gem_check_execbuffer(&exec2)) ++ return -EINVAL; ++ ++ /* Copy in the exec list from userland */ ++ exec_list = kvmalloc_array(count, sizeof(*exec_list), ++ __GFP_NOWARN | GFP_KERNEL); ++ exec2_list = kvmalloc_array(count + 1, eb_element_size(), ++ __GFP_NOWARN | GFP_KERNEL); ++ if (exec_list == NULL || exec2_list == NULL) { ++ DRM_DEBUG("Failed to allocate exec list for %d buffers\n", ++ args->buffer_count); ++ kvfree(exec_list); ++ kvfree(exec2_list); ++ return -ENOMEM; ++ } ++ err = copy_from_user(exec_list, ++ u64_to_user_ptr(args->buffers_ptr), ++ sizeof(*exec_list) * count); ++ if (err) { ++ DRM_DEBUG("copy %d exec entries failed %d\n", ++ args->buffer_count, err); ++ kvfree(exec_list); ++ kvfree(exec2_list); ++ return -EFAULT; ++ } ++ ++ for (i = 0; i < args->buffer_count; i++) { ++ exec2_list[i].handle = exec_list[i].handle; ++ exec2_list[i].relocation_count = exec_list[i].relocation_count; ++ exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr; ++ exec2_list[i].alignment = exec_list[i].alignment; ++ exec2_list[i].offset = exec_list[i].offset; ++ if (INTEL_GEN(to_i915(dev)) < 4) ++ exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE; ++ else ++ exec2_list[i].flags = 0; ++ } ++ ++ err = i915_gem_do_execbuffer(dev, file, &exec2, exec2_list, NULL); ++ if (exec2.flags & __EXEC_HAS_RELOC) { ++ struct drm_i915_gem_exec_object __user *user_exec_list = ++ u64_to_user_ptr(args->buffers_ptr); ++ ++ /* Copy the new buffer offsets back to the user's exec list. */ ++ for (i = 0; i < args->buffer_count; i++) { ++ if (!(exec2_list[i].offset & UPDATE)) ++ continue; ++ ++ exec2_list[i].offset = ++ gen8_canonical_addr(exec2_list[i].offset & PIN_OFFSET_MASK); ++ exec2_list[i].offset &= PIN_OFFSET_MASK; ++ if (__copy_to_user(&user_exec_list[i].offset, ++ &exec2_list[i].offset, ++ sizeof(user_exec_list[i].offset))) ++ break; ++ } ++ } ++ ++ kvfree(exec_list); ++ kvfree(exec2_list); ++ return err; ++} ++ ++int ++i915_gem_execbuffer2_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_i915_gem_execbuffer2 *args = data; ++ struct drm_i915_gem_exec_object2 *exec2_list; ++ struct drm_syncobj **fences = NULL; ++ const size_t count = args->buffer_count; ++ int err; ++ ++ if (!check_buffer_count(count)) { ++ DRM_DEBUG("execbuf2 with %zd buffers\n", count); ++ return -EINVAL; ++ } ++ ++ if (!i915_gem_check_execbuffer(args)) ++ return -EINVAL; ++ ++ /* Allocate an extra slot for use by the command parser */ ++ exec2_list = kvmalloc_array(count + 1, eb_element_size(), ++ __GFP_NOWARN | GFP_KERNEL); ++ if (exec2_list == NULL) { ++ DRM_DEBUG("Failed to allocate exec list for %zd buffers\n", ++ count); ++ return -ENOMEM; ++ } ++ if (copy_from_user(exec2_list, ++ u64_to_user_ptr(args->buffers_ptr), ++ sizeof(*exec2_list) * count)) { ++ DRM_DEBUG("copy %zd exec entries failed\n", count); ++ kvfree(exec2_list); ++ return -EFAULT; ++ } ++ ++ if (args->flags & I915_EXEC_FENCE_ARRAY) { ++ fences = get_fence_array(args, file); ++ if (IS_ERR(fences)) { ++ kvfree(exec2_list); ++ return PTR_ERR(fences); ++ } ++ } ++ ++ err = i915_gem_do_execbuffer(dev, file, args, exec2_list, fences); ++ ++ /* ++ * Now that we have begun execution of the batchbuffer, we ignore ++ * any new error after this point. Also given that we have already ++ * updated the associated relocations, we try to write out the current ++ * object locations irrespective of any error. ++ */ ++ if (args->flags & __EXEC_HAS_RELOC) { ++ struct drm_i915_gem_exec_object2 __user *user_exec_list = ++ u64_to_user_ptr(args->buffers_ptr); ++ unsigned int i; ++ ++ /* Copy the new buffer offsets back to the user's exec list. */ ++ /* ++ * Note: count * sizeof(*user_exec_list) does not overflow, ++ * because we checked 'count' in check_buffer_count(). ++ * ++ * And this range already got effectively checked earlier ++ * when we did the "copy_from_user()" above. ++ */ ++ if (!user_access_begin(user_exec_list, count * sizeof(*user_exec_list))) ++ goto end; ++ ++ for (i = 0; i < args->buffer_count; i++) { ++ if (!(exec2_list[i].offset & UPDATE)) ++ continue; ++ ++ exec2_list[i].offset = ++ gen8_canonical_addr(exec2_list[i].offset & PIN_OFFSET_MASK); ++ unsafe_put_user(exec2_list[i].offset, ++ &user_exec_list[i].offset, ++ end_user); ++ } ++end_user: ++ user_access_end(); ++end:; ++ } ++ ++ args->flags &= ~__I915_EXEC_UNKNOWN_FLAGS; ++ put_fence_array(args, fences); ++ kvfree(exec2_list); ++ return err; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_gem_fence_reg.c b/drivers/gpu/drm/i915_legacy/i915_gem_fence_reg.c +new file mode 100644 +index 000000000000..3084f52e3372 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gem_fence_reg.c +@@ -0,0 +1,785 @@ ++/* ++ * Copyright © 2008-2015 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ */ ++ ++#include ++#include "i915_drv.h" ++ ++/** ++ * DOC: fence register handling ++ * ++ * Important to avoid confusions: "fences" in the i915 driver are not execution ++ * fences used to track command completion but hardware detiler objects which ++ * wrap a given range of the global GTT. Each platform has only a fairly limited ++ * set of these objects. ++ * ++ * Fences are used to detile GTT memory mappings. They're also connected to the ++ * hardware frontbuffer render tracking and hence interact with frontbuffer ++ * compression. Furthermore on older platforms fences are required for tiled ++ * objects used by the display engine. They can also be used by the render ++ * engine - they're required for blitter commands and are optional for render ++ * commands. But on gen4+ both display (with the exception of fbc) and rendering ++ * have their own tiling state bits and don't need fences. ++ * ++ * Also note that fences only support X and Y tiling and hence can't be used for ++ * the fancier new tiling formats like W, Ys and Yf. ++ * ++ * Finally note that because fences are such a restricted resource they're ++ * dynamically associated with objects. Furthermore fence state is committed to ++ * the hardware lazily to avoid unnecessary stalls on gen2/3. Therefore code must ++ * explicitly call i915_gem_object_get_fence() to synchronize fencing status ++ * for cpu access. Also note that some code wants an unfenced view, for those ++ * cases the fence can be removed forcefully with i915_gem_object_put_fence(). ++ * ++ * Internally these functions will synchronize with userspace access by removing ++ * CPU ptes into GTT mmaps (not the GTT ptes themselves) as needed. ++ */ ++ ++#define pipelined 0 ++ ++static void i965_write_fence_reg(struct drm_i915_fence_reg *fence, ++ struct i915_vma *vma) ++{ ++ i915_reg_t fence_reg_lo, fence_reg_hi; ++ int fence_pitch_shift; ++ u64 val; ++ ++ if (INTEL_GEN(fence->i915) >= 6) { ++ fence_reg_lo = FENCE_REG_GEN6_LO(fence->id); ++ fence_reg_hi = FENCE_REG_GEN6_HI(fence->id); ++ fence_pitch_shift = GEN6_FENCE_PITCH_SHIFT; ++ ++ } else { ++ fence_reg_lo = FENCE_REG_965_LO(fence->id); ++ fence_reg_hi = FENCE_REG_965_HI(fence->id); ++ fence_pitch_shift = I965_FENCE_PITCH_SHIFT; ++ } ++ ++ val = 0; ++ if (vma) { ++ unsigned int stride = i915_gem_object_get_stride(vma->obj); ++ ++ GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma)); ++ GEM_BUG_ON(!IS_ALIGNED(vma->node.start, I965_FENCE_PAGE)); ++ GEM_BUG_ON(!IS_ALIGNED(vma->fence_size, I965_FENCE_PAGE)); ++ GEM_BUG_ON(!IS_ALIGNED(stride, 128)); ++ ++ val = (vma->node.start + vma->fence_size - I965_FENCE_PAGE) << 32; ++ val |= vma->node.start; ++ val |= (u64)((stride / 128) - 1) << fence_pitch_shift; ++ if (i915_gem_object_get_tiling(vma->obj) == I915_TILING_Y) ++ val |= BIT(I965_FENCE_TILING_Y_SHIFT); ++ val |= I965_FENCE_REG_VALID; ++ } ++ ++ if (!pipelined) { ++ struct drm_i915_private *dev_priv = fence->i915; ++ ++ /* To w/a incoherency with non-atomic 64-bit register updates, ++ * we split the 64-bit update into two 32-bit writes. In order ++ * for a partial fence not to be evaluated between writes, we ++ * precede the update with write to turn off the fence register, ++ * and only enable the fence as the last step. ++ * ++ * For extra levels of paranoia, we make sure each step lands ++ * before applying the next step. ++ */ ++ I915_WRITE(fence_reg_lo, 0); ++ POSTING_READ(fence_reg_lo); ++ ++ I915_WRITE(fence_reg_hi, upper_32_bits(val)); ++ I915_WRITE(fence_reg_lo, lower_32_bits(val)); ++ POSTING_READ(fence_reg_lo); ++ } ++} ++ ++static void i915_write_fence_reg(struct drm_i915_fence_reg *fence, ++ struct i915_vma *vma) ++{ ++ u32 val; ++ ++ val = 0; ++ if (vma) { ++ unsigned int tiling = i915_gem_object_get_tiling(vma->obj); ++ bool is_y_tiled = tiling == I915_TILING_Y; ++ unsigned int stride = i915_gem_object_get_stride(vma->obj); ++ ++ GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma)); ++ GEM_BUG_ON(vma->node.start & ~I915_FENCE_START_MASK); ++ GEM_BUG_ON(!is_power_of_2(vma->fence_size)); ++ GEM_BUG_ON(!IS_ALIGNED(vma->node.start, vma->fence_size)); ++ ++ if (is_y_tiled && HAS_128_BYTE_Y_TILING(fence->i915)) ++ stride /= 128; ++ else ++ stride /= 512; ++ GEM_BUG_ON(!is_power_of_2(stride)); ++ ++ val = vma->node.start; ++ if (is_y_tiled) ++ val |= BIT(I830_FENCE_TILING_Y_SHIFT); ++ val |= I915_FENCE_SIZE_BITS(vma->fence_size); ++ val |= ilog2(stride) << I830_FENCE_PITCH_SHIFT; ++ ++ val |= I830_FENCE_REG_VALID; ++ } ++ ++ if (!pipelined) { ++ struct drm_i915_private *dev_priv = fence->i915; ++ i915_reg_t reg = FENCE_REG(fence->id); ++ ++ I915_WRITE(reg, val); ++ POSTING_READ(reg); ++ } ++} ++ ++static void i830_write_fence_reg(struct drm_i915_fence_reg *fence, ++ struct i915_vma *vma) ++{ ++ u32 val; ++ ++ val = 0; ++ if (vma) { ++ unsigned int stride = i915_gem_object_get_stride(vma->obj); ++ ++ GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma)); ++ GEM_BUG_ON(vma->node.start & ~I830_FENCE_START_MASK); ++ GEM_BUG_ON(!is_power_of_2(vma->fence_size)); ++ GEM_BUG_ON(!is_power_of_2(stride / 128)); ++ GEM_BUG_ON(!IS_ALIGNED(vma->node.start, vma->fence_size)); ++ ++ val = vma->node.start; ++ if (i915_gem_object_get_tiling(vma->obj) == I915_TILING_Y) ++ val |= BIT(I830_FENCE_TILING_Y_SHIFT); ++ val |= I830_FENCE_SIZE_BITS(vma->fence_size); ++ val |= ilog2(stride / 128) << I830_FENCE_PITCH_SHIFT; ++ val |= I830_FENCE_REG_VALID; ++ } ++ ++ if (!pipelined) { ++ struct drm_i915_private *dev_priv = fence->i915; ++ i915_reg_t reg = FENCE_REG(fence->id); ++ ++ I915_WRITE(reg, val); ++ POSTING_READ(reg); ++ } ++} ++ ++static void fence_write(struct drm_i915_fence_reg *fence, ++ struct i915_vma *vma) ++{ ++ /* Previous access through the fence register is marshalled by ++ * the mb() inside the fault handlers (i915_gem_release_mmaps) ++ * and explicitly managed for internal users. ++ */ ++ ++ if (IS_GEN(fence->i915, 2)) ++ i830_write_fence_reg(fence, vma); ++ else if (IS_GEN(fence->i915, 3)) ++ i915_write_fence_reg(fence, vma); ++ else ++ i965_write_fence_reg(fence, vma); ++ ++ /* Access through the fenced region afterwards is ++ * ordered by the posting reads whilst writing the registers. ++ */ ++ ++ fence->dirty = false; ++} ++ ++static int fence_update(struct drm_i915_fence_reg *fence, ++ struct i915_vma *vma) ++{ ++ intel_wakeref_t wakeref; ++ struct i915_vma *old; ++ int ret; ++ ++ if (vma) { ++ if (!i915_vma_is_map_and_fenceable(vma)) ++ return -EINVAL; ++ ++ if (WARN(!i915_gem_object_get_stride(vma->obj) || ++ !i915_gem_object_get_tiling(vma->obj), ++ "bogus fence setup with stride: 0x%x, tiling mode: %i\n", ++ i915_gem_object_get_stride(vma->obj), ++ i915_gem_object_get_tiling(vma->obj))) ++ return -EINVAL; ++ ++ ret = i915_active_request_retire(&vma->last_fence, ++ &vma->obj->base.dev->struct_mutex); ++ if (ret) ++ return ret; ++ } ++ ++ old = xchg(&fence->vma, NULL); ++ if (old) { ++ ret = i915_active_request_retire(&old->last_fence, ++ &old->obj->base.dev->struct_mutex); ++ if (ret) { ++ fence->vma = old; ++ return ret; ++ } ++ ++ i915_vma_flush_writes(old); ++ ++ /* ++ * Ensure that all userspace CPU access is completed before ++ * stealing the fence. ++ */ ++ if (old != vma) { ++ GEM_BUG_ON(old->fence != fence); ++ i915_vma_revoke_mmap(old); ++ old->fence = NULL; ++ } ++ ++ list_move(&fence->link, &fence->i915->mm.fence_list); ++ } ++ ++ /* ++ * We only need to update the register itself if the device is awake. ++ * If the device is currently powered down, we will defer the write ++ * to the runtime resume, see i915_gem_restore_fences(). ++ * ++ * This only works for removing the fence register, on acquisition ++ * the caller must hold the rpm wakeref. The fence register must ++ * be cleared before we can use any other fences to ensure that ++ * the new fences do not overlap the elided clears, confusing HW. ++ */ ++ wakeref = intel_runtime_pm_get_if_in_use(fence->i915); ++ if (!wakeref) { ++ GEM_BUG_ON(vma); ++ return 0; ++ } ++ ++ WRITE_ONCE(fence->vma, vma); ++ fence_write(fence, vma); ++ ++ if (vma) { ++ vma->fence = fence; ++ list_move_tail(&fence->link, &fence->i915->mm.fence_list); ++ } ++ ++ intel_runtime_pm_put(fence->i915, wakeref); ++ return 0; ++} ++ ++/** ++ * i915_vma_put_fence - force-remove fence for a VMA ++ * @vma: vma to map linearly (not through a fence reg) ++ * ++ * This function force-removes any fence from the given object, which is useful ++ * if the kernel wants to do untiled GTT access. ++ * ++ * Returns: ++ * ++ * 0 on success, negative error code on failure. ++ */ ++int i915_vma_put_fence(struct i915_vma *vma) ++{ ++ struct drm_i915_fence_reg *fence = vma->fence; ++ ++ if (!fence) ++ return 0; ++ ++ if (fence->pin_count) ++ return -EBUSY; ++ ++ return fence_update(fence, NULL); ++} ++ ++static struct drm_i915_fence_reg *fence_find(struct drm_i915_private *dev_priv) ++{ ++ struct drm_i915_fence_reg *fence; ++ ++ list_for_each_entry(fence, &dev_priv->mm.fence_list, link) { ++ GEM_BUG_ON(fence->vma && fence->vma->fence != fence); ++ ++ if (fence->pin_count) ++ continue; ++ ++ return fence; ++ } ++ ++ /* Wait for completion of pending flips which consume fences */ ++ if (intel_has_pending_fb_unpin(dev_priv)) ++ return ERR_PTR(-EAGAIN); ++ ++ return ERR_PTR(-EDEADLK); ++} ++ ++/** ++ * i915_vma_pin_fence - set up fencing for a vma ++ * @vma: vma to map through a fence reg ++ * ++ * When mapping objects through the GTT, userspace wants to be able to write ++ * to them without having to worry about swizzling if the object is tiled. ++ * This function walks the fence regs looking for a free one for @obj, ++ * stealing one if it can't find any. ++ * ++ * It then sets up the reg based on the object's properties: address, pitch ++ * and tiling format. ++ * ++ * For an untiled surface, this removes any existing fence. ++ * ++ * Returns: ++ * ++ * 0 on success, negative error code on failure. ++ */ ++int ++i915_vma_pin_fence(struct i915_vma *vma) ++{ ++ struct drm_i915_fence_reg *fence; ++ struct i915_vma *set = i915_gem_object_is_tiled(vma->obj) ? vma : NULL; ++ int err; ++ ++ /* Note that we revoke fences on runtime suspend. Therefore the user ++ * must keep the device awake whilst using the fence. ++ */ ++ assert_rpm_wakelock_held(vma->vm->i915); ++ ++ /* Just update our place in the LRU if our fence is getting reused. */ ++ if (vma->fence) { ++ fence = vma->fence; ++ GEM_BUG_ON(fence->vma != vma); ++ fence->pin_count++; ++ if (!fence->dirty) { ++ list_move_tail(&fence->link, ++ &fence->i915->mm.fence_list); ++ return 0; ++ } ++ } else if (set) { ++ fence = fence_find(vma->vm->i915); ++ if (IS_ERR(fence)) ++ return PTR_ERR(fence); ++ ++ GEM_BUG_ON(fence->pin_count); ++ fence->pin_count++; ++ } else ++ return 0; ++ ++ err = fence_update(fence, set); ++ if (err) ++ goto out_unpin; ++ ++ GEM_BUG_ON(fence->vma != set); ++ GEM_BUG_ON(vma->fence != (set ? fence : NULL)); ++ ++ if (set) ++ return 0; ++ ++out_unpin: ++ fence->pin_count--; ++ return err; ++} ++ ++/** ++ * i915_reserve_fence - Reserve a fence for vGPU ++ * @dev_priv: i915 device private ++ * ++ * This function walks the fence regs looking for a free one and remove ++ * it from the fence_list. It is used to reserve fence for vGPU to use. ++ */ ++struct drm_i915_fence_reg * ++i915_reserve_fence(struct drm_i915_private *dev_priv) ++{ ++ struct drm_i915_fence_reg *fence; ++ int count; ++ int ret; ++ ++ lockdep_assert_held(&dev_priv->drm.struct_mutex); ++ ++ /* Keep at least one fence available for the display engine. */ ++ count = 0; ++ list_for_each_entry(fence, &dev_priv->mm.fence_list, link) ++ count += !fence->pin_count; ++ if (count <= 1) ++ return ERR_PTR(-ENOSPC); ++ ++ fence = fence_find(dev_priv); ++ if (IS_ERR(fence)) ++ return fence; ++ ++ if (fence->vma) { ++ /* Force-remove fence from VMA */ ++ ret = fence_update(fence, NULL); ++ if (ret) ++ return ERR_PTR(ret); ++ } ++ ++ list_del(&fence->link); ++ return fence; ++} ++ ++/** ++ * i915_unreserve_fence - Reclaim a reserved fence ++ * @fence: the fence reg ++ * ++ * This function add a reserved fence register from vGPU to the fence_list. ++ */ ++void i915_unreserve_fence(struct drm_i915_fence_reg *fence) ++{ ++ lockdep_assert_held(&fence->i915->drm.struct_mutex); ++ ++ list_add(&fence->link, &fence->i915->mm.fence_list); ++} ++ ++/** ++ * i915_gem_restore_fences - restore fence state ++ * @dev_priv: i915 device private ++ * ++ * Restore the hw fence state to match the software tracking again, to be called ++ * after a gpu reset and on resume. Note that on runtime suspend we only cancel ++ * the fences, to be reacquired by the user later. ++ */ ++void i915_gem_restore_fences(struct drm_i915_private *dev_priv) ++{ ++ int i; ++ ++ rcu_read_lock(); /* keep obj alive as we dereference */ ++ for (i = 0; i < dev_priv->num_fence_regs; i++) { ++ struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; ++ struct i915_vma *vma = READ_ONCE(reg->vma); ++ ++ GEM_BUG_ON(vma && vma->fence != reg); ++ ++ /* ++ * Commit delayed tiling changes if we have an object still ++ * attached to the fence, otherwise just clear the fence. ++ */ ++ if (vma && !i915_gem_object_is_tiled(vma->obj)) ++ vma = NULL; ++ ++ fence_write(reg, vma); ++ } ++ rcu_read_unlock(); ++} ++ ++/** ++ * DOC: tiling swizzling details ++ * ++ * The idea behind tiling is to increase cache hit rates by rearranging ++ * pixel data so that a group of pixel accesses are in the same cacheline. ++ * Performance improvement from doing this on the back/depth buffer are on ++ * the order of 30%. ++ * ++ * Intel architectures make this somewhat more complicated, though, by ++ * adjustments made to addressing of data when the memory is in interleaved ++ * mode (matched pairs of DIMMS) to improve memory bandwidth. ++ * For interleaved memory, the CPU sends every sequential 64 bytes ++ * to an alternate memory channel so it can get the bandwidth from both. ++ * ++ * The GPU also rearranges its accesses for increased bandwidth to interleaved ++ * memory, and it matches what the CPU does for non-tiled. However, when tiled ++ * it does it a little differently, since one walks addresses not just in the ++ * X direction but also Y. So, along with alternating channels when bit ++ * 6 of the address flips, it also alternates when other bits flip -- Bits 9 ++ * (every 512 bytes, an X tile scanline) and 10 (every two X tile scanlines) ++ * are common to both the 915 and 965-class hardware. ++ * ++ * The CPU also sometimes XORs in higher bits as well, to improve ++ * bandwidth doing strided access like we do so frequently in graphics. This ++ * is called "Channel XOR Randomization" in the MCH documentation. The result ++ * is that the CPU is XORing in either bit 11 or bit 17 to bit 6 of its address ++ * decode. ++ * ++ * All of this bit 6 XORing has an effect on our memory management, ++ * as we need to make sure that the 3d driver can correctly address object ++ * contents. ++ * ++ * If we don't have interleaved memory, all tiling is safe and no swizzling is ++ * required. ++ * ++ * When bit 17 is XORed in, we simply refuse to tile at all. Bit ++ * 17 is not just a page offset, so as we page an object out and back in, ++ * individual pages in it will have different bit 17 addresses, resulting in ++ * each 64 bytes being swapped with its neighbor! ++ * ++ * Otherwise, if interleaved, we have to tell the 3d driver what the address ++ * swizzling it needs to do is, since it's writing with the CPU to the pages ++ * (bit 6 and potentially bit 11 XORed in), and the GPU is reading from the ++ * pages (bit 6, 9, and 10 XORed in), resulting in a cumulative bit swizzling ++ * required by the CPU of XORing in bit 6, 9, 10, and potentially 11, in order ++ * to match what the GPU expects. ++ */ ++ ++/** ++ * i915_gem_detect_bit_6_swizzle - detect bit 6 swizzling pattern ++ * @dev_priv: i915 device private ++ * ++ * Detects bit 6 swizzling of address lookup between IGD access and CPU ++ * access through main memory. ++ */ ++void ++i915_gem_detect_bit_6_swizzle(struct drm_i915_private *dev_priv) ++{ ++ u32 swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN; ++ u32 swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN; ++ ++ if (INTEL_GEN(dev_priv) >= 8 || IS_VALLEYVIEW(dev_priv)) { ++ /* ++ * On BDW+, swizzling is not used. We leave the CPU memory ++ * controller in charge of optimizing memory accesses without ++ * the extra address manipulation GPU side. ++ * ++ * VLV and CHV don't have GPU swizzling. ++ */ ++ swizzle_x = I915_BIT_6_SWIZZLE_NONE; ++ swizzle_y = I915_BIT_6_SWIZZLE_NONE; ++ } else if (INTEL_GEN(dev_priv) >= 6) { ++ if (dev_priv->preserve_bios_swizzle) { ++ if (I915_READ(DISP_ARB_CTL) & ++ DISP_TILE_SURFACE_SWIZZLING) { ++ swizzle_x = I915_BIT_6_SWIZZLE_9_10; ++ swizzle_y = I915_BIT_6_SWIZZLE_9; ++ } else { ++ swizzle_x = I915_BIT_6_SWIZZLE_NONE; ++ swizzle_y = I915_BIT_6_SWIZZLE_NONE; ++ } ++ } else { ++ u32 dimm_c0, dimm_c1; ++ dimm_c0 = I915_READ(MAD_DIMM_C0); ++ dimm_c1 = I915_READ(MAD_DIMM_C1); ++ dimm_c0 &= MAD_DIMM_A_SIZE_MASK | MAD_DIMM_B_SIZE_MASK; ++ dimm_c1 &= MAD_DIMM_A_SIZE_MASK | MAD_DIMM_B_SIZE_MASK; ++ /* Enable swizzling when the channels are populated ++ * with identically sized dimms. We don't need to check ++ * the 3rd channel because no cpu with gpu attached ++ * ships in that configuration. Also, swizzling only ++ * makes sense for 2 channels anyway. */ ++ if (dimm_c0 == dimm_c1) { ++ swizzle_x = I915_BIT_6_SWIZZLE_9_10; ++ swizzle_y = I915_BIT_6_SWIZZLE_9; ++ } else { ++ swizzle_x = I915_BIT_6_SWIZZLE_NONE; ++ swizzle_y = I915_BIT_6_SWIZZLE_NONE; ++ } ++ } ++ } else if (IS_GEN(dev_priv, 5)) { ++ /* On Ironlake whatever DRAM config, GPU always do ++ * same swizzling setup. ++ */ ++ swizzle_x = I915_BIT_6_SWIZZLE_9_10; ++ swizzle_y = I915_BIT_6_SWIZZLE_9; ++ } else if (IS_GEN(dev_priv, 2)) { ++ /* As far as we know, the 865 doesn't have these bit 6 ++ * swizzling issues. ++ */ ++ swizzle_x = I915_BIT_6_SWIZZLE_NONE; ++ swizzle_y = I915_BIT_6_SWIZZLE_NONE; ++ } else if (IS_G45(dev_priv) || IS_I965G(dev_priv) || IS_G33(dev_priv)) { ++ /* The 965, G33, and newer, have a very flexible memory ++ * configuration. It will enable dual-channel mode ++ * (interleaving) on as much memory as it can, and the GPU ++ * will additionally sometimes enable different bit 6 ++ * swizzling for tiled objects from the CPU. ++ * ++ * Here's what I found on the G965: ++ * slot fill memory size swizzling ++ * 0A 0B 1A 1B 1-ch 2-ch ++ * 512 0 0 0 512 0 O ++ * 512 0 512 0 16 1008 X ++ * 512 0 0 512 16 1008 X ++ * 0 512 0 512 16 1008 X ++ * 1024 1024 1024 0 2048 1024 O ++ * ++ * We could probably detect this based on either the DRB ++ * matching, which was the case for the swizzling required in ++ * the table above, or from the 1-ch value being less than ++ * the minimum size of a rank. ++ * ++ * Reports indicate that the swizzling actually ++ * varies depending upon page placement inside the ++ * channels, i.e. we see swizzled pages where the ++ * banks of memory are paired and unswizzled on the ++ * uneven portion, so leave that as unknown. ++ */ ++ if (I915_READ16(C0DRB3) == I915_READ16(C1DRB3)) { ++ swizzle_x = I915_BIT_6_SWIZZLE_9_10; ++ swizzle_y = I915_BIT_6_SWIZZLE_9; ++ } ++ } else { ++ u32 dcc; ++ ++ /* On 9xx chipsets, channel interleave by the CPU is ++ * determined by DCC. For single-channel, neither the CPU ++ * nor the GPU do swizzling. For dual channel interleaved, ++ * the GPU's interleave is bit 9 and 10 for X tiled, and bit ++ * 9 for Y tiled. The CPU's interleave is independent, and ++ * can be based on either bit 11 (haven't seen this yet) or ++ * bit 17 (common). ++ */ ++ dcc = I915_READ(DCC); ++ switch (dcc & DCC_ADDRESSING_MODE_MASK) { ++ case DCC_ADDRESSING_MODE_SINGLE_CHANNEL: ++ case DCC_ADDRESSING_MODE_DUAL_CHANNEL_ASYMMETRIC: ++ swizzle_x = I915_BIT_6_SWIZZLE_NONE; ++ swizzle_y = I915_BIT_6_SWIZZLE_NONE; ++ break; ++ case DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED: ++ if (dcc & DCC_CHANNEL_XOR_DISABLE) { ++ /* This is the base swizzling by the GPU for ++ * tiled buffers. ++ */ ++ swizzle_x = I915_BIT_6_SWIZZLE_9_10; ++ swizzle_y = I915_BIT_6_SWIZZLE_9; ++ } else if ((dcc & DCC_CHANNEL_XOR_BIT_17) == 0) { ++ /* Bit 11 swizzling by the CPU in addition. */ ++ swizzle_x = I915_BIT_6_SWIZZLE_9_10_11; ++ swizzle_y = I915_BIT_6_SWIZZLE_9_11; ++ } else { ++ /* Bit 17 swizzling by the CPU in addition. */ ++ swizzle_x = I915_BIT_6_SWIZZLE_9_10_17; ++ swizzle_y = I915_BIT_6_SWIZZLE_9_17; ++ } ++ break; ++ } ++ ++ /* check for L-shaped memory aka modified enhanced addressing */ ++ if (IS_GEN(dev_priv, 4) && ++ !(I915_READ(DCC2) & DCC2_MODIFIED_ENHANCED_DISABLE)) { ++ swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN; ++ swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN; ++ } ++ ++ if (dcc == 0xffffffff) { ++ DRM_ERROR("Couldn't read from MCHBAR. " ++ "Disabling tiling.\n"); ++ swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN; ++ swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN; ++ } ++ } ++ ++ if (swizzle_x == I915_BIT_6_SWIZZLE_UNKNOWN || ++ swizzle_y == I915_BIT_6_SWIZZLE_UNKNOWN) { ++ /* Userspace likes to explode if it sees unknown swizzling, ++ * so lie. We will finish the lie when reporting through ++ * the get-tiling-ioctl by reporting the physical swizzle ++ * mode as unknown instead. ++ * ++ * As we don't strictly know what the swizzling is, it may be ++ * bit17 dependent, and so we need to also prevent the pages ++ * from being moved. ++ */ ++ dev_priv->quirks |= QUIRK_PIN_SWIZZLED_PAGES; ++ swizzle_x = I915_BIT_6_SWIZZLE_NONE; ++ swizzle_y = I915_BIT_6_SWIZZLE_NONE; ++ } ++ ++ dev_priv->mm.bit_6_swizzle_x = swizzle_x; ++ dev_priv->mm.bit_6_swizzle_y = swizzle_y; ++} ++ ++/* ++ * Swap every 64 bytes of this page around, to account for it having a new ++ * bit 17 of its physical address and therefore being interpreted differently ++ * by the GPU. ++ */ ++static void ++i915_gem_swizzle_page(struct page *page) ++{ ++ char temp[64]; ++ char *vaddr; ++ int i; ++ ++ vaddr = kmap(page); ++ ++ for (i = 0; i < PAGE_SIZE; i += 128) { ++ memcpy(temp, &vaddr[i], 64); ++ memcpy(&vaddr[i], &vaddr[i + 64], 64); ++ memcpy(&vaddr[i + 64], temp, 64); ++ } ++ ++ kunmap(page); ++} ++ ++/** ++ * i915_gem_object_do_bit_17_swizzle - fixup bit 17 swizzling ++ * @obj: i915 GEM buffer object ++ * @pages: the scattergather list of physical pages ++ * ++ * This function fixes up the swizzling in case any page frame number for this ++ * object has changed in bit 17 since that state has been saved with ++ * i915_gem_object_save_bit_17_swizzle(). ++ * ++ * This is called when pinning backing storage again, since the kernel is free ++ * to move unpinned backing storage around (either by directly moving pages or ++ * by swapping them out and back in again). ++ */ ++void ++i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj, ++ struct sg_table *pages) ++{ ++ struct sgt_iter sgt_iter; ++ struct page *page; ++ int i; ++ ++ if (obj->bit_17 == NULL) ++ return; ++ ++ i = 0; ++ for_each_sgt_page(page, sgt_iter, pages) { ++ char new_bit_17 = page_to_phys(page) >> 17; ++ if ((new_bit_17 & 0x1) != (test_bit(i, obj->bit_17) != 0)) { ++ i915_gem_swizzle_page(page); ++ set_page_dirty(page); ++ } ++ i++; ++ } ++} ++ ++/** ++ * i915_gem_object_save_bit_17_swizzle - save bit 17 swizzling ++ * @obj: i915 GEM buffer object ++ * @pages: the scattergather list of physical pages ++ * ++ * This function saves the bit 17 of each page frame number so that swizzling ++ * can be fixed up later on with i915_gem_object_do_bit_17_swizzle(). This must ++ * be called before the backing storage can be unpinned. ++ */ ++void ++i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj, ++ struct sg_table *pages) ++{ ++ const unsigned int page_count = obj->base.size >> PAGE_SHIFT; ++ struct sgt_iter sgt_iter; ++ struct page *page; ++ int i; ++ ++ if (obj->bit_17 == NULL) { ++ obj->bit_17 = bitmap_zalloc(page_count, GFP_KERNEL); ++ if (obj->bit_17 == NULL) { ++ DRM_ERROR("Failed to allocate memory for bit 17 " ++ "record\n"); ++ return; ++ } ++ } ++ ++ i = 0; ++ ++ for_each_sgt_page(page, sgt_iter, pages) { ++ if (page_to_phys(page) & (1 << 17)) ++ __set_bit(i, obj->bit_17); ++ else ++ __clear_bit(i, obj->bit_17); ++ i++; ++ } ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_gem_fence_reg.h b/drivers/gpu/drm/i915_legacy/i915_gem_fence_reg.h +new file mode 100644 +index 000000000000..09dcaf14121b +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gem_fence_reg.h +@@ -0,0 +1,52 @@ ++/* ++ * Copyright © 2016 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#ifndef __I915_FENCE_REG_H__ ++#define __I915_FENCE_REG_H__ ++ ++#include ++ ++struct drm_i915_private; ++struct i915_vma; ++ ++#define I965_FENCE_PAGE 4096UL ++ ++struct drm_i915_fence_reg { ++ struct list_head link; ++ struct drm_i915_private *i915; ++ struct i915_vma *vma; ++ int pin_count; ++ int id; ++ /** ++ * Whether the tiling parameters for the currently ++ * associated fence register have changed. Note that ++ * for the purposes of tracking tiling changes we also ++ * treat the unfenced register, the register slot that ++ * the object occupies whilst it executes a fenced ++ * command (such as BLT on gen2/3), as a "fence". ++ */ ++ bool dirty; ++}; ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_gem_gtt.c b/drivers/gpu/drm/i915_legacy/i915_gem_gtt.c +new file mode 100644 +index 000000000000..8f460cc4cc1f +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gem_gtt.c +@@ -0,0 +1,3922 @@ ++/* ++ * Copyright © 2010 Daniel Vetter ++ * Copyright © 2011-2014 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#include /* fault-inject.h is not standalone! */ ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include ++ ++#include "i915_drv.h" ++#include "i915_vgpu.h" ++#include "i915_reset.h" ++#include "i915_trace.h" ++#include "intel_drv.h" ++#include "intel_frontbuffer.h" ++ ++#define I915_GFP_ALLOW_FAIL (GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN) ++ ++/** ++ * DOC: Global GTT views ++ * ++ * Background and previous state ++ * ++ * Historically objects could exists (be bound) in global GTT space only as ++ * singular instances with a view representing all of the object's backing pages ++ * in a linear fashion. This view will be called a normal view. ++ * ++ * To support multiple views of the same object, where the number of mapped ++ * pages is not equal to the backing store, or where the layout of the pages ++ * is not linear, concept of a GGTT view was added. ++ * ++ * One example of an alternative view is a stereo display driven by a single ++ * image. In this case we would have a framebuffer looking like this ++ * (2x2 pages): ++ * ++ * 12 ++ * 34 ++ * ++ * Above would represent a normal GGTT view as normally mapped for GPU or CPU ++ * rendering. In contrast, fed to the display engine would be an alternative ++ * view which could look something like this: ++ * ++ * 1212 ++ * 3434 ++ * ++ * In this example both the size and layout of pages in the alternative view is ++ * different from the normal view. ++ * ++ * Implementation and usage ++ * ++ * GGTT views are implemented using VMAs and are distinguished via enum ++ * i915_ggtt_view_type and struct i915_ggtt_view. ++ * ++ * A new flavour of core GEM functions which work with GGTT bound objects were ++ * added with the _ggtt_ infix, and sometimes with _view postfix to avoid ++ * renaming in large amounts of code. They take the struct i915_ggtt_view ++ * parameter encapsulating all metadata required to implement a view. ++ * ++ * As a helper for callers which are only interested in the normal view, ++ * globally const i915_ggtt_view_normal singleton instance exists. All old core ++ * GEM API functions, the ones not taking the view parameter, are operating on, ++ * or with the normal GGTT view. ++ * ++ * Code wanting to add or use a new GGTT view needs to: ++ * ++ * 1. Add a new enum with a suitable name. ++ * 2. Extend the metadata in the i915_ggtt_view structure if required. ++ * 3. Add support to i915_get_vma_pages(). ++ * ++ * New views are required to build a scatter-gather table from within the ++ * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and ++ * exists for the lifetime of an VMA. ++ * ++ * Core API is designed to have copy semantics which means that passed in ++ * struct i915_ggtt_view does not need to be persistent (left around after ++ * calling the core API functions). ++ * ++ */ ++ ++static int ++i915_get_ggtt_vma_pages(struct i915_vma *vma); ++ ++static void gen6_ggtt_invalidate(struct drm_i915_private *dev_priv) ++{ ++ /* ++ * Note that as an uncached mmio write, this will flush the ++ * WCB of the writes into the GGTT before it triggers the invalidate. ++ */ ++ I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); ++} ++ ++static void guc_ggtt_invalidate(struct drm_i915_private *dev_priv) ++{ ++ gen6_ggtt_invalidate(dev_priv); ++ I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE); ++} ++ ++static void gmch_ggtt_invalidate(struct drm_i915_private *dev_priv) ++{ ++ intel_gtt_chipset_flush(); ++} ++ ++static inline void i915_ggtt_invalidate(struct drm_i915_private *i915) ++{ ++ i915->ggtt.invalidate(i915); ++} ++ ++static int ppgtt_bind_vma(struct i915_vma *vma, ++ enum i915_cache_level cache_level, ++ u32 unused) ++{ ++ u32 pte_flags; ++ int err; ++ ++ if (!(vma->flags & I915_VMA_LOCAL_BIND)) { ++ err = vma->vm->allocate_va_range(vma->vm, ++ vma->node.start, vma->size); ++ if (err) ++ return err; ++ } ++ ++ /* Applicable to VLV, and gen8+ */ ++ pte_flags = 0; ++ if (i915_gem_object_is_readonly(vma->obj)) ++ pte_flags |= PTE_READ_ONLY; ++ ++ vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags); ++ ++ return 0; ++} ++ ++static void ppgtt_unbind_vma(struct i915_vma *vma) ++{ ++ vma->vm->clear_range(vma->vm, vma->node.start, vma->size); ++} ++ ++static int ppgtt_set_pages(struct i915_vma *vma) ++{ ++ GEM_BUG_ON(vma->pages); ++ ++ vma->pages = vma->obj->mm.pages; ++ ++ vma->page_sizes = vma->obj->mm.page_sizes; ++ ++ return 0; ++} ++ ++static void clear_pages(struct i915_vma *vma) ++{ ++ GEM_BUG_ON(!vma->pages); ++ ++ if (vma->pages != vma->obj->mm.pages) { ++ sg_free_table(vma->pages); ++ kfree(vma->pages); ++ } ++ vma->pages = NULL; ++ ++ memset(&vma->page_sizes, 0, sizeof(vma->page_sizes)); ++} ++ ++static u64 gen8_pte_encode(dma_addr_t addr, ++ enum i915_cache_level level, ++ u32 flags) ++{ ++ gen8_pte_t pte = addr | _PAGE_PRESENT | _PAGE_RW; ++ ++ if (unlikely(flags & PTE_READ_ONLY)) ++ pte &= ~_PAGE_RW; ++ ++ switch (level) { ++ case I915_CACHE_NONE: ++ pte |= PPAT_UNCACHED; ++ break; ++ case I915_CACHE_WT: ++ pte |= PPAT_DISPLAY_ELLC; ++ break; ++ default: ++ pte |= PPAT_CACHED; ++ break; ++ } ++ ++ return pte; ++} ++ ++static gen8_pde_t gen8_pde_encode(const dma_addr_t addr, ++ const enum i915_cache_level level) ++{ ++ gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW; ++ pde |= addr; ++ if (level != I915_CACHE_NONE) ++ pde |= PPAT_CACHED_PDE; ++ else ++ pde |= PPAT_UNCACHED; ++ return pde; ++} ++ ++#define gen8_pdpe_encode gen8_pde_encode ++#define gen8_pml4e_encode gen8_pde_encode ++ ++static u64 snb_pte_encode(dma_addr_t addr, ++ enum i915_cache_level level, ++ u32 flags) ++{ ++ gen6_pte_t pte = GEN6_PTE_VALID; ++ pte |= GEN6_PTE_ADDR_ENCODE(addr); ++ ++ switch (level) { ++ case I915_CACHE_L3_LLC: ++ case I915_CACHE_LLC: ++ pte |= GEN6_PTE_CACHE_LLC; ++ break; ++ case I915_CACHE_NONE: ++ pte |= GEN6_PTE_UNCACHED; ++ break; ++ default: ++ MISSING_CASE(level); ++ } ++ ++ return pte; ++} ++ ++static u64 ivb_pte_encode(dma_addr_t addr, ++ enum i915_cache_level level, ++ u32 flags) ++{ ++ gen6_pte_t pte = GEN6_PTE_VALID; ++ pte |= GEN6_PTE_ADDR_ENCODE(addr); ++ ++ switch (level) { ++ case I915_CACHE_L3_LLC: ++ pte |= GEN7_PTE_CACHE_L3_LLC; ++ break; ++ case I915_CACHE_LLC: ++ pte |= GEN6_PTE_CACHE_LLC; ++ break; ++ case I915_CACHE_NONE: ++ pte |= GEN6_PTE_UNCACHED; ++ break; ++ default: ++ MISSING_CASE(level); ++ } ++ ++ return pte; ++} ++ ++static u64 byt_pte_encode(dma_addr_t addr, ++ enum i915_cache_level level, ++ u32 flags) ++{ ++ gen6_pte_t pte = GEN6_PTE_VALID; ++ pte |= GEN6_PTE_ADDR_ENCODE(addr); ++ ++ if (!(flags & PTE_READ_ONLY)) ++ pte |= BYT_PTE_WRITEABLE; ++ ++ if (level != I915_CACHE_NONE) ++ pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES; ++ ++ return pte; ++} ++ ++static u64 hsw_pte_encode(dma_addr_t addr, ++ enum i915_cache_level level, ++ u32 flags) ++{ ++ gen6_pte_t pte = GEN6_PTE_VALID; ++ pte |= HSW_PTE_ADDR_ENCODE(addr); ++ ++ if (level != I915_CACHE_NONE) ++ pte |= HSW_WB_LLC_AGE3; ++ ++ return pte; ++} ++ ++static u64 iris_pte_encode(dma_addr_t addr, ++ enum i915_cache_level level, ++ u32 flags) ++{ ++ gen6_pte_t pte = GEN6_PTE_VALID; ++ pte |= HSW_PTE_ADDR_ENCODE(addr); ++ ++ switch (level) { ++ case I915_CACHE_NONE: ++ break; ++ case I915_CACHE_WT: ++ pte |= HSW_WT_ELLC_LLC_AGE3; ++ break; ++ default: ++ pte |= HSW_WB_ELLC_LLC_AGE3; ++ break; ++ } ++ ++ return pte; ++} ++ ++static void stash_init(struct pagestash *stash) ++{ ++ pagevec_init(&stash->pvec); ++ spin_lock_init(&stash->lock); ++} ++ ++static struct page *stash_pop_page(struct pagestash *stash) ++{ ++ struct page *page = NULL; ++ ++ spin_lock(&stash->lock); ++ if (likely(stash->pvec.nr)) ++ page = stash->pvec.pages[--stash->pvec.nr]; ++ spin_unlock(&stash->lock); ++ ++ return page; ++} ++ ++static void stash_push_pagevec(struct pagestash *stash, struct pagevec *pvec) ++{ ++ int nr; ++ ++ spin_lock_nested(&stash->lock, SINGLE_DEPTH_NESTING); ++ ++ nr = min_t(int, pvec->nr, pagevec_space(&stash->pvec)); ++ memcpy(stash->pvec.pages + stash->pvec.nr, ++ pvec->pages + pvec->nr - nr, ++ sizeof(pvec->pages[0]) * nr); ++ stash->pvec.nr += nr; ++ ++ spin_unlock(&stash->lock); ++ ++ pvec->nr -= nr; ++} ++ ++static struct page *vm_alloc_page(struct i915_address_space *vm, gfp_t gfp) ++{ ++ struct pagevec stack; ++ struct page *page; ++ ++ if (I915_SELFTEST_ONLY(should_fail(&vm->fault_attr, 1))) ++ i915_gem_shrink_all(vm->i915); ++ ++ page = stash_pop_page(&vm->free_pages); ++ if (page) ++ return page; ++ ++ if (!vm->pt_kmap_wc) ++ return alloc_page(gfp); ++ ++ /* Look in our global stash of WC pages... */ ++ page = stash_pop_page(&vm->i915->mm.wc_stash); ++ if (page) ++ return page; ++ ++ /* ++ * Otherwise batch allocate pages to amortize cost of set_pages_wc. ++ * ++ * We have to be careful as page allocation may trigger the shrinker ++ * (via direct reclaim) which will fill up the WC stash underneath us. ++ * So we add our WB pages into a temporary pvec on the stack and merge ++ * them into the WC stash after all the allocations are complete. ++ */ ++ pagevec_init(&stack); ++ do { ++ struct page *page; ++ ++ page = alloc_page(gfp); ++ if (unlikely(!page)) ++ break; ++ ++ stack.pages[stack.nr++] = page; ++ } while (pagevec_space(&stack)); ++ ++ if (stack.nr && !set_pages_array_wc(stack.pages, stack.nr)) { ++ page = stack.pages[--stack.nr]; ++ ++ /* Merge spare WC pages to the global stash */ ++ stash_push_pagevec(&vm->i915->mm.wc_stash, &stack); ++ ++ /* Push any surplus WC pages onto the local VM stash */ ++ if (stack.nr) ++ stash_push_pagevec(&vm->free_pages, &stack); ++ } ++ ++ /* Return unwanted leftovers */ ++ if (unlikely(stack.nr)) { ++ WARN_ON_ONCE(set_pages_array_wb(stack.pages, stack.nr)); ++ __pagevec_release(&stack); ++ } ++ ++ return page; ++} ++ ++static void vm_free_pages_release(struct i915_address_space *vm, ++ bool immediate) ++{ ++ struct pagevec *pvec = &vm->free_pages.pvec; ++ struct pagevec stack; ++ ++ lockdep_assert_held(&vm->free_pages.lock); ++ GEM_BUG_ON(!pagevec_count(pvec)); ++ ++ if (vm->pt_kmap_wc) { ++ /* ++ * When we use WC, first fill up the global stash and then ++ * only if full immediately free the overflow. ++ */ ++ stash_push_pagevec(&vm->i915->mm.wc_stash, pvec); ++ ++ /* ++ * As we have made some room in the VM's free_pages, ++ * we can wait for it to fill again. Unless we are ++ * inside i915_address_space_fini() and must ++ * immediately release the pages! ++ */ ++ if (pvec->nr <= (immediate ? 0 : PAGEVEC_SIZE - 1)) ++ return; ++ ++ /* ++ * We have to drop the lock to allow ourselves to sleep, ++ * so take a copy of the pvec and clear the stash for ++ * others to use it as we sleep. ++ */ ++ stack = *pvec; ++ pagevec_reinit(pvec); ++ spin_unlock(&vm->free_pages.lock); ++ ++ pvec = &stack; ++ set_pages_array_wb(pvec->pages, pvec->nr); ++ ++ spin_lock(&vm->free_pages.lock); ++ } ++ ++ __pagevec_release(pvec); ++} ++ ++static void vm_free_page(struct i915_address_space *vm, struct page *page) ++{ ++ /* ++ * On !llc, we need to change the pages back to WB. We only do so ++ * in bulk, so we rarely need to change the page attributes here, ++ * but doing so requires a stop_machine() from deep inside arch/x86/mm. ++ * To make detection of the possible sleep more likely, use an ++ * unconditional might_sleep() for everybody. ++ */ ++ might_sleep(); ++ spin_lock(&vm->free_pages.lock); ++ if (!pagevec_add(&vm->free_pages.pvec, page)) ++ vm_free_pages_release(vm, false); ++ spin_unlock(&vm->free_pages.lock); ++} ++ ++static void i915_address_space_init(struct i915_address_space *vm, int subclass) ++{ ++ /* ++ * The vm->mutex must be reclaim safe (for use in the shrinker). ++ * Do a dummy acquire now under fs_reclaim so that any allocation ++ * attempt holding the lock is immediately reported by lockdep. ++ */ ++ mutex_init(&vm->mutex); ++ lockdep_set_subclass(&vm->mutex, subclass); ++ i915_gem_shrinker_taints_mutex(vm->i915, &vm->mutex); ++ ++ GEM_BUG_ON(!vm->total); ++ drm_mm_init(&vm->mm, 0, vm->total); ++ vm->mm.head_node.color = I915_COLOR_UNEVICTABLE; ++ ++ stash_init(&vm->free_pages); ++ ++ INIT_LIST_HEAD(&vm->unbound_list); ++ INIT_LIST_HEAD(&vm->bound_list); ++} ++ ++static void i915_address_space_fini(struct i915_address_space *vm) ++{ ++ spin_lock(&vm->free_pages.lock); ++ if (pagevec_count(&vm->free_pages.pvec)) ++ vm_free_pages_release(vm, true); ++ GEM_BUG_ON(pagevec_count(&vm->free_pages.pvec)); ++ spin_unlock(&vm->free_pages.lock); ++ ++ drm_mm_takedown(&vm->mm); ++ ++ mutex_destroy(&vm->mutex); ++} ++ ++static int __setup_page_dma(struct i915_address_space *vm, ++ struct i915_page_dma *p, ++ gfp_t gfp) ++{ ++ p->page = vm_alloc_page(vm, gfp | I915_GFP_ALLOW_FAIL); ++ if (unlikely(!p->page)) ++ return -ENOMEM; ++ ++ p->daddr = dma_map_page_attrs(vm->dma, ++ p->page, 0, PAGE_SIZE, ++ PCI_DMA_BIDIRECTIONAL, ++ DMA_ATTR_SKIP_CPU_SYNC | ++ DMA_ATTR_NO_WARN); ++ if (unlikely(dma_mapping_error(vm->dma, p->daddr))) { ++ vm_free_page(vm, p->page); ++ return -ENOMEM; ++ } ++ ++ return 0; ++} ++ ++static int setup_page_dma(struct i915_address_space *vm, ++ struct i915_page_dma *p) ++{ ++ return __setup_page_dma(vm, p, __GFP_HIGHMEM); ++} ++ ++static void cleanup_page_dma(struct i915_address_space *vm, ++ struct i915_page_dma *p) ++{ ++ dma_unmap_page(vm->dma, p->daddr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); ++ vm_free_page(vm, p->page); ++} ++ ++#define kmap_atomic_px(px) kmap_atomic(px_base(px)->page) ++ ++#define setup_px(vm, px) setup_page_dma((vm), px_base(px)) ++#define cleanup_px(vm, px) cleanup_page_dma((vm), px_base(px)) ++#define fill_px(vm, px, v) fill_page_dma((vm), px_base(px), (v)) ++#define fill32_px(vm, px, v) fill_page_dma_32((vm), px_base(px), (v)) ++ ++static void fill_page_dma(struct i915_address_space *vm, ++ struct i915_page_dma *p, ++ const u64 val) ++{ ++ u64 * const vaddr = kmap_atomic(p->page); ++ ++ memset64(vaddr, val, PAGE_SIZE / sizeof(val)); ++ ++ kunmap_atomic(vaddr); ++} ++ ++static void fill_page_dma_32(struct i915_address_space *vm, ++ struct i915_page_dma *p, ++ const u32 v) ++{ ++ fill_page_dma(vm, p, (u64)v << 32 | v); ++} ++ ++static int ++setup_scratch_page(struct i915_address_space *vm, gfp_t gfp) ++{ ++ unsigned long size; ++ ++ /* ++ * In order to utilize 64K pages for an object with a size < 2M, we will ++ * need to support a 64K scratch page, given that every 16th entry for a ++ * page-table operating in 64K mode must point to a properly aligned 64K ++ * region, including any PTEs which happen to point to scratch. ++ * ++ * This is only relevant for the 48b PPGTT where we support ++ * huge-gtt-pages, see also i915_vma_insert(). However, as we share the ++ * scratch (read-only) between all vm, we create one 64k scratch page ++ * for all. ++ */ ++ size = I915_GTT_PAGE_SIZE_4K; ++ if (i915_vm_is_4lvl(vm) && ++ HAS_PAGE_SIZES(vm->i915, I915_GTT_PAGE_SIZE_64K)) { ++ size = I915_GTT_PAGE_SIZE_64K; ++ gfp |= __GFP_NOWARN; ++ } ++ gfp |= __GFP_ZERO | __GFP_RETRY_MAYFAIL; ++ ++ do { ++ int order = get_order(size); ++ struct page *page; ++ dma_addr_t addr; ++ ++ page = alloc_pages(gfp, order); ++ if (unlikely(!page)) ++ goto skip; ++ ++ addr = dma_map_page_attrs(vm->dma, ++ page, 0, size, ++ PCI_DMA_BIDIRECTIONAL, ++ DMA_ATTR_SKIP_CPU_SYNC | ++ DMA_ATTR_NO_WARN); ++ if (unlikely(dma_mapping_error(vm->dma, addr))) ++ goto free_page; ++ ++ if (unlikely(!IS_ALIGNED(addr, size))) ++ goto unmap_page; ++ ++ vm->scratch_page.page = page; ++ vm->scratch_page.daddr = addr; ++ vm->scratch_order = order; ++ return 0; ++ ++unmap_page: ++ dma_unmap_page(vm->dma, addr, size, PCI_DMA_BIDIRECTIONAL); ++free_page: ++ __free_pages(page, order); ++skip: ++ if (size == I915_GTT_PAGE_SIZE_4K) ++ return -ENOMEM; ++ ++ size = I915_GTT_PAGE_SIZE_4K; ++ gfp &= ~__GFP_NOWARN; ++ } while (1); ++} ++ ++static void cleanup_scratch_page(struct i915_address_space *vm) ++{ ++ struct i915_page_dma *p = &vm->scratch_page; ++ int order = vm->scratch_order; ++ ++ dma_unmap_page(vm->dma, p->daddr, BIT(order) << PAGE_SHIFT, ++ PCI_DMA_BIDIRECTIONAL); ++ __free_pages(p->page, order); ++} ++ ++static struct i915_page_table *alloc_pt(struct i915_address_space *vm) ++{ ++ struct i915_page_table *pt; ++ ++ pt = kmalloc(sizeof(*pt), I915_GFP_ALLOW_FAIL); ++ if (unlikely(!pt)) ++ return ERR_PTR(-ENOMEM); ++ ++ if (unlikely(setup_px(vm, pt))) { ++ kfree(pt); ++ return ERR_PTR(-ENOMEM); ++ } ++ ++ pt->used_ptes = 0; ++ return pt; ++} ++ ++static void free_pt(struct i915_address_space *vm, struct i915_page_table *pt) ++{ ++ cleanup_px(vm, pt); ++ kfree(pt); ++} ++ ++static void gen8_initialize_pt(struct i915_address_space *vm, ++ struct i915_page_table *pt) ++{ ++ fill_px(vm, pt, vm->scratch_pte); ++} ++ ++static void gen6_initialize_pt(struct i915_address_space *vm, ++ struct i915_page_table *pt) ++{ ++ fill32_px(vm, pt, vm->scratch_pte); ++} ++ ++static struct i915_page_directory *alloc_pd(struct i915_address_space *vm) ++{ ++ struct i915_page_directory *pd; ++ ++ pd = kzalloc(sizeof(*pd), I915_GFP_ALLOW_FAIL); ++ if (unlikely(!pd)) ++ return ERR_PTR(-ENOMEM); ++ ++ if (unlikely(setup_px(vm, pd))) { ++ kfree(pd); ++ return ERR_PTR(-ENOMEM); ++ } ++ ++ pd->used_pdes = 0; ++ return pd; ++} ++ ++static void free_pd(struct i915_address_space *vm, ++ struct i915_page_directory *pd) ++{ ++ cleanup_px(vm, pd); ++ kfree(pd); ++} ++ ++static void gen8_initialize_pd(struct i915_address_space *vm, ++ struct i915_page_directory *pd) ++{ ++ fill_px(vm, pd, ++ gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC)); ++ memset_p((void **)pd->page_table, vm->scratch_pt, I915_PDES); ++} ++ ++static int __pdp_init(struct i915_address_space *vm, ++ struct i915_page_directory_pointer *pdp) ++{ ++ const unsigned int pdpes = i915_pdpes_per_pdp(vm); ++ ++ pdp->page_directory = kmalloc_array(pdpes, sizeof(*pdp->page_directory), ++ I915_GFP_ALLOW_FAIL); ++ if (unlikely(!pdp->page_directory)) ++ return -ENOMEM; ++ ++ memset_p((void **)pdp->page_directory, vm->scratch_pd, pdpes); ++ ++ return 0; ++} ++ ++static void __pdp_fini(struct i915_page_directory_pointer *pdp) ++{ ++ kfree(pdp->page_directory); ++ pdp->page_directory = NULL; ++} ++ ++static struct i915_page_directory_pointer * ++alloc_pdp(struct i915_address_space *vm) ++{ ++ struct i915_page_directory_pointer *pdp; ++ int ret = -ENOMEM; ++ ++ GEM_BUG_ON(!i915_vm_is_4lvl(vm)); ++ ++ pdp = kzalloc(sizeof(*pdp), GFP_KERNEL); ++ if (!pdp) ++ return ERR_PTR(-ENOMEM); ++ ++ ret = __pdp_init(vm, pdp); ++ if (ret) ++ goto fail_bitmap; ++ ++ ret = setup_px(vm, pdp); ++ if (ret) ++ goto fail_page_m; ++ ++ return pdp; ++ ++fail_page_m: ++ __pdp_fini(pdp); ++fail_bitmap: ++ kfree(pdp); ++ ++ return ERR_PTR(ret); ++} ++ ++static void free_pdp(struct i915_address_space *vm, ++ struct i915_page_directory_pointer *pdp) ++{ ++ __pdp_fini(pdp); ++ ++ if (!i915_vm_is_4lvl(vm)) ++ return; ++ ++ cleanup_px(vm, pdp); ++ kfree(pdp); ++} ++ ++static void gen8_initialize_pdp(struct i915_address_space *vm, ++ struct i915_page_directory_pointer *pdp) ++{ ++ gen8_ppgtt_pdpe_t scratch_pdpe; ++ ++ scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC); ++ ++ fill_px(vm, pdp, scratch_pdpe); ++} ++ ++static void gen8_initialize_pml4(struct i915_address_space *vm, ++ struct i915_pml4 *pml4) ++{ ++ fill_px(vm, pml4, ++ gen8_pml4e_encode(px_dma(vm->scratch_pdp), I915_CACHE_LLC)); ++ memset_p((void **)pml4->pdps, vm->scratch_pdp, GEN8_PML4ES_PER_PML4); ++} ++ ++/* ++ * PDE TLBs are a pain to invalidate on GEN8+. When we modify ++ * the page table structures, we mark them dirty so that ++ * context switching/execlist queuing code takes extra steps ++ * to ensure that tlbs are flushed. ++ */ ++static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt) ++{ ++ ppgtt->pd_dirty_engines = ALL_ENGINES; ++} ++ ++/* Removes entries from a single page table, releasing it if it's empty. ++ * Caller can use the return value to update higher-level entries. ++ */ ++static bool gen8_ppgtt_clear_pt(const struct i915_address_space *vm, ++ struct i915_page_table *pt, ++ u64 start, u64 length) ++{ ++ unsigned int num_entries = gen8_pte_count(start, length); ++ gen8_pte_t *vaddr; ++ ++ GEM_BUG_ON(num_entries > pt->used_ptes); ++ ++ pt->used_ptes -= num_entries; ++ if (!pt->used_ptes) ++ return true; ++ ++ vaddr = kmap_atomic_px(pt); ++ memset64(vaddr + gen8_pte_index(start), vm->scratch_pte, num_entries); ++ kunmap_atomic(vaddr); ++ ++ return false; ++} ++ ++static void gen8_ppgtt_set_pde(struct i915_address_space *vm, ++ struct i915_page_directory *pd, ++ struct i915_page_table *pt, ++ unsigned int pde) ++{ ++ gen8_pde_t *vaddr; ++ ++ pd->page_table[pde] = pt; ++ ++ vaddr = kmap_atomic_px(pd); ++ vaddr[pde] = gen8_pde_encode(px_dma(pt), I915_CACHE_LLC); ++ kunmap_atomic(vaddr); ++} ++ ++static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm, ++ struct i915_page_directory *pd, ++ u64 start, u64 length) ++{ ++ struct i915_page_table *pt; ++ u32 pde; ++ ++ gen8_for_each_pde(pt, pd, start, length, pde) { ++ GEM_BUG_ON(pt == vm->scratch_pt); ++ ++ if (!gen8_ppgtt_clear_pt(vm, pt, start, length)) ++ continue; ++ ++ gen8_ppgtt_set_pde(vm, pd, vm->scratch_pt, pde); ++ GEM_BUG_ON(!pd->used_pdes); ++ pd->used_pdes--; ++ ++ free_pt(vm, pt); ++ } ++ ++ return !pd->used_pdes; ++} ++ ++static void gen8_ppgtt_set_pdpe(struct i915_address_space *vm, ++ struct i915_page_directory_pointer *pdp, ++ struct i915_page_directory *pd, ++ unsigned int pdpe) ++{ ++ gen8_ppgtt_pdpe_t *vaddr; ++ ++ pdp->page_directory[pdpe] = pd; ++ if (!i915_vm_is_4lvl(vm)) ++ return; ++ ++ vaddr = kmap_atomic_px(pdp); ++ vaddr[pdpe] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC); ++ kunmap_atomic(vaddr); ++} ++ ++/* Removes entries from a single page dir pointer, releasing it if it's empty. ++ * Caller can use the return value to update higher-level entries ++ */ ++static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm, ++ struct i915_page_directory_pointer *pdp, ++ u64 start, u64 length) ++{ ++ struct i915_page_directory *pd; ++ unsigned int pdpe; ++ ++ gen8_for_each_pdpe(pd, pdp, start, length, pdpe) { ++ GEM_BUG_ON(pd == vm->scratch_pd); ++ ++ if (!gen8_ppgtt_clear_pd(vm, pd, start, length)) ++ continue; ++ ++ gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe); ++ GEM_BUG_ON(!pdp->used_pdpes); ++ pdp->used_pdpes--; ++ ++ free_pd(vm, pd); ++ } ++ ++ return !pdp->used_pdpes; ++} ++ ++static void gen8_ppgtt_clear_3lvl(struct i915_address_space *vm, ++ u64 start, u64 length) ++{ ++ gen8_ppgtt_clear_pdp(vm, &i915_vm_to_ppgtt(vm)->pdp, start, length); ++} ++ ++static void gen8_ppgtt_set_pml4e(struct i915_pml4 *pml4, ++ struct i915_page_directory_pointer *pdp, ++ unsigned int pml4e) ++{ ++ gen8_ppgtt_pml4e_t *vaddr; ++ ++ pml4->pdps[pml4e] = pdp; ++ ++ vaddr = kmap_atomic_px(pml4); ++ vaddr[pml4e] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC); ++ kunmap_atomic(vaddr); ++} ++ ++/* Removes entries from a single pml4. ++ * This is the top-level structure in 4-level page tables used on gen8+. ++ * Empty entries are always scratch pml4e. ++ */ ++static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm, ++ u64 start, u64 length) ++{ ++ struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); ++ struct i915_pml4 *pml4 = &ppgtt->pml4; ++ struct i915_page_directory_pointer *pdp; ++ unsigned int pml4e; ++ ++ GEM_BUG_ON(!i915_vm_is_4lvl(vm)); ++ ++ gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) { ++ GEM_BUG_ON(pdp == vm->scratch_pdp); ++ ++ if (!gen8_ppgtt_clear_pdp(vm, pdp, start, length)) ++ continue; ++ ++ gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e); ++ ++ free_pdp(vm, pdp); ++ } ++} ++ ++static inline struct sgt_dma { ++ struct scatterlist *sg; ++ dma_addr_t dma, max; ++} sgt_dma(struct i915_vma *vma) { ++ struct scatterlist *sg = vma->pages->sgl; ++ dma_addr_t addr = sg_dma_address(sg); ++ return (struct sgt_dma) { sg, addr, addr + sg->length }; ++} ++ ++struct gen8_insert_pte { ++ u16 pml4e; ++ u16 pdpe; ++ u16 pde; ++ u16 pte; ++}; ++ ++static __always_inline struct gen8_insert_pte gen8_insert_pte(u64 start) ++{ ++ return (struct gen8_insert_pte) { ++ gen8_pml4e_index(start), ++ gen8_pdpe_index(start), ++ gen8_pde_index(start), ++ gen8_pte_index(start), ++ }; ++} ++ ++static __always_inline bool ++gen8_ppgtt_insert_pte_entries(struct i915_hw_ppgtt *ppgtt, ++ struct i915_page_directory_pointer *pdp, ++ struct sgt_dma *iter, ++ struct gen8_insert_pte *idx, ++ enum i915_cache_level cache_level, ++ u32 flags) ++{ ++ struct i915_page_directory *pd; ++ const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level, flags); ++ gen8_pte_t *vaddr; ++ bool ret; ++ ++ GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm)); ++ pd = pdp->page_directory[idx->pdpe]; ++ vaddr = kmap_atomic_px(pd->page_table[idx->pde]); ++ do { ++ vaddr[idx->pte] = pte_encode | iter->dma; ++ ++ iter->dma += I915_GTT_PAGE_SIZE; ++ if (iter->dma >= iter->max) { ++ iter->sg = __sg_next(iter->sg); ++ if (!iter->sg) { ++ ret = false; ++ break; ++ } ++ ++ iter->dma = sg_dma_address(iter->sg); ++ iter->max = iter->dma + iter->sg->length; ++ } ++ ++ if (++idx->pte == GEN8_PTES) { ++ idx->pte = 0; ++ ++ if (++idx->pde == I915_PDES) { ++ idx->pde = 0; ++ ++ /* Limited by sg length for 3lvl */ ++ if (++idx->pdpe == GEN8_PML4ES_PER_PML4) { ++ idx->pdpe = 0; ++ ret = true; ++ break; ++ } ++ ++ GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm)); ++ pd = pdp->page_directory[idx->pdpe]; ++ } ++ ++ kunmap_atomic(vaddr); ++ vaddr = kmap_atomic_px(pd->page_table[idx->pde]); ++ } ++ } while (1); ++ kunmap_atomic(vaddr); ++ ++ return ret; ++} ++ ++static void gen8_ppgtt_insert_3lvl(struct i915_address_space *vm, ++ struct i915_vma *vma, ++ enum i915_cache_level cache_level, ++ u32 flags) ++{ ++ struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); ++ struct sgt_dma iter = sgt_dma(vma); ++ struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start); ++ ++ gen8_ppgtt_insert_pte_entries(ppgtt, &ppgtt->pdp, &iter, &idx, ++ cache_level, flags); ++ ++ vma->page_sizes.gtt = I915_GTT_PAGE_SIZE; ++} ++ ++static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma, ++ struct i915_page_directory_pointer **pdps, ++ struct sgt_dma *iter, ++ enum i915_cache_level cache_level, ++ u32 flags) ++{ ++ const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level, flags); ++ u64 start = vma->node.start; ++ dma_addr_t rem = iter->sg->length; ++ ++ do { ++ struct gen8_insert_pte idx = gen8_insert_pte(start); ++ struct i915_page_directory_pointer *pdp = pdps[idx.pml4e]; ++ struct i915_page_directory *pd = pdp->page_directory[idx.pdpe]; ++ unsigned int page_size; ++ bool maybe_64K = false; ++ gen8_pte_t encode = pte_encode; ++ gen8_pte_t *vaddr; ++ u16 index, max; ++ ++ if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_2M && ++ IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_2M) && ++ rem >= I915_GTT_PAGE_SIZE_2M && !idx.pte) { ++ index = idx.pde; ++ max = I915_PDES; ++ page_size = I915_GTT_PAGE_SIZE_2M; ++ ++ encode |= GEN8_PDE_PS_2M; ++ ++ vaddr = kmap_atomic_px(pd); ++ } else { ++ struct i915_page_table *pt = pd->page_table[idx.pde]; ++ ++ index = idx.pte; ++ max = GEN8_PTES; ++ page_size = I915_GTT_PAGE_SIZE; ++ ++ if (!index && ++ vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K && ++ IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_64K) && ++ (IS_ALIGNED(rem, I915_GTT_PAGE_SIZE_64K) || ++ rem >= (max - index) * I915_GTT_PAGE_SIZE)) ++ maybe_64K = true; ++ ++ vaddr = kmap_atomic_px(pt); ++ } ++ ++ do { ++ GEM_BUG_ON(iter->sg->length < page_size); ++ vaddr[index++] = encode | iter->dma; ++ ++ start += page_size; ++ iter->dma += page_size; ++ rem -= page_size; ++ if (iter->dma >= iter->max) { ++ iter->sg = __sg_next(iter->sg); ++ if (!iter->sg) ++ break; ++ ++ rem = iter->sg->length; ++ iter->dma = sg_dma_address(iter->sg); ++ iter->max = iter->dma + rem; ++ ++ if (maybe_64K && index < max && ++ !(IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_64K) && ++ (IS_ALIGNED(rem, I915_GTT_PAGE_SIZE_64K) || ++ rem >= (max - index) * I915_GTT_PAGE_SIZE))) ++ maybe_64K = false; ++ ++ if (unlikely(!IS_ALIGNED(iter->dma, page_size))) ++ break; ++ } ++ } while (rem >= page_size && index < max); ++ ++ kunmap_atomic(vaddr); ++ ++ /* ++ * Is it safe to mark the 2M block as 64K? -- Either we have ++ * filled whole page-table with 64K entries, or filled part of ++ * it and have reached the end of the sg table and we have ++ * enough padding. ++ */ ++ if (maybe_64K && ++ (index == max || ++ (i915_vm_has_scratch_64K(vma->vm) && ++ !iter->sg && IS_ALIGNED(vma->node.start + ++ vma->node.size, ++ I915_GTT_PAGE_SIZE_2M)))) { ++ vaddr = kmap_atomic_px(pd); ++ vaddr[idx.pde] |= GEN8_PDE_IPS_64K; ++ kunmap_atomic(vaddr); ++ page_size = I915_GTT_PAGE_SIZE_64K; ++ ++ /* ++ * We write all 4K page entries, even when using 64K ++ * pages. In order to verify that the HW isn't cheating ++ * by using the 4K PTE instead of the 64K PTE, we want ++ * to remove all the surplus entries. If the HW skipped ++ * the 64K PTE, it will read/write into the scratch page ++ * instead - which we detect as missing results during ++ * selftests. ++ */ ++ if (I915_SELFTEST_ONLY(vma->vm->scrub_64K)) { ++ u16 i; ++ ++ encode = vma->vm->scratch_pte; ++ vaddr = kmap_atomic_px(pd->page_table[idx.pde]); ++ ++ for (i = 1; i < index; i += 16) ++ memset64(vaddr + i, encode, 15); ++ ++ kunmap_atomic(vaddr); ++ } ++ } ++ ++ vma->page_sizes.gtt |= page_size; ++ } while (iter->sg); ++} ++ ++static void gen8_ppgtt_insert_4lvl(struct i915_address_space *vm, ++ struct i915_vma *vma, ++ enum i915_cache_level cache_level, ++ u32 flags) ++{ ++ struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); ++ struct sgt_dma iter = sgt_dma(vma); ++ struct i915_page_directory_pointer **pdps = ppgtt->pml4.pdps; ++ ++ if (vma->page_sizes.sg > I915_GTT_PAGE_SIZE) { ++ gen8_ppgtt_insert_huge_entries(vma, pdps, &iter, cache_level, ++ flags); ++ } else { ++ struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start); ++ ++ while (gen8_ppgtt_insert_pte_entries(ppgtt, pdps[idx.pml4e++], ++ &iter, &idx, cache_level, ++ flags)) ++ GEM_BUG_ON(idx.pml4e >= GEN8_PML4ES_PER_PML4); ++ ++ vma->page_sizes.gtt = I915_GTT_PAGE_SIZE; ++ } ++} ++ ++static void gen8_free_page_tables(struct i915_address_space *vm, ++ struct i915_page_directory *pd) ++{ ++ int i; ++ ++ for (i = 0; i < I915_PDES; i++) { ++ if (pd->page_table[i] != vm->scratch_pt) ++ free_pt(vm, pd->page_table[i]); ++ } ++} ++ ++static int gen8_init_scratch(struct i915_address_space *vm) ++{ ++ int ret; ++ ++ /* ++ * If everybody agrees to not to write into the scratch page, ++ * we can reuse it for all vm, keeping contexts and processes separate. ++ */ ++ if (vm->has_read_only && ++ vm->i915->kernel_context && ++ vm->i915->kernel_context->ppgtt) { ++ struct i915_address_space *clone = ++ &vm->i915->kernel_context->ppgtt->vm; ++ ++ GEM_BUG_ON(!clone->has_read_only); ++ ++ vm->scratch_order = clone->scratch_order; ++ vm->scratch_pte = clone->scratch_pte; ++ vm->scratch_pt = clone->scratch_pt; ++ vm->scratch_pd = clone->scratch_pd; ++ vm->scratch_pdp = clone->scratch_pdp; ++ return 0; ++ } ++ ++ ret = setup_scratch_page(vm, __GFP_HIGHMEM); ++ if (ret) ++ return ret; ++ ++ vm->scratch_pte = ++ gen8_pte_encode(vm->scratch_page.daddr, ++ I915_CACHE_LLC, ++ vm->has_read_only); ++ ++ vm->scratch_pt = alloc_pt(vm); ++ if (IS_ERR(vm->scratch_pt)) { ++ ret = PTR_ERR(vm->scratch_pt); ++ goto free_scratch_page; ++ } ++ ++ vm->scratch_pd = alloc_pd(vm); ++ if (IS_ERR(vm->scratch_pd)) { ++ ret = PTR_ERR(vm->scratch_pd); ++ goto free_pt; ++ } ++ ++ if (i915_vm_is_4lvl(vm)) { ++ vm->scratch_pdp = alloc_pdp(vm); ++ if (IS_ERR(vm->scratch_pdp)) { ++ ret = PTR_ERR(vm->scratch_pdp); ++ goto free_pd; ++ } ++ } ++ ++ gen8_initialize_pt(vm, vm->scratch_pt); ++ gen8_initialize_pd(vm, vm->scratch_pd); ++ if (i915_vm_is_4lvl(vm)) ++ gen8_initialize_pdp(vm, vm->scratch_pdp); ++ ++ return 0; ++ ++free_pd: ++ free_pd(vm, vm->scratch_pd); ++free_pt: ++ free_pt(vm, vm->scratch_pt); ++free_scratch_page: ++ cleanup_scratch_page(vm); ++ ++ return ret; ++} ++ ++static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create) ++{ ++ struct i915_address_space *vm = &ppgtt->vm; ++ struct drm_i915_private *dev_priv = vm->i915; ++ enum vgt_g2v_type msg; ++ int i; ++ ++ if (i915_vm_is_4lvl(vm)) { ++ const u64 daddr = px_dma(&ppgtt->pml4); ++ ++ I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr)); ++ I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr)); ++ ++ msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE : ++ VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY); ++ } else { ++ for (i = 0; i < GEN8_3LVL_PDPES; i++) { ++ const u64 daddr = i915_page_dir_dma_addr(ppgtt, i); ++ ++ I915_WRITE(vgtif_reg(pdp[i].lo), lower_32_bits(daddr)); ++ I915_WRITE(vgtif_reg(pdp[i].hi), upper_32_bits(daddr)); ++ } ++ ++ msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE : ++ VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY); ++ } ++ ++ I915_WRITE(vgtif_reg(g2v_notify), msg); ++ ++ return 0; ++} ++ ++static void gen8_free_scratch(struct i915_address_space *vm) ++{ ++ if (!vm->scratch_page.daddr) ++ return; ++ ++ if (i915_vm_is_4lvl(vm)) ++ free_pdp(vm, vm->scratch_pdp); ++ free_pd(vm, vm->scratch_pd); ++ free_pt(vm, vm->scratch_pt); ++ cleanup_scratch_page(vm); ++} ++ ++static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm, ++ struct i915_page_directory_pointer *pdp) ++{ ++ const unsigned int pdpes = i915_pdpes_per_pdp(vm); ++ int i; ++ ++ for (i = 0; i < pdpes; i++) { ++ if (pdp->page_directory[i] == vm->scratch_pd) ++ continue; ++ ++ gen8_free_page_tables(vm, pdp->page_directory[i]); ++ free_pd(vm, pdp->page_directory[i]); ++ } ++ ++ free_pdp(vm, pdp); ++} ++ ++static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt) ++{ ++ int i; ++ ++ for (i = 0; i < GEN8_PML4ES_PER_PML4; i++) { ++ if (ppgtt->pml4.pdps[i] == ppgtt->vm.scratch_pdp) ++ continue; ++ ++ gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, ppgtt->pml4.pdps[i]); ++ } ++ ++ cleanup_px(&ppgtt->vm, &ppgtt->pml4); ++} ++ ++static void gen8_ppgtt_cleanup(struct i915_address_space *vm) ++{ ++ struct drm_i915_private *dev_priv = vm->i915; ++ struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); ++ ++ if (intel_vgpu_active(dev_priv)) ++ gen8_ppgtt_notify_vgt(ppgtt, false); ++ ++ if (i915_vm_is_4lvl(vm)) ++ gen8_ppgtt_cleanup_4lvl(ppgtt); ++ else ++ gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, &ppgtt->pdp); ++ ++ gen8_free_scratch(vm); ++} ++ ++static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm, ++ struct i915_page_directory *pd, ++ u64 start, u64 length) ++{ ++ struct i915_page_table *pt; ++ u64 from = start; ++ unsigned int pde; ++ ++ gen8_for_each_pde(pt, pd, start, length, pde) { ++ int count = gen8_pte_count(start, length); ++ ++ if (pt == vm->scratch_pt) { ++ pd->used_pdes++; ++ ++ pt = alloc_pt(vm); ++ if (IS_ERR(pt)) { ++ pd->used_pdes--; ++ goto unwind; ++ } ++ ++ if (count < GEN8_PTES || intel_vgpu_active(vm->i915)) ++ gen8_initialize_pt(vm, pt); ++ ++ gen8_ppgtt_set_pde(vm, pd, pt, pde); ++ GEM_BUG_ON(pd->used_pdes > I915_PDES); ++ } ++ ++ pt->used_ptes += count; ++ } ++ return 0; ++ ++unwind: ++ gen8_ppgtt_clear_pd(vm, pd, from, start - from); ++ return -ENOMEM; ++} ++ ++static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm, ++ struct i915_page_directory_pointer *pdp, ++ u64 start, u64 length) ++{ ++ struct i915_page_directory *pd; ++ u64 from = start; ++ unsigned int pdpe; ++ int ret; ++ ++ gen8_for_each_pdpe(pd, pdp, start, length, pdpe) { ++ if (pd == vm->scratch_pd) { ++ pdp->used_pdpes++; ++ ++ pd = alloc_pd(vm); ++ if (IS_ERR(pd)) { ++ pdp->used_pdpes--; ++ goto unwind; ++ } ++ ++ gen8_initialize_pd(vm, pd); ++ gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe); ++ GEM_BUG_ON(pdp->used_pdpes > i915_pdpes_per_pdp(vm)); ++ } ++ ++ ret = gen8_ppgtt_alloc_pd(vm, pd, start, length); ++ if (unlikely(ret)) ++ goto unwind_pd; ++ } ++ ++ return 0; ++ ++unwind_pd: ++ if (!pd->used_pdes) { ++ gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe); ++ GEM_BUG_ON(!pdp->used_pdpes); ++ pdp->used_pdpes--; ++ free_pd(vm, pd); ++ } ++unwind: ++ gen8_ppgtt_clear_pdp(vm, pdp, from, start - from); ++ return -ENOMEM; ++} ++ ++static int gen8_ppgtt_alloc_3lvl(struct i915_address_space *vm, ++ u64 start, u64 length) ++{ ++ return gen8_ppgtt_alloc_pdp(vm, ++ &i915_vm_to_ppgtt(vm)->pdp, start, length); ++} ++ ++static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm, ++ u64 start, u64 length) ++{ ++ struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); ++ struct i915_pml4 *pml4 = &ppgtt->pml4; ++ struct i915_page_directory_pointer *pdp; ++ u64 from = start; ++ u32 pml4e; ++ int ret; ++ ++ gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) { ++ if (pml4->pdps[pml4e] == vm->scratch_pdp) { ++ pdp = alloc_pdp(vm); ++ if (IS_ERR(pdp)) ++ goto unwind; ++ ++ gen8_initialize_pdp(vm, pdp); ++ gen8_ppgtt_set_pml4e(pml4, pdp, pml4e); ++ } ++ ++ ret = gen8_ppgtt_alloc_pdp(vm, pdp, start, length); ++ if (unlikely(ret)) ++ goto unwind_pdp; ++ } ++ ++ return 0; ++ ++unwind_pdp: ++ if (!pdp->used_pdpes) { ++ gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e); ++ free_pdp(vm, pdp); ++ } ++unwind: ++ gen8_ppgtt_clear_4lvl(vm, from, start - from); ++ return -ENOMEM; ++} ++ ++static int gen8_preallocate_top_level_pdp(struct i915_hw_ppgtt *ppgtt) ++{ ++ struct i915_address_space *vm = &ppgtt->vm; ++ struct i915_page_directory_pointer *pdp = &ppgtt->pdp; ++ struct i915_page_directory *pd; ++ u64 start = 0, length = ppgtt->vm.total; ++ u64 from = start; ++ unsigned int pdpe; ++ ++ gen8_for_each_pdpe(pd, pdp, start, length, pdpe) { ++ pd = alloc_pd(vm); ++ if (IS_ERR(pd)) ++ goto unwind; ++ ++ gen8_initialize_pd(vm, pd); ++ gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe); ++ pdp->used_pdpes++; ++ } ++ ++ pdp->used_pdpes++; /* never remove */ ++ return 0; ++ ++unwind: ++ start -= from; ++ gen8_for_each_pdpe(pd, pdp, from, start, pdpe) { ++ gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe); ++ free_pd(vm, pd); ++ } ++ pdp->used_pdpes = 0; ++ return -ENOMEM; ++} ++ ++static void ppgtt_init(struct drm_i915_private *i915, ++ struct i915_hw_ppgtt *ppgtt) ++{ ++ kref_init(&ppgtt->ref); ++ ++ ppgtt->vm.i915 = i915; ++ ppgtt->vm.dma = &i915->drm.pdev->dev; ++ ppgtt->vm.total = BIT_ULL(INTEL_INFO(i915)->ppgtt_size); ++ ++ i915_address_space_init(&ppgtt->vm, VM_CLASS_PPGTT); ++ ++ ppgtt->vm.vma_ops.bind_vma = ppgtt_bind_vma; ++ ppgtt->vm.vma_ops.unbind_vma = ppgtt_unbind_vma; ++ ppgtt->vm.vma_ops.set_pages = ppgtt_set_pages; ++ ppgtt->vm.vma_ops.clear_pages = clear_pages; ++} ++ ++/* ++ * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers ++ * with a net effect resembling a 2-level page table in normal x86 terms. Each ++ * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address ++ * space. ++ * ++ */ ++static struct i915_hw_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915) ++{ ++ struct i915_hw_ppgtt *ppgtt; ++ int err; ++ ++ ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); ++ if (!ppgtt) ++ return ERR_PTR(-ENOMEM); ++ ++ ppgtt_init(i915, ppgtt); ++ ++ /* ++ * From bdw, there is hw support for read-only pages in the PPGTT. ++ * ++ * Gen11 has HSDES#:1807136187 unresolved. Disable ro support ++ * for now. ++ */ ++ ppgtt->vm.has_read_only = INTEL_GEN(i915) != 11; ++ ++ /* There are only few exceptions for gen >=6. chv and bxt. ++ * And we are not sure about the latter so play safe for now. ++ */ ++ if (IS_CHERRYVIEW(i915) || IS_BROXTON(i915)) ++ ppgtt->vm.pt_kmap_wc = true; ++ ++ err = gen8_init_scratch(&ppgtt->vm); ++ if (err) ++ goto err_free; ++ ++ if (i915_vm_is_4lvl(&ppgtt->vm)) { ++ err = setup_px(&ppgtt->vm, &ppgtt->pml4); ++ if (err) ++ goto err_scratch; ++ ++ gen8_initialize_pml4(&ppgtt->vm, &ppgtt->pml4); ++ ++ ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_4lvl; ++ ppgtt->vm.insert_entries = gen8_ppgtt_insert_4lvl; ++ ppgtt->vm.clear_range = gen8_ppgtt_clear_4lvl; ++ } else { ++ err = __pdp_init(&ppgtt->vm, &ppgtt->pdp); ++ if (err) ++ goto err_scratch; ++ ++ if (intel_vgpu_active(i915)) { ++ err = gen8_preallocate_top_level_pdp(ppgtt); ++ if (err) { ++ __pdp_fini(&ppgtt->pdp); ++ goto err_scratch; ++ } ++ } ++ ++ ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_3lvl; ++ ppgtt->vm.insert_entries = gen8_ppgtt_insert_3lvl; ++ ppgtt->vm.clear_range = gen8_ppgtt_clear_3lvl; ++ } ++ ++ if (intel_vgpu_active(i915)) ++ gen8_ppgtt_notify_vgt(ppgtt, true); ++ ++ ppgtt->vm.cleanup = gen8_ppgtt_cleanup; ++ ++ return ppgtt; ++ ++err_scratch: ++ gen8_free_scratch(&ppgtt->vm); ++err_free: ++ kfree(ppgtt); ++ return ERR_PTR(err); ++} ++ ++/* Write pde (index) from the page directory @pd to the page table @pt */ ++static inline void gen6_write_pde(const struct gen6_hw_ppgtt *ppgtt, ++ const unsigned int pde, ++ const struct i915_page_table *pt) ++{ ++ /* Caller needs to make sure the write completes if necessary */ ++ iowrite32(GEN6_PDE_ADDR_ENCODE(px_dma(pt)) | GEN6_PDE_VALID, ++ ppgtt->pd_addr + pde); ++} ++ ++static void gen7_ppgtt_enable(struct drm_i915_private *dev_priv) ++{ ++ struct intel_engine_cs *engine; ++ u32 ecochk, ecobits; ++ enum intel_engine_id id; ++ ++ ecobits = I915_READ(GAC_ECO_BITS); ++ I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); ++ ++ ecochk = I915_READ(GAM_ECOCHK); ++ if (IS_HASWELL(dev_priv)) { ++ ecochk |= ECOCHK_PPGTT_WB_HSW; ++ } else { ++ ecochk |= ECOCHK_PPGTT_LLC_IVB; ++ ecochk &= ~ECOCHK_PPGTT_GFDT_IVB; ++ } ++ I915_WRITE(GAM_ECOCHK, ecochk); ++ ++ for_each_engine(engine, dev_priv, id) { ++ /* GFX_MODE is per-ring on gen7+ */ ++ I915_WRITE(RING_MODE_GEN7(engine), ++ _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); ++ } ++} ++ ++static void gen6_ppgtt_enable(struct drm_i915_private *dev_priv) ++{ ++ u32 ecochk, gab_ctl, ecobits; ++ ++ ecobits = I915_READ(GAC_ECO_BITS); ++ I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT | ++ ECOBITS_PPGTT_CACHE64B); ++ ++ gab_ctl = I915_READ(GAB_CTL); ++ I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); ++ ++ ecochk = I915_READ(GAM_ECOCHK); ++ I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B); ++ ++ if (HAS_PPGTT(dev_priv)) /* may be disabled for VT-d */ ++ I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); ++} ++ ++/* PPGTT support for Sandybdrige/Gen6 and later */ ++static void gen6_ppgtt_clear_range(struct i915_address_space *vm, ++ u64 start, u64 length) ++{ ++ struct gen6_hw_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm)); ++ unsigned int first_entry = start / I915_GTT_PAGE_SIZE; ++ unsigned int pde = first_entry / GEN6_PTES; ++ unsigned int pte = first_entry % GEN6_PTES; ++ unsigned int num_entries = length / I915_GTT_PAGE_SIZE; ++ const gen6_pte_t scratch_pte = vm->scratch_pte; ++ ++ while (num_entries) { ++ struct i915_page_table *pt = ppgtt->base.pd.page_table[pde++]; ++ const unsigned int count = min(num_entries, GEN6_PTES - pte); ++ gen6_pte_t *vaddr; ++ ++ GEM_BUG_ON(pt == vm->scratch_pt); ++ ++ num_entries -= count; ++ ++ GEM_BUG_ON(count > pt->used_ptes); ++ pt->used_ptes -= count; ++ if (!pt->used_ptes) ++ ppgtt->scan_for_unused_pt = true; ++ ++ /* ++ * Note that the hw doesn't support removing PDE on the fly ++ * (they are cached inside the context with no means to ++ * invalidate the cache), so we can only reset the PTE ++ * entries back to scratch. ++ */ ++ ++ vaddr = kmap_atomic_px(pt); ++ memset32(vaddr + pte, scratch_pte, count); ++ kunmap_atomic(vaddr); ++ ++ pte = 0; ++ } ++} ++ ++static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, ++ struct i915_vma *vma, ++ enum i915_cache_level cache_level, ++ u32 flags) ++{ ++ struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); ++ unsigned first_entry = vma->node.start / I915_GTT_PAGE_SIZE; ++ unsigned act_pt = first_entry / GEN6_PTES; ++ unsigned act_pte = first_entry % GEN6_PTES; ++ const u32 pte_encode = vm->pte_encode(0, cache_level, flags); ++ struct sgt_dma iter = sgt_dma(vma); ++ gen6_pte_t *vaddr; ++ ++ GEM_BUG_ON(ppgtt->pd.page_table[act_pt] == vm->scratch_pt); ++ ++ vaddr = kmap_atomic_px(ppgtt->pd.page_table[act_pt]); ++ do { ++ vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma); ++ ++ iter.dma += I915_GTT_PAGE_SIZE; ++ if (iter.dma == iter.max) { ++ iter.sg = __sg_next(iter.sg); ++ if (!iter.sg) ++ break; ++ ++ iter.dma = sg_dma_address(iter.sg); ++ iter.max = iter.dma + iter.sg->length; ++ } ++ ++ if (++act_pte == GEN6_PTES) { ++ kunmap_atomic(vaddr); ++ vaddr = kmap_atomic_px(ppgtt->pd.page_table[++act_pt]); ++ act_pte = 0; ++ } ++ } while (1); ++ kunmap_atomic(vaddr); ++ ++ vma->page_sizes.gtt = I915_GTT_PAGE_SIZE; ++} ++ ++static int gen6_alloc_va_range(struct i915_address_space *vm, ++ u64 start, u64 length) ++{ ++ struct gen6_hw_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm)); ++ struct i915_page_table *pt; ++ u64 from = start; ++ unsigned int pde; ++ bool flush = false; ++ ++ gen6_for_each_pde(pt, &ppgtt->base.pd, start, length, pde) { ++ const unsigned int count = gen6_pte_count(start, length); ++ ++ if (pt == vm->scratch_pt) { ++ pt = alloc_pt(vm); ++ if (IS_ERR(pt)) ++ goto unwind_out; ++ ++ gen6_initialize_pt(vm, pt); ++ ppgtt->base.pd.page_table[pde] = pt; ++ ++ if (i915_vma_is_bound(ppgtt->vma, ++ I915_VMA_GLOBAL_BIND)) { ++ gen6_write_pde(ppgtt, pde, pt); ++ flush = true; ++ } ++ ++ GEM_BUG_ON(pt->used_ptes); ++ } ++ ++ pt->used_ptes += count; ++ } ++ ++ if (flush) { ++ mark_tlbs_dirty(&ppgtt->base); ++ gen6_ggtt_invalidate(ppgtt->base.vm.i915); ++ } ++ ++ return 0; ++ ++unwind_out: ++ gen6_ppgtt_clear_range(vm, from, start - from); ++ return -ENOMEM; ++} ++ ++static int gen6_ppgtt_init_scratch(struct gen6_hw_ppgtt *ppgtt) ++{ ++ struct i915_address_space * const vm = &ppgtt->base.vm; ++ struct i915_page_table *unused; ++ u32 pde; ++ int ret; ++ ++ ret = setup_scratch_page(vm, __GFP_HIGHMEM); ++ if (ret) ++ return ret; ++ ++ vm->scratch_pte = vm->pte_encode(vm->scratch_page.daddr, ++ I915_CACHE_NONE, ++ PTE_READ_ONLY); ++ ++ vm->scratch_pt = alloc_pt(vm); ++ if (IS_ERR(vm->scratch_pt)) { ++ cleanup_scratch_page(vm); ++ return PTR_ERR(vm->scratch_pt); ++ } ++ ++ gen6_initialize_pt(vm, vm->scratch_pt); ++ gen6_for_all_pdes(unused, &ppgtt->base.pd, pde) ++ ppgtt->base.pd.page_table[pde] = vm->scratch_pt; ++ ++ return 0; ++} ++ ++static void gen6_ppgtt_free_scratch(struct i915_address_space *vm) ++{ ++ free_pt(vm, vm->scratch_pt); ++ cleanup_scratch_page(vm); ++} ++ ++static void gen6_ppgtt_free_pd(struct gen6_hw_ppgtt *ppgtt) ++{ ++ struct i915_page_table *pt; ++ u32 pde; ++ ++ gen6_for_all_pdes(pt, &ppgtt->base.pd, pde) ++ if (pt != ppgtt->base.vm.scratch_pt) ++ free_pt(&ppgtt->base.vm, pt); ++} ++ ++static void gen6_ppgtt_cleanup(struct i915_address_space *vm) ++{ ++ struct gen6_hw_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm)); ++ ++ i915_vma_destroy(ppgtt->vma); ++ ++ gen6_ppgtt_free_pd(ppgtt); ++ gen6_ppgtt_free_scratch(vm); ++} ++ ++static int pd_vma_set_pages(struct i915_vma *vma) ++{ ++ vma->pages = ERR_PTR(-ENODEV); ++ return 0; ++} ++ ++static void pd_vma_clear_pages(struct i915_vma *vma) ++{ ++ GEM_BUG_ON(!vma->pages); ++ ++ vma->pages = NULL; ++} ++ ++static int pd_vma_bind(struct i915_vma *vma, ++ enum i915_cache_level cache_level, ++ u32 unused) ++{ ++ struct i915_ggtt *ggtt = i915_vm_to_ggtt(vma->vm); ++ struct gen6_hw_ppgtt *ppgtt = vma->private; ++ u32 ggtt_offset = i915_ggtt_offset(vma) / I915_GTT_PAGE_SIZE; ++ struct i915_page_table *pt; ++ unsigned int pde; ++ ++ ppgtt->base.pd.base.ggtt_offset = ggtt_offset * sizeof(gen6_pte_t); ++ ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm + ggtt_offset; ++ ++ gen6_for_all_pdes(pt, &ppgtt->base.pd, pde) ++ gen6_write_pde(ppgtt, pde, pt); ++ ++ mark_tlbs_dirty(&ppgtt->base); ++ gen6_ggtt_invalidate(ppgtt->base.vm.i915); ++ ++ return 0; ++} ++ ++static void pd_vma_unbind(struct i915_vma *vma) ++{ ++ struct gen6_hw_ppgtt *ppgtt = vma->private; ++ struct i915_page_table * const scratch_pt = ppgtt->base.vm.scratch_pt; ++ struct i915_page_table *pt; ++ unsigned int pde; ++ ++ if (!ppgtt->scan_for_unused_pt) ++ return; ++ ++ /* Free all no longer used page tables */ ++ gen6_for_all_pdes(pt, &ppgtt->base.pd, pde) { ++ if (pt->used_ptes || pt == scratch_pt) ++ continue; ++ ++ free_pt(&ppgtt->base.vm, pt); ++ ppgtt->base.pd.page_table[pde] = scratch_pt; ++ } ++ ++ ppgtt->scan_for_unused_pt = false; ++} ++ ++static const struct i915_vma_ops pd_vma_ops = { ++ .set_pages = pd_vma_set_pages, ++ .clear_pages = pd_vma_clear_pages, ++ .bind_vma = pd_vma_bind, ++ .unbind_vma = pd_vma_unbind, ++}; ++ ++static struct i915_vma *pd_vma_create(struct gen6_hw_ppgtt *ppgtt, int size) ++{ ++ struct drm_i915_private *i915 = ppgtt->base.vm.i915; ++ struct i915_ggtt *ggtt = &i915->ggtt; ++ struct i915_vma *vma; ++ ++ GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); ++ GEM_BUG_ON(size > ggtt->vm.total); ++ ++ vma = i915_vma_alloc(); ++ if (!vma) ++ return ERR_PTR(-ENOMEM); ++ ++ i915_active_init(i915, &vma->active, NULL); ++ INIT_ACTIVE_REQUEST(&vma->last_fence); ++ ++ vma->vm = &ggtt->vm; ++ vma->ops = &pd_vma_ops; ++ vma->private = ppgtt; ++ ++ vma->size = size; ++ vma->fence_size = size; ++ vma->flags = I915_VMA_GGTT; ++ vma->ggtt_view.type = I915_GGTT_VIEW_ROTATED; /* prevent fencing */ ++ ++ INIT_LIST_HEAD(&vma->obj_link); ++ ++ mutex_lock(&vma->vm->mutex); ++ list_add(&vma->vm_link, &vma->vm->unbound_list); ++ mutex_unlock(&vma->vm->mutex); ++ ++ return vma; ++} ++ ++int gen6_ppgtt_pin(struct i915_hw_ppgtt *base) ++{ ++ struct gen6_hw_ppgtt *ppgtt = to_gen6_ppgtt(base); ++ int err; ++ ++ GEM_BUG_ON(ppgtt->base.vm.closed); ++ ++ /* ++ * Workaround the limited maximum vma->pin_count and the aliasing_ppgtt ++ * which will be pinned into every active context. ++ * (When vma->pin_count becomes atomic, I expect we will naturally ++ * need a larger, unpacked, type and kill this redundancy.) ++ */ ++ if (ppgtt->pin_count++) ++ return 0; ++ ++ /* ++ * PPGTT PDEs reside in the GGTT and consists of 512 entries. The ++ * allocator works in address space sizes, so it's multiplied by page ++ * size. We allocate at the top of the GTT to avoid fragmentation. ++ */ ++ err = i915_vma_pin(ppgtt->vma, ++ 0, GEN6_PD_ALIGN, ++ PIN_GLOBAL | PIN_HIGH); ++ if (err) ++ goto unpin; ++ ++ return 0; ++ ++unpin: ++ ppgtt->pin_count = 0; ++ return err; ++} ++ ++void gen6_ppgtt_unpin(struct i915_hw_ppgtt *base) ++{ ++ struct gen6_hw_ppgtt *ppgtt = to_gen6_ppgtt(base); ++ ++ GEM_BUG_ON(!ppgtt->pin_count); ++ if (--ppgtt->pin_count) ++ return; ++ ++ i915_vma_unpin(ppgtt->vma); ++} ++ ++void gen6_ppgtt_unpin_all(struct i915_hw_ppgtt *base) ++{ ++ struct gen6_hw_ppgtt *ppgtt = to_gen6_ppgtt(base); ++ ++ if (!ppgtt->pin_count) ++ return; ++ ++ ppgtt->pin_count = 0; ++ i915_vma_unpin(ppgtt->vma); ++} ++ ++static struct i915_hw_ppgtt *gen6_ppgtt_create(struct drm_i915_private *i915) ++{ ++ struct i915_ggtt * const ggtt = &i915->ggtt; ++ struct gen6_hw_ppgtt *ppgtt; ++ int err; ++ ++ ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); ++ if (!ppgtt) ++ return ERR_PTR(-ENOMEM); ++ ++ ppgtt_init(i915, &ppgtt->base); ++ ++ ppgtt->base.vm.allocate_va_range = gen6_alloc_va_range; ++ ppgtt->base.vm.clear_range = gen6_ppgtt_clear_range; ++ ppgtt->base.vm.insert_entries = gen6_ppgtt_insert_entries; ++ ppgtt->base.vm.cleanup = gen6_ppgtt_cleanup; ++ ++ ppgtt->base.vm.pte_encode = ggtt->vm.pte_encode; ++ ++ err = gen6_ppgtt_init_scratch(ppgtt); ++ if (err) ++ goto err_free; ++ ++ ppgtt->vma = pd_vma_create(ppgtt, GEN6_PD_SIZE); ++ if (IS_ERR(ppgtt->vma)) { ++ err = PTR_ERR(ppgtt->vma); ++ goto err_scratch; ++ } ++ ++ return &ppgtt->base; ++ ++err_scratch: ++ gen6_ppgtt_free_scratch(&ppgtt->base.vm); ++err_free: ++ kfree(ppgtt); ++ return ERR_PTR(err); ++} ++ ++static void gtt_write_workarounds(struct drm_i915_private *dev_priv) ++{ ++ /* This function is for gtt related workarounds. This function is ++ * called on driver load and after a GPU reset, so you can place ++ * workarounds here even if they get overwritten by GPU reset. ++ */ ++ /* WaIncreaseDefaultTLBEntries:chv,bdw,skl,bxt,kbl,glk,cfl,cnl,icl */ ++ if (IS_BROADWELL(dev_priv)) ++ I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_BDW); ++ else if (IS_CHERRYVIEW(dev_priv)) ++ I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_CHV); ++ else if (IS_GEN9_LP(dev_priv)) ++ I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT); ++ else if (INTEL_GEN(dev_priv) >= 9) ++ I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_SKL); ++ ++ /* ++ * To support 64K PTEs we need to first enable the use of the ++ * Intermediate-Page-Size(IPS) bit of the PDE field via some magical ++ * mmio, otherwise the page-walker will simply ignore the IPS bit. This ++ * shouldn't be needed after GEN10. ++ * ++ * 64K pages were first introduced from BDW+, although technically they ++ * only *work* from gen9+. For pre-BDW we instead have the option for ++ * 32K pages, but we don't currently have any support for it in our ++ * driver. ++ */ ++ if (HAS_PAGE_SIZES(dev_priv, I915_GTT_PAGE_SIZE_64K) && ++ INTEL_GEN(dev_priv) <= 10) ++ I915_WRITE(GEN8_GAMW_ECO_DEV_RW_IA, ++ I915_READ(GEN8_GAMW_ECO_DEV_RW_IA) | ++ GAMW_ECO_ENABLE_64K_IPS_FIELD); ++} ++ ++int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv) ++{ ++ gtt_write_workarounds(dev_priv); ++ ++ if (IS_GEN(dev_priv, 6)) ++ gen6_ppgtt_enable(dev_priv); ++ else if (IS_GEN(dev_priv, 7)) ++ gen7_ppgtt_enable(dev_priv); ++ ++ return 0; ++} ++ ++static struct i915_hw_ppgtt * ++__hw_ppgtt_create(struct drm_i915_private *i915) ++{ ++ if (INTEL_GEN(i915) < 8) ++ return gen6_ppgtt_create(i915); ++ else ++ return gen8_ppgtt_create(i915); ++} ++ ++struct i915_hw_ppgtt * ++i915_ppgtt_create(struct drm_i915_private *i915) ++{ ++ struct i915_hw_ppgtt *ppgtt; ++ ++ ppgtt = __hw_ppgtt_create(i915); ++ if (IS_ERR(ppgtt)) ++ return ppgtt; ++ ++ trace_i915_ppgtt_create(&ppgtt->vm); ++ ++ return ppgtt; ++} ++ ++static void ppgtt_destroy_vma(struct i915_address_space *vm) ++{ ++ struct list_head *phases[] = { ++ &vm->bound_list, ++ &vm->unbound_list, ++ NULL, ++ }, **phase; ++ ++ vm->closed = true; ++ for (phase = phases; *phase; phase++) { ++ struct i915_vma *vma, *vn; ++ ++ list_for_each_entry_safe(vma, vn, *phase, vm_link) ++ i915_vma_destroy(vma); ++ } ++} ++ ++void i915_ppgtt_release(struct kref *kref) ++{ ++ struct i915_hw_ppgtt *ppgtt = ++ container_of(kref, struct i915_hw_ppgtt, ref); ++ ++ trace_i915_ppgtt_release(&ppgtt->vm); ++ ++ ppgtt_destroy_vma(&ppgtt->vm); ++ ++ GEM_BUG_ON(!list_empty(&ppgtt->vm.bound_list)); ++ GEM_BUG_ON(!list_empty(&ppgtt->vm.unbound_list)); ++ ++ ppgtt->vm.cleanup(&ppgtt->vm); ++ i915_address_space_fini(&ppgtt->vm); ++ kfree(ppgtt); ++} ++ ++/* Certain Gen5 chipsets require require idling the GPU before ++ * unmapping anything from the GTT when VT-d is enabled. ++ */ ++static bool needs_idle_maps(struct drm_i915_private *dev_priv) ++{ ++ /* Query intel_iommu to see if we need the workaround. Presumably that ++ * was loaded first. ++ */ ++ return IS_GEN(dev_priv, 5) && IS_MOBILE(dev_priv) && intel_vtd_active(); ++} ++ ++static void gen6_check_faults(struct drm_i915_private *dev_priv) ++{ ++ struct intel_engine_cs *engine; ++ enum intel_engine_id id; ++ u32 fault; ++ ++ for_each_engine(engine, dev_priv, id) { ++ fault = I915_READ(RING_FAULT_REG(engine)); ++ if (fault & RING_FAULT_VALID) { ++ DRM_DEBUG_DRIVER("Unexpected fault\n" ++ "\tAddr: 0x%08lx\n" ++ "\tAddress space: %s\n" ++ "\tSource ID: %d\n" ++ "\tType: %d\n", ++ fault & PAGE_MASK, ++ fault & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT", ++ RING_FAULT_SRCID(fault), ++ RING_FAULT_FAULT_TYPE(fault)); ++ } ++ } ++} ++ ++static void gen8_check_faults(struct drm_i915_private *dev_priv) ++{ ++ u32 fault = I915_READ(GEN8_RING_FAULT_REG); ++ ++ if (fault & RING_FAULT_VALID) { ++ u32 fault_data0, fault_data1; ++ u64 fault_addr; ++ ++ fault_data0 = I915_READ(GEN8_FAULT_TLB_DATA0); ++ fault_data1 = I915_READ(GEN8_FAULT_TLB_DATA1); ++ fault_addr = ((u64)(fault_data1 & FAULT_VA_HIGH_BITS) << 44) | ++ ((u64)fault_data0 << 12); ++ ++ DRM_DEBUG_DRIVER("Unexpected fault\n" ++ "\tAddr: 0x%08x_%08x\n" ++ "\tAddress space: %s\n" ++ "\tEngine ID: %d\n" ++ "\tSource ID: %d\n" ++ "\tType: %d\n", ++ upper_32_bits(fault_addr), ++ lower_32_bits(fault_addr), ++ fault_data1 & FAULT_GTT_SEL ? "GGTT" : "PPGTT", ++ GEN8_RING_FAULT_ENGINE_ID(fault), ++ RING_FAULT_SRCID(fault), ++ RING_FAULT_FAULT_TYPE(fault)); ++ } ++} ++ ++void i915_check_and_clear_faults(struct drm_i915_private *dev_priv) ++{ ++ /* From GEN8 onwards we only have one 'All Engine Fault Register' */ ++ if (INTEL_GEN(dev_priv) >= 8) ++ gen8_check_faults(dev_priv); ++ else if (INTEL_GEN(dev_priv) >= 6) ++ gen6_check_faults(dev_priv); ++ else ++ return; ++ ++ i915_clear_error_registers(dev_priv); ++} ++ ++void i915_gem_suspend_gtt_mappings(struct drm_i915_private *dev_priv) ++{ ++ struct i915_ggtt *ggtt = &dev_priv->ggtt; ++ ++ /* Don't bother messing with faults pre GEN6 as we have little ++ * documentation supporting that it's a good idea. ++ */ ++ if (INTEL_GEN(dev_priv) < 6) ++ return; ++ ++ i915_check_and_clear_faults(dev_priv); ++ ++ ggtt->vm.clear_range(&ggtt->vm, 0, ggtt->vm.total); ++ ++ i915_ggtt_invalidate(dev_priv); ++} ++ ++int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj, ++ struct sg_table *pages) ++{ ++ do { ++ if (dma_map_sg_attrs(&obj->base.dev->pdev->dev, ++ pages->sgl, pages->nents, ++ PCI_DMA_BIDIRECTIONAL, ++ DMA_ATTR_NO_WARN)) ++ return 0; ++ ++ /* ++ * If the DMA remap fails, one cause can be that we have ++ * too many objects pinned in a small remapping table, ++ * such as swiotlb. Incrementally purge all other objects and ++ * try again - if there are no more pages to remove from ++ * the DMA remapper, i915_gem_shrink will return 0. ++ */ ++ GEM_BUG_ON(obj->mm.pages == pages); ++ } while (i915_gem_shrink(to_i915(obj->base.dev), ++ obj->base.size >> PAGE_SHIFT, NULL, ++ I915_SHRINK_BOUND | ++ I915_SHRINK_UNBOUND)); ++ ++ return -ENOSPC; ++} ++ ++static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) ++{ ++ writeq(pte, addr); ++} ++ ++static void gen8_ggtt_insert_page(struct i915_address_space *vm, ++ dma_addr_t addr, ++ u64 offset, ++ enum i915_cache_level level, ++ u32 unused) ++{ ++ struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); ++ gen8_pte_t __iomem *pte = ++ (gen8_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE; ++ ++ gen8_set_pte(pte, gen8_pte_encode(addr, level, 0)); ++ ++ ggtt->invalidate(vm->i915); ++} ++ ++static void gen8_ggtt_insert_entries(struct i915_address_space *vm, ++ struct i915_vma *vma, ++ enum i915_cache_level level, ++ u32 flags) ++{ ++ struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); ++ struct sgt_iter sgt_iter; ++ gen8_pte_t __iomem *gtt_entries; ++ const gen8_pte_t pte_encode = gen8_pte_encode(0, level, 0); ++ dma_addr_t addr; ++ ++ /* ++ * Note that we ignore PTE_READ_ONLY here. The caller must be careful ++ * not to allow the user to override access to a read only page. ++ */ ++ ++ gtt_entries = (gen8_pte_t __iomem *)ggtt->gsm; ++ gtt_entries += vma->node.start / I915_GTT_PAGE_SIZE; ++ for_each_sgt_dma(addr, sgt_iter, vma->pages) ++ gen8_set_pte(gtt_entries++, pte_encode | addr); ++ ++ /* ++ * We want to flush the TLBs only after we're certain all the PTE ++ * updates have finished. ++ */ ++ ggtt->invalidate(vm->i915); ++} ++ ++static void gen6_ggtt_insert_page(struct i915_address_space *vm, ++ dma_addr_t addr, ++ u64 offset, ++ enum i915_cache_level level, ++ u32 flags) ++{ ++ struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); ++ gen6_pte_t __iomem *pte = ++ (gen6_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE; ++ ++ iowrite32(vm->pte_encode(addr, level, flags), pte); ++ ++ ggtt->invalidate(vm->i915); ++} ++ ++/* ++ * Binds an object into the global gtt with the specified cache level. The object ++ * will be accessible to the GPU via commands whose operands reference offsets ++ * within the global GTT as well as accessible by the GPU through the GMADR ++ * mapped BAR (dev_priv->mm.gtt->gtt). ++ */ ++static void gen6_ggtt_insert_entries(struct i915_address_space *vm, ++ struct i915_vma *vma, ++ enum i915_cache_level level, ++ u32 flags) ++{ ++ struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); ++ gen6_pte_t __iomem *entries = (gen6_pte_t __iomem *)ggtt->gsm; ++ unsigned int i = vma->node.start / I915_GTT_PAGE_SIZE; ++ struct sgt_iter iter; ++ dma_addr_t addr; ++ for_each_sgt_dma(addr, iter, vma->pages) ++ iowrite32(vm->pte_encode(addr, level, flags), &entries[i++]); ++ ++ /* ++ * We want to flush the TLBs only after we're certain all the PTE ++ * updates have finished. ++ */ ++ ggtt->invalidate(vm->i915); ++} ++ ++static void nop_clear_range(struct i915_address_space *vm, ++ u64 start, u64 length) ++{ ++} ++ ++static void gen8_ggtt_clear_range(struct i915_address_space *vm, ++ u64 start, u64 length) ++{ ++ struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); ++ unsigned first_entry = start / I915_GTT_PAGE_SIZE; ++ unsigned num_entries = length / I915_GTT_PAGE_SIZE; ++ const gen8_pte_t scratch_pte = vm->scratch_pte; ++ gen8_pte_t __iomem *gtt_base = ++ (gen8_pte_t __iomem *)ggtt->gsm + first_entry; ++ const int max_entries = ggtt_total_entries(ggtt) - first_entry; ++ int i; ++ ++ if (WARN(num_entries > max_entries, ++ "First entry = %d; Num entries = %d (max=%d)\n", ++ first_entry, num_entries, max_entries)) ++ num_entries = max_entries; ++ ++ for (i = 0; i < num_entries; i++) ++ gen8_set_pte(>t_base[i], scratch_pte); ++} ++ ++static void bxt_vtd_ggtt_wa(struct i915_address_space *vm) ++{ ++ struct drm_i915_private *dev_priv = vm->i915; ++ ++ /* ++ * Make sure the internal GAM fifo has been cleared of all GTT ++ * writes before exiting stop_machine(). This guarantees that ++ * any aperture accesses waiting to start in another process ++ * cannot back up behind the GTT writes causing a hang. ++ * The register can be any arbitrary GAM register. ++ */ ++ POSTING_READ(GFX_FLSH_CNTL_GEN6); ++} ++ ++struct insert_page { ++ struct i915_address_space *vm; ++ dma_addr_t addr; ++ u64 offset; ++ enum i915_cache_level level; ++}; ++ ++static int bxt_vtd_ggtt_insert_page__cb(void *_arg) ++{ ++ struct insert_page *arg = _arg; ++ ++ gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset, arg->level, 0); ++ bxt_vtd_ggtt_wa(arg->vm); ++ ++ return 0; ++} ++ ++static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm, ++ dma_addr_t addr, ++ u64 offset, ++ enum i915_cache_level level, ++ u32 unused) ++{ ++ struct insert_page arg = { vm, addr, offset, level }; ++ ++ stop_machine(bxt_vtd_ggtt_insert_page__cb, &arg, NULL); ++} ++ ++struct insert_entries { ++ struct i915_address_space *vm; ++ struct i915_vma *vma; ++ enum i915_cache_level level; ++ u32 flags; ++}; ++ ++static int bxt_vtd_ggtt_insert_entries__cb(void *_arg) ++{ ++ struct insert_entries *arg = _arg; ++ ++ gen8_ggtt_insert_entries(arg->vm, arg->vma, arg->level, arg->flags); ++ bxt_vtd_ggtt_wa(arg->vm); ++ ++ return 0; ++} ++ ++static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm, ++ struct i915_vma *vma, ++ enum i915_cache_level level, ++ u32 flags) ++{ ++ struct insert_entries arg = { vm, vma, level, flags }; ++ ++ stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL); ++} ++ ++struct clear_range { ++ struct i915_address_space *vm; ++ u64 start; ++ u64 length; ++}; ++ ++static int bxt_vtd_ggtt_clear_range__cb(void *_arg) ++{ ++ struct clear_range *arg = _arg; ++ ++ gen8_ggtt_clear_range(arg->vm, arg->start, arg->length); ++ bxt_vtd_ggtt_wa(arg->vm); ++ ++ return 0; ++} ++ ++static void bxt_vtd_ggtt_clear_range__BKL(struct i915_address_space *vm, ++ u64 start, ++ u64 length) ++{ ++ struct clear_range arg = { vm, start, length }; ++ ++ stop_machine(bxt_vtd_ggtt_clear_range__cb, &arg, NULL); ++} ++ ++static void gen6_ggtt_clear_range(struct i915_address_space *vm, ++ u64 start, u64 length) ++{ ++ struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); ++ unsigned first_entry = start / I915_GTT_PAGE_SIZE; ++ unsigned num_entries = length / I915_GTT_PAGE_SIZE; ++ gen6_pte_t scratch_pte, __iomem *gtt_base = ++ (gen6_pte_t __iomem *)ggtt->gsm + first_entry; ++ const int max_entries = ggtt_total_entries(ggtt) - first_entry; ++ int i; ++ ++ if (WARN(num_entries > max_entries, ++ "First entry = %d; Num entries = %d (max=%d)\n", ++ first_entry, num_entries, max_entries)) ++ num_entries = max_entries; ++ ++ scratch_pte = vm->scratch_pte; ++ ++ for (i = 0; i < num_entries; i++) ++ iowrite32(scratch_pte, >t_base[i]); ++} ++ ++static void i915_ggtt_insert_page(struct i915_address_space *vm, ++ dma_addr_t addr, ++ u64 offset, ++ enum i915_cache_level cache_level, ++ u32 unused) ++{ ++ unsigned int flags = (cache_level == I915_CACHE_NONE) ? ++ AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; ++ ++ intel_gtt_insert_page(addr, offset >> PAGE_SHIFT, flags); ++} ++ ++static void i915_ggtt_insert_entries(struct i915_address_space *vm, ++ struct i915_vma *vma, ++ enum i915_cache_level cache_level, ++ u32 unused) ++{ ++ unsigned int flags = (cache_level == I915_CACHE_NONE) ? ++ AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; ++ ++ intel_gtt_insert_sg_entries(vma->pages, vma->node.start >> PAGE_SHIFT, ++ flags); ++} ++ ++static void i915_ggtt_clear_range(struct i915_address_space *vm, ++ u64 start, u64 length) ++{ ++ intel_gtt_clear_range(start >> PAGE_SHIFT, length >> PAGE_SHIFT); ++} ++ ++static int ggtt_bind_vma(struct i915_vma *vma, ++ enum i915_cache_level cache_level, ++ u32 flags) ++{ ++ struct drm_i915_private *i915 = vma->vm->i915; ++ struct drm_i915_gem_object *obj = vma->obj; ++ intel_wakeref_t wakeref; ++ u32 pte_flags; ++ ++ /* Applicable to VLV (gen8+ do not support RO in the GGTT) */ ++ pte_flags = 0; ++ if (i915_gem_object_is_readonly(obj)) ++ pte_flags |= PTE_READ_ONLY; ++ ++ with_intel_runtime_pm(i915, wakeref) ++ vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags); ++ ++ vma->page_sizes.gtt = I915_GTT_PAGE_SIZE; ++ ++ /* ++ * Without aliasing PPGTT there's no difference between ++ * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally ++ * upgrade to both bound if we bind either to avoid double-binding. ++ */ ++ vma->flags |= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; ++ ++ return 0; ++} ++ ++static void ggtt_unbind_vma(struct i915_vma *vma) ++{ ++ struct drm_i915_private *i915 = vma->vm->i915; ++ intel_wakeref_t wakeref; ++ ++ with_intel_runtime_pm(i915, wakeref) ++ vma->vm->clear_range(vma->vm, vma->node.start, vma->size); ++} ++ ++static int aliasing_gtt_bind_vma(struct i915_vma *vma, ++ enum i915_cache_level cache_level, ++ u32 flags) ++{ ++ struct drm_i915_private *i915 = vma->vm->i915; ++ u32 pte_flags; ++ int ret; ++ ++ /* Currently applicable only to VLV */ ++ pte_flags = 0; ++ if (i915_gem_object_is_readonly(vma->obj)) ++ pte_flags |= PTE_READ_ONLY; ++ ++ if (flags & I915_VMA_LOCAL_BIND) { ++ struct i915_hw_ppgtt *appgtt = i915->mm.aliasing_ppgtt; ++ ++ if (!(vma->flags & I915_VMA_LOCAL_BIND)) { ++ ret = appgtt->vm.allocate_va_range(&appgtt->vm, ++ vma->node.start, ++ vma->size); ++ if (ret) ++ return ret; ++ } ++ ++ appgtt->vm.insert_entries(&appgtt->vm, vma, cache_level, ++ pte_flags); ++ } ++ ++ if (flags & I915_VMA_GLOBAL_BIND) { ++ intel_wakeref_t wakeref; ++ ++ with_intel_runtime_pm(i915, wakeref) { ++ vma->vm->insert_entries(vma->vm, vma, ++ cache_level, pte_flags); ++ } ++ } ++ ++ return 0; ++} ++ ++static void aliasing_gtt_unbind_vma(struct i915_vma *vma) ++{ ++ struct drm_i915_private *i915 = vma->vm->i915; ++ ++ if (vma->flags & I915_VMA_GLOBAL_BIND) { ++ struct i915_address_space *vm = vma->vm; ++ intel_wakeref_t wakeref; ++ ++ with_intel_runtime_pm(i915, wakeref) ++ vm->clear_range(vm, vma->node.start, vma->size); ++ } ++ ++ if (vma->flags & I915_VMA_LOCAL_BIND) { ++ struct i915_address_space *vm = &i915->mm.aliasing_ppgtt->vm; ++ ++ vm->clear_range(vm, vma->node.start, vma->size); ++ } ++} ++ ++void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj, ++ struct sg_table *pages) ++{ ++ struct drm_i915_private *dev_priv = to_i915(obj->base.dev); ++ struct device *kdev = &dev_priv->drm.pdev->dev; ++ struct i915_ggtt *ggtt = &dev_priv->ggtt; ++ ++ if (unlikely(ggtt->do_idle_maps)) { ++ if (i915_gem_wait_for_idle(dev_priv, 0, MAX_SCHEDULE_TIMEOUT)) { ++ DRM_ERROR("Failed to wait for idle; VT'd may hang.\n"); ++ /* Wait a bit, in hopes it avoids the hang */ ++ udelay(10); ++ } ++ } ++ ++ dma_unmap_sg(kdev, pages->sgl, pages->nents, PCI_DMA_BIDIRECTIONAL); ++} ++ ++static int ggtt_set_pages(struct i915_vma *vma) ++{ ++ int ret; ++ ++ GEM_BUG_ON(vma->pages); ++ ++ ret = i915_get_ggtt_vma_pages(vma); ++ if (ret) ++ return ret; ++ ++ vma->page_sizes = vma->obj->mm.page_sizes; ++ ++ return 0; ++} ++ ++static void i915_gtt_color_adjust(const struct drm_mm_node *node, ++ unsigned long color, ++ u64 *start, ++ u64 *end) ++{ ++ if (node->allocated && node->color != color) ++ *start += I915_GTT_PAGE_SIZE; ++ ++ /* Also leave a space between the unallocated reserved node after the ++ * GTT and any objects within the GTT, i.e. we use the color adjustment ++ * to insert a guard page to prevent prefetches crossing over the ++ * GTT boundary. ++ */ ++ node = list_next_entry(node, node_list); ++ if (node->color != color) ++ *end -= I915_GTT_PAGE_SIZE; ++} ++ ++int i915_gem_init_aliasing_ppgtt(struct drm_i915_private *i915) ++{ ++ struct i915_ggtt *ggtt = &i915->ggtt; ++ struct i915_hw_ppgtt *ppgtt; ++ int err; ++ ++ ppgtt = i915_ppgtt_create(i915); ++ if (IS_ERR(ppgtt)) ++ return PTR_ERR(ppgtt); ++ ++ if (GEM_WARN_ON(ppgtt->vm.total < ggtt->vm.total)) { ++ err = -ENODEV; ++ goto err_ppgtt; ++ } ++ ++ /* ++ * Note we only pre-allocate as far as the end of the global ++ * GTT. On 48b / 4-level page-tables, the difference is very, ++ * very significant! We have to preallocate as GVT/vgpu does ++ * not like the page directory disappearing. ++ */ ++ err = ppgtt->vm.allocate_va_range(&ppgtt->vm, 0, ggtt->vm.total); ++ if (err) ++ goto err_ppgtt; ++ ++ i915->mm.aliasing_ppgtt = ppgtt; ++ ++ GEM_BUG_ON(ggtt->vm.vma_ops.bind_vma != ggtt_bind_vma); ++ ggtt->vm.vma_ops.bind_vma = aliasing_gtt_bind_vma; ++ ++ GEM_BUG_ON(ggtt->vm.vma_ops.unbind_vma != ggtt_unbind_vma); ++ ggtt->vm.vma_ops.unbind_vma = aliasing_gtt_unbind_vma; ++ ++ return 0; ++ ++err_ppgtt: ++ i915_ppgtt_put(ppgtt); ++ return err; ++} ++ ++void i915_gem_fini_aliasing_ppgtt(struct drm_i915_private *i915) ++{ ++ struct i915_ggtt *ggtt = &i915->ggtt; ++ struct i915_hw_ppgtt *ppgtt; ++ ++ ppgtt = fetch_and_zero(&i915->mm.aliasing_ppgtt); ++ if (!ppgtt) ++ return; ++ ++ i915_ppgtt_put(ppgtt); ++ ++ ggtt->vm.vma_ops.bind_vma = ggtt_bind_vma; ++ ggtt->vm.vma_ops.unbind_vma = ggtt_unbind_vma; ++} ++ ++int i915_gem_init_ggtt(struct drm_i915_private *dev_priv) ++{ ++ /* Let GEM Manage all of the aperture. ++ * ++ * However, leave one page at the end still bound to the scratch page. ++ * There are a number of places where the hardware apparently prefetches ++ * past the end of the object, and we've seen multiple hangs with the ++ * GPU head pointer stuck in a batchbuffer bound at the last page of the ++ * aperture. One page should be enough to keep any prefetching inside ++ * of the aperture. ++ */ ++ struct i915_ggtt *ggtt = &dev_priv->ggtt; ++ unsigned long hole_start, hole_end; ++ struct drm_mm_node *entry; ++ int ret; ++ ++ /* ++ * GuC requires all resources that we're sharing with it to be placed in ++ * non-WOPCM memory. If GuC is not present or not in use we still need a ++ * small bias as ring wraparound at offset 0 sometimes hangs. No idea ++ * why. ++ */ ++ ggtt->pin_bias = max_t(u32, I915_GTT_PAGE_SIZE, ++ intel_guc_reserved_gtt_size(&dev_priv->guc)); ++ ++ ret = intel_vgt_balloon(dev_priv); ++ if (ret) ++ return ret; ++ ++ /* Reserve a mappable slot for our lockless error capture */ ++ ret = drm_mm_insert_node_in_range(&ggtt->vm.mm, &ggtt->error_capture, ++ PAGE_SIZE, 0, I915_COLOR_UNEVICTABLE, ++ 0, ggtt->mappable_end, ++ DRM_MM_INSERT_LOW); ++ if (ret) ++ return ret; ++ ++ /* Clear any non-preallocated blocks */ ++ drm_mm_for_each_hole(entry, &ggtt->vm.mm, hole_start, hole_end) { ++ DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n", ++ hole_start, hole_end); ++ ggtt->vm.clear_range(&ggtt->vm, hole_start, ++ hole_end - hole_start); ++ } ++ ++ /* And finally clear the reserved guard page */ ++ ggtt->vm.clear_range(&ggtt->vm, ggtt->vm.total - PAGE_SIZE, PAGE_SIZE); ++ ++ if (INTEL_PPGTT(dev_priv) == INTEL_PPGTT_ALIASING) { ++ ret = i915_gem_init_aliasing_ppgtt(dev_priv); ++ if (ret) ++ goto err; ++ } ++ ++ return 0; ++ ++err: ++ drm_mm_remove_node(&ggtt->error_capture); ++ return ret; ++} ++ ++/** ++ * i915_ggtt_cleanup_hw - Clean up GGTT hardware initialization ++ * @dev_priv: i915 device ++ */ ++void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv) ++{ ++ struct i915_ggtt *ggtt = &dev_priv->ggtt; ++ struct i915_vma *vma, *vn; ++ struct pagevec *pvec; ++ ++ ggtt->vm.closed = true; ++ ++ mutex_lock(&dev_priv->drm.struct_mutex); ++ i915_gem_fini_aliasing_ppgtt(dev_priv); ++ ++ list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) ++ WARN_ON(i915_vma_unbind(vma)); ++ ++ if (drm_mm_node_allocated(&ggtt->error_capture)) ++ drm_mm_remove_node(&ggtt->error_capture); ++ ++ if (drm_mm_initialized(&ggtt->vm.mm)) { ++ intel_vgt_deballoon(dev_priv); ++ i915_address_space_fini(&ggtt->vm); ++ } ++ ++ ggtt->vm.cleanup(&ggtt->vm); ++ ++ pvec = &dev_priv->mm.wc_stash.pvec; ++ if (pvec->nr) { ++ set_pages_array_wb(pvec->pages, pvec->nr); ++ __pagevec_release(pvec); ++ } ++ ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++ ++ arch_phys_wc_del(ggtt->mtrr); ++ io_mapping_fini(&ggtt->iomap); ++ ++ i915_gem_cleanup_stolen(dev_priv); ++} ++ ++static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) ++{ ++ snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; ++ snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; ++ return snb_gmch_ctl << 20; ++} ++ ++static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl) ++{ ++ bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT; ++ bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK; ++ if (bdw_gmch_ctl) ++ bdw_gmch_ctl = 1 << bdw_gmch_ctl; ++ ++#ifdef CONFIG_X86_32 ++ /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * I915_GTT_PAGE_SIZE */ ++ if (bdw_gmch_ctl > 4) ++ bdw_gmch_ctl = 4; ++#endif ++ ++ return bdw_gmch_ctl << 20; ++} ++ ++static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl) ++{ ++ gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT; ++ gmch_ctrl &= SNB_GMCH_GGMS_MASK; ++ ++ if (gmch_ctrl) ++ return 1 << (20 + gmch_ctrl); ++ ++ return 0; ++} ++ ++static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) ++{ ++ struct drm_i915_private *dev_priv = ggtt->vm.i915; ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ phys_addr_t phys_addr; ++ int ret; ++ ++ /* For Modern GENs the PTEs and register space are split in the BAR */ ++ phys_addr = pci_resource_start(pdev, 0) + pci_resource_len(pdev, 0) / 2; ++ ++ /* ++ * On BXT+/CNL+ writes larger than 64 bit to the GTT pagetable range ++ * will be dropped. For WC mappings in general we have 64 byte burst ++ * writes when the WC buffer is flushed, so we can't use it, but have to ++ * resort to an uncached mapping. The WC issue is easily caught by the ++ * readback check when writing GTT PTE entries. ++ */ ++ if (IS_GEN9_LP(dev_priv) || INTEL_GEN(dev_priv) >= 10) ++ ggtt->gsm = ioremap_nocache(phys_addr, size); ++ else ++ ggtt->gsm = ioremap_wc(phys_addr, size); ++ if (!ggtt->gsm) { ++ DRM_ERROR("Failed to map the ggtt page table\n"); ++ return -ENOMEM; ++ } ++ ++ ret = setup_scratch_page(&ggtt->vm, GFP_DMA32); ++ if (ret) { ++ DRM_ERROR("Scratch setup failed\n"); ++ /* iounmap will also get called at remove, but meh */ ++ iounmap(ggtt->gsm); ++ return ret; ++ } ++ ++ ggtt->vm.scratch_pte = ++ ggtt->vm.pte_encode(ggtt->vm.scratch_page.daddr, ++ I915_CACHE_NONE, 0); ++ ++ return 0; ++} ++ ++static struct intel_ppat_entry * ++__alloc_ppat_entry(struct intel_ppat *ppat, unsigned int index, u8 value) ++{ ++ struct intel_ppat_entry *entry = &ppat->entries[index]; ++ ++ GEM_BUG_ON(index >= ppat->max_entries); ++ GEM_BUG_ON(test_bit(index, ppat->used)); ++ ++ entry->ppat = ppat; ++ entry->value = value; ++ kref_init(&entry->ref); ++ set_bit(index, ppat->used); ++ set_bit(index, ppat->dirty); ++ ++ return entry; ++} ++ ++static void __free_ppat_entry(struct intel_ppat_entry *entry) ++{ ++ struct intel_ppat *ppat = entry->ppat; ++ unsigned int index = entry - ppat->entries; ++ ++ GEM_BUG_ON(index >= ppat->max_entries); ++ GEM_BUG_ON(!test_bit(index, ppat->used)); ++ ++ entry->value = ppat->clear_value; ++ clear_bit(index, ppat->used); ++ set_bit(index, ppat->dirty); ++} ++ ++/** ++ * intel_ppat_get - get a usable PPAT entry ++ * @i915: i915 device instance ++ * @value: the PPAT value required by the caller ++ * ++ * The function tries to search if there is an existing PPAT entry which ++ * matches with the required value. If perfectly matched, the existing PPAT ++ * entry will be used. If only partially matched, it will try to check if ++ * there is any available PPAT index. If yes, it will allocate a new PPAT ++ * index for the required entry and update the HW. If not, the partially ++ * matched entry will be used. ++ */ ++const struct intel_ppat_entry * ++intel_ppat_get(struct drm_i915_private *i915, u8 value) ++{ ++ struct intel_ppat *ppat = &i915->ppat; ++ struct intel_ppat_entry *entry = NULL; ++ unsigned int scanned, best_score; ++ int i; ++ ++ GEM_BUG_ON(!ppat->max_entries); ++ ++ scanned = best_score = 0; ++ for_each_set_bit(i, ppat->used, ppat->max_entries) { ++ unsigned int score; ++ ++ score = ppat->match(ppat->entries[i].value, value); ++ if (score > best_score) { ++ entry = &ppat->entries[i]; ++ if (score == INTEL_PPAT_PERFECT_MATCH) { ++ kref_get(&entry->ref); ++ return entry; ++ } ++ best_score = score; ++ } ++ scanned++; ++ } ++ ++ if (scanned == ppat->max_entries) { ++ if (!entry) ++ return ERR_PTR(-ENOSPC); ++ ++ kref_get(&entry->ref); ++ return entry; ++ } ++ ++ i = find_first_zero_bit(ppat->used, ppat->max_entries); ++ entry = __alloc_ppat_entry(ppat, i, value); ++ ppat->update_hw(i915); ++ return entry; ++} ++ ++static void release_ppat(struct kref *kref) ++{ ++ struct intel_ppat_entry *entry = ++ container_of(kref, struct intel_ppat_entry, ref); ++ struct drm_i915_private *i915 = entry->ppat->i915; ++ ++ __free_ppat_entry(entry); ++ entry->ppat->update_hw(i915); ++} ++ ++/** ++ * intel_ppat_put - put back the PPAT entry got from intel_ppat_get() ++ * @entry: an intel PPAT entry ++ * ++ * Put back the PPAT entry got from intel_ppat_get(). If the PPAT index of the ++ * entry is dynamically allocated, its reference count will be decreased. Once ++ * the reference count becomes into zero, the PPAT index becomes free again. ++ */ ++void intel_ppat_put(const struct intel_ppat_entry *entry) ++{ ++ struct intel_ppat *ppat = entry->ppat; ++ unsigned int index = entry - ppat->entries; ++ ++ GEM_BUG_ON(!ppat->max_entries); ++ ++ kref_put(&ppat->entries[index].ref, release_ppat); ++} ++ ++static void cnl_private_pat_update_hw(struct drm_i915_private *dev_priv) ++{ ++ struct intel_ppat *ppat = &dev_priv->ppat; ++ int i; ++ ++ for_each_set_bit(i, ppat->dirty, ppat->max_entries) { ++ I915_WRITE(GEN10_PAT_INDEX(i), ppat->entries[i].value); ++ clear_bit(i, ppat->dirty); ++ } ++} ++ ++static void bdw_private_pat_update_hw(struct drm_i915_private *dev_priv) ++{ ++ struct intel_ppat *ppat = &dev_priv->ppat; ++ u64 pat = 0; ++ int i; ++ ++ for (i = 0; i < ppat->max_entries; i++) ++ pat |= GEN8_PPAT(i, ppat->entries[i].value); ++ ++ bitmap_clear(ppat->dirty, 0, ppat->max_entries); ++ ++ I915_WRITE(GEN8_PRIVATE_PAT_LO, lower_32_bits(pat)); ++ I915_WRITE(GEN8_PRIVATE_PAT_HI, upper_32_bits(pat)); ++} ++ ++static unsigned int bdw_private_pat_match(u8 src, u8 dst) ++{ ++ unsigned int score = 0; ++ enum { ++ AGE_MATCH = BIT(0), ++ TC_MATCH = BIT(1), ++ CA_MATCH = BIT(2), ++ }; ++ ++ /* Cache attribute has to be matched. */ ++ if (GEN8_PPAT_GET_CA(src) != GEN8_PPAT_GET_CA(dst)) ++ return 0; ++ ++ score |= CA_MATCH; ++ ++ if (GEN8_PPAT_GET_TC(src) == GEN8_PPAT_GET_TC(dst)) ++ score |= TC_MATCH; ++ ++ if (GEN8_PPAT_GET_AGE(src) == GEN8_PPAT_GET_AGE(dst)) ++ score |= AGE_MATCH; ++ ++ if (score == (AGE_MATCH | TC_MATCH | CA_MATCH)) ++ return INTEL_PPAT_PERFECT_MATCH; ++ ++ return score; ++} ++ ++static unsigned int chv_private_pat_match(u8 src, u8 dst) ++{ ++ return (CHV_PPAT_GET_SNOOP(src) == CHV_PPAT_GET_SNOOP(dst)) ? ++ INTEL_PPAT_PERFECT_MATCH : 0; ++} ++ ++static void cnl_setup_private_ppat(struct intel_ppat *ppat) ++{ ++ ppat->max_entries = 8; ++ ppat->update_hw = cnl_private_pat_update_hw; ++ ppat->match = bdw_private_pat_match; ++ ppat->clear_value = GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3); ++ ++ __alloc_ppat_entry(ppat, 0, GEN8_PPAT_WB | GEN8_PPAT_LLC); ++ __alloc_ppat_entry(ppat, 1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC); ++ __alloc_ppat_entry(ppat, 2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC); ++ __alloc_ppat_entry(ppat, 3, GEN8_PPAT_UC); ++ __alloc_ppat_entry(ppat, 4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)); ++ __alloc_ppat_entry(ppat, 5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)); ++ __alloc_ppat_entry(ppat, 6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)); ++ __alloc_ppat_entry(ppat, 7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3)); ++} ++ ++/* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability ++ * bits. When using advanced contexts each context stores its own PAT, but ++ * writing this data shouldn't be harmful even in those cases. */ ++static void bdw_setup_private_ppat(struct intel_ppat *ppat) ++{ ++ ppat->max_entries = 8; ++ ppat->update_hw = bdw_private_pat_update_hw; ++ ppat->match = bdw_private_pat_match; ++ ppat->clear_value = GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3); ++ ++ if (!HAS_PPGTT(ppat->i915)) { ++ /* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry, ++ * so RTL will always use the value corresponding to ++ * pat_sel = 000". ++ * So let's disable cache for GGTT to avoid screen corruptions. ++ * MOCS still can be used though. ++ * - System agent ggtt writes (i.e. cpu gtt mmaps) already work ++ * before this patch, i.e. the same uncached + snooping access ++ * like on gen6/7 seems to be in effect. ++ * - So this just fixes blitter/render access. Again it looks ++ * like it's not just uncached access, but uncached + snooping. ++ * So we can still hold onto all our assumptions wrt cpu ++ * clflushing on LLC machines. ++ */ ++ __alloc_ppat_entry(ppat, 0, GEN8_PPAT_UC); ++ return; ++ } ++ ++ __alloc_ppat_entry(ppat, 0, GEN8_PPAT_WB | GEN8_PPAT_LLC); /* for normal objects, no eLLC */ ++ __alloc_ppat_entry(ppat, 1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC); /* for something pointing to ptes? */ ++ __alloc_ppat_entry(ppat, 2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC); /* for scanout with eLLC */ ++ __alloc_ppat_entry(ppat, 3, GEN8_PPAT_UC); /* Uncached objects, mostly for scanout */ ++ __alloc_ppat_entry(ppat, 4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)); ++ __alloc_ppat_entry(ppat, 5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)); ++ __alloc_ppat_entry(ppat, 6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)); ++ __alloc_ppat_entry(ppat, 7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3)); ++} ++ ++static void chv_setup_private_ppat(struct intel_ppat *ppat) ++{ ++ ppat->max_entries = 8; ++ ppat->update_hw = bdw_private_pat_update_hw; ++ ppat->match = chv_private_pat_match; ++ ppat->clear_value = CHV_PPAT_SNOOP; ++ ++ /* ++ * Map WB on BDW to snooped on CHV. ++ * ++ * Only the snoop bit has meaning for CHV, the rest is ++ * ignored. ++ * ++ * The hardware will never snoop for certain types of accesses: ++ * - CPU GTT (GMADR->GGTT->no snoop->memory) ++ * - PPGTT page tables ++ * - some other special cycles ++ * ++ * As with BDW, we also need to consider the following for GT accesses: ++ * "For GGTT, there is NO pat_sel[2:0] from the entry, ++ * so RTL will always use the value corresponding to ++ * pat_sel = 000". ++ * Which means we must set the snoop bit in PAT entry 0 ++ * in order to keep the global status page working. ++ */ ++ ++ __alloc_ppat_entry(ppat, 0, CHV_PPAT_SNOOP); ++ __alloc_ppat_entry(ppat, 1, 0); ++ __alloc_ppat_entry(ppat, 2, 0); ++ __alloc_ppat_entry(ppat, 3, 0); ++ __alloc_ppat_entry(ppat, 4, CHV_PPAT_SNOOP); ++ __alloc_ppat_entry(ppat, 5, CHV_PPAT_SNOOP); ++ __alloc_ppat_entry(ppat, 6, CHV_PPAT_SNOOP); ++ __alloc_ppat_entry(ppat, 7, CHV_PPAT_SNOOP); ++} ++ ++static void gen6_gmch_remove(struct i915_address_space *vm) ++{ ++ struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); ++ ++ iounmap(ggtt->gsm); ++ cleanup_scratch_page(vm); ++} ++ ++static void setup_private_pat(struct drm_i915_private *dev_priv) ++{ ++ struct intel_ppat *ppat = &dev_priv->ppat; ++ int i; ++ ++ ppat->i915 = dev_priv; ++ ++ if (INTEL_GEN(dev_priv) >= 10) ++ cnl_setup_private_ppat(ppat); ++ else if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv)) ++ chv_setup_private_ppat(ppat); ++ else ++ bdw_setup_private_ppat(ppat); ++ ++ GEM_BUG_ON(ppat->max_entries > INTEL_MAX_PPAT_ENTRIES); ++ ++ for_each_clear_bit(i, ppat->used, ppat->max_entries) { ++ ppat->entries[i].value = ppat->clear_value; ++ ppat->entries[i].ppat = ppat; ++ set_bit(i, ppat->dirty); ++ } ++ ++ ppat->update_hw(dev_priv); ++} ++ ++static int gen8_gmch_probe(struct i915_ggtt *ggtt) ++{ ++ struct drm_i915_private *dev_priv = ggtt->vm.i915; ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ unsigned int size; ++ u16 snb_gmch_ctl; ++ int err; ++ ++ /* TODO: We're not aware of mappable constraints on gen8 yet */ ++ ggtt->gmadr = ++ (struct resource) DEFINE_RES_MEM(pci_resource_start(pdev, 2), ++ pci_resource_len(pdev, 2)); ++ ggtt->mappable_end = resource_size(&ggtt->gmadr); ++ ++ err = pci_set_dma_mask(pdev, DMA_BIT_MASK(39)); ++ if (!err) ++ err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(39)); ++ if (err) ++ DRM_ERROR("Can't set DMA mask/consistent mask (%d)\n", err); ++ ++ pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); ++ if (IS_CHERRYVIEW(dev_priv)) ++ size = chv_get_total_gtt_size(snb_gmch_ctl); ++ else ++ size = gen8_get_total_gtt_size(snb_gmch_ctl); ++ ++ ggtt->vm.total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE; ++ ggtt->vm.cleanup = gen6_gmch_remove; ++ ggtt->vm.insert_page = gen8_ggtt_insert_page; ++ ggtt->vm.clear_range = nop_clear_range; ++ if (intel_scanout_needs_vtd_wa(dev_priv)) ++ ggtt->vm.clear_range = gen8_ggtt_clear_range; ++ ++ ggtt->vm.insert_entries = gen8_ggtt_insert_entries; ++ ++ /* Serialize GTT updates with aperture access on BXT if VT-d is on. */ ++ if (intel_ggtt_update_needs_vtd_wa(dev_priv) || ++ IS_CHERRYVIEW(dev_priv) /* fails with concurrent use/update */) { ++ ggtt->vm.insert_entries = bxt_vtd_ggtt_insert_entries__BKL; ++ ggtt->vm.insert_page = bxt_vtd_ggtt_insert_page__BKL; ++ if (ggtt->vm.clear_range != nop_clear_range) ++ ggtt->vm.clear_range = bxt_vtd_ggtt_clear_range__BKL; ++ ++ /* Prevent recursively calling stop_machine() and deadlocks. */ ++ dev_info(dev_priv->drm.dev, ++ "Disabling error capture for VT-d workaround\n"); ++ i915_disable_error_state(dev_priv, -ENODEV); ++ } ++ ++ ggtt->invalidate = gen6_ggtt_invalidate; ++ ++ ggtt->vm.vma_ops.bind_vma = ggtt_bind_vma; ++ ggtt->vm.vma_ops.unbind_vma = ggtt_unbind_vma; ++ ggtt->vm.vma_ops.set_pages = ggtt_set_pages; ++ ggtt->vm.vma_ops.clear_pages = clear_pages; ++ ++ ggtt->vm.pte_encode = gen8_pte_encode; ++ ++ setup_private_pat(dev_priv); ++ ++ return ggtt_probe_common(ggtt, size); ++} ++ ++static int gen6_gmch_probe(struct i915_ggtt *ggtt) ++{ ++ struct drm_i915_private *dev_priv = ggtt->vm.i915; ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ unsigned int size; ++ u16 snb_gmch_ctl; ++ int err; ++ ++ ggtt->gmadr = ++ (struct resource) DEFINE_RES_MEM(pci_resource_start(pdev, 2), ++ pci_resource_len(pdev, 2)); ++ ggtt->mappable_end = resource_size(&ggtt->gmadr); ++ ++ /* 64/512MB is the current min/max we actually know of, but this is just ++ * a coarse sanity check. ++ */ ++ if (ggtt->mappable_end < (64<<20) || ggtt->mappable_end > (512<<20)) { ++ DRM_ERROR("Unknown GMADR size (%pa)\n", &ggtt->mappable_end); ++ return -ENXIO; ++ } ++ ++ err = pci_set_dma_mask(pdev, DMA_BIT_MASK(40)); ++ if (!err) ++ err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(40)); ++ if (err) ++ DRM_ERROR("Can't set DMA mask/consistent mask (%d)\n", err); ++ pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); ++ ++ size = gen6_get_total_gtt_size(snb_gmch_ctl); ++ ggtt->vm.total = (size / sizeof(gen6_pte_t)) * I915_GTT_PAGE_SIZE; ++ ++ ggtt->vm.clear_range = gen6_ggtt_clear_range; ++ ggtt->vm.insert_page = gen6_ggtt_insert_page; ++ ggtt->vm.insert_entries = gen6_ggtt_insert_entries; ++ ggtt->vm.cleanup = gen6_gmch_remove; ++ ++ ggtt->invalidate = gen6_ggtt_invalidate; ++ ++ if (HAS_EDRAM(dev_priv)) ++ ggtt->vm.pte_encode = iris_pte_encode; ++ else if (IS_HASWELL(dev_priv)) ++ ggtt->vm.pte_encode = hsw_pte_encode; ++ else if (IS_VALLEYVIEW(dev_priv)) ++ ggtt->vm.pte_encode = byt_pte_encode; ++ else if (INTEL_GEN(dev_priv) >= 7) ++ ggtt->vm.pte_encode = ivb_pte_encode; ++ else ++ ggtt->vm.pte_encode = snb_pte_encode; ++ ++ ggtt->vm.vma_ops.bind_vma = ggtt_bind_vma; ++ ggtt->vm.vma_ops.unbind_vma = ggtt_unbind_vma; ++ ggtt->vm.vma_ops.set_pages = ggtt_set_pages; ++ ggtt->vm.vma_ops.clear_pages = clear_pages; ++ ++ return ggtt_probe_common(ggtt, size); ++} ++ ++static void i915_gmch_remove(struct i915_address_space *vm) ++{ ++ intel_gmch_remove(); ++} ++ ++static int i915_gmch_probe(struct i915_ggtt *ggtt) ++{ ++ struct drm_i915_private *dev_priv = ggtt->vm.i915; ++ phys_addr_t gmadr_base; ++ int ret; ++ ++ ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->drm.pdev, NULL); ++ if (!ret) { ++ DRM_ERROR("failed to set up gmch\n"); ++ return -EIO; ++ } ++ ++ intel_gtt_get(&ggtt->vm.total, &gmadr_base, &ggtt->mappable_end); ++ ++ ggtt->gmadr = ++ (struct resource) DEFINE_RES_MEM(gmadr_base, ++ ggtt->mappable_end); ++ ++ ggtt->do_idle_maps = needs_idle_maps(dev_priv); ++ ggtt->vm.insert_page = i915_ggtt_insert_page; ++ ggtt->vm.insert_entries = i915_ggtt_insert_entries; ++ ggtt->vm.clear_range = i915_ggtt_clear_range; ++ ggtt->vm.cleanup = i915_gmch_remove; ++ ++ ggtt->invalidate = gmch_ggtt_invalidate; ++ ++ ggtt->vm.vma_ops.bind_vma = ggtt_bind_vma; ++ ggtt->vm.vma_ops.unbind_vma = ggtt_unbind_vma; ++ ggtt->vm.vma_ops.set_pages = ggtt_set_pages; ++ ggtt->vm.vma_ops.clear_pages = clear_pages; ++ ++ if (unlikely(ggtt->do_idle_maps)) ++ DRM_INFO("applying Ironlake quirks for intel_iommu\n"); ++ ++ return 0; ++} ++ ++/** ++ * i915_ggtt_probe_hw - Probe GGTT hardware location ++ * @dev_priv: i915 device ++ */ ++int i915_ggtt_probe_hw(struct drm_i915_private *dev_priv) ++{ ++ struct i915_ggtt *ggtt = &dev_priv->ggtt; ++ int ret; ++ ++ ggtt->vm.i915 = dev_priv; ++ ggtt->vm.dma = &dev_priv->drm.pdev->dev; ++ ++ if (INTEL_GEN(dev_priv) <= 5) ++ ret = i915_gmch_probe(ggtt); ++ else if (INTEL_GEN(dev_priv) < 8) ++ ret = gen6_gmch_probe(ggtt); ++ else ++ ret = gen8_gmch_probe(ggtt); ++ if (ret) ++ return ret; ++ ++ /* Trim the GGTT to fit the GuC mappable upper range (when enabled). ++ * This is easier than doing range restriction on the fly, as we ++ * currently don't have any bits spare to pass in this upper ++ * restriction! ++ */ ++ if (USES_GUC(dev_priv)) { ++ ggtt->vm.total = min_t(u64, ggtt->vm.total, GUC_GGTT_TOP); ++ ggtt->mappable_end = ++ min_t(u64, ggtt->mappable_end, ggtt->vm.total); ++ } ++ ++ if ((ggtt->vm.total - 1) >> 32) { ++ DRM_ERROR("We never expected a Global GTT with more than 32bits" ++ " of address space! Found %lldM!\n", ++ ggtt->vm.total >> 20); ++ ggtt->vm.total = 1ULL << 32; ++ ggtt->mappable_end = ++ min_t(u64, ggtt->mappable_end, ggtt->vm.total); ++ } ++ ++ if (ggtt->mappable_end > ggtt->vm.total) { ++ DRM_ERROR("mappable aperture extends past end of GGTT," ++ " aperture=%pa, total=%llx\n", ++ &ggtt->mappable_end, ggtt->vm.total); ++ ggtt->mappable_end = ggtt->vm.total; ++ } ++ ++ /* GMADR is the PCI mmio aperture into the global GTT. */ ++ DRM_DEBUG_DRIVER("GGTT size = %lluM\n", ggtt->vm.total >> 20); ++ DRM_DEBUG_DRIVER("GMADR size = %lluM\n", (u64)ggtt->mappable_end >> 20); ++ DRM_DEBUG_DRIVER("DSM size = %lluM\n", ++ (u64)resource_size(&intel_graphics_stolen_res) >> 20); ++ if (intel_vtd_active()) ++ DRM_INFO("VT-d active for gfx access\n"); ++ ++ return 0; ++} ++ ++/** ++ * i915_ggtt_init_hw - Initialize GGTT hardware ++ * @dev_priv: i915 device ++ */ ++int i915_ggtt_init_hw(struct drm_i915_private *dev_priv) ++{ ++ struct i915_ggtt *ggtt = &dev_priv->ggtt; ++ int ret; ++ ++ stash_init(&dev_priv->mm.wc_stash); ++ ++ /* Note that we use page colouring to enforce a guard page at the ++ * end of the address space. This is required as the CS may prefetch ++ * beyond the end of the batch buffer, across the page boundary, ++ * and beyond the end of the GTT if we do not provide a guard. ++ */ ++ mutex_lock(&dev_priv->drm.struct_mutex); ++ i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT); ++ ++ ggtt->vm.is_ggtt = true; ++ ++ /* Only VLV supports read-only GGTT mappings */ ++ ggtt->vm.has_read_only = IS_VALLEYVIEW(dev_priv); ++ ++ if (!HAS_LLC(dev_priv) && !HAS_PPGTT(dev_priv)) ++ ggtt->vm.mm.color_adjust = i915_gtt_color_adjust; ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++ ++ if (!io_mapping_init_wc(&dev_priv->ggtt.iomap, ++ dev_priv->ggtt.gmadr.start, ++ dev_priv->ggtt.mappable_end)) { ++ ret = -EIO; ++ goto out_gtt_cleanup; ++ } ++ ++ ggtt->mtrr = arch_phys_wc_add(ggtt->gmadr.start, ggtt->mappable_end); ++ ++ /* ++ * Initialise stolen early so that we may reserve preallocated ++ * objects for the BIOS to KMS transition. ++ */ ++ ret = i915_gem_init_stolen(dev_priv); ++ if (ret) ++ goto out_gtt_cleanup; ++ ++ return 0; ++ ++out_gtt_cleanup: ++ ggtt->vm.cleanup(&ggtt->vm); ++ return ret; ++} ++ ++int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv) ++{ ++ if (INTEL_GEN(dev_priv) < 6 && !intel_enable_gtt()) ++ return -EIO; ++ ++ return 0; ++} ++ ++void i915_ggtt_enable_guc(struct drm_i915_private *i915) ++{ ++ GEM_BUG_ON(i915->ggtt.invalidate != gen6_ggtt_invalidate); ++ ++ i915->ggtt.invalidate = guc_ggtt_invalidate; ++ ++ i915_ggtt_invalidate(i915); ++} ++ ++void i915_ggtt_disable_guc(struct drm_i915_private *i915) ++{ ++ /* XXX Temporary pardon for error unload */ ++ if (i915->ggtt.invalidate == gen6_ggtt_invalidate) ++ return; ++ ++ /* We should only be called after i915_ggtt_enable_guc() */ ++ GEM_BUG_ON(i915->ggtt.invalidate != guc_ggtt_invalidate); ++ ++ i915->ggtt.invalidate = gen6_ggtt_invalidate; ++ ++ i915_ggtt_invalidate(i915); ++} ++ ++void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv) ++{ ++ struct i915_ggtt *ggtt = &dev_priv->ggtt; ++ struct i915_vma *vma, *vn; ++ ++ i915_check_and_clear_faults(dev_priv); ++ ++ mutex_lock(&ggtt->vm.mutex); ++ ++ /* First fill our portion of the GTT with scratch pages */ ++ ggtt->vm.clear_range(&ggtt->vm, 0, ggtt->vm.total); ++ ggtt->vm.closed = true; /* skip rewriting PTE on VMA unbind */ ++ ++ /* clflush objects bound into the GGTT and rebind them. */ ++ list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) { ++ struct drm_i915_gem_object *obj = vma->obj; ++ ++ if (!(vma->flags & I915_VMA_GLOBAL_BIND)) ++ continue; ++ ++ mutex_unlock(&ggtt->vm.mutex); ++ ++ if (!i915_vma_unbind(vma)) ++ goto lock; ++ ++ WARN_ON(i915_vma_bind(vma, ++ obj ? obj->cache_level : 0, ++ PIN_UPDATE)); ++ if (obj) ++ WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false)); ++ ++lock: ++ mutex_lock(&ggtt->vm.mutex); ++ } ++ ++ ggtt->vm.closed = false; ++ i915_ggtt_invalidate(dev_priv); ++ ++ mutex_unlock(&ggtt->vm.mutex); ++ ++ if (INTEL_GEN(dev_priv) >= 8) { ++ struct intel_ppat *ppat = &dev_priv->ppat; ++ ++ bitmap_set(ppat->dirty, 0, ppat->max_entries); ++ dev_priv->ppat.update_hw(dev_priv); ++ return; ++ } ++} ++ ++static struct scatterlist * ++rotate_pages(struct drm_i915_gem_object *obj, unsigned int offset, ++ unsigned int width, unsigned int height, ++ unsigned int stride, ++ struct sg_table *st, struct scatterlist *sg) ++{ ++ unsigned int column, row; ++ unsigned int src_idx; ++ ++ for (column = 0; column < width; column++) { ++ src_idx = stride * (height - 1) + column + offset; ++ for (row = 0; row < height; row++) { ++ st->nents++; ++ /* We don't need the pages, but need to initialize ++ * the entries so the sg list can be happily traversed. ++ * The only thing we need are DMA addresses. ++ */ ++ sg_set_page(sg, NULL, I915_GTT_PAGE_SIZE, 0); ++ sg_dma_address(sg) = ++ i915_gem_object_get_dma_address(obj, src_idx); ++ sg_dma_len(sg) = I915_GTT_PAGE_SIZE; ++ sg = sg_next(sg); ++ src_idx -= stride; ++ } ++ } ++ ++ return sg; ++} ++ ++static noinline struct sg_table * ++intel_rotate_pages(struct intel_rotation_info *rot_info, ++ struct drm_i915_gem_object *obj) ++{ ++ unsigned int size = intel_rotation_info_size(rot_info); ++ struct sg_table *st; ++ struct scatterlist *sg; ++ int ret = -ENOMEM; ++ int i; ++ ++ /* Allocate target SG list. */ ++ st = kmalloc(sizeof(*st), GFP_KERNEL); ++ if (!st) ++ goto err_st_alloc; ++ ++ ret = sg_alloc_table(st, size, GFP_KERNEL); ++ if (ret) ++ goto err_sg_alloc; ++ ++ st->nents = 0; ++ sg = st->sgl; ++ ++ for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++) { ++ sg = rotate_pages(obj, rot_info->plane[i].offset, ++ rot_info->plane[i].width, rot_info->plane[i].height, ++ rot_info->plane[i].stride, st, sg); ++ } ++ ++ return st; ++ ++err_sg_alloc: ++ kfree(st); ++err_st_alloc: ++ ++ DRM_DEBUG_DRIVER("Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n", ++ obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size); ++ ++ return ERR_PTR(ret); ++} ++ ++static noinline struct sg_table * ++intel_partial_pages(const struct i915_ggtt_view *view, ++ struct drm_i915_gem_object *obj) ++{ ++ struct sg_table *st; ++ struct scatterlist *sg, *iter; ++ unsigned int count = view->partial.size; ++ unsigned int offset; ++ int ret = -ENOMEM; ++ ++ st = kmalloc(sizeof(*st), GFP_KERNEL); ++ if (!st) ++ goto err_st_alloc; ++ ++ ret = sg_alloc_table(st, count, GFP_KERNEL); ++ if (ret) ++ goto err_sg_alloc; ++ ++ iter = i915_gem_object_get_sg(obj, view->partial.offset, &offset); ++ GEM_BUG_ON(!iter); ++ ++ sg = st->sgl; ++ st->nents = 0; ++ do { ++ unsigned int len; ++ ++ len = min(iter->length - (offset << PAGE_SHIFT), ++ count << PAGE_SHIFT); ++ sg_set_page(sg, NULL, len, 0); ++ sg_dma_address(sg) = ++ sg_dma_address(iter) + (offset << PAGE_SHIFT); ++ sg_dma_len(sg) = len; ++ ++ st->nents++; ++ count -= len >> PAGE_SHIFT; ++ if (count == 0) { ++ sg_mark_end(sg); ++ i915_sg_trim(st); /* Drop any unused tail entries. */ ++ ++ return st; ++ } ++ ++ sg = __sg_next(sg); ++ iter = __sg_next(iter); ++ offset = 0; ++ } while (1); ++ ++err_sg_alloc: ++ kfree(st); ++err_st_alloc: ++ return ERR_PTR(ret); ++} ++ ++static int ++i915_get_ggtt_vma_pages(struct i915_vma *vma) ++{ ++ int ret; ++ ++ /* The vma->pages are only valid within the lifespan of the borrowed ++ * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so ++ * must be the vma->pages. A simple rule is that vma->pages must only ++ * be accessed when the obj->mm.pages are pinned. ++ */ ++ GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj)); ++ ++ switch (vma->ggtt_view.type) { ++ default: ++ GEM_BUG_ON(vma->ggtt_view.type); ++ /* fall through */ ++ case I915_GGTT_VIEW_NORMAL: ++ vma->pages = vma->obj->mm.pages; ++ return 0; ++ ++ case I915_GGTT_VIEW_ROTATED: ++ vma->pages = ++ intel_rotate_pages(&vma->ggtt_view.rotated, vma->obj); ++ break; ++ ++ case I915_GGTT_VIEW_PARTIAL: ++ vma->pages = intel_partial_pages(&vma->ggtt_view, vma->obj); ++ break; ++ } ++ ++ ret = 0; ++ if (IS_ERR(vma->pages)) { ++ ret = PTR_ERR(vma->pages); ++ vma->pages = NULL; ++ DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n", ++ vma->ggtt_view.type, ret); ++ } ++ return ret; ++} ++ ++/** ++ * i915_gem_gtt_reserve - reserve a node in an address_space (GTT) ++ * @vm: the &struct i915_address_space ++ * @node: the &struct drm_mm_node (typically i915_vma.mode) ++ * @size: how much space to allocate inside the GTT, ++ * must be #I915_GTT_PAGE_SIZE aligned ++ * @offset: where to insert inside the GTT, ++ * must be #I915_GTT_MIN_ALIGNMENT aligned, and the node ++ * (@offset + @size) must fit within the address space ++ * @color: color to apply to node, if this node is not from a VMA, ++ * color must be #I915_COLOR_UNEVICTABLE ++ * @flags: control search and eviction behaviour ++ * ++ * i915_gem_gtt_reserve() tries to insert the @node at the exact @offset inside ++ * the address space (using @size and @color). If the @node does not fit, it ++ * tries to evict any overlapping nodes from the GTT, including any ++ * neighbouring nodes if the colors do not match (to ensure guard pages between ++ * differing domains). See i915_gem_evict_for_node() for the gory details ++ * on the eviction algorithm. #PIN_NONBLOCK may used to prevent waiting on ++ * evicting active overlapping objects, and any overlapping node that is pinned ++ * or marked as unevictable will also result in failure. ++ * ++ * Returns: 0 on success, -ENOSPC if no suitable hole is found, -EINTR if ++ * asked to wait for eviction and interrupted. ++ */ ++int i915_gem_gtt_reserve(struct i915_address_space *vm, ++ struct drm_mm_node *node, ++ u64 size, u64 offset, unsigned long color, ++ unsigned int flags) ++{ ++ int err; ++ ++ GEM_BUG_ON(!size); ++ GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); ++ GEM_BUG_ON(!IS_ALIGNED(offset, I915_GTT_MIN_ALIGNMENT)); ++ GEM_BUG_ON(range_overflows(offset, size, vm->total)); ++ GEM_BUG_ON(vm == &vm->i915->mm.aliasing_ppgtt->vm); ++ GEM_BUG_ON(drm_mm_node_allocated(node)); ++ ++ node->size = size; ++ node->start = offset; ++ node->color = color; ++ ++ err = drm_mm_reserve_node(&vm->mm, node); ++ if (err != -ENOSPC) ++ return err; ++ ++ if (flags & PIN_NOEVICT) ++ return -ENOSPC; ++ ++ err = i915_gem_evict_for_node(vm, node, flags); ++ if (err == 0) ++ err = drm_mm_reserve_node(&vm->mm, node); ++ ++ return err; ++} ++ ++static u64 random_offset(u64 start, u64 end, u64 len, u64 align) ++{ ++ u64 range, addr; ++ ++ GEM_BUG_ON(range_overflows(start, len, end)); ++ GEM_BUG_ON(round_up(start, align) > round_down(end - len, align)); ++ ++ range = round_down(end - len, align) - round_up(start, align); ++ if (range) { ++ if (sizeof(unsigned long) == sizeof(u64)) { ++ addr = get_random_long(); ++ } else { ++ addr = get_random_int(); ++ if (range > U32_MAX) { ++ addr <<= 32; ++ addr |= get_random_int(); ++ } ++ } ++ div64_u64_rem(addr, range, &addr); ++ start += addr; ++ } ++ ++ return round_up(start, align); ++} ++ ++/** ++ * i915_gem_gtt_insert - insert a node into an address_space (GTT) ++ * @vm: the &struct i915_address_space ++ * @node: the &struct drm_mm_node (typically i915_vma.node) ++ * @size: how much space to allocate inside the GTT, ++ * must be #I915_GTT_PAGE_SIZE aligned ++ * @alignment: required alignment of starting offset, may be 0 but ++ * if specified, this must be a power-of-two and at least ++ * #I915_GTT_MIN_ALIGNMENT ++ * @color: color to apply to node ++ * @start: start of any range restriction inside GTT (0 for all), ++ * must be #I915_GTT_PAGE_SIZE aligned ++ * @end: end of any range restriction inside GTT (U64_MAX for all), ++ * must be #I915_GTT_PAGE_SIZE aligned if not U64_MAX ++ * @flags: control search and eviction behaviour ++ * ++ * i915_gem_gtt_insert() first searches for an available hole into which ++ * is can insert the node. The hole address is aligned to @alignment and ++ * its @size must then fit entirely within the [@start, @end] bounds. The ++ * nodes on either side of the hole must match @color, or else a guard page ++ * will be inserted between the two nodes (or the node evicted). If no ++ * suitable hole is found, first a victim is randomly selected and tested ++ * for eviction, otherwise then the LRU list of objects within the GTT ++ * is scanned to find the first set of replacement nodes to create the hole. ++ * Those old overlapping nodes are evicted from the GTT (and so must be ++ * rebound before any future use). Any node that is currently pinned cannot ++ * be evicted (see i915_vma_pin()). Similar if the node's VMA is currently ++ * active and #PIN_NONBLOCK is specified, that node is also skipped when ++ * searching for an eviction candidate. See i915_gem_evict_something() for ++ * the gory details on the eviction algorithm. ++ * ++ * Returns: 0 on success, -ENOSPC if no suitable hole is found, -EINTR if ++ * asked to wait for eviction and interrupted. ++ */ ++int i915_gem_gtt_insert(struct i915_address_space *vm, ++ struct drm_mm_node *node, ++ u64 size, u64 alignment, unsigned long color, ++ u64 start, u64 end, unsigned int flags) ++{ ++ enum drm_mm_insert_mode mode; ++ u64 offset; ++ int err; ++ ++ lockdep_assert_held(&vm->i915->drm.struct_mutex); ++ GEM_BUG_ON(!size); ++ GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); ++ GEM_BUG_ON(alignment && !is_power_of_2(alignment)); ++ GEM_BUG_ON(alignment && !IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT)); ++ GEM_BUG_ON(start >= end); ++ GEM_BUG_ON(start > 0 && !IS_ALIGNED(start, I915_GTT_PAGE_SIZE)); ++ GEM_BUG_ON(end < U64_MAX && !IS_ALIGNED(end, I915_GTT_PAGE_SIZE)); ++ GEM_BUG_ON(vm == &vm->i915->mm.aliasing_ppgtt->vm); ++ GEM_BUG_ON(drm_mm_node_allocated(node)); ++ ++ if (unlikely(range_overflows(start, size, end))) ++ return -ENOSPC; ++ ++ if (unlikely(round_up(start, alignment) > round_down(end - size, alignment))) ++ return -ENOSPC; ++ ++ mode = DRM_MM_INSERT_BEST; ++ if (flags & PIN_HIGH) ++ mode = DRM_MM_INSERT_HIGHEST; ++ if (flags & PIN_MAPPABLE) ++ mode = DRM_MM_INSERT_LOW; ++ ++ /* We only allocate in PAGE_SIZE/GTT_PAGE_SIZE (4096) chunks, ++ * so we know that we always have a minimum alignment of 4096. ++ * The drm_mm range manager is optimised to return results ++ * with zero alignment, so where possible use the optimal ++ * path. ++ */ ++ BUILD_BUG_ON(I915_GTT_MIN_ALIGNMENT > I915_GTT_PAGE_SIZE); ++ if (alignment <= I915_GTT_MIN_ALIGNMENT) ++ alignment = 0; ++ ++ err = drm_mm_insert_node_in_range(&vm->mm, node, ++ size, alignment, color, ++ start, end, mode); ++ if (err != -ENOSPC) ++ return err; ++ ++ if (mode & DRM_MM_INSERT_ONCE) { ++ err = drm_mm_insert_node_in_range(&vm->mm, node, ++ size, alignment, color, ++ start, end, ++ DRM_MM_INSERT_BEST); ++ if (err != -ENOSPC) ++ return err; ++ } ++ ++ if (flags & PIN_NOEVICT) ++ return -ENOSPC; ++ ++ /* No free space, pick a slot at random. ++ * ++ * There is a pathological case here using a GTT shared between ++ * mmap and GPU (i.e. ggtt/aliasing_ppgtt but not full-ppgtt): ++ * ++ * |<-- 256 MiB aperture -->||<-- 1792 MiB unmappable -->| ++ * (64k objects) (448k objects) ++ * ++ * Now imagine that the eviction LRU is ordered top-down (just because ++ * pathology meets real life), and that we need to evict an object to ++ * make room inside the aperture. The eviction scan then has to walk ++ * the 448k list before it finds one within range. And now imagine that ++ * it has to search for a new hole between every byte inside the memcpy, ++ * for several simultaneous clients. ++ * ++ * On a full-ppgtt system, if we have run out of available space, there ++ * will be lots and lots of objects in the eviction list! Again, ++ * searching that LRU list may be slow if we are also applying any ++ * range restrictions (e.g. restriction to low 4GiB) and so, for ++ * simplicity and similarilty between different GTT, try the single ++ * random replacement first. ++ */ ++ offset = random_offset(start, end, ++ size, alignment ?: I915_GTT_MIN_ALIGNMENT); ++ err = i915_gem_gtt_reserve(vm, node, size, offset, color, flags); ++ if (err != -ENOSPC) ++ return err; ++ ++ /* Randomly selected placement is pinned, do a search */ ++ err = i915_gem_evict_something(vm, size, alignment, color, ++ start, end, flags); ++ if (err) ++ return err; ++ ++ return drm_mm_insert_node_in_range(&vm->mm, node, ++ size, alignment, color, ++ start, end, DRM_MM_INSERT_EVICT); ++} ++ ++#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) ++#include "selftests/mock_gtt.c" ++#include "selftests/i915_gem_gtt.c" ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_gem_gtt.h b/drivers/gpu/drm/i915_legacy/i915_gem_gtt.h +new file mode 100644 +index 000000000000..f597f35b109b +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gem_gtt.h +@@ -0,0 +1,664 @@ ++/* ++ * Copyright © 2014 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ * Please try to maintain the following order within this file unless it makes ++ * sense to do otherwise. From top to bottom: ++ * 1. typedefs ++ * 2. #defines, and macros ++ * 3. structure definitions ++ * 4. function prototypes ++ * ++ * Within each section, please try to order by generation in ascending order, ++ * from top to bottom (ie. gen6 on the top, gen8 on the bottom). ++ */ ++ ++#ifndef __I915_GEM_GTT_H__ ++#define __I915_GEM_GTT_H__ ++ ++#include ++#include ++#include ++ ++#include "i915_request.h" ++#include "i915_reset.h" ++#include "i915_selftest.h" ++#include "i915_timeline.h" ++ ++#define I915_GTT_PAGE_SIZE_4K BIT_ULL(12) ++#define I915_GTT_PAGE_SIZE_64K BIT_ULL(16) ++#define I915_GTT_PAGE_SIZE_2M BIT_ULL(21) ++ ++#define I915_GTT_PAGE_SIZE I915_GTT_PAGE_SIZE_4K ++#define I915_GTT_MAX_PAGE_SIZE I915_GTT_PAGE_SIZE_2M ++ ++#define I915_GTT_PAGE_MASK -I915_GTT_PAGE_SIZE ++ ++#define I915_GTT_MIN_ALIGNMENT I915_GTT_PAGE_SIZE ++ ++#define I915_FENCE_REG_NONE -1 ++#define I915_MAX_NUM_FENCES 32 ++/* 32 fences + sign bit for FENCE_REG_NONE */ ++#define I915_MAX_NUM_FENCE_BITS 6 ++ ++struct drm_i915_file_private; ++struct drm_i915_fence_reg; ++struct i915_vma; ++ ++typedef u32 gen6_pte_t; ++typedef u64 gen8_pte_t; ++typedef u64 gen8_pde_t; ++typedef u64 gen8_ppgtt_pdpe_t; ++typedef u64 gen8_ppgtt_pml4e_t; ++ ++#define ggtt_total_entries(ggtt) ((ggtt)->vm.total >> PAGE_SHIFT) ++ ++/* gen6-hsw has bit 11-4 for physical addr bit 39-32 */ ++#define GEN6_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0xff0)) ++#define GEN6_PTE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) ++#define GEN6_PDE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) ++#define GEN6_PTE_CACHE_LLC (2 << 1) ++#define GEN6_PTE_UNCACHED (1 << 1) ++#define GEN6_PTE_VALID (1 << 0) ++ ++#define I915_PTES(pte_len) ((unsigned int)(PAGE_SIZE / (pte_len))) ++#define I915_PTE_MASK(pte_len) (I915_PTES(pte_len) - 1) ++#define I915_PDES 512 ++#define I915_PDE_MASK (I915_PDES - 1) ++#define NUM_PTE(pde_shift) (1 << (pde_shift - PAGE_SHIFT)) ++ ++#define GEN6_PTES I915_PTES(sizeof(gen6_pte_t)) ++#define GEN6_PD_SIZE (I915_PDES * PAGE_SIZE) ++#define GEN6_PD_ALIGN (PAGE_SIZE * 16) ++#define GEN6_PDE_SHIFT 22 ++#define GEN6_PDE_VALID (1 << 0) ++ ++#define GEN7_PTE_CACHE_L3_LLC (3 << 1) ++ ++#define BYT_PTE_SNOOPED_BY_CPU_CACHES (1 << 2) ++#define BYT_PTE_WRITEABLE (1 << 1) ++ ++/* Cacheability Control is a 4-bit value. The low three bits are stored in bits ++ * 3:1 of the PTE, while the fourth bit is stored in bit 11 of the PTE. ++ */ ++#define HSW_CACHEABILITY_CONTROL(bits) ((((bits) & 0x7) << 1) | \ ++ (((bits) & 0x8) << (11 - 3))) ++#define HSW_WB_LLC_AGE3 HSW_CACHEABILITY_CONTROL(0x2) ++#define HSW_WB_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0x3) ++#define HSW_WB_ELLC_LLC_AGE3 HSW_CACHEABILITY_CONTROL(0x8) ++#define HSW_WB_ELLC_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0xb) ++#define HSW_WT_ELLC_LLC_AGE3 HSW_CACHEABILITY_CONTROL(0x7) ++#define HSW_WT_ELLC_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0x6) ++#define HSW_PTE_UNCACHED (0) ++#define HSW_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0x7f0)) ++#define HSW_PTE_ADDR_ENCODE(addr) HSW_GTT_ADDR_ENCODE(addr) ++ ++/* GEN8 32b style address is defined as a 3 level page table: ++ * 31:30 | 29:21 | 20:12 | 11:0 ++ * PDPE | PDE | PTE | offset ++ * The difference as compared to normal x86 3 level page table is the PDPEs are ++ * programmed via register. ++ */ ++#define GEN8_3LVL_PDPES 4 ++#define GEN8_PDE_SHIFT 21 ++#define GEN8_PDE_MASK 0x1ff ++#define GEN8_PTE_SHIFT 12 ++#define GEN8_PTE_MASK 0x1ff ++#define GEN8_PTES I915_PTES(sizeof(gen8_pte_t)) ++ ++/* GEN8 48b style address is defined as a 4 level page table: ++ * 47:39 | 38:30 | 29:21 | 20:12 | 11:0 ++ * PML4E | PDPE | PDE | PTE | offset ++ */ ++#define GEN8_PML4ES_PER_PML4 512 ++#define GEN8_PML4E_SHIFT 39 ++#define GEN8_PML4E_MASK (GEN8_PML4ES_PER_PML4 - 1) ++#define GEN8_PDPE_SHIFT 30 ++/* NB: GEN8_PDPE_MASK is untrue for 32b platforms, but it has no impact on 32b page ++ * tables */ ++#define GEN8_PDPE_MASK 0x1ff ++ ++#define PPAT_UNCACHED (_PAGE_PWT | _PAGE_PCD) ++#define PPAT_CACHED_PDE 0 /* WB LLC */ ++#define PPAT_CACHED _PAGE_PAT /* WB LLCeLLC */ ++#define PPAT_DISPLAY_ELLC _PAGE_PCD /* WT eLLC */ ++ ++#define CHV_PPAT_SNOOP (1<<6) ++#define GEN8_PPAT_AGE(x) ((x)<<4) ++#define GEN8_PPAT_LLCeLLC (3<<2) ++#define GEN8_PPAT_LLCELLC (2<<2) ++#define GEN8_PPAT_LLC (1<<2) ++#define GEN8_PPAT_WB (3<<0) ++#define GEN8_PPAT_WT (2<<0) ++#define GEN8_PPAT_WC (1<<0) ++#define GEN8_PPAT_UC (0<<0) ++#define GEN8_PPAT_ELLC_OVERRIDE (0<<2) ++#define GEN8_PPAT(i, x) ((u64)(x) << ((i) * 8)) ++ ++#define GEN8_PPAT_GET_CA(x) ((x) & 3) ++#define GEN8_PPAT_GET_TC(x) ((x) & (3 << 2)) ++#define GEN8_PPAT_GET_AGE(x) ((x) & (3 << 4)) ++#define CHV_PPAT_GET_SNOOP(x) ((x) & (1 << 6)) ++ ++#define GEN8_PDE_IPS_64K BIT(11) ++#define GEN8_PDE_PS_2M BIT(7) ++ ++struct sg_table; ++ ++struct intel_rotation_info { ++ struct intel_rotation_plane_info { ++ /* tiles */ ++ unsigned int width, height, stride, offset; ++ } plane[2]; ++} __packed; ++ ++struct intel_partial_info { ++ u64 offset; ++ unsigned int size; ++} __packed; ++ ++enum i915_ggtt_view_type { ++ I915_GGTT_VIEW_NORMAL = 0, ++ I915_GGTT_VIEW_ROTATED = sizeof(struct intel_rotation_info), ++ I915_GGTT_VIEW_PARTIAL = sizeof(struct intel_partial_info), ++}; ++ ++static inline void assert_i915_gem_gtt_types(void) ++{ ++ BUILD_BUG_ON(sizeof(struct intel_rotation_info) != 8*sizeof(unsigned int)); ++ BUILD_BUG_ON(sizeof(struct intel_partial_info) != sizeof(u64) + sizeof(unsigned int)); ++ ++ /* As we encode the size of each branch inside the union into its type, ++ * we have to be careful that each branch has a unique size. ++ */ ++ switch ((enum i915_ggtt_view_type)0) { ++ case I915_GGTT_VIEW_NORMAL: ++ case I915_GGTT_VIEW_PARTIAL: ++ case I915_GGTT_VIEW_ROTATED: ++ /* gcc complains if these are identical cases */ ++ break; ++ } ++} ++ ++struct i915_ggtt_view { ++ enum i915_ggtt_view_type type; ++ union { ++ /* Members need to contain no holes/padding */ ++ struct intel_partial_info partial; ++ struct intel_rotation_info rotated; ++ }; ++}; ++ ++enum i915_cache_level; ++ ++struct i915_vma; ++ ++struct i915_page_dma { ++ struct page *page; ++ union { ++ dma_addr_t daddr; ++ ++ /* For gen6/gen7 only. This is the offset in the GGTT ++ * where the page directory entries for PPGTT begin ++ */ ++ u32 ggtt_offset; ++ }; ++}; ++ ++#define px_base(px) (&(px)->base) ++#define px_dma(px) (px_base(px)->daddr) ++ ++struct i915_page_table { ++ struct i915_page_dma base; ++ unsigned int used_ptes; ++}; ++ ++struct i915_page_directory { ++ struct i915_page_dma base; ++ ++ struct i915_page_table *page_table[I915_PDES]; /* PDEs */ ++ unsigned int used_pdes; ++}; ++ ++struct i915_page_directory_pointer { ++ struct i915_page_dma base; ++ struct i915_page_directory **page_directory; ++ unsigned int used_pdpes; ++}; ++ ++struct i915_pml4 { ++ struct i915_page_dma base; ++ struct i915_page_directory_pointer *pdps[GEN8_PML4ES_PER_PML4]; ++}; ++ ++struct i915_vma_ops { ++ /* Map an object into an address space with the given cache flags. */ ++ int (*bind_vma)(struct i915_vma *vma, ++ enum i915_cache_level cache_level, ++ u32 flags); ++ /* ++ * Unmap an object from an address space. This usually consists of ++ * setting the valid PTE entries to a reserved scratch page. ++ */ ++ void (*unbind_vma)(struct i915_vma *vma); ++ ++ int (*set_pages)(struct i915_vma *vma); ++ void (*clear_pages)(struct i915_vma *vma); ++}; ++ ++struct pagestash { ++ spinlock_t lock; ++ struct pagevec pvec; ++}; ++ ++struct i915_address_space { ++ struct drm_mm mm; ++ struct drm_i915_private *i915; ++ struct device *dma; ++ /* Every address space belongs to a struct file - except for the global ++ * GTT that is owned by the driver (and so @file is set to NULL). In ++ * principle, no information should leak from one context to another ++ * (or between files/processes etc) unless explicitly shared by the ++ * owner. Tracking the owner is important in order to free up per-file ++ * objects along with the file, to aide resource tracking, and to ++ * assign blame. ++ */ ++ struct drm_i915_file_private *file; ++ u64 total; /* size addr space maps (ex. 2GB for ggtt) */ ++ u64 reserved; /* size addr space reserved */ ++ ++ bool closed; ++ ++ struct mutex mutex; /* protects vma and our lists */ ++#define VM_CLASS_GGTT 0 ++#define VM_CLASS_PPGTT 1 ++ ++ u64 scratch_pte; ++ int scratch_order; ++ struct i915_page_dma scratch_page; ++ struct i915_page_table *scratch_pt; ++ struct i915_page_directory *scratch_pd; ++ struct i915_page_directory_pointer *scratch_pdp; /* GEN8+ & 48b PPGTT */ ++ ++ /** ++ * List of vma currently bound. ++ */ ++ struct list_head bound_list; ++ ++ /** ++ * List of vma that are not unbound. ++ */ ++ struct list_head unbound_list; ++ ++ struct pagestash free_pages; ++ ++ /* Global GTT */ ++ bool is_ggtt:1; ++ ++ /* Some systems require uncached updates of the page directories */ ++ bool pt_kmap_wc:1; ++ ++ /* Some systems support read-only mappings for GGTT and/or PPGTT */ ++ bool has_read_only:1; ++ ++ u64 (*pte_encode)(dma_addr_t addr, ++ enum i915_cache_level level, ++ u32 flags); /* Create a valid PTE */ ++#define PTE_READ_ONLY (1<<0) ++ ++ int (*allocate_va_range)(struct i915_address_space *vm, ++ u64 start, u64 length); ++ void (*clear_range)(struct i915_address_space *vm, ++ u64 start, u64 length); ++ void (*insert_page)(struct i915_address_space *vm, ++ dma_addr_t addr, ++ u64 offset, ++ enum i915_cache_level cache_level, ++ u32 flags); ++ void (*insert_entries)(struct i915_address_space *vm, ++ struct i915_vma *vma, ++ enum i915_cache_level cache_level, ++ u32 flags); ++ void (*cleanup)(struct i915_address_space *vm); ++ ++ struct i915_vma_ops vma_ops; ++ ++ I915_SELFTEST_DECLARE(struct fault_attr fault_attr); ++ I915_SELFTEST_DECLARE(bool scrub_64K); ++}; ++ ++#define i915_is_ggtt(vm) ((vm)->is_ggtt) ++ ++static inline bool ++i915_vm_is_4lvl(const struct i915_address_space *vm) ++{ ++ return (vm->total - 1) >> 32; ++} ++ ++static inline bool ++i915_vm_has_scratch_64K(struct i915_address_space *vm) ++{ ++ return vm->scratch_order == get_order(I915_GTT_PAGE_SIZE_64K); ++} ++ ++/* The Graphics Translation Table is the way in which GEN hardware translates a ++ * Graphics Virtual Address into a Physical Address. In addition to the normal ++ * collateral associated with any va->pa translations GEN hardware also has a ++ * portion of the GTT which can be mapped by the CPU and remain both coherent ++ * and correct (in cases like swizzling). That region is referred to as GMADR in ++ * the spec. ++ */ ++struct i915_ggtt { ++ struct i915_address_space vm; ++ ++ struct io_mapping iomap; /* Mapping to our CPU mappable region */ ++ struct resource gmadr; /* GMADR resource */ ++ resource_size_t mappable_end; /* End offset that we can CPU map */ ++ ++ /** "Graphics Stolen Memory" holds the global PTEs */ ++ void __iomem *gsm; ++ void (*invalidate)(struct drm_i915_private *dev_priv); ++ ++ bool do_idle_maps; ++ ++ int mtrr; ++ ++ u32 pin_bias; ++ ++ struct drm_mm_node error_capture; ++}; ++ ++struct i915_hw_ppgtt { ++ struct i915_address_space vm; ++ struct kref ref; ++ ++ intel_engine_mask_t pd_dirty_engines; ++ union { ++ struct i915_pml4 pml4; /* GEN8+ & 48b PPGTT */ ++ struct i915_page_directory_pointer pdp; /* GEN8+ */ ++ struct i915_page_directory pd; /* GEN6-7 */ ++ }; ++ ++ u32 user_handle; ++}; ++ ++struct gen6_hw_ppgtt { ++ struct i915_hw_ppgtt base; ++ ++ struct i915_vma *vma; ++ gen6_pte_t __iomem *pd_addr; ++ ++ unsigned int pin_count; ++ bool scan_for_unused_pt; ++}; ++ ++#define __to_gen6_ppgtt(base) container_of(base, struct gen6_hw_ppgtt, base) ++ ++static inline struct gen6_hw_ppgtt *to_gen6_ppgtt(struct i915_hw_ppgtt *base) ++{ ++ BUILD_BUG_ON(offsetof(struct gen6_hw_ppgtt, base)); ++ return __to_gen6_ppgtt(base); ++} ++ ++/* ++ * gen6_for_each_pde() iterates over every pde from start until start+length. ++ * If start and start+length are not perfectly divisible, the macro will round ++ * down and up as needed. Start=0 and length=2G effectively iterates over ++ * every PDE in the system. The macro modifies ALL its parameters except 'pd', ++ * so each of the other parameters should preferably be a simple variable, or ++ * at most an lvalue with no side-effects! ++ */ ++#define gen6_for_each_pde(pt, pd, start, length, iter) \ ++ for (iter = gen6_pde_index(start); \ ++ length > 0 && iter < I915_PDES && \ ++ (pt = (pd)->page_table[iter], true); \ ++ ({ u32 temp = ALIGN(start+1, 1 << GEN6_PDE_SHIFT); \ ++ temp = min(temp - start, length); \ ++ start += temp, length -= temp; }), ++iter) ++ ++#define gen6_for_all_pdes(pt, pd, iter) \ ++ for (iter = 0; \ ++ iter < I915_PDES && \ ++ (pt = (pd)->page_table[iter], true); \ ++ ++iter) ++ ++static inline u32 i915_pte_index(u64 address, unsigned int pde_shift) ++{ ++ const u32 mask = NUM_PTE(pde_shift) - 1; ++ ++ return (address >> PAGE_SHIFT) & mask; ++} ++ ++/* Helper to counts the number of PTEs within the given length. This count ++ * does not cross a page table boundary, so the max value would be ++ * GEN6_PTES for GEN6, and GEN8_PTES for GEN8. ++*/ ++static inline u32 i915_pte_count(u64 addr, u64 length, unsigned int pde_shift) ++{ ++ const u64 mask = ~((1ULL << pde_shift) - 1); ++ u64 end; ++ ++ GEM_BUG_ON(length == 0); ++ GEM_BUG_ON(offset_in_page(addr | length)); ++ ++ end = addr + length; ++ ++ if ((addr & mask) != (end & mask)) ++ return NUM_PTE(pde_shift) - i915_pte_index(addr, pde_shift); ++ ++ return i915_pte_index(end, pde_shift) - i915_pte_index(addr, pde_shift); ++} ++ ++static inline u32 i915_pde_index(u64 addr, u32 shift) ++{ ++ return (addr >> shift) & I915_PDE_MASK; ++} ++ ++static inline u32 gen6_pte_index(u32 addr) ++{ ++ return i915_pte_index(addr, GEN6_PDE_SHIFT); ++} ++ ++static inline u32 gen6_pte_count(u32 addr, u32 length) ++{ ++ return i915_pte_count(addr, length, GEN6_PDE_SHIFT); ++} ++ ++static inline u32 gen6_pde_index(u32 addr) ++{ ++ return i915_pde_index(addr, GEN6_PDE_SHIFT); ++} ++ ++static inline unsigned int ++i915_pdpes_per_pdp(const struct i915_address_space *vm) ++{ ++ if (i915_vm_is_4lvl(vm)) ++ return GEN8_PML4ES_PER_PML4; ++ ++ return GEN8_3LVL_PDPES; ++} ++ ++/* Equivalent to the gen6 version, For each pde iterates over every pde ++ * between from start until start + length. On gen8+ it simply iterates ++ * over every page directory entry in a page directory. ++ */ ++#define gen8_for_each_pde(pt, pd, start, length, iter) \ ++ for (iter = gen8_pde_index(start); \ ++ length > 0 && iter < I915_PDES && \ ++ (pt = (pd)->page_table[iter], true); \ ++ ({ u64 temp = ALIGN(start+1, 1 << GEN8_PDE_SHIFT); \ ++ temp = min(temp - start, length); \ ++ start += temp, length -= temp; }), ++iter) ++ ++#define gen8_for_each_pdpe(pd, pdp, start, length, iter) \ ++ for (iter = gen8_pdpe_index(start); \ ++ length > 0 && iter < i915_pdpes_per_pdp(vm) && \ ++ (pd = (pdp)->page_directory[iter], true); \ ++ ({ u64 temp = ALIGN(start+1, 1 << GEN8_PDPE_SHIFT); \ ++ temp = min(temp - start, length); \ ++ start += temp, length -= temp; }), ++iter) ++ ++#define gen8_for_each_pml4e(pdp, pml4, start, length, iter) \ ++ for (iter = gen8_pml4e_index(start); \ ++ length > 0 && iter < GEN8_PML4ES_PER_PML4 && \ ++ (pdp = (pml4)->pdps[iter], true); \ ++ ({ u64 temp = ALIGN(start+1, 1ULL << GEN8_PML4E_SHIFT); \ ++ temp = min(temp - start, length); \ ++ start += temp, length -= temp; }), ++iter) ++ ++static inline u32 gen8_pte_index(u64 address) ++{ ++ return i915_pte_index(address, GEN8_PDE_SHIFT); ++} ++ ++static inline u32 gen8_pde_index(u64 address) ++{ ++ return i915_pde_index(address, GEN8_PDE_SHIFT); ++} ++ ++static inline u32 gen8_pdpe_index(u64 address) ++{ ++ return (address >> GEN8_PDPE_SHIFT) & GEN8_PDPE_MASK; ++} ++ ++static inline u32 gen8_pml4e_index(u64 address) ++{ ++ return (address >> GEN8_PML4E_SHIFT) & GEN8_PML4E_MASK; ++} ++ ++static inline u64 gen8_pte_count(u64 address, u64 length) ++{ ++ return i915_pte_count(address, length, GEN8_PDE_SHIFT); ++} ++ ++static inline dma_addr_t ++i915_page_dir_dma_addr(const struct i915_hw_ppgtt *ppgtt, const unsigned n) ++{ ++ return px_dma(ppgtt->pdp.page_directory[n]); ++} ++ ++static inline struct i915_ggtt * ++i915_vm_to_ggtt(struct i915_address_space *vm) ++{ ++ GEM_BUG_ON(!i915_is_ggtt(vm)); ++ return container_of(vm, struct i915_ggtt, vm); ++} ++ ++#define INTEL_MAX_PPAT_ENTRIES 8 ++#define INTEL_PPAT_PERFECT_MATCH (~0U) ++ ++struct intel_ppat; ++ ++struct intel_ppat_entry { ++ struct intel_ppat *ppat; ++ struct kref ref; ++ u8 value; ++}; ++ ++struct intel_ppat { ++ struct intel_ppat_entry entries[INTEL_MAX_PPAT_ENTRIES]; ++ DECLARE_BITMAP(used, INTEL_MAX_PPAT_ENTRIES); ++ DECLARE_BITMAP(dirty, INTEL_MAX_PPAT_ENTRIES); ++ unsigned int max_entries; ++ u8 clear_value; ++ /* ++ * Return a score to show how two PPAT values match, ++ * a INTEL_PPAT_PERFECT_MATCH indicates a perfect match ++ */ ++ unsigned int (*match)(u8 src, u8 dst); ++ void (*update_hw)(struct drm_i915_private *i915); ++ ++ struct drm_i915_private *i915; ++}; ++ ++const struct intel_ppat_entry * ++intel_ppat_get(struct drm_i915_private *i915, u8 value); ++void intel_ppat_put(const struct intel_ppat_entry *entry); ++ ++int i915_gem_init_aliasing_ppgtt(struct drm_i915_private *i915); ++void i915_gem_fini_aliasing_ppgtt(struct drm_i915_private *i915); ++ ++int i915_ggtt_probe_hw(struct drm_i915_private *dev_priv); ++int i915_ggtt_init_hw(struct drm_i915_private *dev_priv); ++int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv); ++void i915_ggtt_enable_guc(struct drm_i915_private *i915); ++void i915_ggtt_disable_guc(struct drm_i915_private *i915); ++int i915_gem_init_ggtt(struct drm_i915_private *dev_priv); ++void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv); ++ ++int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv); ++ ++struct i915_hw_ppgtt *i915_ppgtt_create(struct drm_i915_private *dev_priv); ++void i915_ppgtt_release(struct kref *kref); ++ ++static inline struct i915_hw_ppgtt *i915_ppgtt_get(struct i915_hw_ppgtt *ppgtt) ++{ ++ kref_get(&ppgtt->ref); ++ return ppgtt; ++} ++ ++static inline void i915_ppgtt_put(struct i915_hw_ppgtt *ppgtt) ++{ ++ if (ppgtt) ++ kref_put(&ppgtt->ref, i915_ppgtt_release); ++} ++ ++int gen6_ppgtt_pin(struct i915_hw_ppgtt *base); ++void gen6_ppgtt_unpin(struct i915_hw_ppgtt *base); ++void gen6_ppgtt_unpin_all(struct i915_hw_ppgtt *base); ++ ++void i915_check_and_clear_faults(struct drm_i915_private *dev_priv); ++void i915_gem_suspend_gtt_mappings(struct drm_i915_private *dev_priv); ++void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv); ++ ++int __must_check i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj, ++ struct sg_table *pages); ++void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj, ++ struct sg_table *pages); ++ ++int i915_gem_gtt_reserve(struct i915_address_space *vm, ++ struct drm_mm_node *node, ++ u64 size, u64 offset, unsigned long color, ++ unsigned int flags); ++ ++int i915_gem_gtt_insert(struct i915_address_space *vm, ++ struct drm_mm_node *node, ++ u64 size, u64 alignment, unsigned long color, ++ u64 start, u64 end, unsigned int flags); ++ ++/* Flags used by pin/bind&friends. */ ++#define PIN_NONBLOCK BIT_ULL(0) ++#define PIN_NONFAULT BIT_ULL(1) ++#define PIN_NOEVICT BIT_ULL(2) ++#define PIN_MAPPABLE BIT_ULL(3) ++#define PIN_ZONE_4G BIT_ULL(4) ++#define PIN_HIGH BIT_ULL(5) ++#define PIN_OFFSET_BIAS BIT_ULL(6) ++#define PIN_OFFSET_FIXED BIT_ULL(7) ++ ++#define PIN_MBZ BIT_ULL(8) /* I915_VMA_PIN_OVERFLOW */ ++#define PIN_GLOBAL BIT_ULL(9) /* I915_VMA_GLOBAL_BIND */ ++#define PIN_USER BIT_ULL(10) /* I915_VMA_LOCAL_BIND */ ++#define PIN_UPDATE BIT_ULL(11) ++ ++#define PIN_OFFSET_MASK (-I915_GTT_PAGE_SIZE) ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_gem_internal.c b/drivers/gpu/drm/i915_legacy/i915_gem_internal.c +new file mode 100644 +index 000000000000..ab627ed1269c +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gem_internal.c +@@ -0,0 +1,210 @@ ++/* ++ * Copyright © 2014-2016 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#include ++#include "i915_drv.h" ++ ++#define QUIET (__GFP_NORETRY | __GFP_NOWARN) ++#define MAYFAIL (__GFP_RETRY_MAYFAIL | __GFP_NOWARN) ++ ++/* convert swiotlb segment size into sensible units (pages)! */ ++#define IO_TLB_SEGPAGES (IO_TLB_SEGSIZE << IO_TLB_SHIFT >> PAGE_SHIFT) ++ ++static void internal_free_pages(struct sg_table *st) ++{ ++ struct scatterlist *sg; ++ ++ for (sg = st->sgl; sg; sg = __sg_next(sg)) { ++ if (sg_page(sg)) ++ __free_pages(sg_page(sg), get_order(sg->length)); ++ } ++ ++ sg_free_table(st); ++ kfree(st); ++} ++ ++static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj) ++{ ++ struct drm_i915_private *i915 = to_i915(obj->base.dev); ++ struct sg_table *st; ++ struct scatterlist *sg; ++ unsigned int sg_page_sizes; ++ unsigned int npages; ++ int max_order; ++ gfp_t gfp; ++ ++ max_order = MAX_ORDER; ++#ifdef CONFIG_SWIOTLB ++ if (swiotlb_nr_tbl()) { ++ unsigned int max_segment; ++ ++ max_segment = swiotlb_max_segment(); ++ if (max_segment) { ++ max_segment = max_t(unsigned int, max_segment, ++ PAGE_SIZE) >> PAGE_SHIFT; ++ max_order = min(max_order, ilog2(max_segment)); ++ } ++ } ++#endif ++ ++ gfp = GFP_KERNEL | __GFP_HIGHMEM | __GFP_RECLAIMABLE; ++ if (IS_I965GM(i915) || IS_I965G(i915)) { ++ /* 965gm cannot relocate objects above 4GiB. */ ++ gfp &= ~__GFP_HIGHMEM; ++ gfp |= __GFP_DMA32; ++ } ++ ++create_st: ++ st = kmalloc(sizeof(*st), GFP_KERNEL); ++ if (!st) ++ return -ENOMEM; ++ ++ npages = obj->base.size / PAGE_SIZE; ++ if (sg_alloc_table(st, npages, GFP_KERNEL)) { ++ kfree(st); ++ return -ENOMEM; ++ } ++ ++ sg = st->sgl; ++ st->nents = 0; ++ sg_page_sizes = 0; ++ ++ do { ++ int order = min(fls(npages) - 1, max_order); ++ struct page *page; ++ ++ do { ++ page = alloc_pages(gfp | (order ? QUIET : MAYFAIL), ++ order); ++ if (page) ++ break; ++ if (!order--) ++ goto err; ++ ++ /* Limit subsequent allocations as well */ ++ max_order = order; ++ } while (1); ++ ++ sg_set_page(sg, page, PAGE_SIZE << order, 0); ++ sg_page_sizes |= PAGE_SIZE << order; ++ st->nents++; ++ ++ npages -= 1 << order; ++ if (!npages) { ++ sg_mark_end(sg); ++ break; ++ } ++ ++ sg = __sg_next(sg); ++ } while (1); ++ ++ if (i915_gem_gtt_prepare_pages(obj, st)) { ++ /* Failed to dma-map try again with single page sg segments */ ++ if (get_order(st->sgl->length)) { ++ internal_free_pages(st); ++ max_order = 0; ++ goto create_st; ++ } ++ goto err; ++ } ++ ++ /* Mark the pages as dontneed whilst they are still pinned. As soon ++ * as they are unpinned they are allowed to be reaped by the shrinker, ++ * and the caller is expected to repopulate - the contents of this ++ * object are only valid whilst active and pinned. ++ */ ++ obj->mm.madv = I915_MADV_DONTNEED; ++ ++ __i915_gem_object_set_pages(obj, st, sg_page_sizes); ++ ++ return 0; ++ ++err: ++ sg_set_page(sg, NULL, 0, 0); ++ sg_mark_end(sg); ++ internal_free_pages(st); ++ ++ return -ENOMEM; ++} ++ ++static void i915_gem_object_put_pages_internal(struct drm_i915_gem_object *obj, ++ struct sg_table *pages) ++{ ++ i915_gem_gtt_finish_pages(obj, pages); ++ internal_free_pages(pages); ++ ++ obj->mm.dirty = false; ++ obj->mm.madv = I915_MADV_WILLNEED; ++} ++ ++static const struct drm_i915_gem_object_ops i915_gem_object_internal_ops = { ++ .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | ++ I915_GEM_OBJECT_IS_SHRINKABLE, ++ .get_pages = i915_gem_object_get_pages_internal, ++ .put_pages = i915_gem_object_put_pages_internal, ++}; ++ ++/** ++ * i915_gem_object_create_internal: create an object with volatile pages ++ * @i915: the i915 device ++ * @size: the size in bytes of backing storage to allocate for the object ++ * ++ * Creates a new object that wraps some internal memory for private use. ++ * This object is not backed by swappable storage, and as such its contents ++ * are volatile and only valid whilst pinned. If the object is reaped by the ++ * shrinker, its pages and data will be discarded. Equally, it is not a full ++ * GEM object and so not valid for access from userspace. This makes it useful ++ * for hardware interfaces like ringbuffers (which are pinned from the time ++ * the request is written to the time the hardware stops accessing it), but ++ * not for contexts (which need to be preserved when not active for later ++ * reuse). Note that it is not cleared upon allocation. ++ */ ++struct drm_i915_gem_object * ++i915_gem_object_create_internal(struct drm_i915_private *i915, ++ phys_addr_t size) ++{ ++ struct drm_i915_gem_object *obj; ++ unsigned int cache_level; ++ ++ GEM_BUG_ON(!size); ++ GEM_BUG_ON(!IS_ALIGNED(size, PAGE_SIZE)); ++ ++ if (overflows_type(size, obj->base.size)) ++ return ERR_PTR(-E2BIG); ++ ++ obj = i915_gem_object_alloc(); ++ if (!obj) ++ return ERR_PTR(-ENOMEM); ++ ++ drm_gem_private_object_init(&i915->drm, &obj->base, size); ++ i915_gem_object_init(obj, &i915_gem_object_internal_ops); ++ ++ obj->read_domains = I915_GEM_DOMAIN_CPU; ++ obj->write_domain = I915_GEM_DOMAIN_CPU; ++ ++ cache_level = HAS_LLC(i915) ? I915_CACHE_LLC : I915_CACHE_NONE; ++ i915_gem_object_set_cache_coherency(obj, cache_level); ++ ++ return obj; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_gem_object.c b/drivers/gpu/drm/i915_legacy/i915_gem_object.c +new file mode 100644 +index 000000000000..ac6a5ab84586 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gem_object.c +@@ -0,0 +1,90 @@ ++/* ++ * Copyright © 2017 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#include "i915_drv.h" ++#include "i915_gem_object.h" ++#include "i915_globals.h" ++ ++static struct i915_global_object { ++ struct i915_global base; ++ struct kmem_cache *slab_objects; ++} global; ++ ++struct drm_i915_gem_object *i915_gem_object_alloc(void) ++{ ++ return kmem_cache_zalloc(global.slab_objects, GFP_KERNEL); ++} ++ ++void i915_gem_object_free(struct drm_i915_gem_object *obj) ++{ ++ return kmem_cache_free(global.slab_objects, obj); ++} ++ ++/** ++ * Mark up the object's coherency levels for a given cache_level ++ * @obj: #drm_i915_gem_object ++ * @cache_level: cache level ++ */ ++void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj, ++ unsigned int cache_level) ++{ ++ obj->cache_level = cache_level; ++ ++ if (cache_level != I915_CACHE_NONE) ++ obj->cache_coherent = (I915_BO_CACHE_COHERENT_FOR_READ | ++ I915_BO_CACHE_COHERENT_FOR_WRITE); ++ else if (HAS_LLC(to_i915(obj->base.dev))) ++ obj->cache_coherent = I915_BO_CACHE_COHERENT_FOR_READ; ++ else ++ obj->cache_coherent = 0; ++ ++ obj->cache_dirty = ++ !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE); ++} ++ ++static void i915_global_objects_shrink(void) ++{ ++ kmem_cache_shrink(global.slab_objects); ++} ++ ++static void i915_global_objects_exit(void) ++{ ++ kmem_cache_destroy(global.slab_objects); ++} ++ ++static struct i915_global_object global = { { ++ .shrink = i915_global_objects_shrink, ++ .exit = i915_global_objects_exit, ++} }; ++ ++int __init i915_global_objects_init(void) ++{ ++ global.slab_objects = ++ KMEM_CACHE(drm_i915_gem_object, SLAB_HWCACHE_ALIGN); ++ if (!global.slab_objects) ++ return -ENOMEM; ++ ++ i915_global_register(&global.base); ++ return 0; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_gem_object.h b/drivers/gpu/drm/i915_legacy/i915_gem_object.h +new file mode 100644 +index 000000000000..ca93a40c0c87 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gem_object.h +@@ -0,0 +1,509 @@ ++/* ++ * Copyright © 2016 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#ifndef __I915_GEM_OBJECT_H__ ++#define __I915_GEM_OBJECT_H__ ++ ++#include ++ ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include "i915_request.h" ++#include "i915_selftest.h" ++ ++struct drm_i915_gem_object; ++ ++/* ++ * struct i915_lut_handle tracks the fast lookups from handle to vma used ++ * for execbuf. Although we use a radixtree for that mapping, in order to ++ * remove them as the object or context is closed, we need a secondary list ++ * and a translation entry (i915_lut_handle). ++ */ ++struct i915_lut_handle { ++ struct list_head obj_link; ++ struct list_head ctx_link; ++ struct i915_gem_context *ctx; ++ u32 handle; ++}; ++ ++struct drm_i915_gem_object_ops { ++ unsigned int flags; ++#define I915_GEM_OBJECT_HAS_STRUCT_PAGE BIT(0) ++#define I915_GEM_OBJECT_IS_SHRINKABLE BIT(1) ++#define I915_GEM_OBJECT_IS_PROXY BIT(2) ++#define I915_GEM_OBJECT_ASYNC_CANCEL BIT(3) ++ ++ /* Interface between the GEM object and its backing storage. ++ * get_pages() is called once prior to the use of the associated set ++ * of pages before to binding them into the GTT, and put_pages() is ++ * called after we no longer need them. As we expect there to be ++ * associated cost with migrating pages between the backing storage ++ * and making them available for the GPU (e.g. clflush), we may hold ++ * onto the pages after they are no longer referenced by the GPU ++ * in case they may be used again shortly (for example migrating the ++ * pages to a different memory domain within the GTT). put_pages() ++ * will therefore most likely be called when the object itself is ++ * being released or under memory pressure (where we attempt to ++ * reap pages for the shrinker). ++ */ ++ int (*get_pages)(struct drm_i915_gem_object *); ++ void (*put_pages)(struct drm_i915_gem_object *, struct sg_table *); ++ ++ int (*pwrite)(struct drm_i915_gem_object *, ++ const struct drm_i915_gem_pwrite *); ++ ++ int (*dmabuf_export)(struct drm_i915_gem_object *); ++ void (*release)(struct drm_i915_gem_object *); ++}; ++ ++struct drm_i915_gem_object { ++ struct drm_gem_object base; ++ ++ const struct drm_i915_gem_object_ops *ops; ++ ++ struct { ++ /** ++ * @vma.lock: protect the list/tree of vmas ++ */ ++ spinlock_t lock; ++ ++ /** ++ * @vma.list: List of VMAs backed by this object ++ * ++ * The VMA on this list are ordered by type, all GGTT vma are ++ * placed at the head and all ppGTT vma are placed at the tail. ++ * The different types of GGTT vma are unordered between ++ * themselves, use the @vma.tree (which has a defined order ++ * between all VMA) to quickly find an exact match. ++ */ ++ struct list_head list; ++ ++ /** ++ * @vma.tree: Ordered tree of VMAs backed by this object ++ * ++ * All VMA created for this object are placed in the @vma.tree ++ * for fast retrieval via a binary search in ++ * i915_vma_instance(). They are also added to @vma.list for ++ * easy iteration. ++ */ ++ struct rb_root tree; ++ } vma; ++ ++ /** ++ * @lut_list: List of vma lookup entries in use for this object. ++ * ++ * If this object is closed, we need to remove all of its VMA from ++ * the fast lookup index in associated contexts; @lut_list provides ++ * this translation from object to context->handles_vma. ++ */ ++ struct list_head lut_list; ++ ++ /** Stolen memory for this object, instead of being backed by shmem. */ ++ struct drm_mm_node *stolen; ++ union { ++ struct rcu_head rcu; ++ struct llist_node freed; ++ }; ++ ++ /** ++ * Whether the object is currently in the GGTT mmap. ++ */ ++ unsigned int userfault_count; ++ struct list_head userfault_link; ++ ++ struct list_head batch_pool_link; ++ I915_SELFTEST_DECLARE(struct list_head st_link); ++ ++ unsigned long flags; ++ ++ /** ++ * Have we taken a reference for the object for incomplete GPU ++ * activity? ++ */ ++#define I915_BO_ACTIVE_REF 0 ++ ++ /* ++ * Is the object to be mapped as read-only to the GPU ++ * Only honoured if hardware has relevant pte bit ++ */ ++ unsigned int cache_level:3; ++ unsigned int cache_coherent:2; ++#define I915_BO_CACHE_COHERENT_FOR_READ BIT(0) ++#define I915_BO_CACHE_COHERENT_FOR_WRITE BIT(1) ++ unsigned int cache_dirty:1; ++ ++ /** ++ * @read_domains: Read memory domains. ++ * ++ * These monitor which caches contain read/write data related to the ++ * object. When transitioning from one set of domains to another, ++ * the driver is called to ensure that caches are suitably flushed and ++ * invalidated. ++ */ ++ u16 read_domains; ++ ++ /** ++ * @write_domain: Corresponding unique write memory domain. ++ */ ++ u16 write_domain; ++ ++ atomic_t frontbuffer_bits; ++ unsigned int frontbuffer_ggtt_origin; /* write once */ ++ struct i915_active_request frontbuffer_write; ++ ++ /** Current tiling stride for the object, if it's tiled. */ ++ unsigned int tiling_and_stride; ++#define FENCE_MINIMUM_STRIDE 128 /* See i915_tiling_ok() */ ++#define TILING_MASK (FENCE_MINIMUM_STRIDE-1) ++#define STRIDE_MASK (~TILING_MASK) ++ ++ /** Count of VMA actually bound by this object */ ++ unsigned int bind_count; ++ unsigned int active_count; ++ /** Count of how many global VMA are currently pinned for use by HW */ ++ unsigned int pin_global; ++ ++ struct { ++ struct mutex lock; /* protects the pages and their use */ ++ atomic_t pages_pin_count; ++ ++ struct sg_table *pages; ++ void *mapping; ++ ++ /* TODO: whack some of this into the error state */ ++ struct i915_page_sizes { ++ /** ++ * The sg mask of the pages sg_table. i.e the mask of ++ * of the lengths for each sg entry. ++ */ ++ unsigned int phys; ++ ++ /** ++ * The gtt page sizes we are allowed to use given the ++ * sg mask and the supported page sizes. This will ++ * express the smallest unit we can use for the whole ++ * object, as well as the larger sizes we may be able ++ * to use opportunistically. ++ */ ++ unsigned int sg; ++ ++ /** ++ * The actual gtt page size usage. Since we can have ++ * multiple vma associated with this object we need to ++ * prevent any trampling of state, hence a copy of this ++ * struct also lives in each vma, therefore the gtt ++ * value here should only be read/write through the vma. ++ */ ++ unsigned int gtt; ++ } page_sizes; ++ ++ I915_SELFTEST_DECLARE(unsigned int page_mask); ++ ++ struct i915_gem_object_page_iter { ++ struct scatterlist *sg_pos; ++ unsigned int sg_idx; /* in pages, but 32bit eek! */ ++ ++ struct radix_tree_root radix; ++ struct mutex lock; /* protects this cache */ ++ } get_page; ++ ++ /** ++ * Element within i915->mm.unbound_list or i915->mm.bound_list, ++ * locked by i915->mm.obj_lock. ++ */ ++ struct list_head link; ++ ++ /** ++ * Advice: are the backing pages purgeable? ++ */ ++ unsigned int madv:2; ++ ++ /** ++ * This is set if the object has been written to since the ++ * pages were last acquired. ++ */ ++ bool dirty:1; ++ ++ /** ++ * This is set if the object has been pinned due to unknown ++ * swizzling. ++ */ ++ bool quirked:1; ++ } mm; ++ ++ /** Breadcrumb of last rendering to the buffer. ++ * There can only be one writer, but we allow for multiple readers. ++ * If there is a writer that necessarily implies that all other ++ * read requests are complete - but we may only be lazily clearing ++ * the read requests. A read request is naturally the most recent ++ * request on a ring, so we may have two different write and read ++ * requests on one ring where the write request is older than the ++ * read request. This allows for the CPU to read from an active ++ * buffer by only waiting for the write to complete. ++ */ ++ struct reservation_object *resv; ++ ++ /** References from framebuffers, locks out tiling changes. */ ++ unsigned int framebuffer_references; ++ ++ /** Record of address bit 17 of each page at last unbind. */ ++ unsigned long *bit_17; ++ ++ union { ++ struct i915_gem_userptr { ++ uintptr_t ptr; ++ ++ struct i915_mm_struct *mm; ++ struct i915_mmu_object *mmu_object; ++ struct work_struct *work; ++ } userptr; ++ ++ unsigned long scratch; ++ ++ void *gvt_info; ++ }; ++ ++ /** for phys allocated objects */ ++ struct drm_dma_handle *phys_handle; ++ ++ struct reservation_object __builtin_resv; ++}; ++ ++static inline struct drm_i915_gem_object * ++to_intel_bo(struct drm_gem_object *gem) ++{ ++ /* Assert that to_intel_bo(NULL) == NULL */ ++ BUILD_BUG_ON(offsetof(struct drm_i915_gem_object, base)); ++ ++ return container_of(gem, struct drm_i915_gem_object, base); ++} ++ ++struct drm_i915_gem_object *i915_gem_object_alloc(void); ++void i915_gem_object_free(struct drm_i915_gem_object *obj); ++ ++/** ++ * i915_gem_object_lookup_rcu - look up a temporary GEM object from its handle ++ * @filp: DRM file private date ++ * @handle: userspace handle ++ * ++ * Returns: ++ * ++ * A pointer to the object named by the handle if such exists on @filp, NULL ++ * otherwise. This object is only valid whilst under the RCU read lock, and ++ * note carefully the object may be in the process of being destroyed. ++ */ ++static inline struct drm_i915_gem_object * ++i915_gem_object_lookup_rcu(struct drm_file *file, u32 handle) ++{ ++#ifdef CONFIG_LOCKDEP ++ WARN_ON(debug_locks && !lock_is_held(&rcu_lock_map)); ++#endif ++ return idr_find(&file->object_idr, handle); ++} ++ ++static inline struct drm_i915_gem_object * ++i915_gem_object_lookup(struct drm_file *file, u32 handle) ++{ ++ struct drm_i915_gem_object *obj; ++ ++ rcu_read_lock(); ++ obj = i915_gem_object_lookup_rcu(file, handle); ++ if (obj && !kref_get_unless_zero(&obj->base.refcount)) ++ obj = NULL; ++ rcu_read_unlock(); ++ ++ return obj; ++} ++ ++__deprecated ++extern struct drm_gem_object * ++drm_gem_object_lookup(struct drm_file *file, u32 handle); ++ ++__attribute__((nonnull)) ++static inline struct drm_i915_gem_object * ++i915_gem_object_get(struct drm_i915_gem_object *obj) ++{ ++ drm_gem_object_get(&obj->base); ++ return obj; ++} ++ ++__attribute__((nonnull)) ++static inline void ++i915_gem_object_put(struct drm_i915_gem_object *obj) ++{ ++ __drm_gem_object_put(&obj->base); ++} ++ ++static inline void i915_gem_object_lock(struct drm_i915_gem_object *obj) ++{ ++ reservation_object_lock(obj->resv, NULL); ++} ++ ++static inline void i915_gem_object_unlock(struct drm_i915_gem_object *obj) ++{ ++ reservation_object_unlock(obj->resv); ++} ++ ++static inline void ++i915_gem_object_set_readonly(struct drm_i915_gem_object *obj) ++{ ++ obj->base.vma_node.readonly = true; ++} ++ ++static inline bool ++i915_gem_object_is_readonly(const struct drm_i915_gem_object *obj) ++{ ++ return obj->base.vma_node.readonly; ++} ++ ++static inline bool ++i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj) ++{ ++ return obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE; ++} ++ ++static inline bool ++i915_gem_object_is_shrinkable(const struct drm_i915_gem_object *obj) ++{ ++ return obj->ops->flags & I915_GEM_OBJECT_IS_SHRINKABLE; ++} ++ ++static inline bool ++i915_gem_object_is_proxy(const struct drm_i915_gem_object *obj) ++{ ++ return obj->ops->flags & I915_GEM_OBJECT_IS_PROXY; ++} ++ ++static inline bool ++i915_gem_object_needs_async_cancel(const struct drm_i915_gem_object *obj) ++{ ++ return obj->ops->flags & I915_GEM_OBJECT_ASYNC_CANCEL; ++} ++ ++static inline bool ++i915_gem_object_is_active(const struct drm_i915_gem_object *obj) ++{ ++ return obj->active_count; ++} ++ ++static inline bool ++i915_gem_object_has_active_reference(const struct drm_i915_gem_object *obj) ++{ ++ return test_bit(I915_BO_ACTIVE_REF, &obj->flags); ++} ++ ++static inline void ++i915_gem_object_set_active_reference(struct drm_i915_gem_object *obj) ++{ ++ lockdep_assert_held(&obj->base.dev->struct_mutex); ++ __set_bit(I915_BO_ACTIVE_REF, &obj->flags); ++} ++ ++static inline void ++i915_gem_object_clear_active_reference(struct drm_i915_gem_object *obj) ++{ ++ lockdep_assert_held(&obj->base.dev->struct_mutex); ++ __clear_bit(I915_BO_ACTIVE_REF, &obj->flags); ++} ++ ++void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj); ++ ++static inline bool ++i915_gem_object_is_framebuffer(const struct drm_i915_gem_object *obj) ++{ ++ return READ_ONCE(obj->framebuffer_references); ++} ++ ++static inline unsigned int ++i915_gem_object_get_tiling(const struct drm_i915_gem_object *obj) ++{ ++ return obj->tiling_and_stride & TILING_MASK; ++} ++ ++static inline bool ++i915_gem_object_is_tiled(const struct drm_i915_gem_object *obj) ++{ ++ return i915_gem_object_get_tiling(obj) != I915_TILING_NONE; ++} ++ ++static inline unsigned int ++i915_gem_object_get_stride(const struct drm_i915_gem_object *obj) ++{ ++ return obj->tiling_and_stride & STRIDE_MASK; ++} ++ ++static inline unsigned int ++i915_gem_tile_height(unsigned int tiling) ++{ ++ GEM_BUG_ON(!tiling); ++ return tiling == I915_TILING_Y ? 32 : 8; ++} ++ ++static inline unsigned int ++i915_gem_object_get_tile_height(const struct drm_i915_gem_object *obj) ++{ ++ return i915_gem_tile_height(i915_gem_object_get_tiling(obj)); ++} ++ ++static inline unsigned int ++i915_gem_object_get_tile_row_size(const struct drm_i915_gem_object *obj) ++{ ++ return (i915_gem_object_get_stride(obj) * ++ i915_gem_object_get_tile_height(obj)); ++} ++ ++int i915_gem_object_set_tiling(struct drm_i915_gem_object *obj, ++ unsigned int tiling, unsigned int stride); ++ ++static inline struct intel_engine_cs * ++i915_gem_object_last_write_engine(struct drm_i915_gem_object *obj) ++{ ++ struct intel_engine_cs *engine = NULL; ++ struct dma_fence *fence; ++ ++ rcu_read_lock(); ++ fence = reservation_object_get_excl_rcu(obj->resv); ++ rcu_read_unlock(); ++ ++ if (fence && dma_fence_is_i915(fence) && !dma_fence_is_signaled(fence)) ++ engine = to_request(fence)->engine; ++ dma_fence_put(fence); ++ ++ return engine; ++} ++ ++void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj, ++ unsigned int cache_level); ++void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj); ++ ++void __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj, ++ struct sg_table *pages, ++ bool needs_clflush); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_gem_render_state.c b/drivers/gpu/drm/i915_legacy/i915_gem_render_state.c +new file mode 100644 +index 000000000000..9440024c763f +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gem_render_state.c +@@ -0,0 +1,233 @@ ++/* ++ * Copyright © 2014 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ * Authors: ++ * Mika Kuoppala ++ * ++ */ ++ ++#include "i915_drv.h" ++#include "i915_gem_render_state.h" ++#include "intel_renderstate.h" ++ ++struct intel_render_state { ++ const struct intel_renderstate_rodata *rodata; ++ struct drm_i915_gem_object *obj; ++ struct i915_vma *vma; ++ u32 batch_offset; ++ u32 batch_size; ++ u32 aux_offset; ++ u32 aux_size; ++}; ++ ++static const struct intel_renderstate_rodata * ++render_state_get_rodata(const struct intel_engine_cs *engine) ++{ ++ if (engine->id != RCS0) ++ return NULL; ++ ++ switch (INTEL_GEN(engine->i915)) { ++ case 6: ++ return &gen6_null_state; ++ case 7: ++ return &gen7_null_state; ++ case 8: ++ return &gen8_null_state; ++ case 9: ++ return &gen9_null_state; ++ } ++ ++ return NULL; ++} ++ ++/* ++ * Macro to add commands to auxiliary batch. ++ * This macro only checks for page overflow before inserting the commands, ++ * this is sufficient as the null state generator makes the final batch ++ * with two passes to build command and state separately. At this point ++ * the size of both are known and it compacts them by relocating the state ++ * right after the commands taking care of alignment so we should sufficient ++ * space below them for adding new commands. ++ */ ++#define OUT_BATCH(batch, i, val) \ ++ do { \ ++ if ((i) >= PAGE_SIZE / sizeof(u32)) \ ++ goto err; \ ++ (batch)[(i)++] = (val); \ ++ } while(0) ++ ++static int render_state_setup(struct intel_render_state *so, ++ struct drm_i915_private *i915) ++{ ++ const struct intel_renderstate_rodata *rodata = so->rodata; ++ unsigned int i = 0, reloc_index = 0; ++ unsigned int needs_clflush; ++ u32 *d; ++ int ret; ++ ++ ret = i915_gem_obj_prepare_shmem_write(so->obj, &needs_clflush); ++ if (ret) ++ return ret; ++ ++ d = kmap_atomic(i915_gem_object_get_dirty_page(so->obj, 0)); ++ ++ while (i < rodata->batch_items) { ++ u32 s = rodata->batch[i]; ++ ++ if (i * 4 == rodata->reloc[reloc_index]) { ++ u64 r = s + so->vma->node.start; ++ s = lower_32_bits(r); ++ if (HAS_64BIT_RELOC(i915)) { ++ if (i + 1 >= rodata->batch_items || ++ rodata->batch[i + 1] != 0) ++ goto err; ++ ++ d[i++] = s; ++ s = upper_32_bits(r); ++ } ++ ++ reloc_index++; ++ } ++ ++ d[i++] = s; ++ } ++ ++ if (rodata->reloc[reloc_index] != -1) { ++ DRM_ERROR("only %d relocs resolved\n", reloc_index); ++ goto err; ++ } ++ ++ so->batch_offset = i915_ggtt_offset(so->vma); ++ so->batch_size = rodata->batch_items * sizeof(u32); ++ ++ while (i % CACHELINE_DWORDS) ++ OUT_BATCH(d, i, MI_NOOP); ++ ++ so->aux_offset = i * sizeof(u32); ++ ++ if (HAS_POOLED_EU(i915)) { ++ /* ++ * We always program 3x6 pool config but depending upon which ++ * subslice is disabled HW drops down to appropriate config ++ * shown below. ++ * ++ * In the below table 2x6 config always refers to ++ * fused-down version, native 2x6 is not available and can ++ * be ignored ++ * ++ * SNo subslices config eu pool configuration ++ * ----------------------------------------------------------- ++ * 1 3 subslices enabled (3x6) - 0x00777000 (9+9) ++ * 2 ss0 disabled (2x6) - 0x00777000 (3+9) ++ * 3 ss1 disabled (2x6) - 0x00770000 (6+6) ++ * 4 ss2 disabled (2x6) - 0x00007000 (9+3) ++ */ ++ u32 eu_pool_config = 0x00777000; ++ ++ OUT_BATCH(d, i, GEN9_MEDIA_POOL_STATE); ++ OUT_BATCH(d, i, GEN9_MEDIA_POOL_ENABLE); ++ OUT_BATCH(d, i, eu_pool_config); ++ OUT_BATCH(d, i, 0); ++ OUT_BATCH(d, i, 0); ++ OUT_BATCH(d, i, 0); ++ } ++ ++ OUT_BATCH(d, i, MI_BATCH_BUFFER_END); ++ so->aux_size = i * sizeof(u32) - so->aux_offset; ++ so->aux_offset += so->batch_offset; ++ /* ++ * Since we are sending length, we need to strictly conform to ++ * all requirements. For Gen2 this must be a multiple of 8. ++ */ ++ so->aux_size = ALIGN(so->aux_size, 8); ++ ++ if (needs_clflush) ++ drm_clflush_virt_range(d, i * sizeof(u32)); ++ kunmap_atomic(d); ++ ++ ret = 0; ++out: ++ i915_gem_obj_finish_shmem_access(so->obj); ++ return ret; ++ ++err: ++ kunmap_atomic(d); ++ ret = -EINVAL; ++ goto out; ++} ++ ++#undef OUT_BATCH ++ ++int i915_gem_render_state_emit(struct i915_request *rq) ++{ ++ struct intel_engine_cs *engine = rq->engine; ++ struct intel_render_state so = {}; /* keep the compiler happy */ ++ int err; ++ ++ so.rodata = render_state_get_rodata(engine); ++ if (!so.rodata) ++ return 0; ++ ++ if (so.rodata->batch_items * 4 > PAGE_SIZE) ++ return -EINVAL; ++ ++ so.obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE); ++ if (IS_ERR(so.obj)) ++ return PTR_ERR(so.obj); ++ ++ so.vma = i915_vma_instance(so.obj, &engine->i915->ggtt.vm, NULL); ++ if (IS_ERR(so.vma)) { ++ err = PTR_ERR(so.vma); ++ goto err_obj; ++ } ++ ++ err = i915_vma_pin(so.vma, 0, 0, PIN_GLOBAL | PIN_HIGH); ++ if (err) ++ goto err_vma; ++ ++ err = render_state_setup(&so, rq->i915); ++ if (err) ++ goto err_unpin; ++ ++ err = engine->emit_bb_start(rq, ++ so.batch_offset, so.batch_size, ++ I915_DISPATCH_SECURE); ++ if (err) ++ goto err_unpin; ++ ++ if (so.aux_size > 8) { ++ err = engine->emit_bb_start(rq, ++ so.aux_offset, so.aux_size, ++ I915_DISPATCH_SECURE); ++ if (err) ++ goto err_unpin; ++ } ++ ++ err = i915_vma_move_to_active(so.vma, rq, 0); ++err_unpin: ++ i915_vma_unpin(so.vma); ++err_vma: ++ i915_vma_close(so.vma); ++err_obj: ++ __i915_gem_object_release_unless_active(so.obj); ++ return err; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_gem_render_state.h b/drivers/gpu/drm/i915_legacy/i915_gem_render_state.h +new file mode 100644 +index 000000000000..112cda8fa1a8 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gem_render_state.h +@@ -0,0 +1,31 @@ ++/* ++ * Copyright © 2014 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER ++ * DEALINGS IN THE SOFTWARE. ++ */ ++ ++#ifndef _I915_GEM_RENDER_STATE_H_ ++#define _I915_GEM_RENDER_STATE_H_ ++ ++struct i915_request; ++ ++int i915_gem_render_state_emit(struct i915_request *rq); ++ ++#endif /* _I915_GEM_RENDER_STATE_H_ */ +diff --git a/drivers/gpu/drm/i915_legacy/i915_gem_shrinker.c b/drivers/gpu/drm/i915_legacy/i915_gem_shrinker.c +new file mode 100644 +index 000000000000..6da795c7e62e +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gem_shrinker.c +@@ -0,0 +1,556 @@ ++/* ++ * Copyright © 2008-2015 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "i915_drv.h" ++#include "i915_trace.h" ++ ++static bool shrinker_lock(struct drm_i915_private *i915, ++ unsigned int flags, ++ bool *unlock) ++{ ++ struct mutex *m = &i915->drm.struct_mutex; ++ ++ switch (mutex_trylock_recursive(m)) { ++ case MUTEX_TRYLOCK_RECURSIVE: ++ *unlock = false; ++ return true; ++ ++ case MUTEX_TRYLOCK_FAILED: ++ *unlock = false; ++ if (flags & I915_SHRINK_ACTIVE && ++ mutex_lock_killable_nested(m, I915_MM_SHRINKER) == 0) ++ *unlock = true; ++ return *unlock; ++ ++ case MUTEX_TRYLOCK_SUCCESS: ++ *unlock = true; ++ return true; ++ } ++ ++ BUG(); ++} ++ ++static void shrinker_unlock(struct drm_i915_private *i915, bool unlock) ++{ ++ if (!unlock) ++ return; ++ ++ mutex_unlock(&i915->drm.struct_mutex); ++} ++ ++static bool swap_available(void) ++{ ++ return get_nr_swap_pages() > 0; ++} ++ ++static bool can_release_pages(struct drm_i915_gem_object *obj) ++{ ++ /* Consider only shrinkable ojects. */ ++ if (!i915_gem_object_is_shrinkable(obj)) ++ return false; ++ ++ /* Only report true if by unbinding the object and putting its pages ++ * we can actually make forward progress towards freeing physical ++ * pages. ++ * ++ * If the pages are pinned for any other reason than being bound ++ * to the GPU, simply unbinding from the GPU is not going to succeed ++ * in releasing our pin count on the pages themselves. ++ */ ++ if (atomic_read(&obj->mm.pages_pin_count) > obj->bind_count) ++ return false; ++ ++ /* If any vma are "permanently" pinned, it will prevent us from ++ * reclaiming the obj->mm.pages. We only allow scanout objects to claim ++ * a permanent pin, along with a few others like the context objects. ++ * To simplify the scan, and to avoid walking the list of vma under the ++ * object, we just check the count of its permanently pinned. ++ */ ++ if (READ_ONCE(obj->pin_global)) ++ return false; ++ ++ /* We can only return physical pages to the system if we can either ++ * discard the contents (because the user has marked them as being ++ * purgeable) or if we can move their contents out to swap. ++ */ ++ return swap_available() || obj->mm.madv == I915_MADV_DONTNEED; ++} ++ ++static bool unsafe_drop_pages(struct drm_i915_gem_object *obj) ++{ ++ if (i915_gem_object_unbind(obj) == 0) ++ __i915_gem_object_put_pages(obj, I915_MM_SHRINKER); ++ return !i915_gem_object_has_pages(obj); ++} ++ ++/** ++ * i915_gem_shrink - Shrink buffer object caches ++ * @i915: i915 device ++ * @target: amount of memory to make available, in pages ++ * @nr_scanned: optional output for number of pages scanned (incremental) ++ * @flags: control flags for selecting cache types ++ * ++ * This function is the main interface to the shrinker. It will try to release ++ * up to @target pages of main memory backing storage from buffer objects. ++ * Selection of the specific caches can be done with @flags. This is e.g. useful ++ * when purgeable objects should be removed from caches preferentially. ++ * ++ * Note that it's not guaranteed that released amount is actually available as ++ * free system memory - the pages might still be in-used to due to other reasons ++ * (like cpu mmaps) or the mm core has reused them before we could grab them. ++ * Therefore code that needs to explicitly shrink buffer objects caches (e.g. to ++ * avoid deadlocks in memory reclaim) must fall back to i915_gem_shrink_all(). ++ * ++ * Also note that any kind of pinning (both per-vma address space pins and ++ * backing storage pins at the buffer object level) result in the shrinker code ++ * having to skip the object. ++ * ++ * Returns: ++ * The number of pages of backing storage actually released. ++ */ ++unsigned long ++i915_gem_shrink(struct drm_i915_private *i915, ++ unsigned long target, ++ unsigned long *nr_scanned, ++ unsigned flags) ++{ ++ const struct { ++ struct list_head *list; ++ unsigned int bit; ++ } phases[] = { ++ { &i915->mm.unbound_list, I915_SHRINK_UNBOUND }, ++ { &i915->mm.bound_list, I915_SHRINK_BOUND }, ++ { NULL, 0 }, ++ }, *phase; ++ intel_wakeref_t wakeref = 0; ++ unsigned long count = 0; ++ unsigned long scanned = 0; ++ bool unlock; ++ ++ if (!shrinker_lock(i915, flags, &unlock)) ++ return 0; ++ ++ /* ++ * When shrinking the active list, also consider active contexts. ++ * Active contexts are pinned until they are retired, and so can ++ * not be simply unbound to retire and unpin their pages. To shrink ++ * the contexts, we must wait until the gpu is idle. ++ * ++ * We don't care about errors here; if we cannot wait upon the GPU, ++ * we will free as much as we can and hope to get a second chance. ++ */ ++ if (flags & I915_SHRINK_ACTIVE) ++ i915_gem_wait_for_idle(i915, ++ I915_WAIT_LOCKED, ++ MAX_SCHEDULE_TIMEOUT); ++ ++ trace_i915_gem_shrink(i915, target, flags); ++ i915_retire_requests(i915); ++ ++ /* ++ * Unbinding of objects will require HW access; Let us not wake the ++ * device just to recover a little memory. If absolutely necessary, ++ * we will force the wake during oom-notifier. ++ */ ++ if (flags & I915_SHRINK_BOUND) { ++ wakeref = intel_runtime_pm_get_if_in_use(i915); ++ if (!wakeref) ++ flags &= ~I915_SHRINK_BOUND; ++ } ++ ++ /* ++ * As we may completely rewrite the (un)bound list whilst unbinding ++ * (due to retiring requests) we have to strictly process only ++ * one element of the list at the time, and recheck the list ++ * on every iteration. ++ * ++ * In particular, we must hold a reference whilst removing the ++ * object as we may end up waiting for and/or retiring the objects. ++ * This might release the final reference (held by the active list) ++ * and result in the object being freed from under us. This is ++ * similar to the precautions the eviction code must take whilst ++ * removing objects. ++ * ++ * Also note that although these lists do not hold a reference to ++ * the object we can safely grab one here: The final object ++ * unreferencing and the bound_list are both protected by the ++ * dev->struct_mutex and so we won't ever be able to observe an ++ * object on the bound_list with a reference count equals 0. ++ */ ++ for (phase = phases; phase->list; phase++) { ++ struct list_head still_in_list; ++ struct drm_i915_gem_object *obj; ++ ++ if ((flags & phase->bit) == 0) ++ continue; ++ ++ INIT_LIST_HEAD(&still_in_list); ++ ++ /* ++ * We serialize our access to unreferenced objects through ++ * the use of the struct_mutex. While the objects are not ++ * yet freed (due to RCU then a workqueue) we still want ++ * to be able to shrink their pages, so they remain on ++ * the unbound/bound list until actually freed. ++ */ ++ spin_lock(&i915->mm.obj_lock); ++ while (count < target && ++ (obj = list_first_entry_or_null(phase->list, ++ typeof(*obj), ++ mm.link))) { ++ list_move_tail(&obj->mm.link, &still_in_list); ++ ++ if (flags & I915_SHRINK_PURGEABLE && ++ obj->mm.madv != I915_MADV_DONTNEED) ++ continue; ++ ++ if (flags & I915_SHRINK_VMAPS && ++ !is_vmalloc_addr(obj->mm.mapping)) ++ continue; ++ ++ if (!(flags & I915_SHRINK_ACTIVE) && ++ (i915_gem_object_is_active(obj) || ++ i915_gem_object_is_framebuffer(obj))) ++ continue; ++ ++ if (!can_release_pages(obj)) ++ continue; ++ ++ spin_unlock(&i915->mm.obj_lock); ++ ++ if (unsafe_drop_pages(obj)) { ++ /* May arrive from get_pages on another bo */ ++ mutex_lock_nested(&obj->mm.lock, ++ I915_MM_SHRINKER); ++ if (!i915_gem_object_has_pages(obj)) { ++ __i915_gem_object_invalidate(obj); ++ count += obj->base.size >> PAGE_SHIFT; ++ } ++ mutex_unlock(&obj->mm.lock); ++ } ++ scanned += obj->base.size >> PAGE_SHIFT; ++ ++ spin_lock(&i915->mm.obj_lock); ++ } ++ list_splice_tail(&still_in_list, phase->list); ++ spin_unlock(&i915->mm.obj_lock); ++ } ++ ++ if (flags & I915_SHRINK_BOUND) ++ intel_runtime_pm_put(i915, wakeref); ++ ++ i915_retire_requests(i915); ++ ++ shrinker_unlock(i915, unlock); ++ ++ if (nr_scanned) ++ *nr_scanned += scanned; ++ return count; ++} ++ ++/** ++ * i915_gem_shrink_all - Shrink buffer object caches completely ++ * @i915: i915 device ++ * ++ * This is a simple wraper around i915_gem_shrink() to aggressively shrink all ++ * caches completely. It also first waits for and retires all outstanding ++ * requests to also be able to release backing storage for active objects. ++ * ++ * This should only be used in code to intentionally quiescent the gpu or as a ++ * last-ditch effort when memory seems to have run out. ++ * ++ * Returns: ++ * The number of pages of backing storage actually released. ++ */ ++unsigned long i915_gem_shrink_all(struct drm_i915_private *i915) ++{ ++ intel_wakeref_t wakeref; ++ unsigned long freed = 0; ++ ++ with_intel_runtime_pm(i915, wakeref) { ++ freed = i915_gem_shrink(i915, -1UL, NULL, ++ I915_SHRINK_BOUND | ++ I915_SHRINK_UNBOUND | ++ I915_SHRINK_ACTIVE); ++ } ++ ++ return freed; ++} ++ ++static unsigned long ++i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) ++{ ++ struct drm_i915_private *i915 = ++ container_of(shrinker, struct drm_i915_private, mm.shrinker); ++ struct drm_i915_gem_object *obj; ++ unsigned long num_objects = 0; ++ unsigned long count = 0; ++ ++ spin_lock(&i915->mm.obj_lock); ++ list_for_each_entry(obj, &i915->mm.unbound_list, mm.link) ++ if (can_release_pages(obj)) { ++ count += obj->base.size >> PAGE_SHIFT; ++ num_objects++; ++ } ++ ++ list_for_each_entry(obj, &i915->mm.bound_list, mm.link) ++ if (!i915_gem_object_is_active(obj) && can_release_pages(obj)) { ++ count += obj->base.size >> PAGE_SHIFT; ++ num_objects++; ++ } ++ spin_unlock(&i915->mm.obj_lock); ++ ++ /* Update our preferred vmscan batch size for the next pass. ++ * Our rough guess for an effective batch size is roughly 2 ++ * available GEM objects worth of pages. That is we don't want ++ * the shrinker to fire, until it is worth the cost of freeing an ++ * entire GEM object. ++ */ ++ if (num_objects) { ++ unsigned long avg = 2 * count / num_objects; ++ ++ i915->mm.shrinker.batch = ++ max((i915->mm.shrinker.batch + avg) >> 1, ++ 128ul /* default SHRINK_BATCH */); ++ } ++ ++ return count; ++} ++ ++static unsigned long ++i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) ++{ ++ struct drm_i915_private *i915 = ++ container_of(shrinker, struct drm_i915_private, mm.shrinker); ++ unsigned long freed; ++ bool unlock; ++ ++ sc->nr_scanned = 0; ++ ++ if (!shrinker_lock(i915, 0, &unlock)) ++ return SHRINK_STOP; ++ ++ freed = i915_gem_shrink(i915, ++ sc->nr_to_scan, ++ &sc->nr_scanned, ++ I915_SHRINK_BOUND | ++ I915_SHRINK_UNBOUND | ++ I915_SHRINK_PURGEABLE); ++ if (sc->nr_scanned < sc->nr_to_scan) ++ freed += i915_gem_shrink(i915, ++ sc->nr_to_scan - sc->nr_scanned, ++ &sc->nr_scanned, ++ I915_SHRINK_BOUND | ++ I915_SHRINK_UNBOUND); ++ if (sc->nr_scanned < sc->nr_to_scan && current_is_kswapd()) { ++ intel_wakeref_t wakeref; ++ ++ with_intel_runtime_pm(i915, wakeref) { ++ freed += i915_gem_shrink(i915, ++ sc->nr_to_scan - sc->nr_scanned, ++ &sc->nr_scanned, ++ I915_SHRINK_ACTIVE | ++ I915_SHRINK_BOUND | ++ I915_SHRINK_UNBOUND); ++ } ++ } ++ ++ shrinker_unlock(i915, unlock); ++ ++ return sc->nr_scanned ? freed : SHRINK_STOP; ++} ++ ++static int ++i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr) ++{ ++ struct drm_i915_private *i915 = ++ container_of(nb, struct drm_i915_private, mm.oom_notifier); ++ struct drm_i915_gem_object *obj; ++ unsigned long unevictable, bound, unbound, freed_pages; ++ intel_wakeref_t wakeref; ++ ++ freed_pages = 0; ++ with_intel_runtime_pm(i915, wakeref) ++ freed_pages += i915_gem_shrink(i915, -1UL, NULL, ++ I915_SHRINK_BOUND | ++ I915_SHRINK_UNBOUND); ++ ++ /* Because we may be allocating inside our own driver, we cannot ++ * assert that there are no objects with pinned pages that are not ++ * being pointed to by hardware. ++ */ ++ unbound = bound = unevictable = 0; ++ spin_lock(&i915->mm.obj_lock); ++ list_for_each_entry(obj, &i915->mm.unbound_list, mm.link) { ++ if (!can_release_pages(obj)) ++ unevictable += obj->base.size >> PAGE_SHIFT; ++ else ++ unbound += obj->base.size >> PAGE_SHIFT; ++ } ++ list_for_each_entry(obj, &i915->mm.bound_list, mm.link) { ++ if (!can_release_pages(obj)) ++ unevictable += obj->base.size >> PAGE_SHIFT; ++ else ++ bound += obj->base.size >> PAGE_SHIFT; ++ } ++ spin_unlock(&i915->mm.obj_lock); ++ ++ if (freed_pages || unbound || bound) ++ pr_info("Purging GPU memory, %lu pages freed, " ++ "%lu pages still pinned.\n", ++ freed_pages, unevictable); ++ ++ *(unsigned long *)ptr += freed_pages; ++ return NOTIFY_DONE; ++} ++ ++static int ++i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr) ++{ ++ struct drm_i915_private *i915 = ++ container_of(nb, struct drm_i915_private, mm.vmap_notifier); ++ struct i915_vma *vma, *next; ++ unsigned long freed_pages = 0; ++ intel_wakeref_t wakeref; ++ bool unlock; ++ ++ if (!shrinker_lock(i915, 0, &unlock)) ++ return NOTIFY_DONE; ++ ++ /* Force everything onto the inactive lists */ ++ if (i915_gem_wait_for_idle(i915, ++ I915_WAIT_LOCKED, ++ MAX_SCHEDULE_TIMEOUT)) ++ goto out; ++ ++ with_intel_runtime_pm(i915, wakeref) ++ freed_pages += i915_gem_shrink(i915, -1UL, NULL, ++ I915_SHRINK_BOUND | ++ I915_SHRINK_UNBOUND | ++ I915_SHRINK_VMAPS); ++ ++ /* We also want to clear any cached iomaps as they wrap vmap */ ++ mutex_lock(&i915->ggtt.vm.mutex); ++ list_for_each_entry_safe(vma, next, ++ &i915->ggtt.vm.bound_list, vm_link) { ++ unsigned long count = vma->node.size >> PAGE_SHIFT; ++ ++ if (!vma->iomap || i915_vma_is_active(vma)) ++ continue; ++ ++ mutex_unlock(&i915->ggtt.vm.mutex); ++ if (i915_vma_unbind(vma) == 0) ++ freed_pages += count; ++ mutex_lock(&i915->ggtt.vm.mutex); ++ } ++ mutex_unlock(&i915->ggtt.vm.mutex); ++ ++out: ++ shrinker_unlock(i915, unlock); ++ ++ *(unsigned long *)ptr += freed_pages; ++ return NOTIFY_DONE; ++} ++ ++/** ++ * i915_gem_shrinker_register - Register the i915 shrinker ++ * @i915: i915 device ++ * ++ * This function registers and sets up the i915 shrinker and OOM handler. ++ */ ++void i915_gem_shrinker_register(struct drm_i915_private *i915) ++{ ++ i915->mm.shrinker.scan_objects = i915_gem_shrinker_scan; ++ i915->mm.shrinker.count_objects = i915_gem_shrinker_count; ++ i915->mm.shrinker.seeks = DEFAULT_SEEKS; ++ i915->mm.shrinker.batch = 4096; ++ WARN_ON(register_shrinker(&i915->mm.shrinker)); ++ ++ i915->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom; ++ WARN_ON(register_oom_notifier(&i915->mm.oom_notifier)); ++ ++ i915->mm.vmap_notifier.notifier_call = i915_gem_shrinker_vmap; ++ WARN_ON(register_vmap_purge_notifier(&i915->mm.vmap_notifier)); ++} ++ ++/** ++ * i915_gem_shrinker_unregister - Unregisters the i915 shrinker ++ * @i915: i915 device ++ * ++ * This function unregisters the i915 shrinker and OOM handler. ++ */ ++void i915_gem_shrinker_unregister(struct drm_i915_private *i915) ++{ ++ WARN_ON(unregister_vmap_purge_notifier(&i915->mm.vmap_notifier)); ++ WARN_ON(unregister_oom_notifier(&i915->mm.oom_notifier)); ++ unregister_shrinker(&i915->mm.shrinker); ++} ++ ++void i915_gem_shrinker_taints_mutex(struct drm_i915_private *i915, ++ struct mutex *mutex) ++{ ++ bool unlock = false; ++ ++ if (!IS_ENABLED(CONFIG_LOCKDEP)) ++ return; ++ ++ if (!lockdep_is_held_type(&i915->drm.struct_mutex, -1)) { ++ mutex_acquire(&i915->drm.struct_mutex.dep_map, ++ I915_MM_NORMAL, 0, _RET_IP_); ++ unlock = true; ++ } ++ ++ fs_reclaim_acquire(GFP_KERNEL); ++ ++ /* ++ * As we invariably rely on the struct_mutex within the shrinker, ++ * but have a complicated recursion dance, taint all the mutexes used ++ * within the shrinker with the struct_mutex. For completeness, we ++ * taint with all subclass of struct_mutex, even though we should ++ * only need tainting by I915_MM_NORMAL to catch possible ABBA ++ * deadlocks from using struct_mutex inside @mutex. ++ */ ++ mutex_acquire(&i915->drm.struct_mutex.dep_map, ++ I915_MM_SHRINKER, 0, _RET_IP_); ++ ++ mutex_acquire(&mutex->dep_map, 0, 0, _RET_IP_); ++ mutex_release(&mutex->dep_map, 0, _RET_IP_); ++ ++ mutex_release(&i915->drm.struct_mutex.dep_map, 0, _RET_IP_); ++ ++ fs_reclaim_release(GFP_KERNEL); ++ ++ if (unlock) ++ mutex_release(&i915->drm.struct_mutex.dep_map, 0, _RET_IP_); ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_gem_stolen.c b/drivers/gpu/drm/i915_legacy/i915_gem_stolen.c +new file mode 100644 +index 000000000000..0a8082cfc761 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gem_stolen.c +@@ -0,0 +1,721 @@ ++/* ++ * Copyright © 2008-2012 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ * Authors: ++ * Eric Anholt ++ * Chris Wilson ++ * ++ */ ++ ++#include ++#include "i915_drv.h" ++ ++/* ++ * The BIOS typically reserves some of the system's memory for the exclusive ++ * use of the integrated graphics. This memory is no longer available for ++ * use by the OS and so the user finds that his system has less memory ++ * available than he put in. We refer to this memory as stolen. ++ * ++ * The BIOS will allocate its framebuffer from the stolen memory. Our ++ * goal is try to reuse that object for our own fbcon which must always ++ * be available for panics. Anything else we can reuse the stolen memory ++ * for is a boon. ++ */ ++ ++int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *dev_priv, ++ struct drm_mm_node *node, u64 size, ++ unsigned alignment, u64 start, u64 end) ++{ ++ int ret; ++ ++ if (!drm_mm_initialized(&dev_priv->mm.stolen)) ++ return -ENODEV; ++ ++ /* WaSkipStolenMemoryFirstPage:bdw+ */ ++ if (INTEL_GEN(dev_priv) >= 8 && start < 4096) ++ start = 4096; ++ ++ mutex_lock(&dev_priv->mm.stolen_lock); ++ ret = drm_mm_insert_node_in_range(&dev_priv->mm.stolen, node, ++ size, alignment, 0, ++ start, end, DRM_MM_INSERT_BEST); ++ mutex_unlock(&dev_priv->mm.stolen_lock); ++ ++ return ret; ++} ++ ++int i915_gem_stolen_insert_node(struct drm_i915_private *dev_priv, ++ struct drm_mm_node *node, u64 size, ++ unsigned alignment) ++{ ++ return i915_gem_stolen_insert_node_in_range(dev_priv, node, size, ++ alignment, 0, U64_MAX); ++} ++ ++void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv, ++ struct drm_mm_node *node) ++{ ++ mutex_lock(&dev_priv->mm.stolen_lock); ++ drm_mm_remove_node(node); ++ mutex_unlock(&dev_priv->mm.stolen_lock); ++} ++ ++static int i915_adjust_stolen(struct drm_i915_private *dev_priv, ++ struct resource *dsm) ++{ ++ struct i915_ggtt *ggtt = &dev_priv->ggtt; ++ struct resource *r; ++ ++ if (dsm->start == 0 || dsm->end <= dsm->start) ++ return -EINVAL; ++ ++ /* ++ * TODO: We have yet too encounter the case where the GTT wasn't at the ++ * end of stolen. With that assumption we could simplify this. ++ */ ++ ++ /* Make sure we don't clobber the GTT if it's within stolen memory */ ++ if (INTEL_GEN(dev_priv) <= 4 && ++ !IS_G33(dev_priv) && !IS_PINEVIEW(dev_priv) && !IS_G4X(dev_priv)) { ++ struct resource stolen[2] = {*dsm, *dsm}; ++ struct resource ggtt_res; ++ resource_size_t ggtt_start; ++ ++ ggtt_start = I915_READ(PGTBL_CTL); ++ if (IS_GEN(dev_priv, 4)) ++ ggtt_start = (ggtt_start & PGTBL_ADDRESS_LO_MASK) | ++ (ggtt_start & PGTBL_ADDRESS_HI_MASK) << 28; ++ else ++ ggtt_start &= PGTBL_ADDRESS_LO_MASK; ++ ++ ggtt_res = ++ (struct resource) DEFINE_RES_MEM(ggtt_start, ++ ggtt_total_entries(ggtt) * 4); ++ ++ if (ggtt_res.start >= stolen[0].start && ggtt_res.start < stolen[0].end) ++ stolen[0].end = ggtt_res.start; ++ if (ggtt_res.end > stolen[1].start && ggtt_res.end <= stolen[1].end) ++ stolen[1].start = ggtt_res.end; ++ ++ /* Pick the larger of the two chunks */ ++ if (resource_size(&stolen[0]) > resource_size(&stolen[1])) ++ *dsm = stolen[0]; ++ else ++ *dsm = stolen[1]; ++ ++ if (stolen[0].start != stolen[1].start || ++ stolen[0].end != stolen[1].end) { ++ DRM_DEBUG_DRIVER("GTT within stolen memory at %pR\n", &ggtt_res); ++ DRM_DEBUG_DRIVER("Stolen memory adjusted to %pR\n", dsm); ++ } ++ } ++ ++ /* ++ * Verify that nothing else uses this physical address. Stolen ++ * memory should be reserved by the BIOS and hidden from the ++ * kernel. So if the region is already marked as busy, something ++ * is seriously wrong. ++ */ ++ r = devm_request_mem_region(dev_priv->drm.dev, dsm->start, ++ resource_size(dsm), ++ "Graphics Stolen Memory"); ++ if (r == NULL) { ++ /* ++ * One more attempt but this time requesting region from ++ * start + 1, as we have seen that this resolves the region ++ * conflict with the PCI Bus. ++ * This is a BIOS w/a: Some BIOS wrap stolen in the root ++ * PCI bus, but have an off-by-one error. Hence retry the ++ * reservation starting from 1 instead of 0. ++ * There's also BIOS with off-by-one on the other end. ++ */ ++ r = devm_request_mem_region(dev_priv->drm.dev, dsm->start + 1, ++ resource_size(dsm) - 2, ++ "Graphics Stolen Memory"); ++ /* ++ * GEN3 firmware likes to smash pci bridges into the stolen ++ * range. Apparently this works. ++ */ ++ if (r == NULL && !IS_GEN(dev_priv, 3)) { ++ DRM_ERROR("conflict detected with stolen region: %pR\n", ++ dsm); ++ ++ return -EBUSY; ++ } ++ } ++ ++ return 0; ++} ++ ++void i915_gem_cleanup_stolen(struct drm_i915_private *dev_priv) ++{ ++ if (!drm_mm_initialized(&dev_priv->mm.stolen)) ++ return; ++ ++ drm_mm_takedown(&dev_priv->mm.stolen); ++} ++ ++static void g4x_get_stolen_reserved(struct drm_i915_private *dev_priv, ++ resource_size_t *base, ++ resource_size_t *size) ++{ ++ u32 reg_val = I915_READ(IS_GM45(dev_priv) ? ++ CTG_STOLEN_RESERVED : ++ ELK_STOLEN_RESERVED); ++ resource_size_t stolen_top = dev_priv->dsm.end + 1; ++ ++ DRM_DEBUG_DRIVER("%s_STOLEN_RESERVED = %08x\n", ++ IS_GM45(dev_priv) ? "CTG" : "ELK", reg_val); ++ ++ if ((reg_val & G4X_STOLEN_RESERVED_ENABLE) == 0) ++ return; ++ ++ /* ++ * Whether ILK really reuses the ELK register for this is unclear. ++ * Let's see if we catch anyone with this supposedly enabled on ILK. ++ */ ++ WARN(IS_GEN(dev_priv, 5), "ILK stolen reserved found? 0x%08x\n", ++ reg_val); ++ ++ if (!(reg_val & G4X_STOLEN_RESERVED_ADDR2_MASK)) ++ return; ++ ++ *base = (reg_val & G4X_STOLEN_RESERVED_ADDR2_MASK) << 16; ++ WARN_ON((reg_val & G4X_STOLEN_RESERVED_ADDR1_MASK) < *base); ++ ++ *size = stolen_top - *base; ++} ++ ++static void gen6_get_stolen_reserved(struct drm_i915_private *dev_priv, ++ resource_size_t *base, ++ resource_size_t *size) ++{ ++ u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED); ++ ++ DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val); ++ ++ if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE)) ++ return; ++ ++ *base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK; ++ ++ switch (reg_val & GEN6_STOLEN_RESERVED_SIZE_MASK) { ++ case GEN6_STOLEN_RESERVED_1M: ++ *size = 1024 * 1024; ++ break; ++ case GEN6_STOLEN_RESERVED_512K: ++ *size = 512 * 1024; ++ break; ++ case GEN6_STOLEN_RESERVED_256K: ++ *size = 256 * 1024; ++ break; ++ case GEN6_STOLEN_RESERVED_128K: ++ *size = 128 * 1024; ++ break; ++ default: ++ *size = 1024 * 1024; ++ MISSING_CASE(reg_val & GEN6_STOLEN_RESERVED_SIZE_MASK); ++ } ++} ++ ++static void vlv_get_stolen_reserved(struct drm_i915_private *dev_priv, ++ resource_size_t *base, ++ resource_size_t *size) ++{ ++ u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED); ++ resource_size_t stolen_top = dev_priv->dsm.end + 1; ++ ++ DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val); ++ ++ if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE)) ++ return; ++ ++ switch (reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK) { ++ default: ++ MISSING_CASE(reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK); ++ /* fall through */ ++ case GEN7_STOLEN_RESERVED_1M: ++ *size = 1024 * 1024; ++ break; ++ } ++ ++ /* ++ * On vlv, the ADDR_MASK portion is left as 0 and HW deduces the ++ * reserved location as (top - size). ++ */ ++ *base = stolen_top - *size; ++} ++ ++static void gen7_get_stolen_reserved(struct drm_i915_private *dev_priv, ++ resource_size_t *base, ++ resource_size_t *size) ++{ ++ u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED); ++ ++ DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val); ++ ++ if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE)) ++ return; ++ ++ *base = reg_val & GEN7_STOLEN_RESERVED_ADDR_MASK; ++ ++ switch (reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK) { ++ case GEN7_STOLEN_RESERVED_1M: ++ *size = 1024 * 1024; ++ break; ++ case GEN7_STOLEN_RESERVED_256K: ++ *size = 256 * 1024; ++ break; ++ default: ++ *size = 1024 * 1024; ++ MISSING_CASE(reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK); ++ } ++} ++ ++static void chv_get_stolen_reserved(struct drm_i915_private *dev_priv, ++ resource_size_t *base, ++ resource_size_t *size) ++{ ++ u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED); ++ ++ DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val); ++ ++ if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE)) ++ return; ++ ++ *base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK; ++ ++ switch (reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK) { ++ case GEN8_STOLEN_RESERVED_1M: ++ *size = 1024 * 1024; ++ break; ++ case GEN8_STOLEN_RESERVED_2M: ++ *size = 2 * 1024 * 1024; ++ break; ++ case GEN8_STOLEN_RESERVED_4M: ++ *size = 4 * 1024 * 1024; ++ break; ++ case GEN8_STOLEN_RESERVED_8M: ++ *size = 8 * 1024 * 1024; ++ break; ++ default: ++ *size = 8 * 1024 * 1024; ++ MISSING_CASE(reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK); ++ } ++} ++ ++static void bdw_get_stolen_reserved(struct drm_i915_private *dev_priv, ++ resource_size_t *base, ++ resource_size_t *size) ++{ ++ u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED); ++ resource_size_t stolen_top = dev_priv->dsm.end + 1; ++ ++ DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val); ++ ++ if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE)) ++ return; ++ ++ if (!(reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK)) ++ return; ++ ++ *base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK; ++ *size = stolen_top - *base; ++} ++ ++static void icl_get_stolen_reserved(struct drm_i915_private *dev_priv, ++ resource_size_t *base, ++ resource_size_t *size) ++{ ++ u64 reg_val = I915_READ64(GEN6_STOLEN_RESERVED); ++ ++ DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = 0x%016llx\n", reg_val); ++ ++ *base = reg_val & GEN11_STOLEN_RESERVED_ADDR_MASK; ++ ++ switch (reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK) { ++ case GEN8_STOLEN_RESERVED_1M: ++ *size = 1024 * 1024; ++ break; ++ case GEN8_STOLEN_RESERVED_2M: ++ *size = 2 * 1024 * 1024; ++ break; ++ case GEN8_STOLEN_RESERVED_4M: ++ *size = 4 * 1024 * 1024; ++ break; ++ case GEN8_STOLEN_RESERVED_8M: ++ *size = 8 * 1024 * 1024; ++ break; ++ default: ++ *size = 8 * 1024 * 1024; ++ MISSING_CASE(reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK); ++ } ++} ++ ++int i915_gem_init_stolen(struct drm_i915_private *dev_priv) ++{ ++ resource_size_t reserved_base, stolen_top; ++ resource_size_t reserved_total, reserved_size; ++ ++ mutex_init(&dev_priv->mm.stolen_lock); ++ ++ if (intel_vgpu_active(dev_priv)) { ++ DRM_INFO("iGVT-g active, disabling use of stolen memory\n"); ++ return 0; ++ } ++ ++ if (intel_vtd_active() && INTEL_GEN(dev_priv) < 8) { ++ DRM_INFO("DMAR active, disabling use of stolen memory\n"); ++ return 0; ++ } ++ ++ if (resource_size(&intel_graphics_stolen_res) == 0) ++ return 0; ++ ++ dev_priv->dsm = intel_graphics_stolen_res; ++ ++ if (i915_adjust_stolen(dev_priv, &dev_priv->dsm)) ++ return 0; ++ ++ GEM_BUG_ON(dev_priv->dsm.start == 0); ++ GEM_BUG_ON(dev_priv->dsm.end <= dev_priv->dsm.start); ++ ++ stolen_top = dev_priv->dsm.end + 1; ++ reserved_base = stolen_top; ++ reserved_size = 0; ++ ++ switch (INTEL_GEN(dev_priv)) { ++ case 2: ++ case 3: ++ break; ++ case 4: ++ if (!IS_G4X(dev_priv)) ++ break; ++ /* fall through */ ++ case 5: ++ g4x_get_stolen_reserved(dev_priv, ++ &reserved_base, &reserved_size); ++ break; ++ case 6: ++ gen6_get_stolen_reserved(dev_priv, ++ &reserved_base, &reserved_size); ++ break; ++ case 7: ++ if (IS_VALLEYVIEW(dev_priv)) ++ vlv_get_stolen_reserved(dev_priv, ++ &reserved_base, &reserved_size); ++ else ++ gen7_get_stolen_reserved(dev_priv, ++ &reserved_base, &reserved_size); ++ break; ++ case 8: ++ case 9: ++ case 10: ++ if (IS_LP(dev_priv)) ++ chv_get_stolen_reserved(dev_priv, ++ &reserved_base, &reserved_size); ++ else ++ bdw_get_stolen_reserved(dev_priv, ++ &reserved_base, &reserved_size); ++ break; ++ case 11: ++ default: ++ icl_get_stolen_reserved(dev_priv, &reserved_base, ++ &reserved_size); ++ break; ++ } ++ ++ /* ++ * Our expectation is that the reserved space is at the top of the ++ * stolen region and *never* at the bottom. If we see !reserved_base, ++ * it likely means we failed to read the registers correctly. ++ */ ++ if (!reserved_base) { ++ DRM_ERROR("inconsistent reservation %pa + %pa; ignoring\n", ++ &reserved_base, &reserved_size); ++ reserved_base = stolen_top; ++ reserved_size = 0; ++ } ++ ++ dev_priv->dsm_reserved = ++ (struct resource) DEFINE_RES_MEM(reserved_base, reserved_size); ++ ++ if (!resource_contains(&dev_priv->dsm, &dev_priv->dsm_reserved)) { ++ DRM_ERROR("Stolen reserved area %pR outside stolen memory %pR\n", ++ &dev_priv->dsm_reserved, &dev_priv->dsm); ++ return 0; ++ } ++ ++ /* It is possible for the reserved area to end before the end of stolen ++ * memory, so just consider the start. */ ++ reserved_total = stolen_top - reserved_base; ++ ++ DRM_DEBUG_DRIVER("Memory reserved for graphics device: %lluK, usable: %lluK\n", ++ (u64)resource_size(&dev_priv->dsm) >> 10, ++ ((u64)resource_size(&dev_priv->dsm) - reserved_total) >> 10); ++ ++ dev_priv->stolen_usable_size = ++ resource_size(&dev_priv->dsm) - reserved_total; ++ ++ /* Basic memrange allocator for stolen space. */ ++ drm_mm_init(&dev_priv->mm.stolen, 0, dev_priv->stolen_usable_size); ++ ++ return 0; ++} ++ ++static struct sg_table * ++i915_pages_create_for_stolen(struct drm_device *dev, ++ resource_size_t offset, resource_size_t size) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct sg_table *st; ++ struct scatterlist *sg; ++ ++ GEM_BUG_ON(range_overflows(offset, size, resource_size(&dev_priv->dsm))); ++ ++ /* We hide that we have no struct page backing our stolen object ++ * by wrapping the contiguous physical allocation with a fake ++ * dma mapping in a single scatterlist. ++ */ ++ ++ st = kmalloc(sizeof(*st), GFP_KERNEL); ++ if (st == NULL) ++ return ERR_PTR(-ENOMEM); ++ ++ if (sg_alloc_table(st, 1, GFP_KERNEL)) { ++ kfree(st); ++ return ERR_PTR(-ENOMEM); ++ } ++ ++ sg = st->sgl; ++ sg->offset = 0; ++ sg->length = size; ++ ++ sg_dma_address(sg) = (dma_addr_t)dev_priv->dsm.start + offset; ++ sg_dma_len(sg) = size; ++ ++ return st; ++} ++ ++static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj) ++{ ++ struct sg_table *pages = ++ i915_pages_create_for_stolen(obj->base.dev, ++ obj->stolen->start, ++ obj->stolen->size); ++ if (IS_ERR(pages)) ++ return PTR_ERR(pages); ++ ++ __i915_gem_object_set_pages(obj, pages, obj->stolen->size); ++ ++ return 0; ++} ++ ++static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj, ++ struct sg_table *pages) ++{ ++ /* Should only be called from i915_gem_object_release_stolen() */ ++ sg_free_table(pages); ++ kfree(pages); ++} ++ ++static void ++i915_gem_object_release_stolen(struct drm_i915_gem_object *obj) ++{ ++ struct drm_i915_private *dev_priv = to_i915(obj->base.dev); ++ struct drm_mm_node *stolen = fetch_and_zero(&obj->stolen); ++ ++ GEM_BUG_ON(!stolen); ++ ++ __i915_gem_object_unpin_pages(obj); ++ ++ i915_gem_stolen_remove_node(dev_priv, stolen); ++ kfree(stolen); ++} ++ ++static const struct drm_i915_gem_object_ops i915_gem_object_stolen_ops = { ++ .get_pages = i915_gem_object_get_pages_stolen, ++ .put_pages = i915_gem_object_put_pages_stolen, ++ .release = i915_gem_object_release_stolen, ++}; ++ ++static struct drm_i915_gem_object * ++_i915_gem_object_create_stolen(struct drm_i915_private *dev_priv, ++ struct drm_mm_node *stolen) ++{ ++ struct drm_i915_gem_object *obj; ++ unsigned int cache_level; ++ ++ obj = i915_gem_object_alloc(); ++ if (obj == NULL) ++ return NULL; ++ ++ drm_gem_private_object_init(&dev_priv->drm, &obj->base, stolen->size); ++ i915_gem_object_init(obj, &i915_gem_object_stolen_ops); ++ ++ obj->stolen = stolen; ++ obj->read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT; ++ cache_level = HAS_LLC(dev_priv) ? I915_CACHE_LLC : I915_CACHE_NONE; ++ i915_gem_object_set_cache_coherency(obj, cache_level); ++ ++ if (i915_gem_object_pin_pages(obj)) ++ goto cleanup; ++ ++ return obj; ++ ++cleanup: ++ i915_gem_object_free(obj); ++ return NULL; ++} ++ ++struct drm_i915_gem_object * ++i915_gem_object_create_stolen(struct drm_i915_private *dev_priv, ++ resource_size_t size) ++{ ++ struct drm_i915_gem_object *obj; ++ struct drm_mm_node *stolen; ++ int ret; ++ ++ if (!drm_mm_initialized(&dev_priv->mm.stolen)) ++ return NULL; ++ ++ if (size == 0) ++ return NULL; ++ ++ stolen = kzalloc(sizeof(*stolen), GFP_KERNEL); ++ if (!stolen) ++ return NULL; ++ ++ ret = i915_gem_stolen_insert_node(dev_priv, stolen, size, 4096); ++ if (ret) { ++ kfree(stolen); ++ return NULL; ++ } ++ ++ obj = _i915_gem_object_create_stolen(dev_priv, stolen); ++ if (obj) ++ return obj; ++ ++ i915_gem_stolen_remove_node(dev_priv, stolen); ++ kfree(stolen); ++ return NULL; ++} ++ ++struct drm_i915_gem_object * ++i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv, ++ resource_size_t stolen_offset, ++ resource_size_t gtt_offset, ++ resource_size_t size) ++{ ++ struct i915_ggtt *ggtt = &dev_priv->ggtt; ++ struct drm_i915_gem_object *obj; ++ struct drm_mm_node *stolen; ++ struct i915_vma *vma; ++ int ret; ++ ++ if (!drm_mm_initialized(&dev_priv->mm.stolen)) ++ return NULL; ++ ++ lockdep_assert_held(&dev_priv->drm.struct_mutex); ++ ++ DRM_DEBUG_DRIVER("creating preallocated stolen object: stolen_offset=%pa, gtt_offset=%pa, size=%pa\n", ++ &stolen_offset, >t_offset, &size); ++ ++ /* KISS and expect everything to be page-aligned */ ++ if (WARN_ON(size == 0) || ++ WARN_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)) || ++ WARN_ON(!IS_ALIGNED(stolen_offset, I915_GTT_MIN_ALIGNMENT))) ++ return NULL; ++ ++ stolen = kzalloc(sizeof(*stolen), GFP_KERNEL); ++ if (!stolen) ++ return NULL; ++ ++ stolen->start = stolen_offset; ++ stolen->size = size; ++ mutex_lock(&dev_priv->mm.stolen_lock); ++ ret = drm_mm_reserve_node(&dev_priv->mm.stolen, stolen); ++ mutex_unlock(&dev_priv->mm.stolen_lock); ++ if (ret) { ++ DRM_DEBUG_DRIVER("failed to allocate stolen space\n"); ++ kfree(stolen); ++ return NULL; ++ } ++ ++ obj = _i915_gem_object_create_stolen(dev_priv, stolen); ++ if (obj == NULL) { ++ DRM_DEBUG_DRIVER("failed to allocate stolen object\n"); ++ i915_gem_stolen_remove_node(dev_priv, stolen); ++ kfree(stolen); ++ return NULL; ++ } ++ ++ /* Some objects just need physical mem from stolen space */ ++ if (gtt_offset == I915_GTT_OFFSET_NONE) ++ return obj; ++ ++ ret = i915_gem_object_pin_pages(obj); ++ if (ret) ++ goto err; ++ ++ vma = i915_vma_instance(obj, &ggtt->vm, NULL); ++ if (IS_ERR(vma)) { ++ ret = PTR_ERR(vma); ++ goto err_pages; ++ } ++ ++ /* To simplify the initialisation sequence between KMS and GTT, ++ * we allow construction of the stolen object prior to ++ * setting up the GTT space. The actual reservation will occur ++ * later. ++ */ ++ ret = i915_gem_gtt_reserve(&ggtt->vm, &vma->node, ++ size, gtt_offset, obj->cache_level, ++ 0); ++ if (ret) { ++ DRM_DEBUG_DRIVER("failed to allocate stolen GTT space\n"); ++ goto err_pages; ++ } ++ ++ GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); ++ ++ vma->pages = obj->mm.pages; ++ vma->flags |= I915_VMA_GLOBAL_BIND; ++ __i915_vma_set_map_and_fenceable(vma); ++ ++ mutex_lock(&ggtt->vm.mutex); ++ list_move_tail(&vma->vm_link, &ggtt->vm.bound_list); ++ mutex_unlock(&ggtt->vm.mutex); ++ ++ spin_lock(&dev_priv->mm.obj_lock); ++ list_move_tail(&obj->mm.link, &dev_priv->mm.bound_list); ++ obj->bind_count++; ++ spin_unlock(&dev_priv->mm.obj_lock); ++ ++ return obj; ++ ++err_pages: ++ i915_gem_object_unpin_pages(obj); ++err: ++ i915_gem_object_put(obj); ++ return NULL; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_gem_tiling.c b/drivers/gpu/drm/i915_legacy/i915_gem_tiling.c +new file mode 100644 +index 000000000000..a9b5329dae3b +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gem_tiling.c +@@ -0,0 +1,457 @@ ++/* ++ * Copyright © 2008 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ * Authors: ++ * Eric Anholt ++ * ++ */ ++ ++#include ++#include ++#include ++#include "i915_drv.h" ++ ++/** ++ * DOC: buffer object tiling ++ * ++ * i915_gem_set_tiling_ioctl() and i915_gem_get_tiling_ioctl() is the userspace ++ * interface to declare fence register requirements. ++ * ++ * In principle GEM doesn't care at all about the internal data layout of an ++ * object, and hence it also doesn't care about tiling or swizzling. There's two ++ * exceptions: ++ * ++ * - For X and Y tiling the hardware provides detilers for CPU access, so called ++ * fences. Since there's only a limited amount of them the kernel must manage ++ * these, and therefore userspace must tell the kernel the object tiling if it ++ * wants to use fences for detiling. ++ * - On gen3 and gen4 platforms have a swizzling pattern for tiled objects which ++ * depends upon the physical page frame number. When swapping such objects the ++ * page frame number might change and the kernel must be able to fix this up ++ * and hence now the tiling. Note that on a subset of platforms with ++ * asymmetric memory channel population the swizzling pattern changes in an ++ * unknown way, and for those the kernel simply forbids swapping completely. ++ * ++ * Since neither of this applies for new tiling layouts on modern platforms like ++ * W, Ys and Yf tiling GEM only allows object tiling to be set to X or Y tiled. ++ * Anything else can be handled in userspace entirely without the kernel's ++ * invovlement. ++ */ ++ ++/** ++ * i915_gem_fence_size - required global GTT size for a fence ++ * @i915: i915 device ++ * @size: object size ++ * @tiling: tiling mode ++ * @stride: tiling stride ++ * ++ * Return the required global GTT size for a fence (view of a tiled object), ++ * taking into account potential fence register mapping. ++ */ ++u32 i915_gem_fence_size(struct drm_i915_private *i915, ++ u32 size, unsigned int tiling, unsigned int stride) ++{ ++ u32 ggtt_size; ++ ++ GEM_BUG_ON(!size); ++ ++ if (tiling == I915_TILING_NONE) ++ return size; ++ ++ GEM_BUG_ON(!stride); ++ ++ if (INTEL_GEN(i915) >= 4) { ++ stride *= i915_gem_tile_height(tiling); ++ GEM_BUG_ON(!IS_ALIGNED(stride, I965_FENCE_PAGE)); ++ return roundup(size, stride); ++ } ++ ++ /* Previous chips need a power-of-two fence region when tiling */ ++ if (IS_GEN(i915, 3)) ++ ggtt_size = 1024*1024; ++ else ++ ggtt_size = 512*1024; ++ ++ while (ggtt_size < size) ++ ggtt_size <<= 1; ++ ++ return ggtt_size; ++} ++ ++/** ++ * i915_gem_fence_alignment - required global GTT alignment for a fence ++ * @i915: i915 device ++ * @size: object size ++ * @tiling: tiling mode ++ * @stride: tiling stride ++ * ++ * Return the required global GTT alignment for a fence (a view of a tiled ++ * object), taking into account potential fence register mapping. ++ */ ++u32 i915_gem_fence_alignment(struct drm_i915_private *i915, u32 size, ++ unsigned int tiling, unsigned int stride) ++{ ++ GEM_BUG_ON(!size); ++ ++ /* ++ * Minimum alignment is 4k (GTT page size), but might be greater ++ * if a fence register is needed for the object. ++ */ ++ if (tiling == I915_TILING_NONE) ++ return I915_GTT_MIN_ALIGNMENT; ++ ++ if (INTEL_GEN(i915) >= 4) ++ return I965_FENCE_PAGE; ++ ++ /* ++ * Previous chips need to be aligned to the size of the smallest ++ * fence register that can contain the object. ++ */ ++ return i915_gem_fence_size(i915, size, tiling, stride); ++} ++ ++/* Check pitch constriants for all chips & tiling formats */ ++static bool ++i915_tiling_ok(struct drm_i915_gem_object *obj, ++ unsigned int tiling, unsigned int stride) ++{ ++ struct drm_i915_private *i915 = to_i915(obj->base.dev); ++ unsigned int tile_width; ++ ++ /* Linear is always fine */ ++ if (tiling == I915_TILING_NONE) ++ return true; ++ ++ if (tiling > I915_TILING_LAST) ++ return false; ++ ++ /* check maximum stride & object size */ ++ /* i965+ stores the end address of the gtt mapping in the fence ++ * reg, so dont bother to check the size */ ++ if (INTEL_GEN(i915) >= 7) { ++ if (stride / 128 > GEN7_FENCE_MAX_PITCH_VAL) ++ return false; ++ } else if (INTEL_GEN(i915) >= 4) { ++ if (stride / 128 > I965_FENCE_MAX_PITCH_VAL) ++ return false; ++ } else { ++ if (stride > 8192) ++ return false; ++ ++ if (!is_power_of_2(stride)) ++ return false; ++ } ++ ++ if (IS_GEN(i915, 2) || ++ (tiling == I915_TILING_Y && HAS_128_BYTE_Y_TILING(i915))) ++ tile_width = 128; ++ else ++ tile_width = 512; ++ ++ if (!stride || !IS_ALIGNED(stride, tile_width)) ++ return false; ++ ++ return true; ++} ++ ++static bool i915_vma_fence_prepare(struct i915_vma *vma, ++ int tiling_mode, unsigned int stride) ++{ ++ struct drm_i915_private *i915 = vma->vm->i915; ++ u32 size, alignment; ++ ++ if (!i915_vma_is_map_and_fenceable(vma)) ++ return true; ++ ++ size = i915_gem_fence_size(i915, vma->size, tiling_mode, stride); ++ if (vma->node.size < size) ++ return false; ++ ++ alignment = i915_gem_fence_alignment(i915, vma->size, tiling_mode, stride); ++ if (!IS_ALIGNED(vma->node.start, alignment)) ++ return false; ++ ++ return true; ++} ++ ++/* Make the current GTT allocation valid for the change in tiling. */ ++static int ++i915_gem_object_fence_prepare(struct drm_i915_gem_object *obj, ++ int tiling_mode, unsigned int stride) ++{ ++ struct i915_vma *vma; ++ int ret; ++ ++ if (tiling_mode == I915_TILING_NONE) ++ return 0; ++ ++ for_each_ggtt_vma(vma, obj) { ++ if (i915_vma_fence_prepare(vma, tiling_mode, stride)) ++ continue; ++ ++ ret = i915_vma_unbind(vma); ++ if (ret) ++ return ret; ++ } ++ ++ return 0; ++} ++ ++int ++i915_gem_object_set_tiling(struct drm_i915_gem_object *obj, ++ unsigned int tiling, unsigned int stride) ++{ ++ struct drm_i915_private *i915 = to_i915(obj->base.dev); ++ struct i915_vma *vma; ++ int err; ++ ++ /* Make sure we don't cross-contaminate obj->tiling_and_stride */ ++ BUILD_BUG_ON(I915_TILING_LAST & STRIDE_MASK); ++ ++ GEM_BUG_ON(!i915_tiling_ok(obj, tiling, stride)); ++ GEM_BUG_ON(!stride ^ (tiling == I915_TILING_NONE)); ++ lockdep_assert_held(&i915->drm.struct_mutex); ++ ++ if ((tiling | stride) == obj->tiling_and_stride) ++ return 0; ++ ++ if (i915_gem_object_is_framebuffer(obj)) ++ return -EBUSY; ++ ++ /* We need to rebind the object if its current allocation ++ * no longer meets the alignment restrictions for its new ++ * tiling mode. Otherwise we can just leave it alone, but ++ * need to ensure that any fence register is updated before ++ * the next fenced (either through the GTT or by the BLT unit ++ * on older GPUs) access. ++ * ++ * After updating the tiling parameters, we then flag whether ++ * we need to update an associated fence register. Note this ++ * has to also include the unfenced register the GPU uses ++ * whilst executing a fenced command for an untiled object. ++ */ ++ ++ err = i915_gem_object_fence_prepare(obj, tiling, stride); ++ if (err) ++ return err; ++ ++ i915_gem_object_lock(obj); ++ if (i915_gem_object_is_framebuffer(obj)) { ++ i915_gem_object_unlock(obj); ++ return -EBUSY; ++ } ++ ++ /* If the memory has unknown (i.e. varying) swizzling, we pin the ++ * pages to prevent them being swapped out and causing corruption ++ * due to the change in swizzling. ++ */ ++ mutex_lock(&obj->mm.lock); ++ if (i915_gem_object_has_pages(obj) && ++ obj->mm.madv == I915_MADV_WILLNEED && ++ i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) { ++ if (tiling == I915_TILING_NONE) { ++ GEM_BUG_ON(!obj->mm.quirked); ++ __i915_gem_object_unpin_pages(obj); ++ obj->mm.quirked = false; ++ } ++ if (!i915_gem_object_is_tiled(obj)) { ++ GEM_BUG_ON(obj->mm.quirked); ++ __i915_gem_object_pin_pages(obj); ++ obj->mm.quirked = true; ++ } ++ } ++ mutex_unlock(&obj->mm.lock); ++ ++ for_each_ggtt_vma(vma, obj) { ++ vma->fence_size = ++ i915_gem_fence_size(i915, vma->size, tiling, stride); ++ vma->fence_alignment = ++ i915_gem_fence_alignment(i915, ++ vma->size, tiling, stride); ++ ++ if (vma->fence) ++ vma->fence->dirty = true; ++ } ++ ++ obj->tiling_and_stride = tiling | stride; ++ i915_gem_object_unlock(obj); ++ ++ /* Force the fence to be reacquired for GTT access */ ++ i915_gem_release_mmap(obj); ++ ++ /* Try to preallocate memory required to save swizzling on put-pages */ ++ if (i915_gem_object_needs_bit17_swizzle(obj)) { ++ if (!obj->bit_17) { ++ obj->bit_17 = bitmap_zalloc(obj->base.size >> PAGE_SHIFT, ++ GFP_KERNEL); ++ } ++ } else { ++ bitmap_free(obj->bit_17); ++ obj->bit_17 = NULL; ++ } ++ ++ return 0; ++} ++ ++/** ++ * i915_gem_set_tiling_ioctl - IOCTL handler to set tiling mode ++ * @dev: DRM device ++ * @data: data pointer for the ioctl ++ * @file: DRM file for the ioctl call ++ * ++ * Sets the tiling mode of an object, returning the required swizzling of ++ * bit 6 of addresses in the object. ++ * ++ * Called by the user via ioctl. ++ * ++ * Returns: ++ * Zero on success, negative errno on failure. ++ */ ++int ++i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_i915_gem_set_tiling *args = data; ++ struct drm_i915_gem_object *obj; ++ int err; ++ ++ obj = i915_gem_object_lookup(file, args->handle); ++ if (!obj) ++ return -ENOENT; ++ ++ /* ++ * The tiling mode of proxy objects is handled by its generator, and ++ * not allowed to be changed by userspace. ++ */ ++ if (i915_gem_object_is_proxy(obj)) { ++ err = -ENXIO; ++ goto err; ++ } ++ ++ if (!i915_tiling_ok(obj, args->tiling_mode, args->stride)) { ++ err = -EINVAL; ++ goto err; ++ } ++ ++ if (args->tiling_mode == I915_TILING_NONE) { ++ args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE; ++ args->stride = 0; ++ } else { ++ if (args->tiling_mode == I915_TILING_X) ++ args->swizzle_mode = to_i915(dev)->mm.bit_6_swizzle_x; ++ else ++ args->swizzle_mode = to_i915(dev)->mm.bit_6_swizzle_y; ++ ++ /* Hide bit 17 swizzling from the user. This prevents old Mesa ++ * from aborting the application on sw fallbacks to bit 17, ++ * and we use the pread/pwrite bit17 paths to swizzle for it. ++ * If there was a user that was relying on the swizzle ++ * information for drm_intel_bo_map()ed reads/writes this would ++ * break it, but we don't have any of those. ++ */ ++ if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17) ++ args->swizzle_mode = I915_BIT_6_SWIZZLE_9; ++ if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17) ++ args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10; ++ ++ /* If we can't handle the swizzling, make it untiled. */ ++ if (args->swizzle_mode == I915_BIT_6_SWIZZLE_UNKNOWN) { ++ args->tiling_mode = I915_TILING_NONE; ++ args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE; ++ args->stride = 0; ++ } ++ } ++ ++ err = mutex_lock_interruptible(&dev->struct_mutex); ++ if (err) ++ goto err; ++ ++ err = i915_gem_object_set_tiling(obj, args->tiling_mode, args->stride); ++ mutex_unlock(&dev->struct_mutex); ++ ++ /* We have to maintain this existing ABI... */ ++ args->stride = i915_gem_object_get_stride(obj); ++ args->tiling_mode = i915_gem_object_get_tiling(obj); ++ ++err: ++ i915_gem_object_put(obj); ++ return err; ++} ++ ++/** ++ * i915_gem_get_tiling_ioctl - IOCTL handler to get tiling mode ++ * @dev: DRM device ++ * @data: data pointer for the ioctl ++ * @file: DRM file for the ioctl call ++ * ++ * Returns the current tiling mode and required bit 6 swizzling for the object. ++ * ++ * Called by the user via ioctl. ++ * ++ * Returns: ++ * Zero on success, negative errno on failure. ++ */ ++int ++i915_gem_get_tiling_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_i915_gem_get_tiling *args = data; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct drm_i915_gem_object *obj; ++ int err = -ENOENT; ++ ++ rcu_read_lock(); ++ obj = i915_gem_object_lookup_rcu(file, args->handle); ++ if (obj) { ++ args->tiling_mode = ++ READ_ONCE(obj->tiling_and_stride) & TILING_MASK; ++ err = 0; ++ } ++ rcu_read_unlock(); ++ if (unlikely(err)) ++ return err; ++ ++ switch (args->tiling_mode) { ++ case I915_TILING_X: ++ args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x; ++ break; ++ case I915_TILING_Y: ++ args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y; ++ break; ++ default: ++ case I915_TILING_NONE: ++ args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE; ++ break; ++ } ++ ++ /* Hide bit 17 from the user -- see comment in i915_gem_set_tiling */ ++ if (dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) ++ args->phys_swizzle_mode = I915_BIT_6_SWIZZLE_UNKNOWN; ++ else ++ args->phys_swizzle_mode = args->swizzle_mode; ++ if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17) ++ args->swizzle_mode = I915_BIT_6_SWIZZLE_9; ++ if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17) ++ args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10; ++ ++ return 0; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_gem_userptr.c b/drivers/gpu/drm/i915_legacy/i915_gem_userptr.c +new file mode 100644 +index 000000000000..8079ea3af103 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gem_userptr.c +@@ -0,0 +1,847 @@ ++/* ++ * Copyright © 2012-2014 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#include ++#include "i915_drv.h" ++#include "i915_trace.h" ++#include "intel_drv.h" ++#include ++#include ++#include ++#include ++#include ++ ++struct i915_mm_struct { ++ struct mm_struct *mm; ++ struct drm_i915_private *i915; ++ struct i915_mmu_notifier *mn; ++ struct hlist_node node; ++ struct kref kref; ++ struct work_struct work; ++}; ++ ++#if defined(CONFIG_MMU_NOTIFIER) ++#include ++ ++struct i915_mmu_notifier { ++ spinlock_t lock; ++ struct hlist_node node; ++ struct mmu_notifier mn; ++ struct rb_root_cached objects; ++ struct i915_mm_struct *mm; ++}; ++ ++struct i915_mmu_object { ++ struct i915_mmu_notifier *mn; ++ struct drm_i915_gem_object *obj; ++ struct interval_tree_node it; ++}; ++ ++static void add_object(struct i915_mmu_object *mo) ++{ ++ GEM_BUG_ON(!RB_EMPTY_NODE(&mo->it.rb)); ++ interval_tree_insert(&mo->it, &mo->mn->objects); ++} ++ ++static void del_object(struct i915_mmu_object *mo) ++{ ++ if (RB_EMPTY_NODE(&mo->it.rb)) ++ return; ++ ++ interval_tree_remove(&mo->it, &mo->mn->objects); ++ RB_CLEAR_NODE(&mo->it.rb); ++} ++ ++static void ++__i915_gem_userptr_set_active(struct drm_i915_gem_object *obj, bool value) ++{ ++ struct i915_mmu_object *mo = obj->userptr.mmu_object; ++ ++ /* ++ * During mm_invalidate_range we need to cancel any userptr that ++ * overlaps the range being invalidated. Doing so requires the ++ * struct_mutex, and that risks recursion. In order to cause ++ * recursion, the user must alias the userptr address space with ++ * a GTT mmapping (possible with a MAP_FIXED) - then when we have ++ * to invalidate that mmaping, mm_invalidate_range is called with ++ * the userptr address *and* the struct_mutex held. To prevent that ++ * we set a flag under the i915_mmu_notifier spinlock to indicate ++ * whether this object is valid. ++ */ ++ if (!mo) ++ return; ++ ++ spin_lock(&mo->mn->lock); ++ if (value) ++ add_object(mo); ++ else ++ del_object(mo); ++ spin_unlock(&mo->mn->lock); ++} ++ ++static int ++userptr_mn_invalidate_range_start(struct mmu_notifier *_mn, ++ const struct mmu_notifier_range *range) ++{ ++ struct i915_mmu_notifier *mn = ++ container_of(_mn, struct i915_mmu_notifier, mn); ++ struct interval_tree_node *it; ++ struct mutex *unlock = NULL; ++ unsigned long end; ++ int ret = 0; ++ ++ if (RB_EMPTY_ROOT(&mn->objects.rb_root)) ++ return 0; ++ ++ /* interval ranges are inclusive, but invalidate range is exclusive */ ++ end = range->end - 1; ++ ++ spin_lock(&mn->lock); ++ it = interval_tree_iter_first(&mn->objects, range->start, end); ++ while (it) { ++ struct drm_i915_gem_object *obj; ++ ++ if (!mmu_notifier_range_blockable(range)) { ++ ret = -EAGAIN; ++ break; ++ } ++ ++ /* ++ * The mmu_object is released late when destroying the ++ * GEM object so it is entirely possible to gain a ++ * reference on an object in the process of being freed ++ * since our serialisation is via the spinlock and not ++ * the struct_mutex - and consequently use it after it ++ * is freed and then double free it. To prevent that ++ * use-after-free we only acquire a reference on the ++ * object if it is not in the process of being destroyed. ++ */ ++ obj = container_of(it, struct i915_mmu_object, it)->obj; ++ if (!kref_get_unless_zero(&obj->base.refcount)) { ++ it = interval_tree_iter_next(it, range->start, end); ++ continue; ++ } ++ spin_unlock(&mn->lock); ++ ++ if (!unlock) { ++ unlock = &mn->mm->i915->drm.struct_mutex; ++ ++ switch (mutex_trylock_recursive(unlock)) { ++ default: ++ case MUTEX_TRYLOCK_FAILED: ++ if (mutex_lock_killable_nested(unlock, I915_MM_SHRINKER)) { ++ i915_gem_object_put(obj); ++ return -EINTR; ++ } ++ /* fall through */ ++ case MUTEX_TRYLOCK_SUCCESS: ++ break; ++ ++ case MUTEX_TRYLOCK_RECURSIVE: ++ unlock = ERR_PTR(-EEXIST); ++ break; ++ } ++ } ++ ++ ret = i915_gem_object_unbind(obj); ++ if (ret == 0) ++ ret = __i915_gem_object_put_pages(obj, I915_MM_SHRINKER); ++ i915_gem_object_put(obj); ++ if (ret) ++ goto unlock; ++ ++ spin_lock(&mn->lock); ++ ++ /* ++ * As we do not (yet) protect the mmu from concurrent insertion ++ * over this range, there is no guarantee that this search will ++ * terminate given a pathologic workload. ++ */ ++ it = interval_tree_iter_first(&mn->objects, range->start, end); ++ } ++ spin_unlock(&mn->lock); ++ ++unlock: ++ if (!IS_ERR_OR_NULL(unlock)) ++ mutex_unlock(unlock); ++ ++ return ret; ++ ++} ++ ++static const struct mmu_notifier_ops i915_gem_userptr_notifier = { ++ .invalidate_range_start = userptr_mn_invalidate_range_start, ++}; ++ ++static struct i915_mmu_notifier * ++i915_mmu_notifier_create(struct i915_mm_struct *mm) ++{ ++ struct i915_mmu_notifier *mn; ++ ++ mn = kmalloc(sizeof(*mn), GFP_KERNEL); ++ if (mn == NULL) ++ return ERR_PTR(-ENOMEM); ++ ++ spin_lock_init(&mn->lock); ++ mn->mn.ops = &i915_gem_userptr_notifier; ++ mn->objects = RB_ROOT_CACHED; ++ mn->mm = mm; ++ ++ return mn; ++} ++ ++static void ++i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj) ++{ ++ struct i915_mmu_object *mo; ++ ++ mo = fetch_and_zero(&obj->userptr.mmu_object); ++ if (!mo) ++ return; ++ ++ spin_lock(&mo->mn->lock); ++ del_object(mo); ++ spin_unlock(&mo->mn->lock); ++ kfree(mo); ++} ++ ++static struct i915_mmu_notifier * ++i915_mmu_notifier_find(struct i915_mm_struct *mm) ++{ ++ struct i915_mmu_notifier *mn; ++ int err = 0; ++ ++ mn = mm->mn; ++ if (mn) ++ return mn; ++ ++ mn = i915_mmu_notifier_create(mm); ++ if (IS_ERR(mn)) ++ err = PTR_ERR(mn); ++ ++ down_write(&mm->mm->mmap_sem); ++ mutex_lock(&mm->i915->mm_lock); ++ if (mm->mn == NULL && !err) { ++ /* Protected by mmap_sem (write-lock) */ ++ err = __mmu_notifier_register(&mn->mn, mm->mm); ++ if (!err) { ++ /* Protected by mm_lock */ ++ mm->mn = fetch_and_zero(&mn); ++ } ++ } else if (mm->mn) { ++ /* ++ * Someone else raced and successfully installed the mmu ++ * notifier, we can cancel our own errors. ++ */ ++ err = 0; ++ } ++ mutex_unlock(&mm->i915->mm_lock); ++ up_write(&mm->mm->mmap_sem); ++ ++ if (mn && !IS_ERR(mn)) ++ kfree(mn); ++ ++ return err ? ERR_PTR(err) : mm->mn; ++} ++ ++static int ++i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj, ++ unsigned flags) ++{ ++ struct i915_mmu_notifier *mn; ++ struct i915_mmu_object *mo; ++ ++ if (flags & I915_USERPTR_UNSYNCHRONIZED) ++ return capable(CAP_SYS_ADMIN) ? 0 : -EPERM; ++ ++ if (WARN_ON(obj->userptr.mm == NULL)) ++ return -EINVAL; ++ ++ mn = i915_mmu_notifier_find(obj->userptr.mm); ++ if (IS_ERR(mn)) ++ return PTR_ERR(mn); ++ ++ mo = kzalloc(sizeof(*mo), GFP_KERNEL); ++ if (!mo) ++ return -ENOMEM; ++ ++ mo->mn = mn; ++ mo->obj = obj; ++ mo->it.start = obj->userptr.ptr; ++ mo->it.last = obj->userptr.ptr + obj->base.size - 1; ++ RB_CLEAR_NODE(&mo->it.rb); ++ ++ obj->userptr.mmu_object = mo; ++ return 0; ++} ++ ++static void ++i915_mmu_notifier_free(struct i915_mmu_notifier *mn, ++ struct mm_struct *mm) ++{ ++ if (mn == NULL) ++ return; ++ ++ mmu_notifier_unregister(&mn->mn, mm); ++ kfree(mn); ++} ++ ++#else ++ ++static void ++__i915_gem_userptr_set_active(struct drm_i915_gem_object *obj, bool value) ++{ ++} ++ ++static void ++i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj) ++{ ++} ++ ++static int ++i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj, ++ unsigned flags) ++{ ++ if ((flags & I915_USERPTR_UNSYNCHRONIZED) == 0) ++ return -ENODEV; ++ ++ if (!capable(CAP_SYS_ADMIN)) ++ return -EPERM; ++ ++ return 0; ++} ++ ++static void ++i915_mmu_notifier_free(struct i915_mmu_notifier *mn, ++ struct mm_struct *mm) ++{ ++} ++ ++#endif ++ ++static struct i915_mm_struct * ++__i915_mm_struct_find(struct drm_i915_private *dev_priv, struct mm_struct *real) ++{ ++ struct i915_mm_struct *mm; ++ ++ /* Protected by dev_priv->mm_lock */ ++ hash_for_each_possible(dev_priv->mm_structs, mm, node, (unsigned long)real) ++ if (mm->mm == real) ++ return mm; ++ ++ return NULL; ++} ++ ++static int ++i915_gem_userptr_init__mm_struct(struct drm_i915_gem_object *obj) ++{ ++ struct drm_i915_private *dev_priv = to_i915(obj->base.dev); ++ struct i915_mm_struct *mm; ++ int ret = 0; ++ ++ /* During release of the GEM object we hold the struct_mutex. This ++ * precludes us from calling mmput() at that time as that may be ++ * the last reference and so call exit_mmap(). exit_mmap() will ++ * attempt to reap the vma, and if we were holding a GTT mmap ++ * would then call drm_gem_vm_close() and attempt to reacquire ++ * the struct mutex. So in order to avoid that recursion, we have ++ * to defer releasing the mm reference until after we drop the ++ * struct_mutex, i.e. we need to schedule a worker to do the clean ++ * up. ++ */ ++ mutex_lock(&dev_priv->mm_lock); ++ mm = __i915_mm_struct_find(dev_priv, current->mm); ++ if (mm == NULL) { ++ mm = kmalloc(sizeof(*mm), GFP_KERNEL); ++ if (mm == NULL) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ++ kref_init(&mm->kref); ++ mm->i915 = to_i915(obj->base.dev); ++ ++ mm->mm = current->mm; ++ mmgrab(current->mm); ++ ++ mm->mn = NULL; ++ ++ /* Protected by dev_priv->mm_lock */ ++ hash_add(dev_priv->mm_structs, ++ &mm->node, (unsigned long)mm->mm); ++ } else ++ kref_get(&mm->kref); ++ ++ obj->userptr.mm = mm; ++out: ++ mutex_unlock(&dev_priv->mm_lock); ++ return ret; ++} ++ ++static void ++__i915_mm_struct_free__worker(struct work_struct *work) ++{ ++ struct i915_mm_struct *mm = container_of(work, typeof(*mm), work); ++ i915_mmu_notifier_free(mm->mn, mm->mm); ++ mmdrop(mm->mm); ++ kfree(mm); ++} ++ ++static void ++__i915_mm_struct_free(struct kref *kref) ++{ ++ struct i915_mm_struct *mm = container_of(kref, typeof(*mm), kref); ++ ++ /* Protected by dev_priv->mm_lock */ ++ hash_del(&mm->node); ++ mutex_unlock(&mm->i915->mm_lock); ++ ++ INIT_WORK(&mm->work, __i915_mm_struct_free__worker); ++ queue_work(mm->i915->mm.userptr_wq, &mm->work); ++} ++ ++static void ++i915_gem_userptr_release__mm_struct(struct drm_i915_gem_object *obj) ++{ ++ if (obj->userptr.mm == NULL) ++ return; ++ ++ kref_put_mutex(&obj->userptr.mm->kref, ++ __i915_mm_struct_free, ++ &to_i915(obj->base.dev)->mm_lock); ++ obj->userptr.mm = NULL; ++} ++ ++struct get_pages_work { ++ struct work_struct work; ++ struct drm_i915_gem_object *obj; ++ struct task_struct *task; ++}; ++ ++static struct sg_table * ++__i915_gem_userptr_alloc_pages(struct drm_i915_gem_object *obj, ++ struct page **pvec, int num_pages) ++{ ++ unsigned int max_segment = i915_sg_segment_size(); ++ struct sg_table *st; ++ unsigned int sg_page_sizes; ++ int ret; ++ ++ st = kmalloc(sizeof(*st), GFP_KERNEL); ++ if (!st) ++ return ERR_PTR(-ENOMEM); ++ ++alloc_table: ++ ret = __sg_alloc_table_from_pages(st, pvec, num_pages, ++ 0, num_pages << PAGE_SHIFT, ++ max_segment, ++ GFP_KERNEL); ++ if (ret) { ++ kfree(st); ++ return ERR_PTR(ret); ++ } ++ ++ ret = i915_gem_gtt_prepare_pages(obj, st); ++ if (ret) { ++ sg_free_table(st); ++ ++ if (max_segment > PAGE_SIZE) { ++ max_segment = PAGE_SIZE; ++ goto alloc_table; ++ } ++ ++ kfree(st); ++ return ERR_PTR(ret); ++ } ++ ++ sg_page_sizes = i915_sg_page_sizes(st->sgl); ++ ++ __i915_gem_object_set_pages(obj, st, sg_page_sizes); ++ ++ return st; ++} ++ ++static void ++__i915_gem_userptr_get_pages_worker(struct work_struct *_work) ++{ ++ struct get_pages_work *work = container_of(_work, typeof(*work), work); ++ struct drm_i915_gem_object *obj = work->obj; ++ const int npages = obj->base.size >> PAGE_SHIFT; ++ struct page **pvec; ++ int pinned, ret; ++ ++ ret = -ENOMEM; ++ pinned = 0; ++ ++ pvec = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL); ++ if (pvec != NULL) { ++ struct mm_struct *mm = obj->userptr.mm->mm; ++ unsigned int flags = 0; ++ ++ if (!i915_gem_object_is_readonly(obj)) ++ flags |= FOLL_WRITE; ++ ++ ret = -EFAULT; ++ if (mmget_not_zero(mm)) { ++ down_read(&mm->mmap_sem); ++ while (pinned < npages) { ++ ret = get_user_pages_remote ++ (work->task, mm, ++ obj->userptr.ptr + pinned * PAGE_SIZE, ++ npages - pinned, ++ flags, ++ pvec + pinned, NULL, NULL); ++ if (ret < 0) ++ break; ++ ++ pinned += ret; ++ } ++ up_read(&mm->mmap_sem); ++ mmput(mm); ++ } ++ } ++ ++ mutex_lock(&obj->mm.lock); ++ if (obj->userptr.work == &work->work) { ++ struct sg_table *pages = ERR_PTR(ret); ++ ++ if (pinned == npages) { ++ pages = __i915_gem_userptr_alloc_pages(obj, pvec, ++ npages); ++ if (!IS_ERR(pages)) { ++ pinned = 0; ++ pages = NULL; ++ } ++ } ++ ++ obj->userptr.work = ERR_CAST(pages); ++ if (IS_ERR(pages)) ++ __i915_gem_userptr_set_active(obj, false); ++ } ++ mutex_unlock(&obj->mm.lock); ++ ++ release_pages(pvec, pinned); ++ kvfree(pvec); ++ ++ i915_gem_object_put(obj); ++ put_task_struct(work->task); ++ kfree(work); ++} ++ ++static struct sg_table * ++__i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj) ++{ ++ struct get_pages_work *work; ++ ++ /* Spawn a worker so that we can acquire the ++ * user pages without holding our mutex. Access ++ * to the user pages requires mmap_sem, and we have ++ * a strict lock ordering of mmap_sem, struct_mutex - ++ * we already hold struct_mutex here and so cannot ++ * call gup without encountering a lock inversion. ++ * ++ * Userspace will keep on repeating the operation ++ * (thanks to EAGAIN) until either we hit the fast ++ * path or the worker completes. If the worker is ++ * cancelled or superseded, the task is still run ++ * but the results ignored. (This leads to ++ * complications that we may have a stray object ++ * refcount that we need to be wary of when ++ * checking for existing objects during creation.) ++ * If the worker encounters an error, it reports ++ * that error back to this function through ++ * obj->userptr.work = ERR_PTR. ++ */ ++ work = kmalloc(sizeof(*work), GFP_KERNEL); ++ if (work == NULL) ++ return ERR_PTR(-ENOMEM); ++ ++ obj->userptr.work = &work->work; ++ ++ work->obj = i915_gem_object_get(obj); ++ ++ work->task = current; ++ get_task_struct(work->task); ++ ++ INIT_WORK(&work->work, __i915_gem_userptr_get_pages_worker); ++ queue_work(to_i915(obj->base.dev)->mm.userptr_wq, &work->work); ++ ++ return ERR_PTR(-EAGAIN); ++} ++ ++static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) ++{ ++ const int num_pages = obj->base.size >> PAGE_SHIFT; ++ struct mm_struct *mm = obj->userptr.mm->mm; ++ struct page **pvec; ++ struct sg_table *pages; ++ bool active; ++ int pinned; ++ ++ /* If userspace should engineer that these pages are replaced in ++ * the vma between us binding this page into the GTT and completion ++ * of rendering... Their loss. If they change the mapping of their ++ * pages they need to create a new bo to point to the new vma. ++ * ++ * However, that still leaves open the possibility of the vma ++ * being copied upon fork. Which falls under the same userspace ++ * synchronisation issue as a regular bo, except that this time ++ * the process may not be expecting that a particular piece of ++ * memory is tied to the GPU. ++ * ++ * Fortunately, we can hook into the mmu_notifier in order to ++ * discard the page references prior to anything nasty happening ++ * to the vma (discard or cloning) which should prevent the more ++ * egregious cases from causing harm. ++ */ ++ ++ if (obj->userptr.work) { ++ /* active flag should still be held for the pending work */ ++ if (IS_ERR(obj->userptr.work)) ++ return PTR_ERR(obj->userptr.work); ++ else ++ return -EAGAIN; ++ } ++ ++ pvec = NULL; ++ pinned = 0; ++ ++ if (mm == current->mm) { ++ pvec = kvmalloc_array(num_pages, sizeof(struct page *), ++ GFP_KERNEL | ++ __GFP_NORETRY | ++ __GFP_NOWARN); ++ if (pvec) /* defer to worker if malloc fails */ ++ pinned = __get_user_pages_fast(obj->userptr.ptr, ++ num_pages, ++ !i915_gem_object_is_readonly(obj), ++ pvec); ++ } ++ ++ active = false; ++ if (pinned < 0) { ++ pages = ERR_PTR(pinned); ++ pinned = 0; ++ } else if (pinned < num_pages) { ++ pages = __i915_gem_userptr_get_pages_schedule(obj); ++ active = pages == ERR_PTR(-EAGAIN); ++ } else { ++ pages = __i915_gem_userptr_alloc_pages(obj, pvec, num_pages); ++ active = !IS_ERR(pages); ++ } ++ if (active) ++ __i915_gem_userptr_set_active(obj, true); ++ ++ if (IS_ERR(pages)) ++ release_pages(pvec, pinned); ++ kvfree(pvec); ++ ++ return PTR_ERR_OR_ZERO(pages); ++} ++ ++static void ++i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj, ++ struct sg_table *pages) ++{ ++ struct sgt_iter sgt_iter; ++ struct page *page; ++ ++ /* Cancel any inflight work and force them to restart their gup */ ++ obj->userptr.work = NULL; ++ __i915_gem_userptr_set_active(obj, false); ++ if (!pages) ++ return; ++ ++ __i915_gem_object_release_shmem(obj, pages, true); ++ i915_gem_gtt_finish_pages(obj, pages); ++ ++ for_each_sgt_page(page, sgt_iter, pages) { ++ if (obj->mm.dirty) ++ set_page_dirty(page); ++ ++ mark_page_accessed(page); ++ put_page(page); ++ } ++ obj->mm.dirty = false; ++ ++ sg_free_table(pages); ++ kfree(pages); ++} ++ ++static void ++i915_gem_userptr_release(struct drm_i915_gem_object *obj) ++{ ++ i915_gem_userptr_release__mmu_notifier(obj); ++ i915_gem_userptr_release__mm_struct(obj); ++} ++ ++static int ++i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj) ++{ ++ if (obj->userptr.mmu_object) ++ return 0; ++ ++ return i915_gem_userptr_init__mmu_notifier(obj, 0); ++} ++ ++static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = { ++ .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | ++ I915_GEM_OBJECT_IS_SHRINKABLE | ++ I915_GEM_OBJECT_ASYNC_CANCEL, ++ .get_pages = i915_gem_userptr_get_pages, ++ .put_pages = i915_gem_userptr_put_pages, ++ .dmabuf_export = i915_gem_userptr_dmabuf_export, ++ .release = i915_gem_userptr_release, ++}; ++ ++/* ++ * Creates a new mm object that wraps some normal memory from the process ++ * context - user memory. ++ * ++ * We impose several restrictions upon the memory being mapped ++ * into the GPU. ++ * 1. It must be page aligned (both start/end addresses, i.e ptr and size). ++ * 2. It must be normal system memory, not a pointer into another map of IO ++ * space (e.g. it must not be a GTT mmapping of another object). ++ * 3. We only allow a bo as large as we could in theory map into the GTT, ++ * that is we limit the size to the total size of the GTT. ++ * 4. The bo is marked as being snoopable. The backing pages are left ++ * accessible directly by the CPU, but reads and writes by the GPU may ++ * incur the cost of a snoop (unless you have an LLC architecture). ++ * ++ * Synchronisation between multiple users and the GPU is left to userspace ++ * through the normal set-domain-ioctl. The kernel will enforce that the ++ * GPU relinquishes the VMA before it is returned back to the system ++ * i.e. upon free(), munmap() or process termination. However, the userspace ++ * malloc() library may not immediately relinquish the VMA after free() and ++ * instead reuse it whilst the GPU is still reading and writing to the VMA. ++ * Caveat emptor. ++ * ++ * Also note, that the object created here is not currently a "first class" ++ * object, in that several ioctls are banned. These are the CPU access ++ * ioctls: mmap(), pwrite and pread. In practice, you are expected to use ++ * direct access via your pointer rather than use those ioctls. Another ++ * restriction is that we do not allow userptr surfaces to be pinned to the ++ * hardware and so we reject any attempt to create a framebuffer out of a ++ * userptr. ++ * ++ * If you think this is a good interface to use to pass GPU memory between ++ * drivers, please use dma-buf instead. In fact, wherever possible use ++ * dma-buf instead. ++ */ ++int ++i915_gem_userptr_ioctl(struct drm_device *dev, ++ void *data, ++ struct drm_file *file) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct drm_i915_gem_userptr *args = data; ++ struct drm_i915_gem_object *obj; ++ int ret; ++ u32 handle; ++ ++ if (!HAS_LLC(dev_priv) && !HAS_SNOOP(dev_priv)) { ++ /* We cannot support coherent userptr objects on hw without ++ * LLC and broken snooping. ++ */ ++ return -ENODEV; ++ } ++ ++ if (args->flags & ~(I915_USERPTR_READ_ONLY | ++ I915_USERPTR_UNSYNCHRONIZED)) ++ return -EINVAL; ++ ++ if (!args->user_size) ++ return -EINVAL; ++ ++ if (offset_in_page(args->user_ptr | args->user_size)) ++ return -EINVAL; ++ ++ if (!access_ok((char __user *)(unsigned long)args->user_ptr, args->user_size)) ++ return -EFAULT; ++ ++ if (args->flags & I915_USERPTR_READ_ONLY) { ++ struct i915_hw_ppgtt *ppgtt; ++ ++ /* ++ * On almost all of the older hw, we cannot tell the GPU that ++ * a page is readonly. ++ */ ++ ppgtt = dev_priv->kernel_context->ppgtt; ++ if (!ppgtt || !ppgtt->vm.has_read_only) ++ return -ENODEV; ++ } ++ ++ obj = i915_gem_object_alloc(); ++ if (obj == NULL) ++ return -ENOMEM; ++ ++ drm_gem_private_object_init(dev, &obj->base, args->user_size); ++ i915_gem_object_init(obj, &i915_gem_userptr_ops); ++ obj->read_domains = I915_GEM_DOMAIN_CPU; ++ obj->write_domain = I915_GEM_DOMAIN_CPU; ++ i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC); ++ ++ obj->userptr.ptr = args->user_ptr; ++ if (args->flags & I915_USERPTR_READ_ONLY) ++ i915_gem_object_set_readonly(obj); ++ ++ /* And keep a pointer to the current->mm for resolving the user pages ++ * at binding. This means that we need to hook into the mmu_notifier ++ * in order to detect if the mmu is destroyed. ++ */ ++ ret = i915_gem_userptr_init__mm_struct(obj); ++ if (ret == 0) ++ ret = i915_gem_userptr_init__mmu_notifier(obj, args->flags); ++ if (ret == 0) ++ ret = drm_gem_handle_create(file, &obj->base, &handle); ++ ++ /* drop reference from allocate - handle holds it now */ ++ i915_gem_object_put(obj); ++ if (ret) ++ return ret; ++ ++ args->handle = handle; ++ return 0; ++} ++ ++int i915_gem_init_userptr(struct drm_i915_private *dev_priv) ++{ ++ mutex_init(&dev_priv->mm_lock); ++ hash_init(dev_priv->mm_structs); ++ ++ dev_priv->mm.userptr_wq = ++ alloc_workqueue("i915-userptr-acquire", ++ WQ_HIGHPRI | WQ_UNBOUND, ++ 0); ++ if (!dev_priv->mm.userptr_wq) ++ return -ENOMEM; ++ ++ return 0; ++} ++ ++void i915_gem_cleanup_userptr(struct drm_i915_private *dev_priv) ++{ ++ destroy_workqueue(dev_priv->mm.userptr_wq); ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_gemfs.c b/drivers/gpu/drm/i915_legacy/i915_gemfs.c +new file mode 100644 +index 000000000000..888b7d3f04c3 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gemfs.c +@@ -0,0 +1,75 @@ ++/* ++ * Copyright © 2017 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#include ++#include ++#include ++ ++#include "i915_drv.h" ++#include "i915_gemfs.h" ++ ++int i915_gemfs_init(struct drm_i915_private *i915) ++{ ++ struct file_system_type *type; ++ struct vfsmount *gemfs; ++ ++ type = get_fs_type("tmpfs"); ++ if (!type) ++ return -ENODEV; ++ ++ gemfs = kern_mount(type); ++ if (IS_ERR(gemfs)) ++ return PTR_ERR(gemfs); ++ ++ /* ++ * Enable huge-pages for objects that are at least HPAGE_PMD_SIZE, most ++ * likely 2M. Note that within_size may overallocate huge-pages, if say ++ * we allocate an object of size 2M + 4K, we may get 2M + 2M, but under ++ * memory pressure shmem should split any huge-pages which can be ++ * shrunk. ++ */ ++ ++ if (has_transparent_hugepage()) { ++ struct super_block *sb = gemfs->mnt_sb; ++ /* FIXME: Disabled until we get W/A for read BW issue. */ ++ char options[] = "huge=never"; ++ int flags = 0; ++ int err; ++ ++ err = sb->s_op->remount_fs(sb, &flags, options); ++ if (err) { ++ kern_unmount(gemfs); ++ return err; ++ } ++ } ++ ++ i915->mm.gemfs = gemfs; ++ ++ return 0; ++} ++ ++void i915_gemfs_fini(struct drm_i915_private *i915) ++{ ++ kern_unmount(i915->mm.gemfs); ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_gemfs.h b/drivers/gpu/drm/i915_legacy/i915_gemfs.h +new file mode 100644 +index 000000000000..cca8bdc5b93e +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gemfs.h +@@ -0,0 +1,34 @@ ++/* ++ * Copyright © 2017 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#ifndef __I915_GEMFS_H__ ++#define __I915_GEMFS_H__ ++ ++struct drm_i915_private; ++ ++int i915_gemfs_init(struct drm_i915_private *i915); ++ ++void i915_gemfs_fini(struct drm_i915_private *i915); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_globals.c b/drivers/gpu/drm/i915_legacy/i915_globals.c +new file mode 100644 +index 000000000000..81e5c2ce336b +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_globals.c +@@ -0,0 +1,125 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2019 Intel Corporation ++ */ ++ ++#include ++#include ++ ++#include "i915_active.h" ++#include "i915_gem_context.h" ++#include "i915_gem_object.h" ++#include "i915_globals.h" ++#include "i915_request.h" ++#include "i915_scheduler.h" ++#include "i915_vma.h" ++ ++static LIST_HEAD(globals); ++ ++static atomic_t active; ++static atomic_t epoch; ++static struct park_work { ++ struct rcu_work work; ++ int epoch; ++} park; ++ ++static void i915_globals_shrink(void) ++{ ++ struct i915_global *global; ++ ++ /* ++ * kmem_cache_shrink() discards empty slabs and reorders partially ++ * filled slabs to prioritise allocating from the mostly full slabs, ++ * with the aim of reducing fragmentation. ++ */ ++ list_for_each_entry(global, &globals, link) ++ global->shrink(); ++} ++ ++static void __i915_globals_park(struct work_struct *work) ++{ ++ /* Confirm nothing woke up in the last grace period */ ++ if (park.epoch == atomic_read(&epoch)) ++ i915_globals_shrink(); ++} ++ ++void __init i915_global_register(struct i915_global *global) ++{ ++ GEM_BUG_ON(!global->shrink); ++ GEM_BUG_ON(!global->exit); ++ ++ list_add_tail(&global->link, &globals); ++} ++ ++static void __i915_globals_cleanup(void) ++{ ++ struct i915_global *global, *next; ++ ++ list_for_each_entry_safe_reverse(global, next, &globals, link) ++ global->exit(); ++} ++ ++static __initconst int (* const initfn[])(void) = { ++ i915_global_active_init, ++ i915_global_context_init, ++ i915_global_gem_context_init, ++ i915_global_objects_init, ++ i915_global_request_init, ++ i915_global_scheduler_init, ++ i915_global_vma_init, ++}; ++ ++int __init i915_globals_init(void) ++{ ++ int i; ++ ++ for (i = 0; i < ARRAY_SIZE(initfn); i++) { ++ int err; ++ ++ err = initfn[i](); ++ if (err) { ++ __i915_globals_cleanup(); ++ return err; ++ } ++ } ++ ++ INIT_RCU_WORK(&park.work, __i915_globals_park); ++ return 0; ++} ++ ++void i915_globals_park(void) ++{ ++ /* ++ * Defer shrinking the global slab caches (and other work) until ++ * after a RCU grace period has completed with no activity. This ++ * is to try and reduce the latency impact on the consumers caused ++ * by us shrinking the caches the same time as they are trying to ++ * allocate, with the assumption being that if we idle long enough ++ * for an RCU grace period to elapse since the last use, it is likely ++ * to be longer until we need the caches again. ++ */ ++ if (!atomic_dec_and_test(&active)) ++ return; ++ ++ park.epoch = atomic_inc_return(&epoch); ++ queue_rcu_work(system_wq, &park.work); ++} ++ ++void i915_globals_unpark(void) ++{ ++ atomic_inc(&epoch); ++ atomic_inc(&active); ++} ++ ++void __exit i915_globals_exit(void) ++{ ++ /* Flush any residual park_work */ ++ atomic_inc(&epoch); ++ flush_rcu_work(&park.work); ++ ++ __i915_globals_cleanup(); ++ ++ /* And ensure that our DESTROY_BY_RCU slabs are truly destroyed */ ++ rcu_barrier(); ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_globals.h b/drivers/gpu/drm/i915_legacy/i915_globals.h +new file mode 100644 +index 000000000000..04c1ce107fc0 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_globals.h +@@ -0,0 +1,35 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2019 Intel Corporation ++ */ ++ ++#ifndef _I915_GLOBALS_H_ ++#define _I915_GLOBALS_H_ ++ ++typedef void (*i915_global_func_t)(void); ++ ++struct i915_global { ++ struct list_head link; ++ ++ i915_global_func_t shrink; ++ i915_global_func_t exit; ++}; ++ ++void i915_global_register(struct i915_global *global); ++ ++int i915_globals_init(void); ++void i915_globals_park(void); ++void i915_globals_unpark(void); ++void i915_globals_exit(void); ++ ++/* constructors */ ++int i915_global_active_init(void); ++int i915_global_context_init(void); ++int i915_global_gem_context_init(void); ++int i915_global_objects_init(void); ++int i915_global_request_init(void); ++int i915_global_scheduler_init(void); ++int i915_global_vma_init(void); ++ ++#endif /* _I915_GLOBALS_H_ */ +diff --git a/drivers/gpu/drm/i915_legacy/i915_gpu_error.c b/drivers/gpu/drm/i915_legacy/i915_gpu_error.c +new file mode 100644 +index 000000000000..f51ff683dd2e +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gpu_error.c +@@ -0,0 +1,1876 @@ ++/* ++ * Copyright (c) 2008 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ * Authors: ++ * Eric Anholt ++ * Keith Packard ++ * Mika Kuoppala ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include "i915_gpu_error.h" ++#include "i915_drv.h" ++ ++static inline const struct intel_engine_cs * ++engine_lookup(const struct drm_i915_private *i915, unsigned int id) ++{ ++ if (id >= I915_NUM_ENGINES) ++ return NULL; ++ ++ return i915->engine[id]; ++} ++ ++static inline const char * ++__engine_name(const struct intel_engine_cs *engine) ++{ ++ return engine ? engine->name : ""; ++} ++ ++static const char * ++engine_name(const struct drm_i915_private *i915, unsigned int id) ++{ ++ return __engine_name(engine_lookup(i915, id)); ++} ++ ++static const char *tiling_flag(int tiling) ++{ ++ switch (tiling) { ++ default: ++ case I915_TILING_NONE: return ""; ++ case I915_TILING_X: return " X"; ++ case I915_TILING_Y: return " Y"; ++ } ++} ++ ++static const char *dirty_flag(int dirty) ++{ ++ return dirty ? " dirty" : ""; ++} ++ ++static const char *purgeable_flag(int purgeable) ++{ ++ return purgeable ? " purgeable" : ""; ++} ++ ++static void __sg_set_buf(struct scatterlist *sg, ++ void *addr, unsigned int len, loff_t it) ++{ ++ sg->page_link = (unsigned long)virt_to_page(addr); ++ sg->offset = offset_in_page(addr); ++ sg->length = len; ++ sg->dma_address = it; ++} ++ ++static bool __i915_error_grow(struct drm_i915_error_state_buf *e, size_t len) ++{ ++ if (!len) ++ return false; ++ ++ if (e->bytes + len + 1 <= e->size) ++ return true; ++ ++ if (e->bytes) { ++ __sg_set_buf(e->cur++, e->buf, e->bytes, e->iter); ++ e->iter += e->bytes; ++ e->buf = NULL; ++ e->bytes = 0; ++ } ++ ++ if (e->cur == e->end) { ++ struct scatterlist *sgl; ++ ++ sgl = (typeof(sgl))__get_free_page(GFP_KERNEL); ++ if (!sgl) { ++ e->err = -ENOMEM; ++ return false; ++ } ++ ++ if (e->cur) { ++ e->cur->offset = 0; ++ e->cur->length = 0; ++ e->cur->page_link = ++ (unsigned long)sgl | SG_CHAIN; ++ } else { ++ e->sgl = sgl; ++ } ++ ++ e->cur = sgl; ++ e->end = sgl + SG_MAX_SINGLE_ALLOC - 1; ++ } ++ ++ e->size = ALIGN(len + 1, SZ_64K); ++ e->buf = kmalloc(e->size, GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY); ++ if (!e->buf) { ++ e->size = PAGE_ALIGN(len + 1); ++ e->buf = kmalloc(e->size, GFP_KERNEL); ++ } ++ if (!e->buf) { ++ e->err = -ENOMEM; ++ return false; ++ } ++ ++ return true; ++} ++ ++__printf(2, 0) ++static void i915_error_vprintf(struct drm_i915_error_state_buf *e, ++ const char *fmt, va_list args) ++{ ++ va_list ap; ++ int len; ++ ++ if (e->err) ++ return; ++ ++ va_copy(ap, args); ++ len = vsnprintf(NULL, 0, fmt, ap); ++ va_end(ap); ++ if (len <= 0) { ++ e->err = len; ++ return; ++ } ++ ++ if (!__i915_error_grow(e, len)) ++ return; ++ ++ GEM_BUG_ON(e->bytes >= e->size); ++ len = vscnprintf(e->buf + e->bytes, e->size - e->bytes, fmt, args); ++ if (len < 0) { ++ e->err = len; ++ return; ++ } ++ e->bytes += len; ++} ++ ++static void i915_error_puts(struct drm_i915_error_state_buf *e, const char *str) ++{ ++ unsigned len; ++ ++ if (e->err || !str) ++ return; ++ ++ len = strlen(str); ++ if (!__i915_error_grow(e, len)) ++ return; ++ ++ GEM_BUG_ON(e->bytes + len > e->size); ++ memcpy(e->buf + e->bytes, str, len); ++ e->bytes += len; ++} ++ ++#define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__) ++#define err_puts(e, s) i915_error_puts(e, s) ++ ++static void __i915_printfn_error(struct drm_printer *p, struct va_format *vaf) ++{ ++ i915_error_vprintf(p->arg, vaf->fmt, *vaf->va); ++} ++ ++static inline struct drm_printer ++i915_error_printer(struct drm_i915_error_state_buf *e) ++{ ++ struct drm_printer p = { ++ .printfn = __i915_printfn_error, ++ .arg = e, ++ }; ++ return p; ++} ++ ++#ifdef CONFIG_DRM_I915_COMPRESS_ERROR ++ ++struct compress { ++ struct z_stream_s zstream; ++ void *tmp; ++}; ++ ++static bool compress_init(struct compress *c) ++{ ++ struct z_stream_s *zstream = memset(&c->zstream, 0, sizeof(c->zstream)); ++ ++ zstream->workspace = ++ kmalloc(zlib_deflate_workspacesize(MAX_WBITS, MAX_MEM_LEVEL), ++ GFP_ATOMIC | __GFP_NOWARN); ++ if (!zstream->workspace) ++ return false; ++ ++ if (zlib_deflateInit(zstream, Z_DEFAULT_COMPRESSION) != Z_OK) { ++ kfree(zstream->workspace); ++ return false; ++ } ++ ++ c->tmp = NULL; ++ if (i915_has_memcpy_from_wc()) ++ c->tmp = (void *)__get_free_page(GFP_ATOMIC | __GFP_NOWARN); ++ ++ return true; ++} ++ ++static void *compress_next_page(struct drm_i915_error_object *dst) ++{ ++ unsigned long page; ++ ++ if (dst->page_count >= dst->num_pages) ++ return ERR_PTR(-ENOSPC); ++ ++ page = __get_free_page(GFP_ATOMIC | __GFP_NOWARN); ++ if (!page) ++ return ERR_PTR(-ENOMEM); ++ ++ return dst->pages[dst->page_count++] = (void *)page; ++} ++ ++static int compress_page(struct compress *c, ++ void *src, ++ struct drm_i915_error_object *dst) ++{ ++ struct z_stream_s *zstream = &c->zstream; ++ ++ zstream->next_in = src; ++ if (c->tmp && i915_memcpy_from_wc(c->tmp, src, PAGE_SIZE)) ++ zstream->next_in = c->tmp; ++ zstream->avail_in = PAGE_SIZE; ++ ++ do { ++ if (zstream->avail_out == 0) { ++ zstream->next_out = compress_next_page(dst); ++ if (IS_ERR(zstream->next_out)) ++ return PTR_ERR(zstream->next_out); ++ ++ zstream->avail_out = PAGE_SIZE; ++ } ++ ++ if (zlib_deflate(zstream, Z_NO_FLUSH) != Z_OK) ++ return -EIO; ++ ++ touch_nmi_watchdog(); ++ } while (zstream->avail_in); ++ ++ /* Fallback to uncompressed if we increase size? */ ++ if (0 && zstream->total_out > zstream->total_in) ++ return -E2BIG; ++ ++ return 0; ++} ++ ++static int compress_flush(struct compress *c, ++ struct drm_i915_error_object *dst) ++{ ++ struct z_stream_s *zstream = &c->zstream; ++ ++ do { ++ switch (zlib_deflate(zstream, Z_FINISH)) { ++ case Z_OK: /* more space requested */ ++ zstream->next_out = compress_next_page(dst); ++ if (IS_ERR(zstream->next_out)) ++ return PTR_ERR(zstream->next_out); ++ ++ zstream->avail_out = PAGE_SIZE; ++ break; ++ ++ case Z_STREAM_END: ++ goto end; ++ ++ default: /* any error */ ++ return -EIO; ++ } ++ } while (1); ++ ++end: ++ memset(zstream->next_out, 0, zstream->avail_out); ++ dst->unused = zstream->avail_out; ++ return 0; ++} ++ ++static void compress_fini(struct compress *c, ++ struct drm_i915_error_object *dst) ++{ ++ struct z_stream_s *zstream = &c->zstream; ++ ++ zlib_deflateEnd(zstream); ++ kfree(zstream->workspace); ++ if (c->tmp) ++ free_page((unsigned long)c->tmp); ++} ++ ++static void err_compression_marker(struct drm_i915_error_state_buf *m) ++{ ++ err_puts(m, ":"); ++} ++ ++#else ++ ++struct compress { ++}; ++ ++static bool compress_init(struct compress *c) ++{ ++ return true; ++} ++ ++static int compress_page(struct compress *c, ++ void *src, ++ struct drm_i915_error_object *dst) ++{ ++ unsigned long page; ++ void *ptr; ++ ++ page = __get_free_page(GFP_ATOMIC | __GFP_NOWARN); ++ if (!page) ++ return -ENOMEM; ++ ++ ptr = (void *)page; ++ if (!i915_memcpy_from_wc(ptr, src, PAGE_SIZE)) ++ memcpy(ptr, src, PAGE_SIZE); ++ dst->pages[dst->page_count++] = ptr; ++ ++ return 0; ++} ++ ++static int compress_flush(struct compress *c, ++ struct drm_i915_error_object *dst) ++{ ++ return 0; ++} ++ ++static void compress_fini(struct compress *c, ++ struct drm_i915_error_object *dst) ++{ ++} ++ ++static void err_compression_marker(struct drm_i915_error_state_buf *m) ++{ ++ err_puts(m, "~"); ++} ++ ++#endif ++ ++static void print_error_buffers(struct drm_i915_error_state_buf *m, ++ const char *name, ++ struct drm_i915_error_buffer *err, ++ int count) ++{ ++ err_printf(m, "%s [%d]:\n", name, count); ++ ++ while (count--) { ++ err_printf(m, " %08x_%08x %8u %02x %02x", ++ upper_32_bits(err->gtt_offset), ++ lower_32_bits(err->gtt_offset), ++ err->size, ++ err->read_domains, ++ err->write_domain); ++ err_puts(m, tiling_flag(err->tiling)); ++ err_puts(m, dirty_flag(err->dirty)); ++ err_puts(m, purgeable_flag(err->purgeable)); ++ err_puts(m, err->userptr ? " userptr" : ""); ++ err_puts(m, i915_cache_level_str(m->i915, err->cache_level)); ++ ++ if (err->name) ++ err_printf(m, " (name: %d)", err->name); ++ if (err->fence_reg != I915_FENCE_REG_NONE) ++ err_printf(m, " (fence: %d)", err->fence_reg); ++ ++ err_puts(m, "\n"); ++ err++; ++ } ++} ++ ++static void error_print_instdone(struct drm_i915_error_state_buf *m, ++ const struct drm_i915_error_engine *ee) ++{ ++ int slice; ++ int subslice; ++ ++ err_printf(m, " INSTDONE: 0x%08x\n", ++ ee->instdone.instdone); ++ ++ if (ee->engine_id != RCS0 || INTEL_GEN(m->i915) <= 3) ++ return; ++ ++ err_printf(m, " SC_INSTDONE: 0x%08x\n", ++ ee->instdone.slice_common); ++ ++ if (INTEL_GEN(m->i915) <= 6) ++ return; ++ ++ for_each_instdone_slice_subslice(m->i915, slice, subslice) ++ err_printf(m, " SAMPLER_INSTDONE[%d][%d]: 0x%08x\n", ++ slice, subslice, ++ ee->instdone.sampler[slice][subslice]); ++ ++ for_each_instdone_slice_subslice(m->i915, slice, subslice) ++ err_printf(m, " ROW_INSTDONE[%d][%d]: 0x%08x\n", ++ slice, subslice, ++ ee->instdone.row[slice][subslice]); ++} ++ ++static void error_print_request(struct drm_i915_error_state_buf *m, ++ const char *prefix, ++ const struct drm_i915_error_request *erq, ++ const unsigned long epoch) ++{ ++ if (!erq->seqno) ++ return; ++ ++ err_printf(m, "%s pid %d, seqno %8x:%08x%s%s, prio %d, emitted %dms, start %08x, head %08x, tail %08x\n", ++ prefix, erq->pid, erq->context, erq->seqno, ++ test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, ++ &erq->flags) ? "!" : "", ++ test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, ++ &erq->flags) ? "+" : "", ++ erq->sched_attr.priority, ++ jiffies_to_msecs(erq->jiffies - epoch), ++ erq->start, erq->head, erq->tail); ++} ++ ++static void error_print_context(struct drm_i915_error_state_buf *m, ++ const char *header, ++ const struct drm_i915_error_context *ctx) ++{ ++ err_printf(m, "%s%s[%d] hw_id %d, prio %d, guilty %d active %d\n", ++ header, ctx->comm, ctx->pid, ctx->hw_id, ++ ctx->sched_attr.priority, ctx->guilty, ctx->active); ++} ++ ++static void error_print_engine(struct drm_i915_error_state_buf *m, ++ const struct drm_i915_error_engine *ee, ++ const unsigned long epoch) ++{ ++ int n; ++ ++ err_printf(m, "%s command stream:\n", ++ engine_name(m->i915, ee->engine_id)); ++ err_printf(m, " IDLE?: %s\n", yesno(ee->idle)); ++ err_printf(m, " START: 0x%08x\n", ee->start); ++ err_printf(m, " HEAD: 0x%08x [0x%08x]\n", ee->head, ee->rq_head); ++ err_printf(m, " TAIL: 0x%08x [0x%08x, 0x%08x]\n", ++ ee->tail, ee->rq_post, ee->rq_tail); ++ err_printf(m, " CTL: 0x%08x\n", ee->ctl); ++ err_printf(m, " MODE: 0x%08x\n", ee->mode); ++ err_printf(m, " HWS: 0x%08x\n", ee->hws); ++ err_printf(m, " ACTHD: 0x%08x %08x\n", ++ (u32)(ee->acthd>>32), (u32)ee->acthd); ++ err_printf(m, " IPEIR: 0x%08x\n", ee->ipeir); ++ err_printf(m, " IPEHR: 0x%08x\n", ee->ipehr); ++ ++ error_print_instdone(m, ee); ++ ++ if (ee->batchbuffer) { ++ u64 start = ee->batchbuffer->gtt_offset; ++ u64 end = start + ee->batchbuffer->gtt_size; ++ ++ err_printf(m, " batch: [0x%08x_%08x, 0x%08x_%08x]\n", ++ upper_32_bits(start), lower_32_bits(start), ++ upper_32_bits(end), lower_32_bits(end)); ++ } ++ if (INTEL_GEN(m->i915) >= 4) { ++ err_printf(m, " BBADDR: 0x%08x_%08x\n", ++ (u32)(ee->bbaddr>>32), (u32)ee->bbaddr); ++ err_printf(m, " BB_STATE: 0x%08x\n", ee->bbstate); ++ err_printf(m, " INSTPS: 0x%08x\n", ee->instps); ++ } ++ err_printf(m, " INSTPM: 0x%08x\n", ee->instpm); ++ err_printf(m, " FADDR: 0x%08x %08x\n", upper_32_bits(ee->faddr), ++ lower_32_bits(ee->faddr)); ++ if (INTEL_GEN(m->i915) >= 6) { ++ err_printf(m, " RC PSMI: 0x%08x\n", ee->rc_psmi); ++ err_printf(m, " FAULT_REG: 0x%08x\n", ee->fault_reg); ++ } ++ if (HAS_PPGTT(m->i915)) { ++ err_printf(m, " GFX_MODE: 0x%08x\n", ee->vm_info.gfx_mode); ++ ++ if (INTEL_GEN(m->i915) >= 8) { ++ int i; ++ for (i = 0; i < 4; i++) ++ err_printf(m, " PDP%d: 0x%016llx\n", ++ i, ee->vm_info.pdp[i]); ++ } else { ++ err_printf(m, " PP_DIR_BASE: 0x%08x\n", ++ ee->vm_info.pp_dir_base); ++ } ++ } ++ err_printf(m, " ring->head: 0x%08x\n", ee->cpu_ring_head); ++ err_printf(m, " ring->tail: 0x%08x\n", ee->cpu_ring_tail); ++ err_printf(m, " hangcheck timestamp: %dms (%lu%s)\n", ++ jiffies_to_msecs(ee->hangcheck_timestamp - epoch), ++ ee->hangcheck_timestamp, ++ ee->hangcheck_timestamp == epoch ? "; epoch" : ""); ++ err_printf(m, " engine reset count: %u\n", ee->reset_count); ++ ++ for (n = 0; n < ee->num_ports; n++) { ++ err_printf(m, " ELSP[%d]:", n); ++ error_print_request(m, " ", &ee->execlist[n], epoch); ++ } ++ ++ error_print_context(m, " Active context: ", &ee->context); ++} ++ ++void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...) ++{ ++ va_list args; ++ ++ va_start(args, f); ++ i915_error_vprintf(e, f, args); ++ va_end(args); ++} ++ ++static void print_error_obj(struct drm_i915_error_state_buf *m, ++ struct intel_engine_cs *engine, ++ const char *name, ++ struct drm_i915_error_object *obj) ++{ ++ char out[ASCII85_BUFSZ]; ++ int page; ++ ++ if (!obj) ++ return; ++ ++ if (name) { ++ err_printf(m, "%s --- %s = 0x%08x %08x\n", ++ engine ? engine->name : "global", name, ++ upper_32_bits(obj->gtt_offset), ++ lower_32_bits(obj->gtt_offset)); ++ } ++ ++ err_compression_marker(m); ++ for (page = 0; page < obj->page_count; page++) { ++ int i, len; ++ ++ len = PAGE_SIZE; ++ if (page == obj->page_count - 1) ++ len -= obj->unused; ++ len = ascii85_encode_len(len); ++ ++ for (i = 0; i < len; i++) ++ err_puts(m, ascii85_encode(obj->pages[page][i], out)); ++ } ++ err_puts(m, "\n"); ++} ++ ++static void err_print_capabilities(struct drm_i915_error_state_buf *m, ++ const struct intel_device_info *info, ++ const struct intel_runtime_info *runtime, ++ const struct intel_driver_caps *caps) ++{ ++ struct drm_printer p = i915_error_printer(m); ++ ++ intel_device_info_dump_flags(info, &p); ++ intel_driver_caps_print(caps, &p); ++ intel_device_info_dump_topology(&runtime->sseu, &p); ++} ++ ++static void err_print_params(struct drm_i915_error_state_buf *m, ++ const struct i915_params *params) ++{ ++ struct drm_printer p = i915_error_printer(m); ++ ++ i915_params_dump(params, &p); ++} ++ ++static void err_print_pciid(struct drm_i915_error_state_buf *m, ++ struct drm_i915_private *i915) ++{ ++ struct pci_dev *pdev = i915->drm.pdev; ++ ++ err_printf(m, "PCI ID: 0x%04x\n", pdev->device); ++ err_printf(m, "PCI Revision: 0x%02x\n", pdev->revision); ++ err_printf(m, "PCI Subsystem: %04x:%04x\n", ++ pdev->subsystem_vendor, ++ pdev->subsystem_device); ++} ++ ++static void err_print_uc(struct drm_i915_error_state_buf *m, ++ const struct i915_error_uc *error_uc) ++{ ++ struct drm_printer p = i915_error_printer(m); ++ const struct i915_gpu_state *error = ++ container_of(error_uc, typeof(*error), uc); ++ ++ if (!error->device_info.has_guc) ++ return; ++ ++ intel_uc_fw_dump(&error_uc->guc_fw, &p); ++ intel_uc_fw_dump(&error_uc->huc_fw, &p); ++ print_error_obj(m, NULL, "GuC log buffer", error_uc->guc_log); ++} ++ ++static void err_free_sgl(struct scatterlist *sgl) ++{ ++ while (sgl) { ++ struct scatterlist *sg; ++ ++ for (sg = sgl; !sg_is_chain(sg); sg++) { ++ kfree(sg_virt(sg)); ++ if (sg_is_last(sg)) ++ break; ++ } ++ ++ sg = sg_is_last(sg) ? NULL : sg_chain_ptr(sg); ++ free_page((unsigned long)sgl); ++ sgl = sg; ++ } ++} ++ ++static void __err_print_to_sgl(struct drm_i915_error_state_buf *m, ++ struct i915_gpu_state *error) ++{ ++ struct drm_i915_error_object *obj; ++ struct timespec64 ts; ++ int i, j; ++ ++ if (*error->error_msg) ++ err_printf(m, "%s\n", error->error_msg); ++ err_printf(m, "Kernel: %s %s\n", ++ init_utsname()->release, ++ init_utsname()->machine); ++ ts = ktime_to_timespec64(error->time); ++ err_printf(m, "Time: %lld s %ld us\n", ++ (s64)ts.tv_sec, ts.tv_nsec / NSEC_PER_USEC); ++ ts = ktime_to_timespec64(error->boottime); ++ err_printf(m, "Boottime: %lld s %ld us\n", ++ (s64)ts.tv_sec, ts.tv_nsec / NSEC_PER_USEC); ++ ts = ktime_to_timespec64(error->uptime); ++ err_printf(m, "Uptime: %lld s %ld us\n", ++ (s64)ts.tv_sec, ts.tv_nsec / NSEC_PER_USEC); ++ err_printf(m, "Epoch: %lu jiffies (%u HZ)\n", error->epoch, HZ); ++ err_printf(m, "Capture: %lu jiffies; %d ms ago, %d ms after epoch\n", ++ error->capture, ++ jiffies_to_msecs(jiffies - error->capture), ++ jiffies_to_msecs(error->capture - error->epoch)); ++ ++ for (i = 0; i < ARRAY_SIZE(error->engine); i++) { ++ if (!error->engine[i].context.pid) ++ continue; ++ ++ err_printf(m, "Active process (on ring %s): %s [%d]\n", ++ engine_name(m->i915, i), ++ error->engine[i].context.comm, ++ error->engine[i].context.pid); ++ } ++ err_printf(m, "Reset count: %u\n", error->reset_count); ++ err_printf(m, "Suspend count: %u\n", error->suspend_count); ++ err_printf(m, "Platform: %s\n", intel_platform_name(error->device_info.platform)); ++ err_printf(m, "Subplatform: 0x%x\n", ++ intel_subplatform(&error->runtime_info, ++ error->device_info.platform)); ++ err_print_pciid(m, m->i915); ++ ++ err_printf(m, "IOMMU enabled?: %d\n", error->iommu); ++ ++ if (HAS_CSR(m->i915)) { ++ struct intel_csr *csr = &m->i915->csr; ++ ++ err_printf(m, "DMC loaded: %s\n", ++ yesno(csr->dmc_payload != NULL)); ++ err_printf(m, "DMC fw version: %d.%d\n", ++ CSR_VERSION_MAJOR(csr->version), ++ CSR_VERSION_MINOR(csr->version)); ++ } ++ ++ err_printf(m, "GT awake: %s\n", yesno(error->awake)); ++ err_printf(m, "RPM wakelock: %s\n", yesno(error->wakelock)); ++ err_printf(m, "PM suspended: %s\n", yesno(error->suspended)); ++ err_printf(m, "EIR: 0x%08x\n", error->eir); ++ err_printf(m, "IER: 0x%08x\n", error->ier); ++ for (i = 0; i < error->ngtier; i++) ++ err_printf(m, "GTIER[%d]: 0x%08x\n", i, error->gtier[i]); ++ err_printf(m, "PGTBL_ER: 0x%08x\n", error->pgtbl_er); ++ err_printf(m, "FORCEWAKE: 0x%08x\n", error->forcewake); ++ err_printf(m, "DERRMR: 0x%08x\n", error->derrmr); ++ err_printf(m, "CCID: 0x%08x\n", error->ccid); ++ ++ for (i = 0; i < error->nfence; i++) ++ err_printf(m, " fence[%d] = %08llx\n", i, error->fence[i]); ++ ++ if (INTEL_GEN(m->i915) >= 6) { ++ err_printf(m, "ERROR: 0x%08x\n", error->error); ++ ++ if (INTEL_GEN(m->i915) >= 8) ++ err_printf(m, "FAULT_TLB_DATA: 0x%08x 0x%08x\n", ++ error->fault_data1, error->fault_data0); ++ ++ err_printf(m, "DONE_REG: 0x%08x\n", error->done_reg); ++ } ++ ++ if (IS_GEN(m->i915, 7)) ++ err_printf(m, "ERR_INT: 0x%08x\n", error->err_int); ++ ++ for (i = 0; i < ARRAY_SIZE(error->engine); i++) { ++ if (error->engine[i].engine_id != -1) ++ error_print_engine(m, &error->engine[i], error->epoch); ++ } ++ ++ for (i = 0; i < ARRAY_SIZE(error->active_vm); i++) { ++ char buf[128]; ++ int len, first = 1; ++ ++ if (!error->active_vm[i]) ++ break; ++ ++ len = scnprintf(buf, sizeof(buf), "Active ("); ++ for (j = 0; j < ARRAY_SIZE(error->engine); j++) { ++ if (error->engine[j].vm != error->active_vm[i]) ++ continue; ++ ++ len += scnprintf(buf + len, sizeof(buf), "%s%s", ++ first ? "" : ", ", ++ m->i915->engine[j]->name); ++ first = 0; ++ } ++ scnprintf(buf + len, sizeof(buf), ")"); ++ print_error_buffers(m, buf, ++ error->active_bo[i], ++ error->active_bo_count[i]); ++ } ++ ++ print_error_buffers(m, "Pinned (global)", ++ error->pinned_bo, ++ error->pinned_bo_count); ++ ++ for (i = 0; i < ARRAY_SIZE(error->engine); i++) { ++ const struct drm_i915_error_engine *ee = &error->engine[i]; ++ ++ obj = ee->batchbuffer; ++ if (obj) { ++ err_puts(m, m->i915->engine[i]->name); ++ if (ee->context.pid) ++ err_printf(m, " (submitted by %s [%d])", ++ ee->context.comm, ++ ee->context.pid); ++ err_printf(m, " --- gtt_offset = 0x%08x %08x\n", ++ upper_32_bits(obj->gtt_offset), ++ lower_32_bits(obj->gtt_offset)); ++ print_error_obj(m, m->i915->engine[i], NULL, obj); ++ } ++ ++ for (j = 0; j < ee->user_bo_count; j++) ++ print_error_obj(m, m->i915->engine[i], ++ "user", ee->user_bo[j]); ++ ++ if (ee->num_requests) { ++ err_printf(m, "%s --- %d requests\n", ++ m->i915->engine[i]->name, ++ ee->num_requests); ++ for (j = 0; j < ee->num_requests; j++) ++ error_print_request(m, " ", ++ &ee->requests[j], ++ error->epoch); ++ } ++ ++ print_error_obj(m, m->i915->engine[i], ++ "ringbuffer", ee->ringbuffer); ++ ++ print_error_obj(m, m->i915->engine[i], ++ "HW Status", ee->hws_page); ++ ++ print_error_obj(m, m->i915->engine[i], ++ "HW context", ee->ctx); ++ ++ print_error_obj(m, m->i915->engine[i], ++ "WA context", ee->wa_ctx); ++ ++ print_error_obj(m, m->i915->engine[i], ++ "WA batchbuffer", ee->wa_batchbuffer); ++ ++ print_error_obj(m, m->i915->engine[i], ++ "NULL context", ee->default_state); ++ } ++ ++ if (error->overlay) ++ intel_overlay_print_error_state(m, error->overlay); ++ ++ if (error->display) ++ intel_display_print_error_state(m, error->display); ++ ++ err_print_capabilities(m, &error->device_info, &error->runtime_info, ++ &error->driver_caps); ++ err_print_params(m, &error->params); ++ err_print_uc(m, &error->uc); ++} ++ ++static int err_print_to_sgl(struct i915_gpu_state *error) ++{ ++ struct drm_i915_error_state_buf m; ++ ++ if (IS_ERR(error)) ++ return PTR_ERR(error); ++ ++ if (READ_ONCE(error->sgl)) ++ return 0; ++ ++ memset(&m, 0, sizeof(m)); ++ m.i915 = error->i915; ++ ++ __err_print_to_sgl(&m, error); ++ ++ if (m.buf) { ++ __sg_set_buf(m.cur++, m.buf, m.bytes, m.iter); ++ m.bytes = 0; ++ m.buf = NULL; ++ } ++ if (m.cur) { ++ GEM_BUG_ON(m.end < m.cur); ++ sg_mark_end(m.cur - 1); ++ } ++ GEM_BUG_ON(m.sgl && !m.cur); ++ ++ if (m.err) { ++ err_free_sgl(m.sgl); ++ return m.err; ++ } ++ ++ if (cmpxchg(&error->sgl, NULL, m.sgl)) ++ err_free_sgl(m.sgl); ++ ++ return 0; ++} ++ ++ssize_t i915_gpu_state_copy_to_buffer(struct i915_gpu_state *error, ++ char *buf, loff_t off, size_t rem) ++{ ++ struct scatterlist *sg; ++ size_t count; ++ loff_t pos; ++ int err; ++ ++ if (!error || !rem) ++ return 0; ++ ++ err = err_print_to_sgl(error); ++ if (err) ++ return err; ++ ++ sg = READ_ONCE(error->fit); ++ if (!sg || off < sg->dma_address) ++ sg = error->sgl; ++ if (!sg) ++ return 0; ++ ++ pos = sg->dma_address; ++ count = 0; ++ do { ++ size_t len, start; ++ ++ if (sg_is_chain(sg)) { ++ sg = sg_chain_ptr(sg); ++ GEM_BUG_ON(sg_is_chain(sg)); ++ } ++ ++ len = sg->length; ++ if (pos + len <= off) { ++ pos += len; ++ continue; ++ } ++ ++ start = sg->offset; ++ if (pos < off) { ++ GEM_BUG_ON(off - pos > len); ++ len -= off - pos; ++ start += off - pos; ++ pos = off; ++ } ++ ++ len = min(len, rem); ++ GEM_BUG_ON(!len || len > sg->length); ++ ++ memcpy(buf, page_address(sg_page(sg)) + start, len); ++ ++ count += len; ++ pos += len; ++ ++ buf += len; ++ rem -= len; ++ if (!rem) { ++ WRITE_ONCE(error->fit, sg); ++ break; ++ } ++ } while (!sg_is_last(sg++)); ++ ++ return count; ++} ++ ++static void i915_error_object_free(struct drm_i915_error_object *obj) ++{ ++ int page; ++ ++ if (obj == NULL) ++ return; ++ ++ for (page = 0; page < obj->page_count; page++) ++ free_page((unsigned long)obj->pages[page]); ++ ++ kfree(obj); ++} ++ ++ ++static void cleanup_params(struct i915_gpu_state *error) ++{ ++ i915_params_free(&error->params); ++} ++ ++static void cleanup_uc_state(struct i915_gpu_state *error) ++{ ++ struct i915_error_uc *error_uc = &error->uc; ++ ++ kfree(error_uc->guc_fw.path); ++ kfree(error_uc->huc_fw.path); ++ i915_error_object_free(error_uc->guc_log); ++} ++ ++void __i915_gpu_state_free(struct kref *error_ref) ++{ ++ struct i915_gpu_state *error = ++ container_of(error_ref, typeof(*error), ref); ++ long i, j; ++ ++ for (i = 0; i < ARRAY_SIZE(error->engine); i++) { ++ struct drm_i915_error_engine *ee = &error->engine[i]; ++ ++ for (j = 0; j < ee->user_bo_count; j++) ++ i915_error_object_free(ee->user_bo[j]); ++ kfree(ee->user_bo); ++ ++ i915_error_object_free(ee->batchbuffer); ++ i915_error_object_free(ee->wa_batchbuffer); ++ i915_error_object_free(ee->ringbuffer); ++ i915_error_object_free(ee->hws_page); ++ i915_error_object_free(ee->ctx); ++ i915_error_object_free(ee->wa_ctx); ++ ++ kfree(ee->requests); ++ } ++ ++ for (i = 0; i < ARRAY_SIZE(error->active_bo); i++) ++ kfree(error->active_bo[i]); ++ kfree(error->pinned_bo); ++ ++ kfree(error->overlay); ++ kfree(error->display); ++ ++ cleanup_params(error); ++ cleanup_uc_state(error); ++ ++ err_free_sgl(error->sgl); ++ kfree(error); ++} ++ ++static struct drm_i915_error_object * ++i915_error_object_create(struct drm_i915_private *i915, ++ struct i915_vma *vma) ++{ ++ struct i915_ggtt *ggtt = &i915->ggtt; ++ const u64 slot = ggtt->error_capture.start; ++ struct drm_i915_error_object *dst; ++ struct compress compress; ++ unsigned long num_pages; ++ struct sgt_iter iter; ++ dma_addr_t dma; ++ int ret; ++ ++ if (!vma || !vma->pages) ++ return NULL; ++ ++ num_pages = min_t(u64, vma->size, vma->obj->base.size) >> PAGE_SHIFT; ++ num_pages = DIV_ROUND_UP(10 * num_pages, 8); /* worstcase zlib growth */ ++ dst = kmalloc(sizeof(*dst) + num_pages * sizeof(u32 *), ++ GFP_ATOMIC | __GFP_NOWARN); ++ if (!dst) ++ return NULL; ++ ++ dst->gtt_offset = vma->node.start; ++ dst->gtt_size = vma->node.size; ++ dst->num_pages = num_pages; ++ dst->page_count = 0; ++ dst->unused = 0; ++ ++ if (!compress_init(&compress)) { ++ kfree(dst); ++ return NULL; ++ } ++ ++ ret = -EINVAL; ++ for_each_sgt_dma(dma, iter, vma->pages) { ++ void __iomem *s; ++ ++ ggtt->vm.insert_page(&ggtt->vm, dma, slot, I915_CACHE_NONE, 0); ++ ++ s = io_mapping_map_atomic_wc(&ggtt->iomap, slot); ++ ret = compress_page(&compress, (void __force *)s, dst); ++ io_mapping_unmap_atomic(s); ++ if (ret) ++ break; ++ } ++ ++ if (ret || compress_flush(&compress, dst)) { ++ while (dst->page_count--) ++ free_page((unsigned long)dst->pages[dst->page_count]); ++ kfree(dst); ++ dst = NULL; ++ } ++ ++ compress_fini(&compress, dst); ++ return dst; ++} ++ ++static void capture_bo(struct drm_i915_error_buffer *err, ++ struct i915_vma *vma) ++{ ++ struct drm_i915_gem_object *obj = vma->obj; ++ ++ err->size = obj->base.size; ++ err->name = obj->base.name; ++ ++ err->gtt_offset = vma->node.start; ++ err->read_domains = obj->read_domains; ++ err->write_domain = obj->write_domain; ++ err->fence_reg = vma->fence ? vma->fence->id : -1; ++ err->tiling = i915_gem_object_get_tiling(obj); ++ err->dirty = obj->mm.dirty; ++ err->purgeable = obj->mm.madv != I915_MADV_WILLNEED; ++ err->userptr = obj->userptr.mm != NULL; ++ err->cache_level = obj->cache_level; ++} ++ ++static u32 capture_error_bo(struct drm_i915_error_buffer *err, ++ int count, struct list_head *head, ++ unsigned int flags) ++#define ACTIVE_ONLY BIT(0) ++#define PINNED_ONLY BIT(1) ++{ ++ struct i915_vma *vma; ++ int i = 0; ++ ++ list_for_each_entry(vma, head, vm_link) { ++ if (!vma->obj) ++ continue; ++ ++ if (flags & ACTIVE_ONLY && !i915_vma_is_active(vma)) ++ continue; ++ ++ if (flags & PINNED_ONLY && !i915_vma_is_pinned(vma)) ++ continue; ++ ++ capture_bo(err++, vma); ++ if (++i == count) ++ break; ++ } ++ ++ return i; ++} ++ ++/* ++ * Generate a semi-unique error code. The code is not meant to have meaning, The ++ * code's only purpose is to try to prevent false duplicated bug reports by ++ * grossly estimating a GPU error state. ++ * ++ * TODO Ideally, hashing the batchbuffer would be a very nice way to determine ++ * the hang if we could strip the GTT offset information from it. ++ * ++ * It's only a small step better than a random number in its current form. ++ */ ++static u32 i915_error_generate_code(struct i915_gpu_state *error, ++ intel_engine_mask_t engine_mask) ++{ ++ /* ++ * IPEHR would be an ideal way to detect errors, as it's the gross ++ * measure of "the command that hung." However, has some very common ++ * synchronization commands which almost always appear in the case ++ * strictly a client bug. Use instdone to differentiate those some. ++ */ ++ if (engine_mask) { ++ struct drm_i915_error_engine *ee = ++ &error->engine[ffs(engine_mask)]; ++ ++ return ee->ipehr ^ ee->instdone.instdone; ++ } ++ ++ return 0; ++} ++ ++static void gem_record_fences(struct i915_gpu_state *error) ++{ ++ struct drm_i915_private *dev_priv = error->i915; ++ int i; ++ ++ if (INTEL_GEN(dev_priv) >= 6) { ++ for (i = 0; i < dev_priv->num_fence_regs; i++) ++ error->fence[i] = I915_READ64(FENCE_REG_GEN6_LO(i)); ++ } else if (INTEL_GEN(dev_priv) >= 4) { ++ for (i = 0; i < dev_priv->num_fence_regs; i++) ++ error->fence[i] = I915_READ64(FENCE_REG_965_LO(i)); ++ } else { ++ for (i = 0; i < dev_priv->num_fence_regs; i++) ++ error->fence[i] = I915_READ(FENCE_REG(i)); ++ } ++ error->nfence = i; ++} ++ ++static void error_record_engine_registers(struct i915_gpu_state *error, ++ struct intel_engine_cs *engine, ++ struct drm_i915_error_engine *ee) ++{ ++ struct drm_i915_private *dev_priv = engine->i915; ++ ++ if (INTEL_GEN(dev_priv) >= 6) { ++ ee->rc_psmi = ENGINE_READ(engine, RING_PSMI_CTL); ++ if (INTEL_GEN(dev_priv) >= 8) ++ ee->fault_reg = I915_READ(GEN8_RING_FAULT_REG); ++ else ++ ee->fault_reg = I915_READ(RING_FAULT_REG(engine)); ++ } ++ ++ if (INTEL_GEN(dev_priv) >= 4) { ++ ee->faddr = ENGINE_READ(engine, RING_DMA_FADD); ++ ee->ipeir = ENGINE_READ(engine, RING_IPEIR); ++ ee->ipehr = ENGINE_READ(engine, RING_IPEHR); ++ ee->instps = ENGINE_READ(engine, RING_INSTPS); ++ ee->bbaddr = ENGINE_READ(engine, RING_BBADDR); ++ if (INTEL_GEN(dev_priv) >= 8) { ++ ee->faddr |= (u64)ENGINE_READ(engine, RING_DMA_FADD_UDW) << 32; ++ ee->bbaddr |= (u64)ENGINE_READ(engine, RING_BBADDR_UDW) << 32; ++ } ++ ee->bbstate = ENGINE_READ(engine, RING_BBSTATE); ++ } else { ++ ee->faddr = ENGINE_READ(engine, DMA_FADD_I8XX); ++ ee->ipeir = ENGINE_READ(engine, IPEIR); ++ ee->ipehr = ENGINE_READ(engine, IPEHR); ++ } ++ ++ intel_engine_get_instdone(engine, &ee->instdone); ++ ++ ee->instpm = ENGINE_READ(engine, RING_INSTPM); ++ ee->acthd = intel_engine_get_active_head(engine); ++ ee->start = ENGINE_READ(engine, RING_START); ++ ee->head = ENGINE_READ(engine, RING_HEAD); ++ ee->tail = ENGINE_READ(engine, RING_TAIL); ++ ee->ctl = ENGINE_READ(engine, RING_CTL); ++ if (INTEL_GEN(dev_priv) > 2) ++ ee->mode = ENGINE_READ(engine, RING_MI_MODE); ++ ++ if (!HWS_NEEDS_PHYSICAL(dev_priv)) { ++ i915_reg_t mmio; ++ ++ if (IS_GEN(dev_priv, 7)) { ++ switch (engine->id) { ++ default: ++ MISSING_CASE(engine->id); ++ case RCS0: ++ mmio = RENDER_HWS_PGA_GEN7; ++ break; ++ case BCS0: ++ mmio = BLT_HWS_PGA_GEN7; ++ break; ++ case VCS0: ++ mmio = BSD_HWS_PGA_GEN7; ++ break; ++ case VECS0: ++ mmio = VEBOX_HWS_PGA_GEN7; ++ break; ++ } ++ } else if (IS_GEN(engine->i915, 6)) { ++ mmio = RING_HWS_PGA_GEN6(engine->mmio_base); ++ } else { ++ /* XXX: gen8 returns to sanity */ ++ mmio = RING_HWS_PGA(engine->mmio_base); ++ } ++ ++ ee->hws = I915_READ(mmio); ++ } ++ ++ ee->idle = intel_engine_is_idle(engine); ++ if (!ee->idle) ++ ee->hangcheck_timestamp = engine->hangcheck.action_timestamp; ++ ee->reset_count = i915_reset_engine_count(&dev_priv->gpu_error, ++ engine); ++ ++ if (HAS_PPGTT(dev_priv)) { ++ int i; ++ ++ ee->vm_info.gfx_mode = I915_READ(RING_MODE_GEN7(engine)); ++ ++ if (IS_GEN(dev_priv, 6)) { ++ ee->vm_info.pp_dir_base = ++ ENGINE_READ(engine, RING_PP_DIR_BASE_READ); ++ } else if (IS_GEN(dev_priv, 7)) { ++ ee->vm_info.pp_dir_base = ++ ENGINE_READ(engine, RING_PP_DIR_BASE); ++ } else if (INTEL_GEN(dev_priv) >= 8) { ++ u32 base = engine->mmio_base; ++ ++ for (i = 0; i < 4; i++) { ++ ee->vm_info.pdp[i] = ++ I915_READ(GEN8_RING_PDP_UDW(base, i)); ++ ee->vm_info.pdp[i] <<= 32; ++ ee->vm_info.pdp[i] |= ++ I915_READ(GEN8_RING_PDP_LDW(base, i)); ++ } ++ } ++ } ++} ++ ++static void record_request(struct i915_request *request, ++ struct drm_i915_error_request *erq) ++{ ++ struct i915_gem_context *ctx = request->gem_context; ++ ++ erq->flags = request->fence.flags; ++ erq->context = request->fence.context; ++ erq->seqno = request->fence.seqno; ++ erq->sched_attr = request->sched.attr; ++ erq->jiffies = request->emitted_jiffies; ++ erq->start = i915_ggtt_offset(request->ring->vma); ++ erq->head = request->head; ++ erq->tail = request->tail; ++ ++ rcu_read_lock(); ++ erq->pid = ctx->pid ? pid_nr(ctx->pid) : 0; ++ rcu_read_unlock(); ++} ++ ++static void engine_record_requests(struct intel_engine_cs *engine, ++ struct i915_request *first, ++ struct drm_i915_error_engine *ee) ++{ ++ struct i915_request *request; ++ int count; ++ ++ count = 0; ++ request = first; ++ list_for_each_entry_from(request, &engine->timeline.requests, link) ++ count++; ++ if (!count) ++ return; ++ ++ ee->requests = kcalloc(count, sizeof(*ee->requests), GFP_ATOMIC); ++ if (!ee->requests) ++ return; ++ ++ ee->num_requests = count; ++ ++ count = 0; ++ request = first; ++ list_for_each_entry_from(request, &engine->timeline.requests, link) { ++ if (count >= ee->num_requests) { ++ /* ++ * If the ring request list was changed in ++ * between the point where the error request ++ * list was created and dimensioned and this ++ * point then just exit early to avoid crashes. ++ * ++ * We don't need to communicate that the ++ * request list changed state during error ++ * state capture and that the error state is ++ * slightly incorrect as a consequence since we ++ * are typically only interested in the request ++ * list state at the point of error state ++ * capture, not in any changes happening during ++ * the capture. ++ */ ++ break; ++ } ++ ++ record_request(request, &ee->requests[count++]); ++ } ++ ee->num_requests = count; ++} ++ ++static void error_record_engine_execlists(struct intel_engine_cs *engine, ++ struct drm_i915_error_engine *ee) ++{ ++ const struct intel_engine_execlists * const execlists = &engine->execlists; ++ unsigned int n; ++ ++ for (n = 0; n < execlists_num_ports(execlists); n++) { ++ struct i915_request *rq = port_request(&execlists->port[n]); ++ ++ if (!rq) ++ break; ++ ++ record_request(rq, &ee->execlist[n]); ++ } ++ ++ ee->num_ports = n; ++} ++ ++static void record_context(struct drm_i915_error_context *e, ++ struct i915_gem_context *ctx) ++{ ++ if (ctx->pid) { ++ struct task_struct *task; ++ ++ rcu_read_lock(); ++ task = pid_task(ctx->pid, PIDTYPE_PID); ++ if (task) { ++ strcpy(e->comm, task->comm); ++ e->pid = task->pid; ++ } ++ rcu_read_unlock(); ++ } ++ ++ e->hw_id = ctx->hw_id; ++ e->sched_attr = ctx->sched; ++ e->guilty = atomic_read(&ctx->guilty_count); ++ e->active = atomic_read(&ctx->active_count); ++} ++ ++static void request_record_user_bo(struct i915_request *request, ++ struct drm_i915_error_engine *ee) ++{ ++ struct i915_capture_list *c; ++ struct drm_i915_error_object **bo; ++ long count, max; ++ ++ max = 0; ++ for (c = request->capture_list; c; c = c->next) ++ max++; ++ if (!max) ++ return; ++ ++ bo = kmalloc_array(max, sizeof(*bo), GFP_ATOMIC); ++ if (!bo) { ++ /* If we can't capture everything, try to capture something. */ ++ max = min_t(long, max, PAGE_SIZE / sizeof(*bo)); ++ bo = kmalloc_array(max, sizeof(*bo), GFP_ATOMIC); ++ } ++ if (!bo) ++ return; ++ ++ count = 0; ++ for (c = request->capture_list; c; c = c->next) { ++ bo[count] = i915_error_object_create(request->i915, c->vma); ++ if (!bo[count]) ++ break; ++ if (++count == max) ++ break; ++ } ++ ++ ee->user_bo = bo; ++ ee->user_bo_count = count; ++} ++ ++static struct drm_i915_error_object * ++capture_object(struct drm_i915_private *dev_priv, ++ struct drm_i915_gem_object *obj) ++{ ++ if (obj && i915_gem_object_has_pages(obj)) { ++ struct i915_vma fake = { ++ .node = { .start = U64_MAX, .size = obj->base.size }, ++ .size = obj->base.size, ++ .pages = obj->mm.pages, ++ .obj = obj, ++ }; ++ ++ return i915_error_object_create(dev_priv, &fake); ++ } else { ++ return NULL; ++ } ++} ++ ++static void gem_record_rings(struct i915_gpu_state *error) ++{ ++ struct drm_i915_private *i915 = error->i915; ++ struct i915_ggtt *ggtt = &i915->ggtt; ++ int i; ++ ++ for (i = 0; i < I915_NUM_ENGINES; i++) { ++ struct intel_engine_cs *engine = i915->engine[i]; ++ struct drm_i915_error_engine *ee = &error->engine[i]; ++ struct i915_request *request; ++ ++ ee->engine_id = -1; ++ ++ if (!engine) ++ continue; ++ ++ ee->engine_id = i; ++ ++ error_record_engine_registers(error, engine, ee); ++ error_record_engine_execlists(engine, ee); ++ ++ request = intel_engine_find_active_request(engine); ++ if (request) { ++ struct i915_gem_context *ctx = request->gem_context; ++ struct intel_ring *ring; ++ ++ ee->vm = ctx->ppgtt ? &ctx->ppgtt->vm : &ggtt->vm; ++ ++ record_context(&ee->context, ctx); ++ ++ /* We need to copy these to an anonymous buffer ++ * as the simplest method to avoid being overwritten ++ * by userspace. ++ */ ++ ee->batchbuffer = ++ i915_error_object_create(i915, request->batch); ++ ++ if (HAS_BROKEN_CS_TLB(i915)) ++ ee->wa_batchbuffer = ++ i915_error_object_create(i915, ++ i915->gt.scratch); ++ request_record_user_bo(request, ee); ++ ++ ee->ctx = ++ i915_error_object_create(i915, ++ request->hw_context->state); ++ ++ error->simulated |= ++ i915_gem_context_no_error_capture(ctx); ++ ++ ee->rq_head = request->head; ++ ee->rq_post = request->postfix; ++ ee->rq_tail = request->tail; ++ ++ ring = request->ring; ++ ee->cpu_ring_head = ring->head; ++ ee->cpu_ring_tail = ring->tail; ++ ee->ringbuffer = ++ i915_error_object_create(i915, ring->vma); ++ ++ engine_record_requests(engine, request, ee); ++ } ++ ++ ee->hws_page = ++ i915_error_object_create(i915, ++ engine->status_page.vma); ++ ++ ee->wa_ctx = i915_error_object_create(i915, engine->wa_ctx.vma); ++ ++ ee->default_state = capture_object(i915, engine->default_state); ++ } ++} ++ ++static void gem_capture_vm(struct i915_gpu_state *error, ++ struct i915_address_space *vm, ++ int idx) ++{ ++ struct drm_i915_error_buffer *active_bo; ++ struct i915_vma *vma; ++ int count; ++ ++ count = 0; ++ list_for_each_entry(vma, &vm->bound_list, vm_link) ++ if (i915_vma_is_active(vma)) ++ count++; ++ ++ active_bo = NULL; ++ if (count) ++ active_bo = kcalloc(count, sizeof(*active_bo), GFP_ATOMIC); ++ if (active_bo) ++ count = capture_error_bo(active_bo, ++ count, &vm->bound_list, ++ ACTIVE_ONLY); ++ else ++ count = 0; ++ ++ error->active_vm[idx] = vm; ++ error->active_bo[idx] = active_bo; ++ error->active_bo_count[idx] = count; ++} ++ ++static void capture_active_buffers(struct i915_gpu_state *error) ++{ ++ int cnt = 0, i, j; ++ ++ BUILD_BUG_ON(ARRAY_SIZE(error->engine) > ARRAY_SIZE(error->active_bo)); ++ BUILD_BUG_ON(ARRAY_SIZE(error->active_bo) != ARRAY_SIZE(error->active_vm)); ++ BUILD_BUG_ON(ARRAY_SIZE(error->active_bo) != ARRAY_SIZE(error->active_bo_count)); ++ ++ /* Scan each engine looking for unique active contexts/vm */ ++ for (i = 0; i < ARRAY_SIZE(error->engine); i++) { ++ struct drm_i915_error_engine *ee = &error->engine[i]; ++ bool found; ++ ++ if (!ee->vm) ++ continue; ++ ++ found = false; ++ for (j = 0; j < i && !found; j++) ++ found = error->engine[j].vm == ee->vm; ++ if (!found) ++ gem_capture_vm(error, ee->vm, cnt++); ++ } ++} ++ ++static void capture_pinned_buffers(struct i915_gpu_state *error) ++{ ++ struct i915_address_space *vm = &error->i915->ggtt.vm; ++ struct drm_i915_error_buffer *bo; ++ struct i915_vma *vma; ++ int count; ++ ++ count = 0; ++ list_for_each_entry(vma, &vm->bound_list, vm_link) ++ count++; ++ ++ bo = NULL; ++ if (count) ++ bo = kcalloc(count, sizeof(*bo), GFP_ATOMIC); ++ if (!bo) ++ return; ++ ++ error->pinned_bo_count = ++ capture_error_bo(bo, count, &vm->bound_list, PINNED_ONLY); ++ error->pinned_bo = bo; ++} ++ ++static void capture_uc_state(struct i915_gpu_state *error) ++{ ++ struct drm_i915_private *i915 = error->i915; ++ struct i915_error_uc *error_uc = &error->uc; ++ ++ /* Capturing uC state won't be useful if there is no GuC */ ++ if (!error->device_info.has_guc) ++ return; ++ ++ error_uc->guc_fw = i915->guc.fw; ++ error_uc->huc_fw = i915->huc.fw; ++ ++ /* Non-default firmware paths will be specified by the modparam. ++ * As modparams are generally accesible from the userspace make ++ * explicit copies of the firmware paths. ++ */ ++ error_uc->guc_fw.path = kstrdup(i915->guc.fw.path, GFP_ATOMIC); ++ error_uc->huc_fw.path = kstrdup(i915->huc.fw.path, GFP_ATOMIC); ++ error_uc->guc_log = i915_error_object_create(i915, i915->guc.log.vma); ++} ++ ++/* Capture all registers which don't fit into another category. */ ++static void capture_reg_state(struct i915_gpu_state *error) ++{ ++ struct drm_i915_private *dev_priv = error->i915; ++ int i; ++ ++ /* General organization ++ * 1. Registers specific to a single generation ++ * 2. Registers which belong to multiple generations ++ * 3. Feature specific registers. ++ * 4. Everything else ++ * Please try to follow the order. ++ */ ++ ++ /* 1: Registers specific to a single generation */ ++ if (IS_VALLEYVIEW(dev_priv)) { ++ error->gtier[0] = I915_READ(GTIER); ++ error->ier = I915_READ(VLV_IER); ++ error->forcewake = I915_READ_FW(FORCEWAKE_VLV); ++ } ++ ++ if (IS_GEN(dev_priv, 7)) ++ error->err_int = I915_READ(GEN7_ERR_INT); ++ ++ if (INTEL_GEN(dev_priv) >= 8) { ++ error->fault_data0 = I915_READ(GEN8_FAULT_TLB_DATA0); ++ error->fault_data1 = I915_READ(GEN8_FAULT_TLB_DATA1); ++ } ++ ++ if (IS_GEN(dev_priv, 6)) { ++ error->forcewake = I915_READ_FW(FORCEWAKE); ++ error->gab_ctl = I915_READ(GAB_CTL); ++ error->gfx_mode = I915_READ(GFX_MODE); ++ } ++ ++ /* 2: Registers which belong to multiple generations */ ++ if (INTEL_GEN(dev_priv) >= 7) ++ error->forcewake = I915_READ_FW(FORCEWAKE_MT); ++ ++ if (INTEL_GEN(dev_priv) >= 6) { ++ error->derrmr = I915_READ(DERRMR); ++ error->error = I915_READ(ERROR_GEN6); ++ error->done_reg = I915_READ(DONE_REG); ++ } ++ ++ if (INTEL_GEN(dev_priv) >= 5) ++ error->ccid = I915_READ(CCID(RENDER_RING_BASE)); ++ ++ /* 3: Feature specific registers */ ++ if (IS_GEN_RANGE(dev_priv, 6, 7)) { ++ error->gam_ecochk = I915_READ(GAM_ECOCHK); ++ error->gac_eco = I915_READ(GAC_ECO_BITS); ++ } ++ ++ /* 4: Everything else */ ++ if (INTEL_GEN(dev_priv) >= 11) { ++ error->ier = I915_READ(GEN8_DE_MISC_IER); ++ error->gtier[0] = I915_READ(GEN11_RENDER_COPY_INTR_ENABLE); ++ error->gtier[1] = I915_READ(GEN11_VCS_VECS_INTR_ENABLE); ++ error->gtier[2] = I915_READ(GEN11_GUC_SG_INTR_ENABLE); ++ error->gtier[3] = I915_READ(GEN11_GPM_WGBOXPERF_INTR_ENABLE); ++ error->gtier[4] = I915_READ(GEN11_CRYPTO_RSVD_INTR_ENABLE); ++ error->gtier[5] = I915_READ(GEN11_GUNIT_CSME_INTR_ENABLE); ++ error->ngtier = 6; ++ } else if (INTEL_GEN(dev_priv) >= 8) { ++ error->ier = I915_READ(GEN8_DE_MISC_IER); ++ for (i = 0; i < 4; i++) ++ error->gtier[i] = I915_READ(GEN8_GT_IER(i)); ++ error->ngtier = 4; ++ } else if (HAS_PCH_SPLIT(dev_priv)) { ++ error->ier = I915_READ(DEIER); ++ error->gtier[0] = I915_READ(GTIER); ++ error->ngtier = 1; ++ } else if (IS_GEN(dev_priv, 2)) { ++ error->ier = I915_READ16(GEN2_IER); ++ } else if (!IS_VALLEYVIEW(dev_priv)) { ++ error->ier = I915_READ(GEN2_IER); ++ } ++ error->eir = I915_READ(EIR); ++ error->pgtbl_er = I915_READ(PGTBL_ER); ++} ++ ++static const char * ++error_msg(struct i915_gpu_state *error, ++ intel_engine_mask_t engines, const char *msg) ++{ ++ int len; ++ int i; ++ ++ for (i = 0; i < ARRAY_SIZE(error->engine); i++) ++ if (!error->engine[i].context.pid) ++ engines &= ~BIT(i); ++ ++ len = scnprintf(error->error_msg, sizeof(error->error_msg), ++ "GPU HANG: ecode %d:%x:0x%08x", ++ INTEL_GEN(error->i915), engines, ++ i915_error_generate_code(error, engines)); ++ if (engines) { ++ /* Just show the first executing process, more is confusing */ ++ i = __ffs(engines); ++ len += scnprintf(error->error_msg + len, ++ sizeof(error->error_msg) - len, ++ ", in %s [%d]", ++ error->engine[i].context.comm, ++ error->engine[i].context.pid); ++ } ++ if (msg) ++ len += scnprintf(error->error_msg + len, ++ sizeof(error->error_msg) - len, ++ ", %s", msg); ++ ++ return error->error_msg; ++} ++ ++static void capture_gen_state(struct i915_gpu_state *error) ++{ ++ struct drm_i915_private *i915 = error->i915; ++ ++ error->awake = i915->gt.awake; ++ error->wakelock = atomic_read(&i915->runtime_pm.wakeref_count); ++ error->suspended = i915->runtime_pm.suspended; ++ ++ error->iommu = -1; ++#ifdef CONFIG_INTEL_IOMMU ++ error->iommu = intel_iommu_gfx_mapped; ++#endif ++ error->reset_count = i915_reset_count(&i915->gpu_error); ++ error->suspend_count = i915->suspend_count; ++ ++ memcpy(&error->device_info, ++ INTEL_INFO(i915), ++ sizeof(error->device_info)); ++ memcpy(&error->runtime_info, ++ RUNTIME_INFO(i915), ++ sizeof(error->runtime_info)); ++ error->driver_caps = i915->caps; ++} ++ ++static void capture_params(struct i915_gpu_state *error) ++{ ++ i915_params_copy(&error->params, &i915_modparams); ++} ++ ++static unsigned long capture_find_epoch(const struct i915_gpu_state *error) ++{ ++ unsigned long epoch = error->capture; ++ int i; ++ ++ for (i = 0; i < ARRAY_SIZE(error->engine); i++) { ++ const struct drm_i915_error_engine *ee = &error->engine[i]; ++ ++ if (ee->hangcheck_timestamp && ++ time_before(ee->hangcheck_timestamp, epoch)) ++ epoch = ee->hangcheck_timestamp; ++ } ++ ++ return epoch; ++} ++ ++static void capture_finish(struct i915_gpu_state *error) ++{ ++ struct i915_ggtt *ggtt = &error->i915->ggtt; ++ const u64 slot = ggtt->error_capture.start; ++ ++ ggtt->vm.clear_range(&ggtt->vm, slot, PAGE_SIZE); ++} ++ ++static int capture(void *data) ++{ ++ struct i915_gpu_state *error = data; ++ ++ error->time = ktime_get_real(); ++ error->boottime = ktime_get_boottime(); ++ error->uptime = ktime_sub(ktime_get(), ++ error->i915->gt.last_init_time); ++ error->capture = jiffies; ++ ++ capture_params(error); ++ capture_gen_state(error); ++ capture_uc_state(error); ++ capture_reg_state(error); ++ gem_record_fences(error); ++ gem_record_rings(error); ++ capture_active_buffers(error); ++ capture_pinned_buffers(error); ++ ++ error->overlay = intel_overlay_capture_error_state(error->i915); ++ error->display = intel_display_capture_error_state(error->i915); ++ ++ error->epoch = capture_find_epoch(error); ++ ++ capture_finish(error); ++ return 0; ++} ++ ++#define DAY_AS_SECONDS(x) (24 * 60 * 60 * (x)) ++ ++struct i915_gpu_state * ++i915_capture_gpu_state(struct drm_i915_private *i915) ++{ ++ struct i915_gpu_state *error; ++ ++ /* Check if GPU capture has been disabled */ ++ error = READ_ONCE(i915->gpu_error.first_error); ++ if (IS_ERR(error)) ++ return error; ++ ++ error = kzalloc(sizeof(*error), GFP_ATOMIC); ++ if (!error) { ++ i915_disable_error_state(i915, -ENOMEM); ++ return ERR_PTR(-ENOMEM); ++ } ++ ++ kref_init(&error->ref); ++ error->i915 = i915; ++ ++ stop_machine(capture, error, NULL); ++ ++ return error; ++} ++ ++/** ++ * i915_capture_error_state - capture an error record for later analysis ++ * @i915: i915 device ++ * @engine_mask: the mask of engines triggering the hang ++ * @msg: a message to insert into the error capture header ++ * ++ * Should be called when an error is detected (either a hang or an error ++ * interrupt) to capture error state from the time of the error. Fills ++ * out a structure which becomes available in debugfs for user level tools ++ * to pick up. ++ */ ++void i915_capture_error_state(struct drm_i915_private *i915, ++ intel_engine_mask_t engine_mask, ++ const char *msg) ++{ ++ static bool warned; ++ struct i915_gpu_state *error; ++ unsigned long flags; ++ ++ if (!i915_modparams.error_capture) ++ return; ++ ++ if (READ_ONCE(i915->gpu_error.first_error)) ++ return; ++ ++ error = i915_capture_gpu_state(i915); ++ if (IS_ERR(error)) ++ return; ++ ++ dev_info(i915->drm.dev, "%s\n", error_msg(error, engine_mask, msg)); ++ ++ if (!error->simulated) { ++ spin_lock_irqsave(&i915->gpu_error.lock, flags); ++ if (!i915->gpu_error.first_error) { ++ i915->gpu_error.first_error = error; ++ error = NULL; ++ } ++ spin_unlock_irqrestore(&i915->gpu_error.lock, flags); ++ } ++ ++ if (error) { ++ __i915_gpu_state_free(&error->ref); ++ return; ++ } ++ ++ if (!warned && ++ ktime_get_real_seconds() - DRIVER_TIMESTAMP < DAY_AS_SECONDS(180)) { ++ DRM_INFO("GPU hangs can indicate a bug anywhere in the entire gfx stack, including userspace.\n"); ++ DRM_INFO("Please file a _new_ bug report on bugs.freedesktop.org against DRI -> DRM/Intel\n"); ++ DRM_INFO("drm/i915 developers can then reassign to the right component if it's not a kernel issue.\n"); ++ DRM_INFO("The gpu crash dump is required to analyze gpu hangs, so please always attach it.\n"); ++ DRM_INFO("GPU crash dump saved to /sys/class/drm/card%d/error\n", ++ i915->drm.primary->index); ++ warned = true; ++ } ++} ++ ++struct i915_gpu_state * ++i915_first_error_state(struct drm_i915_private *i915) ++{ ++ struct i915_gpu_state *error; ++ ++ spin_lock_irq(&i915->gpu_error.lock); ++ error = i915->gpu_error.first_error; ++ if (!IS_ERR_OR_NULL(error)) ++ i915_gpu_state_get(error); ++ spin_unlock_irq(&i915->gpu_error.lock); ++ ++ return error; ++} ++ ++void i915_reset_error_state(struct drm_i915_private *i915) ++{ ++ struct i915_gpu_state *error; ++ ++ spin_lock_irq(&i915->gpu_error.lock); ++ error = i915->gpu_error.first_error; ++ if (error != ERR_PTR(-ENODEV)) /* if disabled, always disabled */ ++ i915->gpu_error.first_error = NULL; ++ spin_unlock_irq(&i915->gpu_error.lock); ++ ++ if (!IS_ERR_OR_NULL(error)) ++ i915_gpu_state_put(error); ++} ++ ++void i915_disable_error_state(struct drm_i915_private *i915, int err) ++{ ++ spin_lock_irq(&i915->gpu_error.lock); ++ if (!i915->gpu_error.first_error) ++ i915->gpu_error.first_error = ERR_PTR(err); ++ spin_unlock_irq(&i915->gpu_error.lock); ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_gpu_error.h b/drivers/gpu/drm/i915_legacy/i915_gpu_error.h +new file mode 100644 +index 000000000000..5dc761e85d9d +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_gpu_error.h +@@ -0,0 +1,315 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright � 2008-2018 Intel Corporation ++ */ ++ ++#ifndef _I915_GPU_ERROR_H_ ++#define _I915_GPU_ERROR_H_ ++ ++#include ++#include ++#include ++ ++#include ++ ++#include "intel_device_info.h" ++#include "intel_ringbuffer.h" ++#include "intel_uc_fw.h" ++ ++#include "i915_gem.h" ++#include "i915_gem_gtt.h" ++#include "i915_params.h" ++#include "i915_scheduler.h" ++ ++struct drm_i915_private; ++struct intel_overlay_error_state; ++struct intel_display_error_state; ++ ++struct i915_gpu_state { ++ struct kref ref; ++ ktime_t time; ++ ktime_t boottime; ++ ktime_t uptime; ++ unsigned long capture; ++ unsigned long epoch; ++ ++ struct drm_i915_private *i915; ++ ++ char error_msg[128]; ++ bool simulated; ++ bool awake; ++ bool wakelock; ++ bool suspended; ++ int iommu; ++ u32 reset_count; ++ u32 suspend_count; ++ struct intel_device_info device_info; ++ struct intel_runtime_info runtime_info; ++ struct intel_driver_caps driver_caps; ++ struct i915_params params; ++ ++ struct i915_error_uc { ++ struct intel_uc_fw guc_fw; ++ struct intel_uc_fw huc_fw; ++ struct drm_i915_error_object *guc_log; ++ } uc; ++ ++ /* Generic register state */ ++ u32 eir; ++ u32 pgtbl_er; ++ u32 ier; ++ u32 gtier[6], ngtier; ++ u32 ccid; ++ u32 derrmr; ++ u32 forcewake; ++ u32 error; /* gen6+ */ ++ u32 err_int; /* gen7 */ ++ u32 fault_data0; /* gen8, gen9 */ ++ u32 fault_data1; /* gen8, gen9 */ ++ u32 done_reg; ++ u32 gac_eco; ++ u32 gam_ecochk; ++ u32 gab_ctl; ++ u32 gfx_mode; ++ ++ u32 nfence; ++ u64 fence[I915_MAX_NUM_FENCES]; ++ struct intel_overlay_error_state *overlay; ++ struct intel_display_error_state *display; ++ ++ struct drm_i915_error_engine { ++ int engine_id; ++ /* Software tracked state */ ++ bool idle; ++ unsigned long hangcheck_timestamp; ++ struct i915_address_space *vm; ++ int num_requests; ++ u32 reset_count; ++ ++ /* position of active request inside the ring */ ++ u32 rq_head, rq_post, rq_tail; ++ ++ /* our own tracking of ring head and tail */ ++ u32 cpu_ring_head; ++ u32 cpu_ring_tail; ++ ++ /* Register state */ ++ u32 start; ++ u32 tail; ++ u32 head; ++ u32 ctl; ++ u32 mode; ++ u32 hws; ++ u32 ipeir; ++ u32 ipehr; ++ u32 bbstate; ++ u32 instpm; ++ u32 instps; ++ u64 bbaddr; ++ u64 acthd; ++ u32 fault_reg; ++ u64 faddr; ++ u32 rc_psmi; /* sleep state */ ++ struct intel_instdone instdone; ++ ++ struct drm_i915_error_context { ++ char comm[TASK_COMM_LEN]; ++ pid_t pid; ++ u32 hw_id; ++ int active; ++ int guilty; ++ struct i915_sched_attr sched_attr; ++ } context; ++ ++ struct drm_i915_error_object { ++ u64 gtt_offset; ++ u64 gtt_size; ++ int num_pages; ++ int page_count; ++ int unused; ++ u32 *pages[0]; ++ } *ringbuffer, *batchbuffer, *wa_batchbuffer, *ctx, *hws_page; ++ ++ struct drm_i915_error_object **user_bo; ++ long user_bo_count; ++ ++ struct drm_i915_error_object *wa_ctx; ++ struct drm_i915_error_object *default_state; ++ ++ struct drm_i915_error_request { ++ unsigned long flags; ++ long jiffies; ++ pid_t pid; ++ u32 context; ++ u32 seqno; ++ u32 start; ++ u32 head; ++ u32 tail; ++ struct i915_sched_attr sched_attr; ++ } *requests, execlist[EXECLIST_MAX_PORTS]; ++ unsigned int num_ports; ++ ++ struct { ++ u32 gfx_mode; ++ union { ++ u64 pdp[4]; ++ u32 pp_dir_base; ++ }; ++ } vm_info; ++ } engine[I915_NUM_ENGINES]; ++ ++ struct drm_i915_error_buffer { ++ u32 size; ++ u32 name; ++ u64 gtt_offset; ++ u32 read_domains; ++ u32 write_domain; ++ s32 fence_reg:I915_MAX_NUM_FENCE_BITS; ++ u32 tiling:2; ++ u32 dirty:1; ++ u32 purgeable:1; ++ u32 userptr:1; ++ u32 cache_level:3; ++ } *active_bo[I915_NUM_ENGINES], *pinned_bo; ++ u32 active_bo_count[I915_NUM_ENGINES], pinned_bo_count; ++ struct i915_address_space *active_vm[I915_NUM_ENGINES]; ++ ++ struct scatterlist *sgl, *fit; ++}; ++ ++struct i915_gpu_restart; ++ ++struct i915_gpu_error { ++ /* For hangcheck timer */ ++#define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */ ++#define DRM_I915_HANGCHECK_JIFFIES msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD) ++ ++ struct delayed_work hangcheck_work; ++ ++ /* For reset and error_state handling. */ ++ spinlock_t lock; ++ /* Protected by the above dev->gpu_error.lock. */ ++ struct i915_gpu_state *first_error; ++ ++ atomic_t pending_fb_pin; ++ ++ /** ++ * flags: Control various stages of the GPU reset ++ * ++ * #I915_RESET_BACKOFF - When we start a global reset, we need to ++ * serialise with any other users attempting to do the same, and ++ * any global resources that may be clobber by the reset (such as ++ * FENCE registers). ++ * ++ * #I915_RESET_ENGINE[num_engines] - Since the driver doesn't need to ++ * acquire the struct_mutex to reset an engine, we need an explicit ++ * flag to prevent two concurrent reset attempts in the same engine. ++ * As the number of engines continues to grow, allocate the flags from ++ * the most significant bits. ++ * ++ * #I915_WEDGED - If reset fails and we can no longer use the GPU, ++ * we set the #I915_WEDGED bit. Prior to command submission, e.g. ++ * i915_request_alloc(), this bit is checked and the sequence ++ * aborted (with -EIO reported to userspace) if set. ++ */ ++ unsigned long flags; ++#define I915_RESET_BACKOFF 0 ++#define I915_RESET_MODESET 1 ++#define I915_RESET_ENGINE 2 ++#define I915_WEDGED (BITS_PER_LONG - 1) ++ ++ /** Number of times the device has been reset (global) */ ++ u32 reset_count; ++ ++ /** Number of times an engine has been reset */ ++ u32 reset_engine_count[I915_NUM_ENGINES]; ++ ++ struct mutex wedge_mutex; /* serialises wedging/unwedging */ ++ ++ /** ++ * Waitqueue to signal when a hang is detected. Used to for waiters ++ * to release the struct_mutex for the reset to procede. ++ */ ++ wait_queue_head_t wait_queue; ++ ++ /** ++ * Waitqueue to signal when the reset has completed. Used by clients ++ * that wait for dev_priv->mm.wedged to settle. ++ */ ++ wait_queue_head_t reset_queue; ++ ++ struct srcu_struct reset_backoff_srcu; ++ ++ struct i915_gpu_restart *restart; ++}; ++ ++struct drm_i915_error_state_buf { ++ struct drm_i915_private *i915; ++ struct scatterlist *sgl, *cur, *end; ++ ++ char *buf; ++ size_t bytes; ++ size_t size; ++ loff_t iter; ++ ++ int err; ++}; ++ ++#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) ++ ++__printf(2, 3) ++void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...); ++ ++struct i915_gpu_state *i915_capture_gpu_state(struct drm_i915_private *i915); ++void i915_capture_error_state(struct drm_i915_private *dev_priv, ++ intel_engine_mask_t engine_mask, ++ const char *error_msg); ++ ++static inline struct i915_gpu_state * ++i915_gpu_state_get(struct i915_gpu_state *gpu) ++{ ++ kref_get(&gpu->ref); ++ return gpu; ++} ++ ++ssize_t i915_gpu_state_copy_to_buffer(struct i915_gpu_state *error, ++ char *buf, loff_t offset, size_t count); ++ ++void __i915_gpu_state_free(struct kref *kref); ++static inline void i915_gpu_state_put(struct i915_gpu_state *gpu) ++{ ++ if (gpu) ++ kref_put(&gpu->ref, __i915_gpu_state_free); ++} ++ ++struct i915_gpu_state *i915_first_error_state(struct drm_i915_private *i915); ++void i915_reset_error_state(struct drm_i915_private *i915); ++void i915_disable_error_state(struct drm_i915_private *i915, int err); ++ ++#else ++ ++static inline void i915_capture_error_state(struct drm_i915_private *dev_priv, ++ u32 engine_mask, ++ const char *error_msg) ++{ ++} ++ ++static inline struct i915_gpu_state * ++i915_first_error_state(struct drm_i915_private *i915) ++{ ++ return ERR_PTR(-ENODEV); ++} ++ ++static inline void i915_reset_error_state(struct drm_i915_private *i915) ++{ ++} ++ ++static inline void i915_disable_error_state(struct drm_i915_private *i915, ++ int err) ++{ ++} ++ ++#endif /* IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) */ ++ ++#endif /* _I915_GPU_ERROR_H_ */ +diff --git a/drivers/gpu/drm/i915_legacy/i915_ioc32.c b/drivers/gpu/drm/i915_legacy/i915_ioc32.c +new file mode 100644 +index 000000000000..c1007245f46d +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_ioc32.c +@@ -0,0 +1,96 @@ ++/* ++ * 32-bit ioctl compatibility routines for the i915 DRM. ++ * ++ * Copyright (C) Paul Mackerras 2005 ++ * Copyright (C) Alan Hourihane 2005 ++ * All Rights Reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, ++ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ * Author: Alan Hourihane ++ */ ++#include ++ ++#include ++#include ++#include "i915_drv.h" ++ ++struct drm_i915_getparam32 { ++ s32 param; ++ /* ++ * We screwed up the generic ioctl struct here and used a variable-sized ++ * pointer. Use u32 in the compat struct to match the 32bit pointer ++ * userspace expects. ++ */ ++ u32 value; ++}; ++ ++static int compat_i915_getparam(struct file *file, unsigned int cmd, ++ unsigned long arg) ++{ ++ struct drm_i915_getparam32 req32; ++ drm_i915_getparam_t __user *request; ++ ++ if (copy_from_user(&req32, (void __user *)arg, sizeof(req32))) ++ return -EFAULT; ++ ++ request = compat_alloc_user_space(sizeof(*request)); ++ if (!access_ok(request, sizeof(*request)) || ++ __put_user(req32.param, &request->param) || ++ __put_user((void __user *)(unsigned long)req32.value, ++ &request->value)) ++ return -EFAULT; ++ ++ return drm_ioctl(file, DRM_IOCTL_I915_GETPARAM, ++ (unsigned long)request); ++} ++ ++static drm_ioctl_compat_t *i915_compat_ioctls[] = { ++ [DRM_I915_GETPARAM] = compat_i915_getparam, ++}; ++ ++/** ++ * i915_compat_ioctl - handle the mistakes of the past ++ * @filp: the file pointer ++ * @cmd: the ioctl command (and encoded flags) ++ * @arg: the ioctl argument (from userspace) ++ * ++ * Called whenever a 32-bit process running under a 64-bit kernel ++ * performs an ioctl on /dev/dri/card. ++ */ ++long i915_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) ++{ ++ unsigned int nr = DRM_IOCTL_NR(cmd); ++ drm_ioctl_compat_t *fn = NULL; ++ int ret; ++ ++ if (nr < DRM_COMMAND_BASE || nr >= DRM_COMMAND_END) ++ return drm_compat_ioctl(filp, cmd, arg); ++ ++ if (nr < DRM_COMMAND_BASE + ARRAY_SIZE(i915_compat_ioctls)) ++ fn = i915_compat_ioctls[nr - DRM_COMMAND_BASE]; ++ ++ if (fn != NULL) ++ ret = (*fn) (filp, cmd, arg); ++ else ++ ret = drm_ioctl(filp, cmd, arg); ++ ++ return ret; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_irq.c b/drivers/gpu/drm/i915_legacy/i915_irq.c +new file mode 100644 +index 000000000000..b92cfd69134b +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_irq.c +@@ -0,0 +1,4925 @@ ++/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*- ++ */ ++/* ++ * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. ++ * All Rights Reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the ++ * "Software"), to deal in the Software without restriction, including ++ * without limitation the rights to use, copy, modify, merge, publish, ++ * distribute, sub license, and/or sell copies of the Software, and to ++ * permit persons to whom the Software is furnished to do so, subject to ++ * the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the ++ * next paragraph) shall be included in all copies or substantial portions ++ * of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS ++ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF ++ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. ++ * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR ++ * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, ++ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE ++ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ++ * ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++ ++#include "i915_drv.h" ++#include "i915_trace.h" ++#include "intel_drv.h" ++#include "intel_psr.h" ++ ++/** ++ * DOC: interrupt handling ++ * ++ * These functions provide the basic support for enabling and disabling the ++ * interrupt handling support. There's a lot more functionality in i915_irq.c ++ * and related files, but that will be described in separate chapters. ++ */ ++ ++static const u32 hpd_ilk[HPD_NUM_PINS] = { ++ [HPD_PORT_A] = DE_DP_A_HOTPLUG, ++}; ++ ++static const u32 hpd_ivb[HPD_NUM_PINS] = { ++ [HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB, ++}; ++ ++static const u32 hpd_bdw[HPD_NUM_PINS] = { ++ [HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG, ++}; ++ ++static const u32 hpd_ibx[HPD_NUM_PINS] = { ++ [HPD_CRT] = SDE_CRT_HOTPLUG, ++ [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG, ++ [HPD_PORT_B] = SDE_PORTB_HOTPLUG, ++ [HPD_PORT_C] = SDE_PORTC_HOTPLUG, ++ [HPD_PORT_D] = SDE_PORTD_HOTPLUG ++}; ++ ++static const u32 hpd_cpt[HPD_NUM_PINS] = { ++ [HPD_CRT] = SDE_CRT_HOTPLUG_CPT, ++ [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT, ++ [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT, ++ [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT, ++ [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT ++}; ++ ++static const u32 hpd_spt[HPD_NUM_PINS] = { ++ [HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT, ++ [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT, ++ [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT, ++ [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT, ++ [HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT ++}; ++ ++static const u32 hpd_mask_i915[HPD_NUM_PINS] = { ++ [HPD_CRT] = CRT_HOTPLUG_INT_EN, ++ [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN, ++ [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN, ++ [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN, ++ [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN, ++ [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN ++}; ++ ++static const u32 hpd_status_g4x[HPD_NUM_PINS] = { ++ [HPD_CRT] = CRT_HOTPLUG_INT_STATUS, ++ [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X, ++ [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X, ++ [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS, ++ [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS, ++ [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS ++}; ++ ++static const u32 hpd_status_i915[HPD_NUM_PINS] = { ++ [HPD_CRT] = CRT_HOTPLUG_INT_STATUS, ++ [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915, ++ [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915, ++ [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS, ++ [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS, ++ [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS ++}; ++ ++/* BXT hpd list */ ++static const u32 hpd_bxt[HPD_NUM_PINS] = { ++ [HPD_PORT_A] = BXT_DE_PORT_HP_DDIA, ++ [HPD_PORT_B] = BXT_DE_PORT_HP_DDIB, ++ [HPD_PORT_C] = BXT_DE_PORT_HP_DDIC ++}; ++ ++static const u32 hpd_gen11[HPD_NUM_PINS] = { ++ [HPD_PORT_C] = GEN11_TC1_HOTPLUG | GEN11_TBT1_HOTPLUG, ++ [HPD_PORT_D] = GEN11_TC2_HOTPLUG | GEN11_TBT2_HOTPLUG, ++ [HPD_PORT_E] = GEN11_TC3_HOTPLUG | GEN11_TBT3_HOTPLUG, ++ [HPD_PORT_F] = GEN11_TC4_HOTPLUG | GEN11_TBT4_HOTPLUG ++}; ++ ++static const u32 hpd_icp[HPD_NUM_PINS] = { ++ [HPD_PORT_A] = SDE_DDIA_HOTPLUG_ICP, ++ [HPD_PORT_B] = SDE_DDIB_HOTPLUG_ICP, ++ [HPD_PORT_C] = SDE_TC1_HOTPLUG_ICP, ++ [HPD_PORT_D] = SDE_TC2_HOTPLUG_ICP, ++ [HPD_PORT_E] = SDE_TC3_HOTPLUG_ICP, ++ [HPD_PORT_F] = SDE_TC4_HOTPLUG_ICP ++}; ++ ++static void gen3_irq_reset(struct intel_uncore *uncore, i915_reg_t imr, ++ i915_reg_t iir, i915_reg_t ier) ++{ ++ intel_uncore_write(uncore, imr, 0xffffffff); ++ intel_uncore_posting_read(uncore, imr); ++ ++ intel_uncore_write(uncore, ier, 0); ++ ++ /* IIR can theoretically queue up two events. Be paranoid. */ ++ intel_uncore_write(uncore, iir, 0xffffffff); ++ intel_uncore_posting_read(uncore, iir); ++ intel_uncore_write(uncore, iir, 0xffffffff); ++ intel_uncore_posting_read(uncore, iir); ++} ++ ++static void gen2_irq_reset(struct intel_uncore *uncore) ++{ ++ intel_uncore_write16(uncore, GEN2_IMR, 0xffff); ++ intel_uncore_posting_read16(uncore, GEN2_IMR); ++ ++ intel_uncore_write16(uncore, GEN2_IER, 0); ++ ++ /* IIR can theoretically queue up two events. Be paranoid. */ ++ intel_uncore_write16(uncore, GEN2_IIR, 0xffff); ++ intel_uncore_posting_read16(uncore, GEN2_IIR); ++ intel_uncore_write16(uncore, GEN2_IIR, 0xffff); ++ intel_uncore_posting_read16(uncore, GEN2_IIR); ++} ++ ++#define GEN8_IRQ_RESET_NDX(uncore, type, which) \ ++({ \ ++ unsigned int which_ = which; \ ++ gen3_irq_reset((uncore), GEN8_##type##_IMR(which_), \ ++ GEN8_##type##_IIR(which_), GEN8_##type##_IER(which_)); \ ++}) ++ ++#define GEN3_IRQ_RESET(uncore, type) \ ++ gen3_irq_reset((uncore), type##IMR, type##IIR, type##IER) ++ ++#define GEN2_IRQ_RESET(uncore) \ ++ gen2_irq_reset(uncore) ++ ++/* ++ * We should clear IMR at preinstall/uninstall, and just check at postinstall. ++ */ ++static void gen3_assert_iir_is_zero(struct intel_uncore *uncore, i915_reg_t reg) ++{ ++ u32 val = intel_uncore_read(uncore, reg); ++ ++ if (val == 0) ++ return; ++ ++ WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n", ++ i915_mmio_reg_offset(reg), val); ++ intel_uncore_write(uncore, reg, 0xffffffff); ++ intel_uncore_posting_read(uncore, reg); ++ intel_uncore_write(uncore, reg, 0xffffffff); ++ intel_uncore_posting_read(uncore, reg); ++} ++ ++static void gen2_assert_iir_is_zero(struct intel_uncore *uncore) ++{ ++ u16 val = intel_uncore_read16(uncore, GEN2_IIR); ++ ++ if (val == 0) ++ return; ++ ++ WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n", ++ i915_mmio_reg_offset(GEN2_IIR), val); ++ intel_uncore_write16(uncore, GEN2_IIR, 0xffff); ++ intel_uncore_posting_read16(uncore, GEN2_IIR); ++ intel_uncore_write16(uncore, GEN2_IIR, 0xffff); ++ intel_uncore_posting_read16(uncore, GEN2_IIR); ++} ++ ++static void gen3_irq_init(struct intel_uncore *uncore, ++ i915_reg_t imr, u32 imr_val, ++ i915_reg_t ier, u32 ier_val, ++ i915_reg_t iir) ++{ ++ gen3_assert_iir_is_zero(uncore, iir); ++ ++ intel_uncore_write(uncore, ier, ier_val); ++ intel_uncore_write(uncore, imr, imr_val); ++ intel_uncore_posting_read(uncore, imr); ++} ++ ++static void gen2_irq_init(struct intel_uncore *uncore, ++ u32 imr_val, u32 ier_val) ++{ ++ gen2_assert_iir_is_zero(uncore); ++ ++ intel_uncore_write16(uncore, GEN2_IER, ier_val); ++ intel_uncore_write16(uncore, GEN2_IMR, imr_val); ++ intel_uncore_posting_read16(uncore, GEN2_IMR); ++} ++ ++#define GEN8_IRQ_INIT_NDX(uncore, type, which, imr_val, ier_val) \ ++({ \ ++ unsigned int which_ = which; \ ++ gen3_irq_init((uncore), \ ++ GEN8_##type##_IMR(which_), imr_val, \ ++ GEN8_##type##_IER(which_), ier_val, \ ++ GEN8_##type##_IIR(which_)); \ ++}) ++ ++#define GEN3_IRQ_INIT(uncore, type, imr_val, ier_val) \ ++ gen3_irq_init((uncore), \ ++ type##IMR, imr_val, \ ++ type##IER, ier_val, \ ++ type##IIR) ++ ++#define GEN2_IRQ_INIT(uncore, imr_val, ier_val) \ ++ gen2_irq_init((uncore), imr_val, ier_val) ++ ++static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir); ++static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir); ++ ++/* For display hotplug interrupt */ ++static inline void ++i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv, ++ u32 mask, ++ u32 bits) ++{ ++ u32 val; ++ ++ lockdep_assert_held(&dev_priv->irq_lock); ++ WARN_ON(bits & ~mask); ++ ++ val = I915_READ(PORT_HOTPLUG_EN); ++ val &= ~mask; ++ val |= bits; ++ I915_WRITE(PORT_HOTPLUG_EN, val); ++} ++ ++/** ++ * i915_hotplug_interrupt_update - update hotplug interrupt enable ++ * @dev_priv: driver private ++ * @mask: bits to update ++ * @bits: bits to enable ++ * NOTE: the HPD enable bits are modified both inside and outside ++ * of an interrupt context. To avoid that read-modify-write cycles ++ * interfer, these bits are protected by a spinlock. Since this ++ * function is usually not called from a context where the lock is ++ * held already, this function acquires the lock itself. A non-locking ++ * version is also available. ++ */ ++void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv, ++ u32 mask, ++ u32 bits) ++{ ++ spin_lock_irq(&dev_priv->irq_lock); ++ i915_hotplug_interrupt_update_locked(dev_priv, mask, bits); ++ spin_unlock_irq(&dev_priv->irq_lock); ++} ++ ++static u32 ++gen11_gt_engine_identity(struct drm_i915_private * const i915, ++ const unsigned int bank, const unsigned int bit); ++ ++static bool gen11_reset_one_iir(struct drm_i915_private * const i915, ++ const unsigned int bank, ++ const unsigned int bit) ++{ ++ void __iomem * const regs = i915->uncore.regs; ++ u32 dw; ++ ++ lockdep_assert_held(&i915->irq_lock); ++ ++ dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank)); ++ if (dw & BIT(bit)) { ++ /* ++ * According to the BSpec, DW_IIR bits cannot be cleared without ++ * first servicing the Selector & Shared IIR registers. ++ */ ++ gen11_gt_engine_identity(i915, bank, bit); ++ ++ /* ++ * We locked GT INT DW by reading it. If we want to (try ++ * to) recover from this succesfully, we need to clear ++ * our bit, otherwise we are locking the register for ++ * everybody. ++ */ ++ raw_reg_write(regs, GEN11_GT_INTR_DW(bank), BIT(bit)); ++ ++ return true; ++ } ++ ++ return false; ++} ++ ++/** ++ * ilk_update_display_irq - update DEIMR ++ * @dev_priv: driver private ++ * @interrupt_mask: mask of interrupt bits to update ++ * @enabled_irq_mask: mask of interrupt bits to enable ++ */ ++void ilk_update_display_irq(struct drm_i915_private *dev_priv, ++ u32 interrupt_mask, ++ u32 enabled_irq_mask) ++{ ++ u32 new_val; ++ ++ lockdep_assert_held(&dev_priv->irq_lock); ++ ++ WARN_ON(enabled_irq_mask & ~interrupt_mask); ++ ++ if (WARN_ON(!intel_irqs_enabled(dev_priv))) ++ return; ++ ++ new_val = dev_priv->irq_mask; ++ new_val &= ~interrupt_mask; ++ new_val |= (~enabled_irq_mask & interrupt_mask); ++ ++ if (new_val != dev_priv->irq_mask) { ++ dev_priv->irq_mask = new_val; ++ I915_WRITE(DEIMR, dev_priv->irq_mask); ++ POSTING_READ(DEIMR); ++ } ++} ++ ++/** ++ * ilk_update_gt_irq - update GTIMR ++ * @dev_priv: driver private ++ * @interrupt_mask: mask of interrupt bits to update ++ * @enabled_irq_mask: mask of interrupt bits to enable ++ */ ++static void ilk_update_gt_irq(struct drm_i915_private *dev_priv, ++ u32 interrupt_mask, ++ u32 enabled_irq_mask) ++{ ++ lockdep_assert_held(&dev_priv->irq_lock); ++ ++ WARN_ON(enabled_irq_mask & ~interrupt_mask); ++ ++ if (WARN_ON(!intel_irqs_enabled(dev_priv))) ++ return; ++ ++ dev_priv->gt_irq_mask &= ~interrupt_mask; ++ dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask); ++ I915_WRITE(GTIMR, dev_priv->gt_irq_mask); ++} ++ ++void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, u32 mask) ++{ ++ ilk_update_gt_irq(dev_priv, mask, mask); ++ POSTING_READ_FW(GTIMR); ++} ++ ++void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, u32 mask) ++{ ++ ilk_update_gt_irq(dev_priv, mask, 0); ++} ++ ++static i915_reg_t gen6_pm_iir(struct drm_i915_private *dev_priv) ++{ ++ WARN_ON_ONCE(INTEL_GEN(dev_priv) >= 11); ++ ++ return INTEL_GEN(dev_priv) >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR; ++} ++ ++static void write_pm_imr(struct drm_i915_private *dev_priv) ++{ ++ i915_reg_t reg; ++ u32 mask = dev_priv->pm_imr; ++ ++ if (INTEL_GEN(dev_priv) >= 11) { ++ reg = GEN11_GPM_WGBOXPERF_INTR_MASK; ++ /* pm is in upper half */ ++ mask = mask << 16; ++ } else if (INTEL_GEN(dev_priv) >= 8) { ++ reg = GEN8_GT_IMR(2); ++ } else { ++ reg = GEN6_PMIMR; ++ } ++ ++ I915_WRITE(reg, mask); ++ POSTING_READ(reg); ++} ++ ++static void write_pm_ier(struct drm_i915_private *dev_priv) ++{ ++ i915_reg_t reg; ++ u32 mask = dev_priv->pm_ier; ++ ++ if (INTEL_GEN(dev_priv) >= 11) { ++ reg = GEN11_GPM_WGBOXPERF_INTR_ENABLE; ++ /* pm is in upper half */ ++ mask = mask << 16; ++ } else if (INTEL_GEN(dev_priv) >= 8) { ++ reg = GEN8_GT_IER(2); ++ } else { ++ reg = GEN6_PMIER; ++ } ++ ++ I915_WRITE(reg, mask); ++} ++ ++/** ++ * snb_update_pm_irq - update GEN6_PMIMR ++ * @dev_priv: driver private ++ * @interrupt_mask: mask of interrupt bits to update ++ * @enabled_irq_mask: mask of interrupt bits to enable ++ */ ++static void snb_update_pm_irq(struct drm_i915_private *dev_priv, ++ u32 interrupt_mask, ++ u32 enabled_irq_mask) ++{ ++ u32 new_val; ++ ++ WARN_ON(enabled_irq_mask & ~interrupt_mask); ++ ++ lockdep_assert_held(&dev_priv->irq_lock); ++ ++ new_val = dev_priv->pm_imr; ++ new_val &= ~interrupt_mask; ++ new_val |= (~enabled_irq_mask & interrupt_mask); ++ ++ if (new_val != dev_priv->pm_imr) { ++ dev_priv->pm_imr = new_val; ++ write_pm_imr(dev_priv); ++ } ++} ++ ++void gen6_unmask_pm_irq(struct drm_i915_private *dev_priv, u32 mask) ++{ ++ if (WARN_ON(!intel_irqs_enabled(dev_priv))) ++ return; ++ ++ snb_update_pm_irq(dev_priv, mask, mask); ++} ++ ++static void __gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask) ++{ ++ snb_update_pm_irq(dev_priv, mask, 0); ++} ++ ++void gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask) ++{ ++ if (WARN_ON(!intel_irqs_enabled(dev_priv))) ++ return; ++ ++ __gen6_mask_pm_irq(dev_priv, mask); ++} ++ ++static void gen6_reset_pm_iir(struct drm_i915_private *dev_priv, u32 reset_mask) ++{ ++ i915_reg_t reg = gen6_pm_iir(dev_priv); ++ ++ lockdep_assert_held(&dev_priv->irq_lock); ++ ++ I915_WRITE(reg, reset_mask); ++ I915_WRITE(reg, reset_mask); ++ POSTING_READ(reg); ++} ++ ++static void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, u32 enable_mask) ++{ ++ lockdep_assert_held(&dev_priv->irq_lock); ++ ++ dev_priv->pm_ier |= enable_mask; ++ write_pm_ier(dev_priv); ++ gen6_unmask_pm_irq(dev_priv, enable_mask); ++ /* unmask_pm_irq provides an implicit barrier (POSTING_READ) */ ++} ++ ++static void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, u32 disable_mask) ++{ ++ lockdep_assert_held(&dev_priv->irq_lock); ++ ++ dev_priv->pm_ier &= ~disable_mask; ++ __gen6_mask_pm_irq(dev_priv, disable_mask); ++ write_pm_ier(dev_priv); ++ /* though a barrier is missing here, but don't really need a one */ ++} ++ ++void gen11_reset_rps_interrupts(struct drm_i915_private *dev_priv) ++{ ++ spin_lock_irq(&dev_priv->irq_lock); ++ ++ while (gen11_reset_one_iir(dev_priv, 0, GEN11_GTPM)) ++ ; ++ ++ dev_priv->gt_pm.rps.pm_iir = 0; ++ ++ spin_unlock_irq(&dev_priv->irq_lock); ++} ++ ++void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv) ++{ ++ spin_lock_irq(&dev_priv->irq_lock); ++ gen6_reset_pm_iir(dev_priv, GEN6_PM_RPS_EVENTS); ++ dev_priv->gt_pm.rps.pm_iir = 0; ++ spin_unlock_irq(&dev_priv->irq_lock); ++} ++ ++void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv) ++{ ++ struct intel_rps *rps = &dev_priv->gt_pm.rps; ++ ++ if (READ_ONCE(rps->interrupts_enabled)) ++ return; ++ ++ spin_lock_irq(&dev_priv->irq_lock); ++ WARN_ON_ONCE(rps->pm_iir); ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ WARN_ON_ONCE(gen11_reset_one_iir(dev_priv, 0, GEN11_GTPM)); ++ else ++ WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events); ++ ++ rps->interrupts_enabled = true; ++ gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events); ++ ++ spin_unlock_irq(&dev_priv->irq_lock); ++} ++ ++void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv) ++{ ++ struct intel_rps *rps = &dev_priv->gt_pm.rps; ++ ++ if (!READ_ONCE(rps->interrupts_enabled)) ++ return; ++ ++ spin_lock_irq(&dev_priv->irq_lock); ++ rps->interrupts_enabled = false; ++ ++ I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0u)); ++ ++ gen6_disable_pm_irq(dev_priv, GEN6_PM_RPS_EVENTS); ++ ++ spin_unlock_irq(&dev_priv->irq_lock); ++ synchronize_irq(dev_priv->drm.irq); ++ ++ /* Now that we will not be generating any more work, flush any ++ * outstanding tasks. As we are called on the RPS idle path, ++ * we will reset the GPU to minimum frequencies, so the current ++ * state of the worker can be discarded. ++ */ ++ cancel_work_sync(&rps->work); ++ if (INTEL_GEN(dev_priv) >= 11) ++ gen11_reset_rps_interrupts(dev_priv); ++ else ++ gen6_reset_rps_interrupts(dev_priv); ++} ++ ++void gen9_reset_guc_interrupts(struct drm_i915_private *dev_priv) ++{ ++ assert_rpm_wakelock_held(dev_priv); ++ ++ spin_lock_irq(&dev_priv->irq_lock); ++ gen6_reset_pm_iir(dev_priv, dev_priv->pm_guc_events); ++ spin_unlock_irq(&dev_priv->irq_lock); ++} ++ ++void gen9_enable_guc_interrupts(struct drm_i915_private *dev_priv) ++{ ++ assert_rpm_wakelock_held(dev_priv); ++ ++ spin_lock_irq(&dev_priv->irq_lock); ++ if (!dev_priv->guc.interrupts_enabled) { ++ WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & ++ dev_priv->pm_guc_events); ++ dev_priv->guc.interrupts_enabled = true; ++ gen6_enable_pm_irq(dev_priv, dev_priv->pm_guc_events); ++ } ++ spin_unlock_irq(&dev_priv->irq_lock); ++} ++ ++void gen9_disable_guc_interrupts(struct drm_i915_private *dev_priv) ++{ ++ assert_rpm_wakelock_held(dev_priv); ++ ++ spin_lock_irq(&dev_priv->irq_lock); ++ dev_priv->guc.interrupts_enabled = false; ++ ++ gen6_disable_pm_irq(dev_priv, dev_priv->pm_guc_events); ++ ++ spin_unlock_irq(&dev_priv->irq_lock); ++ synchronize_irq(dev_priv->drm.irq); ++ ++ gen9_reset_guc_interrupts(dev_priv); ++} ++ ++/** ++ * bdw_update_port_irq - update DE port interrupt ++ * @dev_priv: driver private ++ * @interrupt_mask: mask of interrupt bits to update ++ * @enabled_irq_mask: mask of interrupt bits to enable ++ */ ++static void bdw_update_port_irq(struct drm_i915_private *dev_priv, ++ u32 interrupt_mask, ++ u32 enabled_irq_mask) ++{ ++ u32 new_val; ++ u32 old_val; ++ ++ lockdep_assert_held(&dev_priv->irq_lock); ++ ++ WARN_ON(enabled_irq_mask & ~interrupt_mask); ++ ++ if (WARN_ON(!intel_irqs_enabled(dev_priv))) ++ return; ++ ++ old_val = I915_READ(GEN8_DE_PORT_IMR); ++ ++ new_val = old_val; ++ new_val &= ~interrupt_mask; ++ new_val |= (~enabled_irq_mask & interrupt_mask); ++ ++ if (new_val != old_val) { ++ I915_WRITE(GEN8_DE_PORT_IMR, new_val); ++ POSTING_READ(GEN8_DE_PORT_IMR); ++ } ++} ++ ++/** ++ * bdw_update_pipe_irq - update DE pipe interrupt ++ * @dev_priv: driver private ++ * @pipe: pipe whose interrupt to update ++ * @interrupt_mask: mask of interrupt bits to update ++ * @enabled_irq_mask: mask of interrupt bits to enable ++ */ ++void bdw_update_pipe_irq(struct drm_i915_private *dev_priv, ++ enum pipe pipe, ++ u32 interrupt_mask, ++ u32 enabled_irq_mask) ++{ ++ u32 new_val; ++ ++ lockdep_assert_held(&dev_priv->irq_lock); ++ ++ WARN_ON(enabled_irq_mask & ~interrupt_mask); ++ ++ if (WARN_ON(!intel_irqs_enabled(dev_priv))) ++ return; ++ ++ new_val = dev_priv->de_irq_mask[pipe]; ++ new_val &= ~interrupt_mask; ++ new_val |= (~enabled_irq_mask & interrupt_mask); ++ ++ if (new_val != dev_priv->de_irq_mask[pipe]) { ++ dev_priv->de_irq_mask[pipe] = new_val; ++ I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]); ++ POSTING_READ(GEN8_DE_PIPE_IMR(pipe)); ++ } ++} ++ ++/** ++ * ibx_display_interrupt_update - update SDEIMR ++ * @dev_priv: driver private ++ * @interrupt_mask: mask of interrupt bits to update ++ * @enabled_irq_mask: mask of interrupt bits to enable ++ */ ++void ibx_display_interrupt_update(struct drm_i915_private *dev_priv, ++ u32 interrupt_mask, ++ u32 enabled_irq_mask) ++{ ++ u32 sdeimr = I915_READ(SDEIMR); ++ sdeimr &= ~interrupt_mask; ++ sdeimr |= (~enabled_irq_mask & interrupt_mask); ++ ++ WARN_ON(enabled_irq_mask & ~interrupt_mask); ++ ++ lockdep_assert_held(&dev_priv->irq_lock); ++ ++ if (WARN_ON(!intel_irqs_enabled(dev_priv))) ++ return; ++ ++ I915_WRITE(SDEIMR, sdeimr); ++ POSTING_READ(SDEIMR); ++} ++ ++u32 i915_pipestat_enable_mask(struct drm_i915_private *dev_priv, ++ enum pipe pipe) ++{ ++ u32 status_mask = dev_priv->pipestat_irq_mask[pipe]; ++ u32 enable_mask = status_mask << 16; ++ ++ lockdep_assert_held(&dev_priv->irq_lock); ++ ++ if (INTEL_GEN(dev_priv) < 5) ++ goto out; ++ ++ /* ++ * On pipe A we don't support the PSR interrupt yet, ++ * on pipe B and C the same bit MBZ. ++ */ ++ if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV)) ++ return 0; ++ /* ++ * On pipe B and C we don't support the PSR interrupt yet, on pipe ++ * A the same bit is for perf counters which we don't use either. ++ */ ++ if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV)) ++ return 0; ++ ++ enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS | ++ SPRITE0_FLIP_DONE_INT_EN_VLV | ++ SPRITE1_FLIP_DONE_INT_EN_VLV); ++ if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV) ++ enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV; ++ if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV) ++ enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV; ++ ++out: ++ WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK || ++ status_mask & ~PIPESTAT_INT_STATUS_MASK, ++ "pipe %c: enable_mask=0x%x, status_mask=0x%x\n", ++ pipe_name(pipe), enable_mask, status_mask); ++ ++ return enable_mask; ++} ++ ++void i915_enable_pipestat(struct drm_i915_private *dev_priv, ++ enum pipe pipe, u32 status_mask) ++{ ++ i915_reg_t reg = PIPESTAT(pipe); ++ u32 enable_mask; ++ ++ WARN_ONCE(status_mask & ~PIPESTAT_INT_STATUS_MASK, ++ "pipe %c: status_mask=0x%x\n", ++ pipe_name(pipe), status_mask); ++ ++ lockdep_assert_held(&dev_priv->irq_lock); ++ WARN_ON(!intel_irqs_enabled(dev_priv)); ++ ++ if ((dev_priv->pipestat_irq_mask[pipe] & status_mask) == status_mask) ++ return; ++ ++ dev_priv->pipestat_irq_mask[pipe] |= status_mask; ++ enable_mask = i915_pipestat_enable_mask(dev_priv, pipe); ++ ++ I915_WRITE(reg, enable_mask | status_mask); ++ POSTING_READ(reg); ++} ++ ++void i915_disable_pipestat(struct drm_i915_private *dev_priv, ++ enum pipe pipe, u32 status_mask) ++{ ++ i915_reg_t reg = PIPESTAT(pipe); ++ u32 enable_mask; ++ ++ WARN_ONCE(status_mask & ~PIPESTAT_INT_STATUS_MASK, ++ "pipe %c: status_mask=0x%x\n", ++ pipe_name(pipe), status_mask); ++ ++ lockdep_assert_held(&dev_priv->irq_lock); ++ WARN_ON(!intel_irqs_enabled(dev_priv)); ++ ++ if ((dev_priv->pipestat_irq_mask[pipe] & status_mask) == 0) ++ return; ++ ++ dev_priv->pipestat_irq_mask[pipe] &= ~status_mask; ++ enable_mask = i915_pipestat_enable_mask(dev_priv, pipe); ++ ++ I915_WRITE(reg, enable_mask | status_mask); ++ POSTING_READ(reg); ++} ++ ++static bool i915_has_asle(struct drm_i915_private *dev_priv) ++{ ++ if (!dev_priv->opregion.asle) ++ return false; ++ ++ return IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv); ++} ++ ++/** ++ * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion ++ * @dev_priv: i915 device private ++ */ ++static void i915_enable_asle_pipestat(struct drm_i915_private *dev_priv) ++{ ++ if (!i915_has_asle(dev_priv)) ++ return; ++ ++ spin_lock_irq(&dev_priv->irq_lock); ++ ++ i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS); ++ if (INTEL_GEN(dev_priv) >= 4) ++ i915_enable_pipestat(dev_priv, PIPE_A, ++ PIPE_LEGACY_BLC_EVENT_STATUS); ++ ++ spin_unlock_irq(&dev_priv->irq_lock); ++} ++ ++/* ++ * This timing diagram depicts the video signal in and ++ * around the vertical blanking period. ++ * ++ * Assumptions about the fictitious mode used in this example: ++ * vblank_start >= 3 ++ * vsync_start = vblank_start + 1 ++ * vsync_end = vblank_start + 2 ++ * vtotal = vblank_start + 3 ++ * ++ * start of vblank: ++ * latch double buffered registers ++ * increment frame counter (ctg+) ++ * generate start of vblank interrupt (gen4+) ++ * | ++ * | frame start: ++ * | generate frame start interrupt (aka. vblank interrupt) (gmch) ++ * | may be shifted forward 1-3 extra lines via PIPECONF ++ * | | ++ * | | start of vsync: ++ * | | generate vsync interrupt ++ * | | | ++ * ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx ++ * . \hs/ . \hs/ \hs/ \hs/ . \hs/ ++ * ----va---> <-----------------vb--------------------> <--------va------------- ++ * | | <----vs-----> | ++ * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2) ++ * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+) ++ * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi) ++ * | | | ++ * last visible pixel first visible pixel ++ * | increment frame counter (gen3/4) ++ * pixel counter = vblank_start * htotal pixel counter = 0 (gen3/4) ++ * ++ * x = horizontal active ++ * _ = horizontal blanking ++ * hs = horizontal sync ++ * va = vertical active ++ * vb = vertical blanking ++ * vs = vertical sync ++ * vbs = vblank_start (number) ++ * ++ * Summary: ++ * - most events happen at the start of horizontal sync ++ * - frame start happens at the start of horizontal blank, 1-4 lines ++ * (depending on PIPECONF settings) after the start of vblank ++ * - gen3/4 pixel and frame counter are synchronized with the start ++ * of horizontal active on the first line of vertical active ++ */ ++ ++/* Called from drm generic code, passed a 'crtc', which ++ * we use as a pipe index ++ */ ++static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; ++ const struct drm_display_mode *mode = &vblank->hwmode; ++ i915_reg_t high_frame, low_frame; ++ u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal; ++ unsigned long irqflags; ++ ++ /* ++ * On i965gm TV output the frame counter only works up to ++ * the point when we enable the TV encoder. After that the ++ * frame counter ceases to work and reads zero. We need a ++ * vblank wait before enabling the TV encoder and so we ++ * have to enable vblank interrupts while the frame counter ++ * is still in a working state. However the core vblank code ++ * does not like us returning non-zero frame counter values ++ * when we've told it that we don't have a working frame ++ * counter. Thus we must stop non-zero values leaking out. ++ */ ++ if (!vblank->max_vblank_count) ++ return 0; ++ ++ htotal = mode->crtc_htotal; ++ hsync_start = mode->crtc_hsync_start; ++ vbl_start = mode->crtc_vblank_start; ++ if (mode->flags & DRM_MODE_FLAG_INTERLACE) ++ vbl_start = DIV_ROUND_UP(vbl_start, 2); ++ ++ /* Convert to pixel count */ ++ vbl_start *= htotal; ++ ++ /* Start of vblank event occurs at start of hsync */ ++ vbl_start -= htotal - hsync_start; ++ ++ high_frame = PIPEFRAME(pipe); ++ low_frame = PIPEFRAMEPIXEL(pipe); ++ ++ spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); ++ ++ /* ++ * High & low register fields aren't synchronized, so make sure ++ * we get a low value that's stable across two reads of the high ++ * register. ++ */ ++ do { ++ high1 = I915_READ_FW(high_frame) & PIPE_FRAME_HIGH_MASK; ++ low = I915_READ_FW(low_frame); ++ high2 = I915_READ_FW(high_frame) & PIPE_FRAME_HIGH_MASK; ++ } while (high1 != high2); ++ ++ spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); ++ ++ high1 >>= PIPE_FRAME_HIGH_SHIFT; ++ pixel = low & PIPE_PIXEL_MASK; ++ low >>= PIPE_FRAME_LOW_SHIFT; ++ ++ /* ++ * The frame counter increments at beginning of active. ++ * Cook up a vblank counter by also checking the pixel ++ * counter against vblank start. ++ */ ++ return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff; ++} ++ ++static u32 g4x_get_vblank_counter(struct drm_device *dev, unsigned int pipe) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ ++ return I915_READ(PIPE_FRMCOUNT_G4X(pipe)); ++} ++ ++/* ++ * On certain encoders on certain platforms, pipe ++ * scanline register will not work to get the scanline, ++ * since the timings are driven from the PORT or issues ++ * with scanline register updates. ++ * This function will use Framestamp and current ++ * timestamp registers to calculate the scanline. ++ */ ++static u32 __intel_get_crtc_scanline_from_timestamp(struct intel_crtc *crtc) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ struct drm_vblank_crtc *vblank = ++ &crtc->base.dev->vblank[drm_crtc_index(&crtc->base)]; ++ const struct drm_display_mode *mode = &vblank->hwmode; ++ u32 vblank_start = mode->crtc_vblank_start; ++ u32 vtotal = mode->crtc_vtotal; ++ u32 htotal = mode->crtc_htotal; ++ u32 clock = mode->crtc_clock; ++ u32 scanline, scan_prev_time, scan_curr_time, scan_post_time; ++ ++ /* ++ * To avoid the race condition where we might cross into the ++ * next vblank just between the PIPE_FRMTMSTMP and TIMESTAMP_CTR ++ * reads. We make sure we read PIPE_FRMTMSTMP and TIMESTAMP_CTR ++ * during the same frame. ++ */ ++ do { ++ /* ++ * This field provides read back of the display ++ * pipe frame time stamp. The time stamp value ++ * is sampled at every start of vertical blank. ++ */ ++ scan_prev_time = I915_READ_FW(PIPE_FRMTMSTMP(crtc->pipe)); ++ ++ /* ++ * The TIMESTAMP_CTR register has the current ++ * time stamp value. ++ */ ++ scan_curr_time = I915_READ_FW(IVB_TIMESTAMP_CTR); ++ ++ scan_post_time = I915_READ_FW(PIPE_FRMTMSTMP(crtc->pipe)); ++ } while (scan_post_time != scan_prev_time); ++ ++ scanline = div_u64(mul_u32_u32(scan_curr_time - scan_prev_time, ++ clock), 1000 * htotal); ++ scanline = min(scanline, vtotal - 1); ++ scanline = (scanline + vblank_start) % vtotal; ++ ++ return scanline; ++} ++ ++/* I915_READ_FW, only for fast reads of display block, no need for forcewake etc. */ ++static int __intel_get_crtc_scanline(struct intel_crtc *crtc) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ const struct drm_display_mode *mode; ++ struct drm_vblank_crtc *vblank; ++ enum pipe pipe = crtc->pipe; ++ int position, vtotal; ++ ++ if (!crtc->active) ++ return -1; ++ ++ vblank = &crtc->base.dev->vblank[drm_crtc_index(&crtc->base)]; ++ mode = &vblank->hwmode; ++ ++ if (mode->private_flags & I915_MODE_FLAG_GET_SCANLINE_FROM_TIMESTAMP) ++ return __intel_get_crtc_scanline_from_timestamp(crtc); ++ ++ vtotal = mode->crtc_vtotal; ++ if (mode->flags & DRM_MODE_FLAG_INTERLACE) ++ vtotal /= 2; ++ ++ if (IS_GEN(dev_priv, 2)) ++ position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN2; ++ else ++ position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3; ++ ++ /* ++ * On HSW, the DSL reg (0x70000) appears to return 0 if we ++ * read it just before the start of vblank. So try it again ++ * so we don't accidentally end up spanning a vblank frame ++ * increment, causing the pipe_update_end() code to squak at us. ++ * ++ * The nature of this problem means we can't simply check the ISR ++ * bit and return the vblank start value; nor can we use the scanline ++ * debug register in the transcoder as it appears to have the same ++ * problem. We may need to extend this to include other platforms, ++ * but so far testing only shows the problem on HSW. ++ */ ++ if (HAS_DDI(dev_priv) && !position) { ++ int i, temp; ++ ++ for (i = 0; i < 100; i++) { ++ udelay(1); ++ temp = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3; ++ if (temp != position) { ++ position = temp; ++ break; ++ } ++ } ++ } ++ ++ /* ++ * See update_scanline_offset() for the details on the ++ * scanline_offset adjustment. ++ */ ++ return (position + crtc->scanline_offset) % vtotal; ++} ++ ++static bool i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe, ++ bool in_vblank_irq, int *vpos, int *hpos, ++ ktime_t *stime, ktime_t *etime, ++ const struct drm_display_mode *mode) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv, ++ pipe); ++ int position; ++ int vbl_start, vbl_end, hsync_start, htotal, vtotal; ++ unsigned long irqflags; ++ bool use_scanline_counter = INTEL_GEN(dev_priv) >= 5 || ++ IS_G4X(dev_priv) || IS_GEN(dev_priv, 2) || ++ mode->private_flags & I915_MODE_FLAG_USE_SCANLINE_COUNTER; ++ ++ if (WARN_ON(!mode->crtc_clock)) { ++ DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled " ++ "pipe %c\n", pipe_name(pipe)); ++ return false; ++ } ++ ++ htotal = mode->crtc_htotal; ++ hsync_start = mode->crtc_hsync_start; ++ vtotal = mode->crtc_vtotal; ++ vbl_start = mode->crtc_vblank_start; ++ vbl_end = mode->crtc_vblank_end; ++ ++ if (mode->flags & DRM_MODE_FLAG_INTERLACE) { ++ vbl_start = DIV_ROUND_UP(vbl_start, 2); ++ vbl_end /= 2; ++ vtotal /= 2; ++ } ++ ++ /* ++ * Lock uncore.lock, as we will do multiple timing critical raw ++ * register reads, potentially with preemption disabled, so the ++ * following code must not block on uncore.lock. ++ */ ++ spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); ++ ++ /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */ ++ ++ /* Get optional system timestamp before query. */ ++ if (stime) ++ *stime = ktime_get(); ++ ++ if (use_scanline_counter) { ++ /* No obvious pixelcount register. Only query vertical ++ * scanout position from Display scan line register. ++ */ ++ position = __intel_get_crtc_scanline(intel_crtc); ++ } else { ++ /* Have access to pixelcount since start of frame. ++ * We can split this into vertical and horizontal ++ * scanout position. ++ */ ++ position = (I915_READ_FW(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT; ++ ++ /* convert to pixel counts */ ++ vbl_start *= htotal; ++ vbl_end *= htotal; ++ vtotal *= htotal; ++ ++ /* ++ * In interlaced modes, the pixel counter counts all pixels, ++ * so one field will have htotal more pixels. In order to avoid ++ * the reported position from jumping backwards when the pixel ++ * counter is beyond the length of the shorter field, just ++ * clamp the position the length of the shorter field. This ++ * matches how the scanline counter based position works since ++ * the scanline counter doesn't count the two half lines. ++ */ ++ if (position >= vtotal) ++ position = vtotal - 1; ++ ++ /* ++ * Start of vblank interrupt is triggered at start of hsync, ++ * just prior to the first active line of vblank. However we ++ * consider lines to start at the leading edge of horizontal ++ * active. So, should we get here before we've crossed into ++ * the horizontal active of the first line in vblank, we would ++ * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that, ++ * always add htotal-hsync_start to the current pixel position. ++ */ ++ position = (position + htotal - hsync_start) % vtotal; ++ } ++ ++ /* Get optional system timestamp after query. */ ++ if (etime) ++ *etime = ktime_get(); ++ ++ /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */ ++ ++ spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); ++ ++ /* ++ * While in vblank, position will be negative ++ * counting up towards 0 at vbl_end. And outside ++ * vblank, position will be positive counting ++ * up since vbl_end. ++ */ ++ if (position >= vbl_start) ++ position -= vbl_end; ++ else ++ position += vtotal - vbl_end; ++ ++ if (use_scanline_counter) { ++ *vpos = position; ++ *hpos = 0; ++ } else { ++ *vpos = position / htotal; ++ *hpos = position - (*vpos * htotal); ++ } ++ ++ return true; ++} ++ ++int intel_get_crtc_scanline(struct intel_crtc *crtc) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ unsigned long irqflags; ++ int position; ++ ++ spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); ++ position = __intel_get_crtc_scanline(crtc); ++ spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); ++ ++ return position; ++} ++ ++static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv) ++{ ++ u32 busy_up, busy_down, max_avg, min_avg; ++ u8 new_delay; ++ ++ spin_lock(&mchdev_lock); ++ ++ I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS)); ++ ++ new_delay = dev_priv->ips.cur_delay; ++ ++ I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG); ++ busy_up = I915_READ(RCPREVBSYTUPAVG); ++ busy_down = I915_READ(RCPREVBSYTDNAVG); ++ max_avg = I915_READ(RCBMAXAVG); ++ min_avg = I915_READ(RCBMINAVG); ++ ++ /* Handle RCS change request from hw */ ++ if (busy_up > max_avg) { ++ if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay) ++ new_delay = dev_priv->ips.cur_delay - 1; ++ if (new_delay < dev_priv->ips.max_delay) ++ new_delay = dev_priv->ips.max_delay; ++ } else if (busy_down < min_avg) { ++ if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay) ++ new_delay = dev_priv->ips.cur_delay + 1; ++ if (new_delay > dev_priv->ips.min_delay) ++ new_delay = dev_priv->ips.min_delay; ++ } ++ ++ if (ironlake_set_drps(dev_priv, new_delay)) ++ dev_priv->ips.cur_delay = new_delay; ++ ++ spin_unlock(&mchdev_lock); ++ ++ return; ++} ++ ++static void vlv_c0_read(struct drm_i915_private *dev_priv, ++ struct intel_rps_ei *ei) ++{ ++ ei->ktime = ktime_get_raw(); ++ ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT); ++ ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT); ++} ++ ++void gen6_rps_reset_ei(struct drm_i915_private *dev_priv) ++{ ++ memset(&dev_priv->gt_pm.rps.ei, 0, sizeof(dev_priv->gt_pm.rps.ei)); ++} ++ ++static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir) ++{ ++ struct intel_rps *rps = &dev_priv->gt_pm.rps; ++ const struct intel_rps_ei *prev = &rps->ei; ++ struct intel_rps_ei now; ++ u32 events = 0; ++ ++ if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0) ++ return 0; ++ ++ vlv_c0_read(dev_priv, &now); ++ ++ if (prev->ktime) { ++ u64 time, c0; ++ u32 render, media; ++ ++ time = ktime_us_delta(now.ktime, prev->ktime); ++ ++ time *= dev_priv->czclk_freq; ++ ++ /* Workload can be split between render + media, ++ * e.g. SwapBuffers being blitted in X after being rendered in ++ * mesa. To account for this we need to combine both engines ++ * into our activity counter. ++ */ ++ render = now.render_c0 - prev->render_c0; ++ media = now.media_c0 - prev->media_c0; ++ c0 = max(render, media); ++ c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */ ++ ++ if (c0 > time * rps->power.up_threshold) ++ events = GEN6_PM_RP_UP_THRESHOLD; ++ else if (c0 < time * rps->power.down_threshold) ++ events = GEN6_PM_RP_DOWN_THRESHOLD; ++ } ++ ++ rps->ei = now; ++ return events; ++} ++ ++static void gen6_pm_rps_work(struct work_struct *work) ++{ ++ struct drm_i915_private *dev_priv = ++ container_of(work, struct drm_i915_private, gt_pm.rps.work); ++ struct intel_rps *rps = &dev_priv->gt_pm.rps; ++ bool client_boost = false; ++ int new_delay, adj, min, max; ++ u32 pm_iir = 0; ++ ++ spin_lock_irq(&dev_priv->irq_lock); ++ if (rps->interrupts_enabled) { ++ pm_iir = fetch_and_zero(&rps->pm_iir); ++ client_boost = atomic_read(&rps->num_waiters); ++ } ++ spin_unlock_irq(&dev_priv->irq_lock); ++ ++ /* Make sure we didn't queue anything we're not going to process. */ ++ WARN_ON(pm_iir & ~dev_priv->pm_rps_events); ++ if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost) ++ goto out; ++ ++ mutex_lock(&dev_priv->pcu_lock); ++ ++ pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir); ++ ++ adj = rps->last_adj; ++ new_delay = rps->cur_freq; ++ min = rps->min_freq_softlimit; ++ max = rps->max_freq_softlimit; ++ if (client_boost) ++ max = rps->max_freq; ++ if (client_boost && new_delay < rps->boost_freq) { ++ new_delay = rps->boost_freq; ++ adj = 0; ++ } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) { ++ if (adj > 0) ++ adj *= 2; ++ else /* CHV needs even encode values */ ++ adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1; ++ ++ if (new_delay >= rps->max_freq_softlimit) ++ adj = 0; ++ } else if (client_boost) { ++ adj = 0; ++ } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) { ++ if (rps->cur_freq > rps->efficient_freq) ++ new_delay = rps->efficient_freq; ++ else if (rps->cur_freq > rps->min_freq_softlimit) ++ new_delay = rps->min_freq_softlimit; ++ adj = 0; ++ } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) { ++ if (adj < 0) ++ adj *= 2; ++ else /* CHV needs even encode values */ ++ adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1; ++ ++ if (new_delay <= rps->min_freq_softlimit) ++ adj = 0; ++ } else { /* unknown event */ ++ adj = 0; ++ } ++ ++ rps->last_adj = adj; ++ ++ /* ++ * Limit deboosting and boosting to keep ourselves at the extremes ++ * when in the respective power modes (i.e. slowly decrease frequencies ++ * while in the HIGH_POWER zone and slowly increase frequencies while ++ * in the LOW_POWER zone). On idle, we will hit the timeout and drop ++ * to the next level quickly, and conversely if busy we expect to ++ * hit a waitboost and rapidly switch into max power. ++ */ ++ if ((adj < 0 && rps->power.mode == HIGH_POWER) || ++ (adj > 0 && rps->power.mode == LOW_POWER)) ++ rps->last_adj = 0; ++ ++ /* sysfs frequency interfaces may have snuck in while servicing the ++ * interrupt ++ */ ++ new_delay += adj; ++ new_delay = clamp_t(int, new_delay, min, max); ++ ++ if (intel_set_rps(dev_priv, new_delay)) { ++ DRM_DEBUG_DRIVER("Failed to set new GPU frequency\n"); ++ rps->last_adj = 0; ++ } ++ ++ mutex_unlock(&dev_priv->pcu_lock); ++ ++out: ++ /* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */ ++ spin_lock_irq(&dev_priv->irq_lock); ++ if (rps->interrupts_enabled) ++ gen6_unmask_pm_irq(dev_priv, dev_priv->pm_rps_events); ++ spin_unlock_irq(&dev_priv->irq_lock); ++} ++ ++ ++/** ++ * ivybridge_parity_work - Workqueue called when a parity error interrupt ++ * occurred. ++ * @work: workqueue struct ++ * ++ * Doesn't actually do anything except notify userspace. As a consequence of ++ * this event, userspace should try to remap the bad rows since statistically ++ * it is likely the same row is more likely to go bad again. ++ */ ++static void ivybridge_parity_work(struct work_struct *work) ++{ ++ struct drm_i915_private *dev_priv = ++ container_of(work, typeof(*dev_priv), l3_parity.error_work); ++ u32 error_status, row, bank, subbank; ++ char *parity_event[6]; ++ u32 misccpctl; ++ u8 slice = 0; ++ ++ /* We must turn off DOP level clock gating to access the L3 registers. ++ * In order to prevent a get/put style interface, acquire struct mutex ++ * any time we access those registers. ++ */ ++ mutex_lock(&dev_priv->drm.struct_mutex); ++ ++ /* If we've screwed up tracking, just let the interrupt fire again */ ++ if (WARN_ON(!dev_priv->l3_parity.which_slice)) ++ goto out; ++ ++ misccpctl = I915_READ(GEN7_MISCCPCTL); ++ I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE); ++ POSTING_READ(GEN7_MISCCPCTL); ++ ++ while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) { ++ i915_reg_t reg; ++ ++ slice--; ++ if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv))) ++ break; ++ ++ dev_priv->l3_parity.which_slice &= ~(1<drm.primary->kdev->kobj, ++ KOBJ_CHANGE, parity_event); ++ ++ DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n", ++ slice, row, bank, subbank); ++ ++ kfree(parity_event[4]); ++ kfree(parity_event[3]); ++ kfree(parity_event[2]); ++ kfree(parity_event[1]); ++ } ++ ++ I915_WRITE(GEN7_MISCCPCTL, misccpctl); ++ ++out: ++ WARN_ON(dev_priv->l3_parity.which_slice); ++ spin_lock_irq(&dev_priv->irq_lock); ++ gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv)); ++ spin_unlock_irq(&dev_priv->irq_lock); ++ ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++} ++ ++static void ivybridge_parity_error_irq_handler(struct drm_i915_private *dev_priv, ++ u32 iir) ++{ ++ if (!HAS_L3_DPF(dev_priv)) ++ return; ++ ++ spin_lock(&dev_priv->irq_lock); ++ gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv)); ++ spin_unlock(&dev_priv->irq_lock); ++ ++ iir &= GT_PARITY_ERROR(dev_priv); ++ if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1) ++ dev_priv->l3_parity.which_slice |= 1 << 1; ++ ++ if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT) ++ dev_priv->l3_parity.which_slice |= 1 << 0; ++ ++ queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work); ++} ++ ++static void ilk_gt_irq_handler(struct drm_i915_private *dev_priv, ++ u32 gt_iir) ++{ ++ if (gt_iir & GT_RENDER_USER_INTERRUPT) ++ intel_engine_breadcrumbs_irq(dev_priv->engine[RCS0]); ++ if (gt_iir & ILK_BSD_USER_INTERRUPT) ++ intel_engine_breadcrumbs_irq(dev_priv->engine[VCS0]); ++} ++ ++static void snb_gt_irq_handler(struct drm_i915_private *dev_priv, ++ u32 gt_iir) ++{ ++ if (gt_iir & GT_RENDER_USER_INTERRUPT) ++ intel_engine_breadcrumbs_irq(dev_priv->engine[RCS0]); ++ if (gt_iir & GT_BSD_USER_INTERRUPT) ++ intel_engine_breadcrumbs_irq(dev_priv->engine[VCS0]); ++ if (gt_iir & GT_BLT_USER_INTERRUPT) ++ intel_engine_breadcrumbs_irq(dev_priv->engine[BCS0]); ++ ++ if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT | ++ GT_BSD_CS_ERROR_INTERRUPT | ++ GT_RENDER_CS_MASTER_ERROR_INTERRUPT)) ++ DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir); ++ ++ if (gt_iir & GT_PARITY_ERROR(dev_priv)) ++ ivybridge_parity_error_irq_handler(dev_priv, gt_iir); ++} ++ ++static void ++gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir) ++{ ++ bool tasklet = false; ++ ++ if (iir & GT_CONTEXT_SWITCH_INTERRUPT) ++ tasklet = true; ++ ++ if (iir & GT_RENDER_USER_INTERRUPT) { ++ intel_engine_breadcrumbs_irq(engine); ++ tasklet |= intel_engine_needs_breadcrumb_tasklet(engine); ++ } ++ ++ if (tasklet) ++ tasklet_hi_schedule(&engine->execlists.tasklet); ++} ++ ++static void gen8_gt_irq_ack(struct drm_i915_private *i915, ++ u32 master_ctl, u32 gt_iir[4]) ++{ ++ void __iomem * const regs = i915->uncore.regs; ++ ++#define GEN8_GT_IRQS (GEN8_GT_RCS_IRQ | \ ++ GEN8_GT_BCS_IRQ | \ ++ GEN8_GT_VCS0_IRQ | \ ++ GEN8_GT_VCS1_IRQ | \ ++ GEN8_GT_VECS_IRQ | \ ++ GEN8_GT_PM_IRQ | \ ++ GEN8_GT_GUC_IRQ) ++ ++ if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) { ++ gt_iir[0] = raw_reg_read(regs, GEN8_GT_IIR(0)); ++ if (likely(gt_iir[0])) ++ raw_reg_write(regs, GEN8_GT_IIR(0), gt_iir[0]); ++ } ++ ++ if (master_ctl & (GEN8_GT_VCS0_IRQ | GEN8_GT_VCS1_IRQ)) { ++ gt_iir[1] = raw_reg_read(regs, GEN8_GT_IIR(1)); ++ if (likely(gt_iir[1])) ++ raw_reg_write(regs, GEN8_GT_IIR(1), gt_iir[1]); ++ } ++ ++ if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) { ++ gt_iir[2] = raw_reg_read(regs, GEN8_GT_IIR(2)); ++ if (likely(gt_iir[2])) ++ raw_reg_write(regs, GEN8_GT_IIR(2), gt_iir[2]); ++ } ++ ++ if (master_ctl & GEN8_GT_VECS_IRQ) { ++ gt_iir[3] = raw_reg_read(regs, GEN8_GT_IIR(3)); ++ if (likely(gt_iir[3])) ++ raw_reg_write(regs, GEN8_GT_IIR(3), gt_iir[3]); ++ } ++} ++ ++static void gen8_gt_irq_handler(struct drm_i915_private *i915, ++ u32 master_ctl, u32 gt_iir[4]) ++{ ++ if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) { ++ gen8_cs_irq_handler(i915->engine[RCS0], ++ gt_iir[0] >> GEN8_RCS_IRQ_SHIFT); ++ gen8_cs_irq_handler(i915->engine[BCS0], ++ gt_iir[0] >> GEN8_BCS_IRQ_SHIFT); ++ } ++ ++ if (master_ctl & (GEN8_GT_VCS0_IRQ | GEN8_GT_VCS1_IRQ)) { ++ gen8_cs_irq_handler(i915->engine[VCS0], ++ gt_iir[1] >> GEN8_VCS0_IRQ_SHIFT); ++ gen8_cs_irq_handler(i915->engine[VCS1], ++ gt_iir[1] >> GEN8_VCS1_IRQ_SHIFT); ++ } ++ ++ if (master_ctl & GEN8_GT_VECS_IRQ) { ++ gen8_cs_irq_handler(i915->engine[VECS0], ++ gt_iir[3] >> GEN8_VECS_IRQ_SHIFT); ++ } ++ ++ if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) { ++ gen6_rps_irq_handler(i915, gt_iir[2]); ++ gen9_guc_irq_handler(i915, gt_iir[2]); ++ } ++} ++ ++static bool gen11_port_hotplug_long_detect(enum hpd_pin pin, u32 val) ++{ ++ switch (pin) { ++ case HPD_PORT_C: ++ return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC1); ++ case HPD_PORT_D: ++ return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC2); ++ case HPD_PORT_E: ++ return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC3); ++ case HPD_PORT_F: ++ return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC4); ++ default: ++ return false; ++ } ++} ++ ++static bool bxt_port_hotplug_long_detect(enum hpd_pin pin, u32 val) ++{ ++ switch (pin) { ++ case HPD_PORT_A: ++ return val & PORTA_HOTPLUG_LONG_DETECT; ++ case HPD_PORT_B: ++ return val & PORTB_HOTPLUG_LONG_DETECT; ++ case HPD_PORT_C: ++ return val & PORTC_HOTPLUG_LONG_DETECT; ++ default: ++ return false; ++ } ++} ++ ++static bool icp_ddi_port_hotplug_long_detect(enum hpd_pin pin, u32 val) ++{ ++ switch (pin) { ++ case HPD_PORT_A: ++ return val & ICP_DDIA_HPD_LONG_DETECT; ++ case HPD_PORT_B: ++ return val & ICP_DDIB_HPD_LONG_DETECT; ++ default: ++ return false; ++ } ++} ++ ++static bool icp_tc_port_hotplug_long_detect(enum hpd_pin pin, u32 val) ++{ ++ switch (pin) { ++ case HPD_PORT_C: ++ return val & ICP_TC_HPD_LONG_DETECT(PORT_TC1); ++ case HPD_PORT_D: ++ return val & ICP_TC_HPD_LONG_DETECT(PORT_TC2); ++ case HPD_PORT_E: ++ return val & ICP_TC_HPD_LONG_DETECT(PORT_TC3); ++ case HPD_PORT_F: ++ return val & ICP_TC_HPD_LONG_DETECT(PORT_TC4); ++ default: ++ return false; ++ } ++} ++ ++static bool spt_port_hotplug2_long_detect(enum hpd_pin pin, u32 val) ++{ ++ switch (pin) { ++ case HPD_PORT_E: ++ return val & PORTE_HOTPLUG_LONG_DETECT; ++ default: ++ return false; ++ } ++} ++ ++static bool spt_port_hotplug_long_detect(enum hpd_pin pin, u32 val) ++{ ++ switch (pin) { ++ case HPD_PORT_A: ++ return val & PORTA_HOTPLUG_LONG_DETECT; ++ case HPD_PORT_B: ++ return val & PORTB_HOTPLUG_LONG_DETECT; ++ case HPD_PORT_C: ++ return val & PORTC_HOTPLUG_LONG_DETECT; ++ case HPD_PORT_D: ++ return val & PORTD_HOTPLUG_LONG_DETECT; ++ default: ++ return false; ++ } ++} ++ ++static bool ilk_port_hotplug_long_detect(enum hpd_pin pin, u32 val) ++{ ++ switch (pin) { ++ case HPD_PORT_A: ++ return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT; ++ default: ++ return false; ++ } ++} ++ ++static bool pch_port_hotplug_long_detect(enum hpd_pin pin, u32 val) ++{ ++ switch (pin) { ++ case HPD_PORT_B: ++ return val & PORTB_HOTPLUG_LONG_DETECT; ++ case HPD_PORT_C: ++ return val & PORTC_HOTPLUG_LONG_DETECT; ++ case HPD_PORT_D: ++ return val & PORTD_HOTPLUG_LONG_DETECT; ++ default: ++ return false; ++ } ++} ++ ++static bool i9xx_port_hotplug_long_detect(enum hpd_pin pin, u32 val) ++{ ++ switch (pin) { ++ case HPD_PORT_B: ++ return val & PORTB_HOTPLUG_INT_LONG_PULSE; ++ case HPD_PORT_C: ++ return val & PORTC_HOTPLUG_INT_LONG_PULSE; ++ case HPD_PORT_D: ++ return val & PORTD_HOTPLUG_INT_LONG_PULSE; ++ default: ++ return false; ++ } ++} ++ ++/* ++ * Get a bit mask of pins that have triggered, and which ones may be long. ++ * This can be called multiple times with the same masks to accumulate ++ * hotplug detection results from several registers. ++ * ++ * Note that the caller is expected to zero out the masks initially. ++ */ ++static void intel_get_hpd_pins(struct drm_i915_private *dev_priv, ++ u32 *pin_mask, u32 *long_mask, ++ u32 hotplug_trigger, u32 dig_hotplug_reg, ++ const u32 hpd[HPD_NUM_PINS], ++ bool long_pulse_detect(enum hpd_pin pin, u32 val)) ++{ ++ enum hpd_pin pin; ++ ++ for_each_hpd_pin(pin) { ++ if ((hpd[pin] & hotplug_trigger) == 0) ++ continue; ++ ++ *pin_mask |= BIT(pin); ++ ++ if (long_pulse_detect(pin, dig_hotplug_reg)) ++ *long_mask |= BIT(pin); ++ } ++ ++ DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x, long 0x%08x\n", ++ hotplug_trigger, dig_hotplug_reg, *pin_mask, *long_mask); ++ ++} ++ ++static void gmbus_irq_handler(struct drm_i915_private *dev_priv) ++{ ++ wake_up_all(&dev_priv->gmbus_wait_queue); ++} ++ ++static void dp_aux_irq_handler(struct drm_i915_private *dev_priv) ++{ ++ wake_up_all(&dev_priv->gmbus_wait_queue); ++} ++ ++#if defined(CONFIG_DEBUG_FS) ++static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv, ++ enum pipe pipe, ++ u32 crc0, u32 crc1, ++ u32 crc2, u32 crc3, ++ u32 crc4) ++{ ++ struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe]; ++ struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); ++ u32 crcs[5] = { crc0, crc1, crc2, crc3, crc4 }; ++ ++ trace_intel_pipe_crc(crtc, crcs); ++ ++ spin_lock(&pipe_crc->lock); ++ /* ++ * For some not yet identified reason, the first CRC is ++ * bonkers. So let's just wait for the next vblank and read ++ * out the buggy result. ++ * ++ * On GEN8+ sometimes the second CRC is bonkers as well, so ++ * don't trust that one either. ++ */ ++ if (pipe_crc->skipped <= 0 || ++ (INTEL_GEN(dev_priv) >= 8 && pipe_crc->skipped == 1)) { ++ pipe_crc->skipped++; ++ spin_unlock(&pipe_crc->lock); ++ return; ++ } ++ spin_unlock(&pipe_crc->lock); ++ ++ drm_crtc_add_crc_entry(&crtc->base, true, ++ drm_crtc_accurate_vblank_count(&crtc->base), ++ crcs); ++} ++#else ++static inline void ++display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv, ++ enum pipe pipe, ++ u32 crc0, u32 crc1, ++ u32 crc2, u32 crc3, ++ u32 crc4) {} ++#endif ++ ++ ++static void hsw_pipe_crc_irq_handler(struct drm_i915_private *dev_priv, ++ enum pipe pipe) ++{ ++ display_pipe_crc_irq_handler(dev_priv, pipe, ++ I915_READ(PIPE_CRC_RES_1_IVB(pipe)), ++ 0, 0, 0, 0); ++} ++ ++static void ivb_pipe_crc_irq_handler(struct drm_i915_private *dev_priv, ++ enum pipe pipe) ++{ ++ display_pipe_crc_irq_handler(dev_priv, pipe, ++ I915_READ(PIPE_CRC_RES_1_IVB(pipe)), ++ I915_READ(PIPE_CRC_RES_2_IVB(pipe)), ++ I915_READ(PIPE_CRC_RES_3_IVB(pipe)), ++ I915_READ(PIPE_CRC_RES_4_IVB(pipe)), ++ I915_READ(PIPE_CRC_RES_5_IVB(pipe))); ++} ++ ++static void i9xx_pipe_crc_irq_handler(struct drm_i915_private *dev_priv, ++ enum pipe pipe) ++{ ++ u32 res1, res2; ++ ++ if (INTEL_GEN(dev_priv) >= 3) ++ res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe)); ++ else ++ res1 = 0; ++ ++ if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv)) ++ res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe)); ++ else ++ res2 = 0; ++ ++ display_pipe_crc_irq_handler(dev_priv, pipe, ++ I915_READ(PIPE_CRC_RES_RED(pipe)), ++ I915_READ(PIPE_CRC_RES_GREEN(pipe)), ++ I915_READ(PIPE_CRC_RES_BLUE(pipe)), ++ res1, res2); ++} ++ ++/* The RPS events need forcewake, so we add them to a work queue and mask their ++ * IMR bits until the work is done. Other interrupts can be processed without ++ * the work queue. */ ++static void gen11_rps_irq_handler(struct drm_i915_private *i915, u32 pm_iir) ++{ ++ struct intel_rps *rps = &i915->gt_pm.rps; ++ const u32 events = i915->pm_rps_events & pm_iir; ++ ++ lockdep_assert_held(&i915->irq_lock); ++ ++ if (unlikely(!events)) ++ return; ++ ++ gen6_mask_pm_irq(i915, events); ++ ++ if (!rps->interrupts_enabled) ++ return; ++ ++ rps->pm_iir |= events; ++ schedule_work(&rps->work); ++} ++ ++static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir) ++{ ++ struct intel_rps *rps = &dev_priv->gt_pm.rps; ++ ++ if (pm_iir & dev_priv->pm_rps_events) { ++ spin_lock(&dev_priv->irq_lock); ++ gen6_mask_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events); ++ if (rps->interrupts_enabled) { ++ rps->pm_iir |= pm_iir & dev_priv->pm_rps_events; ++ schedule_work(&rps->work); ++ } ++ spin_unlock(&dev_priv->irq_lock); ++ } ++ ++ if (INTEL_GEN(dev_priv) >= 8) ++ return; ++ ++ if (pm_iir & PM_VEBOX_USER_INTERRUPT) ++ intel_engine_breadcrumbs_irq(dev_priv->engine[VECS0]); ++ ++ if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) ++ DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir); ++} ++ ++static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 gt_iir) ++{ ++ if (gt_iir & GEN9_GUC_TO_HOST_INT_EVENT) ++ intel_guc_to_host_event_handler(&dev_priv->guc); ++} ++ ++static void i9xx_pipestat_irq_reset(struct drm_i915_private *dev_priv) ++{ ++ enum pipe pipe; ++ ++ for_each_pipe(dev_priv, pipe) { ++ I915_WRITE(PIPESTAT(pipe), ++ PIPESTAT_INT_STATUS_MASK | ++ PIPE_FIFO_UNDERRUN_STATUS); ++ ++ dev_priv->pipestat_irq_mask[pipe] = 0; ++ } ++} ++ ++static void i9xx_pipestat_irq_ack(struct drm_i915_private *dev_priv, ++ u32 iir, u32 pipe_stats[I915_MAX_PIPES]) ++{ ++ int pipe; ++ ++ spin_lock(&dev_priv->irq_lock); ++ ++ if (!dev_priv->display_irqs_enabled) { ++ spin_unlock(&dev_priv->irq_lock); ++ return; ++ } ++ ++ for_each_pipe(dev_priv, pipe) { ++ i915_reg_t reg; ++ u32 status_mask, enable_mask, iir_bit = 0; ++ ++ /* ++ * PIPESTAT bits get signalled even when the interrupt is ++ * disabled with the mask bits, and some of the status bits do ++ * not generate interrupts at all (like the underrun bit). Hence ++ * we need to be careful that we only handle what we want to ++ * handle. ++ */ ++ ++ /* fifo underruns are filterered in the underrun handler. */ ++ status_mask = PIPE_FIFO_UNDERRUN_STATUS; ++ ++ switch (pipe) { ++ case PIPE_A: ++ iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT; ++ break; ++ case PIPE_B: ++ iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT; ++ break; ++ case PIPE_C: ++ iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT; ++ break; ++ } ++ if (iir & iir_bit) ++ status_mask |= dev_priv->pipestat_irq_mask[pipe]; ++ ++ if (!status_mask) ++ continue; ++ ++ reg = PIPESTAT(pipe); ++ pipe_stats[pipe] = I915_READ(reg) & status_mask; ++ enable_mask = i915_pipestat_enable_mask(dev_priv, pipe); ++ ++ /* ++ * Clear the PIPE*STAT regs before the IIR ++ * ++ * Toggle the enable bits to make sure we get an ++ * edge in the ISR pipe event bit if we don't clear ++ * all the enabled status bits. Otherwise the edge ++ * triggered IIR on i965/g4x wouldn't notice that ++ * an interrupt is still pending. ++ */ ++ if (pipe_stats[pipe]) { ++ I915_WRITE(reg, pipe_stats[pipe]); ++ I915_WRITE(reg, enable_mask); ++ } ++ } ++ spin_unlock(&dev_priv->irq_lock); ++} ++ ++static void i8xx_pipestat_irq_handler(struct drm_i915_private *dev_priv, ++ u16 iir, u32 pipe_stats[I915_MAX_PIPES]) ++{ ++ enum pipe pipe; ++ ++ for_each_pipe(dev_priv, pipe) { ++ if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS) ++ drm_handle_vblank(&dev_priv->drm, pipe); ++ ++ if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS) ++ i9xx_pipe_crc_irq_handler(dev_priv, pipe); ++ ++ if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS) ++ intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe); ++ } ++} ++ ++static void i915_pipestat_irq_handler(struct drm_i915_private *dev_priv, ++ u32 iir, u32 pipe_stats[I915_MAX_PIPES]) ++{ ++ bool blc_event = false; ++ enum pipe pipe; ++ ++ for_each_pipe(dev_priv, pipe) { ++ if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS) ++ drm_handle_vblank(&dev_priv->drm, pipe); ++ ++ if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS) ++ blc_event = true; ++ ++ if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS) ++ i9xx_pipe_crc_irq_handler(dev_priv, pipe); ++ ++ if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS) ++ intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe); ++ } ++ ++ if (blc_event || (iir & I915_ASLE_INTERRUPT)) ++ intel_opregion_asle_intr(dev_priv); ++} ++ ++static void i965_pipestat_irq_handler(struct drm_i915_private *dev_priv, ++ u32 iir, u32 pipe_stats[I915_MAX_PIPES]) ++{ ++ bool blc_event = false; ++ enum pipe pipe; ++ ++ for_each_pipe(dev_priv, pipe) { ++ if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS) ++ drm_handle_vblank(&dev_priv->drm, pipe); ++ ++ if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS) ++ blc_event = true; ++ ++ if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS) ++ i9xx_pipe_crc_irq_handler(dev_priv, pipe); ++ ++ if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS) ++ intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe); ++ } ++ ++ if (blc_event || (iir & I915_ASLE_INTERRUPT)) ++ intel_opregion_asle_intr(dev_priv); ++ ++ if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS) ++ gmbus_irq_handler(dev_priv); ++} ++ ++static void valleyview_pipestat_irq_handler(struct drm_i915_private *dev_priv, ++ u32 pipe_stats[I915_MAX_PIPES]) ++{ ++ enum pipe pipe; ++ ++ for_each_pipe(dev_priv, pipe) { ++ if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS) ++ drm_handle_vblank(&dev_priv->drm, pipe); ++ ++ if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS) ++ i9xx_pipe_crc_irq_handler(dev_priv, pipe); ++ ++ if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS) ++ intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe); ++ } ++ ++ if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS) ++ gmbus_irq_handler(dev_priv); ++} ++ ++static u32 i9xx_hpd_irq_ack(struct drm_i915_private *dev_priv) ++{ ++ u32 hotplug_status = 0, hotplug_status_mask; ++ int i; ++ ++ if (IS_G4X(dev_priv) || ++ IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ hotplug_status_mask = HOTPLUG_INT_STATUS_G4X | ++ DP_AUX_CHANNEL_MASK_INT_STATUS_G4X; ++ else ++ hotplug_status_mask = HOTPLUG_INT_STATUS_I915; ++ ++ /* ++ * We absolutely have to clear all the pending interrupt ++ * bits in PORT_HOTPLUG_STAT. Otherwise the ISR port ++ * interrupt bit won't have an edge, and the i965/g4x ++ * edge triggered IIR will not notice that an interrupt ++ * is still pending. We can't use PORT_HOTPLUG_EN to ++ * guarantee the edge as the act of toggling the enable ++ * bits can itself generate a new hotplug interrupt :( ++ */ ++ for (i = 0; i < 10; i++) { ++ u32 tmp = I915_READ(PORT_HOTPLUG_STAT) & hotplug_status_mask; ++ ++ if (tmp == 0) ++ return hotplug_status; ++ ++ hotplug_status |= tmp; ++ I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status); ++ } ++ ++ WARN_ONCE(1, ++ "PORT_HOTPLUG_STAT did not clear (0x%08x)\n", ++ I915_READ(PORT_HOTPLUG_STAT)); ++ ++ return hotplug_status; ++} ++ ++static void i9xx_hpd_irq_handler(struct drm_i915_private *dev_priv, ++ u32 hotplug_status) ++{ ++ u32 pin_mask = 0, long_mask = 0; ++ ++ if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) || ++ IS_CHERRYVIEW(dev_priv)) { ++ u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X; ++ ++ if (hotplug_trigger) { ++ intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, ++ hotplug_trigger, hotplug_trigger, ++ hpd_status_g4x, ++ i9xx_port_hotplug_long_detect); ++ ++ intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); ++ } ++ ++ if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X) ++ dp_aux_irq_handler(dev_priv); ++ } else { ++ u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915; ++ ++ if (hotplug_trigger) { ++ intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, ++ hotplug_trigger, hotplug_trigger, ++ hpd_status_i915, ++ i9xx_port_hotplug_long_detect); ++ intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); ++ } ++ } ++} ++ ++static irqreturn_t valleyview_irq_handler(int irq, void *arg) ++{ ++ struct drm_device *dev = arg; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ irqreturn_t ret = IRQ_NONE; ++ ++ if (!intel_irqs_enabled(dev_priv)) ++ return IRQ_NONE; ++ ++ /* IRQs are synced during runtime_suspend, we don't require a wakeref */ ++ disable_rpm_wakeref_asserts(dev_priv); ++ ++ do { ++ u32 iir, gt_iir, pm_iir; ++ u32 pipe_stats[I915_MAX_PIPES] = {}; ++ u32 hotplug_status = 0; ++ u32 ier = 0; ++ ++ gt_iir = I915_READ(GTIIR); ++ pm_iir = I915_READ(GEN6_PMIIR); ++ iir = I915_READ(VLV_IIR); ++ ++ if (gt_iir == 0 && pm_iir == 0 && iir == 0) ++ break; ++ ++ ret = IRQ_HANDLED; ++ ++ /* ++ * Theory on interrupt generation, based on empirical evidence: ++ * ++ * x = ((VLV_IIR & VLV_IER) || ++ * (((GT_IIR & GT_IER) || (GEN6_PMIIR & GEN6_PMIER)) && ++ * (VLV_MASTER_IER & MASTER_INTERRUPT_ENABLE))); ++ * ++ * A CPU interrupt will only be raised when 'x' has a 0->1 edge. ++ * Hence we clear MASTER_INTERRUPT_ENABLE and VLV_IER to ++ * guarantee the CPU interrupt will be raised again even if we ++ * don't end up clearing all the VLV_IIR, GT_IIR, GEN6_PMIIR ++ * bits this time around. ++ */ ++ I915_WRITE(VLV_MASTER_IER, 0); ++ ier = I915_READ(VLV_IER); ++ I915_WRITE(VLV_IER, 0); ++ ++ if (gt_iir) ++ I915_WRITE(GTIIR, gt_iir); ++ if (pm_iir) ++ I915_WRITE(GEN6_PMIIR, pm_iir); ++ ++ if (iir & I915_DISPLAY_PORT_INTERRUPT) ++ hotplug_status = i9xx_hpd_irq_ack(dev_priv); ++ ++ /* Call regardless, as some status bits might not be ++ * signalled in iir */ ++ i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats); ++ ++ if (iir & (I915_LPE_PIPE_A_INTERRUPT | ++ I915_LPE_PIPE_B_INTERRUPT)) ++ intel_lpe_audio_irq_handler(dev_priv); ++ ++ /* ++ * VLV_IIR is single buffered, and reflects the level ++ * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last. ++ */ ++ if (iir) ++ I915_WRITE(VLV_IIR, iir); ++ ++ I915_WRITE(VLV_IER, ier); ++ I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE); ++ ++ if (gt_iir) ++ snb_gt_irq_handler(dev_priv, gt_iir); ++ if (pm_iir) ++ gen6_rps_irq_handler(dev_priv, pm_iir); ++ ++ if (hotplug_status) ++ i9xx_hpd_irq_handler(dev_priv, hotplug_status); ++ ++ valleyview_pipestat_irq_handler(dev_priv, pipe_stats); ++ } while (0); ++ ++ enable_rpm_wakeref_asserts(dev_priv); ++ ++ return ret; ++} ++ ++static irqreturn_t cherryview_irq_handler(int irq, void *arg) ++{ ++ struct drm_device *dev = arg; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ irqreturn_t ret = IRQ_NONE; ++ ++ if (!intel_irqs_enabled(dev_priv)) ++ return IRQ_NONE; ++ ++ /* IRQs are synced during runtime_suspend, we don't require a wakeref */ ++ disable_rpm_wakeref_asserts(dev_priv); ++ ++ do { ++ u32 master_ctl, iir; ++ u32 pipe_stats[I915_MAX_PIPES] = {}; ++ u32 hotplug_status = 0; ++ u32 gt_iir[4]; ++ u32 ier = 0; ++ ++ master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL; ++ iir = I915_READ(VLV_IIR); ++ ++ if (master_ctl == 0 && iir == 0) ++ break; ++ ++ ret = IRQ_HANDLED; ++ ++ /* ++ * Theory on interrupt generation, based on empirical evidence: ++ * ++ * x = ((VLV_IIR & VLV_IER) || ++ * ((GEN8_MASTER_IRQ & ~GEN8_MASTER_IRQ_CONTROL) && ++ * (GEN8_MASTER_IRQ & GEN8_MASTER_IRQ_CONTROL))); ++ * ++ * A CPU interrupt will only be raised when 'x' has a 0->1 edge. ++ * Hence we clear GEN8_MASTER_IRQ_CONTROL and VLV_IER to ++ * guarantee the CPU interrupt will be raised again even if we ++ * don't end up clearing all the VLV_IIR and GEN8_MASTER_IRQ_CONTROL ++ * bits this time around. ++ */ ++ I915_WRITE(GEN8_MASTER_IRQ, 0); ++ ier = I915_READ(VLV_IER); ++ I915_WRITE(VLV_IER, 0); ++ ++ gen8_gt_irq_ack(dev_priv, master_ctl, gt_iir); ++ ++ if (iir & I915_DISPLAY_PORT_INTERRUPT) ++ hotplug_status = i9xx_hpd_irq_ack(dev_priv); ++ ++ /* Call regardless, as some status bits might not be ++ * signalled in iir */ ++ i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats); ++ ++ if (iir & (I915_LPE_PIPE_A_INTERRUPT | ++ I915_LPE_PIPE_B_INTERRUPT | ++ I915_LPE_PIPE_C_INTERRUPT)) ++ intel_lpe_audio_irq_handler(dev_priv); ++ ++ /* ++ * VLV_IIR is single buffered, and reflects the level ++ * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last. ++ */ ++ if (iir) ++ I915_WRITE(VLV_IIR, iir); ++ ++ I915_WRITE(VLV_IER, ier); ++ I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL); ++ ++ gen8_gt_irq_handler(dev_priv, master_ctl, gt_iir); ++ ++ if (hotplug_status) ++ i9xx_hpd_irq_handler(dev_priv, hotplug_status); ++ ++ valleyview_pipestat_irq_handler(dev_priv, pipe_stats); ++ } while (0); ++ ++ enable_rpm_wakeref_asserts(dev_priv); ++ ++ return ret; ++} ++ ++static void ibx_hpd_irq_handler(struct drm_i915_private *dev_priv, ++ u32 hotplug_trigger, ++ const u32 hpd[HPD_NUM_PINS]) ++{ ++ u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0; ++ ++ /* ++ * Somehow the PCH doesn't seem to really ack the interrupt to the CPU ++ * unless we touch the hotplug register, even if hotplug_trigger is ++ * zero. Not acking leads to "The master control interrupt lied (SDE)!" ++ * errors. ++ */ ++ dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG); ++ if (!hotplug_trigger) { ++ u32 mask = PORTA_HOTPLUG_STATUS_MASK | ++ PORTD_HOTPLUG_STATUS_MASK | ++ PORTC_HOTPLUG_STATUS_MASK | ++ PORTB_HOTPLUG_STATUS_MASK; ++ dig_hotplug_reg &= ~mask; ++ } ++ ++ I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg); ++ if (!hotplug_trigger) ++ return; ++ ++ intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger, ++ dig_hotplug_reg, hpd, ++ pch_port_hotplug_long_detect); ++ ++ intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); ++} ++ ++static void ibx_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir) ++{ ++ int pipe; ++ u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK; ++ ++ ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ibx); ++ ++ if (pch_iir & SDE_AUDIO_POWER_MASK) { ++ int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >> ++ SDE_AUDIO_POWER_SHIFT); ++ DRM_DEBUG_DRIVER("PCH audio power change on port %d\n", ++ port_name(port)); ++ } ++ ++ if (pch_iir & SDE_AUX_MASK) ++ dp_aux_irq_handler(dev_priv); ++ ++ if (pch_iir & SDE_GMBUS) ++ gmbus_irq_handler(dev_priv); ++ ++ if (pch_iir & SDE_AUDIO_HDCP_MASK) ++ DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n"); ++ ++ if (pch_iir & SDE_AUDIO_TRANS_MASK) ++ DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n"); ++ ++ if (pch_iir & SDE_POISON) ++ DRM_ERROR("PCH poison interrupt\n"); ++ ++ if (pch_iir & SDE_FDI_MASK) ++ for_each_pipe(dev_priv, pipe) ++ DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n", ++ pipe_name(pipe), ++ I915_READ(FDI_RX_IIR(pipe))); ++ ++ if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE)) ++ DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n"); ++ ++ if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR)) ++ DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n"); ++ ++ if (pch_iir & SDE_TRANSA_FIFO_UNDER) ++ intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_A); ++ ++ if (pch_iir & SDE_TRANSB_FIFO_UNDER) ++ intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_B); ++} ++ ++static void ivb_err_int_handler(struct drm_i915_private *dev_priv) ++{ ++ u32 err_int = I915_READ(GEN7_ERR_INT); ++ enum pipe pipe; ++ ++ if (err_int & ERR_INT_POISON) ++ DRM_ERROR("Poison interrupt\n"); ++ ++ for_each_pipe(dev_priv, pipe) { ++ if (err_int & ERR_INT_FIFO_UNDERRUN(pipe)) ++ intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe); ++ ++ if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) { ++ if (IS_IVYBRIDGE(dev_priv)) ++ ivb_pipe_crc_irq_handler(dev_priv, pipe); ++ else ++ hsw_pipe_crc_irq_handler(dev_priv, pipe); ++ } ++ } ++ ++ I915_WRITE(GEN7_ERR_INT, err_int); ++} ++ ++static void cpt_serr_int_handler(struct drm_i915_private *dev_priv) ++{ ++ u32 serr_int = I915_READ(SERR_INT); ++ enum pipe pipe; ++ ++ if (serr_int & SERR_INT_POISON) ++ DRM_ERROR("PCH poison interrupt\n"); ++ ++ for_each_pipe(dev_priv, pipe) ++ if (serr_int & SERR_INT_TRANS_FIFO_UNDERRUN(pipe)) ++ intel_pch_fifo_underrun_irq_handler(dev_priv, pipe); ++ ++ I915_WRITE(SERR_INT, serr_int); ++} ++ ++static void cpt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir) ++{ ++ int pipe; ++ u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT; ++ ++ ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_cpt); ++ ++ if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) { ++ int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >> ++ SDE_AUDIO_POWER_SHIFT_CPT); ++ DRM_DEBUG_DRIVER("PCH audio power change on port %c\n", ++ port_name(port)); ++ } ++ ++ if (pch_iir & SDE_AUX_MASK_CPT) ++ dp_aux_irq_handler(dev_priv); ++ ++ if (pch_iir & SDE_GMBUS_CPT) ++ gmbus_irq_handler(dev_priv); ++ ++ if (pch_iir & SDE_AUDIO_CP_REQ_CPT) ++ DRM_DEBUG_DRIVER("Audio CP request interrupt\n"); ++ ++ if (pch_iir & SDE_AUDIO_CP_CHG_CPT) ++ DRM_DEBUG_DRIVER("Audio CP change interrupt\n"); ++ ++ if (pch_iir & SDE_FDI_MASK_CPT) ++ for_each_pipe(dev_priv, pipe) ++ DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n", ++ pipe_name(pipe), ++ I915_READ(FDI_RX_IIR(pipe))); ++ ++ if (pch_iir & SDE_ERROR_CPT) ++ cpt_serr_int_handler(dev_priv); ++} ++ ++static void icp_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir) ++{ ++ u32 ddi_hotplug_trigger = pch_iir & SDE_DDI_MASK_ICP; ++ u32 tc_hotplug_trigger = pch_iir & SDE_TC_MASK_ICP; ++ u32 pin_mask = 0, long_mask = 0; ++ ++ if (ddi_hotplug_trigger) { ++ u32 dig_hotplug_reg; ++ ++ dig_hotplug_reg = I915_READ(SHOTPLUG_CTL_DDI); ++ I915_WRITE(SHOTPLUG_CTL_DDI, dig_hotplug_reg); ++ ++ intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, ++ ddi_hotplug_trigger, ++ dig_hotplug_reg, hpd_icp, ++ icp_ddi_port_hotplug_long_detect); ++ } ++ ++ if (tc_hotplug_trigger) { ++ u32 dig_hotplug_reg; ++ ++ dig_hotplug_reg = I915_READ(SHOTPLUG_CTL_TC); ++ I915_WRITE(SHOTPLUG_CTL_TC, dig_hotplug_reg); ++ ++ intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, ++ tc_hotplug_trigger, ++ dig_hotplug_reg, hpd_icp, ++ icp_tc_port_hotplug_long_detect); ++ } ++ ++ if (pin_mask) ++ intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); ++ ++ if (pch_iir & SDE_GMBUS_ICP) ++ gmbus_irq_handler(dev_priv); ++} ++ ++static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir) ++{ ++ u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT & ++ ~SDE_PORTE_HOTPLUG_SPT; ++ u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT; ++ u32 pin_mask = 0, long_mask = 0; ++ ++ if (hotplug_trigger) { ++ u32 dig_hotplug_reg; ++ ++ dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG); ++ I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg); ++ ++ intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, ++ hotplug_trigger, dig_hotplug_reg, hpd_spt, ++ spt_port_hotplug_long_detect); ++ } ++ ++ if (hotplug2_trigger) { ++ u32 dig_hotplug_reg; ++ ++ dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2); ++ I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg); ++ ++ intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, ++ hotplug2_trigger, dig_hotplug_reg, hpd_spt, ++ spt_port_hotplug2_long_detect); ++ } ++ ++ if (pin_mask) ++ intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); ++ ++ if (pch_iir & SDE_GMBUS_CPT) ++ gmbus_irq_handler(dev_priv); ++} ++ ++static void ilk_hpd_irq_handler(struct drm_i915_private *dev_priv, ++ u32 hotplug_trigger, ++ const u32 hpd[HPD_NUM_PINS]) ++{ ++ u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0; ++ ++ dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL); ++ I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg); ++ ++ intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger, ++ dig_hotplug_reg, hpd, ++ ilk_port_hotplug_long_detect); ++ ++ intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); ++} ++ ++static void ilk_display_irq_handler(struct drm_i915_private *dev_priv, ++ u32 de_iir) ++{ ++ enum pipe pipe; ++ u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG; ++ ++ if (hotplug_trigger) ++ ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ilk); ++ ++ if (de_iir & DE_AUX_CHANNEL_A) ++ dp_aux_irq_handler(dev_priv); ++ ++ if (de_iir & DE_GSE) ++ intel_opregion_asle_intr(dev_priv); ++ ++ if (de_iir & DE_POISON) ++ DRM_ERROR("Poison interrupt\n"); ++ ++ for_each_pipe(dev_priv, pipe) { ++ if (de_iir & DE_PIPE_VBLANK(pipe)) ++ drm_handle_vblank(&dev_priv->drm, pipe); ++ ++ if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe)) ++ intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe); ++ ++ if (de_iir & DE_PIPE_CRC_DONE(pipe)) ++ i9xx_pipe_crc_irq_handler(dev_priv, pipe); ++ } ++ ++ /* check event from PCH */ ++ if (de_iir & DE_PCH_EVENT) { ++ u32 pch_iir = I915_READ(SDEIIR); ++ ++ if (HAS_PCH_CPT(dev_priv)) ++ cpt_irq_handler(dev_priv, pch_iir); ++ else ++ ibx_irq_handler(dev_priv, pch_iir); ++ ++ /* should clear PCH hotplug event before clear CPU irq */ ++ I915_WRITE(SDEIIR, pch_iir); ++ } ++ ++ if (IS_GEN(dev_priv, 5) && de_iir & DE_PCU_EVENT) ++ ironlake_rps_change_irq_handler(dev_priv); ++} ++ ++static void ivb_display_irq_handler(struct drm_i915_private *dev_priv, ++ u32 de_iir) ++{ ++ enum pipe pipe; ++ u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB; ++ ++ if (hotplug_trigger) ++ ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ivb); ++ ++ if (de_iir & DE_ERR_INT_IVB) ++ ivb_err_int_handler(dev_priv); ++ ++ if (de_iir & DE_EDP_PSR_INT_HSW) { ++ u32 psr_iir = I915_READ(EDP_PSR_IIR); ++ ++ intel_psr_irq_handler(dev_priv, psr_iir); ++ I915_WRITE(EDP_PSR_IIR, psr_iir); ++ } ++ ++ if (de_iir & DE_AUX_CHANNEL_A_IVB) ++ dp_aux_irq_handler(dev_priv); ++ ++ if (de_iir & DE_GSE_IVB) ++ intel_opregion_asle_intr(dev_priv); ++ ++ for_each_pipe(dev_priv, pipe) { ++ if (de_iir & (DE_PIPE_VBLANK_IVB(pipe))) ++ drm_handle_vblank(&dev_priv->drm, pipe); ++ } ++ ++ /* check event from PCH */ ++ if (!HAS_PCH_NOP(dev_priv) && (de_iir & DE_PCH_EVENT_IVB)) { ++ u32 pch_iir = I915_READ(SDEIIR); ++ ++ cpt_irq_handler(dev_priv, pch_iir); ++ ++ /* clear PCH hotplug event before clear CPU irq */ ++ I915_WRITE(SDEIIR, pch_iir); ++ } ++} ++ ++/* ++ * To handle irqs with the minimum potential races with fresh interrupts, we: ++ * 1 - Disable Master Interrupt Control. ++ * 2 - Find the source(s) of the interrupt. ++ * 3 - Clear the Interrupt Identity bits (IIR). ++ * 4 - Process the interrupt(s) that had bits set in the IIRs. ++ * 5 - Re-enable Master Interrupt Control. ++ */ ++static irqreturn_t ironlake_irq_handler(int irq, void *arg) ++{ ++ struct drm_device *dev = arg; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ u32 de_iir, gt_iir, de_ier, sde_ier = 0; ++ irqreturn_t ret = IRQ_NONE; ++ ++ if (!intel_irqs_enabled(dev_priv)) ++ return IRQ_NONE; ++ ++ /* IRQs are synced during runtime_suspend, we don't require a wakeref */ ++ disable_rpm_wakeref_asserts(dev_priv); ++ ++ /* disable master interrupt before clearing iir */ ++ de_ier = I915_READ(DEIER); ++ I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL); ++ ++ /* Disable south interrupts. We'll only write to SDEIIR once, so further ++ * interrupts will will be stored on its back queue, and then we'll be ++ * able to process them after we restore SDEIER (as soon as we restore ++ * it, we'll get an interrupt if SDEIIR still has something to process ++ * due to its back queue). */ ++ if (!HAS_PCH_NOP(dev_priv)) { ++ sde_ier = I915_READ(SDEIER); ++ I915_WRITE(SDEIER, 0); ++ } ++ ++ /* Find, clear, then process each source of interrupt */ ++ ++ gt_iir = I915_READ(GTIIR); ++ if (gt_iir) { ++ I915_WRITE(GTIIR, gt_iir); ++ ret = IRQ_HANDLED; ++ if (INTEL_GEN(dev_priv) >= 6) ++ snb_gt_irq_handler(dev_priv, gt_iir); ++ else ++ ilk_gt_irq_handler(dev_priv, gt_iir); ++ } ++ ++ de_iir = I915_READ(DEIIR); ++ if (de_iir) { ++ I915_WRITE(DEIIR, de_iir); ++ ret = IRQ_HANDLED; ++ if (INTEL_GEN(dev_priv) >= 7) ++ ivb_display_irq_handler(dev_priv, de_iir); ++ else ++ ilk_display_irq_handler(dev_priv, de_iir); ++ } ++ ++ if (INTEL_GEN(dev_priv) >= 6) { ++ u32 pm_iir = I915_READ(GEN6_PMIIR); ++ if (pm_iir) { ++ I915_WRITE(GEN6_PMIIR, pm_iir); ++ ret = IRQ_HANDLED; ++ gen6_rps_irq_handler(dev_priv, pm_iir); ++ } ++ } ++ ++ I915_WRITE(DEIER, de_ier); ++ if (!HAS_PCH_NOP(dev_priv)) ++ I915_WRITE(SDEIER, sde_ier); ++ ++ /* IRQs are synced during runtime_suspend, we don't require a wakeref */ ++ enable_rpm_wakeref_asserts(dev_priv); ++ ++ return ret; ++} ++ ++static void bxt_hpd_irq_handler(struct drm_i915_private *dev_priv, ++ u32 hotplug_trigger, ++ const u32 hpd[HPD_NUM_PINS]) ++{ ++ u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0; ++ ++ dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG); ++ I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg); ++ ++ intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger, ++ dig_hotplug_reg, hpd, ++ bxt_port_hotplug_long_detect); ++ ++ intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); ++} ++ ++static void gen11_hpd_irq_handler(struct drm_i915_private *dev_priv, u32 iir) ++{ ++ u32 pin_mask = 0, long_mask = 0; ++ u32 trigger_tc = iir & GEN11_DE_TC_HOTPLUG_MASK; ++ u32 trigger_tbt = iir & GEN11_DE_TBT_HOTPLUG_MASK; ++ ++ if (trigger_tc) { ++ u32 dig_hotplug_reg; ++ ++ dig_hotplug_reg = I915_READ(GEN11_TC_HOTPLUG_CTL); ++ I915_WRITE(GEN11_TC_HOTPLUG_CTL, dig_hotplug_reg); ++ ++ intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, trigger_tc, ++ dig_hotplug_reg, hpd_gen11, ++ gen11_port_hotplug_long_detect); ++ } ++ ++ if (trigger_tbt) { ++ u32 dig_hotplug_reg; ++ ++ dig_hotplug_reg = I915_READ(GEN11_TBT_HOTPLUG_CTL); ++ I915_WRITE(GEN11_TBT_HOTPLUG_CTL, dig_hotplug_reg); ++ ++ intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, trigger_tbt, ++ dig_hotplug_reg, hpd_gen11, ++ gen11_port_hotplug_long_detect); ++ } ++ ++ if (pin_mask) ++ intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); ++ else ++ DRM_ERROR("Unexpected DE HPD interrupt 0x%08x\n", iir); ++} ++ ++static u32 gen8_de_port_aux_mask(struct drm_i915_private *dev_priv) ++{ ++ u32 mask = GEN8_AUX_CHANNEL_A; ++ ++ if (INTEL_GEN(dev_priv) >= 9) ++ mask |= GEN9_AUX_CHANNEL_B | ++ GEN9_AUX_CHANNEL_C | ++ GEN9_AUX_CHANNEL_D; ++ ++ if (IS_CNL_WITH_PORT_F(dev_priv)) ++ mask |= CNL_AUX_CHANNEL_F; ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ mask |= ICL_AUX_CHANNEL_E | ++ CNL_AUX_CHANNEL_F; ++ ++ return mask; ++} ++ ++static irqreturn_t ++gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl) ++{ ++ irqreturn_t ret = IRQ_NONE; ++ u32 iir; ++ enum pipe pipe; ++ ++ if (master_ctl & GEN8_DE_MISC_IRQ) { ++ iir = I915_READ(GEN8_DE_MISC_IIR); ++ if (iir) { ++ bool found = false; ++ ++ I915_WRITE(GEN8_DE_MISC_IIR, iir); ++ ret = IRQ_HANDLED; ++ ++ if (iir & GEN8_DE_MISC_GSE) { ++ intel_opregion_asle_intr(dev_priv); ++ found = true; ++ } ++ ++ if (iir & GEN8_DE_EDP_PSR) { ++ u32 psr_iir = I915_READ(EDP_PSR_IIR); ++ ++ intel_psr_irq_handler(dev_priv, psr_iir); ++ I915_WRITE(EDP_PSR_IIR, psr_iir); ++ found = true; ++ } ++ ++ if (!found) ++ DRM_ERROR("Unexpected DE Misc interrupt\n"); ++ } ++ else ++ DRM_ERROR("The master control interrupt lied (DE MISC)!\n"); ++ } ++ ++ if (INTEL_GEN(dev_priv) >= 11 && (master_ctl & GEN11_DE_HPD_IRQ)) { ++ iir = I915_READ(GEN11_DE_HPD_IIR); ++ if (iir) { ++ I915_WRITE(GEN11_DE_HPD_IIR, iir); ++ ret = IRQ_HANDLED; ++ gen11_hpd_irq_handler(dev_priv, iir); ++ } else { ++ DRM_ERROR("The master control interrupt lied, (DE HPD)!\n"); ++ } ++ } ++ ++ if (master_ctl & GEN8_DE_PORT_IRQ) { ++ iir = I915_READ(GEN8_DE_PORT_IIR); ++ if (iir) { ++ u32 tmp_mask; ++ bool found = false; ++ ++ I915_WRITE(GEN8_DE_PORT_IIR, iir); ++ ret = IRQ_HANDLED; ++ ++ if (iir & gen8_de_port_aux_mask(dev_priv)) { ++ dp_aux_irq_handler(dev_priv); ++ found = true; ++ } ++ ++ if (IS_GEN9_LP(dev_priv)) { ++ tmp_mask = iir & BXT_DE_PORT_HOTPLUG_MASK; ++ if (tmp_mask) { ++ bxt_hpd_irq_handler(dev_priv, tmp_mask, ++ hpd_bxt); ++ found = true; ++ } ++ } else if (IS_BROADWELL(dev_priv)) { ++ tmp_mask = iir & GEN8_PORT_DP_A_HOTPLUG; ++ if (tmp_mask) { ++ ilk_hpd_irq_handler(dev_priv, ++ tmp_mask, hpd_bdw); ++ found = true; ++ } ++ } ++ ++ if (IS_GEN9_LP(dev_priv) && (iir & BXT_DE_PORT_GMBUS)) { ++ gmbus_irq_handler(dev_priv); ++ found = true; ++ } ++ ++ if (!found) ++ DRM_ERROR("Unexpected DE Port interrupt\n"); ++ } ++ else ++ DRM_ERROR("The master control interrupt lied (DE PORT)!\n"); ++ } ++ ++ for_each_pipe(dev_priv, pipe) { ++ u32 fault_errors; ++ ++ if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe))) ++ continue; ++ ++ iir = I915_READ(GEN8_DE_PIPE_IIR(pipe)); ++ if (!iir) { ++ DRM_ERROR("The master control interrupt lied (DE PIPE)!\n"); ++ continue; ++ } ++ ++ ret = IRQ_HANDLED; ++ I915_WRITE(GEN8_DE_PIPE_IIR(pipe), iir); ++ ++ if (iir & GEN8_PIPE_VBLANK) ++ drm_handle_vblank(&dev_priv->drm, pipe); ++ ++ if (iir & GEN8_PIPE_CDCLK_CRC_DONE) ++ hsw_pipe_crc_irq_handler(dev_priv, pipe); ++ ++ if (iir & GEN8_PIPE_FIFO_UNDERRUN) ++ intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe); ++ ++ fault_errors = iir; ++ if (INTEL_GEN(dev_priv) >= 9) ++ fault_errors &= GEN9_DE_PIPE_IRQ_FAULT_ERRORS; ++ else ++ fault_errors &= GEN8_DE_PIPE_IRQ_FAULT_ERRORS; ++ ++ if (fault_errors) ++ DRM_ERROR("Fault errors on pipe %c: 0x%08x\n", ++ pipe_name(pipe), ++ fault_errors); ++ } ++ ++ if (HAS_PCH_SPLIT(dev_priv) && !HAS_PCH_NOP(dev_priv) && ++ master_ctl & GEN8_DE_PCH_IRQ) { ++ /* ++ * FIXME(BDW): Assume for now that the new interrupt handling ++ * scheme also closed the SDE interrupt handling race we've seen ++ * on older pch-split platforms. But this needs testing. ++ */ ++ iir = I915_READ(SDEIIR); ++ if (iir) { ++ I915_WRITE(SDEIIR, iir); ++ ret = IRQ_HANDLED; ++ ++ if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP) ++ icp_irq_handler(dev_priv, iir); ++ else if (INTEL_PCH_TYPE(dev_priv) >= PCH_SPT) ++ spt_irq_handler(dev_priv, iir); ++ else ++ cpt_irq_handler(dev_priv, iir); ++ } else { ++ /* ++ * Like on previous PCH there seems to be something ++ * fishy going on with forwarding PCH interrupts. ++ */ ++ DRM_DEBUG_DRIVER("The master control interrupt lied (SDE)!\n"); ++ } ++ } ++ ++ return ret; ++} ++ ++static inline u32 gen8_master_intr_disable(void __iomem * const regs) ++{ ++ raw_reg_write(regs, GEN8_MASTER_IRQ, 0); ++ ++ /* ++ * Now with master disabled, get a sample of level indications ++ * for this interrupt. Indications will be cleared on related acks. ++ * New indications can and will light up during processing, ++ * and will generate new interrupt after enabling master. ++ */ ++ return raw_reg_read(regs, GEN8_MASTER_IRQ); ++} ++ ++static inline void gen8_master_intr_enable(void __iomem * const regs) ++{ ++ raw_reg_write(regs, GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL); ++} ++ ++static irqreturn_t gen8_irq_handler(int irq, void *arg) ++{ ++ struct drm_i915_private *dev_priv = to_i915(arg); ++ void __iomem * const regs = dev_priv->uncore.regs; ++ u32 master_ctl; ++ u32 gt_iir[4]; ++ ++ if (!intel_irqs_enabled(dev_priv)) ++ return IRQ_NONE; ++ ++ master_ctl = gen8_master_intr_disable(regs); ++ if (!master_ctl) { ++ gen8_master_intr_enable(regs); ++ return IRQ_NONE; ++ } ++ ++ /* Find, clear, then process each source of interrupt */ ++ gen8_gt_irq_ack(dev_priv, master_ctl, gt_iir); ++ ++ /* IRQs are synced during runtime_suspend, we don't require a wakeref */ ++ if (master_ctl & ~GEN8_GT_IRQS) { ++ disable_rpm_wakeref_asserts(dev_priv); ++ gen8_de_irq_handler(dev_priv, master_ctl); ++ enable_rpm_wakeref_asserts(dev_priv); ++ } ++ ++ gen8_master_intr_enable(regs); ++ ++ gen8_gt_irq_handler(dev_priv, master_ctl, gt_iir); ++ ++ return IRQ_HANDLED; ++} ++ ++static u32 ++gen11_gt_engine_identity(struct drm_i915_private * const i915, ++ const unsigned int bank, const unsigned int bit) ++{ ++ void __iomem * const regs = i915->uncore.regs; ++ u32 timeout_ts; ++ u32 ident; ++ ++ lockdep_assert_held(&i915->irq_lock); ++ ++ raw_reg_write(regs, GEN11_IIR_REG_SELECTOR(bank), BIT(bit)); ++ ++ /* ++ * NB: Specs do not specify how long to spin wait, ++ * so we do ~100us as an educated guess. ++ */ ++ timeout_ts = (local_clock() >> 10) + 100; ++ do { ++ ident = raw_reg_read(regs, GEN11_INTR_IDENTITY_REG(bank)); ++ } while (!(ident & GEN11_INTR_DATA_VALID) && ++ !time_after32(local_clock() >> 10, timeout_ts)); ++ ++ if (unlikely(!(ident & GEN11_INTR_DATA_VALID))) { ++ DRM_ERROR("INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n", ++ bank, bit, ident); ++ return 0; ++ } ++ ++ raw_reg_write(regs, GEN11_INTR_IDENTITY_REG(bank), ++ GEN11_INTR_DATA_VALID); ++ ++ return ident; ++} ++ ++static void ++gen11_other_irq_handler(struct drm_i915_private * const i915, ++ const u8 instance, const u16 iir) ++{ ++ if (instance == OTHER_GTPM_INSTANCE) ++ return gen11_rps_irq_handler(i915, iir); ++ ++ WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n", ++ instance, iir); ++} ++ ++static void ++gen11_engine_irq_handler(struct drm_i915_private * const i915, ++ const u8 class, const u8 instance, const u16 iir) ++{ ++ struct intel_engine_cs *engine; ++ ++ if (instance <= MAX_ENGINE_INSTANCE) ++ engine = i915->engine_class[class][instance]; ++ else ++ engine = NULL; ++ ++ if (likely(engine)) ++ return gen8_cs_irq_handler(engine, iir); ++ ++ WARN_ONCE(1, "unhandled engine interrupt class=0x%x, instance=0x%x\n", ++ class, instance); ++} ++ ++static void ++gen11_gt_identity_handler(struct drm_i915_private * const i915, ++ const u32 identity) ++{ ++ const u8 class = GEN11_INTR_ENGINE_CLASS(identity); ++ const u8 instance = GEN11_INTR_ENGINE_INSTANCE(identity); ++ const u16 intr = GEN11_INTR_ENGINE_INTR(identity); ++ ++ if (unlikely(!intr)) ++ return; ++ ++ if (class <= COPY_ENGINE_CLASS) ++ return gen11_engine_irq_handler(i915, class, instance, intr); ++ ++ if (class == OTHER_CLASS) ++ return gen11_other_irq_handler(i915, instance, intr); ++ ++ WARN_ONCE(1, "unknown interrupt class=0x%x, instance=0x%x, intr=0x%x\n", ++ class, instance, intr); ++} ++ ++static void ++gen11_gt_bank_handler(struct drm_i915_private * const i915, ++ const unsigned int bank) ++{ ++ void __iomem * const regs = i915->uncore.regs; ++ unsigned long intr_dw; ++ unsigned int bit; ++ ++ lockdep_assert_held(&i915->irq_lock); ++ ++ intr_dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank)); ++ ++ for_each_set_bit(bit, &intr_dw, 32) { ++ const u32 ident = gen11_gt_engine_identity(i915, bank, bit); ++ ++ gen11_gt_identity_handler(i915, ident); ++ } ++ ++ /* Clear must be after shared has been served for engine */ ++ raw_reg_write(regs, GEN11_GT_INTR_DW(bank), intr_dw); ++} ++ ++static void ++gen11_gt_irq_handler(struct drm_i915_private * const i915, ++ const u32 master_ctl) ++{ ++ unsigned int bank; ++ ++ spin_lock(&i915->irq_lock); ++ ++ for (bank = 0; bank < 2; bank++) { ++ if (master_ctl & GEN11_GT_DW_IRQ(bank)) ++ gen11_gt_bank_handler(i915, bank); ++ } ++ ++ spin_unlock(&i915->irq_lock); ++} ++ ++static u32 ++gen11_gu_misc_irq_ack(struct drm_i915_private *dev_priv, const u32 master_ctl) ++{ ++ void __iomem * const regs = dev_priv->uncore.regs; ++ u32 iir; ++ ++ if (!(master_ctl & GEN11_GU_MISC_IRQ)) ++ return 0; ++ ++ iir = raw_reg_read(regs, GEN11_GU_MISC_IIR); ++ if (likely(iir)) ++ raw_reg_write(regs, GEN11_GU_MISC_IIR, iir); ++ ++ return iir; ++} ++ ++static void ++gen11_gu_misc_irq_handler(struct drm_i915_private *dev_priv, const u32 iir) ++{ ++ if (iir & GEN11_GU_MISC_GSE) ++ intel_opregion_asle_intr(dev_priv); ++} ++ ++static inline u32 gen11_master_intr_disable(void __iomem * const regs) ++{ ++ raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, 0); ++ ++ /* ++ * Now with master disabled, get a sample of level indications ++ * for this interrupt. Indications will be cleared on related acks. ++ * New indications can and will light up during processing, ++ * and will generate new interrupt after enabling master. ++ */ ++ return raw_reg_read(regs, GEN11_GFX_MSTR_IRQ); ++} ++ ++static inline void gen11_master_intr_enable(void __iomem * const regs) ++{ ++ raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, GEN11_MASTER_IRQ); ++} ++ ++static irqreturn_t gen11_irq_handler(int irq, void *arg) ++{ ++ struct drm_i915_private * const i915 = to_i915(arg); ++ void __iomem * const regs = i915->uncore.regs; ++ u32 master_ctl; ++ u32 gu_misc_iir; ++ ++ if (!intel_irqs_enabled(i915)) ++ return IRQ_NONE; ++ ++ master_ctl = gen11_master_intr_disable(regs); ++ if (!master_ctl) { ++ gen11_master_intr_enable(regs); ++ return IRQ_NONE; ++ } ++ ++ /* Find, clear, then process each source of interrupt. */ ++ gen11_gt_irq_handler(i915, master_ctl); ++ ++ /* IRQs are synced during runtime_suspend, we don't require a wakeref */ ++ if (master_ctl & GEN11_DISPLAY_IRQ) { ++ const u32 disp_ctl = raw_reg_read(regs, GEN11_DISPLAY_INT_CTL); ++ ++ disable_rpm_wakeref_asserts(i915); ++ /* ++ * GEN11_DISPLAY_INT_CTL has same format as GEN8_MASTER_IRQ ++ * for the display related bits. ++ */ ++ gen8_de_irq_handler(i915, disp_ctl); ++ enable_rpm_wakeref_asserts(i915); ++ } ++ ++ gu_misc_iir = gen11_gu_misc_irq_ack(i915, master_ctl); ++ ++ gen11_master_intr_enable(regs); ++ ++ gen11_gu_misc_irq_handler(i915, gu_misc_iir); ++ ++ return IRQ_HANDLED; ++} ++ ++/* Called from drm generic code, passed 'crtc' which ++ * we use as a pipe index ++ */ ++static int i8xx_enable_vblank(struct drm_device *dev, unsigned int pipe) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ unsigned long irqflags; ++ ++ spin_lock_irqsave(&dev_priv->irq_lock, irqflags); ++ i915_enable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS); ++ spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); ++ ++ return 0; ++} ++ ++static int i945gm_enable_vblank(struct drm_device *dev, unsigned int pipe) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ ++ if (dev_priv->i945gm_vblank.enabled++ == 0) ++ schedule_work(&dev_priv->i945gm_vblank.work); ++ ++ return i8xx_enable_vblank(dev, pipe); ++} ++ ++static int i965_enable_vblank(struct drm_device *dev, unsigned int pipe) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ unsigned long irqflags; ++ ++ spin_lock_irqsave(&dev_priv->irq_lock, irqflags); ++ i915_enable_pipestat(dev_priv, pipe, ++ PIPE_START_VBLANK_INTERRUPT_STATUS); ++ spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); ++ ++ return 0; ++} ++ ++static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ unsigned long irqflags; ++ u32 bit = INTEL_GEN(dev_priv) >= 7 ? ++ DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe); ++ ++ spin_lock_irqsave(&dev_priv->irq_lock, irqflags); ++ ilk_enable_display_irq(dev_priv, bit); ++ spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); ++ ++ /* Even though there is no DMC, frame counter can get stuck when ++ * PSR is active as no frames are generated. ++ */ ++ if (HAS_PSR(dev_priv)) ++ drm_vblank_restore(dev, pipe); ++ ++ return 0; ++} ++ ++static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ unsigned long irqflags; ++ ++ spin_lock_irqsave(&dev_priv->irq_lock, irqflags); ++ bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK); ++ spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); ++ ++ /* Even if there is no DMC, frame counter can get stuck when ++ * PSR is active as no frames are generated, so check only for PSR. ++ */ ++ if (HAS_PSR(dev_priv)) ++ drm_vblank_restore(dev, pipe); ++ ++ return 0; ++} ++ ++/* Called from drm generic code, passed 'crtc' which ++ * we use as a pipe index ++ */ ++static void i8xx_disable_vblank(struct drm_device *dev, unsigned int pipe) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ unsigned long irqflags; ++ ++ spin_lock_irqsave(&dev_priv->irq_lock, irqflags); ++ i915_disable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS); ++ spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); ++} ++ ++static void i945gm_disable_vblank(struct drm_device *dev, unsigned int pipe) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ ++ i8xx_disable_vblank(dev, pipe); ++ ++ if (--dev_priv->i945gm_vblank.enabled == 0) ++ schedule_work(&dev_priv->i945gm_vblank.work); ++} ++ ++static void i965_disable_vblank(struct drm_device *dev, unsigned int pipe) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ unsigned long irqflags; ++ ++ spin_lock_irqsave(&dev_priv->irq_lock, irqflags); ++ i915_disable_pipestat(dev_priv, pipe, ++ PIPE_START_VBLANK_INTERRUPT_STATUS); ++ spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); ++} ++ ++static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ unsigned long irqflags; ++ u32 bit = INTEL_GEN(dev_priv) >= 7 ? ++ DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe); ++ ++ spin_lock_irqsave(&dev_priv->irq_lock, irqflags); ++ ilk_disable_display_irq(dev_priv, bit); ++ spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); ++} ++ ++static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ unsigned long irqflags; ++ ++ spin_lock_irqsave(&dev_priv->irq_lock, irqflags); ++ bdw_disable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK); ++ spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); ++} ++ ++static void i945gm_vblank_work_func(struct work_struct *work) ++{ ++ struct drm_i915_private *dev_priv = ++ container_of(work, struct drm_i915_private, i945gm_vblank.work); ++ ++ /* ++ * Vblank interrupts fail to wake up the device from C3, ++ * hence we want to prevent C3 usage while vblank interrupts ++ * are enabled. ++ */ ++ pm_qos_update_request(&dev_priv->i945gm_vblank.pm_qos, ++ READ_ONCE(dev_priv->i945gm_vblank.enabled) ? ++ dev_priv->i945gm_vblank.c3_disable_latency : ++ PM_QOS_DEFAULT_VALUE); ++} ++ ++static int cstate_disable_latency(const char *name) ++{ ++ const struct cpuidle_driver *drv; ++ int i; ++ ++ drv = cpuidle_get_driver(); ++ if (!drv) ++ return 0; ++ ++ for (i = 0; i < drv->state_count; i++) { ++ const struct cpuidle_state *state = &drv->states[i]; ++ ++ if (!strcmp(state->name, name)) ++ return state->exit_latency ? ++ state->exit_latency - 1 : 0; ++ } ++ ++ return 0; ++} ++ ++static void i945gm_vblank_work_init(struct drm_i915_private *dev_priv) ++{ ++ INIT_WORK(&dev_priv->i945gm_vblank.work, ++ i945gm_vblank_work_func); ++ ++ dev_priv->i945gm_vblank.c3_disable_latency = ++ cstate_disable_latency("C3"); ++ pm_qos_add_request(&dev_priv->i945gm_vblank.pm_qos, ++ PM_QOS_CPU_DMA_LATENCY, ++ PM_QOS_DEFAULT_VALUE); ++} ++ ++static void i945gm_vblank_work_fini(struct drm_i915_private *dev_priv) ++{ ++ cancel_work_sync(&dev_priv->i945gm_vblank.work); ++ pm_qos_remove_request(&dev_priv->i945gm_vblank.pm_qos); ++} ++ ++static void ibx_irq_reset(struct drm_i915_private *dev_priv) ++{ ++ struct intel_uncore *uncore = &dev_priv->uncore; ++ ++ if (HAS_PCH_NOP(dev_priv)) ++ return; ++ ++ GEN3_IRQ_RESET(uncore, SDE); ++ ++ if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv)) ++ I915_WRITE(SERR_INT, 0xffffffff); ++} ++ ++/* ++ * SDEIER is also touched by the interrupt handler to work around missed PCH ++ * interrupts. Hence we can't update it after the interrupt handler is enabled - ++ * instead we unconditionally enable all PCH interrupt sources here, but then ++ * only unmask them as needed with SDEIMR. ++ * ++ * This function needs to be called before interrupts are enabled. ++ */ ++static void ibx_irq_pre_postinstall(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ ++ if (HAS_PCH_NOP(dev_priv)) ++ return; ++ ++ WARN_ON(I915_READ(SDEIER) != 0); ++ I915_WRITE(SDEIER, 0xffffffff); ++ POSTING_READ(SDEIER); ++} ++ ++static void gen5_gt_irq_reset(struct drm_i915_private *dev_priv) ++{ ++ struct intel_uncore *uncore = &dev_priv->uncore; ++ ++ GEN3_IRQ_RESET(uncore, GT); ++ if (INTEL_GEN(dev_priv) >= 6) ++ GEN3_IRQ_RESET(uncore, GEN6_PM); ++} ++ ++static void vlv_display_irq_reset(struct drm_i915_private *dev_priv) ++{ ++ struct intel_uncore *uncore = &dev_priv->uncore; ++ ++ if (IS_CHERRYVIEW(dev_priv)) ++ I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV); ++ else ++ I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK); ++ ++ i915_hotplug_interrupt_update_locked(dev_priv, 0xffffffff, 0); ++ I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); ++ ++ i9xx_pipestat_irq_reset(dev_priv); ++ ++ GEN3_IRQ_RESET(uncore, VLV_); ++ dev_priv->irq_mask = ~0u; ++} ++ ++static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv) ++{ ++ struct intel_uncore *uncore = &dev_priv->uncore; ++ ++ u32 pipestat_mask; ++ u32 enable_mask; ++ enum pipe pipe; ++ ++ pipestat_mask = PIPE_CRC_DONE_INTERRUPT_STATUS; ++ ++ i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS); ++ for_each_pipe(dev_priv, pipe) ++ i915_enable_pipestat(dev_priv, pipe, pipestat_mask); ++ ++ enable_mask = I915_DISPLAY_PORT_INTERRUPT | ++ I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | ++ I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | ++ I915_LPE_PIPE_A_INTERRUPT | ++ I915_LPE_PIPE_B_INTERRUPT; ++ ++ if (IS_CHERRYVIEW(dev_priv)) ++ enable_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT | ++ I915_LPE_PIPE_C_INTERRUPT; ++ ++ WARN_ON(dev_priv->irq_mask != ~0u); ++ ++ dev_priv->irq_mask = ~enable_mask; ++ ++ GEN3_IRQ_INIT(uncore, VLV_, dev_priv->irq_mask, enable_mask); ++} ++ ++/* drm_dma.h hooks ++*/ ++static void ironlake_irq_reset(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_uncore *uncore = &dev_priv->uncore; ++ ++ GEN3_IRQ_RESET(uncore, DE); ++ if (IS_GEN(dev_priv, 7)) ++ I915_WRITE(GEN7_ERR_INT, 0xffffffff); ++ ++ if (IS_HASWELL(dev_priv)) { ++ I915_WRITE(EDP_PSR_IMR, 0xffffffff); ++ I915_WRITE(EDP_PSR_IIR, 0xffffffff); ++ } ++ ++ gen5_gt_irq_reset(dev_priv); ++ ++ ibx_irq_reset(dev_priv); ++} ++ ++static void valleyview_irq_reset(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ ++ I915_WRITE(VLV_MASTER_IER, 0); ++ POSTING_READ(VLV_MASTER_IER); ++ ++ gen5_gt_irq_reset(dev_priv); ++ ++ spin_lock_irq(&dev_priv->irq_lock); ++ if (dev_priv->display_irqs_enabled) ++ vlv_display_irq_reset(dev_priv); ++ spin_unlock_irq(&dev_priv->irq_lock); ++} ++ ++static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv) ++{ ++ struct intel_uncore *uncore = &dev_priv->uncore; ++ ++ GEN8_IRQ_RESET_NDX(uncore, GT, 0); ++ GEN8_IRQ_RESET_NDX(uncore, GT, 1); ++ GEN8_IRQ_RESET_NDX(uncore, GT, 2); ++ GEN8_IRQ_RESET_NDX(uncore, GT, 3); ++} ++ ++static void gen8_irq_reset(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_uncore *uncore = &dev_priv->uncore; ++ int pipe; ++ ++ gen8_master_intr_disable(dev_priv->uncore.regs); ++ ++ gen8_gt_irq_reset(dev_priv); ++ ++ I915_WRITE(EDP_PSR_IMR, 0xffffffff); ++ I915_WRITE(EDP_PSR_IIR, 0xffffffff); ++ ++ for_each_pipe(dev_priv, pipe) ++ if (intel_display_power_is_enabled(dev_priv, ++ POWER_DOMAIN_PIPE(pipe))) ++ GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe); ++ ++ GEN3_IRQ_RESET(uncore, GEN8_DE_PORT_); ++ GEN3_IRQ_RESET(uncore, GEN8_DE_MISC_); ++ GEN3_IRQ_RESET(uncore, GEN8_PCU_); ++ ++ if (HAS_PCH_SPLIT(dev_priv)) ++ ibx_irq_reset(dev_priv); ++} ++ ++static void gen11_gt_irq_reset(struct drm_i915_private *dev_priv) ++{ ++ /* Disable RCS, BCS, VCS and VECS class engines. */ ++ I915_WRITE(GEN11_RENDER_COPY_INTR_ENABLE, 0); ++ I915_WRITE(GEN11_VCS_VECS_INTR_ENABLE, 0); ++ ++ /* Restore masks irqs on RCS, BCS, VCS and VECS engines. */ ++ I915_WRITE(GEN11_RCS0_RSVD_INTR_MASK, ~0); ++ I915_WRITE(GEN11_BCS_RSVD_INTR_MASK, ~0); ++ I915_WRITE(GEN11_VCS0_VCS1_INTR_MASK, ~0); ++ I915_WRITE(GEN11_VCS2_VCS3_INTR_MASK, ~0); ++ I915_WRITE(GEN11_VECS0_VECS1_INTR_MASK, ~0); ++ ++ I915_WRITE(GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0); ++ I915_WRITE(GEN11_GPM_WGBOXPERF_INTR_MASK, ~0); ++} ++ ++static void gen11_irq_reset(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = dev->dev_private; ++ struct intel_uncore *uncore = &dev_priv->uncore; ++ int pipe; ++ ++ gen11_master_intr_disable(dev_priv->uncore.regs); ++ ++ gen11_gt_irq_reset(dev_priv); ++ ++ I915_WRITE(GEN11_DISPLAY_INT_CTL, 0); ++ ++ I915_WRITE(EDP_PSR_IMR, 0xffffffff); ++ I915_WRITE(EDP_PSR_IIR, 0xffffffff); ++ ++ for_each_pipe(dev_priv, pipe) ++ if (intel_display_power_is_enabled(dev_priv, ++ POWER_DOMAIN_PIPE(pipe))) ++ GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe); ++ ++ GEN3_IRQ_RESET(uncore, GEN8_DE_PORT_); ++ GEN3_IRQ_RESET(uncore, GEN8_DE_MISC_); ++ GEN3_IRQ_RESET(uncore, GEN11_DE_HPD_); ++ GEN3_IRQ_RESET(uncore, GEN11_GU_MISC_); ++ GEN3_IRQ_RESET(uncore, GEN8_PCU_); ++ ++ if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP) ++ GEN3_IRQ_RESET(uncore, SDE); ++} ++ ++void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv, ++ u8 pipe_mask) ++{ ++ struct intel_uncore *uncore = &dev_priv->uncore; ++ ++ u32 extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN; ++ enum pipe pipe; ++ ++ spin_lock_irq(&dev_priv->irq_lock); ++ ++ if (!intel_irqs_enabled(dev_priv)) { ++ spin_unlock_irq(&dev_priv->irq_lock); ++ return; ++ } ++ ++ for_each_pipe_masked(dev_priv, pipe, pipe_mask) ++ GEN8_IRQ_INIT_NDX(uncore, DE_PIPE, pipe, ++ dev_priv->de_irq_mask[pipe], ++ ~dev_priv->de_irq_mask[pipe] | extra_ier); ++ ++ spin_unlock_irq(&dev_priv->irq_lock); ++} ++ ++void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv, ++ u8 pipe_mask) ++{ ++ struct intel_uncore *uncore = &dev_priv->uncore; ++ enum pipe pipe; ++ ++ spin_lock_irq(&dev_priv->irq_lock); ++ ++ if (!intel_irqs_enabled(dev_priv)) { ++ spin_unlock_irq(&dev_priv->irq_lock); ++ return; ++ } ++ ++ for_each_pipe_masked(dev_priv, pipe, pipe_mask) ++ GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe); ++ ++ spin_unlock_irq(&dev_priv->irq_lock); ++ ++ /* make sure we're done processing display irqs */ ++ synchronize_irq(dev_priv->drm.irq); ++} ++ ++static void cherryview_irq_reset(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_uncore *uncore = &dev_priv->uncore; ++ ++ I915_WRITE(GEN8_MASTER_IRQ, 0); ++ POSTING_READ(GEN8_MASTER_IRQ); ++ ++ gen8_gt_irq_reset(dev_priv); ++ ++ GEN3_IRQ_RESET(uncore, GEN8_PCU_); ++ ++ spin_lock_irq(&dev_priv->irq_lock); ++ if (dev_priv->display_irqs_enabled) ++ vlv_display_irq_reset(dev_priv); ++ spin_unlock_irq(&dev_priv->irq_lock); ++} ++ ++static u32 intel_hpd_enabled_irqs(struct drm_i915_private *dev_priv, ++ const u32 hpd[HPD_NUM_PINS]) ++{ ++ struct intel_encoder *encoder; ++ u32 enabled_irqs = 0; ++ ++ for_each_intel_encoder(&dev_priv->drm, encoder) ++ if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED) ++ enabled_irqs |= hpd[encoder->hpd_pin]; ++ ++ return enabled_irqs; ++} ++ ++static void ibx_hpd_detection_setup(struct drm_i915_private *dev_priv) ++{ ++ u32 hotplug; ++ ++ /* ++ * Enable digital hotplug on the PCH, and configure the DP short pulse ++ * duration to 2ms (which is the minimum in the Display Port spec). ++ * The pulse duration bits are reserved on LPT+. ++ */ ++ hotplug = I915_READ(PCH_PORT_HOTPLUG); ++ hotplug &= ~(PORTB_PULSE_DURATION_MASK | ++ PORTC_PULSE_DURATION_MASK | ++ PORTD_PULSE_DURATION_MASK); ++ hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms; ++ hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms; ++ hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms; ++ /* ++ * When CPU and PCH are on the same package, port A ++ * HPD must be enabled in both north and south. ++ */ ++ if (HAS_PCH_LPT_LP(dev_priv)) ++ hotplug |= PORTA_HOTPLUG_ENABLE; ++ I915_WRITE(PCH_PORT_HOTPLUG, hotplug); ++} ++ ++static void ibx_hpd_irq_setup(struct drm_i915_private *dev_priv) ++{ ++ u32 hotplug_irqs, enabled_irqs; ++ ++ if (HAS_PCH_IBX(dev_priv)) { ++ hotplug_irqs = SDE_HOTPLUG_MASK; ++ enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ibx); ++ } else { ++ hotplug_irqs = SDE_HOTPLUG_MASK_CPT; ++ enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_cpt); ++ } ++ ++ ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs); ++ ++ ibx_hpd_detection_setup(dev_priv); ++} ++ ++static void icp_hpd_detection_setup(struct drm_i915_private *dev_priv) ++{ ++ u32 hotplug; ++ ++ hotplug = I915_READ(SHOTPLUG_CTL_DDI); ++ hotplug |= ICP_DDIA_HPD_ENABLE | ++ ICP_DDIB_HPD_ENABLE; ++ I915_WRITE(SHOTPLUG_CTL_DDI, hotplug); ++ ++ hotplug = I915_READ(SHOTPLUG_CTL_TC); ++ hotplug |= ICP_TC_HPD_ENABLE(PORT_TC1) | ++ ICP_TC_HPD_ENABLE(PORT_TC2) | ++ ICP_TC_HPD_ENABLE(PORT_TC3) | ++ ICP_TC_HPD_ENABLE(PORT_TC4); ++ I915_WRITE(SHOTPLUG_CTL_TC, hotplug); ++} ++ ++static void icp_hpd_irq_setup(struct drm_i915_private *dev_priv) ++{ ++ u32 hotplug_irqs, enabled_irqs; ++ ++ hotplug_irqs = SDE_DDI_MASK_ICP | SDE_TC_MASK_ICP; ++ enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_icp); ++ ++ ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs); ++ ++ icp_hpd_detection_setup(dev_priv); ++} ++ ++static void gen11_hpd_detection_setup(struct drm_i915_private *dev_priv) ++{ ++ u32 hotplug; ++ ++ hotplug = I915_READ(GEN11_TC_HOTPLUG_CTL); ++ hotplug |= GEN11_HOTPLUG_CTL_ENABLE(PORT_TC1) | ++ GEN11_HOTPLUG_CTL_ENABLE(PORT_TC2) | ++ GEN11_HOTPLUG_CTL_ENABLE(PORT_TC3) | ++ GEN11_HOTPLUG_CTL_ENABLE(PORT_TC4); ++ I915_WRITE(GEN11_TC_HOTPLUG_CTL, hotplug); ++ ++ hotplug = I915_READ(GEN11_TBT_HOTPLUG_CTL); ++ hotplug |= GEN11_HOTPLUG_CTL_ENABLE(PORT_TC1) | ++ GEN11_HOTPLUG_CTL_ENABLE(PORT_TC2) | ++ GEN11_HOTPLUG_CTL_ENABLE(PORT_TC3) | ++ GEN11_HOTPLUG_CTL_ENABLE(PORT_TC4); ++ I915_WRITE(GEN11_TBT_HOTPLUG_CTL, hotplug); ++} ++ ++static void gen11_hpd_irq_setup(struct drm_i915_private *dev_priv) ++{ ++ u32 hotplug_irqs, enabled_irqs; ++ u32 val; ++ ++ enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_gen11); ++ hotplug_irqs = GEN11_DE_TC_HOTPLUG_MASK | GEN11_DE_TBT_HOTPLUG_MASK; ++ ++ val = I915_READ(GEN11_DE_HPD_IMR); ++ val &= ~hotplug_irqs; ++ I915_WRITE(GEN11_DE_HPD_IMR, val); ++ POSTING_READ(GEN11_DE_HPD_IMR); ++ ++ gen11_hpd_detection_setup(dev_priv); ++ ++ if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP) ++ icp_hpd_irq_setup(dev_priv); ++} ++ ++static void spt_hpd_detection_setup(struct drm_i915_private *dev_priv) ++{ ++ u32 val, hotplug; ++ ++ /* Display WA #1179 WaHardHangonHotPlug: cnp */ ++ if (HAS_PCH_CNP(dev_priv)) { ++ val = I915_READ(SOUTH_CHICKEN1); ++ val &= ~CHASSIS_CLK_REQ_DURATION_MASK; ++ val |= CHASSIS_CLK_REQ_DURATION(0xf); ++ I915_WRITE(SOUTH_CHICKEN1, val); ++ } ++ ++ /* Enable digital hotplug on the PCH */ ++ hotplug = I915_READ(PCH_PORT_HOTPLUG); ++ hotplug |= PORTA_HOTPLUG_ENABLE | ++ PORTB_HOTPLUG_ENABLE | ++ PORTC_HOTPLUG_ENABLE | ++ PORTD_HOTPLUG_ENABLE; ++ I915_WRITE(PCH_PORT_HOTPLUG, hotplug); ++ ++ hotplug = I915_READ(PCH_PORT_HOTPLUG2); ++ hotplug |= PORTE_HOTPLUG_ENABLE; ++ I915_WRITE(PCH_PORT_HOTPLUG2, hotplug); ++} ++ ++static void spt_hpd_irq_setup(struct drm_i915_private *dev_priv) ++{ ++ u32 hotplug_irqs, enabled_irqs; ++ ++ hotplug_irqs = SDE_HOTPLUG_MASK_SPT; ++ enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_spt); ++ ++ ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs); ++ ++ spt_hpd_detection_setup(dev_priv); ++} ++ ++static void ilk_hpd_detection_setup(struct drm_i915_private *dev_priv) ++{ ++ u32 hotplug; ++ ++ /* ++ * Enable digital hotplug on the CPU, and configure the DP short pulse ++ * duration to 2ms (which is the minimum in the Display Port spec) ++ * The pulse duration bits are reserved on HSW+. ++ */ ++ hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL); ++ hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK; ++ hotplug |= DIGITAL_PORTA_HOTPLUG_ENABLE | ++ DIGITAL_PORTA_PULSE_DURATION_2ms; ++ I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug); ++} ++ ++static void ilk_hpd_irq_setup(struct drm_i915_private *dev_priv) ++{ ++ u32 hotplug_irqs, enabled_irqs; ++ ++ if (INTEL_GEN(dev_priv) >= 8) { ++ hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG; ++ enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bdw); ++ ++ bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs); ++ } else if (INTEL_GEN(dev_priv) >= 7) { ++ hotplug_irqs = DE_DP_A_HOTPLUG_IVB; ++ enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ivb); ++ ++ ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs); ++ } else { ++ hotplug_irqs = DE_DP_A_HOTPLUG; ++ enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ilk); ++ ++ ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs); ++ } ++ ++ ilk_hpd_detection_setup(dev_priv); ++ ++ ibx_hpd_irq_setup(dev_priv); ++} ++ ++static void __bxt_hpd_detection_setup(struct drm_i915_private *dev_priv, ++ u32 enabled_irqs) ++{ ++ u32 hotplug; ++ ++ hotplug = I915_READ(PCH_PORT_HOTPLUG); ++ hotplug |= PORTA_HOTPLUG_ENABLE | ++ PORTB_HOTPLUG_ENABLE | ++ PORTC_HOTPLUG_ENABLE; ++ ++ DRM_DEBUG_KMS("Invert bit setting: hp_ctl:%x hp_port:%x\n", ++ hotplug, enabled_irqs); ++ hotplug &= ~BXT_DDI_HPD_INVERT_MASK; ++ ++ /* ++ * For BXT invert bit has to be set based on AOB design ++ * for HPD detection logic, update it based on VBT fields. ++ */ ++ if ((enabled_irqs & BXT_DE_PORT_HP_DDIA) && ++ intel_bios_is_port_hpd_inverted(dev_priv, PORT_A)) ++ hotplug |= BXT_DDIA_HPD_INVERT; ++ if ((enabled_irqs & BXT_DE_PORT_HP_DDIB) && ++ intel_bios_is_port_hpd_inverted(dev_priv, PORT_B)) ++ hotplug |= BXT_DDIB_HPD_INVERT; ++ if ((enabled_irqs & BXT_DE_PORT_HP_DDIC) && ++ intel_bios_is_port_hpd_inverted(dev_priv, PORT_C)) ++ hotplug |= BXT_DDIC_HPD_INVERT; ++ ++ I915_WRITE(PCH_PORT_HOTPLUG, hotplug); ++} ++ ++static void bxt_hpd_detection_setup(struct drm_i915_private *dev_priv) ++{ ++ __bxt_hpd_detection_setup(dev_priv, BXT_DE_PORT_HOTPLUG_MASK); ++} ++ ++static void bxt_hpd_irq_setup(struct drm_i915_private *dev_priv) ++{ ++ u32 hotplug_irqs, enabled_irqs; ++ ++ enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bxt); ++ hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK; ++ ++ bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs); ++ ++ __bxt_hpd_detection_setup(dev_priv, enabled_irqs); ++} ++ ++static void ibx_irq_postinstall(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ u32 mask; ++ ++ if (HAS_PCH_NOP(dev_priv)) ++ return; ++ ++ if (HAS_PCH_IBX(dev_priv)) ++ mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON; ++ else if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv)) ++ mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT; ++ else ++ mask = SDE_GMBUS_CPT; ++ ++ gen3_assert_iir_is_zero(&dev_priv->uncore, SDEIIR); ++ I915_WRITE(SDEIMR, ~mask); ++ ++ if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) || ++ HAS_PCH_LPT(dev_priv)) ++ ibx_hpd_detection_setup(dev_priv); ++ else ++ spt_hpd_detection_setup(dev_priv); ++} ++ ++static void gen5_gt_irq_postinstall(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_uncore *uncore = &dev_priv->uncore; ++ u32 pm_irqs, gt_irqs; ++ ++ pm_irqs = gt_irqs = 0; ++ ++ dev_priv->gt_irq_mask = ~0; ++ if (HAS_L3_DPF(dev_priv)) { ++ /* L3 parity interrupt is always unmasked. */ ++ dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev_priv); ++ gt_irqs |= GT_PARITY_ERROR(dev_priv); ++ } ++ ++ gt_irqs |= GT_RENDER_USER_INTERRUPT; ++ if (IS_GEN(dev_priv, 5)) { ++ gt_irqs |= ILK_BSD_USER_INTERRUPT; ++ } else { ++ gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT; ++ } ++ ++ GEN3_IRQ_INIT(uncore, GT, dev_priv->gt_irq_mask, gt_irqs); ++ ++ if (INTEL_GEN(dev_priv) >= 6) { ++ /* ++ * RPS interrupts will get enabled/disabled on demand when RPS ++ * itself is enabled/disabled. ++ */ ++ if (HAS_ENGINE(dev_priv, VECS0)) { ++ pm_irqs |= PM_VEBOX_USER_INTERRUPT; ++ dev_priv->pm_ier |= PM_VEBOX_USER_INTERRUPT; ++ } ++ ++ dev_priv->pm_imr = 0xffffffff; ++ GEN3_IRQ_INIT(uncore, GEN6_PM, dev_priv->pm_imr, pm_irqs); ++ } ++} ++ ++static int ironlake_irq_postinstall(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_uncore *uncore = &dev_priv->uncore; ++ u32 display_mask, extra_mask; ++ ++ if (INTEL_GEN(dev_priv) >= 7) { ++ display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB | ++ DE_PCH_EVENT_IVB | DE_AUX_CHANNEL_A_IVB); ++ extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB | ++ DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB | ++ DE_DP_A_HOTPLUG_IVB); ++ } else { ++ display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT | ++ DE_AUX_CHANNEL_A | DE_PIPEB_CRC_DONE | ++ DE_PIPEA_CRC_DONE | DE_POISON); ++ extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT | ++ DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN | ++ DE_DP_A_HOTPLUG); ++ } ++ ++ if (IS_HASWELL(dev_priv)) { ++ gen3_assert_iir_is_zero(uncore, EDP_PSR_IIR); ++ intel_psr_irq_control(dev_priv, dev_priv->psr.debug); ++ display_mask |= DE_EDP_PSR_INT_HSW; ++ } ++ ++ dev_priv->irq_mask = ~display_mask; ++ ++ ibx_irq_pre_postinstall(dev); ++ ++ GEN3_IRQ_INIT(uncore, DE, dev_priv->irq_mask, ++ display_mask | extra_mask); ++ ++ gen5_gt_irq_postinstall(dev); ++ ++ ilk_hpd_detection_setup(dev_priv); ++ ++ ibx_irq_postinstall(dev); ++ ++ if (IS_IRONLAKE_M(dev_priv)) { ++ /* Enable PCU event interrupts ++ * ++ * spinlocking not required here for correctness since interrupt ++ * setup is guaranteed to run in single-threaded context. But we ++ * need it to make the assert_spin_locked happy. */ ++ spin_lock_irq(&dev_priv->irq_lock); ++ ilk_enable_display_irq(dev_priv, DE_PCU_EVENT); ++ spin_unlock_irq(&dev_priv->irq_lock); ++ } ++ ++ return 0; ++} ++ ++void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv) ++{ ++ lockdep_assert_held(&dev_priv->irq_lock); ++ ++ if (dev_priv->display_irqs_enabled) ++ return; ++ ++ dev_priv->display_irqs_enabled = true; ++ ++ if (intel_irqs_enabled(dev_priv)) { ++ vlv_display_irq_reset(dev_priv); ++ vlv_display_irq_postinstall(dev_priv); ++ } ++} ++ ++void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv) ++{ ++ lockdep_assert_held(&dev_priv->irq_lock); ++ ++ if (!dev_priv->display_irqs_enabled) ++ return; ++ ++ dev_priv->display_irqs_enabled = false; ++ ++ if (intel_irqs_enabled(dev_priv)) ++ vlv_display_irq_reset(dev_priv); ++} ++ ++ ++static int valleyview_irq_postinstall(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ ++ gen5_gt_irq_postinstall(dev); ++ ++ spin_lock_irq(&dev_priv->irq_lock); ++ if (dev_priv->display_irqs_enabled) ++ vlv_display_irq_postinstall(dev_priv); ++ spin_unlock_irq(&dev_priv->irq_lock); ++ ++ I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE); ++ POSTING_READ(VLV_MASTER_IER); ++ ++ return 0; ++} ++ ++static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv) ++{ ++ struct intel_uncore *uncore = &dev_priv->uncore; ++ ++ /* These are interrupts we'll toggle with the ring mask register */ ++ u32 gt_interrupts[] = { ++ (GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT | ++ GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT | ++ GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT | ++ GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT), ++ ++ (GT_RENDER_USER_INTERRUPT << GEN8_VCS0_IRQ_SHIFT | ++ GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS0_IRQ_SHIFT | ++ GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT | ++ GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT), ++ ++ 0, ++ ++ (GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT | ++ GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT) ++ }; ++ ++ dev_priv->pm_ier = 0x0; ++ dev_priv->pm_imr = ~dev_priv->pm_ier; ++ GEN8_IRQ_INIT_NDX(uncore, GT, 0, ~gt_interrupts[0], gt_interrupts[0]); ++ GEN8_IRQ_INIT_NDX(uncore, GT, 1, ~gt_interrupts[1], gt_interrupts[1]); ++ /* ++ * RPS interrupts will get enabled/disabled on demand when RPS itself ++ * is enabled/disabled. Same wil be the case for GuC interrupts. ++ */ ++ GEN8_IRQ_INIT_NDX(uncore, GT, 2, dev_priv->pm_imr, dev_priv->pm_ier); ++ GEN8_IRQ_INIT_NDX(uncore, GT, 3, ~gt_interrupts[3], gt_interrupts[3]); ++} ++ ++static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv) ++{ ++ struct intel_uncore *uncore = &dev_priv->uncore; ++ ++ u32 de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE; ++ u32 de_pipe_enables; ++ u32 de_port_masked = GEN8_AUX_CHANNEL_A; ++ u32 de_port_enables; ++ u32 de_misc_masked = GEN8_DE_EDP_PSR; ++ enum pipe pipe; ++ ++ if (INTEL_GEN(dev_priv) <= 10) ++ de_misc_masked |= GEN8_DE_MISC_GSE; ++ ++ if (INTEL_GEN(dev_priv) >= 9) { ++ de_pipe_masked |= GEN9_DE_PIPE_IRQ_FAULT_ERRORS; ++ de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C | ++ GEN9_AUX_CHANNEL_D; ++ if (IS_GEN9_LP(dev_priv)) ++ de_port_masked |= BXT_DE_PORT_GMBUS; ++ } else { ++ de_pipe_masked |= GEN8_DE_PIPE_IRQ_FAULT_ERRORS; ++ } ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ de_port_masked |= ICL_AUX_CHANNEL_E; ++ ++ if (IS_CNL_WITH_PORT_F(dev_priv) || INTEL_GEN(dev_priv) >= 11) ++ de_port_masked |= CNL_AUX_CHANNEL_F; ++ ++ de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK | ++ GEN8_PIPE_FIFO_UNDERRUN; ++ ++ de_port_enables = de_port_masked; ++ if (IS_GEN9_LP(dev_priv)) ++ de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK; ++ else if (IS_BROADWELL(dev_priv)) ++ de_port_enables |= GEN8_PORT_DP_A_HOTPLUG; ++ ++ gen3_assert_iir_is_zero(uncore, EDP_PSR_IIR); ++ intel_psr_irq_control(dev_priv, dev_priv->psr.debug); ++ ++ for_each_pipe(dev_priv, pipe) { ++ dev_priv->de_irq_mask[pipe] = ~de_pipe_masked; ++ ++ if (intel_display_power_is_enabled(dev_priv, ++ POWER_DOMAIN_PIPE(pipe))) ++ GEN8_IRQ_INIT_NDX(uncore, DE_PIPE, pipe, ++ dev_priv->de_irq_mask[pipe], ++ de_pipe_enables); ++ } ++ ++ GEN3_IRQ_INIT(uncore, GEN8_DE_PORT_, ~de_port_masked, de_port_enables); ++ GEN3_IRQ_INIT(uncore, GEN8_DE_MISC_, ~de_misc_masked, de_misc_masked); ++ ++ if (INTEL_GEN(dev_priv) >= 11) { ++ u32 de_hpd_masked = 0; ++ u32 de_hpd_enables = GEN11_DE_TC_HOTPLUG_MASK | ++ GEN11_DE_TBT_HOTPLUG_MASK; ++ ++ GEN3_IRQ_INIT(uncore, GEN11_DE_HPD_, ~de_hpd_masked, ++ de_hpd_enables); ++ gen11_hpd_detection_setup(dev_priv); ++ } else if (IS_GEN9_LP(dev_priv)) { ++ bxt_hpd_detection_setup(dev_priv); ++ } else if (IS_BROADWELL(dev_priv)) { ++ ilk_hpd_detection_setup(dev_priv); ++ } ++} ++ ++static int gen8_irq_postinstall(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ ++ if (HAS_PCH_SPLIT(dev_priv)) ++ ibx_irq_pre_postinstall(dev); ++ ++ gen8_gt_irq_postinstall(dev_priv); ++ gen8_de_irq_postinstall(dev_priv); ++ ++ if (HAS_PCH_SPLIT(dev_priv)) ++ ibx_irq_postinstall(dev); ++ ++ gen8_master_intr_enable(dev_priv->uncore.regs); ++ ++ return 0; ++} ++ ++static void gen11_gt_irq_postinstall(struct drm_i915_private *dev_priv) ++{ ++ const u32 irqs = GT_RENDER_USER_INTERRUPT | GT_CONTEXT_SWITCH_INTERRUPT; ++ ++ BUILD_BUG_ON(irqs & 0xffff0000); ++ ++ /* Enable RCS, BCS, VCS and VECS class interrupts. */ ++ I915_WRITE(GEN11_RENDER_COPY_INTR_ENABLE, irqs << 16 | irqs); ++ I915_WRITE(GEN11_VCS_VECS_INTR_ENABLE, irqs << 16 | irqs); ++ ++ /* Unmask irqs on RCS, BCS, VCS and VECS engines. */ ++ I915_WRITE(GEN11_RCS0_RSVD_INTR_MASK, ~(irqs << 16)); ++ I915_WRITE(GEN11_BCS_RSVD_INTR_MASK, ~(irqs << 16)); ++ I915_WRITE(GEN11_VCS0_VCS1_INTR_MASK, ~(irqs | irqs << 16)); ++ I915_WRITE(GEN11_VCS2_VCS3_INTR_MASK, ~(irqs | irqs << 16)); ++ I915_WRITE(GEN11_VECS0_VECS1_INTR_MASK, ~(irqs | irqs << 16)); ++ ++ /* ++ * RPS interrupts will get enabled/disabled on demand when RPS itself ++ * is enabled/disabled. ++ */ ++ dev_priv->pm_ier = 0x0; ++ dev_priv->pm_imr = ~dev_priv->pm_ier; ++ I915_WRITE(GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0); ++ I915_WRITE(GEN11_GPM_WGBOXPERF_INTR_MASK, ~0); ++} ++ ++static void icp_irq_postinstall(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ u32 mask = SDE_GMBUS_ICP; ++ ++ WARN_ON(I915_READ(SDEIER) != 0); ++ I915_WRITE(SDEIER, 0xffffffff); ++ POSTING_READ(SDEIER); ++ ++ gen3_assert_iir_is_zero(&dev_priv->uncore, SDEIIR); ++ I915_WRITE(SDEIMR, ~mask); ++ ++ icp_hpd_detection_setup(dev_priv); ++} ++ ++static int gen11_irq_postinstall(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = dev->dev_private; ++ struct intel_uncore *uncore = &dev_priv->uncore; ++ u32 gu_misc_masked = GEN11_GU_MISC_GSE; ++ ++ if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP) ++ icp_irq_postinstall(dev); ++ ++ gen11_gt_irq_postinstall(dev_priv); ++ gen8_de_irq_postinstall(dev_priv); ++ ++ GEN3_IRQ_INIT(uncore, GEN11_GU_MISC_, ~gu_misc_masked, gu_misc_masked); ++ ++ I915_WRITE(GEN11_DISPLAY_INT_CTL, GEN11_DISPLAY_IRQ_ENABLE); ++ ++ gen11_master_intr_enable(dev_priv->uncore.regs); ++ POSTING_READ(GEN11_GFX_MSTR_IRQ); ++ ++ return 0; ++} ++ ++static int cherryview_irq_postinstall(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ ++ gen8_gt_irq_postinstall(dev_priv); ++ ++ spin_lock_irq(&dev_priv->irq_lock); ++ if (dev_priv->display_irqs_enabled) ++ vlv_display_irq_postinstall(dev_priv); ++ spin_unlock_irq(&dev_priv->irq_lock); ++ ++ I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL); ++ POSTING_READ(GEN8_MASTER_IRQ); ++ ++ return 0; ++} ++ ++static void i8xx_irq_reset(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_uncore *uncore = &dev_priv->uncore; ++ ++ i9xx_pipestat_irq_reset(dev_priv); ++ ++ GEN2_IRQ_RESET(uncore); ++} ++ ++static int i8xx_irq_postinstall(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_uncore *uncore = &dev_priv->uncore; ++ u16 enable_mask; ++ ++ I915_WRITE16(EMR, ~(I915_ERROR_PAGE_TABLE | ++ I915_ERROR_MEMORY_REFRESH)); ++ ++ /* Unmask the interrupts that we always want on. */ ++ dev_priv->irq_mask = ++ ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | ++ I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | ++ I915_MASTER_ERROR_INTERRUPT); ++ ++ enable_mask = ++ I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | ++ I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | ++ I915_MASTER_ERROR_INTERRUPT | ++ I915_USER_INTERRUPT; ++ ++ GEN2_IRQ_INIT(uncore, dev_priv->irq_mask, enable_mask); ++ ++ /* Interrupt setup is already guaranteed to be single-threaded, this is ++ * just to make the assert_spin_locked check happy. */ ++ spin_lock_irq(&dev_priv->irq_lock); ++ i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS); ++ i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS); ++ spin_unlock_irq(&dev_priv->irq_lock); ++ ++ return 0; ++} ++ ++static void i8xx_error_irq_ack(struct drm_i915_private *dev_priv, ++ u16 *eir, u16 *eir_stuck) ++{ ++ u16 emr; ++ ++ *eir = I915_READ16(EIR); ++ ++ if (*eir) ++ I915_WRITE16(EIR, *eir); ++ ++ *eir_stuck = I915_READ16(EIR); ++ if (*eir_stuck == 0) ++ return; ++ ++ /* ++ * Toggle all EMR bits to make sure we get an edge ++ * in the ISR master error bit if we don't clear ++ * all the EIR bits. Otherwise the edge triggered ++ * IIR on i965/g4x wouldn't notice that an interrupt ++ * is still pending. Also some EIR bits can't be ++ * cleared except by handling the underlying error ++ * (or by a GPU reset) so we mask any bit that ++ * remains set. ++ */ ++ emr = I915_READ16(EMR); ++ I915_WRITE16(EMR, 0xffff); ++ I915_WRITE16(EMR, emr | *eir_stuck); ++} ++ ++static void i8xx_error_irq_handler(struct drm_i915_private *dev_priv, ++ u16 eir, u16 eir_stuck) ++{ ++ DRM_DEBUG("Master Error: EIR 0x%04x\n", eir); ++ ++ if (eir_stuck) ++ DRM_DEBUG_DRIVER("EIR stuck: 0x%04x, masked\n", eir_stuck); ++} ++ ++static void i9xx_error_irq_ack(struct drm_i915_private *dev_priv, ++ u32 *eir, u32 *eir_stuck) ++{ ++ u32 emr; ++ ++ *eir = I915_READ(EIR); ++ ++ I915_WRITE(EIR, *eir); ++ ++ *eir_stuck = I915_READ(EIR); ++ if (*eir_stuck == 0) ++ return; ++ ++ /* ++ * Toggle all EMR bits to make sure we get an edge ++ * in the ISR master error bit if we don't clear ++ * all the EIR bits. Otherwise the edge triggered ++ * IIR on i965/g4x wouldn't notice that an interrupt ++ * is still pending. Also some EIR bits can't be ++ * cleared except by handling the underlying error ++ * (or by a GPU reset) so we mask any bit that ++ * remains set. ++ */ ++ emr = I915_READ(EMR); ++ I915_WRITE(EMR, 0xffffffff); ++ I915_WRITE(EMR, emr | *eir_stuck); ++} ++ ++static void i9xx_error_irq_handler(struct drm_i915_private *dev_priv, ++ u32 eir, u32 eir_stuck) ++{ ++ DRM_DEBUG("Master Error, EIR 0x%08x\n", eir); ++ ++ if (eir_stuck) ++ DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masked\n", eir_stuck); ++} ++ ++static irqreturn_t i8xx_irq_handler(int irq, void *arg) ++{ ++ struct drm_device *dev = arg; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ irqreturn_t ret = IRQ_NONE; ++ ++ if (!intel_irqs_enabled(dev_priv)) ++ return IRQ_NONE; ++ ++ /* IRQs are synced during runtime_suspend, we don't require a wakeref */ ++ disable_rpm_wakeref_asserts(dev_priv); ++ ++ do { ++ u32 pipe_stats[I915_MAX_PIPES] = {}; ++ u16 eir = 0, eir_stuck = 0; ++ u16 iir; ++ ++ iir = I915_READ16(GEN2_IIR); ++ if (iir == 0) ++ break; ++ ++ ret = IRQ_HANDLED; ++ ++ /* Call regardless, as some status bits might not be ++ * signalled in iir */ ++ i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats); ++ ++ if (iir & I915_MASTER_ERROR_INTERRUPT) ++ i8xx_error_irq_ack(dev_priv, &eir, &eir_stuck); ++ ++ I915_WRITE16(GEN2_IIR, iir); ++ ++ if (iir & I915_USER_INTERRUPT) ++ intel_engine_breadcrumbs_irq(dev_priv->engine[RCS0]); ++ ++ if (iir & I915_MASTER_ERROR_INTERRUPT) ++ i8xx_error_irq_handler(dev_priv, eir, eir_stuck); ++ ++ i8xx_pipestat_irq_handler(dev_priv, iir, pipe_stats); ++ } while (0); ++ ++ enable_rpm_wakeref_asserts(dev_priv); ++ ++ return ret; ++} ++ ++static void i915_irq_reset(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_uncore *uncore = &dev_priv->uncore; ++ ++ if (I915_HAS_HOTPLUG(dev_priv)) { ++ i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0); ++ I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); ++ } ++ ++ i9xx_pipestat_irq_reset(dev_priv); ++ ++ GEN3_IRQ_RESET(uncore, GEN2_); ++} ++ ++static int i915_irq_postinstall(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_uncore *uncore = &dev_priv->uncore; ++ u32 enable_mask; ++ ++ I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | ++ I915_ERROR_MEMORY_REFRESH)); ++ ++ /* Unmask the interrupts that we always want on. */ ++ dev_priv->irq_mask = ++ ~(I915_ASLE_INTERRUPT | ++ I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | ++ I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | ++ I915_MASTER_ERROR_INTERRUPT); ++ ++ enable_mask = ++ I915_ASLE_INTERRUPT | ++ I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | ++ I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | ++ I915_MASTER_ERROR_INTERRUPT | ++ I915_USER_INTERRUPT; ++ ++ if (I915_HAS_HOTPLUG(dev_priv)) { ++ /* Enable in IER... */ ++ enable_mask |= I915_DISPLAY_PORT_INTERRUPT; ++ /* and unmask in IMR */ ++ dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT; ++ } ++ ++ GEN3_IRQ_INIT(uncore, GEN2_, dev_priv->irq_mask, enable_mask); ++ ++ /* Interrupt setup is already guaranteed to be single-threaded, this is ++ * just to make the assert_spin_locked check happy. */ ++ spin_lock_irq(&dev_priv->irq_lock); ++ i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS); ++ i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS); ++ spin_unlock_irq(&dev_priv->irq_lock); ++ ++ i915_enable_asle_pipestat(dev_priv); ++ ++ return 0; ++} ++ ++static irqreturn_t i915_irq_handler(int irq, void *arg) ++{ ++ struct drm_device *dev = arg; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ irqreturn_t ret = IRQ_NONE; ++ ++ if (!intel_irqs_enabled(dev_priv)) ++ return IRQ_NONE; ++ ++ /* IRQs are synced during runtime_suspend, we don't require a wakeref */ ++ disable_rpm_wakeref_asserts(dev_priv); ++ ++ do { ++ u32 pipe_stats[I915_MAX_PIPES] = {}; ++ u32 eir = 0, eir_stuck = 0; ++ u32 hotplug_status = 0; ++ u32 iir; ++ ++ iir = I915_READ(GEN2_IIR); ++ if (iir == 0) ++ break; ++ ++ ret = IRQ_HANDLED; ++ ++ if (I915_HAS_HOTPLUG(dev_priv) && ++ iir & I915_DISPLAY_PORT_INTERRUPT) ++ hotplug_status = i9xx_hpd_irq_ack(dev_priv); ++ ++ /* Call regardless, as some status bits might not be ++ * signalled in iir */ ++ i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats); ++ ++ if (iir & I915_MASTER_ERROR_INTERRUPT) ++ i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck); ++ ++ I915_WRITE(GEN2_IIR, iir); ++ ++ if (iir & I915_USER_INTERRUPT) ++ intel_engine_breadcrumbs_irq(dev_priv->engine[RCS0]); ++ ++ if (iir & I915_MASTER_ERROR_INTERRUPT) ++ i9xx_error_irq_handler(dev_priv, eir, eir_stuck); ++ ++ if (hotplug_status) ++ i9xx_hpd_irq_handler(dev_priv, hotplug_status); ++ ++ i915_pipestat_irq_handler(dev_priv, iir, pipe_stats); ++ } while (0); ++ ++ enable_rpm_wakeref_asserts(dev_priv); ++ ++ return ret; ++} ++ ++static void i965_irq_reset(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_uncore *uncore = &dev_priv->uncore; ++ ++ i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0); ++ I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); ++ ++ i9xx_pipestat_irq_reset(dev_priv); ++ ++ GEN3_IRQ_RESET(uncore, GEN2_); ++} ++ ++static int i965_irq_postinstall(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_uncore *uncore = &dev_priv->uncore; ++ u32 enable_mask; ++ u32 error_mask; ++ ++ /* ++ * Enable some error detection, note the instruction error mask ++ * bit is reserved, so we leave it masked. ++ */ ++ if (IS_G4X(dev_priv)) { ++ error_mask = ~(GM45_ERROR_PAGE_TABLE | ++ GM45_ERROR_MEM_PRIV | ++ GM45_ERROR_CP_PRIV | ++ I915_ERROR_MEMORY_REFRESH); ++ } else { ++ error_mask = ~(I915_ERROR_PAGE_TABLE | ++ I915_ERROR_MEMORY_REFRESH); ++ } ++ I915_WRITE(EMR, error_mask); ++ ++ /* Unmask the interrupts that we always want on. */ ++ dev_priv->irq_mask = ++ ~(I915_ASLE_INTERRUPT | ++ I915_DISPLAY_PORT_INTERRUPT | ++ I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | ++ I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | ++ I915_MASTER_ERROR_INTERRUPT); ++ ++ enable_mask = ++ I915_ASLE_INTERRUPT | ++ I915_DISPLAY_PORT_INTERRUPT | ++ I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | ++ I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | ++ I915_MASTER_ERROR_INTERRUPT | ++ I915_USER_INTERRUPT; ++ ++ if (IS_G4X(dev_priv)) ++ enable_mask |= I915_BSD_USER_INTERRUPT; ++ ++ GEN3_IRQ_INIT(uncore, GEN2_, dev_priv->irq_mask, enable_mask); ++ ++ /* Interrupt setup is already guaranteed to be single-threaded, this is ++ * just to make the assert_spin_locked check happy. */ ++ spin_lock_irq(&dev_priv->irq_lock); ++ i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS); ++ i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS); ++ i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS); ++ spin_unlock_irq(&dev_priv->irq_lock); ++ ++ i915_enable_asle_pipestat(dev_priv); ++ ++ return 0; ++} ++ ++static void i915_hpd_irq_setup(struct drm_i915_private *dev_priv) ++{ ++ u32 hotplug_en; ++ ++ lockdep_assert_held(&dev_priv->irq_lock); ++ ++ /* Note HDMI and DP share hotplug bits */ ++ /* enable bits are the same for all generations */ ++ hotplug_en = intel_hpd_enabled_irqs(dev_priv, hpd_mask_i915); ++ /* Programming the CRT detection parameters tends ++ to generate a spurious hotplug event about three ++ seconds later. So just do it once. ++ */ ++ if (IS_G4X(dev_priv)) ++ hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64; ++ hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50; ++ ++ /* Ignore TV since it's buggy */ ++ i915_hotplug_interrupt_update_locked(dev_priv, ++ HOTPLUG_INT_EN_MASK | ++ CRT_HOTPLUG_VOLTAGE_COMPARE_MASK | ++ CRT_HOTPLUG_ACTIVATION_PERIOD_64, ++ hotplug_en); ++} ++ ++static irqreturn_t i965_irq_handler(int irq, void *arg) ++{ ++ struct drm_device *dev = arg; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ irqreturn_t ret = IRQ_NONE; ++ ++ if (!intel_irqs_enabled(dev_priv)) ++ return IRQ_NONE; ++ ++ /* IRQs are synced during runtime_suspend, we don't require a wakeref */ ++ disable_rpm_wakeref_asserts(dev_priv); ++ ++ do { ++ u32 pipe_stats[I915_MAX_PIPES] = {}; ++ u32 eir = 0, eir_stuck = 0; ++ u32 hotplug_status = 0; ++ u32 iir; ++ ++ iir = I915_READ(GEN2_IIR); ++ if (iir == 0) ++ break; ++ ++ ret = IRQ_HANDLED; ++ ++ if (iir & I915_DISPLAY_PORT_INTERRUPT) ++ hotplug_status = i9xx_hpd_irq_ack(dev_priv); ++ ++ /* Call regardless, as some status bits might not be ++ * signalled in iir */ ++ i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats); ++ ++ if (iir & I915_MASTER_ERROR_INTERRUPT) ++ i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck); ++ ++ I915_WRITE(GEN2_IIR, iir); ++ ++ if (iir & I915_USER_INTERRUPT) ++ intel_engine_breadcrumbs_irq(dev_priv->engine[RCS0]); ++ ++ if (iir & I915_BSD_USER_INTERRUPT) ++ intel_engine_breadcrumbs_irq(dev_priv->engine[VCS0]); ++ ++ if (iir & I915_MASTER_ERROR_INTERRUPT) ++ i9xx_error_irq_handler(dev_priv, eir, eir_stuck); ++ ++ if (hotplug_status) ++ i9xx_hpd_irq_handler(dev_priv, hotplug_status); ++ ++ i965_pipestat_irq_handler(dev_priv, iir, pipe_stats); ++ } while (0); ++ ++ enable_rpm_wakeref_asserts(dev_priv); ++ ++ return ret; ++} ++ ++/** ++ * intel_irq_init - initializes irq support ++ * @dev_priv: i915 device instance ++ * ++ * This function initializes all the irq support including work items, timers ++ * and all the vtables. It does not setup the interrupt itself though. ++ */ ++void intel_irq_init(struct drm_i915_private *dev_priv) ++{ ++ struct drm_device *dev = &dev_priv->drm; ++ struct intel_rps *rps = &dev_priv->gt_pm.rps; ++ int i; ++ ++ if (IS_I945GM(dev_priv)) ++ i945gm_vblank_work_init(dev_priv); ++ ++ intel_hpd_init_work(dev_priv); ++ ++ INIT_WORK(&rps->work, gen6_pm_rps_work); ++ ++ INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work); ++ for (i = 0; i < MAX_L3_SLICES; ++i) ++ dev_priv->l3_parity.remap_info[i] = NULL; ++ ++ if (HAS_GUC_SCHED(dev_priv)) ++ dev_priv->pm_guc_events = GEN9_GUC_TO_HOST_INT_EVENT; ++ ++ /* Let's track the enabled rps events */ ++ if (IS_VALLEYVIEW(dev_priv)) ++ /* WaGsvRC0ResidencyMethod:vlv */ ++ dev_priv->pm_rps_events = GEN6_PM_RP_UP_EI_EXPIRED; ++ else ++ dev_priv->pm_rps_events = (GEN6_PM_RP_UP_THRESHOLD | ++ GEN6_PM_RP_DOWN_THRESHOLD | ++ GEN6_PM_RP_DOWN_TIMEOUT); ++ ++ /* We share the register with other engine */ ++ if (INTEL_GEN(dev_priv) > 9) ++ GEM_WARN_ON(dev_priv->pm_rps_events & 0xffff0000); ++ ++ rps->pm_intrmsk_mbz = 0; ++ ++ /* ++ * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer ++ * if GEN6_PM_UP_EI_EXPIRED is masked. ++ * ++ * TODO: verify if this can be reproduced on VLV,CHV. ++ */ ++ if (INTEL_GEN(dev_priv) <= 7) ++ rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED; ++ ++ if (INTEL_GEN(dev_priv) >= 8) ++ rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC; ++ ++ if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv)) ++ dev->driver->get_vblank_counter = g4x_get_vblank_counter; ++ else if (INTEL_GEN(dev_priv) >= 3) ++ dev->driver->get_vblank_counter = i915_get_vblank_counter; ++ ++ dev->vblank_disable_immediate = true; ++ ++ /* Most platforms treat the display irq block as an always-on ++ * power domain. vlv/chv can disable it at runtime and need ++ * special care to avoid writing any of the display block registers ++ * outside of the power domain. We defer setting up the display irqs ++ * in this case to the runtime pm. ++ */ ++ dev_priv->display_irqs_enabled = true; ++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ dev_priv->display_irqs_enabled = false; ++ ++ dev_priv->hotplug.hpd_storm_threshold = HPD_STORM_DEFAULT_THRESHOLD; ++ /* If we have MST support, we want to avoid doing short HPD IRQ storm ++ * detection, as short HPD storms will occur as a natural part of ++ * sideband messaging with MST. ++ * On older platforms however, IRQ storms can occur with both long and ++ * short pulses, as seen on some G4x systems. ++ */ ++ dev_priv->hotplug.hpd_short_storm_enabled = !HAS_DP_MST(dev_priv); ++ ++ dev->driver->get_vblank_timestamp = drm_calc_vbltimestamp_from_scanoutpos; ++ dev->driver->get_scanout_position = i915_get_crtc_scanoutpos; ++ ++ if (IS_CHERRYVIEW(dev_priv)) { ++ dev->driver->irq_handler = cherryview_irq_handler; ++ dev->driver->irq_preinstall = cherryview_irq_reset; ++ dev->driver->irq_postinstall = cherryview_irq_postinstall; ++ dev->driver->irq_uninstall = cherryview_irq_reset; ++ dev->driver->enable_vblank = i965_enable_vblank; ++ dev->driver->disable_vblank = i965_disable_vblank; ++ dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup; ++ } else if (IS_VALLEYVIEW(dev_priv)) { ++ dev->driver->irq_handler = valleyview_irq_handler; ++ dev->driver->irq_preinstall = valleyview_irq_reset; ++ dev->driver->irq_postinstall = valleyview_irq_postinstall; ++ dev->driver->irq_uninstall = valleyview_irq_reset; ++ dev->driver->enable_vblank = i965_enable_vblank; ++ dev->driver->disable_vblank = i965_disable_vblank; ++ dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup; ++ } else if (INTEL_GEN(dev_priv) >= 11) { ++ dev->driver->irq_handler = gen11_irq_handler; ++ dev->driver->irq_preinstall = gen11_irq_reset; ++ dev->driver->irq_postinstall = gen11_irq_postinstall; ++ dev->driver->irq_uninstall = gen11_irq_reset; ++ dev->driver->enable_vblank = gen8_enable_vblank; ++ dev->driver->disable_vblank = gen8_disable_vblank; ++ dev_priv->display.hpd_irq_setup = gen11_hpd_irq_setup; ++ } else if (INTEL_GEN(dev_priv) >= 8) { ++ dev->driver->irq_handler = gen8_irq_handler; ++ dev->driver->irq_preinstall = gen8_irq_reset; ++ dev->driver->irq_postinstall = gen8_irq_postinstall; ++ dev->driver->irq_uninstall = gen8_irq_reset; ++ dev->driver->enable_vblank = gen8_enable_vblank; ++ dev->driver->disable_vblank = gen8_disable_vblank; ++ if (IS_GEN9_LP(dev_priv)) ++ dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup; ++ else if (INTEL_PCH_TYPE(dev_priv) >= PCH_SPT) ++ dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup; ++ else ++ dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup; ++ } else if (HAS_PCH_SPLIT(dev_priv)) { ++ dev->driver->irq_handler = ironlake_irq_handler; ++ dev->driver->irq_preinstall = ironlake_irq_reset; ++ dev->driver->irq_postinstall = ironlake_irq_postinstall; ++ dev->driver->irq_uninstall = ironlake_irq_reset; ++ dev->driver->enable_vblank = ironlake_enable_vblank; ++ dev->driver->disable_vblank = ironlake_disable_vblank; ++ dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup; ++ } else { ++ if (IS_GEN(dev_priv, 2)) { ++ dev->driver->irq_preinstall = i8xx_irq_reset; ++ dev->driver->irq_postinstall = i8xx_irq_postinstall; ++ dev->driver->irq_handler = i8xx_irq_handler; ++ dev->driver->irq_uninstall = i8xx_irq_reset; ++ dev->driver->enable_vblank = i8xx_enable_vblank; ++ dev->driver->disable_vblank = i8xx_disable_vblank; ++ } else if (IS_I945GM(dev_priv)) { ++ dev->driver->irq_preinstall = i915_irq_reset; ++ dev->driver->irq_postinstall = i915_irq_postinstall; ++ dev->driver->irq_uninstall = i915_irq_reset; ++ dev->driver->irq_handler = i915_irq_handler; ++ dev->driver->enable_vblank = i945gm_enable_vblank; ++ dev->driver->disable_vblank = i945gm_disable_vblank; ++ } else if (IS_GEN(dev_priv, 3)) { ++ dev->driver->irq_preinstall = i915_irq_reset; ++ dev->driver->irq_postinstall = i915_irq_postinstall; ++ dev->driver->irq_uninstall = i915_irq_reset; ++ dev->driver->irq_handler = i915_irq_handler; ++ dev->driver->enable_vblank = i8xx_enable_vblank; ++ dev->driver->disable_vblank = i8xx_disable_vblank; ++ } else { ++ dev->driver->irq_preinstall = i965_irq_reset; ++ dev->driver->irq_postinstall = i965_irq_postinstall; ++ dev->driver->irq_uninstall = i965_irq_reset; ++ dev->driver->irq_handler = i965_irq_handler; ++ dev->driver->enable_vblank = i965_enable_vblank; ++ dev->driver->disable_vblank = i965_disable_vblank; ++ } ++ if (I915_HAS_HOTPLUG(dev_priv)) ++ dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup; ++ } ++} ++ ++/** ++ * intel_irq_fini - deinitializes IRQ support ++ * @i915: i915 device instance ++ * ++ * This function deinitializes all the IRQ support. ++ */ ++void intel_irq_fini(struct drm_i915_private *i915) ++{ ++ int i; ++ ++ if (IS_I945GM(i915)) ++ i945gm_vblank_work_fini(i915); ++ ++ for (i = 0; i < MAX_L3_SLICES; ++i) ++ kfree(i915->l3_parity.remap_info[i]); ++} ++ ++/** ++ * intel_irq_install - enables the hardware interrupt ++ * @dev_priv: i915 device instance ++ * ++ * This function enables the hardware interrupt handling, but leaves the hotplug ++ * handling still disabled. It is called after intel_irq_init(). ++ * ++ * In the driver load and resume code we need working interrupts in a few places ++ * but don't want to deal with the hassle of concurrent probe and hotplug ++ * workers. Hence the split into this two-stage approach. ++ */ ++int intel_irq_install(struct drm_i915_private *dev_priv) ++{ ++ /* ++ * We enable some interrupt sources in our postinstall hooks, so mark ++ * interrupts as enabled _before_ actually enabling them to avoid ++ * special cases in our ordering checks. ++ */ ++ dev_priv->runtime_pm.irqs_enabled = true; ++ ++ return drm_irq_install(&dev_priv->drm, dev_priv->drm.pdev->irq); ++} ++ ++/** ++ * intel_irq_uninstall - finilizes all irq handling ++ * @dev_priv: i915 device instance ++ * ++ * This stops interrupt and hotplug handling and unregisters and frees all ++ * resources acquired in the init functions. ++ */ ++void intel_irq_uninstall(struct drm_i915_private *dev_priv) ++{ ++ drm_irq_uninstall(&dev_priv->drm); ++ intel_hpd_cancel_work(dev_priv); ++ dev_priv->runtime_pm.irqs_enabled = false; ++} ++ ++/** ++ * intel_runtime_pm_disable_interrupts - runtime interrupt disabling ++ * @dev_priv: i915 device instance ++ * ++ * This function is used to disable interrupts at runtime, both in the runtime ++ * pm and the system suspend/resume code. ++ */ ++void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv) ++{ ++ dev_priv->drm.driver->irq_uninstall(&dev_priv->drm); ++ dev_priv->runtime_pm.irqs_enabled = false; ++ synchronize_irq(dev_priv->drm.irq); ++} ++ ++/** ++ * intel_runtime_pm_enable_interrupts - runtime interrupt enabling ++ * @dev_priv: i915 device instance ++ * ++ * This function is used to enable interrupts at runtime, both in the runtime ++ * pm and the system suspend/resume code. ++ */ ++void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv) ++{ ++ dev_priv->runtime_pm.irqs_enabled = true; ++ dev_priv->drm.driver->irq_preinstall(&dev_priv->drm); ++ dev_priv->drm.driver->irq_postinstall(&dev_priv->drm); ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_memcpy.c b/drivers/gpu/drm/i915_legacy/i915_memcpy.c +new file mode 100644 +index 000000000000..79f8ec756362 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_memcpy.c +@@ -0,0 +1,106 @@ ++/* ++ * Copyright © 2016 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#include ++#include ++ ++#include "i915_drv.h" ++ ++static DEFINE_STATIC_KEY_FALSE(has_movntdqa); ++ ++#ifdef CONFIG_AS_MOVNTDQA ++static void __memcpy_ntdqa(void *dst, const void *src, unsigned long len) ++{ ++ kernel_fpu_begin(); ++ ++ len >>= 4; ++ while (len >= 4) { ++ asm("movntdqa (%0), %%xmm0\n" ++ "movntdqa 16(%0), %%xmm1\n" ++ "movntdqa 32(%0), %%xmm2\n" ++ "movntdqa 48(%0), %%xmm3\n" ++ "movaps %%xmm0, (%1)\n" ++ "movaps %%xmm1, 16(%1)\n" ++ "movaps %%xmm2, 32(%1)\n" ++ "movaps %%xmm3, 48(%1)\n" ++ :: "r" (src), "r" (dst) : "memory"); ++ src += 64; ++ dst += 64; ++ len -= 4; ++ } ++ while (len--) { ++ asm("movntdqa (%0), %%xmm0\n" ++ "movaps %%xmm0, (%1)\n" ++ :: "r" (src), "r" (dst) : "memory"); ++ src += 16; ++ dst += 16; ++ } ++ ++ kernel_fpu_end(); ++} ++#endif ++ ++/** ++ * i915_memcpy_from_wc: perform an accelerated *aligned* read from WC ++ * @dst: destination pointer ++ * @src: source pointer ++ * @len: how many bytes to copy ++ * ++ * i915_memcpy_from_wc copies @len bytes from @src to @dst using ++ * non-temporal instructions where available. Note that all arguments ++ * (@src, @dst) must be aligned to 16 bytes and @len must be a multiple ++ * of 16. ++ * ++ * To test whether accelerated reads from WC are supported, use ++ * i915_memcpy_from_wc(NULL, NULL, 0); ++ * ++ * Returns true if the copy was successful, false if the preconditions ++ * are not met. ++ */ ++bool i915_memcpy_from_wc(void *dst, const void *src, unsigned long len) ++{ ++ if (unlikely(((unsigned long)dst | (unsigned long)src | len) & 15)) ++ return false; ++ ++#ifdef CONFIG_AS_MOVNTDQA ++ if (static_branch_likely(&has_movntdqa)) { ++ if (likely(len)) ++ __memcpy_ntdqa(dst, src, len); ++ return true; ++ } ++#endif ++ ++ return false; ++} ++ ++void i915_memcpy_init_early(struct drm_i915_private *dev_priv) ++{ ++ /* ++ * Some hypervisors (e.g. KVM) don't support VEX-prefix instructions ++ * emulation. So don't enable movntdqa in hypervisor guest. ++ */ ++ if (static_cpu_has(X86_FEATURE_XMM4_1) && ++ !boot_cpu_has(X86_FEATURE_HYPERVISOR)) ++ static_branch_enable(&has_movntdqa); ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_mm.c b/drivers/gpu/drm/i915_legacy/i915_mm.c +new file mode 100644 +index 000000000000..c23bb29e6d3e +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_mm.c +@@ -0,0 +1,83 @@ ++/* ++ * Copyright © 2014 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#include ++#include ++ ++#include ++ ++#include "i915_drv.h" ++ ++struct remap_pfn { ++ struct mm_struct *mm; ++ unsigned long pfn; ++ pgprot_t prot; ++}; ++ ++static int remap_pfn(pte_t *pte, unsigned long addr, void *data) ++{ ++ struct remap_pfn *r = data; ++ ++ /* Special PTE are not associated with any struct page */ ++ set_pte_at(r->mm, addr, pte, pte_mkspecial(pfn_pte(r->pfn, r->prot))); ++ r->pfn++; ++ ++ return 0; ++} ++ ++/** ++ * remap_io_mapping - remap an IO mapping to userspace ++ * @vma: user vma to map to ++ * @addr: target user address to start at ++ * @pfn: physical address of kernel memory ++ * @size: size of map area ++ * @iomap: the source io_mapping ++ * ++ * Note: this is only safe if the mm semaphore is held when called. ++ */ ++int remap_io_mapping(struct vm_area_struct *vma, ++ unsigned long addr, unsigned long pfn, unsigned long size, ++ struct io_mapping *iomap) ++{ ++ struct remap_pfn r; ++ int err; ++ ++ GEM_BUG_ON((vma->vm_flags & ++ (VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP)) != ++ (VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP)); ++ ++ /* We rely on prevalidation of the io-mapping to skip track_pfn(). */ ++ r.mm = vma->vm_mm; ++ r.pfn = pfn; ++ r.prot = __pgprot((pgprot_val(iomap->prot) & _PAGE_CACHE_MASK) | ++ (pgprot_val(vma->vm_page_prot) & ~_PAGE_CACHE_MASK)); ++ ++ err = apply_to_page_range(r.mm, addr, size, remap_pfn, &r); ++ if (unlikely(err)) { ++ zap_vma_ptes(vma, addr, (r.pfn - pfn) << PAGE_SHIFT); ++ return err; ++ } ++ ++ return 0; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_bdw.c b/drivers/gpu/drm/i915_legacy/i915_oa_bdw.c +new file mode 100644 +index 000000000000..4acdb94555b7 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_bdw.c +@@ -0,0 +1,91 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#include ++ ++#include "i915_drv.h" ++#include "i915_oa_bdw.h" ++ ++static const struct i915_oa_reg b_counter_config_test_oa[] = { ++ { _MMIO(0x2740), 0x00000000 }, ++ { _MMIO(0x2744), 0x00800000 }, ++ { _MMIO(0x2714), 0xf0800000 }, ++ { _MMIO(0x2710), 0x00000000 }, ++ { _MMIO(0x2724), 0xf0800000 }, ++ { _MMIO(0x2720), 0x00000000 }, ++ { _MMIO(0x2770), 0x00000004 }, ++ { _MMIO(0x2774), 0x00000000 }, ++ { _MMIO(0x2778), 0x00000003 }, ++ { _MMIO(0x277c), 0x00000000 }, ++ { _MMIO(0x2780), 0x00000007 }, ++ { _MMIO(0x2784), 0x00000000 }, ++ { _MMIO(0x2788), 0x00100002 }, ++ { _MMIO(0x278c), 0x0000fff7 }, ++ { _MMIO(0x2790), 0x00100002 }, ++ { _MMIO(0x2794), 0x0000ffcf }, ++ { _MMIO(0x2798), 0x00100082 }, ++ { _MMIO(0x279c), 0x0000ffef }, ++ { _MMIO(0x27a0), 0x001000c2 }, ++ { _MMIO(0x27a4), 0x0000ffe7 }, ++ { _MMIO(0x27a8), 0x00100001 }, ++ { _MMIO(0x27ac), 0x0000ffe7 }, ++}; ++ ++static const struct i915_oa_reg flex_eu_config_test_oa[] = { ++}; ++ ++static const struct i915_oa_reg mux_config_test_oa[] = { ++ { _MMIO(0x9840), 0x000000a0 }, ++ { _MMIO(0x9888), 0x198b0000 }, ++ { _MMIO(0x9888), 0x078b0066 }, ++ { _MMIO(0x9888), 0x118b0000 }, ++ { _MMIO(0x9888), 0x258b0000 }, ++ { _MMIO(0x9888), 0x21850008 }, ++ { _MMIO(0x9888), 0x0d834000 }, ++ { _MMIO(0x9888), 0x07844000 }, ++ { _MMIO(0x9888), 0x17804000 }, ++ { _MMIO(0x9888), 0x21800000 }, ++ { _MMIO(0x9888), 0x4f800000 }, ++ { _MMIO(0x9888), 0x41800000 }, ++ { _MMIO(0x9888), 0x31800000 }, ++ { _MMIO(0x9840), 0x00000080 }, ++}; ++ ++static ssize_t ++show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) ++{ ++ return sprintf(buf, "1\n"); ++} ++ ++void ++i915_perf_load_test_config_bdw(struct drm_i915_private *dev_priv) ++{ ++ strlcpy(dev_priv->perf.oa.test_config.uuid, ++ "d6de6f55-e526-4f79-a6a6-d7315c09044e", ++ sizeof(dev_priv->perf.oa.test_config.uuid)); ++ dev_priv->perf.oa.test_config.id = 1; ++ ++ dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa; ++ dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa; ++ dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa; ++ dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.sysfs_metric.name = "d6de6f55-e526-4f79-a6a6-d7315c09044e"; ++ dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs; ++ ++ dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr; ++ ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id"; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_bdw.h b/drivers/gpu/drm/i915_legacy/i915_oa_bdw.h +new file mode 100644 +index 000000000000..0e667f1a8aa1 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_bdw.h +@@ -0,0 +1,15 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#ifndef __I915_OA_BDW_H__ ++#define __I915_OA_BDW_H__ ++ ++extern void i915_perf_load_test_config_bdw(struct drm_i915_private *dev_priv); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_bxt.c b/drivers/gpu/drm/i915_legacy/i915_oa_bxt.c +new file mode 100644 +index 000000000000..a44195c39923 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_bxt.c +@@ -0,0 +1,89 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#include ++ ++#include "i915_drv.h" ++#include "i915_oa_bxt.h" ++ ++static const struct i915_oa_reg b_counter_config_test_oa[] = { ++ { _MMIO(0x2740), 0x00000000 }, ++ { _MMIO(0x2744), 0x00800000 }, ++ { _MMIO(0x2714), 0xf0800000 }, ++ { _MMIO(0x2710), 0x00000000 }, ++ { _MMIO(0x2724), 0xf0800000 }, ++ { _MMIO(0x2720), 0x00000000 }, ++ { _MMIO(0x2770), 0x00000004 }, ++ { _MMIO(0x2774), 0x00000000 }, ++ { _MMIO(0x2778), 0x00000003 }, ++ { _MMIO(0x277c), 0x00000000 }, ++ { _MMIO(0x2780), 0x00000007 }, ++ { _MMIO(0x2784), 0x00000000 }, ++ { _MMIO(0x2788), 0x00100002 }, ++ { _MMIO(0x278c), 0x0000fff7 }, ++ { _MMIO(0x2790), 0x00100002 }, ++ { _MMIO(0x2794), 0x0000ffcf }, ++ { _MMIO(0x2798), 0x00100082 }, ++ { _MMIO(0x279c), 0x0000ffef }, ++ { _MMIO(0x27a0), 0x001000c2 }, ++ { _MMIO(0x27a4), 0x0000ffe7 }, ++ { _MMIO(0x27a8), 0x00100001 }, ++ { _MMIO(0x27ac), 0x0000ffe7 }, ++}; ++ ++static const struct i915_oa_reg flex_eu_config_test_oa[] = { ++}; ++ ++static const struct i915_oa_reg mux_config_test_oa[] = { ++ { _MMIO(0x9840), 0x00000080 }, ++ { _MMIO(0x9888), 0x19800000 }, ++ { _MMIO(0x9888), 0x07800063 }, ++ { _MMIO(0x9888), 0x11800000 }, ++ { _MMIO(0x9888), 0x23810008 }, ++ { _MMIO(0x9888), 0x1d950400 }, ++ { _MMIO(0x9888), 0x0f922000 }, ++ { _MMIO(0x9888), 0x1f908000 }, ++ { _MMIO(0x9888), 0x37900000 }, ++ { _MMIO(0x9888), 0x55900000 }, ++ { _MMIO(0x9888), 0x47900000 }, ++ { _MMIO(0x9888), 0x33900000 }, ++}; ++ ++static ssize_t ++show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) ++{ ++ return sprintf(buf, "1\n"); ++} ++ ++void ++i915_perf_load_test_config_bxt(struct drm_i915_private *dev_priv) ++{ ++ strlcpy(dev_priv->perf.oa.test_config.uuid, ++ "5ee72f5c-092f-421e-8b70-225f7c3e9612", ++ sizeof(dev_priv->perf.oa.test_config.uuid)); ++ dev_priv->perf.oa.test_config.id = 1; ++ ++ dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa; ++ dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa; ++ dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa; ++ dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.sysfs_metric.name = "5ee72f5c-092f-421e-8b70-225f7c3e9612"; ++ dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs; ++ ++ dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr; ++ ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id"; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_bxt.h b/drivers/gpu/drm/i915_legacy/i915_oa_bxt.h +new file mode 100644 +index 000000000000..679e92cf4f1d +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_bxt.h +@@ -0,0 +1,15 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#ifndef __I915_OA_BXT_H__ ++#define __I915_OA_BXT_H__ ++ ++extern void i915_perf_load_test_config_bxt(struct drm_i915_private *dev_priv); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_cflgt2.c b/drivers/gpu/drm/i915_legacy/i915_oa_cflgt2.c +new file mode 100644 +index 000000000000..7f60d51b8761 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_cflgt2.c +@@ -0,0 +1,90 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#include ++ ++#include "i915_drv.h" ++#include "i915_oa_cflgt2.h" ++ ++static const struct i915_oa_reg b_counter_config_test_oa[] = { ++ { _MMIO(0x2740), 0x00000000 }, ++ { _MMIO(0x2744), 0x00800000 }, ++ { _MMIO(0x2714), 0xf0800000 }, ++ { _MMIO(0x2710), 0x00000000 }, ++ { _MMIO(0x2724), 0xf0800000 }, ++ { _MMIO(0x2720), 0x00000000 }, ++ { _MMIO(0x2770), 0x00000004 }, ++ { _MMIO(0x2774), 0x00000000 }, ++ { _MMIO(0x2778), 0x00000003 }, ++ { _MMIO(0x277c), 0x00000000 }, ++ { _MMIO(0x2780), 0x00000007 }, ++ { _MMIO(0x2784), 0x00000000 }, ++ { _MMIO(0x2788), 0x00100002 }, ++ { _MMIO(0x278c), 0x0000fff7 }, ++ { _MMIO(0x2790), 0x00100002 }, ++ { _MMIO(0x2794), 0x0000ffcf }, ++ { _MMIO(0x2798), 0x00100082 }, ++ { _MMIO(0x279c), 0x0000ffef }, ++ { _MMIO(0x27a0), 0x001000c2 }, ++ { _MMIO(0x27a4), 0x0000ffe7 }, ++ { _MMIO(0x27a8), 0x00100001 }, ++ { _MMIO(0x27ac), 0x0000ffe7 }, ++}; ++ ++static const struct i915_oa_reg flex_eu_config_test_oa[] = { ++}; ++ ++static const struct i915_oa_reg mux_config_test_oa[] = { ++ { _MMIO(0x9840), 0x00000080 }, ++ { _MMIO(0x9888), 0x11810000 }, ++ { _MMIO(0x9888), 0x07810013 }, ++ { _MMIO(0x9888), 0x1f810000 }, ++ { _MMIO(0x9888), 0x1d810000 }, ++ { _MMIO(0x9888), 0x1b930040 }, ++ { _MMIO(0x9888), 0x07e54000 }, ++ { _MMIO(0x9888), 0x1f908000 }, ++ { _MMIO(0x9888), 0x11900000 }, ++ { _MMIO(0x9888), 0x37900000 }, ++ { _MMIO(0x9888), 0x53900000 }, ++ { _MMIO(0x9888), 0x45900000 }, ++ { _MMIO(0x9888), 0x33900000 }, ++}; ++ ++static ssize_t ++show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) ++{ ++ return sprintf(buf, "1\n"); ++} ++ ++void ++i915_perf_load_test_config_cflgt2(struct drm_i915_private *dev_priv) ++{ ++ strlcpy(dev_priv->perf.oa.test_config.uuid, ++ "74fb4902-d3d3-4237-9e90-cbdc68d0a446", ++ sizeof(dev_priv->perf.oa.test_config.uuid)); ++ dev_priv->perf.oa.test_config.id = 1; ++ ++ dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa; ++ dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa; ++ dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa; ++ dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.sysfs_metric.name = "74fb4902-d3d3-4237-9e90-cbdc68d0a446"; ++ dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs; ++ ++ dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr; ++ ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id"; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_cflgt2.h b/drivers/gpu/drm/i915_legacy/i915_oa_cflgt2.h +new file mode 100644 +index 000000000000..4d6025559bbe +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_cflgt2.h +@@ -0,0 +1,15 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#ifndef __I915_OA_CFLGT2_H__ ++#define __I915_OA_CFLGT2_H__ ++ ++extern void i915_perf_load_test_config_cflgt2(struct drm_i915_private *dev_priv); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_cflgt3.c b/drivers/gpu/drm/i915_legacy/i915_oa_cflgt3.c +new file mode 100644 +index 000000000000..a92c38e3a0ce +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_cflgt3.c +@@ -0,0 +1,90 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#include ++ ++#include "i915_drv.h" ++#include "i915_oa_cflgt3.h" ++ ++static const struct i915_oa_reg b_counter_config_test_oa[] = { ++ { _MMIO(0x2740), 0x00000000 }, ++ { _MMIO(0x2744), 0x00800000 }, ++ { _MMIO(0x2714), 0xf0800000 }, ++ { _MMIO(0x2710), 0x00000000 }, ++ { _MMIO(0x2724), 0xf0800000 }, ++ { _MMIO(0x2720), 0x00000000 }, ++ { _MMIO(0x2770), 0x00000004 }, ++ { _MMIO(0x2774), 0x00000000 }, ++ { _MMIO(0x2778), 0x00000003 }, ++ { _MMIO(0x277c), 0x00000000 }, ++ { _MMIO(0x2780), 0x00000007 }, ++ { _MMIO(0x2784), 0x00000000 }, ++ { _MMIO(0x2788), 0x00100002 }, ++ { _MMIO(0x278c), 0x0000fff7 }, ++ { _MMIO(0x2790), 0x00100002 }, ++ { _MMIO(0x2794), 0x0000ffcf }, ++ { _MMIO(0x2798), 0x00100082 }, ++ { _MMIO(0x279c), 0x0000ffef }, ++ { _MMIO(0x27a0), 0x001000c2 }, ++ { _MMIO(0x27a4), 0x0000ffe7 }, ++ { _MMIO(0x27a8), 0x00100001 }, ++ { _MMIO(0x27ac), 0x0000ffe7 }, ++}; ++ ++static const struct i915_oa_reg flex_eu_config_test_oa[] = { ++}; ++ ++static const struct i915_oa_reg mux_config_test_oa[] = { ++ { _MMIO(0x9840), 0x00000080 }, ++ { _MMIO(0x9888), 0x11810000 }, ++ { _MMIO(0x9888), 0x07810013 }, ++ { _MMIO(0x9888), 0x1f810000 }, ++ { _MMIO(0x9888), 0x1d810000 }, ++ { _MMIO(0x9888), 0x1b930040 }, ++ { _MMIO(0x9888), 0x07e54000 }, ++ { _MMIO(0x9888), 0x1f908000 }, ++ { _MMIO(0x9888), 0x11900000 }, ++ { _MMIO(0x9888), 0x37900000 }, ++ { _MMIO(0x9888), 0x53900000 }, ++ { _MMIO(0x9888), 0x45900000 }, ++ { _MMIO(0x9888), 0x33900000 }, ++}; ++ ++static ssize_t ++show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) ++{ ++ return sprintf(buf, "1\n"); ++} ++ ++void ++i915_perf_load_test_config_cflgt3(struct drm_i915_private *dev_priv) ++{ ++ strlcpy(dev_priv->perf.oa.test_config.uuid, ++ "577e8e2c-3fa0-4875-8743-3538d585e3b0", ++ sizeof(dev_priv->perf.oa.test_config.uuid)); ++ dev_priv->perf.oa.test_config.id = 1; ++ ++ dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa; ++ dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa; ++ dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa; ++ dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.sysfs_metric.name = "577e8e2c-3fa0-4875-8743-3538d585e3b0"; ++ dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs; ++ ++ dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr; ++ ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id"; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_cflgt3.h b/drivers/gpu/drm/i915_legacy/i915_oa_cflgt3.h +new file mode 100644 +index 000000000000..0697f4077402 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_cflgt3.h +@@ -0,0 +1,15 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#ifndef __I915_OA_CFLGT3_H__ ++#define __I915_OA_CFLGT3_H__ ++ ++extern void i915_perf_load_test_config_cflgt3(struct drm_i915_private *dev_priv); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_chv.c b/drivers/gpu/drm/i915_legacy/i915_oa_chv.c +new file mode 100644 +index 000000000000..71ec889a0114 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_chv.c +@@ -0,0 +1,90 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#include ++ ++#include "i915_drv.h" ++#include "i915_oa_chv.h" ++ ++static const struct i915_oa_reg b_counter_config_test_oa[] = { ++ { _MMIO(0x2740), 0x00000000 }, ++ { _MMIO(0x2744), 0x00800000 }, ++ { _MMIO(0x2714), 0xf0800000 }, ++ { _MMIO(0x2710), 0x00000000 }, ++ { _MMIO(0x2724), 0xf0800000 }, ++ { _MMIO(0x2720), 0x00000000 }, ++ { _MMIO(0x2770), 0x00000004 }, ++ { _MMIO(0x2774), 0x00000000 }, ++ { _MMIO(0x2778), 0x00000003 }, ++ { _MMIO(0x277c), 0x00000000 }, ++ { _MMIO(0x2780), 0x00000007 }, ++ { _MMIO(0x2784), 0x00000000 }, ++ { _MMIO(0x2788), 0x00100002 }, ++ { _MMIO(0x278c), 0x0000fff7 }, ++ { _MMIO(0x2790), 0x00100002 }, ++ { _MMIO(0x2794), 0x0000ffcf }, ++ { _MMIO(0x2798), 0x00100082 }, ++ { _MMIO(0x279c), 0x0000ffef }, ++ { _MMIO(0x27a0), 0x001000c2 }, ++ { _MMIO(0x27a4), 0x0000ffe7 }, ++ { _MMIO(0x27a8), 0x00100001 }, ++ { _MMIO(0x27ac), 0x0000ffe7 }, ++}; ++ ++static const struct i915_oa_reg flex_eu_config_test_oa[] = { ++}; ++ ++static const struct i915_oa_reg mux_config_test_oa[] = { ++ { _MMIO(0x9840), 0x000000a0 }, ++ { _MMIO(0x9888), 0x59800000 }, ++ { _MMIO(0x9888), 0x59800001 }, ++ { _MMIO(0x9888), 0x338b0000 }, ++ { _MMIO(0x9888), 0x258b0066 }, ++ { _MMIO(0x9888), 0x058b0000 }, ++ { _MMIO(0x9888), 0x038b0000 }, ++ { _MMIO(0x9888), 0x03844000 }, ++ { _MMIO(0x9888), 0x47800080 }, ++ { _MMIO(0x9888), 0x57800000 }, ++ { _MMIO(0x1823a4), 0x00000000 }, ++ { _MMIO(0x9888), 0x59800000 }, ++ { _MMIO(0x9840), 0x00000080 }, ++}; ++ ++static ssize_t ++show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) ++{ ++ return sprintf(buf, "1\n"); ++} ++ ++void ++i915_perf_load_test_config_chv(struct drm_i915_private *dev_priv) ++{ ++ strlcpy(dev_priv->perf.oa.test_config.uuid, ++ "4a534b07-cba3-414d-8d60-874830e883aa", ++ sizeof(dev_priv->perf.oa.test_config.uuid)); ++ dev_priv->perf.oa.test_config.id = 1; ++ ++ dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa; ++ dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa; ++ dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa; ++ dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.sysfs_metric.name = "4a534b07-cba3-414d-8d60-874830e883aa"; ++ dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs; ++ ++ dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr; ++ ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id"; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_chv.h b/drivers/gpu/drm/i915_legacy/i915_oa_chv.h +new file mode 100644 +index 000000000000..0986eae3135f +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_chv.h +@@ -0,0 +1,15 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#ifndef __I915_OA_CHV_H__ ++#define __I915_OA_CHV_H__ ++ ++extern void i915_perf_load_test_config_chv(struct drm_i915_private *dev_priv); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_cnl.c b/drivers/gpu/drm/i915_legacy/i915_oa_cnl.c +new file mode 100644 +index 000000000000..5c23d883d6c9 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_cnl.c +@@ -0,0 +1,102 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#include ++ ++#include "i915_drv.h" ++#include "i915_oa_cnl.h" ++ ++static const struct i915_oa_reg b_counter_config_test_oa[] = { ++ { _MMIO(0x2740), 0x00000000 }, ++ { _MMIO(0x2710), 0x00000000 }, ++ { _MMIO(0x2714), 0xf0800000 }, ++ { _MMIO(0x2720), 0x00000000 }, ++ { _MMIO(0x2724), 0xf0800000 }, ++ { _MMIO(0x2770), 0x00000004 }, ++ { _MMIO(0x2774), 0x0000ffff }, ++ { _MMIO(0x2778), 0x00000003 }, ++ { _MMIO(0x277c), 0x0000ffff }, ++ { _MMIO(0x2780), 0x00000007 }, ++ { _MMIO(0x2784), 0x0000ffff }, ++ { _MMIO(0x2788), 0x00100002 }, ++ { _MMIO(0x278c), 0x0000fff7 }, ++ { _MMIO(0x2790), 0x00100002 }, ++ { _MMIO(0x2794), 0x0000ffcf }, ++ { _MMIO(0x2798), 0x00100082 }, ++ { _MMIO(0x279c), 0x0000ffef }, ++ { _MMIO(0x27a0), 0x001000c2 }, ++ { _MMIO(0x27a4), 0x0000ffe7 }, ++ { _MMIO(0x27a8), 0x00100001 }, ++ { _MMIO(0x27ac), 0x0000ffe7 }, ++}; ++ ++static const struct i915_oa_reg flex_eu_config_test_oa[] = { ++}; ++ ++static const struct i915_oa_reg mux_config_test_oa[] = { ++ { _MMIO(0xd04), 0x00000200 }, ++ { _MMIO(0x9884), 0x00000007 }, ++ { _MMIO(0x9888), 0x17060000 }, ++ { _MMIO(0x9840), 0x00000000 }, ++ { _MMIO(0x9884), 0x00000007 }, ++ { _MMIO(0x9888), 0x13034000 }, ++ { _MMIO(0x9884), 0x00000007 }, ++ { _MMIO(0x9888), 0x07060066 }, ++ { _MMIO(0x9884), 0x00000007 }, ++ { _MMIO(0x9888), 0x05060000 }, ++ { _MMIO(0x9884), 0x00000007 }, ++ { _MMIO(0x9888), 0x0f080040 }, ++ { _MMIO(0x9884), 0x00000007 }, ++ { _MMIO(0x9888), 0x07091000 }, ++ { _MMIO(0x9884), 0x00000007 }, ++ { _MMIO(0x9888), 0x0f041000 }, ++ { _MMIO(0x9884), 0x00000007 }, ++ { _MMIO(0x9888), 0x1d004000 }, ++ { _MMIO(0x9884), 0x00000007 }, ++ { _MMIO(0x9888), 0x35000000 }, ++ { _MMIO(0x9884), 0x00000007 }, ++ { _MMIO(0x9888), 0x49000000 }, ++ { _MMIO(0x9884), 0x00000007 }, ++ { _MMIO(0x9888), 0x3d000000 }, ++ { _MMIO(0x9884), 0x00000007 }, ++ { _MMIO(0x9888), 0x31000000 }, ++}; ++ ++static ssize_t ++show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) ++{ ++ return sprintf(buf, "1\n"); ++} ++ ++void ++i915_perf_load_test_config_cnl(struct drm_i915_private *dev_priv) ++{ ++ strlcpy(dev_priv->perf.oa.test_config.uuid, ++ "db41edd4-d8e7-4730-ad11-b9a2d6833503", ++ sizeof(dev_priv->perf.oa.test_config.uuid)); ++ dev_priv->perf.oa.test_config.id = 1; ++ ++ dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa; ++ dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa; ++ dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa; ++ dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.sysfs_metric.name = "db41edd4-d8e7-4730-ad11-b9a2d6833503"; ++ dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs; ++ ++ dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr; ++ ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id"; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_cnl.h b/drivers/gpu/drm/i915_legacy/i915_oa_cnl.h +new file mode 100644 +index 000000000000..e830a406aff2 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_cnl.h +@@ -0,0 +1,15 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#ifndef __I915_OA_CNL_H__ ++#define __I915_OA_CNL_H__ ++ ++extern void i915_perf_load_test_config_cnl(struct drm_i915_private *dev_priv); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_glk.c b/drivers/gpu/drm/i915_legacy/i915_oa_glk.c +new file mode 100644 +index 000000000000..4bdda66df7d2 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_glk.c +@@ -0,0 +1,89 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#include ++ ++#include "i915_drv.h" ++#include "i915_oa_glk.h" ++ ++static const struct i915_oa_reg b_counter_config_test_oa[] = { ++ { _MMIO(0x2740), 0x00000000 }, ++ { _MMIO(0x2744), 0x00800000 }, ++ { _MMIO(0x2714), 0xf0800000 }, ++ { _MMIO(0x2710), 0x00000000 }, ++ { _MMIO(0x2724), 0xf0800000 }, ++ { _MMIO(0x2720), 0x00000000 }, ++ { _MMIO(0x2770), 0x00000004 }, ++ { _MMIO(0x2774), 0x00000000 }, ++ { _MMIO(0x2778), 0x00000003 }, ++ { _MMIO(0x277c), 0x00000000 }, ++ { _MMIO(0x2780), 0x00000007 }, ++ { _MMIO(0x2784), 0x00000000 }, ++ { _MMIO(0x2788), 0x00100002 }, ++ { _MMIO(0x278c), 0x0000fff7 }, ++ { _MMIO(0x2790), 0x00100002 }, ++ { _MMIO(0x2794), 0x0000ffcf }, ++ { _MMIO(0x2798), 0x00100082 }, ++ { _MMIO(0x279c), 0x0000ffef }, ++ { _MMIO(0x27a0), 0x001000c2 }, ++ { _MMIO(0x27a4), 0x0000ffe7 }, ++ { _MMIO(0x27a8), 0x00100001 }, ++ { _MMIO(0x27ac), 0x0000ffe7 }, ++}; ++ ++static const struct i915_oa_reg flex_eu_config_test_oa[] = { ++}; ++ ++static const struct i915_oa_reg mux_config_test_oa[] = { ++ { _MMIO(0x9840), 0x00000080 }, ++ { _MMIO(0x9888), 0x19800000 }, ++ { _MMIO(0x9888), 0x07800063 }, ++ { _MMIO(0x9888), 0x11800000 }, ++ { _MMIO(0x9888), 0x23810008 }, ++ { _MMIO(0x9888), 0x1d950400 }, ++ { _MMIO(0x9888), 0x0f922000 }, ++ { _MMIO(0x9888), 0x1f908000 }, ++ { _MMIO(0x9888), 0x37900000 }, ++ { _MMIO(0x9888), 0x55900000 }, ++ { _MMIO(0x9888), 0x47900000 }, ++ { _MMIO(0x9888), 0x33900000 }, ++}; ++ ++static ssize_t ++show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) ++{ ++ return sprintf(buf, "1\n"); ++} ++ ++void ++i915_perf_load_test_config_glk(struct drm_i915_private *dev_priv) ++{ ++ strlcpy(dev_priv->perf.oa.test_config.uuid, ++ "dd3fd789-e783-4204-8cd0-b671bbccb0cf", ++ sizeof(dev_priv->perf.oa.test_config.uuid)); ++ dev_priv->perf.oa.test_config.id = 1; ++ ++ dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa; ++ dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa; ++ dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa; ++ dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.sysfs_metric.name = "dd3fd789-e783-4204-8cd0-b671bbccb0cf"; ++ dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs; ++ ++ dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr; ++ ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id"; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_glk.h b/drivers/gpu/drm/i915_legacy/i915_oa_glk.h +new file mode 100644 +index 000000000000..06dedf991edb +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_glk.h +@@ -0,0 +1,15 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#ifndef __I915_OA_GLK_H__ ++#define __I915_OA_GLK_H__ ++ ++extern void i915_perf_load_test_config_glk(struct drm_i915_private *dev_priv); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_hsw.c b/drivers/gpu/drm/i915_legacy/i915_oa_hsw.c +new file mode 100644 +index 000000000000..cc6526fdd2bd +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_hsw.c +@@ -0,0 +1,119 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#include ++ ++#include "i915_drv.h" ++#include "i915_oa_hsw.h" ++ ++static const struct i915_oa_reg b_counter_config_render_basic[] = { ++ { _MMIO(0x2724), 0x00800000 }, ++ { _MMIO(0x2720), 0x00000000 }, ++ { _MMIO(0x2714), 0x00800000 }, ++ { _MMIO(0x2710), 0x00000000 }, ++}; ++ ++static const struct i915_oa_reg flex_eu_config_render_basic[] = { ++}; ++ ++static const struct i915_oa_reg mux_config_render_basic[] = { ++ { _MMIO(0x9840), 0x00000080 }, ++ { _MMIO(0x253a4), 0x01600000 }, ++ { _MMIO(0x25440), 0x00100000 }, ++ { _MMIO(0x25128), 0x00000000 }, ++ { _MMIO(0x2691c), 0x00000800 }, ++ { _MMIO(0x26aa0), 0x01500000 }, ++ { _MMIO(0x26b9c), 0x00006000 }, ++ { _MMIO(0x2791c), 0x00000800 }, ++ { _MMIO(0x27aa0), 0x01500000 }, ++ { _MMIO(0x27b9c), 0x00006000 }, ++ { _MMIO(0x2641c), 0x00000400 }, ++ { _MMIO(0x25380), 0x00000010 }, ++ { _MMIO(0x2538c), 0x00000000 }, ++ { _MMIO(0x25384), 0x0800aaaa }, ++ { _MMIO(0x25400), 0x00000004 }, ++ { _MMIO(0x2540c), 0x06029000 }, ++ { _MMIO(0x25410), 0x00000002 }, ++ { _MMIO(0x25404), 0x5c30ffff }, ++ { _MMIO(0x25100), 0x00000016 }, ++ { _MMIO(0x25110), 0x00000400 }, ++ { _MMIO(0x25104), 0x00000000 }, ++ { _MMIO(0x26804), 0x00001211 }, ++ { _MMIO(0x26884), 0x00000100 }, ++ { _MMIO(0x26900), 0x00000002 }, ++ { _MMIO(0x26908), 0x00700000 }, ++ { _MMIO(0x26904), 0x00000000 }, ++ { _MMIO(0x26984), 0x00001022 }, ++ { _MMIO(0x26a04), 0x00000011 }, ++ { _MMIO(0x26a80), 0x00000006 }, ++ { _MMIO(0x26a88), 0x00000c02 }, ++ { _MMIO(0x26a84), 0x00000000 }, ++ { _MMIO(0x26b04), 0x00001000 }, ++ { _MMIO(0x26b80), 0x00000002 }, ++ { _MMIO(0x26b8c), 0x00000007 }, ++ { _MMIO(0x26b84), 0x00000000 }, ++ { _MMIO(0x27804), 0x00004844 }, ++ { _MMIO(0x27884), 0x00000400 }, ++ { _MMIO(0x27900), 0x00000002 }, ++ { _MMIO(0x27908), 0x0e000000 }, ++ { _MMIO(0x27904), 0x00000000 }, ++ { _MMIO(0x27984), 0x00004088 }, ++ { _MMIO(0x27a04), 0x00000044 }, ++ { _MMIO(0x27a80), 0x00000006 }, ++ { _MMIO(0x27a88), 0x00018040 }, ++ { _MMIO(0x27a84), 0x00000000 }, ++ { _MMIO(0x27b04), 0x00004000 }, ++ { _MMIO(0x27b80), 0x00000002 }, ++ { _MMIO(0x27b8c), 0x000000e0 }, ++ { _MMIO(0x27b84), 0x00000000 }, ++ { _MMIO(0x26104), 0x00002222 }, ++ { _MMIO(0x26184), 0x0c006666 }, ++ { _MMIO(0x26284), 0x04000000 }, ++ { _MMIO(0x26304), 0x04000000 }, ++ { _MMIO(0x26400), 0x00000002 }, ++ { _MMIO(0x26410), 0x000000a0 }, ++ { _MMIO(0x26404), 0x00000000 }, ++ { _MMIO(0x25420), 0x04108020 }, ++ { _MMIO(0x25424), 0x1284a420 }, ++ { _MMIO(0x2541c), 0x00000000 }, ++ { _MMIO(0x25428), 0x00042049 }, ++}; ++ ++static ssize_t ++show_render_basic_id(struct device *kdev, struct device_attribute *attr, char *buf) ++{ ++ return sprintf(buf, "1\n"); ++} ++ ++void ++i915_perf_load_test_config_hsw(struct drm_i915_private *dev_priv) ++{ ++ strlcpy(dev_priv->perf.oa.test_config.uuid, ++ "403d8832-1a27-4aa6-a64e-f5389ce7b212", ++ sizeof(dev_priv->perf.oa.test_config.uuid)); ++ dev_priv->perf.oa.test_config.id = 1; ++ ++ dev_priv->perf.oa.test_config.mux_regs = mux_config_render_basic; ++ dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_render_basic); ++ ++ dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_render_basic; ++ dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_render_basic); ++ ++ dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_render_basic; ++ dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_render_basic); ++ ++ dev_priv->perf.oa.test_config.sysfs_metric.name = "403d8832-1a27-4aa6-a64e-f5389ce7b212"; ++ dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs; ++ ++ dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr; ++ ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id"; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_render_basic_id; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_hsw.h b/drivers/gpu/drm/i915_legacy/i915_oa_hsw.h +new file mode 100644 +index 000000000000..3d0c870cd0bd +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_hsw.h +@@ -0,0 +1,15 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#ifndef __I915_OA_HSW_H__ ++#define __I915_OA_HSW_H__ ++ ++extern void i915_perf_load_test_config_hsw(struct drm_i915_private *dev_priv); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_icl.c b/drivers/gpu/drm/i915_legacy/i915_oa_icl.c +new file mode 100644 +index 000000000000..baa51427a543 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_icl.c +@@ -0,0 +1,99 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#include ++ ++#include "i915_drv.h" ++#include "i915_oa_icl.h" ++ ++static const struct i915_oa_reg b_counter_config_test_oa[] = { ++ { _MMIO(0x2740), 0x00000000 }, ++ { _MMIO(0x2710), 0x00000000 }, ++ { _MMIO(0x2714), 0xf0800000 }, ++ { _MMIO(0x2720), 0x00000000 }, ++ { _MMIO(0x2724), 0xf0800000 }, ++ { _MMIO(0x2770), 0x00000004 }, ++ { _MMIO(0x2774), 0x0000ffff }, ++ { _MMIO(0x2778), 0x00000003 }, ++ { _MMIO(0x277c), 0x0000ffff }, ++ { _MMIO(0x2780), 0x00000007 }, ++ { _MMIO(0x2784), 0x0000ffff }, ++ { _MMIO(0x2788), 0x00100002 }, ++ { _MMIO(0x278c), 0x0000fff7 }, ++ { _MMIO(0x2790), 0x00100002 }, ++ { _MMIO(0x2794), 0x0000ffcf }, ++ { _MMIO(0x2798), 0x00100082 }, ++ { _MMIO(0x279c), 0x0000ffef }, ++ { _MMIO(0x27a0), 0x001000c2 }, ++ { _MMIO(0x27a4), 0x0000ffe7 }, ++ { _MMIO(0x27a8), 0x00100001 }, ++ { _MMIO(0x27ac), 0x0000ffe7 }, ++}; ++ ++static const struct i915_oa_reg flex_eu_config_test_oa[] = { ++}; ++ ++static const struct i915_oa_reg mux_config_test_oa[] = { ++ { _MMIO(0xd04), 0x00000200 }, ++ { _MMIO(0x9840), 0x00000000 }, ++ { _MMIO(0x9884), 0x00000000 }, ++ { _MMIO(0x9888), 0x10060000 }, ++ { _MMIO(0x9888), 0x22060000 }, ++ { _MMIO(0x9888), 0x16060000 }, ++ { _MMIO(0x9888), 0x24060000 }, ++ { _MMIO(0x9888), 0x18060000 }, ++ { _MMIO(0x9888), 0x1a060000 }, ++ { _MMIO(0x9888), 0x12060000 }, ++ { _MMIO(0x9888), 0x14060000 }, ++ { _MMIO(0x9888), 0x10060000 }, ++ { _MMIO(0x9888), 0x22060000 }, ++ { _MMIO(0x9884), 0x00000003 }, ++ { _MMIO(0x9888), 0x16130000 }, ++ { _MMIO(0x9888), 0x24000001 }, ++ { _MMIO(0x9888), 0x0e130056 }, ++ { _MMIO(0x9888), 0x10130000 }, ++ { _MMIO(0x9888), 0x1a130000 }, ++ { _MMIO(0x9888), 0x541f0001 }, ++ { _MMIO(0x9888), 0x181f0000 }, ++ { _MMIO(0x9888), 0x4c1f0000 }, ++ { _MMIO(0x9888), 0x301f0000 }, ++}; ++ ++static ssize_t ++show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) ++{ ++ return sprintf(buf, "1\n"); ++} ++ ++void ++i915_perf_load_test_config_icl(struct drm_i915_private *dev_priv) ++{ ++ strlcpy(dev_priv->perf.oa.test_config.uuid, ++ "a291665e-244b-4b76-9b9a-01de9d3c8068", ++ sizeof(dev_priv->perf.oa.test_config.uuid)); ++ dev_priv->perf.oa.test_config.id = 1; ++ ++ dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa; ++ dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa; ++ dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa; ++ dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.sysfs_metric.name = "a291665e-244b-4b76-9b9a-01de9d3c8068"; ++ dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs; ++ ++ dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr; ++ ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id"; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_icl.h b/drivers/gpu/drm/i915_legacy/i915_oa_icl.h +new file mode 100644 +index 000000000000..24eaa97d61ba +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_icl.h +@@ -0,0 +1,15 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#ifndef __I915_OA_ICL_H__ ++#define __I915_OA_ICL_H__ ++ ++extern void i915_perf_load_test_config_icl(struct drm_i915_private *dev_priv); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_kblgt2.c b/drivers/gpu/drm/i915_legacy/i915_oa_kblgt2.c +new file mode 100644 +index 000000000000..168e49ab0d4d +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_kblgt2.c +@@ -0,0 +1,90 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#include ++ ++#include "i915_drv.h" ++#include "i915_oa_kblgt2.h" ++ ++static const struct i915_oa_reg b_counter_config_test_oa[] = { ++ { _MMIO(0x2740), 0x00000000 }, ++ { _MMIO(0x2744), 0x00800000 }, ++ { _MMIO(0x2714), 0xf0800000 }, ++ { _MMIO(0x2710), 0x00000000 }, ++ { _MMIO(0x2724), 0xf0800000 }, ++ { _MMIO(0x2720), 0x00000000 }, ++ { _MMIO(0x2770), 0x00000004 }, ++ { _MMIO(0x2774), 0x00000000 }, ++ { _MMIO(0x2778), 0x00000003 }, ++ { _MMIO(0x277c), 0x00000000 }, ++ { _MMIO(0x2780), 0x00000007 }, ++ { _MMIO(0x2784), 0x00000000 }, ++ { _MMIO(0x2788), 0x00100002 }, ++ { _MMIO(0x278c), 0x0000fff7 }, ++ { _MMIO(0x2790), 0x00100002 }, ++ { _MMIO(0x2794), 0x0000ffcf }, ++ { _MMIO(0x2798), 0x00100082 }, ++ { _MMIO(0x279c), 0x0000ffef }, ++ { _MMIO(0x27a0), 0x001000c2 }, ++ { _MMIO(0x27a4), 0x0000ffe7 }, ++ { _MMIO(0x27a8), 0x00100001 }, ++ { _MMIO(0x27ac), 0x0000ffe7 }, ++}; ++ ++static const struct i915_oa_reg flex_eu_config_test_oa[] = { ++}; ++ ++static const struct i915_oa_reg mux_config_test_oa[] = { ++ { _MMIO(0x9840), 0x00000080 }, ++ { _MMIO(0x9888), 0x11810000 }, ++ { _MMIO(0x9888), 0x07810013 }, ++ { _MMIO(0x9888), 0x1f810000 }, ++ { _MMIO(0x9888), 0x1d810000 }, ++ { _MMIO(0x9888), 0x1b930040 }, ++ { _MMIO(0x9888), 0x07e54000 }, ++ { _MMIO(0x9888), 0x1f908000 }, ++ { _MMIO(0x9888), 0x11900000 }, ++ { _MMIO(0x9888), 0x37900000 }, ++ { _MMIO(0x9888), 0x53900000 }, ++ { _MMIO(0x9888), 0x45900000 }, ++ { _MMIO(0x9888), 0x33900000 }, ++}; ++ ++static ssize_t ++show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) ++{ ++ return sprintf(buf, "1\n"); ++} ++ ++void ++i915_perf_load_test_config_kblgt2(struct drm_i915_private *dev_priv) ++{ ++ strlcpy(dev_priv->perf.oa.test_config.uuid, ++ "baa3c7e4-52b6-4b85-801e-465a94b746dd", ++ sizeof(dev_priv->perf.oa.test_config.uuid)); ++ dev_priv->perf.oa.test_config.id = 1; ++ ++ dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa; ++ dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa; ++ dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa; ++ dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.sysfs_metric.name = "baa3c7e4-52b6-4b85-801e-465a94b746dd"; ++ dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs; ++ ++ dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr; ++ ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id"; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_kblgt2.h b/drivers/gpu/drm/i915_legacy/i915_oa_kblgt2.h +new file mode 100644 +index 000000000000..a55398a904de +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_kblgt2.h +@@ -0,0 +1,15 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#ifndef __I915_OA_KBLGT2_H__ ++#define __I915_OA_KBLGT2_H__ ++ ++extern void i915_perf_load_test_config_kblgt2(struct drm_i915_private *dev_priv); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_kblgt3.c b/drivers/gpu/drm/i915_legacy/i915_oa_kblgt3.c +new file mode 100644 +index 000000000000..6ffa553c388e +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_kblgt3.c +@@ -0,0 +1,90 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#include ++ ++#include "i915_drv.h" ++#include "i915_oa_kblgt3.h" ++ ++static const struct i915_oa_reg b_counter_config_test_oa[] = { ++ { _MMIO(0x2740), 0x00000000 }, ++ { _MMIO(0x2744), 0x00800000 }, ++ { _MMIO(0x2714), 0xf0800000 }, ++ { _MMIO(0x2710), 0x00000000 }, ++ { _MMIO(0x2724), 0xf0800000 }, ++ { _MMIO(0x2720), 0x00000000 }, ++ { _MMIO(0x2770), 0x00000004 }, ++ { _MMIO(0x2774), 0x00000000 }, ++ { _MMIO(0x2778), 0x00000003 }, ++ { _MMIO(0x277c), 0x00000000 }, ++ { _MMIO(0x2780), 0x00000007 }, ++ { _MMIO(0x2784), 0x00000000 }, ++ { _MMIO(0x2788), 0x00100002 }, ++ { _MMIO(0x278c), 0x0000fff7 }, ++ { _MMIO(0x2790), 0x00100002 }, ++ { _MMIO(0x2794), 0x0000ffcf }, ++ { _MMIO(0x2798), 0x00100082 }, ++ { _MMIO(0x279c), 0x0000ffef }, ++ { _MMIO(0x27a0), 0x001000c2 }, ++ { _MMIO(0x27a4), 0x0000ffe7 }, ++ { _MMIO(0x27a8), 0x00100001 }, ++ { _MMIO(0x27ac), 0x0000ffe7 }, ++}; ++ ++static const struct i915_oa_reg flex_eu_config_test_oa[] = { ++}; ++ ++static const struct i915_oa_reg mux_config_test_oa[] = { ++ { _MMIO(0x9840), 0x00000080 }, ++ { _MMIO(0x9888), 0x11810000 }, ++ { _MMIO(0x9888), 0x07810013 }, ++ { _MMIO(0x9888), 0x1f810000 }, ++ { _MMIO(0x9888), 0x1d810000 }, ++ { _MMIO(0x9888), 0x1b930040 }, ++ { _MMIO(0x9888), 0x07e54000 }, ++ { _MMIO(0x9888), 0x1f908000 }, ++ { _MMIO(0x9888), 0x11900000 }, ++ { _MMIO(0x9888), 0x37900000 }, ++ { _MMIO(0x9888), 0x53900000 }, ++ { _MMIO(0x9888), 0x45900000 }, ++ { _MMIO(0x9888), 0x33900000 }, ++}; ++ ++static ssize_t ++show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) ++{ ++ return sprintf(buf, "1\n"); ++} ++ ++void ++i915_perf_load_test_config_kblgt3(struct drm_i915_private *dev_priv) ++{ ++ strlcpy(dev_priv->perf.oa.test_config.uuid, ++ "f1792f32-6db2-4b50-b4b2-557128f1688d", ++ sizeof(dev_priv->perf.oa.test_config.uuid)); ++ dev_priv->perf.oa.test_config.id = 1; ++ ++ dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa; ++ dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa; ++ dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa; ++ dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.sysfs_metric.name = "f1792f32-6db2-4b50-b4b2-557128f1688d"; ++ dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs; ++ ++ dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr; ++ ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id"; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_kblgt3.h b/drivers/gpu/drm/i915_legacy/i915_oa_kblgt3.h +new file mode 100644 +index 000000000000..3ddd3483b7cc +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_kblgt3.h +@@ -0,0 +1,15 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#ifndef __I915_OA_KBLGT3_H__ ++#define __I915_OA_KBLGT3_H__ ++ ++extern void i915_perf_load_test_config_kblgt3(struct drm_i915_private *dev_priv); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_sklgt2.c b/drivers/gpu/drm/i915_legacy/i915_oa_sklgt2.c +new file mode 100644 +index 000000000000..7ce6ee851d43 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_sklgt2.c +@@ -0,0 +1,89 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#include ++ ++#include "i915_drv.h" ++#include "i915_oa_sklgt2.h" ++ ++static const struct i915_oa_reg b_counter_config_test_oa[] = { ++ { _MMIO(0x2740), 0x00000000 }, ++ { _MMIO(0x2714), 0xf0800000 }, ++ { _MMIO(0x2710), 0x00000000 }, ++ { _MMIO(0x2724), 0xf0800000 }, ++ { _MMIO(0x2720), 0x00000000 }, ++ { _MMIO(0x2770), 0x00000004 }, ++ { _MMIO(0x2774), 0x00000000 }, ++ { _MMIO(0x2778), 0x00000003 }, ++ { _MMIO(0x277c), 0x00000000 }, ++ { _MMIO(0x2780), 0x00000007 }, ++ { _MMIO(0x2784), 0x00000000 }, ++ { _MMIO(0x2788), 0x00100002 }, ++ { _MMIO(0x278c), 0x0000fff7 }, ++ { _MMIO(0x2790), 0x00100002 }, ++ { _MMIO(0x2794), 0x0000ffcf }, ++ { _MMIO(0x2798), 0x00100082 }, ++ { _MMIO(0x279c), 0x0000ffef }, ++ { _MMIO(0x27a0), 0x001000c2 }, ++ { _MMIO(0x27a4), 0x0000ffe7 }, ++ { _MMIO(0x27a8), 0x00100001 }, ++ { _MMIO(0x27ac), 0x0000ffe7 }, ++}; ++ ++static const struct i915_oa_reg flex_eu_config_test_oa[] = { ++}; ++ ++static const struct i915_oa_reg mux_config_test_oa[] = { ++ { _MMIO(0x9840), 0x00000080 }, ++ { _MMIO(0x9888), 0x11810000 }, ++ { _MMIO(0x9888), 0x07810016 }, ++ { _MMIO(0x9888), 0x1f810000 }, ++ { _MMIO(0x9888), 0x1d810000 }, ++ { _MMIO(0x9888), 0x1b930040 }, ++ { _MMIO(0x9888), 0x07e54000 }, ++ { _MMIO(0x9888), 0x1f908000 }, ++ { _MMIO(0x9888), 0x11900000 }, ++ { _MMIO(0x9888), 0x37900000 }, ++ { _MMIO(0x9888), 0x53900000 }, ++ { _MMIO(0x9888), 0x45900000 }, ++ { _MMIO(0x9888), 0x33900000 }, ++}; ++ ++static ssize_t ++show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) ++{ ++ return sprintf(buf, "1\n"); ++} ++ ++void ++i915_perf_load_test_config_sklgt2(struct drm_i915_private *dev_priv) ++{ ++ strlcpy(dev_priv->perf.oa.test_config.uuid, ++ "1651949f-0ac0-4cb1-a06f-dafd74a407d1", ++ sizeof(dev_priv->perf.oa.test_config.uuid)); ++ dev_priv->perf.oa.test_config.id = 1; ++ ++ dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa; ++ dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa; ++ dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa; ++ dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.sysfs_metric.name = "1651949f-0ac0-4cb1-a06f-dafd74a407d1"; ++ dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs; ++ ++ dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr; ++ ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id"; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_sklgt2.h b/drivers/gpu/drm/i915_legacy/i915_oa_sklgt2.h +new file mode 100644 +index 000000000000..be6256037239 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_sklgt2.h +@@ -0,0 +1,15 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#ifndef __I915_OA_SKLGT2_H__ ++#define __I915_OA_SKLGT2_H__ ++ ++extern void i915_perf_load_test_config_sklgt2(struct drm_i915_private *dev_priv); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_sklgt3.c b/drivers/gpu/drm/i915_legacy/i915_oa_sklgt3.c +new file mode 100644 +index 000000000000..086ca2631e1c +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_sklgt3.c +@@ -0,0 +1,90 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#include ++ ++#include "i915_drv.h" ++#include "i915_oa_sklgt3.h" ++ ++static const struct i915_oa_reg b_counter_config_test_oa[] = { ++ { _MMIO(0x2740), 0x00000000 }, ++ { _MMIO(0x2744), 0x00800000 }, ++ { _MMIO(0x2714), 0xf0800000 }, ++ { _MMIO(0x2710), 0x00000000 }, ++ { _MMIO(0x2724), 0xf0800000 }, ++ { _MMIO(0x2720), 0x00000000 }, ++ { _MMIO(0x2770), 0x00000004 }, ++ { _MMIO(0x2774), 0x00000000 }, ++ { _MMIO(0x2778), 0x00000003 }, ++ { _MMIO(0x277c), 0x00000000 }, ++ { _MMIO(0x2780), 0x00000007 }, ++ { _MMIO(0x2784), 0x00000000 }, ++ { _MMIO(0x2788), 0x00100002 }, ++ { _MMIO(0x278c), 0x0000fff7 }, ++ { _MMIO(0x2790), 0x00100002 }, ++ { _MMIO(0x2794), 0x0000ffcf }, ++ { _MMIO(0x2798), 0x00100082 }, ++ { _MMIO(0x279c), 0x0000ffef }, ++ { _MMIO(0x27a0), 0x001000c2 }, ++ { _MMIO(0x27a4), 0x0000ffe7 }, ++ { _MMIO(0x27a8), 0x00100001 }, ++ { _MMIO(0x27ac), 0x0000ffe7 }, ++}; ++ ++static const struct i915_oa_reg flex_eu_config_test_oa[] = { ++}; ++ ++static const struct i915_oa_reg mux_config_test_oa[] = { ++ { _MMIO(0x9840), 0x00000080 }, ++ { _MMIO(0x9888), 0x11810000 }, ++ { _MMIO(0x9888), 0x07810013 }, ++ { _MMIO(0x9888), 0x1f810000 }, ++ { _MMIO(0x9888), 0x1d810000 }, ++ { _MMIO(0x9888), 0x1b930040 }, ++ { _MMIO(0x9888), 0x07e54000 }, ++ { _MMIO(0x9888), 0x1f908000 }, ++ { _MMIO(0x9888), 0x11900000 }, ++ { _MMIO(0x9888), 0x37900000 }, ++ { _MMIO(0x9888), 0x53900000 }, ++ { _MMIO(0x9888), 0x45900000 }, ++ { _MMIO(0x9888), 0x33900000 }, ++}; ++ ++static ssize_t ++show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) ++{ ++ return sprintf(buf, "1\n"); ++} ++ ++void ++i915_perf_load_test_config_sklgt3(struct drm_i915_private *dev_priv) ++{ ++ strlcpy(dev_priv->perf.oa.test_config.uuid, ++ "2b985803-d3c9-4629-8a4f-634bfecba0e8", ++ sizeof(dev_priv->perf.oa.test_config.uuid)); ++ dev_priv->perf.oa.test_config.id = 1; ++ ++ dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa; ++ dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa; ++ dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa; ++ dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.sysfs_metric.name = "2b985803-d3c9-4629-8a4f-634bfecba0e8"; ++ dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs; ++ ++ dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr; ++ ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id"; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_sklgt3.h b/drivers/gpu/drm/i915_legacy/i915_oa_sklgt3.h +new file mode 100644 +index 000000000000..650beb068e56 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_sklgt3.h +@@ -0,0 +1,15 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#ifndef __I915_OA_SKLGT3_H__ ++#define __I915_OA_SKLGT3_H__ ++ ++extern void i915_perf_load_test_config_sklgt3(struct drm_i915_private *dev_priv); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_sklgt4.c b/drivers/gpu/drm/i915_legacy/i915_oa_sklgt4.c +new file mode 100644 +index 000000000000..b291a6eb8a87 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_sklgt4.c +@@ -0,0 +1,90 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#include ++ ++#include "i915_drv.h" ++#include "i915_oa_sklgt4.h" ++ ++static const struct i915_oa_reg b_counter_config_test_oa[] = { ++ { _MMIO(0x2740), 0x00000000 }, ++ { _MMIO(0x2744), 0x00800000 }, ++ { _MMIO(0x2714), 0xf0800000 }, ++ { _MMIO(0x2710), 0x00000000 }, ++ { _MMIO(0x2724), 0xf0800000 }, ++ { _MMIO(0x2720), 0x00000000 }, ++ { _MMIO(0x2770), 0x00000004 }, ++ { _MMIO(0x2774), 0x00000000 }, ++ { _MMIO(0x2778), 0x00000003 }, ++ { _MMIO(0x277c), 0x00000000 }, ++ { _MMIO(0x2780), 0x00000007 }, ++ { _MMIO(0x2784), 0x00000000 }, ++ { _MMIO(0x2788), 0x00100002 }, ++ { _MMIO(0x278c), 0x0000fff7 }, ++ { _MMIO(0x2790), 0x00100002 }, ++ { _MMIO(0x2794), 0x0000ffcf }, ++ { _MMIO(0x2798), 0x00100082 }, ++ { _MMIO(0x279c), 0x0000ffef }, ++ { _MMIO(0x27a0), 0x001000c2 }, ++ { _MMIO(0x27a4), 0x0000ffe7 }, ++ { _MMIO(0x27a8), 0x00100001 }, ++ { _MMIO(0x27ac), 0x0000ffe7 }, ++}; ++ ++static const struct i915_oa_reg flex_eu_config_test_oa[] = { ++}; ++ ++static const struct i915_oa_reg mux_config_test_oa[] = { ++ { _MMIO(0x9840), 0x00000080 }, ++ { _MMIO(0x9888), 0x11810000 }, ++ { _MMIO(0x9888), 0x07810013 }, ++ { _MMIO(0x9888), 0x1f810000 }, ++ { _MMIO(0x9888), 0x1d810000 }, ++ { _MMIO(0x9888), 0x1b930040 }, ++ { _MMIO(0x9888), 0x07e54000 }, ++ { _MMIO(0x9888), 0x1f908000 }, ++ { _MMIO(0x9888), 0x11900000 }, ++ { _MMIO(0x9888), 0x37900000 }, ++ { _MMIO(0x9888), 0x53900000 }, ++ { _MMIO(0x9888), 0x45900000 }, ++ { _MMIO(0x9888), 0x33900000 }, ++}; ++ ++static ssize_t ++show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) ++{ ++ return sprintf(buf, "1\n"); ++} ++ ++void ++i915_perf_load_test_config_sklgt4(struct drm_i915_private *dev_priv) ++{ ++ strlcpy(dev_priv->perf.oa.test_config.uuid, ++ "882fa433-1f4a-4a67-a962-c741888fe5f5", ++ sizeof(dev_priv->perf.oa.test_config.uuid)); ++ dev_priv->perf.oa.test_config.id = 1; ++ ++ dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa; ++ dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa; ++ dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa; ++ dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); ++ ++ dev_priv->perf.oa.test_config.sysfs_metric.name = "882fa433-1f4a-4a67-a962-c741888fe5f5"; ++ dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs; ++ ++ dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr; ++ ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id"; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444; ++ dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_oa_sklgt4.h b/drivers/gpu/drm/i915_legacy/i915_oa_sklgt4.h +new file mode 100644 +index 000000000000..8dcf849d131e +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_oa_sklgt4.h +@@ -0,0 +1,15 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ * ++ * Autogenerated file by GPU Top : https://github.com/rib/gputop ++ * DO NOT EDIT manually! ++ */ ++ ++#ifndef __I915_OA_SKLGT4_H__ ++#define __I915_OA_SKLGT4_H__ ++ ++extern void i915_perf_load_test_config_sklgt4(struct drm_i915_private *dev_priv); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_params.c b/drivers/gpu/drm/i915_legacy/i915_params.c +new file mode 100644 +index 000000000000..b5be0abbba35 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_params.c +@@ -0,0 +1,237 @@ ++/* ++ * Copyright © 2014 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the ++ * "Software"), to deal in the Software without restriction, including ++ * without limitation the rights to use, copy, modify, merge, publish, ++ * distribute, sub license, and/or sell copies of the Software, and to ++ * permit persons to whom the Software is furnished to do so, subject to ++ * the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the ++ * next paragraph) shall be included in all copies or substantial portions ++ * of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ */ ++ ++#include ++ ++#include "i915_params.h" ++#include "i915_drv.h" ++ ++#define i915_param_named(name, T, perm, desc) \ ++ module_param_named(name, i915_modparams.name, T, perm); \ ++ MODULE_PARM_DESC(name, desc) ++#define i915_param_named_unsafe(name, T, perm, desc) \ ++ module_param_named_unsafe(name, i915_modparams.name, T, perm); \ ++ MODULE_PARM_DESC(name, desc) ++ ++struct i915_params i915_modparams __read_mostly = { ++#define MEMBER(T, member, value) .member = (value), ++ I915_PARAMS_FOR_EACH(MEMBER) ++#undef MEMBER ++}; ++ ++i915_param_named(modeset, int, 0400, ++ "Use kernel modesetting [KMS] (0=disable, " ++ "1=on, -1=force vga console preference [default])"); ++ ++i915_param_named_unsafe(enable_dc, int, 0400, ++ "Enable power-saving display C-states. " ++ "(-1=auto [default]; 0=disable; 1=up to DC5; 2=up to DC6)"); ++ ++i915_param_named_unsafe(enable_fbc, int, 0600, ++ "Enable frame buffer compression for power savings " ++ "(default: -1 (use per-chip default))"); ++ ++i915_param_named_unsafe(lvds_channel_mode, int, 0400, ++ "Specify LVDS channel mode " ++ "(0=probe BIOS [default], 1=single-channel, 2=dual-channel)"); ++ ++i915_param_named_unsafe(panel_use_ssc, int, 0600, ++ "Use Spread Spectrum Clock with panels [LVDS/eDP] " ++ "(default: auto from VBT)"); ++ ++i915_param_named_unsafe(vbt_sdvo_panel_type, int, 0400, ++ "Override/Ignore selection of SDVO panel mode in the VBT " ++ "(-2=ignore, -1=auto [default], index in VBT BIOS table)"); ++ ++i915_param_named_unsafe(reset, int, 0600, ++ "Attempt GPU resets (0=disabled, 1=full gpu reset, 2=engine reset [default])"); ++ ++i915_param_named_unsafe(vbt_firmware, charp, 0400, ++ "Load VBT from specified file under /lib/firmware"); ++ ++#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) ++i915_param_named(error_capture, bool, 0600, ++ "Record the GPU state following a hang. " ++ "This information in /sys/class/drm/card/error is vital for " ++ "triaging and debugging hangs."); ++#endif ++ ++i915_param_named_unsafe(enable_hangcheck, bool, 0600, ++ "Periodically check GPU activity for detecting hangs. " ++ "WARNING: Disabling this can cause system wide hangs. " ++ "(default: true)"); ++ ++i915_param_named_unsafe(enable_psr, int, 0600, ++ "Enable PSR " ++ "(0=disabled, 1=enabled) " ++ "Default: -1 (use per-chip default)"); ++ ++i915_param_named_unsafe(alpha_support, bool, 0400, ++ "Enable alpha quality driver support for latest hardware. " ++ "See also CONFIG_DRM_I915_ALPHA_SUPPORT."); ++ ++i915_param_named_unsafe(disable_power_well, int, 0400, ++ "Disable display power wells when possible " ++ "(-1=auto [default], 0=power wells always on, 1=power wells disabled when possible)"); ++ ++i915_param_named_unsafe(enable_ips, int, 0600, "Enable IPS (default: true)"); ++ ++i915_param_named(fastboot, int, 0600, ++ "Try to skip unnecessary mode sets at boot time " ++ "(0=disabled, 1=enabled) " ++ "Default: -1 (use per-chip default)"); ++ ++i915_param_named_unsafe(prefault_disable, bool, 0600, ++ "Disable page prefaulting for pread/pwrite/reloc (default:false). " ++ "For developers only."); ++ ++i915_param_named_unsafe(load_detect_test, bool, 0600, ++ "Force-enable the VGA load detect code for testing (default:false). " ++ "For developers only."); ++ ++i915_param_named_unsafe(force_reset_modeset_test, bool, 0600, ++ "Force a modeset during gpu reset for testing (default:false). " ++ "For developers only."); ++ ++i915_param_named_unsafe(invert_brightness, int, 0600, ++ "Invert backlight brightness " ++ "(-1 force normal, 0 machine defaults, 1 force inversion), please " ++ "report PCI device ID, subsystem vendor and subsystem device ID " ++ "to dri-devel@lists.freedesktop.org, if your machine needs it. " ++ "It will then be included in an upcoming module version."); ++ ++i915_param_named(disable_display, bool, 0400, ++ "Disable display (default: false)"); ++ ++i915_param_named(mmio_debug, int, 0600, ++ "Enable the MMIO debug code for the first N failures (default: off). " ++ "This may negatively affect performance."); ++ ++i915_param_named(verbose_state_checks, bool, 0600, ++ "Enable verbose logs (ie. WARN_ON()) in case of unexpected hw state conditions."); ++ ++i915_param_named_unsafe(nuclear_pageflip, bool, 0400, ++ "Force enable atomic functionality on platforms that don't have full support yet."); ++ ++/* WA to get away with the default setting in VBT for early platforms.Will be removed */ ++i915_param_named_unsafe(edp_vswing, int, 0400, ++ "Ignore/Override vswing pre-emph table selection from VBT " ++ "(0=use value from vbt [default], 1=low power swing(200mV)," ++ "2=default swing(400mV))"); ++ ++i915_param_named_unsafe(enable_guc, int, 0400, ++ "Enable GuC load for GuC submission and/or HuC load. " ++ "Required functionality can be selected using bitmask values. " ++ "(-1=auto, 0=disable [default], 1=GuC submission, 2=HuC load)"); ++ ++i915_param_named(guc_log_level, int, 0400, ++ "GuC firmware logging level. Requires GuC to be loaded. " ++ "(-1=auto [default], 0=disable, 1..4=enable with verbosity min..max)"); ++ ++i915_param_named_unsafe(guc_firmware_path, charp, 0400, ++ "GuC firmware path to use instead of the default one"); ++ ++i915_param_named_unsafe(huc_firmware_path, charp, 0400, ++ "HuC firmware path to use instead of the default one"); ++ ++i915_param_named_unsafe(dmc_firmware_path, charp, 0400, ++ "DMC firmware path to use instead of the default one"); ++ ++i915_param_named_unsafe(enable_dp_mst, bool, 0600, ++ "Enable multi-stream transport (MST) for new DisplayPort sinks. (default: true)"); ++ ++#if IS_ENABLED(CONFIG_DRM_I915_DEBUG) ++i915_param_named_unsafe(inject_load_failure, uint, 0400, ++ "Force an error after a number of failure check points (0:disabled (default), N:force failure at the Nth failure check point)"); ++#endif ++ ++i915_param_named(enable_dpcd_backlight, bool, 0600, ++ "Enable support for DPCD backlight control (default:false)"); ++ ++#if IS_ENABLED(CONFIG_DRM_I915_GVT) ++i915_param_named(enable_gvt, bool, 0400, ++ "Enable support for Intel GVT-g graphics virtualization host support(default:false)"); ++#endif ++ ++static __always_inline void _print_param(struct drm_printer *p, ++ const char *name, ++ const char *type, ++ const void *x) ++{ ++ if (!__builtin_strcmp(type, "bool")) ++ drm_printf(p, "i915.%s=%s\n", name, yesno(*(const bool *)x)); ++ else if (!__builtin_strcmp(type, "int")) ++ drm_printf(p, "i915.%s=%d\n", name, *(const int *)x); ++ else if (!__builtin_strcmp(type, "unsigned int")) ++ drm_printf(p, "i915.%s=%u\n", name, *(const unsigned int *)x); ++ else if (!__builtin_strcmp(type, "char *")) ++ drm_printf(p, "i915.%s=%s\n", name, *(const char **)x); ++ else ++ WARN_ONCE(1, "no printer defined for param type %s (i915.%s)\n", ++ type, name); ++} ++ ++/** ++ * i915_params_dump - dump i915 modparams ++ * @params: i915 modparams ++ * @p: the &drm_printer ++ * ++ * Pretty printer for i915 modparams. ++ */ ++void i915_params_dump(const struct i915_params *params, struct drm_printer *p) ++{ ++#define PRINT(T, x, ...) _print_param(p, #x, #T, ¶ms->x); ++ I915_PARAMS_FOR_EACH(PRINT); ++#undef PRINT ++} ++ ++static __always_inline void dup_param(const char *type, void *x) ++{ ++ if (!__builtin_strcmp(type, "char *")) ++ *(void **)x = kstrdup(*(void **)x, GFP_ATOMIC); ++} ++ ++void i915_params_copy(struct i915_params *dest, const struct i915_params *src) ++{ ++ *dest = *src; ++#define DUP(T, x, ...) dup_param(#T, &dest->x); ++ I915_PARAMS_FOR_EACH(DUP); ++#undef DUP ++} ++ ++static __always_inline void free_param(const char *type, void *x) ++{ ++ if (!__builtin_strcmp(type, "char *")) { ++ kfree(*(void **)x); ++ *(void **)x = NULL; ++ } ++} ++ ++/* free the allocated members, *not* the passed in params itself */ ++void i915_params_free(struct i915_params *params) ++{ ++#define FREE(T, x, ...) free_param(#T, ¶ms->x); ++ I915_PARAMS_FOR_EACH(FREE); ++#undef FREE ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_params.h b/drivers/gpu/drm/i915_legacy/i915_params.h +new file mode 100644 +index 000000000000..3f14e9881a0d +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_params.h +@@ -0,0 +1,94 @@ ++/* ++ * Copyright © 2015 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#ifndef _I915_PARAMS_H_ ++#define _I915_PARAMS_H_ ++ ++#include ++#include /* for __read_mostly */ ++ ++struct drm_printer; ++ ++#define ENABLE_GUC_SUBMISSION BIT(0) ++#define ENABLE_GUC_LOAD_HUC BIT(1) ++ ++/* ++ * Invoke param, a function-like macro, for each i915 param, with arguments: ++ * ++ * param(type, name, value) ++ * ++ * type: parameter type, one of {bool, int, unsigned int, char *} ++ * name: name of the parameter ++ * value: initial/default value of the parameter ++ */ ++#define I915_PARAMS_FOR_EACH(param) \ ++ param(char *, vbt_firmware, NULL) \ ++ param(int, modeset, -1) \ ++ param(int, lvds_channel_mode, 0) \ ++ param(int, panel_use_ssc, -1) \ ++ param(int, vbt_sdvo_panel_type, -1) \ ++ param(int, enable_dc, -1) \ ++ param(int, enable_fbc, -1) \ ++ param(int, enable_psr, -1) \ ++ param(int, disable_power_well, -1) \ ++ param(int, enable_ips, 1) \ ++ param(int, invert_brightness, 0) \ ++ param(int, enable_guc, 0) \ ++ param(int, guc_log_level, -1) \ ++ param(char *, guc_firmware_path, NULL) \ ++ param(char *, huc_firmware_path, NULL) \ ++ param(char *, dmc_firmware_path, NULL) \ ++ param(int, mmio_debug, 0) \ ++ param(int, edp_vswing, 0) \ ++ param(int, reset, 2) \ ++ param(unsigned int, inject_load_failure, 0) \ ++ param(int, fastboot, -1) \ ++ /* leave bools at the end to not create holes */ \ ++ param(bool, alpha_support, IS_ENABLED(CONFIG_DRM_I915_ALPHA_SUPPORT)) \ ++ param(bool, enable_hangcheck, true) \ ++ param(bool, prefault_disable, false) \ ++ param(bool, load_detect_test, false) \ ++ param(bool, force_reset_modeset_test, false) \ ++ param(bool, error_capture, true) \ ++ param(bool, disable_display, false) \ ++ param(bool, verbose_state_checks, true) \ ++ param(bool, nuclear_pageflip, false) \ ++ param(bool, enable_dp_mst, true) \ ++ param(bool, enable_dpcd_backlight, false) \ ++ param(bool, enable_gvt, false) ++ ++#define MEMBER(T, member, ...) T member; ++struct i915_params { ++ I915_PARAMS_FOR_EACH(MEMBER); ++}; ++#undef MEMBER ++ ++extern struct i915_params i915_modparams __read_mostly; ++ ++void i915_params_dump(const struct i915_params *params, struct drm_printer *p); ++void i915_params_copy(struct i915_params *dest, const struct i915_params *src); ++void i915_params_free(struct i915_params *params); ++ ++#endif ++ +diff --git a/drivers/gpu/drm/i915_legacy/i915_pci.c b/drivers/gpu/drm/i915_legacy/i915_pci.c +new file mode 100644 +index 000000000000..f893c2cbce15 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_pci.c +@@ -0,0 +1,957 @@ ++/* ++ * Copyright © 2016 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#include ++#include ++#include ++ ++#include ++ ++#include "i915_drv.h" ++#include "i915_globals.h" ++#include "i915_selftest.h" ++#include "intel_fbdev.h" ++ ++#define PLATFORM(x) .platform = (x) ++#define GEN(x) .gen = (x), .gen_mask = BIT((x) - 1) ++ ++#define I845_PIPE_OFFSETS \ ++ .pipe_offsets = { \ ++ [TRANSCODER_A] = PIPE_A_OFFSET, \ ++ }, \ ++ .trans_offsets = { \ ++ [TRANSCODER_A] = TRANSCODER_A_OFFSET, \ ++ } ++ ++#define I9XX_PIPE_OFFSETS \ ++ .pipe_offsets = { \ ++ [TRANSCODER_A] = PIPE_A_OFFSET, \ ++ [TRANSCODER_B] = PIPE_B_OFFSET, \ ++ }, \ ++ .trans_offsets = { \ ++ [TRANSCODER_A] = TRANSCODER_A_OFFSET, \ ++ [TRANSCODER_B] = TRANSCODER_B_OFFSET, \ ++ } ++ ++#define IVB_PIPE_OFFSETS \ ++ .pipe_offsets = { \ ++ [TRANSCODER_A] = PIPE_A_OFFSET, \ ++ [TRANSCODER_B] = PIPE_B_OFFSET, \ ++ [TRANSCODER_C] = PIPE_C_OFFSET, \ ++ }, \ ++ .trans_offsets = { \ ++ [TRANSCODER_A] = TRANSCODER_A_OFFSET, \ ++ [TRANSCODER_B] = TRANSCODER_B_OFFSET, \ ++ [TRANSCODER_C] = TRANSCODER_C_OFFSET, \ ++ } ++ ++#define HSW_PIPE_OFFSETS \ ++ .pipe_offsets = { \ ++ [TRANSCODER_A] = PIPE_A_OFFSET, \ ++ [TRANSCODER_B] = PIPE_B_OFFSET, \ ++ [TRANSCODER_C] = PIPE_C_OFFSET, \ ++ [TRANSCODER_EDP] = PIPE_EDP_OFFSET, \ ++ }, \ ++ .trans_offsets = { \ ++ [TRANSCODER_A] = TRANSCODER_A_OFFSET, \ ++ [TRANSCODER_B] = TRANSCODER_B_OFFSET, \ ++ [TRANSCODER_C] = TRANSCODER_C_OFFSET, \ ++ [TRANSCODER_EDP] = TRANSCODER_EDP_OFFSET, \ ++ } ++ ++#define CHV_PIPE_OFFSETS \ ++ .pipe_offsets = { \ ++ [TRANSCODER_A] = PIPE_A_OFFSET, \ ++ [TRANSCODER_B] = PIPE_B_OFFSET, \ ++ [TRANSCODER_C] = CHV_PIPE_C_OFFSET, \ ++ }, \ ++ .trans_offsets = { \ ++ [TRANSCODER_A] = TRANSCODER_A_OFFSET, \ ++ [TRANSCODER_B] = TRANSCODER_B_OFFSET, \ ++ [TRANSCODER_C] = CHV_TRANSCODER_C_OFFSET, \ ++ } ++ ++#define I845_CURSOR_OFFSETS \ ++ .cursor_offsets = { \ ++ [PIPE_A] = CURSOR_A_OFFSET, \ ++ } ++ ++#define I9XX_CURSOR_OFFSETS \ ++ .cursor_offsets = { \ ++ [PIPE_A] = CURSOR_A_OFFSET, \ ++ [PIPE_B] = CURSOR_B_OFFSET, \ ++ } ++ ++#define CHV_CURSOR_OFFSETS \ ++ .cursor_offsets = { \ ++ [PIPE_A] = CURSOR_A_OFFSET, \ ++ [PIPE_B] = CURSOR_B_OFFSET, \ ++ [PIPE_C] = CHV_CURSOR_C_OFFSET, \ ++ } ++ ++#define IVB_CURSOR_OFFSETS \ ++ .cursor_offsets = { \ ++ [PIPE_A] = CURSOR_A_OFFSET, \ ++ [PIPE_B] = IVB_CURSOR_B_OFFSET, \ ++ [PIPE_C] = IVB_CURSOR_C_OFFSET, \ ++ } ++ ++#define I9XX_COLORS \ ++ .color = { .gamma_lut_size = 256 } ++#define I965_COLORS \ ++ .color = { .gamma_lut_size = 129, \ ++ .gamma_lut_tests = DRM_COLOR_LUT_NON_DECREASING, \ ++ } ++#define ILK_COLORS \ ++ .color = { .gamma_lut_size = 1024 } ++#define IVB_COLORS \ ++ .color = { .degamma_lut_size = 1024, .gamma_lut_size = 1024 } ++#define CHV_COLORS \ ++ .color = { .degamma_lut_size = 65, .gamma_lut_size = 257, \ ++ .degamma_lut_tests = DRM_COLOR_LUT_NON_DECREASING, \ ++ .gamma_lut_tests = DRM_COLOR_LUT_NON_DECREASING, \ ++ } ++#define GLK_COLORS \ ++ .color = { .degamma_lut_size = 33, .gamma_lut_size = 1024, \ ++ .degamma_lut_tests = DRM_COLOR_LUT_NON_DECREASING | \ ++ DRM_COLOR_LUT_EQUAL_CHANNELS, \ ++ } ++ ++/* Keep in gen based order, and chronological order within a gen */ ++ ++#define GEN_DEFAULT_PAGE_SIZES \ ++ .page_sizes = I915_GTT_PAGE_SIZE_4K ++ ++#define I830_FEATURES \ ++ GEN(2), \ ++ .is_mobile = 1, \ ++ .num_pipes = 2, \ ++ .display.has_overlay = 1, \ ++ .display.cursor_needs_physical = 1, \ ++ .display.overlay_needs_physical = 1, \ ++ .display.has_gmch = 1, \ ++ .gpu_reset_clobbers_display = true, \ ++ .hws_needs_physical = 1, \ ++ .unfenced_needs_alignment = 1, \ ++ .engine_mask = BIT(RCS0), \ ++ .has_snoop = true, \ ++ .has_coherent_ggtt = false, \ ++ I9XX_PIPE_OFFSETS, \ ++ I9XX_CURSOR_OFFSETS, \ ++ I9XX_COLORS, \ ++ GEN_DEFAULT_PAGE_SIZES ++ ++#define I845_FEATURES \ ++ GEN(2), \ ++ .num_pipes = 1, \ ++ .display.has_overlay = 1, \ ++ .display.overlay_needs_physical = 1, \ ++ .display.has_gmch = 1, \ ++ .gpu_reset_clobbers_display = true, \ ++ .hws_needs_physical = 1, \ ++ .unfenced_needs_alignment = 1, \ ++ .engine_mask = BIT(RCS0), \ ++ .has_snoop = true, \ ++ .has_coherent_ggtt = false, \ ++ I845_PIPE_OFFSETS, \ ++ I845_CURSOR_OFFSETS, \ ++ I9XX_COLORS, \ ++ GEN_DEFAULT_PAGE_SIZES ++ ++static const struct intel_device_info intel_i830_info = { ++ I830_FEATURES, ++ PLATFORM(INTEL_I830), ++}; ++ ++static const struct intel_device_info intel_i845g_info = { ++ I845_FEATURES, ++ PLATFORM(INTEL_I845G), ++}; ++ ++static const struct intel_device_info intel_i85x_info = { ++ I830_FEATURES, ++ PLATFORM(INTEL_I85X), ++ .display.has_fbc = 1, ++}; ++ ++static const struct intel_device_info intel_i865g_info = { ++ I845_FEATURES, ++ PLATFORM(INTEL_I865G), ++}; ++ ++#define GEN3_FEATURES \ ++ GEN(3), \ ++ .num_pipes = 2, \ ++ .display.has_gmch = 1, \ ++ .gpu_reset_clobbers_display = true, \ ++ .engine_mask = BIT(RCS0), \ ++ .has_snoop = true, \ ++ .has_coherent_ggtt = true, \ ++ I9XX_PIPE_OFFSETS, \ ++ I9XX_CURSOR_OFFSETS, \ ++ I9XX_COLORS, \ ++ GEN_DEFAULT_PAGE_SIZES ++ ++static const struct intel_device_info intel_i915g_info = { ++ GEN3_FEATURES, ++ PLATFORM(INTEL_I915G), ++ .has_coherent_ggtt = false, ++ .display.cursor_needs_physical = 1, ++ .display.has_overlay = 1, ++ .display.overlay_needs_physical = 1, ++ .hws_needs_physical = 1, ++ .unfenced_needs_alignment = 1, ++}; ++ ++static const struct intel_device_info intel_i915gm_info = { ++ GEN3_FEATURES, ++ PLATFORM(INTEL_I915GM), ++ .is_mobile = 1, ++ .display.cursor_needs_physical = 1, ++ .display.has_overlay = 1, ++ .display.overlay_needs_physical = 1, ++ .display.supports_tv = 1, ++ .display.has_fbc = 1, ++ .hws_needs_physical = 1, ++ .unfenced_needs_alignment = 1, ++}; ++ ++static const struct intel_device_info intel_i945g_info = { ++ GEN3_FEATURES, ++ PLATFORM(INTEL_I945G), ++ .display.has_hotplug = 1, ++ .display.cursor_needs_physical = 1, ++ .display.has_overlay = 1, ++ .display.overlay_needs_physical = 1, ++ .hws_needs_physical = 1, ++ .unfenced_needs_alignment = 1, ++}; ++ ++static const struct intel_device_info intel_i945gm_info = { ++ GEN3_FEATURES, ++ PLATFORM(INTEL_I945GM), ++ .is_mobile = 1, ++ .display.has_hotplug = 1, ++ .display.cursor_needs_physical = 1, ++ .display.has_overlay = 1, ++ .display.overlay_needs_physical = 1, ++ .display.supports_tv = 1, ++ .display.has_fbc = 1, ++ .hws_needs_physical = 1, ++ .unfenced_needs_alignment = 1, ++}; ++ ++static const struct intel_device_info intel_g33_info = { ++ GEN3_FEATURES, ++ PLATFORM(INTEL_G33), ++ .display.has_hotplug = 1, ++ .display.has_overlay = 1, ++}; ++ ++static const struct intel_device_info intel_pineview_g_info = { ++ GEN3_FEATURES, ++ PLATFORM(INTEL_PINEVIEW), ++ .display.has_hotplug = 1, ++ .display.has_overlay = 1, ++}; ++ ++static const struct intel_device_info intel_pineview_m_info = { ++ GEN3_FEATURES, ++ PLATFORM(INTEL_PINEVIEW), ++ .is_mobile = 1, ++ .display.has_hotplug = 1, ++ .display.has_overlay = 1, ++}; ++ ++#define GEN4_FEATURES \ ++ GEN(4), \ ++ .num_pipes = 2, \ ++ .display.has_hotplug = 1, \ ++ .display.has_gmch = 1, \ ++ .gpu_reset_clobbers_display = true, \ ++ .engine_mask = BIT(RCS0), \ ++ .has_snoop = true, \ ++ .has_coherent_ggtt = true, \ ++ I9XX_PIPE_OFFSETS, \ ++ I9XX_CURSOR_OFFSETS, \ ++ I965_COLORS, \ ++ GEN_DEFAULT_PAGE_SIZES ++ ++static const struct intel_device_info intel_i965g_info = { ++ GEN4_FEATURES, ++ PLATFORM(INTEL_I965G), ++ .display.has_overlay = 1, ++ .hws_needs_physical = 1, ++ .has_snoop = false, ++}; ++ ++static const struct intel_device_info intel_i965gm_info = { ++ GEN4_FEATURES, ++ PLATFORM(INTEL_I965GM), ++ .is_mobile = 1, ++ .display.has_fbc = 1, ++ .display.has_overlay = 1, ++ .display.supports_tv = 1, ++ .hws_needs_physical = 1, ++ .has_snoop = false, ++}; ++ ++static const struct intel_device_info intel_g45_info = { ++ GEN4_FEATURES, ++ PLATFORM(INTEL_G45), ++ .engine_mask = BIT(RCS0) | BIT(VCS0), ++ .gpu_reset_clobbers_display = false, ++}; ++ ++static const struct intel_device_info intel_gm45_info = { ++ GEN4_FEATURES, ++ PLATFORM(INTEL_GM45), ++ .is_mobile = 1, ++ .display.has_fbc = 1, ++ .display.supports_tv = 1, ++ .engine_mask = BIT(RCS0) | BIT(VCS0), ++ .gpu_reset_clobbers_display = false, ++}; ++ ++#define GEN5_FEATURES \ ++ GEN(5), \ ++ .num_pipes = 2, \ ++ .display.has_hotplug = 1, \ ++ .engine_mask = BIT(RCS0) | BIT(VCS0), \ ++ .has_snoop = true, \ ++ .has_coherent_ggtt = true, \ ++ /* ilk does support rc6, but we do not implement [power] contexts */ \ ++ .has_rc6 = 0, \ ++ I9XX_PIPE_OFFSETS, \ ++ I9XX_CURSOR_OFFSETS, \ ++ ILK_COLORS, \ ++ GEN_DEFAULT_PAGE_SIZES ++ ++static const struct intel_device_info intel_ironlake_d_info = { ++ GEN5_FEATURES, ++ PLATFORM(INTEL_IRONLAKE), ++}; ++ ++static const struct intel_device_info intel_ironlake_m_info = { ++ GEN5_FEATURES, ++ PLATFORM(INTEL_IRONLAKE), ++ .is_mobile = 1, ++ .display.has_fbc = 1, ++}; ++ ++#define GEN6_FEATURES \ ++ GEN(6), \ ++ .num_pipes = 2, \ ++ .display.has_hotplug = 1, \ ++ .display.has_fbc = 1, \ ++ .engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0), \ ++ .has_coherent_ggtt = true, \ ++ .has_llc = 1, \ ++ .has_rc6 = 1, \ ++ .has_rc6p = 1, \ ++ .ppgtt_type = INTEL_PPGTT_ALIASING, \ ++ .ppgtt_size = 31, \ ++ I9XX_PIPE_OFFSETS, \ ++ I9XX_CURSOR_OFFSETS, \ ++ ILK_COLORS, \ ++ GEN_DEFAULT_PAGE_SIZES ++ ++#define SNB_D_PLATFORM \ ++ GEN6_FEATURES, \ ++ PLATFORM(INTEL_SANDYBRIDGE) ++ ++static const struct intel_device_info intel_sandybridge_d_gt1_info = { ++ SNB_D_PLATFORM, ++ .gt = 1, ++}; ++ ++static const struct intel_device_info intel_sandybridge_d_gt2_info = { ++ SNB_D_PLATFORM, ++ .gt = 2, ++}; ++ ++#define SNB_M_PLATFORM \ ++ GEN6_FEATURES, \ ++ PLATFORM(INTEL_SANDYBRIDGE), \ ++ .is_mobile = 1 ++ ++ ++static const struct intel_device_info intel_sandybridge_m_gt1_info = { ++ SNB_M_PLATFORM, ++ .gt = 1, ++}; ++ ++static const struct intel_device_info intel_sandybridge_m_gt2_info = { ++ SNB_M_PLATFORM, ++ .gt = 2, ++}; ++ ++#define GEN7_FEATURES \ ++ GEN(7), \ ++ .num_pipes = 3, \ ++ .display.has_hotplug = 1, \ ++ .display.has_fbc = 1, \ ++ .engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0), \ ++ .has_coherent_ggtt = true, \ ++ .has_llc = 1, \ ++ .has_rc6 = 1, \ ++ .has_rc6p = 1, \ ++ .ppgtt_type = INTEL_PPGTT_FULL, \ ++ .ppgtt_size = 31, \ ++ IVB_PIPE_OFFSETS, \ ++ IVB_CURSOR_OFFSETS, \ ++ IVB_COLORS, \ ++ GEN_DEFAULT_PAGE_SIZES ++ ++#define IVB_D_PLATFORM \ ++ GEN7_FEATURES, \ ++ PLATFORM(INTEL_IVYBRIDGE), \ ++ .has_l3_dpf = 1 ++ ++static const struct intel_device_info intel_ivybridge_d_gt1_info = { ++ IVB_D_PLATFORM, ++ .gt = 1, ++}; ++ ++static const struct intel_device_info intel_ivybridge_d_gt2_info = { ++ IVB_D_PLATFORM, ++ .gt = 2, ++}; ++ ++#define IVB_M_PLATFORM \ ++ GEN7_FEATURES, \ ++ PLATFORM(INTEL_IVYBRIDGE), \ ++ .is_mobile = 1, \ ++ .has_l3_dpf = 1 ++ ++static const struct intel_device_info intel_ivybridge_m_gt1_info = { ++ IVB_M_PLATFORM, ++ .gt = 1, ++}; ++ ++static const struct intel_device_info intel_ivybridge_m_gt2_info = { ++ IVB_M_PLATFORM, ++ .gt = 2, ++}; ++ ++static const struct intel_device_info intel_ivybridge_q_info = { ++ GEN7_FEATURES, ++ PLATFORM(INTEL_IVYBRIDGE), ++ .gt = 2, ++ .num_pipes = 0, /* legal, last one wins */ ++ .has_l3_dpf = 1, ++}; ++ ++static const struct intel_device_info intel_valleyview_info = { ++ PLATFORM(INTEL_VALLEYVIEW), ++ GEN(7), ++ .is_lp = 1, ++ .num_pipes = 2, ++ .has_runtime_pm = 1, ++ .has_rc6 = 1, ++ .display.has_gmch = 1, ++ .display.has_hotplug = 1, ++ .ppgtt_type = INTEL_PPGTT_FULL, ++ .ppgtt_size = 31, ++ .has_snoop = true, ++ .has_coherent_ggtt = false, ++ .engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0), ++ .display_mmio_offset = VLV_DISPLAY_BASE, ++ I9XX_PIPE_OFFSETS, ++ I9XX_CURSOR_OFFSETS, ++ I965_COLORS, ++ GEN_DEFAULT_PAGE_SIZES, ++}; ++ ++#define G75_FEATURES \ ++ GEN7_FEATURES, \ ++ .engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0), \ ++ .display.has_ddi = 1, \ ++ .has_fpga_dbg = 1, \ ++ .display.has_psr = 1, \ ++ .display.has_dp_mst = 1, \ ++ .has_rc6p = 0 /* RC6p removed-by HSW */, \ ++ HSW_PIPE_OFFSETS, \ ++ .has_runtime_pm = 1 ++ ++#define HSW_PLATFORM \ ++ G75_FEATURES, \ ++ PLATFORM(INTEL_HASWELL), \ ++ .has_l3_dpf = 1 ++ ++static const struct intel_device_info intel_haswell_gt1_info = { ++ HSW_PLATFORM, ++ .gt = 1, ++}; ++ ++static const struct intel_device_info intel_haswell_gt2_info = { ++ HSW_PLATFORM, ++ .gt = 2, ++}; ++ ++static const struct intel_device_info intel_haswell_gt3_info = { ++ HSW_PLATFORM, ++ .gt = 3, ++}; ++ ++#define GEN8_FEATURES \ ++ G75_FEATURES, \ ++ GEN(8), \ ++ .page_sizes = I915_GTT_PAGE_SIZE_4K | \ ++ I915_GTT_PAGE_SIZE_2M, \ ++ .has_logical_ring_contexts = 1, \ ++ .ppgtt_type = INTEL_PPGTT_FULL, \ ++ .ppgtt_size = 48, \ ++ .has_64bit_reloc = 1, \ ++ .has_reset_engine = 1 ++ ++#define BDW_PLATFORM \ ++ GEN8_FEATURES, \ ++ PLATFORM(INTEL_BROADWELL) ++ ++static const struct intel_device_info intel_broadwell_gt1_info = { ++ BDW_PLATFORM, ++ .gt = 1, ++}; ++ ++static const struct intel_device_info intel_broadwell_gt2_info = { ++ BDW_PLATFORM, ++ .gt = 2, ++}; ++ ++static const struct intel_device_info intel_broadwell_rsvd_info = { ++ BDW_PLATFORM, ++ .gt = 3, ++ /* According to the device ID those devices are GT3, they were ++ * previously treated as not GT3, keep it like that. ++ */ ++}; ++ ++static const struct intel_device_info intel_broadwell_gt3_info = { ++ BDW_PLATFORM, ++ .gt = 3, ++ .engine_mask = ++ BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS1), ++}; ++ ++static const struct intel_device_info intel_cherryview_info = { ++ PLATFORM(INTEL_CHERRYVIEW), ++ GEN(8), ++ .num_pipes = 3, ++ .display.has_hotplug = 1, ++ .is_lp = 1, ++ .engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0), ++ .has_64bit_reloc = 1, ++ .has_runtime_pm = 1, ++ .has_rc6 = 1, ++ .has_logical_ring_contexts = 1, ++ .display.has_gmch = 1, ++ .ppgtt_type = INTEL_PPGTT_FULL, ++ .ppgtt_size = 32, ++ .has_reset_engine = 1, ++ .has_snoop = true, ++ .has_coherent_ggtt = false, ++ .display_mmio_offset = VLV_DISPLAY_BASE, ++ CHV_PIPE_OFFSETS, ++ CHV_CURSOR_OFFSETS, ++ CHV_COLORS, ++ GEN_DEFAULT_PAGE_SIZES, ++}; ++ ++#define GEN9_DEFAULT_PAGE_SIZES \ ++ .page_sizes = I915_GTT_PAGE_SIZE_4K | \ ++ I915_GTT_PAGE_SIZE_64K | \ ++ I915_GTT_PAGE_SIZE_2M ++ ++#define GEN9_FEATURES \ ++ GEN8_FEATURES, \ ++ GEN(9), \ ++ GEN9_DEFAULT_PAGE_SIZES, \ ++ .has_logical_ring_preemption = 1, \ ++ .display.has_csr = 1, \ ++ .has_guc = 1, \ ++ .display.has_ipc = 1, \ ++ .ddb_size = 896 ++ ++#define SKL_PLATFORM \ ++ GEN9_FEATURES, \ ++ /* Display WA #0477 WaDisableIPC: skl */ \ ++ .display.has_ipc = 0, \ ++ PLATFORM(INTEL_SKYLAKE) ++ ++static const struct intel_device_info intel_skylake_gt1_info = { ++ SKL_PLATFORM, ++ .gt = 1, ++}; ++ ++static const struct intel_device_info intel_skylake_gt2_info = { ++ SKL_PLATFORM, ++ .gt = 2, ++}; ++ ++#define SKL_GT3_PLUS_PLATFORM \ ++ SKL_PLATFORM, \ ++ .engine_mask = \ ++ BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS1) ++ ++ ++static const struct intel_device_info intel_skylake_gt3_info = { ++ SKL_GT3_PLUS_PLATFORM, ++ .gt = 3, ++}; ++ ++static const struct intel_device_info intel_skylake_gt4_info = { ++ SKL_GT3_PLUS_PLATFORM, ++ .gt = 4, ++}; ++ ++#define GEN9_LP_FEATURES \ ++ GEN(9), \ ++ .is_lp = 1, \ ++ .display.has_hotplug = 1, \ ++ .engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0), \ ++ .num_pipes = 3, \ ++ .has_64bit_reloc = 1, \ ++ .display.has_ddi = 1, \ ++ .has_fpga_dbg = 1, \ ++ .display.has_fbc = 1, \ ++ .display.has_psr = 1, \ ++ .has_runtime_pm = 1, \ ++ .display.has_csr = 1, \ ++ .has_rc6 = 1, \ ++ .display.has_dp_mst = 1, \ ++ .has_logical_ring_contexts = 1, \ ++ .has_logical_ring_preemption = 1, \ ++ .has_guc = 1, \ ++ .ppgtt_type = INTEL_PPGTT_FULL, \ ++ .ppgtt_size = 48, \ ++ .has_reset_engine = 1, \ ++ .has_snoop = true, \ ++ .has_coherent_ggtt = false, \ ++ .display.has_ipc = 1, \ ++ HSW_PIPE_OFFSETS, \ ++ IVB_CURSOR_OFFSETS, \ ++ IVB_COLORS, \ ++ GEN9_DEFAULT_PAGE_SIZES ++ ++static const struct intel_device_info intel_broxton_info = { ++ GEN9_LP_FEATURES, ++ PLATFORM(INTEL_BROXTON), ++ .ddb_size = 512, ++}; ++ ++static const struct intel_device_info intel_geminilake_info = { ++ GEN9_LP_FEATURES, ++ PLATFORM(INTEL_GEMINILAKE), ++ .ddb_size = 1024, ++ GLK_COLORS, ++}; ++ ++#define KBL_PLATFORM \ ++ GEN9_FEATURES, \ ++ PLATFORM(INTEL_KABYLAKE) ++ ++static const struct intel_device_info intel_kabylake_gt1_info = { ++ KBL_PLATFORM, ++ .gt = 1, ++}; ++ ++static const struct intel_device_info intel_kabylake_gt2_info = { ++ KBL_PLATFORM, ++ .gt = 2, ++}; ++ ++static const struct intel_device_info intel_kabylake_gt3_info = { ++ KBL_PLATFORM, ++ .gt = 3, ++ .engine_mask = ++ BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS1), ++}; ++ ++#define CFL_PLATFORM \ ++ GEN9_FEATURES, \ ++ PLATFORM(INTEL_COFFEELAKE) ++ ++static const struct intel_device_info intel_coffeelake_gt1_info = { ++ CFL_PLATFORM, ++ .gt = 1, ++}; ++ ++static const struct intel_device_info intel_coffeelake_gt2_info = { ++ CFL_PLATFORM, ++ .gt = 2, ++}; ++ ++static const struct intel_device_info intel_coffeelake_gt3_info = { ++ CFL_PLATFORM, ++ .gt = 3, ++ .engine_mask = ++ BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS1), ++}; ++ ++#define GEN10_FEATURES \ ++ GEN9_FEATURES, \ ++ GEN(10), \ ++ .ddb_size = 1024, \ ++ .has_coherent_ggtt = false, \ ++ GLK_COLORS ++ ++static const struct intel_device_info intel_cannonlake_info = { ++ GEN10_FEATURES, ++ PLATFORM(INTEL_CANNONLAKE), ++ .gt = 2, ++}; ++ ++#define GEN11_FEATURES \ ++ GEN10_FEATURES, \ ++ .pipe_offsets = { \ ++ [TRANSCODER_A] = PIPE_A_OFFSET, \ ++ [TRANSCODER_B] = PIPE_B_OFFSET, \ ++ [TRANSCODER_C] = PIPE_C_OFFSET, \ ++ [TRANSCODER_EDP] = PIPE_EDP_OFFSET, \ ++ [TRANSCODER_DSI_0] = PIPE_DSI0_OFFSET, \ ++ [TRANSCODER_DSI_1] = PIPE_DSI1_OFFSET, \ ++ }, \ ++ .trans_offsets = { \ ++ [TRANSCODER_A] = TRANSCODER_A_OFFSET, \ ++ [TRANSCODER_B] = TRANSCODER_B_OFFSET, \ ++ [TRANSCODER_C] = TRANSCODER_C_OFFSET, \ ++ [TRANSCODER_EDP] = TRANSCODER_EDP_OFFSET, \ ++ [TRANSCODER_DSI_0] = TRANSCODER_DSI0_OFFSET, \ ++ [TRANSCODER_DSI_1] = TRANSCODER_DSI1_OFFSET, \ ++ }, \ ++ GEN(11), \ ++ .ddb_size = 2048, \ ++ .has_logical_ring_elsq = 1, \ ++ .color = { .degamma_lut_size = 33, .gamma_lut_size = 1024 } ++ ++static const struct intel_device_info intel_icelake_11_info = { ++ GEN11_FEATURES, ++ PLATFORM(INTEL_ICELAKE), ++ .engine_mask = ++ BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0) | BIT(VCS2), ++}; ++ ++static const struct intel_device_info intel_elkhartlake_info = { ++ GEN11_FEATURES, ++ PLATFORM(INTEL_ELKHARTLAKE), ++ .is_alpha_support = 1, ++ .engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VCS0), ++ .ppgtt_size = 36, ++}; ++ ++#undef GEN ++#undef PLATFORM ++ ++/* ++ * Make sure any device matches here are from most specific to most ++ * general. For example, since the Quanta match is based on the subsystem ++ * and subvendor IDs, we need it to come before the more general IVB ++ * PCI ID matches, otherwise we'll use the wrong info struct above. ++ */ ++static const struct pci_device_id pciidlist[] = { ++ INTEL_I830_IDS(&intel_i830_info), ++ INTEL_I845G_IDS(&intel_i845g_info), ++ INTEL_I85X_IDS(&intel_i85x_info), ++ INTEL_I865G_IDS(&intel_i865g_info), ++ INTEL_I915G_IDS(&intel_i915g_info), ++ INTEL_I915GM_IDS(&intel_i915gm_info), ++ INTEL_I945G_IDS(&intel_i945g_info), ++ INTEL_I945GM_IDS(&intel_i945gm_info), ++ INTEL_I965G_IDS(&intel_i965g_info), ++ INTEL_G33_IDS(&intel_g33_info), ++ INTEL_I965GM_IDS(&intel_i965gm_info), ++ INTEL_GM45_IDS(&intel_gm45_info), ++ INTEL_G45_IDS(&intel_g45_info), ++ INTEL_PINEVIEW_G_IDS(&intel_pineview_g_info), ++ INTEL_PINEVIEW_M_IDS(&intel_pineview_m_info), ++ INTEL_IRONLAKE_D_IDS(&intel_ironlake_d_info), ++ INTEL_IRONLAKE_M_IDS(&intel_ironlake_m_info), ++ INTEL_SNB_D_GT1_IDS(&intel_sandybridge_d_gt1_info), ++ INTEL_SNB_D_GT2_IDS(&intel_sandybridge_d_gt2_info), ++ INTEL_SNB_M_GT1_IDS(&intel_sandybridge_m_gt1_info), ++ INTEL_SNB_M_GT2_IDS(&intel_sandybridge_m_gt2_info), ++ INTEL_IVB_Q_IDS(&intel_ivybridge_q_info), /* must be first IVB */ ++ INTEL_IVB_M_GT1_IDS(&intel_ivybridge_m_gt1_info), ++ INTEL_IVB_M_GT2_IDS(&intel_ivybridge_m_gt2_info), ++ INTEL_IVB_D_GT1_IDS(&intel_ivybridge_d_gt1_info), ++ INTEL_IVB_D_GT2_IDS(&intel_ivybridge_d_gt2_info), ++ INTEL_HSW_GT1_IDS(&intel_haswell_gt1_info), ++ INTEL_HSW_GT2_IDS(&intel_haswell_gt2_info), ++ INTEL_HSW_GT3_IDS(&intel_haswell_gt3_info), ++ INTEL_VLV_IDS(&intel_valleyview_info), ++ INTEL_BDW_GT1_IDS(&intel_broadwell_gt1_info), ++ INTEL_BDW_GT2_IDS(&intel_broadwell_gt2_info), ++ INTEL_BDW_GT3_IDS(&intel_broadwell_gt3_info), ++ INTEL_BDW_RSVD_IDS(&intel_broadwell_rsvd_info), ++ INTEL_CHV_IDS(&intel_cherryview_info), ++ INTEL_SKL_GT1_IDS(&intel_skylake_gt1_info), ++ INTEL_SKL_GT2_IDS(&intel_skylake_gt2_info), ++ INTEL_SKL_GT3_IDS(&intel_skylake_gt3_info), ++ INTEL_SKL_GT4_IDS(&intel_skylake_gt4_info), ++ INTEL_BXT_IDS(&intel_broxton_info), ++ INTEL_GLK_IDS(&intel_geminilake_info), ++ INTEL_KBL_GT1_IDS(&intel_kabylake_gt1_info), ++ INTEL_KBL_GT2_IDS(&intel_kabylake_gt2_info), ++ INTEL_KBL_GT3_IDS(&intel_kabylake_gt3_info), ++ INTEL_KBL_GT4_IDS(&intel_kabylake_gt3_info), ++ INTEL_AML_KBL_GT2_IDS(&intel_kabylake_gt2_info), ++ INTEL_CFL_S_GT1_IDS(&intel_coffeelake_gt1_info), ++ INTEL_CFL_S_GT2_IDS(&intel_coffeelake_gt2_info), ++ INTEL_CFL_H_GT1_IDS(&intel_coffeelake_gt1_info), ++ INTEL_CFL_H_GT2_IDS(&intel_coffeelake_gt2_info), ++ INTEL_CFL_U_GT2_IDS(&intel_coffeelake_gt2_info), ++ INTEL_CFL_U_GT3_IDS(&intel_coffeelake_gt3_info), ++ INTEL_WHL_U_GT1_IDS(&intel_coffeelake_gt1_info), ++ INTEL_WHL_U_GT2_IDS(&intel_coffeelake_gt2_info), ++ INTEL_AML_CFL_GT2_IDS(&intel_coffeelake_gt2_info), ++ INTEL_WHL_U_GT3_IDS(&intel_coffeelake_gt3_info), ++ INTEL_CML_GT1_IDS(&intel_coffeelake_gt1_info), ++ INTEL_CML_GT2_IDS(&intel_coffeelake_gt2_info), ++ INTEL_CNL_IDS(&intel_cannonlake_info), ++ INTEL_ICL_11_IDS(&intel_icelake_11_info), ++ INTEL_EHL_IDS(&intel_elkhartlake_info), ++ {0, 0, 0} ++}; ++MODULE_DEVICE_TABLE(pci, pciidlist); ++ ++static void i915_pci_remove(struct pci_dev *pdev) ++{ ++ struct drm_device *dev; ++ ++ dev = pci_get_drvdata(pdev); ++ if (!dev) /* driver load aborted, nothing to cleanup */ ++ return; ++ ++ i915_driver_unload(dev); ++ drm_dev_put(dev); ++ ++ pci_set_drvdata(pdev, NULL); ++} ++ ++static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) ++{ ++ struct intel_device_info *intel_info = ++ (struct intel_device_info *) ent->driver_data; ++ int err; ++ ++ if (IS_ALPHA_SUPPORT(intel_info) && !i915_modparams.alpha_support) { ++ DRM_INFO("The driver support for your hardware in this kernel version is alpha quality\n" ++ "See CONFIG_DRM_I915_ALPHA_SUPPORT or i915.alpha_support module parameter\n" ++ "to enable support in this kernel version, or check for kernel updates.\n"); ++ return -ENODEV; ++ } ++ ++ /* Only bind to function 0 of the device. Early generations ++ * used function 1 as a placeholder for multi-head. This causes ++ * us confusion instead, especially on the systems where both ++ * functions have the same PCI-ID! ++ */ ++ if (PCI_FUNC(pdev->devfn)) ++ return -ENODEV; ++ ++ /* ++ * apple-gmux is needed on dual GPU MacBook Pro ++ * to probe the panel if we're the inactive GPU. ++ */ ++ if (vga_switcheroo_client_probe_defer(pdev)) ++ return -EPROBE_DEFER; ++ ++ err = i915_driver_load(pdev, ent); ++ if (err) ++ return err; ++ ++ if (i915_inject_load_failure()) { ++ i915_pci_remove(pdev); ++ return -ENODEV; ++ } ++ ++ err = i915_live_selftests(pdev); ++ if (err) { ++ i915_pci_remove(pdev); ++ return err > 0 ? -ENOTTY : err; ++ } ++ ++ return 0; ++} ++ ++static struct pci_driver i915_pci_driver = { ++ .name = DRIVER_NAME, ++ .id_table = pciidlist, ++ .probe = i915_pci_probe, ++ .remove = i915_pci_remove, ++ .driver.pm = &i915_pm_ops, ++}; ++ ++static int __init i915_init(void) ++{ ++ bool use_kms = true; ++ int err; ++ ++ err = i915_globals_init(); ++ if (err) ++ return err; ++ ++ err = i915_mock_selftests(); ++ if (err) ++ return err > 0 ? 0 : err; ++ ++ /* ++ * Enable KMS by default, unless explicitly overriden by ++ * either the i915.modeset prarameter or by the ++ * vga_text_mode_force boot option. ++ */ ++ ++ if (i915_modparams.modeset == 0) ++ use_kms = false; ++ ++ if (vgacon_text_force() && i915_modparams.modeset == -1) ++ use_kms = false; ++ ++ if (!use_kms) { ++ /* Silently fail loading to not upset userspace. */ ++ DRM_DEBUG_DRIVER("KMS disabled.\n"); ++ return 0; ++ } ++ ++ return pci_register_driver(&i915_pci_driver); ++} ++ ++static void __exit i915_exit(void) ++{ ++ if (!i915_pci_driver.driver.owner) ++ return; ++ ++ pci_unregister_driver(&i915_pci_driver); ++ i915_globals_exit(); ++} ++ ++module_init(i915_init); ++module_exit(i915_exit); ++ ++MODULE_AUTHOR("Tungsten Graphics, Inc."); ++MODULE_AUTHOR("Intel Corporation"); ++ ++MODULE_DESCRIPTION(DRIVER_DESC); ++MODULE_LICENSE("GPL and additional rights"); +diff --git a/drivers/gpu/drm/i915_legacy/i915_perf.c b/drivers/gpu/drm/i915_legacy/i915_perf.c +new file mode 100644 +index 000000000000..235aedc62b4c +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_perf.c +@@ -0,0 +1,3519 @@ ++/* ++ * Copyright © 2015-2016 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ * Authors: ++ * Robert Bragg ++ */ ++ ++ ++/** ++ * DOC: i915 Perf Overview ++ * ++ * Gen graphics supports a large number of performance counters that can help ++ * driver and application developers understand and optimize their use of the ++ * GPU. ++ * ++ * This i915 perf interface enables userspace to configure and open a file ++ * descriptor representing a stream of GPU metrics which can then be read() as ++ * a stream of sample records. ++ * ++ * The interface is particularly suited to exposing buffered metrics that are ++ * captured by DMA from the GPU, unsynchronized with and unrelated to the CPU. ++ * ++ * Streams representing a single context are accessible to applications with a ++ * corresponding drm file descriptor, such that OpenGL can use the interface ++ * without special privileges. Access to system-wide metrics requires root ++ * privileges by default, unless changed via the dev.i915.perf_event_paranoid ++ * sysctl option. ++ * ++ */ ++ ++/** ++ * DOC: i915 Perf History and Comparison with Core Perf ++ * ++ * The interface was initially inspired by the core Perf infrastructure but ++ * some notable differences are: ++ * ++ * i915 perf file descriptors represent a "stream" instead of an "event"; where ++ * a perf event primarily corresponds to a single 64bit value, while a stream ++ * might sample sets of tightly-coupled counters, depending on the ++ * configuration. For example the Gen OA unit isn't designed to support ++ * orthogonal configurations of individual counters; it's configured for a set ++ * of related counters. Samples for an i915 perf stream capturing OA metrics ++ * will include a set of counter values packed in a compact HW specific format. ++ * The OA unit supports a number of different packing formats which can be ++ * selected by the user opening the stream. Perf has support for grouping ++ * events, but each event in the group is configured, validated and ++ * authenticated individually with separate system calls. ++ * ++ * i915 perf stream configurations are provided as an array of u64 (key,value) ++ * pairs, instead of a fixed struct with multiple miscellaneous config members, ++ * interleaved with event-type specific members. ++ * ++ * i915 perf doesn't support exposing metrics via an mmap'd circular buffer. ++ * The supported metrics are being written to memory by the GPU unsynchronized ++ * with the CPU, using HW specific packing formats for counter sets. Sometimes ++ * the constraints on HW configuration require reports to be filtered before it ++ * would be acceptable to expose them to unprivileged applications - to hide ++ * the metrics of other processes/contexts. For these use cases a read() based ++ * interface is a good fit, and provides an opportunity to filter data as it ++ * gets copied from the GPU mapped buffers to userspace buffers. ++ * ++ * ++ * Issues hit with first prototype based on Core Perf ++ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ++ * ++ * The first prototype of this driver was based on the core perf ++ * infrastructure, and while we did make that mostly work, with some changes to ++ * perf, we found we were breaking or working around too many assumptions baked ++ * into perf's currently cpu centric design. ++ * ++ * In the end we didn't see a clear benefit to making perf's implementation and ++ * interface more complex by changing design assumptions while we knew we still ++ * wouldn't be able to use any existing perf based userspace tools. ++ * ++ * Also considering the Gen specific nature of the Observability hardware and ++ * how userspace will sometimes need to combine i915 perf OA metrics with ++ * side-band OA data captured via MI_REPORT_PERF_COUNT commands; we're ++ * expecting the interface to be used by a platform specific userspace such as ++ * OpenGL or tools. This is to say; we aren't inherently missing out on having ++ * a standard vendor/architecture agnostic interface by not using perf. ++ * ++ * ++ * For posterity, in case we might re-visit trying to adapt core perf to be ++ * better suited to exposing i915 metrics these were the main pain points we ++ * hit: ++ * ++ * - The perf based OA PMU driver broke some significant design assumptions: ++ * ++ * Existing perf pmus are used for profiling work on a cpu and we were ++ * introducing the idea of _IS_DEVICE pmus with different security ++ * implications, the need to fake cpu-related data (such as user/kernel ++ * registers) to fit with perf's current design, and adding _DEVICE records ++ * as a way to forward device-specific status records. ++ * ++ * The OA unit writes reports of counters into a circular buffer, without ++ * involvement from the CPU, making our PMU driver the first of a kind. ++ * ++ * Given the way we were periodically forward data from the GPU-mapped, OA ++ * buffer to perf's buffer, those bursts of sample writes looked to perf like ++ * we were sampling too fast and so we had to subvert its throttling checks. ++ * ++ * Perf supports groups of counters and allows those to be read via ++ * transactions internally but transactions currently seem designed to be ++ * explicitly initiated from the cpu (say in response to a userspace read()) ++ * and while we could pull a report out of the OA buffer we can't ++ * trigger a report from the cpu on demand. ++ * ++ * Related to being report based; the OA counters are configured in HW as a ++ * set while perf generally expects counter configurations to be orthogonal. ++ * Although counters can be associated with a group leader as they are ++ * opened, there's no clear precedent for being able to provide group-wide ++ * configuration attributes (for example we want to let userspace choose the ++ * OA unit report format used to capture all counters in a set, or specify a ++ * GPU context to filter metrics on). We avoided using perf's grouping ++ * feature and forwarded OA reports to userspace via perf's 'raw' sample ++ * field. This suited our userspace well considering how coupled the counters ++ * are when dealing with normalizing. It would be inconvenient to split ++ * counters up into separate events, only to require userspace to recombine ++ * them. For Mesa it's also convenient to be forwarded raw, periodic reports ++ * for combining with the side-band raw reports it captures using ++ * MI_REPORT_PERF_COUNT commands. ++ * ++ * - As a side note on perf's grouping feature; there was also some concern ++ * that using PERF_FORMAT_GROUP as a way to pack together counter values ++ * would quite drastically inflate our sample sizes, which would likely ++ * lower the effective sampling resolutions we could use when the available ++ * memory bandwidth is limited. ++ * ++ * With the OA unit's report formats, counters are packed together as 32 ++ * or 40bit values, with the largest report size being 256 bytes. ++ * ++ * PERF_FORMAT_GROUP values are 64bit, but there doesn't appear to be a ++ * documented ordering to the values, implying PERF_FORMAT_ID must also be ++ * used to add a 64bit ID before each value; giving 16 bytes per counter. ++ * ++ * Related to counter orthogonality; we can't time share the OA unit, while ++ * event scheduling is a central design idea within perf for allowing ++ * userspace to open + enable more events than can be configured in HW at any ++ * one time. The OA unit is not designed to allow re-configuration while in ++ * use. We can't reconfigure the OA unit without losing internal OA unit ++ * state which we can't access explicitly to save and restore. Reconfiguring ++ * the OA unit is also relatively slow, involving ~100 register writes. From ++ * userspace Mesa also depends on a stable OA configuration when emitting ++ * MI_REPORT_PERF_COUNT commands and importantly the OA unit can't be ++ * disabled while there are outstanding MI_RPC commands lest we hang the ++ * command streamer. ++ * ++ * The contents of sample records aren't extensible by device drivers (i.e. ++ * the sample_type bits). As an example; Sourab Gupta had been looking to ++ * attach GPU timestamps to our OA samples. We were shoehorning OA reports ++ * into sample records by using the 'raw' field, but it's tricky to pack more ++ * than one thing into this field because events/core.c currently only lets a ++ * pmu give a single raw data pointer plus len which will be copied into the ++ * ring buffer. To include more than the OA report we'd have to copy the ++ * report into an intermediate larger buffer. I'd been considering allowing a ++ * vector of data+len values to be specified for copying the raw data, but ++ * it felt like a kludge to being using the raw field for this purpose. ++ * ++ * - It felt like our perf based PMU was making some technical compromises ++ * just for the sake of using perf: ++ * ++ * perf_event_open() requires events to either relate to a pid or a specific ++ * cpu core, while our device pmu related to neither. Events opened with a ++ * pid will be automatically enabled/disabled according to the scheduling of ++ * that process - so not appropriate for us. When an event is related to a ++ * cpu id, perf ensures pmu methods will be invoked via an inter process ++ * interrupt on that core. To avoid invasive changes our userspace opened OA ++ * perf events for a specific cpu. This was workable but it meant the ++ * majority of the OA driver ran in atomic context, including all OA report ++ * forwarding, which wasn't really necessary in our case and seems to make ++ * our locking requirements somewhat complex as we handled the interaction ++ * with the rest of the i915 driver. ++ */ ++ ++#include ++#include ++#include ++ ++#include "i915_drv.h" ++#include "i915_oa_hsw.h" ++#include "i915_oa_bdw.h" ++#include "i915_oa_chv.h" ++#include "i915_oa_sklgt2.h" ++#include "i915_oa_sklgt3.h" ++#include "i915_oa_sklgt4.h" ++#include "i915_oa_bxt.h" ++#include "i915_oa_kblgt2.h" ++#include "i915_oa_kblgt3.h" ++#include "i915_oa_glk.h" ++#include "i915_oa_cflgt2.h" ++#include "i915_oa_cflgt3.h" ++#include "i915_oa_cnl.h" ++#include "i915_oa_icl.h" ++#include "intel_lrc_reg.h" ++ ++/* HW requires this to be a power of two, between 128k and 16M, though driver ++ * is currently generally designed assuming the largest 16M size is used such ++ * that the overflow cases are unlikely in normal operation. ++ */ ++#define OA_BUFFER_SIZE SZ_16M ++ ++#define OA_TAKEN(tail, head) ((tail - head) & (OA_BUFFER_SIZE - 1)) ++ ++/** ++ * DOC: OA Tail Pointer Race ++ * ++ * There's a HW race condition between OA unit tail pointer register updates and ++ * writes to memory whereby the tail pointer can sometimes get ahead of what's ++ * been written out to the OA buffer so far (in terms of what's visible to the ++ * CPU). ++ * ++ * Although this can be observed explicitly while copying reports to userspace ++ * by checking for a zeroed report-id field in tail reports, we want to account ++ * for this earlier, as part of the oa_buffer_check to avoid lots of redundant ++ * read() attempts. ++ * ++ * In effect we define a tail pointer for reading that lags the real tail ++ * pointer by at least %OA_TAIL_MARGIN_NSEC nanoseconds, which gives enough ++ * time for the corresponding reports to become visible to the CPU. ++ * ++ * To manage this we actually track two tail pointers: ++ * 1) An 'aging' tail with an associated timestamp that is tracked until we ++ * can trust the corresponding data is visible to the CPU; at which point ++ * it is considered 'aged'. ++ * 2) An 'aged' tail that can be used for read()ing. ++ * ++ * The two separate pointers let us decouple read()s from tail pointer aging. ++ * ++ * The tail pointers are checked and updated at a limited rate within a hrtimer ++ * callback (the same callback that is used for delivering EPOLLIN events) ++ * ++ * Initially the tails are marked invalid with %INVALID_TAIL_PTR which ++ * indicates that an updated tail pointer is needed. ++ * ++ * Most of the implementation details for this workaround are in ++ * oa_buffer_check_unlocked() and _append_oa_reports() ++ * ++ * Note for posterity: previously the driver used to define an effective tail ++ * pointer that lagged the real pointer by a 'tail margin' measured in bytes ++ * derived from %OA_TAIL_MARGIN_NSEC and the configured sampling frequency. ++ * This was flawed considering that the OA unit may also automatically generate ++ * non-periodic reports (such as on context switch) or the OA unit may be ++ * enabled without any periodic sampling. ++ */ ++#define OA_TAIL_MARGIN_NSEC 100000ULL ++#define INVALID_TAIL_PTR 0xffffffff ++ ++/* frequency for checking whether the OA unit has written new reports to the ++ * circular OA buffer... ++ */ ++#define POLL_FREQUENCY 200 ++#define POLL_PERIOD (NSEC_PER_SEC / POLL_FREQUENCY) ++ ++/* for sysctl proc_dointvec_minmax of dev.i915.perf_stream_paranoid */ ++static int zero; ++static int one = 1; ++static u32 i915_perf_stream_paranoid = true; ++ ++/* The maximum exponent the hardware accepts is 63 (essentially it selects one ++ * of the 64bit timestamp bits to trigger reports from) but there's currently ++ * no known use case for sampling as infrequently as once per 47 thousand years. ++ * ++ * Since the timestamps included in OA reports are only 32bits it seems ++ * reasonable to limit the OA exponent where it's still possible to account for ++ * overflow in OA report timestamps. ++ */ ++#define OA_EXPONENT_MAX 31 ++ ++#define INVALID_CTX_ID 0xffffffff ++ ++/* On Gen8+ automatically triggered OA reports include a 'reason' field... */ ++#define OAREPORT_REASON_MASK 0x3f ++#define OAREPORT_REASON_SHIFT 19 ++#define OAREPORT_REASON_TIMER (1<<0) ++#define OAREPORT_REASON_CTX_SWITCH (1<<3) ++#define OAREPORT_REASON_CLK_RATIO (1<<5) ++ ++ ++/* For sysctl proc_dointvec_minmax of i915_oa_max_sample_rate ++ * ++ * The highest sampling frequency we can theoretically program the OA unit ++ * with is always half the timestamp frequency: E.g. 6.25Mhz for Haswell. ++ * ++ * Initialized just before we register the sysctl parameter. ++ */ ++static int oa_sample_rate_hard_limit; ++ ++/* Theoretically we can program the OA unit to sample every 160ns but don't ++ * allow that by default unless root... ++ * ++ * The default threshold of 100000Hz is based on perf's similar ++ * kernel.perf_event_max_sample_rate sysctl parameter. ++ */ ++static u32 i915_oa_max_sample_rate = 100000; ++ ++/* XXX: beware if future OA HW adds new report formats that the current ++ * code assumes all reports have a power-of-two size and ~(size - 1) can ++ * be used as a mask to align the OA tail pointer. ++ */ ++static const struct i915_oa_format hsw_oa_formats[I915_OA_FORMAT_MAX] = { ++ [I915_OA_FORMAT_A13] = { 0, 64 }, ++ [I915_OA_FORMAT_A29] = { 1, 128 }, ++ [I915_OA_FORMAT_A13_B8_C8] = { 2, 128 }, ++ /* A29_B8_C8 Disallowed as 192 bytes doesn't factor into buffer size */ ++ [I915_OA_FORMAT_B4_C8] = { 4, 64 }, ++ [I915_OA_FORMAT_A45_B8_C8] = { 5, 256 }, ++ [I915_OA_FORMAT_B4_C8_A16] = { 6, 128 }, ++ [I915_OA_FORMAT_C4_B8] = { 7, 64 }, ++}; ++ ++static const struct i915_oa_format gen8_plus_oa_formats[I915_OA_FORMAT_MAX] = { ++ [I915_OA_FORMAT_A12] = { 0, 64 }, ++ [I915_OA_FORMAT_A12_B8_C8] = { 2, 128 }, ++ [I915_OA_FORMAT_A32u40_A4u32_B8_C8] = { 5, 256 }, ++ [I915_OA_FORMAT_C4_B8] = { 7, 64 }, ++}; ++ ++#define SAMPLE_OA_REPORT (1<<0) ++ ++/** ++ * struct perf_open_properties - for validated properties given to open a stream ++ * @sample_flags: `DRM_I915_PERF_PROP_SAMPLE_*` properties are tracked as flags ++ * @single_context: Whether a single or all gpu contexts should be monitored ++ * @ctx_handle: A gem ctx handle for use with @single_context ++ * @metrics_set: An ID for an OA unit metric set advertised via sysfs ++ * @oa_format: An OA unit HW report format ++ * @oa_periodic: Whether to enable periodic OA unit sampling ++ * @oa_period_exponent: The OA unit sampling period is derived from this ++ * ++ * As read_properties_unlocked() enumerates and validates the properties given ++ * to open a stream of metrics the configuration is built up in the structure ++ * which starts out zero initialized. ++ */ ++struct perf_open_properties { ++ u32 sample_flags; ++ ++ u64 single_context:1; ++ u64 ctx_handle; ++ ++ /* OA sampling state */ ++ int metrics_set; ++ int oa_format; ++ bool oa_periodic; ++ int oa_period_exponent; ++}; ++ ++static void free_oa_config(struct drm_i915_private *dev_priv, ++ struct i915_oa_config *oa_config) ++{ ++ if (!PTR_ERR(oa_config->flex_regs)) ++ kfree(oa_config->flex_regs); ++ if (!PTR_ERR(oa_config->b_counter_regs)) ++ kfree(oa_config->b_counter_regs); ++ if (!PTR_ERR(oa_config->mux_regs)) ++ kfree(oa_config->mux_regs); ++ kfree(oa_config); ++} ++ ++static void put_oa_config(struct drm_i915_private *dev_priv, ++ struct i915_oa_config *oa_config) ++{ ++ if (!atomic_dec_and_test(&oa_config->ref_count)) ++ return; ++ ++ free_oa_config(dev_priv, oa_config); ++} ++ ++static int get_oa_config(struct drm_i915_private *dev_priv, ++ int metrics_set, ++ struct i915_oa_config **out_config) ++{ ++ int ret; ++ ++ if (metrics_set == 1) { ++ *out_config = &dev_priv->perf.oa.test_config; ++ atomic_inc(&dev_priv->perf.oa.test_config.ref_count); ++ return 0; ++ } ++ ++ ret = mutex_lock_interruptible(&dev_priv->perf.metrics_lock); ++ if (ret) ++ return ret; ++ ++ *out_config = idr_find(&dev_priv->perf.metrics_idr, metrics_set); ++ if (!*out_config) ++ ret = -EINVAL; ++ else ++ atomic_inc(&(*out_config)->ref_count); ++ ++ mutex_unlock(&dev_priv->perf.metrics_lock); ++ ++ return ret; ++} ++ ++static u32 gen8_oa_hw_tail_read(struct drm_i915_private *dev_priv) ++{ ++ return I915_READ(GEN8_OATAILPTR) & GEN8_OATAILPTR_MASK; ++} ++ ++static u32 gen7_oa_hw_tail_read(struct drm_i915_private *dev_priv) ++{ ++ u32 oastatus1 = I915_READ(GEN7_OASTATUS1); ++ ++ return oastatus1 & GEN7_OASTATUS1_TAIL_MASK; ++} ++ ++/** ++ * oa_buffer_check_unlocked - check for data and update tail ptr state ++ * @dev_priv: i915 device instance ++ * ++ * This is either called via fops (for blocking reads in user ctx) or the poll ++ * check hrtimer (atomic ctx) to check the OA buffer tail pointer and check ++ * if there is data available for userspace to read. ++ * ++ * This function is central to providing a workaround for the OA unit tail ++ * pointer having a race with respect to what data is visible to the CPU. ++ * It is responsible for reading tail pointers from the hardware and giving ++ * the pointers time to 'age' before they are made available for reading. ++ * (See description of OA_TAIL_MARGIN_NSEC above for further details.) ++ * ++ * Besides returning true when there is data available to read() this function ++ * also has the side effect of updating the oa_buffer.tails[], .aging_timestamp ++ * and .aged_tail_idx state used for reading. ++ * ++ * Note: It's safe to read OA config state here unlocked, assuming that this is ++ * only called while the stream is enabled, while the global OA configuration ++ * can't be modified. ++ * ++ * Returns: %true if the OA buffer contains data, else %false ++ */ ++static bool oa_buffer_check_unlocked(struct drm_i915_private *dev_priv) ++{ ++ int report_size = dev_priv->perf.oa.oa_buffer.format_size; ++ unsigned long flags; ++ unsigned int aged_idx; ++ u32 head, hw_tail, aged_tail, aging_tail; ++ u64 now; ++ ++ /* We have to consider the (unlikely) possibility that read() errors ++ * could result in an OA buffer reset which might reset the head, ++ * tails[] and aged_tail state. ++ */ ++ spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); ++ ++ /* NB: The head we observe here might effectively be a little out of ++ * date (between head and tails[aged_idx].offset if there is currently ++ * a read() in progress. ++ */ ++ head = dev_priv->perf.oa.oa_buffer.head; ++ ++ aged_idx = dev_priv->perf.oa.oa_buffer.aged_tail_idx; ++ aged_tail = dev_priv->perf.oa.oa_buffer.tails[aged_idx].offset; ++ aging_tail = dev_priv->perf.oa.oa_buffer.tails[!aged_idx].offset; ++ ++ hw_tail = dev_priv->perf.oa.ops.oa_hw_tail_read(dev_priv); ++ ++ /* The tail pointer increases in 64 byte increments, ++ * not in report_size steps... ++ */ ++ hw_tail &= ~(report_size - 1); ++ ++ now = ktime_get_mono_fast_ns(); ++ ++ /* Update the aged tail ++ * ++ * Flip the tail pointer available for read()s once the aging tail is ++ * old enough to trust that the corresponding data will be visible to ++ * the CPU... ++ * ++ * Do this before updating the aging pointer in case we may be able to ++ * immediately start aging a new pointer too (if new data has become ++ * available) without needing to wait for a later hrtimer callback. ++ */ ++ if (aging_tail != INVALID_TAIL_PTR && ++ ((now - dev_priv->perf.oa.oa_buffer.aging_timestamp) > ++ OA_TAIL_MARGIN_NSEC)) { ++ ++ aged_idx ^= 1; ++ dev_priv->perf.oa.oa_buffer.aged_tail_idx = aged_idx; ++ ++ aged_tail = aging_tail; ++ ++ /* Mark that we need a new pointer to start aging... */ ++ dev_priv->perf.oa.oa_buffer.tails[!aged_idx].offset = INVALID_TAIL_PTR; ++ aging_tail = INVALID_TAIL_PTR; ++ } ++ ++ /* Update the aging tail ++ * ++ * We throttle aging tail updates until we have a new tail that ++ * represents >= one report more data than is already available for ++ * reading. This ensures there will be enough data for a successful ++ * read once this new pointer has aged and ensures we will give the new ++ * pointer time to age. ++ */ ++ if (aging_tail == INVALID_TAIL_PTR && ++ (aged_tail == INVALID_TAIL_PTR || ++ OA_TAKEN(hw_tail, aged_tail) >= report_size)) { ++ struct i915_vma *vma = dev_priv->perf.oa.oa_buffer.vma; ++ u32 gtt_offset = i915_ggtt_offset(vma); ++ ++ /* Be paranoid and do a bounds check on the pointer read back ++ * from hardware, just in case some spurious hardware condition ++ * could put the tail out of bounds... ++ */ ++ if (hw_tail >= gtt_offset && ++ hw_tail < (gtt_offset + OA_BUFFER_SIZE)) { ++ dev_priv->perf.oa.oa_buffer.tails[!aged_idx].offset = ++ aging_tail = hw_tail; ++ dev_priv->perf.oa.oa_buffer.aging_timestamp = now; ++ } else { ++ DRM_ERROR("Ignoring spurious out of range OA buffer tail pointer = %u\n", ++ hw_tail); ++ } ++ } ++ ++ spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); ++ ++ return aged_tail == INVALID_TAIL_PTR ? ++ false : OA_TAKEN(aged_tail, head) >= report_size; ++} ++ ++/** ++ * append_oa_status - Appends a status record to a userspace read() buffer. ++ * @stream: An i915-perf stream opened for OA metrics ++ * @buf: destination buffer given by userspace ++ * @count: the number of bytes userspace wants to read ++ * @offset: (inout): the current position for writing into @buf ++ * @type: The kind of status to report to userspace ++ * ++ * Writes a status record (such as `DRM_I915_PERF_RECORD_OA_REPORT_LOST`) ++ * into the userspace read() buffer. ++ * ++ * The @buf @offset will only be updated on success. ++ * ++ * Returns: 0 on success, negative error code on failure. ++ */ ++static int append_oa_status(struct i915_perf_stream *stream, ++ char __user *buf, ++ size_t count, ++ size_t *offset, ++ enum drm_i915_perf_record_type type) ++{ ++ struct drm_i915_perf_record_header header = { type, 0, sizeof(header) }; ++ ++ if ((count - *offset) < header.size) ++ return -ENOSPC; ++ ++ if (copy_to_user(buf + *offset, &header, sizeof(header))) ++ return -EFAULT; ++ ++ (*offset) += header.size; ++ ++ return 0; ++} ++ ++/** ++ * append_oa_sample - Copies single OA report into userspace read() buffer. ++ * @stream: An i915-perf stream opened for OA metrics ++ * @buf: destination buffer given by userspace ++ * @count: the number of bytes userspace wants to read ++ * @offset: (inout): the current position for writing into @buf ++ * @report: A single OA report to (optionally) include as part of the sample ++ * ++ * The contents of a sample are configured through `DRM_I915_PERF_PROP_SAMPLE_*` ++ * properties when opening a stream, tracked as `stream->sample_flags`. This ++ * function copies the requested components of a single sample to the given ++ * read() @buf. ++ * ++ * The @buf @offset will only be updated on success. ++ * ++ * Returns: 0 on success, negative error code on failure. ++ */ ++static int append_oa_sample(struct i915_perf_stream *stream, ++ char __user *buf, ++ size_t count, ++ size_t *offset, ++ const u8 *report) ++{ ++ struct drm_i915_private *dev_priv = stream->dev_priv; ++ int report_size = dev_priv->perf.oa.oa_buffer.format_size; ++ struct drm_i915_perf_record_header header; ++ u32 sample_flags = stream->sample_flags; ++ ++ header.type = DRM_I915_PERF_RECORD_SAMPLE; ++ header.pad = 0; ++ header.size = stream->sample_size; ++ ++ if ((count - *offset) < header.size) ++ return -ENOSPC; ++ ++ buf += *offset; ++ if (copy_to_user(buf, &header, sizeof(header))) ++ return -EFAULT; ++ buf += sizeof(header); ++ ++ if (sample_flags & SAMPLE_OA_REPORT) { ++ if (copy_to_user(buf, report, report_size)) ++ return -EFAULT; ++ } ++ ++ (*offset) += header.size; ++ ++ return 0; ++} ++ ++/** ++ * Copies all buffered OA reports into userspace read() buffer. ++ * @stream: An i915-perf stream opened for OA metrics ++ * @buf: destination buffer given by userspace ++ * @count: the number of bytes userspace wants to read ++ * @offset: (inout): the current position for writing into @buf ++ * ++ * Notably any error condition resulting in a short read (-%ENOSPC or ++ * -%EFAULT) will be returned even though one or more records may ++ * have been successfully copied. In this case it's up to the caller ++ * to decide if the error should be squashed before returning to ++ * userspace. ++ * ++ * Note: reports are consumed from the head, and appended to the ++ * tail, so the tail chases the head?... If you think that's mad ++ * and back-to-front you're not alone, but this follows the ++ * Gen PRM naming convention. ++ * ++ * Returns: 0 on success, negative error code on failure. ++ */ ++static int gen8_append_oa_reports(struct i915_perf_stream *stream, ++ char __user *buf, ++ size_t count, ++ size_t *offset) ++{ ++ struct drm_i915_private *dev_priv = stream->dev_priv; ++ int report_size = dev_priv->perf.oa.oa_buffer.format_size; ++ u8 *oa_buf_base = dev_priv->perf.oa.oa_buffer.vaddr; ++ u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma); ++ u32 mask = (OA_BUFFER_SIZE - 1); ++ size_t start_offset = *offset; ++ unsigned long flags; ++ unsigned int aged_tail_idx; ++ u32 head, tail; ++ u32 taken; ++ int ret = 0; ++ ++ if (WARN_ON(!stream->enabled)) ++ return -EIO; ++ ++ spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); ++ ++ head = dev_priv->perf.oa.oa_buffer.head; ++ aged_tail_idx = dev_priv->perf.oa.oa_buffer.aged_tail_idx; ++ tail = dev_priv->perf.oa.oa_buffer.tails[aged_tail_idx].offset; ++ ++ spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); ++ ++ /* ++ * An invalid tail pointer here means we're still waiting for the poll ++ * hrtimer callback to give us a pointer ++ */ ++ if (tail == INVALID_TAIL_PTR) ++ return -EAGAIN; ++ ++ /* ++ * NB: oa_buffer.head/tail include the gtt_offset which we don't want ++ * while indexing relative to oa_buf_base. ++ */ ++ head -= gtt_offset; ++ tail -= gtt_offset; ++ ++ /* ++ * An out of bounds or misaligned head or tail pointer implies a driver ++ * bug since we validate + align the tail pointers we read from the ++ * hardware and we are in full control of the head pointer which should ++ * only be incremented by multiples of the report size (notably also ++ * all a power of two). ++ */ ++ if (WARN_ONCE(head > OA_BUFFER_SIZE || head % report_size || ++ tail > OA_BUFFER_SIZE || tail % report_size, ++ "Inconsistent OA buffer pointers: head = %u, tail = %u\n", ++ head, tail)) ++ return -EIO; ++ ++ ++ for (/* none */; ++ (taken = OA_TAKEN(tail, head)); ++ head = (head + report_size) & mask) { ++ u8 *report = oa_buf_base + head; ++ u32 *report32 = (void *)report; ++ u32 ctx_id; ++ u32 reason; ++ ++ /* ++ * All the report sizes factor neatly into the buffer ++ * size so we never expect to see a report split ++ * between the beginning and end of the buffer. ++ * ++ * Given the initial alignment check a misalignment ++ * here would imply a driver bug that would result ++ * in an overrun. ++ */ ++ if (WARN_ON((OA_BUFFER_SIZE - head) < report_size)) { ++ DRM_ERROR("Spurious OA head ptr: non-integral report offset\n"); ++ break; ++ } ++ ++ /* ++ * The reason field includes flags identifying what ++ * triggered this specific report (mostly timer ++ * triggered or e.g. due to a context switch). ++ * ++ * This field is never expected to be zero so we can ++ * check that the report isn't invalid before copying ++ * it to userspace... ++ */ ++ reason = ((report32[0] >> OAREPORT_REASON_SHIFT) & ++ OAREPORT_REASON_MASK); ++ if (reason == 0) { ++ if (__ratelimit(&dev_priv->perf.oa.spurious_report_rs)) ++ DRM_NOTE("Skipping spurious, invalid OA report\n"); ++ continue; ++ } ++ ++ ctx_id = report32[2] & dev_priv->perf.oa.specific_ctx_id_mask; ++ ++ /* ++ * Squash whatever is in the CTX_ID field if it's marked as ++ * invalid to be sure we avoid false-positive, single-context ++ * filtering below... ++ * ++ * Note: that we don't clear the valid_ctx_bit so userspace can ++ * understand that the ID has been squashed by the kernel. ++ */ ++ if (!(report32[0] & dev_priv->perf.oa.gen8_valid_ctx_bit)) ++ ctx_id = report32[2] = INVALID_CTX_ID; ++ ++ /* ++ * NB: For Gen 8 the OA unit no longer supports clock gating ++ * off for a specific context and the kernel can't securely ++ * stop the counters from updating as system-wide / global ++ * values. ++ * ++ * Automatic reports now include a context ID so reports can be ++ * filtered on the cpu but it's not worth trying to ++ * automatically subtract/hide counter progress for other ++ * contexts while filtering since we can't stop userspace ++ * issuing MI_REPORT_PERF_COUNT commands which would still ++ * provide a side-band view of the real values. ++ * ++ * To allow userspace (such as Mesa/GL_INTEL_performance_query) ++ * to normalize counters for a single filtered context then it ++ * needs be forwarded bookend context-switch reports so that it ++ * can track switches in between MI_REPORT_PERF_COUNT commands ++ * and can itself subtract/ignore the progress of counters ++ * associated with other contexts. Note that the hardware ++ * automatically triggers reports when switching to a new ++ * context which are tagged with the ID of the newly active ++ * context. To avoid the complexity (and likely fragility) of ++ * reading ahead while parsing reports to try and minimize ++ * forwarding redundant context switch reports (i.e. between ++ * other, unrelated contexts) we simply elect to forward them ++ * all. ++ * ++ * We don't rely solely on the reason field to identify context ++ * switches since it's not-uncommon for periodic samples to ++ * identify a switch before any 'context switch' report. ++ */ ++ if (!dev_priv->perf.oa.exclusive_stream->ctx || ++ dev_priv->perf.oa.specific_ctx_id == ctx_id || ++ (dev_priv->perf.oa.oa_buffer.last_ctx_id == ++ dev_priv->perf.oa.specific_ctx_id) || ++ reason & OAREPORT_REASON_CTX_SWITCH) { ++ ++ /* ++ * While filtering for a single context we avoid ++ * leaking the IDs of other contexts. ++ */ ++ if (dev_priv->perf.oa.exclusive_stream->ctx && ++ dev_priv->perf.oa.specific_ctx_id != ctx_id) { ++ report32[2] = INVALID_CTX_ID; ++ } ++ ++ ret = append_oa_sample(stream, buf, count, offset, ++ report); ++ if (ret) ++ break; ++ ++ dev_priv->perf.oa.oa_buffer.last_ctx_id = ctx_id; ++ } ++ ++ /* ++ * The above reason field sanity check is based on ++ * the assumption that the OA buffer is initially ++ * zeroed and we reset the field after copying so the ++ * check is still meaningful once old reports start ++ * being overwritten. ++ */ ++ report32[0] = 0; ++ } ++ ++ if (start_offset != *offset) { ++ spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); ++ ++ /* ++ * We removed the gtt_offset for the copy loop above, indexing ++ * relative to oa_buf_base so put back here... ++ */ ++ head += gtt_offset; ++ ++ I915_WRITE(GEN8_OAHEADPTR, head & GEN8_OAHEADPTR_MASK); ++ dev_priv->perf.oa.oa_buffer.head = head; ++ ++ spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); ++ } ++ ++ return ret; ++} ++ ++/** ++ * gen8_oa_read - copy status records then buffered OA reports ++ * @stream: An i915-perf stream opened for OA metrics ++ * @buf: destination buffer given by userspace ++ * @count: the number of bytes userspace wants to read ++ * @offset: (inout): the current position for writing into @buf ++ * ++ * Checks OA unit status registers and if necessary appends corresponding ++ * status records for userspace (such as for a buffer full condition) and then ++ * initiate appending any buffered OA reports. ++ * ++ * Updates @offset according to the number of bytes successfully copied into ++ * the userspace buffer. ++ * ++ * NB: some data may be successfully copied to the userspace buffer ++ * even if an error is returned, and this is reflected in the ++ * updated @offset. ++ * ++ * Returns: zero on success or a negative error code ++ */ ++static int gen8_oa_read(struct i915_perf_stream *stream, ++ char __user *buf, ++ size_t count, ++ size_t *offset) ++{ ++ struct drm_i915_private *dev_priv = stream->dev_priv; ++ u32 oastatus; ++ int ret; ++ ++ if (WARN_ON(!dev_priv->perf.oa.oa_buffer.vaddr)) ++ return -EIO; ++ ++ oastatus = I915_READ(GEN8_OASTATUS); ++ ++ /* ++ * We treat OABUFFER_OVERFLOW as a significant error: ++ * ++ * Although theoretically we could handle this more gracefully ++ * sometimes, some Gens don't correctly suppress certain ++ * automatically triggered reports in this condition and so we ++ * have to assume that old reports are now being trampled ++ * over. ++ * ++ * Considering how we don't currently give userspace control ++ * over the OA buffer size and always configure a large 16MB ++ * buffer, then a buffer overflow does anyway likely indicate ++ * that something has gone quite badly wrong. ++ */ ++ if (oastatus & GEN8_OASTATUS_OABUFFER_OVERFLOW) { ++ ret = append_oa_status(stream, buf, count, offset, ++ DRM_I915_PERF_RECORD_OA_BUFFER_LOST); ++ if (ret) ++ return ret; ++ ++ DRM_DEBUG("OA buffer overflow (exponent = %d): force restart\n", ++ dev_priv->perf.oa.period_exponent); ++ ++ dev_priv->perf.oa.ops.oa_disable(stream); ++ dev_priv->perf.oa.ops.oa_enable(stream); ++ ++ /* ++ * Note: .oa_enable() is expected to re-init the oabuffer and ++ * reset GEN8_OASTATUS for us ++ */ ++ oastatus = I915_READ(GEN8_OASTATUS); ++ } ++ ++ if (oastatus & GEN8_OASTATUS_REPORT_LOST) { ++ ret = append_oa_status(stream, buf, count, offset, ++ DRM_I915_PERF_RECORD_OA_REPORT_LOST); ++ if (ret) ++ return ret; ++ I915_WRITE(GEN8_OASTATUS, ++ oastatus & ~GEN8_OASTATUS_REPORT_LOST); ++ } ++ ++ return gen8_append_oa_reports(stream, buf, count, offset); ++} ++ ++/** ++ * Copies all buffered OA reports into userspace read() buffer. ++ * @stream: An i915-perf stream opened for OA metrics ++ * @buf: destination buffer given by userspace ++ * @count: the number of bytes userspace wants to read ++ * @offset: (inout): the current position for writing into @buf ++ * ++ * Notably any error condition resulting in a short read (-%ENOSPC or ++ * -%EFAULT) will be returned even though one or more records may ++ * have been successfully copied. In this case it's up to the caller ++ * to decide if the error should be squashed before returning to ++ * userspace. ++ * ++ * Note: reports are consumed from the head, and appended to the ++ * tail, so the tail chases the head?... If you think that's mad ++ * and back-to-front you're not alone, but this follows the ++ * Gen PRM naming convention. ++ * ++ * Returns: 0 on success, negative error code on failure. ++ */ ++static int gen7_append_oa_reports(struct i915_perf_stream *stream, ++ char __user *buf, ++ size_t count, ++ size_t *offset) ++{ ++ struct drm_i915_private *dev_priv = stream->dev_priv; ++ int report_size = dev_priv->perf.oa.oa_buffer.format_size; ++ u8 *oa_buf_base = dev_priv->perf.oa.oa_buffer.vaddr; ++ u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma); ++ u32 mask = (OA_BUFFER_SIZE - 1); ++ size_t start_offset = *offset; ++ unsigned long flags; ++ unsigned int aged_tail_idx; ++ u32 head, tail; ++ u32 taken; ++ int ret = 0; ++ ++ if (WARN_ON(!stream->enabled)) ++ return -EIO; ++ ++ spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); ++ ++ head = dev_priv->perf.oa.oa_buffer.head; ++ aged_tail_idx = dev_priv->perf.oa.oa_buffer.aged_tail_idx; ++ tail = dev_priv->perf.oa.oa_buffer.tails[aged_tail_idx].offset; ++ ++ spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); ++ ++ /* An invalid tail pointer here means we're still waiting for the poll ++ * hrtimer callback to give us a pointer ++ */ ++ if (tail == INVALID_TAIL_PTR) ++ return -EAGAIN; ++ ++ /* NB: oa_buffer.head/tail include the gtt_offset which we don't want ++ * while indexing relative to oa_buf_base. ++ */ ++ head -= gtt_offset; ++ tail -= gtt_offset; ++ ++ /* An out of bounds or misaligned head or tail pointer implies a driver ++ * bug since we validate + align the tail pointers we read from the ++ * hardware and we are in full control of the head pointer which should ++ * only be incremented by multiples of the report size (notably also ++ * all a power of two). ++ */ ++ if (WARN_ONCE(head > OA_BUFFER_SIZE || head % report_size || ++ tail > OA_BUFFER_SIZE || tail % report_size, ++ "Inconsistent OA buffer pointers: head = %u, tail = %u\n", ++ head, tail)) ++ return -EIO; ++ ++ ++ for (/* none */; ++ (taken = OA_TAKEN(tail, head)); ++ head = (head + report_size) & mask) { ++ u8 *report = oa_buf_base + head; ++ u32 *report32 = (void *)report; ++ ++ /* All the report sizes factor neatly into the buffer ++ * size so we never expect to see a report split ++ * between the beginning and end of the buffer. ++ * ++ * Given the initial alignment check a misalignment ++ * here would imply a driver bug that would result ++ * in an overrun. ++ */ ++ if (WARN_ON((OA_BUFFER_SIZE - head) < report_size)) { ++ DRM_ERROR("Spurious OA head ptr: non-integral report offset\n"); ++ break; ++ } ++ ++ /* The report-ID field for periodic samples includes ++ * some undocumented flags related to what triggered ++ * the report and is never expected to be zero so we ++ * can check that the report isn't invalid before ++ * copying it to userspace... ++ */ ++ if (report32[0] == 0) { ++ if (__ratelimit(&dev_priv->perf.oa.spurious_report_rs)) ++ DRM_NOTE("Skipping spurious, invalid OA report\n"); ++ continue; ++ } ++ ++ ret = append_oa_sample(stream, buf, count, offset, report); ++ if (ret) ++ break; ++ ++ /* The above report-id field sanity check is based on ++ * the assumption that the OA buffer is initially ++ * zeroed and we reset the field after copying so the ++ * check is still meaningful once old reports start ++ * being overwritten. ++ */ ++ report32[0] = 0; ++ } ++ ++ if (start_offset != *offset) { ++ spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); ++ ++ /* We removed the gtt_offset for the copy loop above, indexing ++ * relative to oa_buf_base so put back here... ++ */ ++ head += gtt_offset; ++ ++ I915_WRITE(GEN7_OASTATUS2, ++ ((head & GEN7_OASTATUS2_HEAD_MASK) | ++ GEN7_OASTATUS2_MEM_SELECT_GGTT)); ++ dev_priv->perf.oa.oa_buffer.head = head; ++ ++ spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); ++ } ++ ++ return ret; ++} ++ ++/** ++ * gen7_oa_read - copy status records then buffered OA reports ++ * @stream: An i915-perf stream opened for OA metrics ++ * @buf: destination buffer given by userspace ++ * @count: the number of bytes userspace wants to read ++ * @offset: (inout): the current position for writing into @buf ++ * ++ * Checks Gen 7 specific OA unit status registers and if necessary appends ++ * corresponding status records for userspace (such as for a buffer full ++ * condition) and then initiate appending any buffered OA reports. ++ * ++ * Updates @offset according to the number of bytes successfully copied into ++ * the userspace buffer. ++ * ++ * Returns: zero on success or a negative error code ++ */ ++static int gen7_oa_read(struct i915_perf_stream *stream, ++ char __user *buf, ++ size_t count, ++ size_t *offset) ++{ ++ struct drm_i915_private *dev_priv = stream->dev_priv; ++ u32 oastatus1; ++ int ret; ++ ++ if (WARN_ON(!dev_priv->perf.oa.oa_buffer.vaddr)) ++ return -EIO; ++ ++ oastatus1 = I915_READ(GEN7_OASTATUS1); ++ ++ /* XXX: On Haswell we don't have a safe way to clear oastatus1 ++ * bits while the OA unit is enabled (while the tail pointer ++ * may be updated asynchronously) so we ignore status bits ++ * that have already been reported to userspace. ++ */ ++ oastatus1 &= ~dev_priv->perf.oa.gen7_latched_oastatus1; ++ ++ /* We treat OABUFFER_OVERFLOW as a significant error: ++ * ++ * - The status can be interpreted to mean that the buffer is ++ * currently full (with a higher precedence than OA_TAKEN() ++ * which will start to report a near-empty buffer after an ++ * overflow) but it's awkward that we can't clear the status ++ * on Haswell, so without a reset we won't be able to catch ++ * the state again. ++ * ++ * - Since it also implies the HW has started overwriting old ++ * reports it may also affect our sanity checks for invalid ++ * reports when copying to userspace that assume new reports ++ * are being written to cleared memory. ++ * ++ * - In the future we may want to introduce a flight recorder ++ * mode where the driver will automatically maintain a safe ++ * guard band between head/tail, avoiding this overflow ++ * condition, but we avoid the added driver complexity for ++ * now. ++ */ ++ if (unlikely(oastatus1 & GEN7_OASTATUS1_OABUFFER_OVERFLOW)) { ++ ret = append_oa_status(stream, buf, count, offset, ++ DRM_I915_PERF_RECORD_OA_BUFFER_LOST); ++ if (ret) ++ return ret; ++ ++ DRM_DEBUG("OA buffer overflow (exponent = %d): force restart\n", ++ dev_priv->perf.oa.period_exponent); ++ ++ dev_priv->perf.oa.ops.oa_disable(stream); ++ dev_priv->perf.oa.ops.oa_enable(stream); ++ ++ oastatus1 = I915_READ(GEN7_OASTATUS1); ++ } ++ ++ if (unlikely(oastatus1 & GEN7_OASTATUS1_REPORT_LOST)) { ++ ret = append_oa_status(stream, buf, count, offset, ++ DRM_I915_PERF_RECORD_OA_REPORT_LOST); ++ if (ret) ++ return ret; ++ dev_priv->perf.oa.gen7_latched_oastatus1 |= ++ GEN7_OASTATUS1_REPORT_LOST; ++ } ++ ++ return gen7_append_oa_reports(stream, buf, count, offset); ++} ++ ++/** ++ * i915_oa_wait_unlocked - handles blocking IO until OA data available ++ * @stream: An i915-perf stream opened for OA metrics ++ * ++ * Called when userspace tries to read() from a blocking stream FD opened ++ * for OA metrics. It waits until the hrtimer callback finds a non-empty ++ * OA buffer and wakes us. ++ * ++ * Note: it's acceptable to have this return with some false positives ++ * since any subsequent read handling will return -EAGAIN if there isn't ++ * really data ready for userspace yet. ++ * ++ * Returns: zero on success or a negative error code ++ */ ++static int i915_oa_wait_unlocked(struct i915_perf_stream *stream) ++{ ++ struct drm_i915_private *dev_priv = stream->dev_priv; ++ ++ /* We would wait indefinitely if periodic sampling is not enabled */ ++ if (!dev_priv->perf.oa.periodic) ++ return -EIO; ++ ++ return wait_event_interruptible(dev_priv->perf.oa.poll_wq, ++ oa_buffer_check_unlocked(dev_priv)); ++} ++ ++/** ++ * i915_oa_poll_wait - call poll_wait() for an OA stream poll() ++ * @stream: An i915-perf stream opened for OA metrics ++ * @file: An i915 perf stream file ++ * @wait: poll() state table ++ * ++ * For handling userspace polling on an i915 perf stream opened for OA metrics, ++ * this starts a poll_wait with the wait queue that our hrtimer callback wakes ++ * when it sees data ready to read in the circular OA buffer. ++ */ ++static void i915_oa_poll_wait(struct i915_perf_stream *stream, ++ struct file *file, ++ poll_table *wait) ++{ ++ struct drm_i915_private *dev_priv = stream->dev_priv; ++ ++ poll_wait(file, &dev_priv->perf.oa.poll_wq, wait); ++} ++ ++/** ++ * i915_oa_read - just calls through to &i915_oa_ops->read ++ * @stream: An i915-perf stream opened for OA metrics ++ * @buf: destination buffer given by userspace ++ * @count: the number of bytes userspace wants to read ++ * @offset: (inout): the current position for writing into @buf ++ * ++ * Updates @offset according to the number of bytes successfully copied into ++ * the userspace buffer. ++ * ++ * Returns: zero on success or a negative error code ++ */ ++static int i915_oa_read(struct i915_perf_stream *stream, ++ char __user *buf, ++ size_t count, ++ size_t *offset) ++{ ++ struct drm_i915_private *dev_priv = stream->dev_priv; ++ ++ return dev_priv->perf.oa.ops.read(stream, buf, count, offset); ++} ++ ++static struct intel_context *oa_pin_context(struct drm_i915_private *i915, ++ struct i915_gem_context *ctx) ++{ ++ struct intel_engine_cs *engine = i915->engine[RCS0]; ++ struct intel_context *ce; ++ int ret; ++ ++ ret = i915_mutex_lock_interruptible(&i915->drm); ++ if (ret) ++ return ERR_PTR(ret); ++ ++ /* ++ * As the ID is the gtt offset of the context's vma we ++ * pin the vma to ensure the ID remains fixed. ++ * ++ * NB: implied RCS engine... ++ */ ++ ce = intel_context_pin(ctx, engine); ++ mutex_unlock(&i915->drm.struct_mutex); ++ if (IS_ERR(ce)) ++ return ce; ++ ++ i915->perf.oa.pinned_ctx = ce; ++ ++ return ce; ++} ++ ++/** ++ * oa_get_render_ctx_id - determine and hold ctx hw id ++ * @stream: An i915-perf stream opened for OA metrics ++ * ++ * Determine the render context hw id, and ensure it remains fixed for the ++ * lifetime of the stream. This ensures that we don't have to worry about ++ * updating the context ID in OACONTROL on the fly. ++ * ++ * Returns: zero on success or a negative error code ++ */ ++static int oa_get_render_ctx_id(struct i915_perf_stream *stream) ++{ ++ struct drm_i915_private *i915 = stream->dev_priv; ++ struct intel_context *ce; ++ ++ ce = oa_pin_context(i915, stream->ctx); ++ if (IS_ERR(ce)) ++ return PTR_ERR(ce); ++ ++ switch (INTEL_GEN(i915)) { ++ case 7: { ++ /* ++ * On Haswell we don't do any post processing of the reports ++ * and don't need to use the mask. ++ */ ++ i915->perf.oa.specific_ctx_id = i915_ggtt_offset(ce->state); ++ i915->perf.oa.specific_ctx_id_mask = 0; ++ break; ++ } ++ ++ case 8: ++ case 9: ++ case 10: ++ if (USES_GUC_SUBMISSION(i915)) { ++ /* ++ * When using GuC, the context descriptor we write in ++ * i915 is read by GuC and rewritten before it's ++ * actually written into the hardware. The LRCA is ++ * what is put into the context id field of the ++ * context descriptor by GuC. Because it's aligned to ++ * a page, the lower 12bits are always at 0 and ++ * dropped by GuC. They won't be part of the context ++ * ID in the OA reports, so squash those lower bits. ++ */ ++ i915->perf.oa.specific_ctx_id = ++ lower_32_bits(ce->lrc_desc) >> 12; ++ ++ /* ++ * GuC uses the top bit to signal proxy submission, so ++ * ignore that bit. ++ */ ++ i915->perf.oa.specific_ctx_id_mask = ++ (1U << (GEN8_CTX_ID_WIDTH - 1)) - 1; ++ } else { ++ i915->perf.oa.specific_ctx_id_mask = ++ (1U << GEN8_CTX_ID_WIDTH) - 1; ++ i915->perf.oa.specific_ctx_id = ++ upper_32_bits(ce->lrc_desc); ++ i915->perf.oa.specific_ctx_id &= ++ i915->perf.oa.specific_ctx_id_mask; ++ } ++ break; ++ ++ case 11: { ++ i915->perf.oa.specific_ctx_id_mask = ++ ((1U << GEN11_SW_CTX_ID_WIDTH) - 1) << (GEN11_SW_CTX_ID_SHIFT - 32) | ++ ((1U << GEN11_ENGINE_INSTANCE_WIDTH) - 1) << (GEN11_ENGINE_INSTANCE_SHIFT - 32) | ++ ((1 << GEN11_ENGINE_CLASS_WIDTH) - 1) << (GEN11_ENGINE_CLASS_SHIFT - 32); ++ i915->perf.oa.specific_ctx_id = upper_32_bits(ce->lrc_desc); ++ i915->perf.oa.specific_ctx_id &= ++ i915->perf.oa.specific_ctx_id_mask; ++ break; ++ } ++ ++ default: ++ MISSING_CASE(INTEL_GEN(i915)); ++ } ++ ++ DRM_DEBUG_DRIVER("filtering on ctx_id=0x%x ctx_id_mask=0x%x\n", ++ i915->perf.oa.specific_ctx_id, ++ i915->perf.oa.specific_ctx_id_mask); ++ ++ return 0; ++} ++ ++/** ++ * oa_put_render_ctx_id - counterpart to oa_get_render_ctx_id releases hold ++ * @stream: An i915-perf stream opened for OA metrics ++ * ++ * In case anything needed doing to ensure the context HW ID would remain valid ++ * for the lifetime of the stream, then that can be undone here. ++ */ ++static void oa_put_render_ctx_id(struct i915_perf_stream *stream) ++{ ++ struct drm_i915_private *dev_priv = stream->dev_priv; ++ struct intel_context *ce; ++ ++ dev_priv->perf.oa.specific_ctx_id = INVALID_CTX_ID; ++ dev_priv->perf.oa.specific_ctx_id_mask = 0; ++ ++ ce = fetch_and_zero(&dev_priv->perf.oa.pinned_ctx); ++ if (ce) { ++ mutex_lock(&dev_priv->drm.struct_mutex); ++ intel_context_unpin(ce); ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++ } ++} ++ ++static void ++free_oa_buffer(struct drm_i915_private *i915) ++{ ++ mutex_lock(&i915->drm.struct_mutex); ++ ++ i915_vma_unpin_and_release(&i915->perf.oa.oa_buffer.vma, ++ I915_VMA_RELEASE_MAP); ++ ++ mutex_unlock(&i915->drm.struct_mutex); ++ ++ i915->perf.oa.oa_buffer.vaddr = NULL; ++} ++ ++static void i915_oa_stream_destroy(struct i915_perf_stream *stream) ++{ ++ struct drm_i915_private *dev_priv = stream->dev_priv; ++ ++ BUG_ON(stream != dev_priv->perf.oa.exclusive_stream); ++ ++ /* ++ * Unset exclusive_stream first, it will be checked while disabling ++ * the metric set on gen8+. ++ */ ++ mutex_lock(&dev_priv->drm.struct_mutex); ++ dev_priv->perf.oa.exclusive_stream = NULL; ++ dev_priv->perf.oa.ops.disable_metric_set(dev_priv); ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++ ++ free_oa_buffer(dev_priv); ++ ++ intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); ++ intel_runtime_pm_put(dev_priv, stream->wakeref); ++ ++ if (stream->ctx) ++ oa_put_render_ctx_id(stream); ++ ++ put_oa_config(dev_priv, stream->oa_config); ++ ++ if (dev_priv->perf.oa.spurious_report_rs.missed) { ++ DRM_NOTE("%d spurious OA report notices suppressed due to ratelimiting\n", ++ dev_priv->perf.oa.spurious_report_rs.missed); ++ } ++} ++ ++static void gen7_init_oa_buffer(struct drm_i915_private *dev_priv) ++{ ++ u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma); ++ unsigned long flags; ++ ++ spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); ++ ++ /* Pre-DevBDW: OABUFFER must be set with counters off, ++ * before OASTATUS1, but after OASTATUS2 ++ */ ++ I915_WRITE(GEN7_OASTATUS2, ++ gtt_offset | GEN7_OASTATUS2_MEM_SELECT_GGTT); /* head */ ++ dev_priv->perf.oa.oa_buffer.head = gtt_offset; ++ ++ I915_WRITE(GEN7_OABUFFER, gtt_offset); ++ ++ I915_WRITE(GEN7_OASTATUS1, gtt_offset | OABUFFER_SIZE_16M); /* tail */ ++ ++ /* Mark that we need updated tail pointers to read from... */ ++ dev_priv->perf.oa.oa_buffer.tails[0].offset = INVALID_TAIL_PTR; ++ dev_priv->perf.oa.oa_buffer.tails[1].offset = INVALID_TAIL_PTR; ++ ++ spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); ++ ++ /* On Haswell we have to track which OASTATUS1 flags we've ++ * already seen since they can't be cleared while periodic ++ * sampling is enabled. ++ */ ++ dev_priv->perf.oa.gen7_latched_oastatus1 = 0; ++ ++ /* NB: although the OA buffer will initially be allocated ++ * zeroed via shmfs (and so this memset is redundant when ++ * first allocating), we may re-init the OA buffer, either ++ * when re-enabling a stream or in error/reset paths. ++ * ++ * The reason we clear the buffer for each re-init is for the ++ * sanity check in gen7_append_oa_reports() that looks at the ++ * report-id field to make sure it's non-zero which relies on ++ * the assumption that new reports are being written to zeroed ++ * memory... ++ */ ++ memset(dev_priv->perf.oa.oa_buffer.vaddr, 0, OA_BUFFER_SIZE); ++ ++ /* Maybe make ->pollin per-stream state if we support multiple ++ * concurrent streams in the future. ++ */ ++ dev_priv->perf.oa.pollin = false; ++} ++ ++static void gen8_init_oa_buffer(struct drm_i915_private *dev_priv) ++{ ++ u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma); ++ unsigned long flags; ++ ++ spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); ++ ++ I915_WRITE(GEN8_OASTATUS, 0); ++ I915_WRITE(GEN8_OAHEADPTR, gtt_offset); ++ dev_priv->perf.oa.oa_buffer.head = gtt_offset; ++ ++ I915_WRITE(GEN8_OABUFFER_UDW, 0); ++ ++ /* ++ * PRM says: ++ * ++ * "This MMIO must be set before the OATAILPTR ++ * register and after the OAHEADPTR register. This is ++ * to enable proper functionality of the overflow ++ * bit." ++ */ ++ I915_WRITE(GEN8_OABUFFER, gtt_offset | ++ OABUFFER_SIZE_16M | GEN8_OABUFFER_MEM_SELECT_GGTT); ++ I915_WRITE(GEN8_OATAILPTR, gtt_offset & GEN8_OATAILPTR_MASK); ++ ++ /* Mark that we need updated tail pointers to read from... */ ++ dev_priv->perf.oa.oa_buffer.tails[0].offset = INVALID_TAIL_PTR; ++ dev_priv->perf.oa.oa_buffer.tails[1].offset = INVALID_TAIL_PTR; ++ ++ /* ++ * Reset state used to recognise context switches, affecting which ++ * reports we will forward to userspace while filtering for a single ++ * context. ++ */ ++ dev_priv->perf.oa.oa_buffer.last_ctx_id = INVALID_CTX_ID; ++ ++ spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); ++ ++ /* ++ * NB: although the OA buffer will initially be allocated ++ * zeroed via shmfs (and so this memset is redundant when ++ * first allocating), we may re-init the OA buffer, either ++ * when re-enabling a stream or in error/reset paths. ++ * ++ * The reason we clear the buffer for each re-init is for the ++ * sanity check in gen8_append_oa_reports() that looks at the ++ * reason field to make sure it's non-zero which relies on ++ * the assumption that new reports are being written to zeroed ++ * memory... ++ */ ++ memset(dev_priv->perf.oa.oa_buffer.vaddr, 0, OA_BUFFER_SIZE); ++ ++ /* ++ * Maybe make ->pollin per-stream state if we support multiple ++ * concurrent streams in the future. ++ */ ++ dev_priv->perf.oa.pollin = false; ++} ++ ++static int alloc_oa_buffer(struct drm_i915_private *dev_priv) ++{ ++ struct drm_i915_gem_object *bo; ++ struct i915_vma *vma; ++ int ret; ++ ++ if (WARN_ON(dev_priv->perf.oa.oa_buffer.vma)) ++ return -ENODEV; ++ ++ ret = i915_mutex_lock_interruptible(&dev_priv->drm); ++ if (ret) ++ return ret; ++ ++ BUILD_BUG_ON_NOT_POWER_OF_2(OA_BUFFER_SIZE); ++ BUILD_BUG_ON(OA_BUFFER_SIZE < SZ_128K || OA_BUFFER_SIZE > SZ_16M); ++ ++ bo = i915_gem_object_create(dev_priv, OA_BUFFER_SIZE); ++ if (IS_ERR(bo)) { ++ DRM_ERROR("Failed to allocate OA buffer\n"); ++ ret = PTR_ERR(bo); ++ goto unlock; ++ } ++ ++ i915_gem_object_set_cache_coherency(bo, I915_CACHE_LLC); ++ ++ /* PreHSW required 512K alignment, HSW requires 16M */ ++ vma = i915_gem_object_ggtt_pin(bo, NULL, 0, SZ_16M, 0); ++ if (IS_ERR(vma)) { ++ ret = PTR_ERR(vma); ++ goto err_unref; ++ } ++ dev_priv->perf.oa.oa_buffer.vma = vma; ++ ++ dev_priv->perf.oa.oa_buffer.vaddr = ++ i915_gem_object_pin_map(bo, I915_MAP_WB); ++ if (IS_ERR(dev_priv->perf.oa.oa_buffer.vaddr)) { ++ ret = PTR_ERR(dev_priv->perf.oa.oa_buffer.vaddr); ++ goto err_unpin; ++ } ++ ++ DRM_DEBUG_DRIVER("OA Buffer initialized, gtt offset = 0x%x, vaddr = %p\n", ++ i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma), ++ dev_priv->perf.oa.oa_buffer.vaddr); ++ ++ goto unlock; ++ ++err_unpin: ++ __i915_vma_unpin(vma); ++ ++err_unref: ++ i915_gem_object_put(bo); ++ ++ dev_priv->perf.oa.oa_buffer.vaddr = NULL; ++ dev_priv->perf.oa.oa_buffer.vma = NULL; ++ ++unlock: ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++ return ret; ++} ++ ++static void config_oa_regs(struct drm_i915_private *dev_priv, ++ const struct i915_oa_reg *regs, ++ u32 n_regs) ++{ ++ u32 i; ++ ++ for (i = 0; i < n_regs; i++) { ++ const struct i915_oa_reg *reg = regs + i; ++ ++ I915_WRITE(reg->addr, reg->value); ++ } ++} ++ ++static int hsw_enable_metric_set(struct i915_perf_stream *stream) ++{ ++ struct drm_i915_private *dev_priv = stream->dev_priv; ++ const struct i915_oa_config *oa_config = stream->oa_config; ++ ++ /* PRM: ++ * ++ * OA unit is using “crclk” for its functionality. When trunk ++ * level clock gating takes place, OA clock would be gated, ++ * unable to count the events from non-render clock domain. ++ * Render clock gating must be disabled when OA is enabled to ++ * count the events from non-render domain. Unit level clock ++ * gating for RCS should also be disabled. ++ */ ++ I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) & ++ ~GEN7_DOP_CLOCK_GATE_ENABLE)); ++ I915_WRITE(GEN6_UCGCTL1, (I915_READ(GEN6_UCGCTL1) | ++ GEN6_CSUNIT_CLOCK_GATE_DISABLE)); ++ ++ config_oa_regs(dev_priv, oa_config->mux_regs, oa_config->mux_regs_len); ++ ++ /* It apparently takes a fairly long time for a new MUX ++ * configuration to be be applied after these register writes. ++ * This delay duration was derived empirically based on the ++ * render_basic config but hopefully it covers the maximum ++ * configuration latency. ++ * ++ * As a fallback, the checks in _append_oa_reports() to skip ++ * invalid OA reports do also seem to work to discard reports ++ * generated before this config has completed - albeit not ++ * silently. ++ * ++ * Unfortunately this is essentially a magic number, since we ++ * don't currently know of a reliable mechanism for predicting ++ * how long the MUX config will take to apply and besides ++ * seeing invalid reports we don't know of a reliable way to ++ * explicitly check that the MUX config has landed. ++ * ++ * It's even possible we've miss characterized the underlying ++ * problem - it just seems like the simplest explanation why ++ * a delay at this location would mitigate any invalid reports. ++ */ ++ usleep_range(15000, 20000); ++ ++ config_oa_regs(dev_priv, oa_config->b_counter_regs, ++ oa_config->b_counter_regs_len); ++ ++ return 0; ++} ++ ++static void hsw_disable_metric_set(struct drm_i915_private *dev_priv) ++{ ++ I915_WRITE(GEN6_UCGCTL1, (I915_READ(GEN6_UCGCTL1) & ++ ~GEN6_CSUNIT_CLOCK_GATE_DISABLE)); ++ I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) | ++ GEN7_DOP_CLOCK_GATE_ENABLE)); ++ ++ I915_WRITE(GDT_CHICKEN_BITS, (I915_READ(GDT_CHICKEN_BITS) & ++ ~GT_NOA_ENABLE)); ++} ++ ++/* ++ * NB: It must always remain pointer safe to run this even if the OA unit ++ * has been disabled. ++ * ++ * It's fine to put out-of-date values into these per-context registers ++ * in the case that the OA unit has been disabled. ++ */ ++static void ++gen8_update_reg_state_unlocked(struct intel_context *ce, ++ u32 *reg_state, ++ const struct i915_oa_config *oa_config) ++{ ++ struct drm_i915_private *i915 = ce->gem_context->i915; ++ u32 ctx_oactxctrl = i915->perf.oa.ctx_oactxctrl_offset; ++ u32 ctx_flexeu0 = i915->perf.oa.ctx_flexeu0_offset; ++ /* The MMIO offsets for Flex EU registers aren't contiguous */ ++ i915_reg_t flex_regs[] = { ++ EU_PERF_CNTL0, ++ EU_PERF_CNTL1, ++ EU_PERF_CNTL2, ++ EU_PERF_CNTL3, ++ EU_PERF_CNTL4, ++ EU_PERF_CNTL5, ++ EU_PERF_CNTL6, ++ }; ++ int i; ++ ++ CTX_REG(reg_state, ctx_oactxctrl, GEN8_OACTXCONTROL, ++ (i915->perf.oa.period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) | ++ (i915->perf.oa.periodic ? GEN8_OA_TIMER_ENABLE : 0) | ++ GEN8_OA_COUNTER_RESUME); ++ ++ for (i = 0; i < ARRAY_SIZE(flex_regs); i++) { ++ u32 state_offset = ctx_flexeu0 + i * 2; ++ u32 mmio = i915_mmio_reg_offset(flex_regs[i]); ++ ++ /* ++ * This arbitrary default will select the 'EU FPU0 Pipeline ++ * Active' event. In the future it's anticipated that there ++ * will be an explicit 'No Event' we can select, but not yet... ++ */ ++ u32 value = 0; ++ ++ if (oa_config) { ++ u32 j; ++ ++ for (j = 0; j < oa_config->flex_regs_len; j++) { ++ if (i915_mmio_reg_offset(oa_config->flex_regs[j].addr) == mmio) { ++ value = oa_config->flex_regs[j].value; ++ break; ++ } ++ } ++ } ++ ++ CTX_REG(reg_state, state_offset, flex_regs[i], value); ++ } ++ ++ CTX_REG(reg_state, ++ CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE, ++ gen8_make_rpcs(i915, &ce->sseu)); ++} ++ ++/* ++ * Manages updating the per-context aspects of the OA stream ++ * configuration across all contexts. ++ * ++ * The awkward consideration here is that OACTXCONTROL controls the ++ * exponent for periodic sampling which is primarily used for system ++ * wide profiling where we'd like a consistent sampling period even in ++ * the face of context switches. ++ * ++ * Our approach of updating the register state context (as opposed to ++ * say using a workaround batch buffer) ensures that the hardware ++ * won't automatically reload an out-of-date timer exponent even ++ * transiently before a WA BB could be parsed. ++ * ++ * This function needs to: ++ * - Ensure the currently running context's per-context OA state is ++ * updated ++ * - Ensure that all existing contexts will have the correct per-context ++ * OA state if they are scheduled for use. ++ * - Ensure any new contexts will be initialized with the correct ++ * per-context OA state. ++ * ++ * Note: it's only the RCS/Render context that has any OA state. ++ */ ++static int gen8_configure_all_contexts(struct drm_i915_private *dev_priv, ++ const struct i915_oa_config *oa_config) ++{ ++ struct intel_engine_cs *engine = dev_priv->engine[RCS0]; ++ unsigned int map_type = i915_coherent_map_type(dev_priv); ++ struct i915_gem_context *ctx; ++ struct i915_request *rq; ++ int ret; ++ ++ lockdep_assert_held(&dev_priv->drm.struct_mutex); ++ ++ /* ++ * The OA register config is setup through the context image. This image ++ * might be written to by the GPU on context switch (in particular on ++ * lite-restore). This means we can't safely update a context's image, ++ * if this context is scheduled/submitted to run on the GPU. ++ * ++ * We could emit the OA register config through the batch buffer but ++ * this might leave small interval of time where the OA unit is ++ * configured at an invalid sampling period. ++ * ++ * So far the best way to work around this issue seems to be draining ++ * the GPU from any submitted work. ++ */ ++ ret = i915_gem_wait_for_idle(dev_priv, ++ I915_WAIT_LOCKED, ++ MAX_SCHEDULE_TIMEOUT); ++ if (ret) ++ return ret; ++ ++ /* Update all contexts now that we've stalled the submission. */ ++ list_for_each_entry(ctx, &dev_priv->contexts.list, link) { ++ struct intel_context *ce = intel_context_lookup(ctx, engine); ++ u32 *regs; ++ ++ /* OA settings will be set upon first use */ ++ if (!ce || !ce->state) ++ continue; ++ ++ regs = i915_gem_object_pin_map(ce->state->obj, map_type); ++ if (IS_ERR(regs)) ++ return PTR_ERR(regs); ++ ++ ce->state->obj->mm.dirty = true; ++ regs += LRC_STATE_PN * PAGE_SIZE / sizeof(*regs); ++ ++ gen8_update_reg_state_unlocked(ce, regs, oa_config); ++ ++ i915_gem_object_unpin_map(ce->state->obj); ++ } ++ ++ /* ++ * Apply the configuration by doing one context restore of the edited ++ * context image. ++ */ ++ rq = i915_request_alloc(engine, dev_priv->kernel_context); ++ if (IS_ERR(rq)) ++ return PTR_ERR(rq); ++ ++ i915_request_add(rq); ++ ++ return 0; ++} ++ ++static int gen8_enable_metric_set(struct i915_perf_stream *stream) ++{ ++ struct drm_i915_private *dev_priv = stream->dev_priv; ++ const struct i915_oa_config *oa_config = stream->oa_config; ++ int ret; ++ ++ /* ++ * We disable slice/unslice clock ratio change reports on SKL since ++ * they are too noisy. The HW generates a lot of redundant reports ++ * where the ratio hasn't really changed causing a lot of redundant ++ * work to processes and increasing the chances we'll hit buffer ++ * overruns. ++ * ++ * Although we don't currently use the 'disable overrun' OABUFFER ++ * feature it's worth noting that clock ratio reports have to be ++ * disabled before considering to use that feature since the HW doesn't ++ * correctly block these reports. ++ * ++ * Currently none of the high-level metrics we have depend on knowing ++ * this ratio to normalize. ++ * ++ * Note: This register is not power context saved and restored, but ++ * that's OK considering that we disable RC6 while the OA unit is ++ * enabled. ++ * ++ * The _INCLUDE_CLK_RATIO bit allows the slice/unslice frequency to ++ * be read back from automatically triggered reports, as part of the ++ * RPT_ID field. ++ */ ++ if (IS_GEN_RANGE(dev_priv, 9, 11)) { ++ I915_WRITE(GEN8_OA_DEBUG, ++ _MASKED_BIT_ENABLE(GEN9_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS | ++ GEN9_OA_DEBUG_INCLUDE_CLK_RATIO)); ++ } ++ ++ /* ++ * Update all contexts prior writing the mux configurations as we need ++ * to make sure all slices/subslices are ON before writing to NOA ++ * registers. ++ */ ++ ret = gen8_configure_all_contexts(dev_priv, oa_config); ++ if (ret) ++ return ret; ++ ++ config_oa_regs(dev_priv, oa_config->mux_regs, oa_config->mux_regs_len); ++ ++ config_oa_regs(dev_priv, oa_config->b_counter_regs, ++ oa_config->b_counter_regs_len); ++ ++ return 0; ++} ++ ++static void gen8_disable_metric_set(struct drm_i915_private *dev_priv) ++{ ++ /* Reset all contexts' slices/subslices configurations. */ ++ gen8_configure_all_contexts(dev_priv, NULL); ++ ++ I915_WRITE(GDT_CHICKEN_BITS, (I915_READ(GDT_CHICKEN_BITS) & ++ ~GT_NOA_ENABLE)); ++} ++ ++static void gen10_disable_metric_set(struct drm_i915_private *dev_priv) ++{ ++ /* Reset all contexts' slices/subslices configurations. */ ++ gen8_configure_all_contexts(dev_priv, NULL); ++ ++ /* Make sure we disable noa to save power. */ ++ I915_WRITE(RPM_CONFIG1, ++ I915_READ(RPM_CONFIG1) & ~GEN10_GT_NOA_ENABLE); ++} ++ ++static void gen7_oa_enable(struct i915_perf_stream *stream) ++{ ++ struct drm_i915_private *dev_priv = stream->dev_priv; ++ struct i915_gem_context *ctx = stream->ctx; ++ u32 ctx_id = dev_priv->perf.oa.specific_ctx_id; ++ bool periodic = dev_priv->perf.oa.periodic; ++ u32 period_exponent = dev_priv->perf.oa.period_exponent; ++ u32 report_format = dev_priv->perf.oa.oa_buffer.format; ++ ++ /* ++ * Reset buf pointers so we don't forward reports from before now. ++ * ++ * Think carefully if considering trying to avoid this, since it ++ * also ensures status flags and the buffer itself are cleared ++ * in error paths, and we have checks for invalid reports based ++ * on the assumption that certain fields are written to zeroed ++ * memory which this helps maintains. ++ */ ++ gen7_init_oa_buffer(dev_priv); ++ ++ I915_WRITE(GEN7_OACONTROL, ++ (ctx_id & GEN7_OACONTROL_CTX_MASK) | ++ (period_exponent << ++ GEN7_OACONTROL_TIMER_PERIOD_SHIFT) | ++ (periodic ? GEN7_OACONTROL_TIMER_ENABLE : 0) | ++ (report_format << GEN7_OACONTROL_FORMAT_SHIFT) | ++ (ctx ? GEN7_OACONTROL_PER_CTX_ENABLE : 0) | ++ GEN7_OACONTROL_ENABLE); ++} ++ ++static void gen8_oa_enable(struct i915_perf_stream *stream) ++{ ++ struct drm_i915_private *dev_priv = stream->dev_priv; ++ u32 report_format = dev_priv->perf.oa.oa_buffer.format; ++ ++ /* ++ * Reset buf pointers so we don't forward reports from before now. ++ * ++ * Think carefully if considering trying to avoid this, since it ++ * also ensures status flags and the buffer itself are cleared ++ * in error paths, and we have checks for invalid reports based ++ * on the assumption that certain fields are written to zeroed ++ * memory which this helps maintains. ++ */ ++ gen8_init_oa_buffer(dev_priv); ++ ++ /* ++ * Note: we don't rely on the hardware to perform single context ++ * filtering and instead filter on the cpu based on the context-id ++ * field of reports ++ */ ++ I915_WRITE(GEN8_OACONTROL, (report_format << ++ GEN8_OA_REPORT_FORMAT_SHIFT) | ++ GEN8_OA_COUNTER_ENABLE); ++} ++ ++/** ++ * i915_oa_stream_enable - handle `I915_PERF_IOCTL_ENABLE` for OA stream ++ * @stream: An i915 perf stream opened for OA metrics ++ * ++ * [Re]enables hardware periodic sampling according to the period configured ++ * when opening the stream. This also starts a hrtimer that will periodically ++ * check for data in the circular OA buffer for notifying userspace (e.g. ++ * during a read() or poll()). ++ */ ++static void i915_oa_stream_enable(struct i915_perf_stream *stream) ++{ ++ struct drm_i915_private *dev_priv = stream->dev_priv; ++ ++ dev_priv->perf.oa.ops.oa_enable(stream); ++ ++ if (dev_priv->perf.oa.periodic) ++ hrtimer_start(&dev_priv->perf.oa.poll_check_timer, ++ ns_to_ktime(POLL_PERIOD), ++ HRTIMER_MODE_REL_PINNED); ++} ++ ++static void gen7_oa_disable(struct i915_perf_stream *stream) ++{ ++ struct intel_uncore *uncore = &stream->dev_priv->uncore; ++ ++ intel_uncore_write(uncore, GEN7_OACONTROL, 0); ++ if (intel_wait_for_register(uncore, ++ GEN7_OACONTROL, GEN7_OACONTROL_ENABLE, 0, ++ 50)) ++ DRM_ERROR("wait for OA to be disabled timed out\n"); ++} ++ ++static void gen8_oa_disable(struct i915_perf_stream *stream) ++{ ++ struct intel_uncore *uncore = &stream->dev_priv->uncore; ++ ++ intel_uncore_write(uncore, GEN8_OACONTROL, 0); ++ if (intel_wait_for_register(uncore, ++ GEN8_OACONTROL, GEN8_OA_COUNTER_ENABLE, 0, ++ 50)) ++ DRM_ERROR("wait for OA to be disabled timed out\n"); ++} ++ ++/** ++ * i915_oa_stream_disable - handle `I915_PERF_IOCTL_DISABLE` for OA stream ++ * @stream: An i915 perf stream opened for OA metrics ++ * ++ * Stops the OA unit from periodically writing counter reports into the ++ * circular OA buffer. This also stops the hrtimer that periodically checks for ++ * data in the circular OA buffer, for notifying userspace. ++ */ ++static void i915_oa_stream_disable(struct i915_perf_stream *stream) ++{ ++ struct drm_i915_private *dev_priv = stream->dev_priv; ++ ++ dev_priv->perf.oa.ops.oa_disable(stream); ++ ++ if (dev_priv->perf.oa.periodic) ++ hrtimer_cancel(&dev_priv->perf.oa.poll_check_timer); ++} ++ ++static const struct i915_perf_stream_ops i915_oa_stream_ops = { ++ .destroy = i915_oa_stream_destroy, ++ .enable = i915_oa_stream_enable, ++ .disable = i915_oa_stream_disable, ++ .wait_unlocked = i915_oa_wait_unlocked, ++ .poll_wait = i915_oa_poll_wait, ++ .read = i915_oa_read, ++}; ++ ++/** ++ * i915_oa_stream_init - validate combined props for OA stream and init ++ * @stream: An i915 perf stream ++ * @param: The open parameters passed to `DRM_I915_PERF_OPEN` ++ * @props: The property state that configures stream (individually validated) ++ * ++ * While read_properties_unlocked() validates properties in isolation it ++ * doesn't ensure that the combination necessarily makes sense. ++ * ++ * At this point it has been determined that userspace wants a stream of ++ * OA metrics, but still we need to further validate the combined ++ * properties are OK. ++ * ++ * If the configuration makes sense then we can allocate memory for ++ * a circular OA buffer and apply the requested metric set configuration. ++ * ++ * Returns: zero on success or a negative error code. ++ */ ++static int i915_oa_stream_init(struct i915_perf_stream *stream, ++ struct drm_i915_perf_open_param *param, ++ struct perf_open_properties *props) ++{ ++ struct drm_i915_private *dev_priv = stream->dev_priv; ++ int format_size; ++ int ret; ++ ++ /* If the sysfs metrics/ directory wasn't registered for some ++ * reason then don't let userspace try their luck with config ++ * IDs ++ */ ++ if (!dev_priv->perf.metrics_kobj) { ++ DRM_DEBUG("OA metrics weren't advertised via sysfs\n"); ++ return -EINVAL; ++ } ++ ++ if (!(props->sample_flags & SAMPLE_OA_REPORT)) { ++ DRM_DEBUG("Only OA report sampling supported\n"); ++ return -EINVAL; ++ } ++ ++ if (!dev_priv->perf.oa.ops.enable_metric_set) { ++ DRM_DEBUG("OA unit not supported\n"); ++ return -ENODEV; ++ } ++ ++ /* To avoid the complexity of having to accurately filter ++ * counter reports and marshal to the appropriate client ++ * we currently only allow exclusive access ++ */ ++ if (dev_priv->perf.oa.exclusive_stream) { ++ DRM_DEBUG("OA unit already in use\n"); ++ return -EBUSY; ++ } ++ ++ if (!props->oa_format) { ++ DRM_DEBUG("OA report format not specified\n"); ++ return -EINVAL; ++ } ++ ++ /* We set up some ratelimit state to potentially throttle any _NOTES ++ * about spurious, invalid OA reports which we don't forward to ++ * userspace. ++ * ++ * The initialization is associated with opening the stream (not driver ++ * init) considering we print a _NOTE about any throttling when closing ++ * the stream instead of waiting until driver _fini which no one would ++ * ever see. ++ * ++ * Using the same limiting factors as printk_ratelimit() ++ */ ++ ratelimit_state_init(&dev_priv->perf.oa.spurious_report_rs, ++ 5 * HZ, 10); ++ /* Since we use a DRM_NOTE for spurious reports it would be ++ * inconsistent to let __ratelimit() automatically print a warning for ++ * throttling. ++ */ ++ ratelimit_set_flags(&dev_priv->perf.oa.spurious_report_rs, ++ RATELIMIT_MSG_ON_RELEASE); ++ ++ stream->sample_size = sizeof(struct drm_i915_perf_record_header); ++ ++ format_size = dev_priv->perf.oa.oa_formats[props->oa_format].size; ++ ++ stream->sample_flags |= SAMPLE_OA_REPORT; ++ stream->sample_size += format_size; ++ ++ dev_priv->perf.oa.oa_buffer.format_size = format_size; ++ if (WARN_ON(dev_priv->perf.oa.oa_buffer.format_size == 0)) ++ return -EINVAL; ++ ++ dev_priv->perf.oa.oa_buffer.format = ++ dev_priv->perf.oa.oa_formats[props->oa_format].format; ++ ++ dev_priv->perf.oa.periodic = props->oa_periodic; ++ if (dev_priv->perf.oa.periodic) ++ dev_priv->perf.oa.period_exponent = props->oa_period_exponent; ++ ++ if (stream->ctx) { ++ ret = oa_get_render_ctx_id(stream); ++ if (ret) { ++ DRM_DEBUG("Invalid context id to filter with\n"); ++ return ret; ++ } ++ } ++ ++ ret = get_oa_config(dev_priv, props->metrics_set, &stream->oa_config); ++ if (ret) { ++ DRM_DEBUG("Invalid OA config id=%i\n", props->metrics_set); ++ goto err_config; ++ } ++ ++ /* PRM - observability performance counters: ++ * ++ * OACONTROL, performance counter enable, note: ++ * ++ * "When this bit is set, in order to have coherent counts, ++ * RC6 power state and trunk clock gating must be disabled. ++ * This can be achieved by programming MMIO registers as ++ * 0xA094=0 and 0xA090[31]=1" ++ * ++ * In our case we are expecting that taking pm + FORCEWAKE ++ * references will effectively disable RC6. ++ */ ++ stream->wakeref = intel_runtime_pm_get(dev_priv); ++ intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL); ++ ++ ret = alloc_oa_buffer(dev_priv); ++ if (ret) ++ goto err_oa_buf_alloc; ++ ++ ret = i915_mutex_lock_interruptible(&dev_priv->drm); ++ if (ret) ++ goto err_lock; ++ ++ stream->ops = &i915_oa_stream_ops; ++ dev_priv->perf.oa.exclusive_stream = stream; ++ ++ ret = dev_priv->perf.oa.ops.enable_metric_set(stream); ++ if (ret) { ++ DRM_DEBUG("Unable to enable metric set\n"); ++ goto err_enable; ++ } ++ ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++ ++ return 0; ++ ++err_enable: ++ dev_priv->perf.oa.exclusive_stream = NULL; ++ dev_priv->perf.oa.ops.disable_metric_set(dev_priv); ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++ ++err_lock: ++ free_oa_buffer(dev_priv); ++ ++err_oa_buf_alloc: ++ put_oa_config(dev_priv, stream->oa_config); ++ ++ intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); ++ intel_runtime_pm_put(dev_priv, stream->wakeref); ++ ++err_config: ++ if (stream->ctx) ++ oa_put_render_ctx_id(stream); ++ ++ return ret; ++} ++ ++void i915_oa_init_reg_state(struct intel_engine_cs *engine, ++ struct intel_context *ce, ++ u32 *regs) ++{ ++ struct i915_perf_stream *stream; ++ ++ if (engine->class != RENDER_CLASS) ++ return; ++ ++ stream = engine->i915->perf.oa.exclusive_stream; ++ if (stream) ++ gen8_update_reg_state_unlocked(ce, regs, stream->oa_config); ++} ++ ++/** ++ * i915_perf_read_locked - &i915_perf_stream_ops->read with error normalisation ++ * @stream: An i915 perf stream ++ * @file: An i915 perf stream file ++ * @buf: destination buffer given by userspace ++ * @count: the number of bytes userspace wants to read ++ * @ppos: (inout) file seek position (unused) ++ * ++ * Besides wrapping &i915_perf_stream_ops->read this provides a common place to ++ * ensure that if we've successfully copied any data then reporting that takes ++ * precedence over any internal error status, so the data isn't lost. ++ * ++ * For example ret will be -ENOSPC whenever there is more buffered data than ++ * can be copied to userspace, but that's only interesting if we weren't able ++ * to copy some data because it implies the userspace buffer is too small to ++ * receive a single record (and we never split records). ++ * ++ * Another case with ret == -EFAULT is more of a grey area since it would seem ++ * like bad form for userspace to ask us to overrun its buffer, but the user ++ * knows best: ++ * ++ * http://yarchive.net/comp/linux/partial_reads_writes.html ++ * ++ * Returns: The number of bytes copied or a negative error code on failure. ++ */ ++static ssize_t i915_perf_read_locked(struct i915_perf_stream *stream, ++ struct file *file, ++ char __user *buf, ++ size_t count, ++ loff_t *ppos) ++{ ++ /* Note we keep the offset (aka bytes read) separate from any ++ * error status so that the final check for whether we return ++ * the bytes read with a higher precedence than any error (see ++ * comment below) doesn't need to be handled/duplicated in ++ * stream->ops->read() implementations. ++ */ ++ size_t offset = 0; ++ int ret = stream->ops->read(stream, buf, count, &offset); ++ ++ return offset ?: (ret ?: -EAGAIN); ++} ++ ++/** ++ * i915_perf_read - handles read() FOP for i915 perf stream FDs ++ * @file: An i915 perf stream file ++ * @buf: destination buffer given by userspace ++ * @count: the number of bytes userspace wants to read ++ * @ppos: (inout) file seek position (unused) ++ * ++ * The entry point for handling a read() on a stream file descriptor from ++ * userspace. Most of the work is left to the i915_perf_read_locked() and ++ * &i915_perf_stream_ops->read but to save having stream implementations (of ++ * which we might have multiple later) we handle blocking read here. ++ * ++ * We can also consistently treat trying to read from a disabled stream ++ * as an IO error so implementations can assume the stream is enabled ++ * while reading. ++ * ++ * Returns: The number of bytes copied or a negative error code on failure. ++ */ ++static ssize_t i915_perf_read(struct file *file, ++ char __user *buf, ++ size_t count, ++ loff_t *ppos) ++{ ++ struct i915_perf_stream *stream = file->private_data; ++ struct drm_i915_private *dev_priv = stream->dev_priv; ++ ssize_t ret; ++ ++ /* To ensure it's handled consistently we simply treat all reads of a ++ * disabled stream as an error. In particular it might otherwise lead ++ * to a deadlock for blocking file descriptors... ++ */ ++ if (!stream->enabled) ++ return -EIO; ++ ++ if (!(file->f_flags & O_NONBLOCK)) { ++ /* There's the small chance of false positives from ++ * stream->ops->wait_unlocked. ++ * ++ * E.g. with single context filtering since we only wait until ++ * oabuffer has >= 1 report we don't immediately know whether ++ * any reports really belong to the current context ++ */ ++ do { ++ ret = stream->ops->wait_unlocked(stream); ++ if (ret) ++ return ret; ++ ++ mutex_lock(&dev_priv->perf.lock); ++ ret = i915_perf_read_locked(stream, file, ++ buf, count, ppos); ++ mutex_unlock(&dev_priv->perf.lock); ++ } while (ret == -EAGAIN); ++ } else { ++ mutex_lock(&dev_priv->perf.lock); ++ ret = i915_perf_read_locked(stream, file, buf, count, ppos); ++ mutex_unlock(&dev_priv->perf.lock); ++ } ++ ++ /* We allow the poll checking to sometimes report false positive EPOLLIN ++ * events where we might actually report EAGAIN on read() if there's ++ * not really any data available. In this situation though we don't ++ * want to enter a busy loop between poll() reporting a EPOLLIN event ++ * and read() returning -EAGAIN. Clearing the oa.pollin state here ++ * effectively ensures we back off until the next hrtimer callback ++ * before reporting another EPOLLIN event. ++ */ ++ if (ret >= 0 || ret == -EAGAIN) { ++ /* Maybe make ->pollin per-stream state if we support multiple ++ * concurrent streams in the future. ++ */ ++ dev_priv->perf.oa.pollin = false; ++ } ++ ++ return ret; ++} ++ ++static enum hrtimer_restart oa_poll_check_timer_cb(struct hrtimer *hrtimer) ++{ ++ struct drm_i915_private *dev_priv = ++ container_of(hrtimer, typeof(*dev_priv), ++ perf.oa.poll_check_timer); ++ ++ if (oa_buffer_check_unlocked(dev_priv)) { ++ dev_priv->perf.oa.pollin = true; ++ wake_up(&dev_priv->perf.oa.poll_wq); ++ } ++ ++ hrtimer_forward_now(hrtimer, ns_to_ktime(POLL_PERIOD)); ++ ++ return HRTIMER_RESTART; ++} ++ ++/** ++ * i915_perf_poll_locked - poll_wait() with a suitable wait queue for stream ++ * @dev_priv: i915 device instance ++ * @stream: An i915 perf stream ++ * @file: An i915 perf stream file ++ * @wait: poll() state table ++ * ++ * For handling userspace polling on an i915 perf stream, this calls through to ++ * &i915_perf_stream_ops->poll_wait to call poll_wait() with a wait queue that ++ * will be woken for new stream data. ++ * ++ * Note: The &drm_i915_private->perf.lock mutex has been taken to serialize ++ * with any non-file-operation driver hooks. ++ * ++ * Returns: any poll events that are ready without sleeping ++ */ ++static __poll_t i915_perf_poll_locked(struct drm_i915_private *dev_priv, ++ struct i915_perf_stream *stream, ++ struct file *file, ++ poll_table *wait) ++{ ++ __poll_t events = 0; ++ ++ stream->ops->poll_wait(stream, file, wait); ++ ++ /* Note: we don't explicitly check whether there's something to read ++ * here since this path may be very hot depending on what else ++ * userspace is polling, or on the timeout in use. We rely solely on ++ * the hrtimer/oa_poll_check_timer_cb to notify us when there are ++ * samples to read. ++ */ ++ if (dev_priv->perf.oa.pollin) ++ events |= EPOLLIN; ++ ++ return events; ++} ++ ++/** ++ * i915_perf_poll - call poll_wait() with a suitable wait queue for stream ++ * @file: An i915 perf stream file ++ * @wait: poll() state table ++ * ++ * For handling userspace polling on an i915 perf stream, this ensures ++ * poll_wait() gets called with a wait queue that will be woken for new stream ++ * data. ++ * ++ * Note: Implementation deferred to i915_perf_poll_locked() ++ * ++ * Returns: any poll events that are ready without sleeping ++ */ ++static __poll_t i915_perf_poll(struct file *file, poll_table *wait) ++{ ++ struct i915_perf_stream *stream = file->private_data; ++ struct drm_i915_private *dev_priv = stream->dev_priv; ++ __poll_t ret; ++ ++ mutex_lock(&dev_priv->perf.lock); ++ ret = i915_perf_poll_locked(dev_priv, stream, file, wait); ++ mutex_unlock(&dev_priv->perf.lock); ++ ++ return ret; ++} ++ ++/** ++ * i915_perf_enable_locked - handle `I915_PERF_IOCTL_ENABLE` ioctl ++ * @stream: A disabled i915 perf stream ++ * ++ * [Re]enables the associated capture of data for this stream. ++ * ++ * If a stream was previously enabled then there's currently no intention ++ * to provide userspace any guarantee about the preservation of previously ++ * buffered data. ++ */ ++static void i915_perf_enable_locked(struct i915_perf_stream *stream) ++{ ++ if (stream->enabled) ++ return; ++ ++ /* Allow stream->ops->enable() to refer to this */ ++ stream->enabled = true; ++ ++ if (stream->ops->enable) ++ stream->ops->enable(stream); ++} ++ ++/** ++ * i915_perf_disable_locked - handle `I915_PERF_IOCTL_DISABLE` ioctl ++ * @stream: An enabled i915 perf stream ++ * ++ * Disables the associated capture of data for this stream. ++ * ++ * The intention is that disabling an re-enabling a stream will ideally be ++ * cheaper than destroying and re-opening a stream with the same configuration, ++ * though there are no formal guarantees about what state or buffered data ++ * must be retained between disabling and re-enabling a stream. ++ * ++ * Note: while a stream is disabled it's considered an error for userspace ++ * to attempt to read from the stream (-EIO). ++ */ ++static void i915_perf_disable_locked(struct i915_perf_stream *stream) ++{ ++ if (!stream->enabled) ++ return; ++ ++ /* Allow stream->ops->disable() to refer to this */ ++ stream->enabled = false; ++ ++ if (stream->ops->disable) ++ stream->ops->disable(stream); ++} ++ ++/** ++ * i915_perf_ioctl - support ioctl() usage with i915 perf stream FDs ++ * @stream: An i915 perf stream ++ * @cmd: the ioctl request ++ * @arg: the ioctl data ++ * ++ * Note: The &drm_i915_private->perf.lock mutex has been taken to serialize ++ * with any non-file-operation driver hooks. ++ * ++ * Returns: zero on success or a negative error code. Returns -EINVAL for ++ * an unknown ioctl request. ++ */ ++static long i915_perf_ioctl_locked(struct i915_perf_stream *stream, ++ unsigned int cmd, ++ unsigned long arg) ++{ ++ switch (cmd) { ++ case I915_PERF_IOCTL_ENABLE: ++ i915_perf_enable_locked(stream); ++ return 0; ++ case I915_PERF_IOCTL_DISABLE: ++ i915_perf_disable_locked(stream); ++ return 0; ++ } ++ ++ return -EINVAL; ++} ++ ++/** ++ * i915_perf_ioctl - support ioctl() usage with i915 perf stream FDs ++ * @file: An i915 perf stream file ++ * @cmd: the ioctl request ++ * @arg: the ioctl data ++ * ++ * Implementation deferred to i915_perf_ioctl_locked(). ++ * ++ * Returns: zero on success or a negative error code. Returns -EINVAL for ++ * an unknown ioctl request. ++ */ ++static long i915_perf_ioctl(struct file *file, ++ unsigned int cmd, ++ unsigned long arg) ++{ ++ struct i915_perf_stream *stream = file->private_data; ++ struct drm_i915_private *dev_priv = stream->dev_priv; ++ long ret; ++ ++ mutex_lock(&dev_priv->perf.lock); ++ ret = i915_perf_ioctl_locked(stream, cmd, arg); ++ mutex_unlock(&dev_priv->perf.lock); ++ ++ return ret; ++} ++ ++/** ++ * i915_perf_destroy_locked - destroy an i915 perf stream ++ * @stream: An i915 perf stream ++ * ++ * Frees all resources associated with the given i915 perf @stream, disabling ++ * any associated data capture in the process. ++ * ++ * Note: The &drm_i915_private->perf.lock mutex has been taken to serialize ++ * with any non-file-operation driver hooks. ++ */ ++static void i915_perf_destroy_locked(struct i915_perf_stream *stream) ++{ ++ if (stream->enabled) ++ i915_perf_disable_locked(stream); ++ ++ if (stream->ops->destroy) ++ stream->ops->destroy(stream); ++ ++ list_del(&stream->link); ++ ++ if (stream->ctx) ++ i915_gem_context_put(stream->ctx); ++ ++ kfree(stream); ++} ++ ++/** ++ * i915_perf_release - handles userspace close() of a stream file ++ * @inode: anonymous inode associated with file ++ * @file: An i915 perf stream file ++ * ++ * Cleans up any resources associated with an open i915 perf stream file. ++ * ++ * NB: close() can't really fail from the userspace point of view. ++ * ++ * Returns: zero on success or a negative error code. ++ */ ++static int i915_perf_release(struct inode *inode, struct file *file) ++{ ++ struct i915_perf_stream *stream = file->private_data; ++ struct drm_i915_private *dev_priv = stream->dev_priv; ++ ++ mutex_lock(&dev_priv->perf.lock); ++ i915_perf_destroy_locked(stream); ++ mutex_unlock(&dev_priv->perf.lock); ++ ++ return 0; ++} ++ ++ ++static const struct file_operations fops = { ++ .owner = THIS_MODULE, ++ .llseek = no_llseek, ++ .release = i915_perf_release, ++ .poll = i915_perf_poll, ++ .read = i915_perf_read, ++ .unlocked_ioctl = i915_perf_ioctl, ++ /* Our ioctl have no arguments, so it's safe to use the same function ++ * to handle 32bits compatibility. ++ */ ++ .compat_ioctl = i915_perf_ioctl, ++}; ++ ++ ++/** ++ * i915_perf_open_ioctl_locked - DRM ioctl() for userspace to open a stream FD ++ * @dev_priv: i915 device instance ++ * @param: The open parameters passed to 'DRM_I915_PERF_OPEN` ++ * @props: individually validated u64 property value pairs ++ * @file: drm file ++ * ++ * See i915_perf_ioctl_open() for interface details. ++ * ++ * Implements further stream config validation and stream initialization on ++ * behalf of i915_perf_open_ioctl() with the &drm_i915_private->perf.lock mutex ++ * taken to serialize with any non-file-operation driver hooks. ++ * ++ * Note: at this point the @props have only been validated in isolation and ++ * it's still necessary to validate that the combination of properties makes ++ * sense. ++ * ++ * In the case where userspace is interested in OA unit metrics then further ++ * config validation and stream initialization details will be handled by ++ * i915_oa_stream_init(). The code here should only validate config state that ++ * will be relevant to all stream types / backends. ++ * ++ * Returns: zero on success or a negative error code. ++ */ ++static int ++i915_perf_open_ioctl_locked(struct drm_i915_private *dev_priv, ++ struct drm_i915_perf_open_param *param, ++ struct perf_open_properties *props, ++ struct drm_file *file) ++{ ++ struct i915_gem_context *specific_ctx = NULL; ++ struct i915_perf_stream *stream = NULL; ++ unsigned long f_flags = 0; ++ bool privileged_op = true; ++ int stream_fd; ++ int ret; ++ ++ if (props->single_context) { ++ u32 ctx_handle = props->ctx_handle; ++ struct drm_i915_file_private *file_priv = file->driver_priv; ++ ++ specific_ctx = i915_gem_context_lookup(file_priv, ctx_handle); ++ if (!specific_ctx) { ++ DRM_DEBUG("Failed to look up context with ID %u for opening perf stream\n", ++ ctx_handle); ++ ret = -ENOENT; ++ goto err; ++ } ++ } ++ ++ /* ++ * On Haswell the OA unit supports clock gating off for a specific ++ * context and in this mode there's no visibility of metrics for the ++ * rest of the system, which we consider acceptable for a ++ * non-privileged client. ++ * ++ * For Gen8+ the OA unit no longer supports clock gating off for a ++ * specific context and the kernel can't securely stop the counters ++ * from updating as system-wide / global values. Even though we can ++ * filter reports based on the included context ID we can't block ++ * clients from seeing the raw / global counter values via ++ * MI_REPORT_PERF_COUNT commands and so consider it a privileged op to ++ * enable the OA unit by default. ++ */ ++ if (IS_HASWELL(dev_priv) && specific_ctx) ++ privileged_op = false; ++ ++ /* Similar to perf's kernel.perf_paranoid_cpu sysctl option ++ * we check a dev.i915.perf_stream_paranoid sysctl option ++ * to determine if it's ok to access system wide OA counters ++ * without CAP_SYS_ADMIN privileges. ++ */ ++ if (privileged_op && ++ i915_perf_stream_paranoid && !capable(CAP_SYS_ADMIN)) { ++ DRM_DEBUG("Insufficient privileges to open system-wide i915 perf stream\n"); ++ ret = -EACCES; ++ goto err_ctx; ++ } ++ ++ stream = kzalloc(sizeof(*stream), GFP_KERNEL); ++ if (!stream) { ++ ret = -ENOMEM; ++ goto err_ctx; ++ } ++ ++ stream->dev_priv = dev_priv; ++ stream->ctx = specific_ctx; ++ ++ ret = i915_oa_stream_init(stream, param, props); ++ if (ret) ++ goto err_alloc; ++ ++ /* we avoid simply assigning stream->sample_flags = props->sample_flags ++ * to have _stream_init check the combination of sample flags more ++ * thoroughly, but still this is the expected result at this point. ++ */ ++ if (WARN_ON(stream->sample_flags != props->sample_flags)) { ++ ret = -ENODEV; ++ goto err_flags; ++ } ++ ++ list_add(&stream->link, &dev_priv->perf.streams); ++ ++ if (param->flags & I915_PERF_FLAG_FD_CLOEXEC) ++ f_flags |= O_CLOEXEC; ++ if (param->flags & I915_PERF_FLAG_FD_NONBLOCK) ++ f_flags |= O_NONBLOCK; ++ ++ stream_fd = anon_inode_getfd("[i915_perf]", &fops, stream, f_flags); ++ if (stream_fd < 0) { ++ ret = stream_fd; ++ goto err_open; ++ } ++ ++ if (!(param->flags & I915_PERF_FLAG_DISABLED)) ++ i915_perf_enable_locked(stream); ++ ++ return stream_fd; ++ ++err_open: ++ list_del(&stream->link); ++err_flags: ++ if (stream->ops->destroy) ++ stream->ops->destroy(stream); ++err_alloc: ++ kfree(stream); ++err_ctx: ++ if (specific_ctx) ++ i915_gem_context_put(specific_ctx); ++err: ++ return ret; ++} ++ ++static u64 oa_exponent_to_ns(struct drm_i915_private *dev_priv, int exponent) ++{ ++ return div64_u64(1000000000ULL * (2ULL << exponent), ++ 1000ULL * RUNTIME_INFO(dev_priv)->cs_timestamp_frequency_khz); ++} ++ ++/** ++ * read_properties_unlocked - validate + copy userspace stream open properties ++ * @dev_priv: i915 device instance ++ * @uprops: The array of u64 key value pairs given by userspace ++ * @n_props: The number of key value pairs expected in @uprops ++ * @props: The stream configuration built up while validating properties ++ * ++ * Note this function only validates properties in isolation it doesn't ++ * validate that the combination of properties makes sense or that all ++ * properties necessary for a particular kind of stream have been set. ++ * ++ * Note that there currently aren't any ordering requirements for properties so ++ * we shouldn't validate or assume anything about ordering here. This doesn't ++ * rule out defining new properties with ordering requirements in the future. ++ */ ++static int read_properties_unlocked(struct drm_i915_private *dev_priv, ++ u64 __user *uprops, ++ u32 n_props, ++ struct perf_open_properties *props) ++{ ++ u64 __user *uprop = uprops; ++ u32 i; ++ ++ memset(props, 0, sizeof(struct perf_open_properties)); ++ ++ if (!n_props) { ++ DRM_DEBUG("No i915 perf properties given\n"); ++ return -EINVAL; ++ } ++ ++ /* Considering that ID = 0 is reserved and assuming that we don't ++ * (currently) expect any configurations to ever specify duplicate ++ * values for a particular property ID then the last _PROP_MAX value is ++ * one greater than the maximum number of properties we expect to get ++ * from userspace. ++ */ ++ if (n_props >= DRM_I915_PERF_PROP_MAX) { ++ DRM_DEBUG("More i915 perf properties specified than exist\n"); ++ return -EINVAL; ++ } ++ ++ for (i = 0; i < n_props; i++) { ++ u64 oa_period, oa_freq_hz; ++ u64 id, value; ++ int ret; ++ ++ ret = get_user(id, uprop); ++ if (ret) ++ return ret; ++ ++ ret = get_user(value, uprop + 1); ++ if (ret) ++ return ret; ++ ++ if (id == 0 || id >= DRM_I915_PERF_PROP_MAX) { ++ DRM_DEBUG("Unknown i915 perf property ID\n"); ++ return -EINVAL; ++ } ++ ++ switch ((enum drm_i915_perf_property_id)id) { ++ case DRM_I915_PERF_PROP_CTX_HANDLE: ++ props->single_context = 1; ++ props->ctx_handle = value; ++ break; ++ case DRM_I915_PERF_PROP_SAMPLE_OA: ++ if (value) ++ props->sample_flags |= SAMPLE_OA_REPORT; ++ break; ++ case DRM_I915_PERF_PROP_OA_METRICS_SET: ++ if (value == 0) { ++ DRM_DEBUG("Unknown OA metric set ID\n"); ++ return -EINVAL; ++ } ++ props->metrics_set = value; ++ break; ++ case DRM_I915_PERF_PROP_OA_FORMAT: ++ if (value == 0 || value >= I915_OA_FORMAT_MAX) { ++ DRM_DEBUG("Out-of-range OA report format %llu\n", ++ value); ++ return -EINVAL; ++ } ++ if (!dev_priv->perf.oa.oa_formats[value].size) { ++ DRM_DEBUG("Unsupported OA report format %llu\n", ++ value); ++ return -EINVAL; ++ } ++ props->oa_format = value; ++ break; ++ case DRM_I915_PERF_PROP_OA_EXPONENT: ++ if (value > OA_EXPONENT_MAX) { ++ DRM_DEBUG("OA timer exponent too high (> %u)\n", ++ OA_EXPONENT_MAX); ++ return -EINVAL; ++ } ++ ++ /* Theoretically we can program the OA unit to sample ++ * e.g. every 160ns for HSW, 167ns for BDW/SKL or 104ns ++ * for BXT. We don't allow such high sampling ++ * frequencies by default unless root. ++ */ ++ ++ BUILD_BUG_ON(sizeof(oa_period) != 8); ++ oa_period = oa_exponent_to_ns(dev_priv, value); ++ ++ /* This check is primarily to ensure that oa_period <= ++ * UINT32_MAX (before passing to do_div which only ++ * accepts a u32 denominator), but we can also skip ++ * checking anything < 1Hz which implicitly can't be ++ * limited via an integer oa_max_sample_rate. ++ */ ++ if (oa_period <= NSEC_PER_SEC) { ++ u64 tmp = NSEC_PER_SEC; ++ do_div(tmp, oa_period); ++ oa_freq_hz = tmp; ++ } else ++ oa_freq_hz = 0; ++ ++ if (oa_freq_hz > i915_oa_max_sample_rate && ++ !capable(CAP_SYS_ADMIN)) { ++ DRM_DEBUG("OA exponent would exceed the max sampling frequency (sysctl dev.i915.oa_max_sample_rate) %uHz without root privileges\n", ++ i915_oa_max_sample_rate); ++ return -EACCES; ++ } ++ ++ props->oa_periodic = true; ++ props->oa_period_exponent = value; ++ break; ++ case DRM_I915_PERF_PROP_MAX: ++ MISSING_CASE(id); ++ return -EINVAL; ++ } ++ ++ uprop += 2; ++ } ++ ++ return 0; ++} ++ ++/** ++ * i915_perf_open_ioctl - DRM ioctl() for userspace to open a stream FD ++ * @dev: drm device ++ * @data: ioctl data copied from userspace (unvalidated) ++ * @file: drm file ++ * ++ * Validates the stream open parameters given by userspace including flags ++ * and an array of u64 key, value pair properties. ++ * ++ * Very little is assumed up front about the nature of the stream being ++ * opened (for instance we don't assume it's for periodic OA unit metrics). An ++ * i915-perf stream is expected to be a suitable interface for other forms of ++ * buffered data written by the GPU besides periodic OA metrics. ++ * ++ * Note we copy the properties from userspace outside of the i915 perf ++ * mutex to avoid an awkward lockdep with mmap_sem. ++ * ++ * Most of the implementation details are handled by ++ * i915_perf_open_ioctl_locked() after taking the &drm_i915_private->perf.lock ++ * mutex for serializing with any non-file-operation driver hooks. ++ * ++ * Return: A newly opened i915 Perf stream file descriptor or negative ++ * error code on failure. ++ */ ++int i915_perf_open_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_i915_private *dev_priv = dev->dev_private; ++ struct drm_i915_perf_open_param *param = data; ++ struct perf_open_properties props; ++ u32 known_open_flags; ++ int ret; ++ ++ if (!dev_priv->perf.initialized) { ++ DRM_DEBUG("i915 perf interface not available for this system\n"); ++ return -ENOTSUPP; ++ } ++ ++ known_open_flags = I915_PERF_FLAG_FD_CLOEXEC | ++ I915_PERF_FLAG_FD_NONBLOCK | ++ I915_PERF_FLAG_DISABLED; ++ if (param->flags & ~known_open_flags) { ++ DRM_DEBUG("Unknown drm_i915_perf_open_param flag\n"); ++ return -EINVAL; ++ } ++ ++ ret = read_properties_unlocked(dev_priv, ++ u64_to_user_ptr(param->properties_ptr), ++ param->num_properties, ++ &props); ++ if (ret) ++ return ret; ++ ++ mutex_lock(&dev_priv->perf.lock); ++ ret = i915_perf_open_ioctl_locked(dev_priv, param, &props, file); ++ mutex_unlock(&dev_priv->perf.lock); ++ ++ return ret; ++} ++ ++/** ++ * i915_perf_register - exposes i915-perf to userspace ++ * @dev_priv: i915 device instance ++ * ++ * In particular OA metric sets are advertised under a sysfs metrics/ ++ * directory allowing userspace to enumerate valid IDs that can be ++ * used to open an i915-perf stream. ++ */ ++void i915_perf_register(struct drm_i915_private *dev_priv) ++{ ++ int ret; ++ ++ if (!dev_priv->perf.initialized) ++ return; ++ ++ /* To be sure we're synchronized with an attempted ++ * i915_perf_open_ioctl(); considering that we register after ++ * being exposed to userspace. ++ */ ++ mutex_lock(&dev_priv->perf.lock); ++ ++ dev_priv->perf.metrics_kobj = ++ kobject_create_and_add("metrics", ++ &dev_priv->drm.primary->kdev->kobj); ++ if (!dev_priv->perf.metrics_kobj) ++ goto exit; ++ ++ sysfs_attr_init(&dev_priv->perf.oa.test_config.sysfs_metric_id.attr); ++ ++ if (INTEL_GEN(dev_priv) >= 11) { ++ i915_perf_load_test_config_icl(dev_priv); ++ } else if (IS_CANNONLAKE(dev_priv)) { ++ i915_perf_load_test_config_cnl(dev_priv); ++ } else if (IS_COFFEELAKE(dev_priv)) { ++ if (IS_CFL_GT2(dev_priv)) ++ i915_perf_load_test_config_cflgt2(dev_priv); ++ if (IS_CFL_GT3(dev_priv)) ++ i915_perf_load_test_config_cflgt3(dev_priv); ++ } else if (IS_GEMINILAKE(dev_priv)) { ++ i915_perf_load_test_config_glk(dev_priv); ++ } else if (IS_KABYLAKE(dev_priv)) { ++ if (IS_KBL_GT2(dev_priv)) ++ i915_perf_load_test_config_kblgt2(dev_priv); ++ else if (IS_KBL_GT3(dev_priv)) ++ i915_perf_load_test_config_kblgt3(dev_priv); ++ } else if (IS_BROXTON(dev_priv)) { ++ i915_perf_load_test_config_bxt(dev_priv); ++ } else if (IS_SKYLAKE(dev_priv)) { ++ if (IS_SKL_GT2(dev_priv)) ++ i915_perf_load_test_config_sklgt2(dev_priv); ++ else if (IS_SKL_GT3(dev_priv)) ++ i915_perf_load_test_config_sklgt3(dev_priv); ++ else if (IS_SKL_GT4(dev_priv)) ++ i915_perf_load_test_config_sklgt4(dev_priv); ++ } else if (IS_CHERRYVIEW(dev_priv)) { ++ i915_perf_load_test_config_chv(dev_priv); ++ } else if (IS_BROADWELL(dev_priv)) { ++ i915_perf_load_test_config_bdw(dev_priv); ++ } else if (IS_HASWELL(dev_priv)) { ++ i915_perf_load_test_config_hsw(dev_priv); ++} ++ ++ if (dev_priv->perf.oa.test_config.id == 0) ++ goto sysfs_error; ++ ++ ret = sysfs_create_group(dev_priv->perf.metrics_kobj, ++ &dev_priv->perf.oa.test_config.sysfs_metric); ++ if (ret) ++ goto sysfs_error; ++ ++ atomic_set(&dev_priv->perf.oa.test_config.ref_count, 1); ++ ++ goto exit; ++ ++sysfs_error: ++ kobject_put(dev_priv->perf.metrics_kobj); ++ dev_priv->perf.metrics_kobj = NULL; ++ ++exit: ++ mutex_unlock(&dev_priv->perf.lock); ++} ++ ++/** ++ * i915_perf_unregister - hide i915-perf from userspace ++ * @dev_priv: i915 device instance ++ * ++ * i915-perf state cleanup is split up into an 'unregister' and ++ * 'deinit' phase where the interface is first hidden from ++ * userspace by i915_perf_unregister() before cleaning up ++ * remaining state in i915_perf_fini(). ++ */ ++void i915_perf_unregister(struct drm_i915_private *dev_priv) ++{ ++ if (!dev_priv->perf.metrics_kobj) ++ return; ++ ++ sysfs_remove_group(dev_priv->perf.metrics_kobj, ++ &dev_priv->perf.oa.test_config.sysfs_metric); ++ ++ kobject_put(dev_priv->perf.metrics_kobj); ++ dev_priv->perf.metrics_kobj = NULL; ++} ++ ++static bool gen8_is_valid_flex_addr(struct drm_i915_private *dev_priv, u32 addr) ++{ ++ static const i915_reg_t flex_eu_regs[] = { ++ EU_PERF_CNTL0, ++ EU_PERF_CNTL1, ++ EU_PERF_CNTL2, ++ EU_PERF_CNTL3, ++ EU_PERF_CNTL4, ++ EU_PERF_CNTL5, ++ EU_PERF_CNTL6, ++ }; ++ int i; ++ ++ for (i = 0; i < ARRAY_SIZE(flex_eu_regs); i++) { ++ if (i915_mmio_reg_offset(flex_eu_regs[i]) == addr) ++ return true; ++ } ++ return false; ++} ++ ++static bool gen7_is_valid_b_counter_addr(struct drm_i915_private *dev_priv, u32 addr) ++{ ++ return (addr >= i915_mmio_reg_offset(OASTARTTRIG1) && ++ addr <= i915_mmio_reg_offset(OASTARTTRIG8)) || ++ (addr >= i915_mmio_reg_offset(OAREPORTTRIG1) && ++ addr <= i915_mmio_reg_offset(OAREPORTTRIG8)) || ++ (addr >= i915_mmio_reg_offset(OACEC0_0) && ++ addr <= i915_mmio_reg_offset(OACEC7_1)); ++} ++ ++static bool gen7_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr) ++{ ++ return addr == i915_mmio_reg_offset(HALF_SLICE_CHICKEN2) || ++ (addr >= i915_mmio_reg_offset(MICRO_BP0_0) && ++ addr <= i915_mmio_reg_offset(NOA_WRITE)) || ++ (addr >= i915_mmio_reg_offset(OA_PERFCNT1_LO) && ++ addr <= i915_mmio_reg_offset(OA_PERFCNT2_HI)) || ++ (addr >= i915_mmio_reg_offset(OA_PERFMATRIX_LO) && ++ addr <= i915_mmio_reg_offset(OA_PERFMATRIX_HI)); ++} ++ ++static bool gen8_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr) ++{ ++ return gen7_is_valid_mux_addr(dev_priv, addr) || ++ addr == i915_mmio_reg_offset(WAIT_FOR_RC6_EXIT) || ++ (addr >= i915_mmio_reg_offset(RPM_CONFIG0) && ++ addr <= i915_mmio_reg_offset(NOA_CONFIG(8))); ++} ++ ++static bool gen10_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr) ++{ ++ return gen8_is_valid_mux_addr(dev_priv, addr) || ++ addr == i915_mmio_reg_offset(GEN10_NOA_WRITE_HIGH) || ++ (addr >= i915_mmio_reg_offset(OA_PERFCNT3_LO) && ++ addr <= i915_mmio_reg_offset(OA_PERFCNT4_HI)); ++} ++ ++static bool hsw_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr) ++{ ++ return gen7_is_valid_mux_addr(dev_priv, addr) || ++ (addr >= 0x25100 && addr <= 0x2FF90) || ++ (addr >= i915_mmio_reg_offset(HSW_MBVID2_NOA0) && ++ addr <= i915_mmio_reg_offset(HSW_MBVID2_NOA9)) || ++ addr == i915_mmio_reg_offset(HSW_MBVID2_MISR0); ++} ++ ++static bool chv_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr) ++{ ++ return gen7_is_valid_mux_addr(dev_priv, addr) || ++ (addr >= 0x182300 && addr <= 0x1823A4); ++} ++ ++static u32 mask_reg_value(u32 reg, u32 val) ++{ ++ /* HALF_SLICE_CHICKEN2 is programmed with a the ++ * WaDisableSTUnitPowerOptimization workaround. Make sure the value ++ * programmed by userspace doesn't change this. ++ */ ++ if (i915_mmio_reg_offset(HALF_SLICE_CHICKEN2) == reg) ++ val = val & ~_MASKED_BIT_ENABLE(GEN8_ST_PO_DISABLE); ++ ++ /* WAIT_FOR_RC6_EXIT has only one bit fullfilling the function ++ * indicated by its name and a bunch of selection fields used by OA ++ * configs. ++ */ ++ if (i915_mmio_reg_offset(WAIT_FOR_RC6_EXIT) == reg) ++ val = val & ~_MASKED_BIT_ENABLE(HSW_WAIT_FOR_RC6_EXIT_ENABLE); ++ ++ return val; ++} ++ ++static struct i915_oa_reg *alloc_oa_regs(struct drm_i915_private *dev_priv, ++ bool (*is_valid)(struct drm_i915_private *dev_priv, u32 addr), ++ u32 __user *regs, ++ u32 n_regs) ++{ ++ struct i915_oa_reg *oa_regs; ++ int err; ++ u32 i; ++ ++ if (!n_regs) ++ return NULL; ++ ++ if (!access_ok(regs, n_regs * sizeof(u32) * 2)) ++ return ERR_PTR(-EFAULT); ++ ++ /* No is_valid function means we're not allowing any register to be programmed. */ ++ GEM_BUG_ON(!is_valid); ++ if (!is_valid) ++ return ERR_PTR(-EINVAL); ++ ++ oa_regs = kmalloc_array(n_regs, sizeof(*oa_regs), GFP_KERNEL); ++ if (!oa_regs) ++ return ERR_PTR(-ENOMEM); ++ ++ for (i = 0; i < n_regs; i++) { ++ u32 addr, value; ++ ++ err = get_user(addr, regs); ++ if (err) ++ goto addr_err; ++ ++ if (!is_valid(dev_priv, addr)) { ++ DRM_DEBUG("Invalid oa_reg address: %X\n", addr); ++ err = -EINVAL; ++ goto addr_err; ++ } ++ ++ err = get_user(value, regs + 1); ++ if (err) ++ goto addr_err; ++ ++ oa_regs[i].addr = _MMIO(addr); ++ oa_regs[i].value = mask_reg_value(addr, value); ++ ++ regs += 2; ++ } ++ ++ return oa_regs; ++ ++addr_err: ++ kfree(oa_regs); ++ return ERR_PTR(err); ++} ++ ++static ssize_t show_dynamic_id(struct device *dev, ++ struct device_attribute *attr, ++ char *buf) ++{ ++ struct i915_oa_config *oa_config = ++ container_of(attr, typeof(*oa_config), sysfs_metric_id); ++ ++ return sprintf(buf, "%d\n", oa_config->id); ++} ++ ++static int create_dynamic_oa_sysfs_entry(struct drm_i915_private *dev_priv, ++ struct i915_oa_config *oa_config) ++{ ++ sysfs_attr_init(&oa_config->sysfs_metric_id.attr); ++ oa_config->sysfs_metric_id.attr.name = "id"; ++ oa_config->sysfs_metric_id.attr.mode = S_IRUGO; ++ oa_config->sysfs_metric_id.show = show_dynamic_id; ++ oa_config->sysfs_metric_id.store = NULL; ++ ++ oa_config->attrs[0] = &oa_config->sysfs_metric_id.attr; ++ oa_config->attrs[1] = NULL; ++ ++ oa_config->sysfs_metric.name = oa_config->uuid; ++ oa_config->sysfs_metric.attrs = oa_config->attrs; ++ ++ return sysfs_create_group(dev_priv->perf.metrics_kobj, ++ &oa_config->sysfs_metric); ++} ++ ++/** ++ * i915_perf_add_config_ioctl - DRM ioctl() for userspace to add a new OA config ++ * @dev: drm device ++ * @data: ioctl data (pointer to struct drm_i915_perf_oa_config) copied from ++ * userspace (unvalidated) ++ * @file: drm file ++ * ++ * Validates the submitted OA register to be saved into a new OA config that ++ * can then be used for programming the OA unit and its NOA network. ++ * ++ * Returns: A new allocated config number to be used with the perf open ioctl ++ * or a negative error code on failure. ++ */ ++int i915_perf_add_config_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_i915_private *dev_priv = dev->dev_private; ++ struct drm_i915_perf_oa_config *args = data; ++ struct i915_oa_config *oa_config, *tmp; ++ int err, id; ++ ++ if (!dev_priv->perf.initialized) { ++ DRM_DEBUG("i915 perf interface not available for this system\n"); ++ return -ENOTSUPP; ++ } ++ ++ if (!dev_priv->perf.metrics_kobj) { ++ DRM_DEBUG("OA metrics weren't advertised via sysfs\n"); ++ return -EINVAL; ++ } ++ ++ if (i915_perf_stream_paranoid && !capable(CAP_SYS_ADMIN)) { ++ DRM_DEBUG("Insufficient privileges to add i915 OA config\n"); ++ return -EACCES; ++ } ++ ++ if ((!args->mux_regs_ptr || !args->n_mux_regs) && ++ (!args->boolean_regs_ptr || !args->n_boolean_regs) && ++ (!args->flex_regs_ptr || !args->n_flex_regs)) { ++ DRM_DEBUG("No OA registers given\n"); ++ return -EINVAL; ++ } ++ ++ oa_config = kzalloc(sizeof(*oa_config), GFP_KERNEL); ++ if (!oa_config) { ++ DRM_DEBUG("Failed to allocate memory for the OA config\n"); ++ return -ENOMEM; ++ } ++ ++ atomic_set(&oa_config->ref_count, 1); ++ ++ if (!uuid_is_valid(args->uuid)) { ++ DRM_DEBUG("Invalid uuid format for OA config\n"); ++ err = -EINVAL; ++ goto reg_err; ++ } ++ ++ /* Last character in oa_config->uuid will be 0 because oa_config is ++ * kzalloc. ++ */ ++ memcpy(oa_config->uuid, args->uuid, sizeof(args->uuid)); ++ ++ oa_config->mux_regs_len = args->n_mux_regs; ++ oa_config->mux_regs = ++ alloc_oa_regs(dev_priv, ++ dev_priv->perf.oa.ops.is_valid_mux_reg, ++ u64_to_user_ptr(args->mux_regs_ptr), ++ args->n_mux_regs); ++ ++ if (IS_ERR(oa_config->mux_regs)) { ++ DRM_DEBUG("Failed to create OA config for mux_regs\n"); ++ err = PTR_ERR(oa_config->mux_regs); ++ goto reg_err; ++ } ++ ++ oa_config->b_counter_regs_len = args->n_boolean_regs; ++ oa_config->b_counter_regs = ++ alloc_oa_regs(dev_priv, ++ dev_priv->perf.oa.ops.is_valid_b_counter_reg, ++ u64_to_user_ptr(args->boolean_regs_ptr), ++ args->n_boolean_regs); ++ ++ if (IS_ERR(oa_config->b_counter_regs)) { ++ DRM_DEBUG("Failed to create OA config for b_counter_regs\n"); ++ err = PTR_ERR(oa_config->b_counter_regs); ++ goto reg_err; ++ } ++ ++ if (INTEL_GEN(dev_priv) < 8) { ++ if (args->n_flex_regs != 0) { ++ err = -EINVAL; ++ goto reg_err; ++ } ++ } else { ++ oa_config->flex_regs_len = args->n_flex_regs; ++ oa_config->flex_regs = ++ alloc_oa_regs(dev_priv, ++ dev_priv->perf.oa.ops.is_valid_flex_reg, ++ u64_to_user_ptr(args->flex_regs_ptr), ++ args->n_flex_regs); ++ ++ if (IS_ERR(oa_config->flex_regs)) { ++ DRM_DEBUG("Failed to create OA config for flex_regs\n"); ++ err = PTR_ERR(oa_config->flex_regs); ++ goto reg_err; ++ } ++ } ++ ++ err = mutex_lock_interruptible(&dev_priv->perf.metrics_lock); ++ if (err) ++ goto reg_err; ++ ++ /* We shouldn't have too many configs, so this iteration shouldn't be ++ * too costly. ++ */ ++ idr_for_each_entry(&dev_priv->perf.metrics_idr, tmp, id) { ++ if (!strcmp(tmp->uuid, oa_config->uuid)) { ++ DRM_DEBUG("OA config already exists with this uuid\n"); ++ err = -EADDRINUSE; ++ goto sysfs_err; ++ } ++ } ++ ++ err = create_dynamic_oa_sysfs_entry(dev_priv, oa_config); ++ if (err) { ++ DRM_DEBUG("Failed to create sysfs entry for OA config\n"); ++ goto sysfs_err; ++ } ++ ++ /* Config id 0 is invalid, id 1 for kernel stored test config. */ ++ oa_config->id = idr_alloc(&dev_priv->perf.metrics_idr, ++ oa_config, 2, ++ 0, GFP_KERNEL); ++ if (oa_config->id < 0) { ++ DRM_DEBUG("Failed to create sysfs entry for OA config\n"); ++ err = oa_config->id; ++ goto sysfs_err; ++ } ++ ++ mutex_unlock(&dev_priv->perf.metrics_lock); ++ ++ DRM_DEBUG("Added config %s id=%i\n", oa_config->uuid, oa_config->id); ++ ++ return oa_config->id; ++ ++sysfs_err: ++ mutex_unlock(&dev_priv->perf.metrics_lock); ++reg_err: ++ put_oa_config(dev_priv, oa_config); ++ DRM_DEBUG("Failed to add new OA config\n"); ++ return err; ++} ++ ++/** ++ * i915_perf_remove_config_ioctl - DRM ioctl() for userspace to remove an OA config ++ * @dev: drm device ++ * @data: ioctl data (pointer to u64 integer) copied from userspace ++ * @file: drm file ++ * ++ * Configs can be removed while being used, the will stop appearing in sysfs ++ * and their content will be freed when the stream using the config is closed. ++ * ++ * Returns: 0 on success or a negative error code on failure. ++ */ ++int i915_perf_remove_config_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_i915_private *dev_priv = dev->dev_private; ++ u64 *arg = data; ++ struct i915_oa_config *oa_config; ++ int ret; ++ ++ if (!dev_priv->perf.initialized) { ++ DRM_DEBUG("i915 perf interface not available for this system\n"); ++ return -ENOTSUPP; ++ } ++ ++ if (i915_perf_stream_paranoid && !capable(CAP_SYS_ADMIN)) { ++ DRM_DEBUG("Insufficient privileges to remove i915 OA config\n"); ++ return -EACCES; ++ } ++ ++ ret = mutex_lock_interruptible(&dev_priv->perf.metrics_lock); ++ if (ret) ++ goto lock_err; ++ ++ oa_config = idr_find(&dev_priv->perf.metrics_idr, *arg); ++ if (!oa_config) { ++ DRM_DEBUG("Failed to remove unknown OA config\n"); ++ ret = -ENOENT; ++ goto config_err; ++ } ++ ++ GEM_BUG_ON(*arg != oa_config->id); ++ ++ sysfs_remove_group(dev_priv->perf.metrics_kobj, ++ &oa_config->sysfs_metric); ++ ++ idr_remove(&dev_priv->perf.metrics_idr, *arg); ++ ++ DRM_DEBUG("Removed config %s id=%i\n", oa_config->uuid, oa_config->id); ++ ++ put_oa_config(dev_priv, oa_config); ++ ++config_err: ++ mutex_unlock(&dev_priv->perf.metrics_lock); ++lock_err: ++ return ret; ++} ++ ++static struct ctl_table oa_table[] = { ++ { ++ .procname = "perf_stream_paranoid", ++ .data = &i915_perf_stream_paranoid, ++ .maxlen = sizeof(i915_perf_stream_paranoid), ++ .mode = 0644, ++ .proc_handler = proc_dointvec_minmax, ++ .extra1 = &zero, ++ .extra2 = &one, ++ }, ++ { ++ .procname = "oa_max_sample_rate", ++ .data = &i915_oa_max_sample_rate, ++ .maxlen = sizeof(i915_oa_max_sample_rate), ++ .mode = 0644, ++ .proc_handler = proc_dointvec_minmax, ++ .extra1 = &zero, ++ .extra2 = &oa_sample_rate_hard_limit, ++ }, ++ {} ++}; ++ ++static struct ctl_table i915_root[] = { ++ { ++ .procname = "i915", ++ .maxlen = 0, ++ .mode = 0555, ++ .child = oa_table, ++ }, ++ {} ++}; ++ ++static struct ctl_table dev_root[] = { ++ { ++ .procname = "dev", ++ .maxlen = 0, ++ .mode = 0555, ++ .child = i915_root, ++ }, ++ {} ++}; ++ ++/** ++ * i915_perf_init - initialize i915-perf state on module load ++ * @dev_priv: i915 device instance ++ * ++ * Initializes i915-perf state without exposing anything to userspace. ++ * ++ * Note: i915-perf initialization is split into an 'init' and 'register' ++ * phase with the i915_perf_register() exposing state to userspace. ++ */ ++void i915_perf_init(struct drm_i915_private *dev_priv) ++{ ++ if (IS_HASWELL(dev_priv)) { ++ dev_priv->perf.oa.ops.is_valid_b_counter_reg = ++ gen7_is_valid_b_counter_addr; ++ dev_priv->perf.oa.ops.is_valid_mux_reg = ++ hsw_is_valid_mux_addr; ++ dev_priv->perf.oa.ops.is_valid_flex_reg = NULL; ++ dev_priv->perf.oa.ops.enable_metric_set = hsw_enable_metric_set; ++ dev_priv->perf.oa.ops.disable_metric_set = hsw_disable_metric_set; ++ dev_priv->perf.oa.ops.oa_enable = gen7_oa_enable; ++ dev_priv->perf.oa.ops.oa_disable = gen7_oa_disable; ++ dev_priv->perf.oa.ops.read = gen7_oa_read; ++ dev_priv->perf.oa.ops.oa_hw_tail_read = ++ gen7_oa_hw_tail_read; ++ ++ dev_priv->perf.oa.oa_formats = hsw_oa_formats; ++ } else if (HAS_LOGICAL_RING_CONTEXTS(dev_priv)) { ++ /* Note: that although we could theoretically also support the ++ * legacy ringbuffer mode on BDW (and earlier iterations of ++ * this driver, before upstreaming did this) it didn't seem ++ * worth the complexity to maintain now that BDW+ enable ++ * execlist mode by default. ++ */ ++ dev_priv->perf.oa.oa_formats = gen8_plus_oa_formats; ++ ++ dev_priv->perf.oa.ops.oa_enable = gen8_oa_enable; ++ dev_priv->perf.oa.ops.oa_disable = gen8_oa_disable; ++ dev_priv->perf.oa.ops.read = gen8_oa_read; ++ dev_priv->perf.oa.ops.oa_hw_tail_read = gen8_oa_hw_tail_read; ++ ++ if (IS_GEN_RANGE(dev_priv, 8, 9)) { ++ dev_priv->perf.oa.ops.is_valid_b_counter_reg = ++ gen7_is_valid_b_counter_addr; ++ dev_priv->perf.oa.ops.is_valid_mux_reg = ++ gen8_is_valid_mux_addr; ++ dev_priv->perf.oa.ops.is_valid_flex_reg = ++ gen8_is_valid_flex_addr; ++ ++ if (IS_CHERRYVIEW(dev_priv)) { ++ dev_priv->perf.oa.ops.is_valid_mux_reg = ++ chv_is_valid_mux_addr; ++ } ++ ++ dev_priv->perf.oa.ops.enable_metric_set = gen8_enable_metric_set; ++ dev_priv->perf.oa.ops.disable_metric_set = gen8_disable_metric_set; ++ ++ if (IS_GEN(dev_priv, 8)) { ++ dev_priv->perf.oa.ctx_oactxctrl_offset = 0x120; ++ dev_priv->perf.oa.ctx_flexeu0_offset = 0x2ce; ++ ++ dev_priv->perf.oa.gen8_valid_ctx_bit = (1<<25); ++ } else { ++ dev_priv->perf.oa.ctx_oactxctrl_offset = 0x128; ++ dev_priv->perf.oa.ctx_flexeu0_offset = 0x3de; ++ ++ dev_priv->perf.oa.gen8_valid_ctx_bit = (1<<16); ++ } ++ } else if (IS_GEN_RANGE(dev_priv, 10, 11)) { ++ dev_priv->perf.oa.ops.is_valid_b_counter_reg = ++ gen7_is_valid_b_counter_addr; ++ dev_priv->perf.oa.ops.is_valid_mux_reg = ++ gen10_is_valid_mux_addr; ++ dev_priv->perf.oa.ops.is_valid_flex_reg = ++ gen8_is_valid_flex_addr; ++ ++ dev_priv->perf.oa.ops.enable_metric_set = gen8_enable_metric_set; ++ dev_priv->perf.oa.ops.disable_metric_set = gen10_disable_metric_set; ++ ++ if (IS_GEN(dev_priv, 10)) { ++ dev_priv->perf.oa.ctx_oactxctrl_offset = 0x128; ++ dev_priv->perf.oa.ctx_flexeu0_offset = 0x3de; ++ } else { ++ dev_priv->perf.oa.ctx_oactxctrl_offset = 0x124; ++ dev_priv->perf.oa.ctx_flexeu0_offset = 0x78e; ++ } ++ dev_priv->perf.oa.gen8_valid_ctx_bit = (1<<16); ++ } ++ } ++ ++ if (dev_priv->perf.oa.ops.enable_metric_set) { ++ hrtimer_init(&dev_priv->perf.oa.poll_check_timer, ++ CLOCK_MONOTONIC, HRTIMER_MODE_REL); ++ dev_priv->perf.oa.poll_check_timer.function = oa_poll_check_timer_cb; ++ init_waitqueue_head(&dev_priv->perf.oa.poll_wq); ++ ++ INIT_LIST_HEAD(&dev_priv->perf.streams); ++ mutex_init(&dev_priv->perf.lock); ++ spin_lock_init(&dev_priv->perf.oa.oa_buffer.ptr_lock); ++ ++ oa_sample_rate_hard_limit = 1000 * ++ (RUNTIME_INFO(dev_priv)->cs_timestamp_frequency_khz / 2); ++ dev_priv->perf.sysctl_header = register_sysctl_table(dev_root); ++ ++ mutex_init(&dev_priv->perf.metrics_lock); ++ idr_init(&dev_priv->perf.metrics_idr); ++ ++ dev_priv->perf.initialized = true; ++ } ++} ++ ++static int destroy_config(int id, void *p, void *data) ++{ ++ struct drm_i915_private *dev_priv = data; ++ struct i915_oa_config *oa_config = p; ++ ++ put_oa_config(dev_priv, oa_config); ++ ++ return 0; ++} ++ ++/** ++ * i915_perf_fini - Counter part to i915_perf_init() ++ * @dev_priv: i915 device instance ++ */ ++void i915_perf_fini(struct drm_i915_private *dev_priv) ++{ ++ if (!dev_priv->perf.initialized) ++ return; ++ ++ idr_for_each(&dev_priv->perf.metrics_idr, destroy_config, dev_priv); ++ idr_destroy(&dev_priv->perf.metrics_idr); ++ ++ unregister_sysctl_table(dev_priv->perf.sysctl_header); ++ ++ memset(&dev_priv->perf.oa.ops, 0, sizeof(dev_priv->perf.oa.ops)); ++ ++ dev_priv->perf.initialized = false; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_pmu.c b/drivers/gpu/drm/i915_legacy/i915_pmu.c +new file mode 100644 +index 000000000000..46a52da3db29 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_pmu.c +@@ -0,0 +1,1096 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2017-2018 Intel Corporation ++ */ ++ ++#include ++#include ++#include "i915_pmu.h" ++#include "intel_ringbuffer.h" ++#include "i915_drv.h" ++ ++/* Frequency for the sampling timer for events which need it. */ ++#define FREQUENCY 200 ++#define PERIOD max_t(u64, 10000, NSEC_PER_SEC / FREQUENCY) ++ ++#define ENGINE_SAMPLE_MASK \ ++ (BIT(I915_SAMPLE_BUSY) | \ ++ BIT(I915_SAMPLE_WAIT) | \ ++ BIT(I915_SAMPLE_SEMA)) ++ ++#define ENGINE_SAMPLE_BITS (1 << I915_PMU_SAMPLE_BITS) ++ ++static cpumask_t i915_pmu_cpumask; ++ ++static u8 engine_config_sample(u64 config) ++{ ++ return config & I915_PMU_SAMPLE_MASK; ++} ++ ++static u8 engine_event_sample(struct perf_event *event) ++{ ++ return engine_config_sample(event->attr.config); ++} ++ ++static u8 engine_event_class(struct perf_event *event) ++{ ++ return (event->attr.config >> I915_PMU_CLASS_SHIFT) & 0xff; ++} ++ ++static u8 engine_event_instance(struct perf_event *event) ++{ ++ return (event->attr.config >> I915_PMU_SAMPLE_BITS) & 0xff; ++} ++ ++static bool is_engine_config(u64 config) ++{ ++ return config < __I915_PMU_OTHER(0); ++} ++ ++static unsigned int config_enabled_bit(u64 config) ++{ ++ if (is_engine_config(config)) ++ return engine_config_sample(config); ++ else ++ return ENGINE_SAMPLE_BITS + (config - __I915_PMU_OTHER(0)); ++} ++ ++static u64 config_enabled_mask(u64 config) ++{ ++ return BIT_ULL(config_enabled_bit(config)); ++} ++ ++static bool is_engine_event(struct perf_event *event) ++{ ++ return is_engine_config(event->attr.config); ++} ++ ++static unsigned int event_enabled_bit(struct perf_event *event) ++{ ++ return config_enabled_bit(event->attr.config); ++} ++ ++static bool pmu_needs_timer(struct drm_i915_private *i915, bool gpu_active) ++{ ++ u64 enable; ++ ++ /* ++ * Only some counters need the sampling timer. ++ * ++ * We start with a bitmask of all currently enabled events. ++ */ ++ enable = i915->pmu.enable; ++ ++ /* ++ * Mask out all the ones which do not need the timer, or in ++ * other words keep all the ones that could need the timer. ++ */ ++ enable &= config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY) | ++ config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY) | ++ ENGINE_SAMPLE_MASK; ++ ++ /* ++ * When the GPU is idle per-engine counters do not need to be ++ * running so clear those bits out. ++ */ ++ if (!gpu_active) ++ enable &= ~ENGINE_SAMPLE_MASK; ++ /* ++ * Also there is software busyness tracking available we do not ++ * need the timer for I915_SAMPLE_BUSY counter. ++ * ++ * Use RCS as proxy for all engines. ++ */ ++ else if (intel_engine_supports_stats(i915->engine[RCS0])) ++ enable &= ~BIT(I915_SAMPLE_BUSY); ++ ++ /* ++ * If some bits remain it means we need the sampling timer running. ++ */ ++ return enable; ++} ++ ++void i915_pmu_gt_parked(struct drm_i915_private *i915) ++{ ++ if (!i915->pmu.base.event_init) ++ return; ++ ++ spin_lock_irq(&i915->pmu.lock); ++ /* ++ * Signal sampling timer to stop if only engine events are enabled and ++ * GPU went idle. ++ */ ++ i915->pmu.timer_enabled = pmu_needs_timer(i915, false); ++ spin_unlock_irq(&i915->pmu.lock); ++} ++ ++static void __i915_pmu_maybe_start_timer(struct drm_i915_private *i915) ++{ ++ if (!i915->pmu.timer_enabled && pmu_needs_timer(i915, true)) { ++ i915->pmu.timer_enabled = true; ++ i915->pmu.timer_last = ktime_get(); ++ hrtimer_start_range_ns(&i915->pmu.timer, ++ ns_to_ktime(PERIOD), 0, ++ HRTIMER_MODE_REL_PINNED); ++ } ++} ++ ++void i915_pmu_gt_unparked(struct drm_i915_private *i915) ++{ ++ if (!i915->pmu.base.event_init) ++ return; ++ ++ spin_lock_irq(&i915->pmu.lock); ++ /* ++ * Re-enable sampling timer when GPU goes active. ++ */ ++ __i915_pmu_maybe_start_timer(i915); ++ spin_unlock_irq(&i915->pmu.lock); ++} ++ ++static void ++add_sample(struct i915_pmu_sample *sample, u32 val) ++{ ++ sample->cur += val; ++} ++ ++static void ++engines_sample(struct drm_i915_private *dev_priv, unsigned int period_ns) ++{ ++ struct intel_engine_cs *engine; ++ enum intel_engine_id id; ++ intel_wakeref_t wakeref; ++ unsigned long flags; ++ ++ if ((dev_priv->pmu.enable & ENGINE_SAMPLE_MASK) == 0) ++ return; ++ ++ wakeref = 0; ++ if (READ_ONCE(dev_priv->gt.awake)) ++ wakeref = intel_runtime_pm_get_if_in_use(dev_priv); ++ if (!wakeref) ++ return; ++ ++ spin_lock_irqsave(&dev_priv->uncore.lock, flags); ++ for_each_engine(engine, dev_priv, id) { ++ struct intel_engine_pmu *pmu = &engine->pmu; ++ bool busy; ++ u32 val; ++ ++ val = I915_READ_FW(RING_CTL(engine->mmio_base)); ++ if (val == 0) /* powerwell off => engine idle */ ++ continue; ++ ++ if (val & RING_WAIT) ++ add_sample(&pmu->sample[I915_SAMPLE_WAIT], period_ns); ++ if (val & RING_WAIT_SEMAPHORE) ++ add_sample(&pmu->sample[I915_SAMPLE_SEMA], period_ns); ++ ++ /* ++ * While waiting on a semaphore or event, MI_MODE reports the ++ * ring as idle. However, previously using the seqno, and with ++ * execlists sampling, we account for the ring waiting as the ++ * engine being busy. Therefore, we record the sample as being ++ * busy if either waiting or !idle. ++ */ ++ busy = val & (RING_WAIT_SEMAPHORE | RING_WAIT); ++ if (!busy) { ++ val = I915_READ_FW(RING_MI_MODE(engine->mmio_base)); ++ busy = !(val & MODE_IDLE); ++ } ++ if (busy) ++ add_sample(&pmu->sample[I915_SAMPLE_BUSY], period_ns); ++ } ++ spin_unlock_irqrestore(&dev_priv->uncore.lock, flags); ++ ++ intel_runtime_pm_put(dev_priv, wakeref); ++} ++ ++static void ++add_sample_mult(struct i915_pmu_sample *sample, u32 val, u32 mul) ++{ ++ sample->cur += mul_u32_u32(val, mul); ++} ++ ++static void ++frequency_sample(struct drm_i915_private *dev_priv, unsigned int period_ns) ++{ ++ if (dev_priv->pmu.enable & ++ config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY)) { ++ u32 val; ++ ++ val = dev_priv->gt_pm.rps.cur_freq; ++ if (dev_priv->gt.awake) { ++ intel_wakeref_t wakeref; ++ ++ with_intel_runtime_pm_if_in_use(dev_priv, wakeref) ++ val = intel_get_cagf(dev_priv, ++ I915_READ_NOTRACE(GEN6_RPSTAT1)); ++ } ++ ++ add_sample_mult(&dev_priv->pmu.sample[__I915_SAMPLE_FREQ_ACT], ++ intel_gpu_freq(dev_priv, val), ++ period_ns / 1000); ++ } ++ ++ if (dev_priv->pmu.enable & ++ config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY)) { ++ add_sample_mult(&dev_priv->pmu.sample[__I915_SAMPLE_FREQ_REQ], ++ intel_gpu_freq(dev_priv, ++ dev_priv->gt_pm.rps.cur_freq), ++ period_ns / 1000); ++ } ++} ++ ++static enum hrtimer_restart i915_sample(struct hrtimer *hrtimer) ++{ ++ struct drm_i915_private *i915 = ++ container_of(hrtimer, struct drm_i915_private, pmu.timer); ++ unsigned int period_ns; ++ ktime_t now; ++ ++ if (!READ_ONCE(i915->pmu.timer_enabled)) ++ return HRTIMER_NORESTART; ++ ++ now = ktime_get(); ++ period_ns = ktime_to_ns(ktime_sub(now, i915->pmu.timer_last)); ++ i915->pmu.timer_last = now; ++ ++ /* ++ * Strictly speaking the passed in period may not be 100% accurate for ++ * all internal calculation, since some amount of time can be spent on ++ * grabbing the forcewake. However the potential error from timer call- ++ * back delay greatly dominates this so we keep it simple. ++ */ ++ engines_sample(i915, period_ns); ++ frequency_sample(i915, period_ns); ++ ++ hrtimer_forward(hrtimer, now, ns_to_ktime(PERIOD)); ++ ++ return HRTIMER_RESTART; ++} ++ ++static u64 count_interrupts(struct drm_i915_private *i915) ++{ ++ /* open-coded kstat_irqs() */ ++ struct irq_desc *desc = irq_to_desc(i915->drm.pdev->irq); ++ u64 sum = 0; ++ int cpu; ++ ++ if (!desc || !desc->kstat_irqs) ++ return 0; ++ ++ for_each_possible_cpu(cpu) ++ sum += *per_cpu_ptr(desc->kstat_irqs, cpu); ++ ++ return sum; ++} ++ ++static void engine_event_destroy(struct perf_event *event) ++{ ++ struct drm_i915_private *i915 = ++ container_of(event->pmu, typeof(*i915), pmu.base); ++ struct intel_engine_cs *engine; ++ ++ engine = intel_engine_lookup_user(i915, ++ engine_event_class(event), ++ engine_event_instance(event)); ++ if (WARN_ON_ONCE(!engine)) ++ return; ++ ++ if (engine_event_sample(event) == I915_SAMPLE_BUSY && ++ intel_engine_supports_stats(engine)) ++ intel_disable_engine_stats(engine); ++} ++ ++static void i915_pmu_event_destroy(struct perf_event *event) ++{ ++ WARN_ON(event->parent); ++ ++ if (is_engine_event(event)) ++ engine_event_destroy(event); ++} ++ ++static int ++engine_event_status(struct intel_engine_cs *engine, ++ enum drm_i915_pmu_engine_sample sample) ++{ ++ switch (sample) { ++ case I915_SAMPLE_BUSY: ++ case I915_SAMPLE_WAIT: ++ break; ++ case I915_SAMPLE_SEMA: ++ if (INTEL_GEN(engine->i915) < 6) ++ return -ENODEV; ++ break; ++ default: ++ return -ENOENT; ++ } ++ ++ return 0; ++} ++ ++static int ++config_status(struct drm_i915_private *i915, u64 config) ++{ ++ switch (config) { ++ case I915_PMU_ACTUAL_FREQUENCY: ++ if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) ++ /* Requires a mutex for sampling! */ ++ return -ENODEV; ++ /* Fall-through. */ ++ case I915_PMU_REQUESTED_FREQUENCY: ++ if (INTEL_GEN(i915) < 6) ++ return -ENODEV; ++ break; ++ case I915_PMU_INTERRUPTS: ++ break; ++ case I915_PMU_RC6_RESIDENCY: ++ if (!HAS_RC6(i915)) ++ return -ENODEV; ++ break; ++ default: ++ return -ENOENT; ++ } ++ ++ return 0; ++} ++ ++static int engine_event_init(struct perf_event *event) ++{ ++ struct drm_i915_private *i915 = ++ container_of(event->pmu, typeof(*i915), pmu.base); ++ struct intel_engine_cs *engine; ++ u8 sample; ++ int ret; ++ ++ engine = intel_engine_lookup_user(i915, engine_event_class(event), ++ engine_event_instance(event)); ++ if (!engine) ++ return -ENODEV; ++ ++ sample = engine_event_sample(event); ++ ret = engine_event_status(engine, sample); ++ if (ret) ++ return ret; ++ ++ if (sample == I915_SAMPLE_BUSY && intel_engine_supports_stats(engine)) ++ ret = intel_enable_engine_stats(engine); ++ ++ return ret; ++} ++ ++static int i915_pmu_event_init(struct perf_event *event) ++{ ++ struct drm_i915_private *i915 = ++ container_of(event->pmu, typeof(*i915), pmu.base); ++ int ret; ++ ++ if (event->attr.type != event->pmu->type) ++ return -ENOENT; ++ ++ /* unsupported modes and filters */ ++ if (event->attr.sample_period) /* no sampling */ ++ return -EINVAL; ++ ++ if (has_branch_stack(event)) ++ return -EOPNOTSUPP; ++ ++ if (event->cpu < 0) ++ return -EINVAL; ++ ++ /* only allow running on one cpu at a time */ ++ if (!cpumask_test_cpu(event->cpu, &i915_pmu_cpumask)) ++ return -EINVAL; ++ ++ if (is_engine_event(event)) ++ ret = engine_event_init(event); ++ else ++ ret = config_status(i915, event->attr.config); ++ if (ret) ++ return ret; ++ ++ if (!event->parent) ++ event->destroy = i915_pmu_event_destroy; ++ ++ return 0; ++} ++ ++static u64 __get_rc6(struct drm_i915_private *i915) ++{ ++ u64 val; ++ ++ val = intel_rc6_residency_ns(i915, ++ IS_VALLEYVIEW(i915) ? ++ VLV_GT_RENDER_RC6 : ++ GEN6_GT_GFX_RC6); ++ ++ if (HAS_RC6p(i915)) ++ val += intel_rc6_residency_ns(i915, GEN6_GT_GFX_RC6p); ++ ++ if (HAS_RC6pp(i915)) ++ val += intel_rc6_residency_ns(i915, GEN6_GT_GFX_RC6pp); ++ ++ return val; ++} ++ ++static u64 get_rc6(struct drm_i915_private *i915) ++{ ++#if IS_ENABLED(CONFIG_PM) ++ intel_wakeref_t wakeref; ++ unsigned long flags; ++ u64 val; ++ ++ wakeref = intel_runtime_pm_get_if_in_use(i915); ++ if (wakeref) { ++ val = __get_rc6(i915); ++ intel_runtime_pm_put(i915, wakeref); ++ ++ /* ++ * If we are coming back from being runtime suspended we must ++ * be careful not to report a larger value than returned ++ * previously. ++ */ ++ ++ spin_lock_irqsave(&i915->pmu.lock, flags); ++ ++ if (val >= i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur) { ++ i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur = 0; ++ i915->pmu.sample[__I915_SAMPLE_RC6].cur = val; ++ } else { ++ val = i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur; ++ } ++ ++ spin_unlock_irqrestore(&i915->pmu.lock, flags); ++ } else { ++ struct pci_dev *pdev = i915->drm.pdev; ++ struct device *kdev = &pdev->dev; ++ ++ /* ++ * We are runtime suspended. ++ * ++ * Report the delta from when the device was suspended to now, ++ * on top of the last known real value, as the approximated RC6 ++ * counter value. ++ */ ++ spin_lock_irqsave(&i915->pmu.lock, flags); ++ ++ /* ++ * After the above branch intel_runtime_pm_get_if_in_use failed ++ * to get the runtime PM reference we cannot assume we are in ++ * runtime suspend since we can either: a) race with coming out ++ * of it before we took the power.lock, or b) there are other ++ * states than suspended which can bring us here. ++ * ++ * We need to double-check that we are indeed currently runtime ++ * suspended and if not we cannot do better than report the last ++ * known RC6 value. ++ */ ++ if (pm_runtime_status_suspended(kdev)) { ++ val = pm_runtime_suspended_time(kdev); ++ ++ if (!i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur) ++ i915->pmu.suspended_time_last = val; ++ ++ val -= i915->pmu.suspended_time_last; ++ val += i915->pmu.sample[__I915_SAMPLE_RC6].cur; ++ ++ i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur = val; ++ } else if (i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur) { ++ val = i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur; ++ } else { ++ val = i915->pmu.sample[__I915_SAMPLE_RC6].cur; ++ } ++ ++ spin_unlock_irqrestore(&i915->pmu.lock, flags); ++ } ++ ++ return val; ++#else ++ return __get_rc6(i915); ++#endif ++} ++ ++static u64 __i915_pmu_event_read(struct perf_event *event) ++{ ++ struct drm_i915_private *i915 = ++ container_of(event->pmu, typeof(*i915), pmu.base); ++ u64 val = 0; ++ ++ if (is_engine_event(event)) { ++ u8 sample = engine_event_sample(event); ++ struct intel_engine_cs *engine; ++ ++ engine = intel_engine_lookup_user(i915, ++ engine_event_class(event), ++ engine_event_instance(event)); ++ ++ if (WARN_ON_ONCE(!engine)) { ++ /* Do nothing */ ++ } else if (sample == I915_SAMPLE_BUSY && ++ intel_engine_supports_stats(engine)) { ++ val = ktime_to_ns(intel_engine_get_busy_time(engine)); ++ } else { ++ val = engine->pmu.sample[sample].cur; ++ } ++ } else { ++ switch (event->attr.config) { ++ case I915_PMU_ACTUAL_FREQUENCY: ++ val = ++ div_u64(i915->pmu.sample[__I915_SAMPLE_FREQ_ACT].cur, ++ USEC_PER_SEC /* to MHz */); ++ break; ++ case I915_PMU_REQUESTED_FREQUENCY: ++ val = ++ div_u64(i915->pmu.sample[__I915_SAMPLE_FREQ_REQ].cur, ++ USEC_PER_SEC /* to MHz */); ++ break; ++ case I915_PMU_INTERRUPTS: ++ val = count_interrupts(i915); ++ break; ++ case I915_PMU_RC6_RESIDENCY: ++ val = get_rc6(i915); ++ break; ++ } ++ } ++ ++ return val; ++} ++ ++static void i915_pmu_event_read(struct perf_event *event) ++{ ++ struct hw_perf_event *hwc = &event->hw; ++ u64 prev, new; ++ ++again: ++ prev = local64_read(&hwc->prev_count); ++ new = __i915_pmu_event_read(event); ++ ++ if (local64_cmpxchg(&hwc->prev_count, prev, new) != prev) ++ goto again; ++ ++ local64_add(new - prev, &event->count); ++} ++ ++static void i915_pmu_enable(struct perf_event *event) ++{ ++ struct drm_i915_private *i915 = ++ container_of(event->pmu, typeof(*i915), pmu.base); ++ unsigned int bit = event_enabled_bit(event); ++ unsigned long flags; ++ ++ spin_lock_irqsave(&i915->pmu.lock, flags); ++ ++ /* ++ * Update the bitmask of enabled events and increment ++ * the event reference counter. ++ */ ++ BUILD_BUG_ON(ARRAY_SIZE(i915->pmu.enable_count) != I915_PMU_MASK_BITS); ++ GEM_BUG_ON(bit >= ARRAY_SIZE(i915->pmu.enable_count)); ++ GEM_BUG_ON(i915->pmu.enable_count[bit] == ~0); ++ i915->pmu.enable |= BIT_ULL(bit); ++ i915->pmu.enable_count[bit]++; ++ ++ /* ++ * Start the sampling timer if needed and not already enabled. ++ */ ++ __i915_pmu_maybe_start_timer(i915); ++ ++ /* ++ * For per-engine events the bitmask and reference counting ++ * is stored per engine. ++ */ ++ if (is_engine_event(event)) { ++ u8 sample = engine_event_sample(event); ++ struct intel_engine_cs *engine; ++ ++ engine = intel_engine_lookup_user(i915, ++ engine_event_class(event), ++ engine_event_instance(event)); ++ ++ BUILD_BUG_ON(ARRAY_SIZE(engine->pmu.enable_count) != ++ I915_ENGINE_SAMPLE_COUNT); ++ BUILD_BUG_ON(ARRAY_SIZE(engine->pmu.sample) != ++ I915_ENGINE_SAMPLE_COUNT); ++ GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.enable_count)); ++ GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.sample)); ++ GEM_BUG_ON(engine->pmu.enable_count[sample] == ~0); ++ ++ engine->pmu.enable |= BIT(sample); ++ engine->pmu.enable_count[sample]++; ++ } ++ ++ spin_unlock_irqrestore(&i915->pmu.lock, flags); ++ ++ /* ++ * Store the current counter value so we can report the correct delta ++ * for all listeners. Even when the event was already enabled and has ++ * an existing non-zero value. ++ */ ++ local64_set(&event->hw.prev_count, __i915_pmu_event_read(event)); ++} ++ ++static void i915_pmu_disable(struct perf_event *event) ++{ ++ struct drm_i915_private *i915 = ++ container_of(event->pmu, typeof(*i915), pmu.base); ++ unsigned int bit = event_enabled_bit(event); ++ unsigned long flags; ++ ++ spin_lock_irqsave(&i915->pmu.lock, flags); ++ ++ if (is_engine_event(event)) { ++ u8 sample = engine_event_sample(event); ++ struct intel_engine_cs *engine; ++ ++ engine = intel_engine_lookup_user(i915, ++ engine_event_class(event), ++ engine_event_instance(event)); ++ ++ GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.enable_count)); ++ GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.sample)); ++ GEM_BUG_ON(engine->pmu.enable_count[sample] == 0); ++ ++ /* ++ * Decrement the reference count and clear the enabled ++ * bitmask when the last listener on an event goes away. ++ */ ++ if (--engine->pmu.enable_count[sample] == 0) ++ engine->pmu.enable &= ~BIT(sample); ++ } ++ ++ GEM_BUG_ON(bit >= ARRAY_SIZE(i915->pmu.enable_count)); ++ GEM_BUG_ON(i915->pmu.enable_count[bit] == 0); ++ /* ++ * Decrement the reference count and clear the enabled ++ * bitmask when the last listener on an event goes away. ++ */ ++ if (--i915->pmu.enable_count[bit] == 0) { ++ i915->pmu.enable &= ~BIT_ULL(bit); ++ i915->pmu.timer_enabled &= pmu_needs_timer(i915, true); ++ } ++ ++ spin_unlock_irqrestore(&i915->pmu.lock, flags); ++} ++ ++static void i915_pmu_event_start(struct perf_event *event, int flags) ++{ ++ i915_pmu_enable(event); ++ event->hw.state = 0; ++} ++ ++static void i915_pmu_event_stop(struct perf_event *event, int flags) ++{ ++ if (flags & PERF_EF_UPDATE) ++ i915_pmu_event_read(event); ++ i915_pmu_disable(event); ++ event->hw.state = PERF_HES_STOPPED; ++} ++ ++static int i915_pmu_event_add(struct perf_event *event, int flags) ++{ ++ if (flags & PERF_EF_START) ++ i915_pmu_event_start(event, flags); ++ ++ return 0; ++} ++ ++static void i915_pmu_event_del(struct perf_event *event, int flags) ++{ ++ i915_pmu_event_stop(event, PERF_EF_UPDATE); ++} ++ ++static int i915_pmu_event_event_idx(struct perf_event *event) ++{ ++ return 0; ++} ++ ++struct i915_str_attribute { ++ struct device_attribute attr; ++ const char *str; ++}; ++ ++static ssize_t i915_pmu_format_show(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ struct i915_str_attribute *eattr; ++ ++ eattr = container_of(attr, struct i915_str_attribute, attr); ++ return sprintf(buf, "%s\n", eattr->str); ++} ++ ++#define I915_PMU_FORMAT_ATTR(_name, _config) \ ++ (&((struct i915_str_attribute[]) { \ ++ { .attr = __ATTR(_name, 0444, i915_pmu_format_show, NULL), \ ++ .str = _config, } \ ++ })[0].attr.attr) ++ ++static struct attribute *i915_pmu_format_attrs[] = { ++ I915_PMU_FORMAT_ATTR(i915_eventid, "config:0-20"), ++ NULL, ++}; ++ ++static const struct attribute_group i915_pmu_format_attr_group = { ++ .name = "format", ++ .attrs = i915_pmu_format_attrs, ++}; ++ ++struct i915_ext_attribute { ++ struct device_attribute attr; ++ unsigned long val; ++}; ++ ++static ssize_t i915_pmu_event_show(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ struct i915_ext_attribute *eattr; ++ ++ eattr = container_of(attr, struct i915_ext_attribute, attr); ++ return sprintf(buf, "config=0x%lx\n", eattr->val); ++} ++ ++static struct attribute_group i915_pmu_events_attr_group = { ++ .name = "events", ++ /* Patch in attrs at runtime. */ ++}; ++ ++static ssize_t ++i915_pmu_get_attr_cpumask(struct device *dev, ++ struct device_attribute *attr, ++ char *buf) ++{ ++ return cpumap_print_to_pagebuf(true, buf, &i915_pmu_cpumask); ++} ++ ++static DEVICE_ATTR(cpumask, 0444, i915_pmu_get_attr_cpumask, NULL); ++ ++static struct attribute *i915_cpumask_attrs[] = { ++ &dev_attr_cpumask.attr, ++ NULL, ++}; ++ ++static const struct attribute_group i915_pmu_cpumask_attr_group = { ++ .attrs = i915_cpumask_attrs, ++}; ++ ++static const struct attribute_group *i915_pmu_attr_groups[] = { ++ &i915_pmu_format_attr_group, ++ &i915_pmu_events_attr_group, ++ &i915_pmu_cpumask_attr_group, ++ NULL ++}; ++ ++#define __event(__config, __name, __unit) \ ++{ \ ++ .config = (__config), \ ++ .name = (__name), \ ++ .unit = (__unit), \ ++} ++ ++#define __engine_event(__sample, __name) \ ++{ \ ++ .sample = (__sample), \ ++ .name = (__name), \ ++} ++ ++static struct i915_ext_attribute * ++add_i915_attr(struct i915_ext_attribute *attr, const char *name, u64 config) ++{ ++ sysfs_attr_init(&attr->attr.attr); ++ attr->attr.attr.name = name; ++ attr->attr.attr.mode = 0444; ++ attr->attr.show = i915_pmu_event_show; ++ attr->val = config; ++ ++ return ++attr; ++} ++ ++static struct perf_pmu_events_attr * ++add_pmu_attr(struct perf_pmu_events_attr *attr, const char *name, ++ const char *str) ++{ ++ sysfs_attr_init(&attr->attr.attr); ++ attr->attr.attr.name = name; ++ attr->attr.attr.mode = 0444; ++ attr->attr.show = perf_event_sysfs_show; ++ attr->event_str = str; ++ ++ return ++attr; ++} ++ ++static struct attribute ** ++create_event_attributes(struct drm_i915_private *i915) ++{ ++ static const struct { ++ u64 config; ++ const char *name; ++ const char *unit; ++ } events[] = { ++ __event(I915_PMU_ACTUAL_FREQUENCY, "actual-frequency", "MHz"), ++ __event(I915_PMU_REQUESTED_FREQUENCY, "requested-frequency", "MHz"), ++ __event(I915_PMU_INTERRUPTS, "interrupts", NULL), ++ __event(I915_PMU_RC6_RESIDENCY, "rc6-residency", "ns"), ++ }; ++ static const struct { ++ enum drm_i915_pmu_engine_sample sample; ++ char *name; ++ } engine_events[] = { ++ __engine_event(I915_SAMPLE_BUSY, "busy"), ++ __engine_event(I915_SAMPLE_SEMA, "sema"), ++ __engine_event(I915_SAMPLE_WAIT, "wait"), ++ }; ++ unsigned int count = 0; ++ struct perf_pmu_events_attr *pmu_attr = NULL, *pmu_iter; ++ struct i915_ext_attribute *i915_attr = NULL, *i915_iter; ++ struct attribute **attr = NULL, **attr_iter; ++ struct intel_engine_cs *engine; ++ enum intel_engine_id id; ++ unsigned int i; ++ ++ /* Count how many counters we will be exposing. */ ++ for (i = 0; i < ARRAY_SIZE(events); i++) { ++ if (!config_status(i915, events[i].config)) ++ count++; ++ } ++ ++ for_each_engine(engine, i915, id) { ++ for (i = 0; i < ARRAY_SIZE(engine_events); i++) { ++ if (!engine_event_status(engine, ++ engine_events[i].sample)) ++ count++; ++ } ++ } ++ ++ /* Allocate attribute objects and table. */ ++ i915_attr = kcalloc(count, sizeof(*i915_attr), GFP_KERNEL); ++ if (!i915_attr) ++ goto err_alloc; ++ ++ pmu_attr = kcalloc(count, sizeof(*pmu_attr), GFP_KERNEL); ++ if (!pmu_attr) ++ goto err_alloc; ++ ++ /* Max one pointer of each attribute type plus a termination entry. */ ++ attr = kcalloc(count * 2 + 1, sizeof(*attr), GFP_KERNEL); ++ if (!attr) ++ goto err_alloc; ++ ++ i915_iter = i915_attr; ++ pmu_iter = pmu_attr; ++ attr_iter = attr; ++ ++ /* Initialize supported non-engine counters. */ ++ for (i = 0; i < ARRAY_SIZE(events); i++) { ++ char *str; ++ ++ if (config_status(i915, events[i].config)) ++ continue; ++ ++ str = kstrdup(events[i].name, GFP_KERNEL); ++ if (!str) ++ goto err; ++ ++ *attr_iter++ = &i915_iter->attr.attr; ++ i915_iter = add_i915_attr(i915_iter, str, events[i].config); ++ ++ if (events[i].unit) { ++ str = kasprintf(GFP_KERNEL, "%s.unit", events[i].name); ++ if (!str) ++ goto err; ++ ++ *attr_iter++ = &pmu_iter->attr.attr; ++ pmu_iter = add_pmu_attr(pmu_iter, str, events[i].unit); ++ } ++ } ++ ++ /* Initialize supported engine counters. */ ++ for_each_engine(engine, i915, id) { ++ for (i = 0; i < ARRAY_SIZE(engine_events); i++) { ++ char *str; ++ ++ if (engine_event_status(engine, ++ engine_events[i].sample)) ++ continue; ++ ++ str = kasprintf(GFP_KERNEL, "%s-%s", ++ engine->name, engine_events[i].name); ++ if (!str) ++ goto err; ++ ++ *attr_iter++ = &i915_iter->attr.attr; ++ i915_iter = ++ add_i915_attr(i915_iter, str, ++ __I915_PMU_ENGINE(engine->uabi_class, ++ engine->instance, ++ engine_events[i].sample)); ++ ++ str = kasprintf(GFP_KERNEL, "%s-%s.unit", ++ engine->name, engine_events[i].name); ++ if (!str) ++ goto err; ++ ++ *attr_iter++ = &pmu_iter->attr.attr; ++ pmu_iter = add_pmu_attr(pmu_iter, str, "ns"); ++ } ++ } ++ ++ i915->pmu.i915_attr = i915_attr; ++ i915->pmu.pmu_attr = pmu_attr; ++ ++ return attr; ++ ++err:; ++ for (attr_iter = attr; *attr_iter; attr_iter++) ++ kfree((*attr_iter)->name); ++ ++err_alloc: ++ kfree(attr); ++ kfree(i915_attr); ++ kfree(pmu_attr); ++ ++ return NULL; ++} ++ ++static void free_event_attributes(struct drm_i915_private *i915) ++{ ++ struct attribute **attr_iter = i915_pmu_events_attr_group.attrs; ++ ++ for (; *attr_iter; attr_iter++) ++ kfree((*attr_iter)->name); ++ ++ kfree(i915_pmu_events_attr_group.attrs); ++ kfree(i915->pmu.i915_attr); ++ kfree(i915->pmu.pmu_attr); ++ ++ i915_pmu_events_attr_group.attrs = NULL; ++ i915->pmu.i915_attr = NULL; ++ i915->pmu.pmu_attr = NULL; ++} ++ ++static int i915_pmu_cpu_online(unsigned int cpu, struct hlist_node *node) ++{ ++ struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), node); ++ ++ GEM_BUG_ON(!pmu->base.event_init); ++ ++ /* Select the first online CPU as a designated reader. */ ++ if (!cpumask_weight(&i915_pmu_cpumask)) ++ cpumask_set_cpu(cpu, &i915_pmu_cpumask); ++ ++ return 0; ++} ++ ++static int i915_pmu_cpu_offline(unsigned int cpu, struct hlist_node *node) ++{ ++ struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), node); ++ unsigned int target; ++ ++ GEM_BUG_ON(!pmu->base.event_init); ++ ++ if (cpumask_test_and_clear_cpu(cpu, &i915_pmu_cpumask)) { ++ target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu); ++ /* Migrate events if there is a valid target */ ++ if (target < nr_cpu_ids) { ++ cpumask_set_cpu(target, &i915_pmu_cpumask); ++ perf_pmu_migrate_context(&pmu->base, cpu, target); ++ } ++ } ++ ++ return 0; ++} ++ ++static enum cpuhp_state cpuhp_slot = CPUHP_INVALID; ++ ++static int i915_pmu_register_cpuhp_state(struct drm_i915_private *i915) ++{ ++ enum cpuhp_state slot; ++ int ret; ++ ++ ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, ++ "perf/x86/intel/i915:online", ++ i915_pmu_cpu_online, ++ i915_pmu_cpu_offline); ++ if (ret < 0) ++ return ret; ++ ++ slot = ret; ++ ret = cpuhp_state_add_instance(slot, &i915->pmu.node); ++ if (ret) { ++ cpuhp_remove_multi_state(slot); ++ return ret; ++ } ++ ++ cpuhp_slot = slot; ++ return 0; ++} ++ ++static void i915_pmu_unregister_cpuhp_state(struct drm_i915_private *i915) ++{ ++ WARN_ON(cpuhp_slot == CPUHP_INVALID); ++ WARN_ON(cpuhp_state_remove_instance(cpuhp_slot, &i915->pmu.node)); ++ cpuhp_remove_multi_state(cpuhp_slot); ++} ++ ++void i915_pmu_register(struct drm_i915_private *i915) ++{ ++ int ret; ++ ++ if (INTEL_GEN(i915) <= 2) { ++ DRM_INFO("PMU not supported for this GPU."); ++ return; ++ } ++ ++ i915_pmu_events_attr_group.attrs = create_event_attributes(i915); ++ if (!i915_pmu_events_attr_group.attrs) { ++ ret = -ENOMEM; ++ goto err; ++ } ++ ++ i915->pmu.base.attr_groups = i915_pmu_attr_groups; ++ i915->pmu.base.task_ctx_nr = perf_invalid_context; ++ i915->pmu.base.event_init = i915_pmu_event_init; ++ i915->pmu.base.add = i915_pmu_event_add; ++ i915->pmu.base.del = i915_pmu_event_del; ++ i915->pmu.base.start = i915_pmu_event_start; ++ i915->pmu.base.stop = i915_pmu_event_stop; ++ i915->pmu.base.read = i915_pmu_event_read; ++ i915->pmu.base.event_idx = i915_pmu_event_event_idx; ++ ++ spin_lock_init(&i915->pmu.lock); ++ hrtimer_init(&i915->pmu.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); ++ i915->pmu.timer.function = i915_sample; ++ ++ ret = perf_pmu_register(&i915->pmu.base, "i915", -1); ++ if (ret) ++ goto err; ++ ++ ret = i915_pmu_register_cpuhp_state(i915); ++ if (ret) ++ goto err_unreg; ++ ++ return; ++ ++err_unreg: ++ perf_pmu_unregister(&i915->pmu.base); ++err: ++ i915->pmu.base.event_init = NULL; ++ free_event_attributes(i915); ++ DRM_NOTE("Failed to register PMU! (err=%d)\n", ret); ++} ++ ++void i915_pmu_unregister(struct drm_i915_private *i915) ++{ ++ if (!i915->pmu.base.event_init) ++ return; ++ ++ WARN_ON(i915->pmu.enable); ++ ++ hrtimer_cancel(&i915->pmu.timer); ++ ++ i915_pmu_unregister_cpuhp_state(i915); ++ ++ perf_pmu_unregister(&i915->pmu.base); ++ i915->pmu.base.event_init = NULL; ++ free_event_attributes(i915); ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_pmu.h b/drivers/gpu/drm/i915_legacy/i915_pmu.h +new file mode 100644 +index 000000000000..4fc4f2478301 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_pmu.h +@@ -0,0 +1,125 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2017-2018 Intel Corporation ++ */ ++ ++#ifndef __I915_PMU_H__ ++#define __I915_PMU_H__ ++ ++#include ++#include ++#include ++#include ++ ++struct drm_i915_private; ++ ++enum { ++ __I915_SAMPLE_FREQ_ACT = 0, ++ __I915_SAMPLE_FREQ_REQ, ++ __I915_SAMPLE_RC6, ++ __I915_SAMPLE_RC6_ESTIMATED, ++ __I915_NUM_PMU_SAMPLERS ++}; ++ ++/** ++ * How many different events we track in the global PMU mask. ++ * ++ * It is also used to know to needed number of event reference counters. ++ */ ++#define I915_PMU_MASK_BITS \ ++ ((1 << I915_PMU_SAMPLE_BITS) + \ ++ (I915_PMU_LAST + 1 - __I915_PMU_OTHER(0))) ++ ++#define I915_ENGINE_SAMPLE_COUNT (I915_SAMPLE_SEMA + 1) ++ ++struct i915_pmu_sample { ++ u64 cur; ++}; ++ ++struct i915_pmu { ++ /** ++ * @node: List node for CPU hotplug handling. ++ */ ++ struct hlist_node node; ++ /** ++ * @base: PMU base. ++ */ ++ struct pmu base; ++ /** ++ * @lock: Lock protecting enable mask and ref count handling. ++ */ ++ spinlock_t lock; ++ /** ++ * @timer: Timer for internal i915 PMU sampling. ++ */ ++ struct hrtimer timer; ++ /** ++ * @enable: Bitmask of all currently enabled events. ++ * ++ * Bits are derived from uAPI event numbers in a way that low 16 bits ++ * correspond to engine event _sample_ _type_ (I915_SAMPLE_QUEUED is ++ * bit 0), and higher bits correspond to other events (for instance ++ * I915_PMU_ACTUAL_FREQUENCY is bit 16 etc). ++ * ++ * In other words, low 16 bits are not per engine but per engine ++ * sampler type, while the upper bits are directly mapped to other ++ * event types. ++ */ ++ u64 enable; ++ ++ /** ++ * @timer_last: ++ * ++ * Timestmap of the previous timer invocation. ++ */ ++ ktime_t timer_last; ++ ++ /** ++ * @enable_count: Reference counts for the enabled events. ++ * ++ * Array indices are mapped in the same way as bits in the @enable field ++ * and they are used to control sampling on/off when multiple clients ++ * are using the PMU API. ++ */ ++ unsigned int enable_count[I915_PMU_MASK_BITS]; ++ /** ++ * @timer_enabled: Should the internal sampling timer be running. ++ */ ++ bool timer_enabled; ++ /** ++ * @sample: Current and previous (raw) counters for sampling events. ++ * ++ * These counters are updated from the i915 PMU sampling timer. ++ * ++ * Only global counters are held here, while the per-engine ones are in ++ * struct intel_engine_cs. ++ */ ++ struct i915_pmu_sample sample[__I915_NUM_PMU_SAMPLERS]; ++ /** ++ * @suspended_time_last: Cached suspend time from PM core. ++ */ ++ u64 suspended_time_last; ++ /** ++ * @i915_attr: Memory block holding device attributes. ++ */ ++ void *i915_attr; ++ /** ++ * @pmu_attr: Memory block holding device attributes. ++ */ ++ void *pmu_attr; ++}; ++ ++#ifdef CONFIG_PERF_EVENTS ++void i915_pmu_register(struct drm_i915_private *i915); ++void i915_pmu_unregister(struct drm_i915_private *i915); ++void i915_pmu_gt_parked(struct drm_i915_private *i915); ++void i915_pmu_gt_unparked(struct drm_i915_private *i915); ++#else ++static inline void i915_pmu_register(struct drm_i915_private *i915) {} ++static inline void i915_pmu_unregister(struct drm_i915_private *i915) {} ++static inline void i915_pmu_gt_parked(struct drm_i915_private *i915) {} ++static inline void i915_pmu_gt_unparked(struct drm_i915_private *i915) {} ++#endif ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_priolist_types.h b/drivers/gpu/drm/i915_legacy/i915_priolist_types.h +new file mode 100644 +index 000000000000..49709de69875 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_priolist_types.h +@@ -0,0 +1,41 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ */ ++ ++#ifndef _I915_PRIOLIST_TYPES_H_ ++#define _I915_PRIOLIST_TYPES_H_ ++ ++#include ++#include ++ ++#include ++ ++enum { ++ I915_PRIORITY_MIN = I915_CONTEXT_MIN_USER_PRIORITY - 1, ++ I915_PRIORITY_NORMAL = I915_CONTEXT_DEFAULT_PRIORITY, ++ I915_PRIORITY_MAX = I915_CONTEXT_MAX_USER_PRIORITY + 1, ++ ++ I915_PRIORITY_INVALID = INT_MIN ++}; ++ ++#define I915_USER_PRIORITY_SHIFT 2 ++#define I915_USER_PRIORITY(x) ((x) << I915_USER_PRIORITY_SHIFT) ++ ++#define I915_PRIORITY_COUNT BIT(I915_USER_PRIORITY_SHIFT) ++#define I915_PRIORITY_MASK (I915_PRIORITY_COUNT - 1) ++ ++#define I915_PRIORITY_WAIT ((u8)BIT(0)) ++#define I915_PRIORITY_NOSEMAPHORE ((u8)BIT(1)) ++ ++#define __NO_PREEMPTION (I915_PRIORITY_WAIT) ++ ++struct i915_priolist { ++ struct list_head requests[I915_PRIORITY_COUNT]; ++ struct rb_node node; ++ unsigned long used; ++ int priority; ++}; ++ ++#endif /* _I915_PRIOLIST_TYPES_H_ */ +diff --git a/drivers/gpu/drm/i915_legacy/i915_pvinfo.h b/drivers/gpu/drm/i915_legacy/i915_pvinfo.h +new file mode 100644 +index 000000000000..969e514916ab +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_pvinfo.h +@@ -0,0 +1,120 @@ ++/* ++ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ */ ++ ++#ifndef _I915_PVINFO_H_ ++#define _I915_PVINFO_H_ ++ ++/* The MMIO offset of the shared info between guest and host emulator */ ++#define VGT_PVINFO_PAGE 0x78000 ++#define VGT_PVINFO_SIZE 0x1000 ++ ++/* ++ * The following structure pages are defined in GEN MMIO space ++ * for virtualization. (One page for now) ++ */ ++#define VGT_MAGIC 0x4776544776544776ULL /* 'vGTvGTvG' */ ++#define VGT_VERSION_MAJOR 1 ++#define VGT_VERSION_MINOR 0 ++ ++/* ++ * notifications from guest to vgpu device model ++ */ ++enum vgt_g2v_type { ++ VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE = 2, ++ VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY, ++ VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE, ++ VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY, ++ VGT_G2V_EXECLIST_CONTEXT_CREATE, ++ VGT_G2V_EXECLIST_CONTEXT_DESTROY, ++ VGT_G2V_MAX, ++}; ++ ++/* ++ * VGT capabilities type ++ */ ++#define VGT_CAPS_FULL_PPGTT BIT(2) ++#define VGT_CAPS_HWSP_EMULATION BIT(3) ++#define VGT_CAPS_HUGE_GTT BIT(4) ++ ++struct vgt_if { ++ u64 magic; /* VGT_MAGIC */ ++ u16 version_major; ++ u16 version_minor; ++ u32 vgt_id; /* ID of vGT instance */ ++ u32 vgt_caps; /* VGT capabilities */ ++ u32 rsv1[11]; /* pad to offset 0x40 */ ++ /* ++ * Data structure to describe the balooning info of resources. ++ * Each VM can only have one portion of continuous area for now. ++ * (May support scattered resource in future) ++ * (starting from offset 0x40) ++ */ ++ struct { ++ /* Aperture register balooning */ ++ struct { ++ u32 base; ++ u32 size; ++ } mappable_gmadr; /* aperture */ ++ /* GMADR register balooning */ ++ struct { ++ u32 base; ++ u32 size; ++ } nonmappable_gmadr; /* non aperture */ ++ /* allowed fence registers */ ++ u32 fence_num; ++ u32 rsv2[3]; ++ } avail_rs; /* available/assigned resource */ ++ u32 rsv3[0x200 - 24]; /* pad to half page */ ++ /* ++ * The bottom half page is for response from Gfx driver to hypervisor. ++ */ ++ u32 rsv4; ++ u32 display_ready; /* ready for display owner switch */ ++ ++ u32 rsv5[4]; ++ ++ u32 g2v_notify; ++ u32 rsv6[5]; ++ ++ u32 cursor_x_hot; ++ u32 cursor_y_hot; ++ ++ struct { ++ u32 lo; ++ u32 hi; ++ } pdp[4]; ++ ++ u32 execlist_context_descriptor_lo; ++ u32 execlist_context_descriptor_hi; ++ ++ u32 rsv7[0x200 - 24]; /* pad to one page */ ++} __packed; ++ ++#define vgtif_reg(x) \ ++ _MMIO((VGT_PVINFO_PAGE + offsetof(struct vgt_if, x))) ++ ++/* vGPU display status to be used by the host side */ ++#define VGT_DRV_DISPLAY_NOT_READY 0 ++#define VGT_DRV_DISPLAY_READY 1 /* ready for display switch */ ++ ++#endif /* _I915_PVINFO_H_ */ +diff --git a/drivers/gpu/drm/i915_legacy/i915_query.c b/drivers/gpu/drm/i915_legacy/i915_query.c +new file mode 100644 +index 000000000000..782183b78f49 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_query.c +@@ -0,0 +1,144 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ */ ++ ++#include ++ ++#include "i915_drv.h" ++#include "i915_query.h" ++#include ++ ++static int copy_query_item(void *query_hdr, size_t query_sz, ++ u32 total_length, ++ struct drm_i915_query_item *query_item) ++{ ++ if (query_item->length == 0) ++ return total_length; ++ ++ if (query_item->length < total_length) ++ return -EINVAL; ++ ++ if (copy_from_user(query_hdr, u64_to_user_ptr(query_item->data_ptr), ++ query_sz)) ++ return -EFAULT; ++ ++ if (!access_ok(u64_to_user_ptr(query_item->data_ptr), ++ total_length)) ++ return -EFAULT; ++ ++ return 0; ++} ++ ++static int query_topology_info(struct drm_i915_private *dev_priv, ++ struct drm_i915_query_item *query_item) ++{ ++ const struct sseu_dev_info *sseu = &RUNTIME_INFO(dev_priv)->sseu; ++ struct drm_i915_query_topology_info topo; ++ u32 slice_length, subslice_length, eu_length, total_length; ++ int ret; ++ ++ if (query_item->flags != 0) ++ return -EINVAL; ++ ++ if (sseu->max_slices == 0) ++ return -ENODEV; ++ ++ BUILD_BUG_ON(sizeof(u8) != sizeof(sseu->slice_mask)); ++ ++ slice_length = sizeof(sseu->slice_mask); ++ subslice_length = sseu->max_slices * ++ DIV_ROUND_UP(sseu->max_subslices, BITS_PER_BYTE); ++ eu_length = sseu->max_slices * sseu->max_subslices * ++ DIV_ROUND_UP(sseu->max_eus_per_subslice, BITS_PER_BYTE); ++ ++ total_length = sizeof(topo) + slice_length + subslice_length + eu_length; ++ ++ ret = copy_query_item(&topo, sizeof(topo), total_length, ++ query_item); ++ if (ret != 0) ++ return ret; ++ ++ if (topo.flags != 0) ++ return -EINVAL; ++ ++ memset(&topo, 0, sizeof(topo)); ++ topo.max_slices = sseu->max_slices; ++ topo.max_subslices = sseu->max_subslices; ++ topo.max_eus_per_subslice = sseu->max_eus_per_subslice; ++ ++ topo.subslice_offset = slice_length; ++ topo.subslice_stride = DIV_ROUND_UP(sseu->max_subslices, BITS_PER_BYTE); ++ topo.eu_offset = slice_length + subslice_length; ++ topo.eu_stride = ++ DIV_ROUND_UP(sseu->max_eus_per_subslice, BITS_PER_BYTE); ++ ++ if (__copy_to_user(u64_to_user_ptr(query_item->data_ptr), ++ &topo, sizeof(topo))) ++ return -EFAULT; ++ ++ if (__copy_to_user(u64_to_user_ptr(query_item->data_ptr + sizeof(topo)), ++ &sseu->slice_mask, slice_length)) ++ return -EFAULT; ++ ++ if (__copy_to_user(u64_to_user_ptr(query_item->data_ptr + ++ sizeof(topo) + slice_length), ++ sseu->subslice_mask, subslice_length)) ++ return -EFAULT; ++ ++ if (__copy_to_user(u64_to_user_ptr(query_item->data_ptr + ++ sizeof(topo) + ++ slice_length + subslice_length), ++ sseu->eu_mask, eu_length)) ++ return -EFAULT; ++ ++ return total_length; ++} ++ ++static int (* const i915_query_funcs[])(struct drm_i915_private *dev_priv, ++ struct drm_i915_query_item *query_item) = { ++ query_topology_info, ++}; ++ ++int i915_query_ioctl(struct drm_device *dev, void *data, struct drm_file *file) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct drm_i915_query *args = data; ++ struct drm_i915_query_item __user *user_item_ptr = ++ u64_to_user_ptr(args->items_ptr); ++ u32 i; ++ ++ if (args->flags != 0) ++ return -EINVAL; ++ ++ for (i = 0; i < args->num_items; i++, user_item_ptr++) { ++ struct drm_i915_query_item item; ++ unsigned long func_idx; ++ int ret; ++ ++ if (copy_from_user(&item, user_item_ptr, sizeof(item))) ++ return -EFAULT; ++ ++ if (item.query_id == 0) ++ return -EINVAL; ++ ++ if (overflows_type(item.query_id - 1, unsigned long)) ++ return -EINVAL; ++ ++ func_idx = item.query_id - 1; ++ ++ ret = -EINVAL; ++ if (func_idx < ARRAY_SIZE(i915_query_funcs)) { ++ func_idx = array_index_nospec(func_idx, ++ ARRAY_SIZE(i915_query_funcs)); ++ ret = i915_query_funcs[func_idx](dev_priv, &item); ++ } ++ ++ /* Only write the length back to userspace if they differ. */ ++ if (ret != item.length && put_user(ret, &user_item_ptr->length)) ++ return -EFAULT; ++ } ++ ++ return 0; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_query.h b/drivers/gpu/drm/i915_legacy/i915_query.h +new file mode 100644 +index 000000000000..31dcef181f63 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_query.h +@@ -0,0 +1,15 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ */ ++ ++#ifndef _I915_QUERY_H_ ++#define _I915_QUERY_H_ ++ ++struct drm_device; ++struct drm_file; ++ ++int i915_query_ioctl(struct drm_device *dev, void *data, struct drm_file *file); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_reg.h b/drivers/gpu/drm/i915_legacy/i915_reg.h +new file mode 100644 +index 000000000000..cf748b80e640 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_reg.h +@@ -0,0 +1,11424 @@ ++/* Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. ++ * All Rights Reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the ++ * "Software"), to deal in the Software without restriction, including ++ * without limitation the rights to use, copy, modify, merge, publish, ++ * distribute, sub license, and/or sell copies of the Software, and to ++ * permit persons to whom the Software is furnished to do so, subject to ++ * the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the ++ * next paragraph) shall be included in all copies or substantial portions ++ * of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS ++ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF ++ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. ++ * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR ++ * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, ++ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE ++ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++#ifndef _I915_REG_H_ ++#define _I915_REG_H_ ++ ++#include ++#include ++ ++/** ++ * DOC: The i915 register macro definition style guide ++ * ++ * Follow the style described here for new macros, and while changing existing ++ * macros. Do **not** mass change existing definitions just to update the style. ++ * ++ * Layout ++ * ~~~~~~ ++ * ++ * Keep helper macros near the top. For example, _PIPE() and friends. ++ * ++ * Prefix macros that generally should not be used outside of this file with ++ * underscore '_'. For example, _PIPE() and friends, single instances of ++ * registers that are defined solely for the use by function-like macros. ++ * ++ * Avoid using the underscore prefixed macros outside of this file. There are ++ * exceptions, but keep them to a minimum. ++ * ++ * There are two basic types of register definitions: Single registers and ++ * register groups. Register groups are registers which have two or more ++ * instances, for example one per pipe, port, transcoder, etc. Register groups ++ * should be defined using function-like macros. ++ * ++ * For single registers, define the register offset first, followed by register ++ * contents. ++ * ++ * For register groups, define the register instance offsets first, prefixed ++ * with underscore, followed by a function-like macro choosing the right ++ * instance based on the parameter, followed by register contents. ++ * ++ * Define the register contents (i.e. bit and bit field macros) from most ++ * significant to least significant bit. Indent the register content macros ++ * using two extra spaces between ``#define`` and the macro name. ++ * ++ * Define bit fields using ``REG_GENMASK(h, l)``. Define bit field contents ++ * using ``REG_FIELD_PREP(mask, value)``. This will define the values already ++ * shifted in place, so they can be directly OR'd together. For convenience, ++ * function-like macros may be used to define bit fields, but do note that the ++ * macros may be needed to read as well as write the register contents. ++ * ++ * Define bits using ``REG_BIT(N)``. Do **not** add ``_BIT`` suffix to the name. ++ * ++ * Group the register and its contents together without blank lines, separate ++ * from other registers and their contents with one blank line. ++ * ++ * Indent macro values from macro names using TABs. Align values vertically. Use ++ * braces in macro values as needed to avoid unintended precedence after macro ++ * substitution. Use spaces in macro values according to kernel coding ++ * style. Use lower case in hexadecimal values. ++ * ++ * Naming ++ * ~~~~~~ ++ * ++ * Try to name registers according to the specs. If the register name changes in ++ * the specs from platform to another, stick to the original name. ++ * ++ * Try to re-use existing register macro definitions. Only add new macros for ++ * new register offsets, or when the register contents have changed enough to ++ * warrant a full redefinition. ++ * ++ * When a register macro changes for a new platform, prefix the new macro using ++ * the platform acronym or generation. For example, ``SKL_`` or ``GEN8_``. The ++ * prefix signifies the start platform/generation using the register. ++ * ++ * When a bit (field) macro changes or gets added for a new platform, while ++ * retaining the existing register macro, add a platform acronym or generation ++ * suffix to the name. For example, ``_SKL`` or ``_GEN8``. ++ * ++ * Examples ++ * ~~~~~~~~ ++ * ++ * (Note that the values in the example are indented using spaces instead of ++ * TABs to avoid misalignment in generated documentation. Use TABs in the ++ * definitions.):: ++ * ++ * #define _FOO_A 0xf000 ++ * #define _FOO_B 0xf001 ++ * #define FOO(pipe) _MMIO_PIPE(pipe, _FOO_A, _FOO_B) ++ * #define FOO_ENABLE REG_BIT(31) ++ * #define FOO_MODE_MASK REG_GENMASK(19, 16) ++ * #define FOO_MODE_BAR REG_FIELD_PREP(FOO_MODE_MASK, 0) ++ * #define FOO_MODE_BAZ REG_FIELD_PREP(FOO_MODE_MASK, 1) ++ * #define FOO_MODE_QUX_SNB REG_FIELD_PREP(FOO_MODE_MASK, 2) ++ * ++ * #define BAR _MMIO(0xb000) ++ * #define GEN8_BAR _MMIO(0xb888) ++ */ ++ ++/** ++ * REG_BIT() - Prepare a u32 bit value ++ * @__n: 0-based bit number ++ * ++ * Local wrapper for BIT() to force u32, with compile time checks. ++ * ++ * @return: Value with bit @__n set. ++ */ ++#define REG_BIT(__n) \ ++ ((u32)(BIT(__n) + \ ++ BUILD_BUG_ON_ZERO(__builtin_constant_p(__n) && \ ++ ((__n) < 0 || (__n) > 31)))) ++ ++/** ++ * REG_GENMASK() - Prepare a continuous u32 bitmask ++ * @__high: 0-based high bit ++ * @__low: 0-based low bit ++ * ++ * Local wrapper for GENMASK() to force u32, with compile time checks. ++ * ++ * @return: Continuous bitmask from @__high to @__low, inclusive. ++ */ ++#define REG_GENMASK(__high, __low) \ ++ ((u32)(GENMASK(__high, __low) + \ ++ BUILD_BUG_ON_ZERO(__builtin_constant_p(__high) && \ ++ __builtin_constant_p(__low) && \ ++ ((__low) < 0 || (__high) > 31 || (__low) > (__high))))) ++ ++/* ++ * Local integer constant expression version of is_power_of_2(). ++ */ ++#define IS_POWER_OF_2(__x) ((__x) && (((__x) & ((__x) - 1)) == 0)) ++ ++/** ++ * REG_FIELD_PREP() - Prepare a u32 bitfield value ++ * @__mask: shifted mask defining the field's length and position ++ * @__val: value to put in the field ++ ++ * Local copy of FIELD_PREP() to generate an integer constant expression, force ++ * u32 and for consistency with REG_FIELD_GET(), REG_BIT() and REG_GENMASK(). ++ * ++ * @return: @__val masked and shifted into the field defined by @__mask. ++ */ ++#define REG_FIELD_PREP(__mask, __val) \ ++ ((u32)((((typeof(__mask))(__val) << __bf_shf(__mask)) & (__mask)) + \ ++ BUILD_BUG_ON_ZERO(!__is_constexpr(__mask)) + \ ++ BUILD_BUG_ON_ZERO((__mask) == 0 || (__mask) > U32_MAX) + \ ++ BUILD_BUG_ON_ZERO(!IS_POWER_OF_2((__mask) + (1ULL << __bf_shf(__mask)))) + \ ++ BUILD_BUG_ON_ZERO(__builtin_choose_expr(__is_constexpr(__val), (~((__mask) >> __bf_shf(__mask)) & (__val)), 0)))) ++ ++/** ++ * REG_FIELD_GET() - Extract a u32 bitfield value ++ * @__mask: shifted mask defining the field's length and position ++ * @__val: value to extract the bitfield value from ++ * ++ * Local wrapper for FIELD_GET() to force u32 and for consistency with ++ * REG_FIELD_PREP(), REG_BIT() and REG_GENMASK(). ++ * ++ * @return: Masked and shifted value of the field defined by @__mask in @__val. ++ */ ++#define REG_FIELD_GET(__mask, __val) ((u32)FIELD_GET(__mask, __val)) ++ ++typedef struct { ++ u32 reg; ++} i915_reg_t; ++ ++#define _MMIO(r) ((const i915_reg_t){ .reg = (r) }) ++ ++#define INVALID_MMIO_REG _MMIO(0) ++ ++static inline u32 i915_mmio_reg_offset(i915_reg_t reg) ++{ ++ return reg.reg; ++} ++ ++static inline bool i915_mmio_reg_equal(i915_reg_t a, i915_reg_t b) ++{ ++ return i915_mmio_reg_offset(a) == i915_mmio_reg_offset(b); ++} ++ ++static inline bool i915_mmio_reg_valid(i915_reg_t reg) ++{ ++ return !i915_mmio_reg_equal(reg, INVALID_MMIO_REG); ++} ++ ++#define VLV_DISPLAY_BASE 0x180000 ++#define VLV_MIPI_BASE VLV_DISPLAY_BASE ++#define BXT_MIPI_BASE 0x60000 ++ ++#define DISPLAY_MMIO_BASE(dev_priv) (INTEL_INFO(dev_priv)->display_mmio_offset) ++ ++/* ++ * Given the first two numbers __a and __b of arbitrarily many evenly spaced ++ * numbers, pick the 0-based __index'th value. ++ * ++ * Always prefer this over _PICK() if the numbers are evenly spaced. ++ */ ++#define _PICK_EVEN(__index, __a, __b) ((__a) + (__index) * ((__b) - (__a))) ++ ++/* ++ * Given the arbitrary numbers in varargs, pick the 0-based __index'th number. ++ * ++ * Always prefer _PICK_EVEN() over this if the numbers are evenly spaced. ++ */ ++#define _PICK(__index, ...) (((const u32 []){ __VA_ARGS__ })[__index]) ++ ++/* ++ * Named helper wrappers around _PICK_EVEN() and _PICK(). ++ */ ++#define _PIPE(pipe, a, b) _PICK_EVEN(pipe, a, b) ++#define _PLANE(plane, a, b) _PICK_EVEN(plane, a, b) ++#define _TRANS(tran, a, b) _PICK_EVEN(tran, a, b) ++#define _PORT(port, a, b) _PICK_EVEN(port, a, b) ++#define _PLL(pll, a, b) _PICK_EVEN(pll, a, b) ++ ++#define _MMIO_PIPE(pipe, a, b) _MMIO(_PIPE(pipe, a, b)) ++#define _MMIO_PLANE(plane, a, b) _MMIO(_PLANE(plane, a, b)) ++#define _MMIO_TRANS(tran, a, b) _MMIO(_TRANS(tran, a, b)) ++#define _MMIO_PORT(port, a, b) _MMIO(_PORT(port, a, b)) ++#define _MMIO_PLL(pll, a, b) _MMIO(_PLL(pll, a, b)) ++ ++#define _PHY3(phy, ...) _PICK(phy, __VA_ARGS__) ++ ++#define _MMIO_PIPE3(pipe, a, b, c) _MMIO(_PICK(pipe, a, b, c)) ++#define _MMIO_PORT3(pipe, a, b, c) _MMIO(_PICK(pipe, a, b, c)) ++#define _MMIO_PHY3(phy, a, b, c) _MMIO(_PHY3(phy, a, b, c)) ++ ++/* ++ * Device info offset array based helpers for groups of registers with unevenly ++ * spaced base offsets. ++ */ ++#define _MMIO_PIPE2(pipe, reg) _MMIO(INTEL_INFO(dev_priv)->pipe_offsets[pipe] - \ ++ INTEL_INFO(dev_priv)->pipe_offsets[PIPE_A] + (reg) + \ ++ DISPLAY_MMIO_BASE(dev_priv)) ++#define _MMIO_TRANS2(pipe, reg) _MMIO(INTEL_INFO(dev_priv)->trans_offsets[(pipe)] - \ ++ INTEL_INFO(dev_priv)->trans_offsets[TRANSCODER_A] + (reg) + \ ++ DISPLAY_MMIO_BASE(dev_priv)) ++#define _CURSOR2(pipe, reg) _MMIO(INTEL_INFO(dev_priv)->cursor_offsets[(pipe)] - \ ++ INTEL_INFO(dev_priv)->cursor_offsets[PIPE_A] + (reg) + \ ++ DISPLAY_MMIO_BASE(dev_priv)) ++ ++#define __MASKED_FIELD(mask, value) ((mask) << 16 | (value)) ++#define _MASKED_FIELD(mask, value) ({ \ ++ if (__builtin_constant_p(mask)) \ ++ BUILD_BUG_ON_MSG(((mask) & 0xffff0000), "Incorrect mask"); \ ++ if (__builtin_constant_p(value)) \ ++ BUILD_BUG_ON_MSG((value) & 0xffff0000, "Incorrect value"); \ ++ if (__builtin_constant_p(mask) && __builtin_constant_p(value)) \ ++ BUILD_BUG_ON_MSG((value) & ~(mask), \ ++ "Incorrect value for mask"); \ ++ __MASKED_FIELD(mask, value); }) ++#define _MASKED_BIT_ENABLE(a) ({ typeof(a) _a = (a); _MASKED_FIELD(_a, _a); }) ++#define _MASKED_BIT_DISABLE(a) (_MASKED_FIELD((a), 0)) ++ ++/* Engine ID */ ++ ++#define RCS0_HW 0 ++#define VCS0_HW 1 ++#define BCS0_HW 2 ++#define VECS0_HW 3 ++#define VCS1_HW 4 ++#define VCS2_HW 6 ++#define VCS3_HW 7 ++#define VECS1_HW 12 ++ ++/* Engine class */ ++ ++#define RENDER_CLASS 0 ++#define VIDEO_DECODE_CLASS 1 ++#define VIDEO_ENHANCEMENT_CLASS 2 ++#define COPY_ENGINE_CLASS 3 ++#define OTHER_CLASS 4 ++#define MAX_ENGINE_CLASS 4 ++ ++#define OTHER_GTPM_INSTANCE 1 ++#define MAX_ENGINE_INSTANCE 3 ++ ++/* PCI config space */ ++ ++#define MCHBAR_I915 0x44 ++#define MCHBAR_I965 0x48 ++#define MCHBAR_SIZE (4 * 4096) ++ ++#define DEVEN 0x54 ++#define DEVEN_MCHBAR_EN (1 << 28) ++ ++/* BSM in include/drm/i915_drm.h */ ++ ++#define HPLLCC 0xc0 /* 85x only */ ++#define GC_CLOCK_CONTROL_MASK (0x7 << 0) ++#define GC_CLOCK_133_200 (0 << 0) ++#define GC_CLOCK_100_200 (1 << 0) ++#define GC_CLOCK_100_133 (2 << 0) ++#define GC_CLOCK_133_266 (3 << 0) ++#define GC_CLOCK_133_200_2 (4 << 0) ++#define GC_CLOCK_133_266_2 (5 << 0) ++#define GC_CLOCK_166_266 (6 << 0) ++#define GC_CLOCK_166_250 (7 << 0) ++ ++#define I915_GDRST 0xc0 /* PCI config register */ ++#define GRDOM_FULL (0 << 2) ++#define GRDOM_RENDER (1 << 2) ++#define GRDOM_MEDIA (3 << 2) ++#define GRDOM_MASK (3 << 2) ++#define GRDOM_RESET_STATUS (1 << 1) ++#define GRDOM_RESET_ENABLE (1 << 0) ++ ++/* BSpec only has register offset, PCI device and bit found empirically */ ++#define I830_CLOCK_GATE 0xc8 /* device 0 */ ++#define I830_L2_CACHE_CLOCK_GATE_DISABLE (1 << 2) ++ ++#define GCDGMBUS 0xcc ++ ++#define GCFGC2 0xda ++#define GCFGC 0xf0 /* 915+ only */ ++#define GC_LOW_FREQUENCY_ENABLE (1 << 7) ++#define GC_DISPLAY_CLOCK_190_200_MHZ (0 << 4) ++#define GC_DISPLAY_CLOCK_333_320_MHZ (4 << 4) ++#define GC_DISPLAY_CLOCK_267_MHZ_PNV (0 << 4) ++#define GC_DISPLAY_CLOCK_333_MHZ_PNV (1 << 4) ++#define GC_DISPLAY_CLOCK_444_MHZ_PNV (2 << 4) ++#define GC_DISPLAY_CLOCK_200_MHZ_PNV (5 << 4) ++#define GC_DISPLAY_CLOCK_133_MHZ_PNV (6 << 4) ++#define GC_DISPLAY_CLOCK_167_MHZ_PNV (7 << 4) ++#define GC_DISPLAY_CLOCK_MASK (7 << 4) ++#define GM45_GC_RENDER_CLOCK_MASK (0xf << 0) ++#define GM45_GC_RENDER_CLOCK_266_MHZ (8 << 0) ++#define GM45_GC_RENDER_CLOCK_320_MHZ (9 << 0) ++#define GM45_GC_RENDER_CLOCK_400_MHZ (0xb << 0) ++#define GM45_GC_RENDER_CLOCK_533_MHZ (0xc << 0) ++#define I965_GC_RENDER_CLOCK_MASK (0xf << 0) ++#define I965_GC_RENDER_CLOCK_267_MHZ (2 << 0) ++#define I965_GC_RENDER_CLOCK_333_MHZ (3 << 0) ++#define I965_GC_RENDER_CLOCK_444_MHZ (4 << 0) ++#define I965_GC_RENDER_CLOCK_533_MHZ (5 << 0) ++#define I945_GC_RENDER_CLOCK_MASK (7 << 0) ++#define I945_GC_RENDER_CLOCK_166_MHZ (0 << 0) ++#define I945_GC_RENDER_CLOCK_200_MHZ (1 << 0) ++#define I945_GC_RENDER_CLOCK_250_MHZ (3 << 0) ++#define I945_GC_RENDER_CLOCK_400_MHZ (5 << 0) ++#define I915_GC_RENDER_CLOCK_MASK (7 << 0) ++#define I915_GC_RENDER_CLOCK_166_MHZ (0 << 0) ++#define I915_GC_RENDER_CLOCK_200_MHZ (1 << 0) ++#define I915_GC_RENDER_CLOCK_333_MHZ (4 << 0) ++ ++#define ASLE 0xe4 ++#define ASLS 0xfc ++ ++#define SWSCI 0xe8 ++#define SWSCI_SCISEL (1 << 15) ++#define SWSCI_GSSCIE (1 << 0) ++ ++#define LBPC 0xf4 /* legacy/combination backlight modes, also called LBB */ ++ ++ ++#define ILK_GDSR _MMIO(MCHBAR_MIRROR_BASE + 0x2ca4) ++#define ILK_GRDOM_FULL (0 << 1) ++#define ILK_GRDOM_RENDER (1 << 1) ++#define ILK_GRDOM_MEDIA (3 << 1) ++#define ILK_GRDOM_MASK (3 << 1) ++#define ILK_GRDOM_RESET_ENABLE (1 << 0) ++ ++#define GEN6_MBCUNIT_SNPCR _MMIO(0x900c) /* for LLC config */ ++#define GEN6_MBC_SNPCR_SHIFT 21 ++#define GEN6_MBC_SNPCR_MASK (3 << 21) ++#define GEN6_MBC_SNPCR_MAX (0 << 21) ++#define GEN6_MBC_SNPCR_MED (1 << 21) ++#define GEN6_MBC_SNPCR_LOW (2 << 21) ++#define GEN6_MBC_SNPCR_MIN (3 << 21) /* only 1/16th of the cache is shared */ ++ ++#define VLV_G3DCTL _MMIO(0x9024) ++#define VLV_GSCKGCTL _MMIO(0x9028) ++ ++#define GEN6_MBCTL _MMIO(0x0907c) ++#define GEN6_MBCTL_ENABLE_BOOT_FETCH (1 << 4) ++#define GEN6_MBCTL_CTX_FETCH_NEEDED (1 << 3) ++#define GEN6_MBCTL_BME_UPDATE_ENABLE (1 << 2) ++#define GEN6_MBCTL_MAE_UPDATE_ENABLE (1 << 1) ++#define GEN6_MBCTL_BOOT_FETCH_MECH (1 << 0) ++ ++#define GEN6_GDRST _MMIO(0x941c) ++#define GEN6_GRDOM_FULL (1 << 0) ++#define GEN6_GRDOM_RENDER (1 << 1) ++#define GEN6_GRDOM_MEDIA (1 << 2) ++#define GEN6_GRDOM_BLT (1 << 3) ++#define GEN6_GRDOM_VECS (1 << 4) ++#define GEN9_GRDOM_GUC (1 << 5) ++#define GEN8_GRDOM_MEDIA2 (1 << 7) ++/* GEN11 changed all bit defs except for FULL & RENDER */ ++#define GEN11_GRDOM_FULL GEN6_GRDOM_FULL ++#define GEN11_GRDOM_RENDER GEN6_GRDOM_RENDER ++#define GEN11_GRDOM_BLT (1 << 2) ++#define GEN11_GRDOM_GUC (1 << 3) ++#define GEN11_GRDOM_MEDIA (1 << 5) ++#define GEN11_GRDOM_MEDIA2 (1 << 6) ++#define GEN11_GRDOM_MEDIA3 (1 << 7) ++#define GEN11_GRDOM_MEDIA4 (1 << 8) ++#define GEN11_GRDOM_VECS (1 << 13) ++#define GEN11_GRDOM_VECS2 (1 << 14) ++#define GEN11_GRDOM_SFC0 (1 << 17) ++#define GEN11_GRDOM_SFC1 (1 << 18) ++ ++#define GEN11_VCS_SFC_RESET_BIT(instance) (GEN11_GRDOM_SFC0 << ((instance) >> 1)) ++#define GEN11_VECS_SFC_RESET_BIT(instance) (GEN11_GRDOM_SFC0 << (instance)) ++ ++#define GEN11_VCS_SFC_FORCED_LOCK(engine) _MMIO((engine)->mmio_base + 0x88C) ++#define GEN11_VCS_SFC_FORCED_LOCK_BIT (1 << 0) ++#define GEN11_VCS_SFC_LOCK_STATUS(engine) _MMIO((engine)->mmio_base + 0x890) ++#define GEN11_VCS_SFC_USAGE_BIT (1 << 0) ++#define GEN11_VCS_SFC_LOCK_ACK_BIT (1 << 1) ++ ++#define GEN11_VECS_SFC_FORCED_LOCK(engine) _MMIO((engine)->mmio_base + 0x201C) ++#define GEN11_VECS_SFC_FORCED_LOCK_BIT (1 << 0) ++#define GEN11_VECS_SFC_LOCK_ACK(engine) _MMIO((engine)->mmio_base + 0x2018) ++#define GEN11_VECS_SFC_LOCK_ACK_BIT (1 << 0) ++#define GEN11_VECS_SFC_USAGE(engine) _MMIO((engine)->mmio_base + 0x2014) ++#define GEN11_VECS_SFC_USAGE_BIT (1 << 0) ++ ++#define RING_PP_DIR_BASE(base) _MMIO((base) + 0x228) ++#define RING_PP_DIR_BASE_READ(base) _MMIO((base) + 0x518) ++#define RING_PP_DIR_DCLV(base) _MMIO((base) + 0x220) ++#define PP_DIR_DCLV_2G 0xffffffff ++ ++#define GEN8_RING_PDP_UDW(base, n) _MMIO((base) + 0x270 + (n) * 8 + 4) ++#define GEN8_RING_PDP_LDW(base, n) _MMIO((base) + 0x270 + (n) * 8) ++ ++#define GEN8_R_PWR_CLK_STATE _MMIO(0x20C8) ++#define GEN8_RPCS_ENABLE (1 << 31) ++#define GEN8_RPCS_S_CNT_ENABLE (1 << 18) ++#define GEN8_RPCS_S_CNT_SHIFT 15 ++#define GEN8_RPCS_S_CNT_MASK (0x7 << GEN8_RPCS_S_CNT_SHIFT) ++#define GEN11_RPCS_S_CNT_SHIFT 12 ++#define GEN11_RPCS_S_CNT_MASK (0x3f << GEN11_RPCS_S_CNT_SHIFT) ++#define GEN8_RPCS_SS_CNT_ENABLE (1 << 11) ++#define GEN8_RPCS_SS_CNT_SHIFT 8 ++#define GEN8_RPCS_SS_CNT_MASK (0x7 << GEN8_RPCS_SS_CNT_SHIFT) ++#define GEN8_RPCS_EU_MAX_SHIFT 4 ++#define GEN8_RPCS_EU_MAX_MASK (0xf << GEN8_RPCS_EU_MAX_SHIFT) ++#define GEN8_RPCS_EU_MIN_SHIFT 0 ++#define GEN8_RPCS_EU_MIN_MASK (0xf << GEN8_RPCS_EU_MIN_SHIFT) ++ ++#define WAIT_FOR_RC6_EXIT _MMIO(0x20CC) ++/* HSW only */ ++#define HSW_SELECTIVE_READ_ADDRESSING_SHIFT 2 ++#define HSW_SELECTIVE_READ_ADDRESSING_MASK (0x3 << HSW_SLECTIVE_READ_ADDRESSING_SHIFT) ++#define HSW_SELECTIVE_WRITE_ADDRESS_SHIFT 4 ++#define HSW_SELECTIVE_WRITE_ADDRESS_MASK (0x7 << HSW_SELECTIVE_WRITE_ADDRESS_SHIFT) ++/* HSW+ */ ++#define HSW_WAIT_FOR_RC6_EXIT_ENABLE (1 << 0) ++#define HSW_RCS_CONTEXT_ENABLE (1 << 7) ++#define HSW_RCS_INHIBIT (1 << 8) ++/* Gen8 */ ++#define GEN8_SELECTIVE_WRITE_ADDRESS_SHIFT 4 ++#define GEN8_SELECTIVE_WRITE_ADDRESS_MASK (0x3 << GEN8_SELECTIVE_WRITE_ADDRESS_SHIFT) ++#define GEN8_SELECTIVE_WRITE_ADDRESS_SHIFT 4 ++#define GEN8_SELECTIVE_WRITE_ADDRESS_MASK (0x3 << GEN8_SELECTIVE_WRITE_ADDRESS_SHIFT) ++#define GEN8_SELECTIVE_WRITE_ADDRESSING_ENABLE (1 << 6) ++#define GEN8_SELECTIVE_READ_SUBSLICE_SELECT_SHIFT 9 ++#define GEN8_SELECTIVE_READ_SUBSLICE_SELECT_MASK (0x3 << GEN8_SELECTIVE_READ_SUBSLICE_SELECT_SHIFT) ++#define GEN8_SELECTIVE_READ_SLICE_SELECT_SHIFT 11 ++#define GEN8_SELECTIVE_READ_SLICE_SELECT_MASK (0x3 << GEN8_SELECTIVE_READ_SLICE_SELECT_SHIFT) ++#define GEN8_SELECTIVE_READ_ADDRESSING_ENABLE (1 << 13) ++ ++#define GAM_ECOCHK _MMIO(0x4090) ++#define BDW_DISABLE_HDC_INVALIDATION (1 << 25) ++#define ECOCHK_SNB_BIT (1 << 10) ++#define ECOCHK_DIS_TLB (1 << 8) ++#define HSW_ECOCHK_ARB_PRIO_SOL (1 << 6) ++#define ECOCHK_PPGTT_CACHE64B (0x3 << 3) ++#define ECOCHK_PPGTT_CACHE4B (0x0 << 3) ++#define ECOCHK_PPGTT_GFDT_IVB (0x1 << 4) ++#define ECOCHK_PPGTT_LLC_IVB (0x1 << 3) ++#define ECOCHK_PPGTT_UC_HSW (0x1 << 3) ++#define ECOCHK_PPGTT_WT_HSW (0x2 << 3) ++#define ECOCHK_PPGTT_WB_HSW (0x3 << 3) ++ ++#define GAC_ECO_BITS _MMIO(0x14090) ++#define ECOBITS_SNB_BIT (1 << 13) ++#define ECOBITS_PPGTT_CACHE64B (3 << 8) ++#define ECOBITS_PPGTT_CACHE4B (0 << 8) ++ ++#define GAB_CTL _MMIO(0x24000) ++#define GAB_CTL_CONT_AFTER_PAGEFAULT (1 << 8) ++ ++#define GEN6_STOLEN_RESERVED _MMIO(0x1082C0) ++#define GEN6_STOLEN_RESERVED_ADDR_MASK (0xFFF << 20) ++#define GEN7_STOLEN_RESERVED_ADDR_MASK (0x3FFF << 18) ++#define GEN6_STOLEN_RESERVED_SIZE_MASK (3 << 4) ++#define GEN6_STOLEN_RESERVED_1M (0 << 4) ++#define GEN6_STOLEN_RESERVED_512K (1 << 4) ++#define GEN6_STOLEN_RESERVED_256K (2 << 4) ++#define GEN6_STOLEN_RESERVED_128K (3 << 4) ++#define GEN7_STOLEN_RESERVED_SIZE_MASK (1 << 5) ++#define GEN7_STOLEN_RESERVED_1M (0 << 5) ++#define GEN7_STOLEN_RESERVED_256K (1 << 5) ++#define GEN8_STOLEN_RESERVED_SIZE_MASK (3 << 7) ++#define GEN8_STOLEN_RESERVED_1M (0 << 7) ++#define GEN8_STOLEN_RESERVED_2M (1 << 7) ++#define GEN8_STOLEN_RESERVED_4M (2 << 7) ++#define GEN8_STOLEN_RESERVED_8M (3 << 7) ++#define GEN6_STOLEN_RESERVED_ENABLE (1 << 0) ++#define GEN11_STOLEN_RESERVED_ADDR_MASK (0xFFFFFFFFFFFULL << 20) ++ ++/* VGA stuff */ ++ ++#define VGA_ST01_MDA 0x3ba ++#define VGA_ST01_CGA 0x3da ++ ++#define _VGA_MSR_WRITE _MMIO(0x3c2) ++#define VGA_MSR_WRITE 0x3c2 ++#define VGA_MSR_READ 0x3cc ++#define VGA_MSR_MEM_EN (1 << 1) ++#define VGA_MSR_CGA_MODE (1 << 0) ++ ++#define VGA_SR_INDEX 0x3c4 ++#define SR01 1 ++#define VGA_SR_DATA 0x3c5 ++ ++#define VGA_AR_INDEX 0x3c0 ++#define VGA_AR_VID_EN (1 << 5) ++#define VGA_AR_DATA_WRITE 0x3c0 ++#define VGA_AR_DATA_READ 0x3c1 ++ ++#define VGA_GR_INDEX 0x3ce ++#define VGA_GR_DATA 0x3cf ++/* GR05 */ ++#define VGA_GR_MEM_READ_MODE_SHIFT 3 ++#define VGA_GR_MEM_READ_MODE_PLANE 1 ++/* GR06 */ ++#define VGA_GR_MEM_MODE_MASK 0xc ++#define VGA_GR_MEM_MODE_SHIFT 2 ++#define VGA_GR_MEM_A0000_AFFFF 0 ++#define VGA_GR_MEM_A0000_BFFFF 1 ++#define VGA_GR_MEM_B0000_B7FFF 2 ++#define VGA_GR_MEM_B0000_BFFFF 3 ++ ++#define VGA_DACMASK 0x3c6 ++#define VGA_DACRX 0x3c7 ++#define VGA_DACWX 0x3c8 ++#define VGA_DACDATA 0x3c9 ++ ++#define VGA_CR_INDEX_MDA 0x3b4 ++#define VGA_CR_DATA_MDA 0x3b5 ++#define VGA_CR_INDEX_CGA 0x3d4 ++#define VGA_CR_DATA_CGA 0x3d5 ++ ++#define MI_PREDICATE_SRC0 _MMIO(0x2400) ++#define MI_PREDICATE_SRC0_UDW _MMIO(0x2400 + 4) ++#define MI_PREDICATE_SRC1 _MMIO(0x2408) ++#define MI_PREDICATE_SRC1_UDW _MMIO(0x2408 + 4) ++ ++#define MI_PREDICATE_RESULT_2 _MMIO(0x2214) ++#define LOWER_SLICE_ENABLED (1 << 0) ++#define LOWER_SLICE_DISABLED (0 << 0) ++ ++/* ++ * Registers used only by the command parser ++ */ ++#define BCS_SWCTRL _MMIO(0x22200) ++ ++#define GPGPU_THREADS_DISPATCHED _MMIO(0x2290) ++#define GPGPU_THREADS_DISPATCHED_UDW _MMIO(0x2290 + 4) ++#define HS_INVOCATION_COUNT _MMIO(0x2300) ++#define HS_INVOCATION_COUNT_UDW _MMIO(0x2300 + 4) ++#define DS_INVOCATION_COUNT _MMIO(0x2308) ++#define DS_INVOCATION_COUNT_UDW _MMIO(0x2308 + 4) ++#define IA_VERTICES_COUNT _MMIO(0x2310) ++#define IA_VERTICES_COUNT_UDW _MMIO(0x2310 + 4) ++#define IA_PRIMITIVES_COUNT _MMIO(0x2318) ++#define IA_PRIMITIVES_COUNT_UDW _MMIO(0x2318 + 4) ++#define VS_INVOCATION_COUNT _MMIO(0x2320) ++#define VS_INVOCATION_COUNT_UDW _MMIO(0x2320 + 4) ++#define GS_INVOCATION_COUNT _MMIO(0x2328) ++#define GS_INVOCATION_COUNT_UDW _MMIO(0x2328 + 4) ++#define GS_PRIMITIVES_COUNT _MMIO(0x2330) ++#define GS_PRIMITIVES_COUNT_UDW _MMIO(0x2330 + 4) ++#define CL_INVOCATION_COUNT _MMIO(0x2338) ++#define CL_INVOCATION_COUNT_UDW _MMIO(0x2338 + 4) ++#define CL_PRIMITIVES_COUNT _MMIO(0x2340) ++#define CL_PRIMITIVES_COUNT_UDW _MMIO(0x2340 + 4) ++#define PS_INVOCATION_COUNT _MMIO(0x2348) ++#define PS_INVOCATION_COUNT_UDW _MMIO(0x2348 + 4) ++#define PS_DEPTH_COUNT _MMIO(0x2350) ++#define PS_DEPTH_COUNT_UDW _MMIO(0x2350 + 4) ++ ++/* There are the 4 64-bit counter registers, one for each stream output */ ++#define GEN7_SO_NUM_PRIMS_WRITTEN(n) _MMIO(0x5200 + (n) * 8) ++#define GEN7_SO_NUM_PRIMS_WRITTEN_UDW(n) _MMIO(0x5200 + (n) * 8 + 4) ++ ++#define GEN7_SO_PRIM_STORAGE_NEEDED(n) _MMIO(0x5240 + (n) * 8) ++#define GEN7_SO_PRIM_STORAGE_NEEDED_UDW(n) _MMIO(0x5240 + (n) * 8 + 4) ++ ++#define GEN7_3DPRIM_END_OFFSET _MMIO(0x2420) ++#define GEN7_3DPRIM_START_VERTEX _MMIO(0x2430) ++#define GEN7_3DPRIM_VERTEX_COUNT _MMIO(0x2434) ++#define GEN7_3DPRIM_INSTANCE_COUNT _MMIO(0x2438) ++#define GEN7_3DPRIM_START_INSTANCE _MMIO(0x243C) ++#define GEN7_3DPRIM_BASE_VERTEX _MMIO(0x2440) ++ ++#define GEN7_GPGPU_DISPATCHDIMX _MMIO(0x2500) ++#define GEN7_GPGPU_DISPATCHDIMY _MMIO(0x2504) ++#define GEN7_GPGPU_DISPATCHDIMZ _MMIO(0x2508) ++ ++/* There are the 16 64-bit CS General Purpose Registers */ ++#define HSW_CS_GPR(n) _MMIO(0x2600 + (n) * 8) ++#define HSW_CS_GPR_UDW(n) _MMIO(0x2600 + (n) * 8 + 4) ++ ++#define GEN7_OACONTROL _MMIO(0x2360) ++#define GEN7_OACONTROL_CTX_MASK 0xFFFFF000 ++#define GEN7_OACONTROL_TIMER_PERIOD_MASK 0x3F ++#define GEN7_OACONTROL_TIMER_PERIOD_SHIFT 6 ++#define GEN7_OACONTROL_TIMER_ENABLE (1 << 5) ++#define GEN7_OACONTROL_FORMAT_A13 (0 << 2) ++#define GEN7_OACONTROL_FORMAT_A29 (1 << 2) ++#define GEN7_OACONTROL_FORMAT_A13_B8_C8 (2 << 2) ++#define GEN7_OACONTROL_FORMAT_A29_B8_C8 (3 << 2) ++#define GEN7_OACONTROL_FORMAT_B4_C8 (4 << 2) ++#define GEN7_OACONTROL_FORMAT_A45_B8_C8 (5 << 2) ++#define GEN7_OACONTROL_FORMAT_B4_C8_A16 (6 << 2) ++#define GEN7_OACONTROL_FORMAT_C4_B8 (7 << 2) ++#define GEN7_OACONTROL_FORMAT_SHIFT 2 ++#define GEN7_OACONTROL_PER_CTX_ENABLE (1 << 1) ++#define GEN7_OACONTROL_ENABLE (1 << 0) ++ ++#define GEN8_OACTXID _MMIO(0x2364) ++ ++#define GEN8_OA_DEBUG _MMIO(0x2B04) ++#define GEN9_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS (1 << 5) ++#define GEN9_OA_DEBUG_INCLUDE_CLK_RATIO (1 << 6) ++#define GEN9_OA_DEBUG_DISABLE_GO_1_0_REPORTS (1 << 2) ++#define GEN9_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS (1 << 1) ++ ++#define GEN8_OACONTROL _MMIO(0x2B00) ++#define GEN8_OA_REPORT_FORMAT_A12 (0 << 2) ++#define GEN8_OA_REPORT_FORMAT_A12_B8_C8 (2 << 2) ++#define GEN8_OA_REPORT_FORMAT_A36_B8_C8 (5 << 2) ++#define GEN8_OA_REPORT_FORMAT_C4_B8 (7 << 2) ++#define GEN8_OA_REPORT_FORMAT_SHIFT 2 ++#define GEN8_OA_SPECIFIC_CONTEXT_ENABLE (1 << 1) ++#define GEN8_OA_COUNTER_ENABLE (1 << 0) ++ ++#define GEN8_OACTXCONTROL _MMIO(0x2360) ++#define GEN8_OA_TIMER_PERIOD_MASK 0x3F ++#define GEN8_OA_TIMER_PERIOD_SHIFT 2 ++#define GEN8_OA_TIMER_ENABLE (1 << 1) ++#define GEN8_OA_COUNTER_RESUME (1 << 0) ++ ++#define GEN7_OABUFFER _MMIO(0x23B0) /* R/W */ ++#define GEN7_OABUFFER_OVERRUN_DISABLE (1 << 3) ++#define GEN7_OABUFFER_EDGE_TRIGGER (1 << 2) ++#define GEN7_OABUFFER_STOP_RESUME_ENABLE (1 << 1) ++#define GEN7_OABUFFER_RESUME (1 << 0) ++ ++#define GEN8_OABUFFER_UDW _MMIO(0x23b4) ++#define GEN8_OABUFFER _MMIO(0x2b14) ++#define GEN8_OABUFFER_MEM_SELECT_GGTT (1 << 0) /* 0: PPGTT, 1: GGTT */ ++ ++#define GEN7_OASTATUS1 _MMIO(0x2364) ++#define GEN7_OASTATUS1_TAIL_MASK 0xffffffc0 ++#define GEN7_OASTATUS1_COUNTER_OVERFLOW (1 << 2) ++#define GEN7_OASTATUS1_OABUFFER_OVERFLOW (1 << 1) ++#define GEN7_OASTATUS1_REPORT_LOST (1 << 0) ++ ++#define GEN7_OASTATUS2 _MMIO(0x2368) ++#define GEN7_OASTATUS2_HEAD_MASK 0xffffffc0 ++#define GEN7_OASTATUS2_MEM_SELECT_GGTT (1 << 0) /* 0: PPGTT, 1: GGTT */ ++ ++#define GEN8_OASTATUS _MMIO(0x2b08) ++#define GEN8_OASTATUS_OVERRUN_STATUS (1 << 3) ++#define GEN8_OASTATUS_COUNTER_OVERFLOW (1 << 2) ++#define GEN8_OASTATUS_OABUFFER_OVERFLOW (1 << 1) ++#define GEN8_OASTATUS_REPORT_LOST (1 << 0) ++ ++#define GEN8_OAHEADPTR _MMIO(0x2B0C) ++#define GEN8_OAHEADPTR_MASK 0xffffffc0 ++#define GEN8_OATAILPTR _MMIO(0x2B10) ++#define GEN8_OATAILPTR_MASK 0xffffffc0 ++ ++#define OABUFFER_SIZE_128K (0 << 3) ++#define OABUFFER_SIZE_256K (1 << 3) ++#define OABUFFER_SIZE_512K (2 << 3) ++#define OABUFFER_SIZE_1M (3 << 3) ++#define OABUFFER_SIZE_2M (4 << 3) ++#define OABUFFER_SIZE_4M (5 << 3) ++#define OABUFFER_SIZE_8M (6 << 3) ++#define OABUFFER_SIZE_16M (7 << 3) ++ ++/* ++ * Flexible, Aggregate EU Counter Registers. ++ * Note: these aren't contiguous ++ */ ++#define EU_PERF_CNTL0 _MMIO(0xe458) ++#define EU_PERF_CNTL1 _MMIO(0xe558) ++#define EU_PERF_CNTL2 _MMIO(0xe658) ++#define EU_PERF_CNTL3 _MMIO(0xe758) ++#define EU_PERF_CNTL4 _MMIO(0xe45c) ++#define EU_PERF_CNTL5 _MMIO(0xe55c) ++#define EU_PERF_CNTL6 _MMIO(0xe65c) ++ ++/* ++ * OA Boolean state ++ */ ++ ++#define OASTARTTRIG1 _MMIO(0x2710) ++#define OASTARTTRIG1_THRESHOLD_COUNT_MASK_MBZ 0xffff0000 ++#define OASTARTTRIG1_THRESHOLD_MASK 0xffff ++ ++#define OASTARTTRIG2 _MMIO(0x2714) ++#define OASTARTTRIG2_INVERT_A_0 (1 << 0) ++#define OASTARTTRIG2_INVERT_A_1 (1 << 1) ++#define OASTARTTRIG2_INVERT_A_2 (1 << 2) ++#define OASTARTTRIG2_INVERT_A_3 (1 << 3) ++#define OASTARTTRIG2_INVERT_A_4 (1 << 4) ++#define OASTARTTRIG2_INVERT_A_5 (1 << 5) ++#define OASTARTTRIG2_INVERT_A_6 (1 << 6) ++#define OASTARTTRIG2_INVERT_A_7 (1 << 7) ++#define OASTARTTRIG2_INVERT_A_8 (1 << 8) ++#define OASTARTTRIG2_INVERT_A_9 (1 << 9) ++#define OASTARTTRIG2_INVERT_A_10 (1 << 10) ++#define OASTARTTRIG2_INVERT_A_11 (1 << 11) ++#define OASTARTTRIG2_INVERT_A_12 (1 << 12) ++#define OASTARTTRIG2_INVERT_A_13 (1 << 13) ++#define OASTARTTRIG2_INVERT_A_14 (1 << 14) ++#define OASTARTTRIG2_INVERT_A_15 (1 << 15) ++#define OASTARTTRIG2_INVERT_B_0 (1 << 16) ++#define OASTARTTRIG2_INVERT_B_1 (1 << 17) ++#define OASTARTTRIG2_INVERT_B_2 (1 << 18) ++#define OASTARTTRIG2_INVERT_B_3 (1 << 19) ++#define OASTARTTRIG2_INVERT_C_0 (1 << 20) ++#define OASTARTTRIG2_INVERT_C_1 (1 << 21) ++#define OASTARTTRIG2_INVERT_D_0 (1 << 22) ++#define OASTARTTRIG2_THRESHOLD_ENABLE (1 << 23) ++#define OASTARTTRIG2_START_TRIG_FLAG_MBZ (1 << 24) ++#define OASTARTTRIG2_EVENT_SELECT_0 (1 << 28) ++#define OASTARTTRIG2_EVENT_SELECT_1 (1 << 29) ++#define OASTARTTRIG2_EVENT_SELECT_2 (1 << 30) ++#define OASTARTTRIG2_EVENT_SELECT_3 (1 << 31) ++ ++#define OASTARTTRIG3 _MMIO(0x2718) ++#define OASTARTTRIG3_NOA_SELECT_MASK 0xf ++#define OASTARTTRIG3_NOA_SELECT_8_SHIFT 0 ++#define OASTARTTRIG3_NOA_SELECT_9_SHIFT 4 ++#define OASTARTTRIG3_NOA_SELECT_10_SHIFT 8 ++#define OASTARTTRIG3_NOA_SELECT_11_SHIFT 12 ++#define OASTARTTRIG3_NOA_SELECT_12_SHIFT 16 ++#define OASTARTTRIG3_NOA_SELECT_13_SHIFT 20 ++#define OASTARTTRIG3_NOA_SELECT_14_SHIFT 24 ++#define OASTARTTRIG3_NOA_SELECT_15_SHIFT 28 ++ ++#define OASTARTTRIG4 _MMIO(0x271c) ++#define OASTARTTRIG4_NOA_SELECT_MASK 0xf ++#define OASTARTTRIG4_NOA_SELECT_0_SHIFT 0 ++#define OASTARTTRIG4_NOA_SELECT_1_SHIFT 4 ++#define OASTARTTRIG4_NOA_SELECT_2_SHIFT 8 ++#define OASTARTTRIG4_NOA_SELECT_3_SHIFT 12 ++#define OASTARTTRIG4_NOA_SELECT_4_SHIFT 16 ++#define OASTARTTRIG4_NOA_SELECT_5_SHIFT 20 ++#define OASTARTTRIG4_NOA_SELECT_6_SHIFT 24 ++#define OASTARTTRIG4_NOA_SELECT_7_SHIFT 28 ++ ++#define OASTARTTRIG5 _MMIO(0x2720) ++#define OASTARTTRIG5_THRESHOLD_COUNT_MASK_MBZ 0xffff0000 ++#define OASTARTTRIG5_THRESHOLD_MASK 0xffff ++ ++#define OASTARTTRIG6 _MMIO(0x2724) ++#define OASTARTTRIG6_INVERT_A_0 (1 << 0) ++#define OASTARTTRIG6_INVERT_A_1 (1 << 1) ++#define OASTARTTRIG6_INVERT_A_2 (1 << 2) ++#define OASTARTTRIG6_INVERT_A_3 (1 << 3) ++#define OASTARTTRIG6_INVERT_A_4 (1 << 4) ++#define OASTARTTRIG6_INVERT_A_5 (1 << 5) ++#define OASTARTTRIG6_INVERT_A_6 (1 << 6) ++#define OASTARTTRIG6_INVERT_A_7 (1 << 7) ++#define OASTARTTRIG6_INVERT_A_8 (1 << 8) ++#define OASTARTTRIG6_INVERT_A_9 (1 << 9) ++#define OASTARTTRIG6_INVERT_A_10 (1 << 10) ++#define OASTARTTRIG6_INVERT_A_11 (1 << 11) ++#define OASTARTTRIG6_INVERT_A_12 (1 << 12) ++#define OASTARTTRIG6_INVERT_A_13 (1 << 13) ++#define OASTARTTRIG6_INVERT_A_14 (1 << 14) ++#define OASTARTTRIG6_INVERT_A_15 (1 << 15) ++#define OASTARTTRIG6_INVERT_B_0 (1 << 16) ++#define OASTARTTRIG6_INVERT_B_1 (1 << 17) ++#define OASTARTTRIG6_INVERT_B_2 (1 << 18) ++#define OASTARTTRIG6_INVERT_B_3 (1 << 19) ++#define OASTARTTRIG6_INVERT_C_0 (1 << 20) ++#define OASTARTTRIG6_INVERT_C_1 (1 << 21) ++#define OASTARTTRIG6_INVERT_D_0 (1 << 22) ++#define OASTARTTRIG6_THRESHOLD_ENABLE (1 << 23) ++#define OASTARTTRIG6_START_TRIG_FLAG_MBZ (1 << 24) ++#define OASTARTTRIG6_EVENT_SELECT_4 (1 << 28) ++#define OASTARTTRIG6_EVENT_SELECT_5 (1 << 29) ++#define OASTARTTRIG6_EVENT_SELECT_6 (1 << 30) ++#define OASTARTTRIG6_EVENT_SELECT_7 (1 << 31) ++ ++#define OASTARTTRIG7 _MMIO(0x2728) ++#define OASTARTTRIG7_NOA_SELECT_MASK 0xf ++#define OASTARTTRIG7_NOA_SELECT_8_SHIFT 0 ++#define OASTARTTRIG7_NOA_SELECT_9_SHIFT 4 ++#define OASTARTTRIG7_NOA_SELECT_10_SHIFT 8 ++#define OASTARTTRIG7_NOA_SELECT_11_SHIFT 12 ++#define OASTARTTRIG7_NOA_SELECT_12_SHIFT 16 ++#define OASTARTTRIG7_NOA_SELECT_13_SHIFT 20 ++#define OASTARTTRIG7_NOA_SELECT_14_SHIFT 24 ++#define OASTARTTRIG7_NOA_SELECT_15_SHIFT 28 ++ ++#define OASTARTTRIG8 _MMIO(0x272c) ++#define OASTARTTRIG8_NOA_SELECT_MASK 0xf ++#define OASTARTTRIG8_NOA_SELECT_0_SHIFT 0 ++#define OASTARTTRIG8_NOA_SELECT_1_SHIFT 4 ++#define OASTARTTRIG8_NOA_SELECT_2_SHIFT 8 ++#define OASTARTTRIG8_NOA_SELECT_3_SHIFT 12 ++#define OASTARTTRIG8_NOA_SELECT_4_SHIFT 16 ++#define OASTARTTRIG8_NOA_SELECT_5_SHIFT 20 ++#define OASTARTTRIG8_NOA_SELECT_6_SHIFT 24 ++#define OASTARTTRIG8_NOA_SELECT_7_SHIFT 28 ++ ++#define OAREPORTTRIG1 _MMIO(0x2740) ++#define OAREPORTTRIG1_THRESHOLD_MASK 0xffff ++#define OAREPORTTRIG1_EDGE_LEVEL_TRIGER_SELECT_MASK 0xffff0000 /* 0=level */ ++ ++#define OAREPORTTRIG2 _MMIO(0x2744) ++#define OAREPORTTRIG2_INVERT_A_0 (1 << 0) ++#define OAREPORTTRIG2_INVERT_A_1 (1 << 1) ++#define OAREPORTTRIG2_INVERT_A_2 (1 << 2) ++#define OAREPORTTRIG2_INVERT_A_3 (1 << 3) ++#define OAREPORTTRIG2_INVERT_A_4 (1 << 4) ++#define OAREPORTTRIG2_INVERT_A_5 (1 << 5) ++#define OAREPORTTRIG2_INVERT_A_6 (1 << 6) ++#define OAREPORTTRIG2_INVERT_A_7 (1 << 7) ++#define OAREPORTTRIG2_INVERT_A_8 (1 << 8) ++#define OAREPORTTRIG2_INVERT_A_9 (1 << 9) ++#define OAREPORTTRIG2_INVERT_A_10 (1 << 10) ++#define OAREPORTTRIG2_INVERT_A_11 (1 << 11) ++#define OAREPORTTRIG2_INVERT_A_12 (1 << 12) ++#define OAREPORTTRIG2_INVERT_A_13 (1 << 13) ++#define OAREPORTTRIG2_INVERT_A_14 (1 << 14) ++#define OAREPORTTRIG2_INVERT_A_15 (1 << 15) ++#define OAREPORTTRIG2_INVERT_B_0 (1 << 16) ++#define OAREPORTTRIG2_INVERT_B_1 (1 << 17) ++#define OAREPORTTRIG2_INVERT_B_2 (1 << 18) ++#define OAREPORTTRIG2_INVERT_B_3 (1 << 19) ++#define OAREPORTTRIG2_INVERT_C_0 (1 << 20) ++#define OAREPORTTRIG2_INVERT_C_1 (1 << 21) ++#define OAREPORTTRIG2_INVERT_D_0 (1 << 22) ++#define OAREPORTTRIG2_THRESHOLD_ENABLE (1 << 23) ++#define OAREPORTTRIG2_REPORT_TRIGGER_ENABLE (1 << 31) ++ ++#define OAREPORTTRIG3 _MMIO(0x2748) ++#define OAREPORTTRIG3_NOA_SELECT_MASK 0xf ++#define OAREPORTTRIG3_NOA_SELECT_8_SHIFT 0 ++#define OAREPORTTRIG3_NOA_SELECT_9_SHIFT 4 ++#define OAREPORTTRIG3_NOA_SELECT_10_SHIFT 8 ++#define OAREPORTTRIG3_NOA_SELECT_11_SHIFT 12 ++#define OAREPORTTRIG3_NOA_SELECT_12_SHIFT 16 ++#define OAREPORTTRIG3_NOA_SELECT_13_SHIFT 20 ++#define OAREPORTTRIG3_NOA_SELECT_14_SHIFT 24 ++#define OAREPORTTRIG3_NOA_SELECT_15_SHIFT 28 ++ ++#define OAREPORTTRIG4 _MMIO(0x274c) ++#define OAREPORTTRIG4_NOA_SELECT_MASK 0xf ++#define OAREPORTTRIG4_NOA_SELECT_0_SHIFT 0 ++#define OAREPORTTRIG4_NOA_SELECT_1_SHIFT 4 ++#define OAREPORTTRIG4_NOA_SELECT_2_SHIFT 8 ++#define OAREPORTTRIG4_NOA_SELECT_3_SHIFT 12 ++#define OAREPORTTRIG4_NOA_SELECT_4_SHIFT 16 ++#define OAREPORTTRIG4_NOA_SELECT_5_SHIFT 20 ++#define OAREPORTTRIG4_NOA_SELECT_6_SHIFT 24 ++#define OAREPORTTRIG4_NOA_SELECT_7_SHIFT 28 ++ ++#define OAREPORTTRIG5 _MMIO(0x2750) ++#define OAREPORTTRIG5_THRESHOLD_MASK 0xffff ++#define OAREPORTTRIG5_EDGE_LEVEL_TRIGER_SELECT_MASK 0xffff0000 /* 0=level */ ++ ++#define OAREPORTTRIG6 _MMIO(0x2754) ++#define OAREPORTTRIG6_INVERT_A_0 (1 << 0) ++#define OAREPORTTRIG6_INVERT_A_1 (1 << 1) ++#define OAREPORTTRIG6_INVERT_A_2 (1 << 2) ++#define OAREPORTTRIG6_INVERT_A_3 (1 << 3) ++#define OAREPORTTRIG6_INVERT_A_4 (1 << 4) ++#define OAREPORTTRIG6_INVERT_A_5 (1 << 5) ++#define OAREPORTTRIG6_INVERT_A_6 (1 << 6) ++#define OAREPORTTRIG6_INVERT_A_7 (1 << 7) ++#define OAREPORTTRIG6_INVERT_A_8 (1 << 8) ++#define OAREPORTTRIG6_INVERT_A_9 (1 << 9) ++#define OAREPORTTRIG6_INVERT_A_10 (1 << 10) ++#define OAREPORTTRIG6_INVERT_A_11 (1 << 11) ++#define OAREPORTTRIG6_INVERT_A_12 (1 << 12) ++#define OAREPORTTRIG6_INVERT_A_13 (1 << 13) ++#define OAREPORTTRIG6_INVERT_A_14 (1 << 14) ++#define OAREPORTTRIG6_INVERT_A_15 (1 << 15) ++#define OAREPORTTRIG6_INVERT_B_0 (1 << 16) ++#define OAREPORTTRIG6_INVERT_B_1 (1 << 17) ++#define OAREPORTTRIG6_INVERT_B_2 (1 << 18) ++#define OAREPORTTRIG6_INVERT_B_3 (1 << 19) ++#define OAREPORTTRIG6_INVERT_C_0 (1 << 20) ++#define OAREPORTTRIG6_INVERT_C_1 (1 << 21) ++#define OAREPORTTRIG6_INVERT_D_0 (1 << 22) ++#define OAREPORTTRIG6_THRESHOLD_ENABLE (1 << 23) ++#define OAREPORTTRIG6_REPORT_TRIGGER_ENABLE (1 << 31) ++ ++#define OAREPORTTRIG7 _MMIO(0x2758) ++#define OAREPORTTRIG7_NOA_SELECT_MASK 0xf ++#define OAREPORTTRIG7_NOA_SELECT_8_SHIFT 0 ++#define OAREPORTTRIG7_NOA_SELECT_9_SHIFT 4 ++#define OAREPORTTRIG7_NOA_SELECT_10_SHIFT 8 ++#define OAREPORTTRIG7_NOA_SELECT_11_SHIFT 12 ++#define OAREPORTTRIG7_NOA_SELECT_12_SHIFT 16 ++#define OAREPORTTRIG7_NOA_SELECT_13_SHIFT 20 ++#define OAREPORTTRIG7_NOA_SELECT_14_SHIFT 24 ++#define OAREPORTTRIG7_NOA_SELECT_15_SHIFT 28 ++ ++#define OAREPORTTRIG8 _MMIO(0x275c) ++#define OAREPORTTRIG8_NOA_SELECT_MASK 0xf ++#define OAREPORTTRIG8_NOA_SELECT_0_SHIFT 0 ++#define OAREPORTTRIG8_NOA_SELECT_1_SHIFT 4 ++#define OAREPORTTRIG8_NOA_SELECT_2_SHIFT 8 ++#define OAREPORTTRIG8_NOA_SELECT_3_SHIFT 12 ++#define OAREPORTTRIG8_NOA_SELECT_4_SHIFT 16 ++#define OAREPORTTRIG8_NOA_SELECT_5_SHIFT 20 ++#define OAREPORTTRIG8_NOA_SELECT_6_SHIFT 24 ++#define OAREPORTTRIG8_NOA_SELECT_7_SHIFT 28 ++ ++/* CECX_0 */ ++#define OACEC_COMPARE_LESS_OR_EQUAL 6 ++#define OACEC_COMPARE_NOT_EQUAL 5 ++#define OACEC_COMPARE_LESS_THAN 4 ++#define OACEC_COMPARE_GREATER_OR_EQUAL 3 ++#define OACEC_COMPARE_EQUAL 2 ++#define OACEC_COMPARE_GREATER_THAN 1 ++#define OACEC_COMPARE_ANY_EQUAL 0 ++ ++#define OACEC_COMPARE_VALUE_MASK 0xffff ++#define OACEC_COMPARE_VALUE_SHIFT 3 ++ ++#define OACEC_SELECT_NOA (0 << 19) ++#define OACEC_SELECT_PREV (1 << 19) ++#define OACEC_SELECT_BOOLEAN (2 << 19) ++ ++/* CECX_1 */ ++#define OACEC_MASK_MASK 0xffff ++#define OACEC_CONSIDERATIONS_MASK 0xffff ++#define OACEC_CONSIDERATIONS_SHIFT 16 ++ ++#define OACEC0_0 _MMIO(0x2770) ++#define OACEC0_1 _MMIO(0x2774) ++#define OACEC1_0 _MMIO(0x2778) ++#define OACEC1_1 _MMIO(0x277c) ++#define OACEC2_0 _MMIO(0x2780) ++#define OACEC2_1 _MMIO(0x2784) ++#define OACEC3_0 _MMIO(0x2788) ++#define OACEC3_1 _MMIO(0x278c) ++#define OACEC4_0 _MMIO(0x2790) ++#define OACEC4_1 _MMIO(0x2794) ++#define OACEC5_0 _MMIO(0x2798) ++#define OACEC5_1 _MMIO(0x279c) ++#define OACEC6_0 _MMIO(0x27a0) ++#define OACEC6_1 _MMIO(0x27a4) ++#define OACEC7_0 _MMIO(0x27a8) ++#define OACEC7_1 _MMIO(0x27ac) ++ ++/* OA perf counters */ ++#define OA_PERFCNT1_LO _MMIO(0x91B8) ++#define OA_PERFCNT1_HI _MMIO(0x91BC) ++#define OA_PERFCNT2_LO _MMIO(0x91C0) ++#define OA_PERFCNT2_HI _MMIO(0x91C4) ++#define OA_PERFCNT3_LO _MMIO(0x91C8) ++#define OA_PERFCNT3_HI _MMIO(0x91CC) ++#define OA_PERFCNT4_LO _MMIO(0x91D8) ++#define OA_PERFCNT4_HI _MMIO(0x91DC) ++ ++#define OA_PERFMATRIX_LO _MMIO(0x91C8) ++#define OA_PERFMATRIX_HI _MMIO(0x91CC) ++ ++/* RPM unit config (Gen8+) */ ++#define RPM_CONFIG0 _MMIO(0x0D00) ++#define GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT 3 ++#define GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_MASK (1 << GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT) ++#define GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_19_2_MHZ 0 ++#define GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_24_MHZ 1 ++#define GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT 3 ++#define GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_MASK (0x7 << GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT) ++#define GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_24_MHZ 0 ++#define GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_19_2_MHZ 1 ++#define GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_38_4_MHZ 2 ++#define GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_25_MHZ 3 ++#define GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT 1 ++#define GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK (0x3 << GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT) ++ ++#define RPM_CONFIG1 _MMIO(0x0D04) ++#define GEN10_GT_NOA_ENABLE (1 << 9) ++ ++/* GPM unit config (Gen9+) */ ++#define CTC_MODE _MMIO(0xA26C) ++#define CTC_SOURCE_PARAMETER_MASK 1 ++#define CTC_SOURCE_CRYSTAL_CLOCK 0 ++#define CTC_SOURCE_DIVIDE_LOGIC 1 ++#define CTC_SHIFT_PARAMETER_SHIFT 1 ++#define CTC_SHIFT_PARAMETER_MASK (0x3 << CTC_SHIFT_PARAMETER_SHIFT) ++ ++/* RCP unit config (Gen8+) */ ++#define RCP_CONFIG _MMIO(0x0D08) ++ ++/* NOA (HSW) */ ++#define HSW_MBVID2_NOA0 _MMIO(0x9E80) ++#define HSW_MBVID2_NOA1 _MMIO(0x9E84) ++#define HSW_MBVID2_NOA2 _MMIO(0x9E88) ++#define HSW_MBVID2_NOA3 _MMIO(0x9E8C) ++#define HSW_MBVID2_NOA4 _MMIO(0x9E90) ++#define HSW_MBVID2_NOA5 _MMIO(0x9E94) ++#define HSW_MBVID2_NOA6 _MMIO(0x9E98) ++#define HSW_MBVID2_NOA7 _MMIO(0x9E9C) ++#define HSW_MBVID2_NOA8 _MMIO(0x9EA0) ++#define HSW_MBVID2_NOA9 _MMIO(0x9EA4) ++ ++#define HSW_MBVID2_MISR0 _MMIO(0x9EC0) ++ ++/* NOA (Gen8+) */ ++#define NOA_CONFIG(i) _MMIO(0x0D0C + (i) * 4) ++ ++#define MICRO_BP0_0 _MMIO(0x9800) ++#define MICRO_BP0_2 _MMIO(0x9804) ++#define MICRO_BP0_1 _MMIO(0x9808) ++ ++#define MICRO_BP1_0 _MMIO(0x980C) ++#define MICRO_BP1_2 _MMIO(0x9810) ++#define MICRO_BP1_1 _MMIO(0x9814) ++ ++#define MICRO_BP2_0 _MMIO(0x9818) ++#define MICRO_BP2_2 _MMIO(0x981C) ++#define MICRO_BP2_1 _MMIO(0x9820) ++ ++#define MICRO_BP3_0 _MMIO(0x9824) ++#define MICRO_BP3_2 _MMIO(0x9828) ++#define MICRO_BP3_1 _MMIO(0x982C) ++ ++#define MICRO_BP_TRIGGER _MMIO(0x9830) ++#define MICRO_BP3_COUNT_STATUS01 _MMIO(0x9834) ++#define MICRO_BP3_COUNT_STATUS23 _MMIO(0x9838) ++#define MICRO_BP_FIRED_ARMED _MMIO(0x983C) ++ ++#define GDT_CHICKEN_BITS _MMIO(0x9840) ++#define GT_NOA_ENABLE 0x00000080 ++ ++#define NOA_DATA _MMIO(0x986C) ++#define NOA_WRITE _MMIO(0x9888) ++#define GEN10_NOA_WRITE_HIGH _MMIO(0x9884) ++ ++#define _GEN7_PIPEA_DE_LOAD_SL 0x70068 ++#define _GEN7_PIPEB_DE_LOAD_SL 0x71068 ++#define GEN7_PIPE_DE_LOAD_SL(pipe) _MMIO_PIPE(pipe, _GEN7_PIPEA_DE_LOAD_SL, _GEN7_PIPEB_DE_LOAD_SL) ++ ++/* ++ * Reset registers ++ */ ++#define DEBUG_RESET_I830 _MMIO(0x6070) ++#define DEBUG_RESET_FULL (1 << 7) ++#define DEBUG_RESET_RENDER (1 << 8) ++#define DEBUG_RESET_DISPLAY (1 << 9) ++ ++/* ++ * IOSF sideband ++ */ ++#define VLV_IOSF_DOORBELL_REQ _MMIO(VLV_DISPLAY_BASE + 0x2100) ++#define IOSF_DEVFN_SHIFT 24 ++#define IOSF_OPCODE_SHIFT 16 ++#define IOSF_PORT_SHIFT 8 ++#define IOSF_BYTE_ENABLES_SHIFT 4 ++#define IOSF_BAR_SHIFT 1 ++#define IOSF_SB_BUSY (1 << 0) ++#define IOSF_PORT_BUNIT 0x03 ++#define IOSF_PORT_PUNIT 0x04 ++#define IOSF_PORT_NC 0x11 ++#define IOSF_PORT_DPIO 0x12 ++#define IOSF_PORT_GPIO_NC 0x13 ++#define IOSF_PORT_CCK 0x14 ++#define IOSF_PORT_DPIO_2 0x1a ++#define IOSF_PORT_FLISDSI 0x1b ++#define IOSF_PORT_GPIO_SC 0x48 ++#define IOSF_PORT_GPIO_SUS 0xa8 ++#define IOSF_PORT_CCU 0xa9 ++#define CHV_IOSF_PORT_GPIO_N 0x13 ++#define CHV_IOSF_PORT_GPIO_SE 0x48 ++#define CHV_IOSF_PORT_GPIO_E 0xa8 ++#define CHV_IOSF_PORT_GPIO_SW 0xb2 ++#define VLV_IOSF_DATA _MMIO(VLV_DISPLAY_BASE + 0x2104) ++#define VLV_IOSF_ADDR _MMIO(VLV_DISPLAY_BASE + 0x2108) ++ ++/* See configdb bunit SB addr map */ ++#define BUNIT_REG_BISOC 0x11 ++ ++/* PUNIT_REG_*SSPM0 */ ++#define _SSPM0_SSC(val) ((val) << 0) ++#define SSPM0_SSC_MASK _SSPM0_SSC(0x3) ++#define SSPM0_SSC_PWR_ON _SSPM0_SSC(0x0) ++#define SSPM0_SSC_CLK_GATE _SSPM0_SSC(0x1) ++#define SSPM0_SSC_RESET _SSPM0_SSC(0x2) ++#define SSPM0_SSC_PWR_GATE _SSPM0_SSC(0x3) ++#define _SSPM0_SSS(val) ((val) << 24) ++#define SSPM0_SSS_MASK _SSPM0_SSS(0x3) ++#define SSPM0_SSS_PWR_ON _SSPM0_SSS(0x0) ++#define SSPM0_SSS_CLK_GATE _SSPM0_SSS(0x1) ++#define SSPM0_SSS_RESET _SSPM0_SSS(0x2) ++#define SSPM0_SSS_PWR_GATE _SSPM0_SSS(0x3) ++ ++/* PUNIT_REG_*SSPM1 */ ++#define SSPM1_FREQSTAT_SHIFT 24 ++#define SSPM1_FREQSTAT_MASK (0x1f << SSPM1_FREQSTAT_SHIFT) ++#define SSPM1_FREQGUAR_SHIFT 8 ++#define SSPM1_FREQGUAR_MASK (0x1f << SSPM1_FREQGUAR_SHIFT) ++#define SSPM1_FREQ_SHIFT 0 ++#define SSPM1_FREQ_MASK (0x1f << SSPM1_FREQ_SHIFT) ++ ++#define PUNIT_REG_VEDSSPM0 0x32 ++#define PUNIT_REG_VEDSSPM1 0x33 ++ ++#define PUNIT_REG_DSPSSPM 0x36 ++#define DSPFREQSTAT_SHIFT_CHV 24 ++#define DSPFREQSTAT_MASK_CHV (0x1f << DSPFREQSTAT_SHIFT_CHV) ++#define DSPFREQGUAR_SHIFT_CHV 8 ++#define DSPFREQGUAR_MASK_CHV (0x1f << DSPFREQGUAR_SHIFT_CHV) ++#define DSPFREQSTAT_SHIFT 30 ++#define DSPFREQSTAT_MASK (0x3 << DSPFREQSTAT_SHIFT) ++#define DSPFREQGUAR_SHIFT 14 ++#define DSPFREQGUAR_MASK (0x3 << DSPFREQGUAR_SHIFT) ++#define DSP_MAXFIFO_PM5_STATUS (1 << 22) /* chv */ ++#define DSP_AUTO_CDCLK_GATE_DISABLE (1 << 7) /* chv */ ++#define DSP_MAXFIFO_PM5_ENABLE (1 << 6) /* chv */ ++#define _DP_SSC(val, pipe) ((val) << (2 * (pipe))) ++#define DP_SSC_MASK(pipe) _DP_SSC(0x3, (pipe)) ++#define DP_SSC_PWR_ON(pipe) _DP_SSC(0x0, (pipe)) ++#define DP_SSC_CLK_GATE(pipe) _DP_SSC(0x1, (pipe)) ++#define DP_SSC_RESET(pipe) _DP_SSC(0x2, (pipe)) ++#define DP_SSC_PWR_GATE(pipe) _DP_SSC(0x3, (pipe)) ++#define _DP_SSS(val, pipe) ((val) << (2 * (pipe) + 16)) ++#define DP_SSS_MASK(pipe) _DP_SSS(0x3, (pipe)) ++#define DP_SSS_PWR_ON(pipe) _DP_SSS(0x0, (pipe)) ++#define DP_SSS_CLK_GATE(pipe) _DP_SSS(0x1, (pipe)) ++#define DP_SSS_RESET(pipe) _DP_SSS(0x2, (pipe)) ++#define DP_SSS_PWR_GATE(pipe) _DP_SSS(0x3, (pipe)) ++ ++#define PUNIT_REG_ISPSSPM0 0x39 ++#define PUNIT_REG_ISPSSPM1 0x3a ++ ++/* ++ * i915_power_well_id: ++ * ++ * IDs used to look up power wells. Power wells accessed directly bypassing ++ * the power domains framework must be assigned a unique ID. The rest of power ++ * wells must be assigned DISP_PW_ID_NONE. ++ */ ++enum i915_power_well_id { ++ DISP_PW_ID_NONE, ++ ++ VLV_DISP_PW_DISP2D, ++ BXT_DISP_PW_DPIO_CMN_A, ++ VLV_DISP_PW_DPIO_CMN_BC, ++ GLK_DISP_PW_DPIO_CMN_C, ++ CHV_DISP_PW_DPIO_CMN_D, ++ HSW_DISP_PW_GLOBAL, ++ SKL_DISP_PW_MISC_IO, ++ SKL_DISP_PW_1, ++ SKL_DISP_PW_2, ++}; ++ ++#define PUNIT_REG_PWRGT_CTRL 0x60 ++#define PUNIT_REG_PWRGT_STATUS 0x61 ++#define PUNIT_PWRGT_MASK(pw_idx) (3 << ((pw_idx) * 2)) ++#define PUNIT_PWRGT_PWR_ON(pw_idx) (0 << ((pw_idx) * 2)) ++#define PUNIT_PWRGT_CLK_GATE(pw_idx) (1 << ((pw_idx) * 2)) ++#define PUNIT_PWRGT_RESET(pw_idx) (2 << ((pw_idx) * 2)) ++#define PUNIT_PWRGT_PWR_GATE(pw_idx) (3 << ((pw_idx) * 2)) ++ ++#define PUNIT_PWGT_IDX_RENDER 0 ++#define PUNIT_PWGT_IDX_MEDIA 1 ++#define PUNIT_PWGT_IDX_DISP2D 3 ++#define PUNIT_PWGT_IDX_DPIO_CMN_BC 5 ++#define PUNIT_PWGT_IDX_DPIO_TX_B_LANES_01 6 ++#define PUNIT_PWGT_IDX_DPIO_TX_B_LANES_23 7 ++#define PUNIT_PWGT_IDX_DPIO_TX_C_LANES_01 8 ++#define PUNIT_PWGT_IDX_DPIO_TX_C_LANES_23 9 ++#define PUNIT_PWGT_IDX_DPIO_RX0 10 ++#define PUNIT_PWGT_IDX_DPIO_RX1 11 ++#define PUNIT_PWGT_IDX_DPIO_CMN_D 12 ++ ++#define PUNIT_REG_GPU_LFM 0xd3 ++#define PUNIT_REG_GPU_FREQ_REQ 0xd4 ++#define PUNIT_REG_GPU_FREQ_STS 0xd8 ++#define GPLLENABLE (1 << 4) ++#define GENFREQSTATUS (1 << 0) ++#define PUNIT_REG_MEDIA_TURBO_FREQ_REQ 0xdc ++#define PUNIT_REG_CZ_TIMESTAMP 0xce ++ ++#define PUNIT_FUSE_BUS2 0xf6 /* bits 47:40 */ ++#define PUNIT_FUSE_BUS1 0xf5 /* bits 55:48 */ ++ ++#define FB_GFX_FMAX_AT_VMAX_FUSE 0x136 ++#define FB_GFX_FREQ_FUSE_MASK 0xff ++#define FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT 24 ++#define FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT 16 ++#define FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT 8 ++ ++#define FB_GFX_FMIN_AT_VMIN_FUSE 0x137 ++#define FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT 8 ++ ++#define PUNIT_REG_DDR_SETUP2 0x139 ++#define FORCE_DDR_FREQ_REQ_ACK (1 << 8) ++#define FORCE_DDR_LOW_FREQ (1 << 1) ++#define FORCE_DDR_HIGH_FREQ (1 << 0) ++ ++#define PUNIT_GPU_STATUS_REG 0xdb ++#define PUNIT_GPU_STATUS_MAX_FREQ_SHIFT 16 ++#define PUNIT_GPU_STATUS_MAX_FREQ_MASK 0xff ++#define PUNIT_GPU_STATIS_GFX_MIN_FREQ_SHIFT 8 ++#define PUNIT_GPU_STATUS_GFX_MIN_FREQ_MASK 0xff ++ ++#define PUNIT_GPU_DUTYCYCLE_REG 0xdf ++#define PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT 8 ++#define PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK 0xff ++ ++#define IOSF_NC_FB_GFX_FREQ_FUSE 0x1c ++#define FB_GFX_MAX_FREQ_FUSE_SHIFT 3 ++#define FB_GFX_MAX_FREQ_FUSE_MASK 0x000007f8 ++#define FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT 11 ++#define FB_GFX_FGUARANTEED_FREQ_FUSE_MASK 0x0007f800 ++#define IOSF_NC_FB_GFX_FMAX_FUSE_HI 0x34 ++#define FB_FMAX_VMIN_FREQ_HI_MASK 0x00000007 ++#define IOSF_NC_FB_GFX_FMAX_FUSE_LO 0x30 ++#define FB_FMAX_VMIN_FREQ_LO_SHIFT 27 ++#define FB_FMAX_VMIN_FREQ_LO_MASK 0xf8000000 ++ ++#define VLV_TURBO_SOC_OVERRIDE 0x04 ++#define VLV_OVERRIDE_EN 1 ++#define VLV_SOC_TDP_EN (1 << 1) ++#define VLV_BIAS_CPU_125_SOC_875 (6 << 2) ++#define CHV_BIAS_CPU_50_SOC_50 (3 << 2) ++ ++/* vlv2 north clock has */ ++#define CCK_FUSE_REG 0x8 ++#define CCK_FUSE_HPLL_FREQ_MASK 0x3 ++#define CCK_REG_DSI_PLL_FUSE 0x44 ++#define CCK_REG_DSI_PLL_CONTROL 0x48 ++#define DSI_PLL_VCO_EN (1 << 31) ++#define DSI_PLL_LDO_GATE (1 << 30) ++#define DSI_PLL_P1_POST_DIV_SHIFT 17 ++#define DSI_PLL_P1_POST_DIV_MASK (0x1ff << 17) ++#define DSI_PLL_P2_MUX_DSI0_DIV2 (1 << 13) ++#define DSI_PLL_P3_MUX_DSI1_DIV2 (1 << 12) ++#define DSI_PLL_MUX_MASK (3 << 9) ++#define DSI_PLL_MUX_DSI0_DSIPLL (0 << 10) ++#define DSI_PLL_MUX_DSI0_CCK (1 << 10) ++#define DSI_PLL_MUX_DSI1_DSIPLL (0 << 9) ++#define DSI_PLL_MUX_DSI1_CCK (1 << 9) ++#define DSI_PLL_CLK_GATE_MASK (0xf << 5) ++#define DSI_PLL_CLK_GATE_DSI0_DSIPLL (1 << 8) ++#define DSI_PLL_CLK_GATE_DSI1_DSIPLL (1 << 7) ++#define DSI_PLL_CLK_GATE_DSI0_CCK (1 << 6) ++#define DSI_PLL_CLK_GATE_DSI1_CCK (1 << 5) ++#define DSI_PLL_LOCK (1 << 0) ++#define CCK_REG_DSI_PLL_DIVIDER 0x4c ++#define DSI_PLL_LFSR (1 << 31) ++#define DSI_PLL_FRACTION_EN (1 << 30) ++#define DSI_PLL_FRAC_COUNTER_SHIFT 27 ++#define DSI_PLL_FRAC_COUNTER_MASK (7 << 27) ++#define DSI_PLL_USYNC_CNT_SHIFT 18 ++#define DSI_PLL_USYNC_CNT_MASK (0x1ff << 18) ++#define DSI_PLL_N1_DIV_SHIFT 16 ++#define DSI_PLL_N1_DIV_MASK (3 << 16) ++#define DSI_PLL_M1_DIV_SHIFT 0 ++#define DSI_PLL_M1_DIV_MASK (0x1ff << 0) ++#define CCK_CZ_CLOCK_CONTROL 0x62 ++#define CCK_GPLL_CLOCK_CONTROL 0x67 ++#define CCK_DISPLAY_CLOCK_CONTROL 0x6b ++#define CCK_DISPLAY_REF_CLOCK_CONTROL 0x6c ++#define CCK_TRUNK_FORCE_ON (1 << 17) ++#define CCK_TRUNK_FORCE_OFF (1 << 16) ++#define CCK_FREQUENCY_STATUS (0x1f << 8) ++#define CCK_FREQUENCY_STATUS_SHIFT 8 ++#define CCK_FREQUENCY_VALUES (0x1f << 0) ++ ++/* DPIO registers */ ++#define DPIO_DEVFN 0 ++ ++#define DPIO_CTL _MMIO(VLV_DISPLAY_BASE + 0x2110) ++#define DPIO_MODSEL1 (1 << 3) /* if ref clk b == 27 */ ++#define DPIO_MODSEL0 (1 << 2) /* if ref clk a == 27 */ ++#define DPIO_SFR_BYPASS (1 << 1) ++#define DPIO_CMNRST (1 << 0) ++ ++#define DPIO_PHY(pipe) ((pipe) >> 1) ++#define DPIO_PHY_IOSF_PORT(phy) (dev_priv->dpio_phy_iosf_port[phy]) ++ ++/* ++ * Per pipe/PLL DPIO regs ++ */ ++#define _VLV_PLL_DW3_CH0 0x800c ++#define DPIO_POST_DIV_SHIFT (28) /* 3 bits */ ++#define DPIO_POST_DIV_DAC 0 ++#define DPIO_POST_DIV_HDMIDP 1 /* DAC 225-400M rate */ ++#define DPIO_POST_DIV_LVDS1 2 ++#define DPIO_POST_DIV_LVDS2 3 ++#define DPIO_K_SHIFT (24) /* 4 bits */ ++#define DPIO_P1_SHIFT (21) /* 3 bits */ ++#define DPIO_P2_SHIFT (16) /* 5 bits */ ++#define DPIO_N_SHIFT (12) /* 4 bits */ ++#define DPIO_ENABLE_CALIBRATION (1 << 11) ++#define DPIO_M1DIV_SHIFT (8) /* 3 bits */ ++#define DPIO_M2DIV_MASK 0xff ++#define _VLV_PLL_DW3_CH1 0x802c ++#define VLV_PLL_DW3(ch) _PIPE(ch, _VLV_PLL_DW3_CH0, _VLV_PLL_DW3_CH1) ++ ++#define _VLV_PLL_DW5_CH0 0x8014 ++#define DPIO_REFSEL_OVERRIDE 27 ++#define DPIO_PLL_MODESEL_SHIFT 24 /* 3 bits */ ++#define DPIO_BIAS_CURRENT_CTL_SHIFT 21 /* 3 bits, always 0x7 */ ++#define DPIO_PLL_REFCLK_SEL_SHIFT 16 /* 2 bits */ ++#define DPIO_PLL_REFCLK_SEL_MASK 3 ++#define DPIO_DRIVER_CTL_SHIFT 12 /* always set to 0x8 */ ++#define DPIO_CLK_BIAS_CTL_SHIFT 8 /* always set to 0x5 */ ++#define _VLV_PLL_DW5_CH1 0x8034 ++#define VLV_PLL_DW5(ch) _PIPE(ch, _VLV_PLL_DW5_CH0, _VLV_PLL_DW5_CH1) ++ ++#define _VLV_PLL_DW7_CH0 0x801c ++#define _VLV_PLL_DW7_CH1 0x803c ++#define VLV_PLL_DW7(ch) _PIPE(ch, _VLV_PLL_DW7_CH0, _VLV_PLL_DW7_CH1) ++ ++#define _VLV_PLL_DW8_CH0 0x8040 ++#define _VLV_PLL_DW8_CH1 0x8060 ++#define VLV_PLL_DW8(ch) _PIPE(ch, _VLV_PLL_DW8_CH0, _VLV_PLL_DW8_CH1) ++ ++#define VLV_PLL_DW9_BCAST 0xc044 ++#define _VLV_PLL_DW9_CH0 0x8044 ++#define _VLV_PLL_DW9_CH1 0x8064 ++#define VLV_PLL_DW9(ch) _PIPE(ch, _VLV_PLL_DW9_CH0, _VLV_PLL_DW9_CH1) ++ ++#define _VLV_PLL_DW10_CH0 0x8048 ++#define _VLV_PLL_DW10_CH1 0x8068 ++#define VLV_PLL_DW10(ch) _PIPE(ch, _VLV_PLL_DW10_CH0, _VLV_PLL_DW10_CH1) ++ ++#define _VLV_PLL_DW11_CH0 0x804c ++#define _VLV_PLL_DW11_CH1 0x806c ++#define VLV_PLL_DW11(ch) _PIPE(ch, _VLV_PLL_DW11_CH0, _VLV_PLL_DW11_CH1) ++ ++/* Spec for ref block start counts at DW10 */ ++#define VLV_REF_DW13 0x80ac ++ ++#define VLV_CMN_DW0 0x8100 ++ ++/* ++ * Per DDI channel DPIO regs ++ */ ++ ++#define _VLV_PCS_DW0_CH0 0x8200 ++#define _VLV_PCS_DW0_CH1 0x8400 ++#define DPIO_PCS_TX_LANE2_RESET (1 << 16) ++#define DPIO_PCS_TX_LANE1_RESET (1 << 7) ++#define DPIO_LEFT_TXFIFO_RST_MASTER2 (1 << 4) ++#define DPIO_RIGHT_TXFIFO_RST_MASTER2 (1 << 3) ++#define VLV_PCS_DW0(ch) _PORT(ch, _VLV_PCS_DW0_CH0, _VLV_PCS_DW0_CH1) ++ ++#define _VLV_PCS01_DW0_CH0 0x200 ++#define _VLV_PCS23_DW0_CH0 0x400 ++#define _VLV_PCS01_DW0_CH1 0x2600 ++#define _VLV_PCS23_DW0_CH1 0x2800 ++#define VLV_PCS01_DW0(ch) _PORT(ch, _VLV_PCS01_DW0_CH0, _VLV_PCS01_DW0_CH1) ++#define VLV_PCS23_DW0(ch) _PORT(ch, _VLV_PCS23_DW0_CH0, _VLV_PCS23_DW0_CH1) ++ ++#define _VLV_PCS_DW1_CH0 0x8204 ++#define _VLV_PCS_DW1_CH1 0x8404 ++#define CHV_PCS_REQ_SOFTRESET_EN (1 << 23) ++#define DPIO_PCS_CLK_CRI_RXEB_EIOS_EN (1 << 22) ++#define DPIO_PCS_CLK_CRI_RXDIGFILTSG_EN (1 << 21) ++#define DPIO_PCS_CLK_DATAWIDTH_SHIFT (6) ++#define DPIO_PCS_CLK_SOFT_RESET (1 << 5) ++#define VLV_PCS_DW1(ch) _PORT(ch, _VLV_PCS_DW1_CH0, _VLV_PCS_DW1_CH1) ++ ++#define _VLV_PCS01_DW1_CH0 0x204 ++#define _VLV_PCS23_DW1_CH0 0x404 ++#define _VLV_PCS01_DW1_CH1 0x2604 ++#define _VLV_PCS23_DW1_CH1 0x2804 ++#define VLV_PCS01_DW1(ch) _PORT(ch, _VLV_PCS01_DW1_CH0, _VLV_PCS01_DW1_CH1) ++#define VLV_PCS23_DW1(ch) _PORT(ch, _VLV_PCS23_DW1_CH0, _VLV_PCS23_DW1_CH1) ++ ++#define _VLV_PCS_DW8_CH0 0x8220 ++#define _VLV_PCS_DW8_CH1 0x8420 ++#define CHV_PCS_USEDCLKCHANNEL_OVRRIDE (1 << 20) ++#define CHV_PCS_USEDCLKCHANNEL (1 << 21) ++#define VLV_PCS_DW8(ch) _PORT(ch, _VLV_PCS_DW8_CH0, _VLV_PCS_DW8_CH1) ++ ++#define _VLV_PCS01_DW8_CH0 0x0220 ++#define _VLV_PCS23_DW8_CH0 0x0420 ++#define _VLV_PCS01_DW8_CH1 0x2620 ++#define _VLV_PCS23_DW8_CH1 0x2820 ++#define VLV_PCS01_DW8(port) _PORT(port, _VLV_PCS01_DW8_CH0, _VLV_PCS01_DW8_CH1) ++#define VLV_PCS23_DW8(port) _PORT(port, _VLV_PCS23_DW8_CH0, _VLV_PCS23_DW8_CH1) ++ ++#define _VLV_PCS_DW9_CH0 0x8224 ++#define _VLV_PCS_DW9_CH1 0x8424 ++#define DPIO_PCS_TX2MARGIN_MASK (0x7 << 13) ++#define DPIO_PCS_TX2MARGIN_000 (0 << 13) ++#define DPIO_PCS_TX2MARGIN_101 (1 << 13) ++#define DPIO_PCS_TX1MARGIN_MASK (0x7 << 10) ++#define DPIO_PCS_TX1MARGIN_000 (0 << 10) ++#define DPIO_PCS_TX1MARGIN_101 (1 << 10) ++#define VLV_PCS_DW9(ch) _PORT(ch, _VLV_PCS_DW9_CH0, _VLV_PCS_DW9_CH1) ++ ++#define _VLV_PCS01_DW9_CH0 0x224 ++#define _VLV_PCS23_DW9_CH0 0x424 ++#define _VLV_PCS01_DW9_CH1 0x2624 ++#define _VLV_PCS23_DW9_CH1 0x2824 ++#define VLV_PCS01_DW9(ch) _PORT(ch, _VLV_PCS01_DW9_CH0, _VLV_PCS01_DW9_CH1) ++#define VLV_PCS23_DW9(ch) _PORT(ch, _VLV_PCS23_DW9_CH0, _VLV_PCS23_DW9_CH1) ++ ++#define _CHV_PCS_DW10_CH0 0x8228 ++#define _CHV_PCS_DW10_CH1 0x8428 ++#define DPIO_PCS_SWING_CALC_TX0_TX2 (1 << 30) ++#define DPIO_PCS_SWING_CALC_TX1_TX3 (1 << 31) ++#define DPIO_PCS_TX2DEEMP_MASK (0xf << 24) ++#define DPIO_PCS_TX2DEEMP_9P5 (0 << 24) ++#define DPIO_PCS_TX2DEEMP_6P0 (2 << 24) ++#define DPIO_PCS_TX1DEEMP_MASK (0xf << 16) ++#define DPIO_PCS_TX1DEEMP_9P5 (0 << 16) ++#define DPIO_PCS_TX1DEEMP_6P0 (2 << 16) ++#define CHV_PCS_DW10(ch) _PORT(ch, _CHV_PCS_DW10_CH0, _CHV_PCS_DW10_CH1) ++ ++#define _VLV_PCS01_DW10_CH0 0x0228 ++#define _VLV_PCS23_DW10_CH0 0x0428 ++#define _VLV_PCS01_DW10_CH1 0x2628 ++#define _VLV_PCS23_DW10_CH1 0x2828 ++#define VLV_PCS01_DW10(port) _PORT(port, _VLV_PCS01_DW10_CH0, _VLV_PCS01_DW10_CH1) ++#define VLV_PCS23_DW10(port) _PORT(port, _VLV_PCS23_DW10_CH0, _VLV_PCS23_DW10_CH1) ++ ++#define _VLV_PCS_DW11_CH0 0x822c ++#define _VLV_PCS_DW11_CH1 0x842c ++#define DPIO_TX2_STAGGER_MASK(x) ((x) << 24) ++#define DPIO_LANEDESKEW_STRAP_OVRD (1 << 3) ++#define DPIO_LEFT_TXFIFO_RST_MASTER (1 << 1) ++#define DPIO_RIGHT_TXFIFO_RST_MASTER (1 << 0) ++#define VLV_PCS_DW11(ch) _PORT(ch, _VLV_PCS_DW11_CH0, _VLV_PCS_DW11_CH1) ++ ++#define _VLV_PCS01_DW11_CH0 0x022c ++#define _VLV_PCS23_DW11_CH0 0x042c ++#define _VLV_PCS01_DW11_CH1 0x262c ++#define _VLV_PCS23_DW11_CH1 0x282c ++#define VLV_PCS01_DW11(ch) _PORT(ch, _VLV_PCS01_DW11_CH0, _VLV_PCS01_DW11_CH1) ++#define VLV_PCS23_DW11(ch) _PORT(ch, _VLV_PCS23_DW11_CH0, _VLV_PCS23_DW11_CH1) ++ ++#define _VLV_PCS01_DW12_CH0 0x0230 ++#define _VLV_PCS23_DW12_CH0 0x0430 ++#define _VLV_PCS01_DW12_CH1 0x2630 ++#define _VLV_PCS23_DW12_CH1 0x2830 ++#define VLV_PCS01_DW12(ch) _PORT(ch, _VLV_PCS01_DW12_CH0, _VLV_PCS01_DW12_CH1) ++#define VLV_PCS23_DW12(ch) _PORT(ch, _VLV_PCS23_DW12_CH0, _VLV_PCS23_DW12_CH1) ++ ++#define _VLV_PCS_DW12_CH0 0x8230 ++#define _VLV_PCS_DW12_CH1 0x8430 ++#define DPIO_TX2_STAGGER_MULT(x) ((x) << 20) ++#define DPIO_TX1_STAGGER_MULT(x) ((x) << 16) ++#define DPIO_TX1_STAGGER_MASK(x) ((x) << 8) ++#define DPIO_LANESTAGGER_STRAP_OVRD (1 << 6) ++#define DPIO_LANESTAGGER_STRAP(x) ((x) << 0) ++#define VLV_PCS_DW12(ch) _PORT(ch, _VLV_PCS_DW12_CH0, _VLV_PCS_DW12_CH1) ++ ++#define _VLV_PCS_DW14_CH0 0x8238 ++#define _VLV_PCS_DW14_CH1 0x8438 ++#define VLV_PCS_DW14(ch) _PORT(ch, _VLV_PCS_DW14_CH0, _VLV_PCS_DW14_CH1) ++ ++#define _VLV_PCS_DW23_CH0 0x825c ++#define _VLV_PCS_DW23_CH1 0x845c ++#define VLV_PCS_DW23(ch) _PORT(ch, _VLV_PCS_DW23_CH0, _VLV_PCS_DW23_CH1) ++ ++#define _VLV_TX_DW2_CH0 0x8288 ++#define _VLV_TX_DW2_CH1 0x8488 ++#define DPIO_SWING_MARGIN000_SHIFT 16 ++#define DPIO_SWING_MARGIN000_MASK (0xff << DPIO_SWING_MARGIN000_SHIFT) ++#define DPIO_UNIQ_TRANS_SCALE_SHIFT 8 ++#define VLV_TX_DW2(ch) _PORT(ch, _VLV_TX_DW2_CH0, _VLV_TX_DW2_CH1) ++ ++#define _VLV_TX_DW3_CH0 0x828c ++#define _VLV_TX_DW3_CH1 0x848c ++/* The following bit for CHV phy */ ++#define DPIO_TX_UNIQ_TRANS_SCALE_EN (1 << 27) ++#define DPIO_SWING_MARGIN101_SHIFT 16 ++#define DPIO_SWING_MARGIN101_MASK (0xff << DPIO_SWING_MARGIN101_SHIFT) ++#define VLV_TX_DW3(ch) _PORT(ch, _VLV_TX_DW3_CH0, _VLV_TX_DW3_CH1) ++ ++#define _VLV_TX_DW4_CH0 0x8290 ++#define _VLV_TX_DW4_CH1 0x8490 ++#define DPIO_SWING_DEEMPH9P5_SHIFT 24 ++#define DPIO_SWING_DEEMPH9P5_MASK (0xff << DPIO_SWING_DEEMPH9P5_SHIFT) ++#define DPIO_SWING_DEEMPH6P0_SHIFT 16 ++#define DPIO_SWING_DEEMPH6P0_MASK (0xff << DPIO_SWING_DEEMPH6P0_SHIFT) ++#define VLV_TX_DW4(ch) _PORT(ch, _VLV_TX_DW4_CH0, _VLV_TX_DW4_CH1) ++ ++#define _VLV_TX3_DW4_CH0 0x690 ++#define _VLV_TX3_DW4_CH1 0x2a90 ++#define VLV_TX3_DW4(ch) _PORT(ch, _VLV_TX3_DW4_CH0, _VLV_TX3_DW4_CH1) ++ ++#define _VLV_TX_DW5_CH0 0x8294 ++#define _VLV_TX_DW5_CH1 0x8494 ++#define DPIO_TX_OCALINIT_EN (1 << 31) ++#define VLV_TX_DW5(ch) _PORT(ch, _VLV_TX_DW5_CH0, _VLV_TX_DW5_CH1) ++ ++#define _VLV_TX_DW11_CH0 0x82ac ++#define _VLV_TX_DW11_CH1 0x84ac ++#define VLV_TX_DW11(ch) _PORT(ch, _VLV_TX_DW11_CH0, _VLV_TX_DW11_CH1) ++ ++#define _VLV_TX_DW14_CH0 0x82b8 ++#define _VLV_TX_DW14_CH1 0x84b8 ++#define VLV_TX_DW14(ch) _PORT(ch, _VLV_TX_DW14_CH0, _VLV_TX_DW14_CH1) ++ ++/* CHV dpPhy registers */ ++#define _CHV_PLL_DW0_CH0 0x8000 ++#define _CHV_PLL_DW0_CH1 0x8180 ++#define CHV_PLL_DW0(ch) _PIPE(ch, _CHV_PLL_DW0_CH0, _CHV_PLL_DW0_CH1) ++ ++#define _CHV_PLL_DW1_CH0 0x8004 ++#define _CHV_PLL_DW1_CH1 0x8184 ++#define DPIO_CHV_N_DIV_SHIFT 8 ++#define DPIO_CHV_M1_DIV_BY_2 (0 << 0) ++#define CHV_PLL_DW1(ch) _PIPE(ch, _CHV_PLL_DW1_CH0, _CHV_PLL_DW1_CH1) ++ ++#define _CHV_PLL_DW2_CH0 0x8008 ++#define _CHV_PLL_DW2_CH1 0x8188 ++#define CHV_PLL_DW2(ch) _PIPE(ch, _CHV_PLL_DW2_CH0, _CHV_PLL_DW2_CH1) ++ ++#define _CHV_PLL_DW3_CH0 0x800c ++#define _CHV_PLL_DW3_CH1 0x818c ++#define DPIO_CHV_FRAC_DIV_EN (1 << 16) ++#define DPIO_CHV_FIRST_MOD (0 << 8) ++#define DPIO_CHV_SECOND_MOD (1 << 8) ++#define DPIO_CHV_FEEDFWD_GAIN_SHIFT 0 ++#define DPIO_CHV_FEEDFWD_GAIN_MASK (0xF << 0) ++#define CHV_PLL_DW3(ch) _PIPE(ch, _CHV_PLL_DW3_CH0, _CHV_PLL_DW3_CH1) ++ ++#define _CHV_PLL_DW6_CH0 0x8018 ++#define _CHV_PLL_DW6_CH1 0x8198 ++#define DPIO_CHV_GAIN_CTRL_SHIFT 16 ++#define DPIO_CHV_INT_COEFF_SHIFT 8 ++#define DPIO_CHV_PROP_COEFF_SHIFT 0 ++#define CHV_PLL_DW6(ch) _PIPE(ch, _CHV_PLL_DW6_CH0, _CHV_PLL_DW6_CH1) ++ ++#define _CHV_PLL_DW8_CH0 0x8020 ++#define _CHV_PLL_DW8_CH1 0x81A0 ++#define DPIO_CHV_TDC_TARGET_CNT_SHIFT 0 ++#define DPIO_CHV_TDC_TARGET_CNT_MASK (0x3FF << 0) ++#define CHV_PLL_DW8(ch) _PIPE(ch, _CHV_PLL_DW8_CH0, _CHV_PLL_DW8_CH1) ++ ++#define _CHV_PLL_DW9_CH0 0x8024 ++#define _CHV_PLL_DW9_CH1 0x81A4 ++#define DPIO_CHV_INT_LOCK_THRESHOLD_SHIFT 1 /* 3 bits */ ++#define DPIO_CHV_INT_LOCK_THRESHOLD_MASK (7 << 1) ++#define DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE 1 /* 1: coarse & 0 : fine */ ++#define CHV_PLL_DW9(ch) _PIPE(ch, _CHV_PLL_DW9_CH0, _CHV_PLL_DW9_CH1) ++ ++#define _CHV_CMN_DW0_CH0 0x8100 ++#define DPIO_ALLDL_POWERDOWN_SHIFT_CH0 19 ++#define DPIO_ANYDL_POWERDOWN_SHIFT_CH0 18 ++#define DPIO_ALLDL_POWERDOWN (1 << 1) ++#define DPIO_ANYDL_POWERDOWN (1 << 0) ++ ++#define _CHV_CMN_DW5_CH0 0x8114 ++#define CHV_BUFRIGHTENA1_DISABLE (0 << 20) ++#define CHV_BUFRIGHTENA1_NORMAL (1 << 20) ++#define CHV_BUFRIGHTENA1_FORCE (3 << 20) ++#define CHV_BUFRIGHTENA1_MASK (3 << 20) ++#define CHV_BUFLEFTENA1_DISABLE (0 << 22) ++#define CHV_BUFLEFTENA1_NORMAL (1 << 22) ++#define CHV_BUFLEFTENA1_FORCE (3 << 22) ++#define CHV_BUFLEFTENA1_MASK (3 << 22) ++ ++#define _CHV_CMN_DW13_CH0 0x8134 ++#define _CHV_CMN_DW0_CH1 0x8080 ++#define DPIO_CHV_S1_DIV_SHIFT 21 ++#define DPIO_CHV_P1_DIV_SHIFT 13 /* 3 bits */ ++#define DPIO_CHV_P2_DIV_SHIFT 8 /* 5 bits */ ++#define DPIO_CHV_K_DIV_SHIFT 4 ++#define DPIO_PLL_FREQLOCK (1 << 1) ++#define DPIO_PLL_LOCK (1 << 0) ++#define CHV_CMN_DW13(ch) _PIPE(ch, _CHV_CMN_DW13_CH0, _CHV_CMN_DW0_CH1) ++ ++#define _CHV_CMN_DW14_CH0 0x8138 ++#define _CHV_CMN_DW1_CH1 0x8084 ++#define DPIO_AFC_RECAL (1 << 14) ++#define DPIO_DCLKP_EN (1 << 13) ++#define CHV_BUFLEFTENA2_DISABLE (0 << 17) /* CL2 DW1 only */ ++#define CHV_BUFLEFTENA2_NORMAL (1 << 17) /* CL2 DW1 only */ ++#define CHV_BUFLEFTENA2_FORCE (3 << 17) /* CL2 DW1 only */ ++#define CHV_BUFLEFTENA2_MASK (3 << 17) /* CL2 DW1 only */ ++#define CHV_BUFRIGHTENA2_DISABLE (0 << 19) /* CL2 DW1 only */ ++#define CHV_BUFRIGHTENA2_NORMAL (1 << 19) /* CL2 DW1 only */ ++#define CHV_BUFRIGHTENA2_FORCE (3 << 19) /* CL2 DW1 only */ ++#define CHV_BUFRIGHTENA2_MASK (3 << 19) /* CL2 DW1 only */ ++#define CHV_CMN_DW14(ch) _PIPE(ch, _CHV_CMN_DW14_CH0, _CHV_CMN_DW1_CH1) ++ ++#define _CHV_CMN_DW19_CH0 0x814c ++#define _CHV_CMN_DW6_CH1 0x8098 ++#define DPIO_ALLDL_POWERDOWN_SHIFT_CH1 30 /* CL2 DW6 only */ ++#define DPIO_ANYDL_POWERDOWN_SHIFT_CH1 29 /* CL2 DW6 only */ ++#define DPIO_DYNPWRDOWNEN_CH1 (1 << 28) /* CL2 DW6 only */ ++#define CHV_CMN_USEDCLKCHANNEL (1 << 13) ++ ++#define CHV_CMN_DW19(ch) _PIPE(ch, _CHV_CMN_DW19_CH0, _CHV_CMN_DW6_CH1) ++ ++#define CHV_CMN_DW28 0x8170 ++#define DPIO_CL1POWERDOWNEN (1 << 23) ++#define DPIO_DYNPWRDOWNEN_CH0 (1 << 22) ++#define DPIO_SUS_CLK_CONFIG_ON (0 << 0) ++#define DPIO_SUS_CLK_CONFIG_CLKREQ (1 << 0) ++#define DPIO_SUS_CLK_CONFIG_GATE (2 << 0) ++#define DPIO_SUS_CLK_CONFIG_GATE_CLKREQ (3 << 0) ++ ++#define CHV_CMN_DW30 0x8178 ++#define DPIO_CL2_LDOFUSE_PWRENB (1 << 6) ++#define DPIO_LRC_BYPASS (1 << 3) ++ ++#define _TXLANE(ch, lane, offset) ((ch ? 0x2400 : 0) + \ ++ (lane) * 0x200 + (offset)) ++ ++#define CHV_TX_DW0(ch, lane) _TXLANE(ch, lane, 0x80) ++#define CHV_TX_DW1(ch, lane) _TXLANE(ch, lane, 0x84) ++#define CHV_TX_DW2(ch, lane) _TXLANE(ch, lane, 0x88) ++#define CHV_TX_DW3(ch, lane) _TXLANE(ch, lane, 0x8c) ++#define CHV_TX_DW4(ch, lane) _TXLANE(ch, lane, 0x90) ++#define CHV_TX_DW5(ch, lane) _TXLANE(ch, lane, 0x94) ++#define CHV_TX_DW6(ch, lane) _TXLANE(ch, lane, 0x98) ++#define CHV_TX_DW7(ch, lane) _TXLANE(ch, lane, 0x9c) ++#define CHV_TX_DW8(ch, lane) _TXLANE(ch, lane, 0xa0) ++#define CHV_TX_DW9(ch, lane) _TXLANE(ch, lane, 0xa4) ++#define CHV_TX_DW10(ch, lane) _TXLANE(ch, lane, 0xa8) ++#define CHV_TX_DW11(ch, lane) _TXLANE(ch, lane, 0xac) ++#define DPIO_FRC_LATENCY_SHFIT 8 ++#define CHV_TX_DW14(ch, lane) _TXLANE(ch, lane, 0xb8) ++#define DPIO_UPAR_SHIFT 30 ++ ++/* BXT PHY registers */ ++#define _BXT_PHY0_BASE 0x6C000 ++#define _BXT_PHY1_BASE 0x162000 ++#define _BXT_PHY2_BASE 0x163000 ++#define BXT_PHY_BASE(phy) _PHY3((phy), _BXT_PHY0_BASE, \ ++ _BXT_PHY1_BASE, \ ++ _BXT_PHY2_BASE) ++ ++#define _BXT_PHY(phy, reg) \ ++ _MMIO(BXT_PHY_BASE(phy) - _BXT_PHY0_BASE + (reg)) ++ ++#define _BXT_PHY_CH(phy, ch, reg_ch0, reg_ch1) \ ++ (BXT_PHY_BASE(phy) + _PIPE((ch), (reg_ch0) - _BXT_PHY0_BASE, \ ++ (reg_ch1) - _BXT_PHY0_BASE)) ++#define _MMIO_BXT_PHY_CH(phy, ch, reg_ch0, reg_ch1) \ ++ _MMIO(_BXT_PHY_CH(phy, ch, reg_ch0, reg_ch1)) ++ ++#define BXT_P_CR_GT_DISP_PWRON _MMIO(0x138090) ++#define MIPIO_RST_CTRL (1 << 2) ++ ++#define _BXT_PHY_CTL_DDI_A 0x64C00 ++#define _BXT_PHY_CTL_DDI_B 0x64C10 ++#define _BXT_PHY_CTL_DDI_C 0x64C20 ++#define BXT_PHY_CMNLANE_POWERDOWN_ACK (1 << 10) ++#define BXT_PHY_LANE_POWERDOWN_ACK (1 << 9) ++#define BXT_PHY_LANE_ENABLED (1 << 8) ++#define BXT_PHY_CTL(port) _MMIO_PORT(port, _BXT_PHY_CTL_DDI_A, \ ++ _BXT_PHY_CTL_DDI_B) ++ ++#define _PHY_CTL_FAMILY_EDP 0x64C80 ++#define _PHY_CTL_FAMILY_DDI 0x64C90 ++#define _PHY_CTL_FAMILY_DDI_C 0x64CA0 ++#define COMMON_RESET_DIS (1 << 31) ++#define BXT_PHY_CTL_FAMILY(phy) _MMIO_PHY3((phy), _PHY_CTL_FAMILY_DDI, \ ++ _PHY_CTL_FAMILY_EDP, \ ++ _PHY_CTL_FAMILY_DDI_C) ++ ++/* BXT PHY PLL registers */ ++#define _PORT_PLL_A 0x46074 ++#define _PORT_PLL_B 0x46078 ++#define _PORT_PLL_C 0x4607c ++#define PORT_PLL_ENABLE (1 << 31) ++#define PORT_PLL_LOCK (1 << 30) ++#define PORT_PLL_REF_SEL (1 << 27) ++#define PORT_PLL_POWER_ENABLE (1 << 26) ++#define PORT_PLL_POWER_STATE (1 << 25) ++#define BXT_PORT_PLL_ENABLE(port) _MMIO_PORT(port, _PORT_PLL_A, _PORT_PLL_B) ++ ++#define _PORT_PLL_EBB_0_A 0x162034 ++#define _PORT_PLL_EBB_0_B 0x6C034 ++#define _PORT_PLL_EBB_0_C 0x6C340 ++#define PORT_PLL_P1_SHIFT 13 ++#define PORT_PLL_P1_MASK (0x07 << PORT_PLL_P1_SHIFT) ++#define PORT_PLL_P1(x) ((x) << PORT_PLL_P1_SHIFT) ++#define PORT_PLL_P2_SHIFT 8 ++#define PORT_PLL_P2_MASK (0x1f << PORT_PLL_P2_SHIFT) ++#define PORT_PLL_P2(x) ((x) << PORT_PLL_P2_SHIFT) ++#define BXT_PORT_PLL_EBB_0(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ ++ _PORT_PLL_EBB_0_B, \ ++ _PORT_PLL_EBB_0_C) ++ ++#define _PORT_PLL_EBB_4_A 0x162038 ++#define _PORT_PLL_EBB_4_B 0x6C038 ++#define _PORT_PLL_EBB_4_C 0x6C344 ++#define PORT_PLL_10BIT_CLK_ENABLE (1 << 13) ++#define PORT_PLL_RECALIBRATE (1 << 14) ++#define BXT_PORT_PLL_EBB_4(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ ++ _PORT_PLL_EBB_4_B, \ ++ _PORT_PLL_EBB_4_C) ++ ++#define _PORT_PLL_0_A 0x162100 ++#define _PORT_PLL_0_B 0x6C100 ++#define _PORT_PLL_0_C 0x6C380 ++/* PORT_PLL_0_A */ ++#define PORT_PLL_M2_MASK 0xFF ++/* PORT_PLL_1_A */ ++#define PORT_PLL_N_SHIFT 8 ++#define PORT_PLL_N_MASK (0x0F << PORT_PLL_N_SHIFT) ++#define PORT_PLL_N(x) ((x) << PORT_PLL_N_SHIFT) ++/* PORT_PLL_2_A */ ++#define PORT_PLL_M2_FRAC_MASK 0x3FFFFF ++/* PORT_PLL_3_A */ ++#define PORT_PLL_M2_FRAC_ENABLE (1 << 16) ++/* PORT_PLL_6_A */ ++#define PORT_PLL_PROP_COEFF_MASK 0xF ++#define PORT_PLL_INT_COEFF_MASK (0x1F << 8) ++#define PORT_PLL_INT_COEFF(x) ((x) << 8) ++#define PORT_PLL_GAIN_CTL_MASK (0x07 << 16) ++#define PORT_PLL_GAIN_CTL(x) ((x) << 16) ++/* PORT_PLL_8_A */ ++#define PORT_PLL_TARGET_CNT_MASK 0x3FF ++/* PORT_PLL_9_A */ ++#define PORT_PLL_LOCK_THRESHOLD_SHIFT 1 ++#define PORT_PLL_LOCK_THRESHOLD_MASK (0x7 << PORT_PLL_LOCK_THRESHOLD_SHIFT) ++/* PORT_PLL_10_A */ ++#define PORT_PLL_DCO_AMP_OVR_EN_H (1 << 27) ++#define PORT_PLL_DCO_AMP_DEFAULT 15 ++#define PORT_PLL_DCO_AMP_MASK 0x3c00 ++#define PORT_PLL_DCO_AMP(x) ((x) << 10) ++#define _PORT_PLL_BASE(phy, ch) _BXT_PHY_CH(phy, ch, \ ++ _PORT_PLL_0_B, \ ++ _PORT_PLL_0_C) ++#define BXT_PORT_PLL(phy, ch, idx) _MMIO(_PORT_PLL_BASE(phy, ch) + \ ++ (idx) * 4) ++ ++/* BXT PHY common lane registers */ ++#define _PORT_CL1CM_DW0_A 0x162000 ++#define _PORT_CL1CM_DW0_BC 0x6C000 ++#define PHY_POWER_GOOD (1 << 16) ++#define PHY_RESERVED (1 << 7) ++#define BXT_PORT_CL1CM_DW0(phy) _BXT_PHY((phy), _PORT_CL1CM_DW0_BC) ++ ++#define _PORT_CL1CM_DW9_A 0x162024 ++#define _PORT_CL1CM_DW9_BC 0x6C024 ++#define IREF0RC_OFFSET_SHIFT 8 ++#define IREF0RC_OFFSET_MASK (0xFF << IREF0RC_OFFSET_SHIFT) ++#define BXT_PORT_CL1CM_DW9(phy) _BXT_PHY((phy), _PORT_CL1CM_DW9_BC) ++ ++#define _PORT_CL1CM_DW10_A 0x162028 ++#define _PORT_CL1CM_DW10_BC 0x6C028 ++#define IREF1RC_OFFSET_SHIFT 8 ++#define IREF1RC_OFFSET_MASK (0xFF << IREF1RC_OFFSET_SHIFT) ++#define BXT_PORT_CL1CM_DW10(phy) _BXT_PHY((phy), _PORT_CL1CM_DW10_BC) ++ ++#define _PORT_CL1CM_DW28_A 0x162070 ++#define _PORT_CL1CM_DW28_BC 0x6C070 ++#define OCL1_POWER_DOWN_EN (1 << 23) ++#define DW28_OLDO_DYN_PWR_DOWN_EN (1 << 22) ++#define SUS_CLK_CONFIG 0x3 ++#define BXT_PORT_CL1CM_DW28(phy) _BXT_PHY((phy), _PORT_CL1CM_DW28_BC) ++ ++#define _PORT_CL1CM_DW30_A 0x162078 ++#define _PORT_CL1CM_DW30_BC 0x6C078 ++#define OCL2_LDOFUSE_PWR_DIS (1 << 6) ++#define BXT_PORT_CL1CM_DW30(phy) _BXT_PHY((phy), _PORT_CL1CM_DW30_BC) ++ ++/* ++ * CNL/ICL Port/COMBO-PHY Registers ++ */ ++#define _ICL_COMBOPHY_A 0x162000 ++#define _ICL_COMBOPHY_B 0x6C000 ++#define _ICL_COMBOPHY(port) _PICK(port, _ICL_COMBOPHY_A, \ ++ _ICL_COMBOPHY_B) ++ ++/* CNL/ICL Port CL_DW registers */ ++#define _ICL_PORT_CL_DW(dw, port) (_ICL_COMBOPHY(port) + \ ++ 4 * (dw)) ++ ++#define CNL_PORT_CL1CM_DW5 _MMIO(0x162014) ++#define ICL_PORT_CL_DW5(port) _MMIO(_ICL_PORT_CL_DW(5, port)) ++#define CL_POWER_DOWN_ENABLE (1 << 4) ++#define SUS_CLOCK_CONFIG (3 << 0) ++ ++#define ICL_PORT_CL_DW10(port) _MMIO(_ICL_PORT_CL_DW(10, port)) ++#define PG_SEQ_DELAY_OVERRIDE_MASK (3 << 25) ++#define PG_SEQ_DELAY_OVERRIDE_SHIFT 25 ++#define PG_SEQ_DELAY_OVERRIDE_ENABLE (1 << 24) ++#define PWR_UP_ALL_LANES (0x0 << 4) ++#define PWR_DOWN_LN_3_2_1 (0xe << 4) ++#define PWR_DOWN_LN_3_2 (0xc << 4) ++#define PWR_DOWN_LN_3 (0x8 << 4) ++#define PWR_DOWN_LN_2_1_0 (0x7 << 4) ++#define PWR_DOWN_LN_1_0 (0x3 << 4) ++#define PWR_DOWN_LN_1 (0x2 << 4) ++#define PWR_DOWN_LN_3_1 (0xa << 4) ++#define PWR_DOWN_LN_3_1_0 (0xb << 4) ++#define PWR_DOWN_LN_MASK (0xf << 4) ++#define PWR_DOWN_LN_SHIFT 4 ++ ++#define ICL_PORT_CL_DW12(port) _MMIO(_ICL_PORT_CL_DW(12, port)) ++#define ICL_LANE_ENABLE_AUX (1 << 0) ++ ++/* CNL/ICL Port COMP_DW registers */ ++#define _ICL_PORT_COMP 0x100 ++#define _ICL_PORT_COMP_DW(dw, port) (_ICL_COMBOPHY(port) + \ ++ _ICL_PORT_COMP + 4 * (dw)) ++ ++#define CNL_PORT_COMP_DW0 _MMIO(0x162100) ++#define ICL_PORT_COMP_DW0(port) _MMIO(_ICL_PORT_COMP_DW(0, port)) ++#define COMP_INIT (1 << 31) ++ ++#define CNL_PORT_COMP_DW1 _MMIO(0x162104) ++#define ICL_PORT_COMP_DW1(port) _MMIO(_ICL_PORT_COMP_DW(1, port)) ++ ++#define CNL_PORT_COMP_DW3 _MMIO(0x16210c) ++#define ICL_PORT_COMP_DW3(port) _MMIO(_ICL_PORT_COMP_DW(3, port)) ++#define PROCESS_INFO_DOT_0 (0 << 26) ++#define PROCESS_INFO_DOT_1 (1 << 26) ++#define PROCESS_INFO_DOT_4 (2 << 26) ++#define PROCESS_INFO_MASK (7 << 26) ++#define PROCESS_INFO_SHIFT 26 ++#define VOLTAGE_INFO_0_85V (0 << 24) ++#define VOLTAGE_INFO_0_95V (1 << 24) ++#define VOLTAGE_INFO_1_05V (2 << 24) ++#define VOLTAGE_INFO_MASK (3 << 24) ++#define VOLTAGE_INFO_SHIFT 24 ++ ++#define CNL_PORT_COMP_DW9 _MMIO(0x162124) ++#define ICL_PORT_COMP_DW9(port) _MMIO(_ICL_PORT_COMP_DW(9, port)) ++ ++#define CNL_PORT_COMP_DW10 _MMIO(0x162128) ++#define ICL_PORT_COMP_DW10(port) _MMIO(_ICL_PORT_COMP_DW(10, port)) ++ ++/* CNL/ICL Port PCS registers */ ++#define _CNL_PORT_PCS_DW1_GRP_AE 0x162304 ++#define _CNL_PORT_PCS_DW1_GRP_B 0x162384 ++#define _CNL_PORT_PCS_DW1_GRP_C 0x162B04 ++#define _CNL_PORT_PCS_DW1_GRP_D 0x162B84 ++#define _CNL_PORT_PCS_DW1_GRP_F 0x162A04 ++#define _CNL_PORT_PCS_DW1_LN0_AE 0x162404 ++#define _CNL_PORT_PCS_DW1_LN0_B 0x162604 ++#define _CNL_PORT_PCS_DW1_LN0_C 0x162C04 ++#define _CNL_PORT_PCS_DW1_LN0_D 0x162E04 ++#define _CNL_PORT_PCS_DW1_LN0_F 0x162804 ++#define CNL_PORT_PCS_DW1_GRP(port) _MMIO(_PICK(port, \ ++ _CNL_PORT_PCS_DW1_GRP_AE, \ ++ _CNL_PORT_PCS_DW1_GRP_B, \ ++ _CNL_PORT_PCS_DW1_GRP_C, \ ++ _CNL_PORT_PCS_DW1_GRP_D, \ ++ _CNL_PORT_PCS_DW1_GRP_AE, \ ++ _CNL_PORT_PCS_DW1_GRP_F)) ++#define CNL_PORT_PCS_DW1_LN0(port) _MMIO(_PICK(port, \ ++ _CNL_PORT_PCS_DW1_LN0_AE, \ ++ _CNL_PORT_PCS_DW1_LN0_B, \ ++ _CNL_PORT_PCS_DW1_LN0_C, \ ++ _CNL_PORT_PCS_DW1_LN0_D, \ ++ _CNL_PORT_PCS_DW1_LN0_AE, \ ++ _CNL_PORT_PCS_DW1_LN0_F)) ++ ++#define _ICL_PORT_PCS_AUX 0x300 ++#define _ICL_PORT_PCS_GRP 0x600 ++#define _ICL_PORT_PCS_LN(ln) (0x800 + (ln) * 0x100) ++#define _ICL_PORT_PCS_DW_AUX(dw, port) (_ICL_COMBOPHY(port) + \ ++ _ICL_PORT_PCS_AUX + 4 * (dw)) ++#define _ICL_PORT_PCS_DW_GRP(dw, port) (_ICL_COMBOPHY(port) + \ ++ _ICL_PORT_PCS_GRP + 4 * (dw)) ++#define _ICL_PORT_PCS_DW_LN(dw, ln, port) (_ICL_COMBOPHY(port) + \ ++ _ICL_PORT_PCS_LN(ln) + 4 * (dw)) ++#define ICL_PORT_PCS_DW1_AUX(port) _MMIO(_ICL_PORT_PCS_DW_AUX(1, port)) ++#define ICL_PORT_PCS_DW1_GRP(port) _MMIO(_ICL_PORT_PCS_DW_GRP(1, port)) ++#define ICL_PORT_PCS_DW1_LN0(port) _MMIO(_ICL_PORT_PCS_DW_LN(1, 0, port)) ++#define COMMON_KEEPER_EN (1 << 26) ++ ++/* CNL/ICL Port TX registers */ ++#define _CNL_PORT_TX_AE_GRP_OFFSET 0x162340 ++#define _CNL_PORT_TX_B_GRP_OFFSET 0x1623C0 ++#define _CNL_PORT_TX_C_GRP_OFFSET 0x162B40 ++#define _CNL_PORT_TX_D_GRP_OFFSET 0x162BC0 ++#define _CNL_PORT_TX_F_GRP_OFFSET 0x162A40 ++#define _CNL_PORT_TX_AE_LN0_OFFSET 0x162440 ++#define _CNL_PORT_TX_B_LN0_OFFSET 0x162640 ++#define _CNL_PORT_TX_C_LN0_OFFSET 0x162C40 ++#define _CNL_PORT_TX_D_LN0_OFFSET 0x162E40 ++#define _CNL_PORT_TX_F_LN0_OFFSET 0x162840 ++#define _CNL_PORT_TX_DW_GRP(dw, port) (_PICK((port), \ ++ _CNL_PORT_TX_AE_GRP_OFFSET, \ ++ _CNL_PORT_TX_B_GRP_OFFSET, \ ++ _CNL_PORT_TX_B_GRP_OFFSET, \ ++ _CNL_PORT_TX_D_GRP_OFFSET, \ ++ _CNL_PORT_TX_AE_GRP_OFFSET, \ ++ _CNL_PORT_TX_F_GRP_OFFSET) + \ ++ 4 * (dw)) ++#define _CNL_PORT_TX_DW_LN0(dw, port) (_PICK((port), \ ++ _CNL_PORT_TX_AE_LN0_OFFSET, \ ++ _CNL_PORT_TX_B_LN0_OFFSET, \ ++ _CNL_PORT_TX_B_LN0_OFFSET, \ ++ _CNL_PORT_TX_D_LN0_OFFSET, \ ++ _CNL_PORT_TX_AE_LN0_OFFSET, \ ++ _CNL_PORT_TX_F_LN0_OFFSET) + \ ++ 4 * (dw)) ++ ++#define _ICL_PORT_TX_AUX 0x380 ++#define _ICL_PORT_TX_GRP 0x680 ++#define _ICL_PORT_TX_LN(ln) (0x880 + (ln) * 0x100) ++ ++#define _ICL_PORT_TX_DW_AUX(dw, port) (_ICL_COMBOPHY(port) + \ ++ _ICL_PORT_TX_AUX + 4 * (dw)) ++#define _ICL_PORT_TX_DW_GRP(dw, port) (_ICL_COMBOPHY(port) + \ ++ _ICL_PORT_TX_GRP + 4 * (dw)) ++#define _ICL_PORT_TX_DW_LN(dw, ln, port) (_ICL_COMBOPHY(port) + \ ++ _ICL_PORT_TX_LN(ln) + 4 * (dw)) ++ ++#define CNL_PORT_TX_DW2_GRP(port) _MMIO(_CNL_PORT_TX_DW_GRP(2, port)) ++#define CNL_PORT_TX_DW2_LN0(port) _MMIO(_CNL_PORT_TX_DW_LN0(2, port)) ++#define ICL_PORT_TX_DW2_AUX(port) _MMIO(_ICL_PORT_TX_DW_AUX(2, port)) ++#define ICL_PORT_TX_DW2_GRP(port) _MMIO(_ICL_PORT_TX_DW_GRP(2, port)) ++#define ICL_PORT_TX_DW2_LN0(port) _MMIO(_ICL_PORT_TX_DW_LN(2, 0, port)) ++#define SWING_SEL_UPPER(x) (((x) >> 3) << 15) ++#define SWING_SEL_UPPER_MASK (1 << 15) ++#define SWING_SEL_LOWER(x) (((x) & 0x7) << 11) ++#define SWING_SEL_LOWER_MASK (0x7 << 11) ++#define FRC_LATENCY_OPTIM_MASK (0x7 << 8) ++#define FRC_LATENCY_OPTIM_VAL(x) ((x) << 8) ++#define RCOMP_SCALAR(x) ((x) << 0) ++#define RCOMP_SCALAR_MASK (0xFF << 0) ++ ++#define _CNL_PORT_TX_DW4_LN0_AE 0x162450 ++#define _CNL_PORT_TX_DW4_LN1_AE 0x1624D0 ++#define CNL_PORT_TX_DW4_GRP(port) _MMIO(_CNL_PORT_TX_DW_GRP(4, (port))) ++#define CNL_PORT_TX_DW4_LN0(port) _MMIO(_CNL_PORT_TX_DW_LN0(4, (port))) ++#define CNL_PORT_TX_DW4_LN(ln, port) _MMIO(_CNL_PORT_TX_DW_LN0(4, (port)) + \ ++ ((ln) * (_CNL_PORT_TX_DW4_LN1_AE - \ ++ _CNL_PORT_TX_DW4_LN0_AE))) ++#define ICL_PORT_TX_DW4_AUX(port) _MMIO(_ICL_PORT_TX_DW_AUX(4, port)) ++#define ICL_PORT_TX_DW4_GRP(port) _MMIO(_ICL_PORT_TX_DW_GRP(4, port)) ++#define ICL_PORT_TX_DW4_LN0(port) _MMIO(_ICL_PORT_TX_DW_LN(4, 0, port)) ++#define ICL_PORT_TX_DW4_LN(ln, port) _MMIO(_ICL_PORT_TX_DW_LN(4, ln, port)) ++#define LOADGEN_SELECT (1 << 31) ++#define POST_CURSOR_1(x) ((x) << 12) ++#define POST_CURSOR_1_MASK (0x3F << 12) ++#define POST_CURSOR_2(x) ((x) << 6) ++#define POST_CURSOR_2_MASK (0x3F << 6) ++#define CURSOR_COEFF(x) ((x) << 0) ++#define CURSOR_COEFF_MASK (0x3F << 0) ++ ++#define CNL_PORT_TX_DW5_GRP(port) _MMIO(_CNL_PORT_TX_DW_GRP(5, port)) ++#define CNL_PORT_TX_DW5_LN0(port) _MMIO(_CNL_PORT_TX_DW_LN0(5, port)) ++#define ICL_PORT_TX_DW5_AUX(port) _MMIO(_ICL_PORT_TX_DW_AUX(5, port)) ++#define ICL_PORT_TX_DW5_GRP(port) _MMIO(_ICL_PORT_TX_DW_GRP(5, port)) ++#define ICL_PORT_TX_DW5_LN0(port) _MMIO(_ICL_PORT_TX_DW_LN(5, 0, port)) ++#define TX_TRAINING_EN (1 << 31) ++#define TAP2_DISABLE (1 << 30) ++#define TAP3_DISABLE (1 << 29) ++#define SCALING_MODE_SEL(x) ((x) << 18) ++#define SCALING_MODE_SEL_MASK (0x7 << 18) ++#define RTERM_SELECT(x) ((x) << 3) ++#define RTERM_SELECT_MASK (0x7 << 3) ++ ++#define CNL_PORT_TX_DW7_GRP(port) _MMIO(_CNL_PORT_TX_DW_GRP(7, (port))) ++#define CNL_PORT_TX_DW7_LN0(port) _MMIO(_CNL_PORT_TX_DW_LN0(7, (port))) ++#define ICL_PORT_TX_DW7_AUX(port) _MMIO(_ICL_PORT_TX_DW_AUX(7, port)) ++#define ICL_PORT_TX_DW7_GRP(port) _MMIO(_ICL_PORT_TX_DW_GRP(7, port)) ++#define ICL_PORT_TX_DW7_LN0(port) _MMIO(_ICL_PORT_TX_DW_LN(7, 0, port)) ++#define ICL_PORT_TX_DW7_LN(ln, port) _MMIO(_ICL_PORT_TX_DW_LN(7, ln, port)) ++#define N_SCALAR(x) ((x) << 24) ++#define N_SCALAR_MASK (0x7F << 24) ++ ++#define MG_PHY_PORT_LN(ln, port, ln0p1, ln0p2, ln1p1) \ ++ _MMIO(_PORT((port) - PORT_C, ln0p1, ln0p2) + (ln) * ((ln1p1) - (ln0p1))) ++ ++#define MG_TX_LINK_PARAMS_TX1LN0_PORT1 0x16812C ++#define MG_TX_LINK_PARAMS_TX1LN1_PORT1 0x16852C ++#define MG_TX_LINK_PARAMS_TX1LN0_PORT2 0x16912C ++#define MG_TX_LINK_PARAMS_TX1LN1_PORT2 0x16952C ++#define MG_TX_LINK_PARAMS_TX1LN0_PORT3 0x16A12C ++#define MG_TX_LINK_PARAMS_TX1LN1_PORT3 0x16A52C ++#define MG_TX_LINK_PARAMS_TX1LN0_PORT4 0x16B12C ++#define MG_TX_LINK_PARAMS_TX1LN1_PORT4 0x16B52C ++#define MG_TX1_LINK_PARAMS(ln, port) \ ++ MG_PHY_PORT_LN(ln, port, MG_TX_LINK_PARAMS_TX1LN0_PORT1, \ ++ MG_TX_LINK_PARAMS_TX1LN0_PORT2, \ ++ MG_TX_LINK_PARAMS_TX1LN1_PORT1) ++ ++#define MG_TX_LINK_PARAMS_TX2LN0_PORT1 0x1680AC ++#define MG_TX_LINK_PARAMS_TX2LN1_PORT1 0x1684AC ++#define MG_TX_LINK_PARAMS_TX2LN0_PORT2 0x1690AC ++#define MG_TX_LINK_PARAMS_TX2LN1_PORT2 0x1694AC ++#define MG_TX_LINK_PARAMS_TX2LN0_PORT3 0x16A0AC ++#define MG_TX_LINK_PARAMS_TX2LN1_PORT3 0x16A4AC ++#define MG_TX_LINK_PARAMS_TX2LN0_PORT4 0x16B0AC ++#define MG_TX_LINK_PARAMS_TX2LN1_PORT4 0x16B4AC ++#define MG_TX2_LINK_PARAMS(ln, port) \ ++ MG_PHY_PORT_LN(ln, port, MG_TX_LINK_PARAMS_TX2LN0_PORT1, \ ++ MG_TX_LINK_PARAMS_TX2LN0_PORT2, \ ++ MG_TX_LINK_PARAMS_TX2LN1_PORT1) ++#define CRI_USE_FS32 (1 << 5) ++ ++#define MG_TX_PISO_READLOAD_TX1LN0_PORT1 0x16814C ++#define MG_TX_PISO_READLOAD_TX1LN1_PORT1 0x16854C ++#define MG_TX_PISO_READLOAD_TX1LN0_PORT2 0x16914C ++#define MG_TX_PISO_READLOAD_TX1LN1_PORT2 0x16954C ++#define MG_TX_PISO_READLOAD_TX1LN0_PORT3 0x16A14C ++#define MG_TX_PISO_READLOAD_TX1LN1_PORT3 0x16A54C ++#define MG_TX_PISO_READLOAD_TX1LN0_PORT4 0x16B14C ++#define MG_TX_PISO_READLOAD_TX1LN1_PORT4 0x16B54C ++#define MG_TX1_PISO_READLOAD(ln, port) \ ++ MG_PHY_PORT_LN(ln, port, MG_TX_PISO_READLOAD_TX1LN0_PORT1, \ ++ MG_TX_PISO_READLOAD_TX1LN0_PORT2, \ ++ MG_TX_PISO_READLOAD_TX1LN1_PORT1) ++ ++#define MG_TX_PISO_READLOAD_TX2LN0_PORT1 0x1680CC ++#define MG_TX_PISO_READLOAD_TX2LN1_PORT1 0x1684CC ++#define MG_TX_PISO_READLOAD_TX2LN0_PORT2 0x1690CC ++#define MG_TX_PISO_READLOAD_TX2LN1_PORT2 0x1694CC ++#define MG_TX_PISO_READLOAD_TX2LN0_PORT3 0x16A0CC ++#define MG_TX_PISO_READLOAD_TX2LN1_PORT3 0x16A4CC ++#define MG_TX_PISO_READLOAD_TX2LN0_PORT4 0x16B0CC ++#define MG_TX_PISO_READLOAD_TX2LN1_PORT4 0x16B4CC ++#define MG_TX2_PISO_READLOAD(ln, port) \ ++ MG_PHY_PORT_LN(ln, port, MG_TX_PISO_READLOAD_TX2LN0_PORT1, \ ++ MG_TX_PISO_READLOAD_TX2LN0_PORT2, \ ++ MG_TX_PISO_READLOAD_TX2LN1_PORT1) ++#define CRI_CALCINIT (1 << 1) ++ ++#define MG_TX_SWINGCTRL_TX1LN0_PORT1 0x168148 ++#define MG_TX_SWINGCTRL_TX1LN1_PORT1 0x168548 ++#define MG_TX_SWINGCTRL_TX1LN0_PORT2 0x169148 ++#define MG_TX_SWINGCTRL_TX1LN1_PORT2 0x169548 ++#define MG_TX_SWINGCTRL_TX1LN0_PORT3 0x16A148 ++#define MG_TX_SWINGCTRL_TX1LN1_PORT3 0x16A548 ++#define MG_TX_SWINGCTRL_TX1LN0_PORT4 0x16B148 ++#define MG_TX_SWINGCTRL_TX1LN1_PORT4 0x16B548 ++#define MG_TX1_SWINGCTRL(ln, port) \ ++ MG_PHY_PORT_LN(ln, port, MG_TX_SWINGCTRL_TX1LN0_PORT1, \ ++ MG_TX_SWINGCTRL_TX1LN0_PORT2, \ ++ MG_TX_SWINGCTRL_TX1LN1_PORT1) ++ ++#define MG_TX_SWINGCTRL_TX2LN0_PORT1 0x1680C8 ++#define MG_TX_SWINGCTRL_TX2LN1_PORT1 0x1684C8 ++#define MG_TX_SWINGCTRL_TX2LN0_PORT2 0x1690C8 ++#define MG_TX_SWINGCTRL_TX2LN1_PORT2 0x1694C8 ++#define MG_TX_SWINGCTRL_TX2LN0_PORT3 0x16A0C8 ++#define MG_TX_SWINGCTRL_TX2LN1_PORT3 0x16A4C8 ++#define MG_TX_SWINGCTRL_TX2LN0_PORT4 0x16B0C8 ++#define MG_TX_SWINGCTRL_TX2LN1_PORT4 0x16B4C8 ++#define MG_TX2_SWINGCTRL(ln, port) \ ++ MG_PHY_PORT_LN(ln, port, MG_TX_SWINGCTRL_TX2LN0_PORT1, \ ++ MG_TX_SWINGCTRL_TX2LN0_PORT2, \ ++ MG_TX_SWINGCTRL_TX2LN1_PORT1) ++#define CRI_TXDEEMPH_OVERRIDE_17_12(x) ((x) << 0) ++#define CRI_TXDEEMPH_OVERRIDE_17_12_MASK (0x3F << 0) ++ ++#define MG_TX_DRVCTRL_TX1LN0_TXPORT1 0x168144 ++#define MG_TX_DRVCTRL_TX1LN1_TXPORT1 0x168544 ++#define MG_TX_DRVCTRL_TX1LN0_TXPORT2 0x169144 ++#define MG_TX_DRVCTRL_TX1LN1_TXPORT2 0x169544 ++#define MG_TX_DRVCTRL_TX1LN0_TXPORT3 0x16A144 ++#define MG_TX_DRVCTRL_TX1LN1_TXPORT3 0x16A544 ++#define MG_TX_DRVCTRL_TX1LN0_TXPORT4 0x16B144 ++#define MG_TX_DRVCTRL_TX1LN1_TXPORT4 0x16B544 ++#define MG_TX1_DRVCTRL(ln, port) \ ++ MG_PHY_PORT_LN(ln, port, MG_TX_DRVCTRL_TX1LN0_TXPORT1, \ ++ MG_TX_DRVCTRL_TX1LN0_TXPORT2, \ ++ MG_TX_DRVCTRL_TX1LN1_TXPORT1) ++ ++#define MG_TX_DRVCTRL_TX2LN0_PORT1 0x1680C4 ++#define MG_TX_DRVCTRL_TX2LN1_PORT1 0x1684C4 ++#define MG_TX_DRVCTRL_TX2LN0_PORT2 0x1690C4 ++#define MG_TX_DRVCTRL_TX2LN1_PORT2 0x1694C4 ++#define MG_TX_DRVCTRL_TX2LN0_PORT3 0x16A0C4 ++#define MG_TX_DRVCTRL_TX2LN1_PORT3 0x16A4C4 ++#define MG_TX_DRVCTRL_TX2LN0_PORT4 0x16B0C4 ++#define MG_TX_DRVCTRL_TX2LN1_PORT4 0x16B4C4 ++#define MG_TX2_DRVCTRL(ln, port) \ ++ MG_PHY_PORT_LN(ln, port, MG_TX_DRVCTRL_TX2LN0_PORT1, \ ++ MG_TX_DRVCTRL_TX2LN0_PORT2, \ ++ MG_TX_DRVCTRL_TX2LN1_PORT1) ++#define CRI_TXDEEMPH_OVERRIDE_11_6(x) ((x) << 24) ++#define CRI_TXDEEMPH_OVERRIDE_11_6_MASK (0x3F << 24) ++#define CRI_TXDEEMPH_OVERRIDE_EN (1 << 22) ++#define CRI_TXDEEMPH_OVERRIDE_5_0(x) ((x) << 16) ++#define CRI_TXDEEMPH_OVERRIDE_5_0_MASK (0x3F << 16) ++#define CRI_LOADGEN_SEL(x) ((x) << 12) ++#define CRI_LOADGEN_SEL_MASK (0x3 << 12) ++ ++#define MG_CLKHUB_LN0_PORT1 0x16839C ++#define MG_CLKHUB_LN1_PORT1 0x16879C ++#define MG_CLKHUB_LN0_PORT2 0x16939C ++#define MG_CLKHUB_LN1_PORT2 0x16979C ++#define MG_CLKHUB_LN0_PORT3 0x16A39C ++#define MG_CLKHUB_LN1_PORT3 0x16A79C ++#define MG_CLKHUB_LN0_PORT4 0x16B39C ++#define MG_CLKHUB_LN1_PORT4 0x16B79C ++#define MG_CLKHUB(ln, port) \ ++ MG_PHY_PORT_LN(ln, port, MG_CLKHUB_LN0_PORT1, \ ++ MG_CLKHUB_LN0_PORT2, \ ++ MG_CLKHUB_LN1_PORT1) ++#define CFG_LOW_RATE_LKREN_EN (1 << 11) ++ ++#define MG_TX_DCC_TX1LN0_PORT1 0x168110 ++#define MG_TX_DCC_TX1LN1_PORT1 0x168510 ++#define MG_TX_DCC_TX1LN0_PORT2 0x169110 ++#define MG_TX_DCC_TX1LN1_PORT2 0x169510 ++#define MG_TX_DCC_TX1LN0_PORT3 0x16A110 ++#define MG_TX_DCC_TX1LN1_PORT3 0x16A510 ++#define MG_TX_DCC_TX1LN0_PORT4 0x16B110 ++#define MG_TX_DCC_TX1LN1_PORT4 0x16B510 ++#define MG_TX1_DCC(ln, port) \ ++ MG_PHY_PORT_LN(ln, port, MG_TX_DCC_TX1LN0_PORT1, \ ++ MG_TX_DCC_TX1LN0_PORT2, \ ++ MG_TX_DCC_TX1LN1_PORT1) ++#define MG_TX_DCC_TX2LN0_PORT1 0x168090 ++#define MG_TX_DCC_TX2LN1_PORT1 0x168490 ++#define MG_TX_DCC_TX2LN0_PORT2 0x169090 ++#define MG_TX_DCC_TX2LN1_PORT2 0x169490 ++#define MG_TX_DCC_TX2LN0_PORT3 0x16A090 ++#define MG_TX_DCC_TX2LN1_PORT3 0x16A490 ++#define MG_TX_DCC_TX2LN0_PORT4 0x16B090 ++#define MG_TX_DCC_TX2LN1_PORT4 0x16B490 ++#define MG_TX2_DCC(ln, port) \ ++ MG_PHY_PORT_LN(ln, port, MG_TX_DCC_TX2LN0_PORT1, \ ++ MG_TX_DCC_TX2LN0_PORT2, \ ++ MG_TX_DCC_TX2LN1_PORT1) ++#define CFG_AMI_CK_DIV_OVERRIDE_VAL(x) ((x) << 25) ++#define CFG_AMI_CK_DIV_OVERRIDE_VAL_MASK (0x3 << 25) ++#define CFG_AMI_CK_DIV_OVERRIDE_EN (1 << 24) ++ ++#define MG_DP_MODE_LN0_ACU_PORT1 0x1683A0 ++#define MG_DP_MODE_LN1_ACU_PORT1 0x1687A0 ++#define MG_DP_MODE_LN0_ACU_PORT2 0x1693A0 ++#define MG_DP_MODE_LN1_ACU_PORT2 0x1697A0 ++#define MG_DP_MODE_LN0_ACU_PORT3 0x16A3A0 ++#define MG_DP_MODE_LN1_ACU_PORT3 0x16A7A0 ++#define MG_DP_MODE_LN0_ACU_PORT4 0x16B3A0 ++#define MG_DP_MODE_LN1_ACU_PORT4 0x16B7A0 ++#define MG_DP_MODE(ln, port) \ ++ MG_PHY_PORT_LN(ln, port, MG_DP_MODE_LN0_ACU_PORT1, \ ++ MG_DP_MODE_LN0_ACU_PORT2, \ ++ MG_DP_MODE_LN1_ACU_PORT1) ++#define MG_DP_MODE_CFG_DP_X2_MODE (1 << 7) ++#define MG_DP_MODE_CFG_DP_X1_MODE (1 << 6) ++#define MG_DP_MODE_CFG_TR2PWR_GATING (1 << 5) ++#define MG_DP_MODE_CFG_TRPWR_GATING (1 << 4) ++#define MG_DP_MODE_CFG_CLNPWR_GATING (1 << 3) ++#define MG_DP_MODE_CFG_DIGPWR_GATING (1 << 2) ++#define MG_DP_MODE_CFG_GAONPWR_GATING (1 << 1) ++ ++#define MG_MISC_SUS0_PORT1 0x168814 ++#define MG_MISC_SUS0_PORT2 0x169814 ++#define MG_MISC_SUS0_PORT3 0x16A814 ++#define MG_MISC_SUS0_PORT4 0x16B814 ++#define MG_MISC_SUS0(tc_port) \ ++ _MMIO(_PORT(tc_port, MG_MISC_SUS0_PORT1, MG_MISC_SUS0_PORT2)) ++#define MG_MISC_SUS0_SUSCLK_DYNCLKGATE_MODE_MASK (3 << 14) ++#define MG_MISC_SUS0_SUSCLK_DYNCLKGATE_MODE(x) ((x) << 14) ++#define MG_MISC_SUS0_CFG_TR2PWR_GATING (1 << 12) ++#define MG_MISC_SUS0_CFG_CL2PWR_GATING (1 << 11) ++#define MG_MISC_SUS0_CFG_GAONPWR_GATING (1 << 10) ++#define MG_MISC_SUS0_CFG_TRPWR_GATING (1 << 7) ++#define MG_MISC_SUS0_CFG_CL1PWR_GATING (1 << 6) ++#define MG_MISC_SUS0_CFG_DGPWR_GATING (1 << 5) ++ ++/* The spec defines this only for BXT PHY0, but lets assume that this ++ * would exist for PHY1 too if it had a second channel. ++ */ ++#define _PORT_CL2CM_DW6_A 0x162358 ++#define _PORT_CL2CM_DW6_BC 0x6C358 ++#define BXT_PORT_CL2CM_DW6(phy) _BXT_PHY((phy), _PORT_CL2CM_DW6_BC) ++#define DW6_OLDO_DYN_PWR_DOWN_EN (1 << 28) ++ ++#define FIA1_BASE 0x163000 ++ ++/* ICL PHY DFLEX registers */ ++#define PORT_TX_DFLEXDPMLE1 _MMIO(FIA1_BASE + 0x008C0) ++#define DFLEXDPMLE1_DPMLETC_MASK(tc_port) (0xf << (4 * (tc_port))) ++#define DFLEXDPMLE1_DPMLETC_ML0(tc_port) (1 << (4 * (tc_port))) ++#define DFLEXDPMLE1_DPMLETC_ML1_0(tc_port) (3 << (4 * (tc_port))) ++#define DFLEXDPMLE1_DPMLETC_ML3(tc_port) (8 << (4 * (tc_port))) ++#define DFLEXDPMLE1_DPMLETC_ML3_2(tc_port) (12 << (4 * (tc_port))) ++#define DFLEXDPMLE1_DPMLETC_ML3_0(tc_port) (15 << (4 * (tc_port))) ++ ++/* BXT PHY Ref registers */ ++#define _PORT_REF_DW3_A 0x16218C ++#define _PORT_REF_DW3_BC 0x6C18C ++#define GRC_DONE (1 << 22) ++#define BXT_PORT_REF_DW3(phy) _BXT_PHY((phy), _PORT_REF_DW3_BC) ++ ++#define _PORT_REF_DW6_A 0x162198 ++#define _PORT_REF_DW6_BC 0x6C198 ++#define GRC_CODE_SHIFT 24 ++#define GRC_CODE_MASK (0xFF << GRC_CODE_SHIFT) ++#define GRC_CODE_FAST_SHIFT 16 ++#define GRC_CODE_FAST_MASK (0xFF << GRC_CODE_FAST_SHIFT) ++#define GRC_CODE_SLOW_SHIFT 8 ++#define GRC_CODE_SLOW_MASK (0xFF << GRC_CODE_SLOW_SHIFT) ++#define GRC_CODE_NOM_MASK 0xFF ++#define BXT_PORT_REF_DW6(phy) _BXT_PHY((phy), _PORT_REF_DW6_BC) ++ ++#define _PORT_REF_DW8_A 0x1621A0 ++#define _PORT_REF_DW8_BC 0x6C1A0 ++#define GRC_DIS (1 << 15) ++#define GRC_RDY_OVRD (1 << 1) ++#define BXT_PORT_REF_DW8(phy) _BXT_PHY((phy), _PORT_REF_DW8_BC) ++ ++/* BXT PHY PCS registers */ ++#define _PORT_PCS_DW10_LN01_A 0x162428 ++#define _PORT_PCS_DW10_LN01_B 0x6C428 ++#define _PORT_PCS_DW10_LN01_C 0x6C828 ++#define _PORT_PCS_DW10_GRP_A 0x162C28 ++#define _PORT_PCS_DW10_GRP_B 0x6CC28 ++#define _PORT_PCS_DW10_GRP_C 0x6CE28 ++#define BXT_PORT_PCS_DW10_LN01(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ ++ _PORT_PCS_DW10_LN01_B, \ ++ _PORT_PCS_DW10_LN01_C) ++#define BXT_PORT_PCS_DW10_GRP(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ ++ _PORT_PCS_DW10_GRP_B, \ ++ _PORT_PCS_DW10_GRP_C) ++ ++#define TX2_SWING_CALC_INIT (1 << 31) ++#define TX1_SWING_CALC_INIT (1 << 30) ++ ++#define _PORT_PCS_DW12_LN01_A 0x162430 ++#define _PORT_PCS_DW12_LN01_B 0x6C430 ++#define _PORT_PCS_DW12_LN01_C 0x6C830 ++#define _PORT_PCS_DW12_LN23_A 0x162630 ++#define _PORT_PCS_DW12_LN23_B 0x6C630 ++#define _PORT_PCS_DW12_LN23_C 0x6CA30 ++#define _PORT_PCS_DW12_GRP_A 0x162c30 ++#define _PORT_PCS_DW12_GRP_B 0x6CC30 ++#define _PORT_PCS_DW12_GRP_C 0x6CE30 ++#define LANESTAGGER_STRAP_OVRD (1 << 6) ++#define LANE_STAGGER_MASK 0x1F ++#define BXT_PORT_PCS_DW12_LN01(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ ++ _PORT_PCS_DW12_LN01_B, \ ++ _PORT_PCS_DW12_LN01_C) ++#define BXT_PORT_PCS_DW12_LN23(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ ++ _PORT_PCS_DW12_LN23_B, \ ++ _PORT_PCS_DW12_LN23_C) ++#define BXT_PORT_PCS_DW12_GRP(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ ++ _PORT_PCS_DW12_GRP_B, \ ++ _PORT_PCS_DW12_GRP_C) ++ ++/* BXT PHY TX registers */ ++#define _BXT_LANE_OFFSET(lane) (((lane) >> 1) * 0x200 + \ ++ ((lane) & 1) * 0x80) ++ ++#define _PORT_TX_DW2_LN0_A 0x162508 ++#define _PORT_TX_DW2_LN0_B 0x6C508 ++#define _PORT_TX_DW2_LN0_C 0x6C908 ++#define _PORT_TX_DW2_GRP_A 0x162D08 ++#define _PORT_TX_DW2_GRP_B 0x6CD08 ++#define _PORT_TX_DW2_GRP_C 0x6CF08 ++#define BXT_PORT_TX_DW2_LN0(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ ++ _PORT_TX_DW2_LN0_B, \ ++ _PORT_TX_DW2_LN0_C) ++#define BXT_PORT_TX_DW2_GRP(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ ++ _PORT_TX_DW2_GRP_B, \ ++ _PORT_TX_DW2_GRP_C) ++#define MARGIN_000_SHIFT 16 ++#define MARGIN_000 (0xFF << MARGIN_000_SHIFT) ++#define UNIQ_TRANS_SCALE_SHIFT 8 ++#define UNIQ_TRANS_SCALE (0xFF << UNIQ_TRANS_SCALE_SHIFT) ++ ++#define _PORT_TX_DW3_LN0_A 0x16250C ++#define _PORT_TX_DW3_LN0_B 0x6C50C ++#define _PORT_TX_DW3_LN0_C 0x6C90C ++#define _PORT_TX_DW3_GRP_A 0x162D0C ++#define _PORT_TX_DW3_GRP_B 0x6CD0C ++#define _PORT_TX_DW3_GRP_C 0x6CF0C ++#define BXT_PORT_TX_DW3_LN0(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ ++ _PORT_TX_DW3_LN0_B, \ ++ _PORT_TX_DW3_LN0_C) ++#define BXT_PORT_TX_DW3_GRP(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ ++ _PORT_TX_DW3_GRP_B, \ ++ _PORT_TX_DW3_GRP_C) ++#define SCALE_DCOMP_METHOD (1 << 26) ++#define UNIQUE_TRANGE_EN_METHOD (1 << 27) ++ ++#define _PORT_TX_DW4_LN0_A 0x162510 ++#define _PORT_TX_DW4_LN0_B 0x6C510 ++#define _PORT_TX_DW4_LN0_C 0x6C910 ++#define _PORT_TX_DW4_GRP_A 0x162D10 ++#define _PORT_TX_DW4_GRP_B 0x6CD10 ++#define _PORT_TX_DW4_GRP_C 0x6CF10 ++#define BXT_PORT_TX_DW4_LN0(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ ++ _PORT_TX_DW4_LN0_B, \ ++ _PORT_TX_DW4_LN0_C) ++#define BXT_PORT_TX_DW4_GRP(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ ++ _PORT_TX_DW4_GRP_B, \ ++ _PORT_TX_DW4_GRP_C) ++#define DEEMPH_SHIFT 24 ++#define DE_EMPHASIS (0xFF << DEEMPH_SHIFT) ++ ++#define _PORT_TX_DW5_LN0_A 0x162514 ++#define _PORT_TX_DW5_LN0_B 0x6C514 ++#define _PORT_TX_DW5_LN0_C 0x6C914 ++#define _PORT_TX_DW5_GRP_A 0x162D14 ++#define _PORT_TX_DW5_GRP_B 0x6CD14 ++#define _PORT_TX_DW5_GRP_C 0x6CF14 ++#define BXT_PORT_TX_DW5_LN0(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ ++ _PORT_TX_DW5_LN0_B, \ ++ _PORT_TX_DW5_LN0_C) ++#define BXT_PORT_TX_DW5_GRP(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ ++ _PORT_TX_DW5_GRP_B, \ ++ _PORT_TX_DW5_GRP_C) ++#define DCC_DELAY_RANGE_1 (1 << 9) ++#define DCC_DELAY_RANGE_2 (1 << 8) ++ ++#define _PORT_TX_DW14_LN0_A 0x162538 ++#define _PORT_TX_DW14_LN0_B 0x6C538 ++#define _PORT_TX_DW14_LN0_C 0x6C938 ++#define LATENCY_OPTIM_SHIFT 30 ++#define LATENCY_OPTIM (1 << LATENCY_OPTIM_SHIFT) ++#define BXT_PORT_TX_DW14_LN(phy, ch, lane) \ ++ _MMIO(_BXT_PHY_CH(phy, ch, _PORT_TX_DW14_LN0_B, \ ++ _PORT_TX_DW14_LN0_C) + \ ++ _BXT_LANE_OFFSET(lane)) ++ ++/* UAIMI scratch pad register 1 */ ++#define UAIMI_SPR1 _MMIO(0x4F074) ++/* SKL VccIO mask */ ++#define SKL_VCCIO_MASK 0x1 ++/* SKL balance leg register */ ++#define DISPIO_CR_TX_BMU_CR0 _MMIO(0x6C00C) ++/* I_boost values */ ++#define BALANCE_LEG_SHIFT(port) (8 + 3 * (port)) ++#define BALANCE_LEG_MASK(port) (7 << (8 + 3 * (port))) ++/* Balance leg disable bits */ ++#define BALANCE_LEG_DISABLE_SHIFT 23 ++#define BALANCE_LEG_DISABLE(port) (1 << (23 + (port))) ++ ++/* ++ * Fence registers ++ * [0-7] @ 0x2000 gen2,gen3 ++ * [8-15] @ 0x3000 945,g33,pnv ++ * ++ * [0-15] @ 0x3000 gen4,gen5 ++ * ++ * [0-15] @ 0x100000 gen6,vlv,chv ++ * [0-31] @ 0x100000 gen7+ ++ */ ++#define FENCE_REG(i) _MMIO(0x2000 + (((i) & 8) << 9) + ((i) & 7) * 4) ++#define I830_FENCE_START_MASK 0x07f80000 ++#define I830_FENCE_TILING_Y_SHIFT 12 ++#define I830_FENCE_SIZE_BITS(size) ((ffs((size) >> 19) - 1) << 8) ++#define I830_FENCE_PITCH_SHIFT 4 ++#define I830_FENCE_REG_VALID (1 << 0) ++#define I915_FENCE_MAX_PITCH_VAL 4 ++#define I830_FENCE_MAX_PITCH_VAL 6 ++#define I830_FENCE_MAX_SIZE_VAL (1 << 8) ++ ++#define I915_FENCE_START_MASK 0x0ff00000 ++#define I915_FENCE_SIZE_BITS(size) ((ffs((size) >> 20) - 1) << 8) ++ ++#define FENCE_REG_965_LO(i) _MMIO(0x03000 + (i) * 8) ++#define FENCE_REG_965_HI(i) _MMIO(0x03000 + (i) * 8 + 4) ++#define I965_FENCE_PITCH_SHIFT 2 ++#define I965_FENCE_TILING_Y_SHIFT 1 ++#define I965_FENCE_REG_VALID (1 << 0) ++#define I965_FENCE_MAX_PITCH_VAL 0x0400 ++ ++#define FENCE_REG_GEN6_LO(i) _MMIO(0x100000 + (i) * 8) ++#define FENCE_REG_GEN6_HI(i) _MMIO(0x100000 + (i) * 8 + 4) ++#define GEN6_FENCE_PITCH_SHIFT 32 ++#define GEN7_FENCE_MAX_PITCH_VAL 0x0800 ++ ++ ++/* control register for cpu gtt access */ ++#define TILECTL _MMIO(0x101000) ++#define TILECTL_SWZCTL (1 << 0) ++#define TILECTL_TLBPF (1 << 1) ++#define TILECTL_TLB_PREFETCH_DIS (1 << 2) ++#define TILECTL_BACKSNOOP_DIS (1 << 3) ++ ++/* ++ * Instruction and interrupt control regs ++ */ ++#define PGTBL_CTL _MMIO(0x02020) ++#define PGTBL_ADDRESS_LO_MASK 0xfffff000 /* bits [31:12] */ ++#define PGTBL_ADDRESS_HI_MASK 0x000000f0 /* bits [35:32] (gen4) */ ++#define PGTBL_ER _MMIO(0x02024) ++#define PRB0_BASE (0x2030 - 0x30) ++#define PRB1_BASE (0x2040 - 0x30) /* 830,gen3 */ ++#define PRB2_BASE (0x2050 - 0x30) /* gen3 */ ++#define SRB0_BASE (0x2100 - 0x30) /* gen2 */ ++#define SRB1_BASE (0x2110 - 0x30) /* gen2 */ ++#define SRB2_BASE (0x2120 - 0x30) /* 830 */ ++#define SRB3_BASE (0x2130 - 0x30) /* 830 */ ++#define RENDER_RING_BASE 0x02000 ++#define BSD_RING_BASE 0x04000 ++#define GEN6_BSD_RING_BASE 0x12000 ++#define GEN8_BSD2_RING_BASE 0x1c000 ++#define GEN11_BSD_RING_BASE 0x1c0000 ++#define GEN11_BSD2_RING_BASE 0x1c4000 ++#define GEN11_BSD3_RING_BASE 0x1d0000 ++#define GEN11_BSD4_RING_BASE 0x1d4000 ++#define VEBOX_RING_BASE 0x1a000 ++#define GEN11_VEBOX_RING_BASE 0x1c8000 ++#define GEN11_VEBOX2_RING_BASE 0x1d8000 ++#define BLT_RING_BASE 0x22000 ++#define RING_TAIL(base) _MMIO((base) + 0x30) ++#define RING_HEAD(base) _MMIO((base) + 0x34) ++#define RING_START(base) _MMIO((base) + 0x38) ++#define RING_CTL(base) _MMIO((base) + 0x3c) ++#define RING_CTL_SIZE(size) ((size) - PAGE_SIZE) /* in bytes -> pages */ ++#define RING_SYNC_0(base) _MMIO((base) + 0x40) ++#define RING_SYNC_1(base) _MMIO((base) + 0x44) ++#define RING_SYNC_2(base) _MMIO((base) + 0x48) ++#define GEN6_RVSYNC (RING_SYNC_0(RENDER_RING_BASE)) ++#define GEN6_RBSYNC (RING_SYNC_1(RENDER_RING_BASE)) ++#define GEN6_RVESYNC (RING_SYNC_2(RENDER_RING_BASE)) ++#define GEN6_VBSYNC (RING_SYNC_0(GEN6_BSD_RING_BASE)) ++#define GEN6_VRSYNC (RING_SYNC_1(GEN6_BSD_RING_BASE)) ++#define GEN6_VVESYNC (RING_SYNC_2(GEN6_BSD_RING_BASE)) ++#define GEN6_BRSYNC (RING_SYNC_0(BLT_RING_BASE)) ++#define GEN6_BVSYNC (RING_SYNC_1(BLT_RING_BASE)) ++#define GEN6_BVESYNC (RING_SYNC_2(BLT_RING_BASE)) ++#define GEN6_VEBSYNC (RING_SYNC_0(VEBOX_RING_BASE)) ++#define GEN6_VERSYNC (RING_SYNC_1(VEBOX_RING_BASE)) ++#define GEN6_VEVSYNC (RING_SYNC_2(VEBOX_RING_BASE)) ++#define GEN6_NOSYNC INVALID_MMIO_REG ++#define RING_PSMI_CTL(base) _MMIO((base) + 0x50) ++#define RING_MAX_IDLE(base) _MMIO((base) + 0x54) ++#define RING_HWS_PGA(base) _MMIO((base) + 0x80) ++#define RING_HWS_PGA_GEN6(base) _MMIO((base) + 0x2080) ++#define RING_RESET_CTL(base) _MMIO((base) + 0xd0) ++#define RESET_CTL_CAT_ERROR REG_BIT(2) ++#define RESET_CTL_READY_TO_RESET REG_BIT(1) ++#define RESET_CTL_REQUEST_RESET REG_BIT(0) ++ ++#define RING_SEMA_WAIT_POLL(base) _MMIO((base) + 0x24c) ++ ++#define HSW_GTT_CACHE_EN _MMIO(0x4024) ++#define GTT_CACHE_EN_ALL 0xF0007FFF ++#define GEN7_WR_WATERMARK _MMIO(0x4028) ++#define GEN7_GFX_PRIO_CTRL _MMIO(0x402C) ++#define ARB_MODE _MMIO(0x4030) ++#define ARB_MODE_SWIZZLE_SNB (1 << 4) ++#define ARB_MODE_SWIZZLE_IVB (1 << 5) ++#define GEN7_GFX_PEND_TLB0 _MMIO(0x4034) ++#define GEN7_GFX_PEND_TLB1 _MMIO(0x4038) ++/* L3, CVS, ZTLB, RCC, CASC LRA min, max values */ ++#define GEN7_LRA_LIMITS(i) _MMIO(0x403C + (i) * 4) ++#define GEN7_LRA_LIMITS_REG_NUM 13 ++#define GEN7_MEDIA_MAX_REQ_COUNT _MMIO(0x4070) ++#define GEN7_GFX_MAX_REQ_COUNT _MMIO(0x4074) ++ ++#define GAMTARBMODE _MMIO(0x04a08) ++#define ARB_MODE_BWGTLB_DISABLE (1 << 9) ++#define ARB_MODE_SWIZZLE_BDW (1 << 1) ++#define RENDER_HWS_PGA_GEN7 _MMIO(0x04080) ++#define RING_FAULT_REG(engine) _MMIO(0x4094 + 0x100 * (engine)->hw_id) ++#define GEN8_RING_FAULT_REG _MMIO(0x4094) ++#define GEN8_RING_FAULT_ENGINE_ID(x) (((x) >> 12) & 0x7) ++#define RING_FAULT_GTTSEL_MASK (1 << 11) ++#define RING_FAULT_SRCID(x) (((x) >> 3) & 0xff) ++#define RING_FAULT_FAULT_TYPE(x) (((x) >> 1) & 0x3) ++#define RING_FAULT_VALID (1 << 0) ++#define DONE_REG _MMIO(0x40b0) ++#define GEN8_PRIVATE_PAT_LO _MMIO(0x40e0) ++#define GEN8_PRIVATE_PAT_HI _MMIO(0x40e0 + 4) ++#define GEN10_PAT_INDEX(index) _MMIO(0x40e0 + (index) * 4) ++#define BSD_HWS_PGA_GEN7 _MMIO(0x04180) ++#define BLT_HWS_PGA_GEN7 _MMIO(0x04280) ++#define VEBOX_HWS_PGA_GEN7 _MMIO(0x04380) ++#define RING_ACTHD(base) _MMIO((base) + 0x74) ++#define RING_ACTHD_UDW(base) _MMIO((base) + 0x5c) ++#define RING_NOPID(base) _MMIO((base) + 0x94) ++#define RING_IMR(base) _MMIO((base) + 0xa8) ++#define RING_HWSTAM(base) _MMIO((base) + 0x98) ++#define RING_TIMESTAMP(base) _MMIO((base) + 0x358) ++#define RING_TIMESTAMP_UDW(base) _MMIO((base) + 0x358 + 4) ++#define TAIL_ADDR 0x001FFFF8 ++#define HEAD_WRAP_COUNT 0xFFE00000 ++#define HEAD_WRAP_ONE 0x00200000 ++#define HEAD_ADDR 0x001FFFFC ++#define RING_NR_PAGES 0x001FF000 ++#define RING_REPORT_MASK 0x00000006 ++#define RING_REPORT_64K 0x00000002 ++#define RING_REPORT_128K 0x00000004 ++#define RING_NO_REPORT 0x00000000 ++#define RING_VALID_MASK 0x00000001 ++#define RING_VALID 0x00000001 ++#define RING_INVALID 0x00000000 ++#define RING_WAIT_I8XX (1 << 0) /* gen2, PRBx_HEAD */ ++#define RING_WAIT (1 << 11) /* gen3+, PRBx_CTL */ ++#define RING_WAIT_SEMAPHORE (1 << 10) /* gen6+ */ ++ ++#define RING_FORCE_TO_NONPRIV(base, i) _MMIO(((base) + 0x4D0) + (i) * 4) ++#define RING_FORCE_TO_NONPRIV_RW (0 << 28) /* CFL+ & Gen11+ */ ++#define RING_FORCE_TO_NONPRIV_RD (1 << 28) ++#define RING_FORCE_TO_NONPRIV_WR (2 << 28) ++#define RING_FORCE_TO_NONPRIV_RANGE_1 (0 << 0) /* CFL+ & Gen11+ */ ++#define RING_FORCE_TO_NONPRIV_RANGE_4 (1 << 0) ++#define RING_FORCE_TO_NONPRIV_RANGE_16 (2 << 0) ++#define RING_FORCE_TO_NONPRIV_RANGE_64 (3 << 0) ++#define RING_MAX_NONPRIV_SLOTS 12 ++ ++#define GEN7_TLB_RD_ADDR _MMIO(0x4700) ++ ++#define GEN9_GAMT_ECO_REG_RW_IA _MMIO(0x4ab0) ++#define GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS (1 << 18) ++ ++#define GEN8_GAMW_ECO_DEV_RW_IA _MMIO(0x4080) ++#define GAMW_ECO_ENABLE_64K_IPS_FIELD 0xF ++#define GAMW_ECO_DEV_CTX_RELOAD_DISABLE (1 << 7) ++ ++#define GAMT_CHKN_BIT_REG _MMIO(0x4ab8) ++#define GAMT_CHKN_DISABLE_L3_COH_PIPE (1 << 31) ++#define GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING (1 << 28) ++#define GAMT_CHKN_DISABLE_I2M_CYCLE_ON_WR_PORT (1 << 24) ++ ++#if 0 ++#define PRB0_TAIL _MMIO(0x2030) ++#define PRB0_HEAD _MMIO(0x2034) ++#define PRB0_START _MMIO(0x2038) ++#define PRB0_CTL _MMIO(0x203c) ++#define PRB1_TAIL _MMIO(0x2040) /* 915+ only */ ++#define PRB1_HEAD _MMIO(0x2044) /* 915+ only */ ++#define PRB1_START _MMIO(0x2048) /* 915+ only */ ++#define PRB1_CTL _MMIO(0x204c) /* 915+ only */ ++#endif ++#define IPEIR_I965 _MMIO(0x2064) ++#define IPEHR_I965 _MMIO(0x2068) ++#define GEN7_SC_INSTDONE _MMIO(0x7100) ++#define GEN7_SAMPLER_INSTDONE _MMIO(0xe160) ++#define GEN7_ROW_INSTDONE _MMIO(0xe164) ++#define GEN8_MCR_SELECTOR _MMIO(0xfdc) ++#define GEN8_MCR_SLICE(slice) (((slice) & 3) << 26) ++#define GEN8_MCR_SLICE_MASK GEN8_MCR_SLICE(3) ++#define GEN8_MCR_SUBSLICE(subslice) (((subslice) & 3) << 24) ++#define GEN8_MCR_SUBSLICE_MASK GEN8_MCR_SUBSLICE(3) ++#define GEN11_MCR_SLICE(slice) (((slice) & 0xf) << 27) ++#define GEN11_MCR_SLICE_MASK GEN11_MCR_SLICE(0xf) ++#define GEN11_MCR_SUBSLICE(subslice) (((subslice) & 0x7) << 24) ++#define GEN11_MCR_SUBSLICE_MASK GEN11_MCR_SUBSLICE(0x7) ++#define RING_IPEIR(base) _MMIO((base) + 0x64) ++#define RING_IPEHR(base) _MMIO((base) + 0x68) ++/* ++ * On GEN4, only the render ring INSTDONE exists and has a different ++ * layout than the GEN7+ version. ++ * The GEN2 counterpart of this register is GEN2_INSTDONE. ++ */ ++#define RING_INSTDONE(base) _MMIO((base) + 0x6c) ++#define RING_INSTPS(base) _MMIO((base) + 0x70) ++#define RING_DMA_FADD(base) _MMIO((base) + 0x78) ++#define RING_DMA_FADD_UDW(base) _MMIO((base) + 0x60) /* gen8+ */ ++#define RING_INSTPM(base) _MMIO((base) + 0xc0) ++#define RING_MI_MODE(base) _MMIO((base) + 0x9c) ++#define INSTPS _MMIO(0x2070) /* 965+ only */ ++#define GEN4_INSTDONE1 _MMIO(0x207c) /* 965+ only, aka INSTDONE_2 on SNB */ ++#define ACTHD_I965 _MMIO(0x2074) ++#define HWS_PGA _MMIO(0x2080) ++#define HWS_ADDRESS_MASK 0xfffff000 ++#define HWS_START_ADDRESS_SHIFT 4 ++#define PWRCTXA _MMIO(0x2088) /* 965GM+ only */ ++#define PWRCTX_EN (1 << 0) ++#define IPEIR(base) _MMIO((base) + 0x88) ++#define IPEHR(base) _MMIO((base) + 0x8c) ++#define GEN2_INSTDONE _MMIO(0x2090) ++#define NOPID _MMIO(0x2094) ++#define HWSTAM _MMIO(0x2098) ++#define DMA_FADD_I8XX(base) _MMIO((base) + 0xd0) ++#define RING_BBSTATE(base) _MMIO((base) + 0x110) ++#define RING_BB_PPGTT (1 << 5) ++#define RING_SBBADDR(base) _MMIO((base) + 0x114) /* hsw+ */ ++#define RING_SBBSTATE(base) _MMIO((base) + 0x118) /* hsw+ */ ++#define RING_SBBADDR_UDW(base) _MMIO((base) + 0x11c) /* gen8+ */ ++#define RING_BBADDR(base) _MMIO((base) + 0x140) ++#define RING_BBADDR_UDW(base) _MMIO((base) + 0x168) /* gen8+ */ ++#define RING_BB_PER_CTX_PTR(base) _MMIO((base) + 0x1c0) /* gen8+ */ ++#define RING_INDIRECT_CTX(base) _MMIO((base) + 0x1c4) /* gen8+ */ ++#define RING_INDIRECT_CTX_OFFSET(base) _MMIO((base) + 0x1c8) /* gen8+ */ ++#define RING_CTX_TIMESTAMP(base) _MMIO((base) + 0x3a8) /* gen8+ */ ++ ++#define ERROR_GEN6 _MMIO(0x40a0) ++#define GEN7_ERR_INT _MMIO(0x44040) ++#define ERR_INT_POISON (1 << 31) ++#define ERR_INT_MMIO_UNCLAIMED (1 << 13) ++#define ERR_INT_PIPE_CRC_DONE_C (1 << 8) ++#define ERR_INT_FIFO_UNDERRUN_C (1 << 6) ++#define ERR_INT_PIPE_CRC_DONE_B (1 << 5) ++#define ERR_INT_FIFO_UNDERRUN_B (1 << 3) ++#define ERR_INT_PIPE_CRC_DONE_A (1 << 2) ++#define ERR_INT_PIPE_CRC_DONE(pipe) (1 << (2 + (pipe) * 3)) ++#define ERR_INT_FIFO_UNDERRUN_A (1 << 0) ++#define ERR_INT_FIFO_UNDERRUN(pipe) (1 << ((pipe) * 3)) ++ ++#define GEN8_FAULT_TLB_DATA0 _MMIO(0x4b10) ++#define GEN8_FAULT_TLB_DATA1 _MMIO(0x4b14) ++#define FAULT_VA_HIGH_BITS (0xf << 0) ++#define FAULT_GTT_SEL (1 << 4) ++ ++#define FPGA_DBG _MMIO(0x42300) ++#define FPGA_DBG_RM_NOCLAIM (1 << 31) ++ ++#define CLAIM_ER _MMIO(VLV_DISPLAY_BASE + 0x2028) ++#define CLAIM_ER_CLR (1 << 31) ++#define CLAIM_ER_OVERFLOW (1 << 16) ++#define CLAIM_ER_CTR_MASK 0xffff ++ ++#define DERRMR _MMIO(0x44050) ++/* Note that HBLANK events are reserved on bdw+ */ ++#define DERRMR_PIPEA_SCANLINE (1 << 0) ++#define DERRMR_PIPEA_PRI_FLIP_DONE (1 << 1) ++#define DERRMR_PIPEA_SPR_FLIP_DONE (1 << 2) ++#define DERRMR_PIPEA_VBLANK (1 << 3) ++#define DERRMR_PIPEA_HBLANK (1 << 5) ++#define DERRMR_PIPEB_SCANLINE (1 << 8) ++#define DERRMR_PIPEB_PRI_FLIP_DONE (1 << 9) ++#define DERRMR_PIPEB_SPR_FLIP_DONE (1 << 10) ++#define DERRMR_PIPEB_VBLANK (1 << 11) ++#define DERRMR_PIPEB_HBLANK (1 << 13) ++/* Note that PIPEC is not a simple translation of PIPEA/PIPEB */ ++#define DERRMR_PIPEC_SCANLINE (1 << 14) ++#define DERRMR_PIPEC_PRI_FLIP_DONE (1 << 15) ++#define DERRMR_PIPEC_SPR_FLIP_DONE (1 << 20) ++#define DERRMR_PIPEC_VBLANK (1 << 21) ++#define DERRMR_PIPEC_HBLANK (1 << 22) ++ ++ ++/* GM45+ chicken bits -- debug workaround bits that may be required ++ * for various sorts of correct behavior. The top 16 bits of each are ++ * the enables for writing to the corresponding low bit. ++ */ ++#define _3D_CHICKEN _MMIO(0x2084) ++#define _3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB (1 << 10) ++#define _3D_CHICKEN2 _MMIO(0x208c) ++ ++#define FF_SLICE_CHICKEN _MMIO(0x2088) ++#define FF_SLICE_CHICKEN_CL_PROVOKING_VERTEX_FIX (1 << 1) ++ ++/* Disables pipelining of read flushes past the SF-WIZ interface. ++ * Required on all Ironlake steppings according to the B-Spec, but the ++ * particular danger of not doing so is not specified. ++ */ ++# define _3D_CHICKEN2_WM_READ_PIPELINED (1 << 14) ++#define _3D_CHICKEN3 _MMIO(0x2090) ++#define _3D_CHICKEN_SF_PROVOKING_VERTEX_FIX (1 << 12) ++#define _3D_CHICKEN_SF_DISABLE_OBJEND_CULL (1 << 10) ++#define _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE (1 << 5) ++#define _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL (1 << 5) ++#define _3D_CHICKEN_SDE_LIMIT_FIFO_POLY_DEPTH(x) ((x) << 1) /* gen8+ */ ++#define _3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH (1 << 1) /* gen6 */ ++ ++#define MI_MODE _MMIO(0x209c) ++# define VS_TIMER_DISPATCH (1 << 6) ++# define MI_FLUSH_ENABLE (1 << 12) ++# define ASYNC_FLIP_PERF_DISABLE (1 << 14) ++# define MODE_IDLE (1 << 9) ++# define STOP_RING (1 << 8) ++ ++#define GEN6_GT_MODE _MMIO(0x20d0) ++#define GEN7_GT_MODE _MMIO(0x7008) ++#define GEN6_WIZ_HASHING(hi, lo) (((hi) << 9) | ((lo) << 7)) ++#define GEN6_WIZ_HASHING_8x8 GEN6_WIZ_HASHING(0, 0) ++#define GEN6_WIZ_HASHING_8x4 GEN6_WIZ_HASHING(0, 1) ++#define GEN6_WIZ_HASHING_16x4 GEN6_WIZ_HASHING(1, 0) ++#define GEN6_WIZ_HASHING_MASK GEN6_WIZ_HASHING(1, 1) ++#define GEN6_TD_FOUR_ROW_DISPATCH_DISABLE (1 << 5) ++#define GEN9_IZ_HASHING_MASK(slice) (0x3 << ((slice) * 2)) ++#define GEN9_IZ_HASHING(slice, val) ((val) << ((slice) * 2)) ++ ++/* chicken reg for WaConextSwitchWithConcurrentTLBInvalidate */ ++#define GEN9_CSFE_CHICKEN1_RCS _MMIO(0x20D4) ++#define GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE (1 << 2) ++#define GEN11_ENABLE_32_PLANE_MODE (1 << 7) ++ ++/* WaClearTdlStateAckDirtyBits */ ++#define GEN8_STATE_ACK _MMIO(0x20F0) ++#define GEN9_STATE_ACK_SLICE1 _MMIO(0x20F8) ++#define GEN9_STATE_ACK_SLICE2 _MMIO(0x2100) ++#define GEN9_STATE_ACK_TDL0 (1 << 12) ++#define GEN9_STATE_ACK_TDL1 (1 << 13) ++#define GEN9_STATE_ACK_TDL2 (1 << 14) ++#define GEN9_STATE_ACK_TDL3 (1 << 15) ++#define GEN9_SUBSLICE_TDL_ACK_BITS \ ++ (GEN9_STATE_ACK_TDL3 | GEN9_STATE_ACK_TDL2 | \ ++ GEN9_STATE_ACK_TDL1 | GEN9_STATE_ACK_TDL0) ++ ++#define GFX_MODE _MMIO(0x2520) ++#define GFX_MODE_GEN7 _MMIO(0x229c) ++#define RING_MODE_GEN7(engine) _MMIO((engine)->mmio_base + 0x29c) ++#define GFX_RUN_LIST_ENABLE (1 << 15) ++#define GFX_INTERRUPT_STEERING (1 << 14) ++#define GFX_TLB_INVALIDATE_EXPLICIT (1 << 13) ++#define GFX_SURFACE_FAULT_ENABLE (1 << 12) ++#define GFX_REPLAY_MODE (1 << 11) ++#define GFX_PSMI_GRANULARITY (1 << 10) ++#define GFX_PPGTT_ENABLE (1 << 9) ++#define GEN8_GFX_PPGTT_48B (1 << 7) ++ ++#define GFX_FORWARD_VBLANK_MASK (3 << 5) ++#define GFX_FORWARD_VBLANK_NEVER (0 << 5) ++#define GFX_FORWARD_VBLANK_ALWAYS (1 << 5) ++#define GFX_FORWARD_VBLANK_COND (2 << 5) ++ ++#define GEN11_GFX_DISABLE_LEGACY_MODE (1 << 3) ++ ++#define VLV_GU_CTL0 _MMIO(VLV_DISPLAY_BASE + 0x2030) ++#define VLV_GU_CTL1 _MMIO(VLV_DISPLAY_BASE + 0x2034) ++#define SCPD0 _MMIO(0x209c) /* 915+ only */ ++#define GEN2_IER _MMIO(0x20a0) ++#define GEN2_IIR _MMIO(0x20a4) ++#define GEN2_IMR _MMIO(0x20a8) ++#define GEN2_ISR _MMIO(0x20ac) ++#define VLV_GUNIT_CLOCK_GATE _MMIO(VLV_DISPLAY_BASE + 0x2060) ++#define GINT_DIS (1 << 22) ++#define GCFG_DIS (1 << 8) ++#define VLV_GUNIT_CLOCK_GATE2 _MMIO(VLV_DISPLAY_BASE + 0x2064) ++#define VLV_IIR_RW _MMIO(VLV_DISPLAY_BASE + 0x2084) ++#define VLV_IER _MMIO(VLV_DISPLAY_BASE + 0x20a0) ++#define VLV_IIR _MMIO(VLV_DISPLAY_BASE + 0x20a4) ++#define VLV_IMR _MMIO(VLV_DISPLAY_BASE + 0x20a8) ++#define VLV_ISR _MMIO(VLV_DISPLAY_BASE + 0x20ac) ++#define VLV_PCBR _MMIO(VLV_DISPLAY_BASE + 0x2120) ++#define VLV_PCBR_ADDR_SHIFT 12 ++ ++#define DISPLAY_PLANE_FLIP_PENDING(plane) (1 << (11 - (plane))) /* A and B only */ ++#define EIR _MMIO(0x20b0) ++#define EMR _MMIO(0x20b4) ++#define ESR _MMIO(0x20b8) ++#define GM45_ERROR_PAGE_TABLE (1 << 5) ++#define GM45_ERROR_MEM_PRIV (1 << 4) ++#define I915_ERROR_PAGE_TABLE (1 << 4) ++#define GM45_ERROR_CP_PRIV (1 << 3) ++#define I915_ERROR_MEMORY_REFRESH (1 << 1) ++#define I915_ERROR_INSTRUCTION (1 << 0) ++#define INSTPM _MMIO(0x20c0) ++#define INSTPM_SELF_EN (1 << 12) /* 915GM only */ ++#define INSTPM_AGPBUSY_INT_EN (1 << 11) /* gen3: when disabled, pending interrupts ++ will not assert AGPBUSY# and will only ++ be delivered when out of C3. */ ++#define INSTPM_FORCE_ORDERING (1 << 7) /* GEN6+ */ ++#define INSTPM_TLB_INVALIDATE (1 << 9) ++#define INSTPM_SYNC_FLUSH (1 << 5) ++#define ACTHD(base) _MMIO((base) + 0xc8) ++#define MEM_MODE _MMIO(0x20cc) ++#define MEM_DISPLAY_B_TRICKLE_FEED_DISABLE (1 << 3) /* 830 only */ ++#define MEM_DISPLAY_A_TRICKLE_FEED_DISABLE (1 << 2) /* 830/845 only */ ++#define MEM_DISPLAY_TRICKLE_FEED_DISABLE (1 << 2) /* 85x only */ ++#define FW_BLC _MMIO(0x20d8) ++#define FW_BLC2 _MMIO(0x20dc) ++#define FW_BLC_SELF _MMIO(0x20e0) /* 915+ only */ ++#define FW_BLC_SELF_EN_MASK (1 << 31) ++#define FW_BLC_SELF_FIFO_MASK (1 << 16) /* 945 only */ ++#define FW_BLC_SELF_EN (1 << 15) /* 945 only */ ++#define MM_BURST_LENGTH 0x00700000 ++#define MM_FIFO_WATERMARK 0x0001F000 ++#define LM_BURST_LENGTH 0x00000700 ++#define LM_FIFO_WATERMARK 0x0000001F ++#define MI_ARB_STATE _MMIO(0x20e4) /* 915+ only */ ++ ++#define MBUS_ABOX_CTL _MMIO(0x45038) ++#define MBUS_ABOX_BW_CREDIT_MASK (3 << 20) ++#define MBUS_ABOX_BW_CREDIT(x) ((x) << 20) ++#define MBUS_ABOX_B_CREDIT_MASK (0xF << 16) ++#define MBUS_ABOX_B_CREDIT(x) ((x) << 16) ++#define MBUS_ABOX_BT_CREDIT_POOL2_MASK (0x1F << 8) ++#define MBUS_ABOX_BT_CREDIT_POOL2(x) ((x) << 8) ++#define MBUS_ABOX_BT_CREDIT_POOL1_MASK (0x1F << 0) ++#define MBUS_ABOX_BT_CREDIT_POOL1(x) ((x) << 0) ++ ++#define _PIPEA_MBUS_DBOX_CTL 0x7003C ++#define _PIPEB_MBUS_DBOX_CTL 0x7103C ++#define PIPE_MBUS_DBOX_CTL(pipe) _MMIO_PIPE(pipe, _PIPEA_MBUS_DBOX_CTL, \ ++ _PIPEB_MBUS_DBOX_CTL) ++#define MBUS_DBOX_BW_CREDIT_MASK (3 << 14) ++#define MBUS_DBOX_BW_CREDIT(x) ((x) << 14) ++#define MBUS_DBOX_B_CREDIT_MASK (0x1F << 8) ++#define MBUS_DBOX_B_CREDIT(x) ((x) << 8) ++#define MBUS_DBOX_A_CREDIT_MASK (0xF << 0) ++#define MBUS_DBOX_A_CREDIT(x) ((x) << 0) ++ ++#define MBUS_UBOX_CTL _MMIO(0x4503C) ++#define MBUS_BBOX_CTL_S1 _MMIO(0x45040) ++#define MBUS_BBOX_CTL_S2 _MMIO(0x45044) ++ ++/* Make render/texture TLB fetches lower priorty than associated data ++ * fetches. This is not turned on by default ++ */ ++#define MI_ARB_RENDER_TLB_LOW_PRIORITY (1 << 15) ++ ++/* Isoch request wait on GTT enable (Display A/B/C streams). ++ * Make isoch requests stall on the TLB update. May cause ++ * display underruns (test mode only) ++ */ ++#define MI_ARB_ISOCH_WAIT_GTT (1 << 14) ++ ++/* Block grant count for isoch requests when block count is ++ * set to a finite value. ++ */ ++#define MI_ARB_BLOCK_GRANT_MASK (3 << 12) ++#define MI_ARB_BLOCK_GRANT_8 (0 << 12) /* for 3 display planes */ ++#define MI_ARB_BLOCK_GRANT_4 (1 << 12) /* for 2 display planes */ ++#define MI_ARB_BLOCK_GRANT_2 (2 << 12) /* for 1 display plane */ ++#define MI_ARB_BLOCK_GRANT_0 (3 << 12) /* don't use */ ++ ++/* Enable render writes to complete in C2/C3/C4 power states. ++ * If this isn't enabled, render writes are prevented in low ++ * power states. That seems bad to me. ++ */ ++#define MI_ARB_C3_LP_WRITE_ENABLE (1 << 11) ++ ++/* This acknowledges an async flip immediately instead ++ * of waiting for 2TLB fetches. ++ */ ++#define MI_ARB_ASYNC_FLIP_ACK_IMMEDIATE (1 << 10) ++ ++/* Enables non-sequential data reads through arbiter ++ */ ++#define MI_ARB_DUAL_DATA_PHASE_DISABLE (1 << 9) ++ ++/* Disable FSB snooping of cacheable write cycles from binner/render ++ * command stream ++ */ ++#define MI_ARB_CACHE_SNOOP_DISABLE (1 << 8) ++ ++/* Arbiter time slice for non-isoch streams */ ++#define MI_ARB_TIME_SLICE_MASK (7 << 5) ++#define MI_ARB_TIME_SLICE_1 (0 << 5) ++#define MI_ARB_TIME_SLICE_2 (1 << 5) ++#define MI_ARB_TIME_SLICE_4 (2 << 5) ++#define MI_ARB_TIME_SLICE_6 (3 << 5) ++#define MI_ARB_TIME_SLICE_8 (4 << 5) ++#define MI_ARB_TIME_SLICE_10 (5 << 5) ++#define MI_ARB_TIME_SLICE_14 (6 << 5) ++#define MI_ARB_TIME_SLICE_16 (7 << 5) ++ ++/* Low priority grace period page size */ ++#define MI_ARB_LOW_PRIORITY_GRACE_4KB (0 << 4) /* default */ ++#define MI_ARB_LOW_PRIORITY_GRACE_8KB (1 << 4) ++ ++/* Disable display A/B trickle feed */ ++#define MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE (1 << 2) ++ ++/* Set display plane priority */ ++#define MI_ARB_DISPLAY_PRIORITY_A_B (0 << 0) /* display A > display B */ ++#define MI_ARB_DISPLAY_PRIORITY_B_A (1 << 0) /* display B > display A */ ++ ++#define MI_STATE _MMIO(0x20e4) /* gen2 only */ ++#define MI_AGPBUSY_INT_EN (1 << 1) /* 85x only */ ++#define MI_AGPBUSY_830_MODE (1 << 0) /* 85x only */ ++ ++#define CACHE_MODE_0 _MMIO(0x2120) /* 915+ only */ ++#define CM0_PIPELINED_RENDER_FLUSH_DISABLE (1 << 8) ++#define CM0_IZ_OPT_DISABLE (1 << 6) ++#define CM0_ZR_OPT_DISABLE (1 << 5) ++#define CM0_STC_EVICT_DISABLE_LRA_SNB (1 << 5) ++#define CM0_DEPTH_EVICT_DISABLE (1 << 4) ++#define CM0_COLOR_EVICT_DISABLE (1 << 3) ++#define CM0_DEPTH_WRITE_DISABLE (1 << 1) ++#define CM0_RC_OP_FLUSH_DISABLE (1 << 0) ++#define GFX_FLSH_CNTL _MMIO(0x2170) /* 915+ only */ ++#define GFX_FLSH_CNTL_GEN6 _MMIO(0x101008) ++#define GFX_FLSH_CNTL_EN (1 << 0) ++#define ECOSKPD _MMIO(0x21d0) ++#define ECO_GATING_CX_ONLY (1 << 3) ++#define ECO_FLIP_DONE (1 << 0) ++ ++#define CACHE_MODE_0_GEN7 _MMIO(0x7000) /* IVB+ */ ++#define RC_OP_FLUSH_ENABLE (1 << 0) ++#define HIZ_RAW_STALL_OPT_DISABLE (1 << 2) ++#define CACHE_MODE_1 _MMIO(0x7004) /* IVB+ */ ++#define PIXEL_SUBSPAN_COLLECT_OPT_DISABLE (1 << 6) ++#define GEN8_4x4_STC_OPTIMIZATION_DISABLE (1 << 6) ++#define GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE (1 << 1) ++ ++#define GEN6_BLITTER_ECOSKPD _MMIO(0x221d0) ++#define GEN6_BLITTER_LOCK_SHIFT 16 ++#define GEN6_BLITTER_FBC_NOTIFY (1 << 3) ++ ++#define GEN6_RC_SLEEP_PSMI_CONTROL _MMIO(0x2050) ++#define GEN6_PSMI_SLEEP_MSG_DISABLE (1 << 0) ++#define GEN8_RC_SEMA_IDLE_MSG_DISABLE (1 << 12) ++#define GEN8_FF_DOP_CLOCK_GATE_DISABLE (1 << 10) ++ ++#define GEN6_RCS_PWR_FSM _MMIO(0x22ac) ++#define GEN9_RCS_FE_FSM2 _MMIO(0x22a4) ++ ++#define GEN10_CACHE_MODE_SS _MMIO(0xe420) ++#define FLOAT_BLEND_OPTIMIZATION_ENABLE (1 << 4) ++ ++/* Fuse readout registers for GT */ ++#define HSW_PAVP_FUSE1 _MMIO(0x911C) ++#define HSW_F1_EU_DIS_SHIFT 16 ++#define HSW_F1_EU_DIS_MASK (0x3 << HSW_F1_EU_DIS_SHIFT) ++#define HSW_F1_EU_DIS_10EUS 0 ++#define HSW_F1_EU_DIS_8EUS 1 ++#define HSW_F1_EU_DIS_6EUS 2 ++ ++#define CHV_FUSE_GT _MMIO(VLV_DISPLAY_BASE + 0x2168) ++#define CHV_FGT_DISABLE_SS0 (1 << 10) ++#define CHV_FGT_DISABLE_SS1 (1 << 11) ++#define CHV_FGT_EU_DIS_SS0_R0_SHIFT 16 ++#define CHV_FGT_EU_DIS_SS0_R0_MASK (0xf << CHV_FGT_EU_DIS_SS0_R0_SHIFT) ++#define CHV_FGT_EU_DIS_SS0_R1_SHIFT 20 ++#define CHV_FGT_EU_DIS_SS0_R1_MASK (0xf << CHV_FGT_EU_DIS_SS0_R1_SHIFT) ++#define CHV_FGT_EU_DIS_SS1_R0_SHIFT 24 ++#define CHV_FGT_EU_DIS_SS1_R0_MASK (0xf << CHV_FGT_EU_DIS_SS1_R0_SHIFT) ++#define CHV_FGT_EU_DIS_SS1_R1_SHIFT 28 ++#define CHV_FGT_EU_DIS_SS1_R1_MASK (0xf << CHV_FGT_EU_DIS_SS1_R1_SHIFT) ++ ++#define GEN8_FUSE2 _MMIO(0x9120) ++#define GEN8_F2_SS_DIS_SHIFT 21 ++#define GEN8_F2_SS_DIS_MASK (0x7 << GEN8_F2_SS_DIS_SHIFT) ++#define GEN8_F2_S_ENA_SHIFT 25 ++#define GEN8_F2_S_ENA_MASK (0x7 << GEN8_F2_S_ENA_SHIFT) ++ ++#define GEN9_F2_SS_DIS_SHIFT 20 ++#define GEN9_F2_SS_DIS_MASK (0xf << GEN9_F2_SS_DIS_SHIFT) ++ ++#define GEN10_F2_S_ENA_SHIFT 22 ++#define GEN10_F2_S_ENA_MASK (0x3f << GEN10_F2_S_ENA_SHIFT) ++#define GEN10_F2_SS_DIS_SHIFT 18 ++#define GEN10_F2_SS_DIS_MASK (0xf << GEN10_F2_SS_DIS_SHIFT) ++ ++#define GEN10_MIRROR_FUSE3 _MMIO(0x9118) ++#define GEN10_L3BANK_PAIR_COUNT 4 ++#define GEN10_L3BANK_MASK 0x0F ++ ++#define GEN8_EU_DISABLE0 _MMIO(0x9134) ++#define GEN8_EU_DIS0_S0_MASK 0xffffff ++#define GEN8_EU_DIS0_S1_SHIFT 24 ++#define GEN8_EU_DIS0_S1_MASK (0xff << GEN8_EU_DIS0_S1_SHIFT) ++ ++#define GEN8_EU_DISABLE1 _MMIO(0x9138) ++#define GEN8_EU_DIS1_S1_MASK 0xffff ++#define GEN8_EU_DIS1_S2_SHIFT 16 ++#define GEN8_EU_DIS1_S2_MASK (0xffff << GEN8_EU_DIS1_S2_SHIFT) ++ ++#define GEN8_EU_DISABLE2 _MMIO(0x913c) ++#define GEN8_EU_DIS2_S2_MASK 0xff ++ ++#define GEN9_EU_DISABLE(slice) _MMIO(0x9134 + (slice) * 0x4) ++ ++#define GEN10_EU_DISABLE3 _MMIO(0x9140) ++#define GEN10_EU_DIS_SS_MASK 0xff ++ ++#define GEN11_GT_VEBOX_VDBOX_DISABLE _MMIO(0x9140) ++#define GEN11_GT_VDBOX_DISABLE_MASK 0xff ++#define GEN11_GT_VEBOX_DISABLE_SHIFT 16 ++#define GEN11_GT_VEBOX_DISABLE_MASK (0x0f << GEN11_GT_VEBOX_DISABLE_SHIFT) ++ ++#define GEN11_EU_DISABLE _MMIO(0x9134) ++#define GEN11_EU_DIS_MASK 0xFF ++ ++#define GEN11_GT_SLICE_ENABLE _MMIO(0x9138) ++#define GEN11_GT_S_ENA_MASK 0xFF ++ ++#define GEN11_GT_SUBSLICE_DISABLE _MMIO(0x913C) ++ ++#define GEN6_BSD_SLEEP_PSMI_CONTROL _MMIO(0x12050) ++#define GEN6_BSD_SLEEP_MSG_DISABLE (1 << 0) ++#define GEN6_BSD_SLEEP_FLUSH_DISABLE (1 << 2) ++#define GEN6_BSD_SLEEP_INDICATOR (1 << 3) ++#define GEN6_BSD_GO_INDICATOR (1 << 4) ++ ++/* On modern GEN architectures interrupt control consists of two sets ++ * of registers. The first set pertains to the ring generating the ++ * interrupt. The second control is for the functional block generating the ++ * interrupt. These are PM, GT, DE, etc. ++ * ++ * Luckily *knocks on wood* all the ring interrupt bits match up with the ++ * GT interrupt bits, so we don't need to duplicate the defines. ++ * ++ * These defines should cover us well from SNB->HSW with minor exceptions ++ * it can also work on ILK. ++ */ ++#define GT_BLT_FLUSHDW_NOTIFY_INTERRUPT (1 << 26) ++#define GT_BLT_CS_ERROR_INTERRUPT (1 << 25) ++#define GT_BLT_USER_INTERRUPT (1 << 22) ++#define GT_BSD_CS_ERROR_INTERRUPT (1 << 15) ++#define GT_BSD_USER_INTERRUPT (1 << 12) ++#define GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1 (1 << 11) /* hsw+; rsvd on snb, ivb, vlv */ ++#define GT_CONTEXT_SWITCH_INTERRUPT (1 << 8) ++#define GT_RENDER_L3_PARITY_ERROR_INTERRUPT (1 << 5) /* !snb */ ++#define GT_RENDER_PIPECTL_NOTIFY_INTERRUPT (1 << 4) ++#define GT_RENDER_CS_MASTER_ERROR_INTERRUPT (1 << 3) ++#define GT_RENDER_SYNC_STATUS_INTERRUPT (1 << 2) ++#define GT_RENDER_DEBUG_INTERRUPT (1 << 1) ++#define GT_RENDER_USER_INTERRUPT (1 << 0) ++ ++#define PM_VEBOX_CS_ERROR_INTERRUPT (1 << 12) /* hsw+ */ ++#define PM_VEBOX_USER_INTERRUPT (1 << 10) /* hsw+ */ ++ ++#define GT_PARITY_ERROR(dev_priv) \ ++ (GT_RENDER_L3_PARITY_ERROR_INTERRUPT | \ ++ (IS_HASWELL(dev_priv) ? GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1 : 0)) ++ ++/* These are all the "old" interrupts */ ++#define ILK_BSD_USER_INTERRUPT (1 << 5) ++ ++#define I915_PM_INTERRUPT (1 << 31) ++#define I915_ISP_INTERRUPT (1 << 22) ++#define I915_LPE_PIPE_B_INTERRUPT (1 << 21) ++#define I915_LPE_PIPE_A_INTERRUPT (1 << 20) ++#define I915_MIPIC_INTERRUPT (1 << 19) ++#define I915_MIPIA_INTERRUPT (1 << 18) ++#define I915_PIPE_CONTROL_NOTIFY_INTERRUPT (1 << 18) ++#define I915_DISPLAY_PORT_INTERRUPT (1 << 17) ++#define I915_DISPLAY_PIPE_C_HBLANK_INTERRUPT (1 << 16) ++#define I915_MASTER_ERROR_INTERRUPT (1 << 15) ++#define I915_DISPLAY_PIPE_B_HBLANK_INTERRUPT (1 << 14) ++#define I915_GMCH_THERMAL_SENSOR_EVENT_INTERRUPT (1 << 14) /* p-state */ ++#define I915_DISPLAY_PIPE_A_HBLANK_INTERRUPT (1 << 13) ++#define I915_HWB_OOM_INTERRUPT (1 << 13) ++#define I915_LPE_PIPE_C_INTERRUPT (1 << 12) ++#define I915_SYNC_STATUS_INTERRUPT (1 << 12) ++#define I915_MISC_INTERRUPT (1 << 11) ++#define I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT (1 << 11) ++#define I915_DISPLAY_PIPE_C_VBLANK_INTERRUPT (1 << 10) ++#define I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT (1 << 10) ++#define I915_DISPLAY_PIPE_C_EVENT_INTERRUPT (1 << 9) ++#define I915_OVERLAY_PLANE_FLIP_PENDING_INTERRUPT (1 << 9) ++#define I915_DISPLAY_PIPE_C_DPBM_INTERRUPT (1 << 8) ++#define I915_DISPLAY_PLANE_C_FLIP_PENDING_INTERRUPT (1 << 8) ++#define I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT (1 << 7) ++#define I915_DISPLAY_PIPE_A_EVENT_INTERRUPT (1 << 6) ++#define I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT (1 << 5) ++#define I915_DISPLAY_PIPE_B_EVENT_INTERRUPT (1 << 4) ++#define I915_DISPLAY_PIPE_A_DPBM_INTERRUPT (1 << 3) ++#define I915_DISPLAY_PIPE_B_DPBM_INTERRUPT (1 << 2) ++#define I915_DEBUG_INTERRUPT (1 << 2) ++#define I915_WINVALID_INTERRUPT (1 << 1) ++#define I915_USER_INTERRUPT (1 << 1) ++#define I915_ASLE_INTERRUPT (1 << 0) ++#define I915_BSD_USER_INTERRUPT (1 << 25) ++ ++#define I915_HDMI_LPE_AUDIO_BASE (VLV_DISPLAY_BASE + 0x65000) ++#define I915_HDMI_LPE_AUDIO_SIZE 0x1000 ++ ++/* DisplayPort Audio w/ LPE */ ++#define VLV_AUD_CHICKEN_BIT_REG _MMIO(VLV_DISPLAY_BASE + 0x62F38) ++#define VLV_CHICKEN_BIT_DBG_ENABLE (1 << 0) ++ ++#define _VLV_AUD_PORT_EN_B_DBG (VLV_DISPLAY_BASE + 0x62F20) ++#define _VLV_AUD_PORT_EN_C_DBG (VLV_DISPLAY_BASE + 0x62F30) ++#define _VLV_AUD_PORT_EN_D_DBG (VLV_DISPLAY_BASE + 0x62F34) ++#define VLV_AUD_PORT_EN_DBG(port) _MMIO_PORT3((port) - PORT_B, \ ++ _VLV_AUD_PORT_EN_B_DBG, \ ++ _VLV_AUD_PORT_EN_C_DBG, \ ++ _VLV_AUD_PORT_EN_D_DBG) ++#define VLV_AMP_MUTE (1 << 1) ++ ++#define GEN6_BSD_RNCID _MMIO(0x12198) ++ ++#define GEN7_FF_THREAD_MODE _MMIO(0x20a0) ++#define GEN7_FF_SCHED_MASK 0x0077070 ++#define GEN8_FF_DS_REF_CNT_FFME (1 << 19) ++#define GEN7_FF_TS_SCHED_HS1 (0x5 << 16) ++#define GEN7_FF_TS_SCHED_HS0 (0x3 << 16) ++#define GEN7_FF_TS_SCHED_LOAD_BALANCE (0x1 << 16) ++#define GEN7_FF_TS_SCHED_HW (0x0 << 16) /* Default */ ++#define GEN7_FF_VS_REF_CNT_FFME (1 << 15) ++#define GEN7_FF_VS_SCHED_HS1 (0x5 << 12) ++#define GEN7_FF_VS_SCHED_HS0 (0x3 << 12) ++#define GEN7_FF_VS_SCHED_LOAD_BALANCE (0x1 << 12) /* Default */ ++#define GEN7_FF_VS_SCHED_HW (0x0 << 12) ++#define GEN7_FF_DS_SCHED_HS1 (0x5 << 4) ++#define GEN7_FF_DS_SCHED_HS0 (0x3 << 4) ++#define GEN7_FF_DS_SCHED_LOAD_BALANCE (0x1 << 4) /* Default */ ++#define GEN7_FF_DS_SCHED_HW (0x0 << 4) ++ ++/* ++ * Framebuffer compression (915+ only) ++ */ ++ ++#define FBC_CFB_BASE _MMIO(0x3200) /* 4k page aligned */ ++#define FBC_LL_BASE _MMIO(0x3204) /* 4k page aligned */ ++#define FBC_CONTROL _MMIO(0x3208) ++#define FBC_CTL_EN (1 << 31) ++#define FBC_CTL_PERIODIC (1 << 30) ++#define FBC_CTL_INTERVAL_SHIFT (16) ++#define FBC_CTL_UNCOMPRESSIBLE (1 << 14) ++#define FBC_CTL_C3_IDLE (1 << 13) ++#define FBC_CTL_STRIDE_SHIFT (5) ++#define FBC_CTL_FENCENO_SHIFT (0) ++#define FBC_COMMAND _MMIO(0x320c) ++#define FBC_CMD_COMPRESS (1 << 0) ++#define FBC_STATUS _MMIO(0x3210) ++#define FBC_STAT_COMPRESSING (1 << 31) ++#define FBC_STAT_COMPRESSED (1 << 30) ++#define FBC_STAT_MODIFIED (1 << 29) ++#define FBC_STAT_CURRENT_LINE_SHIFT (0) ++#define FBC_CONTROL2 _MMIO(0x3214) ++#define FBC_CTL_FENCE_DBL (0 << 4) ++#define FBC_CTL_IDLE_IMM (0 << 2) ++#define FBC_CTL_IDLE_FULL (1 << 2) ++#define FBC_CTL_IDLE_LINE (2 << 2) ++#define FBC_CTL_IDLE_DEBUG (3 << 2) ++#define FBC_CTL_CPU_FENCE (1 << 1) ++#define FBC_CTL_PLANE(plane) ((plane) << 0) ++#define FBC_FENCE_OFF _MMIO(0x3218) /* BSpec typo has 321Bh */ ++#define FBC_TAG(i) _MMIO(0x3300 + (i) * 4) ++ ++#define FBC_LL_SIZE (1536) ++ ++#define FBC_LLC_READ_CTRL _MMIO(0x9044) ++#define FBC_LLC_FULLY_OPEN (1 << 30) ++ ++/* Framebuffer compression for GM45+ */ ++#define DPFC_CB_BASE _MMIO(0x3200) ++#define DPFC_CONTROL _MMIO(0x3208) ++#define DPFC_CTL_EN (1 << 31) ++#define DPFC_CTL_PLANE(plane) ((plane) << 30) ++#define IVB_DPFC_CTL_PLANE(plane) ((plane) << 29) ++#define DPFC_CTL_FENCE_EN (1 << 29) ++#define IVB_DPFC_CTL_FENCE_EN (1 << 28) ++#define DPFC_CTL_PERSISTENT_MODE (1 << 25) ++#define DPFC_SR_EN (1 << 10) ++#define DPFC_CTL_LIMIT_1X (0 << 6) ++#define DPFC_CTL_LIMIT_2X (1 << 6) ++#define DPFC_CTL_LIMIT_4X (2 << 6) ++#define DPFC_RECOMP_CTL _MMIO(0x320c) ++#define DPFC_RECOMP_STALL_EN (1 << 27) ++#define DPFC_RECOMP_STALL_WM_SHIFT (16) ++#define DPFC_RECOMP_STALL_WM_MASK (0x07ff0000) ++#define DPFC_RECOMP_TIMER_COUNT_SHIFT (0) ++#define DPFC_RECOMP_TIMER_COUNT_MASK (0x0000003f) ++#define DPFC_STATUS _MMIO(0x3210) ++#define DPFC_INVAL_SEG_SHIFT (16) ++#define DPFC_INVAL_SEG_MASK (0x07ff0000) ++#define DPFC_COMP_SEG_SHIFT (0) ++#define DPFC_COMP_SEG_MASK (0x000007ff) ++#define DPFC_STATUS2 _MMIO(0x3214) ++#define DPFC_FENCE_YOFF _MMIO(0x3218) ++#define DPFC_CHICKEN _MMIO(0x3224) ++#define DPFC_HT_MODIFY (1 << 31) ++ ++/* Framebuffer compression for Ironlake */ ++#define ILK_DPFC_CB_BASE _MMIO(0x43200) ++#define ILK_DPFC_CONTROL _MMIO(0x43208) ++#define FBC_CTL_FALSE_COLOR (1 << 10) ++/* The bit 28-8 is reserved */ ++#define DPFC_RESERVED (0x1FFFFF00) ++#define ILK_DPFC_RECOMP_CTL _MMIO(0x4320c) ++#define ILK_DPFC_STATUS _MMIO(0x43210) ++#define ILK_DPFC_COMP_SEG_MASK 0x7ff ++#define IVB_FBC_STATUS2 _MMIO(0x43214) ++#define IVB_FBC_COMP_SEG_MASK 0x7ff ++#define BDW_FBC_COMP_SEG_MASK 0xfff ++#define ILK_DPFC_FENCE_YOFF _MMIO(0x43218) ++#define ILK_DPFC_CHICKEN _MMIO(0x43224) ++#define ILK_DPFC_DISABLE_DUMMY0 (1 << 8) ++#define ILK_DPFC_NUKE_ON_ANY_MODIFICATION (1 << 23) ++#define ILK_FBC_RT_BASE _MMIO(0x2128) ++#define ILK_FBC_RT_VALID (1 << 0) ++#define SNB_FBC_FRONT_BUFFER (1 << 1) ++ ++#define ILK_DISPLAY_CHICKEN1 _MMIO(0x42000) ++#define ILK_FBCQ_DIS (1 << 22) ++#define ILK_PABSTRETCH_DIS (1 << 21) ++ ++ ++/* ++ * Framebuffer compression for Sandybridge ++ * ++ * The following two registers are of type GTTMMADR ++ */ ++#define SNB_DPFC_CTL_SA _MMIO(0x100100) ++#define SNB_CPU_FENCE_ENABLE (1 << 29) ++#define DPFC_CPU_FENCE_OFFSET _MMIO(0x100104) ++ ++/* Framebuffer compression for Ivybridge */ ++#define IVB_FBC_RT_BASE _MMIO(0x7020) ++ ++#define IPS_CTL _MMIO(0x43408) ++#define IPS_ENABLE (1 << 31) ++ ++#define MSG_FBC_REND_STATE _MMIO(0x50380) ++#define FBC_REND_NUKE (1 << 2) ++#define FBC_REND_CACHE_CLEAN (1 << 1) ++ ++/* ++ * GPIO regs ++ */ ++#define GPIO(gpio) _MMIO(dev_priv->gpio_mmio_base + 0x5010 + \ ++ 4 * (gpio)) ++ ++# define GPIO_CLOCK_DIR_MASK (1 << 0) ++# define GPIO_CLOCK_DIR_IN (0 << 1) ++# define GPIO_CLOCK_DIR_OUT (1 << 1) ++# define GPIO_CLOCK_VAL_MASK (1 << 2) ++# define GPIO_CLOCK_VAL_OUT (1 << 3) ++# define GPIO_CLOCK_VAL_IN (1 << 4) ++# define GPIO_CLOCK_PULLUP_DISABLE (1 << 5) ++# define GPIO_DATA_DIR_MASK (1 << 8) ++# define GPIO_DATA_DIR_IN (0 << 9) ++# define GPIO_DATA_DIR_OUT (1 << 9) ++# define GPIO_DATA_VAL_MASK (1 << 10) ++# define GPIO_DATA_VAL_OUT (1 << 11) ++# define GPIO_DATA_VAL_IN (1 << 12) ++# define GPIO_DATA_PULLUP_DISABLE (1 << 13) ++ ++#define GMBUS0 _MMIO(dev_priv->gpio_mmio_base + 0x5100) /* clock/port select */ ++#define GMBUS_AKSV_SELECT (1 << 11) ++#define GMBUS_RATE_100KHZ (0 << 8) ++#define GMBUS_RATE_50KHZ (1 << 8) ++#define GMBUS_RATE_400KHZ (2 << 8) /* reserved on Pineview */ ++#define GMBUS_RATE_1MHZ (3 << 8) /* reserved on Pineview */ ++#define GMBUS_HOLD_EXT (1 << 7) /* 300ns hold time, rsvd on Pineview */ ++#define GMBUS_BYTE_CNT_OVERRIDE (1 << 6) ++#define GMBUS_PIN_DISABLED 0 ++#define GMBUS_PIN_SSC 1 ++#define GMBUS_PIN_VGADDC 2 ++#define GMBUS_PIN_PANEL 3 ++#define GMBUS_PIN_DPD_CHV 3 /* HDMID_CHV */ ++#define GMBUS_PIN_DPC 4 /* HDMIC */ ++#define GMBUS_PIN_DPB 5 /* SDVO, HDMIB */ ++#define GMBUS_PIN_DPD 6 /* HDMID */ ++#define GMBUS_PIN_RESERVED 7 /* 7 reserved */ ++#define GMBUS_PIN_1_BXT 1 /* BXT+ (atom) and CNP+ (big core) */ ++#define GMBUS_PIN_2_BXT 2 ++#define GMBUS_PIN_3_BXT 3 ++#define GMBUS_PIN_4_CNP 4 ++#define GMBUS_PIN_9_TC1_ICP 9 ++#define GMBUS_PIN_10_TC2_ICP 10 ++#define GMBUS_PIN_11_TC3_ICP 11 ++#define GMBUS_PIN_12_TC4_ICP 12 ++ ++#define GMBUS_NUM_PINS 13 /* including 0 */ ++#define GMBUS1 _MMIO(dev_priv->gpio_mmio_base + 0x5104) /* command/status */ ++#define GMBUS_SW_CLR_INT (1 << 31) ++#define GMBUS_SW_RDY (1 << 30) ++#define GMBUS_ENT (1 << 29) /* enable timeout */ ++#define GMBUS_CYCLE_NONE (0 << 25) ++#define GMBUS_CYCLE_WAIT (1 << 25) ++#define GMBUS_CYCLE_INDEX (2 << 25) ++#define GMBUS_CYCLE_STOP (4 << 25) ++#define GMBUS_BYTE_COUNT_SHIFT 16 ++#define GMBUS_BYTE_COUNT_MAX 256U ++#define GEN9_GMBUS_BYTE_COUNT_MAX 511U ++#define GMBUS_SLAVE_INDEX_SHIFT 8 ++#define GMBUS_SLAVE_ADDR_SHIFT 1 ++#define GMBUS_SLAVE_READ (1 << 0) ++#define GMBUS_SLAVE_WRITE (0 << 0) ++#define GMBUS2 _MMIO(dev_priv->gpio_mmio_base + 0x5108) /* status */ ++#define GMBUS_INUSE (1 << 15) ++#define GMBUS_HW_WAIT_PHASE (1 << 14) ++#define GMBUS_STALL_TIMEOUT (1 << 13) ++#define GMBUS_INT (1 << 12) ++#define GMBUS_HW_RDY (1 << 11) ++#define GMBUS_SATOER (1 << 10) ++#define GMBUS_ACTIVE (1 << 9) ++#define GMBUS3 _MMIO(dev_priv->gpio_mmio_base + 0x510c) /* data buffer bytes 3-0 */ ++#define GMBUS4 _MMIO(dev_priv->gpio_mmio_base + 0x5110) /* interrupt mask (Pineview+) */ ++#define GMBUS_SLAVE_TIMEOUT_EN (1 << 4) ++#define GMBUS_NAK_EN (1 << 3) ++#define GMBUS_IDLE_EN (1 << 2) ++#define GMBUS_HW_WAIT_EN (1 << 1) ++#define GMBUS_HW_RDY_EN (1 << 0) ++#define GMBUS5 _MMIO(dev_priv->gpio_mmio_base + 0x5120) /* byte index */ ++#define GMBUS_2BYTE_INDEX_EN (1 << 31) ++ ++/* ++ * Clock control & power management ++ */ ++#define _DPLL_A (DISPLAY_MMIO_BASE(dev_priv) + 0x6014) ++#define _DPLL_B (DISPLAY_MMIO_BASE(dev_priv) + 0x6018) ++#define _CHV_DPLL_C (DISPLAY_MMIO_BASE(dev_priv) + 0x6030) ++#define DPLL(pipe) _MMIO_PIPE3((pipe), _DPLL_A, _DPLL_B, _CHV_DPLL_C) ++ ++#define VGA0 _MMIO(0x6000) ++#define VGA1 _MMIO(0x6004) ++#define VGA_PD _MMIO(0x6010) ++#define VGA0_PD_P2_DIV_4 (1 << 7) ++#define VGA0_PD_P1_DIV_2 (1 << 5) ++#define VGA0_PD_P1_SHIFT 0 ++#define VGA0_PD_P1_MASK (0x1f << 0) ++#define VGA1_PD_P2_DIV_4 (1 << 15) ++#define VGA1_PD_P1_DIV_2 (1 << 13) ++#define VGA1_PD_P1_SHIFT 8 ++#define VGA1_PD_P1_MASK (0x1f << 8) ++#define DPLL_VCO_ENABLE (1 << 31) ++#define DPLL_SDVO_HIGH_SPEED (1 << 30) ++#define DPLL_DVO_2X_MODE (1 << 30) ++#define DPLL_EXT_BUFFER_ENABLE_VLV (1 << 30) ++#define DPLL_SYNCLOCK_ENABLE (1 << 29) ++#define DPLL_REF_CLK_ENABLE_VLV (1 << 29) ++#define DPLL_VGA_MODE_DIS (1 << 28) ++#define DPLLB_MODE_DAC_SERIAL (1 << 26) /* i915 */ ++#define DPLLB_MODE_LVDS (2 << 26) /* i915 */ ++#define DPLL_MODE_MASK (3 << 26) ++#define DPLL_DAC_SERIAL_P2_CLOCK_DIV_10 (0 << 24) /* i915 */ ++#define DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 (1 << 24) /* i915 */ ++#define DPLLB_LVDS_P2_CLOCK_DIV_14 (0 << 24) /* i915 */ ++#define DPLLB_LVDS_P2_CLOCK_DIV_7 (1 << 24) /* i915 */ ++#define DPLL_P2_CLOCK_DIV_MASK 0x03000000 /* i915 */ ++#define DPLL_FPA01_P1_POST_DIV_MASK 0x00ff0000 /* i915 */ ++#define DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW 0x00ff8000 /* Pineview */ ++#define DPLL_LOCK_VLV (1 << 15) ++#define DPLL_INTEGRATED_CRI_CLK_VLV (1 << 14) ++#define DPLL_INTEGRATED_REF_CLK_VLV (1 << 13) ++#define DPLL_SSC_REF_CLK_CHV (1 << 13) ++#define DPLL_PORTC_READY_MASK (0xf << 4) ++#define DPLL_PORTB_READY_MASK (0xf) ++ ++#define DPLL_FPA01_P1_POST_DIV_MASK_I830 0x001f0000 ++ ++/* Additional CHV pll/phy registers */ ++#define DPIO_PHY_STATUS _MMIO(VLV_DISPLAY_BASE + 0x6240) ++#define DPLL_PORTD_READY_MASK (0xf) ++#define DISPLAY_PHY_CONTROL _MMIO(VLV_DISPLAY_BASE + 0x60100) ++#define PHY_CH_POWER_DOWN_OVRD_EN(phy, ch) (1 << (2 * (phy) + (ch) + 27)) ++#define PHY_LDO_DELAY_0NS 0x0 ++#define PHY_LDO_DELAY_200NS 0x1 ++#define PHY_LDO_DELAY_600NS 0x2 ++#define PHY_LDO_SEQ_DELAY(delay, phy) ((delay) << (2 * (phy) + 23)) ++#define PHY_CH_POWER_DOWN_OVRD(mask, phy, ch) ((mask) << (8 * (phy) + 4 * (ch) + 11)) ++#define PHY_CH_SU_PSR 0x1 ++#define PHY_CH_DEEP_PSR 0x7 ++#define PHY_CH_POWER_MODE(mode, phy, ch) ((mode) << (6 * (phy) + 3 * (ch) + 2)) ++#define PHY_COM_LANE_RESET_DEASSERT(phy) (1 << (phy)) ++#define DISPLAY_PHY_STATUS _MMIO(VLV_DISPLAY_BASE + 0x60104) ++#define PHY_POWERGOOD(phy) (((phy) == DPIO_PHY0) ? (1 << 31) : (1 << 30)) ++#define PHY_STATUS_CMN_LDO(phy, ch) (1 << (6 - (6 * (phy) + 3 * (ch)))) ++#define PHY_STATUS_SPLINE_LDO(phy, ch, spline) (1 << (8 - (6 * (phy) + 3 * (ch) + (spline)))) ++ ++/* ++ * The i830 generation, in LVDS mode, defines P1 as the bit number set within ++ * this field (only one bit may be set). ++ */ ++#define DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS 0x003f0000 ++#define DPLL_FPA01_P1_POST_DIV_SHIFT 16 ++#define DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW 15 ++/* i830, required in DVO non-gang */ ++#define PLL_P2_DIVIDE_BY_4 (1 << 23) ++#define PLL_P1_DIVIDE_BY_TWO (1 << 21) /* i830 */ ++#define PLL_REF_INPUT_DREFCLK (0 << 13) ++#define PLL_REF_INPUT_TVCLKINA (1 << 13) /* i830 */ ++#define PLL_REF_INPUT_TVCLKINBC (2 << 13) /* SDVO TVCLKIN */ ++#define PLLB_REF_INPUT_SPREADSPECTRUMIN (3 << 13) ++#define PLL_REF_INPUT_MASK (3 << 13) ++#define PLL_LOAD_PULSE_PHASE_SHIFT 9 ++/* Ironlake */ ++# define PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT 9 ++# define PLL_REF_SDVO_HDMI_MULTIPLIER_MASK (7 << 9) ++# define PLL_REF_SDVO_HDMI_MULTIPLIER(x) (((x) - 1) << 9) ++# define DPLL_FPA1_P1_POST_DIV_SHIFT 0 ++# define DPLL_FPA1_P1_POST_DIV_MASK 0xff ++ ++/* ++ * Parallel to Serial Load Pulse phase selection. ++ * Selects the phase for the 10X DPLL clock for the PCIe ++ * digital display port. The range is 4 to 13; 10 or more ++ * is just a flip delay. The default is 6 ++ */ ++#define PLL_LOAD_PULSE_PHASE_MASK (0xf << PLL_LOAD_PULSE_PHASE_SHIFT) ++#define DISPLAY_RATE_SELECT_FPA1 (1 << 8) ++/* ++ * SDVO multiplier for 945G/GM. Not used on 965. ++ */ ++#define SDVO_MULTIPLIER_MASK 0x000000ff ++#define SDVO_MULTIPLIER_SHIFT_HIRES 4 ++#define SDVO_MULTIPLIER_SHIFT_VGA 0 ++ ++#define _DPLL_A_MD (DISPLAY_MMIO_BASE(dev_priv) + 0x601c) ++#define _DPLL_B_MD (DISPLAY_MMIO_BASE(dev_priv) + 0x6020) ++#define _CHV_DPLL_C_MD (DISPLAY_MMIO_BASE(dev_priv) + 0x603c) ++#define DPLL_MD(pipe) _MMIO_PIPE3((pipe), _DPLL_A_MD, _DPLL_B_MD, _CHV_DPLL_C_MD) ++ ++/* ++ * UDI pixel divider, controlling how many pixels are stuffed into a packet. ++ * ++ * Value is pixels minus 1. Must be set to 1 pixel for SDVO. ++ */ ++#define DPLL_MD_UDI_DIVIDER_MASK 0x3f000000 ++#define DPLL_MD_UDI_DIVIDER_SHIFT 24 ++/* UDI pixel divider for VGA, same as DPLL_MD_UDI_DIVIDER_MASK. */ ++#define DPLL_MD_VGA_UDI_DIVIDER_MASK 0x003f0000 ++#define DPLL_MD_VGA_UDI_DIVIDER_SHIFT 16 ++/* ++ * SDVO/UDI pixel multiplier. ++ * ++ * SDVO requires that the bus clock rate be between 1 and 2 Ghz, and the bus ++ * clock rate is 10 times the DPLL clock. At low resolution/refresh rate ++ * modes, the bus rate would be below the limits, so SDVO allows for stuffing ++ * dummy bytes in the datastream at an increased clock rate, with both sides of ++ * the link knowing how many bytes are fill. ++ * ++ * So, for a mode with a dotclock of 65Mhz, we would want to double the clock ++ * rate to 130Mhz to get a bus rate of 1.30Ghz. The DPLL clock rate would be ++ * set to 130Mhz, and the SDVO multiplier set to 2x in this register and ++ * through an SDVO command. ++ * ++ * This register field has values of multiplication factor minus 1, with ++ * a maximum multiplier of 5 for SDVO. ++ */ ++#define DPLL_MD_UDI_MULTIPLIER_MASK 0x00003f00 ++#define DPLL_MD_UDI_MULTIPLIER_SHIFT 8 ++/* ++ * SDVO/UDI pixel multiplier for VGA, same as DPLL_MD_UDI_MULTIPLIER_MASK. ++ * This best be set to the default value (3) or the CRT won't work. No, ++ * I don't entirely understand what this does... ++ */ ++#define DPLL_MD_VGA_UDI_MULTIPLIER_MASK 0x0000003f ++#define DPLL_MD_VGA_UDI_MULTIPLIER_SHIFT 0 ++ ++#define RAWCLK_FREQ_VLV _MMIO(VLV_DISPLAY_BASE + 0x6024) ++ ++#define _FPA0 0x6040 ++#define _FPA1 0x6044 ++#define _FPB0 0x6048 ++#define _FPB1 0x604c ++#define FP0(pipe) _MMIO_PIPE(pipe, _FPA0, _FPB0) ++#define FP1(pipe) _MMIO_PIPE(pipe, _FPA1, _FPB1) ++#define FP_N_DIV_MASK 0x003f0000 ++#define FP_N_PINEVIEW_DIV_MASK 0x00ff0000 ++#define FP_N_DIV_SHIFT 16 ++#define FP_M1_DIV_MASK 0x00003f00 ++#define FP_M1_DIV_SHIFT 8 ++#define FP_M2_DIV_MASK 0x0000003f ++#define FP_M2_PINEVIEW_DIV_MASK 0x000000ff ++#define FP_M2_DIV_SHIFT 0 ++#define DPLL_TEST _MMIO(0x606c) ++#define DPLLB_TEST_SDVO_DIV_1 (0 << 22) ++#define DPLLB_TEST_SDVO_DIV_2 (1 << 22) ++#define DPLLB_TEST_SDVO_DIV_4 (2 << 22) ++#define DPLLB_TEST_SDVO_DIV_MASK (3 << 22) ++#define DPLLB_TEST_N_BYPASS (1 << 19) ++#define DPLLB_TEST_M_BYPASS (1 << 18) ++#define DPLLB_INPUT_BUFFER_ENABLE (1 << 16) ++#define DPLLA_TEST_N_BYPASS (1 << 3) ++#define DPLLA_TEST_M_BYPASS (1 << 2) ++#define DPLLA_INPUT_BUFFER_ENABLE (1 << 0) ++#define D_STATE _MMIO(0x6104) ++#define DSTATE_GFX_RESET_I830 (1 << 6) ++#define DSTATE_PLL_D3_OFF (1 << 3) ++#define DSTATE_GFX_CLOCK_GATING (1 << 1) ++#define DSTATE_DOT_CLOCK_GATING (1 << 0) ++#define DSPCLK_GATE_D _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x6200) ++# define DPUNIT_B_CLOCK_GATE_DISABLE (1 << 30) /* 965 */ ++# define VSUNIT_CLOCK_GATE_DISABLE (1 << 29) /* 965 */ ++# define VRHUNIT_CLOCK_GATE_DISABLE (1 << 28) /* 965 */ ++# define VRDUNIT_CLOCK_GATE_DISABLE (1 << 27) /* 965 */ ++# define AUDUNIT_CLOCK_GATE_DISABLE (1 << 26) /* 965 */ ++# define DPUNIT_A_CLOCK_GATE_DISABLE (1 << 25) /* 965 */ ++# define DPCUNIT_CLOCK_GATE_DISABLE (1 << 24) /* 965 */ ++# define PNV_GMBUSUNIT_CLOCK_GATE_DISABLE (1 << 24) /* pnv */ ++# define TVRUNIT_CLOCK_GATE_DISABLE (1 << 23) /* 915-945 */ ++# define TVCUNIT_CLOCK_GATE_DISABLE (1 << 22) /* 915-945 */ ++# define TVFUNIT_CLOCK_GATE_DISABLE (1 << 21) /* 915-945 */ ++# define TVEUNIT_CLOCK_GATE_DISABLE (1 << 20) /* 915-945 */ ++# define DVSUNIT_CLOCK_GATE_DISABLE (1 << 19) /* 915-945 */ ++# define DSSUNIT_CLOCK_GATE_DISABLE (1 << 18) /* 915-945 */ ++# define DDBUNIT_CLOCK_GATE_DISABLE (1 << 17) /* 915-945 */ ++# define DPRUNIT_CLOCK_GATE_DISABLE (1 << 16) /* 915-945 */ ++# define DPFUNIT_CLOCK_GATE_DISABLE (1 << 15) /* 915-945 */ ++# define DPBMUNIT_CLOCK_GATE_DISABLE (1 << 14) /* 915-945 */ ++# define DPLSUNIT_CLOCK_GATE_DISABLE (1 << 13) /* 915-945 */ ++# define DPLUNIT_CLOCK_GATE_DISABLE (1 << 12) /* 915-945 */ ++# define DPOUNIT_CLOCK_GATE_DISABLE (1 << 11) ++# define DPBUNIT_CLOCK_GATE_DISABLE (1 << 10) ++# define DCUNIT_CLOCK_GATE_DISABLE (1 << 9) ++# define DPUNIT_CLOCK_GATE_DISABLE (1 << 8) ++# define VRUNIT_CLOCK_GATE_DISABLE (1 << 7) /* 915+: reserved */ ++# define OVHUNIT_CLOCK_GATE_DISABLE (1 << 6) /* 830-865 */ ++# define DPIOUNIT_CLOCK_GATE_DISABLE (1 << 6) /* 915-945 */ ++# define OVFUNIT_CLOCK_GATE_DISABLE (1 << 5) ++# define OVBUNIT_CLOCK_GATE_DISABLE (1 << 4) ++/* ++ * This bit must be set on the 830 to prevent hangs when turning off the ++ * overlay scaler. ++ */ ++# define OVRUNIT_CLOCK_GATE_DISABLE (1 << 3) ++# define OVCUNIT_CLOCK_GATE_DISABLE (1 << 2) ++# define OVUUNIT_CLOCK_GATE_DISABLE (1 << 1) ++# define ZVUNIT_CLOCK_GATE_DISABLE (1 << 0) /* 830 */ ++# define OVLUNIT_CLOCK_GATE_DISABLE (1 << 0) /* 845,865 */ ++ ++#define RENCLK_GATE_D1 _MMIO(0x6204) ++# define BLITTER_CLOCK_GATE_DISABLE (1 << 13) /* 945GM only */ ++# define MPEG_CLOCK_GATE_DISABLE (1 << 12) /* 945GM only */ ++# define PC_FE_CLOCK_GATE_DISABLE (1 << 11) ++# define PC_BE_CLOCK_GATE_DISABLE (1 << 10) ++# define WINDOWER_CLOCK_GATE_DISABLE (1 << 9) ++# define INTERPOLATOR_CLOCK_GATE_DISABLE (1 << 8) ++# define COLOR_CALCULATOR_CLOCK_GATE_DISABLE (1 << 7) ++# define MOTION_COMP_CLOCK_GATE_DISABLE (1 << 6) ++# define MAG_CLOCK_GATE_DISABLE (1 << 5) ++/* This bit must be unset on 855,865 */ ++# define MECI_CLOCK_GATE_DISABLE (1 << 4) ++# define DCMP_CLOCK_GATE_DISABLE (1 << 3) ++# define MEC_CLOCK_GATE_DISABLE (1 << 2) ++# define MECO_CLOCK_GATE_DISABLE (1 << 1) ++/* This bit must be set on 855,865. */ ++# define SV_CLOCK_GATE_DISABLE (1 << 0) ++# define I915_MPEG_CLOCK_GATE_DISABLE (1 << 16) ++# define I915_VLD_IP_PR_CLOCK_GATE_DISABLE (1 << 15) ++# define I915_MOTION_COMP_CLOCK_GATE_DISABLE (1 << 14) ++# define I915_BD_BF_CLOCK_GATE_DISABLE (1 << 13) ++# define I915_SF_SE_CLOCK_GATE_DISABLE (1 << 12) ++# define I915_WM_CLOCK_GATE_DISABLE (1 << 11) ++# define I915_IZ_CLOCK_GATE_DISABLE (1 << 10) ++# define I915_PI_CLOCK_GATE_DISABLE (1 << 9) ++# define I915_DI_CLOCK_GATE_DISABLE (1 << 8) ++# define I915_SH_SV_CLOCK_GATE_DISABLE (1 << 7) ++# define I915_PL_DG_QC_FT_CLOCK_GATE_DISABLE (1 << 6) ++# define I915_SC_CLOCK_GATE_DISABLE (1 << 5) ++# define I915_FL_CLOCK_GATE_DISABLE (1 << 4) ++# define I915_DM_CLOCK_GATE_DISABLE (1 << 3) ++# define I915_PS_CLOCK_GATE_DISABLE (1 << 2) ++# define I915_CC_CLOCK_GATE_DISABLE (1 << 1) ++# define I915_BY_CLOCK_GATE_DISABLE (1 << 0) ++ ++# define I965_RCZ_CLOCK_GATE_DISABLE (1 << 30) ++/* This bit must always be set on 965G/965GM */ ++# define I965_RCC_CLOCK_GATE_DISABLE (1 << 29) ++# define I965_RCPB_CLOCK_GATE_DISABLE (1 << 28) ++# define I965_DAP_CLOCK_GATE_DISABLE (1 << 27) ++# define I965_ROC_CLOCK_GATE_DISABLE (1 << 26) ++# define I965_GW_CLOCK_GATE_DISABLE (1 << 25) ++# define I965_TD_CLOCK_GATE_DISABLE (1 << 24) ++/* This bit must always be set on 965G */ ++# define I965_ISC_CLOCK_GATE_DISABLE (1 << 23) ++# define I965_IC_CLOCK_GATE_DISABLE (1 << 22) ++# define I965_EU_CLOCK_GATE_DISABLE (1 << 21) ++# define I965_IF_CLOCK_GATE_DISABLE (1 << 20) ++# define I965_TC_CLOCK_GATE_DISABLE (1 << 19) ++# define I965_SO_CLOCK_GATE_DISABLE (1 << 17) ++# define I965_FBC_CLOCK_GATE_DISABLE (1 << 16) ++# define I965_MARI_CLOCK_GATE_DISABLE (1 << 15) ++# define I965_MASF_CLOCK_GATE_DISABLE (1 << 14) ++# define I965_MAWB_CLOCK_GATE_DISABLE (1 << 13) ++# define I965_EM_CLOCK_GATE_DISABLE (1 << 12) ++# define I965_UC_CLOCK_GATE_DISABLE (1 << 11) ++# define I965_SI_CLOCK_GATE_DISABLE (1 << 6) ++# define I965_MT_CLOCK_GATE_DISABLE (1 << 5) ++# define I965_PL_CLOCK_GATE_DISABLE (1 << 4) ++# define I965_DG_CLOCK_GATE_DISABLE (1 << 3) ++# define I965_QC_CLOCK_GATE_DISABLE (1 << 2) ++# define I965_FT_CLOCK_GATE_DISABLE (1 << 1) ++# define I965_DM_CLOCK_GATE_DISABLE (1 << 0) ++ ++#define RENCLK_GATE_D2 _MMIO(0x6208) ++#define VF_UNIT_CLOCK_GATE_DISABLE (1 << 9) ++#define GS_UNIT_CLOCK_GATE_DISABLE (1 << 7) ++#define CL_UNIT_CLOCK_GATE_DISABLE (1 << 6) ++ ++#define VDECCLK_GATE_D _MMIO(0x620C) /* g4x only */ ++#define VCP_UNIT_CLOCK_GATE_DISABLE (1 << 4) ++ ++#define RAMCLK_GATE_D _MMIO(0x6210) /* CRL only */ ++#define DEUC _MMIO(0x6214) /* CRL only */ ++ ++#define FW_BLC_SELF_VLV _MMIO(VLV_DISPLAY_BASE + 0x6500) ++#define FW_CSPWRDWNEN (1 << 15) ++ ++#define MI_ARB_VLV _MMIO(VLV_DISPLAY_BASE + 0x6504) ++ ++#define CZCLK_CDCLK_FREQ_RATIO _MMIO(VLV_DISPLAY_BASE + 0x6508) ++#define CDCLK_FREQ_SHIFT 4 ++#define CDCLK_FREQ_MASK (0x1f << CDCLK_FREQ_SHIFT) ++#define CZCLK_FREQ_MASK 0xf ++ ++#define GCI_CONTROL _MMIO(VLV_DISPLAY_BASE + 0x650C) ++#define PFI_CREDIT_63 (9 << 28) /* chv only */ ++#define PFI_CREDIT_31 (8 << 28) /* chv only */ ++#define PFI_CREDIT(x) (((x) - 8) << 28) /* 8-15 */ ++#define PFI_CREDIT_RESEND (1 << 27) ++#define VGA_FAST_MODE_DISABLE (1 << 14) ++ ++#define GMBUSFREQ_VLV _MMIO(VLV_DISPLAY_BASE + 0x6510) ++ ++/* ++ * Palette regs ++ */ ++#define _PALETTE_A 0xa000 ++#define _PALETTE_B 0xa800 ++#define _CHV_PALETTE_C 0xc000 ++#define PALETTE(pipe, i) _MMIO(DISPLAY_MMIO_BASE(dev_priv) + \ ++ _PICK((pipe), _PALETTE_A, \ ++ _PALETTE_B, _CHV_PALETTE_C) + \ ++ (i) * 4) ++ ++/* MCH MMIO space */ ++ ++/* ++ * MCHBAR mirror. ++ * ++ * This mirrors the MCHBAR MMIO space whose location is determined by ++ * device 0 function 0's pci config register 0x44 or 0x48 and matches it in ++ * every way. It is not accessible from the CP register read instructions. ++ * ++ * Starting from Haswell, you can't write registers using the MCHBAR mirror, ++ * just read. ++ */ ++#define MCHBAR_MIRROR_BASE 0x10000 ++ ++#define MCHBAR_MIRROR_BASE_SNB 0x140000 ++ ++#define CTG_STOLEN_RESERVED _MMIO(MCHBAR_MIRROR_BASE + 0x34) ++#define ELK_STOLEN_RESERVED _MMIO(MCHBAR_MIRROR_BASE + 0x48) ++#define G4X_STOLEN_RESERVED_ADDR1_MASK (0xFFFF << 16) ++#define G4X_STOLEN_RESERVED_ADDR2_MASK (0xFFF << 4) ++#define G4X_STOLEN_RESERVED_ENABLE (1 << 0) ++ ++/* Memory controller frequency in MCHBAR for Haswell (possible SNB+) */ ++#define DCLK _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5e04) ++ ++/* 915-945 and GM965 MCH register controlling DRAM channel access */ ++#define DCC _MMIO(MCHBAR_MIRROR_BASE + 0x200) ++#define DCC_ADDRESSING_MODE_SINGLE_CHANNEL (0 << 0) ++#define DCC_ADDRESSING_MODE_DUAL_CHANNEL_ASYMMETRIC (1 << 0) ++#define DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED (2 << 0) ++#define DCC_ADDRESSING_MODE_MASK (3 << 0) ++#define DCC_CHANNEL_XOR_DISABLE (1 << 10) ++#define DCC_CHANNEL_XOR_BIT_17 (1 << 9) ++#define DCC2 _MMIO(MCHBAR_MIRROR_BASE + 0x204) ++#define DCC2_MODIFIED_ENHANCED_DISABLE (1 << 20) ++ ++/* Pineview MCH register contains DDR3 setting */ ++#define CSHRDDR3CTL _MMIO(MCHBAR_MIRROR_BASE + 0x1a8) ++#define CSHRDDR3CTL_DDR3 (1 << 2) ++ ++/* 965 MCH register controlling DRAM channel configuration */ ++#define C0DRB3 _MMIO(MCHBAR_MIRROR_BASE + 0x206) ++#define C1DRB3 _MMIO(MCHBAR_MIRROR_BASE + 0x606) ++ ++/* snb MCH registers for reading the DRAM channel configuration */ ++#define MAD_DIMM_C0 _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5004) ++#define MAD_DIMM_C1 _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5008) ++#define MAD_DIMM_C2 _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x500C) ++#define MAD_DIMM_ECC_MASK (0x3 << 24) ++#define MAD_DIMM_ECC_OFF (0x0 << 24) ++#define MAD_DIMM_ECC_IO_ON_LOGIC_OFF (0x1 << 24) ++#define MAD_DIMM_ECC_IO_OFF_LOGIC_ON (0x2 << 24) ++#define MAD_DIMM_ECC_ON (0x3 << 24) ++#define MAD_DIMM_ENH_INTERLEAVE (0x1 << 22) ++#define MAD_DIMM_RANK_INTERLEAVE (0x1 << 21) ++#define MAD_DIMM_B_WIDTH_X16 (0x1 << 20) /* X8 chips if unset */ ++#define MAD_DIMM_A_WIDTH_X16 (0x1 << 19) /* X8 chips if unset */ ++#define MAD_DIMM_B_DUAL_RANK (0x1 << 18) ++#define MAD_DIMM_A_DUAL_RANK (0x1 << 17) ++#define MAD_DIMM_A_SELECT (0x1 << 16) ++/* DIMM sizes are in multiples of 256mb. */ ++#define MAD_DIMM_B_SIZE_SHIFT 8 ++#define MAD_DIMM_B_SIZE_MASK (0xff << MAD_DIMM_B_SIZE_SHIFT) ++#define MAD_DIMM_A_SIZE_SHIFT 0 ++#define MAD_DIMM_A_SIZE_MASK (0xff << MAD_DIMM_A_SIZE_SHIFT) ++ ++/* snb MCH registers for priority tuning */ ++#define MCH_SSKPD _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5d10) ++#define MCH_SSKPD_WM0_MASK 0x3f ++#define MCH_SSKPD_WM0_VAL 0xc ++ ++#define MCH_SECP_NRG_STTS _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x592c) ++ ++/* Clocking configuration register */ ++#define CLKCFG _MMIO(MCHBAR_MIRROR_BASE + 0xc00) ++#define CLKCFG_FSB_400 (5 << 0) /* hrawclk 100 */ ++#define CLKCFG_FSB_533 (1 << 0) /* hrawclk 133 */ ++#define CLKCFG_FSB_667 (3 << 0) /* hrawclk 166 */ ++#define CLKCFG_FSB_800 (2 << 0) /* hrawclk 200 */ ++#define CLKCFG_FSB_1067 (6 << 0) /* hrawclk 266 */ ++#define CLKCFG_FSB_1067_ALT (0 << 0) /* hrawclk 266 */ ++#define CLKCFG_FSB_1333 (7 << 0) /* hrawclk 333 */ ++/* ++ * Note that on at least on ELK the below value is reported for both ++ * 333 and 400 MHz BIOS FSB setting, but given that the gmch datasheet ++ * lists only 200/266/333 MHz FSB as supported let's decode it as 333 MHz. ++ */ ++#define CLKCFG_FSB_1333_ALT (4 << 0) /* hrawclk 333 */ ++#define CLKCFG_FSB_MASK (7 << 0) ++#define CLKCFG_MEM_533 (1 << 4) ++#define CLKCFG_MEM_667 (2 << 4) ++#define CLKCFG_MEM_800 (3 << 4) ++#define CLKCFG_MEM_MASK (7 << 4) ++ ++#define HPLLVCO _MMIO(MCHBAR_MIRROR_BASE + 0xc38) ++#define HPLLVCO_MOBILE _MMIO(MCHBAR_MIRROR_BASE + 0xc0f) ++ ++#define TSC1 _MMIO(0x11001) ++#define TSE (1 << 0) ++#define TR1 _MMIO(0x11006) ++#define TSFS _MMIO(0x11020) ++#define TSFS_SLOPE_MASK 0x0000ff00 ++#define TSFS_SLOPE_SHIFT 8 ++#define TSFS_INTR_MASK 0x000000ff ++ ++#define CRSTANDVID _MMIO(0x11100) ++#define PXVFREQ(fstart) _MMIO(0x11110 + (fstart) * 4) /* P[0-15]VIDFREQ (0x1114c) (Ironlake) */ ++#define PXVFREQ_PX_MASK 0x7f000000 ++#define PXVFREQ_PX_SHIFT 24 ++#define VIDFREQ_BASE _MMIO(0x11110) ++#define VIDFREQ1 _MMIO(0x11110) /* VIDFREQ1-4 (0x1111c) (Cantiga) */ ++#define VIDFREQ2 _MMIO(0x11114) ++#define VIDFREQ3 _MMIO(0x11118) ++#define VIDFREQ4 _MMIO(0x1111c) ++#define VIDFREQ_P0_MASK 0x1f000000 ++#define VIDFREQ_P0_SHIFT 24 ++#define VIDFREQ_P0_CSCLK_MASK 0x00f00000 ++#define VIDFREQ_P0_CSCLK_SHIFT 20 ++#define VIDFREQ_P0_CRCLK_MASK 0x000f0000 ++#define VIDFREQ_P0_CRCLK_SHIFT 16 ++#define VIDFREQ_P1_MASK 0x00001f00 ++#define VIDFREQ_P1_SHIFT 8 ++#define VIDFREQ_P1_CSCLK_MASK 0x000000f0 ++#define VIDFREQ_P1_CSCLK_SHIFT 4 ++#define VIDFREQ_P1_CRCLK_MASK 0x0000000f ++#define INTTOEXT_BASE_ILK _MMIO(0x11300) ++#define INTTOEXT_BASE _MMIO(0x11120) /* INTTOEXT1-8 (0x1113c) */ ++#define INTTOEXT_MAP3_SHIFT 24 ++#define INTTOEXT_MAP3_MASK (0x1f << INTTOEXT_MAP3_SHIFT) ++#define INTTOEXT_MAP2_SHIFT 16 ++#define INTTOEXT_MAP2_MASK (0x1f << INTTOEXT_MAP2_SHIFT) ++#define INTTOEXT_MAP1_SHIFT 8 ++#define INTTOEXT_MAP1_MASK (0x1f << INTTOEXT_MAP1_SHIFT) ++#define INTTOEXT_MAP0_SHIFT 0 ++#define INTTOEXT_MAP0_MASK (0x1f << INTTOEXT_MAP0_SHIFT) ++#define MEMSWCTL _MMIO(0x11170) /* Ironlake only */ ++#define MEMCTL_CMD_MASK 0xe000 ++#define MEMCTL_CMD_SHIFT 13 ++#define MEMCTL_CMD_RCLK_OFF 0 ++#define MEMCTL_CMD_RCLK_ON 1 ++#define MEMCTL_CMD_CHFREQ 2 ++#define MEMCTL_CMD_CHVID 3 ++#define MEMCTL_CMD_VMMOFF 4 ++#define MEMCTL_CMD_VMMON 5 ++#define MEMCTL_CMD_STS (1 << 12) /* write 1 triggers command, clears ++ when command complete */ ++#define MEMCTL_FREQ_MASK 0x0f00 /* jitter, from 0-15 */ ++#define MEMCTL_FREQ_SHIFT 8 ++#define MEMCTL_SFCAVM (1 << 7) ++#define MEMCTL_TGT_VID_MASK 0x007f ++#define MEMIHYST _MMIO(0x1117c) ++#define MEMINTREN _MMIO(0x11180) /* 16 bits */ ++#define MEMINT_RSEXIT_EN (1 << 8) ++#define MEMINT_CX_SUPR_EN (1 << 7) ++#define MEMINT_CONT_BUSY_EN (1 << 6) ++#define MEMINT_AVG_BUSY_EN (1 << 5) ++#define MEMINT_EVAL_CHG_EN (1 << 4) ++#define MEMINT_MON_IDLE_EN (1 << 3) ++#define MEMINT_UP_EVAL_EN (1 << 2) ++#define MEMINT_DOWN_EVAL_EN (1 << 1) ++#define MEMINT_SW_CMD_EN (1 << 0) ++#define MEMINTRSTR _MMIO(0x11182) /* 16 bits */ ++#define MEM_RSEXIT_MASK 0xc000 ++#define MEM_RSEXIT_SHIFT 14 ++#define MEM_CONT_BUSY_MASK 0x3000 ++#define MEM_CONT_BUSY_SHIFT 12 ++#define MEM_AVG_BUSY_MASK 0x0c00 ++#define MEM_AVG_BUSY_SHIFT 10 ++#define MEM_EVAL_CHG_MASK 0x0300 ++#define MEM_EVAL_BUSY_SHIFT 8 ++#define MEM_MON_IDLE_MASK 0x00c0 ++#define MEM_MON_IDLE_SHIFT 6 ++#define MEM_UP_EVAL_MASK 0x0030 ++#define MEM_UP_EVAL_SHIFT 4 ++#define MEM_DOWN_EVAL_MASK 0x000c ++#define MEM_DOWN_EVAL_SHIFT 2 ++#define MEM_SW_CMD_MASK 0x0003 ++#define MEM_INT_STEER_GFX 0 ++#define MEM_INT_STEER_CMR 1 ++#define MEM_INT_STEER_SMI 2 ++#define MEM_INT_STEER_SCI 3 ++#define MEMINTRSTS _MMIO(0x11184) ++#define MEMINT_RSEXIT (1 << 7) ++#define MEMINT_CONT_BUSY (1 << 6) ++#define MEMINT_AVG_BUSY (1 << 5) ++#define MEMINT_EVAL_CHG (1 << 4) ++#define MEMINT_MON_IDLE (1 << 3) ++#define MEMINT_UP_EVAL (1 << 2) ++#define MEMINT_DOWN_EVAL (1 << 1) ++#define MEMINT_SW_CMD (1 << 0) ++#define MEMMODECTL _MMIO(0x11190) ++#define MEMMODE_BOOST_EN (1 << 31) ++#define MEMMODE_BOOST_FREQ_MASK 0x0f000000 /* jitter for boost, 0-15 */ ++#define MEMMODE_BOOST_FREQ_SHIFT 24 ++#define MEMMODE_IDLE_MODE_MASK 0x00030000 ++#define MEMMODE_IDLE_MODE_SHIFT 16 ++#define MEMMODE_IDLE_MODE_EVAL 0 ++#define MEMMODE_IDLE_MODE_CONT 1 ++#define MEMMODE_HWIDLE_EN (1 << 15) ++#define MEMMODE_SWMODE_EN (1 << 14) ++#define MEMMODE_RCLK_GATE (1 << 13) ++#define MEMMODE_HW_UPDATE (1 << 12) ++#define MEMMODE_FSTART_MASK 0x00000f00 /* starting jitter, 0-15 */ ++#define MEMMODE_FSTART_SHIFT 8 ++#define MEMMODE_FMAX_MASK 0x000000f0 /* max jitter, 0-15 */ ++#define MEMMODE_FMAX_SHIFT 4 ++#define MEMMODE_FMIN_MASK 0x0000000f /* min jitter, 0-15 */ ++#define RCBMAXAVG _MMIO(0x1119c) ++#define MEMSWCTL2 _MMIO(0x1119e) /* Cantiga only */ ++#define SWMEMCMD_RENDER_OFF (0 << 13) ++#define SWMEMCMD_RENDER_ON (1 << 13) ++#define SWMEMCMD_SWFREQ (2 << 13) ++#define SWMEMCMD_TARVID (3 << 13) ++#define SWMEMCMD_VRM_OFF (4 << 13) ++#define SWMEMCMD_VRM_ON (5 << 13) ++#define CMDSTS (1 << 12) ++#define SFCAVM (1 << 11) ++#define SWFREQ_MASK 0x0380 /* P0-7 */ ++#define SWFREQ_SHIFT 7 ++#define TARVID_MASK 0x001f ++#define MEMSTAT_CTG _MMIO(0x111a0) ++#define RCBMINAVG _MMIO(0x111a0) ++#define RCUPEI _MMIO(0x111b0) ++#define RCDNEI _MMIO(0x111b4) ++#define RSTDBYCTL _MMIO(0x111b8) ++#define RS1EN (1 << 31) ++#define RS2EN (1 << 30) ++#define RS3EN (1 << 29) ++#define D3RS3EN (1 << 28) /* Display D3 imlies RS3 */ ++#define SWPROMORSX (1 << 27) /* RSx promotion timers ignored */ ++#define RCWAKERW (1 << 26) /* Resetwarn from PCH causes wakeup */ ++#define DPRSLPVREN (1 << 25) /* Fast voltage ramp enable */ ++#define GFXTGHYST (1 << 24) /* Hysteresis to allow trunk gating */ ++#define RCX_SW_EXIT (1 << 23) /* Leave RSx and prevent re-entry */ ++#define RSX_STATUS_MASK (7 << 20) ++#define RSX_STATUS_ON (0 << 20) ++#define RSX_STATUS_RC1 (1 << 20) ++#define RSX_STATUS_RC1E (2 << 20) ++#define RSX_STATUS_RS1 (3 << 20) ++#define RSX_STATUS_RS2 (4 << 20) /* aka rc6 */ ++#define RSX_STATUS_RSVD (5 << 20) /* deep rc6 unsupported on ilk */ ++#define RSX_STATUS_RS3 (6 << 20) /* rs3 unsupported on ilk */ ++#define RSX_STATUS_RSVD2 (7 << 20) ++#define UWRCRSXE (1 << 19) /* wake counter limit prevents rsx */ ++#define RSCRP (1 << 18) /* rs requests control on rs1/2 reqs */ ++#define JRSC (1 << 17) /* rsx coupled to cpu c-state */ ++#define RS2INC0 (1 << 16) /* allow rs2 in cpu c0 */ ++#define RS1CONTSAV_MASK (3 << 14) ++#define RS1CONTSAV_NO_RS1 (0 << 14) /* rs1 doesn't save/restore context */ ++#define RS1CONTSAV_RSVD (1 << 14) ++#define RS1CONTSAV_SAVE_RS1 (2 << 14) /* rs1 saves context */ ++#define RS1CONTSAV_FULL_RS1 (3 << 14) /* rs1 saves and restores context */ ++#define NORMSLEXLAT_MASK (3 << 12) ++#define SLOW_RS123 (0 << 12) ++#define SLOW_RS23 (1 << 12) ++#define SLOW_RS3 (2 << 12) ++#define NORMAL_RS123 (3 << 12) ++#define RCMODE_TIMEOUT (1 << 11) /* 0 is eval interval method */ ++#define IMPROMOEN (1 << 10) /* promo is immediate or delayed until next idle interval (only for timeout method above) */ ++#define RCENTSYNC (1 << 9) /* rs coupled to cpu c-state (3/6/7) */ ++#define STATELOCK (1 << 7) /* locked to rs_cstate if 0 */ ++#define RS_CSTATE_MASK (3 << 4) ++#define RS_CSTATE_C367_RS1 (0 << 4) ++#define RS_CSTATE_C36_RS1_C7_RS2 (1 << 4) ++#define RS_CSTATE_RSVD (2 << 4) ++#define RS_CSTATE_C367_RS2 (3 << 4) ++#define REDSAVES (1 << 3) /* no context save if was idle during rs0 */ ++#define REDRESTORES (1 << 2) /* no restore if was idle during rs0 */ ++#define VIDCTL _MMIO(0x111c0) ++#define VIDSTS _MMIO(0x111c8) ++#define VIDSTART _MMIO(0x111cc) /* 8 bits */ ++#define MEMSTAT_ILK _MMIO(0x111f8) ++#define MEMSTAT_VID_MASK 0x7f00 ++#define MEMSTAT_VID_SHIFT 8 ++#define MEMSTAT_PSTATE_MASK 0x00f8 ++#define MEMSTAT_PSTATE_SHIFT 3 ++#define MEMSTAT_MON_ACTV (1 << 2) ++#define MEMSTAT_SRC_CTL_MASK 0x0003 ++#define MEMSTAT_SRC_CTL_CORE 0 ++#define MEMSTAT_SRC_CTL_TRB 1 ++#define MEMSTAT_SRC_CTL_THM 2 ++#define MEMSTAT_SRC_CTL_STDBY 3 ++#define RCPREVBSYTUPAVG _MMIO(0x113b8) ++#define RCPREVBSYTDNAVG _MMIO(0x113bc) ++#define PMMISC _MMIO(0x11214) ++#define MCPPCE_EN (1 << 0) /* enable PM_MSG from PCH->MPC */ ++#define SDEW _MMIO(0x1124c) ++#define CSIEW0 _MMIO(0x11250) ++#define CSIEW1 _MMIO(0x11254) ++#define CSIEW2 _MMIO(0x11258) ++#define PEW(i) _MMIO(0x1125c + (i) * 4) /* 5 registers */ ++#define DEW(i) _MMIO(0x11270 + (i) * 4) /* 3 registers */ ++#define MCHAFE _MMIO(0x112c0) ++#define CSIEC _MMIO(0x112e0) ++#define DMIEC _MMIO(0x112e4) ++#define DDREC _MMIO(0x112e8) ++#define PEG0EC _MMIO(0x112ec) ++#define PEG1EC _MMIO(0x112f0) ++#define GFXEC _MMIO(0x112f4) ++#define RPPREVBSYTUPAVG _MMIO(0x113b8) ++#define RPPREVBSYTDNAVG _MMIO(0x113bc) ++#define ECR _MMIO(0x11600) ++#define ECR_GPFE (1 << 31) ++#define ECR_IMONE (1 << 30) ++#define ECR_CAP_MASK 0x0000001f /* Event range, 0-31 */ ++#define OGW0 _MMIO(0x11608) ++#define OGW1 _MMIO(0x1160c) ++#define EG0 _MMIO(0x11610) ++#define EG1 _MMIO(0x11614) ++#define EG2 _MMIO(0x11618) ++#define EG3 _MMIO(0x1161c) ++#define EG4 _MMIO(0x11620) ++#define EG5 _MMIO(0x11624) ++#define EG6 _MMIO(0x11628) ++#define EG7 _MMIO(0x1162c) ++#define PXW(i) _MMIO(0x11664 + (i) * 4) /* 4 registers */ ++#define PXWL(i) _MMIO(0x11680 + (i) * 8) /* 8 registers */ ++#define LCFUSE02 _MMIO(0x116c0) ++#define LCFUSE_HIV_MASK 0x000000ff ++#define CSIPLL0 _MMIO(0x12c10) ++#define DDRMPLL1 _MMIO(0X12c20) ++#define PEG_BAND_GAP_DATA _MMIO(0x14d68) ++ ++#define GEN6_GT_THREAD_STATUS_REG _MMIO(0x13805c) ++#define GEN6_GT_THREAD_STATUS_CORE_MASK 0x7 ++ ++#define GEN6_GT_PERF_STATUS _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5948) ++#define BXT_GT_PERF_STATUS _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x7070) ++#define GEN6_RP_STATE_LIMITS _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5994) ++#define GEN6_RP_STATE_CAP _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5998) ++#define BXT_RP_STATE_CAP _MMIO(0x138170) ++ ++/* ++ * Make these a multiple of magic 25 to avoid SNB (eg. Dell XPS ++ * 8300) freezing up around GPU hangs. Looks as if even ++ * scheduling/timer interrupts start misbehaving if the RPS ++ * EI/thresholds are "bad", leading to a very sluggish or even ++ * frozen machine. ++ */ ++#define INTERVAL_1_28_US(us) roundup(((us) * 100) >> 7, 25) ++#define INTERVAL_1_33_US(us) (((us) * 3) >> 2) ++#define INTERVAL_0_833_US(us) (((us) * 6) / 5) ++#define GT_INTERVAL_FROM_US(dev_priv, us) (INTEL_GEN(dev_priv) >= 9 ? \ ++ (IS_GEN9_LP(dev_priv) ? \ ++ INTERVAL_0_833_US(us) : \ ++ INTERVAL_1_33_US(us)) : \ ++ INTERVAL_1_28_US(us)) ++ ++#define INTERVAL_1_28_TO_US(interval) (((interval) << 7) / 100) ++#define INTERVAL_1_33_TO_US(interval) (((interval) << 2) / 3) ++#define INTERVAL_0_833_TO_US(interval) (((interval) * 5) / 6) ++#define GT_PM_INTERVAL_TO_US(dev_priv, interval) (INTEL_GEN(dev_priv) >= 9 ? \ ++ (IS_GEN9_LP(dev_priv) ? \ ++ INTERVAL_0_833_TO_US(interval) : \ ++ INTERVAL_1_33_TO_US(interval)) : \ ++ INTERVAL_1_28_TO_US(interval)) ++ ++/* ++ * Logical Context regs ++ */ ++#define CCID(base) _MMIO((base) + 0x180) ++#define CCID_EN BIT(0) ++#define CCID_EXTENDED_STATE_RESTORE BIT(2) ++#define CCID_EXTENDED_STATE_SAVE BIT(3) ++/* ++ * Notes on SNB/IVB/VLV context size: ++ * - Power context is saved elsewhere (LLC or stolen) ++ * - Ring/execlist context is saved on SNB, not on IVB ++ * - Extended context size already includes render context size ++ * - We always need to follow the extended context size. ++ * SNB BSpec has comments indicating that we should use the ++ * render context size instead if execlists are disabled, but ++ * based on empirical testing that's just nonsense. ++ * - Pipelined/VF state is saved on SNB/IVB respectively ++ * - GT1 size just indicates how much of render context ++ * doesn't need saving on GT1 ++ */ ++#define CXT_SIZE _MMIO(0x21a0) ++#define GEN6_CXT_POWER_SIZE(cxt_reg) (((cxt_reg) >> 24) & 0x3f) ++#define GEN6_CXT_RING_SIZE(cxt_reg) (((cxt_reg) >> 18) & 0x3f) ++#define GEN6_CXT_RENDER_SIZE(cxt_reg) (((cxt_reg) >> 12) & 0x3f) ++#define GEN6_CXT_EXTENDED_SIZE(cxt_reg) (((cxt_reg) >> 6) & 0x3f) ++#define GEN6_CXT_PIPELINE_SIZE(cxt_reg) (((cxt_reg) >> 0) & 0x3f) ++#define GEN6_CXT_TOTAL_SIZE(cxt_reg) (GEN6_CXT_RING_SIZE(cxt_reg) + \ ++ GEN6_CXT_EXTENDED_SIZE(cxt_reg) + \ ++ GEN6_CXT_PIPELINE_SIZE(cxt_reg)) ++#define GEN7_CXT_SIZE _MMIO(0x21a8) ++#define GEN7_CXT_POWER_SIZE(ctx_reg) (((ctx_reg) >> 25) & 0x7f) ++#define GEN7_CXT_RING_SIZE(ctx_reg) (((ctx_reg) >> 22) & 0x7) ++#define GEN7_CXT_RENDER_SIZE(ctx_reg) (((ctx_reg) >> 16) & 0x3f) ++#define GEN7_CXT_EXTENDED_SIZE(ctx_reg) (((ctx_reg) >> 9) & 0x7f) ++#define GEN7_CXT_GT1_SIZE(ctx_reg) (((ctx_reg) >> 6) & 0x7) ++#define GEN7_CXT_VFSTATE_SIZE(ctx_reg) (((ctx_reg) >> 0) & 0x3f) ++#define GEN7_CXT_TOTAL_SIZE(ctx_reg) (GEN7_CXT_EXTENDED_SIZE(ctx_reg) + \ ++ GEN7_CXT_VFSTATE_SIZE(ctx_reg)) ++ ++enum { ++ INTEL_ADVANCED_CONTEXT = 0, ++ INTEL_LEGACY_32B_CONTEXT, ++ INTEL_ADVANCED_AD_CONTEXT, ++ INTEL_LEGACY_64B_CONTEXT ++}; ++ ++enum { ++ FAULT_AND_HANG = 0, ++ FAULT_AND_HALT, /* Debug only */ ++ FAULT_AND_STREAM, ++ FAULT_AND_CONTINUE /* Unsupported */ ++}; ++ ++#define GEN8_CTX_VALID (1 << 0) ++#define GEN8_CTX_FORCE_PD_RESTORE (1 << 1) ++#define GEN8_CTX_FORCE_RESTORE (1 << 2) ++#define GEN8_CTX_L3LLC_COHERENT (1 << 5) ++#define GEN8_CTX_PRIVILEGE (1 << 8) ++#define GEN8_CTX_ADDRESSING_MODE_SHIFT 3 ++ ++#define GEN8_CTX_ID_SHIFT 32 ++#define GEN8_CTX_ID_WIDTH 21 ++#define GEN11_SW_CTX_ID_SHIFT 37 ++#define GEN11_SW_CTX_ID_WIDTH 11 ++#define GEN11_ENGINE_CLASS_SHIFT 61 ++#define GEN11_ENGINE_CLASS_WIDTH 3 ++#define GEN11_ENGINE_INSTANCE_SHIFT 48 ++#define GEN11_ENGINE_INSTANCE_WIDTH 6 ++ ++#define CHV_CLK_CTL1 _MMIO(0x101100) ++#define VLV_CLK_CTL2 _MMIO(0x101104) ++#define CLK_CTL2_CZCOUNT_30NS_SHIFT 28 ++ ++/* ++ * Overlay regs ++ */ ++ ++#define OVADD _MMIO(0x30000) ++#define DOVSTA _MMIO(0x30008) ++#define OC_BUF (0x3 << 20) ++#define OGAMC5 _MMIO(0x30010) ++#define OGAMC4 _MMIO(0x30014) ++#define OGAMC3 _MMIO(0x30018) ++#define OGAMC2 _MMIO(0x3001c) ++#define OGAMC1 _MMIO(0x30020) ++#define OGAMC0 _MMIO(0x30024) ++ ++/* ++ * GEN9 clock gating regs ++ */ ++#define GEN9_CLKGATE_DIS_0 _MMIO(0x46530) ++#define DARBF_GATING_DIS (1 << 27) ++#define PWM2_GATING_DIS (1 << 14) ++#define PWM1_GATING_DIS (1 << 13) ++ ++#define GEN9_CLKGATE_DIS_4 _MMIO(0x4653C) ++#define BXT_GMBUS_GATING_DIS (1 << 14) ++ ++#define _CLKGATE_DIS_PSL_A 0x46520 ++#define _CLKGATE_DIS_PSL_B 0x46524 ++#define _CLKGATE_DIS_PSL_C 0x46528 ++#define DUPS1_GATING_DIS (1 << 15) ++#define DUPS2_GATING_DIS (1 << 19) ++#define DUPS3_GATING_DIS (1 << 23) ++#define DPF_GATING_DIS (1 << 10) ++#define DPF_RAM_GATING_DIS (1 << 9) ++#define DPFR_GATING_DIS (1 << 8) ++ ++#define CLKGATE_DIS_PSL(pipe) \ ++ _MMIO_PIPE(pipe, _CLKGATE_DIS_PSL_A, _CLKGATE_DIS_PSL_B) ++ ++/* ++ * GEN10 clock gating regs ++ */ ++#define SLICE_UNIT_LEVEL_CLKGATE _MMIO(0x94d4) ++#define SARBUNIT_CLKGATE_DIS (1 << 5) ++#define RCCUNIT_CLKGATE_DIS (1 << 7) ++#define MSCUNIT_CLKGATE_DIS (1 << 10) ++ ++#define SUBSLICE_UNIT_LEVEL_CLKGATE _MMIO(0x9524) ++#define GWUNIT_CLKGATE_DIS (1 << 16) ++ ++#define UNSLICE_UNIT_LEVEL_CLKGATE _MMIO(0x9434) ++#define VFUNIT_CLKGATE_DIS (1 << 20) ++ ++#define INF_UNIT_LEVEL_CLKGATE _MMIO(0x9560) ++#define CGPSF_CLKGATE_DIS (1 << 3) ++ ++/* ++ * Display engine regs ++ */ ++ ++/* Pipe A CRC regs */ ++#define _PIPE_CRC_CTL_A 0x60050 ++#define PIPE_CRC_ENABLE (1 << 31) ++/* skl+ source selection */ ++#define PIPE_CRC_SOURCE_PLANE_1_SKL (0 << 28) ++#define PIPE_CRC_SOURCE_PLANE_2_SKL (2 << 28) ++#define PIPE_CRC_SOURCE_DMUX_SKL (4 << 28) ++#define PIPE_CRC_SOURCE_PLANE_3_SKL (6 << 28) ++#define PIPE_CRC_SOURCE_PLANE_4_SKL (7 << 28) ++#define PIPE_CRC_SOURCE_PLANE_5_SKL (5 << 28) ++#define PIPE_CRC_SOURCE_PLANE_6_SKL (3 << 28) ++#define PIPE_CRC_SOURCE_PLANE_7_SKL (1 << 28) ++/* ivb+ source selection */ ++#define PIPE_CRC_SOURCE_PRIMARY_IVB (0 << 29) ++#define PIPE_CRC_SOURCE_SPRITE_IVB (1 << 29) ++#define PIPE_CRC_SOURCE_PF_IVB (2 << 29) ++/* ilk+ source selection */ ++#define PIPE_CRC_SOURCE_PRIMARY_ILK (0 << 28) ++#define PIPE_CRC_SOURCE_SPRITE_ILK (1 << 28) ++#define PIPE_CRC_SOURCE_PIPE_ILK (2 << 28) ++/* embedded DP port on the north display block, reserved on ivb */ ++#define PIPE_CRC_SOURCE_PORT_A_ILK (4 << 28) ++#define PIPE_CRC_SOURCE_FDI_ILK (5 << 28) /* reserved on ivb */ ++/* vlv source selection */ ++#define PIPE_CRC_SOURCE_PIPE_VLV (0 << 27) ++#define PIPE_CRC_SOURCE_HDMIB_VLV (1 << 27) ++#define PIPE_CRC_SOURCE_HDMIC_VLV (2 << 27) ++/* with DP port the pipe source is invalid */ ++#define PIPE_CRC_SOURCE_DP_D_VLV (3 << 27) ++#define PIPE_CRC_SOURCE_DP_B_VLV (6 << 27) ++#define PIPE_CRC_SOURCE_DP_C_VLV (7 << 27) ++/* gen3+ source selection */ ++#define PIPE_CRC_SOURCE_PIPE_I9XX (0 << 28) ++#define PIPE_CRC_SOURCE_SDVOB_I9XX (1 << 28) ++#define PIPE_CRC_SOURCE_SDVOC_I9XX (2 << 28) ++/* with DP/TV port the pipe source is invalid */ ++#define PIPE_CRC_SOURCE_DP_D_G4X (3 << 28) ++#define PIPE_CRC_SOURCE_TV_PRE (4 << 28) ++#define PIPE_CRC_SOURCE_TV_POST (5 << 28) ++#define PIPE_CRC_SOURCE_DP_B_G4X (6 << 28) ++#define PIPE_CRC_SOURCE_DP_C_G4X (7 << 28) ++/* gen2 doesn't have source selection bits */ ++#define PIPE_CRC_INCLUDE_BORDER_I8XX (1 << 30) ++ ++#define _PIPE_CRC_RES_1_A_IVB 0x60064 ++#define _PIPE_CRC_RES_2_A_IVB 0x60068 ++#define _PIPE_CRC_RES_3_A_IVB 0x6006c ++#define _PIPE_CRC_RES_4_A_IVB 0x60070 ++#define _PIPE_CRC_RES_5_A_IVB 0x60074 ++ ++#define _PIPE_CRC_RES_RED_A 0x60060 ++#define _PIPE_CRC_RES_GREEN_A 0x60064 ++#define _PIPE_CRC_RES_BLUE_A 0x60068 ++#define _PIPE_CRC_RES_RES1_A_I915 0x6006c ++#define _PIPE_CRC_RES_RES2_A_G4X 0x60080 ++ ++/* Pipe B CRC regs */ ++#define _PIPE_CRC_RES_1_B_IVB 0x61064 ++#define _PIPE_CRC_RES_2_B_IVB 0x61068 ++#define _PIPE_CRC_RES_3_B_IVB 0x6106c ++#define _PIPE_CRC_RES_4_B_IVB 0x61070 ++#define _PIPE_CRC_RES_5_B_IVB 0x61074 ++ ++#define PIPE_CRC_CTL(pipe) _MMIO_TRANS2(pipe, _PIPE_CRC_CTL_A) ++#define PIPE_CRC_RES_1_IVB(pipe) _MMIO_TRANS2(pipe, _PIPE_CRC_RES_1_A_IVB) ++#define PIPE_CRC_RES_2_IVB(pipe) _MMIO_TRANS2(pipe, _PIPE_CRC_RES_2_A_IVB) ++#define PIPE_CRC_RES_3_IVB(pipe) _MMIO_TRANS2(pipe, _PIPE_CRC_RES_3_A_IVB) ++#define PIPE_CRC_RES_4_IVB(pipe) _MMIO_TRANS2(pipe, _PIPE_CRC_RES_4_A_IVB) ++#define PIPE_CRC_RES_5_IVB(pipe) _MMIO_TRANS2(pipe, _PIPE_CRC_RES_5_A_IVB) ++ ++#define PIPE_CRC_RES_RED(pipe) _MMIO_TRANS2(pipe, _PIPE_CRC_RES_RED_A) ++#define PIPE_CRC_RES_GREEN(pipe) _MMIO_TRANS2(pipe, _PIPE_CRC_RES_GREEN_A) ++#define PIPE_CRC_RES_BLUE(pipe) _MMIO_TRANS2(pipe, _PIPE_CRC_RES_BLUE_A) ++#define PIPE_CRC_RES_RES1_I915(pipe) _MMIO_TRANS2(pipe, _PIPE_CRC_RES_RES1_A_I915) ++#define PIPE_CRC_RES_RES2_G4X(pipe) _MMIO_TRANS2(pipe, _PIPE_CRC_RES_RES2_A_G4X) ++ ++/* Pipe A timing regs */ ++#define _HTOTAL_A 0x60000 ++#define _HBLANK_A 0x60004 ++#define _HSYNC_A 0x60008 ++#define _VTOTAL_A 0x6000c ++#define _VBLANK_A 0x60010 ++#define _VSYNC_A 0x60014 ++#define _PIPEASRC 0x6001c ++#define _BCLRPAT_A 0x60020 ++#define _VSYNCSHIFT_A 0x60028 ++#define _PIPE_MULT_A 0x6002c ++ ++/* Pipe B timing regs */ ++#define _HTOTAL_B 0x61000 ++#define _HBLANK_B 0x61004 ++#define _HSYNC_B 0x61008 ++#define _VTOTAL_B 0x6100c ++#define _VBLANK_B 0x61010 ++#define _VSYNC_B 0x61014 ++#define _PIPEBSRC 0x6101c ++#define _BCLRPAT_B 0x61020 ++#define _VSYNCSHIFT_B 0x61028 ++#define _PIPE_MULT_B 0x6102c ++ ++/* DSI 0 timing regs */ ++#define _HTOTAL_DSI0 0x6b000 ++#define _HSYNC_DSI0 0x6b008 ++#define _VTOTAL_DSI0 0x6b00c ++#define _VSYNC_DSI0 0x6b014 ++#define _VSYNCSHIFT_DSI0 0x6b028 ++ ++/* DSI 1 timing regs */ ++#define _HTOTAL_DSI1 0x6b800 ++#define _HSYNC_DSI1 0x6b808 ++#define _VTOTAL_DSI1 0x6b80c ++#define _VSYNC_DSI1 0x6b814 ++#define _VSYNCSHIFT_DSI1 0x6b828 ++ ++#define TRANSCODER_A_OFFSET 0x60000 ++#define TRANSCODER_B_OFFSET 0x61000 ++#define TRANSCODER_C_OFFSET 0x62000 ++#define CHV_TRANSCODER_C_OFFSET 0x63000 ++#define TRANSCODER_EDP_OFFSET 0x6f000 ++#define TRANSCODER_DSI0_OFFSET 0x6b000 ++#define TRANSCODER_DSI1_OFFSET 0x6b800 ++ ++#define HTOTAL(trans) _MMIO_TRANS2(trans, _HTOTAL_A) ++#define HBLANK(trans) _MMIO_TRANS2(trans, _HBLANK_A) ++#define HSYNC(trans) _MMIO_TRANS2(trans, _HSYNC_A) ++#define VTOTAL(trans) _MMIO_TRANS2(trans, _VTOTAL_A) ++#define VBLANK(trans) _MMIO_TRANS2(trans, _VBLANK_A) ++#define VSYNC(trans) _MMIO_TRANS2(trans, _VSYNC_A) ++#define BCLRPAT(trans) _MMIO_TRANS2(trans, _BCLRPAT_A) ++#define VSYNCSHIFT(trans) _MMIO_TRANS2(trans, _VSYNCSHIFT_A) ++#define PIPESRC(trans) _MMIO_TRANS2(trans, _PIPEASRC) ++#define PIPE_MULT(trans) _MMIO_TRANS2(trans, _PIPE_MULT_A) ++ ++/* HSW+ eDP PSR registers */ ++#define HSW_EDP_PSR_BASE 0x64800 ++#define BDW_EDP_PSR_BASE 0x6f800 ++#define EDP_PSR_CTL _MMIO(dev_priv->psr_mmio_base + 0) ++#define EDP_PSR_ENABLE (1 << 31) ++#define BDW_PSR_SINGLE_FRAME (1 << 30) ++#define EDP_PSR_RESTORE_PSR_ACTIVE_CTX_MASK (1 << 29) /* SW can't modify */ ++#define EDP_PSR_LINK_STANDBY (1 << 27) ++#define EDP_PSR_MIN_LINK_ENTRY_TIME_MASK (3 << 25) ++#define EDP_PSR_MIN_LINK_ENTRY_TIME_8_LINES (0 << 25) ++#define EDP_PSR_MIN_LINK_ENTRY_TIME_4_LINES (1 << 25) ++#define EDP_PSR_MIN_LINK_ENTRY_TIME_2_LINES (2 << 25) ++#define EDP_PSR_MIN_LINK_ENTRY_TIME_0_LINES (3 << 25) ++#define EDP_PSR_MAX_SLEEP_TIME_SHIFT 20 ++#define EDP_PSR_SKIP_AUX_EXIT (1 << 12) ++#define EDP_PSR_TP1_TP2_SEL (0 << 11) ++#define EDP_PSR_TP1_TP3_SEL (1 << 11) ++#define EDP_PSR_CRC_ENABLE (1 << 10) /* BDW+ */ ++#define EDP_PSR_TP2_TP3_TIME_500us (0 << 8) ++#define EDP_PSR_TP2_TP3_TIME_100us (1 << 8) ++#define EDP_PSR_TP2_TP3_TIME_2500us (2 << 8) ++#define EDP_PSR_TP2_TP3_TIME_0us (3 << 8) ++#define EDP_PSR_TP4_TIME_0US (3 << 6) /* ICL+ */ ++#define EDP_PSR_TP1_TIME_500us (0 << 4) ++#define EDP_PSR_TP1_TIME_100us (1 << 4) ++#define EDP_PSR_TP1_TIME_2500us (2 << 4) ++#define EDP_PSR_TP1_TIME_0us (3 << 4) ++#define EDP_PSR_IDLE_FRAME_SHIFT 0 ++ ++/* Bspec claims those aren't shifted but stay at 0x64800 */ ++#define EDP_PSR_IMR _MMIO(0x64834) ++#define EDP_PSR_IIR _MMIO(0x64838) ++#define EDP_PSR_ERROR(shift) (1 << ((shift) + 2)) ++#define EDP_PSR_POST_EXIT(shift) (1 << ((shift) + 1)) ++#define EDP_PSR_PRE_ENTRY(shift) (1 << (shift)) ++#define EDP_PSR_TRANSCODER_C_SHIFT 24 ++#define EDP_PSR_TRANSCODER_B_SHIFT 16 ++#define EDP_PSR_TRANSCODER_A_SHIFT 8 ++#define EDP_PSR_TRANSCODER_EDP_SHIFT 0 ++ ++#define EDP_PSR_AUX_CTL _MMIO(dev_priv->psr_mmio_base + 0x10) ++#define EDP_PSR_AUX_CTL_TIME_OUT_MASK (3 << 26) ++#define EDP_PSR_AUX_CTL_MESSAGE_SIZE_MASK (0x1f << 20) ++#define EDP_PSR_AUX_CTL_PRECHARGE_2US_MASK (0xf << 16) ++#define EDP_PSR_AUX_CTL_ERROR_INTERRUPT (1 << 11) ++#define EDP_PSR_AUX_CTL_BIT_CLOCK_2X_MASK (0x7ff) ++ ++#define EDP_PSR_AUX_DATA(i) _MMIO(dev_priv->psr_mmio_base + 0x14 + (i) * 4) /* 5 registers */ ++ ++#define EDP_PSR_STATUS _MMIO(dev_priv->psr_mmio_base + 0x40) ++#define EDP_PSR_STATUS_STATE_MASK (7 << 29) ++#define EDP_PSR_STATUS_STATE_SHIFT 29 ++#define EDP_PSR_STATUS_STATE_IDLE (0 << 29) ++#define EDP_PSR_STATUS_STATE_SRDONACK (1 << 29) ++#define EDP_PSR_STATUS_STATE_SRDENT (2 << 29) ++#define EDP_PSR_STATUS_STATE_BUFOFF (3 << 29) ++#define EDP_PSR_STATUS_STATE_BUFON (4 << 29) ++#define EDP_PSR_STATUS_STATE_AUXACK (5 << 29) ++#define EDP_PSR_STATUS_STATE_SRDOFFACK (6 << 29) ++#define EDP_PSR_STATUS_LINK_MASK (3 << 26) ++#define EDP_PSR_STATUS_LINK_FULL_OFF (0 << 26) ++#define EDP_PSR_STATUS_LINK_FULL_ON (1 << 26) ++#define EDP_PSR_STATUS_LINK_STANDBY (2 << 26) ++#define EDP_PSR_STATUS_MAX_SLEEP_TIMER_SHIFT 20 ++#define EDP_PSR_STATUS_MAX_SLEEP_TIMER_MASK 0x1f ++#define EDP_PSR_STATUS_COUNT_SHIFT 16 ++#define EDP_PSR_STATUS_COUNT_MASK 0xf ++#define EDP_PSR_STATUS_AUX_ERROR (1 << 15) ++#define EDP_PSR_STATUS_AUX_SENDING (1 << 12) ++#define EDP_PSR_STATUS_SENDING_IDLE (1 << 9) ++#define EDP_PSR_STATUS_SENDING_TP2_TP3 (1 << 8) ++#define EDP_PSR_STATUS_SENDING_TP1 (1 << 4) ++#define EDP_PSR_STATUS_IDLE_MASK 0xf ++ ++#define EDP_PSR_PERF_CNT _MMIO(dev_priv->psr_mmio_base + 0x44) ++#define EDP_PSR_PERF_CNT_MASK 0xffffff ++ ++#define EDP_PSR_DEBUG _MMIO(dev_priv->psr_mmio_base + 0x60) /* PSR_MASK on SKL+ */ ++#define EDP_PSR_DEBUG_MASK_MAX_SLEEP (1 << 28) ++#define EDP_PSR_DEBUG_MASK_LPSP (1 << 27) ++#define EDP_PSR_DEBUG_MASK_MEMUP (1 << 26) ++#define EDP_PSR_DEBUG_MASK_HPD (1 << 25) ++#define EDP_PSR_DEBUG_MASK_DISP_REG_WRITE (1 << 16) /* Reserved in ICL+ */ ++#define EDP_PSR_DEBUG_EXIT_ON_PIXEL_UNDERRUN (1 << 15) /* SKL+ */ ++ ++#define EDP_PSR2_CTL _MMIO(0x6f900) ++#define EDP_PSR2_ENABLE (1 << 31) ++#define EDP_SU_TRACK_ENABLE (1 << 30) ++#define EDP_Y_COORDINATE_VALID (1 << 26) /* GLK and CNL+ */ ++#define EDP_Y_COORDINATE_ENABLE (1 << 25) /* GLK and CNL+ */ ++#define EDP_MAX_SU_DISABLE_TIME(t) ((t) << 20) ++#define EDP_MAX_SU_DISABLE_TIME_MASK (0x1f << 20) ++#define EDP_PSR2_TP2_TIME_500us (0 << 8) ++#define EDP_PSR2_TP2_TIME_100us (1 << 8) ++#define EDP_PSR2_TP2_TIME_2500us (2 << 8) ++#define EDP_PSR2_TP2_TIME_50us (3 << 8) ++#define EDP_PSR2_TP2_TIME_MASK (3 << 8) ++#define EDP_PSR2_FRAME_BEFORE_SU_SHIFT 4 ++#define EDP_PSR2_FRAME_BEFORE_SU_MASK (0xf << 4) ++#define EDP_PSR2_FRAME_BEFORE_SU(a) ((a) << 4) ++#define EDP_PSR2_IDLE_FRAME_MASK 0xf ++#define EDP_PSR2_IDLE_FRAME_SHIFT 0 ++ ++#define _PSR_EVENT_TRANS_A 0x60848 ++#define _PSR_EVENT_TRANS_B 0x61848 ++#define _PSR_EVENT_TRANS_C 0x62848 ++#define _PSR_EVENT_TRANS_D 0x63848 ++#define _PSR_EVENT_TRANS_EDP 0x6F848 ++#define PSR_EVENT(trans) _MMIO_TRANS2(trans, _PSR_EVENT_TRANS_A) ++#define PSR_EVENT_PSR2_WD_TIMER_EXPIRE (1 << 17) ++#define PSR_EVENT_PSR2_DISABLED (1 << 16) ++#define PSR_EVENT_SU_DIRTY_FIFO_UNDERRUN (1 << 15) ++#define PSR_EVENT_SU_CRC_FIFO_UNDERRUN (1 << 14) ++#define PSR_EVENT_GRAPHICS_RESET (1 << 12) ++#define PSR_EVENT_PCH_INTERRUPT (1 << 11) ++#define PSR_EVENT_MEMORY_UP (1 << 10) ++#define PSR_EVENT_FRONT_BUFFER_MODIFY (1 << 9) ++#define PSR_EVENT_WD_TIMER_EXPIRE (1 << 8) ++#define PSR_EVENT_PIPE_REGISTERS_UPDATE (1 << 6) ++#define PSR_EVENT_REGISTER_UPDATE (1 << 5) /* Reserved in ICL+ */ ++#define PSR_EVENT_HDCP_ENABLE (1 << 4) ++#define PSR_EVENT_KVMR_SESSION_ENABLE (1 << 3) ++#define PSR_EVENT_VBI_ENABLE (1 << 2) ++#define PSR_EVENT_LPSP_MODE_EXIT (1 << 1) ++#define PSR_EVENT_PSR_DISABLE (1 << 0) ++ ++#define EDP_PSR2_STATUS _MMIO(0x6f940) ++#define EDP_PSR2_STATUS_STATE_MASK (0xf << 28) ++#define EDP_PSR2_STATUS_STATE_SHIFT 28 ++ ++#define _PSR2_SU_STATUS_0 0x6F914 ++#define _PSR2_SU_STATUS_1 0x6F918 ++#define _PSR2_SU_STATUS_2 0x6F91C ++#define _PSR2_SU_STATUS(index) _MMIO(_PICK_EVEN((index), _PSR2_SU_STATUS_0, _PSR2_SU_STATUS_1)) ++#define PSR2_SU_STATUS(frame) (_PSR2_SU_STATUS((frame) / 3)) ++#define PSR2_SU_STATUS_SHIFT(frame) (((frame) % 3) * 10) ++#define PSR2_SU_STATUS_MASK(frame) (0x3ff << PSR2_SU_STATUS_SHIFT(frame)) ++#define PSR2_SU_STATUS_FRAMES 8 ++ ++/* VGA port control */ ++#define ADPA _MMIO(0x61100) ++#define PCH_ADPA _MMIO(0xe1100) ++#define VLV_ADPA _MMIO(VLV_DISPLAY_BASE + 0x61100) ++ ++#define ADPA_DAC_ENABLE (1 << 31) ++#define ADPA_DAC_DISABLE 0 ++#define ADPA_PIPE_SEL_SHIFT 30 ++#define ADPA_PIPE_SEL_MASK (1 << 30) ++#define ADPA_PIPE_SEL(pipe) ((pipe) << 30) ++#define ADPA_PIPE_SEL_SHIFT_CPT 29 ++#define ADPA_PIPE_SEL_MASK_CPT (3 << 29) ++#define ADPA_PIPE_SEL_CPT(pipe) ((pipe) << 29) ++#define ADPA_CRT_HOTPLUG_MASK 0x03ff0000 /* bit 25-16 */ ++#define ADPA_CRT_HOTPLUG_MONITOR_NONE (0 << 24) ++#define ADPA_CRT_HOTPLUG_MONITOR_MASK (3 << 24) ++#define ADPA_CRT_HOTPLUG_MONITOR_COLOR (3 << 24) ++#define ADPA_CRT_HOTPLUG_MONITOR_MONO (2 << 24) ++#define ADPA_CRT_HOTPLUG_ENABLE (1 << 23) ++#define ADPA_CRT_HOTPLUG_PERIOD_64 (0 << 22) ++#define ADPA_CRT_HOTPLUG_PERIOD_128 (1 << 22) ++#define ADPA_CRT_HOTPLUG_WARMUP_5MS (0 << 21) ++#define ADPA_CRT_HOTPLUG_WARMUP_10MS (1 << 21) ++#define ADPA_CRT_HOTPLUG_SAMPLE_2S (0 << 20) ++#define ADPA_CRT_HOTPLUG_SAMPLE_4S (1 << 20) ++#define ADPA_CRT_HOTPLUG_VOLTAGE_40 (0 << 18) ++#define ADPA_CRT_HOTPLUG_VOLTAGE_50 (1 << 18) ++#define ADPA_CRT_HOTPLUG_VOLTAGE_60 (2 << 18) ++#define ADPA_CRT_HOTPLUG_VOLTAGE_70 (3 << 18) ++#define ADPA_CRT_HOTPLUG_VOLREF_325MV (0 << 17) ++#define ADPA_CRT_HOTPLUG_VOLREF_475MV (1 << 17) ++#define ADPA_CRT_HOTPLUG_FORCE_TRIGGER (1 << 16) ++#define ADPA_USE_VGA_HVPOLARITY (1 << 15) ++#define ADPA_SETS_HVPOLARITY 0 ++#define ADPA_VSYNC_CNTL_DISABLE (1 << 10) ++#define ADPA_VSYNC_CNTL_ENABLE 0 ++#define ADPA_HSYNC_CNTL_DISABLE (1 << 11) ++#define ADPA_HSYNC_CNTL_ENABLE 0 ++#define ADPA_VSYNC_ACTIVE_HIGH (1 << 4) ++#define ADPA_VSYNC_ACTIVE_LOW 0 ++#define ADPA_HSYNC_ACTIVE_HIGH (1 << 3) ++#define ADPA_HSYNC_ACTIVE_LOW 0 ++#define ADPA_DPMS_MASK (~(3 << 10)) ++#define ADPA_DPMS_ON (0 << 10) ++#define ADPA_DPMS_SUSPEND (1 << 10) ++#define ADPA_DPMS_STANDBY (2 << 10) ++#define ADPA_DPMS_OFF (3 << 10) ++ ++ ++/* Hotplug control (945+ only) */ ++#define PORT_HOTPLUG_EN _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61110) ++#define PORTB_HOTPLUG_INT_EN (1 << 29) ++#define PORTC_HOTPLUG_INT_EN (1 << 28) ++#define PORTD_HOTPLUG_INT_EN (1 << 27) ++#define SDVOB_HOTPLUG_INT_EN (1 << 26) ++#define SDVOC_HOTPLUG_INT_EN (1 << 25) ++#define TV_HOTPLUG_INT_EN (1 << 18) ++#define CRT_HOTPLUG_INT_EN (1 << 9) ++#define HOTPLUG_INT_EN_MASK (PORTB_HOTPLUG_INT_EN | \ ++ PORTC_HOTPLUG_INT_EN | \ ++ PORTD_HOTPLUG_INT_EN | \ ++ SDVOC_HOTPLUG_INT_EN | \ ++ SDVOB_HOTPLUG_INT_EN | \ ++ CRT_HOTPLUG_INT_EN) ++#define CRT_HOTPLUG_FORCE_DETECT (1 << 3) ++#define CRT_HOTPLUG_ACTIVATION_PERIOD_32 (0 << 8) ++/* must use period 64 on GM45 according to docs */ ++#define CRT_HOTPLUG_ACTIVATION_PERIOD_64 (1 << 8) ++#define CRT_HOTPLUG_DAC_ON_TIME_2M (0 << 7) ++#define CRT_HOTPLUG_DAC_ON_TIME_4M (1 << 7) ++#define CRT_HOTPLUG_VOLTAGE_COMPARE_40 (0 << 5) ++#define CRT_HOTPLUG_VOLTAGE_COMPARE_50 (1 << 5) ++#define CRT_HOTPLUG_VOLTAGE_COMPARE_60 (2 << 5) ++#define CRT_HOTPLUG_VOLTAGE_COMPARE_70 (3 << 5) ++#define CRT_HOTPLUG_VOLTAGE_COMPARE_MASK (3 << 5) ++#define CRT_HOTPLUG_DETECT_DELAY_1G (0 << 4) ++#define CRT_HOTPLUG_DETECT_DELAY_2G (1 << 4) ++#define CRT_HOTPLUG_DETECT_VOLTAGE_325MV (0 << 2) ++#define CRT_HOTPLUG_DETECT_VOLTAGE_475MV (1 << 2) ++ ++#define PORT_HOTPLUG_STAT _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61114) ++/* ++ * HDMI/DP bits are g4x+ ++ * ++ * WARNING: Bspec for hpd status bits on gen4 seems to be completely confused. ++ * Please check the detailed lore in the commit message for for experimental ++ * evidence. ++ */ ++/* Bspec says GM45 should match G4X/VLV/CHV, but reality disagrees */ ++#define PORTD_HOTPLUG_LIVE_STATUS_GM45 (1 << 29) ++#define PORTC_HOTPLUG_LIVE_STATUS_GM45 (1 << 28) ++#define PORTB_HOTPLUG_LIVE_STATUS_GM45 (1 << 27) ++/* G4X/VLV/CHV DP/HDMI bits again match Bspec */ ++#define PORTD_HOTPLUG_LIVE_STATUS_G4X (1 << 27) ++#define PORTC_HOTPLUG_LIVE_STATUS_G4X (1 << 28) ++#define PORTB_HOTPLUG_LIVE_STATUS_G4X (1 << 29) ++#define PORTD_HOTPLUG_INT_STATUS (3 << 21) ++#define PORTD_HOTPLUG_INT_LONG_PULSE (2 << 21) ++#define PORTD_HOTPLUG_INT_SHORT_PULSE (1 << 21) ++#define PORTC_HOTPLUG_INT_STATUS (3 << 19) ++#define PORTC_HOTPLUG_INT_LONG_PULSE (2 << 19) ++#define PORTC_HOTPLUG_INT_SHORT_PULSE (1 << 19) ++#define PORTB_HOTPLUG_INT_STATUS (3 << 17) ++#define PORTB_HOTPLUG_INT_LONG_PULSE (2 << 17) ++#define PORTB_HOTPLUG_INT_SHORT_PLUSE (1 << 17) ++/* CRT/TV common between gen3+ */ ++#define CRT_HOTPLUG_INT_STATUS (1 << 11) ++#define TV_HOTPLUG_INT_STATUS (1 << 10) ++#define CRT_HOTPLUG_MONITOR_MASK (3 << 8) ++#define CRT_HOTPLUG_MONITOR_COLOR (3 << 8) ++#define CRT_HOTPLUG_MONITOR_MONO (2 << 8) ++#define CRT_HOTPLUG_MONITOR_NONE (0 << 8) ++#define DP_AUX_CHANNEL_D_INT_STATUS_G4X (1 << 6) ++#define DP_AUX_CHANNEL_C_INT_STATUS_G4X (1 << 5) ++#define DP_AUX_CHANNEL_B_INT_STATUS_G4X (1 << 4) ++#define DP_AUX_CHANNEL_MASK_INT_STATUS_G4X (7 << 4) ++ ++/* SDVO is different across gen3/4 */ ++#define SDVOC_HOTPLUG_INT_STATUS_G4X (1 << 3) ++#define SDVOB_HOTPLUG_INT_STATUS_G4X (1 << 2) ++/* ++ * Bspec seems to be seriously misleaded about the SDVO hpd bits on i965g/gm, ++ * since reality corrobates that they're the same as on gen3. But keep these ++ * bits here (and the comment!) to help any other lost wanderers back onto the ++ * right tracks. ++ */ ++#define SDVOC_HOTPLUG_INT_STATUS_I965 (3 << 4) ++#define SDVOB_HOTPLUG_INT_STATUS_I965 (3 << 2) ++#define SDVOC_HOTPLUG_INT_STATUS_I915 (1 << 7) ++#define SDVOB_HOTPLUG_INT_STATUS_I915 (1 << 6) ++#define HOTPLUG_INT_STATUS_G4X (CRT_HOTPLUG_INT_STATUS | \ ++ SDVOB_HOTPLUG_INT_STATUS_G4X | \ ++ SDVOC_HOTPLUG_INT_STATUS_G4X | \ ++ PORTB_HOTPLUG_INT_STATUS | \ ++ PORTC_HOTPLUG_INT_STATUS | \ ++ PORTD_HOTPLUG_INT_STATUS) ++ ++#define HOTPLUG_INT_STATUS_I915 (CRT_HOTPLUG_INT_STATUS | \ ++ SDVOB_HOTPLUG_INT_STATUS_I915 | \ ++ SDVOC_HOTPLUG_INT_STATUS_I915 | \ ++ PORTB_HOTPLUG_INT_STATUS | \ ++ PORTC_HOTPLUG_INT_STATUS | \ ++ PORTD_HOTPLUG_INT_STATUS) ++ ++/* SDVO and HDMI port control. ++ * The same register may be used for SDVO or HDMI */ ++#define _GEN3_SDVOB 0x61140 ++#define _GEN3_SDVOC 0x61160 ++#define GEN3_SDVOB _MMIO(_GEN3_SDVOB) ++#define GEN3_SDVOC _MMIO(_GEN3_SDVOC) ++#define GEN4_HDMIB GEN3_SDVOB ++#define GEN4_HDMIC GEN3_SDVOC ++#define VLV_HDMIB _MMIO(VLV_DISPLAY_BASE + 0x61140) ++#define VLV_HDMIC _MMIO(VLV_DISPLAY_BASE + 0x61160) ++#define CHV_HDMID _MMIO(VLV_DISPLAY_BASE + 0x6116C) ++#define PCH_SDVOB _MMIO(0xe1140) ++#define PCH_HDMIB PCH_SDVOB ++#define PCH_HDMIC _MMIO(0xe1150) ++#define PCH_HDMID _MMIO(0xe1160) ++ ++#define PORT_DFT_I9XX _MMIO(0x61150) ++#define DC_BALANCE_RESET (1 << 25) ++#define PORT_DFT2_G4X _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61154) ++#define DC_BALANCE_RESET_VLV (1 << 31) ++#define PIPE_SCRAMBLE_RESET_MASK ((1 << 14) | (0x3 << 0)) ++#define PIPE_C_SCRAMBLE_RESET (1 << 14) /* chv */ ++#define PIPE_B_SCRAMBLE_RESET (1 << 1) ++#define PIPE_A_SCRAMBLE_RESET (1 << 0) ++ ++/* Gen 3 SDVO bits: */ ++#define SDVO_ENABLE (1 << 31) ++#define SDVO_PIPE_SEL_SHIFT 30 ++#define SDVO_PIPE_SEL_MASK (1 << 30) ++#define SDVO_PIPE_SEL(pipe) ((pipe) << 30) ++#define SDVO_STALL_SELECT (1 << 29) ++#define SDVO_INTERRUPT_ENABLE (1 << 26) ++/* ++ * 915G/GM SDVO pixel multiplier. ++ * Programmed value is multiplier - 1, up to 5x. ++ * \sa DPLL_MD_UDI_MULTIPLIER_MASK ++ */ ++#define SDVO_PORT_MULTIPLY_MASK (7 << 23) ++#define SDVO_PORT_MULTIPLY_SHIFT 23 ++#define SDVO_PHASE_SELECT_MASK (15 << 19) ++#define SDVO_PHASE_SELECT_DEFAULT (6 << 19) ++#define SDVO_CLOCK_OUTPUT_INVERT (1 << 18) ++#define SDVOC_GANG_MODE (1 << 16) /* Port C only */ ++#define SDVO_BORDER_ENABLE (1 << 7) /* SDVO only */ ++#define SDVOB_PCIE_CONCURRENCY (1 << 3) /* Port B only */ ++#define SDVO_DETECTED (1 << 2) ++/* Bits to be preserved when writing */ ++#define SDVOB_PRESERVE_MASK ((1 << 17) | (1 << 16) | (1 << 14) | \ ++ SDVO_INTERRUPT_ENABLE) ++#define SDVOC_PRESERVE_MASK ((1 << 17) | SDVO_INTERRUPT_ENABLE) ++ ++/* Gen 4 SDVO/HDMI bits: */ ++#define SDVO_COLOR_FORMAT_8bpc (0 << 26) ++#define SDVO_COLOR_FORMAT_MASK (7 << 26) ++#define SDVO_ENCODING_SDVO (0 << 10) ++#define SDVO_ENCODING_HDMI (2 << 10) ++#define HDMI_MODE_SELECT_HDMI (1 << 9) /* HDMI only */ ++#define HDMI_MODE_SELECT_DVI (0 << 9) /* HDMI only */ ++#define HDMI_COLOR_RANGE_16_235 (1 << 8) /* HDMI only */ ++#define SDVO_AUDIO_ENABLE (1 << 6) ++/* VSYNC/HSYNC bits new with 965, default is to be set */ ++#define SDVO_VSYNC_ACTIVE_HIGH (1 << 4) ++#define SDVO_HSYNC_ACTIVE_HIGH (1 << 3) ++ ++/* Gen 5 (IBX) SDVO/HDMI bits: */ ++#define HDMI_COLOR_FORMAT_12bpc (3 << 26) /* HDMI only */ ++#define SDVOB_HOTPLUG_ENABLE (1 << 23) /* SDVO only */ ++ ++/* Gen 6 (CPT) SDVO/HDMI bits: */ ++#define SDVO_PIPE_SEL_SHIFT_CPT 29 ++#define SDVO_PIPE_SEL_MASK_CPT (3 << 29) ++#define SDVO_PIPE_SEL_CPT(pipe) ((pipe) << 29) ++ ++/* CHV SDVO/HDMI bits: */ ++#define SDVO_PIPE_SEL_SHIFT_CHV 24 ++#define SDVO_PIPE_SEL_MASK_CHV (3 << 24) ++#define SDVO_PIPE_SEL_CHV(pipe) ((pipe) << 24) ++ ++ ++/* DVO port control */ ++#define _DVOA 0x61120 ++#define DVOA _MMIO(_DVOA) ++#define _DVOB 0x61140 ++#define DVOB _MMIO(_DVOB) ++#define _DVOC 0x61160 ++#define DVOC _MMIO(_DVOC) ++#define DVO_ENABLE (1 << 31) ++#define DVO_PIPE_SEL_SHIFT 30 ++#define DVO_PIPE_SEL_MASK (1 << 30) ++#define DVO_PIPE_SEL(pipe) ((pipe) << 30) ++#define DVO_PIPE_STALL_UNUSED (0 << 28) ++#define DVO_PIPE_STALL (1 << 28) ++#define DVO_PIPE_STALL_TV (2 << 28) ++#define DVO_PIPE_STALL_MASK (3 << 28) ++#define DVO_USE_VGA_SYNC (1 << 15) ++#define DVO_DATA_ORDER_I740 (0 << 14) ++#define DVO_DATA_ORDER_FP (1 << 14) ++#define DVO_VSYNC_DISABLE (1 << 11) ++#define DVO_HSYNC_DISABLE (1 << 10) ++#define DVO_VSYNC_TRISTATE (1 << 9) ++#define DVO_HSYNC_TRISTATE (1 << 8) ++#define DVO_BORDER_ENABLE (1 << 7) ++#define DVO_DATA_ORDER_GBRG (1 << 6) ++#define DVO_DATA_ORDER_RGGB (0 << 6) ++#define DVO_DATA_ORDER_GBRG_ERRATA (0 << 6) ++#define DVO_DATA_ORDER_RGGB_ERRATA (1 << 6) ++#define DVO_VSYNC_ACTIVE_HIGH (1 << 4) ++#define DVO_HSYNC_ACTIVE_HIGH (1 << 3) ++#define DVO_BLANK_ACTIVE_HIGH (1 << 2) ++#define DVO_OUTPUT_CSTATE_PIXELS (1 << 1) /* SDG only */ ++#define DVO_OUTPUT_SOURCE_SIZE_PIXELS (1 << 0) /* SDG only */ ++#define DVO_PRESERVE_MASK (0x7 << 24) ++#define DVOA_SRCDIM _MMIO(0x61124) ++#define DVOB_SRCDIM _MMIO(0x61144) ++#define DVOC_SRCDIM _MMIO(0x61164) ++#define DVO_SRCDIM_HORIZONTAL_SHIFT 12 ++#define DVO_SRCDIM_VERTICAL_SHIFT 0 ++ ++/* LVDS port control */ ++#define LVDS _MMIO(0x61180) ++/* ++ * Enables the LVDS port. This bit must be set before DPLLs are enabled, as ++ * the DPLL semantics change when the LVDS is assigned to that pipe. ++ */ ++#define LVDS_PORT_EN (1 << 31) ++/* Selects pipe B for LVDS data. Must be set on pre-965. */ ++#define LVDS_PIPE_SEL_SHIFT 30 ++#define LVDS_PIPE_SEL_MASK (1 << 30) ++#define LVDS_PIPE_SEL(pipe) ((pipe) << 30) ++#define LVDS_PIPE_SEL_SHIFT_CPT 29 ++#define LVDS_PIPE_SEL_MASK_CPT (3 << 29) ++#define LVDS_PIPE_SEL_CPT(pipe) ((pipe) << 29) ++/* LVDS dithering flag on 965/g4x platform */ ++#define LVDS_ENABLE_DITHER (1 << 25) ++/* LVDS sync polarity flags. Set to invert (i.e. negative) */ ++#define LVDS_VSYNC_POLARITY (1 << 21) ++#define LVDS_HSYNC_POLARITY (1 << 20) ++ ++/* Enable border for unscaled (or aspect-scaled) display */ ++#define LVDS_BORDER_ENABLE (1 << 15) ++/* ++ * Enables the A0-A2 data pairs and CLKA, containing 18 bits of color data per ++ * pixel. ++ */ ++#define LVDS_A0A2_CLKA_POWER_MASK (3 << 8) ++#define LVDS_A0A2_CLKA_POWER_DOWN (0 << 8) ++#define LVDS_A0A2_CLKA_POWER_UP (3 << 8) ++/* ++ * Controls the A3 data pair, which contains the additional LSBs for 24 bit ++ * mode. Only enabled if LVDS_A0A2_CLKA_POWER_UP also indicates it should be ++ * on. ++ */ ++#define LVDS_A3_POWER_MASK (3 << 6) ++#define LVDS_A3_POWER_DOWN (0 << 6) ++#define LVDS_A3_POWER_UP (3 << 6) ++/* ++ * Controls the CLKB pair. This should only be set when LVDS_B0B3_POWER_UP ++ * is set. ++ */ ++#define LVDS_CLKB_POWER_MASK (3 << 4) ++#define LVDS_CLKB_POWER_DOWN (0 << 4) ++#define LVDS_CLKB_POWER_UP (3 << 4) ++/* ++ * Controls the B0-B3 data pairs. This must be set to match the DPLL p2 ++ * setting for whether we are in dual-channel mode. The B3 pair will ++ * additionally only be powered up when LVDS_A3_POWER_UP is set. ++ */ ++#define LVDS_B0B3_POWER_MASK (3 << 2) ++#define LVDS_B0B3_POWER_DOWN (0 << 2) ++#define LVDS_B0B3_POWER_UP (3 << 2) ++ ++/* Video Data Island Packet control */ ++#define VIDEO_DIP_DATA _MMIO(0x61178) ++/* Read the description of VIDEO_DIP_DATA (before Haswell) or VIDEO_DIP_ECC ++ * (Haswell and newer) to see which VIDEO_DIP_DATA byte corresponds to each byte ++ * of the infoframe structure specified by CEA-861. */ ++#define VIDEO_DIP_DATA_SIZE 32 ++#define VIDEO_DIP_VSC_DATA_SIZE 36 ++#define VIDEO_DIP_PPS_DATA_SIZE 132 ++#define VIDEO_DIP_CTL _MMIO(0x61170) ++/* Pre HSW: */ ++#define VIDEO_DIP_ENABLE (1 << 31) ++#define VIDEO_DIP_PORT(port) ((port) << 29) ++#define VIDEO_DIP_PORT_MASK (3 << 29) ++#define VIDEO_DIP_ENABLE_GCP (1 << 25) /* ilk+ */ ++#define VIDEO_DIP_ENABLE_AVI (1 << 21) ++#define VIDEO_DIP_ENABLE_VENDOR (2 << 21) ++#define VIDEO_DIP_ENABLE_GAMUT (4 << 21) /* ilk+ */ ++#define VIDEO_DIP_ENABLE_SPD (8 << 21) ++#define VIDEO_DIP_SELECT_AVI (0 << 19) ++#define VIDEO_DIP_SELECT_VENDOR (1 << 19) ++#define VIDEO_DIP_SELECT_GAMUT (2 << 19) ++#define VIDEO_DIP_SELECT_SPD (3 << 19) ++#define VIDEO_DIP_SELECT_MASK (3 << 19) ++#define VIDEO_DIP_FREQ_ONCE (0 << 16) ++#define VIDEO_DIP_FREQ_VSYNC (1 << 16) ++#define VIDEO_DIP_FREQ_2VSYNC (2 << 16) ++#define VIDEO_DIP_FREQ_MASK (3 << 16) ++/* HSW and later: */ ++#define DRM_DIP_ENABLE (1 << 28) ++#define PSR_VSC_BIT_7_SET (1 << 27) ++#define VSC_SELECT_MASK (0x3 << 25) ++#define VSC_SELECT_SHIFT 25 ++#define VSC_DIP_HW_HEA_DATA (0 << 25) ++#define VSC_DIP_HW_HEA_SW_DATA (1 << 25) ++#define VSC_DIP_HW_DATA_SW_HEA (2 << 25) ++#define VSC_DIP_SW_HEA_DATA (3 << 25) ++#define VDIP_ENABLE_PPS (1 << 24) ++#define VIDEO_DIP_ENABLE_VSC_HSW (1 << 20) ++#define VIDEO_DIP_ENABLE_GCP_HSW (1 << 16) ++#define VIDEO_DIP_ENABLE_AVI_HSW (1 << 12) ++#define VIDEO_DIP_ENABLE_VS_HSW (1 << 8) ++#define VIDEO_DIP_ENABLE_GMP_HSW (1 << 4) ++#define VIDEO_DIP_ENABLE_SPD_HSW (1 << 0) ++ ++/* Panel power sequencing */ ++#define PPS_BASE 0x61200 ++#define VLV_PPS_BASE (VLV_DISPLAY_BASE + PPS_BASE) ++#define PCH_PPS_BASE 0xC7200 ++ ++#define _MMIO_PPS(pps_idx, reg) _MMIO(dev_priv->pps_mmio_base - \ ++ PPS_BASE + (reg) + \ ++ (pps_idx) * 0x100) ++ ++#define _PP_STATUS 0x61200 ++#define PP_STATUS(pps_idx) _MMIO_PPS(pps_idx, _PP_STATUS) ++#define PP_ON REG_BIT(31) ++ ++#define _PP_CONTROL_1 0xc7204 ++#define _PP_CONTROL_2 0xc7304 ++#define ICP_PP_CONTROL(x) _MMIO(((x) == 1) ? _PP_CONTROL_1 : \ ++ _PP_CONTROL_2) ++#define POWER_CYCLE_DELAY_MASK REG_GENMASK(8, 4) ++#define VDD_OVERRIDE_FORCE REG_BIT(3) ++#define BACKLIGHT_ENABLE REG_BIT(2) ++#define PWR_DOWN_ON_RESET REG_BIT(1) ++#define PWR_STATE_TARGET REG_BIT(0) ++/* ++ * Indicates that all dependencies of the panel are on: ++ * ++ * - PLL enabled ++ * - pipe enabled ++ * - LVDS/DVOB/DVOC on ++ */ ++#define PP_READY REG_BIT(30) ++#define PP_SEQUENCE_MASK REG_GENMASK(29, 28) ++#define PP_SEQUENCE_NONE REG_FIELD_PREP(PP_SEQUENCE_MASK, 0) ++#define PP_SEQUENCE_POWER_UP REG_FIELD_PREP(PP_SEQUENCE_MASK, 1) ++#define PP_SEQUENCE_POWER_DOWN REG_FIELD_PREP(PP_SEQUENCE_MASK, 2) ++#define PP_CYCLE_DELAY_ACTIVE REG_BIT(27) ++#define PP_SEQUENCE_STATE_MASK REG_GENMASK(3, 0) ++#define PP_SEQUENCE_STATE_OFF_IDLE REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0x0) ++#define PP_SEQUENCE_STATE_OFF_S0_1 REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0x1) ++#define PP_SEQUENCE_STATE_OFF_S0_2 REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0x2) ++#define PP_SEQUENCE_STATE_OFF_S0_3 REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0x3) ++#define PP_SEQUENCE_STATE_ON_IDLE REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0x8) ++#define PP_SEQUENCE_STATE_ON_S1_1 REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0x9) ++#define PP_SEQUENCE_STATE_ON_S1_2 REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0xa) ++#define PP_SEQUENCE_STATE_ON_S1_3 REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0xb) ++#define PP_SEQUENCE_STATE_RESET REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0xf) ++ ++#define _PP_CONTROL 0x61204 ++#define PP_CONTROL(pps_idx) _MMIO_PPS(pps_idx, _PP_CONTROL) ++#define PANEL_UNLOCK_MASK REG_GENMASK(31, 16) ++#define PANEL_UNLOCK_REGS REG_FIELD_PREP(PANEL_UNLOCK_MASK, 0xabcd) ++#define BXT_POWER_CYCLE_DELAY_MASK REG_GENMASK(8, 4) ++#define EDP_FORCE_VDD REG_BIT(3) ++#define EDP_BLC_ENABLE REG_BIT(2) ++#define PANEL_POWER_RESET REG_BIT(1) ++#define PANEL_POWER_ON REG_BIT(0) ++ ++#define _PP_ON_DELAYS 0x61208 ++#define PP_ON_DELAYS(pps_idx) _MMIO_PPS(pps_idx, _PP_ON_DELAYS) ++#define PANEL_PORT_SELECT_MASK REG_GENMASK(31, 30) ++#define PANEL_PORT_SELECT_LVDS REG_FIELD_PREP(PANEL_PORT_SELECT_MASK, 0) ++#define PANEL_PORT_SELECT_DPA REG_FIELD_PREP(PANEL_PORT_SELECT_MASK, 1) ++#define PANEL_PORT_SELECT_DPC REG_FIELD_PREP(PANEL_PORT_SELECT_MASK, 2) ++#define PANEL_PORT_SELECT_DPD REG_FIELD_PREP(PANEL_PORT_SELECT_MASK, 3) ++#define PANEL_PORT_SELECT_VLV(port) REG_FIELD_PREP(PANEL_PORT_SELECT_MASK, port) ++#define PANEL_POWER_UP_DELAY_MASK REG_GENMASK(28, 16) ++#define PANEL_LIGHT_ON_DELAY_MASK REG_GENMASK(12, 0) ++ ++#define _PP_OFF_DELAYS 0x6120C ++#define PP_OFF_DELAYS(pps_idx) _MMIO_PPS(pps_idx, _PP_OFF_DELAYS) ++#define PANEL_POWER_DOWN_DELAY_MASK REG_GENMASK(28, 16) ++#define PANEL_LIGHT_OFF_DELAY_MASK REG_GENMASK(12, 0) ++ ++#define _PP_DIVISOR 0x61210 ++#define PP_DIVISOR(pps_idx) _MMIO_PPS(pps_idx, _PP_DIVISOR) ++#define PP_REFERENCE_DIVIDER_MASK REG_GENMASK(31, 8) ++#define PANEL_POWER_CYCLE_DELAY_MASK REG_GENMASK(4, 0) ++ ++/* Panel fitting */ ++#define PFIT_CONTROL _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61230) ++#define PFIT_ENABLE (1 << 31) ++#define PFIT_PIPE_MASK (3 << 29) ++#define PFIT_PIPE_SHIFT 29 ++#define VERT_INTERP_DISABLE (0 << 10) ++#define VERT_INTERP_BILINEAR (1 << 10) ++#define VERT_INTERP_MASK (3 << 10) ++#define VERT_AUTO_SCALE (1 << 9) ++#define HORIZ_INTERP_DISABLE (0 << 6) ++#define HORIZ_INTERP_BILINEAR (1 << 6) ++#define HORIZ_INTERP_MASK (3 << 6) ++#define HORIZ_AUTO_SCALE (1 << 5) ++#define PANEL_8TO6_DITHER_ENABLE (1 << 3) ++#define PFIT_FILTER_FUZZY (0 << 24) ++#define PFIT_SCALING_AUTO (0 << 26) ++#define PFIT_SCALING_PROGRAMMED (1 << 26) ++#define PFIT_SCALING_PILLAR (2 << 26) ++#define PFIT_SCALING_LETTER (3 << 26) ++#define PFIT_PGM_RATIOS _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61234) ++/* Pre-965 */ ++#define PFIT_VERT_SCALE_SHIFT 20 ++#define PFIT_VERT_SCALE_MASK 0xfff00000 ++#define PFIT_HORIZ_SCALE_SHIFT 4 ++#define PFIT_HORIZ_SCALE_MASK 0x0000fff0 ++/* 965+ */ ++#define PFIT_VERT_SCALE_SHIFT_965 16 ++#define PFIT_VERT_SCALE_MASK_965 0x1fff0000 ++#define PFIT_HORIZ_SCALE_SHIFT_965 0 ++#define PFIT_HORIZ_SCALE_MASK_965 0x00001fff ++ ++#define PFIT_AUTO_RATIOS _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61238) ++ ++#define _VLV_BLC_PWM_CTL2_A (DISPLAY_MMIO_BASE(dev_priv) + 0x61250) ++#define _VLV_BLC_PWM_CTL2_B (DISPLAY_MMIO_BASE(dev_priv) + 0x61350) ++#define VLV_BLC_PWM_CTL2(pipe) _MMIO_PIPE(pipe, _VLV_BLC_PWM_CTL2_A, \ ++ _VLV_BLC_PWM_CTL2_B) ++ ++#define _VLV_BLC_PWM_CTL_A (DISPLAY_MMIO_BASE(dev_priv) + 0x61254) ++#define _VLV_BLC_PWM_CTL_B (DISPLAY_MMIO_BASE(dev_priv) + 0x61354) ++#define VLV_BLC_PWM_CTL(pipe) _MMIO_PIPE(pipe, _VLV_BLC_PWM_CTL_A, \ ++ _VLV_BLC_PWM_CTL_B) ++ ++#define _VLV_BLC_HIST_CTL_A (DISPLAY_MMIO_BASE(dev_priv) + 0x61260) ++#define _VLV_BLC_HIST_CTL_B (DISPLAY_MMIO_BASE(dev_priv) + 0x61360) ++#define VLV_BLC_HIST_CTL(pipe) _MMIO_PIPE(pipe, _VLV_BLC_HIST_CTL_A, \ ++ _VLV_BLC_HIST_CTL_B) ++ ++/* Backlight control */ ++#define BLC_PWM_CTL2 _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61250) /* 965+ only */ ++#define BLM_PWM_ENABLE (1 << 31) ++#define BLM_COMBINATION_MODE (1 << 30) /* gen4 only */ ++#define BLM_PIPE_SELECT (1 << 29) ++#define BLM_PIPE_SELECT_IVB (3 << 29) ++#define BLM_PIPE_A (0 << 29) ++#define BLM_PIPE_B (1 << 29) ++#define BLM_PIPE_C (2 << 29) /* ivb + */ ++#define BLM_TRANSCODER_A BLM_PIPE_A /* hsw */ ++#define BLM_TRANSCODER_B BLM_PIPE_B ++#define BLM_TRANSCODER_C BLM_PIPE_C ++#define BLM_TRANSCODER_EDP (3 << 29) ++#define BLM_PIPE(pipe) ((pipe) << 29) ++#define BLM_POLARITY_I965 (1 << 28) /* gen4 only */ ++#define BLM_PHASE_IN_INTERUPT_STATUS (1 << 26) ++#define BLM_PHASE_IN_ENABLE (1 << 25) ++#define BLM_PHASE_IN_INTERUPT_ENABL (1 << 24) ++#define BLM_PHASE_IN_TIME_BASE_SHIFT (16) ++#define BLM_PHASE_IN_TIME_BASE_MASK (0xff << 16) ++#define BLM_PHASE_IN_COUNT_SHIFT (8) ++#define BLM_PHASE_IN_COUNT_MASK (0xff << 8) ++#define BLM_PHASE_IN_INCR_SHIFT (0) ++#define BLM_PHASE_IN_INCR_MASK (0xff << 0) ++#define BLC_PWM_CTL _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61254) ++/* ++ * This is the most significant 15 bits of the number of backlight cycles in a ++ * complete cycle of the modulated backlight control. ++ * ++ * The actual value is this field multiplied by two. ++ */ ++#define BACKLIGHT_MODULATION_FREQ_SHIFT (17) ++#define BACKLIGHT_MODULATION_FREQ_MASK (0x7fff << 17) ++#define BLM_LEGACY_MODE (1 << 16) /* gen2 only */ ++/* ++ * This is the number of cycles out of the backlight modulation cycle for which ++ * the backlight is on. ++ * ++ * This field must be no greater than the number of cycles in the complete ++ * backlight modulation cycle. ++ */ ++#define BACKLIGHT_DUTY_CYCLE_SHIFT (0) ++#define BACKLIGHT_DUTY_CYCLE_MASK (0xffff) ++#define BACKLIGHT_DUTY_CYCLE_MASK_PNV (0xfffe) ++#define BLM_POLARITY_PNV (1 << 0) /* pnv only */ ++ ++#define BLC_HIST_CTL _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61260) ++#define BLM_HISTOGRAM_ENABLE (1 << 31) ++ ++/* New registers for PCH-split platforms. Safe where new bits show up, the ++ * register layout machtes with gen4 BLC_PWM_CTL[12]. */ ++#define BLC_PWM_CPU_CTL2 _MMIO(0x48250) ++#define BLC_PWM_CPU_CTL _MMIO(0x48254) ++ ++#define HSW_BLC_PWM2_CTL _MMIO(0x48350) ++ ++/* PCH CTL1 is totally different, all but the below bits are reserved. CTL2 is ++ * like the normal CTL from gen4 and earlier. Hooray for confusing naming. */ ++#define BLC_PWM_PCH_CTL1 _MMIO(0xc8250) ++#define BLM_PCH_PWM_ENABLE (1 << 31) ++#define BLM_PCH_OVERRIDE_ENABLE (1 << 30) ++#define BLM_PCH_POLARITY (1 << 29) ++#define BLC_PWM_PCH_CTL2 _MMIO(0xc8254) ++ ++#define UTIL_PIN_CTL _MMIO(0x48400) ++#define UTIL_PIN_ENABLE (1 << 31) ++ ++#define UTIL_PIN_PIPE(x) ((x) << 29) ++#define UTIL_PIN_PIPE_MASK (3 << 29) ++#define UTIL_PIN_MODE_PWM (1 << 24) ++#define UTIL_PIN_MODE_MASK (0xf << 24) ++#define UTIL_PIN_POLARITY (1 << 22) ++ ++/* BXT backlight register definition. */ ++#define _BXT_BLC_PWM_CTL1 0xC8250 ++#define BXT_BLC_PWM_ENABLE (1 << 31) ++#define BXT_BLC_PWM_POLARITY (1 << 29) ++#define _BXT_BLC_PWM_FREQ1 0xC8254 ++#define _BXT_BLC_PWM_DUTY1 0xC8258 ++ ++#define _BXT_BLC_PWM_CTL2 0xC8350 ++#define _BXT_BLC_PWM_FREQ2 0xC8354 ++#define _BXT_BLC_PWM_DUTY2 0xC8358 ++ ++#define BXT_BLC_PWM_CTL(controller) _MMIO_PIPE(controller, \ ++ _BXT_BLC_PWM_CTL1, _BXT_BLC_PWM_CTL2) ++#define BXT_BLC_PWM_FREQ(controller) _MMIO_PIPE(controller, \ ++ _BXT_BLC_PWM_FREQ1, _BXT_BLC_PWM_FREQ2) ++#define BXT_BLC_PWM_DUTY(controller) _MMIO_PIPE(controller, \ ++ _BXT_BLC_PWM_DUTY1, _BXT_BLC_PWM_DUTY2) ++ ++#define PCH_GTC_CTL _MMIO(0xe7000) ++#define PCH_GTC_ENABLE (1 << 31) ++ ++/* TV port control */ ++#define TV_CTL _MMIO(0x68000) ++/* Enables the TV encoder */ ++# define TV_ENC_ENABLE (1 << 31) ++/* Sources the TV encoder input from pipe B instead of A. */ ++# define TV_ENC_PIPE_SEL_SHIFT 30 ++# define TV_ENC_PIPE_SEL_MASK (1 << 30) ++# define TV_ENC_PIPE_SEL(pipe) ((pipe) << 30) ++/* Outputs composite video (DAC A only) */ ++# define TV_ENC_OUTPUT_COMPOSITE (0 << 28) ++/* Outputs SVideo video (DAC B/C) */ ++# define TV_ENC_OUTPUT_SVIDEO (1 << 28) ++/* Outputs Component video (DAC A/B/C) */ ++# define TV_ENC_OUTPUT_COMPONENT (2 << 28) ++/* Outputs Composite and SVideo (DAC A/B/C) */ ++# define TV_ENC_OUTPUT_SVIDEO_COMPOSITE (3 << 28) ++# define TV_TRILEVEL_SYNC (1 << 21) ++/* Enables slow sync generation (945GM only) */ ++# define TV_SLOW_SYNC (1 << 20) ++/* Selects 4x oversampling for 480i and 576p */ ++# define TV_OVERSAMPLE_4X (0 << 18) ++/* Selects 2x oversampling for 720p and 1080i */ ++# define TV_OVERSAMPLE_2X (1 << 18) ++/* Selects no oversampling for 1080p */ ++# define TV_OVERSAMPLE_NONE (2 << 18) ++/* Selects 8x oversampling */ ++# define TV_OVERSAMPLE_8X (3 << 18) ++# define TV_OVERSAMPLE_MASK (3 << 18) ++/* Selects progressive mode rather than interlaced */ ++# define TV_PROGRESSIVE (1 << 17) ++/* Sets the colorburst to PAL mode. Required for non-M PAL modes. */ ++# define TV_PAL_BURST (1 << 16) ++/* Field for setting delay of Y compared to C */ ++# define TV_YC_SKEW_MASK (7 << 12) ++/* Enables a fix for 480p/576p standard definition modes on the 915GM only */ ++# define TV_ENC_SDP_FIX (1 << 11) ++/* ++ * Enables a fix for the 915GM only. ++ * ++ * Not sure what it does. ++ */ ++# define TV_ENC_C0_FIX (1 << 10) ++/* Bits that must be preserved by software */ ++# define TV_CTL_SAVE ((1 << 11) | (3 << 9) | (7 << 6) | 0xf) ++# define TV_FUSE_STATE_MASK (3 << 4) ++/* Read-only state that reports all features enabled */ ++# define TV_FUSE_STATE_ENABLED (0 << 4) ++/* Read-only state that reports that Macrovision is disabled in hardware*/ ++# define TV_FUSE_STATE_NO_MACROVISION (1 << 4) ++/* Read-only state that reports that TV-out is disabled in hardware. */ ++# define TV_FUSE_STATE_DISABLED (2 << 4) ++/* Normal operation */ ++# define TV_TEST_MODE_NORMAL (0 << 0) ++/* Encoder test pattern 1 - combo pattern */ ++# define TV_TEST_MODE_PATTERN_1 (1 << 0) ++/* Encoder test pattern 2 - full screen vertical 75% color bars */ ++# define TV_TEST_MODE_PATTERN_2 (2 << 0) ++/* Encoder test pattern 3 - full screen horizontal 75% color bars */ ++# define TV_TEST_MODE_PATTERN_3 (3 << 0) ++/* Encoder test pattern 4 - random noise */ ++# define TV_TEST_MODE_PATTERN_4 (4 << 0) ++/* Encoder test pattern 5 - linear color ramps */ ++# define TV_TEST_MODE_PATTERN_5 (5 << 0) ++/* ++ * This test mode forces the DACs to 50% of full output. ++ * ++ * This is used for load detection in combination with TVDAC_SENSE_MASK ++ */ ++# define TV_TEST_MODE_MONITOR_DETECT (7 << 0) ++# define TV_TEST_MODE_MASK (7 << 0) ++ ++#define TV_DAC _MMIO(0x68004) ++# define TV_DAC_SAVE 0x00ffff00 ++/* ++ * Reports that DAC state change logic has reported change (RO). ++ * ++ * This gets cleared when TV_DAC_STATE_EN is cleared ++*/ ++# define TVDAC_STATE_CHG (1 << 31) ++# define TVDAC_SENSE_MASK (7 << 28) ++/* Reports that DAC A voltage is above the detect threshold */ ++# define TVDAC_A_SENSE (1 << 30) ++/* Reports that DAC B voltage is above the detect threshold */ ++# define TVDAC_B_SENSE (1 << 29) ++/* Reports that DAC C voltage is above the detect threshold */ ++# define TVDAC_C_SENSE (1 << 28) ++/* ++ * Enables DAC state detection logic, for load-based TV detection. ++ * ++ * The PLL of the chosen pipe (in TV_CTL) must be running, and the encoder set ++ * to off, for load detection to work. ++ */ ++# define TVDAC_STATE_CHG_EN (1 << 27) ++/* Sets the DAC A sense value to high */ ++# define TVDAC_A_SENSE_CTL (1 << 26) ++/* Sets the DAC B sense value to high */ ++# define TVDAC_B_SENSE_CTL (1 << 25) ++/* Sets the DAC C sense value to high */ ++# define TVDAC_C_SENSE_CTL (1 << 24) ++/* Overrides the ENC_ENABLE and DAC voltage levels */ ++# define DAC_CTL_OVERRIDE (1 << 7) ++/* Sets the slew rate. Must be preserved in software */ ++# define ENC_TVDAC_SLEW_FAST (1 << 6) ++# define DAC_A_1_3_V (0 << 4) ++# define DAC_A_1_1_V (1 << 4) ++# define DAC_A_0_7_V (2 << 4) ++# define DAC_A_MASK (3 << 4) ++# define DAC_B_1_3_V (0 << 2) ++# define DAC_B_1_1_V (1 << 2) ++# define DAC_B_0_7_V (2 << 2) ++# define DAC_B_MASK (3 << 2) ++# define DAC_C_1_3_V (0 << 0) ++# define DAC_C_1_1_V (1 << 0) ++# define DAC_C_0_7_V (2 << 0) ++# define DAC_C_MASK (3 << 0) ++ ++/* ++ * CSC coefficients are stored in a floating point format with 9 bits of ++ * mantissa and 2 or 3 bits of exponent. The exponent is represented as 2**-n, ++ * where 2-bit exponents are unsigned n, and 3-bit exponents are signed n with ++ * -1 (0x3) being the only legal negative value. ++ */ ++#define TV_CSC_Y _MMIO(0x68010) ++# define TV_RY_MASK 0x07ff0000 ++# define TV_RY_SHIFT 16 ++# define TV_GY_MASK 0x00000fff ++# define TV_GY_SHIFT 0 ++ ++#define TV_CSC_Y2 _MMIO(0x68014) ++# define TV_BY_MASK 0x07ff0000 ++# define TV_BY_SHIFT 16 ++/* ++ * Y attenuation for component video. ++ * ++ * Stored in 1.9 fixed point. ++ */ ++# define TV_AY_MASK 0x000003ff ++# define TV_AY_SHIFT 0 ++ ++#define TV_CSC_U _MMIO(0x68018) ++# define TV_RU_MASK 0x07ff0000 ++# define TV_RU_SHIFT 16 ++# define TV_GU_MASK 0x000007ff ++# define TV_GU_SHIFT 0 ++ ++#define TV_CSC_U2 _MMIO(0x6801c) ++# define TV_BU_MASK 0x07ff0000 ++# define TV_BU_SHIFT 16 ++/* ++ * U attenuation for component video. ++ * ++ * Stored in 1.9 fixed point. ++ */ ++# define TV_AU_MASK 0x000003ff ++# define TV_AU_SHIFT 0 ++ ++#define TV_CSC_V _MMIO(0x68020) ++# define TV_RV_MASK 0x0fff0000 ++# define TV_RV_SHIFT 16 ++# define TV_GV_MASK 0x000007ff ++# define TV_GV_SHIFT 0 ++ ++#define TV_CSC_V2 _MMIO(0x68024) ++# define TV_BV_MASK 0x07ff0000 ++# define TV_BV_SHIFT 16 ++/* ++ * V attenuation for component video. ++ * ++ * Stored in 1.9 fixed point. ++ */ ++# define TV_AV_MASK 0x000007ff ++# define TV_AV_SHIFT 0 ++ ++#define TV_CLR_KNOBS _MMIO(0x68028) ++/* 2s-complement brightness adjustment */ ++# define TV_BRIGHTNESS_MASK 0xff000000 ++# define TV_BRIGHTNESS_SHIFT 24 ++/* Contrast adjustment, as a 2.6 unsigned floating point number */ ++# define TV_CONTRAST_MASK 0x00ff0000 ++# define TV_CONTRAST_SHIFT 16 ++/* Saturation adjustment, as a 2.6 unsigned floating point number */ ++# define TV_SATURATION_MASK 0x0000ff00 ++# define TV_SATURATION_SHIFT 8 ++/* Hue adjustment, as an integer phase angle in degrees */ ++# define TV_HUE_MASK 0x000000ff ++# define TV_HUE_SHIFT 0 ++ ++#define TV_CLR_LEVEL _MMIO(0x6802c) ++/* Controls the DAC level for black */ ++# define TV_BLACK_LEVEL_MASK 0x01ff0000 ++# define TV_BLACK_LEVEL_SHIFT 16 ++/* Controls the DAC level for blanking */ ++# define TV_BLANK_LEVEL_MASK 0x000001ff ++# define TV_BLANK_LEVEL_SHIFT 0 ++ ++#define TV_H_CTL_1 _MMIO(0x68030) ++/* Number of pixels in the hsync. */ ++# define TV_HSYNC_END_MASK 0x1fff0000 ++# define TV_HSYNC_END_SHIFT 16 ++/* Total number of pixels minus one in the line (display and blanking). */ ++# define TV_HTOTAL_MASK 0x00001fff ++# define TV_HTOTAL_SHIFT 0 ++ ++#define TV_H_CTL_2 _MMIO(0x68034) ++/* Enables the colorburst (needed for non-component color) */ ++# define TV_BURST_ENA (1 << 31) ++/* Offset of the colorburst from the start of hsync, in pixels minus one. */ ++# define TV_HBURST_START_SHIFT 16 ++# define TV_HBURST_START_MASK 0x1fff0000 ++/* Length of the colorburst */ ++# define TV_HBURST_LEN_SHIFT 0 ++# define TV_HBURST_LEN_MASK 0x0001fff ++ ++#define TV_H_CTL_3 _MMIO(0x68038) ++/* End of hblank, measured in pixels minus one from start of hsync */ ++# define TV_HBLANK_END_SHIFT 16 ++# define TV_HBLANK_END_MASK 0x1fff0000 ++/* Start of hblank, measured in pixels minus one from start of hsync */ ++# define TV_HBLANK_START_SHIFT 0 ++# define TV_HBLANK_START_MASK 0x0001fff ++ ++#define TV_V_CTL_1 _MMIO(0x6803c) ++/* XXX */ ++# define TV_NBR_END_SHIFT 16 ++# define TV_NBR_END_MASK 0x07ff0000 ++/* XXX */ ++# define TV_VI_END_F1_SHIFT 8 ++# define TV_VI_END_F1_MASK 0x00003f00 ++/* XXX */ ++# define TV_VI_END_F2_SHIFT 0 ++# define TV_VI_END_F2_MASK 0x0000003f ++ ++#define TV_V_CTL_2 _MMIO(0x68040) ++/* Length of vsync, in half lines */ ++# define TV_VSYNC_LEN_MASK 0x07ff0000 ++# define TV_VSYNC_LEN_SHIFT 16 ++/* Offset of the start of vsync in field 1, measured in one less than the ++ * number of half lines. ++ */ ++# define TV_VSYNC_START_F1_MASK 0x00007f00 ++# define TV_VSYNC_START_F1_SHIFT 8 ++/* ++ * Offset of the start of vsync in field 2, measured in one less than the ++ * number of half lines. ++ */ ++# define TV_VSYNC_START_F2_MASK 0x0000007f ++# define TV_VSYNC_START_F2_SHIFT 0 ++ ++#define TV_V_CTL_3 _MMIO(0x68044) ++/* Enables generation of the equalization signal */ ++# define TV_EQUAL_ENA (1 << 31) ++/* Length of vsync, in half lines */ ++# define TV_VEQ_LEN_MASK 0x007f0000 ++# define TV_VEQ_LEN_SHIFT 16 ++/* Offset of the start of equalization in field 1, measured in one less than ++ * the number of half lines. ++ */ ++# define TV_VEQ_START_F1_MASK 0x0007f00 ++# define TV_VEQ_START_F1_SHIFT 8 ++/* ++ * Offset of the start of equalization in field 2, measured in one less than ++ * the number of half lines. ++ */ ++# define TV_VEQ_START_F2_MASK 0x000007f ++# define TV_VEQ_START_F2_SHIFT 0 ++ ++#define TV_V_CTL_4 _MMIO(0x68048) ++/* ++ * Offset to start of vertical colorburst, measured in one less than the ++ * number of lines from vertical start. ++ */ ++# define TV_VBURST_START_F1_MASK 0x003f0000 ++# define TV_VBURST_START_F1_SHIFT 16 ++/* ++ * Offset to the end of vertical colorburst, measured in one less than the ++ * number of lines from the start of NBR. ++ */ ++# define TV_VBURST_END_F1_MASK 0x000000ff ++# define TV_VBURST_END_F1_SHIFT 0 ++ ++#define TV_V_CTL_5 _MMIO(0x6804c) ++/* ++ * Offset to start of vertical colorburst, measured in one less than the ++ * number of lines from vertical start. ++ */ ++# define TV_VBURST_START_F2_MASK 0x003f0000 ++# define TV_VBURST_START_F2_SHIFT 16 ++/* ++ * Offset to the end of vertical colorburst, measured in one less than the ++ * number of lines from the start of NBR. ++ */ ++# define TV_VBURST_END_F2_MASK 0x000000ff ++# define TV_VBURST_END_F2_SHIFT 0 ++ ++#define TV_V_CTL_6 _MMIO(0x68050) ++/* ++ * Offset to start of vertical colorburst, measured in one less than the ++ * number of lines from vertical start. ++ */ ++# define TV_VBURST_START_F3_MASK 0x003f0000 ++# define TV_VBURST_START_F3_SHIFT 16 ++/* ++ * Offset to the end of vertical colorburst, measured in one less than the ++ * number of lines from the start of NBR. ++ */ ++# define TV_VBURST_END_F3_MASK 0x000000ff ++# define TV_VBURST_END_F3_SHIFT 0 ++ ++#define TV_V_CTL_7 _MMIO(0x68054) ++/* ++ * Offset to start of vertical colorburst, measured in one less than the ++ * number of lines from vertical start. ++ */ ++# define TV_VBURST_START_F4_MASK 0x003f0000 ++# define TV_VBURST_START_F4_SHIFT 16 ++/* ++ * Offset to the end of vertical colorburst, measured in one less than the ++ * number of lines from the start of NBR. ++ */ ++# define TV_VBURST_END_F4_MASK 0x000000ff ++# define TV_VBURST_END_F4_SHIFT 0 ++ ++#define TV_SC_CTL_1 _MMIO(0x68060) ++/* Turns on the first subcarrier phase generation DDA */ ++# define TV_SC_DDA1_EN (1 << 31) ++/* Turns on the first subcarrier phase generation DDA */ ++# define TV_SC_DDA2_EN (1 << 30) ++/* Turns on the first subcarrier phase generation DDA */ ++# define TV_SC_DDA3_EN (1 << 29) ++/* Sets the subcarrier DDA to reset frequency every other field */ ++# define TV_SC_RESET_EVERY_2 (0 << 24) ++/* Sets the subcarrier DDA to reset frequency every fourth field */ ++# define TV_SC_RESET_EVERY_4 (1 << 24) ++/* Sets the subcarrier DDA to reset frequency every eighth field */ ++# define TV_SC_RESET_EVERY_8 (2 << 24) ++/* Sets the subcarrier DDA to never reset the frequency */ ++# define TV_SC_RESET_NEVER (3 << 24) ++/* Sets the peak amplitude of the colorburst.*/ ++# define TV_BURST_LEVEL_MASK 0x00ff0000 ++# define TV_BURST_LEVEL_SHIFT 16 ++/* Sets the increment of the first subcarrier phase generation DDA */ ++# define TV_SCDDA1_INC_MASK 0x00000fff ++# define TV_SCDDA1_INC_SHIFT 0 ++ ++#define TV_SC_CTL_2 _MMIO(0x68064) ++/* Sets the rollover for the second subcarrier phase generation DDA */ ++# define TV_SCDDA2_SIZE_MASK 0x7fff0000 ++# define TV_SCDDA2_SIZE_SHIFT 16 ++/* Sets the increent of the second subcarrier phase generation DDA */ ++# define TV_SCDDA2_INC_MASK 0x00007fff ++# define TV_SCDDA2_INC_SHIFT 0 ++ ++#define TV_SC_CTL_3 _MMIO(0x68068) ++/* Sets the rollover for the third subcarrier phase generation DDA */ ++# define TV_SCDDA3_SIZE_MASK 0x7fff0000 ++# define TV_SCDDA3_SIZE_SHIFT 16 ++/* Sets the increent of the third subcarrier phase generation DDA */ ++# define TV_SCDDA3_INC_MASK 0x00007fff ++# define TV_SCDDA3_INC_SHIFT 0 ++ ++#define TV_WIN_POS _MMIO(0x68070) ++/* X coordinate of the display from the start of horizontal active */ ++# define TV_XPOS_MASK 0x1fff0000 ++# define TV_XPOS_SHIFT 16 ++/* Y coordinate of the display from the start of vertical active (NBR) */ ++# define TV_YPOS_MASK 0x00000fff ++# define TV_YPOS_SHIFT 0 ++ ++#define TV_WIN_SIZE _MMIO(0x68074) ++/* Horizontal size of the display window, measured in pixels*/ ++# define TV_XSIZE_MASK 0x1fff0000 ++# define TV_XSIZE_SHIFT 16 ++/* ++ * Vertical size of the display window, measured in pixels. ++ * ++ * Must be even for interlaced modes. ++ */ ++# define TV_YSIZE_MASK 0x00000fff ++# define TV_YSIZE_SHIFT 0 ++ ++#define TV_FILTER_CTL_1 _MMIO(0x68080) ++/* ++ * Enables automatic scaling calculation. ++ * ++ * If set, the rest of the registers are ignored, and the calculated values can ++ * be read back from the register. ++ */ ++# define TV_AUTO_SCALE (1 << 31) ++/* ++ * Disables the vertical filter. ++ * ++ * This is required on modes more than 1024 pixels wide */ ++# define TV_V_FILTER_BYPASS (1 << 29) ++/* Enables adaptive vertical filtering */ ++# define TV_VADAPT (1 << 28) ++# define TV_VADAPT_MODE_MASK (3 << 26) ++/* Selects the least adaptive vertical filtering mode */ ++# define TV_VADAPT_MODE_LEAST (0 << 26) ++/* Selects the moderately adaptive vertical filtering mode */ ++# define TV_VADAPT_MODE_MODERATE (1 << 26) ++/* Selects the most adaptive vertical filtering mode */ ++# define TV_VADAPT_MODE_MOST (3 << 26) ++/* ++ * Sets the horizontal scaling factor. ++ * ++ * This should be the fractional part of the horizontal scaling factor divided ++ * by the oversampling rate. TV_HSCALE should be less than 1, and set to: ++ * ++ * (src width - 1) / ((oversample * dest width) - 1) ++ */ ++# define TV_HSCALE_FRAC_MASK 0x00003fff ++# define TV_HSCALE_FRAC_SHIFT 0 ++ ++#define TV_FILTER_CTL_2 _MMIO(0x68084) ++/* ++ * Sets the integer part of the 3.15 fixed-point vertical scaling factor. ++ * ++ * TV_VSCALE should be (src height - 1) / ((interlace * dest height) - 1) ++ */ ++# define TV_VSCALE_INT_MASK 0x00038000 ++# define TV_VSCALE_INT_SHIFT 15 ++/* ++ * Sets the fractional part of the 3.15 fixed-point vertical scaling factor. ++ * ++ * \sa TV_VSCALE_INT_MASK ++ */ ++# define TV_VSCALE_FRAC_MASK 0x00007fff ++# define TV_VSCALE_FRAC_SHIFT 0 ++ ++#define TV_FILTER_CTL_3 _MMIO(0x68088) ++/* ++ * Sets the integer part of the 3.15 fixed-point vertical scaling factor. ++ * ++ * TV_VSCALE should be (src height - 1) / (1/4 * (dest height - 1)) ++ * ++ * For progressive modes, TV_VSCALE_IP_INT should be set to zeroes. ++ */ ++# define TV_VSCALE_IP_INT_MASK 0x00038000 ++# define TV_VSCALE_IP_INT_SHIFT 15 ++/* ++ * Sets the fractional part of the 3.15 fixed-point vertical scaling factor. ++ * ++ * For progressive modes, TV_VSCALE_IP_INT should be set to zeroes. ++ * ++ * \sa TV_VSCALE_IP_INT_MASK ++ */ ++# define TV_VSCALE_IP_FRAC_MASK 0x00007fff ++# define TV_VSCALE_IP_FRAC_SHIFT 0 ++ ++#define TV_CC_CONTROL _MMIO(0x68090) ++# define TV_CC_ENABLE (1 << 31) ++/* ++ * Specifies which field to send the CC data in. ++ * ++ * CC data is usually sent in field 0. ++ */ ++# define TV_CC_FID_MASK (1 << 27) ++# define TV_CC_FID_SHIFT 27 ++/* Sets the horizontal position of the CC data. Usually 135. */ ++# define TV_CC_HOFF_MASK 0x03ff0000 ++# define TV_CC_HOFF_SHIFT 16 ++/* Sets the vertical position of the CC data. Usually 21 */ ++# define TV_CC_LINE_MASK 0x0000003f ++# define TV_CC_LINE_SHIFT 0 ++ ++#define TV_CC_DATA _MMIO(0x68094) ++# define TV_CC_RDY (1 << 31) ++/* Second word of CC data to be transmitted. */ ++# define TV_CC_DATA_2_MASK 0x007f0000 ++# define TV_CC_DATA_2_SHIFT 16 ++/* First word of CC data to be transmitted. */ ++# define TV_CC_DATA_1_MASK 0x0000007f ++# define TV_CC_DATA_1_SHIFT 0 ++ ++#define TV_H_LUMA(i) _MMIO(0x68100 + (i) * 4) /* 60 registers */ ++#define TV_H_CHROMA(i) _MMIO(0x68200 + (i) * 4) /* 60 registers */ ++#define TV_V_LUMA(i) _MMIO(0x68300 + (i) * 4) /* 43 registers */ ++#define TV_V_CHROMA(i) _MMIO(0x68400 + (i) * 4) /* 43 registers */ ++ ++/* Display Port */ ++#define DP_A _MMIO(0x64000) /* eDP */ ++#define DP_B _MMIO(0x64100) ++#define DP_C _MMIO(0x64200) ++#define DP_D _MMIO(0x64300) ++ ++#define VLV_DP_B _MMIO(VLV_DISPLAY_BASE + 0x64100) ++#define VLV_DP_C _MMIO(VLV_DISPLAY_BASE + 0x64200) ++#define CHV_DP_D _MMIO(VLV_DISPLAY_BASE + 0x64300) ++ ++#define DP_PORT_EN (1 << 31) ++#define DP_PIPE_SEL_SHIFT 30 ++#define DP_PIPE_SEL_MASK (1 << 30) ++#define DP_PIPE_SEL(pipe) ((pipe) << 30) ++#define DP_PIPE_SEL_SHIFT_IVB 29 ++#define DP_PIPE_SEL_MASK_IVB (3 << 29) ++#define DP_PIPE_SEL_IVB(pipe) ((pipe) << 29) ++#define DP_PIPE_SEL_SHIFT_CHV 16 ++#define DP_PIPE_SEL_MASK_CHV (3 << 16) ++#define DP_PIPE_SEL_CHV(pipe) ((pipe) << 16) ++ ++/* Link training mode - select a suitable mode for each stage */ ++#define DP_LINK_TRAIN_PAT_1 (0 << 28) ++#define DP_LINK_TRAIN_PAT_2 (1 << 28) ++#define DP_LINK_TRAIN_PAT_IDLE (2 << 28) ++#define DP_LINK_TRAIN_OFF (3 << 28) ++#define DP_LINK_TRAIN_MASK (3 << 28) ++#define DP_LINK_TRAIN_SHIFT 28 ++ ++/* CPT Link training mode */ ++#define DP_LINK_TRAIN_PAT_1_CPT (0 << 8) ++#define DP_LINK_TRAIN_PAT_2_CPT (1 << 8) ++#define DP_LINK_TRAIN_PAT_IDLE_CPT (2 << 8) ++#define DP_LINK_TRAIN_OFF_CPT (3 << 8) ++#define DP_LINK_TRAIN_MASK_CPT (7 << 8) ++#define DP_LINK_TRAIN_SHIFT_CPT 8 ++ ++/* Signal voltages. These are mostly controlled by the other end */ ++#define DP_VOLTAGE_0_4 (0 << 25) ++#define DP_VOLTAGE_0_6 (1 << 25) ++#define DP_VOLTAGE_0_8 (2 << 25) ++#define DP_VOLTAGE_1_2 (3 << 25) ++#define DP_VOLTAGE_MASK (7 << 25) ++#define DP_VOLTAGE_SHIFT 25 ++ ++/* Signal pre-emphasis levels, like voltages, the other end tells us what ++ * they want ++ */ ++#define DP_PRE_EMPHASIS_0 (0 << 22) ++#define DP_PRE_EMPHASIS_3_5 (1 << 22) ++#define DP_PRE_EMPHASIS_6 (2 << 22) ++#define DP_PRE_EMPHASIS_9_5 (3 << 22) ++#define DP_PRE_EMPHASIS_MASK (7 << 22) ++#define DP_PRE_EMPHASIS_SHIFT 22 ++ ++/* How many wires to use. I guess 3 was too hard */ ++#define DP_PORT_WIDTH(width) (((width) - 1) << 19) ++#define DP_PORT_WIDTH_MASK (7 << 19) ++#define DP_PORT_WIDTH_SHIFT 19 ++ ++/* Mystic DPCD version 1.1 special mode */ ++#define DP_ENHANCED_FRAMING (1 << 18) ++ ++/* eDP */ ++#define DP_PLL_FREQ_270MHZ (0 << 16) ++#define DP_PLL_FREQ_162MHZ (1 << 16) ++#define DP_PLL_FREQ_MASK (3 << 16) ++ ++/* locked once port is enabled */ ++#define DP_PORT_REVERSAL (1 << 15) ++ ++/* eDP */ ++#define DP_PLL_ENABLE (1 << 14) ++ ++/* sends the clock on lane 15 of the PEG for debug */ ++#define DP_CLOCK_OUTPUT_ENABLE (1 << 13) ++ ++#define DP_SCRAMBLING_DISABLE (1 << 12) ++#define DP_SCRAMBLING_DISABLE_IRONLAKE (1 << 7) ++ ++/* limit RGB values to avoid confusing TVs */ ++#define DP_COLOR_RANGE_16_235 (1 << 8) ++ ++/* Turn on the audio link */ ++#define DP_AUDIO_OUTPUT_ENABLE (1 << 6) ++ ++/* vs and hs sync polarity */ ++#define DP_SYNC_VS_HIGH (1 << 4) ++#define DP_SYNC_HS_HIGH (1 << 3) ++ ++/* A fantasy */ ++#define DP_DETECTED (1 << 2) ++ ++/* The aux channel provides a way to talk to the ++ * signal sink for DDC etc. Max packet size supported ++ * is 20 bytes in each direction, hence the 5 fixed ++ * data registers ++ */ ++#define _DPA_AUX_CH_CTL (DISPLAY_MMIO_BASE(dev_priv) + 0x64010) ++#define _DPA_AUX_CH_DATA1 (DISPLAY_MMIO_BASE(dev_priv) + 0x64014) ++#define _DPA_AUX_CH_DATA2 (DISPLAY_MMIO_BASE(dev_priv) + 0x64018) ++#define _DPA_AUX_CH_DATA3 (DISPLAY_MMIO_BASE(dev_priv) + 0x6401c) ++#define _DPA_AUX_CH_DATA4 (DISPLAY_MMIO_BASE(dev_priv) + 0x64020) ++#define _DPA_AUX_CH_DATA5 (DISPLAY_MMIO_BASE(dev_priv) + 0x64024) ++ ++#define _DPB_AUX_CH_CTL (DISPLAY_MMIO_BASE(dev_priv) + 0x64110) ++#define _DPB_AUX_CH_DATA1 (DISPLAY_MMIO_BASE(dev_priv) + 0x64114) ++#define _DPB_AUX_CH_DATA2 (DISPLAY_MMIO_BASE(dev_priv) + 0x64118) ++#define _DPB_AUX_CH_DATA3 (DISPLAY_MMIO_BASE(dev_priv) + 0x6411c) ++#define _DPB_AUX_CH_DATA4 (DISPLAY_MMIO_BASE(dev_priv) + 0x64120) ++#define _DPB_AUX_CH_DATA5 (DISPLAY_MMIO_BASE(dev_priv) + 0x64124) ++ ++#define _DPC_AUX_CH_CTL (DISPLAY_MMIO_BASE(dev_priv) + 0x64210) ++#define _DPC_AUX_CH_DATA1 (DISPLAY_MMIO_BASE(dev_priv) + 0x64214) ++#define _DPC_AUX_CH_DATA2 (DISPLAY_MMIO_BASE(dev_priv) + 0x64218) ++#define _DPC_AUX_CH_DATA3 (DISPLAY_MMIO_BASE(dev_priv) + 0x6421c) ++#define _DPC_AUX_CH_DATA4 (DISPLAY_MMIO_BASE(dev_priv) + 0x64220) ++#define _DPC_AUX_CH_DATA5 (DISPLAY_MMIO_BASE(dev_priv) + 0x64224) ++ ++#define _DPD_AUX_CH_CTL (DISPLAY_MMIO_BASE(dev_priv) + 0x64310) ++#define _DPD_AUX_CH_DATA1 (DISPLAY_MMIO_BASE(dev_priv) + 0x64314) ++#define _DPD_AUX_CH_DATA2 (DISPLAY_MMIO_BASE(dev_priv) + 0x64318) ++#define _DPD_AUX_CH_DATA3 (DISPLAY_MMIO_BASE(dev_priv) + 0x6431c) ++#define _DPD_AUX_CH_DATA4 (DISPLAY_MMIO_BASE(dev_priv) + 0x64320) ++#define _DPD_AUX_CH_DATA5 (DISPLAY_MMIO_BASE(dev_priv) + 0x64324) ++ ++#define _DPE_AUX_CH_CTL (DISPLAY_MMIO_BASE(dev_priv) + 0x64410) ++#define _DPE_AUX_CH_DATA1 (DISPLAY_MMIO_BASE(dev_priv) + 0x64414) ++#define _DPE_AUX_CH_DATA2 (DISPLAY_MMIO_BASE(dev_priv) + 0x64418) ++#define _DPE_AUX_CH_DATA3 (DISPLAY_MMIO_BASE(dev_priv) + 0x6441c) ++#define _DPE_AUX_CH_DATA4 (DISPLAY_MMIO_BASE(dev_priv) + 0x64420) ++#define _DPE_AUX_CH_DATA5 (DISPLAY_MMIO_BASE(dev_priv) + 0x64424) ++ ++#define _DPF_AUX_CH_CTL (DISPLAY_MMIO_BASE(dev_priv) + 0x64510) ++#define _DPF_AUX_CH_DATA1 (DISPLAY_MMIO_BASE(dev_priv) + 0x64514) ++#define _DPF_AUX_CH_DATA2 (DISPLAY_MMIO_BASE(dev_priv) + 0x64518) ++#define _DPF_AUX_CH_DATA3 (DISPLAY_MMIO_BASE(dev_priv) + 0x6451c) ++#define _DPF_AUX_CH_DATA4 (DISPLAY_MMIO_BASE(dev_priv) + 0x64520) ++#define _DPF_AUX_CH_DATA5 (DISPLAY_MMIO_BASE(dev_priv) + 0x64524) ++ ++#define DP_AUX_CH_CTL(aux_ch) _MMIO_PORT(aux_ch, _DPA_AUX_CH_CTL, _DPB_AUX_CH_CTL) ++#define DP_AUX_CH_DATA(aux_ch, i) _MMIO(_PORT(aux_ch, _DPA_AUX_CH_DATA1, _DPB_AUX_CH_DATA1) + (i) * 4) /* 5 registers */ ++ ++#define DP_AUX_CH_CTL_SEND_BUSY (1 << 31) ++#define DP_AUX_CH_CTL_DONE (1 << 30) ++#define DP_AUX_CH_CTL_INTERRUPT (1 << 29) ++#define DP_AUX_CH_CTL_TIME_OUT_ERROR (1 << 28) ++#define DP_AUX_CH_CTL_TIME_OUT_400us (0 << 26) ++#define DP_AUX_CH_CTL_TIME_OUT_600us (1 << 26) ++#define DP_AUX_CH_CTL_TIME_OUT_800us (2 << 26) ++#define DP_AUX_CH_CTL_TIME_OUT_MAX (3 << 26) /* Varies per platform */ ++#define DP_AUX_CH_CTL_TIME_OUT_MASK (3 << 26) ++#define DP_AUX_CH_CTL_RECEIVE_ERROR (1 << 25) ++#define DP_AUX_CH_CTL_MESSAGE_SIZE_MASK (0x1f << 20) ++#define DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT 20 ++#define DP_AUX_CH_CTL_PRECHARGE_2US_MASK (0xf << 16) ++#define DP_AUX_CH_CTL_PRECHARGE_2US_SHIFT 16 ++#define DP_AUX_CH_CTL_AUX_AKSV_SELECT (1 << 15) ++#define DP_AUX_CH_CTL_MANCHESTER_TEST (1 << 14) ++#define DP_AUX_CH_CTL_SYNC_TEST (1 << 13) ++#define DP_AUX_CH_CTL_DEGLITCH_TEST (1 << 12) ++#define DP_AUX_CH_CTL_PRECHARGE_TEST (1 << 11) ++#define DP_AUX_CH_CTL_BIT_CLOCK_2X_MASK (0x7ff) ++#define DP_AUX_CH_CTL_BIT_CLOCK_2X_SHIFT 0 ++#define DP_AUX_CH_CTL_PSR_DATA_AUX_REG_SKL (1 << 14) ++#define DP_AUX_CH_CTL_FS_DATA_AUX_REG_SKL (1 << 13) ++#define DP_AUX_CH_CTL_GTC_DATA_AUX_REG_SKL (1 << 12) ++#define DP_AUX_CH_CTL_TBT_IO (1 << 11) ++#define DP_AUX_CH_CTL_FW_SYNC_PULSE_SKL_MASK (0x1f << 5) ++#define DP_AUX_CH_CTL_FW_SYNC_PULSE_SKL(c) (((c) - 1) << 5) ++#define DP_AUX_CH_CTL_SYNC_PULSE_SKL(c) ((c) - 1) ++ ++/* ++ * Computing GMCH M and N values for the Display Port link ++ * ++ * GMCH M/N = dot clock * bytes per pixel / ls_clk * # of lanes ++ * ++ * ls_clk (we assume) is the DP link clock (1.62 or 2.7 GHz) ++ * ++ * The GMCH value is used internally ++ * ++ * bytes_per_pixel is the number of bytes coming out of the plane, ++ * which is after the LUTs, so we want the bytes for our color format. ++ * For our current usage, this is always 3, one byte for R, G and B. ++ */ ++#define _PIPEA_DATA_M_G4X 0x70050 ++#define _PIPEB_DATA_M_G4X 0x71050 ++ ++/* Transfer unit size for display port - 1, default is 0x3f (for TU size 64) */ ++#define TU_SIZE(x) (((x) - 1) << 25) /* default size 64 */ ++#define TU_SIZE_SHIFT 25 ++#define TU_SIZE_MASK (0x3f << 25) ++ ++#define DATA_LINK_M_N_MASK (0xffffff) ++#define DATA_LINK_N_MAX (0x800000) ++ ++#define _PIPEA_DATA_N_G4X 0x70054 ++#define _PIPEB_DATA_N_G4X 0x71054 ++#define PIPE_GMCH_DATA_N_MASK (0xffffff) ++ ++/* ++ * Computing Link M and N values for the Display Port link ++ * ++ * Link M / N = pixel_clock / ls_clk ++ * ++ * (the DP spec calls pixel_clock the 'strm_clk') ++ * ++ * The Link value is transmitted in the Main Stream ++ * Attributes and VB-ID. ++ */ ++ ++#define _PIPEA_LINK_M_G4X 0x70060 ++#define _PIPEB_LINK_M_G4X 0x71060 ++#define PIPEA_DP_LINK_M_MASK (0xffffff) ++ ++#define _PIPEA_LINK_N_G4X 0x70064 ++#define _PIPEB_LINK_N_G4X 0x71064 ++#define PIPEA_DP_LINK_N_MASK (0xffffff) ++ ++#define PIPE_DATA_M_G4X(pipe) _MMIO_PIPE(pipe, _PIPEA_DATA_M_G4X, _PIPEB_DATA_M_G4X) ++#define PIPE_DATA_N_G4X(pipe) _MMIO_PIPE(pipe, _PIPEA_DATA_N_G4X, _PIPEB_DATA_N_G4X) ++#define PIPE_LINK_M_G4X(pipe) _MMIO_PIPE(pipe, _PIPEA_LINK_M_G4X, _PIPEB_LINK_M_G4X) ++#define PIPE_LINK_N_G4X(pipe) _MMIO_PIPE(pipe, _PIPEA_LINK_N_G4X, _PIPEB_LINK_N_G4X) ++ ++/* Display & cursor control */ ++ ++/* Pipe A */ ++#define _PIPEADSL 0x70000 ++#define DSL_LINEMASK_GEN2 0x00000fff ++#define DSL_LINEMASK_GEN3 0x00001fff ++#define _PIPEACONF 0x70008 ++#define PIPECONF_ENABLE (1 << 31) ++#define PIPECONF_DISABLE 0 ++#define PIPECONF_DOUBLE_WIDE (1 << 30) ++#define I965_PIPECONF_ACTIVE (1 << 30) ++#define PIPECONF_DSI_PLL_LOCKED (1 << 29) /* vlv & pipe A only */ ++#define PIPECONF_FRAME_START_DELAY_MASK (3 << 27) ++#define PIPECONF_SINGLE_WIDE 0 ++#define PIPECONF_PIPE_UNLOCKED 0 ++#define PIPECONF_PIPE_LOCKED (1 << 25) ++#define PIPECONF_FORCE_BORDER (1 << 25) ++#define PIPECONF_GAMMA_MODE_MASK_I9XX (1 << 24) /* gmch */ ++#define PIPECONF_GAMMA_MODE_MASK_ILK (3 << 24) /* ilk-ivb */ ++#define PIPECONF_GAMMA_MODE_8BIT (0 << 24) /* gmch,ilk-ivb */ ++#define PIPECONF_GAMMA_MODE_10BIT (1 << 24) /* gmch,ilk-ivb */ ++#define PIPECONF_GAMMA_MODE_12BIT (2 << 24) /* ilk-ivb */ ++#define PIPECONF_GAMMA_MODE_SPLIT (3 << 24) /* ivb */ ++#define PIPECONF_GAMMA_MODE(x) ((x) << 24) /* pass in GAMMA_MODE_MODE_* */ ++#define PIPECONF_GAMMA_MODE_SHIFT 24 ++#define PIPECONF_INTERLACE_MASK (7 << 21) ++#define PIPECONF_INTERLACE_MASK_HSW (3 << 21) ++/* Note that pre-gen3 does not support interlaced display directly. Panel ++ * fitting must be disabled on pre-ilk for interlaced. */ ++#define PIPECONF_PROGRESSIVE (0 << 21) ++#define PIPECONF_INTERLACE_W_SYNC_SHIFT_PANEL (4 << 21) /* gen4 only */ ++#define PIPECONF_INTERLACE_W_SYNC_SHIFT (5 << 21) /* gen4 only */ ++#define PIPECONF_INTERLACE_W_FIELD_INDICATION (6 << 21) ++#define PIPECONF_INTERLACE_FIELD_0_ONLY (7 << 21) /* gen3 only */ ++/* Ironlake and later have a complete new set of values for interlaced. PFIT ++ * means panel fitter required, PF means progressive fetch, DBL means power ++ * saving pixel doubling. */ ++#define PIPECONF_PFIT_PF_INTERLACED_ILK (1 << 21) ++#define PIPECONF_INTERLACED_ILK (3 << 21) ++#define PIPECONF_INTERLACED_DBL_ILK (4 << 21) /* ilk/snb only */ ++#define PIPECONF_PFIT_PF_INTERLACED_DBL_ILK (5 << 21) /* ilk/snb only */ ++#define PIPECONF_INTERLACE_MODE_MASK (7 << 21) ++#define PIPECONF_EDP_RR_MODE_SWITCH (1 << 20) ++#define PIPECONF_CXSR_DOWNCLOCK (1 << 16) ++#define PIPECONF_EDP_RR_MODE_SWITCH_VLV (1 << 14) ++#define PIPECONF_COLOR_RANGE_SELECT (1 << 13) ++#define PIPECONF_BPC_MASK (0x7 << 5) ++#define PIPECONF_8BPC (0 << 5) ++#define PIPECONF_10BPC (1 << 5) ++#define PIPECONF_6BPC (2 << 5) ++#define PIPECONF_12BPC (3 << 5) ++#define PIPECONF_DITHER_EN (1 << 4) ++#define PIPECONF_DITHER_TYPE_MASK (0x0000000c) ++#define PIPECONF_DITHER_TYPE_SP (0 << 2) ++#define PIPECONF_DITHER_TYPE_ST1 (1 << 2) ++#define PIPECONF_DITHER_TYPE_ST2 (2 << 2) ++#define PIPECONF_DITHER_TYPE_TEMP (3 << 2) ++#define _PIPEASTAT 0x70024 ++#define PIPE_FIFO_UNDERRUN_STATUS (1UL << 31) ++#define SPRITE1_FLIP_DONE_INT_EN_VLV (1UL << 30) ++#define PIPE_CRC_ERROR_ENABLE (1UL << 29) ++#define PIPE_CRC_DONE_ENABLE (1UL << 28) ++#define PERF_COUNTER2_INTERRUPT_EN (1UL << 27) ++#define PIPE_GMBUS_EVENT_ENABLE (1UL << 27) ++#define PLANE_FLIP_DONE_INT_EN_VLV (1UL << 26) ++#define PIPE_HOTPLUG_INTERRUPT_ENABLE (1UL << 26) ++#define PIPE_VSYNC_INTERRUPT_ENABLE (1UL << 25) ++#define PIPE_DISPLAY_LINE_COMPARE_ENABLE (1UL << 24) ++#define PIPE_DPST_EVENT_ENABLE (1UL << 23) ++#define SPRITE0_FLIP_DONE_INT_EN_VLV (1UL << 22) ++#define PIPE_LEGACY_BLC_EVENT_ENABLE (1UL << 22) ++#define PIPE_ODD_FIELD_INTERRUPT_ENABLE (1UL << 21) ++#define PIPE_EVEN_FIELD_INTERRUPT_ENABLE (1UL << 20) ++#define PIPE_B_PSR_INTERRUPT_ENABLE_VLV (1UL << 19) ++#define PERF_COUNTER_INTERRUPT_EN (1UL << 19) ++#define PIPE_HOTPLUG_TV_INTERRUPT_ENABLE (1UL << 18) /* pre-965 */ ++#define PIPE_START_VBLANK_INTERRUPT_ENABLE (1UL << 18) /* 965 or later */ ++#define PIPE_FRAMESTART_INTERRUPT_ENABLE (1UL << 17) ++#define PIPE_VBLANK_INTERRUPT_ENABLE (1UL << 17) ++#define PIPEA_HBLANK_INT_EN_VLV (1UL << 16) ++#define PIPE_OVERLAY_UPDATED_ENABLE (1UL << 16) ++#define SPRITE1_FLIP_DONE_INT_STATUS_VLV (1UL << 15) ++#define SPRITE0_FLIP_DONE_INT_STATUS_VLV (1UL << 14) ++#define PIPE_CRC_ERROR_INTERRUPT_STATUS (1UL << 13) ++#define PIPE_CRC_DONE_INTERRUPT_STATUS (1UL << 12) ++#define PERF_COUNTER2_INTERRUPT_STATUS (1UL << 11) ++#define PIPE_GMBUS_INTERRUPT_STATUS (1UL << 11) ++#define PLANE_FLIP_DONE_INT_STATUS_VLV (1UL << 10) ++#define PIPE_HOTPLUG_INTERRUPT_STATUS (1UL << 10) ++#define PIPE_VSYNC_INTERRUPT_STATUS (1UL << 9) ++#define PIPE_DISPLAY_LINE_COMPARE_STATUS (1UL << 8) ++#define PIPE_DPST_EVENT_STATUS (1UL << 7) ++#define PIPE_A_PSR_STATUS_VLV (1UL << 6) ++#define PIPE_LEGACY_BLC_EVENT_STATUS (1UL << 6) ++#define PIPE_ODD_FIELD_INTERRUPT_STATUS (1UL << 5) ++#define PIPE_EVEN_FIELD_INTERRUPT_STATUS (1UL << 4) ++#define PIPE_B_PSR_STATUS_VLV (1UL << 3) ++#define PERF_COUNTER_INTERRUPT_STATUS (1UL << 3) ++#define PIPE_HOTPLUG_TV_INTERRUPT_STATUS (1UL << 2) /* pre-965 */ ++#define PIPE_START_VBLANK_INTERRUPT_STATUS (1UL << 2) /* 965 or later */ ++#define PIPE_FRAMESTART_INTERRUPT_STATUS (1UL << 1) ++#define PIPE_VBLANK_INTERRUPT_STATUS (1UL << 1) ++#define PIPE_HBLANK_INT_STATUS (1UL << 0) ++#define PIPE_OVERLAY_UPDATED_STATUS (1UL << 0) ++ ++#define PIPESTAT_INT_ENABLE_MASK 0x7fff0000 ++#define PIPESTAT_INT_STATUS_MASK 0x0000ffff ++ ++#define PIPE_A_OFFSET 0x70000 ++#define PIPE_B_OFFSET 0x71000 ++#define PIPE_C_OFFSET 0x72000 ++#define CHV_PIPE_C_OFFSET 0x74000 ++/* ++ * There's actually no pipe EDP. Some pipe registers have ++ * simply shifted from the pipe to the transcoder, while ++ * keeping their original offset. Thus we need PIPE_EDP_OFFSET ++ * to access such registers in transcoder EDP. ++ */ ++#define PIPE_EDP_OFFSET 0x7f000 ++ ++/* ICL DSI 0 and 1 */ ++#define PIPE_DSI0_OFFSET 0x7b000 ++#define PIPE_DSI1_OFFSET 0x7b800 ++ ++#define PIPECONF(pipe) _MMIO_PIPE2(pipe, _PIPEACONF) ++#define PIPEDSL(pipe) _MMIO_PIPE2(pipe, _PIPEADSL) ++#define PIPEFRAME(pipe) _MMIO_PIPE2(pipe, _PIPEAFRAMEHIGH) ++#define PIPEFRAMEPIXEL(pipe) _MMIO_PIPE2(pipe, _PIPEAFRAMEPIXEL) ++#define PIPESTAT(pipe) _MMIO_PIPE2(pipe, _PIPEASTAT) ++ ++#define _PIPEAGCMAX 0x70010 ++#define _PIPEBGCMAX 0x71010 ++#define PIPEGCMAX(pipe, i) _MMIO_PIPE2(pipe, _PIPEAGCMAX + (i) * 4) ++ ++#define _PIPE_MISC_A 0x70030 ++#define _PIPE_MISC_B 0x71030 ++#define PIPEMISC_YUV420_ENABLE (1 << 27) ++#define PIPEMISC_YUV420_MODE_FULL_BLEND (1 << 26) ++#define PIPEMISC_OUTPUT_COLORSPACE_YUV (1 << 11) ++#define PIPEMISC_DITHER_BPC_MASK (7 << 5) ++#define PIPEMISC_DITHER_8_BPC (0 << 5) ++#define PIPEMISC_DITHER_10_BPC (1 << 5) ++#define PIPEMISC_DITHER_6_BPC (2 << 5) ++#define PIPEMISC_DITHER_12_BPC (3 << 5) ++#define PIPEMISC_DITHER_ENABLE (1 << 4) ++#define PIPEMISC_DITHER_TYPE_MASK (3 << 2) ++#define PIPEMISC_DITHER_TYPE_SP (0 << 2) ++#define PIPEMISC(pipe) _MMIO_PIPE2(pipe, _PIPE_MISC_A) ++ ++/* Skylake+ pipe bottom (background) color */ ++#define _SKL_BOTTOM_COLOR_A 0x70034 ++#define SKL_BOTTOM_COLOR_GAMMA_ENABLE (1 << 31) ++#define SKL_BOTTOM_COLOR_CSC_ENABLE (1 << 30) ++#define SKL_BOTTOM_COLOR(pipe) _MMIO_PIPE2(pipe, _SKL_BOTTOM_COLOR_A) ++ ++#define VLV_DPFLIPSTAT _MMIO(VLV_DISPLAY_BASE + 0x70028) ++#define PIPEB_LINE_COMPARE_INT_EN (1 << 29) ++#define PIPEB_HLINE_INT_EN (1 << 28) ++#define PIPEB_VBLANK_INT_EN (1 << 27) ++#define SPRITED_FLIP_DONE_INT_EN (1 << 26) ++#define SPRITEC_FLIP_DONE_INT_EN (1 << 25) ++#define PLANEB_FLIP_DONE_INT_EN (1 << 24) ++#define PIPE_PSR_INT_EN (1 << 22) ++#define PIPEA_LINE_COMPARE_INT_EN (1 << 21) ++#define PIPEA_HLINE_INT_EN (1 << 20) ++#define PIPEA_VBLANK_INT_EN (1 << 19) ++#define SPRITEB_FLIP_DONE_INT_EN (1 << 18) ++#define SPRITEA_FLIP_DONE_INT_EN (1 << 17) ++#define PLANEA_FLIPDONE_INT_EN (1 << 16) ++#define PIPEC_LINE_COMPARE_INT_EN (1 << 13) ++#define PIPEC_HLINE_INT_EN (1 << 12) ++#define PIPEC_VBLANK_INT_EN (1 << 11) ++#define SPRITEF_FLIPDONE_INT_EN (1 << 10) ++#define SPRITEE_FLIPDONE_INT_EN (1 << 9) ++#define PLANEC_FLIPDONE_INT_EN (1 << 8) ++ ++#define DPINVGTT _MMIO(VLV_DISPLAY_BASE + 0x7002c) /* VLV/CHV only */ ++#define SPRITEF_INVALID_GTT_INT_EN (1 << 27) ++#define SPRITEE_INVALID_GTT_INT_EN (1 << 26) ++#define PLANEC_INVALID_GTT_INT_EN (1 << 25) ++#define CURSORC_INVALID_GTT_INT_EN (1 << 24) ++#define CURSORB_INVALID_GTT_INT_EN (1 << 23) ++#define CURSORA_INVALID_GTT_INT_EN (1 << 22) ++#define SPRITED_INVALID_GTT_INT_EN (1 << 21) ++#define SPRITEC_INVALID_GTT_INT_EN (1 << 20) ++#define PLANEB_INVALID_GTT_INT_EN (1 << 19) ++#define SPRITEB_INVALID_GTT_INT_EN (1 << 18) ++#define SPRITEA_INVALID_GTT_INT_EN (1 << 17) ++#define PLANEA_INVALID_GTT_INT_EN (1 << 16) ++#define DPINVGTT_EN_MASK 0xff0000 ++#define DPINVGTT_EN_MASK_CHV 0xfff0000 ++#define SPRITEF_INVALID_GTT_STATUS (1 << 11) ++#define SPRITEE_INVALID_GTT_STATUS (1 << 10) ++#define PLANEC_INVALID_GTT_STATUS (1 << 9) ++#define CURSORC_INVALID_GTT_STATUS (1 << 8) ++#define CURSORB_INVALID_GTT_STATUS (1 << 7) ++#define CURSORA_INVALID_GTT_STATUS (1 << 6) ++#define SPRITED_INVALID_GTT_STATUS (1 << 5) ++#define SPRITEC_INVALID_GTT_STATUS (1 << 4) ++#define PLANEB_INVALID_GTT_STATUS (1 << 3) ++#define SPRITEB_INVALID_GTT_STATUS (1 << 2) ++#define SPRITEA_INVALID_GTT_STATUS (1 << 1) ++#define PLANEA_INVALID_GTT_STATUS (1 << 0) ++#define DPINVGTT_STATUS_MASK 0xff ++#define DPINVGTT_STATUS_MASK_CHV 0xfff ++ ++#define DSPARB _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x70030) ++#define DSPARB_CSTART_MASK (0x7f << 7) ++#define DSPARB_CSTART_SHIFT 7 ++#define DSPARB_BSTART_MASK (0x7f) ++#define DSPARB_BSTART_SHIFT 0 ++#define DSPARB_BEND_SHIFT 9 /* on 855 */ ++#define DSPARB_AEND_SHIFT 0 ++#define DSPARB_SPRITEA_SHIFT_VLV 0 ++#define DSPARB_SPRITEA_MASK_VLV (0xff << 0) ++#define DSPARB_SPRITEB_SHIFT_VLV 8 ++#define DSPARB_SPRITEB_MASK_VLV (0xff << 8) ++#define DSPARB_SPRITEC_SHIFT_VLV 16 ++#define DSPARB_SPRITEC_MASK_VLV (0xff << 16) ++#define DSPARB_SPRITED_SHIFT_VLV 24 ++#define DSPARB_SPRITED_MASK_VLV (0xff << 24) ++#define DSPARB2 _MMIO(VLV_DISPLAY_BASE + 0x70060) /* vlv/chv */ ++#define DSPARB_SPRITEA_HI_SHIFT_VLV 0 ++#define DSPARB_SPRITEA_HI_MASK_VLV (0x1 << 0) ++#define DSPARB_SPRITEB_HI_SHIFT_VLV 4 ++#define DSPARB_SPRITEB_HI_MASK_VLV (0x1 << 4) ++#define DSPARB_SPRITEC_HI_SHIFT_VLV 8 ++#define DSPARB_SPRITEC_HI_MASK_VLV (0x1 << 8) ++#define DSPARB_SPRITED_HI_SHIFT_VLV 12 ++#define DSPARB_SPRITED_HI_MASK_VLV (0x1 << 12) ++#define DSPARB_SPRITEE_HI_SHIFT_VLV 16 ++#define DSPARB_SPRITEE_HI_MASK_VLV (0x1 << 16) ++#define DSPARB_SPRITEF_HI_SHIFT_VLV 20 ++#define DSPARB_SPRITEF_HI_MASK_VLV (0x1 << 20) ++#define DSPARB3 _MMIO(VLV_DISPLAY_BASE + 0x7006c) /* chv */ ++#define DSPARB_SPRITEE_SHIFT_VLV 0 ++#define DSPARB_SPRITEE_MASK_VLV (0xff << 0) ++#define DSPARB_SPRITEF_SHIFT_VLV 8 ++#define DSPARB_SPRITEF_MASK_VLV (0xff << 8) ++ ++/* pnv/gen4/g4x/vlv/chv */ ++#define DSPFW1 _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x70034) ++#define DSPFW_SR_SHIFT 23 ++#define DSPFW_SR_MASK (0x1ff << 23) ++#define DSPFW_CURSORB_SHIFT 16 ++#define DSPFW_CURSORB_MASK (0x3f << 16) ++#define DSPFW_PLANEB_SHIFT 8 ++#define DSPFW_PLANEB_MASK (0x7f << 8) ++#define DSPFW_PLANEB_MASK_VLV (0xff << 8) /* vlv/chv */ ++#define DSPFW_PLANEA_SHIFT 0 ++#define DSPFW_PLANEA_MASK (0x7f << 0) ++#define DSPFW_PLANEA_MASK_VLV (0xff << 0) /* vlv/chv */ ++#define DSPFW2 _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x70038) ++#define DSPFW_FBC_SR_EN (1 << 31) /* g4x */ ++#define DSPFW_FBC_SR_SHIFT 28 ++#define DSPFW_FBC_SR_MASK (0x7 << 28) /* g4x */ ++#define DSPFW_FBC_HPLL_SR_SHIFT 24 ++#define DSPFW_FBC_HPLL_SR_MASK (0xf << 24) /* g4x */ ++#define DSPFW_SPRITEB_SHIFT (16) ++#define DSPFW_SPRITEB_MASK (0x7f << 16) /* g4x */ ++#define DSPFW_SPRITEB_MASK_VLV (0xff << 16) /* vlv/chv */ ++#define DSPFW_CURSORA_SHIFT 8 ++#define DSPFW_CURSORA_MASK (0x3f << 8) ++#define DSPFW_PLANEC_OLD_SHIFT 0 ++#define DSPFW_PLANEC_OLD_MASK (0x7f << 0) /* pre-gen4 sprite C */ ++#define DSPFW_SPRITEA_SHIFT 0 ++#define DSPFW_SPRITEA_MASK (0x7f << 0) /* g4x */ ++#define DSPFW_SPRITEA_MASK_VLV (0xff << 0) /* vlv/chv */ ++#define DSPFW3 _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x7003c) ++#define DSPFW_HPLL_SR_EN (1 << 31) ++#define PINEVIEW_SELF_REFRESH_EN (1 << 30) ++#define DSPFW_CURSOR_SR_SHIFT 24 ++#define DSPFW_CURSOR_SR_MASK (0x3f << 24) ++#define DSPFW_HPLL_CURSOR_SHIFT 16 ++#define DSPFW_HPLL_CURSOR_MASK (0x3f << 16) ++#define DSPFW_HPLL_SR_SHIFT 0 ++#define DSPFW_HPLL_SR_MASK (0x1ff << 0) ++ ++/* vlv/chv */ ++#define DSPFW4 _MMIO(VLV_DISPLAY_BASE + 0x70070) ++#define DSPFW_SPRITEB_WM1_SHIFT 16 ++#define DSPFW_SPRITEB_WM1_MASK (0xff << 16) ++#define DSPFW_CURSORA_WM1_SHIFT 8 ++#define DSPFW_CURSORA_WM1_MASK (0x3f << 8) ++#define DSPFW_SPRITEA_WM1_SHIFT 0 ++#define DSPFW_SPRITEA_WM1_MASK (0xff << 0) ++#define DSPFW5 _MMIO(VLV_DISPLAY_BASE + 0x70074) ++#define DSPFW_PLANEB_WM1_SHIFT 24 ++#define DSPFW_PLANEB_WM1_MASK (0xff << 24) ++#define DSPFW_PLANEA_WM1_SHIFT 16 ++#define DSPFW_PLANEA_WM1_MASK (0xff << 16) ++#define DSPFW_CURSORB_WM1_SHIFT 8 ++#define DSPFW_CURSORB_WM1_MASK (0x3f << 8) ++#define DSPFW_CURSOR_SR_WM1_SHIFT 0 ++#define DSPFW_CURSOR_SR_WM1_MASK (0x3f << 0) ++#define DSPFW6 _MMIO(VLV_DISPLAY_BASE + 0x70078) ++#define DSPFW_SR_WM1_SHIFT 0 ++#define DSPFW_SR_WM1_MASK (0x1ff << 0) ++#define DSPFW7 _MMIO(VLV_DISPLAY_BASE + 0x7007c) ++#define DSPFW7_CHV _MMIO(VLV_DISPLAY_BASE + 0x700b4) /* wtf #1? */ ++#define DSPFW_SPRITED_WM1_SHIFT 24 ++#define DSPFW_SPRITED_WM1_MASK (0xff << 24) ++#define DSPFW_SPRITED_SHIFT 16 ++#define DSPFW_SPRITED_MASK_VLV (0xff << 16) ++#define DSPFW_SPRITEC_WM1_SHIFT 8 ++#define DSPFW_SPRITEC_WM1_MASK (0xff << 8) ++#define DSPFW_SPRITEC_SHIFT 0 ++#define DSPFW_SPRITEC_MASK_VLV (0xff << 0) ++#define DSPFW8_CHV _MMIO(VLV_DISPLAY_BASE + 0x700b8) ++#define DSPFW_SPRITEF_WM1_SHIFT 24 ++#define DSPFW_SPRITEF_WM1_MASK (0xff << 24) ++#define DSPFW_SPRITEF_SHIFT 16 ++#define DSPFW_SPRITEF_MASK_VLV (0xff << 16) ++#define DSPFW_SPRITEE_WM1_SHIFT 8 ++#define DSPFW_SPRITEE_WM1_MASK (0xff << 8) ++#define DSPFW_SPRITEE_SHIFT 0 ++#define DSPFW_SPRITEE_MASK_VLV (0xff << 0) ++#define DSPFW9_CHV _MMIO(VLV_DISPLAY_BASE + 0x7007c) /* wtf #2? */ ++#define DSPFW_PLANEC_WM1_SHIFT 24 ++#define DSPFW_PLANEC_WM1_MASK (0xff << 24) ++#define DSPFW_PLANEC_SHIFT 16 ++#define DSPFW_PLANEC_MASK_VLV (0xff << 16) ++#define DSPFW_CURSORC_WM1_SHIFT 8 ++#define DSPFW_CURSORC_WM1_MASK (0x3f << 16) ++#define DSPFW_CURSORC_SHIFT 0 ++#define DSPFW_CURSORC_MASK (0x3f << 0) ++ ++/* vlv/chv high order bits */ ++#define DSPHOWM _MMIO(VLV_DISPLAY_BASE + 0x70064) ++#define DSPFW_SR_HI_SHIFT 24 ++#define DSPFW_SR_HI_MASK (3 << 24) /* 2 bits for chv, 1 for vlv */ ++#define DSPFW_SPRITEF_HI_SHIFT 23 ++#define DSPFW_SPRITEF_HI_MASK (1 << 23) ++#define DSPFW_SPRITEE_HI_SHIFT 22 ++#define DSPFW_SPRITEE_HI_MASK (1 << 22) ++#define DSPFW_PLANEC_HI_SHIFT 21 ++#define DSPFW_PLANEC_HI_MASK (1 << 21) ++#define DSPFW_SPRITED_HI_SHIFT 20 ++#define DSPFW_SPRITED_HI_MASK (1 << 20) ++#define DSPFW_SPRITEC_HI_SHIFT 16 ++#define DSPFW_SPRITEC_HI_MASK (1 << 16) ++#define DSPFW_PLANEB_HI_SHIFT 12 ++#define DSPFW_PLANEB_HI_MASK (1 << 12) ++#define DSPFW_SPRITEB_HI_SHIFT 8 ++#define DSPFW_SPRITEB_HI_MASK (1 << 8) ++#define DSPFW_SPRITEA_HI_SHIFT 4 ++#define DSPFW_SPRITEA_HI_MASK (1 << 4) ++#define DSPFW_PLANEA_HI_SHIFT 0 ++#define DSPFW_PLANEA_HI_MASK (1 << 0) ++#define DSPHOWM1 _MMIO(VLV_DISPLAY_BASE + 0x70068) ++#define DSPFW_SR_WM1_HI_SHIFT 24 ++#define DSPFW_SR_WM1_HI_MASK (3 << 24) /* 2 bits for chv, 1 for vlv */ ++#define DSPFW_SPRITEF_WM1_HI_SHIFT 23 ++#define DSPFW_SPRITEF_WM1_HI_MASK (1 << 23) ++#define DSPFW_SPRITEE_WM1_HI_SHIFT 22 ++#define DSPFW_SPRITEE_WM1_HI_MASK (1 << 22) ++#define DSPFW_PLANEC_WM1_HI_SHIFT 21 ++#define DSPFW_PLANEC_WM1_HI_MASK (1 << 21) ++#define DSPFW_SPRITED_WM1_HI_SHIFT 20 ++#define DSPFW_SPRITED_WM1_HI_MASK (1 << 20) ++#define DSPFW_SPRITEC_WM1_HI_SHIFT 16 ++#define DSPFW_SPRITEC_WM1_HI_MASK (1 << 16) ++#define DSPFW_PLANEB_WM1_HI_SHIFT 12 ++#define DSPFW_PLANEB_WM1_HI_MASK (1 << 12) ++#define DSPFW_SPRITEB_WM1_HI_SHIFT 8 ++#define DSPFW_SPRITEB_WM1_HI_MASK (1 << 8) ++#define DSPFW_SPRITEA_WM1_HI_SHIFT 4 ++#define DSPFW_SPRITEA_WM1_HI_MASK (1 << 4) ++#define DSPFW_PLANEA_WM1_HI_SHIFT 0 ++#define DSPFW_PLANEA_WM1_HI_MASK (1 << 0) ++ ++/* drain latency register values*/ ++#define VLV_DDL(pipe) _MMIO(VLV_DISPLAY_BASE + 0x70050 + 4 * (pipe)) ++#define DDL_CURSOR_SHIFT 24 ++#define DDL_SPRITE_SHIFT(sprite) (8 + 8 * (sprite)) ++#define DDL_PLANE_SHIFT 0 ++#define DDL_PRECISION_HIGH (1 << 7) ++#define DDL_PRECISION_LOW (0 << 7) ++#define DRAIN_LATENCY_MASK 0x7f ++ ++#define CBR1_VLV _MMIO(VLV_DISPLAY_BASE + 0x70400) ++#define CBR_PND_DEADLINE_DISABLE (1 << 31) ++#define CBR_PWM_CLOCK_MUX_SELECT (1 << 30) ++ ++#define CBR4_VLV _MMIO(VLV_DISPLAY_BASE + 0x70450) ++#define CBR_DPLLBMD_PIPE(pipe) (1 << (7 + (pipe) * 11)) /* pipes B and C */ ++ ++/* FIFO watermark sizes etc */ ++#define G4X_FIFO_LINE_SIZE 64 ++#define I915_FIFO_LINE_SIZE 64 ++#define I830_FIFO_LINE_SIZE 32 ++ ++#define VALLEYVIEW_FIFO_SIZE 255 ++#define G4X_FIFO_SIZE 127 ++#define I965_FIFO_SIZE 512 ++#define I945_FIFO_SIZE 127 ++#define I915_FIFO_SIZE 95 ++#define I855GM_FIFO_SIZE 127 /* In cachelines */ ++#define I830_FIFO_SIZE 95 ++ ++#define VALLEYVIEW_MAX_WM 0xff ++#define G4X_MAX_WM 0x3f ++#define I915_MAX_WM 0x3f ++ ++#define PINEVIEW_DISPLAY_FIFO 512 /* in 64byte unit */ ++#define PINEVIEW_FIFO_LINE_SIZE 64 ++#define PINEVIEW_MAX_WM 0x1ff ++#define PINEVIEW_DFT_WM 0x3f ++#define PINEVIEW_DFT_HPLLOFF_WM 0 ++#define PINEVIEW_GUARD_WM 10 ++#define PINEVIEW_CURSOR_FIFO 64 ++#define PINEVIEW_CURSOR_MAX_WM 0x3f ++#define PINEVIEW_CURSOR_DFT_WM 0 ++#define PINEVIEW_CURSOR_GUARD_WM 5 ++ ++#define VALLEYVIEW_CURSOR_MAX_WM 64 ++#define I965_CURSOR_FIFO 64 ++#define I965_CURSOR_MAX_WM 32 ++#define I965_CURSOR_DFT_WM 8 ++ ++/* Watermark register definitions for SKL */ ++#define _CUR_WM_A_0 0x70140 ++#define _CUR_WM_B_0 0x71140 ++#define _PLANE_WM_1_A_0 0x70240 ++#define _PLANE_WM_1_B_0 0x71240 ++#define _PLANE_WM_2_A_0 0x70340 ++#define _PLANE_WM_2_B_0 0x71340 ++#define _PLANE_WM_TRANS_1_A_0 0x70268 ++#define _PLANE_WM_TRANS_1_B_0 0x71268 ++#define _PLANE_WM_TRANS_2_A_0 0x70368 ++#define _PLANE_WM_TRANS_2_B_0 0x71368 ++#define _CUR_WM_TRANS_A_0 0x70168 ++#define _CUR_WM_TRANS_B_0 0x71168 ++#define PLANE_WM_EN (1 << 31) ++#define PLANE_WM_IGNORE_LINES (1 << 30) ++#define PLANE_WM_LINES_SHIFT 14 ++#define PLANE_WM_LINES_MASK 0x1f ++#define PLANE_WM_BLOCKS_MASK 0x7ff /* skl+: 10 bits, icl+ 11 bits */ ++ ++#define _CUR_WM_0(pipe) _PIPE(pipe, _CUR_WM_A_0, _CUR_WM_B_0) ++#define CUR_WM(pipe, level) _MMIO(_CUR_WM_0(pipe) + ((4) * (level))) ++#define CUR_WM_TRANS(pipe) _MMIO_PIPE(pipe, _CUR_WM_TRANS_A_0, _CUR_WM_TRANS_B_0) ++ ++#define _PLANE_WM_1(pipe) _PIPE(pipe, _PLANE_WM_1_A_0, _PLANE_WM_1_B_0) ++#define _PLANE_WM_2(pipe) _PIPE(pipe, _PLANE_WM_2_A_0, _PLANE_WM_2_B_0) ++#define _PLANE_WM_BASE(pipe, plane) \ ++ _PLANE(plane, _PLANE_WM_1(pipe), _PLANE_WM_2(pipe)) ++#define PLANE_WM(pipe, plane, level) \ ++ _MMIO(_PLANE_WM_BASE(pipe, plane) + ((4) * (level))) ++#define _PLANE_WM_TRANS_1(pipe) \ ++ _PIPE(pipe, _PLANE_WM_TRANS_1_A_0, _PLANE_WM_TRANS_1_B_0) ++#define _PLANE_WM_TRANS_2(pipe) \ ++ _PIPE(pipe, _PLANE_WM_TRANS_2_A_0, _PLANE_WM_TRANS_2_B_0) ++#define PLANE_WM_TRANS(pipe, plane) \ ++ _MMIO(_PLANE(plane, _PLANE_WM_TRANS_1(pipe), _PLANE_WM_TRANS_2(pipe))) ++ ++/* define the Watermark register on Ironlake */ ++#define WM0_PIPEA_ILK _MMIO(0x45100) ++#define WM0_PIPE_PLANE_MASK (0xffff << 16) ++#define WM0_PIPE_PLANE_SHIFT 16 ++#define WM0_PIPE_SPRITE_MASK (0xff << 8) ++#define WM0_PIPE_SPRITE_SHIFT 8 ++#define WM0_PIPE_CURSOR_MASK (0xff) ++ ++#define WM0_PIPEB_ILK _MMIO(0x45104) ++#define WM0_PIPEC_IVB _MMIO(0x45200) ++#define WM1_LP_ILK _MMIO(0x45108) ++#define WM1_LP_SR_EN (1 << 31) ++#define WM1_LP_LATENCY_SHIFT 24 ++#define WM1_LP_LATENCY_MASK (0x7f << 24) ++#define WM1_LP_FBC_MASK (0xf << 20) ++#define WM1_LP_FBC_SHIFT 20 ++#define WM1_LP_FBC_SHIFT_BDW 19 ++#define WM1_LP_SR_MASK (0x7ff << 8) ++#define WM1_LP_SR_SHIFT 8 ++#define WM1_LP_CURSOR_MASK (0xff) ++#define WM2_LP_ILK _MMIO(0x4510c) ++#define WM2_LP_EN (1 << 31) ++#define WM3_LP_ILK _MMIO(0x45110) ++#define WM3_LP_EN (1 << 31) ++#define WM1S_LP_ILK _MMIO(0x45120) ++#define WM2S_LP_IVB _MMIO(0x45124) ++#define WM3S_LP_IVB _MMIO(0x45128) ++#define WM1S_LP_EN (1 << 31) ++ ++#define HSW_WM_LP_VAL(lat, fbc, pri, cur) \ ++ (WM3_LP_EN | ((lat) << WM1_LP_LATENCY_SHIFT) | \ ++ ((fbc) << WM1_LP_FBC_SHIFT) | ((pri) << WM1_LP_SR_SHIFT) | (cur)) ++ ++/* Memory latency timer register */ ++#define MLTR_ILK _MMIO(0x11222) ++#define MLTR_WM1_SHIFT 0 ++#define MLTR_WM2_SHIFT 8 ++/* the unit of memory self-refresh latency time is 0.5us */ ++#define ILK_SRLT_MASK 0x3f ++ ++ ++/* the address where we get all kinds of latency value */ ++#define SSKPD _MMIO(0x5d10) ++#define SSKPD_WM_MASK 0x3f ++#define SSKPD_WM0_SHIFT 0 ++#define SSKPD_WM1_SHIFT 8 ++#define SSKPD_WM2_SHIFT 16 ++#define SSKPD_WM3_SHIFT 24 ++ ++/* ++ * The two pipe frame counter registers are not synchronized, so ++ * reading a stable value is somewhat tricky. The following code ++ * should work: ++ * ++ * do { ++ * high1 = ((INREG(PIPEAFRAMEHIGH) & PIPE_FRAME_HIGH_MASK) >> ++ * PIPE_FRAME_HIGH_SHIFT; ++ * low1 = ((INREG(PIPEAFRAMEPIXEL) & PIPE_FRAME_LOW_MASK) >> ++ * PIPE_FRAME_LOW_SHIFT); ++ * high2 = ((INREG(PIPEAFRAMEHIGH) & PIPE_FRAME_HIGH_MASK) >> ++ * PIPE_FRAME_HIGH_SHIFT); ++ * } while (high1 != high2); ++ * frame = (high1 << 8) | low1; ++ */ ++#define _PIPEAFRAMEHIGH 0x70040 ++#define PIPE_FRAME_HIGH_MASK 0x0000ffff ++#define PIPE_FRAME_HIGH_SHIFT 0 ++#define _PIPEAFRAMEPIXEL 0x70044 ++#define PIPE_FRAME_LOW_MASK 0xff000000 ++#define PIPE_FRAME_LOW_SHIFT 24 ++#define PIPE_PIXEL_MASK 0x00ffffff ++#define PIPE_PIXEL_SHIFT 0 ++/* GM45+ just has to be different */ ++#define _PIPEA_FRMCOUNT_G4X 0x70040 ++#define _PIPEA_FLIPCOUNT_G4X 0x70044 ++#define PIPE_FRMCOUNT_G4X(pipe) _MMIO_PIPE2(pipe, _PIPEA_FRMCOUNT_G4X) ++#define PIPE_FLIPCOUNT_G4X(pipe) _MMIO_PIPE2(pipe, _PIPEA_FLIPCOUNT_G4X) ++ ++/* Cursor A & B regs */ ++#define _CURACNTR 0x70080 ++/* Old style CUR*CNTR flags (desktop 8xx) */ ++#define CURSOR_ENABLE 0x80000000 ++#define CURSOR_GAMMA_ENABLE 0x40000000 ++#define CURSOR_STRIDE_SHIFT 28 ++#define CURSOR_STRIDE(x) ((ffs(x) - 9) << CURSOR_STRIDE_SHIFT) /* 256,512,1k,2k */ ++#define CURSOR_FORMAT_SHIFT 24 ++#define CURSOR_FORMAT_MASK (0x07 << CURSOR_FORMAT_SHIFT) ++#define CURSOR_FORMAT_2C (0x00 << CURSOR_FORMAT_SHIFT) ++#define CURSOR_FORMAT_3C (0x01 << CURSOR_FORMAT_SHIFT) ++#define CURSOR_FORMAT_4C (0x02 << CURSOR_FORMAT_SHIFT) ++#define CURSOR_FORMAT_ARGB (0x04 << CURSOR_FORMAT_SHIFT) ++#define CURSOR_FORMAT_XRGB (0x05 << CURSOR_FORMAT_SHIFT) ++/* New style CUR*CNTR flags */ ++#define MCURSOR_MODE 0x27 ++#define MCURSOR_MODE_DISABLE 0x00 ++#define MCURSOR_MODE_128_32B_AX 0x02 ++#define MCURSOR_MODE_256_32B_AX 0x03 ++#define MCURSOR_MODE_64_32B_AX 0x07 ++#define MCURSOR_MODE_128_ARGB_AX ((1 << 5) | MCURSOR_MODE_128_32B_AX) ++#define MCURSOR_MODE_256_ARGB_AX ((1 << 5) | MCURSOR_MODE_256_32B_AX) ++#define MCURSOR_MODE_64_ARGB_AX ((1 << 5) | MCURSOR_MODE_64_32B_AX) ++#define MCURSOR_PIPE_SELECT_MASK (0x3 << 28) ++#define MCURSOR_PIPE_SELECT_SHIFT 28 ++#define MCURSOR_PIPE_SELECT(pipe) ((pipe) << 28) ++#define MCURSOR_GAMMA_ENABLE (1 << 26) ++#define MCURSOR_PIPE_CSC_ENABLE (1 << 24) /* ilk+ */ ++#define MCURSOR_ROTATE_180 (1 << 15) ++#define MCURSOR_TRICKLE_FEED_DISABLE (1 << 14) ++#define _CURABASE 0x70084 ++#define _CURAPOS 0x70088 ++#define CURSOR_POS_MASK 0x007FF ++#define CURSOR_POS_SIGN 0x8000 ++#define CURSOR_X_SHIFT 0 ++#define CURSOR_Y_SHIFT 16 ++#define CURSIZE _MMIO(0x700a0) /* 845/865 */ ++#define _CUR_FBC_CTL_A 0x700a0 /* ivb+ */ ++#define CUR_FBC_CTL_EN (1 << 31) ++#define _CURASURFLIVE 0x700ac /* g4x+ */ ++#define _CURBCNTR 0x700c0 ++#define _CURBBASE 0x700c4 ++#define _CURBPOS 0x700c8 ++ ++#define _CURBCNTR_IVB 0x71080 ++#define _CURBBASE_IVB 0x71084 ++#define _CURBPOS_IVB 0x71088 ++ ++#define CURCNTR(pipe) _CURSOR2(pipe, _CURACNTR) ++#define CURBASE(pipe) _CURSOR2(pipe, _CURABASE) ++#define CURPOS(pipe) _CURSOR2(pipe, _CURAPOS) ++#define CUR_FBC_CTL(pipe) _CURSOR2(pipe, _CUR_FBC_CTL_A) ++#define CURSURFLIVE(pipe) _CURSOR2(pipe, _CURASURFLIVE) ++ ++#define CURSOR_A_OFFSET 0x70080 ++#define CURSOR_B_OFFSET 0x700c0 ++#define CHV_CURSOR_C_OFFSET 0x700e0 ++#define IVB_CURSOR_B_OFFSET 0x71080 ++#define IVB_CURSOR_C_OFFSET 0x72080 ++ ++/* Display A control */ ++#define _DSPACNTR 0x70180 ++#define DISPLAY_PLANE_ENABLE (1 << 31) ++#define DISPLAY_PLANE_DISABLE 0 ++#define DISPPLANE_GAMMA_ENABLE (1 << 30) ++#define DISPPLANE_GAMMA_DISABLE 0 ++#define DISPPLANE_PIXFORMAT_MASK (0xf << 26) ++#define DISPPLANE_YUV422 (0x0 << 26) ++#define DISPPLANE_8BPP (0x2 << 26) ++#define DISPPLANE_BGRA555 (0x3 << 26) ++#define DISPPLANE_BGRX555 (0x4 << 26) ++#define DISPPLANE_BGRX565 (0x5 << 26) ++#define DISPPLANE_BGRX888 (0x6 << 26) ++#define DISPPLANE_BGRA888 (0x7 << 26) ++#define DISPPLANE_RGBX101010 (0x8 << 26) ++#define DISPPLANE_RGBA101010 (0x9 << 26) ++#define DISPPLANE_BGRX101010 (0xa << 26) ++#define DISPPLANE_RGBX161616 (0xc << 26) ++#define DISPPLANE_RGBX888 (0xe << 26) ++#define DISPPLANE_RGBA888 (0xf << 26) ++#define DISPPLANE_STEREO_ENABLE (1 << 25) ++#define DISPPLANE_STEREO_DISABLE 0 ++#define DISPPLANE_PIPE_CSC_ENABLE (1 << 24) /* ilk+ */ ++#define DISPPLANE_SEL_PIPE_SHIFT 24 ++#define DISPPLANE_SEL_PIPE_MASK (3 << DISPPLANE_SEL_PIPE_SHIFT) ++#define DISPPLANE_SEL_PIPE(pipe) ((pipe) << DISPPLANE_SEL_PIPE_SHIFT) ++#define DISPPLANE_SRC_KEY_ENABLE (1 << 22) ++#define DISPPLANE_SRC_KEY_DISABLE 0 ++#define DISPPLANE_LINE_DOUBLE (1 << 20) ++#define DISPPLANE_NO_LINE_DOUBLE 0 ++#define DISPPLANE_STEREO_POLARITY_FIRST 0 ++#define DISPPLANE_STEREO_POLARITY_SECOND (1 << 18) ++#define DISPPLANE_ALPHA_PREMULTIPLY (1 << 16) /* CHV pipe B */ ++#define DISPPLANE_ROTATE_180 (1 << 15) ++#define DISPPLANE_TRICKLE_FEED_DISABLE (1 << 14) /* Ironlake */ ++#define DISPPLANE_TILED (1 << 10) ++#define DISPPLANE_MIRROR (1 << 8) /* CHV pipe B */ ++#define _DSPAADDR 0x70184 ++#define _DSPASTRIDE 0x70188 ++#define _DSPAPOS 0x7018C /* reserved */ ++#define _DSPASIZE 0x70190 ++#define _DSPASURF 0x7019C /* 965+ only */ ++#define _DSPATILEOFF 0x701A4 /* 965+ only */ ++#define _DSPAOFFSET 0x701A4 /* HSW */ ++#define _DSPASURFLIVE 0x701AC ++ ++#define DSPCNTR(plane) _MMIO_PIPE2(plane, _DSPACNTR) ++#define DSPADDR(plane) _MMIO_PIPE2(plane, _DSPAADDR) ++#define DSPSTRIDE(plane) _MMIO_PIPE2(plane, _DSPASTRIDE) ++#define DSPPOS(plane) _MMIO_PIPE2(plane, _DSPAPOS) ++#define DSPSIZE(plane) _MMIO_PIPE2(plane, _DSPASIZE) ++#define DSPSURF(plane) _MMIO_PIPE2(plane, _DSPASURF) ++#define DSPTILEOFF(plane) _MMIO_PIPE2(plane, _DSPATILEOFF) ++#define DSPLINOFF(plane) DSPADDR(plane) ++#define DSPOFFSET(plane) _MMIO_PIPE2(plane, _DSPAOFFSET) ++#define DSPSURFLIVE(plane) _MMIO_PIPE2(plane, _DSPASURFLIVE) ++ ++/* CHV pipe B blender and primary plane */ ++#define _CHV_BLEND_A 0x60a00 ++#define CHV_BLEND_LEGACY (0 << 30) ++#define CHV_BLEND_ANDROID (1 << 30) ++#define CHV_BLEND_MPO (2 << 30) ++#define CHV_BLEND_MASK (3 << 30) ++#define _CHV_CANVAS_A 0x60a04 ++#define _PRIMPOS_A 0x60a08 ++#define _PRIMSIZE_A 0x60a0c ++#define _PRIMCNSTALPHA_A 0x60a10 ++#define PRIM_CONST_ALPHA_ENABLE (1 << 31) ++ ++#define CHV_BLEND(pipe) _MMIO_TRANS2(pipe, _CHV_BLEND_A) ++#define CHV_CANVAS(pipe) _MMIO_TRANS2(pipe, _CHV_CANVAS_A) ++#define PRIMPOS(plane) _MMIO_TRANS2(plane, _PRIMPOS_A) ++#define PRIMSIZE(plane) _MMIO_TRANS2(plane, _PRIMSIZE_A) ++#define PRIMCNSTALPHA(plane) _MMIO_TRANS2(plane, _PRIMCNSTALPHA_A) ++ ++/* Display/Sprite base address macros */ ++#define DISP_BASEADDR_MASK (0xfffff000) ++#define I915_LO_DISPBASE(val) ((val) & ~DISP_BASEADDR_MASK) ++#define I915_HI_DISPBASE(val) ((val) & DISP_BASEADDR_MASK) ++ ++/* ++ * VBIOS flags ++ * gen2: ++ * [00:06] alm,mgm ++ * [10:16] all ++ * [30:32] alm,mgm ++ * gen3+: ++ * [00:0f] all ++ * [10:1f] all ++ * [30:32] all ++ */ ++#define SWF0(i) _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x70410 + (i) * 4) ++#define SWF1(i) _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x71410 + (i) * 4) ++#define SWF3(i) _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x72414 + (i) * 4) ++#define SWF_ILK(i) _MMIO(0x4F000 + (i) * 4) ++ ++/* Pipe B */ ++#define _PIPEBDSL (DISPLAY_MMIO_BASE(dev_priv) + 0x71000) ++#define _PIPEBCONF (DISPLAY_MMIO_BASE(dev_priv) + 0x71008) ++#define _PIPEBSTAT (DISPLAY_MMIO_BASE(dev_priv) + 0x71024) ++#define _PIPEBFRAMEHIGH 0x71040 ++#define _PIPEBFRAMEPIXEL 0x71044 ++#define _PIPEB_FRMCOUNT_G4X (DISPLAY_MMIO_BASE(dev_priv) + 0x71040) ++#define _PIPEB_FLIPCOUNT_G4X (DISPLAY_MMIO_BASE(dev_priv) + 0x71044) ++ ++ ++/* Display B control */ ++#define _DSPBCNTR (DISPLAY_MMIO_BASE(dev_priv) + 0x71180) ++#define DISPPLANE_ALPHA_TRANS_ENABLE (1 << 15) ++#define DISPPLANE_ALPHA_TRANS_DISABLE 0 ++#define DISPPLANE_SPRITE_ABOVE_DISPLAY 0 ++#define DISPPLANE_SPRITE_ABOVE_OVERLAY (1) ++#define _DSPBADDR (DISPLAY_MMIO_BASE(dev_priv) + 0x71184) ++#define _DSPBSTRIDE (DISPLAY_MMIO_BASE(dev_priv) + 0x71188) ++#define _DSPBPOS (DISPLAY_MMIO_BASE(dev_priv) + 0x7118C) ++#define _DSPBSIZE (DISPLAY_MMIO_BASE(dev_priv) + 0x71190) ++#define _DSPBSURF (DISPLAY_MMIO_BASE(dev_priv) + 0x7119C) ++#define _DSPBTILEOFF (DISPLAY_MMIO_BASE(dev_priv) + 0x711A4) ++#define _DSPBOFFSET (DISPLAY_MMIO_BASE(dev_priv) + 0x711A4) ++#define _DSPBSURFLIVE (DISPLAY_MMIO_BASE(dev_priv) + 0x711AC) ++ ++/* ICL DSI 0 and 1 */ ++#define _PIPEDSI0CONF 0x7b008 ++#define _PIPEDSI1CONF 0x7b808 ++ ++/* Sprite A control */ ++#define _DVSACNTR 0x72180 ++#define DVS_ENABLE (1 << 31) ++#define DVS_GAMMA_ENABLE (1 << 30) ++#define DVS_YUV_RANGE_CORRECTION_DISABLE (1 << 27) ++#define DVS_PIXFORMAT_MASK (3 << 25) ++#define DVS_FORMAT_YUV422 (0 << 25) ++#define DVS_FORMAT_RGBX101010 (1 << 25) ++#define DVS_FORMAT_RGBX888 (2 << 25) ++#define DVS_FORMAT_RGBX161616 (3 << 25) ++#define DVS_PIPE_CSC_ENABLE (1 << 24) ++#define DVS_SOURCE_KEY (1 << 22) ++#define DVS_RGB_ORDER_XBGR (1 << 20) ++#define DVS_YUV_FORMAT_BT709 (1 << 18) ++#define DVS_YUV_BYTE_ORDER_MASK (3 << 16) ++#define DVS_YUV_ORDER_YUYV (0 << 16) ++#define DVS_YUV_ORDER_UYVY (1 << 16) ++#define DVS_YUV_ORDER_YVYU (2 << 16) ++#define DVS_YUV_ORDER_VYUY (3 << 16) ++#define DVS_ROTATE_180 (1 << 15) ++#define DVS_DEST_KEY (1 << 2) ++#define DVS_TRICKLE_FEED_DISABLE (1 << 14) ++#define DVS_TILED (1 << 10) ++#define _DVSALINOFF 0x72184 ++#define _DVSASTRIDE 0x72188 ++#define _DVSAPOS 0x7218c ++#define _DVSASIZE 0x72190 ++#define _DVSAKEYVAL 0x72194 ++#define _DVSAKEYMSK 0x72198 ++#define _DVSASURF 0x7219c ++#define _DVSAKEYMAXVAL 0x721a0 ++#define _DVSATILEOFF 0x721a4 ++#define _DVSASURFLIVE 0x721ac ++#define _DVSASCALE 0x72204 ++#define DVS_SCALE_ENABLE (1 << 31) ++#define DVS_FILTER_MASK (3 << 29) ++#define DVS_FILTER_MEDIUM (0 << 29) ++#define DVS_FILTER_ENHANCING (1 << 29) ++#define DVS_FILTER_SOFTENING (2 << 29) ++#define DVS_VERTICAL_OFFSET_HALF (1 << 28) /* must be enabled below */ ++#define DVS_VERTICAL_OFFSET_ENABLE (1 << 27) ++#define _DVSAGAMC 0x72300 ++ ++#define _DVSBCNTR 0x73180 ++#define _DVSBLINOFF 0x73184 ++#define _DVSBSTRIDE 0x73188 ++#define _DVSBPOS 0x7318c ++#define _DVSBSIZE 0x73190 ++#define _DVSBKEYVAL 0x73194 ++#define _DVSBKEYMSK 0x73198 ++#define _DVSBSURF 0x7319c ++#define _DVSBKEYMAXVAL 0x731a0 ++#define _DVSBTILEOFF 0x731a4 ++#define _DVSBSURFLIVE 0x731ac ++#define _DVSBSCALE 0x73204 ++#define _DVSBGAMC 0x73300 ++ ++#define DVSCNTR(pipe) _MMIO_PIPE(pipe, _DVSACNTR, _DVSBCNTR) ++#define DVSLINOFF(pipe) _MMIO_PIPE(pipe, _DVSALINOFF, _DVSBLINOFF) ++#define DVSSTRIDE(pipe) _MMIO_PIPE(pipe, _DVSASTRIDE, _DVSBSTRIDE) ++#define DVSPOS(pipe) _MMIO_PIPE(pipe, _DVSAPOS, _DVSBPOS) ++#define DVSSURF(pipe) _MMIO_PIPE(pipe, _DVSASURF, _DVSBSURF) ++#define DVSKEYMAX(pipe) _MMIO_PIPE(pipe, _DVSAKEYMAXVAL, _DVSBKEYMAXVAL) ++#define DVSSIZE(pipe) _MMIO_PIPE(pipe, _DVSASIZE, _DVSBSIZE) ++#define DVSSCALE(pipe) _MMIO_PIPE(pipe, _DVSASCALE, _DVSBSCALE) ++#define DVSTILEOFF(pipe) _MMIO_PIPE(pipe, _DVSATILEOFF, _DVSBTILEOFF) ++#define DVSKEYVAL(pipe) _MMIO_PIPE(pipe, _DVSAKEYVAL, _DVSBKEYVAL) ++#define DVSKEYMSK(pipe) _MMIO_PIPE(pipe, _DVSAKEYMSK, _DVSBKEYMSK) ++#define DVSSURFLIVE(pipe) _MMIO_PIPE(pipe, _DVSASURFLIVE, _DVSBSURFLIVE) ++ ++#define _SPRA_CTL 0x70280 ++#define SPRITE_ENABLE (1 << 31) ++#define SPRITE_GAMMA_ENABLE (1 << 30) ++#define SPRITE_YUV_RANGE_CORRECTION_DISABLE (1 << 28) ++#define SPRITE_PIXFORMAT_MASK (7 << 25) ++#define SPRITE_FORMAT_YUV422 (0 << 25) ++#define SPRITE_FORMAT_RGBX101010 (1 << 25) ++#define SPRITE_FORMAT_RGBX888 (2 << 25) ++#define SPRITE_FORMAT_RGBX161616 (3 << 25) ++#define SPRITE_FORMAT_YUV444 (4 << 25) ++#define SPRITE_FORMAT_XR_BGR101010 (5 << 25) /* Extended range */ ++#define SPRITE_PIPE_CSC_ENABLE (1 << 24) ++#define SPRITE_SOURCE_KEY (1 << 22) ++#define SPRITE_RGB_ORDER_RGBX (1 << 20) /* only for 888 and 161616 */ ++#define SPRITE_YUV_TO_RGB_CSC_DISABLE (1 << 19) ++#define SPRITE_YUV_TO_RGB_CSC_FORMAT_BT709 (1 << 18) /* 0 is BT601 */ ++#define SPRITE_YUV_BYTE_ORDER_MASK (3 << 16) ++#define SPRITE_YUV_ORDER_YUYV (0 << 16) ++#define SPRITE_YUV_ORDER_UYVY (1 << 16) ++#define SPRITE_YUV_ORDER_YVYU (2 << 16) ++#define SPRITE_YUV_ORDER_VYUY (3 << 16) ++#define SPRITE_ROTATE_180 (1 << 15) ++#define SPRITE_TRICKLE_FEED_DISABLE (1 << 14) ++#define SPRITE_INT_GAMMA_ENABLE (1 << 13) ++#define SPRITE_TILED (1 << 10) ++#define SPRITE_DEST_KEY (1 << 2) ++#define _SPRA_LINOFF 0x70284 ++#define _SPRA_STRIDE 0x70288 ++#define _SPRA_POS 0x7028c ++#define _SPRA_SIZE 0x70290 ++#define _SPRA_KEYVAL 0x70294 ++#define _SPRA_KEYMSK 0x70298 ++#define _SPRA_SURF 0x7029c ++#define _SPRA_KEYMAX 0x702a0 ++#define _SPRA_TILEOFF 0x702a4 ++#define _SPRA_OFFSET 0x702a4 ++#define _SPRA_SURFLIVE 0x702ac ++#define _SPRA_SCALE 0x70304 ++#define SPRITE_SCALE_ENABLE (1 << 31) ++#define SPRITE_FILTER_MASK (3 << 29) ++#define SPRITE_FILTER_MEDIUM (0 << 29) ++#define SPRITE_FILTER_ENHANCING (1 << 29) ++#define SPRITE_FILTER_SOFTENING (2 << 29) ++#define SPRITE_VERTICAL_OFFSET_HALF (1 << 28) /* must be enabled below */ ++#define SPRITE_VERTICAL_OFFSET_ENABLE (1 << 27) ++#define _SPRA_GAMC 0x70400 ++ ++#define _SPRB_CTL 0x71280 ++#define _SPRB_LINOFF 0x71284 ++#define _SPRB_STRIDE 0x71288 ++#define _SPRB_POS 0x7128c ++#define _SPRB_SIZE 0x71290 ++#define _SPRB_KEYVAL 0x71294 ++#define _SPRB_KEYMSK 0x71298 ++#define _SPRB_SURF 0x7129c ++#define _SPRB_KEYMAX 0x712a0 ++#define _SPRB_TILEOFF 0x712a4 ++#define _SPRB_OFFSET 0x712a4 ++#define _SPRB_SURFLIVE 0x712ac ++#define _SPRB_SCALE 0x71304 ++#define _SPRB_GAMC 0x71400 ++ ++#define SPRCTL(pipe) _MMIO_PIPE(pipe, _SPRA_CTL, _SPRB_CTL) ++#define SPRLINOFF(pipe) _MMIO_PIPE(pipe, _SPRA_LINOFF, _SPRB_LINOFF) ++#define SPRSTRIDE(pipe) _MMIO_PIPE(pipe, _SPRA_STRIDE, _SPRB_STRIDE) ++#define SPRPOS(pipe) _MMIO_PIPE(pipe, _SPRA_POS, _SPRB_POS) ++#define SPRSIZE(pipe) _MMIO_PIPE(pipe, _SPRA_SIZE, _SPRB_SIZE) ++#define SPRKEYVAL(pipe) _MMIO_PIPE(pipe, _SPRA_KEYVAL, _SPRB_KEYVAL) ++#define SPRKEYMSK(pipe) _MMIO_PIPE(pipe, _SPRA_KEYMSK, _SPRB_KEYMSK) ++#define SPRSURF(pipe) _MMIO_PIPE(pipe, _SPRA_SURF, _SPRB_SURF) ++#define SPRKEYMAX(pipe) _MMIO_PIPE(pipe, _SPRA_KEYMAX, _SPRB_KEYMAX) ++#define SPRTILEOFF(pipe) _MMIO_PIPE(pipe, _SPRA_TILEOFF, _SPRB_TILEOFF) ++#define SPROFFSET(pipe) _MMIO_PIPE(pipe, _SPRA_OFFSET, _SPRB_OFFSET) ++#define SPRSCALE(pipe) _MMIO_PIPE(pipe, _SPRA_SCALE, _SPRB_SCALE) ++#define SPRGAMC(pipe) _MMIO_PIPE(pipe, _SPRA_GAMC, _SPRB_GAMC) ++#define SPRSURFLIVE(pipe) _MMIO_PIPE(pipe, _SPRA_SURFLIVE, _SPRB_SURFLIVE) ++ ++#define _SPACNTR (VLV_DISPLAY_BASE + 0x72180) ++#define SP_ENABLE (1 << 31) ++#define SP_GAMMA_ENABLE (1 << 30) ++#define SP_PIXFORMAT_MASK (0xf << 26) ++#define SP_FORMAT_YUV422 (0 << 26) ++#define SP_FORMAT_BGR565 (5 << 26) ++#define SP_FORMAT_BGRX8888 (6 << 26) ++#define SP_FORMAT_BGRA8888 (7 << 26) ++#define SP_FORMAT_RGBX1010102 (8 << 26) ++#define SP_FORMAT_RGBA1010102 (9 << 26) ++#define SP_FORMAT_RGBX8888 (0xe << 26) ++#define SP_FORMAT_RGBA8888 (0xf << 26) ++#define SP_ALPHA_PREMULTIPLY (1 << 23) /* CHV pipe B */ ++#define SP_SOURCE_KEY (1 << 22) ++#define SP_YUV_FORMAT_BT709 (1 << 18) ++#define SP_YUV_BYTE_ORDER_MASK (3 << 16) ++#define SP_YUV_ORDER_YUYV (0 << 16) ++#define SP_YUV_ORDER_UYVY (1 << 16) ++#define SP_YUV_ORDER_YVYU (2 << 16) ++#define SP_YUV_ORDER_VYUY (3 << 16) ++#define SP_ROTATE_180 (1 << 15) ++#define SP_TILED (1 << 10) ++#define SP_MIRROR (1 << 8) /* CHV pipe B */ ++#define _SPALINOFF (VLV_DISPLAY_BASE + 0x72184) ++#define _SPASTRIDE (VLV_DISPLAY_BASE + 0x72188) ++#define _SPAPOS (VLV_DISPLAY_BASE + 0x7218c) ++#define _SPASIZE (VLV_DISPLAY_BASE + 0x72190) ++#define _SPAKEYMINVAL (VLV_DISPLAY_BASE + 0x72194) ++#define _SPAKEYMSK (VLV_DISPLAY_BASE + 0x72198) ++#define _SPASURF (VLV_DISPLAY_BASE + 0x7219c) ++#define _SPAKEYMAXVAL (VLV_DISPLAY_BASE + 0x721a0) ++#define _SPATILEOFF (VLV_DISPLAY_BASE + 0x721a4) ++#define _SPACONSTALPHA (VLV_DISPLAY_BASE + 0x721a8) ++#define SP_CONST_ALPHA_ENABLE (1 << 31) ++#define _SPACLRC0 (VLV_DISPLAY_BASE + 0x721d0) ++#define SP_CONTRAST(x) ((x) << 18) /* u3.6 */ ++#define SP_BRIGHTNESS(x) ((x) & 0xff) /* s8 */ ++#define _SPACLRC1 (VLV_DISPLAY_BASE + 0x721d4) ++#define SP_SH_SIN(x) (((x) & 0x7ff) << 16) /* s4.7 */ ++#define SP_SH_COS(x) (x) /* u3.7 */ ++#define _SPAGAMC (VLV_DISPLAY_BASE + 0x721f4) ++ ++#define _SPBCNTR (VLV_DISPLAY_BASE + 0x72280) ++#define _SPBLINOFF (VLV_DISPLAY_BASE + 0x72284) ++#define _SPBSTRIDE (VLV_DISPLAY_BASE + 0x72288) ++#define _SPBPOS (VLV_DISPLAY_BASE + 0x7228c) ++#define _SPBSIZE (VLV_DISPLAY_BASE + 0x72290) ++#define _SPBKEYMINVAL (VLV_DISPLAY_BASE + 0x72294) ++#define _SPBKEYMSK (VLV_DISPLAY_BASE + 0x72298) ++#define _SPBSURF (VLV_DISPLAY_BASE + 0x7229c) ++#define _SPBKEYMAXVAL (VLV_DISPLAY_BASE + 0x722a0) ++#define _SPBTILEOFF (VLV_DISPLAY_BASE + 0x722a4) ++#define _SPBCONSTALPHA (VLV_DISPLAY_BASE + 0x722a8) ++#define _SPBCLRC0 (VLV_DISPLAY_BASE + 0x722d0) ++#define _SPBCLRC1 (VLV_DISPLAY_BASE + 0x722d4) ++#define _SPBGAMC (VLV_DISPLAY_BASE + 0x722f4) ++ ++#define _MMIO_VLV_SPR(pipe, plane_id, reg_a, reg_b) \ ++ _MMIO_PIPE((pipe) * 2 + (plane_id) - PLANE_SPRITE0, (reg_a), (reg_b)) ++ ++#define SPCNTR(pipe, plane_id) _MMIO_VLV_SPR((pipe), (plane_id), _SPACNTR, _SPBCNTR) ++#define SPLINOFF(pipe, plane_id) _MMIO_VLV_SPR((pipe), (plane_id), _SPALINOFF, _SPBLINOFF) ++#define SPSTRIDE(pipe, plane_id) _MMIO_VLV_SPR((pipe), (plane_id), _SPASTRIDE, _SPBSTRIDE) ++#define SPPOS(pipe, plane_id) _MMIO_VLV_SPR((pipe), (plane_id), _SPAPOS, _SPBPOS) ++#define SPSIZE(pipe, plane_id) _MMIO_VLV_SPR((pipe), (plane_id), _SPASIZE, _SPBSIZE) ++#define SPKEYMINVAL(pipe, plane_id) _MMIO_VLV_SPR((pipe), (plane_id), _SPAKEYMINVAL, _SPBKEYMINVAL) ++#define SPKEYMSK(pipe, plane_id) _MMIO_VLV_SPR((pipe), (plane_id), _SPAKEYMSK, _SPBKEYMSK) ++#define SPSURF(pipe, plane_id) _MMIO_VLV_SPR((pipe), (plane_id), _SPASURF, _SPBSURF) ++#define SPKEYMAXVAL(pipe, plane_id) _MMIO_VLV_SPR((pipe), (plane_id), _SPAKEYMAXVAL, _SPBKEYMAXVAL) ++#define SPTILEOFF(pipe, plane_id) _MMIO_VLV_SPR((pipe), (plane_id), _SPATILEOFF, _SPBTILEOFF) ++#define SPCONSTALPHA(pipe, plane_id) _MMIO_VLV_SPR((pipe), (plane_id), _SPACONSTALPHA, _SPBCONSTALPHA) ++#define SPCLRC0(pipe, plane_id) _MMIO_VLV_SPR((pipe), (plane_id), _SPACLRC0, _SPBCLRC0) ++#define SPCLRC1(pipe, plane_id) _MMIO_VLV_SPR((pipe), (plane_id), _SPACLRC1, _SPBCLRC1) ++#define SPGAMC(pipe, plane_id) _MMIO_VLV_SPR((pipe), (plane_id), _SPAGAMC, _SPBGAMC) ++ ++/* ++ * CHV pipe B sprite CSC ++ * ++ * |cr| |c0 c1 c2| |cr + cr_ioff| |cr_ooff| ++ * |yg| = |c3 c4 c5| x |yg + yg_ioff| + |yg_ooff| ++ * |cb| |c6 c7 c8| |cb + cr_ioff| |cb_ooff| ++ */ ++#define _MMIO_CHV_SPCSC(plane_id, reg) \ ++ _MMIO(VLV_DISPLAY_BASE + ((plane_id) - PLANE_SPRITE0) * 0x1000 + (reg)) ++ ++#define SPCSCYGOFF(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d900) ++#define SPCSCCBOFF(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d904) ++#define SPCSCCROFF(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d908) ++#define SPCSC_OOFF(x) (((x) & 0x7ff) << 16) /* s11 */ ++#define SPCSC_IOFF(x) (((x) & 0x7ff) << 0) /* s11 */ ++ ++#define SPCSCC01(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d90c) ++#define SPCSCC23(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d910) ++#define SPCSCC45(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d914) ++#define SPCSCC67(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d918) ++#define SPCSCC8(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d91c) ++#define SPCSC_C1(x) (((x) & 0x7fff) << 16) /* s3.12 */ ++#define SPCSC_C0(x) (((x) & 0x7fff) << 0) /* s3.12 */ ++ ++#define SPCSCYGICLAMP(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d920) ++#define SPCSCCBICLAMP(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d924) ++#define SPCSCCRICLAMP(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d928) ++#define SPCSC_IMAX(x) (((x) & 0x7ff) << 16) /* s11 */ ++#define SPCSC_IMIN(x) (((x) & 0x7ff) << 0) /* s11 */ ++ ++#define SPCSCYGOCLAMP(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d92c) ++#define SPCSCCBOCLAMP(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d930) ++#define SPCSCCROCLAMP(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d934) ++#define SPCSC_OMAX(x) ((x) << 16) /* u10 */ ++#define SPCSC_OMIN(x) ((x) << 0) /* u10 */ ++ ++/* Skylake plane registers */ ++ ++#define _PLANE_CTL_1_A 0x70180 ++#define _PLANE_CTL_2_A 0x70280 ++#define _PLANE_CTL_3_A 0x70380 ++#define PLANE_CTL_ENABLE (1 << 31) ++#define PLANE_CTL_PIPE_GAMMA_ENABLE (1 << 30) /* Pre-GLK */ ++#define PLANE_CTL_YUV_RANGE_CORRECTION_DISABLE (1 << 28) ++/* ++ * ICL+ uses the same PLANE_CTL_FORMAT bits, but the field definition ++ * expanded to include bit 23 as well. However, the shift-24 based values ++ * correctly map to the same formats in ICL, as long as bit 23 is set to 0 ++ */ ++#define PLANE_CTL_FORMAT_MASK (0xf << 24) ++#define PLANE_CTL_FORMAT_YUV422 (0 << 24) ++#define PLANE_CTL_FORMAT_NV12 (1 << 24) ++#define PLANE_CTL_FORMAT_XRGB_2101010 (2 << 24) ++#define PLANE_CTL_FORMAT_P010 (3 << 24) ++#define PLANE_CTL_FORMAT_XRGB_8888 (4 << 24) ++#define PLANE_CTL_FORMAT_P012 (5 << 24) ++#define PLANE_CTL_FORMAT_XRGB_16161616F (6 << 24) ++#define PLANE_CTL_FORMAT_P016 (7 << 24) ++#define PLANE_CTL_FORMAT_AYUV (8 << 24) ++#define PLANE_CTL_FORMAT_INDEXED (12 << 24) ++#define PLANE_CTL_FORMAT_RGB_565 (14 << 24) ++#define ICL_PLANE_CTL_FORMAT_MASK (0x1f << 23) ++#define PLANE_CTL_PIPE_CSC_ENABLE (1 << 23) /* Pre-GLK */ ++#define PLANE_CTL_FORMAT_Y210 (1 << 23) ++#define PLANE_CTL_FORMAT_Y212 (3 << 23) ++#define PLANE_CTL_FORMAT_Y216 (5 << 23) ++#define PLANE_CTL_FORMAT_Y410 (7 << 23) ++#define PLANE_CTL_FORMAT_Y412 (9 << 23) ++#define PLANE_CTL_FORMAT_Y416 (0xb << 23) ++#define PLANE_CTL_KEY_ENABLE_MASK (0x3 << 21) ++#define PLANE_CTL_KEY_ENABLE_SOURCE (1 << 21) ++#define PLANE_CTL_KEY_ENABLE_DESTINATION (2 << 21) ++#define PLANE_CTL_ORDER_BGRX (0 << 20) ++#define PLANE_CTL_ORDER_RGBX (1 << 20) ++#define PLANE_CTL_YUV420_Y_PLANE (1 << 19) ++#define PLANE_CTL_YUV_TO_RGB_CSC_FORMAT_BT709 (1 << 18) ++#define PLANE_CTL_YUV422_ORDER_MASK (0x3 << 16) ++#define PLANE_CTL_YUV422_YUYV (0 << 16) ++#define PLANE_CTL_YUV422_UYVY (1 << 16) ++#define PLANE_CTL_YUV422_YVYU (2 << 16) ++#define PLANE_CTL_YUV422_VYUY (3 << 16) ++#define PLANE_CTL_RENDER_DECOMPRESSION_ENABLE (1 << 15) ++#define PLANE_CTL_TRICKLE_FEED_DISABLE (1 << 14) ++#define PLANE_CTL_PLANE_GAMMA_DISABLE (1 << 13) /* Pre-GLK */ ++#define PLANE_CTL_TILED_MASK (0x7 << 10) ++#define PLANE_CTL_TILED_LINEAR (0 << 10) ++#define PLANE_CTL_TILED_X (1 << 10) ++#define PLANE_CTL_TILED_Y (4 << 10) ++#define PLANE_CTL_TILED_YF (5 << 10) ++#define PLANE_CTL_FLIP_HORIZONTAL (1 << 8) ++#define PLANE_CTL_ALPHA_MASK (0x3 << 4) /* Pre-GLK */ ++#define PLANE_CTL_ALPHA_DISABLE (0 << 4) ++#define PLANE_CTL_ALPHA_SW_PREMULTIPLY (2 << 4) ++#define PLANE_CTL_ALPHA_HW_PREMULTIPLY (3 << 4) ++#define PLANE_CTL_ROTATE_MASK 0x3 ++#define PLANE_CTL_ROTATE_0 0x0 ++#define PLANE_CTL_ROTATE_90 0x1 ++#define PLANE_CTL_ROTATE_180 0x2 ++#define PLANE_CTL_ROTATE_270 0x3 ++#define _PLANE_STRIDE_1_A 0x70188 ++#define _PLANE_STRIDE_2_A 0x70288 ++#define _PLANE_STRIDE_3_A 0x70388 ++#define _PLANE_POS_1_A 0x7018c ++#define _PLANE_POS_2_A 0x7028c ++#define _PLANE_POS_3_A 0x7038c ++#define _PLANE_SIZE_1_A 0x70190 ++#define _PLANE_SIZE_2_A 0x70290 ++#define _PLANE_SIZE_3_A 0x70390 ++#define _PLANE_SURF_1_A 0x7019c ++#define _PLANE_SURF_2_A 0x7029c ++#define _PLANE_SURF_3_A 0x7039c ++#define _PLANE_OFFSET_1_A 0x701a4 ++#define _PLANE_OFFSET_2_A 0x702a4 ++#define _PLANE_OFFSET_3_A 0x703a4 ++#define _PLANE_KEYVAL_1_A 0x70194 ++#define _PLANE_KEYVAL_2_A 0x70294 ++#define _PLANE_KEYMSK_1_A 0x70198 ++#define _PLANE_KEYMSK_2_A 0x70298 ++#define PLANE_KEYMSK_ALPHA_ENABLE (1 << 31) ++#define _PLANE_KEYMAX_1_A 0x701a0 ++#define _PLANE_KEYMAX_2_A 0x702a0 ++#define PLANE_KEYMAX_ALPHA(a) ((a) << 24) ++#define _PLANE_AUX_DIST_1_A 0x701c0 ++#define _PLANE_AUX_DIST_2_A 0x702c0 ++#define _PLANE_AUX_OFFSET_1_A 0x701c4 ++#define _PLANE_AUX_OFFSET_2_A 0x702c4 ++#define _PLANE_CUS_CTL_1_A 0x701c8 ++#define _PLANE_CUS_CTL_2_A 0x702c8 ++#define PLANE_CUS_ENABLE (1 << 31) ++#define PLANE_CUS_PLANE_6 (0 << 30) ++#define PLANE_CUS_PLANE_7 (1 << 30) ++#define PLANE_CUS_HPHASE_SIGN_NEGATIVE (1 << 19) ++#define PLANE_CUS_HPHASE_0 (0 << 16) ++#define PLANE_CUS_HPHASE_0_25 (1 << 16) ++#define PLANE_CUS_HPHASE_0_5 (2 << 16) ++#define PLANE_CUS_VPHASE_SIGN_NEGATIVE (1 << 15) ++#define PLANE_CUS_VPHASE_0 (0 << 12) ++#define PLANE_CUS_VPHASE_0_25 (1 << 12) ++#define PLANE_CUS_VPHASE_0_5 (2 << 12) ++#define _PLANE_COLOR_CTL_1_A 0x701CC /* GLK+ */ ++#define _PLANE_COLOR_CTL_2_A 0x702CC /* GLK+ */ ++#define _PLANE_COLOR_CTL_3_A 0x703CC /* GLK+ */ ++#define PLANE_COLOR_PIPE_GAMMA_ENABLE (1 << 30) /* Pre-ICL */ ++#define PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE (1 << 28) ++#define PLANE_COLOR_INPUT_CSC_ENABLE (1 << 20) /* ICL+ */ ++#define PLANE_COLOR_PIPE_CSC_ENABLE (1 << 23) /* Pre-ICL */ ++#define PLANE_COLOR_CSC_MODE_BYPASS (0 << 17) ++#define PLANE_COLOR_CSC_MODE_YUV601_TO_RGB709 (1 << 17) ++#define PLANE_COLOR_CSC_MODE_YUV709_TO_RGB709 (2 << 17) ++#define PLANE_COLOR_CSC_MODE_YUV2020_TO_RGB2020 (3 << 17) ++#define PLANE_COLOR_CSC_MODE_RGB709_TO_RGB2020 (4 << 17) ++#define PLANE_COLOR_PLANE_GAMMA_DISABLE (1 << 13) ++#define PLANE_COLOR_ALPHA_MASK (0x3 << 4) ++#define PLANE_COLOR_ALPHA_DISABLE (0 << 4) ++#define PLANE_COLOR_ALPHA_SW_PREMULTIPLY (2 << 4) ++#define PLANE_COLOR_ALPHA_HW_PREMULTIPLY (3 << 4) ++#define _PLANE_BUF_CFG_1_A 0x7027c ++#define _PLANE_BUF_CFG_2_A 0x7037c ++#define _PLANE_NV12_BUF_CFG_1_A 0x70278 ++#define _PLANE_NV12_BUF_CFG_2_A 0x70378 ++ ++/* Input CSC Register Definitions */ ++#define _PLANE_INPUT_CSC_RY_GY_1_A 0x701E0 ++#define _PLANE_INPUT_CSC_RY_GY_2_A 0x702E0 ++ ++#define _PLANE_INPUT_CSC_RY_GY_1_B 0x711E0 ++#define _PLANE_INPUT_CSC_RY_GY_2_B 0x712E0 ++ ++#define _PLANE_INPUT_CSC_RY_GY_1(pipe) \ ++ _PIPE(pipe, _PLANE_INPUT_CSC_RY_GY_1_A, \ ++ _PLANE_INPUT_CSC_RY_GY_1_B) ++#define _PLANE_INPUT_CSC_RY_GY_2(pipe) \ ++ _PIPE(pipe, _PLANE_INPUT_CSC_RY_GY_2_A, \ ++ _PLANE_INPUT_CSC_RY_GY_2_B) ++ ++#define PLANE_INPUT_CSC_COEFF(pipe, plane, index) \ ++ _MMIO_PLANE(plane, _PLANE_INPUT_CSC_RY_GY_1(pipe) + (index) * 4, \ ++ _PLANE_INPUT_CSC_RY_GY_2(pipe) + (index) * 4) ++ ++#define _PLANE_INPUT_CSC_PREOFF_HI_1_A 0x701F8 ++#define _PLANE_INPUT_CSC_PREOFF_HI_2_A 0x702F8 ++ ++#define _PLANE_INPUT_CSC_PREOFF_HI_1_B 0x711F8 ++#define _PLANE_INPUT_CSC_PREOFF_HI_2_B 0x712F8 ++ ++#define _PLANE_INPUT_CSC_PREOFF_HI_1(pipe) \ ++ _PIPE(pipe, _PLANE_INPUT_CSC_PREOFF_HI_1_A, \ ++ _PLANE_INPUT_CSC_PREOFF_HI_1_B) ++#define _PLANE_INPUT_CSC_PREOFF_HI_2(pipe) \ ++ _PIPE(pipe, _PLANE_INPUT_CSC_PREOFF_HI_2_A, \ ++ _PLANE_INPUT_CSC_PREOFF_HI_2_B) ++#define PLANE_INPUT_CSC_PREOFF(pipe, plane, index) \ ++ _MMIO_PLANE(plane, _PLANE_INPUT_CSC_PREOFF_HI_1(pipe) + (index) * 4, \ ++ _PLANE_INPUT_CSC_PREOFF_HI_2(pipe) + (index) * 4) ++ ++#define _PLANE_INPUT_CSC_POSTOFF_HI_1_A 0x70204 ++#define _PLANE_INPUT_CSC_POSTOFF_HI_2_A 0x70304 ++ ++#define _PLANE_INPUT_CSC_POSTOFF_HI_1_B 0x71204 ++#define _PLANE_INPUT_CSC_POSTOFF_HI_2_B 0x71304 ++ ++#define _PLANE_INPUT_CSC_POSTOFF_HI_1(pipe) \ ++ _PIPE(pipe, _PLANE_INPUT_CSC_POSTOFF_HI_1_A, \ ++ _PLANE_INPUT_CSC_POSTOFF_HI_1_B) ++#define _PLANE_INPUT_CSC_POSTOFF_HI_2(pipe) \ ++ _PIPE(pipe, _PLANE_INPUT_CSC_POSTOFF_HI_2_A, \ ++ _PLANE_INPUT_CSC_POSTOFF_HI_2_B) ++#define PLANE_INPUT_CSC_POSTOFF(pipe, plane, index) \ ++ _MMIO_PLANE(plane, _PLANE_INPUT_CSC_POSTOFF_HI_1(pipe) + (index) * 4, \ ++ _PLANE_INPUT_CSC_POSTOFF_HI_2(pipe) + (index) * 4) ++ ++#define _PLANE_CTL_1_B 0x71180 ++#define _PLANE_CTL_2_B 0x71280 ++#define _PLANE_CTL_3_B 0x71380 ++#define _PLANE_CTL_1(pipe) _PIPE(pipe, _PLANE_CTL_1_A, _PLANE_CTL_1_B) ++#define _PLANE_CTL_2(pipe) _PIPE(pipe, _PLANE_CTL_2_A, _PLANE_CTL_2_B) ++#define _PLANE_CTL_3(pipe) _PIPE(pipe, _PLANE_CTL_3_A, _PLANE_CTL_3_B) ++#define PLANE_CTL(pipe, plane) \ ++ _MMIO_PLANE(plane, _PLANE_CTL_1(pipe), _PLANE_CTL_2(pipe)) ++ ++#define _PLANE_STRIDE_1_B 0x71188 ++#define _PLANE_STRIDE_2_B 0x71288 ++#define _PLANE_STRIDE_3_B 0x71388 ++#define _PLANE_STRIDE_1(pipe) \ ++ _PIPE(pipe, _PLANE_STRIDE_1_A, _PLANE_STRIDE_1_B) ++#define _PLANE_STRIDE_2(pipe) \ ++ _PIPE(pipe, _PLANE_STRIDE_2_A, _PLANE_STRIDE_2_B) ++#define _PLANE_STRIDE_3(pipe) \ ++ _PIPE(pipe, _PLANE_STRIDE_3_A, _PLANE_STRIDE_3_B) ++#define PLANE_STRIDE(pipe, plane) \ ++ _MMIO_PLANE(plane, _PLANE_STRIDE_1(pipe), _PLANE_STRIDE_2(pipe)) ++ ++#define _PLANE_POS_1_B 0x7118c ++#define _PLANE_POS_2_B 0x7128c ++#define _PLANE_POS_3_B 0x7138c ++#define _PLANE_POS_1(pipe) _PIPE(pipe, _PLANE_POS_1_A, _PLANE_POS_1_B) ++#define _PLANE_POS_2(pipe) _PIPE(pipe, _PLANE_POS_2_A, _PLANE_POS_2_B) ++#define _PLANE_POS_3(pipe) _PIPE(pipe, _PLANE_POS_3_A, _PLANE_POS_3_B) ++#define PLANE_POS(pipe, plane) \ ++ _MMIO_PLANE(plane, _PLANE_POS_1(pipe), _PLANE_POS_2(pipe)) ++ ++#define _PLANE_SIZE_1_B 0x71190 ++#define _PLANE_SIZE_2_B 0x71290 ++#define _PLANE_SIZE_3_B 0x71390 ++#define _PLANE_SIZE_1(pipe) _PIPE(pipe, _PLANE_SIZE_1_A, _PLANE_SIZE_1_B) ++#define _PLANE_SIZE_2(pipe) _PIPE(pipe, _PLANE_SIZE_2_A, _PLANE_SIZE_2_B) ++#define _PLANE_SIZE_3(pipe) _PIPE(pipe, _PLANE_SIZE_3_A, _PLANE_SIZE_3_B) ++#define PLANE_SIZE(pipe, plane) \ ++ _MMIO_PLANE(plane, _PLANE_SIZE_1(pipe), _PLANE_SIZE_2(pipe)) ++ ++#define _PLANE_SURF_1_B 0x7119c ++#define _PLANE_SURF_2_B 0x7129c ++#define _PLANE_SURF_3_B 0x7139c ++#define _PLANE_SURF_1(pipe) _PIPE(pipe, _PLANE_SURF_1_A, _PLANE_SURF_1_B) ++#define _PLANE_SURF_2(pipe) _PIPE(pipe, _PLANE_SURF_2_A, _PLANE_SURF_2_B) ++#define _PLANE_SURF_3(pipe) _PIPE(pipe, _PLANE_SURF_3_A, _PLANE_SURF_3_B) ++#define PLANE_SURF(pipe, plane) \ ++ _MMIO_PLANE(plane, _PLANE_SURF_1(pipe), _PLANE_SURF_2(pipe)) ++ ++#define _PLANE_OFFSET_1_B 0x711a4 ++#define _PLANE_OFFSET_2_B 0x712a4 ++#define _PLANE_OFFSET_1(pipe) _PIPE(pipe, _PLANE_OFFSET_1_A, _PLANE_OFFSET_1_B) ++#define _PLANE_OFFSET_2(pipe) _PIPE(pipe, _PLANE_OFFSET_2_A, _PLANE_OFFSET_2_B) ++#define PLANE_OFFSET(pipe, plane) \ ++ _MMIO_PLANE(plane, _PLANE_OFFSET_1(pipe), _PLANE_OFFSET_2(pipe)) ++ ++#define _PLANE_KEYVAL_1_B 0x71194 ++#define _PLANE_KEYVAL_2_B 0x71294 ++#define _PLANE_KEYVAL_1(pipe) _PIPE(pipe, _PLANE_KEYVAL_1_A, _PLANE_KEYVAL_1_B) ++#define _PLANE_KEYVAL_2(pipe) _PIPE(pipe, _PLANE_KEYVAL_2_A, _PLANE_KEYVAL_2_B) ++#define PLANE_KEYVAL(pipe, plane) \ ++ _MMIO_PLANE(plane, _PLANE_KEYVAL_1(pipe), _PLANE_KEYVAL_2(pipe)) ++ ++#define _PLANE_KEYMSK_1_B 0x71198 ++#define _PLANE_KEYMSK_2_B 0x71298 ++#define _PLANE_KEYMSK_1(pipe) _PIPE(pipe, _PLANE_KEYMSK_1_A, _PLANE_KEYMSK_1_B) ++#define _PLANE_KEYMSK_2(pipe) _PIPE(pipe, _PLANE_KEYMSK_2_A, _PLANE_KEYMSK_2_B) ++#define PLANE_KEYMSK(pipe, plane) \ ++ _MMIO_PLANE(plane, _PLANE_KEYMSK_1(pipe), _PLANE_KEYMSK_2(pipe)) ++ ++#define _PLANE_KEYMAX_1_B 0x711a0 ++#define _PLANE_KEYMAX_2_B 0x712a0 ++#define _PLANE_KEYMAX_1(pipe) _PIPE(pipe, _PLANE_KEYMAX_1_A, _PLANE_KEYMAX_1_B) ++#define _PLANE_KEYMAX_2(pipe) _PIPE(pipe, _PLANE_KEYMAX_2_A, _PLANE_KEYMAX_2_B) ++#define PLANE_KEYMAX(pipe, plane) \ ++ _MMIO_PLANE(plane, _PLANE_KEYMAX_1(pipe), _PLANE_KEYMAX_2(pipe)) ++ ++#define _PLANE_BUF_CFG_1_B 0x7127c ++#define _PLANE_BUF_CFG_2_B 0x7137c ++#define DDB_ENTRY_MASK 0x7FF /* skl+: 10 bits, icl+ 11 bits */ ++#define DDB_ENTRY_END_SHIFT 16 ++#define _PLANE_BUF_CFG_1(pipe) \ ++ _PIPE(pipe, _PLANE_BUF_CFG_1_A, _PLANE_BUF_CFG_1_B) ++#define _PLANE_BUF_CFG_2(pipe) \ ++ _PIPE(pipe, _PLANE_BUF_CFG_2_A, _PLANE_BUF_CFG_2_B) ++#define PLANE_BUF_CFG(pipe, plane) \ ++ _MMIO_PLANE(plane, _PLANE_BUF_CFG_1(pipe), _PLANE_BUF_CFG_2(pipe)) ++ ++#define _PLANE_NV12_BUF_CFG_1_B 0x71278 ++#define _PLANE_NV12_BUF_CFG_2_B 0x71378 ++#define _PLANE_NV12_BUF_CFG_1(pipe) \ ++ _PIPE(pipe, _PLANE_NV12_BUF_CFG_1_A, _PLANE_NV12_BUF_CFG_1_B) ++#define _PLANE_NV12_BUF_CFG_2(pipe) \ ++ _PIPE(pipe, _PLANE_NV12_BUF_CFG_2_A, _PLANE_NV12_BUF_CFG_2_B) ++#define PLANE_NV12_BUF_CFG(pipe, plane) \ ++ _MMIO_PLANE(plane, _PLANE_NV12_BUF_CFG_1(pipe), _PLANE_NV12_BUF_CFG_2(pipe)) ++ ++#define _PLANE_AUX_DIST_1_B 0x711c0 ++#define _PLANE_AUX_DIST_2_B 0x712c0 ++#define _PLANE_AUX_DIST_1(pipe) \ ++ _PIPE(pipe, _PLANE_AUX_DIST_1_A, _PLANE_AUX_DIST_1_B) ++#define _PLANE_AUX_DIST_2(pipe) \ ++ _PIPE(pipe, _PLANE_AUX_DIST_2_A, _PLANE_AUX_DIST_2_B) ++#define PLANE_AUX_DIST(pipe, plane) \ ++ _MMIO_PLANE(plane, _PLANE_AUX_DIST_1(pipe), _PLANE_AUX_DIST_2(pipe)) ++ ++#define _PLANE_AUX_OFFSET_1_B 0x711c4 ++#define _PLANE_AUX_OFFSET_2_B 0x712c4 ++#define _PLANE_AUX_OFFSET_1(pipe) \ ++ _PIPE(pipe, _PLANE_AUX_OFFSET_1_A, _PLANE_AUX_OFFSET_1_B) ++#define _PLANE_AUX_OFFSET_2(pipe) \ ++ _PIPE(pipe, _PLANE_AUX_OFFSET_2_A, _PLANE_AUX_OFFSET_2_B) ++#define PLANE_AUX_OFFSET(pipe, plane) \ ++ _MMIO_PLANE(plane, _PLANE_AUX_OFFSET_1(pipe), _PLANE_AUX_OFFSET_2(pipe)) ++ ++#define _PLANE_CUS_CTL_1_B 0x711c8 ++#define _PLANE_CUS_CTL_2_B 0x712c8 ++#define _PLANE_CUS_CTL_1(pipe) \ ++ _PIPE(pipe, _PLANE_CUS_CTL_1_A, _PLANE_CUS_CTL_1_B) ++#define _PLANE_CUS_CTL_2(pipe) \ ++ _PIPE(pipe, _PLANE_CUS_CTL_2_A, _PLANE_CUS_CTL_2_B) ++#define PLANE_CUS_CTL(pipe, plane) \ ++ _MMIO_PLANE(plane, _PLANE_CUS_CTL_1(pipe), _PLANE_CUS_CTL_2(pipe)) ++ ++#define _PLANE_COLOR_CTL_1_B 0x711CC ++#define _PLANE_COLOR_CTL_2_B 0x712CC ++#define _PLANE_COLOR_CTL_3_B 0x713CC ++#define _PLANE_COLOR_CTL_1(pipe) \ ++ _PIPE(pipe, _PLANE_COLOR_CTL_1_A, _PLANE_COLOR_CTL_1_B) ++#define _PLANE_COLOR_CTL_2(pipe) \ ++ _PIPE(pipe, _PLANE_COLOR_CTL_2_A, _PLANE_COLOR_CTL_2_B) ++#define PLANE_COLOR_CTL(pipe, plane) \ ++ _MMIO_PLANE(plane, _PLANE_COLOR_CTL_1(pipe), _PLANE_COLOR_CTL_2(pipe)) ++ ++#/* SKL new cursor registers */ ++#define _CUR_BUF_CFG_A 0x7017c ++#define _CUR_BUF_CFG_B 0x7117c ++#define CUR_BUF_CFG(pipe) _MMIO_PIPE(pipe, _CUR_BUF_CFG_A, _CUR_BUF_CFG_B) ++ ++/* VBIOS regs */ ++#define VGACNTRL _MMIO(0x71400) ++# define VGA_DISP_DISABLE (1 << 31) ++# define VGA_2X_MODE (1 << 30) ++# define VGA_PIPE_B_SELECT (1 << 29) ++ ++#define VLV_VGACNTRL _MMIO(VLV_DISPLAY_BASE + 0x71400) ++ ++/* Ironlake */ ++ ++#define CPU_VGACNTRL _MMIO(0x41000) ++ ++#define DIGITAL_PORT_HOTPLUG_CNTRL _MMIO(0x44030) ++#define DIGITAL_PORTA_HOTPLUG_ENABLE (1 << 4) ++#define DIGITAL_PORTA_PULSE_DURATION_2ms (0 << 2) /* pre-HSW */ ++#define DIGITAL_PORTA_PULSE_DURATION_4_5ms (1 << 2) /* pre-HSW */ ++#define DIGITAL_PORTA_PULSE_DURATION_6ms (2 << 2) /* pre-HSW */ ++#define DIGITAL_PORTA_PULSE_DURATION_100ms (3 << 2) /* pre-HSW */ ++#define DIGITAL_PORTA_PULSE_DURATION_MASK (3 << 2) /* pre-HSW */ ++#define DIGITAL_PORTA_HOTPLUG_STATUS_MASK (3 << 0) ++#define DIGITAL_PORTA_HOTPLUG_NO_DETECT (0 << 0) ++#define DIGITAL_PORTA_HOTPLUG_SHORT_DETECT (1 << 0) ++#define DIGITAL_PORTA_HOTPLUG_LONG_DETECT (2 << 0) ++ ++/* refresh rate hardware control */ ++#define RR_HW_CTL _MMIO(0x45300) ++#define RR_HW_LOW_POWER_FRAMES_MASK 0xff ++#define RR_HW_HIGH_POWER_FRAMES_MASK 0xff00 ++ ++#define FDI_PLL_BIOS_0 _MMIO(0x46000) ++#define FDI_PLL_FB_CLOCK_MASK 0xff ++#define FDI_PLL_BIOS_1 _MMIO(0x46004) ++#define FDI_PLL_BIOS_2 _MMIO(0x46008) ++#define DISPLAY_PORT_PLL_BIOS_0 _MMIO(0x4600c) ++#define DISPLAY_PORT_PLL_BIOS_1 _MMIO(0x46010) ++#define DISPLAY_PORT_PLL_BIOS_2 _MMIO(0x46014) ++ ++#define PCH_3DCGDIS0 _MMIO(0x46020) ++# define MARIUNIT_CLOCK_GATE_DISABLE (1 << 18) ++# define SVSMUNIT_CLOCK_GATE_DISABLE (1 << 1) ++ ++#define PCH_3DCGDIS1 _MMIO(0x46024) ++# define VFMUNIT_CLOCK_GATE_DISABLE (1 << 11) ++ ++#define FDI_PLL_FREQ_CTL _MMIO(0x46030) ++#define FDI_PLL_FREQ_CHANGE_REQUEST (1 << 24) ++#define FDI_PLL_FREQ_LOCK_LIMIT_MASK 0xfff00 ++#define FDI_PLL_FREQ_DISABLE_COUNT_LIMIT_MASK 0xff ++ ++ ++#define _PIPEA_DATA_M1 0x60030 ++#define PIPE_DATA_M1_OFFSET 0 ++#define _PIPEA_DATA_N1 0x60034 ++#define PIPE_DATA_N1_OFFSET 0 ++ ++#define _PIPEA_DATA_M2 0x60038 ++#define PIPE_DATA_M2_OFFSET 0 ++#define _PIPEA_DATA_N2 0x6003c ++#define PIPE_DATA_N2_OFFSET 0 ++ ++#define _PIPEA_LINK_M1 0x60040 ++#define PIPE_LINK_M1_OFFSET 0 ++#define _PIPEA_LINK_N1 0x60044 ++#define PIPE_LINK_N1_OFFSET 0 ++ ++#define _PIPEA_LINK_M2 0x60048 ++#define PIPE_LINK_M2_OFFSET 0 ++#define _PIPEA_LINK_N2 0x6004c ++#define PIPE_LINK_N2_OFFSET 0 ++ ++/* PIPEB timing regs are same start from 0x61000 */ ++ ++#define _PIPEB_DATA_M1 0x61030 ++#define _PIPEB_DATA_N1 0x61034 ++#define _PIPEB_DATA_M2 0x61038 ++#define _PIPEB_DATA_N2 0x6103c ++#define _PIPEB_LINK_M1 0x61040 ++#define _PIPEB_LINK_N1 0x61044 ++#define _PIPEB_LINK_M2 0x61048 ++#define _PIPEB_LINK_N2 0x6104c ++ ++#define PIPE_DATA_M1(tran) _MMIO_TRANS2(tran, _PIPEA_DATA_M1) ++#define PIPE_DATA_N1(tran) _MMIO_TRANS2(tran, _PIPEA_DATA_N1) ++#define PIPE_DATA_M2(tran) _MMIO_TRANS2(tran, _PIPEA_DATA_M2) ++#define PIPE_DATA_N2(tran) _MMIO_TRANS2(tran, _PIPEA_DATA_N2) ++#define PIPE_LINK_M1(tran) _MMIO_TRANS2(tran, _PIPEA_LINK_M1) ++#define PIPE_LINK_N1(tran) _MMIO_TRANS2(tran, _PIPEA_LINK_N1) ++#define PIPE_LINK_M2(tran) _MMIO_TRANS2(tran, _PIPEA_LINK_M2) ++#define PIPE_LINK_N2(tran) _MMIO_TRANS2(tran, _PIPEA_LINK_N2) ++ ++/* CPU panel fitter */ ++/* IVB+ has 3 fitters, 0 is 7x5 capable, the other two only 3x3 */ ++#define _PFA_CTL_1 0x68080 ++#define _PFB_CTL_1 0x68880 ++#define PF_ENABLE (1 << 31) ++#define PF_PIPE_SEL_MASK_IVB (3 << 29) ++#define PF_PIPE_SEL_IVB(pipe) ((pipe) << 29) ++#define PF_FILTER_MASK (3 << 23) ++#define PF_FILTER_PROGRAMMED (0 << 23) ++#define PF_FILTER_MED_3x3 (1 << 23) ++#define PF_FILTER_EDGE_ENHANCE (2 << 23) ++#define PF_FILTER_EDGE_SOFTEN (3 << 23) ++#define _PFA_WIN_SZ 0x68074 ++#define _PFB_WIN_SZ 0x68874 ++#define _PFA_WIN_POS 0x68070 ++#define _PFB_WIN_POS 0x68870 ++#define _PFA_VSCALE 0x68084 ++#define _PFB_VSCALE 0x68884 ++#define _PFA_HSCALE 0x68090 ++#define _PFB_HSCALE 0x68890 ++ ++#define PF_CTL(pipe) _MMIO_PIPE(pipe, _PFA_CTL_1, _PFB_CTL_1) ++#define PF_WIN_SZ(pipe) _MMIO_PIPE(pipe, _PFA_WIN_SZ, _PFB_WIN_SZ) ++#define PF_WIN_POS(pipe) _MMIO_PIPE(pipe, _PFA_WIN_POS, _PFB_WIN_POS) ++#define PF_VSCALE(pipe) _MMIO_PIPE(pipe, _PFA_VSCALE, _PFB_VSCALE) ++#define PF_HSCALE(pipe) _MMIO_PIPE(pipe, _PFA_HSCALE, _PFB_HSCALE) ++ ++#define _PSA_CTL 0x68180 ++#define _PSB_CTL 0x68980 ++#define PS_ENABLE (1 << 31) ++#define _PSA_WIN_SZ 0x68174 ++#define _PSB_WIN_SZ 0x68974 ++#define _PSA_WIN_POS 0x68170 ++#define _PSB_WIN_POS 0x68970 ++ ++#define PS_CTL(pipe) _MMIO_PIPE(pipe, _PSA_CTL, _PSB_CTL) ++#define PS_WIN_SZ(pipe) _MMIO_PIPE(pipe, _PSA_WIN_SZ, _PSB_WIN_SZ) ++#define PS_WIN_POS(pipe) _MMIO_PIPE(pipe, _PSA_WIN_POS, _PSB_WIN_POS) ++ ++/* ++ * Skylake scalers ++ */ ++#define _PS_1A_CTRL 0x68180 ++#define _PS_2A_CTRL 0x68280 ++#define _PS_1B_CTRL 0x68980 ++#define _PS_2B_CTRL 0x68A80 ++#define _PS_1C_CTRL 0x69180 ++#define PS_SCALER_EN (1 << 31) ++#define SKL_PS_SCALER_MODE_MASK (3 << 28) ++#define SKL_PS_SCALER_MODE_DYN (0 << 28) ++#define SKL_PS_SCALER_MODE_HQ (1 << 28) ++#define SKL_PS_SCALER_MODE_NV12 (2 << 28) ++#define PS_SCALER_MODE_PLANAR (1 << 29) ++#define PS_SCALER_MODE_NORMAL (0 << 29) ++#define PS_PLANE_SEL_MASK (7 << 25) ++#define PS_PLANE_SEL(plane) (((plane) + 1) << 25) ++#define PS_FILTER_MASK (3 << 23) ++#define PS_FILTER_MEDIUM (0 << 23) ++#define PS_FILTER_EDGE_ENHANCE (2 << 23) ++#define PS_FILTER_BILINEAR (3 << 23) ++#define PS_VERT3TAP (1 << 21) ++#define PS_VERT_INT_INVERT_FIELD1 (0 << 20) ++#define PS_VERT_INT_INVERT_FIELD0 (1 << 20) ++#define PS_PWRUP_PROGRESS (1 << 17) ++#define PS_V_FILTER_BYPASS (1 << 8) ++#define PS_VADAPT_EN (1 << 7) ++#define PS_VADAPT_MODE_MASK (3 << 5) ++#define PS_VADAPT_MODE_LEAST_ADAPT (0 << 5) ++#define PS_VADAPT_MODE_MOD_ADAPT (1 << 5) ++#define PS_VADAPT_MODE_MOST_ADAPT (3 << 5) ++#define PS_PLANE_Y_SEL_MASK (7 << 5) ++#define PS_PLANE_Y_SEL(plane) (((plane) + 1) << 5) ++ ++#define _PS_PWR_GATE_1A 0x68160 ++#define _PS_PWR_GATE_2A 0x68260 ++#define _PS_PWR_GATE_1B 0x68960 ++#define _PS_PWR_GATE_2B 0x68A60 ++#define _PS_PWR_GATE_1C 0x69160 ++#define PS_PWR_GATE_DIS_OVERRIDE (1 << 31) ++#define PS_PWR_GATE_SETTLING_TIME_32 (0 << 3) ++#define PS_PWR_GATE_SETTLING_TIME_64 (1 << 3) ++#define PS_PWR_GATE_SETTLING_TIME_96 (2 << 3) ++#define PS_PWR_GATE_SETTLING_TIME_128 (3 << 3) ++#define PS_PWR_GATE_SLPEN_8 0 ++#define PS_PWR_GATE_SLPEN_16 1 ++#define PS_PWR_GATE_SLPEN_24 2 ++#define PS_PWR_GATE_SLPEN_32 3 ++ ++#define _PS_WIN_POS_1A 0x68170 ++#define _PS_WIN_POS_2A 0x68270 ++#define _PS_WIN_POS_1B 0x68970 ++#define _PS_WIN_POS_2B 0x68A70 ++#define _PS_WIN_POS_1C 0x69170 ++ ++#define _PS_WIN_SZ_1A 0x68174 ++#define _PS_WIN_SZ_2A 0x68274 ++#define _PS_WIN_SZ_1B 0x68974 ++#define _PS_WIN_SZ_2B 0x68A74 ++#define _PS_WIN_SZ_1C 0x69174 ++ ++#define _PS_VSCALE_1A 0x68184 ++#define _PS_VSCALE_2A 0x68284 ++#define _PS_VSCALE_1B 0x68984 ++#define _PS_VSCALE_2B 0x68A84 ++#define _PS_VSCALE_1C 0x69184 ++ ++#define _PS_HSCALE_1A 0x68190 ++#define _PS_HSCALE_2A 0x68290 ++#define _PS_HSCALE_1B 0x68990 ++#define _PS_HSCALE_2B 0x68A90 ++#define _PS_HSCALE_1C 0x69190 ++ ++#define _PS_VPHASE_1A 0x68188 ++#define _PS_VPHASE_2A 0x68288 ++#define _PS_VPHASE_1B 0x68988 ++#define _PS_VPHASE_2B 0x68A88 ++#define _PS_VPHASE_1C 0x69188 ++#define PS_Y_PHASE(x) ((x) << 16) ++#define PS_UV_RGB_PHASE(x) ((x) << 0) ++#define PS_PHASE_MASK (0x7fff << 1) /* u2.13 */ ++#define PS_PHASE_TRIP (1 << 0) ++ ++#define _PS_HPHASE_1A 0x68194 ++#define _PS_HPHASE_2A 0x68294 ++#define _PS_HPHASE_1B 0x68994 ++#define _PS_HPHASE_2B 0x68A94 ++#define _PS_HPHASE_1C 0x69194 ++ ++#define _PS_ECC_STAT_1A 0x681D0 ++#define _PS_ECC_STAT_2A 0x682D0 ++#define _PS_ECC_STAT_1B 0x689D0 ++#define _PS_ECC_STAT_2B 0x68AD0 ++#define _PS_ECC_STAT_1C 0x691D0 ++ ++#define _ID(id, a, b) _PICK_EVEN(id, a, b) ++#define SKL_PS_CTRL(pipe, id) _MMIO_PIPE(pipe, \ ++ _ID(id, _PS_1A_CTRL, _PS_2A_CTRL), \ ++ _ID(id, _PS_1B_CTRL, _PS_2B_CTRL)) ++#define SKL_PS_PWR_GATE(pipe, id) _MMIO_PIPE(pipe, \ ++ _ID(id, _PS_PWR_GATE_1A, _PS_PWR_GATE_2A), \ ++ _ID(id, _PS_PWR_GATE_1B, _PS_PWR_GATE_2B)) ++#define SKL_PS_WIN_POS(pipe, id) _MMIO_PIPE(pipe, \ ++ _ID(id, _PS_WIN_POS_1A, _PS_WIN_POS_2A), \ ++ _ID(id, _PS_WIN_POS_1B, _PS_WIN_POS_2B)) ++#define SKL_PS_WIN_SZ(pipe, id) _MMIO_PIPE(pipe, \ ++ _ID(id, _PS_WIN_SZ_1A, _PS_WIN_SZ_2A), \ ++ _ID(id, _PS_WIN_SZ_1B, _PS_WIN_SZ_2B)) ++#define SKL_PS_VSCALE(pipe, id) _MMIO_PIPE(pipe, \ ++ _ID(id, _PS_VSCALE_1A, _PS_VSCALE_2A), \ ++ _ID(id, _PS_VSCALE_1B, _PS_VSCALE_2B)) ++#define SKL_PS_HSCALE(pipe, id) _MMIO_PIPE(pipe, \ ++ _ID(id, _PS_HSCALE_1A, _PS_HSCALE_2A), \ ++ _ID(id, _PS_HSCALE_1B, _PS_HSCALE_2B)) ++#define SKL_PS_VPHASE(pipe, id) _MMIO_PIPE(pipe, \ ++ _ID(id, _PS_VPHASE_1A, _PS_VPHASE_2A), \ ++ _ID(id, _PS_VPHASE_1B, _PS_VPHASE_2B)) ++#define SKL_PS_HPHASE(pipe, id) _MMIO_PIPE(pipe, \ ++ _ID(id, _PS_HPHASE_1A, _PS_HPHASE_2A), \ ++ _ID(id, _PS_HPHASE_1B, _PS_HPHASE_2B)) ++#define SKL_PS_ECC_STAT(pipe, id) _MMIO_PIPE(pipe, \ ++ _ID(id, _PS_ECC_STAT_1A, _PS_ECC_STAT_2A), \ ++ _ID(id, _PS_ECC_STAT_1B, _PS_ECC_STAT_2B)) ++ ++/* legacy palette */ ++#define _LGC_PALETTE_A 0x4a000 ++#define _LGC_PALETTE_B 0x4a800 ++#define LGC_PALETTE(pipe, i) _MMIO(_PIPE(pipe, _LGC_PALETTE_A, _LGC_PALETTE_B) + (i) * 4) ++ ++/* ilk/snb precision palette */ ++#define _PREC_PALETTE_A 0x4b000 ++#define _PREC_PALETTE_B 0x4c000 ++#define PREC_PALETTE(pipe, i) _MMIO(_PIPE(pipe, _PREC_PALETTE_A, _PREC_PALETTE_B) + (i) * 4) ++ ++#define _PREC_PIPEAGCMAX 0x4d000 ++#define _PREC_PIPEBGCMAX 0x4d010 ++#define PREC_PIPEGCMAX(pipe, i) _MMIO(_PIPE(pipe, _PIPEAGCMAX, _PIPEBGCMAX) + (i) * 4) ++ ++#define _GAMMA_MODE_A 0x4a480 ++#define _GAMMA_MODE_B 0x4ac80 ++#define GAMMA_MODE(pipe) _MMIO_PIPE(pipe, _GAMMA_MODE_A, _GAMMA_MODE_B) ++#define PRE_CSC_GAMMA_ENABLE (1 << 31) ++#define POST_CSC_GAMMA_ENABLE (1 << 30) ++#define GAMMA_MODE_MODE_MASK (3 << 0) ++#define GAMMA_MODE_MODE_8BIT (0 << 0) ++#define GAMMA_MODE_MODE_10BIT (1 << 0) ++#define GAMMA_MODE_MODE_12BIT (2 << 0) ++#define GAMMA_MODE_MODE_SPLIT (3 << 0) ++ ++/* DMC/CSR */ ++#define CSR_PROGRAM(i) _MMIO(0x80000 + (i) * 4) ++#define CSR_SSP_BASE_ADDR_GEN9 0x00002FC0 ++#define CSR_HTP_ADDR_SKL 0x00500034 ++#define CSR_SSP_BASE _MMIO(0x8F074) ++#define CSR_HTP_SKL _MMIO(0x8F004) ++#define CSR_LAST_WRITE _MMIO(0x8F034) ++#define CSR_LAST_WRITE_VALUE 0xc003b400 ++/* MMIO address range for CSR program (0x80000 - 0x82FFF) */ ++#define CSR_MMIO_START_RANGE 0x80000 ++#define CSR_MMIO_END_RANGE 0x8FFFF ++#define SKL_CSR_DC3_DC5_COUNT _MMIO(0x80030) ++#define SKL_CSR_DC5_DC6_COUNT _MMIO(0x8002C) ++#define BXT_CSR_DC3_DC5_COUNT _MMIO(0x80038) ++ ++/* interrupts */ ++#define DE_MASTER_IRQ_CONTROL (1 << 31) ++#define DE_SPRITEB_FLIP_DONE (1 << 29) ++#define DE_SPRITEA_FLIP_DONE (1 << 28) ++#define DE_PLANEB_FLIP_DONE (1 << 27) ++#define DE_PLANEA_FLIP_DONE (1 << 26) ++#define DE_PLANE_FLIP_DONE(plane) (1 << (26 + (plane))) ++#define DE_PCU_EVENT (1 << 25) ++#define DE_GTT_FAULT (1 << 24) ++#define DE_POISON (1 << 23) ++#define DE_PERFORM_COUNTER (1 << 22) ++#define DE_PCH_EVENT (1 << 21) ++#define DE_AUX_CHANNEL_A (1 << 20) ++#define DE_DP_A_HOTPLUG (1 << 19) ++#define DE_GSE (1 << 18) ++#define DE_PIPEB_VBLANK (1 << 15) ++#define DE_PIPEB_EVEN_FIELD (1 << 14) ++#define DE_PIPEB_ODD_FIELD (1 << 13) ++#define DE_PIPEB_LINE_COMPARE (1 << 12) ++#define DE_PIPEB_VSYNC (1 << 11) ++#define DE_PIPEB_CRC_DONE (1 << 10) ++#define DE_PIPEB_FIFO_UNDERRUN (1 << 8) ++#define DE_PIPEA_VBLANK (1 << 7) ++#define DE_PIPE_VBLANK(pipe) (1 << (7 + 8 * (pipe))) ++#define DE_PIPEA_EVEN_FIELD (1 << 6) ++#define DE_PIPEA_ODD_FIELD (1 << 5) ++#define DE_PIPEA_LINE_COMPARE (1 << 4) ++#define DE_PIPEA_VSYNC (1 << 3) ++#define DE_PIPEA_CRC_DONE (1 << 2) ++#define DE_PIPE_CRC_DONE(pipe) (1 << (2 + 8 * (pipe))) ++#define DE_PIPEA_FIFO_UNDERRUN (1 << 0) ++#define DE_PIPE_FIFO_UNDERRUN(pipe) (1 << (8 * (pipe))) ++ ++/* More Ivybridge lolz */ ++#define DE_ERR_INT_IVB (1 << 30) ++#define DE_GSE_IVB (1 << 29) ++#define DE_PCH_EVENT_IVB (1 << 28) ++#define DE_DP_A_HOTPLUG_IVB (1 << 27) ++#define DE_AUX_CHANNEL_A_IVB (1 << 26) ++#define DE_EDP_PSR_INT_HSW (1 << 19) ++#define DE_SPRITEC_FLIP_DONE_IVB (1 << 14) ++#define DE_PLANEC_FLIP_DONE_IVB (1 << 13) ++#define DE_PIPEC_VBLANK_IVB (1 << 10) ++#define DE_SPRITEB_FLIP_DONE_IVB (1 << 9) ++#define DE_PLANEB_FLIP_DONE_IVB (1 << 8) ++#define DE_PIPEB_VBLANK_IVB (1 << 5) ++#define DE_SPRITEA_FLIP_DONE_IVB (1 << 4) ++#define DE_PLANEA_FLIP_DONE_IVB (1 << 3) ++#define DE_PLANE_FLIP_DONE_IVB(plane) (1 << (3 + 5 * (plane))) ++#define DE_PIPEA_VBLANK_IVB (1 << 0) ++#define DE_PIPE_VBLANK_IVB(pipe) (1 << ((pipe) * 5)) ++ ++#define VLV_MASTER_IER _MMIO(0x4400c) /* Gunit master IER */ ++#define MASTER_INTERRUPT_ENABLE (1 << 31) ++ ++#define DEISR _MMIO(0x44000) ++#define DEIMR _MMIO(0x44004) ++#define DEIIR _MMIO(0x44008) ++#define DEIER _MMIO(0x4400c) ++ ++#define GTISR _MMIO(0x44010) ++#define GTIMR _MMIO(0x44014) ++#define GTIIR _MMIO(0x44018) ++#define GTIER _MMIO(0x4401c) ++ ++#define GEN8_MASTER_IRQ _MMIO(0x44200) ++#define GEN8_MASTER_IRQ_CONTROL (1 << 31) ++#define GEN8_PCU_IRQ (1 << 30) ++#define GEN8_DE_PCH_IRQ (1 << 23) ++#define GEN8_DE_MISC_IRQ (1 << 22) ++#define GEN8_DE_PORT_IRQ (1 << 20) ++#define GEN8_DE_PIPE_C_IRQ (1 << 18) ++#define GEN8_DE_PIPE_B_IRQ (1 << 17) ++#define GEN8_DE_PIPE_A_IRQ (1 << 16) ++#define GEN8_DE_PIPE_IRQ(pipe) (1 << (16 + (pipe))) ++#define GEN8_GT_VECS_IRQ (1 << 6) ++#define GEN8_GT_GUC_IRQ (1 << 5) ++#define GEN8_GT_PM_IRQ (1 << 4) ++#define GEN8_GT_VCS1_IRQ (1 << 3) /* NB: VCS2 in bspec! */ ++#define GEN8_GT_VCS0_IRQ (1 << 2) /* NB: VCS1 in bpsec! */ ++#define GEN8_GT_BCS_IRQ (1 << 1) ++#define GEN8_GT_RCS_IRQ (1 << 0) ++ ++#define GEN8_GT_ISR(which) _MMIO(0x44300 + (0x10 * (which))) ++#define GEN8_GT_IMR(which) _MMIO(0x44304 + (0x10 * (which))) ++#define GEN8_GT_IIR(which) _MMIO(0x44308 + (0x10 * (which))) ++#define GEN8_GT_IER(which) _MMIO(0x4430c + (0x10 * (which))) ++ ++#define GEN9_GUC_TO_HOST_INT_EVENT (1 << 31) ++#define GEN9_GUC_EXEC_ERROR_EVENT (1 << 30) ++#define GEN9_GUC_DISPLAY_EVENT (1 << 29) ++#define GEN9_GUC_SEMA_SIGNAL_EVENT (1 << 28) ++#define GEN9_GUC_IOMMU_MSG_EVENT (1 << 27) ++#define GEN9_GUC_DB_RING_EVENT (1 << 26) ++#define GEN9_GUC_DMA_DONE_EVENT (1 << 25) ++#define GEN9_GUC_FATAL_ERROR_EVENT (1 << 24) ++#define GEN9_GUC_NOTIFICATION_EVENT (1 << 23) ++ ++#define GEN8_RCS_IRQ_SHIFT 0 ++#define GEN8_BCS_IRQ_SHIFT 16 ++#define GEN8_VCS0_IRQ_SHIFT 0 /* NB: VCS1 in bspec! */ ++#define GEN8_VCS1_IRQ_SHIFT 16 /* NB: VCS2 in bpsec! */ ++#define GEN8_VECS_IRQ_SHIFT 0 ++#define GEN8_WD_IRQ_SHIFT 16 ++ ++#define GEN8_DE_PIPE_ISR(pipe) _MMIO(0x44400 + (0x10 * (pipe))) ++#define GEN8_DE_PIPE_IMR(pipe) _MMIO(0x44404 + (0x10 * (pipe))) ++#define GEN8_DE_PIPE_IIR(pipe) _MMIO(0x44408 + (0x10 * (pipe))) ++#define GEN8_DE_PIPE_IER(pipe) _MMIO(0x4440c + (0x10 * (pipe))) ++#define GEN8_PIPE_FIFO_UNDERRUN (1 << 31) ++#define GEN8_PIPE_CDCLK_CRC_ERROR (1 << 29) ++#define GEN8_PIPE_CDCLK_CRC_DONE (1 << 28) ++#define GEN8_PIPE_CURSOR_FAULT (1 << 10) ++#define GEN8_PIPE_SPRITE_FAULT (1 << 9) ++#define GEN8_PIPE_PRIMARY_FAULT (1 << 8) ++#define GEN8_PIPE_SPRITE_FLIP_DONE (1 << 5) ++#define GEN8_PIPE_PRIMARY_FLIP_DONE (1 << 4) ++#define GEN8_PIPE_SCAN_LINE_EVENT (1 << 2) ++#define GEN8_PIPE_VSYNC (1 << 1) ++#define GEN8_PIPE_VBLANK (1 << 0) ++#define GEN9_PIPE_CURSOR_FAULT (1 << 11) ++#define GEN9_PIPE_PLANE4_FAULT (1 << 10) ++#define GEN9_PIPE_PLANE3_FAULT (1 << 9) ++#define GEN9_PIPE_PLANE2_FAULT (1 << 8) ++#define GEN9_PIPE_PLANE1_FAULT (1 << 7) ++#define GEN9_PIPE_PLANE4_FLIP_DONE (1 << 6) ++#define GEN9_PIPE_PLANE3_FLIP_DONE (1 << 5) ++#define GEN9_PIPE_PLANE2_FLIP_DONE (1 << 4) ++#define GEN9_PIPE_PLANE1_FLIP_DONE (1 << 3) ++#define GEN9_PIPE_PLANE_FLIP_DONE(p) (1 << (3 + (p))) ++#define GEN8_DE_PIPE_IRQ_FAULT_ERRORS \ ++ (GEN8_PIPE_CURSOR_FAULT | \ ++ GEN8_PIPE_SPRITE_FAULT | \ ++ GEN8_PIPE_PRIMARY_FAULT) ++#define GEN9_DE_PIPE_IRQ_FAULT_ERRORS \ ++ (GEN9_PIPE_CURSOR_FAULT | \ ++ GEN9_PIPE_PLANE4_FAULT | \ ++ GEN9_PIPE_PLANE3_FAULT | \ ++ GEN9_PIPE_PLANE2_FAULT | \ ++ GEN9_PIPE_PLANE1_FAULT) ++ ++#define GEN8_DE_PORT_ISR _MMIO(0x44440) ++#define GEN8_DE_PORT_IMR _MMIO(0x44444) ++#define GEN8_DE_PORT_IIR _MMIO(0x44448) ++#define GEN8_DE_PORT_IER _MMIO(0x4444c) ++#define ICL_AUX_CHANNEL_E (1 << 29) ++#define CNL_AUX_CHANNEL_F (1 << 28) ++#define GEN9_AUX_CHANNEL_D (1 << 27) ++#define GEN9_AUX_CHANNEL_C (1 << 26) ++#define GEN9_AUX_CHANNEL_B (1 << 25) ++#define BXT_DE_PORT_HP_DDIC (1 << 5) ++#define BXT_DE_PORT_HP_DDIB (1 << 4) ++#define BXT_DE_PORT_HP_DDIA (1 << 3) ++#define BXT_DE_PORT_HOTPLUG_MASK (BXT_DE_PORT_HP_DDIA | \ ++ BXT_DE_PORT_HP_DDIB | \ ++ BXT_DE_PORT_HP_DDIC) ++#define GEN8_PORT_DP_A_HOTPLUG (1 << 3) ++#define BXT_DE_PORT_GMBUS (1 << 1) ++#define GEN8_AUX_CHANNEL_A (1 << 0) ++ ++#define GEN8_DE_MISC_ISR _MMIO(0x44460) ++#define GEN8_DE_MISC_IMR _MMIO(0x44464) ++#define GEN8_DE_MISC_IIR _MMIO(0x44468) ++#define GEN8_DE_MISC_IER _MMIO(0x4446c) ++#define GEN8_DE_MISC_GSE (1 << 27) ++#define GEN8_DE_EDP_PSR (1 << 19) ++ ++#define GEN8_PCU_ISR _MMIO(0x444e0) ++#define GEN8_PCU_IMR _MMIO(0x444e4) ++#define GEN8_PCU_IIR _MMIO(0x444e8) ++#define GEN8_PCU_IER _MMIO(0x444ec) ++ ++#define GEN11_GU_MISC_ISR _MMIO(0x444f0) ++#define GEN11_GU_MISC_IMR _MMIO(0x444f4) ++#define GEN11_GU_MISC_IIR _MMIO(0x444f8) ++#define GEN11_GU_MISC_IER _MMIO(0x444fc) ++#define GEN11_GU_MISC_GSE (1 << 27) ++ ++#define GEN11_GFX_MSTR_IRQ _MMIO(0x190010) ++#define GEN11_MASTER_IRQ (1 << 31) ++#define GEN11_PCU_IRQ (1 << 30) ++#define GEN11_GU_MISC_IRQ (1 << 29) ++#define GEN11_DISPLAY_IRQ (1 << 16) ++#define GEN11_GT_DW_IRQ(x) (1 << (x)) ++#define GEN11_GT_DW1_IRQ (1 << 1) ++#define GEN11_GT_DW0_IRQ (1 << 0) ++ ++#define GEN11_DISPLAY_INT_CTL _MMIO(0x44200) ++#define GEN11_DISPLAY_IRQ_ENABLE (1 << 31) ++#define GEN11_AUDIO_CODEC_IRQ (1 << 24) ++#define GEN11_DE_PCH_IRQ (1 << 23) ++#define GEN11_DE_MISC_IRQ (1 << 22) ++#define GEN11_DE_HPD_IRQ (1 << 21) ++#define GEN11_DE_PORT_IRQ (1 << 20) ++#define GEN11_DE_PIPE_C (1 << 18) ++#define GEN11_DE_PIPE_B (1 << 17) ++#define GEN11_DE_PIPE_A (1 << 16) ++ ++#define GEN11_DE_HPD_ISR _MMIO(0x44470) ++#define GEN11_DE_HPD_IMR _MMIO(0x44474) ++#define GEN11_DE_HPD_IIR _MMIO(0x44478) ++#define GEN11_DE_HPD_IER _MMIO(0x4447c) ++#define GEN11_TC4_HOTPLUG (1 << 19) ++#define GEN11_TC3_HOTPLUG (1 << 18) ++#define GEN11_TC2_HOTPLUG (1 << 17) ++#define GEN11_TC1_HOTPLUG (1 << 16) ++#define GEN11_TC_HOTPLUG(tc_port) (1 << ((tc_port) + 16)) ++#define GEN11_DE_TC_HOTPLUG_MASK (GEN11_TC4_HOTPLUG | \ ++ GEN11_TC3_HOTPLUG | \ ++ GEN11_TC2_HOTPLUG | \ ++ GEN11_TC1_HOTPLUG) ++#define GEN11_TBT4_HOTPLUG (1 << 3) ++#define GEN11_TBT3_HOTPLUG (1 << 2) ++#define GEN11_TBT2_HOTPLUG (1 << 1) ++#define GEN11_TBT1_HOTPLUG (1 << 0) ++#define GEN11_TBT_HOTPLUG(tc_port) (1 << (tc_port)) ++#define GEN11_DE_TBT_HOTPLUG_MASK (GEN11_TBT4_HOTPLUG | \ ++ GEN11_TBT3_HOTPLUG | \ ++ GEN11_TBT2_HOTPLUG | \ ++ GEN11_TBT1_HOTPLUG) ++ ++#define GEN11_TBT_HOTPLUG_CTL _MMIO(0x44030) ++#define GEN11_TC_HOTPLUG_CTL _MMIO(0x44038) ++#define GEN11_HOTPLUG_CTL_ENABLE(tc_port) (8 << (tc_port) * 4) ++#define GEN11_HOTPLUG_CTL_LONG_DETECT(tc_port) (2 << (tc_port) * 4) ++#define GEN11_HOTPLUG_CTL_SHORT_DETECT(tc_port) (1 << (tc_port) * 4) ++#define GEN11_HOTPLUG_CTL_NO_DETECT(tc_port) (0 << (tc_port) * 4) ++ ++#define GEN11_GT_INTR_DW0 _MMIO(0x190018) ++#define GEN11_CSME (31) ++#define GEN11_GUNIT (28) ++#define GEN11_GUC (25) ++#define GEN11_WDPERF (20) ++#define GEN11_KCR (19) ++#define GEN11_GTPM (16) ++#define GEN11_BCS (15) ++#define GEN11_RCS0 (0) ++ ++#define GEN11_GT_INTR_DW1 _MMIO(0x19001c) ++#define GEN11_VECS(x) (31 - (x)) ++#define GEN11_VCS(x) (x) ++ ++#define GEN11_GT_INTR_DW(x) _MMIO(0x190018 + ((x) * 4)) ++ ++#define GEN11_INTR_IDENTITY_REG0 _MMIO(0x190060) ++#define GEN11_INTR_IDENTITY_REG1 _MMIO(0x190064) ++#define GEN11_INTR_DATA_VALID (1 << 31) ++#define GEN11_INTR_ENGINE_CLASS(x) (((x) & GENMASK(18, 16)) >> 16) ++#define GEN11_INTR_ENGINE_INSTANCE(x) (((x) & GENMASK(25, 20)) >> 20) ++#define GEN11_INTR_ENGINE_INTR(x) ((x) & 0xffff) ++ ++#define GEN11_INTR_IDENTITY_REG(x) _MMIO(0x190060 + ((x) * 4)) ++ ++#define GEN11_IIR_REG0_SELECTOR _MMIO(0x190070) ++#define GEN11_IIR_REG1_SELECTOR _MMIO(0x190074) ++ ++#define GEN11_IIR_REG_SELECTOR(x) _MMIO(0x190070 + ((x) * 4)) ++ ++#define GEN11_RENDER_COPY_INTR_ENABLE _MMIO(0x190030) ++#define GEN11_VCS_VECS_INTR_ENABLE _MMIO(0x190034) ++#define GEN11_GUC_SG_INTR_ENABLE _MMIO(0x190038) ++#define GEN11_GPM_WGBOXPERF_INTR_ENABLE _MMIO(0x19003c) ++#define GEN11_CRYPTO_RSVD_INTR_ENABLE _MMIO(0x190040) ++#define GEN11_GUNIT_CSME_INTR_ENABLE _MMIO(0x190044) ++ ++#define GEN11_RCS0_RSVD_INTR_MASK _MMIO(0x190090) ++#define GEN11_BCS_RSVD_INTR_MASK _MMIO(0x1900a0) ++#define GEN11_VCS0_VCS1_INTR_MASK _MMIO(0x1900a8) ++#define GEN11_VCS2_VCS3_INTR_MASK _MMIO(0x1900ac) ++#define GEN11_VECS0_VECS1_INTR_MASK _MMIO(0x1900d0) ++#define GEN11_GUC_SG_INTR_MASK _MMIO(0x1900e8) ++#define GEN11_GPM_WGBOXPERF_INTR_MASK _MMIO(0x1900ec) ++#define GEN11_CRYPTO_RSVD_INTR_MASK _MMIO(0x1900f0) ++#define GEN11_GUNIT_CSME_INTR_MASK _MMIO(0x1900f4) ++ ++#define ILK_DISPLAY_CHICKEN2 _MMIO(0x42004) ++/* Required on all Ironlake and Sandybridge according to the B-Spec. */ ++#define ILK_ELPIN_409_SELECT (1 << 25) ++#define ILK_DPARB_GATE (1 << 22) ++#define ILK_VSDPFD_FULL (1 << 21) ++#define FUSE_STRAP _MMIO(0x42014) ++#define ILK_INTERNAL_GRAPHICS_DISABLE (1 << 31) ++#define ILK_INTERNAL_DISPLAY_DISABLE (1 << 30) ++#define ILK_DISPLAY_DEBUG_DISABLE (1 << 29) ++#define IVB_PIPE_C_DISABLE (1 << 28) ++#define ILK_HDCP_DISABLE (1 << 25) ++#define ILK_eDP_A_DISABLE (1 << 24) ++#define HSW_CDCLK_LIMIT (1 << 24) ++#define ILK_DESKTOP (1 << 23) ++ ++#define ILK_DSPCLK_GATE_D _MMIO(0x42020) ++#define ILK_VRHUNIT_CLOCK_GATE_DISABLE (1 << 28) ++#define ILK_DPFCUNIT_CLOCK_GATE_DISABLE (1 << 9) ++#define ILK_DPFCRUNIT_CLOCK_GATE_DISABLE (1 << 8) ++#define ILK_DPFDUNIT_CLOCK_GATE_ENABLE (1 << 7) ++#define ILK_DPARBUNIT_CLOCK_GATE_ENABLE (1 << 5) ++ ++#define IVB_CHICKEN3 _MMIO(0x4200c) ++# define CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE (1 << 5) ++# define CHICKEN3_DGMG_DONE_FIX_DISABLE (1 << 2) ++ ++#define CHICKEN_PAR1_1 _MMIO(0x42080) ++#define SKL_DE_COMPRESSED_HASH_MODE (1 << 15) ++#define DPA_MASK_VBLANK_SRD (1 << 15) ++#define FORCE_ARB_IDLE_PLANES (1 << 14) ++#define SKL_EDP_PSR_FIX_RDWRAP (1 << 3) ++ ++#define CHICKEN_PAR2_1 _MMIO(0x42090) ++#define KVM_CONFIG_CHANGE_NOTIFICATION_SELECT (1 << 14) ++ ++#define CHICKEN_MISC_2 _MMIO(0x42084) ++#define CNL_COMP_PWR_DOWN (1 << 23) ++#define GLK_CL2_PWR_DOWN (1 << 12) ++#define GLK_CL1_PWR_DOWN (1 << 11) ++#define GLK_CL0_PWR_DOWN (1 << 10) ++ ++#define CHICKEN_MISC_4 _MMIO(0x4208c) ++#define FBC_STRIDE_OVERRIDE (1 << 13) ++#define FBC_STRIDE_MASK 0x1FFF ++ ++#define _CHICKEN_PIPESL_1_A 0x420b0 ++#define _CHICKEN_PIPESL_1_B 0x420b4 ++#define HSW_FBCQ_DIS (1 << 22) ++#define BDW_DPRS_MASK_VBLANK_SRD (1 << 0) ++#define CHICKEN_PIPESL_1(pipe) _MMIO_PIPE(pipe, _CHICKEN_PIPESL_1_A, _CHICKEN_PIPESL_1_B) ++ ++#define CHICKEN_TRANS_A _MMIO(0x420c0) ++#define CHICKEN_TRANS_B _MMIO(0x420c4) ++#define CHICKEN_TRANS_C _MMIO(0x420c8) ++#define CHICKEN_TRANS_EDP _MMIO(0x420cc) ++#define VSC_DATA_SEL_SOFTWARE_CONTROL (1 << 25) /* GLK and CNL+ */ ++#define DDI_TRAINING_OVERRIDE_ENABLE (1 << 19) ++#define DDI_TRAINING_OVERRIDE_VALUE (1 << 18) ++#define DDIE_TRAINING_OVERRIDE_ENABLE (1 << 17) /* CHICKEN_TRANS_A only */ ++#define DDIE_TRAINING_OVERRIDE_VALUE (1 << 16) /* CHICKEN_TRANS_A only */ ++#define PSR2_ADD_VERTICAL_LINE_COUNT (1 << 15) ++#define PSR2_VSC_ENABLE_PROG_HEADER (1 << 12) ++ ++#define DISP_ARB_CTL _MMIO(0x45000) ++#define DISP_FBC_MEMORY_WAKE (1 << 31) ++#define DISP_TILE_SURFACE_SWIZZLING (1 << 13) ++#define DISP_FBC_WM_DIS (1 << 15) ++#define DISP_ARB_CTL2 _MMIO(0x45004) ++#define DISP_DATA_PARTITION_5_6 (1 << 6) ++#define DISP_IPC_ENABLE (1 << 3) ++#define DBUF_CTL _MMIO(0x45008) ++#define DBUF_CTL_S1 _MMIO(0x45008) ++#define DBUF_CTL_S2 _MMIO(0x44FE8) ++#define DBUF_POWER_REQUEST (1 << 31) ++#define DBUF_POWER_STATE (1 << 30) ++#define GEN7_MSG_CTL _MMIO(0x45010) ++#define WAIT_FOR_PCH_RESET_ACK (1 << 1) ++#define WAIT_FOR_PCH_FLR_ACK (1 << 0) ++#define HSW_NDE_RSTWRN_OPT _MMIO(0x46408) ++#define RESET_PCH_HANDSHAKE_ENABLE (1 << 4) ++ ++#define GEN8_CHICKEN_DCPR_1 _MMIO(0x46430) ++#define SKL_SELECT_ALTERNATE_DC_EXIT (1 << 30) ++#define MASK_WAKEMEM (1 << 13) ++#define CNL_DDI_CLOCK_REG_ACCESS_ON (1 << 7) ++ ++#define SKL_DFSM _MMIO(0x51000) ++#define SKL_DFSM_CDCLK_LIMIT_MASK (3 << 23) ++#define SKL_DFSM_CDCLK_LIMIT_675 (0 << 23) ++#define SKL_DFSM_CDCLK_LIMIT_540 (1 << 23) ++#define SKL_DFSM_CDCLK_LIMIT_450 (2 << 23) ++#define SKL_DFSM_CDCLK_LIMIT_337_5 (3 << 23) ++#define SKL_DFSM_PIPE_A_DISABLE (1 << 30) ++#define SKL_DFSM_PIPE_B_DISABLE (1 << 21) ++#define SKL_DFSM_PIPE_C_DISABLE (1 << 28) ++ ++#define SKL_DSSM _MMIO(0x51004) ++#define CNL_DSSM_CDCLK_PLL_REFCLK_24MHz (1 << 31) ++#define ICL_DSSM_CDCLK_PLL_REFCLK_MASK (7 << 29) ++#define ICL_DSSM_CDCLK_PLL_REFCLK_24MHz (0 << 29) ++#define ICL_DSSM_CDCLK_PLL_REFCLK_19_2MHz (1 << 29) ++#define ICL_DSSM_CDCLK_PLL_REFCLK_38_4MHz (2 << 29) ++ ++#define GEN7_FF_SLICE_CS_CHICKEN1 _MMIO(0x20e0) ++#define GEN9_FFSC_PERCTX_PREEMPT_CTRL (1 << 14) ++ ++#define FF_SLICE_CS_CHICKEN2 _MMIO(0x20e4) ++#define GEN9_TSG_BARRIER_ACK_DISABLE (1 << 8) ++#define GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE (1 << 10) ++ ++#define GEN9_CS_DEBUG_MODE1 _MMIO(0x20ec) ++#define GEN9_CTX_PREEMPT_REG _MMIO(0x2248) ++#define GEN8_CS_CHICKEN1 _MMIO(0x2580) ++#define GEN9_PREEMPT_3D_OBJECT_LEVEL (1 << 0) ++#define GEN9_PREEMPT_GPGPU_LEVEL(hi, lo) (((hi) << 2) | ((lo) << 1)) ++#define GEN9_PREEMPT_GPGPU_MID_THREAD_LEVEL GEN9_PREEMPT_GPGPU_LEVEL(0, 0) ++#define GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL GEN9_PREEMPT_GPGPU_LEVEL(0, 1) ++#define GEN9_PREEMPT_GPGPU_COMMAND_LEVEL GEN9_PREEMPT_GPGPU_LEVEL(1, 0) ++#define GEN9_PREEMPT_GPGPU_LEVEL_MASK GEN9_PREEMPT_GPGPU_LEVEL(1, 1) ++ ++/* GEN7 chicken */ ++#define GEN7_COMMON_SLICE_CHICKEN1 _MMIO(0x7010) ++ #define GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC ((1 << 10) | (1 << 26)) ++ #define GEN9_RHWO_OPTIMIZATION_DISABLE (1 << 14) ++ ++#define COMMON_SLICE_CHICKEN2 _MMIO(0x7014) ++ #define GEN9_PBE_COMPRESSED_HASH_SELECTION (1 << 13) ++ #define GEN9_DISABLE_GATHER_AT_SET_SHADER_COMMON_SLICE (1 << 12) ++ #define GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION (1 << 8) ++ #define GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE (1 << 0) ++ ++#define GEN8_L3CNTLREG _MMIO(0x7034) ++ #define GEN8_ERRDETBCTRL (1 << 9) ++ ++#define GEN11_COMMON_SLICE_CHICKEN3 _MMIO(0x7304) ++ #define GEN11_BLEND_EMB_FIX_DISABLE_IN_RCC (1 << 11) ++ ++#define HIZ_CHICKEN _MMIO(0x7018) ++# define CHV_HZ_8X8_MODE_IN_1X (1 << 15) ++# define BDW_HIZ_POWER_COMPILER_CLOCK_GATING_DISABLE (1 << 3) ++ ++#define GEN9_SLICE_COMMON_ECO_CHICKEN0 _MMIO(0x7308) ++#define DISABLE_PIXEL_MASK_CAMMING (1 << 14) ++ ++#define GEN9_SLICE_COMMON_ECO_CHICKEN1 _MMIO(0x731c) ++#define GEN11_STATE_CACHE_REDIRECT_TO_CS (1 << 11) ++ ++#define GEN7_SARCHKMD _MMIO(0xB000) ++#define GEN7_DISABLE_DEMAND_PREFETCH (1 << 31) ++#define GEN7_DISABLE_SAMPLER_PREFETCH (1 << 30) ++ ++#define GEN7_L3SQCREG1 _MMIO(0xB010) ++#define VLV_B0_WA_L3SQCREG1_VALUE 0x00D30000 ++ ++#define GEN8_L3SQCREG1 _MMIO(0xB100) ++/* ++ * Note that on CHV the following has an off-by-one error wrt. to BSpec. ++ * Using the formula in BSpec leads to a hang, while the formula here works ++ * fine and matches the formulas for all other platforms. A BSpec change ++ * request has been filed to clarify this. ++ */ ++#define L3_GENERAL_PRIO_CREDITS(x) (((x) >> 1) << 19) ++#define L3_HIGH_PRIO_CREDITS(x) (((x) >> 1) << 14) ++#define L3_PRIO_CREDITS_MASK ((0x1f << 19) | (0x1f << 14)) ++ ++#define GEN7_L3CNTLREG1 _MMIO(0xB01C) ++#define GEN7_WA_FOR_GEN7_L3_CONTROL 0x3C47FF8C ++#define GEN7_L3AGDIS (1 << 19) ++#define GEN7_L3CNTLREG2 _MMIO(0xB020) ++#define GEN7_L3CNTLREG3 _MMIO(0xB024) ++ ++#define GEN7_L3_CHICKEN_MODE_REGISTER _MMIO(0xB030) ++#define GEN7_WA_L3_CHICKEN_MODE 0x20000000 ++#define GEN10_L3_CHICKEN_MODE_REGISTER _MMIO(0xB114) ++#define GEN11_I2M_WRITE_DISABLE (1 << 28) ++ ++#define GEN7_L3SQCREG4 _MMIO(0xb034) ++#define L3SQ_URB_READ_CAM_MATCH_DISABLE (1 << 27) ++ ++#define GEN8_L3SQCREG4 _MMIO(0xb118) ++#define GEN11_LQSC_CLEAN_EVICT_DISABLE (1 << 6) ++#define GEN8_LQSC_RO_PERF_DIS (1 << 27) ++#define GEN8_LQSC_FLUSH_COHERENT_LINES (1 << 21) ++ ++/* GEN8 chicken */ ++#define HDC_CHICKEN0 _MMIO(0x7300) ++#define CNL_HDC_CHICKEN0 _MMIO(0xE5F0) ++#define ICL_HDC_MODE _MMIO(0xE5F4) ++#define HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE (1 << 15) ++#define HDC_FENCE_DEST_SLM_DISABLE (1 << 14) ++#define HDC_DONOT_FETCH_MEM_WHEN_MASKED (1 << 11) ++#define HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT (1 << 5) ++#define HDC_FORCE_NON_COHERENT (1 << 4) ++#define HDC_BARRIER_PERFORMANCE_DISABLE (1 << 10) ++ ++#define GEN8_HDC_CHICKEN1 _MMIO(0x7304) ++ ++/* GEN9 chicken */ ++#define SLICE_ECO_CHICKEN0 _MMIO(0x7308) ++#define PIXEL_MASK_CAMMING_DISABLE (1 << 14) ++ ++#define GEN9_WM_CHICKEN3 _MMIO(0x5588) ++#define GEN9_FACTOR_IN_CLR_VAL_HIZ (1 << 9) ++ ++/* WaCatErrorRejectionIssue */ ++#define GEN7_SQ_CHICKEN_MBCUNIT_CONFIG _MMIO(0x9030) ++#define GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB (1 << 11) ++ ++#define HSW_SCRATCH1 _MMIO(0xb038) ++#define HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE (1 << 27) ++ ++#define BDW_SCRATCH1 _MMIO(0xb11c) ++#define GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE (1 << 2) ++ ++/*GEN11 chicken */ ++#define _PIPEA_CHICKEN 0x70038 ++#define _PIPEB_CHICKEN 0x71038 ++#define _PIPEC_CHICKEN 0x72038 ++#define PIPE_CHICKEN(pipe) _MMIO_PIPE(pipe, _PIPEA_CHICKEN,\ ++ _PIPEB_CHICKEN) ++#define PIXEL_ROUNDING_TRUNC_FB_PASSTHRU (1 << 15) ++#define PER_PIXEL_ALPHA_BYPASS_EN (1 << 7) ++ ++/* PCH */ ++ ++#define PCH_DISPLAY_BASE 0xc0000u ++ ++/* south display engine interrupt: IBX */ ++#define SDE_AUDIO_POWER_D (1 << 27) ++#define SDE_AUDIO_POWER_C (1 << 26) ++#define SDE_AUDIO_POWER_B (1 << 25) ++#define SDE_AUDIO_POWER_SHIFT (25) ++#define SDE_AUDIO_POWER_MASK (7 << SDE_AUDIO_POWER_SHIFT) ++#define SDE_GMBUS (1 << 24) ++#define SDE_AUDIO_HDCP_TRANSB (1 << 23) ++#define SDE_AUDIO_HDCP_TRANSA (1 << 22) ++#define SDE_AUDIO_HDCP_MASK (3 << 22) ++#define SDE_AUDIO_TRANSB (1 << 21) ++#define SDE_AUDIO_TRANSA (1 << 20) ++#define SDE_AUDIO_TRANS_MASK (3 << 20) ++#define SDE_POISON (1 << 19) ++/* 18 reserved */ ++#define SDE_FDI_RXB (1 << 17) ++#define SDE_FDI_RXA (1 << 16) ++#define SDE_FDI_MASK (3 << 16) ++#define SDE_AUXD (1 << 15) ++#define SDE_AUXC (1 << 14) ++#define SDE_AUXB (1 << 13) ++#define SDE_AUX_MASK (7 << 13) ++/* 12 reserved */ ++#define SDE_CRT_HOTPLUG (1 << 11) ++#define SDE_PORTD_HOTPLUG (1 << 10) ++#define SDE_PORTC_HOTPLUG (1 << 9) ++#define SDE_PORTB_HOTPLUG (1 << 8) ++#define SDE_SDVOB_HOTPLUG (1 << 6) ++#define SDE_HOTPLUG_MASK (SDE_CRT_HOTPLUG | \ ++ SDE_SDVOB_HOTPLUG | \ ++ SDE_PORTB_HOTPLUG | \ ++ SDE_PORTC_HOTPLUG | \ ++ SDE_PORTD_HOTPLUG) ++#define SDE_TRANSB_CRC_DONE (1 << 5) ++#define SDE_TRANSB_CRC_ERR (1 << 4) ++#define SDE_TRANSB_FIFO_UNDER (1 << 3) ++#define SDE_TRANSA_CRC_DONE (1 << 2) ++#define SDE_TRANSA_CRC_ERR (1 << 1) ++#define SDE_TRANSA_FIFO_UNDER (1 << 0) ++#define SDE_TRANS_MASK (0x3f) ++ ++/* south display engine interrupt: CPT - CNP */ ++#define SDE_AUDIO_POWER_D_CPT (1 << 31) ++#define SDE_AUDIO_POWER_C_CPT (1 << 30) ++#define SDE_AUDIO_POWER_B_CPT (1 << 29) ++#define SDE_AUDIO_POWER_SHIFT_CPT 29 ++#define SDE_AUDIO_POWER_MASK_CPT (7 << 29) ++#define SDE_AUXD_CPT (1 << 27) ++#define SDE_AUXC_CPT (1 << 26) ++#define SDE_AUXB_CPT (1 << 25) ++#define SDE_AUX_MASK_CPT (7 << 25) ++#define SDE_PORTE_HOTPLUG_SPT (1 << 25) ++#define SDE_PORTA_HOTPLUG_SPT (1 << 24) ++#define SDE_PORTD_HOTPLUG_CPT (1 << 23) ++#define SDE_PORTC_HOTPLUG_CPT (1 << 22) ++#define SDE_PORTB_HOTPLUG_CPT (1 << 21) ++#define SDE_CRT_HOTPLUG_CPT (1 << 19) ++#define SDE_SDVOB_HOTPLUG_CPT (1 << 18) ++#define SDE_HOTPLUG_MASK_CPT (SDE_CRT_HOTPLUG_CPT | \ ++ SDE_SDVOB_HOTPLUG_CPT | \ ++ SDE_PORTD_HOTPLUG_CPT | \ ++ SDE_PORTC_HOTPLUG_CPT | \ ++ SDE_PORTB_HOTPLUG_CPT) ++#define SDE_HOTPLUG_MASK_SPT (SDE_PORTE_HOTPLUG_SPT | \ ++ SDE_PORTD_HOTPLUG_CPT | \ ++ SDE_PORTC_HOTPLUG_CPT | \ ++ SDE_PORTB_HOTPLUG_CPT | \ ++ SDE_PORTA_HOTPLUG_SPT) ++#define SDE_GMBUS_CPT (1 << 17) ++#define SDE_ERROR_CPT (1 << 16) ++#define SDE_AUDIO_CP_REQ_C_CPT (1 << 10) ++#define SDE_AUDIO_CP_CHG_C_CPT (1 << 9) ++#define SDE_FDI_RXC_CPT (1 << 8) ++#define SDE_AUDIO_CP_REQ_B_CPT (1 << 6) ++#define SDE_AUDIO_CP_CHG_B_CPT (1 << 5) ++#define SDE_FDI_RXB_CPT (1 << 4) ++#define SDE_AUDIO_CP_REQ_A_CPT (1 << 2) ++#define SDE_AUDIO_CP_CHG_A_CPT (1 << 1) ++#define SDE_FDI_RXA_CPT (1 << 0) ++#define SDE_AUDIO_CP_REQ_CPT (SDE_AUDIO_CP_REQ_C_CPT | \ ++ SDE_AUDIO_CP_REQ_B_CPT | \ ++ SDE_AUDIO_CP_REQ_A_CPT) ++#define SDE_AUDIO_CP_CHG_CPT (SDE_AUDIO_CP_CHG_C_CPT | \ ++ SDE_AUDIO_CP_CHG_B_CPT | \ ++ SDE_AUDIO_CP_CHG_A_CPT) ++#define SDE_FDI_MASK_CPT (SDE_FDI_RXC_CPT | \ ++ SDE_FDI_RXB_CPT | \ ++ SDE_FDI_RXA_CPT) ++ ++/* south display engine interrupt: ICP */ ++#define SDE_TC4_HOTPLUG_ICP (1 << 27) ++#define SDE_TC3_HOTPLUG_ICP (1 << 26) ++#define SDE_TC2_HOTPLUG_ICP (1 << 25) ++#define SDE_TC1_HOTPLUG_ICP (1 << 24) ++#define SDE_GMBUS_ICP (1 << 23) ++#define SDE_DDIB_HOTPLUG_ICP (1 << 17) ++#define SDE_DDIA_HOTPLUG_ICP (1 << 16) ++#define SDE_TC_HOTPLUG_ICP(tc_port) (1 << ((tc_port) + 24)) ++#define SDE_DDI_HOTPLUG_ICP(port) (1 << ((port) + 16)) ++#define SDE_DDI_MASK_ICP (SDE_DDIB_HOTPLUG_ICP | \ ++ SDE_DDIA_HOTPLUG_ICP) ++#define SDE_TC_MASK_ICP (SDE_TC4_HOTPLUG_ICP | \ ++ SDE_TC3_HOTPLUG_ICP | \ ++ SDE_TC2_HOTPLUG_ICP | \ ++ SDE_TC1_HOTPLUG_ICP) ++ ++#define SDEISR _MMIO(0xc4000) ++#define SDEIMR _MMIO(0xc4004) ++#define SDEIIR _MMIO(0xc4008) ++#define SDEIER _MMIO(0xc400c) ++ ++#define SERR_INT _MMIO(0xc4040) ++#define SERR_INT_POISON (1 << 31) ++#define SERR_INT_TRANS_FIFO_UNDERRUN(pipe) (1 << ((pipe) * 3)) ++ ++/* digital port hotplug */ ++#define PCH_PORT_HOTPLUG _MMIO(0xc4030) /* SHOTPLUG_CTL */ ++#define PORTA_HOTPLUG_ENABLE (1 << 28) /* LPT:LP+ & BXT */ ++#define BXT_DDIA_HPD_INVERT (1 << 27) ++#define PORTA_HOTPLUG_STATUS_MASK (3 << 24) /* SPT+ & BXT */ ++#define PORTA_HOTPLUG_NO_DETECT (0 << 24) /* SPT+ & BXT */ ++#define PORTA_HOTPLUG_SHORT_DETECT (1 << 24) /* SPT+ & BXT */ ++#define PORTA_HOTPLUG_LONG_DETECT (2 << 24) /* SPT+ & BXT */ ++#define PORTD_HOTPLUG_ENABLE (1 << 20) ++#define PORTD_PULSE_DURATION_2ms (0 << 18) /* pre-LPT */ ++#define PORTD_PULSE_DURATION_4_5ms (1 << 18) /* pre-LPT */ ++#define PORTD_PULSE_DURATION_6ms (2 << 18) /* pre-LPT */ ++#define PORTD_PULSE_DURATION_100ms (3 << 18) /* pre-LPT */ ++#define PORTD_PULSE_DURATION_MASK (3 << 18) /* pre-LPT */ ++#define PORTD_HOTPLUG_STATUS_MASK (3 << 16) ++#define PORTD_HOTPLUG_NO_DETECT (0 << 16) ++#define PORTD_HOTPLUG_SHORT_DETECT (1 << 16) ++#define PORTD_HOTPLUG_LONG_DETECT (2 << 16) ++#define PORTC_HOTPLUG_ENABLE (1 << 12) ++#define BXT_DDIC_HPD_INVERT (1 << 11) ++#define PORTC_PULSE_DURATION_2ms (0 << 10) /* pre-LPT */ ++#define PORTC_PULSE_DURATION_4_5ms (1 << 10) /* pre-LPT */ ++#define PORTC_PULSE_DURATION_6ms (2 << 10) /* pre-LPT */ ++#define PORTC_PULSE_DURATION_100ms (3 << 10) /* pre-LPT */ ++#define PORTC_PULSE_DURATION_MASK (3 << 10) /* pre-LPT */ ++#define PORTC_HOTPLUG_STATUS_MASK (3 << 8) ++#define PORTC_HOTPLUG_NO_DETECT (0 << 8) ++#define PORTC_HOTPLUG_SHORT_DETECT (1 << 8) ++#define PORTC_HOTPLUG_LONG_DETECT (2 << 8) ++#define PORTB_HOTPLUG_ENABLE (1 << 4) ++#define BXT_DDIB_HPD_INVERT (1 << 3) ++#define PORTB_PULSE_DURATION_2ms (0 << 2) /* pre-LPT */ ++#define PORTB_PULSE_DURATION_4_5ms (1 << 2) /* pre-LPT */ ++#define PORTB_PULSE_DURATION_6ms (2 << 2) /* pre-LPT */ ++#define PORTB_PULSE_DURATION_100ms (3 << 2) /* pre-LPT */ ++#define PORTB_PULSE_DURATION_MASK (3 << 2) /* pre-LPT */ ++#define PORTB_HOTPLUG_STATUS_MASK (3 << 0) ++#define PORTB_HOTPLUG_NO_DETECT (0 << 0) ++#define PORTB_HOTPLUG_SHORT_DETECT (1 << 0) ++#define PORTB_HOTPLUG_LONG_DETECT (2 << 0) ++#define BXT_DDI_HPD_INVERT_MASK (BXT_DDIA_HPD_INVERT | \ ++ BXT_DDIB_HPD_INVERT | \ ++ BXT_DDIC_HPD_INVERT) ++ ++#define PCH_PORT_HOTPLUG2 _MMIO(0xc403C) /* SHOTPLUG_CTL2 SPT+ */ ++#define PORTE_HOTPLUG_ENABLE (1 << 4) ++#define PORTE_HOTPLUG_STATUS_MASK (3 << 0) ++#define PORTE_HOTPLUG_NO_DETECT (0 << 0) ++#define PORTE_HOTPLUG_SHORT_DETECT (1 << 0) ++#define PORTE_HOTPLUG_LONG_DETECT (2 << 0) ++ ++/* This register is a reuse of PCH_PORT_HOTPLUG register. The ++ * functionality covered in PCH_PORT_HOTPLUG is split into ++ * SHOTPLUG_CTL_DDI and SHOTPLUG_CTL_TC. ++ */ ++ ++#define SHOTPLUG_CTL_DDI _MMIO(0xc4030) ++#define ICP_DDIB_HPD_ENABLE (1 << 7) ++#define ICP_DDIB_HPD_STATUS_MASK (3 << 4) ++#define ICP_DDIB_HPD_NO_DETECT (0 << 4) ++#define ICP_DDIB_HPD_SHORT_DETECT (1 << 4) ++#define ICP_DDIB_HPD_LONG_DETECT (2 << 4) ++#define ICP_DDIB_HPD_SHORT_LONG_DETECT (3 << 4) ++#define ICP_DDIA_HPD_ENABLE (1 << 3) ++#define ICP_DDIA_HPD_OP_DRIVE_1 (1 << 2) ++#define ICP_DDIA_HPD_STATUS_MASK (3 << 0) ++#define ICP_DDIA_HPD_NO_DETECT (0 << 0) ++#define ICP_DDIA_HPD_SHORT_DETECT (1 << 0) ++#define ICP_DDIA_HPD_LONG_DETECT (2 << 0) ++#define ICP_DDIA_HPD_SHORT_LONG_DETECT (3 << 0) ++ ++#define SHOTPLUG_CTL_TC _MMIO(0xc4034) ++#define ICP_TC_HPD_ENABLE(tc_port) (8 << (tc_port) * 4) ++/* Icelake DSC Rate Control Range Parameter Registers */ ++#define DSCA_RC_RANGE_PARAMETERS_0 _MMIO(0x6B240) ++#define DSCA_RC_RANGE_PARAMETERS_0_UDW _MMIO(0x6B240 + 4) ++#define DSCC_RC_RANGE_PARAMETERS_0 _MMIO(0x6BA40) ++#define DSCC_RC_RANGE_PARAMETERS_0_UDW _MMIO(0x6BA40 + 4) ++#define _ICL_DSC0_RC_RANGE_PARAMETERS_0_PB (0x78208) ++#define _ICL_DSC0_RC_RANGE_PARAMETERS_0_UDW_PB (0x78208 + 4) ++#define _ICL_DSC1_RC_RANGE_PARAMETERS_0_PB (0x78308) ++#define _ICL_DSC1_RC_RANGE_PARAMETERS_0_UDW_PB (0x78308 + 4) ++#define _ICL_DSC0_RC_RANGE_PARAMETERS_0_PC (0x78408) ++#define _ICL_DSC0_RC_RANGE_PARAMETERS_0_UDW_PC (0x78408 + 4) ++#define _ICL_DSC1_RC_RANGE_PARAMETERS_0_PC (0x78508) ++#define _ICL_DSC1_RC_RANGE_PARAMETERS_0_UDW_PC (0x78508 + 4) ++#define ICL_DSC0_RC_RANGE_PARAMETERS_0(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_RC_RANGE_PARAMETERS_0_PB, \ ++ _ICL_DSC0_RC_RANGE_PARAMETERS_0_PC) ++#define ICL_DSC0_RC_RANGE_PARAMETERS_0_UDW(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_RC_RANGE_PARAMETERS_0_UDW_PB, \ ++ _ICL_DSC0_RC_RANGE_PARAMETERS_0_UDW_PC) ++#define ICL_DSC1_RC_RANGE_PARAMETERS_0(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_RC_RANGE_PARAMETERS_0_PB, \ ++ _ICL_DSC1_RC_RANGE_PARAMETERS_0_PC) ++#define ICL_DSC1_RC_RANGE_PARAMETERS_0_UDW(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_RC_RANGE_PARAMETERS_0_UDW_PB, \ ++ _ICL_DSC1_RC_RANGE_PARAMETERS_0_UDW_PC) ++#define RC_BPG_OFFSET_SHIFT 10 ++#define RC_MAX_QP_SHIFT 5 ++#define RC_MIN_QP_SHIFT 0 ++ ++#define DSCA_RC_RANGE_PARAMETERS_1 _MMIO(0x6B248) ++#define DSCA_RC_RANGE_PARAMETERS_1_UDW _MMIO(0x6B248 + 4) ++#define DSCC_RC_RANGE_PARAMETERS_1 _MMIO(0x6BA48) ++#define DSCC_RC_RANGE_PARAMETERS_1_UDW _MMIO(0x6BA48 + 4) ++#define _ICL_DSC0_RC_RANGE_PARAMETERS_1_PB (0x78210) ++#define _ICL_DSC0_RC_RANGE_PARAMETERS_1_UDW_PB (0x78210 + 4) ++#define _ICL_DSC1_RC_RANGE_PARAMETERS_1_PB (0x78310) ++#define _ICL_DSC1_RC_RANGE_PARAMETERS_1_UDW_PB (0x78310 + 4) ++#define _ICL_DSC0_RC_RANGE_PARAMETERS_1_PC (0x78410) ++#define _ICL_DSC0_RC_RANGE_PARAMETERS_1_UDW_PC (0x78410 + 4) ++#define _ICL_DSC1_RC_RANGE_PARAMETERS_1_PC (0x78510) ++#define _ICL_DSC1_RC_RANGE_PARAMETERS_1_UDW_PC (0x78510 + 4) ++#define ICL_DSC0_RC_RANGE_PARAMETERS_1(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_RC_RANGE_PARAMETERS_1_PB, \ ++ _ICL_DSC0_RC_RANGE_PARAMETERS_1_PC) ++#define ICL_DSC0_RC_RANGE_PARAMETERS_1_UDW(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_RC_RANGE_PARAMETERS_1_UDW_PB, \ ++ _ICL_DSC0_RC_RANGE_PARAMETERS_1_UDW_PC) ++#define ICL_DSC1_RC_RANGE_PARAMETERS_1(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_RC_RANGE_PARAMETERS_1_PB, \ ++ _ICL_DSC1_RC_RANGE_PARAMETERS_1_PC) ++#define ICL_DSC1_RC_RANGE_PARAMETERS_1_UDW(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_RC_RANGE_PARAMETERS_1_UDW_PB, \ ++ _ICL_DSC1_RC_RANGE_PARAMETERS_1_UDW_PC) ++ ++#define DSCA_RC_RANGE_PARAMETERS_2 _MMIO(0x6B250) ++#define DSCA_RC_RANGE_PARAMETERS_2_UDW _MMIO(0x6B250 + 4) ++#define DSCC_RC_RANGE_PARAMETERS_2 _MMIO(0x6BA50) ++#define DSCC_RC_RANGE_PARAMETERS_2_UDW _MMIO(0x6BA50 + 4) ++#define _ICL_DSC0_RC_RANGE_PARAMETERS_2_PB (0x78218) ++#define _ICL_DSC0_RC_RANGE_PARAMETERS_2_UDW_PB (0x78218 + 4) ++#define _ICL_DSC1_RC_RANGE_PARAMETERS_2_PB (0x78318) ++#define _ICL_DSC1_RC_RANGE_PARAMETERS_2_UDW_PB (0x78318 + 4) ++#define _ICL_DSC0_RC_RANGE_PARAMETERS_2_PC (0x78418) ++#define _ICL_DSC0_RC_RANGE_PARAMETERS_2_UDW_PC (0x78418 + 4) ++#define _ICL_DSC1_RC_RANGE_PARAMETERS_2_PC (0x78518) ++#define _ICL_DSC1_RC_RANGE_PARAMETERS_2_UDW_PC (0x78518 + 4) ++#define ICL_DSC0_RC_RANGE_PARAMETERS_2(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_RC_RANGE_PARAMETERS_2_PB, \ ++ _ICL_DSC0_RC_RANGE_PARAMETERS_2_PC) ++#define ICL_DSC0_RC_RANGE_PARAMETERS_2_UDW(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_RC_RANGE_PARAMETERS_2_UDW_PB, \ ++ _ICL_DSC0_RC_RANGE_PARAMETERS_2_UDW_PC) ++#define ICL_DSC1_RC_RANGE_PARAMETERS_2(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_RC_RANGE_PARAMETERS_2_PB, \ ++ _ICL_DSC1_RC_RANGE_PARAMETERS_2_PC) ++#define ICL_DSC1_RC_RANGE_PARAMETERS_2_UDW(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_RC_RANGE_PARAMETERS_2_UDW_PB, \ ++ _ICL_DSC1_RC_RANGE_PARAMETERS_2_UDW_PC) ++ ++#define DSCA_RC_RANGE_PARAMETERS_3 _MMIO(0x6B258) ++#define DSCA_RC_RANGE_PARAMETERS_3_UDW _MMIO(0x6B258 + 4) ++#define DSCC_RC_RANGE_PARAMETERS_3 _MMIO(0x6BA58) ++#define DSCC_RC_RANGE_PARAMETERS_3_UDW _MMIO(0x6BA58 + 4) ++#define _ICL_DSC0_RC_RANGE_PARAMETERS_3_PB (0x78220) ++#define _ICL_DSC0_RC_RANGE_PARAMETERS_3_UDW_PB (0x78220 + 4) ++#define _ICL_DSC1_RC_RANGE_PARAMETERS_3_PB (0x78320) ++#define _ICL_DSC1_RC_RANGE_PARAMETERS_3_UDW_PB (0x78320 + 4) ++#define _ICL_DSC0_RC_RANGE_PARAMETERS_3_PC (0x78420) ++#define _ICL_DSC0_RC_RANGE_PARAMETERS_3_UDW_PC (0x78420 + 4) ++#define _ICL_DSC1_RC_RANGE_PARAMETERS_3_PC (0x78520) ++#define _ICL_DSC1_RC_RANGE_PARAMETERS_3_UDW_PC (0x78520 + 4) ++#define ICL_DSC0_RC_RANGE_PARAMETERS_3(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_RC_RANGE_PARAMETERS_3_PB, \ ++ _ICL_DSC0_RC_RANGE_PARAMETERS_3_PC) ++#define ICL_DSC0_RC_RANGE_PARAMETERS_3_UDW(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_RC_RANGE_PARAMETERS_3_UDW_PB, \ ++ _ICL_DSC0_RC_RANGE_PARAMETERS_3_UDW_PC) ++#define ICL_DSC1_RC_RANGE_PARAMETERS_3(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_RC_RANGE_PARAMETERS_3_PB, \ ++ _ICL_DSC1_RC_RANGE_PARAMETERS_3_PC) ++#define ICL_DSC1_RC_RANGE_PARAMETERS_3_UDW(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_RC_RANGE_PARAMETERS_3_UDW_PB, \ ++ _ICL_DSC1_RC_RANGE_PARAMETERS_3_UDW_PC) ++ ++#define ICP_TC_HPD_LONG_DETECT(tc_port) (2 << (tc_port) * 4) ++#define ICP_TC_HPD_SHORT_DETECT(tc_port) (1 << (tc_port) * 4) ++ ++#define _PCH_DPLL_A 0xc6014 ++#define _PCH_DPLL_B 0xc6018 ++#define PCH_DPLL(pll) _MMIO((pll) == 0 ? _PCH_DPLL_A : _PCH_DPLL_B) ++ ++#define _PCH_FPA0 0xc6040 ++#define FP_CB_TUNE (0x3 << 22) ++#define _PCH_FPA1 0xc6044 ++#define _PCH_FPB0 0xc6048 ++#define _PCH_FPB1 0xc604c ++#define PCH_FP0(pll) _MMIO((pll) == 0 ? _PCH_FPA0 : _PCH_FPB0) ++#define PCH_FP1(pll) _MMIO((pll) == 0 ? _PCH_FPA1 : _PCH_FPB1) ++ ++#define PCH_DPLL_TEST _MMIO(0xc606c) ++ ++#define PCH_DREF_CONTROL _MMIO(0xC6200) ++#define DREF_CONTROL_MASK 0x7fc3 ++#define DREF_CPU_SOURCE_OUTPUT_DISABLE (0 << 13) ++#define DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD (2 << 13) ++#define DREF_CPU_SOURCE_OUTPUT_NONSPREAD (3 << 13) ++#define DREF_CPU_SOURCE_OUTPUT_MASK (3 << 13) ++#define DREF_SSC_SOURCE_DISABLE (0 << 11) ++#define DREF_SSC_SOURCE_ENABLE (2 << 11) ++#define DREF_SSC_SOURCE_MASK (3 << 11) ++#define DREF_NONSPREAD_SOURCE_DISABLE (0 << 9) ++#define DREF_NONSPREAD_CK505_ENABLE (1 << 9) ++#define DREF_NONSPREAD_SOURCE_ENABLE (2 << 9) ++#define DREF_NONSPREAD_SOURCE_MASK (3 << 9) ++#define DREF_SUPERSPREAD_SOURCE_DISABLE (0 << 7) ++#define DREF_SUPERSPREAD_SOURCE_ENABLE (2 << 7) ++#define DREF_SUPERSPREAD_SOURCE_MASK (3 << 7) ++#define DREF_SSC4_DOWNSPREAD (0 << 6) ++#define DREF_SSC4_CENTERSPREAD (1 << 6) ++#define DREF_SSC1_DISABLE (0 << 1) ++#define DREF_SSC1_ENABLE (1 << 1) ++#define DREF_SSC4_DISABLE (0) ++#define DREF_SSC4_ENABLE (1) ++ ++#define PCH_RAWCLK_FREQ _MMIO(0xc6204) ++#define FDL_TP1_TIMER_SHIFT 12 ++#define FDL_TP1_TIMER_MASK (3 << 12) ++#define FDL_TP2_TIMER_SHIFT 10 ++#define FDL_TP2_TIMER_MASK (3 << 10) ++#define RAWCLK_FREQ_MASK 0x3ff ++#define CNP_RAWCLK_DIV_MASK (0x3ff << 16) ++#define CNP_RAWCLK_DIV(div) ((div) << 16) ++#define CNP_RAWCLK_FRAC_MASK (0xf << 26) ++#define CNP_RAWCLK_DEN(den) ((den) << 26) ++#define ICP_RAWCLK_NUM(num) ((num) << 11) ++ ++#define PCH_DPLL_TMR_CFG _MMIO(0xc6208) ++ ++#define PCH_SSC4_PARMS _MMIO(0xc6210) ++#define PCH_SSC4_AUX_PARMS _MMIO(0xc6214) ++ ++#define PCH_DPLL_SEL _MMIO(0xc7000) ++#define TRANS_DPLLB_SEL(pipe) (1 << ((pipe) * 4)) ++#define TRANS_DPLLA_SEL(pipe) 0 ++#define TRANS_DPLL_ENABLE(pipe) (1 << ((pipe) * 4 + 3)) ++ ++/* transcoder */ ++ ++#define _PCH_TRANS_HTOTAL_A 0xe0000 ++#define TRANS_HTOTAL_SHIFT 16 ++#define TRANS_HACTIVE_SHIFT 0 ++#define _PCH_TRANS_HBLANK_A 0xe0004 ++#define TRANS_HBLANK_END_SHIFT 16 ++#define TRANS_HBLANK_START_SHIFT 0 ++#define _PCH_TRANS_HSYNC_A 0xe0008 ++#define TRANS_HSYNC_END_SHIFT 16 ++#define TRANS_HSYNC_START_SHIFT 0 ++#define _PCH_TRANS_VTOTAL_A 0xe000c ++#define TRANS_VTOTAL_SHIFT 16 ++#define TRANS_VACTIVE_SHIFT 0 ++#define _PCH_TRANS_VBLANK_A 0xe0010 ++#define TRANS_VBLANK_END_SHIFT 16 ++#define TRANS_VBLANK_START_SHIFT 0 ++#define _PCH_TRANS_VSYNC_A 0xe0014 ++#define TRANS_VSYNC_END_SHIFT 16 ++#define TRANS_VSYNC_START_SHIFT 0 ++#define _PCH_TRANS_VSYNCSHIFT_A 0xe0028 ++ ++#define _PCH_TRANSA_DATA_M1 0xe0030 ++#define _PCH_TRANSA_DATA_N1 0xe0034 ++#define _PCH_TRANSA_DATA_M2 0xe0038 ++#define _PCH_TRANSA_DATA_N2 0xe003c ++#define _PCH_TRANSA_LINK_M1 0xe0040 ++#define _PCH_TRANSA_LINK_N1 0xe0044 ++#define _PCH_TRANSA_LINK_M2 0xe0048 ++#define _PCH_TRANSA_LINK_N2 0xe004c ++ ++/* Per-transcoder DIP controls (PCH) */ ++#define _VIDEO_DIP_CTL_A 0xe0200 ++#define _VIDEO_DIP_DATA_A 0xe0208 ++#define _VIDEO_DIP_GCP_A 0xe0210 ++#define GCP_COLOR_INDICATION (1 << 2) ++#define GCP_DEFAULT_PHASE_ENABLE (1 << 1) ++#define GCP_AV_MUTE (1 << 0) ++ ++#define _VIDEO_DIP_CTL_B 0xe1200 ++#define _VIDEO_DIP_DATA_B 0xe1208 ++#define _VIDEO_DIP_GCP_B 0xe1210 ++ ++#define TVIDEO_DIP_CTL(pipe) _MMIO_PIPE(pipe, _VIDEO_DIP_CTL_A, _VIDEO_DIP_CTL_B) ++#define TVIDEO_DIP_DATA(pipe) _MMIO_PIPE(pipe, _VIDEO_DIP_DATA_A, _VIDEO_DIP_DATA_B) ++#define TVIDEO_DIP_GCP(pipe) _MMIO_PIPE(pipe, _VIDEO_DIP_GCP_A, _VIDEO_DIP_GCP_B) ++ ++/* Per-transcoder DIP controls (VLV) */ ++#define _VLV_VIDEO_DIP_CTL_A (VLV_DISPLAY_BASE + 0x60200) ++#define _VLV_VIDEO_DIP_DATA_A (VLV_DISPLAY_BASE + 0x60208) ++#define _VLV_VIDEO_DIP_GDCP_PAYLOAD_A (VLV_DISPLAY_BASE + 0x60210) ++ ++#define _VLV_VIDEO_DIP_CTL_B (VLV_DISPLAY_BASE + 0x61170) ++#define _VLV_VIDEO_DIP_DATA_B (VLV_DISPLAY_BASE + 0x61174) ++#define _VLV_VIDEO_DIP_GDCP_PAYLOAD_B (VLV_DISPLAY_BASE + 0x61178) ++ ++#define _CHV_VIDEO_DIP_CTL_C (VLV_DISPLAY_BASE + 0x611f0) ++#define _CHV_VIDEO_DIP_DATA_C (VLV_DISPLAY_BASE + 0x611f4) ++#define _CHV_VIDEO_DIP_GDCP_PAYLOAD_C (VLV_DISPLAY_BASE + 0x611f8) ++ ++#define VLV_TVIDEO_DIP_CTL(pipe) \ ++ _MMIO_PIPE3((pipe), _VLV_VIDEO_DIP_CTL_A, \ ++ _VLV_VIDEO_DIP_CTL_B, _CHV_VIDEO_DIP_CTL_C) ++#define VLV_TVIDEO_DIP_DATA(pipe) \ ++ _MMIO_PIPE3((pipe), _VLV_VIDEO_DIP_DATA_A, \ ++ _VLV_VIDEO_DIP_DATA_B, _CHV_VIDEO_DIP_DATA_C) ++#define VLV_TVIDEO_DIP_GCP(pipe) \ ++ _MMIO_PIPE3((pipe), _VLV_VIDEO_DIP_GDCP_PAYLOAD_A, \ ++ _VLV_VIDEO_DIP_GDCP_PAYLOAD_B, _CHV_VIDEO_DIP_GDCP_PAYLOAD_C) ++ ++/* Haswell DIP controls */ ++ ++#define _HSW_VIDEO_DIP_CTL_A 0x60200 ++#define _HSW_VIDEO_DIP_AVI_DATA_A 0x60220 ++#define _HSW_VIDEO_DIP_VS_DATA_A 0x60260 ++#define _HSW_VIDEO_DIP_SPD_DATA_A 0x602A0 ++#define _HSW_VIDEO_DIP_GMP_DATA_A 0x602E0 ++#define _HSW_VIDEO_DIP_VSC_DATA_A 0x60320 ++#define _HSW_VIDEO_DIP_AVI_ECC_A 0x60240 ++#define _HSW_VIDEO_DIP_VS_ECC_A 0x60280 ++#define _HSW_VIDEO_DIP_SPD_ECC_A 0x602C0 ++#define _HSW_VIDEO_DIP_GMP_ECC_A 0x60300 ++#define _HSW_VIDEO_DIP_VSC_ECC_A 0x60344 ++#define _HSW_VIDEO_DIP_GCP_A 0x60210 ++ ++#define _HSW_VIDEO_DIP_CTL_B 0x61200 ++#define _HSW_VIDEO_DIP_AVI_DATA_B 0x61220 ++#define _HSW_VIDEO_DIP_VS_DATA_B 0x61260 ++#define _HSW_VIDEO_DIP_SPD_DATA_B 0x612A0 ++#define _HSW_VIDEO_DIP_GMP_DATA_B 0x612E0 ++#define _HSW_VIDEO_DIP_VSC_DATA_B 0x61320 ++#define _HSW_VIDEO_DIP_BVI_ECC_B 0x61240 ++#define _HSW_VIDEO_DIP_VS_ECC_B 0x61280 ++#define _HSW_VIDEO_DIP_SPD_ECC_B 0x612C0 ++#define _HSW_VIDEO_DIP_GMP_ECC_B 0x61300 ++#define _HSW_VIDEO_DIP_VSC_ECC_B 0x61344 ++#define _HSW_VIDEO_DIP_GCP_B 0x61210 ++ ++/* Icelake PPS_DATA and _ECC DIP Registers. ++ * These are available for transcoders B,C and eDP. ++ * Adding the _A so as to reuse the _MMIO_TRANS2 ++ * definition, with which it offsets to the right location. ++ */ ++ ++#define _ICL_VIDEO_DIP_PPS_DATA_A 0x60350 ++#define _ICL_VIDEO_DIP_PPS_DATA_B 0x61350 ++#define _ICL_VIDEO_DIP_PPS_ECC_A 0x603D4 ++#define _ICL_VIDEO_DIP_PPS_ECC_B 0x613D4 ++ ++#define HSW_TVIDEO_DIP_CTL(trans) _MMIO_TRANS2(trans, _HSW_VIDEO_DIP_CTL_A) ++#define HSW_TVIDEO_DIP_GCP(trans) _MMIO_TRANS2(trans, _HSW_VIDEO_DIP_GCP_A) ++#define HSW_TVIDEO_DIP_AVI_DATA(trans, i) _MMIO_TRANS2(trans, _HSW_VIDEO_DIP_AVI_DATA_A + (i) * 4) ++#define HSW_TVIDEO_DIP_VS_DATA(trans, i) _MMIO_TRANS2(trans, _HSW_VIDEO_DIP_VS_DATA_A + (i) * 4) ++#define HSW_TVIDEO_DIP_SPD_DATA(trans, i) _MMIO_TRANS2(trans, _HSW_VIDEO_DIP_SPD_DATA_A + (i) * 4) ++#define HSW_TVIDEO_DIP_GMP_DATA(trans, i) _MMIO_TRANS2(trans, _HSW_VIDEO_DIP_GMP_DATA_A + (i) * 4) ++#define HSW_TVIDEO_DIP_VSC_DATA(trans, i) _MMIO_TRANS2(trans, _HSW_VIDEO_DIP_VSC_DATA_A + (i) * 4) ++#define ICL_VIDEO_DIP_PPS_DATA(trans, i) _MMIO_TRANS2(trans, _ICL_VIDEO_DIP_PPS_DATA_A + (i) * 4) ++#define ICL_VIDEO_DIP_PPS_ECC(trans, i) _MMIO_TRANS2(trans, _ICL_VIDEO_DIP_PPS_ECC_A + (i) * 4) ++ ++#define _HSW_STEREO_3D_CTL_A 0x70020 ++#define S3D_ENABLE (1 << 31) ++#define _HSW_STEREO_3D_CTL_B 0x71020 ++ ++#define HSW_STEREO_3D_CTL(trans) _MMIO_PIPE2(trans, _HSW_STEREO_3D_CTL_A) ++ ++#define _PCH_TRANS_HTOTAL_B 0xe1000 ++#define _PCH_TRANS_HBLANK_B 0xe1004 ++#define _PCH_TRANS_HSYNC_B 0xe1008 ++#define _PCH_TRANS_VTOTAL_B 0xe100c ++#define _PCH_TRANS_VBLANK_B 0xe1010 ++#define _PCH_TRANS_VSYNC_B 0xe1014 ++#define _PCH_TRANS_VSYNCSHIFT_B 0xe1028 ++ ++#define PCH_TRANS_HTOTAL(pipe) _MMIO_PIPE(pipe, _PCH_TRANS_HTOTAL_A, _PCH_TRANS_HTOTAL_B) ++#define PCH_TRANS_HBLANK(pipe) _MMIO_PIPE(pipe, _PCH_TRANS_HBLANK_A, _PCH_TRANS_HBLANK_B) ++#define PCH_TRANS_HSYNC(pipe) _MMIO_PIPE(pipe, _PCH_TRANS_HSYNC_A, _PCH_TRANS_HSYNC_B) ++#define PCH_TRANS_VTOTAL(pipe) _MMIO_PIPE(pipe, _PCH_TRANS_VTOTAL_A, _PCH_TRANS_VTOTAL_B) ++#define PCH_TRANS_VBLANK(pipe) _MMIO_PIPE(pipe, _PCH_TRANS_VBLANK_A, _PCH_TRANS_VBLANK_B) ++#define PCH_TRANS_VSYNC(pipe) _MMIO_PIPE(pipe, _PCH_TRANS_VSYNC_A, _PCH_TRANS_VSYNC_B) ++#define PCH_TRANS_VSYNCSHIFT(pipe) _MMIO_PIPE(pipe, _PCH_TRANS_VSYNCSHIFT_A, _PCH_TRANS_VSYNCSHIFT_B) ++ ++#define _PCH_TRANSB_DATA_M1 0xe1030 ++#define _PCH_TRANSB_DATA_N1 0xe1034 ++#define _PCH_TRANSB_DATA_M2 0xe1038 ++#define _PCH_TRANSB_DATA_N2 0xe103c ++#define _PCH_TRANSB_LINK_M1 0xe1040 ++#define _PCH_TRANSB_LINK_N1 0xe1044 ++#define _PCH_TRANSB_LINK_M2 0xe1048 ++#define _PCH_TRANSB_LINK_N2 0xe104c ++ ++#define PCH_TRANS_DATA_M1(pipe) _MMIO_PIPE(pipe, _PCH_TRANSA_DATA_M1, _PCH_TRANSB_DATA_M1) ++#define PCH_TRANS_DATA_N1(pipe) _MMIO_PIPE(pipe, _PCH_TRANSA_DATA_N1, _PCH_TRANSB_DATA_N1) ++#define PCH_TRANS_DATA_M2(pipe) _MMIO_PIPE(pipe, _PCH_TRANSA_DATA_M2, _PCH_TRANSB_DATA_M2) ++#define PCH_TRANS_DATA_N2(pipe) _MMIO_PIPE(pipe, _PCH_TRANSA_DATA_N2, _PCH_TRANSB_DATA_N2) ++#define PCH_TRANS_LINK_M1(pipe) _MMIO_PIPE(pipe, _PCH_TRANSA_LINK_M1, _PCH_TRANSB_LINK_M1) ++#define PCH_TRANS_LINK_N1(pipe) _MMIO_PIPE(pipe, _PCH_TRANSA_LINK_N1, _PCH_TRANSB_LINK_N1) ++#define PCH_TRANS_LINK_M2(pipe) _MMIO_PIPE(pipe, _PCH_TRANSA_LINK_M2, _PCH_TRANSB_LINK_M2) ++#define PCH_TRANS_LINK_N2(pipe) _MMIO_PIPE(pipe, _PCH_TRANSA_LINK_N2, _PCH_TRANSB_LINK_N2) ++ ++#define _PCH_TRANSACONF 0xf0008 ++#define _PCH_TRANSBCONF 0xf1008 ++#define PCH_TRANSCONF(pipe) _MMIO_PIPE(pipe, _PCH_TRANSACONF, _PCH_TRANSBCONF) ++#define LPT_TRANSCONF PCH_TRANSCONF(PIPE_A) /* lpt has only one transcoder */ ++#define TRANS_DISABLE (0 << 31) ++#define TRANS_ENABLE (1 << 31) ++#define TRANS_STATE_MASK (1 << 30) ++#define TRANS_STATE_DISABLE (0 << 30) ++#define TRANS_STATE_ENABLE (1 << 30) ++#define TRANS_FSYNC_DELAY_HB1 (0 << 27) ++#define TRANS_FSYNC_DELAY_HB2 (1 << 27) ++#define TRANS_FSYNC_DELAY_HB3 (2 << 27) ++#define TRANS_FSYNC_DELAY_HB4 (3 << 27) ++#define TRANS_INTERLACE_MASK (7 << 21) ++#define TRANS_PROGRESSIVE (0 << 21) ++#define TRANS_INTERLACED (3 << 21) ++#define TRANS_LEGACY_INTERLACED_ILK (2 << 21) ++#define TRANS_8BPC (0 << 5) ++#define TRANS_10BPC (1 << 5) ++#define TRANS_6BPC (2 << 5) ++#define TRANS_12BPC (3 << 5) ++ ++#define _TRANSA_CHICKEN1 0xf0060 ++#define _TRANSB_CHICKEN1 0xf1060 ++#define TRANS_CHICKEN1(pipe) _MMIO_PIPE(pipe, _TRANSA_CHICKEN1, _TRANSB_CHICKEN1) ++#define TRANS_CHICKEN1_HDMIUNIT_GC_DISABLE (1 << 10) ++#define TRANS_CHICKEN1_DP0UNIT_GC_DISABLE (1 << 4) ++#define _TRANSA_CHICKEN2 0xf0064 ++#define _TRANSB_CHICKEN2 0xf1064 ++#define TRANS_CHICKEN2(pipe) _MMIO_PIPE(pipe, _TRANSA_CHICKEN2, _TRANSB_CHICKEN2) ++#define TRANS_CHICKEN2_TIMING_OVERRIDE (1 << 31) ++#define TRANS_CHICKEN2_FDI_POLARITY_REVERSED (1 << 29) ++#define TRANS_CHICKEN2_FRAME_START_DELAY_MASK (3 << 27) ++#define TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER (1 << 26) ++#define TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH (1 << 25) ++ ++#define SOUTH_CHICKEN1 _MMIO(0xc2000) ++#define FDIA_PHASE_SYNC_SHIFT_OVR 19 ++#define FDIA_PHASE_SYNC_SHIFT_EN 18 ++#define FDI_PHASE_SYNC_OVR(pipe) (1 << (FDIA_PHASE_SYNC_SHIFT_OVR - ((pipe) * 2))) ++#define FDI_PHASE_SYNC_EN(pipe) (1 << (FDIA_PHASE_SYNC_SHIFT_EN - ((pipe) * 2))) ++#define FDI_BC_BIFURCATION_SELECT (1 << 12) ++#define CHASSIS_CLK_REQ_DURATION_MASK (0xf << 8) ++#define CHASSIS_CLK_REQ_DURATION(x) ((x) << 8) ++#define SPT_PWM_GRANULARITY (1 << 0) ++#define SOUTH_CHICKEN2 _MMIO(0xc2004) ++#define FDI_MPHY_IOSFSB_RESET_STATUS (1 << 13) ++#define FDI_MPHY_IOSFSB_RESET_CTL (1 << 12) ++#define LPT_PWM_GRANULARITY (1 << 5) ++#define DPLS_EDP_PPS_FIX_DIS (1 << 0) ++ ++#define _FDI_RXA_CHICKEN 0xc200c ++#define _FDI_RXB_CHICKEN 0xc2010 ++#define FDI_RX_PHASE_SYNC_POINTER_OVR (1 << 1) ++#define FDI_RX_PHASE_SYNC_POINTER_EN (1 << 0) ++#define FDI_RX_CHICKEN(pipe) _MMIO_PIPE(pipe, _FDI_RXA_CHICKEN, _FDI_RXB_CHICKEN) ++ ++#define SOUTH_DSPCLK_GATE_D _MMIO(0xc2020) ++#define PCH_GMBUSUNIT_CLOCK_GATE_DISABLE (1 << 31) ++#define PCH_DPLUNIT_CLOCK_GATE_DISABLE (1 << 30) ++#define PCH_DPLSUNIT_CLOCK_GATE_DISABLE (1 << 29) ++#define PCH_CPUNIT_CLOCK_GATE_DISABLE (1 << 14) ++#define CNP_PWM_CGE_GATING_DISABLE (1 << 13) ++#define PCH_LP_PARTITION_LEVEL_DISABLE (1 << 12) ++ ++/* CPU: FDI_TX */ ++#define _FDI_TXA_CTL 0x60100 ++#define _FDI_TXB_CTL 0x61100 ++#define FDI_TX_CTL(pipe) _MMIO_PIPE(pipe, _FDI_TXA_CTL, _FDI_TXB_CTL) ++#define FDI_TX_DISABLE (0 << 31) ++#define FDI_TX_ENABLE (1 << 31) ++#define FDI_LINK_TRAIN_PATTERN_1 (0 << 28) ++#define FDI_LINK_TRAIN_PATTERN_2 (1 << 28) ++#define FDI_LINK_TRAIN_PATTERN_IDLE (2 << 28) ++#define FDI_LINK_TRAIN_NONE (3 << 28) ++#define FDI_LINK_TRAIN_VOLTAGE_0_4V (0 << 25) ++#define FDI_LINK_TRAIN_VOLTAGE_0_6V (1 << 25) ++#define FDI_LINK_TRAIN_VOLTAGE_0_8V (2 << 25) ++#define FDI_LINK_TRAIN_VOLTAGE_1_2V (3 << 25) ++#define FDI_LINK_TRAIN_PRE_EMPHASIS_NONE (0 << 22) ++#define FDI_LINK_TRAIN_PRE_EMPHASIS_1_5X (1 << 22) ++#define FDI_LINK_TRAIN_PRE_EMPHASIS_2X (2 << 22) ++#define FDI_LINK_TRAIN_PRE_EMPHASIS_3X (3 << 22) ++/* ILK always use 400mV 0dB for voltage swing and pre-emphasis level. ++ SNB has different settings. */ ++/* SNB A-stepping */ ++#define FDI_LINK_TRAIN_400MV_0DB_SNB_A (0x38 << 22) ++#define FDI_LINK_TRAIN_400MV_6DB_SNB_A (0x02 << 22) ++#define FDI_LINK_TRAIN_600MV_3_5DB_SNB_A (0x01 << 22) ++#define FDI_LINK_TRAIN_800MV_0DB_SNB_A (0x0 << 22) ++/* SNB B-stepping */ ++#define FDI_LINK_TRAIN_400MV_0DB_SNB_B (0x0 << 22) ++#define FDI_LINK_TRAIN_400MV_6DB_SNB_B (0x3a << 22) ++#define FDI_LINK_TRAIN_600MV_3_5DB_SNB_B (0x39 << 22) ++#define FDI_LINK_TRAIN_800MV_0DB_SNB_B (0x38 << 22) ++#define FDI_LINK_TRAIN_VOL_EMP_MASK (0x3f << 22) ++#define FDI_DP_PORT_WIDTH_SHIFT 19 ++#define FDI_DP_PORT_WIDTH_MASK (7 << FDI_DP_PORT_WIDTH_SHIFT) ++#define FDI_DP_PORT_WIDTH(width) (((width) - 1) << FDI_DP_PORT_WIDTH_SHIFT) ++#define FDI_TX_ENHANCE_FRAME_ENABLE (1 << 18) ++/* Ironlake: hardwired to 1 */ ++#define FDI_TX_PLL_ENABLE (1 << 14) ++ ++/* Ivybridge has different bits for lolz */ ++#define FDI_LINK_TRAIN_PATTERN_1_IVB (0 << 8) ++#define FDI_LINK_TRAIN_PATTERN_2_IVB (1 << 8) ++#define FDI_LINK_TRAIN_PATTERN_IDLE_IVB (2 << 8) ++#define FDI_LINK_TRAIN_NONE_IVB (3 << 8) ++ ++/* both Tx and Rx */ ++#define FDI_COMPOSITE_SYNC (1 << 11) ++#define FDI_LINK_TRAIN_AUTO (1 << 10) ++#define FDI_SCRAMBLING_ENABLE (0 << 7) ++#define FDI_SCRAMBLING_DISABLE (1 << 7) ++ ++/* FDI_RX, FDI_X is hard-wired to Transcoder_X */ ++#define _FDI_RXA_CTL 0xf000c ++#define _FDI_RXB_CTL 0xf100c ++#define FDI_RX_CTL(pipe) _MMIO_PIPE(pipe, _FDI_RXA_CTL, _FDI_RXB_CTL) ++#define FDI_RX_ENABLE (1 << 31) ++/* train, dp width same as FDI_TX */ ++#define FDI_FS_ERRC_ENABLE (1 << 27) ++#define FDI_FE_ERRC_ENABLE (1 << 26) ++#define FDI_RX_POLARITY_REVERSED_LPT (1 << 16) ++#define FDI_8BPC (0 << 16) ++#define FDI_10BPC (1 << 16) ++#define FDI_6BPC (2 << 16) ++#define FDI_12BPC (3 << 16) ++#define FDI_RX_LINK_REVERSAL_OVERRIDE (1 << 15) ++#define FDI_DMI_LINK_REVERSE_MASK (1 << 14) ++#define FDI_RX_PLL_ENABLE (1 << 13) ++#define FDI_FS_ERR_CORRECT_ENABLE (1 << 11) ++#define FDI_FE_ERR_CORRECT_ENABLE (1 << 10) ++#define FDI_FS_ERR_REPORT_ENABLE (1 << 9) ++#define FDI_FE_ERR_REPORT_ENABLE (1 << 8) ++#define FDI_RX_ENHANCE_FRAME_ENABLE (1 << 6) ++#define FDI_PCDCLK (1 << 4) ++/* CPT */ ++#define FDI_AUTO_TRAINING (1 << 10) ++#define FDI_LINK_TRAIN_PATTERN_1_CPT (0 << 8) ++#define FDI_LINK_TRAIN_PATTERN_2_CPT (1 << 8) ++#define FDI_LINK_TRAIN_PATTERN_IDLE_CPT (2 << 8) ++#define FDI_LINK_TRAIN_NORMAL_CPT (3 << 8) ++#define FDI_LINK_TRAIN_PATTERN_MASK_CPT (3 << 8) ++ ++#define _FDI_RXA_MISC 0xf0010 ++#define _FDI_RXB_MISC 0xf1010 ++#define FDI_RX_PWRDN_LANE1_MASK (3 << 26) ++#define FDI_RX_PWRDN_LANE1_VAL(x) ((x) << 26) ++#define FDI_RX_PWRDN_LANE0_MASK (3 << 24) ++#define FDI_RX_PWRDN_LANE0_VAL(x) ((x) << 24) ++#define FDI_RX_TP1_TO_TP2_48 (2 << 20) ++#define FDI_RX_TP1_TO_TP2_64 (3 << 20) ++#define FDI_RX_FDI_DELAY_90 (0x90 << 0) ++#define FDI_RX_MISC(pipe) _MMIO_PIPE(pipe, _FDI_RXA_MISC, _FDI_RXB_MISC) ++ ++#define _FDI_RXA_TUSIZE1 0xf0030 ++#define _FDI_RXA_TUSIZE2 0xf0038 ++#define _FDI_RXB_TUSIZE1 0xf1030 ++#define _FDI_RXB_TUSIZE2 0xf1038 ++#define FDI_RX_TUSIZE1(pipe) _MMIO_PIPE(pipe, _FDI_RXA_TUSIZE1, _FDI_RXB_TUSIZE1) ++#define FDI_RX_TUSIZE2(pipe) _MMIO_PIPE(pipe, _FDI_RXA_TUSIZE2, _FDI_RXB_TUSIZE2) ++ ++/* FDI_RX interrupt register format */ ++#define FDI_RX_INTER_LANE_ALIGN (1 << 10) ++#define FDI_RX_SYMBOL_LOCK (1 << 9) /* train 2 */ ++#define FDI_RX_BIT_LOCK (1 << 8) /* train 1 */ ++#define FDI_RX_TRAIN_PATTERN_2_FAIL (1 << 7) ++#define FDI_RX_FS_CODE_ERR (1 << 6) ++#define FDI_RX_FE_CODE_ERR (1 << 5) ++#define FDI_RX_SYMBOL_ERR_RATE_ABOVE (1 << 4) ++#define FDI_RX_HDCP_LINK_FAIL (1 << 3) ++#define FDI_RX_PIXEL_FIFO_OVERFLOW (1 << 2) ++#define FDI_RX_CROSS_CLOCK_OVERFLOW (1 << 1) ++#define FDI_RX_SYMBOL_QUEUE_OVERFLOW (1 << 0) ++ ++#define _FDI_RXA_IIR 0xf0014 ++#define _FDI_RXA_IMR 0xf0018 ++#define _FDI_RXB_IIR 0xf1014 ++#define _FDI_RXB_IMR 0xf1018 ++#define FDI_RX_IIR(pipe) _MMIO_PIPE(pipe, _FDI_RXA_IIR, _FDI_RXB_IIR) ++#define FDI_RX_IMR(pipe) _MMIO_PIPE(pipe, _FDI_RXA_IMR, _FDI_RXB_IMR) ++ ++#define FDI_PLL_CTL_1 _MMIO(0xfe000) ++#define FDI_PLL_CTL_2 _MMIO(0xfe004) ++ ++#define PCH_LVDS _MMIO(0xe1180) ++#define LVDS_DETECTED (1 << 1) ++ ++#define _PCH_DP_B 0xe4100 ++#define PCH_DP_B _MMIO(_PCH_DP_B) ++#define _PCH_DPB_AUX_CH_CTL 0xe4110 ++#define _PCH_DPB_AUX_CH_DATA1 0xe4114 ++#define _PCH_DPB_AUX_CH_DATA2 0xe4118 ++#define _PCH_DPB_AUX_CH_DATA3 0xe411c ++#define _PCH_DPB_AUX_CH_DATA4 0xe4120 ++#define _PCH_DPB_AUX_CH_DATA5 0xe4124 ++ ++#define _PCH_DP_C 0xe4200 ++#define PCH_DP_C _MMIO(_PCH_DP_C) ++#define _PCH_DPC_AUX_CH_CTL 0xe4210 ++#define _PCH_DPC_AUX_CH_DATA1 0xe4214 ++#define _PCH_DPC_AUX_CH_DATA2 0xe4218 ++#define _PCH_DPC_AUX_CH_DATA3 0xe421c ++#define _PCH_DPC_AUX_CH_DATA4 0xe4220 ++#define _PCH_DPC_AUX_CH_DATA5 0xe4224 ++ ++#define _PCH_DP_D 0xe4300 ++#define PCH_DP_D _MMIO(_PCH_DP_D) ++#define _PCH_DPD_AUX_CH_CTL 0xe4310 ++#define _PCH_DPD_AUX_CH_DATA1 0xe4314 ++#define _PCH_DPD_AUX_CH_DATA2 0xe4318 ++#define _PCH_DPD_AUX_CH_DATA3 0xe431c ++#define _PCH_DPD_AUX_CH_DATA4 0xe4320 ++#define _PCH_DPD_AUX_CH_DATA5 0xe4324 ++ ++#define PCH_DP_AUX_CH_CTL(aux_ch) _MMIO_PORT((aux_ch) - AUX_CH_B, _PCH_DPB_AUX_CH_CTL, _PCH_DPC_AUX_CH_CTL) ++#define PCH_DP_AUX_CH_DATA(aux_ch, i) _MMIO(_PORT((aux_ch) - AUX_CH_B, _PCH_DPB_AUX_CH_DATA1, _PCH_DPC_AUX_CH_DATA1) + (i) * 4) /* 5 registers */ ++ ++/* CPT */ ++#define _TRANS_DP_CTL_A 0xe0300 ++#define _TRANS_DP_CTL_B 0xe1300 ++#define _TRANS_DP_CTL_C 0xe2300 ++#define TRANS_DP_CTL(pipe) _MMIO_PIPE(pipe, _TRANS_DP_CTL_A, _TRANS_DP_CTL_B) ++#define TRANS_DP_OUTPUT_ENABLE (1 << 31) ++#define TRANS_DP_PORT_SEL_MASK (3 << 29) ++#define TRANS_DP_PORT_SEL_NONE (3 << 29) ++#define TRANS_DP_PORT_SEL(port) (((port) - PORT_B) << 29) ++#define TRANS_DP_AUDIO_ONLY (1 << 26) ++#define TRANS_DP_ENH_FRAMING (1 << 18) ++#define TRANS_DP_8BPC (0 << 9) ++#define TRANS_DP_10BPC (1 << 9) ++#define TRANS_DP_6BPC (2 << 9) ++#define TRANS_DP_12BPC (3 << 9) ++#define TRANS_DP_BPC_MASK (3 << 9) ++#define TRANS_DP_VSYNC_ACTIVE_HIGH (1 << 4) ++#define TRANS_DP_VSYNC_ACTIVE_LOW 0 ++#define TRANS_DP_HSYNC_ACTIVE_HIGH (1 << 3) ++#define TRANS_DP_HSYNC_ACTIVE_LOW 0 ++#define TRANS_DP_SYNC_MASK (3 << 3) ++ ++/* SNB eDP training params */ ++/* SNB A-stepping */ ++#define EDP_LINK_TRAIN_400MV_0DB_SNB_A (0x38 << 22) ++#define EDP_LINK_TRAIN_400MV_6DB_SNB_A (0x02 << 22) ++#define EDP_LINK_TRAIN_600MV_3_5DB_SNB_A (0x01 << 22) ++#define EDP_LINK_TRAIN_800MV_0DB_SNB_A (0x0 << 22) ++/* SNB B-stepping */ ++#define EDP_LINK_TRAIN_400_600MV_0DB_SNB_B (0x0 << 22) ++#define EDP_LINK_TRAIN_400MV_3_5DB_SNB_B (0x1 << 22) ++#define EDP_LINK_TRAIN_400_600MV_6DB_SNB_B (0x3a << 22) ++#define EDP_LINK_TRAIN_600_800MV_3_5DB_SNB_B (0x39 << 22) ++#define EDP_LINK_TRAIN_800_1200MV_0DB_SNB_B (0x38 << 22) ++#define EDP_LINK_TRAIN_VOL_EMP_MASK_SNB (0x3f << 22) ++ ++/* IVB */ ++#define EDP_LINK_TRAIN_400MV_0DB_IVB (0x24 << 22) ++#define EDP_LINK_TRAIN_400MV_3_5DB_IVB (0x2a << 22) ++#define EDP_LINK_TRAIN_400MV_6DB_IVB (0x2f << 22) ++#define EDP_LINK_TRAIN_600MV_0DB_IVB (0x30 << 22) ++#define EDP_LINK_TRAIN_600MV_3_5DB_IVB (0x36 << 22) ++#define EDP_LINK_TRAIN_800MV_0DB_IVB (0x38 << 22) ++#define EDP_LINK_TRAIN_800MV_3_5DB_IVB (0x3e << 22) ++ ++/* legacy values */ ++#define EDP_LINK_TRAIN_500MV_0DB_IVB (0x00 << 22) ++#define EDP_LINK_TRAIN_1000MV_0DB_IVB (0x20 << 22) ++#define EDP_LINK_TRAIN_500MV_3_5DB_IVB (0x02 << 22) ++#define EDP_LINK_TRAIN_1000MV_3_5DB_IVB (0x22 << 22) ++#define EDP_LINK_TRAIN_1000MV_6DB_IVB (0x23 << 22) ++ ++#define EDP_LINK_TRAIN_VOL_EMP_MASK_IVB (0x3f << 22) ++ ++#define VLV_PMWGICZ _MMIO(0x1300a4) ++ ++#define RC6_LOCATION _MMIO(0xD40) ++#define RC6_CTX_IN_DRAM (1 << 0) ++#define RC6_CTX_BASE _MMIO(0xD48) ++#define RC6_CTX_BASE_MASK 0xFFFFFFF0 ++#define PWRCTX_MAXCNT_RCSUNIT _MMIO(0x2054) ++#define PWRCTX_MAXCNT_VCSUNIT0 _MMIO(0x12054) ++#define PWRCTX_MAXCNT_BCSUNIT _MMIO(0x22054) ++#define PWRCTX_MAXCNT_VECSUNIT _MMIO(0x1A054) ++#define PWRCTX_MAXCNT_VCSUNIT1 _MMIO(0x1C054) ++#define IDLE_TIME_MASK 0xFFFFF ++#define FORCEWAKE _MMIO(0xA18C) ++#define FORCEWAKE_VLV _MMIO(0x1300b0) ++#define FORCEWAKE_ACK_VLV _MMIO(0x1300b4) ++#define FORCEWAKE_MEDIA_VLV _MMIO(0x1300b8) ++#define FORCEWAKE_ACK_MEDIA_VLV _MMIO(0x1300bc) ++#define FORCEWAKE_ACK_HSW _MMIO(0x130044) ++#define FORCEWAKE_ACK _MMIO(0x130090) ++#define VLV_GTLC_WAKE_CTRL _MMIO(0x130090) ++#define VLV_GTLC_RENDER_CTX_EXISTS (1 << 25) ++#define VLV_GTLC_MEDIA_CTX_EXISTS (1 << 24) ++#define VLV_GTLC_ALLOWWAKEREQ (1 << 0) ++ ++#define VLV_GTLC_PW_STATUS _MMIO(0x130094) ++#define VLV_GTLC_ALLOWWAKEACK (1 << 0) ++#define VLV_GTLC_ALLOWWAKEERR (1 << 1) ++#define VLV_GTLC_PW_MEDIA_STATUS_MASK (1 << 5) ++#define VLV_GTLC_PW_RENDER_STATUS_MASK (1 << 7) ++#define FORCEWAKE_MT _MMIO(0xa188) /* multi-threaded */ ++#define FORCEWAKE_MEDIA_GEN9 _MMIO(0xa270) ++#define FORCEWAKE_MEDIA_VDBOX_GEN11(n) _MMIO(0xa540 + (n) * 4) ++#define FORCEWAKE_MEDIA_VEBOX_GEN11(n) _MMIO(0xa560 + (n) * 4) ++#define FORCEWAKE_RENDER_GEN9 _MMIO(0xa278) ++#define FORCEWAKE_BLITTER_GEN9 _MMIO(0xa188) ++#define FORCEWAKE_ACK_MEDIA_GEN9 _MMIO(0x0D88) ++#define FORCEWAKE_ACK_MEDIA_VDBOX_GEN11(n) _MMIO(0x0D50 + (n) * 4) ++#define FORCEWAKE_ACK_MEDIA_VEBOX_GEN11(n) _MMIO(0x0D70 + (n) * 4) ++#define FORCEWAKE_ACK_RENDER_GEN9 _MMIO(0x0D84) ++#define FORCEWAKE_ACK_BLITTER_GEN9 _MMIO(0x130044) ++#define FORCEWAKE_KERNEL BIT(0) ++#define FORCEWAKE_USER BIT(1) ++#define FORCEWAKE_KERNEL_FALLBACK BIT(15) ++#define FORCEWAKE_MT_ACK _MMIO(0x130040) ++#define ECOBUS _MMIO(0xa180) ++#define FORCEWAKE_MT_ENABLE (1 << 5) ++#define VLV_SPAREG2H _MMIO(0xA194) ++#define GEN9_PWRGT_DOMAIN_STATUS _MMIO(0xA2A0) ++#define GEN9_PWRGT_MEDIA_STATUS_MASK (1 << 0) ++#define GEN9_PWRGT_RENDER_STATUS_MASK (1 << 1) ++ ++#define GTFIFODBG _MMIO(0x120000) ++#define GT_FIFO_SBDEDICATE_FREE_ENTRY_CHV (0x1f << 20) ++#define GT_FIFO_FREE_ENTRIES_CHV (0x7f << 13) ++#define GT_FIFO_SBDROPERR (1 << 6) ++#define GT_FIFO_BLOBDROPERR (1 << 5) ++#define GT_FIFO_SB_READ_ABORTERR (1 << 4) ++#define GT_FIFO_DROPERR (1 << 3) ++#define GT_FIFO_OVFERR (1 << 2) ++#define GT_FIFO_IAWRERR (1 << 1) ++#define GT_FIFO_IARDERR (1 << 0) ++ ++#define GTFIFOCTL _MMIO(0x120008) ++#define GT_FIFO_FREE_ENTRIES_MASK 0x7f ++#define GT_FIFO_NUM_RESERVED_ENTRIES 20 ++#define GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL (1 << 12) ++#define GT_FIFO_CTL_RC6_POLICY_STALL (1 << 11) ++ ++#define HSW_IDICR _MMIO(0x9008) ++#define IDIHASHMSK(x) (((x) & 0x3f) << 16) ++#define HSW_EDRAM_CAP _MMIO(0x120010) ++#define EDRAM_ENABLED 0x1 ++#define EDRAM_NUM_BANKS(cap) (((cap) >> 1) & 0xf) ++#define EDRAM_WAYS_IDX(cap) (((cap) >> 5) & 0x7) ++#define EDRAM_SETS_IDX(cap) (((cap) >> 8) & 0x3) ++ ++#define GEN6_UCGCTL1 _MMIO(0x9400) ++# define GEN6_GAMUNIT_CLOCK_GATE_DISABLE (1 << 22) ++# define GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE (1 << 16) ++# define GEN6_BLBUNIT_CLOCK_GATE_DISABLE (1 << 5) ++# define GEN6_CSUNIT_CLOCK_GATE_DISABLE (1 << 7) ++ ++#define GEN6_UCGCTL2 _MMIO(0x9404) ++# define GEN6_VFUNIT_CLOCK_GATE_DISABLE (1 << 31) ++# define GEN7_VDSUNIT_CLOCK_GATE_DISABLE (1 << 30) ++# define GEN7_TDLUNIT_CLOCK_GATE_DISABLE (1 << 22) ++# define GEN6_RCZUNIT_CLOCK_GATE_DISABLE (1 << 13) ++# define GEN6_RCPBUNIT_CLOCK_GATE_DISABLE (1 << 12) ++# define GEN6_RCCUNIT_CLOCK_GATE_DISABLE (1 << 11) ++ ++#define GEN6_UCGCTL3 _MMIO(0x9408) ++# define GEN6_OACSUNIT_CLOCK_GATE_DISABLE (1 << 20) ++ ++#define GEN7_UCGCTL4 _MMIO(0x940c) ++#define GEN7_L3BANK2X_CLOCK_GATE_DISABLE (1 << 25) ++#define GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE (1 << 14) ++ ++#define GEN6_RCGCTL1 _MMIO(0x9410) ++#define GEN6_RCGCTL2 _MMIO(0x9414) ++#define GEN6_RSTCTL _MMIO(0x9420) ++ ++#define GEN8_UCGCTL6 _MMIO(0x9430) ++#define GEN8_GAPSUNIT_CLOCK_GATE_DISABLE (1 << 24) ++#define GEN8_SDEUNIT_CLOCK_GATE_DISABLE (1 << 14) ++#define GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ (1 << 28) ++ ++#define GEN6_GFXPAUSE _MMIO(0xA000) ++#define GEN6_RPNSWREQ _MMIO(0xA008) ++#define GEN6_TURBO_DISABLE (1 << 31) ++#define GEN6_FREQUENCY(x) ((x) << 25) ++#define HSW_FREQUENCY(x) ((x) << 24) ++#define GEN9_FREQUENCY(x) ((x) << 23) ++#define GEN6_OFFSET(x) ((x) << 19) ++#define GEN6_AGGRESSIVE_TURBO (0 << 15) ++#define GEN6_RC_VIDEO_FREQ _MMIO(0xA00C) ++#define GEN6_RC_CONTROL _MMIO(0xA090) ++#define GEN6_RC_CTL_RC6pp_ENABLE (1 << 16) ++#define GEN6_RC_CTL_RC6p_ENABLE (1 << 17) ++#define GEN6_RC_CTL_RC6_ENABLE (1 << 18) ++#define GEN6_RC_CTL_RC1e_ENABLE (1 << 20) ++#define GEN6_RC_CTL_RC7_ENABLE (1 << 22) ++#define VLV_RC_CTL_CTX_RST_PARALLEL (1 << 24) ++#define GEN7_RC_CTL_TO_MODE (1 << 28) ++#define GEN6_RC_CTL_EI_MODE(x) ((x) << 27) ++#define GEN6_RC_CTL_HW_ENABLE (1 << 31) ++#define GEN6_RP_DOWN_TIMEOUT _MMIO(0xA010) ++#define GEN6_RP_INTERRUPT_LIMITS _MMIO(0xA014) ++#define GEN6_RPSTAT1 _MMIO(0xA01C) ++#define GEN6_CAGF_SHIFT 8 ++#define HSW_CAGF_SHIFT 7 ++#define GEN9_CAGF_SHIFT 23 ++#define GEN6_CAGF_MASK (0x7f << GEN6_CAGF_SHIFT) ++#define HSW_CAGF_MASK (0x7f << HSW_CAGF_SHIFT) ++#define GEN9_CAGF_MASK (0x1ff << GEN9_CAGF_SHIFT) ++#define GEN6_RP_CONTROL _MMIO(0xA024) ++#define GEN6_RP_MEDIA_TURBO (1 << 11) ++#define GEN6_RP_MEDIA_MODE_MASK (3 << 9) ++#define GEN6_RP_MEDIA_HW_TURBO_MODE (3 << 9) ++#define GEN6_RP_MEDIA_HW_NORMAL_MODE (2 << 9) ++#define GEN6_RP_MEDIA_HW_MODE (1 << 9) ++#define GEN6_RP_MEDIA_SW_MODE (0 << 9) ++#define GEN6_RP_MEDIA_IS_GFX (1 << 8) ++#define GEN6_RP_ENABLE (1 << 7) ++#define GEN6_RP_UP_IDLE_MIN (0x1 << 3) ++#define GEN6_RP_UP_BUSY_AVG (0x2 << 3) ++#define GEN6_RP_UP_BUSY_CONT (0x4 << 3) ++#define GEN6_RP_DOWN_IDLE_AVG (0x2 << 0) ++#define GEN6_RP_DOWN_IDLE_CONT (0x1 << 0) ++#define GEN6_RP_UP_THRESHOLD _MMIO(0xA02C) ++#define GEN6_RP_DOWN_THRESHOLD _MMIO(0xA030) ++#define GEN6_RP_CUR_UP_EI _MMIO(0xA050) ++#define GEN6_RP_EI_MASK 0xffffff ++#define GEN6_CURICONT_MASK GEN6_RP_EI_MASK ++#define GEN6_RP_CUR_UP _MMIO(0xA054) ++#define GEN6_CURBSYTAVG_MASK GEN6_RP_EI_MASK ++#define GEN6_RP_PREV_UP _MMIO(0xA058) ++#define GEN6_RP_CUR_DOWN_EI _MMIO(0xA05C) ++#define GEN6_CURIAVG_MASK GEN6_RP_EI_MASK ++#define GEN6_RP_CUR_DOWN _MMIO(0xA060) ++#define GEN6_RP_PREV_DOWN _MMIO(0xA064) ++#define GEN6_RP_UP_EI _MMIO(0xA068) ++#define GEN6_RP_DOWN_EI _MMIO(0xA06C) ++#define GEN6_RP_IDLE_HYSTERSIS _MMIO(0xA070) ++#define GEN6_RPDEUHWTC _MMIO(0xA080) ++#define GEN6_RPDEUC _MMIO(0xA084) ++#define GEN6_RPDEUCSW _MMIO(0xA088) ++#define GEN6_RC_STATE _MMIO(0xA094) ++#define RC_SW_TARGET_STATE_SHIFT 16 ++#define RC_SW_TARGET_STATE_MASK (7 << RC_SW_TARGET_STATE_SHIFT) ++#define GEN6_RC1_WAKE_RATE_LIMIT _MMIO(0xA098) ++#define GEN6_RC6_WAKE_RATE_LIMIT _MMIO(0xA09C) ++#define GEN6_RC6pp_WAKE_RATE_LIMIT _MMIO(0xA0A0) ++#define GEN10_MEDIA_WAKE_RATE_LIMIT _MMIO(0xA0A0) ++#define GEN6_RC_EVALUATION_INTERVAL _MMIO(0xA0A8) ++#define GEN6_RC_IDLE_HYSTERSIS _MMIO(0xA0AC) ++#define GEN6_RC_SLEEP _MMIO(0xA0B0) ++#define GEN6_RCUBMABDTMR _MMIO(0xA0B0) ++#define GEN6_RC1e_THRESHOLD _MMIO(0xA0B4) ++#define GEN6_RC6_THRESHOLD _MMIO(0xA0B8) ++#define GEN6_RC6p_THRESHOLD _MMIO(0xA0BC) ++#define VLV_RCEDATA _MMIO(0xA0BC) ++#define GEN6_RC6pp_THRESHOLD _MMIO(0xA0C0) ++#define GEN6_PMINTRMSK _MMIO(0xA168) ++#define GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC (1 << 31) ++#define ARAT_EXPIRED_INTRMSK (1 << 9) ++#define GEN8_MISC_CTRL0 _MMIO(0xA180) ++#define VLV_PWRDWNUPCTL _MMIO(0xA294) ++#define GEN9_MEDIA_PG_IDLE_HYSTERESIS _MMIO(0xA0C4) ++#define GEN9_RENDER_PG_IDLE_HYSTERESIS _MMIO(0xA0C8) ++#define GEN9_PG_ENABLE _MMIO(0xA210) ++#define GEN9_RENDER_PG_ENABLE REG_BIT(0) ++#define GEN9_MEDIA_PG_ENABLE REG_BIT(1) ++#define GEN11_MEDIA_SAMPLER_PG_ENABLE REG_BIT(2) ++#define GEN8_PUSHBUS_CONTROL _MMIO(0xA248) ++#define GEN8_PUSHBUS_ENABLE _MMIO(0xA250) ++#define GEN8_PUSHBUS_SHIFT _MMIO(0xA25C) ++ ++#define VLV_CHICKEN_3 _MMIO(VLV_DISPLAY_BASE + 0x7040C) ++#define PIXEL_OVERLAP_CNT_MASK (3 << 30) ++#define PIXEL_OVERLAP_CNT_SHIFT 30 ++ ++#define GEN6_PMISR _MMIO(0x44020) ++#define GEN6_PMIMR _MMIO(0x44024) /* rps_lock */ ++#define GEN6_PMIIR _MMIO(0x44028) ++#define GEN6_PMIER _MMIO(0x4402C) ++#define GEN6_PM_MBOX_EVENT (1 << 25) ++#define GEN6_PM_THERMAL_EVENT (1 << 24) ++ ++/* ++ * For Gen11 these are in the upper word of the GPM_WGBOXPERF ++ * registers. Shifting is handled on accessing the imr and ier. ++ */ ++#define GEN6_PM_RP_DOWN_TIMEOUT (1 << 6) ++#define GEN6_PM_RP_UP_THRESHOLD (1 << 5) ++#define GEN6_PM_RP_DOWN_THRESHOLD (1 << 4) ++#define GEN6_PM_RP_UP_EI_EXPIRED (1 << 2) ++#define GEN6_PM_RP_DOWN_EI_EXPIRED (1 << 1) ++#define GEN6_PM_RPS_EVENTS (GEN6_PM_RP_UP_EI_EXPIRED | \ ++ GEN6_PM_RP_UP_THRESHOLD | \ ++ GEN6_PM_RP_DOWN_EI_EXPIRED | \ ++ GEN6_PM_RP_DOWN_THRESHOLD | \ ++ GEN6_PM_RP_DOWN_TIMEOUT) ++ ++#define GEN7_GT_SCRATCH(i) _MMIO(0x4F100 + (i) * 4) ++#define GEN7_GT_SCRATCH_REG_NUM 8 ++ ++#define VLV_GTLC_SURVIVABILITY_REG _MMIO(0x130098) ++#define VLV_GFX_CLK_STATUS_BIT (1 << 3) ++#define VLV_GFX_CLK_FORCE_ON_BIT (1 << 2) ++ ++#define GEN6_GT_GFX_RC6_LOCKED _MMIO(0x138104) ++#define VLV_COUNTER_CONTROL _MMIO(0x138104) ++#define VLV_COUNT_RANGE_HIGH (1 << 15) ++#define VLV_MEDIA_RC0_COUNT_EN (1 << 5) ++#define VLV_RENDER_RC0_COUNT_EN (1 << 4) ++#define VLV_MEDIA_RC6_COUNT_EN (1 << 1) ++#define VLV_RENDER_RC6_COUNT_EN (1 << 0) ++#define GEN6_GT_GFX_RC6 _MMIO(0x138108) ++#define VLV_GT_RENDER_RC6 _MMIO(0x138108) ++#define VLV_GT_MEDIA_RC6 _MMIO(0x13810C) ++ ++#define GEN6_GT_GFX_RC6p _MMIO(0x13810C) ++#define GEN6_GT_GFX_RC6pp _MMIO(0x138110) ++#define VLV_RENDER_C0_COUNT _MMIO(0x138118) ++#define VLV_MEDIA_C0_COUNT _MMIO(0x13811C) ++ ++#define GEN6_PCODE_MAILBOX _MMIO(0x138124) ++#define GEN6_PCODE_READY (1 << 31) ++#define GEN6_PCODE_ERROR_MASK 0xFF ++#define GEN6_PCODE_SUCCESS 0x0 ++#define GEN6_PCODE_ILLEGAL_CMD 0x1 ++#define GEN6_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE 0x2 ++#define GEN6_PCODE_TIMEOUT 0x3 ++#define GEN6_PCODE_UNIMPLEMENTED_CMD 0xFF ++#define GEN7_PCODE_TIMEOUT 0x2 ++#define GEN7_PCODE_ILLEGAL_DATA 0x3 ++#define GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE 0x10 ++#define GEN6_PCODE_WRITE_RC6VIDS 0x4 ++#define GEN6_PCODE_READ_RC6VIDS 0x5 ++#define GEN6_ENCODE_RC6_VID(mv) (((mv) - 245) / 5) ++#define GEN6_DECODE_RC6_VID(vids) (((vids) * 5) + 245) ++#define BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ 0x18 ++#define GEN9_PCODE_READ_MEM_LATENCY 0x6 ++#define GEN9_MEM_LATENCY_LEVEL_MASK 0xFF ++#define GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT 8 ++#define GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT 16 ++#define GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT 24 ++#define SKL_PCODE_LOAD_HDCP_KEYS 0x5 ++#define SKL_PCODE_CDCLK_CONTROL 0x7 ++#define SKL_CDCLK_PREPARE_FOR_CHANGE 0x3 ++#define SKL_CDCLK_READY_FOR_CHANGE 0x1 ++#define GEN6_PCODE_WRITE_MIN_FREQ_TABLE 0x8 ++#define GEN6_PCODE_READ_MIN_FREQ_TABLE 0x9 ++#define GEN6_READ_OC_PARAMS 0xc ++#define GEN6_PCODE_READ_D_COMP 0x10 ++#define GEN6_PCODE_WRITE_D_COMP 0x11 ++#define HSW_PCODE_DE_WRITE_FREQ_REQ 0x17 ++#define DISPLAY_IPS_CONTROL 0x19 ++ /* See also IPS_CTL */ ++#define IPS_PCODE_CONTROL (1 << 30) ++#define HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL 0x1A ++#define GEN9_PCODE_SAGV_CONTROL 0x21 ++#define GEN9_SAGV_DISABLE 0x0 ++#define GEN9_SAGV_IS_DISABLED 0x1 ++#define GEN9_SAGV_ENABLE 0x3 ++#define GEN6_PCODE_DATA _MMIO(0x138128) ++#define GEN6_PCODE_FREQ_IA_RATIO_SHIFT 8 ++#define GEN6_PCODE_FREQ_RING_RATIO_SHIFT 16 ++#define GEN6_PCODE_DATA1 _MMIO(0x13812C) ++ ++#define GEN6_GT_CORE_STATUS _MMIO(0x138060) ++#define GEN6_CORE_CPD_STATE_MASK (7 << 4) ++#define GEN6_RCn_MASK 7 ++#define GEN6_RC0 0 ++#define GEN6_RC3 2 ++#define GEN6_RC6 3 ++#define GEN6_RC7 4 ++ ++#define GEN8_GT_SLICE_INFO _MMIO(0x138064) ++#define GEN8_LSLICESTAT_MASK 0x7 ++ ++#define CHV_POWER_SS0_SIG1 _MMIO(0xa720) ++#define CHV_POWER_SS1_SIG1 _MMIO(0xa728) ++#define CHV_SS_PG_ENABLE (1 << 1) ++#define CHV_EU08_PG_ENABLE (1 << 9) ++#define CHV_EU19_PG_ENABLE (1 << 17) ++#define CHV_EU210_PG_ENABLE (1 << 25) ++ ++#define CHV_POWER_SS0_SIG2 _MMIO(0xa724) ++#define CHV_POWER_SS1_SIG2 _MMIO(0xa72c) ++#define CHV_EU311_PG_ENABLE (1 << 1) ++ ++#define GEN9_SLICE_PGCTL_ACK(slice) _MMIO(0x804c + (slice) * 0x4) ++#define GEN10_SLICE_PGCTL_ACK(slice) _MMIO(0x804c + ((slice) / 3) * 0x34 + \ ++ ((slice) % 3) * 0x4) ++#define GEN9_PGCTL_SLICE_ACK (1 << 0) ++#define GEN9_PGCTL_SS_ACK(subslice) (1 << (2 + (subslice) * 2)) ++#define GEN10_PGCTL_VALID_SS_MASK(slice) ((slice) == 0 ? 0x7F : 0x1F) ++ ++#define GEN9_SS01_EU_PGCTL_ACK(slice) _MMIO(0x805c + (slice) * 0x8) ++#define GEN10_SS01_EU_PGCTL_ACK(slice) _MMIO(0x805c + ((slice) / 3) * 0x30 + \ ++ ((slice) % 3) * 0x8) ++#define GEN9_SS23_EU_PGCTL_ACK(slice) _MMIO(0x8060 + (slice) * 0x8) ++#define GEN10_SS23_EU_PGCTL_ACK(slice) _MMIO(0x8060 + ((slice) / 3) * 0x30 + \ ++ ((slice) % 3) * 0x8) ++#define GEN9_PGCTL_SSA_EU08_ACK (1 << 0) ++#define GEN9_PGCTL_SSA_EU19_ACK (1 << 2) ++#define GEN9_PGCTL_SSA_EU210_ACK (1 << 4) ++#define GEN9_PGCTL_SSA_EU311_ACK (1 << 6) ++#define GEN9_PGCTL_SSB_EU08_ACK (1 << 8) ++#define GEN9_PGCTL_SSB_EU19_ACK (1 << 10) ++#define GEN9_PGCTL_SSB_EU210_ACK (1 << 12) ++#define GEN9_PGCTL_SSB_EU311_ACK (1 << 14) ++ ++#define GEN7_MISCCPCTL _MMIO(0x9424) ++#define GEN7_DOP_CLOCK_GATE_ENABLE (1 << 0) ++#define GEN8_DOP_CLOCK_GATE_CFCLK_ENABLE (1 << 2) ++#define GEN8_DOP_CLOCK_GATE_GUC_ENABLE (1 << 4) ++#define GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE (1 << 6) ++ ++#define GEN8_GARBCNTL _MMIO(0xB004) ++#define GEN9_GAPS_TSV_CREDIT_DISABLE (1 << 7) ++#define GEN11_ARBITRATION_PRIO_ORDER_MASK (0x3f << 22) ++#define GEN11_HASH_CTRL_EXCL_MASK (0x7f << 0) ++#define GEN11_HASH_CTRL_EXCL_BIT0 (1 << 0) ++ ++#define GEN11_GLBLINVL _MMIO(0xB404) ++#define GEN11_BANK_HASH_ADDR_EXCL_MASK (0x7f << 5) ++#define GEN11_BANK_HASH_ADDR_EXCL_BIT0 (1 << 5) ++ ++#define GEN10_DFR_RATIO_EN_AND_CHICKEN _MMIO(0x9550) ++#define DFR_DISABLE (1 << 9) ++ ++#define GEN11_GACB_PERF_CTRL _MMIO(0x4B80) ++#define GEN11_HASH_CTRL_MASK (0x3 << 12 | 0xf << 0) ++#define GEN11_HASH_CTRL_BIT0 (1 << 0) ++#define GEN11_HASH_CTRL_BIT4 (1 << 12) ++ ++#define GEN11_LSN_UNSLCVC _MMIO(0xB43C) ++#define GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC (1 << 9) ++#define GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC (1 << 7) ++ ++#define GEN10_SAMPLER_MODE _MMIO(0xE18C) ++ ++/* IVYBRIDGE DPF */ ++#define GEN7_L3CDERRST1(slice) _MMIO(0xB008 + (slice) * 0x200) /* L3CD Error Status 1 */ ++#define GEN7_L3CDERRST1_ROW_MASK (0x7ff << 14) ++#define GEN7_PARITY_ERROR_VALID (1 << 13) ++#define GEN7_L3CDERRST1_BANK_MASK (3 << 11) ++#define GEN7_L3CDERRST1_SUBBANK_MASK (7 << 8) ++#define GEN7_PARITY_ERROR_ROW(reg) \ ++ (((reg) & GEN7_L3CDERRST1_ROW_MASK) >> 14) ++#define GEN7_PARITY_ERROR_BANK(reg) \ ++ (((reg) & GEN7_L3CDERRST1_BANK_MASK) >> 11) ++#define GEN7_PARITY_ERROR_SUBBANK(reg) \ ++ (((reg) & GEN7_L3CDERRST1_SUBBANK_MASK) >> 8) ++#define GEN7_L3CDERRST1_ENABLE (1 << 7) ++ ++#define GEN7_L3LOG(slice, i) _MMIO(0xB070 + (slice) * 0x200 + (i) * 4) ++#define GEN7_L3LOG_SIZE 0x80 ++ ++#define GEN7_HALF_SLICE_CHICKEN1 _MMIO(0xe100) /* IVB GT1 + VLV */ ++#define GEN7_HALF_SLICE_CHICKEN1_GT2 _MMIO(0xf100) ++#define GEN7_MAX_PS_THREAD_DEP (8 << 12) ++#define GEN7_SINGLE_SUBSCAN_DISPATCH_ENABLE (1 << 10) ++#define GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE (1 << 4) ++#define GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE (1 << 3) ++ ++#define GEN9_HALF_SLICE_CHICKEN5 _MMIO(0xe188) ++#define GEN9_DG_MIRROR_FIX_ENABLE (1 << 5) ++#define GEN9_CCS_TLB_PREFETCH_ENABLE (1 << 3) ++ ++#define GEN8_ROW_CHICKEN _MMIO(0xe4f0) ++#define FLOW_CONTROL_ENABLE (1 << 15) ++#define PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE (1 << 8) ++#define STALL_DOP_GATING_DISABLE (1 << 5) ++#define THROTTLE_12_5 (7 << 2) ++#define DISABLE_EARLY_EOT (1 << 1) ++ ++#define GEN7_ROW_CHICKEN2 _MMIO(0xe4f4) ++#define GEN7_ROW_CHICKEN2_GT2 _MMIO(0xf4f4) ++#define DOP_CLOCK_GATING_DISABLE (1 << 0) ++#define PUSH_CONSTANT_DEREF_DISABLE (1 << 8) ++#define GEN11_TDL_CLOCK_GATING_FIX_DISABLE (1 << 1) ++ ++#define HSW_ROW_CHICKEN3 _MMIO(0xe49c) ++#define HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE (1 << 6) ++ ++#define HALF_SLICE_CHICKEN2 _MMIO(0xe180) ++#define GEN8_ST_PO_DISABLE (1 << 13) ++ ++#define HALF_SLICE_CHICKEN3 _MMIO(0xe184) ++#define HSW_SAMPLE_C_PERFORMANCE (1 << 9) ++#define GEN8_CENTROID_PIXEL_OPT_DIS (1 << 8) ++#define GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC (1 << 5) ++#define CNL_FAST_ANISO_L1_BANKING_FIX (1 << 4) ++#define GEN8_SAMPLER_POWER_BYPASS_DIS (1 << 1) ++ ++#define GEN9_HALF_SLICE_CHICKEN7 _MMIO(0xe194) ++#define GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR (1 << 8) ++#define GEN9_ENABLE_YV12_BUGFIX (1 << 4) ++#define GEN9_ENABLE_GPGPU_PREEMPTION (1 << 2) ++ ++/* Audio */ ++#define G4X_AUD_VID_DID _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x62020) ++#define INTEL_AUDIO_DEVCL 0x808629FB ++#define INTEL_AUDIO_DEVBLC 0x80862801 ++#define INTEL_AUDIO_DEVCTG 0x80862802 ++ ++#define G4X_AUD_CNTL_ST _MMIO(0x620B4) ++#define G4X_ELDV_DEVCL_DEVBLC (1 << 13) ++#define G4X_ELDV_DEVCTG (1 << 14) ++#define G4X_ELD_ADDR_MASK (0xf << 5) ++#define G4X_ELD_ACK (1 << 4) ++#define G4X_HDMIW_HDMIEDID _MMIO(0x6210C) ++ ++#define _IBX_HDMIW_HDMIEDID_A 0xE2050 ++#define _IBX_HDMIW_HDMIEDID_B 0xE2150 ++#define IBX_HDMIW_HDMIEDID(pipe) _MMIO_PIPE(pipe, _IBX_HDMIW_HDMIEDID_A, \ ++ _IBX_HDMIW_HDMIEDID_B) ++#define _IBX_AUD_CNTL_ST_A 0xE20B4 ++#define _IBX_AUD_CNTL_ST_B 0xE21B4 ++#define IBX_AUD_CNTL_ST(pipe) _MMIO_PIPE(pipe, _IBX_AUD_CNTL_ST_A, \ ++ _IBX_AUD_CNTL_ST_B) ++#define IBX_ELD_BUFFER_SIZE_MASK (0x1f << 10) ++#define IBX_ELD_ADDRESS_MASK (0x1f << 5) ++#define IBX_ELD_ACK (1 << 4) ++#define IBX_AUD_CNTL_ST2 _MMIO(0xE20C0) ++#define IBX_CP_READY(port) ((1 << 1) << (((port) - 1) * 4)) ++#define IBX_ELD_VALID(port) ((1 << 0) << (((port) - 1) * 4)) ++ ++#define _CPT_HDMIW_HDMIEDID_A 0xE5050 ++#define _CPT_HDMIW_HDMIEDID_B 0xE5150 ++#define CPT_HDMIW_HDMIEDID(pipe) _MMIO_PIPE(pipe, _CPT_HDMIW_HDMIEDID_A, _CPT_HDMIW_HDMIEDID_B) ++#define _CPT_AUD_CNTL_ST_A 0xE50B4 ++#define _CPT_AUD_CNTL_ST_B 0xE51B4 ++#define CPT_AUD_CNTL_ST(pipe) _MMIO_PIPE(pipe, _CPT_AUD_CNTL_ST_A, _CPT_AUD_CNTL_ST_B) ++#define CPT_AUD_CNTRL_ST2 _MMIO(0xE50C0) ++ ++#define _VLV_HDMIW_HDMIEDID_A (VLV_DISPLAY_BASE + 0x62050) ++#define _VLV_HDMIW_HDMIEDID_B (VLV_DISPLAY_BASE + 0x62150) ++#define VLV_HDMIW_HDMIEDID(pipe) _MMIO_PIPE(pipe, _VLV_HDMIW_HDMIEDID_A, _VLV_HDMIW_HDMIEDID_B) ++#define _VLV_AUD_CNTL_ST_A (VLV_DISPLAY_BASE + 0x620B4) ++#define _VLV_AUD_CNTL_ST_B (VLV_DISPLAY_BASE + 0x621B4) ++#define VLV_AUD_CNTL_ST(pipe) _MMIO_PIPE(pipe, _VLV_AUD_CNTL_ST_A, _VLV_AUD_CNTL_ST_B) ++#define VLV_AUD_CNTL_ST2 _MMIO(VLV_DISPLAY_BASE + 0x620C0) ++ ++/* These are the 4 32-bit write offset registers for each stream ++ * output buffer. It determines the offset from the ++ * 3DSTATE_SO_BUFFERs that the next streamed vertex output goes to. ++ */ ++#define GEN7_SO_WRITE_OFFSET(n) _MMIO(0x5280 + (n) * 4) ++ ++#define _IBX_AUD_CONFIG_A 0xe2000 ++#define _IBX_AUD_CONFIG_B 0xe2100 ++#define IBX_AUD_CFG(pipe) _MMIO_PIPE(pipe, _IBX_AUD_CONFIG_A, _IBX_AUD_CONFIG_B) ++#define _CPT_AUD_CONFIG_A 0xe5000 ++#define _CPT_AUD_CONFIG_B 0xe5100 ++#define CPT_AUD_CFG(pipe) _MMIO_PIPE(pipe, _CPT_AUD_CONFIG_A, _CPT_AUD_CONFIG_B) ++#define _VLV_AUD_CONFIG_A (VLV_DISPLAY_BASE + 0x62000) ++#define _VLV_AUD_CONFIG_B (VLV_DISPLAY_BASE + 0x62100) ++#define VLV_AUD_CFG(pipe) _MMIO_PIPE(pipe, _VLV_AUD_CONFIG_A, _VLV_AUD_CONFIG_B) ++ ++#define AUD_CONFIG_N_VALUE_INDEX (1 << 29) ++#define AUD_CONFIG_N_PROG_ENABLE (1 << 28) ++#define AUD_CONFIG_UPPER_N_SHIFT 20 ++#define AUD_CONFIG_UPPER_N_MASK (0xff << 20) ++#define AUD_CONFIG_LOWER_N_SHIFT 4 ++#define AUD_CONFIG_LOWER_N_MASK (0xfff << 4) ++#define AUD_CONFIG_N_MASK (AUD_CONFIG_UPPER_N_MASK | AUD_CONFIG_LOWER_N_MASK) ++#define AUD_CONFIG_N(n) \ ++ (((((n) >> 12) & 0xff) << AUD_CONFIG_UPPER_N_SHIFT) | \ ++ (((n) & 0xfff) << AUD_CONFIG_LOWER_N_SHIFT)) ++#define AUD_CONFIG_PIXEL_CLOCK_HDMI_SHIFT 16 ++#define AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK (0xf << 16) ++#define AUD_CONFIG_PIXEL_CLOCK_HDMI_25175 (0 << 16) ++#define AUD_CONFIG_PIXEL_CLOCK_HDMI_25200 (1 << 16) ++#define AUD_CONFIG_PIXEL_CLOCK_HDMI_27000 (2 << 16) ++#define AUD_CONFIG_PIXEL_CLOCK_HDMI_27027 (3 << 16) ++#define AUD_CONFIG_PIXEL_CLOCK_HDMI_54000 (4 << 16) ++#define AUD_CONFIG_PIXEL_CLOCK_HDMI_54054 (5 << 16) ++#define AUD_CONFIG_PIXEL_CLOCK_HDMI_74176 (6 << 16) ++#define AUD_CONFIG_PIXEL_CLOCK_HDMI_74250 (7 << 16) ++#define AUD_CONFIG_PIXEL_CLOCK_HDMI_148352 (8 << 16) ++#define AUD_CONFIG_PIXEL_CLOCK_HDMI_148500 (9 << 16) ++#define AUD_CONFIG_DISABLE_NCTS (1 << 3) ++ ++/* HSW Audio */ ++#define _HSW_AUD_CONFIG_A 0x65000 ++#define _HSW_AUD_CONFIG_B 0x65100 ++#define HSW_AUD_CFG(pipe) _MMIO_PIPE(pipe, _HSW_AUD_CONFIG_A, _HSW_AUD_CONFIG_B) ++ ++#define _HSW_AUD_MISC_CTRL_A 0x65010 ++#define _HSW_AUD_MISC_CTRL_B 0x65110 ++#define HSW_AUD_MISC_CTRL(pipe) _MMIO_PIPE(pipe, _HSW_AUD_MISC_CTRL_A, _HSW_AUD_MISC_CTRL_B) ++ ++#define _HSW_AUD_M_CTS_ENABLE_A 0x65028 ++#define _HSW_AUD_M_CTS_ENABLE_B 0x65128 ++#define HSW_AUD_M_CTS_ENABLE(pipe) _MMIO_PIPE(pipe, _HSW_AUD_M_CTS_ENABLE_A, _HSW_AUD_M_CTS_ENABLE_B) ++#define AUD_M_CTS_M_VALUE_INDEX (1 << 21) ++#define AUD_M_CTS_M_PROG_ENABLE (1 << 20) ++#define AUD_CONFIG_M_MASK 0xfffff ++ ++#define _HSW_AUD_DIP_ELD_CTRL_ST_A 0x650b4 ++#define _HSW_AUD_DIP_ELD_CTRL_ST_B 0x651b4 ++#define HSW_AUD_DIP_ELD_CTRL(pipe) _MMIO_PIPE(pipe, _HSW_AUD_DIP_ELD_CTRL_ST_A, _HSW_AUD_DIP_ELD_CTRL_ST_B) ++ ++/* Audio Digital Converter */ ++#define _HSW_AUD_DIG_CNVT_1 0x65080 ++#define _HSW_AUD_DIG_CNVT_2 0x65180 ++#define AUD_DIG_CNVT(pipe) _MMIO_PIPE(pipe, _HSW_AUD_DIG_CNVT_1, _HSW_AUD_DIG_CNVT_2) ++#define DIP_PORT_SEL_MASK 0x3 ++ ++#define _HSW_AUD_EDID_DATA_A 0x65050 ++#define _HSW_AUD_EDID_DATA_B 0x65150 ++#define HSW_AUD_EDID_DATA(pipe) _MMIO_PIPE(pipe, _HSW_AUD_EDID_DATA_A, _HSW_AUD_EDID_DATA_B) ++ ++#define HSW_AUD_PIPE_CONV_CFG _MMIO(0x6507c) ++#define HSW_AUD_PIN_ELD_CP_VLD _MMIO(0x650c0) ++#define AUDIO_INACTIVE(trans) ((1 << 3) << ((trans) * 4)) ++#define AUDIO_OUTPUT_ENABLE(trans) ((1 << 2) << ((trans) * 4)) ++#define AUDIO_CP_READY(trans) ((1 << 1) << ((trans) * 4)) ++#define AUDIO_ELD_VALID(trans) ((1 << 0) << ((trans) * 4)) ++ ++#define HSW_AUD_CHICKENBIT _MMIO(0x65f10) ++#define SKL_AUD_CODEC_WAKE_SIGNAL (1 << 15) ++ ++/* ++ * HSW - ICL power wells ++ * ++ * Platforms have up to 3 power well control register sets, each set ++ * controlling up to 16 power wells via a request/status HW flag tuple: ++ * - main (HSW_PWR_WELL_CTL[1-4]) ++ * - AUX (ICL_PWR_WELL_CTL_AUX[1-4]) ++ * - DDI (ICL_PWR_WELL_CTL_DDI[1-4]) ++ * Each control register set consists of up to 4 registers used by different ++ * sources that can request a power well to be enabled: ++ * - BIOS (HSW_PWR_WELL_CTL1/ICL_PWR_WELL_CTL_AUX1/ICL_PWR_WELL_CTL_DDI1) ++ * - DRIVER (HSW_PWR_WELL_CTL2/ICL_PWR_WELL_CTL_AUX2/ICL_PWR_WELL_CTL_DDI2) ++ * - KVMR (HSW_PWR_WELL_CTL3) (only in the main register set) ++ * - DEBUG (HSW_PWR_WELL_CTL4/ICL_PWR_WELL_CTL_AUX4/ICL_PWR_WELL_CTL_DDI4) ++ */ ++#define HSW_PWR_WELL_CTL1 _MMIO(0x45400) ++#define HSW_PWR_WELL_CTL2 _MMIO(0x45404) ++#define HSW_PWR_WELL_CTL3 _MMIO(0x45408) ++#define HSW_PWR_WELL_CTL4 _MMIO(0x4540C) ++#define HSW_PWR_WELL_CTL_REQ(pw_idx) (0x2 << ((pw_idx) * 2)) ++#define HSW_PWR_WELL_CTL_STATE(pw_idx) (0x1 << ((pw_idx) * 2)) ++ ++/* HSW/BDW power well */ ++#define HSW_PW_CTL_IDX_GLOBAL 15 ++ ++/* SKL/BXT/GLK/CNL power wells */ ++#define SKL_PW_CTL_IDX_PW_2 15 ++#define SKL_PW_CTL_IDX_PW_1 14 ++#define CNL_PW_CTL_IDX_AUX_F 12 ++#define CNL_PW_CTL_IDX_AUX_D 11 ++#define GLK_PW_CTL_IDX_AUX_C 10 ++#define GLK_PW_CTL_IDX_AUX_B 9 ++#define GLK_PW_CTL_IDX_AUX_A 8 ++#define CNL_PW_CTL_IDX_DDI_F 6 ++#define SKL_PW_CTL_IDX_DDI_D 4 ++#define SKL_PW_CTL_IDX_DDI_C 3 ++#define SKL_PW_CTL_IDX_DDI_B 2 ++#define SKL_PW_CTL_IDX_DDI_A_E 1 ++#define GLK_PW_CTL_IDX_DDI_A 1 ++#define SKL_PW_CTL_IDX_MISC_IO 0 ++ ++/* ICL - power wells */ ++#define ICL_PW_CTL_IDX_PW_4 3 ++#define ICL_PW_CTL_IDX_PW_3 2 ++#define ICL_PW_CTL_IDX_PW_2 1 ++#define ICL_PW_CTL_IDX_PW_1 0 ++ ++#define ICL_PWR_WELL_CTL_AUX1 _MMIO(0x45440) ++#define ICL_PWR_WELL_CTL_AUX2 _MMIO(0x45444) ++#define ICL_PWR_WELL_CTL_AUX4 _MMIO(0x4544C) ++#define ICL_PW_CTL_IDX_AUX_TBT4 11 ++#define ICL_PW_CTL_IDX_AUX_TBT3 10 ++#define ICL_PW_CTL_IDX_AUX_TBT2 9 ++#define ICL_PW_CTL_IDX_AUX_TBT1 8 ++#define ICL_PW_CTL_IDX_AUX_F 5 ++#define ICL_PW_CTL_IDX_AUX_E 4 ++#define ICL_PW_CTL_IDX_AUX_D 3 ++#define ICL_PW_CTL_IDX_AUX_C 2 ++#define ICL_PW_CTL_IDX_AUX_B 1 ++#define ICL_PW_CTL_IDX_AUX_A 0 ++ ++#define ICL_PWR_WELL_CTL_DDI1 _MMIO(0x45450) ++#define ICL_PWR_WELL_CTL_DDI2 _MMIO(0x45454) ++#define ICL_PWR_WELL_CTL_DDI4 _MMIO(0x4545C) ++#define ICL_PW_CTL_IDX_DDI_F 5 ++#define ICL_PW_CTL_IDX_DDI_E 4 ++#define ICL_PW_CTL_IDX_DDI_D 3 ++#define ICL_PW_CTL_IDX_DDI_C 2 ++#define ICL_PW_CTL_IDX_DDI_B 1 ++#define ICL_PW_CTL_IDX_DDI_A 0 ++ ++/* HSW - power well misc debug registers */ ++#define HSW_PWR_WELL_CTL5 _MMIO(0x45410) ++#define HSW_PWR_WELL_ENABLE_SINGLE_STEP (1 << 31) ++#define HSW_PWR_WELL_PWR_GATE_OVERRIDE (1 << 20) ++#define HSW_PWR_WELL_FORCE_ON (1 << 19) ++#define HSW_PWR_WELL_CTL6 _MMIO(0x45414) ++ ++/* SKL Fuse Status */ ++enum skl_power_gate { ++ SKL_PG0, ++ SKL_PG1, ++ SKL_PG2, ++ ICL_PG3, ++ ICL_PG4, ++}; ++ ++#define SKL_FUSE_STATUS _MMIO(0x42000) ++#define SKL_FUSE_DOWNLOAD_STATUS (1 << 31) ++/* ++ * PG0 is HW controlled, so doesn't have a corresponding power well control knob ++ * SKL_DISP_PW1_IDX..SKL_DISP_PW2_IDX -> PG1..PG2 ++ */ ++#define SKL_PW_CTL_IDX_TO_PG(pw_idx) \ ++ ((pw_idx) - SKL_PW_CTL_IDX_PW_1 + SKL_PG1) ++/* ++ * PG0 is HW controlled, so doesn't have a corresponding power well control knob ++ * ICL_DISP_PW1_IDX..ICL_DISP_PW4_IDX -> PG1..PG4 ++ */ ++#define ICL_PW_CTL_IDX_TO_PG(pw_idx) \ ++ ((pw_idx) - ICL_PW_CTL_IDX_PW_1 + SKL_PG1) ++#define SKL_FUSE_PG_DIST_STATUS(pg) (1 << (27 - (pg))) ++ ++#define _CNL_AUX_REG_IDX(pw_idx) ((pw_idx) - GLK_PW_CTL_IDX_AUX_B) ++#define _CNL_AUX_ANAOVRD1_B 0x162250 ++#define _CNL_AUX_ANAOVRD1_C 0x162210 ++#define _CNL_AUX_ANAOVRD1_D 0x1622D0 ++#define _CNL_AUX_ANAOVRD1_F 0x162A90 ++#define CNL_AUX_ANAOVRD1(pw_idx) _MMIO(_PICK(_CNL_AUX_REG_IDX(pw_idx), \ ++ _CNL_AUX_ANAOVRD1_B, \ ++ _CNL_AUX_ANAOVRD1_C, \ ++ _CNL_AUX_ANAOVRD1_D, \ ++ _CNL_AUX_ANAOVRD1_F)) ++#define CNL_AUX_ANAOVRD1_ENABLE (1 << 16) ++#define CNL_AUX_ANAOVRD1_LDO_BYPASS (1 << 23) ++ ++#define _ICL_AUX_REG_IDX(pw_idx) ((pw_idx) - ICL_PW_CTL_IDX_AUX_A) ++#define _ICL_AUX_ANAOVRD1_A 0x162398 ++#define _ICL_AUX_ANAOVRD1_B 0x6C398 ++#define ICL_AUX_ANAOVRD1(pw_idx) _MMIO(_PICK(_ICL_AUX_REG_IDX(pw_idx), \ ++ _ICL_AUX_ANAOVRD1_A, \ ++ _ICL_AUX_ANAOVRD1_B)) ++#define ICL_AUX_ANAOVRD1_LDO_BYPASS (1 << 7) ++#define ICL_AUX_ANAOVRD1_ENABLE (1 << 0) ++ ++/* HDCP Key Registers */ ++#define HDCP_KEY_CONF _MMIO(0x66c00) ++#define HDCP_AKSV_SEND_TRIGGER BIT(31) ++#define HDCP_CLEAR_KEYS_TRIGGER BIT(30) ++#define HDCP_KEY_LOAD_TRIGGER BIT(8) ++#define HDCP_KEY_STATUS _MMIO(0x66c04) ++#define HDCP_FUSE_IN_PROGRESS BIT(7) ++#define HDCP_FUSE_ERROR BIT(6) ++#define HDCP_FUSE_DONE BIT(5) ++#define HDCP_KEY_LOAD_STATUS BIT(1) ++#define HDCP_KEY_LOAD_DONE BIT(0) ++#define HDCP_AKSV_LO _MMIO(0x66c10) ++#define HDCP_AKSV_HI _MMIO(0x66c14) ++ ++/* HDCP Repeater Registers */ ++#define HDCP_REP_CTL _MMIO(0x66d00) ++#define HDCP_DDIB_REP_PRESENT BIT(30) ++#define HDCP_DDIA_REP_PRESENT BIT(29) ++#define HDCP_DDIC_REP_PRESENT BIT(28) ++#define HDCP_DDID_REP_PRESENT BIT(27) ++#define HDCP_DDIF_REP_PRESENT BIT(26) ++#define HDCP_DDIE_REP_PRESENT BIT(25) ++#define HDCP_DDIB_SHA1_M0 (1 << 20) ++#define HDCP_DDIA_SHA1_M0 (2 << 20) ++#define HDCP_DDIC_SHA1_M0 (3 << 20) ++#define HDCP_DDID_SHA1_M0 (4 << 20) ++#define HDCP_DDIF_SHA1_M0 (5 << 20) ++#define HDCP_DDIE_SHA1_M0 (6 << 20) /* Bspec says 5? */ ++#define HDCP_SHA1_BUSY BIT(16) ++#define HDCP_SHA1_READY BIT(17) ++#define HDCP_SHA1_COMPLETE BIT(18) ++#define HDCP_SHA1_V_MATCH BIT(19) ++#define HDCP_SHA1_TEXT_32 (1 << 1) ++#define HDCP_SHA1_COMPLETE_HASH (2 << 1) ++#define HDCP_SHA1_TEXT_24 (4 << 1) ++#define HDCP_SHA1_TEXT_16 (5 << 1) ++#define HDCP_SHA1_TEXT_8 (6 << 1) ++#define HDCP_SHA1_TEXT_0 (7 << 1) ++#define HDCP_SHA_V_PRIME_H0 _MMIO(0x66d04) ++#define HDCP_SHA_V_PRIME_H1 _MMIO(0x66d08) ++#define HDCP_SHA_V_PRIME_H2 _MMIO(0x66d0C) ++#define HDCP_SHA_V_PRIME_H3 _MMIO(0x66d10) ++#define HDCP_SHA_V_PRIME_H4 _MMIO(0x66d14) ++#define HDCP_SHA_V_PRIME(h) _MMIO((0x66d04 + (h) * 4)) ++#define HDCP_SHA_TEXT _MMIO(0x66d18) ++ ++/* HDCP Auth Registers */ ++#define _PORTA_HDCP_AUTHENC 0x66800 ++#define _PORTB_HDCP_AUTHENC 0x66500 ++#define _PORTC_HDCP_AUTHENC 0x66600 ++#define _PORTD_HDCP_AUTHENC 0x66700 ++#define _PORTE_HDCP_AUTHENC 0x66A00 ++#define _PORTF_HDCP_AUTHENC 0x66900 ++#define _PORT_HDCP_AUTHENC(port, x) _MMIO(_PICK(port, \ ++ _PORTA_HDCP_AUTHENC, \ ++ _PORTB_HDCP_AUTHENC, \ ++ _PORTC_HDCP_AUTHENC, \ ++ _PORTD_HDCP_AUTHENC, \ ++ _PORTE_HDCP_AUTHENC, \ ++ _PORTF_HDCP_AUTHENC) + (x)) ++#define PORT_HDCP_CONF(port) _PORT_HDCP_AUTHENC(port, 0x0) ++#define HDCP_CONF_CAPTURE_AN BIT(0) ++#define HDCP_CONF_AUTH_AND_ENC (BIT(1) | BIT(0)) ++#define PORT_HDCP_ANINIT(port) _PORT_HDCP_AUTHENC(port, 0x4) ++#define PORT_HDCP_ANLO(port) _PORT_HDCP_AUTHENC(port, 0x8) ++#define PORT_HDCP_ANHI(port) _PORT_HDCP_AUTHENC(port, 0xC) ++#define PORT_HDCP_BKSVLO(port) _PORT_HDCP_AUTHENC(port, 0x10) ++#define PORT_HDCP_BKSVHI(port) _PORT_HDCP_AUTHENC(port, 0x14) ++#define PORT_HDCP_RPRIME(port) _PORT_HDCP_AUTHENC(port, 0x18) ++#define PORT_HDCP_STATUS(port) _PORT_HDCP_AUTHENC(port, 0x1C) ++#define HDCP_STATUS_STREAM_A_ENC BIT(31) ++#define HDCP_STATUS_STREAM_B_ENC BIT(30) ++#define HDCP_STATUS_STREAM_C_ENC BIT(29) ++#define HDCP_STATUS_STREAM_D_ENC BIT(28) ++#define HDCP_STATUS_AUTH BIT(21) ++#define HDCP_STATUS_ENC BIT(20) ++#define HDCP_STATUS_RI_MATCH BIT(19) ++#define HDCP_STATUS_R0_READY BIT(18) ++#define HDCP_STATUS_AN_READY BIT(17) ++#define HDCP_STATUS_CIPHER BIT(16) ++#define HDCP_STATUS_FRAME_CNT(x) (((x) >> 8) & 0xff) ++ ++/* HDCP2.2 Registers */ ++#define _PORTA_HDCP2_BASE 0x66800 ++#define _PORTB_HDCP2_BASE 0x66500 ++#define _PORTC_HDCP2_BASE 0x66600 ++#define _PORTD_HDCP2_BASE 0x66700 ++#define _PORTE_HDCP2_BASE 0x66A00 ++#define _PORTF_HDCP2_BASE 0x66900 ++#define _PORT_HDCP2_BASE(port, x) _MMIO(_PICK((port), \ ++ _PORTA_HDCP2_BASE, \ ++ _PORTB_HDCP2_BASE, \ ++ _PORTC_HDCP2_BASE, \ ++ _PORTD_HDCP2_BASE, \ ++ _PORTE_HDCP2_BASE, \ ++ _PORTF_HDCP2_BASE) + (x)) ++ ++#define HDCP2_AUTH_DDI(port) _PORT_HDCP2_BASE(port, 0x98) ++#define AUTH_LINK_AUTHENTICATED BIT(31) ++#define AUTH_LINK_TYPE BIT(30) ++#define AUTH_FORCE_CLR_INPUTCTR BIT(19) ++#define AUTH_CLR_KEYS BIT(18) ++ ++#define HDCP2_CTL_DDI(port) _PORT_HDCP2_BASE(port, 0xB0) ++#define CTL_LINK_ENCRYPTION_REQ BIT(31) ++ ++#define HDCP2_STATUS_DDI(port) _PORT_HDCP2_BASE(port, 0xB4) ++#define STREAM_ENCRYPTION_STATUS_A BIT(31) ++#define STREAM_ENCRYPTION_STATUS_B BIT(30) ++#define STREAM_ENCRYPTION_STATUS_C BIT(29) ++#define LINK_TYPE_STATUS BIT(22) ++#define LINK_AUTH_STATUS BIT(21) ++#define LINK_ENCRYPTION_STATUS BIT(20) ++ ++/* Per-pipe DDI Function Control */ ++#define _TRANS_DDI_FUNC_CTL_A 0x60400 ++#define _TRANS_DDI_FUNC_CTL_B 0x61400 ++#define _TRANS_DDI_FUNC_CTL_C 0x62400 ++#define _TRANS_DDI_FUNC_CTL_EDP 0x6F400 ++#define _TRANS_DDI_FUNC_CTL_DSI0 0x6b400 ++#define _TRANS_DDI_FUNC_CTL_DSI1 0x6bc00 ++#define TRANS_DDI_FUNC_CTL(tran) _MMIO_TRANS2(tran, _TRANS_DDI_FUNC_CTL_A) ++ ++#define TRANS_DDI_FUNC_ENABLE (1 << 31) ++/* Those bits are ignored by pipe EDP since it can only connect to DDI A */ ++#define TRANS_DDI_PORT_MASK (7 << 28) ++#define TRANS_DDI_PORT_SHIFT 28 ++#define TRANS_DDI_SELECT_PORT(x) ((x) << 28) ++#define TRANS_DDI_PORT_NONE (0 << 28) ++#define TRANS_DDI_MODE_SELECT_MASK (7 << 24) ++#define TRANS_DDI_MODE_SELECT_HDMI (0 << 24) ++#define TRANS_DDI_MODE_SELECT_DVI (1 << 24) ++#define TRANS_DDI_MODE_SELECT_DP_SST (2 << 24) ++#define TRANS_DDI_MODE_SELECT_DP_MST (3 << 24) ++#define TRANS_DDI_MODE_SELECT_FDI (4 << 24) ++#define TRANS_DDI_BPC_MASK (7 << 20) ++#define TRANS_DDI_BPC_8 (0 << 20) ++#define TRANS_DDI_BPC_10 (1 << 20) ++#define TRANS_DDI_BPC_6 (2 << 20) ++#define TRANS_DDI_BPC_12 (3 << 20) ++#define TRANS_DDI_PVSYNC (1 << 17) ++#define TRANS_DDI_PHSYNC (1 << 16) ++#define TRANS_DDI_EDP_INPUT_MASK (7 << 12) ++#define TRANS_DDI_EDP_INPUT_A_ON (0 << 12) ++#define TRANS_DDI_EDP_INPUT_A_ONOFF (4 << 12) ++#define TRANS_DDI_EDP_INPUT_B_ONOFF (5 << 12) ++#define TRANS_DDI_EDP_INPUT_C_ONOFF (6 << 12) ++#define TRANS_DDI_HDCP_SIGNALLING (1 << 9) ++#define TRANS_DDI_DP_VC_PAYLOAD_ALLOC (1 << 8) ++#define TRANS_DDI_HDMI_SCRAMBLER_CTS_ENABLE (1 << 7) ++#define TRANS_DDI_HDMI_SCRAMBLER_RESET_FREQ (1 << 6) ++#define TRANS_DDI_BFI_ENABLE (1 << 4) ++#define TRANS_DDI_HIGH_TMDS_CHAR_RATE (1 << 4) ++#define TRANS_DDI_HDMI_SCRAMBLING (1 << 0) ++#define TRANS_DDI_HDMI_SCRAMBLING_MASK (TRANS_DDI_HDMI_SCRAMBLER_CTS_ENABLE \ ++ | TRANS_DDI_HDMI_SCRAMBLER_RESET_FREQ \ ++ | TRANS_DDI_HDMI_SCRAMBLING) ++ ++#define _TRANS_DDI_FUNC_CTL2_A 0x60404 ++#define _TRANS_DDI_FUNC_CTL2_B 0x61404 ++#define _TRANS_DDI_FUNC_CTL2_C 0x62404 ++#define _TRANS_DDI_FUNC_CTL2_EDP 0x6f404 ++#define _TRANS_DDI_FUNC_CTL2_DSI0 0x6b404 ++#define _TRANS_DDI_FUNC_CTL2_DSI1 0x6bc04 ++#define TRANS_DDI_FUNC_CTL2(tran) _MMIO_TRANS2(tran, \ ++ _TRANS_DDI_FUNC_CTL2_A) ++#define PORT_SYNC_MODE_ENABLE (1 << 4) ++#define PORT_SYNC_MODE_MASTER_SELECT(x) ((x) << 0) ++#define PORT_SYNC_MODE_MASTER_SELECT_MASK (0x7 << 0) ++#define PORT_SYNC_MODE_MASTER_SELECT_SHIFT 0 ++ ++/* DisplayPort Transport Control */ ++#define _DP_TP_CTL_A 0x64040 ++#define _DP_TP_CTL_B 0x64140 ++#define DP_TP_CTL(port) _MMIO_PORT(port, _DP_TP_CTL_A, _DP_TP_CTL_B) ++#define DP_TP_CTL_ENABLE (1 << 31) ++#define DP_TP_CTL_FEC_ENABLE (1 << 30) ++#define DP_TP_CTL_MODE_SST (0 << 27) ++#define DP_TP_CTL_MODE_MST (1 << 27) ++#define DP_TP_CTL_FORCE_ACT (1 << 25) ++#define DP_TP_CTL_ENHANCED_FRAME_ENABLE (1 << 18) ++#define DP_TP_CTL_FDI_AUTOTRAIN (1 << 15) ++#define DP_TP_CTL_LINK_TRAIN_MASK (7 << 8) ++#define DP_TP_CTL_LINK_TRAIN_PAT1 (0 << 8) ++#define DP_TP_CTL_LINK_TRAIN_PAT2 (1 << 8) ++#define DP_TP_CTL_LINK_TRAIN_PAT3 (4 << 8) ++#define DP_TP_CTL_LINK_TRAIN_PAT4 (5 << 8) ++#define DP_TP_CTL_LINK_TRAIN_IDLE (2 << 8) ++#define DP_TP_CTL_LINK_TRAIN_NORMAL (3 << 8) ++#define DP_TP_CTL_SCRAMBLE_DISABLE (1 << 7) ++ ++/* DisplayPort Transport Status */ ++#define _DP_TP_STATUS_A 0x64044 ++#define _DP_TP_STATUS_B 0x64144 ++#define DP_TP_STATUS(port) _MMIO_PORT(port, _DP_TP_STATUS_A, _DP_TP_STATUS_B) ++#define DP_TP_STATUS_FEC_ENABLE_LIVE (1 << 28) ++#define DP_TP_STATUS_IDLE_DONE (1 << 25) ++#define DP_TP_STATUS_ACT_SENT (1 << 24) ++#define DP_TP_STATUS_MODE_STATUS_MST (1 << 23) ++#define DP_TP_STATUS_AUTOTRAIN_DONE (1 << 12) ++#define DP_TP_STATUS_PAYLOAD_MAPPING_VC2 (3 << 8) ++#define DP_TP_STATUS_PAYLOAD_MAPPING_VC1 (3 << 4) ++#define DP_TP_STATUS_PAYLOAD_MAPPING_VC0 (3 << 0) ++ ++/* DDI Buffer Control */ ++#define _DDI_BUF_CTL_A 0x64000 ++#define _DDI_BUF_CTL_B 0x64100 ++#define DDI_BUF_CTL(port) _MMIO_PORT(port, _DDI_BUF_CTL_A, _DDI_BUF_CTL_B) ++#define DDI_BUF_CTL_ENABLE (1 << 31) ++#define DDI_BUF_TRANS_SELECT(n) ((n) << 24) ++#define DDI_BUF_EMP_MASK (0xf << 24) ++#define DDI_BUF_PORT_REVERSAL (1 << 16) ++#define DDI_BUF_IS_IDLE (1 << 7) ++#define DDI_A_4_LANES (1 << 4) ++#define DDI_PORT_WIDTH(width) (((width) - 1) << 1) ++#define DDI_PORT_WIDTH_MASK (7 << 1) ++#define DDI_PORT_WIDTH_SHIFT 1 ++#define DDI_INIT_DISPLAY_DETECTED (1 << 0) ++ ++/* DDI Buffer Translations */ ++#define _DDI_BUF_TRANS_A 0x64E00 ++#define _DDI_BUF_TRANS_B 0x64E60 ++#define DDI_BUF_TRANS_LO(port, i) _MMIO(_PORT(port, _DDI_BUF_TRANS_A, _DDI_BUF_TRANS_B) + (i) * 8) ++#define DDI_BUF_BALANCE_LEG_ENABLE (1 << 31) ++#define DDI_BUF_TRANS_HI(port, i) _MMIO(_PORT(port, _DDI_BUF_TRANS_A, _DDI_BUF_TRANS_B) + (i) * 8 + 4) ++ ++/* Sideband Interface (SBI) is programmed indirectly, via ++ * SBI_ADDR, which contains the register offset; and SBI_DATA, ++ * which contains the payload */ ++#define SBI_ADDR _MMIO(0xC6000) ++#define SBI_DATA _MMIO(0xC6004) ++#define SBI_CTL_STAT _MMIO(0xC6008) ++#define SBI_CTL_DEST_ICLK (0x0 << 16) ++#define SBI_CTL_DEST_MPHY (0x1 << 16) ++#define SBI_CTL_OP_IORD (0x2 << 8) ++#define SBI_CTL_OP_IOWR (0x3 << 8) ++#define SBI_CTL_OP_CRRD (0x6 << 8) ++#define SBI_CTL_OP_CRWR (0x7 << 8) ++#define SBI_RESPONSE_FAIL (0x1 << 1) ++#define SBI_RESPONSE_SUCCESS (0x0 << 1) ++#define SBI_BUSY (0x1 << 0) ++#define SBI_READY (0x0 << 0) ++ ++/* SBI offsets */ ++#define SBI_SSCDIVINTPHASE 0x0200 ++#define SBI_SSCDIVINTPHASE6 0x0600 ++#define SBI_SSCDIVINTPHASE_DIVSEL_SHIFT 1 ++#define SBI_SSCDIVINTPHASE_DIVSEL_MASK (0x7f << 1) ++#define SBI_SSCDIVINTPHASE_DIVSEL(x) ((x) << 1) ++#define SBI_SSCDIVINTPHASE_INCVAL_SHIFT 8 ++#define SBI_SSCDIVINTPHASE_INCVAL_MASK (0x7f << 8) ++#define SBI_SSCDIVINTPHASE_INCVAL(x) ((x) << 8) ++#define SBI_SSCDIVINTPHASE_DIR(x) ((x) << 15) ++#define SBI_SSCDIVINTPHASE_PROPAGATE (1 << 0) ++#define SBI_SSCDITHPHASE 0x0204 ++#define SBI_SSCCTL 0x020c ++#define SBI_SSCCTL6 0x060C ++#define SBI_SSCCTL_PATHALT (1 << 3) ++#define SBI_SSCCTL_DISABLE (1 << 0) ++#define SBI_SSCAUXDIV6 0x0610 ++#define SBI_SSCAUXDIV_FINALDIV2SEL_SHIFT 4 ++#define SBI_SSCAUXDIV_FINALDIV2SEL_MASK (1 << 4) ++#define SBI_SSCAUXDIV_FINALDIV2SEL(x) ((x) << 4) ++#define SBI_DBUFF0 0x2a00 ++#define SBI_GEN0 0x1f00 ++#define SBI_GEN0_CFG_BUFFENABLE_DISABLE (1 << 0) ++ ++/* LPT PIXCLK_GATE */ ++#define PIXCLK_GATE _MMIO(0xC6020) ++#define PIXCLK_GATE_UNGATE (1 << 0) ++#define PIXCLK_GATE_GATE (0 << 0) ++ ++/* SPLL */ ++#define SPLL_CTL _MMIO(0x46020) ++#define SPLL_PLL_ENABLE (1 << 31) ++#define SPLL_PLL_SSC (1 << 28) ++#define SPLL_PLL_NON_SSC (2 << 28) ++#define SPLL_PLL_LCPLL (3 << 28) ++#define SPLL_PLL_REF_MASK (3 << 28) ++#define SPLL_PLL_FREQ_810MHz (0 << 26) ++#define SPLL_PLL_FREQ_1350MHz (1 << 26) ++#define SPLL_PLL_FREQ_2700MHz (2 << 26) ++#define SPLL_PLL_FREQ_MASK (3 << 26) ++ ++/* WRPLL */ ++#define _WRPLL_CTL1 0x46040 ++#define _WRPLL_CTL2 0x46060 ++#define WRPLL_CTL(pll) _MMIO_PIPE(pll, _WRPLL_CTL1, _WRPLL_CTL2) ++#define WRPLL_PLL_ENABLE (1 << 31) ++#define WRPLL_PLL_SSC (1 << 28) ++#define WRPLL_PLL_NON_SSC (2 << 28) ++#define WRPLL_PLL_LCPLL (3 << 28) ++#define WRPLL_PLL_REF_MASK (3 << 28) ++/* WRPLL divider programming */ ++#define WRPLL_DIVIDER_REFERENCE(x) ((x) << 0) ++#define WRPLL_DIVIDER_REF_MASK (0xff) ++#define WRPLL_DIVIDER_POST(x) ((x) << 8) ++#define WRPLL_DIVIDER_POST_MASK (0x3f << 8) ++#define WRPLL_DIVIDER_POST_SHIFT 8 ++#define WRPLL_DIVIDER_FEEDBACK(x) ((x) << 16) ++#define WRPLL_DIVIDER_FB_SHIFT 16 ++#define WRPLL_DIVIDER_FB_MASK (0xff << 16) ++ ++/* Port clock selection */ ++#define _PORT_CLK_SEL_A 0x46100 ++#define _PORT_CLK_SEL_B 0x46104 ++#define PORT_CLK_SEL(port) _MMIO_PORT(port, _PORT_CLK_SEL_A, _PORT_CLK_SEL_B) ++#define PORT_CLK_SEL_LCPLL_2700 (0 << 29) ++#define PORT_CLK_SEL_LCPLL_1350 (1 << 29) ++#define PORT_CLK_SEL_LCPLL_810 (2 << 29) ++#define PORT_CLK_SEL_SPLL (3 << 29) ++#define PORT_CLK_SEL_WRPLL(pll) (((pll) + 4) << 29) ++#define PORT_CLK_SEL_WRPLL1 (4 << 29) ++#define PORT_CLK_SEL_WRPLL2 (5 << 29) ++#define PORT_CLK_SEL_NONE (7 << 29) ++#define PORT_CLK_SEL_MASK (7 << 29) ++ ++/* On ICL+ this is the same as PORT_CLK_SEL, but all bits change. */ ++#define DDI_CLK_SEL(port) PORT_CLK_SEL(port) ++#define DDI_CLK_SEL_NONE (0x0 << 28) ++#define DDI_CLK_SEL_MG (0x8 << 28) ++#define DDI_CLK_SEL_TBT_162 (0xC << 28) ++#define DDI_CLK_SEL_TBT_270 (0xD << 28) ++#define DDI_CLK_SEL_TBT_540 (0xE << 28) ++#define DDI_CLK_SEL_TBT_810 (0xF << 28) ++#define DDI_CLK_SEL_MASK (0xF << 28) ++ ++/* Transcoder clock selection */ ++#define _TRANS_CLK_SEL_A 0x46140 ++#define _TRANS_CLK_SEL_B 0x46144 ++#define TRANS_CLK_SEL(tran) _MMIO_TRANS(tran, _TRANS_CLK_SEL_A, _TRANS_CLK_SEL_B) ++/* For each transcoder, we need to select the corresponding port clock */ ++#define TRANS_CLK_SEL_DISABLED (0x0 << 29) ++#define TRANS_CLK_SEL_PORT(x) (((x) + 1) << 29) ++ ++#define CDCLK_FREQ _MMIO(0x46200) ++ ++#define _TRANSA_MSA_MISC 0x60410 ++#define _TRANSB_MSA_MISC 0x61410 ++#define _TRANSC_MSA_MISC 0x62410 ++#define _TRANS_EDP_MSA_MISC 0x6f410 ++#define TRANS_MSA_MISC(tran) _MMIO_TRANS2(tran, _TRANSA_MSA_MISC) ++ ++#define TRANS_MSA_SYNC_CLK (1 << 0) ++#define TRANS_MSA_SAMPLING_444 (2 << 1) ++#define TRANS_MSA_CLRSP_YCBCR (2 << 3) ++#define TRANS_MSA_6_BPC (0 << 5) ++#define TRANS_MSA_8_BPC (1 << 5) ++#define TRANS_MSA_10_BPC (2 << 5) ++#define TRANS_MSA_12_BPC (3 << 5) ++#define TRANS_MSA_16_BPC (4 << 5) ++#define TRANS_MSA_CEA_RANGE (1 << 3) ++ ++/* LCPLL Control */ ++#define LCPLL_CTL _MMIO(0x130040) ++#define LCPLL_PLL_DISABLE (1 << 31) ++#define LCPLL_PLL_LOCK (1 << 30) ++#define LCPLL_CLK_FREQ_MASK (3 << 26) ++#define LCPLL_CLK_FREQ_450 (0 << 26) ++#define LCPLL_CLK_FREQ_54O_BDW (1 << 26) ++#define LCPLL_CLK_FREQ_337_5_BDW (2 << 26) ++#define LCPLL_CLK_FREQ_675_BDW (3 << 26) ++#define LCPLL_CD_CLOCK_DISABLE (1 << 25) ++#define LCPLL_ROOT_CD_CLOCK_DISABLE (1 << 24) ++#define LCPLL_CD2X_CLOCK_DISABLE (1 << 23) ++#define LCPLL_POWER_DOWN_ALLOW (1 << 22) ++#define LCPLL_CD_SOURCE_FCLK (1 << 21) ++#define LCPLL_CD_SOURCE_FCLK_DONE (1 << 19) ++ ++/* ++ * SKL Clocks ++ */ ++ ++/* CDCLK_CTL */ ++#define CDCLK_CTL _MMIO(0x46000) ++#define CDCLK_FREQ_SEL_MASK (3 << 26) ++#define CDCLK_FREQ_450_432 (0 << 26) ++#define CDCLK_FREQ_540 (1 << 26) ++#define CDCLK_FREQ_337_308 (2 << 26) ++#define CDCLK_FREQ_675_617 (3 << 26) ++#define BXT_CDCLK_CD2X_DIV_SEL_MASK (3 << 22) ++#define BXT_CDCLK_CD2X_DIV_SEL_1 (0 << 22) ++#define BXT_CDCLK_CD2X_DIV_SEL_1_5 (1 << 22) ++#define BXT_CDCLK_CD2X_DIV_SEL_2 (2 << 22) ++#define BXT_CDCLK_CD2X_DIV_SEL_4 (3 << 22) ++#define BXT_CDCLK_CD2X_PIPE(pipe) ((pipe) << 20) ++#define CDCLK_DIVMUX_CD_OVERRIDE (1 << 19) ++#define BXT_CDCLK_CD2X_PIPE_NONE BXT_CDCLK_CD2X_PIPE(3) ++#define ICL_CDCLK_CD2X_PIPE_NONE (7 << 19) ++#define BXT_CDCLK_SSA_PRECHARGE_ENABLE (1 << 16) ++#define CDCLK_FREQ_DECIMAL_MASK (0x7ff) ++ ++/* LCPLL_CTL */ ++#define LCPLL1_CTL _MMIO(0x46010) ++#define LCPLL2_CTL _MMIO(0x46014) ++#define LCPLL_PLL_ENABLE (1 << 31) ++ ++/* DPLL control1 */ ++#define DPLL_CTRL1 _MMIO(0x6C058) ++#define DPLL_CTRL1_HDMI_MODE(id) (1 << ((id) * 6 + 5)) ++#define DPLL_CTRL1_SSC(id) (1 << ((id) * 6 + 4)) ++#define DPLL_CTRL1_LINK_RATE_MASK(id) (7 << ((id) * 6 + 1)) ++#define DPLL_CTRL1_LINK_RATE_SHIFT(id) ((id) * 6 + 1) ++#define DPLL_CTRL1_LINK_RATE(linkrate, id) ((linkrate) << ((id) * 6 + 1)) ++#define DPLL_CTRL1_OVERRIDE(id) (1 << ((id) * 6)) ++#define DPLL_CTRL1_LINK_RATE_2700 0 ++#define DPLL_CTRL1_LINK_RATE_1350 1 ++#define DPLL_CTRL1_LINK_RATE_810 2 ++#define DPLL_CTRL1_LINK_RATE_1620 3 ++#define DPLL_CTRL1_LINK_RATE_1080 4 ++#define DPLL_CTRL1_LINK_RATE_2160 5 ++ ++/* DPLL control2 */ ++#define DPLL_CTRL2 _MMIO(0x6C05C) ++#define DPLL_CTRL2_DDI_CLK_OFF(port) (1 << ((port) + 15)) ++#define DPLL_CTRL2_DDI_CLK_SEL_MASK(port) (3 << ((port) * 3 + 1)) ++#define DPLL_CTRL2_DDI_CLK_SEL_SHIFT(port) ((port) * 3 + 1) ++#define DPLL_CTRL2_DDI_CLK_SEL(clk, port) ((clk) << ((port) * 3 + 1)) ++#define DPLL_CTRL2_DDI_SEL_OVERRIDE(port) (1 << ((port) * 3)) ++ ++/* DPLL Status */ ++#define DPLL_STATUS _MMIO(0x6C060) ++#define DPLL_LOCK(id) (1 << ((id) * 8)) ++ ++/* DPLL cfg */ ++#define _DPLL1_CFGCR1 0x6C040 ++#define _DPLL2_CFGCR1 0x6C048 ++#define _DPLL3_CFGCR1 0x6C050 ++#define DPLL_CFGCR1_FREQ_ENABLE (1 << 31) ++#define DPLL_CFGCR1_DCO_FRACTION_MASK (0x7fff << 9) ++#define DPLL_CFGCR1_DCO_FRACTION(x) ((x) << 9) ++#define DPLL_CFGCR1_DCO_INTEGER_MASK (0x1ff) ++ ++#define _DPLL1_CFGCR2 0x6C044 ++#define _DPLL2_CFGCR2 0x6C04C ++#define _DPLL3_CFGCR2 0x6C054 ++#define DPLL_CFGCR2_QDIV_RATIO_MASK (0xff << 8) ++#define DPLL_CFGCR2_QDIV_RATIO(x) ((x) << 8) ++#define DPLL_CFGCR2_QDIV_MODE(x) ((x) << 7) ++#define DPLL_CFGCR2_KDIV_MASK (3 << 5) ++#define DPLL_CFGCR2_KDIV(x) ((x) << 5) ++#define DPLL_CFGCR2_KDIV_5 (0 << 5) ++#define DPLL_CFGCR2_KDIV_2 (1 << 5) ++#define DPLL_CFGCR2_KDIV_3 (2 << 5) ++#define DPLL_CFGCR2_KDIV_1 (3 << 5) ++#define DPLL_CFGCR2_PDIV_MASK (7 << 2) ++#define DPLL_CFGCR2_PDIV(x) ((x) << 2) ++#define DPLL_CFGCR2_PDIV_1 (0 << 2) ++#define DPLL_CFGCR2_PDIV_2 (1 << 2) ++#define DPLL_CFGCR2_PDIV_3 (2 << 2) ++#define DPLL_CFGCR2_PDIV_7 (4 << 2) ++#define DPLL_CFGCR2_CENTRAL_FREQ_MASK (3) ++ ++#define DPLL_CFGCR1(id) _MMIO_PIPE((id) - SKL_DPLL1, _DPLL1_CFGCR1, _DPLL2_CFGCR1) ++#define DPLL_CFGCR2(id) _MMIO_PIPE((id) - SKL_DPLL1, _DPLL1_CFGCR2, _DPLL2_CFGCR2) ++ ++/* ++ * CNL Clocks ++ */ ++#define DPCLKA_CFGCR0 _MMIO(0x6C200) ++#define DPCLKA_CFGCR0_ICL _MMIO(0x164280) ++#define DPCLKA_CFGCR0_DDI_CLK_OFF(port) (1 << ((port) == PORT_F ? 23 : \ ++ (port) + 10)) ++#define ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(port) (1 << ((port) + 10)) ++#define ICL_DPCLKA_CFGCR0_TC_CLK_OFF(tc_port) (1 << ((tc_port) == PORT_TC4 ? \ ++ 21 : (tc_port) + 12)) ++#define DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port) ((port) == PORT_F ? 21 : \ ++ (port) * 2) ++#define DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port) (3 << DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port)) ++#define DPCLKA_CFGCR0_DDI_CLK_SEL(pll, port) ((pll) << DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port)) ++ ++/* CNL PLL */ ++#define DPLL0_ENABLE 0x46010 ++#define DPLL1_ENABLE 0x46014 ++#define PLL_ENABLE (1 << 31) ++#define PLL_LOCK (1 << 30) ++#define PLL_POWER_ENABLE (1 << 27) ++#define PLL_POWER_STATE (1 << 26) ++#define CNL_DPLL_ENABLE(pll) _MMIO_PLL(pll, DPLL0_ENABLE, DPLL1_ENABLE) ++ ++#define TBT_PLL_ENABLE _MMIO(0x46020) ++ ++#define _MG_PLL1_ENABLE 0x46030 ++#define _MG_PLL2_ENABLE 0x46034 ++#define _MG_PLL3_ENABLE 0x46038 ++#define _MG_PLL4_ENABLE 0x4603C ++/* Bits are the same as DPLL0_ENABLE */ ++#define MG_PLL_ENABLE(tc_port) _MMIO_PORT((tc_port), _MG_PLL1_ENABLE, \ ++ _MG_PLL2_ENABLE) ++ ++#define _MG_REFCLKIN_CTL_PORT1 0x16892C ++#define _MG_REFCLKIN_CTL_PORT2 0x16992C ++#define _MG_REFCLKIN_CTL_PORT3 0x16A92C ++#define _MG_REFCLKIN_CTL_PORT4 0x16B92C ++#define MG_REFCLKIN_CTL_OD_2_MUX(x) ((x) << 8) ++#define MG_REFCLKIN_CTL_OD_2_MUX_MASK (0x7 << 8) ++#define MG_REFCLKIN_CTL(tc_port) _MMIO_PORT((tc_port), \ ++ _MG_REFCLKIN_CTL_PORT1, \ ++ _MG_REFCLKIN_CTL_PORT2) ++ ++#define _MG_CLKTOP2_CORECLKCTL1_PORT1 0x1688D8 ++#define _MG_CLKTOP2_CORECLKCTL1_PORT2 0x1698D8 ++#define _MG_CLKTOP2_CORECLKCTL1_PORT3 0x16A8D8 ++#define _MG_CLKTOP2_CORECLKCTL1_PORT4 0x16B8D8 ++#define MG_CLKTOP2_CORECLKCTL1_B_DIVRATIO(x) ((x) << 16) ++#define MG_CLKTOP2_CORECLKCTL1_B_DIVRATIO_MASK (0xff << 16) ++#define MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO(x) ((x) << 8) ++#define MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO_MASK (0xff << 8) ++#define MG_CLKTOP2_CORECLKCTL1(tc_port) _MMIO_PORT((tc_port), \ ++ _MG_CLKTOP2_CORECLKCTL1_PORT1, \ ++ _MG_CLKTOP2_CORECLKCTL1_PORT2) ++ ++#define _MG_CLKTOP2_HSCLKCTL_PORT1 0x1688D4 ++#define _MG_CLKTOP2_HSCLKCTL_PORT2 0x1698D4 ++#define _MG_CLKTOP2_HSCLKCTL_PORT3 0x16A8D4 ++#define _MG_CLKTOP2_HSCLKCTL_PORT4 0x16B8D4 ++#define MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL(x) ((x) << 16) ++#define MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL_MASK (0x1 << 16) ++#define MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL(x) ((x) << 14) ++#define MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL_MASK (0x3 << 14) ++#define MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_MASK (0x3 << 12) ++#define MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_2 (0 << 12) ++#define MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_3 (1 << 12) ++#define MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_5 (2 << 12) ++#define MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_7 (3 << 12) ++#define MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO(x) ((x) << 8) ++#define MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_SHIFT 8 ++#define MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK (0xf << 8) ++#define MG_CLKTOP2_HSCLKCTL(tc_port) _MMIO_PORT((tc_port), \ ++ _MG_CLKTOP2_HSCLKCTL_PORT1, \ ++ _MG_CLKTOP2_HSCLKCTL_PORT2) ++ ++#define _MG_PLL_DIV0_PORT1 0x168A00 ++#define _MG_PLL_DIV0_PORT2 0x169A00 ++#define _MG_PLL_DIV0_PORT3 0x16AA00 ++#define _MG_PLL_DIV0_PORT4 0x16BA00 ++#define MG_PLL_DIV0_FRACNEN_H (1 << 30) ++#define MG_PLL_DIV0_FBDIV_FRAC_MASK (0x3fffff << 8) ++#define MG_PLL_DIV0_FBDIV_FRAC_SHIFT 8 ++#define MG_PLL_DIV0_FBDIV_FRAC(x) ((x) << 8) ++#define MG_PLL_DIV0_FBDIV_INT_MASK (0xff << 0) ++#define MG_PLL_DIV0_FBDIV_INT(x) ((x) << 0) ++#define MG_PLL_DIV0(tc_port) _MMIO_PORT((tc_port), _MG_PLL_DIV0_PORT1, \ ++ _MG_PLL_DIV0_PORT2) ++ ++#define _MG_PLL_DIV1_PORT1 0x168A04 ++#define _MG_PLL_DIV1_PORT2 0x169A04 ++#define _MG_PLL_DIV1_PORT3 0x16AA04 ++#define _MG_PLL_DIV1_PORT4 0x16BA04 ++#define MG_PLL_DIV1_IREF_NDIVRATIO(x) ((x) << 16) ++#define MG_PLL_DIV1_DITHER_DIV_1 (0 << 12) ++#define MG_PLL_DIV1_DITHER_DIV_2 (1 << 12) ++#define MG_PLL_DIV1_DITHER_DIV_4 (2 << 12) ++#define MG_PLL_DIV1_DITHER_DIV_8 (3 << 12) ++#define MG_PLL_DIV1_NDIVRATIO(x) ((x) << 4) ++#define MG_PLL_DIV1_FBPREDIV_MASK (0xf << 0) ++#define MG_PLL_DIV1_FBPREDIV(x) ((x) << 0) ++#define MG_PLL_DIV1(tc_port) _MMIO_PORT((tc_port), _MG_PLL_DIV1_PORT1, \ ++ _MG_PLL_DIV1_PORT2) ++ ++#define _MG_PLL_LF_PORT1 0x168A08 ++#define _MG_PLL_LF_PORT2 0x169A08 ++#define _MG_PLL_LF_PORT3 0x16AA08 ++#define _MG_PLL_LF_PORT4 0x16BA08 ++#define MG_PLL_LF_TDCTARGETCNT(x) ((x) << 24) ++#define MG_PLL_LF_AFCCNTSEL_256 (0 << 20) ++#define MG_PLL_LF_AFCCNTSEL_512 (1 << 20) ++#define MG_PLL_LF_GAINCTRL(x) ((x) << 16) ++#define MG_PLL_LF_INT_COEFF(x) ((x) << 8) ++#define MG_PLL_LF_PROP_COEFF(x) ((x) << 0) ++#define MG_PLL_LF(tc_port) _MMIO_PORT((tc_port), _MG_PLL_LF_PORT1, \ ++ _MG_PLL_LF_PORT2) ++ ++#define _MG_PLL_FRAC_LOCK_PORT1 0x168A0C ++#define _MG_PLL_FRAC_LOCK_PORT2 0x169A0C ++#define _MG_PLL_FRAC_LOCK_PORT3 0x16AA0C ++#define _MG_PLL_FRAC_LOCK_PORT4 0x16BA0C ++#define MG_PLL_FRAC_LOCK_TRUELOCK_CRIT_32 (1 << 18) ++#define MG_PLL_FRAC_LOCK_EARLYLOCK_CRIT_32 (1 << 16) ++#define MG_PLL_FRAC_LOCK_LOCKTHRESH(x) ((x) << 11) ++#define MG_PLL_FRAC_LOCK_DCODITHEREN (1 << 10) ++#define MG_PLL_FRAC_LOCK_FEEDFWRDCAL_EN (1 << 8) ++#define MG_PLL_FRAC_LOCK_FEEDFWRDGAIN(x) ((x) << 0) ++#define MG_PLL_FRAC_LOCK(tc_port) _MMIO_PORT((tc_port), \ ++ _MG_PLL_FRAC_LOCK_PORT1, \ ++ _MG_PLL_FRAC_LOCK_PORT2) ++ ++#define _MG_PLL_SSC_PORT1 0x168A10 ++#define _MG_PLL_SSC_PORT2 0x169A10 ++#define _MG_PLL_SSC_PORT3 0x16AA10 ++#define _MG_PLL_SSC_PORT4 0x16BA10 ++#define MG_PLL_SSC_EN (1 << 28) ++#define MG_PLL_SSC_TYPE(x) ((x) << 26) ++#define MG_PLL_SSC_STEPLENGTH(x) ((x) << 16) ++#define MG_PLL_SSC_STEPNUM(x) ((x) << 10) ++#define MG_PLL_SSC_FLLEN (1 << 9) ++#define MG_PLL_SSC_STEPSIZE(x) ((x) << 0) ++#define MG_PLL_SSC(tc_port) _MMIO_PORT((tc_port), _MG_PLL_SSC_PORT1, \ ++ _MG_PLL_SSC_PORT2) ++ ++#define _MG_PLL_BIAS_PORT1 0x168A14 ++#define _MG_PLL_BIAS_PORT2 0x169A14 ++#define _MG_PLL_BIAS_PORT3 0x16AA14 ++#define _MG_PLL_BIAS_PORT4 0x16BA14 ++#define MG_PLL_BIAS_BIAS_GB_SEL(x) ((x) << 30) ++#define MG_PLL_BIAS_BIAS_GB_SEL_MASK (0x3 << 30) ++#define MG_PLL_BIAS_INIT_DCOAMP(x) ((x) << 24) ++#define MG_PLL_BIAS_INIT_DCOAMP_MASK (0x3f << 24) ++#define MG_PLL_BIAS_BIAS_BONUS(x) ((x) << 16) ++#define MG_PLL_BIAS_BIAS_BONUS_MASK (0xff << 16) ++#define MG_PLL_BIAS_BIASCAL_EN (1 << 15) ++#define MG_PLL_BIAS_CTRIM(x) ((x) << 8) ++#define MG_PLL_BIAS_CTRIM_MASK (0x1f << 8) ++#define MG_PLL_BIAS_VREF_RDAC(x) ((x) << 5) ++#define MG_PLL_BIAS_VREF_RDAC_MASK (0x7 << 5) ++#define MG_PLL_BIAS_IREFTRIM(x) ((x) << 0) ++#define MG_PLL_BIAS_IREFTRIM_MASK (0x1f << 0) ++#define MG_PLL_BIAS(tc_port) _MMIO_PORT((tc_port), _MG_PLL_BIAS_PORT1, \ ++ _MG_PLL_BIAS_PORT2) ++ ++#define _MG_PLL_TDC_COLDST_BIAS_PORT1 0x168A18 ++#define _MG_PLL_TDC_COLDST_BIAS_PORT2 0x169A18 ++#define _MG_PLL_TDC_COLDST_BIAS_PORT3 0x16AA18 ++#define _MG_PLL_TDC_COLDST_BIAS_PORT4 0x16BA18 ++#define MG_PLL_TDC_COLDST_IREFINT_EN (1 << 27) ++#define MG_PLL_TDC_COLDST_REFBIAS_START_PULSE_W(x) ((x) << 17) ++#define MG_PLL_TDC_COLDST_COLDSTART (1 << 16) ++#define MG_PLL_TDC_TDCOVCCORR_EN (1 << 2) ++#define MG_PLL_TDC_TDCSEL(x) ((x) << 0) ++#define MG_PLL_TDC_COLDST_BIAS(tc_port) _MMIO_PORT((tc_port), \ ++ _MG_PLL_TDC_COLDST_BIAS_PORT1, \ ++ _MG_PLL_TDC_COLDST_BIAS_PORT2) ++ ++#define _CNL_DPLL0_CFGCR0 0x6C000 ++#define _CNL_DPLL1_CFGCR0 0x6C080 ++#define DPLL_CFGCR0_HDMI_MODE (1 << 30) ++#define DPLL_CFGCR0_SSC_ENABLE (1 << 29) ++#define DPLL_CFGCR0_SSC_ENABLE_ICL (1 << 25) ++#define DPLL_CFGCR0_LINK_RATE_MASK (0xf << 25) ++#define DPLL_CFGCR0_LINK_RATE_2700 (0 << 25) ++#define DPLL_CFGCR0_LINK_RATE_1350 (1 << 25) ++#define DPLL_CFGCR0_LINK_RATE_810 (2 << 25) ++#define DPLL_CFGCR0_LINK_RATE_1620 (3 << 25) ++#define DPLL_CFGCR0_LINK_RATE_1080 (4 << 25) ++#define DPLL_CFGCR0_LINK_RATE_2160 (5 << 25) ++#define DPLL_CFGCR0_LINK_RATE_3240 (6 << 25) ++#define DPLL_CFGCR0_LINK_RATE_4050 (7 << 25) ++#define DPLL_CFGCR0_DCO_FRACTION_MASK (0x7fff << 10) ++#define DPLL_CFGCR0_DCO_FRACTION_SHIFT (10) ++#define DPLL_CFGCR0_DCO_FRACTION(x) ((x) << 10) ++#define DPLL_CFGCR0_DCO_INTEGER_MASK (0x3ff) ++#define CNL_DPLL_CFGCR0(pll) _MMIO_PLL(pll, _CNL_DPLL0_CFGCR0, _CNL_DPLL1_CFGCR0) ++ ++#define _CNL_DPLL0_CFGCR1 0x6C004 ++#define _CNL_DPLL1_CFGCR1 0x6C084 ++#define DPLL_CFGCR1_QDIV_RATIO_MASK (0xff << 10) ++#define DPLL_CFGCR1_QDIV_RATIO_SHIFT (10) ++#define DPLL_CFGCR1_QDIV_RATIO(x) ((x) << 10) ++#define DPLL_CFGCR1_QDIV_MODE_SHIFT (9) ++#define DPLL_CFGCR1_QDIV_MODE(x) ((x) << 9) ++#define DPLL_CFGCR1_KDIV_MASK (7 << 6) ++#define DPLL_CFGCR1_KDIV_SHIFT (6) ++#define DPLL_CFGCR1_KDIV(x) ((x) << 6) ++#define DPLL_CFGCR1_KDIV_1 (1 << 6) ++#define DPLL_CFGCR1_KDIV_2 (2 << 6) ++#define DPLL_CFGCR1_KDIV_3 (4 << 6) ++#define DPLL_CFGCR1_PDIV_MASK (0xf << 2) ++#define DPLL_CFGCR1_PDIV_SHIFT (2) ++#define DPLL_CFGCR1_PDIV(x) ((x) << 2) ++#define DPLL_CFGCR1_PDIV_2 (1 << 2) ++#define DPLL_CFGCR1_PDIV_3 (2 << 2) ++#define DPLL_CFGCR1_PDIV_5 (4 << 2) ++#define DPLL_CFGCR1_PDIV_7 (8 << 2) ++#define DPLL_CFGCR1_CENTRAL_FREQ (3 << 0) ++#define DPLL_CFGCR1_CENTRAL_FREQ_8400 (3 << 0) ++#define CNL_DPLL_CFGCR1(pll) _MMIO_PLL(pll, _CNL_DPLL0_CFGCR1, _CNL_DPLL1_CFGCR1) ++ ++#define _ICL_DPLL0_CFGCR0 0x164000 ++#define _ICL_DPLL1_CFGCR0 0x164080 ++#define ICL_DPLL_CFGCR0(pll) _MMIO_PLL(pll, _ICL_DPLL0_CFGCR0, \ ++ _ICL_DPLL1_CFGCR0) ++ ++#define _ICL_DPLL0_CFGCR1 0x164004 ++#define _ICL_DPLL1_CFGCR1 0x164084 ++#define ICL_DPLL_CFGCR1(pll) _MMIO_PLL(pll, _ICL_DPLL0_CFGCR1, \ ++ _ICL_DPLL1_CFGCR1) ++ ++/* BXT display engine PLL */ ++#define BXT_DE_PLL_CTL _MMIO(0x6d000) ++#define BXT_DE_PLL_RATIO(x) (x) /* {60,65,100} * 19.2MHz */ ++#define BXT_DE_PLL_RATIO_MASK 0xff ++ ++#define BXT_DE_PLL_ENABLE _MMIO(0x46070) ++#define BXT_DE_PLL_PLL_ENABLE (1 << 31) ++#define BXT_DE_PLL_LOCK (1 << 30) ++#define CNL_CDCLK_PLL_RATIO(x) (x) ++#define CNL_CDCLK_PLL_RATIO_MASK 0xff ++ ++/* GEN9 DC */ ++#define DC_STATE_EN _MMIO(0x45504) ++#define DC_STATE_DISABLE 0 ++#define DC_STATE_EN_UPTO_DC5 (1 << 0) ++#define DC_STATE_EN_DC9 (1 << 3) ++#define DC_STATE_EN_UPTO_DC6 (2 << 0) ++#define DC_STATE_EN_UPTO_DC5_DC6_MASK 0x3 ++ ++#define DC_STATE_DEBUG _MMIO(0x45520) ++#define DC_STATE_DEBUG_MASK_CORES (1 << 0) ++#define DC_STATE_DEBUG_MASK_MEMORY_UP (1 << 1) ++ ++#define BXT_P_CR_MC_BIOS_REQ_0_0_0 _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x7114) ++#define BXT_REQ_DATA_MASK 0x3F ++#define BXT_DRAM_CHANNEL_ACTIVE_SHIFT 12 ++#define BXT_DRAM_CHANNEL_ACTIVE_MASK (0xF << 12) ++#define BXT_MEMORY_FREQ_MULTIPLIER_HZ 133333333 ++ ++#define BXT_D_CR_DRP0_DUNIT8 0x1000 ++#define BXT_D_CR_DRP0_DUNIT9 0x1200 ++#define BXT_D_CR_DRP0_DUNIT_START 8 ++#define BXT_D_CR_DRP0_DUNIT_END 11 ++#define BXT_D_CR_DRP0_DUNIT(x) _MMIO(MCHBAR_MIRROR_BASE_SNB + \ ++ _PICK_EVEN((x) - 8, BXT_D_CR_DRP0_DUNIT8,\ ++ BXT_D_CR_DRP0_DUNIT9)) ++#define BXT_DRAM_RANK_MASK 0x3 ++#define BXT_DRAM_RANK_SINGLE 0x1 ++#define BXT_DRAM_RANK_DUAL 0x3 ++#define BXT_DRAM_WIDTH_MASK (0x3 << 4) ++#define BXT_DRAM_WIDTH_SHIFT 4 ++#define BXT_DRAM_WIDTH_X8 (0x0 << 4) ++#define BXT_DRAM_WIDTH_X16 (0x1 << 4) ++#define BXT_DRAM_WIDTH_X32 (0x2 << 4) ++#define BXT_DRAM_WIDTH_X64 (0x3 << 4) ++#define BXT_DRAM_SIZE_MASK (0x7 << 6) ++#define BXT_DRAM_SIZE_SHIFT 6 ++#define BXT_DRAM_SIZE_4GBIT (0x0 << 6) ++#define BXT_DRAM_SIZE_6GBIT (0x1 << 6) ++#define BXT_DRAM_SIZE_8GBIT (0x2 << 6) ++#define BXT_DRAM_SIZE_12GBIT (0x3 << 6) ++#define BXT_DRAM_SIZE_16GBIT (0x4 << 6) ++#define BXT_DRAM_TYPE_MASK (0x7 << 22) ++#define BXT_DRAM_TYPE_SHIFT 22 ++#define BXT_DRAM_TYPE_DDR3 (0x0 << 22) ++#define BXT_DRAM_TYPE_LPDDR3 (0x1 << 22) ++#define BXT_DRAM_TYPE_LPDDR4 (0x2 << 22) ++#define BXT_DRAM_TYPE_DDR4 (0x4 << 22) ++ ++#define SKL_MEMORY_FREQ_MULTIPLIER_HZ 266666666 ++#define SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5E04) ++#define SKL_REQ_DATA_MASK (0xF << 0) ++ ++#define SKL_MAD_INTER_CHANNEL_0_0_0_MCHBAR_MCMAIN _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5000) ++#define SKL_DRAM_DDR_TYPE_MASK (0x3 << 0) ++#define SKL_DRAM_DDR_TYPE_DDR4 (0 << 0) ++#define SKL_DRAM_DDR_TYPE_DDR3 (1 << 0) ++#define SKL_DRAM_DDR_TYPE_LPDDR3 (2 << 0) ++#define SKL_DRAM_DDR_TYPE_LPDDR4 (3 << 0) ++ ++#define SKL_MAD_DIMM_CH0_0_0_0_MCHBAR_MCMAIN _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x500C) ++#define SKL_MAD_DIMM_CH1_0_0_0_MCHBAR_MCMAIN _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5010) ++#define SKL_DRAM_S_SHIFT 16 ++#define SKL_DRAM_SIZE_MASK 0x3F ++#define SKL_DRAM_WIDTH_MASK (0x3 << 8) ++#define SKL_DRAM_WIDTH_SHIFT 8 ++#define SKL_DRAM_WIDTH_X8 (0x0 << 8) ++#define SKL_DRAM_WIDTH_X16 (0x1 << 8) ++#define SKL_DRAM_WIDTH_X32 (0x2 << 8) ++#define SKL_DRAM_RANK_MASK (0x1 << 10) ++#define SKL_DRAM_RANK_SHIFT 10 ++#define SKL_DRAM_RANK_1 (0x0 << 10) ++#define SKL_DRAM_RANK_2 (0x1 << 10) ++#define SKL_DRAM_RANK_MASK (0x1 << 10) ++#define CNL_DRAM_SIZE_MASK 0x7F ++#define CNL_DRAM_WIDTH_MASK (0x3 << 7) ++#define CNL_DRAM_WIDTH_SHIFT 7 ++#define CNL_DRAM_WIDTH_X8 (0x0 << 7) ++#define CNL_DRAM_WIDTH_X16 (0x1 << 7) ++#define CNL_DRAM_WIDTH_X32 (0x2 << 7) ++#define CNL_DRAM_RANK_MASK (0x3 << 9) ++#define CNL_DRAM_RANK_SHIFT 9 ++#define CNL_DRAM_RANK_1 (0x0 << 9) ++#define CNL_DRAM_RANK_2 (0x1 << 9) ++#define CNL_DRAM_RANK_3 (0x2 << 9) ++#define CNL_DRAM_RANK_4 (0x3 << 9) ++ ++/* Please see hsw_read_dcomp() and hsw_write_dcomp() before using this register, ++ * since on HSW we can't write to it using I915_WRITE. */ ++#define D_COMP_HSW _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5F0C) ++#define D_COMP_BDW _MMIO(0x138144) ++#define D_COMP_RCOMP_IN_PROGRESS (1 << 9) ++#define D_COMP_COMP_FORCE (1 << 8) ++#define D_COMP_COMP_DISABLE (1 << 0) ++ ++/* Pipe WM_LINETIME - watermark line time */ ++#define _PIPE_WM_LINETIME_A 0x45270 ++#define _PIPE_WM_LINETIME_B 0x45274 ++#define PIPE_WM_LINETIME(pipe) _MMIO_PIPE(pipe, _PIPE_WM_LINETIME_A, _PIPE_WM_LINETIME_B) ++#define PIPE_WM_LINETIME_MASK (0x1ff) ++#define PIPE_WM_LINETIME_TIME(x) ((x)) ++#define PIPE_WM_LINETIME_IPS_LINETIME_MASK (0x1ff << 16) ++#define PIPE_WM_LINETIME_IPS_LINETIME(x) ((x) << 16) ++ ++/* SFUSE_STRAP */ ++#define SFUSE_STRAP _MMIO(0xc2014) ++#define SFUSE_STRAP_FUSE_LOCK (1 << 13) ++#define SFUSE_STRAP_RAW_FREQUENCY (1 << 8) ++#define SFUSE_STRAP_DISPLAY_DISABLED (1 << 7) ++#define SFUSE_STRAP_CRT_DISABLED (1 << 6) ++#define SFUSE_STRAP_DDIF_DETECTED (1 << 3) ++#define SFUSE_STRAP_DDIB_DETECTED (1 << 2) ++#define SFUSE_STRAP_DDIC_DETECTED (1 << 1) ++#define SFUSE_STRAP_DDID_DETECTED (1 << 0) ++ ++#define WM_MISC _MMIO(0x45260) ++#define WM_MISC_DATA_PARTITION_5_6 (1 << 0) ++ ++#define WM_DBG _MMIO(0x45280) ++#define WM_DBG_DISALLOW_MULTIPLE_LP (1 << 0) ++#define WM_DBG_DISALLOW_MAXFIFO (1 << 1) ++#define WM_DBG_DISALLOW_SPRITE (1 << 2) ++ ++/* pipe CSC */ ++#define _PIPE_A_CSC_COEFF_RY_GY 0x49010 ++#define _PIPE_A_CSC_COEFF_BY 0x49014 ++#define _PIPE_A_CSC_COEFF_RU_GU 0x49018 ++#define _PIPE_A_CSC_COEFF_BU 0x4901c ++#define _PIPE_A_CSC_COEFF_RV_GV 0x49020 ++#define _PIPE_A_CSC_COEFF_BV 0x49024 ++ ++#define _PIPE_A_CSC_MODE 0x49028 ++#define ICL_CSC_ENABLE (1 << 31) ++#define ICL_OUTPUT_CSC_ENABLE (1 << 30) ++#define CSC_BLACK_SCREEN_OFFSET (1 << 2) ++#define CSC_POSITION_BEFORE_GAMMA (1 << 1) ++#define CSC_MODE_YUV_TO_RGB (1 << 0) ++ ++#define _PIPE_A_CSC_PREOFF_HI 0x49030 ++#define _PIPE_A_CSC_PREOFF_ME 0x49034 ++#define _PIPE_A_CSC_PREOFF_LO 0x49038 ++#define _PIPE_A_CSC_POSTOFF_HI 0x49040 ++#define _PIPE_A_CSC_POSTOFF_ME 0x49044 ++#define _PIPE_A_CSC_POSTOFF_LO 0x49048 ++ ++#define _PIPE_B_CSC_COEFF_RY_GY 0x49110 ++#define _PIPE_B_CSC_COEFF_BY 0x49114 ++#define _PIPE_B_CSC_COEFF_RU_GU 0x49118 ++#define _PIPE_B_CSC_COEFF_BU 0x4911c ++#define _PIPE_B_CSC_COEFF_RV_GV 0x49120 ++#define _PIPE_B_CSC_COEFF_BV 0x49124 ++#define _PIPE_B_CSC_MODE 0x49128 ++#define _PIPE_B_CSC_PREOFF_HI 0x49130 ++#define _PIPE_B_CSC_PREOFF_ME 0x49134 ++#define _PIPE_B_CSC_PREOFF_LO 0x49138 ++#define _PIPE_B_CSC_POSTOFF_HI 0x49140 ++#define _PIPE_B_CSC_POSTOFF_ME 0x49144 ++#define _PIPE_B_CSC_POSTOFF_LO 0x49148 ++ ++#define PIPE_CSC_COEFF_RY_GY(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_COEFF_RY_GY, _PIPE_B_CSC_COEFF_RY_GY) ++#define PIPE_CSC_COEFF_BY(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_COEFF_BY, _PIPE_B_CSC_COEFF_BY) ++#define PIPE_CSC_COEFF_RU_GU(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_COEFF_RU_GU, _PIPE_B_CSC_COEFF_RU_GU) ++#define PIPE_CSC_COEFF_BU(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_COEFF_BU, _PIPE_B_CSC_COEFF_BU) ++#define PIPE_CSC_COEFF_RV_GV(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_COEFF_RV_GV, _PIPE_B_CSC_COEFF_RV_GV) ++#define PIPE_CSC_COEFF_BV(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_COEFF_BV, _PIPE_B_CSC_COEFF_BV) ++#define PIPE_CSC_MODE(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_MODE, _PIPE_B_CSC_MODE) ++#define PIPE_CSC_PREOFF_HI(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_PREOFF_HI, _PIPE_B_CSC_PREOFF_HI) ++#define PIPE_CSC_PREOFF_ME(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_PREOFF_ME, _PIPE_B_CSC_PREOFF_ME) ++#define PIPE_CSC_PREOFF_LO(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_PREOFF_LO, _PIPE_B_CSC_PREOFF_LO) ++#define PIPE_CSC_POSTOFF_HI(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_POSTOFF_HI, _PIPE_B_CSC_POSTOFF_HI) ++#define PIPE_CSC_POSTOFF_ME(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_POSTOFF_ME, _PIPE_B_CSC_POSTOFF_ME) ++#define PIPE_CSC_POSTOFF_LO(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_POSTOFF_LO, _PIPE_B_CSC_POSTOFF_LO) ++ ++/* Pipe Output CSC */ ++#define _PIPE_A_OUTPUT_CSC_COEFF_RY_GY 0x49050 ++#define _PIPE_A_OUTPUT_CSC_COEFF_BY 0x49054 ++#define _PIPE_A_OUTPUT_CSC_COEFF_RU_GU 0x49058 ++#define _PIPE_A_OUTPUT_CSC_COEFF_BU 0x4905c ++#define _PIPE_A_OUTPUT_CSC_COEFF_RV_GV 0x49060 ++#define _PIPE_A_OUTPUT_CSC_COEFF_BV 0x49064 ++#define _PIPE_A_OUTPUT_CSC_PREOFF_HI 0x49068 ++#define _PIPE_A_OUTPUT_CSC_PREOFF_ME 0x4906c ++#define _PIPE_A_OUTPUT_CSC_PREOFF_LO 0x49070 ++#define _PIPE_A_OUTPUT_CSC_POSTOFF_HI 0x49074 ++#define _PIPE_A_OUTPUT_CSC_POSTOFF_ME 0x49078 ++#define _PIPE_A_OUTPUT_CSC_POSTOFF_LO 0x4907c ++ ++#define _PIPE_B_OUTPUT_CSC_COEFF_RY_GY 0x49150 ++#define _PIPE_B_OUTPUT_CSC_COEFF_BY 0x49154 ++#define _PIPE_B_OUTPUT_CSC_COEFF_RU_GU 0x49158 ++#define _PIPE_B_OUTPUT_CSC_COEFF_BU 0x4915c ++#define _PIPE_B_OUTPUT_CSC_COEFF_RV_GV 0x49160 ++#define _PIPE_B_OUTPUT_CSC_COEFF_BV 0x49164 ++#define _PIPE_B_OUTPUT_CSC_PREOFF_HI 0x49168 ++#define _PIPE_B_OUTPUT_CSC_PREOFF_ME 0x4916c ++#define _PIPE_B_OUTPUT_CSC_PREOFF_LO 0x49170 ++#define _PIPE_B_OUTPUT_CSC_POSTOFF_HI 0x49174 ++#define _PIPE_B_OUTPUT_CSC_POSTOFF_ME 0x49178 ++#define _PIPE_B_OUTPUT_CSC_POSTOFF_LO 0x4917c ++ ++#define PIPE_CSC_OUTPUT_COEFF_RY_GY(pipe) _MMIO_PIPE(pipe,\ ++ _PIPE_A_OUTPUT_CSC_COEFF_RY_GY,\ ++ _PIPE_B_OUTPUT_CSC_COEFF_RY_GY) ++#define PIPE_CSC_OUTPUT_COEFF_BY(pipe) _MMIO_PIPE(pipe, \ ++ _PIPE_A_OUTPUT_CSC_COEFF_BY, \ ++ _PIPE_B_OUTPUT_CSC_COEFF_BY) ++#define PIPE_CSC_OUTPUT_COEFF_RU_GU(pipe) _MMIO_PIPE(pipe, \ ++ _PIPE_A_OUTPUT_CSC_COEFF_RU_GU, \ ++ _PIPE_B_OUTPUT_CSC_COEFF_RU_GU) ++#define PIPE_CSC_OUTPUT_COEFF_BU(pipe) _MMIO_PIPE(pipe, \ ++ _PIPE_A_OUTPUT_CSC_COEFF_BU, \ ++ _PIPE_B_OUTPUT_CSC_COEFF_BU) ++#define PIPE_CSC_OUTPUT_COEFF_RV_GV(pipe) _MMIO_PIPE(pipe, \ ++ _PIPE_A_OUTPUT_CSC_COEFF_RV_GV, \ ++ _PIPE_B_OUTPUT_CSC_COEFF_RV_GV) ++#define PIPE_CSC_OUTPUT_COEFF_BV(pipe) _MMIO_PIPE(pipe, \ ++ _PIPE_A_OUTPUT_CSC_COEFF_BV, \ ++ _PIPE_B_OUTPUT_CSC_COEFF_BV) ++#define PIPE_CSC_OUTPUT_PREOFF_HI(pipe) _MMIO_PIPE(pipe, \ ++ _PIPE_A_OUTPUT_CSC_PREOFF_HI, \ ++ _PIPE_B_OUTPUT_CSC_PREOFF_HI) ++#define PIPE_CSC_OUTPUT_PREOFF_ME(pipe) _MMIO_PIPE(pipe, \ ++ _PIPE_A_OUTPUT_CSC_PREOFF_ME, \ ++ _PIPE_B_OUTPUT_CSC_PREOFF_ME) ++#define PIPE_CSC_OUTPUT_PREOFF_LO(pipe) _MMIO_PIPE(pipe, \ ++ _PIPE_A_OUTPUT_CSC_PREOFF_LO, \ ++ _PIPE_B_OUTPUT_CSC_PREOFF_LO) ++#define PIPE_CSC_OUTPUT_POSTOFF_HI(pipe) _MMIO_PIPE(pipe, \ ++ _PIPE_A_OUTPUT_CSC_POSTOFF_HI, \ ++ _PIPE_B_OUTPUT_CSC_POSTOFF_HI) ++#define PIPE_CSC_OUTPUT_POSTOFF_ME(pipe) _MMIO_PIPE(pipe, \ ++ _PIPE_A_OUTPUT_CSC_POSTOFF_ME, \ ++ _PIPE_B_OUTPUT_CSC_POSTOFF_ME) ++#define PIPE_CSC_OUTPUT_POSTOFF_LO(pipe) _MMIO_PIPE(pipe, \ ++ _PIPE_A_OUTPUT_CSC_POSTOFF_LO, \ ++ _PIPE_B_OUTPUT_CSC_POSTOFF_LO) ++ ++/* pipe degamma/gamma LUTs on IVB+ */ ++#define _PAL_PREC_INDEX_A 0x4A400 ++#define _PAL_PREC_INDEX_B 0x4AC00 ++#define _PAL_PREC_INDEX_C 0x4B400 ++#define PAL_PREC_10_12_BIT (0 << 31) ++#define PAL_PREC_SPLIT_MODE (1 << 31) ++#define PAL_PREC_AUTO_INCREMENT (1 << 15) ++#define PAL_PREC_INDEX_VALUE_MASK (0x3ff << 0) ++#define PAL_PREC_INDEX_VALUE(x) ((x) << 0) ++#define _PAL_PREC_DATA_A 0x4A404 ++#define _PAL_PREC_DATA_B 0x4AC04 ++#define _PAL_PREC_DATA_C 0x4B404 ++#define _PAL_PREC_GC_MAX_A 0x4A410 ++#define _PAL_PREC_GC_MAX_B 0x4AC10 ++#define _PAL_PREC_GC_MAX_C 0x4B410 ++#define _PAL_PREC_EXT_GC_MAX_A 0x4A420 ++#define _PAL_PREC_EXT_GC_MAX_B 0x4AC20 ++#define _PAL_PREC_EXT_GC_MAX_C 0x4B420 ++#define _PAL_PREC_EXT2_GC_MAX_A 0x4A430 ++#define _PAL_PREC_EXT2_GC_MAX_B 0x4AC30 ++#define _PAL_PREC_EXT2_GC_MAX_C 0x4B430 ++ ++#define PREC_PAL_INDEX(pipe) _MMIO_PIPE(pipe, _PAL_PREC_INDEX_A, _PAL_PREC_INDEX_B) ++#define PREC_PAL_DATA(pipe) _MMIO_PIPE(pipe, _PAL_PREC_DATA_A, _PAL_PREC_DATA_B) ++#define PREC_PAL_GC_MAX(pipe, i) _MMIO(_PIPE(pipe, _PAL_PREC_GC_MAX_A, _PAL_PREC_GC_MAX_B) + (i) * 4) ++#define PREC_PAL_EXT_GC_MAX(pipe, i) _MMIO(_PIPE(pipe, _PAL_PREC_EXT_GC_MAX_A, _PAL_PREC_EXT_GC_MAX_B) + (i) * 4) ++#define PREC_PAL_EXT2_GC_MAX(pipe, i) _MMIO(_PIPE(pipe, _PAL_PREC_EXT2_GC_MAX_A, _PAL_PREC_EXT2_GC_MAX_B) + (i) * 4) ++ ++#define _PRE_CSC_GAMC_INDEX_A 0x4A484 ++#define _PRE_CSC_GAMC_INDEX_B 0x4AC84 ++#define _PRE_CSC_GAMC_INDEX_C 0x4B484 ++#define PRE_CSC_GAMC_AUTO_INCREMENT (1 << 10) ++#define _PRE_CSC_GAMC_DATA_A 0x4A488 ++#define _PRE_CSC_GAMC_DATA_B 0x4AC88 ++#define _PRE_CSC_GAMC_DATA_C 0x4B488 ++ ++#define PRE_CSC_GAMC_INDEX(pipe) _MMIO_PIPE(pipe, _PRE_CSC_GAMC_INDEX_A, _PRE_CSC_GAMC_INDEX_B) ++#define PRE_CSC_GAMC_DATA(pipe) _MMIO_PIPE(pipe, _PRE_CSC_GAMC_DATA_A, _PRE_CSC_GAMC_DATA_B) ++ ++/* pipe CSC & degamma/gamma LUTs on CHV */ ++#define _CGM_PIPE_A_CSC_COEFF01 (VLV_DISPLAY_BASE + 0x67900) ++#define _CGM_PIPE_A_CSC_COEFF23 (VLV_DISPLAY_BASE + 0x67904) ++#define _CGM_PIPE_A_CSC_COEFF45 (VLV_DISPLAY_BASE + 0x67908) ++#define _CGM_PIPE_A_CSC_COEFF67 (VLV_DISPLAY_BASE + 0x6790C) ++#define _CGM_PIPE_A_CSC_COEFF8 (VLV_DISPLAY_BASE + 0x67910) ++#define _CGM_PIPE_A_DEGAMMA (VLV_DISPLAY_BASE + 0x66000) ++#define _CGM_PIPE_A_GAMMA (VLV_DISPLAY_BASE + 0x67000) ++#define _CGM_PIPE_A_MODE (VLV_DISPLAY_BASE + 0x67A00) ++#define CGM_PIPE_MODE_GAMMA (1 << 2) ++#define CGM_PIPE_MODE_CSC (1 << 1) ++#define CGM_PIPE_MODE_DEGAMMA (1 << 0) ++ ++#define _CGM_PIPE_B_CSC_COEFF01 (VLV_DISPLAY_BASE + 0x69900) ++#define _CGM_PIPE_B_CSC_COEFF23 (VLV_DISPLAY_BASE + 0x69904) ++#define _CGM_PIPE_B_CSC_COEFF45 (VLV_DISPLAY_BASE + 0x69908) ++#define _CGM_PIPE_B_CSC_COEFF67 (VLV_DISPLAY_BASE + 0x6990C) ++#define _CGM_PIPE_B_CSC_COEFF8 (VLV_DISPLAY_BASE + 0x69910) ++#define _CGM_PIPE_B_DEGAMMA (VLV_DISPLAY_BASE + 0x68000) ++#define _CGM_PIPE_B_GAMMA (VLV_DISPLAY_BASE + 0x69000) ++#define _CGM_PIPE_B_MODE (VLV_DISPLAY_BASE + 0x69A00) ++ ++#define CGM_PIPE_CSC_COEFF01(pipe) _MMIO_PIPE(pipe, _CGM_PIPE_A_CSC_COEFF01, _CGM_PIPE_B_CSC_COEFF01) ++#define CGM_PIPE_CSC_COEFF23(pipe) _MMIO_PIPE(pipe, _CGM_PIPE_A_CSC_COEFF23, _CGM_PIPE_B_CSC_COEFF23) ++#define CGM_PIPE_CSC_COEFF45(pipe) _MMIO_PIPE(pipe, _CGM_PIPE_A_CSC_COEFF45, _CGM_PIPE_B_CSC_COEFF45) ++#define CGM_PIPE_CSC_COEFF67(pipe) _MMIO_PIPE(pipe, _CGM_PIPE_A_CSC_COEFF67, _CGM_PIPE_B_CSC_COEFF67) ++#define CGM_PIPE_CSC_COEFF8(pipe) _MMIO_PIPE(pipe, _CGM_PIPE_A_CSC_COEFF8, _CGM_PIPE_B_CSC_COEFF8) ++#define CGM_PIPE_DEGAMMA(pipe, i, w) _MMIO(_PIPE(pipe, _CGM_PIPE_A_DEGAMMA, _CGM_PIPE_B_DEGAMMA) + (i) * 8 + (w) * 4) ++#define CGM_PIPE_GAMMA(pipe, i, w) _MMIO(_PIPE(pipe, _CGM_PIPE_A_GAMMA, _CGM_PIPE_B_GAMMA) + (i) * 8 + (w) * 4) ++#define CGM_PIPE_MODE(pipe) _MMIO_PIPE(pipe, _CGM_PIPE_A_MODE, _CGM_PIPE_B_MODE) ++ ++/* MIPI DSI registers */ ++ ++#define _MIPI_PORT(port, a, c) (((port) == PORT_A) ? a : c) /* ports A and C only */ ++#define _MMIO_MIPI(port, a, c) _MMIO(_MIPI_PORT(port, a, c)) ++ ++/* Gen11 DSI */ ++#define _MMIO_DSI(tc, dsi0, dsi1) _MMIO_TRANS((tc) - TRANSCODER_DSI_0, \ ++ dsi0, dsi1) ++ ++#define MIPIO_TXESC_CLK_DIV1 _MMIO(0x160004) ++#define GLK_TX_ESC_CLK_DIV1_MASK 0x3FF ++#define MIPIO_TXESC_CLK_DIV2 _MMIO(0x160008) ++#define GLK_TX_ESC_CLK_DIV2_MASK 0x3FF ++ ++#define _ICL_DSI_ESC_CLK_DIV0 0x6b090 ++#define _ICL_DSI_ESC_CLK_DIV1 0x6b890 ++#define ICL_DSI_ESC_CLK_DIV(port) _MMIO_PORT((port), \ ++ _ICL_DSI_ESC_CLK_DIV0, \ ++ _ICL_DSI_ESC_CLK_DIV1) ++#define _ICL_DPHY_ESC_CLK_DIV0 0x162190 ++#define _ICL_DPHY_ESC_CLK_DIV1 0x6C190 ++#define ICL_DPHY_ESC_CLK_DIV(port) _MMIO_PORT((port), \ ++ _ICL_DPHY_ESC_CLK_DIV0, \ ++ _ICL_DPHY_ESC_CLK_DIV1) ++#define ICL_BYTE_CLK_PER_ESC_CLK_MASK (0x1f << 16) ++#define ICL_BYTE_CLK_PER_ESC_CLK_SHIFT 16 ++#define ICL_ESC_CLK_DIV_MASK 0x1ff ++#define ICL_ESC_CLK_DIV_SHIFT 0 ++#define DSI_MAX_ESC_CLK 20000 /* in KHz */ ++ ++/* Gen4+ Timestamp and Pipe Frame time stamp registers */ ++#define GEN4_TIMESTAMP _MMIO(0x2358) ++#define ILK_TIMESTAMP_HI _MMIO(0x70070) ++#define IVB_TIMESTAMP_CTR _MMIO(0x44070) ++ ++#define GEN9_TIMESTAMP_OVERRIDE _MMIO(0x44074) ++#define GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DIVIDER_SHIFT 0 ++#define GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DIVIDER_MASK 0x3ff ++#define GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DENOMINATOR_SHIFT 12 ++#define GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DENOMINATOR_MASK (0xf << 12) ++ ++#define _PIPE_FRMTMSTMP_A 0x70048 ++#define PIPE_FRMTMSTMP(pipe) \ ++ _MMIO_PIPE2(pipe, _PIPE_FRMTMSTMP_A) ++ ++/* BXT MIPI clock controls */ ++#define BXT_MAX_VAR_OUTPUT_KHZ 39500 ++ ++#define BXT_MIPI_CLOCK_CTL _MMIO(0x46090) ++#define BXT_MIPI1_DIV_SHIFT 26 ++#define BXT_MIPI2_DIV_SHIFT 10 ++#define BXT_MIPI_DIV_SHIFT(port) \ ++ _MIPI_PORT(port, BXT_MIPI1_DIV_SHIFT, \ ++ BXT_MIPI2_DIV_SHIFT) ++ ++/* TX control divider to select actual TX clock output from (8x/var) */ ++#define BXT_MIPI1_TX_ESCLK_SHIFT 26 ++#define BXT_MIPI2_TX_ESCLK_SHIFT 10 ++#define BXT_MIPI_TX_ESCLK_SHIFT(port) \ ++ _MIPI_PORT(port, BXT_MIPI1_TX_ESCLK_SHIFT, \ ++ BXT_MIPI2_TX_ESCLK_SHIFT) ++#define BXT_MIPI1_TX_ESCLK_FIXDIV_MASK (0x3F << 26) ++#define BXT_MIPI2_TX_ESCLK_FIXDIV_MASK (0x3F << 10) ++#define BXT_MIPI_TX_ESCLK_FIXDIV_MASK(port) \ ++ _MIPI_PORT(port, BXT_MIPI1_TX_ESCLK_FIXDIV_MASK, \ ++ BXT_MIPI2_TX_ESCLK_FIXDIV_MASK) ++#define BXT_MIPI_TX_ESCLK_DIVIDER(port, val) \ ++ (((val) & 0x3F) << BXT_MIPI_TX_ESCLK_SHIFT(port)) ++/* RX upper control divider to select actual RX clock output from 8x */ ++#define BXT_MIPI1_RX_ESCLK_UPPER_SHIFT 21 ++#define BXT_MIPI2_RX_ESCLK_UPPER_SHIFT 5 ++#define BXT_MIPI_RX_ESCLK_UPPER_SHIFT(port) \ ++ _MIPI_PORT(port, BXT_MIPI1_RX_ESCLK_UPPER_SHIFT, \ ++ BXT_MIPI2_RX_ESCLK_UPPER_SHIFT) ++#define BXT_MIPI1_RX_ESCLK_UPPER_FIXDIV_MASK (3 << 21) ++#define BXT_MIPI2_RX_ESCLK_UPPER_FIXDIV_MASK (3 << 5) ++#define BXT_MIPI_RX_ESCLK_UPPER_FIXDIV_MASK(port) \ ++ _MIPI_PORT(port, BXT_MIPI1_RX_ESCLK_UPPER_FIXDIV_MASK, \ ++ BXT_MIPI2_RX_ESCLK_UPPER_FIXDIV_MASK) ++#define BXT_MIPI_RX_ESCLK_UPPER_DIVIDER(port, val) \ ++ (((val) & 3) << BXT_MIPI_RX_ESCLK_UPPER_SHIFT(port)) ++/* 8/3X divider to select the actual 8/3X clock output from 8x */ ++#define BXT_MIPI1_8X_BY3_SHIFT 19 ++#define BXT_MIPI2_8X_BY3_SHIFT 3 ++#define BXT_MIPI_8X_BY3_SHIFT(port) \ ++ _MIPI_PORT(port, BXT_MIPI1_8X_BY3_SHIFT, \ ++ BXT_MIPI2_8X_BY3_SHIFT) ++#define BXT_MIPI1_8X_BY3_DIVIDER_MASK (3 << 19) ++#define BXT_MIPI2_8X_BY3_DIVIDER_MASK (3 << 3) ++#define BXT_MIPI_8X_BY3_DIVIDER_MASK(port) \ ++ _MIPI_PORT(port, BXT_MIPI1_8X_BY3_DIVIDER_MASK, \ ++ BXT_MIPI2_8X_BY3_DIVIDER_MASK) ++#define BXT_MIPI_8X_BY3_DIVIDER(port, val) \ ++ (((val) & 3) << BXT_MIPI_8X_BY3_SHIFT(port)) ++/* RX lower control divider to select actual RX clock output from 8x */ ++#define BXT_MIPI1_RX_ESCLK_LOWER_SHIFT 16 ++#define BXT_MIPI2_RX_ESCLK_LOWER_SHIFT 0 ++#define BXT_MIPI_RX_ESCLK_LOWER_SHIFT(port) \ ++ _MIPI_PORT(port, BXT_MIPI1_RX_ESCLK_LOWER_SHIFT, \ ++ BXT_MIPI2_RX_ESCLK_LOWER_SHIFT) ++#define BXT_MIPI1_RX_ESCLK_LOWER_FIXDIV_MASK (3 << 16) ++#define BXT_MIPI2_RX_ESCLK_LOWER_FIXDIV_MASK (3 << 0) ++#define BXT_MIPI_RX_ESCLK_LOWER_FIXDIV_MASK(port) \ ++ _MIPI_PORT(port, BXT_MIPI1_RX_ESCLK_LOWER_FIXDIV_MASK, \ ++ BXT_MIPI2_RX_ESCLK_LOWER_FIXDIV_MASK) ++#define BXT_MIPI_RX_ESCLK_LOWER_DIVIDER(port, val) \ ++ (((val) & 3) << BXT_MIPI_RX_ESCLK_LOWER_SHIFT(port)) ++ ++#define RX_DIVIDER_BIT_1_2 0x3 ++#define RX_DIVIDER_BIT_3_4 0xC ++ ++/* BXT MIPI mode configure */ ++#define _BXT_MIPIA_TRANS_HACTIVE 0x6B0F8 ++#define _BXT_MIPIC_TRANS_HACTIVE 0x6B8F8 ++#define BXT_MIPI_TRANS_HACTIVE(tc) _MMIO_MIPI(tc, \ ++ _BXT_MIPIA_TRANS_HACTIVE, _BXT_MIPIC_TRANS_HACTIVE) ++ ++#define _BXT_MIPIA_TRANS_VACTIVE 0x6B0FC ++#define _BXT_MIPIC_TRANS_VACTIVE 0x6B8FC ++#define BXT_MIPI_TRANS_VACTIVE(tc) _MMIO_MIPI(tc, \ ++ _BXT_MIPIA_TRANS_VACTIVE, _BXT_MIPIC_TRANS_VACTIVE) ++ ++#define _BXT_MIPIA_TRANS_VTOTAL 0x6B100 ++#define _BXT_MIPIC_TRANS_VTOTAL 0x6B900 ++#define BXT_MIPI_TRANS_VTOTAL(tc) _MMIO_MIPI(tc, \ ++ _BXT_MIPIA_TRANS_VTOTAL, _BXT_MIPIC_TRANS_VTOTAL) ++ ++#define BXT_DSI_PLL_CTL _MMIO(0x161000) ++#define BXT_DSI_PLL_PVD_RATIO_SHIFT 16 ++#define BXT_DSI_PLL_PVD_RATIO_MASK (3 << BXT_DSI_PLL_PVD_RATIO_SHIFT) ++#define BXT_DSI_PLL_PVD_RATIO_1 (1 << BXT_DSI_PLL_PVD_RATIO_SHIFT) ++#define BXT_DSIC_16X_BY1 (0 << 10) ++#define BXT_DSIC_16X_BY2 (1 << 10) ++#define BXT_DSIC_16X_BY3 (2 << 10) ++#define BXT_DSIC_16X_BY4 (3 << 10) ++#define BXT_DSIC_16X_MASK (3 << 10) ++#define BXT_DSIA_16X_BY1 (0 << 8) ++#define BXT_DSIA_16X_BY2 (1 << 8) ++#define BXT_DSIA_16X_BY3 (2 << 8) ++#define BXT_DSIA_16X_BY4 (3 << 8) ++#define BXT_DSIA_16X_MASK (3 << 8) ++#define BXT_DSI_FREQ_SEL_SHIFT 8 ++#define BXT_DSI_FREQ_SEL_MASK (0xF << BXT_DSI_FREQ_SEL_SHIFT) ++ ++#define BXT_DSI_PLL_RATIO_MAX 0x7D ++#define BXT_DSI_PLL_RATIO_MIN 0x22 ++#define GLK_DSI_PLL_RATIO_MAX 0x6F ++#define GLK_DSI_PLL_RATIO_MIN 0x22 ++#define BXT_DSI_PLL_RATIO_MASK 0xFF ++#define BXT_REF_CLOCK_KHZ 19200 ++ ++#define BXT_DSI_PLL_ENABLE _MMIO(0x46080) ++#define BXT_DSI_PLL_DO_ENABLE (1 << 31) ++#define BXT_DSI_PLL_LOCKED (1 << 30) ++ ++#define _MIPIA_PORT_CTRL (VLV_DISPLAY_BASE + 0x61190) ++#define _MIPIC_PORT_CTRL (VLV_DISPLAY_BASE + 0x61700) ++#define MIPI_PORT_CTRL(port) _MMIO_MIPI(port, _MIPIA_PORT_CTRL, _MIPIC_PORT_CTRL) ++ ++ /* BXT port control */ ++#define _BXT_MIPIA_PORT_CTRL 0x6B0C0 ++#define _BXT_MIPIC_PORT_CTRL 0x6B8C0 ++#define BXT_MIPI_PORT_CTRL(tc) _MMIO_MIPI(tc, _BXT_MIPIA_PORT_CTRL, _BXT_MIPIC_PORT_CTRL) ++ ++/* ICL DSI MODE control */ ++#define _ICL_DSI_IO_MODECTL_0 0x6B094 ++#define _ICL_DSI_IO_MODECTL_1 0x6B894 ++#define ICL_DSI_IO_MODECTL(port) _MMIO_PORT(port, \ ++ _ICL_DSI_IO_MODECTL_0, \ ++ _ICL_DSI_IO_MODECTL_1) ++#define COMBO_PHY_MODE_DSI (1 << 0) ++ ++/* Display Stream Splitter Control */ ++#define DSS_CTL1 _MMIO(0x67400) ++#define SPLITTER_ENABLE (1 << 31) ++#define JOINER_ENABLE (1 << 30) ++#define DUAL_LINK_MODE_INTERLEAVE (1 << 24) ++#define DUAL_LINK_MODE_FRONTBACK (0 << 24) ++#define OVERLAP_PIXELS_MASK (0xf << 16) ++#define OVERLAP_PIXELS(pixels) ((pixels) << 16) ++#define LEFT_DL_BUF_TARGET_DEPTH_MASK (0xfff << 0) ++#define LEFT_DL_BUF_TARGET_DEPTH(pixels) ((pixels) << 0) ++#define MAX_DL_BUFFER_TARGET_DEPTH 0x5a0 ++ ++#define DSS_CTL2 _MMIO(0x67404) ++#define LEFT_BRANCH_VDSC_ENABLE (1 << 31) ++#define RIGHT_BRANCH_VDSC_ENABLE (1 << 15) ++#define RIGHT_DL_BUF_TARGET_DEPTH_MASK (0xfff << 0) ++#define RIGHT_DL_BUF_TARGET_DEPTH(pixels) ((pixels) << 0) ++ ++#define _ICL_PIPE_DSS_CTL1_PB 0x78200 ++#define _ICL_PIPE_DSS_CTL1_PC 0x78400 ++#define ICL_PIPE_DSS_CTL1(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_PIPE_DSS_CTL1_PB, \ ++ _ICL_PIPE_DSS_CTL1_PC) ++#define BIG_JOINER_ENABLE (1 << 29) ++#define MASTER_BIG_JOINER_ENABLE (1 << 28) ++#define VGA_CENTERING_ENABLE (1 << 27) ++ ++#define _ICL_PIPE_DSS_CTL2_PB 0x78204 ++#define _ICL_PIPE_DSS_CTL2_PC 0x78404 ++#define ICL_PIPE_DSS_CTL2(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_PIPE_DSS_CTL2_PB, \ ++ _ICL_PIPE_DSS_CTL2_PC) ++ ++#define BXT_P_DSI_REGULATOR_CFG _MMIO(0x160020) ++#define STAP_SELECT (1 << 0) ++ ++#define BXT_P_DSI_REGULATOR_TX_CTRL _MMIO(0x160054) ++#define HS_IO_CTRL_SELECT (1 << 0) ++ ++#define DPI_ENABLE (1 << 31) /* A + C */ ++#define MIPIA_MIPI4DPHY_DELAY_COUNT_SHIFT 27 ++#define MIPIA_MIPI4DPHY_DELAY_COUNT_MASK (0xf << 27) ++#define DUAL_LINK_MODE_SHIFT 26 ++#define DUAL_LINK_MODE_MASK (1 << 26) ++#define DUAL_LINK_MODE_FRONT_BACK (0 << 26) ++#define DUAL_LINK_MODE_PIXEL_ALTERNATIVE (1 << 26) ++#define DITHERING_ENABLE (1 << 25) /* A + C */ ++#define FLOPPED_HSTX (1 << 23) ++#define DE_INVERT (1 << 19) /* XXX */ ++#define MIPIA_FLISDSI_DELAY_COUNT_SHIFT 18 ++#define MIPIA_FLISDSI_DELAY_COUNT_MASK (0xf << 18) ++#define AFE_LATCHOUT (1 << 17) ++#define LP_OUTPUT_HOLD (1 << 16) ++#define MIPIC_FLISDSI_DELAY_COUNT_HIGH_SHIFT 15 ++#define MIPIC_FLISDSI_DELAY_COUNT_HIGH_MASK (1 << 15) ++#define MIPIC_MIPI4DPHY_DELAY_COUNT_SHIFT 11 ++#define MIPIC_MIPI4DPHY_DELAY_COUNT_MASK (0xf << 11) ++#define CSB_SHIFT 9 ++#define CSB_MASK (3 << 9) ++#define CSB_20MHZ (0 << 9) ++#define CSB_10MHZ (1 << 9) ++#define CSB_40MHZ (2 << 9) ++#define BANDGAP_MASK (1 << 8) ++#define BANDGAP_PNW_CIRCUIT (0 << 8) ++#define BANDGAP_LNC_CIRCUIT (1 << 8) ++#define MIPIC_FLISDSI_DELAY_COUNT_LOW_SHIFT 5 ++#define MIPIC_FLISDSI_DELAY_COUNT_LOW_MASK (7 << 5) ++#define TEARING_EFFECT_DELAY (1 << 4) /* A + C */ ++#define TEARING_EFFECT_SHIFT 2 /* A + C */ ++#define TEARING_EFFECT_MASK (3 << 2) ++#define TEARING_EFFECT_OFF (0 << 2) ++#define TEARING_EFFECT_DSI (1 << 2) ++#define TEARING_EFFECT_GPIO (2 << 2) ++#define LANE_CONFIGURATION_SHIFT 0 ++#define LANE_CONFIGURATION_MASK (3 << 0) ++#define LANE_CONFIGURATION_4LANE (0 << 0) ++#define LANE_CONFIGURATION_DUAL_LINK_A (1 << 0) ++#define LANE_CONFIGURATION_DUAL_LINK_B (2 << 0) ++ ++#define _MIPIA_TEARING_CTRL (VLV_DISPLAY_BASE + 0x61194) ++#define _MIPIC_TEARING_CTRL (VLV_DISPLAY_BASE + 0x61704) ++#define MIPI_TEARING_CTRL(port) _MMIO_MIPI(port, _MIPIA_TEARING_CTRL, _MIPIC_TEARING_CTRL) ++#define TEARING_EFFECT_DELAY_SHIFT 0 ++#define TEARING_EFFECT_DELAY_MASK (0xffff << 0) ++ ++/* XXX: all bits reserved */ ++#define _MIPIA_AUTOPWG (VLV_DISPLAY_BASE + 0x611a0) ++ ++/* MIPI DSI Controller and D-PHY registers */ ++ ++#define _MIPIA_DEVICE_READY (dev_priv->mipi_mmio_base + 0xb000) ++#define _MIPIC_DEVICE_READY (dev_priv->mipi_mmio_base + 0xb800) ++#define MIPI_DEVICE_READY(port) _MMIO_MIPI(port, _MIPIA_DEVICE_READY, _MIPIC_DEVICE_READY) ++#define BUS_POSSESSION (1 << 3) /* set to give bus to receiver */ ++#define ULPS_STATE_MASK (3 << 1) ++#define ULPS_STATE_ENTER (2 << 1) ++#define ULPS_STATE_EXIT (1 << 1) ++#define ULPS_STATE_NORMAL_OPERATION (0 << 1) ++#define DEVICE_READY (1 << 0) ++ ++#define _MIPIA_INTR_STAT (dev_priv->mipi_mmio_base + 0xb004) ++#define _MIPIC_INTR_STAT (dev_priv->mipi_mmio_base + 0xb804) ++#define MIPI_INTR_STAT(port) _MMIO_MIPI(port, _MIPIA_INTR_STAT, _MIPIC_INTR_STAT) ++#define _MIPIA_INTR_EN (dev_priv->mipi_mmio_base + 0xb008) ++#define _MIPIC_INTR_EN (dev_priv->mipi_mmio_base + 0xb808) ++#define MIPI_INTR_EN(port) _MMIO_MIPI(port, _MIPIA_INTR_EN, _MIPIC_INTR_EN) ++#define TEARING_EFFECT (1 << 31) ++#define SPL_PKT_SENT_INTERRUPT (1 << 30) ++#define GEN_READ_DATA_AVAIL (1 << 29) ++#define LP_GENERIC_WR_FIFO_FULL (1 << 28) ++#define HS_GENERIC_WR_FIFO_FULL (1 << 27) ++#define RX_PROT_VIOLATION (1 << 26) ++#define RX_INVALID_TX_LENGTH (1 << 25) ++#define ACK_WITH_NO_ERROR (1 << 24) ++#define TURN_AROUND_ACK_TIMEOUT (1 << 23) ++#define LP_RX_TIMEOUT (1 << 22) ++#define HS_TX_TIMEOUT (1 << 21) ++#define DPI_FIFO_UNDERRUN (1 << 20) ++#define LOW_CONTENTION (1 << 19) ++#define HIGH_CONTENTION (1 << 18) ++#define TXDSI_VC_ID_INVALID (1 << 17) ++#define TXDSI_DATA_TYPE_NOT_RECOGNISED (1 << 16) ++#define TXCHECKSUM_ERROR (1 << 15) ++#define TXECC_MULTIBIT_ERROR (1 << 14) ++#define TXECC_SINGLE_BIT_ERROR (1 << 13) ++#define TXFALSE_CONTROL_ERROR (1 << 12) ++#define RXDSI_VC_ID_INVALID (1 << 11) ++#define RXDSI_DATA_TYPE_NOT_REGOGNISED (1 << 10) ++#define RXCHECKSUM_ERROR (1 << 9) ++#define RXECC_MULTIBIT_ERROR (1 << 8) ++#define RXECC_SINGLE_BIT_ERROR (1 << 7) ++#define RXFALSE_CONTROL_ERROR (1 << 6) ++#define RXHS_RECEIVE_TIMEOUT_ERROR (1 << 5) ++#define RX_LP_TX_SYNC_ERROR (1 << 4) ++#define RXEXCAPE_MODE_ENTRY_ERROR (1 << 3) ++#define RXEOT_SYNC_ERROR (1 << 2) ++#define RXSOT_SYNC_ERROR (1 << 1) ++#define RXSOT_ERROR (1 << 0) ++ ++#define _MIPIA_DSI_FUNC_PRG (dev_priv->mipi_mmio_base + 0xb00c) ++#define _MIPIC_DSI_FUNC_PRG (dev_priv->mipi_mmio_base + 0xb80c) ++#define MIPI_DSI_FUNC_PRG(port) _MMIO_MIPI(port, _MIPIA_DSI_FUNC_PRG, _MIPIC_DSI_FUNC_PRG) ++#define CMD_MODE_DATA_WIDTH_MASK (7 << 13) ++#define CMD_MODE_NOT_SUPPORTED (0 << 13) ++#define CMD_MODE_DATA_WIDTH_16_BIT (1 << 13) ++#define CMD_MODE_DATA_WIDTH_9_BIT (2 << 13) ++#define CMD_MODE_DATA_WIDTH_8_BIT (3 << 13) ++#define CMD_MODE_DATA_WIDTH_OPTION1 (4 << 13) ++#define CMD_MODE_DATA_WIDTH_OPTION2 (5 << 13) ++#define VID_MODE_FORMAT_MASK (0xf << 7) ++#define VID_MODE_NOT_SUPPORTED (0 << 7) ++#define VID_MODE_FORMAT_RGB565 (1 << 7) ++#define VID_MODE_FORMAT_RGB666_PACKED (2 << 7) ++#define VID_MODE_FORMAT_RGB666 (3 << 7) ++#define VID_MODE_FORMAT_RGB888 (4 << 7) ++#define CMD_MODE_CHANNEL_NUMBER_SHIFT 5 ++#define CMD_MODE_CHANNEL_NUMBER_MASK (3 << 5) ++#define VID_MODE_CHANNEL_NUMBER_SHIFT 3 ++#define VID_MODE_CHANNEL_NUMBER_MASK (3 << 3) ++#define DATA_LANES_PRG_REG_SHIFT 0 ++#define DATA_LANES_PRG_REG_MASK (7 << 0) ++ ++#define _MIPIA_HS_TX_TIMEOUT (dev_priv->mipi_mmio_base + 0xb010) ++#define _MIPIC_HS_TX_TIMEOUT (dev_priv->mipi_mmio_base + 0xb810) ++#define MIPI_HS_TX_TIMEOUT(port) _MMIO_MIPI(port, _MIPIA_HS_TX_TIMEOUT, _MIPIC_HS_TX_TIMEOUT) ++#define HIGH_SPEED_TX_TIMEOUT_COUNTER_MASK 0xffffff ++ ++#define _MIPIA_LP_RX_TIMEOUT (dev_priv->mipi_mmio_base + 0xb014) ++#define _MIPIC_LP_RX_TIMEOUT (dev_priv->mipi_mmio_base + 0xb814) ++#define MIPI_LP_RX_TIMEOUT(port) _MMIO_MIPI(port, _MIPIA_LP_RX_TIMEOUT, _MIPIC_LP_RX_TIMEOUT) ++#define LOW_POWER_RX_TIMEOUT_COUNTER_MASK 0xffffff ++ ++#define _MIPIA_TURN_AROUND_TIMEOUT (dev_priv->mipi_mmio_base + 0xb018) ++#define _MIPIC_TURN_AROUND_TIMEOUT (dev_priv->mipi_mmio_base + 0xb818) ++#define MIPI_TURN_AROUND_TIMEOUT(port) _MMIO_MIPI(port, _MIPIA_TURN_AROUND_TIMEOUT, _MIPIC_TURN_AROUND_TIMEOUT) ++#define TURN_AROUND_TIMEOUT_MASK 0x3f ++ ++#define _MIPIA_DEVICE_RESET_TIMER (dev_priv->mipi_mmio_base + 0xb01c) ++#define _MIPIC_DEVICE_RESET_TIMER (dev_priv->mipi_mmio_base + 0xb81c) ++#define MIPI_DEVICE_RESET_TIMER(port) _MMIO_MIPI(port, _MIPIA_DEVICE_RESET_TIMER, _MIPIC_DEVICE_RESET_TIMER) ++#define DEVICE_RESET_TIMER_MASK 0xffff ++ ++#define _MIPIA_DPI_RESOLUTION (dev_priv->mipi_mmio_base + 0xb020) ++#define _MIPIC_DPI_RESOLUTION (dev_priv->mipi_mmio_base + 0xb820) ++#define MIPI_DPI_RESOLUTION(port) _MMIO_MIPI(port, _MIPIA_DPI_RESOLUTION, _MIPIC_DPI_RESOLUTION) ++#define VERTICAL_ADDRESS_SHIFT 16 ++#define VERTICAL_ADDRESS_MASK (0xffff << 16) ++#define HORIZONTAL_ADDRESS_SHIFT 0 ++#define HORIZONTAL_ADDRESS_MASK 0xffff ++ ++#define _MIPIA_DBI_FIFO_THROTTLE (dev_priv->mipi_mmio_base + 0xb024) ++#define _MIPIC_DBI_FIFO_THROTTLE (dev_priv->mipi_mmio_base + 0xb824) ++#define MIPI_DBI_FIFO_THROTTLE(port) _MMIO_MIPI(port, _MIPIA_DBI_FIFO_THROTTLE, _MIPIC_DBI_FIFO_THROTTLE) ++#define DBI_FIFO_EMPTY_HALF (0 << 0) ++#define DBI_FIFO_EMPTY_QUARTER (1 << 0) ++#define DBI_FIFO_EMPTY_7_LOCATIONS (2 << 0) ++ ++/* regs below are bits 15:0 */ ++#define _MIPIA_HSYNC_PADDING_COUNT (dev_priv->mipi_mmio_base + 0xb028) ++#define _MIPIC_HSYNC_PADDING_COUNT (dev_priv->mipi_mmio_base + 0xb828) ++#define MIPI_HSYNC_PADDING_COUNT(port) _MMIO_MIPI(port, _MIPIA_HSYNC_PADDING_COUNT, _MIPIC_HSYNC_PADDING_COUNT) ++ ++#define _MIPIA_HBP_COUNT (dev_priv->mipi_mmio_base + 0xb02c) ++#define _MIPIC_HBP_COUNT (dev_priv->mipi_mmio_base + 0xb82c) ++#define MIPI_HBP_COUNT(port) _MMIO_MIPI(port, _MIPIA_HBP_COUNT, _MIPIC_HBP_COUNT) ++ ++#define _MIPIA_HFP_COUNT (dev_priv->mipi_mmio_base + 0xb030) ++#define _MIPIC_HFP_COUNT (dev_priv->mipi_mmio_base + 0xb830) ++#define MIPI_HFP_COUNT(port) _MMIO_MIPI(port, _MIPIA_HFP_COUNT, _MIPIC_HFP_COUNT) ++ ++#define _MIPIA_HACTIVE_AREA_COUNT (dev_priv->mipi_mmio_base + 0xb034) ++#define _MIPIC_HACTIVE_AREA_COUNT (dev_priv->mipi_mmio_base + 0xb834) ++#define MIPI_HACTIVE_AREA_COUNT(port) _MMIO_MIPI(port, _MIPIA_HACTIVE_AREA_COUNT, _MIPIC_HACTIVE_AREA_COUNT) ++ ++#define _MIPIA_VSYNC_PADDING_COUNT (dev_priv->mipi_mmio_base + 0xb038) ++#define _MIPIC_VSYNC_PADDING_COUNT (dev_priv->mipi_mmio_base + 0xb838) ++#define MIPI_VSYNC_PADDING_COUNT(port) _MMIO_MIPI(port, _MIPIA_VSYNC_PADDING_COUNT, _MIPIC_VSYNC_PADDING_COUNT) ++ ++#define _MIPIA_VBP_COUNT (dev_priv->mipi_mmio_base + 0xb03c) ++#define _MIPIC_VBP_COUNT (dev_priv->mipi_mmio_base + 0xb83c) ++#define MIPI_VBP_COUNT(port) _MMIO_MIPI(port, _MIPIA_VBP_COUNT, _MIPIC_VBP_COUNT) ++ ++#define _MIPIA_VFP_COUNT (dev_priv->mipi_mmio_base + 0xb040) ++#define _MIPIC_VFP_COUNT (dev_priv->mipi_mmio_base + 0xb840) ++#define MIPI_VFP_COUNT(port) _MMIO_MIPI(port, _MIPIA_VFP_COUNT, _MIPIC_VFP_COUNT) ++ ++#define _MIPIA_HIGH_LOW_SWITCH_COUNT (dev_priv->mipi_mmio_base + 0xb044) ++#define _MIPIC_HIGH_LOW_SWITCH_COUNT (dev_priv->mipi_mmio_base + 0xb844) ++#define MIPI_HIGH_LOW_SWITCH_COUNT(port) _MMIO_MIPI(port, _MIPIA_HIGH_LOW_SWITCH_COUNT, _MIPIC_HIGH_LOW_SWITCH_COUNT) ++ ++/* regs above are bits 15:0 */ ++ ++#define _MIPIA_DPI_CONTROL (dev_priv->mipi_mmio_base + 0xb048) ++#define _MIPIC_DPI_CONTROL (dev_priv->mipi_mmio_base + 0xb848) ++#define MIPI_DPI_CONTROL(port) _MMIO_MIPI(port, _MIPIA_DPI_CONTROL, _MIPIC_DPI_CONTROL) ++#define DPI_LP_MODE (1 << 6) ++#define BACKLIGHT_OFF (1 << 5) ++#define BACKLIGHT_ON (1 << 4) ++#define COLOR_MODE_OFF (1 << 3) ++#define COLOR_MODE_ON (1 << 2) ++#define TURN_ON (1 << 1) ++#define SHUTDOWN (1 << 0) ++ ++#define _MIPIA_DPI_DATA (dev_priv->mipi_mmio_base + 0xb04c) ++#define _MIPIC_DPI_DATA (dev_priv->mipi_mmio_base + 0xb84c) ++#define MIPI_DPI_DATA(port) _MMIO_MIPI(port, _MIPIA_DPI_DATA, _MIPIC_DPI_DATA) ++#define COMMAND_BYTE_SHIFT 0 ++#define COMMAND_BYTE_MASK (0x3f << 0) ++ ++#define _MIPIA_INIT_COUNT (dev_priv->mipi_mmio_base + 0xb050) ++#define _MIPIC_INIT_COUNT (dev_priv->mipi_mmio_base + 0xb850) ++#define MIPI_INIT_COUNT(port) _MMIO_MIPI(port, _MIPIA_INIT_COUNT, _MIPIC_INIT_COUNT) ++#define MASTER_INIT_TIMER_SHIFT 0 ++#define MASTER_INIT_TIMER_MASK (0xffff << 0) ++ ++#define _MIPIA_MAX_RETURN_PKT_SIZE (dev_priv->mipi_mmio_base + 0xb054) ++#define _MIPIC_MAX_RETURN_PKT_SIZE (dev_priv->mipi_mmio_base + 0xb854) ++#define MIPI_MAX_RETURN_PKT_SIZE(port) _MMIO_MIPI(port, \ ++ _MIPIA_MAX_RETURN_PKT_SIZE, _MIPIC_MAX_RETURN_PKT_SIZE) ++#define MAX_RETURN_PKT_SIZE_SHIFT 0 ++#define MAX_RETURN_PKT_SIZE_MASK (0x3ff << 0) ++ ++#define _MIPIA_VIDEO_MODE_FORMAT (dev_priv->mipi_mmio_base + 0xb058) ++#define _MIPIC_VIDEO_MODE_FORMAT (dev_priv->mipi_mmio_base + 0xb858) ++#define MIPI_VIDEO_MODE_FORMAT(port) _MMIO_MIPI(port, _MIPIA_VIDEO_MODE_FORMAT, _MIPIC_VIDEO_MODE_FORMAT) ++#define RANDOM_DPI_DISPLAY_RESOLUTION (1 << 4) ++#define DISABLE_VIDEO_BTA (1 << 3) ++#define IP_TG_CONFIG (1 << 2) ++#define VIDEO_MODE_NON_BURST_WITH_SYNC_PULSE (1 << 0) ++#define VIDEO_MODE_NON_BURST_WITH_SYNC_EVENTS (2 << 0) ++#define VIDEO_MODE_BURST (3 << 0) ++ ++#define _MIPIA_EOT_DISABLE (dev_priv->mipi_mmio_base + 0xb05c) ++#define _MIPIC_EOT_DISABLE (dev_priv->mipi_mmio_base + 0xb85c) ++#define MIPI_EOT_DISABLE(port) _MMIO_MIPI(port, _MIPIA_EOT_DISABLE, _MIPIC_EOT_DISABLE) ++#define BXT_DEFEATURE_DPI_FIFO_CTR (1 << 9) ++#define BXT_DPHY_DEFEATURE_EN (1 << 8) ++#define LP_RX_TIMEOUT_ERROR_RECOVERY_DISABLE (1 << 7) ++#define HS_RX_TIMEOUT_ERROR_RECOVERY_DISABLE (1 << 6) ++#define LOW_CONTENTION_RECOVERY_DISABLE (1 << 5) ++#define HIGH_CONTENTION_RECOVERY_DISABLE (1 << 4) ++#define TXDSI_TYPE_NOT_RECOGNISED_ERROR_RECOVERY_DISABLE (1 << 3) ++#define TXECC_MULTIBIT_ERROR_RECOVERY_DISABLE (1 << 2) ++#define CLOCKSTOP (1 << 1) ++#define EOT_DISABLE (1 << 0) ++ ++#define _MIPIA_LP_BYTECLK (dev_priv->mipi_mmio_base + 0xb060) ++#define _MIPIC_LP_BYTECLK (dev_priv->mipi_mmio_base + 0xb860) ++#define MIPI_LP_BYTECLK(port) _MMIO_MIPI(port, _MIPIA_LP_BYTECLK, _MIPIC_LP_BYTECLK) ++#define LP_BYTECLK_SHIFT 0 ++#define LP_BYTECLK_MASK (0xffff << 0) ++ ++#define _MIPIA_TLPX_TIME_COUNT (dev_priv->mipi_mmio_base + 0xb0a4) ++#define _MIPIC_TLPX_TIME_COUNT (dev_priv->mipi_mmio_base + 0xb8a4) ++#define MIPI_TLPX_TIME_COUNT(port) _MMIO_MIPI(port, _MIPIA_TLPX_TIME_COUNT, _MIPIC_TLPX_TIME_COUNT) ++ ++#define _MIPIA_CLK_LANE_TIMING (dev_priv->mipi_mmio_base + 0xb098) ++#define _MIPIC_CLK_LANE_TIMING (dev_priv->mipi_mmio_base + 0xb898) ++#define MIPI_CLK_LANE_TIMING(port) _MMIO_MIPI(port, _MIPIA_CLK_LANE_TIMING, _MIPIC_CLK_LANE_TIMING) ++ ++/* bits 31:0 */ ++#define _MIPIA_LP_GEN_DATA (dev_priv->mipi_mmio_base + 0xb064) ++#define _MIPIC_LP_GEN_DATA (dev_priv->mipi_mmio_base + 0xb864) ++#define MIPI_LP_GEN_DATA(port) _MMIO_MIPI(port, _MIPIA_LP_GEN_DATA, _MIPIC_LP_GEN_DATA) ++ ++/* bits 31:0 */ ++#define _MIPIA_HS_GEN_DATA (dev_priv->mipi_mmio_base + 0xb068) ++#define _MIPIC_HS_GEN_DATA (dev_priv->mipi_mmio_base + 0xb868) ++#define MIPI_HS_GEN_DATA(port) _MMIO_MIPI(port, _MIPIA_HS_GEN_DATA, _MIPIC_HS_GEN_DATA) ++ ++#define _MIPIA_LP_GEN_CTRL (dev_priv->mipi_mmio_base + 0xb06c) ++#define _MIPIC_LP_GEN_CTRL (dev_priv->mipi_mmio_base + 0xb86c) ++#define MIPI_LP_GEN_CTRL(port) _MMIO_MIPI(port, _MIPIA_LP_GEN_CTRL, _MIPIC_LP_GEN_CTRL) ++#define _MIPIA_HS_GEN_CTRL (dev_priv->mipi_mmio_base + 0xb070) ++#define _MIPIC_HS_GEN_CTRL (dev_priv->mipi_mmio_base + 0xb870) ++#define MIPI_HS_GEN_CTRL(port) _MMIO_MIPI(port, _MIPIA_HS_GEN_CTRL, _MIPIC_HS_GEN_CTRL) ++#define LONG_PACKET_WORD_COUNT_SHIFT 8 ++#define LONG_PACKET_WORD_COUNT_MASK (0xffff << 8) ++#define SHORT_PACKET_PARAM_SHIFT 8 ++#define SHORT_PACKET_PARAM_MASK (0xffff << 8) ++#define VIRTUAL_CHANNEL_SHIFT 6 ++#define VIRTUAL_CHANNEL_MASK (3 << 6) ++#define DATA_TYPE_SHIFT 0 ++#define DATA_TYPE_MASK (0x3f << 0) ++/* data type values, see include/video/mipi_display.h */ ++ ++#define _MIPIA_GEN_FIFO_STAT (dev_priv->mipi_mmio_base + 0xb074) ++#define _MIPIC_GEN_FIFO_STAT (dev_priv->mipi_mmio_base + 0xb874) ++#define MIPI_GEN_FIFO_STAT(port) _MMIO_MIPI(port, _MIPIA_GEN_FIFO_STAT, _MIPIC_GEN_FIFO_STAT) ++#define DPI_FIFO_EMPTY (1 << 28) ++#define DBI_FIFO_EMPTY (1 << 27) ++#define LP_CTRL_FIFO_EMPTY (1 << 26) ++#define LP_CTRL_FIFO_HALF_EMPTY (1 << 25) ++#define LP_CTRL_FIFO_FULL (1 << 24) ++#define HS_CTRL_FIFO_EMPTY (1 << 18) ++#define HS_CTRL_FIFO_HALF_EMPTY (1 << 17) ++#define HS_CTRL_FIFO_FULL (1 << 16) ++#define LP_DATA_FIFO_EMPTY (1 << 10) ++#define LP_DATA_FIFO_HALF_EMPTY (1 << 9) ++#define LP_DATA_FIFO_FULL (1 << 8) ++#define HS_DATA_FIFO_EMPTY (1 << 2) ++#define HS_DATA_FIFO_HALF_EMPTY (1 << 1) ++#define HS_DATA_FIFO_FULL (1 << 0) ++ ++#define _MIPIA_HS_LS_DBI_ENABLE (dev_priv->mipi_mmio_base + 0xb078) ++#define _MIPIC_HS_LS_DBI_ENABLE (dev_priv->mipi_mmio_base + 0xb878) ++#define MIPI_HS_LP_DBI_ENABLE(port) _MMIO_MIPI(port, _MIPIA_HS_LS_DBI_ENABLE, _MIPIC_HS_LS_DBI_ENABLE) ++#define DBI_HS_LP_MODE_MASK (1 << 0) ++#define DBI_LP_MODE (1 << 0) ++#define DBI_HS_MODE (0 << 0) ++ ++#define _MIPIA_DPHY_PARAM (dev_priv->mipi_mmio_base + 0xb080) ++#define _MIPIC_DPHY_PARAM (dev_priv->mipi_mmio_base + 0xb880) ++#define MIPI_DPHY_PARAM(port) _MMIO_MIPI(port, _MIPIA_DPHY_PARAM, _MIPIC_DPHY_PARAM) ++#define EXIT_ZERO_COUNT_SHIFT 24 ++#define EXIT_ZERO_COUNT_MASK (0x3f << 24) ++#define TRAIL_COUNT_SHIFT 16 ++#define TRAIL_COUNT_MASK (0x1f << 16) ++#define CLK_ZERO_COUNT_SHIFT 8 ++#define CLK_ZERO_COUNT_MASK (0xff << 8) ++#define PREPARE_COUNT_SHIFT 0 ++#define PREPARE_COUNT_MASK (0x3f << 0) ++ ++#define _ICL_DSI_T_INIT_MASTER_0 0x6b088 ++#define _ICL_DSI_T_INIT_MASTER_1 0x6b888 ++#define ICL_DSI_T_INIT_MASTER(port) _MMIO_PORT(port, \ ++ _ICL_DSI_T_INIT_MASTER_0,\ ++ _ICL_DSI_T_INIT_MASTER_1) ++ ++#define _DPHY_CLK_TIMING_PARAM_0 0x162180 ++#define _DPHY_CLK_TIMING_PARAM_1 0x6c180 ++#define DPHY_CLK_TIMING_PARAM(port) _MMIO_PORT(port, \ ++ _DPHY_CLK_TIMING_PARAM_0,\ ++ _DPHY_CLK_TIMING_PARAM_1) ++#define _DSI_CLK_TIMING_PARAM_0 0x6b080 ++#define _DSI_CLK_TIMING_PARAM_1 0x6b880 ++#define DSI_CLK_TIMING_PARAM(port) _MMIO_PORT(port, \ ++ _DSI_CLK_TIMING_PARAM_0,\ ++ _DSI_CLK_TIMING_PARAM_1) ++#define CLK_PREPARE_OVERRIDE (1 << 31) ++#define CLK_PREPARE(x) ((x) << 28) ++#define CLK_PREPARE_MASK (0x7 << 28) ++#define CLK_PREPARE_SHIFT 28 ++#define CLK_ZERO_OVERRIDE (1 << 27) ++#define CLK_ZERO(x) ((x) << 20) ++#define CLK_ZERO_MASK (0xf << 20) ++#define CLK_ZERO_SHIFT 20 ++#define CLK_PRE_OVERRIDE (1 << 19) ++#define CLK_PRE(x) ((x) << 16) ++#define CLK_PRE_MASK (0x3 << 16) ++#define CLK_PRE_SHIFT 16 ++#define CLK_POST_OVERRIDE (1 << 15) ++#define CLK_POST(x) ((x) << 8) ++#define CLK_POST_MASK (0x7 << 8) ++#define CLK_POST_SHIFT 8 ++#define CLK_TRAIL_OVERRIDE (1 << 7) ++#define CLK_TRAIL(x) ((x) << 0) ++#define CLK_TRAIL_MASK (0xf << 0) ++#define CLK_TRAIL_SHIFT 0 ++ ++#define _DPHY_DATA_TIMING_PARAM_0 0x162184 ++#define _DPHY_DATA_TIMING_PARAM_1 0x6c184 ++#define DPHY_DATA_TIMING_PARAM(port) _MMIO_PORT(port, \ ++ _DPHY_DATA_TIMING_PARAM_0,\ ++ _DPHY_DATA_TIMING_PARAM_1) ++#define _DSI_DATA_TIMING_PARAM_0 0x6B084 ++#define _DSI_DATA_TIMING_PARAM_1 0x6B884 ++#define DSI_DATA_TIMING_PARAM(port) _MMIO_PORT(port, \ ++ _DSI_DATA_TIMING_PARAM_0,\ ++ _DSI_DATA_TIMING_PARAM_1) ++#define HS_PREPARE_OVERRIDE (1 << 31) ++#define HS_PREPARE(x) ((x) << 24) ++#define HS_PREPARE_MASK (0x7 << 24) ++#define HS_PREPARE_SHIFT 24 ++#define HS_ZERO_OVERRIDE (1 << 23) ++#define HS_ZERO(x) ((x) << 16) ++#define HS_ZERO_MASK (0xf << 16) ++#define HS_ZERO_SHIFT 16 ++#define HS_TRAIL_OVERRIDE (1 << 15) ++#define HS_TRAIL(x) ((x) << 8) ++#define HS_TRAIL_MASK (0x7 << 8) ++#define HS_TRAIL_SHIFT 8 ++#define HS_EXIT_OVERRIDE (1 << 7) ++#define HS_EXIT(x) ((x) << 0) ++#define HS_EXIT_MASK (0x7 << 0) ++#define HS_EXIT_SHIFT 0 ++ ++#define _DPHY_TA_TIMING_PARAM_0 0x162188 ++#define _DPHY_TA_TIMING_PARAM_1 0x6c188 ++#define DPHY_TA_TIMING_PARAM(port) _MMIO_PORT(port, \ ++ _DPHY_TA_TIMING_PARAM_0,\ ++ _DPHY_TA_TIMING_PARAM_1) ++#define _DSI_TA_TIMING_PARAM_0 0x6b098 ++#define _DSI_TA_TIMING_PARAM_1 0x6b898 ++#define DSI_TA_TIMING_PARAM(port) _MMIO_PORT(port, \ ++ _DSI_TA_TIMING_PARAM_0,\ ++ _DSI_TA_TIMING_PARAM_1) ++#define TA_SURE_OVERRIDE (1 << 31) ++#define TA_SURE(x) ((x) << 16) ++#define TA_SURE_MASK (0x1f << 16) ++#define TA_SURE_SHIFT 16 ++#define TA_GO_OVERRIDE (1 << 15) ++#define TA_GO(x) ((x) << 8) ++#define TA_GO_MASK (0xf << 8) ++#define TA_GO_SHIFT 8 ++#define TA_GET_OVERRIDE (1 << 7) ++#define TA_GET(x) ((x) << 0) ++#define TA_GET_MASK (0xf << 0) ++#define TA_GET_SHIFT 0 ++ ++/* DSI transcoder configuration */ ++#define _DSI_TRANS_FUNC_CONF_0 0x6b030 ++#define _DSI_TRANS_FUNC_CONF_1 0x6b830 ++#define DSI_TRANS_FUNC_CONF(tc) _MMIO_DSI(tc, \ ++ _DSI_TRANS_FUNC_CONF_0,\ ++ _DSI_TRANS_FUNC_CONF_1) ++#define OP_MODE_MASK (0x3 << 28) ++#define OP_MODE_SHIFT 28 ++#define CMD_MODE_NO_GATE (0x0 << 28) ++#define CMD_MODE_TE_GATE (0x1 << 28) ++#define VIDEO_MODE_SYNC_EVENT (0x2 << 28) ++#define VIDEO_MODE_SYNC_PULSE (0x3 << 28) ++#define LINK_READY (1 << 20) ++#define PIX_FMT_MASK (0x3 << 16) ++#define PIX_FMT_SHIFT 16 ++#define PIX_FMT_RGB565 (0x0 << 16) ++#define PIX_FMT_RGB666_PACKED (0x1 << 16) ++#define PIX_FMT_RGB666_LOOSE (0x2 << 16) ++#define PIX_FMT_RGB888 (0x3 << 16) ++#define PIX_FMT_RGB101010 (0x4 << 16) ++#define PIX_FMT_RGB121212 (0x5 << 16) ++#define PIX_FMT_COMPRESSED (0x6 << 16) ++#define BGR_TRANSMISSION (1 << 15) ++#define PIX_VIRT_CHAN(x) ((x) << 12) ++#define PIX_VIRT_CHAN_MASK (0x3 << 12) ++#define PIX_VIRT_CHAN_SHIFT 12 ++#define PIX_BUF_THRESHOLD_MASK (0x3 << 10) ++#define PIX_BUF_THRESHOLD_SHIFT 10 ++#define PIX_BUF_THRESHOLD_1_4 (0x0 << 10) ++#define PIX_BUF_THRESHOLD_1_2 (0x1 << 10) ++#define PIX_BUF_THRESHOLD_3_4 (0x2 << 10) ++#define PIX_BUF_THRESHOLD_FULL (0x3 << 10) ++#define CONTINUOUS_CLK_MASK (0x3 << 8) ++#define CONTINUOUS_CLK_SHIFT 8 ++#define CLK_ENTER_LP_AFTER_DATA (0x0 << 8) ++#define CLK_HS_OR_LP (0x2 << 8) ++#define CLK_HS_CONTINUOUS (0x3 << 8) ++#define LINK_CALIBRATION_MASK (0x3 << 4) ++#define LINK_CALIBRATION_SHIFT 4 ++#define CALIBRATION_DISABLED (0x0 << 4) ++#define CALIBRATION_ENABLED_INITIAL_ONLY (0x2 << 4) ++#define CALIBRATION_ENABLED_INITIAL_PERIODIC (0x3 << 4) ++#define S3D_ORIENTATION_LANDSCAPE (1 << 1) ++#define EOTP_DISABLED (1 << 0) ++ ++#define _DSI_CMD_RXCTL_0 0x6b0d4 ++#define _DSI_CMD_RXCTL_1 0x6b8d4 ++#define DSI_CMD_RXCTL(tc) _MMIO_DSI(tc, \ ++ _DSI_CMD_RXCTL_0,\ ++ _DSI_CMD_RXCTL_1) ++#define READ_UNLOADS_DW (1 << 16) ++#define RECEIVED_UNASSIGNED_TRIGGER (1 << 15) ++#define RECEIVED_ACKNOWLEDGE_TRIGGER (1 << 14) ++#define RECEIVED_TEAR_EFFECT_TRIGGER (1 << 13) ++#define RECEIVED_RESET_TRIGGER (1 << 12) ++#define RECEIVED_PAYLOAD_WAS_LOST (1 << 11) ++#define RECEIVED_CRC_WAS_LOST (1 << 10) ++#define NUMBER_RX_PLOAD_DW_MASK (0xff << 0) ++#define NUMBER_RX_PLOAD_DW_SHIFT 0 ++ ++#define _DSI_CMD_TXCTL_0 0x6b0d0 ++#define _DSI_CMD_TXCTL_1 0x6b8d0 ++#define DSI_CMD_TXCTL(tc) _MMIO_DSI(tc, \ ++ _DSI_CMD_TXCTL_0,\ ++ _DSI_CMD_TXCTL_1) ++#define KEEP_LINK_IN_HS (1 << 24) ++#define FREE_HEADER_CREDIT_MASK (0x1f << 8) ++#define FREE_HEADER_CREDIT_SHIFT 0x8 ++#define FREE_PLOAD_CREDIT_MASK (0xff << 0) ++#define FREE_PLOAD_CREDIT_SHIFT 0 ++#define MAX_HEADER_CREDIT 0x10 ++#define MAX_PLOAD_CREDIT 0x40 ++ ++#define _DSI_CMD_TXHDR_0 0x6b100 ++#define _DSI_CMD_TXHDR_1 0x6b900 ++#define DSI_CMD_TXHDR(tc) _MMIO_DSI(tc, \ ++ _DSI_CMD_TXHDR_0,\ ++ _DSI_CMD_TXHDR_1) ++#define PAYLOAD_PRESENT (1 << 31) ++#define LP_DATA_TRANSFER (1 << 30) ++#define VBLANK_FENCE (1 << 29) ++#define PARAM_WC_MASK (0xffff << 8) ++#define PARAM_WC_LOWER_SHIFT 8 ++#define PARAM_WC_UPPER_SHIFT 16 ++#define VC_MASK (0x3 << 6) ++#define VC_SHIFT 6 ++#define DT_MASK (0x3f << 0) ++#define DT_SHIFT 0 ++ ++#define _DSI_CMD_TXPYLD_0 0x6b104 ++#define _DSI_CMD_TXPYLD_1 0x6b904 ++#define DSI_CMD_TXPYLD(tc) _MMIO_DSI(tc, \ ++ _DSI_CMD_TXPYLD_0,\ ++ _DSI_CMD_TXPYLD_1) ++ ++#define _DSI_LP_MSG_0 0x6b0d8 ++#define _DSI_LP_MSG_1 0x6b8d8 ++#define DSI_LP_MSG(tc) _MMIO_DSI(tc, \ ++ _DSI_LP_MSG_0,\ ++ _DSI_LP_MSG_1) ++#define LPTX_IN_PROGRESS (1 << 17) ++#define LINK_IN_ULPS (1 << 16) ++#define LINK_ULPS_TYPE_LP11 (1 << 8) ++#define LINK_ENTER_ULPS (1 << 0) ++ ++/* DSI timeout registers */ ++#define _DSI_HSTX_TO_0 0x6b044 ++#define _DSI_HSTX_TO_1 0x6b844 ++#define DSI_HSTX_TO(tc) _MMIO_DSI(tc, \ ++ _DSI_HSTX_TO_0,\ ++ _DSI_HSTX_TO_1) ++#define HSTX_TIMEOUT_VALUE_MASK (0xffff << 16) ++#define HSTX_TIMEOUT_VALUE_SHIFT 16 ++#define HSTX_TIMEOUT_VALUE(x) ((x) << 16) ++#define HSTX_TIMED_OUT (1 << 0) ++ ++#define _DSI_LPRX_HOST_TO_0 0x6b048 ++#define _DSI_LPRX_HOST_TO_1 0x6b848 ++#define DSI_LPRX_HOST_TO(tc) _MMIO_DSI(tc, \ ++ _DSI_LPRX_HOST_TO_0,\ ++ _DSI_LPRX_HOST_TO_1) ++#define LPRX_TIMED_OUT (1 << 16) ++#define LPRX_TIMEOUT_VALUE_MASK (0xffff << 0) ++#define LPRX_TIMEOUT_VALUE_SHIFT 0 ++#define LPRX_TIMEOUT_VALUE(x) ((x) << 0) ++ ++#define _DSI_PWAIT_TO_0 0x6b040 ++#define _DSI_PWAIT_TO_1 0x6b840 ++#define DSI_PWAIT_TO(tc) _MMIO_DSI(tc, \ ++ _DSI_PWAIT_TO_0,\ ++ _DSI_PWAIT_TO_1) ++#define PRESET_TIMEOUT_VALUE_MASK (0xffff << 16) ++#define PRESET_TIMEOUT_VALUE_SHIFT 16 ++#define PRESET_TIMEOUT_VALUE(x) ((x) << 16) ++#define PRESPONSE_TIMEOUT_VALUE_MASK (0xffff << 0) ++#define PRESPONSE_TIMEOUT_VALUE_SHIFT 0 ++#define PRESPONSE_TIMEOUT_VALUE(x) ((x) << 0) ++ ++#define _DSI_TA_TO_0 0x6b04c ++#define _DSI_TA_TO_1 0x6b84c ++#define DSI_TA_TO(tc) _MMIO_DSI(tc, \ ++ _DSI_TA_TO_0,\ ++ _DSI_TA_TO_1) ++#define TA_TIMED_OUT (1 << 16) ++#define TA_TIMEOUT_VALUE_MASK (0xffff << 0) ++#define TA_TIMEOUT_VALUE_SHIFT 0 ++#define TA_TIMEOUT_VALUE(x) ((x) << 0) ++ ++/* bits 31:0 */ ++#define _MIPIA_DBI_BW_CTRL (dev_priv->mipi_mmio_base + 0xb084) ++#define _MIPIC_DBI_BW_CTRL (dev_priv->mipi_mmio_base + 0xb884) ++#define MIPI_DBI_BW_CTRL(port) _MMIO_MIPI(port, _MIPIA_DBI_BW_CTRL, _MIPIC_DBI_BW_CTRL) ++ ++#define _MIPIA_CLK_LANE_SWITCH_TIME_CNT (dev_priv->mipi_mmio_base + 0xb088) ++#define _MIPIC_CLK_LANE_SWITCH_TIME_CNT (dev_priv->mipi_mmio_base + 0xb888) ++#define MIPI_CLK_LANE_SWITCH_TIME_CNT(port) _MMIO_MIPI(port, _MIPIA_CLK_LANE_SWITCH_TIME_CNT, _MIPIC_CLK_LANE_SWITCH_TIME_CNT) ++#define LP_HS_SSW_CNT_SHIFT 16 ++#define LP_HS_SSW_CNT_MASK (0xffff << 16) ++#define HS_LP_PWR_SW_CNT_SHIFT 0 ++#define HS_LP_PWR_SW_CNT_MASK (0xffff << 0) ++ ++#define _MIPIA_STOP_STATE_STALL (dev_priv->mipi_mmio_base + 0xb08c) ++#define _MIPIC_STOP_STATE_STALL (dev_priv->mipi_mmio_base + 0xb88c) ++#define MIPI_STOP_STATE_STALL(port) _MMIO_MIPI(port, _MIPIA_STOP_STATE_STALL, _MIPIC_STOP_STATE_STALL) ++#define STOP_STATE_STALL_COUNTER_SHIFT 0 ++#define STOP_STATE_STALL_COUNTER_MASK (0xff << 0) ++ ++#define _MIPIA_INTR_STAT_REG_1 (dev_priv->mipi_mmio_base + 0xb090) ++#define _MIPIC_INTR_STAT_REG_1 (dev_priv->mipi_mmio_base + 0xb890) ++#define MIPI_INTR_STAT_REG_1(port) _MMIO_MIPI(port, _MIPIA_INTR_STAT_REG_1, _MIPIC_INTR_STAT_REG_1) ++#define _MIPIA_INTR_EN_REG_1 (dev_priv->mipi_mmio_base + 0xb094) ++#define _MIPIC_INTR_EN_REG_1 (dev_priv->mipi_mmio_base + 0xb894) ++#define MIPI_INTR_EN_REG_1(port) _MMIO_MIPI(port, _MIPIA_INTR_EN_REG_1, _MIPIC_INTR_EN_REG_1) ++#define RX_CONTENTION_DETECTED (1 << 0) ++ ++/* XXX: only pipe A ?!? */ ++#define MIPIA_DBI_TYPEC_CTRL (dev_priv->mipi_mmio_base + 0xb100) ++#define DBI_TYPEC_ENABLE (1 << 31) ++#define DBI_TYPEC_WIP (1 << 30) ++#define DBI_TYPEC_OPTION_SHIFT 28 ++#define DBI_TYPEC_OPTION_MASK (3 << 28) ++#define DBI_TYPEC_FREQ_SHIFT 24 ++#define DBI_TYPEC_FREQ_MASK (0xf << 24) ++#define DBI_TYPEC_OVERRIDE (1 << 8) ++#define DBI_TYPEC_OVERRIDE_COUNTER_SHIFT 0 ++#define DBI_TYPEC_OVERRIDE_COUNTER_MASK (0xff << 0) ++ ++ ++/* MIPI adapter registers */ ++ ++#define _MIPIA_CTRL (dev_priv->mipi_mmio_base + 0xb104) ++#define _MIPIC_CTRL (dev_priv->mipi_mmio_base + 0xb904) ++#define MIPI_CTRL(port) _MMIO_MIPI(port, _MIPIA_CTRL, _MIPIC_CTRL) ++#define ESCAPE_CLOCK_DIVIDER_SHIFT 5 /* A only */ ++#define ESCAPE_CLOCK_DIVIDER_MASK (3 << 5) ++#define ESCAPE_CLOCK_DIVIDER_1 (0 << 5) ++#define ESCAPE_CLOCK_DIVIDER_2 (1 << 5) ++#define ESCAPE_CLOCK_DIVIDER_4 (2 << 5) ++#define READ_REQUEST_PRIORITY_SHIFT 3 ++#define READ_REQUEST_PRIORITY_MASK (3 << 3) ++#define READ_REQUEST_PRIORITY_LOW (0 << 3) ++#define READ_REQUEST_PRIORITY_HIGH (3 << 3) ++#define RGB_FLIP_TO_BGR (1 << 2) ++ ++#define BXT_PIPE_SELECT_SHIFT 7 ++#define BXT_PIPE_SELECT_MASK (7 << 7) ++#define BXT_PIPE_SELECT(pipe) ((pipe) << 7) ++#define GLK_PHY_STATUS_PORT_READY (1 << 31) /* RO */ ++#define GLK_ULPS_NOT_ACTIVE (1 << 30) /* RO */ ++#define GLK_MIPIIO_RESET_RELEASED (1 << 28) ++#define GLK_CLOCK_LANE_STOP_STATE (1 << 27) /* RO */ ++#define GLK_DATA_LANE_STOP_STATE (1 << 26) /* RO */ ++#define GLK_LP_WAKE (1 << 22) ++#define GLK_LP11_LOW_PWR_MODE (1 << 21) ++#define GLK_LP00_LOW_PWR_MODE (1 << 20) ++#define GLK_FIREWALL_ENABLE (1 << 16) ++#define BXT_PIXEL_OVERLAP_CNT_MASK (0xf << 10) ++#define BXT_PIXEL_OVERLAP_CNT_SHIFT 10 ++#define BXT_DSC_ENABLE (1 << 3) ++#define BXT_RGB_FLIP (1 << 2) ++#define GLK_MIPIIO_PORT_POWERED (1 << 1) /* RO */ ++#define GLK_MIPIIO_ENABLE (1 << 0) ++ ++#define _MIPIA_DATA_ADDRESS (dev_priv->mipi_mmio_base + 0xb108) ++#define _MIPIC_DATA_ADDRESS (dev_priv->mipi_mmio_base + 0xb908) ++#define MIPI_DATA_ADDRESS(port) _MMIO_MIPI(port, _MIPIA_DATA_ADDRESS, _MIPIC_DATA_ADDRESS) ++#define DATA_MEM_ADDRESS_SHIFT 5 ++#define DATA_MEM_ADDRESS_MASK (0x7ffffff << 5) ++#define DATA_VALID (1 << 0) ++ ++#define _MIPIA_DATA_LENGTH (dev_priv->mipi_mmio_base + 0xb10c) ++#define _MIPIC_DATA_LENGTH (dev_priv->mipi_mmio_base + 0xb90c) ++#define MIPI_DATA_LENGTH(port) _MMIO_MIPI(port, _MIPIA_DATA_LENGTH, _MIPIC_DATA_LENGTH) ++#define DATA_LENGTH_SHIFT 0 ++#define DATA_LENGTH_MASK (0xfffff << 0) ++ ++#define _MIPIA_COMMAND_ADDRESS (dev_priv->mipi_mmio_base + 0xb110) ++#define _MIPIC_COMMAND_ADDRESS (dev_priv->mipi_mmio_base + 0xb910) ++#define MIPI_COMMAND_ADDRESS(port) _MMIO_MIPI(port, _MIPIA_COMMAND_ADDRESS, _MIPIC_COMMAND_ADDRESS) ++#define COMMAND_MEM_ADDRESS_SHIFT 5 ++#define COMMAND_MEM_ADDRESS_MASK (0x7ffffff << 5) ++#define AUTO_PWG_ENABLE (1 << 2) ++#define MEMORY_WRITE_DATA_FROM_PIPE_RENDERING (1 << 1) ++#define COMMAND_VALID (1 << 0) ++ ++#define _MIPIA_COMMAND_LENGTH (dev_priv->mipi_mmio_base + 0xb114) ++#define _MIPIC_COMMAND_LENGTH (dev_priv->mipi_mmio_base + 0xb914) ++#define MIPI_COMMAND_LENGTH(port) _MMIO_MIPI(port, _MIPIA_COMMAND_LENGTH, _MIPIC_COMMAND_LENGTH) ++#define COMMAND_LENGTH_SHIFT(n) (8 * (n)) /* n: 0...3 */ ++#define COMMAND_LENGTH_MASK(n) (0xff << (8 * (n))) ++ ++#define _MIPIA_READ_DATA_RETURN0 (dev_priv->mipi_mmio_base + 0xb118) ++#define _MIPIC_READ_DATA_RETURN0 (dev_priv->mipi_mmio_base + 0xb918) ++#define MIPI_READ_DATA_RETURN(port, n) _MMIO(_MIPI(port, _MIPIA_READ_DATA_RETURN0, _MIPIC_READ_DATA_RETURN0) + 4 * (n)) /* n: 0...7 */ ++ ++#define _MIPIA_READ_DATA_VALID (dev_priv->mipi_mmio_base + 0xb138) ++#define _MIPIC_READ_DATA_VALID (dev_priv->mipi_mmio_base + 0xb938) ++#define MIPI_READ_DATA_VALID(port) _MMIO_MIPI(port, _MIPIA_READ_DATA_VALID, _MIPIC_READ_DATA_VALID) ++#define READ_DATA_VALID(n) (1 << (n)) ++ ++/* MOCS (Memory Object Control State) registers */ ++#define GEN9_LNCFCMOCS(i) _MMIO(0xb020 + (i) * 4) /* L3 Cache Control */ ++ ++#define GEN9_GFX_MOCS(i) _MMIO(0xc800 + (i) * 4) /* Graphics MOCS registers */ ++#define GEN9_MFX0_MOCS(i) _MMIO(0xc900 + (i) * 4) /* Media 0 MOCS registers */ ++#define GEN9_MFX1_MOCS(i) _MMIO(0xca00 + (i) * 4) /* Media 1 MOCS registers */ ++#define GEN9_VEBOX_MOCS(i) _MMIO(0xcb00 + (i) * 4) /* Video MOCS registers */ ++#define GEN9_BLT_MOCS(i) _MMIO(0xcc00 + (i) * 4) /* Blitter MOCS registers */ ++/* Media decoder 2 MOCS registers */ ++#define GEN11_MFX2_MOCS(i) _MMIO(0x10000 + (i) * 4) ++ ++#define GEN10_SCRATCH_LNCF2 _MMIO(0xb0a0) ++#define PMFLUSHDONE_LNICRSDROP (1 << 20) ++#define PMFLUSH_GAPL3UNBLOCK (1 << 21) ++#define PMFLUSHDONE_LNEBLK (1 << 22) ++ ++/* gamt regs */ ++#define GEN8_L3_LRA_1_GPGPU _MMIO(0x4dd4) ++#define GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_BDW 0x67F1427F /* max/min for LRA1/2 */ ++#define GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_CHV 0x5FF101FF /* max/min for LRA1/2 */ ++#define GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_SKL 0x67F1427F /* " " */ ++#define GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT 0x5FF101FF /* " " */ ++ ++#define MMCD_MISC_CTRL _MMIO(0x4ddc) /* skl+ */ ++#define MMCD_PCLA (1 << 31) ++#define MMCD_HOTSPOT_EN (1 << 27) ++ ++#define _ICL_PHY_MISC_A 0x64C00 ++#define _ICL_PHY_MISC_B 0x64C04 ++#define ICL_PHY_MISC(port) _MMIO_PORT(port, _ICL_PHY_MISC_A, \ ++ _ICL_PHY_MISC_B) ++#define ICL_PHY_MISC_DE_IO_COMP_PWR_DOWN (1 << 23) ++ ++/* Icelake Display Stream Compression Registers */ ++#define DSCA_PICTURE_PARAMETER_SET_0 _MMIO(0x6B200) ++#define DSCC_PICTURE_PARAMETER_SET_0 _MMIO(0x6BA00) ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_0_PB 0x78270 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_0_PB 0x78370 ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_0_PC 0x78470 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_0_PC 0x78570 ++#define ICL_DSC0_PICTURE_PARAMETER_SET_0(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_0_PB, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_0_PC) ++#define ICL_DSC1_PICTURE_PARAMETER_SET_0(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_0_PB, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_0_PC) ++#define DSC_VBR_ENABLE (1 << 19) ++#define DSC_422_ENABLE (1 << 18) ++#define DSC_COLOR_SPACE_CONVERSION (1 << 17) ++#define DSC_BLOCK_PREDICTION (1 << 16) ++#define DSC_LINE_BUF_DEPTH_SHIFT 12 ++#define DSC_BPC_SHIFT 8 ++#define DSC_VER_MIN_SHIFT 4 ++#define DSC_VER_MAJ (0x1 << 0) ++ ++#define DSCA_PICTURE_PARAMETER_SET_1 _MMIO(0x6B204) ++#define DSCC_PICTURE_PARAMETER_SET_1 _MMIO(0x6BA04) ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_1_PB 0x78274 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_1_PB 0x78374 ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_1_PC 0x78474 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_1_PC 0x78574 ++#define ICL_DSC0_PICTURE_PARAMETER_SET_1(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_1_PB, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_1_PC) ++#define ICL_DSC1_PICTURE_PARAMETER_SET_1(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_1_PB, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_1_PC) ++#define DSC_BPP(bpp) ((bpp) << 0) ++ ++#define DSCA_PICTURE_PARAMETER_SET_2 _MMIO(0x6B208) ++#define DSCC_PICTURE_PARAMETER_SET_2 _MMIO(0x6BA08) ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_2_PB 0x78278 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_2_PB 0x78378 ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_2_PC 0x78478 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_2_PC 0x78578 ++#define ICL_DSC0_PICTURE_PARAMETER_SET_2(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_2_PB, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_2_PC) ++#define ICL_DSC1_PICTURE_PARAMETER_SET_2(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_2_PB, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_2_PC) ++#define DSC_PIC_WIDTH(pic_width) ((pic_width) << 16) ++#define DSC_PIC_HEIGHT(pic_height) ((pic_height) << 0) ++ ++#define DSCA_PICTURE_PARAMETER_SET_3 _MMIO(0x6B20C) ++#define DSCC_PICTURE_PARAMETER_SET_3 _MMIO(0x6BA0C) ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_3_PB 0x7827C ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_3_PB 0x7837C ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_3_PC 0x7847C ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_3_PC 0x7857C ++#define ICL_DSC0_PICTURE_PARAMETER_SET_3(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_3_PB, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_3_PC) ++#define ICL_DSC1_PICTURE_PARAMETER_SET_3(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_3_PB, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_3_PC) ++#define DSC_SLICE_WIDTH(slice_width) ((slice_width) << 16) ++#define DSC_SLICE_HEIGHT(slice_height) ((slice_height) << 0) ++ ++#define DSCA_PICTURE_PARAMETER_SET_4 _MMIO(0x6B210) ++#define DSCC_PICTURE_PARAMETER_SET_4 _MMIO(0x6BA10) ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_4_PB 0x78280 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_4_PB 0x78380 ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_4_PC 0x78480 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_4_PC 0x78580 ++#define ICL_DSC0_PICTURE_PARAMETER_SET_4(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_4_PB, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_4_PC) ++#define ICL_DSC1_PICTURE_PARAMETER_SET_4(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_4_PB, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_4_PC) ++#define DSC_INITIAL_DEC_DELAY(dec_delay) ((dec_delay) << 16) ++#define DSC_INITIAL_XMIT_DELAY(xmit_delay) ((xmit_delay) << 0) ++ ++#define DSCA_PICTURE_PARAMETER_SET_5 _MMIO(0x6B214) ++#define DSCC_PICTURE_PARAMETER_SET_5 _MMIO(0x6BA14) ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_5_PB 0x78284 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_5_PB 0x78384 ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_5_PC 0x78484 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_5_PC 0x78584 ++#define ICL_DSC0_PICTURE_PARAMETER_SET_5(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_5_PB, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_5_PC) ++#define ICL_DSC1_PICTURE_PARAMETER_SET_5(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_5_PB, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_5_PC) ++#define DSC_SCALE_DEC_INT(scale_dec) ((scale_dec) << 16) ++#define DSC_SCALE_INC_INT(scale_inc) ((scale_inc) << 0) ++ ++#define DSCA_PICTURE_PARAMETER_SET_6 _MMIO(0x6B218) ++#define DSCC_PICTURE_PARAMETER_SET_6 _MMIO(0x6BA18) ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_6_PB 0x78288 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_6_PB 0x78388 ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_6_PC 0x78488 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_6_PC 0x78588 ++#define ICL_DSC0_PICTURE_PARAMETER_SET_6(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_6_PB, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_6_PC) ++#define ICL_DSC1_PICTURE_PARAMETER_SET_6(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_6_PB, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_6_PC) ++#define DSC_FLATNESS_MAX_QP(max_qp) ((max_qp) << 24) ++#define DSC_FLATNESS_MIN_QP(min_qp) ((min_qp) << 16) ++#define DSC_FIRST_LINE_BPG_OFFSET(offset) ((offset) << 8) ++#define DSC_INITIAL_SCALE_VALUE(value) ((value) << 0) ++ ++#define DSCA_PICTURE_PARAMETER_SET_7 _MMIO(0x6B21C) ++#define DSCC_PICTURE_PARAMETER_SET_7 _MMIO(0x6BA1C) ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_7_PB 0x7828C ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_7_PB 0x7838C ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_7_PC 0x7848C ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_7_PC 0x7858C ++#define ICL_DSC0_PICTURE_PARAMETER_SET_7(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_7_PB, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_7_PC) ++#define ICL_DSC1_PICTURE_PARAMETER_SET_7(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_7_PB, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_7_PC) ++#define DSC_NFL_BPG_OFFSET(bpg_offset) ((bpg_offset) << 16) ++#define DSC_SLICE_BPG_OFFSET(bpg_offset) ((bpg_offset) << 0) ++ ++#define DSCA_PICTURE_PARAMETER_SET_8 _MMIO(0x6B220) ++#define DSCC_PICTURE_PARAMETER_SET_8 _MMIO(0x6BA20) ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_8_PB 0x78290 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_8_PB 0x78390 ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_8_PC 0x78490 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_8_PC 0x78590 ++#define ICL_DSC0_PICTURE_PARAMETER_SET_8(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_8_PB, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_8_PC) ++#define ICL_DSC1_PICTURE_PARAMETER_SET_8(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_8_PB, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_8_PC) ++#define DSC_INITIAL_OFFSET(initial_offset) ((initial_offset) << 16) ++#define DSC_FINAL_OFFSET(final_offset) ((final_offset) << 0) ++ ++#define DSCA_PICTURE_PARAMETER_SET_9 _MMIO(0x6B224) ++#define DSCC_PICTURE_PARAMETER_SET_9 _MMIO(0x6BA24) ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_9_PB 0x78294 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_9_PB 0x78394 ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_9_PC 0x78494 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_9_PC 0x78594 ++#define ICL_DSC0_PICTURE_PARAMETER_SET_9(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_9_PB, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_9_PC) ++#define ICL_DSC1_PICTURE_PARAMETER_SET_9(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_9_PB, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_9_PC) ++#define DSC_RC_EDGE_FACTOR(rc_edge_fact) ((rc_edge_fact) << 16) ++#define DSC_RC_MODEL_SIZE(rc_model_size) ((rc_model_size) << 0) ++ ++#define DSCA_PICTURE_PARAMETER_SET_10 _MMIO(0x6B228) ++#define DSCC_PICTURE_PARAMETER_SET_10 _MMIO(0x6BA28) ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_10_PB 0x78298 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_10_PB 0x78398 ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_10_PC 0x78498 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_10_PC 0x78598 ++#define ICL_DSC0_PICTURE_PARAMETER_SET_10(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_10_PB, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_10_PC) ++#define ICL_DSC1_PICTURE_PARAMETER_SET_10(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_10_PB, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_10_PC) ++#define DSC_RC_TARGET_OFF_LOW(rc_tgt_off_low) ((rc_tgt_off_low) << 20) ++#define DSC_RC_TARGET_OFF_HIGH(rc_tgt_off_high) ((rc_tgt_off_high) << 16) ++#define DSC_RC_QUANT_INC_LIMIT1(lim) ((lim) << 8) ++#define DSC_RC_QUANT_INC_LIMIT0(lim) ((lim) << 0) ++ ++#define DSCA_PICTURE_PARAMETER_SET_11 _MMIO(0x6B22C) ++#define DSCC_PICTURE_PARAMETER_SET_11 _MMIO(0x6BA2C) ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_11_PB 0x7829C ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_11_PB 0x7839C ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_11_PC 0x7849C ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_11_PC 0x7859C ++#define ICL_DSC0_PICTURE_PARAMETER_SET_11(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_11_PB, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_11_PC) ++#define ICL_DSC1_PICTURE_PARAMETER_SET_11(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_11_PB, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_11_PC) ++ ++#define DSCA_PICTURE_PARAMETER_SET_12 _MMIO(0x6B260) ++#define DSCC_PICTURE_PARAMETER_SET_12 _MMIO(0x6BA60) ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_12_PB 0x782A0 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_12_PB 0x783A0 ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_12_PC 0x784A0 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_12_PC 0x785A0 ++#define ICL_DSC0_PICTURE_PARAMETER_SET_12(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_12_PB, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_12_PC) ++#define ICL_DSC1_PICTURE_PARAMETER_SET_12(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_12_PB, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_12_PC) ++ ++#define DSCA_PICTURE_PARAMETER_SET_13 _MMIO(0x6B264) ++#define DSCC_PICTURE_PARAMETER_SET_13 _MMIO(0x6BA64) ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_13_PB 0x782A4 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_13_PB 0x783A4 ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_13_PC 0x784A4 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_13_PC 0x785A4 ++#define ICL_DSC0_PICTURE_PARAMETER_SET_13(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_13_PB, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_13_PC) ++#define ICL_DSC1_PICTURE_PARAMETER_SET_13(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_13_PB, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_13_PC) ++ ++#define DSCA_PICTURE_PARAMETER_SET_14 _MMIO(0x6B268) ++#define DSCC_PICTURE_PARAMETER_SET_14 _MMIO(0x6BA68) ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_14_PB 0x782A8 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_14_PB 0x783A8 ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_14_PC 0x784A8 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_14_PC 0x785A8 ++#define ICL_DSC0_PICTURE_PARAMETER_SET_14(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_14_PB, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_14_PC) ++#define ICL_DSC1_PICTURE_PARAMETER_SET_14(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_14_PB, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_14_PC) ++ ++#define DSCA_PICTURE_PARAMETER_SET_15 _MMIO(0x6B26C) ++#define DSCC_PICTURE_PARAMETER_SET_15 _MMIO(0x6BA6C) ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_15_PB 0x782AC ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_15_PB 0x783AC ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_15_PC 0x784AC ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_15_PC 0x785AC ++#define ICL_DSC0_PICTURE_PARAMETER_SET_15(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_15_PB, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_15_PC) ++#define ICL_DSC1_PICTURE_PARAMETER_SET_15(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_15_PB, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_15_PC) ++ ++#define DSCA_PICTURE_PARAMETER_SET_16 _MMIO(0x6B270) ++#define DSCC_PICTURE_PARAMETER_SET_16 _MMIO(0x6BA70) ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_16_PB 0x782B0 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_16_PB 0x783B0 ++#define _ICL_DSC0_PICTURE_PARAMETER_SET_16_PC 0x784B0 ++#define _ICL_DSC1_PICTURE_PARAMETER_SET_16_PC 0x785B0 ++#define ICL_DSC0_PICTURE_PARAMETER_SET_16(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_16_PB, \ ++ _ICL_DSC0_PICTURE_PARAMETER_SET_16_PC) ++#define ICL_DSC1_PICTURE_PARAMETER_SET_16(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_16_PB, \ ++ _ICL_DSC1_PICTURE_PARAMETER_SET_16_PC) ++#define DSC_SLICE_ROW_PER_FRAME(slice_row_per_frame) ((slice_row_per_frame) << 20) ++#define DSC_SLICE_PER_LINE(slice_per_line) ((slice_per_line) << 16) ++#define DSC_SLICE_CHUNK_SIZE(slice_chunk_size) ((slice_chunk_size) << 0) ++ ++/* Icelake Rate Control Buffer Threshold Registers */ ++#define DSCA_RC_BUF_THRESH_0 _MMIO(0x6B230) ++#define DSCA_RC_BUF_THRESH_0_UDW _MMIO(0x6B230 + 4) ++#define DSCC_RC_BUF_THRESH_0 _MMIO(0x6BA30) ++#define DSCC_RC_BUF_THRESH_0_UDW _MMIO(0x6BA30 + 4) ++#define _ICL_DSC0_RC_BUF_THRESH_0_PB (0x78254) ++#define _ICL_DSC0_RC_BUF_THRESH_0_UDW_PB (0x78254 + 4) ++#define _ICL_DSC1_RC_BUF_THRESH_0_PB (0x78354) ++#define _ICL_DSC1_RC_BUF_THRESH_0_UDW_PB (0x78354 + 4) ++#define _ICL_DSC0_RC_BUF_THRESH_0_PC (0x78454) ++#define _ICL_DSC0_RC_BUF_THRESH_0_UDW_PC (0x78454 + 4) ++#define _ICL_DSC1_RC_BUF_THRESH_0_PC (0x78554) ++#define _ICL_DSC1_RC_BUF_THRESH_0_UDW_PC (0x78554 + 4) ++#define ICL_DSC0_RC_BUF_THRESH_0(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_RC_BUF_THRESH_0_PB, \ ++ _ICL_DSC0_RC_BUF_THRESH_0_PC) ++#define ICL_DSC0_RC_BUF_THRESH_0_UDW(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_RC_BUF_THRESH_0_UDW_PB, \ ++ _ICL_DSC0_RC_BUF_THRESH_0_UDW_PC) ++#define ICL_DSC1_RC_BUF_THRESH_0(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_RC_BUF_THRESH_0_PB, \ ++ _ICL_DSC1_RC_BUF_THRESH_0_PC) ++#define ICL_DSC1_RC_BUF_THRESH_0_UDW(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_RC_BUF_THRESH_0_UDW_PB, \ ++ _ICL_DSC1_RC_BUF_THRESH_0_UDW_PC) ++ ++#define DSCA_RC_BUF_THRESH_1 _MMIO(0x6B238) ++#define DSCA_RC_BUF_THRESH_1_UDW _MMIO(0x6B238 + 4) ++#define DSCC_RC_BUF_THRESH_1 _MMIO(0x6BA38) ++#define DSCC_RC_BUF_THRESH_1_UDW _MMIO(0x6BA38 + 4) ++#define _ICL_DSC0_RC_BUF_THRESH_1_PB (0x7825C) ++#define _ICL_DSC0_RC_BUF_THRESH_1_UDW_PB (0x7825C + 4) ++#define _ICL_DSC1_RC_BUF_THRESH_1_PB (0x7835C) ++#define _ICL_DSC1_RC_BUF_THRESH_1_UDW_PB (0x7835C + 4) ++#define _ICL_DSC0_RC_BUF_THRESH_1_PC (0x7845C) ++#define _ICL_DSC0_RC_BUF_THRESH_1_UDW_PC (0x7845C + 4) ++#define _ICL_DSC1_RC_BUF_THRESH_1_PC (0x7855C) ++#define _ICL_DSC1_RC_BUF_THRESH_1_UDW_PC (0x7855C + 4) ++#define ICL_DSC0_RC_BUF_THRESH_1(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_RC_BUF_THRESH_1_PB, \ ++ _ICL_DSC0_RC_BUF_THRESH_1_PC) ++#define ICL_DSC0_RC_BUF_THRESH_1_UDW(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC0_RC_BUF_THRESH_1_UDW_PB, \ ++ _ICL_DSC0_RC_BUF_THRESH_1_UDW_PC) ++#define ICL_DSC1_RC_BUF_THRESH_1(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_RC_BUF_THRESH_1_PB, \ ++ _ICL_DSC1_RC_BUF_THRESH_1_PC) ++#define ICL_DSC1_RC_BUF_THRESH_1_UDW(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ ++ _ICL_DSC1_RC_BUF_THRESH_1_UDW_PB, \ ++ _ICL_DSC1_RC_BUF_THRESH_1_UDW_PC) ++ ++#define PORT_TX_DFLEXDPSP _MMIO(FIA1_BASE + 0x008A0) ++#define TC_LIVE_STATE_TBT(tc_port) (1 << ((tc_port) * 8 + 6)) ++#define TC_LIVE_STATE_TC(tc_port) (1 << ((tc_port) * 8 + 5)) ++#define DP_LANE_ASSIGNMENT_SHIFT(tc_port) ((tc_port) * 8) ++#define DP_LANE_ASSIGNMENT_MASK(tc_port) (0xf << ((tc_port) * 8)) ++#define DP_LANE_ASSIGNMENT(tc_port, x) ((x) << ((tc_port) * 8)) ++ ++#define PORT_TX_DFLEXDPPMS _MMIO(FIA1_BASE + 0x00890) ++#define DP_PHY_MODE_STATUS_COMPLETED(tc_port) (1 << (tc_port)) ++ ++#define PORT_TX_DFLEXDPCSSS _MMIO(FIA1_BASE + 0x00894) ++#define DP_PHY_MODE_STATUS_NOT_SAFE(tc_port) (1 << (tc_port)) ++ ++#endif /* _I915_REG_H_ */ +diff --git a/drivers/gpu/drm/i915_legacy/i915_request.c b/drivers/gpu/drm/i915_legacy/i915_request.c +new file mode 100644 +index 000000000000..81b48e273cbd +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_request.c +@@ -0,0 +1,1511 @@ ++/* ++ * Copyright © 2008-2015 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "i915_active.h" ++#include "i915_drv.h" ++#include "i915_globals.h" ++#include "i915_reset.h" ++#include "intel_pm.h" ++ ++struct execute_cb { ++ struct list_head link; ++ struct irq_work work; ++ struct i915_sw_fence *fence; ++}; ++ ++static struct i915_global_request { ++ struct i915_global base; ++ struct kmem_cache *slab_requests; ++ struct kmem_cache *slab_dependencies; ++ struct kmem_cache *slab_execute_cbs; ++} global; ++ ++static const char *i915_fence_get_driver_name(struct dma_fence *fence) ++{ ++ return "i915"; ++} ++ ++static const char *i915_fence_get_timeline_name(struct dma_fence *fence) ++{ ++ /* ++ * The timeline struct (as part of the ppgtt underneath a context) ++ * may be freed when the request is no longer in use by the GPU. ++ * We could extend the life of a context to beyond that of all ++ * fences, possibly keeping the hw resource around indefinitely, ++ * or we just give them a false name. Since ++ * dma_fence_ops.get_timeline_name is a debug feature, the occasional ++ * lie seems justifiable. ++ */ ++ if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) ++ return "signaled"; ++ ++ return to_request(fence)->gem_context->name ?: "[i915]"; ++} ++ ++static bool i915_fence_signaled(struct dma_fence *fence) ++{ ++ return i915_request_completed(to_request(fence)); ++} ++ ++static bool i915_fence_enable_signaling(struct dma_fence *fence) ++{ ++ return i915_request_enable_breadcrumb(to_request(fence)); ++} ++ ++static signed long i915_fence_wait(struct dma_fence *fence, ++ bool interruptible, ++ signed long timeout) ++{ ++ return i915_request_wait(to_request(fence), ++ interruptible | I915_WAIT_PRIORITY, ++ timeout); ++} ++ ++static void i915_fence_release(struct dma_fence *fence) ++{ ++ struct i915_request *rq = to_request(fence); ++ ++ /* ++ * The request is put onto a RCU freelist (i.e. the address ++ * is immediately reused), mark the fences as being freed now. ++ * Otherwise the debugobjects for the fences are only marked as ++ * freed when the slab cache itself is freed, and so we would get ++ * caught trying to reuse dead objects. ++ */ ++ i915_sw_fence_fini(&rq->submit); ++ i915_sw_fence_fini(&rq->semaphore); ++ ++ kmem_cache_free(global.slab_requests, rq); ++} ++ ++const struct dma_fence_ops i915_fence_ops = { ++ .get_driver_name = i915_fence_get_driver_name, ++ .get_timeline_name = i915_fence_get_timeline_name, ++ .enable_signaling = i915_fence_enable_signaling, ++ .signaled = i915_fence_signaled, ++ .wait = i915_fence_wait, ++ .release = i915_fence_release, ++}; ++ ++static inline void ++i915_request_remove_from_client(struct i915_request *request) ++{ ++ struct drm_i915_file_private *file_priv; ++ ++ file_priv = request->file_priv; ++ if (!file_priv) ++ return; ++ ++ spin_lock(&file_priv->mm.lock); ++ if (request->file_priv) { ++ list_del(&request->client_link); ++ request->file_priv = NULL; ++ } ++ spin_unlock(&file_priv->mm.lock); ++} ++ ++static void reserve_gt(struct drm_i915_private *i915) ++{ ++ if (!i915->gt.active_requests++) ++ i915_gem_unpark(i915); ++} ++ ++static void unreserve_gt(struct drm_i915_private *i915) ++{ ++ GEM_BUG_ON(!i915->gt.active_requests); ++ if (!--i915->gt.active_requests) ++ i915_gem_park(i915); ++} ++ ++static void advance_ring(struct i915_request *request) ++{ ++ struct intel_ring *ring = request->ring; ++ unsigned int tail; ++ ++ /* ++ * We know the GPU must have read the request to have ++ * sent us the seqno + interrupt, so use the position ++ * of tail of the request to update the last known position ++ * of the GPU head. ++ * ++ * Note this requires that we are always called in request ++ * completion order. ++ */ ++ GEM_BUG_ON(!list_is_first(&request->ring_link, &ring->request_list)); ++ if (list_is_last(&request->ring_link, &ring->request_list)) { ++ /* ++ * We may race here with execlists resubmitting this request ++ * as we retire it. The resubmission will move the ring->tail ++ * forwards (to request->wa_tail). We either read the ++ * current value that was written to hw, or the value that ++ * is just about to be. Either works, if we miss the last two ++ * noops - they are safe to be replayed on a reset. ++ */ ++ tail = READ_ONCE(request->tail); ++ list_del(&ring->active_link); ++ } else { ++ tail = request->postfix; ++ } ++ list_del_init(&request->ring_link); ++ ++ ring->head = tail; ++} ++ ++static void free_capture_list(struct i915_request *request) ++{ ++ struct i915_capture_list *capture; ++ ++ capture = request->capture_list; ++ while (capture) { ++ struct i915_capture_list *next = capture->next; ++ ++ kfree(capture); ++ capture = next; ++ } ++} ++ ++static void __retire_engine_request(struct intel_engine_cs *engine, ++ struct i915_request *rq) ++{ ++ GEM_TRACE("%s(%s) fence %llx:%lld, current %d\n", ++ __func__, engine->name, ++ rq->fence.context, rq->fence.seqno, ++ hwsp_seqno(rq)); ++ ++ GEM_BUG_ON(!i915_request_completed(rq)); ++ ++ local_irq_disable(); ++ ++ spin_lock(&engine->timeline.lock); ++ GEM_BUG_ON(!list_is_first(&rq->link, &engine->timeline.requests)); ++ list_del_init(&rq->link); ++ spin_unlock(&engine->timeline.lock); ++ ++ spin_lock(&rq->lock); ++ i915_request_mark_complete(rq); ++ if (!i915_request_signaled(rq)) ++ dma_fence_signal_locked(&rq->fence); ++ if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &rq->fence.flags)) ++ i915_request_cancel_breadcrumb(rq); ++ if (rq->waitboost) { ++ GEM_BUG_ON(!atomic_read(&rq->i915->gt_pm.rps.num_waiters)); ++ atomic_dec(&rq->i915->gt_pm.rps.num_waiters); ++ } ++ spin_unlock(&rq->lock); ++ ++ local_irq_enable(); ++ ++ /* ++ * The backing object for the context is done after switching to the ++ * *next* context. Therefore we cannot retire the previous context until ++ * the next context has already started running. However, since we ++ * cannot take the required locks at i915_request_submit() we ++ * defer the unpinning of the active context to now, retirement of ++ * the subsequent request. ++ */ ++ if (engine->last_retired_context) ++ intel_context_unpin(engine->last_retired_context); ++ engine->last_retired_context = rq->hw_context; ++} ++ ++static void __retire_engine_upto(struct intel_engine_cs *engine, ++ struct i915_request *rq) ++{ ++ struct i915_request *tmp; ++ ++ if (list_empty(&rq->link)) ++ return; ++ ++ do { ++ tmp = list_first_entry(&engine->timeline.requests, ++ typeof(*tmp), link); ++ ++ GEM_BUG_ON(tmp->engine != engine); ++ __retire_engine_request(engine, tmp); ++ } while (tmp != rq); ++} ++ ++static void i915_request_retire(struct i915_request *request) ++{ ++ struct i915_active_request *active, *next; ++ ++ GEM_TRACE("%s fence %llx:%lld, current %d\n", ++ request->engine->name, ++ request->fence.context, request->fence.seqno, ++ hwsp_seqno(request)); ++ ++ lockdep_assert_held(&request->i915->drm.struct_mutex); ++ GEM_BUG_ON(!i915_sw_fence_signaled(&request->submit)); ++ GEM_BUG_ON(!i915_request_completed(request)); ++ ++ trace_i915_request_retire(request); ++ ++ advance_ring(request); ++ free_capture_list(request); ++ ++ /* ++ * Walk through the active list, calling retire on each. This allows ++ * objects to track their GPU activity and mark themselves as idle ++ * when their *last* active request is completed (updating state ++ * tracking lists for eviction, active references for GEM, etc). ++ * ++ * As the ->retire() may free the node, we decouple it first and ++ * pass along the auxiliary information (to avoid dereferencing ++ * the node after the callback). ++ */ ++ list_for_each_entry_safe(active, next, &request->active_list, link) { ++ /* ++ * In microbenchmarks or focusing upon time inside the kernel, ++ * we may spend an inordinate amount of time simply handling ++ * the retirement of requests and processing their callbacks. ++ * Of which, this loop itself is particularly hot due to the ++ * cache misses when jumping around the list of ++ * i915_active_request. So we try to keep this loop as ++ * streamlined as possible and also prefetch the next ++ * i915_active_request to try and hide the likely cache miss. ++ */ ++ prefetchw(next); ++ ++ INIT_LIST_HEAD(&active->link); ++ RCU_INIT_POINTER(active->request, NULL); ++ ++ active->retire(active, request); ++ } ++ ++ i915_request_remove_from_client(request); ++ ++ intel_context_unpin(request->hw_context); ++ ++ __retire_engine_upto(request->engine, request); ++ ++ unreserve_gt(request->i915); ++ ++ i915_sched_node_fini(&request->sched); ++ i915_request_put(request); ++} ++ ++void i915_request_retire_upto(struct i915_request *rq) ++{ ++ struct intel_ring *ring = rq->ring; ++ struct i915_request *tmp; ++ ++ GEM_TRACE("%s fence %llx:%lld, current %d\n", ++ rq->engine->name, ++ rq->fence.context, rq->fence.seqno, ++ hwsp_seqno(rq)); ++ ++ lockdep_assert_held(&rq->i915->drm.struct_mutex); ++ GEM_BUG_ON(!i915_request_completed(rq)); ++ ++ if (list_empty(&rq->ring_link)) ++ return; ++ ++ do { ++ tmp = list_first_entry(&ring->request_list, ++ typeof(*tmp), ring_link); ++ ++ i915_request_retire(tmp); ++ } while (tmp != rq); ++} ++ ++static void irq_execute_cb(struct irq_work *wrk) ++{ ++ struct execute_cb *cb = container_of(wrk, typeof(*cb), work); ++ ++ i915_sw_fence_complete(cb->fence); ++ kmem_cache_free(global.slab_execute_cbs, cb); ++} ++ ++static void __notify_execute_cb(struct i915_request *rq) ++{ ++ struct execute_cb *cb; ++ ++ lockdep_assert_held(&rq->lock); ++ ++ if (list_empty(&rq->execute_cb)) ++ return; ++ ++ list_for_each_entry(cb, &rq->execute_cb, link) ++ irq_work_queue(&cb->work); ++ ++ /* ++ * XXX Rollback on __i915_request_unsubmit() ++ * ++ * In the future, perhaps when we have an active time-slicing scheduler, ++ * it will be interesting to unsubmit parallel execution and remove ++ * busywaits from the GPU until their master is restarted. This is ++ * quite hairy, we have to carefully rollback the fence and do a ++ * preempt-to-idle cycle on the target engine, all the while the ++ * master execute_cb may refire. ++ */ ++ INIT_LIST_HEAD(&rq->execute_cb); ++} ++ ++static int ++i915_request_await_execution(struct i915_request *rq, ++ struct i915_request *signal, ++ gfp_t gfp) ++{ ++ struct execute_cb *cb; ++ ++ if (i915_request_is_active(signal)) ++ return 0; ++ ++ cb = kmem_cache_alloc(global.slab_execute_cbs, gfp); ++ if (!cb) ++ return -ENOMEM; ++ ++ cb->fence = &rq->submit; ++ i915_sw_fence_await(cb->fence); ++ init_irq_work(&cb->work, irq_execute_cb); ++ ++ spin_lock_irq(&signal->lock); ++ if (i915_request_is_active(signal)) { ++ i915_sw_fence_complete(cb->fence); ++ kmem_cache_free(global.slab_execute_cbs, cb); ++ } else { ++ list_add_tail(&cb->link, &signal->execute_cb); ++ } ++ spin_unlock_irq(&signal->lock); ++ ++ return 0; ++} ++ ++static void move_to_timeline(struct i915_request *request, ++ struct i915_timeline *timeline) ++{ ++ GEM_BUG_ON(request->timeline == &request->engine->timeline); ++ lockdep_assert_held(&request->engine->timeline.lock); ++ ++ spin_lock(&request->timeline->lock); ++ list_move_tail(&request->link, &timeline->requests); ++ spin_unlock(&request->timeline->lock); ++} ++ ++void __i915_request_submit(struct i915_request *request) ++{ ++ struct intel_engine_cs *engine = request->engine; ++ ++ GEM_TRACE("%s fence %llx:%lld -> current %d\n", ++ engine->name, ++ request->fence.context, request->fence.seqno, ++ hwsp_seqno(request)); ++ ++ GEM_BUG_ON(!irqs_disabled()); ++ lockdep_assert_held(&engine->timeline.lock); ++ ++ if (i915_gem_context_is_banned(request->gem_context)) ++ i915_request_skip(request, -EIO); ++ ++ /* ++ * Are we using semaphores when the gpu is already saturated? ++ * ++ * Using semaphores incurs a cost in having the GPU poll a ++ * memory location, busywaiting for it to change. The continual ++ * memory reads can have a noticeable impact on the rest of the ++ * system with the extra bus traffic, stalling the cpu as it too ++ * tries to access memory across the bus (perf stat -e bus-cycles). ++ * ++ * If we installed a semaphore on this request and we only submit ++ * the request after the signaler completed, that indicates the ++ * system is overloaded and using semaphores at this time only ++ * increases the amount of work we are doing. If so, we disable ++ * further use of semaphores until we are idle again, whence we ++ * optimistically try again. ++ */ ++ if (request->sched.semaphores && ++ i915_sw_fence_signaled(&request->semaphore)) ++ engine->saturated |= request->sched.semaphores; ++ ++ /* We may be recursing from the signal callback of another i915 fence */ ++ spin_lock_nested(&request->lock, SINGLE_DEPTH_NESTING); ++ ++ GEM_BUG_ON(test_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags)); ++ set_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags); ++ ++ if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags) && ++ !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &request->fence.flags) && ++ !i915_request_enable_breadcrumb(request)) ++ intel_engine_queue_breadcrumbs(engine); ++ ++ __notify_execute_cb(request); ++ ++ spin_unlock(&request->lock); ++ ++ engine->emit_fini_breadcrumb(request, ++ request->ring->vaddr + request->postfix); ++ ++ /* Transfer from per-context onto the global per-engine timeline */ ++ move_to_timeline(request, &engine->timeline); ++ ++ trace_i915_request_execute(request); ++} ++ ++void i915_request_submit(struct i915_request *request) ++{ ++ struct intel_engine_cs *engine = request->engine; ++ unsigned long flags; ++ ++ /* Will be called from irq-context when using foreign fences. */ ++ spin_lock_irqsave(&engine->timeline.lock, flags); ++ ++ __i915_request_submit(request); ++ ++ spin_unlock_irqrestore(&engine->timeline.lock, flags); ++} ++ ++void __i915_request_unsubmit(struct i915_request *request) ++{ ++ struct intel_engine_cs *engine = request->engine; ++ ++ GEM_TRACE("%s fence %llx:%lld, current %d\n", ++ engine->name, ++ request->fence.context, request->fence.seqno, ++ hwsp_seqno(request)); ++ ++ GEM_BUG_ON(!irqs_disabled()); ++ lockdep_assert_held(&engine->timeline.lock); ++ ++ /* ++ * Only unwind in reverse order, required so that the per-context list ++ * is kept in seqno/ring order. ++ */ ++ ++ /* We may be recursing from the signal callback of another i915 fence */ ++ spin_lock_nested(&request->lock, SINGLE_DEPTH_NESTING); ++ ++ if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags)) ++ i915_request_cancel_breadcrumb(request); ++ ++ GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags)); ++ clear_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags); ++ ++ spin_unlock(&request->lock); ++ ++ /* Transfer back from the global per-engine timeline to per-context */ ++ move_to_timeline(request, request->timeline); ++ ++ /* ++ * We don't need to wake_up any waiters on request->execute, they ++ * will get woken by any other event or us re-adding this request ++ * to the engine timeline (__i915_request_submit()). The waiters ++ * should be quite adapt at finding that the request now has a new ++ * global_seqno to the one they went to sleep on. ++ */ ++} ++ ++void i915_request_unsubmit(struct i915_request *request) ++{ ++ struct intel_engine_cs *engine = request->engine; ++ unsigned long flags; ++ ++ /* Will be called from irq-context when using foreign fences. */ ++ spin_lock_irqsave(&engine->timeline.lock, flags); ++ ++ __i915_request_unsubmit(request); ++ ++ spin_unlock_irqrestore(&engine->timeline.lock, flags); ++} ++ ++static int __i915_sw_fence_call ++submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) ++{ ++ struct i915_request *request = ++ container_of(fence, typeof(*request), submit); ++ ++ switch (state) { ++ case FENCE_COMPLETE: ++ trace_i915_request_submit(request); ++ /* ++ * We need to serialize use of the submit_request() callback ++ * with its hotplugging performed during an emergency ++ * i915_gem_set_wedged(). We use the RCU mechanism to mark the ++ * critical section in order to force i915_gem_set_wedged() to ++ * wait until the submit_request() is completed before ++ * proceeding. ++ */ ++ rcu_read_lock(); ++ request->engine->submit_request(request); ++ rcu_read_unlock(); ++ break; ++ ++ case FENCE_FREE: ++ i915_request_put(request); ++ break; ++ } ++ ++ return NOTIFY_DONE; ++} ++ ++static int __i915_sw_fence_call ++semaphore_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) ++{ ++ struct i915_request *request = ++ container_of(fence, typeof(*request), semaphore); ++ ++ switch (state) { ++ case FENCE_COMPLETE: ++ i915_schedule_bump_priority(request, I915_PRIORITY_NOSEMAPHORE); ++ break; ++ ++ case FENCE_FREE: ++ i915_request_put(request); ++ break; ++ } ++ ++ return NOTIFY_DONE; ++} ++ ++static void ring_retire_requests(struct intel_ring *ring) ++{ ++ struct i915_request *rq, *rn; ++ ++ list_for_each_entry_safe(rq, rn, &ring->request_list, ring_link) { ++ if (!i915_request_completed(rq)) ++ break; ++ ++ i915_request_retire(rq); ++ } ++} ++ ++static noinline struct i915_request * ++i915_request_alloc_slow(struct intel_context *ce) ++{ ++ struct intel_ring *ring = ce->ring; ++ struct i915_request *rq; ++ ++ if (list_empty(&ring->request_list)) ++ goto out; ++ ++ /* Ratelimit ourselves to prevent oom from malicious clients */ ++ rq = list_last_entry(&ring->request_list, typeof(*rq), ring_link); ++ cond_synchronize_rcu(rq->rcustate); ++ ++ /* Retire our old requests in the hope that we free some */ ++ ring_retire_requests(ring); ++ ++out: ++ return kmem_cache_alloc(global.slab_requests, GFP_KERNEL); ++} ++ ++/** ++ * i915_request_alloc - allocate a request structure ++ * ++ * @engine: engine that we wish to issue the request on. ++ * @ctx: context that the request will be associated with. ++ * ++ * Returns a pointer to the allocated request if successful, ++ * or an error code if not. ++ */ ++struct i915_request * ++i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx) ++{ ++ struct drm_i915_private *i915 = engine->i915; ++ struct intel_context *ce; ++ struct i915_timeline *tl; ++ struct i915_request *rq; ++ u32 seqno; ++ int ret; ++ ++ lockdep_assert_held(&i915->drm.struct_mutex); ++ ++ /* ++ * Preempt contexts are reserved for exclusive use to inject a ++ * preemption context switch. They are never to be used for any trivial ++ * request! ++ */ ++ GEM_BUG_ON(ctx == i915->preempt_context); ++ ++ /* ++ * ABI: Before userspace accesses the GPU (e.g. execbuffer), report ++ * EIO if the GPU is already wedged. ++ */ ++ ret = i915_terminally_wedged(i915); ++ if (ret) ++ return ERR_PTR(ret); ++ ++ /* ++ * Pinning the contexts may generate requests in order to acquire ++ * GGTT space, so do this first before we reserve a seqno for ++ * ourselves. ++ */ ++ ce = intel_context_pin(ctx, engine); ++ if (IS_ERR(ce)) ++ return ERR_CAST(ce); ++ ++ reserve_gt(i915); ++ mutex_lock(&ce->ring->timeline->mutex); ++ ++ /* Move our oldest request to the slab-cache (if not in use!) */ ++ rq = list_first_entry(&ce->ring->request_list, typeof(*rq), ring_link); ++ if (!list_is_last(&rq->ring_link, &ce->ring->request_list) && ++ i915_request_completed(rq)) ++ i915_request_retire(rq); ++ ++ /* ++ * Beware: Dragons be flying overhead. ++ * ++ * We use RCU to look up requests in flight. The lookups may ++ * race with the request being allocated from the slab freelist. ++ * That is the request we are writing to here, may be in the process ++ * of being read by __i915_active_request_get_rcu(). As such, ++ * we have to be very careful when overwriting the contents. During ++ * the RCU lookup, we change chase the request->engine pointer, ++ * read the request->global_seqno and increment the reference count. ++ * ++ * The reference count is incremented atomically. If it is zero, ++ * the lookup knows the request is unallocated and complete. Otherwise, ++ * it is either still in use, or has been reallocated and reset ++ * with dma_fence_init(). This increment is safe for release as we ++ * check that the request we have a reference to and matches the active ++ * request. ++ * ++ * Before we increment the refcount, we chase the request->engine ++ * pointer. We must not call kmem_cache_zalloc() or else we set ++ * that pointer to NULL and cause a crash during the lookup. If ++ * we see the request is completed (based on the value of the ++ * old engine and seqno), the lookup is complete and reports NULL. ++ * If we decide the request is not completed (new engine or seqno), ++ * then we grab a reference and double check that it is still the ++ * active request - which it won't be and restart the lookup. ++ * ++ * Do not use kmem_cache_zalloc() here! ++ */ ++ rq = kmem_cache_alloc(global.slab_requests, ++ GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN); ++ if (unlikely(!rq)) { ++ rq = i915_request_alloc_slow(ce); ++ if (!rq) { ++ ret = -ENOMEM; ++ goto err_unreserve; ++ } ++ } ++ ++ INIT_LIST_HEAD(&rq->active_list); ++ INIT_LIST_HEAD(&rq->execute_cb); ++ ++ tl = ce->ring->timeline; ++ ret = i915_timeline_get_seqno(tl, rq, &seqno); ++ if (ret) ++ goto err_free; ++ ++ rq->i915 = i915; ++ rq->engine = engine; ++ rq->gem_context = ctx; ++ rq->hw_context = ce; ++ rq->ring = ce->ring; ++ rq->timeline = tl; ++ GEM_BUG_ON(rq->timeline == &engine->timeline); ++ rq->hwsp_seqno = tl->hwsp_seqno; ++ rq->hwsp_cacheline = tl->hwsp_cacheline; ++ rq->rcustate = get_state_synchronize_rcu(); /* acts as smp_mb() */ ++ ++ spin_lock_init(&rq->lock); ++ dma_fence_init(&rq->fence, &i915_fence_ops, &rq->lock, ++ tl->fence_context, seqno); ++ ++ /* We bump the ref for the fence chain */ ++ i915_sw_fence_init(&i915_request_get(rq)->submit, submit_notify); ++ i915_sw_fence_init(&i915_request_get(rq)->semaphore, semaphore_notify); ++ ++ i915_sched_node_init(&rq->sched); ++ ++ /* No zalloc, must clear what we need by hand */ ++ rq->file_priv = NULL; ++ rq->batch = NULL; ++ rq->capture_list = NULL; ++ rq->waitboost = false; ++ ++ /* ++ * Reserve space in the ring buffer for all the commands required to ++ * eventually emit this request. This is to guarantee that the ++ * i915_request_add() call can't fail. Note that the reserve may need ++ * to be redone if the request is not actually submitted straight ++ * away, e.g. because a GPU scheduler has deferred it. ++ * ++ * Note that due to how we add reserved_space to intel_ring_begin() ++ * we need to double our request to ensure that if we need to wrap ++ * around inside i915_request_add() there is sufficient space at ++ * the beginning of the ring as well. ++ */ ++ rq->reserved_space = 2 * engine->emit_fini_breadcrumb_dw * sizeof(u32); ++ ++ /* ++ * Record the position of the start of the request so that ++ * should we detect the updated seqno part-way through the ++ * GPU processing the request, we never over-estimate the ++ * position of the head. ++ */ ++ rq->head = rq->ring->emit; ++ ++ ret = engine->request_alloc(rq); ++ if (ret) ++ goto err_unwind; ++ ++ /* Keep a second pin for the dual retirement along engine and ring */ ++ __intel_context_pin(ce); ++ ++ rq->infix = rq->ring->emit; /* end of header; start of user payload */ ++ ++ /* Check that we didn't interrupt ourselves with a new request */ ++ lockdep_assert_held(&rq->timeline->mutex); ++ GEM_BUG_ON(rq->timeline->seqno != rq->fence.seqno); ++ rq->cookie = lockdep_pin_lock(&rq->timeline->mutex); ++ ++ return rq; ++ ++err_unwind: ++ ce->ring->emit = rq->head; ++ ++ /* Make sure we didn't add ourselves to external state before freeing */ ++ GEM_BUG_ON(!list_empty(&rq->active_list)); ++ GEM_BUG_ON(!list_empty(&rq->sched.signalers_list)); ++ GEM_BUG_ON(!list_empty(&rq->sched.waiters_list)); ++ ++err_free: ++ kmem_cache_free(global.slab_requests, rq); ++err_unreserve: ++ mutex_unlock(&ce->ring->timeline->mutex); ++ unreserve_gt(i915); ++ intel_context_unpin(ce); ++ return ERR_PTR(ret); ++} ++ ++static int ++i915_request_await_start(struct i915_request *rq, struct i915_request *signal) ++{ ++ if (list_is_first(&signal->ring_link, &signal->ring->request_list)) ++ return 0; ++ ++ signal = list_prev_entry(signal, ring_link); ++ if (i915_timeline_sync_is_later(rq->timeline, &signal->fence)) ++ return 0; ++ ++ return i915_sw_fence_await_dma_fence(&rq->submit, ++ &signal->fence, 0, ++ I915_FENCE_GFP); ++} ++ ++static intel_engine_mask_t ++already_busywaiting(struct i915_request *rq) ++{ ++ /* ++ * Polling a semaphore causes bus traffic, delaying other users of ++ * both the GPU and CPU. We want to limit the impact on others, ++ * while taking advantage of early submission to reduce GPU ++ * latency. Therefore we restrict ourselves to not using more ++ * than one semaphore from each source, and not using a semaphore ++ * if we have detected the engine is saturated (i.e. would not be ++ * submitted early and cause bus traffic reading an already passed ++ * semaphore). ++ * ++ * See the are-we-too-late? check in __i915_request_submit(). ++ */ ++ return rq->sched.semaphores | rq->engine->saturated; ++} ++ ++static int ++emit_semaphore_wait(struct i915_request *to, ++ struct i915_request *from, ++ gfp_t gfp) ++{ ++ u32 hwsp_offset; ++ u32 *cs; ++ int err; ++ ++ GEM_BUG_ON(!from->timeline->has_initial_breadcrumb); ++ GEM_BUG_ON(INTEL_GEN(to->i915) < 8); ++ ++ /* Just emit the first semaphore we see as request space is limited. */ ++ if (already_busywaiting(to) & from->engine->mask) ++ return i915_sw_fence_await_dma_fence(&to->submit, ++ &from->fence, 0, ++ I915_FENCE_GFP); ++ ++ err = i915_request_await_start(to, from); ++ if (err < 0) ++ return err; ++ ++ /* We need to pin the signaler's HWSP until we are finished reading. */ ++ err = i915_timeline_read_hwsp(from, to, &hwsp_offset); ++ if (err) ++ return err; ++ ++ /* Only submit our spinner after the signaler is running! */ ++ err = i915_request_await_execution(to, from, gfp); ++ if (err) ++ return err; ++ ++ cs = intel_ring_begin(to, 4); ++ if (IS_ERR(cs)) ++ return PTR_ERR(cs); ++ ++ /* ++ * Using greater-than-or-equal here means we have to worry ++ * about seqno wraparound. To side step that issue, we swap ++ * the timeline HWSP upon wrapping, so that everyone listening ++ * for the old (pre-wrap) values do not see the much smaller ++ * (post-wrap) values than they were expecting (and so wait ++ * forever). ++ */ ++ *cs++ = MI_SEMAPHORE_WAIT | ++ MI_SEMAPHORE_GLOBAL_GTT | ++ MI_SEMAPHORE_POLL | ++ MI_SEMAPHORE_SAD_GTE_SDD; ++ *cs++ = from->fence.seqno; ++ *cs++ = hwsp_offset; ++ *cs++ = 0; ++ ++ intel_ring_advance(to, cs); ++ to->sched.semaphores |= from->engine->mask; ++ to->sched.flags |= I915_SCHED_HAS_SEMAPHORE_CHAIN; ++ return 0; ++} ++ ++static int ++i915_request_await_request(struct i915_request *to, struct i915_request *from) ++{ ++ int ret; ++ ++ GEM_BUG_ON(to == from); ++ GEM_BUG_ON(to->timeline == from->timeline); ++ ++ if (i915_request_completed(from)) ++ return 0; ++ ++ if (to->engine->schedule) { ++ ret = i915_sched_node_add_dependency(&to->sched, &from->sched); ++ if (ret < 0) ++ return ret; ++ } ++ ++ if (to->engine == from->engine) { ++ ret = i915_sw_fence_await_sw_fence_gfp(&to->submit, ++ &from->submit, ++ I915_FENCE_GFP); ++ } else if (intel_engine_has_semaphores(to->engine) && ++ to->gem_context->sched.priority >= I915_PRIORITY_NORMAL) { ++ ret = emit_semaphore_wait(to, from, I915_FENCE_GFP); ++ } else { ++ ret = i915_sw_fence_await_dma_fence(&to->submit, ++ &from->fence, 0, ++ I915_FENCE_GFP); ++ } ++ if (ret < 0) ++ return ret; ++ ++ if (to->sched.flags & I915_SCHED_HAS_SEMAPHORE_CHAIN) { ++ ret = i915_sw_fence_await_dma_fence(&to->semaphore, ++ &from->fence, 0, ++ I915_FENCE_GFP); ++ if (ret < 0) ++ return ret; ++ } ++ ++ return 0; ++} ++ ++int ++i915_request_await_dma_fence(struct i915_request *rq, struct dma_fence *fence) ++{ ++ struct dma_fence **child = &fence; ++ unsigned int nchild = 1; ++ int ret; ++ ++ /* ++ * Note that if the fence-array was created in signal-on-any mode, ++ * we should *not* decompose it into its individual fences. However, ++ * we don't currently store which mode the fence-array is operating ++ * in. Fortunately, the only user of signal-on-any is private to ++ * amdgpu and we should not see any incoming fence-array from ++ * sync-file being in signal-on-any mode. ++ */ ++ if (dma_fence_is_array(fence)) { ++ struct dma_fence_array *array = to_dma_fence_array(fence); ++ ++ child = array->fences; ++ nchild = array->num_fences; ++ GEM_BUG_ON(!nchild); ++ } ++ ++ do { ++ fence = *child++; ++ if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) ++ continue; ++ ++ /* ++ * Requests on the same timeline are explicitly ordered, along ++ * with their dependencies, by i915_request_add() which ensures ++ * that requests are submitted in-order through each ring. ++ */ ++ if (fence->context == rq->fence.context) ++ continue; ++ ++ /* Squash repeated waits to the same timelines */ ++ if (fence->context != rq->i915->mm.unordered_timeline && ++ i915_timeline_sync_is_later(rq->timeline, fence)) ++ continue; ++ ++ if (dma_fence_is_i915(fence)) ++ ret = i915_request_await_request(rq, to_request(fence)); ++ else ++ ret = i915_sw_fence_await_dma_fence(&rq->submit, fence, ++ I915_FENCE_TIMEOUT, ++ I915_FENCE_GFP); ++ if (ret < 0) ++ return ret; ++ ++ /* Record the latest fence used against each timeline */ ++ if (fence->context != rq->i915->mm.unordered_timeline) ++ i915_timeline_sync_set(rq->timeline, fence); ++ } while (--nchild); ++ ++ return 0; ++} ++ ++/** ++ * i915_request_await_object - set this request to (async) wait upon a bo ++ * @to: request we are wishing to use ++ * @obj: object which may be in use on another ring. ++ * @write: whether the wait is on behalf of a writer ++ * ++ * This code is meant to abstract object synchronization with the GPU. ++ * Conceptually we serialise writes between engines inside the GPU. ++ * We only allow one engine to write into a buffer at any time, but ++ * multiple readers. To ensure each has a coherent view of memory, we must: ++ * ++ * - If there is an outstanding write request to the object, the new ++ * request must wait for it to complete (either CPU or in hw, requests ++ * on the same ring will be naturally ordered). ++ * ++ * - If we are a write request (pending_write_domain is set), the new ++ * request must wait for outstanding read requests to complete. ++ * ++ * Returns 0 if successful, else propagates up the lower layer error. ++ */ ++int ++i915_request_await_object(struct i915_request *to, ++ struct drm_i915_gem_object *obj, ++ bool write) ++{ ++ struct dma_fence *excl; ++ int ret = 0; ++ ++ if (write) { ++ struct dma_fence **shared; ++ unsigned int count, i; ++ ++ ret = reservation_object_get_fences_rcu(obj->resv, ++ &excl, &count, &shared); ++ if (ret) ++ return ret; ++ ++ for (i = 0; i < count; i++) { ++ ret = i915_request_await_dma_fence(to, shared[i]); ++ if (ret) ++ break; ++ ++ dma_fence_put(shared[i]); ++ } ++ ++ for (; i < count; i++) ++ dma_fence_put(shared[i]); ++ kfree(shared); ++ } else { ++ excl = reservation_object_get_excl_rcu(obj->resv); ++ } ++ ++ if (excl) { ++ if (ret == 0) ++ ret = i915_request_await_dma_fence(to, excl); ++ ++ dma_fence_put(excl); ++ } ++ ++ return ret; ++} ++ ++void i915_request_skip(struct i915_request *rq, int error) ++{ ++ void *vaddr = rq->ring->vaddr; ++ u32 head; ++ ++ GEM_BUG_ON(!IS_ERR_VALUE((long)error)); ++ dma_fence_set_error(&rq->fence, error); ++ ++ /* ++ * As this request likely depends on state from the lost ++ * context, clear out all the user operations leaving the ++ * breadcrumb at the end (so we get the fence notifications). ++ */ ++ head = rq->infix; ++ if (rq->postfix < head) { ++ memset(vaddr + head, 0, rq->ring->size - head); ++ head = 0; ++ } ++ memset(vaddr + head, 0, rq->postfix - head); ++} ++ ++static struct i915_request * ++__i915_request_add_to_timeline(struct i915_request *rq) ++{ ++ struct i915_timeline *timeline = rq->timeline; ++ struct i915_request *prev; ++ ++ /* ++ * Dependency tracking and request ordering along the timeline ++ * is special cased so that we can eliminate redundant ordering ++ * operations while building the request (we know that the timeline ++ * itself is ordered, and here we guarantee it). ++ * ++ * As we know we will need to emit tracking along the timeline, ++ * we embed the hooks into our request struct -- at the cost of ++ * having to have specialised no-allocation interfaces (which will ++ * be beneficial elsewhere). ++ * ++ * A second benefit to open-coding i915_request_await_request is ++ * that we can apply a slight variant of the rules specialised ++ * for timelines that jump between engines (such as virtual engines). ++ * If we consider the case of virtual engine, we must emit a dma-fence ++ * to prevent scheduling of the second request until the first is ++ * complete (to maximise our greedy late load balancing) and this ++ * precludes optimising to use semaphores serialisation of a single ++ * timeline across engines. ++ */ ++ prev = i915_active_request_raw(&timeline->last_request, ++ &rq->i915->drm.struct_mutex); ++ if (prev && !i915_request_completed(prev)) { ++ if (is_power_of_2(prev->engine->mask | rq->engine->mask)) ++ i915_sw_fence_await_sw_fence(&rq->submit, ++ &prev->submit, ++ &rq->submitq); ++ else ++ __i915_sw_fence_await_dma_fence(&rq->submit, ++ &prev->fence, ++ &rq->dmaq); ++ if (rq->engine->schedule) ++ __i915_sched_node_add_dependency(&rq->sched, ++ &prev->sched, ++ &rq->dep, ++ 0); ++ } ++ ++ spin_lock_irq(&timeline->lock); ++ list_add_tail(&rq->link, &timeline->requests); ++ spin_unlock_irq(&timeline->lock); ++ ++ GEM_BUG_ON(timeline->seqno != rq->fence.seqno); ++ __i915_active_request_set(&timeline->last_request, rq); ++ ++ return prev; ++} ++ ++/* ++ * NB: This function is not allowed to fail. Doing so would mean the the ++ * request is not being tracked for completion but the work itself is ++ * going to happen on the hardware. This would be a Bad Thing(tm). ++ */ ++void i915_request_add(struct i915_request *request) ++{ ++ struct intel_engine_cs *engine = request->engine; ++ struct i915_timeline *timeline = request->timeline; ++ struct intel_ring *ring = request->ring; ++ struct i915_request *prev; ++ u32 *cs; ++ ++ GEM_TRACE("%s fence %llx:%lld\n", ++ engine->name, request->fence.context, request->fence.seqno); ++ ++ lockdep_assert_held(&request->timeline->mutex); ++ lockdep_unpin_lock(&request->timeline->mutex, request->cookie); ++ ++ trace_i915_request_add(request); ++ ++ /* ++ * Make sure that no request gazumped us - if it was allocated after ++ * our i915_request_alloc() and called __i915_request_add() before ++ * us, the timeline will hold its seqno which is later than ours. ++ */ ++ GEM_BUG_ON(timeline->seqno != request->fence.seqno); ++ ++ /* ++ * To ensure that this call will not fail, space for its emissions ++ * should already have been reserved in the ring buffer. Let the ring ++ * know that it is time to use that space up. ++ */ ++ GEM_BUG_ON(request->reserved_space > request->ring->space); ++ request->reserved_space = 0; ++ ++ /* ++ * Record the position of the start of the breadcrumb so that ++ * should we detect the updated seqno part-way through the ++ * GPU processing the request, we never over-estimate the ++ * position of the ring's HEAD. ++ */ ++ cs = intel_ring_begin(request, engine->emit_fini_breadcrumb_dw); ++ GEM_BUG_ON(IS_ERR(cs)); ++ request->postfix = intel_ring_offset(request, cs); ++ ++ prev = __i915_request_add_to_timeline(request); ++ ++ list_add_tail(&request->ring_link, &ring->request_list); ++ if (list_is_first(&request->ring_link, &ring->request_list)) ++ list_add(&ring->active_link, &request->i915->gt.active_rings); ++ request->i915->gt.active_engines |= request->engine->mask; ++ request->emitted_jiffies = jiffies; ++ ++ /* ++ * Let the backend know a new request has arrived that may need ++ * to adjust the existing execution schedule due to a high priority ++ * request - i.e. we may want to preempt the current request in order ++ * to run a high priority dependency chain *before* we can execute this ++ * request. ++ * ++ * This is called before the request is ready to run so that we can ++ * decide whether to preempt the entire chain so that it is ready to ++ * run at the earliest possible convenience. ++ */ ++ local_bh_disable(); ++ i915_sw_fence_commit(&request->semaphore); ++ rcu_read_lock(); /* RCU serialisation for set-wedged protection */ ++ if (engine->schedule) { ++ struct i915_sched_attr attr = request->gem_context->sched; ++ ++ /* ++ * Boost actual workloads past semaphores! ++ * ++ * With semaphores we spin on one engine waiting for another, ++ * simply to reduce the latency of starting our work when ++ * the signaler completes. However, if there is any other ++ * work that we could be doing on this engine instead, that ++ * is better utilisation and will reduce the overall duration ++ * of the current work. To avoid PI boosting a semaphore ++ * far in the distance past over useful work, we keep a history ++ * of any semaphore use along our dependency chain. ++ */ ++ if (!(request->sched.flags & I915_SCHED_HAS_SEMAPHORE_CHAIN)) ++ attr.priority |= I915_PRIORITY_NOSEMAPHORE; ++ ++ /* ++ * Boost priorities to new clients (new request flows). ++ * ++ * Allow interactive/synchronous clients to jump ahead of ++ * the bulk clients. (FQ_CODEL) ++ */ ++ if (list_empty(&request->sched.signalers_list)) ++ attr.priority |= I915_PRIORITY_WAIT; ++ ++ engine->schedule(request, &attr); ++ } ++ rcu_read_unlock(); ++ i915_sw_fence_commit(&request->submit); ++ local_bh_enable(); /* Kick the execlists tasklet if just scheduled */ ++ ++ /* ++ * In typical scenarios, we do not expect the previous request on ++ * the timeline to be still tracked by timeline->last_request if it ++ * has been completed. If the completed request is still here, that ++ * implies that request retirement is a long way behind submission, ++ * suggesting that we haven't been retiring frequently enough from ++ * the combination of retire-before-alloc, waiters and the background ++ * retirement worker. So if the last request on this timeline was ++ * already completed, do a catch up pass, flushing the retirement queue ++ * up to this client. Since we have now moved the heaviest operations ++ * during retirement onto secondary workers, such as freeing objects ++ * or contexts, retiring a bunch of requests is mostly list management ++ * (and cache misses), and so we should not be overly penalizing this ++ * client by performing excess work, though we may still performing ++ * work on behalf of others -- but instead we should benefit from ++ * improved resource management. (Well, that's the theory at least.) ++ */ ++ if (prev && i915_request_completed(prev)) ++ i915_request_retire_upto(prev); ++ ++ mutex_unlock(&request->timeline->mutex); ++} ++ ++static unsigned long local_clock_us(unsigned int *cpu) ++{ ++ unsigned long t; ++ ++ /* ++ * Cheaply and approximately convert from nanoseconds to microseconds. ++ * The result and subsequent calculations are also defined in the same ++ * approximate microseconds units. The principal source of timing ++ * error here is from the simple truncation. ++ * ++ * Note that local_clock() is only defined wrt to the current CPU; ++ * the comparisons are no longer valid if we switch CPUs. Instead of ++ * blocking preemption for the entire busywait, we can detect the CPU ++ * switch and use that as indicator of system load and a reason to ++ * stop busywaiting, see busywait_stop(). ++ */ ++ *cpu = get_cpu(); ++ t = local_clock() >> 10; ++ put_cpu(); ++ ++ return t; ++} ++ ++static bool busywait_stop(unsigned long timeout, unsigned int cpu) ++{ ++ unsigned int this_cpu; ++ ++ if (time_after(local_clock_us(&this_cpu), timeout)) ++ return true; ++ ++ return this_cpu != cpu; ++} ++ ++static bool __i915_spin_request(const struct i915_request * const rq, ++ int state, unsigned long timeout_us) ++{ ++ unsigned int cpu; ++ ++ /* ++ * Only wait for the request if we know it is likely to complete. ++ * ++ * We don't track the timestamps around requests, nor the average ++ * request length, so we do not have a good indicator that this ++ * request will complete within the timeout. What we do know is the ++ * order in which requests are executed by the context and so we can ++ * tell if the request has been started. If the request is not even ++ * running yet, it is a fair assumption that it will not complete ++ * within our relatively short timeout. ++ */ ++ if (!i915_request_is_running(rq)) ++ return false; ++ ++ /* ++ * When waiting for high frequency requests, e.g. during synchronous ++ * rendering split between the CPU and GPU, the finite amount of time ++ * required to set up the irq and wait upon it limits the response ++ * rate. By busywaiting on the request completion for a short while we ++ * can service the high frequency waits as quick as possible. However, ++ * if it is a slow request, we want to sleep as quickly as possible. ++ * The tradeoff between waiting and sleeping is roughly the time it ++ * takes to sleep on a request, on the order of a microsecond. ++ */ ++ ++ timeout_us += local_clock_us(&cpu); ++ do { ++ if (i915_request_completed(rq)) ++ return true; ++ ++ if (signal_pending_state(state, current)) ++ break; ++ ++ if (busywait_stop(timeout_us, cpu)) ++ break; ++ ++ cpu_relax(); ++ } while (!need_resched()); ++ ++ return false; ++} ++ ++struct request_wait { ++ struct dma_fence_cb cb; ++ struct task_struct *tsk; ++}; ++ ++static void request_wait_wake(struct dma_fence *fence, struct dma_fence_cb *cb) ++{ ++ struct request_wait *wait = container_of(cb, typeof(*wait), cb); ++ ++ wake_up_process(wait->tsk); ++} ++ ++/** ++ * i915_request_wait - wait until execution of request has finished ++ * @rq: the request to wait upon ++ * @flags: how to wait ++ * @timeout: how long to wait in jiffies ++ * ++ * i915_request_wait() waits for the request to be completed, for a ++ * maximum of @timeout jiffies (with MAX_SCHEDULE_TIMEOUT implying an ++ * unbounded wait). ++ * ++ * If the caller holds the struct_mutex, the caller must pass I915_WAIT_LOCKED ++ * in via the flags, and vice versa if the struct_mutex is not held, the caller ++ * must not specify that the wait is locked. ++ * ++ * Returns the remaining time (in jiffies) if the request completed, which may ++ * be zero or -ETIME if the request is unfinished after the timeout expires. ++ * May return -EINTR is called with I915_WAIT_INTERRUPTIBLE and a signal is ++ * pending before the request completes. ++ */ ++long i915_request_wait(struct i915_request *rq, ++ unsigned int flags, ++ long timeout) ++{ ++ const int state = flags & I915_WAIT_INTERRUPTIBLE ? ++ TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE; ++ struct request_wait wait; ++ ++ might_sleep(); ++ GEM_BUG_ON(timeout < 0); ++ ++ if (i915_request_completed(rq)) ++ return timeout; ++ ++ if (!timeout) ++ return -ETIME; ++ ++ trace_i915_request_wait_begin(rq, flags); ++ ++ /* Optimistic short spin before touching IRQs */ ++ if (__i915_spin_request(rq, state, 5)) ++ goto out; ++ ++ /* ++ * This client is about to stall waiting for the GPU. In many cases ++ * this is undesirable and limits the throughput of the system, as ++ * many clients cannot continue processing user input/output whilst ++ * blocked. RPS autotuning may take tens of milliseconds to respond ++ * to the GPU load and thus incurs additional latency for the client. ++ * We can circumvent that by promoting the GPU frequency to maximum ++ * before we sleep. This makes the GPU throttle up much more quickly ++ * (good for benchmarks and user experience, e.g. window animations), ++ * but at a cost of spending more power processing the workload ++ * (bad for battery). ++ */ ++ if (flags & I915_WAIT_PRIORITY) { ++ if (!i915_request_started(rq) && INTEL_GEN(rq->i915) >= 6) ++ gen6_rps_boost(rq); ++ local_bh_disable(); /* suspend tasklets for reprioritisation */ ++ i915_schedule_bump_priority(rq, I915_PRIORITY_WAIT); ++ local_bh_enable(); /* kick tasklets en masse */ ++ } ++ ++ wait.tsk = current; ++ if (dma_fence_add_callback(&rq->fence, &wait.cb, request_wait_wake)) ++ goto out; ++ ++ for (;;) { ++ set_current_state(state); ++ ++ if (i915_request_completed(rq)) ++ break; ++ ++ if (signal_pending_state(state, current)) { ++ timeout = -ERESTARTSYS; ++ break; ++ } ++ ++ if (!timeout) { ++ timeout = -ETIME; ++ break; ++ } ++ ++ timeout = io_schedule_timeout(timeout); ++ } ++ __set_current_state(TASK_RUNNING); ++ ++ dma_fence_remove_callback(&rq->fence, &wait.cb); ++ ++out: ++ trace_i915_request_wait_end(rq); ++ return timeout; ++} ++ ++void i915_retire_requests(struct drm_i915_private *i915) ++{ ++ struct intel_ring *ring, *tmp; ++ ++ lockdep_assert_held(&i915->drm.struct_mutex); ++ ++ if (!i915->gt.active_requests) ++ return; ++ ++ list_for_each_entry_safe(ring, tmp, ++ &i915->gt.active_rings, active_link) { ++ intel_ring_get(ring); /* last rq holds reference! */ ++ ring_retire_requests(ring); ++ intel_ring_put(ring); ++ } ++} ++ ++#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) ++#include "selftests/mock_request.c" ++#include "selftests/i915_request.c" ++#endif ++ ++static void i915_global_request_shrink(void) ++{ ++ kmem_cache_shrink(global.slab_dependencies); ++ kmem_cache_shrink(global.slab_execute_cbs); ++ kmem_cache_shrink(global.slab_requests); ++} ++ ++static void i915_global_request_exit(void) ++{ ++ kmem_cache_destroy(global.slab_dependencies); ++ kmem_cache_destroy(global.slab_execute_cbs); ++ kmem_cache_destroy(global.slab_requests); ++} ++ ++static struct i915_global_request global = { { ++ .shrink = i915_global_request_shrink, ++ .exit = i915_global_request_exit, ++} }; ++ ++int __init i915_global_request_init(void) ++{ ++ global.slab_requests = KMEM_CACHE(i915_request, ++ SLAB_HWCACHE_ALIGN | ++ SLAB_RECLAIM_ACCOUNT | ++ SLAB_TYPESAFE_BY_RCU); ++ if (!global.slab_requests) ++ return -ENOMEM; ++ ++ global.slab_execute_cbs = KMEM_CACHE(execute_cb, ++ SLAB_HWCACHE_ALIGN | ++ SLAB_RECLAIM_ACCOUNT | ++ SLAB_TYPESAFE_BY_RCU); ++ if (!global.slab_execute_cbs) ++ goto err_requests; ++ ++ global.slab_dependencies = KMEM_CACHE(i915_dependency, ++ SLAB_HWCACHE_ALIGN | ++ SLAB_RECLAIM_ACCOUNT); ++ if (!global.slab_dependencies) ++ goto err_execute_cbs; ++ ++ i915_global_register(&global.base); ++ return 0; ++ ++err_execute_cbs: ++ kmem_cache_destroy(global.slab_execute_cbs); ++err_requests: ++ kmem_cache_destroy(global.slab_requests); ++ return -ENOMEM; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_request.h b/drivers/gpu/drm/i915_legacy/i915_request.h +new file mode 100644 +index 000000000000..a982664618c2 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_request.h +@@ -0,0 +1,423 @@ ++/* ++ * Copyright © 2008-2018 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#ifndef I915_REQUEST_H ++#define I915_REQUEST_H ++ ++#include ++#include ++ ++#include "i915_gem.h" ++#include "i915_scheduler.h" ++#include "i915_selftest.h" ++#include "i915_sw_fence.h" ++ ++#include ++ ++struct drm_file; ++struct drm_i915_gem_object; ++struct i915_request; ++struct i915_timeline; ++struct i915_timeline_cacheline; ++ ++struct i915_capture_list { ++ struct i915_capture_list *next; ++ struct i915_vma *vma; ++}; ++ ++enum { ++ /* ++ * I915_FENCE_FLAG_ACTIVE - this request is currently submitted to HW. ++ * ++ * Set by __i915_request_submit() on handing over to HW, and cleared ++ * by __i915_request_unsubmit() if we preempt this request. ++ * ++ * Finally cleared for consistency on retiring the request, when ++ * we know the HW is no longer running this request. ++ * ++ * See i915_request_is_active() ++ */ ++ I915_FENCE_FLAG_ACTIVE = DMA_FENCE_FLAG_USER_BITS, ++ ++ /* ++ * I915_FENCE_FLAG_SIGNAL - this request is currently on signal_list ++ * ++ * Internal bookkeeping used by the breadcrumb code to track when ++ * a request is on the various signal_list. ++ */ ++ I915_FENCE_FLAG_SIGNAL, ++}; ++ ++/** ++ * Request queue structure. ++ * ++ * The request queue allows us to note sequence numbers that have been emitted ++ * and may be associated with active buffers to be retired. ++ * ++ * By keeping this list, we can avoid having to do questionable sequence ++ * number comparisons on buffer last_read|write_seqno. It also allows an ++ * emission time to be associated with the request for tracking how far ahead ++ * of the GPU the submission is. ++ * ++ * When modifying this structure be very aware that we perform a lockless ++ * RCU lookup of it that may race against reallocation of the struct ++ * from the slab freelist. We intentionally do not zero the structure on ++ * allocation so that the lookup can use the dangling pointers (and is ++ * cogniscent that those pointers may be wrong). Instead, everything that ++ * needs to be initialised must be done so explicitly. ++ * ++ * The requests are reference counted. ++ */ ++struct i915_request { ++ struct dma_fence fence; ++ spinlock_t lock; ++ ++ /** On Which ring this request was generated */ ++ struct drm_i915_private *i915; ++ ++ /** ++ * Context and ring buffer related to this request ++ * Contexts are refcounted, so when this request is associated with a ++ * context, we must increment the context's refcount, to guarantee that ++ * it persists while any request is linked to it. Requests themselves ++ * are also refcounted, so the request will only be freed when the last ++ * reference to it is dismissed, and the code in ++ * i915_request_free() will then decrement the refcount on the ++ * context. ++ */ ++ struct i915_gem_context *gem_context; ++ struct intel_engine_cs *engine; ++ struct intel_context *hw_context; ++ struct intel_ring *ring; ++ struct i915_timeline *timeline; ++ struct list_head signal_link; ++ ++ /* ++ * The rcu epoch of when this request was allocated. Used to judiciously ++ * apply backpressure on future allocations to ensure that under ++ * mempressure there is sufficient RCU ticks for us to reclaim our ++ * RCU protected slabs. ++ */ ++ unsigned long rcustate; ++ ++ /* ++ * We pin the timeline->mutex while constructing the request to ++ * ensure that no caller accidentally drops it during construction. ++ * The timeline->mutex must be held to ensure that only this caller ++ * can use the ring and manipulate the associated timeline during ++ * construction. ++ */ ++ struct pin_cookie cookie; ++ ++ /* ++ * Fences for the various phases in the request's lifetime. ++ * ++ * The submit fence is used to await upon all of the request's ++ * dependencies. When it is signaled, the request is ready to run. ++ * It is used by the driver to then queue the request for execution. ++ */ ++ struct i915_sw_fence submit; ++ union { ++ wait_queue_entry_t submitq; ++ struct i915_sw_dma_fence_cb dmaq; ++ }; ++ struct list_head execute_cb; ++ struct i915_sw_fence semaphore; ++ ++ /* ++ * A list of everyone we wait upon, and everyone who waits upon us. ++ * Even though we will not be submitted to the hardware before the ++ * submit fence is signaled (it waits for all external events as well ++ * as our own requests), the scheduler still needs to know the ++ * dependency tree for the lifetime of the request (from execbuf ++ * to retirement), i.e. bidirectional dependency information for the ++ * request not tied to individual fences. ++ */ ++ struct i915_sched_node sched; ++ struct i915_dependency dep; ++ ++ /* ++ * A convenience pointer to the current breadcrumb value stored in ++ * the HW status page (or our timeline's local equivalent). The full ++ * path would be rq->hw_context->ring->timeline->hwsp_seqno. ++ */ ++ const u32 *hwsp_seqno; ++ ++ /* ++ * If we need to access the timeline's seqno for this request in ++ * another request, we need to keep a read reference to this associated ++ * cacheline, so that we do not free and recycle it before the foreign ++ * observers have completed. Hence, we keep a pointer to the cacheline ++ * inside the timeline's HWSP vma, but it is only valid while this ++ * request has not completed and guarded by the timeline mutex. ++ */ ++ struct i915_timeline_cacheline *hwsp_cacheline; ++ ++ /** Position in the ring of the start of the request */ ++ u32 head; ++ ++ /** Position in the ring of the start of the user packets */ ++ u32 infix; ++ ++ /** ++ * Position in the ring of the start of the postfix. ++ * This is required to calculate the maximum available ring space ++ * without overwriting the postfix. ++ */ ++ u32 postfix; ++ ++ /** Position in the ring of the end of the whole request */ ++ u32 tail; ++ ++ /** Position in the ring of the end of any workarounds after the tail */ ++ u32 wa_tail; ++ ++ /** Preallocate space in the ring for the emitting the request */ ++ u32 reserved_space; ++ ++ /** Batch buffer related to this request if any (used for ++ * error state dump only). ++ */ ++ struct i915_vma *batch; ++ /** ++ * Additional buffers requested by userspace to be captured upon ++ * a GPU hang. The vma/obj on this list are protected by their ++ * active reference - all objects on this list must also be ++ * on the active_list (of their final request). ++ */ ++ struct i915_capture_list *capture_list; ++ struct list_head active_list; ++ ++ /** Time at which this request was emitted, in jiffies. */ ++ unsigned long emitted_jiffies; ++ ++ bool waitboost; ++ ++ /** engine->request_list entry for this request */ ++ struct list_head link; ++ ++ /** ring->request_list entry for this request */ ++ struct list_head ring_link; ++ ++ struct drm_i915_file_private *file_priv; ++ /** file_priv list entry for this request */ ++ struct list_head client_link; ++ ++ I915_SELFTEST_DECLARE(struct { ++ struct list_head link; ++ unsigned long delay; ++ } mock;) ++}; ++ ++#define I915_FENCE_GFP (GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN) ++ ++extern const struct dma_fence_ops i915_fence_ops; ++ ++static inline bool dma_fence_is_i915(const struct dma_fence *fence) ++{ ++ return fence->ops == &i915_fence_ops; ++} ++ ++struct i915_request * __must_check ++i915_request_alloc(struct intel_engine_cs *engine, ++ struct i915_gem_context *ctx); ++void i915_request_retire_upto(struct i915_request *rq); ++ ++static inline struct i915_request * ++to_request(struct dma_fence *fence) ++{ ++ /* We assume that NULL fence/request are interoperable */ ++ BUILD_BUG_ON(offsetof(struct i915_request, fence) != 0); ++ GEM_BUG_ON(fence && !dma_fence_is_i915(fence)); ++ return container_of(fence, struct i915_request, fence); ++} ++ ++static inline struct i915_request * ++i915_request_get(struct i915_request *rq) ++{ ++ return to_request(dma_fence_get(&rq->fence)); ++} ++ ++static inline struct i915_request * ++i915_request_get_rcu(struct i915_request *rq) ++{ ++ return to_request(dma_fence_get_rcu(&rq->fence)); ++} ++ ++static inline void ++i915_request_put(struct i915_request *rq) ++{ ++ dma_fence_put(&rq->fence); ++} ++ ++int i915_request_await_object(struct i915_request *to, ++ struct drm_i915_gem_object *obj, ++ bool write); ++int i915_request_await_dma_fence(struct i915_request *rq, ++ struct dma_fence *fence); ++ ++void i915_request_add(struct i915_request *rq); ++ ++void __i915_request_submit(struct i915_request *request); ++void i915_request_submit(struct i915_request *request); ++ ++void i915_request_skip(struct i915_request *request, int error); ++ ++void __i915_request_unsubmit(struct i915_request *request); ++void i915_request_unsubmit(struct i915_request *request); ++ ++/* Note: part of the intel_breadcrumbs family */ ++bool i915_request_enable_breadcrumb(struct i915_request *request); ++void i915_request_cancel_breadcrumb(struct i915_request *request); ++ ++long i915_request_wait(struct i915_request *rq, ++ unsigned int flags, ++ long timeout) ++ __attribute__((nonnull(1))); ++#define I915_WAIT_INTERRUPTIBLE BIT(0) ++#define I915_WAIT_LOCKED BIT(1) /* struct_mutex held, handle GPU reset */ ++#define I915_WAIT_PRIORITY BIT(2) /* small priority bump for the request */ ++#define I915_WAIT_ALL BIT(3) /* used by i915_gem_object_wait() */ ++#define I915_WAIT_FOR_IDLE_BOOST BIT(4) ++ ++static inline bool i915_request_signaled(const struct i915_request *rq) ++{ ++ /* The request may live longer than its HWSP, so check flags first! */ ++ return test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags); ++} ++ ++static inline bool i915_request_is_active(const struct i915_request *rq) ++{ ++ return test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags); ++} ++ ++/** ++ * Returns true if seq1 is later than seq2. ++ */ ++static inline bool i915_seqno_passed(u32 seq1, u32 seq2) ++{ ++ return (s32)(seq1 - seq2) >= 0; ++} ++ ++static inline u32 __hwsp_seqno(const struct i915_request *rq) ++{ ++ return READ_ONCE(*rq->hwsp_seqno); ++} ++ ++/** ++ * hwsp_seqno - the current breadcrumb value in the HW status page ++ * @rq: the request, to chase the relevant HW status page ++ * ++ * The emphasis in naming here is that hwsp_seqno() is not a property of the ++ * request, but an indication of the current HW state (associated with this ++ * request). Its value will change as the GPU executes more requests. ++ * ++ * Returns the current breadcrumb value in the associated HW status page (or ++ * the local timeline's equivalent) for this request. The request itself ++ * has the associated breadcrumb value of rq->fence.seqno, when the HW ++ * status page has that breadcrumb or later, this request is complete. ++ */ ++static inline u32 hwsp_seqno(const struct i915_request *rq) ++{ ++ u32 seqno; ++ ++ rcu_read_lock(); /* the HWSP may be freed at runtime */ ++ seqno = __hwsp_seqno(rq); ++ rcu_read_unlock(); ++ ++ return seqno; ++} ++ ++static inline bool __i915_request_has_started(const struct i915_request *rq) ++{ ++ return i915_seqno_passed(hwsp_seqno(rq), rq->fence.seqno - 1); ++} ++ ++/** ++ * i915_request_started - check if the request has begun being executed ++ * @rq: the request ++ * ++ * If the timeline is not using initial breadcrumbs, a request is ++ * considered started if the previous request on its timeline (i.e. ++ * context) has been signaled. ++ * ++ * If the timeline is using semaphores, it will also be emitting an ++ * "initial breadcrumb" after the semaphores are complete and just before ++ * it began executing the user payload. A request can therefore be active ++ * on the HW and not yet started as it is still busywaiting on its ++ * dependencies (via HW semaphores). ++ * ++ * If the request has started, its dependencies will have been signaled ++ * (either by fences or by semaphores) and it will have begun processing ++ * the user payload. ++ * ++ * However, even if a request has started, it may have been preempted and ++ * so no longer active, or it may have already completed. ++ * ++ * See also i915_request_is_active(). ++ * ++ * Returns true if the request has begun executing the user payload, or ++ * has completed: ++ */ ++static inline bool i915_request_started(const struct i915_request *rq) ++{ ++ if (i915_request_signaled(rq)) ++ return true; ++ ++ /* Remember: started but may have since been preempted! */ ++ return __i915_request_has_started(rq); ++} ++ ++/** ++ * i915_request_is_running - check if the request may actually be executing ++ * @rq: the request ++ * ++ * Returns true if the request is currently submitted to hardware, has passed ++ * its start point (i.e. the context is setup and not busywaiting). Note that ++ * it may no longer be running by the time the function returns! ++ */ ++static inline bool i915_request_is_running(const struct i915_request *rq) ++{ ++ if (!i915_request_is_active(rq)) ++ return false; ++ ++ return __i915_request_has_started(rq); ++} ++ ++static inline bool i915_request_completed(const struct i915_request *rq) ++{ ++ if (i915_request_signaled(rq)) ++ return true; ++ ++ return i915_seqno_passed(hwsp_seqno(rq), rq->fence.seqno); ++} ++ ++static inline void i915_request_mark_complete(struct i915_request *rq) ++{ ++ rq->hwsp_seqno = (u32 *)&rq->fence.seqno; /* decouple from HWSP */ ++} ++ ++void i915_retire_requests(struct drm_i915_private *i915); ++ ++#endif /* I915_REQUEST_H */ +diff --git a/drivers/gpu/drm/i915_legacy/i915_reset.c b/drivers/gpu/drm/i915_legacy/i915_reset.c +new file mode 100644 +index 000000000000..677d59304e78 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_reset.c +@@ -0,0 +1,1474 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2008-2018 Intel Corporation ++ */ ++ ++#include ++#include ++ ++#include "i915_drv.h" ++#include "i915_gpu_error.h" ++#include "i915_reset.h" ++ ++#include "intel_guc.h" ++ ++#define RESET_MAX_RETRIES 3 ++ ++/* XXX How to handle concurrent GGTT updates using tiling registers? */ ++#define RESET_UNDER_STOP_MACHINE 0 ++ ++static void rmw_set(struct intel_uncore *uncore, i915_reg_t reg, u32 set) ++{ ++ intel_uncore_rmw(uncore, reg, 0, set); ++} ++ ++static void rmw_clear(struct intel_uncore *uncore, i915_reg_t reg, u32 clr) ++{ ++ intel_uncore_rmw(uncore, reg, clr, 0); ++} ++ ++static void rmw_set_fw(struct intel_uncore *uncore, i915_reg_t reg, u32 set) ++{ ++ intel_uncore_rmw_fw(uncore, reg, 0, set); ++} ++ ++static void rmw_clear_fw(struct intel_uncore *uncore, i915_reg_t reg, u32 clr) ++{ ++ intel_uncore_rmw_fw(uncore, reg, clr, 0); ++} ++ ++static void engine_skip_context(struct i915_request *rq) ++{ ++ struct intel_engine_cs *engine = rq->engine; ++ struct i915_gem_context *hung_ctx = rq->gem_context; ++ ++ lockdep_assert_held(&engine->timeline.lock); ++ ++ if (!i915_request_is_active(rq)) ++ return; ++ ++ list_for_each_entry_continue(rq, &engine->timeline.requests, link) ++ if (rq->gem_context == hung_ctx) ++ i915_request_skip(rq, -EIO); ++} ++ ++static void client_mark_guilty(struct drm_i915_file_private *file_priv, ++ const struct i915_gem_context *ctx) ++{ ++ unsigned int score; ++ unsigned long prev_hang; ++ ++ if (i915_gem_context_is_banned(ctx)) ++ score = I915_CLIENT_SCORE_CONTEXT_BAN; ++ else ++ score = 0; ++ ++ prev_hang = xchg(&file_priv->hang_timestamp, jiffies); ++ if (time_before(jiffies, prev_hang + I915_CLIENT_FAST_HANG_JIFFIES)) ++ score += I915_CLIENT_SCORE_HANG_FAST; ++ ++ if (score) { ++ atomic_add(score, &file_priv->ban_score); ++ ++ DRM_DEBUG_DRIVER("client %s: gained %u ban score, now %u\n", ++ ctx->name, score, ++ atomic_read(&file_priv->ban_score)); ++ } ++} ++ ++static bool context_mark_guilty(struct i915_gem_context *ctx) ++{ ++ unsigned long prev_hang; ++ bool banned; ++ int i; ++ ++ atomic_inc(&ctx->guilty_count); ++ ++ /* Cool contexts are too cool to be banned! (Used for reset testing.) */ ++ if (!i915_gem_context_is_bannable(ctx)) ++ return false; ++ ++ /* Record the timestamp for the last N hangs */ ++ prev_hang = ctx->hang_timestamp[0]; ++ for (i = 0; i < ARRAY_SIZE(ctx->hang_timestamp) - 1; i++) ++ ctx->hang_timestamp[i] = ctx->hang_timestamp[i + 1]; ++ ctx->hang_timestamp[i] = jiffies; ++ ++ /* If we have hung N+1 times in rapid succession, we ban the context! */ ++ banned = !i915_gem_context_is_recoverable(ctx); ++ if (time_before(jiffies, prev_hang + CONTEXT_FAST_HANG_JIFFIES)) ++ banned = true; ++ if (banned) { ++ DRM_DEBUG_DRIVER("context %s: guilty %d, banned\n", ++ ctx->name, atomic_read(&ctx->guilty_count)); ++ i915_gem_context_set_banned(ctx); ++ } ++ ++ if (!IS_ERR_OR_NULL(ctx->file_priv)) ++ client_mark_guilty(ctx->file_priv, ctx); ++ ++ return banned; ++} ++ ++static void context_mark_innocent(struct i915_gem_context *ctx) ++{ ++ atomic_inc(&ctx->active_count); ++} ++ ++void i915_reset_request(struct i915_request *rq, bool guilty) ++{ ++ GEM_TRACE("%s rq=%llx:%lld, guilty? %s\n", ++ rq->engine->name, ++ rq->fence.context, ++ rq->fence.seqno, ++ yesno(guilty)); ++ ++ lockdep_assert_held(&rq->engine->timeline.lock); ++ GEM_BUG_ON(i915_request_completed(rq)); ++ ++ if (guilty) { ++ i915_request_skip(rq, -EIO); ++ if (context_mark_guilty(rq->gem_context)) ++ engine_skip_context(rq); ++ } else { ++ dma_fence_set_error(&rq->fence, -EAGAIN); ++ context_mark_innocent(rq->gem_context); ++ } ++} ++ ++static void gen3_stop_engine(struct intel_engine_cs *engine) ++{ ++ struct intel_uncore *uncore = engine->uncore; ++ const u32 base = engine->mmio_base; ++ ++ GEM_TRACE("%s\n", engine->name); ++ ++ if (intel_engine_stop_cs(engine)) ++ GEM_TRACE("%s: timed out on STOP_RING\n", engine->name); ++ ++ intel_uncore_write_fw(uncore, ++ RING_HEAD(base), ++ intel_uncore_read_fw(uncore, RING_TAIL(base))); ++ intel_uncore_posting_read_fw(uncore, RING_HEAD(base)); /* paranoia */ ++ ++ intel_uncore_write_fw(uncore, RING_HEAD(base), 0); ++ intel_uncore_write_fw(uncore, RING_TAIL(base), 0); ++ intel_uncore_posting_read_fw(uncore, RING_TAIL(base)); ++ ++ /* The ring must be empty before it is disabled */ ++ intel_uncore_write_fw(uncore, RING_CTL(base), 0); ++ ++ /* Check acts as a post */ ++ if (intel_uncore_read_fw(uncore, RING_HEAD(base))) ++ GEM_TRACE("%s: ring head [%x] not parked\n", ++ engine->name, ++ intel_uncore_read_fw(uncore, RING_HEAD(base))); ++} ++ ++static void i915_stop_engines(struct drm_i915_private *i915, ++ intel_engine_mask_t engine_mask) ++{ ++ struct intel_engine_cs *engine; ++ intel_engine_mask_t tmp; ++ ++ if (INTEL_GEN(i915) < 3) ++ return; ++ ++ for_each_engine_masked(engine, i915, engine_mask, tmp) ++ gen3_stop_engine(engine); ++} ++ ++static bool i915_in_reset(struct pci_dev *pdev) ++{ ++ u8 gdrst; ++ ++ pci_read_config_byte(pdev, I915_GDRST, &gdrst); ++ return gdrst & GRDOM_RESET_STATUS; ++} ++ ++static int i915_do_reset(struct drm_i915_private *i915, ++ intel_engine_mask_t engine_mask, ++ unsigned int retry) ++{ ++ struct pci_dev *pdev = i915->drm.pdev; ++ int err; ++ ++ /* Assert reset for at least 20 usec, and wait for acknowledgement. */ ++ pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE); ++ udelay(50); ++ err = wait_for_atomic(i915_in_reset(pdev), 50); ++ ++ /* Clear the reset request. */ ++ pci_write_config_byte(pdev, I915_GDRST, 0); ++ udelay(50); ++ if (!err) ++ err = wait_for_atomic(!i915_in_reset(pdev), 50); ++ ++ return err; ++} ++ ++static bool g4x_reset_complete(struct pci_dev *pdev) ++{ ++ u8 gdrst; ++ ++ pci_read_config_byte(pdev, I915_GDRST, &gdrst); ++ return (gdrst & GRDOM_RESET_ENABLE) == 0; ++} ++ ++static int g33_do_reset(struct drm_i915_private *i915, ++ intel_engine_mask_t engine_mask, ++ unsigned int retry) ++{ ++ struct pci_dev *pdev = i915->drm.pdev; ++ ++ pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE); ++ return wait_for_atomic(g4x_reset_complete(pdev), 50); ++} ++ ++static int g4x_do_reset(struct drm_i915_private *i915, ++ intel_engine_mask_t engine_mask, ++ unsigned int retry) ++{ ++ struct pci_dev *pdev = i915->drm.pdev; ++ struct intel_uncore *uncore = &i915->uncore; ++ int ret; ++ ++ /* WaVcpClkGateDisableForMediaReset:ctg,elk */ ++ rmw_set_fw(uncore, VDECCLK_GATE_D, VCP_UNIT_CLOCK_GATE_DISABLE); ++ intel_uncore_posting_read_fw(uncore, VDECCLK_GATE_D); ++ ++ pci_write_config_byte(pdev, I915_GDRST, ++ GRDOM_MEDIA | GRDOM_RESET_ENABLE); ++ ret = wait_for_atomic(g4x_reset_complete(pdev), 50); ++ if (ret) { ++ DRM_DEBUG_DRIVER("Wait for media reset failed\n"); ++ goto out; ++ } ++ ++ pci_write_config_byte(pdev, I915_GDRST, ++ GRDOM_RENDER | GRDOM_RESET_ENABLE); ++ ret = wait_for_atomic(g4x_reset_complete(pdev), 50); ++ if (ret) { ++ DRM_DEBUG_DRIVER("Wait for render reset failed\n"); ++ goto out; ++ } ++ ++out: ++ pci_write_config_byte(pdev, I915_GDRST, 0); ++ ++ rmw_clear_fw(uncore, VDECCLK_GATE_D, VCP_UNIT_CLOCK_GATE_DISABLE); ++ intel_uncore_posting_read_fw(uncore, VDECCLK_GATE_D); ++ ++ return ret; ++} ++ ++static int ironlake_do_reset(struct drm_i915_private *i915, ++ intel_engine_mask_t engine_mask, ++ unsigned int retry) ++{ ++ struct intel_uncore *uncore = &i915->uncore; ++ int ret; ++ ++ intel_uncore_write_fw(uncore, ILK_GDSR, ++ ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE); ++ ret = __intel_wait_for_register_fw(uncore, ILK_GDSR, ++ ILK_GRDOM_RESET_ENABLE, 0, ++ 5000, 0, ++ NULL); ++ if (ret) { ++ DRM_DEBUG_DRIVER("Wait for render reset failed\n"); ++ goto out; ++ } ++ ++ intel_uncore_write_fw(uncore, ILK_GDSR, ++ ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE); ++ ret = __intel_wait_for_register_fw(uncore, ILK_GDSR, ++ ILK_GRDOM_RESET_ENABLE, 0, ++ 5000, 0, ++ NULL); ++ if (ret) { ++ DRM_DEBUG_DRIVER("Wait for media reset failed\n"); ++ goto out; ++ } ++ ++out: ++ intel_uncore_write_fw(uncore, ILK_GDSR, 0); ++ intel_uncore_posting_read_fw(uncore, ILK_GDSR); ++ return ret; ++} ++ ++/* Reset the hardware domains (GENX_GRDOM_*) specified by mask */ ++static int gen6_hw_domain_reset(struct drm_i915_private *i915, ++ u32 hw_domain_mask) ++{ ++ struct intel_uncore *uncore = &i915->uncore; ++ int err; ++ ++ /* ++ * GEN6_GDRST is not in the gt power well, no need to check ++ * for fifo space for the write or forcewake the chip for ++ * the read ++ */ ++ intel_uncore_write_fw(uncore, GEN6_GDRST, hw_domain_mask); ++ ++ /* Wait for the device to ack the reset requests */ ++ err = __intel_wait_for_register_fw(uncore, ++ GEN6_GDRST, hw_domain_mask, 0, ++ 500, 0, ++ NULL); ++ if (err) ++ DRM_DEBUG_DRIVER("Wait for 0x%08x engines reset failed\n", ++ hw_domain_mask); ++ ++ return err; ++} ++ ++static int gen6_reset_engines(struct drm_i915_private *i915, ++ intel_engine_mask_t engine_mask, ++ unsigned int retry) ++{ ++ struct intel_engine_cs *engine; ++ const u32 hw_engine_mask[] = { ++ [RCS0] = GEN6_GRDOM_RENDER, ++ [BCS0] = GEN6_GRDOM_BLT, ++ [VCS0] = GEN6_GRDOM_MEDIA, ++ [VCS1] = GEN8_GRDOM_MEDIA2, ++ [VECS0] = GEN6_GRDOM_VECS, ++ }; ++ u32 hw_mask; ++ ++ if (engine_mask == ALL_ENGINES) { ++ hw_mask = GEN6_GRDOM_FULL; ++ } else { ++ intel_engine_mask_t tmp; ++ ++ hw_mask = 0; ++ for_each_engine_masked(engine, i915, engine_mask, tmp) { ++ GEM_BUG_ON(engine->id >= ARRAY_SIZE(hw_engine_mask)); ++ hw_mask |= hw_engine_mask[engine->id]; ++ } ++ } ++ ++ return gen6_hw_domain_reset(i915, hw_mask); ++} ++ ++static u32 gen11_lock_sfc(struct intel_engine_cs *engine) ++{ ++ struct intel_uncore *uncore = engine->uncore; ++ u8 vdbox_sfc_access = RUNTIME_INFO(engine->i915)->vdbox_sfc_access; ++ i915_reg_t sfc_forced_lock, sfc_forced_lock_ack; ++ u32 sfc_forced_lock_bit, sfc_forced_lock_ack_bit; ++ i915_reg_t sfc_usage; ++ u32 sfc_usage_bit; ++ u32 sfc_reset_bit; ++ ++ switch (engine->class) { ++ case VIDEO_DECODE_CLASS: ++ if ((BIT(engine->instance) & vdbox_sfc_access) == 0) ++ return 0; ++ ++ sfc_forced_lock = GEN11_VCS_SFC_FORCED_LOCK(engine); ++ sfc_forced_lock_bit = GEN11_VCS_SFC_FORCED_LOCK_BIT; ++ ++ sfc_forced_lock_ack = GEN11_VCS_SFC_LOCK_STATUS(engine); ++ sfc_forced_lock_ack_bit = GEN11_VCS_SFC_LOCK_ACK_BIT; ++ ++ sfc_usage = GEN11_VCS_SFC_LOCK_STATUS(engine); ++ sfc_usage_bit = GEN11_VCS_SFC_USAGE_BIT; ++ sfc_reset_bit = GEN11_VCS_SFC_RESET_BIT(engine->instance); ++ break; ++ ++ case VIDEO_ENHANCEMENT_CLASS: ++ sfc_forced_lock = GEN11_VECS_SFC_FORCED_LOCK(engine); ++ sfc_forced_lock_bit = GEN11_VECS_SFC_FORCED_LOCK_BIT; ++ ++ sfc_forced_lock_ack = GEN11_VECS_SFC_LOCK_ACK(engine); ++ sfc_forced_lock_ack_bit = GEN11_VECS_SFC_LOCK_ACK_BIT; ++ ++ sfc_usage = GEN11_VECS_SFC_USAGE(engine); ++ sfc_usage_bit = GEN11_VECS_SFC_USAGE_BIT; ++ sfc_reset_bit = GEN11_VECS_SFC_RESET_BIT(engine->instance); ++ break; ++ ++ default: ++ return 0; ++ } ++ ++ /* ++ * Tell the engine that a software reset is going to happen. The engine ++ * will then try to force lock the SFC (if currently locked, it will ++ * remain so until we tell the engine it is safe to unlock; if currently ++ * unlocked, it will ignore this and all new lock requests). If SFC ++ * ends up being locked to the engine we want to reset, we have to reset ++ * it as well (we will unlock it once the reset sequence is completed). ++ */ ++ rmw_set_fw(uncore, sfc_forced_lock, sfc_forced_lock_bit); ++ ++ if (__intel_wait_for_register_fw(uncore, ++ sfc_forced_lock_ack, ++ sfc_forced_lock_ack_bit, ++ sfc_forced_lock_ack_bit, ++ 1000, 0, NULL)) { ++ DRM_DEBUG_DRIVER("Wait for SFC forced lock ack failed\n"); ++ return 0; ++ } ++ ++ if (intel_uncore_read_fw(uncore, sfc_usage) & sfc_usage_bit) ++ return sfc_reset_bit; ++ ++ return 0; ++} ++ ++static void gen11_unlock_sfc(struct intel_engine_cs *engine) ++{ ++ struct intel_uncore *uncore = engine->uncore; ++ u8 vdbox_sfc_access = RUNTIME_INFO(engine->i915)->vdbox_sfc_access; ++ i915_reg_t sfc_forced_lock; ++ u32 sfc_forced_lock_bit; ++ ++ switch (engine->class) { ++ case VIDEO_DECODE_CLASS: ++ if ((BIT(engine->instance) & vdbox_sfc_access) == 0) ++ return; ++ ++ sfc_forced_lock = GEN11_VCS_SFC_FORCED_LOCK(engine); ++ sfc_forced_lock_bit = GEN11_VCS_SFC_FORCED_LOCK_BIT; ++ break; ++ ++ case VIDEO_ENHANCEMENT_CLASS: ++ sfc_forced_lock = GEN11_VECS_SFC_FORCED_LOCK(engine); ++ sfc_forced_lock_bit = GEN11_VECS_SFC_FORCED_LOCK_BIT; ++ break; ++ ++ default: ++ return; ++ } ++ ++ rmw_clear_fw(uncore, sfc_forced_lock, sfc_forced_lock_bit); ++} ++ ++static int gen11_reset_engines(struct drm_i915_private *i915, ++ intel_engine_mask_t engine_mask, ++ unsigned int retry) ++{ ++ const u32 hw_engine_mask[] = { ++ [RCS0] = GEN11_GRDOM_RENDER, ++ [BCS0] = GEN11_GRDOM_BLT, ++ [VCS0] = GEN11_GRDOM_MEDIA, ++ [VCS1] = GEN11_GRDOM_MEDIA2, ++ [VCS2] = GEN11_GRDOM_MEDIA3, ++ [VCS3] = GEN11_GRDOM_MEDIA4, ++ [VECS0] = GEN11_GRDOM_VECS, ++ [VECS1] = GEN11_GRDOM_VECS2, ++ }; ++ struct intel_engine_cs *engine; ++ intel_engine_mask_t tmp; ++ u32 hw_mask; ++ int ret; ++ ++ if (engine_mask == ALL_ENGINES) { ++ hw_mask = GEN11_GRDOM_FULL; ++ } else { ++ hw_mask = 0; ++ for_each_engine_masked(engine, i915, engine_mask, tmp) { ++ GEM_BUG_ON(engine->id >= ARRAY_SIZE(hw_engine_mask)); ++ hw_mask |= hw_engine_mask[engine->id]; ++ hw_mask |= gen11_lock_sfc(engine); ++ } ++ } ++ ++ ret = gen6_hw_domain_reset(i915, hw_mask); ++ ++ if (engine_mask != ALL_ENGINES) ++ for_each_engine_masked(engine, i915, engine_mask, tmp) ++ gen11_unlock_sfc(engine); ++ ++ return ret; ++} ++ ++static int gen8_engine_reset_prepare(struct intel_engine_cs *engine) ++{ ++ struct intel_uncore *uncore = engine->uncore; ++ const i915_reg_t reg = RING_RESET_CTL(engine->mmio_base); ++ u32 request, mask, ack; ++ int ret; ++ ++ ack = intel_uncore_read_fw(uncore, reg); ++ if (ack & RESET_CTL_CAT_ERROR) { ++ /* ++ * For catastrophic errors, ready-for-reset sequence ++ * needs to be bypassed: HAS#396813 ++ */ ++ request = RESET_CTL_CAT_ERROR; ++ mask = RESET_CTL_CAT_ERROR; ++ ++ /* Catastrophic errors need to be cleared by HW */ ++ ack = 0; ++ } else if (!(ack & RESET_CTL_READY_TO_RESET)) { ++ request = RESET_CTL_REQUEST_RESET; ++ mask = RESET_CTL_READY_TO_RESET; ++ ack = RESET_CTL_READY_TO_RESET; ++ } else { ++ return 0; ++ } ++ ++ intel_uncore_write_fw(uncore, reg, _MASKED_BIT_ENABLE(request)); ++ ret = __intel_wait_for_register_fw(uncore, reg, mask, ack, ++ 700, 0, NULL); ++ if (ret) ++ DRM_ERROR("%s reset request timed out: {request: %08x, RESET_CTL: %08x}\n", ++ engine->name, request, ++ intel_uncore_read_fw(uncore, reg)); ++ ++ return ret; ++} ++ ++static void gen8_engine_reset_cancel(struct intel_engine_cs *engine) ++{ ++ intel_uncore_write_fw(engine->uncore, ++ RING_RESET_CTL(engine->mmio_base), ++ _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET)); ++} ++ ++static int gen8_reset_engines(struct drm_i915_private *i915, ++ intel_engine_mask_t engine_mask, ++ unsigned int retry) ++{ ++ struct intel_engine_cs *engine; ++ const bool reset_non_ready = retry >= 1; ++ intel_engine_mask_t tmp; ++ int ret; ++ ++ for_each_engine_masked(engine, i915, engine_mask, tmp) { ++ ret = gen8_engine_reset_prepare(engine); ++ if (ret && !reset_non_ready) ++ goto skip_reset; ++ ++ /* ++ * If this is not the first failed attempt to prepare, ++ * we decide to proceed anyway. ++ * ++ * By doing so we risk context corruption and with ++ * some gens (kbl), possible system hang if reset ++ * happens during active bb execution. ++ * ++ * We rather take context corruption instead of ++ * failed reset with a wedged driver/gpu. And ++ * active bb execution case should be covered by ++ * i915_stop_engines we have before the reset. ++ */ ++ } ++ ++ if (INTEL_GEN(i915) >= 11) ++ ret = gen11_reset_engines(i915, engine_mask, retry); ++ else ++ ret = gen6_reset_engines(i915, engine_mask, retry); ++ ++skip_reset: ++ for_each_engine_masked(engine, i915, engine_mask, tmp) ++ gen8_engine_reset_cancel(engine); ++ ++ return ret; ++} ++ ++typedef int (*reset_func)(struct drm_i915_private *, ++ intel_engine_mask_t engine_mask, ++ unsigned int retry); ++ ++static reset_func intel_get_gpu_reset(struct drm_i915_private *i915) ++{ ++ if (INTEL_GEN(i915) >= 8) ++ return gen8_reset_engines; ++ else if (INTEL_GEN(i915) >= 6) ++ return gen6_reset_engines; ++ else if (INTEL_GEN(i915) >= 5) ++ return ironlake_do_reset; ++ else if (IS_G4X(i915)) ++ return g4x_do_reset; ++ else if (IS_G33(i915) || IS_PINEVIEW(i915)) ++ return g33_do_reset; ++ else if (INTEL_GEN(i915) >= 3) ++ return i915_do_reset; ++ else ++ return NULL; ++} ++ ++int intel_gpu_reset(struct drm_i915_private *i915, ++ intel_engine_mask_t engine_mask) ++{ ++ const int retries = engine_mask == ALL_ENGINES ? RESET_MAX_RETRIES : 1; ++ reset_func reset; ++ int ret = -ETIMEDOUT; ++ int retry; ++ ++ reset = intel_get_gpu_reset(i915); ++ if (!reset) ++ return -ENODEV; ++ ++ /* ++ * If the power well sleeps during the reset, the reset ++ * request may be dropped and never completes (causing -EIO). ++ */ ++ intel_uncore_forcewake_get(&i915->uncore, FORCEWAKE_ALL); ++ for (retry = 0; ret == -ETIMEDOUT && retry < retries; retry++) { ++ /* ++ * We stop engines, otherwise we might get failed reset and a ++ * dead gpu (on elk). Also as modern gpu as kbl can suffer ++ * from system hang if batchbuffer is progressing when ++ * the reset is issued, regardless of READY_TO_RESET ack. ++ * Thus assume it is best to stop engines on all gens ++ * where we have a gpu reset. ++ * ++ * WaKBLVECSSemaphoreWaitPoll:kbl (on ALL_ENGINES) ++ * ++ * WaMediaResetMainRingCleanup:ctg,elk (presumably) ++ * ++ * FIXME: Wa for more modern gens needs to be validated ++ */ ++ if (retry) ++ i915_stop_engines(i915, engine_mask); ++ ++ GEM_TRACE("engine_mask=%x\n", engine_mask); ++ preempt_disable(); ++ ret = reset(i915, engine_mask, retry); ++ preempt_enable(); ++ } ++ intel_uncore_forcewake_put(&i915->uncore, FORCEWAKE_ALL); ++ ++ return ret; ++} ++ ++bool intel_has_gpu_reset(struct drm_i915_private *i915) ++{ ++ if (USES_GUC(i915)) ++ return false; ++ ++ if (!i915_modparams.reset) ++ return NULL; ++ ++ return intel_get_gpu_reset(i915); ++} ++ ++bool intel_has_reset_engine(struct drm_i915_private *i915) ++{ ++ return INTEL_INFO(i915)->has_reset_engine && i915_modparams.reset >= 2; ++} ++ ++int intel_reset_guc(struct drm_i915_private *i915) ++{ ++ u32 guc_domain = ++ INTEL_GEN(i915) >= 11 ? GEN11_GRDOM_GUC : GEN9_GRDOM_GUC; ++ int ret; ++ ++ GEM_BUG_ON(!HAS_GUC(i915)); ++ ++ intel_uncore_forcewake_get(&i915->uncore, FORCEWAKE_ALL); ++ ret = gen6_hw_domain_reset(i915, guc_domain); ++ intel_uncore_forcewake_put(&i915->uncore, FORCEWAKE_ALL); ++ ++ return ret; ++} ++ ++/* ++ * Ensure irq handler finishes, and not run again. ++ * Also return the active request so that we only search for it once. ++ */ ++static void reset_prepare_engine(struct intel_engine_cs *engine) ++{ ++ /* ++ * During the reset sequence, we must prevent the engine from ++ * entering RC6. As the context state is undefined until we restart ++ * the engine, if it does enter RC6 during the reset, the state ++ * written to the powercontext is undefined and so we may lose ++ * GPU state upon resume, i.e. fail to restart after a reset. ++ */ ++ intel_uncore_forcewake_get(engine->uncore, FORCEWAKE_ALL); ++ engine->reset.prepare(engine); ++} ++ ++static void revoke_mmaps(struct drm_i915_private *i915) ++{ ++ int i; ++ ++ for (i = 0; i < i915->num_fence_regs; i++) { ++ struct drm_vma_offset_node *node; ++ struct i915_vma *vma; ++ u64 vma_offset; ++ ++ vma = READ_ONCE(i915->fence_regs[i].vma); ++ if (!vma) ++ continue; ++ ++ if (!i915_vma_has_userfault(vma)) ++ continue; ++ ++ GEM_BUG_ON(vma->fence != &i915->fence_regs[i]); ++ node = &vma->obj->base.vma_node; ++ vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT; ++ unmap_mapping_range(i915->drm.anon_inode->i_mapping, ++ drm_vma_node_offset_addr(node) + vma_offset, ++ vma->size, ++ 1); ++ } ++} ++ ++static void reset_prepare(struct drm_i915_private *i915) ++{ ++ struct intel_engine_cs *engine; ++ enum intel_engine_id id; ++ ++ for_each_engine(engine, i915, id) ++ reset_prepare_engine(engine); ++ ++ intel_uc_reset_prepare(i915); ++} ++ ++static void gt_revoke(struct drm_i915_private *i915) ++{ ++ revoke_mmaps(i915); ++} ++ ++static int gt_reset(struct drm_i915_private *i915, ++ intel_engine_mask_t stalled_mask) ++{ ++ struct intel_engine_cs *engine; ++ enum intel_engine_id id; ++ int err; ++ ++ /* ++ * Everything depends on having the GTT running, so we need to start ++ * there. ++ */ ++ err = i915_ggtt_enable_hw(i915); ++ if (err) ++ return err; ++ ++ for_each_engine(engine, i915, id) ++ intel_engine_reset(engine, stalled_mask & engine->mask); ++ ++ i915_gem_restore_fences(i915); ++ ++ return err; ++} ++ ++static void reset_finish_engine(struct intel_engine_cs *engine) ++{ ++ engine->reset.finish(engine); ++ intel_uncore_forcewake_put(engine->uncore, FORCEWAKE_ALL); ++} ++ ++struct i915_gpu_restart { ++ struct work_struct work; ++ struct drm_i915_private *i915; ++}; ++ ++static void restart_work(struct work_struct *work) ++{ ++ struct i915_gpu_restart *arg = container_of(work, typeof(*arg), work); ++ struct drm_i915_private *i915 = arg->i915; ++ struct intel_engine_cs *engine; ++ enum intel_engine_id id; ++ intel_wakeref_t wakeref; ++ ++ wakeref = intel_runtime_pm_get(i915); ++ mutex_lock(&i915->drm.struct_mutex); ++ WRITE_ONCE(i915->gpu_error.restart, NULL); ++ ++ for_each_engine(engine, i915, id) { ++ struct i915_request *rq; ++ ++ /* ++ * Ostensibily, we always want a context loaded for powersaving, ++ * so if the engine is idle after the reset, send a request ++ * to load our scratch kernel_context. ++ */ ++ if (!intel_engine_is_idle(engine)) ++ continue; ++ ++ rq = i915_request_alloc(engine, i915->kernel_context); ++ if (!IS_ERR(rq)) ++ i915_request_add(rq); ++ } ++ ++ mutex_unlock(&i915->drm.struct_mutex); ++ intel_runtime_pm_put(i915, wakeref); ++ ++ kfree(arg); ++} ++ ++static void reset_finish(struct drm_i915_private *i915) ++{ ++ struct intel_engine_cs *engine; ++ enum intel_engine_id id; ++ ++ for_each_engine(engine, i915, id) { ++ reset_finish_engine(engine); ++ intel_engine_signal_breadcrumbs(engine); ++ } ++} ++ ++static void reset_restart(struct drm_i915_private *i915) ++{ ++ struct i915_gpu_restart *arg; ++ ++ /* ++ * Following the reset, ensure that we always reload context for ++ * powersaving, and to correct engine->last_retired_context. Since ++ * this requires us to submit a request, queue a worker to do that ++ * task for us to evade any locking here. ++ */ ++ if (READ_ONCE(i915->gpu_error.restart)) ++ return; ++ ++ arg = kmalloc(sizeof(*arg), GFP_KERNEL); ++ if (arg) { ++ arg->i915 = i915; ++ INIT_WORK(&arg->work, restart_work); ++ ++ WRITE_ONCE(i915->gpu_error.restart, arg); ++ queue_work(i915->wq, &arg->work); ++ } ++} ++ ++static void nop_submit_request(struct i915_request *request) ++{ ++ struct intel_engine_cs *engine = request->engine; ++ unsigned long flags; ++ ++ GEM_TRACE("%s fence %llx:%lld -> -EIO\n", ++ engine->name, request->fence.context, request->fence.seqno); ++ dma_fence_set_error(&request->fence, -EIO); ++ ++ spin_lock_irqsave(&engine->timeline.lock, flags); ++ __i915_request_submit(request); ++ i915_request_mark_complete(request); ++ spin_unlock_irqrestore(&engine->timeline.lock, flags); ++ ++ intel_engine_queue_breadcrumbs(engine); ++} ++ ++static void __i915_gem_set_wedged(struct drm_i915_private *i915) ++{ ++ struct i915_gpu_error *error = &i915->gpu_error; ++ struct intel_engine_cs *engine; ++ enum intel_engine_id id; ++ ++ if (test_bit(I915_WEDGED, &error->flags)) ++ return; ++ ++ if (GEM_SHOW_DEBUG() && !intel_engines_are_idle(i915)) { ++ struct drm_printer p = drm_debug_printer(__func__); ++ ++ for_each_engine(engine, i915, id) ++ intel_engine_dump(engine, &p, "%s\n", engine->name); ++ } ++ ++ GEM_TRACE("start\n"); ++ ++ /* ++ * First, stop submission to hw, but do not yet complete requests by ++ * rolling the global seqno forward (since this would complete requests ++ * for which we haven't set the fence error to EIO yet). ++ */ ++ reset_prepare(i915); ++ ++ /* Even if the GPU reset fails, it should still stop the engines */ ++ if (!INTEL_INFO(i915)->gpu_reset_clobbers_display) ++ intel_gpu_reset(i915, ALL_ENGINES); ++ ++ for_each_engine(engine, i915, id) { ++ engine->submit_request = nop_submit_request; ++ engine->schedule = NULL; ++ } ++ i915->caps.scheduler = 0; ++ ++ /* ++ * Make sure no request can slip through without getting completed by ++ * either this call here to intel_engine_write_global_seqno, or the one ++ * in nop_submit_request. ++ */ ++ synchronize_rcu_expedited(); ++ ++ /* Mark all executing requests as skipped */ ++ for_each_engine(engine, i915, id) ++ engine->cancel_requests(engine); ++ ++ reset_finish(i915); ++ ++ smp_mb__before_atomic(); ++ set_bit(I915_WEDGED, &error->flags); ++ ++ GEM_TRACE("end\n"); ++} ++ ++void i915_gem_set_wedged(struct drm_i915_private *i915) ++{ ++ struct i915_gpu_error *error = &i915->gpu_error; ++ intel_wakeref_t wakeref; ++ ++ mutex_lock(&error->wedge_mutex); ++ with_intel_runtime_pm(i915, wakeref) ++ __i915_gem_set_wedged(i915); ++ mutex_unlock(&error->wedge_mutex); ++} ++ ++static bool __i915_gem_unset_wedged(struct drm_i915_private *i915) ++{ ++ struct i915_gpu_error *error = &i915->gpu_error; ++ struct i915_timeline *tl; ++ ++ if (!test_bit(I915_WEDGED, &error->flags)) ++ return true; ++ ++ if (!i915->gt.scratch) /* Never full initialised, recovery impossible */ ++ return false; ++ ++ GEM_TRACE("start\n"); ++ ++ /* ++ * Before unwedging, make sure that all pending operations ++ * are flushed and errored out - we may have requests waiting upon ++ * third party fences. We marked all inflight requests as EIO, and ++ * every execbuf since returned EIO, for consistency we want all ++ * the currently pending requests to also be marked as EIO, which ++ * is done inside our nop_submit_request - and so we must wait. ++ * ++ * No more can be submitted until we reset the wedged bit. ++ */ ++ mutex_lock(&i915->gt.timelines.mutex); ++ list_for_each_entry(tl, &i915->gt.timelines.active_list, link) { ++ struct i915_request *rq; ++ ++ rq = i915_active_request_get_unlocked(&tl->last_request); ++ if (!rq) ++ continue; ++ ++ /* ++ * All internal dependencies (i915_requests) will have ++ * been flushed by the set-wedge, but we may be stuck waiting ++ * for external fences. These should all be capped to 10s ++ * (I915_FENCE_TIMEOUT) so this wait should not be unbounded ++ * in the worst case. ++ */ ++ dma_fence_default_wait(&rq->fence, false, MAX_SCHEDULE_TIMEOUT); ++ i915_request_put(rq); ++ } ++ mutex_unlock(&i915->gt.timelines.mutex); ++ ++ intel_engines_sanitize(i915, false); ++ ++ /* ++ * Undo nop_submit_request. We prevent all new i915 requests from ++ * being queued (by disallowing execbuf whilst wedged) so having ++ * waited for all active requests above, we know the system is idle ++ * and do not have to worry about a thread being inside ++ * engine->submit_request() as we swap over. So unlike installing ++ * the nop_submit_request on reset, we can do this from normal ++ * context and do not require stop_machine(). ++ */ ++ intel_engines_reset_default_submission(i915); ++ ++ GEM_TRACE("end\n"); ++ ++ smp_mb__before_atomic(); /* complete takeover before enabling execbuf */ ++ clear_bit(I915_WEDGED, &i915->gpu_error.flags); ++ ++ return true; ++} ++ ++bool i915_gem_unset_wedged(struct drm_i915_private *i915) ++{ ++ struct i915_gpu_error *error = &i915->gpu_error; ++ bool result; ++ ++ mutex_lock(&error->wedge_mutex); ++ result = __i915_gem_unset_wedged(i915); ++ mutex_unlock(&error->wedge_mutex); ++ ++ return result; ++} ++ ++static int do_reset(struct drm_i915_private *i915, ++ intel_engine_mask_t stalled_mask) ++{ ++ int err, i; ++ ++ gt_revoke(i915); ++ ++ err = intel_gpu_reset(i915, ALL_ENGINES); ++ for (i = 0; err && i < RESET_MAX_RETRIES; i++) { ++ msleep(10 * (i + 1)); ++ err = intel_gpu_reset(i915, ALL_ENGINES); ++ } ++ if (err) ++ return err; ++ ++ return gt_reset(i915, stalled_mask); ++} ++ ++/** ++ * i915_reset - reset chip after a hang ++ * @i915: #drm_i915_private to reset ++ * @stalled_mask: mask of the stalled engines with the guilty requests ++ * @reason: user error message for why we are resetting ++ * ++ * Reset the chip. Useful if a hang is detected. Marks the device as wedged ++ * on failure. ++ * ++ * Procedure is fairly simple: ++ * - reset the chip using the reset reg ++ * - re-init context state ++ * - re-init hardware status page ++ * - re-init ring buffer ++ * - re-init interrupt state ++ * - re-init display ++ */ ++void i915_reset(struct drm_i915_private *i915, ++ intel_engine_mask_t stalled_mask, ++ const char *reason) ++{ ++ struct i915_gpu_error *error = &i915->gpu_error; ++ int ret; ++ ++ GEM_TRACE("flags=%lx\n", error->flags); ++ ++ might_sleep(); ++ assert_rpm_wakelock_held(i915); ++ GEM_BUG_ON(!test_bit(I915_RESET_BACKOFF, &error->flags)); ++ ++ /* Clear any previous failed attempts at recovery. Time to try again. */ ++ if (!__i915_gem_unset_wedged(i915)) ++ return; ++ ++ if (reason) ++ dev_notice(i915->drm.dev, "Resetting chip for %s\n", reason); ++ error->reset_count++; ++ ++ reset_prepare(i915); ++ ++ if (!intel_has_gpu_reset(i915)) { ++ if (i915_modparams.reset) ++ dev_err(i915->drm.dev, "GPU reset not supported\n"); ++ else ++ DRM_DEBUG_DRIVER("GPU reset disabled\n"); ++ goto error; ++ } ++ ++ if (INTEL_INFO(i915)->gpu_reset_clobbers_display) ++ intel_runtime_pm_disable_interrupts(i915); ++ ++ if (do_reset(i915, stalled_mask)) { ++ dev_err(i915->drm.dev, "Failed to reset chip\n"); ++ goto taint; ++ } ++ ++ if (INTEL_INFO(i915)->gpu_reset_clobbers_display) ++ intel_runtime_pm_enable_interrupts(i915); ++ ++ intel_overlay_reset(i915); ++ ++ /* ++ * Next we need to restore the context, but we don't use those ++ * yet either... ++ * ++ * Ring buffer needs to be re-initialized in the KMS case, or if X ++ * was running at the time of the reset (i.e. we weren't VT ++ * switched away). ++ */ ++ ret = i915_gem_init_hw(i915); ++ if (ret) { ++ DRM_ERROR("Failed to initialise HW following reset (%d)\n", ++ ret); ++ goto error; ++ } ++ ++ i915_queue_hangcheck(i915); ++ ++finish: ++ reset_finish(i915); ++ if (!__i915_wedged(error)) ++ reset_restart(i915); ++ return; ++ ++taint: ++ /* ++ * History tells us that if we cannot reset the GPU now, we ++ * never will. This then impacts everything that is run ++ * subsequently. On failing the reset, we mark the driver ++ * as wedged, preventing further execution on the GPU. ++ * We also want to go one step further and add a taint to the ++ * kernel so that any subsequent faults can be traced back to ++ * this failure. This is important for CI, where if the ++ * GPU/driver fails we would like to reboot and restart testing ++ * rather than continue on into oblivion. For everyone else, ++ * the system should still plod along, but they have been warned! ++ */ ++ add_taint(TAINT_WARN, LOCKDEP_STILL_OK); ++error: ++ __i915_gem_set_wedged(i915); ++ goto finish; ++} ++ ++static inline int intel_gt_reset_engine(struct drm_i915_private *i915, ++ struct intel_engine_cs *engine) ++{ ++ return intel_gpu_reset(i915, engine->mask); ++} ++ ++/** ++ * i915_reset_engine - reset GPU engine to recover from a hang ++ * @engine: engine to reset ++ * @msg: reason for GPU reset; or NULL for no dev_notice() ++ * ++ * Reset a specific GPU engine. Useful if a hang is detected. ++ * Returns zero on successful reset or otherwise an error code. ++ * ++ * Procedure is: ++ * - identifies the request that caused the hang and it is dropped ++ * - reset engine (which will force the engine to idle) ++ * - re-init/configure engine ++ */ ++int i915_reset_engine(struct intel_engine_cs *engine, const char *msg) ++{ ++ struct i915_gpu_error *error = &engine->i915->gpu_error; ++ int ret; ++ ++ GEM_TRACE("%s flags=%lx\n", engine->name, error->flags); ++ GEM_BUG_ON(!test_bit(I915_RESET_ENGINE + engine->id, &error->flags)); ++ ++ reset_prepare_engine(engine); ++ ++ if (msg) ++ dev_notice(engine->i915->drm.dev, ++ "Resetting %s for %s\n", engine->name, msg); ++ error->reset_engine_count[engine->id]++; ++ ++ if (!engine->i915->guc.execbuf_client) ++ ret = intel_gt_reset_engine(engine->i915, engine); ++ else ++ ret = intel_guc_reset_engine(&engine->i915->guc, engine); ++ if (ret) { ++ /* If we fail here, we expect to fallback to a global reset */ ++ DRM_DEBUG_DRIVER("%sFailed to reset %s, ret=%d\n", ++ engine->i915->guc.execbuf_client ? "GuC " : "", ++ engine->name, ret); ++ goto out; ++ } ++ ++ /* ++ * The request that caused the hang is stuck on elsp, we know the ++ * active request and can drop it, adjust head to skip the offending ++ * request to resume executing remaining requests in the queue. ++ */ ++ intel_engine_reset(engine, true); ++ ++ /* ++ * The engine and its registers (and workarounds in case of render) ++ * have been reset to their default values. Follow the init_ring ++ * process to program RING_MODE, HWSP and re-enable submission. ++ */ ++ ret = engine->init_hw(engine); ++ if (ret) ++ goto out; ++ ++out: ++ intel_engine_cancel_stop_cs(engine); ++ reset_finish_engine(engine); ++ return ret; ++} ++ ++static void i915_reset_device(struct drm_i915_private *i915, ++ u32 engine_mask, ++ const char *reason) ++{ ++ struct i915_gpu_error *error = &i915->gpu_error; ++ struct kobject *kobj = &i915->drm.primary->kdev->kobj; ++ char *error_event[] = { I915_ERROR_UEVENT "=1", NULL }; ++ char *reset_event[] = { I915_RESET_UEVENT "=1", NULL }; ++ char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL }; ++ struct i915_wedge_me w; ++ ++ kobject_uevent_env(kobj, KOBJ_CHANGE, error_event); ++ ++ DRM_DEBUG_DRIVER("resetting chip\n"); ++ kobject_uevent_env(kobj, KOBJ_CHANGE, reset_event); ++ ++ /* Use a watchdog to ensure that our reset completes */ ++ i915_wedge_on_timeout(&w, i915, 5 * HZ) { ++ intel_prepare_reset(i915); ++ ++ /* Flush everyone using a resource about to be clobbered */ ++ synchronize_srcu_expedited(&error->reset_backoff_srcu); ++ ++ mutex_lock(&error->wedge_mutex); ++ i915_reset(i915, engine_mask, reason); ++ mutex_unlock(&error->wedge_mutex); ++ ++ intel_finish_reset(i915); ++ } ++ ++ if (!test_bit(I915_WEDGED, &error->flags)) ++ kobject_uevent_env(kobj, KOBJ_CHANGE, reset_done_event); ++} ++ ++static void clear_register(struct intel_uncore *uncore, i915_reg_t reg) ++{ ++ intel_uncore_rmw(uncore, reg, 0, 0); ++} ++ ++void i915_clear_error_registers(struct drm_i915_private *i915) ++{ ++ struct intel_uncore *uncore = &i915->uncore; ++ u32 eir; ++ ++ if (!IS_GEN(i915, 2)) ++ clear_register(uncore, PGTBL_ER); ++ ++ if (INTEL_GEN(i915) < 4) ++ clear_register(uncore, IPEIR(RENDER_RING_BASE)); ++ else ++ clear_register(uncore, IPEIR_I965); ++ ++ clear_register(uncore, EIR); ++ eir = intel_uncore_read(uncore, EIR); ++ if (eir) { ++ /* ++ * some errors might have become stuck, ++ * mask them. ++ */ ++ DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir); ++ rmw_set(uncore, EMR, eir); ++ intel_uncore_write(uncore, GEN2_IIR, ++ I915_MASTER_ERROR_INTERRUPT); ++ } ++ ++ if (INTEL_GEN(i915) >= 8) { ++ rmw_clear(uncore, GEN8_RING_FAULT_REG, RING_FAULT_VALID); ++ intel_uncore_posting_read(uncore, GEN8_RING_FAULT_REG); ++ } else if (INTEL_GEN(i915) >= 6) { ++ struct intel_engine_cs *engine; ++ enum intel_engine_id id; ++ ++ for_each_engine(engine, i915, id) { ++ rmw_clear(uncore, ++ RING_FAULT_REG(engine), RING_FAULT_VALID); ++ intel_uncore_posting_read(uncore, ++ RING_FAULT_REG(engine)); ++ } ++ } ++} ++ ++/** ++ * i915_handle_error - handle a gpu error ++ * @i915: i915 device private ++ * @engine_mask: mask representing engines that are hung ++ * @flags: control flags ++ * @fmt: Error message format string ++ * ++ * Do some basic checking of register state at error time and ++ * dump it to the syslog. Also call i915_capture_error_state() to make ++ * sure we get a record and make it available in debugfs. Fire a uevent ++ * so userspace knows something bad happened (should trigger collection ++ * of a ring dump etc.). ++ */ ++void i915_handle_error(struct drm_i915_private *i915, ++ intel_engine_mask_t engine_mask, ++ unsigned long flags, ++ const char *fmt, ...) ++{ ++ struct i915_gpu_error *error = &i915->gpu_error; ++ struct intel_engine_cs *engine; ++ intel_wakeref_t wakeref; ++ intel_engine_mask_t tmp; ++ char error_msg[80]; ++ char *msg = NULL; ++ ++ if (fmt) { ++ va_list args; ++ ++ va_start(args, fmt); ++ vscnprintf(error_msg, sizeof(error_msg), fmt, args); ++ va_end(args); ++ ++ msg = error_msg; ++ } ++ ++ /* ++ * In most cases it's guaranteed that we get here with an RPM ++ * reference held, for example because there is a pending GPU ++ * request that won't finish until the reset is done. This ++ * isn't the case at least when we get here by doing a ++ * simulated reset via debugfs, so get an RPM reference. ++ */ ++ wakeref = intel_runtime_pm_get(i915); ++ ++ engine_mask &= INTEL_INFO(i915)->engine_mask; ++ ++ if (flags & I915_ERROR_CAPTURE) { ++ i915_capture_error_state(i915, engine_mask, msg); ++ i915_clear_error_registers(i915); ++ } ++ ++ /* ++ * Try engine reset when available. We fall back to full reset if ++ * single reset fails. ++ */ ++ if (intel_has_reset_engine(i915) && !__i915_wedged(error)) { ++ for_each_engine_masked(engine, i915, engine_mask, tmp) { ++ BUILD_BUG_ON(I915_RESET_MODESET >= I915_RESET_ENGINE); ++ if (test_and_set_bit(I915_RESET_ENGINE + engine->id, ++ &error->flags)) ++ continue; ++ ++ if (i915_reset_engine(engine, msg) == 0) ++ engine_mask &= ~engine->mask; ++ ++ clear_bit(I915_RESET_ENGINE + engine->id, ++ &error->flags); ++ wake_up_bit(&error->flags, ++ I915_RESET_ENGINE + engine->id); ++ } ++ } ++ ++ if (!engine_mask) ++ goto out; ++ ++ /* Full reset needs the mutex, stop any other user trying to do so. */ ++ if (test_and_set_bit(I915_RESET_BACKOFF, &error->flags)) { ++ wait_event(error->reset_queue, ++ !test_bit(I915_RESET_BACKOFF, &error->flags)); ++ goto out; /* piggy-back on the other reset */ ++ } ++ ++ /* Make sure i915_reset_trylock() sees the I915_RESET_BACKOFF */ ++ synchronize_rcu_expedited(); ++ ++ /* Prevent any other reset-engine attempt. */ ++ for_each_engine(engine, i915, tmp) { ++ while (test_and_set_bit(I915_RESET_ENGINE + engine->id, ++ &error->flags)) ++ wait_on_bit(&error->flags, ++ I915_RESET_ENGINE + engine->id, ++ TASK_UNINTERRUPTIBLE); ++ } ++ ++ i915_reset_device(i915, engine_mask, msg); ++ ++ for_each_engine(engine, i915, tmp) { ++ clear_bit(I915_RESET_ENGINE + engine->id, ++ &error->flags); ++ } ++ ++ clear_bit(I915_RESET_BACKOFF, &error->flags); ++ wake_up_all(&error->reset_queue); ++ ++out: ++ intel_runtime_pm_put(i915, wakeref); ++} ++ ++int i915_reset_trylock(struct drm_i915_private *i915) ++{ ++ struct i915_gpu_error *error = &i915->gpu_error; ++ int srcu; ++ ++ might_lock(&error->reset_backoff_srcu); ++ might_sleep(); ++ ++ rcu_read_lock(); ++ while (test_bit(I915_RESET_BACKOFF, &error->flags)) { ++ rcu_read_unlock(); ++ ++ if (wait_event_interruptible(error->reset_queue, ++ !test_bit(I915_RESET_BACKOFF, ++ &error->flags))) ++ return -EINTR; ++ ++ rcu_read_lock(); ++ } ++ srcu = srcu_read_lock(&error->reset_backoff_srcu); ++ rcu_read_unlock(); ++ ++ return srcu; ++} ++ ++void i915_reset_unlock(struct drm_i915_private *i915, int tag) ++__releases(&i915->gpu_error.reset_backoff_srcu) ++{ ++ struct i915_gpu_error *error = &i915->gpu_error; ++ ++ srcu_read_unlock(&error->reset_backoff_srcu, tag); ++} ++ ++int i915_terminally_wedged(struct drm_i915_private *i915) ++{ ++ struct i915_gpu_error *error = &i915->gpu_error; ++ ++ might_sleep(); ++ ++ if (!__i915_wedged(error)) ++ return 0; ++ ++ /* Reset still in progress? Maybe we will recover? */ ++ if (!test_bit(I915_RESET_BACKOFF, &error->flags)) ++ return -EIO; ++ ++ /* XXX intel_reset_finish() still takes struct_mutex!!! */ ++ if (mutex_is_locked(&i915->drm.struct_mutex)) ++ return -EAGAIN; ++ ++ if (wait_event_interruptible(error->reset_queue, ++ !test_bit(I915_RESET_BACKOFF, ++ &error->flags))) ++ return -EINTR; ++ ++ return __i915_wedged(error) ? -EIO : 0; ++} ++ ++bool i915_reset_flush(struct drm_i915_private *i915) ++{ ++ int err; ++ ++ cancel_delayed_work_sync(&i915->gpu_error.hangcheck_work); ++ ++ flush_workqueue(i915->wq); ++ GEM_BUG_ON(READ_ONCE(i915->gpu_error.restart)); ++ ++ mutex_lock(&i915->drm.struct_mutex); ++ err = i915_gem_wait_for_idle(i915, ++ I915_WAIT_LOCKED | ++ I915_WAIT_FOR_IDLE_BOOST, ++ MAX_SCHEDULE_TIMEOUT); ++ mutex_unlock(&i915->drm.struct_mutex); ++ ++ return !err; ++} ++ ++static void i915_wedge_me(struct work_struct *work) ++{ ++ struct i915_wedge_me *w = container_of(work, typeof(*w), work.work); ++ ++ dev_err(w->i915->drm.dev, ++ "%s timed out, cancelling all in-flight rendering.\n", ++ w->name); ++ i915_gem_set_wedged(w->i915); ++} ++ ++void __i915_init_wedge(struct i915_wedge_me *w, ++ struct drm_i915_private *i915, ++ long timeout, ++ const char *name) ++{ ++ w->i915 = i915; ++ w->name = name; ++ ++ INIT_DELAYED_WORK_ONSTACK(&w->work, i915_wedge_me); ++ schedule_delayed_work(&w->work, timeout); ++} ++ ++void __i915_fini_wedge(struct i915_wedge_me *w) ++{ ++ cancel_delayed_work_sync(&w->work); ++ destroy_delayed_work_on_stack(&w->work); ++ w->i915 = NULL; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_reset.h b/drivers/gpu/drm/i915_legacy/i915_reset.h +new file mode 100644 +index 000000000000..3c0450289b8f +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_reset.h +@@ -0,0 +1,69 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2008-2018 Intel Corporation ++ */ ++ ++#ifndef I915_RESET_H ++#define I915_RESET_H ++ ++#include ++#include ++#include ++ ++#include "intel_engine_types.h" ++ ++struct drm_i915_private; ++struct i915_request; ++struct intel_engine_cs; ++struct intel_guc; ++ ++__printf(4, 5) ++void i915_handle_error(struct drm_i915_private *i915, ++ intel_engine_mask_t engine_mask, ++ unsigned long flags, ++ const char *fmt, ...); ++#define I915_ERROR_CAPTURE BIT(0) ++ ++void i915_clear_error_registers(struct drm_i915_private *i915); ++ ++void i915_reset(struct drm_i915_private *i915, ++ intel_engine_mask_t stalled_mask, ++ const char *reason); ++int i915_reset_engine(struct intel_engine_cs *engine, ++ const char *reason); ++ ++void i915_reset_request(struct i915_request *rq, bool guilty); ++bool i915_reset_flush(struct drm_i915_private *i915); ++ ++int __must_check i915_reset_trylock(struct drm_i915_private *i915); ++void i915_reset_unlock(struct drm_i915_private *i915, int tag); ++ ++int i915_terminally_wedged(struct drm_i915_private *i915); ++ ++bool intel_has_gpu_reset(struct drm_i915_private *i915); ++bool intel_has_reset_engine(struct drm_i915_private *i915); ++ ++int intel_gpu_reset(struct drm_i915_private *i915, ++ intel_engine_mask_t engine_mask); ++ ++int intel_reset_guc(struct drm_i915_private *i915); ++ ++struct i915_wedge_me { ++ struct delayed_work work; ++ struct drm_i915_private *i915; ++ const char *name; ++}; ++ ++void __i915_init_wedge(struct i915_wedge_me *w, ++ struct drm_i915_private *i915, ++ long timeout, ++ const char *name); ++void __i915_fini_wedge(struct i915_wedge_me *w); ++ ++#define i915_wedge_on_timeout(W, DEV, TIMEOUT) \ ++ for (__i915_init_wedge((W), (DEV), (TIMEOUT), __func__); \ ++ (W)->i915; \ ++ __i915_fini_wedge((W))) ++ ++#endif /* I915_RESET_H */ +diff --git a/drivers/gpu/drm/i915_legacy/i915_scheduler.c b/drivers/gpu/drm/i915_legacy/i915_scheduler.c +new file mode 100644 +index 000000000000..108f52e1bf35 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_scheduler.c +@@ -0,0 +1,492 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ */ ++ ++#include ++ ++#include "i915_drv.h" ++#include "i915_globals.h" ++#include "i915_request.h" ++#include "i915_scheduler.h" ++ ++static struct i915_global_scheduler { ++ struct i915_global base; ++ struct kmem_cache *slab_dependencies; ++ struct kmem_cache *slab_priorities; ++} global; ++ ++static DEFINE_SPINLOCK(schedule_lock); ++ ++static const struct i915_request * ++node_to_request(const struct i915_sched_node *node) ++{ ++ return container_of(node, const struct i915_request, sched); ++} ++ ++static inline bool node_started(const struct i915_sched_node *node) ++{ ++ return i915_request_started(node_to_request(node)); ++} ++ ++static inline bool node_signaled(const struct i915_sched_node *node) ++{ ++ return i915_request_completed(node_to_request(node)); ++} ++ ++static inline struct i915_priolist *to_priolist(struct rb_node *rb) ++{ ++ return rb_entry(rb, struct i915_priolist, node); ++} ++ ++static void assert_priolists(struct intel_engine_execlists * const execlists) ++{ ++ struct rb_node *rb; ++ long last_prio, i; ++ ++ if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) ++ return; ++ ++ GEM_BUG_ON(rb_first_cached(&execlists->queue) != ++ rb_first(&execlists->queue.rb_root)); ++ ++ last_prio = (INT_MAX >> I915_USER_PRIORITY_SHIFT) + 1; ++ for (rb = rb_first_cached(&execlists->queue); rb; rb = rb_next(rb)) { ++ const struct i915_priolist *p = to_priolist(rb); ++ ++ GEM_BUG_ON(p->priority >= last_prio); ++ last_prio = p->priority; ++ ++ GEM_BUG_ON(!p->used); ++ for (i = 0; i < ARRAY_SIZE(p->requests); i++) { ++ if (list_empty(&p->requests[i])) ++ continue; ++ ++ GEM_BUG_ON(!(p->used & BIT(i))); ++ } ++ } ++} ++ ++struct list_head * ++i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio) ++{ ++ struct intel_engine_execlists * const execlists = &engine->execlists; ++ struct i915_priolist *p; ++ struct rb_node **parent, *rb; ++ bool first = true; ++ int idx, i; ++ ++ lockdep_assert_held(&engine->timeline.lock); ++ assert_priolists(execlists); ++ ++ /* buckets sorted from highest [in slot 0] to lowest priority */ ++ idx = I915_PRIORITY_COUNT - (prio & I915_PRIORITY_MASK) - 1; ++ prio >>= I915_USER_PRIORITY_SHIFT; ++ if (unlikely(execlists->no_priolist)) ++ prio = I915_PRIORITY_NORMAL; ++ ++find_priolist: ++ /* most positive priority is scheduled first, equal priorities fifo */ ++ rb = NULL; ++ parent = &execlists->queue.rb_root.rb_node; ++ while (*parent) { ++ rb = *parent; ++ p = to_priolist(rb); ++ if (prio > p->priority) { ++ parent = &rb->rb_left; ++ } else if (prio < p->priority) { ++ parent = &rb->rb_right; ++ first = false; ++ } else { ++ goto out; ++ } ++ } ++ ++ if (prio == I915_PRIORITY_NORMAL) { ++ p = &execlists->default_priolist; ++ } else { ++ p = kmem_cache_alloc(global.slab_priorities, GFP_ATOMIC); ++ /* Convert an allocation failure to a priority bump */ ++ if (unlikely(!p)) { ++ prio = I915_PRIORITY_NORMAL; /* recurses just once */ ++ ++ /* To maintain ordering with all rendering, after an ++ * allocation failure we have to disable all scheduling. ++ * Requests will then be executed in fifo, and schedule ++ * will ensure that dependencies are emitted in fifo. ++ * There will be still some reordering with existing ++ * requests, so if userspace lied about their ++ * dependencies that reordering may be visible. ++ */ ++ execlists->no_priolist = true; ++ goto find_priolist; ++ } ++ } ++ ++ p->priority = prio; ++ for (i = 0; i < ARRAY_SIZE(p->requests); i++) ++ INIT_LIST_HEAD(&p->requests[i]); ++ rb_link_node(&p->node, rb, parent); ++ rb_insert_color_cached(&p->node, &execlists->queue, first); ++ p->used = 0; ++ ++out: ++ p->used |= BIT(idx); ++ return &p->requests[idx]; ++} ++ ++void __i915_priolist_free(struct i915_priolist *p) ++{ ++ kmem_cache_free(global.slab_priorities, p); ++} ++ ++struct sched_cache { ++ struct list_head *priolist; ++}; ++ ++static struct intel_engine_cs * ++sched_lock_engine(const struct i915_sched_node *node, ++ struct intel_engine_cs *locked, ++ struct sched_cache *cache) ++{ ++ struct intel_engine_cs *engine = node_to_request(node)->engine; ++ ++ GEM_BUG_ON(!locked); ++ ++ if (engine != locked) { ++ spin_unlock(&locked->timeline.lock); ++ memset(cache, 0, sizeof(*cache)); ++ spin_lock(&engine->timeline.lock); ++ } ++ ++ return engine; ++} ++ ++static bool inflight(const struct i915_request *rq, ++ const struct intel_engine_cs *engine) ++{ ++ const struct i915_request *active; ++ ++ if (!i915_request_is_active(rq)) ++ return false; ++ ++ active = port_request(engine->execlists.port); ++ return active->hw_context == rq->hw_context; ++} ++ ++static void __i915_schedule(struct i915_sched_node *node, ++ const struct i915_sched_attr *attr) ++{ ++ struct intel_engine_cs *engine; ++ struct i915_dependency *dep, *p; ++ struct i915_dependency stack; ++ const int prio = attr->priority; ++ struct sched_cache cache; ++ LIST_HEAD(dfs); ++ ++ /* Needed in order to use the temporary link inside i915_dependency */ ++ lockdep_assert_held(&schedule_lock); ++ GEM_BUG_ON(prio == I915_PRIORITY_INVALID); ++ ++ if (node_signaled(node)) ++ return; ++ ++ if (prio <= READ_ONCE(node->attr.priority)) ++ return; ++ ++ stack.signaler = node; ++ list_add(&stack.dfs_link, &dfs); ++ ++ /* ++ * Recursively bump all dependent priorities to match the new request. ++ * ++ * A naive approach would be to use recursion: ++ * static void update_priorities(struct i915_sched_node *node, prio) { ++ * list_for_each_entry(dep, &node->signalers_list, signal_link) ++ * update_priorities(dep->signal, prio) ++ * queue_request(node); ++ * } ++ * but that may have unlimited recursion depth and so runs a very ++ * real risk of overunning the kernel stack. Instead, we build ++ * a flat list of all dependencies starting with the current request. ++ * As we walk the list of dependencies, we add all of its dependencies ++ * to the end of the list (this may include an already visited ++ * request) and continue to walk onwards onto the new dependencies. The ++ * end result is a topological list of requests in reverse order, the ++ * last element in the list is the request we must execute first. ++ */ ++ list_for_each_entry(dep, &dfs, dfs_link) { ++ struct i915_sched_node *node = dep->signaler; ++ ++ /* If we are already flying, we know we have no signalers */ ++ if (node_started(node)) ++ continue; ++ ++ /* ++ * Within an engine, there can be no cycle, but we may ++ * refer to the same dependency chain multiple times ++ * (redundant dependencies are not eliminated) and across ++ * engines. ++ */ ++ list_for_each_entry(p, &node->signalers_list, signal_link) { ++ GEM_BUG_ON(p == dep); /* no cycles! */ ++ ++ if (node_signaled(p->signaler)) ++ continue; ++ ++ if (prio > READ_ONCE(p->signaler->attr.priority)) ++ list_move_tail(&p->dfs_link, &dfs); ++ } ++ } ++ ++ /* ++ * If we didn't need to bump any existing priorities, and we haven't ++ * yet submitted this request (i.e. there is no potential race with ++ * execlists_submit_request()), we can set our own priority and skip ++ * acquiring the engine locks. ++ */ ++ if (node->attr.priority == I915_PRIORITY_INVALID) { ++ GEM_BUG_ON(!list_empty(&node->link)); ++ node->attr = *attr; ++ ++ if (stack.dfs_link.next == stack.dfs_link.prev) ++ return; ++ ++ __list_del_entry(&stack.dfs_link); ++ } ++ ++ memset(&cache, 0, sizeof(cache)); ++ engine = node_to_request(node)->engine; ++ spin_lock(&engine->timeline.lock); ++ ++ /* Fifo and depth-first replacement ensure our deps execute before us */ ++ list_for_each_entry_safe_reverse(dep, p, &dfs, dfs_link) { ++ INIT_LIST_HEAD(&dep->dfs_link); ++ ++ node = dep->signaler; ++ engine = sched_lock_engine(node, engine, &cache); ++ lockdep_assert_held(&engine->timeline.lock); ++ ++ /* Recheck after acquiring the engine->timeline.lock */ ++ if (prio <= node->attr.priority || node_signaled(node)) ++ continue; ++ ++ node->attr.priority = prio; ++ if (!list_empty(&node->link)) { ++ if (!cache.priolist) ++ cache.priolist = ++ i915_sched_lookup_priolist(engine, ++ prio); ++ list_move_tail(&node->link, cache.priolist); ++ } else { ++ /* ++ * If the request is not in the priolist queue because ++ * it is not yet runnable, then it doesn't contribute ++ * to our preemption decisions. On the other hand, ++ * if the request is on the HW, it too is not in the ++ * queue; but in that case we may still need to reorder ++ * the inflight requests. ++ */ ++ if (!i915_sw_fence_done(&node_to_request(node)->submit)) ++ continue; ++ } ++ ++ if (prio <= engine->execlists.queue_priority_hint) ++ continue; ++ ++ engine->execlists.queue_priority_hint = prio; ++ ++ /* ++ * If we are already the currently executing context, don't ++ * bother evaluating if we should preempt ourselves. ++ */ ++ if (inflight(node_to_request(node), engine)) ++ continue; ++ ++ /* Defer (tasklet) submission until after all of our updates. */ ++ tasklet_hi_schedule(&engine->execlists.tasklet); ++ } ++ ++ spin_unlock(&engine->timeline.lock); ++} ++ ++void i915_schedule(struct i915_request *rq, const struct i915_sched_attr *attr) ++{ ++ spin_lock_irq(&schedule_lock); ++ __i915_schedule(&rq->sched, attr); ++ spin_unlock_irq(&schedule_lock); ++} ++ ++static void __bump_priority(struct i915_sched_node *node, unsigned int bump) ++{ ++ struct i915_sched_attr attr = node->attr; ++ ++ attr.priority |= bump; ++ __i915_schedule(node, &attr); ++} ++ ++void i915_schedule_bump_priority(struct i915_request *rq, unsigned int bump) ++{ ++ unsigned long flags; ++ ++ GEM_BUG_ON(bump & ~I915_PRIORITY_MASK); ++ ++ if (READ_ONCE(rq->sched.attr.priority) == I915_PRIORITY_INVALID) ++ return; ++ ++ spin_lock_irqsave(&schedule_lock, flags); ++ __bump_priority(&rq->sched, bump); ++ spin_unlock_irqrestore(&schedule_lock, flags); ++} ++ ++void i915_sched_node_init(struct i915_sched_node *node) ++{ ++ INIT_LIST_HEAD(&node->signalers_list); ++ INIT_LIST_HEAD(&node->waiters_list); ++ INIT_LIST_HEAD(&node->link); ++ node->attr.priority = I915_PRIORITY_INVALID; ++ node->semaphores = 0; ++ node->flags = 0; ++} ++ ++static struct i915_dependency * ++i915_dependency_alloc(void) ++{ ++ return kmem_cache_alloc(global.slab_dependencies, GFP_KERNEL); ++} ++ ++static void ++i915_dependency_free(struct i915_dependency *dep) ++{ ++ kmem_cache_free(global.slab_dependencies, dep); ++} ++ ++bool __i915_sched_node_add_dependency(struct i915_sched_node *node, ++ struct i915_sched_node *signal, ++ struct i915_dependency *dep, ++ unsigned long flags) ++{ ++ bool ret = false; ++ ++ spin_lock_irq(&schedule_lock); ++ ++ if (!node_signaled(signal)) { ++ INIT_LIST_HEAD(&dep->dfs_link); ++ list_add(&dep->wait_link, &signal->waiters_list); ++ list_add(&dep->signal_link, &node->signalers_list); ++ dep->signaler = signal; ++ dep->flags = flags; ++ ++ /* Keep track of whether anyone on this chain has a semaphore */ ++ if (signal->flags & I915_SCHED_HAS_SEMAPHORE_CHAIN && ++ !node_started(signal)) ++ node->flags |= I915_SCHED_HAS_SEMAPHORE_CHAIN; ++ ++ /* ++ * As we do not allow WAIT to preempt inflight requests, ++ * once we have executed a request, along with triggering ++ * any execution callbacks, we must preserve its ordering ++ * within the non-preemptible FIFO. ++ */ ++ BUILD_BUG_ON(__NO_PREEMPTION & ~I915_PRIORITY_MASK); ++ if (flags & I915_DEPENDENCY_EXTERNAL) ++ __bump_priority(signal, __NO_PREEMPTION); ++ ++ ret = true; ++ } ++ ++ spin_unlock_irq(&schedule_lock); ++ ++ return ret; ++} ++ ++int i915_sched_node_add_dependency(struct i915_sched_node *node, ++ struct i915_sched_node *signal) ++{ ++ struct i915_dependency *dep; ++ ++ dep = i915_dependency_alloc(); ++ if (!dep) ++ return -ENOMEM; ++ ++ if (!__i915_sched_node_add_dependency(node, signal, dep, ++ I915_DEPENDENCY_EXTERNAL | ++ I915_DEPENDENCY_ALLOC)) ++ i915_dependency_free(dep); ++ ++ return 0; ++} ++ ++void i915_sched_node_fini(struct i915_sched_node *node) ++{ ++ struct i915_dependency *dep, *tmp; ++ ++ GEM_BUG_ON(!list_empty(&node->link)); ++ ++ spin_lock_irq(&schedule_lock); ++ ++ /* ++ * Everyone we depended upon (the fences we wait to be signaled) ++ * should retire before us and remove themselves from our list. ++ * However, retirement is run independently on each timeline and ++ * so we may be called out-of-order. ++ */ ++ list_for_each_entry_safe(dep, tmp, &node->signalers_list, signal_link) { ++ GEM_BUG_ON(!node_signaled(dep->signaler)); ++ GEM_BUG_ON(!list_empty(&dep->dfs_link)); ++ ++ list_del(&dep->wait_link); ++ if (dep->flags & I915_DEPENDENCY_ALLOC) ++ i915_dependency_free(dep); ++ } ++ ++ /* Remove ourselves from everyone who depends upon us */ ++ list_for_each_entry_safe(dep, tmp, &node->waiters_list, wait_link) { ++ GEM_BUG_ON(dep->signaler != node); ++ GEM_BUG_ON(!list_empty(&dep->dfs_link)); ++ ++ list_del(&dep->signal_link); ++ if (dep->flags & I915_DEPENDENCY_ALLOC) ++ i915_dependency_free(dep); ++ } ++ ++ spin_unlock_irq(&schedule_lock); ++} ++ ++static void i915_global_scheduler_shrink(void) ++{ ++ kmem_cache_shrink(global.slab_dependencies); ++ kmem_cache_shrink(global.slab_priorities); ++} ++ ++static void i915_global_scheduler_exit(void) ++{ ++ kmem_cache_destroy(global.slab_dependencies); ++ kmem_cache_destroy(global.slab_priorities); ++} ++ ++static struct i915_global_scheduler global = { { ++ .shrink = i915_global_scheduler_shrink, ++ .exit = i915_global_scheduler_exit, ++} }; ++ ++int __init i915_global_scheduler_init(void) ++{ ++ global.slab_dependencies = KMEM_CACHE(i915_dependency, ++ SLAB_HWCACHE_ALIGN); ++ if (!global.slab_dependencies) ++ return -ENOMEM; ++ ++ global.slab_priorities = KMEM_CACHE(i915_priolist, ++ SLAB_HWCACHE_ALIGN); ++ if (!global.slab_priorities) ++ goto err_priorities; ++ ++ i915_global_register(&global.base); ++ return 0; ++ ++err_priorities: ++ kmem_cache_destroy(global.slab_priorities); ++ return -ENOMEM; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_scheduler.h b/drivers/gpu/drm/i915_legacy/i915_scheduler.h +new file mode 100644 +index 000000000000..07d243acf553 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_scheduler.h +@@ -0,0 +1,55 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ */ ++ ++#ifndef _I915_SCHEDULER_H_ ++#define _I915_SCHEDULER_H_ ++ ++#include ++#include ++#include ++ ++#include "i915_scheduler_types.h" ++ ++#define priolist_for_each_request(it, plist, idx) \ ++ for (idx = 0; idx < ARRAY_SIZE((plist)->requests); idx++) \ ++ list_for_each_entry(it, &(plist)->requests[idx], sched.link) ++ ++#define priolist_for_each_request_consume(it, n, plist, idx) \ ++ for (; \ ++ (plist)->used ? (idx = __ffs((plist)->used)), 1 : 0; \ ++ (plist)->used &= ~BIT(idx)) \ ++ list_for_each_entry_safe(it, n, \ ++ &(plist)->requests[idx], \ ++ sched.link) ++ ++void i915_sched_node_init(struct i915_sched_node *node); ++ ++bool __i915_sched_node_add_dependency(struct i915_sched_node *node, ++ struct i915_sched_node *signal, ++ struct i915_dependency *dep, ++ unsigned long flags); ++ ++int i915_sched_node_add_dependency(struct i915_sched_node *node, ++ struct i915_sched_node *signal); ++ ++void i915_sched_node_fini(struct i915_sched_node *node); ++ ++void i915_schedule(struct i915_request *request, ++ const struct i915_sched_attr *attr); ++ ++void i915_schedule_bump_priority(struct i915_request *rq, unsigned int bump); ++ ++struct list_head * ++i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio); ++ ++void __i915_priolist_free(struct i915_priolist *p); ++static inline void i915_priolist_free(struct i915_priolist *p) ++{ ++ if (p->priority != I915_PRIORITY_NORMAL) ++ __i915_priolist_free(p); ++} ++ ++#endif /* _I915_SCHEDULER_H_ */ +diff --git a/drivers/gpu/drm/i915_legacy/i915_scheduler_types.h b/drivers/gpu/drm/i915_legacy/i915_scheduler_types.h +new file mode 100644 +index 000000000000..4f2b2eb7c3e5 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_scheduler_types.h +@@ -0,0 +1,73 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ */ ++ ++#ifndef _I915_SCHEDULER_TYPES_H_ ++#define _I915_SCHEDULER_TYPES_H_ ++ ++#include ++ ++#include "i915_priolist_types.h" ++#include "intel_engine_types.h" ++ ++struct drm_i915_private; ++struct i915_request; ++struct intel_engine_cs; ++ ++struct i915_sched_attr { ++ /** ++ * @priority: execution and service priority ++ * ++ * All clients are equal, but some are more equal than others! ++ * ++ * Requests from a context with a greater (more positive) value of ++ * @priority will be executed before those with a lower @priority ++ * value, forming a simple QoS. ++ * ++ * The &drm_i915_private.kernel_context is assigned the lowest priority. ++ */ ++ int priority; ++}; ++ ++/* ++ * "People assume that time is a strict progression of cause to effect, but ++ * actually, from a nonlinear, non-subjective viewpoint, it's more like a big ++ * ball of wibbly-wobbly, timey-wimey ... stuff." -The Doctor, 2015 ++ * ++ * Requests exist in a complex web of interdependencies. Each request ++ * has to wait for some other request to complete before it is ready to be run ++ * (e.g. we have to wait until the pixels have been rendering into a texture ++ * before we can copy from it). We track the readiness of a request in terms ++ * of fences, but we also need to keep the dependency tree for the lifetime ++ * of the request (beyond the life of an individual fence). We use the tree ++ * at various points to reorder the requests whilst keeping the requests ++ * in order with respect to their various dependencies. ++ * ++ * There is no active component to the "scheduler". As we know the dependency ++ * DAG of each request, we are able to insert it into a sorted queue when it ++ * is ready, and are able to reorder its portion of the graph to accommodate ++ * dynamic priority changes. ++ */ ++struct i915_sched_node { ++ struct list_head signalers_list; /* those before us, we depend upon */ ++ struct list_head waiters_list; /* those after us, they depend upon us */ ++ struct list_head link; ++ struct i915_sched_attr attr; ++ unsigned int flags; ++#define I915_SCHED_HAS_SEMAPHORE_CHAIN BIT(0) ++ intel_engine_mask_t semaphores; ++}; ++ ++struct i915_dependency { ++ struct i915_sched_node *signaler; ++ struct list_head signal_link; ++ struct list_head wait_link; ++ struct list_head dfs_link; ++ unsigned long flags; ++#define I915_DEPENDENCY_ALLOC BIT(0) ++#define I915_DEPENDENCY_EXTERNAL BIT(1) ++}; ++ ++#endif /* _I915_SCHEDULER_TYPES_H_ */ +diff --git a/drivers/gpu/drm/i915_legacy/i915_selftest.h b/drivers/gpu/drm/i915_legacy/i915_selftest.h +new file mode 100644 +index 000000000000..207e21b478f2 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_selftest.h +@@ -0,0 +1,105 @@ ++/* ++ * Copyright © 2016 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ */ ++ ++#ifndef __I915_SELFTEST_H__ ++#define __I915_SELFTEST_H__ ++ ++struct pci_dev; ++struct drm_i915_private; ++ ++struct i915_selftest { ++ unsigned long timeout_jiffies; ++ unsigned int timeout_ms; ++ unsigned int random_seed; ++ char *filter; ++ int mock; ++ int live; ++}; ++ ++#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) ++#include ++ ++extern struct i915_selftest i915_selftest; ++ ++int i915_mock_selftests(void); ++int i915_live_selftests(struct pci_dev *pdev); ++ ++/* We extract the function declarations from i915_mock_selftests.h and ++ * i915_live_selftests.h Add your unit test declarations there! ++ * ++ * Mock unit tests are run very early upon module load, before the driver ++ * is probed. All hardware interactions, as well as other subsystems, must ++ * be "mocked". ++ * ++ * Live unit tests are run after the driver is loaded - all hardware ++ * interactions are real. ++ */ ++#define selftest(name, func) int func(void); ++#include "selftests/i915_mock_selftests.h" ++#undef selftest ++#define selftest(name, func) int func(struct drm_i915_private *i915); ++#include "selftests/i915_live_selftests.h" ++#undef selftest ++ ++struct i915_subtest { ++ int (*func)(void *data); ++ const char *name; ++}; ++ ++int __i915_subtests(const char *caller, ++ const struct i915_subtest *st, ++ unsigned int count, ++ void *data); ++#define i915_subtests(T, data) \ ++ __i915_subtests(__func__, T, ARRAY_SIZE(T), data) ++ ++#define SUBTEST(x) { x, #x } ++ ++#define I915_SELFTEST_DECLARE(x) x ++#define I915_SELFTEST_ONLY(x) unlikely(x) ++ ++#else /* !IS_ENABLED(CONFIG_DRM_I915_SELFTEST) */ ++ ++static inline int i915_mock_selftests(void) { return 0; } ++static inline int i915_live_selftests(struct pci_dev *pdev) { return 0; } ++ ++#define I915_SELFTEST_DECLARE(x) ++#define I915_SELFTEST_ONLY(x) 0 ++ ++#endif ++ ++/* Using the i915_selftest_ prefix becomes a little unwieldy with the helpers. ++ * Instead we use the igt_ shorthand, in reference to the intel-gpu-tools ++ * suite of uabi test cases (which includes a test runner for our selftests). ++ */ ++ ++#define IGT_TIMEOUT(name__) \ ++ unsigned long name__ = jiffies + i915_selftest.timeout_jiffies ++ ++__printf(2, 3) ++bool __igt_timeout(unsigned long timeout, const char *fmt, ...); ++ ++#define igt_timeout(t, fmt, ...) \ ++ __igt_timeout((t), KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__) ++ ++#endif /* !__I915_SELFTEST_H__ */ +diff --git a/drivers/gpu/drm/i915_legacy/i915_suspend.c b/drivers/gpu/drm/i915_legacy/i915_suspend.c +new file mode 100644 +index 000000000000..95f3dab1b229 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_suspend.c +@@ -0,0 +1,150 @@ ++/* ++ * ++ * Copyright 2008 (c) Intel Corporation ++ * Jesse Barnes ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the ++ * "Software"), to deal in the Software without restriction, including ++ * without limitation the rights to use, copy, modify, merge, publish, ++ * distribute, sub license, and/or sell copies of the Software, and to ++ * permit persons to whom the Software is furnished to do so, subject to ++ * the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the ++ * next paragraph) shall be included in all copies or substantial portions ++ * of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS ++ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF ++ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. ++ * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR ++ * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, ++ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE ++ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++#include ++ ++#include "i915_reg.h" ++#include "intel_drv.h" ++#include "intel_fbc.h" ++ ++static void i915_save_display(struct drm_i915_private *dev_priv) ++{ ++ /* Display arbitration control */ ++ if (INTEL_GEN(dev_priv) <= 4) ++ dev_priv->regfile.saveDSPARB = I915_READ(DSPARB); ++ ++ /* save FBC interval */ ++ if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) <= 4 && !IS_G4X(dev_priv)) ++ dev_priv->regfile.saveFBC_CONTROL = I915_READ(FBC_CONTROL); ++} ++ ++static void i915_restore_display(struct drm_i915_private *dev_priv) ++{ ++ /* Display arbitration */ ++ if (INTEL_GEN(dev_priv) <= 4) ++ I915_WRITE(DSPARB, dev_priv->regfile.saveDSPARB); ++ ++ /* only restore FBC info on the platform that supports FBC*/ ++ intel_fbc_global_disable(dev_priv); ++ ++ /* restore FBC interval */ ++ if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) <= 4 && !IS_G4X(dev_priv)) ++ I915_WRITE(FBC_CONTROL, dev_priv->regfile.saveFBC_CONTROL); ++ ++ i915_redisable_vga(dev_priv); ++} ++ ++int i915_save_state(struct drm_i915_private *dev_priv) ++{ ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ int i; ++ ++ mutex_lock(&dev_priv->drm.struct_mutex); ++ ++ i915_save_display(dev_priv); ++ ++ if (IS_GEN(dev_priv, 4)) ++ pci_read_config_word(pdev, GCDGMBUS, ++ &dev_priv->regfile.saveGCDGMBUS); ++ ++ /* Cache mode state */ ++ if (INTEL_GEN(dev_priv) < 7) ++ dev_priv->regfile.saveCACHE_MODE_0 = I915_READ(CACHE_MODE_0); ++ ++ /* Memory Arbitration state */ ++ dev_priv->regfile.saveMI_ARB_STATE = I915_READ(MI_ARB_STATE); ++ ++ /* Scratch space */ ++ if (IS_GEN(dev_priv, 2) && IS_MOBILE(dev_priv)) { ++ for (i = 0; i < 7; i++) { ++ dev_priv->regfile.saveSWF0[i] = I915_READ(SWF0(i)); ++ dev_priv->regfile.saveSWF1[i] = I915_READ(SWF1(i)); ++ } ++ for (i = 0; i < 3; i++) ++ dev_priv->regfile.saveSWF3[i] = I915_READ(SWF3(i)); ++ } else if (IS_GEN(dev_priv, 2)) { ++ for (i = 0; i < 7; i++) ++ dev_priv->regfile.saveSWF1[i] = I915_READ(SWF1(i)); ++ } else if (HAS_GMCH(dev_priv)) { ++ for (i = 0; i < 16; i++) { ++ dev_priv->regfile.saveSWF0[i] = I915_READ(SWF0(i)); ++ dev_priv->regfile.saveSWF1[i] = I915_READ(SWF1(i)); ++ } ++ for (i = 0; i < 3; i++) ++ dev_priv->regfile.saveSWF3[i] = I915_READ(SWF3(i)); ++ } ++ ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++ ++ return 0; ++} ++ ++int i915_restore_state(struct drm_i915_private *dev_priv) ++{ ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ int i; ++ ++ mutex_lock(&dev_priv->drm.struct_mutex); ++ ++ if (IS_GEN(dev_priv, 4)) ++ pci_write_config_word(pdev, GCDGMBUS, ++ dev_priv->regfile.saveGCDGMBUS); ++ i915_restore_display(dev_priv); ++ ++ /* Cache mode state */ ++ if (INTEL_GEN(dev_priv) < 7) ++ I915_WRITE(CACHE_MODE_0, dev_priv->regfile.saveCACHE_MODE_0 | ++ 0xffff0000); ++ ++ /* Memory arbitration state */ ++ I915_WRITE(MI_ARB_STATE, dev_priv->regfile.saveMI_ARB_STATE | 0xffff0000); ++ ++ /* Scratch space */ ++ if (IS_GEN(dev_priv, 2) && IS_MOBILE(dev_priv)) { ++ for (i = 0; i < 7; i++) { ++ I915_WRITE(SWF0(i), dev_priv->regfile.saveSWF0[i]); ++ I915_WRITE(SWF1(i), dev_priv->regfile.saveSWF1[i]); ++ } ++ for (i = 0; i < 3; i++) ++ I915_WRITE(SWF3(i), dev_priv->regfile.saveSWF3[i]); ++ } else if (IS_GEN(dev_priv, 2)) { ++ for (i = 0; i < 7; i++) ++ I915_WRITE(SWF1(i), dev_priv->regfile.saveSWF1[i]); ++ } else if (HAS_GMCH(dev_priv)) { ++ for (i = 0; i < 16; i++) { ++ I915_WRITE(SWF0(i), dev_priv->regfile.saveSWF0[i]); ++ I915_WRITE(SWF1(i), dev_priv->regfile.saveSWF1[i]); ++ } ++ for (i = 0; i < 3; i++) ++ I915_WRITE(SWF3(i), dev_priv->regfile.saveSWF3[i]); ++ } ++ ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++ ++ intel_i2c_reset(dev_priv); ++ ++ return 0; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_sw_fence.c b/drivers/gpu/drm/i915_legacy/i915_sw_fence.c +new file mode 100644 +index 000000000000..5387aafd3424 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_sw_fence.c +@@ -0,0 +1,576 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * (C) Copyright 2016 Intel Corporation ++ */ ++ ++#include ++#include ++#include ++#include ++ ++#include "i915_sw_fence.h" ++#include "i915_selftest.h" ++ ++#define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */ ++ ++static DEFINE_SPINLOCK(i915_sw_fence_lock); ++ ++enum { ++ DEBUG_FENCE_IDLE = 0, ++ DEBUG_FENCE_NOTIFY, ++}; ++ ++static void *i915_sw_fence_debug_hint(void *addr) ++{ ++ return (void *)(((struct i915_sw_fence *)addr)->flags & I915_SW_FENCE_MASK); ++} ++ ++#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS ++ ++static struct debug_obj_descr i915_sw_fence_debug_descr = { ++ .name = "i915_sw_fence", ++ .debug_hint = i915_sw_fence_debug_hint, ++}; ++ ++static inline void debug_fence_init(struct i915_sw_fence *fence) ++{ ++ debug_object_init(fence, &i915_sw_fence_debug_descr); ++} ++ ++static inline void debug_fence_init_onstack(struct i915_sw_fence *fence) ++{ ++ debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr); ++} ++ ++static inline void debug_fence_activate(struct i915_sw_fence *fence) ++{ ++ debug_object_activate(fence, &i915_sw_fence_debug_descr); ++} ++ ++static inline void debug_fence_set_state(struct i915_sw_fence *fence, ++ int old, int new) ++{ ++ debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new); ++} ++ ++static inline void debug_fence_deactivate(struct i915_sw_fence *fence) ++{ ++ debug_object_deactivate(fence, &i915_sw_fence_debug_descr); ++} ++ ++static inline void debug_fence_destroy(struct i915_sw_fence *fence) ++{ ++ debug_object_destroy(fence, &i915_sw_fence_debug_descr); ++} ++ ++static inline void debug_fence_free(struct i915_sw_fence *fence) ++{ ++ debug_object_free(fence, &i915_sw_fence_debug_descr); ++ smp_wmb(); /* flush the change in state before reallocation */ ++} ++ ++static inline void debug_fence_assert(struct i915_sw_fence *fence) ++{ ++ debug_object_assert_init(fence, &i915_sw_fence_debug_descr); ++} ++ ++#else ++ ++static inline void debug_fence_init(struct i915_sw_fence *fence) ++{ ++} ++ ++static inline void debug_fence_init_onstack(struct i915_sw_fence *fence) ++{ ++} ++ ++static inline void debug_fence_activate(struct i915_sw_fence *fence) ++{ ++} ++ ++static inline void debug_fence_set_state(struct i915_sw_fence *fence, ++ int old, int new) ++{ ++} ++ ++static inline void debug_fence_deactivate(struct i915_sw_fence *fence) ++{ ++} ++ ++static inline void debug_fence_destroy(struct i915_sw_fence *fence) ++{ ++} ++ ++static inline void debug_fence_free(struct i915_sw_fence *fence) ++{ ++} ++ ++static inline void debug_fence_assert(struct i915_sw_fence *fence) ++{ ++} ++ ++#endif ++ ++static int __i915_sw_fence_notify(struct i915_sw_fence *fence, ++ enum i915_sw_fence_notify state) ++{ ++ i915_sw_fence_notify_t fn; ++ ++ fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK); ++ return fn(fence, state); ++} ++ ++#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS ++void i915_sw_fence_fini(struct i915_sw_fence *fence) ++{ ++ debug_fence_free(fence); ++} ++#endif ++ ++static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence, ++ struct list_head *continuation) ++{ ++ wait_queue_head_t *x = &fence->wait; ++ wait_queue_entry_t *pos, *next; ++ unsigned long flags; ++ ++ debug_fence_deactivate(fence); ++ atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */ ++ ++ /* ++ * To prevent unbounded recursion as we traverse the graph of ++ * i915_sw_fences, we move the entry list from this, the next ready ++ * fence, to the tail of the original fence's entry list ++ * (and so added to the list to be woken). ++ */ ++ ++ spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation); ++ if (continuation) { ++ list_for_each_entry_safe(pos, next, &x->head, entry) { ++ if (pos->func == autoremove_wake_function) ++ pos->func(pos, TASK_NORMAL, 0, continuation); ++ else ++ list_move_tail(&pos->entry, continuation); ++ } ++ } else { ++ LIST_HEAD(extra); ++ ++ do { ++ list_for_each_entry_safe(pos, next, &x->head, entry) ++ pos->func(pos, TASK_NORMAL, 0, &extra); ++ ++ if (list_empty(&extra)) ++ break; ++ ++ list_splice_tail_init(&extra, &x->head); ++ } while (1); ++ } ++ spin_unlock_irqrestore(&x->lock, flags); ++ ++ debug_fence_assert(fence); ++} ++ ++static void __i915_sw_fence_complete(struct i915_sw_fence *fence, ++ struct list_head *continuation) ++{ ++ debug_fence_assert(fence); ++ ++ if (!atomic_dec_and_test(&fence->pending)) ++ return; ++ ++ debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY); ++ ++ if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE) ++ return; ++ ++ debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE); ++ ++ __i915_sw_fence_wake_up_all(fence, continuation); ++ ++ debug_fence_destroy(fence); ++ __i915_sw_fence_notify(fence, FENCE_FREE); ++} ++ ++void i915_sw_fence_complete(struct i915_sw_fence *fence) ++{ ++ debug_fence_assert(fence); ++ ++ if (WARN_ON(i915_sw_fence_done(fence))) ++ return; ++ ++ __i915_sw_fence_complete(fence, NULL); ++} ++ ++void i915_sw_fence_await(struct i915_sw_fence *fence) ++{ ++ debug_fence_assert(fence); ++ WARN_ON(atomic_inc_return(&fence->pending) <= 1); ++} ++ ++void __i915_sw_fence_init(struct i915_sw_fence *fence, ++ i915_sw_fence_notify_t fn, ++ const char *name, ++ struct lock_class_key *key) ++{ ++ BUG_ON(!fn || (unsigned long)fn & ~I915_SW_FENCE_MASK); ++ ++ debug_fence_init(fence); ++ ++ __init_waitqueue_head(&fence->wait, name, key); ++ atomic_set(&fence->pending, 1); ++ fence->flags = (unsigned long)fn; ++} ++ ++void i915_sw_fence_commit(struct i915_sw_fence *fence) ++{ ++ debug_fence_activate(fence); ++ i915_sw_fence_complete(fence); ++} ++ ++static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key) ++{ ++ list_del(&wq->entry); ++ __i915_sw_fence_complete(wq->private, key); ++ ++ if (wq->flags & I915_SW_FENCE_FLAG_ALLOC) ++ kfree(wq); ++ return 0; ++} ++ ++static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence, ++ const struct i915_sw_fence * const signaler) ++{ ++ wait_queue_entry_t *wq; ++ ++ if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) ++ return false; ++ ++ if (fence == signaler) ++ return true; ++ ++ list_for_each_entry(wq, &fence->wait.head, entry) { ++ if (wq->func != i915_sw_fence_wake) ++ continue; ++ ++ if (__i915_sw_fence_check_if_after(wq->private, signaler)) ++ return true; ++ } ++ ++ return false; ++} ++ ++static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence) ++{ ++ wait_queue_entry_t *wq; ++ ++ if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) ++ return; ++ ++ list_for_each_entry(wq, &fence->wait.head, entry) { ++ if (wq->func != i915_sw_fence_wake) ++ continue; ++ ++ __i915_sw_fence_clear_checked_bit(wq->private); ++ } ++} ++ ++static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence, ++ const struct i915_sw_fence * const signaler) ++{ ++ unsigned long flags; ++ bool err; ++ ++ if (!IS_ENABLED(CONFIG_DRM_I915_SW_FENCE_CHECK_DAG)) ++ return false; ++ ++ spin_lock_irqsave(&i915_sw_fence_lock, flags); ++ err = __i915_sw_fence_check_if_after(fence, signaler); ++ __i915_sw_fence_clear_checked_bit(fence); ++ spin_unlock_irqrestore(&i915_sw_fence_lock, flags); ++ ++ return err; ++} ++ ++static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, ++ struct i915_sw_fence *signaler, ++ wait_queue_entry_t *wq, gfp_t gfp) ++{ ++ unsigned long flags; ++ int pending; ++ ++ debug_fence_assert(fence); ++ might_sleep_if(gfpflags_allow_blocking(gfp)); ++ ++ if (i915_sw_fence_done(signaler)) ++ return 0; ++ ++ debug_fence_assert(signaler); ++ ++ /* The dependency graph must be acyclic. */ ++ if (unlikely(i915_sw_fence_check_if_after(fence, signaler))) ++ return -EINVAL; ++ ++ pending = 0; ++ if (!wq) { ++ wq = kmalloc(sizeof(*wq), gfp); ++ if (!wq) { ++ if (!gfpflags_allow_blocking(gfp)) ++ return -ENOMEM; ++ ++ i915_sw_fence_wait(signaler); ++ return 0; ++ } ++ ++ pending |= I915_SW_FENCE_FLAG_ALLOC; ++ } ++ ++ INIT_LIST_HEAD(&wq->entry); ++ wq->flags = pending; ++ wq->func = i915_sw_fence_wake; ++ wq->private = fence; ++ ++ i915_sw_fence_await(fence); ++ ++ spin_lock_irqsave(&signaler->wait.lock, flags); ++ if (likely(!i915_sw_fence_done(signaler))) { ++ __add_wait_queue_entry_tail(&signaler->wait, wq); ++ pending = 1; ++ } else { ++ i915_sw_fence_wake(wq, 0, 0, NULL); ++ pending = 0; ++ } ++ spin_unlock_irqrestore(&signaler->wait.lock, flags); ++ ++ return pending; ++} ++ ++int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, ++ struct i915_sw_fence *signaler, ++ wait_queue_entry_t *wq) ++{ ++ return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0); ++} ++ ++int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence, ++ struct i915_sw_fence *signaler, ++ gfp_t gfp) ++{ ++ return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp); ++} ++ ++struct i915_sw_dma_fence_cb_timer { ++ struct i915_sw_dma_fence_cb base; ++ struct dma_fence *dma; ++ struct timer_list timer; ++ struct irq_work work; ++ struct rcu_head rcu; ++}; ++ ++static void dma_i915_sw_fence_wake(struct dma_fence *dma, ++ struct dma_fence_cb *data) ++{ ++ struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base); ++ ++ i915_sw_fence_complete(cb->fence); ++ kfree(cb); ++} ++ ++static void timer_i915_sw_fence_wake(struct timer_list *t) ++{ ++ struct i915_sw_dma_fence_cb_timer *cb = from_timer(cb, t, timer); ++ struct i915_sw_fence *fence; ++ ++ fence = xchg(&cb->base.fence, NULL); ++ if (!fence) ++ return; ++ ++ pr_notice("Asynchronous wait on fence %s:%s:%llx timed out (hint:%pS)\n", ++ cb->dma->ops->get_driver_name(cb->dma), ++ cb->dma->ops->get_timeline_name(cb->dma), ++ cb->dma->seqno, ++ i915_sw_fence_debug_hint(fence)); ++ ++ i915_sw_fence_complete(fence); ++} ++ ++static void dma_i915_sw_fence_wake_timer(struct dma_fence *dma, ++ struct dma_fence_cb *data) ++{ ++ struct i915_sw_dma_fence_cb_timer *cb = ++ container_of(data, typeof(*cb), base.base); ++ struct i915_sw_fence *fence; ++ ++ fence = xchg(&cb->base.fence, NULL); ++ if (fence) ++ i915_sw_fence_complete(fence); ++ ++ irq_work_queue(&cb->work); ++} ++ ++static void irq_i915_sw_fence_work(struct irq_work *wrk) ++{ ++ struct i915_sw_dma_fence_cb_timer *cb = ++ container_of(wrk, typeof(*cb), work); ++ ++ del_timer_sync(&cb->timer); ++ dma_fence_put(cb->dma); ++ ++ kfree_rcu(cb, rcu); ++} ++ ++int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, ++ struct dma_fence *dma, ++ unsigned long timeout, ++ gfp_t gfp) ++{ ++ struct i915_sw_dma_fence_cb *cb; ++ dma_fence_func_t func; ++ int ret; ++ ++ debug_fence_assert(fence); ++ might_sleep_if(gfpflags_allow_blocking(gfp)); ++ ++ if (dma_fence_is_signaled(dma)) ++ return 0; ++ ++ cb = kmalloc(timeout ? ++ sizeof(struct i915_sw_dma_fence_cb_timer) : ++ sizeof(struct i915_sw_dma_fence_cb), ++ gfp); ++ if (!cb) { ++ if (!gfpflags_allow_blocking(gfp)) ++ return -ENOMEM; ++ ++ return dma_fence_wait(dma, false); ++ } ++ ++ cb->fence = fence; ++ i915_sw_fence_await(fence); ++ ++ func = dma_i915_sw_fence_wake; ++ if (timeout) { ++ struct i915_sw_dma_fence_cb_timer *timer = ++ container_of(cb, typeof(*timer), base); ++ ++ timer->dma = dma_fence_get(dma); ++ init_irq_work(&timer->work, irq_i915_sw_fence_work); ++ ++ timer_setup(&timer->timer, ++ timer_i915_sw_fence_wake, TIMER_IRQSAFE); ++ mod_timer(&timer->timer, round_jiffies_up(jiffies + timeout)); ++ ++ func = dma_i915_sw_fence_wake_timer; ++ } ++ ++ ret = dma_fence_add_callback(dma, &cb->base, func); ++ if (ret == 0) { ++ ret = 1; ++ } else { ++ func(dma, &cb->base); ++ if (ret == -ENOENT) /* fence already signaled */ ++ ret = 0; ++ } ++ ++ return ret; ++} ++ ++static void __dma_i915_sw_fence_wake(struct dma_fence *dma, ++ struct dma_fence_cb *data) ++{ ++ struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base); ++ ++ i915_sw_fence_complete(cb->fence); ++} ++ ++int __i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, ++ struct dma_fence *dma, ++ struct i915_sw_dma_fence_cb *cb) ++{ ++ int ret; ++ ++ debug_fence_assert(fence); ++ ++ if (dma_fence_is_signaled(dma)) ++ return 0; ++ ++ cb->fence = fence; ++ i915_sw_fence_await(fence); ++ ++ ret = dma_fence_add_callback(dma, &cb->base, __dma_i915_sw_fence_wake); ++ if (ret == 0) { ++ ret = 1; ++ } else { ++ i915_sw_fence_complete(fence); ++ if (ret == -ENOENT) /* fence already signaled */ ++ ret = 0; ++ } ++ ++ return ret; ++} ++ ++int i915_sw_fence_await_reservation(struct i915_sw_fence *fence, ++ struct reservation_object *resv, ++ const struct dma_fence_ops *exclude, ++ bool write, ++ unsigned long timeout, ++ gfp_t gfp) ++{ ++ struct dma_fence *excl; ++ int ret = 0, pending; ++ ++ debug_fence_assert(fence); ++ might_sleep_if(gfpflags_allow_blocking(gfp)); ++ ++ if (write) { ++ struct dma_fence **shared; ++ unsigned int count, i; ++ ++ ret = reservation_object_get_fences_rcu(resv, ++ &excl, &count, &shared); ++ if (ret) ++ return ret; ++ ++ for (i = 0; i < count; i++) { ++ if (shared[i]->ops == exclude) ++ continue; ++ ++ pending = i915_sw_fence_await_dma_fence(fence, ++ shared[i], ++ timeout, ++ gfp); ++ if (pending < 0) { ++ ret = pending; ++ break; ++ } ++ ++ ret |= pending; ++ } ++ ++ for (i = 0; i < count; i++) ++ dma_fence_put(shared[i]); ++ kfree(shared); ++ } else { ++ excl = reservation_object_get_excl_rcu(resv); ++ } ++ ++ if (ret >= 0 && excl && excl->ops != exclude) { ++ pending = i915_sw_fence_await_dma_fence(fence, ++ excl, ++ timeout, ++ gfp); ++ if (pending < 0) ++ ret = pending; ++ else ++ ret |= pending; ++ } ++ ++ dma_fence_put(excl); ++ ++ return ret; ++} ++ ++#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) ++#include "selftests/lib_sw_fence.c" ++#include "selftests/i915_sw_fence.c" ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_sw_fence.h b/drivers/gpu/drm/i915_legacy/i915_sw_fence.h +new file mode 100644 +index 000000000000..9cb5c3b307a6 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_sw_fence.h +@@ -0,0 +1,109 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * i915_sw_fence.h - library routines for N:M synchronisation points ++ * ++ * Copyright (C) 2016 Intel Corporation ++ */ ++ ++#ifndef _I915_SW_FENCE_H_ ++#define _I915_SW_FENCE_H_ ++ ++#include ++#include ++#include ++#include /* for NOTIFY_DONE */ ++#include ++ ++struct completion; ++struct reservation_object; ++ ++struct i915_sw_fence { ++ wait_queue_head_t wait; ++ unsigned long flags; ++ atomic_t pending; ++}; ++ ++#define I915_SW_FENCE_CHECKED_BIT 0 /* used internally for DAG checking */ ++#define I915_SW_FENCE_PRIVATE_BIT 1 /* available for use by owner */ ++#define I915_SW_FENCE_MASK (~3) ++ ++enum i915_sw_fence_notify { ++ FENCE_COMPLETE, ++ FENCE_FREE ++}; ++ ++typedef int (*i915_sw_fence_notify_t)(struct i915_sw_fence *, ++ enum i915_sw_fence_notify state); ++#define __i915_sw_fence_call __aligned(4) ++ ++void __i915_sw_fence_init(struct i915_sw_fence *fence, ++ i915_sw_fence_notify_t fn, ++ const char *name, ++ struct lock_class_key *key); ++#ifdef CONFIG_LOCKDEP ++#define i915_sw_fence_init(fence, fn) \ ++do { \ ++ static struct lock_class_key __key; \ ++ \ ++ __i915_sw_fence_init((fence), (fn), #fence, &__key); \ ++} while (0) ++#else ++#define i915_sw_fence_init(fence, fn) \ ++ __i915_sw_fence_init((fence), (fn), NULL, NULL) ++#endif ++ ++#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS ++void i915_sw_fence_fini(struct i915_sw_fence *fence); ++#else ++static inline void i915_sw_fence_fini(struct i915_sw_fence *fence) {} ++#endif ++ ++void i915_sw_fence_commit(struct i915_sw_fence *fence); ++ ++int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, ++ struct i915_sw_fence *after, ++ wait_queue_entry_t *wq); ++int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence, ++ struct i915_sw_fence *after, ++ gfp_t gfp); ++ ++struct i915_sw_dma_fence_cb { ++ struct dma_fence_cb base; ++ struct i915_sw_fence *fence; ++}; ++ ++int __i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, ++ struct dma_fence *dma, ++ struct i915_sw_dma_fence_cb *cb); ++int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, ++ struct dma_fence *dma, ++ unsigned long timeout, ++ gfp_t gfp); ++ ++int i915_sw_fence_await_reservation(struct i915_sw_fence *fence, ++ struct reservation_object *resv, ++ const struct dma_fence_ops *exclude, ++ bool write, ++ unsigned long timeout, ++ gfp_t gfp); ++ ++void i915_sw_fence_await(struct i915_sw_fence *fence); ++void i915_sw_fence_complete(struct i915_sw_fence *fence); ++ ++static inline bool i915_sw_fence_signaled(const struct i915_sw_fence *fence) ++{ ++ return atomic_read(&fence->pending) <= 0; ++} ++ ++static inline bool i915_sw_fence_done(const struct i915_sw_fence *fence) ++{ ++ return atomic_read(&fence->pending) < 0; ++} ++ ++static inline void i915_sw_fence_wait(struct i915_sw_fence *fence) ++{ ++ wait_event(fence->wait, i915_sw_fence_done(fence)); ++} ++ ++#endif /* _I915_SW_FENCE_H_ */ +diff --git a/drivers/gpu/drm/i915_legacy/i915_syncmap.c b/drivers/gpu/drm/i915_legacy/i915_syncmap.c +new file mode 100644 +index 000000000000..60404dbb2e9f +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_syncmap.c +@@ -0,0 +1,412 @@ ++/* ++ * Copyright © 2017 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#include ++ ++#include "i915_syncmap.h" ++ ++#include "i915_gem.h" /* GEM_BUG_ON() */ ++#include "i915_selftest.h" ++ ++#define SHIFT ilog2(KSYNCMAP) ++#define MASK (KSYNCMAP - 1) ++ ++/* ++ * struct i915_syncmap is a layer of a radixtree that maps a u64 fence ++ * context id to the last u32 fence seqno waited upon from that context. ++ * Unlike lib/radixtree it uses a parent pointer that allows traversal back to ++ * the root. This allows us to access the whole tree via a single pointer ++ * to the most recently used layer. We expect fence contexts to be dense ++ * and most reuse to be on the same i915_gem_context but on neighbouring ++ * engines (i.e. on adjacent contexts) and reuse the same leaf, a very ++ * effective lookup cache. If the new lookup is not on the same leaf, we ++ * expect it to be on the neighbouring branch. ++ * ++ * A leaf holds an array of u32 seqno, and has height 0. The bitmap field ++ * allows us to store whether a particular seqno is valid (i.e. allows us ++ * to distinguish unset from 0). ++ * ++ * A branch holds an array of layer pointers, and has height > 0, and always ++ * has at least 2 layers (either branches or leaves) below it. ++ * ++ * For example, ++ * for x in ++ * 0 1 2 0x10 0x11 0x200 0x201 ++ * 0x500000 0x500001 0x503000 0x503001 ++ * 0xE<<60: ++ * i915_syncmap_set(&sync, x, lower_32_bits(x)); ++ * will build a tree like: ++ * 0xXXXXXXXXXXXXXXXX ++ * 0-> 0x0000000000XXXXXX ++ * | 0-> 0x0000000000000XXX ++ * | | 0-> 0x00000000000000XX ++ * | | | 0-> 0x000000000000000X 0:0, 1:1, 2:2 ++ * | | | 1-> 0x000000000000001X 0:10, 1:11 ++ * | | 2-> 0x000000000000020X 0:200, 1:201 ++ * | 5-> 0x000000000050XXXX ++ * | 0-> 0x000000000050000X 0:500000, 1:500001 ++ * | 3-> 0x000000000050300X 0:503000, 1:503001 ++ * e-> 0xe00000000000000X e:e ++ */ ++ ++struct i915_syncmap { ++ u64 prefix; ++ unsigned int height; ++ unsigned int bitmap; ++ struct i915_syncmap *parent; ++ /* ++ * Following this header is an array of either seqno or child pointers: ++ * union { ++ * u32 seqno[KSYNCMAP]; ++ * struct i915_syncmap *child[KSYNCMAP]; ++ * }; ++ */ ++}; ++ ++/** ++ * i915_syncmap_init -- initialise the #i915_syncmap ++ * @root: pointer to the #i915_syncmap ++ */ ++void i915_syncmap_init(struct i915_syncmap **root) ++{ ++ BUILD_BUG_ON_NOT_POWER_OF_2(KSYNCMAP); ++ BUILD_BUG_ON_NOT_POWER_OF_2(SHIFT); ++ BUILD_BUG_ON(KSYNCMAP > BITS_PER_TYPE((*root)->bitmap)); ++ *root = NULL; ++} ++ ++static inline u32 *__sync_seqno(struct i915_syncmap *p) ++{ ++ GEM_BUG_ON(p->height); ++ return (u32 *)(p + 1); ++} ++ ++static inline struct i915_syncmap **__sync_child(struct i915_syncmap *p) ++{ ++ GEM_BUG_ON(!p->height); ++ return (struct i915_syncmap **)(p + 1); ++} ++ ++static inline unsigned int ++__sync_branch_idx(const struct i915_syncmap *p, u64 id) ++{ ++ return (id >> p->height) & MASK; ++} ++ ++static inline unsigned int ++__sync_leaf_idx(const struct i915_syncmap *p, u64 id) ++{ ++ GEM_BUG_ON(p->height); ++ return id & MASK; ++} ++ ++static inline u64 __sync_branch_prefix(const struct i915_syncmap *p, u64 id) ++{ ++ return id >> p->height >> SHIFT; ++} ++ ++static inline u64 __sync_leaf_prefix(const struct i915_syncmap *p, u64 id) ++{ ++ GEM_BUG_ON(p->height); ++ return id >> SHIFT; ++} ++ ++static inline bool seqno_later(u32 a, u32 b) ++{ ++ return (s32)(a - b) >= 0; ++} ++ ++/** ++ * i915_syncmap_is_later -- compare against the last know sync point ++ * @root: pointer to the #i915_syncmap ++ * @id: the context id (other timeline) we are synchronising to ++ * @seqno: the sequence number along the other timeline ++ * ++ * If we have already synchronised this @root timeline with another (@id) then ++ * we can omit any repeated or earlier synchronisation requests. If the two ++ * timelines are already coupled, we can also omit the dependency between the ++ * two as that is already known via the timeline. ++ * ++ * Returns true if the two timelines are already synchronised wrt to @seqno, ++ * false if not and the synchronisation must be emitted. ++ */ ++bool i915_syncmap_is_later(struct i915_syncmap **root, u64 id, u32 seqno) ++{ ++ struct i915_syncmap *p; ++ unsigned int idx; ++ ++ p = *root; ++ if (!p) ++ return false; ++ ++ if (likely(__sync_leaf_prefix(p, id) == p->prefix)) ++ goto found; ++ ++ /* First climb the tree back to a parent branch */ ++ do { ++ p = p->parent; ++ if (!p) ++ return false; ++ ++ if (__sync_branch_prefix(p, id) == p->prefix) ++ break; ++ } while (1); ++ ++ /* And then descend again until we find our leaf */ ++ do { ++ if (!p->height) ++ break; ++ ++ p = __sync_child(p)[__sync_branch_idx(p, id)]; ++ if (!p) ++ return false; ++ ++ if (__sync_branch_prefix(p, id) != p->prefix) ++ return false; ++ } while (1); ++ ++ *root = p; ++found: ++ idx = __sync_leaf_idx(p, id); ++ if (!(p->bitmap & BIT(idx))) ++ return false; ++ ++ return seqno_later(__sync_seqno(p)[idx], seqno); ++} ++ ++static struct i915_syncmap * ++__sync_alloc_leaf(struct i915_syncmap *parent, u64 id) ++{ ++ struct i915_syncmap *p; ++ ++ p = kmalloc(sizeof(*p) + KSYNCMAP * sizeof(u32), GFP_KERNEL); ++ if (unlikely(!p)) ++ return NULL; ++ ++ p->parent = parent; ++ p->height = 0; ++ p->bitmap = 0; ++ p->prefix = __sync_leaf_prefix(p, id); ++ return p; ++} ++ ++static inline void __sync_set_seqno(struct i915_syncmap *p, u64 id, u32 seqno) ++{ ++ unsigned int idx = __sync_leaf_idx(p, id); ++ ++ p->bitmap |= BIT(idx); ++ __sync_seqno(p)[idx] = seqno; ++} ++ ++static inline void __sync_set_child(struct i915_syncmap *p, ++ unsigned int idx, ++ struct i915_syncmap *child) ++{ ++ p->bitmap |= BIT(idx); ++ __sync_child(p)[idx] = child; ++} ++ ++static noinline int __sync_set(struct i915_syncmap **root, u64 id, u32 seqno) ++{ ++ struct i915_syncmap *p = *root; ++ unsigned int idx; ++ ++ if (!p) { ++ p = __sync_alloc_leaf(NULL, id); ++ if (unlikely(!p)) ++ return -ENOMEM; ++ ++ goto found; ++ } ++ ++ /* Caller handled the likely cached case */ ++ GEM_BUG_ON(__sync_leaf_prefix(p, id) == p->prefix); ++ ++ /* Climb back up the tree until we find a common prefix */ ++ do { ++ if (!p->parent) ++ break; ++ ++ p = p->parent; ++ ++ if (__sync_branch_prefix(p, id) == p->prefix) ++ break; ++ } while (1); ++ ++ /* ++ * No shortcut, we have to descend the tree to find the right layer ++ * containing this fence. ++ * ++ * Each layer in the tree holds 16 (KSYNCMAP) pointers, either fences ++ * or lower layers. Leaf nodes (height = 0) contain the fences, all ++ * other nodes (height > 0) are internal layers that point to a lower ++ * node. Each internal layer has at least 2 descendents. ++ * ++ * Starting at the top, we check whether the current prefix matches. If ++ * it doesn't, we have gone past our target and need to insert a join ++ * into the tree, and a new leaf node for the target as a descendent ++ * of the join, as well as the original layer. ++ * ++ * The matching prefix means we are still following the right branch ++ * of the tree. If it has height 0, we have found our leaf and just ++ * need to replace the fence slot with ourselves. If the height is ++ * not zero, our slot contains the next layer in the tree (unless ++ * it is empty, in which case we can add ourselves as a new leaf). ++ * As descend the tree the prefix grows (and height decreases). ++ */ ++ do { ++ struct i915_syncmap *next; ++ ++ if (__sync_branch_prefix(p, id) != p->prefix) { ++ unsigned int above; ++ ++ /* Insert a join above the current layer */ ++ next = kzalloc(sizeof(*next) + KSYNCMAP * sizeof(next), ++ GFP_KERNEL); ++ if (unlikely(!next)) ++ return -ENOMEM; ++ ++ /* Compute the height at which these two diverge */ ++ above = fls64(__sync_branch_prefix(p, id) ^ p->prefix); ++ above = round_up(above, SHIFT); ++ next->height = above + p->height; ++ next->prefix = __sync_branch_prefix(next, id); ++ ++ /* Insert the join into the parent */ ++ if (p->parent) { ++ idx = __sync_branch_idx(p->parent, id); ++ __sync_child(p->parent)[idx] = next; ++ GEM_BUG_ON(!(p->parent->bitmap & BIT(idx))); ++ } ++ next->parent = p->parent; ++ ++ /* Compute the idx of the other branch, not our id! */ ++ idx = p->prefix >> (above - SHIFT) & MASK; ++ __sync_set_child(next, idx, p); ++ p->parent = next; ++ ++ /* Ascend to the join */ ++ p = next; ++ } else { ++ if (!p->height) ++ break; ++ } ++ ++ /* Descend into the next layer */ ++ GEM_BUG_ON(!p->height); ++ idx = __sync_branch_idx(p, id); ++ next = __sync_child(p)[idx]; ++ if (!next) { ++ next = __sync_alloc_leaf(p, id); ++ if (unlikely(!next)) ++ return -ENOMEM; ++ ++ __sync_set_child(p, idx, next); ++ p = next; ++ break; ++ } ++ ++ p = next; ++ } while (1); ++ ++found: ++ GEM_BUG_ON(p->prefix != __sync_leaf_prefix(p, id)); ++ __sync_set_seqno(p, id, seqno); ++ *root = p; ++ return 0; ++} ++ ++/** ++ * i915_syncmap_set -- mark the most recent syncpoint between contexts ++ * @root: pointer to the #i915_syncmap ++ * @id: the context id (other timeline) we have synchronised to ++ * @seqno: the sequence number along the other timeline ++ * ++ * When we synchronise this @root timeline with another (@id), we also know ++ * that we have synchronized with all previous seqno along that timeline. If ++ * we then have a request to synchronise with the same seqno or older, we can ++ * omit it, see i915_syncmap_is_later() ++ * ++ * Returns 0 on success, or a negative error code. ++ */ ++int i915_syncmap_set(struct i915_syncmap **root, u64 id, u32 seqno) ++{ ++ struct i915_syncmap *p = *root; ++ ++ /* ++ * We expect to be called in sequence following is_later(id), which ++ * should have preloaded the root for us. ++ */ ++ if (likely(p && __sync_leaf_prefix(p, id) == p->prefix)) { ++ __sync_set_seqno(p, id, seqno); ++ return 0; ++ } ++ ++ return __sync_set(root, id, seqno); ++} ++ ++static void __sync_free(struct i915_syncmap *p) ++{ ++ if (p->height) { ++ unsigned int i; ++ ++ while ((i = ffs(p->bitmap))) { ++ p->bitmap &= ~0u << i; ++ __sync_free(__sync_child(p)[i - 1]); ++ } ++ } ++ ++ kfree(p); ++} ++ ++/** ++ * i915_syncmap_free -- free all memory associated with the syncmap ++ * @root: pointer to the #i915_syncmap ++ * ++ * Either when the timeline is to be freed and we no longer need the sync ++ * point tracking, or when the fences are all known to be signaled and the ++ * sync point tracking is redundant, we can free the #i915_syncmap to recover ++ * its allocations. ++ * ++ * Will reinitialise the @root pointer so that the #i915_syncmap is ready for ++ * reuse. ++ */ ++void i915_syncmap_free(struct i915_syncmap **root) ++{ ++ struct i915_syncmap *p; ++ ++ p = *root; ++ if (!p) ++ return; ++ ++ while (p->parent) ++ p = p->parent; ++ ++ __sync_free(p); ++ *root = NULL; ++} ++ ++#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) ++#include "selftests/i915_syncmap.c" ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_syncmap.h b/drivers/gpu/drm/i915_legacy/i915_syncmap.h +new file mode 100644 +index 000000000000..0653f70bee82 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_syncmap.h +@@ -0,0 +1,38 @@ ++/* ++ * Copyright © 2017 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#ifndef __I915_SYNCMAP_H__ ++#define __I915_SYNCMAP_H__ ++ ++#include ++ ++struct i915_syncmap; ++#define KSYNCMAP 16 /* radix of the tree, how many slots in each layer */ ++ ++void i915_syncmap_init(struct i915_syncmap **root); ++int i915_syncmap_set(struct i915_syncmap **root, u64 id, u32 seqno); ++bool i915_syncmap_is_later(struct i915_syncmap **root, u64 id, u32 seqno); ++void i915_syncmap_free(struct i915_syncmap **root); ++ ++#endif /* __I915_SYNCMAP_H__ */ +diff --git a/drivers/gpu/drm/i915_legacy/i915_sysfs.c b/drivers/gpu/drm/i915_legacy/i915_sysfs.c +new file mode 100644 +index 000000000000..41313005af42 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_sysfs.c +@@ -0,0 +1,644 @@ ++/* ++ * Copyright © 2012 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ * Authors: ++ * Ben Widawsky ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++#include "intel_drv.h" ++#include "i915_drv.h" ++ ++static inline struct drm_i915_private *kdev_minor_to_i915(struct device *kdev) ++{ ++ struct drm_minor *minor = dev_get_drvdata(kdev); ++ return to_i915(minor->dev); ++} ++ ++#ifdef CONFIG_PM ++static u32 calc_residency(struct drm_i915_private *dev_priv, ++ i915_reg_t reg) ++{ ++ intel_wakeref_t wakeref; ++ u64 res = 0; ++ ++ with_intel_runtime_pm(dev_priv, wakeref) ++ res = intel_rc6_residency_us(dev_priv, reg); ++ ++ return DIV_ROUND_CLOSEST_ULL(res, 1000); ++} ++ ++static ssize_t ++show_rc6_mask(struct device *kdev, struct device_attribute *attr, char *buf) ++{ ++ struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); ++ unsigned int mask; ++ ++ mask = 0; ++ if (HAS_RC6(dev_priv)) ++ mask |= BIT(0); ++ if (HAS_RC6p(dev_priv)) ++ mask |= BIT(1); ++ if (HAS_RC6pp(dev_priv)) ++ mask |= BIT(2); ++ ++ return snprintf(buf, PAGE_SIZE, "%x\n", mask); ++} ++ ++static ssize_t ++show_rc6_ms(struct device *kdev, struct device_attribute *attr, char *buf) ++{ ++ struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); ++ u32 rc6_residency = calc_residency(dev_priv, GEN6_GT_GFX_RC6); ++ return snprintf(buf, PAGE_SIZE, "%u\n", rc6_residency); ++} ++ ++static ssize_t ++show_rc6p_ms(struct device *kdev, struct device_attribute *attr, char *buf) ++{ ++ struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); ++ u32 rc6p_residency = calc_residency(dev_priv, GEN6_GT_GFX_RC6p); ++ return snprintf(buf, PAGE_SIZE, "%u\n", rc6p_residency); ++} ++ ++static ssize_t ++show_rc6pp_ms(struct device *kdev, struct device_attribute *attr, char *buf) ++{ ++ struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); ++ u32 rc6pp_residency = calc_residency(dev_priv, GEN6_GT_GFX_RC6pp); ++ return snprintf(buf, PAGE_SIZE, "%u\n", rc6pp_residency); ++} ++ ++static ssize_t ++show_media_rc6_ms(struct device *kdev, struct device_attribute *attr, char *buf) ++{ ++ struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); ++ u32 rc6_residency = calc_residency(dev_priv, VLV_GT_MEDIA_RC6); ++ return snprintf(buf, PAGE_SIZE, "%u\n", rc6_residency); ++} ++ ++static DEVICE_ATTR(rc6_enable, S_IRUGO, show_rc6_mask, NULL); ++static DEVICE_ATTR(rc6_residency_ms, S_IRUGO, show_rc6_ms, NULL); ++static DEVICE_ATTR(rc6p_residency_ms, S_IRUGO, show_rc6p_ms, NULL); ++static DEVICE_ATTR(rc6pp_residency_ms, S_IRUGO, show_rc6pp_ms, NULL); ++static DEVICE_ATTR(media_rc6_residency_ms, S_IRUGO, show_media_rc6_ms, NULL); ++ ++static struct attribute *rc6_attrs[] = { ++ &dev_attr_rc6_enable.attr, ++ &dev_attr_rc6_residency_ms.attr, ++ NULL ++}; ++ ++static const struct attribute_group rc6_attr_group = { ++ .name = power_group_name, ++ .attrs = rc6_attrs ++}; ++ ++static struct attribute *rc6p_attrs[] = { ++ &dev_attr_rc6p_residency_ms.attr, ++ &dev_attr_rc6pp_residency_ms.attr, ++ NULL ++}; ++ ++static const struct attribute_group rc6p_attr_group = { ++ .name = power_group_name, ++ .attrs = rc6p_attrs ++}; ++ ++static struct attribute *media_rc6_attrs[] = { ++ &dev_attr_media_rc6_residency_ms.attr, ++ NULL ++}; ++ ++static const struct attribute_group media_rc6_attr_group = { ++ .name = power_group_name, ++ .attrs = media_rc6_attrs ++}; ++#endif ++ ++static int l3_access_valid(struct drm_i915_private *dev_priv, loff_t offset) ++{ ++ if (!HAS_L3_DPF(dev_priv)) ++ return -EPERM; ++ ++ if (offset % 4 != 0) ++ return -EINVAL; ++ ++ if (offset >= GEN7_L3LOG_SIZE) ++ return -ENXIO; ++ ++ return 0; ++} ++ ++static ssize_t ++i915_l3_read(struct file *filp, struct kobject *kobj, ++ struct bin_attribute *attr, char *buf, ++ loff_t offset, size_t count) ++{ ++ struct device *kdev = kobj_to_dev(kobj); ++ struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); ++ struct drm_device *dev = &dev_priv->drm; ++ int slice = (int)(uintptr_t)attr->private; ++ int ret; ++ ++ count = round_down(count, 4); ++ ++ ret = l3_access_valid(dev_priv, offset); ++ if (ret) ++ return ret; ++ ++ count = min_t(size_t, GEN7_L3LOG_SIZE - offset, count); ++ ++ ret = i915_mutex_lock_interruptible(dev); ++ if (ret) ++ return ret; ++ ++ if (dev_priv->l3_parity.remap_info[slice]) ++ memcpy(buf, ++ dev_priv->l3_parity.remap_info[slice] + (offset/4), ++ count); ++ else ++ memset(buf, 0, count); ++ ++ mutex_unlock(&dev->struct_mutex); ++ ++ return count; ++} ++ ++static ssize_t ++i915_l3_write(struct file *filp, struct kobject *kobj, ++ struct bin_attribute *attr, char *buf, ++ loff_t offset, size_t count) ++{ ++ struct device *kdev = kobj_to_dev(kobj); ++ struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); ++ struct drm_device *dev = &dev_priv->drm; ++ struct i915_gem_context *ctx; ++ int slice = (int)(uintptr_t)attr->private; ++ u32 **remap_info; ++ int ret; ++ ++ ret = l3_access_valid(dev_priv, offset); ++ if (ret) ++ return ret; ++ ++ ret = i915_mutex_lock_interruptible(dev); ++ if (ret) ++ return ret; ++ ++ remap_info = &dev_priv->l3_parity.remap_info[slice]; ++ if (!*remap_info) { ++ *remap_info = kzalloc(GEN7_L3LOG_SIZE, GFP_KERNEL); ++ if (!*remap_info) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ } ++ ++ /* TODO: Ideally we really want a GPU reset here to make sure errors ++ * aren't propagated. Since I cannot find a stable way to reset the GPU ++ * at this point it is left as a TODO. ++ */ ++ memcpy(*remap_info + (offset/4), buf, count); ++ ++ /* NB: We defer the remapping until we switch to the context */ ++ list_for_each_entry(ctx, &dev_priv->contexts.list, link) ++ ctx->remap_slice |= (1<struct_mutex); ++ ++ return ret; ++} ++ ++static const struct bin_attribute dpf_attrs = { ++ .attr = {.name = "l3_parity", .mode = (S_IRUSR | S_IWUSR)}, ++ .size = GEN7_L3LOG_SIZE, ++ .read = i915_l3_read, ++ .write = i915_l3_write, ++ .mmap = NULL, ++ .private = (void *)0 ++}; ++ ++static const struct bin_attribute dpf_attrs_1 = { ++ .attr = {.name = "l3_parity_slice_1", .mode = (S_IRUSR | S_IWUSR)}, ++ .size = GEN7_L3LOG_SIZE, ++ .read = i915_l3_read, ++ .write = i915_l3_write, ++ .mmap = NULL, ++ .private = (void *)1 ++}; ++ ++static ssize_t gt_act_freq_mhz_show(struct device *kdev, ++ struct device_attribute *attr, char *buf) ++{ ++ struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); ++ intel_wakeref_t wakeref; ++ int ret; ++ ++ wakeref = intel_runtime_pm_get(dev_priv); ++ ++ mutex_lock(&dev_priv->pcu_lock); ++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { ++ u32 freq; ++ freq = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); ++ ret = intel_gpu_freq(dev_priv, (freq >> 8) & 0xff); ++ } else { ++ ret = intel_gpu_freq(dev_priv, ++ intel_get_cagf(dev_priv, ++ I915_READ(GEN6_RPSTAT1))); ++ } ++ mutex_unlock(&dev_priv->pcu_lock); ++ ++ intel_runtime_pm_put(dev_priv, wakeref); ++ ++ return snprintf(buf, PAGE_SIZE, "%d\n", ret); ++} ++ ++static ssize_t gt_cur_freq_mhz_show(struct device *kdev, ++ struct device_attribute *attr, char *buf) ++{ ++ struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); ++ ++ return snprintf(buf, PAGE_SIZE, "%d\n", ++ intel_gpu_freq(dev_priv, ++ dev_priv->gt_pm.rps.cur_freq)); ++} ++ ++static ssize_t gt_boost_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf) ++{ ++ struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); ++ ++ return snprintf(buf, PAGE_SIZE, "%d\n", ++ intel_gpu_freq(dev_priv, ++ dev_priv->gt_pm.rps.boost_freq)); ++} ++ ++static ssize_t gt_boost_freq_mhz_store(struct device *kdev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); ++ struct intel_rps *rps = &dev_priv->gt_pm.rps; ++ bool boost = false; ++ ssize_t ret; ++ u32 val; ++ ++ ret = kstrtou32(buf, 0, &val); ++ if (ret) ++ return ret; ++ ++ /* Validate against (static) hardware limits */ ++ val = intel_freq_opcode(dev_priv, val); ++ if (val < rps->min_freq || val > rps->max_freq) ++ return -EINVAL; ++ ++ mutex_lock(&dev_priv->pcu_lock); ++ if (val != rps->boost_freq) { ++ rps->boost_freq = val; ++ boost = atomic_read(&rps->num_waiters); ++ } ++ mutex_unlock(&dev_priv->pcu_lock); ++ if (boost) ++ schedule_work(&rps->work); ++ ++ return count; ++} ++ ++static ssize_t vlv_rpe_freq_mhz_show(struct device *kdev, ++ struct device_attribute *attr, char *buf) ++{ ++ struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); ++ ++ return snprintf(buf, PAGE_SIZE, "%d\n", ++ intel_gpu_freq(dev_priv, ++ dev_priv->gt_pm.rps.efficient_freq)); ++} ++ ++static ssize_t gt_max_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf) ++{ ++ struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); ++ ++ return snprintf(buf, PAGE_SIZE, "%d\n", ++ intel_gpu_freq(dev_priv, ++ dev_priv->gt_pm.rps.max_freq_softlimit)); ++} ++ ++static ssize_t gt_max_freq_mhz_store(struct device *kdev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); ++ struct intel_rps *rps = &dev_priv->gt_pm.rps; ++ intel_wakeref_t wakeref; ++ u32 val; ++ ssize_t ret; ++ ++ ret = kstrtou32(buf, 0, &val); ++ if (ret) ++ return ret; ++ ++ wakeref = intel_runtime_pm_get(dev_priv); ++ ++ mutex_lock(&dev_priv->pcu_lock); ++ ++ val = intel_freq_opcode(dev_priv, val); ++ ++ if (val < rps->min_freq || ++ val > rps->max_freq || ++ val < rps->min_freq_softlimit) { ++ mutex_unlock(&dev_priv->pcu_lock); ++ intel_runtime_pm_put(dev_priv, wakeref); ++ return -EINVAL; ++ } ++ ++ if (val > rps->rp0_freq) ++ DRM_DEBUG("User requested overclocking to %d\n", ++ intel_gpu_freq(dev_priv, val)); ++ ++ rps->max_freq_softlimit = val; ++ ++ val = clamp_t(int, rps->cur_freq, ++ rps->min_freq_softlimit, ++ rps->max_freq_softlimit); ++ ++ /* We still need *_set_rps to process the new max_delay and ++ * update the interrupt limits and PMINTRMSK even though ++ * frequency request may be unchanged. */ ++ ret = intel_set_rps(dev_priv, val); ++ ++ mutex_unlock(&dev_priv->pcu_lock); ++ ++ intel_runtime_pm_put(dev_priv, wakeref); ++ ++ return ret ?: count; ++} ++ ++static ssize_t gt_min_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf) ++{ ++ struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); ++ ++ return snprintf(buf, PAGE_SIZE, "%d\n", ++ intel_gpu_freq(dev_priv, ++ dev_priv->gt_pm.rps.min_freq_softlimit)); ++} ++ ++static ssize_t gt_min_freq_mhz_store(struct device *kdev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); ++ struct intel_rps *rps = &dev_priv->gt_pm.rps; ++ intel_wakeref_t wakeref; ++ u32 val; ++ ssize_t ret; ++ ++ ret = kstrtou32(buf, 0, &val); ++ if (ret) ++ return ret; ++ ++ wakeref = intel_runtime_pm_get(dev_priv); ++ ++ mutex_lock(&dev_priv->pcu_lock); ++ ++ val = intel_freq_opcode(dev_priv, val); ++ ++ if (val < rps->min_freq || ++ val > rps->max_freq || ++ val > rps->max_freq_softlimit) { ++ mutex_unlock(&dev_priv->pcu_lock); ++ intel_runtime_pm_put(dev_priv, wakeref); ++ return -EINVAL; ++ } ++ ++ rps->min_freq_softlimit = val; ++ ++ val = clamp_t(int, rps->cur_freq, ++ rps->min_freq_softlimit, ++ rps->max_freq_softlimit); ++ ++ /* We still need *_set_rps to process the new min_delay and ++ * update the interrupt limits and PMINTRMSK even though ++ * frequency request may be unchanged. */ ++ ret = intel_set_rps(dev_priv, val); ++ ++ mutex_unlock(&dev_priv->pcu_lock); ++ ++ intel_runtime_pm_put(dev_priv, wakeref); ++ ++ return ret ?: count; ++} ++ ++static DEVICE_ATTR_RO(gt_act_freq_mhz); ++static DEVICE_ATTR_RO(gt_cur_freq_mhz); ++static DEVICE_ATTR_RW(gt_boost_freq_mhz); ++static DEVICE_ATTR_RW(gt_max_freq_mhz); ++static DEVICE_ATTR_RW(gt_min_freq_mhz); ++ ++static DEVICE_ATTR_RO(vlv_rpe_freq_mhz); ++ ++static ssize_t gt_rp_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf); ++static DEVICE_ATTR(gt_RP0_freq_mhz, S_IRUGO, gt_rp_mhz_show, NULL); ++static DEVICE_ATTR(gt_RP1_freq_mhz, S_IRUGO, gt_rp_mhz_show, NULL); ++static DEVICE_ATTR(gt_RPn_freq_mhz, S_IRUGO, gt_rp_mhz_show, NULL); ++ ++/* For now we have a static number of RP states */ ++static ssize_t gt_rp_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf) ++{ ++ struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); ++ struct intel_rps *rps = &dev_priv->gt_pm.rps; ++ u32 val; ++ ++ if (attr == &dev_attr_gt_RP0_freq_mhz) ++ val = intel_gpu_freq(dev_priv, rps->rp0_freq); ++ else if (attr == &dev_attr_gt_RP1_freq_mhz) ++ val = intel_gpu_freq(dev_priv, rps->rp1_freq); ++ else if (attr == &dev_attr_gt_RPn_freq_mhz) ++ val = intel_gpu_freq(dev_priv, rps->min_freq); ++ else ++ BUG(); ++ ++ return snprintf(buf, PAGE_SIZE, "%d\n", val); ++} ++ ++static const struct attribute * const gen6_attrs[] = { ++ &dev_attr_gt_act_freq_mhz.attr, ++ &dev_attr_gt_cur_freq_mhz.attr, ++ &dev_attr_gt_boost_freq_mhz.attr, ++ &dev_attr_gt_max_freq_mhz.attr, ++ &dev_attr_gt_min_freq_mhz.attr, ++ &dev_attr_gt_RP0_freq_mhz.attr, ++ &dev_attr_gt_RP1_freq_mhz.attr, ++ &dev_attr_gt_RPn_freq_mhz.attr, ++ NULL, ++}; ++ ++static const struct attribute * const vlv_attrs[] = { ++ &dev_attr_gt_act_freq_mhz.attr, ++ &dev_attr_gt_cur_freq_mhz.attr, ++ &dev_attr_gt_boost_freq_mhz.attr, ++ &dev_attr_gt_max_freq_mhz.attr, ++ &dev_attr_gt_min_freq_mhz.attr, ++ &dev_attr_gt_RP0_freq_mhz.attr, ++ &dev_attr_gt_RP1_freq_mhz.attr, ++ &dev_attr_gt_RPn_freq_mhz.attr, ++ &dev_attr_vlv_rpe_freq_mhz.attr, ++ NULL, ++}; ++ ++#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) ++ ++static ssize_t error_state_read(struct file *filp, struct kobject *kobj, ++ struct bin_attribute *attr, char *buf, ++ loff_t off, size_t count) ++{ ++ ++ struct device *kdev = kobj_to_dev(kobj); ++ struct drm_i915_private *i915 = kdev_minor_to_i915(kdev); ++ struct i915_gpu_state *gpu; ++ ssize_t ret; ++ ++ gpu = i915_first_error_state(i915); ++ if (IS_ERR(gpu)) { ++ ret = PTR_ERR(gpu); ++ } else if (gpu) { ++ ret = i915_gpu_state_copy_to_buffer(gpu, buf, off, count); ++ i915_gpu_state_put(gpu); ++ } else { ++ const char *str = "No error state collected\n"; ++ size_t len = strlen(str); ++ ++ ret = min_t(size_t, count, len - off); ++ memcpy(buf, str + off, ret); ++ } ++ ++ return ret; ++} ++ ++static ssize_t error_state_write(struct file *file, struct kobject *kobj, ++ struct bin_attribute *attr, char *buf, ++ loff_t off, size_t count) ++{ ++ struct device *kdev = kobj_to_dev(kobj); ++ struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); ++ ++ DRM_DEBUG_DRIVER("Resetting error state\n"); ++ i915_reset_error_state(dev_priv); ++ ++ return count; ++} ++ ++static const struct bin_attribute error_state_attr = { ++ .attr.name = "error", ++ .attr.mode = S_IRUSR | S_IWUSR, ++ .size = 0, ++ .read = error_state_read, ++ .write = error_state_write, ++}; ++ ++static void i915_setup_error_capture(struct device *kdev) ++{ ++ if (sysfs_create_bin_file(&kdev->kobj, &error_state_attr)) ++ DRM_ERROR("error_state sysfs setup failed\n"); ++} ++ ++static void i915_teardown_error_capture(struct device *kdev) ++{ ++ sysfs_remove_bin_file(&kdev->kobj, &error_state_attr); ++} ++#else ++static void i915_setup_error_capture(struct device *kdev) {} ++static void i915_teardown_error_capture(struct device *kdev) {} ++#endif ++ ++void i915_setup_sysfs(struct drm_i915_private *dev_priv) ++{ ++ struct device *kdev = dev_priv->drm.primary->kdev; ++ int ret; ++ ++#ifdef CONFIG_PM ++ if (HAS_RC6(dev_priv)) { ++ ret = sysfs_merge_group(&kdev->kobj, ++ &rc6_attr_group); ++ if (ret) ++ DRM_ERROR("RC6 residency sysfs setup failed\n"); ++ } ++ if (HAS_RC6p(dev_priv)) { ++ ret = sysfs_merge_group(&kdev->kobj, ++ &rc6p_attr_group); ++ if (ret) ++ DRM_ERROR("RC6p residency sysfs setup failed\n"); ++ } ++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { ++ ret = sysfs_merge_group(&kdev->kobj, ++ &media_rc6_attr_group); ++ if (ret) ++ DRM_ERROR("Media RC6 residency sysfs setup failed\n"); ++ } ++#endif ++ if (HAS_L3_DPF(dev_priv)) { ++ ret = device_create_bin_file(kdev, &dpf_attrs); ++ if (ret) ++ DRM_ERROR("l3 parity sysfs setup failed\n"); ++ ++ if (NUM_L3_SLICES(dev_priv) > 1) { ++ ret = device_create_bin_file(kdev, ++ &dpf_attrs_1); ++ if (ret) ++ DRM_ERROR("l3 parity slice 1 setup failed\n"); ++ } ++ } ++ ++ ret = 0; ++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ ret = sysfs_create_files(&kdev->kobj, vlv_attrs); ++ else if (INTEL_GEN(dev_priv) >= 6) ++ ret = sysfs_create_files(&kdev->kobj, gen6_attrs); ++ if (ret) ++ DRM_ERROR("RPS sysfs setup failed\n"); ++ ++ i915_setup_error_capture(kdev); ++} ++ ++void i915_teardown_sysfs(struct drm_i915_private *dev_priv) ++{ ++ struct device *kdev = dev_priv->drm.primary->kdev; ++ ++ i915_teardown_error_capture(kdev); ++ ++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ sysfs_remove_files(&kdev->kobj, vlv_attrs); ++ else ++ sysfs_remove_files(&kdev->kobj, gen6_attrs); ++ device_remove_bin_file(kdev, &dpf_attrs_1); ++ device_remove_bin_file(kdev, &dpf_attrs); ++#ifdef CONFIG_PM ++ sysfs_unmerge_group(&kdev->kobj, &rc6_attr_group); ++ sysfs_unmerge_group(&kdev->kobj, &rc6p_attr_group); ++#endif ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_timeline.c b/drivers/gpu/drm/i915_legacy/i915_timeline.c +new file mode 100644 +index 000000000000..5fbea0892f33 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_timeline.c +@@ -0,0 +1,579 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2016-2018 Intel Corporation ++ */ ++ ++#include "i915_drv.h" ++ ++#include "i915_active.h" ++#include "i915_syncmap.h" ++#include "i915_timeline.h" ++ ++#define ptr_set_bit(ptr, bit) ((typeof(ptr))((unsigned long)(ptr) | BIT(bit))) ++#define ptr_test_bit(ptr, bit) ((unsigned long)(ptr) & BIT(bit)) ++ ++struct i915_timeline_hwsp { ++ struct i915_gt_timelines *gt; ++ struct list_head free_link; ++ struct i915_vma *vma; ++ u64 free_bitmap; ++}; ++ ++struct i915_timeline_cacheline { ++ struct i915_active active; ++ struct i915_timeline_hwsp *hwsp; ++ void *vaddr; ++#define CACHELINE_BITS 6 ++#define CACHELINE_FREE CACHELINE_BITS ++}; ++ ++static inline struct drm_i915_private * ++hwsp_to_i915(struct i915_timeline_hwsp *hwsp) ++{ ++ return container_of(hwsp->gt, struct drm_i915_private, gt.timelines); ++} ++ ++static struct i915_vma *__hwsp_alloc(struct drm_i915_private *i915) ++{ ++ struct drm_i915_gem_object *obj; ++ struct i915_vma *vma; ++ ++ obj = i915_gem_object_create_internal(i915, PAGE_SIZE); ++ if (IS_ERR(obj)) ++ return ERR_CAST(obj); ++ ++ i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC); ++ ++ vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); ++ if (IS_ERR(vma)) ++ i915_gem_object_put(obj); ++ ++ return vma; ++} ++ ++static struct i915_vma * ++hwsp_alloc(struct i915_timeline *timeline, unsigned int *cacheline) ++{ ++ struct drm_i915_private *i915 = timeline->i915; ++ struct i915_gt_timelines *gt = &i915->gt.timelines; ++ struct i915_timeline_hwsp *hwsp; ++ ++ BUILD_BUG_ON(BITS_PER_TYPE(u64) * CACHELINE_BYTES > PAGE_SIZE); ++ ++ spin_lock(>->hwsp_lock); ++ ++ /* hwsp_free_list only contains HWSP that have available cachelines */ ++ hwsp = list_first_entry_or_null(>->hwsp_free_list, ++ typeof(*hwsp), free_link); ++ if (!hwsp) { ++ struct i915_vma *vma; ++ ++ spin_unlock(>->hwsp_lock); ++ ++ hwsp = kmalloc(sizeof(*hwsp), GFP_KERNEL); ++ if (!hwsp) ++ return ERR_PTR(-ENOMEM); ++ ++ vma = __hwsp_alloc(i915); ++ if (IS_ERR(vma)) { ++ kfree(hwsp); ++ return vma; ++ } ++ ++ vma->private = hwsp; ++ hwsp->vma = vma; ++ hwsp->free_bitmap = ~0ull; ++ hwsp->gt = gt; ++ ++ spin_lock(>->hwsp_lock); ++ list_add(&hwsp->free_link, >->hwsp_free_list); ++ } ++ ++ GEM_BUG_ON(!hwsp->free_bitmap); ++ *cacheline = __ffs64(hwsp->free_bitmap); ++ hwsp->free_bitmap &= ~BIT_ULL(*cacheline); ++ if (!hwsp->free_bitmap) ++ list_del(&hwsp->free_link); ++ ++ spin_unlock(>->hwsp_lock); ++ ++ GEM_BUG_ON(hwsp->vma->private != hwsp); ++ return hwsp->vma; ++} ++ ++static void __idle_hwsp_free(struct i915_timeline_hwsp *hwsp, int cacheline) ++{ ++ struct i915_gt_timelines *gt = hwsp->gt; ++ ++ spin_lock(>->hwsp_lock); ++ ++ /* As a cacheline becomes available, publish the HWSP on the freelist */ ++ if (!hwsp->free_bitmap) ++ list_add_tail(&hwsp->free_link, >->hwsp_free_list); ++ ++ GEM_BUG_ON(cacheline >= BITS_PER_TYPE(hwsp->free_bitmap)); ++ hwsp->free_bitmap |= BIT_ULL(cacheline); ++ ++ /* And if no one is left using it, give the page back to the system */ ++ if (hwsp->free_bitmap == ~0ull) { ++ i915_vma_put(hwsp->vma); ++ list_del(&hwsp->free_link); ++ kfree(hwsp); ++ } ++ ++ spin_unlock(>->hwsp_lock); ++} ++ ++static void __idle_cacheline_free(struct i915_timeline_cacheline *cl) ++{ ++ GEM_BUG_ON(!i915_active_is_idle(&cl->active)); ++ ++ i915_gem_object_unpin_map(cl->hwsp->vma->obj); ++ i915_vma_put(cl->hwsp->vma); ++ __idle_hwsp_free(cl->hwsp, ptr_unmask_bits(cl->vaddr, CACHELINE_BITS)); ++ ++ i915_active_fini(&cl->active); ++ kfree(cl); ++} ++ ++static void __cacheline_retire(struct i915_active *active) ++{ ++ struct i915_timeline_cacheline *cl = ++ container_of(active, typeof(*cl), active); ++ ++ i915_vma_unpin(cl->hwsp->vma); ++ if (ptr_test_bit(cl->vaddr, CACHELINE_FREE)) ++ __idle_cacheline_free(cl); ++} ++ ++static struct i915_timeline_cacheline * ++cacheline_alloc(struct i915_timeline_hwsp *hwsp, unsigned int cacheline) ++{ ++ struct i915_timeline_cacheline *cl; ++ void *vaddr; ++ ++ GEM_BUG_ON(cacheline >= BIT(CACHELINE_BITS)); ++ ++ cl = kmalloc(sizeof(*cl), GFP_KERNEL); ++ if (!cl) ++ return ERR_PTR(-ENOMEM); ++ ++ vaddr = i915_gem_object_pin_map(hwsp->vma->obj, I915_MAP_WB); ++ if (IS_ERR(vaddr)) { ++ kfree(cl); ++ return ERR_CAST(vaddr); ++ } ++ ++ i915_vma_get(hwsp->vma); ++ cl->hwsp = hwsp; ++ cl->vaddr = page_pack_bits(vaddr, cacheline); ++ ++ i915_active_init(hwsp_to_i915(hwsp), &cl->active, __cacheline_retire); ++ ++ return cl; ++} ++ ++static void cacheline_acquire(struct i915_timeline_cacheline *cl) ++{ ++ if (cl && i915_active_acquire(&cl->active)) ++ __i915_vma_pin(cl->hwsp->vma); ++} ++ ++static void cacheline_release(struct i915_timeline_cacheline *cl) ++{ ++ if (cl) ++ i915_active_release(&cl->active); ++} ++ ++static void cacheline_free(struct i915_timeline_cacheline *cl) ++{ ++ GEM_BUG_ON(ptr_test_bit(cl->vaddr, CACHELINE_FREE)); ++ cl->vaddr = ptr_set_bit(cl->vaddr, CACHELINE_FREE); ++ ++ if (i915_active_is_idle(&cl->active)) ++ __idle_cacheline_free(cl); ++} ++ ++int i915_timeline_init(struct drm_i915_private *i915, ++ struct i915_timeline *timeline, ++ struct i915_vma *hwsp) ++{ ++ void *vaddr; ++ ++ /* ++ * Ideally we want a set of engines on a single leaf as we expect ++ * to mostly be tracking synchronisation between engines. It is not ++ * a huge issue if this is not the case, but we may want to mitigate ++ * any page crossing penalties if they become an issue. ++ * ++ * Called during early_init before we know how many engines there are. ++ */ ++ BUILD_BUG_ON(KSYNCMAP < I915_NUM_ENGINES); ++ ++ timeline->i915 = i915; ++ timeline->pin_count = 0; ++ timeline->has_initial_breadcrumb = !hwsp; ++ timeline->hwsp_cacheline = NULL; ++ ++ if (!hwsp) { ++ struct i915_timeline_cacheline *cl; ++ unsigned int cacheline; ++ ++ hwsp = hwsp_alloc(timeline, &cacheline); ++ if (IS_ERR(hwsp)) ++ return PTR_ERR(hwsp); ++ ++ cl = cacheline_alloc(hwsp->private, cacheline); ++ if (IS_ERR(cl)) { ++ __idle_hwsp_free(hwsp->private, cacheline); ++ return PTR_ERR(cl); ++ } ++ ++ timeline->hwsp_cacheline = cl; ++ timeline->hwsp_offset = cacheline * CACHELINE_BYTES; ++ ++ vaddr = page_mask_bits(cl->vaddr); ++ } else { ++ timeline->hwsp_offset = I915_GEM_HWS_SEQNO_ADDR; ++ ++ vaddr = i915_gem_object_pin_map(hwsp->obj, I915_MAP_WB); ++ if (IS_ERR(vaddr)) ++ return PTR_ERR(vaddr); ++ } ++ ++ timeline->hwsp_seqno = ++ memset(vaddr + timeline->hwsp_offset, 0, CACHELINE_BYTES); ++ ++ timeline->hwsp_ggtt = i915_vma_get(hwsp); ++ GEM_BUG_ON(timeline->hwsp_offset >= hwsp->size); ++ ++ timeline->fence_context = dma_fence_context_alloc(1); ++ ++ spin_lock_init(&timeline->lock); ++ mutex_init(&timeline->mutex); ++ ++ INIT_ACTIVE_REQUEST(&timeline->last_request); ++ INIT_LIST_HEAD(&timeline->requests); ++ ++ i915_syncmap_init(&timeline->sync); ++ ++ return 0; ++} ++ ++void i915_timelines_init(struct drm_i915_private *i915) ++{ ++ struct i915_gt_timelines *gt = &i915->gt.timelines; ++ ++ mutex_init(>->mutex); ++ INIT_LIST_HEAD(>->active_list); ++ ++ spin_lock_init(>->hwsp_lock); ++ INIT_LIST_HEAD(>->hwsp_free_list); ++ ++ /* via i915_gem_wait_for_idle() */ ++ i915_gem_shrinker_taints_mutex(i915, >->mutex); ++} ++ ++static void timeline_add_to_active(struct i915_timeline *tl) ++{ ++ struct i915_gt_timelines *gt = &tl->i915->gt.timelines; ++ ++ mutex_lock(>->mutex); ++ list_add(&tl->link, >->active_list); ++ mutex_unlock(>->mutex); ++} ++ ++static void timeline_remove_from_active(struct i915_timeline *tl) ++{ ++ struct i915_gt_timelines *gt = &tl->i915->gt.timelines; ++ ++ mutex_lock(>->mutex); ++ list_del(&tl->link); ++ mutex_unlock(>->mutex); ++} ++ ++/** ++ * i915_timelines_park - called when the driver idles ++ * @i915: the drm_i915_private device ++ * ++ * When the driver is completely idle, we know that all of our sync points ++ * have been signaled and our tracking is then entirely redundant. Any request ++ * to wait upon an older sync point will be completed instantly as we know ++ * the fence is signaled and therefore we will not even look them up in the ++ * sync point map. ++ */ ++void i915_timelines_park(struct drm_i915_private *i915) ++{ ++ struct i915_gt_timelines *gt = &i915->gt.timelines; ++ struct i915_timeline *timeline; ++ ++ mutex_lock(>->mutex); ++ list_for_each_entry(timeline, >->active_list, link) { ++ /* ++ * All known fences are completed so we can scrap ++ * the current sync point tracking and start afresh, ++ * any attempt to wait upon a previous sync point ++ * will be skipped as the fence was signaled. ++ */ ++ i915_syncmap_free(&timeline->sync); ++ } ++ mutex_unlock(>->mutex); ++} ++ ++void i915_timeline_fini(struct i915_timeline *timeline) ++{ ++ GEM_BUG_ON(timeline->pin_count); ++ GEM_BUG_ON(!list_empty(&timeline->requests)); ++ ++ i915_syncmap_free(&timeline->sync); ++ ++ if (timeline->hwsp_cacheline) ++ cacheline_free(timeline->hwsp_cacheline); ++ else ++ i915_gem_object_unpin_map(timeline->hwsp_ggtt->obj); ++ ++ i915_vma_put(timeline->hwsp_ggtt); ++} ++ ++struct i915_timeline * ++i915_timeline_create(struct drm_i915_private *i915, ++ struct i915_vma *global_hwsp) ++{ ++ struct i915_timeline *timeline; ++ int err; ++ ++ timeline = kzalloc(sizeof(*timeline), GFP_KERNEL); ++ if (!timeline) ++ return ERR_PTR(-ENOMEM); ++ ++ err = i915_timeline_init(i915, timeline, global_hwsp); ++ if (err) { ++ kfree(timeline); ++ return ERR_PTR(err); ++ } ++ ++ kref_init(&timeline->kref); ++ ++ return timeline; ++} ++ ++int i915_timeline_pin(struct i915_timeline *tl) ++{ ++ int err; ++ ++ if (tl->pin_count++) ++ return 0; ++ GEM_BUG_ON(!tl->pin_count); ++ ++ err = i915_vma_pin(tl->hwsp_ggtt, 0, 0, PIN_GLOBAL | PIN_HIGH); ++ if (err) ++ goto unpin; ++ ++ tl->hwsp_offset = ++ i915_ggtt_offset(tl->hwsp_ggtt) + ++ offset_in_page(tl->hwsp_offset); ++ ++ cacheline_acquire(tl->hwsp_cacheline); ++ timeline_add_to_active(tl); ++ ++ return 0; ++ ++unpin: ++ tl->pin_count = 0; ++ return err; ++} ++ ++static u32 timeline_advance(struct i915_timeline *tl) ++{ ++ GEM_BUG_ON(!tl->pin_count); ++ GEM_BUG_ON(tl->seqno & tl->has_initial_breadcrumb); ++ ++ return tl->seqno += 1 + tl->has_initial_breadcrumb; ++} ++ ++static void timeline_rollback(struct i915_timeline *tl) ++{ ++ tl->seqno -= 1 + tl->has_initial_breadcrumb; ++} ++ ++static noinline int ++__i915_timeline_get_seqno(struct i915_timeline *tl, ++ struct i915_request *rq, ++ u32 *seqno) ++{ ++ struct i915_timeline_cacheline *cl; ++ unsigned int cacheline; ++ struct i915_vma *vma; ++ void *vaddr; ++ int err; ++ ++ /* ++ * If there is an outstanding GPU reference to this cacheline, ++ * such as it being sampled by a HW semaphore on another timeline, ++ * we cannot wraparound our seqno value (the HW semaphore does ++ * a strict greater-than-or-equals compare, not i915_seqno_passed). ++ * So if the cacheline is still busy, we must detach ourselves ++ * from it and leave it inflight alongside its users. ++ * ++ * However, if nobody is watching and we can guarantee that nobody ++ * will, we could simply reuse the same cacheline. ++ * ++ * if (i915_active_request_is_signaled(&tl->last_request) && ++ * i915_active_is_signaled(&tl->hwsp_cacheline->active)) ++ * return 0; ++ * ++ * That seems unlikely for a busy timeline that needed to wrap in ++ * the first place, so just replace the cacheline. ++ */ ++ ++ vma = hwsp_alloc(tl, &cacheline); ++ if (IS_ERR(vma)) { ++ err = PTR_ERR(vma); ++ goto err_rollback; ++ } ++ ++ err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL | PIN_HIGH); ++ if (err) { ++ __idle_hwsp_free(vma->private, cacheline); ++ goto err_rollback; ++ } ++ ++ cl = cacheline_alloc(vma->private, cacheline); ++ if (IS_ERR(cl)) { ++ err = PTR_ERR(cl); ++ __idle_hwsp_free(vma->private, cacheline); ++ goto err_unpin; ++ } ++ GEM_BUG_ON(cl->hwsp->vma != vma); ++ ++ /* ++ * Attach the old cacheline to the current request, so that we only ++ * free it after the current request is retired, which ensures that ++ * all writes into the cacheline from previous requests are complete. ++ */ ++ err = i915_active_ref(&tl->hwsp_cacheline->active, ++ tl->fence_context, rq); ++ if (err) ++ goto err_cacheline; ++ ++ cacheline_release(tl->hwsp_cacheline); /* ownership now xfered to rq */ ++ cacheline_free(tl->hwsp_cacheline); ++ ++ i915_vma_unpin(tl->hwsp_ggtt); /* binding kept alive by old cacheline */ ++ i915_vma_put(tl->hwsp_ggtt); ++ ++ tl->hwsp_ggtt = i915_vma_get(vma); ++ ++ vaddr = page_mask_bits(cl->vaddr); ++ tl->hwsp_offset = cacheline * CACHELINE_BYTES; ++ tl->hwsp_seqno = ++ memset(vaddr + tl->hwsp_offset, 0, CACHELINE_BYTES); ++ ++ tl->hwsp_offset += i915_ggtt_offset(vma); ++ ++ cacheline_acquire(cl); ++ tl->hwsp_cacheline = cl; ++ ++ *seqno = timeline_advance(tl); ++ GEM_BUG_ON(i915_seqno_passed(*tl->hwsp_seqno, *seqno)); ++ return 0; ++ ++err_cacheline: ++ cacheline_free(cl); ++err_unpin: ++ i915_vma_unpin(vma); ++err_rollback: ++ timeline_rollback(tl); ++ return err; ++} ++ ++int i915_timeline_get_seqno(struct i915_timeline *tl, ++ struct i915_request *rq, ++ u32 *seqno) ++{ ++ *seqno = timeline_advance(tl); ++ ++ /* Replace the HWSP on wraparound for HW semaphores */ ++ if (unlikely(!*seqno && tl->hwsp_cacheline)) ++ return __i915_timeline_get_seqno(tl, rq, seqno); ++ ++ return 0; ++} ++ ++static int cacheline_ref(struct i915_timeline_cacheline *cl, ++ struct i915_request *rq) ++{ ++ return i915_active_ref(&cl->active, rq->fence.context, rq); ++} ++ ++int i915_timeline_read_hwsp(struct i915_request *from, ++ struct i915_request *to, ++ u32 *hwsp) ++{ ++ struct i915_timeline_cacheline *cl = from->hwsp_cacheline; ++ struct i915_timeline *tl = from->timeline; ++ int err; ++ ++ GEM_BUG_ON(to->timeline == tl); ++ ++ mutex_lock_nested(&tl->mutex, SINGLE_DEPTH_NESTING); ++ err = i915_request_completed(from); ++ if (!err) ++ err = cacheline_ref(cl, to); ++ if (!err) { ++ if (likely(cl == tl->hwsp_cacheline)) { ++ *hwsp = tl->hwsp_offset; ++ } else { /* across a seqno wrap, recover the original offset */ ++ *hwsp = i915_ggtt_offset(cl->hwsp->vma) + ++ ptr_unmask_bits(cl->vaddr, CACHELINE_BITS) * ++ CACHELINE_BYTES; ++ } ++ } ++ mutex_unlock(&tl->mutex); ++ ++ return err; ++} ++ ++void i915_timeline_unpin(struct i915_timeline *tl) ++{ ++ GEM_BUG_ON(!tl->pin_count); ++ if (--tl->pin_count) ++ return; ++ ++ timeline_remove_from_active(tl); ++ cacheline_release(tl->hwsp_cacheline); ++ ++ /* ++ * Since this timeline is idle, all bariers upon which we were waiting ++ * must also be complete and so we can discard the last used barriers ++ * without loss of information. ++ */ ++ i915_syncmap_free(&tl->sync); ++ ++ __i915_vma_unpin(tl->hwsp_ggtt); ++} ++ ++void __i915_timeline_free(struct kref *kref) ++{ ++ struct i915_timeline *timeline = ++ container_of(kref, typeof(*timeline), kref); ++ ++ i915_timeline_fini(timeline); ++ kfree(timeline); ++} ++ ++void i915_timelines_fini(struct drm_i915_private *i915) ++{ ++ struct i915_gt_timelines *gt = &i915->gt.timelines; ++ ++ GEM_BUG_ON(!list_empty(>->active_list)); ++ GEM_BUG_ON(!list_empty(>->hwsp_free_list)); ++ ++ mutex_destroy(>->mutex); ++} ++ ++#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) ++#include "selftests/mock_timeline.c" ++#include "selftests/i915_timeline.c" ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_timeline.h b/drivers/gpu/drm/i915_legacy/i915_timeline.h +new file mode 100644 +index 000000000000..27668a1a69a3 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_timeline.h +@@ -0,0 +1,113 @@ ++/* ++ * Copyright © 2016 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#ifndef I915_TIMELINE_H ++#define I915_TIMELINE_H ++ ++#include ++ ++#include "i915_active.h" ++#include "i915_syncmap.h" ++#include "i915_timeline_types.h" ++ ++int i915_timeline_init(struct drm_i915_private *i915, ++ struct i915_timeline *tl, ++ struct i915_vma *hwsp); ++void i915_timeline_fini(struct i915_timeline *tl); ++ ++static inline void ++i915_timeline_set_subclass(struct i915_timeline *timeline, ++ unsigned int subclass) ++{ ++ lockdep_set_subclass(&timeline->lock, subclass); ++ ++ /* ++ * Due to an interesting quirk in lockdep's internal debug tracking, ++ * after setting a subclass we must ensure the lock is used. Otherwise, ++ * nr_unused_locks is incremented once too often. ++ */ ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++ local_irq_disable(); ++ lock_map_acquire(&timeline->lock.dep_map); ++ lock_map_release(&timeline->lock.dep_map); ++ local_irq_enable(); ++#endif ++} ++ ++struct i915_timeline * ++i915_timeline_create(struct drm_i915_private *i915, ++ struct i915_vma *global_hwsp); ++ ++static inline struct i915_timeline * ++i915_timeline_get(struct i915_timeline *timeline) ++{ ++ kref_get(&timeline->kref); ++ return timeline; ++} ++ ++void __i915_timeline_free(struct kref *kref); ++static inline void i915_timeline_put(struct i915_timeline *timeline) ++{ ++ kref_put(&timeline->kref, __i915_timeline_free); ++} ++ ++static inline int __i915_timeline_sync_set(struct i915_timeline *tl, ++ u64 context, u32 seqno) ++{ ++ return i915_syncmap_set(&tl->sync, context, seqno); ++} ++ ++static inline int i915_timeline_sync_set(struct i915_timeline *tl, ++ const struct dma_fence *fence) ++{ ++ return __i915_timeline_sync_set(tl, fence->context, fence->seqno); ++} ++ ++static inline bool __i915_timeline_sync_is_later(struct i915_timeline *tl, ++ u64 context, u32 seqno) ++{ ++ return i915_syncmap_is_later(&tl->sync, context, seqno); ++} ++ ++static inline bool i915_timeline_sync_is_later(struct i915_timeline *tl, ++ const struct dma_fence *fence) ++{ ++ return __i915_timeline_sync_is_later(tl, fence->context, fence->seqno); ++} ++ ++int i915_timeline_pin(struct i915_timeline *tl); ++int i915_timeline_get_seqno(struct i915_timeline *tl, ++ struct i915_request *rq, ++ u32 *seqno); ++void i915_timeline_unpin(struct i915_timeline *tl); ++ ++int i915_timeline_read_hwsp(struct i915_request *from, ++ struct i915_request *until, ++ u32 *hwsp_offset); ++ ++void i915_timelines_init(struct drm_i915_private *i915); ++void i915_timelines_park(struct drm_i915_private *i915); ++void i915_timelines_fini(struct drm_i915_private *i915); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_timeline_types.h b/drivers/gpu/drm/i915_legacy/i915_timeline_types.h +new file mode 100644 +index 000000000000..5256a0b5c5f7 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_timeline_types.h +@@ -0,0 +1,70 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2016 Intel Corporation ++ */ ++ ++#ifndef __I915_TIMELINE_TYPES_H__ ++#define __I915_TIMELINE_TYPES_H__ ++ ++#include ++#include ++#include ++#include ++ ++#include "i915_active_types.h" ++ ++struct drm_i915_private; ++struct i915_vma; ++struct i915_timeline_cacheline; ++struct i915_syncmap; ++ ++struct i915_timeline { ++ u64 fence_context; ++ u32 seqno; ++ ++ spinlock_t lock; ++#define TIMELINE_CLIENT 0 /* default subclass */ ++#define TIMELINE_ENGINE 1 ++ struct mutex mutex; /* protects the flow of requests */ ++ ++ unsigned int pin_count; ++ const u32 *hwsp_seqno; ++ struct i915_vma *hwsp_ggtt; ++ u32 hwsp_offset; ++ ++ struct i915_timeline_cacheline *hwsp_cacheline; ++ ++ bool has_initial_breadcrumb; ++ ++ /** ++ * List of breadcrumbs associated with GPU requests currently ++ * outstanding. ++ */ ++ struct list_head requests; ++ ++ /* Contains an RCU guarded pointer to the last request. No reference is ++ * held to the request, users must carefully acquire a reference to ++ * the request using i915_active_request_get_request_rcu(), or hold the ++ * struct_mutex. ++ */ ++ struct i915_active_request last_request; ++ ++ /** ++ * We track the most recent seqno that we wait on in every context so ++ * that we only have to emit a new await and dependency on a more ++ * recent sync point. As the contexts may be executed out-of-order, we ++ * have to track each individually and can not rely on an absolute ++ * global_seqno. When we know that all tracked fences are completed ++ * (i.e. when the driver is idle), we know that the syncmap is ++ * redundant and we can discard it without loss of generality. ++ */ ++ struct i915_syncmap *sync; ++ ++ struct list_head link; ++ struct drm_i915_private *i915; ++ ++ struct kref kref; ++}; ++ ++#endif /* __I915_TIMELINE_TYPES_H__ */ +diff --git a/drivers/gpu/drm/i915_legacy/i915_trace.h b/drivers/gpu/drm/i915_legacy/i915_trace.h +new file mode 100644 +index 000000000000..ca379eaa3537 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_trace.h +@@ -0,0 +1,1000 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#if !defined(_I915_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ) ++#define _I915_TRACE_H_ ++ ++#include ++#include ++#include ++ ++#include ++ ++#include "i915_drv.h" ++#include "intel_drv.h" ++#include "intel_ringbuffer.h" ++ ++#undef TRACE_SYSTEM ++#define TRACE_SYSTEM i915 ++#define TRACE_INCLUDE_FILE i915_trace ++ ++/* watermark/fifo updates */ ++ ++TRACE_EVENT(intel_pipe_enable, ++ TP_PROTO(struct drm_i915_private *dev_priv, enum pipe pipe), ++ TP_ARGS(dev_priv, pipe), ++ ++ TP_STRUCT__entry( ++ __array(u32, frame, 3) ++ __array(u32, scanline, 3) ++ __field(enum pipe, pipe) ++ ), ++ ++ TP_fast_assign( ++ enum pipe _pipe; ++ for_each_pipe(dev_priv, _pipe) { ++ __entry->frame[_pipe] = ++ dev_priv->drm.driver->get_vblank_counter(&dev_priv->drm, _pipe); ++ __entry->scanline[_pipe] = ++ intel_get_crtc_scanline(intel_get_crtc_for_pipe(dev_priv, _pipe)); ++ } ++ __entry->pipe = pipe; ++ ), ++ ++ TP_printk("pipe %c enable, pipe A: frame=%u, scanline=%u, pipe B: frame=%u, scanline=%u, pipe C: frame=%u, scanline=%u", ++ pipe_name(__entry->pipe), ++ __entry->frame[PIPE_A], __entry->scanline[PIPE_A], ++ __entry->frame[PIPE_B], __entry->scanline[PIPE_B], ++ __entry->frame[PIPE_C], __entry->scanline[PIPE_C]) ++); ++ ++TRACE_EVENT(intel_pipe_disable, ++ TP_PROTO(struct drm_i915_private *dev_priv, enum pipe pipe), ++ TP_ARGS(dev_priv, pipe), ++ ++ TP_STRUCT__entry( ++ __array(u32, frame, 3) ++ __array(u32, scanline, 3) ++ __field(enum pipe, pipe) ++ ), ++ ++ TP_fast_assign( ++ enum pipe _pipe; ++ for_each_pipe(dev_priv, _pipe) { ++ __entry->frame[_pipe] = ++ dev_priv->drm.driver->get_vblank_counter(&dev_priv->drm, _pipe); ++ __entry->scanline[_pipe] = ++ intel_get_crtc_scanline(intel_get_crtc_for_pipe(dev_priv, _pipe)); ++ } ++ __entry->pipe = pipe; ++ ), ++ ++ TP_printk("pipe %c disable, pipe A: frame=%u, scanline=%u, pipe B: frame=%u, scanline=%u, pipe C: frame=%u, scanline=%u", ++ pipe_name(__entry->pipe), ++ __entry->frame[PIPE_A], __entry->scanline[PIPE_A], ++ __entry->frame[PIPE_B], __entry->scanline[PIPE_B], ++ __entry->frame[PIPE_C], __entry->scanline[PIPE_C]) ++); ++ ++TRACE_EVENT(intel_pipe_crc, ++ TP_PROTO(struct intel_crtc *crtc, const u32 *crcs), ++ TP_ARGS(crtc, crcs), ++ ++ TP_STRUCT__entry( ++ __field(enum pipe, pipe) ++ __field(u32, frame) ++ __field(u32, scanline) ++ __array(u32, crcs, 5) ++ ), ++ ++ TP_fast_assign( ++ __entry->pipe = crtc->pipe; ++ __entry->frame = crtc->base.dev->driver->get_vblank_counter(crtc->base.dev, ++ crtc->pipe); ++ __entry->scanline = intel_get_crtc_scanline(crtc); ++ memcpy(__entry->crcs, crcs, sizeof(__entry->crcs)); ++ ), ++ ++ TP_printk("pipe %c, frame=%u, scanline=%u crc=%08x %08x %08x %08x %08x", ++ pipe_name(__entry->pipe), __entry->frame, __entry->scanline, ++ __entry->crcs[0], __entry->crcs[1], __entry->crcs[2], ++ __entry->crcs[3], __entry->crcs[4]) ++); ++ ++TRACE_EVENT(intel_cpu_fifo_underrun, ++ TP_PROTO(struct drm_i915_private *dev_priv, enum pipe pipe), ++ TP_ARGS(dev_priv, pipe), ++ ++ TP_STRUCT__entry( ++ __field(enum pipe, pipe) ++ __field(u32, frame) ++ __field(u32, scanline) ++ ), ++ ++ TP_fast_assign( ++ __entry->pipe = pipe; ++ __entry->frame = dev_priv->drm.driver->get_vblank_counter(&dev_priv->drm, pipe); ++ __entry->scanline = intel_get_crtc_scanline(intel_get_crtc_for_pipe(dev_priv, pipe)); ++ ), ++ ++ TP_printk("pipe %c, frame=%u, scanline=%u", ++ pipe_name(__entry->pipe), ++ __entry->frame, __entry->scanline) ++); ++ ++TRACE_EVENT(intel_pch_fifo_underrun, ++ TP_PROTO(struct drm_i915_private *dev_priv, enum pipe pch_transcoder), ++ TP_ARGS(dev_priv, pch_transcoder), ++ ++ TP_STRUCT__entry( ++ __field(enum pipe, pipe) ++ __field(u32, frame) ++ __field(u32, scanline) ++ ), ++ ++ TP_fast_assign( ++ enum pipe pipe = pch_transcoder; ++ __entry->pipe = pipe; ++ __entry->frame = dev_priv->drm.driver->get_vblank_counter(&dev_priv->drm, pipe); ++ __entry->scanline = intel_get_crtc_scanline(intel_get_crtc_for_pipe(dev_priv, pipe)); ++ ), ++ ++ TP_printk("pch transcoder %c, frame=%u, scanline=%u", ++ pipe_name(__entry->pipe), ++ __entry->frame, __entry->scanline) ++); ++ ++TRACE_EVENT(intel_memory_cxsr, ++ TP_PROTO(struct drm_i915_private *dev_priv, bool old, bool new), ++ TP_ARGS(dev_priv, old, new), ++ ++ TP_STRUCT__entry( ++ __array(u32, frame, 3) ++ __array(u32, scanline, 3) ++ __field(bool, old) ++ __field(bool, new) ++ ), ++ ++ TP_fast_assign( ++ enum pipe pipe; ++ for_each_pipe(dev_priv, pipe) { ++ __entry->frame[pipe] = ++ dev_priv->drm.driver->get_vblank_counter(&dev_priv->drm, pipe); ++ __entry->scanline[pipe] = ++ intel_get_crtc_scanline(intel_get_crtc_for_pipe(dev_priv, pipe)); ++ } ++ __entry->old = old; ++ __entry->new = new; ++ ), ++ ++ TP_printk("%s->%s, pipe A: frame=%u, scanline=%u, pipe B: frame=%u, scanline=%u, pipe C: frame=%u, scanline=%u", ++ onoff(__entry->old), onoff(__entry->new), ++ __entry->frame[PIPE_A], __entry->scanline[PIPE_A], ++ __entry->frame[PIPE_B], __entry->scanline[PIPE_B], ++ __entry->frame[PIPE_C], __entry->scanline[PIPE_C]) ++); ++ ++TRACE_EVENT(g4x_wm, ++ TP_PROTO(struct intel_crtc *crtc, const struct g4x_wm_values *wm), ++ TP_ARGS(crtc, wm), ++ ++ TP_STRUCT__entry( ++ __field(enum pipe, pipe) ++ __field(u32, frame) ++ __field(u32, scanline) ++ __field(u16, primary) ++ __field(u16, sprite) ++ __field(u16, cursor) ++ __field(u16, sr_plane) ++ __field(u16, sr_cursor) ++ __field(u16, sr_fbc) ++ __field(u16, hpll_plane) ++ __field(u16, hpll_cursor) ++ __field(u16, hpll_fbc) ++ __field(bool, cxsr) ++ __field(bool, hpll) ++ __field(bool, fbc) ++ ), ++ ++ TP_fast_assign( ++ __entry->pipe = crtc->pipe; ++ __entry->frame = crtc->base.dev->driver->get_vblank_counter(crtc->base.dev, ++ crtc->pipe); ++ __entry->scanline = intel_get_crtc_scanline(crtc); ++ __entry->primary = wm->pipe[crtc->pipe].plane[PLANE_PRIMARY]; ++ __entry->sprite = wm->pipe[crtc->pipe].plane[PLANE_SPRITE0]; ++ __entry->cursor = wm->pipe[crtc->pipe].plane[PLANE_CURSOR]; ++ __entry->sr_plane = wm->sr.plane; ++ __entry->sr_cursor = wm->sr.cursor; ++ __entry->sr_fbc = wm->sr.fbc; ++ __entry->hpll_plane = wm->hpll.plane; ++ __entry->hpll_cursor = wm->hpll.cursor; ++ __entry->hpll_fbc = wm->hpll.fbc; ++ __entry->cxsr = wm->cxsr; ++ __entry->hpll = wm->hpll_en; ++ __entry->fbc = wm->fbc_en; ++ ), ++ ++ TP_printk("pipe %c, frame=%u, scanline=%u, wm %d/%d/%d, sr %s/%d/%d/%d, hpll %s/%d/%d/%d, fbc %s", ++ pipe_name(__entry->pipe), __entry->frame, __entry->scanline, ++ __entry->primary, __entry->sprite, __entry->cursor, ++ yesno(__entry->cxsr), __entry->sr_plane, __entry->sr_cursor, __entry->sr_fbc, ++ yesno(__entry->hpll), __entry->hpll_plane, __entry->hpll_cursor, __entry->hpll_fbc, ++ yesno(__entry->fbc)) ++); ++ ++TRACE_EVENT(vlv_wm, ++ TP_PROTO(struct intel_crtc *crtc, const struct vlv_wm_values *wm), ++ TP_ARGS(crtc, wm), ++ ++ TP_STRUCT__entry( ++ __field(enum pipe, pipe) ++ __field(u32, frame) ++ __field(u32, scanline) ++ __field(u32, level) ++ __field(u32, cxsr) ++ __field(u32, primary) ++ __field(u32, sprite0) ++ __field(u32, sprite1) ++ __field(u32, cursor) ++ __field(u32, sr_plane) ++ __field(u32, sr_cursor) ++ ), ++ ++ TP_fast_assign( ++ __entry->pipe = crtc->pipe; ++ __entry->frame = crtc->base.dev->driver->get_vblank_counter(crtc->base.dev, ++ crtc->pipe); ++ __entry->scanline = intel_get_crtc_scanline(crtc); ++ __entry->level = wm->level; ++ __entry->cxsr = wm->cxsr; ++ __entry->primary = wm->pipe[crtc->pipe].plane[PLANE_PRIMARY]; ++ __entry->sprite0 = wm->pipe[crtc->pipe].plane[PLANE_SPRITE0]; ++ __entry->sprite1 = wm->pipe[crtc->pipe].plane[PLANE_SPRITE1]; ++ __entry->cursor = wm->pipe[crtc->pipe].plane[PLANE_CURSOR]; ++ __entry->sr_plane = wm->sr.plane; ++ __entry->sr_cursor = wm->sr.cursor; ++ ), ++ ++ TP_printk("pipe %c, frame=%u, scanline=%u, level=%d, cxsr=%d, wm %d/%d/%d/%d, sr %d/%d", ++ pipe_name(__entry->pipe), __entry->frame, ++ __entry->scanline, __entry->level, __entry->cxsr, ++ __entry->primary, __entry->sprite0, __entry->sprite1, __entry->cursor, ++ __entry->sr_plane, __entry->sr_cursor) ++); ++ ++TRACE_EVENT(vlv_fifo_size, ++ TP_PROTO(struct intel_crtc *crtc, u32 sprite0_start, u32 sprite1_start, u32 fifo_size), ++ TP_ARGS(crtc, sprite0_start, sprite1_start, fifo_size), ++ ++ TP_STRUCT__entry( ++ __field(enum pipe, pipe) ++ __field(u32, frame) ++ __field(u32, scanline) ++ __field(u32, sprite0_start) ++ __field(u32, sprite1_start) ++ __field(u32, fifo_size) ++ ), ++ ++ TP_fast_assign( ++ __entry->pipe = crtc->pipe; ++ __entry->frame = crtc->base.dev->driver->get_vblank_counter(crtc->base.dev, ++ crtc->pipe); ++ __entry->scanline = intel_get_crtc_scanline(crtc); ++ __entry->sprite0_start = sprite0_start; ++ __entry->sprite1_start = sprite1_start; ++ __entry->fifo_size = fifo_size; ++ ), ++ ++ TP_printk("pipe %c, frame=%u, scanline=%u, %d/%d/%d", ++ pipe_name(__entry->pipe), __entry->frame, ++ __entry->scanline, __entry->sprite0_start, ++ __entry->sprite1_start, __entry->fifo_size) ++); ++ ++/* plane updates */ ++ ++TRACE_EVENT(intel_update_plane, ++ TP_PROTO(struct drm_plane *plane, struct intel_crtc *crtc), ++ TP_ARGS(plane, crtc), ++ ++ TP_STRUCT__entry( ++ __field(enum pipe, pipe) ++ __field(const char *, name) ++ __field(u32, frame) ++ __field(u32, scanline) ++ __array(int, src, 4) ++ __array(int, dst, 4) ++ ), ++ ++ TP_fast_assign( ++ __entry->pipe = crtc->pipe; ++ __entry->name = plane->name; ++ __entry->frame = crtc->base.dev->driver->get_vblank_counter(crtc->base.dev, ++ crtc->pipe); ++ __entry->scanline = intel_get_crtc_scanline(crtc); ++ memcpy(__entry->src, &plane->state->src, sizeof(__entry->src)); ++ memcpy(__entry->dst, &plane->state->dst, sizeof(__entry->dst)); ++ ), ++ ++ TP_printk("pipe %c, plane %s, frame=%u, scanline=%u, " DRM_RECT_FP_FMT " -> " DRM_RECT_FMT, ++ pipe_name(__entry->pipe), __entry->name, ++ __entry->frame, __entry->scanline, ++ DRM_RECT_FP_ARG((const struct drm_rect *)__entry->src), ++ DRM_RECT_ARG((const struct drm_rect *)__entry->dst)) ++); ++ ++TRACE_EVENT(intel_disable_plane, ++ TP_PROTO(struct drm_plane *plane, struct intel_crtc *crtc), ++ TP_ARGS(plane, crtc), ++ ++ TP_STRUCT__entry( ++ __field(enum pipe, pipe) ++ __field(const char *, name) ++ __field(u32, frame) ++ __field(u32, scanline) ++ ), ++ ++ TP_fast_assign( ++ __entry->pipe = crtc->pipe; ++ __entry->name = plane->name; ++ __entry->frame = crtc->base.dev->driver->get_vblank_counter(crtc->base.dev, ++ crtc->pipe); ++ __entry->scanline = intel_get_crtc_scanline(crtc); ++ ), ++ ++ TP_printk("pipe %c, plane %s, frame=%u, scanline=%u", ++ pipe_name(__entry->pipe), __entry->name, ++ __entry->frame, __entry->scanline) ++); ++ ++/* pipe updates */ ++ ++TRACE_EVENT(i915_pipe_update_start, ++ TP_PROTO(struct intel_crtc *crtc), ++ TP_ARGS(crtc), ++ ++ TP_STRUCT__entry( ++ __field(enum pipe, pipe) ++ __field(u32, frame) ++ __field(u32, scanline) ++ __field(u32, min) ++ __field(u32, max) ++ ), ++ ++ TP_fast_assign( ++ __entry->pipe = crtc->pipe; ++ __entry->frame = crtc->base.dev->driver->get_vblank_counter(crtc->base.dev, ++ crtc->pipe); ++ __entry->scanline = intel_get_crtc_scanline(crtc); ++ __entry->min = crtc->debug.min_vbl; ++ __entry->max = crtc->debug.max_vbl; ++ ), ++ ++ TP_printk("pipe %c, frame=%u, scanline=%u, min=%u, max=%u", ++ pipe_name(__entry->pipe), __entry->frame, ++ __entry->scanline, __entry->min, __entry->max) ++); ++ ++TRACE_EVENT(i915_pipe_update_vblank_evaded, ++ TP_PROTO(struct intel_crtc *crtc), ++ TP_ARGS(crtc), ++ ++ TP_STRUCT__entry( ++ __field(enum pipe, pipe) ++ __field(u32, frame) ++ __field(u32, scanline) ++ __field(u32, min) ++ __field(u32, max) ++ ), ++ ++ TP_fast_assign( ++ __entry->pipe = crtc->pipe; ++ __entry->frame = crtc->debug.start_vbl_count; ++ __entry->scanline = crtc->debug.scanline_start; ++ __entry->min = crtc->debug.min_vbl; ++ __entry->max = crtc->debug.max_vbl; ++ ), ++ ++ TP_printk("pipe %c, frame=%u, scanline=%u, min=%u, max=%u", ++ pipe_name(__entry->pipe), __entry->frame, ++ __entry->scanline, __entry->min, __entry->max) ++); ++ ++TRACE_EVENT(i915_pipe_update_end, ++ TP_PROTO(struct intel_crtc *crtc, u32 frame, int scanline_end), ++ TP_ARGS(crtc, frame, scanline_end), ++ ++ TP_STRUCT__entry( ++ __field(enum pipe, pipe) ++ __field(u32, frame) ++ __field(u32, scanline) ++ ), ++ ++ TP_fast_assign( ++ __entry->pipe = crtc->pipe; ++ __entry->frame = frame; ++ __entry->scanline = scanline_end; ++ ), ++ ++ TP_printk("pipe %c, frame=%u, scanline=%u", ++ pipe_name(__entry->pipe), __entry->frame, ++ __entry->scanline) ++); ++ ++/* object tracking */ ++ ++TRACE_EVENT(i915_gem_object_create, ++ TP_PROTO(struct drm_i915_gem_object *obj), ++ TP_ARGS(obj), ++ ++ TP_STRUCT__entry( ++ __field(struct drm_i915_gem_object *, obj) ++ __field(u64, size) ++ ), ++ ++ TP_fast_assign( ++ __entry->obj = obj; ++ __entry->size = obj->base.size; ++ ), ++ ++ TP_printk("obj=%p, size=0x%llx", __entry->obj, __entry->size) ++); ++ ++TRACE_EVENT(i915_gem_shrink, ++ TP_PROTO(struct drm_i915_private *i915, unsigned long target, unsigned flags), ++ TP_ARGS(i915, target, flags), ++ ++ TP_STRUCT__entry( ++ __field(int, dev) ++ __field(unsigned long, target) ++ __field(unsigned, flags) ++ ), ++ ++ TP_fast_assign( ++ __entry->dev = i915->drm.primary->index; ++ __entry->target = target; ++ __entry->flags = flags; ++ ), ++ ++ TP_printk("dev=%d, target=%lu, flags=%x", ++ __entry->dev, __entry->target, __entry->flags) ++); ++ ++TRACE_EVENT(i915_vma_bind, ++ TP_PROTO(struct i915_vma *vma, unsigned flags), ++ TP_ARGS(vma, flags), ++ ++ TP_STRUCT__entry( ++ __field(struct drm_i915_gem_object *, obj) ++ __field(struct i915_address_space *, vm) ++ __field(u64, offset) ++ __field(u64, size) ++ __field(unsigned, flags) ++ ), ++ ++ TP_fast_assign( ++ __entry->obj = vma->obj; ++ __entry->vm = vma->vm; ++ __entry->offset = vma->node.start; ++ __entry->size = vma->node.size; ++ __entry->flags = flags; ++ ), ++ ++ TP_printk("obj=%p, offset=0x%016llx size=0x%llx%s vm=%p", ++ __entry->obj, __entry->offset, __entry->size, ++ __entry->flags & PIN_MAPPABLE ? ", mappable" : "", ++ __entry->vm) ++); ++ ++TRACE_EVENT(i915_vma_unbind, ++ TP_PROTO(struct i915_vma *vma), ++ TP_ARGS(vma), ++ ++ TP_STRUCT__entry( ++ __field(struct drm_i915_gem_object *, obj) ++ __field(struct i915_address_space *, vm) ++ __field(u64, offset) ++ __field(u64, size) ++ ), ++ ++ TP_fast_assign( ++ __entry->obj = vma->obj; ++ __entry->vm = vma->vm; ++ __entry->offset = vma->node.start; ++ __entry->size = vma->node.size; ++ ), ++ ++ TP_printk("obj=%p, offset=0x%016llx size=0x%llx vm=%p", ++ __entry->obj, __entry->offset, __entry->size, __entry->vm) ++); ++ ++TRACE_EVENT(i915_gem_object_pwrite, ++ TP_PROTO(struct drm_i915_gem_object *obj, u64 offset, u64 len), ++ TP_ARGS(obj, offset, len), ++ ++ TP_STRUCT__entry( ++ __field(struct drm_i915_gem_object *, obj) ++ __field(u64, offset) ++ __field(u64, len) ++ ), ++ ++ TP_fast_assign( ++ __entry->obj = obj; ++ __entry->offset = offset; ++ __entry->len = len; ++ ), ++ ++ TP_printk("obj=%p, offset=0x%llx, len=0x%llx", ++ __entry->obj, __entry->offset, __entry->len) ++); ++ ++TRACE_EVENT(i915_gem_object_pread, ++ TP_PROTO(struct drm_i915_gem_object *obj, u64 offset, u64 len), ++ TP_ARGS(obj, offset, len), ++ ++ TP_STRUCT__entry( ++ __field(struct drm_i915_gem_object *, obj) ++ __field(u64, offset) ++ __field(u64, len) ++ ), ++ ++ TP_fast_assign( ++ __entry->obj = obj; ++ __entry->offset = offset; ++ __entry->len = len; ++ ), ++ ++ TP_printk("obj=%p, offset=0x%llx, len=0x%llx", ++ __entry->obj, __entry->offset, __entry->len) ++); ++ ++TRACE_EVENT(i915_gem_object_fault, ++ TP_PROTO(struct drm_i915_gem_object *obj, u64 index, bool gtt, bool write), ++ TP_ARGS(obj, index, gtt, write), ++ ++ TP_STRUCT__entry( ++ __field(struct drm_i915_gem_object *, obj) ++ __field(u64, index) ++ __field(bool, gtt) ++ __field(bool, write) ++ ), ++ ++ TP_fast_assign( ++ __entry->obj = obj; ++ __entry->index = index; ++ __entry->gtt = gtt; ++ __entry->write = write; ++ ), ++ ++ TP_printk("obj=%p, %s index=%llu %s", ++ __entry->obj, ++ __entry->gtt ? "GTT" : "CPU", ++ __entry->index, ++ __entry->write ? ", writable" : "") ++); ++ ++DECLARE_EVENT_CLASS(i915_gem_object, ++ TP_PROTO(struct drm_i915_gem_object *obj), ++ TP_ARGS(obj), ++ ++ TP_STRUCT__entry( ++ __field(struct drm_i915_gem_object *, obj) ++ ), ++ ++ TP_fast_assign( ++ __entry->obj = obj; ++ ), ++ ++ TP_printk("obj=%p", __entry->obj) ++); ++ ++DEFINE_EVENT(i915_gem_object, i915_gem_object_clflush, ++ TP_PROTO(struct drm_i915_gem_object *obj), ++ TP_ARGS(obj) ++); ++ ++DEFINE_EVENT(i915_gem_object, i915_gem_object_destroy, ++ TP_PROTO(struct drm_i915_gem_object *obj), ++ TP_ARGS(obj) ++); ++ ++TRACE_EVENT(i915_gem_evict, ++ TP_PROTO(struct i915_address_space *vm, u64 size, u64 align, unsigned int flags), ++ TP_ARGS(vm, size, align, flags), ++ ++ TP_STRUCT__entry( ++ __field(u32, dev) ++ __field(struct i915_address_space *, vm) ++ __field(u64, size) ++ __field(u64, align) ++ __field(unsigned int, flags) ++ ), ++ ++ TP_fast_assign( ++ __entry->dev = vm->i915->drm.primary->index; ++ __entry->vm = vm; ++ __entry->size = size; ++ __entry->align = align; ++ __entry->flags = flags; ++ ), ++ ++ TP_printk("dev=%d, vm=%p, size=0x%llx, align=0x%llx %s", ++ __entry->dev, __entry->vm, __entry->size, __entry->align, ++ __entry->flags & PIN_MAPPABLE ? ", mappable" : "") ++); ++ ++TRACE_EVENT(i915_gem_evict_node, ++ TP_PROTO(struct i915_address_space *vm, struct drm_mm_node *node, unsigned int flags), ++ TP_ARGS(vm, node, flags), ++ ++ TP_STRUCT__entry( ++ __field(u32, dev) ++ __field(struct i915_address_space *, vm) ++ __field(u64, start) ++ __field(u64, size) ++ __field(unsigned long, color) ++ __field(unsigned int, flags) ++ ), ++ ++ TP_fast_assign( ++ __entry->dev = vm->i915->drm.primary->index; ++ __entry->vm = vm; ++ __entry->start = node->start; ++ __entry->size = node->size; ++ __entry->color = node->color; ++ __entry->flags = flags; ++ ), ++ ++ TP_printk("dev=%d, vm=%p, start=0x%llx size=0x%llx, color=0x%lx, flags=%x", ++ __entry->dev, __entry->vm, ++ __entry->start, __entry->size, ++ __entry->color, __entry->flags) ++); ++ ++TRACE_EVENT(i915_gem_evict_vm, ++ TP_PROTO(struct i915_address_space *vm), ++ TP_ARGS(vm), ++ ++ TP_STRUCT__entry( ++ __field(u32, dev) ++ __field(struct i915_address_space *, vm) ++ ), ++ ++ TP_fast_assign( ++ __entry->dev = vm->i915->drm.primary->index; ++ __entry->vm = vm; ++ ), ++ ++ TP_printk("dev=%d, vm=%p", __entry->dev, __entry->vm) ++); ++ ++TRACE_EVENT(i915_request_queue, ++ TP_PROTO(struct i915_request *rq, u32 flags), ++ TP_ARGS(rq, flags), ++ ++ TP_STRUCT__entry( ++ __field(u32, dev) ++ __field(u32, hw_id) ++ __field(u64, ctx) ++ __field(u16, class) ++ __field(u16, instance) ++ __field(u32, seqno) ++ __field(u32, flags) ++ ), ++ ++ TP_fast_assign( ++ __entry->dev = rq->i915->drm.primary->index; ++ __entry->hw_id = rq->gem_context->hw_id; ++ __entry->class = rq->engine->uabi_class; ++ __entry->instance = rq->engine->instance; ++ __entry->ctx = rq->fence.context; ++ __entry->seqno = rq->fence.seqno; ++ __entry->flags = flags; ++ ), ++ ++ TP_printk("dev=%u, engine=%u:%u, hw_id=%u, ctx=%llu, seqno=%u, flags=0x%x", ++ __entry->dev, __entry->class, __entry->instance, ++ __entry->hw_id, __entry->ctx, __entry->seqno, ++ __entry->flags) ++); ++ ++DECLARE_EVENT_CLASS(i915_request, ++ TP_PROTO(struct i915_request *rq), ++ TP_ARGS(rq), ++ ++ TP_STRUCT__entry( ++ __field(u32, dev) ++ __field(u32, hw_id) ++ __field(u64, ctx) ++ __field(u16, class) ++ __field(u16, instance) ++ __field(u32, seqno) ++ ), ++ ++ TP_fast_assign( ++ __entry->dev = rq->i915->drm.primary->index; ++ __entry->hw_id = rq->gem_context->hw_id; ++ __entry->class = rq->engine->uabi_class; ++ __entry->instance = rq->engine->instance; ++ __entry->ctx = rq->fence.context; ++ __entry->seqno = rq->fence.seqno; ++ ), ++ ++ TP_printk("dev=%u, engine=%u:%u, hw_id=%u, ctx=%llu, seqno=%u", ++ __entry->dev, __entry->class, __entry->instance, ++ __entry->hw_id, __entry->ctx, __entry->seqno) ++); ++ ++DEFINE_EVENT(i915_request, i915_request_add, ++ TP_PROTO(struct i915_request *rq), ++ TP_ARGS(rq) ++); ++ ++#if defined(CONFIG_DRM_I915_LOW_LEVEL_TRACEPOINTS) ++DEFINE_EVENT(i915_request, i915_request_submit, ++ TP_PROTO(struct i915_request *rq), ++ TP_ARGS(rq) ++); ++ ++DEFINE_EVENT(i915_request, i915_request_execute, ++ TP_PROTO(struct i915_request *rq), ++ TP_ARGS(rq) ++); ++ ++TRACE_EVENT(i915_request_in, ++ TP_PROTO(struct i915_request *rq, unsigned int port), ++ TP_ARGS(rq, port), ++ ++ TP_STRUCT__entry( ++ __field(u32, dev) ++ __field(u32, hw_id) ++ __field(u64, ctx) ++ __field(u16, class) ++ __field(u16, instance) ++ __field(u32, seqno) ++ __field(u32, port) ++ __field(u32, prio) ++ ), ++ ++ TP_fast_assign( ++ __entry->dev = rq->i915->drm.primary->index; ++ __entry->hw_id = rq->gem_context->hw_id; ++ __entry->class = rq->engine->uabi_class; ++ __entry->instance = rq->engine->instance; ++ __entry->ctx = rq->fence.context; ++ __entry->seqno = rq->fence.seqno; ++ __entry->prio = rq->sched.attr.priority; ++ __entry->port = port; ++ ), ++ ++ TP_printk("dev=%u, engine=%u:%u, hw_id=%u, ctx=%llu, seqno=%u, prio=%u, port=%u", ++ __entry->dev, __entry->class, __entry->instance, ++ __entry->hw_id, __entry->ctx, __entry->seqno, ++ __entry->prio, __entry->port) ++); ++ ++TRACE_EVENT(i915_request_out, ++ TP_PROTO(struct i915_request *rq), ++ TP_ARGS(rq), ++ ++ TP_STRUCT__entry( ++ __field(u32, dev) ++ __field(u32, hw_id) ++ __field(u64, ctx) ++ __field(u16, class) ++ __field(u16, instance) ++ __field(u32, seqno) ++ __field(u32, completed) ++ ), ++ ++ TP_fast_assign( ++ __entry->dev = rq->i915->drm.primary->index; ++ __entry->hw_id = rq->gem_context->hw_id; ++ __entry->class = rq->engine->uabi_class; ++ __entry->instance = rq->engine->instance; ++ __entry->ctx = rq->fence.context; ++ __entry->seqno = rq->fence.seqno; ++ __entry->completed = i915_request_completed(rq); ++ ), ++ ++ TP_printk("dev=%u, engine=%u:%u, hw_id=%u, ctx=%llu, seqno=%u, completed?=%u", ++ __entry->dev, __entry->class, __entry->instance, ++ __entry->hw_id, __entry->ctx, __entry->seqno, ++ __entry->completed) ++); ++ ++#else ++#if !defined(TRACE_HEADER_MULTI_READ) ++static inline void ++trace_i915_request_submit(struct i915_request *rq) ++{ ++} ++ ++static inline void ++trace_i915_request_execute(struct i915_request *rq) ++{ ++} ++ ++static inline void ++trace_i915_request_in(struct i915_request *rq, unsigned int port) ++{ ++} ++ ++static inline void ++trace_i915_request_out(struct i915_request *rq) ++{ ++} ++#endif ++#endif ++ ++DEFINE_EVENT(i915_request, i915_request_retire, ++ TP_PROTO(struct i915_request *rq), ++ TP_ARGS(rq) ++); ++ ++TRACE_EVENT(i915_request_wait_begin, ++ TP_PROTO(struct i915_request *rq, unsigned int flags), ++ TP_ARGS(rq, flags), ++ ++ TP_STRUCT__entry( ++ __field(u32, dev) ++ __field(u32, hw_id) ++ __field(u64, ctx) ++ __field(u16, class) ++ __field(u16, instance) ++ __field(u32, seqno) ++ __field(unsigned int, flags) ++ ), ++ ++ /* NB: the blocking information is racy since mutex_is_locked ++ * doesn't check that the current thread holds the lock. The only ++ * other option would be to pass the boolean information of whether ++ * or not the class was blocking down through the stack which is ++ * less desirable. ++ */ ++ TP_fast_assign( ++ __entry->dev = rq->i915->drm.primary->index; ++ __entry->hw_id = rq->gem_context->hw_id; ++ __entry->class = rq->engine->uabi_class; ++ __entry->instance = rq->engine->instance; ++ __entry->ctx = rq->fence.context; ++ __entry->seqno = rq->fence.seqno; ++ __entry->flags = flags; ++ ), ++ ++ TP_printk("dev=%u, engine=%u:%u, hw_id=%u, ctx=%llu, seqno=%u, blocking=%u, flags=0x%x", ++ __entry->dev, __entry->class, __entry->instance, ++ __entry->hw_id, __entry->ctx, __entry->seqno, ++ !!(__entry->flags & I915_WAIT_LOCKED), ++ __entry->flags) ++); ++ ++DEFINE_EVENT(i915_request, i915_request_wait_end, ++ TP_PROTO(struct i915_request *rq), ++ TP_ARGS(rq) ++); ++ ++TRACE_EVENT_CONDITION(i915_reg_rw, ++ TP_PROTO(bool write, i915_reg_t reg, u64 val, int len, bool trace), ++ ++ TP_ARGS(write, reg, val, len, trace), ++ ++ TP_CONDITION(trace), ++ ++ TP_STRUCT__entry( ++ __field(u64, val) ++ __field(u32, reg) ++ __field(u16, write) ++ __field(u16, len) ++ ), ++ ++ TP_fast_assign( ++ __entry->val = (u64)val; ++ __entry->reg = i915_mmio_reg_offset(reg); ++ __entry->write = write; ++ __entry->len = len; ++ ), ++ ++ TP_printk("%s reg=0x%x, len=%d, val=(0x%x, 0x%x)", ++ __entry->write ? "write" : "read", ++ __entry->reg, __entry->len, ++ (u32)(__entry->val & 0xffffffff), ++ (u32)(__entry->val >> 32)) ++); ++ ++TRACE_EVENT(intel_gpu_freq_change, ++ TP_PROTO(u32 freq), ++ TP_ARGS(freq), ++ ++ TP_STRUCT__entry( ++ __field(u32, freq) ++ ), ++ ++ TP_fast_assign( ++ __entry->freq = freq; ++ ), ++ ++ TP_printk("new_freq=%u", __entry->freq) ++); ++ ++/** ++ * DOC: i915_ppgtt_create and i915_ppgtt_release tracepoints ++ * ++ * With full ppgtt enabled each process using drm will allocate at least one ++ * translation table. With these traces it is possible to keep track of the ++ * allocation and of the lifetime of the tables; this can be used during ++ * testing/debug to verify that we are not leaking ppgtts. ++ * These traces identify the ppgtt through the vm pointer, which is also printed ++ * by the i915_vma_bind and i915_vma_unbind tracepoints. ++ */ ++DECLARE_EVENT_CLASS(i915_ppgtt, ++ TP_PROTO(struct i915_address_space *vm), ++ TP_ARGS(vm), ++ ++ TP_STRUCT__entry( ++ __field(struct i915_address_space *, vm) ++ __field(u32, dev) ++ ), ++ ++ TP_fast_assign( ++ __entry->vm = vm; ++ __entry->dev = vm->i915->drm.primary->index; ++ ), ++ ++ TP_printk("dev=%u, vm=%p", __entry->dev, __entry->vm) ++) ++ ++DEFINE_EVENT(i915_ppgtt, i915_ppgtt_create, ++ TP_PROTO(struct i915_address_space *vm), ++ TP_ARGS(vm) ++); ++ ++DEFINE_EVENT(i915_ppgtt, i915_ppgtt_release, ++ TP_PROTO(struct i915_address_space *vm), ++ TP_ARGS(vm) ++); ++ ++/** ++ * DOC: i915_context_create and i915_context_free tracepoints ++ * ++ * These tracepoints are used to track creation and deletion of contexts. ++ * If full ppgtt is enabled, they also print the address of the vm assigned to ++ * the context. ++ */ ++DECLARE_EVENT_CLASS(i915_context, ++ TP_PROTO(struct i915_gem_context *ctx), ++ TP_ARGS(ctx), ++ ++ TP_STRUCT__entry( ++ __field(u32, dev) ++ __field(struct i915_gem_context *, ctx) ++ __field(u32, hw_id) ++ __field(struct i915_address_space *, vm) ++ ), ++ ++ TP_fast_assign( ++ __entry->dev = ctx->i915->drm.primary->index; ++ __entry->ctx = ctx; ++ __entry->hw_id = ctx->hw_id; ++ __entry->vm = ctx->ppgtt ? &ctx->ppgtt->vm : NULL; ++ ), ++ ++ TP_printk("dev=%u, ctx=%p, ctx_vm=%p, hw_id=%u", ++ __entry->dev, __entry->ctx, __entry->vm, __entry->hw_id) ++) ++ ++DEFINE_EVENT(i915_context, i915_context_create, ++ TP_PROTO(struct i915_gem_context *ctx), ++ TP_ARGS(ctx) ++); ++ ++DEFINE_EVENT(i915_context, i915_context_free, ++ TP_PROTO(struct i915_gem_context *ctx), ++ TP_ARGS(ctx) ++); ++ ++#endif /* _I915_TRACE_H_ */ ++ ++/* This part must be outside protection */ ++#undef TRACE_INCLUDE_PATH ++#define TRACE_INCLUDE_PATH ../../drivers/gpu/drm/i915_legacy ++#include +diff --git a/drivers/gpu/drm/i915_legacy/i915_trace_points.c b/drivers/gpu/drm/i915_legacy/i915_trace_points.c +new file mode 100644 +index 000000000000..463a7177997c +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_trace_points.c +@@ -0,0 +1,14 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright © 2009 Intel Corporation ++ * ++ * Authors: ++ * Chris Wilson ++ */ ++ ++#include "i915_drv.h" ++ ++#ifndef __CHECKER__ ++#define CREATE_TRACE_POINTS ++#include "i915_trace.h" ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/i915_user_extensions.c b/drivers/gpu/drm/i915_legacy/i915_user_extensions.c +new file mode 100644 +index 000000000000..c822d0aafd2d +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_user_extensions.c +@@ -0,0 +1,61 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ */ ++ ++#include ++#include ++#include ++ ++#include ++ ++#include "i915_user_extensions.h" ++#include "i915_utils.h" ++ ++int i915_user_extensions(struct i915_user_extension __user *ext, ++ const i915_user_extension_fn *tbl, ++ unsigned int count, ++ void *data) ++{ ++ unsigned int stackdepth = 512; ++ ++ while (ext) { ++ int i, err; ++ u32 name; ++ u64 next; ++ ++ if (!stackdepth--) /* recursion vs useful flexibility */ ++ return -E2BIG; ++ ++ err = check_user_mbz(&ext->flags); ++ if (err) ++ return err; ++ ++ for (i = 0; i < ARRAY_SIZE(ext->rsvd); i++) { ++ err = check_user_mbz(&ext->rsvd[i]); ++ if (err) ++ return err; ++ } ++ ++ if (get_user(name, &ext->name)) ++ return -EFAULT; ++ ++ err = -EINVAL; ++ if (name < count) { ++ name = array_index_nospec(name, count); ++ if (tbl[name]) ++ err = tbl[name](ext, data); ++ } ++ if (err) ++ return err; ++ ++ if (get_user(next, &ext->next_extension) || ++ overflows_type(next, ext)) ++ return -EFAULT; ++ ++ ext = u64_to_user_ptr(next); ++ } ++ ++ return 0; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_user_extensions.h b/drivers/gpu/drm/i915_legacy/i915_user_extensions.h +new file mode 100644 +index 000000000000..a14bf6bba9a1 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_user_extensions.h +@@ -0,0 +1,20 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2018 Intel Corporation ++ */ ++ ++#ifndef I915_USER_EXTENSIONS_H ++#define I915_USER_EXTENSIONS_H ++ ++struct i915_user_extension; ++ ++typedef int (*i915_user_extension_fn)(struct i915_user_extension __user *ext, ++ void *data); ++ ++int i915_user_extensions(struct i915_user_extension __user *ext, ++ const i915_user_extension_fn *tbl, ++ unsigned int count, ++ void *data); ++ ++#endif /* I915_USER_EXTENSIONS_H */ +diff --git a/drivers/gpu/drm/i915_legacy/i915_utils.h b/drivers/gpu/drm/i915_legacy/i915_utils.h +new file mode 100644 +index 000000000000..2dbe8933b50a +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_utils.h +@@ -0,0 +1,192 @@ ++/* ++ * Copyright © 2016 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#ifndef __I915_UTILS_H ++#define __I915_UTILS_H ++ ++#undef WARN_ON ++/* Many gcc seem to no see through this and fall over :( */ ++#if 0 ++#define WARN_ON(x) ({ \ ++ bool __i915_warn_cond = (x); \ ++ if (__builtin_constant_p(__i915_warn_cond)) \ ++ BUILD_BUG_ON(__i915_warn_cond); \ ++ WARN(__i915_warn_cond, "WARN_ON(" #x ")"); }) ++#else ++#define WARN_ON(x) WARN((x), "%s", "WARN_ON(" __stringify(x) ")") ++#endif ++ ++#undef WARN_ON_ONCE ++#define WARN_ON_ONCE(x) WARN_ONCE((x), "%s", "WARN_ON_ONCE(" __stringify(x) ")") ++ ++#define MISSING_CASE(x) WARN(1, "Missing case (%s == %ld)\n", \ ++ __stringify(x), (long)(x)) ++ ++#if defined(GCC_VERSION) && GCC_VERSION >= 70000 ++#define add_overflows_t(T, A, B) \ ++ __builtin_add_overflow_p((A), (B), (T)0) ++#else ++#define add_overflows_t(T, A, B) ({ \ ++ typeof(A) a = (A); \ ++ typeof(B) b = (B); \ ++ (T)(a + b) < a; \ ++}) ++#endif ++ ++#define add_overflows(A, B) \ ++ add_overflows_t(typeof((A) + (B)), (A), (B)) ++ ++#define range_overflows(start, size, max) ({ \ ++ typeof(start) start__ = (start); \ ++ typeof(size) size__ = (size); \ ++ typeof(max) max__ = (max); \ ++ (void)(&start__ == &size__); \ ++ (void)(&start__ == &max__); \ ++ start__ > max__ || size__ > max__ - start__; \ ++}) ++ ++#define range_overflows_t(type, start, size, max) \ ++ range_overflows((type)(start), (type)(size), (type)(max)) ++ ++/* Note we don't consider signbits :| */ ++#define overflows_type(x, T) \ ++ (sizeof(x) > sizeof(T) && (x) >> BITS_PER_TYPE(T)) ++ ++#define ptr_mask_bits(ptr, n) ({ \ ++ unsigned long __v = (unsigned long)(ptr); \ ++ (typeof(ptr))(__v & -BIT(n)); \ ++}) ++ ++#define ptr_unmask_bits(ptr, n) ((unsigned long)(ptr) & (BIT(n) - 1)) ++ ++#define ptr_unpack_bits(ptr, bits, n) ({ \ ++ unsigned long __v = (unsigned long)(ptr); \ ++ *(bits) = __v & (BIT(n) - 1); \ ++ (typeof(ptr))(__v & -BIT(n)); \ ++}) ++ ++#define ptr_pack_bits(ptr, bits, n) ({ \ ++ unsigned long __bits = (bits); \ ++ GEM_BUG_ON(__bits & -BIT(n)); \ ++ ((typeof(ptr))((unsigned long)(ptr) | __bits)); \ ++}) ++ ++#define page_mask_bits(ptr) ptr_mask_bits(ptr, PAGE_SHIFT) ++#define page_unmask_bits(ptr) ptr_unmask_bits(ptr, PAGE_SHIFT) ++#define page_pack_bits(ptr, bits) ptr_pack_bits(ptr, bits, PAGE_SHIFT) ++#define page_unpack_bits(ptr, bits) ptr_unpack_bits(ptr, bits, PAGE_SHIFT) ++ ++#define ptr_offset(ptr, member) offsetof(typeof(*(ptr)), member) ++ ++#define fetch_and_zero(ptr) ({ \ ++ typeof(*ptr) __T = *(ptr); \ ++ *(ptr) = (typeof(*ptr))0; \ ++ __T; \ ++}) ++ ++/* ++ * container_of_user: Extract the superclass from a pointer to a member. ++ * ++ * Exactly like container_of() with the exception that it plays nicely ++ * with sparse for __user @ptr. ++ */ ++#define container_of_user(ptr, type, member) ({ \ ++ void __user *__mptr = (void __user *)(ptr); \ ++ BUILD_BUG_ON_MSG(!__same_type(*(ptr), ((type *)0)->member) && \ ++ !__same_type(*(ptr), void), \ ++ "pointer type mismatch in container_of()"); \ ++ ((type __user *)(__mptr - offsetof(type, member))); }) ++ ++/* ++ * check_user_mbz: Check that a user value exists and is zero ++ * ++ * Frequently in our uABI we reserve space for future extensions, and ++ * two ensure that userspace is prepared we enforce that space must ++ * be zero. (Then any future extension can safely assume a default value ++ * of 0.) ++ * ++ * check_user_mbz() combines checking that the user pointer is accessible ++ * and that the contained value is zero. ++ * ++ * Returns: -EFAULT if not accessible, -EINVAL if !zero, or 0 on success. ++ */ ++#define check_user_mbz(U) ({ \ ++ typeof(*(U)) mbz__; \ ++ get_user(mbz__, (U)) ? -EFAULT : mbz__ ? -EINVAL : 0; \ ++}) ++ ++static inline u64 ptr_to_u64(const void *ptr) ++{ ++ return (uintptr_t)ptr; ++} ++ ++#define u64_to_ptr(T, x) ({ \ ++ typecheck(u64, x); \ ++ (T *)(uintptr_t)(x); \ ++}) ++ ++#define __mask_next_bit(mask) ({ \ ++ int __idx = ffs(mask) - 1; \ ++ mask &= ~BIT(__idx); \ ++ __idx; \ ++}) ++ ++#include ++ ++static inline void __list_del_many(struct list_head *head, ++ struct list_head *first) ++{ ++ first->prev = head; ++ WRITE_ONCE(head->next, first); ++} ++ ++/* ++ * Wait until the work is finally complete, even if it tries to postpone ++ * by requeueing itself. Note, that if the worker never cancels itself, ++ * we will spin forever. ++ */ ++static inline void drain_delayed_work(struct delayed_work *dw) ++{ ++ do { ++ while (flush_delayed_work(dw)) ++ ; ++ } while (delayed_work_pending(dw)); ++} ++ ++static inline const char *yesno(bool v) ++{ ++ return v ? "yes" : "no"; ++} ++ ++static inline const char *onoff(bool v) ++{ ++ return v ? "on" : "off"; ++} ++ ++static inline const char *enableddisabled(bool v) ++{ ++ return v ? "enabled" : "disabled"; ++} ++ ++#endif /* !__I915_UTILS_H */ +diff --git a/drivers/gpu/drm/i915_legacy/i915_vgpu.c b/drivers/gpu/drm/i915_legacy/i915_vgpu.c +new file mode 100644 +index 000000000000..724627afdedc +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_vgpu.c +@@ -0,0 +1,279 @@ ++/* ++ * Copyright(c) 2011-2015 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ */ ++ ++#include "intel_drv.h" ++#include "i915_vgpu.h" ++ ++/** ++ * DOC: Intel GVT-g guest support ++ * ++ * Intel GVT-g is a graphics virtualization technology which shares the ++ * GPU among multiple virtual machines on a time-sharing basis. Each ++ * virtual machine is presented a virtual GPU (vGPU), which has equivalent ++ * features as the underlying physical GPU (pGPU), so i915 driver can run ++ * seamlessly in a virtual machine. This file provides vGPU specific ++ * optimizations when running in a virtual machine, to reduce the complexity ++ * of vGPU emulation and to improve the overall performance. ++ * ++ * A primary function introduced here is so-called "address space ballooning" ++ * technique. Intel GVT-g partitions global graphics memory among multiple VMs, ++ * so each VM can directly access a portion of the memory without hypervisor's ++ * intervention, e.g. filling textures or queuing commands. However with the ++ * partitioning an unmodified i915 driver would assume a smaller graphics ++ * memory starting from address ZERO, then requires vGPU emulation module to ++ * translate the graphics address between 'guest view' and 'host view', for ++ * all registers and command opcodes which contain a graphics memory address. ++ * To reduce the complexity, Intel GVT-g introduces "address space ballooning", ++ * by telling the exact partitioning knowledge to each guest i915 driver, which ++ * then reserves and prevents non-allocated portions from allocation. Thus vGPU ++ * emulation module only needs to scan and validate graphics addresses without ++ * complexity of address translation. ++ * ++ */ ++ ++/** ++ * i915_check_vgpu - detect virtual GPU ++ * @dev_priv: i915 device private ++ * ++ * This function is called at the initialization stage, to detect whether ++ * running on a vGPU. ++ */ ++void i915_check_vgpu(struct drm_i915_private *dev_priv) ++{ ++ struct intel_uncore *uncore = &dev_priv->uncore; ++ u64 magic; ++ u16 version_major; ++ ++ BUILD_BUG_ON(sizeof(struct vgt_if) != VGT_PVINFO_SIZE); ++ ++ magic = __raw_uncore_read64(uncore, vgtif_reg(magic)); ++ if (magic != VGT_MAGIC) ++ return; ++ ++ version_major = __raw_uncore_read16(uncore, vgtif_reg(version_major)); ++ if (version_major < VGT_VERSION_MAJOR) { ++ DRM_INFO("VGT interface version mismatch!\n"); ++ return; ++ } ++ ++ dev_priv->vgpu.caps = __raw_uncore_read32(uncore, vgtif_reg(vgt_caps)); ++ ++ dev_priv->vgpu.active = true; ++ DRM_INFO("Virtual GPU for Intel GVT-g detected.\n"); ++} ++ ++bool intel_vgpu_has_full_ppgtt(struct drm_i915_private *dev_priv) ++{ ++ return dev_priv->vgpu.caps & VGT_CAPS_FULL_PPGTT; ++} ++ ++struct _balloon_info_ { ++ /* ++ * There are up to 2 regions per mappable/unmappable graphic ++ * memory that might be ballooned. Here, index 0/1 is for mappable ++ * graphic memory, 2/3 for unmappable graphic memory. ++ */ ++ struct drm_mm_node space[4]; ++}; ++ ++static struct _balloon_info_ bl_info; ++ ++static void vgt_deballoon_space(struct i915_ggtt *ggtt, ++ struct drm_mm_node *node) ++{ ++ if (!drm_mm_node_allocated(node)) ++ return; ++ ++ DRM_DEBUG_DRIVER("deballoon space: range [0x%llx - 0x%llx] %llu KiB.\n", ++ node->start, ++ node->start + node->size, ++ node->size / 1024); ++ ++ ggtt->vm.reserved -= node->size; ++ drm_mm_remove_node(node); ++} ++ ++/** ++ * intel_vgt_deballoon - deballoon reserved graphics address trunks ++ * @dev_priv: i915 device private data ++ * ++ * This function is called to deallocate the ballooned-out graphic memory, when ++ * driver is unloaded or when ballooning fails. ++ */ ++void intel_vgt_deballoon(struct drm_i915_private *dev_priv) ++{ ++ int i; ++ ++ if (!intel_vgpu_active(dev_priv)) ++ return; ++ ++ DRM_DEBUG("VGT deballoon.\n"); ++ ++ for (i = 0; i < 4; i++) ++ vgt_deballoon_space(&dev_priv->ggtt, &bl_info.space[i]); ++} ++ ++static int vgt_balloon_space(struct i915_ggtt *ggtt, ++ struct drm_mm_node *node, ++ unsigned long start, unsigned long end) ++{ ++ unsigned long size = end - start; ++ int ret; ++ ++ if (start >= end) ++ return -EINVAL; ++ ++ DRM_INFO("balloon space: range [ 0x%lx - 0x%lx ] %lu KiB.\n", ++ start, end, size / 1024); ++ ret = i915_gem_gtt_reserve(&ggtt->vm, node, ++ size, start, I915_COLOR_UNEVICTABLE, ++ 0); ++ if (!ret) ++ ggtt->vm.reserved += size; ++ ++ return ret; ++} ++ ++/** ++ * intel_vgt_balloon - balloon out reserved graphics address trunks ++ * @dev_priv: i915 device private data ++ * ++ * This function is called at the initialization stage, to balloon out the ++ * graphic address space allocated to other vGPUs, by marking these spaces as ++ * reserved. The ballooning related knowledge(starting address and size of ++ * the mappable/unmappable graphic memory) is described in the vgt_if structure ++ * in a reserved mmio range. ++ * ++ * To give an example, the drawing below depicts one typical scenario after ++ * ballooning. Here the vGPU1 has 2 pieces of graphic address spaces ballooned ++ * out each for the mappable and the non-mappable part. From the vGPU1 point of ++ * view, the total size is the same as the physical one, with the start address ++ * of its graphic space being zero. Yet there are some portions ballooned out( ++ * the shadow part, which are marked as reserved by drm allocator). From the ++ * host point of view, the graphic address space is partitioned by multiple ++ * vGPUs in different VMs. :: ++ * ++ * vGPU1 view Host view ++ * 0 ------> +-----------+ +-----------+ ++ * ^ |###########| | vGPU3 | ++ * | |###########| +-----------+ ++ * | |###########| | vGPU2 | ++ * | +-----------+ +-----------+ ++ * mappable GM | available | ==> | vGPU1 | ++ * | +-----------+ +-----------+ ++ * | |###########| | | ++ * v |###########| | Host | ++ * +=======+===========+ +===========+ ++ * ^ |###########| | vGPU3 | ++ * | |###########| +-----------+ ++ * | |###########| | vGPU2 | ++ * | +-----------+ +-----------+ ++ * unmappable GM | available | ==> | vGPU1 | ++ * | +-----------+ +-----------+ ++ * | |###########| | | ++ * | |###########| | Host | ++ * v |###########| | | ++ * total GM size ------> +-----------+ +-----------+ ++ * ++ * Returns: ++ * zero on success, non-zero if configuration invalid or ballooning failed ++ */ ++int intel_vgt_balloon(struct drm_i915_private *dev_priv) ++{ ++ struct i915_ggtt *ggtt = &dev_priv->ggtt; ++ unsigned long ggtt_end = ggtt->vm.total; ++ ++ unsigned long mappable_base, mappable_size, mappable_end; ++ unsigned long unmappable_base, unmappable_size, unmappable_end; ++ int ret; ++ ++ if (!intel_vgpu_active(dev_priv)) ++ return 0; ++ ++ mappable_base = I915_READ(vgtif_reg(avail_rs.mappable_gmadr.base)); ++ mappable_size = I915_READ(vgtif_reg(avail_rs.mappable_gmadr.size)); ++ unmappable_base = I915_READ(vgtif_reg(avail_rs.nonmappable_gmadr.base)); ++ unmappable_size = I915_READ(vgtif_reg(avail_rs.nonmappable_gmadr.size)); ++ ++ mappable_end = mappable_base + mappable_size; ++ unmappable_end = unmappable_base + unmappable_size; ++ ++ DRM_INFO("VGT ballooning configuration:\n"); ++ DRM_INFO("Mappable graphic memory: base 0x%lx size %ldKiB\n", ++ mappable_base, mappable_size / 1024); ++ DRM_INFO("Unmappable graphic memory: base 0x%lx size %ldKiB\n", ++ unmappable_base, unmappable_size / 1024); ++ ++ if (mappable_end > ggtt->mappable_end || ++ unmappable_base < ggtt->mappable_end || ++ unmappable_end > ggtt_end) { ++ DRM_ERROR("Invalid ballooning configuration!\n"); ++ return -EINVAL; ++ } ++ ++ /* Unmappable graphic memory ballooning */ ++ if (unmappable_base > ggtt->mappable_end) { ++ ret = vgt_balloon_space(ggtt, &bl_info.space[2], ++ ggtt->mappable_end, unmappable_base); ++ ++ if (ret) ++ goto err; ++ } ++ ++ if (unmappable_end < ggtt_end) { ++ ret = vgt_balloon_space(ggtt, &bl_info.space[3], ++ unmappable_end, ggtt_end); ++ if (ret) ++ goto err_upon_mappable; ++ } ++ ++ /* Mappable graphic memory ballooning */ ++ if (mappable_base) { ++ ret = vgt_balloon_space(ggtt, &bl_info.space[0], ++ 0, mappable_base); ++ ++ if (ret) ++ goto err_upon_unmappable; ++ } ++ ++ if (mappable_end < ggtt->mappable_end) { ++ ret = vgt_balloon_space(ggtt, &bl_info.space[1], ++ mappable_end, ggtt->mappable_end); ++ ++ if (ret) ++ goto err_below_mappable; ++ } ++ ++ DRM_INFO("VGT balloon successfully\n"); ++ return 0; ++ ++err_below_mappable: ++ vgt_deballoon_space(ggtt, &bl_info.space[0]); ++err_upon_unmappable: ++ vgt_deballoon_space(ggtt, &bl_info.space[3]); ++err_upon_mappable: ++ vgt_deballoon_space(ggtt, &bl_info.space[2]); ++err: ++ DRM_ERROR("VGT balloon fail\n"); ++ return ret; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_vgpu.h b/drivers/gpu/drm/i915_legacy/i915_vgpu.h +new file mode 100644 +index 000000000000..ebe1b7bced98 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_vgpu.h +@@ -0,0 +1,48 @@ ++/* ++ * Copyright(c) 2011-2015 Intel Corporation. All rights reserved. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ */ ++ ++#ifndef _I915_VGPU_H_ ++#define _I915_VGPU_H_ ++ ++#include "i915_pvinfo.h" ++ ++void i915_check_vgpu(struct drm_i915_private *dev_priv); ++ ++bool intel_vgpu_has_full_ppgtt(struct drm_i915_private *dev_priv); ++ ++static inline bool ++intel_vgpu_has_hwsp_emulation(struct drm_i915_private *dev_priv) ++{ ++ return dev_priv->vgpu.caps & VGT_CAPS_HWSP_EMULATION; ++} ++ ++static inline bool ++intel_vgpu_has_huge_gtt(struct drm_i915_private *dev_priv) ++{ ++ return dev_priv->vgpu.caps & VGT_CAPS_HUGE_GTT; ++} ++ ++int intel_vgt_balloon(struct drm_i915_private *dev_priv); ++void intel_vgt_deballoon(struct drm_i915_private *dev_priv); ++ ++#endif /* _I915_VGPU_H_ */ +diff --git a/drivers/gpu/drm/i915_legacy/i915_vma.c b/drivers/gpu/drm/i915_legacy/i915_vma.c +new file mode 100644 +index 000000000000..961268f66c63 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_vma.c +@@ -0,0 +1,1079 @@ ++/* ++ * Copyright © 2016 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#include "i915_vma.h" ++ ++#include "i915_drv.h" ++#include "i915_globals.h" ++#include "intel_ringbuffer.h" ++#include "intel_frontbuffer.h" ++ ++#include ++ ++static struct i915_global_vma { ++ struct i915_global base; ++ struct kmem_cache *slab_vmas; ++} global; ++ ++struct i915_vma *i915_vma_alloc(void) ++{ ++ return kmem_cache_zalloc(global.slab_vmas, GFP_KERNEL); ++} ++ ++void i915_vma_free(struct i915_vma *vma) ++{ ++ return kmem_cache_free(global.slab_vmas, vma); ++} ++ ++#if IS_ENABLED(CONFIG_DRM_I915_ERRLOG_GEM) && IS_ENABLED(CONFIG_DRM_DEBUG_MM) ++ ++#include ++ ++static void vma_print_allocator(struct i915_vma *vma, const char *reason) ++{ ++ unsigned long *entries; ++ unsigned int nr_entries; ++ char buf[512]; ++ ++ if (!vma->node.stack) { ++ DRM_DEBUG_DRIVER("vma.node [%08llx + %08llx] %s: unknown owner\n", ++ vma->node.start, vma->node.size, reason); ++ return; ++ } ++ ++ nr_entries = stack_depot_fetch(vma->node.stack, &entries); ++ stack_trace_snprint(buf, sizeof(buf), entries, nr_entries, 0); ++ DRM_DEBUG_DRIVER("vma.node [%08llx + %08llx] %s: inserted at %s\n", ++ vma->node.start, vma->node.size, reason, buf); ++} ++ ++#else ++ ++static void vma_print_allocator(struct i915_vma *vma, const char *reason) ++{ ++} ++ ++#endif ++ ++static void obj_bump_mru(struct drm_i915_gem_object *obj) ++{ ++ struct drm_i915_private *i915 = to_i915(obj->base.dev); ++ ++ spin_lock(&i915->mm.obj_lock); ++ if (obj->bind_count) ++ list_move_tail(&obj->mm.link, &i915->mm.bound_list); ++ spin_unlock(&i915->mm.obj_lock); ++ ++ obj->mm.dirty = true; /* be paranoid */ ++} ++ ++static void __i915_vma_retire(struct i915_active *ref) ++{ ++ struct i915_vma *vma = container_of(ref, typeof(*vma), active); ++ struct drm_i915_gem_object *obj = vma->obj; ++ ++ GEM_BUG_ON(!i915_gem_object_is_active(obj)); ++ if (--obj->active_count) ++ return; ++ ++ /* Prune the shared fence arrays iff completely idle (inc. external) */ ++ if (reservation_object_trylock(obj->resv)) { ++ if (reservation_object_test_signaled_rcu(obj->resv, true)) ++ reservation_object_add_excl_fence(obj->resv, NULL); ++ reservation_object_unlock(obj->resv); ++ } ++ ++ /* ++ * Bump our place on the bound list to keep it roughly in LRU order ++ * so that we don't steal from recently used but inactive objects ++ * (unless we are forced to ofc!) ++ */ ++ obj_bump_mru(obj); ++ ++ if (i915_gem_object_has_active_reference(obj)) { ++ i915_gem_object_clear_active_reference(obj); ++ i915_gem_object_put(obj); ++ } ++} ++ ++static struct i915_vma * ++vma_create(struct drm_i915_gem_object *obj, ++ struct i915_address_space *vm, ++ const struct i915_ggtt_view *view) ++{ ++ struct i915_vma *vma; ++ struct rb_node *rb, **p; ++ ++ /* The aliasing_ppgtt should never be used directly! */ ++ GEM_BUG_ON(vm == &vm->i915->mm.aliasing_ppgtt->vm); ++ ++ vma = i915_vma_alloc(); ++ if (vma == NULL) ++ return ERR_PTR(-ENOMEM); ++ ++ i915_active_init(vm->i915, &vma->active, __i915_vma_retire); ++ INIT_ACTIVE_REQUEST(&vma->last_fence); ++ ++ vma->vm = vm; ++ vma->ops = &vm->vma_ops; ++ vma->obj = obj; ++ vma->resv = obj->resv; ++ vma->size = obj->base.size; ++ vma->display_alignment = I915_GTT_MIN_ALIGNMENT; ++ ++ if (view && view->type != I915_GGTT_VIEW_NORMAL) { ++ vma->ggtt_view = *view; ++ if (view->type == I915_GGTT_VIEW_PARTIAL) { ++ GEM_BUG_ON(range_overflows_t(u64, ++ view->partial.offset, ++ view->partial.size, ++ obj->base.size >> PAGE_SHIFT)); ++ vma->size = view->partial.size; ++ vma->size <<= PAGE_SHIFT; ++ GEM_BUG_ON(vma->size > obj->base.size); ++ } else if (view->type == I915_GGTT_VIEW_ROTATED) { ++ vma->size = intel_rotation_info_size(&view->rotated); ++ vma->size <<= PAGE_SHIFT; ++ } ++ } ++ ++ if (unlikely(vma->size > vm->total)) ++ goto err_vma; ++ ++ GEM_BUG_ON(!IS_ALIGNED(vma->size, I915_GTT_PAGE_SIZE)); ++ ++ if (i915_is_ggtt(vm)) { ++ if (unlikely(overflows_type(vma->size, u32))) ++ goto err_vma; ++ ++ vma->fence_size = i915_gem_fence_size(vm->i915, vma->size, ++ i915_gem_object_get_tiling(obj), ++ i915_gem_object_get_stride(obj)); ++ if (unlikely(vma->fence_size < vma->size || /* overflow */ ++ vma->fence_size > vm->total)) ++ goto err_vma; ++ ++ GEM_BUG_ON(!IS_ALIGNED(vma->fence_size, I915_GTT_MIN_ALIGNMENT)); ++ ++ vma->fence_alignment = i915_gem_fence_alignment(vm->i915, vma->size, ++ i915_gem_object_get_tiling(obj), ++ i915_gem_object_get_stride(obj)); ++ GEM_BUG_ON(!is_power_of_2(vma->fence_alignment)); ++ ++ vma->flags |= I915_VMA_GGTT; ++ } ++ ++ spin_lock(&obj->vma.lock); ++ ++ rb = NULL; ++ p = &obj->vma.tree.rb_node; ++ while (*p) { ++ struct i915_vma *pos; ++ long cmp; ++ ++ rb = *p; ++ pos = rb_entry(rb, struct i915_vma, obj_node); ++ ++ /* ++ * If the view already exists in the tree, another thread ++ * already created a matching vma, so return the older instance ++ * and dispose of ours. ++ */ ++ cmp = i915_vma_compare(pos, vm, view); ++ if (cmp == 0) { ++ spin_unlock(&obj->vma.lock); ++ i915_vma_free(vma); ++ return pos; ++ } ++ ++ if (cmp < 0) ++ p = &rb->rb_right; ++ else ++ p = &rb->rb_left; ++ } ++ rb_link_node(&vma->obj_node, rb, p); ++ rb_insert_color(&vma->obj_node, &obj->vma.tree); ++ ++ if (i915_vma_is_ggtt(vma)) ++ /* ++ * We put the GGTT vma at the start of the vma-list, followed ++ * by the ppGGTT vma. This allows us to break early when ++ * iterating over only the GGTT vma for an object, see ++ * for_each_ggtt_vma() ++ */ ++ list_add(&vma->obj_link, &obj->vma.list); ++ else ++ list_add_tail(&vma->obj_link, &obj->vma.list); ++ ++ spin_unlock(&obj->vma.lock); ++ ++ mutex_lock(&vm->mutex); ++ list_add(&vma->vm_link, &vm->unbound_list); ++ mutex_unlock(&vm->mutex); ++ ++ return vma; ++ ++err_vma: ++ i915_vma_free(vma); ++ return ERR_PTR(-E2BIG); ++} ++ ++static struct i915_vma * ++vma_lookup(struct drm_i915_gem_object *obj, ++ struct i915_address_space *vm, ++ const struct i915_ggtt_view *view) ++{ ++ struct rb_node *rb; ++ ++ rb = obj->vma.tree.rb_node; ++ while (rb) { ++ struct i915_vma *vma = rb_entry(rb, struct i915_vma, obj_node); ++ long cmp; ++ ++ cmp = i915_vma_compare(vma, vm, view); ++ if (cmp == 0) ++ return vma; ++ ++ if (cmp < 0) ++ rb = rb->rb_right; ++ else ++ rb = rb->rb_left; ++ } ++ ++ return NULL; ++} ++ ++/** ++ * i915_vma_instance - return the singleton instance of the VMA ++ * @obj: parent &struct drm_i915_gem_object to be mapped ++ * @vm: address space in which the mapping is located ++ * @view: additional mapping requirements ++ * ++ * i915_vma_instance() looks up an existing VMA of the @obj in the @vm with ++ * the same @view characteristics. If a match is not found, one is created. ++ * Once created, the VMA is kept until either the object is freed, or the ++ * address space is closed. ++ * ++ * Must be called with struct_mutex held. ++ * ++ * Returns the vma, or an error pointer. ++ */ ++struct i915_vma * ++i915_vma_instance(struct drm_i915_gem_object *obj, ++ struct i915_address_space *vm, ++ const struct i915_ggtt_view *view) ++{ ++ struct i915_vma *vma; ++ ++ GEM_BUG_ON(view && !i915_is_ggtt(vm)); ++ GEM_BUG_ON(vm->closed); ++ ++ spin_lock(&obj->vma.lock); ++ vma = vma_lookup(obj, vm, view); ++ spin_unlock(&obj->vma.lock); ++ ++ /* vma_create() will resolve the race if another creates the vma */ ++ if (unlikely(!vma)) ++ vma = vma_create(obj, vm, view); ++ ++ GEM_BUG_ON(!IS_ERR(vma) && i915_vma_compare(vma, vm, view)); ++ return vma; ++} ++ ++/** ++ * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space. ++ * @vma: VMA to map ++ * @cache_level: mapping cache level ++ * @flags: flags like global or local mapping ++ * ++ * DMA addresses are taken from the scatter-gather table of this object (or of ++ * this VMA in case of non-default GGTT views) and PTE entries set up. ++ * Note that DMA addresses are also the only part of the SG table we care about. ++ */ ++int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level, ++ u32 flags) ++{ ++ u32 bind_flags; ++ u32 vma_flags; ++ int ret; ++ ++ GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); ++ GEM_BUG_ON(vma->size > vma->node.size); ++ ++ if (GEM_DEBUG_WARN_ON(range_overflows(vma->node.start, ++ vma->node.size, ++ vma->vm->total))) ++ return -ENODEV; ++ ++ if (GEM_DEBUG_WARN_ON(!flags)) ++ return -EINVAL; ++ ++ bind_flags = 0; ++ if (flags & PIN_GLOBAL) ++ bind_flags |= I915_VMA_GLOBAL_BIND; ++ if (flags & PIN_USER) ++ bind_flags |= I915_VMA_LOCAL_BIND; ++ ++ vma_flags = vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND); ++ if (flags & PIN_UPDATE) ++ bind_flags |= vma_flags; ++ else ++ bind_flags &= ~vma_flags; ++ if (bind_flags == 0) ++ return 0; ++ ++ GEM_BUG_ON(!vma->pages); ++ ++ trace_i915_vma_bind(vma, bind_flags); ++ ret = vma->ops->bind_vma(vma, cache_level, bind_flags); ++ if (ret) ++ return ret; ++ ++ vma->flags |= bind_flags; ++ return 0; ++} ++ ++void __iomem *i915_vma_pin_iomap(struct i915_vma *vma) ++{ ++ void __iomem *ptr; ++ int err; ++ ++ /* Access through the GTT requires the device to be awake. */ ++ assert_rpm_wakelock_held(vma->vm->i915); ++ ++ lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); ++ if (WARN_ON(!i915_vma_is_map_and_fenceable(vma))) { ++ err = -ENODEV; ++ goto err; ++ } ++ ++ GEM_BUG_ON(!i915_vma_is_ggtt(vma)); ++ GEM_BUG_ON((vma->flags & I915_VMA_GLOBAL_BIND) == 0); ++ ++ ptr = vma->iomap; ++ if (ptr == NULL) { ++ ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->iomap, ++ vma->node.start, ++ vma->node.size); ++ if (ptr == NULL) { ++ err = -ENOMEM; ++ goto err; ++ } ++ ++ vma->iomap = ptr; ++ } ++ ++ __i915_vma_pin(vma); ++ ++ err = i915_vma_pin_fence(vma); ++ if (err) ++ goto err_unpin; ++ ++ i915_vma_set_ggtt_write(vma); ++ return ptr; ++ ++err_unpin: ++ __i915_vma_unpin(vma); ++err: ++ return IO_ERR_PTR(err); ++} ++ ++void i915_vma_flush_writes(struct i915_vma *vma) ++{ ++ if (!i915_vma_has_ggtt_write(vma)) ++ return; ++ ++ i915_gem_flush_ggtt_writes(vma->vm->i915); ++ ++ i915_vma_unset_ggtt_write(vma); ++} ++ ++void i915_vma_unpin_iomap(struct i915_vma *vma) ++{ ++ lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); ++ ++ GEM_BUG_ON(vma->iomap == NULL); ++ ++ i915_vma_flush_writes(vma); ++ ++ i915_vma_unpin_fence(vma); ++ i915_vma_unpin(vma); ++} ++ ++void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags) ++{ ++ struct i915_vma *vma; ++ struct drm_i915_gem_object *obj; ++ ++ vma = fetch_and_zero(p_vma); ++ if (!vma) ++ return; ++ ++ obj = vma->obj; ++ GEM_BUG_ON(!obj); ++ ++ i915_vma_unpin(vma); ++ i915_vma_close(vma); ++ ++ if (flags & I915_VMA_RELEASE_MAP) ++ i915_gem_object_unpin_map(obj); ++ ++ __i915_gem_object_release_unless_active(obj); ++} ++ ++bool i915_vma_misplaced(const struct i915_vma *vma, ++ u64 size, u64 alignment, u64 flags) ++{ ++ if (!drm_mm_node_allocated(&vma->node)) ++ return false; ++ ++ if (vma->node.size < size) ++ return true; ++ ++ GEM_BUG_ON(alignment && !is_power_of_2(alignment)); ++ if (alignment && !IS_ALIGNED(vma->node.start, alignment)) ++ return true; ++ ++ if (flags & PIN_MAPPABLE && !i915_vma_is_map_and_fenceable(vma)) ++ return true; ++ ++ if (flags & PIN_OFFSET_BIAS && ++ vma->node.start < (flags & PIN_OFFSET_MASK)) ++ return true; ++ ++ if (flags & PIN_OFFSET_FIXED && ++ vma->node.start != (flags & PIN_OFFSET_MASK)) ++ return true; ++ ++ return false; ++} ++ ++void __i915_vma_set_map_and_fenceable(struct i915_vma *vma) ++{ ++ bool mappable, fenceable; ++ ++ GEM_BUG_ON(!i915_vma_is_ggtt(vma)); ++ GEM_BUG_ON(!vma->fence_size); ++ ++ /* ++ * Explicitly disable for rotated VMA since the display does not ++ * need the fence and the VMA is not accessible to other users. ++ */ ++ if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED) ++ return; ++ ++ fenceable = (vma->node.size >= vma->fence_size && ++ IS_ALIGNED(vma->node.start, vma->fence_alignment)); ++ ++ mappable = vma->node.start + vma->fence_size <= i915_vm_to_ggtt(vma->vm)->mappable_end; ++ ++ if (mappable && fenceable) ++ vma->flags |= I915_VMA_CAN_FENCE; ++ else ++ vma->flags &= ~I915_VMA_CAN_FENCE; ++} ++ ++static bool color_differs(struct drm_mm_node *node, unsigned long color) ++{ ++ return node->allocated && node->color != color; ++} ++ ++bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long cache_level) ++{ ++ struct drm_mm_node *node = &vma->node; ++ struct drm_mm_node *other; ++ ++ /* ++ * On some machines we have to be careful when putting differing types ++ * of snoopable memory together to avoid the prefetcher crossing memory ++ * domains and dying. During vm initialisation, we decide whether or not ++ * these constraints apply and set the drm_mm.color_adjust ++ * appropriately. ++ */ ++ if (vma->vm->mm.color_adjust == NULL) ++ return true; ++ ++ /* Only valid to be called on an already inserted vma */ ++ GEM_BUG_ON(!drm_mm_node_allocated(node)); ++ GEM_BUG_ON(list_empty(&node->node_list)); ++ ++ other = list_prev_entry(node, node_list); ++ if (color_differs(other, cache_level) && !drm_mm_hole_follows(other)) ++ return false; ++ ++ other = list_next_entry(node, node_list); ++ if (color_differs(other, cache_level) && !drm_mm_hole_follows(node)) ++ return false; ++ ++ return true; ++} ++ ++static void assert_bind_count(const struct drm_i915_gem_object *obj) ++{ ++ /* ++ * Combine the assertion that the object is bound and that we have ++ * pinned its pages. But we should never have bound the object ++ * more than we have pinned its pages. (For complete accuracy, we ++ * assume that no else is pinning the pages, but as a rough assertion ++ * that we will not run into problems later, this will do!) ++ */ ++ GEM_BUG_ON(atomic_read(&obj->mm.pages_pin_count) < obj->bind_count); ++} ++ ++/** ++ * i915_vma_insert - finds a slot for the vma in its address space ++ * @vma: the vma ++ * @size: requested size in bytes (can be larger than the VMA) ++ * @alignment: required alignment ++ * @flags: mask of PIN_* flags to use ++ * ++ * First we try to allocate some free space that meets the requirements for ++ * the VMA. Failiing that, if the flags permit, it will evict an old VMA, ++ * preferrably the oldest idle entry to make room for the new VMA. ++ * ++ * Returns: ++ * 0 on success, negative error code otherwise. ++ */ ++static int ++i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) ++{ ++ struct drm_i915_private *dev_priv = vma->vm->i915; ++ unsigned int cache_level; ++ u64 start, end; ++ int ret; ++ ++ GEM_BUG_ON(i915_vma_is_closed(vma)); ++ GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND)); ++ GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); ++ ++ size = max(size, vma->size); ++ alignment = max(alignment, vma->display_alignment); ++ if (flags & PIN_MAPPABLE) { ++ size = max_t(typeof(size), size, vma->fence_size); ++ alignment = max_t(typeof(alignment), ++ alignment, vma->fence_alignment); ++ } ++ ++ GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); ++ GEM_BUG_ON(!IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT)); ++ GEM_BUG_ON(!is_power_of_2(alignment)); ++ ++ start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0; ++ GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE)); ++ ++ end = vma->vm->total; ++ if (flags & PIN_MAPPABLE) ++ end = min_t(u64, end, dev_priv->ggtt.mappable_end); ++ if (flags & PIN_ZONE_4G) ++ end = min_t(u64, end, (1ULL << 32) - I915_GTT_PAGE_SIZE); ++ GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE)); ++ ++ /* If binding the object/GGTT view requires more space than the entire ++ * aperture has, reject it early before evicting everything in a vain ++ * attempt to find space. ++ */ ++ if (size > end) { ++ DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n", ++ size, flags & PIN_MAPPABLE ? "mappable" : "total", ++ end); ++ return -ENOSPC; ++ } ++ ++ if (vma->obj) { ++ ret = i915_gem_object_pin_pages(vma->obj); ++ if (ret) ++ return ret; ++ ++ cache_level = vma->obj->cache_level; ++ } else { ++ cache_level = 0; ++ } ++ ++ GEM_BUG_ON(vma->pages); ++ ++ ret = vma->ops->set_pages(vma); ++ if (ret) ++ goto err_unpin; ++ ++ if (flags & PIN_OFFSET_FIXED) { ++ u64 offset = flags & PIN_OFFSET_MASK; ++ if (!IS_ALIGNED(offset, alignment) || ++ range_overflows(offset, size, end)) { ++ ret = -EINVAL; ++ goto err_clear; ++ } ++ ++ ret = i915_gem_gtt_reserve(vma->vm, &vma->node, ++ size, offset, cache_level, ++ flags); ++ if (ret) ++ goto err_clear; ++ } else { ++ /* ++ * We only support huge gtt pages through the 48b PPGTT, ++ * however we also don't want to force any alignment for ++ * objects which need to be tightly packed into the low 32bits. ++ * ++ * Note that we assume that GGTT are limited to 4GiB for the ++ * forseeable future. See also i915_ggtt_offset(). ++ */ ++ if (upper_32_bits(end - 1) && ++ vma->page_sizes.sg > I915_GTT_PAGE_SIZE) { ++ /* ++ * We can't mix 64K and 4K PTEs in the same page-table ++ * (2M block), and so to avoid the ugliness and ++ * complexity of coloring we opt for just aligning 64K ++ * objects to 2M. ++ */ ++ u64 page_alignment = ++ rounddown_pow_of_two(vma->page_sizes.sg | ++ I915_GTT_PAGE_SIZE_2M); ++ ++ /* ++ * Check we don't expand for the limited Global GTT ++ * (mappable aperture is even more precious!). This ++ * also checks that we exclude the aliasing-ppgtt. ++ */ ++ GEM_BUG_ON(i915_vma_is_ggtt(vma)); ++ ++ alignment = max(alignment, page_alignment); ++ ++ if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) ++ size = round_up(size, I915_GTT_PAGE_SIZE_2M); ++ } ++ ++ ret = i915_gem_gtt_insert(vma->vm, &vma->node, ++ size, alignment, cache_level, ++ start, end, flags); ++ if (ret) ++ goto err_clear; ++ ++ GEM_BUG_ON(vma->node.start < start); ++ GEM_BUG_ON(vma->node.start + vma->node.size > end); ++ } ++ GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); ++ GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, cache_level)); ++ ++ mutex_lock(&vma->vm->mutex); ++ list_move_tail(&vma->vm_link, &vma->vm->bound_list); ++ mutex_unlock(&vma->vm->mutex); ++ ++ if (vma->obj) { ++ struct drm_i915_gem_object *obj = vma->obj; ++ ++ spin_lock(&dev_priv->mm.obj_lock); ++ list_move_tail(&obj->mm.link, &dev_priv->mm.bound_list); ++ obj->bind_count++; ++ spin_unlock(&dev_priv->mm.obj_lock); ++ ++ assert_bind_count(obj); ++ } ++ ++ return 0; ++ ++err_clear: ++ vma->ops->clear_pages(vma); ++err_unpin: ++ if (vma->obj) ++ i915_gem_object_unpin_pages(vma->obj); ++ return ret; ++} ++ ++static void ++i915_vma_remove(struct i915_vma *vma) ++{ ++ struct drm_i915_private *i915 = vma->vm->i915; ++ ++ GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); ++ GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND)); ++ ++ vma->ops->clear_pages(vma); ++ ++ mutex_lock(&vma->vm->mutex); ++ drm_mm_remove_node(&vma->node); ++ list_move_tail(&vma->vm_link, &vma->vm->unbound_list); ++ mutex_unlock(&vma->vm->mutex); ++ ++ /* ++ * Since the unbound list is global, only move to that list if ++ * no more VMAs exist. ++ */ ++ if (vma->obj) { ++ struct drm_i915_gem_object *obj = vma->obj; ++ ++ spin_lock(&i915->mm.obj_lock); ++ if (--obj->bind_count == 0) ++ list_move_tail(&obj->mm.link, &i915->mm.unbound_list); ++ spin_unlock(&i915->mm.obj_lock); ++ ++ /* ++ * And finally now the object is completely decoupled from this ++ * vma, we can drop its hold on the backing storage and allow ++ * it to be reaped by the shrinker. ++ */ ++ i915_gem_object_unpin_pages(obj); ++ assert_bind_count(obj); ++ } ++} ++ ++int __i915_vma_do_pin(struct i915_vma *vma, ++ u64 size, u64 alignment, u64 flags) ++{ ++ const unsigned int bound = vma->flags; ++ int ret; ++ ++ lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); ++ GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0); ++ GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma)); ++ ++ if (WARN_ON(bound & I915_VMA_PIN_OVERFLOW)) { ++ ret = -EBUSY; ++ goto err_unpin; ++ } ++ ++ if ((bound & I915_VMA_BIND_MASK) == 0) { ++ ret = i915_vma_insert(vma, size, alignment, flags); ++ if (ret) ++ goto err_unpin; ++ } ++ GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); ++ ++ ret = i915_vma_bind(vma, vma->obj ? vma->obj->cache_level : 0, flags); ++ if (ret) ++ goto err_remove; ++ ++ GEM_BUG_ON((vma->flags & I915_VMA_BIND_MASK) == 0); ++ ++ if ((bound ^ vma->flags) & I915_VMA_GLOBAL_BIND) ++ __i915_vma_set_map_and_fenceable(vma); ++ ++ GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags)); ++ return 0; ++ ++err_remove: ++ if ((bound & I915_VMA_BIND_MASK) == 0) { ++ i915_vma_remove(vma); ++ GEM_BUG_ON(vma->pages); ++ GEM_BUG_ON(vma->flags & I915_VMA_BIND_MASK); ++ } ++err_unpin: ++ __i915_vma_unpin(vma); ++ return ret; ++} ++ ++void i915_vma_close(struct i915_vma *vma) ++{ ++ lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); ++ ++ GEM_BUG_ON(i915_vma_is_closed(vma)); ++ vma->flags |= I915_VMA_CLOSED; ++ ++ /* ++ * We defer actually closing, unbinding and destroying the VMA until ++ * the next idle point, or if the object is freed in the meantime. By ++ * postponing the unbind, we allow for it to be resurrected by the ++ * client, avoiding the work required to rebind the VMA. This is ++ * advantageous for DRI, where the client/server pass objects ++ * between themselves, temporarily opening a local VMA to the ++ * object, and then closing it again. The same object is then reused ++ * on the next frame (or two, depending on the depth of the swap queue) ++ * causing us to rebind the VMA once more. This ends up being a lot ++ * of wasted work for the steady state. ++ */ ++ list_add_tail(&vma->closed_link, &vma->vm->i915->gt.closed_vma); ++} ++ ++void i915_vma_reopen(struct i915_vma *vma) ++{ ++ lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); ++ ++ if (vma->flags & I915_VMA_CLOSED) { ++ vma->flags &= ~I915_VMA_CLOSED; ++ list_del(&vma->closed_link); ++ } ++} ++ ++static void __i915_vma_destroy(struct i915_vma *vma) ++{ ++ GEM_BUG_ON(vma->node.allocated); ++ GEM_BUG_ON(vma->fence); ++ ++ GEM_BUG_ON(i915_active_request_isset(&vma->last_fence)); ++ ++ mutex_lock(&vma->vm->mutex); ++ list_del(&vma->vm_link); ++ mutex_unlock(&vma->vm->mutex); ++ ++ if (vma->obj) { ++ struct drm_i915_gem_object *obj = vma->obj; ++ ++ spin_lock(&obj->vma.lock); ++ list_del(&vma->obj_link); ++ rb_erase(&vma->obj_node, &vma->obj->vma.tree); ++ spin_unlock(&obj->vma.lock); ++ } ++ ++ i915_active_fini(&vma->active); ++ ++ i915_vma_free(vma); ++} ++ ++void i915_vma_destroy(struct i915_vma *vma) ++{ ++ lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); ++ ++ GEM_BUG_ON(i915_vma_is_active(vma)); ++ GEM_BUG_ON(i915_vma_is_pinned(vma)); ++ ++ if (i915_vma_is_closed(vma)) ++ list_del(&vma->closed_link); ++ ++ WARN_ON(i915_vma_unbind(vma)); ++ __i915_vma_destroy(vma); ++} ++ ++void i915_vma_parked(struct drm_i915_private *i915) ++{ ++ struct i915_vma *vma, *next; ++ ++ list_for_each_entry_safe(vma, next, &i915->gt.closed_vma, closed_link) { ++ GEM_BUG_ON(!i915_vma_is_closed(vma)); ++ i915_vma_destroy(vma); ++ } ++ ++ GEM_BUG_ON(!list_empty(&i915->gt.closed_vma)); ++} ++ ++static void __i915_vma_iounmap(struct i915_vma *vma) ++{ ++ GEM_BUG_ON(i915_vma_is_pinned(vma)); ++ ++ if (vma->iomap == NULL) ++ return; ++ ++ io_mapping_unmap(vma->iomap); ++ vma->iomap = NULL; ++} ++ ++void i915_vma_revoke_mmap(struct i915_vma *vma) ++{ ++ struct drm_vma_offset_node *node = &vma->obj->base.vma_node; ++ u64 vma_offset; ++ ++ lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); ++ ++ if (!i915_vma_has_userfault(vma)) ++ return; ++ ++ GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma)); ++ GEM_BUG_ON(!vma->obj->userfault_count); ++ ++ vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT; ++ unmap_mapping_range(vma->vm->i915->drm.anon_inode->i_mapping, ++ drm_vma_node_offset_addr(node) + vma_offset, ++ vma->size, ++ 1); ++ ++ i915_vma_unset_userfault(vma); ++ if (!--vma->obj->userfault_count) ++ list_del(&vma->obj->userfault_link); ++} ++ ++static void export_fence(struct i915_vma *vma, ++ struct i915_request *rq, ++ unsigned int flags) ++{ ++ struct reservation_object *resv = vma->resv; ++ ++ /* ++ * Ignore errors from failing to allocate the new fence, we can't ++ * handle an error right now. Worst case should be missed ++ * synchronisation leading to rendering corruption. ++ */ ++ reservation_object_lock(resv, NULL); ++ if (flags & EXEC_OBJECT_WRITE) ++ reservation_object_add_excl_fence(resv, &rq->fence); ++ else if (reservation_object_reserve_shared(resv, 1) == 0) ++ reservation_object_add_shared_fence(resv, &rq->fence); ++ reservation_object_unlock(resv); ++} ++ ++int i915_vma_move_to_active(struct i915_vma *vma, ++ struct i915_request *rq, ++ unsigned int flags) ++{ ++ struct drm_i915_gem_object *obj = vma->obj; ++ ++ lockdep_assert_held(&rq->i915->drm.struct_mutex); ++ GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); ++ ++ /* ++ * Add a reference if we're newly entering the active list. ++ * The order in which we add operations to the retirement queue is ++ * vital here: mark_active adds to the start of the callback list, ++ * such that subsequent callbacks are called first. Therefore we ++ * add the active reference first and queue for it to be dropped ++ * *last*. ++ */ ++ if (!vma->active.count) ++ obj->active_count++; ++ ++ if (unlikely(i915_active_ref(&vma->active, rq->fence.context, rq))) { ++ if (!vma->active.count) ++ obj->active_count--; ++ return -ENOMEM; ++ } ++ ++ GEM_BUG_ON(!i915_vma_is_active(vma)); ++ GEM_BUG_ON(!obj->active_count); ++ ++ obj->write_domain = 0; ++ if (flags & EXEC_OBJECT_WRITE) { ++ obj->write_domain = I915_GEM_DOMAIN_RENDER; ++ ++ if (intel_fb_obj_invalidate(obj, ORIGIN_CS)) ++ __i915_active_request_set(&obj->frontbuffer_write, rq); ++ ++ obj->read_domains = 0; ++ } ++ obj->read_domains |= I915_GEM_GPU_DOMAINS; ++ ++ if (flags & EXEC_OBJECT_NEEDS_FENCE) ++ __i915_active_request_set(&vma->last_fence, rq); ++ ++ export_fence(vma, rq, flags); ++ return 0; ++} ++ ++int i915_vma_unbind(struct i915_vma *vma) ++{ ++ int ret; ++ ++ lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); ++ ++ /* ++ * First wait upon any activity as retiring the request may ++ * have side-effects such as unpinning or even unbinding this vma. ++ */ ++ might_sleep(); ++ if (i915_vma_is_active(vma)) { ++ /* ++ * When a closed VMA is retired, it is unbound - eek. ++ * In order to prevent it from being recursively closed, ++ * take a pin on the vma so that the second unbind is ++ * aborted. ++ * ++ * Even more scary is that the retire callback may free ++ * the object (last active vma). To prevent the explosion ++ * we defer the actual object free to a worker that can ++ * only proceed once it acquires the struct_mutex (which ++ * we currently hold, therefore it cannot free this object ++ * before we are finished). ++ */ ++ __i915_vma_pin(vma); ++ ++ ret = i915_active_wait(&vma->active); ++ if (ret) ++ goto unpin; ++ ++ ret = i915_active_request_retire(&vma->last_fence, ++ &vma->vm->i915->drm.struct_mutex); ++unpin: ++ __i915_vma_unpin(vma); ++ if (ret) ++ return ret; ++ } ++ GEM_BUG_ON(i915_vma_is_active(vma)); ++ ++ if (i915_vma_is_pinned(vma)) { ++ vma_print_allocator(vma, "is pinned"); ++ return -EBUSY; ++ } ++ ++ if (!drm_mm_node_allocated(&vma->node)) ++ return 0; ++ ++ if (i915_vma_is_map_and_fenceable(vma)) { ++ /* ++ * Check that we have flushed all writes through the GGTT ++ * before the unbind, other due to non-strict nature of those ++ * indirect writes they may end up referencing the GGTT PTE ++ * after the unbind. ++ */ ++ i915_vma_flush_writes(vma); ++ GEM_BUG_ON(i915_vma_has_ggtt_write(vma)); ++ ++ /* release the fence reg _after_ flushing */ ++ ret = i915_vma_put_fence(vma); ++ if (ret) ++ return ret; ++ ++ /* Force a pagefault for domain tracking on next user access */ ++ i915_vma_revoke_mmap(vma); ++ ++ __i915_vma_iounmap(vma); ++ vma->flags &= ~I915_VMA_CAN_FENCE; ++ } ++ GEM_BUG_ON(vma->fence); ++ GEM_BUG_ON(i915_vma_has_userfault(vma)); ++ ++ if (likely(!vma->vm->closed)) { ++ trace_i915_vma_unbind(vma); ++ vma->ops->unbind_vma(vma); ++ } ++ vma->flags &= ~(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND); ++ ++ i915_vma_remove(vma); ++ ++ return 0; ++} ++ ++#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) ++#include "selftests/i915_vma.c" ++#endif ++ ++static void i915_global_vma_shrink(void) ++{ ++ kmem_cache_shrink(global.slab_vmas); ++} ++ ++static void i915_global_vma_exit(void) ++{ ++ kmem_cache_destroy(global.slab_vmas); ++} ++ ++static struct i915_global_vma global = { { ++ .shrink = i915_global_vma_shrink, ++ .exit = i915_global_vma_exit, ++} }; ++ ++int __init i915_global_vma_init(void) ++{ ++ global.slab_vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN); ++ if (!global.slab_vmas) ++ return -ENOMEM; ++ ++ i915_global_register(&global.base); ++ return 0; ++} +diff --git a/drivers/gpu/drm/i915_legacy/i915_vma.h b/drivers/gpu/drm/i915_legacy/i915_vma.h +new file mode 100644 +index 000000000000..6eab70953a57 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/i915_vma.h +@@ -0,0 +1,446 @@ ++/* ++ * Copyright © 2016 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#ifndef __I915_VMA_H__ ++#define __I915_VMA_H__ ++ ++#include ++#include ++ ++#include ++ ++#include "i915_gem_gtt.h" ++#include "i915_gem_fence_reg.h" ++#include "i915_gem_object.h" ++ ++#include "i915_active.h" ++#include "i915_request.h" ++ ++enum i915_cache_level; ++ ++/** ++ * A VMA represents a GEM BO that is bound into an address space. Therefore, a ++ * VMA's presence cannot be guaranteed before binding, or after unbinding the ++ * object into/from the address space. ++ * ++ * To make things as simple as possible (ie. no refcounting), a VMA's lifetime ++ * will always be <= an objects lifetime. So object refcounting should cover us. ++ */ ++struct i915_vma { ++ struct drm_mm_node node; ++ struct drm_i915_gem_object *obj; ++ struct i915_address_space *vm; ++ const struct i915_vma_ops *ops; ++ struct drm_i915_fence_reg *fence; ++ struct reservation_object *resv; /** Alias of obj->resv */ ++ struct sg_table *pages; ++ void __iomem *iomap; ++ void *private; /* owned by creator */ ++ u64 size; ++ u64 display_alignment; ++ struct i915_page_sizes page_sizes; ++ ++ u32 fence_size; ++ u32 fence_alignment; ++ ++ /** ++ * Count of the number of times this vma has been opened by different ++ * handles (but same file) for execbuf, i.e. the number of aliases ++ * that exist in the ctx->handle_vmas LUT for this vma. ++ */ ++ unsigned int open_count; ++ unsigned long flags; ++ /** ++ * How many users have pinned this object in GTT space. ++ * ++ * This is a tightly bound, fairly small number of users, so we ++ * stuff inside the flags field so that we can both check for overflow ++ * and detect a no-op i915_vma_pin() in a single check, while also ++ * pinning the vma. ++ * ++ * The worst case display setup would have the same vma pinned for ++ * use on each plane on each crtc, while also building the next atomic ++ * state and holding a pin for the length of the cleanup queue. In the ++ * future, the flip queue may be increased from 1. ++ * Estimated worst case: 3 [qlen] * 4 [max crtcs] * 7 [max planes] = 84 ++ * ++ * For GEM, the number of concurrent users for pwrite/pread is ++ * unbounded. For execbuffer, it is currently one but will in future ++ * be extended to allow multiple clients to pin vma concurrently. ++ * ++ * We also use suballocated pages, with each suballocation claiming ++ * its own pin on the shared vma. At present, this is limited to ++ * exclusive cachelines of a single page, so a maximum of 64 possible ++ * users. ++ */ ++#define I915_VMA_PIN_MASK 0xff ++#define I915_VMA_PIN_OVERFLOW BIT(8) ++ ++ /** Flags and address space this VMA is bound to */ ++#define I915_VMA_GLOBAL_BIND BIT(9) ++#define I915_VMA_LOCAL_BIND BIT(10) ++#define I915_VMA_BIND_MASK (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND | I915_VMA_PIN_OVERFLOW) ++ ++#define I915_VMA_GGTT BIT(11) ++#define I915_VMA_CAN_FENCE BIT(12) ++#define I915_VMA_CLOSED BIT(13) ++#define I915_VMA_USERFAULT_BIT 14 ++#define I915_VMA_USERFAULT BIT(I915_VMA_USERFAULT_BIT) ++#define I915_VMA_GGTT_WRITE BIT(15) ++ ++ struct i915_active active; ++ struct i915_active_request last_fence; ++ ++ /** ++ * Support different GGTT views into the same object. ++ * This means there can be multiple VMA mappings per object and per VM. ++ * i915_ggtt_view_type is used to distinguish between those entries. ++ * The default one of zero (I915_GGTT_VIEW_NORMAL) is default and also ++ * assumed in GEM functions which take no ggtt view parameter. ++ */ ++ struct i915_ggtt_view ggtt_view; ++ ++ /** This object's place on the active/inactive lists */ ++ struct list_head vm_link; ++ ++ struct list_head obj_link; /* Link in the object's VMA list */ ++ struct rb_node obj_node; ++ struct hlist_node obj_hash; ++ ++ /** This vma's place in the execbuf reservation list */ ++ struct list_head exec_link; ++ struct list_head reloc_link; ++ ++ /** This vma's place in the eviction list */ ++ struct list_head evict_link; ++ ++ struct list_head closed_link; ++ ++ /** ++ * Used for performing relocations during execbuffer insertion. ++ */ ++ unsigned int *exec_flags; ++ struct hlist_node exec_node; ++ u32 exec_handle; ++}; ++ ++struct i915_vma * ++i915_vma_instance(struct drm_i915_gem_object *obj, ++ struct i915_address_space *vm, ++ const struct i915_ggtt_view *view); ++ ++void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags); ++#define I915_VMA_RELEASE_MAP BIT(0) ++ ++static inline bool i915_vma_is_active(const struct i915_vma *vma) ++{ ++ return !i915_active_is_idle(&vma->active); ++} ++ ++int __must_check i915_vma_move_to_active(struct i915_vma *vma, ++ struct i915_request *rq, ++ unsigned int flags); ++ ++static inline bool i915_vma_is_ggtt(const struct i915_vma *vma) ++{ ++ return vma->flags & I915_VMA_GGTT; ++} ++ ++static inline bool i915_vma_has_ggtt_write(const struct i915_vma *vma) ++{ ++ return vma->flags & I915_VMA_GGTT_WRITE; ++} ++ ++static inline void i915_vma_set_ggtt_write(struct i915_vma *vma) ++{ ++ GEM_BUG_ON(!i915_vma_is_ggtt(vma)); ++ vma->flags |= I915_VMA_GGTT_WRITE; ++} ++ ++static inline void i915_vma_unset_ggtt_write(struct i915_vma *vma) ++{ ++ vma->flags &= ~I915_VMA_GGTT_WRITE; ++} ++ ++void i915_vma_flush_writes(struct i915_vma *vma); ++ ++static inline bool i915_vma_is_map_and_fenceable(const struct i915_vma *vma) ++{ ++ return vma->flags & I915_VMA_CAN_FENCE; ++} ++ ++static inline bool i915_vma_is_closed(const struct i915_vma *vma) ++{ ++ return vma->flags & I915_VMA_CLOSED; ++} ++ ++static inline bool i915_vma_set_userfault(struct i915_vma *vma) ++{ ++ GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma)); ++ return __test_and_set_bit(I915_VMA_USERFAULT_BIT, &vma->flags); ++} ++ ++static inline void i915_vma_unset_userfault(struct i915_vma *vma) ++{ ++ return __clear_bit(I915_VMA_USERFAULT_BIT, &vma->flags); ++} ++ ++static inline bool i915_vma_has_userfault(const struct i915_vma *vma) ++{ ++ return test_bit(I915_VMA_USERFAULT_BIT, &vma->flags); ++} ++ ++static inline u32 i915_ggtt_offset(const struct i915_vma *vma) ++{ ++ GEM_BUG_ON(!i915_vma_is_ggtt(vma)); ++ GEM_BUG_ON(!vma->node.allocated); ++ GEM_BUG_ON(upper_32_bits(vma->node.start)); ++ GEM_BUG_ON(upper_32_bits(vma->node.start + vma->node.size - 1)); ++ return lower_32_bits(vma->node.start); ++} ++ ++static inline u32 i915_ggtt_pin_bias(struct i915_vma *vma) ++{ ++ return i915_vm_to_ggtt(vma->vm)->pin_bias; ++} ++ ++static inline struct i915_vma *i915_vma_get(struct i915_vma *vma) ++{ ++ i915_gem_object_get(vma->obj); ++ return vma; ++} ++ ++static inline void i915_vma_put(struct i915_vma *vma) ++{ ++ i915_gem_object_put(vma->obj); ++} ++ ++static __always_inline ptrdiff_t ptrdiff(const void *a, const void *b) ++{ ++ return a - b; ++} ++ ++static inline long ++i915_vma_compare(struct i915_vma *vma, ++ struct i915_address_space *vm, ++ const struct i915_ggtt_view *view) ++{ ++ ptrdiff_t cmp; ++ ++ GEM_BUG_ON(view && !i915_is_ggtt(vm)); ++ ++ cmp = ptrdiff(vma->vm, vm); ++ if (cmp) ++ return cmp; ++ ++ BUILD_BUG_ON(I915_GGTT_VIEW_NORMAL != 0); ++ cmp = vma->ggtt_view.type; ++ if (!view) ++ return cmp; ++ ++ cmp -= view->type; ++ if (cmp) ++ return cmp; ++ ++ assert_i915_gem_gtt_types(); ++ ++ /* ggtt_view.type also encodes its size so that we both distinguish ++ * different views using it as a "type" and also use a compact (no ++ * accessing of uninitialised padding bytes) memcmp without storing ++ * an extra parameter or adding more code. ++ * ++ * To ensure that the memcmp is valid for all branches of the union, ++ * even though the code looks like it is just comparing one branch, ++ * we assert above that all branches have the same address, and that ++ * each branch has a unique type/size. ++ */ ++ BUILD_BUG_ON(I915_GGTT_VIEW_NORMAL >= I915_GGTT_VIEW_PARTIAL); ++ BUILD_BUG_ON(I915_GGTT_VIEW_PARTIAL >= I915_GGTT_VIEW_ROTATED); ++ BUILD_BUG_ON(offsetof(typeof(*view), rotated) != ++ offsetof(typeof(*view), partial)); ++ return memcmp(&vma->ggtt_view.partial, &view->partial, view->type); ++} ++ ++int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level, ++ u32 flags); ++bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long cache_level); ++bool i915_vma_misplaced(const struct i915_vma *vma, ++ u64 size, u64 alignment, u64 flags); ++void __i915_vma_set_map_and_fenceable(struct i915_vma *vma); ++void i915_vma_revoke_mmap(struct i915_vma *vma); ++int __must_check i915_vma_unbind(struct i915_vma *vma); ++void i915_vma_unlink_ctx(struct i915_vma *vma); ++void i915_vma_close(struct i915_vma *vma); ++void i915_vma_reopen(struct i915_vma *vma); ++void i915_vma_destroy(struct i915_vma *vma); ++ ++int __i915_vma_do_pin(struct i915_vma *vma, ++ u64 size, u64 alignment, u64 flags); ++static inline int __must_check ++i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) ++{ ++ BUILD_BUG_ON(PIN_MBZ != I915_VMA_PIN_OVERFLOW); ++ BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND); ++ BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND); ++ ++ /* Pin early to prevent the shrinker/eviction logic from destroying ++ * our vma as we insert and bind. ++ */ ++ if (likely(((++vma->flags ^ flags) & I915_VMA_BIND_MASK) == 0)) { ++ GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); ++ GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags)); ++ return 0; ++ } ++ ++ return __i915_vma_do_pin(vma, size, alignment, flags); ++} ++ ++static inline int i915_vma_pin_count(const struct i915_vma *vma) ++{ ++ return vma->flags & I915_VMA_PIN_MASK; ++} ++ ++static inline bool i915_vma_is_pinned(const struct i915_vma *vma) ++{ ++ return i915_vma_pin_count(vma); ++} ++ ++static inline void __i915_vma_pin(struct i915_vma *vma) ++{ ++ vma->flags++; ++ GEM_BUG_ON(vma->flags & I915_VMA_PIN_OVERFLOW); ++} ++ ++static inline void __i915_vma_unpin(struct i915_vma *vma) ++{ ++ vma->flags--; ++} ++ ++static inline void i915_vma_unpin(struct i915_vma *vma) ++{ ++ GEM_BUG_ON(!i915_vma_is_pinned(vma)); ++ GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); ++ __i915_vma_unpin(vma); ++} ++ ++static inline bool i915_vma_is_bound(const struct i915_vma *vma, ++ unsigned int where) ++{ ++ return vma->flags & where; ++} ++ ++/** ++ * i915_vma_pin_iomap - calls ioremap_wc to map the GGTT VMA via the aperture ++ * @vma: VMA to iomap ++ * ++ * The passed in VMA has to be pinned in the global GTT mappable region. ++ * An extra pinning of the VMA is acquired for the return iomapping, ++ * the caller must call i915_vma_unpin_iomap to relinquish the pinning ++ * after the iomapping is no longer required. ++ * ++ * Callers must hold the struct_mutex. ++ * ++ * Returns a valid iomapped pointer or ERR_PTR. ++ */ ++void __iomem *i915_vma_pin_iomap(struct i915_vma *vma); ++#define IO_ERR_PTR(x) ((void __iomem *)ERR_PTR(x)) ++ ++/** ++ * i915_vma_unpin_iomap - unpins the mapping returned from i915_vma_iomap ++ * @vma: VMA to unpin ++ * ++ * Unpins the previously iomapped VMA from i915_vma_pin_iomap(). ++ * ++ * Callers must hold the struct_mutex. This function is only valid to be ++ * called on a VMA previously iomapped by the caller with i915_vma_pin_iomap(). ++ */ ++void i915_vma_unpin_iomap(struct i915_vma *vma); ++ ++static inline struct page *i915_vma_first_page(struct i915_vma *vma) ++{ ++ GEM_BUG_ON(!vma->pages); ++ return sg_page(vma->pages->sgl); ++} ++ ++/** ++ * i915_vma_pin_fence - pin fencing state ++ * @vma: vma to pin fencing for ++ * ++ * This pins the fencing state (whether tiled or untiled) to make sure the ++ * vma (and its object) is ready to be used as a scanout target. Fencing ++ * status must be synchronize first by calling i915_vma_get_fence(): ++ * ++ * The resulting fence pin reference must be released again with ++ * i915_vma_unpin_fence(). ++ * ++ * Returns: ++ * ++ * True if the vma has a fence, false otherwise. ++ */ ++int i915_vma_pin_fence(struct i915_vma *vma); ++int __must_check i915_vma_put_fence(struct i915_vma *vma); ++ ++static inline void __i915_vma_unpin_fence(struct i915_vma *vma) ++{ ++ GEM_BUG_ON(vma->fence->pin_count <= 0); ++ vma->fence->pin_count--; ++} ++ ++/** ++ * i915_vma_unpin_fence - unpin fencing state ++ * @vma: vma to unpin fencing for ++ * ++ * This releases the fence pin reference acquired through ++ * i915_vma_pin_fence. It will handle both objects with and without an ++ * attached fence correctly, callers do not need to distinguish this. ++ */ ++static inline void ++i915_vma_unpin_fence(struct i915_vma *vma) ++{ ++ /* lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); */ ++ if (vma->fence) ++ __i915_vma_unpin_fence(vma); ++} ++ ++void i915_vma_parked(struct drm_i915_private *i915); ++ ++#define for_each_until(cond) if (cond) break; else ++ ++/** ++ * for_each_ggtt_vma - Iterate over the GGTT VMA belonging to an object. ++ * @V: the #i915_vma iterator ++ * @OBJ: the #drm_i915_gem_object ++ * ++ * GGTT VMA are placed at the being of the object's vma_list, see ++ * vma_create(), so we can stop our walk as soon as we see a ppgtt VMA, ++ * or the list is empty ofc. ++ */ ++#define for_each_ggtt_vma(V, OBJ) \ ++ list_for_each_entry(V, &(OBJ)->vma.list, obj_link) \ ++ for_each_until(!i915_vma_is_ggtt(V)) ++ ++struct i915_vma *i915_vma_alloc(void); ++void i915_vma_free(struct i915_vma *vma); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/icl_dsi.c b/drivers/gpu/drm/i915_legacy/icl_dsi.c +new file mode 100644 +index 000000000000..9d962ea1e635 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/icl_dsi.c +@@ -0,0 +1,1464 @@ ++/* ++ * Copyright © 2018 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER ++ * DEALINGS IN THE SOFTWARE. ++ * ++ * Authors: ++ * Madhav Chauhan ++ * Jani Nikula ++ */ ++ ++#include ++#include ++ ++#include "intel_connector.h" ++#include "intel_ddi.h" ++#include "intel_dsi.h" ++#include "intel_panel.h" ++ ++static inline int header_credits_available(struct drm_i915_private *dev_priv, ++ enum transcoder dsi_trans) ++{ ++ return (I915_READ(DSI_CMD_TXCTL(dsi_trans)) & FREE_HEADER_CREDIT_MASK) ++ >> FREE_HEADER_CREDIT_SHIFT; ++} ++ ++static inline int payload_credits_available(struct drm_i915_private *dev_priv, ++ enum transcoder dsi_trans) ++{ ++ return (I915_READ(DSI_CMD_TXCTL(dsi_trans)) & FREE_PLOAD_CREDIT_MASK) ++ >> FREE_PLOAD_CREDIT_SHIFT; ++} ++ ++static void wait_for_header_credits(struct drm_i915_private *dev_priv, ++ enum transcoder dsi_trans) ++{ ++ if (wait_for_us(header_credits_available(dev_priv, dsi_trans) >= ++ MAX_HEADER_CREDIT, 100)) ++ DRM_ERROR("DSI header credits not released\n"); ++} ++ ++static void wait_for_payload_credits(struct drm_i915_private *dev_priv, ++ enum transcoder dsi_trans) ++{ ++ if (wait_for_us(payload_credits_available(dev_priv, dsi_trans) >= ++ MAX_PLOAD_CREDIT, 100)) ++ DRM_ERROR("DSI payload credits not released\n"); ++} ++ ++static enum transcoder dsi_port_to_transcoder(enum port port) ++{ ++ if (port == PORT_A) ++ return TRANSCODER_DSI_0; ++ else ++ return TRANSCODER_DSI_1; ++} ++ ++static void wait_for_cmds_dispatched_to_panel(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ struct mipi_dsi_device *dsi; ++ enum port port; ++ enum transcoder dsi_trans; ++ int ret; ++ ++ /* wait for header/payload credits to be released */ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ dsi_trans = dsi_port_to_transcoder(port); ++ wait_for_header_credits(dev_priv, dsi_trans); ++ wait_for_payload_credits(dev_priv, dsi_trans); ++ } ++ ++ /* send nop DCS command */ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ dsi = intel_dsi->dsi_hosts[port]->device; ++ dsi->mode_flags |= MIPI_DSI_MODE_LPM; ++ dsi->channel = 0; ++ ret = mipi_dsi_dcs_nop(dsi); ++ if (ret < 0) ++ DRM_ERROR("error sending DCS NOP command\n"); ++ } ++ ++ /* wait for header credits to be released */ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ dsi_trans = dsi_port_to_transcoder(port); ++ wait_for_header_credits(dev_priv, dsi_trans); ++ } ++ ++ /* wait for LP TX in progress bit to be cleared */ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ dsi_trans = dsi_port_to_transcoder(port); ++ if (wait_for_us(!(I915_READ(DSI_LP_MSG(dsi_trans)) & ++ LPTX_IN_PROGRESS), 20)) ++ DRM_ERROR("LPTX bit not cleared\n"); ++ } ++} ++ ++static bool add_payld_to_queue(struct intel_dsi_host *host, const u8 *data, ++ u32 len) ++{ ++ struct intel_dsi *intel_dsi = host->intel_dsi; ++ struct drm_i915_private *dev_priv = to_i915(intel_dsi->base.base.dev); ++ enum transcoder dsi_trans = dsi_port_to_transcoder(host->port); ++ int free_credits; ++ int i, j; ++ ++ for (i = 0; i < len; i += 4) { ++ u32 tmp = 0; ++ ++ free_credits = payload_credits_available(dev_priv, dsi_trans); ++ if (free_credits < 1) { ++ DRM_ERROR("Payload credit not available\n"); ++ return false; ++ } ++ ++ for (j = 0; j < min_t(u32, len - i, 4); j++) ++ tmp |= *data++ << 8 * j; ++ ++ I915_WRITE(DSI_CMD_TXPYLD(dsi_trans), tmp); ++ } ++ ++ return true; ++} ++ ++static int dsi_send_pkt_hdr(struct intel_dsi_host *host, ++ struct mipi_dsi_packet pkt, bool enable_lpdt) ++{ ++ struct intel_dsi *intel_dsi = host->intel_dsi; ++ struct drm_i915_private *dev_priv = to_i915(intel_dsi->base.base.dev); ++ enum transcoder dsi_trans = dsi_port_to_transcoder(host->port); ++ u32 tmp; ++ int free_credits; ++ ++ /* check if header credit available */ ++ free_credits = header_credits_available(dev_priv, dsi_trans); ++ if (free_credits < 1) { ++ DRM_ERROR("send pkt header failed, not enough hdr credits\n"); ++ return -1; ++ } ++ ++ tmp = I915_READ(DSI_CMD_TXHDR(dsi_trans)); ++ ++ if (pkt.payload) ++ tmp |= PAYLOAD_PRESENT; ++ else ++ tmp &= ~PAYLOAD_PRESENT; ++ ++ tmp &= ~VBLANK_FENCE; ++ ++ if (enable_lpdt) ++ tmp |= LP_DATA_TRANSFER; ++ ++ tmp &= ~(PARAM_WC_MASK | VC_MASK | DT_MASK); ++ tmp |= ((pkt.header[0] & VC_MASK) << VC_SHIFT); ++ tmp |= ((pkt.header[0] & DT_MASK) << DT_SHIFT); ++ tmp |= (pkt.header[1] << PARAM_WC_LOWER_SHIFT); ++ tmp |= (pkt.header[2] << PARAM_WC_UPPER_SHIFT); ++ I915_WRITE(DSI_CMD_TXHDR(dsi_trans), tmp); ++ ++ return 0; ++} ++ ++static int dsi_send_pkt_payld(struct intel_dsi_host *host, ++ struct mipi_dsi_packet pkt) ++{ ++ /* payload queue can accept *256 bytes*, check limit */ ++ if (pkt.payload_length > MAX_PLOAD_CREDIT * 4) { ++ DRM_ERROR("payload size exceeds max queue limit\n"); ++ return -1; ++ } ++ ++ /* load data into command payload queue */ ++ if (!add_payld_to_queue(host, pkt.payload, ++ pkt.payload_length)) { ++ DRM_ERROR("adding payload to queue failed\n"); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++static void dsi_program_swing_and_deemphasis(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ enum port port; ++ u32 tmp; ++ int lane; ++ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ ++ /* ++ * Program voltage swing and pre-emphasis level values as per ++ * table in BSPEC under DDI buffer programing ++ */ ++ tmp = I915_READ(ICL_PORT_TX_DW5_LN0(port)); ++ tmp &= ~(SCALING_MODE_SEL_MASK | RTERM_SELECT_MASK); ++ tmp |= SCALING_MODE_SEL(0x2); ++ tmp |= TAP2_DISABLE | TAP3_DISABLE; ++ tmp |= RTERM_SELECT(0x6); ++ I915_WRITE(ICL_PORT_TX_DW5_GRP(port), tmp); ++ ++ tmp = I915_READ(ICL_PORT_TX_DW5_AUX(port)); ++ tmp &= ~(SCALING_MODE_SEL_MASK | RTERM_SELECT_MASK); ++ tmp |= SCALING_MODE_SEL(0x2); ++ tmp |= TAP2_DISABLE | TAP3_DISABLE; ++ tmp |= RTERM_SELECT(0x6); ++ I915_WRITE(ICL_PORT_TX_DW5_AUX(port), tmp); ++ ++ tmp = I915_READ(ICL_PORT_TX_DW2_LN0(port)); ++ tmp &= ~(SWING_SEL_LOWER_MASK | SWING_SEL_UPPER_MASK | ++ RCOMP_SCALAR_MASK); ++ tmp |= SWING_SEL_UPPER(0x2); ++ tmp |= SWING_SEL_LOWER(0x2); ++ tmp |= RCOMP_SCALAR(0x98); ++ I915_WRITE(ICL_PORT_TX_DW2_GRP(port), tmp); ++ ++ tmp = I915_READ(ICL_PORT_TX_DW2_AUX(port)); ++ tmp &= ~(SWING_SEL_LOWER_MASK | SWING_SEL_UPPER_MASK | ++ RCOMP_SCALAR_MASK); ++ tmp |= SWING_SEL_UPPER(0x2); ++ tmp |= SWING_SEL_LOWER(0x2); ++ tmp |= RCOMP_SCALAR(0x98); ++ I915_WRITE(ICL_PORT_TX_DW2_AUX(port), tmp); ++ ++ tmp = I915_READ(ICL_PORT_TX_DW4_AUX(port)); ++ tmp &= ~(POST_CURSOR_1_MASK | POST_CURSOR_2_MASK | ++ CURSOR_COEFF_MASK); ++ tmp |= POST_CURSOR_1(0x0); ++ tmp |= POST_CURSOR_2(0x0); ++ tmp |= CURSOR_COEFF(0x3f); ++ I915_WRITE(ICL_PORT_TX_DW4_AUX(port), tmp); ++ ++ for (lane = 0; lane <= 3; lane++) { ++ /* Bspec: must not use GRP register for write */ ++ tmp = I915_READ(ICL_PORT_TX_DW4_LN(lane, port)); ++ tmp &= ~(POST_CURSOR_1_MASK | POST_CURSOR_2_MASK | ++ CURSOR_COEFF_MASK); ++ tmp |= POST_CURSOR_1(0x0); ++ tmp |= POST_CURSOR_2(0x0); ++ tmp |= CURSOR_COEFF(0x3f); ++ I915_WRITE(ICL_PORT_TX_DW4_LN(lane, port), tmp); ++ } ++ } ++} ++ ++static void configure_dual_link_mode(struct intel_encoder *encoder, ++ const struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ u32 dss_ctl1; ++ ++ dss_ctl1 = I915_READ(DSS_CTL1); ++ dss_ctl1 |= SPLITTER_ENABLE; ++ dss_ctl1 &= ~OVERLAP_PIXELS_MASK; ++ dss_ctl1 |= OVERLAP_PIXELS(intel_dsi->pixel_overlap); ++ ++ if (intel_dsi->dual_link == DSI_DUAL_LINK_FRONT_BACK) { ++ const struct drm_display_mode *adjusted_mode = ++ &pipe_config->base.adjusted_mode; ++ u32 dss_ctl2; ++ u16 hactive = adjusted_mode->crtc_hdisplay; ++ u16 dl_buffer_depth; ++ ++ dss_ctl1 &= ~DUAL_LINK_MODE_INTERLEAVE; ++ dl_buffer_depth = hactive / 2 + intel_dsi->pixel_overlap; ++ ++ if (dl_buffer_depth > MAX_DL_BUFFER_TARGET_DEPTH) ++ DRM_ERROR("DL buffer depth exceed max value\n"); ++ ++ dss_ctl1 &= ~LEFT_DL_BUF_TARGET_DEPTH_MASK; ++ dss_ctl1 |= LEFT_DL_BUF_TARGET_DEPTH(dl_buffer_depth); ++ dss_ctl2 = I915_READ(DSS_CTL2); ++ dss_ctl2 &= ~RIGHT_DL_BUF_TARGET_DEPTH_MASK; ++ dss_ctl2 |= RIGHT_DL_BUF_TARGET_DEPTH(dl_buffer_depth); ++ I915_WRITE(DSS_CTL2, dss_ctl2); ++ } else { ++ /* Interleave */ ++ dss_ctl1 |= DUAL_LINK_MODE_INTERLEAVE; ++ } ++ ++ I915_WRITE(DSS_CTL1, dss_ctl1); ++} ++ ++static void gen11_dsi_program_esc_clk_div(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ enum port port; ++ u32 bpp = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format); ++ u32 afe_clk_khz; /* 8X Clock */ ++ u32 esc_clk_div_m; ++ ++ afe_clk_khz = DIV_ROUND_CLOSEST(intel_dsi->pclk * bpp, ++ intel_dsi->lane_count); ++ ++ esc_clk_div_m = DIV_ROUND_UP(afe_clk_khz, DSI_MAX_ESC_CLK); ++ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ I915_WRITE(ICL_DSI_ESC_CLK_DIV(port), ++ esc_clk_div_m & ICL_ESC_CLK_DIV_MASK); ++ POSTING_READ(ICL_DSI_ESC_CLK_DIV(port)); ++ } ++ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ I915_WRITE(ICL_DPHY_ESC_CLK_DIV(port), ++ esc_clk_div_m & ICL_ESC_CLK_DIV_MASK); ++ POSTING_READ(ICL_DPHY_ESC_CLK_DIV(port)); ++ } ++} ++ ++static void get_dsi_io_power_domains(struct drm_i915_private *dev_priv, ++ struct intel_dsi *intel_dsi) ++{ ++ enum port port; ++ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ WARN_ON(intel_dsi->io_wakeref[port]); ++ intel_dsi->io_wakeref[port] = ++ intel_display_power_get(dev_priv, ++ port == PORT_A ? ++ POWER_DOMAIN_PORT_DDI_A_IO : ++ POWER_DOMAIN_PORT_DDI_B_IO); ++ } ++} ++ ++static void gen11_dsi_enable_io_power(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ enum port port; ++ u32 tmp; ++ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ tmp = I915_READ(ICL_DSI_IO_MODECTL(port)); ++ tmp |= COMBO_PHY_MODE_DSI; ++ I915_WRITE(ICL_DSI_IO_MODECTL(port), tmp); ++ } ++ ++ get_dsi_io_power_domains(dev_priv, intel_dsi); ++} ++ ++static void gen11_dsi_power_up_lanes(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ enum port port; ++ u32 tmp; ++ u32 lane_mask; ++ ++ switch (intel_dsi->lane_count) { ++ case 1: ++ lane_mask = PWR_DOWN_LN_3_1_0; ++ break; ++ case 2: ++ lane_mask = PWR_DOWN_LN_3_1; ++ break; ++ case 3: ++ lane_mask = PWR_DOWN_LN_3; ++ break; ++ case 4: ++ default: ++ lane_mask = PWR_UP_ALL_LANES; ++ break; ++ } ++ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ tmp = I915_READ(ICL_PORT_CL_DW10(port)); ++ tmp &= ~PWR_DOWN_LN_MASK; ++ I915_WRITE(ICL_PORT_CL_DW10(port), tmp | lane_mask); ++ } ++} ++ ++static void gen11_dsi_config_phy_lanes_sequence(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ enum port port; ++ u32 tmp; ++ int lane; ++ ++ /* Step 4b(i) set loadgen select for transmit and aux lanes */ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ tmp = I915_READ(ICL_PORT_TX_DW4_AUX(port)); ++ tmp &= ~LOADGEN_SELECT; ++ I915_WRITE(ICL_PORT_TX_DW4_AUX(port), tmp); ++ for (lane = 0; lane <= 3; lane++) { ++ tmp = I915_READ(ICL_PORT_TX_DW4_LN(lane, port)); ++ tmp &= ~LOADGEN_SELECT; ++ if (lane != 2) ++ tmp |= LOADGEN_SELECT; ++ I915_WRITE(ICL_PORT_TX_DW4_LN(lane, port), tmp); ++ } ++ } ++ ++ /* Step 4b(ii) set latency optimization for transmit and aux lanes */ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ tmp = I915_READ(ICL_PORT_TX_DW2_AUX(port)); ++ tmp &= ~FRC_LATENCY_OPTIM_MASK; ++ tmp |= FRC_LATENCY_OPTIM_VAL(0x5); ++ I915_WRITE(ICL_PORT_TX_DW2_AUX(port), tmp); ++ tmp = I915_READ(ICL_PORT_TX_DW2_LN0(port)); ++ tmp &= ~FRC_LATENCY_OPTIM_MASK; ++ tmp |= FRC_LATENCY_OPTIM_VAL(0x5); ++ I915_WRITE(ICL_PORT_TX_DW2_GRP(port), tmp); ++ } ++ ++} ++ ++static void gen11_dsi_voltage_swing_program_seq(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ u32 tmp; ++ enum port port; ++ ++ /* clear common keeper enable bit */ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ tmp = I915_READ(ICL_PORT_PCS_DW1_LN0(port)); ++ tmp &= ~COMMON_KEEPER_EN; ++ I915_WRITE(ICL_PORT_PCS_DW1_GRP(port), tmp); ++ tmp = I915_READ(ICL_PORT_PCS_DW1_AUX(port)); ++ tmp &= ~COMMON_KEEPER_EN; ++ I915_WRITE(ICL_PORT_PCS_DW1_AUX(port), tmp); ++ } ++ ++ /* ++ * Set SUS Clock Config bitfield to 11b ++ * Note: loadgen select program is done ++ * as part of lane phy sequence configuration ++ */ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ tmp = I915_READ(ICL_PORT_CL_DW5(port)); ++ tmp |= SUS_CLOCK_CONFIG; ++ I915_WRITE(ICL_PORT_CL_DW5(port), tmp); ++ } ++ ++ /* Clear training enable to change swing values */ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ tmp = I915_READ(ICL_PORT_TX_DW5_LN0(port)); ++ tmp &= ~TX_TRAINING_EN; ++ I915_WRITE(ICL_PORT_TX_DW5_GRP(port), tmp); ++ tmp = I915_READ(ICL_PORT_TX_DW5_AUX(port)); ++ tmp &= ~TX_TRAINING_EN; ++ I915_WRITE(ICL_PORT_TX_DW5_AUX(port), tmp); ++ } ++ ++ /* Program swing and de-emphasis */ ++ dsi_program_swing_and_deemphasis(encoder); ++ ++ /* Set training enable to trigger update */ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ tmp = I915_READ(ICL_PORT_TX_DW5_LN0(port)); ++ tmp |= TX_TRAINING_EN; ++ I915_WRITE(ICL_PORT_TX_DW5_GRP(port), tmp); ++ tmp = I915_READ(ICL_PORT_TX_DW5_AUX(port)); ++ tmp |= TX_TRAINING_EN; ++ I915_WRITE(ICL_PORT_TX_DW5_AUX(port), tmp); ++ } ++} ++ ++static void gen11_dsi_enable_ddi_buffer(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ u32 tmp; ++ enum port port; ++ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ tmp = I915_READ(DDI_BUF_CTL(port)); ++ tmp |= DDI_BUF_CTL_ENABLE; ++ I915_WRITE(DDI_BUF_CTL(port), tmp); ++ ++ if (wait_for_us(!(I915_READ(DDI_BUF_CTL(port)) & ++ DDI_BUF_IS_IDLE), ++ 500)) ++ DRM_ERROR("DDI port:%c buffer idle\n", port_name(port)); ++ } ++} ++ ++static void gen11_dsi_setup_dphy_timings(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ u32 tmp; ++ enum port port; ++ ++ /* Program T-INIT master registers */ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ tmp = I915_READ(ICL_DSI_T_INIT_MASTER(port)); ++ tmp &= ~MASTER_INIT_TIMER_MASK; ++ tmp |= intel_dsi->init_count; ++ I915_WRITE(ICL_DSI_T_INIT_MASTER(port), tmp); ++ } ++ ++ /* Program DPHY clock lanes timings */ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ I915_WRITE(DPHY_CLK_TIMING_PARAM(port), intel_dsi->dphy_reg); ++ ++ /* shadow register inside display core */ ++ I915_WRITE(DSI_CLK_TIMING_PARAM(port), intel_dsi->dphy_reg); ++ } ++ ++ /* Program DPHY data lanes timings */ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ I915_WRITE(DPHY_DATA_TIMING_PARAM(port), ++ intel_dsi->dphy_data_lane_reg); ++ ++ /* shadow register inside display core */ ++ I915_WRITE(DSI_DATA_TIMING_PARAM(port), ++ intel_dsi->dphy_data_lane_reg); ++ } ++ ++ /* ++ * If DSI link operating at or below an 800 MHz, ++ * TA_SURE should be override and programmed to ++ * a value '0' inside TA_PARAM_REGISTERS otherwise ++ * leave all fields at HW default values. ++ */ ++ if (intel_dsi_bitrate(intel_dsi) <= 800000) { ++ for_each_dsi_port(port, intel_dsi->ports) { ++ tmp = I915_READ(DPHY_TA_TIMING_PARAM(port)); ++ tmp &= ~TA_SURE_MASK; ++ tmp |= TA_SURE_OVERRIDE | TA_SURE(0); ++ I915_WRITE(DPHY_TA_TIMING_PARAM(port), tmp); ++ ++ /* shadow register inside display core */ ++ tmp = I915_READ(DSI_TA_TIMING_PARAM(port)); ++ tmp &= ~TA_SURE_MASK; ++ tmp |= TA_SURE_OVERRIDE | TA_SURE(0); ++ I915_WRITE(DSI_TA_TIMING_PARAM(port), tmp); ++ } ++ } ++} ++ ++static void gen11_dsi_gate_clocks(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ u32 tmp; ++ enum port port; ++ ++ mutex_lock(&dev_priv->dpll_lock); ++ tmp = I915_READ(DPCLKA_CFGCR0_ICL); ++ for_each_dsi_port(port, intel_dsi->ports) { ++ tmp |= DPCLKA_CFGCR0_DDI_CLK_OFF(port); ++ } ++ ++ I915_WRITE(DPCLKA_CFGCR0_ICL, tmp); ++ mutex_unlock(&dev_priv->dpll_lock); ++} ++ ++static void gen11_dsi_ungate_clocks(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ u32 tmp; ++ enum port port; ++ ++ mutex_lock(&dev_priv->dpll_lock); ++ tmp = I915_READ(DPCLKA_CFGCR0_ICL); ++ for_each_dsi_port(port, intel_dsi->ports) { ++ tmp &= ~DPCLKA_CFGCR0_DDI_CLK_OFF(port); ++ } ++ ++ I915_WRITE(DPCLKA_CFGCR0_ICL, tmp); ++ mutex_unlock(&dev_priv->dpll_lock); ++} ++ ++static void gen11_dsi_map_pll(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ struct intel_shared_dpll *pll = crtc_state->shared_dpll; ++ enum port port; ++ u32 val; ++ ++ mutex_lock(&dev_priv->dpll_lock); ++ ++ val = I915_READ(DPCLKA_CFGCR0_ICL); ++ for_each_dsi_port(port, intel_dsi->ports) { ++ val &= ~DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port); ++ val |= DPCLKA_CFGCR0_DDI_CLK_SEL(pll->info->id, port); ++ } ++ I915_WRITE(DPCLKA_CFGCR0_ICL, val); ++ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ val &= ~DPCLKA_CFGCR0_DDI_CLK_OFF(port); ++ } ++ I915_WRITE(DPCLKA_CFGCR0_ICL, val); ++ ++ POSTING_READ(DPCLKA_CFGCR0_ICL); ++ ++ mutex_unlock(&dev_priv->dpll_lock); ++} ++ ++static void ++gen11_dsi_configure_transcoder(struct intel_encoder *encoder, ++ const struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ struct intel_crtc *intel_crtc = to_intel_crtc(pipe_config->base.crtc); ++ enum pipe pipe = intel_crtc->pipe; ++ u32 tmp; ++ enum port port; ++ enum transcoder dsi_trans; ++ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ dsi_trans = dsi_port_to_transcoder(port); ++ tmp = I915_READ(DSI_TRANS_FUNC_CONF(dsi_trans)); ++ ++ if (intel_dsi->eotp_pkt) ++ tmp &= ~EOTP_DISABLED; ++ else ++ tmp |= EOTP_DISABLED; ++ ++ /* enable link calibration if freq > 1.5Gbps */ ++ if (intel_dsi_bitrate(intel_dsi) >= 1500 * 1000) { ++ tmp &= ~LINK_CALIBRATION_MASK; ++ tmp |= CALIBRATION_ENABLED_INITIAL_ONLY; ++ } ++ ++ /* configure continuous clock */ ++ tmp &= ~CONTINUOUS_CLK_MASK; ++ if (intel_dsi->clock_stop) ++ tmp |= CLK_ENTER_LP_AFTER_DATA; ++ else ++ tmp |= CLK_HS_CONTINUOUS; ++ ++ /* configure buffer threshold limit to minimum */ ++ tmp &= ~PIX_BUF_THRESHOLD_MASK; ++ tmp |= PIX_BUF_THRESHOLD_1_4; ++ ++ /* set virtual channel to '0' */ ++ tmp &= ~PIX_VIRT_CHAN_MASK; ++ tmp |= PIX_VIRT_CHAN(0); ++ ++ /* program BGR transmission */ ++ if (intel_dsi->bgr_enabled) ++ tmp |= BGR_TRANSMISSION; ++ ++ /* select pixel format */ ++ tmp &= ~PIX_FMT_MASK; ++ switch (intel_dsi->pixel_format) { ++ default: ++ MISSING_CASE(intel_dsi->pixel_format); ++ /* fallthrough */ ++ case MIPI_DSI_FMT_RGB565: ++ tmp |= PIX_FMT_RGB565; ++ break; ++ case MIPI_DSI_FMT_RGB666_PACKED: ++ tmp |= PIX_FMT_RGB666_PACKED; ++ break; ++ case MIPI_DSI_FMT_RGB666: ++ tmp |= PIX_FMT_RGB666_LOOSE; ++ break; ++ case MIPI_DSI_FMT_RGB888: ++ tmp |= PIX_FMT_RGB888; ++ break; ++ } ++ ++ /* program DSI operation mode */ ++ if (is_vid_mode(intel_dsi)) { ++ tmp &= ~OP_MODE_MASK; ++ switch (intel_dsi->video_mode_format) { ++ default: ++ MISSING_CASE(intel_dsi->video_mode_format); ++ /* fallthrough */ ++ case VIDEO_MODE_NON_BURST_WITH_SYNC_EVENTS: ++ tmp |= VIDEO_MODE_SYNC_EVENT; ++ break; ++ case VIDEO_MODE_NON_BURST_WITH_SYNC_PULSE: ++ tmp |= VIDEO_MODE_SYNC_PULSE; ++ break; ++ } ++ } ++ ++ I915_WRITE(DSI_TRANS_FUNC_CONF(dsi_trans), tmp); ++ } ++ ++ /* enable port sync mode if dual link */ ++ if (intel_dsi->dual_link) { ++ for_each_dsi_port(port, intel_dsi->ports) { ++ dsi_trans = dsi_port_to_transcoder(port); ++ tmp = I915_READ(TRANS_DDI_FUNC_CTL2(dsi_trans)); ++ tmp |= PORT_SYNC_MODE_ENABLE; ++ I915_WRITE(TRANS_DDI_FUNC_CTL2(dsi_trans), tmp); ++ } ++ ++ /* configure stream splitting */ ++ configure_dual_link_mode(encoder, pipe_config); ++ } ++ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ dsi_trans = dsi_port_to_transcoder(port); ++ ++ /* select data lane width */ ++ tmp = I915_READ(TRANS_DDI_FUNC_CTL(dsi_trans)); ++ tmp &= ~DDI_PORT_WIDTH_MASK; ++ tmp |= DDI_PORT_WIDTH(intel_dsi->lane_count); ++ ++ /* select input pipe */ ++ tmp &= ~TRANS_DDI_EDP_INPUT_MASK; ++ switch (pipe) { ++ default: ++ MISSING_CASE(pipe); ++ /* fallthrough */ ++ case PIPE_A: ++ tmp |= TRANS_DDI_EDP_INPUT_A_ON; ++ break; ++ case PIPE_B: ++ tmp |= TRANS_DDI_EDP_INPUT_B_ONOFF; ++ break; ++ case PIPE_C: ++ tmp |= TRANS_DDI_EDP_INPUT_C_ONOFF; ++ break; ++ } ++ ++ /* enable DDI buffer */ ++ tmp |= TRANS_DDI_FUNC_ENABLE; ++ I915_WRITE(TRANS_DDI_FUNC_CTL(dsi_trans), tmp); ++ } ++ ++ /* wait for link ready */ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ dsi_trans = dsi_port_to_transcoder(port); ++ if (wait_for_us((I915_READ(DSI_TRANS_FUNC_CONF(dsi_trans)) & ++ LINK_READY), 2500)) ++ DRM_ERROR("DSI link not ready\n"); ++ } ++} ++ ++static void ++gen11_dsi_set_transcoder_timings(struct intel_encoder *encoder, ++ const struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ const struct drm_display_mode *adjusted_mode = ++ &pipe_config->base.adjusted_mode; ++ enum port port; ++ enum transcoder dsi_trans; ++ /* horizontal timings */ ++ u16 htotal, hactive, hsync_start, hsync_end, hsync_size; ++ u16 hfront_porch, hback_porch; ++ /* vertical timings */ ++ u16 vtotal, vactive, vsync_start, vsync_end, vsync_shift; ++ ++ hactive = adjusted_mode->crtc_hdisplay; ++ htotal = adjusted_mode->crtc_htotal; ++ hsync_start = adjusted_mode->crtc_hsync_start; ++ hsync_end = adjusted_mode->crtc_hsync_end; ++ hsync_size = hsync_end - hsync_start; ++ hfront_porch = (adjusted_mode->crtc_hsync_start - ++ adjusted_mode->crtc_hdisplay); ++ hback_porch = (adjusted_mode->crtc_htotal - ++ adjusted_mode->crtc_hsync_end); ++ vactive = adjusted_mode->crtc_vdisplay; ++ vtotal = adjusted_mode->crtc_vtotal; ++ vsync_start = adjusted_mode->crtc_vsync_start; ++ vsync_end = adjusted_mode->crtc_vsync_end; ++ vsync_shift = hsync_start - htotal / 2; ++ ++ if (intel_dsi->dual_link) { ++ hactive /= 2; ++ if (intel_dsi->dual_link == DSI_DUAL_LINK_FRONT_BACK) ++ hactive += intel_dsi->pixel_overlap; ++ htotal /= 2; ++ } ++ ++ /* minimum hactive as per bspec: 256 pixels */ ++ if (adjusted_mode->crtc_hdisplay < 256) ++ DRM_ERROR("hactive is less then 256 pixels\n"); ++ ++ /* if RGB666 format, then hactive must be multiple of 4 pixels */ ++ if (intel_dsi->pixel_format == MIPI_DSI_FMT_RGB666 && hactive % 4 != 0) ++ DRM_ERROR("hactive pixels are not multiple of 4\n"); ++ ++ /* program TRANS_HTOTAL register */ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ dsi_trans = dsi_port_to_transcoder(port); ++ I915_WRITE(HTOTAL(dsi_trans), ++ (hactive - 1) | ((htotal - 1) << 16)); ++ } ++ ++ /* TRANS_HSYNC register to be programmed only for video mode */ ++ if (intel_dsi->operation_mode == INTEL_DSI_VIDEO_MODE) { ++ if (intel_dsi->video_mode_format == ++ VIDEO_MODE_NON_BURST_WITH_SYNC_PULSE) { ++ /* BSPEC: hsync size should be atleast 16 pixels */ ++ if (hsync_size < 16) ++ DRM_ERROR("hsync size < 16 pixels\n"); ++ } ++ ++ if (hback_porch < 16) ++ DRM_ERROR("hback porch < 16 pixels\n"); ++ ++ if (intel_dsi->dual_link) { ++ hsync_start /= 2; ++ hsync_end /= 2; ++ } ++ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ dsi_trans = dsi_port_to_transcoder(port); ++ I915_WRITE(HSYNC(dsi_trans), ++ (hsync_start - 1) | ((hsync_end - 1) << 16)); ++ } ++ } ++ ++ /* program TRANS_VTOTAL register */ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ dsi_trans = dsi_port_to_transcoder(port); ++ /* ++ * FIXME: Programing this by assuming progressive mode, since ++ * non-interlaced info from VBT is not saved inside ++ * struct drm_display_mode. ++ * For interlace mode: program required pixel minus 2 ++ */ ++ I915_WRITE(VTOTAL(dsi_trans), ++ (vactive - 1) | ((vtotal - 1) << 16)); ++ } ++ ++ if (vsync_end < vsync_start || vsync_end > vtotal) ++ DRM_ERROR("Invalid vsync_end value\n"); ++ ++ if (vsync_start < vactive) ++ DRM_ERROR("vsync_start less than vactive\n"); ++ ++ /* program TRANS_VSYNC register */ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ dsi_trans = dsi_port_to_transcoder(port); ++ I915_WRITE(VSYNC(dsi_trans), ++ (vsync_start - 1) | ((vsync_end - 1) << 16)); ++ } ++ ++ /* ++ * FIXME: It has to be programmed only for interlaced ++ * modes. Put the check condition here once interlaced ++ * info available as described above. ++ * program TRANS_VSYNCSHIFT register ++ */ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ dsi_trans = dsi_port_to_transcoder(port); ++ I915_WRITE(VSYNCSHIFT(dsi_trans), vsync_shift); ++ } ++} ++ ++static void gen11_dsi_enable_transcoder(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ enum port port; ++ enum transcoder dsi_trans; ++ u32 tmp; ++ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ dsi_trans = dsi_port_to_transcoder(port); ++ tmp = I915_READ(PIPECONF(dsi_trans)); ++ tmp |= PIPECONF_ENABLE; ++ I915_WRITE(PIPECONF(dsi_trans), tmp); ++ ++ /* wait for transcoder to be enabled */ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ PIPECONF(dsi_trans), ++ I965_PIPECONF_ACTIVE, ++ I965_PIPECONF_ACTIVE, 10)) ++ DRM_ERROR("DSI transcoder not enabled\n"); ++ } ++} ++ ++static void gen11_dsi_setup_timeouts(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ enum port port; ++ enum transcoder dsi_trans; ++ u32 tmp, hs_tx_timeout, lp_rx_timeout, ta_timeout, divisor, mul; ++ ++ /* ++ * escape clock count calculation: ++ * BYTE_CLK_COUNT = TIME_NS/(8 * UI) ++ * UI (nsec) = (10^6)/Bitrate ++ * TIME_NS = (BYTE_CLK_COUNT * 8 * 10^6)/ Bitrate ++ * ESCAPE_CLK_COUNT = TIME_NS/ESC_CLK_NS ++ */ ++ divisor = intel_dsi_tlpx_ns(intel_dsi) * intel_dsi_bitrate(intel_dsi) * 1000; ++ mul = 8 * 1000000; ++ hs_tx_timeout = DIV_ROUND_UP(intel_dsi->hs_tx_timeout * mul, ++ divisor); ++ lp_rx_timeout = DIV_ROUND_UP(intel_dsi->lp_rx_timeout * mul, divisor); ++ ta_timeout = DIV_ROUND_UP(intel_dsi->turn_arnd_val * mul, divisor); ++ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ dsi_trans = dsi_port_to_transcoder(port); ++ ++ /* program hst_tx_timeout */ ++ tmp = I915_READ(DSI_HSTX_TO(dsi_trans)); ++ tmp &= ~HSTX_TIMEOUT_VALUE_MASK; ++ tmp |= HSTX_TIMEOUT_VALUE(hs_tx_timeout); ++ I915_WRITE(DSI_HSTX_TO(dsi_trans), tmp); ++ ++ /* FIXME: DSI_CALIB_TO */ ++ ++ /* program lp_rx_host timeout */ ++ tmp = I915_READ(DSI_LPRX_HOST_TO(dsi_trans)); ++ tmp &= ~LPRX_TIMEOUT_VALUE_MASK; ++ tmp |= LPRX_TIMEOUT_VALUE(lp_rx_timeout); ++ I915_WRITE(DSI_LPRX_HOST_TO(dsi_trans), tmp); ++ ++ /* FIXME: DSI_PWAIT_TO */ ++ ++ /* program turn around timeout */ ++ tmp = I915_READ(DSI_TA_TO(dsi_trans)); ++ tmp &= ~TA_TIMEOUT_VALUE_MASK; ++ tmp |= TA_TIMEOUT_VALUE(ta_timeout); ++ I915_WRITE(DSI_TA_TO(dsi_trans), tmp); ++ } ++} ++ ++static void ++gen11_dsi_enable_port_and_phy(struct intel_encoder *encoder, ++ const struct intel_crtc_state *pipe_config) ++{ ++ /* step 4a: power up all lanes of the DDI used by DSI */ ++ gen11_dsi_power_up_lanes(encoder); ++ ++ /* step 4b: configure lane sequencing of the Combo-PHY transmitters */ ++ gen11_dsi_config_phy_lanes_sequence(encoder); ++ ++ /* step 4c: configure voltage swing and skew */ ++ gen11_dsi_voltage_swing_program_seq(encoder); ++ ++ /* enable DDI buffer */ ++ gen11_dsi_enable_ddi_buffer(encoder); ++ ++ /* setup D-PHY timings */ ++ gen11_dsi_setup_dphy_timings(encoder); ++ ++ /* step 4h: setup DSI protocol timeouts */ ++ gen11_dsi_setup_timeouts(encoder); ++ ++ /* Step (4h, 4i, 4j, 4k): Configure transcoder */ ++ gen11_dsi_configure_transcoder(encoder, pipe_config); ++ ++ /* Step 4l: Gate DDI clocks */ ++ gen11_dsi_gate_clocks(encoder); ++} ++ ++static void gen11_dsi_powerup_panel(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ struct mipi_dsi_device *dsi; ++ enum port port; ++ enum transcoder dsi_trans; ++ u32 tmp; ++ int ret; ++ ++ /* set maximum return packet size */ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ dsi_trans = dsi_port_to_transcoder(port); ++ ++ /* ++ * FIXME: This uses the number of DW's currently in the payload ++ * receive queue. This is probably not what we want here. ++ */ ++ tmp = I915_READ(DSI_CMD_RXCTL(dsi_trans)); ++ tmp &= NUMBER_RX_PLOAD_DW_MASK; ++ /* multiply "Number Rx Payload DW" by 4 to get max value */ ++ tmp = tmp * 4; ++ dsi = intel_dsi->dsi_hosts[port]->device; ++ ret = mipi_dsi_set_maximum_return_packet_size(dsi, tmp); ++ if (ret < 0) ++ DRM_ERROR("error setting max return pkt size%d\n", tmp); ++ } ++ ++ /* panel power on related mipi dsi vbt sequences */ ++ intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_POWER_ON); ++ intel_dsi_msleep(intel_dsi, intel_dsi->panel_on_delay); ++ intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_DEASSERT_RESET); ++ intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_INIT_OTP); ++ intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_DISPLAY_ON); ++ ++ /* ensure all panel commands dispatched before enabling transcoder */ ++ wait_for_cmds_dispatched_to_panel(encoder); ++} ++ ++static void gen11_dsi_pre_pll_enable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *pipe_config, ++ const struct drm_connector_state *conn_state) ++{ ++ /* step2: enable IO power */ ++ gen11_dsi_enable_io_power(encoder); ++ ++ /* step3: enable DSI PLL */ ++ gen11_dsi_program_esc_clk_div(encoder); ++} ++ ++static void gen11_dsi_pre_enable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *pipe_config, ++ const struct drm_connector_state *conn_state) ++{ ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ ++ /* step3b */ ++ gen11_dsi_map_pll(encoder, pipe_config); ++ ++ /* step4: enable DSI port and DPHY */ ++ gen11_dsi_enable_port_and_phy(encoder, pipe_config); ++ ++ /* step5: program and powerup panel */ ++ gen11_dsi_powerup_panel(encoder); ++ ++ /* step6c: configure transcoder timings */ ++ gen11_dsi_set_transcoder_timings(encoder, pipe_config); ++ ++ /* step6d: enable dsi transcoder */ ++ gen11_dsi_enable_transcoder(encoder); ++ ++ /* step7: enable backlight */ ++ intel_panel_enable_backlight(pipe_config, conn_state); ++ intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_BACKLIGHT_ON); ++} ++ ++static void gen11_dsi_disable_transcoder(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ enum port port; ++ enum transcoder dsi_trans; ++ u32 tmp; ++ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ dsi_trans = dsi_port_to_transcoder(port); ++ ++ /* disable transcoder */ ++ tmp = I915_READ(PIPECONF(dsi_trans)); ++ tmp &= ~PIPECONF_ENABLE; ++ I915_WRITE(PIPECONF(dsi_trans), tmp); ++ ++ /* wait for transcoder to be disabled */ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ PIPECONF(dsi_trans), ++ I965_PIPECONF_ACTIVE, 0, 50)) ++ DRM_ERROR("DSI trancoder not disabled\n"); ++ } ++} ++ ++static void gen11_dsi_powerdown_panel(struct intel_encoder *encoder) ++{ ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ ++ intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_DISPLAY_OFF); ++ intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_ASSERT_RESET); ++ intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_POWER_OFF); ++ ++ /* ensure cmds dispatched to panel */ ++ wait_for_cmds_dispatched_to_panel(encoder); ++} ++ ++static void gen11_dsi_deconfigure_trancoder(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ enum port port; ++ enum transcoder dsi_trans; ++ u32 tmp; ++ ++ /* put dsi link in ULPS */ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ dsi_trans = dsi_port_to_transcoder(port); ++ tmp = I915_READ(DSI_LP_MSG(dsi_trans)); ++ tmp |= LINK_ENTER_ULPS; ++ tmp &= ~LINK_ULPS_TYPE_LP11; ++ I915_WRITE(DSI_LP_MSG(dsi_trans), tmp); ++ ++ if (wait_for_us((I915_READ(DSI_LP_MSG(dsi_trans)) & ++ LINK_IN_ULPS), ++ 10)) ++ DRM_ERROR("DSI link not in ULPS\n"); ++ } ++ ++ /* disable ddi function */ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ dsi_trans = dsi_port_to_transcoder(port); ++ tmp = I915_READ(TRANS_DDI_FUNC_CTL(dsi_trans)); ++ tmp &= ~TRANS_DDI_FUNC_ENABLE; ++ I915_WRITE(TRANS_DDI_FUNC_CTL(dsi_trans), tmp); ++ } ++ ++ /* disable port sync mode if dual link */ ++ if (intel_dsi->dual_link) { ++ for_each_dsi_port(port, intel_dsi->ports) { ++ dsi_trans = dsi_port_to_transcoder(port); ++ tmp = I915_READ(TRANS_DDI_FUNC_CTL2(dsi_trans)); ++ tmp &= ~PORT_SYNC_MODE_ENABLE; ++ I915_WRITE(TRANS_DDI_FUNC_CTL2(dsi_trans), tmp); ++ } ++ } ++} ++ ++static void gen11_dsi_disable_port(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ u32 tmp; ++ enum port port; ++ ++ gen11_dsi_ungate_clocks(encoder); ++ for_each_dsi_port(port, intel_dsi->ports) { ++ tmp = I915_READ(DDI_BUF_CTL(port)); ++ tmp &= ~DDI_BUF_CTL_ENABLE; ++ I915_WRITE(DDI_BUF_CTL(port), tmp); ++ ++ if (wait_for_us((I915_READ(DDI_BUF_CTL(port)) & ++ DDI_BUF_IS_IDLE), ++ 8)) ++ DRM_ERROR("DDI port:%c buffer not idle\n", ++ port_name(port)); ++ } ++ gen11_dsi_gate_clocks(encoder); ++} ++ ++static void gen11_dsi_disable_io_power(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ enum port port; ++ u32 tmp; ++ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ intel_wakeref_t wakeref; ++ ++ wakeref = fetch_and_zero(&intel_dsi->io_wakeref[port]); ++ intel_display_power_put(dev_priv, ++ port == PORT_A ? ++ POWER_DOMAIN_PORT_DDI_A_IO : ++ POWER_DOMAIN_PORT_DDI_B_IO, ++ wakeref); ++ } ++ ++ /* set mode to DDI */ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ tmp = I915_READ(ICL_DSI_IO_MODECTL(port)); ++ tmp &= ~COMBO_PHY_MODE_DSI; ++ I915_WRITE(ICL_DSI_IO_MODECTL(port), tmp); ++ } ++} ++ ++static void gen11_dsi_disable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ ++ /* step1: turn off backlight */ ++ intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_BACKLIGHT_OFF); ++ intel_panel_disable_backlight(old_conn_state); ++ ++ /* step2d,e: disable transcoder and wait */ ++ gen11_dsi_disable_transcoder(encoder); ++ ++ /* step2f,g: powerdown panel */ ++ gen11_dsi_powerdown_panel(encoder); ++ ++ /* step2h,i,j: deconfig trancoder */ ++ gen11_dsi_deconfigure_trancoder(encoder); ++ ++ /* step3: disable port */ ++ gen11_dsi_disable_port(encoder); ++ ++ /* step4: disable IO power */ ++ gen11_dsi_disable_io_power(encoder); ++} ++ ++static void gen11_dsi_get_config(struct intel_encoder *encoder, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ ++ /* FIXME: adapt icl_ddi_clock_get() for DSI and use that? */ ++ pipe_config->port_clock = ++ cnl_calc_wrpll_link(dev_priv, &pipe_config->dpll_hw_state); ++ pipe_config->base.adjusted_mode.crtc_clock = intel_dsi->pclk; ++ pipe_config->output_types |= BIT(INTEL_OUTPUT_DSI); ++} ++ ++static int gen11_dsi_compute_config(struct intel_encoder *encoder, ++ struct intel_crtc_state *pipe_config, ++ struct drm_connector_state *conn_state) ++{ ++ struct intel_dsi *intel_dsi = container_of(encoder, struct intel_dsi, ++ base); ++ struct intel_connector *intel_connector = intel_dsi->attached_connector; ++ struct intel_crtc *crtc = to_intel_crtc(pipe_config->base.crtc); ++ const struct drm_display_mode *fixed_mode = ++ intel_connector->panel.fixed_mode; ++ struct drm_display_mode *adjusted_mode = ++ &pipe_config->base.adjusted_mode; ++ ++ intel_fixed_panel_mode(fixed_mode, adjusted_mode); ++ intel_pch_panel_fitting(crtc, pipe_config, conn_state->scaling_mode); ++ ++ adjusted_mode->flags = 0; ++ ++ /* Dual link goes to trancoder DSI'0' */ ++ if (intel_dsi->ports == BIT(PORT_B)) ++ pipe_config->cpu_transcoder = TRANSCODER_DSI_1; ++ else ++ pipe_config->cpu_transcoder = TRANSCODER_DSI_0; ++ ++ pipe_config->clock_set = true; ++ pipe_config->port_clock = intel_dsi_bitrate(intel_dsi) / 5; ++ ++ return 0; ++} ++ ++static void gen11_dsi_get_power_domains(struct intel_encoder *encoder, ++ struct intel_crtc_state *crtc_state) ++{ ++ get_dsi_io_power_domains(to_i915(encoder->base.dev), ++ enc_to_intel_dsi(&encoder->base)); ++} ++ ++static bool gen11_dsi_get_hw_state(struct intel_encoder *encoder, ++ enum pipe *pipe) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); ++ enum transcoder dsi_trans; ++ intel_wakeref_t wakeref; ++ enum port port; ++ bool ret = false; ++ u32 tmp; ++ ++ wakeref = intel_display_power_get_if_enabled(dev_priv, ++ encoder->power_domain); ++ if (!wakeref) ++ return false; ++ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ dsi_trans = dsi_port_to_transcoder(port); ++ tmp = I915_READ(TRANS_DDI_FUNC_CTL(dsi_trans)); ++ switch (tmp & TRANS_DDI_EDP_INPUT_MASK) { ++ case TRANS_DDI_EDP_INPUT_A_ON: ++ *pipe = PIPE_A; ++ break; ++ case TRANS_DDI_EDP_INPUT_B_ONOFF: ++ *pipe = PIPE_B; ++ break; ++ case TRANS_DDI_EDP_INPUT_C_ONOFF: ++ *pipe = PIPE_C; ++ break; ++ default: ++ DRM_ERROR("Invalid PIPE input\n"); ++ goto out; ++ } ++ ++ tmp = I915_READ(PIPECONF(dsi_trans)); ++ ret = tmp & PIPECONF_ENABLE; ++ } ++out: ++ intel_display_power_put(dev_priv, encoder->power_domain, wakeref); ++ return ret; ++} ++ ++static void gen11_dsi_encoder_destroy(struct drm_encoder *encoder) ++{ ++ intel_encoder_destroy(encoder); ++} ++ ++static const struct drm_encoder_funcs gen11_dsi_encoder_funcs = { ++ .destroy = gen11_dsi_encoder_destroy, ++}; ++ ++static const struct drm_connector_funcs gen11_dsi_connector_funcs = { ++ .late_register = intel_connector_register, ++ .early_unregister = intel_connector_unregister, ++ .destroy = intel_connector_destroy, ++ .fill_modes = drm_helper_probe_single_connector_modes, ++ .atomic_get_property = intel_digital_connector_atomic_get_property, ++ .atomic_set_property = intel_digital_connector_atomic_set_property, ++ .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, ++ .atomic_duplicate_state = intel_digital_connector_duplicate_state, ++}; ++ ++static const struct drm_connector_helper_funcs gen11_dsi_connector_helper_funcs = { ++ .get_modes = intel_dsi_get_modes, ++ .mode_valid = intel_dsi_mode_valid, ++ .atomic_check = intel_digital_connector_atomic_check, ++}; ++ ++static int gen11_dsi_host_attach(struct mipi_dsi_host *host, ++ struct mipi_dsi_device *dsi) ++{ ++ return 0; ++} ++ ++static int gen11_dsi_host_detach(struct mipi_dsi_host *host, ++ struct mipi_dsi_device *dsi) ++{ ++ return 0; ++} ++ ++static ssize_t gen11_dsi_host_transfer(struct mipi_dsi_host *host, ++ const struct mipi_dsi_msg *msg) ++{ ++ struct intel_dsi_host *intel_dsi_host = to_intel_dsi_host(host); ++ struct mipi_dsi_packet dsi_pkt; ++ ssize_t ret; ++ bool enable_lpdt = false; ++ ++ ret = mipi_dsi_create_packet(&dsi_pkt, msg); ++ if (ret < 0) ++ return ret; ++ ++ if (msg->flags & MIPI_DSI_MSG_USE_LPM) ++ enable_lpdt = true; ++ ++ /* send packet header */ ++ ret = dsi_send_pkt_hdr(intel_dsi_host, dsi_pkt, enable_lpdt); ++ if (ret < 0) ++ return ret; ++ ++ /* only long packet contains payload */ ++ if (mipi_dsi_packet_format_is_long(msg->type)) { ++ ret = dsi_send_pkt_payld(intel_dsi_host, dsi_pkt); ++ if (ret < 0) ++ return ret; ++ } ++ ++ //TODO: add payload receive code if needed ++ ++ ret = sizeof(dsi_pkt.header) + dsi_pkt.payload_length; ++ ++ return ret; ++} ++ ++static const struct mipi_dsi_host_ops gen11_dsi_host_ops = { ++ .attach = gen11_dsi_host_attach, ++ .detach = gen11_dsi_host_detach, ++ .transfer = gen11_dsi_host_transfer, ++}; ++ ++void icl_dsi_init(struct drm_i915_private *dev_priv) ++{ ++ struct drm_device *dev = &dev_priv->drm; ++ struct intel_dsi *intel_dsi; ++ struct intel_encoder *encoder; ++ struct intel_connector *intel_connector; ++ struct drm_connector *connector; ++ struct drm_display_mode *fixed_mode; ++ enum port port; ++ ++ if (!intel_bios_is_dsi_present(dev_priv, &port)) ++ return; ++ ++ intel_dsi = kzalloc(sizeof(*intel_dsi), GFP_KERNEL); ++ if (!intel_dsi) ++ return; ++ ++ intel_connector = intel_connector_alloc(); ++ if (!intel_connector) { ++ kfree(intel_dsi); ++ return; ++ } ++ ++ encoder = &intel_dsi->base; ++ intel_dsi->attached_connector = intel_connector; ++ connector = &intel_connector->base; ++ ++ /* register DSI encoder with DRM subsystem */ ++ drm_encoder_init(dev, &encoder->base, &gen11_dsi_encoder_funcs, ++ DRM_MODE_ENCODER_DSI, "DSI %c", port_name(port)); ++ ++ encoder->pre_pll_enable = gen11_dsi_pre_pll_enable; ++ encoder->pre_enable = gen11_dsi_pre_enable; ++ encoder->disable = gen11_dsi_disable; ++ encoder->port = port; ++ encoder->get_config = gen11_dsi_get_config; ++ encoder->update_pipe = intel_panel_update_backlight; ++ encoder->compute_config = gen11_dsi_compute_config; ++ encoder->get_hw_state = gen11_dsi_get_hw_state; ++ encoder->type = INTEL_OUTPUT_DSI; ++ encoder->cloneable = 0; ++ encoder->crtc_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C); ++ encoder->power_domain = POWER_DOMAIN_PORT_DSI; ++ encoder->get_power_domains = gen11_dsi_get_power_domains; ++ ++ /* register DSI connector with DRM subsystem */ ++ drm_connector_init(dev, connector, &gen11_dsi_connector_funcs, ++ DRM_MODE_CONNECTOR_DSI); ++ drm_connector_helper_add(connector, &gen11_dsi_connector_helper_funcs); ++ connector->display_info.subpixel_order = SubPixelHorizontalRGB; ++ connector->interlace_allowed = false; ++ connector->doublescan_allowed = false; ++ intel_connector->get_hw_state = intel_connector_get_hw_state; ++ ++ /* attach connector to encoder */ ++ intel_connector_attach_encoder(intel_connector, encoder); ++ ++ mutex_lock(&dev->mode_config.mutex); ++ fixed_mode = intel_panel_vbt_fixed_mode(intel_connector); ++ mutex_unlock(&dev->mode_config.mutex); ++ ++ if (!fixed_mode) { ++ DRM_ERROR("DSI fixed mode info missing\n"); ++ goto err; ++ } ++ ++ intel_panel_init(&intel_connector->panel, fixed_mode, NULL); ++ intel_panel_setup_backlight(connector, INVALID_PIPE); ++ ++ if (dev_priv->vbt.dsi.config->dual_link) ++ intel_dsi->ports = BIT(PORT_A) | BIT(PORT_B); ++ else ++ intel_dsi->ports = BIT(port); ++ ++ intel_dsi->dcs_backlight_ports = dev_priv->vbt.dsi.bl_ports; ++ intel_dsi->dcs_cabc_ports = dev_priv->vbt.dsi.cabc_ports; ++ ++ for_each_dsi_port(port, intel_dsi->ports) { ++ struct intel_dsi_host *host; ++ ++ host = intel_dsi_host_init(intel_dsi, &gen11_dsi_host_ops, port); ++ if (!host) ++ goto err; ++ ++ intel_dsi->dsi_hosts[port] = host; ++ } ++ ++ if (!intel_dsi_vbt_init(intel_dsi, MIPI_DSI_GENERIC_PANEL_ID)) { ++ DRM_DEBUG_KMS("no device found\n"); ++ goto err; ++ } ++ ++ return; ++ ++err: ++ drm_encoder_cleanup(&encoder->base); ++ kfree(intel_dsi); ++ kfree(intel_connector); ++} +diff --git a/drivers/gpu/drm/i915_legacy/intel_acpi.c b/drivers/gpu/drm/i915_legacy/intel_acpi.c +new file mode 100644 +index 000000000000..9d142d038a7d +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_acpi.c +@@ -0,0 +1,155 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Intel ACPI functions ++ * ++ * _DSM related code stolen from nouveau_acpi.c. ++ */ ++#include ++#include ++#include "i915_drv.h" ++ ++#define INTEL_DSM_REVISION_ID 1 /* For Calpella anyway... */ ++#define INTEL_DSM_FN_PLATFORM_MUX_INFO 1 /* No args */ ++ ++static const guid_t intel_dsm_guid = ++ GUID_INIT(0x7ed873d3, 0xc2d0, 0x4e4f, ++ 0xa8, 0x54, 0x0f, 0x13, 0x17, 0xb0, 0x1c, 0x2c); ++ ++static char *intel_dsm_port_name(u8 id) ++{ ++ switch (id) { ++ case 0: ++ return "Reserved"; ++ case 1: ++ return "Analog VGA"; ++ case 2: ++ return "LVDS"; ++ case 3: ++ return "Reserved"; ++ case 4: ++ return "HDMI/DVI_B"; ++ case 5: ++ return "HDMI/DVI_C"; ++ case 6: ++ return "HDMI/DVI_D"; ++ case 7: ++ return "DisplayPort_A"; ++ case 8: ++ return "DisplayPort_B"; ++ case 9: ++ return "DisplayPort_C"; ++ case 0xa: ++ return "DisplayPort_D"; ++ case 0xb: ++ case 0xc: ++ case 0xd: ++ return "Reserved"; ++ case 0xe: ++ return "WiDi"; ++ default: ++ return "bad type"; ++ } ++} ++ ++static char *intel_dsm_mux_type(u8 type) ++{ ++ switch (type) { ++ case 0: ++ return "unknown"; ++ case 1: ++ return "No MUX, iGPU only"; ++ case 2: ++ return "No MUX, dGPU only"; ++ case 3: ++ return "MUXed between iGPU and dGPU"; ++ default: ++ return "bad type"; ++ } ++} ++ ++static void intel_dsm_platform_mux_info(acpi_handle dhandle) ++{ ++ int i; ++ union acpi_object *pkg, *connector_count; ++ ++ pkg = acpi_evaluate_dsm_typed(dhandle, &intel_dsm_guid, ++ INTEL_DSM_REVISION_ID, INTEL_DSM_FN_PLATFORM_MUX_INFO, ++ NULL, ACPI_TYPE_PACKAGE); ++ if (!pkg) { ++ DRM_DEBUG_DRIVER("failed to evaluate _DSM\n"); ++ return; ++ } ++ ++ connector_count = &pkg->package.elements[0]; ++ DRM_DEBUG_DRIVER("MUX info connectors: %lld\n", ++ (unsigned long long)connector_count->integer.value); ++ for (i = 1; i < pkg->package.count; i++) { ++ union acpi_object *obj = &pkg->package.elements[i]; ++ union acpi_object *connector_id = &obj->package.elements[0]; ++ union acpi_object *info = &obj->package.elements[1]; ++ DRM_DEBUG_DRIVER("Connector id: 0x%016llx\n", ++ (unsigned long long)connector_id->integer.value); ++ DRM_DEBUG_DRIVER(" port id: %s\n", ++ intel_dsm_port_name(info->buffer.pointer[0])); ++ DRM_DEBUG_DRIVER(" display mux info: %s\n", ++ intel_dsm_mux_type(info->buffer.pointer[1])); ++ DRM_DEBUG_DRIVER(" aux/dc mux info: %s\n", ++ intel_dsm_mux_type(info->buffer.pointer[2])); ++ DRM_DEBUG_DRIVER(" hpd mux info: %s\n", ++ intel_dsm_mux_type(info->buffer.pointer[3])); ++ } ++ ++ ACPI_FREE(pkg); ++} ++ ++static acpi_handle intel_dsm_pci_probe(struct pci_dev *pdev) ++{ ++ acpi_handle dhandle; ++ ++ dhandle = ACPI_HANDLE(&pdev->dev); ++ if (!dhandle) ++ return NULL; ++ ++ if (!acpi_check_dsm(dhandle, &intel_dsm_guid, INTEL_DSM_REVISION_ID, ++ 1 << INTEL_DSM_FN_PLATFORM_MUX_INFO)) { ++ DRM_DEBUG_KMS("no _DSM method for intel device\n"); ++ return NULL; ++ } ++ ++ intel_dsm_platform_mux_info(dhandle); ++ ++ return dhandle; ++} ++ ++static bool intel_dsm_detect(void) ++{ ++ acpi_handle dhandle = NULL; ++ char acpi_method_name[255] = { 0 }; ++ struct acpi_buffer buffer = {sizeof(acpi_method_name), acpi_method_name}; ++ struct pci_dev *pdev = NULL; ++ int vga_count = 0; ++ ++ while ((pdev = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, pdev)) != NULL) { ++ vga_count++; ++ dhandle = intel_dsm_pci_probe(pdev) ?: dhandle; ++ } ++ ++ if (vga_count == 2 && dhandle) { ++ acpi_get_name(dhandle, ACPI_FULL_PATHNAME, &buffer); ++ DRM_DEBUG_DRIVER("vga_switcheroo: detected DSM switching method %s handle\n", ++ acpi_method_name); ++ return true; ++ } ++ ++ return false; ++} ++ ++void intel_register_dsm_handler(void) ++{ ++ if (!intel_dsm_detect()) ++ return; ++} ++ ++void intel_unregister_dsm_handler(void) ++{ ++} +diff --git a/drivers/gpu/drm/i915_legacy/intel_atomic.c b/drivers/gpu/drm/i915_legacy/intel_atomic.c +new file mode 100644 +index 000000000000..2986ee1dbf62 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_atomic.c +@@ -0,0 +1,428 @@ ++/* ++ * Copyright © 2015 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER ++ * DEALINGS IN THE SOFTWARE. ++ */ ++ ++/** ++ * DOC: atomic modeset support ++ * ++ * The functions here implement the state management and hardware programming ++ * dispatch required by the atomic modeset infrastructure. ++ * See intel_atomic_plane.c for the plane-specific atomic functionality. ++ */ ++ ++#include ++#include ++#include ++#include ++ ++#include "intel_drv.h" ++#include "intel_hdcp.h" ++#include "intel_sprite.h" ++ ++/** ++ * intel_digital_connector_atomic_get_property - hook for connector->atomic_get_property. ++ * @connector: Connector to get the property for. ++ * @state: Connector state to retrieve the property from. ++ * @property: Property to retrieve. ++ * @val: Return value for the property. ++ * ++ * Returns the atomic property value for a digital connector. ++ */ ++int intel_digital_connector_atomic_get_property(struct drm_connector *connector, ++ const struct drm_connector_state *state, ++ struct drm_property *property, ++ u64 *val) ++{ ++ struct drm_device *dev = connector->dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_digital_connector_state *intel_conn_state = ++ to_intel_digital_connector_state(state); ++ ++ if (property == dev_priv->force_audio_property) ++ *val = intel_conn_state->force_audio; ++ else if (property == dev_priv->broadcast_rgb_property) ++ *val = intel_conn_state->broadcast_rgb; ++ else { ++ DRM_DEBUG_ATOMIC("Unknown property [PROP:%d:%s]\n", ++ property->base.id, property->name); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++/** ++ * intel_digital_connector_atomic_set_property - hook for connector->atomic_set_property. ++ * @connector: Connector to set the property for. ++ * @state: Connector state to set the property on. ++ * @property: Property to set. ++ * @val: New value for the property. ++ * ++ * Sets the atomic property value for a digital connector. ++ */ ++int intel_digital_connector_atomic_set_property(struct drm_connector *connector, ++ struct drm_connector_state *state, ++ struct drm_property *property, ++ u64 val) ++{ ++ struct drm_device *dev = connector->dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_digital_connector_state *intel_conn_state = ++ to_intel_digital_connector_state(state); ++ ++ if (property == dev_priv->force_audio_property) { ++ intel_conn_state->force_audio = val; ++ return 0; ++ } ++ ++ if (property == dev_priv->broadcast_rgb_property) { ++ intel_conn_state->broadcast_rgb = val; ++ return 0; ++ } ++ ++ DRM_DEBUG_ATOMIC("Unknown property [PROP:%d:%s]\n", ++ property->base.id, property->name); ++ return -EINVAL; ++} ++ ++ ++static bool blob_equal(const struct drm_property_blob *a, ++ const struct drm_property_blob *b) ++{ ++ if (a && b) ++ return a->length == b->length && ++ !memcmp(a->data, b->data, a->length); ++ ++ return !a == !b; ++} ++ ++int intel_digital_connector_atomic_check(struct drm_connector *conn, ++ struct drm_atomic_state *state) ++{ ++ struct drm_connector_state *new_state = ++ drm_atomic_get_new_connector_state(state, conn); ++ struct intel_digital_connector_state *new_conn_state = ++ to_intel_digital_connector_state(new_state); ++ struct drm_connector_state *old_state = ++ drm_atomic_get_old_connector_state(state, conn); ++ struct intel_digital_connector_state *old_conn_state = ++ to_intel_digital_connector_state(old_state); ++ struct drm_crtc_state *crtc_state; ++ ++ intel_hdcp_atomic_check(conn, old_state, new_state); ++ ++ if (!new_state->crtc) ++ return 0; ++ ++ crtc_state = drm_atomic_get_new_crtc_state(state, new_state->crtc); ++ ++ /* ++ * These properties are handled by fastset, and might not end ++ * up in a modeset. ++ */ ++ if (new_conn_state->force_audio != old_conn_state->force_audio || ++ new_conn_state->broadcast_rgb != old_conn_state->broadcast_rgb || ++ new_conn_state->base.colorspace != old_conn_state->base.colorspace || ++ new_conn_state->base.picture_aspect_ratio != old_conn_state->base.picture_aspect_ratio || ++ new_conn_state->base.content_type != old_conn_state->base.content_type || ++ new_conn_state->base.scaling_mode != old_conn_state->base.scaling_mode || ++ !blob_equal(new_conn_state->base.hdr_output_metadata, ++ old_conn_state->base.hdr_output_metadata)) ++ crtc_state->mode_changed = true; ++ ++ return 0; ++} ++ ++/** ++ * intel_digital_connector_duplicate_state - duplicate connector state ++ * @connector: digital connector ++ * ++ * Allocates and returns a copy of the connector state (both common and ++ * digital connector specific) for the specified connector. ++ * ++ * Returns: The newly allocated connector state, or NULL on failure. ++ */ ++struct drm_connector_state * ++intel_digital_connector_duplicate_state(struct drm_connector *connector) ++{ ++ struct intel_digital_connector_state *state; ++ ++ state = kmemdup(connector->state, sizeof(*state), GFP_KERNEL); ++ if (!state) ++ return NULL; ++ ++ __drm_atomic_helper_connector_duplicate_state(connector, &state->base); ++ return &state->base; ++} ++ ++/** ++ * intel_crtc_duplicate_state - duplicate crtc state ++ * @crtc: drm crtc ++ * ++ * Allocates and returns a copy of the crtc state (both common and ++ * Intel-specific) for the specified crtc. ++ * ++ * Returns: The newly allocated crtc state, or NULL on failure. ++ */ ++struct drm_crtc_state * ++intel_crtc_duplicate_state(struct drm_crtc *crtc) ++{ ++ struct intel_crtc_state *crtc_state; ++ ++ crtc_state = kmemdup(crtc->state, sizeof(*crtc_state), GFP_KERNEL); ++ if (!crtc_state) ++ return NULL; ++ ++ __drm_atomic_helper_crtc_duplicate_state(crtc, &crtc_state->base); ++ ++ crtc_state->update_pipe = false; ++ crtc_state->disable_lp_wm = false; ++ crtc_state->disable_cxsr = false; ++ crtc_state->update_wm_pre = false; ++ crtc_state->update_wm_post = false; ++ crtc_state->fb_changed = false; ++ crtc_state->fifo_changed = false; ++ crtc_state->wm.need_postvbl_update = false; ++ crtc_state->fb_bits = 0; ++ crtc_state->update_planes = 0; ++ ++ return &crtc_state->base; ++} ++ ++/** ++ * intel_crtc_destroy_state - destroy crtc state ++ * @crtc: drm crtc ++ * @state: the state to destroy ++ * ++ * Destroys the crtc state (both common and Intel-specific) for the ++ * specified crtc. ++ */ ++void ++intel_crtc_destroy_state(struct drm_crtc *crtc, ++ struct drm_crtc_state *state) ++{ ++ drm_atomic_helper_crtc_destroy_state(crtc, state); ++} ++ ++static void intel_atomic_setup_scaler(struct intel_crtc_scaler_state *scaler_state, ++ int num_scalers_need, struct intel_crtc *intel_crtc, ++ const char *name, int idx, ++ struct intel_plane_state *plane_state, ++ int *scaler_id) ++{ ++ struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev); ++ int j; ++ u32 mode; ++ ++ if (*scaler_id < 0) { ++ /* find a free scaler */ ++ for (j = 0; j < intel_crtc->num_scalers; j++) { ++ if (scaler_state->scalers[j].in_use) ++ continue; ++ ++ *scaler_id = j; ++ scaler_state->scalers[*scaler_id].in_use = 1; ++ break; ++ } ++ } ++ ++ if (WARN(*scaler_id < 0, "Cannot find scaler for %s:%d\n", name, idx)) ++ return; ++ ++ /* set scaler mode */ ++ if (plane_state && plane_state->base.fb && ++ plane_state->base.fb->format->is_yuv && ++ plane_state->base.fb->format->num_planes > 1) { ++ struct intel_plane *plane = to_intel_plane(plane_state->base.plane); ++ if (IS_GEN(dev_priv, 9) && ++ !IS_GEMINILAKE(dev_priv)) { ++ mode = SKL_PS_SCALER_MODE_NV12; ++ } else if (icl_is_hdr_plane(dev_priv, plane->id)) { ++ /* ++ * On gen11+'s HDR planes we only use the scaler for ++ * scaling. They have a dedicated chroma upsampler, so ++ * we don't need the scaler to upsample the UV plane. ++ */ ++ mode = PS_SCALER_MODE_NORMAL; ++ } else { ++ mode = PS_SCALER_MODE_PLANAR; ++ ++ if (plane_state->linked_plane) ++ mode |= PS_PLANE_Y_SEL(plane_state->linked_plane->id); ++ } ++ } else if (INTEL_GEN(dev_priv) > 9 || IS_GEMINILAKE(dev_priv)) { ++ mode = PS_SCALER_MODE_NORMAL; ++ } else if (num_scalers_need == 1 && intel_crtc->num_scalers > 1) { ++ /* ++ * when only 1 scaler is in use on a pipe with 2 scalers ++ * scaler 0 operates in high quality (HQ) mode. ++ * In this case use scaler 0 to take advantage of HQ mode ++ */ ++ scaler_state->scalers[*scaler_id].in_use = 0; ++ *scaler_id = 0; ++ scaler_state->scalers[0].in_use = 1; ++ mode = SKL_PS_SCALER_MODE_HQ; ++ } else { ++ mode = SKL_PS_SCALER_MODE_DYN; ++ } ++ ++ DRM_DEBUG_KMS("Attached scaler id %u.%u to %s:%d\n", ++ intel_crtc->pipe, *scaler_id, name, idx); ++ scaler_state->scalers[*scaler_id].mode = mode; ++} ++ ++/** ++ * intel_atomic_setup_scalers() - setup scalers for crtc per staged requests ++ * @dev_priv: i915 device ++ * @intel_crtc: intel crtc ++ * @crtc_state: incoming crtc_state to validate and setup scalers ++ * ++ * This function sets up scalers based on staged scaling requests for ++ * a @crtc and its planes. It is called from crtc level check path. If request ++ * is a supportable request, it attaches scalers to requested planes and crtc. ++ * ++ * This function takes into account the current scaler(s) in use by any planes ++ * not being part of this atomic state ++ * ++ * Returns: ++ * 0 - scalers were setup succesfully ++ * error code - otherwise ++ */ ++int intel_atomic_setup_scalers(struct drm_i915_private *dev_priv, ++ struct intel_crtc *intel_crtc, ++ struct intel_crtc_state *crtc_state) ++{ ++ struct drm_plane *plane = NULL; ++ struct intel_plane *intel_plane; ++ struct intel_plane_state *plane_state = NULL; ++ struct intel_crtc_scaler_state *scaler_state = ++ &crtc_state->scaler_state; ++ struct drm_atomic_state *drm_state = crtc_state->base.state; ++ struct intel_atomic_state *intel_state = to_intel_atomic_state(drm_state); ++ int num_scalers_need; ++ int i; ++ ++ num_scalers_need = hweight32(scaler_state->scaler_users); ++ ++ /* ++ * High level flow: ++ * - staged scaler requests are already in scaler_state->scaler_users ++ * - check whether staged scaling requests can be supported ++ * - add planes using scalers that aren't in current transaction ++ * - assign scalers to requested users ++ * - as part of plane commit, scalers will be committed ++ * (i.e., either attached or detached) to respective planes in hw ++ * - as part of crtc_commit, scaler will be either attached or detached ++ * to crtc in hw ++ */ ++ ++ /* fail if required scalers > available scalers */ ++ if (num_scalers_need > intel_crtc->num_scalers){ ++ DRM_DEBUG_KMS("Too many scaling requests %d > %d\n", ++ num_scalers_need, intel_crtc->num_scalers); ++ return -EINVAL; ++ } ++ ++ /* walkthrough scaler_users bits and start assigning scalers */ ++ for (i = 0; i < sizeof(scaler_state->scaler_users) * 8; i++) { ++ int *scaler_id; ++ const char *name; ++ int idx; ++ ++ /* skip if scaler not required */ ++ if (!(scaler_state->scaler_users & (1 << i))) ++ continue; ++ ++ if (i == SKL_CRTC_INDEX) { ++ name = "CRTC"; ++ idx = intel_crtc->base.base.id; ++ ++ /* panel fitter case: assign as a crtc scaler */ ++ scaler_id = &scaler_state->scaler_id; ++ } else { ++ name = "PLANE"; ++ ++ /* plane scaler case: assign as a plane scaler */ ++ /* find the plane that set the bit as scaler_user */ ++ plane = drm_state->planes[i].ptr; ++ ++ /* ++ * to enable/disable hq mode, add planes that are using scaler ++ * into this transaction ++ */ ++ if (!plane) { ++ struct drm_plane_state *state; ++ plane = drm_plane_from_index(&dev_priv->drm, i); ++ state = drm_atomic_get_plane_state(drm_state, plane); ++ if (IS_ERR(state)) { ++ DRM_DEBUG_KMS("Failed to add [PLANE:%d] to drm_state\n", ++ plane->base.id); ++ return PTR_ERR(state); ++ } ++ ++ /* ++ * the plane is added after plane checks are run, ++ * but since this plane is unchanged just do the ++ * minimum required validation. ++ */ ++ crtc_state->base.planes_changed = true; ++ } ++ ++ intel_plane = to_intel_plane(plane); ++ idx = plane->base.id; ++ ++ /* plane on different crtc cannot be a scaler user of this crtc */ ++ if (WARN_ON(intel_plane->pipe != intel_crtc->pipe)) ++ continue; ++ ++ plane_state = intel_atomic_get_new_plane_state(intel_state, ++ intel_plane); ++ scaler_id = &plane_state->scaler_id; ++ } ++ ++ intel_atomic_setup_scaler(scaler_state, num_scalers_need, ++ intel_crtc, name, idx, ++ plane_state, scaler_id); ++ } ++ ++ return 0; ++} ++ ++struct drm_atomic_state * ++intel_atomic_state_alloc(struct drm_device *dev) ++{ ++ struct intel_atomic_state *state = kzalloc(sizeof(*state), GFP_KERNEL); ++ ++ if (!state || drm_atomic_state_init(dev, &state->base) < 0) { ++ kfree(state); ++ return NULL; ++ } ++ ++ return &state->base; ++} ++ ++void intel_atomic_state_clear(struct drm_atomic_state *s) ++{ ++ struct intel_atomic_state *state = to_intel_atomic_state(s); ++ drm_atomic_state_default_clear(&state->base); ++ state->dpll_set = state->modeset = false; ++} +diff --git a/drivers/gpu/drm/i915_legacy/intel_atomic_plane.c b/drivers/gpu/drm/i915_legacy/intel_atomic_plane.c +new file mode 100644 +index 000000000000..d11681d71add +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_atomic_plane.c +@@ -0,0 +1,373 @@ ++/* ++ * Copyright © 2014 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER ++ * DEALINGS IN THE SOFTWARE. ++ */ ++ ++/** ++ * DOC: atomic plane helpers ++ * ++ * The functions here are used by the atomic plane helper functions to ++ * implement legacy plane updates (i.e., drm_plane->update_plane() and ++ * drm_plane->disable_plane()). This allows plane updates to use the ++ * atomic state infrastructure and perform plane updates as separate ++ * prepare/check/commit/cleanup steps. ++ */ ++ ++#include ++#include ++#include ++ ++#include "intel_atomic_plane.h" ++#include "intel_drv.h" ++#include "intel_pm.h" ++#include "intel_sprite.h" ++ ++struct intel_plane *intel_plane_alloc(void) ++{ ++ struct intel_plane_state *plane_state; ++ struct intel_plane *plane; ++ ++ plane = kzalloc(sizeof(*plane), GFP_KERNEL); ++ if (!plane) ++ return ERR_PTR(-ENOMEM); ++ ++ plane_state = kzalloc(sizeof(*plane_state), GFP_KERNEL); ++ if (!plane_state) { ++ kfree(plane); ++ return ERR_PTR(-ENOMEM); ++ } ++ ++ __drm_atomic_helper_plane_reset(&plane->base, &plane_state->base); ++ plane_state->scaler_id = -1; ++ ++ return plane; ++} ++ ++void intel_plane_free(struct intel_plane *plane) ++{ ++ intel_plane_destroy_state(&plane->base, plane->base.state); ++ kfree(plane); ++} ++ ++/** ++ * intel_plane_duplicate_state - duplicate plane state ++ * @plane: drm plane ++ * ++ * Allocates and returns a copy of the plane state (both common and ++ * Intel-specific) for the specified plane. ++ * ++ * Returns: The newly allocated plane state, or NULL on failure. ++ */ ++struct drm_plane_state * ++intel_plane_duplicate_state(struct drm_plane *plane) ++{ ++ struct drm_plane_state *state; ++ struct intel_plane_state *intel_state; ++ ++ intel_state = kmemdup(plane->state, sizeof(*intel_state), GFP_KERNEL); ++ ++ if (!intel_state) ++ return NULL; ++ ++ state = &intel_state->base; ++ ++ __drm_atomic_helper_plane_duplicate_state(plane, state); ++ ++ intel_state->vma = NULL; ++ intel_state->flags = 0; ++ ++ return state; ++} ++ ++/** ++ * intel_plane_destroy_state - destroy plane state ++ * @plane: drm plane ++ * @state: state object to destroy ++ * ++ * Destroys the plane state (both common and Intel-specific) for the ++ * specified plane. ++ */ ++void ++intel_plane_destroy_state(struct drm_plane *plane, ++ struct drm_plane_state *state) ++{ ++ WARN_ON(to_intel_plane_state(state)->vma); ++ ++ drm_atomic_helper_plane_destroy_state(plane, state); ++} ++ ++int intel_plane_atomic_check_with_state(const struct intel_crtc_state *old_crtc_state, ++ struct intel_crtc_state *new_crtc_state, ++ const struct intel_plane_state *old_plane_state, ++ struct intel_plane_state *new_plane_state) ++{ ++ struct intel_plane *plane = to_intel_plane(new_plane_state->base.plane); ++ int ret; ++ ++ new_crtc_state->active_planes &= ~BIT(plane->id); ++ new_crtc_state->nv12_planes &= ~BIT(plane->id); ++ new_crtc_state->c8_planes &= ~BIT(plane->id); ++ new_plane_state->base.visible = false; ++ ++ if (!new_plane_state->base.crtc && !old_plane_state->base.crtc) ++ return 0; ++ ++ ret = plane->check_plane(new_crtc_state, new_plane_state); ++ if (ret) ++ return ret; ++ ++ /* FIXME pre-g4x don't work like this */ ++ if (new_plane_state->base.visible) ++ new_crtc_state->active_planes |= BIT(plane->id); ++ ++ if (new_plane_state->base.visible && ++ is_planar_yuv_format(new_plane_state->base.fb->format->format)) ++ new_crtc_state->nv12_planes |= BIT(plane->id); ++ ++ if (new_plane_state->base.visible && ++ new_plane_state->base.fb->format->format == DRM_FORMAT_C8) ++ new_crtc_state->c8_planes |= BIT(plane->id); ++ ++ if (new_plane_state->base.visible || old_plane_state->base.visible) ++ new_crtc_state->update_planes |= BIT(plane->id); ++ ++ return intel_plane_atomic_calc_changes(old_crtc_state, ++ &new_crtc_state->base, ++ old_plane_state, ++ &new_plane_state->base); ++} ++ ++static int intel_plane_atomic_check(struct drm_plane *plane, ++ struct drm_plane_state *new_plane_state) ++{ ++ struct drm_atomic_state *state = new_plane_state->state; ++ const struct drm_plane_state *old_plane_state = ++ drm_atomic_get_old_plane_state(state, plane); ++ struct drm_crtc *crtc = new_plane_state->crtc ?: old_plane_state->crtc; ++ const struct drm_crtc_state *old_crtc_state; ++ struct drm_crtc_state *new_crtc_state; ++ ++ new_plane_state->visible = false; ++ if (!crtc) ++ return 0; ++ ++ old_crtc_state = drm_atomic_get_old_crtc_state(state, crtc); ++ new_crtc_state = drm_atomic_get_new_crtc_state(state, crtc); ++ ++ return intel_plane_atomic_check_with_state(to_intel_crtc_state(old_crtc_state), ++ to_intel_crtc_state(new_crtc_state), ++ to_intel_plane_state(old_plane_state), ++ to_intel_plane_state(new_plane_state)); ++} ++ ++static struct intel_plane * ++skl_next_plane_to_commit(struct intel_atomic_state *state, ++ struct intel_crtc *crtc, ++ struct skl_ddb_entry entries_y[I915_MAX_PLANES], ++ struct skl_ddb_entry entries_uv[I915_MAX_PLANES], ++ unsigned int *update_mask) ++{ ++ struct intel_crtc_state *crtc_state = ++ intel_atomic_get_new_crtc_state(state, crtc); ++ struct intel_plane_state *plane_state; ++ struct intel_plane *plane; ++ int i; ++ ++ if (*update_mask == 0) ++ return NULL; ++ ++ for_each_new_intel_plane_in_state(state, plane, plane_state, i) { ++ enum plane_id plane_id = plane->id; ++ ++ if (crtc->pipe != plane->pipe || ++ !(*update_mask & BIT(plane_id))) ++ continue; ++ ++ if (skl_ddb_allocation_overlaps(&crtc_state->wm.skl.plane_ddb_y[plane_id], ++ entries_y, ++ I915_MAX_PLANES, plane_id) || ++ skl_ddb_allocation_overlaps(&crtc_state->wm.skl.plane_ddb_uv[plane_id], ++ entries_uv, ++ I915_MAX_PLANES, plane_id)) ++ continue; ++ ++ *update_mask &= ~BIT(plane_id); ++ entries_y[plane_id] = crtc_state->wm.skl.plane_ddb_y[plane_id]; ++ entries_uv[plane_id] = crtc_state->wm.skl.plane_ddb_uv[plane_id]; ++ ++ return plane; ++ } ++ ++ /* should never happen */ ++ WARN_ON(1); ++ ++ return NULL; ++} ++ ++void intel_update_plane(struct intel_plane *plane, ++ const struct intel_crtc_state *crtc_state, ++ const struct intel_plane_state *plane_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ ++ trace_intel_update_plane(&plane->base, crtc); ++ plane->update_plane(plane, crtc_state, plane_state); ++} ++ ++void intel_update_slave(struct intel_plane *plane, ++ const struct intel_crtc_state *crtc_state, ++ const struct intel_plane_state *plane_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ ++ trace_intel_update_plane(&plane->base, crtc); ++ plane->update_slave(plane, crtc_state, plane_state); ++} ++ ++void intel_disable_plane(struct intel_plane *plane, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ ++ trace_intel_disable_plane(&plane->base, crtc); ++ plane->disable_plane(plane, crtc_state); ++} ++ ++void skl_update_planes_on_crtc(struct intel_atomic_state *state, ++ struct intel_crtc *crtc) ++{ ++ struct intel_crtc_state *old_crtc_state = ++ intel_atomic_get_old_crtc_state(state, crtc); ++ struct intel_crtc_state *new_crtc_state = ++ intel_atomic_get_new_crtc_state(state, crtc); ++ struct skl_ddb_entry entries_y[I915_MAX_PLANES]; ++ struct skl_ddb_entry entries_uv[I915_MAX_PLANES]; ++ u32 update_mask = new_crtc_state->update_planes; ++ struct intel_plane *plane; ++ ++ memcpy(entries_y, old_crtc_state->wm.skl.plane_ddb_y, ++ sizeof(old_crtc_state->wm.skl.plane_ddb_y)); ++ memcpy(entries_uv, old_crtc_state->wm.skl.plane_ddb_uv, ++ sizeof(old_crtc_state->wm.skl.plane_ddb_uv)); ++ ++ while ((plane = skl_next_plane_to_commit(state, crtc, ++ entries_y, entries_uv, ++ &update_mask))) { ++ struct intel_plane_state *new_plane_state = ++ intel_atomic_get_new_plane_state(state, plane); ++ ++ if (new_plane_state->base.visible) { ++ intel_update_plane(plane, new_crtc_state, new_plane_state); ++ } else if (new_plane_state->slave) { ++ struct intel_plane *master = ++ new_plane_state->linked_plane; ++ ++ /* ++ * We update the slave plane from this function because ++ * programming it from the master plane's update_plane ++ * callback runs into issues when the Y plane is ++ * reassigned, disabled or used by a different plane. ++ * ++ * The slave plane is updated with the master plane's ++ * plane_state. ++ */ ++ new_plane_state = ++ intel_atomic_get_new_plane_state(state, master); ++ ++ intel_update_slave(plane, new_crtc_state, new_plane_state); ++ } else { ++ intel_disable_plane(plane, new_crtc_state); ++ } ++ } ++} ++ ++void i9xx_update_planes_on_crtc(struct intel_atomic_state *state, ++ struct intel_crtc *crtc) ++{ ++ struct intel_crtc_state *new_crtc_state = ++ intel_atomic_get_new_crtc_state(state, crtc); ++ u32 update_mask = new_crtc_state->update_planes; ++ struct intel_plane_state *new_plane_state; ++ struct intel_plane *plane; ++ int i; ++ ++ for_each_new_intel_plane_in_state(state, plane, new_plane_state, i) { ++ if (crtc->pipe != plane->pipe || ++ !(update_mask & BIT(plane->id))) ++ continue; ++ ++ if (new_plane_state->base.visible) ++ intel_update_plane(plane, new_crtc_state, new_plane_state); ++ else ++ intel_disable_plane(plane, new_crtc_state); ++ } ++} ++ ++const struct drm_plane_helper_funcs intel_plane_helper_funcs = { ++ .prepare_fb = intel_prepare_plane_fb, ++ .cleanup_fb = intel_cleanup_plane_fb, ++ .atomic_check = intel_plane_atomic_check, ++}; ++ ++/** ++ * intel_plane_atomic_get_property - fetch plane property value ++ * @plane: plane to fetch property for ++ * @state: state containing the property value ++ * @property: property to look up ++ * @val: pointer to write property value into ++ * ++ * The DRM core does not store shadow copies of properties for ++ * atomic-capable drivers. This entrypoint is used to fetch ++ * the current value of a driver-specific plane property. ++ */ ++int ++intel_plane_atomic_get_property(struct drm_plane *plane, ++ const struct drm_plane_state *state, ++ struct drm_property *property, ++ u64 *val) ++{ ++ DRM_DEBUG_KMS("Unknown property [PROP:%d:%s]\n", ++ property->base.id, property->name); ++ return -EINVAL; ++} ++ ++/** ++ * intel_plane_atomic_set_property - set plane property value ++ * @plane: plane to set property for ++ * @state: state to update property value in ++ * @property: property to set ++ * @val: value to set property to ++ * ++ * Writes the specified property value for a plane into the provided atomic ++ * state object. ++ * ++ * Returns 0 on success, -EINVAL on unrecognized properties ++ */ ++int ++intel_plane_atomic_set_property(struct drm_plane *plane, ++ struct drm_plane_state *state, ++ struct drm_property *property, ++ u64 val) ++{ ++ DRM_DEBUG_KMS("Unknown property [PROP:%d:%s]\n", ++ property->base.id, property->name); ++ return -EINVAL; ++} +diff --git a/drivers/gpu/drm/i915_legacy/intel_atomic_plane.h b/drivers/gpu/drm/i915_legacy/intel_atomic_plane.h +new file mode 100644 +index 000000000000..14678620440f +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_atomic_plane.h +@@ -0,0 +1,40 @@ ++/* SPDX-License-Identifier: MIT */ ++/* ++ * Copyright © 2019 Intel Corporation ++ */ ++ ++#ifndef __INTEL_ATOMIC_PLANE_H__ ++#define __INTEL_ATOMIC_PLANE_H__ ++ ++struct drm_plane; ++struct intel_atomic_state; ++struct intel_crtc; ++struct intel_crtc_state; ++struct intel_plane; ++struct intel_plane_state; ++ ++extern const struct drm_plane_helper_funcs intel_plane_helper_funcs; ++ ++void intel_update_plane(struct intel_plane *plane, ++ const struct intel_crtc_state *crtc_state, ++ const struct intel_plane_state *plane_state); ++void intel_update_slave(struct intel_plane *plane, ++ const struct intel_crtc_state *crtc_state, ++ const struct intel_plane_state *plane_state); ++void intel_disable_plane(struct intel_plane *plane, ++ const struct intel_crtc_state *crtc_state); ++struct intel_plane *intel_plane_alloc(void); ++void intel_plane_free(struct intel_plane *plane); ++struct drm_plane_state *intel_plane_duplicate_state(struct drm_plane *plane); ++void intel_plane_destroy_state(struct drm_plane *plane, ++ struct drm_plane_state *state); ++void skl_update_planes_on_crtc(struct intel_atomic_state *state, ++ struct intel_crtc *crtc); ++void i9xx_update_planes_on_crtc(struct intel_atomic_state *state, ++ struct intel_crtc *crtc); ++int intel_plane_atomic_check_with_state(const struct intel_crtc_state *old_crtc_state, ++ struct intel_crtc_state *crtc_state, ++ const struct intel_plane_state *old_plane_state, ++ struct intel_plane_state *intel_state); ++ ++#endif /* __INTEL_ATOMIC_PLANE_H__ */ +diff --git a/drivers/gpu/drm/i915_legacy/intel_audio.c b/drivers/gpu/drm/i915_legacy/intel_audio.c +new file mode 100644 +index 000000000000..bca4cc025d3d +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_audio.c +@@ -0,0 +1,1105 @@ ++/* ++ * Copyright © 2014 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER ++ * DEALINGS IN THE SOFTWARE. ++ */ ++ ++#include ++#include ++ ++#include ++#include ++#include ++ ++#include "i915_drv.h" ++#include "intel_audio.h" ++#include "intel_drv.h" ++ ++/** ++ * DOC: High Definition Audio over HDMI and Display Port ++ * ++ * The graphics and audio drivers together support High Definition Audio over ++ * HDMI and Display Port. The audio programming sequences are divided into audio ++ * codec and controller enable and disable sequences. The graphics driver ++ * handles the audio codec sequences, while the audio driver handles the audio ++ * controller sequences. ++ * ++ * The disable sequences must be performed before disabling the transcoder or ++ * port. The enable sequences may only be performed after enabling the ++ * transcoder and port, and after completed link training. Therefore the audio ++ * enable/disable sequences are part of the modeset sequence. ++ * ++ * The codec and controller sequences could be done either parallel or serial, ++ * but generally the ELDV/PD change in the codec sequence indicates to the audio ++ * driver that the controller sequence should start. Indeed, most of the ++ * co-operation between the graphics and audio drivers is handled via audio ++ * related registers. (The notable exception is the power management, not ++ * covered here.) ++ * ++ * The struct &i915_audio_component is used to interact between the graphics ++ * and audio drivers. The struct &i915_audio_component_ops @ops in it is ++ * defined in graphics driver and called in audio driver. The ++ * struct &i915_audio_component_audio_ops @audio_ops is called from i915 driver. ++ */ ++ ++/* DP N/M table */ ++#define LC_810M 810000 ++#define LC_540M 540000 ++#define LC_270M 270000 ++#define LC_162M 162000 ++ ++struct dp_aud_n_m { ++ int sample_rate; ++ int clock; ++ u16 m; ++ u16 n; ++}; ++ ++/* Values according to DP 1.4 Table 2-104 */ ++static const struct dp_aud_n_m dp_aud_n_m[] = { ++ { 32000, LC_162M, 1024, 10125 }, ++ { 44100, LC_162M, 784, 5625 }, ++ { 48000, LC_162M, 512, 3375 }, ++ { 64000, LC_162M, 2048, 10125 }, ++ { 88200, LC_162M, 1568, 5625 }, ++ { 96000, LC_162M, 1024, 3375 }, ++ { 128000, LC_162M, 4096, 10125 }, ++ { 176400, LC_162M, 3136, 5625 }, ++ { 192000, LC_162M, 2048, 3375 }, ++ { 32000, LC_270M, 1024, 16875 }, ++ { 44100, LC_270M, 784, 9375 }, ++ { 48000, LC_270M, 512, 5625 }, ++ { 64000, LC_270M, 2048, 16875 }, ++ { 88200, LC_270M, 1568, 9375 }, ++ { 96000, LC_270M, 1024, 5625 }, ++ { 128000, LC_270M, 4096, 16875 }, ++ { 176400, LC_270M, 3136, 9375 }, ++ { 192000, LC_270M, 2048, 5625 }, ++ { 32000, LC_540M, 1024, 33750 }, ++ { 44100, LC_540M, 784, 18750 }, ++ { 48000, LC_540M, 512, 11250 }, ++ { 64000, LC_540M, 2048, 33750 }, ++ { 88200, LC_540M, 1568, 18750 }, ++ { 96000, LC_540M, 1024, 11250 }, ++ { 128000, LC_540M, 4096, 33750 }, ++ { 176400, LC_540M, 3136, 18750 }, ++ { 192000, LC_540M, 2048, 11250 }, ++ { 32000, LC_810M, 1024, 50625 }, ++ { 44100, LC_810M, 784, 28125 }, ++ { 48000, LC_810M, 512, 16875 }, ++ { 64000, LC_810M, 2048, 50625 }, ++ { 88200, LC_810M, 1568, 28125 }, ++ { 96000, LC_810M, 1024, 16875 }, ++ { 128000, LC_810M, 4096, 50625 }, ++ { 176400, LC_810M, 3136, 28125 }, ++ { 192000, LC_810M, 2048, 16875 }, ++}; ++ ++static const struct dp_aud_n_m * ++audio_config_dp_get_n_m(const struct intel_crtc_state *crtc_state, int rate) ++{ ++ int i; ++ ++ for (i = 0; i < ARRAY_SIZE(dp_aud_n_m); i++) { ++ if (rate == dp_aud_n_m[i].sample_rate && ++ crtc_state->port_clock == dp_aud_n_m[i].clock) ++ return &dp_aud_n_m[i]; ++ } ++ ++ return NULL; ++} ++ ++static const struct { ++ int clock; ++ u32 config; ++} hdmi_audio_clock[] = { ++ { 25175, AUD_CONFIG_PIXEL_CLOCK_HDMI_25175 }, ++ { 25200, AUD_CONFIG_PIXEL_CLOCK_HDMI_25200 }, /* default per bspec */ ++ { 27000, AUD_CONFIG_PIXEL_CLOCK_HDMI_27000 }, ++ { 27027, AUD_CONFIG_PIXEL_CLOCK_HDMI_27027 }, ++ { 54000, AUD_CONFIG_PIXEL_CLOCK_HDMI_54000 }, ++ { 54054, AUD_CONFIG_PIXEL_CLOCK_HDMI_54054 }, ++ { 74176, AUD_CONFIG_PIXEL_CLOCK_HDMI_74176 }, ++ { 74250, AUD_CONFIG_PIXEL_CLOCK_HDMI_74250 }, ++ { 148352, AUD_CONFIG_PIXEL_CLOCK_HDMI_148352 }, ++ { 148500, AUD_CONFIG_PIXEL_CLOCK_HDMI_148500 }, ++}; ++ ++/* HDMI N/CTS table */ ++#define TMDS_297M 297000 ++#define TMDS_296M 296703 ++#define TMDS_594M 594000 ++#define TMDS_593M 593407 ++ ++static const struct { ++ int sample_rate; ++ int clock; ++ int n; ++ int cts; ++} hdmi_aud_ncts[] = { ++ { 32000, TMDS_296M, 5824, 421875 }, ++ { 32000, TMDS_297M, 3072, 222750 }, ++ { 32000, TMDS_593M, 5824, 843750 }, ++ { 32000, TMDS_594M, 3072, 445500 }, ++ { 44100, TMDS_296M, 4459, 234375 }, ++ { 44100, TMDS_297M, 4704, 247500 }, ++ { 44100, TMDS_593M, 8918, 937500 }, ++ { 44100, TMDS_594M, 9408, 990000 }, ++ { 88200, TMDS_296M, 8918, 234375 }, ++ { 88200, TMDS_297M, 9408, 247500 }, ++ { 88200, TMDS_593M, 17836, 937500 }, ++ { 88200, TMDS_594M, 18816, 990000 }, ++ { 176400, TMDS_296M, 17836, 234375 }, ++ { 176400, TMDS_297M, 18816, 247500 }, ++ { 176400, TMDS_593M, 35672, 937500 }, ++ { 176400, TMDS_594M, 37632, 990000 }, ++ { 48000, TMDS_296M, 5824, 281250 }, ++ { 48000, TMDS_297M, 5120, 247500 }, ++ { 48000, TMDS_593M, 5824, 562500 }, ++ { 48000, TMDS_594M, 6144, 594000 }, ++ { 96000, TMDS_296M, 11648, 281250 }, ++ { 96000, TMDS_297M, 10240, 247500 }, ++ { 96000, TMDS_593M, 11648, 562500 }, ++ { 96000, TMDS_594M, 12288, 594000 }, ++ { 192000, TMDS_296M, 23296, 281250 }, ++ { 192000, TMDS_297M, 20480, 247500 }, ++ { 192000, TMDS_593M, 23296, 562500 }, ++ { 192000, TMDS_594M, 24576, 594000 }, ++}; ++ ++/* get AUD_CONFIG_PIXEL_CLOCK_HDMI_* value for mode */ ++static u32 audio_config_hdmi_pixel_clock(const struct intel_crtc_state *crtc_state) ++{ ++ const struct drm_display_mode *adjusted_mode = ++ &crtc_state->base.adjusted_mode; ++ int i; ++ ++ for (i = 0; i < ARRAY_SIZE(hdmi_audio_clock); i++) { ++ if (adjusted_mode->crtc_clock == hdmi_audio_clock[i].clock) ++ break; ++ } ++ ++ if (i == ARRAY_SIZE(hdmi_audio_clock)) { ++ DRM_DEBUG_KMS("HDMI audio pixel clock setting for %d not found, falling back to defaults\n", ++ adjusted_mode->crtc_clock); ++ i = 1; ++ } ++ ++ DRM_DEBUG_KMS("Configuring HDMI audio for pixel clock %d (0x%08x)\n", ++ hdmi_audio_clock[i].clock, ++ hdmi_audio_clock[i].config); ++ ++ return hdmi_audio_clock[i].config; ++} ++ ++static int audio_config_hdmi_get_n(const struct intel_crtc_state *crtc_state, ++ int rate) ++{ ++ const struct drm_display_mode *adjusted_mode = ++ &crtc_state->base.adjusted_mode; ++ int i; ++ ++ for (i = 0; i < ARRAY_SIZE(hdmi_aud_ncts); i++) { ++ if (rate == hdmi_aud_ncts[i].sample_rate && ++ adjusted_mode->crtc_clock == hdmi_aud_ncts[i].clock) { ++ return hdmi_aud_ncts[i].n; ++ } ++ } ++ return 0; ++} ++ ++static bool intel_eld_uptodate(struct drm_connector *connector, ++ i915_reg_t reg_eldv, u32 bits_eldv, ++ i915_reg_t reg_elda, u32 bits_elda, ++ i915_reg_t reg_edid) ++{ ++ struct drm_i915_private *dev_priv = to_i915(connector->dev); ++ const u8 *eld = connector->eld; ++ u32 tmp; ++ int i; ++ ++ tmp = I915_READ(reg_eldv); ++ tmp &= bits_eldv; ++ ++ if (!tmp) ++ return false; ++ ++ tmp = I915_READ(reg_elda); ++ tmp &= ~bits_elda; ++ I915_WRITE(reg_elda, tmp); ++ ++ for (i = 0; i < drm_eld_size(eld) / 4; i++) ++ if (I915_READ(reg_edid) != *((const u32 *)eld + i)) ++ return false; ++ ++ return true; ++} ++ ++static void g4x_audio_codec_disable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ u32 eldv, tmp; ++ ++ DRM_DEBUG_KMS("Disable audio codec\n"); ++ ++ tmp = I915_READ(G4X_AUD_VID_DID); ++ if (tmp == INTEL_AUDIO_DEVBLC || tmp == INTEL_AUDIO_DEVCL) ++ eldv = G4X_ELDV_DEVCL_DEVBLC; ++ else ++ eldv = G4X_ELDV_DEVCTG; ++ ++ /* Invalidate ELD */ ++ tmp = I915_READ(G4X_AUD_CNTL_ST); ++ tmp &= ~eldv; ++ I915_WRITE(G4X_AUD_CNTL_ST, tmp); ++} ++ ++static void g4x_audio_codec_enable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct drm_connector *connector = conn_state->connector; ++ const u8 *eld = connector->eld; ++ u32 eldv; ++ u32 tmp; ++ int len, i; ++ ++ DRM_DEBUG_KMS("Enable audio codec, %u bytes ELD\n", drm_eld_size(eld)); ++ ++ tmp = I915_READ(G4X_AUD_VID_DID); ++ if (tmp == INTEL_AUDIO_DEVBLC || tmp == INTEL_AUDIO_DEVCL) ++ eldv = G4X_ELDV_DEVCL_DEVBLC; ++ else ++ eldv = G4X_ELDV_DEVCTG; ++ ++ if (intel_eld_uptodate(connector, ++ G4X_AUD_CNTL_ST, eldv, ++ G4X_AUD_CNTL_ST, G4X_ELD_ADDR_MASK, ++ G4X_HDMIW_HDMIEDID)) ++ return; ++ ++ tmp = I915_READ(G4X_AUD_CNTL_ST); ++ tmp &= ~(eldv | G4X_ELD_ADDR_MASK); ++ len = (tmp >> 9) & 0x1f; /* ELD buffer size */ ++ I915_WRITE(G4X_AUD_CNTL_ST, tmp); ++ ++ len = min(drm_eld_size(eld) / 4, len); ++ DRM_DEBUG_DRIVER("ELD size %d\n", len); ++ for (i = 0; i < len; i++) ++ I915_WRITE(G4X_HDMIW_HDMIEDID, *((const u32 *)eld + i)); ++ ++ tmp = I915_READ(G4X_AUD_CNTL_ST); ++ tmp |= eldv; ++ I915_WRITE(G4X_AUD_CNTL_ST, tmp); ++} ++ ++static void ++hsw_dp_audio_config_update(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct i915_audio_component *acomp = dev_priv->audio_component; ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ enum port port = encoder->port; ++ enum pipe pipe = crtc->pipe; ++ const struct dp_aud_n_m *nm; ++ int rate; ++ u32 tmp; ++ ++ rate = acomp ? acomp->aud_sample_rate[port] : 0; ++ nm = audio_config_dp_get_n_m(crtc_state, rate); ++ if (nm) ++ DRM_DEBUG_KMS("using Maud %u, Naud %u\n", nm->m, nm->n); ++ else ++ DRM_DEBUG_KMS("using automatic Maud, Naud\n"); ++ ++ tmp = I915_READ(HSW_AUD_CFG(pipe)); ++ tmp &= ~AUD_CONFIG_N_VALUE_INDEX; ++ tmp &= ~AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK; ++ tmp &= ~AUD_CONFIG_N_PROG_ENABLE; ++ tmp |= AUD_CONFIG_N_VALUE_INDEX; ++ ++ if (nm) { ++ tmp &= ~AUD_CONFIG_N_MASK; ++ tmp |= AUD_CONFIG_N(nm->n); ++ tmp |= AUD_CONFIG_N_PROG_ENABLE; ++ } ++ ++ I915_WRITE(HSW_AUD_CFG(pipe), tmp); ++ ++ tmp = I915_READ(HSW_AUD_M_CTS_ENABLE(pipe)); ++ tmp &= ~AUD_CONFIG_M_MASK; ++ tmp &= ~AUD_M_CTS_M_VALUE_INDEX; ++ tmp &= ~AUD_M_CTS_M_PROG_ENABLE; ++ ++ if (nm) { ++ tmp |= nm->m; ++ tmp |= AUD_M_CTS_M_VALUE_INDEX; ++ tmp |= AUD_M_CTS_M_PROG_ENABLE; ++ } ++ ++ I915_WRITE(HSW_AUD_M_CTS_ENABLE(pipe), tmp); ++} ++ ++static void ++hsw_hdmi_audio_config_update(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct i915_audio_component *acomp = dev_priv->audio_component; ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ enum port port = encoder->port; ++ enum pipe pipe = crtc->pipe; ++ int n, rate; ++ u32 tmp; ++ ++ rate = acomp ? acomp->aud_sample_rate[port] : 0; ++ ++ tmp = I915_READ(HSW_AUD_CFG(pipe)); ++ tmp &= ~AUD_CONFIG_N_VALUE_INDEX; ++ tmp &= ~AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK; ++ tmp &= ~AUD_CONFIG_N_PROG_ENABLE; ++ tmp |= audio_config_hdmi_pixel_clock(crtc_state); ++ ++ n = audio_config_hdmi_get_n(crtc_state, rate); ++ if (n != 0) { ++ DRM_DEBUG_KMS("using N %d\n", n); ++ ++ tmp &= ~AUD_CONFIG_N_MASK; ++ tmp |= AUD_CONFIG_N(n); ++ tmp |= AUD_CONFIG_N_PROG_ENABLE; ++ } else { ++ DRM_DEBUG_KMS("using automatic N\n"); ++ } ++ ++ I915_WRITE(HSW_AUD_CFG(pipe), tmp); ++ ++ /* ++ * Let's disable "Enable CTS or M Prog bit" ++ * and let HW calculate the value ++ */ ++ tmp = I915_READ(HSW_AUD_M_CTS_ENABLE(pipe)); ++ tmp &= ~AUD_M_CTS_M_PROG_ENABLE; ++ tmp &= ~AUD_M_CTS_M_VALUE_INDEX; ++ I915_WRITE(HSW_AUD_M_CTS_ENABLE(pipe), tmp); ++} ++ ++static void ++hsw_audio_config_update(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state) ++{ ++ if (intel_crtc_has_dp_encoder(crtc_state)) ++ hsw_dp_audio_config_update(encoder, crtc_state); ++ else ++ hsw_hdmi_audio_config_update(encoder, crtc_state); ++} ++ ++static void hsw_audio_codec_disable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc); ++ enum pipe pipe = crtc->pipe; ++ u32 tmp; ++ ++ DRM_DEBUG_KMS("Disable audio codec on pipe %c\n", pipe_name(pipe)); ++ ++ mutex_lock(&dev_priv->av_mutex); ++ ++ /* Disable timestamps */ ++ tmp = I915_READ(HSW_AUD_CFG(pipe)); ++ tmp &= ~AUD_CONFIG_N_VALUE_INDEX; ++ tmp |= AUD_CONFIG_N_PROG_ENABLE; ++ tmp &= ~AUD_CONFIG_UPPER_N_MASK; ++ tmp &= ~AUD_CONFIG_LOWER_N_MASK; ++ if (intel_crtc_has_dp_encoder(old_crtc_state)) ++ tmp |= AUD_CONFIG_N_VALUE_INDEX; ++ I915_WRITE(HSW_AUD_CFG(pipe), tmp); ++ ++ /* Invalidate ELD */ ++ tmp = I915_READ(HSW_AUD_PIN_ELD_CP_VLD); ++ tmp &= ~AUDIO_ELD_VALID(pipe); ++ tmp &= ~AUDIO_OUTPUT_ENABLE(pipe); ++ I915_WRITE(HSW_AUD_PIN_ELD_CP_VLD, tmp); ++ ++ mutex_unlock(&dev_priv->av_mutex); ++} ++ ++static void hsw_audio_codec_enable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_connector *connector = conn_state->connector; ++ enum pipe pipe = crtc->pipe; ++ const u8 *eld = connector->eld; ++ u32 tmp; ++ int len, i; ++ ++ DRM_DEBUG_KMS("Enable audio codec on pipe %c, %u bytes ELD\n", ++ pipe_name(pipe), drm_eld_size(eld)); ++ ++ mutex_lock(&dev_priv->av_mutex); ++ ++ /* Enable audio presence detect, invalidate ELD */ ++ tmp = I915_READ(HSW_AUD_PIN_ELD_CP_VLD); ++ tmp |= AUDIO_OUTPUT_ENABLE(pipe); ++ tmp &= ~AUDIO_ELD_VALID(pipe); ++ I915_WRITE(HSW_AUD_PIN_ELD_CP_VLD, tmp); ++ ++ /* ++ * FIXME: We're supposed to wait for vblank here, but we have vblanks ++ * disabled during the mode set. The proper fix would be to push the ++ * rest of the setup into a vblank work item, queued here, but the ++ * infrastructure is not there yet. ++ */ ++ ++ /* Reset ELD write address */ ++ tmp = I915_READ(HSW_AUD_DIP_ELD_CTRL(pipe)); ++ tmp &= ~IBX_ELD_ADDRESS_MASK; ++ I915_WRITE(HSW_AUD_DIP_ELD_CTRL(pipe), tmp); ++ ++ /* Up to 84 bytes of hw ELD buffer */ ++ len = min(drm_eld_size(eld), 84); ++ for (i = 0; i < len / 4; i++) ++ I915_WRITE(HSW_AUD_EDID_DATA(pipe), *((const u32 *)eld + i)); ++ ++ /* ELD valid */ ++ tmp = I915_READ(HSW_AUD_PIN_ELD_CP_VLD); ++ tmp |= AUDIO_ELD_VALID(pipe); ++ I915_WRITE(HSW_AUD_PIN_ELD_CP_VLD, tmp); ++ ++ /* Enable timestamps */ ++ hsw_audio_config_update(encoder, crtc_state); ++ ++ mutex_unlock(&dev_priv->av_mutex); ++} ++ ++static void ilk_audio_codec_disable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc); ++ enum pipe pipe = crtc->pipe; ++ enum port port = encoder->port; ++ u32 tmp, eldv; ++ i915_reg_t aud_config, aud_cntrl_st2; ++ ++ DRM_DEBUG_KMS("Disable audio codec on port %c, pipe %c\n", ++ port_name(port), pipe_name(pipe)); ++ ++ if (WARN_ON(port == PORT_A)) ++ return; ++ ++ if (HAS_PCH_IBX(dev_priv)) { ++ aud_config = IBX_AUD_CFG(pipe); ++ aud_cntrl_st2 = IBX_AUD_CNTL_ST2; ++ } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { ++ aud_config = VLV_AUD_CFG(pipe); ++ aud_cntrl_st2 = VLV_AUD_CNTL_ST2; ++ } else { ++ aud_config = CPT_AUD_CFG(pipe); ++ aud_cntrl_st2 = CPT_AUD_CNTRL_ST2; ++ } ++ ++ /* Disable timestamps */ ++ tmp = I915_READ(aud_config); ++ tmp &= ~AUD_CONFIG_N_VALUE_INDEX; ++ tmp |= AUD_CONFIG_N_PROG_ENABLE; ++ tmp &= ~AUD_CONFIG_UPPER_N_MASK; ++ tmp &= ~AUD_CONFIG_LOWER_N_MASK; ++ if (intel_crtc_has_dp_encoder(old_crtc_state)) ++ tmp |= AUD_CONFIG_N_VALUE_INDEX; ++ I915_WRITE(aud_config, tmp); ++ ++ eldv = IBX_ELD_VALID(port); ++ ++ /* Invalidate ELD */ ++ tmp = I915_READ(aud_cntrl_st2); ++ tmp &= ~eldv; ++ I915_WRITE(aud_cntrl_st2, tmp); ++} ++ ++static void ilk_audio_codec_enable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_connector *connector = conn_state->connector; ++ enum pipe pipe = crtc->pipe; ++ enum port port = encoder->port; ++ const u8 *eld = connector->eld; ++ u32 tmp, eldv; ++ int len, i; ++ i915_reg_t hdmiw_hdmiedid, aud_config, aud_cntl_st, aud_cntrl_st2; ++ ++ DRM_DEBUG_KMS("Enable audio codec on port %c, pipe %c, %u bytes ELD\n", ++ port_name(port), pipe_name(pipe), drm_eld_size(eld)); ++ ++ if (WARN_ON(port == PORT_A)) ++ return; ++ ++ /* ++ * FIXME: We're supposed to wait for vblank here, but we have vblanks ++ * disabled during the mode set. The proper fix would be to push the ++ * rest of the setup into a vblank work item, queued here, but the ++ * infrastructure is not there yet. ++ */ ++ ++ if (HAS_PCH_IBX(dev_priv)) { ++ hdmiw_hdmiedid = IBX_HDMIW_HDMIEDID(pipe); ++ aud_config = IBX_AUD_CFG(pipe); ++ aud_cntl_st = IBX_AUD_CNTL_ST(pipe); ++ aud_cntrl_st2 = IBX_AUD_CNTL_ST2; ++ } else if (IS_VALLEYVIEW(dev_priv) || ++ IS_CHERRYVIEW(dev_priv)) { ++ hdmiw_hdmiedid = VLV_HDMIW_HDMIEDID(pipe); ++ aud_config = VLV_AUD_CFG(pipe); ++ aud_cntl_st = VLV_AUD_CNTL_ST(pipe); ++ aud_cntrl_st2 = VLV_AUD_CNTL_ST2; ++ } else { ++ hdmiw_hdmiedid = CPT_HDMIW_HDMIEDID(pipe); ++ aud_config = CPT_AUD_CFG(pipe); ++ aud_cntl_st = CPT_AUD_CNTL_ST(pipe); ++ aud_cntrl_st2 = CPT_AUD_CNTRL_ST2; ++ } ++ ++ eldv = IBX_ELD_VALID(port); ++ ++ /* Invalidate ELD */ ++ tmp = I915_READ(aud_cntrl_st2); ++ tmp &= ~eldv; ++ I915_WRITE(aud_cntrl_st2, tmp); ++ ++ /* Reset ELD write address */ ++ tmp = I915_READ(aud_cntl_st); ++ tmp &= ~IBX_ELD_ADDRESS_MASK; ++ I915_WRITE(aud_cntl_st, tmp); ++ ++ /* Up to 84 bytes of hw ELD buffer */ ++ len = min(drm_eld_size(eld), 84); ++ for (i = 0; i < len / 4; i++) ++ I915_WRITE(hdmiw_hdmiedid, *((const u32 *)eld + i)); ++ ++ /* ELD valid */ ++ tmp = I915_READ(aud_cntrl_st2); ++ tmp |= eldv; ++ I915_WRITE(aud_cntrl_st2, tmp); ++ ++ /* Enable timestamps */ ++ tmp = I915_READ(aud_config); ++ tmp &= ~AUD_CONFIG_N_VALUE_INDEX; ++ tmp &= ~AUD_CONFIG_N_PROG_ENABLE; ++ tmp &= ~AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK; ++ if (intel_crtc_has_dp_encoder(crtc_state)) ++ tmp |= AUD_CONFIG_N_VALUE_INDEX; ++ else ++ tmp |= audio_config_hdmi_pixel_clock(crtc_state); ++ I915_WRITE(aud_config, tmp); ++} ++ ++/** ++ * intel_audio_codec_enable - Enable the audio codec for HD audio ++ * @encoder: encoder on which to enable audio ++ * @crtc_state: pointer to the current crtc state. ++ * @conn_state: pointer to the current connector state. ++ * ++ * The enable sequences may only be performed after enabling the transcoder and ++ * port, and after completed link training. ++ */ ++void intel_audio_codec_enable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct i915_audio_component *acomp = dev_priv->audio_component; ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_connector *connector = conn_state->connector; ++ const struct drm_display_mode *adjusted_mode = ++ &crtc_state->base.adjusted_mode; ++ enum port port = encoder->port; ++ enum pipe pipe = crtc->pipe; ++ ++ if (!connector->eld[0]) ++ return; ++ ++ DRM_DEBUG_DRIVER("ELD on [CONNECTOR:%d:%s], [ENCODER:%d:%s]\n", ++ connector->base.id, ++ connector->name, ++ connector->encoder->base.id, ++ connector->encoder->name); ++ ++ connector->eld[6] = drm_av_sync_delay(connector, adjusted_mode) / 2; ++ ++ if (dev_priv->display.audio_codec_enable) ++ dev_priv->display.audio_codec_enable(encoder, ++ crtc_state, ++ conn_state); ++ ++ mutex_lock(&dev_priv->av_mutex); ++ encoder->audio_connector = connector; ++ ++ /* referred in audio callbacks */ ++ dev_priv->av_enc_map[pipe] = encoder; ++ mutex_unlock(&dev_priv->av_mutex); ++ ++ if (acomp && acomp->base.audio_ops && ++ acomp->base.audio_ops->pin_eld_notify) { ++ /* audio drivers expect pipe = -1 to indicate Non-MST cases */ ++ if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST)) ++ pipe = -1; ++ acomp->base.audio_ops->pin_eld_notify(acomp->base.audio_ops->audio_ptr, ++ (int) port, (int) pipe); ++ } ++ ++ intel_lpe_audio_notify(dev_priv, pipe, port, connector->eld, ++ crtc_state->port_clock, ++ intel_crtc_has_dp_encoder(crtc_state)); ++} ++ ++/** ++ * intel_audio_codec_disable - Disable the audio codec for HD audio ++ * @encoder: encoder on which to disable audio ++ * @old_crtc_state: pointer to the old crtc state. ++ * @old_conn_state: pointer to the old connector state. ++ * ++ * The disable sequences must be performed before disabling the transcoder or ++ * port. ++ */ ++void intel_audio_codec_disable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct i915_audio_component *acomp = dev_priv->audio_component; ++ struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc); ++ enum port port = encoder->port; ++ enum pipe pipe = crtc->pipe; ++ ++ if (dev_priv->display.audio_codec_disable) ++ dev_priv->display.audio_codec_disable(encoder, ++ old_crtc_state, ++ old_conn_state); ++ ++ mutex_lock(&dev_priv->av_mutex); ++ encoder->audio_connector = NULL; ++ dev_priv->av_enc_map[pipe] = NULL; ++ mutex_unlock(&dev_priv->av_mutex); ++ ++ if (acomp && acomp->base.audio_ops && ++ acomp->base.audio_ops->pin_eld_notify) { ++ /* audio drivers expect pipe = -1 to indicate Non-MST cases */ ++ if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DP_MST)) ++ pipe = -1; ++ acomp->base.audio_ops->pin_eld_notify(acomp->base.audio_ops->audio_ptr, ++ (int) port, (int) pipe); ++ } ++ ++ intel_lpe_audio_notify(dev_priv, pipe, port, NULL, 0, false); ++} ++ ++/** ++ * intel_init_audio_hooks - Set up chip specific audio hooks ++ * @dev_priv: device private ++ */ ++void intel_init_audio_hooks(struct drm_i915_private *dev_priv) ++{ ++ if (IS_G4X(dev_priv)) { ++ dev_priv->display.audio_codec_enable = g4x_audio_codec_enable; ++ dev_priv->display.audio_codec_disable = g4x_audio_codec_disable; ++ } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { ++ dev_priv->display.audio_codec_enable = ilk_audio_codec_enable; ++ dev_priv->display.audio_codec_disable = ilk_audio_codec_disable; ++ } else if (IS_HASWELL(dev_priv) || INTEL_GEN(dev_priv) >= 8) { ++ dev_priv->display.audio_codec_enable = hsw_audio_codec_enable; ++ dev_priv->display.audio_codec_disable = hsw_audio_codec_disable; ++ } else if (HAS_PCH_SPLIT(dev_priv)) { ++ dev_priv->display.audio_codec_enable = ilk_audio_codec_enable; ++ dev_priv->display.audio_codec_disable = ilk_audio_codec_disable; ++ } ++} ++ ++static void glk_force_audio_cdclk(struct drm_i915_private *dev_priv, ++ bool enable) ++{ ++ struct drm_modeset_acquire_ctx ctx; ++ struct drm_atomic_state *state; ++ int ret; ++ ++ drm_modeset_acquire_init(&ctx, 0); ++ state = drm_atomic_state_alloc(&dev_priv->drm); ++ if (WARN_ON(!state)) ++ return; ++ ++ state->acquire_ctx = &ctx; ++ ++retry: ++ to_intel_atomic_state(state)->cdclk.force_min_cdclk_changed = true; ++ to_intel_atomic_state(state)->cdclk.force_min_cdclk = ++ enable ? 2 * 96000 : 0; ++ ++ /* ++ * Protects dev_priv->cdclk.force_min_cdclk ++ * Need to lock this here in case we have no active pipes ++ * and thus wouldn't lock it during the commit otherwise. ++ */ ++ ret = drm_modeset_lock(&dev_priv->drm.mode_config.connection_mutex, ++ &ctx); ++ if (!ret) ++ ret = drm_atomic_commit(state); ++ ++ if (ret == -EDEADLK) { ++ drm_atomic_state_clear(state); ++ drm_modeset_backoff(&ctx); ++ goto retry; ++ } ++ ++ WARN_ON(ret); ++ ++ drm_atomic_state_put(state); ++ ++ drm_modeset_drop_locks(&ctx); ++ drm_modeset_acquire_fini(&ctx); ++} ++ ++static unsigned long i915_audio_component_get_power(struct device *kdev) ++{ ++ struct drm_i915_private *dev_priv = kdev_to_i915(kdev); ++ intel_wakeref_t ret; ++ ++ /* Catch potential impedance mismatches before they occur! */ ++ BUILD_BUG_ON(sizeof(intel_wakeref_t) > sizeof(unsigned long)); ++ ++ ret = intel_display_power_get(dev_priv, POWER_DOMAIN_AUDIO); ++ ++ /* Force CDCLK to 2*BCLK as long as we need audio to be powered. */ ++ if (dev_priv->audio_power_refcount++ == 0) ++ if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv)) ++ glk_force_audio_cdclk(dev_priv, true); ++ ++ return ret; ++} ++ ++static void i915_audio_component_put_power(struct device *kdev, ++ unsigned long cookie) ++{ ++ struct drm_i915_private *dev_priv = kdev_to_i915(kdev); ++ ++ /* Stop forcing CDCLK to 2*BCLK if no need for audio to be powered. */ ++ if (--dev_priv->audio_power_refcount == 0) ++ if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv)) ++ glk_force_audio_cdclk(dev_priv, false); ++ ++ intel_display_power_put(dev_priv, POWER_DOMAIN_AUDIO, cookie); ++} ++ ++static void i915_audio_component_codec_wake_override(struct device *kdev, ++ bool enable) ++{ ++ struct drm_i915_private *dev_priv = kdev_to_i915(kdev); ++ unsigned long cookie; ++ u32 tmp; ++ ++ if (!IS_GEN(dev_priv, 9)) ++ return; ++ ++ cookie = i915_audio_component_get_power(kdev); ++ ++ /* ++ * Enable/disable generating the codec wake signal, overriding the ++ * internal logic to generate the codec wake to controller. ++ */ ++ tmp = I915_READ(HSW_AUD_CHICKENBIT); ++ tmp &= ~SKL_AUD_CODEC_WAKE_SIGNAL; ++ I915_WRITE(HSW_AUD_CHICKENBIT, tmp); ++ usleep_range(1000, 1500); ++ ++ if (enable) { ++ tmp = I915_READ(HSW_AUD_CHICKENBIT); ++ tmp |= SKL_AUD_CODEC_WAKE_SIGNAL; ++ I915_WRITE(HSW_AUD_CHICKENBIT, tmp); ++ usleep_range(1000, 1500); ++ } ++ ++ i915_audio_component_put_power(kdev, cookie); ++} ++ ++/* Get CDCLK in kHz */ ++static int i915_audio_component_get_cdclk_freq(struct device *kdev) ++{ ++ struct drm_i915_private *dev_priv = kdev_to_i915(kdev); ++ ++ if (WARN_ON_ONCE(!HAS_DDI(dev_priv))) ++ return -ENODEV; ++ ++ return dev_priv->cdclk.hw.cdclk; ++} ++ ++/* ++ * get the intel_encoder according to the parameter port and pipe ++ * intel_encoder is saved by the index of pipe ++ * MST & (pipe >= 0): return the av_enc_map[pipe], ++ * when port is matched ++ * MST & (pipe < 0): this is invalid ++ * Non-MST & (pipe >= 0): only pipe = 0 (the first device entry) ++ * will get the right intel_encoder with port matched ++ * Non-MST & (pipe < 0): get the right intel_encoder with port matched ++ */ ++static struct intel_encoder *get_saved_enc(struct drm_i915_private *dev_priv, ++ int port, int pipe) ++{ ++ struct intel_encoder *encoder; ++ ++ /* MST */ ++ if (pipe >= 0) { ++ if (WARN_ON(pipe >= ARRAY_SIZE(dev_priv->av_enc_map))) ++ return NULL; ++ ++ encoder = dev_priv->av_enc_map[pipe]; ++ /* ++ * when bootup, audio driver may not know it is ++ * MST or not. So it will poll all the port & pipe ++ * combinations ++ */ ++ if (encoder != NULL && encoder->port == port && ++ encoder->type == INTEL_OUTPUT_DP_MST) ++ return encoder; ++ } ++ ++ /* Non-MST */ ++ if (pipe > 0) ++ return NULL; ++ ++ for_each_pipe(dev_priv, pipe) { ++ encoder = dev_priv->av_enc_map[pipe]; ++ if (encoder == NULL) ++ continue; ++ ++ if (encoder->type == INTEL_OUTPUT_DP_MST) ++ continue; ++ ++ if (port == encoder->port) ++ return encoder; ++ } ++ ++ return NULL; ++} ++ ++static int i915_audio_component_sync_audio_rate(struct device *kdev, int port, ++ int pipe, int rate) ++{ ++ struct drm_i915_private *dev_priv = kdev_to_i915(kdev); ++ struct i915_audio_component *acomp = dev_priv->audio_component; ++ struct intel_encoder *encoder; ++ struct intel_crtc *crtc; ++ unsigned long cookie; ++ int err = 0; ++ ++ if (!HAS_DDI(dev_priv)) ++ return 0; ++ ++ cookie = i915_audio_component_get_power(kdev); ++ mutex_lock(&dev_priv->av_mutex); ++ ++ /* 1. get the pipe */ ++ encoder = get_saved_enc(dev_priv, port, pipe); ++ if (!encoder || !encoder->base.crtc) { ++ DRM_DEBUG_KMS("Not valid for port %c\n", port_name(port)); ++ err = -ENODEV; ++ goto unlock; ++ } ++ ++ crtc = to_intel_crtc(encoder->base.crtc); ++ ++ /* port must be valid now, otherwise the pipe will be invalid */ ++ acomp->aud_sample_rate[port] = rate; ++ ++ hsw_audio_config_update(encoder, crtc->config); ++ ++ unlock: ++ mutex_unlock(&dev_priv->av_mutex); ++ i915_audio_component_put_power(kdev, cookie); ++ return err; ++} ++ ++static int i915_audio_component_get_eld(struct device *kdev, int port, ++ int pipe, bool *enabled, ++ unsigned char *buf, int max_bytes) ++{ ++ struct drm_i915_private *dev_priv = kdev_to_i915(kdev); ++ struct intel_encoder *intel_encoder; ++ const u8 *eld; ++ int ret = -EINVAL; ++ ++ mutex_lock(&dev_priv->av_mutex); ++ ++ intel_encoder = get_saved_enc(dev_priv, port, pipe); ++ if (!intel_encoder) { ++ DRM_DEBUG_KMS("Not valid for port %c\n", port_name(port)); ++ mutex_unlock(&dev_priv->av_mutex); ++ return ret; ++ } ++ ++ ret = 0; ++ *enabled = intel_encoder->audio_connector != NULL; ++ if (*enabled) { ++ eld = intel_encoder->audio_connector->eld; ++ ret = drm_eld_size(eld); ++ memcpy(buf, eld, min(max_bytes, ret)); ++ } ++ ++ mutex_unlock(&dev_priv->av_mutex); ++ return ret; ++} ++ ++static const struct drm_audio_component_ops i915_audio_component_ops = { ++ .owner = THIS_MODULE, ++ .get_power = i915_audio_component_get_power, ++ .put_power = i915_audio_component_put_power, ++ .codec_wake_override = i915_audio_component_codec_wake_override, ++ .get_cdclk_freq = i915_audio_component_get_cdclk_freq, ++ .sync_audio_rate = i915_audio_component_sync_audio_rate, ++ .get_eld = i915_audio_component_get_eld, ++}; ++ ++static int i915_audio_component_bind(struct device *i915_kdev, ++ struct device *hda_kdev, void *data) ++{ ++ struct i915_audio_component *acomp = data; ++ struct drm_i915_private *dev_priv = kdev_to_i915(i915_kdev); ++ int i; ++ ++ if (WARN_ON(acomp->base.ops || acomp->base.dev)) ++ return -EEXIST; ++ ++ if (WARN_ON(!device_link_add(hda_kdev, i915_kdev, DL_FLAG_STATELESS))) ++ return -ENOMEM; ++ ++ drm_modeset_lock_all(&dev_priv->drm); ++ acomp->base.ops = &i915_audio_component_ops; ++ acomp->base.dev = i915_kdev; ++ BUILD_BUG_ON(MAX_PORTS != I915_MAX_PORTS); ++ for (i = 0; i < ARRAY_SIZE(acomp->aud_sample_rate); i++) ++ acomp->aud_sample_rate[i] = 0; ++ dev_priv->audio_component = acomp; ++ drm_modeset_unlock_all(&dev_priv->drm); ++ ++ return 0; ++} ++ ++static void i915_audio_component_unbind(struct device *i915_kdev, ++ struct device *hda_kdev, void *data) ++{ ++ struct i915_audio_component *acomp = data; ++ struct drm_i915_private *dev_priv = kdev_to_i915(i915_kdev); ++ ++ drm_modeset_lock_all(&dev_priv->drm); ++ acomp->base.ops = NULL; ++ acomp->base.dev = NULL; ++ dev_priv->audio_component = NULL; ++ drm_modeset_unlock_all(&dev_priv->drm); ++ ++ device_link_remove(hda_kdev, i915_kdev); ++} ++ ++static const struct component_ops i915_audio_component_bind_ops = { ++ .bind = i915_audio_component_bind, ++ .unbind = i915_audio_component_unbind, ++}; ++ ++/** ++ * i915_audio_component_init - initialize and register the audio component ++ * @dev_priv: i915 device instance ++ * ++ * This will register with the component framework a child component which ++ * will bind dynamically to the snd_hda_intel driver's corresponding master ++ * component when the latter is registered. During binding the child ++ * initializes an instance of struct i915_audio_component which it receives ++ * from the master. The master can then start to use the interface defined by ++ * this struct. Each side can break the binding at any point by deregistering ++ * its own component after which each side's component unbind callback is ++ * called. ++ * ++ * We ignore any error during registration and continue with reduced ++ * functionality (i.e. without HDMI audio). ++ */ ++static void i915_audio_component_init(struct drm_i915_private *dev_priv) ++{ ++ int ret; ++ ++ ret = component_add_typed(dev_priv->drm.dev, ++ &i915_audio_component_bind_ops, ++ I915_COMPONENT_AUDIO); ++ if (ret < 0) { ++ DRM_ERROR("failed to add audio component (%d)\n", ret); ++ /* continue with reduced functionality */ ++ return; ++ } ++ ++ dev_priv->audio_component_registered = true; ++} ++ ++/** ++ * i915_audio_component_cleanup - deregister the audio component ++ * @dev_priv: i915 device instance ++ * ++ * Deregisters the audio component, breaking any existing binding to the ++ * corresponding snd_hda_intel driver's master component. ++ */ ++static void i915_audio_component_cleanup(struct drm_i915_private *dev_priv) ++{ ++ if (!dev_priv->audio_component_registered) ++ return; ++ ++ component_del(dev_priv->drm.dev, &i915_audio_component_bind_ops); ++ dev_priv->audio_component_registered = false; ++} ++ ++/** ++ * intel_audio_init() - Initialize the audio driver either using ++ * component framework or using lpe audio bridge ++ * @dev_priv: the i915 drm device private data ++ * ++ */ ++void intel_audio_init(struct drm_i915_private *dev_priv) ++{ ++ if (intel_lpe_audio_init(dev_priv) < 0) ++ i915_audio_component_init(dev_priv); ++} ++ ++/** ++ * intel_audio_deinit() - deinitialize the audio driver ++ * @dev_priv: the i915 drm device private data ++ * ++ */ ++void intel_audio_deinit(struct drm_i915_private *dev_priv) ++{ ++ if ((dev_priv)->lpe_audio.platdev != NULL) ++ intel_lpe_audio_teardown(dev_priv); ++ else ++ i915_audio_component_cleanup(dev_priv); ++} +diff --git a/drivers/gpu/drm/i915_legacy/intel_audio.h b/drivers/gpu/drm/i915_legacy/intel_audio.h +new file mode 100644 +index 000000000000..a3657c7a7ba2 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_audio.h +@@ -0,0 +1,24 @@ ++/* SPDX-License-Identifier: MIT */ ++/* ++ * Copyright © 2019 Intel Corporation ++ */ ++ ++#ifndef __INTEL_AUDIO_H__ ++#define __INTEL_AUDIO_H__ ++ ++struct drm_connector_state; ++struct drm_i915_private; ++struct intel_crtc_state; ++struct intel_encoder; ++ ++void intel_init_audio_hooks(struct drm_i915_private *dev_priv); ++void intel_audio_codec_enable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state); ++void intel_audio_codec_disable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state); ++void intel_audio_init(struct drm_i915_private *dev_priv); ++void intel_audio_deinit(struct drm_i915_private *dev_priv); ++ ++#endif /* __INTEL_AUDIO_H__ */ +diff --git a/drivers/gpu/drm/i915_legacy/intel_bios.c b/drivers/gpu/drm/i915_legacy/intel_bios.c +new file mode 100644 +index 000000000000..ee6fa75d65a2 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_bios.c +@@ -0,0 +1,2298 @@ ++/* ++ * Copyright © 2006 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ * ++ * Authors: ++ * Eric Anholt ++ * ++ */ ++ ++#include ++#include ++#include "i915_drv.h" ++ ++#define _INTEL_BIOS_PRIVATE ++#include "intel_vbt_defs.h" ++ ++/** ++ * DOC: Video BIOS Table (VBT) ++ * ++ * The Video BIOS Table, or VBT, provides platform and board specific ++ * configuration information to the driver that is not discoverable or available ++ * through other means. The configuration is mostly related to display ++ * hardware. The VBT is available via the ACPI OpRegion or, on older systems, in ++ * the PCI ROM. ++ * ++ * The VBT consists of a VBT Header (defined as &struct vbt_header), a BDB ++ * Header (&struct bdb_header), and a number of BIOS Data Blocks (BDB) that ++ * contain the actual configuration information. The VBT Header, and thus the ++ * VBT, begins with "$VBT" signature. The VBT Header contains the offset of the ++ * BDB Header. The data blocks are concatenated after the BDB Header. The data ++ * blocks have a 1-byte Block ID, 2-byte Block Size, and Block Size bytes of ++ * data. (Block 53, the MIPI Sequence Block is an exception.) ++ * ++ * The driver parses the VBT during load. The relevant information is stored in ++ * driver private data for ease of use, and the actual VBT is not read after ++ * that. ++ */ ++ ++#define SLAVE_ADDR1 0x70 ++#define SLAVE_ADDR2 0x72 ++ ++/* Get BDB block size given a pointer to Block ID. */ ++static u32 _get_blocksize(const u8 *block_base) ++{ ++ /* The MIPI Sequence Block v3+ has a separate size field. */ ++ if (*block_base == BDB_MIPI_SEQUENCE && *(block_base + 3) >= 3) ++ return *((const u32 *)(block_base + 4)); ++ else ++ return *((const u16 *)(block_base + 1)); ++} ++ ++/* Get BDB block size give a pointer to data after Block ID and Block Size. */ ++static u32 get_blocksize(const void *block_data) ++{ ++ return _get_blocksize(block_data - 3); ++} ++ ++static const void * ++find_section(const void *_bdb, int section_id) ++{ ++ const struct bdb_header *bdb = _bdb; ++ const u8 *base = _bdb; ++ int index = 0; ++ u32 total, current_size; ++ u8 current_id; ++ ++ /* skip to first section */ ++ index += bdb->header_size; ++ total = bdb->bdb_size; ++ ++ /* walk the sections looking for section_id */ ++ while (index + 3 < total) { ++ current_id = *(base + index); ++ current_size = _get_blocksize(base + index); ++ index += 3; ++ ++ if (index + current_size > total) ++ return NULL; ++ ++ if (current_id == section_id) ++ return base + index; ++ ++ index += current_size; ++ } ++ ++ return NULL; ++} ++ ++static void ++fill_detail_timing_data(struct drm_display_mode *panel_fixed_mode, ++ const struct lvds_dvo_timing *dvo_timing) ++{ ++ panel_fixed_mode->hdisplay = (dvo_timing->hactive_hi << 8) | ++ dvo_timing->hactive_lo; ++ panel_fixed_mode->hsync_start = panel_fixed_mode->hdisplay + ++ ((dvo_timing->hsync_off_hi << 8) | dvo_timing->hsync_off_lo); ++ panel_fixed_mode->hsync_end = panel_fixed_mode->hsync_start + ++ ((dvo_timing->hsync_pulse_width_hi << 8) | ++ dvo_timing->hsync_pulse_width_lo); ++ panel_fixed_mode->htotal = panel_fixed_mode->hdisplay + ++ ((dvo_timing->hblank_hi << 8) | dvo_timing->hblank_lo); ++ ++ panel_fixed_mode->vdisplay = (dvo_timing->vactive_hi << 8) | ++ dvo_timing->vactive_lo; ++ panel_fixed_mode->vsync_start = panel_fixed_mode->vdisplay + ++ ((dvo_timing->vsync_off_hi << 4) | dvo_timing->vsync_off_lo); ++ panel_fixed_mode->vsync_end = panel_fixed_mode->vsync_start + ++ ((dvo_timing->vsync_pulse_width_hi << 4) | ++ dvo_timing->vsync_pulse_width_lo); ++ panel_fixed_mode->vtotal = panel_fixed_mode->vdisplay + ++ ((dvo_timing->vblank_hi << 8) | dvo_timing->vblank_lo); ++ panel_fixed_mode->clock = dvo_timing->clock * 10; ++ panel_fixed_mode->type = DRM_MODE_TYPE_PREFERRED; ++ ++ if (dvo_timing->hsync_positive) ++ panel_fixed_mode->flags |= DRM_MODE_FLAG_PHSYNC; ++ else ++ panel_fixed_mode->flags |= DRM_MODE_FLAG_NHSYNC; ++ ++ if (dvo_timing->vsync_positive) ++ panel_fixed_mode->flags |= DRM_MODE_FLAG_PVSYNC; ++ else ++ panel_fixed_mode->flags |= DRM_MODE_FLAG_NVSYNC; ++ ++ panel_fixed_mode->width_mm = (dvo_timing->himage_hi << 8) | ++ dvo_timing->himage_lo; ++ panel_fixed_mode->height_mm = (dvo_timing->vimage_hi << 8) | ++ dvo_timing->vimage_lo; ++ ++ /* Some VBTs have bogus h/vtotal values */ ++ if (panel_fixed_mode->hsync_end > panel_fixed_mode->htotal) ++ panel_fixed_mode->htotal = panel_fixed_mode->hsync_end + 1; ++ if (panel_fixed_mode->vsync_end > panel_fixed_mode->vtotal) ++ panel_fixed_mode->vtotal = panel_fixed_mode->vsync_end + 1; ++ ++ drm_mode_set_name(panel_fixed_mode); ++} ++ ++static const struct lvds_dvo_timing * ++get_lvds_dvo_timing(const struct bdb_lvds_lfp_data *lvds_lfp_data, ++ const struct bdb_lvds_lfp_data_ptrs *lvds_lfp_data_ptrs, ++ int index) ++{ ++ /* ++ * the size of fp_timing varies on the different platform. ++ * So calculate the DVO timing relative offset in LVDS data ++ * entry to get the DVO timing entry ++ */ ++ ++ int lfp_data_size = ++ lvds_lfp_data_ptrs->ptr[1].dvo_timing_offset - ++ lvds_lfp_data_ptrs->ptr[0].dvo_timing_offset; ++ int dvo_timing_offset = ++ lvds_lfp_data_ptrs->ptr[0].dvo_timing_offset - ++ lvds_lfp_data_ptrs->ptr[0].fp_timing_offset; ++ char *entry = (char *)lvds_lfp_data->data + lfp_data_size * index; ++ ++ return (struct lvds_dvo_timing *)(entry + dvo_timing_offset); ++} ++ ++/* get lvds_fp_timing entry ++ * this function may return NULL if the corresponding entry is invalid ++ */ ++static const struct lvds_fp_timing * ++get_lvds_fp_timing(const struct bdb_header *bdb, ++ const struct bdb_lvds_lfp_data *data, ++ const struct bdb_lvds_lfp_data_ptrs *ptrs, ++ int index) ++{ ++ size_t data_ofs = (const u8 *)data - (const u8 *)bdb; ++ u16 data_size = ((const u16 *)data)[-1]; /* stored in header */ ++ size_t ofs; ++ ++ if (index >= ARRAY_SIZE(ptrs->ptr)) ++ return NULL; ++ ofs = ptrs->ptr[index].fp_timing_offset; ++ if (ofs < data_ofs || ++ ofs + sizeof(struct lvds_fp_timing) > data_ofs + data_size) ++ return NULL; ++ return (const struct lvds_fp_timing *)((const u8 *)bdb + ofs); ++} ++ ++/* Try to find integrated panel data */ ++static void ++parse_lfp_panel_data(struct drm_i915_private *dev_priv, ++ const struct bdb_header *bdb) ++{ ++ const struct bdb_lvds_options *lvds_options; ++ const struct bdb_lvds_lfp_data *lvds_lfp_data; ++ const struct bdb_lvds_lfp_data_ptrs *lvds_lfp_data_ptrs; ++ const struct lvds_dvo_timing *panel_dvo_timing; ++ const struct lvds_fp_timing *fp_timing; ++ struct drm_display_mode *panel_fixed_mode; ++ int panel_type; ++ int drrs_mode; ++ int ret; ++ ++ lvds_options = find_section(bdb, BDB_LVDS_OPTIONS); ++ if (!lvds_options) ++ return; ++ ++ dev_priv->vbt.lvds_dither = lvds_options->pixel_dither; ++ ++ ret = intel_opregion_get_panel_type(dev_priv); ++ if (ret >= 0) { ++ WARN_ON(ret > 0xf); ++ panel_type = ret; ++ DRM_DEBUG_KMS("Panel type: %d (OpRegion)\n", panel_type); ++ } else { ++ if (lvds_options->panel_type > 0xf) { ++ DRM_DEBUG_KMS("Invalid VBT panel type 0x%x\n", ++ lvds_options->panel_type); ++ return; ++ } ++ panel_type = lvds_options->panel_type; ++ DRM_DEBUG_KMS("Panel type: %d (VBT)\n", panel_type); ++ } ++ ++ dev_priv->vbt.panel_type = panel_type; ++ ++ drrs_mode = (lvds_options->dps_panel_type_bits ++ >> (panel_type * 2)) & MODE_MASK; ++ /* ++ * VBT has static DRRS = 0 and seamless DRRS = 2. ++ * The below piece of code is required to adjust vbt.drrs_type ++ * to match the enum drrs_support_type. ++ */ ++ switch (drrs_mode) { ++ case 0: ++ dev_priv->vbt.drrs_type = STATIC_DRRS_SUPPORT; ++ DRM_DEBUG_KMS("DRRS supported mode is static\n"); ++ break; ++ case 2: ++ dev_priv->vbt.drrs_type = SEAMLESS_DRRS_SUPPORT; ++ DRM_DEBUG_KMS("DRRS supported mode is seamless\n"); ++ break; ++ default: ++ dev_priv->vbt.drrs_type = DRRS_NOT_SUPPORTED; ++ DRM_DEBUG_KMS("DRRS not supported (VBT input)\n"); ++ break; ++ } ++ ++ lvds_lfp_data = find_section(bdb, BDB_LVDS_LFP_DATA); ++ if (!lvds_lfp_data) ++ return; ++ ++ lvds_lfp_data_ptrs = find_section(bdb, BDB_LVDS_LFP_DATA_PTRS); ++ if (!lvds_lfp_data_ptrs) ++ return; ++ ++ panel_dvo_timing = get_lvds_dvo_timing(lvds_lfp_data, ++ lvds_lfp_data_ptrs, ++ panel_type); ++ ++ panel_fixed_mode = kzalloc(sizeof(*panel_fixed_mode), GFP_KERNEL); ++ if (!panel_fixed_mode) ++ return; ++ ++ fill_detail_timing_data(panel_fixed_mode, panel_dvo_timing); ++ ++ dev_priv->vbt.lfp_lvds_vbt_mode = panel_fixed_mode; ++ ++ DRM_DEBUG_KMS("Found panel mode in BIOS VBT tables:\n"); ++ drm_mode_debug_printmodeline(panel_fixed_mode); ++ ++ fp_timing = get_lvds_fp_timing(bdb, lvds_lfp_data, ++ lvds_lfp_data_ptrs, ++ panel_type); ++ if (fp_timing) { ++ /* check the resolution, just to be sure */ ++ if (fp_timing->x_res == panel_fixed_mode->hdisplay && ++ fp_timing->y_res == panel_fixed_mode->vdisplay) { ++ dev_priv->vbt.bios_lvds_val = fp_timing->lvds_reg_val; ++ DRM_DEBUG_KMS("VBT initial LVDS value %x\n", ++ dev_priv->vbt.bios_lvds_val); ++ } ++ } ++} ++ ++static void ++parse_lfp_backlight(struct drm_i915_private *dev_priv, ++ const struct bdb_header *bdb) ++{ ++ const struct bdb_lfp_backlight_data *backlight_data; ++ const struct bdb_lfp_backlight_data_entry *entry; ++ int panel_type = dev_priv->vbt.panel_type; ++ ++ backlight_data = find_section(bdb, BDB_LVDS_BACKLIGHT); ++ if (!backlight_data) ++ return; ++ ++ if (backlight_data->entry_size != sizeof(backlight_data->data[0])) { ++ DRM_DEBUG_KMS("Unsupported backlight data entry size %u\n", ++ backlight_data->entry_size); ++ return; ++ } ++ ++ entry = &backlight_data->data[panel_type]; ++ ++ dev_priv->vbt.backlight.present = entry->type == BDB_BACKLIGHT_TYPE_PWM; ++ if (!dev_priv->vbt.backlight.present) { ++ DRM_DEBUG_KMS("PWM backlight not present in VBT (type %u)\n", ++ entry->type); ++ return; ++ } ++ ++ dev_priv->vbt.backlight.type = INTEL_BACKLIGHT_DISPLAY_DDI; ++ if (bdb->version >= 191 && ++ get_blocksize(backlight_data) >= sizeof(*backlight_data)) { ++ const struct bdb_lfp_backlight_control_method *method; ++ ++ method = &backlight_data->backlight_control[panel_type]; ++ dev_priv->vbt.backlight.type = method->type; ++ dev_priv->vbt.backlight.controller = method->controller; ++ } ++ ++ dev_priv->vbt.backlight.pwm_freq_hz = entry->pwm_freq_hz; ++ dev_priv->vbt.backlight.active_low_pwm = entry->active_low_pwm; ++ dev_priv->vbt.backlight.min_brightness = entry->min_brightness; ++ DRM_DEBUG_KMS("VBT backlight PWM modulation frequency %u Hz, " ++ "active %s, min brightness %u, level %u, controller %u\n", ++ dev_priv->vbt.backlight.pwm_freq_hz, ++ dev_priv->vbt.backlight.active_low_pwm ? "low" : "high", ++ dev_priv->vbt.backlight.min_brightness, ++ backlight_data->level[panel_type], ++ dev_priv->vbt.backlight.controller); ++} ++ ++/* Try to find sdvo panel data */ ++static void ++parse_sdvo_panel_data(struct drm_i915_private *dev_priv, ++ const struct bdb_header *bdb) ++{ ++ const struct lvds_dvo_timing *dvo_timing; ++ struct drm_display_mode *panel_fixed_mode; ++ int index; ++ ++ index = i915_modparams.vbt_sdvo_panel_type; ++ if (index == -2) { ++ DRM_DEBUG_KMS("Ignore SDVO panel mode from BIOS VBT tables.\n"); ++ return; ++ } ++ ++ if (index == -1) { ++ const struct bdb_sdvo_lvds_options *sdvo_lvds_options; ++ ++ sdvo_lvds_options = find_section(bdb, BDB_SDVO_LVDS_OPTIONS); ++ if (!sdvo_lvds_options) ++ return; ++ ++ index = sdvo_lvds_options->panel_type; ++ } ++ ++ dvo_timing = find_section(bdb, BDB_SDVO_PANEL_DTDS); ++ if (!dvo_timing) ++ return; ++ ++ panel_fixed_mode = kzalloc(sizeof(*panel_fixed_mode), GFP_KERNEL); ++ if (!panel_fixed_mode) ++ return; ++ ++ fill_detail_timing_data(panel_fixed_mode, dvo_timing + index); ++ ++ dev_priv->vbt.sdvo_lvds_vbt_mode = panel_fixed_mode; ++ ++ DRM_DEBUG_KMS("Found SDVO panel mode in BIOS VBT tables:\n"); ++ drm_mode_debug_printmodeline(panel_fixed_mode); ++} ++ ++static int intel_bios_ssc_frequency(struct drm_i915_private *dev_priv, ++ bool alternate) ++{ ++ switch (INTEL_GEN(dev_priv)) { ++ case 2: ++ return alternate ? 66667 : 48000; ++ case 3: ++ case 4: ++ return alternate ? 100000 : 96000; ++ default: ++ return alternate ? 100000 : 120000; ++ } ++} ++ ++static void ++parse_general_features(struct drm_i915_private *dev_priv, ++ const struct bdb_header *bdb) ++{ ++ const struct bdb_general_features *general; ++ ++ general = find_section(bdb, BDB_GENERAL_FEATURES); ++ if (!general) ++ return; ++ ++ dev_priv->vbt.int_tv_support = general->int_tv_support; ++ /* int_crt_support can't be trusted on earlier platforms */ ++ if (bdb->version >= 155 && ++ (HAS_DDI(dev_priv) || IS_VALLEYVIEW(dev_priv))) ++ dev_priv->vbt.int_crt_support = general->int_crt_support; ++ dev_priv->vbt.lvds_use_ssc = general->enable_ssc; ++ dev_priv->vbt.lvds_ssc_freq = ++ intel_bios_ssc_frequency(dev_priv, general->ssc_freq); ++ dev_priv->vbt.display_clock_mode = general->display_clock_mode; ++ dev_priv->vbt.fdi_rx_polarity_inverted = general->fdi_rx_polarity_inverted; ++ if (bdb->version >= 181) { ++ dev_priv->vbt.orientation = general->rotate_180 ? ++ DRM_MODE_PANEL_ORIENTATION_BOTTOM_UP : ++ DRM_MODE_PANEL_ORIENTATION_NORMAL; ++ } else { ++ dev_priv->vbt.orientation = DRM_MODE_PANEL_ORIENTATION_UNKNOWN; ++ } ++ DRM_DEBUG_KMS("BDB_GENERAL_FEATURES int_tv_support %d int_crt_support %d lvds_use_ssc %d lvds_ssc_freq %d display_clock_mode %d fdi_rx_polarity_inverted %d\n", ++ dev_priv->vbt.int_tv_support, ++ dev_priv->vbt.int_crt_support, ++ dev_priv->vbt.lvds_use_ssc, ++ dev_priv->vbt.lvds_ssc_freq, ++ dev_priv->vbt.display_clock_mode, ++ dev_priv->vbt.fdi_rx_polarity_inverted); ++} ++ ++static const struct child_device_config * ++child_device_ptr(const struct bdb_general_definitions *defs, int i) ++{ ++ return (const void *) &defs->devices[i * defs->child_dev_size]; ++} ++ ++static void ++parse_sdvo_device_mapping(struct drm_i915_private *dev_priv, u8 bdb_version) ++{ ++ struct sdvo_device_mapping *mapping; ++ const struct child_device_config *child; ++ int i, count = 0; ++ ++ /* ++ * Only parse SDVO mappings on gens that could have SDVO. This isn't ++ * accurate and doesn't have to be, as long as it's not too strict. ++ */ ++ if (!IS_GEN_RANGE(dev_priv, 3, 7)) { ++ DRM_DEBUG_KMS("Skipping SDVO device mapping\n"); ++ return; ++ } ++ ++ for (i = 0, count = 0; i < dev_priv->vbt.child_dev_num; i++) { ++ child = dev_priv->vbt.child_dev + i; ++ ++ if (child->slave_addr != SLAVE_ADDR1 && ++ child->slave_addr != SLAVE_ADDR2) { ++ /* ++ * If the slave address is neither 0x70 nor 0x72, ++ * it is not a SDVO device. Skip it. ++ */ ++ continue; ++ } ++ if (child->dvo_port != DEVICE_PORT_DVOB && ++ child->dvo_port != DEVICE_PORT_DVOC) { ++ /* skip the incorrect SDVO port */ ++ DRM_DEBUG_KMS("Incorrect SDVO port. Skip it\n"); ++ continue; ++ } ++ DRM_DEBUG_KMS("the SDVO device with slave addr %2x is found on" ++ " %s port\n", ++ child->slave_addr, ++ (child->dvo_port == DEVICE_PORT_DVOB) ? ++ "SDVOB" : "SDVOC"); ++ mapping = &dev_priv->vbt.sdvo_mappings[child->dvo_port - 1]; ++ if (!mapping->initialized) { ++ mapping->dvo_port = child->dvo_port; ++ mapping->slave_addr = child->slave_addr; ++ mapping->dvo_wiring = child->dvo_wiring; ++ mapping->ddc_pin = child->ddc_pin; ++ mapping->i2c_pin = child->i2c_pin; ++ mapping->initialized = 1; ++ DRM_DEBUG_KMS("SDVO device: dvo=%x, addr=%x, wiring=%d, ddc_pin=%d, i2c_pin=%d\n", ++ mapping->dvo_port, ++ mapping->slave_addr, ++ mapping->dvo_wiring, ++ mapping->ddc_pin, ++ mapping->i2c_pin); ++ } else { ++ DRM_DEBUG_KMS("Maybe one SDVO port is shared by " ++ "two SDVO device.\n"); ++ } ++ if (child->slave2_addr) { ++ /* Maybe this is a SDVO device with multiple inputs */ ++ /* And the mapping info is not added */ ++ DRM_DEBUG_KMS("there exists the slave2_addr. Maybe this" ++ " is a SDVO device with multiple inputs.\n"); ++ } ++ count++; ++ } ++ ++ if (!count) { ++ /* No SDVO device info is found */ ++ DRM_DEBUG_KMS("No SDVO device info is found in VBT\n"); ++ } ++} ++ ++static void ++parse_driver_features(struct drm_i915_private *dev_priv, ++ const struct bdb_header *bdb) ++{ ++ const struct bdb_driver_features *driver; ++ ++ driver = find_section(bdb, BDB_DRIVER_FEATURES); ++ if (!driver) ++ return; ++ ++ if (INTEL_GEN(dev_priv) >= 5) { ++ /* ++ * Note that we consider BDB_DRIVER_FEATURE_INT_SDVO_LVDS ++ * to mean "eDP". The VBT spec doesn't agree with that ++ * interpretation, but real world VBTs seem to. ++ */ ++ if (driver->lvds_config != BDB_DRIVER_FEATURE_INT_LVDS) ++ dev_priv->vbt.int_lvds_support = 0; ++ } else { ++ /* ++ * FIXME it's not clear which BDB version has the LVDS config ++ * bits defined. Revision history in the VBT spec says: ++ * "0.92 | Add two definitions for VBT value of LVDS Active ++ * Config (00b and 11b values defined) | 06/13/2005" ++ * but does not the specify the BDB version. ++ * ++ * So far version 134 (on i945gm) is the oldest VBT observed ++ * in the wild with the bits correctly populated. Version ++ * 108 (on i85x) does not have the bits correctly populated. ++ */ ++ if (bdb->version >= 134 && ++ driver->lvds_config != BDB_DRIVER_FEATURE_INT_LVDS && ++ driver->lvds_config != BDB_DRIVER_FEATURE_INT_SDVO_LVDS) ++ dev_priv->vbt.int_lvds_support = 0; ++ } ++ ++ DRM_DEBUG_KMS("DRRS State Enabled:%d\n", driver->drrs_enabled); ++ /* ++ * If DRRS is not supported, drrs_type has to be set to 0. ++ * This is because, VBT is configured in such a way that ++ * static DRRS is 0 and DRRS not supported is represented by ++ * driver->drrs_enabled=false ++ */ ++ if (!driver->drrs_enabled) ++ dev_priv->vbt.drrs_type = DRRS_NOT_SUPPORTED; ++ dev_priv->vbt.psr.enable = driver->psr_enabled; ++} ++ ++static void ++parse_edp(struct drm_i915_private *dev_priv, const struct bdb_header *bdb) ++{ ++ const struct bdb_edp *edp; ++ const struct edp_power_seq *edp_pps; ++ const struct edp_fast_link_params *edp_link_params; ++ int panel_type = dev_priv->vbt.panel_type; ++ ++ edp = find_section(bdb, BDB_EDP); ++ if (!edp) ++ return; ++ ++ switch ((edp->color_depth >> (panel_type * 2)) & 3) { ++ case EDP_18BPP: ++ dev_priv->vbt.edp.bpp = 18; ++ break; ++ case EDP_24BPP: ++ dev_priv->vbt.edp.bpp = 24; ++ break; ++ case EDP_30BPP: ++ dev_priv->vbt.edp.bpp = 30; ++ break; ++ } ++ ++ /* Get the eDP sequencing and link info */ ++ edp_pps = &edp->power_seqs[panel_type]; ++ edp_link_params = &edp->fast_link_params[panel_type]; ++ ++ dev_priv->vbt.edp.pps = *edp_pps; ++ ++ switch (edp_link_params->rate) { ++ case EDP_RATE_1_62: ++ dev_priv->vbt.edp.rate = DP_LINK_BW_1_62; ++ break; ++ case EDP_RATE_2_7: ++ dev_priv->vbt.edp.rate = DP_LINK_BW_2_7; ++ break; ++ default: ++ DRM_DEBUG_KMS("VBT has unknown eDP link rate value %u\n", ++ edp_link_params->rate); ++ break; ++ } ++ ++ switch (edp_link_params->lanes) { ++ case EDP_LANE_1: ++ dev_priv->vbt.edp.lanes = 1; ++ break; ++ case EDP_LANE_2: ++ dev_priv->vbt.edp.lanes = 2; ++ break; ++ case EDP_LANE_4: ++ dev_priv->vbt.edp.lanes = 4; ++ break; ++ default: ++ DRM_DEBUG_KMS("VBT has unknown eDP lane count value %u\n", ++ edp_link_params->lanes); ++ break; ++ } ++ ++ switch (edp_link_params->preemphasis) { ++ case EDP_PREEMPHASIS_NONE: ++ dev_priv->vbt.edp.preemphasis = DP_TRAIN_PRE_EMPH_LEVEL_0; ++ break; ++ case EDP_PREEMPHASIS_3_5dB: ++ dev_priv->vbt.edp.preemphasis = DP_TRAIN_PRE_EMPH_LEVEL_1; ++ break; ++ case EDP_PREEMPHASIS_6dB: ++ dev_priv->vbt.edp.preemphasis = DP_TRAIN_PRE_EMPH_LEVEL_2; ++ break; ++ case EDP_PREEMPHASIS_9_5dB: ++ dev_priv->vbt.edp.preemphasis = DP_TRAIN_PRE_EMPH_LEVEL_3; ++ break; ++ default: ++ DRM_DEBUG_KMS("VBT has unknown eDP pre-emphasis value %u\n", ++ edp_link_params->preemphasis); ++ break; ++ } ++ ++ switch (edp_link_params->vswing) { ++ case EDP_VSWING_0_4V: ++ dev_priv->vbt.edp.vswing = DP_TRAIN_VOLTAGE_SWING_LEVEL_0; ++ break; ++ case EDP_VSWING_0_6V: ++ dev_priv->vbt.edp.vswing = DP_TRAIN_VOLTAGE_SWING_LEVEL_1; ++ break; ++ case EDP_VSWING_0_8V: ++ dev_priv->vbt.edp.vswing = DP_TRAIN_VOLTAGE_SWING_LEVEL_2; ++ break; ++ case EDP_VSWING_1_2V: ++ dev_priv->vbt.edp.vswing = DP_TRAIN_VOLTAGE_SWING_LEVEL_3; ++ break; ++ default: ++ DRM_DEBUG_KMS("VBT has unknown eDP voltage swing value %u\n", ++ edp_link_params->vswing); ++ break; ++ } ++ ++ if (bdb->version >= 173) { ++ u8 vswing; ++ ++ /* Don't read from VBT if module parameter has valid value*/ ++ if (i915_modparams.edp_vswing) { ++ dev_priv->vbt.edp.low_vswing = ++ i915_modparams.edp_vswing == 1; ++ } else { ++ vswing = (edp->edp_vswing_preemph >> (panel_type * 4)) & 0xF; ++ dev_priv->vbt.edp.low_vswing = vswing == 0; ++ } ++ } ++} ++ ++static void ++parse_psr(struct drm_i915_private *dev_priv, const struct bdb_header *bdb) ++{ ++ const struct bdb_psr *psr; ++ const struct psr_table *psr_table; ++ int panel_type = dev_priv->vbt.panel_type; ++ ++ psr = find_section(bdb, BDB_PSR); ++ if (!psr) { ++ DRM_DEBUG_KMS("No PSR BDB found.\n"); ++ return; ++ } ++ ++ psr_table = &psr->psr_table[panel_type]; ++ ++ dev_priv->vbt.psr.full_link = psr_table->full_link; ++ dev_priv->vbt.psr.require_aux_wakeup = psr_table->require_aux_to_wakeup; ++ ++ /* Allowed VBT values goes from 0 to 15 */ ++ dev_priv->vbt.psr.idle_frames = psr_table->idle_frames < 0 ? 0 : ++ psr_table->idle_frames > 15 ? 15 : psr_table->idle_frames; ++ ++ switch (psr_table->lines_to_wait) { ++ case 0: ++ dev_priv->vbt.psr.lines_to_wait = PSR_0_LINES_TO_WAIT; ++ break; ++ case 1: ++ dev_priv->vbt.psr.lines_to_wait = PSR_1_LINE_TO_WAIT; ++ break; ++ case 2: ++ dev_priv->vbt.psr.lines_to_wait = PSR_4_LINES_TO_WAIT; ++ break; ++ case 3: ++ dev_priv->vbt.psr.lines_to_wait = PSR_8_LINES_TO_WAIT; ++ break; ++ default: ++ DRM_DEBUG_KMS("VBT has unknown PSR lines to wait %u\n", ++ psr_table->lines_to_wait); ++ break; ++ } ++ ++ /* ++ * New psr options 0=500us, 1=100us, 2=2500us, 3=0us ++ * Old decimal value is wake up time in multiples of 100 us. ++ */ ++ if (bdb->version >= 205 && ++ (IS_GEN9_BC(dev_priv) || IS_GEMINILAKE(dev_priv) || ++ INTEL_GEN(dev_priv) >= 10)) { ++ switch (psr_table->tp1_wakeup_time) { ++ case 0: ++ dev_priv->vbt.psr.tp1_wakeup_time_us = 500; ++ break; ++ case 1: ++ dev_priv->vbt.psr.tp1_wakeup_time_us = 100; ++ break; ++ case 3: ++ dev_priv->vbt.psr.tp1_wakeup_time_us = 0; ++ break; ++ default: ++ DRM_DEBUG_KMS("VBT tp1 wakeup time value %d is outside range[0-3], defaulting to max value 2500us\n", ++ psr_table->tp1_wakeup_time); ++ /* fallthrough */ ++ case 2: ++ dev_priv->vbt.psr.tp1_wakeup_time_us = 2500; ++ break; ++ } ++ ++ switch (psr_table->tp2_tp3_wakeup_time) { ++ case 0: ++ dev_priv->vbt.psr.tp2_tp3_wakeup_time_us = 500; ++ break; ++ case 1: ++ dev_priv->vbt.psr.tp2_tp3_wakeup_time_us = 100; ++ break; ++ case 3: ++ dev_priv->vbt.psr.tp2_tp3_wakeup_time_us = 0; ++ break; ++ default: ++ DRM_DEBUG_KMS("VBT tp2_tp3 wakeup time value %d is outside range[0-3], defaulting to max value 2500us\n", ++ psr_table->tp2_tp3_wakeup_time); ++ /* fallthrough */ ++ case 2: ++ dev_priv->vbt.psr.tp2_tp3_wakeup_time_us = 2500; ++ break; ++ } ++ } else { ++ dev_priv->vbt.psr.tp1_wakeup_time_us = psr_table->tp1_wakeup_time * 100; ++ dev_priv->vbt.psr.tp2_tp3_wakeup_time_us = psr_table->tp2_tp3_wakeup_time * 100; ++ } ++ ++ if (bdb->version >= 226) { ++ u32 wakeup_time = psr->psr2_tp2_tp3_wakeup_time; ++ ++ wakeup_time = (wakeup_time >> (2 * panel_type)) & 0x3; ++ switch (wakeup_time) { ++ case 0: ++ wakeup_time = 500; ++ break; ++ case 1: ++ wakeup_time = 100; ++ break; ++ case 3: ++ wakeup_time = 50; ++ break; ++ default: ++ case 2: ++ wakeup_time = 2500; ++ break; ++ } ++ dev_priv->vbt.psr.psr2_tp2_tp3_wakeup_time_us = wakeup_time; ++ } else { ++ /* Reusing PSR1 wakeup time for PSR2 in older VBTs */ ++ dev_priv->vbt.psr.psr2_tp2_tp3_wakeup_time_us = dev_priv->vbt.psr.tp2_tp3_wakeup_time_us; ++ } ++} ++ ++static void parse_dsi_backlight_ports(struct drm_i915_private *dev_priv, ++ u16 version, enum port port) ++{ ++ if (!dev_priv->vbt.dsi.config->dual_link || version < 197) { ++ dev_priv->vbt.dsi.bl_ports = BIT(port); ++ if (dev_priv->vbt.dsi.config->cabc_supported) ++ dev_priv->vbt.dsi.cabc_ports = BIT(port); ++ ++ return; ++ } ++ ++ switch (dev_priv->vbt.dsi.config->dl_dcs_backlight_ports) { ++ case DL_DCS_PORT_A: ++ dev_priv->vbt.dsi.bl_ports = BIT(PORT_A); ++ break; ++ case DL_DCS_PORT_C: ++ dev_priv->vbt.dsi.bl_ports = BIT(PORT_C); ++ break; ++ default: ++ case DL_DCS_PORT_A_AND_C: ++ dev_priv->vbt.dsi.bl_ports = BIT(PORT_A) | BIT(PORT_C); ++ break; ++ } ++ ++ if (!dev_priv->vbt.dsi.config->cabc_supported) ++ return; ++ ++ switch (dev_priv->vbt.dsi.config->dl_dcs_cabc_ports) { ++ case DL_DCS_PORT_A: ++ dev_priv->vbt.dsi.cabc_ports = BIT(PORT_A); ++ break; ++ case DL_DCS_PORT_C: ++ dev_priv->vbt.dsi.cabc_ports = BIT(PORT_C); ++ break; ++ default: ++ case DL_DCS_PORT_A_AND_C: ++ dev_priv->vbt.dsi.cabc_ports = ++ BIT(PORT_A) | BIT(PORT_C); ++ break; ++ } ++} ++ ++static void ++parse_mipi_config(struct drm_i915_private *dev_priv, ++ const struct bdb_header *bdb) ++{ ++ const struct bdb_mipi_config *start; ++ const struct mipi_config *config; ++ const struct mipi_pps_data *pps; ++ int panel_type = dev_priv->vbt.panel_type; ++ enum port port; ++ ++ /* parse MIPI blocks only if LFP type is MIPI */ ++ if (!intel_bios_is_dsi_present(dev_priv, &port)) ++ return; ++ ++ /* Initialize this to undefined indicating no generic MIPI support */ ++ dev_priv->vbt.dsi.panel_id = MIPI_DSI_UNDEFINED_PANEL_ID; ++ ++ /* Block #40 is already parsed and panel_fixed_mode is ++ * stored in dev_priv->lfp_lvds_vbt_mode ++ * resuse this when needed ++ */ ++ ++ /* Parse #52 for panel index used from panel_type already ++ * parsed ++ */ ++ start = find_section(bdb, BDB_MIPI_CONFIG); ++ if (!start) { ++ DRM_DEBUG_KMS("No MIPI config BDB found"); ++ return; ++ } ++ ++ DRM_DEBUG_DRIVER("Found MIPI Config block, panel index = %d\n", ++ panel_type); ++ ++ /* ++ * get hold of the correct configuration block and pps data as per ++ * the panel_type as index ++ */ ++ config = &start->config[panel_type]; ++ pps = &start->pps[panel_type]; ++ ++ /* store as of now full data. Trim when we realise all is not needed */ ++ dev_priv->vbt.dsi.config = kmemdup(config, sizeof(struct mipi_config), GFP_KERNEL); ++ if (!dev_priv->vbt.dsi.config) ++ return; ++ ++ dev_priv->vbt.dsi.pps = kmemdup(pps, sizeof(struct mipi_pps_data), GFP_KERNEL); ++ if (!dev_priv->vbt.dsi.pps) { ++ kfree(dev_priv->vbt.dsi.config); ++ return; ++ } ++ ++ parse_dsi_backlight_ports(dev_priv, bdb->version, port); ++ ++ /* FIXME is the 90 vs. 270 correct? */ ++ switch (config->rotation) { ++ case ENABLE_ROTATION_0: ++ /* ++ * Most (all?) VBTs claim 0 degrees despite having ++ * an upside down panel, thus we do not trust this. ++ */ ++ dev_priv->vbt.dsi.orientation = ++ DRM_MODE_PANEL_ORIENTATION_UNKNOWN; ++ break; ++ case ENABLE_ROTATION_90: ++ dev_priv->vbt.dsi.orientation = ++ DRM_MODE_PANEL_ORIENTATION_RIGHT_UP; ++ break; ++ case ENABLE_ROTATION_180: ++ dev_priv->vbt.dsi.orientation = ++ DRM_MODE_PANEL_ORIENTATION_BOTTOM_UP; ++ break; ++ case ENABLE_ROTATION_270: ++ dev_priv->vbt.dsi.orientation = ++ DRM_MODE_PANEL_ORIENTATION_LEFT_UP; ++ break; ++ } ++ ++ /* We have mandatory mipi config blocks. Initialize as generic panel */ ++ dev_priv->vbt.dsi.panel_id = MIPI_DSI_GENERIC_PANEL_ID; ++} ++ ++/* Find the sequence block and size for the given panel. */ ++static const u8 * ++find_panel_sequence_block(const struct bdb_mipi_sequence *sequence, ++ u16 panel_id, u32 *seq_size) ++{ ++ u32 total = get_blocksize(sequence); ++ const u8 *data = &sequence->data[0]; ++ u8 current_id; ++ u32 current_size; ++ int header_size = sequence->version >= 3 ? 5 : 3; ++ int index = 0; ++ int i; ++ ++ /* skip new block size */ ++ if (sequence->version >= 3) ++ data += 4; ++ ++ for (i = 0; i < MAX_MIPI_CONFIGURATIONS && index < total; i++) { ++ if (index + header_size > total) { ++ DRM_ERROR("Invalid sequence block (header)\n"); ++ return NULL; ++ } ++ ++ current_id = *(data + index); ++ if (sequence->version >= 3) ++ current_size = *((const u32 *)(data + index + 1)); ++ else ++ current_size = *((const u16 *)(data + index + 1)); ++ ++ index += header_size; ++ ++ if (index + current_size > total) { ++ DRM_ERROR("Invalid sequence block\n"); ++ return NULL; ++ } ++ ++ if (current_id == panel_id) { ++ *seq_size = current_size; ++ return data + index; ++ } ++ ++ index += current_size; ++ } ++ ++ DRM_ERROR("Sequence block detected but no valid configuration\n"); ++ ++ return NULL; ++} ++ ++static int goto_next_sequence(const u8 *data, int index, int total) ++{ ++ u16 len; ++ ++ /* Skip Sequence Byte. */ ++ for (index = index + 1; index < total; index += len) { ++ u8 operation_byte = *(data + index); ++ index++; ++ ++ switch (operation_byte) { ++ case MIPI_SEQ_ELEM_END: ++ return index; ++ case MIPI_SEQ_ELEM_SEND_PKT: ++ if (index + 4 > total) ++ return 0; ++ ++ len = *((const u16 *)(data + index + 2)) + 4; ++ break; ++ case MIPI_SEQ_ELEM_DELAY: ++ len = 4; ++ break; ++ case MIPI_SEQ_ELEM_GPIO: ++ len = 2; ++ break; ++ case MIPI_SEQ_ELEM_I2C: ++ if (index + 7 > total) ++ return 0; ++ len = *(data + index + 6) + 7; ++ break; ++ default: ++ DRM_ERROR("Unknown operation byte\n"); ++ return 0; ++ } ++ } ++ ++ return 0; ++} ++ ++static int goto_next_sequence_v3(const u8 *data, int index, int total) ++{ ++ int seq_end; ++ u16 len; ++ u32 size_of_sequence; ++ ++ /* ++ * Could skip sequence based on Size of Sequence alone, but also do some ++ * checking on the structure. ++ */ ++ if (total < 5) { ++ DRM_ERROR("Too small sequence size\n"); ++ return 0; ++ } ++ ++ /* Skip Sequence Byte. */ ++ index++; ++ ++ /* ++ * Size of Sequence. Excludes the Sequence Byte and the size itself, ++ * includes MIPI_SEQ_ELEM_END byte, excludes the final MIPI_SEQ_END ++ * byte. ++ */ ++ size_of_sequence = *((const u32 *)(data + index)); ++ index += 4; ++ ++ seq_end = index + size_of_sequence; ++ if (seq_end > total) { ++ DRM_ERROR("Invalid sequence size\n"); ++ return 0; ++ } ++ ++ for (; index < total; index += len) { ++ u8 operation_byte = *(data + index); ++ index++; ++ ++ if (operation_byte == MIPI_SEQ_ELEM_END) { ++ if (index != seq_end) { ++ DRM_ERROR("Invalid element structure\n"); ++ return 0; ++ } ++ return index; ++ } ++ ++ len = *(data + index); ++ index++; ++ ++ /* ++ * FIXME: Would be nice to check elements like for v1/v2 in ++ * goto_next_sequence() above. ++ */ ++ switch (operation_byte) { ++ case MIPI_SEQ_ELEM_SEND_PKT: ++ case MIPI_SEQ_ELEM_DELAY: ++ case MIPI_SEQ_ELEM_GPIO: ++ case MIPI_SEQ_ELEM_I2C: ++ case MIPI_SEQ_ELEM_SPI: ++ case MIPI_SEQ_ELEM_PMIC: ++ break; ++ default: ++ DRM_ERROR("Unknown operation byte %u\n", ++ operation_byte); ++ break; ++ } ++ } ++ ++ return 0; ++} ++ ++/* ++ * Get len of pre-fixed deassert fragment from a v1 init OTP sequence, ++ * skip all delay + gpio operands and stop at the first DSI packet op. ++ */ ++static int get_init_otp_deassert_fragment_len(struct drm_i915_private *dev_priv) ++{ ++ const u8 *data = dev_priv->vbt.dsi.sequence[MIPI_SEQ_INIT_OTP]; ++ int index, len; ++ ++ if (WARN_ON(!data || dev_priv->vbt.dsi.seq_version != 1)) ++ return 0; ++ ++ /* index = 1 to skip sequence byte */ ++ for (index = 1; data[index] != MIPI_SEQ_ELEM_END; index += len) { ++ switch (data[index]) { ++ case MIPI_SEQ_ELEM_SEND_PKT: ++ return index == 1 ? 0 : index; ++ case MIPI_SEQ_ELEM_DELAY: ++ len = 5; /* 1 byte for operand + uint32 */ ++ break; ++ case MIPI_SEQ_ELEM_GPIO: ++ len = 3; /* 1 byte for op, 1 for gpio_nr, 1 for value */ ++ break; ++ default: ++ return 0; ++ } ++ } ++ ++ return 0; ++} ++ ++/* ++ * Some v1 VBT MIPI sequences do the deassert in the init OTP sequence. ++ * The deassert must be done before calling intel_dsi_device_ready, so for ++ * these devices we split the init OTP sequence into a deassert sequence and ++ * the actual init OTP part. ++ */ ++static void fixup_mipi_sequences(struct drm_i915_private *dev_priv) ++{ ++ u8 *init_otp; ++ int len; ++ ++ /* Limit this to VLV for now. */ ++ if (!IS_VALLEYVIEW(dev_priv)) ++ return; ++ ++ /* Limit this to v1 vid-mode sequences */ ++ if (dev_priv->vbt.dsi.config->is_cmd_mode || ++ dev_priv->vbt.dsi.seq_version != 1) ++ return; ++ ++ /* Only do this if there are otp and assert seqs and no deassert seq */ ++ if (!dev_priv->vbt.dsi.sequence[MIPI_SEQ_INIT_OTP] || ++ !dev_priv->vbt.dsi.sequence[MIPI_SEQ_ASSERT_RESET] || ++ dev_priv->vbt.dsi.sequence[MIPI_SEQ_DEASSERT_RESET]) ++ return; ++ ++ /* The deassert-sequence ends at the first DSI packet */ ++ len = get_init_otp_deassert_fragment_len(dev_priv); ++ if (!len) ++ return; ++ ++ DRM_DEBUG_KMS("Using init OTP fragment to deassert reset\n"); ++ ++ /* Copy the fragment, update seq byte and terminate it */ ++ init_otp = (u8 *)dev_priv->vbt.dsi.sequence[MIPI_SEQ_INIT_OTP]; ++ dev_priv->vbt.dsi.deassert_seq = kmemdup(init_otp, len + 1, GFP_KERNEL); ++ if (!dev_priv->vbt.dsi.deassert_seq) ++ return; ++ dev_priv->vbt.dsi.deassert_seq[0] = MIPI_SEQ_DEASSERT_RESET; ++ dev_priv->vbt.dsi.deassert_seq[len] = MIPI_SEQ_ELEM_END; ++ /* Use the copy for deassert */ ++ dev_priv->vbt.dsi.sequence[MIPI_SEQ_DEASSERT_RESET] = ++ dev_priv->vbt.dsi.deassert_seq; ++ /* Replace the last byte of the fragment with init OTP seq byte */ ++ init_otp[len - 1] = MIPI_SEQ_INIT_OTP; ++ /* And make MIPI_MIPI_SEQ_INIT_OTP point to it */ ++ dev_priv->vbt.dsi.sequence[MIPI_SEQ_INIT_OTP] = init_otp + len - 1; ++} ++ ++static void ++parse_mipi_sequence(struct drm_i915_private *dev_priv, ++ const struct bdb_header *bdb) ++{ ++ int panel_type = dev_priv->vbt.panel_type; ++ const struct bdb_mipi_sequence *sequence; ++ const u8 *seq_data; ++ u32 seq_size; ++ u8 *data; ++ int index = 0; ++ ++ /* Only our generic panel driver uses the sequence block. */ ++ if (dev_priv->vbt.dsi.panel_id != MIPI_DSI_GENERIC_PANEL_ID) ++ return; ++ ++ sequence = find_section(bdb, BDB_MIPI_SEQUENCE); ++ if (!sequence) { ++ DRM_DEBUG_KMS("No MIPI Sequence found, parsing complete\n"); ++ return; ++ } ++ ++ /* Fail gracefully for forward incompatible sequence block. */ ++ if (sequence->version >= 4) { ++ DRM_ERROR("Unable to parse MIPI Sequence Block v%u\n", ++ sequence->version); ++ return; ++ } ++ ++ DRM_DEBUG_DRIVER("Found MIPI sequence block v%u\n", sequence->version); ++ ++ seq_data = find_panel_sequence_block(sequence, panel_type, &seq_size); ++ if (!seq_data) ++ return; ++ ++ data = kmemdup(seq_data, seq_size, GFP_KERNEL); ++ if (!data) ++ return; ++ ++ /* Parse the sequences, store pointers to each sequence. */ ++ for (;;) { ++ u8 seq_id = *(data + index); ++ if (seq_id == MIPI_SEQ_END) ++ break; ++ ++ if (seq_id >= MIPI_SEQ_MAX) { ++ DRM_ERROR("Unknown sequence %u\n", seq_id); ++ goto err; ++ } ++ ++ /* Log about presence of sequences we won't run. */ ++ if (seq_id == MIPI_SEQ_TEAR_ON || seq_id == MIPI_SEQ_TEAR_OFF) ++ DRM_DEBUG_KMS("Unsupported sequence %u\n", seq_id); ++ ++ dev_priv->vbt.dsi.sequence[seq_id] = data + index; ++ ++ if (sequence->version >= 3) ++ index = goto_next_sequence_v3(data, index, seq_size); ++ else ++ index = goto_next_sequence(data, index, seq_size); ++ if (!index) { ++ DRM_ERROR("Invalid sequence %u\n", seq_id); ++ goto err; ++ } ++ } ++ ++ dev_priv->vbt.dsi.data = data; ++ dev_priv->vbt.dsi.size = seq_size; ++ dev_priv->vbt.dsi.seq_version = sequence->version; ++ ++ fixup_mipi_sequences(dev_priv); ++ ++ DRM_DEBUG_DRIVER("MIPI related VBT parsing complete\n"); ++ return; ++ ++err: ++ kfree(data); ++ memset(dev_priv->vbt.dsi.sequence, 0, sizeof(dev_priv->vbt.dsi.sequence)); ++} ++ ++static u8 translate_iboost(u8 val) ++{ ++ static const u8 mapping[] = { 1, 3, 7 }; /* See VBT spec */ ++ ++ if (val >= ARRAY_SIZE(mapping)) { ++ DRM_DEBUG_KMS("Unsupported I_boost value found in VBT (%d), display may not work properly\n", val); ++ return 0; ++ } ++ return mapping[val]; ++} ++ ++static void sanitize_ddc_pin(struct drm_i915_private *dev_priv, ++ enum port port) ++{ ++ const struct ddi_vbt_port_info *info = ++ &dev_priv->vbt.ddi_port_info[port]; ++ enum port p; ++ ++ if (!info->alternate_ddc_pin) ++ return; ++ ++ for (p = PORT_A; p < I915_MAX_PORTS; p++) { ++ struct ddi_vbt_port_info *i = &dev_priv->vbt.ddi_port_info[p]; ++ ++ if (p == port || !i->present || ++ info->alternate_ddc_pin != i->alternate_ddc_pin) ++ continue; ++ ++ DRM_DEBUG_KMS("port %c trying to use the same DDC pin (0x%x) as port %c, " ++ "disabling port %c DVI/HDMI support\n", ++ port_name(p), i->alternate_ddc_pin, ++ port_name(port), port_name(p)); ++ ++ /* ++ * If we have multiple ports supposedly sharing the ++ * pin, then dvi/hdmi couldn't exist on the shared ++ * port. Otherwise they share the same ddc bin and ++ * system couldn't communicate with them separately. ++ * ++ * Due to parsing the ports in child device order, ++ * a later device will always clobber an earlier one. ++ */ ++ i->supports_dvi = false; ++ i->supports_hdmi = false; ++ i->alternate_ddc_pin = 0; ++ } ++} ++ ++static void sanitize_aux_ch(struct drm_i915_private *dev_priv, ++ enum port port) ++{ ++ const struct ddi_vbt_port_info *info = ++ &dev_priv->vbt.ddi_port_info[port]; ++ enum port p; ++ ++ if (!info->alternate_aux_channel) ++ return; ++ ++ for (p = PORT_A; p < I915_MAX_PORTS; p++) { ++ struct ddi_vbt_port_info *i = &dev_priv->vbt.ddi_port_info[p]; ++ ++ if (p == port || !i->present || ++ info->alternate_aux_channel != i->alternate_aux_channel) ++ continue; ++ ++ DRM_DEBUG_KMS("port %c trying to use the same AUX CH (0x%x) as port %c, " ++ "disabling port %c DP support\n", ++ port_name(p), i->alternate_aux_channel, ++ port_name(port), port_name(p)); ++ ++ /* ++ * If we have multiple ports supposedlt sharing the ++ * aux channel, then DP couldn't exist on the shared ++ * port. Otherwise they share the same aux channel ++ * and system couldn't communicate with them separately. ++ * ++ * Due to parsing the ports in child device order, ++ * a later device will always clobber an earlier one. ++ */ ++ i->supports_dp = false; ++ i->alternate_aux_channel = 0; ++ } ++} ++ ++static const u8 cnp_ddc_pin_map[] = { ++ [0] = 0, /* N/A */ ++ [DDC_BUS_DDI_B] = GMBUS_PIN_1_BXT, ++ [DDC_BUS_DDI_C] = GMBUS_PIN_2_BXT, ++ [DDC_BUS_DDI_D] = GMBUS_PIN_4_CNP, /* sic */ ++ [DDC_BUS_DDI_F] = GMBUS_PIN_3_BXT, /* sic */ ++}; ++ ++static const u8 icp_ddc_pin_map[] = { ++ [ICL_DDC_BUS_DDI_A] = GMBUS_PIN_1_BXT, ++ [ICL_DDC_BUS_DDI_B] = GMBUS_PIN_2_BXT, ++ [ICL_DDC_BUS_PORT_1] = GMBUS_PIN_9_TC1_ICP, ++ [ICL_DDC_BUS_PORT_2] = GMBUS_PIN_10_TC2_ICP, ++ [ICL_DDC_BUS_PORT_3] = GMBUS_PIN_11_TC3_ICP, ++ [ICL_DDC_BUS_PORT_4] = GMBUS_PIN_12_TC4_ICP, ++}; ++ ++static u8 map_ddc_pin(struct drm_i915_private *dev_priv, u8 vbt_pin) ++{ ++ const u8 *ddc_pin_map; ++ int n_entries; ++ ++ if (HAS_PCH_ICP(dev_priv)) { ++ ddc_pin_map = icp_ddc_pin_map; ++ n_entries = ARRAY_SIZE(icp_ddc_pin_map); ++ } else if (HAS_PCH_CNP(dev_priv)) { ++ ddc_pin_map = cnp_ddc_pin_map; ++ n_entries = ARRAY_SIZE(cnp_ddc_pin_map); ++ } else { ++ /* Assuming direct map */ ++ return vbt_pin; ++ } ++ ++ if (vbt_pin < n_entries && ddc_pin_map[vbt_pin] != 0) ++ return ddc_pin_map[vbt_pin]; ++ ++ DRM_DEBUG_KMS("Ignoring alternate pin: VBT claims DDC pin %d, which is not valid for this platform\n", ++ vbt_pin); ++ return 0; ++} ++ ++static enum port dvo_port_to_port(u8 dvo_port) ++{ ++ /* ++ * Each DDI port can have more than one value on the "DVO Port" field, ++ * so look for all the possible values for each port. ++ */ ++ static const int dvo_ports[][3] = { ++ [PORT_A] = { DVO_PORT_HDMIA, DVO_PORT_DPA, -1}, ++ [PORT_B] = { DVO_PORT_HDMIB, DVO_PORT_DPB, -1}, ++ [PORT_C] = { DVO_PORT_HDMIC, DVO_PORT_DPC, -1}, ++ [PORT_D] = { DVO_PORT_HDMID, DVO_PORT_DPD, -1}, ++ [PORT_E] = { DVO_PORT_CRT, DVO_PORT_HDMIE, DVO_PORT_DPE}, ++ [PORT_F] = { DVO_PORT_HDMIF, DVO_PORT_DPF, -1}, ++ }; ++ enum port port; ++ int i; ++ ++ for (port = PORT_A; port < ARRAY_SIZE(dvo_ports); port++) { ++ for (i = 0; i < ARRAY_SIZE(dvo_ports[port]); i++) { ++ if (dvo_ports[port][i] == -1) ++ break; ++ ++ if (dvo_port == dvo_ports[port][i]) ++ return port; ++ } ++ } ++ ++ return PORT_NONE; ++} ++ ++static void parse_ddi_port(struct drm_i915_private *dev_priv, ++ const struct child_device_config *child, ++ u8 bdb_version) ++{ ++ struct ddi_vbt_port_info *info; ++ bool is_dvi, is_hdmi, is_dp, is_edp, is_crt; ++ enum port port; ++ ++ port = dvo_port_to_port(child->dvo_port); ++ if (port == PORT_NONE) ++ return; ++ ++ info = &dev_priv->vbt.ddi_port_info[port]; ++ ++ if (info->present) { ++ DRM_DEBUG_KMS("More than one child device for port %c in VBT, using the first.\n", ++ port_name(port)); ++ return; ++ } ++ ++ info->present = true; ++ ++ is_dvi = child->device_type & DEVICE_TYPE_TMDS_DVI_SIGNALING; ++ is_dp = child->device_type & DEVICE_TYPE_DISPLAYPORT_OUTPUT; ++ is_crt = child->device_type & DEVICE_TYPE_ANALOG_OUTPUT; ++ is_hdmi = is_dvi && (child->device_type & DEVICE_TYPE_NOT_HDMI_OUTPUT) == 0; ++ is_edp = is_dp && (child->device_type & DEVICE_TYPE_INTERNAL_CONNECTOR); ++ ++ if (port == PORT_A && is_dvi) { ++ DRM_DEBUG_KMS("VBT claims port A supports DVI%s, ignoring\n", ++ is_hdmi ? "/HDMI" : ""); ++ is_dvi = false; ++ is_hdmi = false; ++ } ++ ++ info->supports_dvi = is_dvi; ++ info->supports_hdmi = is_hdmi; ++ info->supports_dp = is_dp; ++ info->supports_edp = is_edp; ++ ++ if (bdb_version >= 195) ++ info->supports_typec_usb = child->dp_usb_type_c; ++ ++ if (bdb_version >= 209) ++ info->supports_tbt = child->tbt; ++ ++ DRM_DEBUG_KMS("Port %c VBT info: DP:%d HDMI:%d DVI:%d EDP:%d CRT:%d TCUSB:%d TBT:%d\n", ++ port_name(port), is_dp, is_hdmi, is_dvi, is_edp, is_crt, ++ info->supports_typec_usb, info->supports_tbt); ++ ++ if (is_edp && is_dvi) ++ DRM_DEBUG_KMS("Internal DP port %c is TMDS compatible\n", ++ port_name(port)); ++ if (is_crt && port != PORT_E) ++ DRM_DEBUG_KMS("Port %c is analog\n", port_name(port)); ++ if (is_crt && (is_dvi || is_dp)) ++ DRM_DEBUG_KMS("Analog port %c is also DP or TMDS compatible\n", ++ port_name(port)); ++ if (is_dvi && (port == PORT_A || port == PORT_E)) ++ DRM_DEBUG_KMS("Port %c is TMDS compatible\n", port_name(port)); ++ if (!is_dvi && !is_dp && !is_crt) ++ DRM_DEBUG_KMS("Port %c is not DP/TMDS/CRT compatible\n", ++ port_name(port)); ++ if (is_edp && (port == PORT_B || port == PORT_C || port == PORT_E)) ++ DRM_DEBUG_KMS("Port %c is internal DP\n", port_name(port)); ++ ++ if (is_dvi) { ++ u8 ddc_pin; ++ ++ ddc_pin = map_ddc_pin(dev_priv, child->ddc_pin); ++ if (intel_gmbus_is_valid_pin(dev_priv, ddc_pin)) { ++ info->alternate_ddc_pin = ddc_pin; ++ sanitize_ddc_pin(dev_priv, port); ++ } else { ++ DRM_DEBUG_KMS("Port %c has invalid DDC pin %d, " ++ "sticking to defaults\n", ++ port_name(port), ddc_pin); ++ } ++ } ++ ++ if (is_dp) { ++ info->alternate_aux_channel = child->aux_channel; ++ ++ sanitize_aux_ch(dev_priv, port); ++ } ++ ++ if (bdb_version >= 158) { ++ /* The VBT HDMI level shift values match the table we have. */ ++ u8 hdmi_level_shift = child->hdmi_level_shifter_value; ++ DRM_DEBUG_KMS("VBT HDMI level shift for port %c: %d\n", ++ port_name(port), ++ hdmi_level_shift); ++ info->hdmi_level_shift = hdmi_level_shift; ++ } ++ ++ if (bdb_version >= 204) { ++ int max_tmds_clock; ++ ++ switch (child->hdmi_max_data_rate) { ++ default: ++ MISSING_CASE(child->hdmi_max_data_rate); ++ /* fall through */ ++ case HDMI_MAX_DATA_RATE_PLATFORM: ++ max_tmds_clock = 0; ++ break; ++ case HDMI_MAX_DATA_RATE_297: ++ max_tmds_clock = 297000; ++ break; ++ case HDMI_MAX_DATA_RATE_165: ++ max_tmds_clock = 165000; ++ break; ++ } ++ ++ if (max_tmds_clock) ++ DRM_DEBUG_KMS("VBT HDMI max TMDS clock for port %c: %d kHz\n", ++ port_name(port), max_tmds_clock); ++ info->max_tmds_clock = max_tmds_clock; ++ } ++ ++ /* Parse the I_boost config for SKL and above */ ++ if (bdb_version >= 196 && child->iboost) { ++ info->dp_boost_level = translate_iboost(child->dp_iboost_level); ++ DRM_DEBUG_KMS("VBT (e)DP boost level for port %c: %d\n", ++ port_name(port), info->dp_boost_level); ++ info->hdmi_boost_level = translate_iboost(child->hdmi_iboost_level); ++ DRM_DEBUG_KMS("VBT HDMI boost level for port %c: %d\n", ++ port_name(port), info->hdmi_boost_level); ++ } ++ ++ /* DP max link rate for CNL+ */ ++ if (bdb_version >= 216) { ++ switch (child->dp_max_link_rate) { ++ default: ++ case VBT_DP_MAX_LINK_RATE_HBR3: ++ info->dp_max_link_rate = 810000; ++ break; ++ case VBT_DP_MAX_LINK_RATE_HBR2: ++ info->dp_max_link_rate = 540000; ++ break; ++ case VBT_DP_MAX_LINK_RATE_HBR: ++ info->dp_max_link_rate = 270000; ++ break; ++ case VBT_DP_MAX_LINK_RATE_LBR: ++ info->dp_max_link_rate = 162000; ++ break; ++ } ++ DRM_DEBUG_KMS("VBT DP max link rate for port %c: %d\n", ++ port_name(port), info->dp_max_link_rate); ++ } ++} ++ ++static void parse_ddi_ports(struct drm_i915_private *dev_priv, u8 bdb_version) ++{ ++ const struct child_device_config *child; ++ int i; ++ ++ if (!HAS_DDI(dev_priv) && !IS_CHERRYVIEW(dev_priv)) ++ return; ++ ++ if (bdb_version < 155) ++ return; ++ ++ for (i = 0; i < dev_priv->vbt.child_dev_num; i++) { ++ child = dev_priv->vbt.child_dev + i; ++ ++ parse_ddi_port(dev_priv, child, bdb_version); ++ } ++} ++ ++static void ++parse_general_definitions(struct drm_i915_private *dev_priv, ++ const struct bdb_header *bdb) ++{ ++ const struct bdb_general_definitions *defs; ++ const struct child_device_config *child; ++ int i, child_device_num, count; ++ u8 expected_size; ++ u16 block_size; ++ int bus_pin; ++ ++ defs = find_section(bdb, BDB_GENERAL_DEFINITIONS); ++ if (!defs) { ++ DRM_DEBUG_KMS("No general definition block is found, no devices defined.\n"); ++ return; ++ } ++ ++ block_size = get_blocksize(defs); ++ if (block_size < sizeof(*defs)) { ++ DRM_DEBUG_KMS("General definitions block too small (%u)\n", ++ block_size); ++ return; ++ } ++ ++ bus_pin = defs->crt_ddc_gmbus_pin; ++ DRM_DEBUG_KMS("crt_ddc_bus_pin: %d\n", bus_pin); ++ if (intel_gmbus_is_valid_pin(dev_priv, bus_pin)) ++ dev_priv->vbt.crt_ddc_pin = bus_pin; ++ ++ if (bdb->version < 106) { ++ expected_size = 22; ++ } else if (bdb->version < 111) { ++ expected_size = 27; ++ } else if (bdb->version < 195) { ++ expected_size = LEGACY_CHILD_DEVICE_CONFIG_SIZE; ++ } else if (bdb->version == 195) { ++ expected_size = 37; ++ } else if (bdb->version <= 215) { ++ expected_size = 38; ++ } else if (bdb->version <= 216) { ++ expected_size = 39; ++ } else { ++ expected_size = sizeof(*child); ++ BUILD_BUG_ON(sizeof(*child) < 39); ++ DRM_DEBUG_DRIVER("Expected child device config size for VBT version %u not known; assuming %u\n", ++ bdb->version, expected_size); ++ } ++ ++ /* Flag an error for unexpected size, but continue anyway. */ ++ if (defs->child_dev_size != expected_size) ++ DRM_ERROR("Unexpected child device config size %u (expected %u for VBT version %u)\n", ++ defs->child_dev_size, expected_size, bdb->version); ++ ++ /* The legacy sized child device config is the minimum we need. */ ++ if (defs->child_dev_size < LEGACY_CHILD_DEVICE_CONFIG_SIZE) { ++ DRM_DEBUG_KMS("Child device config size %u is too small.\n", ++ defs->child_dev_size); ++ return; ++ } ++ ++ /* get the number of child device */ ++ child_device_num = (block_size - sizeof(*defs)) / defs->child_dev_size; ++ count = 0; ++ /* get the number of child device that is present */ ++ for (i = 0; i < child_device_num; i++) { ++ child = child_device_ptr(defs, i); ++ if (!child->device_type) ++ continue; ++ count++; ++ } ++ if (!count) { ++ DRM_DEBUG_KMS("no child dev is parsed from VBT\n"); ++ return; ++ } ++ dev_priv->vbt.child_dev = kcalloc(count, sizeof(*child), GFP_KERNEL); ++ if (!dev_priv->vbt.child_dev) { ++ DRM_DEBUG_KMS("No memory space for child device\n"); ++ return; ++ } ++ ++ dev_priv->vbt.child_dev_num = count; ++ count = 0; ++ for (i = 0; i < child_device_num; i++) { ++ child = child_device_ptr(defs, i); ++ if (!child->device_type) ++ continue; ++ ++ /* ++ * Copy as much as we know (sizeof) and is available ++ * (child_dev_size) of the child device. Accessing the data must ++ * depend on VBT version. ++ */ ++ memcpy(dev_priv->vbt.child_dev + count, child, ++ min_t(size_t, defs->child_dev_size, sizeof(*child))); ++ count++; ++ } ++} ++ ++/* Common defaults which may be overridden by VBT. */ ++static void ++init_vbt_defaults(struct drm_i915_private *dev_priv) ++{ ++ enum port port; ++ ++ dev_priv->vbt.crt_ddc_pin = GMBUS_PIN_VGADDC; ++ ++ /* Default to having backlight */ ++ dev_priv->vbt.backlight.present = true; ++ ++ /* LFP panel data */ ++ dev_priv->vbt.lvds_dither = 1; ++ ++ /* SDVO panel data */ ++ dev_priv->vbt.sdvo_lvds_vbt_mode = NULL; ++ ++ /* general features */ ++ dev_priv->vbt.int_tv_support = 1; ++ dev_priv->vbt.int_crt_support = 1; ++ ++ /* driver features */ ++ dev_priv->vbt.int_lvds_support = 1; ++ ++ /* Default to using SSC */ ++ dev_priv->vbt.lvds_use_ssc = 1; ++ /* ++ * Core/SandyBridge/IvyBridge use alternative (120MHz) reference ++ * clock for LVDS. ++ */ ++ dev_priv->vbt.lvds_ssc_freq = intel_bios_ssc_frequency(dev_priv, ++ !HAS_PCH_SPLIT(dev_priv)); ++ DRM_DEBUG_KMS("Set default to SSC at %d kHz\n", dev_priv->vbt.lvds_ssc_freq); ++ ++ for (port = PORT_A; port < I915_MAX_PORTS; port++) { ++ struct ddi_vbt_port_info *info = ++ &dev_priv->vbt.ddi_port_info[port]; ++ ++ info->hdmi_level_shift = HDMI_LEVEL_SHIFT_UNKNOWN; ++ } ++} ++ ++/* Defaults to initialize only if there is no VBT. */ ++static void ++init_vbt_missing_defaults(struct drm_i915_private *dev_priv) ++{ ++ enum port port; ++ ++ for (port = PORT_A; port < I915_MAX_PORTS; port++) { ++ struct ddi_vbt_port_info *info = ++ &dev_priv->vbt.ddi_port_info[port]; ++ ++ /* ++ * VBT has the TypeC mode (native,TBT/USB) and we don't want ++ * to detect it. ++ */ ++ if (intel_port_is_tc(dev_priv, port)) ++ continue; ++ ++ info->supports_dvi = (port != PORT_A && port != PORT_E); ++ info->supports_hdmi = info->supports_dvi; ++ info->supports_dp = (port != PORT_E); ++ info->supports_edp = (port == PORT_A); ++ } ++} ++ ++static const struct bdb_header *get_bdb_header(const struct vbt_header *vbt) ++{ ++ const void *_vbt = vbt; ++ ++ return _vbt + vbt->bdb_offset; ++} ++ ++/** ++ * intel_bios_is_valid_vbt - does the given buffer contain a valid VBT ++ * @buf: pointer to a buffer to validate ++ * @size: size of the buffer ++ * ++ * Returns true on valid VBT. ++ */ ++bool intel_bios_is_valid_vbt(const void *buf, size_t size) ++{ ++ const struct vbt_header *vbt = buf; ++ const struct bdb_header *bdb; ++ ++ if (!vbt) ++ return false; ++ ++ if (sizeof(struct vbt_header) > size) { ++ DRM_DEBUG_DRIVER("VBT header incomplete\n"); ++ return false; ++ } ++ ++ if (memcmp(vbt->signature, "$VBT", 4)) { ++ DRM_DEBUG_DRIVER("VBT invalid signature\n"); ++ return false; ++ } ++ ++ if (range_overflows_t(size_t, ++ vbt->bdb_offset, ++ sizeof(struct bdb_header), ++ size)) { ++ DRM_DEBUG_DRIVER("BDB header incomplete\n"); ++ return false; ++ } ++ ++ bdb = get_bdb_header(vbt); ++ if (range_overflows_t(size_t, vbt->bdb_offset, bdb->bdb_size, size)) { ++ DRM_DEBUG_DRIVER("BDB incomplete\n"); ++ return false; ++ } ++ ++ return vbt; ++} ++ ++static const struct vbt_header *find_vbt(void __iomem *bios, size_t size) ++{ ++ size_t i; ++ ++ /* Scour memory looking for the VBT signature. */ ++ for (i = 0; i + 4 < size; i++) { ++ void *vbt; ++ ++ if (ioread32(bios + i) != *((const u32 *) "$VBT")) ++ continue; ++ ++ /* ++ * This is the one place where we explicitly discard the address ++ * space (__iomem) of the BIOS/VBT. ++ */ ++ vbt = (void __force *) bios + i; ++ if (intel_bios_is_valid_vbt(vbt, size - i)) ++ return vbt; ++ ++ break; ++ } ++ ++ return NULL; ++} ++ ++/** ++ * intel_bios_init - find VBT and initialize settings from the BIOS ++ * @dev_priv: i915 device instance ++ * ++ * Parse and initialize settings from the Video BIOS Tables (VBT). If the VBT ++ * was not found in ACPI OpRegion, try to find it in PCI ROM first. Also ++ * initialize some defaults if the VBT is not present at all. ++ */ ++void intel_bios_init(struct drm_i915_private *dev_priv) ++{ ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ const struct vbt_header *vbt = dev_priv->opregion.vbt; ++ const struct bdb_header *bdb; ++ u8 __iomem *bios = NULL; ++ ++ if (!HAS_DISPLAY(dev_priv)) { ++ DRM_DEBUG_KMS("Skipping VBT init due to disabled display.\n"); ++ return; ++ } ++ ++ init_vbt_defaults(dev_priv); ++ ++ /* If the OpRegion does not have VBT, look in PCI ROM. */ ++ if (!vbt) { ++ size_t size; ++ ++ bios = pci_map_rom(pdev, &size); ++ if (!bios) ++ goto out; ++ ++ vbt = find_vbt(bios, size); ++ if (!vbt) ++ goto out; ++ ++ DRM_DEBUG_KMS("Found valid VBT in PCI ROM\n"); ++ } ++ ++ bdb = get_bdb_header(vbt); ++ ++ DRM_DEBUG_KMS("VBT signature \"%.*s\", BDB version %d\n", ++ (int)sizeof(vbt->signature), vbt->signature, bdb->version); ++ ++ /* Grab useful general definitions */ ++ parse_general_features(dev_priv, bdb); ++ parse_general_definitions(dev_priv, bdb); ++ parse_lfp_panel_data(dev_priv, bdb); ++ parse_lfp_backlight(dev_priv, bdb); ++ parse_sdvo_panel_data(dev_priv, bdb); ++ parse_driver_features(dev_priv, bdb); ++ parse_edp(dev_priv, bdb); ++ parse_psr(dev_priv, bdb); ++ parse_mipi_config(dev_priv, bdb); ++ parse_mipi_sequence(dev_priv, bdb); ++ ++ /* Further processing on pre-parsed data */ ++ parse_sdvo_device_mapping(dev_priv, bdb->version); ++ parse_ddi_ports(dev_priv, bdb->version); ++ ++out: ++ if (!vbt) { ++ DRM_INFO("Failed to find VBIOS tables (VBT)\n"); ++ init_vbt_missing_defaults(dev_priv); ++ } ++ ++ if (bios) ++ pci_unmap_rom(pdev, bios); ++} ++ ++/** ++ * intel_bios_cleanup - Free any resources allocated by intel_bios_init() ++ * @dev_priv: i915 device instance ++ */ ++void intel_bios_cleanup(struct drm_i915_private *dev_priv) ++{ ++ kfree(dev_priv->vbt.child_dev); ++ dev_priv->vbt.child_dev = NULL; ++ dev_priv->vbt.child_dev_num = 0; ++ kfree(dev_priv->vbt.sdvo_lvds_vbt_mode); ++ dev_priv->vbt.sdvo_lvds_vbt_mode = NULL; ++ kfree(dev_priv->vbt.lfp_lvds_vbt_mode); ++ dev_priv->vbt.lfp_lvds_vbt_mode = NULL; ++ kfree(dev_priv->vbt.dsi.data); ++ dev_priv->vbt.dsi.data = NULL; ++ kfree(dev_priv->vbt.dsi.pps); ++ dev_priv->vbt.dsi.pps = NULL; ++ kfree(dev_priv->vbt.dsi.config); ++ dev_priv->vbt.dsi.config = NULL; ++ kfree(dev_priv->vbt.dsi.deassert_seq); ++ dev_priv->vbt.dsi.deassert_seq = NULL; ++} ++ ++/** ++ * intel_bios_is_tv_present - is integrated TV present in VBT ++ * @dev_priv: i915 device instance ++ * ++ * Return true if TV is present. If no child devices were parsed from VBT, ++ * assume TV is present. ++ */ ++bool intel_bios_is_tv_present(struct drm_i915_private *dev_priv) ++{ ++ const struct child_device_config *child; ++ int i; ++ ++ if (!dev_priv->vbt.int_tv_support) ++ return false; ++ ++ if (!dev_priv->vbt.child_dev_num) ++ return true; ++ ++ for (i = 0; i < dev_priv->vbt.child_dev_num; i++) { ++ child = dev_priv->vbt.child_dev + i; ++ /* ++ * If the device type is not TV, continue. ++ */ ++ switch (child->device_type) { ++ case DEVICE_TYPE_INT_TV: ++ case DEVICE_TYPE_TV: ++ case DEVICE_TYPE_TV_SVIDEO_COMPOSITE: ++ break; ++ default: ++ continue; ++ } ++ /* Only when the addin_offset is non-zero, it is regarded ++ * as present. ++ */ ++ if (child->addin_offset) ++ return true; ++ } ++ ++ return false; ++} ++ ++/** ++ * intel_bios_is_lvds_present - is LVDS present in VBT ++ * @dev_priv: i915 device instance ++ * @i2c_pin: i2c pin for LVDS if present ++ * ++ * Return true if LVDS is present. If no child devices were parsed from VBT, ++ * assume LVDS is present. ++ */ ++bool intel_bios_is_lvds_present(struct drm_i915_private *dev_priv, u8 *i2c_pin) ++{ ++ const struct child_device_config *child; ++ int i; ++ ++ if (!dev_priv->vbt.child_dev_num) ++ return true; ++ ++ for (i = 0; i < dev_priv->vbt.child_dev_num; i++) { ++ child = dev_priv->vbt.child_dev + i; ++ ++ /* If the device type is not LFP, continue. ++ * We have to check both the new identifiers as well as the ++ * old for compatibility with some BIOSes. ++ */ ++ if (child->device_type != DEVICE_TYPE_INT_LFP && ++ child->device_type != DEVICE_TYPE_LFP) ++ continue; ++ ++ if (intel_gmbus_is_valid_pin(dev_priv, child->i2c_pin)) ++ *i2c_pin = child->i2c_pin; ++ ++ /* However, we cannot trust the BIOS writers to populate ++ * the VBT correctly. Since LVDS requires additional ++ * information from AIM blocks, a non-zero addin offset is ++ * a good indicator that the LVDS is actually present. ++ */ ++ if (child->addin_offset) ++ return true; ++ ++ /* But even then some BIOS writers perform some black magic ++ * and instantiate the device without reference to any ++ * additional data. Trust that if the VBT was written into ++ * the OpRegion then they have validated the LVDS's existence. ++ */ ++ if (dev_priv->opregion.vbt) ++ return true; ++ } ++ ++ return false; ++} ++ ++/** ++ * intel_bios_is_port_present - is the specified digital port present ++ * @dev_priv: i915 device instance ++ * @port: port to check ++ * ++ * Return true if the device in %port is present. ++ */ ++bool intel_bios_is_port_present(struct drm_i915_private *dev_priv, enum port port) ++{ ++ const struct child_device_config *child; ++ static const struct { ++ u16 dp, hdmi; ++ } port_mapping[] = { ++ [PORT_B] = { DVO_PORT_DPB, DVO_PORT_HDMIB, }, ++ [PORT_C] = { DVO_PORT_DPC, DVO_PORT_HDMIC, }, ++ [PORT_D] = { DVO_PORT_DPD, DVO_PORT_HDMID, }, ++ [PORT_E] = { DVO_PORT_DPE, DVO_PORT_HDMIE, }, ++ [PORT_F] = { DVO_PORT_DPF, DVO_PORT_HDMIF, }, ++ }; ++ int i; ++ ++ if (HAS_DDI(dev_priv)) { ++ const struct ddi_vbt_port_info *port_info = ++ &dev_priv->vbt.ddi_port_info[port]; ++ ++ return port_info->supports_dp || ++ port_info->supports_dvi || ++ port_info->supports_hdmi; ++ } ++ ++ /* FIXME maybe deal with port A as well? */ ++ if (WARN_ON(port == PORT_A) || port >= ARRAY_SIZE(port_mapping)) ++ return false; ++ ++ if (!dev_priv->vbt.child_dev_num) ++ return false; ++ ++ for (i = 0; i < dev_priv->vbt.child_dev_num; i++) { ++ child = dev_priv->vbt.child_dev + i; ++ ++ if ((child->dvo_port == port_mapping[port].dp || ++ child->dvo_port == port_mapping[port].hdmi) && ++ (child->device_type & (DEVICE_TYPE_TMDS_DVI_SIGNALING | ++ DEVICE_TYPE_DISPLAYPORT_OUTPUT))) ++ return true; ++ } ++ ++ return false; ++} ++ ++/** ++ * intel_bios_is_port_edp - is the device in given port eDP ++ * @dev_priv: i915 device instance ++ * @port: port to check ++ * ++ * Return true if the device in %port is eDP. ++ */ ++bool intel_bios_is_port_edp(struct drm_i915_private *dev_priv, enum port port) ++{ ++ const struct child_device_config *child; ++ static const short port_mapping[] = { ++ [PORT_B] = DVO_PORT_DPB, ++ [PORT_C] = DVO_PORT_DPC, ++ [PORT_D] = DVO_PORT_DPD, ++ [PORT_E] = DVO_PORT_DPE, ++ [PORT_F] = DVO_PORT_DPF, ++ }; ++ int i; ++ ++ if (HAS_DDI(dev_priv)) ++ return dev_priv->vbt.ddi_port_info[port].supports_edp; ++ ++ if (!dev_priv->vbt.child_dev_num) ++ return false; ++ ++ for (i = 0; i < dev_priv->vbt.child_dev_num; i++) { ++ child = dev_priv->vbt.child_dev + i; ++ ++ if (child->dvo_port == port_mapping[port] && ++ (child->device_type & DEVICE_TYPE_eDP_BITS) == ++ (DEVICE_TYPE_eDP & DEVICE_TYPE_eDP_BITS)) ++ return true; ++ } ++ ++ return false; ++} ++ ++static bool child_dev_is_dp_dual_mode(const struct child_device_config *child, ++ enum port port) ++{ ++ static const struct { ++ u16 dp, hdmi; ++ } port_mapping[] = { ++ /* ++ * Buggy VBTs may declare DP ports as having ++ * HDMI type dvo_port :( So let's check both. ++ */ ++ [PORT_B] = { DVO_PORT_DPB, DVO_PORT_HDMIB, }, ++ [PORT_C] = { DVO_PORT_DPC, DVO_PORT_HDMIC, }, ++ [PORT_D] = { DVO_PORT_DPD, DVO_PORT_HDMID, }, ++ [PORT_E] = { DVO_PORT_DPE, DVO_PORT_HDMIE, }, ++ [PORT_F] = { DVO_PORT_DPF, DVO_PORT_HDMIF, }, ++ }; ++ ++ if (port == PORT_A || port >= ARRAY_SIZE(port_mapping)) ++ return false; ++ ++ if ((child->device_type & DEVICE_TYPE_DP_DUAL_MODE_BITS) != ++ (DEVICE_TYPE_DP_DUAL_MODE & DEVICE_TYPE_DP_DUAL_MODE_BITS)) ++ return false; ++ ++ if (child->dvo_port == port_mapping[port].dp) ++ return true; ++ ++ /* Only accept a HDMI dvo_port as DP++ if it has an AUX channel */ ++ if (child->dvo_port == port_mapping[port].hdmi && ++ child->aux_channel != 0) ++ return true; ++ ++ return false; ++} ++ ++bool intel_bios_is_port_dp_dual_mode(struct drm_i915_private *dev_priv, ++ enum port port) ++{ ++ const struct child_device_config *child; ++ int i; ++ ++ for (i = 0; i < dev_priv->vbt.child_dev_num; i++) { ++ child = dev_priv->vbt.child_dev + i; ++ ++ if (child_dev_is_dp_dual_mode(child, port)) ++ return true; ++ } ++ ++ return false; ++} ++ ++/** ++ * intel_bios_is_dsi_present - is DSI present in VBT ++ * @dev_priv: i915 device instance ++ * @port: port for DSI if present ++ * ++ * Return true if DSI is present, and return the port in %port. ++ */ ++bool intel_bios_is_dsi_present(struct drm_i915_private *dev_priv, ++ enum port *port) ++{ ++ const struct child_device_config *child; ++ u8 dvo_port; ++ int i; ++ ++ for (i = 0; i < dev_priv->vbt.child_dev_num; i++) { ++ child = dev_priv->vbt.child_dev + i; ++ ++ if (!(child->device_type & DEVICE_TYPE_MIPI_OUTPUT)) ++ continue; ++ ++ dvo_port = child->dvo_port; ++ ++ if (dvo_port == DVO_PORT_MIPIA || ++ (dvo_port == DVO_PORT_MIPIB && INTEL_GEN(dev_priv) >= 11) || ++ (dvo_port == DVO_PORT_MIPIC && INTEL_GEN(dev_priv) < 11)) { ++ if (port) ++ *port = dvo_port - DVO_PORT_MIPIA; ++ return true; ++ } else if (dvo_port == DVO_PORT_MIPIB || ++ dvo_port == DVO_PORT_MIPIC || ++ dvo_port == DVO_PORT_MIPID) { ++ DRM_DEBUG_KMS("VBT has unsupported DSI port %c\n", ++ port_name(dvo_port - DVO_PORT_MIPIA)); ++ } ++ } ++ ++ return false; ++} ++ ++/** ++ * intel_bios_is_port_hpd_inverted - is HPD inverted for %port ++ * @dev_priv: i915 device instance ++ * @port: port to check ++ * ++ * Return true if HPD should be inverted for %port. ++ */ ++bool ++intel_bios_is_port_hpd_inverted(struct drm_i915_private *dev_priv, ++ enum port port) ++{ ++ const struct child_device_config *child; ++ int i; ++ ++ if (WARN_ON_ONCE(!IS_GEN9_LP(dev_priv))) ++ return false; ++ ++ for (i = 0; i < dev_priv->vbt.child_dev_num; i++) { ++ child = dev_priv->vbt.child_dev + i; ++ ++ if (!child->hpd_invert) ++ continue; ++ ++ switch (child->dvo_port) { ++ case DVO_PORT_DPA: ++ case DVO_PORT_HDMIA: ++ if (port == PORT_A) ++ return true; ++ break; ++ case DVO_PORT_DPB: ++ case DVO_PORT_HDMIB: ++ if (port == PORT_B) ++ return true; ++ break; ++ case DVO_PORT_DPC: ++ case DVO_PORT_HDMIC: ++ if (port == PORT_C) ++ return true; ++ break; ++ default: ++ break; ++ } ++ } ++ ++ return false; ++} ++ ++/** ++ * intel_bios_is_lspcon_present - if LSPCON is attached on %port ++ * @dev_priv: i915 device instance ++ * @port: port to check ++ * ++ * Return true if LSPCON is present on this port ++ */ ++bool ++intel_bios_is_lspcon_present(struct drm_i915_private *dev_priv, ++ enum port port) ++{ ++ const struct child_device_config *child; ++ int i; ++ ++ if (!HAS_LSPCON(dev_priv)) ++ return false; ++ ++ for (i = 0; i < dev_priv->vbt.child_dev_num; i++) { ++ child = dev_priv->vbt.child_dev + i; ++ ++ if (!child->lspcon) ++ continue; ++ ++ switch (child->dvo_port) { ++ case DVO_PORT_DPA: ++ case DVO_PORT_HDMIA: ++ if (port == PORT_A) ++ return true; ++ break; ++ case DVO_PORT_DPB: ++ case DVO_PORT_HDMIB: ++ if (port == PORT_B) ++ return true; ++ break; ++ case DVO_PORT_DPC: ++ case DVO_PORT_HDMIC: ++ if (port == PORT_C) ++ return true; ++ break; ++ case DVO_PORT_DPD: ++ case DVO_PORT_HDMID: ++ if (port == PORT_D) ++ return true; ++ break; ++ case DVO_PORT_DPF: ++ case DVO_PORT_HDMIF: ++ if (port == PORT_F) ++ return true; ++ break; ++ default: ++ break; ++ } ++ } ++ ++ return false; ++} ++ ++enum aux_ch intel_bios_port_aux_ch(struct drm_i915_private *dev_priv, ++ enum port port) ++{ ++ const struct ddi_vbt_port_info *info = ++ &dev_priv->vbt.ddi_port_info[port]; ++ enum aux_ch aux_ch; ++ ++ if (!info->alternate_aux_channel) { ++ aux_ch = (enum aux_ch)port; ++ ++ DRM_DEBUG_KMS("using AUX %c for port %c (platform default)\n", ++ aux_ch_name(aux_ch), port_name(port)); ++ return aux_ch; ++ } ++ ++ switch (info->alternate_aux_channel) { ++ case DP_AUX_A: ++ aux_ch = AUX_CH_A; ++ break; ++ case DP_AUX_B: ++ aux_ch = AUX_CH_B; ++ break; ++ case DP_AUX_C: ++ aux_ch = AUX_CH_C; ++ break; ++ case DP_AUX_D: ++ aux_ch = AUX_CH_D; ++ break; ++ case DP_AUX_E: ++ aux_ch = AUX_CH_E; ++ break; ++ case DP_AUX_F: ++ aux_ch = AUX_CH_F; ++ break; ++ default: ++ MISSING_CASE(info->alternate_aux_channel); ++ aux_ch = AUX_CH_A; ++ break; ++ } ++ ++ DRM_DEBUG_KMS("using AUX %c for port %c (VBT)\n", ++ aux_ch_name(aux_ch), port_name(port)); ++ ++ return aux_ch; ++} +diff --git a/drivers/gpu/drm/i915_legacy/intel_bios.h b/drivers/gpu/drm/i915_legacy/intel_bios.h +new file mode 100644 +index 000000000000..7e3545f65257 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_bios.h +@@ -0,0 +1,223 @@ ++/* ++ * Copyright © 2016 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE ++ * SOFTWARE. ++ */ ++ ++/* ++ * Please use intel_vbt_defs.h for VBT private data, to hide and abstract away ++ * the VBT from the rest of the driver. Add the parsed, clean data to struct ++ * intel_vbt_data within struct drm_i915_private. ++ */ ++ ++#ifndef _INTEL_BIOS_H_ ++#define _INTEL_BIOS_H_ ++ ++enum intel_backlight_type { ++ INTEL_BACKLIGHT_PMIC, ++ INTEL_BACKLIGHT_LPSS, ++ INTEL_BACKLIGHT_DISPLAY_DDI, ++ INTEL_BACKLIGHT_DSI_DCS, ++ INTEL_BACKLIGHT_PANEL_DRIVER_INTERFACE, ++}; ++ ++struct edp_power_seq { ++ u16 t1_t3; ++ u16 t8; ++ u16 t9; ++ u16 t10; ++ u16 t11_t12; ++} __packed; ++ ++/* ++ * MIPI Sequence Block definitions ++ * ++ * Note the VBT spec has AssertReset / DeassertReset swapped from their ++ * usual naming, we use the proper names here to avoid confusion when ++ * reading the code. ++ */ ++enum mipi_seq { ++ MIPI_SEQ_END = 0, ++ MIPI_SEQ_DEASSERT_RESET, /* Spec says MipiAssertResetPin */ ++ MIPI_SEQ_INIT_OTP, ++ MIPI_SEQ_DISPLAY_ON, ++ MIPI_SEQ_DISPLAY_OFF, ++ MIPI_SEQ_ASSERT_RESET, /* Spec says MipiDeassertResetPin */ ++ MIPI_SEQ_BACKLIGHT_ON, /* sequence block v2+ */ ++ MIPI_SEQ_BACKLIGHT_OFF, /* sequence block v2+ */ ++ MIPI_SEQ_TEAR_ON, /* sequence block v2+ */ ++ MIPI_SEQ_TEAR_OFF, /* sequence block v3+ */ ++ MIPI_SEQ_POWER_ON, /* sequence block v3+ */ ++ MIPI_SEQ_POWER_OFF, /* sequence block v3+ */ ++ MIPI_SEQ_MAX ++}; ++ ++enum mipi_seq_element { ++ MIPI_SEQ_ELEM_END = 0, ++ MIPI_SEQ_ELEM_SEND_PKT, ++ MIPI_SEQ_ELEM_DELAY, ++ MIPI_SEQ_ELEM_GPIO, ++ MIPI_SEQ_ELEM_I2C, /* sequence block v2+ */ ++ MIPI_SEQ_ELEM_SPI, /* sequence block v3+ */ ++ MIPI_SEQ_ELEM_PMIC, /* sequence block v3+ */ ++ MIPI_SEQ_ELEM_MAX ++}; ++ ++#define MIPI_DSI_UNDEFINED_PANEL_ID 0 ++#define MIPI_DSI_GENERIC_PANEL_ID 1 ++ ++struct mipi_config { ++ u16 panel_id; ++ ++ /* General Params */ ++ u32 enable_dithering:1; ++ u32 rsvd1:1; ++ u32 is_bridge:1; ++ ++ u32 panel_arch_type:2; ++ u32 is_cmd_mode:1; ++ ++#define NON_BURST_SYNC_PULSE 0x1 ++#define NON_BURST_SYNC_EVENTS 0x2 ++#define BURST_MODE 0x3 ++ u32 video_transfer_mode:2; ++ ++ u32 cabc_supported:1; ++#define PPS_BLC_PMIC 0 ++#define PPS_BLC_SOC 1 ++ u32 pwm_blc:1; ++ ++ /* Bit 13:10 */ ++#define PIXEL_FORMAT_RGB565 0x1 ++#define PIXEL_FORMAT_RGB666 0x2 ++#define PIXEL_FORMAT_RGB666_LOOSELY_PACKED 0x3 ++#define PIXEL_FORMAT_RGB888 0x4 ++ u32 videomode_color_format:4; ++ ++ /* Bit 15:14 */ ++#define ENABLE_ROTATION_0 0x0 ++#define ENABLE_ROTATION_90 0x1 ++#define ENABLE_ROTATION_180 0x2 ++#define ENABLE_ROTATION_270 0x3 ++ u32 rotation:2; ++ u32 bta_enabled:1; ++ u32 rsvd2:15; ++ ++ /* 2 byte Port Description */ ++#define DUAL_LINK_NOT_SUPPORTED 0 ++#define DUAL_LINK_FRONT_BACK 1 ++#define DUAL_LINK_PIXEL_ALT 2 ++ u16 dual_link:2; ++ u16 lane_cnt:2; ++ u16 pixel_overlap:3; ++ u16 rgb_flip:1; ++#define DL_DCS_PORT_A 0x00 ++#define DL_DCS_PORT_C 0x01 ++#define DL_DCS_PORT_A_AND_C 0x02 ++ u16 dl_dcs_cabc_ports:2; ++ u16 dl_dcs_backlight_ports:2; ++ u16 rsvd3:4; ++ ++ u16 rsvd4; ++ ++ u8 rsvd5; ++ u32 target_burst_mode_freq; ++ u32 dsi_ddr_clk; ++ u32 bridge_ref_clk; ++ ++#define BYTE_CLK_SEL_20MHZ 0 ++#define BYTE_CLK_SEL_10MHZ 1 ++#define BYTE_CLK_SEL_5MHZ 2 ++ u8 byte_clk_sel:2; ++ ++ u8 rsvd6:6; ++ ++ /* DPHY Flags */ ++ u16 dphy_param_valid:1; ++ u16 eot_pkt_disabled:1; ++ u16 enable_clk_stop:1; ++ u16 rsvd7:13; ++ ++ u32 hs_tx_timeout; ++ u32 lp_rx_timeout; ++ u32 turn_around_timeout; ++ u32 device_reset_timer; ++ u32 master_init_timer; ++ u32 dbi_bw_timer; ++ u32 lp_byte_clk_val; ++ ++ /* 4 byte Dphy Params */ ++ u32 prepare_cnt:6; ++ u32 rsvd8:2; ++ u32 clk_zero_cnt:8; ++ u32 trail_cnt:5; ++ u32 rsvd9:3; ++ u32 exit_zero_cnt:6; ++ u32 rsvd10:2; ++ ++ u32 clk_lane_switch_cnt; ++ u32 hl_switch_cnt; ++ ++ u32 rsvd11[6]; ++ ++ /* timings based on dphy spec */ ++ u8 tclk_miss; ++ u8 tclk_post; ++ u8 rsvd12; ++ u8 tclk_pre; ++ u8 tclk_prepare; ++ u8 tclk_settle; ++ u8 tclk_term_enable; ++ u8 tclk_trail; ++ u16 tclk_prepare_clkzero; ++ u8 rsvd13; ++ u8 td_term_enable; ++ u8 teot; ++ u8 ths_exit; ++ u8 ths_prepare; ++ u16 ths_prepare_hszero; ++ u8 rsvd14; ++ u8 ths_settle; ++ u8 ths_skip; ++ u8 ths_trail; ++ u8 tinit; ++ u8 tlpx; ++ u8 rsvd15[3]; ++ ++ /* GPIOs */ ++ u8 panel_enable; ++ u8 bl_enable; ++ u8 pwm_enable; ++ u8 reset_r_n; ++ u8 pwr_down_r; ++ u8 stdby_r_n; ++ ++} __packed; ++ ++/* all delays have a unit of 100us */ ++struct mipi_pps_data { ++ u16 panel_on_delay; ++ u16 bl_enable_delay; ++ u16 bl_disable_delay; ++ u16 panel_off_delay; ++ u16 panel_power_cycle_delay; ++} __packed; ++ ++#endif /* _INTEL_BIOS_H_ */ +diff --git a/drivers/gpu/drm/i915_legacy/intel_breadcrumbs.c b/drivers/gpu/drm/i915_legacy/intel_breadcrumbs.c +new file mode 100644 +index 000000000000..832cb6b1e9bd +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_breadcrumbs.c +@@ -0,0 +1,373 @@ ++/* ++ * Copyright © 2015 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#include ++#include ++#include ++ ++#include "i915_drv.h" ++ ++static void irq_enable(struct intel_engine_cs *engine) ++{ ++ if (!engine->irq_enable) ++ return; ++ ++ /* Caller disables interrupts */ ++ spin_lock(&engine->i915->irq_lock); ++ engine->irq_enable(engine); ++ spin_unlock(&engine->i915->irq_lock); ++} ++ ++static void irq_disable(struct intel_engine_cs *engine) ++{ ++ if (!engine->irq_disable) ++ return; ++ ++ /* Caller disables interrupts */ ++ spin_lock(&engine->i915->irq_lock); ++ engine->irq_disable(engine); ++ spin_unlock(&engine->i915->irq_lock); ++} ++ ++static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b) ++{ ++ lockdep_assert_held(&b->irq_lock); ++ ++ GEM_BUG_ON(!b->irq_enabled); ++ if (!--b->irq_enabled) ++ irq_disable(container_of(b, ++ struct intel_engine_cs, ++ breadcrumbs)); ++ ++ b->irq_armed = false; ++} ++ ++void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine) ++{ ++ struct intel_breadcrumbs *b = &engine->breadcrumbs; ++ ++ if (!b->irq_armed) ++ return; ++ ++ spin_lock_irq(&b->irq_lock); ++ if (b->irq_armed) ++ __intel_breadcrumbs_disarm_irq(b); ++ spin_unlock_irq(&b->irq_lock); ++} ++ ++static inline bool __request_completed(const struct i915_request *rq) ++{ ++ return i915_seqno_passed(__hwsp_seqno(rq), rq->fence.seqno); ++} ++ ++static bool ++__dma_fence_signal(struct dma_fence *fence) ++{ ++ return !test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags); ++} ++ ++static void ++__dma_fence_signal__timestamp(struct dma_fence *fence, ktime_t timestamp) ++{ ++ fence->timestamp = timestamp; ++ set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags); ++ trace_dma_fence_signaled(fence); ++} ++ ++static void ++__dma_fence_signal__notify(struct dma_fence *fence) ++{ ++ struct dma_fence_cb *cur, *tmp; ++ ++ lockdep_assert_held(fence->lock); ++ lockdep_assert_irqs_disabled(); ++ ++ list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) { ++ INIT_LIST_HEAD(&cur->node); ++ cur->func(fence, cur); ++ } ++ INIT_LIST_HEAD(&fence->cb_list); ++} ++ ++void intel_engine_breadcrumbs_irq(struct intel_engine_cs *engine) ++{ ++ struct intel_breadcrumbs *b = &engine->breadcrumbs; ++ const ktime_t timestamp = ktime_get(); ++ struct intel_context *ce, *cn; ++ struct list_head *pos, *next; ++ LIST_HEAD(signal); ++ ++ spin_lock(&b->irq_lock); ++ ++ if (b->irq_armed && list_empty(&b->signalers)) ++ __intel_breadcrumbs_disarm_irq(b); ++ ++ list_for_each_entry_safe(ce, cn, &b->signalers, signal_link) { ++ GEM_BUG_ON(list_empty(&ce->signals)); ++ ++ list_for_each_safe(pos, next, &ce->signals) { ++ struct i915_request *rq = ++ list_entry(pos, typeof(*rq), signal_link); ++ ++ if (!__request_completed(rq)) ++ break; ++ ++ GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_SIGNAL, ++ &rq->fence.flags)); ++ clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); ++ ++ if (!__dma_fence_signal(&rq->fence)) ++ continue; ++ ++ /* ++ * Queue for execution after dropping the signaling ++ * spinlock as the callback chain may end up adding ++ * more signalers to the same context or engine. ++ */ ++ i915_request_get(rq); ++ list_add_tail(&rq->signal_link, &signal); ++ } ++ ++ /* ++ * We process the list deletion in bulk, only using a list_add ++ * (not list_move) above but keeping the status of ++ * rq->signal_link known with the I915_FENCE_FLAG_SIGNAL bit. ++ */ ++ if (!list_is_first(pos, &ce->signals)) { ++ /* Advance the list to the first incomplete request */ ++ __list_del_many(&ce->signals, pos); ++ if (&ce->signals == pos) /* now empty */ ++ list_del_init(&ce->signal_link); ++ } ++ } ++ ++ spin_unlock(&b->irq_lock); ++ ++ list_for_each_safe(pos, next, &signal) { ++ struct i915_request *rq = ++ list_entry(pos, typeof(*rq), signal_link); ++ ++ __dma_fence_signal__timestamp(&rq->fence, timestamp); ++ ++ spin_lock(&rq->lock); ++ __dma_fence_signal__notify(&rq->fence); ++ spin_unlock(&rq->lock); ++ ++ i915_request_put(rq); ++ } ++} ++ ++void intel_engine_signal_breadcrumbs(struct intel_engine_cs *engine) ++{ ++ local_irq_disable(); ++ intel_engine_breadcrumbs_irq(engine); ++ local_irq_enable(); ++} ++ ++static void signal_irq_work(struct irq_work *work) ++{ ++ struct intel_engine_cs *engine = ++ container_of(work, typeof(*engine), breadcrumbs.irq_work); ++ ++ intel_engine_breadcrumbs_irq(engine); ++} ++ ++void intel_engine_pin_breadcrumbs_irq(struct intel_engine_cs *engine) ++{ ++ struct intel_breadcrumbs *b = &engine->breadcrumbs; ++ ++ spin_lock_irq(&b->irq_lock); ++ if (!b->irq_enabled++) ++ irq_enable(engine); ++ GEM_BUG_ON(!b->irq_enabled); /* no overflow! */ ++ spin_unlock_irq(&b->irq_lock); ++} ++ ++void intel_engine_unpin_breadcrumbs_irq(struct intel_engine_cs *engine) ++{ ++ struct intel_breadcrumbs *b = &engine->breadcrumbs; ++ ++ spin_lock_irq(&b->irq_lock); ++ GEM_BUG_ON(!b->irq_enabled); /* no underflow! */ ++ if (!--b->irq_enabled) ++ irq_disable(engine); ++ spin_unlock_irq(&b->irq_lock); ++} ++ ++static void __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b) ++{ ++ struct intel_engine_cs *engine = ++ container_of(b, struct intel_engine_cs, breadcrumbs); ++ ++ lockdep_assert_held(&b->irq_lock); ++ if (b->irq_armed) ++ return; ++ ++ /* ++ * The breadcrumb irq will be disarmed on the interrupt after the ++ * waiters are signaled. This gives us a single interrupt window in ++ * which we can add a new waiter and avoid the cost of re-enabling ++ * the irq. ++ */ ++ b->irq_armed = true; ++ ++ /* ++ * Since we are waiting on a request, the GPU should be busy ++ * and should have its own rpm reference. This is tracked ++ * by i915->gt.awake, we can forgo holding our own wakref ++ * for the interrupt as before i915->gt.awake is released (when ++ * the driver is idle) we disarm the breadcrumbs. ++ */ ++ ++ if (!b->irq_enabled++) ++ irq_enable(engine); ++} ++ ++void intel_engine_init_breadcrumbs(struct intel_engine_cs *engine) ++{ ++ struct intel_breadcrumbs *b = &engine->breadcrumbs; ++ ++ spin_lock_init(&b->irq_lock); ++ INIT_LIST_HEAD(&b->signalers); ++ ++ init_irq_work(&b->irq_work, signal_irq_work); ++} ++ ++void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine) ++{ ++ struct intel_breadcrumbs *b = &engine->breadcrumbs; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&b->irq_lock, flags); ++ ++ if (b->irq_enabled) ++ irq_enable(engine); ++ else ++ irq_disable(engine); ++ ++ spin_unlock_irqrestore(&b->irq_lock, flags); ++} ++ ++void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine) ++{ ++} ++ ++bool i915_request_enable_breadcrumb(struct i915_request *rq) ++{ ++ lockdep_assert_held(&rq->lock); ++ lockdep_assert_irqs_disabled(); ++ ++ if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) { ++ struct intel_breadcrumbs *b = &rq->engine->breadcrumbs; ++ struct intel_context *ce = rq->hw_context; ++ struct list_head *pos; ++ ++ spin_lock(&b->irq_lock); ++ GEM_BUG_ON(test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)); ++ ++ __intel_breadcrumbs_arm_irq(b); ++ ++ /* ++ * We keep the seqno in retirement order, so we can break ++ * inside intel_engine_breadcrumbs_irq as soon as we've passed ++ * the last completed request (or seen a request that hasn't ++ * event started). We could iterate the timeline->requests list, ++ * but keeping a separate signalers_list has the advantage of ++ * hopefully being much smaller than the full list and so ++ * provides faster iteration and detection when there are no ++ * more interrupts required for this context. ++ * ++ * We typically expect to add new signalers in order, so we ++ * start looking for our insertion point from the tail of ++ * the list. ++ */ ++ list_for_each_prev(pos, &ce->signals) { ++ struct i915_request *it = ++ list_entry(pos, typeof(*it), signal_link); ++ ++ if (i915_seqno_passed(rq->fence.seqno, it->fence.seqno)) ++ break; ++ } ++ list_add(&rq->signal_link, pos); ++ if (pos == &ce->signals) /* catch transitions from empty list */ ++ list_move_tail(&ce->signal_link, &b->signalers); ++ ++ set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); ++ spin_unlock(&b->irq_lock); ++ } ++ ++ return !__request_completed(rq); ++} ++ ++void i915_request_cancel_breadcrumb(struct i915_request *rq) ++{ ++ struct intel_breadcrumbs *b = &rq->engine->breadcrumbs; ++ ++ lockdep_assert_held(&rq->lock); ++ lockdep_assert_irqs_disabled(); ++ ++ /* ++ * We must wait for b->irq_lock so that we know the interrupt handler ++ * has released its reference to the intel_context and has completed ++ * the DMA_FENCE_FLAG_SIGNALED_BIT/I915_FENCE_FLAG_SIGNAL dance (if ++ * required). ++ */ ++ spin_lock(&b->irq_lock); ++ if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) { ++ struct intel_context *ce = rq->hw_context; ++ ++ list_del(&rq->signal_link); ++ if (list_empty(&ce->signals)) ++ list_del_init(&ce->signal_link); ++ ++ clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); ++ } ++ spin_unlock(&b->irq_lock); ++} ++ ++void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine, ++ struct drm_printer *p) ++{ ++ struct intel_breadcrumbs *b = &engine->breadcrumbs; ++ struct intel_context *ce; ++ struct i915_request *rq; ++ ++ if (list_empty(&b->signalers)) ++ return; ++ ++ drm_printf(p, "Signals:\n"); ++ ++ spin_lock_irq(&b->irq_lock); ++ list_for_each_entry(ce, &b->signalers, signal_link) { ++ list_for_each_entry(rq, &ce->signals, signal_link) { ++ drm_printf(p, "\t[%llx:%llx%s] @ %dms\n", ++ rq->fence.context, rq->fence.seqno, ++ i915_request_completed(rq) ? "!" : ++ i915_request_started(rq) ? "*" : ++ "", ++ jiffies_to_msecs(jiffies - rq->emitted_jiffies)); ++ } ++ } ++ spin_unlock_irq(&b->irq_lock); ++} +diff --git a/drivers/gpu/drm/i915_legacy/intel_cdclk.c b/drivers/gpu/drm/i915_legacy/intel_cdclk.c +new file mode 100644 +index 000000000000..fd5236da039f +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_cdclk.c +@@ -0,0 +1,2904 @@ ++/* ++ * Copyright © 2006-2017 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER ++ * DEALINGS IN THE SOFTWARE. ++ */ ++ ++#include "intel_cdclk.h" ++#include "intel_drv.h" ++ ++/** ++ * DOC: CDCLK / RAWCLK ++ * ++ * The display engine uses several different clocks to do its work. There ++ * are two main clocks involved that aren't directly related to the actual ++ * pixel clock or any symbol/bit clock of the actual output port. These ++ * are the core display clock (CDCLK) and RAWCLK. ++ * ++ * CDCLK clocks most of the display pipe logic, and thus its frequency ++ * must be high enough to support the rate at which pixels are flowing ++ * through the pipes. Downscaling must also be accounted as that increases ++ * the effective pixel rate. ++ * ++ * On several platforms the CDCLK frequency can be changed dynamically ++ * to minimize power consumption for a given display configuration. ++ * Typically changes to the CDCLK frequency require all the display pipes ++ * to be shut down while the frequency is being changed. ++ * ++ * On SKL+ the DMC will toggle the CDCLK off/on during DC5/6 entry/exit. ++ * DMC will not change the active CDCLK frequency however, so that part ++ * will still be performed by the driver directly. ++ * ++ * RAWCLK is a fixed frequency clock, often used by various auxiliary ++ * blocks such as AUX CH or backlight PWM. Hence the only thing we ++ * really need to know about RAWCLK is its frequency so that various ++ * dividers can be programmed correctly. ++ */ ++ ++static void fixed_133mhz_get_cdclk(struct drm_i915_private *dev_priv, ++ struct intel_cdclk_state *cdclk_state) ++{ ++ cdclk_state->cdclk = 133333; ++} ++ ++static void fixed_200mhz_get_cdclk(struct drm_i915_private *dev_priv, ++ struct intel_cdclk_state *cdclk_state) ++{ ++ cdclk_state->cdclk = 200000; ++} ++ ++static void fixed_266mhz_get_cdclk(struct drm_i915_private *dev_priv, ++ struct intel_cdclk_state *cdclk_state) ++{ ++ cdclk_state->cdclk = 266667; ++} ++ ++static void fixed_333mhz_get_cdclk(struct drm_i915_private *dev_priv, ++ struct intel_cdclk_state *cdclk_state) ++{ ++ cdclk_state->cdclk = 333333; ++} ++ ++static void fixed_400mhz_get_cdclk(struct drm_i915_private *dev_priv, ++ struct intel_cdclk_state *cdclk_state) ++{ ++ cdclk_state->cdclk = 400000; ++} ++ ++static void fixed_450mhz_get_cdclk(struct drm_i915_private *dev_priv, ++ struct intel_cdclk_state *cdclk_state) ++{ ++ cdclk_state->cdclk = 450000; ++} ++ ++static void i85x_get_cdclk(struct drm_i915_private *dev_priv, ++ struct intel_cdclk_state *cdclk_state) ++{ ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ u16 hpllcc = 0; ++ ++ /* ++ * 852GM/852GMV only supports 133 MHz and the HPLLCC ++ * encoding is different :( ++ * FIXME is this the right way to detect 852GM/852GMV? ++ */ ++ if (pdev->revision == 0x1) { ++ cdclk_state->cdclk = 133333; ++ return; ++ } ++ ++ pci_bus_read_config_word(pdev->bus, ++ PCI_DEVFN(0, 3), HPLLCC, &hpllcc); ++ ++ /* Assume that the hardware is in the high speed state. This ++ * should be the default. ++ */ ++ switch (hpllcc & GC_CLOCK_CONTROL_MASK) { ++ case GC_CLOCK_133_200: ++ case GC_CLOCK_133_200_2: ++ case GC_CLOCK_100_200: ++ cdclk_state->cdclk = 200000; ++ break; ++ case GC_CLOCK_166_250: ++ cdclk_state->cdclk = 250000; ++ break; ++ case GC_CLOCK_100_133: ++ cdclk_state->cdclk = 133333; ++ break; ++ case GC_CLOCK_133_266: ++ case GC_CLOCK_133_266_2: ++ case GC_CLOCK_166_266: ++ cdclk_state->cdclk = 266667; ++ break; ++ } ++} ++ ++static void i915gm_get_cdclk(struct drm_i915_private *dev_priv, ++ struct intel_cdclk_state *cdclk_state) ++{ ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ u16 gcfgc = 0; ++ ++ pci_read_config_word(pdev, GCFGC, &gcfgc); ++ ++ if (gcfgc & GC_LOW_FREQUENCY_ENABLE) { ++ cdclk_state->cdclk = 133333; ++ return; ++ } ++ ++ switch (gcfgc & GC_DISPLAY_CLOCK_MASK) { ++ case GC_DISPLAY_CLOCK_333_320_MHZ: ++ cdclk_state->cdclk = 333333; ++ break; ++ default: ++ case GC_DISPLAY_CLOCK_190_200_MHZ: ++ cdclk_state->cdclk = 190000; ++ break; ++ } ++} ++ ++static void i945gm_get_cdclk(struct drm_i915_private *dev_priv, ++ struct intel_cdclk_state *cdclk_state) ++{ ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ u16 gcfgc = 0; ++ ++ pci_read_config_word(pdev, GCFGC, &gcfgc); ++ ++ if (gcfgc & GC_LOW_FREQUENCY_ENABLE) { ++ cdclk_state->cdclk = 133333; ++ return; ++ } ++ ++ switch (gcfgc & GC_DISPLAY_CLOCK_MASK) { ++ case GC_DISPLAY_CLOCK_333_320_MHZ: ++ cdclk_state->cdclk = 320000; ++ break; ++ default: ++ case GC_DISPLAY_CLOCK_190_200_MHZ: ++ cdclk_state->cdclk = 200000; ++ break; ++ } ++} ++ ++static unsigned int intel_hpll_vco(struct drm_i915_private *dev_priv) ++{ ++ static const unsigned int blb_vco[8] = { ++ [0] = 3200000, ++ [1] = 4000000, ++ [2] = 5333333, ++ [3] = 4800000, ++ [4] = 6400000, ++ }; ++ static const unsigned int pnv_vco[8] = { ++ [0] = 3200000, ++ [1] = 4000000, ++ [2] = 5333333, ++ [3] = 4800000, ++ [4] = 2666667, ++ }; ++ static const unsigned int cl_vco[8] = { ++ [0] = 3200000, ++ [1] = 4000000, ++ [2] = 5333333, ++ [3] = 6400000, ++ [4] = 3333333, ++ [5] = 3566667, ++ [6] = 4266667, ++ }; ++ static const unsigned int elk_vco[8] = { ++ [0] = 3200000, ++ [1] = 4000000, ++ [2] = 5333333, ++ [3] = 4800000, ++ }; ++ static const unsigned int ctg_vco[8] = { ++ [0] = 3200000, ++ [1] = 4000000, ++ [2] = 5333333, ++ [3] = 6400000, ++ [4] = 2666667, ++ [5] = 4266667, ++ }; ++ const unsigned int *vco_table; ++ unsigned int vco; ++ u8 tmp = 0; ++ ++ /* FIXME other chipsets? */ ++ if (IS_GM45(dev_priv)) ++ vco_table = ctg_vco; ++ else if (IS_G45(dev_priv)) ++ vco_table = elk_vco; ++ else if (IS_I965GM(dev_priv)) ++ vco_table = cl_vco; ++ else if (IS_PINEVIEW(dev_priv)) ++ vco_table = pnv_vco; ++ else if (IS_G33(dev_priv)) ++ vco_table = blb_vco; ++ else ++ return 0; ++ ++ tmp = I915_READ(IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv) ? ++ HPLLVCO_MOBILE : HPLLVCO); ++ ++ vco = vco_table[tmp & 0x7]; ++ if (vco == 0) ++ DRM_ERROR("Bad HPLL VCO (HPLLVCO=0x%02x)\n", tmp); ++ else ++ DRM_DEBUG_KMS("HPLL VCO %u kHz\n", vco); ++ ++ return vco; ++} ++ ++static void g33_get_cdclk(struct drm_i915_private *dev_priv, ++ struct intel_cdclk_state *cdclk_state) ++{ ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ static const u8 div_3200[] = { 12, 10, 8, 7, 5, 16 }; ++ static const u8 div_4000[] = { 14, 12, 10, 8, 6, 20 }; ++ static const u8 div_4800[] = { 20, 14, 12, 10, 8, 24 }; ++ static const u8 div_5333[] = { 20, 16, 12, 12, 8, 28 }; ++ const u8 *div_table; ++ unsigned int cdclk_sel; ++ u16 tmp = 0; ++ ++ cdclk_state->vco = intel_hpll_vco(dev_priv); ++ ++ pci_read_config_word(pdev, GCFGC, &tmp); ++ ++ cdclk_sel = (tmp >> 4) & 0x7; ++ ++ if (cdclk_sel >= ARRAY_SIZE(div_3200)) ++ goto fail; ++ ++ switch (cdclk_state->vco) { ++ case 3200000: ++ div_table = div_3200; ++ break; ++ case 4000000: ++ div_table = div_4000; ++ break; ++ case 4800000: ++ div_table = div_4800; ++ break; ++ case 5333333: ++ div_table = div_5333; ++ break; ++ default: ++ goto fail; ++ } ++ ++ cdclk_state->cdclk = DIV_ROUND_CLOSEST(cdclk_state->vco, ++ div_table[cdclk_sel]); ++ return; ++ ++fail: ++ DRM_ERROR("Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%08x\n", ++ cdclk_state->vco, tmp); ++ cdclk_state->cdclk = 190476; ++} ++ ++static void pnv_get_cdclk(struct drm_i915_private *dev_priv, ++ struct intel_cdclk_state *cdclk_state) ++{ ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ u16 gcfgc = 0; ++ ++ pci_read_config_word(pdev, GCFGC, &gcfgc); ++ ++ switch (gcfgc & GC_DISPLAY_CLOCK_MASK) { ++ case GC_DISPLAY_CLOCK_267_MHZ_PNV: ++ cdclk_state->cdclk = 266667; ++ break; ++ case GC_DISPLAY_CLOCK_333_MHZ_PNV: ++ cdclk_state->cdclk = 333333; ++ break; ++ case GC_DISPLAY_CLOCK_444_MHZ_PNV: ++ cdclk_state->cdclk = 444444; ++ break; ++ case GC_DISPLAY_CLOCK_200_MHZ_PNV: ++ cdclk_state->cdclk = 200000; ++ break; ++ default: ++ DRM_ERROR("Unknown pnv display core clock 0x%04x\n", gcfgc); ++ /* fall through */ ++ case GC_DISPLAY_CLOCK_133_MHZ_PNV: ++ cdclk_state->cdclk = 133333; ++ break; ++ case GC_DISPLAY_CLOCK_167_MHZ_PNV: ++ cdclk_state->cdclk = 166667; ++ break; ++ } ++} ++ ++static void i965gm_get_cdclk(struct drm_i915_private *dev_priv, ++ struct intel_cdclk_state *cdclk_state) ++{ ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ static const u8 div_3200[] = { 16, 10, 8 }; ++ static const u8 div_4000[] = { 20, 12, 10 }; ++ static const u8 div_5333[] = { 24, 16, 14 }; ++ const u8 *div_table; ++ unsigned int cdclk_sel; ++ u16 tmp = 0; ++ ++ cdclk_state->vco = intel_hpll_vco(dev_priv); ++ ++ pci_read_config_word(pdev, GCFGC, &tmp); ++ ++ cdclk_sel = ((tmp >> 8) & 0x1f) - 1; ++ ++ if (cdclk_sel >= ARRAY_SIZE(div_3200)) ++ goto fail; ++ ++ switch (cdclk_state->vco) { ++ case 3200000: ++ div_table = div_3200; ++ break; ++ case 4000000: ++ div_table = div_4000; ++ break; ++ case 5333333: ++ div_table = div_5333; ++ break; ++ default: ++ goto fail; ++ } ++ ++ cdclk_state->cdclk = DIV_ROUND_CLOSEST(cdclk_state->vco, ++ div_table[cdclk_sel]); ++ return; ++ ++fail: ++ DRM_ERROR("Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%04x\n", ++ cdclk_state->vco, tmp); ++ cdclk_state->cdclk = 200000; ++} ++ ++static void gm45_get_cdclk(struct drm_i915_private *dev_priv, ++ struct intel_cdclk_state *cdclk_state) ++{ ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ unsigned int cdclk_sel; ++ u16 tmp = 0; ++ ++ cdclk_state->vco = intel_hpll_vco(dev_priv); ++ ++ pci_read_config_word(pdev, GCFGC, &tmp); ++ ++ cdclk_sel = (tmp >> 12) & 0x1; ++ ++ switch (cdclk_state->vco) { ++ case 2666667: ++ case 4000000: ++ case 5333333: ++ cdclk_state->cdclk = cdclk_sel ? 333333 : 222222; ++ break; ++ case 3200000: ++ cdclk_state->cdclk = cdclk_sel ? 320000 : 228571; ++ break; ++ default: ++ DRM_ERROR("Unable to determine CDCLK. HPLL VCO=%u, CFGC=0x%04x\n", ++ cdclk_state->vco, tmp); ++ cdclk_state->cdclk = 222222; ++ break; ++ } ++} ++ ++static void hsw_get_cdclk(struct drm_i915_private *dev_priv, ++ struct intel_cdclk_state *cdclk_state) ++{ ++ u32 lcpll = I915_READ(LCPLL_CTL); ++ u32 freq = lcpll & LCPLL_CLK_FREQ_MASK; ++ ++ if (lcpll & LCPLL_CD_SOURCE_FCLK) ++ cdclk_state->cdclk = 800000; ++ else if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT) ++ cdclk_state->cdclk = 450000; ++ else if (freq == LCPLL_CLK_FREQ_450) ++ cdclk_state->cdclk = 450000; ++ else if (IS_HSW_ULT(dev_priv)) ++ cdclk_state->cdclk = 337500; ++ else ++ cdclk_state->cdclk = 540000; ++} ++ ++static int vlv_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk) ++{ ++ int freq_320 = (dev_priv->hpll_freq << 1) % 320000 != 0 ? ++ 333333 : 320000; ++ ++ /* ++ * We seem to get an unstable or solid color picture at 200MHz. ++ * Not sure what's wrong. For now use 200MHz only when all pipes ++ * are off. ++ */ ++ if (IS_VALLEYVIEW(dev_priv) && min_cdclk > freq_320) ++ return 400000; ++ else if (min_cdclk > 266667) ++ return freq_320; ++ else if (min_cdclk > 0) ++ return 266667; ++ else ++ return 200000; ++} ++ ++static u8 vlv_calc_voltage_level(struct drm_i915_private *dev_priv, int cdclk) ++{ ++ if (IS_VALLEYVIEW(dev_priv)) { ++ if (cdclk >= 320000) /* jump to highest voltage for 400MHz too */ ++ return 2; ++ else if (cdclk >= 266667) ++ return 1; ++ else ++ return 0; ++ } else { ++ /* ++ * Specs are full of misinformation, but testing on actual ++ * hardware has shown that we just need to write the desired ++ * CCK divider into the Punit register. ++ */ ++ return DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, cdclk) - 1; ++ } ++} ++ ++static void vlv_get_cdclk(struct drm_i915_private *dev_priv, ++ struct intel_cdclk_state *cdclk_state) ++{ ++ u32 val; ++ ++ cdclk_state->vco = vlv_get_hpll_vco(dev_priv); ++ cdclk_state->cdclk = vlv_get_cck_clock(dev_priv, "cdclk", ++ CCK_DISPLAY_CLOCK_CONTROL, ++ cdclk_state->vco); ++ ++ mutex_lock(&dev_priv->pcu_lock); ++ val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM); ++ mutex_unlock(&dev_priv->pcu_lock); ++ ++ if (IS_VALLEYVIEW(dev_priv)) ++ cdclk_state->voltage_level = (val & DSPFREQGUAR_MASK) >> ++ DSPFREQGUAR_SHIFT; ++ else ++ cdclk_state->voltage_level = (val & DSPFREQGUAR_MASK_CHV) >> ++ DSPFREQGUAR_SHIFT_CHV; ++} ++ ++static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv) ++{ ++ unsigned int credits, default_credits; ++ ++ if (IS_CHERRYVIEW(dev_priv)) ++ default_credits = PFI_CREDIT(12); ++ else ++ default_credits = PFI_CREDIT(8); ++ ++ if (dev_priv->cdclk.hw.cdclk >= dev_priv->czclk_freq) { ++ /* CHV suggested value is 31 or 63 */ ++ if (IS_CHERRYVIEW(dev_priv)) ++ credits = PFI_CREDIT_63; ++ else ++ credits = PFI_CREDIT(15); ++ } else { ++ credits = default_credits; ++ } ++ ++ /* ++ * WA - write default credits before re-programming ++ * FIXME: should we also set the resend bit here? ++ */ ++ I915_WRITE(GCI_CONTROL, VGA_FAST_MODE_DISABLE | ++ default_credits); ++ ++ I915_WRITE(GCI_CONTROL, VGA_FAST_MODE_DISABLE | ++ credits | PFI_CREDIT_RESEND); ++ ++ /* ++ * FIXME is this guaranteed to clear ++ * immediately or should we poll for it? ++ */ ++ WARN_ON(I915_READ(GCI_CONTROL) & PFI_CREDIT_RESEND); ++} ++ ++static void vlv_set_cdclk(struct drm_i915_private *dev_priv, ++ const struct intel_cdclk_state *cdclk_state, ++ enum pipe pipe) ++{ ++ int cdclk = cdclk_state->cdclk; ++ u32 val, cmd = cdclk_state->voltage_level; ++ intel_wakeref_t wakeref; ++ ++ switch (cdclk) { ++ case 400000: ++ case 333333: ++ case 320000: ++ case 266667: ++ case 200000: ++ break; ++ default: ++ MISSING_CASE(cdclk); ++ return; ++ } ++ ++ /* There are cases where we can end up here with power domains ++ * off and a CDCLK frequency other than the minimum, like when ++ * issuing a modeset without actually changing any display after ++ * a system suspend. So grab the PIPE-A domain, which covers ++ * the HW blocks needed for the following programming. ++ */ ++ wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_PIPE_A); ++ ++ mutex_lock(&dev_priv->pcu_lock); ++ val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM); ++ val &= ~DSPFREQGUAR_MASK; ++ val |= (cmd << DSPFREQGUAR_SHIFT); ++ vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val); ++ if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) & ++ DSPFREQSTAT_MASK) == (cmd << DSPFREQSTAT_SHIFT), ++ 50)) { ++ DRM_ERROR("timed out waiting for CDclk change\n"); ++ } ++ mutex_unlock(&dev_priv->pcu_lock); ++ ++ mutex_lock(&dev_priv->sb_lock); ++ ++ if (cdclk == 400000) { ++ u32 divider; ++ ++ divider = DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, ++ cdclk) - 1; ++ ++ /* adjust cdclk divider */ ++ val = vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL); ++ val &= ~CCK_FREQUENCY_VALUES; ++ val |= divider; ++ vlv_cck_write(dev_priv, CCK_DISPLAY_CLOCK_CONTROL, val); ++ ++ if (wait_for((vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL) & ++ CCK_FREQUENCY_STATUS) == (divider << CCK_FREQUENCY_STATUS_SHIFT), ++ 50)) ++ DRM_ERROR("timed out waiting for CDclk change\n"); ++ } ++ ++ /* adjust self-refresh exit latency value */ ++ val = vlv_bunit_read(dev_priv, BUNIT_REG_BISOC); ++ val &= ~0x7f; ++ ++ /* ++ * For high bandwidth configs, we set a higher latency in the bunit ++ * so that the core display fetch happens in time to avoid underruns. ++ */ ++ if (cdclk == 400000) ++ val |= 4500 / 250; /* 4.5 usec */ ++ else ++ val |= 3000 / 250; /* 3.0 usec */ ++ vlv_bunit_write(dev_priv, BUNIT_REG_BISOC, val); ++ ++ mutex_unlock(&dev_priv->sb_lock); ++ ++ intel_update_cdclk(dev_priv); ++ ++ vlv_program_pfi_credits(dev_priv); ++ ++ intel_display_power_put(dev_priv, POWER_DOMAIN_PIPE_A, wakeref); ++} ++ ++static void chv_set_cdclk(struct drm_i915_private *dev_priv, ++ const struct intel_cdclk_state *cdclk_state, ++ enum pipe pipe) ++{ ++ int cdclk = cdclk_state->cdclk; ++ u32 val, cmd = cdclk_state->voltage_level; ++ intel_wakeref_t wakeref; ++ ++ switch (cdclk) { ++ case 333333: ++ case 320000: ++ case 266667: ++ case 200000: ++ break; ++ default: ++ MISSING_CASE(cdclk); ++ return; ++ } ++ ++ /* There are cases where we can end up here with power domains ++ * off and a CDCLK frequency other than the minimum, like when ++ * issuing a modeset without actually changing any display after ++ * a system suspend. So grab the PIPE-A domain, which covers ++ * the HW blocks needed for the following programming. ++ */ ++ wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_PIPE_A); ++ ++ mutex_lock(&dev_priv->pcu_lock); ++ val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM); ++ val &= ~DSPFREQGUAR_MASK_CHV; ++ val |= (cmd << DSPFREQGUAR_SHIFT_CHV); ++ vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val); ++ if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) & ++ DSPFREQSTAT_MASK_CHV) == (cmd << DSPFREQSTAT_SHIFT_CHV), ++ 50)) { ++ DRM_ERROR("timed out waiting for CDclk change\n"); ++ } ++ mutex_unlock(&dev_priv->pcu_lock); ++ ++ intel_update_cdclk(dev_priv); ++ ++ vlv_program_pfi_credits(dev_priv); ++ ++ intel_display_power_put(dev_priv, POWER_DOMAIN_PIPE_A, wakeref); ++} ++ ++static int bdw_calc_cdclk(int min_cdclk) ++{ ++ if (min_cdclk > 540000) ++ return 675000; ++ else if (min_cdclk > 450000) ++ return 540000; ++ else if (min_cdclk > 337500) ++ return 450000; ++ else ++ return 337500; ++} ++ ++static u8 bdw_calc_voltage_level(int cdclk) ++{ ++ switch (cdclk) { ++ default: ++ case 337500: ++ return 2; ++ case 450000: ++ return 0; ++ case 540000: ++ return 1; ++ case 675000: ++ return 3; ++ } ++} ++ ++static void bdw_get_cdclk(struct drm_i915_private *dev_priv, ++ struct intel_cdclk_state *cdclk_state) ++{ ++ u32 lcpll = I915_READ(LCPLL_CTL); ++ u32 freq = lcpll & LCPLL_CLK_FREQ_MASK; ++ ++ if (lcpll & LCPLL_CD_SOURCE_FCLK) ++ cdclk_state->cdclk = 800000; ++ else if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT) ++ cdclk_state->cdclk = 450000; ++ else if (freq == LCPLL_CLK_FREQ_450) ++ cdclk_state->cdclk = 450000; ++ else if (freq == LCPLL_CLK_FREQ_54O_BDW) ++ cdclk_state->cdclk = 540000; ++ else if (freq == LCPLL_CLK_FREQ_337_5_BDW) ++ cdclk_state->cdclk = 337500; ++ else ++ cdclk_state->cdclk = 675000; ++ ++ /* ++ * Can't read this out :( Let's assume it's ++ * at least what the CDCLK frequency requires. ++ */ ++ cdclk_state->voltage_level = ++ bdw_calc_voltage_level(cdclk_state->cdclk); ++} ++ ++static void bdw_set_cdclk(struct drm_i915_private *dev_priv, ++ const struct intel_cdclk_state *cdclk_state, ++ enum pipe pipe) ++{ ++ int cdclk = cdclk_state->cdclk; ++ u32 val; ++ int ret; ++ ++ if (WARN((I915_READ(LCPLL_CTL) & ++ (LCPLL_PLL_DISABLE | LCPLL_PLL_LOCK | ++ LCPLL_CD_CLOCK_DISABLE | LCPLL_ROOT_CD_CLOCK_DISABLE | ++ LCPLL_CD2X_CLOCK_DISABLE | LCPLL_POWER_DOWN_ALLOW | ++ LCPLL_CD_SOURCE_FCLK)) != LCPLL_PLL_LOCK, ++ "trying to change cdclk frequency with cdclk not enabled\n")) ++ return; ++ ++ mutex_lock(&dev_priv->pcu_lock); ++ ret = sandybridge_pcode_write(dev_priv, ++ BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ, 0x0); ++ mutex_unlock(&dev_priv->pcu_lock); ++ if (ret) { ++ DRM_ERROR("failed to inform pcode about cdclk change\n"); ++ return; ++ } ++ ++ val = I915_READ(LCPLL_CTL); ++ val |= LCPLL_CD_SOURCE_FCLK; ++ I915_WRITE(LCPLL_CTL, val); ++ ++ /* ++ * According to the spec, it should be enough to poll for this 1 us. ++ * However, extensive testing shows that this can take longer. ++ */ ++ if (wait_for_us(I915_READ(LCPLL_CTL) & ++ LCPLL_CD_SOURCE_FCLK_DONE, 100)) ++ DRM_ERROR("Switching to FCLK failed\n"); ++ ++ val = I915_READ(LCPLL_CTL); ++ val &= ~LCPLL_CLK_FREQ_MASK; ++ ++ switch (cdclk) { ++ default: ++ MISSING_CASE(cdclk); ++ /* fall through */ ++ case 337500: ++ val |= LCPLL_CLK_FREQ_337_5_BDW; ++ break; ++ case 450000: ++ val |= LCPLL_CLK_FREQ_450; ++ break; ++ case 540000: ++ val |= LCPLL_CLK_FREQ_54O_BDW; ++ break; ++ case 675000: ++ val |= LCPLL_CLK_FREQ_675_BDW; ++ break; ++ } ++ ++ I915_WRITE(LCPLL_CTL, val); ++ ++ val = I915_READ(LCPLL_CTL); ++ val &= ~LCPLL_CD_SOURCE_FCLK; ++ I915_WRITE(LCPLL_CTL, val); ++ ++ if (wait_for_us((I915_READ(LCPLL_CTL) & ++ LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1)) ++ DRM_ERROR("Switching back to LCPLL failed\n"); ++ ++ mutex_lock(&dev_priv->pcu_lock); ++ sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ, ++ cdclk_state->voltage_level); ++ mutex_unlock(&dev_priv->pcu_lock); ++ ++ I915_WRITE(CDCLK_FREQ, DIV_ROUND_CLOSEST(cdclk, 1000) - 1); ++ ++ intel_update_cdclk(dev_priv); ++} ++ ++static int skl_calc_cdclk(int min_cdclk, int vco) ++{ ++ if (vco == 8640000) { ++ if (min_cdclk > 540000) ++ return 617143; ++ else if (min_cdclk > 432000) ++ return 540000; ++ else if (min_cdclk > 308571) ++ return 432000; ++ else ++ return 308571; ++ } else { ++ if (min_cdclk > 540000) ++ return 675000; ++ else if (min_cdclk > 450000) ++ return 540000; ++ else if (min_cdclk > 337500) ++ return 450000; ++ else ++ return 337500; ++ } ++} ++ ++static u8 skl_calc_voltage_level(int cdclk) ++{ ++ switch (cdclk) { ++ default: ++ case 308571: ++ case 337500: ++ return 0; ++ case 450000: ++ case 432000: ++ return 1; ++ case 540000: ++ return 2; ++ case 617143: ++ case 675000: ++ return 3; ++ } ++} ++ ++static void skl_dpll0_update(struct drm_i915_private *dev_priv, ++ struct intel_cdclk_state *cdclk_state) ++{ ++ u32 val; ++ ++ cdclk_state->ref = 24000; ++ cdclk_state->vco = 0; ++ ++ val = I915_READ(LCPLL1_CTL); ++ if ((val & LCPLL_PLL_ENABLE) == 0) ++ return; ++ ++ if (WARN_ON((val & LCPLL_PLL_LOCK) == 0)) ++ return; ++ ++ val = I915_READ(DPLL_CTRL1); ++ ++ if (WARN_ON((val & (DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) | ++ DPLL_CTRL1_SSC(SKL_DPLL0) | ++ DPLL_CTRL1_OVERRIDE(SKL_DPLL0))) != ++ DPLL_CTRL1_OVERRIDE(SKL_DPLL0))) ++ return; ++ ++ switch (val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0)) { ++ case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0): ++ case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1350, SKL_DPLL0): ++ case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1620, SKL_DPLL0): ++ case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2700, SKL_DPLL0): ++ cdclk_state->vco = 8100000; ++ break; ++ case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0): ++ case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2160, SKL_DPLL0): ++ cdclk_state->vco = 8640000; ++ break; ++ default: ++ MISSING_CASE(val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0)); ++ break; ++ } ++} ++ ++static void skl_get_cdclk(struct drm_i915_private *dev_priv, ++ struct intel_cdclk_state *cdclk_state) ++{ ++ u32 cdctl; ++ ++ skl_dpll0_update(dev_priv, cdclk_state); ++ ++ cdclk_state->cdclk = cdclk_state->bypass = cdclk_state->ref; ++ ++ if (cdclk_state->vco == 0) ++ goto out; ++ ++ cdctl = I915_READ(CDCLK_CTL); ++ ++ if (cdclk_state->vco == 8640000) { ++ switch (cdctl & CDCLK_FREQ_SEL_MASK) { ++ case CDCLK_FREQ_450_432: ++ cdclk_state->cdclk = 432000; ++ break; ++ case CDCLK_FREQ_337_308: ++ cdclk_state->cdclk = 308571; ++ break; ++ case CDCLK_FREQ_540: ++ cdclk_state->cdclk = 540000; ++ break; ++ case CDCLK_FREQ_675_617: ++ cdclk_state->cdclk = 617143; ++ break; ++ default: ++ MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK); ++ break; ++ } ++ } else { ++ switch (cdctl & CDCLK_FREQ_SEL_MASK) { ++ case CDCLK_FREQ_450_432: ++ cdclk_state->cdclk = 450000; ++ break; ++ case CDCLK_FREQ_337_308: ++ cdclk_state->cdclk = 337500; ++ break; ++ case CDCLK_FREQ_540: ++ cdclk_state->cdclk = 540000; ++ break; ++ case CDCLK_FREQ_675_617: ++ cdclk_state->cdclk = 675000; ++ break; ++ default: ++ MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK); ++ break; ++ } ++ } ++ ++ out: ++ /* ++ * Can't read this out :( Let's assume it's ++ * at least what the CDCLK frequency requires. ++ */ ++ cdclk_state->voltage_level = ++ skl_calc_voltage_level(cdclk_state->cdclk); ++} ++ ++/* convert from kHz to .1 fixpoint MHz with -1MHz offset */ ++static int skl_cdclk_decimal(int cdclk) ++{ ++ return DIV_ROUND_CLOSEST(cdclk - 1000, 500); ++} ++ ++static void skl_set_preferred_cdclk_vco(struct drm_i915_private *dev_priv, ++ int vco) ++{ ++ bool changed = dev_priv->skl_preferred_vco_freq != vco; ++ ++ dev_priv->skl_preferred_vco_freq = vco; ++ ++ if (changed) ++ intel_update_max_cdclk(dev_priv); ++} ++ ++static void skl_dpll0_enable(struct drm_i915_private *dev_priv, int vco) ++{ ++ u32 val; ++ ++ WARN_ON(vco != 8100000 && vco != 8640000); ++ ++ /* ++ * We always enable DPLL0 with the lowest link rate possible, but still ++ * taking into account the VCO required to operate the eDP panel at the ++ * desired frequency. The usual DP link rates operate with a VCO of ++ * 8100 while the eDP 1.4 alternate link rates need a VCO of 8640. ++ * The modeset code is responsible for the selection of the exact link ++ * rate later on, with the constraint of choosing a frequency that ++ * works with vco. ++ */ ++ val = I915_READ(DPLL_CTRL1); ++ ++ val &= ~(DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) | DPLL_CTRL1_SSC(SKL_DPLL0) | ++ DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0)); ++ val |= DPLL_CTRL1_OVERRIDE(SKL_DPLL0); ++ if (vco == 8640000) ++ val |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, ++ SKL_DPLL0); ++ else ++ val |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, ++ SKL_DPLL0); ++ ++ I915_WRITE(DPLL_CTRL1, val); ++ POSTING_READ(DPLL_CTRL1); ++ ++ I915_WRITE(LCPLL1_CTL, I915_READ(LCPLL1_CTL) | LCPLL_PLL_ENABLE); ++ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ LCPLL1_CTL, LCPLL_PLL_LOCK, LCPLL_PLL_LOCK, ++ 5)) ++ DRM_ERROR("DPLL0 not locked\n"); ++ ++ dev_priv->cdclk.hw.vco = vco; ++ ++ /* We'll want to keep using the current vco from now on. */ ++ skl_set_preferred_cdclk_vco(dev_priv, vco); ++} ++ ++static void skl_dpll0_disable(struct drm_i915_private *dev_priv) ++{ ++ I915_WRITE(LCPLL1_CTL, I915_READ(LCPLL1_CTL) & ~LCPLL_PLL_ENABLE); ++ if (intel_wait_for_register(&dev_priv->uncore, ++ LCPLL1_CTL, LCPLL_PLL_LOCK, 0, ++ 1)) ++ DRM_ERROR("Couldn't disable DPLL0\n"); ++ ++ dev_priv->cdclk.hw.vco = 0; ++} ++ ++static void skl_set_cdclk(struct drm_i915_private *dev_priv, ++ const struct intel_cdclk_state *cdclk_state, ++ enum pipe pipe) ++{ ++ int cdclk = cdclk_state->cdclk; ++ int vco = cdclk_state->vco; ++ u32 freq_select, cdclk_ctl; ++ int ret; ++ ++ /* ++ * Based on WA#1183 CDCLK rates 308 and 617MHz CDCLK rates are ++ * unsupported on SKL. In theory this should never happen since only ++ * the eDP1.4 2.16 and 4.32Gbps rates require it, but eDP1.4 is not ++ * supported on SKL either, see the above WA. WARN whenever trying to ++ * use the corresponding VCO freq as that always leads to using the ++ * minimum 308MHz CDCLK. ++ */ ++ WARN_ON_ONCE(IS_SKYLAKE(dev_priv) && vco == 8640000); ++ ++ mutex_lock(&dev_priv->pcu_lock); ++ ret = skl_pcode_request(dev_priv, SKL_PCODE_CDCLK_CONTROL, ++ SKL_CDCLK_PREPARE_FOR_CHANGE, ++ SKL_CDCLK_READY_FOR_CHANGE, ++ SKL_CDCLK_READY_FOR_CHANGE, 3); ++ mutex_unlock(&dev_priv->pcu_lock); ++ if (ret) { ++ DRM_ERROR("Failed to inform PCU about cdclk change (%d)\n", ++ ret); ++ return; ++ } ++ ++ /* Choose frequency for this cdclk */ ++ switch (cdclk) { ++ default: ++ WARN_ON(cdclk != dev_priv->cdclk.hw.bypass); ++ WARN_ON(vco != 0); ++ /* fall through */ ++ case 308571: ++ case 337500: ++ freq_select = CDCLK_FREQ_337_308; ++ break; ++ case 450000: ++ case 432000: ++ freq_select = CDCLK_FREQ_450_432; ++ break; ++ case 540000: ++ freq_select = CDCLK_FREQ_540; ++ break; ++ case 617143: ++ case 675000: ++ freq_select = CDCLK_FREQ_675_617; ++ break; ++ } ++ ++ if (dev_priv->cdclk.hw.vco != 0 && ++ dev_priv->cdclk.hw.vco != vco) ++ skl_dpll0_disable(dev_priv); ++ ++ cdclk_ctl = I915_READ(CDCLK_CTL); ++ ++ if (dev_priv->cdclk.hw.vco != vco) { ++ /* Wa Display #1183: skl,kbl,cfl */ ++ cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK); ++ cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk); ++ I915_WRITE(CDCLK_CTL, cdclk_ctl); ++ } ++ ++ /* Wa Display #1183: skl,kbl,cfl */ ++ cdclk_ctl |= CDCLK_DIVMUX_CD_OVERRIDE; ++ I915_WRITE(CDCLK_CTL, cdclk_ctl); ++ POSTING_READ(CDCLK_CTL); ++ ++ if (dev_priv->cdclk.hw.vco != vco) ++ skl_dpll0_enable(dev_priv, vco); ++ ++ /* Wa Display #1183: skl,kbl,cfl */ ++ cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK); ++ I915_WRITE(CDCLK_CTL, cdclk_ctl); ++ ++ cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk); ++ I915_WRITE(CDCLK_CTL, cdclk_ctl); ++ ++ /* Wa Display #1183: skl,kbl,cfl */ ++ cdclk_ctl &= ~CDCLK_DIVMUX_CD_OVERRIDE; ++ I915_WRITE(CDCLK_CTL, cdclk_ctl); ++ POSTING_READ(CDCLK_CTL); ++ ++ /* inform PCU of the change */ ++ mutex_lock(&dev_priv->pcu_lock); ++ sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL, ++ cdclk_state->voltage_level); ++ mutex_unlock(&dev_priv->pcu_lock); ++ ++ intel_update_cdclk(dev_priv); ++} ++ ++static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv) ++{ ++ u32 cdctl, expected; ++ ++ /* ++ * check if the pre-os initialized the display ++ * There is SWF18 scratchpad register defined which is set by the ++ * pre-os which can be used by the OS drivers to check the status ++ */ ++ if ((I915_READ(SWF_ILK(0x18)) & 0x00FFFFFF) == 0) ++ goto sanitize; ++ ++ intel_update_cdclk(dev_priv); ++ intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK"); ++ ++ /* Is PLL enabled and locked ? */ ++ if (dev_priv->cdclk.hw.vco == 0 || ++ dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.bypass) ++ goto sanitize; ++ ++ /* DPLL okay; verify the cdclock ++ * ++ * Noticed in some instances that the freq selection is correct but ++ * decimal part is programmed wrong from BIOS where pre-os does not ++ * enable display. Verify the same as well. ++ */ ++ cdctl = I915_READ(CDCLK_CTL); ++ expected = (cdctl & CDCLK_FREQ_SEL_MASK) | ++ skl_cdclk_decimal(dev_priv->cdclk.hw.cdclk); ++ if (cdctl == expected) ++ /* All well; nothing to sanitize */ ++ return; ++ ++sanitize: ++ DRM_DEBUG_KMS("Sanitizing cdclk programmed by pre-os\n"); ++ ++ /* force cdclk programming */ ++ dev_priv->cdclk.hw.cdclk = 0; ++ /* force full PLL disable + enable */ ++ dev_priv->cdclk.hw.vco = -1; ++} ++ ++static void skl_init_cdclk(struct drm_i915_private *dev_priv) ++{ ++ struct intel_cdclk_state cdclk_state; ++ ++ skl_sanitize_cdclk(dev_priv); ++ ++ if (dev_priv->cdclk.hw.cdclk != 0 && ++ dev_priv->cdclk.hw.vco != 0) { ++ /* ++ * Use the current vco as our initial ++ * guess as to what the preferred vco is. ++ */ ++ if (dev_priv->skl_preferred_vco_freq == 0) ++ skl_set_preferred_cdclk_vco(dev_priv, ++ dev_priv->cdclk.hw.vco); ++ return; ++ } ++ ++ cdclk_state = dev_priv->cdclk.hw; ++ ++ cdclk_state.vco = dev_priv->skl_preferred_vco_freq; ++ if (cdclk_state.vco == 0) ++ cdclk_state.vco = 8100000; ++ cdclk_state.cdclk = skl_calc_cdclk(0, cdclk_state.vco); ++ cdclk_state.voltage_level = skl_calc_voltage_level(cdclk_state.cdclk); ++ ++ skl_set_cdclk(dev_priv, &cdclk_state, INVALID_PIPE); ++} ++ ++static void skl_uninit_cdclk(struct drm_i915_private *dev_priv) ++{ ++ struct intel_cdclk_state cdclk_state = dev_priv->cdclk.hw; ++ ++ cdclk_state.cdclk = cdclk_state.bypass; ++ cdclk_state.vco = 0; ++ cdclk_state.voltage_level = skl_calc_voltage_level(cdclk_state.cdclk); ++ ++ skl_set_cdclk(dev_priv, &cdclk_state, INVALID_PIPE); ++} ++ ++static int bxt_calc_cdclk(int min_cdclk) ++{ ++ if (min_cdclk > 576000) ++ return 624000; ++ else if (min_cdclk > 384000) ++ return 576000; ++ else if (min_cdclk > 288000) ++ return 384000; ++ else if (min_cdclk > 144000) ++ return 288000; ++ else ++ return 144000; ++} ++ ++static int glk_calc_cdclk(int min_cdclk) ++{ ++ if (min_cdclk > 158400) ++ return 316800; ++ else if (min_cdclk > 79200) ++ return 158400; ++ else ++ return 79200; ++} ++ ++static u8 bxt_calc_voltage_level(int cdclk) ++{ ++ return DIV_ROUND_UP(cdclk, 25000); ++} ++ ++static int bxt_de_pll_vco(struct drm_i915_private *dev_priv, int cdclk) ++{ ++ int ratio; ++ ++ if (cdclk == dev_priv->cdclk.hw.bypass) ++ return 0; ++ ++ switch (cdclk) { ++ default: ++ MISSING_CASE(cdclk); ++ /* fall through */ ++ case 144000: ++ case 288000: ++ case 384000: ++ case 576000: ++ ratio = 60; ++ break; ++ case 624000: ++ ratio = 65; ++ break; ++ } ++ ++ return dev_priv->cdclk.hw.ref * ratio; ++} ++ ++static int glk_de_pll_vco(struct drm_i915_private *dev_priv, int cdclk) ++{ ++ int ratio; ++ ++ if (cdclk == dev_priv->cdclk.hw.bypass) ++ return 0; ++ ++ switch (cdclk) { ++ default: ++ MISSING_CASE(cdclk); ++ /* fall through */ ++ case 79200: ++ case 158400: ++ case 316800: ++ ratio = 33; ++ break; ++ } ++ ++ return dev_priv->cdclk.hw.ref * ratio; ++} ++ ++static void bxt_de_pll_update(struct drm_i915_private *dev_priv, ++ struct intel_cdclk_state *cdclk_state) ++{ ++ u32 val; ++ ++ cdclk_state->ref = 19200; ++ cdclk_state->vco = 0; ++ ++ val = I915_READ(BXT_DE_PLL_ENABLE); ++ if ((val & BXT_DE_PLL_PLL_ENABLE) == 0) ++ return; ++ ++ if (WARN_ON((val & BXT_DE_PLL_LOCK) == 0)) ++ return; ++ ++ val = I915_READ(BXT_DE_PLL_CTL); ++ cdclk_state->vco = (val & BXT_DE_PLL_RATIO_MASK) * cdclk_state->ref; ++} ++ ++static void bxt_get_cdclk(struct drm_i915_private *dev_priv, ++ struct intel_cdclk_state *cdclk_state) ++{ ++ u32 divider; ++ int div; ++ ++ bxt_de_pll_update(dev_priv, cdclk_state); ++ ++ cdclk_state->cdclk = cdclk_state->bypass = cdclk_state->ref; ++ ++ if (cdclk_state->vco == 0) ++ goto out; ++ ++ divider = I915_READ(CDCLK_CTL) & BXT_CDCLK_CD2X_DIV_SEL_MASK; ++ ++ switch (divider) { ++ case BXT_CDCLK_CD2X_DIV_SEL_1: ++ div = 2; ++ break; ++ case BXT_CDCLK_CD2X_DIV_SEL_1_5: ++ WARN(IS_GEMINILAKE(dev_priv), "Unsupported divider\n"); ++ div = 3; ++ break; ++ case BXT_CDCLK_CD2X_DIV_SEL_2: ++ div = 4; ++ break; ++ case BXT_CDCLK_CD2X_DIV_SEL_4: ++ div = 8; ++ break; ++ default: ++ MISSING_CASE(divider); ++ return; ++ } ++ ++ cdclk_state->cdclk = DIV_ROUND_CLOSEST(cdclk_state->vco, div); ++ ++ out: ++ /* ++ * Can't read this out :( Let's assume it's ++ * at least what the CDCLK frequency requires. ++ */ ++ cdclk_state->voltage_level = ++ bxt_calc_voltage_level(cdclk_state->cdclk); ++} ++ ++static void bxt_de_pll_disable(struct drm_i915_private *dev_priv) ++{ ++ I915_WRITE(BXT_DE_PLL_ENABLE, 0); ++ ++ /* Timeout 200us */ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 0, ++ 1)) ++ DRM_ERROR("timeout waiting for DE PLL unlock\n"); ++ ++ dev_priv->cdclk.hw.vco = 0; ++} ++ ++static void bxt_de_pll_enable(struct drm_i915_private *dev_priv, int vco) ++{ ++ int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk.hw.ref); ++ u32 val; ++ ++ val = I915_READ(BXT_DE_PLL_CTL); ++ val &= ~BXT_DE_PLL_RATIO_MASK; ++ val |= BXT_DE_PLL_RATIO(ratio); ++ I915_WRITE(BXT_DE_PLL_CTL, val); ++ ++ I915_WRITE(BXT_DE_PLL_ENABLE, BXT_DE_PLL_PLL_ENABLE); ++ ++ /* Timeout 200us */ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ BXT_DE_PLL_ENABLE, ++ BXT_DE_PLL_LOCK, ++ BXT_DE_PLL_LOCK, ++ 1)) ++ DRM_ERROR("timeout waiting for DE PLL lock\n"); ++ ++ dev_priv->cdclk.hw.vco = vco; ++} ++ ++static void bxt_set_cdclk(struct drm_i915_private *dev_priv, ++ const struct intel_cdclk_state *cdclk_state, ++ enum pipe pipe) ++{ ++ int cdclk = cdclk_state->cdclk; ++ int vco = cdclk_state->vco; ++ u32 val, divider; ++ int ret; ++ ++ /* cdclk = vco / 2 / div{1,1.5,2,4} */ ++ switch (DIV_ROUND_CLOSEST(vco, cdclk)) { ++ default: ++ WARN_ON(cdclk != dev_priv->cdclk.hw.bypass); ++ WARN_ON(vco != 0); ++ /* fall through */ ++ case 2: ++ divider = BXT_CDCLK_CD2X_DIV_SEL_1; ++ break; ++ case 3: ++ WARN(IS_GEMINILAKE(dev_priv), "Unsupported divider\n"); ++ divider = BXT_CDCLK_CD2X_DIV_SEL_1_5; ++ break; ++ case 4: ++ divider = BXT_CDCLK_CD2X_DIV_SEL_2; ++ break; ++ case 8: ++ divider = BXT_CDCLK_CD2X_DIV_SEL_4; ++ break; ++ } ++ ++ /* ++ * Inform power controller of upcoming frequency change. BSpec ++ * requires us to wait up to 150usec, but that leads to timeouts; ++ * the 2ms used here is based on experiment. ++ */ ++ mutex_lock(&dev_priv->pcu_lock); ++ ret = sandybridge_pcode_write_timeout(dev_priv, ++ HSW_PCODE_DE_WRITE_FREQ_REQ, ++ 0x80000000, 150, 2); ++ mutex_unlock(&dev_priv->pcu_lock); ++ ++ if (ret) { ++ DRM_ERROR("PCode CDCLK freq change notify failed (err %d, freq %d)\n", ++ ret, cdclk); ++ return; ++ } ++ ++ if (dev_priv->cdclk.hw.vco != 0 && ++ dev_priv->cdclk.hw.vco != vco) ++ bxt_de_pll_disable(dev_priv); ++ ++ if (dev_priv->cdclk.hw.vco != vco) ++ bxt_de_pll_enable(dev_priv, vco); ++ ++ val = divider | skl_cdclk_decimal(cdclk); ++ if (pipe == INVALID_PIPE) ++ val |= BXT_CDCLK_CD2X_PIPE_NONE; ++ else ++ val |= BXT_CDCLK_CD2X_PIPE(pipe); ++ /* ++ * Disable SSA Precharge when CD clock frequency < 500 MHz, ++ * enable otherwise. ++ */ ++ if (cdclk >= 500000) ++ val |= BXT_CDCLK_SSA_PRECHARGE_ENABLE; ++ I915_WRITE(CDCLK_CTL, val); ++ ++ if (pipe != INVALID_PIPE) ++ intel_wait_for_vblank(dev_priv, pipe); ++ ++ mutex_lock(&dev_priv->pcu_lock); ++ /* ++ * The timeout isn't specified, the 2ms used here is based on ++ * experiment. ++ * FIXME: Waiting for the request completion could be delayed until ++ * the next PCODE request based on BSpec. ++ */ ++ ret = sandybridge_pcode_write_timeout(dev_priv, ++ HSW_PCODE_DE_WRITE_FREQ_REQ, ++ cdclk_state->voltage_level, 150, 2); ++ mutex_unlock(&dev_priv->pcu_lock); ++ ++ if (ret) { ++ DRM_ERROR("PCode CDCLK freq set failed, (err %d, freq %d)\n", ++ ret, cdclk); ++ return; ++ } ++ ++ intel_update_cdclk(dev_priv); ++} ++ ++static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv) ++{ ++ u32 cdctl, expected; ++ ++ intel_update_cdclk(dev_priv); ++ intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK"); ++ ++ if (dev_priv->cdclk.hw.vco == 0 || ++ dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.bypass) ++ goto sanitize; ++ ++ /* DPLL okay; verify the cdclock ++ * ++ * Some BIOS versions leave an incorrect decimal frequency value and ++ * set reserved MBZ bits in CDCLK_CTL at least during exiting from S4, ++ * so sanitize this register. ++ */ ++ cdctl = I915_READ(CDCLK_CTL); ++ /* ++ * Let's ignore the pipe field, since BIOS could have configured the ++ * dividers both synching to an active pipe, or asynchronously ++ * (PIPE_NONE). ++ */ ++ cdctl &= ~BXT_CDCLK_CD2X_PIPE_NONE; ++ ++ expected = (cdctl & BXT_CDCLK_CD2X_DIV_SEL_MASK) | ++ skl_cdclk_decimal(dev_priv->cdclk.hw.cdclk); ++ /* ++ * Disable SSA Precharge when CD clock frequency < 500 MHz, ++ * enable otherwise. ++ */ ++ if (dev_priv->cdclk.hw.cdclk >= 500000) ++ expected |= BXT_CDCLK_SSA_PRECHARGE_ENABLE; ++ ++ if (cdctl == expected) ++ /* All well; nothing to sanitize */ ++ return; ++ ++sanitize: ++ DRM_DEBUG_KMS("Sanitizing cdclk programmed by pre-os\n"); ++ ++ /* force cdclk programming */ ++ dev_priv->cdclk.hw.cdclk = 0; ++ ++ /* force full PLL disable + enable */ ++ dev_priv->cdclk.hw.vco = -1; ++} ++ ++static void bxt_init_cdclk(struct drm_i915_private *dev_priv) ++{ ++ struct intel_cdclk_state cdclk_state; ++ ++ bxt_sanitize_cdclk(dev_priv); ++ ++ if (dev_priv->cdclk.hw.cdclk != 0 && ++ dev_priv->cdclk.hw.vco != 0) ++ return; ++ ++ cdclk_state = dev_priv->cdclk.hw; ++ ++ /* ++ * FIXME: ++ * - The initial CDCLK needs to be read from VBT. ++ * Need to make this change after VBT has changes for BXT. ++ */ ++ if (IS_GEMINILAKE(dev_priv)) { ++ cdclk_state.cdclk = glk_calc_cdclk(0); ++ cdclk_state.vco = glk_de_pll_vco(dev_priv, cdclk_state.cdclk); ++ } else { ++ cdclk_state.cdclk = bxt_calc_cdclk(0); ++ cdclk_state.vco = bxt_de_pll_vco(dev_priv, cdclk_state.cdclk); ++ } ++ cdclk_state.voltage_level = bxt_calc_voltage_level(cdclk_state.cdclk); ++ ++ bxt_set_cdclk(dev_priv, &cdclk_state, INVALID_PIPE); ++} ++ ++static void bxt_uninit_cdclk(struct drm_i915_private *dev_priv) ++{ ++ struct intel_cdclk_state cdclk_state = dev_priv->cdclk.hw; ++ ++ cdclk_state.cdclk = cdclk_state.bypass; ++ cdclk_state.vco = 0; ++ cdclk_state.voltage_level = bxt_calc_voltage_level(cdclk_state.cdclk); ++ ++ bxt_set_cdclk(dev_priv, &cdclk_state, INVALID_PIPE); ++} ++ ++static int cnl_calc_cdclk(int min_cdclk) ++{ ++ if (min_cdclk > 336000) ++ return 528000; ++ else if (min_cdclk > 168000) ++ return 336000; ++ else ++ return 168000; ++} ++ ++static u8 cnl_calc_voltage_level(int cdclk) ++{ ++ switch (cdclk) { ++ default: ++ case 168000: ++ return 0; ++ case 336000: ++ return 1; ++ case 528000: ++ return 2; ++ } ++} ++ ++static void cnl_cdclk_pll_update(struct drm_i915_private *dev_priv, ++ struct intel_cdclk_state *cdclk_state) ++{ ++ u32 val; ++ ++ if (I915_READ(SKL_DSSM) & CNL_DSSM_CDCLK_PLL_REFCLK_24MHz) ++ cdclk_state->ref = 24000; ++ else ++ cdclk_state->ref = 19200; ++ ++ cdclk_state->vco = 0; ++ ++ val = I915_READ(BXT_DE_PLL_ENABLE); ++ if ((val & BXT_DE_PLL_PLL_ENABLE) == 0) ++ return; ++ ++ if (WARN_ON((val & BXT_DE_PLL_LOCK) == 0)) ++ return; ++ ++ cdclk_state->vco = (val & CNL_CDCLK_PLL_RATIO_MASK) * cdclk_state->ref; ++} ++ ++static void cnl_get_cdclk(struct drm_i915_private *dev_priv, ++ struct intel_cdclk_state *cdclk_state) ++{ ++ u32 divider; ++ int div; ++ ++ cnl_cdclk_pll_update(dev_priv, cdclk_state); ++ ++ cdclk_state->cdclk = cdclk_state->bypass = cdclk_state->ref; ++ ++ if (cdclk_state->vco == 0) ++ goto out; ++ ++ divider = I915_READ(CDCLK_CTL) & BXT_CDCLK_CD2X_DIV_SEL_MASK; ++ ++ switch (divider) { ++ case BXT_CDCLK_CD2X_DIV_SEL_1: ++ div = 2; ++ break; ++ case BXT_CDCLK_CD2X_DIV_SEL_2: ++ div = 4; ++ break; ++ default: ++ MISSING_CASE(divider); ++ return; ++ } ++ ++ cdclk_state->cdclk = DIV_ROUND_CLOSEST(cdclk_state->vco, div); ++ ++ out: ++ /* ++ * Can't read this out :( Let's assume it's ++ * at least what the CDCLK frequency requires. ++ */ ++ cdclk_state->voltage_level = ++ cnl_calc_voltage_level(cdclk_state->cdclk); ++} ++ ++static void cnl_cdclk_pll_disable(struct drm_i915_private *dev_priv) ++{ ++ u32 val; ++ ++ val = I915_READ(BXT_DE_PLL_ENABLE); ++ val &= ~BXT_DE_PLL_PLL_ENABLE; ++ I915_WRITE(BXT_DE_PLL_ENABLE, val); ++ ++ /* Timeout 200us */ ++ if (wait_for((I915_READ(BXT_DE_PLL_ENABLE) & BXT_DE_PLL_LOCK) == 0, 1)) ++ DRM_ERROR("timeout waiting for CDCLK PLL unlock\n"); ++ ++ dev_priv->cdclk.hw.vco = 0; ++} ++ ++static void cnl_cdclk_pll_enable(struct drm_i915_private *dev_priv, int vco) ++{ ++ int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk.hw.ref); ++ u32 val; ++ ++ val = CNL_CDCLK_PLL_RATIO(ratio); ++ I915_WRITE(BXT_DE_PLL_ENABLE, val); ++ ++ val |= BXT_DE_PLL_PLL_ENABLE; ++ I915_WRITE(BXT_DE_PLL_ENABLE, val); ++ ++ /* Timeout 200us */ ++ if (wait_for((I915_READ(BXT_DE_PLL_ENABLE) & BXT_DE_PLL_LOCK) != 0, 1)) ++ DRM_ERROR("timeout waiting for CDCLK PLL lock\n"); ++ ++ dev_priv->cdclk.hw.vco = vco; ++} ++ ++static void cnl_set_cdclk(struct drm_i915_private *dev_priv, ++ const struct intel_cdclk_state *cdclk_state, ++ enum pipe pipe) ++{ ++ int cdclk = cdclk_state->cdclk; ++ int vco = cdclk_state->vco; ++ u32 val, divider; ++ int ret; ++ ++ mutex_lock(&dev_priv->pcu_lock); ++ ret = skl_pcode_request(dev_priv, SKL_PCODE_CDCLK_CONTROL, ++ SKL_CDCLK_PREPARE_FOR_CHANGE, ++ SKL_CDCLK_READY_FOR_CHANGE, ++ SKL_CDCLK_READY_FOR_CHANGE, 3); ++ mutex_unlock(&dev_priv->pcu_lock); ++ if (ret) { ++ DRM_ERROR("Failed to inform PCU about cdclk change (%d)\n", ++ ret); ++ return; ++ } ++ ++ /* cdclk = vco / 2 / div{1,2} */ ++ switch (DIV_ROUND_CLOSEST(vco, cdclk)) { ++ default: ++ WARN_ON(cdclk != dev_priv->cdclk.hw.bypass); ++ WARN_ON(vco != 0); ++ /* fall through */ ++ case 2: ++ divider = BXT_CDCLK_CD2X_DIV_SEL_1; ++ break; ++ case 4: ++ divider = BXT_CDCLK_CD2X_DIV_SEL_2; ++ break; ++ } ++ ++ if (dev_priv->cdclk.hw.vco != 0 && ++ dev_priv->cdclk.hw.vco != vco) ++ cnl_cdclk_pll_disable(dev_priv); ++ ++ if (dev_priv->cdclk.hw.vco != vco) ++ cnl_cdclk_pll_enable(dev_priv, vco); ++ ++ val = divider | skl_cdclk_decimal(cdclk); ++ if (pipe == INVALID_PIPE) ++ val |= BXT_CDCLK_CD2X_PIPE_NONE; ++ else ++ val |= BXT_CDCLK_CD2X_PIPE(pipe); ++ I915_WRITE(CDCLK_CTL, val); ++ ++ if (pipe != INVALID_PIPE) ++ intel_wait_for_vblank(dev_priv, pipe); ++ ++ /* inform PCU of the change */ ++ mutex_lock(&dev_priv->pcu_lock); ++ sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL, ++ cdclk_state->voltage_level); ++ mutex_unlock(&dev_priv->pcu_lock); ++ ++ intel_update_cdclk(dev_priv); ++ ++ /* ++ * Can't read out the voltage level :( ++ * Let's just assume everything is as expected. ++ */ ++ dev_priv->cdclk.hw.voltage_level = cdclk_state->voltage_level; ++} ++ ++static int cnl_cdclk_pll_vco(struct drm_i915_private *dev_priv, int cdclk) ++{ ++ int ratio; ++ ++ if (cdclk == dev_priv->cdclk.hw.bypass) ++ return 0; ++ ++ switch (cdclk) { ++ default: ++ MISSING_CASE(cdclk); ++ /* fall through */ ++ case 168000: ++ case 336000: ++ ratio = dev_priv->cdclk.hw.ref == 19200 ? 35 : 28; ++ break; ++ case 528000: ++ ratio = dev_priv->cdclk.hw.ref == 19200 ? 55 : 44; ++ break; ++ } ++ ++ return dev_priv->cdclk.hw.ref * ratio; ++} ++ ++static void cnl_sanitize_cdclk(struct drm_i915_private *dev_priv) ++{ ++ u32 cdctl, expected; ++ ++ intel_update_cdclk(dev_priv); ++ intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK"); ++ ++ if (dev_priv->cdclk.hw.vco == 0 || ++ dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.bypass) ++ goto sanitize; ++ ++ /* DPLL okay; verify the cdclock ++ * ++ * Some BIOS versions leave an incorrect decimal frequency value and ++ * set reserved MBZ bits in CDCLK_CTL at least during exiting from S4, ++ * so sanitize this register. ++ */ ++ cdctl = I915_READ(CDCLK_CTL); ++ /* ++ * Let's ignore the pipe field, since BIOS could have configured the ++ * dividers both synching to an active pipe, or asynchronously ++ * (PIPE_NONE). ++ */ ++ cdctl &= ~BXT_CDCLK_CD2X_PIPE_NONE; ++ ++ expected = (cdctl & BXT_CDCLK_CD2X_DIV_SEL_MASK) | ++ skl_cdclk_decimal(dev_priv->cdclk.hw.cdclk); ++ ++ if (cdctl == expected) ++ /* All well; nothing to sanitize */ ++ return; ++ ++sanitize: ++ DRM_DEBUG_KMS("Sanitizing cdclk programmed by pre-os\n"); ++ ++ /* force cdclk programming */ ++ dev_priv->cdclk.hw.cdclk = 0; ++ ++ /* force full PLL disable + enable */ ++ dev_priv->cdclk.hw.vco = -1; ++} ++ ++static int icl_calc_cdclk(int min_cdclk, unsigned int ref) ++{ ++ int ranges_24[] = { 312000, 552000, 648000 }; ++ int ranges_19_38[] = { 307200, 556800, 652800 }; ++ int *ranges; ++ ++ switch (ref) { ++ default: ++ MISSING_CASE(ref); ++ /* fall through */ ++ case 24000: ++ ranges = ranges_24; ++ break; ++ case 19200: ++ case 38400: ++ ranges = ranges_19_38; ++ break; ++ } ++ ++ if (min_cdclk > ranges[1]) ++ return ranges[2]; ++ else if (min_cdclk > ranges[0]) ++ return ranges[1]; ++ else ++ return ranges[0]; ++} ++ ++static int icl_calc_cdclk_pll_vco(struct drm_i915_private *dev_priv, int cdclk) ++{ ++ int ratio; ++ ++ if (cdclk == dev_priv->cdclk.hw.bypass) ++ return 0; ++ ++ switch (cdclk) { ++ default: ++ MISSING_CASE(cdclk); ++ /* fall through */ ++ case 307200: ++ case 556800: ++ case 652800: ++ WARN_ON(dev_priv->cdclk.hw.ref != 19200 && ++ dev_priv->cdclk.hw.ref != 38400); ++ break; ++ case 312000: ++ case 552000: ++ case 648000: ++ WARN_ON(dev_priv->cdclk.hw.ref != 24000); ++ } ++ ++ ratio = cdclk / (dev_priv->cdclk.hw.ref / 2); ++ ++ return dev_priv->cdclk.hw.ref * ratio; ++} ++ ++static void icl_set_cdclk(struct drm_i915_private *dev_priv, ++ const struct intel_cdclk_state *cdclk_state, ++ enum pipe pipe) ++{ ++ unsigned int cdclk = cdclk_state->cdclk; ++ unsigned int vco = cdclk_state->vco; ++ int ret; ++ ++ mutex_lock(&dev_priv->pcu_lock); ++ ret = skl_pcode_request(dev_priv, SKL_PCODE_CDCLK_CONTROL, ++ SKL_CDCLK_PREPARE_FOR_CHANGE, ++ SKL_CDCLK_READY_FOR_CHANGE, ++ SKL_CDCLK_READY_FOR_CHANGE, 3); ++ mutex_unlock(&dev_priv->pcu_lock); ++ if (ret) { ++ DRM_ERROR("Failed to inform PCU about cdclk change (%d)\n", ++ ret); ++ return; ++ } ++ ++ if (dev_priv->cdclk.hw.vco != 0 && ++ dev_priv->cdclk.hw.vco != vco) ++ cnl_cdclk_pll_disable(dev_priv); ++ ++ if (dev_priv->cdclk.hw.vco != vco) ++ cnl_cdclk_pll_enable(dev_priv, vco); ++ ++ /* ++ * On ICL CD2X_DIV can only be 1, so we'll never end up changing the ++ * divider here synchronized to a pipe while CDCLK is on, nor will we ++ * need the corresponding vblank wait. ++ */ ++ I915_WRITE(CDCLK_CTL, ICL_CDCLK_CD2X_PIPE_NONE | ++ skl_cdclk_decimal(cdclk)); ++ ++ mutex_lock(&dev_priv->pcu_lock); ++ sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL, ++ cdclk_state->voltage_level); ++ mutex_unlock(&dev_priv->pcu_lock); ++ ++ intel_update_cdclk(dev_priv); ++ ++ /* ++ * Can't read out the voltage level :( ++ * Let's just assume everything is as expected. ++ */ ++ dev_priv->cdclk.hw.voltage_level = cdclk_state->voltage_level; ++} ++ ++static u8 icl_calc_voltage_level(int cdclk) ++{ ++ switch (cdclk) { ++ case 50000: ++ case 307200: ++ case 312000: ++ return 0; ++ case 556800: ++ case 552000: ++ return 1; ++ default: ++ MISSING_CASE(cdclk); ++ /* fall through */ ++ case 652800: ++ case 648000: ++ return 2; ++ } ++} ++ ++static void icl_get_cdclk(struct drm_i915_private *dev_priv, ++ struct intel_cdclk_state *cdclk_state) ++{ ++ u32 val; ++ ++ cdclk_state->bypass = 50000; ++ ++ val = I915_READ(SKL_DSSM); ++ switch (val & ICL_DSSM_CDCLK_PLL_REFCLK_MASK) { ++ default: ++ MISSING_CASE(val); ++ /* fall through */ ++ case ICL_DSSM_CDCLK_PLL_REFCLK_24MHz: ++ cdclk_state->ref = 24000; ++ break; ++ case ICL_DSSM_CDCLK_PLL_REFCLK_19_2MHz: ++ cdclk_state->ref = 19200; ++ break; ++ case ICL_DSSM_CDCLK_PLL_REFCLK_38_4MHz: ++ cdclk_state->ref = 38400; ++ break; ++ } ++ ++ val = I915_READ(BXT_DE_PLL_ENABLE); ++ if ((val & BXT_DE_PLL_PLL_ENABLE) == 0 || ++ (val & BXT_DE_PLL_LOCK) == 0) { ++ /* ++ * CDCLK PLL is disabled, the VCO/ratio doesn't matter, but ++ * setting it to zero is a way to signal that. ++ */ ++ cdclk_state->vco = 0; ++ cdclk_state->cdclk = cdclk_state->bypass; ++ goto out; ++ } ++ ++ cdclk_state->vco = (val & BXT_DE_PLL_RATIO_MASK) * cdclk_state->ref; ++ ++ val = I915_READ(CDCLK_CTL); ++ WARN_ON((val & BXT_CDCLK_CD2X_DIV_SEL_MASK) != 0); ++ ++ cdclk_state->cdclk = cdclk_state->vco / 2; ++ ++out: ++ /* ++ * Can't read this out :( Let's assume it's ++ * at least what the CDCLK frequency requires. ++ */ ++ cdclk_state->voltage_level = ++ icl_calc_voltage_level(cdclk_state->cdclk); ++} ++ ++static void icl_init_cdclk(struct drm_i915_private *dev_priv) ++{ ++ struct intel_cdclk_state sanitized_state; ++ u32 val; ++ ++ /* This sets dev_priv->cdclk.hw. */ ++ intel_update_cdclk(dev_priv); ++ intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK"); ++ ++ /* This means CDCLK disabled. */ ++ if (dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.bypass) ++ goto sanitize; ++ ++ val = I915_READ(CDCLK_CTL); ++ ++ if ((val & BXT_CDCLK_CD2X_DIV_SEL_MASK) != 0) ++ goto sanitize; ++ ++ if ((val & CDCLK_FREQ_DECIMAL_MASK) != ++ skl_cdclk_decimal(dev_priv->cdclk.hw.cdclk)) ++ goto sanitize; ++ ++ return; ++ ++sanitize: ++ DRM_DEBUG_KMS("Sanitizing cdclk programmed by pre-os\n"); ++ ++ sanitized_state.ref = dev_priv->cdclk.hw.ref; ++ sanitized_state.cdclk = icl_calc_cdclk(0, sanitized_state.ref); ++ sanitized_state.vco = icl_calc_cdclk_pll_vco(dev_priv, ++ sanitized_state.cdclk); ++ sanitized_state.voltage_level = ++ icl_calc_voltage_level(sanitized_state.cdclk); ++ ++ icl_set_cdclk(dev_priv, &sanitized_state, INVALID_PIPE); ++} ++ ++static void icl_uninit_cdclk(struct drm_i915_private *dev_priv) ++{ ++ struct intel_cdclk_state cdclk_state = dev_priv->cdclk.hw; ++ ++ cdclk_state.cdclk = cdclk_state.bypass; ++ cdclk_state.vco = 0; ++ cdclk_state.voltage_level = icl_calc_voltage_level(cdclk_state.cdclk); ++ ++ icl_set_cdclk(dev_priv, &cdclk_state, INVALID_PIPE); ++} ++ ++static void cnl_init_cdclk(struct drm_i915_private *dev_priv) ++{ ++ struct intel_cdclk_state cdclk_state; ++ ++ cnl_sanitize_cdclk(dev_priv); ++ ++ if (dev_priv->cdclk.hw.cdclk != 0 && ++ dev_priv->cdclk.hw.vco != 0) ++ return; ++ ++ cdclk_state = dev_priv->cdclk.hw; ++ ++ cdclk_state.cdclk = cnl_calc_cdclk(0); ++ cdclk_state.vco = cnl_cdclk_pll_vco(dev_priv, cdclk_state.cdclk); ++ cdclk_state.voltage_level = cnl_calc_voltage_level(cdclk_state.cdclk); ++ ++ cnl_set_cdclk(dev_priv, &cdclk_state, INVALID_PIPE); ++} ++ ++static void cnl_uninit_cdclk(struct drm_i915_private *dev_priv) ++{ ++ struct intel_cdclk_state cdclk_state = dev_priv->cdclk.hw; ++ ++ cdclk_state.cdclk = cdclk_state.bypass; ++ cdclk_state.vco = 0; ++ cdclk_state.voltage_level = cnl_calc_voltage_level(cdclk_state.cdclk); ++ ++ cnl_set_cdclk(dev_priv, &cdclk_state, INVALID_PIPE); ++} ++ ++/** ++ * intel_cdclk_init - Initialize CDCLK ++ * @i915: i915 device ++ * ++ * Initialize CDCLK. This consists mainly of initializing dev_priv->cdclk.hw and ++ * sanitizing the state of the hardware if needed. This is generally done only ++ * during the display core initialization sequence, after which the DMC will ++ * take care of turning CDCLK off/on as needed. ++ */ ++void intel_cdclk_init(struct drm_i915_private *i915) ++{ ++ if (INTEL_GEN(i915) >= 11) ++ icl_init_cdclk(i915); ++ else if (IS_CANNONLAKE(i915)) ++ cnl_init_cdclk(i915); ++ else if (IS_GEN9_BC(i915)) ++ skl_init_cdclk(i915); ++ else if (IS_GEN9_LP(i915)) ++ bxt_init_cdclk(i915); ++} ++ ++/** ++ * intel_cdclk_uninit - Uninitialize CDCLK ++ * @i915: i915 device ++ * ++ * Uninitialize CDCLK. This is done only during the display core ++ * uninitialization sequence. ++ */ ++void intel_cdclk_uninit(struct drm_i915_private *i915) ++{ ++ if (INTEL_GEN(i915) >= 11) ++ icl_uninit_cdclk(i915); ++ else if (IS_CANNONLAKE(i915)) ++ cnl_uninit_cdclk(i915); ++ else if (IS_GEN9_BC(i915)) ++ skl_uninit_cdclk(i915); ++ else if (IS_GEN9_LP(i915)) ++ bxt_uninit_cdclk(i915); ++} ++ ++/** ++ * intel_cdclk_needs_modeset - Determine if two CDCLK states require a modeset on all pipes ++ * @a: first CDCLK state ++ * @b: second CDCLK state ++ * ++ * Returns: ++ * True if the CDCLK states require pipes to be off during reprogramming, false if not. ++ */ ++bool intel_cdclk_needs_modeset(const struct intel_cdclk_state *a, ++ const struct intel_cdclk_state *b) ++{ ++ return a->cdclk != b->cdclk || ++ a->vco != b->vco || ++ a->ref != b->ref; ++} ++ ++/** ++ * intel_cdclk_needs_cd2x_update - Determine if two CDCLK states require a cd2x divider update ++ * @dev_priv: Not a CDCLK state, it's the drm_i915_private! ++ * @a: first CDCLK state ++ * @b: second CDCLK state ++ * ++ * Returns: ++ * True if the CDCLK states require just a cd2x divider update, false if not. ++ */ ++bool intel_cdclk_needs_cd2x_update(struct drm_i915_private *dev_priv, ++ const struct intel_cdclk_state *a, ++ const struct intel_cdclk_state *b) ++{ ++ /* Older hw doesn't have the capability */ ++ if (INTEL_GEN(dev_priv) < 10 && !IS_GEN9_LP(dev_priv)) ++ return false; ++ ++ return a->cdclk != b->cdclk && ++ a->vco == b->vco && ++ a->ref == b->ref; ++} ++ ++/** ++ * intel_cdclk_changed - Determine if two CDCLK states are different ++ * @a: first CDCLK state ++ * @b: second CDCLK state ++ * ++ * Returns: ++ * True if the CDCLK states don't match, false if they do. ++ */ ++bool intel_cdclk_changed(const struct intel_cdclk_state *a, ++ const struct intel_cdclk_state *b) ++{ ++ return intel_cdclk_needs_modeset(a, b) || ++ a->voltage_level != b->voltage_level; ++} ++ ++/** ++ * intel_cdclk_swap_state - make atomic CDCLK configuration effective ++ * @state: atomic state ++ * ++ * This is the CDCLK version of drm_atomic_helper_swap_state() since the ++ * helper does not handle driver-specific global state. ++ * ++ * Similarly to the atomic helpers this function does a complete swap, ++ * i.e. it also puts the old state into @state. This is used by the commit ++ * code to determine how CDCLK has changed (for instance did it increase or ++ * decrease). ++ */ ++void intel_cdclk_swap_state(struct intel_atomic_state *state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(state->base.dev); ++ ++ swap(state->cdclk.logical, dev_priv->cdclk.logical); ++ swap(state->cdclk.actual, dev_priv->cdclk.actual); ++} ++ ++void intel_dump_cdclk_state(const struct intel_cdclk_state *cdclk_state, ++ const char *context) ++{ ++ DRM_DEBUG_DRIVER("%s %d kHz, VCO %d kHz, ref %d kHz, bypass %d kHz, voltage level %d\n", ++ context, cdclk_state->cdclk, cdclk_state->vco, ++ cdclk_state->ref, cdclk_state->bypass, ++ cdclk_state->voltage_level); ++} ++ ++/** ++ * intel_set_cdclk - Push the CDCLK state to the hardware ++ * @dev_priv: i915 device ++ * @cdclk_state: new CDCLK state ++ * @pipe: pipe with which to synchronize the update ++ * ++ * Program the hardware based on the passed in CDCLK state, ++ * if necessary. ++ */ ++static void intel_set_cdclk(struct drm_i915_private *dev_priv, ++ const struct intel_cdclk_state *cdclk_state, ++ enum pipe pipe) ++{ ++ if (!intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_state)) ++ return; ++ ++ if (WARN_ON_ONCE(!dev_priv->display.set_cdclk)) ++ return; ++ ++ intel_dump_cdclk_state(cdclk_state, "Changing CDCLK to"); ++ ++ dev_priv->display.set_cdclk(dev_priv, cdclk_state, pipe); ++ ++ if (WARN(intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_state), ++ "cdclk state doesn't match!\n")) { ++ intel_dump_cdclk_state(&dev_priv->cdclk.hw, "[hw state]"); ++ intel_dump_cdclk_state(cdclk_state, "[sw state]"); ++ } ++} ++ ++/** ++ * intel_set_cdclk_pre_plane_update - Push the CDCLK state to the hardware ++ * @dev_priv: i915 device ++ * @old_state: old CDCLK state ++ * @new_state: new CDCLK state ++ * @pipe: pipe with which to synchronize the update ++ * ++ * Program the hardware before updating the HW plane state based on the passed ++ * in CDCLK state, if necessary. ++ */ ++void ++intel_set_cdclk_pre_plane_update(struct drm_i915_private *dev_priv, ++ const struct intel_cdclk_state *old_state, ++ const struct intel_cdclk_state *new_state, ++ enum pipe pipe) ++{ ++ if (pipe == INVALID_PIPE || old_state->cdclk <= new_state->cdclk) ++ intel_set_cdclk(dev_priv, new_state, pipe); ++} ++ ++/** ++ * intel_set_cdclk_post_plane_update - Push the CDCLK state to the hardware ++ * @dev_priv: i915 device ++ * @old_state: old CDCLK state ++ * @new_state: new CDCLK state ++ * @pipe: pipe with which to synchronize the update ++ * ++ * Program the hardware after updating the HW plane state based on the passed ++ * in CDCLK state, if necessary. ++ */ ++void ++intel_set_cdclk_post_plane_update(struct drm_i915_private *dev_priv, ++ const struct intel_cdclk_state *old_state, ++ const struct intel_cdclk_state *new_state, ++ enum pipe pipe) ++{ ++ if (pipe != INVALID_PIPE && old_state->cdclk > new_state->cdclk) ++ intel_set_cdclk(dev_priv, new_state, pipe); ++} ++ ++static int intel_pixel_rate_to_cdclk(struct drm_i915_private *dev_priv, ++ int pixel_rate) ++{ ++ if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) ++ return DIV_ROUND_UP(pixel_rate, 2); ++ else if (IS_GEN(dev_priv, 9) || ++ IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) ++ return pixel_rate; ++ else if (IS_CHERRYVIEW(dev_priv)) ++ return DIV_ROUND_UP(pixel_rate * 100, 95); ++ else ++ return DIV_ROUND_UP(pixel_rate * 100, 90); ++} ++ ++int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = ++ to_i915(crtc_state->base.crtc->dev); ++ int min_cdclk; ++ ++ if (!crtc_state->base.enable) ++ return 0; ++ ++ min_cdclk = intel_pixel_rate_to_cdclk(dev_priv, crtc_state->pixel_rate); ++ ++ /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */ ++ if (IS_BROADWELL(dev_priv) && hsw_crtc_state_ips_capable(crtc_state)) ++ min_cdclk = DIV_ROUND_UP(min_cdclk * 100, 95); ++ ++ /* BSpec says "Do not use DisplayPort with CDCLK less than 432 MHz, ++ * audio enabled, port width x4, and link rate HBR2 (5.4 GHz), or else ++ * there may be audio corruption or screen corruption." This cdclk ++ * restriction for GLK is 316.8 MHz. ++ */ ++ if (intel_crtc_has_dp_encoder(crtc_state) && ++ crtc_state->has_audio && ++ crtc_state->port_clock >= 540000 && ++ crtc_state->lane_count == 4) { ++ if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv)) { ++ /* Display WA #1145: glk,cnl */ ++ min_cdclk = max(316800, min_cdclk); ++ } else if (IS_GEN(dev_priv, 9) || IS_BROADWELL(dev_priv)) { ++ /* Display WA #1144: skl,bxt */ ++ min_cdclk = max(432000, min_cdclk); ++ } ++ } ++ ++ /* ++ * According to BSpec, "The CD clock frequency must be at least twice ++ * the frequency of the Azalia BCLK." and BCLK is 96 MHz by default. ++ */ ++ if (crtc_state->has_audio && INTEL_GEN(dev_priv) >= 9) ++ min_cdclk = max(2 * 96000, min_cdclk); ++ ++ /* ++ * "For DP audio configuration, cdclk frequency shall be set to ++ * meet the following requirements: ++ * DP Link Frequency(MHz) | Cdclk frequency(MHz) ++ * 270 | 320 or higher ++ * 162 | 200 or higher" ++ */ ++ if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) && ++ intel_crtc_has_dp_encoder(crtc_state) && crtc_state->has_audio) ++ min_cdclk = max(crtc_state->port_clock, min_cdclk); ++ ++ /* ++ * On Valleyview some DSI panels lose (v|h)sync when the clock is lower ++ * than 320000KHz. ++ */ ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) && ++ IS_VALLEYVIEW(dev_priv)) ++ min_cdclk = max(320000, min_cdclk); ++ ++ if (min_cdclk > dev_priv->max_cdclk_freq) { ++ DRM_DEBUG_KMS("required cdclk (%d kHz) exceeds max (%d kHz)\n", ++ min_cdclk, dev_priv->max_cdclk_freq); ++ return -EINVAL; ++ } ++ ++ return min_cdclk; ++} ++ ++static int intel_compute_min_cdclk(struct drm_atomic_state *state) ++{ ++ struct intel_atomic_state *intel_state = to_intel_atomic_state(state); ++ struct drm_i915_private *dev_priv = to_i915(state->dev); ++ struct intel_crtc *crtc; ++ struct intel_crtc_state *crtc_state; ++ int min_cdclk, i; ++ enum pipe pipe; ++ ++ memcpy(intel_state->min_cdclk, dev_priv->min_cdclk, ++ sizeof(intel_state->min_cdclk)); ++ ++ for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) { ++ min_cdclk = intel_crtc_compute_min_cdclk(crtc_state); ++ if (min_cdclk < 0) ++ return min_cdclk; ++ ++ intel_state->min_cdclk[i] = min_cdclk; ++ } ++ ++ min_cdclk = intel_state->cdclk.force_min_cdclk; ++ for_each_pipe(dev_priv, pipe) ++ min_cdclk = max(intel_state->min_cdclk[pipe], min_cdclk); ++ ++ return min_cdclk; ++} ++ ++/* ++ * Note that this functions assumes that 0 is ++ * the lowest voltage value, and higher values ++ * correspond to increasingly higher voltages. ++ * ++ * Should that relationship no longer hold on ++ * future platforms this code will need to be ++ * adjusted. ++ */ ++static u8 cnl_compute_min_voltage_level(struct intel_atomic_state *state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(state->base.dev); ++ struct intel_crtc *crtc; ++ struct intel_crtc_state *crtc_state; ++ u8 min_voltage_level; ++ int i; ++ enum pipe pipe; ++ ++ memcpy(state->min_voltage_level, dev_priv->min_voltage_level, ++ sizeof(state->min_voltage_level)); ++ ++ for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) { ++ if (crtc_state->base.enable) ++ state->min_voltage_level[i] = ++ crtc_state->min_voltage_level; ++ else ++ state->min_voltage_level[i] = 0; ++ } ++ ++ min_voltage_level = 0; ++ for_each_pipe(dev_priv, pipe) ++ min_voltage_level = max(state->min_voltage_level[pipe], ++ min_voltage_level); ++ ++ return min_voltage_level; ++} ++ ++static int vlv_modeset_calc_cdclk(struct drm_atomic_state *state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(state->dev); ++ struct intel_atomic_state *intel_state = to_intel_atomic_state(state); ++ int min_cdclk, cdclk; ++ ++ min_cdclk = intel_compute_min_cdclk(state); ++ if (min_cdclk < 0) ++ return min_cdclk; ++ ++ cdclk = vlv_calc_cdclk(dev_priv, min_cdclk); ++ ++ intel_state->cdclk.logical.cdclk = cdclk; ++ intel_state->cdclk.logical.voltage_level = ++ vlv_calc_voltage_level(dev_priv, cdclk); ++ ++ if (!intel_state->active_crtcs) { ++ cdclk = vlv_calc_cdclk(dev_priv, ++ intel_state->cdclk.force_min_cdclk); ++ ++ intel_state->cdclk.actual.cdclk = cdclk; ++ intel_state->cdclk.actual.voltage_level = ++ vlv_calc_voltage_level(dev_priv, cdclk); ++ } else { ++ intel_state->cdclk.actual = ++ intel_state->cdclk.logical; ++ } ++ ++ return 0; ++} ++ ++static int bdw_modeset_calc_cdclk(struct drm_atomic_state *state) ++{ ++ struct intel_atomic_state *intel_state = to_intel_atomic_state(state); ++ int min_cdclk, cdclk; ++ ++ min_cdclk = intel_compute_min_cdclk(state); ++ if (min_cdclk < 0) ++ return min_cdclk; ++ ++ /* ++ * FIXME should also account for plane ratio ++ * once 64bpp pixel formats are supported. ++ */ ++ cdclk = bdw_calc_cdclk(min_cdclk); ++ ++ intel_state->cdclk.logical.cdclk = cdclk; ++ intel_state->cdclk.logical.voltage_level = ++ bdw_calc_voltage_level(cdclk); ++ ++ if (!intel_state->active_crtcs) { ++ cdclk = bdw_calc_cdclk(intel_state->cdclk.force_min_cdclk); ++ ++ intel_state->cdclk.actual.cdclk = cdclk; ++ intel_state->cdclk.actual.voltage_level = ++ bdw_calc_voltage_level(cdclk); ++ } else { ++ intel_state->cdclk.actual = ++ intel_state->cdclk.logical; ++ } ++ ++ return 0; ++} ++ ++static int skl_dpll0_vco(struct intel_atomic_state *intel_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(intel_state->base.dev); ++ struct intel_crtc *crtc; ++ struct intel_crtc_state *crtc_state; ++ int vco, i; ++ ++ vco = intel_state->cdclk.logical.vco; ++ if (!vco) ++ vco = dev_priv->skl_preferred_vco_freq; ++ ++ for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) { ++ if (!crtc_state->base.enable) ++ continue; ++ ++ if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP)) ++ continue; ++ ++ /* ++ * DPLL0 VCO may need to be adjusted to get the correct ++ * clock for eDP. This will affect cdclk as well. ++ */ ++ switch (crtc_state->port_clock / 2) { ++ case 108000: ++ case 216000: ++ vco = 8640000; ++ break; ++ default: ++ vco = 8100000; ++ break; ++ } ++ } ++ ++ return vco; ++} ++ ++static int skl_modeset_calc_cdclk(struct drm_atomic_state *state) ++{ ++ struct intel_atomic_state *intel_state = to_intel_atomic_state(state); ++ int min_cdclk, cdclk, vco; ++ ++ min_cdclk = intel_compute_min_cdclk(state); ++ if (min_cdclk < 0) ++ return min_cdclk; ++ ++ vco = skl_dpll0_vco(intel_state); ++ ++ /* ++ * FIXME should also account for plane ratio ++ * once 64bpp pixel formats are supported. ++ */ ++ cdclk = skl_calc_cdclk(min_cdclk, vco); ++ ++ intel_state->cdclk.logical.vco = vco; ++ intel_state->cdclk.logical.cdclk = cdclk; ++ intel_state->cdclk.logical.voltage_level = ++ skl_calc_voltage_level(cdclk); ++ ++ if (!intel_state->active_crtcs) { ++ cdclk = skl_calc_cdclk(intel_state->cdclk.force_min_cdclk, vco); ++ ++ intel_state->cdclk.actual.vco = vco; ++ intel_state->cdclk.actual.cdclk = cdclk; ++ intel_state->cdclk.actual.voltage_level = ++ skl_calc_voltage_level(cdclk); ++ } else { ++ intel_state->cdclk.actual = ++ intel_state->cdclk.logical; ++ } ++ ++ return 0; ++} ++ ++static int bxt_modeset_calc_cdclk(struct drm_atomic_state *state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(state->dev); ++ struct intel_atomic_state *intel_state = to_intel_atomic_state(state); ++ int min_cdclk, cdclk, vco; ++ ++ min_cdclk = intel_compute_min_cdclk(state); ++ if (min_cdclk < 0) ++ return min_cdclk; ++ ++ if (IS_GEMINILAKE(dev_priv)) { ++ cdclk = glk_calc_cdclk(min_cdclk); ++ vco = glk_de_pll_vco(dev_priv, cdclk); ++ } else { ++ cdclk = bxt_calc_cdclk(min_cdclk); ++ vco = bxt_de_pll_vco(dev_priv, cdclk); ++ } ++ ++ intel_state->cdclk.logical.vco = vco; ++ intel_state->cdclk.logical.cdclk = cdclk; ++ intel_state->cdclk.logical.voltage_level = ++ bxt_calc_voltage_level(cdclk); ++ ++ if (!intel_state->active_crtcs) { ++ if (IS_GEMINILAKE(dev_priv)) { ++ cdclk = glk_calc_cdclk(intel_state->cdclk.force_min_cdclk); ++ vco = glk_de_pll_vco(dev_priv, cdclk); ++ } else { ++ cdclk = bxt_calc_cdclk(intel_state->cdclk.force_min_cdclk); ++ vco = bxt_de_pll_vco(dev_priv, cdclk); ++ } ++ ++ intel_state->cdclk.actual.vco = vco; ++ intel_state->cdclk.actual.cdclk = cdclk; ++ intel_state->cdclk.actual.voltage_level = ++ bxt_calc_voltage_level(cdclk); ++ } else { ++ intel_state->cdclk.actual = ++ intel_state->cdclk.logical; ++ } ++ ++ return 0; ++} ++ ++static int cnl_modeset_calc_cdclk(struct drm_atomic_state *state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(state->dev); ++ struct intel_atomic_state *intel_state = to_intel_atomic_state(state); ++ int min_cdclk, cdclk, vco; ++ ++ min_cdclk = intel_compute_min_cdclk(state); ++ if (min_cdclk < 0) ++ return min_cdclk; ++ ++ cdclk = cnl_calc_cdclk(min_cdclk); ++ vco = cnl_cdclk_pll_vco(dev_priv, cdclk); ++ ++ intel_state->cdclk.logical.vco = vco; ++ intel_state->cdclk.logical.cdclk = cdclk; ++ intel_state->cdclk.logical.voltage_level = ++ max(cnl_calc_voltage_level(cdclk), ++ cnl_compute_min_voltage_level(intel_state)); ++ ++ if (!intel_state->active_crtcs) { ++ cdclk = cnl_calc_cdclk(intel_state->cdclk.force_min_cdclk); ++ vco = cnl_cdclk_pll_vco(dev_priv, cdclk); ++ ++ intel_state->cdclk.actual.vco = vco; ++ intel_state->cdclk.actual.cdclk = cdclk; ++ intel_state->cdclk.actual.voltage_level = ++ cnl_calc_voltage_level(cdclk); ++ } else { ++ intel_state->cdclk.actual = ++ intel_state->cdclk.logical; ++ } ++ ++ return 0; ++} ++ ++static int icl_modeset_calc_cdclk(struct drm_atomic_state *state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(state->dev); ++ struct intel_atomic_state *intel_state = to_intel_atomic_state(state); ++ unsigned int ref = intel_state->cdclk.logical.ref; ++ int min_cdclk, cdclk, vco; ++ ++ min_cdclk = intel_compute_min_cdclk(state); ++ if (min_cdclk < 0) ++ return min_cdclk; ++ ++ cdclk = icl_calc_cdclk(min_cdclk, ref); ++ vco = icl_calc_cdclk_pll_vco(dev_priv, cdclk); ++ ++ intel_state->cdclk.logical.vco = vco; ++ intel_state->cdclk.logical.cdclk = cdclk; ++ intel_state->cdclk.logical.voltage_level = ++ max(icl_calc_voltage_level(cdclk), ++ cnl_compute_min_voltage_level(intel_state)); ++ ++ if (!intel_state->active_crtcs) { ++ cdclk = icl_calc_cdclk(intel_state->cdclk.force_min_cdclk, ref); ++ vco = icl_calc_cdclk_pll_vco(dev_priv, cdclk); ++ ++ intel_state->cdclk.actual.vco = vco; ++ intel_state->cdclk.actual.cdclk = cdclk; ++ intel_state->cdclk.actual.voltage_level = ++ icl_calc_voltage_level(cdclk); ++ } else { ++ intel_state->cdclk.actual = intel_state->cdclk.logical; ++ } ++ ++ return 0; ++} ++ ++static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv) ++{ ++ int max_cdclk_freq = dev_priv->max_cdclk_freq; ++ ++ if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) ++ return 2 * max_cdclk_freq; ++ else if (IS_GEN(dev_priv, 9) || ++ IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) ++ return max_cdclk_freq; ++ else if (IS_CHERRYVIEW(dev_priv)) ++ return max_cdclk_freq*95/100; ++ else if (INTEL_GEN(dev_priv) < 4) ++ return 2*max_cdclk_freq*90/100; ++ else ++ return max_cdclk_freq*90/100; ++} ++ ++/** ++ * intel_update_max_cdclk - Determine the maximum support CDCLK frequency ++ * @dev_priv: i915 device ++ * ++ * Determine the maximum CDCLK frequency the platform supports, and also ++ * derive the maximum dot clock frequency the maximum CDCLK frequency ++ * allows. ++ */ ++void intel_update_max_cdclk(struct drm_i915_private *dev_priv) ++{ ++ if (INTEL_GEN(dev_priv) >= 11) { ++ if (dev_priv->cdclk.hw.ref == 24000) ++ dev_priv->max_cdclk_freq = 648000; ++ else ++ dev_priv->max_cdclk_freq = 652800; ++ } else if (IS_CANNONLAKE(dev_priv)) { ++ dev_priv->max_cdclk_freq = 528000; ++ } else if (IS_GEN9_BC(dev_priv)) { ++ u32 limit = I915_READ(SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK; ++ int max_cdclk, vco; ++ ++ vco = dev_priv->skl_preferred_vco_freq; ++ WARN_ON(vco != 8100000 && vco != 8640000); ++ ++ /* ++ * Use the lower (vco 8640) cdclk values as a ++ * first guess. skl_calc_cdclk() will correct it ++ * if the preferred vco is 8100 instead. ++ */ ++ if (limit == SKL_DFSM_CDCLK_LIMIT_675) ++ max_cdclk = 617143; ++ else if (limit == SKL_DFSM_CDCLK_LIMIT_540) ++ max_cdclk = 540000; ++ else if (limit == SKL_DFSM_CDCLK_LIMIT_450) ++ max_cdclk = 432000; ++ else ++ max_cdclk = 308571; ++ ++ dev_priv->max_cdclk_freq = skl_calc_cdclk(max_cdclk, vco); ++ } else if (IS_GEMINILAKE(dev_priv)) { ++ dev_priv->max_cdclk_freq = 316800; ++ } else if (IS_BROXTON(dev_priv)) { ++ dev_priv->max_cdclk_freq = 624000; ++ } else if (IS_BROADWELL(dev_priv)) { ++ /* ++ * FIXME with extra cooling we can allow ++ * 540 MHz for ULX and 675 Mhz for ULT. ++ * How can we know if extra cooling is ++ * available? PCI ID, VTB, something else? ++ */ ++ if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT) ++ dev_priv->max_cdclk_freq = 450000; ++ else if (IS_BDW_ULX(dev_priv)) ++ dev_priv->max_cdclk_freq = 450000; ++ else if (IS_BDW_ULT(dev_priv)) ++ dev_priv->max_cdclk_freq = 540000; ++ else ++ dev_priv->max_cdclk_freq = 675000; ++ } else if (IS_CHERRYVIEW(dev_priv)) { ++ dev_priv->max_cdclk_freq = 320000; ++ } else if (IS_VALLEYVIEW(dev_priv)) { ++ dev_priv->max_cdclk_freq = 400000; ++ } else { ++ /* otherwise assume cdclk is fixed */ ++ dev_priv->max_cdclk_freq = dev_priv->cdclk.hw.cdclk; ++ } ++ ++ dev_priv->max_dotclk_freq = intel_compute_max_dotclk(dev_priv); ++ ++ DRM_DEBUG_DRIVER("Max CD clock rate: %d kHz\n", ++ dev_priv->max_cdclk_freq); ++ ++ DRM_DEBUG_DRIVER("Max dotclock rate: %d kHz\n", ++ dev_priv->max_dotclk_freq); ++} ++ ++/** ++ * intel_update_cdclk - Determine the current CDCLK frequency ++ * @dev_priv: i915 device ++ * ++ * Determine the current CDCLK frequency. ++ */ ++void intel_update_cdclk(struct drm_i915_private *dev_priv) ++{ ++ dev_priv->display.get_cdclk(dev_priv, &dev_priv->cdclk.hw); ++ ++ /* ++ * 9:0 CMBUS [sic] CDCLK frequency (cdfreq): ++ * Programmng [sic] note: bit[9:2] should be programmed to the number ++ * of cdclk that generates 4MHz reference clock freq which is used to ++ * generate GMBus clock. This will vary with the cdclk freq. ++ */ ++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ I915_WRITE(GMBUSFREQ_VLV, ++ DIV_ROUND_UP(dev_priv->cdclk.hw.cdclk, 1000)); ++} ++ ++static int cnp_rawclk(struct drm_i915_private *dev_priv) ++{ ++ u32 rawclk; ++ int divider, fraction; ++ ++ if (I915_READ(SFUSE_STRAP) & SFUSE_STRAP_RAW_FREQUENCY) { ++ /* 24 MHz */ ++ divider = 24000; ++ fraction = 0; ++ } else { ++ /* 19.2 MHz */ ++ divider = 19000; ++ fraction = 200; ++ } ++ ++ rawclk = CNP_RAWCLK_DIV(divider / 1000); ++ if (fraction) { ++ int numerator = 1; ++ ++ rawclk |= CNP_RAWCLK_DEN(DIV_ROUND_CLOSEST(numerator * 1000, ++ fraction) - 1); ++ if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP) ++ rawclk |= ICP_RAWCLK_NUM(numerator); ++ } ++ ++ I915_WRITE(PCH_RAWCLK_FREQ, rawclk); ++ return divider + fraction; ++} ++ ++static int pch_rawclk(struct drm_i915_private *dev_priv) ++{ ++ return (I915_READ(PCH_RAWCLK_FREQ) & RAWCLK_FREQ_MASK) * 1000; ++} ++ ++static int vlv_hrawclk(struct drm_i915_private *dev_priv) ++{ ++ /* RAWCLK_FREQ_VLV register updated from power well code */ ++ return vlv_get_cck_clock_hpll(dev_priv, "hrawclk", ++ CCK_DISPLAY_REF_CLOCK_CONTROL); ++} ++ ++static int g4x_hrawclk(struct drm_i915_private *dev_priv) ++{ ++ u32 clkcfg; ++ ++ /* hrawclock is 1/4 the FSB frequency */ ++ clkcfg = I915_READ(CLKCFG); ++ switch (clkcfg & CLKCFG_FSB_MASK) { ++ case CLKCFG_FSB_400: ++ return 100000; ++ case CLKCFG_FSB_533: ++ return 133333; ++ case CLKCFG_FSB_667: ++ return 166667; ++ case CLKCFG_FSB_800: ++ return 200000; ++ case CLKCFG_FSB_1067: ++ case CLKCFG_FSB_1067_ALT: ++ return 266667; ++ case CLKCFG_FSB_1333: ++ case CLKCFG_FSB_1333_ALT: ++ return 333333; ++ default: ++ return 133333; ++ } ++} ++ ++/** ++ * intel_update_rawclk - Determine the current RAWCLK frequency ++ * @dev_priv: i915 device ++ * ++ * Determine the current RAWCLK frequency. RAWCLK is a fixed ++ * frequency clock so this needs to done only once. ++ */ ++void intel_update_rawclk(struct drm_i915_private *dev_priv) ++{ ++ if (INTEL_PCH_TYPE(dev_priv) >= PCH_CNP) ++ dev_priv->rawclk_freq = cnp_rawclk(dev_priv); ++ else if (HAS_PCH_SPLIT(dev_priv)) ++ dev_priv->rawclk_freq = pch_rawclk(dev_priv); ++ else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ dev_priv->rawclk_freq = vlv_hrawclk(dev_priv); ++ else if (IS_G4X(dev_priv) || IS_PINEVIEW(dev_priv)) ++ dev_priv->rawclk_freq = g4x_hrawclk(dev_priv); ++ else ++ /* no rawclk on other platforms, or no need to know it */ ++ return; ++ ++ DRM_DEBUG_DRIVER("rawclk rate: %d kHz\n", dev_priv->rawclk_freq); ++} ++ ++/** ++ * intel_init_cdclk_hooks - Initialize CDCLK related modesetting hooks ++ * @dev_priv: i915 device ++ */ ++void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv) ++{ ++ if (INTEL_GEN(dev_priv) >= 11) { ++ dev_priv->display.set_cdclk = icl_set_cdclk; ++ dev_priv->display.modeset_calc_cdclk = icl_modeset_calc_cdclk; ++ } else if (IS_CANNONLAKE(dev_priv)) { ++ dev_priv->display.set_cdclk = cnl_set_cdclk; ++ dev_priv->display.modeset_calc_cdclk = ++ cnl_modeset_calc_cdclk; ++ } else if (IS_GEN9_LP(dev_priv)) { ++ dev_priv->display.set_cdclk = bxt_set_cdclk; ++ dev_priv->display.modeset_calc_cdclk = ++ bxt_modeset_calc_cdclk; ++ } else if (IS_GEN9_BC(dev_priv)) { ++ dev_priv->display.set_cdclk = skl_set_cdclk; ++ dev_priv->display.modeset_calc_cdclk = ++ skl_modeset_calc_cdclk; ++ } else if (IS_BROADWELL(dev_priv)) { ++ dev_priv->display.set_cdclk = bdw_set_cdclk; ++ dev_priv->display.modeset_calc_cdclk = ++ bdw_modeset_calc_cdclk; ++ } else if (IS_CHERRYVIEW(dev_priv)) { ++ dev_priv->display.set_cdclk = chv_set_cdclk; ++ dev_priv->display.modeset_calc_cdclk = ++ vlv_modeset_calc_cdclk; ++ } else if (IS_VALLEYVIEW(dev_priv)) { ++ dev_priv->display.set_cdclk = vlv_set_cdclk; ++ dev_priv->display.modeset_calc_cdclk = ++ vlv_modeset_calc_cdclk; ++ } ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ dev_priv->display.get_cdclk = icl_get_cdclk; ++ else if (IS_CANNONLAKE(dev_priv)) ++ dev_priv->display.get_cdclk = cnl_get_cdclk; ++ else if (IS_GEN9_LP(dev_priv)) ++ dev_priv->display.get_cdclk = bxt_get_cdclk; ++ else if (IS_GEN9_BC(dev_priv)) ++ dev_priv->display.get_cdclk = skl_get_cdclk; ++ else if (IS_BROADWELL(dev_priv)) ++ dev_priv->display.get_cdclk = bdw_get_cdclk; ++ else if (IS_HASWELL(dev_priv)) ++ dev_priv->display.get_cdclk = hsw_get_cdclk; ++ else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ dev_priv->display.get_cdclk = vlv_get_cdclk; ++ else if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv)) ++ dev_priv->display.get_cdclk = fixed_400mhz_get_cdclk; ++ else if (IS_GEN(dev_priv, 5)) ++ dev_priv->display.get_cdclk = fixed_450mhz_get_cdclk; ++ else if (IS_GM45(dev_priv)) ++ dev_priv->display.get_cdclk = gm45_get_cdclk; ++ else if (IS_G45(dev_priv)) ++ dev_priv->display.get_cdclk = g33_get_cdclk; ++ else if (IS_I965GM(dev_priv)) ++ dev_priv->display.get_cdclk = i965gm_get_cdclk; ++ else if (IS_I965G(dev_priv)) ++ dev_priv->display.get_cdclk = fixed_400mhz_get_cdclk; ++ else if (IS_PINEVIEW(dev_priv)) ++ dev_priv->display.get_cdclk = pnv_get_cdclk; ++ else if (IS_G33(dev_priv)) ++ dev_priv->display.get_cdclk = g33_get_cdclk; ++ else if (IS_I945GM(dev_priv)) ++ dev_priv->display.get_cdclk = i945gm_get_cdclk; ++ else if (IS_I945G(dev_priv)) ++ dev_priv->display.get_cdclk = fixed_400mhz_get_cdclk; ++ else if (IS_I915GM(dev_priv)) ++ dev_priv->display.get_cdclk = i915gm_get_cdclk; ++ else if (IS_I915G(dev_priv)) ++ dev_priv->display.get_cdclk = fixed_333mhz_get_cdclk; ++ else if (IS_I865G(dev_priv)) ++ dev_priv->display.get_cdclk = fixed_266mhz_get_cdclk; ++ else if (IS_I85X(dev_priv)) ++ dev_priv->display.get_cdclk = i85x_get_cdclk; ++ else if (IS_I845G(dev_priv)) ++ dev_priv->display.get_cdclk = fixed_200mhz_get_cdclk; ++ else { /* 830 */ ++ WARN(!IS_I830(dev_priv), ++ "Unknown platform. Assuming 133 MHz CDCLK\n"); ++ dev_priv->display.get_cdclk = fixed_133mhz_get_cdclk; ++ } ++} +diff --git a/drivers/gpu/drm/i915_legacy/intel_cdclk.h b/drivers/gpu/drm/i915_legacy/intel_cdclk.h +new file mode 100644 +index 000000000000..4d6f7f5f8930 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_cdclk.h +@@ -0,0 +1,46 @@ ++/* SPDX-License-Identifier: MIT */ ++/* ++ * Copyright © 2019 Intel Corporation ++ */ ++ ++#ifndef __INTEL_CDCLK_H__ ++#define __INTEL_CDCLK_H__ ++ ++#include ++ ++#include "intel_display.h" ++ ++struct drm_i915_private; ++struct intel_atomic_state; ++struct intel_cdclk_state; ++struct intel_crtc_state; ++ ++int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state); ++void intel_cdclk_init(struct drm_i915_private *i915); ++void intel_cdclk_uninit(struct drm_i915_private *i915); ++void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv); ++void intel_update_max_cdclk(struct drm_i915_private *dev_priv); ++void intel_update_cdclk(struct drm_i915_private *dev_priv); ++void intel_update_rawclk(struct drm_i915_private *dev_priv); ++bool intel_cdclk_needs_cd2x_update(struct drm_i915_private *dev_priv, ++ const struct intel_cdclk_state *a, ++ const struct intel_cdclk_state *b); ++bool intel_cdclk_needs_modeset(const struct intel_cdclk_state *a, ++ const struct intel_cdclk_state *b); ++bool intel_cdclk_changed(const struct intel_cdclk_state *a, ++ const struct intel_cdclk_state *b); ++void intel_cdclk_swap_state(struct intel_atomic_state *state); ++void ++intel_set_cdclk_pre_plane_update(struct drm_i915_private *dev_priv, ++ const struct intel_cdclk_state *old_state, ++ const struct intel_cdclk_state *new_state, ++ enum pipe pipe); ++void ++intel_set_cdclk_post_plane_update(struct drm_i915_private *dev_priv, ++ const struct intel_cdclk_state *old_state, ++ const struct intel_cdclk_state *new_state, ++ enum pipe pipe); ++void intel_dump_cdclk_state(const struct intel_cdclk_state *cdclk_state, ++ const char *context); ++ ++#endif /* __INTEL_CDCLK_H__ */ +diff --git a/drivers/gpu/drm/i915_legacy/intel_color.c b/drivers/gpu/drm/i915_legacy/intel_color.c +new file mode 100644 +index 000000000000..9093daabc290 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_color.c +@@ -0,0 +1,1278 @@ ++/* ++ * Copyright © 2016 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER ++ * DEALINGS IN THE SOFTWARE. ++ * ++ */ ++ ++#include "intel_color.h" ++#include "intel_drv.h" ++ ++#define CTM_COEFF_SIGN (1ULL << 63) ++ ++#define CTM_COEFF_1_0 (1ULL << 32) ++#define CTM_COEFF_2_0 (CTM_COEFF_1_0 << 1) ++#define CTM_COEFF_4_0 (CTM_COEFF_2_0 << 1) ++#define CTM_COEFF_8_0 (CTM_COEFF_4_0 << 1) ++#define CTM_COEFF_0_5 (CTM_COEFF_1_0 >> 1) ++#define CTM_COEFF_0_25 (CTM_COEFF_0_5 >> 1) ++#define CTM_COEFF_0_125 (CTM_COEFF_0_25 >> 1) ++ ++#define CTM_COEFF_LIMITED_RANGE ((235ULL - 16ULL) * CTM_COEFF_1_0 / 255) ++ ++#define CTM_COEFF_NEGATIVE(coeff) (((coeff) & CTM_COEFF_SIGN) != 0) ++#define CTM_COEFF_ABS(coeff) ((coeff) & (CTM_COEFF_SIGN - 1)) ++ ++#define LEGACY_LUT_LENGTH 256 ++/* ++ * Extract the CSC coefficient from a CTM coefficient (in U32.32 fixed point ++ * format). This macro takes the coefficient we want transformed and the ++ * number of fractional bits. ++ * ++ * We only have a 9 bits precision window which slides depending on the value ++ * of the CTM coefficient and we write the value from bit 3. We also round the ++ * value. ++ */ ++#define ILK_CSC_COEFF_FP(coeff, fbits) \ ++ (clamp_val(((coeff) >> (32 - (fbits) - 3)) + 4, 0, 0xfff) & 0xff8) ++ ++#define ILK_CSC_COEFF_LIMITED_RANGE 0x0dc0 ++#define ILK_CSC_COEFF_1_0 0x7800 ++ ++#define ILK_CSC_POSTOFF_LIMITED_RANGE (16 * (1 << 12) / 255) ++ ++static const u16 ilk_csc_off_zero[3] = {}; ++ ++static const u16 ilk_csc_coeff_identity[9] = { ++ ILK_CSC_COEFF_1_0, 0, 0, ++ 0, ILK_CSC_COEFF_1_0, 0, ++ 0, 0, ILK_CSC_COEFF_1_0, ++}; ++ ++static const u16 ilk_csc_postoff_limited_range[3] = { ++ ILK_CSC_POSTOFF_LIMITED_RANGE, ++ ILK_CSC_POSTOFF_LIMITED_RANGE, ++ ILK_CSC_POSTOFF_LIMITED_RANGE, ++}; ++ ++static const u16 ilk_csc_coeff_limited_range[9] = { ++ ILK_CSC_COEFF_LIMITED_RANGE, 0, 0, ++ 0, ILK_CSC_COEFF_LIMITED_RANGE, 0, ++ 0, 0, ILK_CSC_COEFF_LIMITED_RANGE, ++}; ++ ++/* ++ * These values are direct register values specified in the Bspec, ++ * for RGB->YUV conversion matrix (colorspace BT709) ++ */ ++static const u16 ilk_csc_coeff_rgb_to_ycbcr[9] = { ++ 0x1e08, 0x9cc0, 0xb528, ++ 0x2ba8, 0x09d8, 0x37e8, ++ 0xbce8, 0x9ad8, 0x1e08, ++}; ++ ++/* Post offset values for RGB->YCBCR conversion */ ++static const u16 ilk_csc_postoff_rgb_to_ycbcr[3] = { ++ 0x0800, 0x0100, 0x0800, ++}; ++ ++static bool lut_is_legacy(const struct drm_property_blob *lut) ++{ ++ return drm_color_lut_size(lut) == LEGACY_LUT_LENGTH; ++} ++ ++static bool crtc_state_is_legacy_gamma(const struct intel_crtc_state *crtc_state) ++{ ++ return !crtc_state->base.degamma_lut && ++ !crtc_state->base.ctm && ++ crtc_state->base.gamma_lut && ++ lut_is_legacy(crtc_state->base.gamma_lut); ++} ++ ++/* ++ * When using limited range, multiply the matrix given by userspace by ++ * the matrix that we would use for the limited range. ++ */ ++static u64 *ctm_mult_by_limited(u64 *result, const u64 *input) ++{ ++ int i; ++ ++ for (i = 0; i < 9; i++) { ++ u64 user_coeff = input[i]; ++ u32 limited_coeff = CTM_COEFF_LIMITED_RANGE; ++ u32 abs_coeff = clamp_val(CTM_COEFF_ABS(user_coeff), 0, ++ CTM_COEFF_4_0 - 1) >> 2; ++ ++ /* ++ * By scaling every co-efficient with limited range (16-235) ++ * vs full range (0-255) the final o/p will be scaled down to ++ * fit in the limited range supported by the panel. ++ */ ++ result[i] = mul_u32_u32(limited_coeff, abs_coeff) >> 30; ++ result[i] |= user_coeff & CTM_COEFF_SIGN; ++ } ++ ++ return result; ++} ++ ++static void ilk_update_pipe_csc(struct intel_crtc *crtc, ++ const u16 preoff[3], ++ const u16 coeff[9], ++ const u16 postoff[3]) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ ++ I915_WRITE(PIPE_CSC_PREOFF_HI(pipe), preoff[0]); ++ I915_WRITE(PIPE_CSC_PREOFF_ME(pipe), preoff[1]); ++ I915_WRITE(PIPE_CSC_PREOFF_LO(pipe), preoff[2]); ++ ++ I915_WRITE(PIPE_CSC_COEFF_RY_GY(pipe), coeff[0] << 16 | coeff[1]); ++ I915_WRITE(PIPE_CSC_COEFF_BY(pipe), coeff[2] << 16); ++ ++ I915_WRITE(PIPE_CSC_COEFF_RU_GU(pipe), coeff[3] << 16 | coeff[4]); ++ I915_WRITE(PIPE_CSC_COEFF_BU(pipe), coeff[5] << 16); ++ ++ I915_WRITE(PIPE_CSC_COEFF_RV_GV(pipe), coeff[6] << 16 | coeff[7]); ++ I915_WRITE(PIPE_CSC_COEFF_BV(pipe), coeff[8] << 16); ++ ++ if (INTEL_GEN(dev_priv) >= 7) { ++ I915_WRITE(PIPE_CSC_POSTOFF_HI(pipe), postoff[0]); ++ I915_WRITE(PIPE_CSC_POSTOFF_ME(pipe), postoff[1]); ++ I915_WRITE(PIPE_CSC_POSTOFF_LO(pipe), postoff[2]); ++ } ++} ++ ++static void icl_update_output_csc(struct intel_crtc *crtc, ++ const u16 preoff[3], ++ const u16 coeff[9], ++ const u16 postoff[3]) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ ++ I915_WRITE(PIPE_CSC_OUTPUT_PREOFF_HI(pipe), preoff[0]); ++ I915_WRITE(PIPE_CSC_OUTPUT_PREOFF_ME(pipe), preoff[1]); ++ I915_WRITE(PIPE_CSC_OUTPUT_PREOFF_LO(pipe), preoff[2]); ++ ++ I915_WRITE(PIPE_CSC_OUTPUT_COEFF_RY_GY(pipe), coeff[0] << 16 | coeff[1]); ++ I915_WRITE(PIPE_CSC_OUTPUT_COEFF_BY(pipe), coeff[2] << 16); ++ ++ I915_WRITE(PIPE_CSC_OUTPUT_COEFF_RU_GU(pipe), coeff[3] << 16 | coeff[4]); ++ I915_WRITE(PIPE_CSC_OUTPUT_COEFF_BU(pipe), coeff[5] << 16); ++ ++ I915_WRITE(PIPE_CSC_OUTPUT_COEFF_RV_GV(pipe), coeff[6] << 16 | coeff[7]); ++ I915_WRITE(PIPE_CSC_OUTPUT_COEFF_BV(pipe), coeff[8] << 16); ++ ++ I915_WRITE(PIPE_CSC_OUTPUT_POSTOFF_HI(pipe), postoff[0]); ++ I915_WRITE(PIPE_CSC_OUTPUT_POSTOFF_ME(pipe), postoff[1]); ++ I915_WRITE(PIPE_CSC_OUTPUT_POSTOFF_LO(pipe), postoff[2]); ++} ++ ++static bool ilk_csc_limited_range(const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); ++ ++ /* ++ * FIXME if there's a gamma LUT after the CSC, we should ++ * do the range compression using the gamma LUT instead. ++ */ ++ return crtc_state->limited_color_range && ++ (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv) || ++ IS_GEN_RANGE(dev_priv, 9, 10)); ++} ++ ++static void ilk_csc_convert_ctm(const struct intel_crtc_state *crtc_state, ++ u16 coeffs[9]) ++{ ++ const struct drm_color_ctm *ctm = crtc_state->base.ctm->data; ++ const u64 *input; ++ u64 temp[9]; ++ int i; ++ ++ if (ilk_csc_limited_range(crtc_state)) ++ input = ctm_mult_by_limited(temp, ctm->matrix); ++ else ++ input = ctm->matrix; ++ ++ /* ++ * Convert fixed point S31.32 input to format supported by the ++ * hardware. ++ */ ++ for (i = 0; i < 9; i++) { ++ u64 abs_coeff = ((1ULL << 63) - 1) & input[i]; ++ ++ /* ++ * Clamp input value to min/max supported by ++ * hardware. ++ */ ++ abs_coeff = clamp_val(abs_coeff, 0, CTM_COEFF_4_0 - 1); ++ ++ coeffs[i] = 0; ++ ++ /* sign bit */ ++ if (CTM_COEFF_NEGATIVE(input[i])) ++ coeffs[i] |= 1 << 15; ++ ++ if (abs_coeff < CTM_COEFF_0_125) ++ coeffs[i] |= (3 << 12) | ++ ILK_CSC_COEFF_FP(abs_coeff, 12); ++ else if (abs_coeff < CTM_COEFF_0_25) ++ coeffs[i] |= (2 << 12) | ++ ILK_CSC_COEFF_FP(abs_coeff, 11); ++ else if (abs_coeff < CTM_COEFF_0_5) ++ coeffs[i] |= (1 << 12) | ++ ILK_CSC_COEFF_FP(abs_coeff, 10); ++ else if (abs_coeff < CTM_COEFF_1_0) ++ coeffs[i] |= ILK_CSC_COEFF_FP(abs_coeff, 9); ++ else if (abs_coeff < CTM_COEFF_2_0) ++ coeffs[i] |= (7 << 12) | ++ ILK_CSC_COEFF_FP(abs_coeff, 8); ++ else ++ coeffs[i] |= (6 << 12) | ++ ILK_CSC_COEFF_FP(abs_coeff, 7); ++ } ++} ++ ++static void ilk_load_csc_matrix(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ bool limited_color_range = ilk_csc_limited_range(crtc_state); ++ ++ if (crtc_state->base.ctm) { ++ u16 coeff[9]; ++ ++ ilk_csc_convert_ctm(crtc_state, coeff); ++ ilk_update_pipe_csc(crtc, ilk_csc_off_zero, coeff, ++ limited_color_range ? ++ ilk_csc_postoff_limited_range : ++ ilk_csc_off_zero); ++ } else if (crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB) { ++ ilk_update_pipe_csc(crtc, ilk_csc_off_zero, ++ ilk_csc_coeff_rgb_to_ycbcr, ++ ilk_csc_postoff_rgb_to_ycbcr); ++ } else if (limited_color_range) { ++ ilk_update_pipe_csc(crtc, ilk_csc_off_zero, ++ ilk_csc_coeff_limited_range, ++ ilk_csc_postoff_limited_range); ++ } else if (crtc_state->csc_enable) { ++ /* ++ * On GLK+ both pipe CSC and degamma LUT are controlled ++ * by csc_enable. Hence for the cases where the degama ++ * LUT is needed but CSC is not we need to load an ++ * identity matrix. ++ */ ++ WARN_ON(!IS_CANNONLAKE(dev_priv) && !IS_GEMINILAKE(dev_priv)); ++ ++ ilk_update_pipe_csc(crtc, ilk_csc_off_zero, ++ ilk_csc_coeff_identity, ++ ilk_csc_off_zero); ++ } ++ ++ I915_WRITE(PIPE_CSC_MODE(crtc->pipe), crtc_state->csc_mode); ++} ++ ++static void icl_load_csc_matrix(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ ++ if (crtc_state->base.ctm) { ++ u16 coeff[9]; ++ ++ ilk_csc_convert_ctm(crtc_state, coeff); ++ ilk_update_pipe_csc(crtc, ilk_csc_off_zero, ++ coeff, ilk_csc_off_zero); ++ } ++ ++ if (crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB) { ++ icl_update_output_csc(crtc, ilk_csc_off_zero, ++ ilk_csc_coeff_rgb_to_ycbcr, ++ ilk_csc_postoff_rgb_to_ycbcr); ++ } else if (crtc_state->limited_color_range) { ++ icl_update_output_csc(crtc, ilk_csc_off_zero, ++ ilk_csc_coeff_limited_range, ++ ilk_csc_postoff_limited_range); ++ } ++ ++ I915_WRITE(PIPE_CSC_MODE(crtc->pipe), crtc_state->csc_mode); ++} ++ ++/* ++ * Set up the pipe CSC unit on CherryView. ++ */ ++static void cherryview_load_csc_matrix(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ ++ if (crtc_state->base.ctm) { ++ const struct drm_color_ctm *ctm = crtc_state->base.ctm->data; ++ u16 coeffs[9] = {}; ++ int i; ++ ++ for (i = 0; i < ARRAY_SIZE(coeffs); i++) { ++ u64 abs_coeff = ++ ((1ULL << 63) - 1) & ctm->matrix[i]; ++ ++ /* Round coefficient. */ ++ abs_coeff += 1 << (32 - 13); ++ /* Clamp to hardware limits. */ ++ abs_coeff = clamp_val(abs_coeff, 0, CTM_COEFF_8_0 - 1); ++ ++ /* Write coefficients in S3.12 format. */ ++ if (ctm->matrix[i] & (1ULL << 63)) ++ coeffs[i] = 1 << 15; ++ coeffs[i] |= ((abs_coeff >> 32) & 7) << 12; ++ coeffs[i] |= (abs_coeff >> 20) & 0xfff; ++ } ++ ++ I915_WRITE(CGM_PIPE_CSC_COEFF01(pipe), ++ coeffs[1] << 16 | coeffs[0]); ++ I915_WRITE(CGM_PIPE_CSC_COEFF23(pipe), ++ coeffs[3] << 16 | coeffs[2]); ++ I915_WRITE(CGM_PIPE_CSC_COEFF45(pipe), ++ coeffs[5] << 16 | coeffs[4]); ++ I915_WRITE(CGM_PIPE_CSC_COEFF67(pipe), ++ coeffs[7] << 16 | coeffs[6]); ++ I915_WRITE(CGM_PIPE_CSC_COEFF8(pipe), coeffs[8]); ++ } ++ ++ I915_WRITE(CGM_PIPE_MODE(pipe), crtc_state->cgm_mode); ++} ++ ++/* i965+ "10.6" bit interpolated format "even DW" (low 8 bits) */ ++static u32 i965_lut_10p6_ldw(const struct drm_color_lut *color) ++{ ++ return (color->red & 0xff) << 16 | ++ (color->green & 0xff) << 8 | ++ (color->blue & 0xff); ++} ++ ++/* i965+ "10.6" interpolated format "odd DW" (high 8 bits) */ ++static u32 i965_lut_10p6_udw(const struct drm_color_lut *color) ++{ ++ return (color->red >> 8) << 16 | ++ (color->green >> 8) << 8 | ++ (color->blue >> 8); ++} ++ ++static u32 ilk_lut_10(const struct drm_color_lut *color) ++{ ++ return drm_color_lut_extract(color->red, 10) << 20 | ++ drm_color_lut_extract(color->green, 10) << 10 | ++ drm_color_lut_extract(color->blue, 10); ++} ++ ++/* Loads the legacy palette/gamma unit for the CRTC. */ ++static void i9xx_load_luts_internal(const struct intel_crtc_state *crtc_state, ++ const struct drm_property_blob *blob) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ int i; ++ ++ if (HAS_GMCH(dev_priv)) { ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI)) ++ assert_dsi_pll_enabled(dev_priv); ++ else ++ assert_pll_enabled(dev_priv, pipe); ++ } ++ ++ if (blob) { ++ const struct drm_color_lut *lut = blob->data; ++ ++ for (i = 0; i < 256; i++) { ++ u32 word = ++ (drm_color_lut_extract(lut[i].red, 8) << 16) | ++ (drm_color_lut_extract(lut[i].green, 8) << 8) | ++ drm_color_lut_extract(lut[i].blue, 8); ++ ++ if (HAS_GMCH(dev_priv)) ++ I915_WRITE(PALETTE(pipe, i), word); ++ else ++ I915_WRITE(LGC_PALETTE(pipe, i), word); ++ } ++ } ++} ++ ++static void i9xx_load_luts(const struct intel_crtc_state *crtc_state) ++{ ++ i9xx_load_luts_internal(crtc_state, crtc_state->base.gamma_lut); ++} ++ ++static void i9xx_color_commit(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ u32 val; ++ ++ val = I915_READ(PIPECONF(pipe)); ++ val &= ~PIPECONF_GAMMA_MODE_MASK_I9XX; ++ val |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode); ++ I915_WRITE(PIPECONF(pipe), val); ++} ++ ++static void ilk_color_commit(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ u32 val; ++ ++ val = I915_READ(PIPECONF(pipe)); ++ val &= ~PIPECONF_GAMMA_MODE_MASK_ILK; ++ val |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode); ++ I915_WRITE(PIPECONF(pipe), val); ++ ++ ilk_load_csc_matrix(crtc_state); ++} ++ ++static void hsw_color_commit(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ ++ I915_WRITE(GAMMA_MODE(crtc->pipe), crtc_state->gamma_mode); ++ ++ ilk_load_csc_matrix(crtc_state); ++} ++ ++static void skl_color_commit(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ u32 val = 0; ++ ++ /* ++ * We don't (yet) allow userspace to control the pipe background color, ++ * so force it to black, but apply pipe gamma and CSC appropriately ++ * so that its handling will match how we program our planes. ++ */ ++ if (crtc_state->gamma_enable) ++ val |= SKL_BOTTOM_COLOR_GAMMA_ENABLE; ++ if (crtc_state->csc_enable) ++ val |= SKL_BOTTOM_COLOR_CSC_ENABLE; ++ I915_WRITE(SKL_BOTTOM_COLOR(pipe), val); ++ ++ I915_WRITE(GAMMA_MODE(crtc->pipe), crtc_state->gamma_mode); ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ icl_load_csc_matrix(crtc_state); ++ else ++ ilk_load_csc_matrix(crtc_state); ++} ++ ++static void i965_load_lut_10p6(struct intel_crtc *crtc, ++ const struct drm_property_blob *blob) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ const struct drm_color_lut *lut = blob->data; ++ int i, lut_size = drm_color_lut_size(blob); ++ enum pipe pipe = crtc->pipe; ++ ++ for (i = 0; i < lut_size - 1; i++) { ++ I915_WRITE(PALETTE(pipe, 2 * i + 0), ++ i965_lut_10p6_ldw(&lut[i])); ++ I915_WRITE(PALETTE(pipe, 2 * i + 1), ++ i965_lut_10p6_udw(&lut[i])); ++ } ++ ++ I915_WRITE(PIPEGCMAX(pipe, 0), lut[i].red); ++ I915_WRITE(PIPEGCMAX(pipe, 1), lut[i].green); ++ I915_WRITE(PIPEGCMAX(pipe, 2), lut[i].blue); ++} ++ ++static void i965_load_luts(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ const struct drm_property_blob *gamma_lut = crtc_state->base.gamma_lut; ++ ++ if (crtc_state->gamma_mode == GAMMA_MODE_MODE_8BIT) ++ i9xx_load_luts(crtc_state); ++ else ++ i965_load_lut_10p6(crtc, gamma_lut); ++} ++ ++static void ilk_load_lut_10(struct intel_crtc *crtc, ++ const struct drm_property_blob *blob) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ const struct drm_color_lut *lut = blob->data; ++ int i, lut_size = drm_color_lut_size(blob); ++ enum pipe pipe = crtc->pipe; ++ ++ for (i = 0; i < lut_size; i++) ++ I915_WRITE(PREC_PALETTE(pipe, i), ilk_lut_10(&lut[i])); ++} ++ ++static void ilk_load_luts(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ const struct drm_property_blob *gamma_lut = crtc_state->base.gamma_lut; ++ ++ if (crtc_state->gamma_mode == GAMMA_MODE_MODE_8BIT) ++ i9xx_load_luts(crtc_state); ++ else ++ ilk_load_lut_10(crtc, gamma_lut); ++} ++ ++static int ivb_lut_10_size(u32 prec_index) ++{ ++ if (prec_index & PAL_PREC_SPLIT_MODE) ++ return 512; ++ else ++ return 1024; ++} ++ ++/* ++ * IVB/HSW Bspec / PAL_PREC_INDEX: ++ * "Restriction : Index auto increment mode is not ++ * supported and must not be enabled." ++ */ ++static void ivb_load_lut_10(struct intel_crtc *crtc, ++ const struct drm_property_blob *blob, ++ u32 prec_index) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ int hw_lut_size = ivb_lut_10_size(prec_index); ++ const struct drm_color_lut *lut = blob->data; ++ int i, lut_size = drm_color_lut_size(blob); ++ enum pipe pipe = crtc->pipe; ++ ++ for (i = 0; i < hw_lut_size; i++) { ++ /* We discard half the user entries in split gamma mode */ ++ const struct drm_color_lut *entry = ++ &lut[i * (lut_size - 1) / (hw_lut_size - 1)]; ++ ++ I915_WRITE(PREC_PAL_INDEX(pipe), prec_index++); ++ I915_WRITE(PREC_PAL_DATA(pipe), ilk_lut_10(entry)); ++ } ++ ++ /* ++ * Reset the index, otherwise it prevents the legacy palette to be ++ * written properly. ++ */ ++ I915_WRITE(PREC_PAL_INDEX(pipe), 0); ++} ++ ++/* On BDW+ the index auto increment mode actually works */ ++static void bdw_load_lut_10(struct intel_crtc *crtc, ++ const struct drm_property_blob *blob, ++ u32 prec_index) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ int hw_lut_size = ivb_lut_10_size(prec_index); ++ const struct drm_color_lut *lut = blob->data; ++ int i, lut_size = drm_color_lut_size(blob); ++ enum pipe pipe = crtc->pipe; ++ ++ I915_WRITE(PREC_PAL_INDEX(pipe), prec_index | ++ PAL_PREC_AUTO_INCREMENT); ++ ++ for (i = 0; i < hw_lut_size; i++) { ++ /* We discard half the user entries in split gamma mode */ ++ const struct drm_color_lut *entry = ++ &lut[i * (lut_size - 1) / (hw_lut_size - 1)]; ++ ++ I915_WRITE(PREC_PAL_DATA(pipe), ilk_lut_10(entry)); ++ } ++ ++ /* ++ * Reset the index, otherwise it prevents the legacy palette to be ++ * written properly. ++ */ ++ I915_WRITE(PREC_PAL_INDEX(pipe), 0); ++} ++ ++static void ivb_load_lut_10_max(struct intel_crtc *crtc) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ ++ /* Program the max register to clamp values > 1.0. */ ++ I915_WRITE(PREC_PAL_EXT_GC_MAX(pipe, 0), 1 << 16); ++ I915_WRITE(PREC_PAL_EXT_GC_MAX(pipe, 1), 1 << 16); ++ I915_WRITE(PREC_PAL_EXT_GC_MAX(pipe, 2), 1 << 16); ++ ++ /* ++ * Program the gc max 2 register to clamp values > 1.0. ++ * ToDo: Extend the ABI to be able to program values ++ * from 3.0 to 7.0 ++ */ ++ if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) { ++ I915_WRITE(PREC_PAL_EXT2_GC_MAX(pipe, 0), 1 << 16); ++ I915_WRITE(PREC_PAL_EXT2_GC_MAX(pipe, 1), 1 << 16); ++ I915_WRITE(PREC_PAL_EXT2_GC_MAX(pipe, 2), 1 << 16); ++ } ++} ++ ++static void ivb_load_luts(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ const struct drm_property_blob *gamma_lut = crtc_state->base.gamma_lut; ++ const struct drm_property_blob *degamma_lut = crtc_state->base.degamma_lut; ++ ++ if (crtc_state->gamma_mode == GAMMA_MODE_MODE_8BIT) { ++ i9xx_load_luts(crtc_state); ++ } else if (crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT) { ++ ivb_load_lut_10(crtc, degamma_lut, PAL_PREC_SPLIT_MODE | ++ PAL_PREC_INDEX_VALUE(0)); ++ ivb_load_lut_10_max(crtc); ++ ivb_load_lut_10(crtc, gamma_lut, PAL_PREC_SPLIT_MODE | ++ PAL_PREC_INDEX_VALUE(512)); ++ } else { ++ const struct drm_property_blob *blob = gamma_lut ?: degamma_lut; ++ ++ ivb_load_lut_10(crtc, blob, ++ PAL_PREC_INDEX_VALUE(0)); ++ ivb_load_lut_10_max(crtc); ++ } ++} ++ ++static void bdw_load_luts(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ const struct drm_property_blob *gamma_lut = crtc_state->base.gamma_lut; ++ const struct drm_property_blob *degamma_lut = crtc_state->base.degamma_lut; ++ ++ if (crtc_state->gamma_mode == GAMMA_MODE_MODE_8BIT) { ++ i9xx_load_luts(crtc_state); ++ } else if (crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT) { ++ bdw_load_lut_10(crtc, degamma_lut, PAL_PREC_SPLIT_MODE | ++ PAL_PREC_INDEX_VALUE(0)); ++ ivb_load_lut_10_max(crtc); ++ bdw_load_lut_10(crtc, gamma_lut, PAL_PREC_SPLIT_MODE | ++ PAL_PREC_INDEX_VALUE(512)); ++ } else { ++ const struct drm_property_blob *blob = gamma_lut ?: degamma_lut; ++ ++ bdw_load_lut_10(crtc, blob, ++ PAL_PREC_INDEX_VALUE(0)); ++ ivb_load_lut_10_max(crtc); ++ } ++} ++ ++static void glk_load_degamma_lut(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ const u32 lut_size = INTEL_INFO(dev_priv)->color.degamma_lut_size; ++ const struct drm_color_lut *lut = crtc_state->base.degamma_lut->data; ++ u32 i; ++ ++ /* ++ * When setting the auto-increment bit, the hardware seems to ++ * ignore the index bits, so we need to reset it to index 0 ++ * separately. ++ */ ++ I915_WRITE(PRE_CSC_GAMC_INDEX(pipe), 0); ++ I915_WRITE(PRE_CSC_GAMC_INDEX(pipe), PRE_CSC_GAMC_AUTO_INCREMENT); ++ ++ for (i = 0; i < lut_size; i++) { ++ /* ++ * First 33 entries represent range from 0 to 1.0 ++ * 34th and 35th entry will represent extended range ++ * inputs 3.0 and 7.0 respectively, currently clamped ++ * at 1.0. Since the precision is 16bit, the user ++ * value can be directly filled to register. ++ * The pipe degamma table in GLK+ onwards doesn't ++ * support different values per channel, so this just ++ * programs green value which will be equal to Red and ++ * Blue into the lut registers. ++ * ToDo: Extend to max 7.0. Enable 32 bit input value ++ * as compared to just 16 to achieve this. ++ */ ++ I915_WRITE(PRE_CSC_GAMC_DATA(pipe), lut[i].green); ++ } ++ ++ /* Clamp values > 1.0. */ ++ while (i++ < 35) ++ I915_WRITE(PRE_CSC_GAMC_DATA(pipe), 1 << 16); ++} ++ ++static void glk_load_degamma_lut_linear(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ const u32 lut_size = INTEL_INFO(dev_priv)->color.degamma_lut_size; ++ u32 i; ++ ++ /* ++ * When setting the auto-increment bit, the hardware seems to ++ * ignore the index bits, so we need to reset it to index 0 ++ * separately. ++ */ ++ I915_WRITE(PRE_CSC_GAMC_INDEX(pipe), 0); ++ I915_WRITE(PRE_CSC_GAMC_INDEX(pipe), PRE_CSC_GAMC_AUTO_INCREMENT); ++ ++ for (i = 0; i < lut_size; i++) { ++ u32 v = (i << 16) / (lut_size - 1); ++ ++ I915_WRITE(PRE_CSC_GAMC_DATA(pipe), v); ++ } ++ ++ /* Clamp values > 1.0. */ ++ while (i++ < 35) ++ I915_WRITE(PRE_CSC_GAMC_DATA(pipe), 1 << 16); ++} ++ ++static void glk_load_luts(const struct intel_crtc_state *crtc_state) ++{ ++ const struct drm_property_blob *gamma_lut = crtc_state->base.gamma_lut; ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ ++ /* ++ * On GLK+ both pipe CSC and degamma LUT are controlled ++ * by csc_enable. Hence for the cases where the CSC is ++ * needed but degamma LUT is not we need to load a ++ * linear degamma LUT. In fact we'll just always load ++ * the degama LUT so that we don't have to reload ++ * it every time the pipe CSC is being enabled. ++ */ ++ if (crtc_state->base.degamma_lut) ++ glk_load_degamma_lut(crtc_state); ++ else ++ glk_load_degamma_lut_linear(crtc_state); ++ ++ if (crtc_state->gamma_mode == GAMMA_MODE_MODE_8BIT) { ++ i9xx_load_luts(crtc_state); ++ } else { ++ bdw_load_lut_10(crtc, gamma_lut, PAL_PREC_INDEX_VALUE(0)); ++ ivb_load_lut_10_max(crtc); ++ } ++} ++ ++static void icl_load_luts(const struct intel_crtc_state *crtc_state) ++{ ++ const struct drm_property_blob *gamma_lut = crtc_state->base.gamma_lut; ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ ++ if (crtc_state->base.degamma_lut) ++ glk_load_degamma_lut(crtc_state); ++ ++ if ((crtc_state->gamma_mode & GAMMA_MODE_MODE_MASK) == ++ GAMMA_MODE_MODE_8BIT) { ++ i9xx_load_luts(crtc_state); ++ } else { ++ bdw_load_lut_10(crtc, gamma_lut, PAL_PREC_INDEX_VALUE(0)); ++ ivb_load_lut_10_max(crtc); ++ } ++} ++ ++static void cherryview_load_luts(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ const struct drm_property_blob *gamma_lut = crtc_state->base.gamma_lut; ++ const struct drm_property_blob *degamma_lut = crtc_state->base.degamma_lut; ++ enum pipe pipe = crtc->pipe; ++ ++ cherryview_load_csc_matrix(crtc_state); ++ ++ if (crtc_state_is_legacy_gamma(crtc_state)) { ++ i9xx_load_luts(crtc_state); ++ return; ++ } ++ ++ if (degamma_lut) { ++ const struct drm_color_lut *lut = degamma_lut->data; ++ int i, lut_size = INTEL_INFO(dev_priv)->color.degamma_lut_size; ++ ++ for (i = 0; i < lut_size; i++) { ++ u32 word0, word1; ++ ++ /* Write LUT in U0.14 format. */ ++ word0 = ++ (drm_color_lut_extract(lut[i].green, 14) << 16) | ++ drm_color_lut_extract(lut[i].blue, 14); ++ word1 = drm_color_lut_extract(lut[i].red, 14); ++ ++ I915_WRITE(CGM_PIPE_DEGAMMA(pipe, i, 0), word0); ++ I915_WRITE(CGM_PIPE_DEGAMMA(pipe, i, 1), word1); ++ } ++ } ++ ++ if (gamma_lut) { ++ const struct drm_color_lut *lut = gamma_lut->data; ++ int i, lut_size = INTEL_INFO(dev_priv)->color.gamma_lut_size; ++ ++ for (i = 0; i < lut_size; i++) { ++ u32 word0, word1; ++ ++ /* Write LUT in U0.10 format. */ ++ word0 = ++ (drm_color_lut_extract(lut[i].green, 10) << 16) | ++ drm_color_lut_extract(lut[i].blue, 10); ++ word1 = drm_color_lut_extract(lut[i].red, 10); ++ ++ I915_WRITE(CGM_PIPE_GAMMA(pipe, i, 0), word0); ++ I915_WRITE(CGM_PIPE_GAMMA(pipe, i, 1), word1); ++ } ++ } ++} ++ ++void intel_color_load_luts(const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); ++ ++ dev_priv->display.load_luts(crtc_state); ++} ++ ++void intel_color_commit(const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); ++ ++ dev_priv->display.color_commit(crtc_state); ++} ++ ++int intel_color_check(struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); ++ ++ return dev_priv->display.color_check(crtc_state); ++} ++ ++static bool need_plane_update(struct intel_plane *plane, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(plane->base.dev); ++ ++ /* ++ * On pre-SKL the pipe gamma enable and pipe csc enable for ++ * the pipe bottom color are configured via the primary plane. ++ * We have to reconfigure that even if the plane is inactive. ++ */ ++ return crtc_state->active_planes & BIT(plane->id) || ++ (INTEL_GEN(dev_priv) < 9 && ++ plane->id == PLANE_PRIMARY); ++} ++ ++static int ++intel_color_add_affected_planes(struct intel_crtc_state *new_crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ struct intel_atomic_state *state = ++ to_intel_atomic_state(new_crtc_state->base.state); ++ const struct intel_crtc_state *old_crtc_state = ++ intel_atomic_get_old_crtc_state(state, crtc); ++ struct intel_plane *plane; ++ ++ if (!new_crtc_state->base.active || ++ drm_atomic_crtc_needs_modeset(&new_crtc_state->base)) ++ return 0; ++ ++ if (new_crtc_state->gamma_enable == old_crtc_state->gamma_enable && ++ new_crtc_state->csc_enable == old_crtc_state->csc_enable) ++ return 0; ++ ++ for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) { ++ struct intel_plane_state *plane_state; ++ ++ if (!need_plane_update(plane, new_crtc_state)) ++ continue; ++ ++ plane_state = intel_atomic_get_plane_state(state, plane); ++ if (IS_ERR(plane_state)) ++ return PTR_ERR(plane_state); ++ ++ new_crtc_state->update_planes |= BIT(plane->id); ++ } ++ ++ return 0; ++} ++ ++static int check_lut_size(const struct drm_property_blob *lut, int expected) ++{ ++ int len; ++ ++ if (!lut) ++ return 0; ++ ++ len = drm_color_lut_size(lut); ++ if (len != expected) { ++ DRM_DEBUG_KMS("Invalid LUT size; got %d, expected %d\n", ++ len, expected); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static int check_luts(const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); ++ const struct drm_property_blob *gamma_lut = crtc_state->base.gamma_lut; ++ const struct drm_property_blob *degamma_lut = crtc_state->base.degamma_lut; ++ int gamma_length, degamma_length; ++ u32 gamma_tests, degamma_tests; ++ ++ /* Always allow legacy gamma LUT with no further checking. */ ++ if (crtc_state_is_legacy_gamma(crtc_state)) ++ return 0; ++ ++ /* C8 relies on its palette being stored in the legacy LUT */ ++ if (crtc_state->c8_planes) ++ return -EINVAL; ++ ++ degamma_length = INTEL_INFO(dev_priv)->color.degamma_lut_size; ++ gamma_length = INTEL_INFO(dev_priv)->color.gamma_lut_size; ++ degamma_tests = INTEL_INFO(dev_priv)->color.degamma_lut_tests; ++ gamma_tests = INTEL_INFO(dev_priv)->color.gamma_lut_tests; ++ ++ if (check_lut_size(degamma_lut, degamma_length) || ++ check_lut_size(gamma_lut, gamma_length)) ++ return -EINVAL; ++ ++ if (drm_color_lut_check(degamma_lut, degamma_tests) || ++ drm_color_lut_check(gamma_lut, gamma_tests)) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++static u32 i9xx_gamma_mode(struct intel_crtc_state *crtc_state) ++{ ++ if (!crtc_state->gamma_enable || ++ crtc_state_is_legacy_gamma(crtc_state)) ++ return GAMMA_MODE_MODE_8BIT; ++ else ++ return GAMMA_MODE_MODE_10BIT; /* i965+ only */ ++} ++ ++static int i9xx_color_check(struct intel_crtc_state *crtc_state) ++{ ++ int ret; ++ ++ ret = check_luts(crtc_state); ++ if (ret) ++ return ret; ++ ++ crtc_state->gamma_enable = ++ crtc_state->base.gamma_lut && ++ !crtc_state->c8_planes; ++ ++ crtc_state->gamma_mode = i9xx_gamma_mode(crtc_state); ++ ++ ret = intel_color_add_affected_planes(crtc_state); ++ if (ret) ++ return ret; ++ ++ return 0; ++} ++ ++static u32 chv_cgm_mode(const struct intel_crtc_state *crtc_state) ++{ ++ u32 cgm_mode = 0; ++ ++ if (crtc_state_is_legacy_gamma(crtc_state)) ++ return 0; ++ ++ if (crtc_state->base.degamma_lut) ++ cgm_mode |= CGM_PIPE_MODE_DEGAMMA; ++ if (crtc_state->base.ctm) ++ cgm_mode |= CGM_PIPE_MODE_CSC; ++ if (crtc_state->base.gamma_lut) ++ cgm_mode |= CGM_PIPE_MODE_GAMMA; ++ ++ return cgm_mode; ++} ++ ++/* ++ * CHV color pipeline: ++ * u0.10 -> CGM degamma -> u0.14 -> CGM csc -> u0.14 -> CGM gamma -> ++ * u0.10 -> WGC csc -> u0.10 -> pipe gamma -> u0.10 ++ * ++ * We always bypass the WGC csc and use the CGM csc ++ * instead since it has degamma and better precision. ++ */ ++static int chv_color_check(struct intel_crtc_state *crtc_state) ++{ ++ int ret; ++ ++ ret = check_luts(crtc_state); ++ if (ret) ++ return ret; ++ ++ /* ++ * Pipe gamma will be used only for the legacy LUT. ++ * Otherwise we bypass it and use the CGM gamma instead. ++ */ ++ crtc_state->gamma_enable = ++ crtc_state_is_legacy_gamma(crtc_state) && ++ !crtc_state->c8_planes; ++ ++ crtc_state->gamma_mode = GAMMA_MODE_MODE_8BIT; ++ ++ crtc_state->cgm_mode = chv_cgm_mode(crtc_state); ++ ++ ret = intel_color_add_affected_planes(crtc_state); ++ if (ret) ++ return ret; ++ ++ return 0; ++} ++ ++static u32 ilk_gamma_mode(const struct intel_crtc_state *crtc_state) ++{ ++ if (!crtc_state->gamma_enable || ++ crtc_state_is_legacy_gamma(crtc_state)) ++ return GAMMA_MODE_MODE_8BIT; ++ else ++ return GAMMA_MODE_MODE_10BIT; ++} ++ ++static int ilk_color_check(struct intel_crtc_state *crtc_state) ++{ ++ int ret; ++ ++ ret = check_luts(crtc_state); ++ if (ret) ++ return ret; ++ ++ crtc_state->gamma_enable = ++ crtc_state->base.gamma_lut && ++ !crtc_state->c8_planes; ++ ++ /* ++ * We don't expose the ctm on ilk/snb currently, ++ * nor do we enable YCbCr output. Also RGB limited ++ * range output is handled by the hw automagically. ++ */ ++ crtc_state->csc_enable = false; ++ ++ crtc_state->gamma_mode = ilk_gamma_mode(crtc_state); ++ ++ crtc_state->csc_mode = 0; ++ ++ ret = intel_color_add_affected_planes(crtc_state); ++ if (ret) ++ return ret; ++ ++ return 0; ++} ++ ++static u32 ivb_gamma_mode(const struct intel_crtc_state *crtc_state) ++{ ++ if (!crtc_state->gamma_enable || ++ crtc_state_is_legacy_gamma(crtc_state)) ++ return GAMMA_MODE_MODE_8BIT; ++ else if (crtc_state->base.gamma_lut && ++ crtc_state->base.degamma_lut) ++ return GAMMA_MODE_MODE_SPLIT; ++ else ++ return GAMMA_MODE_MODE_10BIT; ++} ++ ++static u32 ivb_csc_mode(const struct intel_crtc_state *crtc_state) ++{ ++ bool limited_color_range = ilk_csc_limited_range(crtc_state); ++ ++ /* ++ * CSC comes after the LUT in degamma, RGB->YCbCr, ++ * and RGB full->limited range mode. ++ */ ++ if (crtc_state->base.degamma_lut || ++ crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB || ++ limited_color_range) ++ return 0; ++ ++ return CSC_POSITION_BEFORE_GAMMA; ++} ++ ++static int ivb_color_check(struct intel_crtc_state *crtc_state) ++{ ++ bool limited_color_range = ilk_csc_limited_range(crtc_state); ++ int ret; ++ ++ ret = check_luts(crtc_state); ++ if (ret) ++ return ret; ++ ++ crtc_state->gamma_enable = ++ (crtc_state->base.gamma_lut || ++ crtc_state->base.degamma_lut) && ++ !crtc_state->c8_planes; ++ ++ crtc_state->csc_enable = ++ crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB || ++ crtc_state->base.ctm || limited_color_range; ++ ++ crtc_state->gamma_mode = ivb_gamma_mode(crtc_state); ++ ++ crtc_state->csc_mode = ivb_csc_mode(crtc_state); ++ ++ ret = intel_color_add_affected_planes(crtc_state); ++ if (ret) ++ return ret; ++ ++ return 0; ++} ++ ++static u32 glk_gamma_mode(const struct intel_crtc_state *crtc_state) ++{ ++ if (!crtc_state->gamma_enable || ++ crtc_state_is_legacy_gamma(crtc_state)) ++ return GAMMA_MODE_MODE_8BIT; ++ else ++ return GAMMA_MODE_MODE_10BIT; ++} ++ ++static int glk_color_check(struct intel_crtc_state *crtc_state) ++{ ++ int ret; ++ ++ ret = check_luts(crtc_state); ++ if (ret) ++ return ret; ++ ++ crtc_state->gamma_enable = ++ crtc_state->base.gamma_lut && ++ !crtc_state->c8_planes; ++ ++ /* On GLK+ degamma LUT is controlled by csc_enable */ ++ crtc_state->csc_enable = ++ crtc_state->base.degamma_lut || ++ crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB || ++ crtc_state->base.ctm || crtc_state->limited_color_range; ++ ++ crtc_state->gamma_mode = glk_gamma_mode(crtc_state); ++ ++ crtc_state->csc_mode = 0; ++ ++ ret = intel_color_add_affected_planes(crtc_state); ++ if (ret) ++ return ret; ++ ++ return 0; ++} ++ ++static u32 icl_gamma_mode(const struct intel_crtc_state *crtc_state) ++{ ++ u32 gamma_mode = 0; ++ ++ if (crtc_state->base.degamma_lut) ++ gamma_mode |= PRE_CSC_GAMMA_ENABLE; ++ ++ if (crtc_state->base.gamma_lut && ++ !crtc_state->c8_planes) ++ gamma_mode |= POST_CSC_GAMMA_ENABLE; ++ ++ if (!crtc_state->base.gamma_lut || ++ crtc_state_is_legacy_gamma(crtc_state)) ++ gamma_mode |= GAMMA_MODE_MODE_8BIT; ++ else ++ gamma_mode |= GAMMA_MODE_MODE_10BIT; ++ ++ return gamma_mode; ++} ++ ++static u32 icl_csc_mode(const struct intel_crtc_state *crtc_state) ++{ ++ u32 csc_mode = 0; ++ ++ if (crtc_state->base.ctm) ++ csc_mode |= ICL_CSC_ENABLE; ++ ++ if (crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB || ++ crtc_state->limited_color_range) ++ csc_mode |= ICL_OUTPUT_CSC_ENABLE; ++ ++ return csc_mode; ++} ++ ++static int icl_color_check(struct intel_crtc_state *crtc_state) ++{ ++ int ret; ++ ++ ret = check_luts(crtc_state); ++ if (ret) ++ return ret; ++ ++ crtc_state->gamma_mode = icl_gamma_mode(crtc_state); ++ ++ crtc_state->csc_mode = icl_csc_mode(crtc_state); ++ ++ return 0; ++} ++ ++void intel_color_init(struct intel_crtc *crtc) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ bool has_ctm = INTEL_INFO(dev_priv)->color.degamma_lut_size != 0; ++ ++ drm_mode_crtc_set_gamma_size(&crtc->base, 256); ++ ++ if (HAS_GMCH(dev_priv)) { ++ if (IS_CHERRYVIEW(dev_priv)) { ++ dev_priv->display.color_check = chv_color_check; ++ dev_priv->display.color_commit = i9xx_color_commit; ++ dev_priv->display.load_luts = cherryview_load_luts; ++ } else if (INTEL_GEN(dev_priv) >= 4) { ++ dev_priv->display.color_check = i9xx_color_check; ++ dev_priv->display.color_commit = i9xx_color_commit; ++ dev_priv->display.load_luts = i965_load_luts; ++ } else { ++ dev_priv->display.color_check = i9xx_color_check; ++ dev_priv->display.color_commit = i9xx_color_commit; ++ dev_priv->display.load_luts = i9xx_load_luts; ++ } ++ } else { ++ if (INTEL_GEN(dev_priv) >= 11) ++ dev_priv->display.color_check = icl_color_check; ++ else if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) ++ dev_priv->display.color_check = glk_color_check; ++ else if (INTEL_GEN(dev_priv) >= 7) ++ dev_priv->display.color_check = ivb_color_check; ++ else ++ dev_priv->display.color_check = ilk_color_check; ++ ++ if (INTEL_GEN(dev_priv) >= 9) ++ dev_priv->display.color_commit = skl_color_commit; ++ else if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) ++ dev_priv->display.color_commit = hsw_color_commit; ++ else ++ dev_priv->display.color_commit = ilk_color_commit; ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ dev_priv->display.load_luts = icl_load_luts; ++ else if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv)) ++ dev_priv->display.load_luts = glk_load_luts; ++ else if (INTEL_GEN(dev_priv) >= 8) ++ dev_priv->display.load_luts = bdw_load_luts; ++ else if (INTEL_GEN(dev_priv) >= 7) ++ dev_priv->display.load_luts = ivb_load_luts; ++ else ++ dev_priv->display.load_luts = ilk_load_luts; ++ } ++ ++ drm_crtc_enable_color_mgmt(&crtc->base, ++ INTEL_INFO(dev_priv)->color.degamma_lut_size, ++ has_ctm, ++ INTEL_INFO(dev_priv)->color.gamma_lut_size); ++} +diff --git a/drivers/gpu/drm/i915_legacy/intel_color.h b/drivers/gpu/drm/i915_legacy/intel_color.h +new file mode 100644 +index 000000000000..b8a3ce609587 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_color.h +@@ -0,0 +1,17 @@ ++/* SPDX-License-Identifier: MIT */ ++/* ++ * Copyright © 2019 Intel Corporation ++ */ ++ ++#ifndef __INTEL_COLOR_H__ ++#define __INTEL_COLOR_H__ ++ ++struct intel_crtc_state; ++struct intel_crtc; ++ ++void intel_color_init(struct intel_crtc *crtc); ++int intel_color_check(struct intel_crtc_state *crtc_state); ++void intel_color_commit(const struct intel_crtc_state *crtc_state); ++void intel_color_load_luts(const struct intel_crtc_state *crtc_state); ++ ++#endif /* __INTEL_COLOR_H__ */ +diff --git a/drivers/gpu/drm/i915_legacy/intel_combo_phy.c b/drivers/gpu/drm/i915_legacy/intel_combo_phy.c +new file mode 100644 +index 000000000000..2bf4359d7e41 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_combo_phy.c +@@ -0,0 +1,255 @@ ++// SPDX-License-Identifier: MIT ++/* ++ * Copyright © 2018 Intel Corporation ++ */ ++ ++#include "intel_drv.h" ++ ++#define for_each_combo_port(__dev_priv, __port) \ ++ for ((__port) = PORT_A; (__port) < I915_MAX_PORTS; (__port)++) \ ++ for_each_if(intel_port_is_combophy(__dev_priv, __port)) ++ ++#define for_each_combo_port_reverse(__dev_priv, __port) \ ++ for ((__port) = I915_MAX_PORTS; (__port)-- > PORT_A;) \ ++ for_each_if(intel_port_is_combophy(__dev_priv, __port)) ++ ++enum { ++ PROCMON_0_85V_DOT_0, ++ PROCMON_0_95V_DOT_0, ++ PROCMON_0_95V_DOT_1, ++ PROCMON_1_05V_DOT_0, ++ PROCMON_1_05V_DOT_1, ++}; ++ ++static const struct cnl_procmon { ++ u32 dw1, dw9, dw10; ++} cnl_procmon_values[] = { ++ [PROCMON_0_85V_DOT_0] = ++ { .dw1 = 0x00000000, .dw9 = 0x62AB67BB, .dw10 = 0x51914F96, }, ++ [PROCMON_0_95V_DOT_0] = ++ { .dw1 = 0x00000000, .dw9 = 0x86E172C7, .dw10 = 0x77CA5EAB, }, ++ [PROCMON_0_95V_DOT_1] = ++ { .dw1 = 0x00000000, .dw9 = 0x93F87FE1, .dw10 = 0x8AE871C5, }, ++ [PROCMON_1_05V_DOT_0] = ++ { .dw1 = 0x00000000, .dw9 = 0x98FA82DD, .dw10 = 0x89E46DC1, }, ++ [PROCMON_1_05V_DOT_1] = ++ { .dw1 = 0x00440000, .dw9 = 0x9A00AB25, .dw10 = 0x8AE38FF1, }, ++}; ++ ++/* ++ * CNL has just one set of registers, while ICL has two sets: one for port A and ++ * the other for port B. The CNL registers are equivalent to the ICL port A ++ * registers, that's why we call the ICL macros even though the function has CNL ++ * on its name. ++ */ ++static const struct cnl_procmon * ++cnl_get_procmon_ref_values(struct drm_i915_private *dev_priv, enum port port) ++{ ++ const struct cnl_procmon *procmon; ++ u32 val; ++ ++ val = I915_READ(ICL_PORT_COMP_DW3(port)); ++ switch (val & (PROCESS_INFO_MASK | VOLTAGE_INFO_MASK)) { ++ default: ++ MISSING_CASE(val); ++ /* fall through */ ++ case VOLTAGE_INFO_0_85V | PROCESS_INFO_DOT_0: ++ procmon = &cnl_procmon_values[PROCMON_0_85V_DOT_0]; ++ break; ++ case VOLTAGE_INFO_0_95V | PROCESS_INFO_DOT_0: ++ procmon = &cnl_procmon_values[PROCMON_0_95V_DOT_0]; ++ break; ++ case VOLTAGE_INFO_0_95V | PROCESS_INFO_DOT_1: ++ procmon = &cnl_procmon_values[PROCMON_0_95V_DOT_1]; ++ break; ++ case VOLTAGE_INFO_1_05V | PROCESS_INFO_DOT_0: ++ procmon = &cnl_procmon_values[PROCMON_1_05V_DOT_0]; ++ break; ++ case VOLTAGE_INFO_1_05V | PROCESS_INFO_DOT_1: ++ procmon = &cnl_procmon_values[PROCMON_1_05V_DOT_1]; ++ break; ++ } ++ ++ return procmon; ++} ++ ++static void cnl_set_procmon_ref_values(struct drm_i915_private *dev_priv, ++ enum port port) ++{ ++ const struct cnl_procmon *procmon; ++ u32 val; ++ ++ procmon = cnl_get_procmon_ref_values(dev_priv, port); ++ ++ val = I915_READ(ICL_PORT_COMP_DW1(port)); ++ val &= ~((0xff << 16) | 0xff); ++ val |= procmon->dw1; ++ I915_WRITE(ICL_PORT_COMP_DW1(port), val); ++ ++ I915_WRITE(ICL_PORT_COMP_DW9(port), procmon->dw9); ++ I915_WRITE(ICL_PORT_COMP_DW10(port), procmon->dw10); ++} ++ ++static bool check_phy_reg(struct drm_i915_private *dev_priv, ++ enum port port, i915_reg_t reg, u32 mask, ++ u32 expected_val) ++{ ++ u32 val = I915_READ(reg); ++ ++ if ((val & mask) != expected_val) { ++ DRM_DEBUG_DRIVER("Port %c combo PHY reg %08x state mismatch: " ++ "current %08x mask %08x expected %08x\n", ++ port_name(port), ++ reg.reg, val, mask, expected_val); ++ return false; ++ } ++ ++ return true; ++} ++ ++static bool cnl_verify_procmon_ref_values(struct drm_i915_private *dev_priv, ++ enum port port) ++{ ++ const struct cnl_procmon *procmon; ++ bool ret; ++ ++ procmon = cnl_get_procmon_ref_values(dev_priv, port); ++ ++ ret = check_phy_reg(dev_priv, port, ICL_PORT_COMP_DW1(port), ++ (0xff << 16) | 0xff, procmon->dw1); ++ ret &= check_phy_reg(dev_priv, port, ICL_PORT_COMP_DW9(port), ++ -1U, procmon->dw9); ++ ret &= check_phy_reg(dev_priv, port, ICL_PORT_COMP_DW10(port), ++ -1U, procmon->dw10); ++ ++ return ret; ++} ++ ++static bool cnl_combo_phy_enabled(struct drm_i915_private *dev_priv) ++{ ++ return !(I915_READ(CHICKEN_MISC_2) & CNL_COMP_PWR_DOWN) && ++ (I915_READ(CNL_PORT_COMP_DW0) & COMP_INIT); ++} ++ ++static bool cnl_combo_phy_verify_state(struct drm_i915_private *dev_priv) ++{ ++ enum port port = PORT_A; ++ bool ret; ++ ++ if (!cnl_combo_phy_enabled(dev_priv)) ++ return false; ++ ++ ret = cnl_verify_procmon_ref_values(dev_priv, port); ++ ++ ret &= check_phy_reg(dev_priv, port, CNL_PORT_CL1CM_DW5, ++ CL_POWER_DOWN_ENABLE, CL_POWER_DOWN_ENABLE); ++ ++ return ret; ++} ++ ++void cnl_combo_phys_init(struct drm_i915_private *dev_priv) ++{ ++ u32 val; ++ ++ val = I915_READ(CHICKEN_MISC_2); ++ val &= ~CNL_COMP_PWR_DOWN; ++ I915_WRITE(CHICKEN_MISC_2, val); ++ ++ /* Dummy PORT_A to get the correct CNL register from the ICL macro */ ++ cnl_set_procmon_ref_values(dev_priv, PORT_A); ++ ++ val = I915_READ(CNL_PORT_COMP_DW0); ++ val |= COMP_INIT; ++ I915_WRITE(CNL_PORT_COMP_DW0, val); ++ ++ val = I915_READ(CNL_PORT_CL1CM_DW5); ++ val |= CL_POWER_DOWN_ENABLE; ++ I915_WRITE(CNL_PORT_CL1CM_DW5, val); ++} ++ ++void cnl_combo_phys_uninit(struct drm_i915_private *dev_priv) ++{ ++ u32 val; ++ ++ if (!cnl_combo_phy_verify_state(dev_priv)) ++ DRM_WARN("Combo PHY HW state changed unexpectedly.\n"); ++ ++ val = I915_READ(CHICKEN_MISC_2); ++ val |= CNL_COMP_PWR_DOWN; ++ I915_WRITE(CHICKEN_MISC_2, val); ++} ++ ++static bool icl_combo_phy_enabled(struct drm_i915_private *dev_priv, ++ enum port port) ++{ ++ return !(I915_READ(ICL_PHY_MISC(port)) & ++ ICL_PHY_MISC_DE_IO_COMP_PWR_DOWN) && ++ (I915_READ(ICL_PORT_COMP_DW0(port)) & COMP_INIT); ++} ++ ++static bool icl_combo_phy_verify_state(struct drm_i915_private *dev_priv, ++ enum port port) ++{ ++ bool ret; ++ ++ if (!icl_combo_phy_enabled(dev_priv, port)) ++ return false; ++ ++ ret = cnl_verify_procmon_ref_values(dev_priv, port); ++ ++ ret &= check_phy_reg(dev_priv, port, ICL_PORT_CL_DW5(port), ++ CL_POWER_DOWN_ENABLE, CL_POWER_DOWN_ENABLE); ++ ++ return ret; ++} ++ ++void icl_combo_phys_init(struct drm_i915_private *dev_priv) ++{ ++ enum port port; ++ ++ for_each_combo_port(dev_priv, port) { ++ u32 val; ++ ++ if (icl_combo_phy_verify_state(dev_priv, port)) { ++ DRM_DEBUG_DRIVER("Port %c combo PHY already enabled, won't reprogram it.\n", ++ port_name(port)); ++ continue; ++ } ++ ++ val = I915_READ(ICL_PHY_MISC(port)); ++ val &= ~ICL_PHY_MISC_DE_IO_COMP_PWR_DOWN; ++ I915_WRITE(ICL_PHY_MISC(port), val); ++ ++ cnl_set_procmon_ref_values(dev_priv, port); ++ ++ val = I915_READ(ICL_PORT_COMP_DW0(port)); ++ val |= COMP_INIT; ++ I915_WRITE(ICL_PORT_COMP_DW0(port), val); ++ ++ val = I915_READ(ICL_PORT_CL_DW5(port)); ++ val |= CL_POWER_DOWN_ENABLE; ++ I915_WRITE(ICL_PORT_CL_DW5(port), val); ++ } ++} ++ ++void icl_combo_phys_uninit(struct drm_i915_private *dev_priv) ++{ ++ enum port port; ++ ++ for_each_combo_port_reverse(dev_priv, port) { ++ u32 val; ++ ++ if (port == PORT_A && ++ !icl_combo_phy_verify_state(dev_priv, port)) ++ DRM_WARN("Port %c combo PHY HW state changed unexpectedly\n", ++ port_name(port)); ++ ++ val = I915_READ(ICL_PHY_MISC(port)); ++ val |= ICL_PHY_MISC_DE_IO_COMP_PWR_DOWN; ++ I915_WRITE(ICL_PHY_MISC(port), val); ++ ++ val = I915_READ(ICL_PORT_COMP_DW0(port)); ++ val &= ~COMP_INIT; ++ I915_WRITE(ICL_PORT_COMP_DW0(port), val); ++ } ++} +diff --git a/drivers/gpu/drm/i915_legacy/intel_connector.c b/drivers/gpu/drm/i915_legacy/intel_connector.c +new file mode 100644 +index 000000000000..073b6c3ab7cc +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_connector.c +@@ -0,0 +1,282 @@ ++/* ++ * Copyright (c) 2007 Dave Airlie ++ * Copyright (c) 2007, 2010 Intel Corporation ++ * Jesse Barnes ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER ++ * DEALINGS IN THE SOFTWARE. ++ */ ++ ++#include ++#include ++ ++#include ++#include ++ ++#include "i915_drv.h" ++#include "intel_connector.h" ++#include "intel_drv.h" ++#include "intel_hdcp.h" ++#include "intel_panel.h" ++ ++int intel_connector_init(struct intel_connector *connector) ++{ ++ struct intel_digital_connector_state *conn_state; ++ ++ /* ++ * Allocate enough memory to hold intel_digital_connector_state, ++ * This might be a few bytes too many, but for connectors that don't ++ * need it we'll free the state and allocate a smaller one on the first ++ * successful commit anyway. ++ */ ++ conn_state = kzalloc(sizeof(*conn_state), GFP_KERNEL); ++ if (!conn_state) ++ return -ENOMEM; ++ ++ __drm_atomic_helper_connector_reset(&connector->base, ++ &conn_state->base); ++ ++ return 0; ++} ++ ++struct intel_connector *intel_connector_alloc(void) ++{ ++ struct intel_connector *connector; ++ ++ connector = kzalloc(sizeof(*connector), GFP_KERNEL); ++ if (!connector) ++ return NULL; ++ ++ if (intel_connector_init(connector) < 0) { ++ kfree(connector); ++ return NULL; ++ } ++ ++ return connector; ++} ++ ++/* ++ * Free the bits allocated by intel_connector_alloc. ++ * This should only be used after intel_connector_alloc has returned ++ * successfully, and before drm_connector_init returns successfully. ++ * Otherwise the destroy callbacks for the connector and the state should ++ * take care of proper cleanup/free (see intel_connector_destroy). ++ */ ++void intel_connector_free(struct intel_connector *connector) ++{ ++ kfree(to_intel_digital_connector_state(connector->base.state)); ++ kfree(connector); ++} ++ ++/* ++ * Connector type independent destroy hook for drm_connector_funcs. ++ */ ++void intel_connector_destroy(struct drm_connector *connector) ++{ ++ struct intel_connector *intel_connector = to_intel_connector(connector); ++ ++ kfree(intel_connector->detect_edid); ++ ++ intel_hdcp_cleanup(intel_connector); ++ ++ if (!IS_ERR_OR_NULL(intel_connector->edid)) ++ kfree(intel_connector->edid); ++ ++ intel_panel_fini(&intel_connector->panel); ++ ++ drm_connector_cleanup(connector); ++ ++ if (intel_connector->port) ++ drm_dp_mst_put_port_malloc(intel_connector->port); ++ ++ kfree(connector); ++} ++ ++int intel_connector_register(struct drm_connector *connector) ++{ ++ struct intel_connector *intel_connector = to_intel_connector(connector); ++ int ret; ++ ++ ret = intel_backlight_device_register(intel_connector); ++ if (ret) ++ goto err; ++ ++ if (i915_inject_load_failure()) { ++ ret = -EFAULT; ++ goto err_backlight; ++ } ++ ++ return 0; ++ ++err_backlight: ++ intel_backlight_device_unregister(intel_connector); ++err: ++ return ret; ++} ++ ++void intel_connector_unregister(struct drm_connector *connector) ++{ ++ struct intel_connector *intel_connector = to_intel_connector(connector); ++ ++ intel_backlight_device_unregister(intel_connector); ++} ++ ++void intel_connector_attach_encoder(struct intel_connector *connector, ++ struct intel_encoder *encoder) ++{ ++ connector->encoder = encoder; ++ drm_connector_attach_encoder(&connector->base, &encoder->base); ++} ++ ++/* ++ * Simple connector->get_hw_state implementation for encoders that support only ++ * one connector and no cloning and hence the encoder state determines the state ++ * of the connector. ++ */ ++bool intel_connector_get_hw_state(struct intel_connector *connector) ++{ ++ enum pipe pipe = 0; ++ struct intel_encoder *encoder = connector->encoder; ++ ++ return encoder->get_hw_state(encoder, &pipe); ++} ++ ++enum pipe intel_connector_get_pipe(struct intel_connector *connector) ++{ ++ struct drm_device *dev = connector->base.dev; ++ ++ WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex)); ++ ++ if (!connector->base.state->crtc) ++ return INVALID_PIPE; ++ ++ return to_intel_crtc(connector->base.state->crtc)->pipe; ++} ++ ++/** ++ * intel_connector_update_modes - update connector from edid ++ * @connector: DRM connector device to use ++ * @edid: previously read EDID information ++ */ ++int intel_connector_update_modes(struct drm_connector *connector, ++ struct edid *edid) ++{ ++ int ret; ++ ++ drm_connector_update_edid_property(connector, edid); ++ ret = drm_add_edid_modes(connector, edid); ++ ++ return ret; ++} ++ ++/** ++ * intel_ddc_get_modes - get modelist from monitor ++ * @connector: DRM connector device to use ++ * @adapter: i2c adapter ++ * ++ * Fetch the EDID information from @connector using the DDC bus. ++ */ ++int intel_ddc_get_modes(struct drm_connector *connector, ++ struct i2c_adapter *adapter) ++{ ++ struct edid *edid; ++ int ret; ++ ++ edid = drm_get_edid(connector, adapter); ++ if (!edid) ++ return 0; ++ ++ ret = intel_connector_update_modes(connector, edid); ++ kfree(edid); ++ ++ return ret; ++} ++ ++static const struct drm_prop_enum_list force_audio_names[] = { ++ { HDMI_AUDIO_OFF_DVI, "force-dvi" }, ++ { HDMI_AUDIO_OFF, "off" }, ++ { HDMI_AUDIO_AUTO, "auto" }, ++ { HDMI_AUDIO_ON, "on" }, ++}; ++ ++void ++intel_attach_force_audio_property(struct drm_connector *connector) ++{ ++ struct drm_device *dev = connector->dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct drm_property *prop; ++ ++ prop = dev_priv->force_audio_property; ++ if (prop == NULL) { ++ prop = drm_property_create_enum(dev, 0, ++ "audio", ++ force_audio_names, ++ ARRAY_SIZE(force_audio_names)); ++ if (prop == NULL) ++ return; ++ ++ dev_priv->force_audio_property = prop; ++ } ++ drm_object_attach_property(&connector->base, prop, 0); ++} ++ ++static const struct drm_prop_enum_list broadcast_rgb_names[] = { ++ { INTEL_BROADCAST_RGB_AUTO, "Automatic" }, ++ { INTEL_BROADCAST_RGB_FULL, "Full" }, ++ { INTEL_BROADCAST_RGB_LIMITED, "Limited 16:235" }, ++}; ++ ++void ++intel_attach_broadcast_rgb_property(struct drm_connector *connector) ++{ ++ struct drm_device *dev = connector->dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct drm_property *prop; ++ ++ prop = dev_priv->broadcast_rgb_property; ++ if (prop == NULL) { ++ prop = drm_property_create_enum(dev, DRM_MODE_PROP_ENUM, ++ "Broadcast RGB", ++ broadcast_rgb_names, ++ ARRAY_SIZE(broadcast_rgb_names)); ++ if (prop == NULL) ++ return; ++ ++ dev_priv->broadcast_rgb_property = prop; ++ } ++ ++ drm_object_attach_property(&connector->base, prop, 0); ++} ++ ++void ++intel_attach_aspect_ratio_property(struct drm_connector *connector) ++{ ++ if (!drm_mode_create_aspect_ratio_property(connector->dev)) ++ drm_object_attach_property(&connector->base, ++ connector->dev->mode_config.aspect_ratio_property, ++ DRM_MODE_PICTURE_ASPECT_NONE); ++} ++ ++void ++intel_attach_colorspace_property(struct drm_connector *connector) ++{ ++ if (!drm_mode_create_colorspace_property(connector)) ++ drm_object_attach_property(&connector->base, ++ connector->colorspace_property, 0); ++} +diff --git a/drivers/gpu/drm/i915_legacy/intel_connector.h b/drivers/gpu/drm/i915_legacy/intel_connector.h +new file mode 100644 +index 000000000000..93a7375c8196 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_connector.h +@@ -0,0 +1,35 @@ ++/* SPDX-License-Identifier: MIT */ ++/* ++ * Copyright © 2019 Intel Corporation ++ */ ++ ++#ifndef __INTEL_CONNECTOR_H__ ++#define __INTEL_CONNECTOR_H__ ++ ++#include "intel_display.h" ++ ++struct drm_connector; ++struct edid; ++struct i2c_adapter; ++struct intel_connector; ++struct intel_encoder; ++ ++int intel_connector_init(struct intel_connector *connector); ++struct intel_connector *intel_connector_alloc(void); ++void intel_connector_free(struct intel_connector *connector); ++void intel_connector_destroy(struct drm_connector *connector); ++int intel_connector_register(struct drm_connector *connector); ++void intel_connector_unregister(struct drm_connector *connector); ++void intel_connector_attach_encoder(struct intel_connector *connector, ++ struct intel_encoder *encoder); ++bool intel_connector_get_hw_state(struct intel_connector *connector); ++enum pipe intel_connector_get_pipe(struct intel_connector *connector); ++int intel_connector_update_modes(struct drm_connector *connector, ++ struct edid *edid); ++int intel_ddc_get_modes(struct drm_connector *c, struct i2c_adapter *adapter); ++void intel_attach_force_audio_property(struct drm_connector *connector); ++void intel_attach_broadcast_rgb_property(struct drm_connector *connector); ++void intel_attach_aspect_ratio_property(struct drm_connector *connector); ++void intel_attach_colorspace_property(struct drm_connector *connector); ++ ++#endif /* __INTEL_CONNECTOR_H__ */ +diff --git a/drivers/gpu/drm/i915_legacy/intel_context.c b/drivers/gpu/drm/i915_legacy/intel_context.c +new file mode 100644 +index 000000000000..8931e0fee873 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_context.c +@@ -0,0 +1,269 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2019 Intel Corporation ++ */ ++ ++#include "i915_drv.h" ++#include "i915_gem_context.h" ++#include "i915_globals.h" ++#include "intel_context.h" ++#include "intel_ringbuffer.h" ++ ++static struct i915_global_context { ++ struct i915_global base; ++ struct kmem_cache *slab_ce; ++} global; ++ ++struct intel_context *intel_context_alloc(void) ++{ ++ return kmem_cache_zalloc(global.slab_ce, GFP_KERNEL); ++} ++ ++void intel_context_free(struct intel_context *ce) ++{ ++ kmem_cache_free(global.slab_ce, ce); ++} ++ ++struct intel_context * ++intel_context_lookup(struct i915_gem_context *ctx, ++ struct intel_engine_cs *engine) ++{ ++ struct intel_context *ce = NULL; ++ struct rb_node *p; ++ ++ spin_lock(&ctx->hw_contexts_lock); ++ p = ctx->hw_contexts.rb_node; ++ while (p) { ++ struct intel_context *this = ++ rb_entry(p, struct intel_context, node); ++ ++ if (this->engine == engine) { ++ GEM_BUG_ON(this->gem_context != ctx); ++ ce = this; ++ break; ++ } ++ ++ if (this->engine < engine) ++ p = p->rb_right; ++ else ++ p = p->rb_left; ++ } ++ spin_unlock(&ctx->hw_contexts_lock); ++ ++ return ce; ++} ++ ++struct intel_context * ++__intel_context_insert(struct i915_gem_context *ctx, ++ struct intel_engine_cs *engine, ++ struct intel_context *ce) ++{ ++ struct rb_node **p, *parent; ++ int err = 0; ++ ++ spin_lock(&ctx->hw_contexts_lock); ++ ++ parent = NULL; ++ p = &ctx->hw_contexts.rb_node; ++ while (*p) { ++ struct intel_context *this; ++ ++ parent = *p; ++ this = rb_entry(parent, struct intel_context, node); ++ ++ if (this->engine == engine) { ++ err = -EEXIST; ++ ce = this; ++ break; ++ } ++ ++ if (this->engine < engine) ++ p = &parent->rb_right; ++ else ++ p = &parent->rb_left; ++ } ++ if (!err) { ++ rb_link_node(&ce->node, parent, p); ++ rb_insert_color(&ce->node, &ctx->hw_contexts); ++ } ++ ++ spin_unlock(&ctx->hw_contexts_lock); ++ ++ return ce; ++} ++ ++void __intel_context_remove(struct intel_context *ce) ++{ ++ struct i915_gem_context *ctx = ce->gem_context; ++ ++ spin_lock(&ctx->hw_contexts_lock); ++ rb_erase(&ce->node, &ctx->hw_contexts); ++ spin_unlock(&ctx->hw_contexts_lock); ++} ++ ++static struct intel_context * ++intel_context_instance(struct i915_gem_context *ctx, ++ struct intel_engine_cs *engine) ++{ ++ struct intel_context *ce, *pos; ++ ++ ce = intel_context_lookup(ctx, engine); ++ if (likely(ce)) ++ return ce; ++ ++ ce = intel_context_alloc(); ++ if (!ce) ++ return ERR_PTR(-ENOMEM); ++ ++ intel_context_init(ce, ctx, engine); ++ ++ pos = __intel_context_insert(ctx, engine, ce); ++ if (unlikely(pos != ce)) /* Beaten! Use their HW context instead */ ++ intel_context_free(ce); ++ ++ GEM_BUG_ON(intel_context_lookup(ctx, engine) != pos); ++ return pos; ++} ++ ++struct intel_context * ++intel_context_pin_lock(struct i915_gem_context *ctx, ++ struct intel_engine_cs *engine) ++ __acquires(ce->pin_mutex) ++{ ++ struct intel_context *ce; ++ ++ ce = intel_context_instance(ctx, engine); ++ if (IS_ERR(ce)) ++ return ce; ++ ++ if (mutex_lock_interruptible(&ce->pin_mutex)) ++ return ERR_PTR(-EINTR); ++ ++ return ce; ++} ++ ++struct intel_context * ++intel_context_pin(struct i915_gem_context *ctx, ++ struct intel_engine_cs *engine) ++{ ++ struct intel_context *ce; ++ int err; ++ ++ ce = intel_context_instance(ctx, engine); ++ if (IS_ERR(ce)) ++ return ce; ++ ++ if (likely(atomic_inc_not_zero(&ce->pin_count))) ++ return ce; ++ ++ if (mutex_lock_interruptible(&ce->pin_mutex)) ++ return ERR_PTR(-EINTR); ++ ++ if (likely(!atomic_read(&ce->pin_count))) { ++ err = ce->ops->pin(ce); ++ if (err) ++ goto err; ++ ++ i915_gem_context_get(ctx); ++ GEM_BUG_ON(ce->gem_context != ctx); ++ ++ mutex_lock(&ctx->mutex); ++ list_add(&ce->active_link, &ctx->active_engines); ++ mutex_unlock(&ctx->mutex); ++ ++ intel_context_get(ce); ++ smp_mb__before_atomic(); /* flush pin before it is visible */ ++ } ++ ++ atomic_inc(&ce->pin_count); ++ GEM_BUG_ON(!intel_context_is_pinned(ce)); /* no overflow! */ ++ ++ mutex_unlock(&ce->pin_mutex); ++ return ce; ++ ++err: ++ mutex_unlock(&ce->pin_mutex); ++ return ERR_PTR(err); ++} ++ ++void intel_context_unpin(struct intel_context *ce) ++{ ++ if (likely(atomic_add_unless(&ce->pin_count, -1, 1))) ++ return; ++ ++ /* We may be called from inside intel_context_pin() to evict another */ ++ intel_context_get(ce); ++ mutex_lock_nested(&ce->pin_mutex, SINGLE_DEPTH_NESTING); ++ ++ if (likely(atomic_dec_and_test(&ce->pin_count))) { ++ ce->ops->unpin(ce); ++ ++ mutex_lock(&ce->gem_context->mutex); ++ list_del(&ce->active_link); ++ mutex_unlock(&ce->gem_context->mutex); ++ ++ i915_gem_context_put(ce->gem_context); ++ intel_context_put(ce); ++ } ++ ++ mutex_unlock(&ce->pin_mutex); ++ intel_context_put(ce); ++} ++ ++static void intel_context_retire(struct i915_active_request *active, ++ struct i915_request *rq) ++{ ++ struct intel_context *ce = ++ container_of(active, typeof(*ce), active_tracker); ++ ++ intel_context_unpin(ce); ++} ++ ++void ++intel_context_init(struct intel_context *ce, ++ struct i915_gem_context *ctx, ++ struct intel_engine_cs *engine) ++{ ++ kref_init(&ce->ref); ++ ++ ce->gem_context = ctx; ++ ce->engine = engine; ++ ce->ops = engine->cops; ++ ++ INIT_LIST_HEAD(&ce->signal_link); ++ INIT_LIST_HEAD(&ce->signals); ++ ++ mutex_init(&ce->pin_mutex); ++ ++ /* Use the whole device by default */ ++ ce->sseu = intel_device_default_sseu(ctx->i915); ++ ++ i915_active_request_init(&ce->active_tracker, ++ NULL, intel_context_retire); ++} ++ ++static void i915_global_context_shrink(void) ++{ ++ kmem_cache_shrink(global.slab_ce); ++} ++ ++static void i915_global_context_exit(void) ++{ ++ kmem_cache_destroy(global.slab_ce); ++} ++ ++static struct i915_global_context global = { { ++ .shrink = i915_global_context_shrink, ++ .exit = i915_global_context_exit, ++} }; ++ ++int __init i915_global_context_init(void) ++{ ++ global.slab_ce = KMEM_CACHE(intel_context, SLAB_HWCACHE_ALIGN); ++ if (!global.slab_ce) ++ return -ENOMEM; ++ ++ i915_global_register(&global.base); ++ return 0; ++} +diff --git a/drivers/gpu/drm/i915_legacy/intel_context.h b/drivers/gpu/drm/i915_legacy/intel_context.h +new file mode 100644 +index 000000000000..ebc861b1a49e +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_context.h +@@ -0,0 +1,87 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2019 Intel Corporation ++ */ ++ ++#ifndef __INTEL_CONTEXT_H__ ++#define __INTEL_CONTEXT_H__ ++ ++#include ++ ++#include "intel_context_types.h" ++#include "intel_engine_types.h" ++ ++struct intel_context *intel_context_alloc(void); ++void intel_context_free(struct intel_context *ce); ++ ++void intel_context_init(struct intel_context *ce, ++ struct i915_gem_context *ctx, ++ struct intel_engine_cs *engine); ++ ++/** ++ * intel_context_lookup - Find the matching HW context for this (ctx, engine) ++ * @ctx - the parent GEM context ++ * @engine - the target HW engine ++ * ++ * May return NULL if the HW context hasn't been instantiated (i.e. unused). ++ */ ++struct intel_context * ++intel_context_lookup(struct i915_gem_context *ctx, ++ struct intel_engine_cs *engine); ++ ++/** ++ * intel_context_pin_lock - Stablises the 'pinned' status of the HW context ++ * @ctx - the parent GEM context ++ * @engine - the target HW engine ++ * ++ * Acquire a lock on the pinned status of the HW context, such that the context ++ * can neither be bound to the GPU or unbound whilst the lock is held, i.e. ++ * intel_context_is_pinned() remains stable. ++ */ ++struct intel_context * ++intel_context_pin_lock(struct i915_gem_context *ctx, ++ struct intel_engine_cs *engine); ++ ++static inline bool ++intel_context_is_pinned(struct intel_context *ce) ++{ ++ return atomic_read(&ce->pin_count); ++} ++ ++static inline void intel_context_pin_unlock(struct intel_context *ce) ++__releases(ce->pin_mutex) ++{ ++ mutex_unlock(&ce->pin_mutex); ++} ++ ++struct intel_context * ++__intel_context_insert(struct i915_gem_context *ctx, ++ struct intel_engine_cs *engine, ++ struct intel_context *ce); ++void ++__intel_context_remove(struct intel_context *ce); ++ ++struct intel_context * ++intel_context_pin(struct i915_gem_context *ctx, struct intel_engine_cs *engine); ++ ++static inline void __intel_context_pin(struct intel_context *ce) ++{ ++ GEM_BUG_ON(!intel_context_is_pinned(ce)); ++ atomic_inc(&ce->pin_count); ++} ++ ++void intel_context_unpin(struct intel_context *ce); ++ ++static inline struct intel_context *intel_context_get(struct intel_context *ce) ++{ ++ kref_get(&ce->ref); ++ return ce; ++} ++ ++static inline void intel_context_put(struct intel_context *ce) ++{ ++ kref_put(&ce->ref, ce->ops->destroy); ++} ++ ++#endif /* __INTEL_CONTEXT_H__ */ +diff --git a/drivers/gpu/drm/i915_legacy/intel_context_types.h b/drivers/gpu/drm/i915_legacy/intel_context_types.h +new file mode 100644 +index 000000000000..fd47b9d49e09 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_context_types.h +@@ -0,0 +1,75 @@ ++/* ++ * SPDX-License-Identifier: MIT ++ * ++ * Copyright © 2019 Intel Corporation ++ */ ++ ++#ifndef __INTEL_CONTEXT_TYPES__ ++#define __INTEL_CONTEXT_TYPES__ ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "i915_active_types.h" ++#include "intel_engine_types.h" ++ ++struct i915_gem_context; ++struct i915_vma; ++struct intel_context; ++struct intel_ring; ++ ++struct intel_context_ops { ++ int (*pin)(struct intel_context *ce); ++ void (*unpin)(struct intel_context *ce); ++ ++ void (*reset)(struct intel_context *ce); ++ void (*destroy)(struct kref *kref); ++}; ++ ++/* ++ * Powergating configuration for a particular (context,engine). ++ */ ++struct intel_sseu { ++ u8 slice_mask; ++ u8 subslice_mask; ++ u8 min_eus_per_subslice; ++ u8 max_eus_per_subslice; ++}; ++ ++struct intel_context { ++ struct kref ref; ++ ++ struct i915_gem_context *gem_context; ++ struct intel_engine_cs *engine; ++ struct intel_engine_cs *active; ++ ++ struct list_head active_link; ++ struct list_head signal_link; ++ struct list_head signals; ++ ++ struct i915_vma *state; ++ struct intel_ring *ring; ++ ++ u32 *lrc_reg_state; ++ u64 lrc_desc; ++ ++ atomic_t pin_count; ++ struct mutex pin_mutex; /* guards pinning and associated on-gpuing */ ++ ++ /** ++ * active_tracker: Active tracker for the external rq activity ++ * on this intel_context object. ++ */ ++ struct i915_active_request active_tracker; ++ ++ const struct intel_context_ops *ops; ++ struct rb_node node; ++ ++ /** sseu: Control eu/slice partitioning */ ++ struct intel_sseu sseu; ++}; ++ ++#endif /* __INTEL_CONTEXT_TYPES__ */ +diff --git a/drivers/gpu/drm/i915_legacy/intel_crt.c b/drivers/gpu/drm/i915_legacy/intel_crt.c +new file mode 100644 +index 000000000000..b665c370111b +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_crt.c +@@ -0,0 +1,1061 @@ ++/* ++ * Copyright © 2006-2007 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER ++ * DEALINGS IN THE SOFTWARE. ++ * ++ * Authors: ++ * Eric Anholt ++ */ ++ ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "i915_drv.h" ++#include "intel_connector.h" ++#include "intel_crt.h" ++#include "intel_ddi.h" ++#include "intel_drv.h" ++ ++/* Here's the desired hotplug mode */ ++#define ADPA_HOTPLUG_BITS (ADPA_CRT_HOTPLUG_PERIOD_128 | \ ++ ADPA_CRT_HOTPLUG_WARMUP_10MS | \ ++ ADPA_CRT_HOTPLUG_SAMPLE_4S | \ ++ ADPA_CRT_HOTPLUG_VOLTAGE_50 | \ ++ ADPA_CRT_HOTPLUG_VOLREF_325MV | \ ++ ADPA_CRT_HOTPLUG_ENABLE) ++ ++struct intel_crt { ++ struct intel_encoder base; ++ /* DPMS state is stored in the connector, which we need in the ++ * encoder's enable/disable callbacks */ ++ struct intel_connector *connector; ++ bool force_hotplug_required; ++ i915_reg_t adpa_reg; ++}; ++ ++static struct intel_crt *intel_encoder_to_crt(struct intel_encoder *encoder) ++{ ++ return container_of(encoder, struct intel_crt, base); ++} ++ ++static struct intel_crt *intel_attached_crt(struct drm_connector *connector) ++{ ++ return intel_encoder_to_crt(intel_attached_encoder(connector)); ++} ++ ++bool intel_crt_port_enabled(struct drm_i915_private *dev_priv, ++ i915_reg_t adpa_reg, enum pipe *pipe) ++{ ++ u32 val; ++ ++ val = I915_READ(adpa_reg); ++ ++ /* asserts want to know the pipe even if the port is disabled */ ++ if (HAS_PCH_CPT(dev_priv)) ++ *pipe = (val & ADPA_PIPE_SEL_MASK_CPT) >> ADPA_PIPE_SEL_SHIFT_CPT; ++ else ++ *pipe = (val & ADPA_PIPE_SEL_MASK) >> ADPA_PIPE_SEL_SHIFT; ++ ++ return val & ADPA_DAC_ENABLE; ++} ++ ++static bool intel_crt_get_hw_state(struct intel_encoder *encoder, ++ enum pipe *pipe) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_crt *crt = intel_encoder_to_crt(encoder); ++ intel_wakeref_t wakeref; ++ bool ret; ++ ++ wakeref = intel_display_power_get_if_enabled(dev_priv, ++ encoder->power_domain); ++ if (!wakeref) ++ return false; ++ ++ ret = intel_crt_port_enabled(dev_priv, crt->adpa_reg, pipe); ++ ++ intel_display_power_put(dev_priv, encoder->power_domain, wakeref); ++ ++ return ret; ++} ++ ++static unsigned int intel_crt_get_flags(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_crt *crt = intel_encoder_to_crt(encoder); ++ u32 tmp, flags = 0; ++ ++ tmp = I915_READ(crt->adpa_reg); ++ ++ if (tmp & ADPA_HSYNC_ACTIVE_HIGH) ++ flags |= DRM_MODE_FLAG_PHSYNC; ++ else ++ flags |= DRM_MODE_FLAG_NHSYNC; ++ ++ if (tmp & ADPA_VSYNC_ACTIVE_HIGH) ++ flags |= DRM_MODE_FLAG_PVSYNC; ++ else ++ flags |= DRM_MODE_FLAG_NVSYNC; ++ ++ return flags; ++} ++ ++static void intel_crt_get_config(struct intel_encoder *encoder, ++ struct intel_crtc_state *pipe_config) ++{ ++ pipe_config->output_types |= BIT(INTEL_OUTPUT_ANALOG); ++ ++ pipe_config->base.adjusted_mode.flags |= intel_crt_get_flags(encoder); ++ ++ pipe_config->base.adjusted_mode.crtc_clock = pipe_config->port_clock; ++} ++ ++static void hsw_crt_get_config(struct intel_encoder *encoder, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ ++ intel_ddi_get_config(encoder, pipe_config); ++ ++ pipe_config->base.adjusted_mode.flags &= ~(DRM_MODE_FLAG_PHSYNC | ++ DRM_MODE_FLAG_NHSYNC | ++ DRM_MODE_FLAG_PVSYNC | ++ DRM_MODE_FLAG_NVSYNC); ++ pipe_config->base.adjusted_mode.flags |= intel_crt_get_flags(encoder); ++ ++ pipe_config->base.adjusted_mode.crtc_clock = lpt_get_iclkip(dev_priv); ++} ++ ++/* Note: The caller is required to filter out dpms modes not supported by the ++ * platform. */ ++static void intel_crt_set_dpms(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state, ++ int mode) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_crt *crt = intel_encoder_to_crt(encoder); ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode; ++ u32 adpa; ++ ++ if (INTEL_GEN(dev_priv) >= 5) ++ adpa = ADPA_HOTPLUG_BITS; ++ else ++ adpa = 0; ++ ++ if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC) ++ adpa |= ADPA_HSYNC_ACTIVE_HIGH; ++ if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC) ++ adpa |= ADPA_VSYNC_ACTIVE_HIGH; ++ ++ /* For CPT allow 3 pipe config, for others just use A or B */ ++ if (HAS_PCH_LPT(dev_priv)) ++ ; /* Those bits don't exist here */ ++ else if (HAS_PCH_CPT(dev_priv)) ++ adpa |= ADPA_PIPE_SEL_CPT(crtc->pipe); ++ else ++ adpa |= ADPA_PIPE_SEL(crtc->pipe); ++ ++ if (!HAS_PCH_SPLIT(dev_priv)) ++ I915_WRITE(BCLRPAT(crtc->pipe), 0); ++ ++ switch (mode) { ++ case DRM_MODE_DPMS_ON: ++ adpa |= ADPA_DAC_ENABLE; ++ break; ++ case DRM_MODE_DPMS_STANDBY: ++ adpa |= ADPA_DAC_ENABLE | ADPA_HSYNC_CNTL_DISABLE; ++ break; ++ case DRM_MODE_DPMS_SUSPEND: ++ adpa |= ADPA_DAC_ENABLE | ADPA_VSYNC_CNTL_DISABLE; ++ break; ++ case DRM_MODE_DPMS_OFF: ++ adpa |= ADPA_HSYNC_CNTL_DISABLE | ADPA_VSYNC_CNTL_DISABLE; ++ break; ++ } ++ ++ I915_WRITE(crt->adpa_reg, adpa); ++} ++ ++static void intel_disable_crt(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ intel_crt_set_dpms(encoder, old_crtc_state, DRM_MODE_DPMS_OFF); ++} ++ ++static void pch_disable_crt(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++} ++ ++static void pch_post_disable_crt(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ intel_disable_crt(encoder, old_crtc_state, old_conn_state); ++} ++ ++static void hsw_disable_crt(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ ++ WARN_ON(!old_crtc_state->has_pch_encoder); ++ ++ intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, false); ++} ++ ++static void hsw_post_disable_crt(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ ++ intel_ddi_disable_pipe_clock(old_crtc_state); ++ ++ pch_post_disable_crt(encoder, old_crtc_state, old_conn_state); ++ ++ lpt_disable_pch_transcoder(dev_priv); ++ lpt_disable_iclkip(dev_priv); ++ ++ intel_ddi_fdi_post_disable(encoder, old_crtc_state, old_conn_state); ++ ++ WARN_ON(!old_crtc_state->has_pch_encoder); ++ ++ intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true); ++} ++ ++static void hsw_pre_pll_enable_crt(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ ++ WARN_ON(!crtc_state->has_pch_encoder); ++ ++ intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, false); ++} ++ ++static void hsw_pre_enable_crt(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ enum pipe pipe = crtc->pipe; ++ ++ WARN_ON(!crtc_state->has_pch_encoder); ++ ++ intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false); ++ ++ dev_priv->display.fdi_link_train(crtc, crtc_state); ++ ++ intel_ddi_enable_pipe_clock(crtc_state); ++} ++ ++static void hsw_enable_crt(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ enum pipe pipe = crtc->pipe; ++ ++ WARN_ON(!crtc_state->has_pch_encoder); ++ ++ intel_crt_set_dpms(encoder, crtc_state, DRM_MODE_DPMS_ON); ++ ++ intel_wait_for_vblank(dev_priv, pipe); ++ intel_wait_for_vblank(dev_priv, pipe); ++ intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); ++ intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true); ++} ++ ++static void intel_enable_crt(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state) ++{ ++ intel_crt_set_dpms(encoder, crtc_state, DRM_MODE_DPMS_ON); ++} ++ ++static enum drm_mode_status ++intel_crt_mode_valid(struct drm_connector *connector, ++ struct drm_display_mode *mode) ++{ ++ struct drm_device *dev = connector->dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ int max_dotclk = dev_priv->max_dotclk_freq; ++ int max_clock; ++ ++ if (mode->flags & DRM_MODE_FLAG_DBLSCAN) ++ return MODE_NO_DBLESCAN; ++ ++ if (mode->clock < 25000) ++ return MODE_CLOCK_LOW; ++ ++ if (HAS_PCH_LPT(dev_priv)) ++ max_clock = 180000; ++ else if (IS_VALLEYVIEW(dev_priv)) ++ /* ++ * 270 MHz due to current DPLL limits, ++ * DAC limit supposedly 355 MHz. ++ */ ++ max_clock = 270000; ++ else if (IS_GEN_RANGE(dev_priv, 3, 4)) ++ max_clock = 400000; ++ else ++ max_clock = 350000; ++ if (mode->clock > max_clock) ++ return MODE_CLOCK_HIGH; ++ ++ if (mode->clock > max_dotclk) ++ return MODE_CLOCK_HIGH; ++ ++ /* The FDI receiver on LPT only supports 8bpc and only has 2 lanes. */ ++ if (HAS_PCH_LPT(dev_priv) && ++ (ironlake_get_lanes_required(mode->clock, 270000, 24) > 2)) ++ return MODE_CLOCK_HIGH; ++ ++ /* HSW/BDW FDI limited to 4k */ ++ if (mode->hdisplay > 4096) ++ return MODE_H_ILLEGAL; ++ ++ return MODE_OK; ++} ++ ++static int intel_crt_compute_config(struct intel_encoder *encoder, ++ struct intel_crtc_state *pipe_config, ++ struct drm_connector_state *conn_state) ++{ ++ struct drm_display_mode *adjusted_mode = ++ &pipe_config->base.adjusted_mode; ++ ++ if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) ++ return -EINVAL; ++ ++ pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; ++ ++ return 0; ++} ++ ++static int pch_crt_compute_config(struct intel_encoder *encoder, ++ struct intel_crtc_state *pipe_config, ++ struct drm_connector_state *conn_state) ++{ ++ struct drm_display_mode *adjusted_mode = ++ &pipe_config->base.adjusted_mode; ++ ++ if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) ++ return -EINVAL; ++ ++ pipe_config->has_pch_encoder = true; ++ pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; ++ ++ return 0; ++} ++ ++static int hsw_crt_compute_config(struct intel_encoder *encoder, ++ struct intel_crtc_state *pipe_config, ++ struct drm_connector_state *conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct drm_display_mode *adjusted_mode = ++ &pipe_config->base.adjusted_mode; ++ ++ if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) ++ return -EINVAL; ++ ++ /* HSW/BDW FDI limited to 4k */ ++ if (adjusted_mode->crtc_hdisplay > 4096 || ++ adjusted_mode->crtc_hblank_start > 4096) ++ return -EINVAL; ++ ++ pipe_config->has_pch_encoder = true; ++ pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; ++ ++ /* LPT FDI RX only supports 8bpc. */ ++ if (HAS_PCH_LPT(dev_priv)) { ++ if (pipe_config->bw_constrained && pipe_config->pipe_bpp < 24) { ++ DRM_DEBUG_KMS("LPT only supports 24bpp\n"); ++ return -EINVAL; ++ } ++ ++ pipe_config->pipe_bpp = 24; ++ } ++ ++ /* FDI must always be 2.7 GHz */ ++ pipe_config->port_clock = 135000 * 2; ++ ++ return 0; ++} ++ ++static bool intel_ironlake_crt_detect_hotplug(struct drm_connector *connector) ++{ ++ struct drm_device *dev = connector->dev; ++ struct intel_crt *crt = intel_attached_crt(connector); ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ u32 adpa; ++ bool ret; ++ ++ /* The first time through, trigger an explicit detection cycle */ ++ if (crt->force_hotplug_required) { ++ bool turn_off_dac = HAS_PCH_SPLIT(dev_priv); ++ u32 save_adpa; ++ ++ crt->force_hotplug_required = 0; ++ ++ save_adpa = adpa = I915_READ(crt->adpa_reg); ++ DRM_DEBUG_KMS("trigger hotplug detect cycle: adpa=0x%x\n", adpa); ++ ++ adpa |= ADPA_CRT_HOTPLUG_FORCE_TRIGGER; ++ if (turn_off_dac) ++ adpa &= ~ADPA_DAC_ENABLE; ++ ++ I915_WRITE(crt->adpa_reg, adpa); ++ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ crt->adpa_reg, ++ ADPA_CRT_HOTPLUG_FORCE_TRIGGER, 0, ++ 1000)) ++ DRM_DEBUG_KMS("timed out waiting for FORCE_TRIGGER"); ++ ++ if (turn_off_dac) { ++ I915_WRITE(crt->adpa_reg, save_adpa); ++ POSTING_READ(crt->adpa_reg); ++ } ++ } ++ ++ /* Check the status to see if both blue and green are on now */ ++ adpa = I915_READ(crt->adpa_reg); ++ if ((adpa & ADPA_CRT_HOTPLUG_MONITOR_MASK) != 0) ++ ret = true; ++ else ++ ret = false; ++ DRM_DEBUG_KMS("ironlake hotplug adpa=0x%x, result %d\n", adpa, ret); ++ ++ return ret; ++} ++ ++static bool valleyview_crt_detect_hotplug(struct drm_connector *connector) ++{ ++ struct drm_device *dev = connector->dev; ++ struct intel_crt *crt = intel_attached_crt(connector); ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ bool reenable_hpd; ++ u32 adpa; ++ bool ret; ++ u32 save_adpa; ++ ++ /* ++ * Doing a force trigger causes a hpd interrupt to get sent, which can ++ * get us stuck in a loop if we're polling: ++ * - We enable power wells and reset the ADPA ++ * - output_poll_exec does force probe on VGA, triggering a hpd ++ * - HPD handler waits for poll to unlock dev->mode_config.mutex ++ * - output_poll_exec shuts off the ADPA, unlocks ++ * dev->mode_config.mutex ++ * - HPD handler runs, resets ADPA and brings us back to the start ++ * ++ * Just disable HPD interrupts here to prevent this ++ */ ++ reenable_hpd = intel_hpd_disable(dev_priv, crt->base.hpd_pin); ++ ++ save_adpa = adpa = I915_READ(crt->adpa_reg); ++ DRM_DEBUG_KMS("trigger hotplug detect cycle: adpa=0x%x\n", adpa); ++ ++ adpa |= ADPA_CRT_HOTPLUG_FORCE_TRIGGER; ++ ++ I915_WRITE(crt->adpa_reg, adpa); ++ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ crt->adpa_reg, ++ ADPA_CRT_HOTPLUG_FORCE_TRIGGER, 0, ++ 1000)) { ++ DRM_DEBUG_KMS("timed out waiting for FORCE_TRIGGER"); ++ I915_WRITE(crt->adpa_reg, save_adpa); ++ } ++ ++ /* Check the status to see if both blue and green are on now */ ++ adpa = I915_READ(crt->adpa_reg); ++ if ((adpa & ADPA_CRT_HOTPLUG_MONITOR_MASK) != 0) ++ ret = true; ++ else ++ ret = false; ++ ++ DRM_DEBUG_KMS("valleyview hotplug adpa=0x%x, result %d\n", adpa, ret); ++ ++ if (reenable_hpd) ++ intel_hpd_enable(dev_priv, crt->base.hpd_pin); ++ ++ return ret; ++} ++ ++static bool intel_crt_detect_hotplug(struct drm_connector *connector) ++{ ++ struct drm_device *dev = connector->dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ u32 stat; ++ bool ret = false; ++ int i, tries = 0; ++ ++ if (HAS_PCH_SPLIT(dev_priv)) ++ return intel_ironlake_crt_detect_hotplug(connector); ++ ++ if (IS_VALLEYVIEW(dev_priv)) ++ return valleyview_crt_detect_hotplug(connector); ++ ++ /* ++ * On 4 series desktop, CRT detect sequence need to be done twice ++ * to get a reliable result. ++ */ ++ ++ if (IS_G45(dev_priv)) ++ tries = 2; ++ else ++ tries = 1; ++ ++ for (i = 0; i < tries ; i++) { ++ /* turn on the FORCE_DETECT */ ++ i915_hotplug_interrupt_update(dev_priv, ++ CRT_HOTPLUG_FORCE_DETECT, ++ CRT_HOTPLUG_FORCE_DETECT); ++ /* wait for FORCE_DETECT to go off */ ++ if (intel_wait_for_register(&dev_priv->uncore, PORT_HOTPLUG_EN, ++ CRT_HOTPLUG_FORCE_DETECT, 0, ++ 1000)) ++ DRM_DEBUG_KMS("timed out waiting for FORCE_DETECT to go off"); ++ } ++ ++ stat = I915_READ(PORT_HOTPLUG_STAT); ++ if ((stat & CRT_HOTPLUG_MONITOR_MASK) != CRT_HOTPLUG_MONITOR_NONE) ++ ret = true; ++ ++ /* clear the interrupt we just generated, if any */ ++ I915_WRITE(PORT_HOTPLUG_STAT, CRT_HOTPLUG_INT_STATUS); ++ ++ i915_hotplug_interrupt_update(dev_priv, CRT_HOTPLUG_FORCE_DETECT, 0); ++ ++ return ret; ++} ++ ++static struct edid *intel_crt_get_edid(struct drm_connector *connector, ++ struct i2c_adapter *i2c) ++{ ++ struct edid *edid; ++ ++ edid = drm_get_edid(connector, i2c); ++ ++ if (!edid && !intel_gmbus_is_forced_bit(i2c)) { ++ DRM_DEBUG_KMS("CRT GMBUS EDID read failed, retry using GPIO bit-banging\n"); ++ intel_gmbus_force_bit(i2c, true); ++ edid = drm_get_edid(connector, i2c); ++ intel_gmbus_force_bit(i2c, false); ++ } ++ ++ return edid; ++} ++ ++/* local version of intel_ddc_get_modes() to use intel_crt_get_edid() */ ++static int intel_crt_ddc_get_modes(struct drm_connector *connector, ++ struct i2c_adapter *adapter) ++{ ++ struct edid *edid; ++ int ret; ++ ++ edid = intel_crt_get_edid(connector, adapter); ++ if (!edid) ++ return 0; ++ ++ ret = intel_connector_update_modes(connector, edid); ++ kfree(edid); ++ ++ return ret; ++} ++ ++static bool intel_crt_detect_ddc(struct drm_connector *connector) ++{ ++ struct intel_crt *crt = intel_attached_crt(connector); ++ struct drm_i915_private *dev_priv = to_i915(crt->base.base.dev); ++ struct edid *edid; ++ struct i2c_adapter *i2c; ++ bool ret = false; ++ ++ BUG_ON(crt->base.type != INTEL_OUTPUT_ANALOG); ++ ++ i2c = intel_gmbus_get_adapter(dev_priv, dev_priv->vbt.crt_ddc_pin); ++ edid = intel_crt_get_edid(connector, i2c); ++ ++ if (edid) { ++ bool is_digital = edid->input & DRM_EDID_INPUT_DIGITAL; ++ ++ /* ++ * This may be a DVI-I connector with a shared DDC ++ * link between analog and digital outputs, so we ++ * have to check the EDID input spec of the attached device. ++ */ ++ if (!is_digital) { ++ DRM_DEBUG_KMS("CRT detected via DDC:0x50 [EDID]\n"); ++ ret = true; ++ } else { ++ DRM_DEBUG_KMS("CRT not detected via DDC:0x50 [EDID reports a digital panel]\n"); ++ } ++ } else { ++ DRM_DEBUG_KMS("CRT not detected via DDC:0x50 [no valid EDID found]\n"); ++ } ++ ++ kfree(edid); ++ ++ return ret; ++} ++ ++static enum drm_connector_status ++intel_crt_load_detect(struct intel_crt *crt, u32 pipe) ++{ ++ struct drm_device *dev = crt->base.base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ u32 save_bclrpat; ++ u32 save_vtotal; ++ u32 vtotal, vactive; ++ u32 vsample; ++ u32 vblank, vblank_start, vblank_end; ++ u32 dsl; ++ i915_reg_t bclrpat_reg, vtotal_reg, ++ vblank_reg, vsync_reg, pipeconf_reg, pipe_dsl_reg; ++ u8 st00; ++ enum drm_connector_status status; ++ ++ DRM_DEBUG_KMS("starting load-detect on CRT\n"); ++ ++ bclrpat_reg = BCLRPAT(pipe); ++ vtotal_reg = VTOTAL(pipe); ++ vblank_reg = VBLANK(pipe); ++ vsync_reg = VSYNC(pipe); ++ pipeconf_reg = PIPECONF(pipe); ++ pipe_dsl_reg = PIPEDSL(pipe); ++ ++ save_bclrpat = I915_READ(bclrpat_reg); ++ save_vtotal = I915_READ(vtotal_reg); ++ vblank = I915_READ(vblank_reg); ++ ++ vtotal = ((save_vtotal >> 16) & 0xfff) + 1; ++ vactive = (save_vtotal & 0x7ff) + 1; ++ ++ vblank_start = (vblank & 0xfff) + 1; ++ vblank_end = ((vblank >> 16) & 0xfff) + 1; ++ ++ /* Set the border color to purple. */ ++ I915_WRITE(bclrpat_reg, 0x500050); ++ ++ if (!IS_GEN(dev_priv, 2)) { ++ u32 pipeconf = I915_READ(pipeconf_reg); ++ I915_WRITE(pipeconf_reg, pipeconf | PIPECONF_FORCE_BORDER); ++ POSTING_READ(pipeconf_reg); ++ /* Wait for next Vblank to substitue ++ * border color for Color info */ ++ intel_wait_for_vblank(dev_priv, pipe); ++ st00 = I915_READ8(_VGA_MSR_WRITE); ++ status = ((st00 & (1 << 4)) != 0) ? ++ connector_status_connected : ++ connector_status_disconnected; ++ ++ I915_WRITE(pipeconf_reg, pipeconf); ++ } else { ++ bool restore_vblank = false; ++ int count, detect; ++ ++ /* ++ * If there isn't any border, add some. ++ * Yes, this will flicker ++ */ ++ if (vblank_start <= vactive && vblank_end >= vtotal) { ++ u32 vsync = I915_READ(vsync_reg); ++ u32 vsync_start = (vsync & 0xffff) + 1; ++ ++ vblank_start = vsync_start; ++ I915_WRITE(vblank_reg, ++ (vblank_start - 1) | ++ ((vblank_end - 1) << 16)); ++ restore_vblank = true; ++ } ++ /* sample in the vertical border, selecting the larger one */ ++ if (vblank_start - vactive >= vtotal - vblank_end) ++ vsample = (vblank_start + vactive) >> 1; ++ else ++ vsample = (vtotal + vblank_end) >> 1; ++ ++ /* ++ * Wait for the border to be displayed ++ */ ++ while (I915_READ(pipe_dsl_reg) >= vactive) ++ ; ++ while ((dsl = I915_READ(pipe_dsl_reg)) <= vsample) ++ ; ++ /* ++ * Watch ST00 for an entire scanline ++ */ ++ detect = 0; ++ count = 0; ++ do { ++ count++; ++ /* Read the ST00 VGA status register */ ++ st00 = I915_READ8(_VGA_MSR_WRITE); ++ if (st00 & (1 << 4)) ++ detect++; ++ } while ((I915_READ(pipe_dsl_reg) == dsl)); ++ ++ /* restore vblank if necessary */ ++ if (restore_vblank) ++ I915_WRITE(vblank_reg, vblank); ++ /* ++ * If more than 3/4 of the scanline detected a monitor, ++ * then it is assumed to be present. This works even on i830, ++ * where there isn't any way to force the border color across ++ * the screen ++ */ ++ status = detect * 4 > count * 3 ? ++ connector_status_connected : ++ connector_status_disconnected; ++ } ++ ++ /* Restore previous settings */ ++ I915_WRITE(bclrpat_reg, save_bclrpat); ++ ++ return status; ++} ++ ++static int intel_spurious_crt_detect_dmi_callback(const struct dmi_system_id *id) ++{ ++ DRM_DEBUG_DRIVER("Skipping CRT detection for %s\n", id->ident); ++ return 1; ++} ++ ++static const struct dmi_system_id intel_spurious_crt_detect[] = { ++ { ++ .callback = intel_spurious_crt_detect_dmi_callback, ++ .ident = "ACER ZGB", ++ .matches = { ++ DMI_MATCH(DMI_SYS_VENDOR, "ACER"), ++ DMI_MATCH(DMI_PRODUCT_NAME, "ZGB"), ++ }, ++ }, ++ { ++ .callback = intel_spurious_crt_detect_dmi_callback, ++ .ident = "Intel DZ77BH-55K", ++ .matches = { ++ DMI_MATCH(DMI_BOARD_VENDOR, "Intel Corporation"), ++ DMI_MATCH(DMI_BOARD_NAME, "DZ77BH-55K"), ++ }, ++ }, ++ { } ++}; ++ ++static int ++intel_crt_detect(struct drm_connector *connector, ++ struct drm_modeset_acquire_ctx *ctx, ++ bool force) ++{ ++ struct drm_i915_private *dev_priv = to_i915(connector->dev); ++ struct intel_crt *crt = intel_attached_crt(connector); ++ struct intel_encoder *intel_encoder = &crt->base; ++ intel_wakeref_t wakeref; ++ int status, ret; ++ struct intel_load_detect_pipe tmp; ++ ++ DRM_DEBUG_KMS("[CONNECTOR:%d:%s] force=%d\n", ++ connector->base.id, connector->name, ++ force); ++ ++ if (i915_modparams.load_detect_test) { ++ wakeref = intel_display_power_get(dev_priv, ++ intel_encoder->power_domain); ++ goto load_detect; ++ } ++ ++ /* Skip machines without VGA that falsely report hotplug events */ ++ if (dmi_check_system(intel_spurious_crt_detect)) ++ return connector_status_disconnected; ++ ++ wakeref = intel_display_power_get(dev_priv, ++ intel_encoder->power_domain); ++ ++ if (I915_HAS_HOTPLUG(dev_priv)) { ++ /* We can not rely on the HPD pin always being correctly wired ++ * up, for example many KVM do not pass it through, and so ++ * only trust an assertion that the monitor is connected. ++ */ ++ if (intel_crt_detect_hotplug(connector)) { ++ DRM_DEBUG_KMS("CRT detected via hotplug\n"); ++ status = connector_status_connected; ++ goto out; ++ } else ++ DRM_DEBUG_KMS("CRT not detected via hotplug\n"); ++ } ++ ++ if (intel_crt_detect_ddc(connector)) { ++ status = connector_status_connected; ++ goto out; ++ } ++ ++ /* Load detection is broken on HPD capable machines. Whoever wants a ++ * broken monitor (without edid) to work behind a broken kvm (that fails ++ * to have the right resistors for HP detection) needs to fix this up. ++ * For now just bail out. */ ++ if (I915_HAS_HOTPLUG(dev_priv)) { ++ status = connector_status_disconnected; ++ goto out; ++ } ++ ++load_detect: ++ if (!force) { ++ status = connector->status; ++ goto out; ++ } ++ ++ /* for pre-945g platforms use load detect */ ++ ret = intel_get_load_detect_pipe(connector, NULL, &tmp, ctx); ++ if (ret > 0) { ++ if (intel_crt_detect_ddc(connector)) ++ status = connector_status_connected; ++ else if (INTEL_GEN(dev_priv) < 4) ++ status = intel_crt_load_detect(crt, ++ to_intel_crtc(connector->state->crtc)->pipe); ++ else if (i915_modparams.load_detect_test) ++ status = connector_status_disconnected; ++ else ++ status = connector_status_unknown; ++ intel_release_load_detect_pipe(connector, &tmp, ctx); ++ } else if (ret == 0) { ++ status = connector_status_unknown; ++ } else { ++ status = ret; ++ } ++ ++out: ++ intel_display_power_put(dev_priv, intel_encoder->power_domain, wakeref); ++ return status; ++} ++ ++static int intel_crt_get_modes(struct drm_connector *connector) ++{ ++ struct drm_device *dev = connector->dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_crt *crt = intel_attached_crt(connector); ++ struct intel_encoder *intel_encoder = &crt->base; ++ intel_wakeref_t wakeref; ++ struct i2c_adapter *i2c; ++ int ret; ++ ++ wakeref = intel_display_power_get(dev_priv, ++ intel_encoder->power_domain); ++ ++ i2c = intel_gmbus_get_adapter(dev_priv, dev_priv->vbt.crt_ddc_pin); ++ ret = intel_crt_ddc_get_modes(connector, i2c); ++ if (ret || !IS_G4X(dev_priv)) ++ goto out; ++ ++ /* Try to probe digital port for output in DVI-I -> VGA mode. */ ++ i2c = intel_gmbus_get_adapter(dev_priv, GMBUS_PIN_DPB); ++ ret = intel_crt_ddc_get_modes(connector, i2c); ++ ++out: ++ intel_display_power_put(dev_priv, intel_encoder->power_domain, wakeref); ++ ++ return ret; ++} ++ ++void intel_crt_reset(struct drm_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->dev); ++ struct intel_crt *crt = intel_encoder_to_crt(to_intel_encoder(encoder)); ++ ++ if (INTEL_GEN(dev_priv) >= 5) { ++ u32 adpa; ++ ++ adpa = I915_READ(crt->adpa_reg); ++ adpa &= ~ADPA_CRT_HOTPLUG_MASK; ++ adpa |= ADPA_HOTPLUG_BITS; ++ I915_WRITE(crt->adpa_reg, adpa); ++ POSTING_READ(crt->adpa_reg); ++ ++ DRM_DEBUG_KMS("crt adpa set to 0x%x\n", adpa); ++ crt->force_hotplug_required = 1; ++ } ++ ++} ++ ++/* ++ * Routines for controlling stuff on the analog port ++ */ ++ ++static const struct drm_connector_funcs intel_crt_connector_funcs = { ++ .fill_modes = drm_helper_probe_single_connector_modes, ++ .late_register = intel_connector_register, ++ .early_unregister = intel_connector_unregister, ++ .destroy = intel_connector_destroy, ++ .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, ++ .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, ++}; ++ ++static const struct drm_connector_helper_funcs intel_crt_connector_helper_funcs = { ++ .detect_ctx = intel_crt_detect, ++ .mode_valid = intel_crt_mode_valid, ++ .get_modes = intel_crt_get_modes, ++}; ++ ++static const struct drm_encoder_funcs intel_crt_enc_funcs = { ++ .reset = intel_crt_reset, ++ .destroy = intel_encoder_destroy, ++}; ++ ++void intel_crt_init(struct drm_i915_private *dev_priv) ++{ ++ struct drm_connector *connector; ++ struct intel_crt *crt; ++ struct intel_connector *intel_connector; ++ i915_reg_t adpa_reg; ++ u32 adpa; ++ ++ if (HAS_PCH_SPLIT(dev_priv)) ++ adpa_reg = PCH_ADPA; ++ else if (IS_VALLEYVIEW(dev_priv)) ++ adpa_reg = VLV_ADPA; ++ else ++ adpa_reg = ADPA; ++ ++ adpa = I915_READ(adpa_reg); ++ if ((adpa & ADPA_DAC_ENABLE) == 0) { ++ /* ++ * On some machines (some IVB at least) CRT can be ++ * fused off, but there's no known fuse bit to ++ * indicate that. On these machine the ADPA register ++ * works normally, except the DAC enable bit won't ++ * take. So the only way to tell is attempt to enable ++ * it and see what happens. ++ */ ++ I915_WRITE(adpa_reg, adpa | ADPA_DAC_ENABLE | ++ ADPA_HSYNC_CNTL_DISABLE | ADPA_VSYNC_CNTL_DISABLE); ++ if ((I915_READ(adpa_reg) & ADPA_DAC_ENABLE) == 0) ++ return; ++ I915_WRITE(adpa_reg, adpa); ++ } ++ ++ crt = kzalloc(sizeof(struct intel_crt), GFP_KERNEL); ++ if (!crt) ++ return; ++ ++ intel_connector = intel_connector_alloc(); ++ if (!intel_connector) { ++ kfree(crt); ++ return; ++ } ++ ++ connector = &intel_connector->base; ++ crt->connector = intel_connector; ++ drm_connector_init(&dev_priv->drm, &intel_connector->base, ++ &intel_crt_connector_funcs, DRM_MODE_CONNECTOR_VGA); ++ ++ drm_encoder_init(&dev_priv->drm, &crt->base.base, &intel_crt_enc_funcs, ++ DRM_MODE_ENCODER_DAC, "CRT"); ++ ++ intel_connector_attach_encoder(intel_connector, &crt->base); ++ ++ crt->base.type = INTEL_OUTPUT_ANALOG; ++ crt->base.cloneable = (1 << INTEL_OUTPUT_DVO) | (1 << INTEL_OUTPUT_HDMI); ++ if (IS_I830(dev_priv)) ++ crt->base.crtc_mask = (1 << 0); ++ else ++ crt->base.crtc_mask = (1 << 0) | (1 << 1) | (1 << 2); ++ ++ if (IS_GEN(dev_priv, 2)) ++ connector->interlace_allowed = 0; ++ else ++ connector->interlace_allowed = 1; ++ connector->doublescan_allowed = 0; ++ ++ crt->adpa_reg = adpa_reg; ++ ++ crt->base.power_domain = POWER_DOMAIN_PORT_CRT; ++ ++ if (I915_HAS_HOTPLUG(dev_priv) && ++ !dmi_check_system(intel_spurious_crt_detect)) { ++ crt->base.hpd_pin = HPD_CRT; ++ crt->base.hotplug = intel_encoder_hotplug; ++ } ++ ++ if (HAS_DDI(dev_priv)) { ++ crt->base.port = PORT_E; ++ crt->base.get_config = hsw_crt_get_config; ++ crt->base.get_hw_state = intel_ddi_get_hw_state; ++ crt->base.compute_config = hsw_crt_compute_config; ++ crt->base.pre_pll_enable = hsw_pre_pll_enable_crt; ++ crt->base.pre_enable = hsw_pre_enable_crt; ++ crt->base.enable = hsw_enable_crt; ++ crt->base.disable = hsw_disable_crt; ++ crt->base.post_disable = hsw_post_disable_crt; ++ } else { ++ if (HAS_PCH_SPLIT(dev_priv)) { ++ crt->base.compute_config = pch_crt_compute_config; ++ crt->base.disable = pch_disable_crt; ++ crt->base.post_disable = pch_post_disable_crt; ++ } else { ++ crt->base.compute_config = intel_crt_compute_config; ++ crt->base.disable = intel_disable_crt; ++ } ++ crt->base.port = PORT_NONE; ++ crt->base.get_config = intel_crt_get_config; ++ crt->base.get_hw_state = intel_crt_get_hw_state; ++ crt->base.enable = intel_enable_crt; ++ } ++ intel_connector->get_hw_state = intel_connector_get_hw_state; ++ ++ drm_connector_helper_add(connector, &intel_crt_connector_helper_funcs); ++ ++ if (!I915_HAS_HOTPLUG(dev_priv)) ++ intel_connector->polled = DRM_CONNECTOR_POLL_CONNECT; ++ ++ /* ++ * Configure the automatic hotplug detection stuff ++ */ ++ crt->force_hotplug_required = 0; ++ ++ /* ++ * TODO: find a proper way to discover whether we need to set the the ++ * polarity and link reversal bits or not, instead of relying on the ++ * BIOS. ++ */ ++ if (HAS_PCH_LPT(dev_priv)) { ++ u32 fdi_config = FDI_RX_POLARITY_REVERSED_LPT | ++ FDI_RX_LINK_REVERSAL_OVERRIDE; ++ ++ dev_priv->fdi_rx_config = I915_READ(FDI_RX_CTL(PIPE_A)) & fdi_config; ++ } ++ ++ intel_crt_reset(&crt->base.base); ++} +diff --git a/drivers/gpu/drm/i915_legacy/intel_crt.h b/drivers/gpu/drm/i915_legacy/intel_crt.h +new file mode 100644 +index 000000000000..1b3fba359efc +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_crt.h +@@ -0,0 +1,21 @@ ++/* SPDX-License-Identifier: MIT */ ++/* ++ * Copyright © 2019 Intel Corporation ++ */ ++ ++#ifndef __INTEL_CRT_H__ ++#define __INTEL_CRT_H__ ++ ++#include "i915_reg.h" ++ ++enum pipe; ++struct drm_encoder; ++struct drm_i915_private; ++struct drm_i915_private; ++ ++bool intel_crt_port_enabled(struct drm_i915_private *dev_priv, ++ i915_reg_t adpa_reg, enum pipe *pipe); ++void intel_crt_init(struct drm_i915_private *dev_priv); ++void intel_crt_reset(struct drm_encoder *encoder); ++ ++#endif /* __INTEL_CRT_H__ */ +diff --git a/drivers/gpu/drm/i915_legacy/intel_csr.c b/drivers/gpu/drm/i915_legacy/intel_csr.c +new file mode 100644 +index 000000000000..96618af47088 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_csr.c +@@ -0,0 +1,615 @@ ++/* ++ * Copyright © 2014 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#include ++ ++#include "i915_drv.h" ++#include "i915_reg.h" ++#include "intel_csr.h" ++ ++/** ++ * DOC: csr support for dmc ++ * ++ * Display Context Save and Restore (CSR) firmware support added from gen9 ++ * onwards to drive newly added DMC (Display microcontroller) in display ++ * engine to save and restore the state of display engine when it enter into ++ * low-power state and comes back to normal. ++ */ ++ ++#define GEN12_CSR_MAX_FW_SIZE ICL_CSR_MAX_FW_SIZE ++ ++#define ICL_CSR_PATH "i915/icl_dmc_ver1_07.bin" ++#define ICL_CSR_VERSION_REQUIRED CSR_VERSION(1, 7) ++#define ICL_CSR_MAX_FW_SIZE 0x6000 ++MODULE_FIRMWARE(ICL_CSR_PATH); ++ ++#define CNL_CSR_PATH "i915/cnl_dmc_ver1_07.bin" ++#define CNL_CSR_VERSION_REQUIRED CSR_VERSION(1, 7) ++#define CNL_CSR_MAX_FW_SIZE GLK_CSR_MAX_FW_SIZE ++MODULE_FIRMWARE(CNL_CSR_PATH); ++ ++#define GLK_CSR_PATH "i915/glk_dmc_ver1_04.bin" ++#define GLK_CSR_VERSION_REQUIRED CSR_VERSION(1, 4) ++#define GLK_CSR_MAX_FW_SIZE 0x4000 ++MODULE_FIRMWARE(GLK_CSR_PATH); ++ ++#define KBL_CSR_PATH "i915/kbl_dmc_ver1_04.bin" ++#define KBL_CSR_VERSION_REQUIRED CSR_VERSION(1, 4) ++#define KBL_CSR_MAX_FW_SIZE BXT_CSR_MAX_FW_SIZE ++MODULE_FIRMWARE(KBL_CSR_PATH); ++ ++#define SKL_CSR_PATH "i915/skl_dmc_ver1_27.bin" ++#define SKL_CSR_VERSION_REQUIRED CSR_VERSION(1, 27) ++#define SKL_CSR_MAX_FW_SIZE BXT_CSR_MAX_FW_SIZE ++MODULE_FIRMWARE(SKL_CSR_PATH); ++ ++#define BXT_CSR_PATH "i915/bxt_dmc_ver1_07.bin" ++#define BXT_CSR_VERSION_REQUIRED CSR_VERSION(1, 7) ++#define BXT_CSR_MAX_FW_SIZE 0x3000 ++MODULE_FIRMWARE(BXT_CSR_PATH); ++ ++#define CSR_DEFAULT_FW_OFFSET 0xFFFFFFFF ++ ++struct intel_css_header { ++ /* 0x09 for DMC */ ++ u32 module_type; ++ ++ /* Includes the DMC specific header in dwords */ ++ u32 header_len; ++ ++ /* always value would be 0x10000 */ ++ u32 header_ver; ++ ++ /* Not used */ ++ u32 module_id; ++ ++ /* Not used */ ++ u32 module_vendor; ++ ++ /* in YYYYMMDD format */ ++ u32 date; ++ ++ /* Size in dwords (CSS_Headerlen + PackageHeaderLen + dmc FWsLen)/4 */ ++ u32 size; ++ ++ /* Not used */ ++ u32 key_size; ++ ++ /* Not used */ ++ u32 modulus_size; ++ ++ /* Not used */ ++ u32 exponent_size; ++ ++ /* Not used */ ++ u32 reserved1[12]; ++ ++ /* Major Minor */ ++ u32 version; ++ ++ /* Not used */ ++ u32 reserved2[8]; ++ ++ /* Not used */ ++ u32 kernel_header_info; ++} __packed; ++ ++struct intel_fw_info { ++ u16 reserved1; ++ ++ /* Stepping (A, B, C, ..., *). * is a wildcard */ ++ char stepping; ++ ++ /* Sub-stepping (0, 1, ..., *). * is a wildcard */ ++ char substepping; ++ ++ u32 offset; ++ u32 reserved2; ++} __packed; ++ ++struct intel_package_header { ++ /* DMC container header length in dwords */ ++ unsigned char header_len; ++ ++ /* always value would be 0x01 */ ++ unsigned char header_ver; ++ ++ unsigned char reserved[10]; ++ ++ /* Number of valid entries in the FWInfo array below */ ++ u32 num_entries; ++ ++ struct intel_fw_info fw_info[20]; ++} __packed; ++ ++struct intel_dmc_header { ++ /* always value would be 0x40403E3E */ ++ u32 signature; ++ ++ /* DMC binary header length */ ++ unsigned char header_len; ++ ++ /* 0x01 */ ++ unsigned char header_ver; ++ ++ /* Reserved */ ++ u16 dmcc_ver; ++ ++ /* Major, Minor */ ++ u32 project; ++ ++ /* Firmware program size (excluding header) in dwords */ ++ u32 fw_size; ++ ++ /* Major Minor version */ ++ u32 fw_version; ++ ++ /* Number of valid MMIO cycles present. */ ++ u32 mmio_count; ++ ++ /* MMIO address */ ++ u32 mmioaddr[8]; ++ ++ /* MMIO data */ ++ u32 mmiodata[8]; ++ ++ /* FW filename */ ++ unsigned char dfile[32]; ++ ++ u32 reserved1[2]; ++} __packed; ++ ++struct stepping_info { ++ char stepping; ++ char substepping; ++}; ++ ++static const struct stepping_info skl_stepping_info[] = { ++ {'A', '0'}, {'B', '0'}, {'C', '0'}, ++ {'D', '0'}, {'E', '0'}, {'F', '0'}, ++ {'G', '0'}, {'H', '0'}, {'I', '0'}, ++ {'J', '0'}, {'K', '0'} ++}; ++ ++static const struct stepping_info bxt_stepping_info[] = { ++ {'A', '0'}, {'A', '1'}, {'A', '2'}, ++ {'B', '0'}, {'B', '1'}, {'B', '2'} ++}; ++ ++static const struct stepping_info icl_stepping_info[] = { ++ {'A', '0'}, {'A', '1'}, {'A', '2'}, ++ {'B', '0'}, {'B', '2'}, ++ {'C', '0'} ++}; ++ ++static const struct stepping_info no_stepping_info = { '*', '*' }; ++ ++static const struct stepping_info * ++intel_get_stepping_info(struct drm_i915_private *dev_priv) ++{ ++ const struct stepping_info *si; ++ unsigned int size; ++ ++ if (IS_ICELAKE(dev_priv)) { ++ size = ARRAY_SIZE(icl_stepping_info); ++ si = icl_stepping_info; ++ } else if (IS_SKYLAKE(dev_priv)) { ++ size = ARRAY_SIZE(skl_stepping_info); ++ si = skl_stepping_info; ++ } else if (IS_BROXTON(dev_priv)) { ++ size = ARRAY_SIZE(bxt_stepping_info); ++ si = bxt_stepping_info; ++ } else { ++ size = 0; ++ si = NULL; ++ } ++ ++ if (INTEL_REVID(dev_priv) < size) ++ return si + INTEL_REVID(dev_priv); ++ ++ return &no_stepping_info; ++} ++ ++static void gen9_set_dc_state_debugmask(struct drm_i915_private *dev_priv) ++{ ++ u32 val, mask; ++ ++ mask = DC_STATE_DEBUG_MASK_MEMORY_UP; ++ ++ if (IS_GEN9_LP(dev_priv)) ++ mask |= DC_STATE_DEBUG_MASK_CORES; ++ ++ /* The below bit doesn't need to be cleared ever afterwards */ ++ val = I915_READ(DC_STATE_DEBUG); ++ if ((val & mask) != mask) { ++ val |= mask; ++ I915_WRITE(DC_STATE_DEBUG, val); ++ POSTING_READ(DC_STATE_DEBUG); ++ } ++} ++ ++/** ++ * intel_csr_load_program() - write the firmware from memory to register. ++ * @dev_priv: i915 drm device. ++ * ++ * CSR firmware is read from a .bin file and kept in internal memory one time. ++ * Everytime display comes back from low power state this function is called to ++ * copy the firmware from internal memory to registers. ++ */ ++void intel_csr_load_program(struct drm_i915_private *dev_priv) ++{ ++ u32 *payload = dev_priv->csr.dmc_payload; ++ u32 i, fw_size; ++ ++ if (!HAS_CSR(dev_priv)) { ++ DRM_ERROR("No CSR support available for this platform\n"); ++ return; ++ } ++ ++ if (!dev_priv->csr.dmc_payload) { ++ DRM_ERROR("Tried to program CSR with empty payload\n"); ++ return; ++ } ++ ++ fw_size = dev_priv->csr.dmc_fw_size; ++ assert_rpm_wakelock_held(dev_priv); ++ ++ preempt_disable(); ++ ++ for (i = 0; i < fw_size; i++) ++ I915_WRITE_FW(CSR_PROGRAM(i), payload[i]); ++ ++ preempt_enable(); ++ ++ for (i = 0; i < dev_priv->csr.mmio_count; i++) { ++ I915_WRITE(dev_priv->csr.mmioaddr[i], ++ dev_priv->csr.mmiodata[i]); ++ } ++ ++ dev_priv->csr.dc_state = 0; ++ ++ gen9_set_dc_state_debugmask(dev_priv); ++} ++ ++static u32 *parse_csr_fw(struct drm_i915_private *dev_priv, ++ const struct firmware *fw) ++{ ++ struct intel_css_header *css_header; ++ struct intel_package_header *package_header; ++ struct intel_dmc_header *dmc_header; ++ struct intel_csr *csr = &dev_priv->csr; ++ const struct stepping_info *si = intel_get_stepping_info(dev_priv); ++ u32 dmc_offset = CSR_DEFAULT_FW_OFFSET, readcount = 0, nbytes; ++ u32 i; ++ u32 *dmc_payload; ++ size_t fsize; ++ ++ if (!fw) ++ return NULL; ++ ++ fsize = sizeof(struct intel_css_header) + ++ sizeof(struct intel_package_header) + ++ sizeof(struct intel_dmc_header); ++ if (fsize > fw->size) ++ goto error_truncated; ++ ++ /* Extract CSS Header information*/ ++ css_header = (struct intel_css_header *)fw->data; ++ if (sizeof(struct intel_css_header) != ++ (css_header->header_len * 4)) { ++ DRM_ERROR("DMC firmware has wrong CSS header length " ++ "(%u bytes)\n", ++ (css_header->header_len * 4)); ++ return NULL; ++ } ++ ++ if (csr->required_version && ++ css_header->version != csr->required_version) { ++ DRM_INFO("Refusing to load DMC firmware v%u.%u," ++ " please use v%u.%u\n", ++ CSR_VERSION_MAJOR(css_header->version), ++ CSR_VERSION_MINOR(css_header->version), ++ CSR_VERSION_MAJOR(csr->required_version), ++ CSR_VERSION_MINOR(csr->required_version)); ++ return NULL; ++ } ++ ++ csr->version = css_header->version; ++ ++ readcount += sizeof(struct intel_css_header); ++ ++ /* Extract Package Header information*/ ++ package_header = (struct intel_package_header *) ++ &fw->data[readcount]; ++ if (sizeof(struct intel_package_header) != ++ (package_header->header_len * 4)) { ++ DRM_ERROR("DMC firmware has wrong package header length " ++ "(%u bytes)\n", ++ (package_header->header_len * 4)); ++ return NULL; ++ } ++ readcount += sizeof(struct intel_package_header); ++ ++ /* Search for dmc_offset to find firware binary. */ ++ for (i = 0; i < package_header->num_entries; i++) { ++ if (package_header->fw_info[i].substepping == '*' && ++ si->stepping == package_header->fw_info[i].stepping) { ++ dmc_offset = package_header->fw_info[i].offset; ++ break; ++ } else if (si->stepping == package_header->fw_info[i].stepping && ++ si->substepping == package_header->fw_info[i].substepping) { ++ dmc_offset = package_header->fw_info[i].offset; ++ break; ++ } else if (package_header->fw_info[i].stepping == '*' && ++ package_header->fw_info[i].substepping == '*') ++ dmc_offset = package_header->fw_info[i].offset; ++ } ++ if (dmc_offset == CSR_DEFAULT_FW_OFFSET) { ++ DRM_ERROR("DMC firmware not supported for %c stepping\n", ++ si->stepping); ++ return NULL; ++ } ++ /* Convert dmc_offset into number of bytes. By default it is in dwords*/ ++ dmc_offset *= 4; ++ readcount += dmc_offset; ++ fsize += dmc_offset; ++ if (fsize > fw->size) ++ goto error_truncated; ++ ++ /* Extract dmc_header information. */ ++ dmc_header = (struct intel_dmc_header *)&fw->data[readcount]; ++ if (sizeof(struct intel_dmc_header) != (dmc_header->header_len)) { ++ DRM_ERROR("DMC firmware has wrong dmc header length " ++ "(%u bytes)\n", ++ (dmc_header->header_len)); ++ return NULL; ++ } ++ readcount += sizeof(struct intel_dmc_header); ++ ++ /* Cache the dmc header info. */ ++ if (dmc_header->mmio_count > ARRAY_SIZE(csr->mmioaddr)) { ++ DRM_ERROR("DMC firmware has wrong mmio count %u\n", ++ dmc_header->mmio_count); ++ return NULL; ++ } ++ csr->mmio_count = dmc_header->mmio_count; ++ for (i = 0; i < dmc_header->mmio_count; i++) { ++ if (dmc_header->mmioaddr[i] < CSR_MMIO_START_RANGE || ++ dmc_header->mmioaddr[i] > CSR_MMIO_END_RANGE) { ++ DRM_ERROR("DMC firmware has wrong mmio address 0x%x\n", ++ dmc_header->mmioaddr[i]); ++ return NULL; ++ } ++ csr->mmioaddr[i] = _MMIO(dmc_header->mmioaddr[i]); ++ csr->mmiodata[i] = dmc_header->mmiodata[i]; ++ } ++ ++ /* fw_size is in dwords, so multiplied by 4 to convert into bytes. */ ++ nbytes = dmc_header->fw_size * 4; ++ fsize += nbytes; ++ if (fsize > fw->size) ++ goto error_truncated; ++ ++ if (nbytes > csr->max_fw_size) { ++ DRM_ERROR("DMC FW too big (%u bytes)\n", nbytes); ++ return NULL; ++ } ++ csr->dmc_fw_size = dmc_header->fw_size; ++ ++ dmc_payload = kmalloc(nbytes, GFP_KERNEL); ++ if (!dmc_payload) { ++ DRM_ERROR("Memory allocation failed for dmc payload\n"); ++ return NULL; ++ } ++ ++ return memcpy(dmc_payload, &fw->data[readcount], nbytes); ++ ++error_truncated: ++ DRM_ERROR("Truncated DMC firmware, rejecting.\n"); ++ return NULL; ++} ++ ++static void intel_csr_runtime_pm_get(struct drm_i915_private *dev_priv) ++{ ++ WARN_ON(dev_priv->csr.wakeref); ++ dev_priv->csr.wakeref = ++ intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); ++} ++ ++static void intel_csr_runtime_pm_put(struct drm_i915_private *dev_priv) ++{ ++ intel_wakeref_t wakeref __maybe_unused = ++ fetch_and_zero(&dev_priv->csr.wakeref); ++ ++ intel_display_power_put(dev_priv, POWER_DOMAIN_INIT, wakeref); ++} ++ ++static void csr_load_work_fn(struct work_struct *work) ++{ ++ struct drm_i915_private *dev_priv; ++ struct intel_csr *csr; ++ const struct firmware *fw = NULL; ++ ++ dev_priv = container_of(work, typeof(*dev_priv), csr.work); ++ csr = &dev_priv->csr; ++ ++ request_firmware(&fw, dev_priv->csr.fw_path, &dev_priv->drm.pdev->dev); ++ if (fw) ++ dev_priv->csr.dmc_payload = parse_csr_fw(dev_priv, fw); ++ ++ if (dev_priv->csr.dmc_payload) { ++ intel_csr_load_program(dev_priv); ++ intel_csr_runtime_pm_put(dev_priv); ++ ++ DRM_INFO("Finished loading DMC firmware %s (v%u.%u)\n", ++ dev_priv->csr.fw_path, ++ CSR_VERSION_MAJOR(csr->version), ++ CSR_VERSION_MINOR(csr->version)); ++ } else { ++ dev_notice(dev_priv->drm.dev, ++ "Failed to load DMC firmware %s." ++ " Disabling runtime power management.\n", ++ csr->fw_path); ++ dev_notice(dev_priv->drm.dev, "DMC firmware homepage: %s", ++ INTEL_UC_FIRMWARE_URL); ++ } ++ ++ release_firmware(fw); ++} ++ ++/** ++ * intel_csr_ucode_init() - initialize the firmware loading. ++ * @dev_priv: i915 drm device. ++ * ++ * This function is called at the time of loading the display driver to read ++ * firmware from a .bin file and copied into a internal memory. ++ */ ++void intel_csr_ucode_init(struct drm_i915_private *dev_priv) ++{ ++ struct intel_csr *csr = &dev_priv->csr; ++ ++ INIT_WORK(&dev_priv->csr.work, csr_load_work_fn); ++ ++ if (!HAS_CSR(dev_priv)) ++ return; ++ ++ /* ++ * Obtain a runtime pm reference, until CSR is loaded, to avoid entering ++ * runtime-suspend. ++ * ++ * On error, we return with the rpm wakeref held to prevent runtime ++ * suspend as runtime suspend *requires* a working CSR for whatever ++ * reason. ++ */ ++ intel_csr_runtime_pm_get(dev_priv); ++ ++ if (INTEL_GEN(dev_priv) >= 12) { ++ /* Allow to load fw via parameter using the last known size */ ++ csr->max_fw_size = GEN12_CSR_MAX_FW_SIZE; ++ } else if (IS_GEN(dev_priv, 11)) { ++ csr->fw_path = ICL_CSR_PATH; ++ csr->required_version = ICL_CSR_VERSION_REQUIRED; ++ csr->max_fw_size = ICL_CSR_MAX_FW_SIZE; ++ } else if (IS_CANNONLAKE(dev_priv)) { ++ csr->fw_path = CNL_CSR_PATH; ++ csr->required_version = CNL_CSR_VERSION_REQUIRED; ++ csr->max_fw_size = CNL_CSR_MAX_FW_SIZE; ++ } else if (IS_GEMINILAKE(dev_priv)) { ++ csr->fw_path = GLK_CSR_PATH; ++ csr->required_version = GLK_CSR_VERSION_REQUIRED; ++ csr->max_fw_size = GLK_CSR_MAX_FW_SIZE; ++ } else if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) { ++ csr->fw_path = KBL_CSR_PATH; ++ csr->required_version = KBL_CSR_VERSION_REQUIRED; ++ csr->max_fw_size = KBL_CSR_MAX_FW_SIZE; ++ } else if (IS_SKYLAKE(dev_priv)) { ++ csr->fw_path = SKL_CSR_PATH; ++ csr->required_version = SKL_CSR_VERSION_REQUIRED; ++ csr->max_fw_size = SKL_CSR_MAX_FW_SIZE; ++ } else if (IS_BROXTON(dev_priv)) { ++ csr->fw_path = BXT_CSR_PATH; ++ csr->required_version = BXT_CSR_VERSION_REQUIRED; ++ csr->max_fw_size = BXT_CSR_MAX_FW_SIZE; ++ } ++ ++ if (i915_modparams.dmc_firmware_path) { ++ if (strlen(i915_modparams.dmc_firmware_path) == 0) { ++ csr->fw_path = NULL; ++ DRM_INFO("Disabling CSR firmware and runtime PM\n"); ++ return; ++ } ++ ++ csr->fw_path = i915_modparams.dmc_firmware_path; ++ /* Bypass version check for firmware override. */ ++ csr->required_version = 0; ++ } ++ ++ if (csr->fw_path == NULL) { ++ DRM_DEBUG_KMS("No known CSR firmware for platform, disabling runtime PM\n"); ++ WARN_ON(!IS_ALPHA_SUPPORT(INTEL_INFO(dev_priv))); ++ ++ return; ++ } ++ ++ DRM_DEBUG_KMS("Loading %s\n", csr->fw_path); ++ schedule_work(&dev_priv->csr.work); ++} ++ ++/** ++ * intel_csr_ucode_suspend() - prepare CSR firmware before system suspend ++ * @dev_priv: i915 drm device ++ * ++ * Prepare the DMC firmware before entering system suspend. This includes ++ * flushing pending work items and releasing any resources acquired during ++ * init. ++ */ ++void intel_csr_ucode_suspend(struct drm_i915_private *dev_priv) ++{ ++ if (!HAS_CSR(dev_priv)) ++ return; ++ ++ flush_work(&dev_priv->csr.work); ++ ++ /* Drop the reference held in case DMC isn't loaded. */ ++ if (!dev_priv->csr.dmc_payload) ++ intel_csr_runtime_pm_put(dev_priv); ++} ++ ++/** ++ * intel_csr_ucode_resume() - init CSR firmware during system resume ++ * @dev_priv: i915 drm device ++ * ++ * Reinitialize the DMC firmware during system resume, reacquiring any ++ * resources released in intel_csr_ucode_suspend(). ++ */ ++void intel_csr_ucode_resume(struct drm_i915_private *dev_priv) ++{ ++ if (!HAS_CSR(dev_priv)) ++ return; ++ ++ /* ++ * Reacquire the reference to keep RPM disabled in case DMC isn't ++ * loaded. ++ */ ++ if (!dev_priv->csr.dmc_payload) ++ intel_csr_runtime_pm_get(dev_priv); ++} ++ ++/** ++ * intel_csr_ucode_fini() - unload the CSR firmware. ++ * @dev_priv: i915 drm device. ++ * ++ * Firmmware unloading includes freeing the internal memory and reset the ++ * firmware loading status. ++ */ ++void intel_csr_ucode_fini(struct drm_i915_private *dev_priv) ++{ ++ if (!HAS_CSR(dev_priv)) ++ return; ++ ++ intel_csr_ucode_suspend(dev_priv); ++ WARN_ON(dev_priv->csr.wakeref); ++ ++ kfree(dev_priv->csr.dmc_payload); ++} +diff --git a/drivers/gpu/drm/i915_legacy/intel_csr.h b/drivers/gpu/drm/i915_legacy/intel_csr.h +new file mode 100644 +index 000000000000..17a32c1e8a35 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_csr.h +@@ -0,0 +1,17 @@ ++/* SPDX-License-Identifier: MIT */ ++/* ++ * Copyright © 2019 Intel Corporation ++ */ ++ ++#ifndef __INTEL_CSR_H__ ++#define __INTEL_CSR_H__ ++ ++struct drm_i915_private; ++ ++void intel_csr_ucode_init(struct drm_i915_private *i915); ++void intel_csr_load_program(struct drm_i915_private *i915); ++void intel_csr_ucode_fini(struct drm_i915_private *i915); ++void intel_csr_ucode_suspend(struct drm_i915_private *i915); ++void intel_csr_ucode_resume(struct drm_i915_private *i915); ++ ++#endif /* __INTEL_CSR_H__ */ +diff --git a/drivers/gpu/drm/i915_legacy/intel_ddi.c b/drivers/gpu/drm/i915_legacy/intel_ddi.c +new file mode 100644 +index 000000000000..f181c26f62fd +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_ddi.c +@@ -0,0 +1,4286 @@ ++/* ++ * Copyright © 2012 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ * Authors: ++ * Eugeni Dodonov ++ * ++ */ ++ ++#include ++ ++#include "i915_drv.h" ++#include "intel_audio.h" ++#include "intel_connector.h" ++#include "intel_ddi.h" ++#include "intel_dp.h" ++#include "intel_drv.h" ++#include "intel_dsi.h" ++#include "intel_hdcp.h" ++#include "intel_hdmi.h" ++#include "intel_lspcon.h" ++#include "intel_panel.h" ++#include "intel_psr.h" ++ ++struct ddi_buf_trans { ++ u32 trans1; /* balance leg enable, de-emph level */ ++ u32 trans2; /* vref sel, vswing */ ++ u8 i_boost; /* SKL: I_boost; valid: 0x0, 0x1, 0x3, 0x7 */ ++}; ++ ++static const u8 index_to_dp_signal_levels[] = { ++ [0] = DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_0, ++ [1] = DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_1, ++ [2] = DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_2, ++ [3] = DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_3, ++ [4] = DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_0, ++ [5] = DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_1, ++ [6] = DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_2, ++ [7] = DP_TRAIN_VOLTAGE_SWING_LEVEL_2 | DP_TRAIN_PRE_EMPH_LEVEL_0, ++ [8] = DP_TRAIN_VOLTAGE_SWING_LEVEL_2 | DP_TRAIN_PRE_EMPH_LEVEL_1, ++ [9] = DP_TRAIN_VOLTAGE_SWING_LEVEL_3 | DP_TRAIN_PRE_EMPH_LEVEL_0, ++}; ++ ++/* HDMI/DVI modes ignore everything but the last 2 items. So we share ++ * them for both DP and FDI transports, allowing those ports to ++ * automatically adapt to HDMI connections as well ++ */ ++static const struct ddi_buf_trans hsw_ddi_translations_dp[] = { ++ { 0x00FFFFFF, 0x0006000E, 0x0 }, ++ { 0x00D75FFF, 0x0005000A, 0x0 }, ++ { 0x00C30FFF, 0x00040006, 0x0 }, ++ { 0x80AAAFFF, 0x000B0000, 0x0 }, ++ { 0x00FFFFFF, 0x0005000A, 0x0 }, ++ { 0x00D75FFF, 0x000C0004, 0x0 }, ++ { 0x80C30FFF, 0x000B0000, 0x0 }, ++ { 0x00FFFFFF, 0x00040006, 0x0 }, ++ { 0x80D75FFF, 0x000B0000, 0x0 }, ++}; ++ ++static const struct ddi_buf_trans hsw_ddi_translations_fdi[] = { ++ { 0x00FFFFFF, 0x0007000E, 0x0 }, ++ { 0x00D75FFF, 0x000F000A, 0x0 }, ++ { 0x00C30FFF, 0x00060006, 0x0 }, ++ { 0x00AAAFFF, 0x001E0000, 0x0 }, ++ { 0x00FFFFFF, 0x000F000A, 0x0 }, ++ { 0x00D75FFF, 0x00160004, 0x0 }, ++ { 0x00C30FFF, 0x001E0000, 0x0 }, ++ { 0x00FFFFFF, 0x00060006, 0x0 }, ++ { 0x00D75FFF, 0x001E0000, 0x0 }, ++}; ++ ++static const struct ddi_buf_trans hsw_ddi_translations_hdmi[] = { ++ /* Idx NT mV d T mV d db */ ++ { 0x00FFFFFF, 0x0006000E, 0x0 },/* 0: 400 400 0 */ ++ { 0x00E79FFF, 0x000E000C, 0x0 },/* 1: 400 500 2 */ ++ { 0x00D75FFF, 0x0005000A, 0x0 },/* 2: 400 600 3.5 */ ++ { 0x00FFFFFF, 0x0005000A, 0x0 },/* 3: 600 600 0 */ ++ { 0x00E79FFF, 0x001D0007, 0x0 },/* 4: 600 750 2 */ ++ { 0x00D75FFF, 0x000C0004, 0x0 },/* 5: 600 900 3.5 */ ++ { 0x00FFFFFF, 0x00040006, 0x0 },/* 6: 800 800 0 */ ++ { 0x80E79FFF, 0x00030002, 0x0 },/* 7: 800 1000 2 */ ++ { 0x00FFFFFF, 0x00140005, 0x0 },/* 8: 850 850 0 */ ++ { 0x00FFFFFF, 0x000C0004, 0x0 },/* 9: 900 900 0 */ ++ { 0x00FFFFFF, 0x001C0003, 0x0 },/* 10: 950 950 0 */ ++ { 0x80FFFFFF, 0x00030002, 0x0 },/* 11: 1000 1000 0 */ ++}; ++ ++static const struct ddi_buf_trans bdw_ddi_translations_edp[] = { ++ { 0x00FFFFFF, 0x00000012, 0x0 }, ++ { 0x00EBAFFF, 0x00020011, 0x0 }, ++ { 0x00C71FFF, 0x0006000F, 0x0 }, ++ { 0x00AAAFFF, 0x000E000A, 0x0 }, ++ { 0x00FFFFFF, 0x00020011, 0x0 }, ++ { 0x00DB6FFF, 0x0005000F, 0x0 }, ++ { 0x00BEEFFF, 0x000A000C, 0x0 }, ++ { 0x00FFFFFF, 0x0005000F, 0x0 }, ++ { 0x00DB6FFF, 0x000A000C, 0x0 }, ++}; ++ ++static const struct ddi_buf_trans bdw_ddi_translations_dp[] = { ++ { 0x00FFFFFF, 0x0007000E, 0x0 }, ++ { 0x00D75FFF, 0x000E000A, 0x0 }, ++ { 0x00BEFFFF, 0x00140006, 0x0 }, ++ { 0x80B2CFFF, 0x001B0002, 0x0 }, ++ { 0x00FFFFFF, 0x000E000A, 0x0 }, ++ { 0x00DB6FFF, 0x00160005, 0x0 }, ++ { 0x80C71FFF, 0x001A0002, 0x0 }, ++ { 0x00F7DFFF, 0x00180004, 0x0 }, ++ { 0x80D75FFF, 0x001B0002, 0x0 }, ++}; ++ ++static const struct ddi_buf_trans bdw_ddi_translations_fdi[] = { ++ { 0x00FFFFFF, 0x0001000E, 0x0 }, ++ { 0x00D75FFF, 0x0004000A, 0x0 }, ++ { 0x00C30FFF, 0x00070006, 0x0 }, ++ { 0x00AAAFFF, 0x000C0000, 0x0 }, ++ { 0x00FFFFFF, 0x0004000A, 0x0 }, ++ { 0x00D75FFF, 0x00090004, 0x0 }, ++ { 0x00C30FFF, 0x000C0000, 0x0 }, ++ { 0x00FFFFFF, 0x00070006, 0x0 }, ++ { 0x00D75FFF, 0x000C0000, 0x0 }, ++}; ++ ++static const struct ddi_buf_trans bdw_ddi_translations_hdmi[] = { ++ /* Idx NT mV d T mV df db */ ++ { 0x00FFFFFF, 0x0007000E, 0x0 },/* 0: 400 400 0 */ ++ { 0x00D75FFF, 0x000E000A, 0x0 },/* 1: 400 600 3.5 */ ++ { 0x00BEFFFF, 0x00140006, 0x0 },/* 2: 400 800 6 */ ++ { 0x00FFFFFF, 0x0009000D, 0x0 },/* 3: 450 450 0 */ ++ { 0x00FFFFFF, 0x000E000A, 0x0 },/* 4: 600 600 0 */ ++ { 0x00D7FFFF, 0x00140006, 0x0 },/* 5: 600 800 2.5 */ ++ { 0x80CB2FFF, 0x001B0002, 0x0 },/* 6: 600 1000 4.5 */ ++ { 0x00FFFFFF, 0x00140006, 0x0 },/* 7: 800 800 0 */ ++ { 0x80E79FFF, 0x001B0002, 0x0 },/* 8: 800 1000 2 */ ++ { 0x80FFFFFF, 0x001B0002, 0x0 },/* 9: 1000 1000 0 */ ++}; ++ ++/* Skylake H and S */ ++static const struct ddi_buf_trans skl_ddi_translations_dp[] = { ++ { 0x00002016, 0x000000A0, 0x0 }, ++ { 0x00005012, 0x0000009B, 0x0 }, ++ { 0x00007011, 0x00000088, 0x0 }, ++ { 0x80009010, 0x000000C0, 0x1 }, ++ { 0x00002016, 0x0000009B, 0x0 }, ++ { 0x00005012, 0x00000088, 0x0 }, ++ { 0x80007011, 0x000000C0, 0x1 }, ++ { 0x00002016, 0x000000DF, 0x0 }, ++ { 0x80005012, 0x000000C0, 0x1 }, ++}; ++ ++/* Skylake U */ ++static const struct ddi_buf_trans skl_u_ddi_translations_dp[] = { ++ { 0x0000201B, 0x000000A2, 0x0 }, ++ { 0x00005012, 0x00000088, 0x0 }, ++ { 0x80007011, 0x000000CD, 0x1 }, ++ { 0x80009010, 0x000000C0, 0x1 }, ++ { 0x0000201B, 0x0000009D, 0x0 }, ++ { 0x80005012, 0x000000C0, 0x1 }, ++ { 0x80007011, 0x000000C0, 0x1 }, ++ { 0x00002016, 0x00000088, 0x0 }, ++ { 0x80005012, 0x000000C0, 0x1 }, ++}; ++ ++/* Skylake Y */ ++static const struct ddi_buf_trans skl_y_ddi_translations_dp[] = { ++ { 0x00000018, 0x000000A2, 0x0 }, ++ { 0x00005012, 0x00000088, 0x0 }, ++ { 0x80007011, 0x000000CD, 0x3 }, ++ { 0x80009010, 0x000000C0, 0x3 }, ++ { 0x00000018, 0x0000009D, 0x0 }, ++ { 0x80005012, 0x000000C0, 0x3 }, ++ { 0x80007011, 0x000000C0, 0x3 }, ++ { 0x00000018, 0x00000088, 0x0 }, ++ { 0x80005012, 0x000000C0, 0x3 }, ++}; ++ ++/* Kabylake H and S */ ++static const struct ddi_buf_trans kbl_ddi_translations_dp[] = { ++ { 0x00002016, 0x000000A0, 0x0 }, ++ { 0x00005012, 0x0000009B, 0x0 }, ++ { 0x00007011, 0x00000088, 0x0 }, ++ { 0x80009010, 0x000000C0, 0x1 }, ++ { 0x00002016, 0x0000009B, 0x0 }, ++ { 0x00005012, 0x00000088, 0x0 }, ++ { 0x80007011, 0x000000C0, 0x1 }, ++ { 0x00002016, 0x00000097, 0x0 }, ++ { 0x80005012, 0x000000C0, 0x1 }, ++}; ++ ++/* Kabylake U */ ++static const struct ddi_buf_trans kbl_u_ddi_translations_dp[] = { ++ { 0x0000201B, 0x000000A1, 0x0 }, ++ { 0x00005012, 0x00000088, 0x0 }, ++ { 0x80007011, 0x000000CD, 0x3 }, ++ { 0x80009010, 0x000000C0, 0x3 }, ++ { 0x0000201B, 0x0000009D, 0x0 }, ++ { 0x80005012, 0x000000C0, 0x3 }, ++ { 0x80007011, 0x000000C0, 0x3 }, ++ { 0x00002016, 0x0000004F, 0x0 }, ++ { 0x80005012, 0x000000C0, 0x3 }, ++}; ++ ++/* Kabylake Y */ ++static const struct ddi_buf_trans kbl_y_ddi_translations_dp[] = { ++ { 0x00001017, 0x000000A1, 0x0 }, ++ { 0x00005012, 0x00000088, 0x0 }, ++ { 0x80007011, 0x000000CD, 0x3 }, ++ { 0x8000800F, 0x000000C0, 0x3 }, ++ { 0x00001017, 0x0000009D, 0x0 }, ++ { 0x80005012, 0x000000C0, 0x3 }, ++ { 0x80007011, 0x000000C0, 0x3 }, ++ { 0x00001017, 0x0000004C, 0x0 }, ++ { 0x80005012, 0x000000C0, 0x3 }, ++}; ++ ++/* ++ * Skylake/Kabylake H and S ++ * eDP 1.4 low vswing translation parameters ++ */ ++static const struct ddi_buf_trans skl_ddi_translations_edp[] = { ++ { 0x00000018, 0x000000A8, 0x0 }, ++ { 0x00004013, 0x000000A9, 0x0 }, ++ { 0x00007011, 0x000000A2, 0x0 }, ++ { 0x00009010, 0x0000009C, 0x0 }, ++ { 0x00000018, 0x000000A9, 0x0 }, ++ { 0x00006013, 0x000000A2, 0x0 }, ++ { 0x00007011, 0x000000A6, 0x0 }, ++ { 0x00000018, 0x000000AB, 0x0 }, ++ { 0x00007013, 0x0000009F, 0x0 }, ++ { 0x00000018, 0x000000DF, 0x0 }, ++}; ++ ++/* ++ * Skylake/Kabylake U ++ * eDP 1.4 low vswing translation parameters ++ */ ++static const struct ddi_buf_trans skl_u_ddi_translations_edp[] = { ++ { 0x00000018, 0x000000A8, 0x0 }, ++ { 0x00004013, 0x000000A9, 0x0 }, ++ { 0x00007011, 0x000000A2, 0x0 }, ++ { 0x00009010, 0x0000009C, 0x0 }, ++ { 0x00000018, 0x000000A9, 0x0 }, ++ { 0x00006013, 0x000000A2, 0x0 }, ++ { 0x00007011, 0x000000A6, 0x0 }, ++ { 0x00002016, 0x000000AB, 0x0 }, ++ { 0x00005013, 0x0000009F, 0x0 }, ++ { 0x00000018, 0x000000DF, 0x0 }, ++}; ++ ++/* ++ * Skylake/Kabylake Y ++ * eDP 1.4 low vswing translation parameters ++ */ ++static const struct ddi_buf_trans skl_y_ddi_translations_edp[] = { ++ { 0x00000018, 0x000000A8, 0x0 }, ++ { 0x00004013, 0x000000AB, 0x0 }, ++ { 0x00007011, 0x000000A4, 0x0 }, ++ { 0x00009010, 0x000000DF, 0x0 }, ++ { 0x00000018, 0x000000AA, 0x0 }, ++ { 0x00006013, 0x000000A4, 0x0 }, ++ { 0x00007011, 0x0000009D, 0x0 }, ++ { 0x00000018, 0x000000A0, 0x0 }, ++ { 0x00006012, 0x000000DF, 0x0 }, ++ { 0x00000018, 0x0000008A, 0x0 }, ++}; ++ ++/* Skylake/Kabylake U, H and S */ ++static const struct ddi_buf_trans skl_ddi_translations_hdmi[] = { ++ { 0x00000018, 0x000000AC, 0x0 }, ++ { 0x00005012, 0x0000009D, 0x0 }, ++ { 0x00007011, 0x00000088, 0x0 }, ++ { 0x00000018, 0x000000A1, 0x0 }, ++ { 0x00000018, 0x00000098, 0x0 }, ++ { 0x00004013, 0x00000088, 0x0 }, ++ { 0x80006012, 0x000000CD, 0x1 }, ++ { 0x00000018, 0x000000DF, 0x0 }, ++ { 0x80003015, 0x000000CD, 0x1 }, /* Default */ ++ { 0x80003015, 0x000000C0, 0x1 }, ++ { 0x80000018, 0x000000C0, 0x1 }, ++}; ++ ++/* Skylake/Kabylake Y */ ++static const struct ddi_buf_trans skl_y_ddi_translations_hdmi[] = { ++ { 0x00000018, 0x000000A1, 0x0 }, ++ { 0x00005012, 0x000000DF, 0x0 }, ++ { 0x80007011, 0x000000CB, 0x3 }, ++ { 0x00000018, 0x000000A4, 0x0 }, ++ { 0x00000018, 0x0000009D, 0x0 }, ++ { 0x00004013, 0x00000080, 0x0 }, ++ { 0x80006013, 0x000000C0, 0x3 }, ++ { 0x00000018, 0x0000008A, 0x0 }, ++ { 0x80003015, 0x000000C0, 0x3 }, /* Default */ ++ { 0x80003015, 0x000000C0, 0x3 }, ++ { 0x80000018, 0x000000C0, 0x3 }, ++}; ++ ++struct bxt_ddi_buf_trans { ++ u8 margin; /* swing value */ ++ u8 scale; /* scale value */ ++ u8 enable; /* scale enable */ ++ u8 deemphasis; ++}; ++ ++static const struct bxt_ddi_buf_trans bxt_ddi_translations_dp[] = { ++ /* Idx NT mV diff db */ ++ { 52, 0x9A, 0, 128, }, /* 0: 400 0 */ ++ { 78, 0x9A, 0, 85, }, /* 1: 400 3.5 */ ++ { 104, 0x9A, 0, 64, }, /* 2: 400 6 */ ++ { 154, 0x9A, 0, 43, }, /* 3: 400 9.5 */ ++ { 77, 0x9A, 0, 128, }, /* 4: 600 0 */ ++ { 116, 0x9A, 0, 85, }, /* 5: 600 3.5 */ ++ { 154, 0x9A, 0, 64, }, /* 6: 600 6 */ ++ { 102, 0x9A, 0, 128, }, /* 7: 800 0 */ ++ { 154, 0x9A, 0, 85, }, /* 8: 800 3.5 */ ++ { 154, 0x9A, 1, 128, }, /* 9: 1200 0 */ ++}; ++ ++static const struct bxt_ddi_buf_trans bxt_ddi_translations_edp[] = { ++ /* Idx NT mV diff db */ ++ { 26, 0, 0, 128, }, /* 0: 200 0 */ ++ { 38, 0, 0, 112, }, /* 1: 200 1.5 */ ++ { 48, 0, 0, 96, }, /* 2: 200 4 */ ++ { 54, 0, 0, 69, }, /* 3: 200 6 */ ++ { 32, 0, 0, 128, }, /* 4: 250 0 */ ++ { 48, 0, 0, 104, }, /* 5: 250 1.5 */ ++ { 54, 0, 0, 85, }, /* 6: 250 4 */ ++ { 43, 0, 0, 128, }, /* 7: 300 0 */ ++ { 54, 0, 0, 101, }, /* 8: 300 1.5 */ ++ { 48, 0, 0, 128, }, /* 9: 300 0 */ ++}; ++ ++/* BSpec has 2 recommended values - entries 0 and 8. ++ * Using the entry with higher vswing. ++ */ ++static const struct bxt_ddi_buf_trans bxt_ddi_translations_hdmi[] = { ++ /* Idx NT mV diff db */ ++ { 52, 0x9A, 0, 128, }, /* 0: 400 0 */ ++ { 52, 0x9A, 0, 85, }, /* 1: 400 3.5 */ ++ { 52, 0x9A, 0, 64, }, /* 2: 400 6 */ ++ { 42, 0x9A, 0, 43, }, /* 3: 400 9.5 */ ++ { 77, 0x9A, 0, 128, }, /* 4: 600 0 */ ++ { 77, 0x9A, 0, 85, }, /* 5: 600 3.5 */ ++ { 77, 0x9A, 0, 64, }, /* 6: 600 6 */ ++ { 102, 0x9A, 0, 128, }, /* 7: 800 0 */ ++ { 102, 0x9A, 0, 85, }, /* 8: 800 3.5 */ ++ { 154, 0x9A, 1, 128, }, /* 9: 1200 0 */ ++}; ++ ++struct cnl_ddi_buf_trans { ++ u8 dw2_swing_sel; ++ u8 dw7_n_scalar; ++ u8 dw4_cursor_coeff; ++ u8 dw4_post_cursor_2; ++ u8 dw4_post_cursor_1; ++}; ++ ++/* Voltage Swing Programming for VccIO 0.85V for DP */ ++static const struct cnl_ddi_buf_trans cnl_ddi_translations_dp_0_85V[] = { ++ /* NT mV Trans mV db */ ++ { 0xA, 0x5D, 0x3F, 0x00, 0x00 }, /* 350 350 0.0 */ ++ { 0xA, 0x6A, 0x38, 0x00, 0x07 }, /* 350 500 3.1 */ ++ { 0xB, 0x7A, 0x32, 0x00, 0x0D }, /* 350 700 6.0 */ ++ { 0x6, 0x7C, 0x2D, 0x00, 0x12 }, /* 350 900 8.2 */ ++ { 0xA, 0x69, 0x3F, 0x00, 0x00 }, /* 500 500 0.0 */ ++ { 0xB, 0x7A, 0x36, 0x00, 0x09 }, /* 500 700 2.9 */ ++ { 0x6, 0x7C, 0x30, 0x00, 0x0F }, /* 500 900 5.1 */ ++ { 0xB, 0x7D, 0x3C, 0x00, 0x03 }, /* 650 725 0.9 */ ++ { 0x6, 0x7C, 0x34, 0x00, 0x0B }, /* 600 900 3.5 */ ++ { 0x6, 0x7B, 0x3F, 0x00, 0x00 }, /* 900 900 0.0 */ ++}; ++ ++/* Voltage Swing Programming for VccIO 0.85V for HDMI */ ++static const struct cnl_ddi_buf_trans cnl_ddi_translations_hdmi_0_85V[] = { ++ /* NT mV Trans mV db */ ++ { 0xA, 0x60, 0x3F, 0x00, 0x00 }, /* 450 450 0.0 */ ++ { 0xB, 0x73, 0x36, 0x00, 0x09 }, /* 450 650 3.2 */ ++ { 0x6, 0x7F, 0x31, 0x00, 0x0E }, /* 450 850 5.5 */ ++ { 0xB, 0x73, 0x3F, 0x00, 0x00 }, /* 650 650 0.0 */ ++ { 0x6, 0x7F, 0x37, 0x00, 0x08 }, /* 650 850 2.3 */ ++ { 0x6, 0x7F, 0x3F, 0x00, 0x00 }, /* 850 850 0.0 */ ++ { 0x6, 0x7F, 0x35, 0x00, 0x0A }, /* 600 850 3.0 */ ++}; ++ ++/* Voltage Swing Programming for VccIO 0.85V for eDP */ ++static const struct cnl_ddi_buf_trans cnl_ddi_translations_edp_0_85V[] = { ++ /* NT mV Trans mV db */ ++ { 0xA, 0x66, 0x3A, 0x00, 0x05 }, /* 384 500 2.3 */ ++ { 0x0, 0x7F, 0x38, 0x00, 0x07 }, /* 153 200 2.3 */ ++ { 0x8, 0x7F, 0x38, 0x00, 0x07 }, /* 192 250 2.3 */ ++ { 0x1, 0x7F, 0x38, 0x00, 0x07 }, /* 230 300 2.3 */ ++ { 0x9, 0x7F, 0x38, 0x00, 0x07 }, /* 269 350 2.3 */ ++ { 0xA, 0x66, 0x3C, 0x00, 0x03 }, /* 446 500 1.0 */ ++ { 0xB, 0x70, 0x3C, 0x00, 0x03 }, /* 460 600 2.3 */ ++ { 0xC, 0x75, 0x3C, 0x00, 0x03 }, /* 537 700 2.3 */ ++ { 0x2, 0x7F, 0x3F, 0x00, 0x00 }, /* 400 400 0.0 */ ++}; ++ ++/* Voltage Swing Programming for VccIO 0.95V for DP */ ++static const struct cnl_ddi_buf_trans cnl_ddi_translations_dp_0_95V[] = { ++ /* NT mV Trans mV db */ ++ { 0xA, 0x5D, 0x3F, 0x00, 0x00 }, /* 350 350 0.0 */ ++ { 0xA, 0x6A, 0x38, 0x00, 0x07 }, /* 350 500 3.1 */ ++ { 0xB, 0x7A, 0x32, 0x00, 0x0D }, /* 350 700 6.0 */ ++ { 0x6, 0x7C, 0x2D, 0x00, 0x12 }, /* 350 900 8.2 */ ++ { 0xA, 0x69, 0x3F, 0x00, 0x00 }, /* 500 500 0.0 */ ++ { 0xB, 0x7A, 0x36, 0x00, 0x09 }, /* 500 700 2.9 */ ++ { 0x6, 0x7C, 0x30, 0x00, 0x0F }, /* 500 900 5.1 */ ++ { 0xB, 0x7D, 0x3C, 0x00, 0x03 }, /* 650 725 0.9 */ ++ { 0x6, 0x7C, 0x34, 0x00, 0x0B }, /* 600 900 3.5 */ ++ { 0x6, 0x7B, 0x3F, 0x00, 0x00 }, /* 900 900 0.0 */ ++}; ++ ++/* Voltage Swing Programming for VccIO 0.95V for HDMI */ ++static const struct cnl_ddi_buf_trans cnl_ddi_translations_hdmi_0_95V[] = { ++ /* NT mV Trans mV db */ ++ { 0xA, 0x5C, 0x3F, 0x00, 0x00 }, /* 400 400 0.0 */ ++ { 0xB, 0x69, 0x37, 0x00, 0x08 }, /* 400 600 3.5 */ ++ { 0x5, 0x76, 0x31, 0x00, 0x0E }, /* 400 800 6.0 */ ++ { 0xA, 0x5E, 0x3F, 0x00, 0x00 }, /* 450 450 0.0 */ ++ { 0xB, 0x69, 0x3F, 0x00, 0x00 }, /* 600 600 0.0 */ ++ { 0xB, 0x79, 0x35, 0x00, 0x0A }, /* 600 850 3.0 */ ++ { 0x6, 0x7D, 0x32, 0x00, 0x0D }, /* 600 1000 4.4 */ ++ { 0x5, 0x76, 0x3F, 0x00, 0x00 }, /* 800 800 0.0 */ ++ { 0x6, 0x7D, 0x39, 0x00, 0x06 }, /* 800 1000 1.9 */ ++ { 0x6, 0x7F, 0x39, 0x00, 0x06 }, /* 850 1050 1.8 */ ++ { 0x6, 0x7F, 0x3F, 0x00, 0x00 }, /* 1050 1050 0.0 */ ++}; ++ ++/* Voltage Swing Programming for VccIO 0.95V for eDP */ ++static const struct cnl_ddi_buf_trans cnl_ddi_translations_edp_0_95V[] = { ++ /* NT mV Trans mV db */ ++ { 0xA, 0x61, 0x3A, 0x00, 0x05 }, /* 384 500 2.3 */ ++ { 0x0, 0x7F, 0x38, 0x00, 0x07 }, /* 153 200 2.3 */ ++ { 0x8, 0x7F, 0x38, 0x00, 0x07 }, /* 192 250 2.3 */ ++ { 0x1, 0x7F, 0x38, 0x00, 0x07 }, /* 230 300 2.3 */ ++ { 0x9, 0x7F, 0x38, 0x00, 0x07 }, /* 269 350 2.3 */ ++ { 0xA, 0x61, 0x3C, 0x00, 0x03 }, /* 446 500 1.0 */ ++ { 0xB, 0x68, 0x39, 0x00, 0x06 }, /* 460 600 2.3 */ ++ { 0xC, 0x6E, 0x39, 0x00, 0x06 }, /* 537 700 2.3 */ ++ { 0x4, 0x7F, 0x3A, 0x00, 0x05 }, /* 460 600 2.3 */ ++ { 0x2, 0x7F, 0x3F, 0x00, 0x00 }, /* 400 400 0.0 */ ++}; ++ ++/* Voltage Swing Programming for VccIO 1.05V for DP */ ++static const struct cnl_ddi_buf_trans cnl_ddi_translations_dp_1_05V[] = { ++ /* NT mV Trans mV db */ ++ { 0xA, 0x58, 0x3F, 0x00, 0x00 }, /* 400 400 0.0 */ ++ { 0xB, 0x64, 0x37, 0x00, 0x08 }, /* 400 600 3.5 */ ++ { 0x5, 0x70, 0x31, 0x00, 0x0E }, /* 400 800 6.0 */ ++ { 0x6, 0x7F, 0x2C, 0x00, 0x13 }, /* 400 1050 8.4 */ ++ { 0xB, 0x64, 0x3F, 0x00, 0x00 }, /* 600 600 0.0 */ ++ { 0x5, 0x73, 0x35, 0x00, 0x0A }, /* 600 850 3.0 */ ++ { 0x6, 0x7F, 0x30, 0x00, 0x0F }, /* 550 1050 5.6 */ ++ { 0x5, 0x76, 0x3E, 0x00, 0x01 }, /* 850 900 0.5 */ ++ { 0x6, 0x7F, 0x36, 0x00, 0x09 }, /* 750 1050 2.9 */ ++ { 0x6, 0x7F, 0x3F, 0x00, 0x00 }, /* 1050 1050 0.0 */ ++}; ++ ++/* Voltage Swing Programming for VccIO 1.05V for HDMI */ ++static const struct cnl_ddi_buf_trans cnl_ddi_translations_hdmi_1_05V[] = { ++ /* NT mV Trans mV db */ ++ { 0xA, 0x58, 0x3F, 0x00, 0x00 }, /* 400 400 0.0 */ ++ { 0xB, 0x64, 0x37, 0x00, 0x08 }, /* 400 600 3.5 */ ++ { 0x5, 0x70, 0x31, 0x00, 0x0E }, /* 400 800 6.0 */ ++ { 0xA, 0x5B, 0x3F, 0x00, 0x00 }, /* 450 450 0.0 */ ++ { 0xB, 0x64, 0x3F, 0x00, 0x00 }, /* 600 600 0.0 */ ++ { 0x5, 0x73, 0x35, 0x00, 0x0A }, /* 600 850 3.0 */ ++ { 0x6, 0x7C, 0x32, 0x00, 0x0D }, /* 600 1000 4.4 */ ++ { 0x5, 0x70, 0x3F, 0x00, 0x00 }, /* 800 800 0.0 */ ++ { 0x6, 0x7C, 0x39, 0x00, 0x06 }, /* 800 1000 1.9 */ ++ { 0x6, 0x7F, 0x39, 0x00, 0x06 }, /* 850 1050 1.8 */ ++ { 0x6, 0x7F, 0x3F, 0x00, 0x00 }, /* 1050 1050 0.0 */ ++}; ++ ++/* Voltage Swing Programming for VccIO 1.05V for eDP */ ++static const struct cnl_ddi_buf_trans cnl_ddi_translations_edp_1_05V[] = { ++ /* NT mV Trans mV db */ ++ { 0xA, 0x5E, 0x3A, 0x00, 0x05 }, /* 384 500 2.3 */ ++ { 0x0, 0x7F, 0x38, 0x00, 0x07 }, /* 153 200 2.3 */ ++ { 0x8, 0x7F, 0x38, 0x00, 0x07 }, /* 192 250 2.3 */ ++ { 0x1, 0x7F, 0x38, 0x00, 0x07 }, /* 230 300 2.3 */ ++ { 0x9, 0x7F, 0x38, 0x00, 0x07 }, /* 269 350 2.3 */ ++ { 0xA, 0x5E, 0x3C, 0x00, 0x03 }, /* 446 500 1.0 */ ++ { 0xB, 0x64, 0x39, 0x00, 0x06 }, /* 460 600 2.3 */ ++ { 0xE, 0x6A, 0x39, 0x00, 0x06 }, /* 537 700 2.3 */ ++ { 0x2, 0x7F, 0x3F, 0x00, 0x00 }, /* 400 400 0.0 */ ++}; ++ ++/* icl_combo_phy_ddi_translations */ ++static const struct cnl_ddi_buf_trans icl_combo_phy_ddi_translations_dp_hbr2[] = { ++ /* NT mV Trans mV db */ ++ { 0xA, 0x35, 0x3F, 0x00, 0x00 }, /* 350 350 0.0 */ ++ { 0xA, 0x4F, 0x37, 0x00, 0x08 }, /* 350 500 3.1 */ ++ { 0xC, 0x71, 0x2F, 0x00, 0x10 }, /* 350 700 6.0 */ ++ { 0x6, 0x7F, 0x2B, 0x00, 0x14 }, /* 350 900 8.2 */ ++ { 0xA, 0x4C, 0x3F, 0x00, 0x00 }, /* 500 500 0.0 */ ++ { 0xC, 0x73, 0x34, 0x00, 0x0B }, /* 500 700 2.9 */ ++ { 0x6, 0x7F, 0x2F, 0x00, 0x10 }, /* 500 900 5.1 */ ++ { 0xC, 0x6C, 0x3C, 0x00, 0x03 }, /* 650 700 0.6 */ ++ { 0x6, 0x7F, 0x35, 0x00, 0x0A }, /* 600 900 3.5 */ ++ { 0x6, 0x7F, 0x3F, 0x00, 0x00 }, /* 900 900 0.0 */ ++}; ++ ++static const struct cnl_ddi_buf_trans icl_combo_phy_ddi_translations_edp_hbr2[] = { ++ /* NT mV Trans mV db */ ++ { 0x0, 0x7F, 0x3F, 0x00, 0x00 }, /* 200 200 0.0 */ ++ { 0x8, 0x7F, 0x38, 0x00, 0x07 }, /* 200 250 1.9 */ ++ { 0x1, 0x7F, 0x33, 0x00, 0x0C }, /* 200 300 3.5 */ ++ { 0x9, 0x7F, 0x31, 0x00, 0x0E }, /* 200 350 4.9 */ ++ { 0x8, 0x7F, 0x3F, 0x00, 0x00 }, /* 250 250 0.0 */ ++ { 0x1, 0x7F, 0x38, 0x00, 0x07 }, /* 250 300 1.6 */ ++ { 0x9, 0x7F, 0x35, 0x00, 0x0A }, /* 250 350 2.9 */ ++ { 0x1, 0x7F, 0x3F, 0x00, 0x00 }, /* 300 300 0.0 */ ++ { 0x9, 0x7F, 0x38, 0x00, 0x07 }, /* 300 350 1.3 */ ++ { 0x9, 0x7F, 0x3F, 0x00, 0x00 }, /* 350 350 0.0 */ ++}; ++ ++static const struct cnl_ddi_buf_trans icl_combo_phy_ddi_translations_edp_hbr3[] = { ++ /* NT mV Trans mV db */ ++ { 0xA, 0x35, 0x3F, 0x00, 0x00 }, /* 350 350 0.0 */ ++ { 0xA, 0x4F, 0x37, 0x00, 0x08 }, /* 350 500 3.1 */ ++ { 0xC, 0x71, 0x2F, 0x00, 0x10 }, /* 350 700 6.0 */ ++ { 0x6, 0x7F, 0x2B, 0x00, 0x14 }, /* 350 900 8.2 */ ++ { 0xA, 0x4C, 0x3F, 0x00, 0x00 }, /* 500 500 0.0 */ ++ { 0xC, 0x73, 0x34, 0x00, 0x0B }, /* 500 700 2.9 */ ++ { 0x6, 0x7F, 0x2F, 0x00, 0x10 }, /* 500 900 5.1 */ ++ { 0xC, 0x6C, 0x3C, 0x00, 0x03 }, /* 650 700 0.6 */ ++ { 0x6, 0x7F, 0x35, 0x00, 0x0A }, /* 600 900 3.5 */ ++ { 0x6, 0x7F, 0x3F, 0x00, 0x00 }, /* 900 900 0.0 */ ++}; ++ ++static const struct cnl_ddi_buf_trans icl_combo_phy_ddi_translations_hdmi[] = { ++ /* NT mV Trans mV db */ ++ { 0xA, 0x60, 0x3F, 0x00, 0x00 }, /* 450 450 0.0 */ ++ { 0xB, 0x73, 0x36, 0x00, 0x09 }, /* 450 650 3.2 */ ++ { 0x6, 0x7F, 0x31, 0x00, 0x0E }, /* 450 850 5.5 */ ++ { 0xB, 0x73, 0x3F, 0x00, 0x00 }, /* 650 650 0.0 ALS */ ++ { 0x6, 0x7F, 0x37, 0x00, 0x08 }, /* 650 850 2.3 */ ++ { 0x6, 0x7F, 0x3F, 0x00, 0x00 }, /* 850 850 0.0 */ ++ { 0x6, 0x7F, 0x35, 0x00, 0x0A }, /* 600 850 3.0 */ ++}; ++ ++struct icl_mg_phy_ddi_buf_trans { ++ u32 cri_txdeemph_override_5_0; ++ u32 cri_txdeemph_override_11_6; ++ u32 cri_txdeemph_override_17_12; ++}; ++ ++static const struct icl_mg_phy_ddi_buf_trans icl_mg_phy_ddi_translations[] = { ++ /* Voltage swing pre-emphasis */ ++ { 0x0, 0x1B, 0x00 }, /* 0 0 */ ++ { 0x0, 0x23, 0x08 }, /* 0 1 */ ++ { 0x0, 0x2D, 0x12 }, /* 0 2 */ ++ { 0x0, 0x00, 0x00 }, /* 0 3 */ ++ { 0x0, 0x23, 0x00 }, /* 1 0 */ ++ { 0x0, 0x2B, 0x09 }, /* 1 1 */ ++ { 0x0, 0x2E, 0x11 }, /* 1 2 */ ++ { 0x0, 0x2F, 0x00 }, /* 2 0 */ ++ { 0x0, 0x33, 0x0C }, /* 2 1 */ ++ { 0x0, 0x00, 0x00 }, /* 3 0 */ ++}; ++ ++static const struct ddi_buf_trans * ++bdw_get_buf_trans_edp(struct drm_i915_private *dev_priv, int *n_entries) ++{ ++ if (dev_priv->vbt.edp.low_vswing) { ++ *n_entries = ARRAY_SIZE(bdw_ddi_translations_edp); ++ return bdw_ddi_translations_edp; ++ } else { ++ *n_entries = ARRAY_SIZE(bdw_ddi_translations_dp); ++ return bdw_ddi_translations_dp; ++ } ++} ++ ++static const struct ddi_buf_trans * ++skl_get_buf_trans_dp(struct drm_i915_private *dev_priv, int *n_entries) ++{ ++ if (IS_SKL_ULX(dev_priv)) { ++ *n_entries = ARRAY_SIZE(skl_y_ddi_translations_dp); ++ return skl_y_ddi_translations_dp; ++ } else if (IS_SKL_ULT(dev_priv)) { ++ *n_entries = ARRAY_SIZE(skl_u_ddi_translations_dp); ++ return skl_u_ddi_translations_dp; ++ } else { ++ *n_entries = ARRAY_SIZE(skl_ddi_translations_dp); ++ return skl_ddi_translations_dp; ++ } ++} ++ ++static const struct ddi_buf_trans * ++kbl_get_buf_trans_dp(struct drm_i915_private *dev_priv, int *n_entries) ++{ ++ if (IS_KBL_ULX(dev_priv) || IS_AML_ULX(dev_priv)) { ++ *n_entries = ARRAY_SIZE(kbl_y_ddi_translations_dp); ++ return kbl_y_ddi_translations_dp; ++ } else if (IS_KBL_ULT(dev_priv) || IS_CFL_ULT(dev_priv)) { ++ *n_entries = ARRAY_SIZE(kbl_u_ddi_translations_dp); ++ return kbl_u_ddi_translations_dp; ++ } else { ++ *n_entries = ARRAY_SIZE(kbl_ddi_translations_dp); ++ return kbl_ddi_translations_dp; ++ } ++} ++ ++static const struct ddi_buf_trans * ++skl_get_buf_trans_edp(struct drm_i915_private *dev_priv, int *n_entries) ++{ ++ if (dev_priv->vbt.edp.low_vswing) { ++ if (IS_SKL_ULX(dev_priv) || IS_KBL_ULX(dev_priv) || IS_AML_ULX(dev_priv)) { ++ *n_entries = ARRAY_SIZE(skl_y_ddi_translations_edp); ++ return skl_y_ddi_translations_edp; ++ } else if (IS_SKL_ULT(dev_priv) || IS_KBL_ULT(dev_priv) || ++ IS_CFL_ULT(dev_priv)) { ++ *n_entries = ARRAY_SIZE(skl_u_ddi_translations_edp); ++ return skl_u_ddi_translations_edp; ++ } else { ++ *n_entries = ARRAY_SIZE(skl_ddi_translations_edp); ++ return skl_ddi_translations_edp; ++ } ++ } ++ ++ if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) ++ return kbl_get_buf_trans_dp(dev_priv, n_entries); ++ else ++ return skl_get_buf_trans_dp(dev_priv, n_entries); ++} ++ ++static const struct ddi_buf_trans * ++skl_get_buf_trans_hdmi(struct drm_i915_private *dev_priv, int *n_entries) ++{ ++ if (IS_SKL_ULX(dev_priv) || IS_KBL_ULX(dev_priv) || IS_AML_ULX(dev_priv)) { ++ *n_entries = ARRAY_SIZE(skl_y_ddi_translations_hdmi); ++ return skl_y_ddi_translations_hdmi; ++ } else { ++ *n_entries = ARRAY_SIZE(skl_ddi_translations_hdmi); ++ return skl_ddi_translations_hdmi; ++ } ++} ++ ++static int skl_buf_trans_num_entries(enum port port, int n_entries) ++{ ++ /* Only DDIA and DDIE can select the 10th register with DP */ ++ if (port == PORT_A || port == PORT_E) ++ return min(n_entries, 10); ++ else ++ return min(n_entries, 9); ++} ++ ++static const struct ddi_buf_trans * ++intel_ddi_get_buf_trans_dp(struct drm_i915_private *dev_priv, ++ enum port port, int *n_entries) ++{ ++ if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) { ++ const struct ddi_buf_trans *ddi_translations = ++ kbl_get_buf_trans_dp(dev_priv, n_entries); ++ *n_entries = skl_buf_trans_num_entries(port, *n_entries); ++ return ddi_translations; ++ } else if (IS_SKYLAKE(dev_priv)) { ++ const struct ddi_buf_trans *ddi_translations = ++ skl_get_buf_trans_dp(dev_priv, n_entries); ++ *n_entries = skl_buf_trans_num_entries(port, *n_entries); ++ return ddi_translations; ++ } else if (IS_BROADWELL(dev_priv)) { ++ *n_entries = ARRAY_SIZE(bdw_ddi_translations_dp); ++ return bdw_ddi_translations_dp; ++ } else if (IS_HASWELL(dev_priv)) { ++ *n_entries = ARRAY_SIZE(hsw_ddi_translations_dp); ++ return hsw_ddi_translations_dp; ++ } ++ ++ *n_entries = 0; ++ return NULL; ++} ++ ++static const struct ddi_buf_trans * ++intel_ddi_get_buf_trans_edp(struct drm_i915_private *dev_priv, ++ enum port port, int *n_entries) ++{ ++ if (IS_GEN9_BC(dev_priv)) { ++ const struct ddi_buf_trans *ddi_translations = ++ skl_get_buf_trans_edp(dev_priv, n_entries); ++ *n_entries = skl_buf_trans_num_entries(port, *n_entries); ++ return ddi_translations; ++ } else if (IS_BROADWELL(dev_priv)) { ++ return bdw_get_buf_trans_edp(dev_priv, n_entries); ++ } else if (IS_HASWELL(dev_priv)) { ++ *n_entries = ARRAY_SIZE(hsw_ddi_translations_dp); ++ return hsw_ddi_translations_dp; ++ } ++ ++ *n_entries = 0; ++ return NULL; ++} ++ ++static const struct ddi_buf_trans * ++intel_ddi_get_buf_trans_fdi(struct drm_i915_private *dev_priv, ++ int *n_entries) ++{ ++ if (IS_BROADWELL(dev_priv)) { ++ *n_entries = ARRAY_SIZE(bdw_ddi_translations_fdi); ++ return bdw_ddi_translations_fdi; ++ } else if (IS_HASWELL(dev_priv)) { ++ *n_entries = ARRAY_SIZE(hsw_ddi_translations_fdi); ++ return hsw_ddi_translations_fdi; ++ } ++ ++ *n_entries = 0; ++ return NULL; ++} ++ ++static const struct ddi_buf_trans * ++intel_ddi_get_buf_trans_hdmi(struct drm_i915_private *dev_priv, ++ int *n_entries) ++{ ++ if (IS_GEN9_BC(dev_priv)) { ++ return skl_get_buf_trans_hdmi(dev_priv, n_entries); ++ } else if (IS_BROADWELL(dev_priv)) { ++ *n_entries = ARRAY_SIZE(bdw_ddi_translations_hdmi); ++ return bdw_ddi_translations_hdmi; ++ } else if (IS_HASWELL(dev_priv)) { ++ *n_entries = ARRAY_SIZE(hsw_ddi_translations_hdmi); ++ return hsw_ddi_translations_hdmi; ++ } ++ ++ *n_entries = 0; ++ return NULL; ++} ++ ++static const struct bxt_ddi_buf_trans * ++bxt_get_buf_trans_dp(struct drm_i915_private *dev_priv, int *n_entries) ++{ ++ *n_entries = ARRAY_SIZE(bxt_ddi_translations_dp); ++ return bxt_ddi_translations_dp; ++} ++ ++static const struct bxt_ddi_buf_trans * ++bxt_get_buf_trans_edp(struct drm_i915_private *dev_priv, int *n_entries) ++{ ++ if (dev_priv->vbt.edp.low_vswing) { ++ *n_entries = ARRAY_SIZE(bxt_ddi_translations_edp); ++ return bxt_ddi_translations_edp; ++ } ++ ++ return bxt_get_buf_trans_dp(dev_priv, n_entries); ++} ++ ++static const struct bxt_ddi_buf_trans * ++bxt_get_buf_trans_hdmi(struct drm_i915_private *dev_priv, int *n_entries) ++{ ++ *n_entries = ARRAY_SIZE(bxt_ddi_translations_hdmi); ++ return bxt_ddi_translations_hdmi; ++} ++ ++static const struct cnl_ddi_buf_trans * ++cnl_get_buf_trans_hdmi(struct drm_i915_private *dev_priv, int *n_entries) ++{ ++ u32 voltage = I915_READ(CNL_PORT_COMP_DW3) & VOLTAGE_INFO_MASK; ++ ++ if (voltage == VOLTAGE_INFO_0_85V) { ++ *n_entries = ARRAY_SIZE(cnl_ddi_translations_hdmi_0_85V); ++ return cnl_ddi_translations_hdmi_0_85V; ++ } else if (voltage == VOLTAGE_INFO_0_95V) { ++ *n_entries = ARRAY_SIZE(cnl_ddi_translations_hdmi_0_95V); ++ return cnl_ddi_translations_hdmi_0_95V; ++ } else if (voltage == VOLTAGE_INFO_1_05V) { ++ *n_entries = ARRAY_SIZE(cnl_ddi_translations_hdmi_1_05V); ++ return cnl_ddi_translations_hdmi_1_05V; ++ } else { ++ *n_entries = 1; /* shut up gcc */ ++ MISSING_CASE(voltage); ++ } ++ return NULL; ++} ++ ++static const struct cnl_ddi_buf_trans * ++cnl_get_buf_trans_dp(struct drm_i915_private *dev_priv, int *n_entries) ++{ ++ u32 voltage = I915_READ(CNL_PORT_COMP_DW3) & VOLTAGE_INFO_MASK; ++ ++ if (voltage == VOLTAGE_INFO_0_85V) { ++ *n_entries = ARRAY_SIZE(cnl_ddi_translations_dp_0_85V); ++ return cnl_ddi_translations_dp_0_85V; ++ } else if (voltage == VOLTAGE_INFO_0_95V) { ++ *n_entries = ARRAY_SIZE(cnl_ddi_translations_dp_0_95V); ++ return cnl_ddi_translations_dp_0_95V; ++ } else if (voltage == VOLTAGE_INFO_1_05V) { ++ *n_entries = ARRAY_SIZE(cnl_ddi_translations_dp_1_05V); ++ return cnl_ddi_translations_dp_1_05V; ++ } else { ++ *n_entries = 1; /* shut up gcc */ ++ MISSING_CASE(voltage); ++ } ++ return NULL; ++} ++ ++static const struct cnl_ddi_buf_trans * ++cnl_get_buf_trans_edp(struct drm_i915_private *dev_priv, int *n_entries) ++{ ++ u32 voltage = I915_READ(CNL_PORT_COMP_DW3) & VOLTAGE_INFO_MASK; ++ ++ if (dev_priv->vbt.edp.low_vswing) { ++ if (voltage == VOLTAGE_INFO_0_85V) { ++ *n_entries = ARRAY_SIZE(cnl_ddi_translations_edp_0_85V); ++ return cnl_ddi_translations_edp_0_85V; ++ } else if (voltage == VOLTAGE_INFO_0_95V) { ++ *n_entries = ARRAY_SIZE(cnl_ddi_translations_edp_0_95V); ++ return cnl_ddi_translations_edp_0_95V; ++ } else if (voltage == VOLTAGE_INFO_1_05V) { ++ *n_entries = ARRAY_SIZE(cnl_ddi_translations_edp_1_05V); ++ return cnl_ddi_translations_edp_1_05V; ++ } else { ++ *n_entries = 1; /* shut up gcc */ ++ MISSING_CASE(voltage); ++ } ++ return NULL; ++ } else { ++ return cnl_get_buf_trans_dp(dev_priv, n_entries); ++ } ++} ++ ++static const struct cnl_ddi_buf_trans * ++icl_get_combo_buf_trans(struct drm_i915_private *dev_priv, enum port port, ++ int type, int rate, int *n_entries) ++{ ++ if (type == INTEL_OUTPUT_HDMI) { ++ *n_entries = ARRAY_SIZE(icl_combo_phy_ddi_translations_hdmi); ++ return icl_combo_phy_ddi_translations_hdmi; ++ } else if (rate > 540000 && type == INTEL_OUTPUT_EDP) { ++ *n_entries = ARRAY_SIZE(icl_combo_phy_ddi_translations_edp_hbr3); ++ return icl_combo_phy_ddi_translations_edp_hbr3; ++ } else if (type == INTEL_OUTPUT_EDP && dev_priv->vbt.edp.low_vswing) { ++ *n_entries = ARRAY_SIZE(icl_combo_phy_ddi_translations_edp_hbr2); ++ return icl_combo_phy_ddi_translations_edp_hbr2; ++ } ++ ++ *n_entries = ARRAY_SIZE(icl_combo_phy_ddi_translations_dp_hbr2); ++ return icl_combo_phy_ddi_translations_dp_hbr2; ++} ++ ++static int intel_ddi_hdmi_level(struct drm_i915_private *dev_priv, enum port port) ++{ ++ int n_entries, level, default_entry; ++ ++ level = dev_priv->vbt.ddi_port_info[port].hdmi_level_shift; ++ ++ if (INTEL_GEN(dev_priv) >= 11) { ++ if (intel_port_is_combophy(dev_priv, port)) ++ icl_get_combo_buf_trans(dev_priv, port, INTEL_OUTPUT_HDMI, ++ 0, &n_entries); ++ else ++ n_entries = ARRAY_SIZE(icl_mg_phy_ddi_translations); ++ default_entry = n_entries - 1; ++ } else if (IS_CANNONLAKE(dev_priv)) { ++ cnl_get_buf_trans_hdmi(dev_priv, &n_entries); ++ default_entry = n_entries - 1; ++ } else if (IS_GEN9_LP(dev_priv)) { ++ bxt_get_buf_trans_hdmi(dev_priv, &n_entries); ++ default_entry = n_entries - 1; ++ } else if (IS_GEN9_BC(dev_priv)) { ++ intel_ddi_get_buf_trans_hdmi(dev_priv, &n_entries); ++ default_entry = 8; ++ } else if (IS_BROADWELL(dev_priv)) { ++ intel_ddi_get_buf_trans_hdmi(dev_priv, &n_entries); ++ default_entry = 7; ++ } else if (IS_HASWELL(dev_priv)) { ++ intel_ddi_get_buf_trans_hdmi(dev_priv, &n_entries); ++ default_entry = 6; ++ } else { ++ WARN(1, "ddi translation table missing\n"); ++ return 0; ++ } ++ ++ /* Choose a good default if VBT is badly populated */ ++ if (level == HDMI_LEVEL_SHIFT_UNKNOWN || level >= n_entries) ++ level = default_entry; ++ ++ if (WARN_ON_ONCE(n_entries == 0)) ++ return 0; ++ if (WARN_ON_ONCE(level >= n_entries)) ++ level = n_entries - 1; ++ ++ return level; ++} ++ ++/* ++ * Starting with Haswell, DDI port buffers must be programmed with correct ++ * values in advance. This function programs the correct values for ++ * DP/eDP/FDI use cases. ++ */ ++static void intel_prepare_dp_ddi_buffers(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ u32 iboost_bit = 0; ++ int i, n_entries; ++ enum port port = encoder->port; ++ const struct ddi_buf_trans *ddi_translations; ++ ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) ++ ddi_translations = intel_ddi_get_buf_trans_fdi(dev_priv, ++ &n_entries); ++ else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP)) ++ ddi_translations = intel_ddi_get_buf_trans_edp(dev_priv, port, ++ &n_entries); ++ else ++ ddi_translations = intel_ddi_get_buf_trans_dp(dev_priv, port, ++ &n_entries); ++ ++ /* If we're boosting the current, set bit 31 of trans1 */ ++ if (IS_GEN9_BC(dev_priv) && ++ dev_priv->vbt.ddi_port_info[port].dp_boost_level) ++ iboost_bit = DDI_BUF_BALANCE_LEG_ENABLE; ++ ++ for (i = 0; i < n_entries; i++) { ++ I915_WRITE(DDI_BUF_TRANS_LO(port, i), ++ ddi_translations[i].trans1 | iboost_bit); ++ I915_WRITE(DDI_BUF_TRANS_HI(port, i), ++ ddi_translations[i].trans2); ++ } ++} ++ ++/* ++ * Starting with Haswell, DDI port buffers must be programmed with correct ++ * values in advance. This function programs the correct values for ++ * HDMI/DVI use cases. ++ */ ++static void intel_prepare_hdmi_ddi_buffers(struct intel_encoder *encoder, ++ int level) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ u32 iboost_bit = 0; ++ int n_entries; ++ enum port port = encoder->port; ++ const struct ddi_buf_trans *ddi_translations; ++ ++ ddi_translations = intel_ddi_get_buf_trans_hdmi(dev_priv, &n_entries); ++ ++ if (WARN_ON_ONCE(!ddi_translations)) ++ return; ++ if (WARN_ON_ONCE(level >= n_entries)) ++ level = n_entries - 1; ++ ++ /* If we're boosting the current, set bit 31 of trans1 */ ++ if (IS_GEN9_BC(dev_priv) && ++ dev_priv->vbt.ddi_port_info[port].hdmi_boost_level) ++ iboost_bit = DDI_BUF_BALANCE_LEG_ENABLE; ++ ++ /* Entry 9 is for HDMI: */ ++ I915_WRITE(DDI_BUF_TRANS_LO(port, 9), ++ ddi_translations[level].trans1 | iboost_bit); ++ I915_WRITE(DDI_BUF_TRANS_HI(port, 9), ++ ddi_translations[level].trans2); ++} ++ ++static void intel_wait_ddi_buf_idle(struct drm_i915_private *dev_priv, ++ enum port port) ++{ ++ i915_reg_t reg = DDI_BUF_CTL(port); ++ int i; ++ ++ for (i = 0; i < 16; i++) { ++ udelay(1); ++ if (I915_READ(reg) & DDI_BUF_IS_IDLE) ++ return; ++ } ++ DRM_ERROR("Timeout waiting for DDI BUF %c idle bit\n", port_name(port)); ++} ++ ++static u32 hsw_pll_to_ddi_pll_sel(const struct intel_shared_dpll *pll) ++{ ++ switch (pll->info->id) { ++ case DPLL_ID_WRPLL1: ++ return PORT_CLK_SEL_WRPLL1; ++ case DPLL_ID_WRPLL2: ++ return PORT_CLK_SEL_WRPLL2; ++ case DPLL_ID_SPLL: ++ return PORT_CLK_SEL_SPLL; ++ case DPLL_ID_LCPLL_810: ++ return PORT_CLK_SEL_LCPLL_810; ++ case DPLL_ID_LCPLL_1350: ++ return PORT_CLK_SEL_LCPLL_1350; ++ case DPLL_ID_LCPLL_2700: ++ return PORT_CLK_SEL_LCPLL_2700; ++ default: ++ MISSING_CASE(pll->info->id); ++ return PORT_CLK_SEL_NONE; ++ } ++} ++ ++static u32 icl_pll_to_ddi_clk_sel(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state) ++{ ++ const struct intel_shared_dpll *pll = crtc_state->shared_dpll; ++ int clock = crtc_state->port_clock; ++ const enum intel_dpll_id id = pll->info->id; ++ ++ switch (id) { ++ default: ++ /* ++ * DPLL_ID_ICL_DPLL0 and DPLL_ID_ICL_DPLL1 should not be used ++ * here, so do warn if this get passed in ++ */ ++ MISSING_CASE(id); ++ return DDI_CLK_SEL_NONE; ++ case DPLL_ID_ICL_TBTPLL: ++ switch (clock) { ++ case 162000: ++ return DDI_CLK_SEL_TBT_162; ++ case 270000: ++ return DDI_CLK_SEL_TBT_270; ++ case 540000: ++ return DDI_CLK_SEL_TBT_540; ++ case 810000: ++ return DDI_CLK_SEL_TBT_810; ++ default: ++ MISSING_CASE(clock); ++ return DDI_CLK_SEL_NONE; ++ } ++ case DPLL_ID_ICL_MGPLL1: ++ case DPLL_ID_ICL_MGPLL2: ++ case DPLL_ID_ICL_MGPLL3: ++ case DPLL_ID_ICL_MGPLL4: ++ return DDI_CLK_SEL_MG; ++ } ++} ++ ++/* Starting with Haswell, different DDI ports can work in FDI mode for ++ * connection to the PCH-located connectors. For this, it is necessary to train ++ * both the DDI port and PCH receiver for the desired DDI buffer settings. ++ * ++ * The recommended port to work in FDI mode is DDI E, which we use here. Also, ++ * please note that when FDI mode is active on DDI E, it shares 2 lines with ++ * DDI A (which is used for eDP) ++ */ ++ ++void hsw_fdi_link_train(struct intel_crtc *crtc, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_encoder *encoder; ++ u32 temp, i, rx_ctl_val, ddi_pll_sel; ++ ++ for_each_encoder_on_crtc(dev, &crtc->base, encoder) { ++ WARN_ON(encoder->type != INTEL_OUTPUT_ANALOG); ++ intel_prepare_dp_ddi_buffers(encoder, crtc_state); ++ } ++ ++ /* Set the FDI_RX_MISC pwrdn lanes and the 2 workarounds listed at the ++ * mode set "sequence for CRT port" document: ++ * - TP1 to TP2 time with the default value ++ * - FDI delay to 90h ++ * ++ * WaFDIAutoLinkSetTimingOverrride:hsw ++ */ ++ I915_WRITE(FDI_RX_MISC(PIPE_A), FDI_RX_PWRDN_LANE1_VAL(2) | ++ FDI_RX_PWRDN_LANE0_VAL(2) | ++ FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90); ++ ++ /* Enable the PCH Receiver FDI PLL */ ++ rx_ctl_val = dev_priv->fdi_rx_config | FDI_RX_ENHANCE_FRAME_ENABLE | ++ FDI_RX_PLL_ENABLE | ++ FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes); ++ I915_WRITE(FDI_RX_CTL(PIPE_A), rx_ctl_val); ++ POSTING_READ(FDI_RX_CTL(PIPE_A)); ++ udelay(220); ++ ++ /* Switch from Rawclk to PCDclk */ ++ rx_ctl_val |= FDI_PCDCLK; ++ I915_WRITE(FDI_RX_CTL(PIPE_A), rx_ctl_val); ++ ++ /* Configure Port Clock Select */ ++ ddi_pll_sel = hsw_pll_to_ddi_pll_sel(crtc_state->shared_dpll); ++ I915_WRITE(PORT_CLK_SEL(PORT_E), ddi_pll_sel); ++ WARN_ON(ddi_pll_sel != PORT_CLK_SEL_SPLL); ++ ++ /* Start the training iterating through available voltages and emphasis, ++ * testing each value twice. */ ++ for (i = 0; i < ARRAY_SIZE(hsw_ddi_translations_fdi) * 2; i++) { ++ /* Configure DP_TP_CTL with auto-training */ ++ I915_WRITE(DP_TP_CTL(PORT_E), ++ DP_TP_CTL_FDI_AUTOTRAIN | ++ DP_TP_CTL_ENHANCED_FRAME_ENABLE | ++ DP_TP_CTL_LINK_TRAIN_PAT1 | ++ DP_TP_CTL_ENABLE); ++ ++ /* Configure and enable DDI_BUF_CTL for DDI E with next voltage. ++ * DDI E does not support port reversal, the functionality is ++ * achieved on the PCH side in FDI_RX_CTL, so no need to set the ++ * port reversal bit */ ++ I915_WRITE(DDI_BUF_CTL(PORT_E), ++ DDI_BUF_CTL_ENABLE | ++ ((crtc_state->fdi_lanes - 1) << 1) | ++ DDI_BUF_TRANS_SELECT(i / 2)); ++ POSTING_READ(DDI_BUF_CTL(PORT_E)); ++ ++ udelay(600); ++ ++ /* Program PCH FDI Receiver TU */ ++ I915_WRITE(FDI_RX_TUSIZE1(PIPE_A), TU_SIZE(64)); ++ ++ /* Enable PCH FDI Receiver with auto-training */ ++ rx_ctl_val |= FDI_RX_ENABLE | FDI_LINK_TRAIN_AUTO; ++ I915_WRITE(FDI_RX_CTL(PIPE_A), rx_ctl_val); ++ POSTING_READ(FDI_RX_CTL(PIPE_A)); ++ ++ /* Wait for FDI receiver lane calibration */ ++ udelay(30); ++ ++ /* Unset FDI_RX_MISC pwrdn lanes */ ++ temp = I915_READ(FDI_RX_MISC(PIPE_A)); ++ temp &= ~(FDI_RX_PWRDN_LANE1_MASK | FDI_RX_PWRDN_LANE0_MASK); ++ I915_WRITE(FDI_RX_MISC(PIPE_A), temp); ++ POSTING_READ(FDI_RX_MISC(PIPE_A)); ++ ++ /* Wait for FDI auto training time */ ++ udelay(5); ++ ++ temp = I915_READ(DP_TP_STATUS(PORT_E)); ++ if (temp & DP_TP_STATUS_AUTOTRAIN_DONE) { ++ DRM_DEBUG_KMS("FDI link training done on step %d\n", i); ++ break; ++ } ++ ++ /* ++ * Leave things enabled even if we failed to train FDI. ++ * Results in less fireworks from the state checker. ++ */ ++ if (i == ARRAY_SIZE(hsw_ddi_translations_fdi) * 2 - 1) { ++ DRM_ERROR("FDI link training failed!\n"); ++ break; ++ } ++ ++ rx_ctl_val &= ~FDI_RX_ENABLE; ++ I915_WRITE(FDI_RX_CTL(PIPE_A), rx_ctl_val); ++ POSTING_READ(FDI_RX_CTL(PIPE_A)); ++ ++ temp = I915_READ(DDI_BUF_CTL(PORT_E)); ++ temp &= ~DDI_BUF_CTL_ENABLE; ++ I915_WRITE(DDI_BUF_CTL(PORT_E), temp); ++ POSTING_READ(DDI_BUF_CTL(PORT_E)); ++ ++ /* Disable DP_TP_CTL and FDI_RX_CTL and retry */ ++ temp = I915_READ(DP_TP_CTL(PORT_E)); ++ temp &= ~(DP_TP_CTL_ENABLE | DP_TP_CTL_LINK_TRAIN_MASK); ++ temp |= DP_TP_CTL_LINK_TRAIN_PAT1; ++ I915_WRITE(DP_TP_CTL(PORT_E), temp); ++ POSTING_READ(DP_TP_CTL(PORT_E)); ++ ++ intel_wait_ddi_buf_idle(dev_priv, PORT_E); ++ ++ /* Reset FDI_RX_MISC pwrdn lanes */ ++ temp = I915_READ(FDI_RX_MISC(PIPE_A)); ++ temp &= ~(FDI_RX_PWRDN_LANE1_MASK | FDI_RX_PWRDN_LANE0_MASK); ++ temp |= FDI_RX_PWRDN_LANE1_VAL(2) | FDI_RX_PWRDN_LANE0_VAL(2); ++ I915_WRITE(FDI_RX_MISC(PIPE_A), temp); ++ POSTING_READ(FDI_RX_MISC(PIPE_A)); ++ } ++ ++ /* Enable normal pixel sending for FDI */ ++ I915_WRITE(DP_TP_CTL(PORT_E), ++ DP_TP_CTL_FDI_AUTOTRAIN | ++ DP_TP_CTL_LINK_TRAIN_NORMAL | ++ DP_TP_CTL_ENHANCED_FRAME_ENABLE | ++ DP_TP_CTL_ENABLE); ++} ++ ++static void intel_ddi_init_dp_buf_reg(struct intel_encoder *encoder) ++{ ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ struct intel_digital_port *intel_dig_port = ++ enc_to_dig_port(&encoder->base); ++ ++ intel_dp->DP = intel_dig_port->saved_port_bits | ++ DDI_BUF_CTL_ENABLE | DDI_BUF_TRANS_SELECT(0); ++ intel_dp->DP |= DDI_PORT_WIDTH(intel_dp->lane_count); ++} ++ ++static struct intel_encoder * ++intel_ddi_get_crtc_encoder(struct intel_crtc *crtc) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct intel_encoder *encoder, *ret = NULL; ++ int num_encoders = 0; ++ ++ for_each_encoder_on_crtc(dev, &crtc->base, encoder) { ++ ret = encoder; ++ num_encoders++; ++ } ++ ++ if (num_encoders != 1) ++ WARN(1, "%d encoders on crtc for pipe %c\n", num_encoders, ++ pipe_name(crtc->pipe)); ++ ++ BUG_ON(ret == NULL); ++ return ret; ++} ++ ++#define LC_FREQ 2700 ++ ++static int hsw_ddi_calc_wrpll_link(struct drm_i915_private *dev_priv, ++ i915_reg_t reg) ++{ ++ int refclk = LC_FREQ; ++ int n, p, r; ++ u32 wrpll; ++ ++ wrpll = I915_READ(reg); ++ switch (wrpll & WRPLL_PLL_REF_MASK) { ++ case WRPLL_PLL_SSC: ++ case WRPLL_PLL_NON_SSC: ++ /* ++ * We could calculate spread here, but our checking ++ * code only cares about 5% accuracy, and spread is a max of ++ * 0.5% downspread. ++ */ ++ refclk = 135; ++ break; ++ case WRPLL_PLL_LCPLL: ++ refclk = LC_FREQ; ++ break; ++ default: ++ WARN(1, "bad wrpll refclk\n"); ++ return 0; ++ } ++ ++ r = wrpll & WRPLL_DIVIDER_REF_MASK; ++ p = (wrpll & WRPLL_DIVIDER_POST_MASK) >> WRPLL_DIVIDER_POST_SHIFT; ++ n = (wrpll & WRPLL_DIVIDER_FB_MASK) >> WRPLL_DIVIDER_FB_SHIFT; ++ ++ /* Convert to KHz, p & r have a fixed point portion */ ++ return (refclk * n * 100) / (p * r); ++} ++ ++static int skl_calc_wrpll_link(const struct intel_dpll_hw_state *pll_state) ++{ ++ u32 p0, p1, p2, dco_freq; ++ ++ p0 = pll_state->cfgcr2 & DPLL_CFGCR2_PDIV_MASK; ++ p2 = pll_state->cfgcr2 & DPLL_CFGCR2_KDIV_MASK; ++ ++ if (pll_state->cfgcr2 & DPLL_CFGCR2_QDIV_MODE(1)) ++ p1 = (pll_state->cfgcr2 & DPLL_CFGCR2_QDIV_RATIO_MASK) >> 8; ++ else ++ p1 = 1; ++ ++ ++ switch (p0) { ++ case DPLL_CFGCR2_PDIV_1: ++ p0 = 1; ++ break; ++ case DPLL_CFGCR2_PDIV_2: ++ p0 = 2; ++ break; ++ case DPLL_CFGCR2_PDIV_3: ++ p0 = 3; ++ break; ++ case DPLL_CFGCR2_PDIV_7: ++ p0 = 7; ++ break; ++ } ++ ++ switch (p2) { ++ case DPLL_CFGCR2_KDIV_5: ++ p2 = 5; ++ break; ++ case DPLL_CFGCR2_KDIV_2: ++ p2 = 2; ++ break; ++ case DPLL_CFGCR2_KDIV_3: ++ p2 = 3; ++ break; ++ case DPLL_CFGCR2_KDIV_1: ++ p2 = 1; ++ break; ++ } ++ ++ dco_freq = (pll_state->cfgcr1 & DPLL_CFGCR1_DCO_INTEGER_MASK) ++ * 24 * 1000; ++ ++ dco_freq += (((pll_state->cfgcr1 & DPLL_CFGCR1_DCO_FRACTION_MASK) >> 9) ++ * 24 * 1000) / 0x8000; ++ ++ if (WARN_ON(p0 == 0 || p1 == 0 || p2 == 0)) ++ return 0; ++ ++ return dco_freq / (p0 * p1 * p2 * 5); ++} ++ ++int cnl_calc_wrpll_link(struct drm_i915_private *dev_priv, ++ struct intel_dpll_hw_state *pll_state) ++{ ++ u32 p0, p1, p2, dco_freq, ref_clock; ++ ++ p0 = pll_state->cfgcr1 & DPLL_CFGCR1_PDIV_MASK; ++ p2 = pll_state->cfgcr1 & DPLL_CFGCR1_KDIV_MASK; ++ ++ if (pll_state->cfgcr1 & DPLL_CFGCR1_QDIV_MODE(1)) ++ p1 = (pll_state->cfgcr1 & DPLL_CFGCR1_QDIV_RATIO_MASK) >> ++ DPLL_CFGCR1_QDIV_RATIO_SHIFT; ++ else ++ p1 = 1; ++ ++ ++ switch (p0) { ++ case DPLL_CFGCR1_PDIV_2: ++ p0 = 2; ++ break; ++ case DPLL_CFGCR1_PDIV_3: ++ p0 = 3; ++ break; ++ case DPLL_CFGCR1_PDIV_5: ++ p0 = 5; ++ break; ++ case DPLL_CFGCR1_PDIV_7: ++ p0 = 7; ++ break; ++ } ++ ++ switch (p2) { ++ case DPLL_CFGCR1_KDIV_1: ++ p2 = 1; ++ break; ++ case DPLL_CFGCR1_KDIV_2: ++ p2 = 2; ++ break; ++ case DPLL_CFGCR1_KDIV_3: ++ p2 = 3; ++ break; ++ } ++ ++ ref_clock = cnl_hdmi_pll_ref_clock(dev_priv); ++ ++ dco_freq = (pll_state->cfgcr0 & DPLL_CFGCR0_DCO_INTEGER_MASK) ++ * ref_clock; ++ ++ dco_freq += (((pll_state->cfgcr0 & DPLL_CFGCR0_DCO_FRACTION_MASK) >> ++ DPLL_CFGCR0_DCO_FRACTION_SHIFT) * ref_clock) / 0x8000; ++ ++ if (WARN_ON(p0 == 0 || p1 == 0 || p2 == 0)) ++ return 0; ++ ++ return dco_freq / (p0 * p1 * p2 * 5); ++} ++ ++static int icl_calc_tbt_pll_link(struct drm_i915_private *dev_priv, ++ enum port port) ++{ ++ u32 val = I915_READ(DDI_CLK_SEL(port)) & DDI_CLK_SEL_MASK; ++ ++ switch (val) { ++ case DDI_CLK_SEL_NONE: ++ return 0; ++ case DDI_CLK_SEL_TBT_162: ++ return 162000; ++ case DDI_CLK_SEL_TBT_270: ++ return 270000; ++ case DDI_CLK_SEL_TBT_540: ++ return 540000; ++ case DDI_CLK_SEL_TBT_810: ++ return 810000; ++ default: ++ MISSING_CASE(val); ++ return 0; ++ } ++} ++ ++static int icl_calc_mg_pll_link(struct drm_i915_private *dev_priv, ++ const struct intel_dpll_hw_state *pll_state) ++{ ++ u32 m1, m2_int, m2_frac, div1, div2, ref_clock; ++ u64 tmp; ++ ++ ref_clock = dev_priv->cdclk.hw.ref; ++ ++ m1 = pll_state->mg_pll_div1 & MG_PLL_DIV1_FBPREDIV_MASK; ++ m2_int = pll_state->mg_pll_div0 & MG_PLL_DIV0_FBDIV_INT_MASK; ++ m2_frac = (pll_state->mg_pll_div0 & MG_PLL_DIV0_FRACNEN_H) ? ++ (pll_state->mg_pll_div0 & MG_PLL_DIV0_FBDIV_FRAC_MASK) >> ++ MG_PLL_DIV0_FBDIV_FRAC_SHIFT : 0; ++ ++ switch (pll_state->mg_clktop2_hsclkctl & ++ MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_MASK) { ++ case MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_2: ++ div1 = 2; ++ break; ++ case MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_3: ++ div1 = 3; ++ break; ++ case MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_5: ++ div1 = 5; ++ break; ++ case MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_7: ++ div1 = 7; ++ break; ++ default: ++ MISSING_CASE(pll_state->mg_clktop2_hsclkctl); ++ return 0; ++ } ++ ++ div2 = (pll_state->mg_clktop2_hsclkctl & ++ MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK) >> ++ MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_SHIFT; ++ ++ /* div2 value of 0 is same as 1 means no div */ ++ if (div2 == 0) ++ div2 = 1; ++ ++ /* ++ * Adjust the original formula to delay the division by 2^22 in order to ++ * minimize possible rounding errors. ++ */ ++ tmp = (u64)m1 * m2_int * ref_clock + ++ (((u64)m1 * m2_frac * ref_clock) >> 22); ++ tmp = div_u64(tmp, 5 * div1 * div2); ++ ++ return tmp; ++} ++ ++static void ddi_dotclock_get(struct intel_crtc_state *pipe_config) ++{ ++ int dotclock; ++ ++ if (pipe_config->has_pch_encoder) ++ dotclock = intel_dotclock_calculate(pipe_config->port_clock, ++ &pipe_config->fdi_m_n); ++ else if (intel_crtc_has_dp_encoder(pipe_config)) ++ dotclock = intel_dotclock_calculate(pipe_config->port_clock, ++ &pipe_config->dp_m_n); ++ else if (pipe_config->has_hdmi_sink && pipe_config->pipe_bpp == 36) ++ dotclock = pipe_config->port_clock * 2 / 3; ++ else ++ dotclock = pipe_config->port_clock; ++ ++ if (pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420) ++ dotclock *= 2; ++ ++ if (pipe_config->pixel_multiplier) ++ dotclock /= pipe_config->pixel_multiplier; ++ ++ pipe_config->base.adjusted_mode.crtc_clock = dotclock; ++} ++ ++static void icl_ddi_clock_get(struct intel_encoder *encoder, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dpll_hw_state *pll_state = &pipe_config->dpll_hw_state; ++ enum port port = encoder->port; ++ int link_clock; ++ ++ if (intel_port_is_combophy(dev_priv, port)) { ++ link_clock = cnl_calc_wrpll_link(dev_priv, pll_state); ++ } else { ++ enum intel_dpll_id pll_id = intel_get_shared_dpll_id(dev_priv, ++ pipe_config->shared_dpll); ++ ++ if (pll_id == DPLL_ID_ICL_TBTPLL) ++ link_clock = icl_calc_tbt_pll_link(dev_priv, port); ++ else ++ link_clock = icl_calc_mg_pll_link(dev_priv, pll_state); ++ } ++ ++ pipe_config->port_clock = link_clock; ++ ++ ddi_dotclock_get(pipe_config); ++} ++ ++static void cnl_ddi_clock_get(struct intel_encoder *encoder, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dpll_hw_state *pll_state = &pipe_config->dpll_hw_state; ++ int link_clock; ++ ++ if (pll_state->cfgcr0 & DPLL_CFGCR0_HDMI_MODE) { ++ link_clock = cnl_calc_wrpll_link(dev_priv, pll_state); ++ } else { ++ link_clock = pll_state->cfgcr0 & DPLL_CFGCR0_LINK_RATE_MASK; ++ ++ switch (link_clock) { ++ case DPLL_CFGCR0_LINK_RATE_810: ++ link_clock = 81000; ++ break; ++ case DPLL_CFGCR0_LINK_RATE_1080: ++ link_clock = 108000; ++ break; ++ case DPLL_CFGCR0_LINK_RATE_1350: ++ link_clock = 135000; ++ break; ++ case DPLL_CFGCR0_LINK_RATE_1620: ++ link_clock = 162000; ++ break; ++ case DPLL_CFGCR0_LINK_RATE_2160: ++ link_clock = 216000; ++ break; ++ case DPLL_CFGCR0_LINK_RATE_2700: ++ link_clock = 270000; ++ break; ++ case DPLL_CFGCR0_LINK_RATE_3240: ++ link_clock = 324000; ++ break; ++ case DPLL_CFGCR0_LINK_RATE_4050: ++ link_clock = 405000; ++ break; ++ default: ++ WARN(1, "Unsupported link rate\n"); ++ break; ++ } ++ link_clock *= 2; ++ } ++ ++ pipe_config->port_clock = link_clock; ++ ++ ddi_dotclock_get(pipe_config); ++} ++ ++static void skl_ddi_clock_get(struct intel_encoder *encoder, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct intel_dpll_hw_state *pll_state = &pipe_config->dpll_hw_state; ++ int link_clock; ++ ++ /* ++ * ctrl1 register is already shifted for each pll, just use 0 to get ++ * the internal shift for each field ++ */ ++ if (pll_state->ctrl1 & DPLL_CTRL1_HDMI_MODE(0)) { ++ link_clock = skl_calc_wrpll_link(pll_state); ++ } else { ++ link_clock = pll_state->ctrl1 & DPLL_CTRL1_LINK_RATE_MASK(0); ++ link_clock >>= DPLL_CTRL1_LINK_RATE_SHIFT(0); ++ ++ switch (link_clock) { ++ case DPLL_CTRL1_LINK_RATE_810: ++ link_clock = 81000; ++ break; ++ case DPLL_CTRL1_LINK_RATE_1080: ++ link_clock = 108000; ++ break; ++ case DPLL_CTRL1_LINK_RATE_1350: ++ link_clock = 135000; ++ break; ++ case DPLL_CTRL1_LINK_RATE_1620: ++ link_clock = 162000; ++ break; ++ case DPLL_CTRL1_LINK_RATE_2160: ++ link_clock = 216000; ++ break; ++ case DPLL_CTRL1_LINK_RATE_2700: ++ link_clock = 270000; ++ break; ++ default: ++ WARN(1, "Unsupported link rate\n"); ++ break; ++ } ++ link_clock *= 2; ++ } ++ ++ pipe_config->port_clock = link_clock; ++ ++ ddi_dotclock_get(pipe_config); ++} ++ ++static void hsw_ddi_clock_get(struct intel_encoder *encoder, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ int link_clock = 0; ++ u32 val, pll; ++ ++ val = hsw_pll_to_ddi_pll_sel(pipe_config->shared_dpll); ++ switch (val & PORT_CLK_SEL_MASK) { ++ case PORT_CLK_SEL_LCPLL_810: ++ link_clock = 81000; ++ break; ++ case PORT_CLK_SEL_LCPLL_1350: ++ link_clock = 135000; ++ break; ++ case PORT_CLK_SEL_LCPLL_2700: ++ link_clock = 270000; ++ break; ++ case PORT_CLK_SEL_WRPLL1: ++ link_clock = hsw_ddi_calc_wrpll_link(dev_priv, WRPLL_CTL(0)); ++ break; ++ case PORT_CLK_SEL_WRPLL2: ++ link_clock = hsw_ddi_calc_wrpll_link(dev_priv, WRPLL_CTL(1)); ++ break; ++ case PORT_CLK_SEL_SPLL: ++ pll = I915_READ(SPLL_CTL) & SPLL_PLL_FREQ_MASK; ++ if (pll == SPLL_PLL_FREQ_810MHz) ++ link_clock = 81000; ++ else if (pll == SPLL_PLL_FREQ_1350MHz) ++ link_clock = 135000; ++ else if (pll == SPLL_PLL_FREQ_2700MHz) ++ link_clock = 270000; ++ else { ++ WARN(1, "bad spll freq\n"); ++ return; ++ } ++ break; ++ default: ++ WARN(1, "bad port clock sel\n"); ++ return; ++ } ++ ++ pipe_config->port_clock = link_clock * 2; ++ ++ ddi_dotclock_get(pipe_config); ++} ++ ++static int bxt_calc_pll_link(const struct intel_dpll_hw_state *pll_state) ++{ ++ struct dpll clock; ++ ++ clock.m1 = 2; ++ clock.m2 = (pll_state->pll0 & PORT_PLL_M2_MASK) << 22; ++ if (pll_state->pll3 & PORT_PLL_M2_FRAC_ENABLE) ++ clock.m2 |= pll_state->pll2 & PORT_PLL_M2_FRAC_MASK; ++ clock.n = (pll_state->pll1 & PORT_PLL_N_MASK) >> PORT_PLL_N_SHIFT; ++ clock.p1 = (pll_state->ebb0 & PORT_PLL_P1_MASK) >> PORT_PLL_P1_SHIFT; ++ clock.p2 = (pll_state->ebb0 & PORT_PLL_P2_MASK) >> PORT_PLL_P2_SHIFT; ++ ++ return chv_calc_dpll_params(100000, &clock); ++} ++ ++static void bxt_ddi_clock_get(struct intel_encoder *encoder, ++ struct intel_crtc_state *pipe_config) ++{ ++ pipe_config->port_clock = ++ bxt_calc_pll_link(&pipe_config->dpll_hw_state); ++ ++ ddi_dotclock_get(pipe_config); ++} ++ ++static void intel_ddi_clock_get(struct intel_encoder *encoder, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ icl_ddi_clock_get(encoder, pipe_config); ++ else if (IS_CANNONLAKE(dev_priv)) ++ cnl_ddi_clock_get(encoder, pipe_config); ++ else if (IS_GEN9_LP(dev_priv)) ++ bxt_ddi_clock_get(encoder, pipe_config); ++ else if (IS_GEN9_BC(dev_priv)) ++ skl_ddi_clock_get(encoder, pipe_config); ++ else if (INTEL_GEN(dev_priv) <= 8) ++ hsw_ddi_clock_get(encoder, pipe_config); ++} ++ ++void intel_ddi_set_pipe_settings(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; ++ u32 temp; ++ ++ if (!intel_crtc_has_dp_encoder(crtc_state)) ++ return; ++ ++ WARN_ON(transcoder_is_dsi(cpu_transcoder)); ++ ++ temp = TRANS_MSA_SYNC_CLK; ++ ++ if (crtc_state->limited_color_range) ++ temp |= TRANS_MSA_CEA_RANGE; ++ ++ switch (crtc_state->pipe_bpp) { ++ case 18: ++ temp |= TRANS_MSA_6_BPC; ++ break; ++ case 24: ++ temp |= TRANS_MSA_8_BPC; ++ break; ++ case 30: ++ temp |= TRANS_MSA_10_BPC; ++ break; ++ case 36: ++ temp |= TRANS_MSA_12_BPC; ++ break; ++ default: ++ MISSING_CASE(crtc_state->pipe_bpp); ++ break; ++ } ++ ++ /* ++ * As per DP 1.2 spec section 2.3.4.3 while sending ++ * YCBCR 444 signals we should program MSA MISC1/0 fields with ++ * colorspace information. The output colorspace encoding is BT601. ++ */ ++ if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444) ++ temp |= TRANS_MSA_SAMPLING_444 | TRANS_MSA_CLRSP_YCBCR; ++ I915_WRITE(TRANS_MSA_MISC(cpu_transcoder), temp); ++} ++ ++void intel_ddi_set_vc_payload_alloc(const struct intel_crtc_state *crtc_state, ++ bool state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; ++ u32 temp; ++ ++ temp = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder)); ++ if (state == true) ++ temp |= TRANS_DDI_DP_VC_PAYLOAD_ALLOC; ++ else ++ temp &= ~TRANS_DDI_DP_VC_PAYLOAD_ALLOC; ++ I915_WRITE(TRANS_DDI_FUNC_CTL(cpu_transcoder), temp); ++} ++ ++void intel_ddi_enable_transcoder_func(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct intel_encoder *encoder = intel_ddi_get_crtc_encoder(crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; ++ enum port port = encoder->port; ++ u32 temp; ++ ++ /* Enable TRANS_DDI_FUNC_CTL for the pipe to work in HDMI mode */ ++ temp = TRANS_DDI_FUNC_ENABLE; ++ temp |= TRANS_DDI_SELECT_PORT(port); ++ ++ switch (crtc_state->pipe_bpp) { ++ case 18: ++ temp |= TRANS_DDI_BPC_6; ++ break; ++ case 24: ++ temp |= TRANS_DDI_BPC_8; ++ break; ++ case 30: ++ temp |= TRANS_DDI_BPC_10; ++ break; ++ case 36: ++ temp |= TRANS_DDI_BPC_12; ++ break; ++ default: ++ BUG(); ++ } ++ ++ if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_PVSYNC) ++ temp |= TRANS_DDI_PVSYNC; ++ if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_PHSYNC) ++ temp |= TRANS_DDI_PHSYNC; ++ ++ if (cpu_transcoder == TRANSCODER_EDP) { ++ switch (pipe) { ++ case PIPE_A: ++ /* On Haswell, can only use the always-on power well for ++ * eDP when not using the panel fitter, and when not ++ * using motion blur mitigation (which we don't ++ * support). */ ++ if (IS_HASWELL(dev_priv) && ++ (crtc_state->pch_pfit.enabled || ++ crtc_state->pch_pfit.force_thru)) ++ temp |= TRANS_DDI_EDP_INPUT_A_ONOFF; ++ else ++ temp |= TRANS_DDI_EDP_INPUT_A_ON; ++ break; ++ case PIPE_B: ++ temp |= TRANS_DDI_EDP_INPUT_B_ONOFF; ++ break; ++ case PIPE_C: ++ temp |= TRANS_DDI_EDP_INPUT_C_ONOFF; ++ break; ++ default: ++ BUG(); ++ break; ++ } ++ } ++ ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) { ++ if (crtc_state->has_hdmi_sink) ++ temp |= TRANS_DDI_MODE_SELECT_HDMI; ++ else ++ temp |= TRANS_DDI_MODE_SELECT_DVI; ++ ++ if (crtc_state->hdmi_scrambling) ++ temp |= TRANS_DDI_HDMI_SCRAMBLING; ++ if (crtc_state->hdmi_high_tmds_clock_ratio) ++ temp |= TRANS_DDI_HIGH_TMDS_CHAR_RATE; ++ } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) { ++ temp |= TRANS_DDI_MODE_SELECT_FDI; ++ temp |= (crtc_state->fdi_lanes - 1) << 1; ++ } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST)) { ++ temp |= TRANS_DDI_MODE_SELECT_DP_MST; ++ temp |= DDI_PORT_WIDTH(crtc_state->lane_count); ++ } else { ++ temp |= TRANS_DDI_MODE_SELECT_DP_SST; ++ temp |= DDI_PORT_WIDTH(crtc_state->lane_count); ++ } ++ ++ I915_WRITE(TRANS_DDI_FUNC_CTL(cpu_transcoder), temp); ++} ++ ++void intel_ddi_disable_transcoder_func(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; ++ i915_reg_t reg = TRANS_DDI_FUNC_CTL(cpu_transcoder); ++ u32 val = I915_READ(reg); ++ ++ val &= ~(TRANS_DDI_FUNC_ENABLE | TRANS_DDI_PORT_MASK | TRANS_DDI_DP_VC_PAYLOAD_ALLOC); ++ val |= TRANS_DDI_PORT_NONE; ++ I915_WRITE(reg, val); ++ ++ if (dev_priv->quirks & QUIRK_INCREASE_DDI_DISABLED_TIME && ++ intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) { ++ DRM_DEBUG_KMS("Quirk Increase DDI disabled time\n"); ++ /* Quirk time at 100ms for reliable operation */ ++ msleep(100); ++ } ++} ++ ++int intel_ddi_toggle_hdcp_signalling(struct intel_encoder *intel_encoder, ++ bool enable) ++{ ++ struct drm_device *dev = intel_encoder->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ intel_wakeref_t wakeref; ++ enum pipe pipe = 0; ++ int ret = 0; ++ u32 tmp; ++ ++ wakeref = intel_display_power_get_if_enabled(dev_priv, ++ intel_encoder->power_domain); ++ if (WARN_ON(!wakeref)) ++ return -ENXIO; ++ ++ if (WARN_ON(!intel_encoder->get_hw_state(intel_encoder, &pipe))) { ++ ret = -EIO; ++ goto out; ++ } ++ ++ tmp = I915_READ(TRANS_DDI_FUNC_CTL(pipe)); ++ if (enable) ++ tmp |= TRANS_DDI_HDCP_SIGNALLING; ++ else ++ tmp &= ~TRANS_DDI_HDCP_SIGNALLING; ++ I915_WRITE(TRANS_DDI_FUNC_CTL(pipe), tmp); ++out: ++ intel_display_power_put(dev_priv, intel_encoder->power_domain, wakeref); ++ return ret; ++} ++ ++bool intel_ddi_connector_get_hw_state(struct intel_connector *intel_connector) ++{ ++ struct drm_device *dev = intel_connector->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_encoder *encoder = intel_connector->encoder; ++ int type = intel_connector->base.connector_type; ++ enum port port = encoder->port; ++ enum transcoder cpu_transcoder; ++ intel_wakeref_t wakeref; ++ enum pipe pipe = 0; ++ u32 tmp; ++ bool ret; ++ ++ wakeref = intel_display_power_get_if_enabled(dev_priv, ++ encoder->power_domain); ++ if (!wakeref) ++ return false; ++ ++ if (!encoder->get_hw_state(encoder, &pipe)) { ++ ret = false; ++ goto out; ++ } ++ ++ if (HAS_TRANSCODER_EDP(dev_priv) && port == PORT_A) ++ cpu_transcoder = TRANSCODER_EDP; ++ else ++ cpu_transcoder = (enum transcoder) pipe; ++ ++ tmp = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder)); ++ ++ switch (tmp & TRANS_DDI_MODE_SELECT_MASK) { ++ case TRANS_DDI_MODE_SELECT_HDMI: ++ case TRANS_DDI_MODE_SELECT_DVI: ++ ret = type == DRM_MODE_CONNECTOR_HDMIA; ++ break; ++ ++ case TRANS_DDI_MODE_SELECT_DP_SST: ++ ret = type == DRM_MODE_CONNECTOR_eDP || ++ type == DRM_MODE_CONNECTOR_DisplayPort; ++ break; ++ ++ case TRANS_DDI_MODE_SELECT_DP_MST: ++ /* if the transcoder is in MST state then ++ * connector isn't connected */ ++ ret = false; ++ break; ++ ++ case TRANS_DDI_MODE_SELECT_FDI: ++ ret = type == DRM_MODE_CONNECTOR_VGA; ++ break; ++ ++ default: ++ ret = false; ++ break; ++ } ++ ++out: ++ intel_display_power_put(dev_priv, encoder->power_domain, wakeref); ++ ++ return ret; ++} ++ ++static void intel_ddi_get_encoder_pipes(struct intel_encoder *encoder, ++ u8 *pipe_mask, bool *is_dp_mst) ++{ ++ struct drm_device *dev = encoder->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ enum port port = encoder->port; ++ intel_wakeref_t wakeref; ++ enum pipe p; ++ u32 tmp; ++ u8 mst_pipe_mask; ++ ++ *pipe_mask = 0; ++ *is_dp_mst = false; ++ ++ wakeref = intel_display_power_get_if_enabled(dev_priv, ++ encoder->power_domain); ++ if (!wakeref) ++ return; ++ ++ tmp = I915_READ(DDI_BUF_CTL(port)); ++ if (!(tmp & DDI_BUF_CTL_ENABLE)) ++ goto out; ++ ++ if (HAS_TRANSCODER_EDP(dev_priv) && port == PORT_A) { ++ tmp = I915_READ(TRANS_DDI_FUNC_CTL(TRANSCODER_EDP)); ++ ++ switch (tmp & TRANS_DDI_EDP_INPUT_MASK) { ++ default: ++ MISSING_CASE(tmp & TRANS_DDI_EDP_INPUT_MASK); ++ /* fallthrough */ ++ case TRANS_DDI_EDP_INPUT_A_ON: ++ case TRANS_DDI_EDP_INPUT_A_ONOFF: ++ *pipe_mask = BIT(PIPE_A); ++ break; ++ case TRANS_DDI_EDP_INPUT_B_ONOFF: ++ *pipe_mask = BIT(PIPE_B); ++ break; ++ case TRANS_DDI_EDP_INPUT_C_ONOFF: ++ *pipe_mask = BIT(PIPE_C); ++ break; ++ } ++ ++ goto out; ++ } ++ ++ mst_pipe_mask = 0; ++ for_each_pipe(dev_priv, p) { ++ enum transcoder cpu_transcoder = (enum transcoder)p; ++ ++ tmp = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder)); ++ ++ if ((tmp & TRANS_DDI_PORT_MASK) != TRANS_DDI_SELECT_PORT(port)) ++ continue; ++ ++ if ((tmp & TRANS_DDI_MODE_SELECT_MASK) == ++ TRANS_DDI_MODE_SELECT_DP_MST) ++ mst_pipe_mask |= BIT(p); ++ ++ *pipe_mask |= BIT(p); ++ } ++ ++ if (!*pipe_mask) ++ DRM_DEBUG_KMS("No pipe for ddi port %c found\n", ++ port_name(port)); ++ ++ if (!mst_pipe_mask && hweight8(*pipe_mask) > 1) { ++ DRM_DEBUG_KMS("Multiple pipes for non DP-MST port %c (pipe_mask %02x)\n", ++ port_name(port), *pipe_mask); ++ *pipe_mask = BIT(ffs(*pipe_mask) - 1); ++ } ++ ++ if (mst_pipe_mask && mst_pipe_mask != *pipe_mask) ++ DRM_DEBUG_KMS("Conflicting MST and non-MST encoders for port %c (pipe_mask %02x mst_pipe_mask %02x)\n", ++ port_name(port), *pipe_mask, mst_pipe_mask); ++ else ++ *is_dp_mst = mst_pipe_mask; ++ ++out: ++ if (*pipe_mask && IS_GEN9_LP(dev_priv)) { ++ tmp = I915_READ(BXT_PHY_CTL(port)); ++ if ((tmp & (BXT_PHY_CMNLANE_POWERDOWN_ACK | ++ BXT_PHY_LANE_POWERDOWN_ACK | ++ BXT_PHY_LANE_ENABLED)) != BXT_PHY_LANE_ENABLED) ++ DRM_ERROR("Port %c enabled but PHY powered down? " ++ "(PHY_CTL %08x)\n", port_name(port), tmp); ++ } ++ ++ intel_display_power_put(dev_priv, encoder->power_domain, wakeref); ++} ++ ++bool intel_ddi_get_hw_state(struct intel_encoder *encoder, ++ enum pipe *pipe) ++{ ++ u8 pipe_mask; ++ bool is_mst; ++ ++ intel_ddi_get_encoder_pipes(encoder, &pipe_mask, &is_mst); ++ ++ if (is_mst || !pipe_mask) ++ return false; ++ ++ *pipe = ffs(pipe_mask) - 1; ++ ++ return true; ++} ++ ++static inline enum intel_display_power_domain ++intel_ddi_main_link_aux_domain(struct intel_digital_port *dig_port) ++{ ++ /* CNL+ HW requires corresponding AUX IOs to be powered up for PSR with ++ * DC states enabled at the same time, while for driver initiated AUX ++ * transfers we need the same AUX IOs to be powered but with DC states ++ * disabled. Accordingly use the AUX power domain here which leaves DC ++ * states enabled. ++ * However, for non-A AUX ports the corresponding non-EDP transcoders ++ * would have already enabled power well 2 and DC_OFF. This means we can ++ * acquire a wider POWER_DOMAIN_AUX_{B,C,D,F} reference instead of a ++ * specific AUX_IO reference without powering up any extra wells. ++ * Note that PSR is enabled only on Port A even though this function ++ * returns the correct domain for other ports too. ++ */ ++ return dig_port->aux_ch == AUX_CH_A ? POWER_DOMAIN_AUX_IO_A : ++ intel_aux_power_domain(dig_port); ++} ++ ++static void intel_ddi_get_power_domains(struct intel_encoder *encoder, ++ struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_digital_port *dig_port; ++ ++ /* ++ * TODO: Add support for MST encoders. Atm, the following should never ++ * happen since fake-MST encoders don't set their get_power_domains() ++ * hook. ++ */ ++ if (WARN_ON(intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST))) ++ return; ++ ++ dig_port = enc_to_dig_port(&encoder->base); ++ intel_display_power_get(dev_priv, dig_port->ddi_io_power_domain); ++ ++ /* ++ * AUX power is only needed for (e)DP mode, and for HDMI mode on TC ++ * ports. ++ */ ++ if (intel_crtc_has_dp_encoder(crtc_state) || ++ intel_port_is_tc(dev_priv, encoder->port)) ++ intel_display_power_get(dev_priv, ++ intel_ddi_main_link_aux_domain(dig_port)); ++ ++ /* ++ * VDSC power is needed when DSC is enabled ++ */ ++ if (crtc_state->dsc_params.compression_enable) ++ intel_display_power_get(dev_priv, ++ intel_dsc_power_domain(crtc_state)); ++} ++ ++void intel_ddi_enable_pipe_clock(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ struct intel_encoder *encoder = intel_ddi_get_crtc_encoder(crtc); ++ enum port port = encoder->port; ++ enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; ++ ++ if (cpu_transcoder != TRANSCODER_EDP) ++ I915_WRITE(TRANS_CLK_SEL(cpu_transcoder), ++ TRANS_CLK_SEL_PORT(port)); ++} ++ ++void intel_ddi_disable_pipe_clock(const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); ++ enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; ++ ++ if (cpu_transcoder != TRANSCODER_EDP) ++ I915_WRITE(TRANS_CLK_SEL(cpu_transcoder), ++ TRANS_CLK_SEL_DISABLED); ++} ++ ++static void _skl_ddi_set_iboost(struct drm_i915_private *dev_priv, ++ enum port port, u8 iboost) ++{ ++ u32 tmp; ++ ++ tmp = I915_READ(DISPIO_CR_TX_BMU_CR0); ++ tmp &= ~(BALANCE_LEG_MASK(port) | BALANCE_LEG_DISABLE(port)); ++ if (iboost) ++ tmp |= iboost << BALANCE_LEG_SHIFT(port); ++ else ++ tmp |= BALANCE_LEG_DISABLE(port); ++ I915_WRITE(DISPIO_CR_TX_BMU_CR0, tmp); ++} ++ ++static void skl_ddi_set_iboost(struct intel_encoder *encoder, ++ int level, enum intel_output_type type) ++{ ++ struct intel_digital_port *intel_dig_port = enc_to_dig_port(&encoder->base); ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ enum port port = encoder->port; ++ u8 iboost; ++ ++ if (type == INTEL_OUTPUT_HDMI) ++ iboost = dev_priv->vbt.ddi_port_info[port].hdmi_boost_level; ++ else ++ iboost = dev_priv->vbt.ddi_port_info[port].dp_boost_level; ++ ++ if (iboost == 0) { ++ const struct ddi_buf_trans *ddi_translations; ++ int n_entries; ++ ++ if (type == INTEL_OUTPUT_HDMI) ++ ddi_translations = intel_ddi_get_buf_trans_hdmi(dev_priv, &n_entries); ++ else if (type == INTEL_OUTPUT_EDP) ++ ddi_translations = intel_ddi_get_buf_trans_edp(dev_priv, port, &n_entries); ++ else ++ ddi_translations = intel_ddi_get_buf_trans_dp(dev_priv, port, &n_entries); ++ ++ if (WARN_ON_ONCE(!ddi_translations)) ++ return; ++ if (WARN_ON_ONCE(level >= n_entries)) ++ level = n_entries - 1; ++ ++ iboost = ddi_translations[level].i_boost; ++ } ++ ++ /* Make sure that the requested I_boost is valid */ ++ if (iboost && iboost != 0x1 && iboost != 0x3 && iboost != 0x7) { ++ DRM_ERROR("Invalid I_boost value %u\n", iboost); ++ return; ++ } ++ ++ _skl_ddi_set_iboost(dev_priv, port, iboost); ++ ++ if (port == PORT_A && intel_dig_port->max_lanes == 4) ++ _skl_ddi_set_iboost(dev_priv, PORT_E, iboost); ++} ++ ++static void bxt_ddi_vswing_sequence(struct intel_encoder *encoder, ++ int level, enum intel_output_type type) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ const struct bxt_ddi_buf_trans *ddi_translations; ++ enum port port = encoder->port; ++ int n_entries; ++ ++ if (type == INTEL_OUTPUT_HDMI) ++ ddi_translations = bxt_get_buf_trans_hdmi(dev_priv, &n_entries); ++ else if (type == INTEL_OUTPUT_EDP) ++ ddi_translations = bxt_get_buf_trans_edp(dev_priv, &n_entries); ++ else ++ ddi_translations = bxt_get_buf_trans_dp(dev_priv, &n_entries); ++ ++ if (WARN_ON_ONCE(!ddi_translations)) ++ return; ++ if (WARN_ON_ONCE(level >= n_entries)) ++ level = n_entries - 1; ++ ++ bxt_ddi_phy_set_signal_level(dev_priv, port, ++ ddi_translations[level].margin, ++ ddi_translations[level].scale, ++ ddi_translations[level].enable, ++ ddi_translations[level].deemphasis); ++} ++ ++u8 intel_ddi_dp_voltage_max(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ enum port port = encoder->port; ++ int n_entries; ++ ++ if (INTEL_GEN(dev_priv) >= 11) { ++ if (intel_port_is_combophy(dev_priv, port)) ++ icl_get_combo_buf_trans(dev_priv, port, encoder->type, ++ intel_dp->link_rate, &n_entries); ++ else ++ n_entries = ARRAY_SIZE(icl_mg_phy_ddi_translations); ++ } else if (IS_CANNONLAKE(dev_priv)) { ++ if (encoder->type == INTEL_OUTPUT_EDP) ++ cnl_get_buf_trans_edp(dev_priv, &n_entries); ++ else ++ cnl_get_buf_trans_dp(dev_priv, &n_entries); ++ } else if (IS_GEN9_LP(dev_priv)) { ++ if (encoder->type == INTEL_OUTPUT_EDP) ++ bxt_get_buf_trans_edp(dev_priv, &n_entries); ++ else ++ bxt_get_buf_trans_dp(dev_priv, &n_entries); ++ } else { ++ if (encoder->type == INTEL_OUTPUT_EDP) ++ intel_ddi_get_buf_trans_edp(dev_priv, port, &n_entries); ++ else ++ intel_ddi_get_buf_trans_dp(dev_priv, port, &n_entries); ++ } ++ ++ if (WARN_ON(n_entries < 1)) ++ n_entries = 1; ++ if (WARN_ON(n_entries > ARRAY_SIZE(index_to_dp_signal_levels))) ++ n_entries = ARRAY_SIZE(index_to_dp_signal_levels); ++ ++ return index_to_dp_signal_levels[n_entries - 1] & ++ DP_TRAIN_VOLTAGE_SWING_MASK; ++} ++ ++/* ++ * We assume that the full set of pre-emphasis values can be ++ * used on all DDI platforms. Should that change we need to ++ * rethink this code. ++ */ ++u8 intel_ddi_dp_pre_emphasis_max(struct intel_encoder *encoder, u8 voltage_swing) ++{ ++ switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) { ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_0: ++ return DP_TRAIN_PRE_EMPH_LEVEL_3; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_1: ++ return DP_TRAIN_PRE_EMPH_LEVEL_2; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_2: ++ return DP_TRAIN_PRE_EMPH_LEVEL_1; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_3: ++ default: ++ return DP_TRAIN_PRE_EMPH_LEVEL_0; ++ } ++} ++ ++static void cnl_ddi_vswing_program(struct intel_encoder *encoder, ++ int level, enum intel_output_type type) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ const struct cnl_ddi_buf_trans *ddi_translations; ++ enum port port = encoder->port; ++ int n_entries, ln; ++ u32 val; ++ ++ if (type == INTEL_OUTPUT_HDMI) ++ ddi_translations = cnl_get_buf_trans_hdmi(dev_priv, &n_entries); ++ else if (type == INTEL_OUTPUT_EDP) ++ ddi_translations = cnl_get_buf_trans_edp(dev_priv, &n_entries); ++ else ++ ddi_translations = cnl_get_buf_trans_dp(dev_priv, &n_entries); ++ ++ if (WARN_ON_ONCE(!ddi_translations)) ++ return; ++ if (WARN_ON_ONCE(level >= n_entries)) ++ level = n_entries - 1; ++ ++ /* Set PORT_TX_DW5 Scaling Mode Sel to 010b. */ ++ val = I915_READ(CNL_PORT_TX_DW5_LN0(port)); ++ val &= ~SCALING_MODE_SEL_MASK; ++ val |= SCALING_MODE_SEL(2); ++ I915_WRITE(CNL_PORT_TX_DW5_GRP(port), val); ++ ++ /* Program PORT_TX_DW2 */ ++ val = I915_READ(CNL_PORT_TX_DW2_LN0(port)); ++ val &= ~(SWING_SEL_LOWER_MASK | SWING_SEL_UPPER_MASK | ++ RCOMP_SCALAR_MASK); ++ val |= SWING_SEL_UPPER(ddi_translations[level].dw2_swing_sel); ++ val |= SWING_SEL_LOWER(ddi_translations[level].dw2_swing_sel); ++ /* Rcomp scalar is fixed as 0x98 for every table entry */ ++ val |= RCOMP_SCALAR(0x98); ++ I915_WRITE(CNL_PORT_TX_DW2_GRP(port), val); ++ ++ /* Program PORT_TX_DW4 */ ++ /* We cannot write to GRP. It would overrite individual loadgen */ ++ for (ln = 0; ln < 4; ln++) { ++ val = I915_READ(CNL_PORT_TX_DW4_LN(ln, port)); ++ val &= ~(POST_CURSOR_1_MASK | POST_CURSOR_2_MASK | ++ CURSOR_COEFF_MASK); ++ val |= POST_CURSOR_1(ddi_translations[level].dw4_post_cursor_1); ++ val |= POST_CURSOR_2(ddi_translations[level].dw4_post_cursor_2); ++ val |= CURSOR_COEFF(ddi_translations[level].dw4_cursor_coeff); ++ I915_WRITE(CNL_PORT_TX_DW4_LN(ln, port), val); ++ } ++ ++ /* Program PORT_TX_DW5 */ ++ /* All DW5 values are fixed for every table entry */ ++ val = I915_READ(CNL_PORT_TX_DW5_LN0(port)); ++ val &= ~RTERM_SELECT_MASK; ++ val |= RTERM_SELECT(6); ++ val |= TAP3_DISABLE; ++ I915_WRITE(CNL_PORT_TX_DW5_GRP(port), val); ++ ++ /* Program PORT_TX_DW7 */ ++ val = I915_READ(CNL_PORT_TX_DW7_LN0(port)); ++ val &= ~N_SCALAR_MASK; ++ val |= N_SCALAR(ddi_translations[level].dw7_n_scalar); ++ I915_WRITE(CNL_PORT_TX_DW7_GRP(port), val); ++} ++ ++static void cnl_ddi_vswing_sequence(struct intel_encoder *encoder, ++ int level, enum intel_output_type type) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ enum port port = encoder->port; ++ int width, rate, ln; ++ u32 val; ++ ++ if (type == INTEL_OUTPUT_HDMI) { ++ width = 4; ++ rate = 0; /* Rate is always < than 6GHz for HDMI */ ++ } else { ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ ++ width = intel_dp->lane_count; ++ rate = intel_dp->link_rate; ++ } ++ ++ /* ++ * 1. If port type is eDP or DP, ++ * set PORT_PCS_DW1 cmnkeeper_enable to 1b, ++ * else clear to 0b. ++ */ ++ val = I915_READ(CNL_PORT_PCS_DW1_LN0(port)); ++ if (type != INTEL_OUTPUT_HDMI) ++ val |= COMMON_KEEPER_EN; ++ else ++ val &= ~COMMON_KEEPER_EN; ++ I915_WRITE(CNL_PORT_PCS_DW1_GRP(port), val); ++ ++ /* 2. Program loadgen select */ ++ /* ++ * Program PORT_TX_DW4_LN depending on Bit rate and used lanes ++ * <= 6 GHz and 4 lanes (LN0=0, LN1=1, LN2=1, LN3=1) ++ * <= 6 GHz and 1,2 lanes (LN0=0, LN1=1, LN2=1, LN3=0) ++ * > 6 GHz (LN0=0, LN1=0, LN2=0, LN3=0) ++ */ ++ for (ln = 0; ln <= 3; ln++) { ++ val = I915_READ(CNL_PORT_TX_DW4_LN(ln, port)); ++ val &= ~LOADGEN_SELECT; ++ ++ if ((rate <= 600000 && width == 4 && ln >= 1) || ++ (rate <= 600000 && width < 4 && (ln == 1 || ln == 2))) { ++ val |= LOADGEN_SELECT; ++ } ++ I915_WRITE(CNL_PORT_TX_DW4_LN(ln, port), val); ++ } ++ ++ /* 3. Set PORT_CL_DW5 SUS Clock Config to 11b */ ++ val = I915_READ(CNL_PORT_CL1CM_DW5); ++ val |= SUS_CLOCK_CONFIG; ++ I915_WRITE(CNL_PORT_CL1CM_DW5, val); ++ ++ /* 4. Clear training enable to change swing values */ ++ val = I915_READ(CNL_PORT_TX_DW5_LN0(port)); ++ val &= ~TX_TRAINING_EN; ++ I915_WRITE(CNL_PORT_TX_DW5_GRP(port), val); ++ ++ /* 5. Program swing and de-emphasis */ ++ cnl_ddi_vswing_program(encoder, level, type); ++ ++ /* 6. Set training enable to trigger update */ ++ val = I915_READ(CNL_PORT_TX_DW5_LN0(port)); ++ val |= TX_TRAINING_EN; ++ I915_WRITE(CNL_PORT_TX_DW5_GRP(port), val); ++} ++ ++static void icl_ddi_combo_vswing_program(struct drm_i915_private *dev_priv, ++ u32 level, enum port port, int type, ++ int rate) ++{ ++ const struct cnl_ddi_buf_trans *ddi_translations = NULL; ++ u32 n_entries, val; ++ int ln; ++ ++ ddi_translations = icl_get_combo_buf_trans(dev_priv, port, type, ++ rate, &n_entries); ++ if (!ddi_translations) ++ return; ++ ++ if (level >= n_entries) { ++ DRM_DEBUG_KMS("DDI translation not found for level %d. Using %d instead.", level, n_entries - 1); ++ level = n_entries - 1; ++ } ++ ++ /* Set PORT_TX_DW5 */ ++ val = I915_READ(ICL_PORT_TX_DW5_LN0(port)); ++ val &= ~(SCALING_MODE_SEL_MASK | RTERM_SELECT_MASK | ++ TAP2_DISABLE | TAP3_DISABLE); ++ val |= SCALING_MODE_SEL(0x2); ++ val |= RTERM_SELECT(0x6); ++ val |= TAP3_DISABLE; ++ I915_WRITE(ICL_PORT_TX_DW5_GRP(port), val); ++ ++ /* Program PORT_TX_DW2 */ ++ val = I915_READ(ICL_PORT_TX_DW2_LN0(port)); ++ val &= ~(SWING_SEL_LOWER_MASK | SWING_SEL_UPPER_MASK | ++ RCOMP_SCALAR_MASK); ++ val |= SWING_SEL_UPPER(ddi_translations[level].dw2_swing_sel); ++ val |= SWING_SEL_LOWER(ddi_translations[level].dw2_swing_sel); ++ /* Program Rcomp scalar for every table entry */ ++ val |= RCOMP_SCALAR(0x98); ++ I915_WRITE(ICL_PORT_TX_DW2_GRP(port), val); ++ ++ /* Program PORT_TX_DW4 */ ++ /* We cannot write to GRP. It would overwrite individual loadgen. */ ++ for (ln = 0; ln <= 3; ln++) { ++ val = I915_READ(ICL_PORT_TX_DW4_LN(ln, port)); ++ val &= ~(POST_CURSOR_1_MASK | POST_CURSOR_2_MASK | ++ CURSOR_COEFF_MASK); ++ val |= POST_CURSOR_1(ddi_translations[level].dw4_post_cursor_1); ++ val |= POST_CURSOR_2(ddi_translations[level].dw4_post_cursor_2); ++ val |= CURSOR_COEFF(ddi_translations[level].dw4_cursor_coeff); ++ I915_WRITE(ICL_PORT_TX_DW4_LN(ln, port), val); ++ } ++ ++ /* Program PORT_TX_DW7 */ ++ val = I915_READ(ICL_PORT_TX_DW7_LN0(port)); ++ val &= ~N_SCALAR_MASK; ++ val |= N_SCALAR(ddi_translations[level].dw7_n_scalar); ++ I915_WRITE(ICL_PORT_TX_DW7_GRP(port), val); ++} ++ ++static void icl_combo_phy_ddi_vswing_sequence(struct intel_encoder *encoder, ++ u32 level, ++ enum intel_output_type type) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ enum port port = encoder->port; ++ int width = 0; ++ int rate = 0; ++ u32 val; ++ int ln = 0; ++ ++ if (type == INTEL_OUTPUT_HDMI) { ++ width = 4; ++ /* Rate is always < than 6GHz for HDMI */ ++ } else { ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ ++ width = intel_dp->lane_count; ++ rate = intel_dp->link_rate; ++ } ++ ++ /* ++ * 1. If port type is eDP or DP, ++ * set PORT_PCS_DW1 cmnkeeper_enable to 1b, ++ * else clear to 0b. ++ */ ++ val = I915_READ(ICL_PORT_PCS_DW1_LN0(port)); ++ if (type == INTEL_OUTPUT_HDMI) ++ val &= ~COMMON_KEEPER_EN; ++ else ++ val |= COMMON_KEEPER_EN; ++ I915_WRITE(ICL_PORT_PCS_DW1_GRP(port), val); ++ ++ /* 2. Program loadgen select */ ++ /* ++ * Program PORT_TX_DW4_LN depending on Bit rate and used lanes ++ * <= 6 GHz and 4 lanes (LN0=0, LN1=1, LN2=1, LN3=1) ++ * <= 6 GHz and 1,2 lanes (LN0=0, LN1=1, LN2=1, LN3=0) ++ * > 6 GHz (LN0=0, LN1=0, LN2=0, LN3=0) ++ */ ++ for (ln = 0; ln <= 3; ln++) { ++ val = I915_READ(ICL_PORT_TX_DW4_LN(ln, port)); ++ val &= ~LOADGEN_SELECT; ++ ++ if ((rate <= 600000 && width == 4 && ln >= 1) || ++ (rate <= 600000 && width < 4 && (ln == 1 || ln == 2))) { ++ val |= LOADGEN_SELECT; ++ } ++ I915_WRITE(ICL_PORT_TX_DW4_LN(ln, port), val); ++ } ++ ++ /* 3. Set PORT_CL_DW5 SUS Clock Config to 11b */ ++ val = I915_READ(ICL_PORT_CL_DW5(port)); ++ val |= SUS_CLOCK_CONFIG; ++ I915_WRITE(ICL_PORT_CL_DW5(port), val); ++ ++ /* 4. Clear training enable to change swing values */ ++ val = I915_READ(ICL_PORT_TX_DW5_LN0(port)); ++ val &= ~TX_TRAINING_EN; ++ I915_WRITE(ICL_PORT_TX_DW5_GRP(port), val); ++ ++ /* 5. Program swing and de-emphasis */ ++ icl_ddi_combo_vswing_program(dev_priv, level, port, type, rate); ++ ++ /* 6. Set training enable to trigger update */ ++ val = I915_READ(ICL_PORT_TX_DW5_LN0(port)); ++ val |= TX_TRAINING_EN; ++ I915_WRITE(ICL_PORT_TX_DW5_GRP(port), val); ++} ++ ++static void icl_mg_phy_ddi_vswing_sequence(struct intel_encoder *encoder, ++ int link_clock, ++ u32 level) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ enum port port = encoder->port; ++ const struct icl_mg_phy_ddi_buf_trans *ddi_translations; ++ u32 n_entries, val; ++ int ln; ++ ++ n_entries = ARRAY_SIZE(icl_mg_phy_ddi_translations); ++ ddi_translations = icl_mg_phy_ddi_translations; ++ /* The table does not have values for level 3 and level 9. */ ++ if (level >= n_entries || level == 3 || level == 9) { ++ DRM_DEBUG_KMS("DDI translation not found for level %d. Using %d instead.", ++ level, n_entries - 2); ++ level = n_entries - 2; ++ } ++ ++ /* Set MG_TX_LINK_PARAMS cri_use_fs32 to 0. */ ++ for (ln = 0; ln < 2; ln++) { ++ val = I915_READ(MG_TX1_LINK_PARAMS(ln, port)); ++ val &= ~CRI_USE_FS32; ++ I915_WRITE(MG_TX1_LINK_PARAMS(ln, port), val); ++ ++ val = I915_READ(MG_TX2_LINK_PARAMS(ln, port)); ++ val &= ~CRI_USE_FS32; ++ I915_WRITE(MG_TX2_LINK_PARAMS(ln, port), val); ++ } ++ ++ /* Program MG_TX_SWINGCTRL with values from vswing table */ ++ for (ln = 0; ln < 2; ln++) { ++ val = I915_READ(MG_TX1_SWINGCTRL(ln, port)); ++ val &= ~CRI_TXDEEMPH_OVERRIDE_17_12_MASK; ++ val |= CRI_TXDEEMPH_OVERRIDE_17_12( ++ ddi_translations[level].cri_txdeemph_override_17_12); ++ I915_WRITE(MG_TX1_SWINGCTRL(ln, port), val); ++ ++ val = I915_READ(MG_TX2_SWINGCTRL(ln, port)); ++ val &= ~CRI_TXDEEMPH_OVERRIDE_17_12_MASK; ++ val |= CRI_TXDEEMPH_OVERRIDE_17_12( ++ ddi_translations[level].cri_txdeemph_override_17_12); ++ I915_WRITE(MG_TX2_SWINGCTRL(ln, port), val); ++ } ++ ++ /* Program MG_TX_DRVCTRL with values from vswing table */ ++ for (ln = 0; ln < 2; ln++) { ++ val = I915_READ(MG_TX1_DRVCTRL(ln, port)); ++ val &= ~(CRI_TXDEEMPH_OVERRIDE_11_6_MASK | ++ CRI_TXDEEMPH_OVERRIDE_5_0_MASK); ++ val |= CRI_TXDEEMPH_OVERRIDE_5_0( ++ ddi_translations[level].cri_txdeemph_override_5_0) | ++ CRI_TXDEEMPH_OVERRIDE_11_6( ++ ddi_translations[level].cri_txdeemph_override_11_6) | ++ CRI_TXDEEMPH_OVERRIDE_EN; ++ I915_WRITE(MG_TX1_DRVCTRL(ln, port), val); ++ ++ val = I915_READ(MG_TX2_DRVCTRL(ln, port)); ++ val &= ~(CRI_TXDEEMPH_OVERRIDE_11_6_MASK | ++ CRI_TXDEEMPH_OVERRIDE_5_0_MASK); ++ val |= CRI_TXDEEMPH_OVERRIDE_5_0( ++ ddi_translations[level].cri_txdeemph_override_5_0) | ++ CRI_TXDEEMPH_OVERRIDE_11_6( ++ ddi_translations[level].cri_txdeemph_override_11_6) | ++ CRI_TXDEEMPH_OVERRIDE_EN; ++ I915_WRITE(MG_TX2_DRVCTRL(ln, port), val); ++ ++ /* FIXME: Program CRI_LOADGEN_SEL after the spec is updated */ ++ } ++ ++ /* ++ * Program MG_CLKHUB with value from frequency table ++ * In case of Legacy mode on MG PHY, both TX1 and TX2 enabled so use the ++ * values from table for which TX1 and TX2 enabled. ++ */ ++ for (ln = 0; ln < 2; ln++) { ++ val = I915_READ(MG_CLKHUB(ln, port)); ++ if (link_clock < 300000) ++ val |= CFG_LOW_RATE_LKREN_EN; ++ else ++ val &= ~CFG_LOW_RATE_LKREN_EN; ++ I915_WRITE(MG_CLKHUB(ln, port), val); ++ } ++ ++ /* Program the MG_TX_DCC based on the link frequency */ ++ for (ln = 0; ln < 2; ln++) { ++ val = I915_READ(MG_TX1_DCC(ln, port)); ++ val &= ~CFG_AMI_CK_DIV_OVERRIDE_VAL_MASK; ++ if (link_clock <= 500000) { ++ val &= ~CFG_AMI_CK_DIV_OVERRIDE_EN; ++ } else { ++ val |= CFG_AMI_CK_DIV_OVERRIDE_EN | ++ CFG_AMI_CK_DIV_OVERRIDE_VAL(1); ++ } ++ I915_WRITE(MG_TX1_DCC(ln, port), val); ++ ++ val = I915_READ(MG_TX2_DCC(ln, port)); ++ val &= ~CFG_AMI_CK_DIV_OVERRIDE_VAL_MASK; ++ if (link_clock <= 500000) { ++ val &= ~CFG_AMI_CK_DIV_OVERRIDE_EN; ++ } else { ++ val |= CFG_AMI_CK_DIV_OVERRIDE_EN | ++ CFG_AMI_CK_DIV_OVERRIDE_VAL(1); ++ } ++ I915_WRITE(MG_TX2_DCC(ln, port), val); ++ } ++ ++ /* Program MG_TX_PISO_READLOAD with values from vswing table */ ++ for (ln = 0; ln < 2; ln++) { ++ val = I915_READ(MG_TX1_PISO_READLOAD(ln, port)); ++ val |= CRI_CALCINIT; ++ I915_WRITE(MG_TX1_PISO_READLOAD(ln, port), val); ++ ++ val = I915_READ(MG_TX2_PISO_READLOAD(ln, port)); ++ val |= CRI_CALCINIT; ++ I915_WRITE(MG_TX2_PISO_READLOAD(ln, port), val); ++ } ++} ++ ++static void icl_ddi_vswing_sequence(struct intel_encoder *encoder, ++ int link_clock, ++ u32 level, ++ enum intel_output_type type) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ enum port port = encoder->port; ++ ++ if (intel_port_is_combophy(dev_priv, port)) ++ icl_combo_phy_ddi_vswing_sequence(encoder, level, type); ++ else ++ icl_mg_phy_ddi_vswing_sequence(encoder, link_clock, level); ++} ++ ++static u32 translate_signal_level(int signal_levels) ++{ ++ int i; ++ ++ for (i = 0; i < ARRAY_SIZE(index_to_dp_signal_levels); i++) { ++ if (index_to_dp_signal_levels[i] == signal_levels) ++ return i; ++ } ++ ++ WARN(1, "Unsupported voltage swing/pre-emphasis level: 0x%x\n", ++ signal_levels); ++ ++ return 0; ++} ++ ++static u32 intel_ddi_dp_level(struct intel_dp *intel_dp) ++{ ++ u8 train_set = intel_dp->train_set[0]; ++ int signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK | ++ DP_TRAIN_PRE_EMPHASIS_MASK); ++ ++ return translate_signal_level(signal_levels); ++} ++ ++u32 bxt_signal_levels(struct intel_dp *intel_dp) ++{ ++ struct intel_digital_port *dport = dp_to_dig_port(intel_dp); ++ struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev); ++ struct intel_encoder *encoder = &dport->base; ++ int level = intel_ddi_dp_level(intel_dp); ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ icl_ddi_vswing_sequence(encoder, intel_dp->link_rate, ++ level, encoder->type); ++ else if (IS_CANNONLAKE(dev_priv)) ++ cnl_ddi_vswing_sequence(encoder, level, encoder->type); ++ else ++ bxt_ddi_vswing_sequence(encoder, level, encoder->type); ++ ++ return 0; ++} ++ ++u32 ddi_signal_levels(struct intel_dp *intel_dp) ++{ ++ struct intel_digital_port *dport = dp_to_dig_port(intel_dp); ++ struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev); ++ struct intel_encoder *encoder = &dport->base; ++ int level = intel_ddi_dp_level(intel_dp); ++ ++ if (IS_GEN9_BC(dev_priv)) ++ skl_ddi_set_iboost(encoder, level, encoder->type); ++ ++ return DDI_BUF_TRANS_SELECT(level); ++} ++ ++static inline ++u32 icl_dpclka_cfgcr0_clk_off(struct drm_i915_private *dev_priv, ++ enum port port) ++{ ++ if (intel_port_is_combophy(dev_priv, port)) { ++ return ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(port); ++ } else if (intel_port_is_tc(dev_priv, port)) { ++ enum tc_port tc_port = intel_port_to_tc(dev_priv, port); ++ ++ return ICL_DPCLKA_CFGCR0_TC_CLK_OFF(tc_port); ++ } ++ ++ return 0; ++} ++ ++static void icl_map_plls_to_ports(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_shared_dpll *pll = crtc_state->shared_dpll; ++ enum port port = encoder->port; ++ u32 val; ++ ++ mutex_lock(&dev_priv->dpll_lock); ++ ++ val = I915_READ(DPCLKA_CFGCR0_ICL); ++ WARN_ON((val & icl_dpclka_cfgcr0_clk_off(dev_priv, port)) == 0); ++ ++ if (intel_port_is_combophy(dev_priv, port)) { ++ val &= ~DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port); ++ val |= DPCLKA_CFGCR0_DDI_CLK_SEL(pll->info->id, port); ++ I915_WRITE(DPCLKA_CFGCR0_ICL, val); ++ POSTING_READ(DPCLKA_CFGCR0_ICL); ++ } ++ ++ val &= ~icl_dpclka_cfgcr0_clk_off(dev_priv, port); ++ I915_WRITE(DPCLKA_CFGCR0_ICL, val); ++ ++ mutex_unlock(&dev_priv->dpll_lock); ++} ++ ++static void icl_unmap_plls_to_ports(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ enum port port = encoder->port; ++ u32 val; ++ ++ mutex_lock(&dev_priv->dpll_lock); ++ ++ val = I915_READ(DPCLKA_CFGCR0_ICL); ++ val |= icl_dpclka_cfgcr0_clk_off(dev_priv, port); ++ I915_WRITE(DPCLKA_CFGCR0_ICL, val); ++ ++ mutex_unlock(&dev_priv->dpll_lock); ++} ++ ++void icl_sanitize_encoder_pll_mapping(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ u32 val; ++ enum port port; ++ u32 port_mask; ++ bool ddi_clk_needed; ++ ++ /* ++ * In case of DP MST, we sanitize the primary encoder only, not the ++ * virtual ones. ++ */ ++ if (encoder->type == INTEL_OUTPUT_DP_MST) ++ return; ++ ++ if (!encoder->base.crtc && intel_encoder_is_dp(encoder)) { ++ u8 pipe_mask; ++ bool is_mst; ++ ++ intel_ddi_get_encoder_pipes(encoder, &pipe_mask, &is_mst); ++ /* ++ * In the unlikely case that BIOS enables DP in MST mode, just ++ * warn since our MST HW readout is incomplete. ++ */ ++ if (WARN_ON(is_mst)) ++ return; ++ } ++ ++ port_mask = BIT(encoder->port); ++ ddi_clk_needed = encoder->base.crtc; ++ ++ if (encoder->type == INTEL_OUTPUT_DSI) { ++ struct intel_encoder *other_encoder; ++ ++ port_mask = intel_dsi_encoder_ports(encoder); ++ /* ++ * Sanity check that we haven't incorrectly registered another ++ * encoder using any of the ports of this DSI encoder. ++ */ ++ for_each_intel_encoder(&dev_priv->drm, other_encoder) { ++ if (other_encoder == encoder) ++ continue; ++ ++ if (WARN_ON(port_mask & BIT(other_encoder->port))) ++ return; ++ } ++ /* ++ * For DSI we keep the ddi clocks gated ++ * except during enable/disable sequence. ++ */ ++ ddi_clk_needed = false; ++ } ++ ++ val = I915_READ(DPCLKA_CFGCR0_ICL); ++ for_each_port_masked(port, port_mask) { ++ bool ddi_clk_ungated = !(val & ++ icl_dpclka_cfgcr0_clk_off(dev_priv, ++ port)); ++ ++ if (ddi_clk_needed == ddi_clk_ungated) ++ continue; ++ ++ /* ++ * Punt on the case now where clock is gated, but it would ++ * be needed by the port. Something else is really broken then. ++ */ ++ if (WARN_ON(ddi_clk_needed)) ++ continue; ++ ++ DRM_NOTE("Port %c is disabled/in DSI mode with an ungated DDI clock, gate it\n", ++ port_name(port)); ++ val |= icl_dpclka_cfgcr0_clk_off(dev_priv, port); ++ I915_WRITE(DPCLKA_CFGCR0_ICL, val); ++ } ++} ++ ++static void intel_ddi_clk_select(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ enum port port = encoder->port; ++ u32 val; ++ const struct intel_shared_dpll *pll = crtc_state->shared_dpll; ++ ++ if (WARN_ON(!pll)) ++ return; ++ ++ mutex_lock(&dev_priv->dpll_lock); ++ ++ if (INTEL_GEN(dev_priv) >= 11) { ++ if (!intel_port_is_combophy(dev_priv, port)) ++ I915_WRITE(DDI_CLK_SEL(port), ++ icl_pll_to_ddi_clk_sel(encoder, crtc_state)); ++ } else if (IS_CANNONLAKE(dev_priv)) { ++ /* Configure DPCLKA_CFGCR0 to map the DPLL to the DDI. */ ++ val = I915_READ(DPCLKA_CFGCR0); ++ val &= ~DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port); ++ val |= DPCLKA_CFGCR0_DDI_CLK_SEL(pll->info->id, port); ++ I915_WRITE(DPCLKA_CFGCR0, val); ++ ++ /* ++ * Configure DPCLKA_CFGCR0 to turn on the clock for the DDI. ++ * This step and the step before must be done with separate ++ * register writes. ++ */ ++ val = I915_READ(DPCLKA_CFGCR0); ++ val &= ~DPCLKA_CFGCR0_DDI_CLK_OFF(port); ++ I915_WRITE(DPCLKA_CFGCR0, val); ++ } else if (IS_GEN9_BC(dev_priv)) { ++ /* DDI -> PLL mapping */ ++ val = I915_READ(DPLL_CTRL2); ++ ++ val &= ~(DPLL_CTRL2_DDI_CLK_OFF(port) | ++ DPLL_CTRL2_DDI_CLK_SEL_MASK(port)); ++ val |= (DPLL_CTRL2_DDI_CLK_SEL(pll->info->id, port) | ++ DPLL_CTRL2_DDI_SEL_OVERRIDE(port)); ++ ++ I915_WRITE(DPLL_CTRL2, val); ++ ++ } else if (INTEL_GEN(dev_priv) < 9) { ++ I915_WRITE(PORT_CLK_SEL(port), hsw_pll_to_ddi_pll_sel(pll)); ++ } ++ ++ mutex_unlock(&dev_priv->dpll_lock); ++} ++ ++static void intel_ddi_clk_disable(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ enum port port = encoder->port; ++ ++ if (INTEL_GEN(dev_priv) >= 11) { ++ if (!intel_port_is_combophy(dev_priv, port)) ++ I915_WRITE(DDI_CLK_SEL(port), DDI_CLK_SEL_NONE); ++ } else if (IS_CANNONLAKE(dev_priv)) { ++ I915_WRITE(DPCLKA_CFGCR0, I915_READ(DPCLKA_CFGCR0) | ++ DPCLKA_CFGCR0_DDI_CLK_OFF(port)); ++ } else if (IS_GEN9_BC(dev_priv)) { ++ I915_WRITE(DPLL_CTRL2, I915_READ(DPLL_CTRL2) | ++ DPLL_CTRL2_DDI_CLK_OFF(port)); ++ } else if (INTEL_GEN(dev_priv) < 9) { ++ I915_WRITE(PORT_CLK_SEL(port), PORT_CLK_SEL_NONE); ++ } ++} ++ ++static void icl_enable_phy_clock_gating(struct intel_digital_port *dig_port) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); ++ enum port port = dig_port->base.port; ++ enum tc_port tc_port = intel_port_to_tc(dev_priv, port); ++ u32 val; ++ int ln; ++ ++ if (tc_port == PORT_TC_NONE) ++ return; ++ ++ for (ln = 0; ln < 2; ln++) { ++ val = I915_READ(MG_DP_MODE(ln, port)); ++ val |= MG_DP_MODE_CFG_TR2PWR_GATING | ++ MG_DP_MODE_CFG_TRPWR_GATING | ++ MG_DP_MODE_CFG_CLNPWR_GATING | ++ MG_DP_MODE_CFG_DIGPWR_GATING | ++ MG_DP_MODE_CFG_GAONPWR_GATING; ++ I915_WRITE(MG_DP_MODE(ln, port), val); ++ } ++ ++ val = I915_READ(MG_MISC_SUS0(tc_port)); ++ val |= MG_MISC_SUS0_SUSCLK_DYNCLKGATE_MODE(3) | ++ MG_MISC_SUS0_CFG_TR2PWR_GATING | ++ MG_MISC_SUS0_CFG_CL2PWR_GATING | ++ MG_MISC_SUS0_CFG_GAONPWR_GATING | ++ MG_MISC_SUS0_CFG_TRPWR_GATING | ++ MG_MISC_SUS0_CFG_CL1PWR_GATING | ++ MG_MISC_SUS0_CFG_DGPWR_GATING; ++ I915_WRITE(MG_MISC_SUS0(tc_port), val); ++} ++ ++static void icl_disable_phy_clock_gating(struct intel_digital_port *dig_port) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); ++ enum port port = dig_port->base.port; ++ enum tc_port tc_port = intel_port_to_tc(dev_priv, port); ++ u32 val; ++ int ln; ++ ++ if (tc_port == PORT_TC_NONE) ++ return; ++ ++ for (ln = 0; ln < 2; ln++) { ++ val = I915_READ(MG_DP_MODE(ln, port)); ++ val &= ~(MG_DP_MODE_CFG_TR2PWR_GATING | ++ MG_DP_MODE_CFG_TRPWR_GATING | ++ MG_DP_MODE_CFG_CLNPWR_GATING | ++ MG_DP_MODE_CFG_DIGPWR_GATING | ++ MG_DP_MODE_CFG_GAONPWR_GATING); ++ I915_WRITE(MG_DP_MODE(ln, port), val); ++ } ++ ++ val = I915_READ(MG_MISC_SUS0(tc_port)); ++ val &= ~(MG_MISC_SUS0_SUSCLK_DYNCLKGATE_MODE_MASK | ++ MG_MISC_SUS0_CFG_TR2PWR_GATING | ++ MG_MISC_SUS0_CFG_CL2PWR_GATING | ++ MG_MISC_SUS0_CFG_GAONPWR_GATING | ++ MG_MISC_SUS0_CFG_TRPWR_GATING | ++ MG_MISC_SUS0_CFG_CL1PWR_GATING | ++ MG_MISC_SUS0_CFG_DGPWR_GATING); ++ I915_WRITE(MG_MISC_SUS0(tc_port), val); ++} ++ ++static void icl_program_mg_dp_mode(struct intel_digital_port *intel_dig_port) ++{ ++ struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev); ++ enum port port = intel_dig_port->base.port; ++ enum tc_port tc_port = intel_port_to_tc(dev_priv, port); ++ u32 ln0, ln1, lane_info; ++ ++ if (tc_port == PORT_TC_NONE || intel_dig_port->tc_type == TC_PORT_TBT) ++ return; ++ ++ ln0 = I915_READ(MG_DP_MODE(0, port)); ++ ln1 = I915_READ(MG_DP_MODE(1, port)); ++ ++ switch (intel_dig_port->tc_type) { ++ case TC_PORT_TYPEC: ++ ln0 &= ~(MG_DP_MODE_CFG_DP_X1_MODE | MG_DP_MODE_CFG_DP_X2_MODE); ++ ln1 &= ~(MG_DP_MODE_CFG_DP_X1_MODE | MG_DP_MODE_CFG_DP_X2_MODE); ++ ++ lane_info = (I915_READ(PORT_TX_DFLEXDPSP) & ++ DP_LANE_ASSIGNMENT_MASK(tc_port)) >> ++ DP_LANE_ASSIGNMENT_SHIFT(tc_port); ++ ++ switch (lane_info) { ++ case 0x1: ++ case 0x4: ++ break; ++ case 0x2: ++ ln0 |= MG_DP_MODE_CFG_DP_X1_MODE; ++ break; ++ case 0x3: ++ ln0 |= MG_DP_MODE_CFG_DP_X1_MODE | ++ MG_DP_MODE_CFG_DP_X2_MODE; ++ break; ++ case 0x8: ++ ln1 |= MG_DP_MODE_CFG_DP_X1_MODE; ++ break; ++ case 0xC: ++ ln1 |= MG_DP_MODE_CFG_DP_X1_MODE | ++ MG_DP_MODE_CFG_DP_X2_MODE; ++ break; ++ case 0xF: ++ ln0 |= MG_DP_MODE_CFG_DP_X1_MODE | ++ MG_DP_MODE_CFG_DP_X2_MODE; ++ ln1 |= MG_DP_MODE_CFG_DP_X1_MODE | ++ MG_DP_MODE_CFG_DP_X2_MODE; ++ break; ++ default: ++ MISSING_CASE(lane_info); ++ } ++ break; ++ ++ case TC_PORT_LEGACY: ++ ln0 |= MG_DP_MODE_CFG_DP_X1_MODE | MG_DP_MODE_CFG_DP_X2_MODE; ++ ln1 |= MG_DP_MODE_CFG_DP_X1_MODE | MG_DP_MODE_CFG_DP_X2_MODE; ++ break; ++ ++ default: ++ MISSING_CASE(intel_dig_port->tc_type); ++ return; ++ } ++ ++ I915_WRITE(MG_DP_MODE(0, port), ln0); ++ I915_WRITE(MG_DP_MODE(1, port), ln1); ++} ++ ++static void intel_dp_sink_set_fec_ready(struct intel_dp *intel_dp, ++ const struct intel_crtc_state *crtc_state) ++{ ++ if (!crtc_state->fec_enable) ++ return; ++ ++ if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_FEC_CONFIGURATION, DP_FEC_READY) <= 0) ++ DRM_DEBUG_KMS("Failed to set FEC_READY in the sink\n"); ++} ++ ++static void intel_ddi_enable_fec(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ enum port port = encoder->port; ++ u32 val; ++ ++ if (!crtc_state->fec_enable) ++ return; ++ ++ val = I915_READ(DP_TP_CTL(port)); ++ val |= DP_TP_CTL_FEC_ENABLE; ++ I915_WRITE(DP_TP_CTL(port), val); ++ ++ if (intel_wait_for_register(&dev_priv->uncore, DP_TP_STATUS(port), ++ DP_TP_STATUS_FEC_ENABLE_LIVE, ++ DP_TP_STATUS_FEC_ENABLE_LIVE, ++ 1)) ++ DRM_ERROR("Timed out waiting for FEC Enable Status\n"); ++} ++ ++static void intel_ddi_disable_fec_state(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ enum port port = encoder->port; ++ u32 val; ++ ++ if (!crtc_state->fec_enable) ++ return; ++ ++ val = I915_READ(DP_TP_CTL(port)); ++ val &= ~DP_TP_CTL_FEC_ENABLE; ++ I915_WRITE(DP_TP_CTL(port), val); ++ POSTING_READ(DP_TP_CTL(port)); ++} ++ ++static void intel_ddi_pre_enable_dp(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state) ++{ ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ enum port port = encoder->port; ++ struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base); ++ bool is_mst = intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST); ++ int level = intel_ddi_dp_level(intel_dp); ++ ++ WARN_ON(is_mst && (port == PORT_A || port == PORT_E)); ++ ++ intel_dp_set_link_params(intel_dp, crtc_state->port_clock, ++ crtc_state->lane_count, is_mst); ++ ++ intel_edp_panel_on(intel_dp); ++ ++ intel_ddi_clk_select(encoder, crtc_state); ++ ++ intel_display_power_get(dev_priv, dig_port->ddi_io_power_domain); ++ ++ icl_program_mg_dp_mode(dig_port); ++ icl_disable_phy_clock_gating(dig_port); ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ icl_ddi_vswing_sequence(encoder, crtc_state->port_clock, ++ level, encoder->type); ++ else if (IS_CANNONLAKE(dev_priv)) ++ cnl_ddi_vswing_sequence(encoder, level, encoder->type); ++ else if (IS_GEN9_LP(dev_priv)) ++ bxt_ddi_vswing_sequence(encoder, level, encoder->type); ++ else ++ intel_prepare_dp_ddi_buffers(encoder, crtc_state); ++ ++ intel_ddi_init_dp_buf_reg(encoder); ++ if (!is_mst) ++ intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON); ++ intel_dp_sink_set_decompression_state(intel_dp, crtc_state, ++ true); ++ intel_dp_sink_set_fec_ready(intel_dp, crtc_state); ++ intel_dp_start_link_train(intel_dp); ++ if (port != PORT_A || INTEL_GEN(dev_priv) >= 9) ++ intel_dp_stop_link_train(intel_dp); ++ ++ intel_ddi_enable_fec(encoder, crtc_state); ++ ++ icl_enable_phy_clock_gating(dig_port); ++ ++ if (!is_mst) ++ intel_ddi_enable_pipe_clock(crtc_state); ++ ++ intel_dsc_enable(encoder, crtc_state); ++} ++ ++static void intel_ddi_pre_enable_hdmi(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state) ++{ ++ struct intel_digital_port *intel_dig_port = enc_to_dig_port(&encoder->base); ++ struct intel_hdmi *intel_hdmi = &intel_dig_port->hdmi; ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ enum port port = encoder->port; ++ int level = intel_ddi_hdmi_level(dev_priv, port); ++ struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base); ++ ++ intel_dp_dual_mode_set_tmds_output(intel_hdmi, true); ++ intel_ddi_clk_select(encoder, crtc_state); ++ ++ intel_display_power_get(dev_priv, dig_port->ddi_io_power_domain); ++ ++ icl_program_mg_dp_mode(dig_port); ++ icl_disable_phy_clock_gating(dig_port); ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ icl_ddi_vswing_sequence(encoder, crtc_state->port_clock, ++ level, INTEL_OUTPUT_HDMI); ++ else if (IS_CANNONLAKE(dev_priv)) ++ cnl_ddi_vswing_sequence(encoder, level, INTEL_OUTPUT_HDMI); ++ else if (IS_GEN9_LP(dev_priv)) ++ bxt_ddi_vswing_sequence(encoder, level, INTEL_OUTPUT_HDMI); ++ else ++ intel_prepare_hdmi_ddi_buffers(encoder, level); ++ ++ icl_enable_phy_clock_gating(dig_port); ++ ++ if (IS_GEN9_BC(dev_priv)) ++ skl_ddi_set_iboost(encoder, level, INTEL_OUTPUT_HDMI); ++ ++ intel_ddi_enable_pipe_clock(crtc_state); ++ ++ intel_dig_port->set_infoframes(encoder, ++ crtc_state->has_infoframe, ++ crtc_state, conn_state); ++} ++ ++static void intel_ddi_pre_enable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ ++ /* ++ * When called from DP MST code: ++ * - conn_state will be NULL ++ * - encoder will be the main encoder (ie. mst->primary) ++ * - the main connector associated with this port ++ * won't be active or linked to a crtc ++ * - crtc_state will be the state of the first stream to ++ * be activated on this port, and it may not be the same ++ * stream that will be deactivated last, but each stream ++ * should have a state that is identical when it comes to ++ * the DP link parameteres ++ */ ++ ++ WARN_ON(crtc_state->has_pch_encoder); ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ icl_map_plls_to_ports(encoder, crtc_state); ++ ++ intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); ++ ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) { ++ intel_ddi_pre_enable_hdmi(encoder, crtc_state, conn_state); ++ } else { ++ struct intel_lspcon *lspcon = ++ enc_to_intel_lspcon(&encoder->base); ++ ++ intel_ddi_pre_enable_dp(encoder, crtc_state, conn_state); ++ if (lspcon->active) { ++ struct intel_digital_port *dig_port = ++ enc_to_dig_port(&encoder->base); ++ ++ dig_port->set_infoframes(encoder, ++ crtc_state->has_infoframe, ++ crtc_state, conn_state); ++ } ++ } ++} ++ ++static void intel_disable_ddi_buf(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ enum port port = encoder->port; ++ bool wait = false; ++ u32 val; ++ ++ val = I915_READ(DDI_BUF_CTL(port)); ++ if (val & DDI_BUF_CTL_ENABLE) { ++ val &= ~DDI_BUF_CTL_ENABLE; ++ I915_WRITE(DDI_BUF_CTL(port), val); ++ wait = true; ++ } ++ ++ val = I915_READ(DP_TP_CTL(port)); ++ val &= ~(DP_TP_CTL_ENABLE | DP_TP_CTL_LINK_TRAIN_MASK); ++ val |= DP_TP_CTL_LINK_TRAIN_PAT1; ++ I915_WRITE(DP_TP_CTL(port), val); ++ ++ /* Disable FEC in DP Sink */ ++ intel_ddi_disable_fec_state(encoder, crtc_state); ++ ++ if (wait) ++ intel_wait_ddi_buf_idle(dev_priv, port); ++} ++ ++static void intel_ddi_post_disable_dp(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base); ++ struct intel_dp *intel_dp = &dig_port->dp; ++ bool is_mst = intel_crtc_has_type(old_crtc_state, ++ INTEL_OUTPUT_DP_MST); ++ ++ if (!is_mst) { ++ intel_ddi_disable_pipe_clock(old_crtc_state); ++ /* ++ * Power down sink before disabling the port, otherwise we end ++ * up getting interrupts from the sink on detecting link loss. ++ */ ++ intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF); ++ } ++ ++ intel_disable_ddi_buf(encoder, old_crtc_state); ++ ++ intel_edp_panel_vdd_on(intel_dp); ++ intel_edp_panel_off(intel_dp); ++ ++ intel_display_power_put_unchecked(dev_priv, ++ dig_port->ddi_io_power_domain); ++ ++ intel_ddi_clk_disable(encoder); ++} ++ ++static void intel_ddi_post_disable_hdmi(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base); ++ struct intel_hdmi *intel_hdmi = &dig_port->hdmi; ++ ++ dig_port->set_infoframes(encoder, false, ++ old_crtc_state, old_conn_state); ++ ++ intel_ddi_disable_pipe_clock(old_crtc_state); ++ ++ intel_disable_ddi_buf(encoder, old_crtc_state); ++ ++ intel_display_power_put_unchecked(dev_priv, ++ dig_port->ddi_io_power_domain); ++ ++ intel_ddi_clk_disable(encoder); ++ ++ intel_dp_dual_mode_set_tmds_output(intel_hdmi, false); ++} ++ ++static void intel_ddi_post_disable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ ++ /* ++ * When called from DP MST code: ++ * - old_conn_state will be NULL ++ * - encoder will be the main encoder (ie. mst->primary) ++ * - the main connector associated with this port ++ * won't be active or linked to a crtc ++ * - old_crtc_state will be the state of the last stream to ++ * be deactivated on this port, and it may not be the same ++ * stream that was activated last, but each stream ++ * should have a state that is identical when it comes to ++ * the DP link parameteres ++ */ ++ ++ if (intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_HDMI)) ++ intel_ddi_post_disable_hdmi(encoder, ++ old_crtc_state, old_conn_state); ++ else ++ intel_ddi_post_disable_dp(encoder, ++ old_crtc_state, old_conn_state); ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ icl_unmap_plls_to_ports(encoder); ++} ++ ++void intel_ddi_fdi_post_disable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ u32 val; ++ ++ /* ++ * Bspec lists this as both step 13 (before DDI_BUF_CTL disable) ++ * and step 18 (after clearing PORT_CLK_SEL). Based on a BUN, ++ * step 13 is the correct place for it. Step 18 is where it was ++ * originally before the BUN. ++ */ ++ val = I915_READ(FDI_RX_CTL(PIPE_A)); ++ val &= ~FDI_RX_ENABLE; ++ I915_WRITE(FDI_RX_CTL(PIPE_A), val); ++ ++ intel_disable_ddi_buf(encoder, old_crtc_state); ++ intel_ddi_clk_disable(encoder); ++ ++ val = I915_READ(FDI_RX_MISC(PIPE_A)); ++ val &= ~(FDI_RX_PWRDN_LANE1_MASK | FDI_RX_PWRDN_LANE0_MASK); ++ val |= FDI_RX_PWRDN_LANE1_VAL(2) | FDI_RX_PWRDN_LANE0_VAL(2); ++ I915_WRITE(FDI_RX_MISC(PIPE_A), val); ++ ++ val = I915_READ(FDI_RX_CTL(PIPE_A)); ++ val &= ~FDI_PCDCLK; ++ I915_WRITE(FDI_RX_CTL(PIPE_A), val); ++ ++ val = I915_READ(FDI_RX_CTL(PIPE_A)); ++ val &= ~FDI_RX_PLL_ENABLE; ++ I915_WRITE(FDI_RX_CTL(PIPE_A), val); ++} ++ ++static void intel_enable_ddi_dp(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ enum port port = encoder->port; ++ ++ if (port == PORT_A && INTEL_GEN(dev_priv) < 9) ++ intel_dp_stop_link_train(intel_dp); ++ ++ intel_edp_backlight_on(crtc_state, conn_state); ++ intel_psr_enable(intel_dp, crtc_state); ++ intel_edp_drrs_enable(intel_dp, crtc_state); ++ ++ if (crtc_state->has_audio) ++ intel_audio_codec_enable(encoder, crtc_state, conn_state); ++} ++ ++static i915_reg_t ++gen9_chicken_trans_reg_by_port(struct drm_i915_private *dev_priv, ++ enum port port) ++{ ++ static const i915_reg_t regs[] = { ++ [PORT_A] = CHICKEN_TRANS_EDP, ++ [PORT_B] = CHICKEN_TRANS_A, ++ [PORT_C] = CHICKEN_TRANS_B, ++ [PORT_D] = CHICKEN_TRANS_C, ++ [PORT_E] = CHICKEN_TRANS_A, ++ }; ++ ++ WARN_ON(INTEL_GEN(dev_priv) < 9); ++ ++ if (WARN_ON(port < PORT_A || port > PORT_E)) ++ port = PORT_A; ++ ++ return regs[port]; ++} ++ ++static void intel_enable_ddi_hdmi(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base); ++ struct drm_connector *connector = conn_state->connector; ++ enum port port = encoder->port; ++ ++ if (!intel_hdmi_handle_sink_scrambling(encoder, connector, ++ crtc_state->hdmi_high_tmds_clock_ratio, ++ crtc_state->hdmi_scrambling)) ++ DRM_ERROR("[CONNECTOR:%d:%s] Failed to configure sink scrambling/TMDS bit clock ratio\n", ++ connector->base.id, connector->name); ++ ++ /* Display WA #1143: skl,kbl,cfl */ ++ if (IS_GEN9_BC(dev_priv)) { ++ /* ++ * For some reason these chicken bits have been ++ * stuffed into a transcoder register, event though ++ * the bits affect a specific DDI port rather than ++ * a specific transcoder. ++ */ ++ i915_reg_t reg = gen9_chicken_trans_reg_by_port(dev_priv, port); ++ u32 val; ++ ++ val = I915_READ(reg); ++ ++ if (port == PORT_E) ++ val |= DDIE_TRAINING_OVERRIDE_ENABLE | ++ DDIE_TRAINING_OVERRIDE_VALUE; ++ else ++ val |= DDI_TRAINING_OVERRIDE_ENABLE | ++ DDI_TRAINING_OVERRIDE_VALUE; ++ ++ I915_WRITE(reg, val); ++ POSTING_READ(reg); ++ ++ udelay(1); ++ ++ if (port == PORT_E) ++ val &= ~(DDIE_TRAINING_OVERRIDE_ENABLE | ++ DDIE_TRAINING_OVERRIDE_VALUE); ++ else ++ val &= ~(DDI_TRAINING_OVERRIDE_ENABLE | ++ DDI_TRAINING_OVERRIDE_VALUE); ++ ++ I915_WRITE(reg, val); ++ } ++ ++ /* In HDMI/DVI mode, the port width, and swing/emphasis values ++ * are ignored so nothing special needs to be done besides ++ * enabling the port. ++ */ ++ I915_WRITE(DDI_BUF_CTL(port), ++ dig_port->saved_port_bits | DDI_BUF_CTL_ENABLE); ++ ++ if (crtc_state->has_audio) ++ intel_audio_codec_enable(encoder, crtc_state, conn_state); ++} ++ ++static void intel_enable_ddi(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state) ++{ ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) ++ intel_enable_ddi_hdmi(encoder, crtc_state, conn_state); ++ else ++ intel_enable_ddi_dp(encoder, crtc_state, conn_state); ++ ++ /* Enable hdcp if it's desired */ ++ if (conn_state->content_protection == ++ DRM_MODE_CONTENT_PROTECTION_DESIRED) ++ intel_hdcp_enable(to_intel_connector(conn_state->connector)); ++} ++ ++static void intel_disable_ddi_dp(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ ++ intel_dp->link_trained = false; ++ ++ if (old_crtc_state->has_audio) ++ intel_audio_codec_disable(encoder, ++ old_crtc_state, old_conn_state); ++ ++ intel_edp_drrs_disable(intel_dp, old_crtc_state); ++ intel_psr_disable(intel_dp, old_crtc_state); ++ intel_edp_backlight_off(old_conn_state); ++ /* Disable the decompression in DP Sink */ ++ intel_dp_sink_set_decompression_state(intel_dp, old_crtc_state, ++ false); ++} ++ ++static void intel_disable_ddi_hdmi(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ struct drm_connector *connector = old_conn_state->connector; ++ ++ if (old_crtc_state->has_audio) ++ intel_audio_codec_disable(encoder, ++ old_crtc_state, old_conn_state); ++ ++ if (!intel_hdmi_handle_sink_scrambling(encoder, connector, ++ false, false)) ++ DRM_DEBUG_KMS("[CONNECTOR:%d:%s] Failed to reset sink scrambling/TMDS bit clock ratio\n", ++ connector->base.id, connector->name); ++} ++ ++static void intel_disable_ddi(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ intel_hdcp_disable(to_intel_connector(old_conn_state->connector)); ++ ++ if (intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_HDMI)) ++ intel_disable_ddi_hdmi(encoder, old_crtc_state, old_conn_state); ++ else ++ intel_disable_ddi_dp(encoder, old_crtc_state, old_conn_state); ++} ++ ++static void intel_ddi_update_pipe_dp(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state) ++{ ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ ++ intel_ddi_set_pipe_settings(crtc_state); ++ ++ intel_psr_update(intel_dp, crtc_state); ++ intel_edp_drrs_enable(intel_dp, crtc_state); ++ ++ intel_panel_update_backlight(encoder, crtc_state, conn_state); ++} ++ ++static void intel_ddi_update_pipe(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state) ++{ ++ if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) ++ intel_ddi_update_pipe_dp(encoder, crtc_state, conn_state); ++ ++ if (conn_state->content_protection == ++ DRM_MODE_CONTENT_PROTECTION_DESIRED) ++ intel_hdcp_enable(to_intel_connector(conn_state->connector)); ++ else if (conn_state->content_protection == ++ DRM_MODE_CONTENT_PROTECTION_UNDESIRED) ++ intel_hdcp_disable(to_intel_connector(conn_state->connector)); ++} ++ ++static void intel_ddi_set_fia_lane_count(struct intel_encoder *encoder, ++ const struct intel_crtc_state *pipe_config, ++ enum port port) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base); ++ enum tc_port tc_port = intel_port_to_tc(dev_priv, port); ++ u32 val = I915_READ(PORT_TX_DFLEXDPMLE1); ++ bool lane_reversal = dig_port->saved_port_bits & DDI_BUF_PORT_REVERSAL; ++ ++ val &= ~DFLEXDPMLE1_DPMLETC_MASK(tc_port); ++ switch (pipe_config->lane_count) { ++ case 1: ++ val |= (lane_reversal) ? DFLEXDPMLE1_DPMLETC_ML3(tc_port) : ++ DFLEXDPMLE1_DPMLETC_ML0(tc_port); ++ break; ++ case 2: ++ val |= (lane_reversal) ? DFLEXDPMLE1_DPMLETC_ML3_2(tc_port) : ++ DFLEXDPMLE1_DPMLETC_ML1_0(tc_port); ++ break; ++ case 4: ++ val |= DFLEXDPMLE1_DPMLETC_ML3_0(tc_port); ++ break; ++ default: ++ MISSING_CASE(pipe_config->lane_count); ++ } ++ I915_WRITE(PORT_TX_DFLEXDPMLE1, val); ++} ++ ++static void ++intel_ddi_pre_pll_enable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base); ++ enum port port = encoder->port; ++ ++ if (intel_crtc_has_dp_encoder(crtc_state) || ++ intel_port_is_tc(dev_priv, encoder->port)) ++ intel_display_power_get(dev_priv, ++ intel_ddi_main_link_aux_domain(dig_port)); ++ ++ if (IS_GEN9_LP(dev_priv)) ++ bxt_ddi_phy_set_lane_optim_mask(encoder, ++ crtc_state->lane_lat_optim_mask); ++ ++ /* ++ * Program the lane count for static/dynamic connections on Type-C ports. ++ * Skip this step for TBT. ++ */ ++ if (dig_port->tc_type == TC_PORT_UNKNOWN || ++ dig_port->tc_type == TC_PORT_TBT) ++ return; ++ ++ intel_ddi_set_fia_lane_count(encoder, crtc_state, port); ++} ++ ++static void ++intel_ddi_post_pll_disable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base); ++ ++ if (intel_crtc_has_dp_encoder(crtc_state) || ++ intel_port_is_tc(dev_priv, encoder->port)) ++ intel_display_power_put_unchecked(dev_priv, ++ intel_ddi_main_link_aux_domain(dig_port)); ++} ++ ++void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp) ++{ ++ struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); ++ struct drm_i915_private *dev_priv = ++ to_i915(intel_dig_port->base.base.dev); ++ enum port port = intel_dig_port->base.port; ++ u32 val; ++ bool wait = false; ++ ++ if (I915_READ(DP_TP_CTL(port)) & DP_TP_CTL_ENABLE) { ++ val = I915_READ(DDI_BUF_CTL(port)); ++ if (val & DDI_BUF_CTL_ENABLE) { ++ val &= ~DDI_BUF_CTL_ENABLE; ++ I915_WRITE(DDI_BUF_CTL(port), val); ++ wait = true; ++ } ++ ++ val = I915_READ(DP_TP_CTL(port)); ++ val &= ~(DP_TP_CTL_ENABLE | DP_TP_CTL_LINK_TRAIN_MASK); ++ val |= DP_TP_CTL_LINK_TRAIN_PAT1; ++ I915_WRITE(DP_TP_CTL(port), val); ++ POSTING_READ(DP_TP_CTL(port)); ++ ++ if (wait) ++ intel_wait_ddi_buf_idle(dev_priv, port); ++ } ++ ++ val = DP_TP_CTL_ENABLE | ++ DP_TP_CTL_LINK_TRAIN_PAT1 | DP_TP_CTL_SCRAMBLE_DISABLE; ++ if (intel_dp->link_mst) ++ val |= DP_TP_CTL_MODE_MST; ++ else { ++ val |= DP_TP_CTL_MODE_SST; ++ if (drm_dp_enhanced_frame_cap(intel_dp->dpcd)) ++ val |= DP_TP_CTL_ENHANCED_FRAME_ENABLE; ++ } ++ I915_WRITE(DP_TP_CTL(port), val); ++ POSTING_READ(DP_TP_CTL(port)); ++ ++ intel_dp->DP |= DDI_BUF_CTL_ENABLE; ++ I915_WRITE(DDI_BUF_CTL(port), intel_dp->DP); ++ POSTING_READ(DDI_BUF_CTL(port)); ++ ++ udelay(600); ++} ++ ++static bool intel_ddi_is_audio_enabled(struct drm_i915_private *dev_priv, ++ enum transcoder cpu_transcoder) ++{ ++ if (cpu_transcoder == TRANSCODER_EDP) ++ return false; ++ ++ if (!intel_display_power_is_enabled(dev_priv, POWER_DOMAIN_AUDIO)) ++ return false; ++ ++ return I915_READ(HSW_AUD_PIN_ELD_CP_VLD) & ++ AUDIO_OUTPUT_ENABLE(cpu_transcoder); ++} ++ ++void intel_ddi_compute_min_voltage_level(struct drm_i915_private *dev_priv, ++ struct intel_crtc_state *crtc_state) ++{ ++ if (INTEL_GEN(dev_priv) >= 11 && crtc_state->port_clock > 594000) ++ crtc_state->min_voltage_level = 1; ++ else if (IS_CANNONLAKE(dev_priv) && crtc_state->port_clock > 594000) ++ crtc_state->min_voltage_level = 2; ++} ++ ++void intel_ddi_get_config(struct intel_encoder *encoder, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_crtc *intel_crtc = to_intel_crtc(pipe_config->base.crtc); ++ enum transcoder cpu_transcoder = pipe_config->cpu_transcoder; ++ struct intel_digital_port *intel_dig_port; ++ u32 temp, flags = 0; ++ ++ /* XXX: DSI transcoder paranoia */ ++ if (WARN_ON(transcoder_is_dsi(cpu_transcoder))) ++ return; ++ ++ temp = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder)); ++ if (temp & TRANS_DDI_PHSYNC) ++ flags |= DRM_MODE_FLAG_PHSYNC; ++ else ++ flags |= DRM_MODE_FLAG_NHSYNC; ++ if (temp & TRANS_DDI_PVSYNC) ++ flags |= DRM_MODE_FLAG_PVSYNC; ++ else ++ flags |= DRM_MODE_FLAG_NVSYNC; ++ ++ pipe_config->base.adjusted_mode.flags |= flags; ++ ++ switch (temp & TRANS_DDI_BPC_MASK) { ++ case TRANS_DDI_BPC_6: ++ pipe_config->pipe_bpp = 18; ++ break; ++ case TRANS_DDI_BPC_8: ++ pipe_config->pipe_bpp = 24; ++ break; ++ case TRANS_DDI_BPC_10: ++ pipe_config->pipe_bpp = 30; ++ break; ++ case TRANS_DDI_BPC_12: ++ pipe_config->pipe_bpp = 36; ++ break; ++ default: ++ break; ++ } ++ ++ switch (temp & TRANS_DDI_MODE_SELECT_MASK) { ++ case TRANS_DDI_MODE_SELECT_HDMI: ++ pipe_config->has_hdmi_sink = true; ++ intel_dig_port = enc_to_dig_port(&encoder->base); ++ ++ pipe_config->infoframes.enable |= ++ intel_hdmi_infoframes_enabled(encoder, pipe_config); ++ ++ if (pipe_config->infoframes.enable) ++ pipe_config->has_infoframe = true; ++ ++ if (temp & TRANS_DDI_HDMI_SCRAMBLING) ++ pipe_config->hdmi_scrambling = true; ++ if (temp & TRANS_DDI_HIGH_TMDS_CHAR_RATE) ++ pipe_config->hdmi_high_tmds_clock_ratio = true; ++ /* fall through */ ++ case TRANS_DDI_MODE_SELECT_DVI: ++ pipe_config->output_types |= BIT(INTEL_OUTPUT_HDMI); ++ pipe_config->lane_count = 4; ++ break; ++ case TRANS_DDI_MODE_SELECT_FDI: ++ pipe_config->output_types |= BIT(INTEL_OUTPUT_ANALOG); ++ break; ++ case TRANS_DDI_MODE_SELECT_DP_SST: ++ if (encoder->type == INTEL_OUTPUT_EDP) ++ pipe_config->output_types |= BIT(INTEL_OUTPUT_EDP); ++ else ++ pipe_config->output_types |= BIT(INTEL_OUTPUT_DP); ++ pipe_config->lane_count = ++ ((temp & DDI_PORT_WIDTH_MASK) >> DDI_PORT_WIDTH_SHIFT) + 1; ++ intel_dp_get_m_n(intel_crtc, pipe_config); ++ break; ++ case TRANS_DDI_MODE_SELECT_DP_MST: ++ pipe_config->output_types |= BIT(INTEL_OUTPUT_DP_MST); ++ pipe_config->lane_count = ++ ((temp & DDI_PORT_WIDTH_MASK) >> DDI_PORT_WIDTH_SHIFT) + 1; ++ intel_dp_get_m_n(intel_crtc, pipe_config); ++ break; ++ default: ++ break; ++ } ++ ++ pipe_config->has_audio = ++ intel_ddi_is_audio_enabled(dev_priv, cpu_transcoder); ++ ++ if (encoder->type == INTEL_OUTPUT_EDP && dev_priv->vbt.edp.bpp && ++ pipe_config->pipe_bpp > dev_priv->vbt.edp.bpp) { ++ /* ++ * This is a big fat ugly hack. ++ * ++ * Some machines in UEFI boot mode provide us a VBT that has 18 ++ * bpp and 1.62 GHz link bandwidth for eDP, which for reasons ++ * unknown we fail to light up. Yet the same BIOS boots up with ++ * 24 bpp and 2.7 GHz link. Use the same bpp as the BIOS uses as ++ * max, not what it tells us to use. ++ * ++ * Note: This will still be broken if the eDP panel is not lit ++ * up by the BIOS, and thus we can't get the mode at module ++ * load. ++ */ ++ DRM_DEBUG_KMS("pipe has %d bpp for eDP panel, overriding BIOS-provided max %d bpp\n", ++ pipe_config->pipe_bpp, dev_priv->vbt.edp.bpp); ++ dev_priv->vbt.edp.bpp = pipe_config->pipe_bpp; ++ } ++ ++ intel_ddi_clock_get(encoder, pipe_config); ++ ++ if (IS_GEN9_LP(dev_priv)) ++ pipe_config->lane_lat_optim_mask = ++ bxt_ddi_phy_get_lane_lat_optim_mask(encoder); ++ ++ intel_ddi_compute_min_voltage_level(dev_priv, pipe_config); ++ ++ intel_hdmi_read_gcp_infoframe(encoder, pipe_config); ++ ++ intel_read_infoframe(encoder, pipe_config, ++ HDMI_INFOFRAME_TYPE_AVI, ++ &pipe_config->infoframes.avi); ++ intel_read_infoframe(encoder, pipe_config, ++ HDMI_INFOFRAME_TYPE_SPD, ++ &pipe_config->infoframes.spd); ++ intel_read_infoframe(encoder, pipe_config, ++ HDMI_INFOFRAME_TYPE_VENDOR, ++ &pipe_config->infoframes.hdmi); ++} ++ ++static enum intel_output_type ++intel_ddi_compute_output_type(struct intel_encoder *encoder, ++ struct intel_crtc_state *crtc_state, ++ struct drm_connector_state *conn_state) ++{ ++ switch (conn_state->connector->connector_type) { ++ case DRM_MODE_CONNECTOR_HDMIA: ++ return INTEL_OUTPUT_HDMI; ++ case DRM_MODE_CONNECTOR_eDP: ++ return INTEL_OUTPUT_EDP; ++ case DRM_MODE_CONNECTOR_DisplayPort: ++ return INTEL_OUTPUT_DP; ++ default: ++ MISSING_CASE(conn_state->connector->connector_type); ++ return INTEL_OUTPUT_UNUSED; ++ } ++} ++ ++static int intel_ddi_compute_config(struct intel_encoder *encoder, ++ struct intel_crtc_state *pipe_config, ++ struct drm_connector_state *conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ enum port port = encoder->port; ++ int ret; ++ ++ if (HAS_TRANSCODER_EDP(dev_priv) && port == PORT_A) ++ pipe_config->cpu_transcoder = TRANSCODER_EDP; ++ ++ if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_HDMI)) ++ ret = intel_hdmi_compute_config(encoder, pipe_config, conn_state); ++ else ++ ret = intel_dp_compute_config(encoder, pipe_config, conn_state); ++ if (ret) ++ return ret; ++ ++ if (IS_GEN9_LP(dev_priv)) ++ pipe_config->lane_lat_optim_mask = ++ bxt_ddi_phy_calc_lane_lat_optim_mask(pipe_config->lane_count); ++ ++ intel_ddi_compute_min_voltage_level(dev_priv, pipe_config); ++ ++ return 0; ++ ++} ++ ++static void intel_ddi_encoder_suspend(struct intel_encoder *encoder) ++{ ++ struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base); ++ struct drm_i915_private *i915 = to_i915(encoder->base.dev); ++ ++ intel_dp_encoder_suspend(encoder); ++ ++ /* ++ * TODO: disconnect also from USB DP alternate mode once we have a ++ * way to handle the modeset restore in that mode during resume ++ * even if the sink has disappeared while being suspended. ++ */ ++ if (dig_port->tc_legacy_port) ++ icl_tc_phy_disconnect(i915, dig_port); ++} ++ ++static void intel_ddi_encoder_reset(struct drm_encoder *drm_encoder) ++{ ++ struct intel_digital_port *dig_port = enc_to_dig_port(drm_encoder); ++ struct drm_i915_private *i915 = to_i915(drm_encoder->dev); ++ ++ if (intel_port_is_tc(i915, dig_port->base.port)) ++ intel_digital_port_connected(&dig_port->base); ++ ++ intel_dp_encoder_reset(drm_encoder); ++} ++ ++static void intel_ddi_encoder_destroy(struct drm_encoder *encoder) ++{ ++ struct intel_digital_port *dig_port = enc_to_dig_port(encoder); ++ struct drm_i915_private *i915 = to_i915(encoder->dev); ++ ++ intel_dp_encoder_flush_work(encoder); ++ ++ if (intel_port_is_tc(i915, dig_port->base.port)) ++ icl_tc_phy_disconnect(i915, dig_port); ++ ++ drm_encoder_cleanup(encoder); ++ kfree(dig_port); ++} ++ ++static const struct drm_encoder_funcs intel_ddi_funcs = { ++ .reset = intel_ddi_encoder_reset, ++ .destroy = intel_ddi_encoder_destroy, ++}; ++ ++static struct intel_connector * ++intel_ddi_init_dp_connector(struct intel_digital_port *intel_dig_port) ++{ ++ struct intel_connector *connector; ++ enum port port = intel_dig_port->base.port; ++ ++ connector = intel_connector_alloc(); ++ if (!connector) ++ return NULL; ++ ++ intel_dig_port->dp.output_reg = DDI_BUF_CTL(port); ++ if (!intel_dp_init_connector(intel_dig_port, connector)) { ++ kfree(connector); ++ return NULL; ++ } ++ ++ return connector; ++} ++ ++static int modeset_pipe(struct drm_crtc *crtc, ++ struct drm_modeset_acquire_ctx *ctx) ++{ ++ struct drm_atomic_state *state; ++ struct drm_crtc_state *crtc_state; ++ int ret; ++ ++ state = drm_atomic_state_alloc(crtc->dev); ++ if (!state) ++ return -ENOMEM; ++ ++ state->acquire_ctx = ctx; ++ ++ crtc_state = drm_atomic_get_crtc_state(state, crtc); ++ if (IS_ERR(crtc_state)) { ++ ret = PTR_ERR(crtc_state); ++ goto out; ++ } ++ ++ crtc_state->connectors_changed = true; ++ ++ ret = drm_atomic_commit(state); ++out: ++ drm_atomic_state_put(state); ++ ++ return ret; ++} ++ ++static int intel_hdmi_reset_link(struct intel_encoder *encoder, ++ struct drm_modeset_acquire_ctx *ctx) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_hdmi *hdmi = enc_to_intel_hdmi(&encoder->base); ++ struct intel_connector *connector = hdmi->attached_connector; ++ struct i2c_adapter *adapter = ++ intel_gmbus_get_adapter(dev_priv, hdmi->ddc_bus); ++ struct drm_connector_state *conn_state; ++ struct intel_crtc_state *crtc_state; ++ struct intel_crtc *crtc; ++ u8 config; ++ int ret; ++ ++ if (!connector || connector->base.status != connector_status_connected) ++ return 0; ++ ++ ret = drm_modeset_lock(&dev_priv->drm.mode_config.connection_mutex, ++ ctx); ++ if (ret) ++ return ret; ++ ++ conn_state = connector->base.state; ++ ++ crtc = to_intel_crtc(conn_state->crtc); ++ if (!crtc) ++ return 0; ++ ++ ret = drm_modeset_lock(&crtc->base.mutex, ctx); ++ if (ret) ++ return ret; ++ ++ crtc_state = to_intel_crtc_state(crtc->base.state); ++ ++ WARN_ON(!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)); ++ ++ if (!crtc_state->base.active) ++ return 0; ++ ++ if (!crtc_state->hdmi_high_tmds_clock_ratio && ++ !crtc_state->hdmi_scrambling) ++ return 0; ++ ++ if (conn_state->commit && ++ !try_wait_for_completion(&conn_state->commit->hw_done)) ++ return 0; ++ ++ ret = drm_scdc_readb(adapter, SCDC_TMDS_CONFIG, &config); ++ if (ret < 0) { ++ DRM_ERROR("Failed to read TMDS config: %d\n", ret); ++ return 0; ++ } ++ ++ if (!!(config & SCDC_TMDS_BIT_CLOCK_RATIO_BY_40) == ++ crtc_state->hdmi_high_tmds_clock_ratio && ++ !!(config & SCDC_SCRAMBLING_ENABLE) == ++ crtc_state->hdmi_scrambling) ++ return 0; ++ ++ /* ++ * HDMI 2.0 says that one should not send scrambled data ++ * prior to configuring the sink scrambling, and that ++ * TMDS clock/data transmission should be suspended when ++ * changing the TMDS clock rate in the sink. So let's ++ * just do a full modeset here, even though some sinks ++ * would be perfectly happy if were to just reconfigure ++ * the SCDC settings on the fly. ++ */ ++ return modeset_pipe(&crtc->base, ctx); ++} ++ ++static bool intel_ddi_hotplug(struct intel_encoder *encoder, ++ struct intel_connector *connector) ++{ ++ struct drm_modeset_acquire_ctx ctx; ++ bool changed; ++ int ret; ++ ++ changed = intel_encoder_hotplug(encoder, connector); ++ ++ drm_modeset_acquire_init(&ctx, 0); ++ ++ for (;;) { ++ if (connector->base.connector_type == DRM_MODE_CONNECTOR_HDMIA) ++ ret = intel_hdmi_reset_link(encoder, &ctx); ++ else ++ ret = intel_dp_retrain_link(encoder, &ctx); ++ ++ if (ret == -EDEADLK) { ++ drm_modeset_backoff(&ctx); ++ continue; ++ } ++ ++ break; ++ } ++ ++ drm_modeset_drop_locks(&ctx); ++ drm_modeset_acquire_fini(&ctx); ++ WARN(ret, "Acquiring modeset locks failed with %i\n", ret); ++ ++ return changed; ++} ++ ++static struct intel_connector * ++intel_ddi_init_hdmi_connector(struct intel_digital_port *intel_dig_port) ++{ ++ struct intel_connector *connector; ++ enum port port = intel_dig_port->base.port; ++ ++ connector = intel_connector_alloc(); ++ if (!connector) ++ return NULL; ++ ++ intel_dig_port->hdmi.hdmi_reg = DDI_BUF_CTL(port); ++ intel_hdmi_init_connector(intel_dig_port, connector); ++ ++ return connector; ++} ++ ++static bool intel_ddi_a_force_4_lanes(struct intel_digital_port *dport) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev); ++ ++ if (dport->base.port != PORT_A) ++ return false; ++ ++ if (dport->saved_port_bits & DDI_A_4_LANES) ++ return false; ++ ++ /* Broxton/Geminilake: Bspec says that DDI_A_4_LANES is the only ++ * supported configuration ++ */ ++ if (IS_GEN9_LP(dev_priv)) ++ return true; ++ ++ /* Cannonlake: Most of SKUs don't support DDI_E, and the only ++ * one who does also have a full A/E split called ++ * DDI_F what makes DDI_E useless. However for this ++ * case let's trust VBT info. ++ */ ++ if (IS_CANNONLAKE(dev_priv) && ++ !intel_bios_is_port_present(dev_priv, PORT_E)) ++ return true; ++ ++ return false; ++} ++ ++static int ++intel_ddi_max_lanes(struct intel_digital_port *intel_dport) ++{ ++ struct drm_i915_private *dev_priv = to_i915(intel_dport->base.base.dev); ++ enum port port = intel_dport->base.port; ++ int max_lanes = 4; ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ return max_lanes; ++ ++ if (port == PORT_A || port == PORT_E) { ++ if (I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES) ++ max_lanes = port == PORT_A ? 4 : 0; ++ else ++ /* Both A and E share 2 lanes */ ++ max_lanes = 2; ++ } ++ ++ /* ++ * Some BIOS might fail to set this bit on port A if eDP ++ * wasn't lit up at boot. Force this bit set when needed ++ * so we use the proper lane count for our calculations. ++ */ ++ if (intel_ddi_a_force_4_lanes(intel_dport)) { ++ DRM_DEBUG_KMS("Forcing DDI_A_4_LANES for port A\n"); ++ intel_dport->saved_port_bits |= DDI_A_4_LANES; ++ max_lanes = 4; ++ } ++ ++ return max_lanes; ++} ++ ++void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port) ++{ ++ struct ddi_vbt_port_info *port_info = ++ &dev_priv->vbt.ddi_port_info[port]; ++ struct intel_digital_port *intel_dig_port; ++ struct intel_encoder *intel_encoder; ++ struct drm_encoder *encoder; ++ bool init_hdmi, init_dp, init_lspcon = false; ++ enum pipe pipe; ++ ++ init_hdmi = port_info->supports_dvi || port_info->supports_hdmi; ++ init_dp = port_info->supports_dp; ++ ++ if (intel_bios_is_lspcon_present(dev_priv, port)) { ++ /* ++ * Lspcon device needs to be driven with DP connector ++ * with special detection sequence. So make sure DP ++ * is initialized before lspcon. ++ */ ++ init_dp = true; ++ init_lspcon = true; ++ init_hdmi = false; ++ DRM_DEBUG_KMS("VBT says port %c has lspcon\n", port_name(port)); ++ } ++ ++ if (!init_dp && !init_hdmi) { ++ DRM_DEBUG_KMS("VBT says port %c is not DVI/HDMI/DP compatible, respect it\n", ++ port_name(port)); ++ return; ++ } ++ ++ intel_dig_port = kzalloc(sizeof(*intel_dig_port), GFP_KERNEL); ++ if (!intel_dig_port) ++ return; ++ ++ intel_encoder = &intel_dig_port->base; ++ encoder = &intel_encoder->base; ++ ++ drm_encoder_init(&dev_priv->drm, encoder, &intel_ddi_funcs, ++ DRM_MODE_ENCODER_TMDS, "DDI %c", port_name(port)); ++ ++ intel_encoder->hotplug = intel_ddi_hotplug; ++ intel_encoder->compute_output_type = intel_ddi_compute_output_type; ++ intel_encoder->compute_config = intel_ddi_compute_config; ++ intel_encoder->enable = intel_enable_ddi; ++ intel_encoder->pre_pll_enable = intel_ddi_pre_pll_enable; ++ intel_encoder->post_pll_disable = intel_ddi_post_pll_disable; ++ intel_encoder->pre_enable = intel_ddi_pre_enable; ++ intel_encoder->disable = intel_disable_ddi; ++ intel_encoder->post_disable = intel_ddi_post_disable; ++ intel_encoder->update_pipe = intel_ddi_update_pipe; ++ intel_encoder->get_hw_state = intel_ddi_get_hw_state; ++ intel_encoder->get_config = intel_ddi_get_config; ++ intel_encoder->suspend = intel_ddi_encoder_suspend; ++ intel_encoder->get_power_domains = intel_ddi_get_power_domains; ++ intel_encoder->type = INTEL_OUTPUT_DDI; ++ intel_encoder->power_domain = intel_port_to_power_domain(port); ++ intel_encoder->port = port; ++ intel_encoder->cloneable = 0; ++ for_each_pipe(dev_priv, pipe) ++ intel_encoder->crtc_mask |= BIT(pipe); ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ intel_dig_port->saved_port_bits = I915_READ(DDI_BUF_CTL(port)) & ++ DDI_BUF_PORT_REVERSAL; ++ else ++ intel_dig_port->saved_port_bits = I915_READ(DDI_BUF_CTL(port)) & ++ (DDI_BUF_PORT_REVERSAL | DDI_A_4_LANES); ++ intel_dig_port->dp.output_reg = INVALID_MMIO_REG; ++ intel_dig_port->max_lanes = intel_ddi_max_lanes(intel_dig_port); ++ intel_dig_port->aux_ch = intel_bios_port_aux_ch(dev_priv, port); ++ ++ intel_dig_port->tc_legacy_port = intel_port_is_tc(dev_priv, port) && ++ !port_info->supports_typec_usb && ++ !port_info->supports_tbt; ++ ++ switch (port) { ++ case PORT_A: ++ intel_dig_port->ddi_io_power_domain = ++ POWER_DOMAIN_PORT_DDI_A_IO; ++ break; ++ case PORT_B: ++ intel_dig_port->ddi_io_power_domain = ++ POWER_DOMAIN_PORT_DDI_B_IO; ++ break; ++ case PORT_C: ++ intel_dig_port->ddi_io_power_domain = ++ POWER_DOMAIN_PORT_DDI_C_IO; ++ break; ++ case PORT_D: ++ intel_dig_port->ddi_io_power_domain = ++ POWER_DOMAIN_PORT_DDI_D_IO; ++ break; ++ case PORT_E: ++ intel_dig_port->ddi_io_power_domain = ++ POWER_DOMAIN_PORT_DDI_E_IO; ++ break; ++ case PORT_F: ++ intel_dig_port->ddi_io_power_domain = ++ POWER_DOMAIN_PORT_DDI_F_IO; ++ break; ++ default: ++ MISSING_CASE(port); ++ } ++ ++ if (init_dp) { ++ if (!intel_ddi_init_dp_connector(intel_dig_port)) ++ goto err; ++ ++ intel_dig_port->hpd_pulse = intel_dp_hpd_pulse; ++ } ++ ++ /* In theory we don't need the encoder->type check, but leave it just in ++ * case we have some really bad VBTs... */ ++ if (intel_encoder->type != INTEL_OUTPUT_EDP && init_hdmi) { ++ if (!intel_ddi_init_hdmi_connector(intel_dig_port)) ++ goto err; ++ } ++ ++ if (init_lspcon) { ++ if (lspcon_init(intel_dig_port)) ++ /* TODO: handle hdmi info frame part */ ++ DRM_DEBUG_KMS("LSPCON init success on port %c\n", ++ port_name(port)); ++ else ++ /* ++ * LSPCON init faied, but DP init was success, so ++ * lets try to drive as DP++ port. ++ */ ++ DRM_ERROR("LSPCON init failed on port %c\n", ++ port_name(port)); ++ } ++ ++ intel_infoframe_init(intel_dig_port); ++ ++ if (intel_port_is_tc(dev_priv, port)) ++ intel_digital_port_connected(intel_encoder); ++ ++ return; ++ ++err: ++ drm_encoder_cleanup(encoder); ++ kfree(intel_dig_port); ++} +diff --git a/drivers/gpu/drm/i915_legacy/intel_ddi.h b/drivers/gpu/drm/i915_legacy/intel_ddi.h +new file mode 100644 +index 000000000000..9cf69175942e +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_ddi.h +@@ -0,0 +1,53 @@ ++/* SPDX-License-Identifier: MIT */ ++/* ++ * Copyright © 2019 Intel Corporation ++ */ ++ ++#ifndef __INTEL_DDI_H__ ++#define __INTEL_DDI_H__ ++ ++#include ++ ++#include "intel_display.h" ++ ++struct drm_connector_state; ++struct drm_i915_private; ++struct intel_connector; ++struct intel_crtc; ++struct intel_crtc_state; ++struct intel_dp; ++struct intel_dpll_hw_state; ++struct intel_encoder; ++ ++void intel_ddi_fdi_post_disable(struct intel_encoder *intel_encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state); ++void hsw_fdi_link_train(struct intel_crtc *crtc, ++ const struct intel_crtc_state *crtc_state); ++void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port); ++bool intel_ddi_get_hw_state(struct intel_encoder *encoder, enum pipe *pipe); ++void intel_ddi_enable_transcoder_func(const struct intel_crtc_state *crtc_state); ++void intel_ddi_disable_transcoder_func(const struct intel_crtc_state *crtc_state); ++void intel_ddi_enable_pipe_clock(const struct intel_crtc_state *crtc_state); ++void intel_ddi_disable_pipe_clock(const struct intel_crtc_state *crtc_state); ++void intel_ddi_set_pipe_settings(const struct intel_crtc_state *crtc_state); ++void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp); ++bool intel_ddi_connector_get_hw_state(struct intel_connector *intel_connector); ++void intel_ddi_get_config(struct intel_encoder *encoder, ++ struct intel_crtc_state *pipe_config); ++void intel_ddi_set_vc_payload_alloc(const struct intel_crtc_state *crtc_state, ++ bool state); ++void intel_ddi_compute_min_voltage_level(struct drm_i915_private *dev_priv, ++ struct intel_crtc_state *crtc_state); ++u32 bxt_signal_levels(struct intel_dp *intel_dp); ++u32 ddi_signal_levels(struct intel_dp *intel_dp); ++u8 intel_ddi_dp_voltage_max(struct intel_encoder *encoder); ++u8 intel_ddi_dp_pre_emphasis_max(struct intel_encoder *encoder, ++ u8 voltage_swing); ++int intel_ddi_toggle_hdcp_signalling(struct intel_encoder *intel_encoder, ++ bool enable); ++void icl_sanitize_encoder_pll_mapping(struct intel_encoder *encoder); ++int cnl_calc_wrpll_link(struct drm_i915_private *dev_priv, ++ struct intel_dpll_hw_state *state); ++ ++#endif /* __INTEL_DDI_H__ */ +diff --git a/drivers/gpu/drm/i915_legacy/intel_device_info.c b/drivers/gpu/drm/i915_legacy/intel_device_info.c +new file mode 100644 +index 000000000000..6af480b95bc6 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_device_info.c +@@ -0,0 +1,1019 @@ ++/* ++ * Copyright © 2016 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#include ++ ++#include "intel_device_info.h" ++#include "i915_drv.h" ++ ++#define PLATFORM_NAME(x) [INTEL_##x] = #x ++static const char * const platform_names[] = { ++ PLATFORM_NAME(I830), ++ PLATFORM_NAME(I845G), ++ PLATFORM_NAME(I85X), ++ PLATFORM_NAME(I865G), ++ PLATFORM_NAME(I915G), ++ PLATFORM_NAME(I915GM), ++ PLATFORM_NAME(I945G), ++ PLATFORM_NAME(I945GM), ++ PLATFORM_NAME(G33), ++ PLATFORM_NAME(PINEVIEW), ++ PLATFORM_NAME(I965G), ++ PLATFORM_NAME(I965GM), ++ PLATFORM_NAME(G45), ++ PLATFORM_NAME(GM45), ++ PLATFORM_NAME(IRONLAKE), ++ PLATFORM_NAME(SANDYBRIDGE), ++ PLATFORM_NAME(IVYBRIDGE), ++ PLATFORM_NAME(VALLEYVIEW), ++ PLATFORM_NAME(HASWELL), ++ PLATFORM_NAME(BROADWELL), ++ PLATFORM_NAME(CHERRYVIEW), ++ PLATFORM_NAME(SKYLAKE), ++ PLATFORM_NAME(BROXTON), ++ PLATFORM_NAME(KABYLAKE), ++ PLATFORM_NAME(GEMINILAKE), ++ PLATFORM_NAME(COFFEELAKE), ++ PLATFORM_NAME(CANNONLAKE), ++ PLATFORM_NAME(ICELAKE), ++ PLATFORM_NAME(ELKHARTLAKE), ++}; ++#undef PLATFORM_NAME ++ ++const char *intel_platform_name(enum intel_platform platform) ++{ ++ BUILD_BUG_ON(ARRAY_SIZE(platform_names) != INTEL_MAX_PLATFORMS); ++ ++ if (WARN_ON_ONCE(platform >= ARRAY_SIZE(platform_names) || ++ platform_names[platform] == NULL)) ++ return ""; ++ ++ return platform_names[platform]; ++} ++ ++void intel_device_info_dump_flags(const struct intel_device_info *info, ++ struct drm_printer *p) ++{ ++#define PRINT_FLAG(name) drm_printf(p, "%s: %s\n", #name, yesno(info->name)); ++ DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG); ++#undef PRINT_FLAG ++ ++#define PRINT_FLAG(name) drm_printf(p, "%s: %s\n", #name, yesno(info->display.name)); ++ DEV_INFO_DISPLAY_FOR_EACH_FLAG(PRINT_FLAG); ++#undef PRINT_FLAG ++} ++ ++static void sseu_dump(const struct sseu_dev_info *sseu, struct drm_printer *p) ++{ ++ int s; ++ ++ drm_printf(p, "slice total: %u, mask=%04x\n", ++ hweight8(sseu->slice_mask), sseu->slice_mask); ++ drm_printf(p, "subslice total: %u\n", sseu_subslice_total(sseu)); ++ for (s = 0; s < sseu->max_slices; s++) { ++ drm_printf(p, "slice%d: %u subslices, mask=%04x\n", ++ s, hweight8(sseu->subslice_mask[s]), ++ sseu->subslice_mask[s]); ++ } ++ drm_printf(p, "EU total: %u\n", sseu->eu_total); ++ drm_printf(p, "EU per subslice: %u\n", sseu->eu_per_subslice); ++ drm_printf(p, "has slice power gating: %s\n", ++ yesno(sseu->has_slice_pg)); ++ drm_printf(p, "has subslice power gating: %s\n", ++ yesno(sseu->has_subslice_pg)); ++ drm_printf(p, "has EU power gating: %s\n", yesno(sseu->has_eu_pg)); ++} ++ ++void intel_device_info_dump_runtime(const struct intel_runtime_info *info, ++ struct drm_printer *p) ++{ ++ sseu_dump(&info->sseu, p); ++ ++ drm_printf(p, "CS timestamp frequency: %u kHz\n", ++ info->cs_timestamp_frequency_khz); ++} ++ ++void intel_device_info_dump_topology(const struct sseu_dev_info *sseu, ++ struct drm_printer *p) ++{ ++ int s, ss; ++ ++ if (sseu->max_slices == 0) { ++ drm_printf(p, "Unavailable\n"); ++ return; ++ } ++ ++ for (s = 0; s < sseu->max_slices; s++) { ++ drm_printf(p, "slice%d: %u subslice(s) (0x%hhx):\n", ++ s, hweight8(sseu->subslice_mask[s]), ++ sseu->subslice_mask[s]); ++ ++ for (ss = 0; ss < sseu->max_subslices; ss++) { ++ u16 enabled_eus = sseu_get_eus(sseu, s, ss); ++ ++ drm_printf(p, "\tsubslice%d: %u EUs (0x%hx)\n", ++ ss, hweight16(enabled_eus), enabled_eus); ++ } ++ } ++} ++ ++static u16 compute_eu_total(const struct sseu_dev_info *sseu) ++{ ++ u16 i, total = 0; ++ ++ for (i = 0; i < ARRAY_SIZE(sseu->eu_mask); i++) ++ total += hweight8(sseu->eu_mask[i]); ++ ++ return total; ++} ++ ++static void gen11_sseu_info_init(struct drm_i915_private *dev_priv) ++{ ++ struct sseu_dev_info *sseu = &RUNTIME_INFO(dev_priv)->sseu; ++ u8 s_en; ++ u32 ss_en, ss_en_mask; ++ u8 eu_en; ++ int s; ++ ++ if (IS_ELKHARTLAKE(dev_priv)) { ++ sseu->max_slices = 1; ++ sseu->max_subslices = 4; ++ sseu->max_eus_per_subslice = 8; ++ } else { ++ sseu->max_slices = 1; ++ sseu->max_subslices = 8; ++ sseu->max_eus_per_subslice = 8; ++ } ++ ++ s_en = I915_READ(GEN11_GT_SLICE_ENABLE) & GEN11_GT_S_ENA_MASK; ++ ss_en = ~I915_READ(GEN11_GT_SUBSLICE_DISABLE); ++ ss_en_mask = BIT(sseu->max_subslices) - 1; ++ eu_en = ~(I915_READ(GEN11_EU_DISABLE) & GEN11_EU_DIS_MASK); ++ ++ for (s = 0; s < sseu->max_slices; s++) { ++ if (s_en & BIT(s)) { ++ int ss_idx = sseu->max_subslices * s; ++ int ss; ++ ++ sseu->slice_mask |= BIT(s); ++ sseu->subslice_mask[s] = (ss_en >> ss_idx) & ss_en_mask; ++ for (ss = 0; ss < sseu->max_subslices; ss++) { ++ if (sseu->subslice_mask[s] & BIT(ss)) ++ sseu_set_eus(sseu, s, ss, eu_en); ++ } ++ } ++ } ++ sseu->eu_per_subslice = hweight8(eu_en); ++ sseu->eu_total = compute_eu_total(sseu); ++ ++ /* ICL has no power gating restrictions. */ ++ sseu->has_slice_pg = 1; ++ sseu->has_subslice_pg = 1; ++ sseu->has_eu_pg = 1; ++} ++ ++static void gen10_sseu_info_init(struct drm_i915_private *dev_priv) ++{ ++ struct sseu_dev_info *sseu = &RUNTIME_INFO(dev_priv)->sseu; ++ const u32 fuse2 = I915_READ(GEN8_FUSE2); ++ int s, ss; ++ const int eu_mask = 0xff; ++ u32 subslice_mask, eu_en; ++ ++ sseu->slice_mask = (fuse2 & GEN10_F2_S_ENA_MASK) >> ++ GEN10_F2_S_ENA_SHIFT; ++ sseu->max_slices = 6; ++ sseu->max_subslices = 4; ++ sseu->max_eus_per_subslice = 8; ++ ++ subslice_mask = (1 << 4) - 1; ++ subslice_mask &= ~((fuse2 & GEN10_F2_SS_DIS_MASK) >> ++ GEN10_F2_SS_DIS_SHIFT); ++ ++ /* ++ * Slice0 can have up to 3 subslices, but there are only 2 in ++ * slice1/2. ++ */ ++ sseu->subslice_mask[0] = subslice_mask; ++ for (s = 1; s < sseu->max_slices; s++) ++ sseu->subslice_mask[s] = subslice_mask & 0x3; ++ ++ /* Slice0 */ ++ eu_en = ~I915_READ(GEN8_EU_DISABLE0); ++ for (ss = 0; ss < sseu->max_subslices; ss++) ++ sseu_set_eus(sseu, 0, ss, (eu_en >> (8 * ss)) & eu_mask); ++ /* Slice1 */ ++ sseu_set_eus(sseu, 1, 0, (eu_en >> 24) & eu_mask); ++ eu_en = ~I915_READ(GEN8_EU_DISABLE1); ++ sseu_set_eus(sseu, 1, 1, eu_en & eu_mask); ++ /* Slice2 */ ++ sseu_set_eus(sseu, 2, 0, (eu_en >> 8) & eu_mask); ++ sseu_set_eus(sseu, 2, 1, (eu_en >> 16) & eu_mask); ++ /* Slice3 */ ++ sseu_set_eus(sseu, 3, 0, (eu_en >> 24) & eu_mask); ++ eu_en = ~I915_READ(GEN8_EU_DISABLE2); ++ sseu_set_eus(sseu, 3, 1, eu_en & eu_mask); ++ /* Slice4 */ ++ sseu_set_eus(sseu, 4, 0, (eu_en >> 8) & eu_mask); ++ sseu_set_eus(sseu, 4, 1, (eu_en >> 16) & eu_mask); ++ /* Slice5 */ ++ sseu_set_eus(sseu, 5, 0, (eu_en >> 24) & eu_mask); ++ eu_en = ~I915_READ(GEN10_EU_DISABLE3); ++ sseu_set_eus(sseu, 5, 1, eu_en & eu_mask); ++ ++ /* Do a second pass where we mark the subslices disabled if all their ++ * eus are off. ++ */ ++ for (s = 0; s < sseu->max_slices; s++) { ++ for (ss = 0; ss < sseu->max_subslices; ss++) { ++ if (sseu_get_eus(sseu, s, ss) == 0) ++ sseu->subslice_mask[s] &= ~BIT(ss); ++ } ++ } ++ ++ sseu->eu_total = compute_eu_total(sseu); ++ ++ /* ++ * CNL is expected to always have a uniform distribution ++ * of EU across subslices with the exception that any one ++ * EU in any one subslice may be fused off for die ++ * recovery. ++ */ ++ sseu->eu_per_subslice = sseu_subslice_total(sseu) ? ++ DIV_ROUND_UP(sseu->eu_total, ++ sseu_subslice_total(sseu)) : 0; ++ ++ /* No restrictions on Power Gating */ ++ sseu->has_slice_pg = 1; ++ sseu->has_subslice_pg = 1; ++ sseu->has_eu_pg = 1; ++} ++ ++static void cherryview_sseu_info_init(struct drm_i915_private *dev_priv) ++{ ++ struct sseu_dev_info *sseu = &RUNTIME_INFO(dev_priv)->sseu; ++ u32 fuse; ++ ++ fuse = I915_READ(CHV_FUSE_GT); ++ ++ sseu->slice_mask = BIT(0); ++ sseu->max_slices = 1; ++ sseu->max_subslices = 2; ++ sseu->max_eus_per_subslice = 8; ++ ++ if (!(fuse & CHV_FGT_DISABLE_SS0)) { ++ u8 disabled_mask = ++ ((fuse & CHV_FGT_EU_DIS_SS0_R0_MASK) >> ++ CHV_FGT_EU_DIS_SS0_R0_SHIFT) | ++ (((fuse & CHV_FGT_EU_DIS_SS0_R1_MASK) >> ++ CHV_FGT_EU_DIS_SS0_R1_SHIFT) << 4); ++ ++ sseu->subslice_mask[0] |= BIT(0); ++ sseu_set_eus(sseu, 0, 0, ~disabled_mask); ++ } ++ ++ if (!(fuse & CHV_FGT_DISABLE_SS1)) { ++ u8 disabled_mask = ++ ((fuse & CHV_FGT_EU_DIS_SS1_R0_MASK) >> ++ CHV_FGT_EU_DIS_SS1_R0_SHIFT) | ++ (((fuse & CHV_FGT_EU_DIS_SS1_R1_MASK) >> ++ CHV_FGT_EU_DIS_SS1_R1_SHIFT) << 4); ++ ++ sseu->subslice_mask[0] |= BIT(1); ++ sseu_set_eus(sseu, 0, 1, ~disabled_mask); ++ } ++ ++ sseu->eu_total = compute_eu_total(sseu); ++ ++ /* ++ * CHV expected to always have a uniform distribution of EU ++ * across subslices. ++ */ ++ sseu->eu_per_subslice = sseu_subslice_total(sseu) ? ++ sseu->eu_total / sseu_subslice_total(sseu) : ++ 0; ++ /* ++ * CHV supports subslice power gating on devices with more than ++ * one subslice, and supports EU power gating on devices with ++ * more than one EU pair per subslice. ++ */ ++ sseu->has_slice_pg = 0; ++ sseu->has_subslice_pg = sseu_subslice_total(sseu) > 1; ++ sseu->has_eu_pg = (sseu->eu_per_subslice > 2); ++} ++ ++static void gen9_sseu_info_init(struct drm_i915_private *dev_priv) ++{ ++ struct intel_device_info *info = mkwrite_device_info(dev_priv); ++ struct sseu_dev_info *sseu = &RUNTIME_INFO(dev_priv)->sseu; ++ int s, ss; ++ u32 fuse2, eu_disable, subslice_mask; ++ const u8 eu_mask = 0xff; ++ ++ fuse2 = I915_READ(GEN8_FUSE2); ++ sseu->slice_mask = (fuse2 & GEN8_F2_S_ENA_MASK) >> GEN8_F2_S_ENA_SHIFT; ++ ++ /* BXT has a single slice and at most 3 subslices. */ ++ sseu->max_slices = IS_GEN9_LP(dev_priv) ? 1 : 3; ++ sseu->max_subslices = IS_GEN9_LP(dev_priv) ? 3 : 4; ++ sseu->max_eus_per_subslice = 8; ++ ++ /* ++ * The subslice disable field is global, i.e. it applies ++ * to each of the enabled slices. ++ */ ++ subslice_mask = (1 << sseu->max_subslices) - 1; ++ subslice_mask &= ~((fuse2 & GEN9_F2_SS_DIS_MASK) >> ++ GEN9_F2_SS_DIS_SHIFT); ++ ++ /* ++ * Iterate through enabled slices and subslices to ++ * count the total enabled EU. ++ */ ++ for (s = 0; s < sseu->max_slices; s++) { ++ if (!(sseu->slice_mask & BIT(s))) ++ /* skip disabled slice */ ++ continue; ++ ++ sseu->subslice_mask[s] = subslice_mask; ++ ++ eu_disable = I915_READ(GEN9_EU_DISABLE(s)); ++ for (ss = 0; ss < sseu->max_subslices; ss++) { ++ int eu_per_ss; ++ u8 eu_disabled_mask; ++ ++ if (!(sseu->subslice_mask[s] & BIT(ss))) ++ /* skip disabled subslice */ ++ continue; ++ ++ eu_disabled_mask = (eu_disable >> (ss * 8)) & eu_mask; ++ ++ sseu_set_eus(sseu, s, ss, ~eu_disabled_mask); ++ ++ eu_per_ss = sseu->max_eus_per_subslice - ++ hweight8(eu_disabled_mask); ++ ++ /* ++ * Record which subslice(s) has(have) 7 EUs. we ++ * can tune the hash used to spread work among ++ * subslices if they are unbalanced. ++ */ ++ if (eu_per_ss == 7) ++ sseu->subslice_7eu[s] |= BIT(ss); ++ } ++ } ++ ++ sseu->eu_total = compute_eu_total(sseu); ++ ++ /* ++ * SKL is expected to always have a uniform distribution ++ * of EU across subslices with the exception that any one ++ * EU in any one subslice may be fused off for die ++ * recovery. BXT is expected to be perfectly uniform in EU ++ * distribution. ++ */ ++ sseu->eu_per_subslice = sseu_subslice_total(sseu) ? ++ DIV_ROUND_UP(sseu->eu_total, ++ sseu_subslice_total(sseu)) : 0; ++ /* ++ * SKL+ supports slice power gating on devices with more than ++ * one slice, and supports EU power gating on devices with ++ * more than one EU pair per subslice. BXT+ supports subslice ++ * power gating on devices with more than one subslice, and ++ * supports EU power gating on devices with more than one EU ++ * pair per subslice. ++ */ ++ sseu->has_slice_pg = ++ !IS_GEN9_LP(dev_priv) && hweight8(sseu->slice_mask) > 1; ++ sseu->has_subslice_pg = ++ IS_GEN9_LP(dev_priv) && sseu_subslice_total(sseu) > 1; ++ sseu->has_eu_pg = sseu->eu_per_subslice > 2; ++ ++ if (IS_GEN9_LP(dev_priv)) { ++#define IS_SS_DISABLED(ss) (!(sseu->subslice_mask[0] & BIT(ss))) ++ info->has_pooled_eu = hweight8(sseu->subslice_mask[0]) == 3; ++ ++ sseu->min_eu_in_pool = 0; ++ if (info->has_pooled_eu) { ++ if (IS_SS_DISABLED(2) || IS_SS_DISABLED(0)) ++ sseu->min_eu_in_pool = 3; ++ else if (IS_SS_DISABLED(1)) ++ sseu->min_eu_in_pool = 6; ++ else ++ sseu->min_eu_in_pool = 9; ++ } ++#undef IS_SS_DISABLED ++ } ++} ++ ++static void broadwell_sseu_info_init(struct drm_i915_private *dev_priv) ++{ ++ struct sseu_dev_info *sseu = &RUNTIME_INFO(dev_priv)->sseu; ++ int s, ss; ++ u32 fuse2, subslice_mask, eu_disable[3]; /* s_max */ ++ ++ fuse2 = I915_READ(GEN8_FUSE2); ++ sseu->slice_mask = (fuse2 & GEN8_F2_S_ENA_MASK) >> GEN8_F2_S_ENA_SHIFT; ++ sseu->max_slices = 3; ++ sseu->max_subslices = 3; ++ sseu->max_eus_per_subslice = 8; ++ ++ /* ++ * The subslice disable field is global, i.e. it applies ++ * to each of the enabled slices. ++ */ ++ subslice_mask = GENMASK(sseu->max_subslices - 1, 0); ++ subslice_mask &= ~((fuse2 & GEN8_F2_SS_DIS_MASK) >> ++ GEN8_F2_SS_DIS_SHIFT); ++ ++ eu_disable[0] = I915_READ(GEN8_EU_DISABLE0) & GEN8_EU_DIS0_S0_MASK; ++ eu_disable[1] = (I915_READ(GEN8_EU_DISABLE0) >> GEN8_EU_DIS0_S1_SHIFT) | ++ ((I915_READ(GEN8_EU_DISABLE1) & GEN8_EU_DIS1_S1_MASK) << ++ (32 - GEN8_EU_DIS0_S1_SHIFT)); ++ eu_disable[2] = (I915_READ(GEN8_EU_DISABLE1) >> GEN8_EU_DIS1_S2_SHIFT) | ++ ((I915_READ(GEN8_EU_DISABLE2) & GEN8_EU_DIS2_S2_MASK) << ++ (32 - GEN8_EU_DIS1_S2_SHIFT)); ++ ++ /* ++ * Iterate through enabled slices and subslices to ++ * count the total enabled EU. ++ */ ++ for (s = 0; s < sseu->max_slices; s++) { ++ if (!(sseu->slice_mask & BIT(s))) ++ /* skip disabled slice */ ++ continue; ++ ++ sseu->subslice_mask[s] = subslice_mask; ++ ++ for (ss = 0; ss < sseu->max_subslices; ss++) { ++ u8 eu_disabled_mask; ++ u32 n_disabled; ++ ++ if (!(sseu->subslice_mask[s] & BIT(ss))) ++ /* skip disabled subslice */ ++ continue; ++ ++ eu_disabled_mask = ++ eu_disable[s] >> (ss * sseu->max_eus_per_subslice); ++ ++ sseu_set_eus(sseu, s, ss, ~eu_disabled_mask); ++ ++ n_disabled = hweight8(eu_disabled_mask); ++ ++ /* ++ * Record which subslices have 7 EUs. ++ */ ++ if (sseu->max_eus_per_subslice - n_disabled == 7) ++ sseu->subslice_7eu[s] |= 1 << ss; ++ } ++ } ++ ++ sseu->eu_total = compute_eu_total(sseu); ++ ++ /* ++ * BDW is expected to always have a uniform distribution of EU across ++ * subslices with the exception that any one EU in any one subslice may ++ * be fused off for die recovery. ++ */ ++ sseu->eu_per_subslice = sseu_subslice_total(sseu) ? ++ DIV_ROUND_UP(sseu->eu_total, ++ sseu_subslice_total(sseu)) : 0; ++ ++ /* ++ * BDW supports slice power gating on devices with more than ++ * one slice. ++ */ ++ sseu->has_slice_pg = hweight8(sseu->slice_mask) > 1; ++ sseu->has_subslice_pg = 0; ++ sseu->has_eu_pg = 0; ++} ++ ++static void haswell_sseu_info_init(struct drm_i915_private *dev_priv) ++{ ++ struct sseu_dev_info *sseu = &RUNTIME_INFO(dev_priv)->sseu; ++ u32 fuse1; ++ int s, ss; ++ ++ /* ++ * There isn't a register to tell us how many slices/subslices. We ++ * work off the PCI-ids here. ++ */ ++ switch (INTEL_INFO(dev_priv)->gt) { ++ default: ++ MISSING_CASE(INTEL_INFO(dev_priv)->gt); ++ /* fall through */ ++ case 1: ++ sseu->slice_mask = BIT(0); ++ sseu->subslice_mask[0] = BIT(0); ++ break; ++ case 2: ++ sseu->slice_mask = BIT(0); ++ sseu->subslice_mask[0] = BIT(0) | BIT(1); ++ break; ++ case 3: ++ sseu->slice_mask = BIT(0) | BIT(1); ++ sseu->subslice_mask[0] = BIT(0) | BIT(1); ++ sseu->subslice_mask[1] = BIT(0) | BIT(1); ++ break; ++ } ++ ++ sseu->max_slices = hweight8(sseu->slice_mask); ++ sseu->max_subslices = hweight8(sseu->subslice_mask[0]); ++ ++ fuse1 = I915_READ(HSW_PAVP_FUSE1); ++ switch ((fuse1 & HSW_F1_EU_DIS_MASK) >> HSW_F1_EU_DIS_SHIFT) { ++ default: ++ MISSING_CASE((fuse1 & HSW_F1_EU_DIS_MASK) >> ++ HSW_F1_EU_DIS_SHIFT); ++ /* fall through */ ++ case HSW_F1_EU_DIS_10EUS: ++ sseu->eu_per_subslice = 10; ++ break; ++ case HSW_F1_EU_DIS_8EUS: ++ sseu->eu_per_subslice = 8; ++ break; ++ case HSW_F1_EU_DIS_6EUS: ++ sseu->eu_per_subslice = 6; ++ break; ++ } ++ sseu->max_eus_per_subslice = sseu->eu_per_subslice; ++ ++ for (s = 0; s < sseu->max_slices; s++) { ++ for (ss = 0; ss < sseu->max_subslices; ss++) { ++ sseu_set_eus(sseu, s, ss, ++ (1UL << sseu->eu_per_subslice) - 1); ++ } ++ } ++ ++ sseu->eu_total = compute_eu_total(sseu); ++ ++ /* No powergating for you. */ ++ sseu->has_slice_pg = 0; ++ sseu->has_subslice_pg = 0; ++ sseu->has_eu_pg = 0; ++} ++ ++static u32 read_reference_ts_freq(struct drm_i915_private *dev_priv) ++{ ++ u32 ts_override = I915_READ(GEN9_TIMESTAMP_OVERRIDE); ++ u32 base_freq, frac_freq; ++ ++ base_freq = ((ts_override & GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DIVIDER_MASK) >> ++ GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DIVIDER_SHIFT) + 1; ++ base_freq *= 1000; ++ ++ frac_freq = ((ts_override & ++ GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DENOMINATOR_MASK) >> ++ GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DENOMINATOR_SHIFT); ++ frac_freq = 1000 / (frac_freq + 1); ++ ++ return base_freq + frac_freq; ++} ++ ++static u32 gen10_get_crystal_clock_freq(struct drm_i915_private *dev_priv, ++ u32 rpm_config_reg) ++{ ++ u32 f19_2_mhz = 19200; ++ u32 f24_mhz = 24000; ++ u32 crystal_clock = (rpm_config_reg & ++ GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_MASK) >> ++ GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT; ++ ++ switch (crystal_clock) { ++ case GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_19_2_MHZ: ++ return f19_2_mhz; ++ case GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_24_MHZ: ++ return f24_mhz; ++ default: ++ MISSING_CASE(crystal_clock); ++ return 0; ++ } ++} ++ ++static u32 gen11_get_crystal_clock_freq(struct drm_i915_private *dev_priv, ++ u32 rpm_config_reg) ++{ ++ u32 f19_2_mhz = 19200; ++ u32 f24_mhz = 24000; ++ u32 f25_mhz = 25000; ++ u32 f38_4_mhz = 38400; ++ u32 crystal_clock = (rpm_config_reg & ++ GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_MASK) >> ++ GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT; ++ ++ switch (crystal_clock) { ++ case GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_24_MHZ: ++ return f24_mhz; ++ case GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_19_2_MHZ: ++ return f19_2_mhz; ++ case GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_38_4_MHZ: ++ return f38_4_mhz; ++ case GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_25_MHZ: ++ return f25_mhz; ++ default: ++ MISSING_CASE(crystal_clock); ++ return 0; ++ } ++} ++ ++static u32 read_timestamp_frequency(struct drm_i915_private *dev_priv) ++{ ++ u32 f12_5_mhz = 12500; ++ u32 f19_2_mhz = 19200; ++ u32 f24_mhz = 24000; ++ ++ if (INTEL_GEN(dev_priv) <= 4) { ++ /* PRMs say: ++ * ++ * "The value in this register increments once every 16 ++ * hclks." (through the “Clocking Configuration” ++ * (“CLKCFG”) MCHBAR register) ++ */ ++ return dev_priv->rawclk_freq / 16; ++ } else if (INTEL_GEN(dev_priv) <= 8) { ++ /* PRMs say: ++ * ++ * "The PCU TSC counts 10ns increments; this timestamp ++ * reflects bits 38:3 of the TSC (i.e. 80ns granularity, ++ * rolling over every 1.5 hours). ++ */ ++ return f12_5_mhz; ++ } else if (INTEL_GEN(dev_priv) <= 9) { ++ u32 ctc_reg = I915_READ(CTC_MODE); ++ u32 freq = 0; ++ ++ if ((ctc_reg & CTC_SOURCE_PARAMETER_MASK) == CTC_SOURCE_DIVIDE_LOGIC) { ++ freq = read_reference_ts_freq(dev_priv); ++ } else { ++ freq = IS_GEN9_LP(dev_priv) ? f19_2_mhz : f24_mhz; ++ ++ /* Now figure out how the command stream's timestamp ++ * register increments from this frequency (it might ++ * increment only every few clock cycle). ++ */ ++ freq >>= 3 - ((ctc_reg & CTC_SHIFT_PARAMETER_MASK) >> ++ CTC_SHIFT_PARAMETER_SHIFT); ++ } ++ ++ return freq; ++ } else if (INTEL_GEN(dev_priv) <= 11) { ++ u32 ctc_reg = I915_READ(CTC_MODE); ++ u32 freq = 0; ++ ++ /* First figure out the reference frequency. There are 2 ways ++ * we can compute the frequency, either through the ++ * TIMESTAMP_OVERRIDE register or through RPM_CONFIG. CTC_MODE ++ * tells us which one we should use. ++ */ ++ if ((ctc_reg & CTC_SOURCE_PARAMETER_MASK) == CTC_SOURCE_DIVIDE_LOGIC) { ++ freq = read_reference_ts_freq(dev_priv); ++ } else { ++ u32 rpm_config_reg = I915_READ(RPM_CONFIG0); ++ ++ if (INTEL_GEN(dev_priv) <= 10) ++ freq = gen10_get_crystal_clock_freq(dev_priv, ++ rpm_config_reg); ++ else ++ freq = gen11_get_crystal_clock_freq(dev_priv, ++ rpm_config_reg); ++ ++ /* Now figure out how the command stream's timestamp ++ * register increments from this frequency (it might ++ * increment only every few clock cycle). ++ */ ++ freq >>= 3 - ((rpm_config_reg & ++ GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK) >> ++ GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT); ++ } ++ ++ return freq; ++ } ++ ++ MISSING_CASE("Unknown gen, unable to read command streamer timestamp frequency\n"); ++ return 0; ++} ++ ++#undef INTEL_VGA_DEVICE ++#define INTEL_VGA_DEVICE(id, info) (id) ++ ++static const u16 subplatform_ult_ids[] = { ++ INTEL_HSW_ULT_GT1_IDS(0), ++ INTEL_HSW_ULT_GT2_IDS(0), ++ INTEL_HSW_ULT_GT3_IDS(0), ++ INTEL_BDW_ULT_GT1_IDS(0), ++ INTEL_BDW_ULT_GT2_IDS(0), ++ INTEL_BDW_ULT_GT3_IDS(0), ++ INTEL_BDW_ULT_RSVD_IDS(0), ++ INTEL_SKL_ULT_GT1_IDS(0), ++ INTEL_SKL_ULT_GT2_IDS(0), ++ INTEL_SKL_ULT_GT3_IDS(0), ++ INTEL_KBL_ULT_GT1_IDS(0), ++ INTEL_KBL_ULT_GT2_IDS(0), ++ INTEL_KBL_ULT_GT3_IDS(0), ++ INTEL_CFL_U_GT2_IDS(0), ++ INTEL_CFL_U_GT3_IDS(0), ++ INTEL_WHL_U_GT1_IDS(0), ++ INTEL_WHL_U_GT2_IDS(0), ++ INTEL_WHL_U_GT3_IDS(0) ++}; ++ ++static const u16 subplatform_ulx_ids[] = { ++ INTEL_HSW_ULX_GT1_IDS(0), ++ INTEL_HSW_ULX_GT2_IDS(0), ++ INTEL_BDW_ULX_GT1_IDS(0), ++ INTEL_BDW_ULX_GT2_IDS(0), ++ INTEL_BDW_ULX_GT3_IDS(0), ++ INTEL_BDW_ULX_RSVD_IDS(0), ++ INTEL_SKL_ULX_GT1_IDS(0), ++ INTEL_SKL_ULX_GT2_IDS(0), ++ INTEL_KBL_ULX_GT1_IDS(0), ++ INTEL_KBL_ULX_GT2_IDS(0) ++}; ++ ++static const u16 subplatform_aml_ids[] = { ++ INTEL_AML_KBL_GT2_IDS(0), ++ INTEL_AML_CFL_GT2_IDS(0) ++}; ++ ++static const u16 subplatform_portf_ids[] = { ++ INTEL_CNL_PORT_F_IDS(0), ++ INTEL_ICL_PORT_F_IDS(0) ++}; ++ ++static bool find_devid(u16 id, const u16 *p, unsigned int num) ++{ ++ for (; num; num--, p++) { ++ if (*p == id) ++ return true; ++ } ++ ++ return false; ++} ++ ++void intel_device_info_subplatform_init(struct drm_i915_private *i915) ++{ ++ const struct intel_device_info *info = INTEL_INFO(i915); ++ const struct intel_runtime_info *rinfo = RUNTIME_INFO(i915); ++ const unsigned int pi = __platform_mask_index(rinfo, info->platform); ++ const unsigned int pb = __platform_mask_bit(rinfo, info->platform); ++ u16 devid = INTEL_DEVID(i915); ++ u32 mask = 0; ++ ++ /* Make sure IS_ checks are working. */ ++ RUNTIME_INFO(i915)->platform_mask[pi] = BIT(pb); ++ ++ /* Find and mark subplatform bits based on the PCI device id. */ ++ if (find_devid(devid, subplatform_ult_ids, ++ ARRAY_SIZE(subplatform_ult_ids))) { ++ mask = BIT(INTEL_SUBPLATFORM_ULT); ++ } else if (find_devid(devid, subplatform_ulx_ids, ++ ARRAY_SIZE(subplatform_ulx_ids))) { ++ mask = BIT(INTEL_SUBPLATFORM_ULX); ++ if (IS_HASWELL(i915) || IS_BROADWELL(i915)) { ++ /* ULX machines are also considered ULT. */ ++ mask |= BIT(INTEL_SUBPLATFORM_ULT); ++ } ++ } else if (find_devid(devid, subplatform_aml_ids, ++ ARRAY_SIZE(subplatform_aml_ids))) { ++ mask = BIT(INTEL_SUBPLATFORM_AML); ++ } else if (find_devid(devid, subplatform_portf_ids, ++ ARRAY_SIZE(subplatform_portf_ids))) { ++ mask = BIT(INTEL_SUBPLATFORM_PORTF); ++ } ++ ++ GEM_BUG_ON(mask & ~INTEL_SUBPLATFORM_BITS); ++ ++ RUNTIME_INFO(i915)->platform_mask[pi] |= mask; ++} ++ ++/** ++ * intel_device_info_runtime_init - initialize runtime info ++ * @dev_priv: the i915 device ++ * ++ * Determine various intel_device_info fields at runtime. ++ * ++ * Use it when either: ++ * - it's judged too laborious to fill n static structures with the limit ++ * when a simple if statement does the job, ++ * - run-time checks (eg read fuse/strap registers) are needed. ++ * ++ * This function needs to be called: ++ * - after the MMIO has been setup as we are reading registers, ++ * - after the PCH has been detected, ++ * - before the first usage of the fields it can tweak. ++ */ ++void intel_device_info_runtime_init(struct drm_i915_private *dev_priv) ++{ ++ struct intel_device_info *info = mkwrite_device_info(dev_priv); ++ struct intel_runtime_info *runtime = RUNTIME_INFO(dev_priv); ++ enum pipe pipe; ++ ++ if (INTEL_GEN(dev_priv) >= 10) { ++ for_each_pipe(dev_priv, pipe) ++ runtime->num_scalers[pipe] = 2; ++ } else if (IS_GEN(dev_priv, 9)) { ++ runtime->num_scalers[PIPE_A] = 2; ++ runtime->num_scalers[PIPE_B] = 2; ++ runtime->num_scalers[PIPE_C] = 1; ++ } ++ ++ BUILD_BUG_ON(BITS_PER_TYPE(intel_engine_mask_t) < I915_NUM_ENGINES); ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ for_each_pipe(dev_priv, pipe) ++ runtime->num_sprites[pipe] = 6; ++ else if (IS_GEN(dev_priv, 10) || IS_GEMINILAKE(dev_priv)) ++ for_each_pipe(dev_priv, pipe) ++ runtime->num_sprites[pipe] = 3; ++ else if (IS_BROXTON(dev_priv)) { ++ /* ++ * Skylake and Broxton currently don't expose the topmost plane as its ++ * use is exclusive with the legacy cursor and we only want to expose ++ * one of those, not both. Until we can safely expose the topmost plane ++ * as a DRM_PLANE_TYPE_CURSOR with all the features exposed/supported, ++ * we don't expose the topmost plane at all to prevent ABI breakage ++ * down the line. ++ */ ++ ++ runtime->num_sprites[PIPE_A] = 2; ++ runtime->num_sprites[PIPE_B] = 2; ++ runtime->num_sprites[PIPE_C] = 1; ++ } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { ++ for_each_pipe(dev_priv, pipe) ++ runtime->num_sprites[pipe] = 2; ++ } else if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv)) { ++ for_each_pipe(dev_priv, pipe) ++ runtime->num_sprites[pipe] = 1; ++ } ++ ++ if (i915_modparams.disable_display) { ++ DRM_INFO("Display disabled (module parameter)\n"); ++ info->num_pipes = 0; ++ } else if (HAS_DISPLAY(dev_priv) && ++ (IS_GEN_RANGE(dev_priv, 7, 8)) && ++ HAS_PCH_SPLIT(dev_priv)) { ++ u32 fuse_strap = I915_READ(FUSE_STRAP); ++ u32 sfuse_strap = I915_READ(SFUSE_STRAP); ++ ++ /* ++ * SFUSE_STRAP is supposed to have a bit signalling the display ++ * is fused off. Unfortunately it seems that, at least in ++ * certain cases, fused off display means that PCH display ++ * reads don't land anywhere. In that case, we read 0s. ++ * ++ * On CPT/PPT, we can detect this case as SFUSE_STRAP_FUSE_LOCK ++ * should be set when taking over after the firmware. ++ */ ++ if (fuse_strap & ILK_INTERNAL_DISPLAY_DISABLE || ++ sfuse_strap & SFUSE_STRAP_DISPLAY_DISABLED || ++ (HAS_PCH_CPT(dev_priv) && ++ !(sfuse_strap & SFUSE_STRAP_FUSE_LOCK))) { ++ DRM_INFO("Display fused off, disabling\n"); ++ info->num_pipes = 0; ++ } else if (fuse_strap & IVB_PIPE_C_DISABLE) { ++ DRM_INFO("PipeC fused off\n"); ++ info->num_pipes -= 1; ++ } ++ } else if (HAS_DISPLAY(dev_priv) && INTEL_GEN(dev_priv) >= 9) { ++ u32 dfsm = I915_READ(SKL_DFSM); ++ u8 disabled_mask = 0; ++ bool invalid; ++ int num_bits; ++ ++ if (dfsm & SKL_DFSM_PIPE_A_DISABLE) ++ disabled_mask |= BIT(PIPE_A); ++ if (dfsm & SKL_DFSM_PIPE_B_DISABLE) ++ disabled_mask |= BIT(PIPE_B); ++ if (dfsm & SKL_DFSM_PIPE_C_DISABLE) ++ disabled_mask |= BIT(PIPE_C); ++ ++ num_bits = hweight8(disabled_mask); ++ ++ switch (disabled_mask) { ++ case BIT(PIPE_A): ++ case BIT(PIPE_B): ++ case BIT(PIPE_A) | BIT(PIPE_B): ++ case BIT(PIPE_A) | BIT(PIPE_C): ++ invalid = true; ++ break; ++ default: ++ invalid = false; ++ } ++ ++ if (num_bits > info->num_pipes || invalid) ++ DRM_ERROR("invalid pipe fuse configuration: 0x%x\n", ++ disabled_mask); ++ else ++ info->num_pipes -= num_bits; ++ } ++ ++ /* Initialize slice/subslice/EU info */ ++ if (IS_HASWELL(dev_priv)) ++ haswell_sseu_info_init(dev_priv); ++ else if (IS_CHERRYVIEW(dev_priv)) ++ cherryview_sseu_info_init(dev_priv); ++ else if (IS_BROADWELL(dev_priv)) ++ broadwell_sseu_info_init(dev_priv); ++ else if (IS_GEN(dev_priv, 9)) ++ gen9_sseu_info_init(dev_priv); ++ else if (IS_GEN(dev_priv, 10)) ++ gen10_sseu_info_init(dev_priv); ++ else if (INTEL_GEN(dev_priv) >= 11) ++ gen11_sseu_info_init(dev_priv); ++ ++ if (IS_GEN(dev_priv, 6) && intel_vtd_active()) { ++ DRM_INFO("Disabling ppGTT for VT-d support\n"); ++ info->ppgtt_type = INTEL_PPGTT_NONE; ++ } ++ ++ /* Initialize command stream timestamp frequency */ ++ runtime->cs_timestamp_frequency_khz = read_timestamp_frequency(dev_priv); ++} ++ ++void intel_driver_caps_print(const struct intel_driver_caps *caps, ++ struct drm_printer *p) ++{ ++ drm_printf(p, "Has logical contexts? %s\n", ++ yesno(caps->has_logical_contexts)); ++ drm_printf(p, "scheduler: %x\n", caps->scheduler); ++} ++ ++/* ++ * Determine which engines are fused off in our particular hardware. Since the ++ * fuse register is in the blitter powerwell, we need forcewake to be ready at ++ * this point (but later we need to prune the forcewake domains for engines that ++ * are indeed fused off). ++ */ ++void intel_device_info_init_mmio(struct drm_i915_private *dev_priv) ++{ ++ struct intel_device_info *info = mkwrite_device_info(dev_priv); ++ unsigned int logical_vdbox = 0; ++ unsigned int i; ++ u32 media_fuse; ++ u16 vdbox_mask; ++ u16 vebox_mask; ++ ++ if (INTEL_GEN(dev_priv) < 11) ++ return; ++ ++ media_fuse = ~I915_READ(GEN11_GT_VEBOX_VDBOX_DISABLE); ++ ++ vdbox_mask = media_fuse & GEN11_GT_VDBOX_DISABLE_MASK; ++ vebox_mask = (media_fuse & GEN11_GT_VEBOX_DISABLE_MASK) >> ++ GEN11_GT_VEBOX_DISABLE_SHIFT; ++ ++ for (i = 0; i < I915_MAX_VCS; i++) { ++ if (!HAS_ENGINE(dev_priv, _VCS(i))) ++ continue; ++ ++ if (!(BIT(i) & vdbox_mask)) { ++ info->engine_mask &= ~BIT(_VCS(i)); ++ DRM_DEBUG_DRIVER("vcs%u fused off\n", i); ++ continue; ++ } ++ ++ /* ++ * In Gen11, only even numbered logical VDBOXes are ++ * hooked up to an SFC (Scaler & Format Converter) unit. ++ */ ++ if (logical_vdbox++ % 2 == 0) ++ RUNTIME_INFO(dev_priv)->vdbox_sfc_access |= BIT(i); ++ } ++ DRM_DEBUG_DRIVER("vdbox enable: %04x, instances: %04lx\n", ++ vdbox_mask, VDBOX_MASK(dev_priv)); ++ GEM_BUG_ON(vdbox_mask != VDBOX_MASK(dev_priv)); ++ ++ for (i = 0; i < I915_MAX_VECS; i++) { ++ if (!HAS_ENGINE(dev_priv, _VECS(i))) ++ continue; ++ ++ if (!(BIT(i) & vebox_mask)) { ++ info->engine_mask &= ~BIT(_VECS(i)); ++ DRM_DEBUG_DRIVER("vecs%u fused off\n", i); ++ } ++ } ++ DRM_DEBUG_DRIVER("vebox enable: %04x, instances: %04lx\n", ++ vebox_mask, VEBOX_MASK(dev_priv)); ++ GEM_BUG_ON(vebox_mask != VEBOX_MASK(dev_priv)); ++} +diff --git a/drivers/gpu/drm/i915_legacy/intel_device_info.h b/drivers/gpu/drm/i915_legacy/intel_device_info.h +new file mode 100644 +index 000000000000..0e579f158016 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_device_info.h +@@ -0,0 +1,307 @@ ++/* ++ * Copyright © 2014-2017 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#ifndef _INTEL_DEVICE_INFO_H_ ++#define _INTEL_DEVICE_INFO_H_ ++ ++#include ++ ++#include "intel_engine_types.h" ++#include "intel_display.h" ++ ++struct drm_printer; ++struct drm_i915_private; ++ ++/* Keep in gen based order, and chronological order within a gen */ ++enum intel_platform { ++ INTEL_PLATFORM_UNINITIALIZED = 0, ++ /* gen2 */ ++ INTEL_I830, ++ INTEL_I845G, ++ INTEL_I85X, ++ INTEL_I865G, ++ /* gen3 */ ++ INTEL_I915G, ++ INTEL_I915GM, ++ INTEL_I945G, ++ INTEL_I945GM, ++ INTEL_G33, ++ INTEL_PINEVIEW, ++ /* gen4 */ ++ INTEL_I965G, ++ INTEL_I965GM, ++ INTEL_G45, ++ INTEL_GM45, ++ /* gen5 */ ++ INTEL_IRONLAKE, ++ /* gen6 */ ++ INTEL_SANDYBRIDGE, ++ /* gen7 */ ++ INTEL_IVYBRIDGE, ++ INTEL_VALLEYVIEW, ++ INTEL_HASWELL, ++ /* gen8 */ ++ INTEL_BROADWELL, ++ INTEL_CHERRYVIEW, ++ /* gen9 */ ++ INTEL_SKYLAKE, ++ INTEL_BROXTON, ++ INTEL_KABYLAKE, ++ INTEL_GEMINILAKE, ++ INTEL_COFFEELAKE, ++ /* gen10 */ ++ INTEL_CANNONLAKE, ++ /* gen11 */ ++ INTEL_ICELAKE, ++ INTEL_ELKHARTLAKE, ++ INTEL_MAX_PLATFORMS ++}; ++ ++/* ++ * Subplatform bits share the same namespace per parent platform. In other words ++ * it is fine for the same bit to be used on multiple parent platforms. ++ */ ++ ++#define INTEL_SUBPLATFORM_BITS (3) ++ ++/* HSW/BDW/SKL/KBL/CFL */ ++#define INTEL_SUBPLATFORM_ULT (0) ++#define INTEL_SUBPLATFORM_ULX (1) ++#define INTEL_SUBPLATFORM_AML (2) ++ ++/* CNL/ICL */ ++#define INTEL_SUBPLATFORM_PORTF (0) ++ ++enum intel_ppgtt_type { ++ INTEL_PPGTT_NONE = I915_GEM_PPGTT_NONE, ++ INTEL_PPGTT_ALIASING = I915_GEM_PPGTT_ALIASING, ++ INTEL_PPGTT_FULL = I915_GEM_PPGTT_FULL, ++}; ++ ++#define DEV_INFO_FOR_EACH_FLAG(func) \ ++ func(is_mobile); \ ++ func(is_lp); \ ++ func(is_alpha_support); \ ++ /* Keep has_* in alphabetical order */ \ ++ func(has_64bit_reloc); \ ++ func(gpu_reset_clobbers_display); \ ++ func(has_reset_engine); \ ++ func(has_fpga_dbg); \ ++ func(has_guc); \ ++ func(has_guc_ct); \ ++ func(has_l3_dpf); \ ++ func(has_llc); \ ++ func(has_logical_ring_contexts); \ ++ func(has_logical_ring_elsq); \ ++ func(has_logical_ring_preemption); \ ++ func(has_pooled_eu); \ ++ func(has_rc6); \ ++ func(has_rc6p); \ ++ func(has_runtime_pm); \ ++ func(has_snoop); \ ++ func(has_coherent_ggtt); \ ++ func(unfenced_needs_alignment); \ ++ func(hws_needs_physical); ++ ++#define DEV_INFO_DISPLAY_FOR_EACH_FLAG(func) \ ++ /* Keep in alphabetical order */ \ ++ func(cursor_needs_physical); \ ++ func(has_csr); \ ++ func(has_ddi); \ ++ func(has_dp_mst); \ ++ func(has_fbc); \ ++ func(has_gmch); \ ++ func(has_hotplug); \ ++ func(has_ipc); \ ++ func(has_overlay); \ ++ func(has_psr); \ ++ func(overlay_needs_physical); \ ++ func(supports_tv); ++ ++#define GEN_MAX_SLICES (6) /* CNL upper bound */ ++#define GEN_MAX_SUBSLICES (8) /* ICL upper bound */ ++ ++struct sseu_dev_info { ++ u8 slice_mask; ++ u8 subslice_mask[GEN_MAX_SLICES]; ++ u16 eu_total; ++ u8 eu_per_subslice; ++ u8 min_eu_in_pool; ++ /* For each slice, which subslice(s) has(have) 7 EUs (bitfield)? */ ++ u8 subslice_7eu[3]; ++ u8 has_slice_pg:1; ++ u8 has_subslice_pg:1; ++ u8 has_eu_pg:1; ++ ++ /* Topology fields */ ++ u8 max_slices; ++ u8 max_subslices; ++ u8 max_eus_per_subslice; ++ ++ /* We don't have more than 8 eus per subslice at the moment and as we ++ * store eus enabled using bits, no need to multiply by eus per ++ * subslice. ++ */ ++ u8 eu_mask[GEN_MAX_SLICES * GEN_MAX_SUBSLICES]; ++}; ++ ++struct intel_device_info { ++ u16 gen_mask; ++ ++ u8 gen; ++ u8 gt; /* GT number, 0 if undefined */ ++ intel_engine_mask_t engine_mask; /* Engines supported by the HW */ ++ ++ enum intel_platform platform; ++ ++ enum intel_ppgtt_type ppgtt_type; ++ unsigned int ppgtt_size; /* log2, e.g. 31/32/48 bits */ ++ ++ unsigned int page_sizes; /* page sizes supported by the HW */ ++ ++ u32 display_mmio_offset; ++ ++ u8 num_pipes; ++ ++#define DEFINE_FLAG(name) u8 name:1 ++ DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG); ++#undef DEFINE_FLAG ++ ++ struct { ++#define DEFINE_FLAG(name) u8 name:1 ++ DEV_INFO_DISPLAY_FOR_EACH_FLAG(DEFINE_FLAG); ++#undef DEFINE_FLAG ++ } display; ++ ++ u16 ddb_size; /* in blocks */ ++ ++ /* Register offsets for the various display pipes and transcoders */ ++ int pipe_offsets[I915_MAX_TRANSCODERS]; ++ int trans_offsets[I915_MAX_TRANSCODERS]; ++ int cursor_offsets[I915_MAX_PIPES]; ++ ++ struct color_luts { ++ u16 degamma_lut_size; ++ u16 gamma_lut_size; ++ u32 degamma_lut_tests; ++ u32 gamma_lut_tests; ++ } color; ++}; ++ ++struct intel_runtime_info { ++ /* ++ * Platform mask is used for optimizing or-ed IS_PLATFORM calls into ++ * into single runtime conditionals, and also to provide groundwork ++ * for future per platform, or per SKU build optimizations. ++ * ++ * Array can be extended when necessary if the corresponding ++ * BUILD_BUG_ON is hit. ++ */ ++ u32 platform_mask[2]; ++ ++ u16 device_id; ++ ++ u8 num_sprites[I915_MAX_PIPES]; ++ u8 num_scalers[I915_MAX_PIPES]; ++ ++ u8 num_engines; ++ ++ /* Slice/subslice/EU info */ ++ struct sseu_dev_info sseu; ++ ++ u32 cs_timestamp_frequency_khz; ++ ++ /* Media engine access to SFC per instance */ ++ u8 vdbox_sfc_access; ++}; ++ ++struct intel_driver_caps { ++ unsigned int scheduler; ++ bool has_logical_contexts:1; ++}; ++ ++static inline unsigned int sseu_subslice_total(const struct sseu_dev_info *sseu) ++{ ++ unsigned int i, total = 0; ++ ++ for (i = 0; i < ARRAY_SIZE(sseu->subslice_mask); i++) ++ total += hweight8(sseu->subslice_mask[i]); ++ ++ return total; ++} ++ ++static inline int sseu_eu_idx(const struct sseu_dev_info *sseu, ++ int slice, int subslice) ++{ ++ int subslice_stride = DIV_ROUND_UP(sseu->max_eus_per_subslice, ++ BITS_PER_BYTE); ++ int slice_stride = sseu->max_subslices * subslice_stride; ++ ++ return slice * slice_stride + subslice * subslice_stride; ++} ++ ++static inline u16 sseu_get_eus(const struct sseu_dev_info *sseu, ++ int slice, int subslice) ++{ ++ int i, offset = sseu_eu_idx(sseu, slice, subslice); ++ u16 eu_mask = 0; ++ ++ for (i = 0; ++ i < DIV_ROUND_UP(sseu->max_eus_per_subslice, BITS_PER_BYTE); i++) { ++ eu_mask |= ((u16) sseu->eu_mask[offset + i]) << ++ (i * BITS_PER_BYTE); ++ } ++ ++ return eu_mask; ++} ++ ++static inline void sseu_set_eus(struct sseu_dev_info *sseu, ++ int slice, int subslice, u16 eu_mask) ++{ ++ int i, offset = sseu_eu_idx(sseu, slice, subslice); ++ ++ for (i = 0; ++ i < DIV_ROUND_UP(sseu->max_eus_per_subslice, BITS_PER_BYTE); i++) { ++ sseu->eu_mask[offset + i] = ++ (eu_mask >> (BITS_PER_BYTE * i)) & 0xff; ++ } ++} ++ ++const char *intel_platform_name(enum intel_platform platform); ++ ++void intel_device_info_subplatform_init(struct drm_i915_private *dev_priv); ++void intel_device_info_runtime_init(struct drm_i915_private *dev_priv); ++void intel_device_info_dump_flags(const struct intel_device_info *info, ++ struct drm_printer *p); ++void intel_device_info_dump_runtime(const struct intel_runtime_info *info, ++ struct drm_printer *p); ++void intel_device_info_dump_topology(const struct sseu_dev_info *sseu, ++ struct drm_printer *p); ++ ++void intel_device_info_init_mmio(struct drm_i915_private *dev_priv); ++ ++void intel_driver_caps_print(const struct intel_driver_caps *caps, ++ struct drm_printer *p); ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/intel_display.c b/drivers/gpu/drm/i915_legacy/intel_display.c +new file mode 100644 +index 000000000000..75105a2c59ea +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_display.c +@@ -0,0 +1,16814 @@ ++/* ++ * Copyright © 2006-2007 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER ++ * DEALINGS IN THE SOFTWARE. ++ * ++ * Authors: ++ * Eric Anholt ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "i915_drv.h" ++#include "i915_gem_clflush.h" ++#include "i915_reset.h" ++#include "i915_trace.h" ++#include "intel_atomic_plane.h" ++#include "intel_color.h" ++#include "intel_cdclk.h" ++#include "intel_crt.h" ++#include "intel_ddi.h" ++#include "intel_dp.h" ++#include "intel_drv.h" ++#include "intel_dsi.h" ++#include "intel_dvo.h" ++#include "intel_fbc.h" ++#include "intel_fbdev.h" ++#include "intel_frontbuffer.h" ++#include "intel_hdcp.h" ++#include "intel_hdmi.h" ++#include "intel_lvds.h" ++#include "intel_pipe_crc.h" ++#include "intel_pm.h" ++#include "intel_psr.h" ++#include "intel_sdvo.h" ++#include "intel_sprite.h" ++#include "intel_tv.h" ++ ++/* Primary plane formats for gen <= 3 */ ++static const u32 i8xx_primary_formats[] = { ++ DRM_FORMAT_C8, ++ DRM_FORMAT_RGB565, ++ DRM_FORMAT_XRGB1555, ++ DRM_FORMAT_XRGB8888, ++}; ++ ++/* Primary plane formats for gen >= 4 */ ++static const u32 i965_primary_formats[] = { ++ DRM_FORMAT_C8, ++ DRM_FORMAT_RGB565, ++ DRM_FORMAT_XRGB8888, ++ DRM_FORMAT_XBGR8888, ++ DRM_FORMAT_XRGB2101010, ++ DRM_FORMAT_XBGR2101010, ++}; ++ ++static const u64 i9xx_format_modifiers[] = { ++ I915_FORMAT_MOD_X_TILED, ++ DRM_FORMAT_MOD_LINEAR, ++ DRM_FORMAT_MOD_INVALID ++}; ++ ++/* Cursor formats */ ++static const u32 intel_cursor_formats[] = { ++ DRM_FORMAT_ARGB8888, ++}; ++ ++static const u64 cursor_format_modifiers[] = { ++ DRM_FORMAT_MOD_LINEAR, ++ DRM_FORMAT_MOD_INVALID ++}; ++ ++static void i9xx_crtc_clock_get(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config); ++static void ironlake_pch_clock_get(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config); ++ ++static int intel_framebuffer_init(struct intel_framebuffer *ifb, ++ struct drm_i915_gem_object *obj, ++ struct drm_mode_fb_cmd2 *mode_cmd); ++static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state); ++static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state); ++static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state, ++ const struct intel_link_m_n *m_n, ++ const struct intel_link_m_n *m2_n2); ++static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state); ++static void ironlake_set_pipeconf(const struct intel_crtc_state *crtc_state); ++static void haswell_set_pipeconf(const struct intel_crtc_state *crtc_state); ++static void haswell_set_pipemisc(const struct intel_crtc_state *crtc_state); ++static void vlv_prepare_pll(struct intel_crtc *crtc, ++ const struct intel_crtc_state *pipe_config); ++static void chv_prepare_pll(struct intel_crtc *crtc, ++ const struct intel_crtc_state *pipe_config); ++static void intel_begin_crtc_commit(struct intel_atomic_state *, struct intel_crtc *); ++static void intel_finish_crtc_commit(struct intel_atomic_state *, struct intel_crtc *); ++static void intel_crtc_init_scalers(struct intel_crtc *crtc, ++ struct intel_crtc_state *crtc_state); ++static void skylake_pfit_enable(const struct intel_crtc_state *crtc_state); ++static void ironlake_pfit_disable(const struct intel_crtc_state *old_crtc_state); ++static void ironlake_pfit_enable(const struct intel_crtc_state *crtc_state); ++static void intel_modeset_setup_hw_state(struct drm_device *dev, ++ struct drm_modeset_acquire_ctx *ctx); ++static void intel_pre_disable_primary_noatomic(struct drm_crtc *crtc); ++ ++struct intel_limit { ++ struct { ++ int min, max; ++ } dot, vco, n, m, m1, m2, p, p1; ++ ++ struct { ++ int dot_limit; ++ int p2_slow, p2_fast; ++ } p2; ++}; ++ ++/* returns HPLL frequency in kHz */ ++int vlv_get_hpll_vco(struct drm_i915_private *dev_priv) ++{ ++ int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 }; ++ ++ /* Obtain SKU information */ ++ mutex_lock(&dev_priv->sb_lock); ++ hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) & ++ CCK_FUSE_HPLL_FREQ_MASK; ++ mutex_unlock(&dev_priv->sb_lock); ++ ++ return vco_freq[hpll_freq] * 1000; ++} ++ ++int vlv_get_cck_clock(struct drm_i915_private *dev_priv, ++ const char *name, u32 reg, int ref_freq) ++{ ++ u32 val; ++ int divider; ++ ++ mutex_lock(&dev_priv->sb_lock); ++ val = vlv_cck_read(dev_priv, reg); ++ mutex_unlock(&dev_priv->sb_lock); ++ ++ divider = val & CCK_FREQUENCY_VALUES; ++ ++ WARN((val & CCK_FREQUENCY_STATUS) != ++ (divider << CCK_FREQUENCY_STATUS_SHIFT), ++ "%s change in progress\n", name); ++ ++ return DIV_ROUND_CLOSEST(ref_freq << 1, divider + 1); ++} ++ ++int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv, ++ const char *name, u32 reg) ++{ ++ if (dev_priv->hpll_freq == 0) ++ dev_priv->hpll_freq = vlv_get_hpll_vco(dev_priv); ++ ++ return vlv_get_cck_clock(dev_priv, name, reg, ++ dev_priv->hpll_freq); ++} ++ ++static void intel_update_czclk(struct drm_i915_private *dev_priv) ++{ ++ if (!(IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))) ++ return; ++ ++ dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk", ++ CCK_CZ_CLOCK_CONTROL); ++ ++ DRM_DEBUG_DRIVER("CZ clock rate: %d kHz\n", dev_priv->czclk_freq); ++} ++ ++static inline u32 /* units of 100MHz */ ++intel_fdi_link_freq(struct drm_i915_private *dev_priv, ++ const struct intel_crtc_state *pipe_config) ++{ ++ if (HAS_DDI(dev_priv)) ++ return pipe_config->port_clock; /* SPLL */ ++ else ++ return dev_priv->fdi_pll_freq; ++} ++ ++static const struct intel_limit intel_limits_i8xx_dac = { ++ .dot = { .min = 25000, .max = 350000 }, ++ .vco = { .min = 908000, .max = 1512000 }, ++ .n = { .min = 2, .max = 16 }, ++ .m = { .min = 96, .max = 140 }, ++ .m1 = { .min = 18, .max = 26 }, ++ .m2 = { .min = 6, .max = 16 }, ++ .p = { .min = 4, .max = 128 }, ++ .p1 = { .min = 2, .max = 33 }, ++ .p2 = { .dot_limit = 165000, ++ .p2_slow = 4, .p2_fast = 2 }, ++}; ++ ++static const struct intel_limit intel_limits_i8xx_dvo = { ++ .dot = { .min = 25000, .max = 350000 }, ++ .vco = { .min = 908000, .max = 1512000 }, ++ .n = { .min = 2, .max = 16 }, ++ .m = { .min = 96, .max = 140 }, ++ .m1 = { .min = 18, .max = 26 }, ++ .m2 = { .min = 6, .max = 16 }, ++ .p = { .min = 4, .max = 128 }, ++ .p1 = { .min = 2, .max = 33 }, ++ .p2 = { .dot_limit = 165000, ++ .p2_slow = 4, .p2_fast = 4 }, ++}; ++ ++static const struct intel_limit intel_limits_i8xx_lvds = { ++ .dot = { .min = 25000, .max = 350000 }, ++ .vco = { .min = 908000, .max = 1512000 }, ++ .n = { .min = 2, .max = 16 }, ++ .m = { .min = 96, .max = 140 }, ++ .m1 = { .min = 18, .max = 26 }, ++ .m2 = { .min = 6, .max = 16 }, ++ .p = { .min = 4, .max = 128 }, ++ .p1 = { .min = 1, .max = 6 }, ++ .p2 = { .dot_limit = 165000, ++ .p2_slow = 14, .p2_fast = 7 }, ++}; ++ ++static const struct intel_limit intel_limits_i9xx_sdvo = { ++ .dot = { .min = 20000, .max = 400000 }, ++ .vco = { .min = 1400000, .max = 2800000 }, ++ .n = { .min = 1, .max = 6 }, ++ .m = { .min = 70, .max = 120 }, ++ .m1 = { .min = 8, .max = 18 }, ++ .m2 = { .min = 3, .max = 7 }, ++ .p = { .min = 5, .max = 80 }, ++ .p1 = { .min = 1, .max = 8 }, ++ .p2 = { .dot_limit = 200000, ++ .p2_slow = 10, .p2_fast = 5 }, ++}; ++ ++static const struct intel_limit intel_limits_i9xx_lvds = { ++ .dot = { .min = 20000, .max = 400000 }, ++ .vco = { .min = 1400000, .max = 2800000 }, ++ .n = { .min = 1, .max = 6 }, ++ .m = { .min = 70, .max = 120 }, ++ .m1 = { .min = 8, .max = 18 }, ++ .m2 = { .min = 3, .max = 7 }, ++ .p = { .min = 7, .max = 98 }, ++ .p1 = { .min = 1, .max = 8 }, ++ .p2 = { .dot_limit = 112000, ++ .p2_slow = 14, .p2_fast = 7 }, ++}; ++ ++ ++static const struct intel_limit intel_limits_g4x_sdvo = { ++ .dot = { .min = 25000, .max = 270000 }, ++ .vco = { .min = 1750000, .max = 3500000}, ++ .n = { .min = 1, .max = 4 }, ++ .m = { .min = 104, .max = 138 }, ++ .m1 = { .min = 17, .max = 23 }, ++ .m2 = { .min = 5, .max = 11 }, ++ .p = { .min = 10, .max = 30 }, ++ .p1 = { .min = 1, .max = 3}, ++ .p2 = { .dot_limit = 270000, ++ .p2_slow = 10, ++ .p2_fast = 10 ++ }, ++}; ++ ++static const struct intel_limit intel_limits_g4x_hdmi = { ++ .dot = { .min = 22000, .max = 400000 }, ++ .vco = { .min = 1750000, .max = 3500000}, ++ .n = { .min = 1, .max = 4 }, ++ .m = { .min = 104, .max = 138 }, ++ .m1 = { .min = 16, .max = 23 }, ++ .m2 = { .min = 5, .max = 11 }, ++ .p = { .min = 5, .max = 80 }, ++ .p1 = { .min = 1, .max = 8}, ++ .p2 = { .dot_limit = 165000, ++ .p2_slow = 10, .p2_fast = 5 }, ++}; ++ ++static const struct intel_limit intel_limits_g4x_single_channel_lvds = { ++ .dot = { .min = 20000, .max = 115000 }, ++ .vco = { .min = 1750000, .max = 3500000 }, ++ .n = { .min = 1, .max = 3 }, ++ .m = { .min = 104, .max = 138 }, ++ .m1 = { .min = 17, .max = 23 }, ++ .m2 = { .min = 5, .max = 11 }, ++ .p = { .min = 28, .max = 112 }, ++ .p1 = { .min = 2, .max = 8 }, ++ .p2 = { .dot_limit = 0, ++ .p2_slow = 14, .p2_fast = 14 ++ }, ++}; ++ ++static const struct intel_limit intel_limits_g4x_dual_channel_lvds = { ++ .dot = { .min = 80000, .max = 224000 }, ++ .vco = { .min = 1750000, .max = 3500000 }, ++ .n = { .min = 1, .max = 3 }, ++ .m = { .min = 104, .max = 138 }, ++ .m1 = { .min = 17, .max = 23 }, ++ .m2 = { .min = 5, .max = 11 }, ++ .p = { .min = 14, .max = 42 }, ++ .p1 = { .min = 2, .max = 6 }, ++ .p2 = { .dot_limit = 0, ++ .p2_slow = 7, .p2_fast = 7 ++ }, ++}; ++ ++static const struct intel_limit intel_limits_pineview_sdvo = { ++ .dot = { .min = 20000, .max = 400000}, ++ .vco = { .min = 1700000, .max = 3500000 }, ++ /* Pineview's Ncounter is a ring counter */ ++ .n = { .min = 3, .max = 6 }, ++ .m = { .min = 2, .max = 256 }, ++ /* Pineview only has one combined m divider, which we treat as m2. */ ++ .m1 = { .min = 0, .max = 0 }, ++ .m2 = { .min = 0, .max = 254 }, ++ .p = { .min = 5, .max = 80 }, ++ .p1 = { .min = 1, .max = 8 }, ++ .p2 = { .dot_limit = 200000, ++ .p2_slow = 10, .p2_fast = 5 }, ++}; ++ ++static const struct intel_limit intel_limits_pineview_lvds = { ++ .dot = { .min = 20000, .max = 400000 }, ++ .vco = { .min = 1700000, .max = 3500000 }, ++ .n = { .min = 3, .max = 6 }, ++ .m = { .min = 2, .max = 256 }, ++ .m1 = { .min = 0, .max = 0 }, ++ .m2 = { .min = 0, .max = 254 }, ++ .p = { .min = 7, .max = 112 }, ++ .p1 = { .min = 1, .max = 8 }, ++ .p2 = { .dot_limit = 112000, ++ .p2_slow = 14, .p2_fast = 14 }, ++}; ++ ++/* Ironlake / Sandybridge ++ * ++ * We calculate clock using (register_value + 2) for N/M1/M2, so here ++ * the range value for them is (actual_value - 2). ++ */ ++static const struct intel_limit intel_limits_ironlake_dac = { ++ .dot = { .min = 25000, .max = 350000 }, ++ .vco = { .min = 1760000, .max = 3510000 }, ++ .n = { .min = 1, .max = 5 }, ++ .m = { .min = 79, .max = 127 }, ++ .m1 = { .min = 12, .max = 22 }, ++ .m2 = { .min = 5, .max = 9 }, ++ .p = { .min = 5, .max = 80 }, ++ .p1 = { .min = 1, .max = 8 }, ++ .p2 = { .dot_limit = 225000, ++ .p2_slow = 10, .p2_fast = 5 }, ++}; ++ ++static const struct intel_limit intel_limits_ironlake_single_lvds = { ++ .dot = { .min = 25000, .max = 350000 }, ++ .vco = { .min = 1760000, .max = 3510000 }, ++ .n = { .min = 1, .max = 3 }, ++ .m = { .min = 79, .max = 118 }, ++ .m1 = { .min = 12, .max = 22 }, ++ .m2 = { .min = 5, .max = 9 }, ++ .p = { .min = 28, .max = 112 }, ++ .p1 = { .min = 2, .max = 8 }, ++ .p2 = { .dot_limit = 225000, ++ .p2_slow = 14, .p2_fast = 14 }, ++}; ++ ++static const struct intel_limit intel_limits_ironlake_dual_lvds = { ++ .dot = { .min = 25000, .max = 350000 }, ++ .vco = { .min = 1760000, .max = 3510000 }, ++ .n = { .min = 1, .max = 3 }, ++ .m = { .min = 79, .max = 127 }, ++ .m1 = { .min = 12, .max = 22 }, ++ .m2 = { .min = 5, .max = 9 }, ++ .p = { .min = 14, .max = 56 }, ++ .p1 = { .min = 2, .max = 8 }, ++ .p2 = { .dot_limit = 225000, ++ .p2_slow = 7, .p2_fast = 7 }, ++}; ++ ++/* LVDS 100mhz refclk limits. */ ++static const struct intel_limit intel_limits_ironlake_single_lvds_100m = { ++ .dot = { .min = 25000, .max = 350000 }, ++ .vco = { .min = 1760000, .max = 3510000 }, ++ .n = { .min = 1, .max = 2 }, ++ .m = { .min = 79, .max = 126 }, ++ .m1 = { .min = 12, .max = 22 }, ++ .m2 = { .min = 5, .max = 9 }, ++ .p = { .min = 28, .max = 112 }, ++ .p1 = { .min = 2, .max = 8 }, ++ .p2 = { .dot_limit = 225000, ++ .p2_slow = 14, .p2_fast = 14 }, ++}; ++ ++static const struct intel_limit intel_limits_ironlake_dual_lvds_100m = { ++ .dot = { .min = 25000, .max = 350000 }, ++ .vco = { .min = 1760000, .max = 3510000 }, ++ .n = { .min = 1, .max = 3 }, ++ .m = { .min = 79, .max = 126 }, ++ .m1 = { .min = 12, .max = 22 }, ++ .m2 = { .min = 5, .max = 9 }, ++ .p = { .min = 14, .max = 42 }, ++ .p1 = { .min = 2, .max = 6 }, ++ .p2 = { .dot_limit = 225000, ++ .p2_slow = 7, .p2_fast = 7 }, ++}; ++ ++static const struct intel_limit intel_limits_vlv = { ++ /* ++ * These are the data rate limits (measured in fast clocks) ++ * since those are the strictest limits we have. The fast ++ * clock and actual rate limits are more relaxed, so checking ++ * them would make no difference. ++ */ ++ .dot = { .min = 25000 * 5, .max = 270000 * 5 }, ++ .vco = { .min = 4000000, .max = 6000000 }, ++ .n = { .min = 1, .max = 7 }, ++ .m1 = { .min = 2, .max = 3 }, ++ .m2 = { .min = 11, .max = 156 }, ++ .p1 = { .min = 2, .max = 3 }, ++ .p2 = { .p2_slow = 2, .p2_fast = 20 }, /* slow=min, fast=max */ ++}; ++ ++static const struct intel_limit intel_limits_chv = { ++ /* ++ * These are the data rate limits (measured in fast clocks) ++ * since those are the strictest limits we have. The fast ++ * clock and actual rate limits are more relaxed, so checking ++ * them would make no difference. ++ */ ++ .dot = { .min = 25000 * 5, .max = 540000 * 5}, ++ .vco = { .min = 4800000, .max = 6480000 }, ++ .n = { .min = 1, .max = 1 }, ++ .m1 = { .min = 2, .max = 2 }, ++ .m2 = { .min = 24 << 22, .max = 175 << 22 }, ++ .p1 = { .min = 2, .max = 4 }, ++ .p2 = { .p2_slow = 1, .p2_fast = 14 }, ++}; ++ ++static const struct intel_limit intel_limits_bxt = { ++ /* FIXME: find real dot limits */ ++ .dot = { .min = 0, .max = INT_MAX }, ++ .vco = { .min = 4800000, .max = 6700000 }, ++ .n = { .min = 1, .max = 1 }, ++ .m1 = { .min = 2, .max = 2 }, ++ /* FIXME: find real m2 limits */ ++ .m2 = { .min = 2 << 22, .max = 255 << 22 }, ++ .p1 = { .min = 2, .max = 4 }, ++ .p2 = { .p2_slow = 1, .p2_fast = 20 }, ++}; ++ ++static void ++skl_wa_827(struct drm_i915_private *dev_priv, int pipe, bool enable) ++{ ++ if (enable) ++ I915_WRITE(CLKGATE_DIS_PSL(pipe), ++ I915_READ(CLKGATE_DIS_PSL(pipe)) | ++ DUPS1_GATING_DIS | DUPS2_GATING_DIS); ++ else ++ I915_WRITE(CLKGATE_DIS_PSL(pipe), ++ I915_READ(CLKGATE_DIS_PSL(pipe)) & ++ ~(DUPS1_GATING_DIS | DUPS2_GATING_DIS)); ++} ++ ++static bool ++needs_modeset(const struct drm_crtc_state *state) ++{ ++ return drm_atomic_crtc_needs_modeset(state); ++} ++ ++/* ++ * Platform specific helpers to calculate the port PLL loopback- (clock.m), ++ * and post-divider (clock.p) values, pre- (clock.vco) and post-divided fast ++ * (clock.dot) clock rates. This fast dot clock is fed to the port's IO logic. ++ * The helpers' return value is the rate of the clock that is fed to the ++ * display engine's pipe which can be the above fast dot clock rate or a ++ * divided-down version of it. ++ */ ++/* m1 is reserved as 0 in Pineview, n is a ring counter */ ++static int pnv_calc_dpll_params(int refclk, struct dpll *clock) ++{ ++ clock->m = clock->m2 + 2; ++ clock->p = clock->p1 * clock->p2; ++ if (WARN_ON(clock->n == 0 || clock->p == 0)) ++ return 0; ++ clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n); ++ clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p); ++ ++ return clock->dot; ++} ++ ++static u32 i9xx_dpll_compute_m(struct dpll *dpll) ++{ ++ return 5 * (dpll->m1 + 2) + (dpll->m2 + 2); ++} ++ ++static int i9xx_calc_dpll_params(int refclk, struct dpll *clock) ++{ ++ clock->m = i9xx_dpll_compute_m(clock); ++ clock->p = clock->p1 * clock->p2; ++ if (WARN_ON(clock->n + 2 == 0 || clock->p == 0)) ++ return 0; ++ clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n + 2); ++ clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p); ++ ++ return clock->dot; ++} ++ ++static int vlv_calc_dpll_params(int refclk, struct dpll *clock) ++{ ++ clock->m = clock->m1 * clock->m2; ++ clock->p = clock->p1 * clock->p2; ++ if (WARN_ON(clock->n == 0 || clock->p == 0)) ++ return 0; ++ clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n); ++ clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p); ++ ++ return clock->dot / 5; ++} ++ ++int chv_calc_dpll_params(int refclk, struct dpll *clock) ++{ ++ clock->m = clock->m1 * clock->m2; ++ clock->p = clock->p1 * clock->p2; ++ if (WARN_ON(clock->n == 0 || clock->p == 0)) ++ return 0; ++ clock->vco = DIV_ROUND_CLOSEST_ULL((u64)refclk * clock->m, ++ clock->n << 22); ++ clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p); ++ ++ return clock->dot / 5; ++} ++ ++#define INTELPllInvalid(s) do { /* DRM_DEBUG(s); */ return false; } while (0) ++ ++/* ++ * Returns whether the given set of divisors are valid for a given refclk with ++ * the given connectors. ++ */ ++static bool intel_PLL_is_valid(struct drm_i915_private *dev_priv, ++ const struct intel_limit *limit, ++ const struct dpll *clock) ++{ ++ if (clock->n < limit->n.min || limit->n.max < clock->n) ++ INTELPllInvalid("n out of range\n"); ++ if (clock->p1 < limit->p1.min || limit->p1.max < clock->p1) ++ INTELPllInvalid("p1 out of range\n"); ++ if (clock->m2 < limit->m2.min || limit->m2.max < clock->m2) ++ INTELPllInvalid("m2 out of range\n"); ++ if (clock->m1 < limit->m1.min || limit->m1.max < clock->m1) ++ INTELPllInvalid("m1 out of range\n"); ++ ++ if (!IS_PINEVIEW(dev_priv) && !IS_VALLEYVIEW(dev_priv) && ++ !IS_CHERRYVIEW(dev_priv) && !IS_GEN9_LP(dev_priv)) ++ if (clock->m1 <= clock->m2) ++ INTELPllInvalid("m1 <= m2\n"); ++ ++ if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) && ++ !IS_GEN9_LP(dev_priv)) { ++ if (clock->p < limit->p.min || limit->p.max < clock->p) ++ INTELPllInvalid("p out of range\n"); ++ if (clock->m < limit->m.min || limit->m.max < clock->m) ++ INTELPllInvalid("m out of range\n"); ++ } ++ ++ if (clock->vco < limit->vco.min || limit->vco.max < clock->vco) ++ INTELPllInvalid("vco out of range\n"); ++ /* XXX: We may need to be checking "Dot clock" depending on the multiplier, ++ * connector, etc., rather than just a single range. ++ */ ++ if (clock->dot < limit->dot.min || limit->dot.max < clock->dot) ++ INTELPllInvalid("dot out of range\n"); ++ ++ return true; ++} ++ ++static int ++i9xx_select_p2_div(const struct intel_limit *limit, ++ const struct intel_crtc_state *crtc_state, ++ int target) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); ++ ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) { ++ /* ++ * For LVDS just rely on its current settings for dual-channel. ++ * We haven't figured out how to reliably set up different ++ * single/dual channel state, if we even can. ++ */ ++ if (intel_is_dual_link_lvds(dev_priv)) ++ return limit->p2.p2_fast; ++ else ++ return limit->p2.p2_slow; ++ } else { ++ if (target < limit->p2.dot_limit) ++ return limit->p2.p2_slow; ++ else ++ return limit->p2.p2_fast; ++ } ++} ++ ++/* ++ * Returns a set of divisors for the desired target clock with the given ++ * refclk, or FALSE. The returned values represent the clock equation: ++ * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2. ++ * ++ * Target and reference clocks are specified in kHz. ++ * ++ * If match_clock is provided, then best_clock P divider must match the P ++ * divider from @match_clock used for LVDS downclocking. ++ */ ++static bool ++i9xx_find_best_dpll(const struct intel_limit *limit, ++ struct intel_crtc_state *crtc_state, ++ int target, int refclk, struct dpll *match_clock, ++ struct dpll *best_clock) ++{ ++ struct drm_device *dev = crtc_state->base.crtc->dev; ++ struct dpll clock; ++ int err = target; ++ ++ memset(best_clock, 0, sizeof(*best_clock)); ++ ++ clock.p2 = i9xx_select_p2_div(limit, crtc_state, target); ++ ++ for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max; ++ clock.m1++) { ++ for (clock.m2 = limit->m2.min; ++ clock.m2 <= limit->m2.max; clock.m2++) { ++ if (clock.m2 >= clock.m1) ++ break; ++ for (clock.n = limit->n.min; ++ clock.n <= limit->n.max; clock.n++) { ++ for (clock.p1 = limit->p1.min; ++ clock.p1 <= limit->p1.max; clock.p1++) { ++ int this_err; ++ ++ i9xx_calc_dpll_params(refclk, &clock); ++ if (!intel_PLL_is_valid(to_i915(dev), ++ limit, ++ &clock)) ++ continue; ++ if (match_clock && ++ clock.p != match_clock->p) ++ continue; ++ ++ this_err = abs(clock.dot - target); ++ if (this_err < err) { ++ *best_clock = clock; ++ err = this_err; ++ } ++ } ++ } ++ } ++ } ++ ++ return (err != target); ++} ++ ++/* ++ * Returns a set of divisors for the desired target clock with the given ++ * refclk, or FALSE. The returned values represent the clock equation: ++ * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2. ++ * ++ * Target and reference clocks are specified in kHz. ++ * ++ * If match_clock is provided, then best_clock P divider must match the P ++ * divider from @match_clock used for LVDS downclocking. ++ */ ++static bool ++pnv_find_best_dpll(const struct intel_limit *limit, ++ struct intel_crtc_state *crtc_state, ++ int target, int refclk, struct dpll *match_clock, ++ struct dpll *best_clock) ++{ ++ struct drm_device *dev = crtc_state->base.crtc->dev; ++ struct dpll clock; ++ int err = target; ++ ++ memset(best_clock, 0, sizeof(*best_clock)); ++ ++ clock.p2 = i9xx_select_p2_div(limit, crtc_state, target); ++ ++ for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max; ++ clock.m1++) { ++ for (clock.m2 = limit->m2.min; ++ clock.m2 <= limit->m2.max; clock.m2++) { ++ for (clock.n = limit->n.min; ++ clock.n <= limit->n.max; clock.n++) { ++ for (clock.p1 = limit->p1.min; ++ clock.p1 <= limit->p1.max; clock.p1++) { ++ int this_err; ++ ++ pnv_calc_dpll_params(refclk, &clock); ++ if (!intel_PLL_is_valid(to_i915(dev), ++ limit, ++ &clock)) ++ continue; ++ if (match_clock && ++ clock.p != match_clock->p) ++ continue; ++ ++ this_err = abs(clock.dot - target); ++ if (this_err < err) { ++ *best_clock = clock; ++ err = this_err; ++ } ++ } ++ } ++ } ++ } ++ ++ return (err != target); ++} ++ ++/* ++ * Returns a set of divisors for the desired target clock with the given ++ * refclk, or FALSE. The returned values represent the clock equation: ++ * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2. ++ * ++ * Target and reference clocks are specified in kHz. ++ * ++ * If match_clock is provided, then best_clock P divider must match the P ++ * divider from @match_clock used for LVDS downclocking. ++ */ ++static bool ++g4x_find_best_dpll(const struct intel_limit *limit, ++ struct intel_crtc_state *crtc_state, ++ int target, int refclk, struct dpll *match_clock, ++ struct dpll *best_clock) ++{ ++ struct drm_device *dev = crtc_state->base.crtc->dev; ++ struct dpll clock; ++ int max_n; ++ bool found = false; ++ /* approximately equals target * 0.00585 */ ++ int err_most = (target >> 8) + (target >> 9); ++ ++ memset(best_clock, 0, sizeof(*best_clock)); ++ ++ clock.p2 = i9xx_select_p2_div(limit, crtc_state, target); ++ ++ max_n = limit->n.max; ++ /* based on hardware requirement, prefer smaller n to precision */ ++ for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) { ++ /* based on hardware requirement, prefere larger m1,m2 */ ++ for (clock.m1 = limit->m1.max; ++ clock.m1 >= limit->m1.min; clock.m1--) { ++ for (clock.m2 = limit->m2.max; ++ clock.m2 >= limit->m2.min; clock.m2--) { ++ for (clock.p1 = limit->p1.max; ++ clock.p1 >= limit->p1.min; clock.p1--) { ++ int this_err; ++ ++ i9xx_calc_dpll_params(refclk, &clock); ++ if (!intel_PLL_is_valid(to_i915(dev), ++ limit, ++ &clock)) ++ continue; ++ ++ this_err = abs(clock.dot - target); ++ if (this_err < err_most) { ++ *best_clock = clock; ++ err_most = this_err; ++ max_n = clock.n; ++ found = true; ++ } ++ } ++ } ++ } ++ } ++ return found; ++} ++ ++/* ++ * Check if the calculated PLL configuration is more optimal compared to the ++ * best configuration and error found so far. Return the calculated error. ++ */ ++static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq, ++ const struct dpll *calculated_clock, ++ const struct dpll *best_clock, ++ unsigned int best_error_ppm, ++ unsigned int *error_ppm) ++{ ++ /* ++ * For CHV ignore the error and consider only the P value. ++ * Prefer a bigger P value based on HW requirements. ++ */ ++ if (IS_CHERRYVIEW(to_i915(dev))) { ++ *error_ppm = 0; ++ ++ return calculated_clock->p > best_clock->p; ++ } ++ ++ if (WARN_ON_ONCE(!target_freq)) ++ return false; ++ ++ *error_ppm = div_u64(1000000ULL * ++ abs(target_freq - calculated_clock->dot), ++ target_freq); ++ /* ++ * Prefer a better P value over a better (smaller) error if the error ++ * is small. Ensure this preference for future configurations too by ++ * setting the error to 0. ++ */ ++ if (*error_ppm < 100 && calculated_clock->p > best_clock->p) { ++ *error_ppm = 0; ++ ++ return true; ++ } ++ ++ return *error_ppm + 10 < best_error_ppm; ++} ++ ++/* ++ * Returns a set of divisors for the desired target clock with the given ++ * refclk, or FALSE. The returned values represent the clock equation: ++ * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2. ++ */ ++static bool ++vlv_find_best_dpll(const struct intel_limit *limit, ++ struct intel_crtc_state *crtc_state, ++ int target, int refclk, struct dpll *match_clock, ++ struct dpll *best_clock) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_device *dev = crtc->base.dev; ++ struct dpll clock; ++ unsigned int bestppm = 1000000; ++ /* min update 19.2 MHz */ ++ int max_n = min(limit->n.max, refclk / 19200); ++ bool found = false; ++ ++ target *= 5; /* fast clock */ ++ ++ memset(best_clock, 0, sizeof(*best_clock)); ++ ++ /* based on hardware requirement, prefer smaller n to precision */ ++ for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) { ++ for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) { ++ for (clock.p2 = limit->p2.p2_fast; clock.p2 >= limit->p2.p2_slow; ++ clock.p2 -= clock.p2 > 10 ? 2 : 1) { ++ clock.p = clock.p1 * clock.p2; ++ /* based on hardware requirement, prefer bigger m1,m2 values */ ++ for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max; clock.m1++) { ++ unsigned int ppm; ++ ++ clock.m2 = DIV_ROUND_CLOSEST(target * clock.p * clock.n, ++ refclk * clock.m1); ++ ++ vlv_calc_dpll_params(refclk, &clock); ++ ++ if (!intel_PLL_is_valid(to_i915(dev), ++ limit, ++ &clock)) ++ continue; ++ ++ if (!vlv_PLL_is_optimal(dev, target, ++ &clock, ++ best_clock, ++ bestppm, &ppm)) ++ continue; ++ ++ *best_clock = clock; ++ bestppm = ppm; ++ found = true; ++ } ++ } ++ } ++ } ++ ++ return found; ++} ++ ++/* ++ * Returns a set of divisors for the desired target clock with the given ++ * refclk, or FALSE. The returned values represent the clock equation: ++ * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2. ++ */ ++static bool ++chv_find_best_dpll(const struct intel_limit *limit, ++ struct intel_crtc_state *crtc_state, ++ int target, int refclk, struct dpll *match_clock, ++ struct dpll *best_clock) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_device *dev = crtc->base.dev; ++ unsigned int best_error_ppm; ++ struct dpll clock; ++ u64 m2; ++ int found = false; ++ ++ memset(best_clock, 0, sizeof(*best_clock)); ++ best_error_ppm = 1000000; ++ ++ /* ++ * Based on hardware doc, the n always set to 1, and m1 always ++ * set to 2. If requires to support 200Mhz refclk, we need to ++ * revisit this because n may not 1 anymore. ++ */ ++ clock.n = 1, clock.m1 = 2; ++ target *= 5; /* fast clock */ ++ ++ for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) { ++ for (clock.p2 = limit->p2.p2_fast; ++ clock.p2 >= limit->p2.p2_slow; ++ clock.p2 -= clock.p2 > 10 ? 2 : 1) { ++ unsigned int error_ppm; ++ ++ clock.p = clock.p1 * clock.p2; ++ ++ m2 = DIV_ROUND_CLOSEST_ULL(((u64)target * clock.p * ++ clock.n) << 22, refclk * clock.m1); ++ ++ if (m2 > INT_MAX/clock.m1) ++ continue; ++ ++ clock.m2 = m2; ++ ++ chv_calc_dpll_params(refclk, &clock); ++ ++ if (!intel_PLL_is_valid(to_i915(dev), limit, &clock)) ++ continue; ++ ++ if (!vlv_PLL_is_optimal(dev, target, &clock, best_clock, ++ best_error_ppm, &error_ppm)) ++ continue; ++ ++ *best_clock = clock; ++ best_error_ppm = error_ppm; ++ found = true; ++ } ++ } ++ ++ return found; ++} ++ ++bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, ++ struct dpll *best_clock) ++{ ++ int refclk = 100000; ++ const struct intel_limit *limit = &intel_limits_bxt; ++ ++ return chv_find_best_dpll(limit, crtc_state, ++ crtc_state->port_clock, refclk, ++ NULL, best_clock); ++} ++ ++bool intel_crtc_active(struct intel_crtc *crtc) ++{ ++ /* Be paranoid as we can arrive here with only partial ++ * state retrieved from the hardware during setup. ++ * ++ * We can ditch the adjusted_mode.crtc_clock check as soon ++ * as Haswell has gained clock readout/fastboot support. ++ * ++ * We can ditch the crtc->primary->state->fb check as soon as we can ++ * properly reconstruct framebuffers. ++ * ++ * FIXME: The intel_crtc->active here should be switched to ++ * crtc->state->active once we have proper CRTC states wired up ++ * for atomic. ++ */ ++ return crtc->active && crtc->base.primary->state->fb && ++ crtc->config->base.adjusted_mode.crtc_clock; ++} ++ ++enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv, ++ enum pipe pipe) ++{ ++ struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); ++ ++ return crtc->config->cpu_transcoder; ++} ++ ++static bool pipe_scanline_is_moving(struct drm_i915_private *dev_priv, ++ enum pipe pipe) ++{ ++ i915_reg_t reg = PIPEDSL(pipe); ++ u32 line1, line2; ++ u32 line_mask; ++ ++ if (IS_GEN(dev_priv, 2)) ++ line_mask = DSL_LINEMASK_GEN2; ++ else ++ line_mask = DSL_LINEMASK_GEN3; ++ ++ line1 = I915_READ(reg) & line_mask; ++ msleep(5); ++ line2 = I915_READ(reg) & line_mask; ++ ++ return line1 != line2; ++} ++ ++static void wait_for_pipe_scanline_moving(struct intel_crtc *crtc, bool state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ ++ /* Wait for the display line to settle/start moving */ ++ if (wait_for(pipe_scanline_is_moving(dev_priv, pipe) == state, 100)) ++ DRM_ERROR("pipe %c scanline %s wait timed out\n", ++ pipe_name(pipe), onoff(state)); ++} ++ ++static void intel_wait_for_pipe_scanline_stopped(struct intel_crtc *crtc) ++{ ++ wait_for_pipe_scanline_moving(crtc, false); ++} ++ ++static void intel_wait_for_pipe_scanline_moving(struct intel_crtc *crtc) ++{ ++ wait_for_pipe_scanline_moving(crtc, true); ++} ++ ++static void ++intel_wait_for_pipe_off(const struct intel_crtc_state *old_crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ ++ if (INTEL_GEN(dev_priv) >= 4) { ++ enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder; ++ i915_reg_t reg = PIPECONF(cpu_transcoder); ++ ++ /* Wait for the Pipe State to go off */ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ reg, I965_PIPECONF_ACTIVE, 0, ++ 100)) ++ WARN(1, "pipe_off wait timed out\n"); ++ } else { ++ intel_wait_for_pipe_scanline_stopped(crtc); ++ } ++} ++ ++/* Only for pre-ILK configs */ ++void assert_pll(struct drm_i915_private *dev_priv, ++ enum pipe pipe, bool state) ++{ ++ u32 val; ++ bool cur_state; ++ ++ val = I915_READ(DPLL(pipe)); ++ cur_state = !!(val & DPLL_VCO_ENABLE); ++ I915_STATE_WARN(cur_state != state, ++ "PLL state assertion failure (expected %s, current %s)\n", ++ onoff(state), onoff(cur_state)); ++} ++ ++/* XXX: the dsi pll is shared between MIPI DSI ports */ ++void assert_dsi_pll(struct drm_i915_private *dev_priv, bool state) ++{ ++ u32 val; ++ bool cur_state; ++ ++ mutex_lock(&dev_priv->sb_lock); ++ val = vlv_cck_read(dev_priv, CCK_REG_DSI_PLL_CONTROL); ++ mutex_unlock(&dev_priv->sb_lock); ++ ++ cur_state = val & DSI_PLL_VCO_EN; ++ I915_STATE_WARN(cur_state != state, ++ "DSI PLL state assertion failure (expected %s, current %s)\n", ++ onoff(state), onoff(cur_state)); ++} ++ ++static void assert_fdi_tx(struct drm_i915_private *dev_priv, ++ enum pipe pipe, bool state) ++{ ++ bool cur_state; ++ enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv, ++ pipe); ++ ++ if (HAS_DDI(dev_priv)) { ++ /* DDI does not have a specific FDI_TX register */ ++ u32 val = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder)); ++ cur_state = !!(val & TRANS_DDI_FUNC_ENABLE); ++ } else { ++ u32 val = I915_READ(FDI_TX_CTL(pipe)); ++ cur_state = !!(val & FDI_TX_ENABLE); ++ } ++ I915_STATE_WARN(cur_state != state, ++ "FDI TX state assertion failure (expected %s, current %s)\n", ++ onoff(state), onoff(cur_state)); ++} ++#define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true) ++#define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false) ++ ++static void assert_fdi_rx(struct drm_i915_private *dev_priv, ++ enum pipe pipe, bool state) ++{ ++ u32 val; ++ bool cur_state; ++ ++ val = I915_READ(FDI_RX_CTL(pipe)); ++ cur_state = !!(val & FDI_RX_ENABLE); ++ I915_STATE_WARN(cur_state != state, ++ "FDI RX state assertion failure (expected %s, current %s)\n", ++ onoff(state), onoff(cur_state)); ++} ++#define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true) ++#define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false) ++ ++static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv, ++ enum pipe pipe) ++{ ++ u32 val; ++ ++ /* ILK FDI PLL is always enabled */ ++ if (IS_GEN(dev_priv, 5)) ++ return; ++ ++ /* On Haswell, DDI ports are responsible for the FDI PLL setup */ ++ if (HAS_DDI(dev_priv)) ++ return; ++ ++ val = I915_READ(FDI_TX_CTL(pipe)); ++ I915_STATE_WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n"); ++} ++ ++void assert_fdi_rx_pll(struct drm_i915_private *dev_priv, ++ enum pipe pipe, bool state) ++{ ++ u32 val; ++ bool cur_state; ++ ++ val = I915_READ(FDI_RX_CTL(pipe)); ++ cur_state = !!(val & FDI_RX_PLL_ENABLE); ++ I915_STATE_WARN(cur_state != state, ++ "FDI RX PLL assertion failure (expected %s, current %s)\n", ++ onoff(state), onoff(cur_state)); ++} ++ ++void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe) ++{ ++ i915_reg_t pp_reg; ++ u32 val; ++ enum pipe panel_pipe = INVALID_PIPE; ++ bool locked = true; ++ ++ if (WARN_ON(HAS_DDI(dev_priv))) ++ return; ++ ++ if (HAS_PCH_SPLIT(dev_priv)) { ++ u32 port_sel; ++ ++ pp_reg = PP_CONTROL(0); ++ port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK; ++ ++ switch (port_sel) { ++ case PANEL_PORT_SELECT_LVDS: ++ intel_lvds_port_enabled(dev_priv, PCH_LVDS, &panel_pipe); ++ break; ++ case PANEL_PORT_SELECT_DPA: ++ intel_dp_port_enabled(dev_priv, DP_A, PORT_A, &panel_pipe); ++ break; ++ case PANEL_PORT_SELECT_DPC: ++ intel_dp_port_enabled(dev_priv, PCH_DP_C, PORT_C, &panel_pipe); ++ break; ++ case PANEL_PORT_SELECT_DPD: ++ intel_dp_port_enabled(dev_priv, PCH_DP_D, PORT_D, &panel_pipe); ++ break; ++ default: ++ MISSING_CASE(port_sel); ++ break; ++ } ++ } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { ++ /* presumably write lock depends on pipe, not port select */ ++ pp_reg = PP_CONTROL(pipe); ++ panel_pipe = pipe; ++ } else { ++ u32 port_sel; ++ ++ pp_reg = PP_CONTROL(0); ++ port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK; ++ ++ WARN_ON(port_sel != PANEL_PORT_SELECT_LVDS); ++ intel_lvds_port_enabled(dev_priv, LVDS, &panel_pipe); ++ } ++ ++ val = I915_READ(pp_reg); ++ if (!(val & PANEL_POWER_ON) || ++ ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS)) ++ locked = false; ++ ++ I915_STATE_WARN(panel_pipe == pipe && locked, ++ "panel assertion failure, pipe %c regs locked\n", ++ pipe_name(pipe)); ++} ++ ++void assert_pipe(struct drm_i915_private *dev_priv, ++ enum pipe pipe, bool state) ++{ ++ bool cur_state; ++ enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv, ++ pipe); ++ enum intel_display_power_domain power_domain; ++ intel_wakeref_t wakeref; ++ ++ /* we keep both pipes enabled on 830 */ ++ if (IS_I830(dev_priv)) ++ state = true; ++ ++ power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder); ++ wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain); ++ if (wakeref) { ++ u32 val = I915_READ(PIPECONF(cpu_transcoder)); ++ cur_state = !!(val & PIPECONF_ENABLE); ++ ++ intel_display_power_put(dev_priv, power_domain, wakeref); ++ } else { ++ cur_state = false; ++ } ++ ++ I915_STATE_WARN(cur_state != state, ++ "pipe %c assertion failure (expected %s, current %s)\n", ++ pipe_name(pipe), onoff(state), onoff(cur_state)); ++} ++ ++static void assert_plane(struct intel_plane *plane, bool state) ++{ ++ enum pipe pipe; ++ bool cur_state; ++ ++ cur_state = plane->get_hw_state(plane, &pipe); ++ ++ I915_STATE_WARN(cur_state != state, ++ "%s assertion failure (expected %s, current %s)\n", ++ plane->base.name, onoff(state), onoff(cur_state)); ++} ++ ++#define assert_plane_enabled(p) assert_plane(p, true) ++#define assert_plane_disabled(p) assert_plane(p, false) ++ ++static void assert_planes_disabled(struct intel_crtc *crtc) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ struct intel_plane *plane; ++ ++ for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) ++ assert_plane_disabled(plane); ++} ++ ++static void assert_vblank_disabled(struct drm_crtc *crtc) ++{ ++ if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0)) ++ drm_crtc_vblank_put(crtc); ++} ++ ++void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv, ++ enum pipe pipe) ++{ ++ u32 val; ++ bool enabled; ++ ++ val = I915_READ(PCH_TRANSCONF(pipe)); ++ enabled = !!(val & TRANS_ENABLE); ++ I915_STATE_WARN(enabled, ++ "transcoder assertion failed, should be off on pipe %c but is still active\n", ++ pipe_name(pipe)); ++} ++ ++static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv, ++ enum pipe pipe, enum port port, ++ i915_reg_t dp_reg) ++{ ++ enum pipe port_pipe; ++ bool state; ++ ++ state = intel_dp_port_enabled(dev_priv, dp_reg, port, &port_pipe); ++ ++ I915_STATE_WARN(state && port_pipe == pipe, ++ "PCH DP %c enabled on transcoder %c, should be disabled\n", ++ port_name(port), pipe_name(pipe)); ++ ++ I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B, ++ "IBX PCH DP %c still using transcoder B\n", ++ port_name(port)); ++} ++ ++static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv, ++ enum pipe pipe, enum port port, ++ i915_reg_t hdmi_reg) ++{ ++ enum pipe port_pipe; ++ bool state; ++ ++ state = intel_sdvo_port_enabled(dev_priv, hdmi_reg, &port_pipe); ++ ++ I915_STATE_WARN(state && port_pipe == pipe, ++ "PCH HDMI %c enabled on transcoder %c, should be disabled\n", ++ port_name(port), pipe_name(pipe)); ++ ++ I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B, ++ "IBX PCH HDMI %c still using transcoder B\n", ++ port_name(port)); ++} ++ ++static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv, ++ enum pipe pipe) ++{ ++ enum pipe port_pipe; ++ ++ assert_pch_dp_disabled(dev_priv, pipe, PORT_B, PCH_DP_B); ++ assert_pch_dp_disabled(dev_priv, pipe, PORT_C, PCH_DP_C); ++ assert_pch_dp_disabled(dev_priv, pipe, PORT_D, PCH_DP_D); ++ ++ I915_STATE_WARN(intel_crt_port_enabled(dev_priv, PCH_ADPA, &port_pipe) && ++ port_pipe == pipe, ++ "PCH VGA enabled on transcoder %c, should be disabled\n", ++ pipe_name(pipe)); ++ ++ I915_STATE_WARN(intel_lvds_port_enabled(dev_priv, PCH_LVDS, &port_pipe) && ++ port_pipe == pipe, ++ "PCH LVDS enabled on transcoder %c, should be disabled\n", ++ pipe_name(pipe)); ++ ++ /* PCH SDVOB multiplex with HDMIB */ ++ assert_pch_hdmi_disabled(dev_priv, pipe, PORT_B, PCH_HDMIB); ++ assert_pch_hdmi_disabled(dev_priv, pipe, PORT_C, PCH_HDMIC); ++ assert_pch_hdmi_disabled(dev_priv, pipe, PORT_D, PCH_HDMID); ++} ++ ++static void _vlv_enable_pll(struct intel_crtc *crtc, ++ const struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ ++ I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll); ++ POSTING_READ(DPLL(pipe)); ++ udelay(150); ++ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ DPLL(pipe), ++ DPLL_LOCK_VLV, ++ DPLL_LOCK_VLV, ++ 1)) ++ DRM_ERROR("DPLL %d failed to lock\n", pipe); ++} ++ ++static void vlv_enable_pll(struct intel_crtc *crtc, ++ const struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ ++ assert_pipe_disabled(dev_priv, pipe); ++ ++ /* PLL is protected by panel, make sure we can write it */ ++ assert_panel_unlocked(dev_priv, pipe); ++ ++ if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) ++ _vlv_enable_pll(crtc, pipe_config); ++ ++ I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md); ++ POSTING_READ(DPLL_MD(pipe)); ++} ++ ++ ++static void _chv_enable_pll(struct intel_crtc *crtc, ++ const struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ enum dpio_channel port = vlv_pipe_to_channel(pipe); ++ u32 tmp; ++ ++ mutex_lock(&dev_priv->sb_lock); ++ ++ /* Enable back the 10bit clock to display controller */ ++ tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)); ++ tmp |= DPIO_DCLKP_EN; ++ vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), tmp); ++ ++ mutex_unlock(&dev_priv->sb_lock); ++ ++ /* ++ * Need to wait > 100ns between dclkp clock enable bit and PLL enable. ++ */ ++ udelay(1); ++ ++ /* Enable PLL */ ++ I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll); ++ ++ /* Check PLL is locked */ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ DPLL(pipe), DPLL_LOCK_VLV, DPLL_LOCK_VLV, ++ 1)) ++ DRM_ERROR("PLL %d failed to lock\n", pipe); ++} ++ ++static void chv_enable_pll(struct intel_crtc *crtc, ++ const struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ ++ assert_pipe_disabled(dev_priv, pipe); ++ ++ /* PLL is protected by panel, make sure we can write it */ ++ assert_panel_unlocked(dev_priv, pipe); ++ ++ if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) ++ _chv_enable_pll(crtc, pipe_config); ++ ++ if (pipe != PIPE_A) { ++ /* ++ * WaPixelRepeatModeFixForC0:chv ++ * ++ * DPLLCMD is AWOL. Use chicken bits to propagate ++ * the value from DPLLBMD to either pipe B or C. ++ */ ++ I915_WRITE(CBR4_VLV, CBR_DPLLBMD_PIPE(pipe)); ++ I915_WRITE(DPLL_MD(PIPE_B), pipe_config->dpll_hw_state.dpll_md); ++ I915_WRITE(CBR4_VLV, 0); ++ dev_priv->chv_dpll_md[pipe] = pipe_config->dpll_hw_state.dpll_md; ++ ++ /* ++ * DPLLB VGA mode also seems to cause problems. ++ * We should always have it disabled. ++ */ ++ WARN_ON((I915_READ(DPLL(PIPE_B)) & DPLL_VGA_MODE_DIS) == 0); ++ } else { ++ I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md); ++ POSTING_READ(DPLL_MD(pipe)); ++ } ++} ++ ++static bool i9xx_has_pps(struct drm_i915_private *dev_priv) ++{ ++ if (IS_I830(dev_priv)) ++ return false; ++ ++ return IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv); ++} ++ ++static void i9xx_enable_pll(struct intel_crtc *crtc, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ i915_reg_t reg = DPLL(crtc->pipe); ++ u32 dpll = crtc_state->dpll_hw_state.dpll; ++ int i; ++ ++ assert_pipe_disabled(dev_priv, crtc->pipe); ++ ++ /* PLL is protected by panel, make sure we can write it */ ++ if (i9xx_has_pps(dev_priv)) ++ assert_panel_unlocked(dev_priv, crtc->pipe); ++ ++ /* ++ * Apparently we need to have VGA mode enabled prior to changing ++ * the P1/P2 dividers. Otherwise the DPLL will keep using the old ++ * dividers, even though the register value does change. ++ */ ++ I915_WRITE(reg, dpll & ~DPLL_VGA_MODE_DIS); ++ I915_WRITE(reg, dpll); ++ ++ /* Wait for the clocks to stabilize. */ ++ POSTING_READ(reg); ++ udelay(150); ++ ++ if (INTEL_GEN(dev_priv) >= 4) { ++ I915_WRITE(DPLL_MD(crtc->pipe), ++ crtc_state->dpll_hw_state.dpll_md); ++ } else { ++ /* The pixel multiplier can only be updated once the ++ * DPLL is enabled and the clocks are stable. ++ * ++ * So write it again. ++ */ ++ I915_WRITE(reg, dpll); ++ } ++ ++ /* We do this three times for luck */ ++ for (i = 0; i < 3; i++) { ++ I915_WRITE(reg, dpll); ++ POSTING_READ(reg); ++ udelay(150); /* wait for warmup */ ++ } ++} ++ ++static void i9xx_disable_pll(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ ++ /* Don't disable pipe or pipe PLLs if needed */ ++ if (IS_I830(dev_priv)) ++ return; ++ ++ /* Make sure the pipe isn't still relying on us */ ++ assert_pipe_disabled(dev_priv, pipe); ++ ++ I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS); ++ POSTING_READ(DPLL(pipe)); ++} ++ ++static void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe) ++{ ++ u32 val; ++ ++ /* Make sure the pipe isn't still relying on us */ ++ assert_pipe_disabled(dev_priv, pipe); ++ ++ val = DPLL_INTEGRATED_REF_CLK_VLV | ++ DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS; ++ if (pipe != PIPE_A) ++ val |= DPLL_INTEGRATED_CRI_CLK_VLV; ++ ++ I915_WRITE(DPLL(pipe), val); ++ POSTING_READ(DPLL(pipe)); ++} ++ ++static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe) ++{ ++ enum dpio_channel port = vlv_pipe_to_channel(pipe); ++ u32 val; ++ ++ /* Make sure the pipe isn't still relying on us */ ++ assert_pipe_disabled(dev_priv, pipe); ++ ++ val = DPLL_SSC_REF_CLK_CHV | ++ DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS; ++ if (pipe != PIPE_A) ++ val |= DPLL_INTEGRATED_CRI_CLK_VLV; ++ ++ I915_WRITE(DPLL(pipe), val); ++ POSTING_READ(DPLL(pipe)); ++ ++ mutex_lock(&dev_priv->sb_lock); ++ ++ /* Disable 10bit clock to display controller */ ++ val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)); ++ val &= ~DPIO_DCLKP_EN; ++ vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), val); ++ ++ mutex_unlock(&dev_priv->sb_lock); ++} ++ ++void vlv_wait_port_ready(struct drm_i915_private *dev_priv, ++ struct intel_digital_port *dport, ++ unsigned int expected_mask) ++{ ++ u32 port_mask; ++ i915_reg_t dpll_reg; ++ ++ switch (dport->base.port) { ++ case PORT_B: ++ port_mask = DPLL_PORTB_READY_MASK; ++ dpll_reg = DPLL(0); ++ break; ++ case PORT_C: ++ port_mask = DPLL_PORTC_READY_MASK; ++ dpll_reg = DPLL(0); ++ expected_mask <<= 4; ++ break; ++ case PORT_D: ++ port_mask = DPLL_PORTD_READY_MASK; ++ dpll_reg = DPIO_PHY_STATUS; ++ break; ++ default: ++ BUG(); ++ } ++ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ dpll_reg, port_mask, expected_mask, ++ 1000)) ++ WARN(1, "timed out waiting for port %c ready: got 0x%x, expected 0x%x\n", ++ port_name(dport->base.port), ++ I915_READ(dpll_reg) & port_mask, expected_mask); ++} ++ ++static void ironlake_enable_pch_transcoder(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ i915_reg_t reg; ++ u32 val, pipeconf_val; ++ ++ /* Make sure PCH DPLL is enabled */ ++ assert_shared_dpll_enabled(dev_priv, crtc_state->shared_dpll); ++ ++ /* FDI must be feeding us bits for PCH ports */ ++ assert_fdi_tx_enabled(dev_priv, pipe); ++ assert_fdi_rx_enabled(dev_priv, pipe); ++ ++ if (HAS_PCH_CPT(dev_priv)) { ++ /* Workaround: Set the timing override bit before enabling the ++ * pch transcoder. */ ++ reg = TRANS_CHICKEN2(pipe); ++ val = I915_READ(reg); ++ val |= TRANS_CHICKEN2_TIMING_OVERRIDE; ++ I915_WRITE(reg, val); ++ } ++ ++ reg = PCH_TRANSCONF(pipe); ++ val = I915_READ(reg); ++ pipeconf_val = I915_READ(PIPECONF(pipe)); ++ ++ if (HAS_PCH_IBX(dev_priv)) { ++ /* ++ * Make the BPC in transcoder be consistent with ++ * that in pipeconf reg. For HDMI we must use 8bpc ++ * here for both 8bpc and 12bpc. ++ */ ++ val &= ~PIPECONF_BPC_MASK; ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) ++ val |= PIPECONF_8BPC; ++ else ++ val |= pipeconf_val & PIPECONF_BPC_MASK; ++ } ++ ++ val &= ~TRANS_INTERLACE_MASK; ++ if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK) { ++ if (HAS_PCH_IBX(dev_priv) && ++ intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO)) ++ val |= TRANS_LEGACY_INTERLACED_ILK; ++ else ++ val |= TRANS_INTERLACED; ++ } else { ++ val |= TRANS_PROGRESSIVE; ++ } ++ ++ I915_WRITE(reg, val | TRANS_ENABLE); ++ if (intel_wait_for_register(&dev_priv->uncore, ++ reg, TRANS_STATE_ENABLE, TRANS_STATE_ENABLE, ++ 100)) ++ DRM_ERROR("failed to enable transcoder %c\n", pipe_name(pipe)); ++} ++ ++static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv, ++ enum transcoder cpu_transcoder) ++{ ++ u32 val, pipeconf_val; ++ ++ /* FDI must be feeding us bits for PCH ports */ ++ assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder); ++ assert_fdi_rx_enabled(dev_priv, PIPE_A); ++ ++ /* Workaround: set timing override bit. */ ++ val = I915_READ(TRANS_CHICKEN2(PIPE_A)); ++ val |= TRANS_CHICKEN2_TIMING_OVERRIDE; ++ I915_WRITE(TRANS_CHICKEN2(PIPE_A), val); ++ ++ val = TRANS_ENABLE; ++ pipeconf_val = I915_READ(PIPECONF(cpu_transcoder)); ++ ++ if ((pipeconf_val & PIPECONF_INTERLACE_MASK_HSW) == ++ PIPECONF_INTERLACED_ILK) ++ val |= TRANS_INTERLACED; ++ else ++ val |= TRANS_PROGRESSIVE; ++ ++ I915_WRITE(LPT_TRANSCONF, val); ++ if (intel_wait_for_register(&dev_priv->uncore, ++ LPT_TRANSCONF, ++ TRANS_STATE_ENABLE, ++ TRANS_STATE_ENABLE, ++ 100)) ++ DRM_ERROR("Failed to enable PCH transcoder\n"); ++} ++ ++static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv, ++ enum pipe pipe) ++{ ++ i915_reg_t reg; ++ u32 val; ++ ++ /* FDI relies on the transcoder */ ++ assert_fdi_tx_disabled(dev_priv, pipe); ++ assert_fdi_rx_disabled(dev_priv, pipe); ++ ++ /* Ports must be off as well */ ++ assert_pch_ports_disabled(dev_priv, pipe); ++ ++ reg = PCH_TRANSCONF(pipe); ++ val = I915_READ(reg); ++ val &= ~TRANS_ENABLE; ++ I915_WRITE(reg, val); ++ /* wait for PCH transcoder off, transcoder state */ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ reg, TRANS_STATE_ENABLE, 0, ++ 50)) ++ DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe)); ++ ++ if (HAS_PCH_CPT(dev_priv)) { ++ /* Workaround: Clear the timing override chicken bit again. */ ++ reg = TRANS_CHICKEN2(pipe); ++ val = I915_READ(reg); ++ val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE; ++ I915_WRITE(reg, val); ++ } ++} ++ ++void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv) ++{ ++ u32 val; ++ ++ val = I915_READ(LPT_TRANSCONF); ++ val &= ~TRANS_ENABLE; ++ I915_WRITE(LPT_TRANSCONF, val); ++ /* wait for PCH transcoder off, transcoder state */ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ LPT_TRANSCONF, TRANS_STATE_ENABLE, 0, ++ 50)) ++ DRM_ERROR("Failed to disable PCH transcoder\n"); ++ ++ /* Workaround: clear timing override bit. */ ++ val = I915_READ(TRANS_CHICKEN2(PIPE_A)); ++ val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE; ++ I915_WRITE(TRANS_CHICKEN2(PIPE_A), val); ++} ++ ++enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ ++ if (HAS_PCH_LPT(dev_priv)) ++ return PIPE_A; ++ else ++ return crtc->pipe; ++} ++ ++static u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); ++ ++ /* ++ * On i965gm the hardware frame counter reads ++ * zero when the TV encoder is enabled :( ++ */ ++ if (IS_I965GM(dev_priv) && ++ (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT))) ++ return 0; ++ ++ if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv)) ++ return 0xffffffff; /* full 32 bit counter */ ++ else if (INTEL_GEN(dev_priv) >= 3) ++ return 0xffffff; /* only 24 bits of frame count */ ++ else ++ return 0; /* Gen2 doesn't have a hardware frame counter */ ++} ++ ++static void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ ++ drm_crtc_set_max_vblank_count(&crtc->base, ++ intel_crtc_max_vblank_count(crtc_state)); ++ drm_crtc_vblank_on(&crtc->base); ++} ++ ++static void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder; ++ enum pipe pipe = crtc->pipe; ++ i915_reg_t reg; ++ u32 val; ++ ++ DRM_DEBUG_KMS("enabling pipe %c\n", pipe_name(pipe)); ++ ++ assert_planes_disabled(crtc); ++ ++ /* ++ * A pipe without a PLL won't actually be able to drive bits from ++ * a plane. On ILK+ the pipe PLLs are integrated, so we don't ++ * need the check. ++ */ ++ if (HAS_GMCH(dev_priv)) { ++ if (intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI)) ++ assert_dsi_pll_enabled(dev_priv); ++ else ++ assert_pll_enabled(dev_priv, pipe); ++ } else { ++ if (new_crtc_state->has_pch_encoder) { ++ /* if driving the PCH, we need FDI enabled */ ++ assert_fdi_rx_pll_enabled(dev_priv, ++ intel_crtc_pch_transcoder(crtc)); ++ assert_fdi_tx_pll_enabled(dev_priv, ++ (enum pipe) cpu_transcoder); ++ } ++ /* FIXME: assert CPU port conditions for SNB+ */ ++ } ++ ++ trace_intel_pipe_enable(dev_priv, pipe); ++ ++ reg = PIPECONF(cpu_transcoder); ++ val = I915_READ(reg); ++ if (val & PIPECONF_ENABLE) { ++ /* we keep both pipes enabled on 830 */ ++ WARN_ON(!IS_I830(dev_priv)); ++ return; ++ } ++ ++ I915_WRITE(reg, val | PIPECONF_ENABLE); ++ POSTING_READ(reg); ++ ++ /* ++ * Until the pipe starts PIPEDSL reads will return a stale value, ++ * which causes an apparent vblank timestamp jump when PIPEDSL ++ * resets to its proper value. That also messes up the frame count ++ * when it's derived from the timestamps. So let's wait for the ++ * pipe to start properly before we call drm_crtc_vblank_on() ++ */ ++ if (intel_crtc_max_vblank_count(new_crtc_state) == 0) ++ intel_wait_for_pipe_scanline_moving(crtc); ++} ++ ++static void intel_disable_pipe(const struct intel_crtc_state *old_crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder; ++ enum pipe pipe = crtc->pipe; ++ i915_reg_t reg; ++ u32 val; ++ ++ DRM_DEBUG_KMS("disabling pipe %c\n", pipe_name(pipe)); ++ ++ /* ++ * Make sure planes won't keep trying to pump pixels to us, ++ * or we might hang the display. ++ */ ++ assert_planes_disabled(crtc); ++ ++ trace_intel_pipe_disable(dev_priv, pipe); ++ ++ reg = PIPECONF(cpu_transcoder); ++ val = I915_READ(reg); ++ if ((val & PIPECONF_ENABLE) == 0) ++ return; ++ ++ /* ++ * Double wide has implications for planes ++ * so best keep it disabled when not needed. ++ */ ++ if (old_crtc_state->double_wide) ++ val &= ~PIPECONF_DOUBLE_WIDE; ++ ++ /* Don't disable pipe or pipe PLLs if needed */ ++ if (!IS_I830(dev_priv)) ++ val &= ~PIPECONF_ENABLE; ++ ++ I915_WRITE(reg, val); ++ if ((val & PIPECONF_ENABLE) == 0) ++ intel_wait_for_pipe_off(old_crtc_state); ++} ++ ++static unsigned int intel_tile_size(const struct drm_i915_private *dev_priv) ++{ ++ return IS_GEN(dev_priv, 2) ? 2048 : 4096; ++} ++ ++static unsigned int ++intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane) ++{ ++ struct drm_i915_private *dev_priv = to_i915(fb->dev); ++ unsigned int cpp = fb->format->cpp[color_plane]; ++ ++ switch (fb->modifier) { ++ case DRM_FORMAT_MOD_LINEAR: ++ return cpp; ++ case I915_FORMAT_MOD_X_TILED: ++ if (IS_GEN(dev_priv, 2)) ++ return 128; ++ else ++ return 512; ++ case I915_FORMAT_MOD_Y_TILED_CCS: ++ if (color_plane == 1) ++ return 128; ++ /* fall through */ ++ case I915_FORMAT_MOD_Y_TILED: ++ if (IS_GEN(dev_priv, 2) || HAS_128_BYTE_Y_TILING(dev_priv)) ++ return 128; ++ else ++ return 512; ++ case I915_FORMAT_MOD_Yf_TILED_CCS: ++ if (color_plane == 1) ++ return 128; ++ /* fall through */ ++ case I915_FORMAT_MOD_Yf_TILED: ++ switch (cpp) { ++ case 1: ++ return 64; ++ case 2: ++ case 4: ++ return 128; ++ case 8: ++ case 16: ++ return 256; ++ default: ++ MISSING_CASE(cpp); ++ return cpp; ++ } ++ break; ++ default: ++ MISSING_CASE(fb->modifier); ++ return cpp; ++ } ++} ++ ++static unsigned int ++intel_tile_height(const struct drm_framebuffer *fb, int color_plane) ++{ ++ if (fb->modifier == DRM_FORMAT_MOD_LINEAR) ++ return 1; ++ else ++ return intel_tile_size(to_i915(fb->dev)) / ++ intel_tile_width_bytes(fb, color_plane); ++} ++ ++/* Return the tile dimensions in pixel units */ ++static void intel_tile_dims(const struct drm_framebuffer *fb, int color_plane, ++ unsigned int *tile_width, ++ unsigned int *tile_height) ++{ ++ unsigned int tile_width_bytes = intel_tile_width_bytes(fb, color_plane); ++ unsigned int cpp = fb->format->cpp[color_plane]; ++ ++ *tile_width = tile_width_bytes / cpp; ++ *tile_height = intel_tile_size(to_i915(fb->dev)) / tile_width_bytes; ++} ++ ++unsigned int ++intel_fb_align_height(const struct drm_framebuffer *fb, ++ int color_plane, unsigned int height) ++{ ++ unsigned int tile_height = intel_tile_height(fb, color_plane); ++ ++ return ALIGN(height, tile_height); ++} ++ ++unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info) ++{ ++ unsigned int size = 0; ++ int i; ++ ++ for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++) ++ size += rot_info->plane[i].width * rot_info->plane[i].height; ++ ++ return size; ++} ++ ++static void ++intel_fill_fb_ggtt_view(struct i915_ggtt_view *view, ++ const struct drm_framebuffer *fb, ++ unsigned int rotation) ++{ ++ view->type = I915_GGTT_VIEW_NORMAL; ++ if (drm_rotation_90_or_270(rotation)) { ++ view->type = I915_GGTT_VIEW_ROTATED; ++ view->rotated = to_intel_framebuffer(fb)->rot_info; ++ } ++} ++ ++static unsigned int intel_cursor_alignment(const struct drm_i915_private *dev_priv) ++{ ++ if (IS_I830(dev_priv)) ++ return 16 * 1024; ++ else if (IS_I85X(dev_priv)) ++ return 256; ++ else if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) ++ return 32; ++ else ++ return 4 * 1024; ++} ++ ++static unsigned int intel_linear_alignment(const struct drm_i915_private *dev_priv) ++{ ++ if (INTEL_GEN(dev_priv) >= 9) ++ return 256 * 1024; ++ else if (IS_I965G(dev_priv) || IS_I965GM(dev_priv) || ++ IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ return 128 * 1024; ++ else if (INTEL_GEN(dev_priv) >= 4) ++ return 4 * 1024; ++ else ++ return 0; ++} ++ ++static unsigned int intel_surf_alignment(const struct drm_framebuffer *fb, ++ int color_plane) ++{ ++ struct drm_i915_private *dev_priv = to_i915(fb->dev); ++ ++ /* AUX_DIST needs only 4K alignment */ ++ if (color_plane == 1) ++ return 4096; ++ ++ switch (fb->modifier) { ++ case DRM_FORMAT_MOD_LINEAR: ++ return intel_linear_alignment(dev_priv); ++ case I915_FORMAT_MOD_X_TILED: ++ if (INTEL_GEN(dev_priv) >= 9) ++ return 256 * 1024; ++ return 0; ++ case I915_FORMAT_MOD_Y_TILED_CCS: ++ case I915_FORMAT_MOD_Yf_TILED_CCS: ++ case I915_FORMAT_MOD_Y_TILED: ++ case I915_FORMAT_MOD_Yf_TILED: ++ return 1 * 1024 * 1024; ++ default: ++ MISSING_CASE(fb->modifier); ++ return 0; ++ } ++} ++ ++static bool intel_plane_uses_fence(const struct intel_plane_state *plane_state) ++{ ++ struct intel_plane *plane = to_intel_plane(plane_state->base.plane); ++ struct drm_i915_private *dev_priv = to_i915(plane->base.dev); ++ ++ return INTEL_GEN(dev_priv) < 4 || plane->has_fbc; ++} ++ ++struct i915_vma * ++intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, ++ const struct i915_ggtt_view *view, ++ bool uses_fence, ++ unsigned long *out_flags) ++{ ++ struct drm_device *dev = fb->dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct drm_i915_gem_object *obj = intel_fb_obj(fb); ++ intel_wakeref_t wakeref; ++ struct i915_vma *vma; ++ unsigned int pinctl; ++ u32 alignment; ++ ++ WARN_ON(!mutex_is_locked(&dev->struct_mutex)); ++ ++ alignment = intel_surf_alignment(fb, 0); ++ ++ /* Note that the w/a also requires 64 PTE of padding following the ++ * bo. We currently fill all unused PTE with the shadow page and so ++ * we should always have valid PTE following the scanout preventing ++ * the VT-d warning. ++ */ ++ if (intel_scanout_needs_vtd_wa(dev_priv) && alignment < 256 * 1024) ++ alignment = 256 * 1024; ++ ++ /* ++ * Global gtt pte registers are special registers which actually forward ++ * writes to a chunk of system memory. Which means that there is no risk ++ * that the register values disappear as soon as we call ++ * intel_runtime_pm_put(), so it is correct to wrap only the ++ * pin/unpin/fence and not more. ++ */ ++ wakeref = intel_runtime_pm_get(dev_priv); ++ ++ atomic_inc(&dev_priv->gpu_error.pending_fb_pin); ++ ++ pinctl = 0; ++ ++ /* Valleyview is definitely limited to scanning out the first ++ * 512MiB. Lets presume this behaviour was inherited from the ++ * g4x display engine and that all earlier gen are similarly ++ * limited. Testing suggests that it is a little more ++ * complicated than this. For example, Cherryview appears quite ++ * happy to scanout from anywhere within its global aperture. ++ */ ++ if (HAS_GMCH(dev_priv)) ++ pinctl |= PIN_MAPPABLE; ++ ++ vma = i915_gem_object_pin_to_display_plane(obj, ++ alignment, view, pinctl); ++ if (IS_ERR(vma)) ++ goto err; ++ ++ if (uses_fence && i915_vma_is_map_and_fenceable(vma)) { ++ int ret; ++ ++ /* Install a fence for tiled scan-out. Pre-i965 always needs a ++ * fence, whereas 965+ only requires a fence if using ++ * framebuffer compression. For simplicity, we always, when ++ * possible, install a fence as the cost is not that onerous. ++ * ++ * If we fail to fence the tiled scanout, then either the ++ * modeset will reject the change (which is highly unlikely as ++ * the affected systems, all but one, do not have unmappable ++ * space) or we will not be able to enable full powersaving ++ * techniques (also likely not to apply due to various limits ++ * FBC and the like impose on the size of the buffer, which ++ * presumably we violated anyway with this unmappable buffer). ++ * Anyway, it is presumably better to stumble onwards with ++ * something and try to run the system in a "less than optimal" ++ * mode that matches the user configuration. ++ */ ++ ret = i915_vma_pin_fence(vma); ++ if (ret != 0 && INTEL_GEN(dev_priv) < 4) { ++ i915_gem_object_unpin_from_display_plane(vma); ++ vma = ERR_PTR(ret); ++ goto err; ++ } ++ ++ if (ret == 0 && vma->fence) ++ *out_flags |= PLANE_HAS_FENCE; ++ } ++ ++ i915_vma_get(vma); ++err: ++ atomic_dec(&dev_priv->gpu_error.pending_fb_pin); ++ ++ intel_runtime_pm_put(dev_priv, wakeref); ++ return vma; ++} ++ ++void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags) ++{ ++ lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); ++ ++ if (flags & PLANE_HAS_FENCE) ++ i915_vma_unpin_fence(vma); ++ i915_gem_object_unpin_from_display_plane(vma); ++ i915_vma_put(vma); ++} ++ ++static int intel_fb_pitch(const struct drm_framebuffer *fb, int color_plane, ++ unsigned int rotation) ++{ ++ if (drm_rotation_90_or_270(rotation)) ++ return to_intel_framebuffer(fb)->rotated[color_plane].pitch; ++ else ++ return fb->pitches[color_plane]; ++} ++ ++/* ++ * Convert the x/y offsets into a linear offset. ++ * Only valid with 0/180 degree rotation, which is fine since linear ++ * offset is only used with linear buffers on pre-hsw and tiled buffers ++ * with gen2/3, and 90/270 degree rotations isn't supported on any of them. ++ */ ++u32 intel_fb_xy_to_linear(int x, int y, ++ const struct intel_plane_state *state, ++ int color_plane) ++{ ++ const struct drm_framebuffer *fb = state->base.fb; ++ unsigned int cpp = fb->format->cpp[color_plane]; ++ unsigned int pitch = state->color_plane[color_plane].stride; ++ ++ return y * pitch + x * cpp; ++} ++ ++/* ++ * Add the x/y offsets derived from fb->offsets[] to the user ++ * specified plane src x/y offsets. The resulting x/y offsets ++ * specify the start of scanout from the beginning of the gtt mapping. ++ */ ++void intel_add_fb_offsets(int *x, int *y, ++ const struct intel_plane_state *state, ++ int color_plane) ++ ++{ ++ const struct intel_framebuffer *intel_fb = to_intel_framebuffer(state->base.fb); ++ unsigned int rotation = state->base.rotation; ++ ++ if (drm_rotation_90_or_270(rotation)) { ++ *x += intel_fb->rotated[color_plane].x; ++ *y += intel_fb->rotated[color_plane].y; ++ } else { ++ *x += intel_fb->normal[color_plane].x; ++ *y += intel_fb->normal[color_plane].y; ++ } ++} ++ ++static u32 intel_adjust_tile_offset(int *x, int *y, ++ unsigned int tile_width, ++ unsigned int tile_height, ++ unsigned int tile_size, ++ unsigned int pitch_tiles, ++ u32 old_offset, ++ u32 new_offset) ++{ ++ unsigned int pitch_pixels = pitch_tiles * tile_width; ++ unsigned int tiles; ++ ++ WARN_ON(old_offset & (tile_size - 1)); ++ WARN_ON(new_offset & (tile_size - 1)); ++ WARN_ON(new_offset > old_offset); ++ ++ tiles = (old_offset - new_offset) / tile_size; ++ ++ *y += tiles / pitch_tiles * tile_height; ++ *x += tiles % pitch_tiles * tile_width; ++ ++ /* minimize x in case it got needlessly big */ ++ *y += *x / pitch_pixels * tile_height; ++ *x %= pitch_pixels; ++ ++ return new_offset; ++} ++ ++static bool is_surface_linear(u64 modifier, int color_plane) ++{ ++ return modifier == DRM_FORMAT_MOD_LINEAR; ++} ++ ++static u32 intel_adjust_aligned_offset(int *x, int *y, ++ const struct drm_framebuffer *fb, ++ int color_plane, ++ unsigned int rotation, ++ unsigned int pitch, ++ u32 old_offset, u32 new_offset) ++{ ++ struct drm_i915_private *dev_priv = to_i915(fb->dev); ++ unsigned int cpp = fb->format->cpp[color_plane]; ++ ++ WARN_ON(new_offset > old_offset); ++ ++ if (!is_surface_linear(fb->modifier, color_plane)) { ++ unsigned int tile_size, tile_width, tile_height; ++ unsigned int pitch_tiles; ++ ++ tile_size = intel_tile_size(dev_priv); ++ intel_tile_dims(fb, color_plane, &tile_width, &tile_height); ++ ++ if (drm_rotation_90_or_270(rotation)) { ++ pitch_tiles = pitch / tile_height; ++ swap(tile_width, tile_height); ++ } else { ++ pitch_tiles = pitch / (tile_width * cpp); ++ } ++ ++ intel_adjust_tile_offset(x, y, tile_width, tile_height, ++ tile_size, pitch_tiles, ++ old_offset, new_offset); ++ } else { ++ old_offset += *y * pitch + *x * cpp; ++ ++ *y = (old_offset - new_offset) / pitch; ++ *x = ((old_offset - new_offset) - *y * pitch) / cpp; ++ } ++ ++ return new_offset; ++} ++ ++/* ++ * Adjust the tile offset by moving the difference into ++ * the x/y offsets. ++ */ ++static u32 intel_plane_adjust_aligned_offset(int *x, int *y, ++ const struct intel_plane_state *state, ++ int color_plane, ++ u32 old_offset, u32 new_offset) ++{ ++ return intel_adjust_aligned_offset(x, y, state->base.fb, color_plane, ++ state->base.rotation, ++ state->color_plane[color_plane].stride, ++ old_offset, new_offset); ++} ++ ++/* ++ * Computes the aligned offset to the base tile and adjusts ++ * x, y. bytes per pixel is assumed to be a power-of-two. ++ * ++ * In the 90/270 rotated case, x and y are assumed ++ * to be already rotated to match the rotated GTT view, and ++ * pitch is the tile_height aligned framebuffer height. ++ * ++ * This function is used when computing the derived information ++ * under intel_framebuffer, so using any of that information ++ * here is not allowed. Anything under drm_framebuffer can be ++ * used. This is why the user has to pass in the pitch since it ++ * is specified in the rotated orientation. ++ */ ++static u32 intel_compute_aligned_offset(struct drm_i915_private *dev_priv, ++ int *x, int *y, ++ const struct drm_framebuffer *fb, ++ int color_plane, ++ unsigned int pitch, ++ unsigned int rotation, ++ u32 alignment) ++{ ++ unsigned int cpp = fb->format->cpp[color_plane]; ++ u32 offset, offset_aligned; ++ ++ if (alignment) ++ alignment--; ++ ++ if (!is_surface_linear(fb->modifier, color_plane)) { ++ unsigned int tile_size, tile_width, tile_height; ++ unsigned int tile_rows, tiles, pitch_tiles; ++ ++ tile_size = intel_tile_size(dev_priv); ++ intel_tile_dims(fb, color_plane, &tile_width, &tile_height); ++ ++ if (drm_rotation_90_or_270(rotation)) { ++ pitch_tiles = pitch / tile_height; ++ swap(tile_width, tile_height); ++ } else { ++ pitch_tiles = pitch / (tile_width * cpp); ++ } ++ ++ tile_rows = *y / tile_height; ++ *y %= tile_height; ++ ++ tiles = *x / tile_width; ++ *x %= tile_width; ++ ++ offset = (tile_rows * pitch_tiles + tiles) * tile_size; ++ offset_aligned = offset & ~alignment; ++ ++ intel_adjust_tile_offset(x, y, tile_width, tile_height, ++ tile_size, pitch_tiles, ++ offset, offset_aligned); ++ } else { ++ offset = *y * pitch + *x * cpp; ++ offset_aligned = offset & ~alignment; ++ ++ *y = (offset & alignment) / pitch; ++ *x = ((offset & alignment) - *y * pitch) / cpp; ++ } ++ ++ return offset_aligned; ++} ++ ++static u32 intel_plane_compute_aligned_offset(int *x, int *y, ++ const struct intel_plane_state *state, ++ int color_plane) ++{ ++ struct intel_plane *intel_plane = to_intel_plane(state->base.plane); ++ struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev); ++ const struct drm_framebuffer *fb = state->base.fb; ++ unsigned int rotation = state->base.rotation; ++ int pitch = state->color_plane[color_plane].stride; ++ u32 alignment; ++ ++ if (intel_plane->id == PLANE_CURSOR) ++ alignment = intel_cursor_alignment(dev_priv); ++ else ++ alignment = intel_surf_alignment(fb, color_plane); ++ ++ return intel_compute_aligned_offset(dev_priv, x, y, fb, color_plane, ++ pitch, rotation, alignment); ++} ++ ++/* Convert the fb->offset[] into x/y offsets */ ++static int intel_fb_offset_to_xy(int *x, int *y, ++ const struct drm_framebuffer *fb, ++ int color_plane) ++{ ++ struct drm_i915_private *dev_priv = to_i915(fb->dev); ++ unsigned int height; ++ ++ if (fb->modifier != DRM_FORMAT_MOD_LINEAR && ++ fb->offsets[color_plane] % intel_tile_size(dev_priv)) { ++ DRM_DEBUG_KMS("Misaligned offset 0x%08x for color plane %d\n", ++ fb->offsets[color_plane], color_plane); ++ return -EINVAL; ++ } ++ ++ height = drm_framebuffer_plane_height(fb->height, fb, color_plane); ++ height = ALIGN(height, intel_tile_height(fb, color_plane)); ++ ++ /* Catch potential overflows early */ ++ if (add_overflows_t(u32, mul_u32_u32(height, fb->pitches[color_plane]), ++ fb->offsets[color_plane])) { ++ DRM_DEBUG_KMS("Bad offset 0x%08x or pitch %d for color plane %d\n", ++ fb->offsets[color_plane], fb->pitches[color_plane], ++ color_plane); ++ return -ERANGE; ++ } ++ ++ *x = 0; ++ *y = 0; ++ ++ intel_adjust_aligned_offset(x, y, ++ fb, color_plane, DRM_MODE_ROTATE_0, ++ fb->pitches[color_plane], ++ fb->offsets[color_plane], 0); ++ ++ return 0; ++} ++ ++static unsigned int intel_fb_modifier_to_tiling(u64 fb_modifier) ++{ ++ switch (fb_modifier) { ++ case I915_FORMAT_MOD_X_TILED: ++ return I915_TILING_X; ++ case I915_FORMAT_MOD_Y_TILED: ++ case I915_FORMAT_MOD_Y_TILED_CCS: ++ return I915_TILING_Y; ++ default: ++ return I915_TILING_NONE; ++ } ++} ++ ++/* ++ * From the Sky Lake PRM: ++ * "The Color Control Surface (CCS) contains the compression status of ++ * the cache-line pairs. The compression state of the cache-line pair ++ * is specified by 2 bits in the CCS. Each CCS cache-line represents ++ * an area on the main surface of 16 x16 sets of 128 byte Y-tiled ++ * cache-line-pairs. CCS is always Y tiled." ++ * ++ * Since cache line pairs refers to horizontally adjacent cache lines, ++ * each cache line in the CCS corresponds to an area of 32x16 cache ++ * lines on the main surface. Since each pixel is 4 bytes, this gives ++ * us a ratio of one byte in the CCS for each 8x16 pixels in the ++ * main surface. ++ */ ++static const struct drm_format_info ccs_formats[] = { ++ { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2, ++ .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, }, ++ { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2, ++ .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, }, ++ { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2, ++ .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, }, ++ { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2, ++ .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, }, ++}; ++ ++static const struct drm_format_info * ++lookup_format_info(const struct drm_format_info formats[], ++ int num_formats, u32 format) ++{ ++ int i; ++ ++ for (i = 0; i < num_formats; i++) { ++ if (formats[i].format == format) ++ return &formats[i]; ++ } ++ ++ return NULL; ++} ++ ++static const struct drm_format_info * ++intel_get_format_info(const struct drm_mode_fb_cmd2 *cmd) ++{ ++ switch (cmd->modifier[0]) { ++ case I915_FORMAT_MOD_Y_TILED_CCS: ++ case I915_FORMAT_MOD_Yf_TILED_CCS: ++ return lookup_format_info(ccs_formats, ++ ARRAY_SIZE(ccs_formats), ++ cmd->pixel_format); ++ default: ++ return NULL; ++ } ++} ++ ++bool is_ccs_modifier(u64 modifier) ++{ ++ return modifier == I915_FORMAT_MOD_Y_TILED_CCS || ++ modifier == I915_FORMAT_MOD_Yf_TILED_CCS; ++} ++ ++static int ++intel_fill_fb_info(struct drm_i915_private *dev_priv, ++ struct drm_framebuffer *fb) ++{ ++ struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); ++ struct intel_rotation_info *rot_info = &intel_fb->rot_info; ++ struct drm_i915_gem_object *obj = intel_fb_obj(fb); ++ u32 gtt_offset_rotated = 0; ++ unsigned int max_size = 0; ++ int i, num_planes = fb->format->num_planes; ++ unsigned int tile_size = intel_tile_size(dev_priv); ++ ++ for (i = 0; i < num_planes; i++) { ++ unsigned int width, height; ++ unsigned int cpp, size; ++ u32 offset; ++ int x, y; ++ int ret; ++ ++ cpp = fb->format->cpp[i]; ++ width = drm_framebuffer_plane_width(fb->width, fb, i); ++ height = drm_framebuffer_plane_height(fb->height, fb, i); ++ ++ ret = intel_fb_offset_to_xy(&x, &y, fb, i); ++ if (ret) { ++ DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n", ++ i, fb->offsets[i]); ++ return ret; ++ } ++ ++ if (is_ccs_modifier(fb->modifier) && i == 1) { ++ int hsub = fb->format->hsub; ++ int vsub = fb->format->vsub; ++ int tile_width, tile_height; ++ int main_x, main_y; ++ int ccs_x, ccs_y; ++ ++ intel_tile_dims(fb, i, &tile_width, &tile_height); ++ tile_width *= hsub; ++ tile_height *= vsub; ++ ++ ccs_x = (x * hsub) % tile_width; ++ ccs_y = (y * vsub) % tile_height; ++ main_x = intel_fb->normal[0].x % tile_width; ++ main_y = intel_fb->normal[0].y % tile_height; ++ ++ /* ++ * CCS doesn't have its own x/y offset register, so the intra CCS tile ++ * x/y offsets must match between CCS and the main surface. ++ */ ++ if (main_x != ccs_x || main_y != ccs_y) { ++ DRM_DEBUG_KMS("Bad CCS x/y (main %d,%d ccs %d,%d) full (main %d,%d ccs %d,%d)\n", ++ main_x, main_y, ++ ccs_x, ccs_y, ++ intel_fb->normal[0].x, ++ intel_fb->normal[0].y, ++ x, y); ++ return -EINVAL; ++ } ++ } ++ ++ /* ++ * The fence (if used) is aligned to the start of the object ++ * so having the framebuffer wrap around across the edge of the ++ * fenced region doesn't really work. We have no API to configure ++ * the fence start offset within the object (nor could we probably ++ * on gen2/3). So it's just easier if we just require that the ++ * fb layout agrees with the fence layout. We already check that the ++ * fb stride matches the fence stride elsewhere. ++ */ ++ if (i == 0 && i915_gem_object_is_tiled(obj) && ++ (x + width) * cpp > fb->pitches[i]) { ++ DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n", ++ i, fb->offsets[i]); ++ return -EINVAL; ++ } ++ ++ /* ++ * First pixel of the framebuffer from ++ * the start of the normal gtt mapping. ++ */ ++ intel_fb->normal[i].x = x; ++ intel_fb->normal[i].y = y; ++ ++ offset = intel_compute_aligned_offset(dev_priv, &x, &y, fb, i, ++ fb->pitches[i], ++ DRM_MODE_ROTATE_0, ++ tile_size); ++ offset /= tile_size; ++ ++ if (!is_surface_linear(fb->modifier, i)) { ++ unsigned int tile_width, tile_height; ++ unsigned int pitch_tiles; ++ struct drm_rect r; ++ ++ intel_tile_dims(fb, i, &tile_width, &tile_height); ++ ++ rot_info->plane[i].offset = offset; ++ rot_info->plane[i].stride = DIV_ROUND_UP(fb->pitches[i], tile_width * cpp); ++ rot_info->plane[i].width = DIV_ROUND_UP(x + width, tile_width); ++ rot_info->plane[i].height = DIV_ROUND_UP(y + height, tile_height); ++ ++ intel_fb->rotated[i].pitch = ++ rot_info->plane[i].height * tile_height; ++ ++ /* how many tiles does this plane need */ ++ size = rot_info->plane[i].stride * rot_info->plane[i].height; ++ /* ++ * If the plane isn't horizontally tile aligned, ++ * we need one more tile. ++ */ ++ if (x != 0) ++ size++; ++ ++ /* rotate the x/y offsets to match the GTT view */ ++ r.x1 = x; ++ r.y1 = y; ++ r.x2 = x + width; ++ r.y2 = y + height; ++ drm_rect_rotate(&r, ++ rot_info->plane[i].width * tile_width, ++ rot_info->plane[i].height * tile_height, ++ DRM_MODE_ROTATE_270); ++ x = r.x1; ++ y = r.y1; ++ ++ /* rotate the tile dimensions to match the GTT view */ ++ pitch_tiles = intel_fb->rotated[i].pitch / tile_height; ++ swap(tile_width, tile_height); ++ ++ /* ++ * We only keep the x/y offsets, so push all of the ++ * gtt offset into the x/y offsets. ++ */ ++ intel_adjust_tile_offset(&x, &y, ++ tile_width, tile_height, ++ tile_size, pitch_tiles, ++ gtt_offset_rotated * tile_size, 0); ++ ++ gtt_offset_rotated += rot_info->plane[i].width * rot_info->plane[i].height; ++ ++ /* ++ * First pixel of the framebuffer from ++ * the start of the rotated gtt mapping. ++ */ ++ intel_fb->rotated[i].x = x; ++ intel_fb->rotated[i].y = y; ++ } else { ++ size = DIV_ROUND_UP((y + height) * fb->pitches[i] + ++ x * cpp, tile_size); ++ } ++ ++ /* how many tiles in total needed in the bo */ ++ max_size = max(max_size, offset + size); ++ } ++ ++ if (mul_u32_u32(max_size, tile_size) > obj->base.size) { ++ DRM_DEBUG_KMS("fb too big for bo (need %llu bytes, have %zu bytes)\n", ++ mul_u32_u32(max_size, tile_size), obj->base.size); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static int i9xx_format_to_fourcc(int format) ++{ ++ switch (format) { ++ case DISPPLANE_8BPP: ++ return DRM_FORMAT_C8; ++ case DISPPLANE_BGRX555: ++ return DRM_FORMAT_XRGB1555; ++ case DISPPLANE_BGRX565: ++ return DRM_FORMAT_RGB565; ++ default: ++ case DISPPLANE_BGRX888: ++ return DRM_FORMAT_XRGB8888; ++ case DISPPLANE_RGBX888: ++ return DRM_FORMAT_XBGR8888; ++ case DISPPLANE_BGRX101010: ++ return DRM_FORMAT_XRGB2101010; ++ case DISPPLANE_RGBX101010: ++ return DRM_FORMAT_XBGR2101010; ++ } ++} ++ ++int skl_format_to_fourcc(int format, bool rgb_order, bool alpha) ++{ ++ switch (format) { ++ case PLANE_CTL_FORMAT_RGB_565: ++ return DRM_FORMAT_RGB565; ++ case PLANE_CTL_FORMAT_NV12: ++ return DRM_FORMAT_NV12; ++ case PLANE_CTL_FORMAT_P010: ++ return DRM_FORMAT_P010; ++ case PLANE_CTL_FORMAT_P012: ++ return DRM_FORMAT_P012; ++ case PLANE_CTL_FORMAT_P016: ++ return DRM_FORMAT_P016; ++ case PLANE_CTL_FORMAT_Y210: ++ return DRM_FORMAT_Y210; ++ case PLANE_CTL_FORMAT_Y212: ++ return DRM_FORMAT_Y212; ++ case PLANE_CTL_FORMAT_Y216: ++ return DRM_FORMAT_Y216; ++ case PLANE_CTL_FORMAT_Y410: ++ return DRM_FORMAT_XVYU2101010; ++ case PLANE_CTL_FORMAT_Y412: ++ return DRM_FORMAT_XVYU12_16161616; ++ case PLANE_CTL_FORMAT_Y416: ++ return DRM_FORMAT_XVYU16161616; ++ default: ++ case PLANE_CTL_FORMAT_XRGB_8888: ++ if (rgb_order) { ++ if (alpha) ++ return DRM_FORMAT_ABGR8888; ++ else ++ return DRM_FORMAT_XBGR8888; ++ } else { ++ if (alpha) ++ return DRM_FORMAT_ARGB8888; ++ else ++ return DRM_FORMAT_XRGB8888; ++ } ++ case PLANE_CTL_FORMAT_XRGB_2101010: ++ if (rgb_order) ++ return DRM_FORMAT_XBGR2101010; ++ else ++ return DRM_FORMAT_XRGB2101010; ++ case PLANE_CTL_FORMAT_XRGB_16161616F: ++ if (rgb_order) { ++ if (alpha) ++ return DRM_FORMAT_ABGR16161616F; ++ else ++ return DRM_FORMAT_XBGR16161616F; ++ } else { ++ if (alpha) ++ return DRM_FORMAT_ARGB16161616F; ++ else ++ return DRM_FORMAT_XRGB16161616F; ++ } ++ } ++} ++ ++static bool ++intel_alloc_initial_plane_obj(struct intel_crtc *crtc, ++ struct intel_initial_plane_config *plane_config) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct drm_i915_gem_object *obj = NULL; ++ struct drm_mode_fb_cmd2 mode_cmd = { 0 }; ++ struct drm_framebuffer *fb = &plane_config->fb->base; ++ u32 base_aligned = round_down(plane_config->base, PAGE_SIZE); ++ u32 size_aligned = round_up(plane_config->base + plane_config->size, ++ PAGE_SIZE); ++ ++ size_aligned -= base_aligned; ++ ++ if (plane_config->size == 0) ++ return false; ++ ++ /* If the FB is too big, just don't use it since fbdev is not very ++ * important and we should probably use that space with FBC or other ++ * features. */ ++ if (size_aligned * 2 > dev_priv->stolen_usable_size) ++ return false; ++ ++ switch (fb->modifier) { ++ case DRM_FORMAT_MOD_LINEAR: ++ case I915_FORMAT_MOD_X_TILED: ++ case I915_FORMAT_MOD_Y_TILED: ++ break; ++ default: ++ DRM_DEBUG_DRIVER("Unsupported modifier for initial FB: 0x%llx\n", ++ fb->modifier); ++ return false; ++ } ++ ++ mutex_lock(&dev->struct_mutex); ++ obj = i915_gem_object_create_stolen_for_preallocated(dev_priv, ++ base_aligned, ++ base_aligned, ++ size_aligned); ++ mutex_unlock(&dev->struct_mutex); ++ if (!obj) ++ return false; ++ ++ switch (plane_config->tiling) { ++ case I915_TILING_NONE: ++ break; ++ case I915_TILING_X: ++ case I915_TILING_Y: ++ obj->tiling_and_stride = fb->pitches[0] | plane_config->tiling; ++ break; ++ default: ++ MISSING_CASE(plane_config->tiling); ++ return false; ++ } ++ ++ mode_cmd.pixel_format = fb->format->format; ++ mode_cmd.width = fb->width; ++ mode_cmd.height = fb->height; ++ mode_cmd.pitches[0] = fb->pitches[0]; ++ mode_cmd.modifier[0] = fb->modifier; ++ mode_cmd.flags = DRM_MODE_FB_MODIFIERS; ++ ++ if (intel_framebuffer_init(to_intel_framebuffer(fb), obj, &mode_cmd)) { ++ DRM_DEBUG_KMS("intel fb init failed\n"); ++ goto out_unref_obj; ++ } ++ ++ ++ DRM_DEBUG_KMS("initial plane fb obj %p\n", obj); ++ return true; ++ ++out_unref_obj: ++ i915_gem_object_put(obj); ++ return false; ++} ++ ++static void ++intel_set_plane_visible(struct intel_crtc_state *crtc_state, ++ struct intel_plane_state *plane_state, ++ bool visible) ++{ ++ struct intel_plane *plane = to_intel_plane(plane_state->base.plane); ++ ++ plane_state->base.visible = visible; ++ ++ if (visible) ++ crtc_state->base.plane_mask |= drm_plane_mask(&plane->base); ++ else ++ crtc_state->base.plane_mask &= ~drm_plane_mask(&plane->base); ++} ++ ++static void fixup_active_planes(struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); ++ struct drm_plane *plane; ++ ++ /* ++ * Active_planes aliases if multiple "primary" or cursor planes ++ * have been used on the same (or wrong) pipe. plane_mask uses ++ * unique ids, hence we can use that to reconstruct active_planes. ++ */ ++ crtc_state->active_planes = 0; ++ ++ drm_for_each_plane_mask(plane, &dev_priv->drm, ++ crtc_state->base.plane_mask) ++ crtc_state->active_planes |= BIT(to_intel_plane(plane)->id); ++} ++ ++static void intel_plane_disable_noatomic(struct intel_crtc *crtc, ++ struct intel_plane *plane) ++{ ++ struct intel_crtc_state *crtc_state = ++ to_intel_crtc_state(crtc->base.state); ++ struct intel_plane_state *plane_state = ++ to_intel_plane_state(plane->base.state); ++ ++ DRM_DEBUG_KMS("Disabling [PLANE:%d:%s] on [CRTC:%d:%s]\n", ++ plane->base.base.id, plane->base.name, ++ crtc->base.base.id, crtc->base.name); ++ ++ intel_set_plane_visible(crtc_state, plane_state, false); ++ fixup_active_planes(crtc_state); ++ ++ if (plane->id == PLANE_PRIMARY) ++ intel_pre_disable_primary_noatomic(&crtc->base); ++ ++ intel_disable_plane(plane, crtc_state); ++} ++ ++static void ++intel_find_initial_plane_obj(struct intel_crtc *intel_crtc, ++ struct intel_initial_plane_config *plane_config) ++{ ++ struct drm_device *dev = intel_crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct drm_crtc *c; ++ struct drm_i915_gem_object *obj; ++ struct drm_plane *primary = intel_crtc->base.primary; ++ struct drm_plane_state *plane_state = primary->state; ++ struct intel_plane *intel_plane = to_intel_plane(primary); ++ struct intel_plane_state *intel_state = ++ to_intel_plane_state(plane_state); ++ struct drm_framebuffer *fb; ++ ++ if (!plane_config->fb) ++ return; ++ ++ if (intel_alloc_initial_plane_obj(intel_crtc, plane_config)) { ++ fb = &plane_config->fb->base; ++ goto valid_fb; ++ } ++ ++ kfree(plane_config->fb); ++ ++ /* ++ * Failed to alloc the obj, check to see if we should share ++ * an fb with another CRTC instead ++ */ ++ for_each_crtc(dev, c) { ++ struct intel_plane_state *state; ++ ++ if (c == &intel_crtc->base) ++ continue; ++ ++ if (!to_intel_crtc(c)->active) ++ continue; ++ ++ state = to_intel_plane_state(c->primary->state); ++ if (!state->vma) ++ continue; ++ ++ if (intel_plane_ggtt_offset(state) == plane_config->base) { ++ fb = state->base.fb; ++ drm_framebuffer_get(fb); ++ goto valid_fb; ++ } ++ } ++ ++ /* ++ * We've failed to reconstruct the BIOS FB. Current display state ++ * indicates that the primary plane is visible, but has a NULL FB, ++ * which will lead to problems later if we don't fix it up. The ++ * simplest solution is to just disable the primary plane now and ++ * pretend the BIOS never had it enabled. ++ */ ++ intel_plane_disable_noatomic(intel_crtc, intel_plane); ++ ++ return; ++ ++valid_fb: ++ intel_state->base.rotation = plane_config->rotation; ++ intel_fill_fb_ggtt_view(&intel_state->view, fb, ++ intel_state->base.rotation); ++ intel_state->color_plane[0].stride = ++ intel_fb_pitch(fb, 0, intel_state->base.rotation); ++ ++ mutex_lock(&dev->struct_mutex); ++ intel_state->vma = ++ intel_pin_and_fence_fb_obj(fb, ++ &intel_state->view, ++ intel_plane_uses_fence(intel_state), ++ &intel_state->flags); ++ mutex_unlock(&dev->struct_mutex); ++ if (IS_ERR(intel_state->vma)) { ++ DRM_ERROR("failed to pin boot fb on pipe %d: %li\n", ++ intel_crtc->pipe, PTR_ERR(intel_state->vma)); ++ ++ intel_state->vma = NULL; ++ drm_framebuffer_put(fb); ++ return; ++ } ++ ++ obj = intel_fb_obj(fb); ++ intel_fb_obj_flush(obj, ORIGIN_DIRTYFB); ++ ++ plane_state->src_x = 0; ++ plane_state->src_y = 0; ++ plane_state->src_w = fb->width << 16; ++ plane_state->src_h = fb->height << 16; ++ ++ plane_state->crtc_x = 0; ++ plane_state->crtc_y = 0; ++ plane_state->crtc_w = fb->width; ++ plane_state->crtc_h = fb->height; ++ ++ intel_state->base.src = drm_plane_state_src(plane_state); ++ intel_state->base.dst = drm_plane_state_dest(plane_state); ++ ++ if (i915_gem_object_is_tiled(obj)) ++ dev_priv->preserve_bios_swizzle = true; ++ ++ plane_state->fb = fb; ++ plane_state->crtc = &intel_crtc->base; ++ ++ atomic_or(to_intel_plane(primary)->frontbuffer_bit, ++ &obj->frontbuffer_bits); ++} ++ ++static int skl_max_plane_width(const struct drm_framebuffer *fb, ++ int color_plane, ++ unsigned int rotation) ++{ ++ int cpp = fb->format->cpp[color_plane]; ++ ++ switch (fb->modifier) { ++ case DRM_FORMAT_MOD_LINEAR: ++ case I915_FORMAT_MOD_X_TILED: ++ switch (cpp) { ++ case 8: ++ return 4096; ++ case 4: ++ case 2: ++ case 1: ++ return 8192; ++ default: ++ MISSING_CASE(cpp); ++ break; ++ } ++ break; ++ case I915_FORMAT_MOD_Y_TILED_CCS: ++ case I915_FORMAT_MOD_Yf_TILED_CCS: ++ /* FIXME AUX plane? */ ++ case I915_FORMAT_MOD_Y_TILED: ++ case I915_FORMAT_MOD_Yf_TILED: ++ switch (cpp) { ++ case 8: ++ return 2048; ++ case 4: ++ return 4096; ++ case 2: ++ case 1: ++ return 8192; ++ default: ++ MISSING_CASE(cpp); ++ break; ++ } ++ break; ++ default: ++ MISSING_CASE(fb->modifier); ++ } ++ ++ return 2048; ++} ++ ++static bool skl_check_main_ccs_coordinates(struct intel_plane_state *plane_state, ++ int main_x, int main_y, u32 main_offset) ++{ ++ const struct drm_framebuffer *fb = plane_state->base.fb; ++ int hsub = fb->format->hsub; ++ int vsub = fb->format->vsub; ++ int aux_x = plane_state->color_plane[1].x; ++ int aux_y = plane_state->color_plane[1].y; ++ u32 aux_offset = plane_state->color_plane[1].offset; ++ u32 alignment = intel_surf_alignment(fb, 1); ++ ++ while (aux_offset >= main_offset && aux_y <= main_y) { ++ int x, y; ++ ++ if (aux_x == main_x && aux_y == main_y) ++ break; ++ ++ if (aux_offset == 0) ++ break; ++ ++ x = aux_x / hsub; ++ y = aux_y / vsub; ++ aux_offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 1, ++ aux_offset, aux_offset - alignment); ++ aux_x = x * hsub + aux_x % hsub; ++ aux_y = y * vsub + aux_y % vsub; ++ } ++ ++ if (aux_x != main_x || aux_y != main_y) ++ return false; ++ ++ plane_state->color_plane[1].offset = aux_offset; ++ plane_state->color_plane[1].x = aux_x; ++ plane_state->color_plane[1].y = aux_y; ++ ++ return true; ++} ++ ++static int skl_check_main_surface(struct intel_plane_state *plane_state) ++{ ++ const struct drm_framebuffer *fb = plane_state->base.fb; ++ unsigned int rotation = plane_state->base.rotation; ++ int x = plane_state->base.src.x1 >> 16; ++ int y = plane_state->base.src.y1 >> 16; ++ int w = drm_rect_width(&plane_state->base.src) >> 16; ++ int h = drm_rect_height(&plane_state->base.src) >> 16; ++ int max_width = skl_max_plane_width(fb, 0, rotation); ++ int max_height = 4096; ++ u32 alignment, offset, aux_offset = plane_state->color_plane[1].offset; ++ ++ if (w > max_width || h > max_height) { ++ DRM_DEBUG_KMS("requested Y/RGB source size %dx%d too big (limit %dx%d)\n", ++ w, h, max_width, max_height); ++ return -EINVAL; ++ } ++ ++ intel_add_fb_offsets(&x, &y, plane_state, 0); ++ offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 0); ++ alignment = intel_surf_alignment(fb, 0); ++ ++ /* ++ * AUX surface offset is specified as the distance from the ++ * main surface offset, and it must be non-negative. Make ++ * sure that is what we will get. ++ */ ++ if (offset > aux_offset) ++ offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0, ++ offset, aux_offset & ~(alignment - 1)); ++ ++ /* ++ * When using an X-tiled surface, the plane blows up ++ * if the x offset + width exceed the stride. ++ * ++ * TODO: linear and Y-tiled seem fine, Yf untested, ++ */ ++ if (fb->modifier == I915_FORMAT_MOD_X_TILED) { ++ int cpp = fb->format->cpp[0]; ++ ++ while ((x + w) * cpp > plane_state->color_plane[0].stride) { ++ if (offset == 0) { ++ DRM_DEBUG_KMS("Unable to find suitable display surface offset due to X-tiling\n"); ++ return -EINVAL; ++ } ++ ++ offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0, ++ offset, offset - alignment); ++ } ++ } ++ ++ /* ++ * CCS AUX surface doesn't have its own x/y offsets, we must make sure ++ * they match with the main surface x/y offsets. ++ */ ++ if (is_ccs_modifier(fb->modifier)) { ++ while (!skl_check_main_ccs_coordinates(plane_state, x, y, offset)) { ++ if (offset == 0) ++ break; ++ ++ offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0, ++ offset, offset - alignment); ++ } ++ ++ if (x != plane_state->color_plane[1].x || y != plane_state->color_plane[1].y) { ++ DRM_DEBUG_KMS("Unable to find suitable display surface offset due to CCS\n"); ++ return -EINVAL; ++ } ++ } ++ ++ plane_state->color_plane[0].offset = offset; ++ plane_state->color_plane[0].x = x; ++ plane_state->color_plane[0].y = y; ++ ++ return 0; ++} ++ ++static int skl_check_nv12_aux_surface(struct intel_plane_state *plane_state) ++{ ++ const struct drm_framebuffer *fb = plane_state->base.fb; ++ unsigned int rotation = plane_state->base.rotation; ++ int max_width = skl_max_plane_width(fb, 1, rotation); ++ int max_height = 4096; ++ int x = plane_state->base.src.x1 >> 17; ++ int y = plane_state->base.src.y1 >> 17; ++ int w = drm_rect_width(&plane_state->base.src) >> 17; ++ int h = drm_rect_height(&plane_state->base.src) >> 17; ++ u32 offset; ++ ++ intel_add_fb_offsets(&x, &y, plane_state, 1); ++ offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1); ++ ++ /* FIXME not quite sure how/if these apply to the chroma plane */ ++ if (w > max_width || h > max_height) { ++ DRM_DEBUG_KMS("CbCr source size %dx%d too big (limit %dx%d)\n", ++ w, h, max_width, max_height); ++ return -EINVAL; ++ } ++ ++ plane_state->color_plane[1].offset = offset; ++ plane_state->color_plane[1].x = x; ++ plane_state->color_plane[1].y = y; ++ ++ return 0; ++} ++ ++static int skl_check_ccs_aux_surface(struct intel_plane_state *plane_state) ++{ ++ const struct drm_framebuffer *fb = plane_state->base.fb; ++ int src_x = plane_state->base.src.x1 >> 16; ++ int src_y = plane_state->base.src.y1 >> 16; ++ int hsub = fb->format->hsub; ++ int vsub = fb->format->vsub; ++ int x = src_x / hsub; ++ int y = src_y / vsub; ++ u32 offset; ++ ++ intel_add_fb_offsets(&x, &y, plane_state, 1); ++ offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1); ++ ++ plane_state->color_plane[1].offset = offset; ++ plane_state->color_plane[1].x = x * hsub + src_x % hsub; ++ plane_state->color_plane[1].y = y * vsub + src_y % vsub; ++ ++ return 0; ++} ++ ++int skl_check_plane_surface(struct intel_plane_state *plane_state) ++{ ++ const struct drm_framebuffer *fb = plane_state->base.fb; ++ unsigned int rotation = plane_state->base.rotation; ++ int ret; ++ ++ intel_fill_fb_ggtt_view(&plane_state->view, fb, rotation); ++ plane_state->color_plane[0].stride = intel_fb_pitch(fb, 0, rotation); ++ plane_state->color_plane[1].stride = intel_fb_pitch(fb, 1, rotation); ++ ++ ret = intel_plane_check_stride(plane_state); ++ if (ret) ++ return ret; ++ ++ if (!plane_state->base.visible) ++ return 0; ++ ++ /* Rotate src coordinates to match rotated GTT view */ ++ if (drm_rotation_90_or_270(rotation)) ++ drm_rect_rotate(&plane_state->base.src, ++ fb->width << 16, fb->height << 16, ++ DRM_MODE_ROTATE_270); ++ ++ /* ++ * Handle the AUX surface first since ++ * the main surface setup depends on it. ++ */ ++ if (is_planar_yuv_format(fb->format->format)) { ++ ret = skl_check_nv12_aux_surface(plane_state); ++ if (ret) ++ return ret; ++ } else if (is_ccs_modifier(fb->modifier)) { ++ ret = skl_check_ccs_aux_surface(plane_state); ++ if (ret) ++ return ret; ++ } else { ++ plane_state->color_plane[1].offset = ~0xfff; ++ plane_state->color_plane[1].x = 0; ++ plane_state->color_plane[1].y = 0; ++ } ++ ++ ret = skl_check_main_surface(plane_state); ++ if (ret) ++ return ret; ++ ++ return 0; ++} ++ ++unsigned int ++i9xx_plane_max_stride(struct intel_plane *plane, ++ u32 pixel_format, u64 modifier, ++ unsigned int rotation) ++{ ++ struct drm_i915_private *dev_priv = to_i915(plane->base.dev); ++ ++ if (!HAS_GMCH(dev_priv)) { ++ return 32*1024; ++ } else if (INTEL_GEN(dev_priv) >= 4) { ++ if (modifier == I915_FORMAT_MOD_X_TILED) ++ return 16*1024; ++ else ++ return 32*1024; ++ } else if (INTEL_GEN(dev_priv) >= 3) { ++ if (modifier == I915_FORMAT_MOD_X_TILED) ++ return 8*1024; ++ else ++ return 16*1024; ++ } else { ++ if (plane->i9xx_plane == PLANE_C) ++ return 4*1024; ++ else ++ return 8*1024; ++ } ++} ++ ++static u32 i9xx_plane_ctl_crtc(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ u32 dspcntr = 0; ++ ++ if (crtc_state->gamma_enable) ++ dspcntr |= DISPPLANE_GAMMA_ENABLE; ++ ++ if (crtc_state->csc_enable) ++ dspcntr |= DISPPLANE_PIPE_CSC_ENABLE; ++ ++ if (INTEL_GEN(dev_priv) < 5) ++ dspcntr |= DISPPLANE_SEL_PIPE(crtc->pipe); ++ ++ return dspcntr; ++} ++ ++static u32 i9xx_plane_ctl(const struct intel_crtc_state *crtc_state, ++ const struct intel_plane_state *plane_state) ++{ ++ struct drm_i915_private *dev_priv = ++ to_i915(plane_state->base.plane->dev); ++ const struct drm_framebuffer *fb = plane_state->base.fb; ++ unsigned int rotation = plane_state->base.rotation; ++ u32 dspcntr; ++ ++ dspcntr = DISPLAY_PLANE_ENABLE; ++ ++ if (IS_G4X(dev_priv) || IS_GEN(dev_priv, 5) || ++ IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv)) ++ dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE; ++ ++ switch (fb->format->format) { ++ case DRM_FORMAT_C8: ++ dspcntr |= DISPPLANE_8BPP; ++ break; ++ case DRM_FORMAT_XRGB1555: ++ dspcntr |= DISPPLANE_BGRX555; ++ break; ++ case DRM_FORMAT_RGB565: ++ dspcntr |= DISPPLANE_BGRX565; ++ break; ++ case DRM_FORMAT_XRGB8888: ++ dspcntr |= DISPPLANE_BGRX888; ++ break; ++ case DRM_FORMAT_XBGR8888: ++ dspcntr |= DISPPLANE_RGBX888; ++ break; ++ case DRM_FORMAT_XRGB2101010: ++ dspcntr |= DISPPLANE_BGRX101010; ++ break; ++ case DRM_FORMAT_XBGR2101010: ++ dspcntr |= DISPPLANE_RGBX101010; ++ break; ++ default: ++ MISSING_CASE(fb->format->format); ++ return 0; ++ } ++ ++ if (INTEL_GEN(dev_priv) >= 4 && ++ fb->modifier == I915_FORMAT_MOD_X_TILED) ++ dspcntr |= DISPPLANE_TILED; ++ ++ if (rotation & DRM_MODE_ROTATE_180) ++ dspcntr |= DISPPLANE_ROTATE_180; ++ ++ if (rotation & DRM_MODE_REFLECT_X) ++ dspcntr |= DISPPLANE_MIRROR; ++ ++ return dspcntr; ++} ++ ++int i9xx_check_plane_surface(struct intel_plane_state *plane_state) ++{ ++ struct drm_i915_private *dev_priv = ++ to_i915(plane_state->base.plane->dev); ++ const struct drm_framebuffer *fb = plane_state->base.fb; ++ unsigned int rotation = plane_state->base.rotation; ++ int src_x = plane_state->base.src.x1 >> 16; ++ int src_y = plane_state->base.src.y1 >> 16; ++ u32 offset; ++ int ret; ++ ++ intel_fill_fb_ggtt_view(&plane_state->view, fb, rotation); ++ plane_state->color_plane[0].stride = intel_fb_pitch(fb, 0, rotation); ++ ++ ret = intel_plane_check_stride(plane_state); ++ if (ret) ++ return ret; ++ ++ intel_add_fb_offsets(&src_x, &src_y, plane_state, 0); ++ ++ if (INTEL_GEN(dev_priv) >= 4) ++ offset = intel_plane_compute_aligned_offset(&src_x, &src_y, ++ plane_state, 0); ++ else ++ offset = 0; ++ ++ /* HSW/BDW do this automagically in hardware */ ++ if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)) { ++ int src_w = drm_rect_width(&plane_state->base.src) >> 16; ++ int src_h = drm_rect_height(&plane_state->base.src) >> 16; ++ ++ if (rotation & DRM_MODE_ROTATE_180) { ++ src_x += src_w - 1; ++ src_y += src_h - 1; ++ } else if (rotation & DRM_MODE_REFLECT_X) { ++ src_x += src_w - 1; ++ } ++ } ++ ++ plane_state->color_plane[0].offset = offset; ++ plane_state->color_plane[0].x = src_x; ++ plane_state->color_plane[0].y = src_y; ++ ++ return 0; ++} ++ ++static int ++i9xx_plane_check(struct intel_crtc_state *crtc_state, ++ struct intel_plane_state *plane_state) ++{ ++ int ret; ++ ++ ret = chv_plane_check_rotation(plane_state); ++ if (ret) ++ return ret; ++ ++ ret = drm_atomic_helper_check_plane_state(&plane_state->base, ++ &crtc_state->base, ++ DRM_PLANE_HELPER_NO_SCALING, ++ DRM_PLANE_HELPER_NO_SCALING, ++ false, true); ++ if (ret) ++ return ret; ++ ++ if (!plane_state->base.visible) ++ return 0; ++ ++ ret = intel_plane_check_src_coordinates(plane_state); ++ if (ret) ++ return ret; ++ ++ ret = i9xx_check_plane_surface(plane_state); ++ if (ret) ++ return ret; ++ ++ plane_state->ctl = i9xx_plane_ctl(crtc_state, plane_state); ++ ++ return 0; ++} ++ ++static void i9xx_update_plane(struct intel_plane *plane, ++ const struct intel_crtc_state *crtc_state, ++ const struct intel_plane_state *plane_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(plane->base.dev); ++ enum i9xx_plane_id i9xx_plane = plane->i9xx_plane; ++ u32 linear_offset; ++ int x = plane_state->color_plane[0].x; ++ int y = plane_state->color_plane[0].y; ++ unsigned long irqflags; ++ u32 dspaddr_offset; ++ u32 dspcntr; ++ ++ dspcntr = plane_state->ctl | i9xx_plane_ctl_crtc(crtc_state); ++ ++ linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0); ++ ++ if (INTEL_GEN(dev_priv) >= 4) ++ dspaddr_offset = plane_state->color_plane[0].offset; ++ else ++ dspaddr_offset = linear_offset; ++ ++ spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); ++ ++ I915_WRITE_FW(DSPSTRIDE(i9xx_plane), plane_state->color_plane[0].stride); ++ ++ if (INTEL_GEN(dev_priv) < 4) { ++ /* pipesrc and dspsize control the size that is scaled from, ++ * which should always be the user's requested size. ++ */ ++ I915_WRITE_FW(DSPPOS(i9xx_plane), 0); ++ I915_WRITE_FW(DSPSIZE(i9xx_plane), ++ ((crtc_state->pipe_src_h - 1) << 16) | ++ (crtc_state->pipe_src_w - 1)); ++ } else if (IS_CHERRYVIEW(dev_priv) && i9xx_plane == PLANE_B) { ++ I915_WRITE_FW(PRIMPOS(i9xx_plane), 0); ++ I915_WRITE_FW(PRIMSIZE(i9xx_plane), ++ ((crtc_state->pipe_src_h - 1) << 16) | ++ (crtc_state->pipe_src_w - 1)); ++ I915_WRITE_FW(PRIMCNSTALPHA(i9xx_plane), 0); ++ } ++ ++ if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { ++ I915_WRITE_FW(DSPOFFSET(i9xx_plane), (y << 16) | x); ++ } else if (INTEL_GEN(dev_priv) >= 4) { ++ I915_WRITE_FW(DSPLINOFF(i9xx_plane), linear_offset); ++ I915_WRITE_FW(DSPTILEOFF(i9xx_plane), (y << 16) | x); ++ } ++ ++ /* ++ * The control register self-arms if the plane was previously ++ * disabled. Try to make the plane enable atomic by writing ++ * the control register just before the surface register. ++ */ ++ I915_WRITE_FW(DSPCNTR(i9xx_plane), dspcntr); ++ if (INTEL_GEN(dev_priv) >= 4) ++ I915_WRITE_FW(DSPSURF(i9xx_plane), ++ intel_plane_ggtt_offset(plane_state) + ++ dspaddr_offset); ++ else ++ I915_WRITE_FW(DSPADDR(i9xx_plane), ++ intel_plane_ggtt_offset(plane_state) + ++ dspaddr_offset); ++ ++ spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); ++} ++ ++static void i9xx_disable_plane(struct intel_plane *plane, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(plane->base.dev); ++ enum i9xx_plane_id i9xx_plane = plane->i9xx_plane; ++ unsigned long irqflags; ++ u32 dspcntr; ++ ++ /* ++ * DSPCNTR pipe gamma enable on g4x+ and pipe csc ++ * enable on ilk+ affect the pipe bottom color as ++ * well, so we must configure them even if the plane ++ * is disabled. ++ * ++ * On pre-g4x there is no way to gamma correct the ++ * pipe bottom color but we'll keep on doing this ++ * anyway so that the crtc state readout works correctly. ++ */ ++ dspcntr = i9xx_plane_ctl_crtc(crtc_state); ++ ++ spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); ++ ++ I915_WRITE_FW(DSPCNTR(i9xx_plane), dspcntr); ++ if (INTEL_GEN(dev_priv) >= 4) ++ I915_WRITE_FW(DSPSURF(i9xx_plane), 0); ++ else ++ I915_WRITE_FW(DSPADDR(i9xx_plane), 0); ++ ++ spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); ++} ++ ++static bool i9xx_plane_get_hw_state(struct intel_plane *plane, ++ enum pipe *pipe) ++{ ++ struct drm_i915_private *dev_priv = to_i915(plane->base.dev); ++ enum intel_display_power_domain power_domain; ++ enum i9xx_plane_id i9xx_plane = plane->i9xx_plane; ++ intel_wakeref_t wakeref; ++ bool ret; ++ u32 val; ++ ++ /* ++ * Not 100% correct for planes that can move between pipes, ++ * but that's only the case for gen2-4 which don't have any ++ * display power wells. ++ */ ++ power_domain = POWER_DOMAIN_PIPE(plane->pipe); ++ wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain); ++ if (!wakeref) ++ return false; ++ ++ val = I915_READ(DSPCNTR(i9xx_plane)); ++ ++ ret = val & DISPLAY_PLANE_ENABLE; ++ ++ if (INTEL_GEN(dev_priv) >= 5) ++ *pipe = plane->pipe; ++ else ++ *pipe = (val & DISPPLANE_SEL_PIPE_MASK) >> ++ DISPPLANE_SEL_PIPE_SHIFT; ++ ++ intel_display_power_put(dev_priv, power_domain, wakeref); ++ ++ return ret; ++} ++ ++static u32 ++intel_fb_stride_alignment(const struct drm_framebuffer *fb, int color_plane) ++{ ++ if (fb->modifier == DRM_FORMAT_MOD_LINEAR) ++ return 64; ++ else ++ return intel_tile_width_bytes(fb, color_plane); ++} ++ ++static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id) ++{ ++ struct drm_device *dev = intel_crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ ++ I915_WRITE(SKL_PS_CTRL(intel_crtc->pipe, id), 0); ++ I915_WRITE(SKL_PS_WIN_POS(intel_crtc->pipe, id), 0); ++ I915_WRITE(SKL_PS_WIN_SZ(intel_crtc->pipe, id), 0); ++} ++ ++/* ++ * This function detaches (aka. unbinds) unused scalers in hardware ++ */ ++static void skl_detach_scalers(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc); ++ const struct intel_crtc_scaler_state *scaler_state = ++ &crtc_state->scaler_state; ++ int i; ++ ++ /* loop through and disable scalers that aren't in use */ ++ for (i = 0; i < intel_crtc->num_scalers; i++) { ++ if (!scaler_state->scalers[i].in_use) ++ skl_detach_scaler(intel_crtc, i); ++ } ++} ++ ++static unsigned int skl_plane_stride_mult(const struct drm_framebuffer *fb, ++ int color_plane, unsigned int rotation) ++{ ++ /* ++ * The stride is either expressed as a multiple of 64 bytes chunks for ++ * linear buffers or in number of tiles for tiled buffers. ++ */ ++ if (fb->modifier == DRM_FORMAT_MOD_LINEAR) ++ return 64; ++ else if (drm_rotation_90_or_270(rotation)) ++ return intel_tile_height(fb, color_plane); ++ else ++ return intel_tile_width_bytes(fb, color_plane); ++} ++ ++u32 skl_plane_stride(const struct intel_plane_state *plane_state, ++ int color_plane) ++{ ++ const struct drm_framebuffer *fb = plane_state->base.fb; ++ unsigned int rotation = plane_state->base.rotation; ++ u32 stride = plane_state->color_plane[color_plane].stride; ++ ++ if (color_plane >= fb->format->num_planes) ++ return 0; ++ ++ return stride / skl_plane_stride_mult(fb, color_plane, rotation); ++} ++ ++static u32 skl_plane_ctl_format(u32 pixel_format) ++{ ++ switch (pixel_format) { ++ case DRM_FORMAT_C8: ++ return PLANE_CTL_FORMAT_INDEXED; ++ case DRM_FORMAT_RGB565: ++ return PLANE_CTL_FORMAT_RGB_565; ++ case DRM_FORMAT_XBGR8888: ++ case DRM_FORMAT_ABGR8888: ++ return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX; ++ case DRM_FORMAT_XRGB8888: ++ case DRM_FORMAT_ARGB8888: ++ return PLANE_CTL_FORMAT_XRGB_8888; ++ case DRM_FORMAT_XRGB2101010: ++ return PLANE_CTL_FORMAT_XRGB_2101010; ++ case DRM_FORMAT_XBGR2101010: ++ return PLANE_CTL_ORDER_RGBX | PLANE_CTL_FORMAT_XRGB_2101010; ++ case DRM_FORMAT_XBGR16161616F: ++ case DRM_FORMAT_ABGR16161616F: ++ return PLANE_CTL_FORMAT_XRGB_16161616F | PLANE_CTL_ORDER_RGBX; ++ case DRM_FORMAT_XRGB16161616F: ++ case DRM_FORMAT_ARGB16161616F: ++ return PLANE_CTL_FORMAT_XRGB_16161616F; ++ case DRM_FORMAT_YUYV: ++ return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YUYV; ++ case DRM_FORMAT_YVYU: ++ return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YVYU; ++ case DRM_FORMAT_UYVY: ++ return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_UYVY; ++ case DRM_FORMAT_VYUY: ++ return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_VYUY; ++ case DRM_FORMAT_NV12: ++ return PLANE_CTL_FORMAT_NV12; ++ case DRM_FORMAT_P010: ++ return PLANE_CTL_FORMAT_P010; ++ case DRM_FORMAT_P012: ++ return PLANE_CTL_FORMAT_P012; ++ case DRM_FORMAT_P016: ++ return PLANE_CTL_FORMAT_P016; ++ case DRM_FORMAT_Y210: ++ return PLANE_CTL_FORMAT_Y210; ++ case DRM_FORMAT_Y212: ++ return PLANE_CTL_FORMAT_Y212; ++ case DRM_FORMAT_Y216: ++ return PLANE_CTL_FORMAT_Y216; ++ case DRM_FORMAT_XVYU2101010: ++ return PLANE_CTL_FORMAT_Y410; ++ case DRM_FORMAT_XVYU12_16161616: ++ return PLANE_CTL_FORMAT_Y412; ++ case DRM_FORMAT_XVYU16161616: ++ return PLANE_CTL_FORMAT_Y416; ++ default: ++ MISSING_CASE(pixel_format); ++ } ++ ++ return 0; ++} ++ ++static u32 skl_plane_ctl_alpha(const struct intel_plane_state *plane_state) ++{ ++ if (!plane_state->base.fb->format->has_alpha) ++ return PLANE_CTL_ALPHA_DISABLE; ++ ++ switch (plane_state->base.pixel_blend_mode) { ++ case DRM_MODE_BLEND_PIXEL_NONE: ++ return PLANE_CTL_ALPHA_DISABLE; ++ case DRM_MODE_BLEND_PREMULTI: ++ return PLANE_CTL_ALPHA_SW_PREMULTIPLY; ++ case DRM_MODE_BLEND_COVERAGE: ++ return PLANE_CTL_ALPHA_HW_PREMULTIPLY; ++ default: ++ MISSING_CASE(plane_state->base.pixel_blend_mode); ++ return PLANE_CTL_ALPHA_DISABLE; ++ } ++} ++ ++static u32 glk_plane_color_ctl_alpha(const struct intel_plane_state *plane_state) ++{ ++ if (!plane_state->base.fb->format->has_alpha) ++ return PLANE_COLOR_ALPHA_DISABLE; ++ ++ switch (plane_state->base.pixel_blend_mode) { ++ case DRM_MODE_BLEND_PIXEL_NONE: ++ return PLANE_COLOR_ALPHA_DISABLE; ++ case DRM_MODE_BLEND_PREMULTI: ++ return PLANE_COLOR_ALPHA_SW_PREMULTIPLY; ++ case DRM_MODE_BLEND_COVERAGE: ++ return PLANE_COLOR_ALPHA_HW_PREMULTIPLY; ++ default: ++ MISSING_CASE(plane_state->base.pixel_blend_mode); ++ return PLANE_COLOR_ALPHA_DISABLE; ++ } ++} ++ ++static u32 skl_plane_ctl_tiling(u64 fb_modifier) ++{ ++ switch (fb_modifier) { ++ case DRM_FORMAT_MOD_LINEAR: ++ break; ++ case I915_FORMAT_MOD_X_TILED: ++ return PLANE_CTL_TILED_X; ++ case I915_FORMAT_MOD_Y_TILED: ++ return PLANE_CTL_TILED_Y; ++ case I915_FORMAT_MOD_Y_TILED_CCS: ++ return PLANE_CTL_TILED_Y | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE; ++ case I915_FORMAT_MOD_Yf_TILED: ++ return PLANE_CTL_TILED_YF; ++ case I915_FORMAT_MOD_Yf_TILED_CCS: ++ return PLANE_CTL_TILED_YF | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE; ++ default: ++ MISSING_CASE(fb_modifier); ++ } ++ ++ return 0; ++} ++ ++static u32 skl_plane_ctl_rotate(unsigned int rotate) ++{ ++ switch (rotate) { ++ case DRM_MODE_ROTATE_0: ++ break; ++ /* ++ * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr ++ * while i915 HW rotation is clockwise, thats why this swapping. ++ */ ++ case DRM_MODE_ROTATE_90: ++ return PLANE_CTL_ROTATE_270; ++ case DRM_MODE_ROTATE_180: ++ return PLANE_CTL_ROTATE_180; ++ case DRM_MODE_ROTATE_270: ++ return PLANE_CTL_ROTATE_90; ++ default: ++ MISSING_CASE(rotate); ++ } ++ ++ return 0; ++} ++ ++static u32 cnl_plane_ctl_flip(unsigned int reflect) ++{ ++ switch (reflect) { ++ case 0: ++ break; ++ case DRM_MODE_REFLECT_X: ++ return PLANE_CTL_FLIP_HORIZONTAL; ++ case DRM_MODE_REFLECT_Y: ++ default: ++ MISSING_CASE(reflect); ++ } ++ ++ return 0; ++} ++ ++u32 skl_plane_ctl_crtc(const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); ++ u32 plane_ctl = 0; ++ ++ if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) ++ return plane_ctl; ++ ++ if (crtc_state->gamma_enable) ++ plane_ctl |= PLANE_CTL_PIPE_GAMMA_ENABLE; ++ ++ if (crtc_state->csc_enable) ++ plane_ctl |= PLANE_CTL_PIPE_CSC_ENABLE; ++ ++ return plane_ctl; ++} ++ ++u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state, ++ const struct intel_plane_state *plane_state) ++{ ++ struct drm_i915_private *dev_priv = ++ to_i915(plane_state->base.plane->dev); ++ const struct drm_framebuffer *fb = plane_state->base.fb; ++ unsigned int rotation = plane_state->base.rotation; ++ const struct drm_intel_sprite_colorkey *key = &plane_state->ckey; ++ u32 plane_ctl; ++ ++ plane_ctl = PLANE_CTL_ENABLE; ++ ++ if (INTEL_GEN(dev_priv) < 10 && !IS_GEMINILAKE(dev_priv)) { ++ plane_ctl |= skl_plane_ctl_alpha(plane_state); ++ plane_ctl |= PLANE_CTL_PLANE_GAMMA_DISABLE; ++ ++ if (plane_state->base.color_encoding == DRM_COLOR_YCBCR_BT709) ++ plane_ctl |= PLANE_CTL_YUV_TO_RGB_CSC_FORMAT_BT709; ++ ++ if (plane_state->base.color_range == DRM_COLOR_YCBCR_FULL_RANGE) ++ plane_ctl |= PLANE_CTL_YUV_RANGE_CORRECTION_DISABLE; ++ } ++ ++ plane_ctl |= skl_plane_ctl_format(fb->format->format); ++ plane_ctl |= skl_plane_ctl_tiling(fb->modifier); ++ plane_ctl |= skl_plane_ctl_rotate(rotation & DRM_MODE_ROTATE_MASK); ++ ++ if (INTEL_GEN(dev_priv) >= 10) ++ plane_ctl |= cnl_plane_ctl_flip(rotation & ++ DRM_MODE_REFLECT_MASK); ++ ++ if (key->flags & I915_SET_COLORKEY_DESTINATION) ++ plane_ctl |= PLANE_CTL_KEY_ENABLE_DESTINATION; ++ else if (key->flags & I915_SET_COLORKEY_SOURCE) ++ plane_ctl |= PLANE_CTL_KEY_ENABLE_SOURCE; ++ ++ return plane_ctl; ++} ++ ++u32 glk_plane_color_ctl_crtc(const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); ++ u32 plane_color_ctl = 0; ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ return plane_color_ctl; ++ ++ if (crtc_state->gamma_enable) ++ plane_color_ctl |= PLANE_COLOR_PIPE_GAMMA_ENABLE; ++ ++ if (crtc_state->csc_enable) ++ plane_color_ctl |= PLANE_COLOR_PIPE_CSC_ENABLE; ++ ++ return plane_color_ctl; ++} ++ ++u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state, ++ const struct intel_plane_state *plane_state) ++{ ++ struct drm_i915_private *dev_priv = ++ to_i915(plane_state->base.plane->dev); ++ const struct drm_framebuffer *fb = plane_state->base.fb; ++ struct intel_plane *plane = to_intel_plane(plane_state->base.plane); ++ u32 plane_color_ctl = 0; ++ ++ plane_color_ctl |= PLANE_COLOR_PLANE_GAMMA_DISABLE; ++ plane_color_ctl |= glk_plane_color_ctl_alpha(plane_state); ++ ++ if (fb->format->is_yuv && !icl_is_hdr_plane(dev_priv, plane->id)) { ++ if (plane_state->base.color_encoding == DRM_COLOR_YCBCR_BT709) ++ plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV709_TO_RGB709; ++ else ++ plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV601_TO_RGB709; ++ ++ if (plane_state->base.color_range == DRM_COLOR_YCBCR_FULL_RANGE) ++ plane_color_ctl |= PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE; ++ } else if (fb->format->is_yuv) { ++ plane_color_ctl |= PLANE_COLOR_INPUT_CSC_ENABLE; ++ } ++ ++ return plane_color_ctl; ++} ++ ++static int ++__intel_display_resume(struct drm_device *dev, ++ struct drm_atomic_state *state, ++ struct drm_modeset_acquire_ctx *ctx) ++{ ++ struct drm_crtc_state *crtc_state; ++ struct drm_crtc *crtc; ++ int i, ret; ++ ++ intel_modeset_setup_hw_state(dev, ctx); ++ i915_redisable_vga(to_i915(dev)); ++ ++ if (!state) ++ return 0; ++ ++ /* ++ * We've duplicated the state, pointers to the old state are invalid. ++ * ++ * Don't attempt to use the old state until we commit the duplicated state. ++ */ ++ for_each_new_crtc_in_state(state, crtc, crtc_state, i) { ++ /* ++ * Force recalculation even if we restore ++ * current state. With fast modeset this may not result ++ * in a modeset when the state is compatible. ++ */ ++ crtc_state->mode_changed = true; ++ } ++ ++ /* ignore any reset values/BIOS leftovers in the WM registers */ ++ if (!HAS_GMCH(to_i915(dev))) ++ to_intel_atomic_state(state)->skip_intermediate_wm = true; ++ ++ ret = drm_atomic_helper_commit_duplicated_state(state, ctx); ++ ++ WARN_ON(ret == -EDEADLK); ++ return ret; ++} ++ ++static bool gpu_reset_clobbers_display(struct drm_i915_private *dev_priv) ++{ ++ return (INTEL_INFO(dev_priv)->gpu_reset_clobbers_display && ++ intel_has_gpu_reset(dev_priv)); ++} ++ ++void intel_prepare_reset(struct drm_i915_private *dev_priv) ++{ ++ struct drm_device *dev = &dev_priv->drm; ++ struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx; ++ struct drm_atomic_state *state; ++ int ret; ++ ++ /* reset doesn't touch the display */ ++ if (!i915_modparams.force_reset_modeset_test && ++ !gpu_reset_clobbers_display(dev_priv)) ++ return; ++ ++ /* We have a modeset vs reset deadlock, defensively unbreak it. */ ++ set_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags); ++ wake_up_all(&dev_priv->gpu_error.wait_queue); ++ ++ if (atomic_read(&dev_priv->gpu_error.pending_fb_pin)) { ++ DRM_DEBUG_KMS("Modeset potentially stuck, unbreaking through wedging\n"); ++ i915_gem_set_wedged(dev_priv); ++ } ++ ++ /* ++ * Need mode_config.mutex so that we don't ++ * trample ongoing ->detect() and whatnot. ++ */ ++ mutex_lock(&dev->mode_config.mutex); ++ drm_modeset_acquire_init(ctx, 0); ++ while (1) { ++ ret = drm_modeset_lock_all_ctx(dev, ctx); ++ if (ret != -EDEADLK) ++ break; ++ ++ drm_modeset_backoff(ctx); ++ } ++ /* ++ * Disabling the crtcs gracefully seems nicer. Also the ++ * g33 docs say we should at least disable all the planes. ++ */ ++ state = drm_atomic_helper_duplicate_state(dev, ctx); ++ if (IS_ERR(state)) { ++ ret = PTR_ERR(state); ++ DRM_ERROR("Duplicating state failed with %i\n", ret); ++ return; ++ } ++ ++ ret = drm_atomic_helper_disable_all(dev, ctx); ++ if (ret) { ++ DRM_ERROR("Suspending crtc's failed with %i\n", ret); ++ drm_atomic_state_put(state); ++ return; ++ } ++ ++ dev_priv->modeset_restore_state = state; ++ state->acquire_ctx = ctx; ++} ++ ++void intel_finish_reset(struct drm_i915_private *dev_priv) ++{ ++ struct drm_device *dev = &dev_priv->drm; ++ struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx; ++ struct drm_atomic_state *state; ++ int ret; ++ ++ /* reset doesn't touch the display */ ++ if (!test_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags)) ++ return; ++ ++ state = fetch_and_zero(&dev_priv->modeset_restore_state); ++ if (!state) ++ goto unlock; ++ ++ /* reset doesn't touch the display */ ++ if (!gpu_reset_clobbers_display(dev_priv)) { ++ /* for testing only restore the display */ ++ ret = __intel_display_resume(dev, state, ctx); ++ if (ret) ++ DRM_ERROR("Restoring old state failed with %i\n", ret); ++ } else { ++ /* ++ * The display has been reset as well, ++ * so need a full re-initialization. ++ */ ++ intel_pps_unlock_regs_wa(dev_priv); ++ intel_modeset_init_hw(dev); ++ intel_init_clock_gating(dev_priv); ++ ++ spin_lock_irq(&dev_priv->irq_lock); ++ if (dev_priv->display.hpd_irq_setup) ++ dev_priv->display.hpd_irq_setup(dev_priv); ++ spin_unlock_irq(&dev_priv->irq_lock); ++ ++ ret = __intel_display_resume(dev, state, ctx); ++ if (ret) ++ DRM_ERROR("Restoring old state failed with %i\n", ret); ++ ++ intel_hpd_init(dev_priv); ++ } ++ ++ drm_atomic_state_put(state); ++unlock: ++ drm_modeset_drop_locks(ctx); ++ drm_modeset_acquire_fini(ctx); ++ mutex_unlock(&dev->mode_config.mutex); ++ ++ clear_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags); ++} ++ ++static void icl_set_pipe_chicken(struct intel_crtc *crtc) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ u32 tmp; ++ ++ tmp = I915_READ(PIPE_CHICKEN(pipe)); ++ ++ /* ++ * Display WA #1153: icl ++ * enable hardware to bypass the alpha math ++ * and rounding for per-pixel values 00 and 0xff ++ */ ++ tmp |= PER_PIXEL_ALPHA_BYPASS_EN; ++ /* ++ * Display WA # 1605353570: icl ++ * Set the pixel rounding bit to 1 for allowing ++ * passthrough of Frame buffer pixels unmodified ++ * across pipe ++ */ ++ tmp |= PIXEL_ROUNDING_TRUNC_FB_PASSTHRU; ++ I915_WRITE(PIPE_CHICKEN(pipe), tmp); ++} ++ ++static void intel_update_pipe_config(const struct intel_crtc_state *old_crtc_state, ++ const struct intel_crtc_state *new_crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ ++ /* drm_atomic_helper_update_legacy_modeset_state might not be called. */ ++ crtc->base.mode = new_crtc_state->base.mode; ++ ++ /* ++ * Update pipe size and adjust fitter if needed: the reason for this is ++ * that in compute_mode_changes we check the native mode (not the pfit ++ * mode) to see if we can flip rather than do a full mode set. In the ++ * fastboot case, we'll flip, but if we don't update the pipesrc and ++ * pfit state, we'll end up with a big fb scanned out into the wrong ++ * sized surface. ++ */ ++ ++ I915_WRITE(PIPESRC(crtc->pipe), ++ ((new_crtc_state->pipe_src_w - 1) << 16) | ++ (new_crtc_state->pipe_src_h - 1)); ++ ++ /* on skylake this is done by detaching scalers */ ++ if (INTEL_GEN(dev_priv) >= 9) { ++ skl_detach_scalers(new_crtc_state); ++ ++ if (new_crtc_state->pch_pfit.enabled) ++ skylake_pfit_enable(new_crtc_state); ++ } else if (HAS_PCH_SPLIT(dev_priv)) { ++ if (new_crtc_state->pch_pfit.enabled) ++ ironlake_pfit_enable(new_crtc_state); ++ else if (old_crtc_state->pch_pfit.enabled) ++ ironlake_pfit_disable(old_crtc_state); ++ } ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ icl_set_pipe_chicken(crtc); ++} ++ ++static void intel_fdi_normal_train(struct intel_crtc *crtc) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ int pipe = crtc->pipe; ++ i915_reg_t reg; ++ u32 temp; ++ ++ /* enable normal train */ ++ reg = FDI_TX_CTL(pipe); ++ temp = I915_READ(reg); ++ if (IS_IVYBRIDGE(dev_priv)) { ++ temp &= ~FDI_LINK_TRAIN_NONE_IVB; ++ temp |= FDI_LINK_TRAIN_NONE_IVB | FDI_TX_ENHANCE_FRAME_ENABLE; ++ } else { ++ temp &= ~FDI_LINK_TRAIN_NONE; ++ temp |= FDI_LINK_TRAIN_NONE | FDI_TX_ENHANCE_FRAME_ENABLE; ++ } ++ I915_WRITE(reg, temp); ++ ++ reg = FDI_RX_CTL(pipe); ++ temp = I915_READ(reg); ++ if (HAS_PCH_CPT(dev_priv)) { ++ temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT; ++ temp |= FDI_LINK_TRAIN_NORMAL_CPT; ++ } else { ++ temp &= ~FDI_LINK_TRAIN_NONE; ++ temp |= FDI_LINK_TRAIN_NONE; ++ } ++ I915_WRITE(reg, temp | FDI_RX_ENHANCE_FRAME_ENABLE); ++ ++ /* wait one idle pattern time */ ++ POSTING_READ(reg); ++ udelay(1000); ++ ++ /* IVB wants error correction enabled */ ++ if (IS_IVYBRIDGE(dev_priv)) ++ I915_WRITE(reg, I915_READ(reg) | FDI_FS_ERRC_ENABLE | ++ FDI_FE_ERRC_ENABLE); ++} ++ ++/* The FDI link training functions for ILK/Ibexpeak. */ ++static void ironlake_fdi_link_train(struct intel_crtc *crtc, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ int pipe = crtc->pipe; ++ i915_reg_t reg; ++ u32 temp, tries; ++ ++ /* FDI needs bits from pipe first */ ++ assert_pipe_enabled(dev_priv, pipe); ++ ++ /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit ++ for train result */ ++ reg = FDI_RX_IMR(pipe); ++ temp = I915_READ(reg); ++ temp &= ~FDI_RX_SYMBOL_LOCK; ++ temp &= ~FDI_RX_BIT_LOCK; ++ I915_WRITE(reg, temp); ++ I915_READ(reg); ++ udelay(150); ++ ++ /* enable CPU FDI TX and PCH FDI RX */ ++ reg = FDI_TX_CTL(pipe); ++ temp = I915_READ(reg); ++ temp &= ~FDI_DP_PORT_WIDTH_MASK; ++ temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes); ++ temp &= ~FDI_LINK_TRAIN_NONE; ++ temp |= FDI_LINK_TRAIN_PATTERN_1; ++ I915_WRITE(reg, temp | FDI_TX_ENABLE); ++ ++ reg = FDI_RX_CTL(pipe); ++ temp = I915_READ(reg); ++ temp &= ~FDI_LINK_TRAIN_NONE; ++ temp |= FDI_LINK_TRAIN_PATTERN_1; ++ I915_WRITE(reg, temp | FDI_RX_ENABLE); ++ ++ POSTING_READ(reg); ++ udelay(150); ++ ++ /* Ironlake workaround, enable clock pointer after FDI enable*/ ++ I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR); ++ I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR | ++ FDI_RX_PHASE_SYNC_POINTER_EN); ++ ++ reg = FDI_RX_IIR(pipe); ++ for (tries = 0; tries < 5; tries++) { ++ temp = I915_READ(reg); ++ DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp); ++ ++ if ((temp & FDI_RX_BIT_LOCK)) { ++ DRM_DEBUG_KMS("FDI train 1 done.\n"); ++ I915_WRITE(reg, temp | FDI_RX_BIT_LOCK); ++ break; ++ } ++ } ++ if (tries == 5) ++ DRM_ERROR("FDI train 1 fail!\n"); ++ ++ /* Train 2 */ ++ reg = FDI_TX_CTL(pipe); ++ temp = I915_READ(reg); ++ temp &= ~FDI_LINK_TRAIN_NONE; ++ temp |= FDI_LINK_TRAIN_PATTERN_2; ++ I915_WRITE(reg, temp); ++ ++ reg = FDI_RX_CTL(pipe); ++ temp = I915_READ(reg); ++ temp &= ~FDI_LINK_TRAIN_NONE; ++ temp |= FDI_LINK_TRAIN_PATTERN_2; ++ I915_WRITE(reg, temp); ++ ++ POSTING_READ(reg); ++ udelay(150); ++ ++ reg = FDI_RX_IIR(pipe); ++ for (tries = 0; tries < 5; tries++) { ++ temp = I915_READ(reg); ++ DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp); ++ ++ if (temp & FDI_RX_SYMBOL_LOCK) { ++ I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK); ++ DRM_DEBUG_KMS("FDI train 2 done.\n"); ++ break; ++ } ++ } ++ if (tries == 5) ++ DRM_ERROR("FDI train 2 fail!\n"); ++ ++ DRM_DEBUG_KMS("FDI train done\n"); ++ ++} ++ ++static const int snb_b_fdi_train_param[] = { ++ FDI_LINK_TRAIN_400MV_0DB_SNB_B, ++ FDI_LINK_TRAIN_400MV_6DB_SNB_B, ++ FDI_LINK_TRAIN_600MV_3_5DB_SNB_B, ++ FDI_LINK_TRAIN_800MV_0DB_SNB_B, ++}; ++ ++/* The FDI link training functions for SNB/Cougarpoint. */ ++static void gen6_fdi_link_train(struct intel_crtc *crtc, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ int pipe = crtc->pipe; ++ i915_reg_t reg; ++ u32 temp, i, retry; ++ ++ /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit ++ for train result */ ++ reg = FDI_RX_IMR(pipe); ++ temp = I915_READ(reg); ++ temp &= ~FDI_RX_SYMBOL_LOCK; ++ temp &= ~FDI_RX_BIT_LOCK; ++ I915_WRITE(reg, temp); ++ ++ POSTING_READ(reg); ++ udelay(150); ++ ++ /* enable CPU FDI TX and PCH FDI RX */ ++ reg = FDI_TX_CTL(pipe); ++ temp = I915_READ(reg); ++ temp &= ~FDI_DP_PORT_WIDTH_MASK; ++ temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes); ++ temp &= ~FDI_LINK_TRAIN_NONE; ++ temp |= FDI_LINK_TRAIN_PATTERN_1; ++ temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK; ++ /* SNB-B */ ++ temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B; ++ I915_WRITE(reg, temp | FDI_TX_ENABLE); ++ ++ I915_WRITE(FDI_RX_MISC(pipe), ++ FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90); ++ ++ reg = FDI_RX_CTL(pipe); ++ temp = I915_READ(reg); ++ if (HAS_PCH_CPT(dev_priv)) { ++ temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT; ++ temp |= FDI_LINK_TRAIN_PATTERN_1_CPT; ++ } else { ++ temp &= ~FDI_LINK_TRAIN_NONE; ++ temp |= FDI_LINK_TRAIN_PATTERN_1; ++ } ++ I915_WRITE(reg, temp | FDI_RX_ENABLE); ++ ++ POSTING_READ(reg); ++ udelay(150); ++ ++ for (i = 0; i < 4; i++) { ++ reg = FDI_TX_CTL(pipe); ++ temp = I915_READ(reg); ++ temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK; ++ temp |= snb_b_fdi_train_param[i]; ++ I915_WRITE(reg, temp); ++ ++ POSTING_READ(reg); ++ udelay(500); ++ ++ for (retry = 0; retry < 5; retry++) { ++ reg = FDI_RX_IIR(pipe); ++ temp = I915_READ(reg); ++ DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp); ++ if (temp & FDI_RX_BIT_LOCK) { ++ I915_WRITE(reg, temp | FDI_RX_BIT_LOCK); ++ DRM_DEBUG_KMS("FDI train 1 done.\n"); ++ break; ++ } ++ udelay(50); ++ } ++ if (retry < 5) ++ break; ++ } ++ if (i == 4) ++ DRM_ERROR("FDI train 1 fail!\n"); ++ ++ /* Train 2 */ ++ reg = FDI_TX_CTL(pipe); ++ temp = I915_READ(reg); ++ temp &= ~FDI_LINK_TRAIN_NONE; ++ temp |= FDI_LINK_TRAIN_PATTERN_2; ++ if (IS_GEN(dev_priv, 6)) { ++ temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK; ++ /* SNB-B */ ++ temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B; ++ } ++ I915_WRITE(reg, temp); ++ ++ reg = FDI_RX_CTL(pipe); ++ temp = I915_READ(reg); ++ if (HAS_PCH_CPT(dev_priv)) { ++ temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT; ++ temp |= FDI_LINK_TRAIN_PATTERN_2_CPT; ++ } else { ++ temp &= ~FDI_LINK_TRAIN_NONE; ++ temp |= FDI_LINK_TRAIN_PATTERN_2; ++ } ++ I915_WRITE(reg, temp); ++ ++ POSTING_READ(reg); ++ udelay(150); ++ ++ for (i = 0; i < 4; i++) { ++ reg = FDI_TX_CTL(pipe); ++ temp = I915_READ(reg); ++ temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK; ++ temp |= snb_b_fdi_train_param[i]; ++ I915_WRITE(reg, temp); ++ ++ POSTING_READ(reg); ++ udelay(500); ++ ++ for (retry = 0; retry < 5; retry++) { ++ reg = FDI_RX_IIR(pipe); ++ temp = I915_READ(reg); ++ DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp); ++ if (temp & FDI_RX_SYMBOL_LOCK) { ++ I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK); ++ DRM_DEBUG_KMS("FDI train 2 done.\n"); ++ break; ++ } ++ udelay(50); ++ } ++ if (retry < 5) ++ break; ++ } ++ if (i == 4) ++ DRM_ERROR("FDI train 2 fail!\n"); ++ ++ DRM_DEBUG_KMS("FDI train done.\n"); ++} ++ ++/* Manual link training for Ivy Bridge A0 parts */ ++static void ivb_manual_fdi_link_train(struct intel_crtc *crtc, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ int pipe = crtc->pipe; ++ i915_reg_t reg; ++ u32 temp, i, j; ++ ++ /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit ++ for train result */ ++ reg = FDI_RX_IMR(pipe); ++ temp = I915_READ(reg); ++ temp &= ~FDI_RX_SYMBOL_LOCK; ++ temp &= ~FDI_RX_BIT_LOCK; ++ I915_WRITE(reg, temp); ++ ++ POSTING_READ(reg); ++ udelay(150); ++ ++ DRM_DEBUG_KMS("FDI_RX_IIR before link train 0x%x\n", ++ I915_READ(FDI_RX_IIR(pipe))); ++ ++ /* Try each vswing and preemphasis setting twice before moving on */ ++ for (j = 0; j < ARRAY_SIZE(snb_b_fdi_train_param) * 2; j++) { ++ /* disable first in case we need to retry */ ++ reg = FDI_TX_CTL(pipe); ++ temp = I915_READ(reg); ++ temp &= ~(FDI_LINK_TRAIN_AUTO | FDI_LINK_TRAIN_NONE_IVB); ++ temp &= ~FDI_TX_ENABLE; ++ I915_WRITE(reg, temp); ++ ++ reg = FDI_RX_CTL(pipe); ++ temp = I915_READ(reg); ++ temp &= ~FDI_LINK_TRAIN_AUTO; ++ temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT; ++ temp &= ~FDI_RX_ENABLE; ++ I915_WRITE(reg, temp); ++ ++ /* enable CPU FDI TX and PCH FDI RX */ ++ reg = FDI_TX_CTL(pipe); ++ temp = I915_READ(reg); ++ temp &= ~FDI_DP_PORT_WIDTH_MASK; ++ temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes); ++ temp |= FDI_LINK_TRAIN_PATTERN_1_IVB; ++ temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK; ++ temp |= snb_b_fdi_train_param[j/2]; ++ temp |= FDI_COMPOSITE_SYNC; ++ I915_WRITE(reg, temp | FDI_TX_ENABLE); ++ ++ I915_WRITE(FDI_RX_MISC(pipe), ++ FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90); ++ ++ reg = FDI_RX_CTL(pipe); ++ temp = I915_READ(reg); ++ temp |= FDI_LINK_TRAIN_PATTERN_1_CPT; ++ temp |= FDI_COMPOSITE_SYNC; ++ I915_WRITE(reg, temp | FDI_RX_ENABLE); ++ ++ POSTING_READ(reg); ++ udelay(1); /* should be 0.5us */ ++ ++ for (i = 0; i < 4; i++) { ++ reg = FDI_RX_IIR(pipe); ++ temp = I915_READ(reg); ++ DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp); ++ ++ if (temp & FDI_RX_BIT_LOCK || ++ (I915_READ(reg) & FDI_RX_BIT_LOCK)) { ++ I915_WRITE(reg, temp | FDI_RX_BIT_LOCK); ++ DRM_DEBUG_KMS("FDI train 1 done, level %i.\n", ++ i); ++ break; ++ } ++ udelay(1); /* should be 0.5us */ ++ } ++ if (i == 4) { ++ DRM_DEBUG_KMS("FDI train 1 fail on vswing %d\n", j / 2); ++ continue; ++ } ++ ++ /* Train 2 */ ++ reg = FDI_TX_CTL(pipe); ++ temp = I915_READ(reg); ++ temp &= ~FDI_LINK_TRAIN_NONE_IVB; ++ temp |= FDI_LINK_TRAIN_PATTERN_2_IVB; ++ I915_WRITE(reg, temp); ++ ++ reg = FDI_RX_CTL(pipe); ++ temp = I915_READ(reg); ++ temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT; ++ temp |= FDI_LINK_TRAIN_PATTERN_2_CPT; ++ I915_WRITE(reg, temp); ++ ++ POSTING_READ(reg); ++ udelay(2); /* should be 1.5us */ ++ ++ for (i = 0; i < 4; i++) { ++ reg = FDI_RX_IIR(pipe); ++ temp = I915_READ(reg); ++ DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp); ++ ++ if (temp & FDI_RX_SYMBOL_LOCK || ++ (I915_READ(reg) & FDI_RX_SYMBOL_LOCK)) { ++ I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK); ++ DRM_DEBUG_KMS("FDI train 2 done, level %i.\n", ++ i); ++ goto train_done; ++ } ++ udelay(2); /* should be 1.5us */ ++ } ++ if (i == 4) ++ DRM_DEBUG_KMS("FDI train 2 fail on vswing %d\n", j / 2); ++ } ++ ++train_done: ++ DRM_DEBUG_KMS("FDI train done.\n"); ++} ++ ++static void ironlake_fdi_pll_enable(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev); ++ int pipe = intel_crtc->pipe; ++ i915_reg_t reg; ++ u32 temp; ++ ++ /* enable PCH FDI RX PLL, wait warmup plus DMI latency */ ++ reg = FDI_RX_CTL(pipe); ++ temp = I915_READ(reg); ++ temp &= ~(FDI_DP_PORT_WIDTH_MASK | (0x7 << 16)); ++ temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes); ++ temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11; ++ I915_WRITE(reg, temp | FDI_RX_PLL_ENABLE); ++ ++ POSTING_READ(reg); ++ udelay(200); ++ ++ /* Switch from Rawclk to PCDclk */ ++ temp = I915_READ(reg); ++ I915_WRITE(reg, temp | FDI_PCDCLK); ++ ++ POSTING_READ(reg); ++ udelay(200); ++ ++ /* Enable CPU FDI TX PLL, always on for Ironlake */ ++ reg = FDI_TX_CTL(pipe); ++ temp = I915_READ(reg); ++ if ((temp & FDI_TX_PLL_ENABLE) == 0) { ++ I915_WRITE(reg, temp | FDI_TX_PLL_ENABLE); ++ ++ POSTING_READ(reg); ++ udelay(100); ++ } ++} ++ ++static void ironlake_fdi_pll_disable(struct intel_crtc *intel_crtc) ++{ ++ struct drm_device *dev = intel_crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ int pipe = intel_crtc->pipe; ++ i915_reg_t reg; ++ u32 temp; ++ ++ /* Switch from PCDclk to Rawclk */ ++ reg = FDI_RX_CTL(pipe); ++ temp = I915_READ(reg); ++ I915_WRITE(reg, temp & ~FDI_PCDCLK); ++ ++ /* Disable CPU FDI TX PLL */ ++ reg = FDI_TX_CTL(pipe); ++ temp = I915_READ(reg); ++ I915_WRITE(reg, temp & ~FDI_TX_PLL_ENABLE); ++ ++ POSTING_READ(reg); ++ udelay(100); ++ ++ reg = FDI_RX_CTL(pipe); ++ temp = I915_READ(reg); ++ I915_WRITE(reg, temp & ~FDI_RX_PLL_ENABLE); ++ ++ /* Wait for the clocks to turn off. */ ++ POSTING_READ(reg); ++ udelay(100); ++} ++ ++static void ironlake_fdi_disable(struct drm_crtc *crtc) ++{ ++ struct drm_device *dev = crtc->dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_crtc *intel_crtc = to_intel_crtc(crtc); ++ int pipe = intel_crtc->pipe; ++ i915_reg_t reg; ++ u32 temp; ++ ++ /* disable CPU FDI tx and PCH FDI rx */ ++ reg = FDI_TX_CTL(pipe); ++ temp = I915_READ(reg); ++ I915_WRITE(reg, temp & ~FDI_TX_ENABLE); ++ POSTING_READ(reg); ++ ++ reg = FDI_RX_CTL(pipe); ++ temp = I915_READ(reg); ++ temp &= ~(0x7 << 16); ++ temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11; ++ I915_WRITE(reg, temp & ~FDI_RX_ENABLE); ++ ++ POSTING_READ(reg); ++ udelay(100); ++ ++ /* Ironlake workaround, disable clock pointer after downing FDI */ ++ if (HAS_PCH_IBX(dev_priv)) ++ I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR); ++ ++ /* still set train pattern 1 */ ++ reg = FDI_TX_CTL(pipe); ++ temp = I915_READ(reg); ++ temp &= ~FDI_LINK_TRAIN_NONE; ++ temp |= FDI_LINK_TRAIN_PATTERN_1; ++ I915_WRITE(reg, temp); ++ ++ reg = FDI_RX_CTL(pipe); ++ temp = I915_READ(reg); ++ if (HAS_PCH_CPT(dev_priv)) { ++ temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT; ++ temp |= FDI_LINK_TRAIN_PATTERN_1_CPT; ++ } else { ++ temp &= ~FDI_LINK_TRAIN_NONE; ++ temp |= FDI_LINK_TRAIN_PATTERN_1; ++ } ++ /* BPC in FDI rx is consistent with that in PIPECONF */ ++ temp &= ~(0x07 << 16); ++ temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11; ++ I915_WRITE(reg, temp); ++ ++ POSTING_READ(reg); ++ udelay(100); ++} ++ ++bool intel_has_pending_fb_unpin(struct drm_i915_private *dev_priv) ++{ ++ struct drm_crtc *crtc; ++ bool cleanup_done; ++ ++ drm_for_each_crtc(crtc, &dev_priv->drm) { ++ struct drm_crtc_commit *commit; ++ spin_lock(&crtc->commit_lock); ++ commit = list_first_entry_or_null(&crtc->commit_list, ++ struct drm_crtc_commit, commit_entry); ++ cleanup_done = commit ? ++ try_wait_for_completion(&commit->cleanup_done) : true; ++ spin_unlock(&crtc->commit_lock); ++ ++ if (cleanup_done) ++ continue; ++ ++ drm_crtc_wait_one_vblank(crtc); ++ ++ return true; ++ } ++ ++ return false; ++} ++ ++void lpt_disable_iclkip(struct drm_i915_private *dev_priv) ++{ ++ u32 temp; ++ ++ I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_GATE); ++ ++ mutex_lock(&dev_priv->sb_lock); ++ ++ temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK); ++ temp |= SBI_SSCCTL_DISABLE; ++ intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK); ++ ++ mutex_unlock(&dev_priv->sb_lock); ++} ++ ++/* Program iCLKIP clock to the desired frequency */ ++static void lpt_program_iclkip(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ int clock = crtc_state->base.adjusted_mode.crtc_clock; ++ u32 divsel, phaseinc, auxdiv, phasedir = 0; ++ u32 temp; ++ ++ lpt_disable_iclkip(dev_priv); ++ ++ /* The iCLK virtual clock root frequency is in MHz, ++ * but the adjusted_mode->crtc_clock in in KHz. To get the ++ * divisors, it is necessary to divide one by another, so we ++ * convert the virtual clock precision to KHz here for higher ++ * precision. ++ */ ++ for (auxdiv = 0; auxdiv < 2; auxdiv++) { ++ u32 iclk_virtual_root_freq = 172800 * 1000; ++ u32 iclk_pi_range = 64; ++ u32 desired_divisor; ++ ++ desired_divisor = DIV_ROUND_CLOSEST(iclk_virtual_root_freq, ++ clock << auxdiv); ++ divsel = (desired_divisor / iclk_pi_range) - 2; ++ phaseinc = desired_divisor % iclk_pi_range; ++ ++ /* ++ * Near 20MHz is a corner case which is ++ * out of range for the 7-bit divisor ++ */ ++ if (divsel <= 0x7f) ++ break; ++ } ++ ++ /* This should not happen with any sane values */ ++ WARN_ON(SBI_SSCDIVINTPHASE_DIVSEL(divsel) & ++ ~SBI_SSCDIVINTPHASE_DIVSEL_MASK); ++ WARN_ON(SBI_SSCDIVINTPHASE_DIR(phasedir) & ++ ~SBI_SSCDIVINTPHASE_INCVAL_MASK); ++ ++ DRM_DEBUG_KMS("iCLKIP clock: found settings for %dKHz refresh rate: auxdiv=%x, divsel=%x, phasedir=%x, phaseinc=%x\n", ++ clock, ++ auxdiv, ++ divsel, ++ phasedir, ++ phaseinc); ++ ++ mutex_lock(&dev_priv->sb_lock); ++ ++ /* Program SSCDIVINTPHASE6 */ ++ temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK); ++ temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK; ++ temp |= SBI_SSCDIVINTPHASE_DIVSEL(divsel); ++ temp &= ~SBI_SSCDIVINTPHASE_INCVAL_MASK; ++ temp |= SBI_SSCDIVINTPHASE_INCVAL(phaseinc); ++ temp |= SBI_SSCDIVINTPHASE_DIR(phasedir); ++ temp |= SBI_SSCDIVINTPHASE_PROPAGATE; ++ intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE6, temp, SBI_ICLK); ++ ++ /* Program SSCAUXDIV */ ++ temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK); ++ temp &= ~SBI_SSCAUXDIV_FINALDIV2SEL(1); ++ temp |= SBI_SSCAUXDIV_FINALDIV2SEL(auxdiv); ++ intel_sbi_write(dev_priv, SBI_SSCAUXDIV6, temp, SBI_ICLK); ++ ++ /* Enable modulator and associated divider */ ++ temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK); ++ temp &= ~SBI_SSCCTL_DISABLE; ++ intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK); ++ ++ mutex_unlock(&dev_priv->sb_lock); ++ ++ /* Wait for initialization time */ ++ udelay(24); ++ ++ I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_UNGATE); ++} ++ ++int lpt_get_iclkip(struct drm_i915_private *dev_priv) ++{ ++ u32 divsel, phaseinc, auxdiv; ++ u32 iclk_virtual_root_freq = 172800 * 1000; ++ u32 iclk_pi_range = 64; ++ u32 desired_divisor; ++ u32 temp; ++ ++ if ((I915_READ(PIXCLK_GATE) & PIXCLK_GATE_UNGATE) == 0) ++ return 0; ++ ++ mutex_lock(&dev_priv->sb_lock); ++ ++ temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK); ++ if (temp & SBI_SSCCTL_DISABLE) { ++ mutex_unlock(&dev_priv->sb_lock); ++ return 0; ++ } ++ ++ temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK); ++ divsel = (temp & SBI_SSCDIVINTPHASE_DIVSEL_MASK) >> ++ SBI_SSCDIVINTPHASE_DIVSEL_SHIFT; ++ phaseinc = (temp & SBI_SSCDIVINTPHASE_INCVAL_MASK) >> ++ SBI_SSCDIVINTPHASE_INCVAL_SHIFT; ++ ++ temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK); ++ auxdiv = (temp & SBI_SSCAUXDIV_FINALDIV2SEL_MASK) >> ++ SBI_SSCAUXDIV_FINALDIV2SEL_SHIFT; ++ ++ mutex_unlock(&dev_priv->sb_lock); ++ ++ desired_divisor = (divsel + 2) * iclk_pi_range + phaseinc; ++ ++ return DIV_ROUND_CLOSEST(iclk_virtual_root_freq, ++ desired_divisor << auxdiv); ++} ++ ++static void ironlake_pch_transcoder_set_timings(const struct intel_crtc_state *crtc_state, ++ enum pipe pch_transcoder) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; ++ ++ I915_WRITE(PCH_TRANS_HTOTAL(pch_transcoder), ++ I915_READ(HTOTAL(cpu_transcoder))); ++ I915_WRITE(PCH_TRANS_HBLANK(pch_transcoder), ++ I915_READ(HBLANK(cpu_transcoder))); ++ I915_WRITE(PCH_TRANS_HSYNC(pch_transcoder), ++ I915_READ(HSYNC(cpu_transcoder))); ++ ++ I915_WRITE(PCH_TRANS_VTOTAL(pch_transcoder), ++ I915_READ(VTOTAL(cpu_transcoder))); ++ I915_WRITE(PCH_TRANS_VBLANK(pch_transcoder), ++ I915_READ(VBLANK(cpu_transcoder))); ++ I915_WRITE(PCH_TRANS_VSYNC(pch_transcoder), ++ I915_READ(VSYNC(cpu_transcoder))); ++ I915_WRITE(PCH_TRANS_VSYNCSHIFT(pch_transcoder), ++ I915_READ(VSYNCSHIFT(cpu_transcoder))); ++} ++ ++static void cpt_set_fdi_bc_bifurcation(struct drm_i915_private *dev_priv, bool enable) ++{ ++ u32 temp; ++ ++ temp = I915_READ(SOUTH_CHICKEN1); ++ if (!!(temp & FDI_BC_BIFURCATION_SELECT) == enable) ++ return; ++ ++ WARN_ON(I915_READ(FDI_RX_CTL(PIPE_B)) & FDI_RX_ENABLE); ++ WARN_ON(I915_READ(FDI_RX_CTL(PIPE_C)) & FDI_RX_ENABLE); ++ ++ temp &= ~FDI_BC_BIFURCATION_SELECT; ++ if (enable) ++ temp |= FDI_BC_BIFURCATION_SELECT; ++ ++ DRM_DEBUG_KMS("%sabling fdi C rx\n", enable ? "en" : "dis"); ++ I915_WRITE(SOUTH_CHICKEN1, temp); ++ POSTING_READ(SOUTH_CHICKEN1); ++} ++ ++static void ivybridge_update_fdi_bc_bifurcation(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ ++ switch (crtc->pipe) { ++ case PIPE_A: ++ break; ++ case PIPE_B: ++ if (crtc_state->fdi_lanes > 2) ++ cpt_set_fdi_bc_bifurcation(dev_priv, false); ++ else ++ cpt_set_fdi_bc_bifurcation(dev_priv, true); ++ ++ break; ++ case PIPE_C: ++ cpt_set_fdi_bc_bifurcation(dev_priv, true); ++ ++ break; ++ default: ++ BUG(); ++ } ++} ++ ++/* ++ * Finds the encoder associated with the given CRTC. This can only be ++ * used when we know that the CRTC isn't feeding multiple encoders! ++ */ ++static struct intel_encoder * ++intel_get_crtc_new_encoder(const struct intel_atomic_state *state, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ const struct drm_connector_state *connector_state; ++ const struct drm_connector *connector; ++ struct intel_encoder *encoder = NULL; ++ int num_encoders = 0; ++ int i; ++ ++ for_each_new_connector_in_state(&state->base, connector, connector_state, i) { ++ if (connector_state->crtc != &crtc->base) ++ continue; ++ ++ encoder = to_intel_encoder(connector_state->best_encoder); ++ num_encoders++; ++ } ++ ++ WARN(num_encoders != 1, "%d encoders for pipe %c\n", ++ num_encoders, pipe_name(crtc->pipe)); ++ ++ return encoder; ++} ++ ++/* ++ * Enable PCH resources required for PCH ports: ++ * - PCH PLLs ++ * - FDI training & RX/TX ++ * - update transcoder timings ++ * - DP transcoding bits ++ * - transcoder ++ */ ++static void ironlake_pch_enable(const struct intel_atomic_state *state, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ int pipe = crtc->pipe; ++ u32 temp; ++ ++ assert_pch_transcoder_disabled(dev_priv, pipe); ++ ++ if (IS_IVYBRIDGE(dev_priv)) ++ ivybridge_update_fdi_bc_bifurcation(crtc_state); ++ ++ /* Write the TU size bits before fdi link training, so that error ++ * detection works. */ ++ I915_WRITE(FDI_RX_TUSIZE1(pipe), ++ I915_READ(PIPE_DATA_M1(pipe)) & TU_SIZE_MASK); ++ ++ /* For PCH output, training FDI link */ ++ dev_priv->display.fdi_link_train(crtc, crtc_state); ++ ++ /* We need to program the right clock selection before writing the pixel ++ * mutliplier into the DPLL. */ ++ if (HAS_PCH_CPT(dev_priv)) { ++ u32 sel; ++ ++ temp = I915_READ(PCH_DPLL_SEL); ++ temp |= TRANS_DPLL_ENABLE(pipe); ++ sel = TRANS_DPLLB_SEL(pipe); ++ if (crtc_state->shared_dpll == ++ intel_get_shared_dpll_by_id(dev_priv, DPLL_ID_PCH_PLL_B)) ++ temp |= sel; ++ else ++ temp &= ~sel; ++ I915_WRITE(PCH_DPLL_SEL, temp); ++ } ++ ++ /* XXX: pch pll's can be enabled any time before we enable the PCH ++ * transcoder, and we actually should do this to not upset any PCH ++ * transcoder that already use the clock when we share it. ++ * ++ * Note that enable_shared_dpll tries to do the right thing, but ++ * get_shared_dpll unconditionally resets the pll - we need that to have ++ * the right LVDS enable sequence. */ ++ intel_enable_shared_dpll(crtc_state); ++ ++ /* set transcoder timing, panel must allow it */ ++ assert_panel_unlocked(dev_priv, pipe); ++ ironlake_pch_transcoder_set_timings(crtc_state, pipe); ++ ++ intel_fdi_normal_train(crtc); ++ ++ /* For PCH DP, enable TRANS_DP_CTL */ ++ if (HAS_PCH_CPT(dev_priv) && ++ intel_crtc_has_dp_encoder(crtc_state)) { ++ const struct drm_display_mode *adjusted_mode = ++ &crtc_state->base.adjusted_mode; ++ u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5; ++ i915_reg_t reg = TRANS_DP_CTL(pipe); ++ enum port port; ++ ++ temp = I915_READ(reg); ++ temp &= ~(TRANS_DP_PORT_SEL_MASK | ++ TRANS_DP_SYNC_MASK | ++ TRANS_DP_BPC_MASK); ++ temp |= TRANS_DP_OUTPUT_ENABLE; ++ temp |= bpc << 9; /* same format but at 11:9 */ ++ ++ if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC) ++ temp |= TRANS_DP_HSYNC_ACTIVE_HIGH; ++ if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC) ++ temp |= TRANS_DP_VSYNC_ACTIVE_HIGH; ++ ++ port = intel_get_crtc_new_encoder(state, crtc_state)->port; ++ WARN_ON(port < PORT_B || port > PORT_D); ++ temp |= TRANS_DP_PORT_SEL(port); ++ ++ I915_WRITE(reg, temp); ++ } ++ ++ ironlake_enable_pch_transcoder(crtc_state); ++} ++ ++static void lpt_pch_enable(const struct intel_atomic_state *state, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; ++ ++ assert_pch_transcoder_disabled(dev_priv, PIPE_A); ++ ++ lpt_program_iclkip(crtc_state); ++ ++ /* Set transcoder timing. */ ++ ironlake_pch_transcoder_set_timings(crtc_state, PIPE_A); ++ ++ lpt_enable_pch_transcoder(dev_priv, cpu_transcoder); ++} ++ ++static void cpt_verify_modeset(struct drm_device *dev, int pipe) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ i915_reg_t dslreg = PIPEDSL(pipe); ++ u32 temp; ++ ++ temp = I915_READ(dslreg); ++ udelay(500); ++ if (wait_for(I915_READ(dslreg) != temp, 5)) { ++ if (wait_for(I915_READ(dslreg) != temp, 5)) ++ DRM_ERROR("mode set failed: pipe %c stuck\n", pipe_name(pipe)); ++ } ++} ++ ++/* ++ * The hardware phase 0.0 refers to the center of the pixel. ++ * We want to start from the top/left edge which is phase ++ * -0.5. That matches how the hardware calculates the scaling ++ * factors (from top-left of the first pixel to bottom-right ++ * of the last pixel, as opposed to the pixel centers). ++ * ++ * For 4:2:0 subsampled chroma planes we obviously have to ++ * adjust that so that the chroma sample position lands in ++ * the right spot. ++ * ++ * Note that for packed YCbCr 4:2:2 formats there is no way to ++ * control chroma siting. The hardware simply replicates the ++ * chroma samples for both of the luma samples, and thus we don't ++ * actually get the expected MPEG2 chroma siting convention :( ++ * The same behaviour is observed on pre-SKL platforms as well. ++ * ++ * Theory behind the formula (note that we ignore sub-pixel ++ * source coordinates): ++ * s = source sample position ++ * d = destination sample position ++ * ++ * Downscaling 4:1: ++ * -0.5 ++ * | 0.0 ++ * | | 1.5 (initial phase) ++ * | | | ++ * v v v ++ * | s | s | s | s | ++ * | d | ++ * ++ * Upscaling 1:4: ++ * -0.5 ++ * | -0.375 (initial phase) ++ * | | 0.0 ++ * | | | ++ * v v v ++ * | s | ++ * | d | d | d | d | ++ */ ++u16 skl_scaler_calc_phase(int sub, int scale, bool chroma_cosited) ++{ ++ int phase = -0x8000; ++ u16 trip = 0; ++ ++ if (chroma_cosited) ++ phase += (sub - 1) * 0x8000 / sub; ++ ++ phase += scale / (2 * sub); ++ ++ /* ++ * Hardware initial phase limited to [-0.5:1.5]. ++ * Since the max hardware scale factor is 3.0, we ++ * should never actually excdeed 1.0 here. ++ */ ++ WARN_ON(phase < -0x8000 || phase > 0x18000); ++ ++ if (phase < 0) ++ phase = 0x10000 + phase; ++ else ++ trip = PS_PHASE_TRIP; ++ ++ return ((phase >> 2) & PS_PHASE_MASK) | trip; ++} ++ ++static int ++skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach, ++ unsigned int scaler_user, int *scaler_id, ++ int src_w, int src_h, int dst_w, int dst_h, ++ const struct drm_format_info *format, bool need_scaler) ++{ ++ struct intel_crtc_scaler_state *scaler_state = ++ &crtc_state->scaler_state; ++ struct intel_crtc *intel_crtc = ++ to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev); ++ const struct drm_display_mode *adjusted_mode = ++ &crtc_state->base.adjusted_mode; ++ ++ /* ++ * Src coordinates are already rotated by 270 degrees for ++ * the 90/270 degree plane rotation cases (to match the ++ * GTT mapping), hence no need to account for rotation here. ++ */ ++ if (src_w != dst_w || src_h != dst_h) ++ need_scaler = true; ++ ++ /* ++ * Scaling/fitting not supported in IF-ID mode in GEN9+ ++ * TODO: Interlace fetch mode doesn't support YUV420 planar formats. ++ * Once NV12 is enabled, handle it here while allocating scaler ++ * for NV12. ++ */ ++ if (INTEL_GEN(dev_priv) >= 9 && crtc_state->base.enable && ++ need_scaler && adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) { ++ DRM_DEBUG_KMS("Pipe/Plane scaling not supported with IF-ID mode\n"); ++ return -EINVAL; ++ } ++ ++ /* ++ * if plane is being disabled or scaler is no more required or force detach ++ * - free scaler binded to this plane/crtc ++ * - in order to do this, update crtc->scaler_usage ++ * ++ * Here scaler state in crtc_state is set free so that ++ * scaler can be assigned to other user. Actual register ++ * update to free the scaler is done in plane/panel-fit programming. ++ * For this purpose crtc/plane_state->scaler_id isn't reset here. ++ */ ++ if (force_detach || !need_scaler) { ++ if (*scaler_id >= 0) { ++ scaler_state->scaler_users &= ~(1 << scaler_user); ++ scaler_state->scalers[*scaler_id].in_use = 0; ++ ++ DRM_DEBUG_KMS("scaler_user index %u.%u: " ++ "Staged freeing scaler id %d scaler_users = 0x%x\n", ++ intel_crtc->pipe, scaler_user, *scaler_id, ++ scaler_state->scaler_users); ++ *scaler_id = -1; ++ } ++ return 0; ++ } ++ ++ if (format && is_planar_yuv_format(format->format) && ++ (src_h < SKL_MIN_YUV_420_SRC_H || src_w < SKL_MIN_YUV_420_SRC_W)) { ++ DRM_DEBUG_KMS("Planar YUV: src dimensions not met\n"); ++ return -EINVAL; ++ } ++ ++ /* range checks */ ++ if (src_w < SKL_MIN_SRC_W || src_h < SKL_MIN_SRC_H || ++ dst_w < SKL_MIN_DST_W || dst_h < SKL_MIN_DST_H || ++ (INTEL_GEN(dev_priv) >= 11 && ++ (src_w > ICL_MAX_SRC_W || src_h > ICL_MAX_SRC_H || ++ dst_w > ICL_MAX_DST_W || dst_h > ICL_MAX_DST_H)) || ++ (INTEL_GEN(dev_priv) < 11 && ++ (src_w > SKL_MAX_SRC_W || src_h > SKL_MAX_SRC_H || ++ dst_w > SKL_MAX_DST_W || dst_h > SKL_MAX_DST_H))) { ++ DRM_DEBUG_KMS("scaler_user index %u.%u: src %ux%u dst %ux%u " ++ "size is out of scaler range\n", ++ intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h); ++ return -EINVAL; ++ } ++ ++ /* mark this plane as a scaler user in crtc_state */ ++ scaler_state->scaler_users |= (1 << scaler_user); ++ DRM_DEBUG_KMS("scaler_user index %u.%u: " ++ "staged scaling request for %ux%u->%ux%u scaler_users = 0x%x\n", ++ intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h, ++ scaler_state->scaler_users); ++ ++ return 0; ++} ++ ++/** ++ * skl_update_scaler_crtc - Stages update to scaler state for a given crtc. ++ * ++ * @state: crtc's scaler state ++ * ++ * Return ++ * 0 - scaler_usage updated successfully ++ * error - requested scaling cannot be supported or other error condition ++ */ ++int skl_update_scaler_crtc(struct intel_crtc_state *state) ++{ ++ const struct drm_display_mode *adjusted_mode = &state->base.adjusted_mode; ++ bool need_scaler = false; ++ ++ if (state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420) ++ need_scaler = true; ++ ++ return skl_update_scaler(state, !state->base.active, SKL_CRTC_INDEX, ++ &state->scaler_state.scaler_id, ++ state->pipe_src_w, state->pipe_src_h, ++ adjusted_mode->crtc_hdisplay, ++ adjusted_mode->crtc_vdisplay, NULL, need_scaler); ++} ++ ++/** ++ * skl_update_scaler_plane - Stages update to scaler state for a given plane. ++ * @crtc_state: crtc's scaler state ++ * @plane_state: atomic plane state to update ++ * ++ * Return ++ * 0 - scaler_usage updated successfully ++ * error - requested scaling cannot be supported or other error condition ++ */ ++static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state, ++ struct intel_plane_state *plane_state) ++{ ++ struct intel_plane *intel_plane = ++ to_intel_plane(plane_state->base.plane); ++ struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev); ++ struct drm_framebuffer *fb = plane_state->base.fb; ++ int ret; ++ bool force_detach = !fb || !plane_state->base.visible; ++ bool need_scaler = false; ++ ++ /* Pre-gen11 and SDR planes always need a scaler for planar formats. */ ++ if (!icl_is_hdr_plane(dev_priv, intel_plane->id) && ++ fb && is_planar_yuv_format(fb->format->format)) ++ need_scaler = true; ++ ++ ret = skl_update_scaler(crtc_state, force_detach, ++ drm_plane_index(&intel_plane->base), ++ &plane_state->scaler_id, ++ drm_rect_width(&plane_state->base.src) >> 16, ++ drm_rect_height(&plane_state->base.src) >> 16, ++ drm_rect_width(&plane_state->base.dst), ++ drm_rect_height(&plane_state->base.dst), ++ fb ? fb->format : NULL, need_scaler); ++ ++ if (ret || plane_state->scaler_id < 0) ++ return ret; ++ ++ /* check colorkey */ ++ if (plane_state->ckey.flags) { ++ DRM_DEBUG_KMS("[PLANE:%d:%s] scaling with color key not allowed", ++ intel_plane->base.base.id, ++ intel_plane->base.name); ++ return -EINVAL; ++ } ++ ++ /* Check src format */ ++ switch (fb->format->format) { ++ case DRM_FORMAT_RGB565: ++ case DRM_FORMAT_XBGR8888: ++ case DRM_FORMAT_XRGB8888: ++ case DRM_FORMAT_ABGR8888: ++ case DRM_FORMAT_ARGB8888: ++ case DRM_FORMAT_XRGB2101010: ++ case DRM_FORMAT_XBGR2101010: ++ case DRM_FORMAT_XBGR16161616F: ++ case DRM_FORMAT_ABGR16161616F: ++ case DRM_FORMAT_XRGB16161616F: ++ case DRM_FORMAT_ARGB16161616F: ++ case DRM_FORMAT_YUYV: ++ case DRM_FORMAT_YVYU: ++ case DRM_FORMAT_UYVY: ++ case DRM_FORMAT_VYUY: ++ case DRM_FORMAT_NV12: ++ case DRM_FORMAT_P010: ++ case DRM_FORMAT_P012: ++ case DRM_FORMAT_P016: ++ case DRM_FORMAT_Y210: ++ case DRM_FORMAT_Y212: ++ case DRM_FORMAT_Y216: ++ case DRM_FORMAT_XVYU2101010: ++ case DRM_FORMAT_XVYU12_16161616: ++ case DRM_FORMAT_XVYU16161616: ++ break; ++ default: ++ DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d unsupported scaling format 0x%x\n", ++ intel_plane->base.base.id, intel_plane->base.name, ++ fb->base.id, fb->format->format); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static void skylake_scaler_disable(struct intel_crtc *crtc) ++{ ++ int i; ++ ++ for (i = 0; i < crtc->num_scalers; i++) ++ skl_detach_scaler(crtc, i); ++} ++ ++static void skylake_pfit_enable(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ const struct intel_crtc_scaler_state *scaler_state = ++ &crtc_state->scaler_state; ++ ++ if (crtc_state->pch_pfit.enabled) { ++ u16 uv_rgb_hphase, uv_rgb_vphase; ++ int pfit_w, pfit_h, hscale, vscale; ++ int id; ++ ++ if (WARN_ON(crtc_state->scaler_state.scaler_id < 0)) ++ return; ++ ++ pfit_w = (crtc_state->pch_pfit.size >> 16) & 0xFFFF; ++ pfit_h = crtc_state->pch_pfit.size & 0xFFFF; ++ ++ hscale = (crtc_state->pipe_src_w << 16) / pfit_w; ++ vscale = (crtc_state->pipe_src_h << 16) / pfit_h; ++ ++ uv_rgb_hphase = skl_scaler_calc_phase(1, hscale, false); ++ uv_rgb_vphase = skl_scaler_calc_phase(1, vscale, false); ++ ++ id = scaler_state->scaler_id; ++ I915_WRITE(SKL_PS_CTRL(pipe, id), PS_SCALER_EN | ++ PS_FILTER_MEDIUM | scaler_state->scalers[id].mode); ++ I915_WRITE_FW(SKL_PS_VPHASE(pipe, id), ++ PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_vphase)); ++ I915_WRITE_FW(SKL_PS_HPHASE(pipe, id), ++ PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_hphase)); ++ I915_WRITE(SKL_PS_WIN_POS(pipe, id), crtc_state->pch_pfit.pos); ++ I915_WRITE(SKL_PS_WIN_SZ(pipe, id), crtc_state->pch_pfit.size); ++ } ++} ++ ++static void ironlake_pfit_enable(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ int pipe = crtc->pipe; ++ ++ if (crtc_state->pch_pfit.enabled) { ++ /* Force use of hard-coded filter coefficients ++ * as some pre-programmed values are broken, ++ * e.g. x201. ++ */ ++ if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv)) ++ I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3 | ++ PF_PIPE_SEL_IVB(pipe)); ++ else ++ I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3); ++ I915_WRITE(PF_WIN_POS(pipe), crtc_state->pch_pfit.pos); ++ I915_WRITE(PF_WIN_SZ(pipe), crtc_state->pch_pfit.size); ++ } ++} ++ ++void hsw_enable_ips(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ ++ if (!crtc_state->ips_enabled) ++ return; ++ ++ /* ++ * We can only enable IPS after we enable a plane and wait for a vblank ++ * This function is called from post_plane_update, which is run after ++ * a vblank wait. ++ */ ++ WARN_ON(!(crtc_state->active_planes & ~BIT(PLANE_CURSOR))); ++ ++ if (IS_BROADWELL(dev_priv)) { ++ mutex_lock(&dev_priv->pcu_lock); ++ WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, ++ IPS_ENABLE | IPS_PCODE_CONTROL)); ++ mutex_unlock(&dev_priv->pcu_lock); ++ /* Quoting Art Runyan: "its not safe to expect any particular ++ * value in IPS_CTL bit 31 after enabling IPS through the ++ * mailbox." Moreover, the mailbox may return a bogus state, ++ * so we need to just enable it and continue on. ++ */ ++ } else { ++ I915_WRITE(IPS_CTL, IPS_ENABLE); ++ /* The bit only becomes 1 in the next vblank, so this wait here ++ * is essentially intel_wait_for_vblank. If we don't have this ++ * and don't wait for vblanks until the end of crtc_enable, then ++ * the HW state readout code will complain that the expected ++ * IPS_CTL value is not the one we read. */ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ IPS_CTL, IPS_ENABLE, IPS_ENABLE, ++ 50)) ++ DRM_ERROR("Timed out waiting for IPS enable\n"); ++ } ++} ++ ++void hsw_disable_ips(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ ++ if (!crtc_state->ips_enabled) ++ return; ++ ++ if (IS_BROADWELL(dev_priv)) { ++ mutex_lock(&dev_priv->pcu_lock); ++ WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0)); ++ mutex_unlock(&dev_priv->pcu_lock); ++ /* ++ * Wait for PCODE to finish disabling IPS. The BSpec specified ++ * 42ms timeout value leads to occasional timeouts so use 100ms ++ * instead. ++ */ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ IPS_CTL, IPS_ENABLE, 0, ++ 100)) ++ DRM_ERROR("Timed out waiting for IPS disable\n"); ++ } else { ++ I915_WRITE(IPS_CTL, 0); ++ POSTING_READ(IPS_CTL); ++ } ++ ++ /* We need to wait for a vblank before we can disable the plane. */ ++ intel_wait_for_vblank(dev_priv, crtc->pipe); ++} ++ ++static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc) ++{ ++ if (intel_crtc->overlay) { ++ struct drm_device *dev = intel_crtc->base.dev; ++ ++ mutex_lock(&dev->struct_mutex); ++ (void) intel_overlay_switch_off(intel_crtc->overlay); ++ mutex_unlock(&dev->struct_mutex); ++ } ++ ++ /* Let userspace switch the overlay on again. In most cases userspace ++ * has to recompute where to put it anyway. ++ */ ++} ++ ++/** ++ * intel_post_enable_primary - Perform operations after enabling primary plane ++ * @crtc: the CRTC whose primary plane was just enabled ++ * @new_crtc_state: the enabling state ++ * ++ * Performs potentially sleeping operations that must be done after the primary ++ * plane is enabled, such as updating FBC and IPS. Note that this may be ++ * called due to an explicit primary plane update, or due to an implicit ++ * re-enable that is caused when a sprite plane is updated to no longer ++ * completely hide the primary plane. ++ */ ++static void ++intel_post_enable_primary(struct drm_crtc *crtc, ++ const struct intel_crtc_state *new_crtc_state) ++{ ++ struct drm_device *dev = crtc->dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_crtc *intel_crtc = to_intel_crtc(crtc); ++ int pipe = intel_crtc->pipe; ++ ++ /* ++ * Gen2 reports pipe underruns whenever all planes are disabled. ++ * So don't enable underrun reporting before at least some planes ++ * are enabled. ++ * FIXME: Need to fix the logic to work when we turn off all planes ++ * but leave the pipe running. ++ */ ++ if (IS_GEN(dev_priv, 2)) ++ intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); ++ ++ /* Underruns don't always raise interrupts, so check manually. */ ++ intel_check_cpu_fifo_underruns(dev_priv); ++ intel_check_pch_fifo_underruns(dev_priv); ++} ++ ++/* FIXME get rid of this and use pre_plane_update */ ++static void ++intel_pre_disable_primary_noatomic(struct drm_crtc *crtc) ++{ ++ struct drm_device *dev = crtc->dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_crtc *intel_crtc = to_intel_crtc(crtc); ++ int pipe = intel_crtc->pipe; ++ ++ /* ++ * Gen2 reports pipe underruns whenever all planes are disabled. ++ * So disable underrun reporting before all the planes get disabled. ++ */ ++ if (IS_GEN(dev_priv, 2)) ++ intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false); ++ ++ hsw_disable_ips(to_intel_crtc_state(crtc->state)); ++ ++ /* ++ * Vblank time updates from the shadow to live plane control register ++ * are blocked if the memory self-refresh mode is active at that ++ * moment. So to make sure the plane gets truly disabled, disable ++ * first the self-refresh mode. The self-refresh enable bit in turn ++ * will be checked/applied by the HW only at the next frame start ++ * event which is after the vblank start event, so we need to have a ++ * wait-for-vblank between disabling the plane and the pipe. ++ */ ++ if (HAS_GMCH(dev_priv) && ++ intel_set_memory_cxsr(dev_priv, false)) ++ intel_wait_for_vblank(dev_priv, pipe); ++} ++ ++static bool hsw_pre_update_disable_ips(const struct intel_crtc_state *old_crtc_state, ++ const struct intel_crtc_state *new_crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ ++ if (!old_crtc_state->ips_enabled) ++ return false; ++ ++ if (needs_modeset(&new_crtc_state->base)) ++ return true; ++ ++ /* ++ * Workaround : Do not read or write the pipe palette/gamma data while ++ * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled. ++ * ++ * Disable IPS before we program the LUT. ++ */ ++ if (IS_HASWELL(dev_priv) && ++ (new_crtc_state->base.color_mgmt_changed || ++ new_crtc_state->update_pipe) && ++ new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT) ++ return true; ++ ++ return !new_crtc_state->ips_enabled; ++} ++ ++static bool hsw_post_update_enable_ips(const struct intel_crtc_state *old_crtc_state, ++ const struct intel_crtc_state *new_crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ ++ if (!new_crtc_state->ips_enabled) ++ return false; ++ ++ if (needs_modeset(&new_crtc_state->base)) ++ return true; ++ ++ /* ++ * Workaround : Do not read or write the pipe palette/gamma data while ++ * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled. ++ * ++ * Re-enable IPS after the LUT has been programmed. ++ */ ++ if (IS_HASWELL(dev_priv) && ++ (new_crtc_state->base.color_mgmt_changed || ++ new_crtc_state->update_pipe) && ++ new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT) ++ return true; ++ ++ /* ++ * We can't read out IPS on broadwell, assume the worst and ++ * forcibly enable IPS on the first fastset. ++ */ ++ if (new_crtc_state->update_pipe && ++ old_crtc_state->base.adjusted_mode.private_flags & I915_MODE_FLAG_INHERITED) ++ return true; ++ ++ return !old_crtc_state->ips_enabled; ++} ++ ++static bool needs_nv12_wa(struct drm_i915_private *dev_priv, ++ const struct intel_crtc_state *crtc_state) ++{ ++ if (!crtc_state->nv12_planes) ++ return false; ++ ++ /* WA Display #0827: Gen9:all */ ++ if (IS_GEN(dev_priv, 9) && !IS_GEMINILAKE(dev_priv)) ++ return true; ++ ++ return false; ++} ++ ++static void intel_post_plane_update(struct intel_crtc_state *old_crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc); ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct drm_atomic_state *old_state = old_crtc_state->base.state; ++ struct intel_crtc_state *pipe_config = ++ intel_atomic_get_new_crtc_state(to_intel_atomic_state(old_state), ++ crtc); ++ struct drm_plane *primary = crtc->base.primary; ++ struct drm_plane_state *old_primary_state = ++ drm_atomic_get_old_plane_state(old_state, primary); ++ ++ intel_frontbuffer_flip(to_i915(crtc->base.dev), pipe_config->fb_bits); ++ ++ if (pipe_config->update_wm_post && pipe_config->base.active) ++ intel_update_watermarks(crtc); ++ ++ if (hsw_post_update_enable_ips(old_crtc_state, pipe_config)) ++ hsw_enable_ips(pipe_config); ++ ++ if (old_primary_state) { ++ struct drm_plane_state *new_primary_state = ++ drm_atomic_get_new_plane_state(old_state, primary); ++ ++ intel_fbc_post_update(crtc); ++ ++ if (new_primary_state->visible && ++ (needs_modeset(&pipe_config->base) || ++ !old_primary_state->visible)) ++ intel_post_enable_primary(&crtc->base, pipe_config); ++ } ++ ++ /* Display WA 827 */ ++ if (needs_nv12_wa(dev_priv, old_crtc_state) && ++ !needs_nv12_wa(dev_priv, pipe_config)) { ++ skl_wa_827(dev_priv, crtc->pipe, false); ++ } ++} ++ ++static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc); ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct drm_atomic_state *old_state = old_crtc_state->base.state; ++ struct drm_plane *primary = crtc->base.primary; ++ struct drm_plane_state *old_primary_state = ++ drm_atomic_get_old_plane_state(old_state, primary); ++ bool modeset = needs_modeset(&pipe_config->base); ++ struct intel_atomic_state *old_intel_state = ++ to_intel_atomic_state(old_state); ++ ++ if (hsw_pre_update_disable_ips(old_crtc_state, pipe_config)) ++ hsw_disable_ips(old_crtc_state); ++ ++ if (old_primary_state) { ++ struct intel_plane_state *new_primary_state = ++ intel_atomic_get_new_plane_state(old_intel_state, ++ to_intel_plane(primary)); ++ ++ intel_fbc_pre_update(crtc, pipe_config, new_primary_state); ++ /* ++ * Gen2 reports pipe underruns whenever all planes are disabled. ++ * So disable underrun reporting before all the planes get disabled. ++ */ ++ if (IS_GEN(dev_priv, 2) && old_primary_state->visible && ++ (modeset || !new_primary_state->base.visible)) ++ intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false); ++ } ++ ++ /* Display WA 827 */ ++ if (!needs_nv12_wa(dev_priv, old_crtc_state) && ++ needs_nv12_wa(dev_priv, pipe_config)) { ++ skl_wa_827(dev_priv, crtc->pipe, true); ++ } ++ ++ /* ++ * Vblank time updates from the shadow to live plane control register ++ * are blocked if the memory self-refresh mode is active at that ++ * moment. So to make sure the plane gets truly disabled, disable ++ * first the self-refresh mode. The self-refresh enable bit in turn ++ * will be checked/applied by the HW only at the next frame start ++ * event which is after the vblank start event, so we need to have a ++ * wait-for-vblank between disabling the plane and the pipe. ++ */ ++ if (HAS_GMCH(dev_priv) && old_crtc_state->base.active && ++ pipe_config->disable_cxsr && intel_set_memory_cxsr(dev_priv, false)) ++ intel_wait_for_vblank(dev_priv, crtc->pipe); ++ ++ /* ++ * IVB workaround: must disable low power watermarks for at least ++ * one frame before enabling scaling. LP watermarks can be re-enabled ++ * when scaling is disabled. ++ * ++ * WaCxSRDisabledForSpriteScaling:ivb ++ */ ++ if (pipe_config->disable_lp_wm && ilk_disable_lp_wm(dev) && ++ old_crtc_state->base.active) ++ intel_wait_for_vblank(dev_priv, crtc->pipe); ++ ++ /* ++ * If we're doing a modeset, we're done. No need to do any pre-vblank ++ * watermark programming here. ++ */ ++ if (needs_modeset(&pipe_config->base)) ++ return; ++ ++ /* ++ * For platforms that support atomic watermarks, program the ++ * 'intermediate' watermarks immediately. On pre-gen9 platforms, these ++ * will be the intermediate values that are safe for both pre- and ++ * post- vblank; when vblank happens, the 'active' values will be set ++ * to the final 'target' values and we'll do this again to get the ++ * optimal watermarks. For gen9+ platforms, the values we program here ++ * will be the final target values which will get automatically latched ++ * at vblank time; no further programming will be necessary. ++ * ++ * If a platform hasn't been transitioned to atomic watermarks yet, ++ * we'll continue to update watermarks the old way, if flags tell ++ * us to. ++ */ ++ if (dev_priv->display.initial_watermarks != NULL) ++ dev_priv->display.initial_watermarks(old_intel_state, ++ pipe_config); ++ else if (pipe_config->update_wm_pre) ++ intel_update_watermarks(crtc); ++} ++ ++static void intel_crtc_disable_planes(struct intel_atomic_state *state, ++ struct intel_crtc *crtc) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ const struct intel_crtc_state *new_crtc_state = ++ intel_atomic_get_new_crtc_state(state, crtc); ++ unsigned int update_mask = new_crtc_state->update_planes; ++ const struct intel_plane_state *old_plane_state; ++ struct intel_plane *plane; ++ unsigned fb_bits = 0; ++ int i; ++ ++ intel_crtc_dpms_overlay_disable(crtc); ++ ++ for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) { ++ if (crtc->pipe != plane->pipe || ++ !(update_mask & BIT(plane->id))) ++ continue; ++ ++ intel_disable_plane(plane, new_crtc_state); ++ ++ if (old_plane_state->base.visible) ++ fb_bits |= plane->frontbuffer_bit; ++ } ++ ++ intel_frontbuffer_flip(dev_priv, fb_bits); ++} ++ ++static void intel_encoders_pre_pll_enable(struct drm_crtc *crtc, ++ struct intel_crtc_state *crtc_state, ++ struct drm_atomic_state *old_state) ++{ ++ struct drm_connector_state *conn_state; ++ struct drm_connector *conn; ++ int i; ++ ++ for_each_new_connector_in_state(old_state, conn, conn_state, i) { ++ struct intel_encoder *encoder = ++ to_intel_encoder(conn_state->best_encoder); ++ ++ if (conn_state->crtc != crtc) ++ continue; ++ ++ if (encoder->pre_pll_enable) ++ encoder->pre_pll_enable(encoder, crtc_state, conn_state); ++ } ++} ++ ++static void intel_encoders_pre_enable(struct drm_crtc *crtc, ++ struct intel_crtc_state *crtc_state, ++ struct drm_atomic_state *old_state) ++{ ++ struct drm_connector_state *conn_state; ++ struct drm_connector *conn; ++ int i; ++ ++ for_each_new_connector_in_state(old_state, conn, conn_state, i) { ++ struct intel_encoder *encoder = ++ to_intel_encoder(conn_state->best_encoder); ++ ++ if (conn_state->crtc != crtc) ++ continue; ++ ++ if (encoder->pre_enable) ++ encoder->pre_enable(encoder, crtc_state, conn_state); ++ } ++} ++ ++static void intel_encoders_enable(struct drm_crtc *crtc, ++ struct intel_crtc_state *crtc_state, ++ struct drm_atomic_state *old_state) ++{ ++ struct drm_connector_state *conn_state; ++ struct drm_connector *conn; ++ int i; ++ ++ for_each_new_connector_in_state(old_state, conn, conn_state, i) { ++ struct intel_encoder *encoder = ++ to_intel_encoder(conn_state->best_encoder); ++ ++ if (conn_state->crtc != crtc) ++ continue; ++ ++ if (encoder->enable) ++ encoder->enable(encoder, crtc_state, conn_state); ++ intel_opregion_notify_encoder(encoder, true); ++ } ++} ++ ++static void intel_encoders_disable(struct drm_crtc *crtc, ++ struct intel_crtc_state *old_crtc_state, ++ struct drm_atomic_state *old_state) ++{ ++ struct drm_connector_state *old_conn_state; ++ struct drm_connector *conn; ++ int i; ++ ++ for_each_old_connector_in_state(old_state, conn, old_conn_state, i) { ++ struct intel_encoder *encoder = ++ to_intel_encoder(old_conn_state->best_encoder); ++ ++ if (old_conn_state->crtc != crtc) ++ continue; ++ ++ intel_opregion_notify_encoder(encoder, false); ++ if (encoder->disable) ++ encoder->disable(encoder, old_crtc_state, old_conn_state); ++ } ++} ++ ++static void intel_encoders_post_disable(struct drm_crtc *crtc, ++ struct intel_crtc_state *old_crtc_state, ++ struct drm_atomic_state *old_state) ++{ ++ struct drm_connector_state *old_conn_state; ++ struct drm_connector *conn; ++ int i; ++ ++ for_each_old_connector_in_state(old_state, conn, old_conn_state, i) { ++ struct intel_encoder *encoder = ++ to_intel_encoder(old_conn_state->best_encoder); ++ ++ if (old_conn_state->crtc != crtc) ++ continue; ++ ++ if (encoder->post_disable) ++ encoder->post_disable(encoder, old_crtc_state, old_conn_state); ++ } ++} ++ ++static void intel_encoders_post_pll_disable(struct drm_crtc *crtc, ++ struct intel_crtc_state *old_crtc_state, ++ struct drm_atomic_state *old_state) ++{ ++ struct drm_connector_state *old_conn_state; ++ struct drm_connector *conn; ++ int i; ++ ++ for_each_old_connector_in_state(old_state, conn, old_conn_state, i) { ++ struct intel_encoder *encoder = ++ to_intel_encoder(old_conn_state->best_encoder); ++ ++ if (old_conn_state->crtc != crtc) ++ continue; ++ ++ if (encoder->post_pll_disable) ++ encoder->post_pll_disable(encoder, old_crtc_state, old_conn_state); ++ } ++} ++ ++static void intel_encoders_update_pipe(struct drm_crtc *crtc, ++ struct intel_crtc_state *crtc_state, ++ struct drm_atomic_state *old_state) ++{ ++ struct drm_connector_state *conn_state; ++ struct drm_connector *conn; ++ int i; ++ ++ for_each_new_connector_in_state(old_state, conn, conn_state, i) { ++ struct intel_encoder *encoder = ++ to_intel_encoder(conn_state->best_encoder); ++ ++ if (conn_state->crtc != crtc) ++ continue; ++ ++ if (encoder->update_pipe) ++ encoder->update_pipe(encoder, crtc_state, conn_state); ++ } ++} ++ ++static void intel_disable_primary_plane(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct intel_plane *plane = to_intel_plane(crtc->base.primary); ++ ++ plane->disable_plane(plane, crtc_state); ++} ++ ++static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config, ++ struct drm_atomic_state *old_state) ++{ ++ struct drm_crtc *crtc = pipe_config->base.crtc; ++ struct drm_device *dev = crtc->dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_crtc *intel_crtc = to_intel_crtc(crtc); ++ int pipe = intel_crtc->pipe; ++ struct intel_atomic_state *old_intel_state = ++ to_intel_atomic_state(old_state); ++ ++ if (WARN_ON(intel_crtc->active)) ++ return; ++ ++ /* ++ * Sometimes spurious CPU pipe underruns happen during FDI ++ * training, at least with VGA+HDMI cloning. Suppress them. ++ * ++ * On ILK we get an occasional spurious CPU pipe underruns ++ * between eDP port A enable and vdd enable. Also PCH port ++ * enable seems to result in the occasional CPU pipe underrun. ++ * ++ * Spurious PCH underruns also occur during PCH enabling. ++ */ ++ intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false); ++ intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false); ++ ++ if (pipe_config->has_pch_encoder) ++ intel_prepare_shared_dpll(pipe_config); ++ ++ if (intel_crtc_has_dp_encoder(pipe_config)) ++ intel_dp_set_m_n(pipe_config, M1_N1); ++ ++ intel_set_pipe_timings(pipe_config); ++ intel_set_pipe_src_size(pipe_config); ++ ++ if (pipe_config->has_pch_encoder) { ++ intel_cpu_transcoder_set_m_n(pipe_config, ++ &pipe_config->fdi_m_n, NULL); ++ } ++ ++ ironlake_set_pipeconf(pipe_config); ++ ++ intel_crtc->active = true; ++ ++ intel_encoders_pre_enable(crtc, pipe_config, old_state); ++ ++ if (pipe_config->has_pch_encoder) { ++ /* Note: FDI PLL enabling _must_ be done before we enable the ++ * cpu pipes, hence this is separate from all the other fdi/pch ++ * enabling. */ ++ ironlake_fdi_pll_enable(pipe_config); ++ } else { ++ assert_fdi_tx_disabled(dev_priv, pipe); ++ assert_fdi_rx_disabled(dev_priv, pipe); ++ } ++ ++ ironlake_pfit_enable(pipe_config); ++ ++ /* ++ * On ILK+ LUT must be loaded before the pipe is running but with ++ * clocks enabled ++ */ ++ intel_color_load_luts(pipe_config); ++ intel_color_commit(pipe_config); ++ /* update DSPCNTR to configure gamma for pipe bottom color */ ++ intel_disable_primary_plane(pipe_config); ++ ++ if (dev_priv->display.initial_watermarks != NULL) ++ dev_priv->display.initial_watermarks(old_intel_state, pipe_config); ++ intel_enable_pipe(pipe_config); ++ ++ if (pipe_config->has_pch_encoder) ++ ironlake_pch_enable(old_intel_state, pipe_config); ++ ++ assert_vblank_disabled(crtc); ++ intel_crtc_vblank_on(pipe_config); ++ ++ intel_encoders_enable(crtc, pipe_config, old_state); ++ ++ if (HAS_PCH_CPT(dev_priv)) ++ cpt_verify_modeset(dev, intel_crtc->pipe); ++ ++ /* ++ * Must wait for vblank to avoid spurious PCH FIFO underruns. ++ * And a second vblank wait is needed at least on ILK with ++ * some interlaced HDMI modes. Let's do the double wait always ++ * in case there are more corner cases we don't know about. ++ */ ++ if (pipe_config->has_pch_encoder) { ++ intel_wait_for_vblank(dev_priv, pipe); ++ intel_wait_for_vblank(dev_priv, pipe); ++ } ++ intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); ++ intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true); ++} ++ ++/* IPS only exists on ULT machines and is tied to pipe A. */ ++static bool hsw_crtc_supports_ips(struct intel_crtc *crtc) ++{ ++ return HAS_IPS(to_i915(crtc->base.dev)) && crtc->pipe == PIPE_A; ++} ++ ++static void glk_pipe_scaler_clock_gating_wa(struct drm_i915_private *dev_priv, ++ enum pipe pipe, bool apply) ++{ ++ u32 val = I915_READ(CLKGATE_DIS_PSL(pipe)); ++ u32 mask = DPF_GATING_DIS | DPF_RAM_GATING_DIS | DPFR_GATING_DIS; ++ ++ if (apply) ++ val |= mask; ++ else ++ val &= ~mask; ++ ++ I915_WRITE(CLKGATE_DIS_PSL(pipe), val); ++} ++ ++static void icl_pipe_mbus_enable(struct intel_crtc *crtc) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ u32 val; ++ ++ val = MBUS_DBOX_A_CREDIT(2); ++ val |= MBUS_DBOX_BW_CREDIT(1); ++ val |= MBUS_DBOX_B_CREDIT(8); ++ ++ I915_WRITE(PIPE_MBUS_DBOX_CTL(pipe), val); ++} ++ ++static void haswell_crtc_enable(struct intel_crtc_state *pipe_config, ++ struct drm_atomic_state *old_state) ++{ ++ struct drm_crtc *crtc = pipe_config->base.crtc; ++ struct drm_i915_private *dev_priv = to_i915(crtc->dev); ++ struct intel_crtc *intel_crtc = to_intel_crtc(crtc); ++ int pipe = intel_crtc->pipe, hsw_workaround_pipe; ++ enum transcoder cpu_transcoder = pipe_config->cpu_transcoder; ++ struct intel_atomic_state *old_intel_state = ++ to_intel_atomic_state(old_state); ++ bool psl_clkgate_wa; ++ ++ if (WARN_ON(intel_crtc->active)) ++ return; ++ ++ intel_encoders_pre_pll_enable(crtc, pipe_config, old_state); ++ ++ if (pipe_config->shared_dpll) ++ intel_enable_shared_dpll(pipe_config); ++ ++ intel_encoders_pre_enable(crtc, pipe_config, old_state); ++ ++ if (intel_crtc_has_dp_encoder(pipe_config)) ++ intel_dp_set_m_n(pipe_config, M1_N1); ++ ++ if (!transcoder_is_dsi(cpu_transcoder)) ++ intel_set_pipe_timings(pipe_config); ++ ++ intel_set_pipe_src_size(pipe_config); ++ ++ if (cpu_transcoder != TRANSCODER_EDP && ++ !transcoder_is_dsi(cpu_transcoder)) { ++ I915_WRITE(PIPE_MULT(cpu_transcoder), ++ pipe_config->pixel_multiplier - 1); ++ } ++ ++ if (pipe_config->has_pch_encoder) { ++ intel_cpu_transcoder_set_m_n(pipe_config, ++ &pipe_config->fdi_m_n, NULL); ++ } ++ ++ if (!transcoder_is_dsi(cpu_transcoder)) ++ haswell_set_pipeconf(pipe_config); ++ ++ haswell_set_pipemisc(pipe_config); ++ ++ intel_crtc->active = true; ++ ++ /* Display WA #1180: WaDisableScalarClockGating: glk, cnl */ ++ psl_clkgate_wa = (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) && ++ pipe_config->pch_pfit.enabled; ++ if (psl_clkgate_wa) ++ glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, true); ++ ++ if (INTEL_GEN(dev_priv) >= 9) ++ skylake_pfit_enable(pipe_config); ++ else ++ ironlake_pfit_enable(pipe_config); ++ ++ /* ++ * On ILK+ LUT must be loaded before the pipe is running but with ++ * clocks enabled ++ */ ++ intel_color_load_luts(pipe_config); ++ intel_color_commit(pipe_config); ++ /* update DSPCNTR to configure gamma/csc for pipe bottom color */ ++ if (INTEL_GEN(dev_priv) < 9) ++ intel_disable_primary_plane(pipe_config); ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ icl_set_pipe_chicken(intel_crtc); ++ ++ intel_ddi_set_pipe_settings(pipe_config); ++ if (!transcoder_is_dsi(cpu_transcoder)) ++ intel_ddi_enable_transcoder_func(pipe_config); ++ ++ if (dev_priv->display.initial_watermarks != NULL) ++ dev_priv->display.initial_watermarks(old_intel_state, pipe_config); ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ icl_pipe_mbus_enable(intel_crtc); ++ ++ /* XXX: Do the pipe assertions at the right place for BXT DSI. */ ++ if (!transcoder_is_dsi(cpu_transcoder)) ++ intel_enable_pipe(pipe_config); ++ ++ if (pipe_config->has_pch_encoder) ++ lpt_pch_enable(old_intel_state, pipe_config); ++ ++ if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DP_MST)) ++ intel_ddi_set_vc_payload_alloc(pipe_config, true); ++ ++ assert_vblank_disabled(crtc); ++ intel_crtc_vblank_on(pipe_config); ++ ++ intel_encoders_enable(crtc, pipe_config, old_state); ++ ++ if (psl_clkgate_wa) { ++ intel_wait_for_vblank(dev_priv, pipe); ++ glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, false); ++ } ++ ++ /* If we change the relative order between pipe/planes enabling, we need ++ * to change the workaround. */ ++ hsw_workaround_pipe = pipe_config->hsw_workaround_pipe; ++ if (IS_HASWELL(dev_priv) && hsw_workaround_pipe != INVALID_PIPE) { ++ intel_wait_for_vblank(dev_priv, hsw_workaround_pipe); ++ intel_wait_for_vblank(dev_priv, hsw_workaround_pipe); ++ } ++} ++ ++static void ironlake_pfit_disable(const struct intel_crtc_state *old_crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ ++ /* To avoid upsetting the power well on haswell only disable the pfit if ++ * it's in use. The hw state code will make sure we get this right. */ ++ if (old_crtc_state->pch_pfit.enabled) { ++ I915_WRITE(PF_CTL(pipe), 0); ++ I915_WRITE(PF_WIN_POS(pipe), 0); ++ I915_WRITE(PF_WIN_SZ(pipe), 0); ++ } ++} ++ ++static void ironlake_crtc_disable(struct intel_crtc_state *old_crtc_state, ++ struct drm_atomic_state *old_state) ++{ ++ struct drm_crtc *crtc = old_crtc_state->base.crtc; ++ struct drm_device *dev = crtc->dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_crtc *intel_crtc = to_intel_crtc(crtc); ++ int pipe = intel_crtc->pipe; ++ ++ /* ++ * Sometimes spurious CPU pipe underruns happen when the ++ * pipe is already disabled, but FDI RX/TX is still enabled. ++ * Happens at least with VGA+HDMI cloning. Suppress them. ++ */ ++ intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false); ++ intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false); ++ ++ intel_encoders_disable(crtc, old_crtc_state, old_state); ++ ++ drm_crtc_vblank_off(crtc); ++ assert_vblank_disabled(crtc); ++ ++ intel_disable_pipe(old_crtc_state); ++ ++ ironlake_pfit_disable(old_crtc_state); ++ ++ if (old_crtc_state->has_pch_encoder) ++ ironlake_fdi_disable(crtc); ++ ++ intel_encoders_post_disable(crtc, old_crtc_state, old_state); ++ ++ if (old_crtc_state->has_pch_encoder) { ++ ironlake_disable_pch_transcoder(dev_priv, pipe); ++ ++ if (HAS_PCH_CPT(dev_priv)) { ++ i915_reg_t reg; ++ u32 temp; ++ ++ /* disable TRANS_DP_CTL */ ++ reg = TRANS_DP_CTL(pipe); ++ temp = I915_READ(reg); ++ temp &= ~(TRANS_DP_OUTPUT_ENABLE | ++ TRANS_DP_PORT_SEL_MASK); ++ temp |= TRANS_DP_PORT_SEL_NONE; ++ I915_WRITE(reg, temp); ++ ++ /* disable DPLL_SEL */ ++ temp = I915_READ(PCH_DPLL_SEL); ++ temp &= ~(TRANS_DPLL_ENABLE(pipe) | TRANS_DPLLB_SEL(pipe)); ++ I915_WRITE(PCH_DPLL_SEL, temp); ++ } ++ ++ ironlake_fdi_pll_disable(intel_crtc); ++ } ++ ++ intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); ++ intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true); ++} ++ ++static void haswell_crtc_disable(struct intel_crtc_state *old_crtc_state, ++ struct drm_atomic_state *old_state) ++{ ++ struct drm_crtc *crtc = old_crtc_state->base.crtc; ++ struct drm_i915_private *dev_priv = to_i915(crtc->dev); ++ struct intel_crtc *intel_crtc = to_intel_crtc(crtc); ++ enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder; ++ ++ intel_encoders_disable(crtc, old_crtc_state, old_state); ++ ++ drm_crtc_vblank_off(crtc); ++ assert_vblank_disabled(crtc); ++ ++ /* XXX: Do the pipe assertions at the right place for BXT DSI. */ ++ if (!transcoder_is_dsi(cpu_transcoder)) ++ intel_disable_pipe(old_crtc_state); ++ ++ if (intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DP_MST)) ++ intel_ddi_set_vc_payload_alloc(old_crtc_state, false); ++ ++ if (!transcoder_is_dsi(cpu_transcoder)) ++ intel_ddi_disable_transcoder_func(old_crtc_state); ++ ++ intel_dsc_disable(old_crtc_state); ++ ++ if (INTEL_GEN(dev_priv) >= 9) ++ skylake_scaler_disable(intel_crtc); ++ else ++ ironlake_pfit_disable(old_crtc_state); ++ ++ intel_encoders_post_disable(crtc, old_crtc_state, old_state); ++ ++ intel_encoders_post_pll_disable(crtc, old_crtc_state, old_state); ++} ++ ++static void i9xx_pfit_enable(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ ++ if (!crtc_state->gmch_pfit.control) ++ return; ++ ++ /* ++ * The panel fitter should only be adjusted whilst the pipe is disabled, ++ * according to register description and PRM. ++ */ ++ WARN_ON(I915_READ(PFIT_CONTROL) & PFIT_ENABLE); ++ assert_pipe_disabled(dev_priv, crtc->pipe); ++ ++ I915_WRITE(PFIT_PGM_RATIOS, crtc_state->gmch_pfit.pgm_ratios); ++ I915_WRITE(PFIT_CONTROL, crtc_state->gmch_pfit.control); ++ ++ /* Border color in case we don't scale up to the full screen. Black by ++ * default, change to something else for debugging. */ ++ I915_WRITE(BCLRPAT(crtc->pipe), 0); ++} ++ ++bool intel_port_is_combophy(struct drm_i915_private *dev_priv, enum port port) ++{ ++ if (port == PORT_NONE) ++ return false; ++ ++ if (IS_ELKHARTLAKE(dev_priv)) ++ return port <= PORT_C; ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ return port <= PORT_B; ++ ++ return false; ++} ++ ++bool intel_port_is_tc(struct drm_i915_private *dev_priv, enum port port) ++{ ++ if (INTEL_GEN(dev_priv) >= 11 && !IS_ELKHARTLAKE(dev_priv)) ++ return port >= PORT_C && port <= PORT_F; ++ ++ return false; ++} ++ ++enum tc_port intel_port_to_tc(struct drm_i915_private *dev_priv, enum port port) ++{ ++ if (!intel_port_is_tc(dev_priv, port)) ++ return PORT_TC_NONE; ++ ++ return port - PORT_C; ++} ++ ++enum intel_display_power_domain intel_port_to_power_domain(enum port port) ++{ ++ switch (port) { ++ case PORT_A: ++ return POWER_DOMAIN_PORT_DDI_A_LANES; ++ case PORT_B: ++ return POWER_DOMAIN_PORT_DDI_B_LANES; ++ case PORT_C: ++ return POWER_DOMAIN_PORT_DDI_C_LANES; ++ case PORT_D: ++ return POWER_DOMAIN_PORT_DDI_D_LANES; ++ case PORT_E: ++ return POWER_DOMAIN_PORT_DDI_E_LANES; ++ case PORT_F: ++ return POWER_DOMAIN_PORT_DDI_F_LANES; ++ default: ++ MISSING_CASE(port); ++ return POWER_DOMAIN_PORT_OTHER; ++ } ++} ++ ++enum intel_display_power_domain ++intel_aux_power_domain(struct intel_digital_port *dig_port) ++{ ++ switch (dig_port->aux_ch) { ++ case AUX_CH_A: ++ return POWER_DOMAIN_AUX_A; ++ case AUX_CH_B: ++ return POWER_DOMAIN_AUX_B; ++ case AUX_CH_C: ++ return POWER_DOMAIN_AUX_C; ++ case AUX_CH_D: ++ return POWER_DOMAIN_AUX_D; ++ case AUX_CH_E: ++ return POWER_DOMAIN_AUX_E; ++ case AUX_CH_F: ++ return POWER_DOMAIN_AUX_F; ++ default: ++ MISSING_CASE(dig_port->aux_ch); ++ return POWER_DOMAIN_AUX_A; ++ } ++} ++ ++static u64 get_crtc_power_domains(struct drm_crtc *crtc, ++ struct intel_crtc_state *crtc_state) ++{ ++ struct drm_device *dev = crtc->dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct drm_encoder *encoder; ++ struct intel_crtc *intel_crtc = to_intel_crtc(crtc); ++ enum pipe pipe = intel_crtc->pipe; ++ u64 mask; ++ enum transcoder transcoder = crtc_state->cpu_transcoder; ++ ++ if (!crtc_state->base.active) ++ return 0; ++ ++ mask = BIT_ULL(POWER_DOMAIN_PIPE(pipe)); ++ mask |= BIT_ULL(POWER_DOMAIN_TRANSCODER(transcoder)); ++ if (crtc_state->pch_pfit.enabled || ++ crtc_state->pch_pfit.force_thru) ++ mask |= BIT_ULL(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe)); ++ ++ drm_for_each_encoder_mask(encoder, dev, crtc_state->base.encoder_mask) { ++ struct intel_encoder *intel_encoder = to_intel_encoder(encoder); ++ ++ mask |= BIT_ULL(intel_encoder->power_domain); ++ } ++ ++ if (HAS_DDI(dev_priv) && crtc_state->has_audio) ++ mask |= BIT_ULL(POWER_DOMAIN_AUDIO); ++ ++ if (crtc_state->shared_dpll) ++ mask |= BIT_ULL(POWER_DOMAIN_PLLS); ++ ++ return mask; ++} ++ ++static u64 ++modeset_get_crtc_power_domains(struct drm_crtc *crtc, ++ struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->dev); ++ struct intel_crtc *intel_crtc = to_intel_crtc(crtc); ++ enum intel_display_power_domain domain; ++ u64 domains, new_domains, old_domains; ++ ++ old_domains = intel_crtc->enabled_power_domains; ++ intel_crtc->enabled_power_domains = new_domains = ++ get_crtc_power_domains(crtc, crtc_state); ++ ++ domains = new_domains & ~old_domains; ++ ++ for_each_power_domain(domain, domains) ++ intel_display_power_get(dev_priv, domain); ++ ++ return old_domains & ~new_domains; ++} ++ ++static void modeset_put_power_domains(struct drm_i915_private *dev_priv, ++ u64 domains) ++{ ++ enum intel_display_power_domain domain; ++ ++ for_each_power_domain(domain, domains) ++ intel_display_power_put_unchecked(dev_priv, domain); ++} ++ ++static void valleyview_crtc_enable(struct intel_crtc_state *pipe_config, ++ struct drm_atomic_state *old_state) ++{ ++ struct intel_atomic_state *old_intel_state = ++ to_intel_atomic_state(old_state); ++ struct drm_crtc *crtc = pipe_config->base.crtc; ++ struct drm_device *dev = crtc->dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_crtc *intel_crtc = to_intel_crtc(crtc); ++ int pipe = intel_crtc->pipe; ++ ++ if (WARN_ON(intel_crtc->active)) ++ return; ++ ++ if (intel_crtc_has_dp_encoder(pipe_config)) ++ intel_dp_set_m_n(pipe_config, M1_N1); ++ ++ intel_set_pipe_timings(pipe_config); ++ intel_set_pipe_src_size(pipe_config); ++ ++ if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) { ++ I915_WRITE(CHV_BLEND(pipe), CHV_BLEND_LEGACY); ++ I915_WRITE(CHV_CANVAS(pipe), 0); ++ } ++ ++ i9xx_set_pipeconf(pipe_config); ++ ++ intel_crtc->active = true; ++ ++ intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); ++ ++ intel_encoders_pre_pll_enable(crtc, pipe_config, old_state); ++ ++ if (IS_CHERRYVIEW(dev_priv)) { ++ chv_prepare_pll(intel_crtc, pipe_config); ++ chv_enable_pll(intel_crtc, pipe_config); ++ } else { ++ vlv_prepare_pll(intel_crtc, pipe_config); ++ vlv_enable_pll(intel_crtc, pipe_config); ++ } ++ ++ intel_encoders_pre_enable(crtc, pipe_config, old_state); ++ ++ i9xx_pfit_enable(pipe_config); ++ ++ intel_color_load_luts(pipe_config); ++ intel_color_commit(pipe_config); ++ /* update DSPCNTR to configure gamma for pipe bottom color */ ++ intel_disable_primary_plane(pipe_config); ++ ++ dev_priv->display.initial_watermarks(old_intel_state, ++ pipe_config); ++ intel_enable_pipe(pipe_config); ++ ++ assert_vblank_disabled(crtc); ++ intel_crtc_vblank_on(pipe_config); ++ ++ intel_encoders_enable(crtc, pipe_config, old_state); ++} ++ ++static void i9xx_set_pll_dividers(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ ++ I915_WRITE(FP0(crtc->pipe), crtc_state->dpll_hw_state.fp0); ++ I915_WRITE(FP1(crtc->pipe), crtc_state->dpll_hw_state.fp1); ++} ++ ++static void i9xx_crtc_enable(struct intel_crtc_state *pipe_config, ++ struct drm_atomic_state *old_state) ++{ ++ struct intel_atomic_state *old_intel_state = ++ to_intel_atomic_state(old_state); ++ struct drm_crtc *crtc = pipe_config->base.crtc; ++ struct drm_device *dev = crtc->dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_crtc *intel_crtc = to_intel_crtc(crtc); ++ enum pipe pipe = intel_crtc->pipe; ++ ++ if (WARN_ON(intel_crtc->active)) ++ return; ++ ++ i9xx_set_pll_dividers(pipe_config); ++ ++ if (intel_crtc_has_dp_encoder(pipe_config)) ++ intel_dp_set_m_n(pipe_config, M1_N1); ++ ++ intel_set_pipe_timings(pipe_config); ++ intel_set_pipe_src_size(pipe_config); ++ ++ i9xx_set_pipeconf(pipe_config); ++ ++ intel_crtc->active = true; ++ ++ if (!IS_GEN(dev_priv, 2)) ++ intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); ++ ++ intel_encoders_pre_enable(crtc, pipe_config, old_state); ++ ++ i9xx_enable_pll(intel_crtc, pipe_config); ++ ++ i9xx_pfit_enable(pipe_config); ++ ++ intel_color_load_luts(pipe_config); ++ intel_color_commit(pipe_config); ++ /* update DSPCNTR to configure gamma for pipe bottom color */ ++ intel_disable_primary_plane(pipe_config); ++ ++ if (dev_priv->display.initial_watermarks != NULL) ++ dev_priv->display.initial_watermarks(old_intel_state, ++ pipe_config); ++ else ++ intel_update_watermarks(intel_crtc); ++ intel_enable_pipe(pipe_config); ++ ++ assert_vblank_disabled(crtc); ++ intel_crtc_vblank_on(pipe_config); ++ ++ intel_encoders_enable(crtc, pipe_config, old_state); ++} ++ ++static void i9xx_pfit_disable(const struct intel_crtc_state *old_crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ ++ if (!old_crtc_state->gmch_pfit.control) ++ return; ++ ++ assert_pipe_disabled(dev_priv, crtc->pipe); ++ ++ DRM_DEBUG_KMS("disabling pfit, current: 0x%08x\n", ++ I915_READ(PFIT_CONTROL)); ++ I915_WRITE(PFIT_CONTROL, 0); ++} ++ ++static void i9xx_crtc_disable(struct intel_crtc_state *old_crtc_state, ++ struct drm_atomic_state *old_state) ++{ ++ struct drm_crtc *crtc = old_crtc_state->base.crtc; ++ struct drm_device *dev = crtc->dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_crtc *intel_crtc = to_intel_crtc(crtc); ++ int pipe = intel_crtc->pipe; ++ ++ /* ++ * On gen2 planes are double buffered but the pipe isn't, so we must ++ * wait for planes to fully turn off before disabling the pipe. ++ */ ++ if (IS_GEN(dev_priv, 2)) ++ intel_wait_for_vblank(dev_priv, pipe); ++ ++ intel_encoders_disable(crtc, old_crtc_state, old_state); ++ ++ drm_crtc_vblank_off(crtc); ++ assert_vblank_disabled(crtc); ++ ++ intel_disable_pipe(old_crtc_state); ++ ++ i9xx_pfit_disable(old_crtc_state); ++ ++ intel_encoders_post_disable(crtc, old_crtc_state, old_state); ++ ++ if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DSI)) { ++ if (IS_CHERRYVIEW(dev_priv)) ++ chv_disable_pll(dev_priv, pipe); ++ else if (IS_VALLEYVIEW(dev_priv)) ++ vlv_disable_pll(dev_priv, pipe); ++ else ++ i9xx_disable_pll(old_crtc_state); ++ } ++ ++ intel_encoders_post_pll_disable(crtc, old_crtc_state, old_state); ++ ++ if (!IS_GEN(dev_priv, 2)) ++ intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false); ++ ++ if (!dev_priv->display.initial_watermarks) ++ intel_update_watermarks(intel_crtc); ++ ++ /* clock the pipe down to 640x480@60 to potentially save power */ ++ if (IS_I830(dev_priv)) ++ i830_enable_pipe(dev_priv, pipe); ++} ++ ++static void intel_crtc_disable_noatomic(struct drm_crtc *crtc, ++ struct drm_modeset_acquire_ctx *ctx) ++{ ++ struct intel_encoder *encoder; ++ struct intel_crtc *intel_crtc = to_intel_crtc(crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->dev); ++ enum intel_display_power_domain domain; ++ struct intel_plane *plane; ++ u64 domains; ++ struct drm_atomic_state *state; ++ struct intel_crtc_state *crtc_state; ++ int ret; ++ ++ if (!intel_crtc->active) ++ return; ++ ++ for_each_intel_plane_on_crtc(&dev_priv->drm, intel_crtc, plane) { ++ const struct intel_plane_state *plane_state = ++ to_intel_plane_state(plane->base.state); ++ ++ if (plane_state->base.visible) ++ intel_plane_disable_noatomic(intel_crtc, plane); ++ } ++ ++ state = drm_atomic_state_alloc(crtc->dev); ++ if (!state) { ++ DRM_DEBUG_KMS("failed to disable [CRTC:%d:%s], out of memory", ++ crtc->base.id, crtc->name); ++ return; ++ } ++ ++ state->acquire_ctx = ctx; ++ ++ /* Everything's already locked, -EDEADLK can't happen. */ ++ crtc_state = intel_atomic_get_crtc_state(state, intel_crtc); ++ ret = drm_atomic_add_affected_connectors(state, crtc); ++ ++ WARN_ON(IS_ERR(crtc_state) || ret); ++ ++ dev_priv->display.crtc_disable(crtc_state, state); ++ ++ drm_atomic_state_put(state); ++ ++ DRM_DEBUG_KMS("[CRTC:%d:%s] hw state adjusted, was enabled, now disabled\n", ++ crtc->base.id, crtc->name); ++ ++ WARN_ON(drm_atomic_set_mode_for_crtc(crtc->state, NULL) < 0); ++ crtc->state->active = false; ++ intel_crtc->active = false; ++ crtc->enabled = false; ++ crtc->state->connector_mask = 0; ++ crtc->state->encoder_mask = 0; ++ ++ for_each_encoder_on_crtc(crtc->dev, crtc, encoder) ++ encoder->base.crtc = NULL; ++ ++ intel_fbc_disable(intel_crtc); ++ intel_update_watermarks(intel_crtc); ++ intel_disable_shared_dpll(to_intel_crtc_state(crtc->state)); ++ ++ domains = intel_crtc->enabled_power_domains; ++ for_each_power_domain(domain, domains) ++ intel_display_power_put_unchecked(dev_priv, domain); ++ intel_crtc->enabled_power_domains = 0; ++ ++ dev_priv->active_crtcs &= ~(1 << intel_crtc->pipe); ++ dev_priv->min_cdclk[intel_crtc->pipe] = 0; ++ dev_priv->min_voltage_level[intel_crtc->pipe] = 0; ++} ++ ++/* ++ * turn all crtc's off, but do not adjust state ++ * This has to be paired with a call to intel_modeset_setup_hw_state. ++ */ ++int intel_display_suspend(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct drm_atomic_state *state; ++ int ret; ++ ++ state = drm_atomic_helper_suspend(dev); ++ ret = PTR_ERR_OR_ZERO(state); ++ if (ret) ++ DRM_ERROR("Suspending crtc's failed with %i\n", ret); ++ else ++ dev_priv->modeset_restore_state = state; ++ return ret; ++} ++ ++void intel_encoder_destroy(struct drm_encoder *encoder) ++{ ++ struct intel_encoder *intel_encoder = to_intel_encoder(encoder); ++ ++ drm_encoder_cleanup(encoder); ++ kfree(intel_encoder); ++} ++ ++/* Cross check the actual hw state with our own modeset state tracking (and it's ++ * internal consistency). */ ++static void intel_connector_verify_state(struct drm_crtc_state *crtc_state, ++ struct drm_connector_state *conn_state) ++{ ++ struct intel_connector *connector = to_intel_connector(conn_state->connector); ++ ++ DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", ++ connector->base.base.id, ++ connector->base.name); ++ ++ if (connector->get_hw_state(connector)) { ++ struct intel_encoder *encoder = connector->encoder; ++ ++ I915_STATE_WARN(!crtc_state, ++ "connector enabled without attached crtc\n"); ++ ++ if (!crtc_state) ++ return; ++ ++ I915_STATE_WARN(!crtc_state->active, ++ "connector is active, but attached crtc isn't\n"); ++ ++ if (!encoder || encoder->type == INTEL_OUTPUT_DP_MST) ++ return; ++ ++ I915_STATE_WARN(conn_state->best_encoder != &encoder->base, ++ "atomic encoder doesn't match attached encoder\n"); ++ ++ I915_STATE_WARN(conn_state->crtc != encoder->base.crtc, ++ "attached encoder crtc differs from connector crtc\n"); ++ } else { ++ I915_STATE_WARN(crtc_state && crtc_state->active, ++ "attached crtc is active, but connector isn't\n"); ++ I915_STATE_WARN(!crtc_state && conn_state->best_encoder, ++ "best encoder set without crtc!\n"); ++ } ++} ++ ++static int pipe_required_fdi_lanes(struct intel_crtc_state *crtc_state) ++{ ++ if (crtc_state->base.enable && crtc_state->has_pch_encoder) ++ return crtc_state->fdi_lanes; ++ ++ return 0; ++} ++ ++static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct drm_atomic_state *state = pipe_config->base.state; ++ struct intel_crtc *other_crtc; ++ struct intel_crtc_state *other_crtc_state; ++ ++ DRM_DEBUG_KMS("checking fdi config on pipe %c, lanes %i\n", ++ pipe_name(pipe), pipe_config->fdi_lanes); ++ if (pipe_config->fdi_lanes > 4) { ++ DRM_DEBUG_KMS("invalid fdi lane config on pipe %c: %i lanes\n", ++ pipe_name(pipe), pipe_config->fdi_lanes); ++ return -EINVAL; ++ } ++ ++ if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { ++ if (pipe_config->fdi_lanes > 2) { ++ DRM_DEBUG_KMS("only 2 lanes on haswell, required: %i lanes\n", ++ pipe_config->fdi_lanes); ++ return -EINVAL; ++ } else { ++ return 0; ++ } ++ } ++ ++ if (INTEL_INFO(dev_priv)->num_pipes == 2) ++ return 0; ++ ++ /* Ivybridge 3 pipe is really complicated */ ++ switch (pipe) { ++ case PIPE_A: ++ return 0; ++ case PIPE_B: ++ if (pipe_config->fdi_lanes <= 2) ++ return 0; ++ ++ other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_C); ++ other_crtc_state = ++ intel_atomic_get_crtc_state(state, other_crtc); ++ if (IS_ERR(other_crtc_state)) ++ return PTR_ERR(other_crtc_state); ++ ++ if (pipe_required_fdi_lanes(other_crtc_state) > 0) { ++ DRM_DEBUG_KMS("invalid shared fdi lane config on pipe %c: %i lanes\n", ++ pipe_name(pipe), pipe_config->fdi_lanes); ++ return -EINVAL; ++ } ++ return 0; ++ case PIPE_C: ++ if (pipe_config->fdi_lanes > 2) { ++ DRM_DEBUG_KMS("only 2 lanes on pipe %c: required %i lanes\n", ++ pipe_name(pipe), pipe_config->fdi_lanes); ++ return -EINVAL; ++ } ++ ++ other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_B); ++ other_crtc_state = ++ intel_atomic_get_crtc_state(state, other_crtc); ++ if (IS_ERR(other_crtc_state)) ++ return PTR_ERR(other_crtc_state); ++ ++ if (pipe_required_fdi_lanes(other_crtc_state) > 2) { ++ DRM_DEBUG_KMS("fdi link B uses too many lanes to enable link C\n"); ++ return -EINVAL; ++ } ++ return 0; ++ default: ++ BUG(); ++ } ++} ++ ++#define RETRY 1 ++static int ironlake_fdi_compute_config(struct intel_crtc *intel_crtc, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_device *dev = intel_crtc->base.dev; ++ const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; ++ int lane, link_bw, fdi_dotclock, ret; ++ bool needs_recompute = false; ++ ++retry: ++ /* FDI is a binary signal running at ~2.7GHz, encoding ++ * each output octet as 10 bits. The actual frequency ++ * is stored as a divider into a 100MHz clock, and the ++ * mode pixel clock is stored in units of 1KHz. ++ * Hence the bw of each lane in terms of the mode signal ++ * is: ++ */ ++ link_bw = intel_fdi_link_freq(to_i915(dev), pipe_config); ++ ++ fdi_dotclock = adjusted_mode->crtc_clock; ++ ++ lane = ironlake_get_lanes_required(fdi_dotclock, link_bw, ++ pipe_config->pipe_bpp); ++ ++ pipe_config->fdi_lanes = lane; ++ ++ intel_link_compute_m_n(pipe_config->pipe_bpp, lane, fdi_dotclock, ++ link_bw, &pipe_config->fdi_m_n, false); ++ ++ ret = ironlake_check_fdi_lanes(dev, intel_crtc->pipe, pipe_config); ++ if (ret == -EDEADLK) ++ return ret; ++ ++ if (ret == -EINVAL && pipe_config->pipe_bpp > 6*3) { ++ pipe_config->pipe_bpp -= 2*3; ++ DRM_DEBUG_KMS("fdi link bw constraint, reducing pipe bpp to %i\n", ++ pipe_config->pipe_bpp); ++ needs_recompute = true; ++ pipe_config->bw_constrained = true; ++ ++ goto retry; ++ } ++ ++ if (needs_recompute) ++ return RETRY; ++ ++ return ret; ++} ++ ++bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ ++ /* IPS only exists on ULT machines and is tied to pipe A. */ ++ if (!hsw_crtc_supports_ips(crtc)) ++ return false; ++ ++ if (!i915_modparams.enable_ips) ++ return false; ++ ++ if (crtc_state->pipe_bpp > 24) ++ return false; ++ ++ /* ++ * We compare against max which means we must take ++ * the increased cdclk requirement into account when ++ * calculating the new cdclk. ++ * ++ * Should measure whether using a lower cdclk w/o IPS ++ */ ++ if (IS_BROADWELL(dev_priv) && ++ crtc_state->pixel_rate > dev_priv->max_cdclk_freq * 95 / 100) ++ return false; ++ ++ return true; ++} ++ ++static bool hsw_compute_ips_config(struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = ++ to_i915(crtc_state->base.crtc->dev); ++ struct intel_atomic_state *intel_state = ++ to_intel_atomic_state(crtc_state->base.state); ++ ++ if (!hsw_crtc_state_ips_capable(crtc_state)) ++ return false; ++ ++ /* ++ * When IPS gets enabled, the pipe CRC changes. Since IPS gets ++ * enabled and disabled dynamically based on package C states, ++ * user space can't make reliable use of the CRCs, so let's just ++ * completely disable it. ++ */ ++ if (crtc_state->crc_enabled) ++ return false; ++ ++ /* IPS should be fine as long as at least one plane is enabled. */ ++ if (!(crtc_state->active_planes & ~BIT(PLANE_CURSOR))) ++ return false; ++ ++ /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */ ++ if (IS_BROADWELL(dev_priv) && ++ crtc_state->pixel_rate > intel_state->cdclk.logical.cdclk * 95 / 100) ++ return false; ++ ++ return true; ++} ++ ++static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc) ++{ ++ const struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ ++ /* GDG double wide on either pipe, otherwise pipe A only */ ++ return INTEL_GEN(dev_priv) < 4 && ++ (crtc->pipe == PIPE_A || IS_I915G(dev_priv)); ++} ++ ++static u32 ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config) ++{ ++ u32 pixel_rate; ++ ++ pixel_rate = pipe_config->base.adjusted_mode.crtc_clock; ++ ++ /* ++ * We only use IF-ID interlacing. If we ever use ++ * PF-ID we'll need to adjust the pixel_rate here. ++ */ ++ ++ if (pipe_config->pch_pfit.enabled) { ++ u64 pipe_w, pipe_h, pfit_w, pfit_h; ++ u32 pfit_size = pipe_config->pch_pfit.size; ++ ++ pipe_w = pipe_config->pipe_src_w; ++ pipe_h = pipe_config->pipe_src_h; ++ ++ pfit_w = (pfit_size >> 16) & 0xFFFF; ++ pfit_h = pfit_size & 0xFFFF; ++ if (pipe_w < pfit_w) ++ pipe_w = pfit_w; ++ if (pipe_h < pfit_h) ++ pipe_h = pfit_h; ++ ++ if (WARN_ON(!pfit_w || !pfit_h)) ++ return pixel_rate; ++ ++ pixel_rate = div_u64((u64)pixel_rate * pipe_w * pipe_h, ++ pfit_w * pfit_h); ++ } ++ ++ return pixel_rate; ++} ++ ++static void intel_crtc_compute_pixel_rate(struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); ++ ++ if (HAS_GMCH(dev_priv)) ++ /* FIXME calculate proper pipe pixel rate for GMCH pfit */ ++ crtc_state->pixel_rate = ++ crtc_state->base.adjusted_mode.crtc_clock; ++ else ++ crtc_state->pixel_rate = ++ ilk_pipe_pixel_rate(crtc_state); ++} ++ ++static int intel_crtc_compute_config(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; ++ int clock_limit = dev_priv->max_dotclk_freq; ++ ++ if (INTEL_GEN(dev_priv) < 4) { ++ clock_limit = dev_priv->max_cdclk_freq * 9 / 10; ++ ++ /* ++ * Enable double wide mode when the dot clock ++ * is > 90% of the (display) core speed. ++ */ ++ if (intel_crtc_supports_double_wide(crtc) && ++ adjusted_mode->crtc_clock > clock_limit) { ++ clock_limit = dev_priv->max_dotclk_freq; ++ pipe_config->double_wide = true; ++ } ++ } ++ ++ if (adjusted_mode->crtc_clock > clock_limit) { ++ DRM_DEBUG_KMS("requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n", ++ adjusted_mode->crtc_clock, clock_limit, ++ yesno(pipe_config->double_wide)); ++ return -EINVAL; ++ } ++ ++ if ((pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 || ++ pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR444) && ++ pipe_config->base.ctm) { ++ /* ++ * There is only one pipe CSC unit per pipe, and we need that ++ * for output conversion from RGB->YCBCR. So if CTM is already ++ * applied we can't support YCBCR420 output. ++ */ ++ DRM_DEBUG_KMS("YCBCR420 and CTM together are not possible\n"); ++ return -EINVAL; ++ } ++ ++ /* ++ * Pipe horizontal size must be even in: ++ * - DVO ganged mode ++ * - LVDS dual channel mode ++ * - Double wide pipe ++ */ ++ if (pipe_config->pipe_src_w & 1) { ++ if (pipe_config->double_wide) { ++ DRM_DEBUG_KMS("Odd pipe source width not supported with double wide pipe\n"); ++ return -EINVAL; ++ } ++ ++ if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_LVDS) && ++ intel_is_dual_link_lvds(dev_priv)) { ++ DRM_DEBUG_KMS("Odd pipe source width not supported with dual link LVDS\n"); ++ return -EINVAL; ++ } ++ } ++ ++ /* Cantiga+ cannot handle modes with a hsync front porch of 0. ++ * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw. ++ */ ++ if ((INTEL_GEN(dev_priv) > 4 || IS_G4X(dev_priv)) && ++ adjusted_mode->crtc_hsync_start == adjusted_mode->crtc_hdisplay) ++ return -EINVAL; ++ ++ intel_crtc_compute_pixel_rate(pipe_config); ++ ++ if (pipe_config->has_pch_encoder) ++ return ironlake_fdi_compute_config(crtc, pipe_config); ++ ++ return 0; ++} ++ ++static void ++intel_reduce_m_n_ratio(u32 *num, u32 *den) ++{ ++ while (*num > DATA_LINK_M_N_MASK || ++ *den > DATA_LINK_M_N_MASK) { ++ *num >>= 1; ++ *den >>= 1; ++ } ++} ++ ++static void compute_m_n(unsigned int m, unsigned int n, ++ u32 *ret_m, u32 *ret_n, ++ bool constant_n) ++{ ++ /* ++ * Several DP dongles in particular seem to be fussy about ++ * too large link M/N values. Give N value as 0x8000 that ++ * should be acceptable by specific devices. 0x8000 is the ++ * specified fixed N value for asynchronous clock mode, ++ * which the devices expect also in synchronous clock mode. ++ */ ++ if (constant_n) ++ *ret_n = 0x8000; ++ else ++ *ret_n = min_t(unsigned int, roundup_pow_of_two(n), DATA_LINK_N_MAX); ++ ++ *ret_m = div_u64((u64)m * *ret_n, n); ++ intel_reduce_m_n_ratio(ret_m, ret_n); ++} ++ ++void ++intel_link_compute_m_n(u16 bits_per_pixel, int nlanes, ++ int pixel_clock, int link_clock, ++ struct intel_link_m_n *m_n, ++ bool constant_n) ++{ ++ m_n->tu = 64; ++ ++ compute_m_n(bits_per_pixel * pixel_clock, ++ link_clock * nlanes * 8, ++ &m_n->gmch_m, &m_n->gmch_n, ++ constant_n); ++ ++ compute_m_n(pixel_clock, link_clock, ++ &m_n->link_m, &m_n->link_n, ++ constant_n); ++} ++ ++static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv) ++{ ++ if (i915_modparams.panel_use_ssc >= 0) ++ return i915_modparams.panel_use_ssc != 0; ++ return dev_priv->vbt.lvds_use_ssc ++ && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE); ++} ++ ++static u32 pnv_dpll_compute_fp(struct dpll *dpll) ++{ ++ return (1 << dpll->n) << 16 | dpll->m2; ++} ++ ++static u32 i9xx_dpll_compute_fp(struct dpll *dpll) ++{ ++ return dpll->n << 16 | dpll->m1 << 8 | dpll->m2; ++} ++ ++static void i9xx_update_pll_dividers(struct intel_crtc *crtc, ++ struct intel_crtc_state *crtc_state, ++ struct dpll *reduced_clock) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ u32 fp, fp2 = 0; ++ ++ if (IS_PINEVIEW(dev_priv)) { ++ fp = pnv_dpll_compute_fp(&crtc_state->dpll); ++ if (reduced_clock) ++ fp2 = pnv_dpll_compute_fp(reduced_clock); ++ } else { ++ fp = i9xx_dpll_compute_fp(&crtc_state->dpll); ++ if (reduced_clock) ++ fp2 = i9xx_dpll_compute_fp(reduced_clock); ++ } ++ ++ crtc_state->dpll_hw_state.fp0 = fp; ++ ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) && ++ reduced_clock) { ++ crtc_state->dpll_hw_state.fp1 = fp2; ++ } else { ++ crtc_state->dpll_hw_state.fp1 = fp; ++ } ++} ++ ++static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv, enum pipe ++ pipe) ++{ ++ u32 reg_val; ++ ++ /* ++ * PLLB opamp always calibrates to max value of 0x3f, force enable it ++ * and set it to a reasonable value instead. ++ */ ++ reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1)); ++ reg_val &= 0xffffff00; ++ reg_val |= 0x00000030; ++ vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val); ++ ++ reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13); ++ reg_val &= 0x00ffffff; ++ reg_val |= 0x8c000000; ++ vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val); ++ ++ reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1)); ++ reg_val &= 0xffffff00; ++ vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val); ++ ++ reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13); ++ reg_val &= 0x00ffffff; ++ reg_val |= 0xb0000000; ++ vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val); ++} ++ ++static void intel_pch_transcoder_set_m_n(const struct intel_crtc_state *crtc_state, ++ const struct intel_link_m_n *m_n) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ ++ I915_WRITE(PCH_TRANS_DATA_M1(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m); ++ I915_WRITE(PCH_TRANS_DATA_N1(pipe), m_n->gmch_n); ++ I915_WRITE(PCH_TRANS_LINK_M1(pipe), m_n->link_m); ++ I915_WRITE(PCH_TRANS_LINK_N1(pipe), m_n->link_n); ++} ++ ++static bool transcoder_has_m2_n2(struct drm_i915_private *dev_priv, ++ enum transcoder transcoder) ++{ ++ if (IS_HASWELL(dev_priv)) ++ return transcoder == TRANSCODER_EDP; ++ ++ /* ++ * Strictly speaking some registers are available before ++ * gen7, but we only support DRRS on gen7+ ++ */ ++ return IS_GEN(dev_priv, 7) || IS_CHERRYVIEW(dev_priv); ++} ++ ++static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state, ++ const struct intel_link_m_n *m_n, ++ const struct intel_link_m_n *m2_n2) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ enum transcoder transcoder = crtc_state->cpu_transcoder; ++ ++ if (INTEL_GEN(dev_priv) >= 5) { ++ I915_WRITE(PIPE_DATA_M1(transcoder), TU_SIZE(m_n->tu) | m_n->gmch_m); ++ I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n); ++ I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m); ++ I915_WRITE(PIPE_LINK_N1(transcoder), m_n->link_n); ++ /* ++ * M2_N2 registers are set only if DRRS is supported ++ * (to make sure the registers are not unnecessarily accessed). ++ */ ++ if (m2_n2 && crtc_state->has_drrs && ++ transcoder_has_m2_n2(dev_priv, transcoder)) { ++ I915_WRITE(PIPE_DATA_M2(transcoder), ++ TU_SIZE(m2_n2->tu) | m2_n2->gmch_m); ++ I915_WRITE(PIPE_DATA_N2(transcoder), m2_n2->gmch_n); ++ I915_WRITE(PIPE_LINK_M2(transcoder), m2_n2->link_m); ++ I915_WRITE(PIPE_LINK_N2(transcoder), m2_n2->link_n); ++ } ++ } else { ++ I915_WRITE(PIPE_DATA_M_G4X(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m); ++ I915_WRITE(PIPE_DATA_N_G4X(pipe), m_n->gmch_n); ++ I915_WRITE(PIPE_LINK_M_G4X(pipe), m_n->link_m); ++ I915_WRITE(PIPE_LINK_N_G4X(pipe), m_n->link_n); ++ } ++} ++ ++void intel_dp_set_m_n(const struct intel_crtc_state *crtc_state, enum link_m_n_set m_n) ++{ ++ const struct intel_link_m_n *dp_m_n, *dp_m2_n2 = NULL; ++ ++ if (m_n == M1_N1) { ++ dp_m_n = &crtc_state->dp_m_n; ++ dp_m2_n2 = &crtc_state->dp_m2_n2; ++ } else if (m_n == M2_N2) { ++ ++ /* ++ * M2_N2 registers are not supported. Hence m2_n2 divider value ++ * needs to be programmed into M1_N1. ++ */ ++ dp_m_n = &crtc_state->dp_m2_n2; ++ } else { ++ DRM_ERROR("Unsupported divider value\n"); ++ return; ++ } ++ ++ if (crtc_state->has_pch_encoder) ++ intel_pch_transcoder_set_m_n(crtc_state, &crtc_state->dp_m_n); ++ else ++ intel_cpu_transcoder_set_m_n(crtc_state, dp_m_n, dp_m2_n2); ++} ++ ++static void vlv_compute_dpll(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config) ++{ ++ pipe_config->dpll_hw_state.dpll = DPLL_INTEGRATED_REF_CLK_VLV | ++ DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS; ++ if (crtc->pipe != PIPE_A) ++ pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV; ++ ++ /* DPLL not used with DSI, but still need the rest set up */ ++ if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI)) ++ pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE | ++ DPLL_EXT_BUFFER_ENABLE_VLV; ++ ++ pipe_config->dpll_hw_state.dpll_md = ++ (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT; ++} ++ ++static void chv_compute_dpll(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config) ++{ ++ pipe_config->dpll_hw_state.dpll = DPLL_SSC_REF_CLK_CHV | ++ DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS; ++ if (crtc->pipe != PIPE_A) ++ pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV; ++ ++ /* DPLL not used with DSI, but still need the rest set up */ ++ if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI)) ++ pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE; ++ ++ pipe_config->dpll_hw_state.dpll_md = ++ (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT; ++} ++ ++static void vlv_prepare_pll(struct intel_crtc *crtc, ++ const struct intel_crtc_state *pipe_config) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ enum pipe pipe = crtc->pipe; ++ u32 mdiv; ++ u32 bestn, bestm1, bestm2, bestp1, bestp2; ++ u32 coreclk, reg_val; ++ ++ /* Enable Refclk */ ++ I915_WRITE(DPLL(pipe), ++ pipe_config->dpll_hw_state.dpll & ++ ~(DPLL_VCO_ENABLE | DPLL_EXT_BUFFER_ENABLE_VLV)); ++ ++ /* No need to actually set up the DPLL with DSI */ ++ if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0) ++ return; ++ ++ mutex_lock(&dev_priv->sb_lock); ++ ++ bestn = pipe_config->dpll.n; ++ bestm1 = pipe_config->dpll.m1; ++ bestm2 = pipe_config->dpll.m2; ++ bestp1 = pipe_config->dpll.p1; ++ bestp2 = pipe_config->dpll.p2; ++ ++ /* See eDP HDMI DPIO driver vbios notes doc */ ++ ++ /* PLL B needs special handling */ ++ if (pipe == PIPE_B) ++ vlv_pllb_recal_opamp(dev_priv, pipe); ++ ++ /* Set up Tx target for periodic Rcomp update */ ++ vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9_BCAST, 0x0100000f); ++ ++ /* Disable target IRef on PLL */ ++ reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW8(pipe)); ++ reg_val &= 0x00ffffff; ++ vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW8(pipe), reg_val); ++ ++ /* Disable fast lock */ ++ vlv_dpio_write(dev_priv, pipe, VLV_CMN_DW0, 0x610); ++ ++ /* Set idtafcrecal before PLL is enabled */ ++ mdiv = ((bestm1 << DPIO_M1DIV_SHIFT) | (bestm2 & DPIO_M2DIV_MASK)); ++ mdiv |= ((bestp1 << DPIO_P1_SHIFT) | (bestp2 << DPIO_P2_SHIFT)); ++ mdiv |= ((bestn << DPIO_N_SHIFT)); ++ mdiv |= (1 << DPIO_K_SHIFT); ++ ++ /* ++ * Post divider depends on pixel clock rate, DAC vs digital (and LVDS, ++ * but we don't support that). ++ * Note: don't use the DAC post divider as it seems unstable. ++ */ ++ mdiv |= (DPIO_POST_DIV_HDMIDP << DPIO_POST_DIV_SHIFT); ++ vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv); ++ ++ mdiv |= DPIO_ENABLE_CALIBRATION; ++ vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv); ++ ++ /* Set HBR and RBR LPF coefficients */ ++ if (pipe_config->port_clock == 162000 || ++ intel_crtc_has_type(pipe_config, INTEL_OUTPUT_ANALOG) || ++ intel_crtc_has_type(pipe_config, INTEL_OUTPUT_HDMI)) ++ vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe), ++ 0x009f0003); ++ else ++ vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe), ++ 0x00d0000f); ++ ++ if (intel_crtc_has_dp_encoder(pipe_config)) { ++ /* Use SSC source */ ++ if (pipe == PIPE_A) ++ vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe), ++ 0x0df40000); ++ else ++ vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe), ++ 0x0df70000); ++ } else { /* HDMI or VGA */ ++ /* Use bend source */ ++ if (pipe == PIPE_A) ++ vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe), ++ 0x0df70000); ++ else ++ vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe), ++ 0x0df40000); ++ } ++ ++ coreclk = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW7(pipe)); ++ coreclk = (coreclk & 0x0000ff00) | 0x01c00000; ++ if (intel_crtc_has_dp_encoder(pipe_config)) ++ coreclk |= 0x01000000; ++ vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW7(pipe), coreclk); ++ ++ vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW11(pipe), 0x87871000); ++ mutex_unlock(&dev_priv->sb_lock); ++} ++ ++static void chv_prepare_pll(struct intel_crtc *crtc, ++ const struct intel_crtc_state *pipe_config) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ enum pipe pipe = crtc->pipe; ++ enum dpio_channel port = vlv_pipe_to_channel(pipe); ++ u32 loopfilter, tribuf_calcntr; ++ u32 bestn, bestm1, bestm2, bestp1, bestp2, bestm2_frac; ++ u32 dpio_val; ++ int vco; ++ ++ /* Enable Refclk and SSC */ ++ I915_WRITE(DPLL(pipe), ++ pipe_config->dpll_hw_state.dpll & ~DPLL_VCO_ENABLE); ++ ++ /* No need to actually set up the DPLL with DSI */ ++ if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0) ++ return; ++ ++ bestn = pipe_config->dpll.n; ++ bestm2_frac = pipe_config->dpll.m2 & 0x3fffff; ++ bestm1 = pipe_config->dpll.m1; ++ bestm2 = pipe_config->dpll.m2 >> 22; ++ bestp1 = pipe_config->dpll.p1; ++ bestp2 = pipe_config->dpll.p2; ++ vco = pipe_config->dpll.vco; ++ dpio_val = 0; ++ loopfilter = 0; ++ ++ mutex_lock(&dev_priv->sb_lock); ++ ++ /* p1 and p2 divider */ ++ vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW13(port), ++ 5 << DPIO_CHV_S1_DIV_SHIFT | ++ bestp1 << DPIO_CHV_P1_DIV_SHIFT | ++ bestp2 << DPIO_CHV_P2_DIV_SHIFT | ++ 1 << DPIO_CHV_K_DIV_SHIFT); ++ ++ /* Feedback post-divider - m2 */ ++ vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW0(port), bestm2); ++ ++ /* Feedback refclk divider - n and m1 */ ++ vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW1(port), ++ DPIO_CHV_M1_DIV_BY_2 | ++ 1 << DPIO_CHV_N_DIV_SHIFT); ++ ++ /* M2 fraction division */ ++ vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW2(port), bestm2_frac); ++ ++ /* M2 fraction division enable */ ++ dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port)); ++ dpio_val &= ~(DPIO_CHV_FEEDFWD_GAIN_MASK | DPIO_CHV_FRAC_DIV_EN); ++ dpio_val |= (2 << DPIO_CHV_FEEDFWD_GAIN_SHIFT); ++ if (bestm2_frac) ++ dpio_val |= DPIO_CHV_FRAC_DIV_EN; ++ vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW3(port), dpio_val); ++ ++ /* Program digital lock detect threshold */ ++ dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW9(port)); ++ dpio_val &= ~(DPIO_CHV_INT_LOCK_THRESHOLD_MASK | ++ DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE); ++ dpio_val |= (0x5 << DPIO_CHV_INT_LOCK_THRESHOLD_SHIFT); ++ if (!bestm2_frac) ++ dpio_val |= DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE; ++ vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW9(port), dpio_val); ++ ++ /* Loop filter */ ++ if (vco == 5400000) { ++ loopfilter |= (0x3 << DPIO_CHV_PROP_COEFF_SHIFT); ++ loopfilter |= (0x8 << DPIO_CHV_INT_COEFF_SHIFT); ++ loopfilter |= (0x1 << DPIO_CHV_GAIN_CTRL_SHIFT); ++ tribuf_calcntr = 0x9; ++ } else if (vco <= 6200000) { ++ loopfilter |= (0x5 << DPIO_CHV_PROP_COEFF_SHIFT); ++ loopfilter |= (0xB << DPIO_CHV_INT_COEFF_SHIFT); ++ loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT); ++ tribuf_calcntr = 0x9; ++ } else if (vco <= 6480000) { ++ loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT); ++ loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT); ++ loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT); ++ tribuf_calcntr = 0x8; ++ } else { ++ /* Not supported. Apply the same limits as in the max case */ ++ loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT); ++ loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT); ++ loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT); ++ tribuf_calcntr = 0; ++ } ++ vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW6(port), loopfilter); ++ ++ dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW8(port)); ++ dpio_val &= ~DPIO_CHV_TDC_TARGET_CNT_MASK; ++ dpio_val |= (tribuf_calcntr << DPIO_CHV_TDC_TARGET_CNT_SHIFT); ++ vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW8(port), dpio_val); ++ ++ /* AFC Recal */ ++ vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), ++ vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)) | ++ DPIO_AFC_RECAL); ++ ++ mutex_unlock(&dev_priv->sb_lock); ++} ++ ++/** ++ * vlv_force_pll_on - forcibly enable just the PLL ++ * @dev_priv: i915 private structure ++ * @pipe: pipe PLL to enable ++ * @dpll: PLL configuration ++ * ++ * Enable the PLL for @pipe using the supplied @dpll config. To be used ++ * in cases where we need the PLL enabled even when @pipe is not going to ++ * be enabled. ++ */ ++int vlv_force_pll_on(struct drm_i915_private *dev_priv, enum pipe pipe, ++ const struct dpll *dpll) ++{ ++ struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); ++ struct intel_crtc_state *pipe_config; ++ ++ pipe_config = kzalloc(sizeof(*pipe_config), GFP_KERNEL); ++ if (!pipe_config) ++ return -ENOMEM; ++ ++ pipe_config->base.crtc = &crtc->base; ++ pipe_config->pixel_multiplier = 1; ++ pipe_config->dpll = *dpll; ++ ++ if (IS_CHERRYVIEW(dev_priv)) { ++ chv_compute_dpll(crtc, pipe_config); ++ chv_prepare_pll(crtc, pipe_config); ++ chv_enable_pll(crtc, pipe_config); ++ } else { ++ vlv_compute_dpll(crtc, pipe_config); ++ vlv_prepare_pll(crtc, pipe_config); ++ vlv_enable_pll(crtc, pipe_config); ++ } ++ ++ kfree(pipe_config); ++ ++ return 0; ++} ++ ++/** ++ * vlv_force_pll_off - forcibly disable just the PLL ++ * @dev_priv: i915 private structure ++ * @pipe: pipe PLL to disable ++ * ++ * Disable the PLL for @pipe. To be used in cases where we need ++ * the PLL enabled even when @pipe is not going to be enabled. ++ */ ++void vlv_force_pll_off(struct drm_i915_private *dev_priv, enum pipe pipe) ++{ ++ if (IS_CHERRYVIEW(dev_priv)) ++ chv_disable_pll(dev_priv, pipe); ++ else ++ vlv_disable_pll(dev_priv, pipe); ++} ++ ++static void i9xx_compute_dpll(struct intel_crtc *crtc, ++ struct intel_crtc_state *crtc_state, ++ struct dpll *reduced_clock) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ u32 dpll; ++ struct dpll *clock = &crtc_state->dpll; ++ ++ i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock); ++ ++ dpll = DPLL_VGA_MODE_DIS; ++ ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) ++ dpll |= DPLLB_MODE_LVDS; ++ else ++ dpll |= DPLLB_MODE_DAC_SERIAL; ++ ++ if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) || ++ IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) { ++ dpll |= (crtc_state->pixel_multiplier - 1) ++ << SDVO_MULTIPLIER_SHIFT_HIRES; ++ } ++ ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) || ++ intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) ++ dpll |= DPLL_SDVO_HIGH_SPEED; ++ ++ if (intel_crtc_has_dp_encoder(crtc_state)) ++ dpll |= DPLL_SDVO_HIGH_SPEED; ++ ++ /* compute bitmask from p1 value */ ++ if (IS_PINEVIEW(dev_priv)) ++ dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW; ++ else { ++ dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT; ++ if (IS_G4X(dev_priv) && reduced_clock) ++ dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT; ++ } ++ switch (clock->p2) { ++ case 5: ++ dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5; ++ break; ++ case 7: ++ dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7; ++ break; ++ case 10: ++ dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10; ++ break; ++ case 14: ++ dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14; ++ break; ++ } ++ if (INTEL_GEN(dev_priv) >= 4) ++ dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT); ++ ++ if (crtc_state->sdvo_tv_clock) ++ dpll |= PLL_REF_INPUT_TVCLKINBC; ++ else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) && ++ intel_panel_use_ssc(dev_priv)) ++ dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN; ++ else ++ dpll |= PLL_REF_INPUT_DREFCLK; ++ ++ dpll |= DPLL_VCO_ENABLE; ++ crtc_state->dpll_hw_state.dpll = dpll; ++ ++ if (INTEL_GEN(dev_priv) >= 4) { ++ u32 dpll_md = (crtc_state->pixel_multiplier - 1) ++ << DPLL_MD_UDI_MULTIPLIER_SHIFT; ++ crtc_state->dpll_hw_state.dpll_md = dpll_md; ++ } ++} ++ ++static void i8xx_compute_dpll(struct intel_crtc *crtc, ++ struct intel_crtc_state *crtc_state, ++ struct dpll *reduced_clock) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ u32 dpll; ++ struct dpll *clock = &crtc_state->dpll; ++ ++ i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock); ++ ++ dpll = DPLL_VGA_MODE_DIS; ++ ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) { ++ dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT; ++ } else { ++ if (clock->p1 == 2) ++ dpll |= PLL_P1_DIVIDE_BY_TWO; ++ else ++ dpll |= (clock->p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT; ++ if (clock->p2 == 4) ++ dpll |= PLL_P2_DIVIDE_BY_4; ++ } ++ ++ /* ++ * Bspec: ++ * "[Almador Errata}: For the correct operation of the muxed DVO pins ++ * (GDEVSELB/I2Cdata, GIRDBY/I2CClk) and (GFRAMEB/DVI_Data, ++ * GTRDYB/DVI_Clk): Bit 31 (DPLL VCO Enable) and Bit 30 (2X Clock ++ * Enable) must be set to “1” in both the DPLL A Control Register ++ * (06014h-06017h) and DPLL B Control Register (06018h-0601Bh)." ++ * ++ * For simplicity We simply keep both bits always enabled in ++ * both DPLLS. The spec says we should disable the DVO 2X clock ++ * when not needed, but this seems to work fine in practice. ++ */ ++ if (IS_I830(dev_priv) || ++ intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO)) ++ dpll |= DPLL_DVO_2X_MODE; ++ ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) && ++ intel_panel_use_ssc(dev_priv)) ++ dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN; ++ else ++ dpll |= PLL_REF_INPUT_DREFCLK; ++ ++ dpll |= DPLL_VCO_ENABLE; ++ crtc_state->dpll_hw_state.dpll = dpll; ++} ++ ++static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; ++ const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode; ++ u32 crtc_vtotal, crtc_vblank_end; ++ int vsyncshift = 0; ++ ++ /* We need to be careful not to changed the adjusted mode, for otherwise ++ * the hw state checker will get angry at the mismatch. */ ++ crtc_vtotal = adjusted_mode->crtc_vtotal; ++ crtc_vblank_end = adjusted_mode->crtc_vblank_end; ++ ++ if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) { ++ /* the chip adds 2 halflines automatically */ ++ crtc_vtotal -= 1; ++ crtc_vblank_end -= 1; ++ ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO)) ++ vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2; ++ else ++ vsyncshift = adjusted_mode->crtc_hsync_start - ++ adjusted_mode->crtc_htotal / 2; ++ if (vsyncshift < 0) ++ vsyncshift += adjusted_mode->crtc_htotal; ++ } ++ ++ if (INTEL_GEN(dev_priv) > 3) ++ I915_WRITE(VSYNCSHIFT(cpu_transcoder), vsyncshift); ++ ++ I915_WRITE(HTOTAL(cpu_transcoder), ++ (adjusted_mode->crtc_hdisplay - 1) | ++ ((adjusted_mode->crtc_htotal - 1) << 16)); ++ I915_WRITE(HBLANK(cpu_transcoder), ++ (adjusted_mode->crtc_hblank_start - 1) | ++ ((adjusted_mode->crtc_hblank_end - 1) << 16)); ++ I915_WRITE(HSYNC(cpu_transcoder), ++ (adjusted_mode->crtc_hsync_start - 1) | ++ ((adjusted_mode->crtc_hsync_end - 1) << 16)); ++ ++ I915_WRITE(VTOTAL(cpu_transcoder), ++ (adjusted_mode->crtc_vdisplay - 1) | ++ ((crtc_vtotal - 1) << 16)); ++ I915_WRITE(VBLANK(cpu_transcoder), ++ (adjusted_mode->crtc_vblank_start - 1) | ++ ((crtc_vblank_end - 1) << 16)); ++ I915_WRITE(VSYNC(cpu_transcoder), ++ (adjusted_mode->crtc_vsync_start - 1) | ++ ((adjusted_mode->crtc_vsync_end - 1) << 16)); ++ ++ /* Workaround: when the EDP input selection is B, the VTOTAL_B must be ++ * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is ++ * documented on the DDI_FUNC_CTL register description, EDP Input Select ++ * bits. */ ++ if (IS_HASWELL(dev_priv) && cpu_transcoder == TRANSCODER_EDP && ++ (pipe == PIPE_B || pipe == PIPE_C)) ++ I915_WRITE(VTOTAL(pipe), I915_READ(VTOTAL(cpu_transcoder))); ++ ++} ++ ++static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ ++ /* pipesrc controls the size that is scaled from, which should ++ * always be the user's requested size. ++ */ ++ I915_WRITE(PIPESRC(pipe), ++ ((crtc_state->pipe_src_w - 1) << 16) | ++ (crtc_state->pipe_src_h - 1)); ++} ++ ++static void intel_get_pipe_timings(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ enum transcoder cpu_transcoder = pipe_config->cpu_transcoder; ++ u32 tmp; ++ ++ tmp = I915_READ(HTOTAL(cpu_transcoder)); ++ pipe_config->base.adjusted_mode.crtc_hdisplay = (tmp & 0xffff) + 1; ++ pipe_config->base.adjusted_mode.crtc_htotal = ((tmp >> 16) & 0xffff) + 1; ++ tmp = I915_READ(HBLANK(cpu_transcoder)); ++ pipe_config->base.adjusted_mode.crtc_hblank_start = (tmp & 0xffff) + 1; ++ pipe_config->base.adjusted_mode.crtc_hblank_end = ((tmp >> 16) & 0xffff) + 1; ++ tmp = I915_READ(HSYNC(cpu_transcoder)); ++ pipe_config->base.adjusted_mode.crtc_hsync_start = (tmp & 0xffff) + 1; ++ pipe_config->base.adjusted_mode.crtc_hsync_end = ((tmp >> 16) & 0xffff) + 1; ++ ++ tmp = I915_READ(VTOTAL(cpu_transcoder)); ++ pipe_config->base.adjusted_mode.crtc_vdisplay = (tmp & 0xffff) + 1; ++ pipe_config->base.adjusted_mode.crtc_vtotal = ((tmp >> 16) & 0xffff) + 1; ++ tmp = I915_READ(VBLANK(cpu_transcoder)); ++ pipe_config->base.adjusted_mode.crtc_vblank_start = (tmp & 0xffff) + 1; ++ pipe_config->base.adjusted_mode.crtc_vblank_end = ((tmp >> 16) & 0xffff) + 1; ++ tmp = I915_READ(VSYNC(cpu_transcoder)); ++ pipe_config->base.adjusted_mode.crtc_vsync_start = (tmp & 0xffff) + 1; ++ pipe_config->base.adjusted_mode.crtc_vsync_end = ((tmp >> 16) & 0xffff) + 1; ++ ++ if (I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK) { ++ pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_INTERLACE; ++ pipe_config->base.adjusted_mode.crtc_vtotal += 1; ++ pipe_config->base.adjusted_mode.crtc_vblank_end += 1; ++ } ++} ++ ++static void intel_get_pipe_src_size(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ u32 tmp; ++ ++ tmp = I915_READ(PIPESRC(crtc->pipe)); ++ pipe_config->pipe_src_h = (tmp & 0xffff) + 1; ++ pipe_config->pipe_src_w = ((tmp >> 16) & 0xffff) + 1; ++ ++ pipe_config->base.mode.vdisplay = pipe_config->pipe_src_h; ++ pipe_config->base.mode.hdisplay = pipe_config->pipe_src_w; ++} ++ ++void intel_mode_from_pipe_config(struct drm_display_mode *mode, ++ struct intel_crtc_state *pipe_config) ++{ ++ mode->hdisplay = pipe_config->base.adjusted_mode.crtc_hdisplay; ++ mode->htotal = pipe_config->base.adjusted_mode.crtc_htotal; ++ mode->hsync_start = pipe_config->base.adjusted_mode.crtc_hsync_start; ++ mode->hsync_end = pipe_config->base.adjusted_mode.crtc_hsync_end; ++ ++ mode->vdisplay = pipe_config->base.adjusted_mode.crtc_vdisplay; ++ mode->vtotal = pipe_config->base.adjusted_mode.crtc_vtotal; ++ mode->vsync_start = pipe_config->base.adjusted_mode.crtc_vsync_start; ++ mode->vsync_end = pipe_config->base.adjusted_mode.crtc_vsync_end; ++ ++ mode->flags = pipe_config->base.adjusted_mode.flags; ++ mode->type = DRM_MODE_TYPE_DRIVER; ++ ++ mode->clock = pipe_config->base.adjusted_mode.crtc_clock; ++ ++ mode->hsync = drm_mode_hsync(mode); ++ mode->vrefresh = drm_mode_vrefresh(mode); ++ drm_mode_set_name(mode); ++} ++ ++static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ u32 pipeconf; ++ ++ pipeconf = 0; ++ ++ /* we keep both pipes enabled on 830 */ ++ if (IS_I830(dev_priv)) ++ pipeconf |= I915_READ(PIPECONF(crtc->pipe)) & PIPECONF_ENABLE; ++ ++ if (crtc_state->double_wide) ++ pipeconf |= PIPECONF_DOUBLE_WIDE; ++ ++ /* only g4x and later have fancy bpc/dither controls */ ++ if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) || ++ IS_CHERRYVIEW(dev_priv)) { ++ /* Bspec claims that we can't use dithering for 30bpp pipes. */ ++ if (crtc_state->dither && crtc_state->pipe_bpp != 30) ++ pipeconf |= PIPECONF_DITHER_EN | ++ PIPECONF_DITHER_TYPE_SP; ++ ++ switch (crtc_state->pipe_bpp) { ++ case 18: ++ pipeconf |= PIPECONF_6BPC; ++ break; ++ case 24: ++ pipeconf |= PIPECONF_8BPC; ++ break; ++ case 30: ++ pipeconf |= PIPECONF_10BPC; ++ break; ++ default: ++ /* Case prevented by intel_choose_pipe_bpp_dither. */ ++ BUG(); ++ } ++ } ++ ++ if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) { ++ if (INTEL_GEN(dev_priv) < 4 || ++ intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO)) ++ pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION; ++ else ++ pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT; ++ } else { ++ pipeconf |= PIPECONF_PROGRESSIVE; ++ } ++ ++ if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) && ++ crtc_state->limited_color_range) ++ pipeconf |= PIPECONF_COLOR_RANGE_SELECT; ++ ++ pipeconf |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode); ++ ++ I915_WRITE(PIPECONF(crtc->pipe), pipeconf); ++ POSTING_READ(PIPECONF(crtc->pipe)); ++} ++ ++static int i8xx_crtc_compute_clock(struct intel_crtc *crtc, ++ struct intel_crtc_state *crtc_state) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ const struct intel_limit *limit; ++ int refclk = 48000; ++ ++ memset(&crtc_state->dpll_hw_state, 0, ++ sizeof(crtc_state->dpll_hw_state)); ++ ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) { ++ if (intel_panel_use_ssc(dev_priv)) { ++ refclk = dev_priv->vbt.lvds_ssc_freq; ++ DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk); ++ } ++ ++ limit = &intel_limits_i8xx_lvds; ++ } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO)) { ++ limit = &intel_limits_i8xx_dvo; ++ } else { ++ limit = &intel_limits_i8xx_dac; ++ } ++ ++ if (!crtc_state->clock_set && ++ !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock, ++ refclk, NULL, &crtc_state->dpll)) { ++ DRM_ERROR("Couldn't find PLL settings for mode!\n"); ++ return -EINVAL; ++ } ++ ++ i8xx_compute_dpll(crtc, crtc_state, NULL); ++ ++ return 0; ++} ++ ++static int g4x_crtc_compute_clock(struct intel_crtc *crtc, ++ struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ const struct intel_limit *limit; ++ int refclk = 96000; ++ ++ memset(&crtc_state->dpll_hw_state, 0, ++ sizeof(crtc_state->dpll_hw_state)); ++ ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) { ++ if (intel_panel_use_ssc(dev_priv)) { ++ refclk = dev_priv->vbt.lvds_ssc_freq; ++ DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk); ++ } ++ ++ if (intel_is_dual_link_lvds(dev_priv)) ++ limit = &intel_limits_g4x_dual_channel_lvds; ++ else ++ limit = &intel_limits_g4x_single_channel_lvds; ++ } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) || ++ intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) { ++ limit = &intel_limits_g4x_hdmi; ++ } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO)) { ++ limit = &intel_limits_g4x_sdvo; ++ } else { ++ /* The option is for other outputs */ ++ limit = &intel_limits_i9xx_sdvo; ++ } ++ ++ if (!crtc_state->clock_set && ++ !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock, ++ refclk, NULL, &crtc_state->dpll)) { ++ DRM_ERROR("Couldn't find PLL settings for mode!\n"); ++ return -EINVAL; ++ } ++ ++ i9xx_compute_dpll(crtc, crtc_state, NULL); ++ ++ return 0; ++} ++ ++static int pnv_crtc_compute_clock(struct intel_crtc *crtc, ++ struct intel_crtc_state *crtc_state) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ const struct intel_limit *limit; ++ int refclk = 96000; ++ ++ memset(&crtc_state->dpll_hw_state, 0, ++ sizeof(crtc_state->dpll_hw_state)); ++ ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) { ++ if (intel_panel_use_ssc(dev_priv)) { ++ refclk = dev_priv->vbt.lvds_ssc_freq; ++ DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk); ++ } ++ ++ limit = &intel_limits_pineview_lvds; ++ } else { ++ limit = &intel_limits_pineview_sdvo; ++ } ++ ++ if (!crtc_state->clock_set && ++ !pnv_find_best_dpll(limit, crtc_state, crtc_state->port_clock, ++ refclk, NULL, &crtc_state->dpll)) { ++ DRM_ERROR("Couldn't find PLL settings for mode!\n"); ++ return -EINVAL; ++ } ++ ++ i9xx_compute_dpll(crtc, crtc_state, NULL); ++ ++ return 0; ++} ++ ++static int i9xx_crtc_compute_clock(struct intel_crtc *crtc, ++ struct intel_crtc_state *crtc_state) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ const struct intel_limit *limit; ++ int refclk = 96000; ++ ++ memset(&crtc_state->dpll_hw_state, 0, ++ sizeof(crtc_state->dpll_hw_state)); ++ ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) { ++ if (intel_panel_use_ssc(dev_priv)) { ++ refclk = dev_priv->vbt.lvds_ssc_freq; ++ DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk); ++ } ++ ++ limit = &intel_limits_i9xx_lvds; ++ } else { ++ limit = &intel_limits_i9xx_sdvo; ++ } ++ ++ if (!crtc_state->clock_set && ++ !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock, ++ refclk, NULL, &crtc_state->dpll)) { ++ DRM_ERROR("Couldn't find PLL settings for mode!\n"); ++ return -EINVAL; ++ } ++ ++ i9xx_compute_dpll(crtc, crtc_state, NULL); ++ ++ return 0; ++} ++ ++static int chv_crtc_compute_clock(struct intel_crtc *crtc, ++ struct intel_crtc_state *crtc_state) ++{ ++ int refclk = 100000; ++ const struct intel_limit *limit = &intel_limits_chv; ++ ++ memset(&crtc_state->dpll_hw_state, 0, ++ sizeof(crtc_state->dpll_hw_state)); ++ ++ if (!crtc_state->clock_set && ++ !chv_find_best_dpll(limit, crtc_state, crtc_state->port_clock, ++ refclk, NULL, &crtc_state->dpll)) { ++ DRM_ERROR("Couldn't find PLL settings for mode!\n"); ++ return -EINVAL; ++ } ++ ++ chv_compute_dpll(crtc, crtc_state); ++ ++ return 0; ++} ++ ++static int vlv_crtc_compute_clock(struct intel_crtc *crtc, ++ struct intel_crtc_state *crtc_state) ++{ ++ int refclk = 100000; ++ const struct intel_limit *limit = &intel_limits_vlv; ++ ++ memset(&crtc_state->dpll_hw_state, 0, ++ sizeof(crtc_state->dpll_hw_state)); ++ ++ if (!crtc_state->clock_set && ++ !vlv_find_best_dpll(limit, crtc_state, crtc_state->port_clock, ++ refclk, NULL, &crtc_state->dpll)) { ++ DRM_ERROR("Couldn't find PLL settings for mode!\n"); ++ return -EINVAL; ++ } ++ ++ vlv_compute_dpll(crtc, crtc_state); ++ ++ return 0; ++} ++ ++static bool i9xx_has_pfit(struct drm_i915_private *dev_priv) ++{ ++ if (IS_I830(dev_priv)) ++ return false; ++ ++ return INTEL_GEN(dev_priv) >= 4 || ++ IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv); ++} ++ ++static void i9xx_get_pfit_config(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ u32 tmp; ++ ++ if (!i9xx_has_pfit(dev_priv)) ++ return; ++ ++ tmp = I915_READ(PFIT_CONTROL); ++ if (!(tmp & PFIT_ENABLE)) ++ return; ++ ++ /* Check whether the pfit is attached to our pipe. */ ++ if (INTEL_GEN(dev_priv) < 4) { ++ if (crtc->pipe != PIPE_B) ++ return; ++ } else { ++ if ((tmp & PFIT_PIPE_MASK) != (crtc->pipe << PFIT_PIPE_SHIFT)) ++ return; ++ } ++ ++ pipe_config->gmch_pfit.control = tmp; ++ pipe_config->gmch_pfit.pgm_ratios = I915_READ(PFIT_PGM_RATIOS); ++} ++ ++static void vlv_crtc_clock_get(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ int pipe = pipe_config->cpu_transcoder; ++ struct dpll clock; ++ u32 mdiv; ++ int refclk = 100000; ++ ++ /* In case of DSI, DPLL will not be used */ ++ if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0) ++ return; ++ ++ mutex_lock(&dev_priv->sb_lock); ++ mdiv = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW3(pipe)); ++ mutex_unlock(&dev_priv->sb_lock); ++ ++ clock.m1 = (mdiv >> DPIO_M1DIV_SHIFT) & 7; ++ clock.m2 = mdiv & DPIO_M2DIV_MASK; ++ clock.n = (mdiv >> DPIO_N_SHIFT) & 0xf; ++ clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7; ++ clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f; ++ ++ pipe_config->port_clock = vlv_calc_dpll_params(refclk, &clock); ++} ++ ++static void ++i9xx_get_initial_plane_config(struct intel_crtc *crtc, ++ struct intel_initial_plane_config *plane_config) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_plane *plane = to_intel_plane(crtc->base.primary); ++ enum i9xx_plane_id i9xx_plane = plane->i9xx_plane; ++ enum pipe pipe; ++ u32 val, base, offset; ++ int fourcc, pixel_format; ++ unsigned int aligned_height; ++ struct drm_framebuffer *fb; ++ struct intel_framebuffer *intel_fb; ++ ++ if (!plane->get_hw_state(plane, &pipe)) ++ return; ++ ++ WARN_ON(pipe != crtc->pipe); ++ ++ intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL); ++ if (!intel_fb) { ++ DRM_DEBUG_KMS("failed to alloc fb\n"); ++ return; ++ } ++ ++ fb = &intel_fb->base; ++ ++ fb->dev = dev; ++ ++ val = I915_READ(DSPCNTR(i9xx_plane)); ++ ++ if (INTEL_GEN(dev_priv) >= 4) { ++ if (val & DISPPLANE_TILED) { ++ plane_config->tiling = I915_TILING_X; ++ fb->modifier = I915_FORMAT_MOD_X_TILED; ++ } ++ ++ if (val & DISPPLANE_ROTATE_180) ++ plane_config->rotation = DRM_MODE_ROTATE_180; ++ } ++ ++ if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B && ++ val & DISPPLANE_MIRROR) ++ plane_config->rotation |= DRM_MODE_REFLECT_X; ++ ++ pixel_format = val & DISPPLANE_PIXFORMAT_MASK; ++ fourcc = i9xx_format_to_fourcc(pixel_format); ++ fb->format = drm_format_info(fourcc); ++ ++ if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { ++ offset = I915_READ(DSPOFFSET(i9xx_plane)); ++ base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000; ++ } else if (INTEL_GEN(dev_priv) >= 4) { ++ if (plane_config->tiling) ++ offset = I915_READ(DSPTILEOFF(i9xx_plane)); ++ else ++ offset = I915_READ(DSPLINOFF(i9xx_plane)); ++ base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000; ++ } else { ++ base = I915_READ(DSPADDR(i9xx_plane)); ++ } ++ plane_config->base = base; ++ ++ val = I915_READ(PIPESRC(pipe)); ++ fb->width = ((val >> 16) & 0xfff) + 1; ++ fb->height = ((val >> 0) & 0xfff) + 1; ++ ++ val = I915_READ(DSPSTRIDE(i9xx_plane)); ++ fb->pitches[0] = val & 0xffffffc0; ++ ++ aligned_height = intel_fb_align_height(fb, 0, fb->height); ++ ++ plane_config->size = fb->pitches[0] * aligned_height; ++ ++ DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n", ++ crtc->base.name, plane->base.name, fb->width, fb->height, ++ fb->format->cpp[0] * 8, base, fb->pitches[0], ++ plane_config->size); ++ ++ plane_config->fb = intel_fb; ++} ++ ++static void chv_crtc_clock_get(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ int pipe = pipe_config->cpu_transcoder; ++ enum dpio_channel port = vlv_pipe_to_channel(pipe); ++ struct dpll clock; ++ u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3; ++ int refclk = 100000; ++ ++ /* In case of DSI, DPLL will not be used */ ++ if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0) ++ return; ++ ++ mutex_lock(&dev_priv->sb_lock); ++ cmn_dw13 = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW13(port)); ++ pll_dw0 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW0(port)); ++ pll_dw1 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW1(port)); ++ pll_dw2 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW2(port)); ++ pll_dw3 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port)); ++ mutex_unlock(&dev_priv->sb_lock); ++ ++ clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0; ++ clock.m2 = (pll_dw0 & 0xff) << 22; ++ if (pll_dw3 & DPIO_CHV_FRAC_DIV_EN) ++ clock.m2 |= pll_dw2 & 0x3fffff; ++ clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf; ++ clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7; ++ clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f; ++ ++ pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock); ++} ++ ++static void intel_get_crtc_ycbcr_config(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum intel_output_format output = INTEL_OUTPUT_FORMAT_RGB; ++ ++ pipe_config->lspcon_downsampling = false; ++ ++ if (IS_BROADWELL(dev_priv) || INTEL_GEN(dev_priv) >= 9) { ++ u32 tmp = I915_READ(PIPEMISC(crtc->pipe)); ++ ++ if (tmp & PIPEMISC_OUTPUT_COLORSPACE_YUV) { ++ bool ycbcr420_enabled = tmp & PIPEMISC_YUV420_ENABLE; ++ bool blend = tmp & PIPEMISC_YUV420_MODE_FULL_BLEND; ++ ++ if (ycbcr420_enabled) { ++ /* We support 4:2:0 in full blend mode only */ ++ if (!blend) ++ output = INTEL_OUTPUT_FORMAT_INVALID; ++ else if (!(IS_GEMINILAKE(dev_priv) || ++ INTEL_GEN(dev_priv) >= 10)) ++ output = INTEL_OUTPUT_FORMAT_INVALID; ++ else ++ output = INTEL_OUTPUT_FORMAT_YCBCR420; ++ } else { ++ /* ++ * Currently there is no interface defined to ++ * check user preference between RGB/YCBCR444 ++ * or YCBCR420. So the only possible case for ++ * YCBCR444 usage is driving YCBCR420 output ++ * with LSPCON, when pipe is configured for ++ * YCBCR444 output and LSPCON takes care of ++ * downsampling it. ++ */ ++ pipe_config->lspcon_downsampling = true; ++ output = INTEL_OUTPUT_FORMAT_YCBCR444; ++ } ++ } ++ } ++ ++ pipe_config->output_format = output; ++} ++ ++static void i9xx_get_pipe_color_config(struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct intel_plane *plane = to_intel_plane(crtc->base.primary); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum i9xx_plane_id i9xx_plane = plane->i9xx_plane; ++ u32 tmp; ++ ++ tmp = I915_READ(DSPCNTR(i9xx_plane)); ++ ++ if (tmp & DISPPLANE_GAMMA_ENABLE) ++ crtc_state->gamma_enable = true; ++ ++ if (!HAS_GMCH(dev_priv) && ++ tmp & DISPPLANE_PIPE_CSC_ENABLE) ++ crtc_state->csc_enable = true; ++} ++ ++static bool i9xx_get_pipe_config(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum intel_display_power_domain power_domain; ++ intel_wakeref_t wakeref; ++ u32 tmp; ++ bool ret; ++ ++ power_domain = POWER_DOMAIN_PIPE(crtc->pipe); ++ wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain); ++ if (!wakeref) ++ return false; ++ ++ pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; ++ pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe; ++ pipe_config->shared_dpll = NULL; ++ ++ ret = false; ++ ++ tmp = I915_READ(PIPECONF(crtc->pipe)); ++ if (!(tmp & PIPECONF_ENABLE)) ++ goto out; ++ ++ if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) || ++ IS_CHERRYVIEW(dev_priv)) { ++ switch (tmp & PIPECONF_BPC_MASK) { ++ case PIPECONF_6BPC: ++ pipe_config->pipe_bpp = 18; ++ break; ++ case PIPECONF_8BPC: ++ pipe_config->pipe_bpp = 24; ++ break; ++ case PIPECONF_10BPC: ++ pipe_config->pipe_bpp = 30; ++ break; ++ default: ++ break; ++ } ++ } ++ ++ if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) && ++ (tmp & PIPECONF_COLOR_RANGE_SELECT)) ++ pipe_config->limited_color_range = true; ++ ++ pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_I9XX) >> ++ PIPECONF_GAMMA_MODE_SHIFT; ++ ++ if (IS_CHERRYVIEW(dev_priv)) ++ pipe_config->cgm_mode = I915_READ(CGM_PIPE_MODE(crtc->pipe)); ++ ++ i9xx_get_pipe_color_config(pipe_config); ++ ++ if (INTEL_GEN(dev_priv) < 4) ++ pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE; ++ ++ intel_get_pipe_timings(crtc, pipe_config); ++ intel_get_pipe_src_size(crtc, pipe_config); ++ ++ i9xx_get_pfit_config(crtc, pipe_config); ++ ++ if (INTEL_GEN(dev_priv) >= 4) { ++ /* No way to read it out on pipes B and C */ ++ if (IS_CHERRYVIEW(dev_priv) && crtc->pipe != PIPE_A) ++ tmp = dev_priv->chv_dpll_md[crtc->pipe]; ++ else ++ tmp = I915_READ(DPLL_MD(crtc->pipe)); ++ pipe_config->pixel_multiplier = ++ ((tmp & DPLL_MD_UDI_MULTIPLIER_MASK) ++ >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1; ++ pipe_config->dpll_hw_state.dpll_md = tmp; ++ } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) || ++ IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) { ++ tmp = I915_READ(DPLL(crtc->pipe)); ++ pipe_config->pixel_multiplier = ++ ((tmp & SDVO_MULTIPLIER_MASK) ++ >> SDVO_MULTIPLIER_SHIFT_HIRES) + 1; ++ } else { ++ /* Note that on i915G/GM the pixel multiplier is in the sdvo ++ * port and will be fixed up in the encoder->get_config ++ * function. */ ++ pipe_config->pixel_multiplier = 1; ++ } ++ pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(crtc->pipe)); ++ if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) { ++ pipe_config->dpll_hw_state.fp0 = I915_READ(FP0(crtc->pipe)); ++ pipe_config->dpll_hw_state.fp1 = I915_READ(FP1(crtc->pipe)); ++ } else { ++ /* Mask out read-only status bits. */ ++ pipe_config->dpll_hw_state.dpll &= ~(DPLL_LOCK_VLV | ++ DPLL_PORTC_READY_MASK | ++ DPLL_PORTB_READY_MASK); ++ } ++ ++ if (IS_CHERRYVIEW(dev_priv)) ++ chv_crtc_clock_get(crtc, pipe_config); ++ else if (IS_VALLEYVIEW(dev_priv)) ++ vlv_crtc_clock_get(crtc, pipe_config); ++ else ++ i9xx_crtc_clock_get(crtc, pipe_config); ++ ++ /* ++ * Normally the dotclock is filled in by the encoder .get_config() ++ * but in case the pipe is enabled w/o any ports we need a sane ++ * default. ++ */ ++ pipe_config->base.adjusted_mode.crtc_clock = ++ pipe_config->port_clock / pipe_config->pixel_multiplier; ++ ++ ret = true; ++ ++out: ++ intel_display_power_put(dev_priv, power_domain, wakeref); ++ ++ return ret; ++} ++ ++static void ironlake_init_pch_refclk(struct drm_i915_private *dev_priv) ++{ ++ struct intel_encoder *encoder; ++ int i; ++ u32 val, final; ++ bool has_lvds = false; ++ bool has_cpu_edp = false; ++ bool has_panel = false; ++ bool has_ck505 = false; ++ bool can_ssc = false; ++ bool using_ssc_source = false; ++ ++ /* We need to take the global config into account */ ++ for_each_intel_encoder(&dev_priv->drm, encoder) { ++ switch (encoder->type) { ++ case INTEL_OUTPUT_LVDS: ++ has_panel = true; ++ has_lvds = true; ++ break; ++ case INTEL_OUTPUT_EDP: ++ has_panel = true; ++ if (encoder->port == PORT_A) ++ has_cpu_edp = true; ++ break; ++ default: ++ break; ++ } ++ } ++ ++ if (HAS_PCH_IBX(dev_priv)) { ++ has_ck505 = dev_priv->vbt.display_clock_mode; ++ can_ssc = has_ck505; ++ } else { ++ has_ck505 = false; ++ can_ssc = true; ++ } ++ ++ /* Check if any DPLLs are using the SSC source */ ++ for (i = 0; i < dev_priv->num_shared_dpll; i++) { ++ u32 temp = I915_READ(PCH_DPLL(i)); ++ ++ if (!(temp & DPLL_VCO_ENABLE)) ++ continue; ++ ++ if ((temp & PLL_REF_INPUT_MASK) == ++ PLLB_REF_INPUT_SPREADSPECTRUMIN) { ++ using_ssc_source = true; ++ break; ++ } ++ } ++ ++ DRM_DEBUG_KMS("has_panel %d has_lvds %d has_ck505 %d using_ssc_source %d\n", ++ has_panel, has_lvds, has_ck505, using_ssc_source); ++ ++ /* Ironlake: try to setup display ref clock before DPLL ++ * enabling. This is only under driver's control after ++ * PCH B stepping, previous chipset stepping should be ++ * ignoring this setting. ++ */ ++ val = I915_READ(PCH_DREF_CONTROL); ++ ++ /* As we must carefully and slowly disable/enable each source in turn, ++ * compute the final state we want first and check if we need to ++ * make any changes at all. ++ */ ++ final = val; ++ final &= ~DREF_NONSPREAD_SOURCE_MASK; ++ if (has_ck505) ++ final |= DREF_NONSPREAD_CK505_ENABLE; ++ else ++ final |= DREF_NONSPREAD_SOURCE_ENABLE; ++ ++ final &= ~DREF_SSC_SOURCE_MASK; ++ final &= ~DREF_CPU_SOURCE_OUTPUT_MASK; ++ final &= ~DREF_SSC1_ENABLE; ++ ++ if (has_panel) { ++ final |= DREF_SSC_SOURCE_ENABLE; ++ ++ if (intel_panel_use_ssc(dev_priv) && can_ssc) ++ final |= DREF_SSC1_ENABLE; ++ ++ if (has_cpu_edp) { ++ if (intel_panel_use_ssc(dev_priv) && can_ssc) ++ final |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD; ++ else ++ final |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD; ++ } else ++ final |= DREF_CPU_SOURCE_OUTPUT_DISABLE; ++ } else if (using_ssc_source) { ++ final |= DREF_SSC_SOURCE_ENABLE; ++ final |= DREF_SSC1_ENABLE; ++ } ++ ++ if (final == val) ++ return; ++ ++ /* Always enable nonspread source */ ++ val &= ~DREF_NONSPREAD_SOURCE_MASK; ++ ++ if (has_ck505) ++ val |= DREF_NONSPREAD_CK505_ENABLE; ++ else ++ val |= DREF_NONSPREAD_SOURCE_ENABLE; ++ ++ if (has_panel) { ++ val &= ~DREF_SSC_SOURCE_MASK; ++ val |= DREF_SSC_SOURCE_ENABLE; ++ ++ /* SSC must be turned on before enabling the CPU output */ ++ if (intel_panel_use_ssc(dev_priv) && can_ssc) { ++ DRM_DEBUG_KMS("Using SSC on panel\n"); ++ val |= DREF_SSC1_ENABLE; ++ } else ++ val &= ~DREF_SSC1_ENABLE; ++ ++ /* Get SSC going before enabling the outputs */ ++ I915_WRITE(PCH_DREF_CONTROL, val); ++ POSTING_READ(PCH_DREF_CONTROL); ++ udelay(200); ++ ++ val &= ~DREF_CPU_SOURCE_OUTPUT_MASK; ++ ++ /* Enable CPU source on CPU attached eDP */ ++ if (has_cpu_edp) { ++ if (intel_panel_use_ssc(dev_priv) && can_ssc) { ++ DRM_DEBUG_KMS("Using SSC on eDP\n"); ++ val |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD; ++ } else ++ val |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD; ++ } else ++ val |= DREF_CPU_SOURCE_OUTPUT_DISABLE; ++ ++ I915_WRITE(PCH_DREF_CONTROL, val); ++ POSTING_READ(PCH_DREF_CONTROL); ++ udelay(200); ++ } else { ++ DRM_DEBUG_KMS("Disabling CPU source output\n"); ++ ++ val &= ~DREF_CPU_SOURCE_OUTPUT_MASK; ++ ++ /* Turn off CPU output */ ++ val |= DREF_CPU_SOURCE_OUTPUT_DISABLE; ++ ++ I915_WRITE(PCH_DREF_CONTROL, val); ++ POSTING_READ(PCH_DREF_CONTROL); ++ udelay(200); ++ ++ if (!using_ssc_source) { ++ DRM_DEBUG_KMS("Disabling SSC source\n"); ++ ++ /* Turn off the SSC source */ ++ val &= ~DREF_SSC_SOURCE_MASK; ++ val |= DREF_SSC_SOURCE_DISABLE; ++ ++ /* Turn off SSC1 */ ++ val &= ~DREF_SSC1_ENABLE; ++ ++ I915_WRITE(PCH_DREF_CONTROL, val); ++ POSTING_READ(PCH_DREF_CONTROL); ++ udelay(200); ++ } ++ } ++ ++ BUG_ON(val != final); ++} ++ ++static void lpt_reset_fdi_mphy(struct drm_i915_private *dev_priv) ++{ ++ u32 tmp; ++ ++ tmp = I915_READ(SOUTH_CHICKEN2); ++ tmp |= FDI_MPHY_IOSFSB_RESET_CTL; ++ I915_WRITE(SOUTH_CHICKEN2, tmp); ++ ++ if (wait_for_us(I915_READ(SOUTH_CHICKEN2) & ++ FDI_MPHY_IOSFSB_RESET_STATUS, 100)) ++ DRM_ERROR("FDI mPHY reset assert timeout\n"); ++ ++ tmp = I915_READ(SOUTH_CHICKEN2); ++ tmp &= ~FDI_MPHY_IOSFSB_RESET_CTL; ++ I915_WRITE(SOUTH_CHICKEN2, tmp); ++ ++ if (wait_for_us((I915_READ(SOUTH_CHICKEN2) & ++ FDI_MPHY_IOSFSB_RESET_STATUS) == 0, 100)) ++ DRM_ERROR("FDI mPHY reset de-assert timeout\n"); ++} ++ ++/* WaMPhyProgramming:hsw */ ++static void lpt_program_fdi_mphy(struct drm_i915_private *dev_priv) ++{ ++ u32 tmp; ++ ++ tmp = intel_sbi_read(dev_priv, 0x8008, SBI_MPHY); ++ tmp &= ~(0xFF << 24); ++ tmp |= (0x12 << 24); ++ intel_sbi_write(dev_priv, 0x8008, tmp, SBI_MPHY); ++ ++ tmp = intel_sbi_read(dev_priv, 0x2008, SBI_MPHY); ++ tmp |= (1 << 11); ++ intel_sbi_write(dev_priv, 0x2008, tmp, SBI_MPHY); ++ ++ tmp = intel_sbi_read(dev_priv, 0x2108, SBI_MPHY); ++ tmp |= (1 << 11); ++ intel_sbi_write(dev_priv, 0x2108, tmp, SBI_MPHY); ++ ++ tmp = intel_sbi_read(dev_priv, 0x206C, SBI_MPHY); ++ tmp |= (1 << 24) | (1 << 21) | (1 << 18); ++ intel_sbi_write(dev_priv, 0x206C, tmp, SBI_MPHY); ++ ++ tmp = intel_sbi_read(dev_priv, 0x216C, SBI_MPHY); ++ tmp |= (1 << 24) | (1 << 21) | (1 << 18); ++ intel_sbi_write(dev_priv, 0x216C, tmp, SBI_MPHY); ++ ++ tmp = intel_sbi_read(dev_priv, 0x2080, SBI_MPHY); ++ tmp &= ~(7 << 13); ++ tmp |= (5 << 13); ++ intel_sbi_write(dev_priv, 0x2080, tmp, SBI_MPHY); ++ ++ tmp = intel_sbi_read(dev_priv, 0x2180, SBI_MPHY); ++ tmp &= ~(7 << 13); ++ tmp |= (5 << 13); ++ intel_sbi_write(dev_priv, 0x2180, tmp, SBI_MPHY); ++ ++ tmp = intel_sbi_read(dev_priv, 0x208C, SBI_MPHY); ++ tmp &= ~0xFF; ++ tmp |= 0x1C; ++ intel_sbi_write(dev_priv, 0x208C, tmp, SBI_MPHY); ++ ++ tmp = intel_sbi_read(dev_priv, 0x218C, SBI_MPHY); ++ tmp &= ~0xFF; ++ tmp |= 0x1C; ++ intel_sbi_write(dev_priv, 0x218C, tmp, SBI_MPHY); ++ ++ tmp = intel_sbi_read(dev_priv, 0x2098, SBI_MPHY); ++ tmp &= ~(0xFF << 16); ++ tmp |= (0x1C << 16); ++ intel_sbi_write(dev_priv, 0x2098, tmp, SBI_MPHY); ++ ++ tmp = intel_sbi_read(dev_priv, 0x2198, SBI_MPHY); ++ tmp &= ~(0xFF << 16); ++ tmp |= (0x1C << 16); ++ intel_sbi_write(dev_priv, 0x2198, tmp, SBI_MPHY); ++ ++ tmp = intel_sbi_read(dev_priv, 0x20C4, SBI_MPHY); ++ tmp |= (1 << 27); ++ intel_sbi_write(dev_priv, 0x20C4, tmp, SBI_MPHY); ++ ++ tmp = intel_sbi_read(dev_priv, 0x21C4, SBI_MPHY); ++ tmp |= (1 << 27); ++ intel_sbi_write(dev_priv, 0x21C4, tmp, SBI_MPHY); ++ ++ tmp = intel_sbi_read(dev_priv, 0x20EC, SBI_MPHY); ++ tmp &= ~(0xF << 28); ++ tmp |= (4 << 28); ++ intel_sbi_write(dev_priv, 0x20EC, tmp, SBI_MPHY); ++ ++ tmp = intel_sbi_read(dev_priv, 0x21EC, SBI_MPHY); ++ tmp &= ~(0xF << 28); ++ tmp |= (4 << 28); ++ intel_sbi_write(dev_priv, 0x21EC, tmp, SBI_MPHY); ++} ++ ++/* Implements 3 different sequences from BSpec chapter "Display iCLK ++ * Programming" based on the parameters passed: ++ * - Sequence to enable CLKOUT_DP ++ * - Sequence to enable CLKOUT_DP without spread ++ * - Sequence to enable CLKOUT_DP for FDI usage and configure PCH FDI I/O ++ */ ++static void lpt_enable_clkout_dp(struct drm_i915_private *dev_priv, ++ bool with_spread, bool with_fdi) ++{ ++ u32 reg, tmp; ++ ++ if (WARN(with_fdi && !with_spread, "FDI requires downspread\n")) ++ with_spread = true; ++ if (WARN(HAS_PCH_LPT_LP(dev_priv) && ++ with_fdi, "LP PCH doesn't have FDI\n")) ++ with_fdi = false; ++ ++ mutex_lock(&dev_priv->sb_lock); ++ ++ tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK); ++ tmp &= ~SBI_SSCCTL_DISABLE; ++ tmp |= SBI_SSCCTL_PATHALT; ++ intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK); ++ ++ udelay(24); ++ ++ if (with_spread) { ++ tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK); ++ tmp &= ~SBI_SSCCTL_PATHALT; ++ intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK); ++ ++ if (with_fdi) { ++ lpt_reset_fdi_mphy(dev_priv); ++ lpt_program_fdi_mphy(dev_priv); ++ } ++ } ++ ++ reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0; ++ tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK); ++ tmp |= SBI_GEN0_CFG_BUFFENABLE_DISABLE; ++ intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK); ++ ++ mutex_unlock(&dev_priv->sb_lock); ++} ++ ++/* Sequence to disable CLKOUT_DP */ ++static void lpt_disable_clkout_dp(struct drm_i915_private *dev_priv) ++{ ++ u32 reg, tmp; ++ ++ mutex_lock(&dev_priv->sb_lock); ++ ++ reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0; ++ tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK); ++ tmp &= ~SBI_GEN0_CFG_BUFFENABLE_DISABLE; ++ intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK); ++ ++ tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK); ++ if (!(tmp & SBI_SSCCTL_DISABLE)) { ++ if (!(tmp & SBI_SSCCTL_PATHALT)) { ++ tmp |= SBI_SSCCTL_PATHALT; ++ intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK); ++ udelay(32); ++ } ++ tmp |= SBI_SSCCTL_DISABLE; ++ intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK); ++ } ++ ++ mutex_unlock(&dev_priv->sb_lock); ++} ++ ++#define BEND_IDX(steps) ((50 + (steps)) / 5) ++ ++static const u16 sscdivintphase[] = { ++ [BEND_IDX( 50)] = 0x3B23, ++ [BEND_IDX( 45)] = 0x3B23, ++ [BEND_IDX( 40)] = 0x3C23, ++ [BEND_IDX( 35)] = 0x3C23, ++ [BEND_IDX( 30)] = 0x3D23, ++ [BEND_IDX( 25)] = 0x3D23, ++ [BEND_IDX( 20)] = 0x3E23, ++ [BEND_IDX( 15)] = 0x3E23, ++ [BEND_IDX( 10)] = 0x3F23, ++ [BEND_IDX( 5)] = 0x3F23, ++ [BEND_IDX( 0)] = 0x0025, ++ [BEND_IDX( -5)] = 0x0025, ++ [BEND_IDX(-10)] = 0x0125, ++ [BEND_IDX(-15)] = 0x0125, ++ [BEND_IDX(-20)] = 0x0225, ++ [BEND_IDX(-25)] = 0x0225, ++ [BEND_IDX(-30)] = 0x0325, ++ [BEND_IDX(-35)] = 0x0325, ++ [BEND_IDX(-40)] = 0x0425, ++ [BEND_IDX(-45)] = 0x0425, ++ [BEND_IDX(-50)] = 0x0525, ++}; ++ ++/* ++ * Bend CLKOUT_DP ++ * steps -50 to 50 inclusive, in steps of 5 ++ * < 0 slow down the clock, > 0 speed up the clock, 0 == no bend (135MHz) ++ * change in clock period = -(steps / 10) * 5.787 ps ++ */ ++static void lpt_bend_clkout_dp(struct drm_i915_private *dev_priv, int steps) ++{ ++ u32 tmp; ++ int idx = BEND_IDX(steps); ++ ++ if (WARN_ON(steps % 5 != 0)) ++ return; ++ ++ if (WARN_ON(idx >= ARRAY_SIZE(sscdivintphase))) ++ return; ++ ++ mutex_lock(&dev_priv->sb_lock); ++ ++ if (steps % 10 != 0) ++ tmp = 0xAAAAAAAB; ++ else ++ tmp = 0x00000000; ++ intel_sbi_write(dev_priv, SBI_SSCDITHPHASE, tmp, SBI_ICLK); ++ ++ tmp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE, SBI_ICLK); ++ tmp &= 0xffff0000; ++ tmp |= sscdivintphase[idx]; ++ intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE, tmp, SBI_ICLK); ++ ++ mutex_unlock(&dev_priv->sb_lock); ++} ++ ++#undef BEND_IDX ++ ++static void lpt_init_pch_refclk(struct drm_i915_private *dev_priv) ++{ ++ struct intel_encoder *encoder; ++ bool has_vga = false; ++ ++ for_each_intel_encoder(&dev_priv->drm, encoder) { ++ switch (encoder->type) { ++ case INTEL_OUTPUT_ANALOG: ++ has_vga = true; ++ break; ++ default: ++ break; ++ } ++ } ++ ++ if (has_vga) { ++ lpt_bend_clkout_dp(dev_priv, 0); ++ lpt_enable_clkout_dp(dev_priv, true, true); ++ } else { ++ lpt_disable_clkout_dp(dev_priv); ++ } ++} ++ ++/* ++ * Initialize reference clocks when the driver loads ++ */ ++void intel_init_pch_refclk(struct drm_i915_private *dev_priv) ++{ ++ if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) ++ ironlake_init_pch_refclk(dev_priv); ++ else if (HAS_PCH_LPT(dev_priv)) ++ lpt_init_pch_refclk(dev_priv); ++} ++ ++static void ironlake_set_pipeconf(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ u32 val; ++ ++ val = 0; ++ ++ switch (crtc_state->pipe_bpp) { ++ case 18: ++ val |= PIPECONF_6BPC; ++ break; ++ case 24: ++ val |= PIPECONF_8BPC; ++ break; ++ case 30: ++ val |= PIPECONF_10BPC; ++ break; ++ case 36: ++ val |= PIPECONF_12BPC; ++ break; ++ default: ++ /* Case prevented by intel_choose_pipe_bpp_dither. */ ++ BUG(); ++ } ++ ++ if (crtc_state->dither) ++ val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP); ++ ++ if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) ++ val |= PIPECONF_INTERLACED_ILK; ++ else ++ val |= PIPECONF_PROGRESSIVE; ++ ++ if (crtc_state->limited_color_range) ++ val |= PIPECONF_COLOR_RANGE_SELECT; ++ ++ val |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode); ++ ++ I915_WRITE(PIPECONF(pipe), val); ++ POSTING_READ(PIPECONF(pipe)); ++} ++ ++static void haswell_set_pipeconf(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; ++ u32 val = 0; ++ ++ if (IS_HASWELL(dev_priv) && crtc_state->dither) ++ val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP); ++ ++ if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) ++ val |= PIPECONF_INTERLACED_ILK; ++ else ++ val |= PIPECONF_PROGRESSIVE; ++ ++ I915_WRITE(PIPECONF(cpu_transcoder), val); ++ POSTING_READ(PIPECONF(cpu_transcoder)); ++} ++ ++static void haswell_set_pipemisc(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev); ++ ++ if (IS_BROADWELL(dev_priv) || INTEL_GEN(dev_priv) >= 9) { ++ u32 val = 0; ++ ++ switch (crtc_state->pipe_bpp) { ++ case 18: ++ val |= PIPEMISC_DITHER_6_BPC; ++ break; ++ case 24: ++ val |= PIPEMISC_DITHER_8_BPC; ++ break; ++ case 30: ++ val |= PIPEMISC_DITHER_10_BPC; ++ break; ++ case 36: ++ val |= PIPEMISC_DITHER_12_BPC; ++ break; ++ default: ++ /* Case prevented by pipe_config_set_bpp. */ ++ BUG(); ++ } ++ ++ if (crtc_state->dither) ++ val |= PIPEMISC_DITHER_ENABLE | PIPEMISC_DITHER_TYPE_SP; ++ ++ if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 || ++ crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444) ++ val |= PIPEMISC_OUTPUT_COLORSPACE_YUV; ++ ++ if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420) ++ val |= PIPEMISC_YUV420_ENABLE | ++ PIPEMISC_YUV420_MODE_FULL_BLEND; ++ ++ I915_WRITE(PIPEMISC(intel_crtc->pipe), val); ++ } ++} ++ ++int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp) ++{ ++ /* ++ * Account for spread spectrum to avoid ++ * oversubscribing the link. Max center spread ++ * is 2.5%; use 5% for safety's sake. ++ */ ++ u32 bps = target_clock * bpp * 21 / 20; ++ return DIV_ROUND_UP(bps, link_bw * 8); ++} ++ ++static bool ironlake_needs_fb_cb_tune(struct dpll *dpll, int factor) ++{ ++ return i9xx_dpll_compute_m(dpll) < factor * dpll->n; ++} ++ ++static void ironlake_compute_dpll(struct intel_crtc *crtc, ++ struct intel_crtc_state *crtc_state, ++ struct dpll *reduced_clock) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ u32 dpll, fp, fp2; ++ int factor; ++ ++ /* Enable autotuning of the PLL clock (if permissible) */ ++ factor = 21; ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) { ++ if ((intel_panel_use_ssc(dev_priv) && ++ dev_priv->vbt.lvds_ssc_freq == 100000) || ++ (HAS_PCH_IBX(dev_priv) && ++ intel_is_dual_link_lvds(dev_priv))) ++ factor = 25; ++ } else if (crtc_state->sdvo_tv_clock) { ++ factor = 20; ++ } ++ ++ fp = i9xx_dpll_compute_fp(&crtc_state->dpll); ++ ++ if (ironlake_needs_fb_cb_tune(&crtc_state->dpll, factor)) ++ fp |= FP_CB_TUNE; ++ ++ if (reduced_clock) { ++ fp2 = i9xx_dpll_compute_fp(reduced_clock); ++ ++ if (reduced_clock->m < factor * reduced_clock->n) ++ fp2 |= FP_CB_TUNE; ++ } else { ++ fp2 = fp; ++ } ++ ++ dpll = 0; ++ ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) ++ dpll |= DPLLB_MODE_LVDS; ++ else ++ dpll |= DPLLB_MODE_DAC_SERIAL; ++ ++ dpll |= (crtc_state->pixel_multiplier - 1) ++ << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT; ++ ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) || ++ intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) ++ dpll |= DPLL_SDVO_HIGH_SPEED; ++ ++ if (intel_crtc_has_dp_encoder(crtc_state)) ++ dpll |= DPLL_SDVO_HIGH_SPEED; ++ ++ /* ++ * The high speed IO clock is only really required for ++ * SDVO/HDMI/DP, but we also enable it for CRT to make it ++ * possible to share the DPLL between CRT and HDMI. Enabling ++ * the clock needlessly does no real harm, except use up a ++ * bit of power potentially. ++ * ++ * We'll limit this to IVB with 3 pipes, since it has only two ++ * DPLLs and so DPLL sharing is the only way to get three pipes ++ * driving PCH ports at the same time. On SNB we could do this, ++ * and potentially avoid enabling the second DPLL, but it's not ++ * clear if it''s a win or loss power wise. No point in doing ++ * this on ILK at all since it has a fixed DPLL<->pipe mapping. ++ */ ++ if (INTEL_INFO(dev_priv)->num_pipes == 3 && ++ intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) ++ dpll |= DPLL_SDVO_HIGH_SPEED; ++ ++ /* compute bitmask from p1 value */ ++ dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT; ++ /* also FPA1 */ ++ dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT; ++ ++ switch (crtc_state->dpll.p2) { ++ case 5: ++ dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5; ++ break; ++ case 7: ++ dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7; ++ break; ++ case 10: ++ dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10; ++ break; ++ case 14: ++ dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14; ++ break; ++ } ++ ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) && ++ intel_panel_use_ssc(dev_priv)) ++ dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN; ++ else ++ dpll |= PLL_REF_INPUT_DREFCLK; ++ ++ dpll |= DPLL_VCO_ENABLE; ++ ++ crtc_state->dpll_hw_state.dpll = dpll; ++ crtc_state->dpll_hw_state.fp0 = fp; ++ crtc_state->dpll_hw_state.fp1 = fp2; ++} ++ ++static int ironlake_crtc_compute_clock(struct intel_crtc *crtc, ++ struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ const struct intel_limit *limit; ++ int refclk = 120000; ++ ++ memset(&crtc_state->dpll_hw_state, 0, ++ sizeof(crtc_state->dpll_hw_state)); ++ ++ /* CPU eDP is the only output that doesn't need a PCH PLL of its own. */ ++ if (!crtc_state->has_pch_encoder) ++ return 0; ++ ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) { ++ if (intel_panel_use_ssc(dev_priv)) { ++ DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", ++ dev_priv->vbt.lvds_ssc_freq); ++ refclk = dev_priv->vbt.lvds_ssc_freq; ++ } ++ ++ if (intel_is_dual_link_lvds(dev_priv)) { ++ if (refclk == 100000) ++ limit = &intel_limits_ironlake_dual_lvds_100m; ++ else ++ limit = &intel_limits_ironlake_dual_lvds; ++ } else { ++ if (refclk == 100000) ++ limit = &intel_limits_ironlake_single_lvds_100m; ++ else ++ limit = &intel_limits_ironlake_single_lvds; ++ } ++ } else { ++ limit = &intel_limits_ironlake_dac; ++ } ++ ++ if (!crtc_state->clock_set && ++ !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock, ++ refclk, NULL, &crtc_state->dpll)) { ++ DRM_ERROR("Couldn't find PLL settings for mode!\n"); ++ return -EINVAL; ++ } ++ ++ ironlake_compute_dpll(crtc, crtc_state, NULL); ++ ++ if (!intel_get_shared_dpll(crtc_state, NULL)) { ++ DRM_DEBUG_KMS("failed to find PLL for pipe %c\n", ++ pipe_name(crtc->pipe)); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc, ++ struct intel_link_m_n *m_n) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ enum pipe pipe = crtc->pipe; ++ ++ m_n->link_m = I915_READ(PCH_TRANS_LINK_M1(pipe)); ++ m_n->link_n = I915_READ(PCH_TRANS_LINK_N1(pipe)); ++ m_n->gmch_m = I915_READ(PCH_TRANS_DATA_M1(pipe)) ++ & ~TU_SIZE_MASK; ++ m_n->gmch_n = I915_READ(PCH_TRANS_DATA_N1(pipe)); ++ m_n->tu = ((I915_READ(PCH_TRANS_DATA_M1(pipe)) ++ & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1; ++} ++ ++static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc, ++ enum transcoder transcoder, ++ struct intel_link_m_n *m_n, ++ struct intel_link_m_n *m2_n2) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ enum pipe pipe = crtc->pipe; ++ ++ if (INTEL_GEN(dev_priv) >= 5) { ++ m_n->link_m = I915_READ(PIPE_LINK_M1(transcoder)); ++ m_n->link_n = I915_READ(PIPE_LINK_N1(transcoder)); ++ m_n->gmch_m = I915_READ(PIPE_DATA_M1(transcoder)) ++ & ~TU_SIZE_MASK; ++ m_n->gmch_n = I915_READ(PIPE_DATA_N1(transcoder)); ++ m_n->tu = ((I915_READ(PIPE_DATA_M1(transcoder)) ++ & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1; ++ ++ if (m2_n2 && transcoder_has_m2_n2(dev_priv, transcoder)) { ++ m2_n2->link_m = I915_READ(PIPE_LINK_M2(transcoder)); ++ m2_n2->link_n = I915_READ(PIPE_LINK_N2(transcoder)); ++ m2_n2->gmch_m = I915_READ(PIPE_DATA_M2(transcoder)) ++ & ~TU_SIZE_MASK; ++ m2_n2->gmch_n = I915_READ(PIPE_DATA_N2(transcoder)); ++ m2_n2->tu = ((I915_READ(PIPE_DATA_M2(transcoder)) ++ & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1; ++ } ++ } else { ++ m_n->link_m = I915_READ(PIPE_LINK_M_G4X(pipe)); ++ m_n->link_n = I915_READ(PIPE_LINK_N_G4X(pipe)); ++ m_n->gmch_m = I915_READ(PIPE_DATA_M_G4X(pipe)) ++ & ~TU_SIZE_MASK; ++ m_n->gmch_n = I915_READ(PIPE_DATA_N_G4X(pipe)); ++ m_n->tu = ((I915_READ(PIPE_DATA_M_G4X(pipe)) ++ & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1; ++ } ++} ++ ++void intel_dp_get_m_n(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config) ++{ ++ if (pipe_config->has_pch_encoder) ++ intel_pch_transcoder_get_m_n(crtc, &pipe_config->dp_m_n); ++ else ++ intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder, ++ &pipe_config->dp_m_n, ++ &pipe_config->dp_m2_n2); ++} ++ ++static void ironlake_get_fdi_m_n_config(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config) ++{ ++ intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder, ++ &pipe_config->fdi_m_n, NULL); ++} ++ ++static void skylake_get_pfit_config(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_crtc_scaler_state *scaler_state = &pipe_config->scaler_state; ++ u32 ps_ctrl = 0; ++ int id = -1; ++ int i; ++ ++ /* find scaler attached to this pipe */ ++ for (i = 0; i < crtc->num_scalers; i++) { ++ ps_ctrl = I915_READ(SKL_PS_CTRL(crtc->pipe, i)); ++ if (ps_ctrl & PS_SCALER_EN && !(ps_ctrl & PS_PLANE_SEL_MASK)) { ++ id = i; ++ pipe_config->pch_pfit.enabled = true; ++ pipe_config->pch_pfit.pos = I915_READ(SKL_PS_WIN_POS(crtc->pipe, i)); ++ pipe_config->pch_pfit.size = I915_READ(SKL_PS_WIN_SZ(crtc->pipe, i)); ++ scaler_state->scalers[i].in_use = true; ++ break; ++ } ++ } ++ ++ scaler_state->scaler_id = id; ++ if (id >= 0) { ++ scaler_state->scaler_users |= (1 << SKL_CRTC_INDEX); ++ } else { ++ scaler_state->scaler_users &= ~(1 << SKL_CRTC_INDEX); ++ } ++} ++ ++static void ++skylake_get_initial_plane_config(struct intel_crtc *crtc, ++ struct intel_initial_plane_config *plane_config) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_plane *plane = to_intel_plane(crtc->base.primary); ++ enum plane_id plane_id = plane->id; ++ enum pipe pipe; ++ u32 val, base, offset, stride_mult, tiling, alpha; ++ int fourcc, pixel_format; ++ unsigned int aligned_height; ++ struct drm_framebuffer *fb; ++ struct intel_framebuffer *intel_fb; ++ ++ if (!plane->get_hw_state(plane, &pipe)) ++ return; ++ ++ WARN_ON(pipe != crtc->pipe); ++ ++ intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL); ++ if (!intel_fb) { ++ DRM_DEBUG_KMS("failed to alloc fb\n"); ++ return; ++ } ++ ++ fb = &intel_fb->base; ++ ++ fb->dev = dev; ++ ++ val = I915_READ(PLANE_CTL(pipe, plane_id)); ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ pixel_format = val & ICL_PLANE_CTL_FORMAT_MASK; ++ else ++ pixel_format = val & PLANE_CTL_FORMAT_MASK; ++ ++ if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) { ++ alpha = I915_READ(PLANE_COLOR_CTL(pipe, plane_id)); ++ alpha &= PLANE_COLOR_ALPHA_MASK; ++ } else { ++ alpha = val & PLANE_CTL_ALPHA_MASK; ++ } ++ ++ fourcc = skl_format_to_fourcc(pixel_format, ++ val & PLANE_CTL_ORDER_RGBX, alpha); ++ fb->format = drm_format_info(fourcc); ++ ++ tiling = val & PLANE_CTL_TILED_MASK; ++ switch (tiling) { ++ case PLANE_CTL_TILED_LINEAR: ++ fb->modifier = DRM_FORMAT_MOD_LINEAR; ++ break; ++ case PLANE_CTL_TILED_X: ++ plane_config->tiling = I915_TILING_X; ++ fb->modifier = I915_FORMAT_MOD_X_TILED; ++ break; ++ case PLANE_CTL_TILED_Y: ++ plane_config->tiling = I915_TILING_Y; ++ if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE) ++ fb->modifier = I915_FORMAT_MOD_Y_TILED_CCS; ++ else ++ fb->modifier = I915_FORMAT_MOD_Y_TILED; ++ break; ++ case PLANE_CTL_TILED_YF: ++ if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE) ++ fb->modifier = I915_FORMAT_MOD_Yf_TILED_CCS; ++ else ++ fb->modifier = I915_FORMAT_MOD_Yf_TILED; ++ break; ++ default: ++ MISSING_CASE(tiling); ++ goto error; ++ } ++ ++ /* ++ * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr ++ * while i915 HW rotation is clockwise, thats why this swapping. ++ */ ++ switch (val & PLANE_CTL_ROTATE_MASK) { ++ case PLANE_CTL_ROTATE_0: ++ plane_config->rotation = DRM_MODE_ROTATE_0; ++ break; ++ case PLANE_CTL_ROTATE_90: ++ plane_config->rotation = DRM_MODE_ROTATE_270; ++ break; ++ case PLANE_CTL_ROTATE_180: ++ plane_config->rotation = DRM_MODE_ROTATE_180; ++ break; ++ case PLANE_CTL_ROTATE_270: ++ plane_config->rotation = DRM_MODE_ROTATE_90; ++ break; ++ } ++ ++ if (INTEL_GEN(dev_priv) >= 10 && ++ val & PLANE_CTL_FLIP_HORIZONTAL) ++ plane_config->rotation |= DRM_MODE_REFLECT_X; ++ ++ base = I915_READ(PLANE_SURF(pipe, plane_id)) & 0xfffff000; ++ plane_config->base = base; ++ ++ offset = I915_READ(PLANE_OFFSET(pipe, plane_id)); ++ ++ val = I915_READ(PLANE_SIZE(pipe, plane_id)); ++ fb->height = ((val >> 16) & 0xfff) + 1; ++ fb->width = ((val >> 0) & 0x1fff) + 1; ++ ++ val = I915_READ(PLANE_STRIDE(pipe, plane_id)); ++ stride_mult = skl_plane_stride_mult(fb, 0, DRM_MODE_ROTATE_0); ++ fb->pitches[0] = (val & 0x3ff) * stride_mult; ++ ++ aligned_height = intel_fb_align_height(fb, 0, fb->height); ++ ++ plane_config->size = fb->pitches[0] * aligned_height; ++ ++ DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n", ++ crtc->base.name, plane->base.name, fb->width, fb->height, ++ fb->format->cpp[0] * 8, base, fb->pitches[0], ++ plane_config->size); ++ ++ plane_config->fb = intel_fb; ++ return; ++ ++error: ++ kfree(intel_fb); ++} ++ ++static void ironlake_get_pfit_config(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ u32 tmp; ++ ++ tmp = I915_READ(PF_CTL(crtc->pipe)); ++ ++ if (tmp & PF_ENABLE) { ++ pipe_config->pch_pfit.enabled = true; ++ pipe_config->pch_pfit.pos = I915_READ(PF_WIN_POS(crtc->pipe)); ++ pipe_config->pch_pfit.size = I915_READ(PF_WIN_SZ(crtc->pipe)); ++ ++ /* We currently do not free assignements of panel fitters on ++ * ivb/hsw (since we don't use the higher upscaling modes which ++ * differentiates them) so just WARN about this case for now. */ ++ if (IS_GEN(dev_priv, 7)) { ++ WARN_ON((tmp & PF_PIPE_SEL_MASK_IVB) != ++ PF_PIPE_SEL_IVB(crtc->pipe)); ++ } ++ } ++} ++ ++static bool ironlake_get_pipe_config(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ enum intel_display_power_domain power_domain; ++ intel_wakeref_t wakeref; ++ u32 tmp; ++ bool ret; ++ ++ power_domain = POWER_DOMAIN_PIPE(crtc->pipe); ++ wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain); ++ if (!wakeref) ++ return false; ++ ++ pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; ++ pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe; ++ pipe_config->shared_dpll = NULL; ++ ++ ret = false; ++ tmp = I915_READ(PIPECONF(crtc->pipe)); ++ if (!(tmp & PIPECONF_ENABLE)) ++ goto out; ++ ++ switch (tmp & PIPECONF_BPC_MASK) { ++ case PIPECONF_6BPC: ++ pipe_config->pipe_bpp = 18; ++ break; ++ case PIPECONF_8BPC: ++ pipe_config->pipe_bpp = 24; ++ break; ++ case PIPECONF_10BPC: ++ pipe_config->pipe_bpp = 30; ++ break; ++ case PIPECONF_12BPC: ++ pipe_config->pipe_bpp = 36; ++ break; ++ default: ++ break; ++ } ++ ++ if (tmp & PIPECONF_COLOR_RANGE_SELECT) ++ pipe_config->limited_color_range = true; ++ ++ pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_ILK) >> ++ PIPECONF_GAMMA_MODE_SHIFT; ++ ++ pipe_config->csc_mode = I915_READ(PIPE_CSC_MODE(crtc->pipe)); ++ ++ i9xx_get_pipe_color_config(pipe_config); ++ ++ if (I915_READ(PCH_TRANSCONF(crtc->pipe)) & TRANS_ENABLE) { ++ struct intel_shared_dpll *pll; ++ enum intel_dpll_id pll_id; ++ ++ pipe_config->has_pch_encoder = true; ++ ++ tmp = I915_READ(FDI_RX_CTL(crtc->pipe)); ++ pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >> ++ FDI_DP_PORT_WIDTH_SHIFT) + 1; ++ ++ ironlake_get_fdi_m_n_config(crtc, pipe_config); ++ ++ if (HAS_PCH_IBX(dev_priv)) { ++ /* ++ * The pipe->pch transcoder and pch transcoder->pll ++ * mapping is fixed. ++ */ ++ pll_id = (enum intel_dpll_id) crtc->pipe; ++ } else { ++ tmp = I915_READ(PCH_DPLL_SEL); ++ if (tmp & TRANS_DPLLB_SEL(crtc->pipe)) ++ pll_id = DPLL_ID_PCH_PLL_B; ++ else ++ pll_id= DPLL_ID_PCH_PLL_A; ++ } ++ ++ pipe_config->shared_dpll = ++ intel_get_shared_dpll_by_id(dev_priv, pll_id); ++ pll = pipe_config->shared_dpll; ++ ++ WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll, ++ &pipe_config->dpll_hw_state)); ++ ++ tmp = pipe_config->dpll_hw_state.dpll; ++ pipe_config->pixel_multiplier = ++ ((tmp & PLL_REF_SDVO_HDMI_MULTIPLIER_MASK) ++ >> PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT) + 1; ++ ++ ironlake_pch_clock_get(crtc, pipe_config); ++ } else { ++ pipe_config->pixel_multiplier = 1; ++ } ++ ++ intel_get_pipe_timings(crtc, pipe_config); ++ intel_get_pipe_src_size(crtc, pipe_config); ++ ++ ironlake_get_pfit_config(crtc, pipe_config); ++ ++ ret = true; ++ ++out: ++ intel_display_power_put(dev_priv, power_domain, wakeref); ++ ++ return ret; ++} ++ ++static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv) ++{ ++ struct drm_device *dev = &dev_priv->drm; ++ struct intel_crtc *crtc; ++ ++ for_each_intel_crtc(dev, crtc) ++ I915_STATE_WARN(crtc->active, "CRTC for pipe %c enabled\n", ++ pipe_name(crtc->pipe)); ++ ++ I915_STATE_WARN(I915_READ(HSW_PWR_WELL_CTL2), ++ "Display power well on\n"); ++ I915_STATE_WARN(I915_READ(SPLL_CTL) & SPLL_PLL_ENABLE, "SPLL enabled\n"); ++ I915_STATE_WARN(I915_READ(WRPLL_CTL(0)) & WRPLL_PLL_ENABLE, "WRPLL1 enabled\n"); ++ I915_STATE_WARN(I915_READ(WRPLL_CTL(1)) & WRPLL_PLL_ENABLE, "WRPLL2 enabled\n"); ++ I915_STATE_WARN(I915_READ(PP_STATUS(0)) & PP_ON, "Panel power on\n"); ++ I915_STATE_WARN(I915_READ(BLC_PWM_CPU_CTL2) & BLM_PWM_ENABLE, ++ "CPU PWM1 enabled\n"); ++ if (IS_HASWELL(dev_priv)) ++ I915_STATE_WARN(I915_READ(HSW_BLC_PWM2_CTL) & BLM_PWM_ENABLE, ++ "CPU PWM2 enabled\n"); ++ I915_STATE_WARN(I915_READ(BLC_PWM_PCH_CTL1) & BLM_PCH_PWM_ENABLE, ++ "PCH PWM1 enabled\n"); ++ I915_STATE_WARN(I915_READ(UTIL_PIN_CTL) & UTIL_PIN_ENABLE, ++ "Utility pin enabled\n"); ++ I915_STATE_WARN(I915_READ(PCH_GTC_CTL) & PCH_GTC_ENABLE, "PCH GTC enabled\n"); ++ ++ /* ++ * In theory we can still leave IRQs enabled, as long as only the HPD ++ * interrupts remain enabled. We used to check for that, but since it's ++ * gen-specific and since we only disable LCPLL after we fully disable ++ * the interrupts, the check below should be enough. ++ */ ++ I915_STATE_WARN(intel_irqs_enabled(dev_priv), "IRQs enabled\n"); ++} ++ ++static u32 hsw_read_dcomp(struct drm_i915_private *dev_priv) ++{ ++ if (IS_HASWELL(dev_priv)) ++ return I915_READ(D_COMP_HSW); ++ else ++ return I915_READ(D_COMP_BDW); ++} ++ ++static void hsw_write_dcomp(struct drm_i915_private *dev_priv, u32 val) ++{ ++ if (IS_HASWELL(dev_priv)) { ++ mutex_lock(&dev_priv->pcu_lock); ++ if (sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_D_COMP, ++ val)) ++ DRM_DEBUG_KMS("Failed to write to D_COMP\n"); ++ mutex_unlock(&dev_priv->pcu_lock); ++ } else { ++ I915_WRITE(D_COMP_BDW, val); ++ POSTING_READ(D_COMP_BDW); ++ } ++} ++ ++/* ++ * This function implements pieces of two sequences from BSpec: ++ * - Sequence for display software to disable LCPLL ++ * - Sequence for display software to allow package C8+ ++ * The steps implemented here are just the steps that actually touch the LCPLL ++ * register. Callers should take care of disabling all the display engine ++ * functions, doing the mode unset, fixing interrupts, etc. ++ */ ++static void hsw_disable_lcpll(struct drm_i915_private *dev_priv, ++ bool switch_to_fclk, bool allow_power_down) ++{ ++ u32 val; ++ ++ assert_can_disable_lcpll(dev_priv); ++ ++ val = I915_READ(LCPLL_CTL); ++ ++ if (switch_to_fclk) { ++ val |= LCPLL_CD_SOURCE_FCLK; ++ I915_WRITE(LCPLL_CTL, val); ++ ++ if (wait_for_us(I915_READ(LCPLL_CTL) & ++ LCPLL_CD_SOURCE_FCLK_DONE, 1)) ++ DRM_ERROR("Switching to FCLK failed\n"); ++ ++ val = I915_READ(LCPLL_CTL); ++ } ++ ++ val |= LCPLL_PLL_DISABLE; ++ I915_WRITE(LCPLL_CTL, val); ++ POSTING_READ(LCPLL_CTL); ++ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ LCPLL_CTL, LCPLL_PLL_LOCK, 0, 1)) ++ DRM_ERROR("LCPLL still locked\n"); ++ ++ val = hsw_read_dcomp(dev_priv); ++ val |= D_COMP_COMP_DISABLE; ++ hsw_write_dcomp(dev_priv, val); ++ ndelay(100); ++ ++ if (wait_for((hsw_read_dcomp(dev_priv) & D_COMP_RCOMP_IN_PROGRESS) == 0, ++ 1)) ++ DRM_ERROR("D_COMP RCOMP still in progress\n"); ++ ++ if (allow_power_down) { ++ val = I915_READ(LCPLL_CTL); ++ val |= LCPLL_POWER_DOWN_ALLOW; ++ I915_WRITE(LCPLL_CTL, val); ++ POSTING_READ(LCPLL_CTL); ++ } ++} ++ ++/* ++ * Fully restores LCPLL, disallowing power down and switching back to LCPLL ++ * source. ++ */ ++static void hsw_restore_lcpll(struct drm_i915_private *dev_priv) ++{ ++ u32 val; ++ ++ val = I915_READ(LCPLL_CTL); ++ ++ if ((val & (LCPLL_PLL_LOCK | LCPLL_PLL_DISABLE | LCPLL_CD_SOURCE_FCLK | ++ LCPLL_POWER_DOWN_ALLOW)) == LCPLL_PLL_LOCK) ++ return; ++ ++ /* ++ * Make sure we're not on PC8 state before disabling PC8, otherwise ++ * we'll hang the machine. To prevent PC8 state, just enable force_wake. ++ */ ++ intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL); ++ ++ if (val & LCPLL_POWER_DOWN_ALLOW) { ++ val &= ~LCPLL_POWER_DOWN_ALLOW; ++ I915_WRITE(LCPLL_CTL, val); ++ POSTING_READ(LCPLL_CTL); ++ } ++ ++ val = hsw_read_dcomp(dev_priv); ++ val |= D_COMP_COMP_FORCE; ++ val &= ~D_COMP_COMP_DISABLE; ++ hsw_write_dcomp(dev_priv, val); ++ ++ val = I915_READ(LCPLL_CTL); ++ val &= ~LCPLL_PLL_DISABLE; ++ I915_WRITE(LCPLL_CTL, val); ++ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ LCPLL_CTL, LCPLL_PLL_LOCK, LCPLL_PLL_LOCK, ++ 5)) ++ DRM_ERROR("LCPLL not locked yet\n"); ++ ++ if (val & LCPLL_CD_SOURCE_FCLK) { ++ val = I915_READ(LCPLL_CTL); ++ val &= ~LCPLL_CD_SOURCE_FCLK; ++ I915_WRITE(LCPLL_CTL, val); ++ ++ if (wait_for_us((I915_READ(LCPLL_CTL) & ++ LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1)) ++ DRM_ERROR("Switching back to LCPLL failed\n"); ++ } ++ ++ intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); ++ ++ intel_update_cdclk(dev_priv); ++ intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK"); ++} ++ ++/* ++ * Package states C8 and deeper are really deep PC states that can only be ++ * reached when all the devices on the system allow it, so even if the graphics ++ * device allows PC8+, it doesn't mean the system will actually get to these ++ * states. Our driver only allows PC8+ when going into runtime PM. ++ * ++ * The requirements for PC8+ are that all the outputs are disabled, the power ++ * well is disabled and most interrupts are disabled, and these are also ++ * requirements for runtime PM. When these conditions are met, we manually do ++ * the other conditions: disable the interrupts, clocks and switch LCPLL refclk ++ * to Fclk. If we're in PC8+ and we get an non-hotplug interrupt, we can hard ++ * hang the machine. ++ * ++ * When we really reach PC8 or deeper states (not just when we allow it) we lose ++ * the state of some registers, so when we come back from PC8+ we need to ++ * restore this state. We don't get into PC8+ if we're not in RC6, so we don't ++ * need to take care of the registers kept by RC6. Notice that this happens even ++ * if we don't put the device in PCI D3 state (which is what currently happens ++ * because of the runtime PM support). ++ * ++ * For more, read "Display Sequences for Package C8" on the hardware ++ * documentation. ++ */ ++void hsw_enable_pc8(struct drm_i915_private *dev_priv) ++{ ++ u32 val; ++ ++ DRM_DEBUG_KMS("Enabling package C8+\n"); ++ ++ if (HAS_PCH_LPT_LP(dev_priv)) { ++ val = I915_READ(SOUTH_DSPCLK_GATE_D); ++ val &= ~PCH_LP_PARTITION_LEVEL_DISABLE; ++ I915_WRITE(SOUTH_DSPCLK_GATE_D, val); ++ } ++ ++ lpt_disable_clkout_dp(dev_priv); ++ hsw_disable_lcpll(dev_priv, true, true); ++} ++ ++void hsw_disable_pc8(struct drm_i915_private *dev_priv) ++{ ++ u32 val; ++ ++ DRM_DEBUG_KMS("Disabling package C8+\n"); ++ ++ hsw_restore_lcpll(dev_priv); ++ lpt_init_pch_refclk(dev_priv); ++ ++ if (HAS_PCH_LPT_LP(dev_priv)) { ++ val = I915_READ(SOUTH_DSPCLK_GATE_D); ++ val |= PCH_LP_PARTITION_LEVEL_DISABLE; ++ I915_WRITE(SOUTH_DSPCLK_GATE_D, val); ++ } ++} ++ ++static int haswell_crtc_compute_clock(struct intel_crtc *crtc, ++ struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ struct intel_atomic_state *state = ++ to_intel_atomic_state(crtc_state->base.state); ++ ++ if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) || ++ INTEL_GEN(dev_priv) >= 11) { ++ struct intel_encoder *encoder = ++ intel_get_crtc_new_encoder(state, crtc_state); ++ ++ if (!intel_get_shared_dpll(crtc_state, encoder)) { ++ DRM_DEBUG_KMS("failed to find PLL for pipe %c\n", ++ pipe_name(crtc->pipe)); ++ return -EINVAL; ++ } ++ } ++ ++ return 0; ++} ++ ++static void cannonlake_get_ddi_pll(struct drm_i915_private *dev_priv, ++ enum port port, ++ struct intel_crtc_state *pipe_config) ++{ ++ enum intel_dpll_id id; ++ u32 temp; ++ ++ temp = I915_READ(DPCLKA_CFGCR0) & DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port); ++ id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port); ++ ++ if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL2)) ++ return; ++ ++ pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id); ++} ++ ++static void icelake_get_ddi_pll(struct drm_i915_private *dev_priv, ++ enum port port, ++ struct intel_crtc_state *pipe_config) ++{ ++ enum intel_dpll_id id; ++ u32 temp; ++ ++ /* TODO: TBT pll not implemented. */ ++ if (intel_port_is_combophy(dev_priv, port)) { ++ temp = I915_READ(DPCLKA_CFGCR0_ICL) & ++ DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port); ++ id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port); ++ } else if (intel_port_is_tc(dev_priv, port)) { ++ id = icl_tc_port_to_pll_id(intel_port_to_tc(dev_priv, port)); ++ } else { ++ WARN(1, "Invalid port %x\n", port); ++ return; ++ } ++ ++ pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id); ++} ++ ++static void bxt_get_ddi_pll(struct drm_i915_private *dev_priv, ++ enum port port, ++ struct intel_crtc_state *pipe_config) ++{ ++ enum intel_dpll_id id; ++ ++ switch (port) { ++ case PORT_A: ++ id = DPLL_ID_SKL_DPLL0; ++ break; ++ case PORT_B: ++ id = DPLL_ID_SKL_DPLL1; ++ break; ++ case PORT_C: ++ id = DPLL_ID_SKL_DPLL2; ++ break; ++ default: ++ DRM_ERROR("Incorrect port type\n"); ++ return; ++ } ++ ++ pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id); ++} ++ ++static void skylake_get_ddi_pll(struct drm_i915_private *dev_priv, ++ enum port port, ++ struct intel_crtc_state *pipe_config) ++{ ++ enum intel_dpll_id id; ++ u32 temp; ++ ++ temp = I915_READ(DPLL_CTRL2) & DPLL_CTRL2_DDI_CLK_SEL_MASK(port); ++ id = temp >> (port * 3 + 1); ++ ++ if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL3)) ++ return; ++ ++ pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id); ++} ++ ++static void haswell_get_ddi_pll(struct drm_i915_private *dev_priv, ++ enum port port, ++ struct intel_crtc_state *pipe_config) ++{ ++ enum intel_dpll_id id; ++ u32 ddi_pll_sel = I915_READ(PORT_CLK_SEL(port)); ++ ++ switch (ddi_pll_sel) { ++ case PORT_CLK_SEL_WRPLL1: ++ id = DPLL_ID_WRPLL1; ++ break; ++ case PORT_CLK_SEL_WRPLL2: ++ id = DPLL_ID_WRPLL2; ++ break; ++ case PORT_CLK_SEL_SPLL: ++ id = DPLL_ID_SPLL; ++ break; ++ case PORT_CLK_SEL_LCPLL_810: ++ id = DPLL_ID_LCPLL_810; ++ break; ++ case PORT_CLK_SEL_LCPLL_1350: ++ id = DPLL_ID_LCPLL_1350; ++ break; ++ case PORT_CLK_SEL_LCPLL_2700: ++ id = DPLL_ID_LCPLL_2700; ++ break; ++ default: ++ MISSING_CASE(ddi_pll_sel); ++ /* fall through */ ++ case PORT_CLK_SEL_NONE: ++ return; ++ } ++ ++ pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id); ++} ++ ++static bool hsw_get_transcoder_state(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config, ++ u64 *power_domain_mask, ++ intel_wakeref_t *wakerefs) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ enum intel_display_power_domain power_domain; ++ unsigned long panel_transcoder_mask = 0; ++ unsigned long enabled_panel_transcoders = 0; ++ enum transcoder panel_transcoder; ++ intel_wakeref_t wf; ++ u32 tmp; ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ panel_transcoder_mask |= ++ BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1); ++ ++ if (HAS_TRANSCODER_EDP(dev_priv)) ++ panel_transcoder_mask |= BIT(TRANSCODER_EDP); ++ ++ /* ++ * The pipe->transcoder mapping is fixed with the exception of the eDP ++ * and DSI transcoders handled below. ++ */ ++ pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe; ++ ++ /* ++ * XXX: Do intel_display_power_get_if_enabled before reading this (for ++ * consistency and less surprising code; it's in always on power). ++ */ ++ for_each_set_bit(panel_transcoder, ++ &panel_transcoder_mask, ++ ARRAY_SIZE(INTEL_INFO(dev_priv)->trans_offsets)) { ++ enum pipe trans_pipe; ++ ++ tmp = I915_READ(TRANS_DDI_FUNC_CTL(panel_transcoder)); ++ if (!(tmp & TRANS_DDI_FUNC_ENABLE)) ++ continue; ++ ++ /* ++ * Log all enabled ones, only use the first one. ++ * ++ * FIXME: This won't work for two separate DSI displays. ++ */ ++ enabled_panel_transcoders |= BIT(panel_transcoder); ++ if (enabled_panel_transcoders != BIT(panel_transcoder)) ++ continue; ++ ++ switch (tmp & TRANS_DDI_EDP_INPUT_MASK) { ++ default: ++ WARN(1, "unknown pipe linked to transcoder %s\n", ++ transcoder_name(panel_transcoder)); ++ /* fall through */ ++ case TRANS_DDI_EDP_INPUT_A_ONOFF: ++ case TRANS_DDI_EDP_INPUT_A_ON: ++ trans_pipe = PIPE_A; ++ break; ++ case TRANS_DDI_EDP_INPUT_B_ONOFF: ++ trans_pipe = PIPE_B; ++ break; ++ case TRANS_DDI_EDP_INPUT_C_ONOFF: ++ trans_pipe = PIPE_C; ++ break; ++ } ++ ++ if (trans_pipe == crtc->pipe) ++ pipe_config->cpu_transcoder = panel_transcoder; ++ } ++ ++ /* ++ * Valid combos: none, eDP, DSI0, DSI1, DSI0+DSI1 ++ */ ++ WARN_ON((enabled_panel_transcoders & BIT(TRANSCODER_EDP)) && ++ enabled_panel_transcoders != BIT(TRANSCODER_EDP)); ++ ++ power_domain = POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder); ++ WARN_ON(*power_domain_mask & BIT_ULL(power_domain)); ++ ++ wf = intel_display_power_get_if_enabled(dev_priv, power_domain); ++ if (!wf) ++ return false; ++ ++ wakerefs[power_domain] = wf; ++ *power_domain_mask |= BIT_ULL(power_domain); ++ ++ tmp = I915_READ(PIPECONF(pipe_config->cpu_transcoder)); ++ ++ return tmp & PIPECONF_ENABLE; ++} ++ ++static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config, ++ u64 *power_domain_mask, ++ intel_wakeref_t *wakerefs) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ enum intel_display_power_domain power_domain; ++ enum transcoder cpu_transcoder; ++ intel_wakeref_t wf; ++ enum port port; ++ u32 tmp; ++ ++ for_each_port_masked(port, BIT(PORT_A) | BIT(PORT_C)) { ++ if (port == PORT_A) ++ cpu_transcoder = TRANSCODER_DSI_A; ++ else ++ cpu_transcoder = TRANSCODER_DSI_C; ++ ++ power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder); ++ WARN_ON(*power_domain_mask & BIT_ULL(power_domain)); ++ ++ wf = intel_display_power_get_if_enabled(dev_priv, power_domain); ++ if (!wf) ++ continue; ++ ++ wakerefs[power_domain] = wf; ++ *power_domain_mask |= BIT_ULL(power_domain); ++ ++ /* ++ * The PLL needs to be enabled with a valid divider ++ * configuration, otherwise accessing DSI registers will hang ++ * the machine. See BSpec North Display Engine ++ * registers/MIPI[BXT]. We can break out here early, since we ++ * need the same DSI PLL to be enabled for both DSI ports. ++ */ ++ if (!bxt_dsi_pll_is_enabled(dev_priv)) ++ break; ++ ++ /* XXX: this works for video mode only */ ++ tmp = I915_READ(BXT_MIPI_PORT_CTRL(port)); ++ if (!(tmp & DPI_ENABLE)) ++ continue; ++ ++ tmp = I915_READ(MIPI_CTRL(port)); ++ if ((tmp & BXT_PIPE_SELECT_MASK) != BXT_PIPE_SELECT(crtc->pipe)) ++ continue; ++ ++ pipe_config->cpu_transcoder = cpu_transcoder; ++ break; ++ } ++ ++ return transcoder_is_dsi(pipe_config->cpu_transcoder); ++} ++ ++static void haswell_get_ddi_port_state(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ struct intel_shared_dpll *pll; ++ enum port port; ++ u32 tmp; ++ ++ tmp = I915_READ(TRANS_DDI_FUNC_CTL(pipe_config->cpu_transcoder)); ++ ++ port = (tmp & TRANS_DDI_PORT_MASK) >> TRANS_DDI_PORT_SHIFT; ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ icelake_get_ddi_pll(dev_priv, port, pipe_config); ++ else if (IS_CANNONLAKE(dev_priv)) ++ cannonlake_get_ddi_pll(dev_priv, port, pipe_config); ++ else if (IS_GEN9_BC(dev_priv)) ++ skylake_get_ddi_pll(dev_priv, port, pipe_config); ++ else if (IS_GEN9_LP(dev_priv)) ++ bxt_get_ddi_pll(dev_priv, port, pipe_config); ++ else ++ haswell_get_ddi_pll(dev_priv, port, pipe_config); ++ ++ pll = pipe_config->shared_dpll; ++ if (pll) { ++ WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll, ++ &pipe_config->dpll_hw_state)); ++ } ++ ++ /* ++ * Haswell has only FDI/PCH transcoder A. It is which is connected to ++ * DDI E. So just check whether this pipe is wired to DDI E and whether ++ * the PCH transcoder is on. ++ */ ++ if (INTEL_GEN(dev_priv) < 9 && ++ (port == PORT_E) && I915_READ(LPT_TRANSCONF) & TRANS_ENABLE) { ++ pipe_config->has_pch_encoder = true; ++ ++ tmp = I915_READ(FDI_RX_CTL(PIPE_A)); ++ pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >> ++ FDI_DP_PORT_WIDTH_SHIFT) + 1; ++ ++ ironlake_get_fdi_m_n_config(crtc, pipe_config); ++ } ++} ++ ++static bool haswell_get_pipe_config(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ intel_wakeref_t wakerefs[POWER_DOMAIN_NUM], wf; ++ enum intel_display_power_domain power_domain; ++ u64 power_domain_mask; ++ bool active; ++ ++ intel_crtc_init_scalers(crtc, pipe_config); ++ ++ power_domain = POWER_DOMAIN_PIPE(crtc->pipe); ++ wf = intel_display_power_get_if_enabled(dev_priv, power_domain); ++ if (!wf) ++ return false; ++ ++ wakerefs[power_domain] = wf; ++ power_domain_mask = BIT_ULL(power_domain); ++ ++ pipe_config->shared_dpll = NULL; ++ ++ active = hsw_get_transcoder_state(crtc, pipe_config, ++ &power_domain_mask, wakerefs); ++ ++ if (IS_GEN9_LP(dev_priv) && ++ bxt_get_dsi_transcoder_state(crtc, pipe_config, ++ &power_domain_mask, wakerefs)) { ++ WARN_ON(active); ++ active = true; ++ } ++ ++ if (!active) ++ goto out; ++ ++ if (!transcoder_is_dsi(pipe_config->cpu_transcoder) || ++ INTEL_GEN(dev_priv) >= 11) { ++ haswell_get_ddi_port_state(crtc, pipe_config); ++ intel_get_pipe_timings(crtc, pipe_config); ++ } ++ ++ intel_get_pipe_src_size(crtc, pipe_config); ++ intel_get_crtc_ycbcr_config(crtc, pipe_config); ++ ++ pipe_config->gamma_mode = I915_READ(GAMMA_MODE(crtc->pipe)); ++ ++ pipe_config->csc_mode = I915_READ(PIPE_CSC_MODE(crtc->pipe)); ++ ++ if (INTEL_GEN(dev_priv) >= 9) { ++ u32 tmp = I915_READ(SKL_BOTTOM_COLOR(crtc->pipe)); ++ ++ if (tmp & SKL_BOTTOM_COLOR_GAMMA_ENABLE) ++ pipe_config->gamma_enable = true; ++ ++ if (tmp & SKL_BOTTOM_COLOR_CSC_ENABLE) ++ pipe_config->csc_enable = true; ++ } else { ++ i9xx_get_pipe_color_config(pipe_config); ++ } ++ ++ power_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe); ++ WARN_ON(power_domain_mask & BIT_ULL(power_domain)); ++ ++ wf = intel_display_power_get_if_enabled(dev_priv, power_domain); ++ if (wf) { ++ wakerefs[power_domain] = wf; ++ power_domain_mask |= BIT_ULL(power_domain); ++ ++ if (INTEL_GEN(dev_priv) >= 9) ++ skylake_get_pfit_config(crtc, pipe_config); ++ else ++ ironlake_get_pfit_config(crtc, pipe_config); ++ } ++ ++ if (hsw_crtc_supports_ips(crtc)) { ++ if (IS_HASWELL(dev_priv)) ++ pipe_config->ips_enabled = I915_READ(IPS_CTL) & IPS_ENABLE; ++ else { ++ /* ++ * We cannot readout IPS state on broadwell, set to ++ * true so we can set it to a defined state on first ++ * commit. ++ */ ++ pipe_config->ips_enabled = true; ++ } ++ } ++ ++ if (pipe_config->cpu_transcoder != TRANSCODER_EDP && ++ !transcoder_is_dsi(pipe_config->cpu_transcoder)) { ++ pipe_config->pixel_multiplier = ++ I915_READ(PIPE_MULT(pipe_config->cpu_transcoder)) + 1; ++ } else { ++ pipe_config->pixel_multiplier = 1; ++ } ++ ++out: ++ for_each_power_domain(power_domain, power_domain_mask) ++ intel_display_power_put(dev_priv, ++ power_domain, wakerefs[power_domain]); ++ ++ return active; ++} ++ ++static u32 intel_cursor_base(const struct intel_plane_state *plane_state) ++{ ++ struct drm_i915_private *dev_priv = ++ to_i915(plane_state->base.plane->dev); ++ const struct drm_framebuffer *fb = plane_state->base.fb; ++ const struct drm_i915_gem_object *obj = intel_fb_obj(fb); ++ u32 base; ++ ++ if (INTEL_INFO(dev_priv)->display.cursor_needs_physical) ++ base = obj->phys_handle->busaddr; ++ else ++ base = intel_plane_ggtt_offset(plane_state); ++ ++ base += plane_state->color_plane[0].offset; ++ ++ /* ILK+ do this automagically */ ++ if (HAS_GMCH(dev_priv) && ++ plane_state->base.rotation & DRM_MODE_ROTATE_180) ++ base += (plane_state->base.crtc_h * ++ plane_state->base.crtc_w - 1) * fb->format->cpp[0]; ++ ++ return base; ++} ++ ++static u32 intel_cursor_position(const struct intel_plane_state *plane_state) ++{ ++ int x = plane_state->base.crtc_x; ++ int y = plane_state->base.crtc_y; ++ u32 pos = 0; ++ ++ if (x < 0) { ++ pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT; ++ x = -x; ++ } ++ pos |= x << CURSOR_X_SHIFT; ++ ++ if (y < 0) { ++ pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT; ++ y = -y; ++ } ++ pos |= y << CURSOR_Y_SHIFT; ++ ++ return pos; ++} ++ ++static bool intel_cursor_size_ok(const struct intel_plane_state *plane_state) ++{ ++ const struct drm_mode_config *config = ++ &plane_state->base.plane->dev->mode_config; ++ int width = plane_state->base.crtc_w; ++ int height = plane_state->base.crtc_h; ++ ++ return width > 0 && width <= config->cursor_width && ++ height > 0 && height <= config->cursor_height; ++} ++ ++static int intel_cursor_check_surface(struct intel_plane_state *plane_state) ++{ ++ const struct drm_framebuffer *fb = plane_state->base.fb; ++ unsigned int rotation = plane_state->base.rotation; ++ int src_x, src_y; ++ u32 offset; ++ int ret; ++ ++ intel_fill_fb_ggtt_view(&plane_state->view, fb, rotation); ++ plane_state->color_plane[0].stride = intel_fb_pitch(fb, 0, rotation); ++ ++ ret = intel_plane_check_stride(plane_state); ++ if (ret) ++ return ret; ++ ++ src_x = plane_state->base.src_x >> 16; ++ src_y = plane_state->base.src_y >> 16; ++ ++ intel_add_fb_offsets(&src_x, &src_y, plane_state, 0); ++ offset = intel_plane_compute_aligned_offset(&src_x, &src_y, ++ plane_state, 0); ++ ++ if (src_x != 0 || src_y != 0) { ++ DRM_DEBUG_KMS("Arbitrary cursor panning not supported\n"); ++ return -EINVAL; ++ } ++ ++ plane_state->color_plane[0].offset = offset; ++ ++ return 0; ++} ++ ++static int intel_check_cursor(struct intel_crtc_state *crtc_state, ++ struct intel_plane_state *plane_state) ++{ ++ const struct drm_framebuffer *fb = plane_state->base.fb; ++ int ret; ++ ++ if (fb && fb->modifier != DRM_FORMAT_MOD_LINEAR) { ++ DRM_DEBUG_KMS("cursor cannot be tiled\n"); ++ return -EINVAL; ++ } ++ ++ ret = drm_atomic_helper_check_plane_state(&plane_state->base, ++ &crtc_state->base, ++ DRM_PLANE_HELPER_NO_SCALING, ++ DRM_PLANE_HELPER_NO_SCALING, ++ true, true); ++ if (ret) ++ return ret; ++ ++ if (!plane_state->base.visible) ++ return 0; ++ ++ ret = intel_plane_check_src_coordinates(plane_state); ++ if (ret) ++ return ret; ++ ++ ret = intel_cursor_check_surface(plane_state); ++ if (ret) ++ return ret; ++ ++ return 0; ++} ++ ++static unsigned int ++i845_cursor_max_stride(struct intel_plane *plane, ++ u32 pixel_format, u64 modifier, ++ unsigned int rotation) ++{ ++ return 2048; ++} ++ ++static u32 i845_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state) ++{ ++ u32 cntl = 0; ++ ++ if (crtc_state->gamma_enable) ++ cntl |= CURSOR_GAMMA_ENABLE; ++ ++ return cntl; ++} ++ ++static u32 i845_cursor_ctl(const struct intel_crtc_state *crtc_state, ++ const struct intel_plane_state *plane_state) ++{ ++ return CURSOR_ENABLE | ++ CURSOR_FORMAT_ARGB | ++ CURSOR_STRIDE(plane_state->color_plane[0].stride); ++} ++ ++static bool i845_cursor_size_ok(const struct intel_plane_state *plane_state) ++{ ++ int width = plane_state->base.crtc_w; ++ ++ /* ++ * 845g/865g are only limited by the width of their cursors, ++ * the height is arbitrary up to the precision of the register. ++ */ ++ return intel_cursor_size_ok(plane_state) && IS_ALIGNED(width, 64); ++} ++ ++static int i845_check_cursor(struct intel_crtc_state *crtc_state, ++ struct intel_plane_state *plane_state) ++{ ++ const struct drm_framebuffer *fb = plane_state->base.fb; ++ int ret; ++ ++ ret = intel_check_cursor(crtc_state, plane_state); ++ if (ret) ++ return ret; ++ ++ /* if we want to turn off the cursor ignore width and height */ ++ if (!fb) ++ return 0; ++ ++ /* Check for which cursor types we support */ ++ if (!i845_cursor_size_ok(plane_state)) { ++ DRM_DEBUG("Cursor dimension %dx%d not supported\n", ++ plane_state->base.crtc_w, ++ plane_state->base.crtc_h); ++ return -EINVAL; ++ } ++ ++ WARN_ON(plane_state->base.visible && ++ plane_state->color_plane[0].stride != fb->pitches[0]); ++ ++ switch (fb->pitches[0]) { ++ case 256: ++ case 512: ++ case 1024: ++ case 2048: ++ break; ++ default: ++ DRM_DEBUG_KMS("Invalid cursor stride (%u)\n", ++ fb->pitches[0]); ++ return -EINVAL; ++ } ++ ++ plane_state->ctl = i845_cursor_ctl(crtc_state, plane_state); ++ ++ return 0; ++} ++ ++static void i845_update_cursor(struct intel_plane *plane, ++ const struct intel_crtc_state *crtc_state, ++ const struct intel_plane_state *plane_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(plane->base.dev); ++ u32 cntl = 0, base = 0, pos = 0, size = 0; ++ unsigned long irqflags; ++ ++ if (plane_state && plane_state->base.visible) { ++ unsigned int width = plane_state->base.crtc_w; ++ unsigned int height = plane_state->base.crtc_h; ++ ++ cntl = plane_state->ctl | ++ i845_cursor_ctl_crtc(crtc_state); ++ ++ size = (height << 12) | width; ++ ++ base = intel_cursor_base(plane_state); ++ pos = intel_cursor_position(plane_state); ++ } ++ ++ spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); ++ ++ /* On these chipsets we can only modify the base/size/stride ++ * whilst the cursor is disabled. ++ */ ++ if (plane->cursor.base != base || ++ plane->cursor.size != size || ++ plane->cursor.cntl != cntl) { ++ I915_WRITE_FW(CURCNTR(PIPE_A), 0); ++ I915_WRITE_FW(CURBASE(PIPE_A), base); ++ I915_WRITE_FW(CURSIZE, size); ++ I915_WRITE_FW(CURPOS(PIPE_A), pos); ++ I915_WRITE_FW(CURCNTR(PIPE_A), cntl); ++ ++ plane->cursor.base = base; ++ plane->cursor.size = size; ++ plane->cursor.cntl = cntl; ++ } else { ++ I915_WRITE_FW(CURPOS(PIPE_A), pos); ++ } ++ ++ spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); ++} ++ ++static void i845_disable_cursor(struct intel_plane *plane, ++ const struct intel_crtc_state *crtc_state) ++{ ++ i845_update_cursor(plane, crtc_state, NULL); ++} ++ ++static bool i845_cursor_get_hw_state(struct intel_plane *plane, ++ enum pipe *pipe) ++{ ++ struct drm_i915_private *dev_priv = to_i915(plane->base.dev); ++ enum intel_display_power_domain power_domain; ++ intel_wakeref_t wakeref; ++ bool ret; ++ ++ power_domain = POWER_DOMAIN_PIPE(PIPE_A); ++ wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain); ++ if (!wakeref) ++ return false; ++ ++ ret = I915_READ(CURCNTR(PIPE_A)) & CURSOR_ENABLE; ++ ++ *pipe = PIPE_A; ++ ++ intel_display_power_put(dev_priv, power_domain, wakeref); ++ ++ return ret; ++} ++ ++static unsigned int ++i9xx_cursor_max_stride(struct intel_plane *plane, ++ u32 pixel_format, u64 modifier, ++ unsigned int rotation) ++{ ++ return plane->base.dev->mode_config.cursor_width * 4; ++} ++ ++static u32 i9xx_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ u32 cntl = 0; ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ return cntl; ++ ++ if (crtc_state->gamma_enable) ++ cntl = MCURSOR_GAMMA_ENABLE; ++ ++ if (crtc_state->csc_enable) ++ cntl |= MCURSOR_PIPE_CSC_ENABLE; ++ ++ if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) ++ cntl |= MCURSOR_PIPE_SELECT(crtc->pipe); ++ ++ return cntl; ++} ++ ++static u32 i9xx_cursor_ctl(const struct intel_crtc_state *crtc_state, ++ const struct intel_plane_state *plane_state) ++{ ++ struct drm_i915_private *dev_priv = ++ to_i915(plane_state->base.plane->dev); ++ u32 cntl = 0; ++ ++ if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv)) ++ cntl |= MCURSOR_TRICKLE_FEED_DISABLE; ++ ++ switch (plane_state->base.crtc_w) { ++ case 64: ++ cntl |= MCURSOR_MODE_64_ARGB_AX; ++ break; ++ case 128: ++ cntl |= MCURSOR_MODE_128_ARGB_AX; ++ break; ++ case 256: ++ cntl |= MCURSOR_MODE_256_ARGB_AX; ++ break; ++ default: ++ MISSING_CASE(plane_state->base.crtc_w); ++ return 0; ++ } ++ ++ if (plane_state->base.rotation & DRM_MODE_ROTATE_180) ++ cntl |= MCURSOR_ROTATE_180; ++ ++ return cntl; ++} ++ ++static bool i9xx_cursor_size_ok(const struct intel_plane_state *plane_state) ++{ ++ struct drm_i915_private *dev_priv = ++ to_i915(plane_state->base.plane->dev); ++ int width = plane_state->base.crtc_w; ++ int height = plane_state->base.crtc_h; ++ ++ if (!intel_cursor_size_ok(plane_state)) ++ return false; ++ ++ /* Cursor width is limited to a few power-of-two sizes */ ++ switch (width) { ++ case 256: ++ case 128: ++ case 64: ++ break; ++ default: ++ return false; ++ } ++ ++ /* ++ * IVB+ have CUR_FBC_CTL which allows an arbitrary cursor ++ * height from 8 lines up to the cursor width, when the ++ * cursor is not rotated. Everything else requires square ++ * cursors. ++ */ ++ if (HAS_CUR_FBC(dev_priv) && ++ plane_state->base.rotation & DRM_MODE_ROTATE_0) { ++ if (height < 8 || height > width) ++ return false; ++ } else { ++ if (height != width) ++ return false; ++ } ++ ++ return true; ++} ++ ++static int i9xx_check_cursor(struct intel_crtc_state *crtc_state, ++ struct intel_plane_state *plane_state) ++{ ++ struct intel_plane *plane = to_intel_plane(plane_state->base.plane); ++ struct drm_i915_private *dev_priv = to_i915(plane->base.dev); ++ const struct drm_framebuffer *fb = plane_state->base.fb; ++ enum pipe pipe = plane->pipe; ++ int ret; ++ ++ ret = intel_check_cursor(crtc_state, plane_state); ++ if (ret) ++ return ret; ++ ++ /* if we want to turn off the cursor ignore width and height */ ++ if (!fb) ++ return 0; ++ ++ /* Check for which cursor types we support */ ++ if (!i9xx_cursor_size_ok(plane_state)) { ++ DRM_DEBUG("Cursor dimension %dx%d not supported\n", ++ plane_state->base.crtc_w, ++ plane_state->base.crtc_h); ++ return -EINVAL; ++ } ++ ++ WARN_ON(plane_state->base.visible && ++ plane_state->color_plane[0].stride != fb->pitches[0]); ++ ++ if (fb->pitches[0] != plane_state->base.crtc_w * fb->format->cpp[0]) { ++ DRM_DEBUG_KMS("Invalid cursor stride (%u) (cursor width %d)\n", ++ fb->pitches[0], plane_state->base.crtc_w); ++ return -EINVAL; ++ } ++ ++ /* ++ * There's something wrong with the cursor on CHV pipe C. ++ * If it straddles the left edge of the screen then ++ * moving it away from the edge or disabling it often ++ * results in a pipe underrun, and often that can lead to ++ * dead pipe (constant underrun reported, and it scans ++ * out just a solid color). To recover from that, the ++ * display power well must be turned off and on again. ++ * Refuse the put the cursor into that compromised position. ++ */ ++ if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_C && ++ plane_state->base.visible && plane_state->base.crtc_x < 0) { ++ DRM_DEBUG_KMS("CHV cursor C not allowed to straddle the left screen edge\n"); ++ return -EINVAL; ++ } ++ ++ plane_state->ctl = i9xx_cursor_ctl(crtc_state, plane_state); ++ ++ return 0; ++} ++ ++static void i9xx_update_cursor(struct intel_plane *plane, ++ const struct intel_crtc_state *crtc_state, ++ const struct intel_plane_state *plane_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(plane->base.dev); ++ enum pipe pipe = plane->pipe; ++ u32 cntl = 0, base = 0, pos = 0, fbc_ctl = 0; ++ unsigned long irqflags; ++ ++ if (plane_state && plane_state->base.visible) { ++ cntl = plane_state->ctl | ++ i9xx_cursor_ctl_crtc(crtc_state); ++ ++ if (plane_state->base.crtc_h != plane_state->base.crtc_w) ++ fbc_ctl = CUR_FBC_CTL_EN | (plane_state->base.crtc_h - 1); ++ ++ base = intel_cursor_base(plane_state); ++ pos = intel_cursor_position(plane_state); ++ } ++ ++ spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); ++ ++ /* ++ * On some platforms writing CURCNTR first will also ++ * cause CURPOS to be armed by the CURBASE write. ++ * Without the CURCNTR write the CURPOS write would ++ * arm itself. Thus we always update CURCNTR before ++ * CURPOS. ++ * ++ * On other platforms CURPOS always requires the ++ * CURBASE write to arm the update. Additonally ++ * a write to any of the cursor register will cancel ++ * an already armed cursor update. Thus leaving out ++ * the CURBASE write after CURPOS could lead to a ++ * cursor that doesn't appear to move, or even change ++ * shape. Thus we always write CURBASE. ++ * ++ * The other registers are armed by by the CURBASE write ++ * except when the plane is getting enabled at which time ++ * the CURCNTR write arms the update. ++ */ ++ ++ if (INTEL_GEN(dev_priv) >= 9) ++ skl_write_cursor_wm(plane, crtc_state); ++ ++ if (plane->cursor.base != base || ++ plane->cursor.size != fbc_ctl || ++ plane->cursor.cntl != cntl) { ++ if (HAS_CUR_FBC(dev_priv)) ++ I915_WRITE_FW(CUR_FBC_CTL(pipe), fbc_ctl); ++ I915_WRITE_FW(CURCNTR(pipe), cntl); ++ I915_WRITE_FW(CURPOS(pipe), pos); ++ I915_WRITE_FW(CURBASE(pipe), base); ++ ++ plane->cursor.base = base; ++ plane->cursor.size = fbc_ctl; ++ plane->cursor.cntl = cntl; ++ } else { ++ I915_WRITE_FW(CURPOS(pipe), pos); ++ I915_WRITE_FW(CURBASE(pipe), base); ++ } ++ ++ spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); ++} ++ ++static void i9xx_disable_cursor(struct intel_plane *plane, ++ const struct intel_crtc_state *crtc_state) ++{ ++ i9xx_update_cursor(plane, crtc_state, NULL); ++} ++ ++static bool i9xx_cursor_get_hw_state(struct intel_plane *plane, ++ enum pipe *pipe) ++{ ++ struct drm_i915_private *dev_priv = to_i915(plane->base.dev); ++ enum intel_display_power_domain power_domain; ++ intel_wakeref_t wakeref; ++ bool ret; ++ u32 val; ++ ++ /* ++ * Not 100% correct for planes that can move between pipes, ++ * but that's only the case for gen2-3 which don't have any ++ * display power wells. ++ */ ++ power_domain = POWER_DOMAIN_PIPE(plane->pipe); ++ wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain); ++ if (!wakeref) ++ return false; ++ ++ val = I915_READ(CURCNTR(plane->pipe)); ++ ++ ret = val & MCURSOR_MODE; ++ ++ if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv)) ++ *pipe = plane->pipe; ++ else ++ *pipe = (val & MCURSOR_PIPE_SELECT_MASK) >> ++ MCURSOR_PIPE_SELECT_SHIFT; ++ ++ intel_display_power_put(dev_priv, power_domain, wakeref); ++ ++ return ret; ++} ++ ++/* VESA 640x480x72Hz mode to set on the pipe */ ++static const struct drm_display_mode load_detect_mode = { ++ DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664, ++ 704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC), ++}; ++ ++struct drm_framebuffer * ++intel_framebuffer_create(struct drm_i915_gem_object *obj, ++ struct drm_mode_fb_cmd2 *mode_cmd) ++{ ++ struct intel_framebuffer *intel_fb; ++ int ret; ++ ++ intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL); ++ if (!intel_fb) ++ return ERR_PTR(-ENOMEM); ++ ++ ret = intel_framebuffer_init(intel_fb, obj, mode_cmd); ++ if (ret) ++ goto err; ++ ++ return &intel_fb->base; ++ ++err: ++ kfree(intel_fb); ++ return ERR_PTR(ret); ++} ++ ++static int intel_modeset_disable_planes(struct drm_atomic_state *state, ++ struct drm_crtc *crtc) ++{ ++ struct drm_plane *plane; ++ struct drm_plane_state *plane_state; ++ int ret, i; ++ ++ ret = drm_atomic_add_affected_planes(state, crtc); ++ if (ret) ++ return ret; ++ ++ for_each_new_plane_in_state(state, plane, plane_state, i) { ++ if (plane_state->crtc != crtc) ++ continue; ++ ++ ret = drm_atomic_set_crtc_for_plane(plane_state, NULL); ++ if (ret) ++ return ret; ++ ++ drm_atomic_set_fb_for_plane(plane_state, NULL); ++ } ++ ++ return 0; ++} ++ ++int intel_get_load_detect_pipe(struct drm_connector *connector, ++ const struct drm_display_mode *mode, ++ struct intel_load_detect_pipe *old, ++ struct drm_modeset_acquire_ctx *ctx) ++{ ++ struct intel_crtc *intel_crtc; ++ struct intel_encoder *intel_encoder = ++ intel_attached_encoder(connector); ++ struct drm_crtc *possible_crtc; ++ struct drm_encoder *encoder = &intel_encoder->base; ++ struct drm_crtc *crtc = NULL; ++ struct drm_device *dev = encoder->dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct drm_mode_config *config = &dev->mode_config; ++ struct drm_atomic_state *state = NULL, *restore_state = NULL; ++ struct drm_connector_state *connector_state; ++ struct intel_crtc_state *crtc_state; ++ int ret, i = -1; ++ ++ DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n", ++ connector->base.id, connector->name, ++ encoder->base.id, encoder->name); ++ ++ old->restore_state = NULL; ++ ++ WARN_ON(!drm_modeset_is_locked(&config->connection_mutex)); ++ ++ /* ++ * Algorithm gets a little messy: ++ * ++ * - if the connector already has an assigned crtc, use it (but make ++ * sure it's on first) ++ * ++ * - try to find the first unused crtc that can drive this connector, ++ * and use that if we find one ++ */ ++ ++ /* See if we already have a CRTC for this connector */ ++ if (connector->state->crtc) { ++ crtc = connector->state->crtc; ++ ++ ret = drm_modeset_lock(&crtc->mutex, ctx); ++ if (ret) ++ goto fail; ++ ++ /* Make sure the crtc and connector are running */ ++ goto found; ++ } ++ ++ /* Find an unused one (if possible) */ ++ for_each_crtc(dev, possible_crtc) { ++ i++; ++ if (!(encoder->possible_crtcs & (1 << i))) ++ continue; ++ ++ ret = drm_modeset_lock(&possible_crtc->mutex, ctx); ++ if (ret) ++ goto fail; ++ ++ if (possible_crtc->state->enable) { ++ drm_modeset_unlock(&possible_crtc->mutex); ++ continue; ++ } ++ ++ crtc = possible_crtc; ++ break; ++ } ++ ++ /* ++ * If we didn't find an unused CRTC, don't use any. ++ */ ++ if (!crtc) { ++ DRM_DEBUG_KMS("no pipe available for load-detect\n"); ++ ret = -ENODEV; ++ goto fail; ++ } ++ ++found: ++ intel_crtc = to_intel_crtc(crtc); ++ ++ state = drm_atomic_state_alloc(dev); ++ restore_state = drm_atomic_state_alloc(dev); ++ if (!state || !restore_state) { ++ ret = -ENOMEM; ++ goto fail; ++ } ++ ++ state->acquire_ctx = ctx; ++ restore_state->acquire_ctx = ctx; ++ ++ connector_state = drm_atomic_get_connector_state(state, connector); ++ if (IS_ERR(connector_state)) { ++ ret = PTR_ERR(connector_state); ++ goto fail; ++ } ++ ++ ret = drm_atomic_set_crtc_for_connector(connector_state, crtc); ++ if (ret) ++ goto fail; ++ ++ crtc_state = intel_atomic_get_crtc_state(state, intel_crtc); ++ if (IS_ERR(crtc_state)) { ++ ret = PTR_ERR(crtc_state); ++ goto fail; ++ } ++ ++ crtc_state->base.active = crtc_state->base.enable = true; ++ ++ if (!mode) ++ mode = &load_detect_mode; ++ ++ ret = drm_atomic_set_mode_for_crtc(&crtc_state->base, mode); ++ if (ret) ++ goto fail; ++ ++ ret = intel_modeset_disable_planes(state, crtc); ++ if (ret) ++ goto fail; ++ ++ ret = PTR_ERR_OR_ZERO(drm_atomic_get_connector_state(restore_state, connector)); ++ if (!ret) ++ ret = PTR_ERR_OR_ZERO(drm_atomic_get_crtc_state(restore_state, crtc)); ++ if (!ret) ++ ret = drm_atomic_add_affected_planes(restore_state, crtc); ++ if (ret) { ++ DRM_DEBUG_KMS("Failed to create a copy of old state to restore: %i\n", ret); ++ goto fail; ++ } ++ ++ ret = drm_atomic_commit(state); ++ if (ret) { ++ DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n"); ++ goto fail; ++ } ++ ++ old->restore_state = restore_state; ++ drm_atomic_state_put(state); ++ ++ /* let the connector get through one full cycle before testing */ ++ intel_wait_for_vblank(dev_priv, intel_crtc->pipe); ++ return true; ++ ++fail: ++ if (state) { ++ drm_atomic_state_put(state); ++ state = NULL; ++ } ++ if (restore_state) { ++ drm_atomic_state_put(restore_state); ++ restore_state = NULL; ++ } ++ ++ if (ret == -EDEADLK) ++ return ret; ++ ++ return false; ++} ++ ++void intel_release_load_detect_pipe(struct drm_connector *connector, ++ struct intel_load_detect_pipe *old, ++ struct drm_modeset_acquire_ctx *ctx) ++{ ++ struct intel_encoder *intel_encoder = ++ intel_attached_encoder(connector); ++ struct drm_encoder *encoder = &intel_encoder->base; ++ struct drm_atomic_state *state = old->restore_state; ++ int ret; ++ ++ DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n", ++ connector->base.id, connector->name, ++ encoder->base.id, encoder->name); ++ ++ if (!state) ++ return; ++ ++ ret = drm_atomic_helper_commit_duplicated_state(state, ctx); ++ if (ret) ++ DRM_DEBUG_KMS("Couldn't release load detect pipe: %i\n", ret); ++ drm_atomic_state_put(state); ++} ++ ++static int i9xx_pll_refclk(struct drm_device *dev, ++ const struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ u32 dpll = pipe_config->dpll_hw_state.dpll; ++ ++ if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN) ++ return dev_priv->vbt.lvds_ssc_freq; ++ else if (HAS_PCH_SPLIT(dev_priv)) ++ return 120000; ++ else if (!IS_GEN(dev_priv, 2)) ++ return 96000; ++ else ++ return 48000; ++} ++ ++/* Returns the clock of the currently programmed mode of the given pipe. */ ++static void i9xx_crtc_clock_get(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ int pipe = pipe_config->cpu_transcoder; ++ u32 dpll = pipe_config->dpll_hw_state.dpll; ++ u32 fp; ++ struct dpll clock; ++ int port_clock; ++ int refclk = i9xx_pll_refclk(dev, pipe_config); ++ ++ if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0) ++ fp = pipe_config->dpll_hw_state.fp0; ++ else ++ fp = pipe_config->dpll_hw_state.fp1; ++ ++ clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT; ++ if (IS_PINEVIEW(dev_priv)) { ++ clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1; ++ clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT; ++ } else { ++ clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT; ++ clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT; ++ } ++ ++ if (!IS_GEN(dev_priv, 2)) { ++ if (IS_PINEVIEW(dev_priv)) ++ clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >> ++ DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW); ++ else ++ clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >> ++ DPLL_FPA01_P1_POST_DIV_SHIFT); ++ ++ switch (dpll & DPLL_MODE_MASK) { ++ case DPLLB_MODE_DAC_SERIAL: ++ clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ? ++ 5 : 10; ++ break; ++ case DPLLB_MODE_LVDS: ++ clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ? ++ 7 : 14; ++ break; ++ default: ++ DRM_DEBUG_KMS("Unknown DPLL mode %08x in programmed " ++ "mode\n", (int)(dpll & DPLL_MODE_MASK)); ++ return; ++ } ++ ++ if (IS_PINEVIEW(dev_priv)) ++ port_clock = pnv_calc_dpll_params(refclk, &clock); ++ else ++ port_clock = i9xx_calc_dpll_params(refclk, &clock); ++ } else { ++ u32 lvds = IS_I830(dev_priv) ? 0 : I915_READ(LVDS); ++ bool is_lvds = (pipe == 1) && (lvds & LVDS_PORT_EN); ++ ++ if (is_lvds) { ++ clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >> ++ DPLL_FPA01_P1_POST_DIV_SHIFT); ++ ++ if (lvds & LVDS_CLKB_POWER_UP) ++ clock.p2 = 7; ++ else ++ clock.p2 = 14; ++ } else { ++ if (dpll & PLL_P1_DIVIDE_BY_TWO) ++ clock.p1 = 2; ++ else { ++ clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >> ++ DPLL_FPA01_P1_POST_DIV_SHIFT) + 2; ++ } ++ if (dpll & PLL_P2_DIVIDE_BY_4) ++ clock.p2 = 4; ++ else ++ clock.p2 = 2; ++ } ++ ++ port_clock = i9xx_calc_dpll_params(refclk, &clock); ++ } ++ ++ /* ++ * This value includes pixel_multiplier. We will use ++ * port_clock to compute adjusted_mode.crtc_clock in the ++ * encoder's get_config() function. ++ */ ++ pipe_config->port_clock = port_clock; ++} ++ ++int intel_dotclock_calculate(int link_freq, ++ const struct intel_link_m_n *m_n) ++{ ++ /* ++ * The calculation for the data clock is: ++ * pixel_clock = ((m/n)*(link_clock * nr_lanes))/bpp ++ * But we want to avoid losing precison if possible, so: ++ * pixel_clock = ((m * link_clock * nr_lanes)/(n*bpp)) ++ * ++ * and the link clock is simpler: ++ * link_clock = (m * link_clock) / n ++ */ ++ ++ if (!m_n->link_n) ++ return 0; ++ ++ return div_u64(mul_u32_u32(m_n->link_m, link_freq), m_n->link_n); ++} ++ ++static void ironlake_pch_clock_get(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ ++ /* read out port_clock from the DPLL */ ++ i9xx_crtc_clock_get(crtc, pipe_config); ++ ++ /* ++ * In case there is an active pipe without active ports, ++ * we may need some idea for the dotclock anyway. ++ * Calculate one based on the FDI configuration. ++ */ ++ pipe_config->base.adjusted_mode.crtc_clock = ++ intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config), ++ &pipe_config->fdi_m_n); ++} ++ ++/* Returns the currently programmed mode of the given encoder. */ ++struct drm_display_mode * ++intel_encoder_current_mode(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_crtc_state *crtc_state; ++ struct drm_display_mode *mode; ++ struct intel_crtc *crtc; ++ enum pipe pipe; ++ ++ if (!encoder->get_hw_state(encoder, &pipe)) ++ return NULL; ++ ++ crtc = intel_get_crtc_for_pipe(dev_priv, pipe); ++ ++ mode = kzalloc(sizeof(*mode), GFP_KERNEL); ++ if (!mode) ++ return NULL; ++ ++ crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL); ++ if (!crtc_state) { ++ kfree(mode); ++ return NULL; ++ } ++ ++ crtc_state->base.crtc = &crtc->base; ++ ++ if (!dev_priv->display.get_pipe_config(crtc, crtc_state)) { ++ kfree(crtc_state); ++ kfree(mode); ++ return NULL; ++ } ++ ++ encoder->get_config(encoder, crtc_state); ++ ++ intel_mode_from_pipe_config(mode, crtc_state); ++ ++ kfree(crtc_state); ++ ++ return mode; ++} ++ ++static void intel_crtc_destroy(struct drm_crtc *crtc) ++{ ++ struct intel_crtc *intel_crtc = to_intel_crtc(crtc); ++ ++ drm_crtc_cleanup(crtc); ++ kfree(intel_crtc); ++} ++ ++/** ++ * intel_wm_need_update - Check whether watermarks need updating ++ * @cur: current plane state ++ * @new: new plane state ++ * ++ * Check current plane state versus the new one to determine whether ++ * watermarks need to be recalculated. ++ * ++ * Returns true or false. ++ */ ++static bool intel_wm_need_update(struct intel_plane_state *cur, ++ struct intel_plane_state *new) ++{ ++ /* Update watermarks on tiling or size changes. */ ++ if (new->base.visible != cur->base.visible) ++ return true; ++ ++ if (!cur->base.fb || !new->base.fb) ++ return false; ++ ++ if (cur->base.fb->modifier != new->base.fb->modifier || ++ cur->base.rotation != new->base.rotation || ++ drm_rect_width(&new->base.src) != drm_rect_width(&cur->base.src) || ++ drm_rect_height(&new->base.src) != drm_rect_height(&cur->base.src) || ++ drm_rect_width(&new->base.dst) != drm_rect_width(&cur->base.dst) || ++ drm_rect_height(&new->base.dst) != drm_rect_height(&cur->base.dst)) ++ return true; ++ ++ return false; ++} ++ ++static bool needs_scaling(const struct intel_plane_state *state) ++{ ++ int src_w = drm_rect_width(&state->base.src) >> 16; ++ int src_h = drm_rect_height(&state->base.src) >> 16; ++ int dst_w = drm_rect_width(&state->base.dst); ++ int dst_h = drm_rect_height(&state->base.dst); ++ ++ return (src_w != dst_w || src_h != dst_h); ++} ++ ++int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_state, ++ struct drm_crtc_state *crtc_state, ++ const struct intel_plane_state *old_plane_state, ++ struct drm_plane_state *plane_state) ++{ ++ struct intel_crtc_state *pipe_config = to_intel_crtc_state(crtc_state); ++ struct drm_crtc *crtc = crtc_state->crtc; ++ struct intel_crtc *intel_crtc = to_intel_crtc(crtc); ++ struct intel_plane *plane = to_intel_plane(plane_state->plane); ++ struct drm_device *dev = crtc->dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ bool mode_changed = needs_modeset(crtc_state); ++ bool was_crtc_enabled = old_crtc_state->base.active; ++ bool is_crtc_enabled = crtc_state->active; ++ bool turn_off, turn_on, visible, was_visible; ++ struct drm_framebuffer *fb = plane_state->fb; ++ int ret; ++ ++ if (INTEL_GEN(dev_priv) >= 9 && plane->id != PLANE_CURSOR) { ++ ret = skl_update_scaler_plane( ++ to_intel_crtc_state(crtc_state), ++ to_intel_plane_state(plane_state)); ++ if (ret) ++ return ret; ++ } ++ ++ was_visible = old_plane_state->base.visible; ++ visible = plane_state->visible; ++ ++ if (!was_crtc_enabled && WARN_ON(was_visible)) ++ was_visible = false; ++ ++ /* ++ * Visibility is calculated as if the crtc was on, but ++ * after scaler setup everything depends on it being off ++ * when the crtc isn't active. ++ * ++ * FIXME this is wrong for watermarks. Watermarks should also ++ * be computed as if the pipe would be active. Perhaps move ++ * per-plane wm computation to the .check_plane() hook, and ++ * only combine the results from all planes in the current place? ++ */ ++ if (!is_crtc_enabled) { ++ plane_state->visible = visible = false; ++ to_intel_crtc_state(crtc_state)->active_planes &= ~BIT(plane->id); ++ } ++ ++ if (!was_visible && !visible) ++ return 0; ++ ++ if (fb != old_plane_state->base.fb) ++ pipe_config->fb_changed = true; ++ ++ turn_off = was_visible && (!visible || mode_changed); ++ turn_on = visible && (!was_visible || mode_changed); ++ ++ DRM_DEBUG_ATOMIC("[CRTC:%d:%s] has [PLANE:%d:%s] with fb %i\n", ++ intel_crtc->base.base.id, intel_crtc->base.name, ++ plane->base.base.id, plane->base.name, ++ fb ? fb->base.id : -1); ++ ++ DRM_DEBUG_ATOMIC("[PLANE:%d:%s] visible %i -> %i, off %i, on %i, ms %i\n", ++ plane->base.base.id, plane->base.name, ++ was_visible, visible, ++ turn_off, turn_on, mode_changed); ++ ++ if (turn_on) { ++ if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) ++ pipe_config->update_wm_pre = true; ++ ++ /* must disable cxsr around plane enable/disable */ ++ if (plane->id != PLANE_CURSOR) ++ pipe_config->disable_cxsr = true; ++ } else if (turn_off) { ++ if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) ++ pipe_config->update_wm_post = true; ++ ++ /* must disable cxsr around plane enable/disable */ ++ if (plane->id != PLANE_CURSOR) ++ pipe_config->disable_cxsr = true; ++ } else if (intel_wm_need_update(to_intel_plane_state(plane->base.state), ++ to_intel_plane_state(plane_state))) { ++ if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) { ++ /* FIXME bollocks */ ++ pipe_config->update_wm_pre = true; ++ pipe_config->update_wm_post = true; ++ } ++ } ++ ++ if (visible || was_visible) ++ pipe_config->fb_bits |= plane->frontbuffer_bit; ++ ++ /* ++ * ILK/SNB DVSACNTR/Sprite Enable ++ * IVB SPR_CTL/Sprite Enable ++ * "When in Self Refresh Big FIFO mode, a write to enable the ++ * plane will be internally buffered and delayed while Big FIFO ++ * mode is exiting." ++ * ++ * Which means that enabling the sprite can take an extra frame ++ * when we start in big FIFO mode (LP1+). Thus we need to drop ++ * down to LP0 and wait for vblank in order to make sure the ++ * sprite gets enabled on the next vblank after the register write. ++ * Doing otherwise would risk enabling the sprite one frame after ++ * we've already signalled flip completion. We can resume LP1+ ++ * once the sprite has been enabled. ++ * ++ * ++ * WaCxSRDisabledForSpriteScaling:ivb ++ * IVB SPR_SCALE/Scaling Enable ++ * "Low Power watermarks must be disabled for at least one ++ * frame before enabling sprite scaling, and kept disabled ++ * until sprite scaling is disabled." ++ * ++ * ILK/SNB DVSASCALE/Scaling Enable ++ * "When in Self Refresh Big FIFO mode, scaling enable will be ++ * masked off while Big FIFO mode is exiting." ++ * ++ * Despite the w/a only being listed for IVB we assume that ++ * the ILK/SNB note has similar ramifications, hence we apply ++ * the w/a on all three platforms. ++ * ++ * With experimental results seems this is needed also for primary ++ * plane, not only sprite plane. ++ */ ++ if (plane->id != PLANE_CURSOR && ++ (IS_GEN_RANGE(dev_priv, 5, 6) || ++ IS_IVYBRIDGE(dev_priv)) && ++ (turn_on || (!needs_scaling(old_plane_state) && ++ needs_scaling(to_intel_plane_state(plane_state))))) ++ pipe_config->disable_lp_wm = true; ++ ++ return 0; ++} ++ ++static bool encoders_cloneable(const struct intel_encoder *a, ++ const struct intel_encoder *b) ++{ ++ /* masks could be asymmetric, so check both ways */ ++ return a == b || (a->cloneable & (1 << b->type) && ++ b->cloneable & (1 << a->type)); ++} ++ ++static bool check_single_encoder_cloning(struct drm_atomic_state *state, ++ struct intel_crtc *crtc, ++ struct intel_encoder *encoder) ++{ ++ struct intel_encoder *source_encoder; ++ struct drm_connector *connector; ++ struct drm_connector_state *connector_state; ++ int i; ++ ++ for_each_new_connector_in_state(state, connector, connector_state, i) { ++ if (connector_state->crtc != &crtc->base) ++ continue; ++ ++ source_encoder = ++ to_intel_encoder(connector_state->best_encoder); ++ if (!encoders_cloneable(encoder, source_encoder)) ++ return false; ++ } ++ ++ return true; ++} ++ ++static int icl_add_linked_planes(struct intel_atomic_state *state) ++{ ++ struct intel_plane *plane, *linked; ++ struct intel_plane_state *plane_state, *linked_plane_state; ++ int i; ++ ++ for_each_new_intel_plane_in_state(state, plane, plane_state, i) { ++ linked = plane_state->linked_plane; ++ ++ if (!linked) ++ continue; ++ ++ linked_plane_state = intel_atomic_get_plane_state(state, linked); ++ if (IS_ERR(linked_plane_state)) ++ return PTR_ERR(linked_plane_state); ++ ++ WARN_ON(linked_plane_state->linked_plane != plane); ++ WARN_ON(linked_plane_state->slave == plane_state->slave); ++ } ++ ++ return 0; ++} ++ ++static int icl_check_nv12_planes(struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->base.state); ++ struct intel_plane *plane, *linked; ++ struct intel_plane_state *plane_state; ++ int i; ++ ++ if (INTEL_GEN(dev_priv) < 11) ++ return 0; ++ ++ /* ++ * Destroy all old plane links and make the slave plane invisible ++ * in the crtc_state->active_planes mask. ++ */ ++ for_each_new_intel_plane_in_state(state, plane, plane_state, i) { ++ if (plane->pipe != crtc->pipe || !plane_state->linked_plane) ++ continue; ++ ++ plane_state->linked_plane = NULL; ++ if (plane_state->slave && !plane_state->base.visible) { ++ crtc_state->active_planes &= ~BIT(plane->id); ++ crtc_state->update_planes |= BIT(plane->id); ++ } ++ ++ plane_state->slave = false; ++ } ++ ++ if (!crtc_state->nv12_planes) ++ return 0; ++ ++ for_each_new_intel_plane_in_state(state, plane, plane_state, i) { ++ struct intel_plane_state *linked_state = NULL; ++ ++ if (plane->pipe != crtc->pipe || ++ !(crtc_state->nv12_planes & BIT(plane->id))) ++ continue; ++ ++ for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, linked) { ++ if (!icl_is_nv12_y_plane(linked->id)) ++ continue; ++ ++ if (crtc_state->active_planes & BIT(linked->id)) ++ continue; ++ ++ linked_state = intel_atomic_get_plane_state(state, linked); ++ if (IS_ERR(linked_state)) ++ return PTR_ERR(linked_state); ++ ++ break; ++ } ++ ++ if (!linked_state) { ++ DRM_DEBUG_KMS("Need %d free Y planes for planar YUV\n", ++ hweight8(crtc_state->nv12_planes)); ++ ++ return -EINVAL; ++ } ++ ++ plane_state->linked_plane = linked; ++ ++ linked_state->slave = true; ++ linked_state->linked_plane = plane; ++ crtc_state->active_planes |= BIT(linked->id); ++ crtc_state->update_planes |= BIT(linked->id); ++ DRM_DEBUG_KMS("Using %s as Y plane for %s\n", linked->base.name, plane->base.name); ++ } ++ ++ return 0; ++} ++ ++static int intel_crtc_atomic_check(struct drm_crtc *crtc, ++ struct drm_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->dev); ++ struct intel_crtc *intel_crtc = to_intel_crtc(crtc); ++ struct intel_crtc_state *pipe_config = ++ to_intel_crtc_state(crtc_state); ++ int ret; ++ bool mode_changed = needs_modeset(crtc_state); ++ ++ if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv) && ++ mode_changed && !crtc_state->active) ++ pipe_config->update_wm_post = true; ++ ++ if (mode_changed && crtc_state->enable && ++ dev_priv->display.crtc_compute_clock && ++ !WARN_ON(pipe_config->shared_dpll)) { ++ ret = dev_priv->display.crtc_compute_clock(intel_crtc, ++ pipe_config); ++ if (ret) ++ return ret; ++ } ++ ++ if (mode_changed || pipe_config->update_pipe || ++ crtc_state->color_mgmt_changed) { ++ ret = intel_color_check(pipe_config); ++ if (ret) ++ return ret; ++ } ++ ++ ret = 0; ++ if (dev_priv->display.compute_pipe_wm) { ++ ret = dev_priv->display.compute_pipe_wm(pipe_config); ++ if (ret) { ++ DRM_DEBUG_KMS("Target pipe watermarks are invalid\n"); ++ return ret; ++ } ++ } ++ ++ if (dev_priv->display.compute_intermediate_wm) { ++ if (WARN_ON(!dev_priv->display.compute_pipe_wm)) ++ return 0; ++ ++ /* ++ * Calculate 'intermediate' watermarks that satisfy both the ++ * old state and the new state. We can program these ++ * immediately. ++ */ ++ ret = dev_priv->display.compute_intermediate_wm(pipe_config); ++ if (ret) { ++ DRM_DEBUG_KMS("No valid intermediate pipe watermarks are possible\n"); ++ return ret; ++ } ++ } ++ ++ if (INTEL_GEN(dev_priv) >= 9) { ++ if (mode_changed || pipe_config->update_pipe) ++ ret = skl_update_scaler_crtc(pipe_config); ++ ++ if (!ret) ++ ret = icl_check_nv12_planes(pipe_config); ++ if (!ret) ++ ret = skl_check_pipe_max_pixel_rate(intel_crtc, ++ pipe_config); ++ if (!ret) ++ ret = intel_atomic_setup_scalers(dev_priv, intel_crtc, ++ pipe_config); ++ } ++ ++ if (HAS_IPS(dev_priv)) ++ pipe_config->ips_enabled = hsw_compute_ips_config(pipe_config); ++ ++ return ret; ++} ++ ++static const struct drm_crtc_helper_funcs intel_helper_funcs = { ++ .atomic_check = intel_crtc_atomic_check, ++}; ++ ++static void intel_modeset_update_connector_atomic_state(struct drm_device *dev) ++{ ++ struct intel_connector *connector; ++ struct drm_connector_list_iter conn_iter; ++ ++ drm_connector_list_iter_begin(dev, &conn_iter); ++ for_each_intel_connector_iter(connector, &conn_iter) { ++ if (connector->base.state->crtc) ++ drm_connector_put(&connector->base); ++ ++ if (connector->base.encoder) { ++ connector->base.state->best_encoder = ++ connector->base.encoder; ++ connector->base.state->crtc = ++ connector->base.encoder->crtc; ++ ++ drm_connector_get(&connector->base); ++ } else { ++ connector->base.state->best_encoder = NULL; ++ connector->base.state->crtc = NULL; ++ } ++ } ++ drm_connector_list_iter_end(&conn_iter); ++} ++ ++static int ++compute_sink_pipe_bpp(const struct drm_connector_state *conn_state, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_connector *connector = conn_state->connector; ++ const struct drm_display_info *info = &connector->display_info; ++ int bpp; ++ ++ switch (conn_state->max_bpc) { ++ case 6 ... 7: ++ bpp = 6 * 3; ++ break; ++ case 8 ... 9: ++ bpp = 8 * 3; ++ break; ++ case 10 ... 11: ++ bpp = 10 * 3; ++ break; ++ case 12: ++ bpp = 12 * 3; ++ break; ++ default: ++ return -EINVAL; ++ } ++ ++ if (bpp < pipe_config->pipe_bpp) { ++ DRM_DEBUG_KMS("[CONNECTOR:%d:%s] Limiting display bpp to %d instead of " ++ "EDID bpp %d, requested bpp %d, max platform bpp %d\n", ++ connector->base.id, connector->name, ++ bpp, 3 * info->bpc, 3 * conn_state->max_requested_bpc, ++ pipe_config->pipe_bpp); ++ ++ pipe_config->pipe_bpp = bpp; ++ } ++ ++ return 0; ++} ++ ++static int ++compute_baseline_pipe_bpp(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ struct drm_atomic_state *state = pipe_config->base.state; ++ struct drm_connector *connector; ++ struct drm_connector_state *connector_state; ++ int bpp, i; ++ ++ if ((IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) || ++ IS_CHERRYVIEW(dev_priv))) ++ bpp = 10*3; ++ else if (INTEL_GEN(dev_priv) >= 5) ++ bpp = 12*3; ++ else ++ bpp = 8*3; ++ ++ pipe_config->pipe_bpp = bpp; ++ ++ /* Clamp display bpp to connector max bpp */ ++ for_each_new_connector_in_state(state, connector, connector_state, i) { ++ int ret; ++ ++ if (connector_state->crtc != &crtc->base) ++ continue; ++ ++ ret = compute_sink_pipe_bpp(connector_state, pipe_config); ++ if (ret) ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static void intel_dump_crtc_timings(const struct drm_display_mode *mode) ++{ ++ DRM_DEBUG_KMS("crtc timings: %d %d %d %d %d %d %d %d %d, " ++ "type: 0x%x flags: 0x%x\n", ++ mode->crtc_clock, ++ mode->crtc_hdisplay, mode->crtc_hsync_start, ++ mode->crtc_hsync_end, mode->crtc_htotal, ++ mode->crtc_vdisplay, mode->crtc_vsync_start, ++ mode->crtc_vsync_end, mode->crtc_vtotal, mode->type, mode->flags); ++} ++ ++static inline void ++intel_dump_m_n_config(struct intel_crtc_state *pipe_config, char *id, ++ unsigned int lane_count, struct intel_link_m_n *m_n) ++{ ++ DRM_DEBUG_KMS("%s: lanes: %i; gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n", ++ id, lane_count, ++ m_n->gmch_m, m_n->gmch_n, ++ m_n->link_m, m_n->link_n, m_n->tu); ++} ++ ++static void ++intel_dump_infoframe(struct drm_i915_private *dev_priv, ++ const union hdmi_infoframe *frame) ++{ ++ if ((drm_debug & DRM_UT_KMS) == 0) ++ return; ++ ++ hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, frame); ++} ++ ++#define OUTPUT_TYPE(x) [INTEL_OUTPUT_ ## x] = #x ++ ++static const char * const output_type_str[] = { ++ OUTPUT_TYPE(UNUSED), ++ OUTPUT_TYPE(ANALOG), ++ OUTPUT_TYPE(DVO), ++ OUTPUT_TYPE(SDVO), ++ OUTPUT_TYPE(LVDS), ++ OUTPUT_TYPE(TVOUT), ++ OUTPUT_TYPE(HDMI), ++ OUTPUT_TYPE(DP), ++ OUTPUT_TYPE(EDP), ++ OUTPUT_TYPE(DSI), ++ OUTPUT_TYPE(DDI), ++ OUTPUT_TYPE(DP_MST), ++}; ++ ++#undef OUTPUT_TYPE ++ ++static void snprintf_output_types(char *buf, size_t len, ++ unsigned int output_types) ++{ ++ char *str = buf; ++ int i; ++ ++ str[0] = '\0'; ++ ++ for (i = 0; i < ARRAY_SIZE(output_type_str); i++) { ++ int r; ++ ++ if ((output_types & BIT(i)) == 0) ++ continue; ++ ++ r = snprintf(str, len, "%s%s", ++ str != buf ? "," : "", output_type_str[i]); ++ if (r >= len) ++ break; ++ str += r; ++ len -= r; ++ ++ output_types &= ~BIT(i); ++ } ++ ++ WARN_ON_ONCE(output_types != 0); ++} ++ ++static const char * const output_format_str[] = { ++ [INTEL_OUTPUT_FORMAT_INVALID] = "Invalid", ++ [INTEL_OUTPUT_FORMAT_RGB] = "RGB", ++ [INTEL_OUTPUT_FORMAT_YCBCR420] = "YCBCR4:2:0", ++ [INTEL_OUTPUT_FORMAT_YCBCR444] = "YCBCR4:4:4", ++}; ++ ++static const char *output_formats(enum intel_output_format format) ++{ ++ if (format >= ARRAY_SIZE(output_format_str)) ++ format = INTEL_OUTPUT_FORMAT_INVALID; ++ return output_format_str[format]; ++} ++ ++static void intel_dump_pipe_config(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config, ++ const char *context) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct drm_plane *plane; ++ struct intel_plane *intel_plane; ++ struct intel_plane_state *state; ++ struct drm_framebuffer *fb; ++ char buf[64]; ++ ++ DRM_DEBUG_KMS("[CRTC:%d:%s]%s\n", ++ crtc->base.base.id, crtc->base.name, context); ++ ++ snprintf_output_types(buf, sizeof(buf), pipe_config->output_types); ++ DRM_DEBUG_KMS("output_types: %s (0x%x)\n", ++ buf, pipe_config->output_types); ++ ++ DRM_DEBUG_KMS("output format: %s\n", ++ output_formats(pipe_config->output_format)); ++ ++ DRM_DEBUG_KMS("cpu_transcoder: %s, pipe bpp: %i, dithering: %i\n", ++ transcoder_name(pipe_config->cpu_transcoder), ++ pipe_config->pipe_bpp, pipe_config->dither); ++ ++ if (pipe_config->has_pch_encoder) ++ intel_dump_m_n_config(pipe_config, "fdi", ++ pipe_config->fdi_lanes, ++ &pipe_config->fdi_m_n); ++ ++ if (intel_crtc_has_dp_encoder(pipe_config)) { ++ intel_dump_m_n_config(pipe_config, "dp m_n", ++ pipe_config->lane_count, &pipe_config->dp_m_n); ++ if (pipe_config->has_drrs) ++ intel_dump_m_n_config(pipe_config, "dp m2_n2", ++ pipe_config->lane_count, ++ &pipe_config->dp_m2_n2); ++ } ++ ++ DRM_DEBUG_KMS("audio: %i, infoframes: %i\n", ++ pipe_config->has_audio, pipe_config->has_infoframe); ++ ++ DRM_DEBUG_KMS("infoframes enabled: 0x%x\n", ++ pipe_config->infoframes.enable); ++ ++ if (pipe_config->infoframes.enable & ++ intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GENERAL_CONTROL)) ++ DRM_DEBUG_KMS("GCP: 0x%x\n", pipe_config->infoframes.gcp); ++ if (pipe_config->infoframes.enable & ++ intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_AVI)) ++ intel_dump_infoframe(dev_priv, &pipe_config->infoframes.avi); ++ if (pipe_config->infoframes.enable & ++ intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_SPD)) ++ intel_dump_infoframe(dev_priv, &pipe_config->infoframes.spd); ++ if (pipe_config->infoframes.enable & ++ intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_VENDOR)) ++ intel_dump_infoframe(dev_priv, &pipe_config->infoframes.hdmi); ++ ++ DRM_DEBUG_KMS("requested mode:\n"); ++ drm_mode_debug_printmodeline(&pipe_config->base.mode); ++ DRM_DEBUG_KMS("adjusted mode:\n"); ++ drm_mode_debug_printmodeline(&pipe_config->base.adjusted_mode); ++ intel_dump_crtc_timings(&pipe_config->base.adjusted_mode); ++ DRM_DEBUG_KMS("port clock: %d, pipe src size: %dx%d, pixel rate %d\n", ++ pipe_config->port_clock, ++ pipe_config->pipe_src_w, pipe_config->pipe_src_h, ++ pipe_config->pixel_rate); ++ ++ if (INTEL_GEN(dev_priv) >= 9) ++ DRM_DEBUG_KMS("num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n", ++ crtc->num_scalers, ++ pipe_config->scaler_state.scaler_users, ++ pipe_config->scaler_state.scaler_id); ++ ++ if (HAS_GMCH(dev_priv)) ++ DRM_DEBUG_KMS("gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n", ++ pipe_config->gmch_pfit.control, ++ pipe_config->gmch_pfit.pgm_ratios, ++ pipe_config->gmch_pfit.lvds_border_bits); ++ else ++ DRM_DEBUG_KMS("pch pfit: pos: 0x%08x, size: 0x%08x, %s\n", ++ pipe_config->pch_pfit.pos, ++ pipe_config->pch_pfit.size, ++ enableddisabled(pipe_config->pch_pfit.enabled)); ++ ++ DRM_DEBUG_KMS("ips: %i, double wide: %i\n", ++ pipe_config->ips_enabled, pipe_config->double_wide); ++ ++ intel_dpll_dump_hw_state(dev_priv, &pipe_config->dpll_hw_state); ++ ++ DRM_DEBUG_KMS("planes on this crtc\n"); ++ list_for_each_entry(plane, &dev->mode_config.plane_list, head) { ++ struct drm_format_name_buf format_name; ++ intel_plane = to_intel_plane(plane); ++ if (intel_plane->pipe != crtc->pipe) ++ continue; ++ ++ state = to_intel_plane_state(plane->state); ++ fb = state->base.fb; ++ if (!fb) { ++ DRM_DEBUG_KMS("[PLANE:%d:%s] disabled, scaler_id = %d\n", ++ plane->base.id, plane->name, state->scaler_id); ++ continue; ++ } ++ ++ DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d, fb = %ux%u format = %s\n", ++ plane->base.id, plane->name, ++ fb->base.id, fb->width, fb->height, ++ drm_get_format_name(fb->format->format, &format_name)); ++ if (INTEL_GEN(dev_priv) >= 9) ++ DRM_DEBUG_KMS("\tscaler:%d src %dx%d+%d+%d dst %dx%d+%d+%d\n", ++ state->scaler_id, ++ state->base.src.x1 >> 16, ++ state->base.src.y1 >> 16, ++ drm_rect_width(&state->base.src) >> 16, ++ drm_rect_height(&state->base.src) >> 16, ++ state->base.dst.x1, state->base.dst.y1, ++ drm_rect_width(&state->base.dst), ++ drm_rect_height(&state->base.dst)); ++ } ++} ++ ++static bool check_digital_port_conflicts(struct drm_atomic_state *state) ++{ ++ struct drm_device *dev = state->dev; ++ struct drm_connector *connector; ++ struct drm_connector_list_iter conn_iter; ++ unsigned int used_ports = 0; ++ unsigned int used_mst_ports = 0; ++ bool ret = true; ++ ++ /* ++ * Walk the connector list instead of the encoder ++ * list to detect the problem on ddi platforms ++ * where there's just one encoder per digital port. ++ */ ++ drm_connector_list_iter_begin(dev, &conn_iter); ++ drm_for_each_connector_iter(connector, &conn_iter) { ++ struct drm_connector_state *connector_state; ++ struct intel_encoder *encoder; ++ ++ connector_state = drm_atomic_get_new_connector_state(state, connector); ++ if (!connector_state) ++ connector_state = connector->state; ++ ++ if (!connector_state->best_encoder) ++ continue; ++ ++ encoder = to_intel_encoder(connector_state->best_encoder); ++ ++ WARN_ON(!connector_state->crtc); ++ ++ switch (encoder->type) { ++ unsigned int port_mask; ++ case INTEL_OUTPUT_DDI: ++ if (WARN_ON(!HAS_DDI(to_i915(dev)))) ++ break; ++ /* else: fall through */ ++ case INTEL_OUTPUT_DP: ++ case INTEL_OUTPUT_HDMI: ++ case INTEL_OUTPUT_EDP: ++ port_mask = 1 << encoder->port; ++ ++ /* the same port mustn't appear more than once */ ++ if (used_ports & port_mask) ++ ret = false; ++ ++ used_ports |= port_mask; ++ break; ++ case INTEL_OUTPUT_DP_MST: ++ used_mst_ports |= ++ 1 << encoder->port; ++ break; ++ default: ++ break; ++ } ++ } ++ drm_connector_list_iter_end(&conn_iter); ++ ++ /* can't mix MST and SST/HDMI on the same port */ ++ if (used_ports & used_mst_ports) ++ return false; ++ ++ return ret; ++} ++ ++static int ++clear_intel_crtc_state(struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = ++ to_i915(crtc_state->base.crtc->dev); ++ struct intel_crtc_state *saved_state; ++ ++ saved_state = kzalloc(sizeof(*saved_state), GFP_KERNEL); ++ if (!saved_state) ++ return -ENOMEM; ++ ++ /* FIXME: before the switch to atomic started, a new pipe_config was ++ * kzalloc'd. Code that depends on any field being zero should be ++ * fixed, so that the crtc_state can be safely duplicated. For now, ++ * only fields that are know to not cause problems are preserved. */ ++ ++ saved_state->scaler_state = crtc_state->scaler_state; ++ saved_state->shared_dpll = crtc_state->shared_dpll; ++ saved_state->dpll_hw_state = crtc_state->dpll_hw_state; ++ saved_state->pch_pfit.force_thru = crtc_state->pch_pfit.force_thru; ++ saved_state->crc_enabled = crtc_state->crc_enabled; ++ if (IS_G4X(dev_priv) || ++ IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ saved_state->wm = crtc_state->wm; ++ ++ /* Keep base drm_crtc_state intact, only clear our extended struct */ ++ BUILD_BUG_ON(offsetof(struct intel_crtc_state, base)); ++ memcpy(&crtc_state->base + 1, &saved_state->base + 1, ++ sizeof(*crtc_state) - sizeof(crtc_state->base)); ++ ++ kfree(saved_state); ++ return 0; ++} ++ ++static int ++intel_modeset_pipe_config(struct drm_crtc *crtc, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_atomic_state *state = pipe_config->base.state; ++ struct intel_encoder *encoder; ++ struct drm_connector *connector; ++ struct drm_connector_state *connector_state; ++ int base_bpp, ret; ++ int i; ++ bool retry = true; ++ ++ ret = clear_intel_crtc_state(pipe_config); ++ if (ret) ++ return ret; ++ ++ pipe_config->cpu_transcoder = ++ (enum transcoder) to_intel_crtc(crtc)->pipe; ++ ++ /* ++ * Sanitize sync polarity flags based on requested ones. If neither ++ * positive or negative polarity is requested, treat this as meaning ++ * negative polarity. ++ */ ++ if (!(pipe_config->base.adjusted_mode.flags & ++ (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC))) ++ pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NHSYNC; ++ ++ if (!(pipe_config->base.adjusted_mode.flags & ++ (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC))) ++ pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NVSYNC; ++ ++ ret = compute_baseline_pipe_bpp(to_intel_crtc(crtc), ++ pipe_config); ++ if (ret) ++ return ret; ++ ++ base_bpp = pipe_config->pipe_bpp; ++ ++ /* ++ * Determine the real pipe dimensions. Note that stereo modes can ++ * increase the actual pipe size due to the frame doubling and ++ * insertion of additional space for blanks between the frame. This ++ * is stored in the crtc timings. We use the requested mode to do this ++ * computation to clearly distinguish it from the adjusted mode, which ++ * can be changed by the connectors in the below retry loop. ++ */ ++ drm_mode_get_hv_timing(&pipe_config->base.mode, ++ &pipe_config->pipe_src_w, ++ &pipe_config->pipe_src_h); ++ ++ for_each_new_connector_in_state(state, connector, connector_state, i) { ++ if (connector_state->crtc != crtc) ++ continue; ++ ++ encoder = to_intel_encoder(connector_state->best_encoder); ++ ++ if (!check_single_encoder_cloning(state, to_intel_crtc(crtc), encoder)) { ++ DRM_DEBUG_KMS("rejecting invalid cloning configuration\n"); ++ return -EINVAL; ++ } ++ ++ /* ++ * Determine output_types before calling the .compute_config() ++ * hooks so that the hooks can use this information safely. ++ */ ++ if (encoder->compute_output_type) ++ pipe_config->output_types |= ++ BIT(encoder->compute_output_type(encoder, pipe_config, ++ connector_state)); ++ else ++ pipe_config->output_types |= BIT(encoder->type); ++ } ++ ++encoder_retry: ++ /* Ensure the port clock defaults are reset when retrying. */ ++ pipe_config->port_clock = 0; ++ pipe_config->pixel_multiplier = 1; ++ ++ /* Fill in default crtc timings, allow encoders to overwrite them. */ ++ drm_mode_set_crtcinfo(&pipe_config->base.adjusted_mode, ++ CRTC_STEREO_DOUBLE); ++ ++ /* Pass our mode to the connectors and the CRTC to give them a chance to ++ * adjust it according to limitations or connector properties, and also ++ * a chance to reject the mode entirely. ++ */ ++ for_each_new_connector_in_state(state, connector, connector_state, i) { ++ if (connector_state->crtc != crtc) ++ continue; ++ ++ encoder = to_intel_encoder(connector_state->best_encoder); ++ ret = encoder->compute_config(encoder, pipe_config, ++ connector_state); ++ if (ret < 0) { ++ if (ret != -EDEADLK) ++ DRM_DEBUG_KMS("Encoder config failure: %d\n", ++ ret); ++ return ret; ++ } ++ } ++ ++ /* Set default port clock if not overwritten by the encoder. Needs to be ++ * done afterwards in case the encoder adjusts the mode. */ ++ if (!pipe_config->port_clock) ++ pipe_config->port_clock = pipe_config->base.adjusted_mode.crtc_clock ++ * pipe_config->pixel_multiplier; ++ ++ ret = intel_crtc_compute_config(to_intel_crtc(crtc), pipe_config); ++ if (ret == -EDEADLK) ++ return ret; ++ if (ret < 0) { ++ DRM_DEBUG_KMS("CRTC fixup failed\n"); ++ return ret; ++ } ++ ++ if (ret == RETRY) { ++ if (WARN(!retry, "loop in pipe configuration computation\n")) ++ return -EINVAL; ++ ++ DRM_DEBUG_KMS("CRTC bw constrained, retrying\n"); ++ retry = false; ++ goto encoder_retry; ++ } ++ ++ /* Dithering seems to not pass-through bits correctly when it should, so ++ * only enable it on 6bpc panels and when its not a compliance ++ * test requesting 6bpc video pattern. ++ */ ++ pipe_config->dither = (pipe_config->pipe_bpp == 6*3) && ++ !pipe_config->dither_force_disable; ++ DRM_DEBUG_KMS("hw max bpp: %i, pipe bpp: %i, dithering: %i\n", ++ base_bpp, pipe_config->pipe_bpp, pipe_config->dither); ++ ++ return 0; ++} ++ ++bool intel_fuzzy_clock_check(int clock1, int clock2) ++{ ++ int diff; ++ ++ if (clock1 == clock2) ++ return true; ++ ++ if (!clock1 || !clock2) ++ return false; ++ ++ diff = abs(clock1 - clock2); ++ ++ if (((((diff + clock1 + clock2) * 100)) / (clock1 + clock2)) < 105) ++ return true; ++ ++ return false; ++} ++ ++static bool ++intel_compare_m_n(unsigned int m, unsigned int n, ++ unsigned int m2, unsigned int n2, ++ bool exact) ++{ ++ if (m == m2 && n == n2) ++ return true; ++ ++ if (exact || !m || !n || !m2 || !n2) ++ return false; ++ ++ BUILD_BUG_ON(DATA_LINK_M_N_MASK > INT_MAX); ++ ++ if (n > n2) { ++ while (n > n2) { ++ m2 <<= 1; ++ n2 <<= 1; ++ } ++ } else if (n < n2) { ++ while (n < n2) { ++ m <<= 1; ++ n <<= 1; ++ } ++ } ++ ++ if (n != n2) ++ return false; ++ ++ return intel_fuzzy_clock_check(m, m2); ++} ++ ++static bool ++intel_compare_link_m_n(const struct intel_link_m_n *m_n, ++ struct intel_link_m_n *m2_n2, ++ bool adjust) ++{ ++ if (m_n->tu == m2_n2->tu && ++ intel_compare_m_n(m_n->gmch_m, m_n->gmch_n, ++ m2_n2->gmch_m, m2_n2->gmch_n, !adjust) && ++ intel_compare_m_n(m_n->link_m, m_n->link_n, ++ m2_n2->link_m, m2_n2->link_n, !adjust)) { ++ return true; ++ } ++ ++ return false; ++} ++ ++static bool ++intel_compare_infoframe(const union hdmi_infoframe *a, ++ const union hdmi_infoframe *b) ++{ ++ return memcmp(a, b, sizeof(*a)) == 0; ++} ++ ++static void ++pipe_config_infoframe_err(struct drm_i915_private *dev_priv, ++ bool adjust, const char *name, ++ const union hdmi_infoframe *a, ++ const union hdmi_infoframe *b) ++{ ++ if (adjust) { ++ if ((drm_debug & DRM_UT_KMS) == 0) ++ return; ++ ++ drm_dbg(DRM_UT_KMS, "mismatch in %s infoframe", name); ++ drm_dbg(DRM_UT_KMS, "expected:"); ++ hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, a); ++ drm_dbg(DRM_UT_KMS, "found"); ++ hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, b); ++ } else { ++ drm_err("mismatch in %s infoframe", name); ++ drm_err("expected:"); ++ hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, a); ++ drm_err("found"); ++ hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, b); ++ } ++} ++ ++static void __printf(3, 4) ++pipe_config_err(bool adjust, const char *name, const char *format, ...) ++{ ++ struct va_format vaf; ++ va_list args; ++ ++ va_start(args, format); ++ vaf.fmt = format; ++ vaf.va = &args; ++ ++ if (adjust) ++ drm_dbg(DRM_UT_KMS, "mismatch in %s %pV", name, &vaf); ++ else ++ drm_err("mismatch in %s %pV", name, &vaf); ++ ++ va_end(args); ++} ++ ++static bool fastboot_enabled(struct drm_i915_private *dev_priv) ++{ ++ if (i915_modparams.fastboot != -1) ++ return i915_modparams.fastboot; ++ ++ /* Enable fastboot by default on Skylake and newer */ ++ if (INTEL_GEN(dev_priv) >= 9) ++ return true; ++ ++ /* Enable fastboot by default on VLV and CHV */ ++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ return true; ++ ++ /* Disabled by default on all others */ ++ return false; ++} ++ ++static bool ++intel_pipe_config_compare(struct drm_i915_private *dev_priv, ++ struct intel_crtc_state *current_config, ++ struct intel_crtc_state *pipe_config, ++ bool adjust) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(current_config->base.crtc); ++ bool ret = true; ++ bool fixup_inherited = adjust && ++ (current_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED) && ++ !(pipe_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED); ++ ++ if (fixup_inherited && !fastboot_enabled(dev_priv)) { ++ DRM_DEBUG_KMS("initial modeset and fastboot not set\n"); ++ ret = false; ++ } ++ ++#define PIPE_CONF_CHECK_X(name) do { \ ++ if (current_config->name != pipe_config->name) { \ ++ pipe_config_err(adjust, __stringify(name), \ ++ "(expected 0x%08x, found 0x%08x)\n", \ ++ current_config->name, \ ++ pipe_config->name); \ ++ ret = false; \ ++ } \ ++} while (0) ++ ++#define PIPE_CONF_CHECK_I(name) do { \ ++ if (current_config->name != pipe_config->name) { \ ++ pipe_config_err(adjust, __stringify(name), \ ++ "(expected %i, found %i)\n", \ ++ current_config->name, \ ++ pipe_config->name); \ ++ ret = false; \ ++ } \ ++} while (0) ++ ++#define PIPE_CONF_CHECK_BOOL(name) do { \ ++ if (current_config->name != pipe_config->name) { \ ++ pipe_config_err(adjust, __stringify(name), \ ++ "(expected %s, found %s)\n", \ ++ yesno(current_config->name), \ ++ yesno(pipe_config->name)); \ ++ ret = false; \ ++ } \ ++} while (0) ++ ++/* ++ * Checks state where we only read out the enabling, but not the entire ++ * state itself (like full infoframes or ELD for audio). These states ++ * require a full modeset on bootup to fix up. ++ */ ++#define PIPE_CONF_CHECK_BOOL_INCOMPLETE(name) do { \ ++ if (!fixup_inherited || (!current_config->name && !pipe_config->name)) { \ ++ PIPE_CONF_CHECK_BOOL(name); \ ++ } else { \ ++ pipe_config_err(adjust, __stringify(name), \ ++ "unable to verify whether state matches exactly, forcing modeset (expected %s, found %s)\n", \ ++ yesno(current_config->name), \ ++ yesno(pipe_config->name)); \ ++ ret = false; \ ++ } \ ++} while (0) ++ ++#define PIPE_CONF_CHECK_P(name) do { \ ++ if (current_config->name != pipe_config->name) { \ ++ pipe_config_err(adjust, __stringify(name), \ ++ "(expected %p, found %p)\n", \ ++ current_config->name, \ ++ pipe_config->name); \ ++ ret = false; \ ++ } \ ++} while (0) ++ ++#define PIPE_CONF_CHECK_M_N(name) do { \ ++ if (!intel_compare_link_m_n(¤t_config->name, \ ++ &pipe_config->name,\ ++ adjust)) { \ ++ pipe_config_err(adjust, __stringify(name), \ ++ "(expected tu %i gmch %i/%i link %i/%i, " \ ++ "found tu %i, gmch %i/%i link %i/%i)\n", \ ++ current_config->name.tu, \ ++ current_config->name.gmch_m, \ ++ current_config->name.gmch_n, \ ++ current_config->name.link_m, \ ++ current_config->name.link_n, \ ++ pipe_config->name.tu, \ ++ pipe_config->name.gmch_m, \ ++ pipe_config->name.gmch_n, \ ++ pipe_config->name.link_m, \ ++ pipe_config->name.link_n); \ ++ ret = false; \ ++ } \ ++} while (0) ++ ++/* This is required for BDW+ where there is only one set of registers for ++ * switching between high and low RR. ++ * This macro can be used whenever a comparison has to be made between one ++ * hw state and multiple sw state variables. ++ */ ++#define PIPE_CONF_CHECK_M_N_ALT(name, alt_name) do { \ ++ if (!intel_compare_link_m_n(¤t_config->name, \ ++ &pipe_config->name, adjust) && \ ++ !intel_compare_link_m_n(¤t_config->alt_name, \ ++ &pipe_config->name, adjust)) { \ ++ pipe_config_err(adjust, __stringify(name), \ ++ "(expected tu %i gmch %i/%i link %i/%i, " \ ++ "or tu %i gmch %i/%i link %i/%i, " \ ++ "found tu %i, gmch %i/%i link %i/%i)\n", \ ++ current_config->name.tu, \ ++ current_config->name.gmch_m, \ ++ current_config->name.gmch_n, \ ++ current_config->name.link_m, \ ++ current_config->name.link_n, \ ++ current_config->alt_name.tu, \ ++ current_config->alt_name.gmch_m, \ ++ current_config->alt_name.gmch_n, \ ++ current_config->alt_name.link_m, \ ++ current_config->alt_name.link_n, \ ++ pipe_config->name.tu, \ ++ pipe_config->name.gmch_m, \ ++ pipe_config->name.gmch_n, \ ++ pipe_config->name.link_m, \ ++ pipe_config->name.link_n); \ ++ ret = false; \ ++ } \ ++} while (0) ++ ++#define PIPE_CONF_CHECK_FLAGS(name, mask) do { \ ++ if ((current_config->name ^ pipe_config->name) & (mask)) { \ ++ pipe_config_err(adjust, __stringify(name), \ ++ "(%x) (expected %i, found %i)\n", \ ++ (mask), \ ++ current_config->name & (mask), \ ++ pipe_config->name & (mask)); \ ++ ret = false; \ ++ } \ ++} while (0) ++ ++#define PIPE_CONF_CHECK_CLOCK_FUZZY(name) do { \ ++ if (!intel_fuzzy_clock_check(current_config->name, pipe_config->name)) { \ ++ pipe_config_err(adjust, __stringify(name), \ ++ "(expected %i, found %i)\n", \ ++ current_config->name, \ ++ pipe_config->name); \ ++ ret = false; \ ++ } \ ++} while (0) ++ ++#define PIPE_CONF_CHECK_INFOFRAME(name) do { \ ++ if (!intel_compare_infoframe(¤t_config->infoframes.name, \ ++ &pipe_config->infoframes.name)) { \ ++ pipe_config_infoframe_err(dev_priv, adjust, __stringify(name), \ ++ ¤t_config->infoframes.name, \ ++ &pipe_config->infoframes.name); \ ++ ret = false; \ ++ } \ ++} while (0) ++ ++#define PIPE_CONF_QUIRK(quirk) \ ++ ((current_config->quirks | pipe_config->quirks) & (quirk)) ++ ++ PIPE_CONF_CHECK_I(cpu_transcoder); ++ ++ PIPE_CONF_CHECK_BOOL(has_pch_encoder); ++ PIPE_CONF_CHECK_I(fdi_lanes); ++ PIPE_CONF_CHECK_M_N(fdi_m_n); ++ ++ PIPE_CONF_CHECK_I(lane_count); ++ PIPE_CONF_CHECK_X(lane_lat_optim_mask); ++ ++ if (INTEL_GEN(dev_priv) < 8) { ++ PIPE_CONF_CHECK_M_N(dp_m_n); ++ ++ if (current_config->has_drrs) ++ PIPE_CONF_CHECK_M_N(dp_m2_n2); ++ } else ++ PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2); ++ ++ PIPE_CONF_CHECK_X(output_types); ++ ++ PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hdisplay); ++ PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_htotal); ++ PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_start); ++ PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_end); ++ PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_start); ++ PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_end); ++ ++ PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vdisplay); ++ PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vtotal); ++ PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_start); ++ PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_end); ++ PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_start); ++ PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_end); ++ ++ PIPE_CONF_CHECK_I(pixel_multiplier); ++ PIPE_CONF_CHECK_I(output_format); ++ PIPE_CONF_CHECK_BOOL(has_hdmi_sink); ++ if ((INTEL_GEN(dev_priv) < 8 && !IS_HASWELL(dev_priv)) || ++ IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ PIPE_CONF_CHECK_BOOL(limited_color_range); ++ ++ PIPE_CONF_CHECK_BOOL(hdmi_scrambling); ++ PIPE_CONF_CHECK_BOOL(hdmi_high_tmds_clock_ratio); ++ PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_infoframe); ++ ++ PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_audio); ++ ++ PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags, ++ DRM_MODE_FLAG_INTERLACE); ++ ++ if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS)) { ++ PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags, ++ DRM_MODE_FLAG_PHSYNC); ++ PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags, ++ DRM_MODE_FLAG_NHSYNC); ++ PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags, ++ DRM_MODE_FLAG_PVSYNC); ++ PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags, ++ DRM_MODE_FLAG_NVSYNC); ++ } ++ ++ PIPE_CONF_CHECK_X(gmch_pfit.control); ++ /* pfit ratios are autocomputed by the hw on gen4+ */ ++ if (INTEL_GEN(dev_priv) < 4) ++ PIPE_CONF_CHECK_X(gmch_pfit.pgm_ratios); ++ PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits); ++ ++ /* ++ * Changing the EDP transcoder input mux ++ * (A_ONOFF vs. A_ON) requires a full modeset. ++ */ ++ if (IS_HASWELL(dev_priv) && crtc->pipe == PIPE_A && ++ current_config->cpu_transcoder == TRANSCODER_EDP) ++ PIPE_CONF_CHECK_BOOL(pch_pfit.enabled); ++ ++ if (!adjust) { ++ PIPE_CONF_CHECK_I(pipe_src_w); ++ PIPE_CONF_CHECK_I(pipe_src_h); ++ ++ PIPE_CONF_CHECK_BOOL(pch_pfit.enabled); ++ if (current_config->pch_pfit.enabled) { ++ PIPE_CONF_CHECK_X(pch_pfit.pos); ++ PIPE_CONF_CHECK_X(pch_pfit.size); ++ } ++ ++ PIPE_CONF_CHECK_I(scaler_state.scaler_id); ++ PIPE_CONF_CHECK_CLOCK_FUZZY(pixel_rate); ++ ++ PIPE_CONF_CHECK_X(gamma_mode); ++ if (IS_CHERRYVIEW(dev_priv)) ++ PIPE_CONF_CHECK_X(cgm_mode); ++ else ++ PIPE_CONF_CHECK_X(csc_mode); ++ PIPE_CONF_CHECK_BOOL(gamma_enable); ++ PIPE_CONF_CHECK_BOOL(csc_enable); ++ } ++ ++ PIPE_CONF_CHECK_BOOL(double_wide); ++ ++ PIPE_CONF_CHECK_P(shared_dpll); ++ PIPE_CONF_CHECK_X(dpll_hw_state.dpll); ++ PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md); ++ PIPE_CONF_CHECK_X(dpll_hw_state.fp0); ++ PIPE_CONF_CHECK_X(dpll_hw_state.fp1); ++ PIPE_CONF_CHECK_X(dpll_hw_state.wrpll); ++ PIPE_CONF_CHECK_X(dpll_hw_state.spll); ++ PIPE_CONF_CHECK_X(dpll_hw_state.ctrl1); ++ PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1); ++ PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2); ++ PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr0); ++ PIPE_CONF_CHECK_X(dpll_hw_state.ebb0); ++ PIPE_CONF_CHECK_X(dpll_hw_state.ebb4); ++ PIPE_CONF_CHECK_X(dpll_hw_state.pll0); ++ PIPE_CONF_CHECK_X(dpll_hw_state.pll1); ++ PIPE_CONF_CHECK_X(dpll_hw_state.pll2); ++ PIPE_CONF_CHECK_X(dpll_hw_state.pll3); ++ PIPE_CONF_CHECK_X(dpll_hw_state.pll6); ++ PIPE_CONF_CHECK_X(dpll_hw_state.pll8); ++ PIPE_CONF_CHECK_X(dpll_hw_state.pll9); ++ PIPE_CONF_CHECK_X(dpll_hw_state.pll10); ++ PIPE_CONF_CHECK_X(dpll_hw_state.pcsdw12); ++ PIPE_CONF_CHECK_X(dpll_hw_state.mg_refclkin_ctl); ++ PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_coreclkctl1); ++ PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_hsclkctl); ++ PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div0); ++ PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div1); ++ PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_lf); ++ PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_frac_lock); ++ PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_ssc); ++ PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_bias); ++ PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_tdc_coldst_bias); ++ ++ PIPE_CONF_CHECK_X(dsi_pll.ctrl); ++ PIPE_CONF_CHECK_X(dsi_pll.div); ++ ++ if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) ++ PIPE_CONF_CHECK_I(pipe_bpp); ++ ++ PIPE_CONF_CHECK_CLOCK_FUZZY(base.adjusted_mode.crtc_clock); ++ PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock); ++ ++ PIPE_CONF_CHECK_I(min_voltage_level); ++ ++ PIPE_CONF_CHECK_X(infoframes.enable); ++ PIPE_CONF_CHECK_X(infoframes.gcp); ++ PIPE_CONF_CHECK_INFOFRAME(avi); ++ PIPE_CONF_CHECK_INFOFRAME(spd); ++ PIPE_CONF_CHECK_INFOFRAME(hdmi); ++ ++#undef PIPE_CONF_CHECK_X ++#undef PIPE_CONF_CHECK_I ++#undef PIPE_CONF_CHECK_BOOL ++#undef PIPE_CONF_CHECK_BOOL_INCOMPLETE ++#undef PIPE_CONF_CHECK_P ++#undef PIPE_CONF_CHECK_FLAGS ++#undef PIPE_CONF_CHECK_CLOCK_FUZZY ++#undef PIPE_CONF_QUIRK ++ ++ return ret; ++} ++ ++static void intel_pipe_config_sanity_check(struct drm_i915_private *dev_priv, ++ const struct intel_crtc_state *pipe_config) ++{ ++ if (pipe_config->has_pch_encoder) { ++ int fdi_dotclock = intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config), ++ &pipe_config->fdi_m_n); ++ int dotclock = pipe_config->base.adjusted_mode.crtc_clock; ++ ++ /* ++ * FDI already provided one idea for the dotclock. ++ * Yell if the encoder disagrees. ++ */ ++ WARN(!intel_fuzzy_clock_check(fdi_dotclock, dotclock), ++ "FDI dotclock and encoder dotclock mismatch, fdi: %i, encoder: %i\n", ++ fdi_dotclock, dotclock); ++ } ++} ++ ++static void verify_wm_state(struct drm_crtc *crtc, ++ struct drm_crtc_state *new_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->dev); ++ struct skl_hw_state { ++ struct skl_ddb_entry ddb_y[I915_MAX_PLANES]; ++ struct skl_ddb_entry ddb_uv[I915_MAX_PLANES]; ++ struct skl_ddb_allocation ddb; ++ struct skl_pipe_wm wm; ++ } *hw; ++ struct skl_ddb_allocation *sw_ddb; ++ struct skl_pipe_wm *sw_wm; ++ struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry; ++ struct intel_crtc *intel_crtc = to_intel_crtc(crtc); ++ const enum pipe pipe = intel_crtc->pipe; ++ int plane, level, max_level = ilk_wm_max_level(dev_priv); ++ ++ if (INTEL_GEN(dev_priv) < 9 || !new_state->active) ++ return; ++ ++ hw = kzalloc(sizeof(*hw), GFP_KERNEL); ++ if (!hw) ++ return; ++ ++ skl_pipe_wm_get_hw_state(intel_crtc, &hw->wm); ++ sw_wm = &to_intel_crtc_state(new_state)->wm.skl.optimal; ++ ++ skl_pipe_ddb_get_hw_state(intel_crtc, hw->ddb_y, hw->ddb_uv); ++ ++ skl_ddb_get_hw_state(dev_priv, &hw->ddb); ++ sw_ddb = &dev_priv->wm.skl_hw.ddb; ++ ++ if (INTEL_GEN(dev_priv) >= 11 && ++ hw->ddb.enabled_slices != sw_ddb->enabled_slices) ++ DRM_ERROR("mismatch in DBUF Slices (expected %u, got %u)\n", ++ sw_ddb->enabled_slices, ++ hw->ddb.enabled_slices); ++ ++ /* planes */ ++ for_each_universal_plane(dev_priv, pipe, plane) { ++ struct skl_plane_wm *hw_plane_wm, *sw_plane_wm; ++ ++ hw_plane_wm = &hw->wm.planes[plane]; ++ sw_plane_wm = &sw_wm->planes[plane]; ++ ++ /* Watermarks */ ++ for (level = 0; level <= max_level; level++) { ++ if (skl_wm_level_equals(&hw_plane_wm->wm[level], ++ &sw_plane_wm->wm[level])) ++ continue; ++ ++ DRM_ERROR("mismatch in WM pipe %c plane %d level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n", ++ pipe_name(pipe), plane + 1, level, ++ sw_plane_wm->wm[level].plane_en, ++ sw_plane_wm->wm[level].plane_res_b, ++ sw_plane_wm->wm[level].plane_res_l, ++ hw_plane_wm->wm[level].plane_en, ++ hw_plane_wm->wm[level].plane_res_b, ++ hw_plane_wm->wm[level].plane_res_l); ++ } ++ ++ if (!skl_wm_level_equals(&hw_plane_wm->trans_wm, ++ &sw_plane_wm->trans_wm)) { ++ DRM_ERROR("mismatch in trans WM pipe %c plane %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n", ++ pipe_name(pipe), plane + 1, ++ sw_plane_wm->trans_wm.plane_en, ++ sw_plane_wm->trans_wm.plane_res_b, ++ sw_plane_wm->trans_wm.plane_res_l, ++ hw_plane_wm->trans_wm.plane_en, ++ hw_plane_wm->trans_wm.plane_res_b, ++ hw_plane_wm->trans_wm.plane_res_l); ++ } ++ ++ /* DDB */ ++ hw_ddb_entry = &hw->ddb_y[plane]; ++ sw_ddb_entry = &to_intel_crtc_state(new_state)->wm.skl.plane_ddb_y[plane]; ++ ++ if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) { ++ DRM_ERROR("mismatch in DDB state pipe %c plane %d (expected (%u,%u), found (%u,%u))\n", ++ pipe_name(pipe), plane + 1, ++ sw_ddb_entry->start, sw_ddb_entry->end, ++ hw_ddb_entry->start, hw_ddb_entry->end); ++ } ++ } ++ ++ /* ++ * cursor ++ * If the cursor plane isn't active, we may not have updated it's ddb ++ * allocation. In that case since the ddb allocation will be updated ++ * once the plane becomes visible, we can skip this check ++ */ ++ if (1) { ++ struct skl_plane_wm *hw_plane_wm, *sw_plane_wm; ++ ++ hw_plane_wm = &hw->wm.planes[PLANE_CURSOR]; ++ sw_plane_wm = &sw_wm->planes[PLANE_CURSOR]; ++ ++ /* Watermarks */ ++ for (level = 0; level <= max_level; level++) { ++ if (skl_wm_level_equals(&hw_plane_wm->wm[level], ++ &sw_plane_wm->wm[level])) ++ continue; ++ ++ DRM_ERROR("mismatch in WM pipe %c cursor level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n", ++ pipe_name(pipe), level, ++ sw_plane_wm->wm[level].plane_en, ++ sw_plane_wm->wm[level].plane_res_b, ++ sw_plane_wm->wm[level].plane_res_l, ++ hw_plane_wm->wm[level].plane_en, ++ hw_plane_wm->wm[level].plane_res_b, ++ hw_plane_wm->wm[level].plane_res_l); ++ } ++ ++ if (!skl_wm_level_equals(&hw_plane_wm->trans_wm, ++ &sw_plane_wm->trans_wm)) { ++ DRM_ERROR("mismatch in trans WM pipe %c cursor (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n", ++ pipe_name(pipe), ++ sw_plane_wm->trans_wm.plane_en, ++ sw_plane_wm->trans_wm.plane_res_b, ++ sw_plane_wm->trans_wm.plane_res_l, ++ hw_plane_wm->trans_wm.plane_en, ++ hw_plane_wm->trans_wm.plane_res_b, ++ hw_plane_wm->trans_wm.plane_res_l); ++ } ++ ++ /* DDB */ ++ hw_ddb_entry = &hw->ddb_y[PLANE_CURSOR]; ++ sw_ddb_entry = &to_intel_crtc_state(new_state)->wm.skl.plane_ddb_y[PLANE_CURSOR]; ++ ++ if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) { ++ DRM_ERROR("mismatch in DDB state pipe %c cursor (expected (%u,%u), found (%u,%u))\n", ++ pipe_name(pipe), ++ sw_ddb_entry->start, sw_ddb_entry->end, ++ hw_ddb_entry->start, hw_ddb_entry->end); ++ } ++ } ++ ++ kfree(hw); ++} ++ ++static void ++verify_connector_state(struct drm_device *dev, ++ struct drm_atomic_state *state, ++ struct drm_crtc *crtc) ++{ ++ struct drm_connector *connector; ++ struct drm_connector_state *new_conn_state; ++ int i; ++ ++ for_each_new_connector_in_state(state, connector, new_conn_state, i) { ++ struct drm_encoder *encoder = connector->encoder; ++ struct drm_crtc_state *crtc_state = NULL; ++ ++ if (new_conn_state->crtc != crtc) ++ continue; ++ ++ if (crtc) ++ crtc_state = drm_atomic_get_new_crtc_state(state, new_conn_state->crtc); ++ ++ intel_connector_verify_state(crtc_state, new_conn_state); ++ ++ I915_STATE_WARN(new_conn_state->best_encoder != encoder, ++ "connector's atomic encoder doesn't match legacy encoder\n"); ++ } ++} ++ ++static void ++verify_encoder_state(struct drm_device *dev, struct drm_atomic_state *state) ++{ ++ struct intel_encoder *encoder; ++ struct drm_connector *connector; ++ struct drm_connector_state *old_conn_state, *new_conn_state; ++ int i; ++ ++ for_each_intel_encoder(dev, encoder) { ++ bool enabled = false, found = false; ++ enum pipe pipe; ++ ++ DRM_DEBUG_KMS("[ENCODER:%d:%s]\n", ++ encoder->base.base.id, ++ encoder->base.name); ++ ++ for_each_oldnew_connector_in_state(state, connector, old_conn_state, ++ new_conn_state, i) { ++ if (old_conn_state->best_encoder == &encoder->base) ++ found = true; ++ ++ if (new_conn_state->best_encoder != &encoder->base) ++ continue; ++ found = enabled = true; ++ ++ I915_STATE_WARN(new_conn_state->crtc != ++ encoder->base.crtc, ++ "connector's crtc doesn't match encoder crtc\n"); ++ } ++ ++ if (!found) ++ continue; ++ ++ I915_STATE_WARN(!!encoder->base.crtc != enabled, ++ "encoder's enabled state mismatch " ++ "(expected %i, found %i)\n", ++ !!encoder->base.crtc, enabled); ++ ++ if (!encoder->base.crtc) { ++ bool active; ++ ++ active = encoder->get_hw_state(encoder, &pipe); ++ I915_STATE_WARN(active, ++ "encoder detached but still enabled on pipe %c.\n", ++ pipe_name(pipe)); ++ } ++ } ++} ++ ++static void ++verify_crtc_state(struct drm_crtc *crtc, ++ struct drm_crtc_state *old_crtc_state, ++ struct drm_crtc_state *new_crtc_state) ++{ ++ struct drm_device *dev = crtc->dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_encoder *encoder; ++ struct intel_crtc *intel_crtc = to_intel_crtc(crtc); ++ struct intel_crtc_state *pipe_config, *sw_config; ++ struct drm_atomic_state *old_state; ++ bool active; ++ ++ old_state = old_crtc_state->state; ++ __drm_atomic_helper_crtc_destroy_state(old_crtc_state); ++ pipe_config = to_intel_crtc_state(old_crtc_state); ++ memset(pipe_config, 0, sizeof(*pipe_config)); ++ pipe_config->base.crtc = crtc; ++ pipe_config->base.state = old_state; ++ ++ DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.id, crtc->name); ++ ++ active = dev_priv->display.get_pipe_config(intel_crtc, pipe_config); ++ ++ /* we keep both pipes enabled on 830 */ ++ if (IS_I830(dev_priv)) ++ active = new_crtc_state->active; ++ ++ I915_STATE_WARN(new_crtc_state->active != active, ++ "crtc active state doesn't match with hw state " ++ "(expected %i, found %i)\n", new_crtc_state->active, active); ++ ++ I915_STATE_WARN(intel_crtc->active != new_crtc_state->active, ++ "transitional active state does not match atomic hw state " ++ "(expected %i, found %i)\n", new_crtc_state->active, intel_crtc->active); ++ ++ for_each_encoder_on_crtc(dev, crtc, encoder) { ++ enum pipe pipe; ++ ++ active = encoder->get_hw_state(encoder, &pipe); ++ I915_STATE_WARN(active != new_crtc_state->active, ++ "[ENCODER:%i] active %i with crtc active %i\n", ++ encoder->base.base.id, active, new_crtc_state->active); ++ ++ I915_STATE_WARN(active && intel_crtc->pipe != pipe, ++ "Encoder connected to wrong pipe %c\n", ++ pipe_name(pipe)); ++ ++ if (active) ++ encoder->get_config(encoder, pipe_config); ++ } ++ ++ intel_crtc_compute_pixel_rate(pipe_config); ++ ++ if (!new_crtc_state->active) ++ return; ++ ++ intel_pipe_config_sanity_check(dev_priv, pipe_config); ++ ++ sw_config = to_intel_crtc_state(new_crtc_state); ++ if (!intel_pipe_config_compare(dev_priv, sw_config, ++ pipe_config, false)) { ++ I915_STATE_WARN(1, "pipe state doesn't match!\n"); ++ intel_dump_pipe_config(intel_crtc, pipe_config, ++ "[hw state]"); ++ intel_dump_pipe_config(intel_crtc, sw_config, ++ "[sw state]"); ++ } ++} ++ ++static void ++intel_verify_planes(struct intel_atomic_state *state) ++{ ++ struct intel_plane *plane; ++ const struct intel_plane_state *plane_state; ++ int i; ++ ++ for_each_new_intel_plane_in_state(state, plane, ++ plane_state, i) ++ assert_plane(plane, plane_state->slave || ++ plane_state->base.visible); ++} ++ ++static void ++verify_single_dpll_state(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll, ++ struct drm_crtc *crtc, ++ struct drm_crtc_state *new_state) ++{ ++ struct intel_dpll_hw_state dpll_hw_state; ++ unsigned int crtc_mask; ++ bool active; ++ ++ memset(&dpll_hw_state, 0, sizeof(dpll_hw_state)); ++ ++ DRM_DEBUG_KMS("%s\n", pll->info->name); ++ ++ active = pll->info->funcs->get_hw_state(dev_priv, pll, &dpll_hw_state); ++ ++ if (!(pll->info->flags & INTEL_DPLL_ALWAYS_ON)) { ++ I915_STATE_WARN(!pll->on && pll->active_mask, ++ "pll in active use but not on in sw tracking\n"); ++ I915_STATE_WARN(pll->on && !pll->active_mask, ++ "pll is on but not used by any active crtc\n"); ++ I915_STATE_WARN(pll->on != active, ++ "pll on state mismatch (expected %i, found %i)\n", ++ pll->on, active); ++ } ++ ++ if (!crtc) { ++ I915_STATE_WARN(pll->active_mask & ~pll->state.crtc_mask, ++ "more active pll users than references: %x vs %x\n", ++ pll->active_mask, pll->state.crtc_mask); ++ ++ return; ++ } ++ ++ crtc_mask = drm_crtc_mask(crtc); ++ ++ if (new_state->active) ++ I915_STATE_WARN(!(pll->active_mask & crtc_mask), ++ "pll active mismatch (expected pipe %c in active mask 0x%02x)\n", ++ pipe_name(drm_crtc_index(crtc)), pll->active_mask); ++ else ++ I915_STATE_WARN(pll->active_mask & crtc_mask, ++ "pll active mismatch (didn't expect pipe %c in active mask 0x%02x)\n", ++ pipe_name(drm_crtc_index(crtc)), pll->active_mask); ++ ++ I915_STATE_WARN(!(pll->state.crtc_mask & crtc_mask), ++ "pll enabled crtcs mismatch (expected 0x%x in 0x%02x)\n", ++ crtc_mask, pll->state.crtc_mask); ++ ++ I915_STATE_WARN(pll->on && memcmp(&pll->state.hw_state, ++ &dpll_hw_state, ++ sizeof(dpll_hw_state)), ++ "pll hw state mismatch\n"); ++} ++ ++static void ++verify_shared_dpll_state(struct drm_device *dev, struct drm_crtc *crtc, ++ struct drm_crtc_state *old_crtc_state, ++ struct drm_crtc_state *new_crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_crtc_state *old_state = to_intel_crtc_state(old_crtc_state); ++ struct intel_crtc_state *new_state = to_intel_crtc_state(new_crtc_state); ++ ++ if (new_state->shared_dpll) ++ verify_single_dpll_state(dev_priv, new_state->shared_dpll, crtc, new_crtc_state); ++ ++ if (old_state->shared_dpll && ++ old_state->shared_dpll != new_state->shared_dpll) { ++ unsigned int crtc_mask = drm_crtc_mask(crtc); ++ struct intel_shared_dpll *pll = old_state->shared_dpll; ++ ++ I915_STATE_WARN(pll->active_mask & crtc_mask, ++ "pll active mismatch (didn't expect pipe %c in active mask)\n", ++ pipe_name(drm_crtc_index(crtc))); ++ I915_STATE_WARN(pll->state.crtc_mask & crtc_mask, ++ "pll enabled crtcs mismatch (found %x in enabled mask)\n", ++ pipe_name(drm_crtc_index(crtc))); ++ } ++} ++ ++static void ++intel_modeset_verify_crtc(struct drm_crtc *crtc, ++ struct drm_atomic_state *state, ++ struct drm_crtc_state *old_state, ++ struct drm_crtc_state *new_state) ++{ ++ if (!needs_modeset(new_state) && ++ !to_intel_crtc_state(new_state)->update_pipe) ++ return; ++ ++ verify_wm_state(crtc, new_state); ++ verify_connector_state(crtc->dev, state, crtc); ++ verify_crtc_state(crtc, old_state, new_state); ++ verify_shared_dpll_state(crtc->dev, crtc, old_state, new_state); ++} ++ ++static void ++verify_disabled_dpll_state(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ int i; ++ ++ for (i = 0; i < dev_priv->num_shared_dpll; i++) ++ verify_single_dpll_state(dev_priv, &dev_priv->shared_dplls[i], NULL, NULL); ++} ++ ++static void ++intel_modeset_verify_disabled(struct drm_device *dev, ++ struct drm_atomic_state *state) ++{ ++ verify_encoder_state(dev, state); ++ verify_connector_state(dev, state, NULL); ++ verify_disabled_dpll_state(dev); ++} ++ ++static void update_scanline_offset(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ ++ /* ++ * The scanline counter increments at the leading edge of hsync. ++ * ++ * On most platforms it starts counting from vtotal-1 on the ++ * first active line. That means the scanline counter value is ++ * always one less than what we would expect. Ie. just after ++ * start of vblank, which also occurs at start of hsync (on the ++ * last active line), the scanline counter will read vblank_start-1. ++ * ++ * On gen2 the scanline counter starts counting from 1 instead ++ * of vtotal-1, so we have to subtract one (or rather add vtotal-1 ++ * to keep the value positive), instead of adding one. ++ * ++ * On HSW+ the behaviour of the scanline counter depends on the output ++ * type. For DP ports it behaves like most other platforms, but on HDMI ++ * there's an extra 1 line difference. So we need to add two instead of ++ * one to the value. ++ * ++ * On VLV/CHV DSI the scanline counter would appear to increment ++ * approx. 1/3 of a scanline before start of vblank. Unfortunately ++ * that means we can't tell whether we're in vblank or not while ++ * we're on that particular line. We must still set scanline_offset ++ * to 1 so that the vblank timestamps come out correct when we query ++ * the scanline counter from within the vblank interrupt handler. ++ * However if queried just before the start of vblank we'll get an ++ * answer that's slightly in the future. ++ */ ++ if (IS_GEN(dev_priv, 2)) { ++ const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode; ++ int vtotal; ++ ++ vtotal = adjusted_mode->crtc_vtotal; ++ if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) ++ vtotal /= 2; ++ ++ crtc->scanline_offset = vtotal - 1; ++ } else if (HAS_DDI(dev_priv) && ++ intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) { ++ crtc->scanline_offset = 2; ++ } else ++ crtc->scanline_offset = 1; ++} ++ ++static void intel_modeset_clear_plls(struct drm_atomic_state *state) ++{ ++ struct drm_device *dev = state->dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct drm_crtc *crtc; ++ struct drm_crtc_state *old_crtc_state, *new_crtc_state; ++ int i; ++ ++ if (!dev_priv->display.crtc_compute_clock) ++ return; ++ ++ for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { ++ struct intel_crtc *intel_crtc = to_intel_crtc(crtc); ++ struct intel_shared_dpll *old_dpll = ++ to_intel_crtc_state(old_crtc_state)->shared_dpll; ++ ++ if (!needs_modeset(new_crtc_state)) ++ continue; ++ ++ to_intel_crtc_state(new_crtc_state)->shared_dpll = NULL; ++ ++ if (!old_dpll) ++ continue; ++ ++ intel_release_shared_dpll(old_dpll, intel_crtc, state); ++ } ++} ++ ++/* ++ * This implements the workaround described in the "notes" section of the mode ++ * set sequence documentation. When going from no pipes or single pipe to ++ * multiple pipes, and planes are enabled after the pipe, we need to wait at ++ * least 2 vblanks on the first pipe before enabling planes on the second pipe. ++ */ ++static int haswell_mode_set_planes_workaround(struct drm_atomic_state *state) ++{ ++ struct drm_crtc_state *crtc_state; ++ struct intel_crtc *intel_crtc; ++ struct drm_crtc *crtc; ++ struct intel_crtc_state *first_crtc_state = NULL; ++ struct intel_crtc_state *other_crtc_state = NULL; ++ enum pipe first_pipe = INVALID_PIPE, enabled_pipe = INVALID_PIPE; ++ int i; ++ ++ /* look at all crtc's that are going to be enabled in during modeset */ ++ for_each_new_crtc_in_state(state, crtc, crtc_state, i) { ++ intel_crtc = to_intel_crtc(crtc); ++ ++ if (!crtc_state->active || !needs_modeset(crtc_state)) ++ continue; ++ ++ if (first_crtc_state) { ++ other_crtc_state = to_intel_crtc_state(crtc_state); ++ break; ++ } else { ++ first_crtc_state = to_intel_crtc_state(crtc_state); ++ first_pipe = intel_crtc->pipe; ++ } ++ } ++ ++ /* No workaround needed? */ ++ if (!first_crtc_state) ++ return 0; ++ ++ /* w/a possibly needed, check how many crtc's are already enabled. */ ++ for_each_intel_crtc(state->dev, intel_crtc) { ++ struct intel_crtc_state *pipe_config; ++ ++ pipe_config = intel_atomic_get_crtc_state(state, intel_crtc); ++ if (IS_ERR(pipe_config)) ++ return PTR_ERR(pipe_config); ++ ++ pipe_config->hsw_workaround_pipe = INVALID_PIPE; ++ ++ if (!pipe_config->base.active || ++ needs_modeset(&pipe_config->base)) ++ continue; ++ ++ /* 2 or more enabled crtcs means no need for w/a */ ++ if (enabled_pipe != INVALID_PIPE) ++ return 0; ++ ++ enabled_pipe = intel_crtc->pipe; ++ } ++ ++ if (enabled_pipe != INVALID_PIPE) ++ first_crtc_state->hsw_workaround_pipe = enabled_pipe; ++ else if (other_crtc_state) ++ other_crtc_state->hsw_workaround_pipe = first_pipe; ++ ++ return 0; ++} ++ ++static int intel_lock_all_pipes(struct drm_atomic_state *state) ++{ ++ struct drm_crtc *crtc; ++ ++ /* Add all pipes to the state */ ++ for_each_crtc(state->dev, crtc) { ++ struct drm_crtc_state *crtc_state; ++ ++ crtc_state = drm_atomic_get_crtc_state(state, crtc); ++ if (IS_ERR(crtc_state)) ++ return PTR_ERR(crtc_state); ++ } ++ ++ return 0; ++} ++ ++static int intel_modeset_all_pipes(struct drm_atomic_state *state) ++{ ++ struct drm_crtc *crtc; ++ ++ /* ++ * Add all pipes to the state, and force ++ * a modeset on all the active ones. ++ */ ++ for_each_crtc(state->dev, crtc) { ++ struct drm_crtc_state *crtc_state; ++ int ret; ++ ++ crtc_state = drm_atomic_get_crtc_state(state, crtc); ++ if (IS_ERR(crtc_state)) ++ return PTR_ERR(crtc_state); ++ ++ if (!crtc_state->active || needs_modeset(crtc_state)) ++ continue; ++ ++ crtc_state->mode_changed = true; ++ ++ ret = drm_atomic_add_affected_connectors(state, crtc); ++ if (ret) ++ return ret; ++ ++ ret = drm_atomic_add_affected_planes(state, crtc); ++ if (ret) ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static int intel_modeset_checks(struct drm_atomic_state *state) ++{ ++ struct intel_atomic_state *intel_state = to_intel_atomic_state(state); ++ struct drm_i915_private *dev_priv = to_i915(state->dev); ++ struct drm_crtc *crtc; ++ struct drm_crtc_state *old_crtc_state, *new_crtc_state; ++ int ret = 0, i; ++ ++ if (!check_digital_port_conflicts(state)) { ++ DRM_DEBUG_KMS("rejecting conflicting digital port configuration\n"); ++ return -EINVAL; ++ } ++ ++ /* keep the current setting */ ++ if (!intel_state->cdclk.force_min_cdclk_changed) ++ intel_state->cdclk.force_min_cdclk = ++ dev_priv->cdclk.force_min_cdclk; ++ ++ intel_state->modeset = true; ++ intel_state->active_crtcs = dev_priv->active_crtcs; ++ intel_state->cdclk.logical = dev_priv->cdclk.logical; ++ intel_state->cdclk.actual = dev_priv->cdclk.actual; ++ intel_state->cdclk.pipe = INVALID_PIPE; ++ ++ for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { ++ if (new_crtc_state->active) ++ intel_state->active_crtcs |= 1 << i; ++ else ++ intel_state->active_crtcs &= ~(1 << i); ++ ++ if (old_crtc_state->active != new_crtc_state->active) ++ intel_state->active_pipe_changes |= drm_crtc_mask(crtc); ++ } ++ ++ /* ++ * See if the config requires any additional preparation, e.g. ++ * to adjust global state with pipes off. We need to do this ++ * here so we can get the modeset_pipe updated config for the new ++ * mode set on this crtc. For other crtcs we need to use the ++ * adjusted_mode bits in the crtc directly. ++ */ ++ if (dev_priv->display.modeset_calc_cdclk) { ++ enum pipe pipe; ++ ++ ret = dev_priv->display.modeset_calc_cdclk(state); ++ if (ret < 0) ++ return ret; ++ ++ /* ++ * Writes to dev_priv->cdclk.logical must protected by ++ * holding all the crtc locks, even if we don't end up ++ * touching the hardware ++ */ ++ if (intel_cdclk_changed(&dev_priv->cdclk.logical, ++ &intel_state->cdclk.logical)) { ++ ret = intel_lock_all_pipes(state); ++ if (ret < 0) ++ return ret; ++ } ++ ++ if (is_power_of_2(intel_state->active_crtcs)) { ++ struct drm_crtc *crtc; ++ struct drm_crtc_state *crtc_state; ++ ++ pipe = ilog2(intel_state->active_crtcs); ++ crtc = &intel_get_crtc_for_pipe(dev_priv, pipe)->base; ++ crtc_state = drm_atomic_get_new_crtc_state(state, crtc); ++ if (crtc_state && needs_modeset(crtc_state)) ++ pipe = INVALID_PIPE; ++ } else { ++ pipe = INVALID_PIPE; ++ } ++ ++ /* All pipes must be switched off while we change the cdclk. */ ++ if (pipe != INVALID_PIPE && ++ intel_cdclk_needs_cd2x_update(dev_priv, ++ &dev_priv->cdclk.actual, ++ &intel_state->cdclk.actual)) { ++ ret = intel_lock_all_pipes(state); ++ if (ret < 0) ++ return ret; ++ ++ intel_state->cdclk.pipe = pipe; ++ } else if (intel_cdclk_needs_modeset(&dev_priv->cdclk.actual, ++ &intel_state->cdclk.actual)) { ++ ret = intel_modeset_all_pipes(state); ++ if (ret < 0) ++ return ret; ++ ++ intel_state->cdclk.pipe = INVALID_PIPE; ++ } ++ ++ DRM_DEBUG_KMS("New cdclk calculated to be logical %u kHz, actual %u kHz\n", ++ intel_state->cdclk.logical.cdclk, ++ intel_state->cdclk.actual.cdclk); ++ DRM_DEBUG_KMS("New voltage level calculated to be logical %u, actual %u\n", ++ intel_state->cdclk.logical.voltage_level, ++ intel_state->cdclk.actual.voltage_level); ++ } ++ ++ intel_modeset_clear_plls(state); ++ ++ if (IS_HASWELL(dev_priv)) ++ return haswell_mode_set_planes_workaround(state); ++ ++ return 0; ++} ++ ++/* ++ * Handle calculation of various watermark data at the end of the atomic check ++ * phase. The code here should be run after the per-crtc and per-plane 'check' ++ * handlers to ensure that all derived state has been updated. ++ */ ++static int calc_watermark_data(struct intel_atomic_state *state) ++{ ++ struct drm_device *dev = state->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ ++ /* Is there platform-specific watermark information to calculate? */ ++ if (dev_priv->display.compute_global_watermarks) ++ return dev_priv->display.compute_global_watermarks(state); ++ ++ return 0; ++} ++ ++static void intel_crtc_check_fastset(struct intel_crtc_state *old_crtc_state, ++ struct intel_crtc_state *new_crtc_state) ++{ ++ struct drm_i915_private *dev_priv = ++ to_i915(new_crtc_state->base.crtc->dev); ++ ++ if (!intel_pipe_config_compare(dev_priv, old_crtc_state, ++ new_crtc_state, true)) ++ return; ++ ++ new_crtc_state->base.mode_changed = false; ++ new_crtc_state->update_pipe = true; ++ ++ /* ++ * If we're not doing the full modeset we want to ++ * keep the current M/N values as they may be ++ * sufficiently different to the computed values ++ * to cause problems. ++ * ++ * FIXME: should really copy more fuzzy state here ++ */ ++ new_crtc_state->fdi_m_n = old_crtc_state->fdi_m_n; ++ new_crtc_state->dp_m_n = old_crtc_state->dp_m_n; ++ new_crtc_state->dp_m2_n2 = old_crtc_state->dp_m2_n2; ++ new_crtc_state->has_drrs = old_crtc_state->has_drrs; ++} ++ ++/** ++ * intel_atomic_check - validate state object ++ * @dev: drm device ++ * @state: state to validate ++ */ ++static int intel_atomic_check(struct drm_device *dev, ++ struct drm_atomic_state *state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_atomic_state *intel_state = to_intel_atomic_state(state); ++ struct drm_crtc *crtc; ++ struct drm_crtc_state *old_crtc_state, *crtc_state; ++ int ret, i; ++ bool any_ms = intel_state->cdclk.force_min_cdclk_changed; ++ ++ /* Catch I915_MODE_FLAG_INHERITED */ ++ for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, ++ crtc_state, i) { ++ if (crtc_state->mode.private_flags != ++ old_crtc_state->mode.private_flags) ++ crtc_state->mode_changed = true; ++ } ++ ++ ret = drm_atomic_helper_check_modeset(dev, state); ++ if (ret) ++ return ret; ++ ++ for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, crtc_state, i) { ++ struct intel_crtc_state *pipe_config = ++ to_intel_crtc_state(crtc_state); ++ ++ if (!needs_modeset(crtc_state)) ++ continue; ++ ++ if (!crtc_state->enable) { ++ any_ms = true; ++ continue; ++ } ++ ++ ret = intel_modeset_pipe_config(crtc, pipe_config); ++ if (ret == -EDEADLK) ++ return ret; ++ if (ret) { ++ intel_dump_pipe_config(to_intel_crtc(crtc), ++ pipe_config, "[failed]"); ++ return ret; ++ } ++ ++ intel_crtc_check_fastset(to_intel_crtc_state(old_crtc_state), ++ pipe_config); ++ ++ if (needs_modeset(crtc_state)) ++ any_ms = true; ++ ++ intel_dump_pipe_config(to_intel_crtc(crtc), pipe_config, ++ needs_modeset(crtc_state) ? ++ "[modeset]" : "[fastset]"); ++ } ++ ++ ret = drm_dp_mst_atomic_check(state); ++ if (ret) ++ return ret; ++ ++ if (any_ms) { ++ ret = intel_modeset_checks(state); ++ ++ if (ret) ++ return ret; ++ } else { ++ intel_state->cdclk.logical = dev_priv->cdclk.logical; ++ } ++ ++ ret = icl_add_linked_planes(intel_state); ++ if (ret) ++ return ret; ++ ++ ret = drm_atomic_helper_check_planes(dev, state); ++ if (ret) ++ return ret; ++ ++ intel_fbc_choose_crtc(dev_priv, intel_state); ++ return calc_watermark_data(intel_state); ++} ++ ++static int intel_atomic_prepare_commit(struct drm_device *dev, ++ struct drm_atomic_state *state) ++{ ++ return drm_atomic_helper_prepare_planes(dev, state); ++} ++ ++u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(&crtc->base)]; ++ ++ if (!vblank->max_vblank_count) ++ return (u32)drm_crtc_accurate_vblank_count(&crtc->base); ++ ++ return dev->driver->get_vblank_counter(dev, crtc->pipe); ++} ++ ++static void intel_update_crtc(struct drm_crtc *crtc, ++ struct drm_atomic_state *state, ++ struct drm_crtc_state *old_crtc_state, ++ struct drm_crtc_state *new_crtc_state) ++{ ++ struct drm_device *dev = crtc->dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_crtc *intel_crtc = to_intel_crtc(crtc); ++ struct intel_crtc_state *pipe_config = to_intel_crtc_state(new_crtc_state); ++ bool modeset = needs_modeset(new_crtc_state); ++ struct intel_plane_state *new_plane_state = ++ intel_atomic_get_new_plane_state(to_intel_atomic_state(state), ++ to_intel_plane(crtc->primary)); ++ ++ if (modeset) { ++ update_scanline_offset(pipe_config); ++ dev_priv->display.crtc_enable(pipe_config, state); ++ ++ /* vblanks work again, re-enable pipe CRC. */ ++ intel_crtc_enable_pipe_crc(intel_crtc); ++ } else { ++ intel_pre_plane_update(to_intel_crtc_state(old_crtc_state), ++ pipe_config); ++ ++ if (pipe_config->update_pipe) ++ intel_encoders_update_pipe(crtc, pipe_config, state); ++ } ++ ++ if (pipe_config->update_pipe && !pipe_config->enable_fbc) ++ intel_fbc_disable(intel_crtc); ++ else if (new_plane_state) ++ intel_fbc_enable(intel_crtc, pipe_config, new_plane_state); ++ ++ intel_begin_crtc_commit(to_intel_atomic_state(state), intel_crtc); ++ ++ if (INTEL_GEN(dev_priv) >= 9) ++ skl_update_planes_on_crtc(to_intel_atomic_state(state), intel_crtc); ++ else ++ i9xx_update_planes_on_crtc(to_intel_atomic_state(state), intel_crtc); ++ ++ intel_finish_crtc_commit(to_intel_atomic_state(state), intel_crtc); ++} ++ ++static void intel_update_crtcs(struct drm_atomic_state *state) ++{ ++ struct drm_crtc *crtc; ++ struct drm_crtc_state *old_crtc_state, *new_crtc_state; ++ int i; ++ ++ for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { ++ if (!new_crtc_state->active) ++ continue; ++ ++ intel_update_crtc(crtc, state, old_crtc_state, ++ new_crtc_state); ++ } ++} ++ ++static void skl_update_crtcs(struct drm_atomic_state *state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(state->dev); ++ struct intel_atomic_state *intel_state = to_intel_atomic_state(state); ++ struct drm_crtc *crtc; ++ struct intel_crtc *intel_crtc; ++ struct drm_crtc_state *old_crtc_state, *new_crtc_state; ++ struct intel_crtc_state *cstate; ++ unsigned int updated = 0; ++ bool progress; ++ enum pipe pipe; ++ int i; ++ u8 hw_enabled_slices = dev_priv->wm.skl_hw.ddb.enabled_slices; ++ u8 required_slices = intel_state->wm_results.ddb.enabled_slices; ++ struct skl_ddb_entry entries[I915_MAX_PIPES] = {}; ++ ++ for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) ++ /* ignore allocations for crtc's that have been turned off. */ ++ if (new_crtc_state->active) ++ entries[i] = to_intel_crtc_state(old_crtc_state)->wm.skl.ddb; ++ ++ /* If 2nd DBuf slice required, enable it here */ ++ if (INTEL_GEN(dev_priv) >= 11 && required_slices > hw_enabled_slices) ++ icl_dbuf_slices_update(dev_priv, required_slices); ++ ++ /* ++ * Whenever the number of active pipes changes, we need to make sure we ++ * update the pipes in the right order so that their ddb allocations ++ * never overlap with eachother inbetween CRTC updates. Otherwise we'll ++ * cause pipe underruns and other bad stuff. ++ */ ++ do { ++ progress = false; ++ ++ for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { ++ bool vbl_wait = false; ++ unsigned int cmask = drm_crtc_mask(crtc); ++ ++ intel_crtc = to_intel_crtc(crtc); ++ cstate = to_intel_crtc_state(new_crtc_state); ++ pipe = intel_crtc->pipe; ++ ++ if (updated & cmask || !cstate->base.active) ++ continue; ++ ++ if (skl_ddb_allocation_overlaps(&cstate->wm.skl.ddb, ++ entries, ++ INTEL_INFO(dev_priv)->num_pipes, i)) ++ continue; ++ ++ updated |= cmask; ++ entries[i] = cstate->wm.skl.ddb; ++ ++ /* ++ * If this is an already active pipe, it's DDB changed, ++ * and this isn't the last pipe that needs updating ++ * then we need to wait for a vblank to pass for the ++ * new ddb allocation to take effect. ++ */ ++ if (!skl_ddb_entry_equal(&cstate->wm.skl.ddb, ++ &to_intel_crtc_state(old_crtc_state)->wm.skl.ddb) && ++ !new_crtc_state->active_changed && ++ intel_state->wm_results.dirty_pipes != updated) ++ vbl_wait = true; ++ ++ intel_update_crtc(crtc, state, old_crtc_state, ++ new_crtc_state); ++ ++ if (vbl_wait) ++ intel_wait_for_vblank(dev_priv, pipe); ++ ++ progress = true; ++ } ++ } while (progress); ++ ++ /* If 2nd DBuf slice is no more required disable it */ ++ if (INTEL_GEN(dev_priv) >= 11 && required_slices < hw_enabled_slices) ++ icl_dbuf_slices_update(dev_priv, required_slices); ++} ++ ++static void intel_atomic_helper_free_state(struct drm_i915_private *dev_priv) ++{ ++ struct intel_atomic_state *state, *next; ++ struct llist_node *freed; ++ ++ freed = llist_del_all(&dev_priv->atomic_helper.free_list); ++ llist_for_each_entry_safe(state, next, freed, freed) ++ drm_atomic_state_put(&state->base); ++} ++ ++static void intel_atomic_helper_free_state_worker(struct work_struct *work) ++{ ++ struct drm_i915_private *dev_priv = ++ container_of(work, typeof(*dev_priv), atomic_helper.free_work); ++ ++ intel_atomic_helper_free_state(dev_priv); ++} ++ ++static void intel_atomic_commit_fence_wait(struct intel_atomic_state *intel_state) ++{ ++ struct wait_queue_entry wait_fence, wait_reset; ++ struct drm_i915_private *dev_priv = to_i915(intel_state->base.dev); ++ ++ init_wait_entry(&wait_fence, 0); ++ init_wait_entry(&wait_reset, 0); ++ for (;;) { ++ prepare_to_wait(&intel_state->commit_ready.wait, ++ &wait_fence, TASK_UNINTERRUPTIBLE); ++ prepare_to_wait(&dev_priv->gpu_error.wait_queue, ++ &wait_reset, TASK_UNINTERRUPTIBLE); ++ ++ ++ if (i915_sw_fence_done(&intel_state->commit_ready) ++ || test_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags)) ++ break; ++ ++ schedule(); ++ } ++ finish_wait(&intel_state->commit_ready.wait, &wait_fence); ++ finish_wait(&dev_priv->gpu_error.wait_queue, &wait_reset); ++} ++ ++static void intel_atomic_cleanup_work(struct work_struct *work) ++{ ++ struct drm_atomic_state *state = ++ container_of(work, struct drm_atomic_state, commit_work); ++ struct drm_i915_private *i915 = to_i915(state->dev); ++ ++ drm_atomic_helper_cleanup_planes(&i915->drm, state); ++ drm_atomic_helper_commit_cleanup_done(state); ++ drm_atomic_state_put(state); ++ ++ intel_atomic_helper_free_state(i915); ++} ++ ++static void intel_atomic_commit_tail(struct drm_atomic_state *state) ++{ ++ struct drm_device *dev = state->dev; ++ struct intel_atomic_state *intel_state = to_intel_atomic_state(state); ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct drm_crtc_state *old_crtc_state, *new_crtc_state; ++ struct intel_crtc_state *new_intel_crtc_state, *old_intel_crtc_state; ++ struct drm_crtc *crtc; ++ struct intel_crtc *intel_crtc; ++ u64 put_domains[I915_MAX_PIPES] = {}; ++ intel_wakeref_t wakeref = 0; ++ int i; ++ ++ intel_atomic_commit_fence_wait(intel_state); ++ ++ drm_atomic_helper_wait_for_dependencies(state); ++ ++ if (intel_state->modeset) ++ wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET); ++ ++ for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { ++ old_intel_crtc_state = to_intel_crtc_state(old_crtc_state); ++ new_intel_crtc_state = to_intel_crtc_state(new_crtc_state); ++ intel_crtc = to_intel_crtc(crtc); ++ ++ if (needs_modeset(new_crtc_state) || ++ to_intel_crtc_state(new_crtc_state)->update_pipe) { ++ ++ put_domains[intel_crtc->pipe] = ++ modeset_get_crtc_power_domains(crtc, ++ new_intel_crtc_state); ++ } ++ ++ if (!needs_modeset(new_crtc_state)) ++ continue; ++ ++ intel_pre_plane_update(old_intel_crtc_state, new_intel_crtc_state); ++ ++ if (old_crtc_state->active) { ++ intel_crtc_disable_planes(intel_state, intel_crtc); ++ ++ /* ++ * We need to disable pipe CRC before disabling the pipe, ++ * or we race against vblank off. ++ */ ++ intel_crtc_disable_pipe_crc(intel_crtc); ++ ++ dev_priv->display.crtc_disable(old_intel_crtc_state, state); ++ intel_crtc->active = false; ++ intel_fbc_disable(intel_crtc); ++ intel_disable_shared_dpll(old_intel_crtc_state); ++ ++ /* ++ * Underruns don't always raise ++ * interrupts, so check manually. ++ */ ++ intel_check_cpu_fifo_underruns(dev_priv); ++ intel_check_pch_fifo_underruns(dev_priv); ++ ++ /* FIXME unify this for all platforms */ ++ if (!new_crtc_state->active && ++ !HAS_GMCH(dev_priv) && ++ dev_priv->display.initial_watermarks) ++ dev_priv->display.initial_watermarks(intel_state, ++ new_intel_crtc_state); ++ } ++ } ++ ++ /* FIXME: Eventually get rid of our intel_crtc->config pointer */ ++ for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) ++ to_intel_crtc(crtc)->config = to_intel_crtc_state(new_crtc_state); ++ ++ if (intel_state->modeset) { ++ drm_atomic_helper_update_legacy_modeset_state(state->dev, state); ++ ++ intel_set_cdclk_pre_plane_update(dev_priv, ++ &intel_state->cdclk.actual, ++ &dev_priv->cdclk.actual, ++ intel_state->cdclk.pipe); ++ ++ /* ++ * SKL workaround: bspec recommends we disable the SAGV when we ++ * have more then one pipe enabled ++ */ ++ if (!intel_can_enable_sagv(state)) ++ intel_disable_sagv(dev_priv); ++ ++ intel_modeset_verify_disabled(dev, state); ++ } ++ ++ /* Complete the events for pipes that have now been disabled */ ++ for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) { ++ bool modeset = needs_modeset(new_crtc_state); ++ ++ /* Complete events for now disable pipes here. */ ++ if (modeset && !new_crtc_state->active && new_crtc_state->event) { ++ spin_lock_irq(&dev->event_lock); ++ drm_crtc_send_vblank_event(crtc, new_crtc_state->event); ++ spin_unlock_irq(&dev->event_lock); ++ ++ new_crtc_state->event = NULL; ++ } ++ } ++ ++ /* Now enable the clocks, plane, pipe, and connectors that we set up. */ ++ dev_priv->display.update_crtcs(state); ++ ++ if (intel_state->modeset) ++ intel_set_cdclk_post_plane_update(dev_priv, ++ &intel_state->cdclk.actual, ++ &dev_priv->cdclk.actual, ++ intel_state->cdclk.pipe); ++ ++ /* FIXME: We should call drm_atomic_helper_commit_hw_done() here ++ * already, but still need the state for the delayed optimization. To ++ * fix this: ++ * - wrap the optimization/post_plane_update stuff into a per-crtc work. ++ * - schedule that vblank worker _before_ calling hw_done ++ * - at the start of commit_tail, cancel it _synchrously ++ * - switch over to the vblank wait helper in the core after that since ++ * we don't need out special handling any more. ++ */ ++ drm_atomic_helper_wait_for_flip_done(dev, state); ++ ++ for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) { ++ new_intel_crtc_state = to_intel_crtc_state(new_crtc_state); ++ ++ if (new_crtc_state->active && ++ !needs_modeset(new_crtc_state) && ++ (new_intel_crtc_state->base.color_mgmt_changed || ++ new_intel_crtc_state->update_pipe)) ++ intel_color_load_luts(new_intel_crtc_state); ++ } ++ ++ /* ++ * Now that the vblank has passed, we can go ahead and program the ++ * optimal watermarks on platforms that need two-step watermark ++ * programming. ++ * ++ * TODO: Move this (and other cleanup) to an async worker eventually. ++ */ ++ for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) { ++ new_intel_crtc_state = to_intel_crtc_state(new_crtc_state); ++ ++ if (dev_priv->display.optimize_watermarks) ++ dev_priv->display.optimize_watermarks(intel_state, ++ new_intel_crtc_state); ++ } ++ ++ for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { ++ intel_post_plane_update(to_intel_crtc_state(old_crtc_state)); ++ ++ if (put_domains[i]) ++ modeset_put_power_domains(dev_priv, put_domains[i]); ++ ++ intel_modeset_verify_crtc(crtc, state, old_crtc_state, new_crtc_state); ++ } ++ ++ if (intel_state->modeset) ++ intel_verify_planes(intel_state); ++ ++ if (intel_state->modeset && intel_can_enable_sagv(state)) ++ intel_enable_sagv(dev_priv); ++ ++ drm_atomic_helper_commit_hw_done(state); ++ ++ if (intel_state->modeset) { ++ /* As one of the primary mmio accessors, KMS has a high ++ * likelihood of triggering bugs in unclaimed access. After we ++ * finish modesetting, see if an error has been flagged, and if ++ * so enable debugging for the next modeset - and hope we catch ++ * the culprit. ++ */ ++ intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore); ++ intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET, wakeref); ++ } ++ ++ /* ++ * Defer the cleanup of the old state to a separate worker to not ++ * impede the current task (userspace for blocking modesets) that ++ * are executed inline. For out-of-line asynchronous modesets/flips, ++ * deferring to a new worker seems overkill, but we would place a ++ * schedule point (cond_resched()) here anyway to keep latencies ++ * down. ++ */ ++ INIT_WORK(&state->commit_work, intel_atomic_cleanup_work); ++ queue_work(system_highpri_wq, &state->commit_work); ++} ++ ++static void intel_atomic_commit_work(struct work_struct *work) ++{ ++ struct drm_atomic_state *state = ++ container_of(work, struct drm_atomic_state, commit_work); ++ ++ intel_atomic_commit_tail(state); ++} ++ ++static int __i915_sw_fence_call ++intel_atomic_commit_ready(struct i915_sw_fence *fence, ++ enum i915_sw_fence_notify notify) ++{ ++ struct intel_atomic_state *state = ++ container_of(fence, struct intel_atomic_state, commit_ready); ++ ++ switch (notify) { ++ case FENCE_COMPLETE: ++ /* we do blocking waits in the worker, nothing to do here */ ++ break; ++ case FENCE_FREE: ++ { ++ struct intel_atomic_helper *helper = ++ &to_i915(state->base.dev)->atomic_helper; ++ ++ if (llist_add(&state->freed, &helper->free_list)) ++ schedule_work(&helper->free_work); ++ break; ++ } ++ } ++ ++ return NOTIFY_DONE; ++} ++ ++static void intel_atomic_track_fbs(struct drm_atomic_state *state) ++{ ++ struct drm_plane_state *old_plane_state, *new_plane_state; ++ struct drm_plane *plane; ++ int i; ++ ++ for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i) ++ i915_gem_track_fb(intel_fb_obj(old_plane_state->fb), ++ intel_fb_obj(new_plane_state->fb), ++ to_intel_plane(plane)->frontbuffer_bit); ++} ++ ++/** ++ * intel_atomic_commit - commit validated state object ++ * @dev: DRM device ++ * @state: the top-level driver state object ++ * @nonblock: nonblocking commit ++ * ++ * This function commits a top-level state object that has been validated ++ * with drm_atomic_helper_check(). ++ * ++ * RETURNS ++ * Zero for success or -errno. ++ */ ++static int intel_atomic_commit(struct drm_device *dev, ++ struct drm_atomic_state *state, ++ bool nonblock) ++{ ++ struct intel_atomic_state *intel_state = to_intel_atomic_state(state); ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ int ret = 0; ++ ++ drm_atomic_state_get(state); ++ i915_sw_fence_init(&intel_state->commit_ready, ++ intel_atomic_commit_ready); ++ ++ /* ++ * The intel_legacy_cursor_update() fast path takes care ++ * of avoiding the vblank waits for simple cursor ++ * movement and flips. For cursor on/off and size changes, ++ * we want to perform the vblank waits so that watermark ++ * updates happen during the correct frames. Gen9+ have ++ * double buffered watermarks and so shouldn't need this. ++ * ++ * Unset state->legacy_cursor_update before the call to ++ * drm_atomic_helper_setup_commit() because otherwise ++ * drm_atomic_helper_wait_for_flip_done() is a noop and ++ * we get FIFO underruns because we didn't wait ++ * for vblank. ++ * ++ * FIXME doing watermarks and fb cleanup from a vblank worker ++ * (assuming we had any) would solve these problems. ++ */ ++ if (INTEL_GEN(dev_priv) < 9 && state->legacy_cursor_update) { ++ struct intel_crtc_state *new_crtc_state; ++ struct intel_crtc *crtc; ++ int i; ++ ++ for_each_new_intel_crtc_in_state(intel_state, crtc, new_crtc_state, i) ++ if (new_crtc_state->wm.need_postvbl_update || ++ new_crtc_state->update_wm_post) ++ state->legacy_cursor_update = false; ++ } ++ ++ ret = intel_atomic_prepare_commit(dev, state); ++ if (ret) { ++ DRM_DEBUG_ATOMIC("Preparing state failed with %i\n", ret); ++ i915_sw_fence_commit(&intel_state->commit_ready); ++ return ret; ++ } ++ ++ ret = drm_atomic_helper_setup_commit(state, nonblock); ++ if (!ret) ++ ret = drm_atomic_helper_swap_state(state, true); ++ ++ if (ret) { ++ i915_sw_fence_commit(&intel_state->commit_ready); ++ ++ drm_atomic_helper_cleanup_planes(dev, state); ++ return ret; ++ } ++ dev_priv->wm.distrust_bios_wm = false; ++ intel_shared_dpll_swap_state(state); ++ intel_atomic_track_fbs(state); ++ ++ if (intel_state->modeset) { ++ memcpy(dev_priv->min_cdclk, intel_state->min_cdclk, ++ sizeof(intel_state->min_cdclk)); ++ memcpy(dev_priv->min_voltage_level, ++ intel_state->min_voltage_level, ++ sizeof(intel_state->min_voltage_level)); ++ dev_priv->active_crtcs = intel_state->active_crtcs; ++ dev_priv->cdclk.force_min_cdclk = ++ intel_state->cdclk.force_min_cdclk; ++ ++ intel_cdclk_swap_state(intel_state); ++ } ++ ++ drm_atomic_state_get(state); ++ INIT_WORK(&state->commit_work, intel_atomic_commit_work); ++ ++ i915_sw_fence_commit(&intel_state->commit_ready); ++ if (nonblock && intel_state->modeset) { ++ queue_work(dev_priv->modeset_wq, &state->commit_work); ++ } else if (nonblock) { ++ queue_work(system_unbound_wq, &state->commit_work); ++ } else { ++ if (intel_state->modeset) ++ flush_workqueue(dev_priv->modeset_wq); ++ intel_atomic_commit_tail(state); ++ } ++ ++ return 0; ++} ++ ++static const struct drm_crtc_funcs intel_crtc_funcs = { ++ .gamma_set = drm_atomic_helper_legacy_gamma_set, ++ .set_config = drm_atomic_helper_set_config, ++ .destroy = intel_crtc_destroy, ++ .page_flip = drm_atomic_helper_page_flip, ++ .atomic_duplicate_state = intel_crtc_duplicate_state, ++ .atomic_destroy_state = intel_crtc_destroy_state, ++ .set_crc_source = intel_crtc_set_crc_source, ++ .verify_crc_source = intel_crtc_verify_crc_source, ++ .get_crc_sources = intel_crtc_get_crc_sources, ++}; ++ ++struct wait_rps_boost { ++ struct wait_queue_entry wait; ++ ++ struct drm_crtc *crtc; ++ struct i915_request *request; ++}; ++ ++static int do_rps_boost(struct wait_queue_entry *_wait, ++ unsigned mode, int sync, void *key) ++{ ++ struct wait_rps_boost *wait = container_of(_wait, typeof(*wait), wait); ++ struct i915_request *rq = wait->request; ++ ++ /* ++ * If we missed the vblank, but the request is already running it ++ * is reasonable to assume that it will complete before the next ++ * vblank without our intervention, so leave RPS alone. ++ */ ++ if (!i915_request_started(rq)) ++ gen6_rps_boost(rq); ++ i915_request_put(rq); ++ ++ drm_crtc_vblank_put(wait->crtc); ++ ++ list_del(&wait->wait.entry); ++ kfree(wait); ++ return 1; ++} ++ ++static void add_rps_boost_after_vblank(struct drm_crtc *crtc, ++ struct dma_fence *fence) ++{ ++ struct wait_rps_boost *wait; ++ ++ if (!dma_fence_is_i915(fence)) ++ return; ++ ++ if (INTEL_GEN(to_i915(crtc->dev)) < 6) ++ return; ++ ++ if (drm_crtc_vblank_get(crtc)) ++ return; ++ ++ wait = kmalloc(sizeof(*wait), GFP_KERNEL); ++ if (!wait) { ++ drm_crtc_vblank_put(crtc); ++ return; ++ } ++ ++ wait->request = to_request(dma_fence_get(fence)); ++ wait->crtc = crtc; ++ ++ wait->wait.func = do_rps_boost; ++ wait->wait.flags = 0; ++ ++ add_wait_queue(drm_crtc_vblank_waitqueue(crtc), &wait->wait); ++} ++ ++static int intel_plane_pin_fb(struct intel_plane_state *plane_state) ++{ ++ struct intel_plane *plane = to_intel_plane(plane_state->base.plane); ++ struct drm_i915_private *dev_priv = to_i915(plane->base.dev); ++ struct drm_framebuffer *fb = plane_state->base.fb; ++ struct i915_vma *vma; ++ ++ if (plane->id == PLANE_CURSOR && ++ INTEL_INFO(dev_priv)->display.cursor_needs_physical) { ++ struct drm_i915_gem_object *obj = intel_fb_obj(fb); ++ const int align = intel_cursor_alignment(dev_priv); ++ int err; ++ ++ err = i915_gem_object_attach_phys(obj, align); ++ if (err) ++ return err; ++ } ++ ++ vma = intel_pin_and_fence_fb_obj(fb, ++ &plane_state->view, ++ intel_plane_uses_fence(plane_state), ++ &plane_state->flags); ++ if (IS_ERR(vma)) ++ return PTR_ERR(vma); ++ ++ plane_state->vma = vma; ++ ++ return 0; ++} ++ ++static void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state) ++{ ++ struct i915_vma *vma; ++ ++ vma = fetch_and_zero(&old_plane_state->vma); ++ if (vma) ++ intel_unpin_fb_vma(vma, old_plane_state->flags); ++} ++ ++static void fb_obj_bump_render_priority(struct drm_i915_gem_object *obj) ++{ ++ struct i915_sched_attr attr = { ++ .priority = I915_PRIORITY_DISPLAY, ++ }; ++ ++ i915_gem_object_wait_priority(obj, 0, &attr); ++} ++ ++/** ++ * intel_prepare_plane_fb - Prepare fb for usage on plane ++ * @plane: drm plane to prepare for ++ * @new_state: the plane state being prepared ++ * ++ * Prepares a framebuffer for usage on a display plane. Generally this ++ * involves pinning the underlying object and updating the frontbuffer tracking ++ * bits. Some older platforms need special physical address handling for ++ * cursor planes. ++ * ++ * Must be called with struct_mutex held. ++ * ++ * Returns 0 on success, negative error code on failure. ++ */ ++int ++intel_prepare_plane_fb(struct drm_plane *plane, ++ struct drm_plane_state *new_state) ++{ ++ struct intel_atomic_state *intel_state = ++ to_intel_atomic_state(new_state->state); ++ struct drm_i915_private *dev_priv = to_i915(plane->dev); ++ struct drm_framebuffer *fb = new_state->fb; ++ struct drm_i915_gem_object *obj = intel_fb_obj(fb); ++ struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb); ++ int ret; ++ ++ if (old_obj) { ++ struct drm_crtc_state *crtc_state = ++ drm_atomic_get_new_crtc_state(new_state->state, ++ plane->state->crtc); ++ ++ /* Big Hammer, we also need to ensure that any pending ++ * MI_WAIT_FOR_EVENT inside a user batch buffer on the ++ * current scanout is retired before unpinning the old ++ * framebuffer. Note that we rely on userspace rendering ++ * into the buffer attached to the pipe they are waiting ++ * on. If not, userspace generates a GPU hang with IPEHR ++ * point to the MI_WAIT_FOR_EVENT. ++ * ++ * This should only fail upon a hung GPU, in which case we ++ * can safely continue. ++ */ ++ if (needs_modeset(crtc_state)) { ++ ret = i915_sw_fence_await_reservation(&intel_state->commit_ready, ++ old_obj->resv, NULL, ++ false, 0, ++ GFP_KERNEL); ++ if (ret < 0) ++ return ret; ++ } ++ } ++ ++ if (new_state->fence) { /* explicit fencing */ ++ ret = i915_sw_fence_await_dma_fence(&intel_state->commit_ready, ++ new_state->fence, ++ I915_FENCE_TIMEOUT, ++ GFP_KERNEL); ++ if (ret < 0) ++ return ret; ++ } ++ ++ if (!obj) ++ return 0; ++ ++ ret = i915_gem_object_pin_pages(obj); ++ if (ret) ++ return ret; ++ ++ ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex); ++ if (ret) { ++ i915_gem_object_unpin_pages(obj); ++ return ret; ++ } ++ ++ ret = intel_plane_pin_fb(to_intel_plane_state(new_state)); ++ ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++ i915_gem_object_unpin_pages(obj); ++ if (ret) ++ return ret; ++ ++ fb_obj_bump_render_priority(obj); ++ intel_fb_obj_flush(obj, ORIGIN_DIRTYFB); ++ ++ if (!new_state->fence) { /* implicit fencing */ ++ struct dma_fence *fence; ++ ++ ret = i915_sw_fence_await_reservation(&intel_state->commit_ready, ++ obj->resv, NULL, ++ false, I915_FENCE_TIMEOUT, ++ GFP_KERNEL); ++ if (ret < 0) ++ return ret; ++ ++ fence = reservation_object_get_excl_rcu(obj->resv); ++ if (fence) { ++ add_rps_boost_after_vblank(new_state->crtc, fence); ++ dma_fence_put(fence); ++ } ++ } else { ++ add_rps_boost_after_vblank(new_state->crtc, new_state->fence); ++ } ++ ++ /* ++ * We declare pageflips to be interactive and so merit a small bias ++ * towards upclocking to deliver the frame on time. By only changing ++ * the RPS thresholds to sample more regularly and aim for higher ++ * clocks we can hopefully deliver low power workloads (like kodi) ++ * that are not quite steady state without resorting to forcing ++ * maximum clocks following a vblank miss (see do_rps_boost()). ++ */ ++ if (!intel_state->rps_interactive) { ++ intel_rps_mark_interactive(dev_priv, true); ++ intel_state->rps_interactive = true; ++ } ++ ++ return 0; ++} ++ ++/** ++ * intel_cleanup_plane_fb - Cleans up an fb after plane use ++ * @plane: drm plane to clean up for ++ * @old_state: the state from the previous modeset ++ * ++ * Cleans up a framebuffer that has just been removed from a plane. ++ * ++ * Must be called with struct_mutex held. ++ */ ++void ++intel_cleanup_plane_fb(struct drm_plane *plane, ++ struct drm_plane_state *old_state) ++{ ++ struct intel_atomic_state *intel_state = ++ to_intel_atomic_state(old_state->state); ++ struct drm_i915_private *dev_priv = to_i915(plane->dev); ++ ++ if (intel_state->rps_interactive) { ++ intel_rps_mark_interactive(dev_priv, false); ++ intel_state->rps_interactive = false; ++ } ++ ++ /* Should only be called after a successful intel_prepare_plane_fb()! */ ++ mutex_lock(&dev_priv->drm.struct_mutex); ++ intel_plane_unpin_fb(to_intel_plane_state(old_state)); ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++} ++ ++int ++skl_max_scale(const struct intel_crtc_state *crtc_state, ++ u32 pixel_format) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ int max_scale, mult; ++ int crtc_clock, max_dotclk, tmpclk1, tmpclk2; ++ ++ if (!crtc_state->base.enable) ++ return DRM_PLANE_HELPER_NO_SCALING; ++ ++ crtc_clock = crtc_state->base.adjusted_mode.crtc_clock; ++ max_dotclk = to_intel_atomic_state(crtc_state->base.state)->cdclk.logical.cdclk; ++ ++ if (IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10) ++ max_dotclk *= 2; ++ ++ if (WARN_ON_ONCE(!crtc_clock || max_dotclk < crtc_clock)) ++ return DRM_PLANE_HELPER_NO_SCALING; ++ ++ /* ++ * skl max scale is lower of: ++ * close to 3 but not 3, -1 is for that purpose ++ * or ++ * cdclk/crtc_clock ++ */ ++ mult = is_planar_yuv_format(pixel_format) ? 2 : 3; ++ tmpclk1 = (1 << 16) * mult - 1; ++ tmpclk2 = (1 << 8) * ((max_dotclk << 8) / crtc_clock); ++ max_scale = min(tmpclk1, tmpclk2); ++ ++ return max_scale; ++} ++ ++static void intel_begin_crtc_commit(struct intel_atomic_state *state, ++ struct intel_crtc *crtc) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ struct intel_crtc_state *old_crtc_state = ++ intel_atomic_get_old_crtc_state(state, crtc); ++ struct intel_crtc_state *new_crtc_state = ++ intel_atomic_get_new_crtc_state(state, crtc); ++ bool modeset = needs_modeset(&new_crtc_state->base); ++ ++ /* Perform vblank evasion around commit operation */ ++ intel_pipe_update_start(new_crtc_state); ++ ++ if (modeset) ++ goto out; ++ ++ if (new_crtc_state->base.color_mgmt_changed || ++ new_crtc_state->update_pipe) ++ intel_color_commit(new_crtc_state); ++ ++ if (new_crtc_state->update_pipe) ++ intel_update_pipe_config(old_crtc_state, new_crtc_state); ++ else if (INTEL_GEN(dev_priv) >= 9) ++ skl_detach_scalers(new_crtc_state); ++ ++out: ++ if (dev_priv->display.atomic_update_watermarks) ++ dev_priv->display.atomic_update_watermarks(state, ++ new_crtc_state); ++} ++ ++void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc, ++ struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ ++ if (!IS_GEN(dev_priv, 2)) ++ intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true); ++ ++ if (crtc_state->has_pch_encoder) { ++ enum pipe pch_transcoder = ++ intel_crtc_pch_transcoder(crtc); ++ ++ intel_set_pch_fifo_underrun_reporting(dev_priv, pch_transcoder, true); ++ } ++} ++ ++static void intel_finish_crtc_commit(struct intel_atomic_state *state, ++ struct intel_crtc *crtc) ++{ ++ struct intel_crtc_state *old_crtc_state = ++ intel_atomic_get_old_crtc_state(state, crtc); ++ struct intel_crtc_state *new_crtc_state = ++ intel_atomic_get_new_crtc_state(state, crtc); ++ ++ intel_pipe_update_end(new_crtc_state); ++ ++ if (new_crtc_state->update_pipe && ++ !needs_modeset(&new_crtc_state->base) && ++ old_crtc_state->base.mode.private_flags & I915_MODE_FLAG_INHERITED) ++ intel_crtc_arm_fifo_underrun(crtc, new_crtc_state); ++} ++ ++/** ++ * intel_plane_destroy - destroy a plane ++ * @plane: plane to destroy ++ * ++ * Common destruction function for all types of planes (primary, cursor, ++ * sprite). ++ */ ++void intel_plane_destroy(struct drm_plane *plane) ++{ ++ drm_plane_cleanup(plane); ++ kfree(to_intel_plane(plane)); ++} ++ ++static bool i8xx_plane_format_mod_supported(struct drm_plane *_plane, ++ u32 format, u64 modifier) ++{ ++ switch (modifier) { ++ case DRM_FORMAT_MOD_LINEAR: ++ case I915_FORMAT_MOD_X_TILED: ++ break; ++ default: ++ return false; ++ } ++ ++ switch (format) { ++ case DRM_FORMAT_C8: ++ case DRM_FORMAT_RGB565: ++ case DRM_FORMAT_XRGB1555: ++ case DRM_FORMAT_XRGB8888: ++ return modifier == DRM_FORMAT_MOD_LINEAR || ++ modifier == I915_FORMAT_MOD_X_TILED; ++ default: ++ return false; ++ } ++} ++ ++static bool i965_plane_format_mod_supported(struct drm_plane *_plane, ++ u32 format, u64 modifier) ++{ ++ switch (modifier) { ++ case DRM_FORMAT_MOD_LINEAR: ++ case I915_FORMAT_MOD_X_TILED: ++ break; ++ default: ++ return false; ++ } ++ ++ switch (format) { ++ case DRM_FORMAT_C8: ++ case DRM_FORMAT_RGB565: ++ case DRM_FORMAT_XRGB8888: ++ case DRM_FORMAT_XBGR8888: ++ case DRM_FORMAT_XRGB2101010: ++ case DRM_FORMAT_XBGR2101010: ++ return modifier == DRM_FORMAT_MOD_LINEAR || ++ modifier == I915_FORMAT_MOD_X_TILED; ++ default: ++ return false; ++ } ++} ++ ++static bool intel_cursor_format_mod_supported(struct drm_plane *_plane, ++ u32 format, u64 modifier) ++{ ++ return modifier == DRM_FORMAT_MOD_LINEAR && ++ format == DRM_FORMAT_ARGB8888; ++} ++ ++static const struct drm_plane_funcs i965_plane_funcs = { ++ .update_plane = drm_atomic_helper_update_plane, ++ .disable_plane = drm_atomic_helper_disable_plane, ++ .destroy = intel_plane_destroy, ++ .atomic_get_property = intel_plane_atomic_get_property, ++ .atomic_set_property = intel_plane_atomic_set_property, ++ .atomic_duplicate_state = intel_plane_duplicate_state, ++ .atomic_destroy_state = intel_plane_destroy_state, ++ .format_mod_supported = i965_plane_format_mod_supported, ++}; ++ ++static const struct drm_plane_funcs i8xx_plane_funcs = { ++ .update_plane = drm_atomic_helper_update_plane, ++ .disable_plane = drm_atomic_helper_disable_plane, ++ .destroy = intel_plane_destroy, ++ .atomic_get_property = intel_plane_atomic_get_property, ++ .atomic_set_property = intel_plane_atomic_set_property, ++ .atomic_duplicate_state = intel_plane_duplicate_state, ++ .atomic_destroy_state = intel_plane_destroy_state, ++ .format_mod_supported = i8xx_plane_format_mod_supported, ++}; ++ ++static int ++intel_legacy_cursor_update(struct drm_plane *plane, ++ struct drm_crtc *crtc, ++ struct drm_framebuffer *fb, ++ int crtc_x, int crtc_y, ++ unsigned int crtc_w, unsigned int crtc_h, ++ u32 src_x, u32 src_y, ++ u32 src_w, u32 src_h, ++ struct drm_modeset_acquire_ctx *ctx) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc->dev); ++ int ret; ++ struct drm_plane_state *old_plane_state, *new_plane_state; ++ struct intel_plane *intel_plane = to_intel_plane(plane); ++ struct drm_framebuffer *old_fb; ++ struct intel_crtc_state *crtc_state = ++ to_intel_crtc_state(crtc->state); ++ struct intel_crtc_state *new_crtc_state; ++ ++ /* ++ * When crtc is inactive or there is a modeset pending, ++ * wait for it to complete in the slowpath ++ */ ++ if (!crtc_state->base.active || needs_modeset(&crtc_state->base) || ++ crtc_state->update_pipe) ++ goto slow; ++ ++ old_plane_state = plane->state; ++ /* ++ * Don't do an async update if there is an outstanding commit modifying ++ * the plane. This prevents our async update's changes from getting ++ * overridden by a previous synchronous update's state. ++ */ ++ if (old_plane_state->commit && ++ !try_wait_for_completion(&old_plane_state->commit->hw_done)) ++ goto slow; ++ ++ /* ++ * If any parameters change that may affect watermarks, ++ * take the slowpath. Only changing fb or position should be ++ * in the fastpath. ++ */ ++ if (old_plane_state->crtc != crtc || ++ old_plane_state->src_w != src_w || ++ old_plane_state->src_h != src_h || ++ old_plane_state->crtc_w != crtc_w || ++ old_plane_state->crtc_h != crtc_h || ++ !old_plane_state->fb != !fb) ++ goto slow; ++ ++ new_plane_state = intel_plane_duplicate_state(plane); ++ if (!new_plane_state) ++ return -ENOMEM; ++ ++ new_crtc_state = to_intel_crtc_state(intel_crtc_duplicate_state(crtc)); ++ if (!new_crtc_state) { ++ ret = -ENOMEM; ++ goto out_free; ++ } ++ ++ drm_atomic_set_fb_for_plane(new_plane_state, fb); ++ ++ new_plane_state->src_x = src_x; ++ new_plane_state->src_y = src_y; ++ new_plane_state->src_w = src_w; ++ new_plane_state->src_h = src_h; ++ new_plane_state->crtc_x = crtc_x; ++ new_plane_state->crtc_y = crtc_y; ++ new_plane_state->crtc_w = crtc_w; ++ new_plane_state->crtc_h = crtc_h; ++ ++ ret = intel_plane_atomic_check_with_state(crtc_state, new_crtc_state, ++ to_intel_plane_state(old_plane_state), ++ to_intel_plane_state(new_plane_state)); ++ if (ret) ++ goto out_free; ++ ++ ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex); ++ if (ret) ++ goto out_free; ++ ++ ret = intel_plane_pin_fb(to_intel_plane_state(new_plane_state)); ++ if (ret) ++ goto out_unlock; ++ ++ intel_fb_obj_flush(intel_fb_obj(fb), ORIGIN_FLIP); ++ ++ old_fb = old_plane_state->fb; ++ i915_gem_track_fb(intel_fb_obj(old_fb), intel_fb_obj(fb), ++ intel_plane->frontbuffer_bit); ++ ++ /* Swap plane state */ ++ plane->state = new_plane_state; ++ ++ /* ++ * We cannot swap crtc_state as it may be in use by an atomic commit or ++ * page flip that's running simultaneously. If we swap crtc_state and ++ * destroy the old state, we will cause a use-after-free there. ++ * ++ * Only update active_planes, which is needed for our internal ++ * bookkeeping. Either value will do the right thing when updating ++ * planes atomically. If the cursor was part of the atomic update then ++ * we would have taken the slowpath. ++ */ ++ crtc_state->active_planes = new_crtc_state->active_planes; ++ ++ if (plane->state->visible) ++ intel_update_plane(intel_plane, crtc_state, ++ to_intel_plane_state(plane->state)); ++ else ++ intel_disable_plane(intel_plane, crtc_state); ++ ++ intel_plane_unpin_fb(to_intel_plane_state(old_plane_state)); ++ ++out_unlock: ++ mutex_unlock(&dev_priv->drm.struct_mutex); ++out_free: ++ if (new_crtc_state) ++ intel_crtc_destroy_state(crtc, &new_crtc_state->base); ++ if (ret) ++ intel_plane_destroy_state(plane, new_plane_state); ++ else ++ intel_plane_destroy_state(plane, old_plane_state); ++ return ret; ++ ++slow: ++ return drm_atomic_helper_update_plane(plane, crtc, fb, ++ crtc_x, crtc_y, crtc_w, crtc_h, ++ src_x, src_y, src_w, src_h, ctx); ++} ++ ++static const struct drm_plane_funcs intel_cursor_plane_funcs = { ++ .update_plane = intel_legacy_cursor_update, ++ .disable_plane = drm_atomic_helper_disable_plane, ++ .destroy = intel_plane_destroy, ++ .atomic_get_property = intel_plane_atomic_get_property, ++ .atomic_set_property = intel_plane_atomic_set_property, ++ .atomic_duplicate_state = intel_plane_duplicate_state, ++ .atomic_destroy_state = intel_plane_destroy_state, ++ .format_mod_supported = intel_cursor_format_mod_supported, ++}; ++ ++static bool i9xx_plane_has_fbc(struct drm_i915_private *dev_priv, ++ enum i9xx_plane_id i9xx_plane) ++{ ++ if (!HAS_FBC(dev_priv)) ++ return false; ++ ++ if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) ++ return i9xx_plane == PLANE_A; /* tied to pipe A */ ++ else if (IS_IVYBRIDGE(dev_priv)) ++ return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B || ++ i9xx_plane == PLANE_C; ++ else if (INTEL_GEN(dev_priv) >= 4) ++ return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B; ++ else ++ return i9xx_plane == PLANE_A; ++} ++ ++static struct intel_plane * ++intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe) ++{ ++ struct intel_plane *plane; ++ const struct drm_plane_funcs *plane_funcs; ++ unsigned int supported_rotations; ++ unsigned int possible_crtcs; ++ const u64 *modifiers; ++ const u32 *formats; ++ int num_formats; ++ int ret; ++ ++ if (INTEL_GEN(dev_priv) >= 9) ++ return skl_universal_plane_create(dev_priv, pipe, ++ PLANE_PRIMARY); ++ ++ plane = intel_plane_alloc(); ++ if (IS_ERR(plane)) ++ return plane; ++ ++ plane->pipe = pipe; ++ /* ++ * On gen2/3 only plane A can do FBC, but the panel fitter and LVDS ++ * port is hooked to pipe B. Hence we want plane A feeding pipe B. ++ */ ++ if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) < 4) ++ plane->i9xx_plane = (enum i9xx_plane_id) !pipe; ++ else ++ plane->i9xx_plane = (enum i9xx_plane_id) pipe; ++ plane->id = PLANE_PRIMARY; ++ plane->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, plane->id); ++ ++ plane->has_fbc = i9xx_plane_has_fbc(dev_priv, plane->i9xx_plane); ++ if (plane->has_fbc) { ++ struct intel_fbc *fbc = &dev_priv->fbc; ++ ++ fbc->possible_framebuffer_bits |= plane->frontbuffer_bit; ++ } ++ ++ if (INTEL_GEN(dev_priv) >= 4) { ++ formats = i965_primary_formats; ++ num_formats = ARRAY_SIZE(i965_primary_formats); ++ modifiers = i9xx_format_modifiers; ++ ++ plane->max_stride = i9xx_plane_max_stride; ++ plane->update_plane = i9xx_update_plane; ++ plane->disable_plane = i9xx_disable_plane; ++ plane->get_hw_state = i9xx_plane_get_hw_state; ++ plane->check_plane = i9xx_plane_check; ++ ++ plane_funcs = &i965_plane_funcs; ++ } else { ++ formats = i8xx_primary_formats; ++ num_formats = ARRAY_SIZE(i8xx_primary_formats); ++ modifiers = i9xx_format_modifiers; ++ ++ plane->max_stride = i9xx_plane_max_stride; ++ plane->update_plane = i9xx_update_plane; ++ plane->disable_plane = i9xx_disable_plane; ++ plane->get_hw_state = i9xx_plane_get_hw_state; ++ plane->check_plane = i9xx_plane_check; ++ ++ plane_funcs = &i8xx_plane_funcs; ++ } ++ ++ possible_crtcs = BIT(pipe); ++ ++ if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv)) ++ ret = drm_universal_plane_init(&dev_priv->drm, &plane->base, ++ possible_crtcs, plane_funcs, ++ formats, num_formats, modifiers, ++ DRM_PLANE_TYPE_PRIMARY, ++ "primary %c", pipe_name(pipe)); ++ else ++ ret = drm_universal_plane_init(&dev_priv->drm, &plane->base, ++ possible_crtcs, plane_funcs, ++ formats, num_formats, modifiers, ++ DRM_PLANE_TYPE_PRIMARY, ++ "plane %c", ++ plane_name(plane->i9xx_plane)); ++ if (ret) ++ goto fail; ++ ++ if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) { ++ supported_rotations = ++ DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180 | ++ DRM_MODE_REFLECT_X; ++ } else if (INTEL_GEN(dev_priv) >= 4) { ++ supported_rotations = ++ DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180; ++ } else { ++ supported_rotations = DRM_MODE_ROTATE_0; ++ } ++ ++ if (INTEL_GEN(dev_priv) >= 4) ++ drm_plane_create_rotation_property(&plane->base, ++ DRM_MODE_ROTATE_0, ++ supported_rotations); ++ ++ drm_plane_helper_add(&plane->base, &intel_plane_helper_funcs); ++ ++ return plane; ++ ++fail: ++ intel_plane_free(plane); ++ ++ return ERR_PTR(ret); ++} ++ ++static struct intel_plane * ++intel_cursor_plane_create(struct drm_i915_private *dev_priv, ++ enum pipe pipe) ++{ ++ unsigned int possible_crtcs; ++ struct intel_plane *cursor; ++ int ret; ++ ++ cursor = intel_plane_alloc(); ++ if (IS_ERR(cursor)) ++ return cursor; ++ ++ cursor->pipe = pipe; ++ cursor->i9xx_plane = (enum i9xx_plane_id) pipe; ++ cursor->id = PLANE_CURSOR; ++ cursor->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, cursor->id); ++ ++ if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) { ++ cursor->max_stride = i845_cursor_max_stride; ++ cursor->update_plane = i845_update_cursor; ++ cursor->disable_plane = i845_disable_cursor; ++ cursor->get_hw_state = i845_cursor_get_hw_state; ++ cursor->check_plane = i845_check_cursor; ++ } else { ++ cursor->max_stride = i9xx_cursor_max_stride; ++ cursor->update_plane = i9xx_update_cursor; ++ cursor->disable_plane = i9xx_disable_cursor; ++ cursor->get_hw_state = i9xx_cursor_get_hw_state; ++ cursor->check_plane = i9xx_check_cursor; ++ } ++ ++ cursor->cursor.base = ~0; ++ cursor->cursor.cntl = ~0; ++ ++ if (IS_I845G(dev_priv) || IS_I865G(dev_priv) || HAS_CUR_FBC(dev_priv)) ++ cursor->cursor.size = ~0; ++ ++ possible_crtcs = BIT(pipe); ++ ++ ret = drm_universal_plane_init(&dev_priv->drm, &cursor->base, ++ possible_crtcs, &intel_cursor_plane_funcs, ++ intel_cursor_formats, ++ ARRAY_SIZE(intel_cursor_formats), ++ cursor_format_modifiers, ++ DRM_PLANE_TYPE_CURSOR, ++ "cursor %c", pipe_name(pipe)); ++ if (ret) ++ goto fail; ++ ++ if (INTEL_GEN(dev_priv) >= 4) ++ drm_plane_create_rotation_property(&cursor->base, ++ DRM_MODE_ROTATE_0, ++ DRM_MODE_ROTATE_0 | ++ DRM_MODE_ROTATE_180); ++ ++ drm_plane_helper_add(&cursor->base, &intel_plane_helper_funcs); ++ ++ return cursor; ++ ++fail: ++ intel_plane_free(cursor); ++ ++ return ERR_PTR(ret); ++} ++ ++static void intel_crtc_init_scalers(struct intel_crtc *crtc, ++ struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc_scaler_state *scaler_state = ++ &crtc_state->scaler_state; ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ int i; ++ ++ crtc->num_scalers = RUNTIME_INFO(dev_priv)->num_scalers[crtc->pipe]; ++ if (!crtc->num_scalers) ++ return; ++ ++ for (i = 0; i < crtc->num_scalers; i++) { ++ struct intel_scaler *scaler = &scaler_state->scalers[i]; ++ ++ scaler->in_use = 0; ++ scaler->mode = 0; ++ } ++ ++ scaler_state->scaler_id = -1; ++} ++ ++static int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe) ++{ ++ struct intel_crtc *intel_crtc; ++ struct intel_crtc_state *crtc_state = NULL; ++ struct intel_plane *primary = NULL; ++ struct intel_plane *cursor = NULL; ++ int sprite, ret; ++ ++ intel_crtc = kzalloc(sizeof(*intel_crtc), GFP_KERNEL); ++ if (!intel_crtc) ++ return -ENOMEM; ++ ++ crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL); ++ if (!crtc_state) { ++ ret = -ENOMEM; ++ goto fail; ++ } ++ intel_crtc->config = crtc_state; ++ intel_crtc->base.state = &crtc_state->base; ++ crtc_state->base.crtc = &intel_crtc->base; ++ ++ primary = intel_primary_plane_create(dev_priv, pipe); ++ if (IS_ERR(primary)) { ++ ret = PTR_ERR(primary); ++ goto fail; ++ } ++ intel_crtc->plane_ids_mask |= BIT(primary->id); ++ ++ for_each_sprite(dev_priv, pipe, sprite) { ++ struct intel_plane *plane; ++ ++ plane = intel_sprite_plane_create(dev_priv, pipe, sprite); ++ if (IS_ERR(plane)) { ++ ret = PTR_ERR(plane); ++ goto fail; ++ } ++ intel_crtc->plane_ids_mask |= BIT(plane->id); ++ } ++ ++ cursor = intel_cursor_plane_create(dev_priv, pipe); ++ if (IS_ERR(cursor)) { ++ ret = PTR_ERR(cursor); ++ goto fail; ++ } ++ intel_crtc->plane_ids_mask |= BIT(cursor->id); ++ ++ ret = drm_crtc_init_with_planes(&dev_priv->drm, &intel_crtc->base, ++ &primary->base, &cursor->base, ++ &intel_crtc_funcs, ++ "pipe %c", pipe_name(pipe)); ++ if (ret) ++ goto fail; ++ ++ intel_crtc->pipe = pipe; ++ ++ /* initialize shared scalers */ ++ intel_crtc_init_scalers(intel_crtc, crtc_state); ++ ++ BUG_ON(pipe >= ARRAY_SIZE(dev_priv->pipe_to_crtc_mapping) || ++ dev_priv->pipe_to_crtc_mapping[pipe] != NULL); ++ dev_priv->pipe_to_crtc_mapping[pipe] = intel_crtc; ++ ++ if (INTEL_GEN(dev_priv) < 9) { ++ enum i9xx_plane_id i9xx_plane = primary->i9xx_plane; ++ ++ BUG_ON(i9xx_plane >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) || ++ dev_priv->plane_to_crtc_mapping[i9xx_plane] != NULL); ++ dev_priv->plane_to_crtc_mapping[i9xx_plane] = intel_crtc; ++ } ++ ++ drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs); ++ ++ intel_color_init(intel_crtc); ++ ++ WARN_ON(drm_crtc_index(&intel_crtc->base) != intel_crtc->pipe); ++ ++ return 0; ++ ++fail: ++ /* ++ * drm_mode_config_cleanup() will free up any ++ * crtcs/planes already initialized. ++ */ ++ kfree(crtc_state); ++ kfree(intel_crtc); ++ ++ return ret; ++} ++ ++int intel_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data; ++ struct drm_crtc *drmmode_crtc; ++ struct intel_crtc *crtc; ++ ++ drmmode_crtc = drm_crtc_find(dev, file, pipe_from_crtc_id->crtc_id); ++ if (!drmmode_crtc) ++ return -ENOENT; ++ ++ crtc = to_intel_crtc(drmmode_crtc); ++ pipe_from_crtc_id->pipe = crtc->pipe; ++ ++ return 0; ++} ++ ++static int intel_encoder_clones(struct intel_encoder *encoder) ++{ ++ struct drm_device *dev = encoder->base.dev; ++ struct intel_encoder *source_encoder; ++ int index_mask = 0; ++ int entry = 0; ++ ++ for_each_intel_encoder(dev, source_encoder) { ++ if (encoders_cloneable(encoder, source_encoder)) ++ index_mask |= (1 << entry); ++ ++ entry++; ++ } ++ ++ return index_mask; ++} ++ ++static bool ilk_has_edp_a(struct drm_i915_private *dev_priv) ++{ ++ if (!IS_MOBILE(dev_priv)) ++ return false; ++ ++ if ((I915_READ(DP_A) & DP_DETECTED) == 0) ++ return false; ++ ++ if (IS_GEN(dev_priv, 5) && (I915_READ(FUSE_STRAP) & ILK_eDP_A_DISABLE)) ++ return false; ++ ++ return true; ++} ++ ++static bool intel_ddi_crt_present(struct drm_i915_private *dev_priv) ++{ ++ if (INTEL_GEN(dev_priv) >= 9) ++ return false; ++ ++ if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv)) ++ return false; ++ ++ if (HAS_PCH_LPT_H(dev_priv) && ++ I915_READ(SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED) ++ return false; ++ ++ /* DDI E can't be used if DDI A requires 4 lanes */ ++ if (I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES) ++ return false; ++ ++ if (!dev_priv->vbt.int_crt_support) ++ return false; ++ ++ return true; ++} ++ ++void intel_pps_unlock_regs_wa(struct drm_i915_private *dev_priv) ++{ ++ int pps_num; ++ int pps_idx; ++ ++ if (HAS_DDI(dev_priv)) ++ return; ++ /* ++ * This w/a is needed at least on CPT/PPT, but to be sure apply it ++ * everywhere where registers can be write protected. ++ */ ++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ pps_num = 2; ++ else ++ pps_num = 1; ++ ++ for (pps_idx = 0; pps_idx < pps_num; pps_idx++) { ++ u32 val = I915_READ(PP_CONTROL(pps_idx)); ++ ++ val = (val & ~PANEL_UNLOCK_MASK) | PANEL_UNLOCK_REGS; ++ I915_WRITE(PP_CONTROL(pps_idx), val); ++ } ++} ++ ++static void intel_pps_init(struct drm_i915_private *dev_priv) ++{ ++ if (HAS_PCH_SPLIT(dev_priv) || IS_GEN9_LP(dev_priv)) ++ dev_priv->pps_mmio_base = PCH_PPS_BASE; ++ else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ dev_priv->pps_mmio_base = VLV_PPS_BASE; ++ else ++ dev_priv->pps_mmio_base = PPS_BASE; ++ ++ intel_pps_unlock_regs_wa(dev_priv); ++} ++ ++static void intel_setup_outputs(struct drm_i915_private *dev_priv) ++{ ++ struct intel_encoder *encoder; ++ bool dpd_is_edp = false; ++ ++ intel_pps_init(dev_priv); ++ ++ if (!HAS_DISPLAY(dev_priv)) ++ return; ++ ++ if (IS_ELKHARTLAKE(dev_priv)) { ++ intel_ddi_init(dev_priv, PORT_A); ++ intel_ddi_init(dev_priv, PORT_B); ++ intel_ddi_init(dev_priv, PORT_C); ++ icl_dsi_init(dev_priv); ++ } else if (INTEL_GEN(dev_priv) >= 11) { ++ intel_ddi_init(dev_priv, PORT_A); ++ intel_ddi_init(dev_priv, PORT_B); ++ intel_ddi_init(dev_priv, PORT_C); ++ intel_ddi_init(dev_priv, PORT_D); ++ intel_ddi_init(dev_priv, PORT_E); ++ /* ++ * On some ICL SKUs port F is not present. No strap bits for ++ * this, so rely on VBT. ++ * Work around broken VBTs on SKUs known to have no port F. ++ */ ++ if (IS_ICL_WITH_PORT_F(dev_priv) && ++ intel_bios_is_port_present(dev_priv, PORT_F)) ++ intel_ddi_init(dev_priv, PORT_F); ++ ++ icl_dsi_init(dev_priv); ++ } else if (IS_GEN9_LP(dev_priv)) { ++ /* ++ * FIXME: Broxton doesn't support port detection via the ++ * DDI_BUF_CTL_A or SFUSE_STRAP registers, find another way to ++ * detect the ports. ++ */ ++ intel_ddi_init(dev_priv, PORT_A); ++ intel_ddi_init(dev_priv, PORT_B); ++ intel_ddi_init(dev_priv, PORT_C); ++ ++ vlv_dsi_init(dev_priv); ++ } else if (HAS_DDI(dev_priv)) { ++ int found; ++ ++ if (intel_ddi_crt_present(dev_priv)) ++ intel_crt_init(dev_priv); ++ ++ /* ++ * Haswell uses DDI functions to detect digital outputs. ++ * On SKL pre-D0 the strap isn't connected, so we assume ++ * it's there. ++ */ ++ found = I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED; ++ /* WaIgnoreDDIAStrap: skl */ ++ if (found || IS_GEN9_BC(dev_priv)) ++ intel_ddi_init(dev_priv, PORT_A); ++ ++ /* DDI B, C, D, and F detection is indicated by the SFUSE_STRAP ++ * register */ ++ found = I915_READ(SFUSE_STRAP); ++ ++ if (found & SFUSE_STRAP_DDIB_DETECTED) ++ intel_ddi_init(dev_priv, PORT_B); ++ if (found & SFUSE_STRAP_DDIC_DETECTED) ++ intel_ddi_init(dev_priv, PORT_C); ++ if (found & SFUSE_STRAP_DDID_DETECTED) ++ intel_ddi_init(dev_priv, PORT_D); ++ if (found & SFUSE_STRAP_DDIF_DETECTED) ++ intel_ddi_init(dev_priv, PORT_F); ++ /* ++ * On SKL we don't have a way to detect DDI-E so we rely on VBT. ++ */ ++ if (IS_GEN9_BC(dev_priv) && ++ intel_bios_is_port_present(dev_priv, PORT_E)) ++ intel_ddi_init(dev_priv, PORT_E); ++ ++ } else if (HAS_PCH_SPLIT(dev_priv)) { ++ int found; ++ ++ /* ++ * intel_edp_init_connector() depends on this completing first, ++ * to prevent the registration of both eDP and LVDS and the ++ * incorrect sharing of the PPS. ++ */ ++ intel_lvds_init(dev_priv); ++ intel_crt_init(dev_priv); ++ ++ dpd_is_edp = intel_dp_is_port_edp(dev_priv, PORT_D); ++ ++ if (ilk_has_edp_a(dev_priv)) ++ intel_dp_init(dev_priv, DP_A, PORT_A); ++ ++ if (I915_READ(PCH_HDMIB) & SDVO_DETECTED) { ++ /* PCH SDVOB multiplex with HDMIB */ ++ found = intel_sdvo_init(dev_priv, PCH_SDVOB, PORT_B); ++ if (!found) ++ intel_hdmi_init(dev_priv, PCH_HDMIB, PORT_B); ++ if (!found && (I915_READ(PCH_DP_B) & DP_DETECTED)) ++ intel_dp_init(dev_priv, PCH_DP_B, PORT_B); ++ } ++ ++ if (I915_READ(PCH_HDMIC) & SDVO_DETECTED) ++ intel_hdmi_init(dev_priv, PCH_HDMIC, PORT_C); ++ ++ if (!dpd_is_edp && I915_READ(PCH_HDMID) & SDVO_DETECTED) ++ intel_hdmi_init(dev_priv, PCH_HDMID, PORT_D); ++ ++ if (I915_READ(PCH_DP_C) & DP_DETECTED) ++ intel_dp_init(dev_priv, PCH_DP_C, PORT_C); ++ ++ if (I915_READ(PCH_DP_D) & DP_DETECTED) ++ intel_dp_init(dev_priv, PCH_DP_D, PORT_D); ++ } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { ++ bool has_edp, has_port; ++ ++ if (IS_VALLEYVIEW(dev_priv) && dev_priv->vbt.int_crt_support) ++ intel_crt_init(dev_priv); ++ ++ /* ++ * The DP_DETECTED bit is the latched state of the DDC ++ * SDA pin at boot. However since eDP doesn't require DDC ++ * (no way to plug in a DP->HDMI dongle) the DDC pins for ++ * eDP ports may have been muxed to an alternate function. ++ * Thus we can't rely on the DP_DETECTED bit alone to detect ++ * eDP ports. Consult the VBT as well as DP_DETECTED to ++ * detect eDP ports. ++ * ++ * Sadly the straps seem to be missing sometimes even for HDMI ++ * ports (eg. on Voyo V3 - CHT x7-Z8700), so check both strap ++ * and VBT for the presence of the port. Additionally we can't ++ * trust the port type the VBT declares as we've seen at least ++ * HDMI ports that the VBT claim are DP or eDP. ++ */ ++ has_edp = intel_dp_is_port_edp(dev_priv, PORT_B); ++ has_port = intel_bios_is_port_present(dev_priv, PORT_B); ++ if (I915_READ(VLV_DP_B) & DP_DETECTED || has_port) ++ has_edp &= intel_dp_init(dev_priv, VLV_DP_B, PORT_B); ++ if ((I915_READ(VLV_HDMIB) & SDVO_DETECTED || has_port) && !has_edp) ++ intel_hdmi_init(dev_priv, VLV_HDMIB, PORT_B); ++ ++ has_edp = intel_dp_is_port_edp(dev_priv, PORT_C); ++ has_port = intel_bios_is_port_present(dev_priv, PORT_C); ++ if (I915_READ(VLV_DP_C) & DP_DETECTED || has_port) ++ has_edp &= intel_dp_init(dev_priv, VLV_DP_C, PORT_C); ++ if ((I915_READ(VLV_HDMIC) & SDVO_DETECTED || has_port) && !has_edp) ++ intel_hdmi_init(dev_priv, VLV_HDMIC, PORT_C); ++ ++ if (IS_CHERRYVIEW(dev_priv)) { ++ /* ++ * eDP not supported on port D, ++ * so no need to worry about it ++ */ ++ has_port = intel_bios_is_port_present(dev_priv, PORT_D); ++ if (I915_READ(CHV_DP_D) & DP_DETECTED || has_port) ++ intel_dp_init(dev_priv, CHV_DP_D, PORT_D); ++ if (I915_READ(CHV_HDMID) & SDVO_DETECTED || has_port) ++ intel_hdmi_init(dev_priv, CHV_HDMID, PORT_D); ++ } ++ ++ vlv_dsi_init(dev_priv); ++ } else if (IS_PINEVIEW(dev_priv)) { ++ intel_lvds_init(dev_priv); ++ intel_crt_init(dev_priv); ++ } else if (IS_GEN_RANGE(dev_priv, 3, 4)) { ++ bool found = false; ++ ++ if (IS_MOBILE(dev_priv)) ++ intel_lvds_init(dev_priv); ++ ++ intel_crt_init(dev_priv); ++ ++ if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) { ++ DRM_DEBUG_KMS("probing SDVOB\n"); ++ found = intel_sdvo_init(dev_priv, GEN3_SDVOB, PORT_B); ++ if (!found && IS_G4X(dev_priv)) { ++ DRM_DEBUG_KMS("probing HDMI on SDVOB\n"); ++ intel_hdmi_init(dev_priv, GEN4_HDMIB, PORT_B); ++ } ++ ++ if (!found && IS_G4X(dev_priv)) ++ intel_dp_init(dev_priv, DP_B, PORT_B); ++ } ++ ++ /* Before G4X SDVOC doesn't have its own detect register */ ++ ++ if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) { ++ DRM_DEBUG_KMS("probing SDVOC\n"); ++ found = intel_sdvo_init(dev_priv, GEN3_SDVOC, PORT_C); ++ } ++ ++ if (!found && (I915_READ(GEN3_SDVOC) & SDVO_DETECTED)) { ++ ++ if (IS_G4X(dev_priv)) { ++ DRM_DEBUG_KMS("probing HDMI on SDVOC\n"); ++ intel_hdmi_init(dev_priv, GEN4_HDMIC, PORT_C); ++ } ++ if (IS_G4X(dev_priv)) ++ intel_dp_init(dev_priv, DP_C, PORT_C); ++ } ++ ++ if (IS_G4X(dev_priv) && (I915_READ(DP_D) & DP_DETECTED)) ++ intel_dp_init(dev_priv, DP_D, PORT_D); ++ ++ if (SUPPORTS_TV(dev_priv)) ++ intel_tv_init(dev_priv); ++ } else if (IS_GEN(dev_priv, 2)) { ++ if (IS_I85X(dev_priv)) ++ intel_lvds_init(dev_priv); ++ ++ intel_crt_init(dev_priv); ++ intel_dvo_init(dev_priv); ++ } ++ ++ intel_psr_init(dev_priv); ++ ++ for_each_intel_encoder(&dev_priv->drm, encoder) { ++ encoder->base.possible_crtcs = encoder->crtc_mask; ++ encoder->base.possible_clones = ++ intel_encoder_clones(encoder); ++ } ++ ++ intel_init_pch_refclk(dev_priv); ++ ++ drm_helper_move_panel_connectors_to_head(&dev_priv->drm); ++} ++ ++static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb) ++{ ++ struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); ++ struct drm_i915_gem_object *obj = intel_fb_obj(fb); ++ ++ drm_framebuffer_cleanup(fb); ++ ++ i915_gem_object_lock(obj); ++ WARN_ON(!obj->framebuffer_references--); ++ i915_gem_object_unlock(obj); ++ ++ i915_gem_object_put(obj); ++ ++ kfree(intel_fb); ++} ++ ++static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb, ++ struct drm_file *file, ++ unsigned int *handle) ++{ ++ struct drm_i915_gem_object *obj = intel_fb_obj(fb); ++ ++ if (obj->userptr.mm) { ++ DRM_DEBUG("attempting to use a userptr for a framebuffer, denied\n"); ++ return -EINVAL; ++ } ++ ++ return drm_gem_handle_create(file, &obj->base, handle); ++} ++ ++static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb, ++ struct drm_file *file, ++ unsigned flags, unsigned color, ++ struct drm_clip_rect *clips, ++ unsigned num_clips) ++{ ++ struct drm_i915_gem_object *obj = intel_fb_obj(fb); ++ ++ i915_gem_object_flush_if_display(obj); ++ intel_fb_obj_flush(obj, ORIGIN_DIRTYFB); ++ ++ return 0; ++} ++ ++static const struct drm_framebuffer_funcs intel_fb_funcs = { ++ .destroy = intel_user_framebuffer_destroy, ++ .create_handle = intel_user_framebuffer_create_handle, ++ .dirty = intel_user_framebuffer_dirty, ++}; ++ ++static ++u32 intel_fb_pitch_limit(struct drm_i915_private *dev_priv, ++ u32 pixel_format, u64 fb_modifier) ++{ ++ struct intel_crtc *crtc; ++ struct intel_plane *plane; ++ ++ /* ++ * We assume the primary plane for pipe A has ++ * the highest stride limits of them all. ++ */ ++ crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_A); ++ plane = to_intel_plane(crtc->base.primary); ++ ++ return plane->max_stride(plane, pixel_format, fb_modifier, ++ DRM_MODE_ROTATE_0); ++} ++ ++static int intel_framebuffer_init(struct intel_framebuffer *intel_fb, ++ struct drm_i915_gem_object *obj, ++ struct drm_mode_fb_cmd2 *mode_cmd) ++{ ++ struct drm_i915_private *dev_priv = to_i915(obj->base.dev); ++ struct drm_framebuffer *fb = &intel_fb->base; ++ u32 pitch_limit; ++ unsigned int tiling, stride; ++ int ret = -EINVAL; ++ int i; ++ ++ i915_gem_object_lock(obj); ++ obj->framebuffer_references++; ++ tiling = i915_gem_object_get_tiling(obj); ++ stride = i915_gem_object_get_stride(obj); ++ i915_gem_object_unlock(obj); ++ ++ if (mode_cmd->flags & DRM_MODE_FB_MODIFIERS) { ++ /* ++ * If there's a fence, enforce that ++ * the fb modifier and tiling mode match. ++ */ ++ if (tiling != I915_TILING_NONE && ++ tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) { ++ DRM_DEBUG_KMS("tiling_mode doesn't match fb modifier\n"); ++ goto err; ++ } ++ } else { ++ if (tiling == I915_TILING_X) { ++ mode_cmd->modifier[0] = I915_FORMAT_MOD_X_TILED; ++ } else if (tiling == I915_TILING_Y) { ++ DRM_DEBUG_KMS("No Y tiling for legacy addfb\n"); ++ goto err; ++ } ++ } ++ ++ if (!drm_any_plane_has_format(&dev_priv->drm, ++ mode_cmd->pixel_format, ++ mode_cmd->modifier[0])) { ++ struct drm_format_name_buf format_name; ++ ++ DRM_DEBUG_KMS("unsupported pixel format %s / modifier 0x%llx\n", ++ drm_get_format_name(mode_cmd->pixel_format, ++ &format_name), ++ mode_cmd->modifier[0]); ++ goto err; ++ } ++ ++ /* ++ * gen2/3 display engine uses the fence if present, ++ * so the tiling mode must match the fb modifier exactly. ++ */ ++ if (INTEL_GEN(dev_priv) < 4 && ++ tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) { ++ DRM_DEBUG_KMS("tiling_mode must match fb modifier exactly on gen2/3\n"); ++ goto err; ++ } ++ ++ pitch_limit = intel_fb_pitch_limit(dev_priv, mode_cmd->pixel_format, ++ mode_cmd->modifier[0]); ++ if (mode_cmd->pitches[0] > pitch_limit) { ++ DRM_DEBUG_KMS("%s pitch (%u) must be at most %d\n", ++ mode_cmd->modifier[0] != DRM_FORMAT_MOD_LINEAR ? ++ "tiled" : "linear", ++ mode_cmd->pitches[0], pitch_limit); ++ goto err; ++ } ++ ++ /* ++ * If there's a fence, enforce that ++ * the fb pitch and fence stride match. ++ */ ++ if (tiling != I915_TILING_NONE && mode_cmd->pitches[0] != stride) { ++ DRM_DEBUG_KMS("pitch (%d) must match tiling stride (%d)\n", ++ mode_cmd->pitches[0], stride); ++ goto err; ++ } ++ ++ /* FIXME need to adjust LINOFF/TILEOFF accordingly. */ ++ if (mode_cmd->offsets[0] != 0) ++ goto err; ++ ++ drm_helper_mode_fill_fb_struct(&dev_priv->drm, fb, mode_cmd); ++ ++ for (i = 0; i < fb->format->num_planes; i++) { ++ u32 stride_alignment; ++ ++ if (mode_cmd->handles[i] != mode_cmd->handles[0]) { ++ DRM_DEBUG_KMS("bad plane %d handle\n", i); ++ goto err; ++ } ++ ++ stride_alignment = intel_fb_stride_alignment(fb, i); ++ ++ /* ++ * Display WA #0531: skl,bxt,kbl,glk ++ * ++ * Render decompression and plane width > 3840 ++ * combined with horizontal panning requires the ++ * plane stride to be a multiple of 4. We'll just ++ * require the entire fb to accommodate that to avoid ++ * potential runtime errors at plane configuration time. ++ */ ++ if (IS_GEN(dev_priv, 9) && i == 0 && fb->width > 3840 && ++ is_ccs_modifier(fb->modifier)) ++ stride_alignment *= 4; ++ ++ if (fb->pitches[i] & (stride_alignment - 1)) { ++ DRM_DEBUG_KMS("plane %d pitch (%d) must be at least %u byte aligned\n", ++ i, fb->pitches[i], stride_alignment); ++ goto err; ++ } ++ ++ fb->obj[i] = &obj->base; ++ } ++ ++ ret = intel_fill_fb_info(dev_priv, fb); ++ if (ret) ++ goto err; ++ ++ ret = drm_framebuffer_init(&dev_priv->drm, fb, &intel_fb_funcs); ++ if (ret) { ++ DRM_ERROR("framebuffer init failed %d\n", ret); ++ goto err; ++ } ++ ++ return 0; ++ ++err: ++ i915_gem_object_lock(obj); ++ obj->framebuffer_references--; ++ i915_gem_object_unlock(obj); ++ return ret; ++} ++ ++static struct drm_framebuffer * ++intel_user_framebuffer_create(struct drm_device *dev, ++ struct drm_file *filp, ++ const struct drm_mode_fb_cmd2 *user_mode_cmd) ++{ ++ struct drm_framebuffer *fb; ++ struct drm_i915_gem_object *obj; ++ struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd; ++ ++ obj = i915_gem_object_lookup(filp, mode_cmd.handles[0]); ++ if (!obj) ++ return ERR_PTR(-ENOENT); ++ ++ fb = intel_framebuffer_create(obj, &mode_cmd); ++ if (IS_ERR(fb)) ++ i915_gem_object_put(obj); ++ ++ return fb; ++} ++ ++static void intel_atomic_state_free(struct drm_atomic_state *state) ++{ ++ struct intel_atomic_state *intel_state = to_intel_atomic_state(state); ++ ++ drm_atomic_state_default_release(state); ++ ++ i915_sw_fence_fini(&intel_state->commit_ready); ++ ++ kfree(state); ++} ++ ++static enum drm_mode_status ++intel_mode_valid(struct drm_device *dev, ++ const struct drm_display_mode *mode) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ int hdisplay_max, htotal_max; ++ int vdisplay_max, vtotal_max; ++ ++ /* ++ * Can't reject DBLSCAN here because Xorg ddxen can add piles ++ * of DBLSCAN modes to the output's mode list when they detect ++ * the scaling mode property on the connector. And they don't ++ * ask the kernel to validate those modes in any way until ++ * modeset time at which point the client gets a protocol error. ++ * So in order to not upset those clients we silently ignore the ++ * DBLSCAN flag on such connectors. For other connectors we will ++ * reject modes with the DBLSCAN flag in encoder->compute_config(). ++ * And we always reject DBLSCAN modes in connector->mode_valid() ++ * as we never want such modes on the connector's mode list. ++ */ ++ ++ if (mode->vscan > 1) ++ return MODE_NO_VSCAN; ++ ++ if (mode->flags & DRM_MODE_FLAG_HSKEW) ++ return MODE_H_ILLEGAL; ++ ++ if (mode->flags & (DRM_MODE_FLAG_CSYNC | ++ DRM_MODE_FLAG_NCSYNC | ++ DRM_MODE_FLAG_PCSYNC)) ++ return MODE_HSYNC; ++ ++ if (mode->flags & (DRM_MODE_FLAG_BCAST | ++ DRM_MODE_FLAG_PIXMUX | ++ DRM_MODE_FLAG_CLKDIV2)) ++ return MODE_BAD; ++ ++ if (INTEL_GEN(dev_priv) >= 9 || ++ IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) { ++ hdisplay_max = 8192; /* FDI max 4096 handled elsewhere */ ++ vdisplay_max = 4096; ++ htotal_max = 8192; ++ vtotal_max = 8192; ++ } else if (INTEL_GEN(dev_priv) >= 3) { ++ hdisplay_max = 4096; ++ vdisplay_max = 4096; ++ htotal_max = 8192; ++ vtotal_max = 8192; ++ } else { ++ hdisplay_max = 2048; ++ vdisplay_max = 2048; ++ htotal_max = 4096; ++ vtotal_max = 4096; ++ } ++ ++ if (mode->hdisplay > hdisplay_max || ++ mode->hsync_start > htotal_max || ++ mode->hsync_end > htotal_max || ++ mode->htotal > htotal_max) ++ return MODE_H_ILLEGAL; ++ ++ if (mode->vdisplay > vdisplay_max || ++ mode->vsync_start > vtotal_max || ++ mode->vsync_end > vtotal_max || ++ mode->vtotal > vtotal_max) ++ return MODE_V_ILLEGAL; ++ ++ return MODE_OK; ++} ++ ++static const struct drm_mode_config_funcs intel_mode_funcs = { ++ .fb_create = intel_user_framebuffer_create, ++ .get_format_info = intel_get_format_info, ++ .output_poll_changed = intel_fbdev_output_poll_changed, ++ .mode_valid = intel_mode_valid, ++ .atomic_check = intel_atomic_check, ++ .atomic_commit = intel_atomic_commit, ++ .atomic_state_alloc = intel_atomic_state_alloc, ++ .atomic_state_clear = intel_atomic_state_clear, ++ .atomic_state_free = intel_atomic_state_free, ++}; ++ ++/** ++ * intel_init_display_hooks - initialize the display modesetting hooks ++ * @dev_priv: device private ++ */ ++void intel_init_display_hooks(struct drm_i915_private *dev_priv) ++{ ++ intel_init_cdclk_hooks(dev_priv); ++ ++ if (INTEL_GEN(dev_priv) >= 9) { ++ dev_priv->display.get_pipe_config = haswell_get_pipe_config; ++ dev_priv->display.get_initial_plane_config = ++ skylake_get_initial_plane_config; ++ dev_priv->display.crtc_compute_clock = ++ haswell_crtc_compute_clock; ++ dev_priv->display.crtc_enable = haswell_crtc_enable; ++ dev_priv->display.crtc_disable = haswell_crtc_disable; ++ } else if (HAS_DDI(dev_priv)) { ++ dev_priv->display.get_pipe_config = haswell_get_pipe_config; ++ dev_priv->display.get_initial_plane_config = ++ i9xx_get_initial_plane_config; ++ dev_priv->display.crtc_compute_clock = ++ haswell_crtc_compute_clock; ++ dev_priv->display.crtc_enable = haswell_crtc_enable; ++ dev_priv->display.crtc_disable = haswell_crtc_disable; ++ } else if (HAS_PCH_SPLIT(dev_priv)) { ++ dev_priv->display.get_pipe_config = ironlake_get_pipe_config; ++ dev_priv->display.get_initial_plane_config = ++ i9xx_get_initial_plane_config; ++ dev_priv->display.crtc_compute_clock = ++ ironlake_crtc_compute_clock; ++ dev_priv->display.crtc_enable = ironlake_crtc_enable; ++ dev_priv->display.crtc_disable = ironlake_crtc_disable; ++ } else if (IS_CHERRYVIEW(dev_priv)) { ++ dev_priv->display.get_pipe_config = i9xx_get_pipe_config; ++ dev_priv->display.get_initial_plane_config = ++ i9xx_get_initial_plane_config; ++ dev_priv->display.crtc_compute_clock = chv_crtc_compute_clock; ++ dev_priv->display.crtc_enable = valleyview_crtc_enable; ++ dev_priv->display.crtc_disable = i9xx_crtc_disable; ++ } else if (IS_VALLEYVIEW(dev_priv)) { ++ dev_priv->display.get_pipe_config = i9xx_get_pipe_config; ++ dev_priv->display.get_initial_plane_config = ++ i9xx_get_initial_plane_config; ++ dev_priv->display.crtc_compute_clock = vlv_crtc_compute_clock; ++ dev_priv->display.crtc_enable = valleyview_crtc_enable; ++ dev_priv->display.crtc_disable = i9xx_crtc_disable; ++ } else if (IS_G4X(dev_priv)) { ++ dev_priv->display.get_pipe_config = i9xx_get_pipe_config; ++ dev_priv->display.get_initial_plane_config = ++ i9xx_get_initial_plane_config; ++ dev_priv->display.crtc_compute_clock = g4x_crtc_compute_clock; ++ dev_priv->display.crtc_enable = i9xx_crtc_enable; ++ dev_priv->display.crtc_disable = i9xx_crtc_disable; ++ } else if (IS_PINEVIEW(dev_priv)) { ++ dev_priv->display.get_pipe_config = i9xx_get_pipe_config; ++ dev_priv->display.get_initial_plane_config = ++ i9xx_get_initial_plane_config; ++ dev_priv->display.crtc_compute_clock = pnv_crtc_compute_clock; ++ dev_priv->display.crtc_enable = i9xx_crtc_enable; ++ dev_priv->display.crtc_disable = i9xx_crtc_disable; ++ } else if (!IS_GEN(dev_priv, 2)) { ++ dev_priv->display.get_pipe_config = i9xx_get_pipe_config; ++ dev_priv->display.get_initial_plane_config = ++ i9xx_get_initial_plane_config; ++ dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock; ++ dev_priv->display.crtc_enable = i9xx_crtc_enable; ++ dev_priv->display.crtc_disable = i9xx_crtc_disable; ++ } else { ++ dev_priv->display.get_pipe_config = i9xx_get_pipe_config; ++ dev_priv->display.get_initial_plane_config = ++ i9xx_get_initial_plane_config; ++ dev_priv->display.crtc_compute_clock = i8xx_crtc_compute_clock; ++ dev_priv->display.crtc_enable = i9xx_crtc_enable; ++ dev_priv->display.crtc_disable = i9xx_crtc_disable; ++ } ++ ++ if (IS_GEN(dev_priv, 5)) { ++ dev_priv->display.fdi_link_train = ironlake_fdi_link_train; ++ } else if (IS_GEN(dev_priv, 6)) { ++ dev_priv->display.fdi_link_train = gen6_fdi_link_train; ++ } else if (IS_IVYBRIDGE(dev_priv)) { ++ /* FIXME: detect B0+ stepping and use auto training */ ++ dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train; ++ } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { ++ dev_priv->display.fdi_link_train = hsw_fdi_link_train; ++ } ++ ++ if (INTEL_GEN(dev_priv) >= 9) ++ dev_priv->display.update_crtcs = skl_update_crtcs; ++ else ++ dev_priv->display.update_crtcs = intel_update_crtcs; ++} ++ ++/* Disable the VGA plane that we never use */ ++static void i915_disable_vga(struct drm_i915_private *dev_priv) ++{ ++ struct pci_dev *pdev = dev_priv->drm.pdev; ++ u8 sr1; ++ i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv); ++ ++ /* WaEnableVGAAccessThroughIOPort:ctg,elk,ilk,snb,ivb,vlv,hsw */ ++ vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO); ++ outb(SR01, VGA_SR_INDEX); ++ sr1 = inb(VGA_SR_DATA); ++ outb(sr1 | 1<<5, VGA_SR_DATA); ++ vga_put(pdev, VGA_RSRC_LEGACY_IO); ++ udelay(300); ++ ++ I915_WRITE(vga_reg, VGA_DISP_DISABLE); ++ POSTING_READ(vga_reg); ++} ++ ++void intel_modeset_init_hw(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ ++ intel_update_cdclk(dev_priv); ++ intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK"); ++ dev_priv->cdclk.logical = dev_priv->cdclk.actual = dev_priv->cdclk.hw; ++} ++ ++/* ++ * Calculate what we think the watermarks should be for the state we've read ++ * out of the hardware and then immediately program those watermarks so that ++ * we ensure the hardware settings match our internal state. ++ * ++ * We can calculate what we think WM's should be by creating a duplicate of the ++ * current state (which was constructed during hardware readout) and running it ++ * through the atomic check code to calculate new watermark values in the ++ * state object. ++ */ ++static void sanitize_watermarks(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct drm_atomic_state *state; ++ struct intel_atomic_state *intel_state; ++ struct drm_crtc *crtc; ++ struct drm_crtc_state *cstate; ++ struct drm_modeset_acquire_ctx ctx; ++ int ret; ++ int i; ++ ++ /* Only supported on platforms that use atomic watermark design */ ++ if (!dev_priv->display.optimize_watermarks) ++ return; ++ ++ /* ++ * We need to hold connection_mutex before calling duplicate_state so ++ * that the connector loop is protected. ++ */ ++ drm_modeset_acquire_init(&ctx, 0); ++retry: ++ ret = drm_modeset_lock_all_ctx(dev, &ctx); ++ if (ret == -EDEADLK) { ++ drm_modeset_backoff(&ctx); ++ goto retry; ++ } else if (WARN_ON(ret)) { ++ goto fail; ++ } ++ ++ state = drm_atomic_helper_duplicate_state(dev, &ctx); ++ if (WARN_ON(IS_ERR(state))) ++ goto fail; ++ ++ intel_state = to_intel_atomic_state(state); ++ ++ /* ++ * Hardware readout is the only time we don't want to calculate ++ * intermediate watermarks (since we don't trust the current ++ * watermarks). ++ */ ++ if (!HAS_GMCH(dev_priv)) ++ intel_state->skip_intermediate_wm = true; ++ ++ ret = intel_atomic_check(dev, state); ++ if (ret) { ++ /* ++ * If we fail here, it means that the hardware appears to be ++ * programmed in a way that shouldn't be possible, given our ++ * understanding of watermark requirements. This might mean a ++ * mistake in the hardware readout code or a mistake in the ++ * watermark calculations for a given platform. Raise a WARN ++ * so that this is noticeable. ++ * ++ * If this actually happens, we'll have to just leave the ++ * BIOS-programmed watermarks untouched and hope for the best. ++ */ ++ WARN(true, "Could not determine valid watermarks for inherited state\n"); ++ goto put_state; ++ } ++ ++ /* Write calculated watermark values back */ ++ for_each_new_crtc_in_state(state, crtc, cstate, i) { ++ struct intel_crtc_state *cs = to_intel_crtc_state(cstate); ++ ++ cs->wm.need_postvbl_update = true; ++ dev_priv->display.optimize_watermarks(intel_state, cs); ++ ++ to_intel_crtc_state(crtc->state)->wm = cs->wm; ++ } ++ ++put_state: ++ drm_atomic_state_put(state); ++fail: ++ drm_modeset_drop_locks(&ctx); ++ drm_modeset_acquire_fini(&ctx); ++} ++ ++static void intel_update_fdi_pll_freq(struct drm_i915_private *dev_priv) ++{ ++ if (IS_GEN(dev_priv, 5)) { ++ u32 fdi_pll_clk = ++ I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK; ++ ++ dev_priv->fdi_pll_freq = (fdi_pll_clk + 2) * 10000; ++ } else if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv)) { ++ dev_priv->fdi_pll_freq = 270000; ++ } else { ++ return; ++ } ++ ++ DRM_DEBUG_DRIVER("FDI PLL freq=%d\n", dev_priv->fdi_pll_freq); ++} ++ ++static int intel_initial_commit(struct drm_device *dev) ++{ ++ struct drm_atomic_state *state = NULL; ++ struct drm_modeset_acquire_ctx ctx; ++ struct drm_crtc *crtc; ++ struct drm_crtc_state *crtc_state; ++ int ret = 0; ++ ++ state = drm_atomic_state_alloc(dev); ++ if (!state) ++ return -ENOMEM; ++ ++ drm_modeset_acquire_init(&ctx, 0); ++ ++retry: ++ state->acquire_ctx = &ctx; ++ ++ drm_for_each_crtc(crtc, dev) { ++ crtc_state = drm_atomic_get_crtc_state(state, crtc); ++ if (IS_ERR(crtc_state)) { ++ ret = PTR_ERR(crtc_state); ++ goto out; ++ } ++ ++ if (crtc_state->active) { ++ ret = drm_atomic_add_affected_planes(state, crtc); ++ if (ret) ++ goto out; ++ ++ /* ++ * FIXME hack to force a LUT update to avoid the ++ * plane update forcing the pipe gamma on without ++ * having a proper LUT loaded. Remove once we ++ * have readout for pipe gamma enable. ++ */ ++ crtc_state->color_mgmt_changed = true; ++ } ++ } ++ ++ ret = drm_atomic_commit(state); ++ ++out: ++ if (ret == -EDEADLK) { ++ drm_atomic_state_clear(state); ++ drm_modeset_backoff(&ctx); ++ goto retry; ++ } ++ ++ drm_atomic_state_put(state); ++ ++ drm_modeset_drop_locks(&ctx); ++ drm_modeset_acquire_fini(&ctx); ++ ++ return ret; ++} ++ ++int intel_modeset_init(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct i915_ggtt *ggtt = &dev_priv->ggtt; ++ enum pipe pipe; ++ struct intel_crtc *crtc; ++ int ret; ++ ++ dev_priv->modeset_wq = alloc_ordered_workqueue("i915_modeset", 0); ++ ++ drm_mode_config_init(dev); ++ ++ dev->mode_config.min_width = 0; ++ dev->mode_config.min_height = 0; ++ ++ dev->mode_config.preferred_depth = 24; ++ dev->mode_config.prefer_shadow = 1; ++ ++ dev->mode_config.allow_fb_modifiers = true; ++ ++ dev->mode_config.funcs = &intel_mode_funcs; ++ ++ init_llist_head(&dev_priv->atomic_helper.free_list); ++ INIT_WORK(&dev_priv->atomic_helper.free_work, ++ intel_atomic_helper_free_state_worker); ++ ++ intel_init_quirks(dev_priv); ++ ++ intel_fbc_init(dev_priv); ++ ++ intel_init_pm(dev_priv); ++ ++ /* ++ * There may be no VBT; and if the BIOS enabled SSC we can ++ * just keep using it to avoid unnecessary flicker. Whereas if the ++ * BIOS isn't using it, don't assume it will work even if the VBT ++ * indicates as much. ++ */ ++ if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) { ++ bool bios_lvds_use_ssc = !!(I915_READ(PCH_DREF_CONTROL) & ++ DREF_SSC1_ENABLE); ++ ++ if (dev_priv->vbt.lvds_use_ssc != bios_lvds_use_ssc) { ++ DRM_DEBUG_KMS("SSC %sabled by BIOS, overriding VBT which says %sabled\n", ++ bios_lvds_use_ssc ? "en" : "dis", ++ dev_priv->vbt.lvds_use_ssc ? "en" : "dis"); ++ dev_priv->vbt.lvds_use_ssc = bios_lvds_use_ssc; ++ } ++ } ++ ++ /* maximum framebuffer dimensions */ ++ if (IS_GEN(dev_priv, 2)) { ++ dev->mode_config.max_width = 2048; ++ dev->mode_config.max_height = 2048; ++ } else if (IS_GEN(dev_priv, 3)) { ++ dev->mode_config.max_width = 4096; ++ dev->mode_config.max_height = 4096; ++ } else { ++ dev->mode_config.max_width = 8192; ++ dev->mode_config.max_height = 8192; ++ } ++ ++ if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) { ++ dev->mode_config.cursor_width = IS_I845G(dev_priv) ? 64 : 512; ++ dev->mode_config.cursor_height = 1023; ++ } else if (IS_GEN(dev_priv, 2)) { ++ dev->mode_config.cursor_width = 64; ++ dev->mode_config.cursor_height = 64; ++ } else { ++ dev->mode_config.cursor_width = 256; ++ dev->mode_config.cursor_height = 256; ++ } ++ ++ dev->mode_config.fb_base = ggtt->gmadr.start; ++ ++ DRM_DEBUG_KMS("%d display pipe%s available.\n", ++ INTEL_INFO(dev_priv)->num_pipes, ++ INTEL_INFO(dev_priv)->num_pipes > 1 ? "s" : ""); ++ ++ for_each_pipe(dev_priv, pipe) { ++ ret = intel_crtc_init(dev_priv, pipe); ++ if (ret) { ++ drm_mode_config_cleanup(dev); ++ return ret; ++ } ++ } ++ ++ intel_shared_dpll_init(dev); ++ intel_update_fdi_pll_freq(dev_priv); ++ ++ intel_update_czclk(dev_priv); ++ intel_modeset_init_hw(dev); ++ ++ intel_hdcp_component_init(dev_priv); ++ ++ if (dev_priv->max_cdclk_freq == 0) ++ intel_update_max_cdclk(dev_priv); ++ ++ /* Just disable it once at startup */ ++ i915_disable_vga(dev_priv); ++ intel_setup_outputs(dev_priv); ++ ++ drm_modeset_lock_all(dev); ++ intel_modeset_setup_hw_state(dev, dev->mode_config.acquire_ctx); ++ drm_modeset_unlock_all(dev); ++ ++ for_each_intel_crtc(dev, crtc) { ++ struct intel_initial_plane_config plane_config = {}; ++ ++ if (!crtc->active) ++ continue; ++ ++ /* ++ * Note that reserving the BIOS fb up front prevents us ++ * from stuffing other stolen allocations like the ring ++ * on top. This prevents some ugliness at boot time, and ++ * can even allow for smooth boot transitions if the BIOS ++ * fb is large enough for the active pipe configuration. ++ */ ++ dev_priv->display.get_initial_plane_config(crtc, ++ &plane_config); ++ ++ /* ++ * If the fb is shared between multiple heads, we'll ++ * just get the first one. ++ */ ++ intel_find_initial_plane_obj(crtc, &plane_config); ++ } ++ ++ /* ++ * Make sure hardware watermarks really match the state we read out. ++ * Note that we need to do this after reconstructing the BIOS fb's ++ * since the watermark calculation done here will use pstate->fb. ++ */ ++ if (!HAS_GMCH(dev_priv)) ++ sanitize_watermarks(dev); ++ ++ /* ++ * Force all active planes to recompute their states. So that on ++ * mode_setcrtc after probe, all the intel_plane_state variables ++ * are already calculated and there is no assert_plane warnings ++ * during bootup. ++ */ ++ ret = intel_initial_commit(dev); ++ if (ret) ++ DRM_DEBUG_KMS("Initial commit in probe failed.\n"); ++ ++ return 0; ++} ++ ++void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe) ++{ ++ struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); ++ /* 640x480@60Hz, ~25175 kHz */ ++ struct dpll clock = { ++ .m1 = 18, ++ .m2 = 7, ++ .p1 = 13, ++ .p2 = 4, ++ .n = 2, ++ }; ++ u32 dpll, fp; ++ int i; ++ ++ WARN_ON(i9xx_calc_dpll_params(48000, &clock) != 25154); ++ ++ DRM_DEBUG_KMS("enabling pipe %c due to force quirk (vco=%d dot=%d)\n", ++ pipe_name(pipe), clock.vco, clock.dot); ++ ++ fp = i9xx_dpll_compute_fp(&clock); ++ dpll = DPLL_DVO_2X_MODE | ++ DPLL_VGA_MODE_DIS | ++ ((clock.p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT) | ++ PLL_P2_DIVIDE_BY_4 | ++ PLL_REF_INPUT_DREFCLK | ++ DPLL_VCO_ENABLE; ++ ++ I915_WRITE(FP0(pipe), fp); ++ I915_WRITE(FP1(pipe), fp); ++ ++ I915_WRITE(HTOTAL(pipe), (640 - 1) | ((800 - 1) << 16)); ++ I915_WRITE(HBLANK(pipe), (640 - 1) | ((800 - 1) << 16)); ++ I915_WRITE(HSYNC(pipe), (656 - 1) | ((752 - 1) << 16)); ++ I915_WRITE(VTOTAL(pipe), (480 - 1) | ((525 - 1) << 16)); ++ I915_WRITE(VBLANK(pipe), (480 - 1) | ((525 - 1) << 16)); ++ I915_WRITE(VSYNC(pipe), (490 - 1) | ((492 - 1) << 16)); ++ I915_WRITE(PIPESRC(pipe), ((640 - 1) << 16) | (480 - 1)); ++ ++ /* ++ * Apparently we need to have VGA mode enabled prior to changing ++ * the P1/P2 dividers. Otherwise the DPLL will keep using the old ++ * dividers, even though the register value does change. ++ */ ++ I915_WRITE(DPLL(pipe), dpll & ~DPLL_VGA_MODE_DIS); ++ I915_WRITE(DPLL(pipe), dpll); ++ ++ /* Wait for the clocks to stabilize. */ ++ POSTING_READ(DPLL(pipe)); ++ udelay(150); ++ ++ /* The pixel multiplier can only be updated once the ++ * DPLL is enabled and the clocks are stable. ++ * ++ * So write it again. ++ */ ++ I915_WRITE(DPLL(pipe), dpll); ++ ++ /* We do this three times for luck */ ++ for (i = 0; i < 3 ; i++) { ++ I915_WRITE(DPLL(pipe), dpll); ++ POSTING_READ(DPLL(pipe)); ++ udelay(150); /* wait for warmup */ ++ } ++ ++ I915_WRITE(PIPECONF(pipe), PIPECONF_ENABLE | PIPECONF_PROGRESSIVE); ++ POSTING_READ(PIPECONF(pipe)); ++ ++ intel_wait_for_pipe_scanline_moving(crtc); ++} ++ ++void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe) ++{ ++ struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); ++ ++ DRM_DEBUG_KMS("disabling pipe %c due to force quirk\n", ++ pipe_name(pipe)); ++ ++ WARN_ON(I915_READ(DSPCNTR(PLANE_A)) & DISPLAY_PLANE_ENABLE); ++ WARN_ON(I915_READ(DSPCNTR(PLANE_B)) & DISPLAY_PLANE_ENABLE); ++ WARN_ON(I915_READ(DSPCNTR(PLANE_C)) & DISPLAY_PLANE_ENABLE); ++ WARN_ON(I915_READ(CURCNTR(PIPE_A)) & MCURSOR_MODE); ++ WARN_ON(I915_READ(CURCNTR(PIPE_B)) & MCURSOR_MODE); ++ ++ I915_WRITE(PIPECONF(pipe), 0); ++ POSTING_READ(PIPECONF(pipe)); ++ ++ intel_wait_for_pipe_scanline_stopped(crtc); ++ ++ I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS); ++ POSTING_READ(DPLL(pipe)); ++} ++ ++static void ++intel_sanitize_plane_mapping(struct drm_i915_private *dev_priv) ++{ ++ struct intel_crtc *crtc; ++ ++ if (INTEL_GEN(dev_priv) >= 4) ++ return; ++ ++ for_each_intel_crtc(&dev_priv->drm, crtc) { ++ struct intel_plane *plane = ++ to_intel_plane(crtc->base.primary); ++ struct intel_crtc *plane_crtc; ++ enum pipe pipe; ++ ++ if (!plane->get_hw_state(plane, &pipe)) ++ continue; ++ ++ if (pipe == crtc->pipe) ++ continue; ++ ++ DRM_DEBUG_KMS("[PLANE:%d:%s] attached to the wrong pipe, disabling plane\n", ++ plane->base.base.id, plane->base.name); ++ ++ plane_crtc = intel_get_crtc_for_pipe(dev_priv, pipe); ++ intel_plane_disable_noatomic(plane_crtc, plane); ++ } ++} ++ ++static bool intel_crtc_has_encoders(struct intel_crtc *crtc) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct intel_encoder *encoder; ++ ++ for_each_encoder_on_crtc(dev, &crtc->base, encoder) ++ return true; ++ ++ return false; ++} ++ ++static struct intel_connector *intel_encoder_find_connector(struct intel_encoder *encoder) ++{ ++ struct drm_device *dev = encoder->base.dev; ++ struct intel_connector *connector; ++ ++ for_each_connector_on_encoder(dev, &encoder->base, connector) ++ return connector; ++ ++ return NULL; ++} ++ ++static bool has_pch_trancoder(struct drm_i915_private *dev_priv, ++ enum pipe pch_transcoder) ++{ ++ return HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) || ++ (HAS_PCH_LPT_H(dev_priv) && pch_transcoder == PIPE_A); ++} ++ ++static void intel_sanitize_crtc(struct intel_crtc *crtc, ++ struct drm_modeset_acquire_ctx *ctx) ++{ ++ struct drm_device *dev = crtc->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state); ++ enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; ++ ++ /* Clear any frame start delays used for debugging left by the BIOS */ ++ if (crtc->active && !transcoder_is_dsi(cpu_transcoder)) { ++ i915_reg_t reg = PIPECONF(cpu_transcoder); ++ ++ I915_WRITE(reg, ++ I915_READ(reg) & ~PIPECONF_FRAME_START_DELAY_MASK); ++ } ++ ++ if (crtc_state->base.active) { ++ struct intel_plane *plane; ++ ++ /* Disable everything but the primary plane */ ++ for_each_intel_plane_on_crtc(dev, crtc, plane) { ++ const struct intel_plane_state *plane_state = ++ to_intel_plane_state(plane->base.state); ++ ++ if (plane_state->base.visible && ++ plane->base.type != DRM_PLANE_TYPE_PRIMARY) ++ intel_plane_disable_noatomic(crtc, plane); ++ } ++ ++ /* ++ * Disable any background color set by the BIOS, but enable the ++ * gamma and CSC to match how we program our planes. ++ */ ++ if (INTEL_GEN(dev_priv) >= 9) ++ I915_WRITE(SKL_BOTTOM_COLOR(crtc->pipe), ++ SKL_BOTTOM_COLOR_GAMMA_ENABLE | ++ SKL_BOTTOM_COLOR_CSC_ENABLE); ++ } ++ ++ /* Adjust the state of the output pipe according to whether we ++ * have active connectors/encoders. */ ++ if (crtc_state->base.active && !intel_crtc_has_encoders(crtc)) ++ intel_crtc_disable_noatomic(&crtc->base, ctx); ++ ++ if (crtc_state->base.active || HAS_GMCH(dev_priv)) { ++ /* ++ * We start out with underrun reporting disabled to avoid races. ++ * For correct bookkeeping mark this on active crtcs. ++ * ++ * Also on gmch platforms we dont have any hardware bits to ++ * disable the underrun reporting. Which means we need to start ++ * out with underrun reporting disabled also on inactive pipes, ++ * since otherwise we'll complain about the garbage we read when ++ * e.g. coming up after runtime pm. ++ * ++ * No protection against concurrent access is required - at ++ * worst a fifo underrun happens which also sets this to false. ++ */ ++ crtc->cpu_fifo_underrun_disabled = true; ++ /* ++ * We track the PCH trancoder underrun reporting state ++ * within the crtc. With crtc for pipe A housing the underrun ++ * reporting state for PCH transcoder A, crtc for pipe B housing ++ * it for PCH transcoder B, etc. LPT-H has only PCH transcoder A, ++ * and marking underrun reporting as disabled for the non-existing ++ * PCH transcoders B and C would prevent enabling the south ++ * error interrupt (see cpt_can_enable_serr_int()). ++ */ ++ if (has_pch_trancoder(dev_priv, crtc->pipe)) ++ crtc->pch_fifo_underrun_disabled = true; ++ } ++} ++ ++static bool has_bogus_dpll_config(const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); ++ ++ /* ++ * Some SNB BIOSen (eg. ASUS K53SV) are known to misprogram ++ * the hardware when a high res displays plugged in. DPLL P ++ * divider is zero, and the pipe timings are bonkers. We'll ++ * try to disable everything in that case. ++ * ++ * FIXME would be nice to be able to sanitize this state ++ * without several WARNs, but for now let's take the easy ++ * road. ++ */ ++ return IS_GEN(dev_priv, 6) && ++ crtc_state->base.active && ++ crtc_state->shared_dpll && ++ crtc_state->port_clock == 0; ++} ++ ++static void intel_sanitize_encoder(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_connector *connector; ++ struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); ++ struct intel_crtc_state *crtc_state = crtc ? ++ to_intel_crtc_state(crtc->base.state) : NULL; ++ ++ /* We need to check both for a crtc link (meaning that the ++ * encoder is active and trying to read from a pipe) and the ++ * pipe itself being active. */ ++ bool has_active_crtc = crtc_state && ++ crtc_state->base.active; ++ ++ if (crtc_state && has_bogus_dpll_config(crtc_state)) { ++ DRM_DEBUG_KMS("BIOS has misprogrammed the hardware. Disabling pipe %c\n", ++ pipe_name(crtc->pipe)); ++ has_active_crtc = false; ++ } ++ ++ connector = intel_encoder_find_connector(encoder); ++ if (connector && !has_active_crtc) { ++ DRM_DEBUG_KMS("[ENCODER:%d:%s] has active connectors but no active pipe!\n", ++ encoder->base.base.id, ++ encoder->base.name); ++ ++ /* Connector is active, but has no active pipe. This is ++ * fallout from our resume register restoring. Disable ++ * the encoder manually again. */ ++ if (crtc_state) { ++ struct drm_encoder *best_encoder; ++ ++ DRM_DEBUG_KMS("[ENCODER:%d:%s] manually disabled\n", ++ encoder->base.base.id, ++ encoder->base.name); ++ ++ /* avoid oopsing in case the hooks consult best_encoder */ ++ best_encoder = connector->base.state->best_encoder; ++ connector->base.state->best_encoder = &encoder->base; ++ ++ if (encoder->disable) ++ encoder->disable(encoder, crtc_state, ++ connector->base.state); ++ if (encoder->post_disable) ++ encoder->post_disable(encoder, crtc_state, ++ connector->base.state); ++ ++ connector->base.state->best_encoder = best_encoder; ++ } ++ encoder->base.crtc = NULL; ++ ++ /* Inconsistent output/port/pipe state happens presumably due to ++ * a bug in one of the get_hw_state functions. Or someplace else ++ * in our code, like the register restore mess on resume. Clamp ++ * things to off as a safer default. */ ++ ++ connector->base.dpms = DRM_MODE_DPMS_OFF; ++ connector->base.encoder = NULL; ++ } ++ ++ /* notify opregion of the sanitized encoder state */ ++ intel_opregion_notify_encoder(encoder, connector && has_active_crtc); ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ icl_sanitize_encoder_pll_mapping(encoder); ++} ++ ++void i915_redisable_vga_power_on(struct drm_i915_private *dev_priv) ++{ ++ i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv); ++ ++ if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) { ++ DRM_DEBUG_KMS("Something enabled VGA plane, disabling it\n"); ++ i915_disable_vga(dev_priv); ++ } ++} ++ ++void i915_redisable_vga(struct drm_i915_private *dev_priv) ++{ ++ intel_wakeref_t wakeref; ++ ++ /* ++ * This function can be called both from intel_modeset_setup_hw_state or ++ * at a very early point in our resume sequence, where the power well ++ * structures are not yet restored. Since this function is at a very ++ * paranoid "someone might have enabled VGA while we were not looking" ++ * level, just check if the power well is enabled instead of trying to ++ * follow the "don't touch the power well if we don't need it" policy ++ * the rest of the driver uses. ++ */ ++ wakeref = intel_display_power_get_if_enabled(dev_priv, ++ POWER_DOMAIN_VGA); ++ if (!wakeref) ++ return; ++ ++ i915_redisable_vga_power_on(dev_priv); ++ ++ intel_display_power_put(dev_priv, POWER_DOMAIN_VGA, wakeref); ++} ++ ++/* FIXME read out full plane state for all planes */ ++static void readout_plane_state(struct drm_i915_private *dev_priv) ++{ ++ struct intel_plane *plane; ++ struct intel_crtc *crtc; ++ ++ for_each_intel_plane(&dev_priv->drm, plane) { ++ struct intel_plane_state *plane_state = ++ to_intel_plane_state(plane->base.state); ++ struct intel_crtc_state *crtc_state; ++ enum pipe pipe = PIPE_A; ++ bool visible; ++ ++ visible = plane->get_hw_state(plane, &pipe); ++ ++ crtc = intel_get_crtc_for_pipe(dev_priv, pipe); ++ crtc_state = to_intel_crtc_state(crtc->base.state); ++ ++ intel_set_plane_visible(crtc_state, plane_state, visible); ++ ++ DRM_DEBUG_KMS("[PLANE:%d:%s] hw state readout: %s, pipe %c\n", ++ plane->base.base.id, plane->base.name, ++ enableddisabled(visible), pipe_name(pipe)); ++ } ++ ++ for_each_intel_crtc(&dev_priv->drm, crtc) { ++ struct intel_crtc_state *crtc_state = ++ to_intel_crtc_state(crtc->base.state); ++ ++ fixup_active_planes(crtc_state); ++ } ++} ++ ++static void intel_modeset_readout_hw_state(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ enum pipe pipe; ++ struct intel_crtc *crtc; ++ struct intel_encoder *encoder; ++ struct intel_connector *connector; ++ struct drm_connector_list_iter conn_iter; ++ int i; ++ ++ dev_priv->active_crtcs = 0; ++ ++ for_each_intel_crtc(dev, crtc) { ++ struct intel_crtc_state *crtc_state = ++ to_intel_crtc_state(crtc->base.state); ++ ++ __drm_atomic_helper_crtc_destroy_state(&crtc_state->base); ++ memset(crtc_state, 0, sizeof(*crtc_state)); ++ crtc_state->base.crtc = &crtc->base; ++ ++ crtc_state->base.active = crtc_state->base.enable = ++ dev_priv->display.get_pipe_config(crtc, crtc_state); ++ ++ crtc->base.enabled = crtc_state->base.enable; ++ crtc->active = crtc_state->base.active; ++ ++ if (crtc_state->base.active) ++ dev_priv->active_crtcs |= 1 << crtc->pipe; ++ ++ DRM_DEBUG_KMS("[CRTC:%d:%s] hw state readout: %s\n", ++ crtc->base.base.id, crtc->base.name, ++ enableddisabled(crtc_state->base.active)); ++ } ++ ++ readout_plane_state(dev_priv); ++ ++ for (i = 0; i < dev_priv->num_shared_dpll; i++) { ++ struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i]; ++ ++ pll->on = pll->info->funcs->get_hw_state(dev_priv, pll, ++ &pll->state.hw_state); ++ pll->state.crtc_mask = 0; ++ for_each_intel_crtc(dev, crtc) { ++ struct intel_crtc_state *crtc_state = ++ to_intel_crtc_state(crtc->base.state); ++ ++ if (crtc_state->base.active && ++ crtc_state->shared_dpll == pll) ++ pll->state.crtc_mask |= 1 << crtc->pipe; ++ } ++ pll->active_mask = pll->state.crtc_mask; ++ ++ DRM_DEBUG_KMS("%s hw state readout: crtc_mask 0x%08x, on %i\n", ++ pll->info->name, pll->state.crtc_mask, pll->on); ++ } ++ ++ for_each_intel_encoder(dev, encoder) { ++ pipe = 0; ++ ++ if (encoder->get_hw_state(encoder, &pipe)) { ++ struct intel_crtc_state *crtc_state; ++ ++ crtc = intel_get_crtc_for_pipe(dev_priv, pipe); ++ crtc_state = to_intel_crtc_state(crtc->base.state); ++ ++ encoder->base.crtc = &crtc->base; ++ encoder->get_config(encoder, crtc_state); ++ } else { ++ encoder->base.crtc = NULL; ++ } ++ ++ DRM_DEBUG_KMS("[ENCODER:%d:%s] hw state readout: %s, pipe %c\n", ++ encoder->base.base.id, encoder->base.name, ++ enableddisabled(encoder->base.crtc), ++ pipe_name(pipe)); ++ } ++ ++ drm_connector_list_iter_begin(dev, &conn_iter); ++ for_each_intel_connector_iter(connector, &conn_iter) { ++ if (connector->get_hw_state(connector)) { ++ connector->base.dpms = DRM_MODE_DPMS_ON; ++ ++ encoder = connector->encoder; ++ connector->base.encoder = &encoder->base; ++ ++ if (encoder->base.crtc && ++ encoder->base.crtc->state->active) { ++ /* ++ * This has to be done during hardware readout ++ * because anything calling .crtc_disable may ++ * rely on the connector_mask being accurate. ++ */ ++ encoder->base.crtc->state->connector_mask |= ++ drm_connector_mask(&connector->base); ++ encoder->base.crtc->state->encoder_mask |= ++ drm_encoder_mask(&encoder->base); ++ } ++ ++ } else { ++ connector->base.dpms = DRM_MODE_DPMS_OFF; ++ connector->base.encoder = NULL; ++ } ++ DRM_DEBUG_KMS("[CONNECTOR:%d:%s] hw state readout: %s\n", ++ connector->base.base.id, connector->base.name, ++ enableddisabled(connector->base.encoder)); ++ } ++ drm_connector_list_iter_end(&conn_iter); ++ ++ for_each_intel_crtc(dev, crtc) { ++ struct intel_crtc_state *crtc_state = ++ to_intel_crtc_state(crtc->base.state); ++ int min_cdclk = 0; ++ ++ memset(&crtc->base.mode, 0, sizeof(crtc->base.mode)); ++ if (crtc_state->base.active) { ++ intel_mode_from_pipe_config(&crtc->base.mode, crtc_state); ++ crtc->base.mode.hdisplay = crtc_state->pipe_src_w; ++ crtc->base.mode.vdisplay = crtc_state->pipe_src_h; ++ intel_mode_from_pipe_config(&crtc_state->base.adjusted_mode, crtc_state); ++ WARN_ON(drm_atomic_set_mode_for_crtc(crtc->base.state, &crtc->base.mode)); ++ ++ /* ++ * The initial mode needs to be set in order to keep ++ * the atomic core happy. It wants a valid mode if the ++ * crtc's enabled, so we do the above call. ++ * ++ * But we don't set all the derived state fully, hence ++ * set a flag to indicate that a full recalculation is ++ * needed on the next commit. ++ */ ++ crtc_state->base.mode.private_flags = I915_MODE_FLAG_INHERITED; ++ ++ intel_crtc_compute_pixel_rate(crtc_state); ++ ++ if (dev_priv->display.modeset_calc_cdclk) { ++ min_cdclk = intel_crtc_compute_min_cdclk(crtc_state); ++ if (WARN_ON(min_cdclk < 0)) ++ min_cdclk = 0; ++ } ++ ++ drm_calc_timestamping_constants(&crtc->base, ++ &crtc_state->base.adjusted_mode); ++ update_scanline_offset(crtc_state); ++ } ++ ++ dev_priv->min_cdclk[crtc->pipe] = min_cdclk; ++ dev_priv->min_voltage_level[crtc->pipe] = ++ crtc_state->min_voltage_level; ++ ++ intel_pipe_config_sanity_check(dev_priv, crtc_state); ++ } ++} ++ ++static void ++get_encoder_power_domains(struct drm_i915_private *dev_priv) ++{ ++ struct intel_encoder *encoder; ++ ++ for_each_intel_encoder(&dev_priv->drm, encoder) { ++ struct intel_crtc_state *crtc_state; ++ ++ if (!encoder->get_power_domains) ++ continue; ++ ++ /* ++ * MST-primary and inactive encoders don't have a crtc state ++ * and neither of these require any power domain references. ++ */ ++ if (!encoder->base.crtc) ++ continue; ++ ++ crtc_state = to_intel_crtc_state(encoder->base.crtc->state); ++ encoder->get_power_domains(encoder, crtc_state); ++ } ++} ++ ++static void intel_early_display_was(struct drm_i915_private *dev_priv) ++{ ++ /* Display WA #1185 WaDisableDARBFClkGating:cnl,glk */ ++ if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv)) ++ I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) | ++ DARBF_GATING_DIS); ++ ++ if (IS_HASWELL(dev_priv)) { ++ /* ++ * WaRsPkgCStateDisplayPMReq:hsw ++ * System hang if this isn't done before disabling all planes! ++ */ ++ I915_WRITE(CHICKEN_PAR1_1, ++ I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES); ++ } ++} ++ ++static void ibx_sanitize_pch_hdmi_port(struct drm_i915_private *dev_priv, ++ enum port port, i915_reg_t hdmi_reg) ++{ ++ u32 val = I915_READ(hdmi_reg); ++ ++ if (val & SDVO_ENABLE || ++ (val & SDVO_PIPE_SEL_MASK) == SDVO_PIPE_SEL(PIPE_A)) ++ return; ++ ++ DRM_DEBUG_KMS("Sanitizing transcoder select for HDMI %c\n", ++ port_name(port)); ++ ++ val &= ~SDVO_PIPE_SEL_MASK; ++ val |= SDVO_PIPE_SEL(PIPE_A); ++ ++ I915_WRITE(hdmi_reg, val); ++} ++ ++static void ibx_sanitize_pch_dp_port(struct drm_i915_private *dev_priv, ++ enum port port, i915_reg_t dp_reg) ++{ ++ u32 val = I915_READ(dp_reg); ++ ++ if (val & DP_PORT_EN || ++ (val & DP_PIPE_SEL_MASK) == DP_PIPE_SEL(PIPE_A)) ++ return; ++ ++ DRM_DEBUG_KMS("Sanitizing transcoder select for DP %c\n", ++ port_name(port)); ++ ++ val &= ~DP_PIPE_SEL_MASK; ++ val |= DP_PIPE_SEL(PIPE_A); ++ ++ I915_WRITE(dp_reg, val); ++} ++ ++static void ibx_sanitize_pch_ports(struct drm_i915_private *dev_priv) ++{ ++ /* ++ * The BIOS may select transcoder B on some of the PCH ++ * ports even it doesn't enable the port. This would trip ++ * assert_pch_dp_disabled() and assert_pch_hdmi_disabled(). ++ * Sanitize the transcoder select bits to prevent that. We ++ * assume that the BIOS never actually enabled the port, ++ * because if it did we'd actually have to toggle the port ++ * on and back off to make the transcoder A select stick ++ * (see. intel_dp_link_down(), intel_disable_hdmi(), ++ * intel_disable_sdvo()). ++ */ ++ ibx_sanitize_pch_dp_port(dev_priv, PORT_B, PCH_DP_B); ++ ibx_sanitize_pch_dp_port(dev_priv, PORT_C, PCH_DP_C); ++ ibx_sanitize_pch_dp_port(dev_priv, PORT_D, PCH_DP_D); ++ ++ /* PCH SDVOB multiplex with HDMIB */ ++ ibx_sanitize_pch_hdmi_port(dev_priv, PORT_B, PCH_HDMIB); ++ ibx_sanitize_pch_hdmi_port(dev_priv, PORT_C, PCH_HDMIC); ++ ibx_sanitize_pch_hdmi_port(dev_priv, PORT_D, PCH_HDMID); ++} ++ ++/* Scan out the current hw modeset state, ++ * and sanitizes it to the current state ++ */ ++static void ++intel_modeset_setup_hw_state(struct drm_device *dev, ++ struct drm_modeset_acquire_ctx *ctx) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_crtc_state *crtc_state; ++ struct intel_encoder *encoder; ++ struct intel_crtc *crtc; ++ intel_wakeref_t wakeref; ++ int i; ++ ++ wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); ++ ++ intel_early_display_was(dev_priv); ++ intel_modeset_readout_hw_state(dev); ++ ++ /* HW state is read out, now we need to sanitize this mess. */ ++ get_encoder_power_domains(dev_priv); ++ ++ if (HAS_PCH_IBX(dev_priv)) ++ ibx_sanitize_pch_ports(dev_priv); ++ ++ /* ++ * intel_sanitize_plane_mapping() may need to do vblank ++ * waits, so we need vblank interrupts restored beforehand. ++ */ ++ for_each_intel_crtc(&dev_priv->drm, crtc) { ++ crtc_state = to_intel_crtc_state(crtc->base.state); ++ ++ drm_crtc_vblank_reset(&crtc->base); ++ ++ if (crtc_state->base.active) ++ intel_crtc_vblank_on(crtc_state); ++ } ++ ++ intel_sanitize_plane_mapping(dev_priv); ++ ++ for_each_intel_encoder(dev, encoder) ++ intel_sanitize_encoder(encoder); ++ ++ for_each_intel_crtc(&dev_priv->drm, crtc) { ++ crtc_state = to_intel_crtc_state(crtc->base.state); ++ intel_sanitize_crtc(crtc, ctx); ++ intel_dump_pipe_config(crtc, crtc_state, ++ "[setup_hw_state]"); ++ } ++ ++ intel_modeset_update_connector_atomic_state(dev); ++ ++ for (i = 0; i < dev_priv->num_shared_dpll; i++) { ++ struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i]; ++ ++ if (!pll->on || pll->active_mask) ++ continue; ++ ++ DRM_DEBUG_KMS("%s enabled but not in use, disabling\n", ++ pll->info->name); ++ ++ pll->info->funcs->disable(dev_priv, pll); ++ pll->on = false; ++ } ++ ++ if (IS_G4X(dev_priv)) { ++ g4x_wm_get_hw_state(dev_priv); ++ g4x_wm_sanitize(dev_priv); ++ } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { ++ vlv_wm_get_hw_state(dev_priv); ++ vlv_wm_sanitize(dev_priv); ++ } else if (INTEL_GEN(dev_priv) >= 9) { ++ skl_wm_get_hw_state(dev_priv); ++ } else if (HAS_PCH_SPLIT(dev_priv)) { ++ ilk_wm_get_hw_state(dev_priv); ++ } ++ ++ for_each_intel_crtc(dev, crtc) { ++ u64 put_domains; ++ ++ crtc_state = to_intel_crtc_state(crtc->base.state); ++ put_domains = modeset_get_crtc_power_domains(&crtc->base, crtc_state); ++ if (WARN_ON(put_domains)) ++ modeset_put_power_domains(dev_priv, put_domains); ++ } ++ ++ intel_display_power_put(dev_priv, POWER_DOMAIN_INIT, wakeref); ++ ++ intel_fbc_init_pipe_state(dev_priv); ++} ++ ++void intel_display_resume(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct drm_atomic_state *state = dev_priv->modeset_restore_state; ++ struct drm_modeset_acquire_ctx ctx; ++ int ret; ++ ++ dev_priv->modeset_restore_state = NULL; ++ if (state) ++ state->acquire_ctx = &ctx; ++ ++ drm_modeset_acquire_init(&ctx, 0); ++ ++ while (1) { ++ ret = drm_modeset_lock_all_ctx(dev, &ctx); ++ if (ret != -EDEADLK) ++ break; ++ ++ drm_modeset_backoff(&ctx); ++ } ++ ++ if (!ret) ++ ret = __intel_display_resume(dev, state, &ctx); ++ ++ intel_enable_ipc(dev_priv); ++ drm_modeset_drop_locks(&ctx); ++ drm_modeset_acquire_fini(&ctx); ++ ++ if (ret) ++ DRM_ERROR("Restoring old state failed with %i\n", ret); ++ if (state) ++ drm_atomic_state_put(state); ++} ++ ++static void intel_hpd_poll_fini(struct drm_device *dev) ++{ ++ struct intel_connector *connector; ++ struct drm_connector_list_iter conn_iter; ++ ++ /* Kill all the work that may have been queued by hpd. */ ++ drm_connector_list_iter_begin(dev, &conn_iter); ++ for_each_intel_connector_iter(connector, &conn_iter) { ++ if (connector->modeset_retry_work.func) ++ cancel_work_sync(&connector->modeset_retry_work); ++ if (connector->hdcp.shim) { ++ cancel_delayed_work_sync(&connector->hdcp.check_work); ++ cancel_work_sync(&connector->hdcp.prop_work); ++ } ++ } ++ drm_connector_list_iter_end(&conn_iter); ++} ++ ++void intel_modeset_cleanup(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ ++ flush_workqueue(dev_priv->modeset_wq); ++ ++ flush_work(&dev_priv->atomic_helper.free_work); ++ WARN_ON(!llist_empty(&dev_priv->atomic_helper.free_list)); ++ ++ /* ++ * Interrupts and polling as the first thing to avoid creating havoc. ++ * Too much stuff here (turning of connectors, ...) would ++ * experience fancy races otherwise. ++ */ ++ intel_irq_uninstall(dev_priv); ++ ++ /* ++ * Due to the hpd irq storm handling the hotplug work can re-arm the ++ * poll handlers. Hence disable polling after hpd handling is shut down. ++ */ ++ intel_hpd_poll_fini(dev); ++ ++ /* poll work can call into fbdev, hence clean that up afterwards */ ++ intel_fbdev_fini(dev_priv); ++ ++ intel_unregister_dsm_handler(); ++ ++ intel_fbc_global_disable(dev_priv); ++ ++ /* flush any delayed tasks or pending work */ ++ flush_scheduled_work(); ++ ++ intel_hdcp_component_fini(dev_priv); ++ ++ drm_mode_config_cleanup(dev); ++ ++ intel_overlay_cleanup(dev_priv); ++ ++ intel_teardown_gmbus(dev_priv); ++ ++ destroy_workqueue(dev_priv->modeset_wq); ++ ++ intel_fbc_cleanup_cfb(dev_priv); ++} ++ ++/* ++ * set vga decode state - true == enable VGA decode ++ */ ++int intel_modeset_vga_set_state(struct drm_i915_private *dev_priv, bool state) ++{ ++ unsigned reg = INTEL_GEN(dev_priv) >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL; ++ u16 gmch_ctrl; ++ ++ if (pci_read_config_word(dev_priv->bridge_dev, reg, &gmch_ctrl)) { ++ DRM_ERROR("failed to read control word\n"); ++ return -EIO; ++ } ++ ++ if (!!(gmch_ctrl & INTEL_GMCH_VGA_DISABLE) == !state) ++ return 0; ++ ++ if (state) ++ gmch_ctrl &= ~INTEL_GMCH_VGA_DISABLE; ++ else ++ gmch_ctrl |= INTEL_GMCH_VGA_DISABLE; ++ ++ if (pci_write_config_word(dev_priv->bridge_dev, reg, gmch_ctrl)) { ++ DRM_ERROR("failed to write control word\n"); ++ return -EIO; ++ } ++ ++ return 0; ++} ++ ++#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) ++ ++struct intel_display_error_state { ++ ++ u32 power_well_driver; ++ ++ struct intel_cursor_error_state { ++ u32 control; ++ u32 position; ++ u32 base; ++ u32 size; ++ } cursor[I915_MAX_PIPES]; ++ ++ struct intel_pipe_error_state { ++ bool power_domain_on; ++ u32 source; ++ u32 stat; ++ } pipe[I915_MAX_PIPES]; ++ ++ struct intel_plane_error_state { ++ u32 control; ++ u32 stride; ++ u32 size; ++ u32 pos; ++ u32 addr; ++ u32 surface; ++ u32 tile_offset; ++ } plane[I915_MAX_PIPES]; ++ ++ struct intel_transcoder_error_state { ++ bool available; ++ bool power_domain_on; ++ enum transcoder cpu_transcoder; ++ ++ u32 conf; ++ ++ u32 htotal; ++ u32 hblank; ++ u32 hsync; ++ u32 vtotal; ++ u32 vblank; ++ u32 vsync; ++ } transcoder[4]; ++}; ++ ++struct intel_display_error_state * ++intel_display_capture_error_state(struct drm_i915_private *dev_priv) ++{ ++ struct intel_display_error_state *error; ++ int transcoders[] = { ++ TRANSCODER_A, ++ TRANSCODER_B, ++ TRANSCODER_C, ++ TRANSCODER_EDP, ++ }; ++ int i; ++ ++ BUILD_BUG_ON(ARRAY_SIZE(transcoders) != ARRAY_SIZE(error->transcoder)); ++ ++ if (!HAS_DISPLAY(dev_priv)) ++ return NULL; ++ ++ error = kzalloc(sizeof(*error), GFP_ATOMIC); ++ if (error == NULL) ++ return NULL; ++ ++ if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) ++ error->power_well_driver = I915_READ(HSW_PWR_WELL_CTL2); ++ ++ for_each_pipe(dev_priv, i) { ++ error->pipe[i].power_domain_on = ++ __intel_display_power_is_enabled(dev_priv, ++ POWER_DOMAIN_PIPE(i)); ++ if (!error->pipe[i].power_domain_on) ++ continue; ++ ++ error->cursor[i].control = I915_READ(CURCNTR(i)); ++ error->cursor[i].position = I915_READ(CURPOS(i)); ++ error->cursor[i].base = I915_READ(CURBASE(i)); ++ ++ error->plane[i].control = I915_READ(DSPCNTR(i)); ++ error->plane[i].stride = I915_READ(DSPSTRIDE(i)); ++ if (INTEL_GEN(dev_priv) <= 3) { ++ error->plane[i].size = I915_READ(DSPSIZE(i)); ++ error->plane[i].pos = I915_READ(DSPPOS(i)); ++ } ++ if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv)) ++ error->plane[i].addr = I915_READ(DSPADDR(i)); ++ if (INTEL_GEN(dev_priv) >= 4) { ++ error->plane[i].surface = I915_READ(DSPSURF(i)); ++ error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i)); ++ } ++ ++ error->pipe[i].source = I915_READ(PIPESRC(i)); ++ ++ if (HAS_GMCH(dev_priv)) ++ error->pipe[i].stat = I915_READ(PIPESTAT(i)); ++ } ++ ++ for (i = 0; i < ARRAY_SIZE(error->transcoder); i++) { ++ enum transcoder cpu_transcoder = transcoders[i]; ++ ++ if (!INTEL_INFO(dev_priv)->trans_offsets[cpu_transcoder]) ++ continue; ++ ++ error->transcoder[i].available = true; ++ error->transcoder[i].power_domain_on = ++ __intel_display_power_is_enabled(dev_priv, ++ POWER_DOMAIN_TRANSCODER(cpu_transcoder)); ++ if (!error->transcoder[i].power_domain_on) ++ continue; ++ ++ error->transcoder[i].cpu_transcoder = cpu_transcoder; ++ ++ error->transcoder[i].conf = I915_READ(PIPECONF(cpu_transcoder)); ++ error->transcoder[i].htotal = I915_READ(HTOTAL(cpu_transcoder)); ++ error->transcoder[i].hblank = I915_READ(HBLANK(cpu_transcoder)); ++ error->transcoder[i].hsync = I915_READ(HSYNC(cpu_transcoder)); ++ error->transcoder[i].vtotal = I915_READ(VTOTAL(cpu_transcoder)); ++ error->transcoder[i].vblank = I915_READ(VBLANK(cpu_transcoder)); ++ error->transcoder[i].vsync = I915_READ(VSYNC(cpu_transcoder)); ++ } ++ ++ return error; ++} ++ ++#define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__) ++ ++void ++intel_display_print_error_state(struct drm_i915_error_state_buf *m, ++ struct intel_display_error_state *error) ++{ ++ struct drm_i915_private *dev_priv = m->i915; ++ int i; ++ ++ if (!error) ++ return; ++ ++ err_printf(m, "Num Pipes: %d\n", INTEL_INFO(dev_priv)->num_pipes); ++ if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) ++ err_printf(m, "PWR_WELL_CTL2: %08x\n", ++ error->power_well_driver); ++ for_each_pipe(dev_priv, i) { ++ err_printf(m, "Pipe [%d]:\n", i); ++ err_printf(m, " Power: %s\n", ++ onoff(error->pipe[i].power_domain_on)); ++ err_printf(m, " SRC: %08x\n", error->pipe[i].source); ++ err_printf(m, " STAT: %08x\n", error->pipe[i].stat); ++ ++ err_printf(m, "Plane [%d]:\n", i); ++ err_printf(m, " CNTR: %08x\n", error->plane[i].control); ++ err_printf(m, " STRIDE: %08x\n", error->plane[i].stride); ++ if (INTEL_GEN(dev_priv) <= 3) { ++ err_printf(m, " SIZE: %08x\n", error->plane[i].size); ++ err_printf(m, " POS: %08x\n", error->plane[i].pos); ++ } ++ if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv)) ++ err_printf(m, " ADDR: %08x\n", error->plane[i].addr); ++ if (INTEL_GEN(dev_priv) >= 4) { ++ err_printf(m, " SURF: %08x\n", error->plane[i].surface); ++ err_printf(m, " TILEOFF: %08x\n", error->plane[i].tile_offset); ++ } ++ ++ err_printf(m, "Cursor [%d]:\n", i); ++ err_printf(m, " CNTR: %08x\n", error->cursor[i].control); ++ err_printf(m, " POS: %08x\n", error->cursor[i].position); ++ err_printf(m, " BASE: %08x\n", error->cursor[i].base); ++ } ++ ++ for (i = 0; i < ARRAY_SIZE(error->transcoder); i++) { ++ if (!error->transcoder[i].available) ++ continue; ++ ++ err_printf(m, "CPU transcoder: %s\n", ++ transcoder_name(error->transcoder[i].cpu_transcoder)); ++ err_printf(m, " Power: %s\n", ++ onoff(error->transcoder[i].power_domain_on)); ++ err_printf(m, " CONF: %08x\n", error->transcoder[i].conf); ++ err_printf(m, " HTOTAL: %08x\n", error->transcoder[i].htotal); ++ err_printf(m, " HBLANK: %08x\n", error->transcoder[i].hblank); ++ err_printf(m, " HSYNC: %08x\n", error->transcoder[i].hsync); ++ err_printf(m, " VTOTAL: %08x\n", error->transcoder[i].vtotal); ++ err_printf(m, " VBLANK: %08x\n", error->transcoder[i].vblank); ++ err_printf(m, " VSYNC: %08x\n", error->transcoder[i].vsync); ++ } ++} ++ ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/intel_display.h b/drivers/gpu/drm/i915_legacy/intel_display.h +new file mode 100644 +index 000000000000..2220588e86ac +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_display.h +@@ -0,0 +1,435 @@ ++/* ++ * Copyright © 2006-2017 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#ifndef _INTEL_DISPLAY_H_ ++#define _INTEL_DISPLAY_H_ ++ ++#include ++#include ++ ++enum i915_gpio { ++ GPIOA, ++ GPIOB, ++ GPIOC, ++ GPIOD, ++ GPIOE, ++ GPIOF, ++ GPIOG, ++ GPIOH, ++ __GPIOI_UNUSED, ++ GPIOJ, ++ GPIOK, ++ GPIOL, ++ GPIOM, ++}; ++ ++/* ++ * Keep the pipe enum values fixed: the code assumes that PIPE_A=0, the ++ * rest have consecutive values and match the enum values of transcoders ++ * with a 1:1 transcoder -> pipe mapping. ++ */ ++enum pipe { ++ INVALID_PIPE = -1, ++ ++ PIPE_A = 0, ++ PIPE_B, ++ PIPE_C, ++ _PIPE_EDP, ++ ++ I915_MAX_PIPES = _PIPE_EDP ++}; ++ ++#define pipe_name(p) ((p) + 'A') ++ ++enum transcoder { ++ /* ++ * The following transcoders have a 1:1 transcoder -> pipe mapping, ++ * keep their values fixed: the code assumes that TRANSCODER_A=0, the ++ * rest have consecutive values and match the enum values of the pipes ++ * they map to. ++ */ ++ TRANSCODER_A = PIPE_A, ++ TRANSCODER_B = PIPE_B, ++ TRANSCODER_C = PIPE_C, ++ ++ /* ++ * The following transcoders can map to any pipe, their enum value ++ * doesn't need to stay fixed. ++ */ ++ TRANSCODER_EDP, ++ TRANSCODER_DSI_0, ++ TRANSCODER_DSI_1, ++ TRANSCODER_DSI_A = TRANSCODER_DSI_0, /* legacy DSI */ ++ TRANSCODER_DSI_C = TRANSCODER_DSI_1, /* legacy DSI */ ++ ++ I915_MAX_TRANSCODERS ++}; ++ ++static inline const char *transcoder_name(enum transcoder transcoder) ++{ ++ switch (transcoder) { ++ case TRANSCODER_A: ++ return "A"; ++ case TRANSCODER_B: ++ return "B"; ++ case TRANSCODER_C: ++ return "C"; ++ case TRANSCODER_EDP: ++ return "EDP"; ++ case TRANSCODER_DSI_A: ++ return "DSI A"; ++ case TRANSCODER_DSI_C: ++ return "DSI C"; ++ default: ++ return ""; ++ } ++} ++ ++static inline bool transcoder_is_dsi(enum transcoder transcoder) ++{ ++ return transcoder == TRANSCODER_DSI_A || transcoder == TRANSCODER_DSI_C; ++} ++ ++/* ++ * Global legacy plane identifier. Valid only for primary/sprite ++ * planes on pre-g4x, and only for primary planes on g4x-bdw. ++ */ ++enum i9xx_plane_id { ++ PLANE_A, ++ PLANE_B, ++ PLANE_C, ++}; ++ ++#define plane_name(p) ((p) + 'A') ++#define sprite_name(p, s) ((p) * RUNTIME_INFO(dev_priv)->num_sprites[(p)] + (s) + 'A') ++ ++/* ++ * Per-pipe plane identifier. ++ * I915_MAX_PLANES in the enum below is the maximum (across all platforms) ++ * number of planes per CRTC. Not all platforms really have this many planes, ++ * which means some arrays of size I915_MAX_PLANES may have unused entries ++ * between the topmost sprite plane and the cursor plane. ++ * ++ * This is expected to be passed to various register macros ++ * (eg. PLANE_CTL(), PS_PLANE_SEL(), etc.) so adjust with care. ++ */ ++enum plane_id { ++ PLANE_PRIMARY, ++ PLANE_SPRITE0, ++ PLANE_SPRITE1, ++ PLANE_SPRITE2, ++ PLANE_SPRITE3, ++ PLANE_SPRITE4, ++ PLANE_SPRITE5, ++ PLANE_CURSOR, ++ ++ I915_MAX_PLANES, ++}; ++ ++#define for_each_plane_id_on_crtc(__crtc, __p) \ ++ for ((__p) = PLANE_PRIMARY; (__p) < I915_MAX_PLANES; (__p)++) \ ++ for_each_if((__crtc)->plane_ids_mask & BIT(__p)) ++ ++/* ++ * Ports identifier referenced from other drivers. ++ * Expected to remain stable over time ++ */ ++static inline const char *port_identifier(enum port port) ++{ ++ switch (port) { ++ case PORT_A: ++ return "Port A"; ++ case PORT_B: ++ return "Port B"; ++ case PORT_C: ++ return "Port C"; ++ case PORT_D: ++ return "Port D"; ++ case PORT_E: ++ return "Port E"; ++ case PORT_F: ++ return "Port F"; ++ default: ++ return ""; ++ } ++} ++ ++enum tc_port { ++ PORT_TC_NONE = -1, ++ ++ PORT_TC1 = 0, ++ PORT_TC2, ++ PORT_TC3, ++ PORT_TC4, ++ ++ I915_MAX_TC_PORTS ++}; ++ ++enum tc_port_type { ++ TC_PORT_UNKNOWN = 0, ++ TC_PORT_TYPEC, ++ TC_PORT_TBT, ++ TC_PORT_LEGACY, ++}; ++ ++enum dpio_channel { ++ DPIO_CH0, ++ DPIO_CH1 ++}; ++ ++enum dpio_phy { ++ DPIO_PHY0, ++ DPIO_PHY1, ++ DPIO_PHY2, ++}; ++ ++#define I915_NUM_PHYS_VLV 2 ++ ++enum aux_ch { ++ AUX_CH_A, ++ AUX_CH_B, ++ AUX_CH_C, ++ AUX_CH_D, ++ AUX_CH_E, /* ICL+ */ ++ AUX_CH_F, ++}; ++ ++#define aux_ch_name(a) ((a) + 'A') ++ ++enum intel_display_power_domain { ++ POWER_DOMAIN_PIPE_A, ++ POWER_DOMAIN_PIPE_B, ++ POWER_DOMAIN_PIPE_C, ++ POWER_DOMAIN_PIPE_A_PANEL_FITTER, ++ POWER_DOMAIN_PIPE_B_PANEL_FITTER, ++ POWER_DOMAIN_PIPE_C_PANEL_FITTER, ++ POWER_DOMAIN_TRANSCODER_A, ++ POWER_DOMAIN_TRANSCODER_B, ++ POWER_DOMAIN_TRANSCODER_C, ++ POWER_DOMAIN_TRANSCODER_EDP, ++ POWER_DOMAIN_TRANSCODER_EDP_VDSC, ++ POWER_DOMAIN_TRANSCODER_DSI_A, ++ POWER_DOMAIN_TRANSCODER_DSI_C, ++ POWER_DOMAIN_PORT_DDI_A_LANES, ++ POWER_DOMAIN_PORT_DDI_B_LANES, ++ POWER_DOMAIN_PORT_DDI_C_LANES, ++ POWER_DOMAIN_PORT_DDI_D_LANES, ++ POWER_DOMAIN_PORT_DDI_E_LANES, ++ POWER_DOMAIN_PORT_DDI_F_LANES, ++ POWER_DOMAIN_PORT_DDI_A_IO, ++ POWER_DOMAIN_PORT_DDI_B_IO, ++ POWER_DOMAIN_PORT_DDI_C_IO, ++ POWER_DOMAIN_PORT_DDI_D_IO, ++ POWER_DOMAIN_PORT_DDI_E_IO, ++ POWER_DOMAIN_PORT_DDI_F_IO, ++ POWER_DOMAIN_PORT_DSI, ++ POWER_DOMAIN_PORT_CRT, ++ POWER_DOMAIN_PORT_OTHER, ++ POWER_DOMAIN_VGA, ++ POWER_DOMAIN_AUDIO, ++ POWER_DOMAIN_PLLS, ++ POWER_DOMAIN_AUX_A, ++ POWER_DOMAIN_AUX_B, ++ POWER_DOMAIN_AUX_C, ++ POWER_DOMAIN_AUX_D, ++ POWER_DOMAIN_AUX_E, ++ POWER_DOMAIN_AUX_F, ++ POWER_DOMAIN_AUX_IO_A, ++ POWER_DOMAIN_AUX_TBT1, ++ POWER_DOMAIN_AUX_TBT2, ++ POWER_DOMAIN_AUX_TBT3, ++ POWER_DOMAIN_AUX_TBT4, ++ POWER_DOMAIN_GMBUS, ++ POWER_DOMAIN_MODESET, ++ POWER_DOMAIN_GT_IRQ, ++ POWER_DOMAIN_INIT, ++ ++ POWER_DOMAIN_NUM, ++}; ++ ++#define POWER_DOMAIN_PIPE(pipe) ((pipe) + POWER_DOMAIN_PIPE_A) ++#define POWER_DOMAIN_PIPE_PANEL_FITTER(pipe) \ ++ ((pipe) + POWER_DOMAIN_PIPE_A_PANEL_FITTER) ++#define POWER_DOMAIN_TRANSCODER(tran) \ ++ ((tran) == TRANSCODER_EDP ? POWER_DOMAIN_TRANSCODER_EDP : \ ++ (tran) + POWER_DOMAIN_TRANSCODER_A) ++ ++/* Used by dp and fdi links */ ++struct intel_link_m_n { ++ u32 tu; ++ u32 gmch_m; ++ u32 gmch_n; ++ u32 link_m; ++ u32 link_n; ++}; ++ ++#define for_each_pipe(__dev_priv, __p) \ ++ for ((__p) = 0; (__p) < INTEL_INFO(__dev_priv)->num_pipes; (__p)++) ++ ++#define for_each_pipe_masked(__dev_priv, __p, __mask) \ ++ for ((__p) = 0; (__p) < INTEL_INFO(__dev_priv)->num_pipes; (__p)++) \ ++ for_each_if((__mask) & BIT(__p)) ++ ++#define for_each_cpu_transcoder_masked(__dev_priv, __t, __mask) \ ++ for ((__t) = 0; (__t) < I915_MAX_TRANSCODERS; (__t)++) \ ++ for_each_if ((__mask) & (1 << (__t))) ++ ++#define for_each_universal_plane(__dev_priv, __pipe, __p) \ ++ for ((__p) = 0; \ ++ (__p) < RUNTIME_INFO(__dev_priv)->num_sprites[(__pipe)] + 1; \ ++ (__p)++) ++ ++#define for_each_sprite(__dev_priv, __p, __s) \ ++ for ((__s) = 0; \ ++ (__s) < RUNTIME_INFO(__dev_priv)->num_sprites[(__p)]; \ ++ (__s)++) ++ ++#define for_each_port_masked(__port, __ports_mask) \ ++ for ((__port) = PORT_A; (__port) < I915_MAX_PORTS; (__port)++) \ ++ for_each_if((__ports_mask) & BIT(__port)) ++ ++#define for_each_crtc(dev, crtc) \ ++ list_for_each_entry(crtc, &(dev)->mode_config.crtc_list, head) ++ ++#define for_each_intel_plane(dev, intel_plane) \ ++ list_for_each_entry(intel_plane, \ ++ &(dev)->mode_config.plane_list, \ ++ base.head) ++ ++#define for_each_intel_plane_mask(dev, intel_plane, plane_mask) \ ++ list_for_each_entry(intel_plane, \ ++ &(dev)->mode_config.plane_list, \ ++ base.head) \ ++ for_each_if((plane_mask) & \ ++ drm_plane_mask(&intel_plane->base))) ++ ++#define for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) \ ++ list_for_each_entry(intel_plane, \ ++ &(dev)->mode_config.plane_list, \ ++ base.head) \ ++ for_each_if((intel_plane)->pipe == (intel_crtc)->pipe) ++ ++#define for_each_intel_crtc(dev, intel_crtc) \ ++ list_for_each_entry(intel_crtc, \ ++ &(dev)->mode_config.crtc_list, \ ++ base.head) ++ ++#define for_each_intel_crtc_mask(dev, intel_crtc, crtc_mask) \ ++ list_for_each_entry(intel_crtc, \ ++ &(dev)->mode_config.crtc_list, \ ++ base.head) \ ++ for_each_if((crtc_mask) & drm_crtc_mask(&intel_crtc->base)) ++ ++#define for_each_intel_encoder(dev, intel_encoder) \ ++ list_for_each_entry(intel_encoder, \ ++ &(dev)->mode_config.encoder_list, \ ++ base.head) ++ ++#define for_each_intel_dp(dev, intel_encoder) \ ++ for_each_intel_encoder(dev, intel_encoder) \ ++ for_each_if(intel_encoder_is_dp(intel_encoder)) ++ ++#define for_each_intel_connector_iter(intel_connector, iter) \ ++ while ((intel_connector = to_intel_connector(drm_connector_list_iter_next(iter)))) ++ ++#define for_each_encoder_on_crtc(dev, __crtc, intel_encoder) \ ++ list_for_each_entry((intel_encoder), &(dev)->mode_config.encoder_list, base.head) \ ++ for_each_if((intel_encoder)->base.crtc == (__crtc)) ++ ++#define for_each_connector_on_encoder(dev, __encoder, intel_connector) \ ++ list_for_each_entry((intel_connector), &(dev)->mode_config.connector_list, base.head) \ ++ for_each_if((intel_connector)->base.encoder == (__encoder)) ++ ++#define for_each_power_domain(domain, mask) \ ++ for ((domain) = 0; (domain) < POWER_DOMAIN_NUM; (domain)++) \ ++ for_each_if(BIT_ULL(domain) & (mask)) ++ ++#define for_each_power_well(__dev_priv, __power_well) \ ++ for ((__power_well) = (__dev_priv)->power_domains.power_wells; \ ++ (__power_well) - (__dev_priv)->power_domains.power_wells < \ ++ (__dev_priv)->power_domains.power_well_count; \ ++ (__power_well)++) ++ ++#define for_each_power_well_reverse(__dev_priv, __power_well) \ ++ for ((__power_well) = (__dev_priv)->power_domains.power_wells + \ ++ (__dev_priv)->power_domains.power_well_count - 1; \ ++ (__power_well) - (__dev_priv)->power_domains.power_wells >= 0; \ ++ (__power_well)--) ++ ++#define for_each_power_domain_well(__dev_priv, __power_well, __domain_mask) \ ++ for_each_power_well(__dev_priv, __power_well) \ ++ for_each_if((__power_well)->desc->domains & (__domain_mask)) ++ ++#define for_each_power_domain_well_reverse(__dev_priv, __power_well, __domain_mask) \ ++ for_each_power_well_reverse(__dev_priv, __power_well) \ ++ for_each_if((__power_well)->desc->domains & (__domain_mask)) ++ ++#define for_each_old_intel_plane_in_state(__state, plane, old_plane_state, __i) \ ++ for ((__i) = 0; \ ++ (__i) < (__state)->base.dev->mode_config.num_total_plane && \ ++ ((plane) = to_intel_plane((__state)->base.planes[__i].ptr), \ ++ (old_plane_state) = to_intel_plane_state((__state)->base.planes[__i].old_state), 1); \ ++ (__i)++) \ ++ for_each_if(plane) ++ ++#define for_each_new_intel_plane_in_state(__state, plane, new_plane_state, __i) \ ++ for ((__i) = 0; \ ++ (__i) < (__state)->base.dev->mode_config.num_total_plane && \ ++ ((plane) = to_intel_plane((__state)->base.planes[__i].ptr), \ ++ (new_plane_state) = to_intel_plane_state((__state)->base.planes[__i].new_state), 1); \ ++ (__i)++) \ ++ for_each_if(plane) ++ ++#define for_each_new_intel_crtc_in_state(__state, crtc, new_crtc_state, __i) \ ++ for ((__i) = 0; \ ++ (__i) < (__state)->base.dev->mode_config.num_crtc && \ ++ ((crtc) = to_intel_crtc((__state)->base.crtcs[__i].ptr), \ ++ (new_crtc_state) = to_intel_crtc_state((__state)->base.crtcs[__i].new_state), 1); \ ++ (__i)++) \ ++ for_each_if(crtc) ++ ++#define for_each_oldnew_intel_plane_in_state(__state, plane, old_plane_state, new_plane_state, __i) \ ++ for ((__i) = 0; \ ++ (__i) < (__state)->base.dev->mode_config.num_total_plane && \ ++ ((plane) = to_intel_plane((__state)->base.planes[__i].ptr), \ ++ (old_plane_state) = to_intel_plane_state((__state)->base.planes[__i].old_state), \ ++ (new_plane_state) = to_intel_plane_state((__state)->base.planes[__i].new_state), 1); \ ++ (__i)++) \ ++ for_each_if(plane) ++ ++#define for_each_oldnew_intel_crtc_in_state(__state, crtc, old_crtc_state, new_crtc_state, __i) \ ++ for ((__i) = 0; \ ++ (__i) < (__state)->base.dev->mode_config.num_crtc && \ ++ ((crtc) = to_intel_crtc((__state)->base.crtcs[__i].ptr), \ ++ (old_crtc_state) = to_intel_crtc_state((__state)->base.crtcs[__i].old_state), \ ++ (new_crtc_state) = to_intel_crtc_state((__state)->base.crtcs[__i].new_state), 1); \ ++ (__i)++) \ ++ for_each_if(crtc) ++ ++void intel_link_compute_m_n(u16 bpp, int nlanes, ++ int pixel_clock, int link_clock, ++ struct intel_link_m_n *m_n, ++ bool constant_n); ++bool is_ccs_modifier(u64 modifier); ++#endif +diff --git a/drivers/gpu/drm/i915_legacy/intel_dp.c b/drivers/gpu/drm/i915_legacy/intel_dp.c +new file mode 100644 +index 000000000000..560274d1c50b +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_dp.c +@@ -0,0 +1,7405 @@ ++/* ++ * Copyright © 2008 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ * Authors: ++ * Keith Packard ++ * ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "i915_drv.h" ++#include "intel_audio.h" ++#include "intel_connector.h" ++#include "intel_ddi.h" ++#include "intel_dp.h" ++#include "intel_drv.h" ++#include "intel_hdcp.h" ++#include "intel_hdmi.h" ++#include "intel_lspcon.h" ++#include "intel_lvds.h" ++#include "intel_panel.h" ++#include "intel_psr.h" ++ ++#define DP_DPRX_ESI_LEN 14 ++ ++/* DP DSC small joiner has 2 FIFOs each of 640 x 6 bytes */ ++#define DP_DSC_MAX_SMALL_JOINER_RAM_BUFFER 61440 ++#define DP_DSC_MIN_SUPPORTED_BPC 8 ++#define DP_DSC_MAX_SUPPORTED_BPC 10 ++ ++/* DP DSC throughput values used for slice count calculations KPixels/s */ ++#define DP_DSC_PEAK_PIXEL_RATE 2720000 ++#define DP_DSC_MAX_ENC_THROUGHPUT_0 340000 ++#define DP_DSC_MAX_ENC_THROUGHPUT_1 400000 ++ ++/* DP DSC FEC Overhead factor = (100 - 2.4)/100 */ ++#define DP_DSC_FEC_OVERHEAD_FACTOR 976 ++ ++/* Compliance test status bits */ ++#define INTEL_DP_RESOLUTION_SHIFT_MASK 0 ++#define INTEL_DP_RESOLUTION_PREFERRED (1 << INTEL_DP_RESOLUTION_SHIFT_MASK) ++#define INTEL_DP_RESOLUTION_STANDARD (2 << INTEL_DP_RESOLUTION_SHIFT_MASK) ++#define INTEL_DP_RESOLUTION_FAILSAFE (3 << INTEL_DP_RESOLUTION_SHIFT_MASK) ++ ++struct dp_link_dpll { ++ int clock; ++ struct dpll dpll; ++}; ++ ++static const struct dp_link_dpll g4x_dpll[] = { ++ { 162000, ++ { .p1 = 2, .p2 = 10, .n = 2, .m1 = 23, .m2 = 8 } }, ++ { 270000, ++ { .p1 = 1, .p2 = 10, .n = 1, .m1 = 14, .m2 = 2 } } ++}; ++ ++static const struct dp_link_dpll pch_dpll[] = { ++ { 162000, ++ { .p1 = 2, .p2 = 10, .n = 1, .m1 = 12, .m2 = 9 } }, ++ { 270000, ++ { .p1 = 1, .p2 = 10, .n = 2, .m1 = 14, .m2 = 8 } } ++}; ++ ++static const struct dp_link_dpll vlv_dpll[] = { ++ { 162000, ++ { .p1 = 3, .p2 = 2, .n = 5, .m1 = 3, .m2 = 81 } }, ++ { 270000, ++ { .p1 = 2, .p2 = 2, .n = 1, .m1 = 2, .m2 = 27 } } ++}; ++ ++/* ++ * CHV supports eDP 1.4 that have more link rates. ++ * Below only provides the fixed rate but exclude variable rate. ++ */ ++static const struct dp_link_dpll chv_dpll[] = { ++ /* ++ * CHV requires to program fractional division for m2. ++ * m2 is stored in fixed point format using formula below ++ * (m2_int << 22) | m2_fraction ++ */ ++ { 162000, /* m2_int = 32, m2_fraction = 1677722 */ ++ { .p1 = 4, .p2 = 2, .n = 1, .m1 = 2, .m2 = 0x819999a } }, ++ { 270000, /* m2_int = 27, m2_fraction = 0 */ ++ { .p1 = 4, .p2 = 1, .n = 1, .m1 = 2, .m2 = 0x6c00000 } }, ++}; ++ ++/* Constants for DP DSC configurations */ ++static const u8 valid_dsc_bpp[] = {6, 8, 10, 12, 15}; ++ ++/* With Single pipe configuration, HW is capable of supporting maximum ++ * of 4 slices per line. ++ */ ++static const u8 valid_dsc_slicecount[] = {1, 2, 4}; ++ ++/** ++ * intel_dp_is_edp - is the given port attached to an eDP panel (either CPU or PCH) ++ * @intel_dp: DP struct ++ * ++ * If a CPU or PCH DP output is attached to an eDP panel, this function ++ * will return true, and false otherwise. ++ */ ++bool intel_dp_is_edp(struct intel_dp *intel_dp) ++{ ++ struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); ++ ++ return intel_dig_port->base.type == INTEL_OUTPUT_EDP; ++} ++ ++static struct intel_dp *intel_attached_dp(struct drm_connector *connector) ++{ ++ return enc_to_intel_dp(&intel_attached_encoder(connector)->base); ++} ++ ++static void intel_dp_link_down(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state); ++static bool edp_panel_vdd_on(struct intel_dp *intel_dp); ++static void edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync); ++static void vlv_init_panel_power_sequencer(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state); ++static void vlv_steal_power_sequencer(struct drm_i915_private *dev_priv, ++ enum pipe pipe); ++static void intel_dp_unset_edid(struct intel_dp *intel_dp); ++ ++/* update sink rates from dpcd */ ++static void intel_dp_set_sink_rates(struct intel_dp *intel_dp) ++{ ++ static const int dp_rates[] = { ++ 162000, 270000, 540000, 810000 ++ }; ++ int i, max_rate; ++ ++ max_rate = drm_dp_bw_code_to_link_rate(intel_dp->dpcd[DP_MAX_LINK_RATE]); ++ ++ for (i = 0; i < ARRAY_SIZE(dp_rates); i++) { ++ if (dp_rates[i] > max_rate) ++ break; ++ intel_dp->sink_rates[i] = dp_rates[i]; ++ } ++ ++ intel_dp->num_sink_rates = i; ++} ++ ++/* Get length of rates array potentially limited by max_rate. */ ++static int intel_dp_rate_limit_len(const int *rates, int len, int max_rate) ++{ ++ int i; ++ ++ /* Limit results by potentially reduced max rate */ ++ for (i = 0; i < len; i++) { ++ if (rates[len - i - 1] <= max_rate) ++ return len - i; ++ } ++ ++ return 0; ++} ++ ++/* Get length of common rates array potentially limited by max_rate. */ ++static int intel_dp_common_len_rate_limit(const struct intel_dp *intel_dp, ++ int max_rate) ++{ ++ return intel_dp_rate_limit_len(intel_dp->common_rates, ++ intel_dp->num_common_rates, max_rate); ++} ++ ++/* Theoretical max between source and sink */ ++static int intel_dp_max_common_rate(struct intel_dp *intel_dp) ++{ ++ return intel_dp->common_rates[intel_dp->num_common_rates - 1]; ++} ++ ++static int intel_dp_get_fia_supported_lane_count(struct intel_dp *intel_dp) ++{ ++ struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); ++ struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); ++ enum tc_port tc_port = intel_port_to_tc(dev_priv, dig_port->base.port); ++ u32 lane_info; ++ ++ if (tc_port == PORT_TC_NONE || dig_port->tc_type != TC_PORT_TYPEC) ++ return 4; ++ ++ lane_info = (I915_READ(PORT_TX_DFLEXDPSP) & ++ DP_LANE_ASSIGNMENT_MASK(tc_port)) >> ++ DP_LANE_ASSIGNMENT_SHIFT(tc_port); ++ ++ switch (lane_info) { ++ default: ++ MISSING_CASE(lane_info); ++ case 1: ++ case 2: ++ case 4: ++ case 8: ++ return 1; ++ case 3: ++ case 12: ++ return 2; ++ case 15: ++ return 4; ++ } ++} ++ ++/* Theoretical max between source and sink */ ++static int intel_dp_max_common_lane_count(struct intel_dp *intel_dp) ++{ ++ struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); ++ int source_max = intel_dig_port->max_lanes; ++ int sink_max = drm_dp_max_lane_count(intel_dp->dpcd); ++ int fia_max = intel_dp_get_fia_supported_lane_count(intel_dp); ++ ++ return min3(source_max, sink_max, fia_max); ++} ++ ++int intel_dp_max_lane_count(struct intel_dp *intel_dp) ++{ ++ return intel_dp->max_link_lane_count; ++} ++ ++int ++intel_dp_link_required(int pixel_clock, int bpp) ++{ ++ /* pixel_clock is in kHz, divide bpp by 8 for bit to Byte conversion */ ++ return DIV_ROUND_UP(pixel_clock * bpp, 8); ++} ++ ++int ++intel_dp_max_data_rate(int max_link_clock, int max_lanes) ++{ ++ /* max_link_clock is the link symbol clock (LS_Clk) in kHz and not the ++ * link rate that is generally expressed in Gbps. Since, 8 bits of data ++ * is transmitted every LS_Clk per lane, there is no need to account for ++ * the channel encoding that is done in the PHY layer here. ++ */ ++ ++ return max_link_clock * max_lanes; ++} ++ ++static int ++intel_dp_downstream_max_dotclock(struct intel_dp *intel_dp) ++{ ++ struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); ++ struct intel_encoder *encoder = &intel_dig_port->base; ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ int max_dotclk = dev_priv->max_dotclk_freq; ++ int ds_max_dotclk; ++ ++ int type = intel_dp->downstream_ports[0] & DP_DS_PORT_TYPE_MASK; ++ ++ if (type != DP_DS_PORT_TYPE_VGA) ++ return max_dotclk; ++ ++ ds_max_dotclk = drm_dp_downstream_max_clock(intel_dp->dpcd, ++ intel_dp->downstream_ports); ++ ++ if (ds_max_dotclk != 0) ++ max_dotclk = min(max_dotclk, ds_max_dotclk); ++ ++ return max_dotclk; ++} ++ ++static int cnl_max_source_rate(struct intel_dp *intel_dp) ++{ ++ struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); ++ struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); ++ enum port port = dig_port->base.port; ++ ++ u32 voltage = I915_READ(CNL_PORT_COMP_DW3) & VOLTAGE_INFO_MASK; ++ ++ /* Low voltage SKUs are limited to max of 5.4G */ ++ if (voltage == VOLTAGE_INFO_0_85V) ++ return 540000; ++ ++ /* For this SKU 8.1G is supported in all ports */ ++ if (IS_CNL_WITH_PORT_F(dev_priv)) ++ return 810000; ++ ++ /* For other SKUs, max rate on ports A and D is 5.4G */ ++ if (port == PORT_A || port == PORT_D) ++ return 540000; ++ ++ return 810000; ++} ++ ++static int icl_max_source_rate(struct intel_dp *intel_dp) ++{ ++ struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); ++ struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); ++ enum port port = dig_port->base.port; ++ ++ if (intel_port_is_combophy(dev_priv, port) && ++ !intel_dp_is_edp(intel_dp)) ++ return 540000; ++ ++ return 810000; ++} ++ ++static void ++intel_dp_set_source_rates(struct intel_dp *intel_dp) ++{ ++ /* The values must be in increasing order */ ++ static const int cnl_rates[] = { ++ 162000, 216000, 270000, 324000, 432000, 540000, 648000, 810000 ++ }; ++ static const int bxt_rates[] = { ++ 162000, 216000, 243000, 270000, 324000, 432000, 540000 ++ }; ++ static const int skl_rates[] = { ++ 162000, 216000, 270000, 324000, 432000, 540000 ++ }; ++ static const int hsw_rates[] = { ++ 162000, 270000, 540000 ++ }; ++ static const int g4x_rates[] = { ++ 162000, 270000 ++ }; ++ struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); ++ struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); ++ const struct ddi_vbt_port_info *info = ++ &dev_priv->vbt.ddi_port_info[dig_port->base.port]; ++ const int *source_rates; ++ int size, max_rate = 0, vbt_max_rate = info->dp_max_link_rate; ++ ++ /* This should only be done once */ ++ WARN_ON(intel_dp->source_rates || intel_dp->num_source_rates); ++ ++ if (INTEL_GEN(dev_priv) >= 10) { ++ source_rates = cnl_rates; ++ size = ARRAY_SIZE(cnl_rates); ++ if (IS_GEN(dev_priv, 10)) ++ max_rate = cnl_max_source_rate(intel_dp); ++ else ++ max_rate = icl_max_source_rate(intel_dp); ++ } else if (IS_GEN9_LP(dev_priv)) { ++ source_rates = bxt_rates; ++ size = ARRAY_SIZE(bxt_rates); ++ } else if (IS_GEN9_BC(dev_priv)) { ++ source_rates = skl_rates; ++ size = ARRAY_SIZE(skl_rates); ++ } else if ((IS_HASWELL(dev_priv) && !IS_HSW_ULX(dev_priv)) || ++ IS_BROADWELL(dev_priv)) { ++ source_rates = hsw_rates; ++ size = ARRAY_SIZE(hsw_rates); ++ } else { ++ source_rates = g4x_rates; ++ size = ARRAY_SIZE(g4x_rates); ++ } ++ ++ if (max_rate && vbt_max_rate) ++ max_rate = min(max_rate, vbt_max_rate); ++ else if (vbt_max_rate) ++ max_rate = vbt_max_rate; ++ ++ if (max_rate) ++ size = intel_dp_rate_limit_len(source_rates, size, max_rate); ++ ++ intel_dp->source_rates = source_rates; ++ intel_dp->num_source_rates = size; ++} ++ ++static int intersect_rates(const int *source_rates, int source_len, ++ const int *sink_rates, int sink_len, ++ int *common_rates) ++{ ++ int i = 0, j = 0, k = 0; ++ ++ while (i < source_len && j < sink_len) { ++ if (source_rates[i] == sink_rates[j]) { ++ if (WARN_ON(k >= DP_MAX_SUPPORTED_RATES)) ++ return k; ++ common_rates[k] = source_rates[i]; ++ ++k; ++ ++i; ++ ++j; ++ } else if (source_rates[i] < sink_rates[j]) { ++ ++i; ++ } else { ++ ++j; ++ } ++ } ++ return k; ++} ++ ++/* return index of rate in rates array, or -1 if not found */ ++static int intel_dp_rate_index(const int *rates, int len, int rate) ++{ ++ int i; ++ ++ for (i = 0; i < len; i++) ++ if (rate == rates[i]) ++ return i; ++ ++ return -1; ++} ++ ++static void intel_dp_set_common_rates(struct intel_dp *intel_dp) ++{ ++ WARN_ON(!intel_dp->num_source_rates || !intel_dp->num_sink_rates); ++ ++ intel_dp->num_common_rates = intersect_rates(intel_dp->source_rates, ++ intel_dp->num_source_rates, ++ intel_dp->sink_rates, ++ intel_dp->num_sink_rates, ++ intel_dp->common_rates); ++ ++ /* Paranoia, there should always be something in common. */ ++ if (WARN_ON(intel_dp->num_common_rates == 0)) { ++ intel_dp->common_rates[0] = 162000; ++ intel_dp->num_common_rates = 1; ++ } ++} ++ ++static bool intel_dp_link_params_valid(struct intel_dp *intel_dp, int link_rate, ++ u8 lane_count) ++{ ++ /* ++ * FIXME: we need to synchronize the current link parameters with ++ * hardware readout. Currently fast link training doesn't work on ++ * boot-up. ++ */ ++ if (link_rate == 0 || ++ link_rate > intel_dp->max_link_rate) ++ return false; ++ ++ if (lane_count == 0 || ++ lane_count > intel_dp_max_lane_count(intel_dp)) ++ return false; ++ ++ return true; ++} ++ ++static bool intel_dp_can_link_train_fallback_for_edp(struct intel_dp *intel_dp, ++ int link_rate, ++ u8 lane_count) ++{ ++ const struct drm_display_mode *fixed_mode = ++ intel_dp->attached_connector->panel.fixed_mode; ++ int mode_rate, max_rate; ++ ++ mode_rate = intel_dp_link_required(fixed_mode->clock, 18); ++ max_rate = intel_dp_max_data_rate(link_rate, lane_count); ++ if (mode_rate > max_rate) ++ return false; ++ ++ return true; ++} ++ ++int intel_dp_get_link_train_fallback_values(struct intel_dp *intel_dp, ++ int link_rate, u8 lane_count) ++{ ++ int index; ++ ++ index = intel_dp_rate_index(intel_dp->common_rates, ++ intel_dp->num_common_rates, ++ link_rate); ++ if (index > 0) { ++ if (intel_dp_is_edp(intel_dp) && ++ !intel_dp_can_link_train_fallback_for_edp(intel_dp, ++ intel_dp->common_rates[index - 1], ++ lane_count)) { ++ DRM_DEBUG_KMS("Retrying Link training for eDP with same parameters\n"); ++ return 0; ++ } ++ intel_dp->max_link_rate = intel_dp->common_rates[index - 1]; ++ intel_dp->max_link_lane_count = lane_count; ++ } else if (lane_count > 1) { ++ if (intel_dp_is_edp(intel_dp) && ++ !intel_dp_can_link_train_fallback_for_edp(intel_dp, ++ intel_dp_max_common_rate(intel_dp), ++ lane_count >> 1)) { ++ DRM_DEBUG_KMS("Retrying Link training for eDP with same parameters\n"); ++ return 0; ++ } ++ intel_dp->max_link_rate = intel_dp_max_common_rate(intel_dp); ++ intel_dp->max_link_lane_count = lane_count >> 1; ++ } else { ++ DRM_ERROR("Link Training Unsuccessful\n"); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++static enum drm_mode_status ++intel_dp_mode_valid(struct drm_connector *connector, ++ struct drm_display_mode *mode) ++{ ++ struct intel_dp *intel_dp = intel_attached_dp(connector); ++ struct intel_connector *intel_connector = to_intel_connector(connector); ++ struct drm_display_mode *fixed_mode = intel_connector->panel.fixed_mode; ++ struct drm_i915_private *dev_priv = to_i915(connector->dev); ++ int target_clock = mode->clock; ++ int max_rate, mode_rate, max_lanes, max_link_clock; ++ int max_dotclk; ++ u16 dsc_max_output_bpp = 0; ++ u8 dsc_slice_count = 0; ++ ++ if (mode->flags & DRM_MODE_FLAG_DBLSCAN) ++ return MODE_NO_DBLESCAN; ++ ++ max_dotclk = intel_dp_downstream_max_dotclock(intel_dp); ++ ++ if (intel_dp_is_edp(intel_dp) && fixed_mode) { ++ if (mode->hdisplay > fixed_mode->hdisplay) ++ return MODE_PANEL; ++ ++ if (mode->vdisplay > fixed_mode->vdisplay) ++ return MODE_PANEL; ++ ++ target_clock = fixed_mode->clock; ++ } ++ ++ max_link_clock = intel_dp_max_link_rate(intel_dp); ++ max_lanes = intel_dp_max_lane_count(intel_dp); ++ ++ max_rate = intel_dp_max_data_rate(max_link_clock, max_lanes); ++ mode_rate = intel_dp_link_required(target_clock, 18); ++ ++ /* ++ * Output bpp is stored in 6.4 format so right shift by 4 to get the ++ * integer value since we support only integer values of bpp. ++ */ ++ if ((INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) && ++ drm_dp_sink_supports_dsc(intel_dp->dsc_dpcd)) { ++ if (intel_dp_is_edp(intel_dp)) { ++ dsc_max_output_bpp = ++ drm_edp_dsc_sink_output_bpp(intel_dp->dsc_dpcd) >> 4; ++ dsc_slice_count = ++ drm_dp_dsc_sink_max_slice_count(intel_dp->dsc_dpcd, ++ true); ++ } else if (drm_dp_sink_supports_fec(intel_dp->fec_capable)) { ++ dsc_max_output_bpp = ++ intel_dp_dsc_get_output_bpp(max_link_clock, ++ max_lanes, ++ target_clock, ++ mode->hdisplay) >> 4; ++ dsc_slice_count = ++ intel_dp_dsc_get_slice_count(intel_dp, ++ target_clock, ++ mode->hdisplay); ++ } ++ } ++ ++ if ((mode_rate > max_rate && !(dsc_max_output_bpp && dsc_slice_count)) || ++ target_clock > max_dotclk) ++ return MODE_CLOCK_HIGH; ++ ++ if (mode->clock < 10000) ++ return MODE_CLOCK_LOW; ++ ++ if (mode->flags & DRM_MODE_FLAG_DBLCLK) ++ return MODE_H_ILLEGAL; ++ ++ return MODE_OK; ++} ++ ++u32 intel_dp_pack_aux(const u8 *src, int src_bytes) ++{ ++ int i; ++ u32 v = 0; ++ ++ if (src_bytes > 4) ++ src_bytes = 4; ++ for (i = 0; i < src_bytes; i++) ++ v |= ((u32)src[i]) << ((3 - i) * 8); ++ return v; ++} ++ ++static void intel_dp_unpack_aux(u32 src, u8 *dst, int dst_bytes) ++{ ++ int i; ++ if (dst_bytes > 4) ++ dst_bytes = 4; ++ for (i = 0; i < dst_bytes; i++) ++ dst[i] = src >> ((3-i) * 8); ++} ++ ++static void ++intel_dp_init_panel_power_sequencer(struct intel_dp *intel_dp); ++static void ++intel_dp_init_panel_power_sequencer_registers(struct intel_dp *intel_dp, ++ bool force_disable_vdd); ++static void ++intel_dp_pps_init(struct intel_dp *intel_dp); ++ ++static intel_wakeref_t ++pps_lock(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ intel_wakeref_t wakeref; ++ ++ /* ++ * See intel_power_sequencer_reset() why we need ++ * a power domain reference here. ++ */ ++ wakeref = intel_display_power_get(dev_priv, ++ intel_aux_power_domain(dp_to_dig_port(intel_dp))); ++ ++ mutex_lock(&dev_priv->pps_mutex); ++ ++ return wakeref; ++} ++ ++static intel_wakeref_t ++pps_unlock(struct intel_dp *intel_dp, intel_wakeref_t wakeref) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ ++ mutex_unlock(&dev_priv->pps_mutex); ++ intel_display_power_put(dev_priv, ++ intel_aux_power_domain(dp_to_dig_port(intel_dp)), ++ wakeref); ++ return 0; ++} ++ ++#define with_pps_lock(dp, wf) \ ++ for ((wf) = pps_lock(dp); (wf); (wf) = pps_unlock((dp), (wf))) ++ ++static void ++vlv_power_sequencer_kick(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); ++ enum pipe pipe = intel_dp->pps_pipe; ++ bool pll_enabled, release_cl_override = false; ++ enum dpio_phy phy = DPIO_PHY(pipe); ++ enum dpio_channel ch = vlv_pipe_to_channel(pipe); ++ u32 DP; ++ ++ if (WARN(I915_READ(intel_dp->output_reg) & DP_PORT_EN, ++ "skipping pipe %c power sequencer kick due to port %c being active\n", ++ pipe_name(pipe), port_name(intel_dig_port->base.port))) ++ return; ++ ++ DRM_DEBUG_KMS("kicking pipe %c power sequencer for port %c\n", ++ pipe_name(pipe), port_name(intel_dig_port->base.port)); ++ ++ /* Preserve the BIOS-computed detected bit. This is ++ * supposed to be read-only. ++ */ ++ DP = I915_READ(intel_dp->output_reg) & DP_DETECTED; ++ DP |= DP_VOLTAGE_0_4 | DP_PRE_EMPHASIS_0; ++ DP |= DP_PORT_WIDTH(1); ++ DP |= DP_LINK_TRAIN_PAT_1; ++ ++ if (IS_CHERRYVIEW(dev_priv)) ++ DP |= DP_PIPE_SEL_CHV(pipe); ++ else ++ DP |= DP_PIPE_SEL(pipe); ++ ++ pll_enabled = I915_READ(DPLL(pipe)) & DPLL_VCO_ENABLE; ++ ++ /* ++ * The DPLL for the pipe must be enabled for this to work. ++ * So enable temporarily it if it's not already enabled. ++ */ ++ if (!pll_enabled) { ++ release_cl_override = IS_CHERRYVIEW(dev_priv) && ++ !chv_phy_powergate_ch(dev_priv, phy, ch, true); ++ ++ if (vlv_force_pll_on(dev_priv, pipe, IS_CHERRYVIEW(dev_priv) ? ++ &chv_dpll[0].dpll : &vlv_dpll[0].dpll)) { ++ DRM_ERROR("Failed to force on pll for pipe %c!\n", ++ pipe_name(pipe)); ++ return; ++ } ++ } ++ ++ /* ++ * Similar magic as in intel_dp_enable_port(). ++ * We _must_ do this port enable + disable trick ++ * to make this power sequencer lock onto the port. ++ * Otherwise even VDD force bit won't work. ++ */ ++ I915_WRITE(intel_dp->output_reg, DP); ++ POSTING_READ(intel_dp->output_reg); ++ ++ I915_WRITE(intel_dp->output_reg, DP | DP_PORT_EN); ++ POSTING_READ(intel_dp->output_reg); ++ ++ I915_WRITE(intel_dp->output_reg, DP & ~DP_PORT_EN); ++ POSTING_READ(intel_dp->output_reg); ++ ++ if (!pll_enabled) { ++ vlv_force_pll_off(dev_priv, pipe); ++ ++ if (release_cl_override) ++ chv_phy_powergate_ch(dev_priv, phy, ch, false); ++ } ++} ++ ++static enum pipe vlv_find_free_pps(struct drm_i915_private *dev_priv) ++{ ++ struct intel_encoder *encoder; ++ unsigned int pipes = (1 << PIPE_A) | (1 << PIPE_B); ++ ++ /* ++ * We don't have power sequencer currently. ++ * Pick one that's not used by other ports. ++ */ ++ for_each_intel_dp(&dev_priv->drm, encoder) { ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ ++ if (encoder->type == INTEL_OUTPUT_EDP) { ++ WARN_ON(intel_dp->active_pipe != INVALID_PIPE && ++ intel_dp->active_pipe != intel_dp->pps_pipe); ++ ++ if (intel_dp->pps_pipe != INVALID_PIPE) ++ pipes &= ~(1 << intel_dp->pps_pipe); ++ } else { ++ WARN_ON(intel_dp->pps_pipe != INVALID_PIPE); ++ ++ if (intel_dp->active_pipe != INVALID_PIPE) ++ pipes &= ~(1 << intel_dp->active_pipe); ++ } ++ } ++ ++ if (pipes == 0) ++ return INVALID_PIPE; ++ ++ return ffs(pipes) - 1; ++} ++ ++static enum pipe ++vlv_power_sequencer_pipe(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); ++ enum pipe pipe; ++ ++ lockdep_assert_held(&dev_priv->pps_mutex); ++ ++ /* We should never land here with regular DP ports */ ++ WARN_ON(!intel_dp_is_edp(intel_dp)); ++ ++ WARN_ON(intel_dp->active_pipe != INVALID_PIPE && ++ intel_dp->active_pipe != intel_dp->pps_pipe); ++ ++ if (intel_dp->pps_pipe != INVALID_PIPE) ++ return intel_dp->pps_pipe; ++ ++ pipe = vlv_find_free_pps(dev_priv); ++ ++ /* ++ * Didn't find one. This should not happen since there ++ * are two power sequencers and up to two eDP ports. ++ */ ++ if (WARN_ON(pipe == INVALID_PIPE)) ++ pipe = PIPE_A; ++ ++ vlv_steal_power_sequencer(dev_priv, pipe); ++ intel_dp->pps_pipe = pipe; ++ ++ DRM_DEBUG_KMS("picked pipe %c power sequencer for port %c\n", ++ pipe_name(intel_dp->pps_pipe), ++ port_name(intel_dig_port->base.port)); ++ ++ /* init power sequencer on this pipe and port */ ++ intel_dp_init_panel_power_sequencer(intel_dp); ++ intel_dp_init_panel_power_sequencer_registers(intel_dp, true); ++ ++ /* ++ * Even vdd force doesn't work until we've made ++ * the power sequencer lock in on the port. ++ */ ++ vlv_power_sequencer_kick(intel_dp); ++ ++ return intel_dp->pps_pipe; ++} ++ ++static int ++bxt_power_sequencer_idx(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ int backlight_controller = dev_priv->vbt.backlight.controller; ++ ++ lockdep_assert_held(&dev_priv->pps_mutex); ++ ++ /* We should never land here with regular DP ports */ ++ WARN_ON(!intel_dp_is_edp(intel_dp)); ++ ++ if (!intel_dp->pps_reset) ++ return backlight_controller; ++ ++ intel_dp->pps_reset = false; ++ ++ /* ++ * Only the HW needs to be reprogrammed, the SW state is fixed and ++ * has been setup during connector init. ++ */ ++ intel_dp_init_panel_power_sequencer_registers(intel_dp, false); ++ ++ return backlight_controller; ++} ++ ++typedef bool (*vlv_pipe_check)(struct drm_i915_private *dev_priv, ++ enum pipe pipe); ++ ++static bool vlv_pipe_has_pp_on(struct drm_i915_private *dev_priv, ++ enum pipe pipe) ++{ ++ return I915_READ(PP_STATUS(pipe)) & PP_ON; ++} ++ ++static bool vlv_pipe_has_vdd_on(struct drm_i915_private *dev_priv, ++ enum pipe pipe) ++{ ++ return I915_READ(PP_CONTROL(pipe)) & EDP_FORCE_VDD; ++} ++ ++static bool vlv_pipe_any(struct drm_i915_private *dev_priv, ++ enum pipe pipe) ++{ ++ return true; ++} ++ ++static enum pipe ++vlv_initial_pps_pipe(struct drm_i915_private *dev_priv, ++ enum port port, ++ vlv_pipe_check pipe_check) ++{ ++ enum pipe pipe; ++ ++ for (pipe = PIPE_A; pipe <= PIPE_B; pipe++) { ++ u32 port_sel = I915_READ(PP_ON_DELAYS(pipe)) & ++ PANEL_PORT_SELECT_MASK; ++ ++ if (port_sel != PANEL_PORT_SELECT_VLV(port)) ++ continue; ++ ++ if (!pipe_check(dev_priv, pipe)) ++ continue; ++ ++ return pipe; ++ } ++ ++ return INVALID_PIPE; ++} ++ ++static void ++vlv_initial_power_sequencer_setup(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); ++ enum port port = intel_dig_port->base.port; ++ ++ lockdep_assert_held(&dev_priv->pps_mutex); ++ ++ /* try to find a pipe with this port selected */ ++ /* first pick one where the panel is on */ ++ intel_dp->pps_pipe = vlv_initial_pps_pipe(dev_priv, port, ++ vlv_pipe_has_pp_on); ++ /* didn't find one? pick one where vdd is on */ ++ if (intel_dp->pps_pipe == INVALID_PIPE) ++ intel_dp->pps_pipe = vlv_initial_pps_pipe(dev_priv, port, ++ vlv_pipe_has_vdd_on); ++ /* didn't find one? pick one with just the correct port */ ++ if (intel_dp->pps_pipe == INVALID_PIPE) ++ intel_dp->pps_pipe = vlv_initial_pps_pipe(dev_priv, port, ++ vlv_pipe_any); ++ ++ /* didn't find one? just let vlv_power_sequencer_pipe() pick one when needed */ ++ if (intel_dp->pps_pipe == INVALID_PIPE) { ++ DRM_DEBUG_KMS("no initial power sequencer for port %c\n", ++ port_name(port)); ++ return; ++ } ++ ++ DRM_DEBUG_KMS("initial power sequencer for port %c: pipe %c\n", ++ port_name(port), pipe_name(intel_dp->pps_pipe)); ++ ++ intel_dp_init_panel_power_sequencer(intel_dp); ++ intel_dp_init_panel_power_sequencer_registers(intel_dp, false); ++} ++ ++void intel_power_sequencer_reset(struct drm_i915_private *dev_priv) ++{ ++ struct intel_encoder *encoder; ++ ++ if (WARN_ON(!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) && ++ !IS_GEN9_LP(dev_priv))) ++ return; ++ ++ /* ++ * We can't grab pps_mutex here due to deadlock with power_domain ++ * mutex when power_domain functions are called while holding pps_mutex. ++ * That also means that in order to use pps_pipe the code needs to ++ * hold both a power domain reference and pps_mutex, and the power domain ++ * reference get/put must be done while _not_ holding pps_mutex. ++ * pps_{lock,unlock}() do these steps in the correct order, so one ++ * should use them always. ++ */ ++ ++ for_each_intel_dp(&dev_priv->drm, encoder) { ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ ++ WARN_ON(intel_dp->active_pipe != INVALID_PIPE); ++ ++ if (encoder->type != INTEL_OUTPUT_EDP) ++ continue; ++ ++ if (IS_GEN9_LP(dev_priv)) ++ intel_dp->pps_reset = true; ++ else ++ intel_dp->pps_pipe = INVALID_PIPE; ++ } ++} ++ ++struct pps_registers { ++ i915_reg_t pp_ctrl; ++ i915_reg_t pp_stat; ++ i915_reg_t pp_on; ++ i915_reg_t pp_off; ++ i915_reg_t pp_div; ++}; ++ ++static void intel_pps_get_registers(struct intel_dp *intel_dp, ++ struct pps_registers *regs) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ int pps_idx = 0; ++ ++ memset(regs, 0, sizeof(*regs)); ++ ++ if (IS_GEN9_LP(dev_priv)) ++ pps_idx = bxt_power_sequencer_idx(intel_dp); ++ else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ pps_idx = vlv_power_sequencer_pipe(intel_dp); ++ ++ regs->pp_ctrl = PP_CONTROL(pps_idx); ++ regs->pp_stat = PP_STATUS(pps_idx); ++ regs->pp_on = PP_ON_DELAYS(pps_idx); ++ regs->pp_off = PP_OFF_DELAYS(pps_idx); ++ ++ /* Cycle delay moved from PP_DIVISOR to PP_CONTROL */ ++ if (IS_GEN9_LP(dev_priv) || INTEL_PCH_TYPE(dev_priv) >= PCH_CNP) ++ regs->pp_div = INVALID_MMIO_REG; ++ else ++ regs->pp_div = PP_DIVISOR(pps_idx); ++} ++ ++static i915_reg_t ++_pp_ctrl_reg(struct intel_dp *intel_dp) ++{ ++ struct pps_registers regs; ++ ++ intel_pps_get_registers(intel_dp, ®s); ++ ++ return regs.pp_ctrl; ++} ++ ++static i915_reg_t ++_pp_stat_reg(struct intel_dp *intel_dp) ++{ ++ struct pps_registers regs; ++ ++ intel_pps_get_registers(intel_dp, ®s); ++ ++ return regs.pp_stat; ++} ++ ++/* Reboot notifier handler to shutdown panel power to guarantee T12 timing ++ This function only applicable when panel PM state is not to be tracked */ ++static int edp_notify_handler(struct notifier_block *this, unsigned long code, ++ void *unused) ++{ ++ struct intel_dp *intel_dp = container_of(this, typeof(* intel_dp), ++ edp_notifier); ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ intel_wakeref_t wakeref; ++ ++ if (!intel_dp_is_edp(intel_dp) || code != SYS_RESTART) ++ return 0; ++ ++ with_pps_lock(intel_dp, wakeref) { ++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { ++ enum pipe pipe = vlv_power_sequencer_pipe(intel_dp); ++ i915_reg_t pp_ctrl_reg, pp_div_reg; ++ u32 pp_div; ++ ++ pp_ctrl_reg = PP_CONTROL(pipe); ++ pp_div_reg = PP_DIVISOR(pipe); ++ pp_div = I915_READ(pp_div_reg); ++ pp_div &= PP_REFERENCE_DIVIDER_MASK; ++ ++ /* 0x1F write to PP_DIV_REG sets max cycle delay */ ++ I915_WRITE(pp_div_reg, pp_div | 0x1F); ++ I915_WRITE(pp_ctrl_reg, PANEL_UNLOCK_REGS); ++ msleep(intel_dp->panel_power_cycle_delay); ++ } ++ } ++ ++ return 0; ++} ++ ++static bool edp_have_panel_power(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ ++ lockdep_assert_held(&dev_priv->pps_mutex); ++ ++ if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) && ++ intel_dp->pps_pipe == INVALID_PIPE) ++ return false; ++ ++ return (I915_READ(_pp_stat_reg(intel_dp)) & PP_ON) != 0; ++} ++ ++static bool edp_have_panel_vdd(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ ++ lockdep_assert_held(&dev_priv->pps_mutex); ++ ++ if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) && ++ intel_dp->pps_pipe == INVALID_PIPE) ++ return false; ++ ++ return I915_READ(_pp_ctrl_reg(intel_dp)) & EDP_FORCE_VDD; ++} ++ ++static void ++intel_dp_check_edp(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ ++ if (!intel_dp_is_edp(intel_dp)) ++ return; ++ ++ if (!edp_have_panel_power(intel_dp) && !edp_have_panel_vdd(intel_dp)) { ++ WARN(1, "eDP powered off while attempting aux channel communication.\n"); ++ DRM_DEBUG_KMS("Status 0x%08x Control 0x%08x\n", ++ I915_READ(_pp_stat_reg(intel_dp)), ++ I915_READ(_pp_ctrl_reg(intel_dp))); ++ } ++} ++ ++static u32 ++intel_dp_aux_wait_done(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ i915_reg_t ch_ctl = intel_dp->aux_ch_ctl_reg(intel_dp); ++ u32 status; ++ bool done; ++ ++#define C (((status = I915_READ_NOTRACE(ch_ctl)) & DP_AUX_CH_CTL_SEND_BUSY) == 0) ++ done = wait_event_timeout(dev_priv->gmbus_wait_queue, C, ++ msecs_to_jiffies_timeout(10)); ++ ++ /* just trace the final value */ ++ trace_i915_reg_rw(false, ch_ctl, status, sizeof(status), true); ++ ++ if (!done) ++ DRM_ERROR("dp aux hw did not signal timeout!\n"); ++#undef C ++ ++ return status; ++} ++ ++static u32 g4x_get_aux_clock_divider(struct intel_dp *intel_dp, int index) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ ++ if (index) ++ return 0; ++ ++ /* ++ * The clock divider is based off the hrawclk, and would like to run at ++ * 2MHz. So, take the hrawclk value and divide by 2000 and use that ++ */ ++ return DIV_ROUND_CLOSEST(dev_priv->rawclk_freq, 2000); ++} ++ ++static u32 ilk_get_aux_clock_divider(struct intel_dp *intel_dp, int index) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); ++ ++ if (index) ++ return 0; ++ ++ /* ++ * The clock divider is based off the cdclk or PCH rawclk, and would ++ * like to run at 2MHz. So, take the cdclk or PCH rawclk value and ++ * divide by 2000 and use that ++ */ ++ if (dig_port->aux_ch == AUX_CH_A) ++ return DIV_ROUND_CLOSEST(dev_priv->cdclk.hw.cdclk, 2000); ++ else ++ return DIV_ROUND_CLOSEST(dev_priv->rawclk_freq, 2000); ++} ++ ++static u32 hsw_get_aux_clock_divider(struct intel_dp *intel_dp, int index) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); ++ ++ if (dig_port->aux_ch != AUX_CH_A && HAS_PCH_LPT_H(dev_priv)) { ++ /* Workaround for non-ULT HSW */ ++ switch (index) { ++ case 0: return 63; ++ case 1: return 72; ++ default: return 0; ++ } ++ } ++ ++ return ilk_get_aux_clock_divider(intel_dp, index); ++} ++ ++static u32 skl_get_aux_clock_divider(struct intel_dp *intel_dp, int index) ++{ ++ /* ++ * SKL doesn't need us to program the AUX clock divider (Hardware will ++ * derive the clock from CDCLK automatically). We still implement the ++ * get_aux_clock_divider vfunc to plug-in into the existing code. ++ */ ++ return index ? 0 : 1; ++} ++ ++static u32 g4x_get_aux_send_ctl(struct intel_dp *intel_dp, ++ int send_bytes, ++ u32 aux_clock_divider) ++{ ++ struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); ++ struct drm_i915_private *dev_priv = ++ to_i915(intel_dig_port->base.base.dev); ++ u32 precharge, timeout; ++ ++ if (IS_GEN(dev_priv, 6)) ++ precharge = 3; ++ else ++ precharge = 5; ++ ++ if (IS_BROADWELL(dev_priv)) ++ timeout = DP_AUX_CH_CTL_TIME_OUT_600us; ++ else ++ timeout = DP_AUX_CH_CTL_TIME_OUT_400us; ++ ++ return DP_AUX_CH_CTL_SEND_BUSY | ++ DP_AUX_CH_CTL_DONE | ++ DP_AUX_CH_CTL_INTERRUPT | ++ DP_AUX_CH_CTL_TIME_OUT_ERROR | ++ timeout | ++ DP_AUX_CH_CTL_RECEIVE_ERROR | ++ (send_bytes << DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT) | ++ (precharge << DP_AUX_CH_CTL_PRECHARGE_2US_SHIFT) | ++ (aux_clock_divider << DP_AUX_CH_CTL_BIT_CLOCK_2X_SHIFT); ++} ++ ++static u32 skl_get_aux_send_ctl(struct intel_dp *intel_dp, ++ int send_bytes, ++ u32 unused) ++{ ++ struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); ++ u32 ret; ++ ++ ret = DP_AUX_CH_CTL_SEND_BUSY | ++ DP_AUX_CH_CTL_DONE | ++ DP_AUX_CH_CTL_INTERRUPT | ++ DP_AUX_CH_CTL_TIME_OUT_ERROR | ++ DP_AUX_CH_CTL_TIME_OUT_MAX | ++ DP_AUX_CH_CTL_RECEIVE_ERROR | ++ (send_bytes << DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT) | ++ DP_AUX_CH_CTL_FW_SYNC_PULSE_SKL(32) | ++ DP_AUX_CH_CTL_SYNC_PULSE_SKL(32); ++ ++ if (intel_dig_port->tc_type == TC_PORT_TBT) ++ ret |= DP_AUX_CH_CTL_TBT_IO; ++ ++ return ret; ++} ++ ++static int ++intel_dp_aux_xfer(struct intel_dp *intel_dp, ++ const u8 *send, int send_bytes, ++ u8 *recv, int recv_size, ++ u32 aux_send_ctl_flags) ++{ ++ struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); ++ struct drm_i915_private *dev_priv = ++ to_i915(intel_dig_port->base.base.dev); ++ i915_reg_t ch_ctl, ch_data[5]; ++ u32 aux_clock_divider; ++ intel_wakeref_t wakeref; ++ int i, ret, recv_bytes; ++ int try, clock = 0; ++ u32 status; ++ bool vdd; ++ ++ ch_ctl = intel_dp->aux_ch_ctl_reg(intel_dp); ++ for (i = 0; i < ARRAY_SIZE(ch_data); i++) ++ ch_data[i] = intel_dp->aux_ch_data_reg(intel_dp, i); ++ ++ wakeref = pps_lock(intel_dp); ++ ++ /* ++ * We will be called with VDD already enabled for dpcd/edid/oui reads. ++ * In such cases we want to leave VDD enabled and it's up to upper layers ++ * to turn it off. But for eg. i2c-dev access we need to turn it on/off ++ * ourselves. ++ */ ++ vdd = edp_panel_vdd_on(intel_dp); ++ ++ /* dp aux is extremely sensitive to irq latency, hence request the ++ * lowest possible wakeup latency and so prevent the cpu from going into ++ * deep sleep states. ++ */ ++ pm_qos_update_request(&dev_priv->pm_qos, 0); ++ ++ intel_dp_check_edp(intel_dp); ++ ++ /* Try to wait for any previous AUX channel activity */ ++ for (try = 0; try < 3; try++) { ++ status = I915_READ_NOTRACE(ch_ctl); ++ if ((status & DP_AUX_CH_CTL_SEND_BUSY) == 0) ++ break; ++ msleep(1); ++ } ++ /* just trace the final value */ ++ trace_i915_reg_rw(false, ch_ctl, status, sizeof(status), true); ++ ++ if (try == 3) { ++ static u32 last_status = -1; ++ const u32 status = I915_READ(ch_ctl); ++ ++ if (status != last_status) { ++ WARN(1, "dp_aux_ch not started status 0x%08x\n", ++ status); ++ last_status = status; ++ } ++ ++ ret = -EBUSY; ++ goto out; ++ } ++ ++ /* Only 5 data registers! */ ++ if (WARN_ON(send_bytes > 20 || recv_size > 20)) { ++ ret = -E2BIG; ++ goto out; ++ } ++ ++ while ((aux_clock_divider = intel_dp->get_aux_clock_divider(intel_dp, clock++))) { ++ u32 send_ctl = intel_dp->get_aux_send_ctl(intel_dp, ++ send_bytes, ++ aux_clock_divider); ++ ++ send_ctl |= aux_send_ctl_flags; ++ ++ /* Must try at least 3 times according to DP spec */ ++ for (try = 0; try < 5; try++) { ++ /* Load the send data into the aux channel data registers */ ++ for (i = 0; i < send_bytes; i += 4) ++ I915_WRITE(ch_data[i >> 2], ++ intel_dp_pack_aux(send + i, ++ send_bytes - i)); ++ ++ /* Send the command and wait for it to complete */ ++ I915_WRITE(ch_ctl, send_ctl); ++ ++ status = intel_dp_aux_wait_done(intel_dp); ++ ++ /* Clear done status and any errors */ ++ I915_WRITE(ch_ctl, ++ status | ++ DP_AUX_CH_CTL_DONE | ++ DP_AUX_CH_CTL_TIME_OUT_ERROR | ++ DP_AUX_CH_CTL_RECEIVE_ERROR); ++ ++ /* DP CTS 1.2 Core Rev 1.1, 4.2.1.1 & 4.2.1.2 ++ * 400us delay required for errors and timeouts ++ * Timeout errors from the HW already meet this ++ * requirement so skip to next iteration ++ */ ++ if (status & DP_AUX_CH_CTL_TIME_OUT_ERROR) ++ continue; ++ ++ if (status & DP_AUX_CH_CTL_RECEIVE_ERROR) { ++ usleep_range(400, 500); ++ continue; ++ } ++ if (status & DP_AUX_CH_CTL_DONE) ++ goto done; ++ } ++ } ++ ++ if ((status & DP_AUX_CH_CTL_DONE) == 0) { ++ DRM_ERROR("dp_aux_ch not done status 0x%08x\n", status); ++ ret = -EBUSY; ++ goto out; ++ } ++ ++done: ++ /* Check for timeout or receive error. ++ * Timeouts occur when the sink is not connected ++ */ ++ if (status & DP_AUX_CH_CTL_RECEIVE_ERROR) { ++ DRM_ERROR("dp_aux_ch receive error status 0x%08x\n", status); ++ ret = -EIO; ++ goto out; ++ } ++ ++ /* Timeouts occur when the device isn't connected, so they're ++ * "normal" -- don't fill the kernel log with these */ ++ if (status & DP_AUX_CH_CTL_TIME_OUT_ERROR) { ++ DRM_DEBUG_KMS("dp_aux_ch timeout status 0x%08x\n", status); ++ ret = -ETIMEDOUT; ++ goto out; ++ } ++ ++ /* Unload any bytes sent back from the other side */ ++ recv_bytes = ((status & DP_AUX_CH_CTL_MESSAGE_SIZE_MASK) >> ++ DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT); ++ ++ /* ++ * By BSpec: "Message sizes of 0 or >20 are not allowed." ++ * We have no idea of what happened so we return -EBUSY so ++ * drm layer takes care for the necessary retries. ++ */ ++ if (recv_bytes == 0 || recv_bytes > 20) { ++ DRM_DEBUG_KMS("Forbidden recv_bytes = %d on aux transaction\n", ++ recv_bytes); ++ ret = -EBUSY; ++ goto out; ++ } ++ ++ if (recv_bytes > recv_size) ++ recv_bytes = recv_size; ++ ++ for (i = 0; i < recv_bytes; i += 4) ++ intel_dp_unpack_aux(I915_READ(ch_data[i >> 2]), ++ recv + i, recv_bytes - i); ++ ++ ret = recv_bytes; ++out: ++ pm_qos_update_request(&dev_priv->pm_qos, PM_QOS_DEFAULT_VALUE); ++ ++ if (vdd) ++ edp_panel_vdd_off(intel_dp, false); ++ ++ pps_unlock(intel_dp, wakeref); ++ ++ return ret; ++} ++ ++#define BARE_ADDRESS_SIZE 3 ++#define HEADER_SIZE (BARE_ADDRESS_SIZE + 1) ++ ++static void ++intel_dp_aux_header(u8 txbuf[HEADER_SIZE], ++ const struct drm_dp_aux_msg *msg) ++{ ++ txbuf[0] = (msg->request << 4) | ((msg->address >> 16) & 0xf); ++ txbuf[1] = (msg->address >> 8) & 0xff; ++ txbuf[2] = msg->address & 0xff; ++ txbuf[3] = msg->size - 1; ++} ++ ++static ssize_t ++intel_dp_aux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg) ++{ ++ struct intel_dp *intel_dp = container_of(aux, struct intel_dp, aux); ++ u8 txbuf[20], rxbuf[20]; ++ size_t txsize, rxsize; ++ int ret; ++ ++ intel_dp_aux_header(txbuf, msg); ++ ++ switch (msg->request & ~DP_AUX_I2C_MOT) { ++ case DP_AUX_NATIVE_WRITE: ++ case DP_AUX_I2C_WRITE: ++ case DP_AUX_I2C_WRITE_STATUS_UPDATE: ++ txsize = msg->size ? HEADER_SIZE + msg->size : BARE_ADDRESS_SIZE; ++ rxsize = 2; /* 0 or 1 data bytes */ ++ ++ if (WARN_ON(txsize > 20)) ++ return -E2BIG; ++ ++ WARN_ON(!msg->buffer != !msg->size); ++ ++ if (msg->buffer) ++ memcpy(txbuf + HEADER_SIZE, msg->buffer, msg->size); ++ ++ ret = intel_dp_aux_xfer(intel_dp, txbuf, txsize, ++ rxbuf, rxsize, 0); ++ if (ret > 0) { ++ msg->reply = rxbuf[0] >> 4; ++ ++ if (ret > 1) { ++ /* Number of bytes written in a short write. */ ++ ret = clamp_t(int, rxbuf[1], 0, msg->size); ++ } else { ++ /* Return payload size. */ ++ ret = msg->size; ++ } ++ } ++ break; ++ ++ case DP_AUX_NATIVE_READ: ++ case DP_AUX_I2C_READ: ++ txsize = msg->size ? HEADER_SIZE : BARE_ADDRESS_SIZE; ++ rxsize = msg->size + 1; ++ ++ if (WARN_ON(rxsize > 20)) ++ return -E2BIG; ++ ++ ret = intel_dp_aux_xfer(intel_dp, txbuf, txsize, ++ rxbuf, rxsize, 0); ++ if (ret > 0) { ++ msg->reply = rxbuf[0] >> 4; ++ /* ++ * Assume happy day, and copy the data. The caller is ++ * expected to check msg->reply before touching it. ++ * ++ * Return payload size. ++ */ ++ ret--; ++ memcpy(msg->buffer, rxbuf + 1, ret); ++ } ++ break; ++ ++ default: ++ ret = -EINVAL; ++ break; ++ } ++ ++ return ret; ++} ++ ++ ++static i915_reg_t g4x_aux_ctl_reg(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); ++ enum aux_ch aux_ch = dig_port->aux_ch; ++ ++ switch (aux_ch) { ++ case AUX_CH_B: ++ case AUX_CH_C: ++ case AUX_CH_D: ++ return DP_AUX_CH_CTL(aux_ch); ++ default: ++ MISSING_CASE(aux_ch); ++ return DP_AUX_CH_CTL(AUX_CH_B); ++ } ++} ++ ++static i915_reg_t g4x_aux_data_reg(struct intel_dp *intel_dp, int index) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); ++ enum aux_ch aux_ch = dig_port->aux_ch; ++ ++ switch (aux_ch) { ++ case AUX_CH_B: ++ case AUX_CH_C: ++ case AUX_CH_D: ++ return DP_AUX_CH_DATA(aux_ch, index); ++ default: ++ MISSING_CASE(aux_ch); ++ return DP_AUX_CH_DATA(AUX_CH_B, index); ++ } ++} ++ ++static i915_reg_t ilk_aux_ctl_reg(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); ++ enum aux_ch aux_ch = dig_port->aux_ch; ++ ++ switch (aux_ch) { ++ case AUX_CH_A: ++ return DP_AUX_CH_CTL(aux_ch); ++ case AUX_CH_B: ++ case AUX_CH_C: ++ case AUX_CH_D: ++ return PCH_DP_AUX_CH_CTL(aux_ch); ++ default: ++ MISSING_CASE(aux_ch); ++ return DP_AUX_CH_CTL(AUX_CH_A); ++ } ++} ++ ++static i915_reg_t ilk_aux_data_reg(struct intel_dp *intel_dp, int index) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); ++ enum aux_ch aux_ch = dig_port->aux_ch; ++ ++ switch (aux_ch) { ++ case AUX_CH_A: ++ return DP_AUX_CH_DATA(aux_ch, index); ++ case AUX_CH_B: ++ case AUX_CH_C: ++ case AUX_CH_D: ++ return PCH_DP_AUX_CH_DATA(aux_ch, index); ++ default: ++ MISSING_CASE(aux_ch); ++ return DP_AUX_CH_DATA(AUX_CH_A, index); ++ } ++} ++ ++static i915_reg_t skl_aux_ctl_reg(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); ++ enum aux_ch aux_ch = dig_port->aux_ch; ++ ++ switch (aux_ch) { ++ case AUX_CH_A: ++ case AUX_CH_B: ++ case AUX_CH_C: ++ case AUX_CH_D: ++ case AUX_CH_E: ++ case AUX_CH_F: ++ return DP_AUX_CH_CTL(aux_ch); ++ default: ++ MISSING_CASE(aux_ch); ++ return DP_AUX_CH_CTL(AUX_CH_A); ++ } ++} ++ ++static i915_reg_t skl_aux_data_reg(struct intel_dp *intel_dp, int index) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); ++ enum aux_ch aux_ch = dig_port->aux_ch; ++ ++ switch (aux_ch) { ++ case AUX_CH_A: ++ case AUX_CH_B: ++ case AUX_CH_C: ++ case AUX_CH_D: ++ case AUX_CH_E: ++ case AUX_CH_F: ++ return DP_AUX_CH_DATA(aux_ch, index); ++ default: ++ MISSING_CASE(aux_ch); ++ return DP_AUX_CH_DATA(AUX_CH_A, index); ++ } ++} ++ ++static void ++intel_dp_aux_fini(struct intel_dp *intel_dp) ++{ ++ kfree(intel_dp->aux.name); ++} ++ ++static void ++intel_dp_aux_init(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); ++ struct intel_encoder *encoder = &dig_port->base; ++ ++ if (INTEL_GEN(dev_priv) >= 9) { ++ intel_dp->aux_ch_ctl_reg = skl_aux_ctl_reg; ++ intel_dp->aux_ch_data_reg = skl_aux_data_reg; ++ } else if (HAS_PCH_SPLIT(dev_priv)) { ++ intel_dp->aux_ch_ctl_reg = ilk_aux_ctl_reg; ++ intel_dp->aux_ch_data_reg = ilk_aux_data_reg; ++ } else { ++ intel_dp->aux_ch_ctl_reg = g4x_aux_ctl_reg; ++ intel_dp->aux_ch_data_reg = g4x_aux_data_reg; ++ } ++ ++ if (INTEL_GEN(dev_priv) >= 9) ++ intel_dp->get_aux_clock_divider = skl_get_aux_clock_divider; ++ else if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) ++ intel_dp->get_aux_clock_divider = hsw_get_aux_clock_divider; ++ else if (HAS_PCH_SPLIT(dev_priv)) ++ intel_dp->get_aux_clock_divider = ilk_get_aux_clock_divider; ++ else ++ intel_dp->get_aux_clock_divider = g4x_get_aux_clock_divider; ++ ++ if (INTEL_GEN(dev_priv) >= 9) ++ intel_dp->get_aux_send_ctl = skl_get_aux_send_ctl; ++ else ++ intel_dp->get_aux_send_ctl = g4x_get_aux_send_ctl; ++ ++ drm_dp_aux_init(&intel_dp->aux); ++ ++ /* Failure to allocate our preferred name is not critical */ ++ intel_dp->aux.name = kasprintf(GFP_KERNEL, "DPDDC-%c", ++ port_name(encoder->port)); ++ intel_dp->aux.transfer = intel_dp_aux_transfer; ++} ++ ++bool intel_dp_source_supports_hbr2(struct intel_dp *intel_dp) ++{ ++ int max_rate = intel_dp->source_rates[intel_dp->num_source_rates - 1]; ++ ++ return max_rate >= 540000; ++} ++ ++bool intel_dp_source_supports_hbr3(struct intel_dp *intel_dp) ++{ ++ int max_rate = intel_dp->source_rates[intel_dp->num_source_rates - 1]; ++ ++ return max_rate >= 810000; ++} ++ ++static void ++intel_dp_set_clock(struct intel_encoder *encoder, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ const struct dp_link_dpll *divisor = NULL; ++ int i, count = 0; ++ ++ if (IS_G4X(dev_priv)) { ++ divisor = g4x_dpll; ++ count = ARRAY_SIZE(g4x_dpll); ++ } else if (HAS_PCH_SPLIT(dev_priv)) { ++ divisor = pch_dpll; ++ count = ARRAY_SIZE(pch_dpll); ++ } else if (IS_CHERRYVIEW(dev_priv)) { ++ divisor = chv_dpll; ++ count = ARRAY_SIZE(chv_dpll); ++ } else if (IS_VALLEYVIEW(dev_priv)) { ++ divisor = vlv_dpll; ++ count = ARRAY_SIZE(vlv_dpll); ++ } ++ ++ if (divisor && count) { ++ for (i = 0; i < count; i++) { ++ if (pipe_config->port_clock == divisor[i].clock) { ++ pipe_config->dpll = divisor[i].dpll; ++ pipe_config->clock_set = true; ++ break; ++ } ++ } ++ } ++} ++ ++static void snprintf_int_array(char *str, size_t len, ++ const int *array, int nelem) ++{ ++ int i; ++ ++ str[0] = '\0'; ++ ++ for (i = 0; i < nelem; i++) { ++ int r = snprintf(str, len, "%s%d", i ? ", " : "", array[i]); ++ if (r >= len) ++ return; ++ str += r; ++ len -= r; ++ } ++} ++ ++static void intel_dp_print_rates(struct intel_dp *intel_dp) ++{ ++ char str[128]; /* FIXME: too big for stack? */ ++ ++ if ((drm_debug & DRM_UT_KMS) == 0) ++ return; ++ ++ snprintf_int_array(str, sizeof(str), ++ intel_dp->source_rates, intel_dp->num_source_rates); ++ DRM_DEBUG_KMS("source rates: %s\n", str); ++ ++ snprintf_int_array(str, sizeof(str), ++ intel_dp->sink_rates, intel_dp->num_sink_rates); ++ DRM_DEBUG_KMS("sink rates: %s\n", str); ++ ++ snprintf_int_array(str, sizeof(str), ++ intel_dp->common_rates, intel_dp->num_common_rates); ++ DRM_DEBUG_KMS("common rates: %s\n", str); ++} ++ ++int ++intel_dp_max_link_rate(struct intel_dp *intel_dp) ++{ ++ int len; ++ ++ len = intel_dp_common_len_rate_limit(intel_dp, intel_dp->max_link_rate); ++ if (WARN_ON(len <= 0)) ++ return 162000; ++ ++ return intel_dp->common_rates[len - 1]; ++} ++ ++int intel_dp_rate_select(struct intel_dp *intel_dp, int rate) ++{ ++ int i = intel_dp_rate_index(intel_dp->sink_rates, ++ intel_dp->num_sink_rates, rate); ++ ++ if (WARN_ON(i < 0)) ++ i = 0; ++ ++ return i; ++} ++ ++void intel_dp_compute_rate(struct intel_dp *intel_dp, int port_clock, ++ u8 *link_bw, u8 *rate_select) ++{ ++ /* eDP 1.4 rate select method. */ ++ if (intel_dp->use_rate_select) { ++ *link_bw = 0; ++ *rate_select = ++ intel_dp_rate_select(intel_dp, port_clock); ++ } else { ++ *link_bw = drm_dp_link_rate_to_bw_code(port_clock); ++ *rate_select = 0; ++ } ++} ++ ++static bool intel_dp_source_supports_fec(struct intel_dp *intel_dp, ++ const struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ ++ return INTEL_GEN(dev_priv) >= 11 && ++ pipe_config->cpu_transcoder != TRANSCODER_A; ++} ++ ++static bool intel_dp_supports_fec(struct intel_dp *intel_dp, ++ const struct intel_crtc_state *pipe_config) ++{ ++ return intel_dp_source_supports_fec(intel_dp, pipe_config) && ++ drm_dp_sink_supports_fec(intel_dp->fec_capable); ++} ++ ++static bool intel_dp_source_supports_dsc(struct intel_dp *intel_dp, ++ const struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ ++ return INTEL_GEN(dev_priv) >= 10 && ++ pipe_config->cpu_transcoder != TRANSCODER_A; ++} ++ ++static bool intel_dp_supports_dsc(struct intel_dp *intel_dp, ++ const struct intel_crtc_state *pipe_config) ++{ ++ if (!intel_dp_is_edp(intel_dp) && !pipe_config->fec_enable) ++ return false; ++ ++ return intel_dp_source_supports_dsc(intel_dp, pipe_config) && ++ drm_dp_sink_supports_dsc(intel_dp->dsc_dpcd); ++} ++ ++static int intel_dp_compute_bpp(struct intel_dp *intel_dp, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ struct intel_connector *intel_connector = intel_dp->attached_connector; ++ int bpp, bpc; ++ ++ bpp = pipe_config->pipe_bpp; ++ bpc = drm_dp_downstream_max_bpc(intel_dp->dpcd, intel_dp->downstream_ports); ++ ++ if (bpc > 0) ++ bpp = min(bpp, 3*bpc); ++ ++ if (intel_dp_is_edp(intel_dp)) { ++ /* Get bpp from vbt only for panels that dont have bpp in edid */ ++ if (intel_connector->base.display_info.bpc == 0 && ++ dev_priv->vbt.edp.bpp && dev_priv->vbt.edp.bpp < bpp) { ++ DRM_DEBUG_KMS("clamping bpp for eDP panel to BIOS-provided %i\n", ++ dev_priv->vbt.edp.bpp); ++ bpp = dev_priv->vbt.edp.bpp; ++ } ++ } ++ ++ return bpp; ++} ++ ++/* Adjust link config limits based on compliance test requests. */ ++void ++intel_dp_adjust_compliance_config(struct intel_dp *intel_dp, ++ struct intel_crtc_state *pipe_config, ++ struct link_config_limits *limits) ++{ ++ /* For DP Compliance we override the computed bpp for the pipe */ ++ if (intel_dp->compliance.test_data.bpc != 0) { ++ int bpp = 3 * intel_dp->compliance.test_data.bpc; ++ ++ limits->min_bpp = limits->max_bpp = bpp; ++ pipe_config->dither_force_disable = bpp == 6 * 3; ++ ++ DRM_DEBUG_KMS("Setting pipe_bpp to %d\n", bpp); ++ } ++ ++ /* Use values requested by Compliance Test Request */ ++ if (intel_dp->compliance.test_type == DP_TEST_LINK_TRAINING) { ++ int index; ++ ++ /* Validate the compliance test data since max values ++ * might have changed due to link train fallback. ++ */ ++ if (intel_dp_link_params_valid(intel_dp, intel_dp->compliance.test_link_rate, ++ intel_dp->compliance.test_lane_count)) { ++ index = intel_dp_rate_index(intel_dp->common_rates, ++ intel_dp->num_common_rates, ++ intel_dp->compliance.test_link_rate); ++ if (index >= 0) ++ limits->min_clock = limits->max_clock = index; ++ limits->min_lane_count = limits->max_lane_count = ++ intel_dp->compliance.test_lane_count; ++ } ++ } ++} ++ ++/* Optimize link config in order: max bpp, min clock, min lanes */ ++static int ++intel_dp_compute_link_config_wide(struct intel_dp *intel_dp, ++ struct intel_crtc_state *pipe_config, ++ const struct link_config_limits *limits) ++{ ++ struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; ++ int bpp, clock, lane_count; ++ int mode_rate, link_clock, link_avail; ++ ++ for (bpp = limits->max_bpp; bpp >= limits->min_bpp; bpp -= 2 * 3) { ++ mode_rate = intel_dp_link_required(adjusted_mode->crtc_clock, ++ bpp); ++ ++ for (clock = limits->min_clock; clock <= limits->max_clock; clock++) { ++ for (lane_count = limits->min_lane_count; ++ lane_count <= limits->max_lane_count; ++ lane_count <<= 1) { ++ link_clock = intel_dp->common_rates[clock]; ++ link_avail = intel_dp_max_data_rate(link_clock, ++ lane_count); ++ ++ if (mode_rate <= link_avail) { ++ pipe_config->lane_count = lane_count; ++ pipe_config->pipe_bpp = bpp; ++ pipe_config->port_clock = link_clock; ++ ++ return 0; ++ } ++ } ++ } ++ } ++ ++ return -EINVAL; ++} ++ ++static int intel_dp_dsc_compute_bpp(struct intel_dp *intel_dp, u8 dsc_max_bpc) ++{ ++ int i, num_bpc; ++ u8 dsc_bpc[3] = {0}; ++ ++ num_bpc = drm_dp_dsc_sink_supported_input_bpcs(intel_dp->dsc_dpcd, ++ dsc_bpc); ++ for (i = 0; i < num_bpc; i++) { ++ if (dsc_max_bpc >= dsc_bpc[i]) ++ return dsc_bpc[i] * 3; ++ } ++ ++ return 0; ++} ++ ++static int intel_dp_dsc_compute_config(struct intel_dp *intel_dp, ++ struct intel_crtc_state *pipe_config, ++ struct drm_connector_state *conn_state, ++ struct link_config_limits *limits) ++{ ++ struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); ++ struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); ++ struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; ++ u8 dsc_max_bpc; ++ int pipe_bpp; ++ int ret; ++ ++ pipe_config->fec_enable = !intel_dp_is_edp(intel_dp) && ++ intel_dp_supports_fec(intel_dp, pipe_config); ++ ++ if (!intel_dp_supports_dsc(intel_dp, pipe_config)) ++ return -EINVAL; ++ ++ dsc_max_bpc = min_t(u8, DP_DSC_MAX_SUPPORTED_BPC, ++ conn_state->max_requested_bpc); ++ ++ pipe_bpp = intel_dp_dsc_compute_bpp(intel_dp, dsc_max_bpc); ++ if (pipe_bpp < DP_DSC_MIN_SUPPORTED_BPC * 3) { ++ DRM_DEBUG_KMS("No DSC support for less than 8bpc\n"); ++ return -EINVAL; ++ } ++ ++ /* ++ * For now enable DSC for max bpp, max link rate, max lane count. ++ * Optimize this later for the minimum possible link rate/lane count ++ * with DSC enabled for the requested mode. ++ */ ++ pipe_config->pipe_bpp = pipe_bpp; ++ pipe_config->port_clock = intel_dp->common_rates[limits->max_clock]; ++ pipe_config->lane_count = limits->max_lane_count; ++ ++ if (intel_dp_is_edp(intel_dp)) { ++ pipe_config->dsc_params.compressed_bpp = ++ min_t(u16, drm_edp_dsc_sink_output_bpp(intel_dp->dsc_dpcd) >> 4, ++ pipe_config->pipe_bpp); ++ pipe_config->dsc_params.slice_count = ++ drm_dp_dsc_sink_max_slice_count(intel_dp->dsc_dpcd, ++ true); ++ } else { ++ u16 dsc_max_output_bpp; ++ u8 dsc_dp_slice_count; ++ ++ dsc_max_output_bpp = ++ intel_dp_dsc_get_output_bpp(pipe_config->port_clock, ++ pipe_config->lane_count, ++ adjusted_mode->crtc_clock, ++ adjusted_mode->crtc_hdisplay); ++ dsc_dp_slice_count = ++ intel_dp_dsc_get_slice_count(intel_dp, ++ adjusted_mode->crtc_clock, ++ adjusted_mode->crtc_hdisplay); ++ if (!dsc_max_output_bpp || !dsc_dp_slice_count) { ++ DRM_DEBUG_KMS("Compressed BPP/Slice Count not supported\n"); ++ return -EINVAL; ++ } ++ pipe_config->dsc_params.compressed_bpp = min_t(u16, ++ dsc_max_output_bpp >> 4, ++ pipe_config->pipe_bpp); ++ pipe_config->dsc_params.slice_count = dsc_dp_slice_count; ++ } ++ /* ++ * VDSC engine operates at 1 Pixel per clock, so if peak pixel rate ++ * is greater than the maximum Cdclock and if slice count is even ++ * then we need to use 2 VDSC instances. ++ */ ++ if (adjusted_mode->crtc_clock > dev_priv->max_cdclk_freq) { ++ if (pipe_config->dsc_params.slice_count > 1) { ++ pipe_config->dsc_params.dsc_split = true; ++ } else { ++ DRM_DEBUG_KMS("Cannot split stream to use 2 VDSC instances\n"); ++ return -EINVAL; ++ } ++ } ++ ++ ret = intel_dp_compute_dsc_params(intel_dp, pipe_config); ++ if (ret < 0) { ++ DRM_DEBUG_KMS("Cannot compute valid DSC parameters for Input Bpp = %d " ++ "Compressed BPP = %d\n", ++ pipe_config->pipe_bpp, ++ pipe_config->dsc_params.compressed_bpp); ++ return ret; ++ } ++ ++ pipe_config->dsc_params.compression_enable = true; ++ DRM_DEBUG_KMS("DP DSC computed with Input Bpp = %d " ++ "Compressed Bpp = %d Slice Count = %d\n", ++ pipe_config->pipe_bpp, ++ pipe_config->dsc_params.compressed_bpp, ++ pipe_config->dsc_params.slice_count); ++ ++ return 0; ++} ++ ++int intel_dp_min_bpp(const struct intel_crtc_state *crtc_state) ++{ ++ if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_RGB) ++ return 6 * 3; ++ else ++ return 8 * 3; ++} ++ ++static int ++intel_dp_compute_link_config(struct intel_encoder *encoder, ++ struct intel_crtc_state *pipe_config, ++ struct drm_connector_state *conn_state) ++{ ++ struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ struct link_config_limits limits; ++ int common_len; ++ int ret; ++ ++ common_len = intel_dp_common_len_rate_limit(intel_dp, ++ intel_dp->max_link_rate); ++ ++ /* No common link rates between source and sink */ ++ WARN_ON(common_len <= 0); ++ ++ limits.min_clock = 0; ++ limits.max_clock = common_len - 1; ++ ++ limits.min_lane_count = 1; ++ limits.max_lane_count = intel_dp_max_lane_count(intel_dp); ++ ++ limits.min_bpp = intel_dp_min_bpp(pipe_config); ++ limits.max_bpp = intel_dp_compute_bpp(intel_dp, pipe_config); ++ ++ if (intel_dp_is_edp(intel_dp)) { ++ /* ++ * Use the maximum clock and number of lanes the eDP panel ++ * advertizes being capable of. The panels are generally ++ * designed to support only a single clock and lane ++ * configuration, and typically these values correspond to the ++ * native resolution of the panel. ++ */ ++ limits.min_lane_count = limits.max_lane_count; ++ limits.min_clock = limits.max_clock; ++ } ++ ++ intel_dp_adjust_compliance_config(intel_dp, pipe_config, &limits); ++ ++ DRM_DEBUG_KMS("DP link computation with max lane count %i " ++ "max rate %d max bpp %d pixel clock %iKHz\n", ++ limits.max_lane_count, ++ intel_dp->common_rates[limits.max_clock], ++ limits.max_bpp, adjusted_mode->crtc_clock); ++ ++ /* ++ * Optimize for slow and wide. This is the place to add alternative ++ * optimization policy. ++ */ ++ ret = intel_dp_compute_link_config_wide(intel_dp, pipe_config, &limits); ++ ++ /* enable compression if the mode doesn't fit available BW */ ++ DRM_DEBUG_KMS("Force DSC en = %d\n", intel_dp->force_dsc_en); ++ if (ret || intel_dp->force_dsc_en) { ++ ret = intel_dp_dsc_compute_config(intel_dp, pipe_config, ++ conn_state, &limits); ++ if (ret < 0) ++ return ret; ++ } ++ ++ if (pipe_config->dsc_params.compression_enable) { ++ DRM_DEBUG_KMS("DP lane count %d clock %d Input bpp %d Compressed bpp %d\n", ++ pipe_config->lane_count, pipe_config->port_clock, ++ pipe_config->pipe_bpp, ++ pipe_config->dsc_params.compressed_bpp); ++ ++ DRM_DEBUG_KMS("DP link rate required %i available %i\n", ++ intel_dp_link_required(adjusted_mode->crtc_clock, ++ pipe_config->dsc_params.compressed_bpp), ++ intel_dp_max_data_rate(pipe_config->port_clock, ++ pipe_config->lane_count)); ++ } else { ++ DRM_DEBUG_KMS("DP lane count %d clock %d bpp %d\n", ++ pipe_config->lane_count, pipe_config->port_clock, ++ pipe_config->pipe_bpp); ++ ++ DRM_DEBUG_KMS("DP link rate required %i available %i\n", ++ intel_dp_link_required(adjusted_mode->crtc_clock, ++ pipe_config->pipe_bpp), ++ intel_dp_max_data_rate(pipe_config->port_clock, ++ pipe_config->lane_count)); ++ } ++ return 0; ++} ++ ++bool intel_dp_limited_color_range(const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state) ++{ ++ const struct intel_digital_connector_state *intel_conn_state = ++ to_intel_digital_connector_state(conn_state); ++ const struct drm_display_mode *adjusted_mode = ++ &crtc_state->base.adjusted_mode; ++ ++ if (intel_conn_state->broadcast_rgb == INTEL_BROADCAST_RGB_AUTO) { ++ /* ++ * See: ++ * CEA-861-E - 5.1 Default Encoding Parameters ++ * VESA DisplayPort Ver.1.2a - 5.1.1.1 Video Colorimetry ++ */ ++ return crtc_state->pipe_bpp != 18 && ++ drm_default_rgb_quant_range(adjusted_mode) == ++ HDMI_QUANTIZATION_RANGE_LIMITED; ++ } else { ++ return intel_conn_state->broadcast_rgb == ++ INTEL_BROADCAST_RGB_LIMITED; ++ } ++} ++ ++int ++intel_dp_compute_config(struct intel_encoder *encoder, ++ struct intel_crtc_state *pipe_config, ++ struct drm_connector_state *conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ struct intel_lspcon *lspcon = enc_to_intel_lspcon(&encoder->base); ++ enum port port = encoder->port; ++ struct intel_crtc *intel_crtc = to_intel_crtc(pipe_config->base.crtc); ++ struct intel_connector *intel_connector = intel_dp->attached_connector; ++ struct intel_digital_connector_state *intel_conn_state = ++ to_intel_digital_connector_state(conn_state); ++ bool constant_n = drm_dp_has_quirk(&intel_dp->desc, ++ DP_DPCD_QUIRK_CONSTANT_N); ++ int ret, output_bpp; ++ ++ if (HAS_PCH_SPLIT(dev_priv) && !HAS_DDI(dev_priv) && port != PORT_A) ++ pipe_config->has_pch_encoder = true; ++ ++ pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; ++ if (lspcon->active) ++ lspcon_ycbcr420_config(&intel_connector->base, pipe_config); ++ ++ pipe_config->has_drrs = false; ++ if (IS_G4X(dev_priv) || port == PORT_A) ++ pipe_config->has_audio = false; ++ else if (intel_conn_state->force_audio == HDMI_AUDIO_AUTO) ++ pipe_config->has_audio = intel_dp->has_audio; ++ else ++ pipe_config->has_audio = intel_conn_state->force_audio == HDMI_AUDIO_ON; ++ ++ if (intel_dp_is_edp(intel_dp) && intel_connector->panel.fixed_mode) { ++ intel_fixed_panel_mode(intel_connector->panel.fixed_mode, ++ adjusted_mode); ++ ++ if (INTEL_GEN(dev_priv) >= 9) { ++ ret = skl_update_scaler_crtc(pipe_config); ++ if (ret) ++ return ret; ++ } ++ ++ if (HAS_GMCH(dev_priv)) ++ intel_gmch_panel_fitting(intel_crtc, pipe_config, ++ conn_state->scaling_mode); ++ else ++ intel_pch_panel_fitting(intel_crtc, pipe_config, ++ conn_state->scaling_mode); ++ } ++ ++ if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) ++ return -EINVAL; ++ ++ if (HAS_GMCH(dev_priv) && ++ adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) ++ return -EINVAL; ++ ++ if (adjusted_mode->flags & DRM_MODE_FLAG_DBLCLK) ++ return -EINVAL; ++ ++ ret = intel_dp_compute_link_config(encoder, pipe_config, conn_state); ++ if (ret < 0) ++ return ret; ++ ++ pipe_config->limited_color_range = ++ intel_dp_limited_color_range(pipe_config, conn_state); ++ ++ if (pipe_config->dsc_params.compression_enable) ++ output_bpp = pipe_config->dsc_params.compressed_bpp; ++ else ++ output_bpp = pipe_config->pipe_bpp; ++ ++ intel_link_compute_m_n(output_bpp, ++ pipe_config->lane_count, ++ adjusted_mode->crtc_clock, ++ pipe_config->port_clock, ++ &pipe_config->dp_m_n, ++ constant_n); ++ ++ if (intel_connector->panel.downclock_mode != NULL && ++ dev_priv->drrs.type == SEAMLESS_DRRS_SUPPORT) { ++ pipe_config->has_drrs = true; ++ intel_link_compute_m_n(output_bpp, ++ pipe_config->lane_count, ++ intel_connector->panel.downclock_mode->clock, ++ pipe_config->port_clock, ++ &pipe_config->dp_m2_n2, ++ constant_n); ++ } ++ ++ if (!HAS_DDI(dev_priv)) ++ intel_dp_set_clock(encoder, pipe_config); ++ ++ intel_psr_compute_config(intel_dp, pipe_config); ++ ++ return 0; ++} ++ ++void intel_dp_set_link_params(struct intel_dp *intel_dp, ++ int link_rate, u8 lane_count, ++ bool link_mst) ++{ ++ intel_dp->link_trained = false; ++ intel_dp->link_rate = link_rate; ++ intel_dp->lane_count = lane_count; ++ intel_dp->link_mst = link_mst; ++} ++ ++static void intel_dp_prepare(struct intel_encoder *encoder, ++ const struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ enum port port = encoder->port; ++ struct intel_crtc *crtc = to_intel_crtc(pipe_config->base.crtc); ++ const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; ++ ++ intel_dp_set_link_params(intel_dp, pipe_config->port_clock, ++ pipe_config->lane_count, ++ intel_crtc_has_type(pipe_config, ++ INTEL_OUTPUT_DP_MST)); ++ ++ /* ++ * There are four kinds of DP registers: ++ * ++ * IBX PCH ++ * SNB CPU ++ * IVB CPU ++ * CPT PCH ++ * ++ * IBX PCH and CPU are the same for almost everything, ++ * except that the CPU DP PLL is configured in this ++ * register ++ * ++ * CPT PCH is quite different, having many bits moved ++ * to the TRANS_DP_CTL register instead. That ++ * configuration happens (oddly) in ironlake_pch_enable ++ */ ++ ++ /* Preserve the BIOS-computed detected bit. This is ++ * supposed to be read-only. ++ */ ++ intel_dp->DP = I915_READ(intel_dp->output_reg) & DP_DETECTED; ++ ++ /* Handle DP bits in common between all three register formats */ ++ intel_dp->DP |= DP_VOLTAGE_0_4 | DP_PRE_EMPHASIS_0; ++ intel_dp->DP |= DP_PORT_WIDTH(pipe_config->lane_count); ++ ++ /* Split out the IBX/CPU vs CPT settings */ ++ ++ if (IS_IVYBRIDGE(dev_priv) && port == PORT_A) { ++ if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC) ++ intel_dp->DP |= DP_SYNC_HS_HIGH; ++ if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC) ++ intel_dp->DP |= DP_SYNC_VS_HIGH; ++ intel_dp->DP |= DP_LINK_TRAIN_OFF_CPT; ++ ++ if (drm_dp_enhanced_frame_cap(intel_dp->dpcd)) ++ intel_dp->DP |= DP_ENHANCED_FRAMING; ++ ++ intel_dp->DP |= DP_PIPE_SEL_IVB(crtc->pipe); ++ } else if (HAS_PCH_CPT(dev_priv) && port != PORT_A) { ++ u32 trans_dp; ++ ++ intel_dp->DP |= DP_LINK_TRAIN_OFF_CPT; ++ ++ trans_dp = I915_READ(TRANS_DP_CTL(crtc->pipe)); ++ if (drm_dp_enhanced_frame_cap(intel_dp->dpcd)) ++ trans_dp |= TRANS_DP_ENH_FRAMING; ++ else ++ trans_dp &= ~TRANS_DP_ENH_FRAMING; ++ I915_WRITE(TRANS_DP_CTL(crtc->pipe), trans_dp); ++ } else { ++ if (IS_G4X(dev_priv) && pipe_config->limited_color_range) ++ intel_dp->DP |= DP_COLOR_RANGE_16_235; ++ ++ if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC) ++ intel_dp->DP |= DP_SYNC_HS_HIGH; ++ if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC) ++ intel_dp->DP |= DP_SYNC_VS_HIGH; ++ intel_dp->DP |= DP_LINK_TRAIN_OFF; ++ ++ if (drm_dp_enhanced_frame_cap(intel_dp->dpcd)) ++ intel_dp->DP |= DP_ENHANCED_FRAMING; ++ ++ if (IS_CHERRYVIEW(dev_priv)) ++ intel_dp->DP |= DP_PIPE_SEL_CHV(crtc->pipe); ++ else ++ intel_dp->DP |= DP_PIPE_SEL(crtc->pipe); ++ } ++} ++ ++#define IDLE_ON_MASK (PP_ON | PP_SEQUENCE_MASK | 0 | PP_SEQUENCE_STATE_MASK) ++#define IDLE_ON_VALUE (PP_ON | PP_SEQUENCE_NONE | 0 | PP_SEQUENCE_STATE_ON_IDLE) ++ ++#define IDLE_OFF_MASK (PP_ON | PP_SEQUENCE_MASK | 0 | 0) ++#define IDLE_OFF_VALUE (0 | PP_SEQUENCE_NONE | 0 | 0) ++ ++#define IDLE_CYCLE_MASK (PP_ON | PP_SEQUENCE_MASK | PP_CYCLE_DELAY_ACTIVE | PP_SEQUENCE_STATE_MASK) ++#define IDLE_CYCLE_VALUE (0 | PP_SEQUENCE_NONE | 0 | PP_SEQUENCE_STATE_OFF_IDLE) ++ ++static void intel_pps_verify_state(struct intel_dp *intel_dp); ++ ++static void wait_panel_status(struct intel_dp *intel_dp, ++ u32 mask, ++ u32 value) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ i915_reg_t pp_stat_reg, pp_ctrl_reg; ++ ++ lockdep_assert_held(&dev_priv->pps_mutex); ++ ++ intel_pps_verify_state(intel_dp); ++ ++ pp_stat_reg = _pp_stat_reg(intel_dp); ++ pp_ctrl_reg = _pp_ctrl_reg(intel_dp); ++ ++ DRM_DEBUG_KMS("mask %08x value %08x status %08x control %08x\n", ++ mask, value, ++ I915_READ(pp_stat_reg), ++ I915_READ(pp_ctrl_reg)); ++ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ pp_stat_reg, mask, value, ++ 5000)) ++ DRM_ERROR("Panel status timeout: status %08x control %08x\n", ++ I915_READ(pp_stat_reg), ++ I915_READ(pp_ctrl_reg)); ++ ++ DRM_DEBUG_KMS("Wait complete\n"); ++} ++ ++static void wait_panel_on(struct intel_dp *intel_dp) ++{ ++ DRM_DEBUG_KMS("Wait for panel power on\n"); ++ wait_panel_status(intel_dp, IDLE_ON_MASK, IDLE_ON_VALUE); ++} ++ ++static void wait_panel_off(struct intel_dp *intel_dp) ++{ ++ DRM_DEBUG_KMS("Wait for panel power off time\n"); ++ wait_panel_status(intel_dp, IDLE_OFF_MASK, IDLE_OFF_VALUE); ++} ++ ++static void wait_panel_power_cycle(struct intel_dp *intel_dp) ++{ ++ ktime_t panel_power_on_time; ++ s64 panel_power_off_duration; ++ ++ DRM_DEBUG_KMS("Wait for panel power cycle\n"); ++ ++ /* take the difference of currrent time and panel power off time ++ * and then make panel wait for t11_t12 if needed. */ ++ panel_power_on_time = ktime_get_boottime(); ++ panel_power_off_duration = ktime_ms_delta(panel_power_on_time, intel_dp->panel_power_off_time); ++ ++ /* When we disable the VDD override bit last we have to do the manual ++ * wait. */ ++ if (panel_power_off_duration < (s64)intel_dp->panel_power_cycle_delay) ++ wait_remaining_ms_from_jiffies(jiffies, ++ intel_dp->panel_power_cycle_delay - panel_power_off_duration); ++ ++ wait_panel_status(intel_dp, IDLE_CYCLE_MASK, IDLE_CYCLE_VALUE); ++} ++ ++static void wait_backlight_on(struct intel_dp *intel_dp) ++{ ++ wait_remaining_ms_from_jiffies(intel_dp->last_power_on, ++ intel_dp->backlight_on_delay); ++} ++ ++static void edp_wait_backlight_off(struct intel_dp *intel_dp) ++{ ++ wait_remaining_ms_from_jiffies(intel_dp->last_backlight_off, ++ intel_dp->backlight_off_delay); ++} ++ ++/* Read the current pp_control value, unlocking the register if it ++ * is locked ++ */ ++ ++static u32 ironlake_get_pp_control(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ u32 control; ++ ++ lockdep_assert_held(&dev_priv->pps_mutex); ++ ++ control = I915_READ(_pp_ctrl_reg(intel_dp)); ++ if (WARN_ON(!HAS_DDI(dev_priv) && ++ (control & PANEL_UNLOCK_MASK) != PANEL_UNLOCK_REGS)) { ++ control &= ~PANEL_UNLOCK_MASK; ++ control |= PANEL_UNLOCK_REGS; ++ } ++ return control; ++} ++ ++/* ++ * Must be paired with edp_panel_vdd_off(). ++ * Must hold pps_mutex around the whole on/off sequence. ++ * Can be nested with intel_edp_panel_vdd_{on,off}() calls. ++ */ ++static bool edp_panel_vdd_on(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); ++ u32 pp; ++ i915_reg_t pp_stat_reg, pp_ctrl_reg; ++ bool need_to_disable = !intel_dp->want_panel_vdd; ++ ++ lockdep_assert_held(&dev_priv->pps_mutex); ++ ++ if (!intel_dp_is_edp(intel_dp)) ++ return false; ++ ++ cancel_delayed_work(&intel_dp->panel_vdd_work); ++ intel_dp->want_panel_vdd = true; ++ ++ if (edp_have_panel_vdd(intel_dp)) ++ return need_to_disable; ++ ++ intel_display_power_get(dev_priv, ++ intel_aux_power_domain(intel_dig_port)); ++ ++ DRM_DEBUG_KMS("Turning eDP port %c VDD on\n", ++ port_name(intel_dig_port->base.port)); ++ ++ if (!edp_have_panel_power(intel_dp)) ++ wait_panel_power_cycle(intel_dp); ++ ++ pp = ironlake_get_pp_control(intel_dp); ++ pp |= EDP_FORCE_VDD; ++ ++ pp_stat_reg = _pp_stat_reg(intel_dp); ++ pp_ctrl_reg = _pp_ctrl_reg(intel_dp); ++ ++ I915_WRITE(pp_ctrl_reg, pp); ++ POSTING_READ(pp_ctrl_reg); ++ DRM_DEBUG_KMS("PP_STATUS: 0x%08x PP_CONTROL: 0x%08x\n", ++ I915_READ(pp_stat_reg), I915_READ(pp_ctrl_reg)); ++ /* ++ * If the panel wasn't on, delay before accessing aux channel ++ */ ++ if (!edp_have_panel_power(intel_dp)) { ++ DRM_DEBUG_KMS("eDP port %c panel power wasn't enabled\n", ++ port_name(intel_dig_port->base.port)); ++ msleep(intel_dp->panel_power_up_delay); ++ } ++ ++ return need_to_disable; ++} ++ ++/* ++ * Must be paired with intel_edp_panel_vdd_off() or ++ * intel_edp_panel_off(). ++ * Nested calls to these functions are not allowed since ++ * we drop the lock. Caller must use some higher level ++ * locking to prevent nested calls from other threads. ++ */ ++void intel_edp_panel_vdd_on(struct intel_dp *intel_dp) ++{ ++ intel_wakeref_t wakeref; ++ bool vdd; ++ ++ if (!intel_dp_is_edp(intel_dp)) ++ return; ++ ++ vdd = false; ++ with_pps_lock(intel_dp, wakeref) ++ vdd = edp_panel_vdd_on(intel_dp); ++ I915_STATE_WARN(!vdd, "eDP port %c VDD already requested on\n", ++ port_name(dp_to_dig_port(intel_dp)->base.port)); ++} ++ ++static void edp_panel_vdd_off_sync(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ struct intel_digital_port *intel_dig_port = ++ dp_to_dig_port(intel_dp); ++ u32 pp; ++ i915_reg_t pp_stat_reg, pp_ctrl_reg; ++ ++ lockdep_assert_held(&dev_priv->pps_mutex); ++ ++ WARN_ON(intel_dp->want_panel_vdd); ++ ++ if (!edp_have_panel_vdd(intel_dp)) ++ return; ++ ++ DRM_DEBUG_KMS("Turning eDP port %c VDD off\n", ++ port_name(intel_dig_port->base.port)); ++ ++ pp = ironlake_get_pp_control(intel_dp); ++ pp &= ~EDP_FORCE_VDD; ++ ++ pp_ctrl_reg = _pp_ctrl_reg(intel_dp); ++ pp_stat_reg = _pp_stat_reg(intel_dp); ++ ++ I915_WRITE(pp_ctrl_reg, pp); ++ POSTING_READ(pp_ctrl_reg); ++ ++ /* Make sure sequencer is idle before allowing subsequent activity */ ++ DRM_DEBUG_KMS("PP_STATUS: 0x%08x PP_CONTROL: 0x%08x\n", ++ I915_READ(pp_stat_reg), I915_READ(pp_ctrl_reg)); ++ ++ if ((pp & PANEL_POWER_ON) == 0) ++ intel_dp->panel_power_off_time = ktime_get_boottime(); ++ ++ intel_display_power_put_unchecked(dev_priv, ++ intel_aux_power_domain(intel_dig_port)); ++} ++ ++static void edp_panel_vdd_work(struct work_struct *__work) ++{ ++ struct intel_dp *intel_dp = ++ container_of(to_delayed_work(__work), ++ struct intel_dp, panel_vdd_work); ++ intel_wakeref_t wakeref; ++ ++ with_pps_lock(intel_dp, wakeref) { ++ if (!intel_dp->want_panel_vdd) ++ edp_panel_vdd_off_sync(intel_dp); ++ } ++} ++ ++static void edp_panel_vdd_schedule_off(struct intel_dp *intel_dp) ++{ ++ unsigned long delay; ++ ++ /* ++ * Queue the timer to fire a long time from now (relative to the power ++ * down delay) to keep the panel power up across a sequence of ++ * operations. ++ */ ++ delay = msecs_to_jiffies(intel_dp->panel_power_cycle_delay * 5); ++ schedule_delayed_work(&intel_dp->panel_vdd_work, delay); ++} ++ ++/* ++ * Must be paired with edp_panel_vdd_on(). ++ * Must hold pps_mutex around the whole on/off sequence. ++ * Can be nested with intel_edp_panel_vdd_{on,off}() calls. ++ */ ++static void edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ ++ lockdep_assert_held(&dev_priv->pps_mutex); ++ ++ if (!intel_dp_is_edp(intel_dp)) ++ return; ++ ++ I915_STATE_WARN(!intel_dp->want_panel_vdd, "eDP port %c VDD not forced on", ++ port_name(dp_to_dig_port(intel_dp)->base.port)); ++ ++ intel_dp->want_panel_vdd = false; ++ ++ if (sync) ++ edp_panel_vdd_off_sync(intel_dp); ++ else ++ edp_panel_vdd_schedule_off(intel_dp); ++} ++ ++static void edp_panel_on(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ u32 pp; ++ i915_reg_t pp_ctrl_reg; ++ ++ lockdep_assert_held(&dev_priv->pps_mutex); ++ ++ if (!intel_dp_is_edp(intel_dp)) ++ return; ++ ++ DRM_DEBUG_KMS("Turn eDP port %c panel power on\n", ++ port_name(dp_to_dig_port(intel_dp)->base.port)); ++ ++ if (WARN(edp_have_panel_power(intel_dp), ++ "eDP port %c panel power already on\n", ++ port_name(dp_to_dig_port(intel_dp)->base.port))) ++ return; ++ ++ wait_panel_power_cycle(intel_dp); ++ ++ pp_ctrl_reg = _pp_ctrl_reg(intel_dp); ++ pp = ironlake_get_pp_control(intel_dp); ++ if (IS_GEN(dev_priv, 5)) { ++ /* ILK workaround: disable reset around power sequence */ ++ pp &= ~PANEL_POWER_RESET; ++ I915_WRITE(pp_ctrl_reg, pp); ++ POSTING_READ(pp_ctrl_reg); ++ } ++ ++ pp |= PANEL_POWER_ON; ++ if (!IS_GEN(dev_priv, 5)) ++ pp |= PANEL_POWER_RESET; ++ ++ I915_WRITE(pp_ctrl_reg, pp); ++ POSTING_READ(pp_ctrl_reg); ++ ++ wait_panel_on(intel_dp); ++ intel_dp->last_power_on = jiffies; ++ ++ if (IS_GEN(dev_priv, 5)) { ++ pp |= PANEL_POWER_RESET; /* restore panel reset bit */ ++ I915_WRITE(pp_ctrl_reg, pp); ++ POSTING_READ(pp_ctrl_reg); ++ } ++} ++ ++void intel_edp_panel_on(struct intel_dp *intel_dp) ++{ ++ intel_wakeref_t wakeref; ++ ++ if (!intel_dp_is_edp(intel_dp)) ++ return; ++ ++ with_pps_lock(intel_dp, wakeref) ++ edp_panel_on(intel_dp); ++} ++ ++ ++static void edp_panel_off(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); ++ u32 pp; ++ i915_reg_t pp_ctrl_reg; ++ ++ lockdep_assert_held(&dev_priv->pps_mutex); ++ ++ if (!intel_dp_is_edp(intel_dp)) ++ return; ++ ++ DRM_DEBUG_KMS("Turn eDP port %c panel power off\n", ++ port_name(dig_port->base.port)); ++ ++ WARN(!intel_dp->want_panel_vdd, "Need eDP port %c VDD to turn off panel\n", ++ port_name(dig_port->base.port)); ++ ++ pp = ironlake_get_pp_control(intel_dp); ++ /* We need to switch off panel power _and_ force vdd, for otherwise some ++ * panels get very unhappy and cease to work. */ ++ pp &= ~(PANEL_POWER_ON | PANEL_POWER_RESET | EDP_FORCE_VDD | ++ EDP_BLC_ENABLE); ++ ++ pp_ctrl_reg = _pp_ctrl_reg(intel_dp); ++ ++ intel_dp->want_panel_vdd = false; ++ ++ I915_WRITE(pp_ctrl_reg, pp); ++ POSTING_READ(pp_ctrl_reg); ++ ++ wait_panel_off(intel_dp); ++ intel_dp->panel_power_off_time = ktime_get_boottime(); ++ ++ /* We got a reference when we enabled the VDD. */ ++ intel_display_power_put_unchecked(dev_priv, intel_aux_power_domain(dig_port)); ++} ++ ++void intel_edp_panel_off(struct intel_dp *intel_dp) ++{ ++ intel_wakeref_t wakeref; ++ ++ if (!intel_dp_is_edp(intel_dp)) ++ return; ++ ++ with_pps_lock(intel_dp, wakeref) ++ edp_panel_off(intel_dp); ++} ++ ++/* Enable backlight in the panel power control. */ ++static void _intel_edp_backlight_on(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ intel_wakeref_t wakeref; ++ ++ /* ++ * If we enable the backlight right away following a panel power ++ * on, we may see slight flicker as the panel syncs with the eDP ++ * link. So delay a bit to make sure the image is solid before ++ * allowing it to appear. ++ */ ++ wait_backlight_on(intel_dp); ++ ++ with_pps_lock(intel_dp, wakeref) { ++ i915_reg_t pp_ctrl_reg = _pp_ctrl_reg(intel_dp); ++ u32 pp; ++ ++ pp = ironlake_get_pp_control(intel_dp); ++ pp |= EDP_BLC_ENABLE; ++ ++ I915_WRITE(pp_ctrl_reg, pp); ++ POSTING_READ(pp_ctrl_reg); ++ } ++} ++ ++/* Enable backlight PWM and backlight PP control. */ ++void intel_edp_backlight_on(const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state) ++{ ++ struct intel_dp *intel_dp = enc_to_intel_dp(conn_state->best_encoder); ++ ++ if (!intel_dp_is_edp(intel_dp)) ++ return; ++ ++ DRM_DEBUG_KMS("\n"); ++ ++ intel_panel_enable_backlight(crtc_state, conn_state); ++ _intel_edp_backlight_on(intel_dp); ++} ++ ++/* Disable backlight in the panel power control. */ ++static void _intel_edp_backlight_off(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ intel_wakeref_t wakeref; ++ ++ if (!intel_dp_is_edp(intel_dp)) ++ return; ++ ++ with_pps_lock(intel_dp, wakeref) { ++ i915_reg_t pp_ctrl_reg = _pp_ctrl_reg(intel_dp); ++ u32 pp; ++ ++ pp = ironlake_get_pp_control(intel_dp); ++ pp &= ~EDP_BLC_ENABLE; ++ ++ I915_WRITE(pp_ctrl_reg, pp); ++ POSTING_READ(pp_ctrl_reg); ++ } ++ ++ intel_dp->last_backlight_off = jiffies; ++ edp_wait_backlight_off(intel_dp); ++} ++ ++/* Disable backlight PP control and backlight PWM. */ ++void intel_edp_backlight_off(const struct drm_connector_state *old_conn_state) ++{ ++ struct intel_dp *intel_dp = enc_to_intel_dp(old_conn_state->best_encoder); ++ ++ if (!intel_dp_is_edp(intel_dp)) ++ return; ++ ++ DRM_DEBUG_KMS("\n"); ++ ++ _intel_edp_backlight_off(intel_dp); ++ intel_panel_disable_backlight(old_conn_state); ++} ++ ++/* ++ * Hook for controlling the panel power control backlight through the bl_power ++ * sysfs attribute. Take care to handle multiple calls. ++ */ ++static void intel_edp_backlight_power(struct intel_connector *connector, ++ bool enable) ++{ ++ struct intel_dp *intel_dp = intel_attached_dp(&connector->base); ++ intel_wakeref_t wakeref; ++ bool is_enabled; ++ ++ is_enabled = false; ++ with_pps_lock(intel_dp, wakeref) ++ is_enabled = ironlake_get_pp_control(intel_dp) & EDP_BLC_ENABLE; ++ if (is_enabled == enable) ++ return; ++ ++ DRM_DEBUG_KMS("panel power control backlight %s\n", ++ enable ? "enable" : "disable"); ++ ++ if (enable) ++ _intel_edp_backlight_on(intel_dp); ++ else ++ _intel_edp_backlight_off(intel_dp); ++} ++ ++static void assert_dp_port(struct intel_dp *intel_dp, bool state) ++{ ++ struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); ++ struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); ++ bool cur_state = I915_READ(intel_dp->output_reg) & DP_PORT_EN; ++ ++ I915_STATE_WARN(cur_state != state, ++ "DP port %c state assertion failure (expected %s, current %s)\n", ++ port_name(dig_port->base.port), ++ onoff(state), onoff(cur_state)); ++} ++#define assert_dp_port_disabled(d) assert_dp_port((d), false) ++ ++static void assert_edp_pll(struct drm_i915_private *dev_priv, bool state) ++{ ++ bool cur_state = I915_READ(DP_A) & DP_PLL_ENABLE; ++ ++ I915_STATE_WARN(cur_state != state, ++ "eDP PLL state assertion failure (expected %s, current %s)\n", ++ onoff(state), onoff(cur_state)); ++} ++#define assert_edp_pll_enabled(d) assert_edp_pll((d), true) ++#define assert_edp_pll_disabled(d) assert_edp_pll((d), false) ++ ++static void ironlake_edp_pll_on(struct intel_dp *intel_dp, ++ const struct intel_crtc_state *pipe_config) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(pipe_config->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ ++ assert_pipe_disabled(dev_priv, crtc->pipe); ++ assert_dp_port_disabled(intel_dp); ++ assert_edp_pll_disabled(dev_priv); ++ ++ DRM_DEBUG_KMS("enabling eDP PLL for clock %d\n", ++ pipe_config->port_clock); ++ ++ intel_dp->DP &= ~DP_PLL_FREQ_MASK; ++ ++ if (pipe_config->port_clock == 162000) ++ intel_dp->DP |= DP_PLL_FREQ_162MHZ; ++ else ++ intel_dp->DP |= DP_PLL_FREQ_270MHZ; ++ ++ I915_WRITE(DP_A, intel_dp->DP); ++ POSTING_READ(DP_A); ++ udelay(500); ++ ++ /* ++ * [DevILK] Work around required when enabling DP PLL ++ * while a pipe is enabled going to FDI: ++ * 1. Wait for the start of vertical blank on the enabled pipe going to FDI ++ * 2. Program DP PLL enable ++ */ ++ if (IS_GEN(dev_priv, 5)) ++ intel_wait_for_vblank_if_active(dev_priv, !crtc->pipe); ++ ++ intel_dp->DP |= DP_PLL_ENABLE; ++ ++ I915_WRITE(DP_A, intel_dp->DP); ++ POSTING_READ(DP_A); ++ udelay(200); ++} ++ ++static void ironlake_edp_pll_off(struct intel_dp *intel_dp, ++ const struct intel_crtc_state *old_crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ ++ assert_pipe_disabled(dev_priv, crtc->pipe); ++ assert_dp_port_disabled(intel_dp); ++ assert_edp_pll_enabled(dev_priv); ++ ++ DRM_DEBUG_KMS("disabling eDP PLL\n"); ++ ++ intel_dp->DP &= ~DP_PLL_ENABLE; ++ ++ I915_WRITE(DP_A, intel_dp->DP); ++ POSTING_READ(DP_A); ++ udelay(200); ++} ++ ++static bool downstream_hpd_needs_d0(struct intel_dp *intel_dp) ++{ ++ /* ++ * DPCD 1.2+ should support BRANCH_DEVICE_CTRL, and thus ++ * be capable of signalling downstream hpd with a long pulse. ++ * Whether or not that means D3 is safe to use is not clear, ++ * but let's assume so until proven otherwise. ++ * ++ * FIXME should really check all downstream ports... ++ */ ++ return intel_dp->dpcd[DP_DPCD_REV] == 0x11 && ++ intel_dp->dpcd[DP_DOWNSTREAMPORT_PRESENT] & DP_DWN_STRM_PORT_PRESENT && ++ intel_dp->downstream_ports[0] & DP_DS_PORT_HPD; ++} ++ ++void intel_dp_sink_set_decompression_state(struct intel_dp *intel_dp, ++ const struct intel_crtc_state *crtc_state, ++ bool enable) ++{ ++ int ret; ++ ++ if (!crtc_state->dsc_params.compression_enable) ++ return; ++ ++ ret = drm_dp_dpcd_writeb(&intel_dp->aux, DP_DSC_ENABLE, ++ enable ? DP_DECOMPRESSION_EN : 0); ++ if (ret < 0) ++ DRM_DEBUG_KMS("Failed to %s sink decompression state\n", ++ enable ? "enable" : "disable"); ++} ++ ++/* If the sink supports it, try to set the power state appropriately */ ++void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode) ++{ ++ int ret, i; ++ ++ /* Should have a valid DPCD by this point */ ++ if (intel_dp->dpcd[DP_DPCD_REV] < 0x11) ++ return; ++ ++ if (mode != DRM_MODE_DPMS_ON) { ++ if (downstream_hpd_needs_d0(intel_dp)) ++ return; ++ ++ ret = drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, ++ DP_SET_POWER_D3); ++ } else { ++ struct intel_lspcon *lspcon = dp_to_lspcon(intel_dp); ++ ++ /* ++ * When turning on, we need to retry for 1ms to give the sink ++ * time to wake up. ++ */ ++ for (i = 0; i < 3; i++) { ++ ret = drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, ++ DP_SET_POWER_D0); ++ if (ret == 1) ++ break; ++ msleep(1); ++ } ++ ++ if (ret == 1 && lspcon->active) ++ lspcon_wait_pcon_mode(lspcon); ++ } ++ ++ if (ret != 1) ++ DRM_DEBUG_KMS("failed to %s sink power state\n", ++ mode == DRM_MODE_DPMS_ON ? "enable" : "disable"); ++} ++ ++static bool cpt_dp_port_selected(struct drm_i915_private *dev_priv, ++ enum port port, enum pipe *pipe) ++{ ++ enum pipe p; ++ ++ for_each_pipe(dev_priv, p) { ++ u32 val = I915_READ(TRANS_DP_CTL(p)); ++ ++ if ((val & TRANS_DP_PORT_SEL_MASK) == TRANS_DP_PORT_SEL(port)) { ++ *pipe = p; ++ return true; ++ } ++ } ++ ++ DRM_DEBUG_KMS("No pipe for DP port %c found\n", port_name(port)); ++ ++ /* must initialize pipe to something for the asserts */ ++ *pipe = PIPE_A; ++ ++ return false; ++} ++ ++bool intel_dp_port_enabled(struct drm_i915_private *dev_priv, ++ i915_reg_t dp_reg, enum port port, ++ enum pipe *pipe) ++{ ++ bool ret; ++ u32 val; ++ ++ val = I915_READ(dp_reg); ++ ++ ret = val & DP_PORT_EN; ++ ++ /* asserts want to know the pipe even if the port is disabled */ ++ if (IS_IVYBRIDGE(dev_priv) && port == PORT_A) ++ *pipe = (val & DP_PIPE_SEL_MASK_IVB) >> DP_PIPE_SEL_SHIFT_IVB; ++ else if (HAS_PCH_CPT(dev_priv) && port != PORT_A) ++ ret &= cpt_dp_port_selected(dev_priv, port, pipe); ++ else if (IS_CHERRYVIEW(dev_priv)) ++ *pipe = (val & DP_PIPE_SEL_MASK_CHV) >> DP_PIPE_SEL_SHIFT_CHV; ++ else ++ *pipe = (val & DP_PIPE_SEL_MASK) >> DP_PIPE_SEL_SHIFT; ++ ++ return ret; ++} ++ ++static bool intel_dp_get_hw_state(struct intel_encoder *encoder, ++ enum pipe *pipe) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ intel_wakeref_t wakeref; ++ bool ret; ++ ++ wakeref = intel_display_power_get_if_enabled(dev_priv, ++ encoder->power_domain); ++ if (!wakeref) ++ return false; ++ ++ ret = intel_dp_port_enabled(dev_priv, intel_dp->output_reg, ++ encoder->port, pipe); ++ ++ intel_display_power_put(dev_priv, encoder->power_domain, wakeref); ++ ++ return ret; ++} ++ ++static void intel_dp_get_config(struct intel_encoder *encoder, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ u32 tmp, flags = 0; ++ enum port port = encoder->port; ++ struct intel_crtc *crtc = to_intel_crtc(pipe_config->base.crtc); ++ ++ if (encoder->type == INTEL_OUTPUT_EDP) ++ pipe_config->output_types |= BIT(INTEL_OUTPUT_EDP); ++ else ++ pipe_config->output_types |= BIT(INTEL_OUTPUT_DP); ++ ++ tmp = I915_READ(intel_dp->output_reg); ++ ++ pipe_config->has_audio = tmp & DP_AUDIO_OUTPUT_ENABLE && port != PORT_A; ++ ++ if (HAS_PCH_CPT(dev_priv) && port != PORT_A) { ++ u32 trans_dp = I915_READ(TRANS_DP_CTL(crtc->pipe)); ++ ++ if (trans_dp & TRANS_DP_HSYNC_ACTIVE_HIGH) ++ flags |= DRM_MODE_FLAG_PHSYNC; ++ else ++ flags |= DRM_MODE_FLAG_NHSYNC; ++ ++ if (trans_dp & TRANS_DP_VSYNC_ACTIVE_HIGH) ++ flags |= DRM_MODE_FLAG_PVSYNC; ++ else ++ flags |= DRM_MODE_FLAG_NVSYNC; ++ } else { ++ if (tmp & DP_SYNC_HS_HIGH) ++ flags |= DRM_MODE_FLAG_PHSYNC; ++ else ++ flags |= DRM_MODE_FLAG_NHSYNC; ++ ++ if (tmp & DP_SYNC_VS_HIGH) ++ flags |= DRM_MODE_FLAG_PVSYNC; ++ else ++ flags |= DRM_MODE_FLAG_NVSYNC; ++ } ++ ++ pipe_config->base.adjusted_mode.flags |= flags; ++ ++ if (IS_G4X(dev_priv) && tmp & DP_COLOR_RANGE_16_235) ++ pipe_config->limited_color_range = true; ++ ++ pipe_config->lane_count = ++ ((tmp & DP_PORT_WIDTH_MASK) >> DP_PORT_WIDTH_SHIFT) + 1; ++ ++ intel_dp_get_m_n(crtc, pipe_config); ++ ++ if (port == PORT_A) { ++ if ((I915_READ(DP_A) & DP_PLL_FREQ_MASK) == DP_PLL_FREQ_162MHZ) ++ pipe_config->port_clock = 162000; ++ else ++ pipe_config->port_clock = 270000; ++ } ++ ++ pipe_config->base.adjusted_mode.crtc_clock = ++ intel_dotclock_calculate(pipe_config->port_clock, ++ &pipe_config->dp_m_n); ++ ++ if (intel_dp_is_edp(intel_dp) && dev_priv->vbt.edp.bpp && ++ pipe_config->pipe_bpp > dev_priv->vbt.edp.bpp) { ++ /* ++ * This is a big fat ugly hack. ++ * ++ * Some machines in UEFI boot mode provide us a VBT that has 18 ++ * bpp and 1.62 GHz link bandwidth for eDP, which for reasons ++ * unknown we fail to light up. Yet the same BIOS boots up with ++ * 24 bpp and 2.7 GHz link. Use the same bpp as the BIOS uses as ++ * max, not what it tells us to use. ++ * ++ * Note: This will still be broken if the eDP panel is not lit ++ * up by the BIOS, and thus we can't get the mode at module ++ * load. ++ */ ++ DRM_DEBUG_KMS("pipe has %d bpp for eDP panel, overriding BIOS-provided max %d bpp\n", ++ pipe_config->pipe_bpp, dev_priv->vbt.edp.bpp); ++ dev_priv->vbt.edp.bpp = pipe_config->pipe_bpp; ++ } ++} ++ ++static void intel_disable_dp(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ ++ intel_dp->link_trained = false; ++ ++ if (old_crtc_state->has_audio) ++ intel_audio_codec_disable(encoder, ++ old_crtc_state, old_conn_state); ++ ++ /* Make sure the panel is off before trying to change the mode. But also ++ * ensure that we have vdd while we switch off the panel. */ ++ intel_edp_panel_vdd_on(intel_dp); ++ intel_edp_backlight_off(old_conn_state); ++ intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF); ++ intel_edp_panel_off(intel_dp); ++} ++ ++static void g4x_disable_dp(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ intel_disable_dp(encoder, old_crtc_state, old_conn_state); ++} ++ ++static void vlv_disable_dp(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ intel_disable_dp(encoder, old_crtc_state, old_conn_state); ++} ++ ++static void g4x_post_disable_dp(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ enum port port = encoder->port; ++ ++ /* ++ * Bspec does not list a specific disable sequence for g4x DP. ++ * Follow the ilk+ sequence (disable pipe before the port) for ++ * g4x DP as it does not suffer from underruns like the normal ++ * g4x modeset sequence (disable pipe after the port). ++ */ ++ intel_dp_link_down(encoder, old_crtc_state); ++ ++ /* Only ilk+ has port A */ ++ if (port == PORT_A) ++ ironlake_edp_pll_off(intel_dp, old_crtc_state); ++} ++ ++static void vlv_post_disable_dp(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ intel_dp_link_down(encoder, old_crtc_state); ++} ++ ++static void chv_post_disable_dp(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ ++ intel_dp_link_down(encoder, old_crtc_state); ++ ++ mutex_lock(&dev_priv->sb_lock); ++ ++ /* Assert data lane reset */ ++ chv_data_lane_soft_reset(encoder, old_crtc_state, true); ++ ++ mutex_unlock(&dev_priv->sb_lock); ++} ++ ++static void ++_intel_dp_set_link_train(struct intel_dp *intel_dp, ++ u32 *DP, ++ u8 dp_train_pat) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); ++ enum port port = intel_dig_port->base.port; ++ u8 train_pat_mask = drm_dp_training_pattern_mask(intel_dp->dpcd); ++ ++ if (dp_train_pat & train_pat_mask) ++ DRM_DEBUG_KMS("Using DP training pattern TPS%d\n", ++ dp_train_pat & train_pat_mask); ++ ++ if (HAS_DDI(dev_priv)) { ++ u32 temp = I915_READ(DP_TP_CTL(port)); ++ ++ if (dp_train_pat & DP_LINK_SCRAMBLING_DISABLE) ++ temp |= DP_TP_CTL_SCRAMBLE_DISABLE; ++ else ++ temp &= ~DP_TP_CTL_SCRAMBLE_DISABLE; ++ ++ temp &= ~DP_TP_CTL_LINK_TRAIN_MASK; ++ switch (dp_train_pat & train_pat_mask) { ++ case DP_TRAINING_PATTERN_DISABLE: ++ temp |= DP_TP_CTL_LINK_TRAIN_NORMAL; ++ ++ break; ++ case DP_TRAINING_PATTERN_1: ++ temp |= DP_TP_CTL_LINK_TRAIN_PAT1; ++ break; ++ case DP_TRAINING_PATTERN_2: ++ temp |= DP_TP_CTL_LINK_TRAIN_PAT2; ++ break; ++ case DP_TRAINING_PATTERN_3: ++ temp |= DP_TP_CTL_LINK_TRAIN_PAT3; ++ break; ++ case DP_TRAINING_PATTERN_4: ++ temp |= DP_TP_CTL_LINK_TRAIN_PAT4; ++ break; ++ } ++ I915_WRITE(DP_TP_CTL(port), temp); ++ ++ } else if ((IS_IVYBRIDGE(dev_priv) && port == PORT_A) || ++ (HAS_PCH_CPT(dev_priv) && port != PORT_A)) { ++ *DP &= ~DP_LINK_TRAIN_MASK_CPT; ++ ++ switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) { ++ case DP_TRAINING_PATTERN_DISABLE: ++ *DP |= DP_LINK_TRAIN_OFF_CPT; ++ break; ++ case DP_TRAINING_PATTERN_1: ++ *DP |= DP_LINK_TRAIN_PAT_1_CPT; ++ break; ++ case DP_TRAINING_PATTERN_2: ++ *DP |= DP_LINK_TRAIN_PAT_2_CPT; ++ break; ++ case DP_TRAINING_PATTERN_3: ++ DRM_DEBUG_KMS("TPS3 not supported, using TPS2 instead\n"); ++ *DP |= DP_LINK_TRAIN_PAT_2_CPT; ++ break; ++ } ++ ++ } else { ++ *DP &= ~DP_LINK_TRAIN_MASK; ++ ++ switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) { ++ case DP_TRAINING_PATTERN_DISABLE: ++ *DP |= DP_LINK_TRAIN_OFF; ++ break; ++ case DP_TRAINING_PATTERN_1: ++ *DP |= DP_LINK_TRAIN_PAT_1; ++ break; ++ case DP_TRAINING_PATTERN_2: ++ *DP |= DP_LINK_TRAIN_PAT_2; ++ break; ++ case DP_TRAINING_PATTERN_3: ++ DRM_DEBUG_KMS("TPS3 not supported, using TPS2 instead\n"); ++ *DP |= DP_LINK_TRAIN_PAT_2; ++ break; ++ } ++ } ++} ++ ++static void intel_dp_enable_port(struct intel_dp *intel_dp, ++ const struct intel_crtc_state *old_crtc_state) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ ++ /* enable with pattern 1 (as per spec) */ ++ ++ intel_dp_program_link_training_pattern(intel_dp, DP_TRAINING_PATTERN_1); ++ ++ /* ++ * Magic for VLV/CHV. We _must_ first set up the register ++ * without actually enabling the port, and then do another ++ * write to enable the port. Otherwise link training will ++ * fail when the power sequencer is freshly used for this port. ++ */ ++ intel_dp->DP |= DP_PORT_EN; ++ if (old_crtc_state->has_audio) ++ intel_dp->DP |= DP_AUDIO_OUTPUT_ENABLE; ++ ++ I915_WRITE(intel_dp->output_reg, intel_dp->DP); ++ POSTING_READ(intel_dp->output_reg); ++} ++ ++static void intel_enable_dp(struct intel_encoder *encoder, ++ const struct intel_crtc_state *pipe_config, ++ const struct drm_connector_state *conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ struct intel_crtc *crtc = to_intel_crtc(pipe_config->base.crtc); ++ u32 dp_reg = I915_READ(intel_dp->output_reg); ++ enum pipe pipe = crtc->pipe; ++ intel_wakeref_t wakeref; ++ ++ if (WARN_ON(dp_reg & DP_PORT_EN)) ++ return; ++ ++ with_pps_lock(intel_dp, wakeref) { ++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ vlv_init_panel_power_sequencer(encoder, pipe_config); ++ ++ intel_dp_enable_port(intel_dp, pipe_config); ++ ++ edp_panel_vdd_on(intel_dp); ++ edp_panel_on(intel_dp); ++ edp_panel_vdd_off(intel_dp, true); ++ } ++ ++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { ++ unsigned int lane_mask = 0x0; ++ ++ if (IS_CHERRYVIEW(dev_priv)) ++ lane_mask = intel_dp_unused_lane_mask(pipe_config->lane_count); ++ ++ vlv_wait_port_ready(dev_priv, dp_to_dig_port(intel_dp), ++ lane_mask); ++ } ++ ++ intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON); ++ intel_dp_start_link_train(intel_dp); ++ intel_dp_stop_link_train(intel_dp); ++ ++ if (pipe_config->has_audio) { ++ DRM_DEBUG_DRIVER("Enabling DP audio on pipe %c\n", ++ pipe_name(pipe)); ++ intel_audio_codec_enable(encoder, pipe_config, conn_state); ++ } ++} ++ ++static void g4x_enable_dp(struct intel_encoder *encoder, ++ const struct intel_crtc_state *pipe_config, ++ const struct drm_connector_state *conn_state) ++{ ++ intel_enable_dp(encoder, pipe_config, conn_state); ++ intel_edp_backlight_on(pipe_config, conn_state); ++} ++ ++static void vlv_enable_dp(struct intel_encoder *encoder, ++ const struct intel_crtc_state *pipe_config, ++ const struct drm_connector_state *conn_state) ++{ ++ intel_edp_backlight_on(pipe_config, conn_state); ++} ++ ++static void g4x_pre_enable_dp(struct intel_encoder *encoder, ++ const struct intel_crtc_state *pipe_config, ++ const struct drm_connector_state *conn_state) ++{ ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ enum port port = encoder->port; ++ ++ intel_dp_prepare(encoder, pipe_config); ++ ++ /* Only ilk+ has port A */ ++ if (port == PORT_A) ++ ironlake_edp_pll_on(intel_dp, pipe_config); ++} ++ ++static void vlv_detach_power_sequencer(struct intel_dp *intel_dp) ++{ ++ struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); ++ struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev); ++ enum pipe pipe = intel_dp->pps_pipe; ++ i915_reg_t pp_on_reg = PP_ON_DELAYS(pipe); ++ ++ WARN_ON(intel_dp->active_pipe != INVALID_PIPE); ++ ++ if (WARN_ON(pipe != PIPE_A && pipe != PIPE_B)) ++ return; ++ ++ edp_panel_vdd_off_sync(intel_dp); ++ ++ /* ++ * VLV seems to get confused when multiple power sequencers ++ * have the same port selected (even if only one has power/vdd ++ * enabled). The failure manifests as vlv_wait_port_ready() failing ++ * CHV on the other hand doesn't seem to mind having the same port ++ * selected in multiple power sequencers, but let's clear the ++ * port select always when logically disconnecting a power sequencer ++ * from a port. ++ */ ++ DRM_DEBUG_KMS("detaching pipe %c power sequencer from port %c\n", ++ pipe_name(pipe), port_name(intel_dig_port->base.port)); ++ I915_WRITE(pp_on_reg, 0); ++ POSTING_READ(pp_on_reg); ++ ++ intel_dp->pps_pipe = INVALID_PIPE; ++} ++ ++static void vlv_steal_power_sequencer(struct drm_i915_private *dev_priv, ++ enum pipe pipe) ++{ ++ struct intel_encoder *encoder; ++ ++ lockdep_assert_held(&dev_priv->pps_mutex); ++ ++ for_each_intel_dp(&dev_priv->drm, encoder) { ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ enum port port = encoder->port; ++ ++ WARN(intel_dp->active_pipe == pipe, ++ "stealing pipe %c power sequencer from active (e)DP port %c\n", ++ pipe_name(pipe), port_name(port)); ++ ++ if (intel_dp->pps_pipe != pipe) ++ continue; ++ ++ DRM_DEBUG_KMS("stealing pipe %c power sequencer from port %c\n", ++ pipe_name(pipe), port_name(port)); ++ ++ /* make sure vdd is off before we steal it */ ++ vlv_detach_power_sequencer(intel_dp); ++ } ++} ++ ++static void vlv_init_panel_power_sequencer(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ ++ lockdep_assert_held(&dev_priv->pps_mutex); ++ ++ WARN_ON(intel_dp->active_pipe != INVALID_PIPE); ++ ++ if (intel_dp->pps_pipe != INVALID_PIPE && ++ intel_dp->pps_pipe != crtc->pipe) { ++ /* ++ * If another power sequencer was being used on this ++ * port previously make sure to turn off vdd there while ++ * we still have control of it. ++ */ ++ vlv_detach_power_sequencer(intel_dp); ++ } ++ ++ /* ++ * We may be stealing the power ++ * sequencer from another port. ++ */ ++ vlv_steal_power_sequencer(dev_priv, crtc->pipe); ++ ++ intel_dp->active_pipe = crtc->pipe; ++ ++ if (!intel_dp_is_edp(intel_dp)) ++ return; ++ ++ /* now it's all ours */ ++ intel_dp->pps_pipe = crtc->pipe; ++ ++ DRM_DEBUG_KMS("initializing pipe %c power sequencer for port %c\n", ++ pipe_name(intel_dp->pps_pipe), port_name(encoder->port)); ++ ++ /* init power sequencer on this pipe and port */ ++ intel_dp_init_panel_power_sequencer(intel_dp); ++ intel_dp_init_panel_power_sequencer_registers(intel_dp, true); ++} ++ ++static void vlv_pre_enable_dp(struct intel_encoder *encoder, ++ const struct intel_crtc_state *pipe_config, ++ const struct drm_connector_state *conn_state) ++{ ++ vlv_phy_pre_encoder_enable(encoder, pipe_config); ++ ++ intel_enable_dp(encoder, pipe_config, conn_state); ++} ++ ++static void vlv_dp_pre_pll_enable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *pipe_config, ++ const struct drm_connector_state *conn_state) ++{ ++ intel_dp_prepare(encoder, pipe_config); ++ ++ vlv_phy_pre_pll_enable(encoder, pipe_config); ++} ++ ++static void chv_pre_enable_dp(struct intel_encoder *encoder, ++ const struct intel_crtc_state *pipe_config, ++ const struct drm_connector_state *conn_state) ++{ ++ chv_phy_pre_encoder_enable(encoder, pipe_config); ++ ++ intel_enable_dp(encoder, pipe_config, conn_state); ++ ++ /* Second common lane will stay alive on its own now */ ++ chv_phy_release_cl2_override(encoder); ++} ++ ++static void chv_dp_pre_pll_enable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *pipe_config, ++ const struct drm_connector_state *conn_state) ++{ ++ intel_dp_prepare(encoder, pipe_config); ++ ++ chv_phy_pre_pll_enable(encoder, pipe_config); ++} ++ ++static void chv_dp_post_pll_disable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ chv_phy_post_pll_disable(encoder, old_crtc_state); ++} ++ ++/* ++ * Fetch AUX CH registers 0x202 - 0x207 which contain ++ * link status information ++ */ ++bool ++intel_dp_get_link_status(struct intel_dp *intel_dp, u8 link_status[DP_LINK_STATUS_SIZE]) ++{ ++ return drm_dp_dpcd_read(&intel_dp->aux, DP_LANE0_1_STATUS, link_status, ++ DP_LINK_STATUS_SIZE) == DP_LINK_STATUS_SIZE; ++} ++ ++/* These are source-specific values. */ ++u8 ++intel_dp_voltage_max(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; ++ enum port port = encoder->port; ++ ++ if (HAS_DDI(dev_priv)) ++ return intel_ddi_dp_voltage_max(encoder); ++ else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ return DP_TRAIN_VOLTAGE_SWING_LEVEL_3; ++ else if (IS_IVYBRIDGE(dev_priv) && port == PORT_A) ++ return DP_TRAIN_VOLTAGE_SWING_LEVEL_2; ++ else if (HAS_PCH_CPT(dev_priv) && port != PORT_A) ++ return DP_TRAIN_VOLTAGE_SWING_LEVEL_3; ++ else ++ return DP_TRAIN_VOLTAGE_SWING_LEVEL_2; ++} ++ ++u8 ++intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, u8 voltage_swing) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; ++ enum port port = encoder->port; ++ ++ if (HAS_DDI(dev_priv)) { ++ return intel_ddi_dp_pre_emphasis_max(encoder, voltage_swing); ++ } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { ++ switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) { ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_0: ++ return DP_TRAIN_PRE_EMPH_LEVEL_3; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_1: ++ return DP_TRAIN_PRE_EMPH_LEVEL_2; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_2: ++ return DP_TRAIN_PRE_EMPH_LEVEL_1; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_3: ++ default: ++ return DP_TRAIN_PRE_EMPH_LEVEL_0; ++ } ++ } else if (IS_IVYBRIDGE(dev_priv) && port == PORT_A) { ++ switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) { ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_0: ++ return DP_TRAIN_PRE_EMPH_LEVEL_2; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_1: ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_2: ++ return DP_TRAIN_PRE_EMPH_LEVEL_1; ++ default: ++ return DP_TRAIN_PRE_EMPH_LEVEL_0; ++ } ++ } else { ++ switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) { ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_0: ++ return DP_TRAIN_PRE_EMPH_LEVEL_2; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_1: ++ return DP_TRAIN_PRE_EMPH_LEVEL_2; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_2: ++ return DP_TRAIN_PRE_EMPH_LEVEL_1; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_3: ++ default: ++ return DP_TRAIN_PRE_EMPH_LEVEL_0; ++ } ++ } ++} ++ ++static u32 vlv_signal_levels(struct intel_dp *intel_dp) ++{ ++ struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; ++ unsigned long demph_reg_value, preemph_reg_value, ++ uniqtranscale_reg_value; ++ u8 train_set = intel_dp->train_set[0]; ++ ++ switch (train_set & DP_TRAIN_PRE_EMPHASIS_MASK) { ++ case DP_TRAIN_PRE_EMPH_LEVEL_0: ++ preemph_reg_value = 0x0004000; ++ switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) { ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_0: ++ demph_reg_value = 0x2B405555; ++ uniqtranscale_reg_value = 0x552AB83A; ++ break; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_1: ++ demph_reg_value = 0x2B404040; ++ uniqtranscale_reg_value = 0x5548B83A; ++ break; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_2: ++ demph_reg_value = 0x2B245555; ++ uniqtranscale_reg_value = 0x5560B83A; ++ break; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_3: ++ demph_reg_value = 0x2B405555; ++ uniqtranscale_reg_value = 0x5598DA3A; ++ break; ++ default: ++ return 0; ++ } ++ break; ++ case DP_TRAIN_PRE_EMPH_LEVEL_1: ++ preemph_reg_value = 0x0002000; ++ switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) { ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_0: ++ demph_reg_value = 0x2B404040; ++ uniqtranscale_reg_value = 0x5552B83A; ++ break; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_1: ++ demph_reg_value = 0x2B404848; ++ uniqtranscale_reg_value = 0x5580B83A; ++ break; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_2: ++ demph_reg_value = 0x2B404040; ++ uniqtranscale_reg_value = 0x55ADDA3A; ++ break; ++ default: ++ return 0; ++ } ++ break; ++ case DP_TRAIN_PRE_EMPH_LEVEL_2: ++ preemph_reg_value = 0x0000000; ++ switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) { ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_0: ++ demph_reg_value = 0x2B305555; ++ uniqtranscale_reg_value = 0x5570B83A; ++ break; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_1: ++ demph_reg_value = 0x2B2B4040; ++ uniqtranscale_reg_value = 0x55ADDA3A; ++ break; ++ default: ++ return 0; ++ } ++ break; ++ case DP_TRAIN_PRE_EMPH_LEVEL_3: ++ preemph_reg_value = 0x0006000; ++ switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) { ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_0: ++ demph_reg_value = 0x1B405555; ++ uniqtranscale_reg_value = 0x55ADDA3A; ++ break; ++ default: ++ return 0; ++ } ++ break; ++ default: ++ return 0; ++ } ++ ++ vlv_set_phy_signal_level(encoder, demph_reg_value, preemph_reg_value, ++ uniqtranscale_reg_value, 0); ++ ++ return 0; ++} ++ ++static u32 chv_signal_levels(struct intel_dp *intel_dp) ++{ ++ struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; ++ u32 deemph_reg_value, margin_reg_value; ++ bool uniq_trans_scale = false; ++ u8 train_set = intel_dp->train_set[0]; ++ ++ switch (train_set & DP_TRAIN_PRE_EMPHASIS_MASK) { ++ case DP_TRAIN_PRE_EMPH_LEVEL_0: ++ switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) { ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_0: ++ deemph_reg_value = 128; ++ margin_reg_value = 52; ++ break; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_1: ++ deemph_reg_value = 128; ++ margin_reg_value = 77; ++ break; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_2: ++ deemph_reg_value = 128; ++ margin_reg_value = 102; ++ break; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_3: ++ deemph_reg_value = 128; ++ margin_reg_value = 154; ++ uniq_trans_scale = true; ++ break; ++ default: ++ return 0; ++ } ++ break; ++ case DP_TRAIN_PRE_EMPH_LEVEL_1: ++ switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) { ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_0: ++ deemph_reg_value = 85; ++ margin_reg_value = 78; ++ break; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_1: ++ deemph_reg_value = 85; ++ margin_reg_value = 116; ++ break; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_2: ++ deemph_reg_value = 85; ++ margin_reg_value = 154; ++ break; ++ default: ++ return 0; ++ } ++ break; ++ case DP_TRAIN_PRE_EMPH_LEVEL_2: ++ switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) { ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_0: ++ deemph_reg_value = 64; ++ margin_reg_value = 104; ++ break; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_1: ++ deemph_reg_value = 64; ++ margin_reg_value = 154; ++ break; ++ default: ++ return 0; ++ } ++ break; ++ case DP_TRAIN_PRE_EMPH_LEVEL_3: ++ switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) { ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_0: ++ deemph_reg_value = 43; ++ margin_reg_value = 154; ++ break; ++ default: ++ return 0; ++ } ++ break; ++ default: ++ return 0; ++ } ++ ++ chv_set_phy_signal_level(encoder, deemph_reg_value, ++ margin_reg_value, uniq_trans_scale); ++ ++ return 0; ++} ++ ++static u32 ++g4x_signal_levels(u8 train_set) ++{ ++ u32 signal_levels = 0; ++ ++ switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) { ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_0: ++ default: ++ signal_levels |= DP_VOLTAGE_0_4; ++ break; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_1: ++ signal_levels |= DP_VOLTAGE_0_6; ++ break; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_2: ++ signal_levels |= DP_VOLTAGE_0_8; ++ break; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_3: ++ signal_levels |= DP_VOLTAGE_1_2; ++ break; ++ } ++ switch (train_set & DP_TRAIN_PRE_EMPHASIS_MASK) { ++ case DP_TRAIN_PRE_EMPH_LEVEL_0: ++ default: ++ signal_levels |= DP_PRE_EMPHASIS_0; ++ break; ++ case DP_TRAIN_PRE_EMPH_LEVEL_1: ++ signal_levels |= DP_PRE_EMPHASIS_3_5; ++ break; ++ case DP_TRAIN_PRE_EMPH_LEVEL_2: ++ signal_levels |= DP_PRE_EMPHASIS_6; ++ break; ++ case DP_TRAIN_PRE_EMPH_LEVEL_3: ++ signal_levels |= DP_PRE_EMPHASIS_9_5; ++ break; ++ } ++ return signal_levels; ++} ++ ++/* SNB CPU eDP voltage swing and pre-emphasis control */ ++static u32 ++snb_cpu_edp_signal_levels(u8 train_set) ++{ ++ int signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK | ++ DP_TRAIN_PRE_EMPHASIS_MASK); ++ switch (signal_levels) { ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_0: ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_0: ++ return EDP_LINK_TRAIN_400_600MV_0DB_SNB_B; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_1: ++ return EDP_LINK_TRAIN_400MV_3_5DB_SNB_B; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_2: ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_2: ++ return EDP_LINK_TRAIN_400_600MV_6DB_SNB_B; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_1: ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_2 | DP_TRAIN_PRE_EMPH_LEVEL_1: ++ return EDP_LINK_TRAIN_600_800MV_3_5DB_SNB_B; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_2 | DP_TRAIN_PRE_EMPH_LEVEL_0: ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_3 | DP_TRAIN_PRE_EMPH_LEVEL_0: ++ return EDP_LINK_TRAIN_800_1200MV_0DB_SNB_B; ++ default: ++ DRM_DEBUG_KMS("Unsupported voltage swing/pre-emphasis level:" ++ "0x%x\n", signal_levels); ++ return EDP_LINK_TRAIN_400_600MV_0DB_SNB_B; ++ } ++} ++ ++/* IVB CPU eDP voltage swing and pre-emphasis control */ ++static u32 ++ivb_cpu_edp_signal_levels(u8 train_set) ++{ ++ int signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK | ++ DP_TRAIN_PRE_EMPHASIS_MASK); ++ switch (signal_levels) { ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_0: ++ return EDP_LINK_TRAIN_400MV_0DB_IVB; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_1: ++ return EDP_LINK_TRAIN_400MV_3_5DB_IVB; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_2: ++ return EDP_LINK_TRAIN_400MV_6DB_IVB; ++ ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_0: ++ return EDP_LINK_TRAIN_600MV_0DB_IVB; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_1: ++ return EDP_LINK_TRAIN_600MV_3_5DB_IVB; ++ ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_2 | DP_TRAIN_PRE_EMPH_LEVEL_0: ++ return EDP_LINK_TRAIN_800MV_0DB_IVB; ++ case DP_TRAIN_VOLTAGE_SWING_LEVEL_2 | DP_TRAIN_PRE_EMPH_LEVEL_1: ++ return EDP_LINK_TRAIN_800MV_3_5DB_IVB; ++ ++ default: ++ DRM_DEBUG_KMS("Unsupported voltage swing/pre-emphasis level:" ++ "0x%x\n", signal_levels); ++ return EDP_LINK_TRAIN_500MV_0DB_IVB; ++ } ++} ++ ++void ++intel_dp_set_signal_levels(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); ++ enum port port = intel_dig_port->base.port; ++ u32 signal_levels, mask = 0; ++ u8 train_set = intel_dp->train_set[0]; ++ ++ if (IS_GEN9_LP(dev_priv) || INTEL_GEN(dev_priv) >= 10) { ++ signal_levels = bxt_signal_levels(intel_dp); ++ } else if (HAS_DDI(dev_priv)) { ++ signal_levels = ddi_signal_levels(intel_dp); ++ mask = DDI_BUF_EMP_MASK; ++ } else if (IS_CHERRYVIEW(dev_priv)) { ++ signal_levels = chv_signal_levels(intel_dp); ++ } else if (IS_VALLEYVIEW(dev_priv)) { ++ signal_levels = vlv_signal_levels(intel_dp); ++ } else if (IS_IVYBRIDGE(dev_priv) && port == PORT_A) { ++ signal_levels = ivb_cpu_edp_signal_levels(train_set); ++ mask = EDP_LINK_TRAIN_VOL_EMP_MASK_IVB; ++ } else if (IS_GEN(dev_priv, 6) && port == PORT_A) { ++ signal_levels = snb_cpu_edp_signal_levels(train_set); ++ mask = EDP_LINK_TRAIN_VOL_EMP_MASK_SNB; ++ } else { ++ signal_levels = g4x_signal_levels(train_set); ++ mask = DP_VOLTAGE_MASK | DP_PRE_EMPHASIS_MASK; ++ } ++ ++ if (mask) ++ DRM_DEBUG_KMS("Using signal levels %08x\n", signal_levels); ++ ++ DRM_DEBUG_KMS("Using vswing level %d\n", ++ train_set & DP_TRAIN_VOLTAGE_SWING_MASK); ++ DRM_DEBUG_KMS("Using pre-emphasis level %d\n", ++ (train_set & DP_TRAIN_PRE_EMPHASIS_MASK) >> ++ DP_TRAIN_PRE_EMPHASIS_SHIFT); ++ ++ intel_dp->DP = (intel_dp->DP & ~mask) | signal_levels; ++ ++ I915_WRITE(intel_dp->output_reg, intel_dp->DP); ++ POSTING_READ(intel_dp->output_reg); ++} ++ ++void ++intel_dp_program_link_training_pattern(struct intel_dp *intel_dp, ++ u8 dp_train_pat) ++{ ++ struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); ++ struct drm_i915_private *dev_priv = ++ to_i915(intel_dig_port->base.base.dev); ++ ++ _intel_dp_set_link_train(intel_dp, &intel_dp->DP, dp_train_pat); ++ ++ I915_WRITE(intel_dp->output_reg, intel_dp->DP); ++ POSTING_READ(intel_dp->output_reg); ++} ++ ++void intel_dp_set_idle_link_train(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); ++ enum port port = intel_dig_port->base.port; ++ u32 val; ++ ++ if (!HAS_DDI(dev_priv)) ++ return; ++ ++ val = I915_READ(DP_TP_CTL(port)); ++ val &= ~DP_TP_CTL_LINK_TRAIN_MASK; ++ val |= DP_TP_CTL_LINK_TRAIN_IDLE; ++ I915_WRITE(DP_TP_CTL(port), val); ++ ++ /* ++ * On PORT_A we can have only eDP in SST mode. There the only reason ++ * we need to set idle transmission mode is to work around a HW issue ++ * where we enable the pipe while not in idle link-training mode. ++ * In this case there is requirement to wait for a minimum number of ++ * idle patterns to be sent. ++ */ ++ if (port == PORT_A) ++ return; ++ ++ if (intel_wait_for_register(&dev_priv->uncore, DP_TP_STATUS(port), ++ DP_TP_STATUS_IDLE_DONE, ++ DP_TP_STATUS_IDLE_DONE, ++ 1)) ++ DRM_ERROR("Timed out waiting for DP idle patterns\n"); ++} ++ ++static void ++intel_dp_link_down(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc); ++ enum port port = encoder->port; ++ u32 DP = intel_dp->DP; ++ ++ if (WARN_ON(HAS_DDI(dev_priv))) ++ return; ++ ++ if (WARN_ON((I915_READ(intel_dp->output_reg) & DP_PORT_EN) == 0)) ++ return; ++ ++ DRM_DEBUG_KMS("\n"); ++ ++ if ((IS_IVYBRIDGE(dev_priv) && port == PORT_A) || ++ (HAS_PCH_CPT(dev_priv) && port != PORT_A)) { ++ DP &= ~DP_LINK_TRAIN_MASK_CPT; ++ DP |= DP_LINK_TRAIN_PAT_IDLE_CPT; ++ } else { ++ DP &= ~DP_LINK_TRAIN_MASK; ++ DP |= DP_LINK_TRAIN_PAT_IDLE; ++ } ++ I915_WRITE(intel_dp->output_reg, DP); ++ POSTING_READ(intel_dp->output_reg); ++ ++ DP &= ~(DP_PORT_EN | DP_AUDIO_OUTPUT_ENABLE); ++ I915_WRITE(intel_dp->output_reg, DP); ++ POSTING_READ(intel_dp->output_reg); ++ ++ /* ++ * HW workaround for IBX, we need to move the port ++ * to transcoder A after disabling it to allow the ++ * matching HDMI port to be enabled on transcoder A. ++ */ ++ if (HAS_PCH_IBX(dev_priv) && crtc->pipe == PIPE_B && port != PORT_A) { ++ /* ++ * We get CPU/PCH FIFO underruns on the other pipe when ++ * doing the workaround. Sweep them under the rug. ++ */ ++ intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, false); ++ intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, false); ++ ++ /* always enable with pattern 1 (as per spec) */ ++ DP &= ~(DP_PIPE_SEL_MASK | DP_LINK_TRAIN_MASK); ++ DP |= DP_PORT_EN | DP_PIPE_SEL(PIPE_A) | ++ DP_LINK_TRAIN_PAT_1; ++ I915_WRITE(intel_dp->output_reg, DP); ++ POSTING_READ(intel_dp->output_reg); ++ ++ DP &= ~DP_PORT_EN; ++ I915_WRITE(intel_dp->output_reg, DP); ++ POSTING_READ(intel_dp->output_reg); ++ ++ intel_wait_for_vblank_if_active(dev_priv, PIPE_A); ++ intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, true); ++ intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true); ++ } ++ ++ msleep(intel_dp->panel_power_down_delay); ++ ++ intel_dp->DP = DP; ++ ++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { ++ intel_wakeref_t wakeref; ++ ++ with_pps_lock(intel_dp, wakeref) ++ intel_dp->active_pipe = INVALID_PIPE; ++ } ++} ++ ++static void ++intel_dp_extended_receiver_capabilities(struct intel_dp *intel_dp) ++{ ++ u8 dpcd_ext[6]; ++ ++ /* ++ * Prior to DP1.3 the bit represented by ++ * DP_EXTENDED_RECEIVER_CAP_FIELD_PRESENT was reserved. ++ * if it is set DP_DPCD_REV at 0000h could be at a value less than ++ * the true capability of the panel. The only way to check is to ++ * then compare 0000h and 2200h. ++ */ ++ if (!(intel_dp->dpcd[DP_TRAINING_AUX_RD_INTERVAL] & ++ DP_EXTENDED_RECEIVER_CAP_FIELD_PRESENT)) ++ return; ++ ++ if (drm_dp_dpcd_read(&intel_dp->aux, DP_DP13_DPCD_REV, ++ &dpcd_ext, sizeof(dpcd_ext)) != sizeof(dpcd_ext)) { ++ DRM_ERROR("DPCD failed read at extended capabilities\n"); ++ return; ++ } ++ ++ if (intel_dp->dpcd[DP_DPCD_REV] > dpcd_ext[DP_DPCD_REV]) { ++ DRM_DEBUG_KMS("DPCD extended DPCD rev less than base DPCD rev\n"); ++ return; ++ } ++ ++ if (!memcmp(intel_dp->dpcd, dpcd_ext, sizeof(dpcd_ext))) ++ return; ++ ++ DRM_DEBUG_KMS("Base DPCD: %*ph\n", ++ (int)sizeof(intel_dp->dpcd), intel_dp->dpcd); ++ ++ memcpy(intel_dp->dpcd, dpcd_ext, sizeof(dpcd_ext)); ++} ++ ++bool ++intel_dp_read_dpcd(struct intel_dp *intel_dp) ++{ ++ if (drm_dp_dpcd_read(&intel_dp->aux, 0x000, intel_dp->dpcd, ++ sizeof(intel_dp->dpcd)) < 0) ++ return false; /* aux transfer failed */ ++ ++ intel_dp_extended_receiver_capabilities(intel_dp); ++ ++ DRM_DEBUG_KMS("DPCD: %*ph\n", (int) sizeof(intel_dp->dpcd), intel_dp->dpcd); ++ ++ return intel_dp->dpcd[DP_DPCD_REV] != 0; ++} ++ ++static void intel_dp_get_dsc_sink_cap(struct intel_dp *intel_dp) ++{ ++ /* ++ * Clear the cached register set to avoid using stale values ++ * for the sinks that do not support DSC. ++ */ ++ memset(intel_dp->dsc_dpcd, 0, sizeof(intel_dp->dsc_dpcd)); ++ ++ /* Clear fec_capable to avoid using stale values */ ++ intel_dp->fec_capable = 0; ++ ++ /* Cache the DSC DPCD if eDP or DP rev >= 1.4 */ ++ if (intel_dp->dpcd[DP_DPCD_REV] >= 0x14 || ++ intel_dp->edp_dpcd[0] >= DP_EDP_14) { ++ if (drm_dp_dpcd_read(&intel_dp->aux, DP_DSC_SUPPORT, ++ intel_dp->dsc_dpcd, ++ sizeof(intel_dp->dsc_dpcd)) < 0) ++ DRM_ERROR("Failed to read DPCD register 0x%x\n", ++ DP_DSC_SUPPORT); ++ ++ DRM_DEBUG_KMS("DSC DPCD: %*ph\n", ++ (int)sizeof(intel_dp->dsc_dpcd), ++ intel_dp->dsc_dpcd); ++ ++ /* FEC is supported only on DP 1.4 */ ++ if (!intel_dp_is_edp(intel_dp) && ++ drm_dp_dpcd_readb(&intel_dp->aux, DP_FEC_CAPABILITY, ++ &intel_dp->fec_capable) < 0) ++ DRM_ERROR("Failed to read FEC DPCD register\n"); ++ ++ DRM_DEBUG_KMS("FEC CAPABILITY: %x\n", intel_dp->fec_capable); ++ } ++} ++ ++static bool ++intel_edp_init_dpcd(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = ++ to_i915(dp_to_dig_port(intel_dp)->base.base.dev); ++ ++ /* this function is meant to be called only once */ ++ WARN_ON(intel_dp->dpcd[DP_DPCD_REV] != 0); ++ ++ if (!intel_dp_read_dpcd(intel_dp)) ++ return false; ++ ++ drm_dp_read_desc(&intel_dp->aux, &intel_dp->desc, ++ drm_dp_is_branch(intel_dp->dpcd)); ++ ++ if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11) ++ dev_priv->no_aux_handshake = intel_dp->dpcd[DP_MAX_DOWNSPREAD] & ++ DP_NO_AUX_HANDSHAKE_LINK_TRAINING; ++ ++ /* ++ * Read the eDP display control registers. ++ * ++ * Do this independent of DP_DPCD_DISPLAY_CONTROL_CAPABLE bit in ++ * DP_EDP_CONFIGURATION_CAP, because some buggy displays do not have it ++ * set, but require eDP 1.4+ detection (e.g. for supported link rates ++ * method). The display control registers should read zero if they're ++ * not supported anyway. ++ */ ++ if (drm_dp_dpcd_read(&intel_dp->aux, DP_EDP_DPCD_REV, ++ intel_dp->edp_dpcd, sizeof(intel_dp->edp_dpcd)) == ++ sizeof(intel_dp->edp_dpcd)) ++ DRM_DEBUG_KMS("eDP DPCD: %*ph\n", (int) sizeof(intel_dp->edp_dpcd), ++ intel_dp->edp_dpcd); ++ ++ /* ++ * This has to be called after intel_dp->edp_dpcd is filled, PSR checks ++ * for SET_POWER_CAPABLE bit in intel_dp->edp_dpcd[1] ++ */ ++ intel_psr_init_dpcd(intel_dp); ++ ++ /* Read the eDP 1.4+ supported link rates. */ ++ if (intel_dp->edp_dpcd[0] >= DP_EDP_14) { ++ __le16 sink_rates[DP_MAX_SUPPORTED_RATES]; ++ int i; ++ ++ drm_dp_dpcd_read(&intel_dp->aux, DP_SUPPORTED_LINK_RATES, ++ sink_rates, sizeof(sink_rates)); ++ ++ for (i = 0; i < ARRAY_SIZE(sink_rates); i++) { ++ int val = le16_to_cpu(sink_rates[i]); ++ ++ if (val == 0) ++ break; ++ ++ /* Value read multiplied by 200kHz gives the per-lane ++ * link rate in kHz. The source rates are, however, ++ * stored in terms of LS_Clk kHz. The full conversion ++ * back to symbols is ++ * (val * 200kHz)*(8/10 ch. encoding)*(1/8 bit to Byte) ++ */ ++ intel_dp->sink_rates[i] = (val * 200) / 10; ++ } ++ intel_dp->num_sink_rates = i; ++ } ++ ++ /* ++ * Use DP_LINK_RATE_SET if DP_SUPPORTED_LINK_RATES are available, ++ * default to DP_MAX_LINK_RATE and DP_LINK_BW_SET otherwise. ++ */ ++ if (intel_dp->num_sink_rates) ++ intel_dp->use_rate_select = true; ++ else ++ intel_dp_set_sink_rates(intel_dp); ++ ++ intel_dp_set_common_rates(intel_dp); ++ ++ /* Read the eDP DSC DPCD registers */ ++ if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) ++ intel_dp_get_dsc_sink_cap(intel_dp); ++ ++ return true; ++} ++ ++ ++static bool ++intel_dp_get_dpcd(struct intel_dp *intel_dp) ++{ ++ if (!intel_dp_read_dpcd(intel_dp)) ++ return false; ++ ++ /* Don't clobber cached eDP rates. */ ++ if (!intel_dp_is_edp(intel_dp)) { ++ intel_dp_set_sink_rates(intel_dp); ++ intel_dp_set_common_rates(intel_dp); ++ } ++ ++ /* ++ * Some eDP panels do not set a valid value for sink count, that is why ++ * it don't care about read it here and in intel_edp_init_dpcd(). ++ */ ++ if (!intel_dp_is_edp(intel_dp)) { ++ u8 count; ++ ssize_t r; ++ ++ r = drm_dp_dpcd_readb(&intel_dp->aux, DP_SINK_COUNT, &count); ++ if (r < 1) ++ return false; ++ ++ /* ++ * Sink count can change between short pulse hpd hence ++ * a member variable in intel_dp will track any changes ++ * between short pulse interrupts. ++ */ ++ intel_dp->sink_count = DP_GET_SINK_COUNT(count); ++ ++ /* ++ * SINK_COUNT == 0 and DOWNSTREAM_PORT_PRESENT == 1 implies that ++ * a dongle is present but no display. Unless we require to know ++ * if a dongle is present or not, we don't need to update ++ * downstream port information. So, an early return here saves ++ * time from performing other operations which are not required. ++ */ ++ if (!intel_dp->sink_count) ++ return false; ++ } ++ ++ if (!drm_dp_is_branch(intel_dp->dpcd)) ++ return true; /* native DP sink */ ++ ++ if (intel_dp->dpcd[DP_DPCD_REV] == 0x10) ++ return true; /* no per-port downstream info */ ++ ++ if (drm_dp_dpcd_read(&intel_dp->aux, DP_DOWNSTREAM_PORT_0, ++ intel_dp->downstream_ports, ++ DP_MAX_DOWNSTREAM_PORTS) < 0) ++ return false; /* downstream port status fetch failed */ ++ ++ return true; ++} ++ ++static bool ++intel_dp_sink_can_mst(struct intel_dp *intel_dp) ++{ ++ u8 mstm_cap; ++ ++ if (intel_dp->dpcd[DP_DPCD_REV] < 0x12) ++ return false; ++ ++ if (drm_dp_dpcd_readb(&intel_dp->aux, DP_MSTM_CAP, &mstm_cap) != 1) ++ return false; ++ ++ return mstm_cap & DP_MST_CAP; ++} ++ ++static bool ++intel_dp_can_mst(struct intel_dp *intel_dp) ++{ ++ return i915_modparams.enable_dp_mst && ++ intel_dp->can_mst && ++ intel_dp_sink_can_mst(intel_dp); ++} ++ ++static void ++intel_dp_configure_mst(struct intel_dp *intel_dp) ++{ ++ struct intel_encoder *encoder = ++ &dp_to_dig_port(intel_dp)->base; ++ bool sink_can_mst = intel_dp_sink_can_mst(intel_dp); ++ ++ DRM_DEBUG_KMS("MST support? port %c: %s, sink: %s, modparam: %s\n", ++ port_name(encoder->port), yesno(intel_dp->can_mst), ++ yesno(sink_can_mst), yesno(i915_modparams.enable_dp_mst)); ++ ++ if (!intel_dp->can_mst) ++ return; ++ ++ intel_dp->is_mst = sink_can_mst && ++ i915_modparams.enable_dp_mst; ++ ++ drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr, ++ intel_dp->is_mst); ++} ++ ++static bool ++intel_dp_get_sink_irq_esi(struct intel_dp *intel_dp, u8 *sink_irq_vector) ++{ ++ return drm_dp_dpcd_read(&intel_dp->aux, DP_SINK_COUNT_ESI, ++ sink_irq_vector, DP_DPRX_ESI_LEN) == ++ DP_DPRX_ESI_LEN; ++} ++ ++u16 intel_dp_dsc_get_output_bpp(int link_clock, u8 lane_count, ++ int mode_clock, int mode_hdisplay) ++{ ++ u16 bits_per_pixel, max_bpp_small_joiner_ram; ++ int i; ++ ++ /* ++ * Available Link Bandwidth(Kbits/sec) = (NumberOfLanes)* ++ * (LinkSymbolClock)* 8 * ((100-FECOverhead)/100)*(TimeSlotsPerMTP) ++ * FECOverhead = 2.4%, for SST -> TimeSlotsPerMTP is 1, ++ * for MST -> TimeSlotsPerMTP has to be calculated ++ */ ++ bits_per_pixel = (link_clock * lane_count * 8 * ++ DP_DSC_FEC_OVERHEAD_FACTOR) / ++ mode_clock; ++ ++ /* Small Joiner Check: output bpp <= joiner RAM (bits) / Horiz. width */ ++ max_bpp_small_joiner_ram = DP_DSC_MAX_SMALL_JOINER_RAM_BUFFER / ++ mode_hdisplay; ++ ++ /* ++ * Greatest allowed DSC BPP = MIN (output BPP from avaialble Link BW ++ * check, output bpp from small joiner RAM check) ++ */ ++ bits_per_pixel = min(bits_per_pixel, max_bpp_small_joiner_ram); ++ ++ /* Error out if the max bpp is less than smallest allowed valid bpp */ ++ if (bits_per_pixel < valid_dsc_bpp[0]) { ++ DRM_DEBUG_KMS("Unsupported BPP %d\n", bits_per_pixel); ++ return 0; ++ } ++ ++ /* Find the nearest match in the array of known BPPs from VESA */ ++ for (i = 0; i < ARRAY_SIZE(valid_dsc_bpp) - 1; i++) { ++ if (bits_per_pixel < valid_dsc_bpp[i + 1]) ++ break; ++ } ++ bits_per_pixel = valid_dsc_bpp[i]; ++ ++ /* ++ * Compressed BPP in U6.4 format so multiply by 16, for Gen 11, ++ * fractional part is 0 ++ */ ++ return bits_per_pixel << 4; ++} ++ ++u8 intel_dp_dsc_get_slice_count(struct intel_dp *intel_dp, ++ int mode_clock, ++ int mode_hdisplay) ++{ ++ u8 min_slice_count, i; ++ int max_slice_width; ++ ++ if (mode_clock <= DP_DSC_PEAK_PIXEL_RATE) ++ min_slice_count = DIV_ROUND_UP(mode_clock, ++ DP_DSC_MAX_ENC_THROUGHPUT_0); ++ else ++ min_slice_count = DIV_ROUND_UP(mode_clock, ++ DP_DSC_MAX_ENC_THROUGHPUT_1); ++ ++ max_slice_width = drm_dp_dsc_sink_max_slice_width(intel_dp->dsc_dpcd); ++ if (max_slice_width < DP_DSC_MIN_SLICE_WIDTH_VALUE) { ++ DRM_DEBUG_KMS("Unsupported slice width %d by DP DSC Sink device\n", ++ max_slice_width); ++ return 0; ++ } ++ /* Also take into account max slice width */ ++ min_slice_count = min_t(u8, min_slice_count, ++ DIV_ROUND_UP(mode_hdisplay, ++ max_slice_width)); ++ ++ /* Find the closest match to the valid slice count values */ ++ for (i = 0; i < ARRAY_SIZE(valid_dsc_slicecount); i++) { ++ if (valid_dsc_slicecount[i] > ++ drm_dp_dsc_sink_max_slice_count(intel_dp->dsc_dpcd, ++ false)) ++ break; ++ if (min_slice_count <= valid_dsc_slicecount[i]) ++ return valid_dsc_slicecount[i]; ++ } ++ ++ DRM_DEBUG_KMS("Unsupported Slice Count %d\n", min_slice_count); ++ return 0; ++} ++ ++static u8 intel_dp_autotest_link_training(struct intel_dp *intel_dp) ++{ ++ int status = 0; ++ int test_link_rate; ++ u8 test_lane_count, test_link_bw; ++ /* (DP CTS 1.2) ++ * 4.3.1.11 ++ */ ++ /* Read the TEST_LANE_COUNT and TEST_LINK_RTAE fields (DP CTS 3.1.4) */ ++ status = drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_LANE_COUNT, ++ &test_lane_count); ++ ++ if (status <= 0) { ++ DRM_DEBUG_KMS("Lane count read failed\n"); ++ return DP_TEST_NAK; ++ } ++ test_lane_count &= DP_MAX_LANE_COUNT_MASK; ++ ++ status = drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_LINK_RATE, ++ &test_link_bw); ++ if (status <= 0) { ++ DRM_DEBUG_KMS("Link Rate read failed\n"); ++ return DP_TEST_NAK; ++ } ++ test_link_rate = drm_dp_bw_code_to_link_rate(test_link_bw); ++ ++ /* Validate the requested link rate and lane count */ ++ if (!intel_dp_link_params_valid(intel_dp, test_link_rate, ++ test_lane_count)) ++ return DP_TEST_NAK; ++ ++ intel_dp->compliance.test_lane_count = test_lane_count; ++ intel_dp->compliance.test_link_rate = test_link_rate; ++ ++ return DP_TEST_ACK; ++} ++ ++static u8 intel_dp_autotest_video_pattern(struct intel_dp *intel_dp) ++{ ++ u8 test_pattern; ++ u8 test_misc; ++ __be16 h_width, v_height; ++ int status = 0; ++ ++ /* Read the TEST_PATTERN (DP CTS 3.1.5) */ ++ status = drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_PATTERN, ++ &test_pattern); ++ if (status <= 0) { ++ DRM_DEBUG_KMS("Test pattern read failed\n"); ++ return DP_TEST_NAK; ++ } ++ if (test_pattern != DP_COLOR_RAMP) ++ return DP_TEST_NAK; ++ ++ status = drm_dp_dpcd_read(&intel_dp->aux, DP_TEST_H_WIDTH_HI, ++ &h_width, 2); ++ if (status <= 0) { ++ DRM_DEBUG_KMS("H Width read failed\n"); ++ return DP_TEST_NAK; ++ } ++ ++ status = drm_dp_dpcd_read(&intel_dp->aux, DP_TEST_V_HEIGHT_HI, ++ &v_height, 2); ++ if (status <= 0) { ++ DRM_DEBUG_KMS("V Height read failed\n"); ++ return DP_TEST_NAK; ++ } ++ ++ status = drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_MISC0, ++ &test_misc); ++ if (status <= 0) { ++ DRM_DEBUG_KMS("TEST MISC read failed\n"); ++ return DP_TEST_NAK; ++ } ++ if ((test_misc & DP_TEST_COLOR_FORMAT_MASK) != DP_COLOR_FORMAT_RGB) ++ return DP_TEST_NAK; ++ if (test_misc & DP_TEST_DYNAMIC_RANGE_CEA) ++ return DP_TEST_NAK; ++ switch (test_misc & DP_TEST_BIT_DEPTH_MASK) { ++ case DP_TEST_BIT_DEPTH_6: ++ intel_dp->compliance.test_data.bpc = 6; ++ break; ++ case DP_TEST_BIT_DEPTH_8: ++ intel_dp->compliance.test_data.bpc = 8; ++ break; ++ default: ++ return DP_TEST_NAK; ++ } ++ ++ intel_dp->compliance.test_data.video_pattern = test_pattern; ++ intel_dp->compliance.test_data.hdisplay = be16_to_cpu(h_width); ++ intel_dp->compliance.test_data.vdisplay = be16_to_cpu(v_height); ++ /* Set test active flag here so userspace doesn't interrupt things */ ++ intel_dp->compliance.test_active = 1; ++ ++ return DP_TEST_ACK; ++} ++ ++static u8 intel_dp_autotest_edid(struct intel_dp *intel_dp) ++{ ++ u8 test_result = DP_TEST_ACK; ++ struct intel_connector *intel_connector = intel_dp->attached_connector; ++ struct drm_connector *connector = &intel_connector->base; ++ ++ if (intel_connector->detect_edid == NULL || ++ connector->edid_corrupt || ++ intel_dp->aux.i2c_defer_count > 6) { ++ /* Check EDID read for NACKs, DEFERs and corruption ++ * (DP CTS 1.2 Core r1.1) ++ * 4.2.2.4 : Failed EDID read, I2C_NAK ++ * 4.2.2.5 : Failed EDID read, I2C_DEFER ++ * 4.2.2.6 : EDID corruption detected ++ * Use failsafe mode for all cases ++ */ ++ if (intel_dp->aux.i2c_nack_count > 0 || ++ intel_dp->aux.i2c_defer_count > 0) ++ DRM_DEBUG_KMS("EDID read had %d NACKs, %d DEFERs\n", ++ intel_dp->aux.i2c_nack_count, ++ intel_dp->aux.i2c_defer_count); ++ intel_dp->compliance.test_data.edid = INTEL_DP_RESOLUTION_FAILSAFE; ++ } else { ++ struct edid *block = intel_connector->detect_edid; ++ ++ /* We have to write the checksum ++ * of the last block read ++ */ ++ block += intel_connector->detect_edid->extensions; ++ ++ if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_EDID_CHECKSUM, ++ block->checksum) <= 0) ++ DRM_DEBUG_KMS("Failed to write EDID checksum\n"); ++ ++ test_result = DP_TEST_ACK | DP_TEST_EDID_CHECKSUM_WRITE; ++ intel_dp->compliance.test_data.edid = INTEL_DP_RESOLUTION_PREFERRED; ++ } ++ ++ /* Set test active flag here so userspace doesn't interrupt things */ ++ intel_dp->compliance.test_active = 1; ++ ++ return test_result; ++} ++ ++static u8 intel_dp_autotest_phy_pattern(struct intel_dp *intel_dp) ++{ ++ u8 test_result = DP_TEST_NAK; ++ return test_result; ++} ++ ++static void intel_dp_handle_test_request(struct intel_dp *intel_dp) ++{ ++ u8 response = DP_TEST_NAK; ++ u8 request = 0; ++ int status; ++ ++ status = drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_REQUEST, &request); ++ if (status <= 0) { ++ DRM_DEBUG_KMS("Could not read test request from sink\n"); ++ goto update_status; ++ } ++ ++ switch (request) { ++ case DP_TEST_LINK_TRAINING: ++ DRM_DEBUG_KMS("LINK_TRAINING test requested\n"); ++ response = intel_dp_autotest_link_training(intel_dp); ++ break; ++ case DP_TEST_LINK_VIDEO_PATTERN: ++ DRM_DEBUG_KMS("TEST_PATTERN test requested\n"); ++ response = intel_dp_autotest_video_pattern(intel_dp); ++ break; ++ case DP_TEST_LINK_EDID_READ: ++ DRM_DEBUG_KMS("EDID test requested\n"); ++ response = intel_dp_autotest_edid(intel_dp); ++ break; ++ case DP_TEST_LINK_PHY_TEST_PATTERN: ++ DRM_DEBUG_KMS("PHY_PATTERN test requested\n"); ++ response = intel_dp_autotest_phy_pattern(intel_dp); ++ break; ++ default: ++ DRM_DEBUG_KMS("Invalid test request '%02x'\n", request); ++ break; ++ } ++ ++ if (response & DP_TEST_ACK) ++ intel_dp->compliance.test_type = request; ++ ++update_status: ++ status = drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_RESPONSE, response); ++ if (status <= 0) ++ DRM_DEBUG_KMS("Could not write test response to sink\n"); ++} ++ ++static int ++intel_dp_check_mst_status(struct intel_dp *intel_dp) ++{ ++ bool bret; ++ ++ if (intel_dp->is_mst) { ++ u8 esi[DP_DPRX_ESI_LEN] = { 0 }; ++ int ret = 0; ++ int retry; ++ bool handled; ++ ++ WARN_ON_ONCE(intel_dp->active_mst_links < 0); ++ bret = intel_dp_get_sink_irq_esi(intel_dp, esi); ++go_again: ++ if (bret == true) { ++ ++ /* check link status - esi[10] = 0x200c */ ++ if (intel_dp->active_mst_links > 0 && ++ !drm_dp_channel_eq_ok(&esi[10], intel_dp->lane_count)) { ++ DRM_DEBUG_KMS("channel EQ not ok, retraining\n"); ++ intel_dp_start_link_train(intel_dp); ++ intel_dp_stop_link_train(intel_dp); ++ } ++ ++ DRM_DEBUG_KMS("got esi %3ph\n", esi); ++ ret = drm_dp_mst_hpd_irq(&intel_dp->mst_mgr, esi, &handled); ++ ++ if (handled) { ++ for (retry = 0; retry < 3; retry++) { ++ int wret; ++ wret = drm_dp_dpcd_write(&intel_dp->aux, ++ DP_SINK_COUNT_ESI+1, ++ &esi[1], 3); ++ if (wret == 3) { ++ break; ++ } ++ } ++ ++ bret = intel_dp_get_sink_irq_esi(intel_dp, esi); ++ if (bret == true) { ++ DRM_DEBUG_KMS("got esi2 %3ph\n", esi); ++ goto go_again; ++ } ++ } else ++ ret = 0; ++ ++ return ret; ++ } else { ++ DRM_DEBUG_KMS("failed to get ESI - device may have failed\n"); ++ intel_dp->is_mst = false; ++ drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr, ++ intel_dp->is_mst); ++ } ++ } ++ return -EINVAL; ++} ++ ++static bool ++intel_dp_needs_link_retrain(struct intel_dp *intel_dp) ++{ ++ u8 link_status[DP_LINK_STATUS_SIZE]; ++ ++ if (!intel_dp->link_trained) ++ return false; ++ ++ /* ++ * While PSR source HW is enabled, it will control main-link sending ++ * frames, enabling and disabling it so trying to do a retrain will fail ++ * as the link would or not be on or it could mix training patterns ++ * and frame data at the same time causing retrain to fail. ++ * Also when exiting PSR, HW will retrain the link anyways fixing ++ * any link status error. ++ */ ++ if (intel_psr_enabled(intel_dp)) ++ return false; ++ ++ if (!intel_dp_get_link_status(intel_dp, link_status)) ++ return false; ++ ++ /* ++ * Validate the cached values of intel_dp->link_rate and ++ * intel_dp->lane_count before attempting to retrain. ++ */ ++ if (!intel_dp_link_params_valid(intel_dp, intel_dp->link_rate, ++ intel_dp->lane_count)) ++ return false; ++ ++ /* Retrain if Channel EQ or CR not ok */ ++ return !drm_dp_channel_eq_ok(link_status, intel_dp->lane_count); ++} ++ ++int intel_dp_retrain_link(struct intel_encoder *encoder, ++ struct drm_modeset_acquire_ctx *ctx) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ struct intel_connector *connector = intel_dp->attached_connector; ++ struct drm_connector_state *conn_state; ++ struct intel_crtc_state *crtc_state; ++ struct intel_crtc *crtc; ++ int ret; ++ ++ /* FIXME handle the MST connectors as well */ ++ ++ if (!connector || connector->base.status != connector_status_connected) ++ return 0; ++ ++ ret = drm_modeset_lock(&dev_priv->drm.mode_config.connection_mutex, ++ ctx); ++ if (ret) ++ return ret; ++ ++ conn_state = connector->base.state; ++ ++ crtc = to_intel_crtc(conn_state->crtc); ++ if (!crtc) ++ return 0; ++ ++ ret = drm_modeset_lock(&crtc->base.mutex, ctx); ++ if (ret) ++ return ret; ++ ++ crtc_state = to_intel_crtc_state(crtc->base.state); ++ ++ WARN_ON(!intel_crtc_has_dp_encoder(crtc_state)); ++ ++ if (!crtc_state->base.active) ++ return 0; ++ ++ if (conn_state->commit && ++ !try_wait_for_completion(&conn_state->commit->hw_done)) ++ return 0; ++ ++ if (!intel_dp_needs_link_retrain(intel_dp)) ++ return 0; ++ ++ /* Suppress underruns caused by re-training */ ++ intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false); ++ if (crtc_state->has_pch_encoder) ++ intel_set_pch_fifo_underrun_reporting(dev_priv, ++ intel_crtc_pch_transcoder(crtc), false); ++ ++ intel_dp_start_link_train(intel_dp); ++ intel_dp_stop_link_train(intel_dp); ++ ++ /* Keep underrun reporting disabled until things are stable */ ++ intel_wait_for_vblank(dev_priv, crtc->pipe); ++ ++ intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true); ++ if (crtc_state->has_pch_encoder) ++ intel_set_pch_fifo_underrun_reporting(dev_priv, ++ intel_crtc_pch_transcoder(crtc), true); ++ ++ return 0; ++} ++ ++/* ++ * If display is now connected check links status, ++ * there has been known issues of link loss triggering ++ * long pulse. ++ * ++ * Some sinks (eg. ASUS PB287Q) seem to perform some ++ * weird HPD ping pong during modesets. So we can apparently ++ * end up with HPD going low during a modeset, and then ++ * going back up soon after. And once that happens we must ++ * retrain the link to get a picture. That's in case no ++ * userspace component reacted to intermittent HPD dip. ++ */ ++static bool intel_dp_hotplug(struct intel_encoder *encoder, ++ struct intel_connector *connector) ++{ ++ struct drm_modeset_acquire_ctx ctx; ++ bool changed; ++ int ret; ++ ++ changed = intel_encoder_hotplug(encoder, connector); ++ ++ drm_modeset_acquire_init(&ctx, 0); ++ ++ for (;;) { ++ ret = intel_dp_retrain_link(encoder, &ctx); ++ ++ if (ret == -EDEADLK) { ++ drm_modeset_backoff(&ctx); ++ continue; ++ } ++ ++ break; ++ } ++ ++ drm_modeset_drop_locks(&ctx); ++ drm_modeset_acquire_fini(&ctx); ++ WARN(ret, "Acquiring modeset locks failed with %i\n", ret); ++ ++ return changed; ++} ++ ++static void intel_dp_check_service_irq(struct intel_dp *intel_dp) ++{ ++ u8 val; ++ ++ if (intel_dp->dpcd[DP_DPCD_REV] < 0x11) ++ return; ++ ++ if (drm_dp_dpcd_readb(&intel_dp->aux, ++ DP_DEVICE_SERVICE_IRQ_VECTOR, &val) != 1 || !val) ++ return; ++ ++ drm_dp_dpcd_writeb(&intel_dp->aux, DP_DEVICE_SERVICE_IRQ_VECTOR, val); ++ ++ if (val & DP_AUTOMATED_TEST_REQUEST) ++ intel_dp_handle_test_request(intel_dp); ++ ++ if (val & DP_CP_IRQ) ++ intel_hdcp_handle_cp_irq(intel_dp->attached_connector); ++ ++ if (val & DP_SINK_SPECIFIC_IRQ) ++ DRM_DEBUG_DRIVER("Sink specific irq unhandled\n"); ++} ++ ++/* ++ * According to DP spec ++ * 5.1.2: ++ * 1. Read DPCD ++ * 2. Configure link according to Receiver Capabilities ++ * 3. Use Link Training from 2.5.3.3 and 3.5.1.3 ++ * 4. Check link status on receipt of hot-plug interrupt ++ * ++ * intel_dp_short_pulse - handles short pulse interrupts ++ * when full detection is not required. ++ * Returns %true if short pulse is handled and full detection ++ * is NOT required and %false otherwise. ++ */ ++static bool ++intel_dp_short_pulse(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ u8 old_sink_count = intel_dp->sink_count; ++ bool ret; ++ ++ /* ++ * Clearing compliance test variables to allow capturing ++ * of values for next automated test request. ++ */ ++ memset(&intel_dp->compliance, 0, sizeof(intel_dp->compliance)); ++ ++ /* ++ * Now read the DPCD to see if it's actually running ++ * If the current value of sink count doesn't match with ++ * the value that was stored earlier or dpcd read failed ++ * we need to do full detection ++ */ ++ ret = intel_dp_get_dpcd(intel_dp); ++ ++ if ((old_sink_count != intel_dp->sink_count) || !ret) { ++ /* No need to proceed if we are going to do full detect */ ++ return false; ++ } ++ ++ intel_dp_check_service_irq(intel_dp); ++ ++ /* Handle CEC interrupts, if any */ ++ drm_dp_cec_irq(&intel_dp->aux); ++ ++ /* defer to the hotplug work for link retraining if needed */ ++ if (intel_dp_needs_link_retrain(intel_dp)) ++ return false; ++ ++ intel_psr_short_pulse(intel_dp); ++ ++ if (intel_dp->compliance.test_type == DP_TEST_LINK_TRAINING) { ++ DRM_DEBUG_KMS("Link Training Compliance Test requested\n"); ++ /* Send a Hotplug Uevent to userspace to start modeset */ ++ drm_kms_helper_hotplug_event(&dev_priv->drm); ++ } ++ ++ return true; ++} ++ ++/* XXX this is probably wrong for multiple downstream ports */ ++static enum drm_connector_status ++intel_dp_detect_dpcd(struct intel_dp *intel_dp) ++{ ++ struct intel_lspcon *lspcon = dp_to_lspcon(intel_dp); ++ u8 *dpcd = intel_dp->dpcd; ++ u8 type; ++ ++ if (lspcon->active) ++ lspcon_resume(lspcon); ++ ++ if (!intel_dp_get_dpcd(intel_dp)) ++ return connector_status_disconnected; ++ ++ if (intel_dp_is_edp(intel_dp)) ++ return connector_status_connected; ++ ++ /* if there's no downstream port, we're done */ ++ if (!drm_dp_is_branch(dpcd)) ++ return connector_status_connected; ++ ++ /* If we're HPD-aware, SINK_COUNT changes dynamically */ ++ if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11 && ++ intel_dp->downstream_ports[0] & DP_DS_PORT_HPD) { ++ ++ return intel_dp->sink_count ? ++ connector_status_connected : connector_status_disconnected; ++ } ++ ++ if (intel_dp_can_mst(intel_dp)) ++ return connector_status_connected; ++ ++ /* If no HPD, poke DDC gently */ ++ if (drm_probe_ddc(&intel_dp->aux.ddc)) ++ return connector_status_connected; ++ ++ /* Well we tried, say unknown for unreliable port types */ ++ if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11) { ++ type = intel_dp->downstream_ports[0] & DP_DS_PORT_TYPE_MASK; ++ if (type == DP_DS_PORT_TYPE_VGA || ++ type == DP_DS_PORT_TYPE_NON_EDID) ++ return connector_status_unknown; ++ } else { ++ type = intel_dp->dpcd[DP_DOWNSTREAMPORT_PRESENT] & ++ DP_DWN_STRM_PORT_TYPE_MASK; ++ if (type == DP_DWN_STRM_PORT_TYPE_ANALOG || ++ type == DP_DWN_STRM_PORT_TYPE_OTHER) ++ return connector_status_unknown; ++ } ++ ++ /* Anything else is out of spec, warn and ignore */ ++ DRM_DEBUG_KMS("Broken DP branch device, ignoring\n"); ++ return connector_status_disconnected; ++} ++ ++static enum drm_connector_status ++edp_detect(struct intel_dp *intel_dp) ++{ ++ return connector_status_connected; ++} ++ ++static bool ibx_digital_port_connected(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ u32 bit; ++ ++ switch (encoder->hpd_pin) { ++ case HPD_PORT_B: ++ bit = SDE_PORTB_HOTPLUG; ++ break; ++ case HPD_PORT_C: ++ bit = SDE_PORTC_HOTPLUG; ++ break; ++ case HPD_PORT_D: ++ bit = SDE_PORTD_HOTPLUG; ++ break; ++ default: ++ MISSING_CASE(encoder->hpd_pin); ++ return false; ++ } ++ ++ return I915_READ(SDEISR) & bit; ++} ++ ++static bool cpt_digital_port_connected(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ u32 bit; ++ ++ switch (encoder->hpd_pin) { ++ case HPD_PORT_B: ++ bit = SDE_PORTB_HOTPLUG_CPT; ++ break; ++ case HPD_PORT_C: ++ bit = SDE_PORTC_HOTPLUG_CPT; ++ break; ++ case HPD_PORT_D: ++ bit = SDE_PORTD_HOTPLUG_CPT; ++ break; ++ default: ++ MISSING_CASE(encoder->hpd_pin); ++ return false; ++ } ++ ++ return I915_READ(SDEISR) & bit; ++} ++ ++static bool spt_digital_port_connected(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ u32 bit; ++ ++ switch (encoder->hpd_pin) { ++ case HPD_PORT_A: ++ bit = SDE_PORTA_HOTPLUG_SPT; ++ break; ++ case HPD_PORT_E: ++ bit = SDE_PORTE_HOTPLUG_SPT; ++ break; ++ default: ++ return cpt_digital_port_connected(encoder); ++ } ++ ++ return I915_READ(SDEISR) & bit; ++} ++ ++static bool g4x_digital_port_connected(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ u32 bit; ++ ++ switch (encoder->hpd_pin) { ++ case HPD_PORT_B: ++ bit = PORTB_HOTPLUG_LIVE_STATUS_G4X; ++ break; ++ case HPD_PORT_C: ++ bit = PORTC_HOTPLUG_LIVE_STATUS_G4X; ++ break; ++ case HPD_PORT_D: ++ bit = PORTD_HOTPLUG_LIVE_STATUS_G4X; ++ break; ++ default: ++ MISSING_CASE(encoder->hpd_pin); ++ return false; ++ } ++ ++ return I915_READ(PORT_HOTPLUG_STAT) & bit; ++} ++ ++static bool gm45_digital_port_connected(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ u32 bit; ++ ++ switch (encoder->hpd_pin) { ++ case HPD_PORT_B: ++ bit = PORTB_HOTPLUG_LIVE_STATUS_GM45; ++ break; ++ case HPD_PORT_C: ++ bit = PORTC_HOTPLUG_LIVE_STATUS_GM45; ++ break; ++ case HPD_PORT_D: ++ bit = PORTD_HOTPLUG_LIVE_STATUS_GM45; ++ break; ++ default: ++ MISSING_CASE(encoder->hpd_pin); ++ return false; ++ } ++ ++ return I915_READ(PORT_HOTPLUG_STAT) & bit; ++} ++ ++static bool ilk_digital_port_connected(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ ++ if (encoder->hpd_pin == HPD_PORT_A) ++ return I915_READ(DEISR) & DE_DP_A_HOTPLUG; ++ else ++ return ibx_digital_port_connected(encoder); ++} ++ ++static bool snb_digital_port_connected(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ ++ if (encoder->hpd_pin == HPD_PORT_A) ++ return I915_READ(DEISR) & DE_DP_A_HOTPLUG; ++ else ++ return cpt_digital_port_connected(encoder); ++} ++ ++static bool ivb_digital_port_connected(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ ++ if (encoder->hpd_pin == HPD_PORT_A) ++ return I915_READ(DEISR) & DE_DP_A_HOTPLUG_IVB; ++ else ++ return cpt_digital_port_connected(encoder); ++} ++ ++static bool bdw_digital_port_connected(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ ++ if (encoder->hpd_pin == HPD_PORT_A) ++ return I915_READ(GEN8_DE_PORT_ISR) & GEN8_PORT_DP_A_HOTPLUG; ++ else ++ return cpt_digital_port_connected(encoder); ++} ++ ++static bool bxt_digital_port_connected(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ u32 bit; ++ ++ switch (encoder->hpd_pin) { ++ case HPD_PORT_A: ++ bit = BXT_DE_PORT_HP_DDIA; ++ break; ++ case HPD_PORT_B: ++ bit = BXT_DE_PORT_HP_DDIB; ++ break; ++ case HPD_PORT_C: ++ bit = BXT_DE_PORT_HP_DDIC; ++ break; ++ default: ++ MISSING_CASE(encoder->hpd_pin); ++ return false; ++ } ++ ++ return I915_READ(GEN8_DE_PORT_ISR) & bit; ++} ++ ++static bool icl_combo_port_connected(struct drm_i915_private *dev_priv, ++ struct intel_digital_port *intel_dig_port) ++{ ++ enum port port = intel_dig_port->base.port; ++ ++ return I915_READ(SDEISR) & SDE_DDI_HOTPLUG_ICP(port); ++} ++ ++static const char *tc_type_name(enum tc_port_type type) ++{ ++ static const char * const names[] = { ++ [TC_PORT_UNKNOWN] = "unknown", ++ [TC_PORT_LEGACY] = "legacy", ++ [TC_PORT_TYPEC] = "typec", ++ [TC_PORT_TBT] = "tbt", ++ }; ++ ++ if (WARN_ON(type >= ARRAY_SIZE(names))) ++ type = TC_PORT_UNKNOWN; ++ ++ return names[type]; ++} ++ ++static void icl_update_tc_port_type(struct drm_i915_private *dev_priv, ++ struct intel_digital_port *intel_dig_port, ++ bool is_legacy, bool is_typec, bool is_tbt) ++{ ++ enum port port = intel_dig_port->base.port; ++ enum tc_port_type old_type = intel_dig_port->tc_type; ++ ++ WARN_ON(is_legacy + is_typec + is_tbt != 1); ++ ++ if (is_legacy) ++ intel_dig_port->tc_type = TC_PORT_LEGACY; ++ else if (is_typec) ++ intel_dig_port->tc_type = TC_PORT_TYPEC; ++ else if (is_tbt) ++ intel_dig_port->tc_type = TC_PORT_TBT; ++ else ++ return; ++ ++ /* Types are not supposed to be changed at runtime. */ ++ WARN_ON(old_type != TC_PORT_UNKNOWN && ++ old_type != intel_dig_port->tc_type); ++ ++ if (old_type != intel_dig_port->tc_type) ++ DRM_DEBUG_KMS("Port %c has TC type %s\n", port_name(port), ++ tc_type_name(intel_dig_port->tc_type)); ++} ++ ++/* ++ * This function implements the first part of the Connect Flow described by our ++ * specification, Gen11 TypeC Programming chapter. The rest of the flow (reading ++ * lanes, EDID, etc) is done as needed in the typical places. ++ * ++ * Unlike the other ports, type-C ports are not available to use as soon as we ++ * get a hotplug. The type-C PHYs can be shared between multiple controllers: ++ * display, USB, etc. As a result, handshaking through FIA is required around ++ * connect and disconnect to cleanly transfer ownership with the controller and ++ * set the type-C power state. ++ * ++ * We could opt to only do the connect flow when we actually try to use the AUX ++ * channels or do a modeset, then immediately run the disconnect flow after ++ * usage, but there are some implications on this for a dynamic environment: ++ * things may go away or change behind our backs. So for now our driver is ++ * always trying to acquire ownership of the controller as soon as it gets an ++ * interrupt (or polls state and sees a port is connected) and only gives it ++ * back when it sees a disconnect. Implementation of a more fine-grained model ++ * will require a lot of coordination with user space and thorough testing for ++ * the extra possible cases. ++ */ ++static bool icl_tc_phy_connect(struct drm_i915_private *dev_priv, ++ struct intel_digital_port *dig_port) ++{ ++ enum tc_port tc_port = intel_port_to_tc(dev_priv, dig_port->base.port); ++ u32 val; ++ ++ if (dig_port->tc_type != TC_PORT_LEGACY && ++ dig_port->tc_type != TC_PORT_TYPEC) ++ return true; ++ ++ val = I915_READ(PORT_TX_DFLEXDPPMS); ++ if (!(val & DP_PHY_MODE_STATUS_COMPLETED(tc_port))) { ++ DRM_DEBUG_KMS("DP PHY for TC port %d not ready\n", tc_port); ++ WARN_ON(dig_port->tc_legacy_port); ++ return false; ++ } ++ ++ /* ++ * This function may be called many times in a row without an HPD event ++ * in between, so try to avoid the write when we can. ++ */ ++ val = I915_READ(PORT_TX_DFLEXDPCSSS); ++ if (!(val & DP_PHY_MODE_STATUS_NOT_SAFE(tc_port))) { ++ val |= DP_PHY_MODE_STATUS_NOT_SAFE(tc_port); ++ I915_WRITE(PORT_TX_DFLEXDPCSSS, val); ++ } ++ ++ /* ++ * Now we have to re-check the live state, in case the port recently ++ * became disconnected. Not necessary for legacy mode. ++ */ ++ if (dig_port->tc_type == TC_PORT_TYPEC && ++ !(I915_READ(PORT_TX_DFLEXDPSP) & TC_LIVE_STATE_TC(tc_port))) { ++ DRM_DEBUG_KMS("TC PHY %d sudden disconnect.\n", tc_port); ++ icl_tc_phy_disconnect(dev_priv, dig_port); ++ return false; ++ } ++ ++ return true; ++} ++ ++/* ++ * See the comment at the connect function. This implements the Disconnect ++ * Flow. ++ */ ++void icl_tc_phy_disconnect(struct drm_i915_private *dev_priv, ++ struct intel_digital_port *dig_port) ++{ ++ enum tc_port tc_port = intel_port_to_tc(dev_priv, dig_port->base.port); ++ ++ if (dig_port->tc_type == TC_PORT_UNKNOWN) ++ return; ++ ++ /* ++ * TBT disconnection flow is read the live status, what was done in ++ * caller. ++ */ ++ if (dig_port->tc_type == TC_PORT_TYPEC || ++ dig_port->tc_type == TC_PORT_LEGACY) { ++ u32 val; ++ ++ val = I915_READ(PORT_TX_DFLEXDPCSSS); ++ val &= ~DP_PHY_MODE_STATUS_NOT_SAFE(tc_port); ++ I915_WRITE(PORT_TX_DFLEXDPCSSS, val); ++ } ++ ++ DRM_DEBUG_KMS("Port %c TC type %s disconnected\n", ++ port_name(dig_port->base.port), ++ tc_type_name(dig_port->tc_type)); ++ ++ dig_port->tc_type = TC_PORT_UNKNOWN; ++} ++ ++/* ++ * The type-C ports are different because even when they are connected, they may ++ * not be available/usable by the graphics driver: see the comment on ++ * icl_tc_phy_connect(). So in our driver instead of adding the additional ++ * concept of "usable" and make everything check for "connected and usable" we ++ * define a port as "connected" when it is not only connected, but also when it ++ * is usable by the rest of the driver. That maintains the old assumption that ++ * connected ports are usable, and avoids exposing to the users objects they ++ * can't really use. ++ */ ++static bool icl_tc_port_connected(struct drm_i915_private *dev_priv, ++ struct intel_digital_port *intel_dig_port) ++{ ++ enum port port = intel_dig_port->base.port; ++ enum tc_port tc_port = intel_port_to_tc(dev_priv, port); ++ bool is_legacy, is_typec, is_tbt; ++ u32 dpsp; ++ ++ /* ++ * WARN if we got a legacy port HPD, but VBT didn't mark the port as ++ * legacy. Treat the port as legacy from now on. ++ */ ++ if (WARN_ON(!intel_dig_port->tc_legacy_port && ++ I915_READ(SDEISR) & SDE_TC_HOTPLUG_ICP(tc_port))) ++ intel_dig_port->tc_legacy_port = true; ++ is_legacy = intel_dig_port->tc_legacy_port; ++ ++ /* ++ * The spec says we shouldn't be using the ISR bits for detecting ++ * between TC and TBT. We should use DFLEXDPSP. ++ */ ++ dpsp = I915_READ(PORT_TX_DFLEXDPSP); ++ is_typec = dpsp & TC_LIVE_STATE_TC(tc_port); ++ is_tbt = dpsp & TC_LIVE_STATE_TBT(tc_port); ++ ++ if (!is_legacy && !is_typec && !is_tbt) { ++ icl_tc_phy_disconnect(dev_priv, intel_dig_port); ++ ++ return false; ++ } ++ ++ icl_update_tc_port_type(dev_priv, intel_dig_port, is_legacy, is_typec, ++ is_tbt); ++ ++ if (!icl_tc_phy_connect(dev_priv, intel_dig_port)) ++ return false; ++ ++ return true; ++} ++ ++static bool icl_digital_port_connected(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base); ++ ++ if (intel_port_is_combophy(dev_priv, encoder->port)) ++ return icl_combo_port_connected(dev_priv, dig_port); ++ else if (intel_port_is_tc(dev_priv, encoder->port)) ++ return icl_tc_port_connected(dev_priv, dig_port); ++ else ++ MISSING_CASE(encoder->hpd_pin); ++ ++ return false; ++} ++ ++/* ++ * intel_digital_port_connected - is the specified port connected? ++ * @encoder: intel_encoder ++ * ++ * In cases where there's a connector physically connected but it can't be used ++ * by our hardware we also return false, since the rest of the driver should ++ * pretty much treat the port as disconnected. This is relevant for type-C ++ * (starting on ICL) where there's ownership involved. ++ * ++ * Return %true if port is connected, %false otherwise. ++ */ ++bool intel_digital_port_connected(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ ++ if (HAS_GMCH(dev_priv)) { ++ if (IS_GM45(dev_priv)) ++ return gm45_digital_port_connected(encoder); ++ else ++ return g4x_digital_port_connected(encoder); ++ } ++ ++ if (INTEL_GEN(dev_priv) >= 11) ++ return icl_digital_port_connected(encoder); ++ else if (IS_GEN(dev_priv, 10) || IS_GEN9_BC(dev_priv)) ++ return spt_digital_port_connected(encoder); ++ else if (IS_GEN9_LP(dev_priv)) ++ return bxt_digital_port_connected(encoder); ++ else if (IS_GEN(dev_priv, 8)) ++ return bdw_digital_port_connected(encoder); ++ else if (IS_GEN(dev_priv, 7)) ++ return ivb_digital_port_connected(encoder); ++ else if (IS_GEN(dev_priv, 6)) ++ return snb_digital_port_connected(encoder); ++ else if (IS_GEN(dev_priv, 5)) ++ return ilk_digital_port_connected(encoder); ++ ++ MISSING_CASE(INTEL_GEN(dev_priv)); ++ return false; ++} ++ ++static struct edid * ++intel_dp_get_edid(struct intel_dp *intel_dp) ++{ ++ struct intel_connector *intel_connector = intel_dp->attached_connector; ++ ++ /* use cached edid if we have one */ ++ if (intel_connector->edid) { ++ /* invalid edid */ ++ if (IS_ERR(intel_connector->edid)) ++ return NULL; ++ ++ return drm_edid_duplicate(intel_connector->edid); ++ } else ++ return drm_get_edid(&intel_connector->base, ++ &intel_dp->aux.ddc); ++} ++ ++static void ++intel_dp_set_edid(struct intel_dp *intel_dp) ++{ ++ struct intel_connector *intel_connector = intel_dp->attached_connector; ++ struct edid *edid; ++ ++ intel_dp_unset_edid(intel_dp); ++ edid = intel_dp_get_edid(intel_dp); ++ intel_connector->detect_edid = edid; ++ ++ intel_dp->has_audio = drm_detect_monitor_audio(edid); ++ drm_dp_cec_set_edid(&intel_dp->aux, edid); ++} ++ ++static void ++intel_dp_unset_edid(struct intel_dp *intel_dp) ++{ ++ struct intel_connector *intel_connector = intel_dp->attached_connector; ++ ++ drm_dp_cec_unset_edid(&intel_dp->aux); ++ kfree(intel_connector->detect_edid); ++ intel_connector->detect_edid = NULL; ++ ++ intel_dp->has_audio = false; ++} ++ ++static int ++intel_dp_detect(struct drm_connector *connector, ++ struct drm_modeset_acquire_ctx *ctx, ++ bool force) ++{ ++ struct drm_i915_private *dev_priv = to_i915(connector->dev); ++ struct intel_dp *intel_dp = intel_attached_dp(connector); ++ struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); ++ struct intel_encoder *encoder = &dig_port->base; ++ enum drm_connector_status status; ++ enum intel_display_power_domain aux_domain = ++ intel_aux_power_domain(dig_port); ++ intel_wakeref_t wakeref; ++ ++ DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", ++ connector->base.id, connector->name); ++ WARN_ON(!drm_modeset_is_locked(&dev_priv->drm.mode_config.connection_mutex)); ++ ++ wakeref = intel_display_power_get(dev_priv, aux_domain); ++ ++ /* Can't disconnect eDP */ ++ if (intel_dp_is_edp(intel_dp)) ++ status = edp_detect(intel_dp); ++ else if (intel_digital_port_connected(encoder)) ++ status = intel_dp_detect_dpcd(intel_dp); ++ else ++ status = connector_status_disconnected; ++ ++ if (status == connector_status_disconnected) { ++ memset(&intel_dp->compliance, 0, sizeof(intel_dp->compliance)); ++ memset(intel_dp->dsc_dpcd, 0, sizeof(intel_dp->dsc_dpcd)); ++ ++ if (intel_dp->is_mst) { ++ DRM_DEBUG_KMS("MST device may have disappeared %d vs %d\n", ++ intel_dp->is_mst, ++ intel_dp->mst_mgr.mst_state); ++ intel_dp->is_mst = false; ++ drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr, ++ intel_dp->is_mst); ++ } ++ ++ goto out; ++ } ++ ++ if (intel_dp->reset_link_params) { ++ /* Initial max link lane count */ ++ intel_dp->max_link_lane_count = intel_dp_max_common_lane_count(intel_dp); ++ ++ /* Initial max link rate */ ++ intel_dp->max_link_rate = intel_dp_max_common_rate(intel_dp); ++ ++ intel_dp->reset_link_params = false; ++ } ++ ++ intel_dp_print_rates(intel_dp); ++ ++ /* Read DP Sink DSC Cap DPCD regs for DP v1.4 */ ++ if (INTEL_GEN(dev_priv) >= 11) ++ intel_dp_get_dsc_sink_cap(intel_dp); ++ ++ drm_dp_read_desc(&intel_dp->aux, &intel_dp->desc, ++ drm_dp_is_branch(intel_dp->dpcd)); ++ ++ intel_dp_configure_mst(intel_dp); ++ ++ if (intel_dp->is_mst) { ++ /* ++ * If we are in MST mode then this connector ++ * won't appear connected or have anything ++ * with EDID on it ++ */ ++ status = connector_status_disconnected; ++ goto out; ++ } ++ ++ /* ++ * Some external monitors do not signal loss of link synchronization ++ * with an IRQ_HPD, so force a link status check. ++ */ ++ if (!intel_dp_is_edp(intel_dp)) { ++ int ret; ++ ++ ret = intel_dp_retrain_link(encoder, ctx); ++ if (ret) { ++ intel_display_power_put(dev_priv, aux_domain, wakeref); ++ return ret; ++ } ++ } ++ ++ /* ++ * Clearing NACK and defer counts to get their exact values ++ * while reading EDID which are required by Compliance tests ++ * 4.2.2.4 and 4.2.2.5 ++ */ ++ intel_dp->aux.i2c_nack_count = 0; ++ intel_dp->aux.i2c_defer_count = 0; ++ ++ intel_dp_set_edid(intel_dp); ++ if (intel_dp_is_edp(intel_dp) || ++ to_intel_connector(connector)->detect_edid) ++ status = connector_status_connected; ++ ++ intel_dp_check_service_irq(intel_dp); ++ ++out: ++ if (status != connector_status_connected && !intel_dp->is_mst) ++ intel_dp_unset_edid(intel_dp); ++ ++ intel_display_power_put(dev_priv, aux_domain, wakeref); ++ return status; ++} ++ ++static void ++intel_dp_force(struct drm_connector *connector) ++{ ++ struct intel_dp *intel_dp = intel_attached_dp(connector); ++ struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); ++ struct intel_encoder *intel_encoder = &dig_port->base; ++ struct drm_i915_private *dev_priv = to_i915(intel_encoder->base.dev); ++ enum intel_display_power_domain aux_domain = ++ intel_aux_power_domain(dig_port); ++ intel_wakeref_t wakeref; ++ ++ DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", ++ connector->base.id, connector->name); ++ intel_dp_unset_edid(intel_dp); ++ ++ if (connector->status != connector_status_connected) ++ return; ++ ++ wakeref = intel_display_power_get(dev_priv, aux_domain); ++ ++ intel_dp_set_edid(intel_dp); ++ ++ intel_display_power_put(dev_priv, aux_domain, wakeref); ++} ++ ++static int intel_dp_get_modes(struct drm_connector *connector) ++{ ++ struct intel_connector *intel_connector = to_intel_connector(connector); ++ struct edid *edid; ++ ++ edid = intel_connector->detect_edid; ++ if (edid) { ++ int ret = intel_connector_update_modes(connector, edid); ++ if (ret) ++ return ret; ++ } ++ ++ /* if eDP has no EDID, fall back to fixed mode */ ++ if (intel_dp_is_edp(intel_attached_dp(connector)) && ++ intel_connector->panel.fixed_mode) { ++ struct drm_display_mode *mode; ++ ++ mode = drm_mode_duplicate(connector->dev, ++ intel_connector->panel.fixed_mode); ++ if (mode) { ++ drm_mode_probed_add(connector, mode); ++ return 1; ++ } ++ } ++ ++ return 0; ++} ++ ++static int ++intel_dp_connector_register(struct drm_connector *connector) ++{ ++ struct intel_dp *intel_dp = intel_attached_dp(connector); ++ struct drm_device *dev = connector->dev; ++ int ret; ++ ++ ret = intel_connector_register(connector); ++ if (ret) ++ return ret; ++ ++ i915_debugfs_connector_add(connector); ++ ++ DRM_DEBUG_KMS("registering %s bus for %s\n", ++ intel_dp->aux.name, connector->kdev->kobj.name); ++ ++ intel_dp->aux.dev = connector->kdev; ++ ret = drm_dp_aux_register(&intel_dp->aux); ++ if (!ret) ++ drm_dp_cec_register_connector(&intel_dp->aux, ++ connector->name, dev->dev); ++ return ret; ++} ++ ++static void ++intel_dp_connector_unregister(struct drm_connector *connector) ++{ ++ struct intel_dp *intel_dp = intel_attached_dp(connector); ++ ++ drm_dp_cec_unregister_connector(&intel_dp->aux); ++ drm_dp_aux_unregister(&intel_dp->aux); ++ intel_connector_unregister(connector); ++} ++ ++void intel_dp_encoder_flush_work(struct drm_encoder *encoder) ++{ ++ struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); ++ struct intel_dp *intel_dp = &intel_dig_port->dp; ++ ++ intel_dp_mst_encoder_cleanup(intel_dig_port); ++ if (intel_dp_is_edp(intel_dp)) { ++ intel_wakeref_t wakeref; ++ ++ cancel_delayed_work_sync(&intel_dp->panel_vdd_work); ++ /* ++ * vdd might still be enabled do to the delayed vdd off. ++ * Make sure vdd is actually turned off here. ++ */ ++ with_pps_lock(intel_dp, wakeref) ++ edp_panel_vdd_off_sync(intel_dp); ++ ++ if (intel_dp->edp_notifier.notifier_call) { ++ unregister_reboot_notifier(&intel_dp->edp_notifier); ++ intel_dp->edp_notifier.notifier_call = NULL; ++ } ++ } ++ ++ intel_dp_aux_fini(intel_dp); ++} ++ ++static void intel_dp_encoder_destroy(struct drm_encoder *encoder) ++{ ++ intel_dp_encoder_flush_work(encoder); ++ ++ drm_encoder_cleanup(encoder); ++ kfree(enc_to_dig_port(encoder)); ++} ++ ++void intel_dp_encoder_suspend(struct intel_encoder *intel_encoder) ++{ ++ struct intel_dp *intel_dp = enc_to_intel_dp(&intel_encoder->base); ++ intel_wakeref_t wakeref; ++ ++ if (!intel_dp_is_edp(intel_dp)) ++ return; ++ ++ /* ++ * vdd might still be enabled do to the delayed vdd off. ++ * Make sure vdd is actually turned off here. ++ */ ++ cancel_delayed_work_sync(&intel_dp->panel_vdd_work); ++ with_pps_lock(intel_dp, wakeref) ++ edp_panel_vdd_off_sync(intel_dp); ++} ++ ++static void intel_dp_hdcp_wait_for_cp_irq(struct intel_hdcp *hdcp, int timeout) ++{ ++ long ret; ++ ++#define C (hdcp->cp_irq_count_cached != atomic_read(&hdcp->cp_irq_count)) ++ ret = wait_event_interruptible_timeout(hdcp->cp_irq_queue, C, ++ msecs_to_jiffies(timeout)); ++ ++ if (!ret) ++ DRM_DEBUG_KMS("Timedout at waiting for CP_IRQ\n"); ++} ++ ++static ++int intel_dp_hdcp_write_an_aksv(struct intel_digital_port *intel_dig_port, ++ u8 *an) ++{ ++ struct intel_dp *intel_dp = enc_to_intel_dp(&intel_dig_port->base.base); ++ static const struct drm_dp_aux_msg msg = { ++ .request = DP_AUX_NATIVE_WRITE, ++ .address = DP_AUX_HDCP_AKSV, ++ .size = DRM_HDCP_KSV_LEN, ++ }; ++ u8 txbuf[HEADER_SIZE + DRM_HDCP_KSV_LEN] = {}, rxbuf[2], reply = 0; ++ ssize_t dpcd_ret; ++ int ret; ++ ++ /* Output An first, that's easy */ ++ dpcd_ret = drm_dp_dpcd_write(&intel_dig_port->dp.aux, DP_AUX_HDCP_AN, ++ an, DRM_HDCP_AN_LEN); ++ if (dpcd_ret != DRM_HDCP_AN_LEN) { ++ DRM_DEBUG_KMS("Failed to write An over DP/AUX (%zd)\n", ++ dpcd_ret); ++ return dpcd_ret >= 0 ? -EIO : dpcd_ret; ++ } ++ ++ /* ++ * Since Aksv is Oh-So-Secret, we can't access it in software. So in ++ * order to get it on the wire, we need to create the AUX header as if ++ * we were writing the data, and then tickle the hardware to output the ++ * data once the header is sent out. ++ */ ++ intel_dp_aux_header(txbuf, &msg); ++ ++ ret = intel_dp_aux_xfer(intel_dp, txbuf, HEADER_SIZE + msg.size, ++ rxbuf, sizeof(rxbuf), ++ DP_AUX_CH_CTL_AUX_AKSV_SELECT); ++ if (ret < 0) { ++ DRM_DEBUG_KMS("Write Aksv over DP/AUX failed (%d)\n", ret); ++ return ret; ++ } else if (ret == 0) { ++ DRM_DEBUG_KMS("Aksv write over DP/AUX was empty\n"); ++ return -EIO; ++ } ++ ++ reply = (rxbuf[0] >> 4) & DP_AUX_NATIVE_REPLY_MASK; ++ if (reply != DP_AUX_NATIVE_REPLY_ACK) { ++ DRM_DEBUG_KMS("Aksv write: no DP_AUX_NATIVE_REPLY_ACK %x\n", ++ reply); ++ return -EIO; ++ } ++ return 0; ++} ++ ++static int intel_dp_hdcp_read_bksv(struct intel_digital_port *intel_dig_port, ++ u8 *bksv) ++{ ++ ssize_t ret; ++ ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, DP_AUX_HDCP_BKSV, bksv, ++ DRM_HDCP_KSV_LEN); ++ if (ret != DRM_HDCP_KSV_LEN) { ++ DRM_DEBUG_KMS("Read Bksv from DP/AUX failed (%zd)\n", ret); ++ return ret >= 0 ? -EIO : ret; ++ } ++ return 0; ++} ++ ++static int intel_dp_hdcp_read_bstatus(struct intel_digital_port *intel_dig_port, ++ u8 *bstatus) ++{ ++ ssize_t ret; ++ /* ++ * For some reason the HDMI and DP HDCP specs call this register ++ * definition by different names. In the HDMI spec, it's called BSTATUS, ++ * but in DP it's called BINFO. ++ */ ++ ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, DP_AUX_HDCP_BINFO, ++ bstatus, DRM_HDCP_BSTATUS_LEN); ++ if (ret != DRM_HDCP_BSTATUS_LEN) { ++ DRM_DEBUG_KMS("Read bstatus from DP/AUX failed (%zd)\n", ret); ++ return ret >= 0 ? -EIO : ret; ++ } ++ return 0; ++} ++ ++static ++int intel_dp_hdcp_read_bcaps(struct intel_digital_port *intel_dig_port, ++ u8 *bcaps) ++{ ++ ssize_t ret; ++ ++ ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, DP_AUX_HDCP_BCAPS, ++ bcaps, 1); ++ if (ret != 1) { ++ DRM_DEBUG_KMS("Read bcaps from DP/AUX failed (%zd)\n", ret); ++ return ret >= 0 ? -EIO : ret; ++ } ++ ++ return 0; ++} ++ ++static ++int intel_dp_hdcp_repeater_present(struct intel_digital_port *intel_dig_port, ++ bool *repeater_present) ++{ ++ ssize_t ret; ++ u8 bcaps; ++ ++ ret = intel_dp_hdcp_read_bcaps(intel_dig_port, &bcaps); ++ if (ret) ++ return ret; ++ ++ *repeater_present = bcaps & DP_BCAPS_REPEATER_PRESENT; ++ return 0; ++} ++ ++static ++int intel_dp_hdcp_read_ri_prime(struct intel_digital_port *intel_dig_port, ++ u8 *ri_prime) ++{ ++ ssize_t ret; ++ ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, DP_AUX_HDCP_RI_PRIME, ++ ri_prime, DRM_HDCP_RI_LEN); ++ if (ret != DRM_HDCP_RI_LEN) { ++ DRM_DEBUG_KMS("Read Ri' from DP/AUX failed (%zd)\n", ret); ++ return ret >= 0 ? -EIO : ret; ++ } ++ return 0; ++} ++ ++static ++int intel_dp_hdcp_read_ksv_ready(struct intel_digital_port *intel_dig_port, ++ bool *ksv_ready) ++{ ++ ssize_t ret; ++ u8 bstatus; ++ ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, DP_AUX_HDCP_BSTATUS, ++ &bstatus, 1); ++ if (ret != 1) { ++ DRM_DEBUG_KMS("Read bstatus from DP/AUX failed (%zd)\n", ret); ++ return ret >= 0 ? -EIO : ret; ++ } ++ *ksv_ready = bstatus & DP_BSTATUS_READY; ++ return 0; ++} ++ ++static ++int intel_dp_hdcp_read_ksv_fifo(struct intel_digital_port *intel_dig_port, ++ int num_downstream, u8 *ksv_fifo) ++{ ++ ssize_t ret; ++ int i; ++ ++ /* KSV list is read via 15 byte window (3 entries @ 5 bytes each) */ ++ for (i = 0; i < num_downstream; i += 3) { ++ size_t len = min(num_downstream - i, 3) * DRM_HDCP_KSV_LEN; ++ ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, ++ DP_AUX_HDCP_KSV_FIFO, ++ ksv_fifo + i * DRM_HDCP_KSV_LEN, ++ len); ++ if (ret != len) { ++ DRM_DEBUG_KMS("Read ksv[%d] from DP/AUX failed (%zd)\n", ++ i, ret); ++ return ret >= 0 ? -EIO : ret; ++ } ++ } ++ return 0; ++} ++ ++static ++int intel_dp_hdcp_read_v_prime_part(struct intel_digital_port *intel_dig_port, ++ int i, u32 *part) ++{ ++ ssize_t ret; ++ ++ if (i >= DRM_HDCP_V_PRIME_NUM_PARTS) ++ return -EINVAL; ++ ++ ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, ++ DP_AUX_HDCP_V_PRIME(i), part, ++ DRM_HDCP_V_PRIME_PART_LEN); ++ if (ret != DRM_HDCP_V_PRIME_PART_LEN) { ++ DRM_DEBUG_KMS("Read v'[%d] from DP/AUX failed (%zd)\n", i, ret); ++ return ret >= 0 ? -EIO : ret; ++ } ++ return 0; ++} ++ ++static ++int intel_dp_hdcp_toggle_signalling(struct intel_digital_port *intel_dig_port, ++ bool enable) ++{ ++ /* Not used for single stream DisplayPort setups */ ++ return 0; ++} ++ ++static ++bool intel_dp_hdcp_check_link(struct intel_digital_port *intel_dig_port) ++{ ++ ssize_t ret; ++ u8 bstatus; ++ ++ ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, DP_AUX_HDCP_BSTATUS, ++ &bstatus, 1); ++ if (ret != 1) { ++ DRM_DEBUG_KMS("Read bstatus from DP/AUX failed (%zd)\n", ret); ++ return false; ++ } ++ ++ return !(bstatus & (DP_BSTATUS_LINK_FAILURE | DP_BSTATUS_REAUTH_REQ)); ++} ++ ++static ++int intel_dp_hdcp_capable(struct intel_digital_port *intel_dig_port, ++ bool *hdcp_capable) ++{ ++ ssize_t ret; ++ u8 bcaps; ++ ++ ret = intel_dp_hdcp_read_bcaps(intel_dig_port, &bcaps); ++ if (ret) ++ return ret; ++ ++ *hdcp_capable = bcaps & DP_BCAPS_HDCP_CAPABLE; ++ return 0; ++} ++ ++struct hdcp2_dp_errata_stream_type { ++ u8 msg_id; ++ u8 stream_type; ++} __packed; ++ ++static struct hdcp2_dp_msg_data { ++ u8 msg_id; ++ u32 offset; ++ bool msg_detectable; ++ u32 timeout; ++ u32 timeout2; /* Added for non_paired situation */ ++ } hdcp2_msg_data[] = { ++ {HDCP_2_2_AKE_INIT, DP_HDCP_2_2_AKE_INIT_OFFSET, false, 0, 0}, ++ {HDCP_2_2_AKE_SEND_CERT, DP_HDCP_2_2_AKE_SEND_CERT_OFFSET, ++ false, HDCP_2_2_CERT_TIMEOUT_MS, 0}, ++ {HDCP_2_2_AKE_NO_STORED_KM, DP_HDCP_2_2_AKE_NO_STORED_KM_OFFSET, ++ false, 0, 0}, ++ {HDCP_2_2_AKE_STORED_KM, DP_HDCP_2_2_AKE_STORED_KM_OFFSET, ++ false, 0, 0}, ++ {HDCP_2_2_AKE_SEND_HPRIME, DP_HDCP_2_2_AKE_SEND_HPRIME_OFFSET, ++ true, HDCP_2_2_HPRIME_PAIRED_TIMEOUT_MS, ++ HDCP_2_2_HPRIME_NO_PAIRED_TIMEOUT_MS}, ++ {HDCP_2_2_AKE_SEND_PAIRING_INFO, ++ DP_HDCP_2_2_AKE_SEND_PAIRING_INFO_OFFSET, true, ++ HDCP_2_2_PAIRING_TIMEOUT_MS, 0}, ++ {HDCP_2_2_LC_INIT, DP_HDCP_2_2_LC_INIT_OFFSET, false, 0, 0}, ++ {HDCP_2_2_LC_SEND_LPRIME, DP_HDCP_2_2_LC_SEND_LPRIME_OFFSET, ++ false, HDCP_2_2_DP_LPRIME_TIMEOUT_MS, 0}, ++ {HDCP_2_2_SKE_SEND_EKS, DP_HDCP_2_2_SKE_SEND_EKS_OFFSET, false, ++ 0, 0}, ++ {HDCP_2_2_REP_SEND_RECVID_LIST, ++ DP_HDCP_2_2_REP_SEND_RECVID_LIST_OFFSET, true, ++ HDCP_2_2_RECVID_LIST_TIMEOUT_MS, 0}, ++ {HDCP_2_2_REP_SEND_ACK, DP_HDCP_2_2_REP_SEND_ACK_OFFSET, false, ++ 0, 0}, ++ {HDCP_2_2_REP_STREAM_MANAGE, ++ DP_HDCP_2_2_REP_STREAM_MANAGE_OFFSET, false, ++ 0, 0}, ++ {HDCP_2_2_REP_STREAM_READY, DP_HDCP_2_2_REP_STREAM_READY_OFFSET, ++ false, HDCP_2_2_STREAM_READY_TIMEOUT_MS, 0}, ++/* local define to shovel this through the write_2_2 interface */ ++#define HDCP_2_2_ERRATA_DP_STREAM_TYPE 50 ++ {HDCP_2_2_ERRATA_DP_STREAM_TYPE, ++ DP_HDCP_2_2_REG_STREAM_TYPE_OFFSET, false, ++ 0, 0}, ++ }; ++ ++static inline ++int intel_dp_hdcp2_read_rx_status(struct intel_digital_port *intel_dig_port, ++ u8 *rx_status) ++{ ++ ssize_t ret; ++ ++ ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, ++ DP_HDCP_2_2_REG_RXSTATUS_OFFSET, rx_status, ++ HDCP_2_2_DP_RXSTATUS_LEN); ++ if (ret != HDCP_2_2_DP_RXSTATUS_LEN) { ++ DRM_DEBUG_KMS("Read bstatus from DP/AUX failed (%zd)\n", ret); ++ return ret >= 0 ? -EIO : ret; ++ } ++ ++ return 0; ++} ++ ++static ++int hdcp2_detect_msg_availability(struct intel_digital_port *intel_dig_port, ++ u8 msg_id, bool *msg_ready) ++{ ++ u8 rx_status; ++ int ret; ++ ++ *msg_ready = false; ++ ret = intel_dp_hdcp2_read_rx_status(intel_dig_port, &rx_status); ++ if (ret < 0) ++ return ret; ++ ++ switch (msg_id) { ++ case HDCP_2_2_AKE_SEND_HPRIME: ++ if (HDCP_2_2_DP_RXSTATUS_H_PRIME(rx_status)) ++ *msg_ready = true; ++ break; ++ case HDCP_2_2_AKE_SEND_PAIRING_INFO: ++ if (HDCP_2_2_DP_RXSTATUS_PAIRING(rx_status)) ++ *msg_ready = true; ++ break; ++ case HDCP_2_2_REP_SEND_RECVID_LIST: ++ if (HDCP_2_2_DP_RXSTATUS_READY(rx_status)) ++ *msg_ready = true; ++ break; ++ default: ++ DRM_ERROR("Unidentified msg_id: %d\n", msg_id); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static ssize_t ++intel_dp_hdcp2_wait_for_msg(struct intel_digital_port *intel_dig_port, ++ struct hdcp2_dp_msg_data *hdcp2_msg_data) ++{ ++ struct intel_dp *dp = &intel_dig_port->dp; ++ struct intel_hdcp *hdcp = &dp->attached_connector->hdcp; ++ u8 msg_id = hdcp2_msg_data->msg_id; ++ int ret, timeout; ++ bool msg_ready = false; ++ ++ if (msg_id == HDCP_2_2_AKE_SEND_HPRIME && !hdcp->is_paired) ++ timeout = hdcp2_msg_data->timeout2; ++ else ++ timeout = hdcp2_msg_data->timeout; ++ ++ /* ++ * There is no way to detect the CERT, LPRIME and STREAM_READY ++ * availability. So Wait for timeout and read the msg. ++ */ ++ if (!hdcp2_msg_data->msg_detectable) { ++ mdelay(timeout); ++ ret = 0; ++ } else { ++ /* ++ * As we want to check the msg availability at timeout, Ignoring ++ * the timeout at wait for CP_IRQ. ++ */ ++ intel_dp_hdcp_wait_for_cp_irq(hdcp, timeout); ++ ret = hdcp2_detect_msg_availability(intel_dig_port, ++ msg_id, &msg_ready); ++ if (!msg_ready) ++ ret = -ETIMEDOUT; ++ } ++ ++ if (ret) ++ DRM_DEBUG_KMS("msg_id %d, ret %d, timeout(mSec): %d\n", ++ hdcp2_msg_data->msg_id, ret, timeout); ++ ++ return ret; ++} ++ ++static struct hdcp2_dp_msg_data *get_hdcp2_dp_msg_data(u8 msg_id) ++{ ++ int i; ++ ++ for (i = 0; i < ARRAY_SIZE(hdcp2_msg_data); i++) ++ if (hdcp2_msg_data[i].msg_id == msg_id) ++ return &hdcp2_msg_data[i]; ++ ++ return NULL; ++} ++ ++static ++int intel_dp_hdcp2_write_msg(struct intel_digital_port *intel_dig_port, ++ void *buf, size_t size) ++{ ++ struct intel_dp *dp = &intel_dig_port->dp; ++ struct intel_hdcp *hdcp = &dp->attached_connector->hdcp; ++ unsigned int offset; ++ u8 *byte = buf; ++ ssize_t ret, bytes_to_write, len; ++ struct hdcp2_dp_msg_data *hdcp2_msg_data; ++ ++ hdcp2_msg_data = get_hdcp2_dp_msg_data(*byte); ++ if (!hdcp2_msg_data) ++ return -EINVAL; ++ ++ offset = hdcp2_msg_data->offset; ++ ++ /* No msg_id in DP HDCP2.2 msgs */ ++ bytes_to_write = size - 1; ++ byte++; ++ ++ hdcp->cp_irq_count_cached = atomic_read(&hdcp->cp_irq_count); ++ ++ while (bytes_to_write) { ++ len = bytes_to_write > DP_AUX_MAX_PAYLOAD_BYTES ? ++ DP_AUX_MAX_PAYLOAD_BYTES : bytes_to_write; ++ ++ ret = drm_dp_dpcd_write(&intel_dig_port->dp.aux, ++ offset, (void *)byte, len); ++ if (ret < 0) ++ return ret; ++ ++ bytes_to_write -= ret; ++ byte += ret; ++ offset += ret; ++ } ++ ++ return size; ++} ++ ++static ++ssize_t get_receiver_id_list_size(struct intel_digital_port *intel_dig_port) ++{ ++ u8 rx_info[HDCP_2_2_RXINFO_LEN]; ++ u32 dev_cnt; ++ ssize_t ret; ++ ++ ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, ++ DP_HDCP_2_2_REG_RXINFO_OFFSET, ++ (void *)rx_info, HDCP_2_2_RXINFO_LEN); ++ if (ret != HDCP_2_2_RXINFO_LEN) ++ return ret >= 0 ? -EIO : ret; ++ ++ dev_cnt = (HDCP_2_2_DEV_COUNT_HI(rx_info[0]) << 4 | ++ HDCP_2_2_DEV_COUNT_LO(rx_info[1])); ++ ++ if (dev_cnt > HDCP_2_2_MAX_DEVICE_COUNT) ++ dev_cnt = HDCP_2_2_MAX_DEVICE_COUNT; ++ ++ ret = sizeof(struct hdcp2_rep_send_receiverid_list) - ++ HDCP_2_2_RECEIVER_IDS_MAX_LEN + ++ (dev_cnt * HDCP_2_2_RECEIVER_ID_LEN); ++ ++ return ret; ++} ++ ++static ++int intel_dp_hdcp2_read_msg(struct intel_digital_port *intel_dig_port, ++ u8 msg_id, void *buf, size_t size) ++{ ++ unsigned int offset; ++ u8 *byte = buf; ++ ssize_t ret, bytes_to_recv, len; ++ struct hdcp2_dp_msg_data *hdcp2_msg_data; ++ ++ hdcp2_msg_data = get_hdcp2_dp_msg_data(msg_id); ++ if (!hdcp2_msg_data) ++ return -EINVAL; ++ offset = hdcp2_msg_data->offset; ++ ++ ret = intel_dp_hdcp2_wait_for_msg(intel_dig_port, hdcp2_msg_data); ++ if (ret < 0) ++ return ret; ++ ++ if (msg_id == HDCP_2_2_REP_SEND_RECVID_LIST) { ++ ret = get_receiver_id_list_size(intel_dig_port); ++ if (ret < 0) ++ return ret; ++ ++ size = ret; ++ } ++ bytes_to_recv = size - 1; ++ ++ /* DP adaptation msgs has no msg_id */ ++ byte++; ++ ++ while (bytes_to_recv) { ++ len = bytes_to_recv > DP_AUX_MAX_PAYLOAD_BYTES ? ++ DP_AUX_MAX_PAYLOAD_BYTES : bytes_to_recv; ++ ++ ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, offset, ++ (void *)byte, len); ++ if (ret < 0) { ++ DRM_DEBUG_KMS("msg_id %d, ret %zd\n", msg_id, ret); ++ return ret; ++ } ++ ++ bytes_to_recv -= ret; ++ byte += ret; ++ offset += ret; ++ } ++ byte = buf; ++ *byte = msg_id; ++ ++ return size; ++} ++ ++static ++int intel_dp_hdcp2_config_stream_type(struct intel_digital_port *intel_dig_port, ++ bool is_repeater, u8 content_type) ++{ ++ struct hdcp2_dp_errata_stream_type stream_type_msg; ++ ++ if (is_repeater) ++ return 0; ++ ++ /* ++ * Errata for DP: As Stream type is used for encryption, Receiver ++ * should be communicated with stream type for the decryption of the ++ * content. ++ * Repeater will be communicated with stream type as a part of it's ++ * auth later in time. ++ */ ++ stream_type_msg.msg_id = HDCP_2_2_ERRATA_DP_STREAM_TYPE; ++ stream_type_msg.stream_type = content_type; ++ ++ return intel_dp_hdcp2_write_msg(intel_dig_port, &stream_type_msg, ++ sizeof(stream_type_msg)); ++} ++ ++static ++int intel_dp_hdcp2_check_link(struct intel_digital_port *intel_dig_port) ++{ ++ u8 rx_status; ++ int ret; ++ ++ ret = intel_dp_hdcp2_read_rx_status(intel_dig_port, &rx_status); ++ if (ret) ++ return ret; ++ ++ if (HDCP_2_2_DP_RXSTATUS_REAUTH_REQ(rx_status)) ++ ret = HDCP_REAUTH_REQUEST; ++ else if (HDCP_2_2_DP_RXSTATUS_LINK_FAILED(rx_status)) ++ ret = HDCP_LINK_INTEGRITY_FAILURE; ++ else if (HDCP_2_2_DP_RXSTATUS_READY(rx_status)) ++ ret = HDCP_TOPOLOGY_CHANGE; ++ ++ return ret; ++} ++ ++static ++int intel_dp_hdcp2_capable(struct intel_digital_port *intel_dig_port, ++ bool *capable) ++{ ++ u8 rx_caps[3]; ++ int ret; ++ ++ *capable = false; ++ ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, ++ DP_HDCP_2_2_REG_RX_CAPS_OFFSET, ++ rx_caps, HDCP_2_2_RXCAPS_LEN); ++ if (ret != HDCP_2_2_RXCAPS_LEN) ++ return ret >= 0 ? -EIO : ret; ++ ++ if (rx_caps[0] == HDCP_2_2_RX_CAPS_VERSION_VAL && ++ HDCP_2_2_DP_HDCP_CAPABLE(rx_caps[2])) ++ *capable = true; ++ ++ return 0; ++} ++ ++static const struct intel_hdcp_shim intel_dp_hdcp_shim = { ++ .write_an_aksv = intel_dp_hdcp_write_an_aksv, ++ .read_bksv = intel_dp_hdcp_read_bksv, ++ .read_bstatus = intel_dp_hdcp_read_bstatus, ++ .repeater_present = intel_dp_hdcp_repeater_present, ++ .read_ri_prime = intel_dp_hdcp_read_ri_prime, ++ .read_ksv_ready = intel_dp_hdcp_read_ksv_ready, ++ .read_ksv_fifo = intel_dp_hdcp_read_ksv_fifo, ++ .read_v_prime_part = intel_dp_hdcp_read_v_prime_part, ++ .toggle_signalling = intel_dp_hdcp_toggle_signalling, ++ .check_link = intel_dp_hdcp_check_link, ++ .hdcp_capable = intel_dp_hdcp_capable, ++ .write_2_2_msg = intel_dp_hdcp2_write_msg, ++ .read_2_2_msg = intel_dp_hdcp2_read_msg, ++ .config_stream_type = intel_dp_hdcp2_config_stream_type, ++ .check_2_2_link = intel_dp_hdcp2_check_link, ++ .hdcp_2_2_capable = intel_dp_hdcp2_capable, ++ .protocol = HDCP_PROTOCOL_DP, ++}; ++ ++static void intel_edp_panel_vdd_sanitize(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); ++ ++ lockdep_assert_held(&dev_priv->pps_mutex); ++ ++ if (!edp_have_panel_vdd(intel_dp)) ++ return; ++ ++ /* ++ * The VDD bit needs a power domain reference, so if the bit is ++ * already enabled when we boot or resume, grab this reference and ++ * schedule a vdd off, so we don't hold on to the reference ++ * indefinitely. ++ */ ++ DRM_DEBUG_KMS("VDD left on by BIOS, adjusting state tracking\n"); ++ intel_display_power_get(dev_priv, intel_aux_power_domain(dig_port)); ++ ++ edp_panel_vdd_schedule_off(intel_dp); ++} ++ ++static enum pipe vlv_active_pipe(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; ++ enum pipe pipe; ++ ++ if (intel_dp_port_enabled(dev_priv, intel_dp->output_reg, ++ encoder->port, &pipe)) ++ return pipe; ++ ++ return INVALID_PIPE; ++} ++ ++void intel_dp_encoder_reset(struct drm_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->dev); ++ struct intel_dp *intel_dp = enc_to_intel_dp(encoder); ++ struct intel_lspcon *lspcon = dp_to_lspcon(intel_dp); ++ intel_wakeref_t wakeref; ++ ++ if (!HAS_DDI(dev_priv)) ++ intel_dp->DP = I915_READ(intel_dp->output_reg); ++ ++ if (lspcon->active) ++ lspcon_resume(lspcon); ++ ++ intel_dp->reset_link_params = true; ++ ++ with_pps_lock(intel_dp, wakeref) { ++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ intel_dp->active_pipe = vlv_active_pipe(intel_dp); ++ ++ if (intel_dp_is_edp(intel_dp)) { ++ /* ++ * Reinit the power sequencer, in case BIOS did ++ * something nasty with it. ++ */ ++ intel_dp_pps_init(intel_dp); ++ intel_edp_panel_vdd_sanitize(intel_dp); ++ } ++ } ++} ++ ++static const struct drm_connector_funcs intel_dp_connector_funcs = { ++ .force = intel_dp_force, ++ .fill_modes = drm_helper_probe_single_connector_modes, ++ .atomic_get_property = intel_digital_connector_atomic_get_property, ++ .atomic_set_property = intel_digital_connector_atomic_set_property, ++ .late_register = intel_dp_connector_register, ++ .early_unregister = intel_dp_connector_unregister, ++ .destroy = intel_connector_destroy, ++ .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, ++ .atomic_duplicate_state = intel_digital_connector_duplicate_state, ++}; ++ ++static const struct drm_connector_helper_funcs intel_dp_connector_helper_funcs = { ++ .detect_ctx = intel_dp_detect, ++ .get_modes = intel_dp_get_modes, ++ .mode_valid = intel_dp_mode_valid, ++ .atomic_check = intel_digital_connector_atomic_check, ++}; ++ ++static const struct drm_encoder_funcs intel_dp_enc_funcs = { ++ .reset = intel_dp_encoder_reset, ++ .destroy = intel_dp_encoder_destroy, ++}; ++ ++enum irqreturn ++intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd) ++{ ++ struct intel_dp *intel_dp = &intel_dig_port->dp; ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ enum irqreturn ret = IRQ_NONE; ++ intel_wakeref_t wakeref; ++ ++ if (long_hpd && intel_dig_port->base.type == INTEL_OUTPUT_EDP) { ++ /* ++ * vdd off can generate a long pulse on eDP which ++ * would require vdd on to handle it, and thus we ++ * would end up in an endless cycle of ++ * "vdd off -> long hpd -> vdd on -> detect -> vdd off -> ..." ++ */ ++ DRM_DEBUG_KMS("ignoring long hpd on eDP port %c\n", ++ port_name(intel_dig_port->base.port)); ++ return IRQ_HANDLED; ++ } ++ ++ DRM_DEBUG_KMS("got hpd irq on port %c - %s\n", ++ port_name(intel_dig_port->base.port), ++ long_hpd ? "long" : "short"); ++ ++ if (long_hpd) { ++ intel_dp->reset_link_params = true; ++ return IRQ_NONE; ++ } ++ ++ wakeref = intel_display_power_get(dev_priv, ++ intel_aux_power_domain(intel_dig_port)); ++ ++ if (intel_dp->is_mst) { ++ if (intel_dp_check_mst_status(intel_dp) == -EINVAL) { ++ /* ++ * If we were in MST mode, and device is not ++ * there, get out of MST mode ++ */ ++ DRM_DEBUG_KMS("MST device may have disappeared %d vs %d\n", ++ intel_dp->is_mst, intel_dp->mst_mgr.mst_state); ++ intel_dp->is_mst = false; ++ drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr, ++ intel_dp->is_mst); ++ goto put_power; ++ } ++ } ++ ++ if (!intel_dp->is_mst) { ++ bool handled; ++ ++ handled = intel_dp_short_pulse(intel_dp); ++ ++ if (!handled) ++ goto put_power; ++ } ++ ++ ret = IRQ_HANDLED; ++ ++put_power: ++ intel_display_power_put(dev_priv, ++ intel_aux_power_domain(intel_dig_port), ++ wakeref); ++ ++ return ret; ++} ++ ++/* check the VBT to see whether the eDP is on another port */ ++bool intel_dp_is_port_edp(struct drm_i915_private *dev_priv, enum port port) ++{ ++ /* ++ * eDP not supported on g4x. so bail out early just ++ * for a bit extra safety in case the VBT is bonkers. ++ */ ++ if (INTEL_GEN(dev_priv) < 5) ++ return false; ++ ++ if (INTEL_GEN(dev_priv) < 9 && port == PORT_A) ++ return true; ++ ++ return intel_bios_is_port_edp(dev_priv, port); ++} ++ ++static void ++intel_dp_add_properties(struct intel_dp *intel_dp, struct drm_connector *connector) ++{ ++ struct drm_i915_private *dev_priv = to_i915(connector->dev); ++ enum port port = dp_to_dig_port(intel_dp)->base.port; ++ ++ if (!IS_G4X(dev_priv) && port != PORT_A) ++ intel_attach_force_audio_property(connector); ++ ++ intel_attach_broadcast_rgb_property(connector); ++ if (HAS_GMCH(dev_priv)) ++ drm_connector_attach_max_bpc_property(connector, 6, 10); ++ else if (INTEL_GEN(dev_priv) >= 5) ++ drm_connector_attach_max_bpc_property(connector, 6, 12); ++ ++ if (intel_dp_is_edp(intel_dp)) { ++ u32 allowed_scalers; ++ ++ allowed_scalers = BIT(DRM_MODE_SCALE_ASPECT) | BIT(DRM_MODE_SCALE_FULLSCREEN); ++ if (!HAS_GMCH(dev_priv)) ++ allowed_scalers |= BIT(DRM_MODE_SCALE_CENTER); ++ ++ drm_connector_attach_scaling_mode_property(connector, allowed_scalers); ++ ++ connector->state->scaling_mode = DRM_MODE_SCALE_ASPECT; ++ ++ } ++} ++ ++static void intel_dp_init_panel_power_timestamps(struct intel_dp *intel_dp) ++{ ++ intel_dp->panel_power_off_time = ktime_get_boottime(); ++ intel_dp->last_power_on = jiffies; ++ intel_dp->last_backlight_off = jiffies; ++} ++ ++static void ++intel_pps_readout_hw_state(struct intel_dp *intel_dp, struct edp_power_seq *seq) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ u32 pp_on, pp_off, pp_ctl; ++ struct pps_registers regs; ++ ++ intel_pps_get_registers(intel_dp, ®s); ++ ++ pp_ctl = ironlake_get_pp_control(intel_dp); ++ ++ /* Ensure PPS is unlocked */ ++ if (!HAS_DDI(dev_priv)) ++ I915_WRITE(regs.pp_ctrl, pp_ctl); ++ ++ pp_on = I915_READ(regs.pp_on); ++ pp_off = I915_READ(regs.pp_off); ++ ++ /* Pull timing values out of registers */ ++ seq->t1_t3 = REG_FIELD_GET(PANEL_POWER_UP_DELAY_MASK, pp_on); ++ seq->t8 = REG_FIELD_GET(PANEL_LIGHT_ON_DELAY_MASK, pp_on); ++ seq->t9 = REG_FIELD_GET(PANEL_LIGHT_OFF_DELAY_MASK, pp_off); ++ seq->t10 = REG_FIELD_GET(PANEL_POWER_DOWN_DELAY_MASK, pp_off); ++ ++ if (i915_mmio_reg_valid(regs.pp_div)) { ++ u32 pp_div; ++ ++ pp_div = I915_READ(regs.pp_div); ++ ++ seq->t11_t12 = REG_FIELD_GET(PANEL_POWER_CYCLE_DELAY_MASK, pp_div) * 1000; ++ } else { ++ seq->t11_t12 = REG_FIELD_GET(BXT_POWER_CYCLE_DELAY_MASK, pp_ctl) * 1000; ++ } ++} ++ ++static void ++intel_pps_dump_state(const char *state_name, const struct edp_power_seq *seq) ++{ ++ DRM_DEBUG_KMS("%s t1_t3 %d t8 %d t9 %d t10 %d t11_t12 %d\n", ++ state_name, ++ seq->t1_t3, seq->t8, seq->t9, seq->t10, seq->t11_t12); ++} ++ ++static void ++intel_pps_verify_state(struct intel_dp *intel_dp) ++{ ++ struct edp_power_seq hw; ++ struct edp_power_seq *sw = &intel_dp->pps_delays; ++ ++ intel_pps_readout_hw_state(intel_dp, &hw); ++ ++ if (hw.t1_t3 != sw->t1_t3 || hw.t8 != sw->t8 || hw.t9 != sw->t9 || ++ hw.t10 != sw->t10 || hw.t11_t12 != sw->t11_t12) { ++ DRM_ERROR("PPS state mismatch\n"); ++ intel_pps_dump_state("sw", sw); ++ intel_pps_dump_state("hw", &hw); ++ } ++} ++ ++static void ++intel_dp_init_panel_power_sequencer(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ struct edp_power_seq cur, vbt, spec, ++ *final = &intel_dp->pps_delays; ++ ++ lockdep_assert_held(&dev_priv->pps_mutex); ++ ++ /* already initialized? */ ++ if (final->t11_t12 != 0) ++ return; ++ ++ intel_pps_readout_hw_state(intel_dp, &cur); ++ ++ intel_pps_dump_state("cur", &cur); ++ ++ vbt = dev_priv->vbt.edp.pps; ++ /* On Toshiba Satellite P50-C-18C system the VBT T12 delay ++ * of 500ms appears to be too short. Ocassionally the panel ++ * just fails to power back on. Increasing the delay to 800ms ++ * seems sufficient to avoid this problem. ++ */ ++ if (dev_priv->quirks & QUIRK_INCREASE_T12_DELAY) { ++ vbt.t11_t12 = max_t(u16, vbt.t11_t12, 1300 * 10); ++ DRM_DEBUG_KMS("Increasing T12 panel delay as per the quirk to %d\n", ++ vbt.t11_t12); ++ } ++ /* T11_T12 delay is special and actually in units of 100ms, but zero ++ * based in the hw (so we need to add 100 ms). But the sw vbt ++ * table multiplies it with 1000 to make it in units of 100usec, ++ * too. */ ++ vbt.t11_t12 += 100 * 10; ++ ++ /* Upper limits from eDP 1.3 spec. Note that we use the clunky units of ++ * our hw here, which are all in 100usec. */ ++ spec.t1_t3 = 210 * 10; ++ spec.t8 = 50 * 10; /* no limit for t8, use t7 instead */ ++ spec.t9 = 50 * 10; /* no limit for t9, make it symmetric with t8 */ ++ spec.t10 = 500 * 10; ++ /* This one is special and actually in units of 100ms, but zero ++ * based in the hw (so we need to add 100 ms). But the sw vbt ++ * table multiplies it with 1000 to make it in units of 100usec, ++ * too. */ ++ spec.t11_t12 = (510 + 100) * 10; ++ ++ intel_pps_dump_state("vbt", &vbt); ++ ++ /* Use the max of the register settings and vbt. If both are ++ * unset, fall back to the spec limits. */ ++#define assign_final(field) final->field = (max(cur.field, vbt.field) == 0 ? \ ++ spec.field : \ ++ max(cur.field, vbt.field)) ++ assign_final(t1_t3); ++ assign_final(t8); ++ assign_final(t9); ++ assign_final(t10); ++ assign_final(t11_t12); ++#undef assign_final ++ ++#define get_delay(field) (DIV_ROUND_UP(final->field, 10)) ++ intel_dp->panel_power_up_delay = get_delay(t1_t3); ++ intel_dp->backlight_on_delay = get_delay(t8); ++ intel_dp->backlight_off_delay = get_delay(t9); ++ intel_dp->panel_power_down_delay = get_delay(t10); ++ intel_dp->panel_power_cycle_delay = get_delay(t11_t12); ++#undef get_delay ++ ++ DRM_DEBUG_KMS("panel power up delay %d, power down delay %d, power cycle delay %d\n", ++ intel_dp->panel_power_up_delay, intel_dp->panel_power_down_delay, ++ intel_dp->panel_power_cycle_delay); ++ ++ DRM_DEBUG_KMS("backlight on delay %d, off delay %d\n", ++ intel_dp->backlight_on_delay, intel_dp->backlight_off_delay); ++ ++ /* ++ * We override the HW backlight delays to 1 because we do manual waits ++ * on them. For T8, even BSpec recommends doing it. For T9, if we ++ * don't do this, we'll end up waiting for the backlight off delay ++ * twice: once when we do the manual sleep, and once when we disable ++ * the panel and wait for the PP_STATUS bit to become zero. ++ */ ++ final->t8 = 1; ++ final->t9 = 1; ++ ++ /* ++ * HW has only a 100msec granularity for t11_t12 so round it up ++ * accordingly. ++ */ ++ final->t11_t12 = roundup(final->t11_t12, 100 * 10); ++} ++ ++static void ++intel_dp_init_panel_power_sequencer_registers(struct intel_dp *intel_dp, ++ bool force_disable_vdd) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ u32 pp_on, pp_off, port_sel = 0; ++ int div = dev_priv->rawclk_freq / 1000; ++ struct pps_registers regs; ++ enum port port = dp_to_dig_port(intel_dp)->base.port; ++ const struct edp_power_seq *seq = &intel_dp->pps_delays; ++ ++ lockdep_assert_held(&dev_priv->pps_mutex); ++ ++ intel_pps_get_registers(intel_dp, ®s); ++ ++ /* ++ * On some VLV machines the BIOS can leave the VDD ++ * enabled even on power sequencers which aren't ++ * hooked up to any port. This would mess up the ++ * power domain tracking the first time we pick ++ * one of these power sequencers for use since ++ * edp_panel_vdd_on() would notice that the VDD was ++ * already on and therefore wouldn't grab the power ++ * domain reference. Disable VDD first to avoid this. ++ * This also avoids spuriously turning the VDD on as ++ * soon as the new power sequencer gets initialized. ++ */ ++ if (force_disable_vdd) { ++ u32 pp = ironlake_get_pp_control(intel_dp); ++ ++ WARN(pp & PANEL_POWER_ON, "Panel power already on\n"); ++ ++ if (pp & EDP_FORCE_VDD) ++ DRM_DEBUG_KMS("VDD already on, disabling first\n"); ++ ++ pp &= ~EDP_FORCE_VDD; ++ ++ I915_WRITE(regs.pp_ctrl, pp); ++ } ++ ++ pp_on = REG_FIELD_PREP(PANEL_POWER_UP_DELAY_MASK, seq->t1_t3) | ++ REG_FIELD_PREP(PANEL_LIGHT_ON_DELAY_MASK, seq->t8); ++ pp_off = REG_FIELD_PREP(PANEL_LIGHT_OFF_DELAY_MASK, seq->t9) | ++ REG_FIELD_PREP(PANEL_POWER_DOWN_DELAY_MASK, seq->t10); ++ ++ /* Haswell doesn't have any port selection bits for the panel ++ * power sequencer any more. */ ++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { ++ port_sel = PANEL_PORT_SELECT_VLV(port); ++ } else if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) { ++ switch (port) { ++ case PORT_A: ++ port_sel = PANEL_PORT_SELECT_DPA; ++ break; ++ case PORT_C: ++ port_sel = PANEL_PORT_SELECT_DPC; ++ break; ++ case PORT_D: ++ port_sel = PANEL_PORT_SELECT_DPD; ++ break; ++ default: ++ MISSING_CASE(port); ++ break; ++ } ++ } ++ ++ pp_on |= port_sel; ++ ++ I915_WRITE(regs.pp_on, pp_on); ++ I915_WRITE(regs.pp_off, pp_off); ++ ++ /* ++ * Compute the divisor for the pp clock, simply match the Bspec formula. ++ */ ++ if (i915_mmio_reg_valid(regs.pp_div)) { ++ I915_WRITE(regs.pp_div, ++ REG_FIELD_PREP(PP_REFERENCE_DIVIDER_MASK, (100 * div) / 2 - 1) | ++ REG_FIELD_PREP(PANEL_POWER_CYCLE_DELAY_MASK, DIV_ROUND_UP(seq->t11_t12, 1000))); ++ } else { ++ u32 pp_ctl; ++ ++ pp_ctl = I915_READ(regs.pp_ctrl); ++ pp_ctl &= ~BXT_POWER_CYCLE_DELAY_MASK; ++ pp_ctl |= REG_FIELD_PREP(BXT_POWER_CYCLE_DELAY_MASK, DIV_ROUND_UP(seq->t11_t12, 1000)); ++ I915_WRITE(regs.pp_ctrl, pp_ctl); ++ } ++ ++ DRM_DEBUG_KMS("panel power sequencer register settings: PP_ON %#x, PP_OFF %#x, PP_DIV %#x\n", ++ I915_READ(regs.pp_on), ++ I915_READ(regs.pp_off), ++ i915_mmio_reg_valid(regs.pp_div) ? ++ I915_READ(regs.pp_div) : ++ (I915_READ(regs.pp_ctrl) & BXT_POWER_CYCLE_DELAY_MASK)); ++} ++ ++static void intel_dp_pps_init(struct intel_dp *intel_dp) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ ++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { ++ vlv_initial_power_sequencer_setup(intel_dp); ++ } else { ++ intel_dp_init_panel_power_sequencer(intel_dp); ++ intel_dp_init_panel_power_sequencer_registers(intel_dp, false); ++ } ++} ++ ++/** ++ * intel_dp_set_drrs_state - program registers for RR switch to take effect ++ * @dev_priv: i915 device ++ * @crtc_state: a pointer to the active intel_crtc_state ++ * @refresh_rate: RR to be programmed ++ * ++ * This function gets called when refresh rate (RR) has to be changed from ++ * one frequency to another. Switches can be between high and low RR ++ * supported by the panel or to any other RR based on media playback (in ++ * this case, RR value needs to be passed from user space). ++ * ++ * The caller of this function needs to take a lock on dev_priv->drrs. ++ */ ++static void intel_dp_set_drrs_state(struct drm_i915_private *dev_priv, ++ const struct intel_crtc_state *crtc_state, ++ int refresh_rate) ++{ ++ struct intel_encoder *encoder; ++ struct intel_digital_port *dig_port = NULL; ++ struct intel_dp *intel_dp = dev_priv->drrs.dp; ++ struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc); ++ enum drrs_refresh_rate_type index = DRRS_HIGH_RR; ++ ++ if (refresh_rate <= 0) { ++ DRM_DEBUG_KMS("Refresh rate should be positive non-zero.\n"); ++ return; ++ } ++ ++ if (intel_dp == NULL) { ++ DRM_DEBUG_KMS("DRRS not supported.\n"); ++ return; ++ } ++ ++ dig_port = dp_to_dig_port(intel_dp); ++ encoder = &dig_port->base; ++ ++ if (!intel_crtc) { ++ DRM_DEBUG_KMS("DRRS: intel_crtc not initialized\n"); ++ return; ++ } ++ ++ if (dev_priv->drrs.type < SEAMLESS_DRRS_SUPPORT) { ++ DRM_DEBUG_KMS("Only Seamless DRRS supported.\n"); ++ return; ++ } ++ ++ if (intel_dp->attached_connector->panel.downclock_mode->vrefresh == ++ refresh_rate) ++ index = DRRS_LOW_RR; ++ ++ if (index == dev_priv->drrs.refresh_rate_type) { ++ DRM_DEBUG_KMS( ++ "DRRS requested for previously set RR...ignoring\n"); ++ return; ++ } ++ ++ if (!crtc_state->base.active) { ++ DRM_DEBUG_KMS("eDP encoder disabled. CRTC not Active\n"); ++ return; ++ } ++ ++ if (INTEL_GEN(dev_priv) >= 8 && !IS_CHERRYVIEW(dev_priv)) { ++ switch (index) { ++ case DRRS_HIGH_RR: ++ intel_dp_set_m_n(crtc_state, M1_N1); ++ break; ++ case DRRS_LOW_RR: ++ intel_dp_set_m_n(crtc_state, M2_N2); ++ break; ++ case DRRS_MAX_RR: ++ default: ++ DRM_ERROR("Unsupported refreshrate type\n"); ++ } ++ } else if (INTEL_GEN(dev_priv) > 6) { ++ i915_reg_t reg = PIPECONF(crtc_state->cpu_transcoder); ++ u32 val; ++ ++ val = I915_READ(reg); ++ if (index > DRRS_HIGH_RR) { ++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ val |= PIPECONF_EDP_RR_MODE_SWITCH_VLV; ++ else ++ val |= PIPECONF_EDP_RR_MODE_SWITCH; ++ } else { ++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ val &= ~PIPECONF_EDP_RR_MODE_SWITCH_VLV; ++ else ++ val &= ~PIPECONF_EDP_RR_MODE_SWITCH; ++ } ++ I915_WRITE(reg, val); ++ } ++ ++ dev_priv->drrs.refresh_rate_type = index; ++ ++ DRM_DEBUG_KMS("eDP Refresh Rate set to : %dHz\n", refresh_rate); ++} ++ ++/** ++ * intel_edp_drrs_enable - init drrs struct if supported ++ * @intel_dp: DP struct ++ * @crtc_state: A pointer to the active crtc state. ++ * ++ * Initializes frontbuffer_bits and drrs.dp ++ */ ++void intel_edp_drrs_enable(struct intel_dp *intel_dp, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ ++ if (!crtc_state->has_drrs) { ++ DRM_DEBUG_KMS("Panel doesn't support DRRS\n"); ++ return; ++ } ++ ++ if (dev_priv->psr.enabled) { ++ DRM_DEBUG_KMS("PSR enabled. Not enabling DRRS.\n"); ++ return; ++ } ++ ++ mutex_lock(&dev_priv->drrs.mutex); ++ if (dev_priv->drrs.dp) { ++ DRM_DEBUG_KMS("DRRS already enabled\n"); ++ goto unlock; ++ } ++ ++ dev_priv->drrs.busy_frontbuffer_bits = 0; ++ ++ dev_priv->drrs.dp = intel_dp; ++ ++unlock: ++ mutex_unlock(&dev_priv->drrs.mutex); ++} ++ ++/** ++ * intel_edp_drrs_disable - Disable DRRS ++ * @intel_dp: DP struct ++ * @old_crtc_state: Pointer to old crtc_state. ++ * ++ */ ++void intel_edp_drrs_disable(struct intel_dp *intel_dp, ++ const struct intel_crtc_state *old_crtc_state) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ ++ if (!old_crtc_state->has_drrs) ++ return; ++ ++ mutex_lock(&dev_priv->drrs.mutex); ++ if (!dev_priv->drrs.dp) { ++ mutex_unlock(&dev_priv->drrs.mutex); ++ return; ++ } ++ ++ if (dev_priv->drrs.refresh_rate_type == DRRS_LOW_RR) ++ intel_dp_set_drrs_state(dev_priv, old_crtc_state, ++ intel_dp->attached_connector->panel.fixed_mode->vrefresh); ++ ++ dev_priv->drrs.dp = NULL; ++ mutex_unlock(&dev_priv->drrs.mutex); ++ ++ cancel_delayed_work_sync(&dev_priv->drrs.work); ++} ++ ++static void intel_edp_drrs_downclock_work(struct work_struct *work) ++{ ++ struct drm_i915_private *dev_priv = ++ container_of(work, typeof(*dev_priv), drrs.work.work); ++ struct intel_dp *intel_dp; ++ ++ mutex_lock(&dev_priv->drrs.mutex); ++ ++ intel_dp = dev_priv->drrs.dp; ++ ++ if (!intel_dp) ++ goto unlock; ++ ++ /* ++ * The delayed work can race with an invalidate hence we need to ++ * recheck. ++ */ ++ ++ if (dev_priv->drrs.busy_frontbuffer_bits) ++ goto unlock; ++ ++ if (dev_priv->drrs.refresh_rate_type != DRRS_LOW_RR) { ++ struct drm_crtc *crtc = dp_to_dig_port(intel_dp)->base.base.crtc; ++ ++ intel_dp_set_drrs_state(dev_priv, to_intel_crtc(crtc)->config, ++ intel_dp->attached_connector->panel.downclock_mode->vrefresh); ++ } ++ ++unlock: ++ mutex_unlock(&dev_priv->drrs.mutex); ++} ++ ++/** ++ * intel_edp_drrs_invalidate - Disable Idleness DRRS ++ * @dev_priv: i915 device ++ * @frontbuffer_bits: frontbuffer plane tracking bits ++ * ++ * This function gets called everytime rendering on the given planes start. ++ * Hence DRRS needs to be Upclocked, i.e. (LOW_RR -> HIGH_RR). ++ * ++ * Dirty frontbuffers relevant to DRRS are tracked in busy_frontbuffer_bits. ++ */ ++void intel_edp_drrs_invalidate(struct drm_i915_private *dev_priv, ++ unsigned int frontbuffer_bits) ++{ ++ struct drm_crtc *crtc; ++ enum pipe pipe; ++ ++ if (dev_priv->drrs.type == DRRS_NOT_SUPPORTED) ++ return; ++ ++ cancel_delayed_work(&dev_priv->drrs.work); ++ ++ mutex_lock(&dev_priv->drrs.mutex); ++ if (!dev_priv->drrs.dp) { ++ mutex_unlock(&dev_priv->drrs.mutex); ++ return; ++ } ++ ++ crtc = dp_to_dig_port(dev_priv->drrs.dp)->base.base.crtc; ++ pipe = to_intel_crtc(crtc)->pipe; ++ ++ frontbuffer_bits &= INTEL_FRONTBUFFER_ALL_MASK(pipe); ++ dev_priv->drrs.busy_frontbuffer_bits |= frontbuffer_bits; ++ ++ /* invalidate means busy screen hence upclock */ ++ if (frontbuffer_bits && dev_priv->drrs.refresh_rate_type == DRRS_LOW_RR) ++ intel_dp_set_drrs_state(dev_priv, to_intel_crtc(crtc)->config, ++ dev_priv->drrs.dp->attached_connector->panel.fixed_mode->vrefresh); ++ ++ mutex_unlock(&dev_priv->drrs.mutex); ++} ++ ++/** ++ * intel_edp_drrs_flush - Restart Idleness DRRS ++ * @dev_priv: i915 device ++ * @frontbuffer_bits: frontbuffer plane tracking bits ++ * ++ * This function gets called every time rendering on the given planes has ++ * completed or flip on a crtc is completed. So DRRS should be upclocked ++ * (LOW_RR -> HIGH_RR). And also Idleness detection should be started again, ++ * if no other planes are dirty. ++ * ++ * Dirty frontbuffers relevant to DRRS are tracked in busy_frontbuffer_bits. ++ */ ++void intel_edp_drrs_flush(struct drm_i915_private *dev_priv, ++ unsigned int frontbuffer_bits) ++{ ++ struct drm_crtc *crtc; ++ enum pipe pipe; ++ ++ if (dev_priv->drrs.type == DRRS_NOT_SUPPORTED) ++ return; ++ ++ cancel_delayed_work(&dev_priv->drrs.work); ++ ++ mutex_lock(&dev_priv->drrs.mutex); ++ if (!dev_priv->drrs.dp) { ++ mutex_unlock(&dev_priv->drrs.mutex); ++ return; ++ } ++ ++ crtc = dp_to_dig_port(dev_priv->drrs.dp)->base.base.crtc; ++ pipe = to_intel_crtc(crtc)->pipe; ++ ++ frontbuffer_bits &= INTEL_FRONTBUFFER_ALL_MASK(pipe); ++ dev_priv->drrs.busy_frontbuffer_bits &= ~frontbuffer_bits; ++ ++ /* flush means busy screen hence upclock */ ++ if (frontbuffer_bits && dev_priv->drrs.refresh_rate_type == DRRS_LOW_RR) ++ intel_dp_set_drrs_state(dev_priv, to_intel_crtc(crtc)->config, ++ dev_priv->drrs.dp->attached_connector->panel.fixed_mode->vrefresh); ++ ++ /* ++ * flush also means no more activity hence schedule downclock, if all ++ * other fbs are quiescent too ++ */ ++ if (!dev_priv->drrs.busy_frontbuffer_bits) ++ schedule_delayed_work(&dev_priv->drrs.work, ++ msecs_to_jiffies(1000)); ++ mutex_unlock(&dev_priv->drrs.mutex); ++} ++ ++/** ++ * DOC: Display Refresh Rate Switching (DRRS) ++ * ++ * Display Refresh Rate Switching (DRRS) is a power conservation feature ++ * which enables swtching between low and high refresh rates, ++ * dynamically, based on the usage scenario. This feature is applicable ++ * for internal panels. ++ * ++ * Indication that the panel supports DRRS is given by the panel EDID, which ++ * would list multiple refresh rates for one resolution. ++ * ++ * DRRS is of 2 types - static and seamless. ++ * Static DRRS involves changing refresh rate (RR) by doing a full modeset ++ * (may appear as a blink on screen) and is used in dock-undock scenario. ++ * Seamless DRRS involves changing RR without any visual effect to the user ++ * and can be used during normal system usage. This is done by programming ++ * certain registers. ++ * ++ * Support for static/seamless DRRS may be indicated in the VBT based on ++ * inputs from the panel spec. ++ * ++ * DRRS saves power by switching to low RR based on usage scenarios. ++ * ++ * The implementation is based on frontbuffer tracking implementation. When ++ * there is a disturbance on the screen triggered by user activity or a periodic ++ * system activity, DRRS is disabled (RR is changed to high RR). When there is ++ * no movement on screen, after a timeout of 1 second, a switch to low RR is ++ * made. ++ * ++ * For integration with frontbuffer tracking code, intel_edp_drrs_invalidate() ++ * and intel_edp_drrs_flush() are called. ++ * ++ * DRRS can be further extended to support other internal panels and also ++ * the scenario of video playback wherein RR is set based on the rate ++ * requested by userspace. ++ */ ++ ++/** ++ * intel_dp_drrs_init - Init basic DRRS work and mutex. ++ * @connector: eDP connector ++ * @fixed_mode: preferred mode of panel ++ * ++ * This function is called only once at driver load to initialize basic ++ * DRRS stuff. ++ * ++ * Returns: ++ * Downclock mode if panel supports it, else return NULL. ++ * DRRS support is determined by the presence of downclock mode (apart ++ * from VBT setting). ++ */ ++static struct drm_display_mode * ++intel_dp_drrs_init(struct intel_connector *connector, ++ struct drm_display_mode *fixed_mode) ++{ ++ struct drm_i915_private *dev_priv = to_i915(connector->base.dev); ++ struct drm_display_mode *downclock_mode = NULL; ++ ++ INIT_DELAYED_WORK(&dev_priv->drrs.work, intel_edp_drrs_downclock_work); ++ mutex_init(&dev_priv->drrs.mutex); ++ ++ if (INTEL_GEN(dev_priv) <= 6) { ++ DRM_DEBUG_KMS("DRRS supported for Gen7 and above\n"); ++ return NULL; ++ } ++ ++ if (dev_priv->vbt.drrs_type != SEAMLESS_DRRS_SUPPORT) { ++ DRM_DEBUG_KMS("VBT doesn't support DRRS\n"); ++ return NULL; ++ } ++ ++ downclock_mode = intel_panel_edid_downclock_mode(connector, fixed_mode); ++ if (!downclock_mode) { ++ DRM_DEBUG_KMS("Downclock mode is not found. DRRS not supported\n"); ++ return NULL; ++ } ++ ++ dev_priv->drrs.type = dev_priv->vbt.drrs_type; ++ ++ dev_priv->drrs.refresh_rate_type = DRRS_HIGH_RR; ++ DRM_DEBUG_KMS("seamless DRRS supported for eDP panel.\n"); ++ return downclock_mode; ++} ++ ++static bool intel_edp_init_connector(struct intel_dp *intel_dp, ++ struct intel_connector *intel_connector) ++{ ++ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ++ struct drm_device *dev = &dev_priv->drm; ++ struct drm_connector *connector = &intel_connector->base; ++ struct drm_display_mode *fixed_mode = NULL; ++ struct drm_display_mode *downclock_mode = NULL; ++ bool has_dpcd; ++ enum pipe pipe = INVALID_PIPE; ++ intel_wakeref_t wakeref; ++ struct edid *edid; ++ ++ if (!intel_dp_is_edp(intel_dp)) ++ return true; ++ ++ INIT_DELAYED_WORK(&intel_dp->panel_vdd_work, edp_panel_vdd_work); ++ ++ /* ++ * On IBX/CPT we may get here with LVDS already registered. Since the ++ * driver uses the only internal power sequencer available for both ++ * eDP and LVDS bail out early in this case to prevent interfering ++ * with an already powered-on LVDS power sequencer. ++ */ ++ if (intel_get_lvds_encoder(dev_priv)) { ++ WARN_ON(!(HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))); ++ DRM_INFO("LVDS was detected, not registering eDP\n"); ++ ++ return false; ++ } ++ ++ with_pps_lock(intel_dp, wakeref) { ++ intel_dp_init_panel_power_timestamps(intel_dp); ++ intel_dp_pps_init(intel_dp); ++ intel_edp_panel_vdd_sanitize(intel_dp); ++ } ++ ++ /* Cache DPCD and EDID for edp. */ ++ has_dpcd = intel_edp_init_dpcd(intel_dp); ++ ++ if (!has_dpcd) { ++ /* if this fails, presume the device is a ghost */ ++ DRM_INFO("failed to retrieve link info, disabling eDP\n"); ++ goto out_vdd_off; ++ } ++ ++ mutex_lock(&dev->mode_config.mutex); ++ edid = drm_get_edid(connector, &intel_dp->aux.ddc); ++ if (edid) { ++ if (drm_add_edid_modes(connector, edid)) { ++ drm_connector_update_edid_property(connector, ++ edid); ++ } else { ++ kfree(edid); ++ edid = ERR_PTR(-EINVAL); ++ } ++ } else { ++ edid = ERR_PTR(-ENOENT); ++ } ++ intel_connector->edid = edid; ++ ++ fixed_mode = intel_panel_edid_fixed_mode(intel_connector); ++ if (fixed_mode) ++ downclock_mode = intel_dp_drrs_init(intel_connector, fixed_mode); ++ ++ /* fallback to VBT if available for eDP */ ++ if (!fixed_mode) ++ fixed_mode = intel_panel_vbt_fixed_mode(intel_connector); ++ mutex_unlock(&dev->mode_config.mutex); ++ ++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { ++ intel_dp->edp_notifier.notifier_call = edp_notify_handler; ++ register_reboot_notifier(&intel_dp->edp_notifier); ++ ++ /* ++ * Figure out the current pipe for the initial backlight setup. ++ * If the current pipe isn't valid, try the PPS pipe, and if that ++ * fails just assume pipe A. ++ */ ++ pipe = vlv_active_pipe(intel_dp); ++ ++ if (pipe != PIPE_A && pipe != PIPE_B) ++ pipe = intel_dp->pps_pipe; ++ ++ if (pipe != PIPE_A && pipe != PIPE_B) ++ pipe = PIPE_A; ++ ++ DRM_DEBUG_KMS("using pipe %c for initial backlight setup\n", ++ pipe_name(pipe)); ++ } ++ ++ intel_panel_init(&intel_connector->panel, fixed_mode, downclock_mode); ++ intel_connector->panel.backlight.power = intel_edp_backlight_power; ++ intel_panel_setup_backlight(connector, pipe); ++ ++ if (fixed_mode) ++ drm_connector_init_panel_orientation_property( ++ connector, fixed_mode->hdisplay, fixed_mode->vdisplay); ++ ++ return true; ++ ++out_vdd_off: ++ cancel_delayed_work_sync(&intel_dp->panel_vdd_work); ++ /* ++ * vdd might still be enabled do to the delayed vdd off. ++ * Make sure vdd is actually turned off here. ++ */ ++ with_pps_lock(intel_dp, wakeref) ++ edp_panel_vdd_off_sync(intel_dp); ++ ++ return false; ++} ++ ++static void intel_dp_modeset_retry_work_fn(struct work_struct *work) ++{ ++ struct intel_connector *intel_connector; ++ struct drm_connector *connector; ++ ++ intel_connector = container_of(work, typeof(*intel_connector), ++ modeset_retry_work); ++ connector = &intel_connector->base; ++ DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", connector->base.id, ++ connector->name); ++ ++ /* Grab the locks before changing connector property*/ ++ mutex_lock(&connector->dev->mode_config.mutex); ++ /* Set connector link status to BAD and send a Uevent to notify ++ * userspace to do a modeset. ++ */ ++ drm_connector_set_link_status_property(connector, ++ DRM_MODE_LINK_STATUS_BAD); ++ mutex_unlock(&connector->dev->mode_config.mutex); ++ /* Send Hotplug uevent so userspace can reprobe */ ++ drm_kms_helper_hotplug_event(connector->dev); ++} ++ ++bool ++intel_dp_init_connector(struct intel_digital_port *intel_dig_port, ++ struct intel_connector *intel_connector) ++{ ++ struct drm_connector *connector = &intel_connector->base; ++ struct intel_dp *intel_dp = &intel_dig_port->dp; ++ struct intel_encoder *intel_encoder = &intel_dig_port->base; ++ struct drm_device *dev = intel_encoder->base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ enum port port = intel_encoder->port; ++ int type; ++ ++ /* Initialize the work for modeset in case of link train failure */ ++ INIT_WORK(&intel_connector->modeset_retry_work, ++ intel_dp_modeset_retry_work_fn); ++ ++ if (WARN(intel_dig_port->max_lanes < 1, ++ "Not enough lanes (%d) for DP on port %c\n", ++ intel_dig_port->max_lanes, port_name(port))) ++ return false; ++ ++ intel_dp_set_source_rates(intel_dp); ++ ++ intel_dp->reset_link_params = true; ++ intel_dp->pps_pipe = INVALID_PIPE; ++ intel_dp->active_pipe = INVALID_PIPE; ++ ++ /* intel_dp vfuncs */ ++ if (HAS_DDI(dev_priv)) ++ intel_dp->prepare_link_retrain = intel_ddi_prepare_link_retrain; ++ ++ /* Preserve the current hw state. */ ++ intel_dp->DP = I915_READ(intel_dp->output_reg); ++ intel_dp->attached_connector = intel_connector; ++ ++ if (intel_dp_is_port_edp(dev_priv, port)) ++ type = DRM_MODE_CONNECTOR_eDP; ++ else ++ type = DRM_MODE_CONNECTOR_DisplayPort; ++ ++ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ++ intel_dp->active_pipe = vlv_active_pipe(intel_dp); ++ ++ /* ++ * For eDP we always set the encoder type to INTEL_OUTPUT_EDP, but ++ * for DP the encoder type can be set by the caller to ++ * INTEL_OUTPUT_UNKNOWN for DDI, so don't rewrite it. ++ */ ++ if (type == DRM_MODE_CONNECTOR_eDP) ++ intel_encoder->type = INTEL_OUTPUT_EDP; ++ ++ /* eDP only on port B and/or C on vlv/chv */ ++ if (WARN_ON((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) && ++ intel_dp_is_edp(intel_dp) && ++ port != PORT_B && port != PORT_C)) ++ return false; ++ ++ DRM_DEBUG_KMS("Adding %s connector on port %c\n", ++ type == DRM_MODE_CONNECTOR_eDP ? "eDP" : "DP", ++ port_name(port)); ++ ++ drm_connector_init(dev, connector, &intel_dp_connector_funcs, type); ++ drm_connector_helper_add(connector, &intel_dp_connector_helper_funcs); ++ ++ if (!HAS_GMCH(dev_priv)) ++ connector->interlace_allowed = true; ++ connector->doublescan_allowed = 0; ++ ++ intel_encoder->hpd_pin = intel_hpd_pin_default(dev_priv, port); ++ ++ intel_dp_aux_init(intel_dp); ++ ++ intel_connector_attach_encoder(intel_connector, intel_encoder); ++ ++ if (HAS_DDI(dev_priv)) ++ intel_connector->get_hw_state = intel_ddi_connector_get_hw_state; ++ else ++ intel_connector->get_hw_state = intel_connector_get_hw_state; ++ ++ /* init MST on ports that can support it */ ++ if (HAS_DP_MST(dev_priv) && !intel_dp_is_edp(intel_dp) && ++ (port == PORT_B || port == PORT_C || ++ port == PORT_D || port == PORT_F)) ++ intel_dp_mst_encoder_init(intel_dig_port, ++ intel_connector->base.base.id); ++ ++ if (!intel_edp_init_connector(intel_dp, intel_connector)) { ++ intel_dp_aux_fini(intel_dp); ++ intel_dp_mst_encoder_cleanup(intel_dig_port); ++ goto fail; ++ } ++ ++ intel_dp_add_properties(intel_dp, connector); ++ ++ if (is_hdcp_supported(dev_priv, port) && !intel_dp_is_edp(intel_dp)) { ++ int ret = intel_hdcp_init(intel_connector, &intel_dp_hdcp_shim); ++ if (ret) ++ DRM_DEBUG_KMS("HDCP init failed, skipping.\n"); ++ } ++ ++ /* For G4X desktop chip, PEG_BAND_GAP_DATA 3:0 must first be written ++ * 0xd. Failure to do so will result in spurious interrupts being ++ * generated on the port when a cable is not attached. ++ */ ++ if (IS_G45(dev_priv)) { ++ u32 temp = I915_READ(PEG_BAND_GAP_DATA); ++ I915_WRITE(PEG_BAND_GAP_DATA, (temp & ~0xf) | 0xd); ++ } ++ ++ return true; ++ ++fail: ++ drm_connector_cleanup(connector); ++ ++ return false; ++} ++ ++bool intel_dp_init(struct drm_i915_private *dev_priv, ++ i915_reg_t output_reg, ++ enum port port) ++{ ++ struct intel_digital_port *intel_dig_port; ++ struct intel_encoder *intel_encoder; ++ struct drm_encoder *encoder; ++ struct intel_connector *intel_connector; ++ ++ intel_dig_port = kzalloc(sizeof(*intel_dig_port), GFP_KERNEL); ++ if (!intel_dig_port) ++ return false; ++ ++ intel_connector = intel_connector_alloc(); ++ if (!intel_connector) ++ goto err_connector_alloc; ++ ++ intel_encoder = &intel_dig_port->base; ++ encoder = &intel_encoder->base; ++ ++ if (drm_encoder_init(&dev_priv->drm, &intel_encoder->base, ++ &intel_dp_enc_funcs, DRM_MODE_ENCODER_TMDS, ++ "DP %c", port_name(port))) ++ goto err_encoder_init; ++ ++ intel_encoder->hotplug = intel_dp_hotplug; ++ intel_encoder->compute_config = intel_dp_compute_config; ++ intel_encoder->get_hw_state = intel_dp_get_hw_state; ++ intel_encoder->get_config = intel_dp_get_config; ++ intel_encoder->update_pipe = intel_panel_update_backlight; ++ intel_encoder->suspend = intel_dp_encoder_suspend; ++ if (IS_CHERRYVIEW(dev_priv)) { ++ intel_encoder->pre_pll_enable = chv_dp_pre_pll_enable; ++ intel_encoder->pre_enable = chv_pre_enable_dp; ++ intel_encoder->enable = vlv_enable_dp; ++ intel_encoder->disable = vlv_disable_dp; ++ intel_encoder->post_disable = chv_post_disable_dp; ++ intel_encoder->post_pll_disable = chv_dp_post_pll_disable; ++ } else if (IS_VALLEYVIEW(dev_priv)) { ++ intel_encoder->pre_pll_enable = vlv_dp_pre_pll_enable; ++ intel_encoder->pre_enable = vlv_pre_enable_dp; ++ intel_encoder->enable = vlv_enable_dp; ++ intel_encoder->disable = vlv_disable_dp; ++ intel_encoder->post_disable = vlv_post_disable_dp; ++ } else { ++ intel_encoder->pre_enable = g4x_pre_enable_dp; ++ intel_encoder->enable = g4x_enable_dp; ++ intel_encoder->disable = g4x_disable_dp; ++ intel_encoder->post_disable = g4x_post_disable_dp; ++ } ++ ++ intel_dig_port->dp.output_reg = output_reg; ++ intel_dig_port->max_lanes = 4; ++ ++ intel_encoder->type = INTEL_OUTPUT_DP; ++ intel_encoder->power_domain = intel_port_to_power_domain(port); ++ if (IS_CHERRYVIEW(dev_priv)) { ++ if (port == PORT_D) ++ intel_encoder->crtc_mask = 1 << 2; ++ else ++ intel_encoder->crtc_mask = (1 << 0) | (1 << 1); ++ } else { ++ intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2); ++ } ++ intel_encoder->cloneable = 0; ++ intel_encoder->port = port; ++ ++ intel_dig_port->hpd_pulse = intel_dp_hpd_pulse; ++ ++ if (port != PORT_A) ++ intel_infoframe_init(intel_dig_port); ++ ++ intel_dig_port->aux_ch = intel_bios_port_aux_ch(dev_priv, port); ++ if (!intel_dp_init_connector(intel_dig_port, intel_connector)) ++ goto err_init_connector; ++ ++ return true; ++ ++err_init_connector: ++ drm_encoder_cleanup(encoder); ++err_encoder_init: ++ kfree(intel_connector); ++err_connector_alloc: ++ kfree(intel_dig_port); ++ return false; ++} ++ ++void intel_dp_mst_suspend(struct drm_i915_private *dev_priv) ++{ ++ struct intel_encoder *encoder; ++ ++ for_each_intel_encoder(&dev_priv->drm, encoder) { ++ struct intel_dp *intel_dp; ++ ++ if (encoder->type != INTEL_OUTPUT_DDI) ++ continue; ++ ++ intel_dp = enc_to_intel_dp(&encoder->base); ++ ++ if (!intel_dp->can_mst) ++ continue; ++ ++ if (intel_dp->is_mst) ++ drm_dp_mst_topology_mgr_suspend(&intel_dp->mst_mgr); ++ } ++} ++ ++void intel_dp_mst_resume(struct drm_i915_private *dev_priv) ++{ ++ struct intel_encoder *encoder; ++ ++ for_each_intel_encoder(&dev_priv->drm, encoder) { ++ struct intel_dp *intel_dp; ++ int ret; ++ ++ if (encoder->type != INTEL_OUTPUT_DDI) ++ continue; ++ ++ intel_dp = enc_to_intel_dp(&encoder->base); ++ ++ if (!intel_dp->can_mst) ++ continue; ++ ++ ret = drm_dp_mst_topology_mgr_resume(&intel_dp->mst_mgr); ++ if (ret) { ++ intel_dp->is_mst = false; ++ drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr, ++ false); ++ } ++ } ++} +diff --git a/drivers/gpu/drm/i915_legacy/intel_dp.h b/drivers/gpu/drm/i915_legacy/intel_dp.h +new file mode 100644 +index 000000000000..5e9e8d13de6e +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_dp.h +@@ -0,0 +1,122 @@ ++/* SPDX-License-Identifier: MIT */ ++/* ++ * Copyright © 2019 Intel Corporation ++ */ ++ ++#ifndef __INTEL_DP_H__ ++#define __INTEL_DP_H__ ++ ++#include ++ ++#include ++ ++#include "i915_reg.h" ++ ++enum pipe; ++struct drm_connector_state; ++struct drm_encoder; ++struct drm_i915_private; ++struct drm_modeset_acquire_ctx; ++struct intel_connector; ++struct intel_crtc_state; ++struct intel_digital_port; ++struct intel_dp; ++struct intel_encoder; ++ ++struct link_config_limits { ++ int min_clock, max_clock; ++ int min_lane_count, max_lane_count; ++ int min_bpp, max_bpp; ++}; ++ ++void intel_dp_adjust_compliance_config(struct intel_dp *intel_dp, ++ struct intel_crtc_state *pipe_config, ++ struct link_config_limits *limits); ++bool intel_dp_limited_color_range(const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state); ++int intel_dp_min_bpp(const struct intel_crtc_state *crtc_state); ++bool intel_dp_port_enabled(struct drm_i915_private *dev_priv, ++ i915_reg_t dp_reg, enum port port, ++ enum pipe *pipe); ++bool intel_dp_init(struct drm_i915_private *dev_priv, i915_reg_t output_reg, ++ enum port port); ++bool intel_dp_init_connector(struct intel_digital_port *intel_dig_port, ++ struct intel_connector *intel_connector); ++void intel_dp_set_link_params(struct intel_dp *intel_dp, ++ int link_rate, u8 lane_count, ++ bool link_mst); ++int intel_dp_get_link_train_fallback_values(struct intel_dp *intel_dp, ++ int link_rate, u8 lane_count); ++int intel_dp_retrain_link(struct intel_encoder *encoder, ++ struct drm_modeset_acquire_ctx *ctx); ++void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode); ++void intel_dp_sink_set_decompression_state(struct intel_dp *intel_dp, ++ const struct intel_crtc_state *crtc_state, ++ bool enable); ++void intel_dp_encoder_reset(struct drm_encoder *encoder); ++void intel_dp_encoder_suspend(struct intel_encoder *intel_encoder); ++void intel_dp_encoder_flush_work(struct drm_encoder *encoder); ++int intel_dp_compute_config(struct intel_encoder *encoder, ++ struct intel_crtc_state *pipe_config, ++ struct drm_connector_state *conn_state); ++bool intel_dp_is_edp(struct intel_dp *intel_dp); ++bool intel_dp_is_port_edp(struct drm_i915_private *dev_priv, enum port port); ++enum irqreturn intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, ++ bool long_hpd); ++void intel_edp_backlight_on(const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state); ++void intel_edp_backlight_off(const struct drm_connector_state *conn_state); ++void intel_edp_panel_vdd_on(struct intel_dp *intel_dp); ++void intel_edp_panel_on(struct intel_dp *intel_dp); ++void intel_edp_panel_off(struct intel_dp *intel_dp); ++void intel_dp_mst_suspend(struct drm_i915_private *dev_priv); ++void intel_dp_mst_resume(struct drm_i915_private *dev_priv); ++int intel_dp_max_link_rate(struct intel_dp *intel_dp); ++int intel_dp_max_lane_count(struct intel_dp *intel_dp); ++int intel_dp_rate_select(struct intel_dp *intel_dp, int rate); ++void intel_power_sequencer_reset(struct drm_i915_private *dev_priv); ++u32 intel_dp_pack_aux(const u8 *src, int src_bytes); ++ ++void intel_edp_drrs_enable(struct intel_dp *intel_dp, ++ const struct intel_crtc_state *crtc_state); ++void intel_edp_drrs_disable(struct intel_dp *intel_dp, ++ const struct intel_crtc_state *crtc_state); ++void intel_edp_drrs_invalidate(struct drm_i915_private *dev_priv, ++ unsigned int frontbuffer_bits); ++void intel_edp_drrs_flush(struct drm_i915_private *dev_priv, ++ unsigned int frontbuffer_bits); ++ ++void ++intel_dp_program_link_training_pattern(struct intel_dp *intel_dp, ++ u8 dp_train_pat); ++void ++intel_dp_set_signal_levels(struct intel_dp *intel_dp); ++void intel_dp_set_idle_link_train(struct intel_dp *intel_dp); ++u8 ++intel_dp_voltage_max(struct intel_dp *intel_dp); ++u8 ++intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, u8 voltage_swing); ++void intel_dp_compute_rate(struct intel_dp *intel_dp, int port_clock, ++ u8 *link_bw, u8 *rate_select); ++bool intel_dp_source_supports_hbr2(struct intel_dp *intel_dp); ++bool intel_dp_source_supports_hbr3(struct intel_dp *intel_dp); ++bool ++intel_dp_get_link_status(struct intel_dp *intel_dp, u8 *link_status); ++u16 intel_dp_dsc_get_output_bpp(int link_clock, u8 lane_count, ++ int mode_clock, int mode_hdisplay); ++u8 intel_dp_dsc_get_slice_count(struct intel_dp *intel_dp, int mode_clock, ++ int mode_hdisplay); ++ ++bool intel_dp_read_dpcd(struct intel_dp *intel_dp); ++int intel_dp_link_required(int pixel_clock, int bpp); ++int intel_dp_max_data_rate(int max_link_clock, int max_lanes); ++bool intel_digital_port_connected(struct intel_encoder *encoder); ++void icl_tc_phy_disconnect(struct drm_i915_private *dev_priv, ++ struct intel_digital_port *dig_port); ++ ++static inline unsigned int intel_dp_unused_lane_mask(int lane_count) ++{ ++ return ~((1 << lane_count) - 1) & 0xf; ++} ++ ++#endif /* __INTEL_DP_H__ */ +diff --git a/drivers/gpu/drm/i915_legacy/intel_dp_aux_backlight.c b/drivers/gpu/drm/i915_legacy/intel_dp_aux_backlight.c +new file mode 100644 +index 000000000000..357136f17f85 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_dp_aux_backlight.c +@@ -0,0 +1,280 @@ ++/* ++ * Copyright © 2015 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#include "intel_drv.h" ++ ++static void set_aux_backlight_enable(struct intel_dp *intel_dp, bool enable) ++{ ++ u8 reg_val = 0; ++ ++ /* Early return when display use other mechanism to enable backlight. */ ++ if (!(intel_dp->edp_dpcd[1] & DP_EDP_BACKLIGHT_AUX_ENABLE_CAP)) ++ return; ++ ++ if (drm_dp_dpcd_readb(&intel_dp->aux, DP_EDP_DISPLAY_CONTROL_REGISTER, ++ ®_val) < 0) { ++ DRM_DEBUG_KMS("Failed to read DPCD register 0x%x\n", ++ DP_EDP_DISPLAY_CONTROL_REGISTER); ++ return; ++ } ++ if (enable) ++ reg_val |= DP_EDP_BACKLIGHT_ENABLE; ++ else ++ reg_val &= ~(DP_EDP_BACKLIGHT_ENABLE); ++ ++ if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_EDP_DISPLAY_CONTROL_REGISTER, ++ reg_val) != 1) { ++ DRM_DEBUG_KMS("Failed to %s aux backlight\n", ++ enable ? "enable" : "disable"); ++ } ++} ++ ++/* ++ * Read the current backlight value from DPCD register(s) based ++ * on if 8-bit(MSB) or 16-bit(MSB and LSB) values are supported ++ */ ++static u32 intel_dp_aux_get_backlight(struct intel_connector *connector) ++{ ++ struct intel_dp *intel_dp = enc_to_intel_dp(&connector->encoder->base); ++ u8 read_val[2] = { 0x0 }; ++ u16 level = 0; ++ ++ if (drm_dp_dpcd_read(&intel_dp->aux, DP_EDP_BACKLIGHT_BRIGHTNESS_MSB, ++ &read_val, sizeof(read_val)) < 0) { ++ DRM_DEBUG_KMS("Failed to read DPCD register 0x%x\n", ++ DP_EDP_BACKLIGHT_BRIGHTNESS_MSB); ++ return 0; ++ } ++ level = read_val[0]; ++ if (intel_dp->edp_dpcd[2] & DP_EDP_BACKLIGHT_BRIGHTNESS_BYTE_COUNT) ++ level = (read_val[0] << 8 | read_val[1]); ++ ++ return level; ++} ++ ++/* ++ * Sends the current backlight level over the aux channel, checking if its using ++ * 8-bit or 16 bit value (MSB and LSB) ++ */ ++static void ++intel_dp_aux_set_backlight(const struct drm_connector_state *conn_state, u32 level) ++{ ++ struct intel_connector *connector = to_intel_connector(conn_state->connector); ++ struct intel_dp *intel_dp = enc_to_intel_dp(&connector->encoder->base); ++ u8 vals[2] = { 0x0 }; ++ ++ vals[0] = level; ++ ++ /* Write the MSB and/or LSB */ ++ if (intel_dp->edp_dpcd[2] & DP_EDP_BACKLIGHT_BRIGHTNESS_BYTE_COUNT) { ++ vals[0] = (level & 0xFF00) >> 8; ++ vals[1] = (level & 0xFF); ++ } ++ if (drm_dp_dpcd_write(&intel_dp->aux, DP_EDP_BACKLIGHT_BRIGHTNESS_MSB, ++ vals, sizeof(vals)) < 0) { ++ DRM_DEBUG_KMS("Failed to write aux backlight level\n"); ++ return; ++ } ++} ++ ++/* ++ * Set PWM Frequency divider to match desired frequency in vbt. ++ * The PWM Frequency is calculated as 27Mhz / (F x P). ++ * - Where F = PWM Frequency Pre-Divider value programmed by field 7:0 of the ++ * EDP_BACKLIGHT_FREQ_SET register (DPCD Address 00728h) ++ * - Where P = 2^Pn, where Pn is the value programmed by field 4:0 of the ++ * EDP_PWMGEN_BIT_COUNT register (DPCD Address 00724h) ++ */ ++static bool intel_dp_aux_set_pwm_freq(struct intel_connector *connector) ++{ ++ struct drm_i915_private *dev_priv = to_i915(connector->base.dev); ++ struct intel_dp *intel_dp = enc_to_intel_dp(&connector->encoder->base); ++ int freq, fxp, fxp_min, fxp_max, fxp_actual, f = 1; ++ u8 pn, pn_min, pn_max; ++ ++ /* Find desired value of (F x P) ++ * Note that, if F x P is out of supported range, the maximum value or ++ * minimum value will applied automatically. So no need to check that. ++ */ ++ freq = dev_priv->vbt.backlight.pwm_freq_hz; ++ DRM_DEBUG_KMS("VBT defined backlight frequency %u Hz\n", freq); ++ if (!freq) { ++ DRM_DEBUG_KMS("Use panel default backlight frequency\n"); ++ return false; ++ } ++ ++ fxp = DIV_ROUND_CLOSEST(KHz(DP_EDP_BACKLIGHT_FREQ_BASE_KHZ), freq); ++ ++ /* Use highest possible value of Pn for more granularity of brightness ++ * adjustment while satifying the conditions below. ++ * - Pn is in the range of Pn_min and Pn_max ++ * - F is in the range of 1 and 255 ++ * - FxP is within 25% of desired value. ++ * Note: 25% is arbitrary value and may need some tweak. ++ */ ++ if (drm_dp_dpcd_readb(&intel_dp->aux, ++ DP_EDP_PWMGEN_BIT_COUNT_CAP_MIN, &pn_min) != 1) { ++ DRM_DEBUG_KMS("Failed to read pwmgen bit count cap min\n"); ++ return false; ++ } ++ if (drm_dp_dpcd_readb(&intel_dp->aux, ++ DP_EDP_PWMGEN_BIT_COUNT_CAP_MAX, &pn_max) != 1) { ++ DRM_DEBUG_KMS("Failed to read pwmgen bit count cap max\n"); ++ return false; ++ } ++ pn_min &= DP_EDP_PWMGEN_BIT_COUNT_MASK; ++ pn_max &= DP_EDP_PWMGEN_BIT_COUNT_MASK; ++ ++ fxp_min = DIV_ROUND_CLOSEST(fxp * 3, 4); ++ fxp_max = DIV_ROUND_CLOSEST(fxp * 5, 4); ++ if (fxp_min < (1 << pn_min) || (255 << pn_max) < fxp_max) { ++ DRM_DEBUG_KMS("VBT defined backlight frequency out of range\n"); ++ return false; ++ } ++ ++ for (pn = pn_max; pn >= pn_min; pn--) { ++ f = clamp(DIV_ROUND_CLOSEST(fxp, 1 << pn), 1, 255); ++ fxp_actual = f << pn; ++ if (fxp_min <= fxp_actual && fxp_actual <= fxp_max) ++ break; ++ } ++ ++ if (drm_dp_dpcd_writeb(&intel_dp->aux, ++ DP_EDP_PWMGEN_BIT_COUNT, pn) < 0) { ++ DRM_DEBUG_KMS("Failed to write aux pwmgen bit count\n"); ++ return false; ++ } ++ if (drm_dp_dpcd_writeb(&intel_dp->aux, ++ DP_EDP_BACKLIGHT_FREQ_SET, (u8) f) < 0) { ++ DRM_DEBUG_KMS("Failed to write aux backlight freq\n"); ++ return false; ++ } ++ return true; ++} ++ ++static void intel_dp_aux_enable_backlight(const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state) ++{ ++ struct intel_connector *connector = to_intel_connector(conn_state->connector); ++ struct intel_dp *intel_dp = enc_to_intel_dp(&connector->encoder->base); ++ u8 dpcd_buf, new_dpcd_buf, edp_backlight_mode; ++ ++ if (drm_dp_dpcd_readb(&intel_dp->aux, ++ DP_EDP_BACKLIGHT_MODE_SET_REGISTER, &dpcd_buf) != 1) { ++ DRM_DEBUG_KMS("Failed to read DPCD register 0x%x\n", ++ DP_EDP_BACKLIGHT_MODE_SET_REGISTER); ++ return; ++ } ++ ++ new_dpcd_buf = dpcd_buf; ++ edp_backlight_mode = dpcd_buf & DP_EDP_BACKLIGHT_CONTROL_MODE_MASK; ++ ++ switch (edp_backlight_mode) { ++ case DP_EDP_BACKLIGHT_CONTROL_MODE_PWM: ++ case DP_EDP_BACKLIGHT_CONTROL_MODE_PRESET: ++ case DP_EDP_BACKLIGHT_CONTROL_MODE_PRODUCT: ++ new_dpcd_buf &= ~DP_EDP_BACKLIGHT_CONTROL_MODE_MASK; ++ new_dpcd_buf |= DP_EDP_BACKLIGHT_CONTROL_MODE_DPCD; ++ break; ++ ++ /* Do nothing when it is already DPCD mode */ ++ case DP_EDP_BACKLIGHT_CONTROL_MODE_DPCD: ++ default: ++ break; ++ } ++ ++ if (intel_dp->edp_dpcd[2] & DP_EDP_BACKLIGHT_FREQ_AUX_SET_CAP) ++ if (intel_dp_aux_set_pwm_freq(connector)) ++ new_dpcd_buf |= DP_EDP_BACKLIGHT_FREQ_AUX_SET_ENABLE; ++ ++ if (new_dpcd_buf != dpcd_buf) { ++ if (drm_dp_dpcd_writeb(&intel_dp->aux, ++ DP_EDP_BACKLIGHT_MODE_SET_REGISTER, new_dpcd_buf) < 0) { ++ DRM_DEBUG_KMS("Failed to write aux backlight mode\n"); ++ } ++ } ++ ++ set_aux_backlight_enable(intel_dp, true); ++ intel_dp_aux_set_backlight(conn_state, connector->panel.backlight.level); ++} ++ ++static void intel_dp_aux_disable_backlight(const struct drm_connector_state *old_conn_state) ++{ ++ set_aux_backlight_enable(enc_to_intel_dp(old_conn_state->best_encoder), false); ++} ++ ++static int intel_dp_aux_setup_backlight(struct intel_connector *connector, ++ enum pipe pipe) ++{ ++ struct intel_dp *intel_dp = enc_to_intel_dp(&connector->encoder->base); ++ struct intel_panel *panel = &connector->panel; ++ ++ if (intel_dp->edp_dpcd[2] & DP_EDP_BACKLIGHT_BRIGHTNESS_BYTE_COUNT) ++ panel->backlight.max = 0xFFFF; ++ else ++ panel->backlight.max = 0xFF; ++ ++ panel->backlight.min = 0; ++ panel->backlight.level = intel_dp_aux_get_backlight(connector); ++ ++ panel->backlight.enabled = panel->backlight.level != 0; ++ ++ return 0; ++} ++ ++static bool ++intel_dp_aux_display_control_capable(struct intel_connector *connector) ++{ ++ struct intel_dp *intel_dp = enc_to_intel_dp(&connector->encoder->base); ++ ++ /* Check the eDP Display control capabilities registers to determine if ++ * the panel can support backlight control over the aux channel ++ */ ++ if (intel_dp->edp_dpcd[1] & DP_EDP_TCON_BACKLIGHT_ADJUSTMENT_CAP && ++ (intel_dp->edp_dpcd[2] & DP_EDP_BACKLIGHT_BRIGHTNESS_AUX_SET_CAP) && ++ !(intel_dp->edp_dpcd[2] & DP_EDP_BACKLIGHT_BRIGHTNESS_PWM_PIN_CAP)) { ++ DRM_DEBUG_KMS("AUX Backlight Control Supported!\n"); ++ return true; ++ } ++ return false; ++} ++ ++int intel_dp_aux_init_backlight_funcs(struct intel_connector *intel_connector) ++{ ++ struct intel_panel *panel = &intel_connector->panel; ++ ++ if (!i915_modparams.enable_dpcd_backlight) ++ return -ENODEV; ++ ++ if (!intel_dp_aux_display_control_capable(intel_connector)) ++ return -ENODEV; ++ ++ panel->backlight.setup = intel_dp_aux_setup_backlight; ++ panel->backlight.enable = intel_dp_aux_enable_backlight; ++ panel->backlight.disable = intel_dp_aux_disable_backlight; ++ panel->backlight.set = intel_dp_aux_set_backlight; ++ panel->backlight.get = intel_dp_aux_get_backlight; ++ ++ return 0; ++} +diff --git a/drivers/gpu/drm/i915_legacy/intel_dp_link_training.c b/drivers/gpu/drm/i915_legacy/intel_dp_link_training.c +new file mode 100644 +index 000000000000..54b069333e2f +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_dp_link_training.c +@@ -0,0 +1,381 @@ ++/* ++ * Copyright © 2008-2015 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ */ ++ ++#include "intel_dp.h" ++#include "intel_drv.h" ++ ++static void ++intel_dp_dump_link_status(const u8 link_status[DP_LINK_STATUS_SIZE]) ++{ ++ ++ DRM_DEBUG_KMS("ln0_1:0x%x ln2_3:0x%x align:0x%x sink:0x%x adj_req0_1:0x%x adj_req2_3:0x%x", ++ link_status[0], link_status[1], link_status[2], ++ link_status[3], link_status[4], link_status[5]); ++} ++ ++static void ++intel_get_adjust_train(struct intel_dp *intel_dp, ++ const u8 link_status[DP_LINK_STATUS_SIZE]) ++{ ++ u8 v = 0; ++ u8 p = 0; ++ int lane; ++ u8 voltage_max; ++ u8 preemph_max; ++ ++ for (lane = 0; lane < intel_dp->lane_count; lane++) { ++ u8 this_v = drm_dp_get_adjust_request_voltage(link_status, lane); ++ u8 this_p = drm_dp_get_adjust_request_pre_emphasis(link_status, lane); ++ ++ if (this_v > v) ++ v = this_v; ++ if (this_p > p) ++ p = this_p; ++ } ++ ++ voltage_max = intel_dp_voltage_max(intel_dp); ++ if (v >= voltage_max) ++ v = voltage_max | DP_TRAIN_MAX_SWING_REACHED; ++ ++ preemph_max = intel_dp_pre_emphasis_max(intel_dp, v); ++ if (p >= preemph_max) ++ p = preemph_max | DP_TRAIN_MAX_PRE_EMPHASIS_REACHED; ++ ++ for (lane = 0; lane < 4; lane++) ++ intel_dp->train_set[lane] = v | p; ++} ++ ++static bool ++intel_dp_set_link_train(struct intel_dp *intel_dp, ++ u8 dp_train_pat) ++{ ++ u8 buf[sizeof(intel_dp->train_set) + 1]; ++ int ret, len; ++ ++ intel_dp_program_link_training_pattern(intel_dp, dp_train_pat); ++ ++ buf[0] = dp_train_pat; ++ if ((dp_train_pat & DP_TRAINING_PATTERN_MASK) == ++ DP_TRAINING_PATTERN_DISABLE) { ++ /* don't write DP_TRAINING_LANEx_SET on disable */ ++ len = 1; ++ } else { ++ /* DP_TRAINING_LANEx_SET follow DP_TRAINING_PATTERN_SET */ ++ memcpy(buf + 1, intel_dp->train_set, intel_dp->lane_count); ++ len = intel_dp->lane_count + 1; ++ } ++ ++ ret = drm_dp_dpcd_write(&intel_dp->aux, DP_TRAINING_PATTERN_SET, ++ buf, len); ++ ++ return ret == len; ++} ++ ++static bool ++intel_dp_reset_link_train(struct intel_dp *intel_dp, ++ u8 dp_train_pat) ++{ ++ memset(intel_dp->train_set, 0, sizeof(intel_dp->train_set)); ++ intel_dp_set_signal_levels(intel_dp); ++ return intel_dp_set_link_train(intel_dp, dp_train_pat); ++} ++ ++static bool ++intel_dp_update_link_train(struct intel_dp *intel_dp) ++{ ++ int ret; ++ ++ intel_dp_set_signal_levels(intel_dp); ++ ++ ret = drm_dp_dpcd_write(&intel_dp->aux, DP_TRAINING_LANE0_SET, ++ intel_dp->train_set, intel_dp->lane_count); ++ ++ return ret == intel_dp->lane_count; ++} ++ ++static bool intel_dp_link_max_vswing_reached(struct intel_dp *intel_dp) ++{ ++ int lane; ++ ++ for (lane = 0; lane < intel_dp->lane_count; lane++) ++ if ((intel_dp->train_set[lane] & ++ DP_TRAIN_MAX_SWING_REACHED) == 0) ++ return false; ++ ++ return true; ++} ++ ++/* Enable corresponding port and start training pattern 1 */ ++static bool ++intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp) ++{ ++ u8 voltage; ++ int voltage_tries, cr_tries, max_cr_tries; ++ bool max_vswing_reached = false; ++ u8 link_config[2]; ++ u8 link_bw, rate_select; ++ ++ if (intel_dp->prepare_link_retrain) ++ intel_dp->prepare_link_retrain(intel_dp); ++ ++ intel_dp_compute_rate(intel_dp, intel_dp->link_rate, ++ &link_bw, &rate_select); ++ ++ if (link_bw) ++ DRM_DEBUG_KMS("Using LINK_BW_SET value %02x\n", link_bw); ++ else ++ DRM_DEBUG_KMS("Using LINK_RATE_SET value %02x\n", rate_select); ++ ++ /* Write the link configuration data */ ++ link_config[0] = link_bw; ++ link_config[1] = intel_dp->lane_count; ++ if (drm_dp_enhanced_frame_cap(intel_dp->dpcd)) ++ link_config[1] |= DP_LANE_COUNT_ENHANCED_FRAME_EN; ++ drm_dp_dpcd_write(&intel_dp->aux, DP_LINK_BW_SET, link_config, 2); ++ ++ /* eDP 1.4 rate select method. */ ++ if (!link_bw) ++ drm_dp_dpcd_write(&intel_dp->aux, DP_LINK_RATE_SET, ++ &rate_select, 1); ++ ++ link_config[0] = 0; ++ link_config[1] = DP_SET_ANSI_8B10B; ++ drm_dp_dpcd_write(&intel_dp->aux, DP_DOWNSPREAD_CTRL, link_config, 2); ++ ++ intel_dp->DP |= DP_PORT_EN; ++ ++ /* clock recovery */ ++ if (!intel_dp_reset_link_train(intel_dp, ++ DP_TRAINING_PATTERN_1 | ++ DP_LINK_SCRAMBLING_DISABLE)) { ++ DRM_ERROR("failed to enable link training\n"); ++ return false; ++ } ++ ++ /* ++ * The DP 1.4 spec defines the max clock recovery retries value ++ * as 10 but for pre-DP 1.4 devices we set a very tolerant ++ * retry limit of 80 (4 voltage levels x 4 preemphasis levels x ++ * x 5 identical voltage retries). Since the previous specs didn't ++ * define a limit and created the possibility of an infinite loop ++ * we want to prevent any sync from triggering that corner case. ++ */ ++ if (intel_dp->dpcd[DP_DPCD_REV] >= DP_DPCD_REV_14) ++ max_cr_tries = 10; ++ else ++ max_cr_tries = 80; ++ ++ voltage_tries = 1; ++ for (cr_tries = 0; cr_tries < max_cr_tries; ++cr_tries) { ++ u8 link_status[DP_LINK_STATUS_SIZE]; ++ ++ drm_dp_link_train_clock_recovery_delay(intel_dp->dpcd); ++ ++ if (!intel_dp_get_link_status(intel_dp, link_status)) { ++ DRM_ERROR("failed to get link status\n"); ++ return false; ++ } ++ ++ if (drm_dp_clock_recovery_ok(link_status, intel_dp->lane_count)) { ++ DRM_DEBUG_KMS("clock recovery OK\n"); ++ return true; ++ } ++ ++ if (voltage_tries == 5) { ++ DRM_DEBUG_KMS("Same voltage tried 5 times\n"); ++ return false; ++ } ++ ++ if (max_vswing_reached) { ++ DRM_DEBUG_KMS("Max Voltage Swing reached\n"); ++ return false; ++ } ++ ++ voltage = intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK; ++ ++ /* Update training set as requested by target */ ++ intel_get_adjust_train(intel_dp, link_status); ++ if (!intel_dp_update_link_train(intel_dp)) { ++ DRM_ERROR("failed to update link training\n"); ++ return false; ++ } ++ ++ if ((intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK) == ++ voltage) ++ ++voltage_tries; ++ else ++ voltage_tries = 1; ++ ++ if (intel_dp_link_max_vswing_reached(intel_dp)) ++ max_vswing_reached = true; ++ ++ } ++ DRM_ERROR("Failed clock recovery %d times, giving up!\n", max_cr_tries); ++ return false; ++} ++ ++/* ++ * Pick training pattern for channel equalization. Training pattern 4 for HBR3 ++ * or for 1.4 devices that support it, training Pattern 3 for HBR2 ++ * or 1.2 devices that support it, Training Pattern 2 otherwise. ++ */ ++static u32 intel_dp_training_pattern(struct intel_dp *intel_dp) ++{ ++ bool source_tps3, sink_tps3, source_tps4, sink_tps4; ++ ++ /* ++ * Intel platforms that support HBR3 also support TPS4. It is mandatory ++ * for all downstream devices that support HBR3. There are no known eDP ++ * panels that support TPS4 as of Feb 2018 as per VESA eDP_v1.4b_E1 ++ * specification. ++ */ ++ source_tps4 = intel_dp_source_supports_hbr3(intel_dp); ++ sink_tps4 = drm_dp_tps4_supported(intel_dp->dpcd); ++ if (source_tps4 && sink_tps4) { ++ return DP_TRAINING_PATTERN_4; ++ } else if (intel_dp->link_rate == 810000) { ++ if (!source_tps4) ++ DRM_DEBUG_KMS("8.1 Gbps link rate without source HBR3/TPS4 support\n"); ++ if (!sink_tps4) ++ DRM_DEBUG_KMS("8.1 Gbps link rate without sink TPS4 support\n"); ++ } ++ /* ++ * Intel platforms that support HBR2 also support TPS3. TPS3 support is ++ * also mandatory for downstream devices that support HBR2. However, not ++ * all sinks follow the spec. ++ */ ++ source_tps3 = intel_dp_source_supports_hbr2(intel_dp); ++ sink_tps3 = drm_dp_tps3_supported(intel_dp->dpcd); ++ if (source_tps3 && sink_tps3) { ++ return DP_TRAINING_PATTERN_3; ++ } else if (intel_dp->link_rate >= 540000) { ++ if (!source_tps3) ++ DRM_DEBUG_KMS(">=5.4/6.48 Gbps link rate without source HBR2/TPS3 support\n"); ++ if (!sink_tps3) ++ DRM_DEBUG_KMS(">=5.4/6.48 Gbps link rate without sink TPS3 support\n"); ++ } ++ ++ return DP_TRAINING_PATTERN_2; ++} ++ ++static bool ++intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp) ++{ ++ int tries; ++ u32 training_pattern; ++ u8 link_status[DP_LINK_STATUS_SIZE]; ++ bool channel_eq = false; ++ ++ training_pattern = intel_dp_training_pattern(intel_dp); ++ /* Scrambling is disabled for TPS2/3 and enabled for TPS4 */ ++ if (training_pattern != DP_TRAINING_PATTERN_4) ++ training_pattern |= DP_LINK_SCRAMBLING_DISABLE; ++ ++ /* channel equalization */ ++ if (!intel_dp_set_link_train(intel_dp, ++ training_pattern)) { ++ DRM_ERROR("failed to start channel equalization\n"); ++ return false; ++ } ++ ++ for (tries = 0; tries < 5; tries++) { ++ ++ drm_dp_link_train_channel_eq_delay(intel_dp->dpcd); ++ if (!intel_dp_get_link_status(intel_dp, link_status)) { ++ DRM_ERROR("failed to get link status\n"); ++ break; ++ } ++ ++ /* Make sure clock is still ok */ ++ if (!drm_dp_clock_recovery_ok(link_status, ++ intel_dp->lane_count)) { ++ intel_dp_dump_link_status(link_status); ++ DRM_DEBUG_KMS("Clock recovery check failed, cannot " ++ "continue channel equalization\n"); ++ break; ++ } ++ ++ if (drm_dp_channel_eq_ok(link_status, ++ intel_dp->lane_count)) { ++ channel_eq = true; ++ DRM_DEBUG_KMS("Channel EQ done. DP Training " ++ "successful\n"); ++ break; ++ } ++ ++ /* Update training set as requested by target */ ++ intel_get_adjust_train(intel_dp, link_status); ++ if (!intel_dp_update_link_train(intel_dp)) { ++ DRM_ERROR("failed to update link training\n"); ++ break; ++ } ++ } ++ ++ /* Try 5 times, else fail and try at lower BW */ ++ if (tries == 5) { ++ intel_dp_dump_link_status(link_status); ++ DRM_DEBUG_KMS("Channel equalization failed 5 times\n"); ++ } ++ ++ intel_dp_set_idle_link_train(intel_dp); ++ ++ return channel_eq; ++ ++} ++ ++void intel_dp_stop_link_train(struct intel_dp *intel_dp) ++{ ++ intel_dp->link_trained = true; ++ ++ intel_dp_set_link_train(intel_dp, ++ DP_TRAINING_PATTERN_DISABLE); ++} ++ ++void ++intel_dp_start_link_train(struct intel_dp *intel_dp) ++{ ++ struct intel_connector *intel_connector = intel_dp->attached_connector; ++ ++ if (!intel_dp_link_training_clock_recovery(intel_dp)) ++ goto failure_handling; ++ if (!intel_dp_link_training_channel_equalization(intel_dp)) ++ goto failure_handling; ++ ++ DRM_DEBUG_KMS("[CONNECTOR:%d:%s] Link Training Passed at Link Rate = %d, Lane count = %d", ++ intel_connector->base.base.id, ++ intel_connector->base.name, ++ intel_dp->link_rate, intel_dp->lane_count); ++ return; ++ ++ failure_handling: ++ DRM_DEBUG_KMS("[CONNECTOR:%d:%s] Link Training failed at link rate = %d, lane count = %d", ++ intel_connector->base.base.id, ++ intel_connector->base.name, ++ intel_dp->link_rate, intel_dp->lane_count); ++ if (!intel_dp_get_link_train_fallback_values(intel_dp, ++ intel_dp->link_rate, ++ intel_dp->lane_count)) ++ /* Schedule a Hotplug Uevent to userspace to start modeset */ ++ schedule_work(&intel_connector->modeset_retry_work); ++ return; ++} +diff --git a/drivers/gpu/drm/i915_legacy/intel_dp_mst.c b/drivers/gpu/drm/i915_legacy/intel_dp_mst.c +new file mode 100644 +index 000000000000..ffdc80bfde15 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_dp_mst.c +@@ -0,0 +1,678 @@ ++/* ++ * Copyright © 2008 Intel Corporation ++ * 2014 Red Hat Inc. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#include ++#include ++#include ++ ++#include "i915_drv.h" ++#include "intel_audio.h" ++#include "intel_connector.h" ++#include "intel_ddi.h" ++#include "intel_dp.h" ++#include "intel_drv.h" ++ ++static int intel_dp_mst_compute_link_config(struct intel_encoder *encoder, ++ struct intel_crtc_state *crtc_state, ++ struct drm_connector_state *conn_state, ++ struct link_config_limits *limits) ++{ ++ struct drm_atomic_state *state = crtc_state->base.state; ++ struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base); ++ struct intel_dp *intel_dp = &intel_mst->primary->dp; ++ struct intel_connector *connector = ++ to_intel_connector(conn_state->connector); ++ const struct drm_display_mode *adjusted_mode = ++ &crtc_state->base.adjusted_mode; ++ void *port = connector->port; ++ bool constant_n = drm_dp_has_quirk(&intel_dp->desc, ++ DP_DPCD_QUIRK_CONSTANT_N); ++ int bpp, slots = -EINVAL; ++ ++ crtc_state->lane_count = limits->max_lane_count; ++ crtc_state->port_clock = limits->max_clock; ++ ++ for (bpp = limits->max_bpp; bpp >= limits->min_bpp; bpp -= 2 * 3) { ++ crtc_state->pipe_bpp = bpp; ++ ++ crtc_state->pbn = drm_dp_calc_pbn_mode(adjusted_mode->crtc_clock, ++ crtc_state->pipe_bpp); ++ ++ slots = drm_dp_atomic_find_vcpi_slots(state, &intel_dp->mst_mgr, ++ port, crtc_state->pbn); ++ if (slots == -EDEADLK) ++ return slots; ++ if (slots >= 0) ++ break; ++ } ++ ++ if (slots < 0) { ++ DRM_DEBUG_KMS("failed finding vcpi slots:%d\n", slots); ++ return slots; ++ } ++ ++ intel_link_compute_m_n(crtc_state->pipe_bpp, ++ crtc_state->lane_count, ++ adjusted_mode->crtc_clock, ++ crtc_state->port_clock, ++ &crtc_state->dp_m_n, ++ constant_n); ++ crtc_state->dp_m_n.tu = slots; ++ ++ return 0; ++} ++ ++static int intel_dp_mst_compute_config(struct intel_encoder *encoder, ++ struct intel_crtc_state *pipe_config, ++ struct drm_connector_state *conn_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base); ++ struct intel_dp *intel_dp = &intel_mst->primary->dp; ++ struct intel_connector *connector = ++ to_intel_connector(conn_state->connector); ++ struct intel_digital_connector_state *intel_conn_state = ++ to_intel_digital_connector_state(conn_state); ++ const struct drm_display_mode *adjusted_mode = ++ &pipe_config->base.adjusted_mode; ++ void *port = connector->port; ++ struct link_config_limits limits; ++ int ret; ++ ++ if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) ++ return -EINVAL; ++ ++ pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; ++ pipe_config->has_pch_encoder = false; ++ ++ if (intel_conn_state->force_audio == HDMI_AUDIO_AUTO) ++ pipe_config->has_audio = ++ drm_dp_mst_port_has_audio(&intel_dp->mst_mgr, port); ++ else ++ pipe_config->has_audio = ++ intel_conn_state->force_audio == HDMI_AUDIO_ON; ++ ++ /* ++ * for MST we always configure max link bw - the spec doesn't ++ * seem to suggest we should do otherwise. ++ */ ++ limits.min_clock = ++ limits.max_clock = intel_dp_max_link_rate(intel_dp); ++ ++ limits.min_lane_count = ++ limits.max_lane_count = intel_dp_max_lane_count(intel_dp); ++ ++ limits.min_bpp = intel_dp_min_bpp(pipe_config); ++ /* ++ * FIXME: If all the streams can't fit into the link with ++ * their current pipe_bpp we should reduce pipe_bpp across ++ * the board until things start to fit. Until then we ++ * limit to <= 8bpc since that's what was hardcoded for all ++ * MST streams previously. This hack should be removed once ++ * we have the proper retry logic in place. ++ */ ++ limits.max_bpp = min(pipe_config->pipe_bpp, 24); ++ ++ intel_dp_adjust_compliance_config(intel_dp, pipe_config, &limits); ++ ++ ret = intel_dp_mst_compute_link_config(encoder, pipe_config, ++ conn_state, &limits); ++ if (ret) ++ return ret; ++ ++ pipe_config->limited_color_range = ++ intel_dp_limited_color_range(pipe_config, conn_state); ++ ++ if (IS_GEN9_LP(dev_priv)) ++ pipe_config->lane_lat_optim_mask = ++ bxt_ddi_phy_calc_lane_lat_optim_mask(pipe_config->lane_count); ++ ++ intel_ddi_compute_min_voltage_level(dev_priv, pipe_config); ++ ++ return 0; ++} ++ ++static int ++intel_dp_mst_atomic_check(struct drm_connector *connector, ++ struct drm_atomic_state *state) ++{ ++ struct drm_connector_state *new_conn_state = ++ drm_atomic_get_new_connector_state(state, connector); ++ struct drm_connector_state *old_conn_state = ++ drm_atomic_get_old_connector_state(state, connector); ++ struct intel_connector *intel_connector = ++ to_intel_connector(connector); ++ struct drm_crtc *new_crtc = new_conn_state->crtc; ++ struct drm_crtc_state *crtc_state; ++ struct drm_dp_mst_topology_mgr *mgr; ++ int ret; ++ ++ ret = intel_digital_connector_atomic_check(connector, state); ++ if (ret) ++ return ret; ++ ++ if (!old_conn_state->crtc) ++ return 0; ++ ++ /* We only want to free VCPI if this state disables the CRTC on this ++ * connector ++ */ ++ if (new_crtc) { ++ crtc_state = drm_atomic_get_new_crtc_state(state, new_crtc); ++ ++ if (!crtc_state || ++ !drm_atomic_crtc_needs_modeset(crtc_state) || ++ crtc_state->enable) ++ return 0; ++ } ++ ++ mgr = &enc_to_mst(old_conn_state->best_encoder)->primary->dp.mst_mgr; ++ ret = drm_dp_atomic_release_vcpi_slots(state, mgr, ++ intel_connector->port); ++ ++ return ret; ++} ++ ++static void intel_mst_disable_dp(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base); ++ struct intel_digital_port *intel_dig_port = intel_mst->primary; ++ struct intel_dp *intel_dp = &intel_dig_port->dp; ++ struct intel_connector *connector = ++ to_intel_connector(old_conn_state->connector); ++ int ret; ++ ++ DRM_DEBUG_KMS("active links %d\n", intel_dp->active_mst_links); ++ ++ drm_dp_mst_reset_vcpi_slots(&intel_dp->mst_mgr, connector->port); ++ ++ ret = drm_dp_update_payload_part1(&intel_dp->mst_mgr); ++ if (ret) { ++ DRM_ERROR("failed to update payload %d\n", ret); ++ } ++ if (old_crtc_state->has_audio) ++ intel_audio_codec_disable(encoder, ++ old_crtc_state, old_conn_state); ++} ++ ++static void intel_mst_post_disable_dp(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base); ++ struct intel_digital_port *intel_dig_port = intel_mst->primary; ++ struct intel_dp *intel_dp = &intel_dig_port->dp; ++ struct intel_connector *connector = ++ to_intel_connector(old_conn_state->connector); ++ ++ intel_ddi_disable_pipe_clock(old_crtc_state); ++ ++ /* this can fail */ ++ drm_dp_check_act_status(&intel_dp->mst_mgr); ++ /* and this can also fail */ ++ drm_dp_update_payload_part2(&intel_dp->mst_mgr); ++ ++ drm_dp_mst_deallocate_vcpi(&intel_dp->mst_mgr, connector->port); ++ ++ /* ++ * Power down mst path before disabling the port, otherwise we end ++ * up getting interrupts from the sink upon detecting link loss. ++ */ ++ drm_dp_send_power_updown_phy(&intel_dp->mst_mgr, connector->port, ++ false); ++ ++ intel_dp->active_mst_links--; ++ ++ intel_mst->connector = NULL; ++ if (intel_dp->active_mst_links == 0) { ++ intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF); ++ intel_dig_port->base.post_disable(&intel_dig_port->base, ++ old_crtc_state, NULL); ++ } ++ ++ DRM_DEBUG_KMS("active links %d\n", intel_dp->active_mst_links); ++} ++ ++static void intel_mst_pre_pll_enable_dp(struct intel_encoder *encoder, ++ const struct intel_crtc_state *pipe_config, ++ const struct drm_connector_state *conn_state) ++{ ++ struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base); ++ struct intel_digital_port *intel_dig_port = intel_mst->primary; ++ struct intel_dp *intel_dp = &intel_dig_port->dp; ++ ++ if (intel_dp->active_mst_links == 0) ++ intel_dig_port->base.pre_pll_enable(&intel_dig_port->base, ++ pipe_config, NULL); ++} ++ ++static void intel_mst_post_pll_disable_dp(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state, ++ const struct drm_connector_state *old_conn_state) ++{ ++ struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base); ++ struct intel_digital_port *intel_dig_port = intel_mst->primary; ++ struct intel_dp *intel_dp = &intel_dig_port->dp; ++ ++ if (intel_dp->active_mst_links == 0) ++ intel_dig_port->base.post_pll_disable(&intel_dig_port->base, ++ old_crtc_state, ++ old_conn_state); ++} ++ ++static void intel_mst_pre_enable_dp(struct intel_encoder *encoder, ++ const struct intel_crtc_state *pipe_config, ++ const struct drm_connector_state *conn_state) ++{ ++ struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base); ++ struct intel_digital_port *intel_dig_port = intel_mst->primary; ++ struct intel_dp *intel_dp = &intel_dig_port->dp; ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ enum port port = intel_dig_port->base.port; ++ struct intel_connector *connector = ++ to_intel_connector(conn_state->connector); ++ int ret; ++ u32 temp; ++ ++ /* MST encoders are bound to a crtc, not to a connector, ++ * force the mapping here for get_hw_state. ++ */ ++ connector->encoder = encoder; ++ intel_mst->connector = connector; ++ ++ DRM_DEBUG_KMS("active links %d\n", intel_dp->active_mst_links); ++ ++ if (intel_dp->active_mst_links == 0) ++ intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON); ++ ++ drm_dp_send_power_updown_phy(&intel_dp->mst_mgr, connector->port, true); ++ ++ if (intel_dp->active_mst_links == 0) ++ intel_dig_port->base.pre_enable(&intel_dig_port->base, ++ pipe_config, NULL); ++ ++ ret = drm_dp_mst_allocate_vcpi(&intel_dp->mst_mgr, ++ connector->port, ++ pipe_config->pbn, ++ pipe_config->dp_m_n.tu); ++ if (!ret) ++ DRM_ERROR("failed to allocate vcpi\n"); ++ ++ intel_dp->active_mst_links++; ++ temp = I915_READ(DP_TP_STATUS(port)); ++ I915_WRITE(DP_TP_STATUS(port), temp); ++ ++ ret = drm_dp_update_payload_part1(&intel_dp->mst_mgr); ++ ++ intel_ddi_enable_pipe_clock(pipe_config); ++} ++ ++static void intel_mst_enable_dp(struct intel_encoder *encoder, ++ const struct intel_crtc_state *pipe_config, ++ const struct drm_connector_state *conn_state) ++{ ++ struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base); ++ struct intel_digital_port *intel_dig_port = intel_mst->primary; ++ struct intel_dp *intel_dp = &intel_dig_port->dp; ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ enum port port = intel_dig_port->base.port; ++ ++ DRM_DEBUG_KMS("active links %d\n", intel_dp->active_mst_links); ++ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ DP_TP_STATUS(port), ++ DP_TP_STATUS_ACT_SENT, ++ DP_TP_STATUS_ACT_SENT, ++ 1)) ++ DRM_ERROR("Timed out waiting for ACT sent\n"); ++ ++ drm_dp_check_act_status(&intel_dp->mst_mgr); ++ ++ drm_dp_update_payload_part2(&intel_dp->mst_mgr); ++ if (pipe_config->has_audio) ++ intel_audio_codec_enable(encoder, pipe_config, conn_state); ++} ++ ++static bool intel_dp_mst_enc_get_hw_state(struct intel_encoder *encoder, ++ enum pipe *pipe) ++{ ++ struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base); ++ *pipe = intel_mst->pipe; ++ if (intel_mst->connector) ++ return true; ++ return false; ++} ++ ++static void intel_dp_mst_enc_get_config(struct intel_encoder *encoder, ++ struct intel_crtc_state *pipe_config) ++{ ++ struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base); ++ struct intel_digital_port *intel_dig_port = intel_mst->primary; ++ ++ intel_ddi_get_config(&intel_dig_port->base, pipe_config); ++} ++ ++static int intel_dp_mst_get_ddc_modes(struct drm_connector *connector) ++{ ++ struct intel_connector *intel_connector = to_intel_connector(connector); ++ struct intel_dp *intel_dp = intel_connector->mst_port; ++ struct edid *edid; ++ int ret; ++ ++ if (drm_connector_is_unregistered(connector)) ++ return intel_connector_update_modes(connector, NULL); ++ ++ edid = drm_dp_mst_get_edid(connector, &intel_dp->mst_mgr, intel_connector->port); ++ ret = intel_connector_update_modes(connector, edid); ++ kfree(edid); ++ ++ return ret; ++} ++ ++static enum drm_connector_status ++intel_dp_mst_detect(struct drm_connector *connector, bool force) ++{ ++ struct intel_connector *intel_connector = to_intel_connector(connector); ++ struct intel_dp *intel_dp = intel_connector->mst_port; ++ ++ if (drm_connector_is_unregistered(connector)) ++ return connector_status_disconnected; ++ return drm_dp_mst_detect_port(connector, &intel_dp->mst_mgr, ++ intel_connector->port); ++} ++ ++static const struct drm_connector_funcs intel_dp_mst_connector_funcs = { ++ .detect = intel_dp_mst_detect, ++ .fill_modes = drm_helper_probe_single_connector_modes, ++ .atomic_get_property = intel_digital_connector_atomic_get_property, ++ .atomic_set_property = intel_digital_connector_atomic_set_property, ++ .late_register = intel_connector_register, ++ .early_unregister = intel_connector_unregister, ++ .destroy = intel_connector_destroy, ++ .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, ++ .atomic_duplicate_state = intel_digital_connector_duplicate_state, ++}; ++ ++static int intel_dp_mst_get_modes(struct drm_connector *connector) ++{ ++ return intel_dp_mst_get_ddc_modes(connector); ++} ++ ++static enum drm_mode_status ++intel_dp_mst_mode_valid(struct drm_connector *connector, ++ struct drm_display_mode *mode) ++{ ++ struct intel_connector *intel_connector = to_intel_connector(connector); ++ struct intel_dp *intel_dp = intel_connector->mst_port; ++ int max_dotclk = to_i915(connector->dev)->max_dotclk_freq; ++ int max_rate, mode_rate, max_lanes, max_link_clock; ++ ++ if (drm_connector_is_unregistered(connector)) ++ return MODE_ERROR; ++ ++ if (mode->flags & DRM_MODE_FLAG_DBLSCAN) ++ return MODE_NO_DBLESCAN; ++ ++ max_link_clock = intel_dp_max_link_rate(intel_dp); ++ max_lanes = intel_dp_max_lane_count(intel_dp); ++ ++ max_rate = intel_dp_max_data_rate(max_link_clock, max_lanes); ++ mode_rate = intel_dp_link_required(mode->clock, 18); ++ ++ /* TODO - validate mode against available PBN for link */ ++ if (mode->clock < 10000) ++ return MODE_CLOCK_LOW; ++ ++ if (mode->flags & DRM_MODE_FLAG_DBLCLK) ++ return MODE_H_ILLEGAL; ++ ++ if (mode_rate > max_rate || mode->clock > max_dotclk) ++ return MODE_CLOCK_HIGH; ++ ++ return MODE_OK; ++} ++ ++static struct drm_encoder *intel_mst_atomic_best_encoder(struct drm_connector *connector, ++ struct drm_connector_state *state) ++{ ++ struct intel_connector *intel_connector = to_intel_connector(connector); ++ struct intel_dp *intel_dp = intel_connector->mst_port; ++ struct intel_crtc *crtc = to_intel_crtc(state->crtc); ++ ++ return &intel_dp->mst_encoders[crtc->pipe]->base.base; ++} ++ ++static const struct drm_connector_helper_funcs intel_dp_mst_connector_helper_funcs = { ++ .get_modes = intel_dp_mst_get_modes, ++ .mode_valid = intel_dp_mst_mode_valid, ++ .atomic_best_encoder = intel_mst_atomic_best_encoder, ++ .atomic_check = intel_dp_mst_atomic_check, ++}; ++ ++static void intel_dp_mst_encoder_destroy(struct drm_encoder *encoder) ++{ ++ struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); ++ ++ drm_encoder_cleanup(encoder); ++ kfree(intel_mst); ++} ++ ++static const struct drm_encoder_funcs intel_dp_mst_enc_funcs = { ++ .destroy = intel_dp_mst_encoder_destroy, ++}; ++ ++static bool intel_dp_mst_get_hw_state(struct intel_connector *connector) ++{ ++ if (connector->encoder && connector->base.state->crtc) { ++ enum pipe pipe; ++ if (!connector->encoder->get_hw_state(connector->encoder, &pipe)) ++ return false; ++ return true; ++ } ++ return false; ++} ++ ++static struct drm_connector *intel_dp_add_mst_connector(struct drm_dp_mst_topology_mgr *mgr, struct drm_dp_mst_port *port, const char *pathprop) ++{ ++ struct intel_dp *intel_dp = container_of(mgr, struct intel_dp, mst_mgr); ++ struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); ++ struct drm_device *dev = intel_dig_port->base.base.dev; ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ struct intel_connector *intel_connector; ++ struct drm_connector *connector; ++ enum pipe pipe; ++ int ret; ++ ++ intel_connector = intel_connector_alloc(); ++ if (!intel_connector) ++ return NULL; ++ ++ intel_connector->get_hw_state = intel_dp_mst_get_hw_state; ++ intel_connector->mst_port = intel_dp; ++ intel_connector->port = port; ++ drm_dp_mst_get_port_malloc(port); ++ ++ connector = &intel_connector->base; ++ ret = drm_connector_init(dev, connector, &intel_dp_mst_connector_funcs, ++ DRM_MODE_CONNECTOR_DisplayPort); ++ if (ret) { ++ intel_connector_free(intel_connector); ++ return NULL; ++ } ++ ++ drm_connector_helper_add(connector, &intel_dp_mst_connector_helper_funcs); ++ ++ for_each_pipe(dev_priv, pipe) { ++ struct drm_encoder *enc = ++ &intel_dp->mst_encoders[pipe]->base.base; ++ ++ ret = drm_connector_attach_encoder(&intel_connector->base, enc); ++ if (ret) ++ goto err; ++ } ++ ++ drm_object_attach_property(&connector->base, dev->mode_config.path_property, 0); ++ drm_object_attach_property(&connector->base, dev->mode_config.tile_property, 0); ++ ++ ret = drm_connector_set_path_property(connector, pathprop); ++ if (ret) ++ goto err; ++ ++ intel_attach_force_audio_property(connector); ++ intel_attach_broadcast_rgb_property(connector); ++ ++ /* ++ * Reuse the prop from the SST connector because we're ++ * not allowed to create new props after device registration. ++ */ ++ connector->max_bpc_property = ++ intel_dp->attached_connector->base.max_bpc_property; ++ if (connector->max_bpc_property) ++ drm_connector_attach_max_bpc_property(connector, 6, 12); ++ ++ return connector; ++ ++err: ++ drm_connector_cleanup(connector); ++ return NULL; ++} ++ ++static void intel_dp_register_mst_connector(struct drm_connector *connector) ++{ ++ struct drm_i915_private *dev_priv = to_i915(connector->dev); ++ ++ if (dev_priv->fbdev) ++ drm_fb_helper_add_one_connector(&dev_priv->fbdev->helper, ++ connector); ++ ++ drm_connector_register(connector); ++} ++ ++static void intel_dp_destroy_mst_connector(struct drm_dp_mst_topology_mgr *mgr, ++ struct drm_connector *connector) ++{ ++ struct drm_i915_private *dev_priv = to_i915(connector->dev); ++ ++ DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", connector->base.id, connector->name); ++ drm_connector_unregister(connector); ++ ++ if (dev_priv->fbdev) ++ drm_fb_helper_remove_one_connector(&dev_priv->fbdev->helper, ++ connector); ++ ++ drm_connector_put(connector); ++} ++ ++static const struct drm_dp_mst_topology_cbs mst_cbs = { ++ .add_connector = intel_dp_add_mst_connector, ++ .register_connector = intel_dp_register_mst_connector, ++ .destroy_connector = intel_dp_destroy_mst_connector, ++}; ++ ++static struct intel_dp_mst_encoder * ++intel_dp_create_fake_mst_encoder(struct intel_digital_port *intel_dig_port, enum pipe pipe) ++{ ++ struct intel_dp_mst_encoder *intel_mst; ++ struct intel_encoder *intel_encoder; ++ struct drm_device *dev = intel_dig_port->base.base.dev; ++ ++ intel_mst = kzalloc(sizeof(*intel_mst), GFP_KERNEL); ++ ++ if (!intel_mst) ++ return NULL; ++ ++ intel_mst->pipe = pipe; ++ intel_encoder = &intel_mst->base; ++ intel_mst->primary = intel_dig_port; ++ ++ drm_encoder_init(dev, &intel_encoder->base, &intel_dp_mst_enc_funcs, ++ DRM_MODE_ENCODER_DPMST, "DP-MST %c", pipe_name(pipe)); ++ ++ intel_encoder->type = INTEL_OUTPUT_DP_MST; ++ intel_encoder->power_domain = intel_dig_port->base.power_domain; ++ intel_encoder->port = intel_dig_port->base.port; ++ intel_encoder->crtc_mask = 0x7; ++ intel_encoder->cloneable = 0; ++ ++ intel_encoder->compute_config = intel_dp_mst_compute_config; ++ intel_encoder->disable = intel_mst_disable_dp; ++ intel_encoder->post_disable = intel_mst_post_disable_dp; ++ intel_encoder->pre_pll_enable = intel_mst_pre_pll_enable_dp; ++ intel_encoder->post_pll_disable = intel_mst_post_pll_disable_dp; ++ intel_encoder->pre_enable = intel_mst_pre_enable_dp; ++ intel_encoder->enable = intel_mst_enable_dp; ++ intel_encoder->get_hw_state = intel_dp_mst_enc_get_hw_state; ++ intel_encoder->get_config = intel_dp_mst_enc_get_config; ++ ++ return intel_mst; ++ ++} ++ ++static bool ++intel_dp_create_fake_mst_encoders(struct intel_digital_port *intel_dig_port) ++{ ++ struct intel_dp *intel_dp = &intel_dig_port->dp; ++ struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev); ++ enum pipe pipe; ++ ++ for_each_pipe(dev_priv, pipe) ++ intel_dp->mst_encoders[pipe] = intel_dp_create_fake_mst_encoder(intel_dig_port, pipe); ++ return true; ++} ++ ++int ++intel_dp_mst_encoder_init(struct intel_digital_port *intel_dig_port, int conn_base_id) ++{ ++ struct intel_dp *intel_dp = &intel_dig_port->dp; ++ struct drm_device *dev = intel_dig_port->base.base.dev; ++ int ret; ++ ++ intel_dp->can_mst = true; ++ intel_dp->mst_mgr.cbs = &mst_cbs; ++ ++ /* create encoders */ ++ intel_dp_create_fake_mst_encoders(intel_dig_port); ++ ret = drm_dp_mst_topology_mgr_init(&intel_dp->mst_mgr, dev, ++ &intel_dp->aux, 16, 3, conn_base_id); ++ if (ret) { ++ intel_dp->can_mst = false; ++ return ret; ++ } ++ return 0; ++} ++ ++void ++intel_dp_mst_encoder_cleanup(struct intel_digital_port *intel_dig_port) ++{ ++ struct intel_dp *intel_dp = &intel_dig_port->dp; ++ ++ if (!intel_dp->can_mst) ++ return; ++ ++ drm_dp_mst_topology_mgr_destroy(&intel_dp->mst_mgr); ++ /* encoders will get killed by normal cleanup */ ++} +diff --git a/drivers/gpu/drm/i915_legacy/intel_dpio_phy.c b/drivers/gpu/drm/i915_legacy/intel_dpio_phy.c +new file mode 100644 +index 000000000000..ab4ac7158b79 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_dpio_phy.c +@@ -0,0 +1,1082 @@ ++/* ++ * Copyright © 2014-2016 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER ++ * DEALINGS IN THE SOFTWARE. ++ */ ++ ++#include "intel_dp.h" ++#include "intel_drv.h" ++ ++/** ++ * DOC: DPIO ++ * ++ * VLV, CHV and BXT have slightly peculiar display PHYs for driving DP/HDMI ++ * ports. DPIO is the name given to such a display PHY. These PHYs ++ * don't follow the standard programming model using direct MMIO ++ * registers, and instead their registers must be accessed trough IOSF ++ * sideband. VLV has one such PHY for driving ports B and C, and CHV ++ * adds another PHY for driving port D. Each PHY responds to specific ++ * IOSF-SB port. ++ * ++ * Each display PHY is made up of one or two channels. Each channel ++ * houses a common lane part which contains the PLL and other common ++ * logic. CH0 common lane also contains the IOSF-SB logic for the ++ * Common Register Interface (CRI) ie. the DPIO registers. CRI clock ++ * must be running when any DPIO registers are accessed. ++ * ++ * In addition to having their own registers, the PHYs are also ++ * controlled through some dedicated signals from the display ++ * controller. These include PLL reference clock enable, PLL enable, ++ * and CRI clock selection, for example. ++ * ++ * Eeach channel also has two splines (also called data lanes), and ++ * each spline is made up of one Physical Access Coding Sub-Layer ++ * (PCS) block and two TX lanes. So each channel has two PCS blocks ++ * and four TX lanes. The TX lanes are used as DP lanes or TMDS ++ * data/clock pairs depending on the output type. ++ * ++ * Additionally the PHY also contains an AUX lane with AUX blocks ++ * for each channel. This is used for DP AUX communication, but ++ * this fact isn't really relevant for the driver since AUX is ++ * controlled from the display controller side. No DPIO registers ++ * need to be accessed during AUX communication, ++ * ++ * Generally on VLV/CHV the common lane corresponds to the pipe and ++ * the spline (PCS/TX) corresponds to the port. ++ * ++ * For dual channel PHY (VLV/CHV): ++ * ++ * pipe A == CMN/PLL/REF CH0 ++ * ++ * pipe B == CMN/PLL/REF CH1 ++ * ++ * port B == PCS/TX CH0 ++ * ++ * port C == PCS/TX CH1 ++ * ++ * This is especially important when we cross the streams ++ * ie. drive port B with pipe B, or port C with pipe A. ++ * ++ * For single channel PHY (CHV): ++ * ++ * pipe C == CMN/PLL/REF CH0 ++ * ++ * port D == PCS/TX CH0 ++ * ++ * On BXT the entire PHY channel corresponds to the port. That means ++ * the PLL is also now associated with the port rather than the pipe, ++ * and so the clock needs to be routed to the appropriate transcoder. ++ * Port A PLL is directly connected to transcoder EDP and port B/C ++ * PLLs can be routed to any transcoder A/B/C. ++ * ++ * Note: DDI0 is digital port B, DD1 is digital port C, and DDI2 is ++ * digital port D (CHV) or port A (BXT). :: ++ * ++ * ++ * Dual channel PHY (VLV/CHV/BXT) ++ * --------------------------------- ++ * | CH0 | CH1 | ++ * | CMN/PLL/REF | CMN/PLL/REF | ++ * |---------------|---------------| Display PHY ++ * | PCS01 | PCS23 | PCS01 | PCS23 | ++ * |-------|-------|-------|-------| ++ * |TX0|TX1|TX2|TX3|TX0|TX1|TX2|TX3| ++ * --------------------------------- ++ * | DDI0 | DDI1 | DP/HDMI ports ++ * --------------------------------- ++ * ++ * Single channel PHY (CHV/BXT) ++ * ----------------- ++ * | CH0 | ++ * | CMN/PLL/REF | ++ * |---------------| Display PHY ++ * | PCS01 | PCS23 | ++ * |-------|-------| ++ * |TX0|TX1|TX2|TX3| ++ * ----------------- ++ * | DDI2 | DP/HDMI port ++ * ----------------- ++ */ ++ ++/** ++ * struct bxt_ddi_phy_info - Hold info for a broxton DDI phy ++ */ ++struct bxt_ddi_phy_info { ++ /** ++ * @dual_channel: true if this phy has a second channel. ++ */ ++ bool dual_channel; ++ ++ /** ++ * @rcomp_phy: If -1, indicates this phy has its own rcomp resistor. ++ * Otherwise the GRC value will be copied from the phy indicated by ++ * this field. ++ */ ++ enum dpio_phy rcomp_phy; ++ ++ /** ++ * @reset_delay: delay in us to wait before setting the common reset ++ * bit in BXT_PHY_CTL_FAMILY, which effectively enables the phy. ++ */ ++ int reset_delay; ++ ++ /** ++ * @pwron_mask: Mask with the appropriate bit set that would cause the ++ * punit to power this phy if written to BXT_P_CR_GT_DISP_PWRON. ++ */ ++ u32 pwron_mask; ++ ++ /** ++ * @channel: struct containing per channel information. ++ */ ++ struct { ++ /** ++ * @channel.port: which port maps to this channel. ++ */ ++ enum port port; ++ } channel[2]; ++}; ++ ++static const struct bxt_ddi_phy_info bxt_ddi_phy_info[] = { ++ [DPIO_PHY0] = { ++ .dual_channel = true, ++ .rcomp_phy = DPIO_PHY1, ++ .pwron_mask = BIT(0), ++ ++ .channel = { ++ [DPIO_CH0] = { .port = PORT_B }, ++ [DPIO_CH1] = { .port = PORT_C }, ++ } ++ }, ++ [DPIO_PHY1] = { ++ .dual_channel = false, ++ .rcomp_phy = -1, ++ .pwron_mask = BIT(1), ++ ++ .channel = { ++ [DPIO_CH0] = { .port = PORT_A }, ++ } ++ }, ++}; ++ ++static const struct bxt_ddi_phy_info glk_ddi_phy_info[] = { ++ [DPIO_PHY0] = { ++ .dual_channel = false, ++ .rcomp_phy = DPIO_PHY1, ++ .pwron_mask = BIT(0), ++ .reset_delay = 20, ++ ++ .channel = { ++ [DPIO_CH0] = { .port = PORT_B }, ++ } ++ }, ++ [DPIO_PHY1] = { ++ .dual_channel = false, ++ .rcomp_phy = -1, ++ .pwron_mask = BIT(3), ++ .reset_delay = 20, ++ ++ .channel = { ++ [DPIO_CH0] = { .port = PORT_A }, ++ } ++ }, ++ [DPIO_PHY2] = { ++ .dual_channel = false, ++ .rcomp_phy = DPIO_PHY1, ++ .pwron_mask = BIT(1), ++ .reset_delay = 20, ++ ++ .channel = { ++ [DPIO_CH0] = { .port = PORT_C }, ++ } ++ }, ++}; ++ ++static const struct bxt_ddi_phy_info * ++bxt_get_phy_list(struct drm_i915_private *dev_priv, int *count) ++{ ++ if (IS_GEMINILAKE(dev_priv)) { ++ *count = ARRAY_SIZE(glk_ddi_phy_info); ++ return glk_ddi_phy_info; ++ } else { ++ *count = ARRAY_SIZE(bxt_ddi_phy_info); ++ return bxt_ddi_phy_info; ++ } ++} ++ ++static const struct bxt_ddi_phy_info * ++bxt_get_phy_info(struct drm_i915_private *dev_priv, enum dpio_phy phy) ++{ ++ int count; ++ const struct bxt_ddi_phy_info *phy_list = ++ bxt_get_phy_list(dev_priv, &count); ++ ++ return &phy_list[phy]; ++} ++ ++void bxt_port_to_phy_channel(struct drm_i915_private *dev_priv, enum port port, ++ enum dpio_phy *phy, enum dpio_channel *ch) ++{ ++ const struct bxt_ddi_phy_info *phy_info, *phys; ++ int i, count; ++ ++ phys = bxt_get_phy_list(dev_priv, &count); ++ ++ for (i = 0; i < count; i++) { ++ phy_info = &phys[i]; ++ ++ if (port == phy_info->channel[DPIO_CH0].port) { ++ *phy = i; ++ *ch = DPIO_CH0; ++ return; ++ } ++ ++ if (phy_info->dual_channel && ++ port == phy_info->channel[DPIO_CH1].port) { ++ *phy = i; ++ *ch = DPIO_CH1; ++ return; ++ } ++ } ++ ++ WARN(1, "PHY not found for PORT %c", port_name(port)); ++ *phy = DPIO_PHY0; ++ *ch = DPIO_CH0; ++} ++ ++void bxt_ddi_phy_set_signal_level(struct drm_i915_private *dev_priv, ++ enum port port, u32 margin, u32 scale, ++ u32 enable, u32 deemphasis) ++{ ++ u32 val; ++ enum dpio_phy phy; ++ enum dpio_channel ch; ++ ++ bxt_port_to_phy_channel(dev_priv, port, &phy, &ch); ++ ++ /* ++ * While we write to the group register to program all lanes at once we ++ * can read only lane registers and we pick lanes 0/1 for that. ++ */ ++ val = I915_READ(BXT_PORT_PCS_DW10_LN01(phy, ch)); ++ val &= ~(TX2_SWING_CALC_INIT | TX1_SWING_CALC_INIT); ++ I915_WRITE(BXT_PORT_PCS_DW10_GRP(phy, ch), val); ++ ++ val = I915_READ(BXT_PORT_TX_DW2_LN0(phy, ch)); ++ val &= ~(MARGIN_000 | UNIQ_TRANS_SCALE); ++ val |= margin << MARGIN_000_SHIFT | scale << UNIQ_TRANS_SCALE_SHIFT; ++ I915_WRITE(BXT_PORT_TX_DW2_GRP(phy, ch), val); ++ ++ val = I915_READ(BXT_PORT_TX_DW3_LN0(phy, ch)); ++ val &= ~SCALE_DCOMP_METHOD; ++ if (enable) ++ val |= SCALE_DCOMP_METHOD; ++ ++ if ((val & UNIQUE_TRANGE_EN_METHOD) && !(val & SCALE_DCOMP_METHOD)) ++ DRM_ERROR("Disabled scaling while ouniqetrangenmethod was set"); ++ ++ I915_WRITE(BXT_PORT_TX_DW3_GRP(phy, ch), val); ++ ++ val = I915_READ(BXT_PORT_TX_DW4_LN0(phy, ch)); ++ val &= ~DE_EMPHASIS; ++ val |= deemphasis << DEEMPH_SHIFT; ++ I915_WRITE(BXT_PORT_TX_DW4_GRP(phy, ch), val); ++ ++ val = I915_READ(BXT_PORT_PCS_DW10_LN01(phy, ch)); ++ val |= TX2_SWING_CALC_INIT | TX1_SWING_CALC_INIT; ++ I915_WRITE(BXT_PORT_PCS_DW10_GRP(phy, ch), val); ++} ++ ++bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv, ++ enum dpio_phy phy) ++{ ++ const struct bxt_ddi_phy_info *phy_info; ++ ++ phy_info = bxt_get_phy_info(dev_priv, phy); ++ ++ if (!(I915_READ(BXT_P_CR_GT_DISP_PWRON) & phy_info->pwron_mask)) ++ return false; ++ ++ if ((I915_READ(BXT_PORT_CL1CM_DW0(phy)) & ++ (PHY_POWER_GOOD | PHY_RESERVED)) != PHY_POWER_GOOD) { ++ DRM_DEBUG_DRIVER("DDI PHY %d powered, but power hasn't settled\n", ++ phy); ++ ++ return false; ++ } ++ ++ if (!(I915_READ(BXT_PHY_CTL_FAMILY(phy)) & COMMON_RESET_DIS)) { ++ DRM_DEBUG_DRIVER("DDI PHY %d powered, but still in reset\n", ++ phy); ++ ++ return false; ++ } ++ ++ return true; ++} ++ ++static u32 bxt_get_grc(struct drm_i915_private *dev_priv, enum dpio_phy phy) ++{ ++ u32 val = I915_READ(BXT_PORT_REF_DW6(phy)); ++ ++ return (val & GRC_CODE_MASK) >> GRC_CODE_SHIFT; ++} ++ ++static void bxt_phy_wait_grc_done(struct drm_i915_private *dev_priv, ++ enum dpio_phy phy) ++{ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ BXT_PORT_REF_DW3(phy), ++ GRC_DONE, GRC_DONE, ++ 10)) ++ DRM_ERROR("timeout waiting for PHY%d GRC\n", phy); ++} ++ ++static void _bxt_ddi_phy_init(struct drm_i915_private *dev_priv, ++ enum dpio_phy phy) ++{ ++ const struct bxt_ddi_phy_info *phy_info; ++ u32 val; ++ ++ phy_info = bxt_get_phy_info(dev_priv, phy); ++ ++ if (bxt_ddi_phy_is_enabled(dev_priv, phy)) { ++ /* Still read out the GRC value for state verification */ ++ if (phy_info->rcomp_phy != -1) ++ dev_priv->bxt_phy_grc = bxt_get_grc(dev_priv, phy); ++ ++ if (bxt_ddi_phy_verify_state(dev_priv, phy)) { ++ DRM_DEBUG_DRIVER("DDI PHY %d already enabled, " ++ "won't reprogram it\n", phy); ++ return; ++ } ++ ++ DRM_DEBUG_DRIVER("DDI PHY %d enabled with invalid state, " ++ "force reprogramming it\n", phy); ++ } ++ ++ val = I915_READ(BXT_P_CR_GT_DISP_PWRON); ++ val |= phy_info->pwron_mask; ++ I915_WRITE(BXT_P_CR_GT_DISP_PWRON, val); ++ ++ /* ++ * The PHY registers start out inaccessible and respond to reads with ++ * all 1s. Eventually they become accessible as they power up, then ++ * the reserved bit will give the default 0. Poll on the reserved bit ++ * becoming 0 to find when the PHY is accessible. ++ * The flag should get set in 100us according to the HW team, but ++ * use 1ms due to occasional timeouts observed with that. ++ */ ++ if (intel_wait_for_register_fw(&dev_priv->uncore, ++ BXT_PORT_CL1CM_DW0(phy), ++ PHY_RESERVED | PHY_POWER_GOOD, ++ PHY_POWER_GOOD, ++ 1)) ++ DRM_ERROR("timeout during PHY%d power on\n", phy); ++ ++ /* Program PLL Rcomp code offset */ ++ val = I915_READ(BXT_PORT_CL1CM_DW9(phy)); ++ val &= ~IREF0RC_OFFSET_MASK; ++ val |= 0xE4 << IREF0RC_OFFSET_SHIFT; ++ I915_WRITE(BXT_PORT_CL1CM_DW9(phy), val); ++ ++ val = I915_READ(BXT_PORT_CL1CM_DW10(phy)); ++ val &= ~IREF1RC_OFFSET_MASK; ++ val |= 0xE4 << IREF1RC_OFFSET_SHIFT; ++ I915_WRITE(BXT_PORT_CL1CM_DW10(phy), val); ++ ++ /* Program power gating */ ++ val = I915_READ(BXT_PORT_CL1CM_DW28(phy)); ++ val |= OCL1_POWER_DOWN_EN | DW28_OLDO_DYN_PWR_DOWN_EN | ++ SUS_CLK_CONFIG; ++ I915_WRITE(BXT_PORT_CL1CM_DW28(phy), val); ++ ++ if (phy_info->dual_channel) { ++ val = I915_READ(BXT_PORT_CL2CM_DW6(phy)); ++ val |= DW6_OLDO_DYN_PWR_DOWN_EN; ++ I915_WRITE(BXT_PORT_CL2CM_DW6(phy), val); ++ } ++ ++ if (phy_info->rcomp_phy != -1) { ++ u32 grc_code; ++ ++ bxt_phy_wait_grc_done(dev_priv, phy_info->rcomp_phy); ++ ++ /* ++ * PHY0 isn't connected to an RCOMP resistor so copy over ++ * the corresponding calibrated value from PHY1, and disable ++ * the automatic calibration on PHY0. ++ */ ++ val = dev_priv->bxt_phy_grc = bxt_get_grc(dev_priv, ++ phy_info->rcomp_phy); ++ grc_code = val << GRC_CODE_FAST_SHIFT | ++ val << GRC_CODE_SLOW_SHIFT | ++ val; ++ I915_WRITE(BXT_PORT_REF_DW6(phy), grc_code); ++ ++ val = I915_READ(BXT_PORT_REF_DW8(phy)); ++ val |= GRC_DIS | GRC_RDY_OVRD; ++ I915_WRITE(BXT_PORT_REF_DW8(phy), val); ++ } ++ ++ if (phy_info->reset_delay) ++ udelay(phy_info->reset_delay); ++ ++ val = I915_READ(BXT_PHY_CTL_FAMILY(phy)); ++ val |= COMMON_RESET_DIS; ++ I915_WRITE(BXT_PHY_CTL_FAMILY(phy), val); ++} ++ ++void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy) ++{ ++ const struct bxt_ddi_phy_info *phy_info; ++ u32 val; ++ ++ phy_info = bxt_get_phy_info(dev_priv, phy); ++ ++ val = I915_READ(BXT_PHY_CTL_FAMILY(phy)); ++ val &= ~COMMON_RESET_DIS; ++ I915_WRITE(BXT_PHY_CTL_FAMILY(phy), val); ++ ++ val = I915_READ(BXT_P_CR_GT_DISP_PWRON); ++ val &= ~phy_info->pwron_mask; ++ I915_WRITE(BXT_P_CR_GT_DISP_PWRON, val); ++} ++ ++void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy) ++{ ++ const struct bxt_ddi_phy_info *phy_info = ++ bxt_get_phy_info(dev_priv, phy); ++ enum dpio_phy rcomp_phy = phy_info->rcomp_phy; ++ bool was_enabled; ++ ++ lockdep_assert_held(&dev_priv->power_domains.lock); ++ ++ was_enabled = true; ++ if (rcomp_phy != -1) ++ was_enabled = bxt_ddi_phy_is_enabled(dev_priv, rcomp_phy); ++ ++ /* ++ * We need to copy the GRC calibration value from rcomp_phy, ++ * so make sure it's powered up. ++ */ ++ if (!was_enabled) ++ _bxt_ddi_phy_init(dev_priv, rcomp_phy); ++ ++ _bxt_ddi_phy_init(dev_priv, phy); ++ ++ if (!was_enabled) ++ bxt_ddi_phy_uninit(dev_priv, rcomp_phy); ++} ++ ++static bool __printf(6, 7) ++__phy_reg_verify_state(struct drm_i915_private *dev_priv, enum dpio_phy phy, ++ i915_reg_t reg, u32 mask, u32 expected, ++ const char *reg_fmt, ...) ++{ ++ struct va_format vaf; ++ va_list args; ++ u32 val; ++ ++ val = I915_READ(reg); ++ if ((val & mask) == expected) ++ return true; ++ ++ va_start(args, reg_fmt); ++ vaf.fmt = reg_fmt; ++ vaf.va = &args; ++ ++ DRM_DEBUG_DRIVER("DDI PHY %d reg %pV [%08x] state mismatch: " ++ "current %08x, expected %08x (mask %08x)\n", ++ phy, &vaf, reg.reg, val, (val & ~mask) | expected, ++ mask); ++ ++ va_end(args); ++ ++ return false; ++} ++ ++bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv, ++ enum dpio_phy phy) ++{ ++ const struct bxt_ddi_phy_info *phy_info; ++ u32 mask; ++ bool ok; ++ ++ phy_info = bxt_get_phy_info(dev_priv, phy); ++ ++#define _CHK(reg, mask, exp, fmt, ...) \ ++ __phy_reg_verify_state(dev_priv, phy, reg, mask, exp, fmt, \ ++ ## __VA_ARGS__) ++ ++ if (!bxt_ddi_phy_is_enabled(dev_priv, phy)) ++ return false; ++ ++ ok = true; ++ ++ /* PLL Rcomp code offset */ ++ ok &= _CHK(BXT_PORT_CL1CM_DW9(phy), ++ IREF0RC_OFFSET_MASK, 0xe4 << IREF0RC_OFFSET_SHIFT, ++ "BXT_PORT_CL1CM_DW9(%d)", phy); ++ ok &= _CHK(BXT_PORT_CL1CM_DW10(phy), ++ IREF1RC_OFFSET_MASK, 0xe4 << IREF1RC_OFFSET_SHIFT, ++ "BXT_PORT_CL1CM_DW10(%d)", phy); ++ ++ /* Power gating */ ++ mask = OCL1_POWER_DOWN_EN | DW28_OLDO_DYN_PWR_DOWN_EN | SUS_CLK_CONFIG; ++ ok &= _CHK(BXT_PORT_CL1CM_DW28(phy), mask, mask, ++ "BXT_PORT_CL1CM_DW28(%d)", phy); ++ ++ if (phy_info->dual_channel) ++ ok &= _CHK(BXT_PORT_CL2CM_DW6(phy), ++ DW6_OLDO_DYN_PWR_DOWN_EN, DW6_OLDO_DYN_PWR_DOWN_EN, ++ "BXT_PORT_CL2CM_DW6(%d)", phy); ++ ++ if (phy_info->rcomp_phy != -1) { ++ u32 grc_code = dev_priv->bxt_phy_grc; ++ ++ grc_code = grc_code << GRC_CODE_FAST_SHIFT | ++ grc_code << GRC_CODE_SLOW_SHIFT | ++ grc_code; ++ mask = GRC_CODE_FAST_MASK | GRC_CODE_SLOW_MASK | ++ GRC_CODE_NOM_MASK; ++ ok &= _CHK(BXT_PORT_REF_DW6(phy), mask, grc_code, ++ "BXT_PORT_REF_DW6(%d)", phy); ++ ++ mask = GRC_DIS | GRC_RDY_OVRD; ++ ok &= _CHK(BXT_PORT_REF_DW8(phy), mask, mask, ++ "BXT_PORT_REF_DW8(%d)", phy); ++ } ++ ++ return ok; ++#undef _CHK ++} ++ ++u8 ++bxt_ddi_phy_calc_lane_lat_optim_mask(u8 lane_count) ++{ ++ switch (lane_count) { ++ case 1: ++ return 0; ++ case 2: ++ return BIT(2) | BIT(0); ++ case 4: ++ return BIT(3) | BIT(2) | BIT(0); ++ default: ++ MISSING_CASE(lane_count); ++ ++ return 0; ++ } ++} ++ ++void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder, ++ u8 lane_lat_optim_mask) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ enum port port = encoder->port; ++ enum dpio_phy phy; ++ enum dpio_channel ch; ++ int lane; ++ ++ bxt_port_to_phy_channel(dev_priv, port, &phy, &ch); ++ ++ for (lane = 0; lane < 4; lane++) { ++ u32 val = I915_READ(BXT_PORT_TX_DW14_LN(phy, ch, lane)); ++ ++ /* ++ * Note that on CHV this flag is called UPAR, but has ++ * the same function. ++ */ ++ val &= ~LATENCY_OPTIM; ++ if (lane_lat_optim_mask & BIT(lane)) ++ val |= LATENCY_OPTIM; ++ ++ I915_WRITE(BXT_PORT_TX_DW14_LN(phy, ch, lane), val); ++ } ++} ++ ++u8 ++bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ enum port port = encoder->port; ++ enum dpio_phy phy; ++ enum dpio_channel ch; ++ int lane; ++ u8 mask; ++ ++ bxt_port_to_phy_channel(dev_priv, port, &phy, &ch); ++ ++ mask = 0; ++ for (lane = 0; lane < 4; lane++) { ++ u32 val = I915_READ(BXT_PORT_TX_DW14_LN(phy, ch, lane)); ++ ++ if (val & LATENCY_OPTIM) ++ mask |= BIT(lane); ++ } ++ ++ return mask; ++} ++ ++ ++void chv_set_phy_signal_level(struct intel_encoder *encoder, ++ u32 deemph_reg_value, u32 margin_reg_value, ++ bool uniq_trans_scale) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); ++ struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); ++ enum dpio_channel ch = vlv_dport_to_channel(dport); ++ enum pipe pipe = intel_crtc->pipe; ++ u32 val; ++ int i; ++ ++ mutex_lock(&dev_priv->sb_lock); ++ ++ /* Clear calc init */ ++ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch)); ++ val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3); ++ val &= ~(DPIO_PCS_TX1DEEMP_MASK | DPIO_PCS_TX2DEEMP_MASK); ++ val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5; ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val); ++ ++ if (intel_crtc->config->lane_count > 2) { ++ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch)); ++ val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3); ++ val &= ~(DPIO_PCS_TX1DEEMP_MASK | DPIO_PCS_TX2DEEMP_MASK); ++ val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5; ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val); ++ } ++ ++ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW9(ch)); ++ val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK); ++ val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000; ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW9(ch), val); ++ ++ if (intel_crtc->config->lane_count > 2) { ++ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW9(ch)); ++ val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK); ++ val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000; ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW9(ch), val); ++ } ++ ++ /* Program swing deemph */ ++ for (i = 0; i < intel_crtc->config->lane_count; i++) { ++ val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW4(ch, i)); ++ val &= ~DPIO_SWING_DEEMPH9P5_MASK; ++ val |= deemph_reg_value << DPIO_SWING_DEEMPH9P5_SHIFT; ++ vlv_dpio_write(dev_priv, pipe, CHV_TX_DW4(ch, i), val); ++ } ++ ++ /* Program swing margin */ ++ for (i = 0; i < intel_crtc->config->lane_count; i++) { ++ val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW2(ch, i)); ++ ++ val &= ~DPIO_SWING_MARGIN000_MASK; ++ val |= margin_reg_value << DPIO_SWING_MARGIN000_SHIFT; ++ ++ /* ++ * Supposedly this value shouldn't matter when unique transition ++ * scale is disabled, but in fact it does matter. Let's just ++ * always program the same value and hope it's OK. ++ */ ++ val &= ~(0xff << DPIO_UNIQ_TRANS_SCALE_SHIFT); ++ val |= 0x9a << DPIO_UNIQ_TRANS_SCALE_SHIFT; ++ ++ vlv_dpio_write(dev_priv, pipe, CHV_TX_DW2(ch, i), val); ++ } ++ ++ /* ++ * The document said it needs to set bit 27 for ch0 and bit 26 ++ * for ch1. Might be a typo in the doc. ++ * For now, for this unique transition scale selection, set bit ++ * 27 for ch0 and ch1. ++ */ ++ for (i = 0; i < intel_crtc->config->lane_count; i++) { ++ val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW3(ch, i)); ++ if (uniq_trans_scale) ++ val |= DPIO_TX_UNIQ_TRANS_SCALE_EN; ++ else ++ val &= ~DPIO_TX_UNIQ_TRANS_SCALE_EN; ++ vlv_dpio_write(dev_priv, pipe, CHV_TX_DW3(ch, i), val); ++ } ++ ++ /* Start swing calculation */ ++ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch)); ++ val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3; ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val); ++ ++ if (intel_crtc->config->lane_count > 2) { ++ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch)); ++ val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3; ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val); ++ } ++ ++ mutex_unlock(&dev_priv->sb_lock); ++ ++} ++ ++void chv_data_lane_soft_reset(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state, ++ bool reset) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base)); ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ enum pipe pipe = crtc->pipe; ++ u32 val; ++ ++ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch)); ++ if (reset) ++ val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); ++ else ++ val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET; ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val); ++ ++ if (crtc_state->lane_count > 2) { ++ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch)); ++ if (reset) ++ val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); ++ else ++ val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET; ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val); ++ } ++ ++ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW1(ch)); ++ val |= CHV_PCS_REQ_SOFTRESET_EN; ++ if (reset) ++ val &= ~DPIO_PCS_CLK_SOFT_RESET; ++ else ++ val |= DPIO_PCS_CLK_SOFT_RESET; ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW1(ch), val); ++ ++ if (crtc_state->lane_count > 2) { ++ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch)); ++ val |= CHV_PCS_REQ_SOFTRESET_EN; ++ if (reset) ++ val &= ~DPIO_PCS_CLK_SOFT_RESET; ++ else ++ val |= DPIO_PCS_CLK_SOFT_RESET; ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW1(ch), val); ++ } ++} ++ ++void chv_phy_pre_pll_enable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ enum dpio_channel ch = vlv_dport_to_channel(dport); ++ enum pipe pipe = crtc->pipe; ++ unsigned int lane_mask = ++ intel_dp_unused_lane_mask(crtc_state->lane_count); ++ u32 val; ++ ++ /* ++ * Must trick the second common lane into life. ++ * Otherwise we can't even access the PLL. ++ */ ++ if (ch == DPIO_CH0 && pipe == PIPE_B) ++ dport->release_cl2_override = ++ !chv_phy_powergate_ch(dev_priv, DPIO_PHY0, DPIO_CH1, true); ++ ++ chv_phy_powergate_lanes(encoder, true, lane_mask); ++ ++ mutex_lock(&dev_priv->sb_lock); ++ ++ /* Assert data lane reset */ ++ chv_data_lane_soft_reset(encoder, crtc_state, true); ++ ++ /* program left/right clock distribution */ ++ if (pipe != PIPE_B) { ++ val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0); ++ val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK); ++ if (ch == DPIO_CH0) ++ val |= CHV_BUFLEFTENA1_FORCE; ++ if (ch == DPIO_CH1) ++ val |= CHV_BUFRIGHTENA1_FORCE; ++ vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val); ++ } else { ++ val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1); ++ val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK); ++ if (ch == DPIO_CH0) ++ val |= CHV_BUFLEFTENA2_FORCE; ++ if (ch == DPIO_CH1) ++ val |= CHV_BUFRIGHTENA2_FORCE; ++ vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val); ++ } ++ ++ /* program clock channel usage */ ++ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW8(ch)); ++ val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE; ++ if (pipe != PIPE_B) ++ val &= ~CHV_PCS_USEDCLKCHANNEL; ++ else ++ val |= CHV_PCS_USEDCLKCHANNEL; ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW8(ch), val); ++ ++ if (crtc_state->lane_count > 2) { ++ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW8(ch)); ++ val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE; ++ if (pipe != PIPE_B) ++ val &= ~CHV_PCS_USEDCLKCHANNEL; ++ else ++ val |= CHV_PCS_USEDCLKCHANNEL; ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW8(ch), val); ++ } ++ ++ /* ++ * This a a bit weird since generally CL ++ * matches the pipe, but here we need to ++ * pick the CL based on the port. ++ */ ++ val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW19(ch)); ++ if (pipe != PIPE_B) ++ val &= ~CHV_CMN_USEDCLKCHANNEL; ++ else ++ val |= CHV_CMN_USEDCLKCHANNEL; ++ vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW19(ch), val); ++ ++ mutex_unlock(&dev_priv->sb_lock); ++} ++ ++void chv_phy_pre_encoder_enable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ struct intel_digital_port *dport = dp_to_dig_port(intel_dp); ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ enum dpio_channel ch = vlv_dport_to_channel(dport); ++ enum pipe pipe = crtc->pipe; ++ int data, i, stagger; ++ u32 val; ++ ++ mutex_lock(&dev_priv->sb_lock); ++ ++ /* allow hardware to manage TX FIFO reset source */ ++ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW11(ch)); ++ val &= ~DPIO_LANEDESKEW_STRAP_OVRD; ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val); ++ ++ if (crtc_state->lane_count > 2) { ++ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch)); ++ val &= ~DPIO_LANEDESKEW_STRAP_OVRD; ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val); ++ } ++ ++ /* Program Tx lane latency optimal setting*/ ++ for (i = 0; i < crtc_state->lane_count; i++) { ++ /* Set the upar bit */ ++ if (crtc_state->lane_count == 1) ++ data = 0x0; ++ else ++ data = (i == 1) ? 0x0 : 0x1; ++ vlv_dpio_write(dev_priv, pipe, CHV_TX_DW14(ch, i), ++ data << DPIO_UPAR_SHIFT); ++ } ++ ++ /* Data lane stagger programming */ ++ if (crtc_state->port_clock > 270000) ++ stagger = 0x18; ++ else if (crtc_state->port_clock > 135000) ++ stagger = 0xd; ++ else if (crtc_state->port_clock > 67500) ++ stagger = 0x7; ++ else if (crtc_state->port_clock > 33750) ++ stagger = 0x4; ++ else ++ stagger = 0x2; ++ ++ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW11(ch)); ++ val |= DPIO_TX2_STAGGER_MASK(0x1f); ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val); ++ ++ if (crtc_state->lane_count > 2) { ++ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch)); ++ val |= DPIO_TX2_STAGGER_MASK(0x1f); ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val); ++ } ++ ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW12(ch), ++ DPIO_LANESTAGGER_STRAP(stagger) | ++ DPIO_LANESTAGGER_STRAP_OVRD | ++ DPIO_TX1_STAGGER_MASK(0x1f) | ++ DPIO_TX1_STAGGER_MULT(6) | ++ DPIO_TX2_STAGGER_MULT(0)); ++ ++ if (crtc_state->lane_count > 2) { ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW12(ch), ++ DPIO_LANESTAGGER_STRAP(stagger) | ++ DPIO_LANESTAGGER_STRAP_OVRD | ++ DPIO_TX1_STAGGER_MASK(0x1f) | ++ DPIO_TX1_STAGGER_MULT(7) | ++ DPIO_TX2_STAGGER_MULT(5)); ++ } ++ ++ /* Deassert data lane reset */ ++ chv_data_lane_soft_reset(encoder, crtc_state, false); ++ ++ mutex_unlock(&dev_priv->sb_lock); ++} ++ ++void chv_phy_release_cl2_override(struct intel_encoder *encoder) ++{ ++ struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ ++ if (dport->release_cl2_override) { ++ chv_phy_powergate_ch(dev_priv, DPIO_PHY0, DPIO_CH1, false); ++ dport->release_cl2_override = false; ++ } ++} ++ ++void chv_phy_post_pll_disable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ enum pipe pipe = to_intel_crtc(old_crtc_state->base.crtc)->pipe; ++ u32 val; ++ ++ mutex_lock(&dev_priv->sb_lock); ++ ++ /* disable left/right clock distribution */ ++ if (pipe != PIPE_B) { ++ val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0); ++ val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK); ++ vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val); ++ } else { ++ val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1); ++ val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK); ++ vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val); ++ } ++ ++ mutex_unlock(&dev_priv->sb_lock); ++ ++ /* ++ * Leave the power down bit cleared for at least one ++ * lane so that chv_powergate_phy_ch() will power ++ * on something when the channel is otherwise unused. ++ * When the port is off and the override is removed ++ * the lanes power down anyway, so otherwise it doesn't ++ * really matter what the state of power down bits is ++ * after this. ++ */ ++ chv_phy_powergate_lanes(encoder, false, 0x0); ++} ++ ++void vlv_set_phy_signal_level(struct intel_encoder *encoder, ++ u32 demph_reg_value, u32 preemph_reg_value, ++ u32 uniqtranscale_reg_value, u32 tx3_demph) ++{ ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); ++ struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); ++ enum dpio_channel port = vlv_dport_to_channel(dport); ++ enum pipe pipe = intel_crtc->pipe; ++ ++ mutex_lock(&dev_priv->sb_lock); ++ vlv_dpio_write(dev_priv, pipe, VLV_TX_DW5(port), 0x00000000); ++ vlv_dpio_write(dev_priv, pipe, VLV_TX_DW4(port), demph_reg_value); ++ vlv_dpio_write(dev_priv, pipe, VLV_TX_DW2(port), ++ uniqtranscale_reg_value); ++ vlv_dpio_write(dev_priv, pipe, VLV_TX_DW3(port), 0x0C782040); ++ ++ if (tx3_demph) ++ vlv_dpio_write(dev_priv, pipe, VLV_TX3_DW4(port), tx3_demph); ++ ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW11(port), 0x00030000); ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW9(port), preemph_reg_value); ++ vlv_dpio_write(dev_priv, pipe, VLV_TX_DW5(port), DPIO_TX_OCALINIT_EN); ++ mutex_unlock(&dev_priv->sb_lock); ++} ++ ++void vlv_phy_pre_pll_enable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ enum dpio_channel port = vlv_dport_to_channel(dport); ++ enum pipe pipe = crtc->pipe; ++ ++ /* Program Tx lane resets to default */ ++ mutex_lock(&dev_priv->sb_lock); ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW0(port), ++ DPIO_PCS_TX_LANE2_RESET | ++ DPIO_PCS_TX_LANE1_RESET); ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW1(port), ++ DPIO_PCS_CLK_CRI_RXEB_EIOS_EN | ++ DPIO_PCS_CLK_CRI_RXDIGFILTSG_EN | ++ (1<sb_lock); ++} ++ ++void vlv_phy_pre_encoder_enable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); ++ struct intel_digital_port *dport = dp_to_dig_port(intel_dp); ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ enum dpio_channel port = vlv_dport_to_channel(dport); ++ enum pipe pipe = crtc->pipe; ++ u32 val; ++ ++ mutex_lock(&dev_priv->sb_lock); ++ ++ /* Enable clock channels for this port */ ++ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW8(port)); ++ val = 0; ++ if (pipe) ++ val |= (1<<21); ++ else ++ val &= ~(1<<21); ++ val |= 0x001000c4; ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW8(port), val); ++ ++ /* Program lane clock */ ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW14(port), 0x00760018); ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW23(port), 0x00400888); ++ ++ mutex_unlock(&dev_priv->sb_lock); ++} ++ ++void vlv_phy_reset_lanes(struct intel_encoder *encoder, ++ const struct intel_crtc_state *old_crtc_state) ++{ ++ struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); ++ struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); ++ struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc); ++ enum dpio_channel port = vlv_dport_to_channel(dport); ++ enum pipe pipe = crtc->pipe; ++ ++ mutex_lock(&dev_priv->sb_lock); ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW0(port), 0x00000000); ++ vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW1(port), 0x00e00060); ++ mutex_unlock(&dev_priv->sb_lock); ++} +diff --git a/drivers/gpu/drm/i915_legacy/intel_dpll_mgr.c b/drivers/gpu/drm/i915_legacy/intel_dpll_mgr.c +new file mode 100644 +index 000000000000..e01c057ce50b +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_dpll_mgr.c +@@ -0,0 +1,3382 @@ ++/* ++ * Copyright © 2006-2016 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER ++ * DEALINGS IN THE SOFTWARE. ++ */ ++ ++#include "intel_drv.h" ++ ++/** ++ * DOC: Display PLLs ++ * ++ * Display PLLs used for driving outputs vary by platform. While some have ++ * per-pipe or per-encoder dedicated PLLs, others allow the use of any PLL ++ * from a pool. In the latter scenario, it is possible that multiple pipes ++ * share a PLL if their configurations match. ++ * ++ * This file provides an abstraction over display PLLs. The function ++ * intel_shared_dpll_init() initializes the PLLs for the given platform. The ++ * users of a PLL are tracked and that tracking is integrated with the atomic ++ * modest interface. During an atomic operation, a PLL can be requested for a ++ * given CRTC and encoder configuration by calling intel_get_shared_dpll() and ++ * a previously used PLL can be released with intel_release_shared_dpll(). ++ * Changes to the users are first staged in the atomic state, and then made ++ * effective by calling intel_shared_dpll_swap_state() during the atomic ++ * commit phase. ++ */ ++ ++static void ++intel_atomic_duplicate_dpll_state(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll_state *shared_dpll) ++{ ++ enum intel_dpll_id i; ++ ++ /* Copy shared dpll state */ ++ for (i = 0; i < dev_priv->num_shared_dpll; i++) { ++ struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i]; ++ ++ shared_dpll[i] = pll->state; ++ } ++} ++ ++static struct intel_shared_dpll_state * ++intel_atomic_get_shared_dpll_state(struct drm_atomic_state *s) ++{ ++ struct intel_atomic_state *state = to_intel_atomic_state(s); ++ ++ WARN_ON(!drm_modeset_is_locked(&s->dev->mode_config.connection_mutex)); ++ ++ if (!state->dpll_set) { ++ state->dpll_set = true; ++ ++ intel_atomic_duplicate_dpll_state(to_i915(s->dev), ++ state->shared_dpll); ++ } ++ ++ return state->shared_dpll; ++} ++ ++/** ++ * intel_get_shared_dpll_by_id - get a DPLL given its id ++ * @dev_priv: i915 device instance ++ * @id: pll id ++ * ++ * Returns: ++ * A pointer to the DPLL with @id ++ */ ++struct intel_shared_dpll * ++intel_get_shared_dpll_by_id(struct drm_i915_private *dev_priv, ++ enum intel_dpll_id id) ++{ ++ return &dev_priv->shared_dplls[id]; ++} ++ ++/** ++ * intel_get_shared_dpll_id - get the id of a DPLL ++ * @dev_priv: i915 device instance ++ * @pll: the DPLL ++ * ++ * Returns: ++ * The id of @pll ++ */ ++enum intel_dpll_id ++intel_get_shared_dpll_id(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++ if (WARN_ON(pll < dev_priv->shared_dplls|| ++ pll > &dev_priv->shared_dplls[dev_priv->num_shared_dpll])) ++ return -1; ++ ++ return (enum intel_dpll_id) (pll - dev_priv->shared_dplls); ++} ++ ++/* For ILK+ */ ++void assert_shared_dpll(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll, ++ bool state) ++{ ++ bool cur_state; ++ struct intel_dpll_hw_state hw_state; ++ ++ if (WARN(!pll, "asserting DPLL %s with no DPLL\n", onoff(state))) ++ return; ++ ++ cur_state = pll->info->funcs->get_hw_state(dev_priv, pll, &hw_state); ++ I915_STATE_WARN(cur_state != state, ++ "%s assertion failure (expected %s, current %s)\n", ++ pll->info->name, onoff(state), onoff(cur_state)); ++} ++ ++/** ++ * intel_prepare_shared_dpll - call a dpll's prepare hook ++ * @crtc_state: CRTC, and its state, which has a shared dpll ++ * ++ * This calls the PLL's prepare hook if it has one and if the PLL is not ++ * already enabled. The prepare hook is platform specific. ++ */ ++void intel_prepare_shared_dpll(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ struct intel_shared_dpll *pll = crtc_state->shared_dpll; ++ ++ if (WARN_ON(pll == NULL)) ++ return; ++ ++ mutex_lock(&dev_priv->dpll_lock); ++ WARN_ON(!pll->state.crtc_mask); ++ if (!pll->active_mask) { ++ DRM_DEBUG_DRIVER("setting up %s\n", pll->info->name); ++ WARN_ON(pll->on); ++ assert_shared_dpll_disabled(dev_priv, pll); ++ ++ pll->info->funcs->prepare(dev_priv, pll); ++ } ++ mutex_unlock(&dev_priv->dpll_lock); ++} ++ ++/** ++ * intel_enable_shared_dpll - enable a CRTC's shared DPLL ++ * @crtc_state: CRTC, and its state, which has a shared DPLL ++ * ++ * Enable the shared DPLL used by @crtc. ++ */ ++void intel_enable_shared_dpll(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ struct intel_shared_dpll *pll = crtc_state->shared_dpll; ++ unsigned int crtc_mask = drm_crtc_mask(&crtc->base); ++ unsigned int old_mask; ++ ++ if (WARN_ON(pll == NULL)) ++ return; ++ ++ mutex_lock(&dev_priv->dpll_lock); ++ old_mask = pll->active_mask; ++ ++ if (WARN_ON(!(pll->state.crtc_mask & crtc_mask)) || ++ WARN_ON(pll->active_mask & crtc_mask)) ++ goto out; ++ ++ pll->active_mask |= crtc_mask; ++ ++ DRM_DEBUG_KMS("enable %s (active %x, on? %d) for crtc %d\n", ++ pll->info->name, pll->active_mask, pll->on, ++ crtc->base.base.id); ++ ++ if (old_mask) { ++ WARN_ON(!pll->on); ++ assert_shared_dpll_enabled(dev_priv, pll); ++ goto out; ++ } ++ WARN_ON(pll->on); ++ ++ DRM_DEBUG_KMS("enabling %s\n", pll->info->name); ++ pll->info->funcs->enable(dev_priv, pll); ++ pll->on = true; ++ ++out: ++ mutex_unlock(&dev_priv->dpll_lock); ++} ++ ++/** ++ * intel_disable_shared_dpll - disable a CRTC's shared DPLL ++ * @crtc_state: CRTC, and its state, which has a shared DPLL ++ * ++ * Disable the shared DPLL used by @crtc. ++ */ ++void intel_disable_shared_dpll(const struct intel_crtc_state *crtc_state) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ struct intel_shared_dpll *pll = crtc_state->shared_dpll; ++ unsigned int crtc_mask = drm_crtc_mask(&crtc->base); ++ ++ /* PCH only available on ILK+ */ ++ if (INTEL_GEN(dev_priv) < 5) ++ return; ++ ++ if (pll == NULL) ++ return; ++ ++ mutex_lock(&dev_priv->dpll_lock); ++ if (WARN_ON(!(pll->active_mask & crtc_mask))) ++ goto out; ++ ++ DRM_DEBUG_KMS("disable %s (active %x, on? %d) for crtc %d\n", ++ pll->info->name, pll->active_mask, pll->on, ++ crtc->base.base.id); ++ ++ assert_shared_dpll_enabled(dev_priv, pll); ++ WARN_ON(!pll->on); ++ ++ pll->active_mask &= ~crtc_mask; ++ if (pll->active_mask) ++ goto out; ++ ++ DRM_DEBUG_KMS("disabling %s\n", pll->info->name); ++ pll->info->funcs->disable(dev_priv, pll); ++ pll->on = false; ++ ++out: ++ mutex_unlock(&dev_priv->dpll_lock); ++} ++ ++static struct intel_shared_dpll * ++intel_find_shared_dpll(struct intel_crtc_state *crtc_state, ++ enum intel_dpll_id range_min, ++ enum intel_dpll_id range_max) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ struct intel_shared_dpll *pll, *unused_pll = NULL; ++ struct intel_shared_dpll_state *shared_dpll; ++ enum intel_dpll_id i; ++ ++ shared_dpll = intel_atomic_get_shared_dpll_state(crtc_state->base.state); ++ ++ for (i = range_min; i <= range_max; i++) { ++ pll = &dev_priv->shared_dplls[i]; ++ ++ /* Only want to check enabled timings first */ ++ if (shared_dpll[i].crtc_mask == 0) { ++ if (!unused_pll) ++ unused_pll = pll; ++ continue; ++ } ++ ++ if (memcmp(&crtc_state->dpll_hw_state, ++ &shared_dpll[i].hw_state, ++ sizeof(crtc_state->dpll_hw_state)) == 0) { ++ DRM_DEBUG_KMS("[CRTC:%d:%s] sharing existing %s (crtc mask 0x%08x, active %x)\n", ++ crtc->base.base.id, crtc->base.name, ++ pll->info->name, ++ shared_dpll[i].crtc_mask, ++ pll->active_mask); ++ return pll; ++ } ++ } ++ ++ /* Ok no matching timings, maybe there's a free one? */ ++ if (unused_pll) { ++ DRM_DEBUG_KMS("[CRTC:%d:%s] allocated %s\n", ++ crtc->base.base.id, crtc->base.name, ++ unused_pll->info->name); ++ return unused_pll; ++ } ++ ++ return NULL; ++} ++ ++static void ++intel_reference_shared_dpll(struct intel_shared_dpll *pll, ++ struct intel_crtc_state *crtc_state) ++{ ++ struct intel_shared_dpll_state *shared_dpll; ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ const enum intel_dpll_id id = pll->info->id; ++ ++ shared_dpll = intel_atomic_get_shared_dpll_state(crtc_state->base.state); ++ ++ if (shared_dpll[id].crtc_mask == 0) ++ shared_dpll[id].hw_state = ++ crtc_state->dpll_hw_state; ++ ++ crtc_state->shared_dpll = pll; ++ DRM_DEBUG_DRIVER("using %s for pipe %c\n", pll->info->name, ++ pipe_name(crtc->pipe)); ++ ++ shared_dpll[id].crtc_mask |= 1 << crtc->pipe; ++} ++ ++/** ++ * intel_shared_dpll_swap_state - make atomic DPLL configuration effective ++ * @state: atomic state ++ * ++ * This is the dpll version of drm_atomic_helper_swap_state() since the ++ * helper does not handle driver-specific global state. ++ * ++ * For consistency with atomic helpers this function does a complete swap, ++ * i.e. it also puts the current state into @state, even though there is no ++ * need for that at this moment. ++ */ ++void intel_shared_dpll_swap_state(struct drm_atomic_state *state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(state->dev); ++ struct intel_shared_dpll_state *shared_dpll; ++ struct intel_shared_dpll *pll; ++ enum intel_dpll_id i; ++ ++ if (!to_intel_atomic_state(state)->dpll_set) ++ return; ++ ++ shared_dpll = to_intel_atomic_state(state)->shared_dpll; ++ for (i = 0; i < dev_priv->num_shared_dpll; i++) { ++ struct intel_shared_dpll_state tmp; ++ ++ pll = &dev_priv->shared_dplls[i]; ++ ++ tmp = pll->state; ++ pll->state = shared_dpll[i]; ++ shared_dpll[i] = tmp; ++ } ++} ++ ++static bool ibx_pch_dpll_get_hw_state(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll, ++ struct intel_dpll_hw_state *hw_state) ++{ ++ const enum intel_dpll_id id = pll->info->id; ++ intel_wakeref_t wakeref; ++ u32 val; ++ ++ wakeref = intel_display_power_get_if_enabled(dev_priv, ++ POWER_DOMAIN_PLLS); ++ if (!wakeref) ++ return false; ++ ++ val = I915_READ(PCH_DPLL(id)); ++ hw_state->dpll = val; ++ hw_state->fp0 = I915_READ(PCH_FP0(id)); ++ hw_state->fp1 = I915_READ(PCH_FP1(id)); ++ ++ intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS, wakeref); ++ ++ return val & DPLL_VCO_ENABLE; ++} ++ ++static void ibx_pch_dpll_prepare(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++ const enum intel_dpll_id id = pll->info->id; ++ ++ I915_WRITE(PCH_FP0(id), pll->state.hw_state.fp0); ++ I915_WRITE(PCH_FP1(id), pll->state.hw_state.fp1); ++} ++ ++static void ibx_assert_pch_refclk_enabled(struct drm_i915_private *dev_priv) ++{ ++ u32 val; ++ bool enabled; ++ ++ I915_STATE_WARN_ON(!(HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))); ++ ++ val = I915_READ(PCH_DREF_CONTROL); ++ enabled = !!(val & (DREF_SSC_SOURCE_MASK | DREF_NONSPREAD_SOURCE_MASK | ++ DREF_SUPERSPREAD_SOURCE_MASK)); ++ I915_STATE_WARN(!enabled, "PCH refclk assertion failure, should be active but is disabled\n"); ++} ++ ++static void ibx_pch_dpll_enable(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++ const enum intel_dpll_id id = pll->info->id; ++ ++ /* PCH refclock must be enabled first */ ++ ibx_assert_pch_refclk_enabled(dev_priv); ++ ++ I915_WRITE(PCH_DPLL(id), pll->state.hw_state.dpll); ++ ++ /* Wait for the clocks to stabilize. */ ++ POSTING_READ(PCH_DPLL(id)); ++ udelay(150); ++ ++ /* The pixel multiplier can only be updated once the ++ * DPLL is enabled and the clocks are stable. ++ * ++ * So write it again. ++ */ ++ I915_WRITE(PCH_DPLL(id), pll->state.hw_state.dpll); ++ POSTING_READ(PCH_DPLL(id)); ++ udelay(200); ++} ++ ++static void ibx_pch_dpll_disable(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++ const enum intel_dpll_id id = pll->info->id; ++ ++ I915_WRITE(PCH_DPLL(id), 0); ++ POSTING_READ(PCH_DPLL(id)); ++ udelay(200); ++} ++ ++static struct intel_shared_dpll * ++ibx_get_dpll(struct intel_crtc_state *crtc_state, ++ struct intel_encoder *encoder) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ struct intel_shared_dpll *pll; ++ enum intel_dpll_id i; ++ ++ if (HAS_PCH_IBX(dev_priv)) { ++ /* Ironlake PCH has a fixed PLL->PCH pipe mapping. */ ++ i = (enum intel_dpll_id) crtc->pipe; ++ pll = &dev_priv->shared_dplls[i]; ++ ++ DRM_DEBUG_KMS("[CRTC:%d:%s] using pre-allocated %s\n", ++ crtc->base.base.id, crtc->base.name, ++ pll->info->name); ++ } else { ++ pll = intel_find_shared_dpll(crtc_state, ++ DPLL_ID_PCH_PLL_A, ++ DPLL_ID_PCH_PLL_B); ++ } ++ ++ if (!pll) ++ return NULL; ++ ++ /* reference the pll */ ++ intel_reference_shared_dpll(pll, crtc_state); ++ ++ return pll; ++} ++ ++static void ibx_dump_hw_state(struct drm_i915_private *dev_priv, ++ struct intel_dpll_hw_state *hw_state) ++{ ++ DRM_DEBUG_KMS("dpll_hw_state: dpll: 0x%x, dpll_md: 0x%x, " ++ "fp0: 0x%x, fp1: 0x%x\n", ++ hw_state->dpll, ++ hw_state->dpll_md, ++ hw_state->fp0, ++ hw_state->fp1); ++} ++ ++static const struct intel_shared_dpll_funcs ibx_pch_dpll_funcs = { ++ .prepare = ibx_pch_dpll_prepare, ++ .enable = ibx_pch_dpll_enable, ++ .disable = ibx_pch_dpll_disable, ++ .get_hw_state = ibx_pch_dpll_get_hw_state, ++}; ++ ++static void hsw_ddi_wrpll_enable(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++ const enum intel_dpll_id id = pll->info->id; ++ ++ I915_WRITE(WRPLL_CTL(id), pll->state.hw_state.wrpll); ++ POSTING_READ(WRPLL_CTL(id)); ++ udelay(20); ++} ++ ++static void hsw_ddi_spll_enable(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++ I915_WRITE(SPLL_CTL, pll->state.hw_state.spll); ++ POSTING_READ(SPLL_CTL); ++ udelay(20); ++} ++ ++static void hsw_ddi_wrpll_disable(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++ const enum intel_dpll_id id = pll->info->id; ++ u32 val; ++ ++ val = I915_READ(WRPLL_CTL(id)); ++ I915_WRITE(WRPLL_CTL(id), val & ~WRPLL_PLL_ENABLE); ++ POSTING_READ(WRPLL_CTL(id)); ++} ++ ++static void hsw_ddi_spll_disable(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++ u32 val; ++ ++ val = I915_READ(SPLL_CTL); ++ I915_WRITE(SPLL_CTL, val & ~SPLL_PLL_ENABLE); ++ POSTING_READ(SPLL_CTL); ++} ++ ++static bool hsw_ddi_wrpll_get_hw_state(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll, ++ struct intel_dpll_hw_state *hw_state) ++{ ++ const enum intel_dpll_id id = pll->info->id; ++ intel_wakeref_t wakeref; ++ u32 val; ++ ++ wakeref = intel_display_power_get_if_enabled(dev_priv, ++ POWER_DOMAIN_PLLS); ++ if (!wakeref) ++ return false; ++ ++ val = I915_READ(WRPLL_CTL(id)); ++ hw_state->wrpll = val; ++ ++ intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS, wakeref); ++ ++ return val & WRPLL_PLL_ENABLE; ++} ++ ++static bool hsw_ddi_spll_get_hw_state(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll, ++ struct intel_dpll_hw_state *hw_state) ++{ ++ intel_wakeref_t wakeref; ++ u32 val; ++ ++ wakeref = intel_display_power_get_if_enabled(dev_priv, ++ POWER_DOMAIN_PLLS); ++ if (!wakeref) ++ return false; ++ ++ val = I915_READ(SPLL_CTL); ++ hw_state->spll = val; ++ ++ intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS, wakeref); ++ ++ return val & SPLL_PLL_ENABLE; ++} ++ ++#define LC_FREQ 2700 ++#define LC_FREQ_2K U64_C(LC_FREQ * 2000) ++ ++#define P_MIN 2 ++#define P_MAX 64 ++#define P_INC 2 ++ ++/* Constraints for PLL good behavior */ ++#define REF_MIN 48 ++#define REF_MAX 400 ++#define VCO_MIN 2400 ++#define VCO_MAX 4800 ++ ++struct hsw_wrpll_rnp { ++ unsigned p, n2, r2; ++}; ++ ++static unsigned hsw_wrpll_get_budget_for_freq(int clock) ++{ ++ unsigned budget; ++ ++ switch (clock) { ++ case 25175000: ++ case 25200000: ++ case 27000000: ++ case 27027000: ++ case 37762500: ++ case 37800000: ++ case 40500000: ++ case 40541000: ++ case 54000000: ++ case 54054000: ++ case 59341000: ++ case 59400000: ++ case 72000000: ++ case 74176000: ++ case 74250000: ++ case 81000000: ++ case 81081000: ++ case 89012000: ++ case 89100000: ++ case 108000000: ++ case 108108000: ++ case 111264000: ++ case 111375000: ++ case 148352000: ++ case 148500000: ++ case 162000000: ++ case 162162000: ++ case 222525000: ++ case 222750000: ++ case 296703000: ++ case 297000000: ++ budget = 0; ++ break; ++ case 233500000: ++ case 245250000: ++ case 247750000: ++ case 253250000: ++ case 298000000: ++ budget = 1500; ++ break; ++ case 169128000: ++ case 169500000: ++ case 179500000: ++ case 202000000: ++ budget = 2000; ++ break; ++ case 256250000: ++ case 262500000: ++ case 270000000: ++ case 272500000: ++ case 273750000: ++ case 280750000: ++ case 281250000: ++ case 286000000: ++ case 291750000: ++ budget = 4000; ++ break; ++ case 267250000: ++ case 268500000: ++ budget = 5000; ++ break; ++ default: ++ budget = 1000; ++ break; ++ } ++ ++ return budget; ++} ++ ++static void hsw_wrpll_update_rnp(u64 freq2k, unsigned int budget, ++ unsigned int r2, unsigned int n2, ++ unsigned int p, ++ struct hsw_wrpll_rnp *best) ++{ ++ u64 a, b, c, d, diff, diff_best; ++ ++ /* No best (r,n,p) yet */ ++ if (best->p == 0) { ++ best->p = p; ++ best->n2 = n2; ++ best->r2 = r2; ++ return; ++ } ++ ++ /* ++ * Output clock is (LC_FREQ_2K / 2000) * N / (P * R), which compares to ++ * freq2k. ++ * ++ * delta = 1e6 * ++ * abs(freq2k - (LC_FREQ_2K * n2/(p * r2))) / ++ * freq2k; ++ * ++ * and we would like delta <= budget. ++ * ++ * If the discrepancy is above the PPM-based budget, always prefer to ++ * improve upon the previous solution. However, if you're within the ++ * budget, try to maximize Ref * VCO, that is N / (P * R^2). ++ */ ++ a = freq2k * budget * p * r2; ++ b = freq2k * budget * best->p * best->r2; ++ diff = abs_diff(freq2k * p * r2, LC_FREQ_2K * n2); ++ diff_best = abs_diff(freq2k * best->p * best->r2, ++ LC_FREQ_2K * best->n2); ++ c = 1000000 * diff; ++ d = 1000000 * diff_best; ++ ++ if (a < c && b < d) { ++ /* If both are above the budget, pick the closer */ ++ if (best->p * best->r2 * diff < p * r2 * diff_best) { ++ best->p = p; ++ best->n2 = n2; ++ best->r2 = r2; ++ } ++ } else if (a >= c && b < d) { ++ /* If A is below the threshold but B is above it? Update. */ ++ best->p = p; ++ best->n2 = n2; ++ best->r2 = r2; ++ } else if (a >= c && b >= d) { ++ /* Both are below the limit, so pick the higher n2/(r2*r2) */ ++ if (n2 * best->r2 * best->r2 > best->n2 * r2 * r2) { ++ best->p = p; ++ best->n2 = n2; ++ best->r2 = r2; ++ } ++ } ++ /* Otherwise a < c && b >= d, do nothing */ ++} ++ ++static void ++hsw_ddi_calculate_wrpll(int clock /* in Hz */, ++ unsigned *r2_out, unsigned *n2_out, unsigned *p_out) ++{ ++ u64 freq2k; ++ unsigned p, n2, r2; ++ struct hsw_wrpll_rnp best = { 0, 0, 0 }; ++ unsigned budget; ++ ++ freq2k = clock / 100; ++ ++ budget = hsw_wrpll_get_budget_for_freq(clock); ++ ++ /* Special case handling for 540 pixel clock: bypass WR PLL entirely ++ * and directly pass the LC PLL to it. */ ++ if (freq2k == 5400000) { ++ *n2_out = 2; ++ *p_out = 1; ++ *r2_out = 2; ++ return; ++ } ++ ++ /* ++ * Ref = LC_FREQ / R, where Ref is the actual reference input seen by ++ * the WR PLL. ++ * ++ * We want R so that REF_MIN <= Ref <= REF_MAX. ++ * Injecting R2 = 2 * R gives: ++ * REF_MAX * r2 > LC_FREQ * 2 and ++ * REF_MIN * r2 < LC_FREQ * 2 ++ * ++ * Which means the desired boundaries for r2 are: ++ * LC_FREQ * 2 / REF_MAX < r2 < LC_FREQ * 2 / REF_MIN ++ * ++ */ ++ for (r2 = LC_FREQ * 2 / REF_MAX + 1; ++ r2 <= LC_FREQ * 2 / REF_MIN; ++ r2++) { ++ ++ /* ++ * VCO = N * Ref, that is: VCO = N * LC_FREQ / R ++ * ++ * Once again we want VCO_MIN <= VCO <= VCO_MAX. ++ * Injecting R2 = 2 * R and N2 = 2 * N, we get: ++ * VCO_MAX * r2 > n2 * LC_FREQ and ++ * VCO_MIN * r2 < n2 * LC_FREQ) ++ * ++ * Which means the desired boundaries for n2 are: ++ * VCO_MIN * r2 / LC_FREQ < n2 < VCO_MAX * r2 / LC_FREQ ++ */ ++ for (n2 = VCO_MIN * r2 / LC_FREQ + 1; ++ n2 <= VCO_MAX * r2 / LC_FREQ; ++ n2++) { ++ ++ for (p = P_MIN; p <= P_MAX; p += P_INC) ++ hsw_wrpll_update_rnp(freq2k, budget, ++ r2, n2, p, &best); ++ } ++ } ++ ++ *n2_out = best.n2; ++ *p_out = best.p; ++ *r2_out = best.r2; ++} ++ ++static struct intel_shared_dpll *hsw_ddi_hdmi_get_dpll(struct intel_crtc_state *crtc_state) ++{ ++ struct intel_shared_dpll *pll; ++ u32 val; ++ unsigned int p, n2, r2; ++ ++ hsw_ddi_calculate_wrpll(crtc_state->port_clock * 1000, &r2, &n2, &p); ++ ++ val = WRPLL_PLL_ENABLE | WRPLL_PLL_LCPLL | ++ WRPLL_DIVIDER_REFERENCE(r2) | WRPLL_DIVIDER_FEEDBACK(n2) | ++ WRPLL_DIVIDER_POST(p); ++ ++ crtc_state->dpll_hw_state.wrpll = val; ++ ++ pll = intel_find_shared_dpll(crtc_state, ++ DPLL_ID_WRPLL1, DPLL_ID_WRPLL2); ++ ++ if (!pll) ++ return NULL; ++ ++ return pll; ++} ++ ++static struct intel_shared_dpll * ++hsw_ddi_dp_get_dpll(struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); ++ struct intel_shared_dpll *pll; ++ enum intel_dpll_id pll_id; ++ int clock = crtc_state->port_clock; ++ ++ switch (clock / 2) { ++ case 81000: ++ pll_id = DPLL_ID_LCPLL_810; ++ break; ++ case 135000: ++ pll_id = DPLL_ID_LCPLL_1350; ++ break; ++ case 270000: ++ pll_id = DPLL_ID_LCPLL_2700; ++ break; ++ default: ++ DRM_DEBUG_KMS("Invalid clock for DP: %d\n", clock); ++ return NULL; ++ } ++ ++ pll = intel_get_shared_dpll_by_id(dev_priv, pll_id); ++ ++ if (!pll) ++ return NULL; ++ ++ return pll; ++} ++ ++static struct intel_shared_dpll * ++hsw_get_dpll(struct intel_crtc_state *crtc_state, ++ struct intel_encoder *encoder) ++{ ++ struct intel_shared_dpll *pll; ++ ++ memset(&crtc_state->dpll_hw_state, 0, ++ sizeof(crtc_state->dpll_hw_state)); ++ ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) { ++ pll = hsw_ddi_hdmi_get_dpll(crtc_state); ++ } else if (intel_crtc_has_dp_encoder(crtc_state)) { ++ pll = hsw_ddi_dp_get_dpll(crtc_state); ++ } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) { ++ if (WARN_ON(crtc_state->port_clock / 2 != 135000)) ++ return NULL; ++ ++ crtc_state->dpll_hw_state.spll = ++ SPLL_PLL_ENABLE | SPLL_PLL_FREQ_1350MHz | SPLL_PLL_SSC; ++ ++ pll = intel_find_shared_dpll(crtc_state, ++ DPLL_ID_SPLL, DPLL_ID_SPLL); ++ } else { ++ return NULL; ++ } ++ ++ if (!pll) ++ return NULL; ++ ++ intel_reference_shared_dpll(pll, crtc_state); ++ ++ return pll; ++} ++ ++static void hsw_dump_hw_state(struct drm_i915_private *dev_priv, ++ struct intel_dpll_hw_state *hw_state) ++{ ++ DRM_DEBUG_KMS("dpll_hw_state: wrpll: 0x%x spll: 0x%x\n", ++ hw_state->wrpll, hw_state->spll); ++} ++ ++static const struct intel_shared_dpll_funcs hsw_ddi_wrpll_funcs = { ++ .enable = hsw_ddi_wrpll_enable, ++ .disable = hsw_ddi_wrpll_disable, ++ .get_hw_state = hsw_ddi_wrpll_get_hw_state, ++}; ++ ++static const struct intel_shared_dpll_funcs hsw_ddi_spll_funcs = { ++ .enable = hsw_ddi_spll_enable, ++ .disable = hsw_ddi_spll_disable, ++ .get_hw_state = hsw_ddi_spll_get_hw_state, ++}; ++ ++static void hsw_ddi_lcpll_enable(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++} ++ ++static void hsw_ddi_lcpll_disable(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++} ++ ++static bool hsw_ddi_lcpll_get_hw_state(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll, ++ struct intel_dpll_hw_state *hw_state) ++{ ++ return true; ++} ++ ++static const struct intel_shared_dpll_funcs hsw_ddi_lcpll_funcs = { ++ .enable = hsw_ddi_lcpll_enable, ++ .disable = hsw_ddi_lcpll_disable, ++ .get_hw_state = hsw_ddi_lcpll_get_hw_state, ++}; ++ ++struct skl_dpll_regs { ++ i915_reg_t ctl, cfgcr1, cfgcr2; ++}; ++ ++/* this array is indexed by the *shared* pll id */ ++static const struct skl_dpll_regs skl_dpll_regs[4] = { ++ { ++ /* DPLL 0 */ ++ .ctl = LCPLL1_CTL, ++ /* DPLL 0 doesn't support HDMI mode */ ++ }, ++ { ++ /* DPLL 1 */ ++ .ctl = LCPLL2_CTL, ++ .cfgcr1 = DPLL_CFGCR1(SKL_DPLL1), ++ .cfgcr2 = DPLL_CFGCR2(SKL_DPLL1), ++ }, ++ { ++ /* DPLL 2 */ ++ .ctl = WRPLL_CTL(0), ++ .cfgcr1 = DPLL_CFGCR1(SKL_DPLL2), ++ .cfgcr2 = DPLL_CFGCR2(SKL_DPLL2), ++ }, ++ { ++ /* DPLL 3 */ ++ .ctl = WRPLL_CTL(1), ++ .cfgcr1 = DPLL_CFGCR1(SKL_DPLL3), ++ .cfgcr2 = DPLL_CFGCR2(SKL_DPLL3), ++ }, ++}; ++ ++static void skl_ddi_pll_write_ctrl1(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++ const enum intel_dpll_id id = pll->info->id; ++ u32 val; ++ ++ val = I915_READ(DPLL_CTRL1); ++ ++ val &= ~(DPLL_CTRL1_HDMI_MODE(id) | ++ DPLL_CTRL1_SSC(id) | ++ DPLL_CTRL1_LINK_RATE_MASK(id)); ++ val |= pll->state.hw_state.ctrl1 << (id * 6); ++ ++ I915_WRITE(DPLL_CTRL1, val); ++ POSTING_READ(DPLL_CTRL1); ++} ++ ++static void skl_ddi_pll_enable(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++ const struct skl_dpll_regs *regs = skl_dpll_regs; ++ const enum intel_dpll_id id = pll->info->id; ++ ++ skl_ddi_pll_write_ctrl1(dev_priv, pll); ++ ++ I915_WRITE(regs[id].cfgcr1, pll->state.hw_state.cfgcr1); ++ I915_WRITE(regs[id].cfgcr2, pll->state.hw_state.cfgcr2); ++ POSTING_READ(regs[id].cfgcr1); ++ POSTING_READ(regs[id].cfgcr2); ++ ++ /* the enable bit is always bit 31 */ ++ I915_WRITE(regs[id].ctl, ++ I915_READ(regs[id].ctl) | LCPLL_PLL_ENABLE); ++ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ DPLL_STATUS, ++ DPLL_LOCK(id), ++ DPLL_LOCK(id), ++ 5)) ++ DRM_ERROR("DPLL %d not locked\n", id); ++} ++ ++static void skl_ddi_dpll0_enable(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++ skl_ddi_pll_write_ctrl1(dev_priv, pll); ++} ++ ++static void skl_ddi_pll_disable(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++ const struct skl_dpll_regs *regs = skl_dpll_regs; ++ const enum intel_dpll_id id = pll->info->id; ++ ++ /* the enable bit is always bit 31 */ ++ I915_WRITE(regs[id].ctl, ++ I915_READ(regs[id].ctl) & ~LCPLL_PLL_ENABLE); ++ POSTING_READ(regs[id].ctl); ++} ++ ++static void skl_ddi_dpll0_disable(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++} ++ ++static bool skl_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll, ++ struct intel_dpll_hw_state *hw_state) ++{ ++ u32 val; ++ const struct skl_dpll_regs *regs = skl_dpll_regs; ++ const enum intel_dpll_id id = pll->info->id; ++ intel_wakeref_t wakeref; ++ bool ret; ++ ++ wakeref = intel_display_power_get_if_enabled(dev_priv, ++ POWER_DOMAIN_PLLS); ++ if (!wakeref) ++ return false; ++ ++ ret = false; ++ ++ val = I915_READ(regs[id].ctl); ++ if (!(val & LCPLL_PLL_ENABLE)) ++ goto out; ++ ++ val = I915_READ(DPLL_CTRL1); ++ hw_state->ctrl1 = (val >> (id * 6)) & 0x3f; ++ ++ /* avoid reading back stale values if HDMI mode is not enabled */ ++ if (val & DPLL_CTRL1_HDMI_MODE(id)) { ++ hw_state->cfgcr1 = I915_READ(regs[id].cfgcr1); ++ hw_state->cfgcr2 = I915_READ(regs[id].cfgcr2); ++ } ++ ret = true; ++ ++out: ++ intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS, wakeref); ++ ++ return ret; ++} ++ ++static bool skl_ddi_dpll0_get_hw_state(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll, ++ struct intel_dpll_hw_state *hw_state) ++{ ++ const struct skl_dpll_regs *regs = skl_dpll_regs; ++ const enum intel_dpll_id id = pll->info->id; ++ intel_wakeref_t wakeref; ++ u32 val; ++ bool ret; ++ ++ wakeref = intel_display_power_get_if_enabled(dev_priv, ++ POWER_DOMAIN_PLLS); ++ if (!wakeref) ++ return false; ++ ++ ret = false; ++ ++ /* DPLL0 is always enabled since it drives CDCLK */ ++ val = I915_READ(regs[id].ctl); ++ if (WARN_ON(!(val & LCPLL_PLL_ENABLE))) ++ goto out; ++ ++ val = I915_READ(DPLL_CTRL1); ++ hw_state->ctrl1 = (val >> (id * 6)) & 0x3f; ++ ++ ret = true; ++ ++out: ++ intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS, wakeref); ++ ++ return ret; ++} ++ ++struct skl_wrpll_context { ++ u64 min_deviation; /* current minimal deviation */ ++ u64 central_freq; /* chosen central freq */ ++ u64 dco_freq; /* chosen dco freq */ ++ unsigned int p; /* chosen divider */ ++}; ++ ++static void skl_wrpll_context_init(struct skl_wrpll_context *ctx) ++{ ++ memset(ctx, 0, sizeof(*ctx)); ++ ++ ctx->min_deviation = U64_MAX; ++} ++ ++/* DCO freq must be within +1%/-6% of the DCO central freq */ ++#define SKL_DCO_MAX_PDEVIATION 100 ++#define SKL_DCO_MAX_NDEVIATION 600 ++ ++static void skl_wrpll_try_divider(struct skl_wrpll_context *ctx, ++ u64 central_freq, ++ u64 dco_freq, ++ unsigned int divider) ++{ ++ u64 deviation; ++ ++ deviation = div64_u64(10000 * abs_diff(dco_freq, central_freq), ++ central_freq); ++ ++ /* positive deviation */ ++ if (dco_freq >= central_freq) { ++ if (deviation < SKL_DCO_MAX_PDEVIATION && ++ deviation < ctx->min_deviation) { ++ ctx->min_deviation = deviation; ++ ctx->central_freq = central_freq; ++ ctx->dco_freq = dco_freq; ++ ctx->p = divider; ++ } ++ /* negative deviation */ ++ } else if (deviation < SKL_DCO_MAX_NDEVIATION && ++ deviation < ctx->min_deviation) { ++ ctx->min_deviation = deviation; ++ ctx->central_freq = central_freq; ++ ctx->dco_freq = dco_freq; ++ ctx->p = divider; ++ } ++} ++ ++static void skl_wrpll_get_multipliers(unsigned int p, ++ unsigned int *p0 /* out */, ++ unsigned int *p1 /* out */, ++ unsigned int *p2 /* out */) ++{ ++ /* even dividers */ ++ if (p % 2 == 0) { ++ unsigned int half = p / 2; ++ ++ if (half == 1 || half == 2 || half == 3 || half == 5) { ++ *p0 = 2; ++ *p1 = 1; ++ *p2 = half; ++ } else if (half % 2 == 0) { ++ *p0 = 2; ++ *p1 = half / 2; ++ *p2 = 2; ++ } else if (half % 3 == 0) { ++ *p0 = 3; ++ *p1 = half / 3; ++ *p2 = 2; ++ } else if (half % 7 == 0) { ++ *p0 = 7; ++ *p1 = half / 7; ++ *p2 = 2; ++ } ++ } else if (p == 3 || p == 9) { /* 3, 5, 7, 9, 15, 21, 35 */ ++ *p0 = 3; ++ *p1 = 1; ++ *p2 = p / 3; ++ } else if (p == 5 || p == 7) { ++ *p0 = p; ++ *p1 = 1; ++ *p2 = 1; ++ } else if (p == 15) { ++ *p0 = 3; ++ *p1 = 1; ++ *p2 = 5; ++ } else if (p == 21) { ++ *p0 = 7; ++ *p1 = 1; ++ *p2 = 3; ++ } else if (p == 35) { ++ *p0 = 7; ++ *p1 = 1; ++ *p2 = 5; ++ } ++} ++ ++struct skl_wrpll_params { ++ u32 dco_fraction; ++ u32 dco_integer; ++ u32 qdiv_ratio; ++ u32 qdiv_mode; ++ u32 kdiv; ++ u32 pdiv; ++ u32 central_freq; ++}; ++ ++static void skl_wrpll_params_populate(struct skl_wrpll_params *params, ++ u64 afe_clock, ++ u64 central_freq, ++ u32 p0, u32 p1, u32 p2) ++{ ++ u64 dco_freq; ++ ++ switch (central_freq) { ++ case 9600000000ULL: ++ params->central_freq = 0; ++ break; ++ case 9000000000ULL: ++ params->central_freq = 1; ++ break; ++ case 8400000000ULL: ++ params->central_freq = 3; ++ } ++ ++ switch (p0) { ++ case 1: ++ params->pdiv = 0; ++ break; ++ case 2: ++ params->pdiv = 1; ++ break; ++ case 3: ++ params->pdiv = 2; ++ break; ++ case 7: ++ params->pdiv = 4; ++ break; ++ default: ++ WARN(1, "Incorrect PDiv\n"); ++ } ++ ++ switch (p2) { ++ case 5: ++ params->kdiv = 0; ++ break; ++ case 2: ++ params->kdiv = 1; ++ break; ++ case 3: ++ params->kdiv = 2; ++ break; ++ case 1: ++ params->kdiv = 3; ++ break; ++ default: ++ WARN(1, "Incorrect KDiv\n"); ++ } ++ ++ params->qdiv_ratio = p1; ++ params->qdiv_mode = (params->qdiv_ratio == 1) ? 0 : 1; ++ ++ dco_freq = p0 * p1 * p2 * afe_clock; ++ ++ /* ++ * Intermediate values are in Hz. ++ * Divide by MHz to match bsepc ++ */ ++ params->dco_integer = div_u64(dco_freq, 24 * MHz(1)); ++ params->dco_fraction = ++ div_u64((div_u64(dco_freq, 24) - ++ params->dco_integer * MHz(1)) * 0x8000, MHz(1)); ++} ++ ++static bool ++skl_ddi_calculate_wrpll(int clock /* in Hz */, ++ struct skl_wrpll_params *wrpll_params) ++{ ++ u64 afe_clock = clock * 5; /* AFE Clock is 5x Pixel clock */ ++ u64 dco_central_freq[3] = { 8400000000ULL, ++ 9000000000ULL, ++ 9600000000ULL }; ++ static const int even_dividers[] = { 4, 6, 8, 10, 12, 14, 16, 18, 20, ++ 24, 28, 30, 32, 36, 40, 42, 44, ++ 48, 52, 54, 56, 60, 64, 66, 68, ++ 70, 72, 76, 78, 80, 84, 88, 90, ++ 92, 96, 98 }; ++ static const int odd_dividers[] = { 3, 5, 7, 9, 15, 21, 35 }; ++ static const struct { ++ const int *list; ++ int n_dividers; ++ } dividers[] = { ++ { even_dividers, ARRAY_SIZE(even_dividers) }, ++ { odd_dividers, ARRAY_SIZE(odd_dividers) }, ++ }; ++ struct skl_wrpll_context ctx; ++ unsigned int dco, d, i; ++ unsigned int p0, p1, p2; ++ ++ skl_wrpll_context_init(&ctx); ++ ++ for (d = 0; d < ARRAY_SIZE(dividers); d++) { ++ for (dco = 0; dco < ARRAY_SIZE(dco_central_freq); dco++) { ++ for (i = 0; i < dividers[d].n_dividers; i++) { ++ unsigned int p = dividers[d].list[i]; ++ u64 dco_freq = p * afe_clock; ++ ++ skl_wrpll_try_divider(&ctx, ++ dco_central_freq[dco], ++ dco_freq, ++ p); ++ /* ++ * Skip the remaining dividers if we're sure to ++ * have found the definitive divider, we can't ++ * improve a 0 deviation. ++ */ ++ if (ctx.min_deviation == 0) ++ goto skip_remaining_dividers; ++ } ++ } ++ ++skip_remaining_dividers: ++ /* ++ * If a solution is found with an even divider, prefer ++ * this one. ++ */ ++ if (d == 0 && ctx.p) ++ break; ++ } ++ ++ if (!ctx.p) { ++ DRM_DEBUG_DRIVER("No valid divider found for %dHz\n", clock); ++ return false; ++ } ++ ++ /* ++ * gcc incorrectly analyses that these can be used without being ++ * initialized. To be fair, it's hard to guess. ++ */ ++ p0 = p1 = p2 = 0; ++ skl_wrpll_get_multipliers(ctx.p, &p0, &p1, &p2); ++ skl_wrpll_params_populate(wrpll_params, afe_clock, ctx.central_freq, ++ p0, p1, p2); ++ ++ return true; ++} ++ ++static bool skl_ddi_hdmi_pll_dividers(struct intel_crtc_state *crtc_state) ++{ ++ u32 ctrl1, cfgcr1, cfgcr2; ++ struct skl_wrpll_params wrpll_params = { 0, }; ++ ++ /* ++ * See comment in intel_dpll_hw_state to understand why we always use 0 ++ * as the DPLL id in this function. ++ */ ++ ctrl1 = DPLL_CTRL1_OVERRIDE(0); ++ ++ ctrl1 |= DPLL_CTRL1_HDMI_MODE(0); ++ ++ if (!skl_ddi_calculate_wrpll(crtc_state->port_clock * 1000, ++ &wrpll_params)) ++ return false; ++ ++ cfgcr1 = DPLL_CFGCR1_FREQ_ENABLE | ++ DPLL_CFGCR1_DCO_FRACTION(wrpll_params.dco_fraction) | ++ wrpll_params.dco_integer; ++ ++ cfgcr2 = DPLL_CFGCR2_QDIV_RATIO(wrpll_params.qdiv_ratio) | ++ DPLL_CFGCR2_QDIV_MODE(wrpll_params.qdiv_mode) | ++ DPLL_CFGCR2_KDIV(wrpll_params.kdiv) | ++ DPLL_CFGCR2_PDIV(wrpll_params.pdiv) | ++ wrpll_params.central_freq; ++ ++ memset(&crtc_state->dpll_hw_state, 0, ++ sizeof(crtc_state->dpll_hw_state)); ++ ++ crtc_state->dpll_hw_state.ctrl1 = ctrl1; ++ crtc_state->dpll_hw_state.cfgcr1 = cfgcr1; ++ crtc_state->dpll_hw_state.cfgcr2 = cfgcr2; ++ return true; ++} ++ ++static bool ++skl_ddi_dp_set_dpll_hw_state(struct intel_crtc_state *crtc_state) ++{ ++ u32 ctrl1; ++ ++ /* ++ * See comment in intel_dpll_hw_state to understand why we always use 0 ++ * as the DPLL id in this function. ++ */ ++ ctrl1 = DPLL_CTRL1_OVERRIDE(0); ++ switch (crtc_state->port_clock / 2) { ++ case 81000: ++ ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, 0); ++ break; ++ case 135000: ++ ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1350, 0); ++ break; ++ case 270000: ++ ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2700, 0); ++ break; ++ /* eDP 1.4 rates */ ++ case 162000: ++ ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1620, 0); ++ break; ++ case 108000: ++ ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, 0); ++ break; ++ case 216000: ++ ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2160, 0); ++ break; ++ } ++ ++ memset(&crtc_state->dpll_hw_state, 0, ++ sizeof(crtc_state->dpll_hw_state)); ++ ++ crtc_state->dpll_hw_state.ctrl1 = ctrl1; ++ ++ return true; ++} ++ ++static struct intel_shared_dpll * ++skl_get_dpll(struct intel_crtc_state *crtc_state, ++ struct intel_encoder *encoder) ++{ ++ struct intel_shared_dpll *pll; ++ bool bret; ++ ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) { ++ bret = skl_ddi_hdmi_pll_dividers(crtc_state); ++ if (!bret) { ++ DRM_DEBUG_KMS("Could not get HDMI pll dividers.\n"); ++ return NULL; ++ } ++ } else if (intel_crtc_has_dp_encoder(crtc_state)) { ++ bret = skl_ddi_dp_set_dpll_hw_state(crtc_state); ++ if (!bret) { ++ DRM_DEBUG_KMS("Could not set DP dpll HW state.\n"); ++ return NULL; ++ } ++ } else { ++ return NULL; ++ } ++ ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP)) ++ pll = intel_find_shared_dpll(crtc_state, ++ DPLL_ID_SKL_DPLL0, ++ DPLL_ID_SKL_DPLL0); ++ else ++ pll = intel_find_shared_dpll(crtc_state, ++ DPLL_ID_SKL_DPLL1, ++ DPLL_ID_SKL_DPLL3); ++ if (!pll) ++ return NULL; ++ ++ intel_reference_shared_dpll(pll, crtc_state); ++ ++ return pll; ++} ++ ++static void skl_dump_hw_state(struct drm_i915_private *dev_priv, ++ struct intel_dpll_hw_state *hw_state) ++{ ++ DRM_DEBUG_KMS("dpll_hw_state: " ++ "ctrl1: 0x%x, cfgcr1: 0x%x, cfgcr2: 0x%x\n", ++ hw_state->ctrl1, ++ hw_state->cfgcr1, ++ hw_state->cfgcr2); ++} ++ ++static const struct intel_shared_dpll_funcs skl_ddi_pll_funcs = { ++ .enable = skl_ddi_pll_enable, ++ .disable = skl_ddi_pll_disable, ++ .get_hw_state = skl_ddi_pll_get_hw_state, ++}; ++ ++static const struct intel_shared_dpll_funcs skl_ddi_dpll0_funcs = { ++ .enable = skl_ddi_dpll0_enable, ++ .disable = skl_ddi_dpll0_disable, ++ .get_hw_state = skl_ddi_dpll0_get_hw_state, ++}; ++ ++static void bxt_ddi_pll_enable(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++ u32 temp; ++ enum port port = (enum port)pll->info->id; /* 1:1 port->PLL mapping */ ++ enum dpio_phy phy; ++ enum dpio_channel ch; ++ ++ bxt_port_to_phy_channel(dev_priv, port, &phy, &ch); ++ ++ /* Non-SSC reference */ ++ temp = I915_READ(BXT_PORT_PLL_ENABLE(port)); ++ temp |= PORT_PLL_REF_SEL; ++ I915_WRITE(BXT_PORT_PLL_ENABLE(port), temp); ++ ++ if (IS_GEMINILAKE(dev_priv)) { ++ temp = I915_READ(BXT_PORT_PLL_ENABLE(port)); ++ temp |= PORT_PLL_POWER_ENABLE; ++ I915_WRITE(BXT_PORT_PLL_ENABLE(port), temp); ++ ++ if (wait_for_us((I915_READ(BXT_PORT_PLL_ENABLE(port)) & ++ PORT_PLL_POWER_STATE), 200)) ++ DRM_ERROR("Power state not set for PLL:%d\n", port); ++ } ++ ++ /* Disable 10 bit clock */ ++ temp = I915_READ(BXT_PORT_PLL_EBB_4(phy, ch)); ++ temp &= ~PORT_PLL_10BIT_CLK_ENABLE; ++ I915_WRITE(BXT_PORT_PLL_EBB_4(phy, ch), temp); ++ ++ /* Write P1 & P2 */ ++ temp = I915_READ(BXT_PORT_PLL_EBB_0(phy, ch)); ++ temp &= ~(PORT_PLL_P1_MASK | PORT_PLL_P2_MASK); ++ temp |= pll->state.hw_state.ebb0; ++ I915_WRITE(BXT_PORT_PLL_EBB_0(phy, ch), temp); ++ ++ /* Write M2 integer */ ++ temp = I915_READ(BXT_PORT_PLL(phy, ch, 0)); ++ temp &= ~PORT_PLL_M2_MASK; ++ temp |= pll->state.hw_state.pll0; ++ I915_WRITE(BXT_PORT_PLL(phy, ch, 0), temp); ++ ++ /* Write N */ ++ temp = I915_READ(BXT_PORT_PLL(phy, ch, 1)); ++ temp &= ~PORT_PLL_N_MASK; ++ temp |= pll->state.hw_state.pll1; ++ I915_WRITE(BXT_PORT_PLL(phy, ch, 1), temp); ++ ++ /* Write M2 fraction */ ++ temp = I915_READ(BXT_PORT_PLL(phy, ch, 2)); ++ temp &= ~PORT_PLL_M2_FRAC_MASK; ++ temp |= pll->state.hw_state.pll2; ++ I915_WRITE(BXT_PORT_PLL(phy, ch, 2), temp); ++ ++ /* Write M2 fraction enable */ ++ temp = I915_READ(BXT_PORT_PLL(phy, ch, 3)); ++ temp &= ~PORT_PLL_M2_FRAC_ENABLE; ++ temp |= pll->state.hw_state.pll3; ++ I915_WRITE(BXT_PORT_PLL(phy, ch, 3), temp); ++ ++ /* Write coeff */ ++ temp = I915_READ(BXT_PORT_PLL(phy, ch, 6)); ++ temp &= ~PORT_PLL_PROP_COEFF_MASK; ++ temp &= ~PORT_PLL_INT_COEFF_MASK; ++ temp &= ~PORT_PLL_GAIN_CTL_MASK; ++ temp |= pll->state.hw_state.pll6; ++ I915_WRITE(BXT_PORT_PLL(phy, ch, 6), temp); ++ ++ /* Write calibration val */ ++ temp = I915_READ(BXT_PORT_PLL(phy, ch, 8)); ++ temp &= ~PORT_PLL_TARGET_CNT_MASK; ++ temp |= pll->state.hw_state.pll8; ++ I915_WRITE(BXT_PORT_PLL(phy, ch, 8), temp); ++ ++ temp = I915_READ(BXT_PORT_PLL(phy, ch, 9)); ++ temp &= ~PORT_PLL_LOCK_THRESHOLD_MASK; ++ temp |= pll->state.hw_state.pll9; ++ I915_WRITE(BXT_PORT_PLL(phy, ch, 9), temp); ++ ++ temp = I915_READ(BXT_PORT_PLL(phy, ch, 10)); ++ temp &= ~PORT_PLL_DCO_AMP_OVR_EN_H; ++ temp &= ~PORT_PLL_DCO_AMP_MASK; ++ temp |= pll->state.hw_state.pll10; ++ I915_WRITE(BXT_PORT_PLL(phy, ch, 10), temp); ++ ++ /* Recalibrate with new settings */ ++ temp = I915_READ(BXT_PORT_PLL_EBB_4(phy, ch)); ++ temp |= PORT_PLL_RECALIBRATE; ++ I915_WRITE(BXT_PORT_PLL_EBB_4(phy, ch), temp); ++ temp &= ~PORT_PLL_10BIT_CLK_ENABLE; ++ temp |= pll->state.hw_state.ebb4; ++ I915_WRITE(BXT_PORT_PLL_EBB_4(phy, ch), temp); ++ ++ /* Enable PLL */ ++ temp = I915_READ(BXT_PORT_PLL_ENABLE(port)); ++ temp |= PORT_PLL_ENABLE; ++ I915_WRITE(BXT_PORT_PLL_ENABLE(port), temp); ++ POSTING_READ(BXT_PORT_PLL_ENABLE(port)); ++ ++ if (wait_for_us((I915_READ(BXT_PORT_PLL_ENABLE(port)) & PORT_PLL_LOCK), ++ 200)) ++ DRM_ERROR("PLL %d not locked\n", port); ++ ++ if (IS_GEMINILAKE(dev_priv)) { ++ temp = I915_READ(BXT_PORT_TX_DW5_LN0(phy, ch)); ++ temp |= DCC_DELAY_RANGE_2; ++ I915_WRITE(BXT_PORT_TX_DW5_GRP(phy, ch), temp); ++ } ++ ++ /* ++ * While we write to the group register to program all lanes at once we ++ * can read only lane registers and we pick lanes 0/1 for that. ++ */ ++ temp = I915_READ(BXT_PORT_PCS_DW12_LN01(phy, ch)); ++ temp &= ~LANE_STAGGER_MASK; ++ temp &= ~LANESTAGGER_STRAP_OVRD; ++ temp |= pll->state.hw_state.pcsdw12; ++ I915_WRITE(BXT_PORT_PCS_DW12_GRP(phy, ch), temp); ++} ++ ++static void bxt_ddi_pll_disable(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++ enum port port = (enum port)pll->info->id; /* 1:1 port->PLL mapping */ ++ u32 temp; ++ ++ temp = I915_READ(BXT_PORT_PLL_ENABLE(port)); ++ temp &= ~PORT_PLL_ENABLE; ++ I915_WRITE(BXT_PORT_PLL_ENABLE(port), temp); ++ POSTING_READ(BXT_PORT_PLL_ENABLE(port)); ++ ++ if (IS_GEMINILAKE(dev_priv)) { ++ temp = I915_READ(BXT_PORT_PLL_ENABLE(port)); ++ temp &= ~PORT_PLL_POWER_ENABLE; ++ I915_WRITE(BXT_PORT_PLL_ENABLE(port), temp); ++ ++ if (wait_for_us(!(I915_READ(BXT_PORT_PLL_ENABLE(port)) & ++ PORT_PLL_POWER_STATE), 200)) ++ DRM_ERROR("Power state not reset for PLL:%d\n", port); ++ } ++} ++ ++static bool bxt_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll, ++ struct intel_dpll_hw_state *hw_state) ++{ ++ enum port port = (enum port)pll->info->id; /* 1:1 port->PLL mapping */ ++ intel_wakeref_t wakeref; ++ enum dpio_phy phy; ++ enum dpio_channel ch; ++ u32 val; ++ bool ret; ++ ++ bxt_port_to_phy_channel(dev_priv, port, &phy, &ch); ++ ++ wakeref = intel_display_power_get_if_enabled(dev_priv, ++ POWER_DOMAIN_PLLS); ++ if (!wakeref) ++ return false; ++ ++ ret = false; ++ ++ val = I915_READ(BXT_PORT_PLL_ENABLE(port)); ++ if (!(val & PORT_PLL_ENABLE)) ++ goto out; ++ ++ hw_state->ebb0 = I915_READ(BXT_PORT_PLL_EBB_0(phy, ch)); ++ hw_state->ebb0 &= PORT_PLL_P1_MASK | PORT_PLL_P2_MASK; ++ ++ hw_state->ebb4 = I915_READ(BXT_PORT_PLL_EBB_4(phy, ch)); ++ hw_state->ebb4 &= PORT_PLL_10BIT_CLK_ENABLE; ++ ++ hw_state->pll0 = I915_READ(BXT_PORT_PLL(phy, ch, 0)); ++ hw_state->pll0 &= PORT_PLL_M2_MASK; ++ ++ hw_state->pll1 = I915_READ(BXT_PORT_PLL(phy, ch, 1)); ++ hw_state->pll1 &= PORT_PLL_N_MASK; ++ ++ hw_state->pll2 = I915_READ(BXT_PORT_PLL(phy, ch, 2)); ++ hw_state->pll2 &= PORT_PLL_M2_FRAC_MASK; ++ ++ hw_state->pll3 = I915_READ(BXT_PORT_PLL(phy, ch, 3)); ++ hw_state->pll3 &= PORT_PLL_M2_FRAC_ENABLE; ++ ++ hw_state->pll6 = I915_READ(BXT_PORT_PLL(phy, ch, 6)); ++ hw_state->pll6 &= PORT_PLL_PROP_COEFF_MASK | ++ PORT_PLL_INT_COEFF_MASK | ++ PORT_PLL_GAIN_CTL_MASK; ++ ++ hw_state->pll8 = I915_READ(BXT_PORT_PLL(phy, ch, 8)); ++ hw_state->pll8 &= PORT_PLL_TARGET_CNT_MASK; ++ ++ hw_state->pll9 = I915_READ(BXT_PORT_PLL(phy, ch, 9)); ++ hw_state->pll9 &= PORT_PLL_LOCK_THRESHOLD_MASK; ++ ++ hw_state->pll10 = I915_READ(BXT_PORT_PLL(phy, ch, 10)); ++ hw_state->pll10 &= PORT_PLL_DCO_AMP_OVR_EN_H | ++ PORT_PLL_DCO_AMP_MASK; ++ ++ /* ++ * While we write to the group register to program all lanes at once we ++ * can read only lane registers. We configure all lanes the same way, so ++ * here just read out lanes 0/1 and output a note if lanes 2/3 differ. ++ */ ++ hw_state->pcsdw12 = I915_READ(BXT_PORT_PCS_DW12_LN01(phy, ch)); ++ if (I915_READ(BXT_PORT_PCS_DW12_LN23(phy, ch)) != hw_state->pcsdw12) ++ DRM_DEBUG_DRIVER("lane stagger config different for lane 01 (%08x) and 23 (%08x)\n", ++ hw_state->pcsdw12, ++ I915_READ(BXT_PORT_PCS_DW12_LN23(phy, ch))); ++ hw_state->pcsdw12 &= LANE_STAGGER_MASK | LANESTAGGER_STRAP_OVRD; ++ ++ ret = true; ++ ++out: ++ intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS, wakeref); ++ ++ return ret; ++} ++ ++/* bxt clock parameters */ ++struct bxt_clk_div { ++ int clock; ++ u32 p1; ++ u32 p2; ++ u32 m2_int; ++ u32 m2_frac; ++ bool m2_frac_en; ++ u32 n; ++ ++ int vco; ++}; ++ ++/* pre-calculated values for DP linkrates */ ++static const struct bxt_clk_div bxt_dp_clk_val[] = { ++ {162000, 4, 2, 32, 1677722, 1, 1}, ++ {270000, 4, 1, 27, 0, 0, 1}, ++ {540000, 2, 1, 27, 0, 0, 1}, ++ {216000, 3, 2, 32, 1677722, 1, 1}, ++ {243000, 4, 1, 24, 1258291, 1, 1}, ++ {324000, 4, 1, 32, 1677722, 1, 1}, ++ {432000, 3, 1, 32, 1677722, 1, 1} ++}; ++ ++static bool ++bxt_ddi_hdmi_pll_dividers(struct intel_crtc_state *crtc_state, ++ struct bxt_clk_div *clk_div) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct dpll best_clock; ++ ++ /* Calculate HDMI div */ ++ /* ++ * FIXME: tie the following calculation into ++ * i9xx_crtc_compute_clock ++ */ ++ if (!bxt_find_best_dpll(crtc_state, &best_clock)) { ++ DRM_DEBUG_DRIVER("no PLL dividers found for clock %d pipe %c\n", ++ crtc_state->port_clock, ++ pipe_name(crtc->pipe)); ++ return false; ++ } ++ ++ clk_div->p1 = best_clock.p1; ++ clk_div->p2 = best_clock.p2; ++ WARN_ON(best_clock.m1 != 2); ++ clk_div->n = best_clock.n; ++ clk_div->m2_int = best_clock.m2 >> 22; ++ clk_div->m2_frac = best_clock.m2 & ((1 << 22) - 1); ++ clk_div->m2_frac_en = clk_div->m2_frac != 0; ++ ++ clk_div->vco = best_clock.vco; ++ ++ return true; ++} ++ ++static void bxt_ddi_dp_pll_dividers(struct intel_crtc_state *crtc_state, ++ struct bxt_clk_div *clk_div) ++{ ++ int clock = crtc_state->port_clock; ++ int i; ++ ++ *clk_div = bxt_dp_clk_val[0]; ++ for (i = 0; i < ARRAY_SIZE(bxt_dp_clk_val); ++i) { ++ if (bxt_dp_clk_val[i].clock == clock) { ++ *clk_div = bxt_dp_clk_val[i]; ++ break; ++ } ++ } ++ ++ clk_div->vco = clock * 10 / 2 * clk_div->p1 * clk_div->p2; ++} ++ ++static bool bxt_ddi_set_dpll_hw_state(struct intel_crtc_state *crtc_state, ++ const struct bxt_clk_div *clk_div) ++{ ++ struct intel_dpll_hw_state *dpll_hw_state = &crtc_state->dpll_hw_state; ++ int clock = crtc_state->port_clock; ++ int vco = clk_div->vco; ++ u32 prop_coef, int_coef, gain_ctl, targ_cnt; ++ u32 lanestagger; ++ ++ memset(dpll_hw_state, 0, sizeof(*dpll_hw_state)); ++ ++ if (vco >= 6200000 && vco <= 6700000) { ++ prop_coef = 4; ++ int_coef = 9; ++ gain_ctl = 3; ++ targ_cnt = 8; ++ } else if ((vco > 5400000 && vco < 6200000) || ++ (vco >= 4800000 && vco < 5400000)) { ++ prop_coef = 5; ++ int_coef = 11; ++ gain_ctl = 3; ++ targ_cnt = 9; ++ } else if (vco == 5400000) { ++ prop_coef = 3; ++ int_coef = 8; ++ gain_ctl = 1; ++ targ_cnt = 9; ++ } else { ++ DRM_ERROR("Invalid VCO\n"); ++ return false; ++ } ++ ++ if (clock > 270000) ++ lanestagger = 0x18; ++ else if (clock > 135000) ++ lanestagger = 0x0d; ++ else if (clock > 67000) ++ lanestagger = 0x07; ++ else if (clock > 33000) ++ lanestagger = 0x04; ++ else ++ lanestagger = 0x02; ++ ++ dpll_hw_state->ebb0 = PORT_PLL_P1(clk_div->p1) | PORT_PLL_P2(clk_div->p2); ++ dpll_hw_state->pll0 = clk_div->m2_int; ++ dpll_hw_state->pll1 = PORT_PLL_N(clk_div->n); ++ dpll_hw_state->pll2 = clk_div->m2_frac; ++ ++ if (clk_div->m2_frac_en) ++ dpll_hw_state->pll3 = PORT_PLL_M2_FRAC_ENABLE; ++ ++ dpll_hw_state->pll6 = prop_coef | PORT_PLL_INT_COEFF(int_coef); ++ dpll_hw_state->pll6 |= PORT_PLL_GAIN_CTL(gain_ctl); ++ ++ dpll_hw_state->pll8 = targ_cnt; ++ ++ dpll_hw_state->pll9 = 5 << PORT_PLL_LOCK_THRESHOLD_SHIFT; ++ ++ dpll_hw_state->pll10 = ++ PORT_PLL_DCO_AMP(PORT_PLL_DCO_AMP_DEFAULT) ++ | PORT_PLL_DCO_AMP_OVR_EN_H; ++ ++ dpll_hw_state->ebb4 = PORT_PLL_10BIT_CLK_ENABLE; ++ ++ dpll_hw_state->pcsdw12 = LANESTAGGER_STRAP_OVRD | lanestagger; ++ ++ return true; ++} ++ ++static bool ++bxt_ddi_dp_set_dpll_hw_state(struct intel_crtc_state *crtc_state) ++{ ++ struct bxt_clk_div clk_div = {}; ++ ++ bxt_ddi_dp_pll_dividers(crtc_state, &clk_div); ++ ++ return bxt_ddi_set_dpll_hw_state(crtc_state, &clk_div); ++} ++ ++static bool ++bxt_ddi_hdmi_set_dpll_hw_state(struct intel_crtc_state *crtc_state) ++{ ++ struct bxt_clk_div clk_div = {}; ++ ++ bxt_ddi_hdmi_pll_dividers(crtc_state, &clk_div); ++ ++ return bxt_ddi_set_dpll_hw_state(crtc_state, &clk_div); ++} ++ ++static struct intel_shared_dpll * ++bxt_get_dpll(struct intel_crtc_state *crtc_state, ++ struct intel_encoder *encoder) ++{ ++ struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); ++ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); ++ struct intel_shared_dpll *pll; ++ enum intel_dpll_id id; ++ ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) && ++ !bxt_ddi_hdmi_set_dpll_hw_state(crtc_state)) ++ return NULL; ++ ++ if (intel_crtc_has_dp_encoder(crtc_state) && ++ !bxt_ddi_dp_set_dpll_hw_state(crtc_state)) ++ return NULL; ++ ++ /* 1:1 mapping between ports and PLLs */ ++ id = (enum intel_dpll_id) encoder->port; ++ pll = intel_get_shared_dpll_by_id(dev_priv, id); ++ ++ DRM_DEBUG_KMS("[CRTC:%d:%s] using pre-allocated %s\n", ++ crtc->base.base.id, crtc->base.name, pll->info->name); ++ ++ intel_reference_shared_dpll(pll, crtc_state); ++ ++ return pll; ++} ++ ++static void bxt_dump_hw_state(struct drm_i915_private *dev_priv, ++ struct intel_dpll_hw_state *hw_state) ++{ ++ DRM_DEBUG_KMS("dpll_hw_state: ebb0: 0x%x, ebb4: 0x%x," ++ "pll0: 0x%x, pll1: 0x%x, pll2: 0x%x, pll3: 0x%x, " ++ "pll6: 0x%x, pll8: 0x%x, pll9: 0x%x, pll10: 0x%x, pcsdw12: 0x%x\n", ++ hw_state->ebb0, ++ hw_state->ebb4, ++ hw_state->pll0, ++ hw_state->pll1, ++ hw_state->pll2, ++ hw_state->pll3, ++ hw_state->pll6, ++ hw_state->pll8, ++ hw_state->pll9, ++ hw_state->pll10, ++ hw_state->pcsdw12); ++} ++ ++static const struct intel_shared_dpll_funcs bxt_ddi_pll_funcs = { ++ .enable = bxt_ddi_pll_enable, ++ .disable = bxt_ddi_pll_disable, ++ .get_hw_state = bxt_ddi_pll_get_hw_state, ++}; ++ ++static void intel_ddi_pll_init(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ ++ if (INTEL_GEN(dev_priv) < 9) { ++ u32 val = I915_READ(LCPLL_CTL); ++ ++ /* ++ * The LCPLL register should be turned on by the BIOS. For now ++ * let's just check its state and print errors in case ++ * something is wrong. Don't even try to turn it on. ++ */ ++ ++ if (val & LCPLL_CD_SOURCE_FCLK) ++ DRM_ERROR("CDCLK source is not LCPLL\n"); ++ ++ if (val & LCPLL_PLL_DISABLE) ++ DRM_ERROR("LCPLL is disabled\n"); ++ } ++} ++ ++struct intel_dpll_mgr { ++ const struct dpll_info *dpll_info; ++ ++ struct intel_shared_dpll *(*get_dpll)(struct intel_crtc_state *crtc_state, ++ struct intel_encoder *encoder); ++ ++ void (*dump_hw_state)(struct drm_i915_private *dev_priv, ++ struct intel_dpll_hw_state *hw_state); ++}; ++ ++static const struct dpll_info pch_plls[] = { ++ { "PCH DPLL A", &ibx_pch_dpll_funcs, DPLL_ID_PCH_PLL_A, 0 }, ++ { "PCH DPLL B", &ibx_pch_dpll_funcs, DPLL_ID_PCH_PLL_B, 0 }, ++ { }, ++}; ++ ++static const struct intel_dpll_mgr pch_pll_mgr = { ++ .dpll_info = pch_plls, ++ .get_dpll = ibx_get_dpll, ++ .dump_hw_state = ibx_dump_hw_state, ++}; ++ ++static const struct dpll_info hsw_plls[] = { ++ { "WRPLL 1", &hsw_ddi_wrpll_funcs, DPLL_ID_WRPLL1, 0 }, ++ { "WRPLL 2", &hsw_ddi_wrpll_funcs, DPLL_ID_WRPLL2, 0 }, ++ { "SPLL", &hsw_ddi_spll_funcs, DPLL_ID_SPLL, 0 }, ++ { "LCPLL 810", &hsw_ddi_lcpll_funcs, DPLL_ID_LCPLL_810, INTEL_DPLL_ALWAYS_ON }, ++ { "LCPLL 1350", &hsw_ddi_lcpll_funcs, DPLL_ID_LCPLL_1350, INTEL_DPLL_ALWAYS_ON }, ++ { "LCPLL 2700", &hsw_ddi_lcpll_funcs, DPLL_ID_LCPLL_2700, INTEL_DPLL_ALWAYS_ON }, ++ { }, ++}; ++ ++static const struct intel_dpll_mgr hsw_pll_mgr = { ++ .dpll_info = hsw_plls, ++ .get_dpll = hsw_get_dpll, ++ .dump_hw_state = hsw_dump_hw_state, ++}; ++ ++static const struct dpll_info skl_plls[] = { ++ { "DPLL 0", &skl_ddi_dpll0_funcs, DPLL_ID_SKL_DPLL0, INTEL_DPLL_ALWAYS_ON }, ++ { "DPLL 1", &skl_ddi_pll_funcs, DPLL_ID_SKL_DPLL1, 0 }, ++ { "DPLL 2", &skl_ddi_pll_funcs, DPLL_ID_SKL_DPLL2, 0 }, ++ { "DPLL 3", &skl_ddi_pll_funcs, DPLL_ID_SKL_DPLL3, 0 }, ++ { }, ++}; ++ ++static const struct intel_dpll_mgr skl_pll_mgr = { ++ .dpll_info = skl_plls, ++ .get_dpll = skl_get_dpll, ++ .dump_hw_state = skl_dump_hw_state, ++}; ++ ++static const struct dpll_info bxt_plls[] = { ++ { "PORT PLL A", &bxt_ddi_pll_funcs, DPLL_ID_SKL_DPLL0, 0 }, ++ { "PORT PLL B", &bxt_ddi_pll_funcs, DPLL_ID_SKL_DPLL1, 0 }, ++ { "PORT PLL C", &bxt_ddi_pll_funcs, DPLL_ID_SKL_DPLL2, 0 }, ++ { }, ++}; ++ ++static const struct intel_dpll_mgr bxt_pll_mgr = { ++ .dpll_info = bxt_plls, ++ .get_dpll = bxt_get_dpll, ++ .dump_hw_state = bxt_dump_hw_state, ++}; ++ ++static void cnl_ddi_pll_enable(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++ const enum intel_dpll_id id = pll->info->id; ++ u32 val; ++ ++ /* 1. Enable DPLL power in DPLL_ENABLE. */ ++ val = I915_READ(CNL_DPLL_ENABLE(id)); ++ val |= PLL_POWER_ENABLE; ++ I915_WRITE(CNL_DPLL_ENABLE(id), val); ++ ++ /* 2. Wait for DPLL power state enabled in DPLL_ENABLE. */ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ CNL_DPLL_ENABLE(id), ++ PLL_POWER_STATE, ++ PLL_POWER_STATE, ++ 5)) ++ DRM_ERROR("PLL %d Power not enabled\n", id); ++ ++ /* ++ * 3. Configure DPLL_CFGCR0 to set SSC enable/disable, ++ * select DP mode, and set DP link rate. ++ */ ++ val = pll->state.hw_state.cfgcr0; ++ I915_WRITE(CNL_DPLL_CFGCR0(id), val); ++ ++ /* 4. Reab back to ensure writes completed */ ++ POSTING_READ(CNL_DPLL_CFGCR0(id)); ++ ++ /* 3. Configure DPLL_CFGCR0 */ ++ /* Avoid touch CFGCR1 if HDMI mode is not enabled */ ++ if (pll->state.hw_state.cfgcr0 & DPLL_CFGCR0_HDMI_MODE) { ++ val = pll->state.hw_state.cfgcr1; ++ I915_WRITE(CNL_DPLL_CFGCR1(id), val); ++ /* 4. Reab back to ensure writes completed */ ++ POSTING_READ(CNL_DPLL_CFGCR1(id)); ++ } ++ ++ /* ++ * 5. If the frequency will result in a change to the voltage ++ * requirement, follow the Display Voltage Frequency Switching ++ * Sequence Before Frequency Change ++ * ++ * Note: DVFS is actually handled via the cdclk code paths, ++ * hence we do nothing here. ++ */ ++ ++ /* 6. Enable DPLL in DPLL_ENABLE. */ ++ val = I915_READ(CNL_DPLL_ENABLE(id)); ++ val |= PLL_ENABLE; ++ I915_WRITE(CNL_DPLL_ENABLE(id), val); ++ ++ /* 7. Wait for PLL lock status in DPLL_ENABLE. */ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ CNL_DPLL_ENABLE(id), ++ PLL_LOCK, ++ PLL_LOCK, ++ 5)) ++ DRM_ERROR("PLL %d not locked\n", id); ++ ++ /* ++ * 8. If the frequency will result in a change to the voltage ++ * requirement, follow the Display Voltage Frequency Switching ++ * Sequence After Frequency Change ++ * ++ * Note: DVFS is actually handled via the cdclk code paths, ++ * hence we do nothing here. ++ */ ++ ++ /* ++ * 9. turn on the clock for the DDI and map the DPLL to the DDI ++ * Done at intel_ddi_clk_select ++ */ ++} ++ ++static void cnl_ddi_pll_disable(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++ const enum intel_dpll_id id = pll->info->id; ++ u32 val; ++ ++ /* ++ * 1. Configure DPCLKA_CFGCR0 to turn off the clock for the DDI. ++ * Done at intel_ddi_post_disable ++ */ ++ ++ /* ++ * 2. If the frequency will result in a change to the voltage ++ * requirement, follow the Display Voltage Frequency Switching ++ * Sequence Before Frequency Change ++ * ++ * Note: DVFS is actually handled via the cdclk code paths, ++ * hence we do nothing here. ++ */ ++ ++ /* 3. Disable DPLL through DPLL_ENABLE. */ ++ val = I915_READ(CNL_DPLL_ENABLE(id)); ++ val &= ~PLL_ENABLE; ++ I915_WRITE(CNL_DPLL_ENABLE(id), val); ++ ++ /* 4. Wait for PLL not locked status in DPLL_ENABLE. */ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ CNL_DPLL_ENABLE(id), ++ PLL_LOCK, ++ 0, ++ 5)) ++ DRM_ERROR("PLL %d locked\n", id); ++ ++ /* ++ * 5. If the frequency will result in a change to the voltage ++ * requirement, follow the Display Voltage Frequency Switching ++ * Sequence After Frequency Change ++ * ++ * Note: DVFS is actually handled via the cdclk code paths, ++ * hence we do nothing here. ++ */ ++ ++ /* 6. Disable DPLL power in DPLL_ENABLE. */ ++ val = I915_READ(CNL_DPLL_ENABLE(id)); ++ val &= ~PLL_POWER_ENABLE; ++ I915_WRITE(CNL_DPLL_ENABLE(id), val); ++ ++ /* 7. Wait for DPLL power state disabled in DPLL_ENABLE. */ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ CNL_DPLL_ENABLE(id), ++ PLL_POWER_STATE, ++ 0, ++ 5)) ++ DRM_ERROR("PLL %d Power not disabled\n", id); ++} ++ ++static bool cnl_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll, ++ struct intel_dpll_hw_state *hw_state) ++{ ++ const enum intel_dpll_id id = pll->info->id; ++ intel_wakeref_t wakeref; ++ u32 val; ++ bool ret; ++ ++ wakeref = intel_display_power_get_if_enabled(dev_priv, ++ POWER_DOMAIN_PLLS); ++ if (!wakeref) ++ return false; ++ ++ ret = false; ++ ++ val = I915_READ(CNL_DPLL_ENABLE(id)); ++ if (!(val & PLL_ENABLE)) ++ goto out; ++ ++ val = I915_READ(CNL_DPLL_CFGCR0(id)); ++ hw_state->cfgcr0 = val; ++ ++ /* avoid reading back stale values if HDMI mode is not enabled */ ++ if (val & DPLL_CFGCR0_HDMI_MODE) { ++ hw_state->cfgcr1 = I915_READ(CNL_DPLL_CFGCR1(id)); ++ } ++ ret = true; ++ ++out: ++ intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS, wakeref); ++ ++ return ret; ++} ++ ++static void cnl_wrpll_get_multipliers(int bestdiv, int *pdiv, ++ int *qdiv, int *kdiv) ++{ ++ /* even dividers */ ++ if (bestdiv % 2 == 0) { ++ if (bestdiv == 2) { ++ *pdiv = 2; ++ *qdiv = 1; ++ *kdiv = 1; ++ } else if (bestdiv % 4 == 0) { ++ *pdiv = 2; ++ *qdiv = bestdiv / 4; ++ *kdiv = 2; ++ } else if (bestdiv % 6 == 0) { ++ *pdiv = 3; ++ *qdiv = bestdiv / 6; ++ *kdiv = 2; ++ } else if (bestdiv % 5 == 0) { ++ *pdiv = 5; ++ *qdiv = bestdiv / 10; ++ *kdiv = 2; ++ } else if (bestdiv % 14 == 0) { ++ *pdiv = 7; ++ *qdiv = bestdiv / 14; ++ *kdiv = 2; ++ } ++ } else { ++ if (bestdiv == 3 || bestdiv == 5 || bestdiv == 7) { ++ *pdiv = bestdiv; ++ *qdiv = 1; ++ *kdiv = 1; ++ } else { /* 9, 15, 21 */ ++ *pdiv = bestdiv / 3; ++ *qdiv = 1; ++ *kdiv = 3; ++ } ++ } ++} ++ ++static void cnl_wrpll_params_populate(struct skl_wrpll_params *params, ++ u32 dco_freq, u32 ref_freq, ++ int pdiv, int qdiv, int kdiv) ++{ ++ u32 dco; ++ ++ switch (kdiv) { ++ case 1: ++ params->kdiv = 1; ++ break; ++ case 2: ++ params->kdiv = 2; ++ break; ++ case 3: ++ params->kdiv = 4; ++ break; ++ default: ++ WARN(1, "Incorrect KDiv\n"); ++ } ++ ++ switch (pdiv) { ++ case 2: ++ params->pdiv = 1; ++ break; ++ case 3: ++ params->pdiv = 2; ++ break; ++ case 5: ++ params->pdiv = 4; ++ break; ++ case 7: ++ params->pdiv = 8; ++ break; ++ default: ++ WARN(1, "Incorrect PDiv\n"); ++ } ++ ++ WARN_ON(kdiv != 2 && qdiv != 1); ++ ++ params->qdiv_ratio = qdiv; ++ params->qdiv_mode = (qdiv == 1) ? 0 : 1; ++ ++ dco = div_u64((u64)dco_freq << 15, ref_freq); ++ ++ params->dco_integer = dco >> 15; ++ params->dco_fraction = dco & 0x7fff; ++} ++ ++int cnl_hdmi_pll_ref_clock(struct drm_i915_private *dev_priv) ++{ ++ int ref_clock = dev_priv->cdclk.hw.ref; ++ ++ /* ++ * For ICL+, the spec states: if reference frequency is 38.4, ++ * use 19.2 because the DPLL automatically divides that by 2. ++ */ ++ if (INTEL_GEN(dev_priv) >= 11 && ref_clock == 38400) ++ ref_clock = 19200; ++ ++ return ref_clock; ++} ++ ++static bool ++cnl_ddi_calculate_wrpll(struct intel_crtc_state *crtc_state, ++ struct skl_wrpll_params *wrpll_params) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); ++ u32 afe_clock = crtc_state->port_clock * 5; ++ u32 ref_clock; ++ u32 dco_min = 7998000; ++ u32 dco_max = 10000000; ++ u32 dco_mid = (dco_min + dco_max) / 2; ++ static const int dividers[] = { 2, 4, 6, 8, 10, 12, 14, 16, ++ 18, 20, 24, 28, 30, 32, 36, 40, ++ 42, 44, 48, 50, 52, 54, 56, 60, ++ 64, 66, 68, 70, 72, 76, 78, 80, ++ 84, 88, 90, 92, 96, 98, 100, 102, ++ 3, 5, 7, 9, 15, 21 }; ++ u32 dco, best_dco = 0, dco_centrality = 0; ++ u32 best_dco_centrality = U32_MAX; /* Spec meaning of 999999 MHz */ ++ int d, best_div = 0, pdiv = 0, qdiv = 0, kdiv = 0; ++ ++ for (d = 0; d < ARRAY_SIZE(dividers); d++) { ++ dco = afe_clock * dividers[d]; ++ ++ if ((dco <= dco_max) && (dco >= dco_min)) { ++ dco_centrality = abs(dco - dco_mid); ++ ++ if (dco_centrality < best_dco_centrality) { ++ best_dco_centrality = dco_centrality; ++ best_div = dividers[d]; ++ best_dco = dco; ++ } ++ } ++ } ++ ++ if (best_div == 0) ++ return false; ++ ++ cnl_wrpll_get_multipliers(best_div, &pdiv, &qdiv, &kdiv); ++ ++ ref_clock = cnl_hdmi_pll_ref_clock(dev_priv); ++ ++ cnl_wrpll_params_populate(wrpll_params, best_dco, ref_clock, ++ pdiv, qdiv, kdiv); ++ ++ return true; ++} ++ ++static bool cnl_ddi_hdmi_pll_dividers(struct intel_crtc_state *crtc_state) ++{ ++ u32 cfgcr0, cfgcr1; ++ struct skl_wrpll_params wrpll_params = { 0, }; ++ ++ cfgcr0 = DPLL_CFGCR0_HDMI_MODE; ++ ++ if (!cnl_ddi_calculate_wrpll(crtc_state, &wrpll_params)) ++ return false; ++ ++ cfgcr0 |= DPLL_CFGCR0_DCO_FRACTION(wrpll_params.dco_fraction) | ++ wrpll_params.dco_integer; ++ ++ cfgcr1 = DPLL_CFGCR1_QDIV_RATIO(wrpll_params.qdiv_ratio) | ++ DPLL_CFGCR1_QDIV_MODE(wrpll_params.qdiv_mode) | ++ DPLL_CFGCR1_KDIV(wrpll_params.kdiv) | ++ DPLL_CFGCR1_PDIV(wrpll_params.pdiv) | ++ DPLL_CFGCR1_CENTRAL_FREQ; ++ ++ memset(&crtc_state->dpll_hw_state, 0, ++ sizeof(crtc_state->dpll_hw_state)); ++ ++ crtc_state->dpll_hw_state.cfgcr0 = cfgcr0; ++ crtc_state->dpll_hw_state.cfgcr1 = cfgcr1; ++ return true; ++} ++ ++static bool ++cnl_ddi_dp_set_dpll_hw_state(struct intel_crtc_state *crtc_state) ++{ ++ u32 cfgcr0; ++ ++ cfgcr0 = DPLL_CFGCR0_SSC_ENABLE; ++ ++ switch (crtc_state->port_clock / 2) { ++ case 81000: ++ cfgcr0 |= DPLL_CFGCR0_LINK_RATE_810; ++ break; ++ case 135000: ++ cfgcr0 |= DPLL_CFGCR0_LINK_RATE_1350; ++ break; ++ case 270000: ++ cfgcr0 |= DPLL_CFGCR0_LINK_RATE_2700; ++ break; ++ /* eDP 1.4 rates */ ++ case 162000: ++ cfgcr0 |= DPLL_CFGCR0_LINK_RATE_1620; ++ break; ++ case 108000: ++ cfgcr0 |= DPLL_CFGCR0_LINK_RATE_1080; ++ break; ++ case 216000: ++ cfgcr0 |= DPLL_CFGCR0_LINK_RATE_2160; ++ break; ++ case 324000: ++ /* Some SKUs may require elevated I/O voltage to support this */ ++ cfgcr0 |= DPLL_CFGCR0_LINK_RATE_3240; ++ break; ++ case 405000: ++ /* Some SKUs may require elevated I/O voltage to support this */ ++ cfgcr0 |= DPLL_CFGCR0_LINK_RATE_4050; ++ break; ++ } ++ ++ memset(&crtc_state->dpll_hw_state, 0, ++ sizeof(crtc_state->dpll_hw_state)); ++ ++ crtc_state->dpll_hw_state.cfgcr0 = cfgcr0; ++ ++ return true; ++} ++ ++static struct intel_shared_dpll * ++cnl_get_dpll(struct intel_crtc_state *crtc_state, ++ struct intel_encoder *encoder) ++{ ++ struct intel_shared_dpll *pll; ++ bool bret; ++ ++ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) { ++ bret = cnl_ddi_hdmi_pll_dividers(crtc_state); ++ if (!bret) { ++ DRM_DEBUG_KMS("Could not get HDMI pll dividers.\n"); ++ return NULL; ++ } ++ } else if (intel_crtc_has_dp_encoder(crtc_state)) { ++ bret = cnl_ddi_dp_set_dpll_hw_state(crtc_state); ++ if (!bret) { ++ DRM_DEBUG_KMS("Could not set DP dpll HW state.\n"); ++ return NULL; ++ } ++ } else { ++ DRM_DEBUG_KMS("Skip DPLL setup for output_types 0x%x\n", ++ crtc_state->output_types); ++ return NULL; ++ } ++ ++ pll = intel_find_shared_dpll(crtc_state, ++ DPLL_ID_SKL_DPLL0, ++ DPLL_ID_SKL_DPLL2); ++ if (!pll) { ++ DRM_DEBUG_KMS("No PLL selected\n"); ++ return NULL; ++ } ++ ++ intel_reference_shared_dpll(pll, crtc_state); ++ ++ return pll; ++} ++ ++static void cnl_dump_hw_state(struct drm_i915_private *dev_priv, ++ struct intel_dpll_hw_state *hw_state) ++{ ++ DRM_DEBUG_KMS("dpll_hw_state: " ++ "cfgcr0: 0x%x, cfgcr1: 0x%x\n", ++ hw_state->cfgcr0, ++ hw_state->cfgcr1); ++} ++ ++static const struct intel_shared_dpll_funcs cnl_ddi_pll_funcs = { ++ .enable = cnl_ddi_pll_enable, ++ .disable = cnl_ddi_pll_disable, ++ .get_hw_state = cnl_ddi_pll_get_hw_state, ++}; ++ ++static const struct dpll_info cnl_plls[] = { ++ { "DPLL 0", &cnl_ddi_pll_funcs, DPLL_ID_SKL_DPLL0, 0 }, ++ { "DPLL 1", &cnl_ddi_pll_funcs, DPLL_ID_SKL_DPLL1, 0 }, ++ { "DPLL 2", &cnl_ddi_pll_funcs, DPLL_ID_SKL_DPLL2, 0 }, ++ { }, ++}; ++ ++static const struct intel_dpll_mgr cnl_pll_mgr = { ++ .dpll_info = cnl_plls, ++ .get_dpll = cnl_get_dpll, ++ .dump_hw_state = cnl_dump_hw_state, ++}; ++ ++struct icl_combo_pll_params { ++ int clock; ++ struct skl_wrpll_params wrpll; ++}; ++ ++/* ++ * These values alrea already adjusted: they're the bits we write to the ++ * registers, not the logical values. ++ */ ++static const struct icl_combo_pll_params icl_dp_combo_pll_24MHz_values[] = { ++ { 540000, ++ { .dco_integer = 0x151, .dco_fraction = 0x4000, /* [0]: 5.4 */ ++ .pdiv = 0x2 /* 3 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0, }, }, ++ { 270000, ++ { .dco_integer = 0x151, .dco_fraction = 0x4000, /* [1]: 2.7 */ ++ .pdiv = 0x2 /* 3 */, .kdiv = 2, .qdiv_mode = 0, .qdiv_ratio = 0, }, }, ++ { 162000, ++ { .dco_integer = 0x151, .dco_fraction = 0x4000, /* [2]: 1.62 */ ++ .pdiv = 0x4 /* 5 */, .kdiv = 2, .qdiv_mode = 0, .qdiv_ratio = 0, }, }, ++ { 324000, ++ { .dco_integer = 0x151, .dco_fraction = 0x4000, /* [3]: 3.24 */ ++ .pdiv = 0x4 /* 5 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0, }, }, ++ { 216000, ++ { .dco_integer = 0x168, .dco_fraction = 0x0000, /* [4]: 2.16 */ ++ .pdiv = 0x1 /* 2 */, .kdiv = 2, .qdiv_mode = 1, .qdiv_ratio = 2, }, }, ++ { 432000, ++ { .dco_integer = 0x168, .dco_fraction = 0x0000, /* [5]: 4.32 */ ++ .pdiv = 0x1 /* 2 */, .kdiv = 2, .qdiv_mode = 0, .qdiv_ratio = 0, }, }, ++ { 648000, ++ { .dco_integer = 0x195, .dco_fraction = 0x0000, /* [6]: 6.48 */ ++ .pdiv = 0x2 /* 3 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0, }, }, ++ { 810000, ++ { .dco_integer = 0x151, .dco_fraction = 0x4000, /* [7]: 8.1 */ ++ .pdiv = 0x1 /* 2 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0, }, }, ++}; ++ ++ ++/* Also used for 38.4 MHz values. */ ++static const struct icl_combo_pll_params icl_dp_combo_pll_19_2MHz_values[] = { ++ { 540000, ++ { .dco_integer = 0x1A5, .dco_fraction = 0x7000, /* [0]: 5.4 */ ++ .pdiv = 0x2 /* 3 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0, }, }, ++ { 270000, ++ { .dco_integer = 0x1A5, .dco_fraction = 0x7000, /* [1]: 2.7 */ ++ .pdiv = 0x2 /* 3 */, .kdiv = 2, .qdiv_mode = 0, .qdiv_ratio = 0, }, }, ++ { 162000, ++ { .dco_integer = 0x1A5, .dco_fraction = 0x7000, /* [2]: 1.62 */ ++ .pdiv = 0x4 /* 5 */, .kdiv = 2, .qdiv_mode = 0, .qdiv_ratio = 0, }, }, ++ { 324000, ++ { .dco_integer = 0x1A5, .dco_fraction = 0x7000, /* [3]: 3.24 */ ++ .pdiv = 0x4 /* 5 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0, }, }, ++ { 216000, ++ { .dco_integer = 0x1C2, .dco_fraction = 0x0000, /* [4]: 2.16 */ ++ .pdiv = 0x1 /* 2 */, .kdiv = 2, .qdiv_mode = 1, .qdiv_ratio = 2, }, }, ++ { 432000, ++ { .dco_integer = 0x1C2, .dco_fraction = 0x0000, /* [5]: 4.32 */ ++ .pdiv = 0x1 /* 2 */, .kdiv = 2, .qdiv_mode = 0, .qdiv_ratio = 0, }, }, ++ { 648000, ++ { .dco_integer = 0x1FA, .dco_fraction = 0x2000, /* [6]: 6.48 */ ++ .pdiv = 0x2 /* 3 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0, }, }, ++ { 810000, ++ { .dco_integer = 0x1A5, .dco_fraction = 0x7000, /* [7]: 8.1 */ ++ .pdiv = 0x1 /* 2 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0, }, }, ++}; ++ ++static const struct skl_wrpll_params icl_tbt_pll_24MHz_values = { ++ .dco_integer = 0x151, .dco_fraction = 0x4000, ++ .pdiv = 0x4 /* 5 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0, ++}; ++ ++static const struct skl_wrpll_params icl_tbt_pll_19_2MHz_values = { ++ .dco_integer = 0x1A5, .dco_fraction = 0x7000, ++ .pdiv = 0x4 /* 5 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0, ++}; ++ ++static bool icl_calc_dp_combo_pll(struct intel_crtc_state *crtc_state, ++ struct skl_wrpll_params *pll_params) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); ++ const struct icl_combo_pll_params *params = ++ dev_priv->cdclk.hw.ref == 24000 ? ++ icl_dp_combo_pll_24MHz_values : ++ icl_dp_combo_pll_19_2MHz_values; ++ int clock = crtc_state->port_clock; ++ int i; ++ ++ for (i = 0; i < ARRAY_SIZE(icl_dp_combo_pll_24MHz_values); i++) { ++ if (clock == params[i].clock) { ++ *pll_params = params[i].wrpll; ++ return true; ++ } ++ } ++ ++ MISSING_CASE(clock); ++ return false; ++} ++ ++static bool icl_calc_tbt_pll(struct intel_crtc_state *crtc_state, ++ struct skl_wrpll_params *pll_params) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); ++ ++ *pll_params = dev_priv->cdclk.hw.ref == 24000 ? ++ icl_tbt_pll_24MHz_values : icl_tbt_pll_19_2MHz_values; ++ return true; ++} ++ ++static bool icl_calc_dpll_state(struct intel_crtc_state *crtc_state, ++ struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); ++ u32 cfgcr0, cfgcr1; ++ struct skl_wrpll_params pll_params = { 0 }; ++ bool ret; ++ ++ if (intel_port_is_tc(dev_priv, encoder->port)) ++ ret = icl_calc_tbt_pll(crtc_state, &pll_params); ++ else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) || ++ intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI)) ++ ret = cnl_ddi_calculate_wrpll(crtc_state, &pll_params); ++ else ++ ret = icl_calc_dp_combo_pll(crtc_state, &pll_params); ++ ++ if (!ret) ++ return false; ++ ++ cfgcr0 = DPLL_CFGCR0_DCO_FRACTION(pll_params.dco_fraction) | ++ pll_params.dco_integer; ++ ++ cfgcr1 = DPLL_CFGCR1_QDIV_RATIO(pll_params.qdiv_ratio) | ++ DPLL_CFGCR1_QDIV_MODE(pll_params.qdiv_mode) | ++ DPLL_CFGCR1_KDIV(pll_params.kdiv) | ++ DPLL_CFGCR1_PDIV(pll_params.pdiv) | ++ DPLL_CFGCR1_CENTRAL_FREQ_8400; ++ ++ memset(&crtc_state->dpll_hw_state, 0, ++ sizeof(crtc_state->dpll_hw_state)); ++ ++ crtc_state->dpll_hw_state.cfgcr0 = cfgcr0; ++ crtc_state->dpll_hw_state.cfgcr1 = cfgcr1; ++ ++ return true; ++} ++ ++ ++static enum tc_port icl_pll_id_to_tc_port(enum intel_dpll_id id) ++{ ++ return id - DPLL_ID_ICL_MGPLL1; ++} ++ ++enum intel_dpll_id icl_tc_port_to_pll_id(enum tc_port tc_port) ++{ ++ return tc_port + DPLL_ID_ICL_MGPLL1; ++} ++ ++static bool icl_mg_pll_find_divisors(int clock_khz, bool is_dp, bool use_ssc, ++ u32 *target_dco_khz, ++ struct intel_dpll_hw_state *state) ++{ ++ u32 dco_min_freq, dco_max_freq; ++ int div1_vals[] = {7, 5, 3, 2}; ++ unsigned int i; ++ int div2; ++ ++ dco_min_freq = is_dp ? 8100000 : use_ssc ? 8000000 : 7992000; ++ dco_max_freq = is_dp ? 8100000 : 10000000; ++ ++ for (i = 0; i < ARRAY_SIZE(div1_vals); i++) { ++ int div1 = div1_vals[i]; ++ ++ for (div2 = 10; div2 > 0; div2--) { ++ int dco = div1 * div2 * clock_khz * 5; ++ int a_divratio, tlinedrv, inputsel; ++ u32 hsdiv; ++ ++ if (dco < dco_min_freq || dco > dco_max_freq) ++ continue; ++ ++ if (div2 >= 2) { ++ a_divratio = is_dp ? 10 : 5; ++ tlinedrv = 2; ++ } else { ++ a_divratio = 5; ++ tlinedrv = 0; ++ } ++ inputsel = is_dp ? 0 : 1; ++ ++ switch (div1) { ++ default: ++ MISSING_CASE(div1); ++ /* fall through */ ++ case 2: ++ hsdiv = MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_2; ++ break; ++ case 3: ++ hsdiv = MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_3; ++ break; ++ case 5: ++ hsdiv = MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_5; ++ break; ++ case 7: ++ hsdiv = MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_7; ++ break; ++ } ++ ++ *target_dco_khz = dco; ++ ++ state->mg_refclkin_ctl = MG_REFCLKIN_CTL_OD_2_MUX(1); ++ ++ state->mg_clktop2_coreclkctl1 = ++ MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO(a_divratio); ++ ++ state->mg_clktop2_hsclkctl = ++ MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL(tlinedrv) | ++ MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL(inputsel) | ++ hsdiv | ++ MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO(div2); ++ ++ return true; ++ } ++ } ++ ++ return false; ++} ++ ++/* ++ * The specification for this function uses real numbers, so the math had to be ++ * adapted to integer-only calculation, that's why it looks so different. ++ */ ++static bool icl_calc_mg_pll_state(struct intel_crtc_state *crtc_state) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); ++ struct intel_dpll_hw_state *pll_state = &crtc_state->dpll_hw_state; ++ int refclk_khz = dev_priv->cdclk.hw.ref; ++ int clock = crtc_state->port_clock; ++ u32 dco_khz, m1div, m2div_int, m2div_rem, m2div_frac; ++ u32 iref_ndiv, iref_trim, iref_pulse_w; ++ u32 prop_coeff, int_coeff; ++ u32 tdc_targetcnt, feedfwgain; ++ u64 ssc_stepsize, ssc_steplen, ssc_steplog; ++ u64 tmp; ++ bool use_ssc = false; ++ bool is_dp = !intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI); ++ ++ memset(pll_state, 0, sizeof(*pll_state)); ++ ++ if (!icl_mg_pll_find_divisors(clock, is_dp, use_ssc, &dco_khz, ++ pll_state)) { ++ DRM_DEBUG_KMS("Failed to find divisors for clock %d\n", clock); ++ return false; ++ } ++ ++ m1div = 2; ++ m2div_int = dco_khz / (refclk_khz * m1div); ++ if (m2div_int > 255) { ++ m1div = 4; ++ m2div_int = dco_khz / (refclk_khz * m1div); ++ if (m2div_int > 255) { ++ DRM_DEBUG_KMS("Failed to find mdiv for clock %d\n", ++ clock); ++ return false; ++ } ++ } ++ m2div_rem = dco_khz % (refclk_khz * m1div); ++ ++ tmp = (u64)m2div_rem * (1 << 22); ++ do_div(tmp, refclk_khz * m1div); ++ m2div_frac = tmp; ++ ++ switch (refclk_khz) { ++ case 19200: ++ iref_ndiv = 1; ++ iref_trim = 28; ++ iref_pulse_w = 1; ++ break; ++ case 24000: ++ iref_ndiv = 1; ++ iref_trim = 25; ++ iref_pulse_w = 2; ++ break; ++ case 38400: ++ iref_ndiv = 2; ++ iref_trim = 28; ++ iref_pulse_w = 1; ++ break; ++ default: ++ MISSING_CASE(refclk_khz); ++ return false; ++ } ++ ++ /* ++ * tdc_res = 0.000003 ++ * tdc_targetcnt = int(2 / (tdc_res * 8 * 50 * 1.1) / refclk_mhz + 0.5) ++ * ++ * The multiplication by 1000 is due to refclk MHz to KHz conversion. It ++ * was supposed to be a division, but we rearranged the operations of ++ * the formula to avoid early divisions so we don't multiply the ++ * rounding errors. ++ * ++ * 0.000003 * 8 * 50 * 1.1 = 0.00132, also known as 132 / 100000, which ++ * we also rearrange to work with integers. ++ * ++ * The 0.5 transformed to 5 results in a multiplication by 10 and the ++ * last division by 10. ++ */ ++ tdc_targetcnt = (2 * 1000 * 100000 * 10 / (132 * refclk_khz) + 5) / 10; ++ ++ /* ++ * Here we divide dco_khz by 10 in order to allow the dividend to fit in ++ * 32 bits. That's not a problem since we round the division down ++ * anyway. ++ */ ++ feedfwgain = (use_ssc || m2div_rem > 0) ? ++ m1div * 1000000 * 100 / (dco_khz * 3 / 10) : 0; ++ ++ if (dco_khz >= 9000000) { ++ prop_coeff = 5; ++ int_coeff = 10; ++ } else { ++ prop_coeff = 4; ++ int_coeff = 8; ++ } ++ ++ if (use_ssc) { ++ tmp = (u64)dco_khz * 47 * 32; ++ do_div(tmp, refclk_khz * m1div * 10000); ++ ssc_stepsize = tmp; ++ ++ tmp = (u64)dco_khz * 1000; ++ ssc_steplen = DIV_ROUND_UP_ULL(tmp, 32 * 2 * 32); ++ } else { ++ ssc_stepsize = 0; ++ ssc_steplen = 0; ++ } ++ ssc_steplog = 4; ++ ++ pll_state->mg_pll_div0 = (m2div_rem > 0 ? MG_PLL_DIV0_FRACNEN_H : 0) | ++ MG_PLL_DIV0_FBDIV_FRAC(m2div_frac) | ++ MG_PLL_DIV0_FBDIV_INT(m2div_int); ++ ++ pll_state->mg_pll_div1 = MG_PLL_DIV1_IREF_NDIVRATIO(iref_ndiv) | ++ MG_PLL_DIV1_DITHER_DIV_2 | ++ MG_PLL_DIV1_NDIVRATIO(1) | ++ MG_PLL_DIV1_FBPREDIV(m1div); ++ ++ pll_state->mg_pll_lf = MG_PLL_LF_TDCTARGETCNT(tdc_targetcnt) | ++ MG_PLL_LF_AFCCNTSEL_512 | ++ MG_PLL_LF_GAINCTRL(1) | ++ MG_PLL_LF_INT_COEFF(int_coeff) | ++ MG_PLL_LF_PROP_COEFF(prop_coeff); ++ ++ pll_state->mg_pll_frac_lock = MG_PLL_FRAC_LOCK_TRUELOCK_CRIT_32 | ++ MG_PLL_FRAC_LOCK_EARLYLOCK_CRIT_32 | ++ MG_PLL_FRAC_LOCK_LOCKTHRESH(10) | ++ MG_PLL_FRAC_LOCK_DCODITHEREN | ++ MG_PLL_FRAC_LOCK_FEEDFWRDGAIN(feedfwgain); ++ if (use_ssc || m2div_rem > 0) ++ pll_state->mg_pll_frac_lock |= MG_PLL_FRAC_LOCK_FEEDFWRDCAL_EN; ++ ++ pll_state->mg_pll_ssc = (use_ssc ? MG_PLL_SSC_EN : 0) | ++ MG_PLL_SSC_TYPE(2) | ++ MG_PLL_SSC_STEPLENGTH(ssc_steplen) | ++ MG_PLL_SSC_STEPNUM(ssc_steplog) | ++ MG_PLL_SSC_FLLEN | ++ MG_PLL_SSC_STEPSIZE(ssc_stepsize); ++ ++ pll_state->mg_pll_tdc_coldst_bias = MG_PLL_TDC_COLDST_COLDSTART | ++ MG_PLL_TDC_COLDST_IREFINT_EN | ++ MG_PLL_TDC_COLDST_REFBIAS_START_PULSE_W(iref_pulse_w) | ++ MG_PLL_TDC_TDCOVCCORR_EN | ++ MG_PLL_TDC_TDCSEL(3); ++ ++ pll_state->mg_pll_bias = MG_PLL_BIAS_BIAS_GB_SEL(3) | ++ MG_PLL_BIAS_INIT_DCOAMP(0x3F) | ++ MG_PLL_BIAS_BIAS_BONUS(10) | ++ MG_PLL_BIAS_BIASCAL_EN | ++ MG_PLL_BIAS_CTRIM(12) | ++ MG_PLL_BIAS_VREF_RDAC(4) | ++ MG_PLL_BIAS_IREFTRIM(iref_trim); ++ ++ if (refclk_khz == 38400) { ++ pll_state->mg_pll_tdc_coldst_bias_mask = MG_PLL_TDC_COLDST_COLDSTART; ++ pll_state->mg_pll_bias_mask = 0; ++ } else { ++ pll_state->mg_pll_tdc_coldst_bias_mask = -1U; ++ pll_state->mg_pll_bias_mask = -1U; ++ } ++ ++ pll_state->mg_pll_tdc_coldst_bias &= pll_state->mg_pll_tdc_coldst_bias_mask; ++ pll_state->mg_pll_bias &= pll_state->mg_pll_bias_mask; ++ ++ return true; ++} ++ ++static struct intel_shared_dpll * ++icl_get_dpll(struct intel_crtc_state *crtc_state, ++ struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); ++ struct intel_digital_port *intel_dig_port; ++ struct intel_shared_dpll *pll; ++ enum port port = encoder->port; ++ enum intel_dpll_id min, max; ++ bool ret; ++ ++ if (intel_port_is_combophy(dev_priv, port)) { ++ min = DPLL_ID_ICL_DPLL0; ++ max = DPLL_ID_ICL_DPLL1; ++ ret = icl_calc_dpll_state(crtc_state, encoder); ++ } else if (intel_port_is_tc(dev_priv, port)) { ++ if (encoder->type == INTEL_OUTPUT_DP_MST) { ++ struct intel_dp_mst_encoder *mst_encoder; ++ ++ mst_encoder = enc_to_mst(&encoder->base); ++ intel_dig_port = mst_encoder->primary; ++ } else { ++ intel_dig_port = enc_to_dig_port(&encoder->base); ++ } ++ ++ if (intel_dig_port->tc_type == TC_PORT_TBT) { ++ min = DPLL_ID_ICL_TBTPLL; ++ max = min; ++ ret = icl_calc_dpll_state(crtc_state, encoder); ++ } else { ++ enum tc_port tc_port; ++ ++ tc_port = intel_port_to_tc(dev_priv, port); ++ min = icl_tc_port_to_pll_id(tc_port); ++ max = min; ++ ret = icl_calc_mg_pll_state(crtc_state); ++ } ++ } else { ++ MISSING_CASE(port); ++ return NULL; ++ } ++ ++ if (!ret) { ++ DRM_DEBUG_KMS("Could not calculate PLL state.\n"); ++ return NULL; ++ } ++ ++ ++ pll = intel_find_shared_dpll(crtc_state, min, max); ++ if (!pll) { ++ DRM_DEBUG_KMS("No PLL selected\n"); ++ return NULL; ++ } ++ ++ intel_reference_shared_dpll(pll, crtc_state); ++ ++ return pll; ++} ++ ++static bool mg_pll_get_hw_state(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll, ++ struct intel_dpll_hw_state *hw_state) ++{ ++ const enum intel_dpll_id id = pll->info->id; ++ enum tc_port tc_port = icl_pll_id_to_tc_port(id); ++ intel_wakeref_t wakeref; ++ bool ret = false; ++ u32 val; ++ ++ wakeref = intel_display_power_get_if_enabled(dev_priv, ++ POWER_DOMAIN_PLLS); ++ if (!wakeref) ++ return false; ++ ++ val = I915_READ(MG_PLL_ENABLE(tc_port)); ++ if (!(val & PLL_ENABLE)) ++ goto out; ++ ++ hw_state->mg_refclkin_ctl = I915_READ(MG_REFCLKIN_CTL(tc_port)); ++ hw_state->mg_refclkin_ctl &= MG_REFCLKIN_CTL_OD_2_MUX_MASK; ++ ++ hw_state->mg_clktop2_coreclkctl1 = ++ I915_READ(MG_CLKTOP2_CORECLKCTL1(tc_port)); ++ hw_state->mg_clktop2_coreclkctl1 &= ++ MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO_MASK; ++ ++ hw_state->mg_clktop2_hsclkctl = ++ I915_READ(MG_CLKTOP2_HSCLKCTL(tc_port)); ++ hw_state->mg_clktop2_hsclkctl &= ++ MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL_MASK | ++ MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL_MASK | ++ MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_MASK | ++ MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK; ++ ++ hw_state->mg_pll_div0 = I915_READ(MG_PLL_DIV0(tc_port)); ++ hw_state->mg_pll_div1 = I915_READ(MG_PLL_DIV1(tc_port)); ++ hw_state->mg_pll_lf = I915_READ(MG_PLL_LF(tc_port)); ++ hw_state->mg_pll_frac_lock = I915_READ(MG_PLL_FRAC_LOCK(tc_port)); ++ hw_state->mg_pll_ssc = I915_READ(MG_PLL_SSC(tc_port)); ++ ++ hw_state->mg_pll_bias = I915_READ(MG_PLL_BIAS(tc_port)); ++ hw_state->mg_pll_tdc_coldst_bias = ++ I915_READ(MG_PLL_TDC_COLDST_BIAS(tc_port)); ++ ++ if (dev_priv->cdclk.hw.ref == 38400) { ++ hw_state->mg_pll_tdc_coldst_bias_mask = MG_PLL_TDC_COLDST_COLDSTART; ++ hw_state->mg_pll_bias_mask = 0; ++ } else { ++ hw_state->mg_pll_tdc_coldst_bias_mask = -1U; ++ hw_state->mg_pll_bias_mask = -1U; ++ } ++ ++ hw_state->mg_pll_tdc_coldst_bias &= hw_state->mg_pll_tdc_coldst_bias_mask; ++ hw_state->mg_pll_bias &= hw_state->mg_pll_bias_mask; ++ ++ ret = true; ++out: ++ intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS, wakeref); ++ return ret; ++} ++ ++static bool icl_pll_get_hw_state(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll, ++ struct intel_dpll_hw_state *hw_state, ++ i915_reg_t enable_reg) ++{ ++ const enum intel_dpll_id id = pll->info->id; ++ intel_wakeref_t wakeref; ++ bool ret = false; ++ u32 val; ++ ++ wakeref = intel_display_power_get_if_enabled(dev_priv, ++ POWER_DOMAIN_PLLS); ++ if (!wakeref) ++ return false; ++ ++ val = I915_READ(enable_reg); ++ if (!(val & PLL_ENABLE)) ++ goto out; ++ ++ hw_state->cfgcr0 = I915_READ(ICL_DPLL_CFGCR0(id)); ++ hw_state->cfgcr1 = I915_READ(ICL_DPLL_CFGCR1(id)); ++ ++ ret = true; ++out: ++ intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS, wakeref); ++ return ret; ++} ++ ++static bool combo_pll_get_hw_state(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll, ++ struct intel_dpll_hw_state *hw_state) ++{ ++ return icl_pll_get_hw_state(dev_priv, pll, hw_state, ++ CNL_DPLL_ENABLE(pll->info->id)); ++} ++ ++static bool tbt_pll_get_hw_state(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll, ++ struct intel_dpll_hw_state *hw_state) ++{ ++ return icl_pll_get_hw_state(dev_priv, pll, hw_state, TBT_PLL_ENABLE); ++} ++ ++static void icl_dpll_write(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++ struct intel_dpll_hw_state *hw_state = &pll->state.hw_state; ++ const enum intel_dpll_id id = pll->info->id; ++ ++ I915_WRITE(ICL_DPLL_CFGCR0(id), hw_state->cfgcr0); ++ I915_WRITE(ICL_DPLL_CFGCR1(id), hw_state->cfgcr1); ++ POSTING_READ(ICL_DPLL_CFGCR1(id)); ++} ++ ++static void icl_mg_pll_write(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++ struct intel_dpll_hw_state *hw_state = &pll->state.hw_state; ++ enum tc_port tc_port = icl_pll_id_to_tc_port(pll->info->id); ++ u32 val; ++ ++ /* ++ * Some of the following registers have reserved fields, so program ++ * these with RMW based on a mask. The mask can be fixed or generated ++ * during the calc/readout phase if the mask depends on some other HW ++ * state like refclk, see icl_calc_mg_pll_state(). ++ */ ++ val = I915_READ(MG_REFCLKIN_CTL(tc_port)); ++ val &= ~MG_REFCLKIN_CTL_OD_2_MUX_MASK; ++ val |= hw_state->mg_refclkin_ctl; ++ I915_WRITE(MG_REFCLKIN_CTL(tc_port), val); ++ ++ val = I915_READ(MG_CLKTOP2_CORECLKCTL1(tc_port)); ++ val &= ~MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO_MASK; ++ val |= hw_state->mg_clktop2_coreclkctl1; ++ I915_WRITE(MG_CLKTOP2_CORECLKCTL1(tc_port), val); ++ ++ val = I915_READ(MG_CLKTOP2_HSCLKCTL(tc_port)); ++ val &= ~(MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL_MASK | ++ MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL_MASK | ++ MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_MASK | ++ MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK); ++ val |= hw_state->mg_clktop2_hsclkctl; ++ I915_WRITE(MG_CLKTOP2_HSCLKCTL(tc_port), val); ++ ++ I915_WRITE(MG_PLL_DIV0(tc_port), hw_state->mg_pll_div0); ++ I915_WRITE(MG_PLL_DIV1(tc_port), hw_state->mg_pll_div1); ++ I915_WRITE(MG_PLL_LF(tc_port), hw_state->mg_pll_lf); ++ I915_WRITE(MG_PLL_FRAC_LOCK(tc_port), hw_state->mg_pll_frac_lock); ++ I915_WRITE(MG_PLL_SSC(tc_port), hw_state->mg_pll_ssc); ++ ++ val = I915_READ(MG_PLL_BIAS(tc_port)); ++ val &= ~hw_state->mg_pll_bias_mask; ++ val |= hw_state->mg_pll_bias; ++ I915_WRITE(MG_PLL_BIAS(tc_port), val); ++ ++ val = I915_READ(MG_PLL_TDC_COLDST_BIAS(tc_port)); ++ val &= ~hw_state->mg_pll_tdc_coldst_bias_mask; ++ val |= hw_state->mg_pll_tdc_coldst_bias; ++ I915_WRITE(MG_PLL_TDC_COLDST_BIAS(tc_port), val); ++ ++ POSTING_READ(MG_PLL_TDC_COLDST_BIAS(tc_port)); ++} ++ ++static void icl_pll_power_enable(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll, ++ i915_reg_t enable_reg) ++{ ++ u32 val; ++ ++ val = I915_READ(enable_reg); ++ val |= PLL_POWER_ENABLE; ++ I915_WRITE(enable_reg, val); ++ ++ /* ++ * The spec says we need to "wait" but it also says it should be ++ * immediate. ++ */ ++ if (intel_wait_for_register(&dev_priv->uncore, enable_reg, ++ PLL_POWER_STATE, PLL_POWER_STATE, 1)) ++ DRM_ERROR("PLL %d Power not enabled\n", pll->info->id); ++} ++ ++static void icl_pll_enable(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll, ++ i915_reg_t enable_reg) ++{ ++ u32 val; ++ ++ val = I915_READ(enable_reg); ++ val |= PLL_ENABLE; ++ I915_WRITE(enable_reg, val); ++ ++ /* Timeout is actually 600us. */ ++ if (intel_wait_for_register(&dev_priv->uncore, enable_reg, ++ PLL_LOCK, PLL_LOCK, 1)) ++ DRM_ERROR("PLL %d not locked\n", pll->info->id); ++} ++ ++static void combo_pll_enable(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++ i915_reg_t enable_reg = CNL_DPLL_ENABLE(pll->info->id); ++ ++ icl_pll_power_enable(dev_priv, pll, enable_reg); ++ ++ icl_dpll_write(dev_priv, pll); ++ ++ /* ++ * DVFS pre sequence would be here, but in our driver the cdclk code ++ * paths should already be setting the appropriate voltage, hence we do ++ * nothing here. ++ */ ++ ++ icl_pll_enable(dev_priv, pll, enable_reg); ++ ++ /* DVFS post sequence would be here. See the comment above. */ ++} ++ ++static void tbt_pll_enable(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++ icl_pll_power_enable(dev_priv, pll, TBT_PLL_ENABLE); ++ ++ icl_dpll_write(dev_priv, pll); ++ ++ /* ++ * DVFS pre sequence would be here, but in our driver the cdclk code ++ * paths should already be setting the appropriate voltage, hence we do ++ * nothing here. ++ */ ++ ++ icl_pll_enable(dev_priv, pll, TBT_PLL_ENABLE); ++ ++ /* DVFS post sequence would be here. See the comment above. */ ++} ++ ++static void mg_pll_enable(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++ i915_reg_t enable_reg = ++ MG_PLL_ENABLE(icl_pll_id_to_tc_port(pll->info->id)); ++ ++ icl_pll_power_enable(dev_priv, pll, enable_reg); ++ ++ icl_mg_pll_write(dev_priv, pll); ++ ++ /* ++ * DVFS pre sequence would be here, but in our driver the cdclk code ++ * paths should already be setting the appropriate voltage, hence we do ++ * nothing here. ++ */ ++ ++ icl_pll_enable(dev_priv, pll, enable_reg); ++ ++ /* DVFS post sequence would be here. See the comment above. */ ++} ++ ++static void icl_pll_disable(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll, ++ i915_reg_t enable_reg) ++{ ++ u32 val; ++ ++ /* The first steps are done by intel_ddi_post_disable(). */ ++ ++ /* ++ * DVFS pre sequence would be here, but in our driver the cdclk code ++ * paths should already be setting the appropriate voltage, hence we do ++ * nothign here. ++ */ ++ ++ val = I915_READ(enable_reg); ++ val &= ~PLL_ENABLE; ++ I915_WRITE(enable_reg, val); ++ ++ /* Timeout is actually 1us. */ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ enable_reg, PLL_LOCK, 0, 1)) ++ DRM_ERROR("PLL %d locked\n", pll->info->id); ++ ++ /* DVFS post sequence would be here. See the comment above. */ ++ ++ val = I915_READ(enable_reg); ++ val &= ~PLL_POWER_ENABLE; ++ I915_WRITE(enable_reg, val); ++ ++ /* ++ * The spec says we need to "wait" but it also says it should be ++ * immediate. ++ */ ++ if (intel_wait_for_register(&dev_priv->uncore, ++ enable_reg, PLL_POWER_STATE, 0, 1)) ++ DRM_ERROR("PLL %d Power not disabled\n", pll->info->id); ++} ++ ++static void combo_pll_disable(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++ icl_pll_disable(dev_priv, pll, CNL_DPLL_ENABLE(pll->info->id)); ++} ++ ++static void tbt_pll_disable(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++ icl_pll_disable(dev_priv, pll, TBT_PLL_ENABLE); ++} ++ ++static void mg_pll_disable(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll) ++{ ++ i915_reg_t enable_reg = ++ MG_PLL_ENABLE(icl_pll_id_to_tc_port(pll->info->id)); ++ ++ icl_pll_disable(dev_priv, pll, enable_reg); ++} ++ ++static void icl_dump_hw_state(struct drm_i915_private *dev_priv, ++ struct intel_dpll_hw_state *hw_state) ++{ ++ DRM_DEBUG_KMS("dpll_hw_state: cfgcr0: 0x%x, cfgcr1: 0x%x, " ++ "mg_refclkin_ctl: 0x%x, hg_clktop2_coreclkctl1: 0x%x, " ++ "mg_clktop2_hsclkctl: 0x%x, mg_pll_div0: 0x%x, " ++ "mg_pll_div2: 0x%x, mg_pll_lf: 0x%x, " ++ "mg_pll_frac_lock: 0x%x, mg_pll_ssc: 0x%x, " ++ "mg_pll_bias: 0x%x, mg_pll_tdc_coldst_bias: 0x%x\n", ++ hw_state->cfgcr0, hw_state->cfgcr1, ++ hw_state->mg_refclkin_ctl, ++ hw_state->mg_clktop2_coreclkctl1, ++ hw_state->mg_clktop2_hsclkctl, ++ hw_state->mg_pll_div0, ++ hw_state->mg_pll_div1, ++ hw_state->mg_pll_lf, ++ hw_state->mg_pll_frac_lock, ++ hw_state->mg_pll_ssc, ++ hw_state->mg_pll_bias, ++ hw_state->mg_pll_tdc_coldst_bias); ++} ++ ++static const struct intel_shared_dpll_funcs combo_pll_funcs = { ++ .enable = combo_pll_enable, ++ .disable = combo_pll_disable, ++ .get_hw_state = combo_pll_get_hw_state, ++}; ++ ++static const struct intel_shared_dpll_funcs tbt_pll_funcs = { ++ .enable = tbt_pll_enable, ++ .disable = tbt_pll_disable, ++ .get_hw_state = tbt_pll_get_hw_state, ++}; ++ ++static const struct intel_shared_dpll_funcs mg_pll_funcs = { ++ .enable = mg_pll_enable, ++ .disable = mg_pll_disable, ++ .get_hw_state = mg_pll_get_hw_state, ++}; ++ ++static const struct dpll_info icl_plls[] = { ++ { "DPLL 0", &combo_pll_funcs, DPLL_ID_ICL_DPLL0, 0 }, ++ { "DPLL 1", &combo_pll_funcs, DPLL_ID_ICL_DPLL1, 0 }, ++ { "TBT PLL", &tbt_pll_funcs, DPLL_ID_ICL_TBTPLL, 0 }, ++ { "MG PLL 1", &mg_pll_funcs, DPLL_ID_ICL_MGPLL1, 0 }, ++ { "MG PLL 2", &mg_pll_funcs, DPLL_ID_ICL_MGPLL2, 0 }, ++ { "MG PLL 3", &mg_pll_funcs, DPLL_ID_ICL_MGPLL3, 0 }, ++ { "MG PLL 4", &mg_pll_funcs, DPLL_ID_ICL_MGPLL4, 0 }, ++ { }, ++}; ++ ++static const struct intel_dpll_mgr icl_pll_mgr = { ++ .dpll_info = icl_plls, ++ .get_dpll = icl_get_dpll, ++ .dump_hw_state = icl_dump_hw_state, ++}; ++ ++static const struct dpll_info ehl_plls[] = { ++ { "DPLL 0", &combo_pll_funcs, DPLL_ID_ICL_DPLL0, 0 }, ++ { "DPLL 1", &combo_pll_funcs, DPLL_ID_ICL_DPLL1, 0 }, ++ { }, ++}; ++ ++static const struct intel_dpll_mgr ehl_pll_mgr = { ++ .dpll_info = ehl_plls, ++ .get_dpll = icl_get_dpll, ++ .dump_hw_state = icl_dump_hw_state, ++}; ++ ++/** ++ * intel_shared_dpll_init - Initialize shared DPLLs ++ * @dev: drm device ++ * ++ * Initialize shared DPLLs for @dev. ++ */ ++void intel_shared_dpll_init(struct drm_device *dev) ++{ ++ struct drm_i915_private *dev_priv = to_i915(dev); ++ const struct intel_dpll_mgr *dpll_mgr = NULL; ++ const struct dpll_info *dpll_info; ++ int i; ++ ++ if (IS_ELKHARTLAKE(dev_priv)) ++ dpll_mgr = &ehl_pll_mgr; ++ else if (INTEL_GEN(dev_priv) >= 11) ++ dpll_mgr = &icl_pll_mgr; ++ else if (IS_CANNONLAKE(dev_priv)) ++ dpll_mgr = &cnl_pll_mgr; ++ else if (IS_GEN9_BC(dev_priv)) ++ dpll_mgr = &skl_pll_mgr; ++ else if (IS_GEN9_LP(dev_priv)) ++ dpll_mgr = &bxt_pll_mgr; ++ else if (HAS_DDI(dev_priv)) ++ dpll_mgr = &hsw_pll_mgr; ++ else if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) ++ dpll_mgr = &pch_pll_mgr; ++ ++ if (!dpll_mgr) { ++ dev_priv->num_shared_dpll = 0; ++ return; ++ } ++ ++ dpll_info = dpll_mgr->dpll_info; ++ ++ for (i = 0; dpll_info[i].name; i++) { ++ WARN_ON(i != dpll_info[i].id); ++ dev_priv->shared_dplls[i].info = &dpll_info[i]; ++ } ++ ++ dev_priv->dpll_mgr = dpll_mgr; ++ dev_priv->num_shared_dpll = i; ++ mutex_init(&dev_priv->dpll_lock); ++ ++ BUG_ON(dev_priv->num_shared_dpll > I915_NUM_PLLS); ++ ++ /* FIXME: Move this to a more suitable place */ ++ if (HAS_DDI(dev_priv)) ++ intel_ddi_pll_init(dev); ++} ++ ++/** ++ * intel_get_shared_dpll - get a shared DPLL for CRTC and encoder combination ++ * @crtc_state: atomic state for the crtc ++ * @encoder: encoder ++ * ++ * Find an appropriate DPLL for the given CRTC and encoder combination. A ++ * reference from the @crtc_state to the returned pll is registered in the ++ * atomic state. That configuration is made effective by calling ++ * intel_shared_dpll_swap_state(). The reference should be released by calling ++ * intel_release_shared_dpll(). ++ * ++ * Returns: ++ * A shared DPLL to be used by @crtc_state and @encoder. ++ */ ++struct intel_shared_dpll * ++intel_get_shared_dpll(struct intel_crtc_state *crtc_state, ++ struct intel_encoder *encoder) ++{ ++ struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); ++ const struct intel_dpll_mgr *dpll_mgr = dev_priv->dpll_mgr; ++ ++ if (WARN_ON(!dpll_mgr)) ++ return NULL; ++ ++ return dpll_mgr->get_dpll(crtc_state, encoder); ++} ++ ++/** ++ * intel_release_shared_dpll - end use of DPLL by CRTC in atomic state ++ * @dpll: dpll in use by @crtc ++ * @crtc: crtc ++ * @state: atomic state ++ * ++ * This function releases the reference from @crtc to @dpll from the ++ * atomic @state. The new configuration is made effective by calling ++ * intel_shared_dpll_swap_state(). ++ */ ++void intel_release_shared_dpll(struct intel_shared_dpll *dpll, ++ struct intel_crtc *crtc, ++ struct drm_atomic_state *state) ++{ ++ struct intel_shared_dpll_state *shared_dpll_state; ++ ++ shared_dpll_state = intel_atomic_get_shared_dpll_state(state); ++ shared_dpll_state[dpll->info->id].crtc_mask &= ~(1 << crtc->pipe); ++} ++ ++/** ++ * intel_shared_dpll_dump_hw_state - write hw_state to dmesg ++ * @dev_priv: i915 drm device ++ * @hw_state: hw state to be written to the log ++ * ++ * Write the relevant values in @hw_state to dmesg using DRM_DEBUG_KMS. ++ */ ++void intel_dpll_dump_hw_state(struct drm_i915_private *dev_priv, ++ struct intel_dpll_hw_state *hw_state) ++{ ++ if (dev_priv->dpll_mgr) { ++ dev_priv->dpll_mgr->dump_hw_state(dev_priv, hw_state); ++ } else { ++ /* fallback for platforms that don't use the shared dpll ++ * infrastructure ++ */ ++ DRM_DEBUG_KMS("dpll_hw_state: dpll: 0x%x, dpll_md: 0x%x, " ++ "fp0: 0x%x, fp1: 0x%x\n", ++ hw_state->dpll, ++ hw_state->dpll_md, ++ hw_state->fp0, ++ hw_state->fp1); ++ } ++} +diff --git a/drivers/gpu/drm/i915_legacy/intel_dpll_mgr.h b/drivers/gpu/drm/i915_legacy/intel_dpll_mgr.h +new file mode 100644 +index 000000000000..bd8124cc81ed +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_dpll_mgr.h +@@ -0,0 +1,347 @@ ++/* ++ * Copyright © 2012-2016 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ * ++ */ ++ ++#ifndef _INTEL_DPLL_MGR_H_ ++#define _INTEL_DPLL_MGR_H_ ++ ++/*FIXME: Move this to a more appropriate place. */ ++#define abs_diff(a, b) ({ \ ++ typeof(a) __a = (a); \ ++ typeof(b) __b = (b); \ ++ (void) (&__a == &__b); \ ++ __a > __b ? (__a - __b) : (__b - __a); }) ++ ++struct drm_i915_private; ++struct intel_crtc; ++struct intel_crtc_state; ++struct intel_encoder; ++ ++struct intel_shared_dpll; ++struct intel_dpll_mgr; ++ ++/** ++ * enum intel_dpll_id - possible DPLL ids ++ * ++ * Enumeration of possible IDs for a DPLL. Real shared dpll ids must be >= 0. ++ */ ++enum intel_dpll_id { ++ /** ++ * @DPLL_ID_PRIVATE: non-shared dpll in use ++ */ ++ DPLL_ID_PRIVATE = -1, ++ ++ /** ++ * @DPLL_ID_PCH_PLL_A: DPLL A in ILK, SNB and IVB ++ */ ++ DPLL_ID_PCH_PLL_A = 0, ++ /** ++ * @DPLL_ID_PCH_PLL_B: DPLL B in ILK, SNB and IVB ++ */ ++ DPLL_ID_PCH_PLL_B = 1, ++ ++ ++ /** ++ * @DPLL_ID_WRPLL1: HSW and BDW WRPLL1 ++ */ ++ DPLL_ID_WRPLL1 = 0, ++ /** ++ * @DPLL_ID_WRPLL2: HSW and BDW WRPLL2 ++ */ ++ DPLL_ID_WRPLL2 = 1, ++ /** ++ * @DPLL_ID_SPLL: HSW and BDW SPLL ++ */ ++ DPLL_ID_SPLL = 2, ++ /** ++ * @DPLL_ID_LCPLL_810: HSW and BDW 0.81 GHz LCPLL ++ */ ++ DPLL_ID_LCPLL_810 = 3, ++ /** ++ * @DPLL_ID_LCPLL_1350: HSW and BDW 1.35 GHz LCPLL ++ */ ++ DPLL_ID_LCPLL_1350 = 4, ++ /** ++ * @DPLL_ID_LCPLL_2700: HSW and BDW 2.7 GHz LCPLL ++ */ ++ DPLL_ID_LCPLL_2700 = 5, ++ ++ ++ /** ++ * @DPLL_ID_SKL_DPLL0: SKL and later DPLL0 ++ */ ++ DPLL_ID_SKL_DPLL0 = 0, ++ /** ++ * @DPLL_ID_SKL_DPLL1: SKL and later DPLL1 ++ */ ++ DPLL_ID_SKL_DPLL1 = 1, ++ /** ++ * @DPLL_ID_SKL_DPLL2: SKL and later DPLL2 ++ */ ++ DPLL_ID_SKL_DPLL2 = 2, ++ /** ++ * @DPLL_ID_SKL_DPLL3: SKL and later DPLL3 ++ */ ++ DPLL_ID_SKL_DPLL3 = 3, ++ ++ ++ /** ++ * @DPLL_ID_ICL_DPLL0: ICL combo PHY DPLL0 ++ */ ++ DPLL_ID_ICL_DPLL0 = 0, ++ /** ++ * @DPLL_ID_ICL_DPLL1: ICL combo PHY DPLL1 ++ */ ++ DPLL_ID_ICL_DPLL1 = 1, ++ /** ++ * @DPLL_ID_ICL_TBTPLL: ICL TBT PLL ++ */ ++ DPLL_ID_ICL_TBTPLL = 2, ++ /** ++ * @DPLL_ID_ICL_MGPLL1: ICL MG PLL 1 port 1 (C) ++ */ ++ DPLL_ID_ICL_MGPLL1 = 3, ++ /** ++ * @DPLL_ID_ICL_MGPLL2: ICL MG PLL 1 port 2 (D) ++ */ ++ DPLL_ID_ICL_MGPLL2 = 4, ++ /** ++ * @DPLL_ID_ICL_MGPLL3: ICL MG PLL 1 port 3 (E) ++ */ ++ DPLL_ID_ICL_MGPLL3 = 5, ++ /** ++ * @DPLL_ID_ICL_MGPLL4: ICL MG PLL 1 port 4 (F) ++ */ ++ DPLL_ID_ICL_MGPLL4 = 6, ++}; ++#define I915_NUM_PLLS 7 ++ ++struct intel_dpll_hw_state { ++ /* i9xx, pch plls */ ++ u32 dpll; ++ u32 dpll_md; ++ u32 fp0; ++ u32 fp1; ++ ++ /* hsw, bdw */ ++ u32 wrpll; ++ u32 spll; ++ ++ /* skl */ ++ /* ++ * DPLL_CTRL1 has 6 bits for each each this DPLL. We store those in ++ * lower part of ctrl1 and they get shifted into position when writing ++ * the register. This allows us to easily compare the state to share ++ * the DPLL. ++ */ ++ u32 ctrl1; ++ /* HDMI only, 0 when used for DP */ ++ u32 cfgcr1, cfgcr2; ++ ++ /* cnl */ ++ u32 cfgcr0; ++ /* CNL also uses cfgcr1 */ ++ ++ /* bxt */ ++ u32 ebb0, ebb4, pll0, pll1, pll2, pll3, pll6, pll8, pll9, pll10, pcsdw12; ++ ++ /* ++ * ICL uses the following, already defined: ++ * u32 cfgcr0, cfgcr1; ++ */ ++ u32 mg_refclkin_ctl; ++ u32 mg_clktop2_coreclkctl1; ++ u32 mg_clktop2_hsclkctl; ++ u32 mg_pll_div0; ++ u32 mg_pll_div1; ++ u32 mg_pll_lf; ++ u32 mg_pll_frac_lock; ++ u32 mg_pll_ssc; ++ u32 mg_pll_bias; ++ u32 mg_pll_tdc_coldst_bias; ++ u32 mg_pll_bias_mask; ++ u32 mg_pll_tdc_coldst_bias_mask; ++}; ++ ++/** ++ * struct intel_shared_dpll_state - hold the DPLL atomic state ++ * ++ * This structure holds an atomic state for the DPLL, that can represent ++ * either its current state (in struct &intel_shared_dpll) or a desired ++ * future state which would be applied by an atomic mode set (stored in ++ * a struct &intel_atomic_state). ++ * ++ * See also intel_get_shared_dpll() and intel_release_shared_dpll(). ++ */ ++struct intel_shared_dpll_state { ++ /** ++ * @crtc_mask: mask of CRTC using this DPLL, active or not ++ */ ++ unsigned crtc_mask; ++ ++ /** ++ * @hw_state: hardware configuration for the DPLL stored in ++ * struct &intel_dpll_hw_state. ++ */ ++ struct intel_dpll_hw_state hw_state; ++}; ++ ++/** ++ * struct intel_shared_dpll_funcs - platform specific hooks for managing DPLLs ++ */ ++struct intel_shared_dpll_funcs { ++ /** ++ * @prepare: ++ * ++ * Optional hook to perform operations prior to enabling the PLL. ++ * Called from intel_prepare_shared_dpll() function unless the PLL ++ * is already enabled. ++ */ ++ void (*prepare)(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll); ++ ++ /** ++ * @enable: ++ * ++ * Hook for enabling the pll, called from intel_enable_shared_dpll() ++ * if the pll is not already enabled. ++ */ ++ void (*enable)(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll); ++ ++ /** ++ * @disable: ++ * ++ * Hook for disabling the pll, called from intel_disable_shared_dpll() ++ * only when it is safe to disable the pll, i.e., there are no more ++ * tracked users for it. ++ */ ++ void (*disable)(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll); ++ ++ /** ++ * @get_hw_state: ++ * ++ * Hook for reading the values currently programmed to the DPLL ++ * registers. This is used for initial hw state readout and state ++ * verification after a mode set. ++ */ ++ bool (*get_hw_state)(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll, ++ struct intel_dpll_hw_state *hw_state); ++}; ++ ++/** ++ * struct dpll_info - display PLL platform specific info ++ */ ++struct dpll_info { ++ /** ++ * @name: DPLL name; used for logging ++ */ ++ const char *name; ++ ++ /** ++ * @funcs: platform specific hooks ++ */ ++ const struct intel_shared_dpll_funcs *funcs; ++ ++ /** ++ * @id: unique indentifier for this DPLL; should match the index in the ++ * dev_priv->shared_dplls array ++ */ ++ enum intel_dpll_id id; ++ ++#define INTEL_DPLL_ALWAYS_ON (1 << 0) ++ /** ++ * @flags: ++ * ++ * INTEL_DPLL_ALWAYS_ON ++ * Inform the state checker that the DPLL is kept enabled even if ++ * not in use by any CRTC. ++ */ ++ u32 flags; ++}; ++ ++/** ++ * struct intel_shared_dpll - display PLL with tracked state and users ++ */ ++struct intel_shared_dpll { ++ /** ++ * @state: ++ * ++ * Store the state for the pll, including the its hw state ++ * and CRTCs using it. ++ */ ++ struct intel_shared_dpll_state state; ++ ++ /** ++ * @active_mask: mask of active CRTCs (i.e. DPMS on) using this DPLL ++ */ ++ unsigned active_mask; ++ ++ /** ++ * @on: is the PLL actually active? Disabled during modeset ++ */ ++ bool on; ++ ++ /** ++ * @info: platform specific info ++ */ ++ const struct dpll_info *info; ++}; ++ ++#define SKL_DPLL0 0 ++#define SKL_DPLL1 1 ++#define SKL_DPLL2 2 ++#define SKL_DPLL3 3 ++ ++/* shared dpll functions */ ++struct intel_shared_dpll * ++intel_get_shared_dpll_by_id(struct drm_i915_private *dev_priv, ++ enum intel_dpll_id id); ++enum intel_dpll_id ++intel_get_shared_dpll_id(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll); ++void assert_shared_dpll(struct drm_i915_private *dev_priv, ++ struct intel_shared_dpll *pll, ++ bool state); ++#define assert_shared_dpll_enabled(d, p) assert_shared_dpll(d, p, true) ++#define assert_shared_dpll_disabled(d, p) assert_shared_dpll(d, p, false) ++struct intel_shared_dpll *intel_get_shared_dpll(struct intel_crtc_state *state, ++ struct intel_encoder *encoder); ++void intel_release_shared_dpll(struct intel_shared_dpll *dpll, ++ struct intel_crtc *crtc, ++ struct drm_atomic_state *state); ++void intel_prepare_shared_dpll(const struct intel_crtc_state *crtc_state); ++void intel_enable_shared_dpll(const struct intel_crtc_state *crtc_state); ++void intel_disable_shared_dpll(const struct intel_crtc_state *crtc_state); ++void intel_shared_dpll_swap_state(struct drm_atomic_state *state); ++void intel_shared_dpll_init(struct drm_device *dev); ++ ++void intel_dpll_dump_hw_state(struct drm_i915_private *dev_priv, ++ struct intel_dpll_hw_state *hw_state); ++int cnl_hdmi_pll_ref_clock(struct drm_i915_private *dev_priv); ++enum intel_dpll_id icl_tc_port_to_pll_id(enum tc_port tc_port); ++bool intel_dpll_is_combophy(enum intel_dpll_id id); ++ ++#endif /* _INTEL_DPLL_MGR_H_ */ +diff --git a/drivers/gpu/drm/i915_legacy/intel_drv.h b/drivers/gpu/drm/i915_legacy/intel_drv.h +new file mode 100644 +index 000000000000..2db759e8e454 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_drv.h +@@ -0,0 +1,2045 @@ ++/* ++ * Copyright (c) 2006 Dave Airlie ++ * Copyright (c) 2007-2008 Intel Corporation ++ * Jesse Barnes ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS ++ * IN THE SOFTWARE. ++ */ ++#ifndef __INTEL_DRV_H__ ++#define __INTEL_DRV_H__ ++ ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "i915_drv.h" ++ ++struct drm_printer; ++ ++/** ++ * __wait_for - magic wait macro ++ * ++ * Macro to help avoid open coding check/wait/timeout patterns. Note that it's ++ * important that we check the condition again after having timed out, since the ++ * timeout could be due to preemption or similar and we've never had a chance to ++ * check the condition before the timeout. ++ */ ++#define __wait_for(OP, COND, US, Wmin, Wmax) ({ \ ++ const ktime_t end__ = ktime_add_ns(ktime_get_raw(), 1000ll * (US)); \ ++ long wait__ = (Wmin); /* recommended min for usleep is 10 us */ \ ++ int ret__; \ ++ might_sleep(); \ ++ for (;;) { \ ++ const bool expired__ = ktime_after(ktime_get_raw(), end__); \ ++ OP; \ ++ /* Guarantee COND check prior to timeout */ \ ++ barrier(); \ ++ if (COND) { \ ++ ret__ = 0; \ ++ break; \ ++ } \ ++ if (expired__) { \ ++ ret__ = -ETIMEDOUT; \ ++ break; \ ++ } \ ++ usleep_range(wait__, wait__ * 2); \ ++ if (wait__ < (Wmax)) \ ++ wait__ <<= 1; \ ++ } \ ++ ret__; \ ++}) ++ ++#define _wait_for(COND, US, Wmin, Wmax) __wait_for(, (COND), (US), (Wmin), \ ++ (Wmax)) ++#define wait_for(COND, MS) _wait_for((COND), (MS) * 1000, 10, 1000) ++ ++/* If CONFIG_PREEMPT_COUNT is disabled, in_atomic() always reports false. */ ++#if defined(CONFIG_DRM_I915_DEBUG) && defined(CONFIG_PREEMPT_COUNT) ++# define _WAIT_FOR_ATOMIC_CHECK(ATOMIC) WARN_ON_ONCE((ATOMIC) && !in_atomic()) ++#else ++# define _WAIT_FOR_ATOMIC_CHECK(ATOMIC) do { } while (0) ++#endif ++ ++#define _wait_for_atomic(COND, US, ATOMIC) \ ++({ \ ++ int cpu, ret, timeout = (US) * 1000; \ ++ u64 base; \ ++ _WAIT_FOR_ATOMIC_CHECK(ATOMIC); \ ++ if (!(ATOMIC)) { \ ++ preempt_disable(); \ ++ cpu = smp_processor_id(); \ ++ } \ ++ base = local_clock(); \ ++ for (;;) { \ ++ u64 now = local_clock(); \ ++ if (!(ATOMIC)) \ ++ preempt_enable(); \ ++ /* Guarantee COND check prior to timeout */ \ ++ barrier(); \ ++ if (COND) { \ ++ ret = 0; \ ++ break; \ ++ } \ ++ if (now - base >= timeout) { \ ++ ret = -ETIMEDOUT; \ ++ break; \ ++ } \ ++ cpu_relax(); \ ++ if (!(ATOMIC)) { \ ++ preempt_disable(); \ ++ if (unlikely(cpu != smp_processor_id())) { \ ++ timeout -= now - base; \ ++ cpu = smp_processor_id(); \ ++ base = local_clock(); \ ++ } \ ++ } \ ++ } \ ++ ret; \ ++}) ++ ++#define wait_for_us(COND, US) \ ++({ \ ++ int ret__; \ ++ BUILD_BUG_ON(!__builtin_constant_p(US)); \ ++ if ((US) > 10) \ ++ ret__ = _wait_for((COND), (US), 10, 10); \ ++ else \ ++ ret__ = _wait_for_atomic((COND), (US), 0); \ ++ ret__; \ ++}) ++ ++#define wait_for_atomic_us(COND, US) \ ++({ \ ++ BUILD_BUG_ON(!__builtin_constant_p(US)); \ ++ BUILD_BUG_ON((US) > 50000); \ ++ _wait_for_atomic((COND), (US), 1); \ ++}) ++ ++#define wait_for_atomic(COND, MS) wait_for_atomic_us((COND), (MS) * 1000) ++ ++#define KHz(x) (1000 * (x)) ++#define MHz(x) KHz(1000 * (x)) ++ ++#define KBps(x) (1000 * (x)) ++#define MBps(x) KBps(1000 * (x)) ++#define GBps(x) ((u64)1000 * MBps((x))) ++ ++/* ++ * Display related stuff ++ */ ++ ++/* store information about an Ixxx DVO */ ++/* The i830->i865 use multiple DVOs with multiple i2cs */ ++/* the i915, i945 have a single sDVO i2c bus - which is different */ ++#define MAX_OUTPUTS 6 ++/* maximum connectors per crtcs in the mode set */ ++ ++#define INTEL_I2C_BUS_DVO 1 ++#define INTEL_I2C_BUS_SDVO 2 ++ ++/* these are outputs from the chip - integrated only ++ external chips are via DVO or SDVO output */ ++enum intel_output_type { ++ INTEL_OUTPUT_UNUSED = 0, ++ INTEL_OUTPUT_ANALOG = 1, ++ INTEL_OUTPUT_DVO = 2, ++ INTEL_OUTPUT_SDVO = 3, ++ INTEL_OUTPUT_LVDS = 4, ++ INTEL_OUTPUT_TVOUT = 5, ++ INTEL_OUTPUT_HDMI = 6, ++ INTEL_OUTPUT_DP = 7, ++ INTEL_OUTPUT_EDP = 8, ++ INTEL_OUTPUT_DSI = 9, ++ INTEL_OUTPUT_DDI = 10, ++ INTEL_OUTPUT_DP_MST = 11, ++}; ++ ++#define INTEL_DVO_CHIP_NONE 0 ++#define INTEL_DVO_CHIP_LVDS 1 ++#define INTEL_DVO_CHIP_TMDS 2 ++#define INTEL_DVO_CHIP_TVOUT 4 ++ ++#define INTEL_DSI_VIDEO_MODE 0 ++#define INTEL_DSI_COMMAND_MODE 1 ++ ++struct intel_framebuffer { ++ struct drm_framebuffer base; ++ struct intel_rotation_info rot_info; ++ ++ /* for each plane in the normal GTT view */ ++ struct { ++ unsigned int x, y; ++ } normal[2]; ++ /* for each plane in the rotated GTT view */ ++ struct { ++ unsigned int x, y; ++ unsigned int pitch; /* pixels */ ++ } rotated[2]; ++}; ++ ++struct intel_fbdev { ++ struct drm_fb_helper helper; ++ struct intel_framebuffer *fb; ++ struct i915_vma *vma; ++ unsigned long vma_flags; ++ async_cookie_t cookie; ++ int preferred_bpp; ++ ++ /* Whether or not fbdev hpd processing is temporarily suspended */ ++ bool hpd_suspended : 1; ++ /* Set when a hotplug was received while HPD processing was ++ * suspended ++ */ ++ bool hpd_waiting : 1; ++ ++ /* Protects hpd_suspended */ ++ struct mutex hpd_lock; ++}; ++ ++struct intel_encoder { ++ struct drm_encoder base; ++ ++ enum intel_output_type type; ++ enum port port; ++ unsigned int cloneable; ++ bool (*hotplug)(struct intel_encoder *encoder, ++ struct intel_connector *connector); ++ enum intel_output_type (*compute_output_type)(struct intel_encoder *, ++ struct intel_crtc_state *, ++ struct drm_connector_state *); ++ int (*compute_config)(struct intel_encoder *, ++ struct intel_crtc_state *, ++ struct drm_connector_state *); ++ void (*pre_pll_enable)(struct intel_encoder *, ++ const struct intel_crtc_state *, ++ const struct drm_connector_state *); ++ void (*pre_enable)(struct intel_encoder *, ++ const struct intel_crtc_state *, ++ const struct drm_connector_state *); ++ void (*enable)(struct intel_encoder *, ++ const struct intel_crtc_state *, ++ const struct drm_connector_state *); ++ void (*disable)(struct intel_encoder *, ++ const struct intel_crtc_state *, ++ const struct drm_connector_state *); ++ void (*post_disable)(struct intel_encoder *, ++ const struct intel_crtc_state *, ++ const struct drm_connector_state *); ++ void (*post_pll_disable)(struct intel_encoder *, ++ const struct intel_crtc_state *, ++ const struct drm_connector_state *); ++ void (*update_pipe)(struct intel_encoder *, ++ const struct intel_crtc_state *, ++ const struct drm_connector_state *); ++ /* Read out the current hw state of this connector, returning true if ++ * the encoder is active. If the encoder is enabled it also set the pipe ++ * it is connected to in the pipe parameter. */ ++ bool (*get_hw_state)(struct intel_encoder *, enum pipe *pipe); ++ /* Reconstructs the equivalent mode flags for the current hardware ++ * state. This must be called _after_ display->get_pipe_config has ++ * pre-filled the pipe config. Note that intel_encoder->base.crtc must ++ * be set correctly before calling this function. */ ++ void (*get_config)(struct intel_encoder *, ++ struct intel_crtc_state *pipe_config); ++ /* ++ * Acquires the power domains needed for an active encoder during ++ * hardware state readout. ++ */ ++ void (*get_power_domains)(struct intel_encoder *encoder, ++ struct intel_crtc_state *crtc_state); ++ /* ++ * Called during system suspend after all pending requests for the ++ * encoder are flushed (for example for DP AUX transactions) and ++ * device interrupts are disabled. ++ */ ++ void (*suspend)(struct intel_encoder *); ++ int crtc_mask; ++ enum hpd_pin hpd_pin; ++ enum intel_display_power_domain power_domain; ++ /* for communication with audio component; protected by av_mutex */ ++ const struct drm_connector *audio_connector; ++}; ++ ++struct intel_panel { ++ struct drm_display_mode *fixed_mode; ++ struct drm_display_mode *downclock_mode; ++ ++ /* backlight */ ++ struct { ++ bool present; ++ u32 level; ++ u32 min; ++ u32 max; ++ bool enabled; ++ bool combination_mode; /* gen 2/4 only */ ++ bool active_low_pwm; ++ bool alternate_pwm_increment; /* lpt+ */ ++ ++ /* PWM chip */ ++ bool util_pin_active_low; /* bxt+ */ ++ u8 controller; /* bxt+ only */ ++ struct pwm_device *pwm; ++ ++ struct backlight_device *device; ++ ++ /* Connector and platform specific backlight functions */ ++ int (*setup)(struct intel_connector *connector, enum pipe pipe); ++ u32 (*get)(struct intel_connector *connector); ++ void (*set)(const struct drm_connector_state *conn_state, u32 level); ++ void (*disable)(const struct drm_connector_state *conn_state); ++ void (*enable)(const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state); ++ u32 (*hz_to_pwm)(struct intel_connector *connector, u32 hz); ++ void (*power)(struct intel_connector *, bool enable); ++ } backlight; ++}; ++ ++struct intel_digital_port; ++ ++enum check_link_response { ++ HDCP_LINK_PROTECTED = 0, ++ HDCP_TOPOLOGY_CHANGE, ++ HDCP_LINK_INTEGRITY_FAILURE, ++ HDCP_REAUTH_REQUEST ++}; ++ ++/* ++ * This structure serves as a translation layer between the generic HDCP code ++ * and the bus-specific code. What that means is that HDCP over HDMI differs ++ * from HDCP over DP, so to account for these differences, we need to ++ * communicate with the receiver through this shim. ++ * ++ * For completeness, the 2 buses differ in the following ways: ++ * - DP AUX vs. DDC ++ * HDCP registers on the receiver are set via DP AUX for DP, and ++ * they are set via DDC for HDMI. ++ * - Receiver register offsets ++ * The offsets of the registers are different for DP vs. HDMI ++ * - Receiver register masks/offsets ++ * For instance, the ready bit for the KSV fifo is in a different ++ * place on DP vs HDMI ++ * - Receiver register names ++ * Seriously. In the DP spec, the 16-bit register containing ++ * downstream information is called BINFO, on HDMI it's called ++ * BSTATUS. To confuse matters further, DP has a BSTATUS register ++ * with a completely different definition. ++ * - KSV FIFO ++ * On HDMI, the ksv fifo is read all at once, whereas on DP it must ++ * be read 3 keys at a time ++ * - Aksv output ++ * Since Aksv is hidden in hardware, there's different procedures ++ * to send it over DP AUX vs DDC ++ */ ++struct intel_hdcp_shim { ++ /* Outputs the transmitter's An and Aksv values to the receiver. */ ++ int (*write_an_aksv)(struct intel_digital_port *intel_dig_port, u8 *an); ++ ++ /* Reads the receiver's key selection vector */ ++ int (*read_bksv)(struct intel_digital_port *intel_dig_port, u8 *bksv); ++ ++ /* ++ * Reads BINFO from DP receivers and BSTATUS from HDMI receivers. The ++ * definitions are the same in the respective specs, but the names are ++ * different. Call it BSTATUS since that's the name the HDMI spec ++ * uses and it was there first. ++ */ ++ int (*read_bstatus)(struct intel_digital_port *intel_dig_port, ++ u8 *bstatus); ++ ++ /* Determines whether a repeater is present downstream */ ++ int (*repeater_present)(struct intel_digital_port *intel_dig_port, ++ bool *repeater_present); ++ ++ /* Reads the receiver's Ri' value */ ++ int (*read_ri_prime)(struct intel_digital_port *intel_dig_port, u8 *ri); ++ ++ /* Determines if the receiver's KSV FIFO is ready for consumption */ ++ int (*read_ksv_ready)(struct intel_digital_port *intel_dig_port, ++ bool *ksv_ready); ++ ++ /* Reads the ksv fifo for num_downstream devices */ ++ int (*read_ksv_fifo)(struct intel_digital_port *intel_dig_port, ++ int num_downstream, u8 *ksv_fifo); ++ ++ /* Reads a 32-bit part of V' from the receiver */ ++ int (*read_v_prime_part)(struct intel_digital_port *intel_dig_port, ++ int i, u32 *part); ++ ++ /* Enables HDCP signalling on the port */ ++ int (*toggle_signalling)(struct intel_digital_port *intel_dig_port, ++ bool enable); ++ ++ /* Ensures the link is still protected */ ++ bool (*check_link)(struct intel_digital_port *intel_dig_port); ++ ++ /* Detects panel's hdcp capability. This is optional for HDMI. */ ++ int (*hdcp_capable)(struct intel_digital_port *intel_dig_port, ++ bool *hdcp_capable); ++ ++ /* HDCP adaptation(DP/HDMI) required on the port */ ++ enum hdcp_wired_protocol protocol; ++ ++ /* Detects whether sink is HDCP2.2 capable */ ++ int (*hdcp_2_2_capable)(struct intel_digital_port *intel_dig_port, ++ bool *capable); ++ ++ /* Write HDCP2.2 messages */ ++ int (*write_2_2_msg)(struct intel_digital_port *intel_dig_port, ++ void *buf, size_t size); ++ ++ /* Read HDCP2.2 messages */ ++ int (*read_2_2_msg)(struct intel_digital_port *intel_dig_port, ++ u8 msg_id, void *buf, size_t size); ++ ++ /* ++ * Implementation of DP HDCP2.2 Errata for the communication of stream ++ * type to Receivers. In DP HDCP2.2 Stream type is one of the input to ++ * the HDCP2.2 Cipher for En/De-Cryption. Not applicable for HDMI. ++ */ ++ int (*config_stream_type)(struct intel_digital_port *intel_dig_port, ++ bool is_repeater, u8 type); ++ ++ /* HDCP2.2 Link Integrity Check */ ++ int (*check_2_2_link)(struct intel_digital_port *intel_dig_port); ++}; ++ ++struct intel_hdcp { ++ const struct intel_hdcp_shim *shim; ++ /* Mutex for hdcp state of the connector */ ++ struct mutex mutex; ++ u64 value; ++ struct delayed_work check_work; ++ struct work_struct prop_work; ++ ++ /* HDCP1.4 Encryption status */ ++ bool hdcp_encrypted; ++ ++ /* HDCP2.2 related definitions */ ++ /* Flag indicates whether this connector supports HDCP2.2 or not. */ ++ bool hdcp2_supported; ++ ++ /* HDCP2.2 Encryption status */ ++ bool hdcp2_encrypted; ++ ++ /* ++ * Content Stream Type defined by content owner. TYPE0(0x0) content can ++ * flow in the link protected by HDCP2.2 or HDCP1.4, where as TYPE1(0x1) ++ * content can flow only through a link protected by HDCP2.2. ++ */ ++ u8 content_type; ++ struct hdcp_port_data port_data; ++ ++ bool is_paired; ++ bool is_repeater; ++ ++ /* ++ * Count of ReceiverID_List received. Initialized to 0 at AKE_INIT. ++ * Incremented after processing the RepeaterAuth_Send_ReceiverID_List. ++ * When it rolls over re-auth has to be triggered. ++ */ ++ u32 seq_num_v; ++ ++ /* ++ * Count of RepeaterAuth_Stream_Manage msg propagated. ++ * Initialized to 0 on AKE_INIT. Incremented after every successful ++ * transmission of RepeaterAuth_Stream_Manage message. When it rolls ++ * over re-Auth has to be triggered. ++ */ ++ u32 seq_num_m; ++ ++ /* ++ * Work queue to signal the CP_IRQ. Used for the waiters to read the ++ * available information from HDCP DP sink. ++ */ ++ wait_queue_head_t cp_irq_queue; ++ atomic_t cp_irq_count; ++ int cp_irq_count_cached; ++}; ++ ++struct intel_connector { ++ struct drm_connector base; ++ /* ++ * The fixed encoder this connector is connected to. ++ */ ++ struct intel_encoder *encoder; ++ ++ /* ACPI device id for ACPI and driver cooperation */ ++ u32 acpi_device_id; ++ ++ /* Reads out the current hw, returning true if the connector is enabled ++ * and active (i.e. dpms ON state). */ ++ bool (*get_hw_state)(struct intel_connector *); ++ ++ /* Panel info for eDP and LVDS */ ++ struct intel_panel panel; ++ ++ /* Cached EDID for eDP and LVDS. May hold ERR_PTR for invalid EDID. */ ++ struct edid *edid; ++ struct edid *detect_edid; ++ ++ /* since POLL and HPD connectors may use the same HPD line keep the native ++ state of connector->polled in case hotplug storm detection changes it */ ++ u8 polled; ++ ++ void *port; /* store this opaque as its illegal to dereference it */ ++ ++ struct intel_dp *mst_port; ++ ++ /* Work struct to schedule a uevent on link train failure */ ++ struct work_struct modeset_retry_work; ++ ++ struct intel_hdcp hdcp; ++}; ++ ++struct intel_digital_connector_state { ++ struct drm_connector_state base; ++ ++ enum hdmi_force_audio force_audio; ++ int broadcast_rgb; ++}; ++ ++#define to_intel_digital_connector_state(x) container_of(x, struct intel_digital_connector_state, base) ++ ++struct dpll { ++ /* given values */ ++ int n; ++ int m1, m2; ++ int p1, p2; ++ /* derived values */ ++ int dot; ++ int vco; ++ int m; ++ int p; ++}; ++ ++struct intel_atomic_state { ++ struct drm_atomic_state base; ++ ++ struct { ++ /* ++ * Logical state of cdclk (used for all scaling, watermark, ++ * etc. calculations and checks). This is computed as if all ++ * enabled crtcs were active. ++ */ ++ struct intel_cdclk_state logical; ++ ++ /* ++ * Actual state of cdclk, can be different from the logical ++ * state only when all crtc's are DPMS off. ++ */ ++ struct intel_cdclk_state actual; ++ ++ int force_min_cdclk; ++ bool force_min_cdclk_changed; ++ /* pipe to which cd2x update is synchronized */ ++ enum pipe pipe; ++ } cdclk; ++ ++ bool dpll_set, modeset; ++ ++ /* ++ * Does this transaction change the pipes that are active? This mask ++ * tracks which CRTC's have changed their active state at the end of ++ * the transaction (not counting the temporary disable during modesets). ++ * This mask should only be non-zero when intel_state->modeset is true, ++ * but the converse is not necessarily true; simply changing a mode may ++ * not flip the final active status of any CRTC's ++ */ ++ unsigned int active_pipe_changes; ++ ++ unsigned int active_crtcs; ++ /* minimum acceptable cdclk for each pipe */ ++ int min_cdclk[I915_MAX_PIPES]; ++ /* minimum acceptable voltage level for each pipe */ ++ u8 min_voltage_level[I915_MAX_PIPES]; ++ ++ struct intel_shared_dpll_state shared_dpll[I915_NUM_PLLS]; ++ ++ /* ++ * Current watermarks can't be trusted during hardware readout, so ++ * don't bother calculating intermediate watermarks. ++ */ ++ bool skip_intermediate_wm; ++ ++ bool rps_interactive; ++ ++ /* Gen9+ only */ ++ struct skl_ddb_values wm_results; ++ ++ struct i915_sw_fence commit_ready; ++ ++ struct llist_node freed; ++}; ++ ++struct intel_plane_state { ++ struct drm_plane_state base; ++ struct i915_ggtt_view view; ++ struct i915_vma *vma; ++ unsigned long flags; ++#define PLANE_HAS_FENCE BIT(0) ++ ++ struct { ++ u32 offset; ++ /* ++ * Plane stride in: ++ * bytes for 0/180 degree rotation ++ * pixels for 90/270 degree rotation ++ */ ++ u32 stride; ++ int x, y; ++ } color_plane[2]; ++ ++ /* plane control register */ ++ u32 ctl; ++ ++ /* plane color control register */ ++ u32 color_ctl; ++ ++ /* ++ * scaler_id ++ * = -1 : not using a scaler ++ * >= 0 : using a scalers ++ * ++ * plane requiring a scaler: ++ * - During check_plane, its bit is set in ++ * crtc_state->scaler_state.scaler_users by calling helper function ++ * update_scaler_plane. ++ * - scaler_id indicates the scaler it got assigned. ++ * ++ * plane doesn't require a scaler: ++ * - this can happen when scaling is no more required or plane simply ++ * got disabled. ++ * - During check_plane, corresponding bit is reset in ++ * crtc_state->scaler_state.scaler_users by calling helper function ++ * update_scaler_plane. ++ */ ++ int scaler_id; ++ ++ /* ++ * linked_plane: ++ * ++ * ICL planar formats require 2 planes that are updated as pairs. ++ * This member is used to make sure the other plane is also updated ++ * when required, and for update_slave() to find the correct ++ * plane_state to pass as argument. ++ */ ++ struct intel_plane *linked_plane; ++ ++ /* ++ * slave: ++ * If set don't update use the linked plane's state for updating ++ * this plane during atomic commit with the update_slave() callback. ++ * ++ * It's also used by the watermark code to ignore wm calculations on ++ * this plane. They're calculated by the linked plane's wm code. ++ */ ++ u32 slave; ++ ++ struct drm_intel_sprite_colorkey ckey; ++}; ++ ++struct intel_initial_plane_config { ++ struct intel_framebuffer *fb; ++ unsigned int tiling; ++ int size; ++ u32 base; ++ u8 rotation; ++}; ++ ++#define SKL_MIN_SRC_W 8 ++#define SKL_MAX_SRC_W 4096 ++#define SKL_MIN_SRC_H 8 ++#define SKL_MAX_SRC_H 4096 ++#define SKL_MIN_DST_W 8 ++#define SKL_MAX_DST_W 4096 ++#define SKL_MIN_DST_H 8 ++#define SKL_MAX_DST_H 4096 ++#define ICL_MAX_SRC_W 5120 ++#define ICL_MAX_SRC_H 4096 ++#define ICL_MAX_DST_W 5120 ++#define ICL_MAX_DST_H 4096 ++#define SKL_MIN_YUV_420_SRC_W 16 ++#define SKL_MIN_YUV_420_SRC_H 16 ++ ++struct intel_scaler { ++ int in_use; ++ u32 mode; ++}; ++ ++struct intel_crtc_scaler_state { ++#define SKL_NUM_SCALERS 2 ++ struct intel_scaler scalers[SKL_NUM_SCALERS]; ++ ++ /* ++ * scaler_users: keeps track of users requesting scalers on this crtc. ++ * ++ * If a bit is set, a user is using a scaler. ++ * Here user can be a plane or crtc as defined below: ++ * bits 0-30 - plane (bit position is index from drm_plane_index) ++ * bit 31 - crtc ++ * ++ * Instead of creating a new index to cover planes and crtc, using ++ * existing drm_plane_index for planes which is well less than 31 ++ * planes and bit 31 for crtc. This should be fine to cover all ++ * our platforms. ++ * ++ * intel_atomic_setup_scalers will setup available scalers to users ++ * requesting scalers. It will gracefully fail if request exceeds ++ * avilability. ++ */ ++#define SKL_CRTC_INDEX 31 ++ unsigned scaler_users; ++ ++ /* scaler used by crtc for panel fitting purpose */ ++ int scaler_id; ++}; ++ ++/* drm_mode->private_flags */ ++#define I915_MODE_FLAG_INHERITED (1<<0) ++/* Flag to get scanline using frame time stamps */ ++#define I915_MODE_FLAG_GET_SCANLINE_FROM_TIMESTAMP (1<<1) ++/* Flag to use the scanline counter instead of the pixel counter */ ++#define I915_MODE_FLAG_USE_SCANLINE_COUNTER (1<<2) ++ ++struct intel_pipe_wm { ++ struct intel_wm_level wm[5]; ++ u32 linetime; ++ bool fbc_wm_enabled; ++ bool pipe_enabled; ++ bool sprites_enabled; ++ bool sprites_scaled; ++}; ++ ++struct skl_plane_wm { ++ struct skl_wm_level wm[8]; ++ struct skl_wm_level uv_wm[8]; ++ struct skl_wm_level trans_wm; ++ bool is_planar; ++}; ++ ++struct skl_pipe_wm { ++ struct skl_plane_wm planes[I915_MAX_PLANES]; ++ u32 linetime; ++}; ++ ++enum vlv_wm_level { ++ VLV_WM_LEVEL_PM2, ++ VLV_WM_LEVEL_PM5, ++ VLV_WM_LEVEL_DDR_DVFS, ++ NUM_VLV_WM_LEVELS, ++}; ++ ++struct vlv_wm_state { ++ struct g4x_pipe_wm wm[NUM_VLV_WM_LEVELS]; ++ struct g4x_sr_wm sr[NUM_VLV_WM_LEVELS]; ++ u8 num_levels; ++ bool cxsr; ++}; ++ ++struct vlv_fifo_state { ++ u16 plane[I915_MAX_PLANES]; ++}; ++ ++enum g4x_wm_level { ++ G4X_WM_LEVEL_NORMAL, ++ G4X_WM_LEVEL_SR, ++ G4X_WM_LEVEL_HPLL, ++ NUM_G4X_WM_LEVELS, ++}; ++ ++struct g4x_wm_state { ++ struct g4x_pipe_wm wm; ++ struct g4x_sr_wm sr; ++ struct g4x_sr_wm hpll; ++ bool cxsr; ++ bool hpll_en; ++ bool fbc_en; ++}; ++ ++struct intel_crtc_wm_state { ++ union { ++ struct { ++ /* ++ * Intermediate watermarks; these can be ++ * programmed immediately since they satisfy ++ * both the current configuration we're ++ * switching away from and the new ++ * configuration we're switching to. ++ */ ++ struct intel_pipe_wm intermediate; ++ ++ /* ++ * Optimal watermarks, programmed post-vblank ++ * when this state is committed. ++ */ ++ struct intel_pipe_wm optimal; ++ } ilk; ++ ++ struct { ++ /* gen9+ only needs 1-step wm programming */ ++ struct skl_pipe_wm optimal; ++ struct skl_ddb_entry ddb; ++ struct skl_ddb_entry plane_ddb_y[I915_MAX_PLANES]; ++ struct skl_ddb_entry plane_ddb_uv[I915_MAX_PLANES]; ++ } skl; ++ ++ struct { ++ /* "raw" watermarks (not inverted) */ ++ struct g4x_pipe_wm raw[NUM_VLV_WM_LEVELS]; ++ /* intermediate watermarks (inverted) */ ++ struct vlv_wm_state intermediate; ++ /* optimal watermarks (inverted) */ ++ struct vlv_wm_state optimal; ++ /* display FIFO split */ ++ struct vlv_fifo_state fifo_state; ++ } vlv; ++ ++ struct { ++ /* "raw" watermarks */ ++ struct g4x_pipe_wm raw[NUM_G4X_WM_LEVELS]; ++ /* intermediate watermarks */ ++ struct g4x_wm_state intermediate; ++ /* optimal watermarks */ ++ struct g4x_wm_state optimal; ++ } g4x; ++ }; ++ ++ /* ++ * Platforms with two-step watermark programming will need to ++ * update watermark programming post-vblank to switch from the ++ * safe intermediate watermarks to the optimal final ++ * watermarks. ++ */ ++ bool need_postvbl_update; ++}; ++ ++enum intel_output_format { ++ INTEL_OUTPUT_FORMAT_INVALID, ++ INTEL_OUTPUT_FORMAT_RGB, ++ INTEL_OUTPUT_FORMAT_YCBCR420, ++ INTEL_OUTPUT_FORMAT_YCBCR444, ++}; ++ ++struct intel_crtc_state { ++ struct drm_crtc_state base; ++ ++ /** ++ * quirks - bitfield with hw state readout quirks ++ * ++ * For various reasons the hw state readout code might not be able to ++ * completely faithfully read out the current state. These cases are ++ * tracked with quirk flags so that fastboot and state checker can act ++ * accordingly. ++ */ ++#define PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS (1<<0) /* unreliable sync mode.flags */ ++ unsigned long quirks; ++ ++ unsigned fb_bits; /* framebuffers to flip */ ++ bool update_pipe; /* can a fast modeset be performed? */ ++ bool disable_cxsr; ++ bool update_wm_pre, update_wm_post; /* watermarks are updated */ ++ bool fb_changed; /* fb on any of the planes is changed */ ++ bool fifo_changed; /* FIFO split is changed */ ++ ++ /* Pipe source size (ie. panel fitter input size) ++ * All planes will be positioned inside this space, ++ * and get clipped at the edges. */ ++ int pipe_src_w, pipe_src_h; ++ ++ /* ++ * Pipe pixel rate, adjusted for ++ * panel fitter/pipe scaler downscaling. ++ */ ++ unsigned int pixel_rate; ++ ++ /* Whether to set up the PCH/FDI. Note that we never allow sharing ++ * between pch encoders and cpu encoders. */ ++ bool has_pch_encoder; ++ ++ /* Are we sending infoframes on the attached port */ ++ bool has_infoframe; ++ ++ /* CPU Transcoder for the pipe. Currently this can only differ from the ++ * pipe on Haswell and later (where we have a special eDP transcoder) ++ * and Broxton (where we have special DSI transcoders). */ ++ enum transcoder cpu_transcoder; ++ ++ /* ++ * Use reduced/limited/broadcast rbg range, compressing from the full ++ * range fed into the crtcs. ++ */ ++ bool limited_color_range; ++ ++ /* Bitmask of encoder types (enum intel_output_type) ++ * driven by the pipe. ++ */ ++ unsigned int output_types; ++ ++ /* Whether we should send NULL infoframes. Required for audio. */ ++ bool has_hdmi_sink; ++ ++ /* Audio enabled on this pipe. Only valid if either has_hdmi_sink or ++ * has_dp_encoder is set. */ ++ bool has_audio; ++ ++ /* ++ * Enable dithering, used when the selected pipe bpp doesn't match the ++ * plane bpp. ++ */ ++ bool dither; ++ ++ /* ++ * Dither gets enabled for 18bpp which causes CRC mismatch errors for ++ * compliance video pattern tests. ++ * Disable dither only if it is a compliance test request for ++ * 18bpp. ++ */ ++ bool dither_force_disable; ++ ++ /* Controls for the clock computation, to override various stages. */ ++ bool clock_set; ++ ++ /* SDVO TV has a bunch of special case. To make multifunction encoders ++ * work correctly, we need to track this at runtime.*/ ++ bool sdvo_tv_clock; ++ ++ /* ++ * crtc bandwidth limit, don't increase pipe bpp or clock if not really ++ * required. This is set in the 2nd loop of calling encoder's ++ * ->compute_config if the first pick doesn't work out. ++ */ ++ bool bw_constrained; ++ ++ /* Settings for the intel dpll used on pretty much everything but ++ * haswell. */ ++ struct dpll dpll; ++ ++ /* Selected dpll when shared or NULL. */ ++ struct intel_shared_dpll *shared_dpll; ++ ++ /* Actual register state of the dpll, for shared dpll cross-checking. */ ++ struct intel_dpll_hw_state dpll_hw_state; ++ ++ /* DSI PLL registers */ ++ struct { ++ u32 ctrl, div; ++ } dsi_pll; ++ ++ int pipe_bpp; ++ struct intel_link_m_n dp_m_n; ++ ++ /* m2_n2 for eDP downclock */ ++ struct intel_link_m_n dp_m2_n2; ++ bool has_drrs; ++ ++ bool has_psr; ++ bool has_psr2; ++ ++ /* ++ * Frequence the dpll for the port should run at. Differs from the ++ * adjusted dotclock e.g. for DP or 12bpc hdmi mode. This is also ++ * already multiplied by pixel_multiplier. ++ */ ++ int port_clock; ++ ++ /* Used by SDVO (and if we ever fix it, HDMI). */ ++ unsigned pixel_multiplier; ++ ++ u8 lane_count; ++ ++ /* ++ * Used by platforms having DP/HDMI PHY with programmable lane ++ * latency optimization. ++ */ ++ u8 lane_lat_optim_mask; ++ ++ /* minimum acceptable voltage level */ ++ u8 min_voltage_level; ++ ++ /* Panel fitter controls for gen2-gen4 + VLV */ ++ struct { ++ u32 control; ++ u32 pgm_ratios; ++ u32 lvds_border_bits; ++ } gmch_pfit; ++ ++ /* Panel fitter placement and size for Ironlake+ */ ++ struct { ++ u32 pos; ++ u32 size; ++ bool enabled; ++ bool force_thru; ++ } pch_pfit; ++ ++ /* FDI configuration, only valid if has_pch_encoder is set. */ ++ int fdi_lanes; ++ struct intel_link_m_n fdi_m_n; ++ ++ bool ips_enabled; ++ ++ bool crc_enabled; ++ ++ bool enable_fbc; ++ ++ bool double_wide; ++ ++ int pbn; ++ ++ struct intel_crtc_scaler_state scaler_state; ++ ++ /* w/a for waiting 2 vblanks during crtc enable */ ++ enum pipe hsw_workaround_pipe; ++ ++ /* IVB sprite scaling w/a (WaCxSRDisabledForSpriteScaling:ivb) */ ++ bool disable_lp_wm; ++ ++ struct intel_crtc_wm_state wm; ++ ++ /* Gamma mode programmed on the pipe */ ++ u32 gamma_mode; ++ ++ union { ++ /* CSC mode programmed on the pipe */ ++ u32 csc_mode; ++ ++ /* CHV CGM mode */ ++ u32 cgm_mode; ++ }; ++ ++ /* bitmask of visible planes (enum plane_id) */ ++ u8 active_planes; ++ u8 nv12_planes; ++ u8 c8_planes; ++ ++ /* bitmask of planes that will be updated during the commit */ ++ u8 update_planes; ++ ++ struct { ++ u32 enable; ++ u32 gcp; ++ union hdmi_infoframe avi; ++ union hdmi_infoframe spd; ++ union hdmi_infoframe hdmi; ++ } infoframes; ++ ++ /* HDMI scrambling status */ ++ bool hdmi_scrambling; ++ ++ /* HDMI High TMDS char rate ratio */ ++ bool hdmi_high_tmds_clock_ratio; ++ ++ /* Output format RGB/YCBCR etc */ ++ enum intel_output_format output_format; ++ ++ /* Output down scaling is done in LSPCON device */ ++ bool lspcon_downsampling; ++ ++ /* enable pipe gamma? */ ++ bool gamma_enable; ++ ++ /* enable pipe csc? */ ++ bool csc_enable; ++ ++ /* Display Stream compression state */ ++ struct { ++ bool compression_enable; ++ bool dsc_split; ++ u16 compressed_bpp; ++ u8 slice_count; ++ } dsc_params; ++ struct drm_dsc_config dp_dsc_cfg; ++ ++ /* Forward Error correction State */ ++ bool fec_enable; ++}; ++ ++struct intel_crtc { ++ struct drm_crtc base; ++ enum pipe pipe; ++ /* ++ * Whether the crtc and the connected output pipeline is active. Implies ++ * that crtc->enabled is set, i.e. the current mode configuration has ++ * some outputs connected to this crtc. ++ */ ++ bool active; ++ u8 plane_ids_mask; ++ unsigned long long enabled_power_domains; ++ struct intel_overlay *overlay; ++ ++ struct intel_crtc_state *config; ++ ++ /* Access to these should be protected by dev_priv->irq_lock. */ ++ bool cpu_fifo_underrun_disabled; ++ bool pch_fifo_underrun_disabled; ++ ++ /* per-pipe watermark state */ ++ struct { ++ /* watermarks currently being used */ ++ union { ++ struct intel_pipe_wm ilk; ++ struct vlv_wm_state vlv; ++ struct g4x_wm_state g4x; ++ } active; ++ } wm; ++ ++ int scanline_offset; ++ ++ struct { ++ unsigned start_vbl_count; ++ ktime_t start_vbl_time; ++ int min_vbl, max_vbl; ++ int scanline_start; ++ } debug; ++ ++ /* scalers available on this crtc */ ++ int num_scalers; ++}; ++ ++struct intel_plane { ++ struct drm_plane base; ++ enum i9xx_plane_id i9xx_plane; ++ enum plane_id id; ++ enum pipe pipe; ++ bool has_fbc; ++ bool has_ccs; ++ u32 frontbuffer_bit; ++ ++ struct { ++ u32 base, cntl, size; ++ } cursor; ++ ++ /* ++ * NOTE: Do not place new plane state fields here (e.g., when adding ++ * new plane properties). New runtime state should now be placed in ++ * the intel_plane_state structure and accessed via plane_state. ++ */ ++ ++ unsigned int (*max_stride)(struct intel_plane *plane, ++ u32 pixel_format, u64 modifier, ++ unsigned int rotation); ++ void (*update_plane)(struct intel_plane *plane, ++ const struct intel_crtc_state *crtc_state, ++ const struct intel_plane_state *plane_state); ++ void (*update_slave)(struct intel_plane *plane, ++ const struct intel_crtc_state *crtc_state, ++ const struct intel_plane_state *plane_state); ++ void (*disable_plane)(struct intel_plane *plane, ++ const struct intel_crtc_state *crtc_state); ++ bool (*get_hw_state)(struct intel_plane *plane, enum pipe *pipe); ++ int (*check_plane)(struct intel_crtc_state *crtc_state, ++ struct intel_plane_state *plane_state); ++}; ++ ++struct intel_watermark_params { ++ u16 fifo_size; ++ u16 max_wm; ++ u8 default_wm; ++ u8 guard_size; ++ u8 cacheline_size; ++}; ++ ++struct cxsr_latency { ++ bool is_desktop : 1; ++ bool is_ddr3 : 1; ++ u16 fsb_freq; ++ u16 mem_freq; ++ u16 display_sr; ++ u16 display_hpll_disable; ++ u16 cursor_sr; ++ u16 cursor_hpll_disable; ++}; ++ ++#define to_intel_atomic_state(x) container_of(x, struct intel_atomic_state, base) ++#define to_intel_crtc(x) container_of(x, struct intel_crtc, base) ++#define to_intel_crtc_state(x) container_of(x, struct intel_crtc_state, base) ++#define to_intel_connector(x) container_of(x, struct intel_connector, base) ++#define to_intel_encoder(x) container_of(x, struct intel_encoder, base) ++#define to_intel_framebuffer(x) container_of(x, struct intel_framebuffer, base) ++#define to_intel_plane(x) container_of(x, struct intel_plane, base) ++#define to_intel_plane_state(x) container_of(x, struct intel_plane_state, base) ++#define intel_fb_obj(x) ((x) ? to_intel_bo((x)->obj[0]) : NULL) ++ ++struct intel_hdmi { ++ i915_reg_t hdmi_reg; ++ int ddc_bus; ++ struct { ++ enum drm_dp_dual_mode_type type; ++ int max_tmds_clock; ++ } dp_dual_mode; ++ bool has_hdmi_sink; ++ bool has_audio; ++ struct intel_connector *attached_connector; ++ struct cec_notifier *cec_notifier; ++}; ++ ++struct intel_dp_mst_encoder; ++#define DP_MAX_DOWNSTREAM_PORTS 0x10 ++ ++/* ++ * enum link_m_n_set: ++ * When platform provides two set of M_N registers for dp, we can ++ * program them and switch between them incase of DRRS. ++ * But When only one such register is provided, we have to program the ++ * required divider value on that registers itself based on the DRRS state. ++ * ++ * M1_N1 : Program dp_m_n on M1_N1 registers ++ * dp_m2_n2 on M2_N2 registers (If supported) ++ * ++ * M2_N2 : Program dp_m2_n2 on M1_N1 registers ++ * M2_N2 registers are not supported ++ */ ++ ++enum link_m_n_set { ++ /* Sets the m1_n1 and m2_n2 */ ++ M1_N1 = 0, ++ M2_N2 ++}; ++ ++struct intel_dp_compliance_data { ++ unsigned long edid; ++ u8 video_pattern; ++ u16 hdisplay, vdisplay; ++ u8 bpc; ++}; ++ ++struct intel_dp_compliance { ++ unsigned long test_type; ++ struct intel_dp_compliance_data test_data; ++ bool test_active; ++ int test_link_rate; ++ u8 test_lane_count; ++}; ++ ++struct intel_dp { ++ i915_reg_t output_reg; ++ u32 DP; ++ int link_rate; ++ u8 lane_count; ++ u8 sink_count; ++ bool link_mst; ++ bool link_trained; ++ bool has_audio; ++ bool reset_link_params; ++ u8 dpcd[DP_RECEIVER_CAP_SIZE]; ++ u8 psr_dpcd[EDP_PSR_RECEIVER_CAP_SIZE]; ++ u8 downstream_ports[DP_MAX_DOWNSTREAM_PORTS]; ++ u8 edp_dpcd[EDP_DISPLAY_CTL_CAP_SIZE]; ++ u8 dsc_dpcd[DP_DSC_RECEIVER_CAP_SIZE]; ++ u8 fec_capable; ++ /* source rates */ ++ int num_source_rates; ++ const int *source_rates; ++ /* sink rates as reported by DP_MAX_LINK_RATE/DP_SUPPORTED_LINK_RATES */ ++ int num_sink_rates; ++ int sink_rates[DP_MAX_SUPPORTED_RATES]; ++ bool use_rate_select; ++ /* intersection of source and sink rates */ ++ int num_common_rates; ++ int common_rates[DP_MAX_SUPPORTED_RATES]; ++ /* Max lane count for the current link */ ++ int max_link_lane_count; ++ /* Max rate for the current link */ ++ int max_link_rate; ++ /* sink or branch descriptor */ ++ struct drm_dp_desc desc; ++ struct drm_dp_aux aux; ++ u8 train_set[4]; ++ int panel_power_up_delay; ++ int panel_power_down_delay; ++ int panel_power_cycle_delay; ++ int backlight_on_delay; ++ int backlight_off_delay; ++ struct delayed_work panel_vdd_work; ++ bool want_panel_vdd; ++ unsigned long last_power_on; ++ unsigned long last_backlight_off; ++ ktime_t panel_power_off_time; ++ ++ struct notifier_block edp_notifier; ++ ++ /* ++ * Pipe whose power sequencer is currently locked into ++ * this port. Only relevant on VLV/CHV. ++ */ ++ enum pipe pps_pipe; ++ /* ++ * Pipe currently driving the port. Used for preventing ++ * the use of the PPS for any pipe currentrly driving ++ * external DP as that will mess things up on VLV. ++ */ ++ enum pipe active_pipe; ++ /* ++ * Set if the sequencer may be reset due to a power transition, ++ * requiring a reinitialization. Only relevant on BXT. ++ */ ++ bool pps_reset; ++ struct edp_power_seq pps_delays; ++ ++ bool can_mst; /* this port supports mst */ ++ bool is_mst; ++ int active_mst_links; ++ /* connector directly attached - won't be use for modeset in mst world */ ++ struct intel_connector *attached_connector; ++ ++ /* mst connector list */ ++ struct intel_dp_mst_encoder *mst_encoders[I915_MAX_PIPES]; ++ struct drm_dp_mst_topology_mgr mst_mgr; ++ ++ u32 (*get_aux_clock_divider)(struct intel_dp *dp, int index); ++ /* ++ * This function returns the value we have to program the AUX_CTL ++ * register with to kick off an AUX transaction. ++ */ ++ u32 (*get_aux_send_ctl)(struct intel_dp *dp, int send_bytes, ++ u32 aux_clock_divider); ++ ++ i915_reg_t (*aux_ch_ctl_reg)(struct intel_dp *dp); ++ i915_reg_t (*aux_ch_data_reg)(struct intel_dp *dp, int index); ++ ++ /* This is called before a link training is starterd */ ++ void (*prepare_link_retrain)(struct intel_dp *intel_dp); ++ ++ /* Displayport compliance testing */ ++ struct intel_dp_compliance compliance; ++ ++ /* Display stream compression testing */ ++ bool force_dsc_en; ++}; ++ ++enum lspcon_vendor { ++ LSPCON_VENDOR_MCA, ++ LSPCON_VENDOR_PARADE ++}; ++ ++struct intel_lspcon { ++ bool active; ++ enum drm_lspcon_mode mode; ++ enum lspcon_vendor vendor; ++}; ++ ++struct intel_digital_port { ++ struct intel_encoder base; ++ u32 saved_port_bits; ++ struct intel_dp dp; ++ struct intel_hdmi hdmi; ++ struct intel_lspcon lspcon; ++ enum irqreturn (*hpd_pulse)(struct intel_digital_port *, bool); ++ bool release_cl2_override; ++ u8 max_lanes; ++ /* Used for DP and ICL+ TypeC/DP and TypeC/HDMI ports. */ ++ enum aux_ch aux_ch; ++ enum intel_display_power_domain ddi_io_power_domain; ++ bool tc_legacy_port:1; ++ enum tc_port_type tc_type; ++ ++ void (*write_infoframe)(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state, ++ unsigned int type, ++ const void *frame, ssize_t len); ++ void (*read_infoframe)(struct intel_encoder *encoder, ++ const struct intel_crtc_state *crtc_state, ++ unsigned int type, ++ void *frame, ssize_t len); ++ void (*set_infoframes)(struct intel_encoder *encoder, ++ bool enable, ++ const struct intel_crtc_state *crtc_state, ++ const struct drm_connector_state *conn_state); ++ u32 (*infoframes_enabled)(struct intel_encoder *encoder, ++ const struct intel_crtc_state *pipe_config); ++}; ++ ++struct intel_dp_mst_encoder { ++ struct intel_encoder base; ++ enum pipe pipe; ++ struct intel_digital_port *primary; ++ struct intel_connector *connector; ++}; ++ ++static inline enum dpio_channel ++vlv_dport_to_channel(struct intel_digital_port *dport) ++{ ++ switch (dport->base.port) { ++ case PORT_B: ++ case PORT_D: ++ return DPIO_CH0; ++ case PORT_C: ++ return DPIO_CH1; ++ default: ++ BUG(); ++ } ++} ++ ++static inline enum dpio_phy ++vlv_dport_to_phy(struct intel_digital_port *dport) ++{ ++ switch (dport->base.port) { ++ case PORT_B: ++ case PORT_C: ++ return DPIO_PHY0; ++ case PORT_D: ++ return DPIO_PHY1; ++ default: ++ BUG(); ++ } ++} ++ ++static inline enum dpio_channel ++vlv_pipe_to_channel(enum pipe pipe) ++{ ++ switch (pipe) { ++ case PIPE_A: ++ case PIPE_C: ++ return DPIO_CH0; ++ case PIPE_B: ++ return DPIO_CH1; ++ default: ++ BUG(); ++ } ++} ++ ++static inline struct intel_crtc * ++intel_get_crtc_for_pipe(struct drm_i915_private *dev_priv, enum pipe pipe) ++{ ++ return dev_priv->pipe_to_crtc_mapping[pipe]; ++} ++ ++static inline struct intel_crtc * ++intel_get_crtc_for_plane(struct drm_i915_private *dev_priv, enum i9xx_plane_id plane) ++{ ++ return dev_priv->plane_to_crtc_mapping[plane]; ++} ++ ++struct intel_load_detect_pipe { ++ struct drm_atomic_state *restore_state; ++}; ++ ++static inline struct intel_encoder * ++intel_attached_encoder(struct drm_connector *connector) ++{ ++ return to_intel_connector(connector)->encoder; ++} ++ ++static inline bool intel_encoder_is_dig_port(struct intel_encoder *encoder) ++{ ++ switch (encoder->type) { ++ case INTEL_OUTPUT_DDI: ++ case INTEL_OUTPUT_DP: ++ case INTEL_OUTPUT_EDP: ++ case INTEL_OUTPUT_HDMI: ++ return true; ++ default: ++ return false; ++ } ++} ++ ++static inline struct intel_digital_port * ++enc_to_dig_port(struct drm_encoder *encoder) ++{ ++ struct intel_encoder *intel_encoder = to_intel_encoder(encoder); ++ ++ if (intel_encoder_is_dig_port(intel_encoder)) ++ return container_of(encoder, struct intel_digital_port, ++ base.base); ++ else ++ return NULL; ++} ++ ++static inline struct intel_digital_port * ++conn_to_dig_port(struct intel_connector *connector) ++{ ++ return enc_to_dig_port(&intel_attached_encoder(&connector->base)->base); ++} ++ ++static inline struct intel_dp_mst_encoder * ++enc_to_mst(struct drm_encoder *encoder) ++{ ++ return container_of(encoder, struct intel_dp_mst_encoder, base.base); ++} ++ ++static inline struct intel_dp *enc_to_intel_dp(struct drm_encoder *encoder) ++{ ++ return &enc_to_dig_port(encoder)->dp; ++} ++ ++static inline bool intel_encoder_is_dp(struct intel_encoder *encoder) ++{ ++ switch (encoder->type) { ++ case INTEL_OUTPUT_DP: ++ case INTEL_OUTPUT_EDP: ++ return true; ++ case INTEL_OUTPUT_DDI: ++ /* Skip pure HDMI/DVI DDI encoders */ ++ return i915_mmio_reg_valid(enc_to_intel_dp(&encoder->base)->output_reg); ++ default: ++ return false; ++ } ++} ++ ++static inline struct intel_lspcon * ++enc_to_intel_lspcon(struct drm_encoder *encoder) ++{ ++ return &enc_to_dig_port(encoder)->lspcon; ++} ++ ++static inline struct intel_digital_port * ++dp_to_dig_port(struct intel_dp *intel_dp) ++{ ++ return container_of(intel_dp, struct intel_digital_port, dp); ++} ++ ++static inline struct intel_lspcon * ++dp_to_lspcon(struct intel_dp *intel_dp) ++{ ++ return &dp_to_dig_port(intel_dp)->lspcon; ++} ++ ++static inline struct drm_i915_private * ++dp_to_i915(struct intel_dp *intel_dp) ++{ ++ return to_i915(dp_to_dig_port(intel_dp)->base.base.dev); ++} ++ ++static inline struct intel_digital_port * ++hdmi_to_dig_port(struct intel_hdmi *intel_hdmi) ++{ ++ return container_of(intel_hdmi, struct intel_digital_port, hdmi); ++} ++ ++static inline struct intel_plane_state * ++intel_atomic_get_plane_state(struct intel_atomic_state *state, ++ struct intel_plane *plane) ++{ ++ struct drm_plane_state *ret = ++ drm_atomic_get_plane_state(&state->base, &plane->base); ++ ++ if (IS_ERR(ret)) ++ return ERR_CAST(ret); ++ ++ return to_intel_plane_state(ret); ++} ++ ++static inline struct intel_plane_state * ++intel_atomic_get_old_plane_state(struct intel_atomic_state *state, ++ struct intel_plane *plane) ++{ ++ return to_intel_plane_state(drm_atomic_get_old_plane_state(&state->base, ++ &plane->base)); ++} ++ ++static inline struct intel_plane_state * ++intel_atomic_get_new_plane_state(struct intel_atomic_state *state, ++ struct intel_plane *plane) ++{ ++ return to_intel_plane_state(drm_atomic_get_new_plane_state(&state->base, ++ &plane->base)); ++} ++ ++static inline struct intel_crtc_state * ++intel_atomic_get_old_crtc_state(struct intel_atomic_state *state, ++ struct intel_crtc *crtc) ++{ ++ return to_intel_crtc_state(drm_atomic_get_old_crtc_state(&state->base, ++ &crtc->base)); ++} ++ ++static inline struct intel_crtc_state * ++intel_atomic_get_new_crtc_state(struct intel_atomic_state *state, ++ struct intel_crtc *crtc) ++{ ++ return to_intel_crtc_state(drm_atomic_get_new_crtc_state(&state->base, ++ &crtc->base)); ++} ++ ++/* intel_fifo_underrun.c */ ++bool intel_set_cpu_fifo_underrun_reporting(struct drm_i915_private *dev_priv, ++ enum pipe pipe, bool enable); ++bool intel_set_pch_fifo_underrun_reporting(struct drm_i915_private *dev_priv, ++ enum pipe pch_transcoder, ++ bool enable); ++void intel_cpu_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv, ++ enum pipe pipe); ++void intel_pch_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv, ++ enum pipe pch_transcoder); ++void intel_check_cpu_fifo_underruns(struct drm_i915_private *dev_priv); ++void intel_check_pch_fifo_underruns(struct drm_i915_private *dev_priv); ++ ++/* i915_irq.c */ ++void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, u32 mask); ++void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, u32 mask); ++void gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask); ++void gen6_unmask_pm_irq(struct drm_i915_private *dev_priv, u32 mask); ++void gen11_reset_rps_interrupts(struct drm_i915_private *dev_priv); ++void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv); ++void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv); ++void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv); ++void gen6_rps_reset_ei(struct drm_i915_private *dev_priv); ++ ++static inline u32 gen6_sanitize_rps_pm_mask(const struct drm_i915_private *i915, ++ u32 mask) ++{ ++ return mask & ~i915->gt_pm.rps.pm_intrmsk_mbz; ++} ++ ++void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv); ++void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv); ++static inline bool intel_irqs_enabled(struct drm_i915_private *dev_priv) ++{ ++ /* ++ * We only use drm_irq_uninstall() at unload and VT switch, so ++ * this is the only thing we need to check. ++ */ ++ return dev_priv->runtime_pm.irqs_enabled; ++} ++ ++int intel_get_crtc_scanline(struct intel_crtc *crtc); ++void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv, ++ u8 pipe_mask); ++void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv, ++ u8 pipe_mask); ++void gen9_reset_guc_interrupts(struct drm_i915_private *dev_priv); ++void gen9_enable_guc_interrupts(struct drm_i915_private *dev_priv); ++void gen9_disable_guc_interrupts(struct drm_i915_private *dev_priv); ++ ++/* intel_display.c */ ++void intel_plane_destroy(struct drm_plane *plane); ++void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe); ++void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe); ++enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc); ++int vlv_get_hpll_vco(struct drm_i915_private *dev_priv); ++int vlv_get_cck_clock(struct drm_i915_private *dev_priv, ++ const char *name, u32 reg, int ref_freq); ++int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv, ++ const char *name, u32 reg); ++void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv); ++void lpt_disable_iclkip(struct drm_i915_private *dev_priv); ++void intel_init_display_hooks(struct drm_i915_private *dev_priv); ++unsigned int intel_fb_xy_to_linear(int x, int y, ++ const struct intel_plane_state *state, ++ int plane); ++unsigned int intel_fb_align_height(const struct drm_framebuffer *fb, ++ int color_plane, unsigned int height); ++void intel_add_fb_offsets(int *x, int *y, ++ const struct intel_plane_state *state, int plane); ++unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info); ++bool intel_has_pending_fb_unpin(struct drm_i915_private *dev_priv); ++void intel_mark_busy(struct drm_i915_private *dev_priv); ++void intel_mark_idle(struct drm_i915_private *dev_priv); ++int intel_display_suspend(struct drm_device *dev); ++void intel_pps_unlock_regs_wa(struct drm_i915_private *dev_priv); ++void intel_encoder_destroy(struct drm_encoder *encoder); ++struct drm_display_mode * ++intel_encoder_current_mode(struct intel_encoder *encoder); ++bool intel_port_is_combophy(struct drm_i915_private *dev_priv, enum port port); ++bool intel_port_is_tc(struct drm_i915_private *dev_priv, enum port port); ++enum tc_port intel_port_to_tc(struct drm_i915_private *dev_priv, ++ enum port port); ++int intel_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file_priv); ++enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv, ++ enum pipe pipe); ++static inline bool ++intel_crtc_has_type(const struct intel_crtc_state *crtc_state, ++ enum intel_output_type type) ++{ ++ return crtc_state->output_types & (1 << type); ++} ++static inline bool ++intel_crtc_has_dp_encoder(const struct intel_crtc_state *crtc_state) ++{ ++ return crtc_state->output_types & ++ ((1 << INTEL_OUTPUT_DP) | ++ (1 << INTEL_OUTPUT_DP_MST) | ++ (1 << INTEL_OUTPUT_EDP)); ++} ++static inline void ++intel_wait_for_vblank(struct drm_i915_private *dev_priv, enum pipe pipe) ++{ ++ drm_wait_one_vblank(&dev_priv->drm, pipe); ++} ++static inline void ++intel_wait_for_vblank_if_active(struct drm_i915_private *dev_priv, int pipe) ++{ ++ const struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); ++ ++ if (crtc->active) ++ intel_wait_for_vblank(dev_priv, pipe); ++} ++ ++u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc); ++ ++int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp); ++void vlv_wait_port_ready(struct drm_i915_private *dev_priv, ++ struct intel_digital_port *dport, ++ unsigned int expected_mask); ++int intel_get_load_detect_pipe(struct drm_connector *connector, ++ const struct drm_display_mode *mode, ++ struct intel_load_detect_pipe *old, ++ struct drm_modeset_acquire_ctx *ctx); ++void intel_release_load_detect_pipe(struct drm_connector *connector, ++ struct intel_load_detect_pipe *old, ++ struct drm_modeset_acquire_ctx *ctx); ++struct i915_vma * ++intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, ++ const struct i915_ggtt_view *view, ++ bool uses_fence, ++ unsigned long *out_flags); ++void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags); ++struct drm_framebuffer * ++intel_framebuffer_create(struct drm_i915_gem_object *obj, ++ struct drm_mode_fb_cmd2 *mode_cmd); ++int intel_prepare_plane_fb(struct drm_plane *plane, ++ struct drm_plane_state *new_state); ++void intel_cleanup_plane_fb(struct drm_plane *plane, ++ struct drm_plane_state *old_state); ++int intel_plane_atomic_get_property(struct drm_plane *plane, ++ const struct drm_plane_state *state, ++ struct drm_property *property, ++ u64 *val); ++int intel_plane_atomic_set_property(struct drm_plane *plane, ++ struct drm_plane_state *state, ++ struct drm_property *property, ++ u64 val); ++int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_state, ++ struct drm_crtc_state *crtc_state, ++ const struct intel_plane_state *old_plane_state, ++ struct drm_plane_state *plane_state); ++ ++void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv, ++ enum pipe pipe); ++ ++int vlv_force_pll_on(struct drm_i915_private *dev_priv, enum pipe pipe, ++ const struct dpll *dpll); ++void vlv_force_pll_off(struct drm_i915_private *dev_priv, enum pipe pipe); ++int lpt_get_iclkip(struct drm_i915_private *dev_priv); ++bool intel_fuzzy_clock_check(int clock1, int clock2); ++ ++/* modesetting asserts */ ++void assert_panel_unlocked(struct drm_i915_private *dev_priv, ++ enum pipe pipe); ++void assert_pll(struct drm_i915_private *dev_priv, ++ enum pipe pipe, bool state); ++#define assert_pll_enabled(d, p) assert_pll(d, p, true) ++#define assert_pll_disabled(d, p) assert_pll(d, p, false) ++void assert_dsi_pll(struct drm_i915_private *dev_priv, bool state); ++#define assert_dsi_pll_enabled(d) assert_dsi_pll(d, true) ++#define assert_dsi_pll_disabled(d) assert_dsi_pll(d, false) ++void assert_fdi_rx_pll(struct drm_i915_private *dev_priv, ++ enum pipe pipe, bool state); ++#define assert_fdi_rx_pll_enabled(d, p) assert_fdi_rx_pll(d, p, true) ++#define assert_fdi_rx_pll_disabled(d, p) assert_fdi_rx_pll(d, p, false) ++void assert_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, bool state); ++#define assert_pipe_enabled(d, p) assert_pipe(d, p, true) ++#define assert_pipe_disabled(d, p) assert_pipe(d, p, false) ++void intel_prepare_reset(struct drm_i915_private *dev_priv); ++void intel_finish_reset(struct drm_i915_private *dev_priv); ++void hsw_enable_pc8(struct drm_i915_private *dev_priv); ++void hsw_disable_pc8(struct drm_i915_private *dev_priv); ++void gen9_sanitize_dc_state(struct drm_i915_private *dev_priv); ++void bxt_enable_dc9(struct drm_i915_private *dev_priv); ++void bxt_disable_dc9(struct drm_i915_private *dev_priv); ++void gen9_enable_dc5(struct drm_i915_private *dev_priv); ++unsigned int skl_cdclk_get_vco(unsigned int freq); ++void skl_enable_dc6(struct drm_i915_private *dev_priv); ++void intel_dp_get_m_n(struct intel_crtc *crtc, ++ struct intel_crtc_state *pipe_config); ++void intel_dp_set_m_n(const struct intel_crtc_state *crtc_state, ++ enum link_m_n_set m_n); ++int intel_dotclock_calculate(int link_freq, const struct intel_link_m_n *m_n); ++bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, ++ struct dpll *best_clock); ++int chv_calc_dpll_params(int refclk, struct dpll *pll_clock); ++ ++bool intel_crtc_active(struct intel_crtc *crtc); ++bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state); ++void hsw_enable_ips(const struct intel_crtc_state *crtc_state); ++void hsw_disable_ips(const struct intel_crtc_state *crtc_state); ++enum intel_display_power_domain intel_port_to_power_domain(enum port port); ++enum intel_display_power_domain ++intel_aux_power_domain(struct intel_digital_port *dig_port); ++void intel_mode_from_pipe_config(struct drm_display_mode *mode, ++ struct intel_crtc_state *pipe_config); ++void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc, ++ struct intel_crtc_state *crtc_state); ++ ++u16 skl_scaler_calc_phase(int sub, int scale, bool chroma_center); ++int skl_update_scaler_crtc(struct intel_crtc_state *crtc_state); ++int skl_max_scale(const struct intel_crtc_state *crtc_state, ++ u32 pixel_format); ++ ++static inline u32 intel_plane_ggtt_offset(const struct intel_plane_state *state) ++{ ++ return i915_ggtt_offset(state->vma); ++} ++ ++u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state, ++ const struct intel_plane_state *plane_state); ++u32 glk_plane_color_ctl_crtc(const struct intel_crtc_state *crtc_state); ++u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state, ++ const struct intel_plane_state *plane_state); ++u32 skl_plane_ctl_crtc(const struct intel_crtc_state *crtc_state); ++u32 skl_plane_stride(const struct intel_plane_state *plane_state, ++ int plane); ++int skl_check_plane_surface(struct intel_plane_state *plane_state); ++int i9xx_check_plane_surface(struct intel_plane_state *plane_state); ++int skl_format_to_fourcc(int format, bool rgb_order, bool alpha); ++unsigned int i9xx_plane_max_stride(struct intel_plane *plane, ++ u32 pixel_format, u64 modifier, ++ unsigned int rotation); ++ ++/* intel_dp_link_training.c */ ++void intel_dp_start_link_train(struct intel_dp *intel_dp); ++void intel_dp_stop_link_train(struct intel_dp *intel_dp); ++ ++/* intel_vdsc.c */ ++int intel_dp_compute_dsc_params(struct intel_dp *intel_dp, ++ struct intel_crtc_state *pipe_config); ++enum intel_display_power_domain ++intel_dsc_power_domain(const struct intel_crtc_state *crtc_state); ++ ++/* intel_dp_aux_backlight.c */ ++int intel_dp_aux_init_backlight_funcs(struct intel_connector *intel_connector); ++ ++/* intel_dp_mst.c */ ++int intel_dp_mst_encoder_init(struct intel_digital_port *intel_dig_port, int conn_id); ++void intel_dp_mst_encoder_cleanup(struct intel_digital_port *intel_dig_port); ++/* vlv_dsi.c */ ++void vlv_dsi_init(struct drm_i915_private *dev_priv); ++ ++/* icl_dsi.c */ ++void icl_dsi_init(struct drm_i915_private *dev_priv); ++ ++/* intel_dsi_dcs_backlight.c */ ++int intel_dsi_dcs_init_backlight_funcs(struct intel_connector *intel_connector); ++ ++/* intel_hotplug.c */ ++void intel_hpd_poll_init(struct drm_i915_private *dev_priv); ++bool intel_encoder_hotplug(struct intel_encoder *encoder, ++ struct intel_connector *connector); ++ ++/* intel_overlay.c */ ++void intel_overlay_setup(struct drm_i915_private *dev_priv); ++void intel_overlay_cleanup(struct drm_i915_private *dev_priv); ++int intel_overlay_switch_off(struct intel_overlay *overlay); ++int intel_overlay_put_image_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file_priv); ++int intel_overlay_attrs_ioctl(struct drm_device *dev, void *data, ++ struct drm_file *file_priv); ++void intel_overlay_reset(struct drm_i915_private *dev_priv); ++ ++/* intel_quirks.c */ ++void intel_init_quirks(struct drm_i915_private *dev_priv); ++ ++/* intel_runtime_pm.c */ ++void intel_runtime_pm_init_early(struct drm_i915_private *dev_priv); ++int intel_power_domains_init(struct drm_i915_private *); ++void intel_power_domains_cleanup(struct drm_i915_private *dev_priv); ++void intel_power_domains_init_hw(struct drm_i915_private *dev_priv, bool resume); ++void intel_power_domains_fini_hw(struct drm_i915_private *dev_priv); ++void icl_display_core_init(struct drm_i915_private *dev_priv, bool resume); ++void icl_display_core_uninit(struct drm_i915_private *dev_priv); ++void intel_power_domains_enable(struct drm_i915_private *dev_priv); ++void intel_power_domains_disable(struct drm_i915_private *dev_priv); ++ ++enum i915_drm_suspend_mode { ++ I915_DRM_SUSPEND_IDLE, ++ I915_DRM_SUSPEND_MEM, ++ I915_DRM_SUSPEND_HIBERNATE, ++}; ++ ++void intel_power_domains_suspend(struct drm_i915_private *dev_priv, ++ enum i915_drm_suspend_mode); ++void intel_power_domains_resume(struct drm_i915_private *dev_priv); ++void bxt_display_core_init(struct drm_i915_private *dev_priv, bool resume); ++void bxt_display_core_uninit(struct drm_i915_private *dev_priv); ++void intel_runtime_pm_enable(struct drm_i915_private *dev_priv); ++void intel_runtime_pm_disable(struct drm_i915_private *dev_priv); ++void intel_runtime_pm_cleanup(struct drm_i915_private *dev_priv); ++const char * ++intel_display_power_domain_str(enum intel_display_power_domain domain); ++ ++bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv, ++ enum intel_display_power_domain domain); ++bool __intel_display_power_is_enabled(struct drm_i915_private *dev_priv, ++ enum intel_display_power_domain domain); ++intel_wakeref_t intel_display_power_get(struct drm_i915_private *dev_priv, ++ enum intel_display_power_domain domain); ++intel_wakeref_t ++intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv, ++ enum intel_display_power_domain domain); ++void intel_display_power_put_unchecked(struct drm_i915_private *dev_priv, ++ enum intel_display_power_domain domain); ++#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM) ++void intel_display_power_put(struct drm_i915_private *dev_priv, ++ enum intel_display_power_domain domain, ++ intel_wakeref_t wakeref); ++#else ++#define intel_display_power_put(i915, domain, wakeref) \ ++ intel_display_power_put_unchecked(i915, domain) ++#endif ++void icl_dbuf_slices_update(struct drm_i915_private *dev_priv, ++ u8 req_slices); ++ ++static inline void ++assert_rpm_device_not_suspended(struct i915_runtime_pm *rpm) ++{ ++ WARN_ONCE(rpm->suspended, ++ "Device suspended during HW access\n"); ++} ++ ++static inline void ++__assert_rpm_wakelock_held(struct i915_runtime_pm *rpm) ++{ ++ assert_rpm_device_not_suspended(rpm); ++ WARN_ONCE(!atomic_read(&rpm->wakeref_count), ++ "RPM wakelock ref not held during HW access"); ++} ++ ++static inline void ++assert_rpm_wakelock_held(struct drm_i915_private *i915) ++{ ++ __assert_rpm_wakelock_held(&i915->runtime_pm); ++} ++ ++/** ++ * disable_rpm_wakeref_asserts - disable the RPM assert checks ++ * @i915: i915 device instance ++ * ++ * This function disable asserts that check if we hold an RPM wakelock ++ * reference, while keeping the device-not-suspended checks still enabled. ++ * It's meant to be used only in special circumstances where our rule about ++ * the wakelock refcount wrt. the device power state doesn't hold. According ++ * to this rule at any point where we access the HW or want to keep the HW in ++ * an active state we must hold an RPM wakelock reference acquired via one of ++ * the intel_runtime_pm_get() helpers. Currently there are a few special spots ++ * where this rule doesn't hold: the IRQ and suspend/resume handlers, the ++ * forcewake release timer, and the GPU RPS and hangcheck works. All other ++ * users should avoid using this function. ++ * ++ * Any calls to this function must have a symmetric call to ++ * enable_rpm_wakeref_asserts(). ++ */ ++static inline void ++disable_rpm_wakeref_asserts(struct drm_i915_private *i915) ++{ ++ atomic_inc(&i915->runtime_pm.wakeref_count); ++} ++ ++/** ++ * enable_rpm_wakeref_asserts - re-enable the RPM assert checks ++ * @i915: i915 device instance ++ * ++ * This function re-enables the RPM assert checks after disabling them with ++ * disable_rpm_wakeref_asserts. It's meant to be used only in special ++ * circumstances otherwise its use should be avoided. ++ * ++ * Any calls to this function must have a symmetric call to ++ * disable_rpm_wakeref_asserts(). ++ */ ++static inline void ++enable_rpm_wakeref_asserts(struct drm_i915_private *i915) ++{ ++ atomic_dec(&i915->runtime_pm.wakeref_count); ++} ++ ++intel_wakeref_t intel_runtime_pm_get(struct drm_i915_private *i915); ++intel_wakeref_t intel_runtime_pm_get_if_in_use(struct drm_i915_private *i915); ++intel_wakeref_t intel_runtime_pm_get_noresume(struct drm_i915_private *i915); ++ ++#define with_intel_runtime_pm(i915, wf) \ ++ for ((wf) = intel_runtime_pm_get(i915); (wf); \ ++ intel_runtime_pm_put((i915), (wf)), (wf) = 0) ++ ++#define with_intel_runtime_pm_if_in_use(i915, wf) \ ++ for ((wf) = intel_runtime_pm_get_if_in_use(i915); (wf); \ ++ intel_runtime_pm_put((i915), (wf)), (wf) = 0) ++ ++void intel_runtime_pm_put_unchecked(struct drm_i915_private *i915); ++#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM) ++void intel_runtime_pm_put(struct drm_i915_private *i915, intel_wakeref_t wref); ++#else ++#define intel_runtime_pm_put(i915, wref) intel_runtime_pm_put_unchecked(i915) ++#endif ++ ++#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM) ++void print_intel_runtime_pm_wakeref(struct drm_i915_private *i915, ++ struct drm_printer *p); ++#else ++static inline void print_intel_runtime_pm_wakeref(struct drm_i915_private *i915, ++ struct drm_printer *p) ++{ ++} ++#endif ++ ++void chv_phy_powergate_lanes(struct intel_encoder *encoder, ++ bool override, unsigned int mask); ++bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy, ++ enum dpio_channel ch, bool override); ++ ++/* intel_atomic.c */ ++int intel_digital_connector_atomic_get_property(struct drm_connector *connector, ++ const struct drm_connector_state *state, ++ struct drm_property *property, ++ u64 *val); ++int intel_digital_connector_atomic_set_property(struct drm_connector *connector, ++ struct drm_connector_state *state, ++ struct drm_property *property, ++ u64 val); ++int intel_digital_connector_atomic_check(struct drm_connector *conn, ++ struct drm_atomic_state *state); ++struct drm_connector_state * ++intel_digital_connector_duplicate_state(struct drm_connector *connector); ++ ++struct drm_crtc_state *intel_crtc_duplicate_state(struct drm_crtc *crtc); ++void intel_crtc_destroy_state(struct drm_crtc *crtc, ++ struct drm_crtc_state *state); ++struct drm_atomic_state *intel_atomic_state_alloc(struct drm_device *dev); ++void intel_atomic_state_clear(struct drm_atomic_state *); ++ ++static inline struct intel_crtc_state * ++intel_atomic_get_crtc_state(struct drm_atomic_state *state, ++ struct intel_crtc *crtc) ++{ ++ struct drm_crtc_state *crtc_state; ++ crtc_state = drm_atomic_get_crtc_state(state, &crtc->base); ++ if (IS_ERR(crtc_state)) ++ return ERR_CAST(crtc_state); ++ ++ return to_intel_crtc_state(crtc_state); ++} ++ ++int intel_atomic_setup_scalers(struct drm_i915_private *dev_priv, ++ struct intel_crtc *intel_crtc, ++ struct intel_crtc_state *crtc_state); ++ ++#endif /* __INTEL_DRV_H__ */ +diff --git a/drivers/gpu/drm/i915_legacy/intel_dsi.c b/drivers/gpu/drm/i915_legacy/intel_dsi.c +new file mode 100644 +index 000000000000..5fec02aceaed +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_dsi.c +@@ -0,0 +1,128 @@ ++// SPDX-License-Identifier: MIT ++/* ++ * Copyright © 2018 Intel Corporation ++ */ ++ ++#include ++#include "intel_dsi.h" ++ ++int intel_dsi_bitrate(const struct intel_dsi *intel_dsi) ++{ ++ int bpp = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format); ++ ++ if (WARN_ON(bpp < 0)) ++ bpp = 16; ++ ++ return intel_dsi->pclk * bpp / intel_dsi->lane_count; ++} ++ ++int intel_dsi_tlpx_ns(const struct intel_dsi *intel_dsi) ++{ ++ switch (intel_dsi->escape_clk_div) { ++ default: ++ case 0: ++ return 50; ++ case 1: ++ return 100; ++ case 2: ++ return 200; ++ } ++} ++ ++int intel_dsi_get_modes(struct drm_connector *connector) ++{ ++ struct intel_connector *intel_connector = to_intel_connector(connector); ++ struct drm_display_mode *mode; ++ ++ DRM_DEBUG_KMS("\n"); ++ ++ if (!intel_connector->panel.fixed_mode) { ++ DRM_DEBUG_KMS("no fixed mode\n"); ++ return 0; ++ } ++ ++ mode = drm_mode_duplicate(connector->dev, ++ intel_connector->panel.fixed_mode); ++ if (!mode) { ++ DRM_DEBUG_KMS("drm_mode_duplicate failed\n"); ++ return 0; ++ } ++ ++ drm_mode_probed_add(connector, mode); ++ return 1; ++} ++ ++enum drm_mode_status intel_dsi_mode_valid(struct drm_connector *connector, ++ struct drm_display_mode *mode) ++{ ++ struct intel_connector *intel_connector = to_intel_connector(connector); ++ const struct drm_display_mode *fixed_mode = intel_connector->panel.fixed_mode; ++ int max_dotclk = to_i915(connector->dev)->max_dotclk_freq; ++ ++ DRM_DEBUG_KMS("\n"); ++ ++ if (mode->flags & DRM_MODE_FLAG_DBLSCAN) ++ return MODE_NO_DBLESCAN; ++ ++ if (fixed_mode) { ++ if (mode->hdisplay > fixed_mode->hdisplay) ++ return MODE_PANEL; ++ if (mode->vdisplay > fixed_mode->vdisplay) ++ return MODE_PANEL; ++ if (fixed_mode->clock > max_dotclk) ++ return MODE_CLOCK_HIGH; ++ } ++ ++ return MODE_OK; ++} ++ ++struct intel_dsi_host *intel_dsi_host_init(struct intel_dsi *intel_dsi, ++ const struct mipi_dsi_host_ops *funcs, ++ enum port port) ++{ ++ struct intel_dsi_host *host; ++ struct mipi_dsi_device *device; ++ ++ host = kzalloc(sizeof(*host), GFP_KERNEL); ++ if (!host) ++ return NULL; ++ ++ host->base.ops = funcs; ++ host->intel_dsi = intel_dsi; ++ host->port = port; ++ ++ /* ++ * We should call mipi_dsi_host_register(&host->base) here, but we don't ++ * have a host->dev, and we don't have OF stuff either. So just use the ++ * dsi framework as a library and hope for the best. Create the dsi ++ * devices by ourselves here too. Need to be careful though, because we ++ * don't initialize any of the driver model devices here. ++ */ ++ device = kzalloc(sizeof(*device), GFP_KERNEL); ++ if (!device) { ++ kfree(host); ++ return NULL; ++ } ++ ++ device->host = &host->base; ++ host->device = device; ++ ++ return host; ++} ++ ++enum drm_panel_orientation ++intel_dsi_get_panel_orientation(struct intel_connector *connector) ++{ ++ struct drm_i915_private *dev_priv = to_i915(connector->base.dev); ++ enum drm_panel_orientation orientation; ++ ++ orientation = dev_priv->vbt.dsi.orientation; ++ if (orientation != DRM_MODE_PANEL_ORIENTATION_UNKNOWN) ++ return orientation; ++ ++ orientation = dev_priv->vbt.orientation; ++ if (orientation != DRM_MODE_PANEL_ORIENTATION_UNKNOWN) ++ return orientation; ++ ++ return DRM_MODE_PANEL_ORIENTATION_NORMAL; ++} +diff --git a/drivers/gpu/drm/i915_legacy/intel_dsi.h b/drivers/gpu/drm/i915_legacy/intel_dsi.h +new file mode 100644 +index 000000000000..705a609050c0 +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_dsi.h +@@ -0,0 +1,196 @@ ++/* ++ * Copyright © 2013 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER ++ * DEALINGS IN THE SOFTWARE. ++ */ ++ ++#ifndef _INTEL_DSI_H ++#define _INTEL_DSI_H ++ ++#include ++#include ++#include "intel_drv.h" ++ ++/* Dual Link support */ ++#define DSI_DUAL_LINK_NONE 0 ++#define DSI_DUAL_LINK_FRONT_BACK 1 ++#define DSI_DUAL_LINK_PIXEL_ALT 2 ++ ++struct intel_dsi_host; ++ ++struct intel_dsi { ++ struct intel_encoder base; ++ ++ struct intel_dsi_host *dsi_hosts[I915_MAX_PORTS]; ++ intel_wakeref_t io_wakeref[I915_MAX_PORTS]; ++ ++ /* GPIO Desc for CRC based Panel control */ ++ struct gpio_desc *gpio_panel; ++ ++ struct intel_connector *attached_connector; ++ ++ /* bit mask of ports being driven */ ++ u16 ports; ++ ++ /* if true, use HS mode, otherwise LP */ ++ bool hs; ++ ++ /* virtual channel */ ++ int channel; ++ ++ /* Video mode or command mode */ ++ u16 operation_mode; ++ ++ /* number of DSI lanes */ ++ unsigned int lane_count; ++ ++ /* ++ * video mode pixel format ++ * ++ * XXX: consolidate on .format in struct mipi_dsi_device. ++ */ ++ enum mipi_dsi_pixel_format pixel_format; ++ ++ /* video mode format for MIPI_VIDEO_MODE_FORMAT register */ ++ u32 video_mode_format; ++ ++ /* eot for MIPI_EOT_DISABLE register */ ++ u8 eotp_pkt; ++ u8 clock_stop; ++ ++ u8 escape_clk_div; ++ u8 dual_link; ++ ++ u16 dcs_backlight_ports; ++ u16 dcs_cabc_ports; ++ ++ /* RGB or BGR */ ++ bool bgr_enabled; ++ ++ u8 pixel_overlap; ++ u32 port_bits; ++ u32 bw_timer; ++ u32 dphy_reg; ++ ++ /* data lanes dphy timing */ ++ u32 dphy_data_lane_reg; ++ u32 video_frmt_cfg_bits; ++ u16 lp_byte_clk; ++ ++ /* timeouts in byte clocks */ ++ u16 hs_tx_timeout; ++ u16 lp_rx_timeout; ++ u16 turn_arnd_val; ++ u16 rst_timer_val; ++ u16 hs_to_lp_count; ++ u16 clk_lp_to_hs_count; ++ u16 clk_hs_to_lp_count; ++ ++ u16 init_count; ++ u32 pclk; ++ u16 burst_mode_ratio; ++ ++ /* all delays in ms */ ++ u16 backlight_off_delay; ++ u16 backlight_on_delay; ++ u16 panel_on_delay; ++ u16 panel_off_delay; ++ u16 panel_pwr_cycle_delay; ++}; ++ ++struct intel_dsi_host { ++ struct mipi_dsi_host base; ++ struct intel_dsi *intel_dsi; ++ enum port port; ++ ++ /* our little hack */ ++ struct mipi_dsi_device *device; ++}; ++ ++static inline struct intel_dsi_host *to_intel_dsi_host(struct mipi_dsi_host *h) ++{ ++ return container_of(h, struct intel_dsi_host, base); ++} ++ ++#define for_each_dsi_port(__port, __ports_mask) for_each_port_masked(__port, __ports_mask) ++ ++static inline struct intel_dsi *enc_to_intel_dsi(struct drm_encoder *encoder) ++{ ++ return container_of(encoder, struct intel_dsi, base.base); ++} ++ ++static inline bool is_vid_mode(struct intel_dsi *intel_dsi) ++{ ++ return intel_dsi->operation_mode == INTEL_DSI_VIDEO_MODE; ++} ++ ++static inline bool is_cmd_mode(struct intel_dsi *intel_dsi) ++{ ++ return intel_dsi->operation_mode == INTEL_DSI_COMMAND_MODE; ++} ++ ++static inline u16 intel_dsi_encoder_ports(struct intel_encoder *encoder) ++{ ++ return enc_to_intel_dsi(&encoder->base)->ports; ++} ++ ++/* intel_dsi.c */ ++int intel_dsi_bitrate(const struct intel_dsi *intel_dsi); ++int intel_dsi_tlpx_ns(const struct intel_dsi *intel_dsi); ++enum drm_panel_orientation ++intel_dsi_get_panel_orientation(struct intel_connector *connector); ++ ++/* vlv_dsi.c */ ++void vlv_dsi_wait_for_fifo_empty(struct intel_dsi *intel_dsi, enum port port); ++enum mipi_dsi_pixel_format pixel_format_from_register_bits(u32 fmt); ++int intel_dsi_get_modes(struct drm_connector *connector); ++enum drm_mode_status intel_dsi_mode_valid(struct drm_connector *connector, ++ struct drm_display_mode *mode); ++struct intel_dsi_host *intel_dsi_host_init(struct intel_dsi *intel_dsi, ++ const struct mipi_dsi_host_ops *funcs, ++ enum port port); ++ ++/* vlv_dsi_pll.c */ ++int vlv_dsi_pll_compute(struct intel_encoder *encoder, ++ struct intel_crtc_state *config); ++void vlv_dsi_pll_enable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *config); ++void vlv_dsi_pll_disable(struct intel_encoder *encoder); ++u32 vlv_dsi_get_pclk(struct intel_encoder *encoder, ++ struct intel_crtc_state *config); ++void vlv_dsi_reset_clocks(struct intel_encoder *encoder, enum port port); ++ ++bool bxt_dsi_pll_is_enabled(struct drm_i915_private *dev_priv); ++int bxt_dsi_pll_compute(struct intel_encoder *encoder, ++ struct intel_crtc_state *config); ++void bxt_dsi_pll_enable(struct intel_encoder *encoder, ++ const struct intel_crtc_state *config); ++void bxt_dsi_pll_disable(struct intel_encoder *encoder); ++u32 bxt_dsi_get_pclk(struct intel_encoder *encoder, ++ struct intel_crtc_state *config); ++void bxt_dsi_reset_clocks(struct intel_encoder *encoder, enum port port); ++ ++/* intel_dsi_vbt.c */ ++bool intel_dsi_vbt_init(struct intel_dsi *intel_dsi, u16 panel_id); ++void intel_dsi_vbt_exec_sequence(struct intel_dsi *intel_dsi, ++ enum mipi_seq seq_id); ++void intel_dsi_msleep(struct intel_dsi *intel_dsi, int msec); ++ ++#endif /* _INTEL_DSI_H */ +diff --git a/drivers/gpu/drm/i915_legacy/intel_dsi_dcs_backlight.c b/drivers/gpu/drm/i915_legacy/intel_dsi_dcs_backlight.c +new file mode 100644 +index 000000000000..150a156f3b1e +--- /dev/null ++++ b/drivers/gpu/drm/i915_legacy/intel_dsi_dcs_backlight.c +@@ -0,0 +1,177 @@ ++/* ++ * Copyright © 2016 Intel Corporation ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice (including the next ++ * paragraph) shall be included in all copies or substantial portions of the ++ * Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER ++ * DEALINGS IN THE SOFTWARE. ++ * ++ * Author: Deepak M ++ */ ++ ++#include "intel_drv.h" ++#include "intel_dsi.h" ++#include "i915_drv.h" ++#include