Skip to content

Commit 852be13

Browse files
jlawrynodanvet
jlawryno
authored and
danvet
committed
accel/ivpu: Add PM support
 - Implement cold and warm firmware boot flows - Add hang recovery support - Add runtime power management support Co-developed-by: Krystian Pradzynski <[email protected]> Signed-off-by: Krystian Pradzynski <[email protected]> Signed-off-by: Jacek Lawrynowicz <[email protected]> Reviewed-by: Jeffrey Hugo <[email protected]> Signed-off-by: Daniel Vetter <[email protected]> Link: https://patchwork.freedesktop.org/patch/msgid/[email protected]
1 parent cd72722 commit 852be13

File tree

10 files changed

+451
-8
lines changed

10 files changed

+451
-8
lines changed

‎drivers/accel/ivpu/Makefile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ intel_vpu-y := \
1010
ivpu_job.o \
1111
ivpu_jsm_msg.o \
1212
ivpu_mmu.o \
13-
ivpu_mmu_context.o
13+
ivpu_mmu_context.o \
14+
ivpu_pm.o
1415

1516
obj-$(CONFIG_DRM_ACCEL_IVPU) += intel_vpu.o

‎drivers/accel/ivpu/ivpu_drv.c

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include "ivpu_jsm_msg.h"
2525
#include "ivpu_mmu.h"
2626
#include "ivpu_mmu_context.h"
27+
#include "ivpu_pm.h"
2728

2829
#ifndef DRIVER_VERSION_STR
2930
#define DRIVER_VERSION_STR __stringify(DRM_IVPU_DRIVER_MAJOR) "." \
@@ -462,6 +463,10 @@ static int ivpu_dev_init(struct ivpu_device *vdev)
462463
if (!vdev->ipc)
463464
return -ENOMEM;
464465

466+
vdev->pm = drmm_kzalloc(&vdev->drm, sizeof(*vdev->pm), GFP_KERNEL);
467+
if (!vdev->pm)
468+
return -ENOMEM;
469+
465470
vdev->hw->ops = &ivpu_hw_mtl_ops;
466471
vdev->platform = IVPU_PLATFORM_INVALID;
467472
vdev->context_xa_limit.min = IVPU_GLOBAL_CONTEXT_MMU_SSID + 1;
@@ -521,6 +526,12 @@ static int ivpu_dev_init(struct ivpu_device *vdev)
521526
goto err_fw_fini;
522527
}
523528

529+
ret = ivpu_pm_init(vdev);
530+
if (ret) {
531+
ivpu_err(vdev, "Failed to initialize PM: %d\n", ret);
532+
goto err_ipc_fini;
533+
}
534+
524535
ret = ivpu_job_done_thread_init(vdev);
525536
if (ret) {
526537
ivpu_err(vdev, "Failed to initialize job done thread: %d\n", ret);
@@ -539,6 +550,8 @@ static int ivpu_dev_init(struct ivpu_device *vdev)
539550
goto err_job_done_thread_fini;
540551
}
541552

553+
ivpu_pm_enable(vdev);
554+
542555
return 0;
543556

544557
err_job_done_thread_fini:
@@ -559,6 +572,7 @@ static int ivpu_dev_init(struct ivpu_device *vdev)
559572

560573
static void ivpu_dev_fini(struct ivpu_device *vdev)
561574
{
575+
ivpu_pm_disable(vdev);
562576
ivpu_shutdown(vdev);
563577
ivpu_job_done_thread_fini(vdev);
564578
ivpu_ipc_fini(vdev);
@@ -611,11 +625,25 @@ static void ivpu_remove(struct pci_dev *pdev)
611625
ivpu_dev_fini(vdev);
612626
}
613627

628+
static const struct dev_pm_ops ivpu_drv_pci_pm = {
629+
SET_SYSTEM_SLEEP_PM_OPS(ivpu_pm_suspend_cb, ivpu_pm_resume_cb)
630+
SET_RUNTIME_PM_OPS(ivpu_pm_runtime_suspend_cb, ivpu_pm_runtime_resume_cb, NULL)
631+
};
632+
633+
static const struct pci_error_handlers ivpu_drv_pci_err = {
634+
.reset_prepare = ivpu_pm_reset_prepare_cb,
635+
.reset_done = ivpu_pm_reset_done_cb,
636+
};
637+
614638
static struct pci_driver ivpu_pci_driver = {
615639
.name = KBUILD_MODNAME,
616640
.id_table = ivpu_pci_ids,
617641
.probe = ivpu_probe,
618642
.remove = ivpu_remove,
643+
.driver = {
644+
.pm = &ivpu_drv_pci_pm,
645+
},
646+
.err_handler = &ivpu_drv_pci_err,
619647
};
620648

621649
module_pci_driver(ivpu_pci_driver);

‎drivers/accel/ivpu/ivpu_drv.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ struct ivpu_hw_info;
7676
struct ivpu_mmu_info;
7777
struct ivpu_fw_info;
7878
struct ivpu_ipc_info;
79+
struct ivpu_pm_info;
7980

8081
struct ivpu_device {
8182
struct drm_device drm;
@@ -89,6 +90,7 @@ struct ivpu_device {
8990
struct ivpu_mmu_info *mmu;
9091
struct ivpu_fw_info *fw;
9192
struct ivpu_ipc_info *ipc;
93+
struct ivpu_pm_info *pm;
9294

9395
struct ivpu_mmu_context gctx;
9496
struct xarray context_xa;

‎drivers/accel/ivpu/ivpu_fw.c

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#include "ivpu_gem.h"
1515
#include "ivpu_hw.h"
1616
#include "ivpu_ipc.h"
17+
#include "ivpu_pm.h"
1718

1819
#define FW_GLOBAL_MEM_START (2ull * SZ_1G)
1920
#define FW_GLOBAL_MEM_END (3ull * SZ_1G)
@@ -361,9 +362,12 @@ void ivpu_fw_boot_params_setup(struct ivpu_device *vdev, struct vpu_boot_params
361362
/* In case of warm boot we only have to reset the entrypoint addr */
362363
if (!ivpu_fw_is_cold_boot(vdev)) {
363364
boot_params->save_restore_ret_address = 0;
365+
vdev->pm->is_warmboot = true;
364366
return;
365367
}
366368

369+
vdev->pm->is_warmboot = false;
370+
367371
boot_params->magic = VPU_BOOT_PARAMS_MAGIC;
368372
boot_params->vpu_id = to_pci_dev(vdev->drm.dev)->bus->number;
369373
boot_params->frequency = ivpu_hw_reg_pll_freq_get(vdev);

‎drivers/accel/ivpu/ivpu_hw_mtl.c

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include "ivpu_hw.h"
1111
#include "ivpu_ipc.h"
1212
#include "ivpu_mmu.h"
13+
#include "ivpu_pm.h"
1314

1415
#define TILE_FUSE_ENABLE_BOTH 0x0
1516
#define TILE_FUSE_ENABLE_UPPER 0x1
@@ -921,18 +922,23 @@ static void ivpu_hw_mtl_irq_disable(struct ivpu_device *vdev)
921922
static void ivpu_hw_mtl_irq_wdt_nce_handler(struct ivpu_device *vdev)
922923
{
923924
ivpu_err_ratelimited(vdev, "WDT NCE irq\n");
925+
926+
ivpu_pm_schedule_recovery(vdev);
924927
}
925928

926929
static void ivpu_hw_mtl_irq_wdt_mss_handler(struct ivpu_device *vdev)
927930
{
928931
ivpu_err_ratelimited(vdev, "WDT MSS irq\n");
929932

930933
ivpu_hw_wdt_disable(vdev);
934+
ivpu_pm_schedule_recovery(vdev);
931935
}
932936

933937
static void ivpu_hw_mtl_irq_noc_firewall_handler(struct ivpu_device *vdev)
934938
{
935939
ivpu_err_ratelimited(vdev, "NOC Firewall irq\n");
940+
941+
ivpu_pm_schedule_recovery(vdev);
936942
}
937943

938944
/* Handler for IRQs from VPU core (irqV) */
@@ -970,6 +976,7 @@ static u32 ivpu_hw_mtl_irqv_handler(struct ivpu_device *vdev, int irq)
970976
static u32 ivpu_hw_mtl_irqb_handler(struct ivpu_device *vdev, int irq)
971977
{
972978
u32 status = REGB_RD32(MTL_BUTTRESS_INTERRUPT_STAT) & BUTTRESS_IRQ_MASK;
979+
bool schedule_recovery = false;
973980

974981
if (status == 0)
975982
return 0;
@@ -983,6 +990,7 @@ static u32 ivpu_hw_mtl_irqb_handler(struct ivpu_device *vdev, int irq)
983990
if (REG_TEST_FLD(MTL_BUTTRESS_INTERRUPT_STAT, ATS_ERR, status)) {
984991
ivpu_err(vdev, "ATS_ERR irq 0x%016llx", REGB_RD64(MTL_BUTTRESS_ATS_ERR_LOG_0));
985992
REGB_WR32(MTL_BUTTRESS_ATS_ERR_CLEAR, 0x1);
993+
schedule_recovery = true;
986994
}
987995

988996
if (REG_TEST_FLD(MTL_BUTTRESS_INTERRUPT_STAT, UFI_ERR, status)) {
@@ -993,6 +1001,7 @@ static u32 ivpu_hw_mtl_irqb_handler(struct ivpu_device *vdev, int irq)
9931001
REG_GET_FLD(MTL_BUTTRESS_UFI_ERR_LOG, AXI_ID, ufi_log),
9941002
REG_GET_FLD(MTL_BUTTRESS_UFI_ERR_LOG, CQ_ID, ufi_log));
9951003
REGB_WR32(MTL_BUTTRESS_UFI_ERR_CLEAR, 0x1);
1004+
schedule_recovery = true;
9961005
}
9971006

9981007
/*
@@ -1005,6 +1014,9 @@ static u32 ivpu_hw_mtl_irqb_handler(struct ivpu_device *vdev, int irq)
10051014
/* Re-enable global interrupt */
10061015
REGB_WR32(MTL_BUTTRESS_GLOBAL_INT_MASK, 0x0);
10071016

1017+
if (schedule_recovery)
1018+
ivpu_pm_schedule_recovery(vdev);
1019+
10081020
return status;
10091021
}
10101022

‎drivers/accel/ivpu/ivpu_ipc.c

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#include "ivpu_hw_reg_io.h"
1515
#include "ivpu_ipc.h"
1616
#include "ivpu_jsm_msg.h"
17+
#include "ivpu_pm.h"
1718

1819
#define IPC_MAX_RX_MSG 128
1920
#define IS_KTHREAD() (get_current()->flags & PF_KTHREAD)
@@ -294,18 +295,27 @@ int ivpu_ipc_send_receive(struct ivpu_device *vdev, struct vpu_jsm_msg *req,
294295
{
295296
struct vpu_jsm_msg hb_req = { .type = VPU_JSM_MSG_QUERY_ENGINE_HB };
296297
struct vpu_jsm_msg hb_resp;
297-
int ret;
298+
int ret, hb_ret;
299+
300+
ret = ivpu_rpm_get(vdev);
301+
if (ret < 0)
302+
return ret;
298303

299304
ret = ivpu_ipc_send_receive_internal(vdev, req, expected_resp_type, resp,
300305
channel, timeout_ms);
301306
if (ret != -ETIMEDOUT)
302-
return ret;
307+
goto rpm_put;
303308

304-
ret = ivpu_ipc_send_receive_internal(vdev, &hb_req, VPU_JSM_MSG_QUERY_ENGINE_HB_DONE,
305-
&hb_resp, VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm);
306-
if (ret == -ETIMEDOUT)
309+
hb_ret = ivpu_ipc_send_receive_internal(vdev, &hb_req, VPU_JSM_MSG_QUERY_ENGINE_HB_DONE,
310+
&hb_resp, VPU_IPC_CHAN_ASYNC_CMD,
311+
vdev->timeout.jsm);
312+
if (hb_ret == -ETIMEDOUT) {
307313
ivpu_hw_diagnose_failure(vdev);
314+
ivpu_pm_schedule_recovery(vdev);
315+
}
308316

317+
rpm_put:
318+
ivpu_rpm_put(vdev);
309319
return ret;
310320
}
311321

‎drivers/accel/ivpu/ivpu_job.c

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
#include "ivpu_ipc.h"
1818
#include "ivpu_job.h"
1919
#include "ivpu_jsm_msg.h"
20+
#include "ivpu_pm.h"
2021

2122
#define CMD_BUF_IDX 0
2223
#define JOB_ID_JOB_MASK GENMASK(7, 0)
@@ -270,6 +271,9 @@ static void job_release(struct kref *ref)
270271

271272
ivpu_dbg(vdev, KREF, "Job released: id %u\n", job->job_id);
272273
kfree(job);
274+
275+
/* Allow the VPU to get suspended, must be called after ivpu_file_priv_put() */
276+
ivpu_rpm_put(vdev);
273277
}
274278

275279
static void job_put(struct ivpu_job *job)
@@ -286,11 +290,16 @@ ivpu_create_job(struct ivpu_file_priv *file_priv, u32 engine_idx, u32 bo_count)
286290
struct ivpu_device *vdev = file_priv->vdev;
287291
struct ivpu_job *job;
288292
size_t buf_size;
293+
int ret;
294+
295+
ret = ivpu_rpm_get(vdev);
296+
if (ret < 0)
297+
return NULL;
289298

290299
buf_size = sizeof(*job) + bo_count * sizeof(struct ivpu_bo *);
291300
job = kzalloc(buf_size, GFP_KERNEL);
292301
if (!job)
293-
return NULL;
302+
goto err_rpm_put;
294303

295304
kref_init(&job->ref);
296305

@@ -311,6 +320,8 @@ ivpu_create_job(struct ivpu_file_priv *file_priv, u32 engine_idx, u32 bo_count)
311320

312321
err_free_job:
313322
kfree(job);
323+
err_rpm_put:
324+
ivpu_rpm_put(vdev);
314325
return NULL;
315326
}
316327

@@ -565,6 +576,7 @@ static int ivpu_job_done_thread(void *arg)
565576
if (jobs_submitted && !xa_empty(&vdev->submitted_jobs_xa)) {
566577
ivpu_err(vdev, "TDR detected, timeout %d ms", timeout);
567578
ivpu_hw_diagnose_failure(vdev);
579+
ivpu_pm_schedule_recovery(vdev);
568580
}
569581
}
570582
}

‎drivers/accel/ivpu/ivpu_mmu.c

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include "ivpu_hw_reg_io.h"
1212
#include "ivpu_mmu.h"
1313
#include "ivpu_mmu_context.h"
14+
#include "ivpu_pm.h"
1415

1516
#define IVPU_MMU_IDR0_REF 0x080f3e0f
1617
#define IVPU_MMU_IDR0_REF_SIMICS 0x080f3e1f
@@ -814,6 +815,7 @@ static u32 *ivpu_mmu_get_event(struct ivpu_device *vdev)
814815

815816
void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev)
816817
{
818+
bool schedule_recovery = false;
817819
u32 *event;
818820
u32 ssid;
819821

@@ -823,9 +825,14 @@ void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev)
823825
ivpu_mmu_dump_event(vdev, event);
824826

825827
ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, event[0]);
826-
if (ssid != IVPU_GLOBAL_CONTEXT_MMU_SSID)
828+
if (ssid == IVPU_GLOBAL_CONTEXT_MMU_SSID)
829+
schedule_recovery = true;
830+
else
827831
ivpu_mmu_user_context_mark_invalid(vdev, ssid);
828832
}
833+
834+
if (schedule_recovery)
835+
ivpu_pm_schedule_recovery(vdev);
829836
}
830837

831838
void ivpu_mmu_irq_gerr_handler(struct ivpu_device *vdev)

0 commit comments

Comments
 (0)