UbiquitousOS
/
OpenCloudOS-Kernel
mirror of https://gitee.com/OpenCloudOS/OpenCloudOS-Kernel.git
11
0
Fork 0
Code Issues Projects Releases Wiki Activity

drm/amdkfd: CRIU checkpoint and restore queue control stack 

Checkpoint contents of queue control stacks on CRIU dump and restore them
during CRIU restore.

Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: David Yat Sin <david.yatsin@amd.com>
Signed-off-by: Rajneesh Bhardwaj <rajneesh.bhardwaj@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
This commit is contained in:
David Yat Sin 2021-01-25 14:09:32 -05:00 committed by Alex Deucher
parent 42c6c48214
commit 3a9822d7bd
11 changed files with 138 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
View File

@ -311,7 +311,7 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
p->pasid,
dev->id);
err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, &queue_id, NULL, NULL,
err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, &queue_id, NULL, NULL, NULL,
&doorbell_offset_in_process);
if (err != 0)
goto err_create_queue;

2
drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c
View File

@ -185,7 +185,7 @@ static int dbgdev_register_diq(struct kfd_dbgdev *dbgdev)
properties.type = KFD_QUEUE_TYPE_DIQ;
status = pqm_create_queue(dbgdev->pqm, dbgdev->dev, NULL,
&properties, &qid, NULL, NULL, NULL);
&properties, &qid, NULL, NULL, NULL, NULL);
if (status) {
pr_err("Failed to create DIQ\n");

22
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
View File

@ -323,7 +323,7 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
struct queue *q,
struct qcm_process_device *qpd,
const struct kfd_criu_queue_priv_data *qd,
const void *restore_mqd)
const void *restore_mqd, const void *restore_ctl_stack)
{
struct mqd_manager *mqd_mgr;
int retval;
@ -385,7 +385,8 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
if (qd)
mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr,
&q->properties, restore_mqd);
&q->properties, restore_mqd, restore_ctl_stack,
qd->ctl_stack_size);
else
mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
&q->gart_mqd_addr, &q->properties);
@ -1342,7 +1343,7 @@ static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
struct qcm_process_device *qpd,
const struct kfd_criu_queue_priv_data *qd,
const void *restore_mqd)
const void *restore_mqd, const void *restore_ctl_stack)
{
int retval;
struct mqd_manager *mqd_mgr;
@ -1391,7 +1392,8 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
if (qd)
mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr,
&q->properties, restore_mqd);
&q->properties, restore_mqd, restore_ctl_stack,
qd->ctl_stack_size);
else
mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
&q->gart_mqd_addr, &q->properties);
@ -1799,7 +1801,8 @@ static int get_wave_state(struct device_queue_manager *dqm,
static void get_queue_checkpoint_info(struct device_queue_manager *dqm,
const struct queue *q,
u32 *mqd_size)
u32 *mqd_size,
u32 *ctl_stack_size)
{
struct mqd_manager *mqd_mgr;
enum KFD_MQD_TYPE mqd_type =
@ -1808,13 +1811,18 @@ static void get_queue_checkpoint_info(struct device_queue_manager *dqm,
dqm_lock(dqm);
mqd_mgr = dqm->mqd_mgrs[mqd_type];
*mqd_size = mqd_mgr->mqd_size;
*ctl_stack_size = 0;
if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE && mqd_mgr->get_checkpoint_info)
mqd_mgr->get_checkpoint_info(mqd_mgr, q->mqd, ctl_stack_size);
dqm_unlock(dqm);
}
static int checkpoint_mqd(struct device_queue_manager *dqm,
const struct queue *q,
void *mqd)
void *mqd,
void *ctl_stack)
{
struct mqd_manager *mqd_mgr;
int r = 0;
@ -1834,7 +1842,7 @@ static int checkpoint_mqd(struct device_queue_manager *dqm,
goto dqm_unlock;
}
mqd_mgr->checkpoint_mqd(mqd_mgr, q->mqd, mqd);
mqd_mgr->checkpoint_mqd(mqd_mgr, q->mqd, mqd, ctl_stack);
dqm_unlock:
dqm_unlock(dqm);

9
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
View File

@ -93,7 +93,8 @@ struct device_queue_manager_ops {
struct queue *q,
struct qcm_process_device *qpd,
const struct kfd_criu_queue_priv_data *qd,
const void *restore_mqd);
const void *restore_mqd,
const void *restore_ctl_stack);
int (*destroy_queue)(struct device_queue_manager *dqm,
struct qcm_process_device *qpd,
@ -145,11 +146,13 @@ struct device_queue_manager_ops {
int (*reset_queues)(struct device_queue_manager *dqm,
uint16_t pasid);
void (*get_queue_checkpoint_info)(struct device_queue_manager *dqm,
const struct queue *q, u32 *mqd_size);
const struct queue *q, u32 *mqd_size,
u32 *ctl_stack_size);
int (*checkpoint_mqd)(struct device_queue_manager *dqm,
const struct queue *q,
void *mqd);
void *mqd,
void *ctl_stack);
};
struct device_queue_manager_asic_ops {

11
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
View File

@ -100,12 +100,19 @@ struct mqd_manager {
u32 *ctl_stack_used_size,
u32 *save_area_used_size);
void (*checkpoint_mqd)(struct mqd_manager *mm, void *mqd, void *mqd_dst);
void (*get_checkpoint_info)(struct mqd_manager *mm, void *mqd, uint32_t *ctl_stack_size);
void (*checkpoint_mqd)(struct mqd_manager *mm,
void *mqd,
void *mqd_dst,
void *ctl_stack_dst);
void (*restore_mqd)(struct mqd_manager *mm, void **mqd,
struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *p,
const void *mqd_src);
const void *mqd_src,
const void *ctl_stack_src,
const u32 ctl_stack_size);
#if defined(CONFIG_DEBUG_FS)
int (*debugfs_show_mqd)(struct seq_file *m, void *data);

13
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
View File

@ -280,7 +280,7 @@ static int destroy_mqd(struct mqd_manager *mm, void *mqd,
pipe_id, queue_id);
}
static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst)
static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst)
{
struct cik_mqd *m;
@ -292,7 +292,8 @@ static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst)
static void restore_mqd(struct mqd_manager *mm, void **mqd,
struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *qp,
const void *mqd_src)
const void *mqd_src,
const void *ctl_stack_src, const u32 ctl_stack_size)
{
uint64_t addr;
struct cik_mqd *m;
@ -314,7 +315,10 @@ static void restore_mqd(struct mqd_manager *mm, void **mqd,
qp->is_active = 0;
}
static void checkpoint_mqd_sdma(struct mqd_manager *mm, void *mqd, void *mqd_dst)
static void checkpoint_mqd_sdma(struct mqd_manager *mm,
void *mqd,
void *mqd_dst,
void *ctl_stack_dst)
{
struct cik_sdma_rlc_registers *m;
@ -326,7 +330,8 @@ static void checkpoint_mqd_sdma(struct mqd_manager *mm, void *mqd, void *mqd_dst
static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd,
struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *qp,
const void *mqd_src)
const void *mqd_src,
const void *ctl_stack_src, const u32 ctl_stack_size)
{
uint64_t addr;
struct cik_sdma_rlc_registers *m;

14
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
View File

@ -285,7 +285,7 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd,
return 0;
}
static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst)
static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst)
{
struct v10_compute_mqd *m;
@ -297,7 +297,8 @@ static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst)
static void restore_mqd(struct mqd_manager *mm, void **mqd,
struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *qp,
const void *mqd_src)
const void *mqd_src,
const void *ctl_stack_src, const u32 ctl_stack_size)
{
uint64_t addr;
struct v10_compute_mqd *m;
@ -408,7 +409,10 @@ static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->adev, mqd);
}
static void checkpoint_mqd_sdma(struct mqd_manager *mm, void *mqd, void *mqd_dst)
static void checkpoint_mqd_sdma(struct mqd_manager *mm,
void *mqd,
void *mqd_dst,
void *ctl_stack_dst)
{
struct v10_sdma_mqd *m;
@ -420,7 +424,9 @@ static void checkpoint_mqd_sdma(struct mqd_manager *mm, void *mqd, void *mqd_dst
static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd,
struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *qp,
const void *mqd_src)
const void *mqd_src,
const void *ctl_stack_src,
const u32 ctl_stack_size)
{
uint64_t addr;
struct v10_sdma_mqd *m;

29
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
View File

@ -340,22 +340,34 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd,
return 0;
}
static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst)
static void get_checkpoint_info(struct mqd_manager *mm, void *mqd, u32 *ctl_stack_size)
{
struct v9_mqd *m = get_mqd(mqd);
*ctl_stack_size = m->cp_hqd_cntl_stack_size;
}
static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst)
{
struct v9_mqd *m;
/* Control stack is located one page after MQD. */
void *ctl_stack = (void *)((uintptr_t)mqd + PAGE_SIZE);
m = get_mqd(mqd);
memcpy(mqd_dst, m, sizeof(struct v9_mqd));
memcpy(ctl_stack_dst, ctl_stack, m->cp_hqd_cntl_stack_size);
}
static void restore_mqd(struct mqd_manager *mm, void **mqd,
struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *qp,
const void *mqd_src)
const void *mqd_src,
const void *ctl_stack_src, u32 ctl_stack_size)
{
uint64_t addr;
struct v9_mqd *m;
void *ctl_stack;
m = (struct v9_mqd *) mqd_mem_obj->cpu_ptr;
addr = mqd_mem_obj->gpu_addr;
@ -366,6 +378,10 @@ static void restore_mqd(struct mqd_manager *mm, void **mqd,
if (gart_addr)
*gart_addr = addr;
/* Control stack is located one page after MQD. */
ctl_stack = (void *)((uintptr_t)*mqd + PAGE_SIZE);
memcpy(ctl_stack, ctl_stack_src, ctl_stack_size);
m->cp_hqd_pq_doorbell_control =
qp->doorbell_off <<
CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
@ -463,7 +479,10 @@ static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->adev, mqd);
}
static void checkpoint_mqd_sdma(struct mqd_manager *mm, void *mqd, void *mqd_dst)
static void checkpoint_mqd_sdma(struct mqd_manager *mm,
void *mqd,
void *mqd_dst,
void *ctl_stack_dst)
{
struct v9_sdma_mqd *m;
@ -475,7 +494,8 @@ static void checkpoint_mqd_sdma(struct mqd_manager *mm, void *mqd, void *mqd_dst
static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd,
struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *qp,
const void *mqd_src)
const void *mqd_src,
const void *ctl_stack_src, const u32 ctl_stack_size)
{
uint64_t addr;
struct v9_sdma_mqd *m;
@ -537,6 +557,7 @@ struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
mqd->destroy_mqd = destroy_mqd;
mqd->is_occupied = is_occupied;
mqd->get_wave_state = get_wave_state;
mqd->get_checkpoint_info = get_checkpoint_info;
mqd->checkpoint_mqd = checkpoint_mqd;
mqd->restore_mqd = restore_mqd;
mqd->mqd_size = sizeof(struct v9_mqd);

22
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
View File

@ -306,7 +306,13 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd,
return 0;
}
static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst)
static void get_checkpoint_info(struct mqd_manager *mm, void *mqd, u32 *ctl_stack_size)
{
/* Control stack is stored in user mode */
*ctl_stack_size = 0;
}
static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst)
{
struct vi_mqd *m;
@ -318,7 +324,8 @@ static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst)
static void restore_mqd(struct mqd_manager *mm, void **mqd,
struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *qp,
const void *mqd_src)
const void *mqd_src,
const void *ctl_stack_src, const u32 ctl_stack_size)
{
uint64_t addr;
struct vi_mqd *m;
@ -341,7 +348,6 @@ static void restore_mqd(struct mqd_manager *mm, void **mqd,
qp->is_active = 0;
}
static void init_mqd_hiq(struct mqd_manager *mm, void **mqd,
struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *q)
@ -435,7 +441,10 @@ static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->adev, mqd);
}
static void checkpoint_mqd_sdma(struct mqd_manager *mm, void *mqd, void *mqd_dst)
static void checkpoint_mqd_sdma(struct mqd_manager *mm,
void *mqd,
void *mqd_dst,
void *ctl_stack_dst)
{
struct vi_sdma_mqd *m;
@ -447,7 +456,8 @@ static void checkpoint_mqd_sdma(struct mqd_manager *mm, void *mqd, void *mqd_dst
static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd,
struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *qp,
const void *mqd_src)
const void *mqd_src,
const void *ctl_stack_src, const u32 ctl_stack_size)
{
uint64_t addr;
struct vi_sdma_mqd *m;
@ -469,6 +479,7 @@ static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd,
#if defined(CONFIG_DEBUG_FS)
static int debugfs_show_mqd(struct seq_file *m, void *data)
{
seq_hex_dump(m, " ", DUMP_PREFIX_OFFSET, 32, 4,
@ -509,6 +520,7 @@ struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
mqd->destroy_mqd = destroy_mqd;
mqd->is_occupied = is_occupied;
mqd->get_wave_state = get_wave_state;
mqd->get_checkpoint_info = get_checkpoint_info;
mqd->checkpoint_mqd = checkpoint_mqd;
mqd->restore_mqd = restore_mqd;
mqd->mqd_size = sizeof(struct vi_mqd);

5
drivers/gpu/drm/amd/amdkfd/kfd_priv.h
View File

@ -1159,6 +1159,7 @@ int pqm_create_queue(struct process_queue_manager *pqm,
unsigned int *qid,
const struct kfd_criu_queue_priv_data *q_data,
const void *restore_mqd,
const void *restore_ctl_stack,
uint32_t *p_doorbell_offset_in_process);
int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid);
int pqm_update_queue_properties(struct process_queue_manager *pqm, unsigned int qid,
@ -1183,8 +1184,8 @@ int amdkfd_fence_wait_timeout(uint64_t *fence_addr,
int pqm_get_queue_checkpoint_info(struct process_queue_manager *pqm,
unsigned int qid,
u32 *mqd_size);
u32 *mqd_size,
u32 *ctl_stack_size);
/* Packet Manager */
#define KFD_FENCE_COMPLETED (100)

62
drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
View File

@ -209,6 +209,7 @@ int pqm_create_queue(struct process_queue_manager *pqm,
unsigned int *qid,
const struct kfd_criu_queue_priv_data *q_data,
const void *restore_mqd,
const void *restore_ctl_stack,
uint32_t *p_doorbell_offset_in_process)
{
int retval;
@ -273,7 +274,8 @@ int pqm_create_queue(struct process_queue_manager *pqm,
goto err_create_queue;
pqn->q = q;
pqn->kq = NULL;
retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data, restore_mqd);
retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data,
restore_mqd, restore_ctl_stack);
print_queue(q);
break;
@ -293,7 +295,8 @@ int pqm_create_queue(struct process_queue_manager *pqm,
goto err_create_queue;
pqn->q = q;
pqn->kq = NULL;
retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data, restore_mqd);
retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data,
restore_mqd, restore_ctl_stack);
print_queue(q);
break;
case KFD_QUEUE_TYPE_DIQ:
@ -518,11 +521,17 @@ int pqm_get_wave_state(struct process_queue_manager *pqm,
save_area_used_size);
}
static int get_queue_data_sizes(struct kfd_process_device *pdd, struct queue *q, uint32_t *mqd_size)
static int get_queue_data_sizes(struct kfd_process_device *pdd,
struct queue *q,
uint32_t *mqd_size,
uint32_t *ctl_stack_size)
{
int ret;
ret = pqm_get_queue_checkpoint_info(&pdd->process->pqm, q->properties.queue_id, mqd_size);
ret = pqm_get_queue_checkpoint_info(&pdd->process->pqm,
q->properties.queue_id,
mqd_size,
ctl_stack_size);
if (ret)
pr_err("Failed to get queue dump info (%d)\n", ret);
@ -548,14 +557,15 @@ int kfd_process_get_queue_info(struct kfd_process *p,
if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
q->properties.type == KFD_QUEUE_TYPE_SDMA ||
q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
uint32_t mqd_size;
uint32_t mqd_size, ctl_stack_size;
*num_queues = *num_queues + 1;
ret = get_queue_data_sizes(pdd, q, &mqd_size);
ret = get_queue_data_sizes(pdd, q, &mqd_size, &ctl_stack_size);
if (ret)
return ret;
extra_data_sizes += mqd_size;
extra_data_sizes += mqd_size + ctl_stack_size;
} else {
pr_err("Unsupported queue type (%d)\n", q->properties.type);
return -EOPNOTSUPP;
@ -568,7 +578,10 @@ int kfd_process_get_queue_info(struct kfd_process *p,
return 0;
}
static int pqm_checkpoint_mqd(struct process_queue_manager *pqm, unsigned int qid, void *mqd)
static int pqm_checkpoint_mqd(struct process_queue_manager *pqm,
unsigned int qid,
void *mqd,
void *ctl_stack)
{
struct process_queue_node *pqn;
@ -583,17 +596,19 @@ static int pqm_checkpoint_mqd(struct process_queue_manager *pqm, unsigned int qi
return -EOPNOTSUPP;
}
return pqn->q->device->dqm->ops.checkpoint_mqd(pqn->q->device->dqm, pqn->q, mqd);
return pqn->q->device->dqm->ops.checkpoint_mqd(pqn->q->device->dqm,
pqn->q, mqd, ctl_stack);
}
static int criu_checkpoint_queue(struct kfd_process_device *pdd,
struct queue *q,
struct kfd_criu_queue_priv_data *q_data)
{
uint8_t *mqd;
uint8_t *mqd, *ctl_stack;
int ret;
mqd = (void *)(q_data + 1);
ctl_stack = mqd + q_data->mqd_size;
q_data->gpu_id = pdd->dev->id;
q_data->type = q->properties.type;
@ -620,7 +635,7 @@ static int criu_checkpoint_queue(struct kfd_process_device *pdd,
q_data->ctx_save_restore_area_size =
q->properties.ctx_save_restore_area_size;
ret = pqm_checkpoint_mqd(&pdd->process->pqm, q->properties.queue_id, mqd);
ret = pqm_checkpoint_mqd(&pdd->process->pqm, q->properties.queue_id, mqd, ctl_stack);
if (ret) {
pr_err("Failed checkpoint queue_mqd (%d)\n", ret);
return ret;
@ -644,6 +659,7 @@ static int criu_checkpoint_queues_device(struct kfd_process_device *pdd,
struct kfd_criu_queue_priv_data *q_data;
uint64_t q_data_size;
uint32_t mqd_size;
uint32_t ctl_stack_size;
if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE &&
q->properties.type != KFD_QUEUE_TYPE_SDMA &&
@ -654,11 +670,11 @@ static int criu_checkpoint_queues_device(struct kfd_process_device *pdd,
break;
}
ret = get_queue_data_sizes(pdd, q, &mqd_size);
ret = get_queue_data_sizes(pdd, q, &mqd_size, &ctl_stack_size);
if (ret)
break;
q_data_size = sizeof(*q_data) + mqd_size;
q_data_size = sizeof(*q_data) + mqd_size + ctl_stack_size;
/* Increase local buffer space if needed */
if (q_private_data_size < q_data_size) {
@ -674,8 +690,9 @@ static int criu_checkpoint_queues_device(struct kfd_process_device *pdd,
q_data = (struct kfd_criu_queue_priv_data *)q_private_data;
/* data stored in this order: priv_data, mqd */
/* data stored in this order: priv_data, mqd, ctl_stack */
q_data->mqd_size = mqd_size;
q_data->ctl_stack_size = ctl_stack_size;
ret = criu_checkpoint_queue(pdd, q, q_data);
if (ret)
@ -746,8 +763,8 @@ int kfd_criu_restore_queue(struct kfd_process *p,
uint64_t *priv_data_offset,
uint64_t max_priv_data_size)
{
uint8_t *mqd, *ctl_stack, *q_extra_data = NULL;
struct kfd_criu_queue_priv_data *q_data;
uint8_t *mqd, *q_extra_data = NULL;
struct kfd_process_device *pdd;
uint64_t q_extra_data_size;
struct queue_properties qp;
@ -769,7 +786,7 @@ int kfd_criu_restore_queue(struct kfd_process *p,
}
*priv_data_offset += sizeof(*q_data);
q_extra_data_size = q_data->mqd_size;
q_extra_data_size = q_data->ctl_stack_size + q_data->mqd_size;
if (*priv_data_offset + q_extra_data_size > max_priv_data_size) {
ret = -EINVAL;
@ -805,15 +822,17 @@ int kfd_criu_restore_queue(struct kfd_process *p,
ret = -EFAULT;
return ret;
}
/* data stored in this order: mqd */
/* data stored in this order: mqd, ctl_stack */
mqd = q_extra_data;
ctl_stack = mqd + q_data->mqd_size;
memset(&qp, 0, sizeof(qp));
set_queue_properties_from_criu(&qp, q_data);
print_queue_properties(&qp);
ret = pqm_create_queue(&p->pqm, pdd->dev, NULL, &qp, &queue_id, q_data, mqd, NULL);
ret = pqm_create_queue(&p->pqm, pdd->dev, NULL, &qp, &queue_id, q_data, mqd, ctl_stack,
NULL);
if (ret) {
pr_err("Failed to create new queue err:%d\n", ret);
ret = -EINVAL;
@ -832,7 +851,8 @@ exit:
int pqm_get_queue_checkpoint_info(struct process_queue_manager *pqm,
unsigned int qid,
uint32_t *mqd_size)
uint32_t *mqd_size,
uint32_t *ctl_stack_size)
{
struct process_queue_node *pqn;
@ -847,7 +867,9 @@ int pqm_get_queue_checkpoint_info(struct process_queue_manager *pqm,
return -EOPNOTSUPP;
}
pqn->q->device->dqm->ops.get_queue_checkpoint_info(pqn->q->device->dqm, pqn->q, mqd_size);
pqn->q->device->dqm->ops.get_queue_checkpoint_info(pqn->q->device->dqm,
pqn->q, mqd_size,
ctl_stack_size);
return 0;
}