ubistorage
/
treesls
mirror of https://ipads.se.sjtu.edu.cn:1312/opensource/treesls.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Enable time traveling ckpt

This commit is contained in:
wfn 2024-09-20 00:38:38 +08:00
parent 3cb6b6fabf
commit 997aa9edd8
47 changed files with 1197 additions and 209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
kernel/arch/x86_64/irq/irq_entry.c
View File

@ -18,6 +18,7 @@
#include <mm/vmspace.h>
#include <common/vars.h>
#include <sched/fpu.h>
#include <mm/nvm.h>
/* idt and idtr */
struct gate_desc idt[T_NUM] __attribute__((aligned(16)));
@ -170,14 +171,10 @@ void handle_reset_sched(u32 cpuid)
void handle_wait_in_kernel(u32 cpuid) {
save_fpu_state(current_thread);
// struct thread *fpu_owner = (struct thread *)(cpu_info[cpuid].fpu_owner);
// if(fpu_owner != NULL) {
// save_fpu_state(fpu_owner);
// }
/* Save FS for thread from */
if (likely((current_thread) &&
(current_thread->thread_ctx->type > TYPE_KERNEL))) {
current_thread->thread_ctx->tls_base_reg[TLS_FS] = __builtin_ia32_rdfsbase64();
}
mark_in_kernel_ipi_tx(cpuid);
@ -211,6 +208,15 @@ void handle_irq(int irqno)
// kinfo("CPU %d: receive IPI on TLB.\n", smp_get_cpu_id());
handle_ipi();
arch_ack_irq();
/* re-schedule if restored */
if (current_resched_flag == true) {
current_resched_flag = false;
sched();
eret_to_thread(switch_context());
/* should never return */
BUG_ON(1);
}
return;
case IRQ_IPI_RESET_SCHED:
handle_ipi();

11
kernel/arch/x86_64/sched/extended_state.c
View File

@ -36,9 +36,14 @@ struct xsave_area {
#define STATE_AREA_SIZE (sizeof(struct xsave_area))
u64 get_fpu_state_size()
{
return STATE_AREA_SIZE;
}
void *alloc_fpu_state()
{
return kzalloc(STATE_AREA_SIZE);
return kzalloc(get_fpu_state_size());
}
#if USE_INSTRUCTION == USE_XSAVEOPT
@ -47,7 +52,7 @@ void arch_init_thread_fpu(struct thread_ctx *ctx)
struct xsave_area *area;
/* should be aligned to 64 */
ctx->fpu_state = (void *)kzalloc(STATE_AREA_SIZE);
ctx->fpu_state = alloc_fpu_state();
ctx->is_fpu_owner = -1;
area = (struct xsave_area *)ctx->fpu_state;
@ -65,7 +70,7 @@ void arch_init_thread_fpu(struct thread_ctx *ctx)
void arch_init_thread_fpu(struct thread_ctx *ctx)
{
/* should be aligned to 64 */
ctx->fpu_state = (void *)kzalloc(STATE_AREA_SIZE);
ctx->fpu_state = alloc_fpu_state();
ctx->is_fpu_owner = -1;
}
#endif

1
kernel/ckpt/CMakeLists.txt
View File

@ -2,6 +2,7 @@ target_sources(${kernel_target}
PRIVATE ckpt.c
restore.c
ckpt_ipi.c
ckpt_time_traveling.c
ckpt_ws.c
ckpt_object_pool.c
ckpt_page.c

15
kernel/ckpt/ckpt.c
View File

@ -139,14 +139,19 @@ int sys_whole_ckpt(u64 ckpt_name, u64 name_len)
#endif
finish_process_active_list();
#endif
r = ckpt_ws_put(data, ckpt_name, name_len);
if (!r) {
goto out_fail;
}
/* TODO(MOK): remove the following two lines*/
second_latest_ws_data = latest_ws_data;
latest_ws_data = data;
/* issue an async copy */
if (ckpt_name && name_len) {
init_async_copying_task(data->ckpt_root_obj_root,
ckpt_name,
name_len,
get_current_ckpt_version() - 1);
}
smp_mb();
sys_ipi_start_all();
@ -243,7 +248,7 @@ int sys_whole_ckpt_for_test(u64 ckpt_name, u64 name_len, u64 log_level)
#ifdef REPORT
u64 wait_migrate_finish = plat_get_mono_time() - timer_start - recycle_time - ipi_time - object_time - fmap_time;
#endif
r = ckpt_ws_put(data, ckpt_name, name_len);
r = ckpt_ws_put_from_userspace(data, ckpt_name, name_len);
if (!r) {
goto out_fail;
}

62
kernel/ckpt/ckpt_object_pool.c
View File

@ -25,6 +25,16 @@ const obj_restore_func obj_restore_tbl[TYPE_NR] = {
[TYPE_VMSPACE] = vmspace_restore,
};
struct ckpt_ws_data *init_ckpt_ws_data()
{
struct ckpt_ws_data *data;
data = kzalloc(sizeof(*data));
if (!data) {
return NULL;
}
return data;
}
struct ckpt_ws_data *get_ckpt_ws_data()
{
struct ckpt_ws_data *data;
@ -35,7 +45,7 @@ struct ckpt_ws_data *get_ckpt_ws_data()
/* update data version number */
data->version_number = data->version_number + 2;
} else {
data = kzalloc(sizeof(*data));
data = init_ckpt_ws_data();
if (!data) {
goto out_fail;
}
@ -98,6 +108,24 @@ struct ckpt_object *ckpt_obj_alloc(u64 type)
return object;
}
struct ckpt_obj_root *ckpt_obj_root_alloc()
{
struct ckpt_obj_root *root;
root = kmalloc(sizeof(*root));
if (!root)
return NULL;
#if OBJ_OVERWRITE == 1
root->ckpt_objs[0] = NULL;
root->ckpt_objs[1] = NULL;
#endif
root->cow = false;
root->refcnt = 0;
root->flip_flag = system_current_flip_flag ^ 1;
return root;
}
struct ckpt_obj_root *ckpt_obj_root_get(struct object *obj, int alloc)
{
if (!CKPT_GLOBAL_OBJ_MAP || !obj) {
@ -107,19 +135,9 @@ struct ckpt_obj_root *ckpt_obj_root_get(struct object *obj, int alloc)
struct ckpt_obj_root *root = obj->obj_root;
if (!root && alloc) {
root = kmalloc(sizeof(*root));
if (!root)
return NULL;
root->obj = obj;
#if OBJ_OVERWRITE == 1
root->ckpt_objs[0] = NULL;
root->ckpt_objs[1] = NULL;
#endif
root->cow = false;
root->refcnt = 0;
root->flip_flag = system_current_flip_flag ^ 1;
root = ckpt_obj_root_alloc();
obj->obj_root = root;
root->obj = obj;
}
return root;
@ -299,19 +317,31 @@ struct object *restore_obj_get_by_cap_group(struct ckpt_obj_root *ckpt_obj_root,
} else if (alloc) {
ckpt_obj_root->flip_flag = system_current_flip_flag;
ckpt_obj = get_second_latest_ckpt_obj(
ckpt_obj_root, get_current_ckpt_version());
/* get ckpt obj */
if (ckpt_obj_root->time_traveling) {
/* time traveling obj always use the first ckpt_obj */
ckpt_obj = ckpt_obj_root->ckpt_objs[0];
} else {
ckpt_obj = get_second_latest_ckpt_obj(
ckpt_obj_root,
get_current_ckpt_version());
}
BUG_ON(!ckpt_obj);
obj = ckpt_obj_root->obj;
/* mark obj as restored */
kvs_put(obj_map, (kvs_key_t *)&ckpt_obj_root,
(kvs_value_t *)(&obj));
#ifdef RESTORE_REPORT
DECLTMR;
start();
#endif
/* Invoke the object-specific restore routine */
// printk("restore type %d\n", obj->type);
func = obj_restore_tbl[obj->type];
if (func) {
BUG_ON(func(obj, ckpt_obj, obj_map));
BUG_ON(func(obj, ckpt_obj, obj_map, ckpt_obj_root->time_traveling));
} else {
WARN("obj restore func is NULL");
BUG_ON(1);

3
kernel/ckpt/ckpt_object_pool.h
View File

@ -14,8 +14,11 @@ struct object *restore_obj_get(struct ckpt_obj_root *ckpt_obj_root);
struct object *restore_obj_get_by_cap_group(struct ckpt_obj_root *ckpt_obj_root, struct kvs *obj_map,int alloc);
struct ckpt_obj_root *ckpt_obj_root_alloc();
struct ckpt_obj_root *ckpt_obj_root_get(struct object *obj, int alloc);
struct ckpt_obj_root *get_copied_obj_root(struct ckpt_obj_root *ckpt_obj_root, struct kvs *obj_map);
struct ckpt_object *get_latest_ckpt_obj(struct ckpt_obj_root *ckpt_obj_root, u64 version_number);
struct ckpt_object *get_second_latest_ckpt_obj(struct ckpt_obj_root *ckpt_obj_root, u64 version_number);

57
kernel/ckpt/ckpt_objects.h
View File

@ -20,28 +20,49 @@
#include <ckpt/hot_pages_tracker.h>
/* Copy policy */
int cap_group_copy_ckpt(struct cap_group *cap_group, struct ckpt_cap_group *ckpt_cap_group);
int cap_group_restore(struct object *, struct ckpt_object *, struct kvs* obj_map);
int cap_group_copy_ckpt(struct cap_group *cap_group,
struct ckpt_cap_group *ckpt_cap_group);
int cap_group_restore(struct object *, struct ckpt_object *,
struct kvs *obj_map, bool time_traveling);
int ckpt_cap_group_copy(struct ckpt_object *src_obj,
struct ckpt_object *dst_obj, struct kvs *);
int thread_ckpt(struct thread *target, struct ckpt_thread *ckpt_thread);
int thread_restore(struct object *, struct ckpt_object *, struct kvs *obj_map,
bool time_traveling);
int ckpt_thread_copy(struct ckpt_object *src, struct ckpt_object *dst,
struct kvs *);
int thread_ckpt(struct thread *target,struct ckpt_thread *ckpt_thread);
int thread_restore(struct object *, struct ckpt_object *, struct kvs* obj_map);
int vmspace_ckpt(struct vmspace *target_vmspace,
struct ckpt_vmspace *ckpt_vmspace);
int vmspace_restore(struct object *, struct ckpt_object *, struct kvs *obj_map,
bool time_traveling);
int ckpt_vmspace_copy(struct ckpt_object *src_obj, struct ckpt_object *dst_obj,
struct kvs *obj_map);
int connection_ckpt(struct ipc_connection *conn,
struct ckpt_ipc_connection *ckpt_conn, int alloc);
int connection_restore(struct object *, struct ckpt_object *,
struct kvs *obj_map, bool time_traveling);
int ckpt_connection_copy(struct ckpt_object *src_obj,
struct ckpt_object *dst_obj, struct kvs *obj_map);
int vmspace_ckpt(struct vmspace *target_vmspace, struct ckpt_vmspace *ckpt_vmspace);
int vmspace_restore(struct object *, struct ckpt_object *, struct kvs* obj_map);
int connection_ckpt(struct ipc_connection *conn,struct ckpt_ipc_connection *ckpt_conn, int alloc);
int connection_restore(struct object *, struct ckpt_object *, struct kvs* obj_map);
int notification_ckpt(struct notification *notifc, struct ckpt_notification *ckpt_notifc, int alloc);
int notification_restore(struct object *, struct ckpt_object *, struct kvs* obj_map);
int irq_ckpt(struct irq_notification *irq_notifc, struct ckpt_irq_notification *ckpt_irq_notifc, int alloc);
int irq_restore(struct object *, struct ckpt_object *, struct kvs* obj_map);
int notification_ckpt(struct notification *notifc,
struct ckpt_notification *ckpt_notifc, int alloc);
int notification_restore(struct object *, struct ckpt_object *,
struct kvs *obj_map, bool time_traveling);
int ckpt_notification_copy(struct ckpt_object *src_obj,
struct ckpt_object *dst_obj, struct kvs *obj_map);
int irq_ckpt(struct irq_notification *irq_notifc,
struct ckpt_irq_notification *ckpt_irq_notifc, int alloc);
int irq_restore(struct object *, struct ckpt_object *, struct kvs *obj_map,
bool time_traveling);
int ckpt_irq_copy(struct ckpt_object *src_obj, struct ckpt_object *dst_obj,
struct kvs *obj_map);
int pmo_ckpt(struct pmobject *pmo, struct ckpt_pmobject *ckpt_pmo);
int pmo_restore(struct object *, struct ckpt_object *, struct kvs* obj_map);
int pmo_restore(struct object *, struct ckpt_object *, struct kvs *obj_map,
bool time_traveling);
int ckpt_pmo_copy(struct ckpt_object *src_obj, struct ckpt_object *dst_obj,
struct kvs *obj_map);

46
kernel/ckpt/ckpt_objects/cap_group.c
View File

@ -172,7 +172,8 @@ extern int cap_group_init(struct cap_group *cap_group, unsigned int size,
u64 badge);
int cap_group_restore(struct object *cap_group_obj,
struct ckpt_object *ckpt_cap_group_obj,
struct kvs *obj_map)
struct kvs *obj_map,
bool time_traveling)
{
struct cap_group *cap_group = (struct cap_group *)cap_group_obj->opaque;
struct ckpt_cap_group *ckpt_cap_group = (struct ckpt_cap_group *)ckpt_cap_group_obj->opaque;
@ -191,3 +192,46 @@ int cap_group_restore(struct object *cap_group_obj,
/* Restore slot table */
return slot_table_restore(cap_group,ckpt_cap_group,obj_map);
}
int ckpt_slot_table_copy(struct ckpt_object_slot *src_slots,
struct ckpt_object_slot *dst_slots, int size,
struct kvs *obj_map)
{
int i;
struct ckpt_obj_root *copied_obj_root;
for (i = 0; i < size; i++) {
copied_obj_root =
get_copied_obj_root(src_slots[i].obj_root, obj_map);
BUG_ON(!copied_obj_root);
dst_slots[i].slot_id = src_slots[i].slot_id;
dst_slots[i].obj_root = copied_obj_root;
}
return 0;
}
int ckpt_cap_group_copy(struct ckpt_object *src_obj,
struct ckpt_object *dst_obj, struct kvs *obj_map)
{
struct ckpt_cap_group *src_cap_group, *dst_cap_group;
src_cap_group = (struct ckpt_cap_group *)src_obj->opaque;
dst_cap_group = (struct ckpt_cap_group *)dst_obj->opaque;
/* Copy the contents of src_cap_group to dst_cap_group */
memcpy(dst_cap_group, src_cap_group, sizeof(struct ckpt_cap_group));
dst_cap_group->slots = (struct ckpt_object_slot *)kmalloc(
(dst_cap_group->table_size) * sizeof(struct ckpt_object_slot));
kinfo("copy cap group: size %u (%u), badge %lx, name %s\n",
dst_cap_group->table_size,
src_cap_group->table_size,
dst_cap_group->badge,
dst_cap_group->cap_group_name);
/* Copy slot table */
return ckpt_slot_table_copy(src_cap_group->slots,
dst_cap_group->slots,
src_cap_group->table_size,
obj_map);
}

34
kernel/ckpt/ckpt_objects/connection.c
View File

@ -50,7 +50,10 @@ out_fail:
return r;
}
int connection_restore(struct object *conn_obj, struct ckpt_object *ckpt_conn_obj, struct kvs *obj_map)
int connection_restore(struct object *conn_obj,
struct ckpt_object *ckpt_conn_obj,
struct kvs *obj_map,
bool time_traveling)
{
int r;
struct ipc_connection *conn = (struct ipc_connection *)conn_obj->opaque;
@ -91,3 +94,32 @@ int connection_restore(struct object *conn_obj, struct ckpt_object *ckpt_conn_ob
out_fail:
return r;
}
int ckpt_connection_copy(struct ckpt_object *src_obj, struct ckpt_object *dst_obj, struct kvs *obj_map)
{
struct ckpt_ipc_connection *src, *dst;
src = (struct ckpt_ipc_connection *)src_obj->opaque;
dst = (struct ckpt_ipc_connection *)dst_obj->opaque;
/* Copy the contents of src to dst */
memcpy(dst, src, sizeof(struct ckpt_ipc_connection));
/* Update the object roots using the object map */
if (src->current_client_thread_root) {
dst->current_client_thread_root = get_copied_obj_root(
src->current_client_thread_root, obj_map);
if (!dst->current_client_thread_root) {
return -ENOMEM;
}
}
if (src->server_handler_thread_root) {
dst->server_handler_thread_root = get_copied_obj_root(
src->server_handler_thread_root, obj_map);
if (!dst->server_handler_thread_root) {
return -ENOMEM;
}
}
return 0;
}

31
kernel/ckpt/ckpt_objects/irq.c
View File

@ -13,7 +13,10 @@ int irq_ckpt(struct irq_notification *irq_notifc, struct ckpt_irq_notification *
return notification_ckpt(&irq_notifc->notifc,&ckpt_irq_notifc->notifc, alloc);
}
int irq_restore(struct object *irq_obj, struct ckpt_object *ckpt_irq_obj, struct kvs *obj_map)
int irq_restore(struct object *irq_obj,
struct ckpt_object *ckpt_irq_obj,
struct kvs *obj_map,
bool time_traveling)
{
int i, r;
struct irq_notification *irq_notifc = (struct irq_notification *)irq_obj->opaque;
@ -50,3 +53,29 @@ int irq_restore(struct object *irq_obj, struct ckpt_object *ckpt_irq_obj, struct
out_fail:
return r;
}
int ckpt_irq_copy(struct ckpt_object *src_obj, struct ckpt_object *dst_obj, struct kvs *obj_map)
{
struct ckpt_irq_notification *src_irq, *dst_irq;
int r;
src_irq = (struct ckpt_irq_notification *)src_obj->opaque;
dst_irq = (struct ckpt_irq_notification *)dst_obj->opaque;
/* Copy basic fields */
dst_irq->intr_vector = src_irq->intr_vector;
dst_irq->status = src_irq->status;
dst_irq->user_handler_ready = src_irq->user_handler_ready;
/* Copy the notification part */
r = ckpt_notification_copy(
(struct ckpt_object *)&src_irq->notifc,
(struct ckpt_object *)&dst_irq->notifc,
obj_map
);
if (r) {
return r;
}
return 0;
}

38
kernel/ckpt/ckpt_objects/notification.c
View File

@ -38,7 +38,10 @@ out_fail:
return r;
}
int notification_restore(struct object *notifc_obj, struct ckpt_object *ckpt_notifc_obj, struct kvs *obj_map)
int notification_restore(struct object *notifc_obj,
struct ckpt_object *ckpt_notifc_obj,
struct kvs *obj_map,
bool time_traveling)
{
int r, i;
struct notification *notifc = (struct notification *)notifc_obj->opaque;
@ -76,4 +79,35 @@ int notification_restore(struct object *notifc_obj, struct ckpt_object *ckpt_not
return 0;
out_fail:
return r;
}
}
int ckpt_notification_copy(struct ckpt_object *src_obj, struct ckpt_object *dst_obj, struct kvs *obj_map)
{
struct ckpt_notification *src, *dst;
int i;
src = (struct ckpt_notification *)src_obj->opaque;
dst = (struct ckpt_notification *)dst_obj->opaque;
/* Copy basic fields */
dst->not_delivered_notifc_count = src->not_delivered_notifc_count;
dst->waiting_threads_count = src->waiting_threads_count;
dst->state = src->state;
/* Allocate memory for waiting_thread_roots */
dst->waiting_thread_roots = kmalloc(src->waiting_threads_count * sizeof(struct ckpt_obj_root *));
if (!dst->waiting_thread_roots) {
return -ENOMEM;
}
/* Copy waiting thread roots */
for (i = 0; i < src->waiting_threads_count; i++) {
dst->waiting_thread_roots[i] = get_copied_obj_root(src->waiting_thread_roots[i], obj_map);
if (!dst->waiting_thread_roots[i]) {
kfree(dst->waiting_thread_roots);
return -ENOMEM;
}
}
return 0;
}

327
kernel/ckpt/ckpt_objects/pmobject.c
View File

@ -1,6 +1,7 @@
#include "../ckpt_objects.h"
#include "../ckpt_object_pool.h"
#include "../ckpt_page.h"
#include "../ckpt_ws.h"
extern int pgtbl_deep_copy(vaddr_t *src_pgtbl, vaddr_t *dst_pgtbl);
@ -304,7 +305,10 @@ int choose_nvm_ckpt_page_idx(struct ckpt_page_pair *page_pair)
}
}
static int __radix_pmo_restore(struct pmobject *pmo, struct radix_node *page_node, struct radix_node *ckpt_page_node, int node_level, u64 prefix)
static int __radix_pmo_restore(struct pmobject *pmo,
struct radix_node *page_node,
struct radix_node *ckpt_page_node,
int node_level, u64 prefix, bool time_traveling)
{
int err;
int i;
@ -316,16 +320,37 @@ static int __radix_pmo_restore(struct pmobject *pmo, struct radix_node *page_nod
prefix <<= RADIX_NODE_BITS;
if(node_level == RADIX_LEVELS - 1) {
for(i = 0;i < RADIX_NODE_SIZE;i++) {
if((page_pair = ckpt_page_node->values[i]) || page_node->values[i]) {
if (node_level == RADIX_LEVELS - 1) {
for (i = 0;i < RADIX_NODE_SIZE;i++) {
page_pair = ckpt_page_node->values[i];
/* ckpt radix or runtime radix have value */
if (page_pair || page_node->values[i]) {
/* meta */
pa = (paddr_t)page_node->values[i];
BUG_ON(!pa);
va = (vaddr_t)phys_to_virt(pa);
/* set page info */
struct page* page = virt_to_page((void*)va);
page->track_info = NULL;
page_type_t type = get_page_type(page);
/* set page info */
page->track_info = NULL;
/* restore page */
/* case 1: restore from time traveling */
if (time_traveling) {
/* use page_pair[0] to restore */
if (page_pair) {
pagecpy((void*)va, (void*)page_pair->pages[0].va);
}
/* clear runtime page */
else if (!page_pair && va) {
kfree((void*)va);
page_node->values[i] = 0;
}
return 0;
}
/* case 2: restore from normal NVM page */
if (type == NVM_PAGE) {
if (page_pair) {
// int idx = choose_ckpt_page_idx(page_pair);
@ -349,22 +374,20 @@ static int __radix_pmo_restore(struct pmobject *pmo, struct radix_node *page_nod
}
#endif
}
} else {
}
/* case 3: restore from normal DRAM page */
else {
BUG_ON(!page_pair);
int idx = choose_ckpt_page_idx(page_pair);
int stale_idx = 1 - idx;
/* Use page_pair->pages[stale_idx] as runtime memory and page_pair->pages[idx] as backup */
/* Use page_pair->pages[stale_idx] as runtime memory */
/* Use page_pair->pages[idx] as backup */
BUG_ON(page_pair->pages[idx].version_number == 0);
va = page_pair->pages[stale_idx].va;
clear_ckpt_page(page_pair, stale_idx);
pagecpy((void*)va, (void*)page_pair->pages[idx].va);
page_node->values[i] = (void*)virt_to_phys((void*)va);
struct page *page = virt_to_page((void*)va);
@ -395,7 +418,8 @@ static int __radix_pmo_restore(struct pmobject *pmo, struct radix_node *page_nod
page_node->children[i],
ckpt_page_node->children[i],
node_level + 1,
prefix | i);
prefix | i,
time_traveling);
if (err)
return err;
}
@ -404,7 +428,7 @@ static int __radix_pmo_restore(struct pmobject *pmo, struct radix_node *page_nod
return 0;
}
int radix_pmo_restore(struct pmobject *pmo, struct ckpt_pmobject *ckpt_pmo)
int radix_pmo_restore(struct pmobject *pmo, struct ckpt_pmobject *ckpt_pmo, bool time_traveling)
{
int r = 0;
struct radix *pmo_radix = pmo->radix;
@ -428,46 +452,82 @@ int radix_pmo_restore(struct pmobject *pmo, struct ckpt_pmobject *ckpt_pmo)
}
pmo_radix->root = new_node;
}
r = __radix_pmo_restore(pmo, pmo_radix->root, ckpt_page_radix->root, 0, 0);
unlock(&ckpt_page_radix->radix_lock);
r = __radix_pmo_restore(pmo,
pmo_radix->root,
ckpt_page_radix->root,
0,
0,
time_traveling);
unlock(&ckpt_page_radix->radix_lock);
unlock(&pmo_radix->radix_lock);
return r;
}
static int __continuous_pmo_restore(struct pmobject *pmo, struct radix_node *ckpt_page_node, int node_level, u64 prefix)
static int __continuous_pmo_restore(struct pmobject *pmo,
struct radix_node *ckpt_page_node,
int node_level, u64 prefix,
bool time_traveling)
{
int err;
int i;
int err, i, idx, stale_idx;
struct ckpt_page_pair *page_pair;
paddr_t pa;
vaddr_t va;
u64 index;
u64 current_version = get_current_ckpt_version();
struct page *page;
prefix <<= RADIX_NODE_BITS;
if (node_level == RADIX_LEVELS - 1) {
for (i = 0;i < RADIX_NODE_SIZE;i++) {
if ((page_pair = ckpt_page_node->values[i])) {
page_pair = ckpt_page_node->values[i];
if (page_pair) {
index = prefix | i;
/* meta */
pa = (paddr_t)(pmo->start + index * PAGE_SIZE);
va = (vaddr_t)phys_to_virt(pa);
/* set page info */
struct page* page = virt_to_page((void*)va);
page->track_info = NULL;
page = virt_to_page((void*)va);
int idx = choose_ckpt_page_idx(page_pair);
int stale_idx = 1 - idx;
/* set page info */
page->track_info = NULL;
/**
* case 1: time traveling
* always use the first page to restore
*/
if (time_traveling) {
pagecpy((void*)va, (void*)page_pair->pages[0].va);
return 0;
}
/* case 2: non-time traveling */
idx = choose_ckpt_page_idx(page_pair);
stale_idx = 1 - idx;
clear_ckpt_page(page_pair, stale_idx);
/* use dram_cache to know page type */
if (pmo->dram_cache.array == NULL || pmo->dram_cache.array[index] == 0) {
/* page type == NVM */
/**
* case 2.1: page type == NVM
* Figure 6(a): va is the newest page unless
* page_pair->pages[idx].version_number == current_version
*/
BUG_ON(page_pair->pages[stale_idx].va == page_pair->pages[idx].va);
BUG_ON(va == page_pair->pages[idx].va || va != page_pair->pages[stale_idx].va);
BUG_ON(va == page_pair->pages[idx].va ||
va != page_pair->pages[stale_idx].va);
if (page_pair->pages[idx].version_number == current_version) {
pagecpy((void*)va, (void*)page_pair->pages[idx].va);
}
} else {
pagecpy((void*)page_pair->pages[stale_idx].va, (void*)page_pair->pages[idx].va);
/**
* case 2.2: page type == DRAM
* this page must be checkpointed, page_pair has the newest page
* Figure 6(b) DRAM-to-NVM:
* 1. copy two page_pair as the same
* 2. clear the stale page
*/
pagecpy((void*)page_pair->pages[stale_idx].va,
(void*)page_pair->pages[idx].va);
if (va == page_pair->pages[idx].va) {
page_pair->pages[stale_idx].version_number = current_version;
clear_ckpt_page(page_pair, idx);
@ -483,7 +543,8 @@ static int __continuous_pmo_restore(struct pmobject *pmo, struct radix_node *ckp
err = __continuous_pmo_restore(pmo,
ckpt_page_node->children[i],
node_level + 1,
prefix | i);
prefix | i,
time_traveling);
if (err)
return err;
}
@ -492,7 +553,7 @@ static int __continuous_pmo_restore(struct pmobject *pmo, struct radix_node *ckp
return 0;
}
int continuous_pmo_restore(struct pmobject *pmo, struct radix *ckpt_page_radix)
int continuous_pmo_restore(struct pmobject *pmo, struct radix *ckpt_page_radix, bool time_traveling)
{
int r = 0;
struct radix_node *new_node;
@ -505,32 +566,36 @@ int continuous_pmo_restore(struct pmobject *pmo, struct radix *ckpt_page_radix)
ckpt_page_radix->root = new_node;
}
r = __continuous_pmo_restore(pmo, ckpt_page_radix->root, 0, 0);
r = __continuous_pmo_restore(pmo, ckpt_page_radix->root, 0, 0, time_traveling);
unlock(&ckpt_page_radix->radix_lock);
return r;
}
static int __init_ckpt_page_radix(struct radix_node *page_node, struct radix_node *ckpt_page_node, int node_level)
static int __init_ckpt_page_radix(struct radix_node *page_node,
struct radix_node *ckpt_page_node,
int node_level)
{
int err;
int i;
struct radix_node *new;
struct page *page;
struct ckpt_page_pair *page_pair;
if (node_level == RADIX_LEVELS - 1) {
for (i = 0;i < RADIX_NODE_SIZE;i++) {
if (page_node->values[i]) {
struct ckpt_page_pair *page_pair = kzalloc(sizeof(*page_pair));
page_pair = kzalloc(sizeof(*page_pair));
if (!page_pair) {
return -ENOMEM;
}
// printk("%s: page=%p\n", __func__, virt_to_page((void*)phys_to_virt(page_node->values[i])));
page_pair->pages[0].va = (vaddr_t)get_pages(0);
page_pair->pages[1].va = phys_to_virt(page_node->values[i]);
pagecpy((void*)page_pair->pages[0].va, (void*)phys_to_virt(page_node->values[i]));
pagecpy((void *)page_pair->pages[0].va,
(void *)phys_to_virt(page_node->values[i]));
page_pair->pages[0].version_number = get_current_ckpt_version();
ckpt_page_node->values[i] = page_pair;
page = (struct page*)virt_to_page((void*)phys_to_virt(page_node->values[i]));
page = (struct page *)virt_to_page(
(void *)phys_to_virt(page_node->values[i]));
page->page_pair = (u64)page_pair;
}
}
@ -570,14 +635,10 @@ int init_ckpt_page_radix(struct ckpt_pmobject* ckpt_pmo, struct pmobject *pmo)
ckpt_page_radix = ckpt_pmo->radix;
/* copy all pages in pmo to ckpt page in ckpt pmo*/
if(use_radix(pmo)) {
// lock(&pmo_radix->radix_lock);
// lock(&ckpt_page_radix->radix_lock);
if (use_radix(pmo)) {
r = __init_ckpt_page_radix(pmo_radix->root,
ckpt_page_radix->root,
0);
// unlock(&ckpt_page_radix->radix_lock);
// unlock(&pmo_radix->radix_lock);
}
return r;
@ -585,27 +646,27 @@ int init_ckpt_page_radix(struct ckpt_pmobject* ckpt_pmo, struct pmobject *pmo)
int pmo_ckpt(struct pmobject *pmo, struct ckpt_pmobject *ckpt_pmo)
{
/*Step 1: init ckpt pmo */
/* Step 1: init ckpt pmo */
int r;
u64 current_ckpt_version;
u64 current_ckpt_version = get_current_ckpt_version();
struct ckpt_obj_root *pmo_root;
struct ckpt_object *latest_ckpt_obj;
struct ckpt_pmobject *latest_ckpt_pmo;
ckpt_pmo->private = pmo->private;
ckpt_pmo->size = pmo->size;
ckpt_pmo->type = pmo->type;
current_ckpt_version = get_current_ckpt_version();
if (use_continuous_pages(pmo) || use_radix(pmo)) {
if (unlikely(!ckpt_pmo->radix)) {
/* If the radix tree is not created, try to reuse the
* previous version of the radix tree */
struct ckpt_obj_root *pmo_root;
struct ckpt_object *latest_ckpt_obj;
pmo_root = ckpt_obj_root_get(container_of(pmo, struct object,opaque), false);
if (pmo_root && (latest_ckpt_obj =
get_latest_ckpt_obj(pmo_root, current_ckpt_version))) {
struct ckpt_pmobject *latest_ckpt_pmo =
(struct ckpt_pmobject*)latest_ckpt_obj->opaque;
pmo_root = ckpt_obj_root_get(container_of(pmo, struct object, opaque),
false);
latest_ckpt_obj = get_latest_ckpt_obj(pmo_root,
current_ckpt_version);
if (pmo_root && latest_ckpt_obj) {
latest_ckpt_pmo = (struct ckpt_pmobject*)latest_ckpt_obj->opaque;
BUG_ON(!latest_ckpt_pmo->radix);
ckpt_pmo->radix = latest_ckpt_pmo->radix;
} else {
@ -646,7 +707,10 @@ int pmo_ckpt(struct pmobject *pmo, struct ckpt_pmobject *ckpt_pmo)
return 0;
}
int pmo_restore(struct object *pmo_obj, struct ckpt_object *ckpt_pmo_obj, struct kvs* obj_map)
int pmo_restore(struct object *pmo_obj,
struct ckpt_object *ckpt_pmo_obj,
struct kvs *obj_map,
bool time_traveling)
{
int r = 0;
struct ckpt_pmobject *ckpt_pmo = (struct ckpt_pmobject *)ckpt_pmo_obj->opaque;
@ -672,8 +736,11 @@ int pmo_restore(struct object *pmo_obj, struct ckpt_object *ckpt_pmo_obj, struct
struct page *sp;
if (use_continuous_pages(pmo)) {
continuous_pmo_restore(pmo, ckpt_pmo->radix);
pmo->dram_cache.array = NULL;
continuous_pmo_restore(pmo, ckpt_pmo->radix, time_traveling);
/* clear dram cache */
if (pmo->dram_cache.array) {
clear_dram_cache(pmo);
}
lock_init(&pmo->dram_cache.lock);
pmo_start_va = phys_to_virt(pmo->start);
@ -695,8 +762,7 @@ int pmo_restore(struct object *pmo_obj, struct ckpt_object *ckpt_pmo_obj, struct
} else if (use_radix(pmo)) {
/* restore radix tree */
lock_init(&pmo->radix->radix_lock);
r = radix_pmo_restore(pmo, ckpt_pmo);
r = radix_pmo_restore(pmo, ckpt_pmo, time_traveling);
#ifdef PMO_CHECKSUM
if (!is_external_sync_pmo(pmo)) {
printk("[pmo %lx, type:%d]\n", pmo, pmo->type);
@ -734,17 +800,15 @@ static int __radix_deep_copy_with_hybird_mem(struct radix_node *src, struct radi
void *src_pa = src->values[i];
void *src_va = (void*)phys_to_virt(src_pa);
struct page *page = virt_to_page(src_va);
if(get_page_type(page) == NVM_PAGE) {
if (get_page_type(page) == NVM_PAGE) {
lock(&page->lock);
if(page->track_info) {
if (page->track_info->active) {
delete_from_active_list(page->track_info);
}
if (page->track_info && page->track_info->active) {
delete_from_active_list(page->track_info);
}
atomic_fetch_add_64(&page->ref_cnt,1);
atomic_fetch_add_64(&page->ref_cnt, 1);
unlock(&page->lock);
dst->values[i] = src_pa;
}else {
} else {
void *newpage = get_pages(0);
BUG_ON(!newpage);
pagecpy(newpage,
@ -803,4 +867,133 @@ int radix_deep_copy_with_hybird_mem(struct radix *src,struct radix *dst)
out:
unlock(&src->radix_lock);
return r;
}
}
static int __ckpt_radix_deep_copy(struct radix_node *src, struct radix_node *dst, u64 version_number, int node_level)
{
int err;
int i;
struct radix_node *new;
struct ckpt_page_pair *src_page_pair, *dst_page_pair;
struct ckpt_page *page;
if (node_level == RADIX_LEVELS - 1) {
for (i = 0; i < RADIX_NODE_SIZE; i++) {
src_page_pair = src->values[i];
if (src_page_pair) {
page = get_ckpt_page_with_version(src_page_pair, version_number);
/* get or alloc page's dst page pair */
if (page->tt_page) {
dst_page_pair = page->tt_page;
dst_page_pair->refcnt++;
} else {
dst_page_pair = kzalloc(sizeof(struct ckpt_page_pair));
dst_page_pair->type = CKPT_PP_TT;
dst_page_pair->refcnt = 1;
page->tt_page = dst_page_pair;
}
/* copy according to type */
if (src_page_pair->type == CKPT_PP_DRAM) {
/* direct copt dram cached page */
void *newpage = get_pages(0);
BUG_ON(!newpage);
pagecpy(newpage, (void *)page->va);
dst_page_pair->pages[0].va = (u64)newpage;
} else if (src_page_pair->type == CKPT_PP_NVM) {
/* delay copy when page is modified */
/* first use the same va with src ckpt page */
dst_page_pair->pages[0].va = page->va;
}
/* set dst page pair */
dst->values[i] = dst_page_pair;
}
}
return 0;
}
for (i = 0; i < RADIX_NODE_SIZE; i++) {
if (src->children[i]) {
new = kzalloc(sizeof(struct radix_node));
if (IS_ERR(new)) {
return -ENOMEM;
}
dst->children[i] = new;
err = __ckpt_radix_deep_copy(src->children[i],
dst->children[i],
version_number,
node_level + 1);
if (err) {
return err;
}
}
}
return 0;
}
int ckpt_radix_deep_copy(struct radix *src, struct radix *dst, u64 version_number)
{
int r;
struct radix_node *new;
BUG_ON(!(src && dst));
r = 0;
/* don't need to lock dst */
lock(&src->radix_lock);
if (!src->root) {
goto out;
}
if (!dst->root) {
new = kzalloc(sizeof(struct radix_node));
if (IS_ERR(new)) {
r = -ENOMEM;
}
dst->root = new;
}
r = __ckpt_radix_deep_copy(src->root, dst->root, version_number, 0);
out:
unlock(&src->radix_lock);
return r;
}
int ckpt_pmo_copy(struct ckpt_object *src_obj, struct ckpt_object *dst_obj, struct kvs *obj_map)
{
struct ckpt_pmobject *src_pmo, *dst_pmo;
int r;
src_pmo = (struct ckpt_pmobject *)src_obj->opaque;
dst_pmo = (struct ckpt_pmobject *)dst_obj->opaque;
/* Copy basic fields */
dst_pmo->size = src_pmo->size;
dst_pmo->start = src_pmo->start;
dst_pmo->type = src_pmo->type;
/* Copy radix tree */
if (src_pmo->radix) {
/* init radix tree */
dst_pmo->radix = kzalloc(sizeof(struct radix));
if (!dst_pmo->radix) {
return -ENOMEM;
}
init_radix(dst_pmo->radix);
/* Deep copy the radix tree */
r = ckpt_radix_deep_copy(src_pmo->radix,
dst_pmo->radix,
latest_node->version_number);
if (r) {
kfree(dst_pmo->radix);
return r;
}
} else {
dst_pmo->radix = NULL;
}
return 0;
}

146
kernel/ckpt/ckpt_objects/thread.c
View File

@ -1,5 +1,8 @@
#include "../ckpt_objects.h"
#include "../ckpt_object_pool.h"
#include "../ckpt_ws.h"
#include "common/kvstore.h"
#include "sched/sched.h"
void* alloc_ckpt_ipc_config(int config_type)
{
@ -127,7 +130,63 @@ void ipc_config_ckpt(void* config, void* ckpt_config, int config_type)
}
}
int thread_ckpt(struct thread *target,struct ckpt_thread *ckpt_thread)
void ckpt_ipc_config_copy(void* src_ipc_config, void* dst_ipc_config, int config_type, struct kvs *obj_map)
{
switch(config_type) {
case IPC_SERVER_REGISTER_CB:{
struct ckpt_ipc_server_register_cb_config *src =
(struct ckpt_ipc_server_register_cb_config *)src_ipc_config;
struct ckpt_ipc_server_register_cb_config *dst =
(struct ckpt_ipc_server_register_cb_config *)dst_ipc_config;
memcpy(dst, src, sizeof(struct ckpt_ipc_server_register_cb_config));
break;
}
case IPC_SERVER_HANDLER:{
struct ckpt_ipc_server_handler_config *src =
(struct ckpt_ipc_server_handler_config *)src_ipc_config;
struct ckpt_ipc_server_handler_config *dst =
(struct ckpt_ipc_server_handler_config *)dst_ipc_config;
struct ckpt_obj_root *ckpt_obj_root;
memcpy(dst, src, sizeof(struct ckpt_ipc_server_handler_config));
if (src->active_conn_root) {
ckpt_obj_root = get_copied_obj_root(src->active_conn_root, obj_map);
BUG_ON(ckpt_obj_root == NULL);
dst->active_conn_root = ckpt_obj_root;
} else {
dst->active_conn_root = NULL;
}
break;
}
case IPC_SERVER:{
struct ckpt_ipc_server_config *src =
(struct ckpt_ipc_server_config *)src_ipc_config;
struct ckpt_ipc_server_config *dst =
(struct ckpt_ipc_server_config *)dst_ipc_config;
struct ckpt_obj_root *ckpt_thread_obj_root;
dst->config_type = src->config_type;
dst->declared_ipc_routine_entry =
src->declared_ipc_routine_entry;
if (src->register_cb_thread_root) {
ckpt_thread_obj_root = get_copied_obj_root(src->register_cb_thread_root, obj_map);
BUG_ON(ckpt_thread_obj_root == NULL);
dst->register_cb_thread_root = ckpt_thread_obj_root;
} else {
dst->register_cb_thread_root = NULL;
}
break;
}
default:
BUG_ON(1);
}
}
int thread_ckpt(struct thread *target, struct ckpt_thread *ckpt_thread)
{
/* Copy thread_ctx data */
thread_ctx_ckpt(&ckpt_thread->thread_ctx,target->thread_ctx);
@ -142,11 +201,9 @@ int thread_ckpt(struct thread *target,struct ckpt_thread *ckpt_thread)
ckpt_config = (struct ipc_config *)ckpt_thread->general_ipc_config;
if (!ckpt_config) {
/* we can not reuse it if the pointer of config is NULL*/
ckpt_config = alloc_ckpt_ipc_config(config_type);
} else if (config_type
!= ((struct ipc_config *)ckpt_thread->general_ipc_config)
->config_type) {
/* we can not reuse it if the pointer of config is NULL*/
ckpt_config = alloc_ckpt_ipc_config(config_type);
} else if (config_type != ((struct ipc_config *)ckpt_thread->general_ipc_config)->config_type) {
/* we can not reuse it if the type of ipc config in checkpoint
* is different from the target thread*/
kfree(ckpt_config);
@ -181,7 +238,7 @@ int thread_ckpt(struct thread *target,struct ckpt_thread *ckpt_thread)
}
static void thread_ctx_restore(struct ckpt_thread_ctx *ckpt_ctx,struct thread_ctx *target_ctx)
static void thread_ctx_restore(struct ckpt_thread_ctx *ckpt_ctx, struct thread_ctx *target_ctx)
{
int i;
BUG_ON(ckpt_ctx == NULL || target_ctx == NULL);
@ -215,7 +272,9 @@ static void thread_ctx_restore(struct ckpt_thread_ctx *ckpt_ctx,struct thread_ct
}
}
void thread_sleep_state_restore(struct ckpt_sleep_state *ckpt_sleep_state, struct sleep_state *sleep_state, struct kvs *obj_map)
void thread_sleep_state_restore(struct ckpt_sleep_state *ckpt_sleep_state,
struct sleep_state *sleep_state,
struct kvs *obj_map)
{
/* Copy fields */
sleep_state->cb = ckpt_sleep_state->cb;
@ -229,7 +288,10 @@ void thread_sleep_state_restore(struct ckpt_sleep_state *ckpt_sleep_state, struc
lock_init(&sleep_state->queue_lock);
}
int thread_restore(struct object *thread_obj, struct ckpt_object *ckpt_thread_obj, struct kvs *obj_map)
int thread_restore(struct object *thread_obj,
struct ckpt_object *ckpt_thread_obj,
struct kvs *obj_map,
bool time_traveling)
{
struct ckpt_thread *ckpt_thread = (struct ckpt_thread *)ckpt_thread_obj->opaque;
struct thread *target = (struct thread *)thread_obj->opaque;
@ -271,11 +333,11 @@ int thread_restore(struct object *thread_obj, struct ckpt_object *ckpt_thread_ob
new_config->ipc_routine_entry = ckpt_config->ipc_routine_entry;
new_config->ipc_routine_stack = ckpt_config->ipc_routine_stack;
new_config->ipc_lock = ckpt_config->ipc_lock;
if(ckpt_config->active_conn_root) {
if (ckpt_config->active_conn_root) {
new_conn_obj = restore_obj_get(ckpt_config->active_conn_root);
BUG_ON(new_conn_obj == NULL);
new_config->active_conn = (struct ipc_connection*)new_conn_obj->opaque;
}else {
} else {
new_config->active_conn = NULL;
}
@ -303,7 +365,7 @@ int thread_restore(struct object *thread_obj, struct ckpt_object *ckpt_thread_ob
default:
BUG_ON(1);
}
}else {
} else {
target->general_ipc_config = NULL;
}
@ -318,12 +380,11 @@ int thread_restore(struct object *thread_obj, struct ckpt_object *ckpt_thread_ob
cap_obj = restore_obj_get(ckpt_thread->vmspace_root);
BUG_ON(!cap_obj);
struct vmspace* thread_vmspace = (struct vmspace*)cap_obj->opaque;
// struct vmspace* thread_vmspace = obj_get(thread_cap_group,VMSPACE_OBJ_ID,TYPE_VMSPACE);
// BUG_ON(!thread_vmspace);
BUG_ON(!thread_vmspace);
target->vmspace = thread_vmspace;
// obj_put(thread_vmspace);
kdebug("[Restore] restore thread %lx from %s, type=%u, rip=%lx, rsp=%lx, "
"state=%u, cpuid=%u, kernel_stack_state=%u, id=%lx\n",
kdebug("[Restore] restore thread %lx from %s, type=%u, rip=%lx, rsp=%lx,\n"
"\tstate=%u, cpuid=%u, kernel_stack_state=%u, id=%lx\n",
target,
thread_cap_group->cap_group_name,
target->thread_ctx->type,
@ -349,7 +410,6 @@ int thread_restore(struct object *thread_obj, struct ckpt_object *ckpt_thread_ob
case TS_RUNNING:
case TS_READY:{
target->thread_ctx->state = TS_INTER;
// kinfo("thread %lx enqueue\n",target);
BUG_ON(sched_enqueue(target));
break;
}
@ -361,5 +421,55 @@ int thread_restore(struct object *thread_obj, struct ckpt_object *ckpt_thread_ob
break;
}
}
if (target == current_thread) {
target->thread_ctx->state = TS_RUNNING;
}
return 0;
}
#define STATE_AREA_SIZE (sizeof(struct xsave_area))
int ckpt_thread_copy(struct ckpt_object *src_obj, struct ckpt_object *dst_obj, struct kvs *obj_map)
{
struct ckpt_thread *src = (struct ckpt_thread *)src_obj->opaque;
struct ckpt_thread *dst = (struct ckpt_thread *)dst_obj->opaque;
/* Copy thread_ctx */
memcpy(&dst->thread_ctx,
&src->thread_ctx,
sizeof(struct ckpt_thread_ctx));
/* Copt thread_ctx->fpu_state */
dst->thread_ctx.fpu_state = alloc_fpu_state();
memcpy(dst->thread_ctx.fpu_state,
src->thread_ctx.fpu_state,
get_fpu_state_size());
/* Copy sleep_state */
memcpy(&dst->sleep_state,
&src->sleep_state,
sizeof(struct ckpt_sleep_state));
/* Copy cap_group_root and vmspace_root */
dst->cap_group_root = get_copied_obj_root(src->cap_group_root, obj_map);
dst->vmspace_root = get_copied_obj_root(src->vmspace_root, obj_map);
/* Copy general_ipc_config */
if (src->general_ipc_config) {
int config_type = ((struct ipc_config*)src->general_ipc_config)->config_type;
dst->general_ipc_config = alloc_ckpt_ipc_config(config_type);
if (!dst->general_ipc_config) {
return -ENOMEM;
}
ckpt_ipc_config_copy(src->general_ipc_config,
dst->general_ipc_config,
config_type,
obj_map);
} else {
dst->general_ipc_config = NULL;
}
return 0;
}

47
kernel/ckpt/ckpt_objects/vmspace.c
View File

@ -187,7 +187,10 @@ out_fail:
return NULL;
}
int vmspace_restore(struct object *vm_obj, struct ckpt_object *ckpt_vm_obj, struct kvs* obj_map)
int vmspace_restore(struct object *vm_obj,
struct ckpt_object *ckpt_vm_obj,
struct kvs *obj_map,
bool time_traveling)
{
int r;
struct vmspace *target_vmspace = (struct vmspace *)vm_obj->opaque;
@ -231,3 +234,45 @@ int vmspace_restore(struct object *vm_obj, struct ckpt_object *ckpt_vm_obj, stru
#endif
return r;
}
int ckpt_vmspace_copy(struct ckpt_object *src_obj, struct ckpt_object *dst_obj, struct kvs *obj_map)
{
struct ckpt_vmspace *src_vmspace, *dst_vmspace;
int i, r;
src_vmspace = (struct ckpt_vmspace *)src_obj->opaque;
dst_vmspace = (struct ckpt_vmspace *)dst_obj->opaque;
/* Copy basic fields */
dst_vmspace->pcid = src_vmspace->pcid;
dst_vmspace->user_current_mmap_addr = src_vmspace->user_current_mmap_addr;
dst_vmspace->vmr_count = src_vmspace->vmr_count;
dst_vmspace->heap_vmr_idx = src_vmspace->heap_vmr_idx;
/* Allocate memory for ckpt_vmrs */
dst_vmspace->ckpt_vmrs = kmalloc(src_vmspace->vmr_count * sizeof(struct ckpt_vmregion));
if (!dst_vmspace->ckpt_vmrs) {
return -ENOMEM;
}
/* Copy each vmregion */
for (i = 0; i < src_vmspace->vmr_count; i++) {
memcpy(&dst_vmspace->ckpt_vmrs[i], &src_vmspace->ckpt_vmrs[i], sizeof(struct ckpt_vmregion));
/* Update the pmo_root using the object map */
if (src_vmspace->ckpt_vmrs[i].pmo_root) {
dst_vmspace->ckpt_vmrs[i].pmo_root = get_copied_obj_root(
src_vmspace->ckpt_vmrs[i].pmo_root, obj_map);
if (!dst_vmspace->ckpt_vmrs[i].pmo_root) {
r = -ENOMEM;
goto out_free_vmrs;
}
}
}
return 0;
out_free_vmrs:
kfree(dst_vmspace->ckpt_vmrs);
return r;
}

36
kernel/ckpt/ckpt_page.c
View File

@ -1,9 +1,10 @@
#include <common/radix.h>
#include "ckpt_page.h"
#include <mm/mm.h>
#include <mm/nvm.h>
#include <common/list.h>
#include "ckpt_page.h"
/*
* when the version number of page_0 and page_1 are equal,
* we treat page_0 as the new one.
@ -20,6 +21,19 @@ inline struct ckpt_page* get_second_latest_ckpt_page(struct ckpt_page_pair *page
&page_pair->pages[1] : &page_pair->pages[0];
}
inline struct ckpt_page* get_ckpt_page_with_version(struct ckpt_page_pair *page_pair, u64 version_number)
{
/* return the largest version less than @version_number */
if (page_pair->pages[0].version_number <= version_number &&
(page_pair->pages[1].version_number > version_number ||
page_pair->pages[0].version_number > page_pair->pages[1].version_number)) {
return &page_pair->pages[0];
} else if (page_pair->pages[1].version_number <= version_number) {
return &page_pair->pages[1];
}
BUG("No legal page\n");
}
struct ckpt_page_pair *get_page_pair(struct page *page, u64 index)
{
struct ckpt_page_pair *page_pair;
@ -118,19 +132,28 @@ int ckpt_nvm_page(struct pmobject *pmo, void *kva, u64 index)
u64 current_version = get_current_ckpt_version();
struct ckpt_obj_root* obj_root = container_of(pmo, struct object, opaque)->obj_root;
if (!obj_root) {
/* page has not been checkpointed */
return -1;
}
struct page *page = virt_to_page(kva);
struct ckpt_page_pair *page_pair = get_page_pair(page, index);
struct ckpt_page *ckpt_page = get_latest_ckpt_page(page_pair);
/* nvm page use COW method, so we should update the latest ckpt page */
// BUG_ON(ckpt_page->va != page_pair->pages[0].va);
/* dram page use selective copy or direct-copy method,so we should update the second latest ckpt page */
// ckpt_page = get_second_latest_ckpt_page(page_pair);
if (ckpt_page->version_number != current_version) {
/* should never get the runtime page */
BUG_ON((void*)ckpt_page->va == kva);
/* if the page is refered by some time traveling ckpt */
if (ckpt_page->tt_page) {
/**
* (1) the time traveling ckpt will use the old page
* (2) the two-version ckpt will use a new page
*/
kinfo("ckpt_nvm_page: page %lx is refered by time traveling ckpt\n", kva);
ckpt_page->va = (vaddr_t)get_pages(0);
ckpt_page->tt_page = NULL;
}
pagecpy_nt((void*)ckpt_page->va, kva);
ckpt_page->version_number = current_version;
}
@ -154,6 +177,7 @@ void ckpt_dram_cached_page(struct pmobject *pmo, void *kva, u64 index)
BUG_ON(page->pmo != pmo);
page_pair = get_page_pair(page, index);
page_pair->type = CKPT_PP_DRAM;
BUG_ON(!page_pair);
/* dram-cached page use selective copy or direct-copy method,

1
kernel/ckpt/ckpt_page.h
View File

@ -5,6 +5,7 @@
struct ckpt_page* get_latest_ckpt_page(struct ckpt_page_pair *page_pair);
struct ckpt_page *get_second_latest_ckpt_page(struct ckpt_page_pair *page_pair);
struct ckpt_page* get_ckpt_page_with_version(struct ckpt_page_pair *page_pair, u64 version_number);
struct ckpt_page_pair *get_page_pair(struct page *page, u64 index);
void free_page_pair(struct ckpt_page_pair *page_pair);

185
kernel/ckpt/ckpt_time_traveling.c Normal file
View File

@ -0,0 +1,185 @@
#include <common/kvstore.h>
#include <common/errno.h>
#include <common/util.h>
#include <object/thread.h>
#include <ckpt/ckpt_data.h>
#include <ckpt/ckpt.h>
#include <arch/mmu.h>
#include <object/cap_group.h>
#include <mm/mm.h>
#include <mm/kmalloc.h>
#include <mm/nvm.h>
#include <object/user_fault.h>
#include "ckpt_object_pool.h"
#include "ckpt_objects.h"
#include "ckpt_ws.h"
bool async_copying = false;
struct time_travel_node *latest_node = NULL;
extern struct ckpt_ws_data *init_ckpt_ws_data();
extern int recycle_create_ckpt(struct ckpt_recycle_data* recycle_data);
int init_async_copying_task(struct ckpt_obj_root *src_root_obj, u64 ckpt_name,
u64 name_len, u64 version_number)
{
int r = 0;
if (latest_node) {
BUG_ON(latest_node->finished == false);
} else {
latest_node = kzalloc(sizeof(*latest_node));
if (!latest_node)
return -ENOMEM;
}
async_copying = true;
latest_node->finished = false;
latest_node->version_number = version_number;
latest_node->src_root_obj = src_root_obj;
latest_node->ckpt_data = init_ckpt_ws_data();
if (!latest_node->ckpt_data) {
r = -ENOMEM;
goto out_fail;
}
latest_node->obj_map = new_kvs(KVS_SIZE);
if (!latest_node->obj_map) {
r = -ENOMEM;
goto out_fail_free_ckpt;
}
/* init ckpt name */
if (name_len > MAX_CKPT_NAME_LEN) {
kinfo("[CKPT WS] only support name len < %d, truncked\n",
MAX_CKPT_NAME_LEN);
latest_node->ckpt_name_len = MAX_CKPT_NAME_LEN;
} else
latest_node->ckpt_name_len = name_len;
r = copy_from_user((char *)latest_node->ckpt_name,
(char *)ckpt_name,
latest_node->ckpt_name_len);
if (r) {
kinfo("[INIT ASYNC] Could not copy ckpt name from user.\n");
return r;
}
return r;
out_fail_free_ckpt:
kfree(latest_node->ckpt_data);
out_fail:
async_copying = false;
latest_node->version_number = -1;
return r;
}
const obj_copy_func obj_copy_tbl[TYPE_NR] = {
[0 ... TYPE_NR - 1] = NULL,
[TYPE_CAP_GROUP] = ckpt_cap_group_copy,
[TYPE_THREAD] = ckpt_thread_copy,
[TYPE_CONNECTION] = ckpt_connection_copy,
[TYPE_NOTIFICATION] = ckpt_notification_copy,
[TYPE_IRQ] = ckpt_irq_copy,
[TYPE_PMO] = ckpt_pmo_copy,
[TYPE_VMSPACE] = ckpt_vmspace_copy,
};
struct ckpt_obj_root *get_copied_obj_root(struct ckpt_obj_root *ckpt_obj_root,
struct kvs *obj_map)
{
struct ckpt_obj_root *copied_obj_root = NULL;
struct ckpt_obj_root **copied_obj_root_ptr =
(struct ckpt_obj_root **)kvs_get(obj_map,
(kvs_key_t *)(&ckpt_obj_root));
struct ckpt_object *src_obj, *dst_obj;
obj_copy_func func;
if (copied_obj_root_ptr) {
/* object is already copyed */
copied_obj_root = *copied_obj_root_ptr;
// kinfo("kvs_get: obj_map=%p key=%p, value=%p\n", obj_map,
// ckpt_obj_root, copied_obj_root);
} else {
// kinfo("kvs_get: obj_map=%p key=%p FAIL\n", obj_map,
// ckpt_obj_root);
/* get src_obj and dst_obj to be copied */
src_obj = get_latest_ckpt_obj(ckpt_obj_root,
latest_node->version_number);
BUG_ON(!src_obj);
copied_obj_root = ckpt_obj_root_alloc();
BUG_ON(!copied_obj_root);
copied_obj_root->obj = ckpt_obj_root->obj;
dst_obj = ckpt_obj_alloc(src_obj->type);
if (!dst_obj) {
goto out_fail;
}
copied_obj_root->ckpt_objs[0] = dst_obj; // always use first
/* add ckpt_obj_root <=> copied_obj_root */
/* must put into kvs before calling function */
// kinfo("kvs_put: obj_map=%p, key=%p, value=%p\n", obj_map,
// ckpt_obj_root, copied_obj_root);
kvs_put(obj_map,
(kvs_key_t *)(&ckpt_obj_root),
(kvs_value_t *)(&copied_obj_root));
/* excute copy function */
func = obj_copy_tbl[src_obj->type];
if (func) {
BUG_ON(func(src_obj, dst_obj, obj_map));
} else {
WARN("obj copy func is NULL");
BUG_ON(1);
}
}
copied_obj_root->time_traveling = true;
return copied_obj_root;
out_fail:
return NULL;
}
int sys_copy_time_traveling_data()
{
int r = 0;
struct ckpt_obj_root *copied_ckpt_obj_root;
struct ckpt_ws_data *data;
if (!async_copying) {
return r;
}
data = latest_node->ckpt_data;
/* async copy */
BUG_ON(!latest_node || !latest_node->obj_map);
copied_ckpt_obj_root = get_copied_obj_root(latest_node->src_root_obj,
latest_node->obj_map);
if (!copied_ckpt_obj_root)
goto out_fail;
data->ckpt_root_obj_root = copied_ckpt_obj_root;
/* copy recycle data and fmap pool */
recycle_create_ckpt(&data->recycle_data);
fmap_fault_pool_create_ckpt(&data->ckpt_fmap_fault_pool_list);
/* put ckpt data into mapping and mark finished */
r = ckpt_ws_put(latest_node->ckpt_data,
latest_node->ckpt_name,
latest_node->ckpt_name_len);
if (!r) {
goto out_fail;
}
async_copying = false;
latest_node->finished = true;
out_fail:
return r;
}

64
kernel/ckpt/ckpt_ws.c
View File

@ -156,12 +156,51 @@ struct ckpt_ws_data *ckpt_ws_get_latest()
latest = list_entry(CKPT_WS_TABLE->ckpt_ws_list.next,
struct ckpt_ws_info, node);
kinfo("[CKPT WS] latest ckpt(id=%lx) found\n", (u64)latest);
kinfo("[CKPT WS] latest ckpt (id=%lx) found\n", (u64)latest);
return ckpt_ws_get((u64)latest);
}
/* put ckpt data */
u64 ckpt_ws_put(struct ckpt_ws_data *ckpt_data, u64 name_buf, u64 name_len)
u64 ckpt_ws_put(struct ckpt_ws_data *ckpt_data, char *name, u64 name_len)
{
struct ckpt_ws_info *info;
int ret = 0;
info = (struct ckpt_ws_info *)kmalloc(sizeof(*info));
if (!info) {
kinfo("[CKPT WS] can not allocate memory for ckpt info.\n");
return 0;
}
info->ckpt_data = ckpt_data;
info->ts = (timestamp_t)plat_get_mono_time();
if (name_len) {
if (name_len > MAX_CKPT_NAME_LEN) {
name_len = MAX_CKPT_NAME_LEN;
}
memcpy(info->name, name, name_len);
}
info->name_len = name_len;
/* add pair(info, data) */
kvs_put(CKPT_WS_TABLE->ckpt_ws_kvs, (kvs_key_t *)&info,
(kvs_value_t *)&ckpt_data);
/* add pair(hash(name), info*) to kvs */
ret = __name_kvs_put(info);
if (ret) {
kinfo("[CKPT WS] error during add name kvs\n");
return 0;
}
/* add to list is atomic point */
list_add(&(info->node), &(CKPT_WS_TABLE->ckpt_ws_list));
return (u64)info;
}
/* put ckpt data */
u64 ckpt_ws_put_from_userspace(struct ckpt_ws_data *ckpt_data, u64 name_buf, u64 name_len)
{
struct ckpt_ws_info *info;
int ret = 0;
@ -218,20 +257,31 @@ u64 ckpt_ws_put(struct ckpt_ws_data *ckpt_data, u64 name_buf, u64 name_len)
struct ckpt_ws_info *ckpt_ws_query_by_name(char *name, u64 name_len)
{
ckpt_ws_info_list_t *info_list_head;
ckpt_ws_info_list_t **info_list_head_ptr;
u64 hash_val, ckpt_id;
BUG_ON(!name || name_len ==0);
BUG_ON(!CKPT_WS_TABLE || !CKPT_WS_TABLE->name_kvs);
/* get info list */
hash_val = __name_hash(name, name_len);
info_list_head = *((ckpt_ws_info_list_t **)kvs_get(CKPT_WS_TABLE->name_kvs,
(kvs_key_t *)&hash_val));
ckpt_id = __query_info_list(&(info_list_head->list), name, name_len);
info_list_head_ptr = ((ckpt_ws_info_list_t **)kvs_get(
CKPT_WS_TABLE->name_kvs,
(kvs_key_t *)&hash_val));
if (!info_list_head_ptr) {
goto not_found;
}
info_list_head = *info_list_head_ptr;
ckpt_id = __query_info_list(&(info_list_head->list),
name, name_len);
if (!ckpt_id) {
kinfo("[CKPT WS] no ckpt found with name %s\n", name);
return NULL;
goto not_found;
}
return (struct ckpt_ws_info *)ckpt_id;
not_found:
// kinfo("[CKPT WS] no ckpt found with name %s\n", name);
return NULL;
}

18
kernel/ckpt/ckpt_ws.h
View File

@ -8,6 +8,19 @@ typedef u64 timestamp_t;
#define KVS_SIZE (503)
struct time_travel_node {
int version_number;
struct ckpt_ws_data *ckpt_data;
struct kvs *obj_map;
struct ckpt_obj_root *src_root_obj;
char ckpt_name[MAX_CKPT_NAME_LEN];
u64 ckpt_name_len;
bool finished;
};
extern bool async_copying;
extern struct time_travel_node *latest_node;
/*
* store ckpt_ws_info in list, sorted by ts
* use ckpt_id as key to map to ckpt_data
@ -32,9 +45,12 @@ int ckpt_ws_init(void);
/* get and put */
struct ckpt_ws_data *ckpt_ws_get(u64 ckpt_id);
struct ckpt_ws_data *ckpt_ws_get_latest();
u64 ckpt_ws_put(struct ckpt_ws_data *ckpt_data, u64 name_buf, u64 len);
u64 ckpt_ws_put(struct ckpt_ws_data *ckpt_data, char *name, u64 len);
u64 ckpt_ws_put_from_userspace(struct ckpt_ws_data *ckpt_data, u64 name_buf, u64 name_len);
/* query utils */
struct ckpt_ws_info *ckpt_ws_query_by_name(char *name, u64 name_len);
ckpt_ws_info_list_t ckpt_ws_query_by_time(timestamp_t st, timestamp_t et);
ckpt_ws_info_list_t ckpt_ws_query_latest(u64 cnt);
int init_async_copying_task(struct ckpt_obj_root *src_root_obj, u64 ckpt_name, u64 name_len, u64 version_number);

7
kernel/ckpt/hybird_mem.c
View File

@ -92,8 +92,8 @@ static inline void move_runtime_nvm_page_to_backup(struct pmobject *pmo, void *k
BUG_ON(!ckpt_obj);
ckpt_pmo = (struct ckpt_pmobject*)ckpt_obj->opaque;
page_pair = radix_get(ckpt_pmo->radix, index);
if (!page_pair)
BUG("page(%p) pmo->type=%d\n", virt_to_page(kva), pmo->type);
BUG_ON(!page_pair);
page_pair->type = CKPT_PP_DRAM;
ckpt_page = get_second_latest_ckpt_page(page_pair);
BUG_ON(ckpt_page->va != (vaddr_t)kva);
@ -119,6 +119,7 @@ move_runtime_nvm_page_from_backup(struct pmobject *pmo, u64 index)
ckpt_pmo = (struct ckpt_pmobject*)ckpt_obj->opaque;
page_pair = radix_get(ckpt_pmo->radix, index);
BUG_ON(!page_pair);
page_pair->type = CKPT_PP_NVM;
ckpt_page = get_latest_ckpt_page(page_pair);
old_ckpt_page = get_second_latest_ckpt_page(page_pair);
@ -219,7 +220,7 @@ static int migrate_page(struct page *old_page, bool to_dram)
/* 2. update pte to new_pa */
for_each_in_list(item, struct reverse_node, node, &(pmo->reverse_list)) {
vmr = item->vmr;
if ((vmr == NULL))
if (vmr == NULL)
continue;
BUG_ON(vmr->pmo != pmo);
#ifdef CHECK_MIGRATE

14
kernel/ckpt/restore.c
View File

@ -87,17 +87,27 @@ int sys_whole_restore(u64 ckpt_name, u64 name_len)
goto out_fail;
}
set_current_ckpt_version(data->version_number);
/* clear the sched queues first */
sched_clear();
/* restore root_cap_group obj */
obj = restore_obj_get_by_cap_group(ckpt_obj_root, obj_map, true);
root_cap_group_obj_for_ckpt = obj;
root_cap_group = (struct cap_group *)(obj->opaque);
/* restore other system variables */
recycle_restore(&data->recycle_data, obj_map);
fmap_fault_pool_restore(&data->ckpt_fmap_fault_pool_list, obj_map);
second_latest_ws_data = NULL;
latest_ws_data = data;
/* free the tmp obj map */
kfree(obj_map);
if (!data->ckpt_root_obj_root->time_traveling) {
second_latest_ws_data = NULL;
latest_ws_data = data;
}
#ifdef RESTORE_REPORT
int tcnt = 0;
for (int i = 0; i < TYPE_NR; i++) {

4
kernel/include/ckpt/ckpt.h
View File

@ -10,13 +10,15 @@
#define FREE_MEM_LEVEL (4)
#define DETAIL_LEVEL (5)
typedef int (*obj_restore_func)(struct object *, struct ckpt_object *, struct kvs*);
typedef int (*obj_restore_func)(struct object *, struct ckpt_object *, struct kvs*, bool time_traveling);
typedef int (*obj_copy_func)(struct ckpt_object *, struct ckpt_object *, struct kvs*);
int ckpt_metadata_init(void);
int sys_whole_ckpt(u64 ckpt_name, u64 name_len);
int sys_whole_ckpt_for_test(u64 ckpt_name, u64 name_len, u64 log_level);
int sys_whole_restore(u64 ckpt_name, u64 name_len);
int sys_copy_time_traveling_data();
void sys_ipi_stop_all();
void sys_ipi_start_all();

14
kernel/include/ckpt/ckpt_data.h
View File

@ -103,8 +103,19 @@ struct ckpt_object_slot {
struct ckpt_page {
vaddr_t va;
u64 version_number;
/* NULL == no refered; !NULL == cow to this */
struct ckpt_page_pair *tt_page;
};
enum ckpt_page_pair_type {
CKPT_PP_DRAM,
CKPT_PP_NVM,
CKPT_PP_TT,
};
struct ckpt_page_pair {
u8 type;
u64 refcnt;
struct ckpt_page pages[2];
};
@ -116,6 +127,8 @@ struct ckpt_pmobject {
pmo_type_t type;
void *private;
u64 flags;
/* list of ckpt object that refers to this radix tree */
struct list_head *cow_obj_list;
#ifdef PMO_CHECKSUM
u64 checksum;
struct radix *radix_backup;
@ -300,6 +313,7 @@ struct ckpt_obj_root {
bool cow;
int flip_flag;
u64 refcnt;
bool time_traveling; /* not belong to two-latest version */
};
struct ckpt_ipc_server_handler_config {

2
kernel/include/common/debug.h
View File

@ -15,7 +15,7 @@
* ChCore will check each free operation in the slab allocator.
* Please refer to kernel/mm/slab.c.
*/
#define DETECTING_DOUBLE_FREE_IN_SLAB OFF
#define DETECTING_DOUBLE_FREE_IN_SLAB ON
/*
* When BACKTRACE_FUNC is ON,

2
kernel/include/irq/timer.h
View File

@ -44,7 +44,7 @@ struct sleep_state {
extern struct list_head sleep_lists[PLAT_CPU_NUM];
extern u64 tick_per_us;
void sleep_state_enqueue(struct sleep_state *sleep_state); // TODO(FN): rename?
void sleep_state_enqueue(struct sleep_state *sleep_state);
int enqueue_sleeper(struct thread *thread, const struct timespec *timeout,
timer_cb cb);
bool try_dequeue_sleeper(struct thread *thread);

1
kernel/include/mm/buddy.h
View File

@ -24,6 +24,7 @@ enum pageflags {
PG_cached, /* page is moved from NVM to DRAM */
PG_patched, /* page has patch */
PG_flagnum,
PG_ckpt, /* ckpt page*/
};
enum page_type {

1
kernel/include/mm/vmspace.h
View File

@ -198,6 +198,7 @@ void switch_vmspace_to(struct vmspace *);
void commit_page_to_pmo(struct pmobject *pmo, u64 index, paddr_t pa);
void commit_dram_cached_page(struct pmobject *pmo, u64 index, paddr_t pa);
void clear_dram_cached_page(struct pmobject *pmo, u64 index);
void clear_dram_cache(struct pmobject *pmo);
paddr_t get_page_from_pmo(struct pmobject *pmo, u64 index);

4
kernel/include/object/thread.h
View File

@ -11,6 +11,10 @@
extern struct thread *current_threads[PLAT_CPU_NUM];
#define current_thread (current_threads[smp_get_cpu_id()])
extern bool resched_flags[PLAT_CPU_NUM];
#define current_resched_flag (resched_flags[smp_get_cpu_id()])
#ifdef CHCORE_KERNEL_RT
/* RT (kernel PREEMT): allocate the stack for each thread */
#define DEFAULT_KERNEL_STACK_SZ (0x1000)

5
kernel/include/sched/context.h
View File

@ -1,7 +1,12 @@
#pragma once
#include <sched/sched.h>
#include <common/types.h>
#define STATE_AREA_SIZE (sizeof(struct xsave_area))
void *alloc_fpu_state();
u64 get_fpu_state_size();
struct thread_ctx *create_thread_ctx(u32 type);
void destroy_thread_ctx(struct thread *thread);
void init_thread_ctx(struct thread *thread, u64 stack, u64 func, u32 prio, u32 type, s32 aff);

7
kernel/include/sched/sched.h
View File

@ -146,6 +146,8 @@ struct sched_ops {
int (*sched_dequeue)(struct thread * thread);
/* Debug tools */
void (*sched_top)(void);
/* CKPT restore tools */
void (*sched_clear)(void);
};
/* Provided Scheduling Policies */
@ -172,6 +174,11 @@ static inline int sched_dequeue(struct thread *thread)
return cur_sched_ops->sched_dequeue(thread);
}
static inline void sched_clear(void)
{
cur_sched_ops->sched_clear();
}
/* Syscalls */
void sys_yield(void);
void sys_top(void);

37
kernel/irq/timer.c
View File

@ -104,7 +104,9 @@ void handle_timer_irq(void)
BUG_ON(wakeup_thread->sleep_state.cb == sleep_timer_cb
&& wakeup_thread->thread_ctx->state != TS_WAITING);
kdebug("wake up t:%p at:%ld\n", wakeup_thread, current_tick);
BUG_ON(wakeup_thread->sleep_state.cb == NULL);
if (wakeup_thread->sleep_state.cb == NULL) {
BUG("wake up a thread %p: cb == NULL\n", wakeup_thread);
}
wakeup_thread->sleep_state.cb(wakeup_thread);
wakeup_thread->sleep_state.cb = NULL;
@ -156,17 +158,40 @@ int sys_clock_gettime(clockid_t clock, struct timespec *ts)
return 0;
}
void sleep_state_enqueue(struct sleep_state *sleep_state) {
void printk_sleep_state_queue(struct sleep_state *sleep_state)
{
struct sleep_state *iter;
struct list_head *local_sleep_list = &time_states[sleep_state->sleep_cpu].sleep_list;
kinfo("[printk_sleep_state_queue] sleep_state queue on cpu:%ld\n", sleep_state->sleep_cpu);
for_each_in_list(iter, struct sleep_state, sleep_node,
local_sleep_list) {
if (iter->wakeup_tick > sleep_state->wakeup_tick)
break;
BUG_ON(!sleep_state->cb);
kinfo("\t\tsleep_state:%p, wakeup_tick:%ld\n",
iter,
iter->wakeup_tick);
}
}
void sleep_state_enqueue(struct sleep_state *sleep_state) {
struct sleep_state *iter, *insert;
bool find_insert = false;
struct list_head *local_sleep_list = &(time_states[sleep_state->sleep_cpu].sleep_list);
insert = container_of(local_sleep_list, struct sleep_state, sleep_node);
for_each_in_list(iter, struct sleep_state, sleep_node,
local_sleep_list) {
if (iter->wakeup_tick > sleep_state->wakeup_tick) {
if (!find_insert) {
insert = iter;
find_insert = true;
}
}
if (iter == sleep_state) {
// kinfo("sleep state already in list\n");
return;
}
}
list_append(&sleep_state->sleep_node, &iter->sleep_node);
list_append(&sleep_state->sleep_node, &insert->sleep_node);
}
int enqueue_sleeper(struct thread *thread, const struct timespec *timeout,

4
kernel/lib/printk.c
View File

@ -360,11 +360,11 @@ void printk(const char *fmt, ...)
{
extern void lock(struct lock *lock);
extern void unlock(struct lock *lock);
// lock(&printk_lock);
lock(&printk_lock);
va_list va;
va_start(va, fmt);
simple_vsprintf(NULL, fmt, va);
va_end(va);
// unlock(&printk_lock);
unlock(&printk_lock);
}

6
kernel/mm/kmalloc.c
View File

@ -94,9 +94,13 @@ void *kmalloc(size_t size)
addr = alloc_in_slab(size);
} else {
if (size <= BUDDY_PAGE_SIZE)
order = 0;
order = 0;
else
order = size_to_page_order(size);
#if TRACK_THREAD_MM == ON
if (current_thread)
current_thread->mm_size += (BUDDY_PAGE_SIZE * (1 << order));
#endif
addr = get_pages(order);
}

67
kernel/mm/pgfault_handler.c
View File

@ -124,13 +124,17 @@ int handle_trans_fault(struct vmspace *vmspace, vaddr_t fault_addr, int present,
case PMO_RING_BUFFER_RADIX:
case PMO_ANONYM:
case PMO_SHM: {
vmr_prop_t perm;
perm = vmr->perm;
vmr_prop_t perm = vmr->perm;
/* Get the offset in the pmo for faulting addr */
offset = ROUND_DOWN(fault_addr, PAGE_SIZE) - vmr->start;
/* Get the index in the pmo radix for faulting addr */
index = offset / PAGE_SIZE;
/* Round down the fault address */
fault_addr = ROUND_DOWN(fault_addr, PAGE_SIZE);
/* Boundary check */
if ((offset >= pmo->size) && (pmo->type == PMO_FILE)) {
static int warn = 1;
@ -142,24 +146,17 @@ int handle_trans_fault(struct vmspace *vmspace, vaddr_t fault_addr, int present,
warn = 0;
}
/*
/**
* FIXME: we simply allow it now by adding new pages.
* For PMO_FILE, users can mmap a memory that is larger
* than the file size. If they accesses bytes beyond
* file size, SIGBUS should be triggered on LInux.
*
* TODO: why setting all the perm here?
*/
perm = VMR_READ | VMR_WRITE | VMR_EXEC;
} else {
BUG_ON(offset >= pmo->size);
}
/* Get the index in the pmo radix for faulting addr */
index = offset / PAGE_SIZE;
fault_addr = ROUND_DOWN(fault_addr, PAGE_SIZE);
pa = get_page_from_pmo(pmo, index);
/* PMO_FILE fault means user fault */
@ -182,9 +179,11 @@ int handle_trans_fault(struct vmspace *vmspace, vaddr_t fault_addr, int present,
BUG_ON(new_va == NULL);
pa = virt_to_phys(new_va);
BUG_ON(pa == 0);
/* Clear to 0 for the newly allocated page */
memset(new_va, 0, PAGE_SIZE);
/*
/**
* Record the physical page in the radix tree:
* the offset is used as index in the radix tree
*/
@ -197,29 +196,35 @@ int handle_trans_fault(struct vmspace *vmspace, vaddr_t fault_addr, int present,
perm, &pte);
unlock(&vmspace->pgtbl_lock);
#ifdef EXT_SYNC_ENABLED
/* do not persist pages belong to external sync pmo */
if (is_external_sync_pmo(pmo))
break;
#endif
#ifdef OMIT_BENCHMARK
/* omit track page fault of redis-benchmark */
/* omit track page fault of benchmark */
if (need_omit(vmspace)) {
break;
}
#endif
#ifndef OMIT_PF
page = virt_to_page(new_va);
if ((vmspace->flags & VM_FLAG_PRESERVE) && !is_external_sync_pmo(pmo)) {
if ((vmspace->flags & VM_FLAG_PRESERVE)
#ifdef EXT_SYNC_ENABLED
&& !is_external_sync_pmo(pmo)
#endif
) {
#ifdef PMO_CHECKSUM
page->ckpt_version_number = get_current_ckpt_version() + 1;
#endif
#ifndef OMIT_BENCHMARK
#ifndef OMIT_BENCHMARK
ckpt_nvm_page(pmo, new_va, index);
#endif
#endif
add_pte_patch_to_pool(vmspace, (pte_t *)pte, page);
}
#endif
} else {
/*
/**
* pa != 0: the faulting address has be committed a
* physical page.
*
@ -240,12 +245,17 @@ int handle_trans_fault(struct vmspace *vmspace, vaddr_t fault_addr, int present,
* needs to add the mapping in the page table.
* Repeated mapping operations are harmless.
*/
/* For PMO_FILE, we simply set all the perm now. */
if (pmo->type == PMO_FILE) {
perm = VMR_READ | VMR_WRITE | VMR_EXEC;
}
#ifndef OMIT_PF
if ((vmspace->flags & VM_FLAG_PRESERVE) && !write && !is_external_sync_pmo(pmo)) {
if ((vmspace->flags & VM_FLAG_PRESERVE) && !write
#ifdef EXT_SYNC_ENABLED
&& !is_external_sync_pmo(pmo)
#endif
) {
/* Read preserved page, map as read-only */
perm &= ~VMR_WRITE;
}
@ -320,11 +330,13 @@ int handle_trans_fault(struct vmspace *vmspace, vaddr_t fault_addr, int present,
lock(&vmspace->pgtbl_lock);
map_page_in_pgtbl(vmspace->pgtbl, fault_addr, pa, perm, &pte);
unlock(&vmspace->pgtbl_lock);
/* skip add mapping in the page table */
skip_add_mapping:
#ifdef EXT_SYNC_ENABLED
/* do not persist pages belong to external sync pmo */
if (is_external_sync_pmo(pmo))
break;
#endif
#ifndef OMIT_PF
#ifdef OMIT_BENCHMARK
/* omit track page fault of benchmarks */
@ -334,11 +346,8 @@ skip_add_mapping:
#endif
if (write && (vmspace->flags & VM_FLAG_PRESERVE)) {
page = virt_to_page((void*)phys_to_virt(pa));
BUG_ON(unlikely(!page));
if (unlikely(get_page_type(page) != NVM_PAGE)) {
/* Dram page will be marked as unwritable after fork */
break;
}
BUG_ON(!page);
BUG_ON(get_page_type(page) != NVM_PAGE);
#ifndef OMIT_MEMCPY
/* copy page to ckpt_page */
if (pmo != page->pmo) {
@ -349,19 +358,19 @@ skip_add_mapping:
/* Add pte patch */
add_pte_patch_to_pool(vmspace, pte, page);
#ifdef HYBRID_MEM
if(!ckpt_ret)
if (!ckpt_ret)
track_access(page);
#else
UNUSED(ckpt_ret);
#endif
#ifdef REPORT_RUNTIME
#ifdef REPORT_RUNTIME
pf_count++;
pf_tot_time += stop();
LOG("[ckpt=%llu] [page fault count] page=%p, pte=%p\n", CKPT_VERSION_NUMBER, page, pte);
#endif
#endif
}
#else
UNUSED(ckpt_ret);
UNUSED(ckpt_ret);
#endif
}
break;

2
kernel/mm/slab.c
View File

@ -111,6 +111,8 @@ static struct slab_header *init_slab_cache(int order, int size)
}
slot->next_free = NULL;
// printk("slab: init slab cache, order=%d, cnt=%lu\n", order, cnt);
return slab;
}

24
kernel/object/memory.c
View File

@ -303,11 +303,11 @@ int pmo_clone(struct pmobject *dst_pmo, struct pmobject *src_pmo, bool *is_cow)
case PMO_DATA_NOCACHE: {
lock_init(&(dst_pmo->dram_cache.lock));
if(src_pmo->dram_cache.array != NULL) {
if (src_pmo->dram_cache.array != NULL) {
/* Just copy */
*is_cow = false;
void *new_va = kmalloc(dst_pmo->size);
if(new_va == NULL) {
if (new_va == NULL) {
return -ENOMEM;
}
dst_pmo->start = (paddr_t)virt_to_phys(new_va);
@ -316,11 +316,11 @@ int pmo_clone(struct pmobject *dst_pmo, struct pmobject *src_pmo, bool *is_cow)
array = src_pmo->dram_cache.array;
page_num = DIV_ROUND_UP(src_pmo->size, PAGE_SIZE);
for(i = 0; i < page_num; i++) {
for (i = 0; i < page_num; i++) {
u64 src_pa, dst_pa = dst_pmo->start + i * PAGE_SIZE;
if(array[i] != 0) {
if (array[i] != 0) {
src_pa = array[i];
}else {
} else {
src_pa = src_pmo->start + i * PAGE_SIZE;
}
memcpy((void*)phys_to_virt(dst_pa), (void*)phys_to_virt(src_pa), PAGE_SIZE);
@ -797,6 +797,14 @@ void clear_dram_cached_page(struct pmobject *pmo, u64 index)
pmo->dram_cache.array[index] = 0;
}
void clear_dram_cache(struct pmobject *pmo)
{
BUG_ON(!use_continuous_pages(pmo));
BUG_ON(!pmo->dram_cache.array);
kfree(pmo->dram_cache.array);
pmo->dram_cache.array = NULL;
}
/* Record the physical page allocated to a pmo */
void commit_page_to_pmo(struct pmobject *pmo, u64 index, paddr_t pa)
{
@ -1160,7 +1168,11 @@ inline bool is_external_sync_pmo(struct pmobject *pmo)
inline bool is_shared_pmo(struct pmobject *pmo)
{
if (pmo->type == PMO_SHM || is_external_sync_pmo(pmo))
if (pmo->type == PMO_SHM
#ifdef EXT_SYNC_ENABLED
|| is_external_sync_pmo(pmo)
#endif
)
return true;
return false;
}

26
kernel/sched/policy_rr.c
View File

@ -259,7 +259,8 @@ int rr_sched(void)
/* do nothing */
break;
default:
kinfo("thread state: %d\n", old->thread_ctx->state);
// kinfo("thread state: %s\n", thread_state[old->thread_ctx->state]);
print_thread(old);
BUG_ON(1);
break;
}
@ -345,10 +346,31 @@ void rr_top(void)
}
}
void rr_sched_clear()
{
/* clear all queues */
int i;
for (i = 0; i < PLAT_CPU_NUM; i++) {
lock(&(rr_ready_queue_meta[i].queue_lock));
rr_ready_queue_meta[i].queue_len = 0;
init_list_head(&(rr_ready_queue_meta[i].queue_head));
unlock(&(rr_ready_queue_meta[i].queue_lock));
}
/* clear all current threads and mark resched */
for (i = 0; i < PLAT_CPU_NUM; i++) {
if (i == smp_get_cpu_id())
continue;
current_threads[i] = NULL;
resched_flags[i] = true;
}
}
struct sched_ops rr = {
.sched_init = rr_sched_init,
.sched = rr_sched,
.sched_enqueue = rr_sched_enqueue,
.sched_dequeue = rr_sched_dequeue,
.sched_top = rr_top
.sched_top = rr_top,
.sched_clear = rr_sched_clear,
};

1
kernel/sched/sched.c
View File

@ -20,6 +20,7 @@
struct thread *current_threads[PLAT_CPU_NUM];
struct thread idle_threads[PLAT_CPU_NUM];
bool resched_flags[PLAT_CPU_NUM];
#if PLAT_CPU_NUM <= 32
u32 resched_bitmaps[PLAT_CPU_NUM];

2
kernel/sls_config.cmake
View File

@ -1,5 +1,5 @@
# SLS Basic Configruations
set(SLS_RESTORE ON)
set(SLS_RESTORE OFF)
set(SLS_EXT_SYNC OFF)
set(SLS_HYBRID_MEM ON)

1
kernel/syscall/syscall.c
View File

@ -359,6 +359,7 @@ const void *syscall_table[NR_SYSCALL] = {
[SYS_whole_restore] = sys_whole_restore,
[SYS_whole_ckpt_for_test] = sys_whole_ckpt_for_test,
[SYS_register_external_ringbuf] = sys_register_external_ringbuf,
[SYS_copy_time_traveling_data] = sys_copy_time_traveling_data,
/* IPI */
[SYS_ipi_stop_all] = sys_ipi_stop_all,

1
kernel/syscall/syscall_num.h
View File

@ -104,6 +104,7 @@
#define SYS_shutdown 243
#define SYS_whole_ckpt_for_test 244
#define SYS_register_external_ringbuf 245
#define SYS_copy_time_traveling_data 249
/* IPI */
#define SYS_ipi_stop_all 246

4
user/CMakeLists.txt
View File

@ -29,8 +29,8 @@ add_custom_target(
|| echo "No custom ramdisk found, skipping"
# archive ramdisk into ramdisk.cpio
COMMAND find . | cpio -o -H newc > ${CMAKE_CURRENT_BINARY_DIR}/ramdisk.cpio
DEPENDS rocksdb)
# FIXME: ramdisk should wait for all things being copied, add more to DEPENDS.
)
# FIXME: ramdisk should wait for all things being copied, add more to DEPENDS.
include(CommonTools)
include(UserTools)

1
user/musl-1.1.24/include/chcore/defs.h
View File

@ -190,6 +190,7 @@
#define CHCORE_SYS_shutdown 243
#define CHCORE_SYS_whole_ckpt_for_test 244
#define CHCORE_SYS_register_external_ringbuf 245
#define CHCORE_SYS_copy_time_traveling_data 249
/* IPI */
#define CHCORE_SYS_ipi_stop_all 246

1
user/musl-1.1.24/include/chcore/syscall.h
View File

@ -71,6 +71,7 @@ int usys_whole_ckpt(char *ckpt_name, u64 name_len);
int usys_whole_ckpt_for_test(char *ckpt_name, u64 name_len, u64 log_level);
int usys_whole_restore(char *ckpt_name, u64 name_len);
int usys_register_external_ringbuf(u64 buffer);
int usys_copy_time_traveling_data();
void usys_ipi_stop_all(void);
void usys_ipi_start_all(void);

5
user/musl-1.1.24/src/chcore-port/syscall.c
View File

@ -301,6 +301,11 @@ int usys_whole_restore(char *ckpt_name, u64 name_len)
return chcore_syscall2(CHCORE_SYS_whole_restore, (u64)ckpt_name, name_len);
}
int usys_copy_time_traveling_data()
{
return chcore_syscall0(CHCORE_SYS_copy_time_traveling_data);
}
int usys_register_external_ringbuf(u64 buffer)
{
return chcore_syscall1(CHCORE_SYS_register_external_ringbuf, buffer);

1
user/sample-apps/apps/treesls/CMakeLists.txt
View File

@ -9,6 +9,7 @@ add_executable(checkpoint.bin checkpoint.c)
# test applications
add_executable(test_crash_hello.bin test_crash_hello.c)
add_executable(test_crash_counter.bin test_crash_counter.c)
add_executable(test_time_traveling.bin test_time_traveling.c)
# adjust config
add_executable(set_poll_loop_time.bin set_poll_loop_time.c)

24
user/sample-apps/apps/treesls/test_time_traveling.c Normal file
View File

@ -0,0 +1,24 @@
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <chcore/type.h>
#include <chcore/syscall.h>
int main(int argc, char *argv[], char *envp[])
{
int cnt = 0;
usys_set_affinity(-1, 5);
usys_yield();
while (1) {
printf("[%d] hello world\n", cnt);
usys_whole_ckpt(0, 0);
usys_whole_ckpt("test_crash_hello", 17);
printf("usys whole ckpt finished\n");
usys_copy_time_traveling_data();
printf("usys copy tt finished\n");
usys_whole_restore("test_crash_hello", 17);
cnt++;
sleep(1);
}
}