diff --git a/arch/x86/include/asm/vdso.h b/arch/x86/include/asm/vdso.h index 0e72befc9..56761fa3e 100644 --- a/arch/x86/include/asm/vdso.h +++ b/arch/x86/include/asm/vdso.h @@ -149,8 +149,8 @@ extern int vdso_init(void); extern int vdso_do_park(struct vdso_symtable *sym_rt, unsigned long park_at, unsigned long park_size); extern int vdso_fill_symtable(char *mem, size_t size, struct vdso_symtable *t); extern int vdso_proxify(char *who, struct vdso_symtable *sym_rt, - VmaEntry *vdso_vma, VmaEntry *vvar_vma, - unsigned long vdso_rt_parked_at); + unsigned long vdso_rt_parked_at, size_t index, + VmaEntry *vmas, size_t nr_vmas); extern int vdso_redirect_calls(void *base_to, void *base_from, struct vdso_symtable *to, struct vdso_symtable *from); extern int parasite_fixup_vdso(struct parasite_ctl *ctl, pid_t pid, diff --git a/arch/x86/vdso-pie.c b/arch/x86/vdso-pie.c index 05a78fb81..0a55d7173 100644 --- a/arch/x86/vdso-pie.c +++ b/arch/x86/vdso-pie.c @@ -291,13 +291,33 @@ int vdso_do_park(struct vdso_symtable *sym_rt, unsigned long park_at, unsigned l } int vdso_proxify(char *who, struct vdso_symtable *sym_rt, - VmaEntry *vdso_vma, VmaEntry *vvar_vma, - unsigned long vdso_rt_parked_at) + unsigned long vdso_rt_parked_at, size_t index, + VmaEntry *vmas, size_t nr_vmas) { + VmaEntry *vma_vdso = NULL, *vma_vvar = NULL; struct vdso_symtable s = VDSO_SYMTABLE_INIT; - size_t size = vma_entry_len(vdso_vma); bool remap_rt = false; + /* + * Figue out which kind of vdso tuple we get. + */ + if (vma_entry_is(&vmas[index], VMA_AREA_VDSO)) + vma_vdso = &vmas[index]; + else if (vma_entry_is(&vmas[index], VMA_AREA_VVAR)) + vma_vvar = &vmas[index]; + + if (index < (nr_vmas - 1)) { + if (vma_entry_is(&vmas[index + 1], VMA_AREA_VDSO)) + vma_vdso = &vmas[index + 1]; + else if (vma_entry_is(&vmas[index + 1], VMA_AREA_VVAR)) + vma_vvar = &vmas[index + 1]; + } + + if (!vma_vdso) { + pr_err("Can't find vDSO area in image\n"); + return -1; + } + /* * vDSO mark overwrites Elf program header of proxy vDSO thus * it must never ever be greater in size. @@ -305,35 +325,35 @@ int vdso_proxify(char *who, struct vdso_symtable *sym_rt, BUILD_BUG_ON(sizeof(struct vdso_mark) > sizeof(Elf64_Phdr)); /* - * Find symbols in dumpee vdso. + * Find symbols in vDSO zone read from image. */ - if (vdso_fill_symtable((void *)vdso_vma->start, size, &s)) + if (vdso_fill_symtable((void *)vma_vdso->start, vma_entry_len(vma_vdso), &s)) return -1; /* - * Try to figure out if the vDSO in image has the same symbols - * as run time vDSO, if yes we might try to reuse runtime vDSO - * instead of one in image. + * Proxification strategy * - * In case if VVAR area is present at least it must have same - * size as dumped one for inplace remap, also the order of zones - * must be matching. + * - There might be two vDSO zones: vdso code and optionally vvar data + * - To be able to use in-place remapping we need + * + * a) Size and order of vDSO zones are to match + * b) Symbols offsets must match + * c) Have same number of vDSO zones */ - if (size == vdso_vma_size(sym_rt)) { + if (vma_entry_len(vma_vdso) == vdso_vma_size(sym_rt)) { size_t i; for (i = 0; i < ARRAY_SIZE(s.symbols); i++) { if (s.symbols[i].offset != sym_rt->symbols[i].offset) break; - } - if (i == ARRAY_SIZE(s.symbols)) { - remap_rt = true; + } - if (vvar_vma && sym_rt->vvar_start != VVAR_BAD_ADDR) { - remap_rt = (vvar_vma_size(sym_rt) == vma_entry_len(vvar_vma)); + if (i == ARRAY_SIZE(s.symbols)) { + if (vma_vvar && sym_rt->vvar_start != VVAR_BAD_ADDR) { + remap_rt = (vvar_vma_size(sym_rt) == vma_entry_len(vma_vvar)); if (remap_rt) { long delta_rt = sym_rt->vvar_start - sym_rt->vma_start; - long delta_this = vvar_vma->start - vdso_vma->start; + long delta_this = vma_vvar->start - vma_vdso->start; remap_rt = (delta_rt ^ delta_this) < 0 ? false : true; } @@ -341,39 +361,49 @@ int vdso_proxify(char *who, struct vdso_symtable *sym_rt, } } + pr_debug("image [vdso] %lx-%lx [vvar] %lx-%lx\n", + vma_vdso->start, vma_vdso->end, + vma_vvar ? vma_vvar->start : VVAR_BAD_ADDR, + vma_vvar ? vma_vvar->end : VVAR_BAD_ADDR); + /* - * Easy case -- the vdso from image has same offsets and size + * Easy case -- the vdso from image has same offsets, order and size * as runtime, so we simply remap runtime vdso to dumpee position - * without generating any proxy. Note we may remap VVAR vdso as - * well which might not yet been mapped by a caller code. So - * drop VMA_AREA_REGULAR from it and caller would not touch it - * anymore. + * without generating any proxy. + * + * Note we may remap VVAR vdso as well which might not yet been mapped + * by a caller code. So drop VMA_AREA_REGULAR from it and caller would + * not touch it anymore. */ if (remap_rt) { - unsigned long vvar_rt_parked_at = VVAR_BAD_ADDR; int ret = 0; pr_info("Runtime vdso/vvar matches dumpee, remap inplace\n"); - if (sys_munmap((void *)vdso_vma->start, vma_entry_len(vdso_vma))) { + if (sys_munmap((void *)vma_vdso->start, vma_entry_len(vma_vdso))) { pr_err("Failed to unmap %s\n", who); return -1; } - if (vvar_vma) { - if (sys_munmap((void *)vvar_vma->start, vma_entry_len(vvar_vma))) { + if (vma_vvar) { + if (sys_munmap((void *)vma_vvar->start, vma_entry_len(vma_vvar))) { pr_err("Failed to unmap %s\n", who); return -1; } - - vvar_rt_parked_at = ALIGN(vvar_vma_size(sym_rt), PAGE_SIZE); - vvar_rt_parked_at+= vdso_rt_parked_at; - - ret = vdso_remap(who, vvar_rt_parked_at, vvar_vma->start, vma_entry_len(vvar_vma)); - vvar_vma->status &= ~VMA_AREA_REGULAR; } - ret |= vdso_remap(who, vdso_rt_parked_at, vdso_vma->start, vma_entry_len(vdso_vma)); + if (vma_vvar) { + if (vma_vdso->start < vma_vvar->start) { + ret = vdso_remap(who, vdso_rt_parked_at, vma_vdso->start, vdso_vma_size(sym_rt)); + vdso_rt_parked_at += vdso_vma_size(sym_rt); + ret |= vdso_remap(who, vdso_rt_parked_at, vma_vvar->start, vvar_vma_size(sym_rt)); + } else { + ret = vdso_remap(who, vdso_rt_parked_at, vma_vvar->start, vvar_vma_size(sym_rt)); + vdso_rt_parked_at += vvar_vma_size(sym_rt); + ret |= vdso_remap(who, vdso_rt_parked_at, vma_vdso->start, vdso_vma_size(sym_rt)); + } + } else + ret = vdso_remap(who, vdso_rt_parked_at, vma_vdso->start, vdso_vma_size(sym_rt)); return ret; } @@ -385,8 +415,15 @@ int vdso_proxify(char *who, struct vdso_symtable *sym_rt, */ pr_info("Runtime vdso mismatches dumpee, generate proxy\n"); + /* + * Don't forget to shift if vvar is before vdso. + */ + if (sym_rt->vvar_start != VDSO_BAD_ADDR && + sym_rt->vvar_start < sym_rt->vma_start) + vdso_rt_parked_at += vvar_vma_size(sym_rt); + if (vdso_redirect_calls((void *)vdso_rt_parked_at, - (void *)vdso_vma->start, + (void *)vma_vdso->start, sym_rt, &s)) { pr_err("Failed to proxify dumpee contents\n"); return -1; @@ -398,7 +435,7 @@ int vdso_proxify(char *who, struct vdso_symtable *sym_rt, * it's auto-generated every new session if proxy required. */ sys_mprotect((void *)vdso_rt_parked_at, vdso_vma_size(sym_rt), PROT_WRITE); - vdso_put_mark((void *)vdso_rt_parked_at, vdso_vma->start, vvar_vma ? vvar_vma->start : VVAR_BAD_ADDR); + vdso_put_mark((void *)vdso_rt_parked_at, vma_vdso->start, vma_vvar ? vma_vvar->start : VVAR_BAD_ADDR); sys_mprotect((void *)vdso_rt_parked_at, vdso_vma_size(sym_rt), VDSO_PROT); return 0; } diff --git a/include/vdso.h b/include/vdso.h index 52949561c..9fc174dbf 100644 --- a/include/vdso.h +++ b/include/vdso.h @@ -17,8 +17,8 @@ #define vdso_vma_size(t) (0) #define vdso_do_park(sym_rt, park_at, park_size) (0) #define vdso_remap(who, from, to, size) (0) -#define vdso_proxify(who, sym_rt, vdso_vma, \ - vvar_vma, vdso_rt_parked_at) (0) +#define vdso_proxify(who, sym_rt, vdso_rt_parked_at, \ + index, vmas, nr_vmas) (0) #endif /* CONFIG_VDSO */ diff --git a/pie/restorer.c b/pie/restorer.c index 7f8277670..1b05f59ed 100644 --- a/pie/restorer.c +++ b/pie/restorer.c @@ -729,18 +729,11 @@ long __export_restore_task(struct task_restore_args *args) vma_entry->start, vma_entry_len(vma_entry))) goto core_restore_end; #ifdef CONFIG_VDSO - if (vma_entry_is(vma_entry, VMA_AREA_VDSO)) { - VmaEntry *vma_vvar; - - if (i + 1 < args->nr_vmas) { - vma_vvar = args->tgt_vmas + i + 1; - if (!vma_entry_is(vma_entry, VMA_AREA_VVAR)) - vma_vvar = NULL; - } else - vma_vvar = NULL; + if (vma_entry_is(vma_entry, VMA_AREA_VDSO) || + vma_entry_is(vma_entry, VMA_AREA_VVAR)) { if (vdso_proxify("left dumpee", &args->vdso_sym_rt, - vma_entry, vma_vvar, - args->vdso_rt_parked_at)) + args->vdso_rt_parked_at, + i, args->tgt_vmas, args->nr_vmas)) goto core_restore_end; } #endif @@ -766,18 +759,11 @@ long __export_restore_task(struct task_restore_args *args) vma_entry->start, vma_entry_len(vma_entry))) goto core_restore_end; #ifdef CONFIG_VDSO - if (vma_entry_is(vma_entry, VMA_AREA_VDSO)) { - VmaEntry *vma_vvar; - - if (i + 1 < args->nr_vmas) { - vma_vvar = args->tgt_vmas + i + 1; - if (!vma_entry_is(vma_entry, VMA_AREA_VVAR)) - vma_vvar = NULL; - } else - vma_vvar = NULL; + if (vma_entry_is(vma_entry, VMA_AREA_VDSO) || + vma_entry_is(vma_entry, VMA_AREA_VVAR)) { if (vdso_proxify("right dumpee", &args->vdso_sym_rt, - vma_entry, vma_vvar, - args->vdso_rt_parked_at)) + args->vdso_rt_parked_at, + i, args->tgt_vmas, args->nr_vmas)) goto core_restore_end; } #endif