This is the mail archive of the
systemtap@sourceware.org
mailing list for the systemtap project.
[RFC PATCH]djprobe: new patch for the latest kprobes
- From: Masami Hiramatsu <hiramatu at sdl dot hitachi dot co dot jp>
- To: systemtap at sources dot redhat dot com, Satoshi Oshima <soshima at redhat dot com>, Hideo Aoki <haoki at redhat dot com>, Yumiko Sugita <sugita at sdl dot hitachi dot co dot jp>
- Date: Fri, 26 Aug 2005 12:44:30 +0900
- Subject: [RFC PATCH]djprobe: new patch for the latest kprobes
Hi,
Here is a new djprobe patch that includes some changes.
Changes:
- On UP kernel, djprobe changes code directly to jump code.
- Change kprobe's get/free_inst_slot and use it to allocate djprobe stub
code.
- Port sync_core() from x86-64 and use it instead of flush_icache_all().
- Remove CONFIG_DJPROBES and define ARCH_SUPPORTS_DJPROBES.(like kretprobes)
- register_djprobe and register_kprobe return -EBUSY when specified
address is already used by each other. (This is a temporary implementation.)
TODO:
- Support multiple djprobes insertion on an address.
- Support multiple kprobes and djprobes insertion on an address.
I think the best way to solve confliction of djprobes and kprobes is to
get together those probes as a kprobe. Because djprobes can not handle
any fault handlers.
This patch againsts 2.6.13-rc5-mm1.
I will post the djprobe patch to LKML after it is reviewed.
BTW, I marry tomorrow and leave in today for my honeymoon, and come back
5 Sep. So, I'm sorry that I will not reply until 5 Sep.
--
Masami HIRAMATSU
2nd Research Dept.
Hitachi, Ltd., Systems Development Laboratory
E-mail: hiramatu@sdl.hitachi.co.jp
diff -Narup linux-2.6.13-rc5-mm1/arch/i386/kernel/Makefile linux-2.6.13-rc5-mm1.djp/arch/i386/kernel/Makefile
--- linux-2.6.13-rc5-mm1/arch/i386/kernel/Makefile 2005-08-23 20:23:05.000000000 +0900
+++ linux-2.6.13-rc5-mm1.djp/arch/i386/kernel/Makefile 2005-08-26 09:41:35.000000000 +0900
@@ -28,7 +28,7 @@ obj-$(CONFIG_X86_REBOOTFIXUPS) += reboot
obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o crash.o
obj-$(CONFIG_X86_NUMAQ) += numaq.o
obj-$(CONFIG_X86_SUMMIT_NUMA) += summit.o
-obj-$(CONFIG_KPROBES) += kprobes.o
+obj-$(CONFIG_KPROBES) += kprobes.o stub_djprobe.o
obj-$(CONFIG_MODULES) += module.o
obj-y += sysenter.o vsyscall.o
obj-$(CONFIG_ACPI_SRAT) += srat.o
diff -Narup linux-2.6.13-rc5-mm1/arch/i386/kernel/kprobes.c linux-2.6.13-rc5-mm1.djp/arch/i386/kernel/kprobes.c
--- linux-2.6.13-rc5-mm1/arch/i386/kernel/kprobes.c 2005-08-23 20:23:05.000000000 +0900
+++ linux-2.6.13-rc5-mm1.djp/arch/i386/kernel/kprobes.c 2005-08-26 09:41:35.000000000 +0900
@@ -36,6 +36,7 @@
#include <asm/cacheflush.h>
#include <asm/kdebug.h>
#include <asm/desc.h>
+#include <asm/processor.h>
static struct kprobe *current_kprobe;
static unsigned long kprobe_status, kprobe_old_eflags, kprobe_saved_eflags;
@@ -542,3 +543,140 @@ int __init arch_init_kprobes(void)
{
return register_kprobe(&trampoline_p);
}
+
+/*
+ * DJProbe (Direct Jump Probe): Insertion and Recovery (SMP)
+ * Insertion:
+ * 1. Djprobes makes an instance and copies replacement instructions to it.
+ * 2. It registers the instance and the bypass handler to Kprobes at the
+ * specified address.
+ * 3. It schedules workqueues to check safety of the all cpus.
+ * 4. After the final worker finished checking all cpus, it replace int3 to
+ * relative jump instruction.
+ * Recovery:
+ * 1. Djprobe recovers a kprobe's int3 and original code except the first byte
+ * at the specified address.
+ * 2. It schedules workqueues to check safety of the all cpus.
+ * 3. After the final worker finished checking all cpus, it unregisters the
+ * instance from Kprobes and releases it.
+ */
+
+static void __local_sync_core(void * info)
+{
+ sync_core();
+}
+
+/* On pentium series, Unsynchronized cross-modifying code
+ operations can cause unexpected instruction execution results.
+ So after code modified, we should synchronize it on each processor.*/
+static inline void smp_sync_core(void)
+{
+ on_each_cpu(__local_sync_core, NULL, 1,1);
+}
+
+/* jmp code manipulators */
+struct __arch_jmp_op {
+ char op;
+ long raddr;
+} __attribute__((packed));
+/* insert jmp code */
+static inline void __set_jmp_op(void *from, void *to)
+{
+ struct __arch_jmp_op *jop;
+ jop = (struct __arch_jmp_op *)from;
+ jop->raddr=(long)(to) - ((long)(from) + 5);
+ smp_sync_core();
+ jop->op = RELATIVEJUMP_INSTRUCTION;
+}
+/* switch back to the kprobe */
+static inline void __set_breakpoint_op(void *dest, void *orig)
+{
+ struct __arch_jmp_op *jop = (struct __arch_jmp_op *)dest,
+ *jop2 = (struct __arch_jmp_op *)orig;
+
+ jop->op = BREAKPOINT_INSTRUCTION;
+ smp_sync_core();
+ jop->raddr = jop2->raddr;
+}
+/*
+ * djprobe call back function: called from stub code.
+ */
+static void asmlinkage djprobe_callback(struct djprobe_instance * djpi,
+ struct pt_regs *regs)
+{
+ /*TODO: use list*/
+ if (djpi->djp && djpi->djp->handler)
+ djpi->djp->handler(djpi->djp, regs);
+}
+
+/*
+ * Copy post processing instructions
+ * Target instructions MUST be relocatable.
+ */
+int __kprobes arch_prepare_djprobe_instance(struct djprobe_instance *djpi,
+ unsigned long size)
+{
+ kprobe_opcode_t *stub;
+ stub = djpi->stub.insn;
+
+ /* copy arch-dep-instance from template */
+ memcpy((void*)stub, (void*)&arch_tmpl_stub_entry, ARCH_STUB_SIZE);
+
+ /* set probe information */
+ *((long*)(stub + ARCH_STUB_VAL_IDX)) = (long)djpi;
+ /* set probe function */
+ *((long*)(stub + ARCH_STUB_CALL_IDX)) = (long)djprobe_callback;
+
+ /* copy instructions into the middle of axporbe instance */
+ memcpy((void*)(stub + ARCH_STUB_INST_IDX),
+ (void*)djpi->kp.addr, size);
+ djpi->stub.size = size;
+
+ /* set returning jmp instruction at the tail of axporbe instance*/
+ __set_jmp_op(stub + ARCH_STUB_END_IDX,
+ (void*)((long)djpi->kp.addr + size));
+
+ return 0;
+}
+
+/* Insert "jmp" instruction into the probing point. */
+void __kprobes arch_install_djprobe_instance(struct djprobe_instance *djpi)
+{
+ kprobe_opcode_t *stub;
+ stub = djpi->stub.insn;
+ __set_jmp_op((void*)djpi->kp.addr, (void*)stub);
+}
+/* Write back original instructions & kprobe */
+void __kprobes arch_uninstall_djprobe_instance(struct djprobe_instance *djpi)
+{
+ kprobe_opcode_t *stub;
+ stub = &djpi->stub.insn[ARCH_STUB_INST_IDX];
+ __set_breakpoint_op((void*)djpi->kp.addr, (void*)stub);
+}
+
+/*
+ * djprobe handler : switch to bypass code
+ */
+int __kprobes djprobe_bypass_handler(struct kprobe * kp, struct pt_regs * regs)
+{
+ struct djprobe_instance *djpi =
+ container_of(kp,struct djprobe_instance, kp);
+ kprobe_opcode_t *stub = djpi->stub.insn;
+
+ if (DJPI_EMPTY(djpi)) {
+ /* fixup dummy instruction */
+ kp->ainsn.insn[0] = djpi->stub.insn[ARCH_STUB_INST_IDX];
+ return 0;
+ } else {
+ regs->eip = (unsigned long)stub;
+ regs->eflags |= TF_MASK;
+ regs->eflags &= ~IF_MASK;
+ /*
+ * dummy return code :
+ * This code is to avoid to be changed eip value by
+ * resume_execute() of kprobes
+ */
+ kp->ainsn.insn[0] = RETURN_INSTRUCTION;
+ return 1; /* already prepared */
+ }
+}
diff -Narup linux-2.6.13-rc5-mm1/arch/i386/kernel/stub_djprobe.S linux-2.6.13-rc5-mm1.djp/arch/i386/kernel/stub_djprobe.S
--- linux-2.6.13-rc5-mm1/arch/i386/kernel/stub_djprobe.S 1970-01-01 09:00:00.000000000 +0900
+++ linux-2.6.13-rc5-mm1.djp/arch/i386/kernel/stub_djprobe.S 2005-08-26 09:41:35.000000000 +0900
@@ -0,0 +1,78 @@
+/*
+ * linux/arch/i386/stub_djprobe.S
+ *
+ * Copyright (C) HITACHI,LTD. 2005
+ * Created by Masami Hiramatsu <hiramatu@sdl.hitachi.co.jp>
+ */
+
+#include <linux/config.h>
+
+# jmp into this function from other functions.
+.global arch_tmpl_stub_entry
+arch_tmpl_stub_entry:
+ nop
+ subl $8, %esp #skip segment registers.
+ pushf
+ subl $20, %esp #skip segment registers.
+ pushl %eax
+ pushl %ebp
+ pushl %edi
+ pushl %esi
+ pushl %edx
+ pushl %ecx
+ pushl %ebx
+
+ movl %esp, %eax
+ pushl %eax
+ addl $60, %eax
+ movl %eax, 56(%esp)
+.global arch_tmpl_stub_val
+arch_tmpl_stub_val:
+ movl $0xffffffff, %eax
+ pushl %eax
+.global arch_tmpl_stub_call
+arch_tmpl_stub_call:
+ movl $0xffffffff, %eax
+ call *%eax
+ addl $8, %esp
+
+ popl %ebx
+ popl %ecx
+ popl %edx
+ popl %esi
+ popl %edi
+ popl %ebp
+ popl %eax
+ addl $20, %esp
+ popf
+ addl $8, %esp
+.global arch_tmpl_stub_inst
+arch_tmpl_stub_inst:
+ nop
+ nop
+ nop
+ nop
+ nop
+ nop
+ nop
+ nop
+ nop
+ nop
+ nop
+ nop
+ nop
+ nop
+ nop
+ nop
+ nop
+ nop
+ nop
+ nop
+
+.global arch_tmpl_stub_end
+arch_tmpl_stub_end:
+ nop
+ nop
+ nop
+ nop
+ nop
diff -Narup linux-2.6.13-rc5-mm1/include/asm-i386/kprobes.h linux-2.6.13-rc5-mm1.djp/include/asm-i386/kprobes.h
--- linux-2.6.13-rc5-mm1/include/asm-i386/kprobes.h 2005-08-23 20:22:36.000000000 +0900
+++ linux-2.6.13-rc5-mm1.djp/include/asm-i386/kprobes.h 2005-08-26 09:41:35.000000000 +0900
@@ -31,6 +31,8 @@ struct pt_regs;
typedef u8 kprobe_opcode_t;
#define BREAKPOINT_INSTRUCTION 0xcc
+#define RELATIVEJUMP_INSTRUCTION 0xe9
+#define RETURN_INSTRUCTION 0xc3
#define MAX_INSN_SIZE 16
#define MAX_STACK_SIZE 64
#define MIN_STACK_SIZE(ADDR) (((MAX_STACK_SIZE) < \
@@ -40,6 +42,29 @@ typedef u8 kprobe_opcode_t;
#define JPROBE_ENTRY(pentry) (kprobe_opcode_t *)pentry
#define ARCH_SUPPORTS_KRETPROBES
+#define ARCH_SUPPORTS_DJPROBES
+
+/* stub template code */
+extern long arch_tmpl_stub_entry;
+extern long arch_tmpl_stub_val;
+extern long arch_tmpl_stub_call;
+extern long arch_tmpl_stub_inst;
+extern long arch_tmpl_stub_end;
+
+#define ARCH_STUB_VAL_IDX ((long)&arch_tmpl_stub_val - (long)&arch_tmpl_stub_entry + 1)
+#define ARCH_STUB_CALL_IDX ((long)&arch_tmpl_stub_call - (long)&arch_tmpl_stub_entry + 1)
+#define ARCH_STUB_INST_IDX ((long)&arch_tmpl_stub_inst - (long)&arch_tmpl_stub_entry)
+#define ARCH_STUB_END_IDX ((long)&arch_tmpl_stub_end - (long)&arch_tmpl_stub_entry)
+#define ARCH_STUB_SIZE ((long)&arch_tmpl_stub_end - (long)&arch_tmpl_stub_entry + 5)
+
+#define ARCH_STUB_INSN_SIZE 80
+#define ARCH_STUB_INSN_MAX 20
+#define ARCH_STUB_INSN_MIN 5
+
+struct arch_djprobe_stub {
+ kprobe_opcode_t *insn;
+ int size;
+};
void kretprobe_trampoline(void);
diff -Narup linux-2.6.13-rc5-mm1/include/asm-i386/processor.h linux-2.6.13-rc5-mm1.djp/include/asm-i386/processor.h
--- linux-2.6.13-rc5-mm1/include/asm-i386/processor.h 2005-08-23 20:23:10.000000000 +0900
+++ linux-2.6.13-rc5-mm1.djp/include/asm-i386/processor.h 2005-08-26 09:41:35.000000000 +0900
@@ -433,6 +433,12 @@ unsigned long get_wchan(struct task_stru
#define KSTK_EIP(task) (task_pt_regs(task)->eip)
#define KSTK_ESP(task) (task_pt_regs(task)->esp)
+/* Serializing instruction for self-modifying code */
+extern inline void sync_core(void)
+{
+ int tmp;
+ asm volatile("cpuid" : "=a" (tmp) : "0" (1) : "ebx","ecx","edx","memory");
+}
struct microcode_header {
unsigned int hdrver;
diff -Narup linux-2.6.13-rc5-mm1/include/linux/kprobes.h linux-2.6.13-rc5-mm1.djp/include/linux/kprobes.h
--- linux-2.6.13-rc5-mm1/include/linux/kprobes.h 2005-08-23 20:23:10.000000000 +0900
+++ linux-2.6.13-rc5-mm1.djp/include/linux/kprobes.h 2005-08-26 09:41:35.000000000 +0900
@@ -28,6 +28,8 @@
* 2005-May Hien Nguyen <hien@us.ibm.com> and Jim Keniston
* <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
* <prasanna@in.ibm.com> added function-return probes.
+ * 2005-Aug Masami HIRAMATSU <hiramatu@sdl.hitachi.co.jp> added direct
+ * jump probe (djprobe) interface to reduce overhead.
*/
#include <linux/config.h>
#include <linux/list.h>
@@ -141,6 +143,40 @@ struct kretprobe_instance {
struct task_struct *task;
};
+/* djprobe's instance (internal use)*/
+struct djprobe_instance {
+ struct djprobe *djp;
+ struct arch_djprobe_stub stub;
+
+ struct kprobe kp;
+ struct list_head list; /* list of djprobe_instances */
+ cpumask_t checked_cpus;
+};
+#define DJPI_EMPTY(djpi) (djpi->djp==NULL)
+#define DJPI_CHECKED(djpi) (cpus_equal(djpi->checked_cpus, cpu_online_map))
+
+struct djprobe;
+typedef void (*djprobe_handler_t)(struct djprobe *, struct pt_regs *);
+/*
+ * Direct Jump probe interface structure
+ */
+struct djprobe {
+#ifndef ARCH_SUPPORTS_DJPROBES
+ struct kprobe kp;
+#endif /* ARCH_SUPPORTS_DJPROBES */
+ /* location of the probe point */
+ void * addr;
+
+ /* sum of length of the replacing codes */
+ int size;
+
+ /* probing handler (pre-executed) */
+ djprobe_handler_t handler;
+
+ /* pointer for instance */
+ struct djprobe_instance * inst;
+};
+
#ifdef CONFIG_KPROBES
/* Locks kprobe: irq must be disabled */
void lock_kprobes(void);
@@ -182,6 +218,18 @@ struct kretprobe_instance *get_free_rp_i
void add_rp_inst(struct kretprobe_instance *ri);
void kprobe_flush_task(struct task_struct *tk);
void recycle_rp_inst(struct kretprobe_instance *ri);
+
+#ifdef ARCH_SUPPORTS_DJPROBES
+extern int arch_prepare_djprobe_instance(struct djprobe_instance *djpi,
+ unsigned long size);
+extern int djprobe_bypass_handler(struct kprobe * kp, struct pt_regs * regs);
+extern void arch_install_djprobe_instance(struct djprobe_instance *djpi);
+extern void arch_uninstall_djprobe_instance(struct djprobe_instance *djpi);
+#endif /* ARCH_SUPPORTS_DJPROBES */
+
+int register_djprobe(struct djprobe *p);
+void unregister_djprobe(struct djprobe *p);
+
#else /* CONFIG_KPROBES */
static inline int kprobe_running(void)
{
@@ -214,5 +262,12 @@ static inline void unregister_kretprobe(
static inline void kprobe_flush_task(struct task_struct *tk)
{
}
+static inline int register_djprobe(struct djprobe *p)
+{
+ return -ENOSYS;
+}
+static inline void unregister_djprobe(struct djprobe *p)
+{
+}
#endif /* CONFIG_KPROBES */
#endif /* _LINUX_KPROBES_H */
diff -Narup linux-2.6.13-rc5-mm1/kernel/kprobes.c linux-2.6.13-rc5-mm1.djp/kernel/kprobes.c
--- linux-2.6.13-rc5-mm1/kernel/kprobes.c 2005-08-23 20:23:11.000000000 +0900
+++ linux-2.6.13-rc5-mm1.djp/kernel/kprobes.c 2005-08-26 09:42:16.000000000 +0900
@@ -30,6 +30,8 @@
* 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston
* <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
* <prasanna@in.ibm.com> added function-return probes.
+ * 2005-Aug Masami HIRAMATSU <hiramatu@sdl.hitachi.co.jp> added direct
+ * jump probe (djprobe) interface to reduce overhead.
*/
#include <linux/kprobes.h>
#include <linux/spinlock.h>
@@ -58,44 +60,53 @@ static struct kprobe *curr_kprobe;
* stepping on the instruction on a vmalloced/kmalloced/data page
* is a recipe for disaster
*/
-#define INSNS_PER_PAGE (PAGE_SIZE/(MAX_INSN_SIZE * sizeof(kprobe_opcode_t)))
+#define INSNS_PER_PAGE(size) (PAGE_SIZE/(size * sizeof(kprobe_opcode_t)))
struct kprobe_insn_page {
struct hlist_node hlist;
kprobe_opcode_t *insns; /* Page of instruction slots */
- char slot_used[INSNS_PER_PAGE];
int nused;
+ char slot_used[1];
};
-static struct hlist_head kprobe_insn_pages;
+struct kprobe_insn_page_list {
+ struct hlist_head list;
+ int insn_size; /* Size of an instruction slot */
+};
+
+static struct kprobe_insn_page_list kprobe_insn_pages = {
+ HLIST_HEAD_INIT, MAX_INSN_SIZE};
/**
- * get_insn_slot() - Find a slot on an executable page for an instruction.
+ * __get_insn_slot() - Find a slot on an executable page for an instruction.
* We allocate an executable page if there's no room on existing ones.
*/
-kprobe_opcode_t __kprobes *get_insn_slot(void)
+static kprobe_opcode_t
+__kprobes *__get_insn_slot(struct kprobe_insn_page_list *pages)
{
struct kprobe_insn_page *kip;
struct hlist_node *pos;
+ int ninsns = INSNS_PER_PAGE(pages->insn_size);
- hlist_for_each(pos, &kprobe_insn_pages) {
+ hlist_for_each(pos, &pages->list) {
kip = hlist_entry(pos, struct kprobe_insn_page, hlist);
- if (kip->nused < INSNS_PER_PAGE) {
+ if (kip->nused < ninsns ) {
int i;
- for (i = 0; i < INSNS_PER_PAGE; i++) {
+ for (i = 0; i < ninsns; i++) {
if (!kip->slot_used[i]) {
kip->slot_used[i] = 1;
kip->nused++;
- return kip->insns + (i * MAX_INSN_SIZE);
+ return kip->insns + (i * pages->insn_size);
}
}
/* Surprise! No unused slots. Fix kip->nused. */
- kip->nused = INSNS_PER_PAGE;
+ kip->nused = ninsns;
}
}
/* All out of space. Need to allocate a new page. Use slot 0.*/
- kip = kmalloc(sizeof(struct kprobe_insn_page), GFP_KERNEL);
+ kip = kmalloc(sizeof(struct kprobe_insn_page)+sizeof(char)*(ninsns-1),
+ GFP_KERNEL);
if (!kip) {
return NULL;
}
@@ -111,23 +122,25 @@ kprobe_opcode_t __kprobes *get_insn_slot
return NULL;
}
INIT_HLIST_NODE(&kip->hlist);
- hlist_add_head(&kip->hlist, &kprobe_insn_pages);
- memset(kip->slot_used, 0, INSNS_PER_PAGE);
+ hlist_add_head(&kip->hlist, &pages->list);
+ memset(kip->slot_used, 0, ninsns);
kip->slot_used[0] = 1;
kip->nused = 1;
return kip->insns;
}
-void __kprobes free_insn_slot(kprobe_opcode_t *slot)
+static void __kprobes __free_insn_slot(struct kprobe_insn_page_list *pages,
+ kprobe_opcode_t *slot)
{
struct kprobe_insn_page *kip;
struct hlist_node *pos;
+ int ninsns = INSNS_PER_PAGE(pages->insn_size);
- hlist_for_each(pos, &kprobe_insn_pages) {
+ hlist_for_each(pos, &pages->list) {
kip = hlist_entry(pos, struct kprobe_insn_page, hlist);
if (kip->insns <= slot &&
- slot < kip->insns + (INSNS_PER_PAGE * MAX_INSN_SIZE)) {
- int i = (slot - kip->insns) / MAX_INSN_SIZE;
+ slot < kip->insns + (ninsns * pages->insn_size)) {
+ int i = (slot - kip->insns) / pages->insn_size;
kip->slot_used[i] = 0;
kip->nused--;
if (kip->nused == 0) {
@@ -138,10 +151,10 @@ void __kprobes free_insn_slot(kprobe_opc
* next time somebody inserts a probe.
*/
hlist_del(&kip->hlist);
- if (hlist_empty(&kprobe_insn_pages)) {
+ if (hlist_empty(&pages->list)) {
INIT_HLIST_NODE(&kip->hlist);
hlist_add_head(&kip->hlist,
- &kprobe_insn_pages);
+ &pages->list);
} else {
module_free(NULL, kip->insns);
kfree(kip);
@@ -152,6 +165,15 @@ void __kprobes free_insn_slot(kprobe_opc
}
}
+kprobe_opcode_t __kprobes *get_insn_slot(void)
+{
+ return __get_insn_slot(&kprobe_insn_pages);
+}
+void __kprobes free_insn_slot(kprobe_opcode_t *slot)
+{
+ __free_insn_slot(&kprobe_insn_pages, slot);
+}
+
/* Locks kprobe: irqs must be disabled */
void __kprobes lock_kprobes(void)
{
@@ -461,6 +483,12 @@ int __kprobes register_kprobe(struct kpr
old_p = get_kprobe(p->addr);
p->nmissed = 0;
if (old_p) {
+#ifdef ARCH_SUPPORTS_DJPROBES
+ if (old_p->pre_handler == djprobe_bypass_handler) {
+ ret = -EBUSY;
+ goto out;
+ }
+#endif
ret = register_aggr_kprobe(old_p, p);
goto out;
}
@@ -577,6 +605,200 @@ void __kprobes unregister_kretprobe(stru
spin_unlock_irqrestore(&kprobe_lock, flags);
}
+#ifdef ARCH_SUPPORTS_DJPROBES
+/*
+ * The djprobe do not refer instances list when probe function called.
+ * This list is operated on registering and unregistering djprobe.
+ */
+static LIST_HEAD(djprobe_list);
+static DEFINE_SPINLOCK(djprobe_lock);
+/* Instruction pages for djprobe's stub code */
+static struct kprobe_insn_page_list djprobe_insn_pages = {
+ HLIST_HEAD_INIT, ARCH_STUB_INSN_SIZE};
+
+static inline void free_djprobe_instance_nl(struct djprobe_instance *djpi)
+{
+ list_del(&djpi->list);
+ if (djpi->kp.addr) unregister_kprobe(&(djpi->kp));
+ __free_insn_slot(&djprobe_insn_pages, djpi->stub.insn);
+ kfree(djpi);
+}
+
+#ifdef CONFIG_SMP
+static void __kprobes work_check_djprobe_instances(void *data);
+static DEFINE_PER_CPU(struct work_struct, djprobe_check_works);
+
+static inline void init_djprobe_check_works(void)
+{
+ int cpu;
+ struct work_struct *wk;
+ for_each_online_cpu(cpu) {
+ wk = &per_cpu(djprobe_check_works, cpu);
+ INIT_WORK(wk, work_check_djprobe_instances, NULL);
+ }
+}
+
+static inline void schedule_check_djprobe_instances(void)
+{
+ int cpu;
+ struct work_struct *wk;
+ for_each_online_cpu(cpu) {
+ wk = &per_cpu(djprobe_check_works, cpu);
+ schedule_delayed_work_on(cpu, wk, 0);
+ }
+}
+
+static void __kprobes work_check_djprobe_instances(void *data)
+{
+ struct list_head *pos;
+ struct djprobe_instance *djpi;
+
+ spin_lock(&djprobe_lock);
+ list_for_each(pos, &djprobe_list) {
+ djpi = container_of(pos, struct djprobe_instance, list);
+ if (!DJPI_CHECKED(djpi)) {
+ cpu_set(smp_processor_id(), djpi->checked_cpus);
+ if (DJPI_CHECKED(djpi)) {
+ if (DJPI_EMPTY(djpi)) {
+ pos = pos->prev; /* pos is going to be freed */
+ free_djprobe_instance_nl(djpi);
+ } else {
+ arch_install_djprobe_instance(djpi);
+ }
+ }
+ }
+ }
+ spin_unlock(&djprobe_lock);
+}
+#endif /*CONFIG_SMP*/
+
+/* Use kprobe to check safety and install */
+static int __kprobes install_djprobe_instance(struct djprobe_instance *djpi)
+{
+ int ret;
+ djpi->kp.pre_handler = djprobe_bypass_handler;
+ ret = register_kprobe(&(djpi->kp));
+ if (ret == 0) {
+#ifdef CONFIG_SMP
+ cpus_clear(djpi->checked_cpus);
+ schedule_check_djprobe_instances();
+#else /* CONFIG_SMP */
+ arch_install_djprobe_instance(djpi);
+#endif /* CONFIG_SMP */
+ }
+ return ret;
+}
+/* Use kprobe to check safety and release */
+static void __kprobes uninstall_djprobe_instance(struct djprobe_instance *djpi)
+{
+ arch_uninstall_djprobe_instance(djpi);
+#ifdef CONFIG_SMP
+ cpus_clear(djpi->checked_cpus);
+ schedule_check_djprobe_instances();
+#else /* CONFIG_SMP */
+ free_djprobe_instance_nl(djpi);
+#endif /* CONFIG_SMP */
+}
+
+int __kprobes register_djprobe(struct djprobe * djp)
+{
+ struct djprobe_instance *djpi;
+ struct kprobe *kp;
+ int ret = 0;
+
+ if (djp == NULL || djp->addr == NULL ||
+ djp->size > ARCH_STUB_INSN_MAX ||
+ djp->size < ARCH_STUB_INSN_MIN ||
+ djp->inst != NULL)
+ return -EINVAL;
+
+ if ((ret = in_kprobes_functions((unsigned long) djp->addr)) != 0)
+ return ret;
+
+ spin_lock(&djprobe_lock);
+
+ kp = get_kprobe(djp->addr);
+ if (kp != NULL) {
+ if (kp->pre_handler == djprobe_bypass_handler) {
+ djpi = container_of(kp,struct djprobe_instance, kp);
+ if (DJPI_EMPTY(djpi)) {
+ djp->inst = djpi;
+ djpi->djp = djp; /*TODO: use list*/
+ cpus_clear(djpi->checked_cpus);
+ goto out;
+ }
+ }
+ ret = -EBUSY; /* already used or kprobes were inserted */
+ goto out;
+ }
+ /* make a new instance */
+ djpi = kmalloc(sizeof(struct djprobe_instance),GFP_KERNEL);
+ if (djpi == NULL) {
+ ret = -ENOMEM; /* memory allocation error */
+ goto out;
+ }
+
+ /* initialize */
+ memset(djpi, 0, sizeof(struct djprobe_instance));
+ djpi->stub.insn = __get_insn_slot(&djprobe_insn_pages);
+ if (djpi->stub.insn == NULL) {
+ kfree(djpi);
+ ret = -ENOMEM; /* memory allocation error */
+ goto out;
+ }
+ INIT_LIST_HEAD(&djpi->list);
+ djpi->kp.addr = djp->addr;
+ arch_prepare_djprobe_instance(djpi, djp->size); /*TODO : remove size*/
+ list_add(&djpi->list, &djprobe_list);
+
+ /* attach */
+ djp->inst = djpi;
+ djpi->djp = djp; /*TODO: use list*/
+
+ ret = install_djprobe_instance(djpi);
+ if (ret < 0) { /* failed to install */
+ djp->inst = NULL;
+ djpi->kp.addr = NULL;
+ free_djprobe_instance_nl(djpi);
+ }
+out:
+ spin_unlock(&djprobe_lock);
+ return ret;
+}
+
+void __kprobes unregister_djprobe(struct djprobe * djp)
+{
+ struct djprobe_instance *djpi;
+ if (djp == NULL || djp->inst == NULL)
+ return ;
+
+ djpi = djp->inst;
+ spin_lock(&djprobe_lock);
+ djp->inst = NULL;
+ djpi->djp = NULL; /*TODO: use list*/
+ if (DJPI_EMPTY(djpi)) {
+ uninstall_djprobe_instance(djpi);
+ }
+ spin_unlock(&djprobe_lock);
+}
+
+#else /* ARCH_SUPPORTS_DJPROBES */
+int __kprobes register_djprobe(struct djprobe *p)
+{
+ if (p!=NULL) {
+ p->kp.addr = p->addr;
+ p->kp.pre_handler = (kprobe_pre_handler_t)p->handler;
+ return register_kprobe(&p->kp);
+ }
+ return -EINVAL;
+}
+
+void __kprobes unregister_djprobe(struct djprobe *p)
+{
+ unregister_kprobe(&p->kp);
+}
+#endif /* ARCH_SUPPORTS_DJPROBES */
+
static int __init init_kprobes(void)
{
int i, err = 0;
@@ -588,6 +810,9 @@ static int __init init_kprobes(void)
INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
}
+#if defined(ARCH_SUPPORTS_DJPROBE) && defined(CONFIG_SMP)
+ init_djprobe_check_works();
+#endif
err = arch_init_kprobes();
if (!err)
err = register_die_notifier(&kprobe_exceptions_nb);
@@ -604,4 +829,6 @@ EXPORT_SYMBOL_GPL(unregister_jprobe);
EXPORT_SYMBOL_GPL(jprobe_return);
EXPORT_SYMBOL_GPL(register_kretprobe);
EXPORT_SYMBOL_GPL(unregister_kretprobe);
+EXPORT_SYMBOL_GPL(register_djprobe);
+EXPORT_SYMBOL_GPL(unregister_djprobe);