From cca0f7b4927f295e63bc554030a04fc5b145242a Mon Sep 17 00:00:00 2001
From: Sascha Wildner <saw@online.de>
Date: Thu, 3 Nov 2011 21:12:29 +0100
Subject: [PATCH 15/18] hwpmc: Add hwpmc hooks.

In-collaboration-with: aggelos
---
 sys/kern/kern_clock.c                |    9 +++++-
 sys/kern/kern_exec.c                 |   25 +++++++++++++++++
 sys/kern/kern_exit.c                 |   13 ++++++++-
 sys/kern/kern_linker.c               |   24 ++++++++++++++++
 sys/kern/lwkt_thread.c               |   20 ++++++++++++++
 sys/platform/pc32/i386/exception.s   |    7 ++++-
 sys/platform/pc32/i386/genassym.c    |    9 +++++-
 sys/platform/pc32/i386/trap.c        |   18 ++++++++++++-
 sys/platform/pc64/x86_64/exception.S |    6 ++++
 sys/platform/pc64/x86_64/genassym.c  |    8 +++++
 sys/platform/pc64/x86_64/trap.c      |   16 +++++++++++
 sys/vm/vm_mmap.c                     |   49 ++++++++++++++++++++++++++++++++++
 12 files changed, 199 insertions(+), 5 deletions(-)

diff --git a/sys/kern/kern_clock.c b/sys/kern/kern_clock.c
index 60408ce..67b0e7e 100644
--- a/sys/kern/kern_clock.c
+++ b/sys/kern/kern_clock.c
@@ -70,7 +70,6 @@
  *
  *	@(#)kern_clock.c	8.5 (Berkeley) 1/21/94
  * $FreeBSD: src/sys/kern/kern_clock.c,v 1.105.2.10 2002/10/17 13:19:40 maxim Exp $
- * $DragonFly: src/sys/kern/kern_clock.c,v 1.62 2008/09/09 04:06:13 dillon Exp $
  */
 
 #include "opt_ntp.h"
@@ -109,6 +108,10 @@
 #include <sys/gmon.h>
 #endif
 
+#ifdef HWPMC_HOOKS
+#include <sys/pmckern.h>
+#endif
+
 #ifdef DEVICE_POLLING
 extern void init_device_poll_pcpu(int);
 #endif
@@ -701,6 +704,10 @@ statclock(systimer_t info, int in_ipi, struct intrframe *frame)
 			}
 		}
 	}
+#ifdef	HWPMC_HOOKS
+	if (PMC_CPU_HAS_SAMPLES(mycpuid))
+		PMC_CALL_HOOK_UNLOCKED(curthread, PMC_FN_DO_SAMPLES, NULL);
+#endif
 }
 
 #ifdef DEBUG_PCTRACK
diff --git a/sys/kern/kern_exec.c b/sys/kern/kern_exec.c
index dc7047e..7c539bf 100644
--- a/sys/kern/kern_exec.c
+++ b/sys/kern/kern_exec.c
@@ -74,6 +74,10 @@
 #include <sys/thread2.h>
 #include <sys/mplock2.h>
 
+#ifdef	HWPMC_HOOKS
+#include <sys/pmckern.h>
+#endif
+
 MALLOC_DEFINE(M_PARGS, "proc-args", "Process arguments");
 MALLOC_DEFINE(M_EXECARGS, "exec-args", "Exec arguments");
 
@@ -202,6 +206,10 @@ kern_execve(struct nlookupdata *nd, struct image_args *args)
 	struct image_params image_params, *imgp;
 	struct vattr attr;
 	int (*img_first) (struct image_params *);
+	int credential_changing;
+#ifdef HWPMC_HOOKS
+	struct pmckern_procexec pe;
+#endif
 
 	if (debug_execve_args) {
 		kprintf("%s()\n", __func__);
@@ -423,6 +431,7 @@ interpret:
 		wakeup((caddr_t)p->p_pptr);
 	}
 
+	credential_changing = 0;
 	/*
 	 * Implement image setuid/setgid.
 	 *
@@ -438,6 +447,7 @@ interpret:
 		 * root.  Record any set-id flags first to make sure that
 		 * we do not regain any tracing during a possible block.
 		 */
+		credential_changing = 1;
 		setsugid();
 		if (p->p_tracenode && ktrace_suid == 0 &&
 		    priv_check(td, PRIV_ROOT) != 0) {
@@ -515,6 +525,21 @@ interpret:
 	/* clear "fork but no exec" flag, as we _are_ execing */
 	p->p_acflag &= ~AFORK;
 
+#ifdef	HWPMC_HOOKS
+	/*
+	 * Check if system-wide sampling is in effect or if the
+	 * current process is using PMCs.  If so, do exec() time
+	 * processing.  This processing needs to happen AFTER the
+	 * P_INEXEC flag is cleared.
+	 */
+	if (PMC_SYSTEM_SAMPLING_ACTIVE() || PMC_PROC_IS_USING_PMCS(p)) {
+		pe.pm_credentialschanged = credential_changing;
+		pe.pm_entryaddr = imgp->entry_addr;
+
+		PMC_CALL_HOOK_X(td, PMC_FN_PROCESS_EXEC, (void *) &pe);
+	}
+#endif
+
 	/* Set values passed into the program in registers. */
 	exec_setregs(imgp->entry_addr, (u_long)(uintptr_t)stack_base,
 		     imgp->ps_strings);
diff --git a/sys/kern/kern_exit.c b/sys/kern/kern_exit.c
index f3ffb2b..c3dd688 100644
--- a/sys/kern/kern_exit.c
+++ b/sys/kern/kern_exit.c
@@ -37,7 +37,6 @@
  *
  *	@(#)kern_exit.c	8.7 (Berkeley) 2/12/94
  * $FreeBSD: src/sys/kern/kern_exit.c,v 1.92.2.11 2003/01/13 22:51:16 dillon Exp $
- * $DragonFly: src/sys/kern/kern_exit.c,v 1.91 2008/05/18 20:02:02 nth Exp $
  */
 
 #include "opt_compat.h"
@@ -83,6 +82,10 @@
 #include <sys/sysref2.h>
 #include <sys/mplock2.h>
 
+#ifdef	HWPMC_HOOKS
+#include <sys/pmckern.h>
+#endif
+
 static void reaplwps(void *context, int dummy);
 static void reaplwp(struct lwp *lp);
 static void killlwps(struct lwp *lp);
@@ -654,6 +657,14 @@ lwp_exit(int masterexit)
 	dsched_exit_thread(td);
 	biosched_done(curthread);
 
+#ifdef	HWPMC_HOOKS
+	/*
+	 * If this thread is part of a process that is being tracked by hwpmc(4),
+	 * inform the module of the thread's impending exit.
+	 */
+	if (PMC_PROC_IS_USING_PMCS(td->td_proc))
+		PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT);
+#endif
 	/*
 	 * We have to use the reaper for all the LWPs except the one doing
 	 * the master exit.  The LWP doing the master exit can just be
diff --git a/sys/kern/kern_linker.c b/sys/kern/kern_linker.c
index 79bc231..344bcb1 100644
--- a/sys/kern/kern_linker.c
+++ b/sys/kern/kern_linker.c
@@ -50,6 +50,10 @@
 
 #include <sys/mplock2.h>
 
+#ifdef HWPMC_HOOKS
+#include <sys/pmckern.h>
+#endif
+
 #ifdef _KERNEL_VIRTUAL
 #include <dlfcn.h>
 #endif
@@ -763,6 +767,9 @@ linker_ddb_symbol_values(c_linker_sym_t sym, linker_symval_t *symval)
 int
 sys_kldload(struct kldload_args *uap)
 {
+#ifdef HWPMC_HOOKS
+    struct pmckern_map_in pkm;
+#endif
     struct thread *td = curthread;
     char *file;
     char *kldname, *modname;
@@ -800,6 +807,11 @@ sys_kldload(struct kldload_args *uap)
     if (error)
 	goto out;
 
+#ifdef HWPMC_HOOKS
+    pkm.pm_file = lf->filename;
+    pkm.pm_address = (uintptr_t) lf->address;
+    PMC_CALL_HOOK(td, PMC_FN_KLD_LOAD, (void *) &pkm);
+#endif
     lf->userrefs++;
     uap->sysmsg_result = lf->id;
 
@@ -815,6 +827,9 @@ out:
 int
 sys_kldunload(struct kldunload_args *uap)
 {
+#ifdef HWPMC_HOOKS
+	struct pmckern_map_out pkm;
+#endif
     struct thread *td = curthread;
     linker_file_t lf;
     int error = 0;
@@ -834,6 +849,11 @@ sys_kldunload(struct kldunload_args *uap)
 	    error = EBUSY;
 	    goto out;
 	}
+#ifdef HWPMC_HOOKS
+	/* Save data needed by hwpmc(4) before unloading. */
+	pkm.pm_address = (uintptr_t) lf->address;
+	pkm.pm_size = lf->size;
+#endif
 	lf->userrefs--;
 	error = linker_file_unload(lf);
 	if (error)
@@ -841,6 +861,10 @@ sys_kldunload(struct kldunload_args *uap)
     } else {
 	error = ENOENT;
     }
+#ifdef HWPMC_HOOKS
+    if (error == 0)
+	PMC_CALL_HOOK(td, PMC_FN_KLD_UNLOAD, (void *) &pkm);
+#endif
 out:
     rel_mplock();
     return error;
diff --git a/sys/kern/lwkt_thread.c b/sys/kern/lwkt_thread.c
index 389d1f0..71e3f94 100644
--- a/sys/kern/lwkt_thread.c
+++ b/sys/kern/lwkt_thread.c
@@ -72,6 +72,10 @@
 #include <machine/stdarg.h>
 #include <machine/smp.h>
 
+#ifdef HWPMC_HOOKS
+#include <sys/pmckern.h>
+#endif
+
 #if !defined(KTR_CTXSW)
 #define KTR_CTXSW KTR_ALL
 #endif
@@ -932,8 +936,16 @@ haveidle:
 	++switch_count;
 	KTR_LOG(ctxsw_sw, gd->gd_cpuid, ntd);
 	ntd->td_flags |= TDF_RUNNING;
+#ifdef	HWPMC_HOOKS
+	if (td->td_proc && PMC_PROC_IS_USING_PMCS(td->td_proc))
+	    PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT);
+#endif
 	lwkt_switch_return(td->td_switch(ntd));
 	/* ntd invalid, td_switch() can return a different thread_t */
+#ifdef	HWPMC_HOOKS
+	if (td->td_proc && PMC_PROC_IS_USING_PMCS(td->td_proc))
+	    PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_IN);
+#endif
     }
 
     /*
@@ -1092,7 +1104,15 @@ lwkt_preempt(thread_t ntd, int critcount)
 
     KKASSERT((ntd->td_flags & TDF_RUNNING) == 0);
     ntd->td_flags |= TDF_RUNNING;
+#ifdef	HWPMC_HOOKS
+	if (td->td_proc && PMC_PROC_IS_USING_PMCS(td->td_proc))
+	    PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT);
+#endif
     xtd = td->td_switch(ntd);
+#ifdef	HWPMC_HOOKS
+	if (td->td_proc && PMC_PROC_IS_USING_PMCS(td->td_proc))
+	    PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_IN);
+#endif
     KKASSERT(xtd == ntd);
     lwkt_switch_return(xtd);
     gd->gd_intr_nesting_level = save_gd_intr_nesting_level;
diff --git a/sys/platform/pc32/i386/exception.s b/sys/platform/pc32/i386/exception.s
index b0039c8..a78d2ae 100644
--- a/sys/platform/pc32/i386/exception.s
+++ b/sys/platform/pc32/i386/exception.s
@@ -31,7 +31,6 @@
  * SUCH DAMAGE.
  *
  * $FreeBSD: src/sys/i386/i386/exception.s,v 1.65.2.3 2001/08/15 01:23:49 peter Exp $
- * $DragonFly: src/sys/platform/pc32/i386/exception.s,v 1.32 2008/05/08 01:21:04 dillon Exp $
  */
 
 #include "use_npx.h"
@@ -45,6 +44,9 @@
 #include "assym.s"
 
 	.text
+#ifdef HWPMC_HOOKS
+	ENTRY(start_exceptions)
+#endif
 
 	.globl	lwkt_switch_return
 
@@ -943,3 +945,6 @@ ENTRY(fork_trampoline)
  */
 #include "vm86bios.s"
 
+#ifdef HWPMC_HOOKS
+	ENTRY(end_exceptions)
+#endif
diff --git a/sys/platform/pc32/i386/genassym.c b/sys/platform/pc32/i386/genassym.c
index 84e9f15..77f955e 100644
--- a/sys/platform/pc32/i386/genassym.c
+++ b/sys/platform/pc32/i386/genassym.c
@@ -35,7 +35,6 @@
  *
  *	from: @(#)genassym.c	5.11 (Berkeley) 5/10/91
  * $FreeBSD: src/sys/i386/i386/genassym.c,v 1.86.2.3 2002/03/03 05:42:49 nyan Exp $
- * $DragonFly: src/sys/platform/pc32/i386/genassym.c,v 1.58 2008/05/09 06:35:11 dillon Exp $
  */
 
 #include <sys/param.h>
@@ -73,6 +72,10 @@
 #include <machine/globaldata.h>
 #include <machine/pmap.h>
 
+#ifdef	HWPMC_HOOKS
+#include <sys/pmckern.h>
+#endif
+
 ASSYM(P_VMSPACE, offsetof(struct proc, p_vmspace));
 ASSYM(VM_PMAP, offsetof(struct vmspace, vm_pmap));
 ASSYM(PM_ACTIVE, offsetof(struct pmap, pm_active));
@@ -241,3 +244,7 @@ ASSYM(IOAPIC_IRQI_SIZE, sizeof(struct ioapic_irqinfo));
 ASSYM(IOAPIC_IRQI_SZSHIFT, IOAPIC_IRQI_SZSHIFT);
 ASSYM(IOAPIC_IRQI_FLAG_LEVEL, IOAPIC_IRQI_FLAG_LEVEL);
 ASSYM(IOAPIC_IRQI_FLAG_MASKED, IOAPIC_IRQI_FLAG_MASKED);
+
+#ifdef	HWPMC_HOOKS
+ASSYM(PMC_FN_USER_CALLCHAIN, PMC_FN_USER_CALLCHAIN);
+#endif
diff --git a/sys/platform/pc32/i386/trap.c b/sys/platform/pc32/i386/trap.c
index 724806f..61fe0b6 100644
--- a/sys/platform/pc32/i386/trap.c
+++ b/sys/platform/pc32/i386/trap.c
@@ -36,7 +36,6 @@
  *
  *	from: @(#)trap.c	7.4 (Berkeley) 5/13/91
  * $FreeBSD: src/sys/i386/i386/trap.c,v 1.147.2.11 2003/02/27 19:09:59 luoqi Exp $
- * $DragonFly: src/sys/platform/pc32/i386/trap.c,v 1.115 2008/09/09 04:06:17 dillon Exp $
  */
 
 /*
@@ -110,6 +109,10 @@
 #include <sys/thread2.h>
 #include <sys/mplock2.h>
 
+#ifdef HWPMC_HOOKS
+#include <sys/pmckern.h>
+#endif
+
 #ifdef SMP
 
 #define MAKEMPSAFE(have_mplock)			\
@@ -487,6 +490,19 @@ restart:
 	type = frame->tf_trapno;
 	code = frame->tf_err;
 
+#ifdef	HWPMC_HOOKS
+	/*
+	 * CPU PMCs interrupt using an NMI so we check for that first.
+	 * If the HWPMC module is active, 'pmc_hook' will point to
+	 * the function to be called.  A return value of '1' from the
+	 * hook means that the NMI was handled by it and that we can
+	 * return immediately.
+	 */
+	if (type == T_NMI && pmc_intr &&
+	    (*pmc_intr)(mycpuid, frame))
+	    return;
+#endif
+
 	if (in_vm86call) {
 		if (frame->tf_eflags & PSL_VM &&
 		    (type == T_PROTFLT || type == T_STKFLT)) {
diff --git a/sys/platform/pc64/x86_64/exception.S b/sys/platform/pc64/x86_64/exception.S
index 52bb3a6..91ac0bd 100644
--- a/sys/platform/pc64/x86_64/exception.S
+++ b/sys/platform/pc64/x86_64/exception.S
@@ -47,6 +47,9 @@
 #include "assym.s"
 
 	.text
+#ifdef HWPMC_HOOKS
+	ENTRY(start_exceptions)
+#endif
 
 	.globl	lwkt_switch_return
 
@@ -532,3 +535,6 @@ MCOUNT_LABEL(bintr)
 	.text
 MCOUNT_LABEL(eintr)
 
+#ifdef HWPMC_HOOKS
+	ENTRY(end_exceptions)
+#endif
diff --git a/sys/platform/pc64/x86_64/genassym.c b/sys/platform/pc64/x86_64/genassym.c
index 677a3a3..7c16ea6 100644
--- a/sys/platform/pc64/x86_64/genassym.c
+++ b/sys/platform/pc64/x86_64/genassym.c
@@ -74,6 +74,10 @@
 #include <machine/pcb.h>
 #include <machine/pmap.h>
 
+#ifdef	HWPMC_HOOKS
+#include <sys/pmckern.h>
+#endif
+
 ASSYM(P_VMSPACE, offsetof(struct proc, p_vmspace));
 ASSYM(VM_PMAP, offsetof(struct vmspace, vm_pmap));
 ASSYM(PM_ACTIVE, offsetof(struct pmap, pm_active));
@@ -247,3 +251,7 @@ ASSYM(IOAPIC_IRQI_SIZE, sizeof(struct ioapic_irqinfo));
 ASSYM(IOAPIC_IRQI_SZSHIFT, IOAPIC_IRQI_SZSHIFT);
 ASSYM(IOAPIC_IRQI_FLAG_LEVEL, IOAPIC_IRQI_FLAG_LEVEL);
 ASSYM(IOAPIC_IRQI_FLAG_MASKED, IOAPIC_IRQI_FLAG_MASKED);
+
+#ifdef	HWPMC_HOOKS
+ASSYM(PMC_FN_USER_CALLCHAIN, PMC_FN_USER_CALLCHAIN);
+#endif
diff --git a/sys/platform/pc64/x86_64/trap.c b/sys/platform/pc64/x86_64/trap.c
index f74bc24..9863a5d 100644
--- a/sys/platform/pc64/x86_64/trap.c
+++ b/sys/platform/pc64/x86_64/trap.c
@@ -90,6 +90,10 @@
 #include <sys/thread2.h>
 #include <sys/mplock2.h>
 
+#ifdef HWPMC_HOOKS
+#include <sys/pmckern.h>
+#endif
+
 #ifdef SMP
 
 #define MAKEMPSAFE(have_mplock)			\
@@ -456,6 +460,18 @@ trap(struct trapframe *frame)
 	type = frame->tf_trapno;
 	code = frame->tf_err;
 
+#ifdef	HWPMC_HOOKS
+	/*
+	 * CPU PMCs interrupt using an NMI.  If the PMC module is
+	 * active, pass the 'rip' value to the PMC module's interrupt
+	 * handler.  A return value of '1' from the handler means that
+	 * the NMI was handled by it and we can return immediately.
+	 */
+	if (type == T_NMI && pmc_intr &&
+	    (*pmc_intr)(mycpuid, frame))
+		return;
+#endif
+
 	if (ISPL(frame->tf_cs) == SEL_UPL) {
 		/* user trap */
 
diff --git a/sys/vm/vm_mmap.c b/sys/vm/vm_mmap.c
index 9e790d2..c5a2695 100644
--- a/sys/vm/vm_mmap.c
+++ b/sys/vm/vm_mmap.c
@@ -79,6 +79,10 @@
 #include <sys/thread.h>
 #include <sys/thread2.h>
 
+#ifdef HWPMC_HOOKS
+#include <sys/pmckern.h>
+#endif
+
 static int max_proc_mmap;
 SYSCTL_INT(_vm, OID_AUTO, max_proc_mmap, CTLFLAG_RW, &max_proc_mmap, 0, "");
 int vkernel_enable;
@@ -151,6 +155,9 @@ int
 kern_mmap(struct vmspace *vms, caddr_t uaddr, size_t ulen,
 	  int uprot, int uflags, int fd, off_t upos, void **res)
 {
+#ifdef HWPMC_HOOKS
+	struct pmckern_map_in pkm;
+#endif
 	struct thread *td = curthread;
  	struct proc *p = td->td_proc;
 	struct file *fp = NULL;
@@ -160,6 +167,7 @@ kern_mmap(struct vmspace *vms, caddr_t uaddr, size_t ulen,
 	vm_size_t size, pageoff;
 	vm_prot_t prot, maxprot;
 	void *handle;
+	objtype_t handle_type;
 	int flags, error;
 	off_t pos;
 	vm_object_t obj;
@@ -258,6 +266,7 @@ kern_mmap(struct vmspace *vms, caddr_t uaddr, size_t ulen,
 		 * Mapping blank space is trivial.
 		 */
 		handle = NULL;
+		handle_type = OBJT_DEFAULT;
 		maxprot = VM_PROT_ALL;
 	} else {
 		/*
@@ -321,6 +330,7 @@ kern_mmap(struct vmspace *vms, caddr_t uaddr, size_t ulen,
 		 */
 		if (vp->v_type == VCHR && iszerodev(vp->v_rdev)) {
 			handle = NULL;
+			handle_type = OBJT_DEFAULT;
 			maxprot = VM_PROT_ALL;
 			flags |= MAP_ANON;
 			pos = 0;
@@ -379,6 +389,7 @@ kern_mmap(struct vmspace *vms, caddr_t uaddr, size_t ulen,
 				maxprot |= VM_PROT_WRITE;
 			}
 			handle = (void *)vp;
+			handle_type = OBJT_VNODE;
 		}
 	}
 
@@ -398,6 +409,15 @@ kern_mmap(struct vmspace *vms, caddr_t uaddr, size_t ulen,
 
 	error = vm_mmap(&vms->vm_map, &addr, size, prot, maxprot,
 			flags, handle, pos);
+#ifdef HWPMC_HOOKS
+	/* inform hwpmc(4) if an executable is being mapped */
+	if (error == 0 && handle_type == OBJT_VNODE &&
+	    (prot & PROT_EXEC)) {
+		pkm.pm_file = handle;
+		pkm.pm_address = (uintptr_t) addr;
+		PMC_CALL_HOOK(td, PMC_FN_MMAP, (void *) &pkm);
+	}
+#endif
 	if (error == 0)
 		*res = (void *)(addr + pageoff);
 
@@ -525,6 +545,10 @@ done:
 int
 sys_munmap(struct munmap_args *uap)
 {
+#ifdef HWPMC_HOOKS
+	struct pmckern_map_out pkm;
+	vm_map_entry_t entry;
+#endif
 	struct proc *p = curproc;
 	vm_offset_t addr;
 	vm_offset_t tmpaddr;
@@ -569,8 +593,33 @@ sys_munmap(struct munmap_args *uap)
 		lwkt_reltoken(&map->token);
 		return (EINVAL);
 	}
+#ifdef HWPMC_HOOKS
+	/*
+	 * Inform hwpmc if the address range being unmapped contains
+	 * an executable region.
+	 */
+	pkm.pm_address = (uintptr_t) NULL;
+	vm_map_lock_read(map);
+	if (vm_map_lookup_entry(map, addr, &entry)) {
+		for (;
+		     entry != &map->header && entry->start < addr + size;
+		     entry = entry->next) {
+			if (vm_map_check_protection(map, entry->start,
+				entry->end, VM_PROT_EXECUTE, FALSE) == TRUE) {
+				pkm.pm_address = (uintptr_t) addr;
+				pkm.pm_size = (size_t) size;
+				break;
+			}
+		}
+	}
+	vm_map_unlock_read(map);
+#endif
 	/* returns nothing but KERN_SUCCESS anyway */
 	vm_map_remove(map, addr, addr + size);
+#ifdef HWPMC_HOOKS
+	if (pkm.pm_address != (uintptr_t) NULL)
+		PMC_CALL_HOOK(curthread, PMC_FN_MUNMAP, (void *) &pkm);
+#endif
 	lwkt_reltoken(&map->token);
 	return (0);
 }
-- 
1.7.0.5

