From 84986d0c6ab57fb86f0fd24c17ca33709486de3f Mon Sep 17 00:00:00 2001
From: Sascha Wildner <saw@online.de>
Date: Sat, 25 Jun 2011 21:52:25 +0200
Subject: [PATCH 10/18] hwpmc: Locking adjustments.

* Convert the mutex pool FreeBSD is using to protect struct pmc to a
  spinlock in the struct.

* Convert the global sx lock for protecting internal data structures
  to a lockmgr lock.

* For FreeBSD's proctree_lock, grab our proc_token.

* Convert PROC_LOCK and PROC_UNLOCK to taking/releasing the process'
  p_token.

* pmc_bufferlist_mtx -> pmc_bufferlist_lock (spinlock)

* pmc_kthread_mtx -> pmc_kthread_lock (lockmgr)

* pmc_processhash_mtx -> pmc_processhash_lock (spinlock)

* po_mtx -> po_lock (spinlock)

* pc_mtx -> pc_lock (spinlock) for P4

* Remove FreeBSD's critical sections since we don't preempt kernel code.

* pfind() PHOLD()s the process, so PRELE() the process after calls to
  pfind().

* Remove PPAUSE and PWAIT flags.

In-collaboration-with: aggelos
---
 share/man/man4/hwpmc.4             |    3 -
 sys/dev/misc/hwpmc/hwpmc_logging.c |   97 ++++++-------
 sys/dev/misc/hwpmc/hwpmc_mod.c     |  265 +++++++++++++++---------------------
 sys/dev/misc/hwpmc/hwpmc_piv.c     |   22 ++--
 sys/kern/kern_pmc.c                |   17 ++-
 sys/sys/pmc.h                      |   10 +-
 sys/sys/pmckern.h                  |   12 +-
 7 files changed, 186 insertions(+), 240 deletions(-)

diff --git a/share/man/man4/hwpmc.4 b/share/man/man4/hwpmc.4
index a7990c9..f797767 100644
--- a/share/man/man4/hwpmc.4
+++ b/share/man/man4/hwpmc.4
@@ -423,9 +423,6 @@ The size in kilobytes of each log buffer used by
 .Nm Ns 's
 logging function.
 The default buffer size is 4KB.
-.It Va kern.hwpmc.mtxpoolsize Pq integer, read-only
-The size of the spin mutex pool used by the PMC driver.
-The default is 32.
 .It Va kern.hwpmc.nbuffers Pq integer, read-only
 The number of log buffers used by
 .Nm
diff --git a/sys/dev/misc/hwpmc/hwpmc_logging.c b/sys/dev/misc/hwpmc/hwpmc_logging.c
index 4496e5d..2fd7d1a 100644
--- a/sys/dev/misc/hwpmc/hwpmc_logging.c
+++ b/sys/dev/misc/hwpmc/hwpmc_logging.c
@@ -84,8 +84,8 @@ SYSCTL_INT(_kern_hwpmc, OID_AUTO, nbuffers, CTLFLAG_RD,
 
 TAILQ_HEAD(, pmclog_buffer) pmc_bufferlist =
 	TAILQ_HEAD_INITIALIZER(pmc_bufferlist);
-static struct mtx pmc_bufferlist_mtx;	/* spin lock */
-static struct mtx pmc_kthread_mtx;	/* sleep lock */
+static struct spinlock pmc_bufferlist_lock;	/* spin lock */
+static struct lock pmc_kthread_lock;		/* sleep lock */
 
 #define	PMCLOG_INIT_BUFFER_DESCRIPTOR(D) do {				\
 		const int __roundup = roundup(sizeof(*D),		\
@@ -197,15 +197,15 @@ pmclog_get_buffer(struct pmc_owner *po)
 {
 	struct pmclog_buffer *plb;
 
-	mtx_assert(&po->po_mtx, MA_OWNED);
+	assert_spin_locked_self(&po->po_lock);
 
 	KASSERT(po->po_curbuf == NULL,
 	    ("[pmclog,%d] po=%p current buffer still valid", __LINE__, po));
 
-	mtx_lock_spin(&pmc_bufferlist_mtx);
+	spin_lock(&pmc_bufferlist_lock);
 	if ((plb = TAILQ_FIRST(&pmc_bufferlist)) != NULL)
 		TAILQ_REMOVE(&pmc_bufferlist, plb, plb_next);
-	mtx_unlock_spin(&pmc_bufferlist_mtx);
+	spin_unlock(&pmc_bufferlist_lock);
 
 	PMCDBG(LOG,GTB,1, "po=%p plb=%p", po, plb);
 
@@ -254,9 +254,9 @@ pmclog_loop(void *arg)
 	mycred = td->td_ucred;
 	last_buffer = 0;
 
-	PROC_LOCK(p);
+	lwkt_gettoken(&p->p_token);
 	ownercred = crhold(p->p_ucred);
-	PROC_UNLOCK(p);
+	lwkt_reltoken(&p->p_token);
 
 	PMCDBG(LOG,INI,1, "po=%p kt=%p", po, po->po_kthread);
 	KASSERT(po->po_kthread == curthread->td_proc,
@@ -272,7 +272,7 @@ pmclog_loop(void *arg)
 	 * is deconfigured.
 	 */
 
-	mtx_lock(&pmc_kthread_mtx);
+	lockmgr(&pmc_kthread_lock, LK_EXCLUSIVE);
 
 	for (;;) {
 
@@ -281,11 +281,11 @@ pmclog_loop(void *arg)
 			break;
 
 		if (lb == NULL) { /* look for a fresh buffer to write */
-			mtx_lock_spin(&po->po_mtx);
+			spin_lock(&po->po_lock);
 			if ((lb = TAILQ_FIRST(&po->po_logbuffers)) == NULL) {
-				mtx_unlock_spin(&po->po_mtx);
+				spin_unlock(&po->po_lock);
 
-				(void) msleep(po, &pmc_kthread_mtx, PWAIT,
+				(void) lksleep(po, &pmc_kthread_lock, 0,
 				    "pmcloop", 0);
 				continue;
 			}
@@ -293,10 +293,10 @@ pmclog_loop(void *arg)
 			TAILQ_REMOVE(&po->po_logbuffers, lb, plb_next);
 			if (po->po_flags & PMC_PO_SHUTDOWN)
 				last_buffer = TAILQ_EMPTY(&po->po_logbuffers);
-			mtx_unlock_spin(&po->po_mtx);
+			spin_unlock(&po->po_lock);
 		}
 
-		mtx_unlock(&pmc_kthread_mtx);
+		lockmgr(&pmc_kthread_lock, LK_RELEASE);
 
 		/* process the request */
 		PMCDBG(LOG,WRI,2, "po=%p base=%p ptr=%p", po,
@@ -322,11 +322,11 @@ pmclog_loop(void *arg)
 		if (error) {
 			/* XXX some errors are recoverable */
 			/* send a SIGIO to the owner and exit */
-			PROC_LOCK(p);
+			lwkt_gettoken(&p->p_token);
 			psignal(p, SIGIO);
-			PROC_UNLOCK(p);
+			lwkt_reltoken(&p->p_token);
 
-			mtx_lock(&pmc_kthread_mtx);
+			lockmgr(&pmc_kthread_lock, LK_EXCLUSIVE);
 
 			po->po_error = error; /* save for flush log */
 
@@ -343,29 +343,29 @@ pmclog_loop(void *arg)
 			fo_close(po->po_file, curthread);
 		}
 
-		mtx_lock(&pmc_kthread_mtx);
+		lockmgr(&pmc_kthread_lock, LK_EXCLUSIVE);
 
 		/* put the used buffer back into the global pool */
 		PMCLOG_INIT_BUFFER_DESCRIPTOR(lb);
 
-		mtx_lock_spin(&pmc_bufferlist_mtx);
+		spin_lock(&pmc_bufferlist_lock);
 		TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next);
-		mtx_unlock_spin(&pmc_bufferlist_mtx);
+		spin_unlock(&pmc_bufferlist_lock);
 
 		lb = NULL;
 	}
 
 	po->po_kthread = NULL;
 
-	mtx_unlock(&pmc_kthread_mtx);
+	lockmgr(&pmc_kthread_lock, LK_RELEASE);
 
 	/* return the current I/O buffer to the global pool */
 	if (lb) {
 		PMCLOG_INIT_BUFFER_DESCRIPTOR(lb);
 
-		mtx_lock_spin(&pmc_bufferlist_mtx);
+		spin_lock(&pmc_bufferlist_lock);
 		TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next);
-		mtx_unlock_spin(&pmc_bufferlist_mtx);
+		spin_unlock(&pmc_bufferlist_lock);
 	}
 
 	/*
@@ -395,7 +395,7 @@ pmclog_release(struct pmc_owner *po)
 	if (po->po_curbuf->plb_ptr >= po->po_curbuf->plb_fence)
 		pmclog_schedule_io(po);
 
-	mtx_unlock_spin(&po->po_mtx);
+	spin_unlock(&po->po_lock);
 
 	PMCDBG(LOG,REL,1, "po=%p", po);
 }
@@ -422,17 +422,17 @@ pmclog_reserve(struct pmc_owner *po, int length)
 	KASSERT(length % sizeof(uint32_t) == 0,
 	    ("[pmclog,%d] length not a multiple of word size", __LINE__));
 
-	mtx_lock_spin(&po->po_mtx);
+	spin_lock(&po->po_lock);
 
 	/* No more data when shutdown in progress. */
 	if (po->po_flags & PMC_PO_SHUTDOWN) {
-		mtx_unlock_spin(&po->po_mtx);
+		spin_unlock(&po->po_lock);
 		return (NULL);
 	}
 
 	if (po->po_curbuf == NULL)
 		if (pmclog_get_buffer(po) != 0) {
-			mtx_unlock_spin(&po->po_mtx);
+			spin_unlock(&po->po_lock);
 			return (NULL);
 		}
 
@@ -467,7 +467,7 @@ pmclog_reserve(struct pmc_owner *po, int length)
 	pmclog_schedule_io(po);
 
 	if (pmclog_get_buffer(po) != 0) {
-		mtx_unlock_spin(&po->po_mtx);
+		spin_unlock(&po->po_lock);
 		return (NULL);
 	}
 
@@ -515,7 +515,7 @@ pmclog_schedule_io(struct pmc_owner *po)
 
 	PMCDBG(LOG,SIO, 1, "po=%p", po);
 
-	mtx_assert(&po->po_mtx, MA_OWNED);
+	assert_spin_locked_self(&po->po_lock);
 
 	/*
 	 * Add the current buffer to the tail of the buffer list and
@@ -542,12 +542,12 @@ pmclog_stop_kthread(struct pmc_owner *po)
 	if (po->po_file != NULL)
 		fo_close(po->po_file, curthread);
 
-	mtx_lock(&pmc_kthread_mtx);
+	lockmgr(&pmc_kthread_lock, LK_EXCLUSIVE);
 	po->po_flags &= ~PMC_PO_OWNS_LOGFILE;
 	wakeup_one(po);
 	if (po->po_kthread)
-		msleep(po->po_kthread, &pmc_kthread_mtx, PPAUSE, "pmckstp", 0);
-	mtx_unlock(&pmc_kthread_mtx);
+		lksleep(po->po_kthread, &pmc_kthread_lock, 0, "pmckstp", 0);
+	lockmgr(&pmc_kthread_lock, LK_RELEASE);
 }
 
 /*
@@ -567,12 +567,6 @@ pmclog_configure_log(struct pmc_mdep *md, struct pmc_owner *po, int logfd)
 	int error;
 	struct proc *p;
 
-	/*
-	 * As long as it is possible to get a LOR between pmc_sx lock and
-	 * proctree/allproc sx locks used for adding a new process, assure
-	 * the former is not held here.
-	 */
-	sx_assert(&pmc_sx, SA_UNLOCKED);
 	PMCDBG(LOG,CFG,1, "config po=%p logfd=%d", po, logfd);
 
 	p = po->po_owner;
@@ -601,9 +595,9 @@ pmclog_configure_log(struct pmc_mdep *md, struct pmc_owner *po, int logfd)
 		goto error;
 
 	/* mark process as using HWPMCs */
-	PROC_LOCK(p);
+	lwkt_gettoken(&p->p_token);
 	p->p_flag |= P_HWPMC;
-	PROC_UNLOCK(p);
+	lwkt_reltoken(&p->p_token);
 
 	/* create a log initialization entry */
 	PMCLOG_RESERVE_WITH_ERROR(po, INITIALIZE,
@@ -663,17 +657,17 @@ pmclog_deconfigure_log(struct pmc_owner *po)
 	while ((lb = TAILQ_FIRST(&po->po_logbuffers)) != NULL) {
 		TAILQ_REMOVE(&po->po_logbuffers, lb, plb_next);
 		PMCLOG_INIT_BUFFER_DESCRIPTOR(lb);
-		mtx_lock_spin(&pmc_bufferlist_mtx);
+		spin_lock(&pmc_bufferlist_lock);
 		TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next);
-		mtx_unlock_spin(&pmc_bufferlist_mtx);
+		spin_unlock(&pmc_bufferlist_lock);
 	}
 
 	/* return the 'current' buffer to the global pool */
 	if ((lb = po->po_curbuf) != NULL) {
 		PMCLOG_INIT_BUFFER_DESCRIPTOR(lb);
-		mtx_lock_spin(&pmc_bufferlist_mtx);
+		spin_lock(&pmc_bufferlist_lock);
 		TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next);
-		mtx_unlock_spin(&pmc_bufferlist_mtx);
+		spin_unlock(&pmc_bufferlist_lock);
 	}
 
 	/* drop a reference to the fd */
@@ -707,7 +701,7 @@ pmclog_flush(struct pmc_owner *po)
 	/*
 	 * Check that we do have an active log file.
 	 */
-	mtx_lock(&pmc_kthread_mtx);
+	lockmgr(&pmc_kthread_lock, LK_EXCLUSIVE);
 	if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) {
 		error = EINVAL;
 		goto error;
@@ -716,10 +710,10 @@ pmclog_flush(struct pmc_owner *po)
 	/*
 	 * Schedule the current buffer if any.
 	 */
-	mtx_lock_spin(&po->po_mtx);
+	spin_lock(&po->po_lock);
 	if (po->po_curbuf)
 		pmclog_schedule_io(po);
-	mtx_unlock_spin(&po->po_mtx);
+	spin_unlock(&po->po_lock);
 
 	/*
 	 * Initiate shutdown: no new data queued,
@@ -728,7 +722,7 @@ pmclog_flush(struct pmc_owner *po)
 	po->po_flags |= PMC_PO_SHUTDOWN;
 
  error:
-	mtx_unlock(&pmc_kthread_mtx);
+	lockmgr(&pmc_kthread_lock, LK_RELEASE);
 
 	return (error);
 }
@@ -998,9 +992,8 @@ pmclog_initialize(void)
 		PMCLOG_INIT_BUFFER_DESCRIPTOR(plb);
 		TAILQ_INSERT_HEAD(&pmc_bufferlist, plb, plb_next);
 	}
-	mtx_init(&pmc_bufferlist_mtx, "pmc-buffer-list", "pmc-leaf",
-	    MTX_SPIN);
-	mtx_init(&pmc_kthread_mtx, "pmc-kthread", "pmc-sleep", MTX_DEF);
+	spin_init(&pmc_bufferlist_lock);
+	lockinit(&pmc_kthread_lock, "pmc-sleep", 0, 0);
 }
 
 /*
@@ -1014,8 +1007,8 @@ pmclog_shutdown(void)
 {
 	struct pmclog_buffer *plb;
 
-	mtx_destroy(&pmc_kthread_mtx);
-	mtx_destroy(&pmc_bufferlist_mtx);
+	lockuninit(&pmc_kthread_lock);
+	spin_uninit(&pmc_bufferlist_lock);
 
 	while ((plb = TAILQ_FIRST(&pmc_bufferlist)) != NULL) {
 		TAILQ_REMOVE(&pmc_bufferlist, plb, plb_next);
diff --git a/sys/dev/misc/hwpmc/hwpmc_mod.c b/sys/dev/misc/hwpmc/hwpmc_mod.c
index fa03e8c..e24ba80 100644
--- a/sys/dev/misc/hwpmc/hwpmc_mod.c
+++ b/sys/dev/misc/hwpmc/hwpmc_mod.c
@@ -89,7 +89,6 @@ pmc_value_t		*pmc_pcpu_saved; /* saved PMC values: CSW handling */
 
 #define	PMC_PCPU_SAVED(C,R)	pmc_pcpu_saved[(R) + md->pmd_npmc*(C)]
 
-struct mtx_pool		*pmc_mtxpool;
 static int		*pmc_pmcdisp;	 /* PMC row dispositions */
 
 #define	PMC_ROW_DISP_IS_FREE(R)		(pmc_pmcdisp[(R)] == 0)
@@ -140,7 +139,7 @@ static struct pmc_mdep  *md;
  * Hash tables mapping owner processes and target threads to PMCs.
  */
 
-struct mtx pmc_processhash_mtx;		/* spin mutex */
+struct spinlock pmc_processhash_lock;	/* spin lock */
 static u_long pmc_processhashmask;
 static LIST_HEAD(pmc_processhash, pmc_process)	*pmc_processhash;
 
@@ -258,17 +257,6 @@ TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "nsamples", &pmc_nsamples);
 SYSCTL_INT(_kern_hwpmc, OID_AUTO, nsamples, CTLFLAG_RD,
     &pmc_nsamples, 0, "number of PC samples per CPU");
 
-
-/*
- * kern.hwpmc.mtxpoolsize -- number of mutexes in the mutex pool.
- */
-
-static int pmc_mtxpool_size = PMC_MTXPOOL_SIZE;
-TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "mtxpoolsize", &pmc_mtxpool_size);
-SYSCTL_INT(_kern_hwpmc, OID_AUTO, mtxpoolsize, CTLFLAG_TUN|CTLFLAG_RD,
-    &pmc_mtxpool_size, 0, "size of spin mutex pool");
-
-
 /*
  * security.bsd.unprivileged_syspmcs -- allow non-root processes to
  * allocate system-wide PMCs.
@@ -546,7 +534,7 @@ pmc_ri_to_classdep(struct pmc_mdep *md, int ri, int *adjri)
  *
  * The driver uses four locking strategies for its operation:
  *
- * - The global SX lock "pmc_sx" is used to protect internal
+ * - The global lock "pmc_lock" is used to protect internal
  *   data structures.
  *
  *   Calls into the module by syscall() start with this lock being
@@ -555,23 +543,22 @@ pmc_ri_to_classdep(struct pmc_mdep *md, int ri, int *adjri)
  *   concurrent readers into the module.  Calls into the module from
  *   other parts of the kernel acquire the lock in shared mode.
  *
- *   This SX lock is held in exclusive mode for any operations that
+ *   This lockmgr lock is held in exclusive mode for any operations that
  *   modify the linkages between the driver's internal data structures.
  *
  *   The 'pmc_hook' function pointer is also protected by this lock.
- *   It is only examined with the sx lock held in exclusive mode.  The
- *   kernel module is allowed to be unloaded only with the sx lock held
+ *   It is only examined with the lock held in exclusive mode.  The
+ *   kernel module is allowed to be unloaded only with the lock held
  *   in exclusive mode.  In normal syscall handling, after acquiring the
- *   pmc_sx lock we first check that 'pmc_hook' is non-null before
+ *   pmc_lock lock we first check that 'pmc_hook' is non-null before
  *   proceeding.  This prevents races between the thread unloading the module
  *   and other threads seeking to use the module.
  *
  * - Lookups of target process structures and owner process structures
- *   cannot use the global "pmc_sx" SX lock because these lookups need
+ *   cannot use the global "pmc_lock" lock because these lookups need
  *   to happen during context switches and in other critical sections
  *   where sleeping is not allowed.  We protect these lookup tables
- *   with their own private spin-mutexes, "pmc_processhash_mtx" and
- *   "pmc_ownerhash_mtx".
+ *   with its own private spinlock, "pmc_processhash_lock".
  *
  * - Interrupt handlers work in a lock free manner.  At interrupt
  *   time, handlers look at the PMC pointer (phw->phw_pmc) configured
@@ -625,13 +612,7 @@ pmc_ri_to_classdep(struct pmc_mdep *md, int ri, int *adjri)
  *     pause() till the runcount comes to zero.
  *
  * The contents of a PMC descriptor (struct pmc) are protected using
- * a spin-mutex.  In order to save space, we use a mutex pool.
- *
- * In terms of lock types used by witness(4), we use:
- * - Type "pmc-sx", used by the global SX lock.
- * - Type "pmc-sleep", for sleep mutexes used by logger threads.
- * - Type "pmc-per-proc", for protecting PMC owner descriptors.
- * - Type "pmc-leaf", used for all other spin mutexes.
+ * a spinlock in struct pmc.
  */
 
 /*
@@ -730,7 +711,7 @@ pmc_remove_owner(struct pmc_owner *po)
 {
 	struct pmc *pm, *tmp;
 
-	sx_assert(&pmc_sx, SX_XLOCKED);
+	KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE);
 
 	PMCDBG(OWN,ORM,1, "remove-owner po=%p", po);
 
@@ -789,7 +770,7 @@ pmc_link_target_process(struct pmc *pm, struct pmc_process *pp)
 	int ri;
 	struct pmc_target *pt;
 
-	sx_assert(&pmc_sx, SX_XLOCKED);
+	KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE);
 
 	KASSERT(pm != NULL && pp != NULL,
 	    ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
@@ -844,7 +825,7 @@ pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp)
 	struct proc *p;
 	struct pmc_target *ptgt;
 
-	sx_assert(&pmc_sx, SX_XLOCKED);
+	KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE);
 
 	KASSERT(pm != NULL && pp != NULL,
 	    ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
@@ -887,9 +868,9 @@ pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp)
 	/* if the PMC now lacks targets, send the owner a SIGIO */
 	if (LIST_EMPTY(&pm->pm_targets)) {
 		p = pm->pm_owner->po_owner;
-		PROC_LOCK(p);
+		lwkt_gettoken(&p->p_token);
 		psignal(p, SIGIO);
-		PROC_UNLOCK(p);
+		lwkt_reltoken(&p->p_token);
 
 		PMCDBG(PRC,SIG,2, "signalling proc=%p signal=%d", p,
 		    SIGIO);
@@ -914,15 +895,15 @@ pmc_can_attach(struct pmc *pm, struct proc *t)
 	if ((o = pm->pm_owner->po_owner) == t)
 		return 0;
 
-	PROC_LOCK(o);
+	lwkt_gettoken(&o->p_token);
 	oc = o->p_ucred;
 	crhold(oc);
-	PROC_UNLOCK(o);
+	lwkt_reltoken(&o->p_token);
 
-	PROC_LOCK(t);
+	lwkt_gettoken(&t->p_token);
 	tc = t->p_ucred;
 	crhold(tc);
-	PROC_UNLOCK(t);
+	lwkt_reltoken(&t->p_token);
 
 	/*
 	 * The effective uid of the PMC owner should match at least one
@@ -962,7 +943,7 @@ pmc_attach_one_process(struct proc *p, struct pmc *pm)
 	char *fullpath, *freepath;
 	struct pmc_process	*pp;
 
-	sx_assert(&pmc_sx, SX_XLOCKED);
+	KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE);
 
 	PMCDBG(PRC,ATT,2, "attach-one pm=%p ri=%d proc=%p (%d, %s)", pm,
 	    PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
@@ -1010,9 +991,9 @@ pmc_attach_one_process(struct proc *p, struct pmc *pm)
 			pmc_log_process_mappings(pm->pm_owner, p);
 	}
 	/* mark process as using HWPMCs */
-	PROC_LOCK(p);
+	lwkt_gettoken(&p->p_token);
 	p->p_flag |= P_HWPMC;
-	PROC_UNLOCK(p);
+	lwkt_reltoken(&p->p_token);
 
 	return 0;
 }
@@ -1027,7 +1008,7 @@ pmc_attach_process(struct proc *p, struct pmc *pm)
 	int error;
 	struct proc *top;
 
-	sx_assert(&pmc_sx, SX_XLOCKED);
+	KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE);
 
 	PMCDBG(PRC,ATT,1, "attach pm=%p ri=%d proc=%p (%d, %s)", pm,
 	    PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
@@ -1049,7 +1030,7 @@ pmc_attach_process(struct proc *p, struct pmc *pm)
 	 * this PMC.
 	 */
 
-	sx_slock(&proctree_lock);
+	lwkt_gettoken(&proc_token);
 
 	top = p;
 
@@ -1073,7 +1054,7 @@ pmc_attach_process(struct proc *p, struct pmc *pm)
 		(void) pmc_detach_process(top, pm);
 
  done:
-	sx_sunlock(&proctree_lock);
+	lwkt_reltoken(&proc_token);
 	return error;
 }
 
@@ -1089,7 +1070,7 @@ pmc_detach_one_process(struct proc *p, struct pmc *pm, int flags)
 	int ri;
 	struct pmc_process *pp;
 
-	sx_assert(&pmc_sx, SX_XLOCKED);
+	KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE);
 
 	KASSERT(pm != NULL,
 	    ("[pmc,%d] null pm pointer", __LINE__));
@@ -1128,9 +1109,9 @@ pmc_detach_one_process(struct proc *p, struct pmc *pm, int flags)
 	if (flags & PMC_FLAG_REMOVE)
 		kfree(pp, M_PMC);
 
-	PROC_LOCK(p);
+	lwkt_gettoken(&p->p_token);
 	p->p_flag &= ~P_HWPMC;
-	PROC_UNLOCK(p);
+	lwkt_reltoken(&p->p_token);
 
 	return 0;
 }
@@ -1144,7 +1125,7 @@ pmc_detach_process(struct proc *p, struct pmc *pm)
 {
 	struct proc *top;
 
-	sx_assert(&pmc_sx, SX_XLOCKED);
+	KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE);
 
 	PMCDBG(PRC,ATT,1, "detach pm=%p ri=%d proc=%p (%d, %s)", pm,
 	    PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
@@ -1158,7 +1139,7 @@ pmc_detach_process(struct proc *p, struct pmc *pm)
 	 * partially attached proc tree.
 	 */
 
-	sx_slock(&proctree_lock);
+	lwkt_gettoken(&proc_token);
 
 	top = p;
 
@@ -1179,7 +1160,7 @@ pmc_detach_process(struct proc *p, struct pmc *pm)
 	}
 
  done:
-	sx_sunlock(&proctree_lock);
+	lwkt_reltoken(&proc_token);
 
 	if (LIST_EMPTY(&pm->pm_targets))
 		pm->pm_flags &= ~PMC_F_ATTACH_DONE;
@@ -1213,8 +1194,6 @@ pmc_process_csw_in(struct thread *td)
 	KASSERT(pp->pp_proc == td->td_proc,
 	    ("[pmc,%d] not my thread state", __LINE__));
 
-	critical_enter(); /* no preemption from this point */
-
 	cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */
 
 	PMCDBG(CSW,SWI,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
@@ -1270,18 +1249,18 @@ pmc_process_csw_in(struct thread *td)
 		 * inherited across descendants.
 		 */
 		if (PMC_TO_MODE(pm) == PMC_MODE_TS) {
-			mtx_pool_lock_spin(pmc_mtxpool, pm);
+			spin_lock(&pm->pm_lock);
 			newvalue = PMC_PCPU_SAVED(cpu,ri) =
 			    pp->pp_pmcs[ri].pp_pmcval;
-			mtx_pool_unlock_spin(pmc_mtxpool, pm);
+			spin_unlock(&pm->pm_lock);
 		} else {
 			KASSERT(PMC_TO_MODE(pm) == PMC_MODE_TC,
 			    ("[pmc,%d] illegal mode=%d", __LINE__,
 			    PMC_TO_MODE(pm)));
-			mtx_pool_lock_spin(pmc_mtxpool, pm);
+			spin_lock(&pm->pm_lock);
 			newvalue = PMC_PCPU_SAVED(cpu, ri) =
 			    pm->pm_gv.pm_savedvalue;
-			mtx_pool_unlock_spin(pmc_mtxpool, pm);
+			spin_unlock(&pm->pm_lock);
 		}
 
 		PMCDBG(CSW,SWI,1,"cpu=%d ri=%d new=%jd", cpu, ri, newvalue);
@@ -1296,9 +1275,6 @@ pmc_process_csw_in(struct thread *td)
 	 */
 
 	(void) (*md->pmd_switch_in)(pc, pp);
-
-	critical_exit();
-
 }
 
 /*
@@ -1341,8 +1317,6 @@ pmc_process_csw_out(struct thread *td)
 	 * save PMCs
 	 */
 
-	critical_enter();
-
 	cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */
 
 	PMCDBG(CSW,SWO,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
@@ -1422,12 +1396,12 @@ pmc_process_csw_out(struct thread *td)
 				 */
 				if (tmp < 0)
 					tmp += pm->pm_sc.pm_reloadcount;
-				mtx_pool_lock_spin(pmc_mtxpool, pm);
+				spin_lock(&pm->pm_lock);
 				pp->pp_pmcs[ri].pp_pmcval -= tmp;
 				if ((int64_t) pp->pp_pmcs[ri].pp_pmcval < 0)
 					pp->pp_pmcs[ri].pp_pmcval +=
 					    pm->pm_sc.pm_reloadcount;
-				mtx_pool_unlock_spin(pmc_mtxpool, pm);
+				spin_unlock(&pm->pm_lock);
 
 			} else {
 
@@ -1443,10 +1417,10 @@ pmc_process_csw_out(struct thread *td)
 				     "incr=%jx", __LINE__, cpu, ri,
 				     newvalue, PMC_PCPU_SAVED(cpu,ri), tmp));
 
-				mtx_pool_lock_spin(pmc_mtxpool, pm);
+				spin_lock(&pm->pm_lock);
 				pm->pm_gv.pm_savedvalue += tmp;
 				pp->pp_pmcs[ri].pp_pmcval += tmp;
-				mtx_pool_unlock_spin(pmc_mtxpool, pm);
+				spin_unlock(&pm->pm_lock);
 
 				if (pm->pm_flags & PMC_F_LOG_PROCCSW)
 					pmclog_process_proccsw(pm, pp, tmp);
@@ -1463,8 +1437,6 @@ pmc_process_csw_out(struct thread *td)
 	 */
 
 	(void) (*md->pmd_switch_out)(pc, pp);
-
-	critical_exit();
 }
 
 /*
@@ -1475,8 +1447,9 @@ static void
 pmc_process_kld_load(struct pmckern_map_in *pkm)
 {
 	struct pmc_owner *po;
+	int lkstatus __debugvar = lockstatus(&pmc_lock, curthread);
 
-	sx_assert(&pmc_sx, SX_LOCKED);
+	KKASSERT(lkstatus == LK_SHARED || lkstatus == LK_EXCLUSIVE);
 
 	/*
 	 * Notify owners of system sampling PMCs about KLD operations.
@@ -1499,7 +1472,7 @@ pmc_process_kld_unload(struct pmckern_map_out *pkm)
 {
 	struct pmc_owner *po;
 
-	sx_assert(&pmc_sx, SX_LOCKED);
+	KKASSERT(lockstatus(&pmc_lock, curthread) == LK_SHARED);
 
 	LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
 	    if (po->po_flags & PMC_PO_OWNS_LOGFILE)
@@ -1592,8 +1565,9 @@ pmc_log_kernel_mappings(struct pmc *pm)
 {
 	struct pmc_owner *po;
 	struct pmckern_map_in *km, *kmbase;
+	int lkstatus __debugvar = lockstatus(&pmc_lock, curthread);
 
-	sx_assert(&pmc_sx, SX_LOCKED);
+	KKASSERT(lkstatus == LK_SHARED || lkstatus == LK_EXCLUSIVE);
 	KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
 	    ("[pmc,%d] non-sampling PMC (%p) desires mapping information",
 		__LINE__, (void *) pm));
@@ -1786,14 +1760,14 @@ pmc_log_all_process_mappings(struct pmc_owner *po)
 {
 	struct proc *p, *top;
 
-	sx_assert(&pmc_sx, SX_XLOCKED);
+	KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE);
 
 	if ((p = pfind(1)) == NULL)
 		panic("[pmc,%d] Cannot find init", __LINE__);
 
-	PROC_UNLOCK(p);
+	PRELE(p);		/* pfind() returns a locked process */
 
-	sx_slock(&proctree_lock);
+	lwkt_gettoken(&proc_token);
 
 	top = p;
 
@@ -1812,7 +1786,7 @@ pmc_log_all_process_mappings(struct pmc_owner *po)
 		}
 	}
  done:
-	sx_sunlock(&proctree_lock);
+	lwkt_reltoken(&proc_token);
 }
 
 /*
@@ -1839,6 +1813,7 @@ const char *pmc_hooknames[] = {
 static int
 pmc_hook_handler(struct thread *td, int function, void *arg)
 {
+	int lkstatus __debugvar;
 
 	PMCDBG(MOD,PMH,1, "hook td=%p func=%d \"%s\" arg=%p", td, function,
 	    pmc_hooknames[function], arg);
@@ -1861,7 +1836,7 @@ pmc_hook_handler(struct thread *td, int function, void *arg)
 		struct pmc_process *pp;
 		struct pmckern_procexec *pk;
 
-		sx_assert(&pmc_sx, SX_XLOCKED);
+		KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE);
 
 		p = td->td_proc;
 		pmc_getfilename(p->p_textvp, &fullpath, &freepath);
@@ -1874,9 +1849,9 @@ pmc_hook_handler(struct thread *td, int function, void *arg)
 			    pmclog_process_procexec(po, PMC_ID_INVALID,
 				p->p_pid, pk->pm_entryaddr, fullpath);
 
-		PROC_LOCK(p);
+		lwkt_gettoken(&p->p_token);
 		is_using_hwpmcs = p->p_flag & P_HWPMC;
-		PROC_UNLOCK(p);
+		lwkt_reltoken(&p->p_token);
 
 		if (!is_using_hwpmcs) {
 			if (freepath)
@@ -1996,22 +1971,26 @@ pmc_hook_handler(struct thread *td, int function, void *arg)
 
 
 	case PMC_FN_KLD_LOAD:
-		sx_assert(&pmc_sx, SX_LOCKED);
+		lkstatus = lockstatus(&pmc_lock, curthread);
+		KKASSERT(lkstatus == LK_SHARED || lkstatus == LK_EXCLUSIVE);
 		pmc_process_kld_load((struct pmckern_map_in *) arg);
 		break;
 
 	case PMC_FN_KLD_UNLOAD:
-		sx_assert(&pmc_sx, SX_LOCKED);
+		lkstatus = lockstatus(&pmc_lock, curthread);
+		KKASSERT(lkstatus == LK_SHARED || lkstatus == LK_EXCLUSIVE);
 		pmc_process_kld_unload((struct pmckern_map_out *) arg);
 		break;
 
 	case PMC_FN_MMAP:
-		sx_assert(&pmc_sx, SX_LOCKED);
+		lkstatus = lockstatus(&pmc_lock, curthread);
+		KKASSERT(lkstatus == LK_SHARED || lkstatus == LK_EXCLUSIVE);
 		pmc_process_mmap(td, (struct pmckern_map_in *) arg);
 		break;
 
 	case PMC_FN_MUNMAP:
-		sx_assert(&pmc_sx, SX_LOCKED);
+		lkstatus = lockstatus(&pmc_lock, curthread);
+		KKASSERT(lkstatus == LK_SHARED || lkstatus == LK_EXCLUSIVE);
 		pmc_process_munmap(td, (struct pmckern_map_out *) arg);
 		break;
 
@@ -2061,7 +2040,7 @@ pmc_allocate_owner_descriptor(struct proc *p)
 	LIST_INSERT_HEAD(poh, po, po_next); /* insert into hash table */
 
 	TAILQ_INIT(&po->po_logbuffers);
-	mtx_init(&po->po_mtx, "pmc-owner-mtx", "pmc-per-proc", MTX_SPIN);
+	spin_init(&po->po_lock);
 
 	PMCDBG(OWN,ALL,1, "allocate-owner proc=%p (%d, %s) pmc-owner=%p",
 	    p, p->p_pid, p->p_comm, po);
@@ -2076,7 +2055,7 @@ pmc_destroy_owner_descriptor(struct pmc_owner *po)
 	PMCDBG(OWN,REL,1, "destroy-owner po=%p proc=%p (%d, %s)",
 	    po, po->po_owner, po->po_owner->p_pid, po->po_owner->p_comm);
 
-	mtx_destroy(&po->po_mtx);
+	spin_uninit(&po->po_lock);
 	kfree(po, M_PMC);
 }
 
@@ -2105,7 +2084,7 @@ pmc_find_process_descriptor(struct proc *p, uint32_t mode)
 		ppnew = kmalloc(sizeof(struct pmc_process) + md->pmd_npmc *
 		    sizeof(struct pmc_targetstate), M_PMC, M_WAITOK|M_ZERO);
 
-	mtx_lock_spin(&pmc_processhash_mtx);
+	spin_lock(&pmc_processhash_lock);
 	LIST_FOREACH(pp, pph, pp_next)
 	    if (pp->pp_proc == p)
 		    break;
@@ -2120,7 +2099,7 @@ pmc_find_process_descriptor(struct proc *p, uint32_t mode)
 		pp = ppnew;
 		ppnew = NULL;
 	}
-	mtx_unlock_spin(&pmc_processhash_mtx);
+	spin_unlock(&pmc_processhash_lock);
 
 	if (pp != NULL && ppnew != NULL)
 		kfree(ppnew, M_PMC);
@@ -2139,9 +2118,9 @@ pmc_remove_process_descriptor(struct pmc_process *pp)
 	    ("[pmc,%d] Removing process descriptor %p with count %d",
 		__LINE__, pp, pp->pp_refcnt));
 
-	mtx_lock_spin(&pmc_processhash_mtx);
+	spin_lock(&pmc_processhash_lock);
 	LIST_REMOVE(pp, pp_next);
-	mtx_unlock_spin(&pmc_processhash_mtx);
+	spin_unlock(&pmc_processhash_lock);
 }
 
 
@@ -2187,6 +2166,7 @@ pmc_allocate_pmc_descriptor(void)
 	if (pmc != NULL) {
 		pmc->pm_owner = NULL;
 		LIST_INIT(&pmc->pm_targets);
+		spin_init(&pmc->pm_lock);
 	}
 
 	PMCDBG(PMC,ALL,1, "allocate-pmc -> pmc=%p", pmc);
@@ -2215,6 +2195,7 @@ pmc_destroy_pmc_descriptor(struct pmc *pm)
 	    ("[pmc,%d] pmc has non-zero run count %d", __LINE__,
 		pm->pm_runcount));
 #endif
+	spin_uninit(&pm->pm_lock);
 }
 
 static void
@@ -2266,7 +2247,7 @@ pmc_release_pmc_descriptor(struct pmc *pm)
 	struct pmc_classdep *pcd;
 	struct pmc_target *ptgt, *tmp;
 
-	sx_assert(&pmc_sx, SX_XLOCKED);
+	KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE);
 
 	KASSERT(pm, ("[pmc,%d] null pmc", __LINE__));
 
@@ -2304,16 +2285,12 @@ pmc_release_pmc_descriptor(struct pmc *pm)
 				__LINE__, ri, phw->phw_pmc, pm));
 			PMCDBG(PMC,REL,2, "stopping cpu=%d ri=%d", cpu, ri);
 
-			critical_enter();
 			pcd->pcd_stop_pmc(cpu, adjri);
-			critical_exit();
 		}
 
 		PMCDBG(PMC,REL,2, "decfg cpu=%d ri=%d", cpu, ri);
 
-		critical_enter();
 		pcd->pcd_config_pmc(cpu, adjri, NULL);
-		critical_exit();
 
 		/* adjust the global and process count of SS mode PMCs */
 		if (mode == PMC_MODE_SS && pm->pm_state == PMC_STATE_RUNNING) {
@@ -2414,7 +2391,7 @@ pmc_register_owner(struct proc *p, struct pmc *pmc)
 {
 	struct pmc_owner *po;
 
-	sx_assert(&pmc_sx, SX_XLOCKED);
+	KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE);
 
 	if ((po = pmc_find_owner_descriptor(p)) == NULL)
 		if ((po = pmc_allocate_owner_descriptor(p)) == NULL)
@@ -2426,9 +2403,9 @@ pmc_register_owner(struct proc *p, struct pmc *pmc)
 
 	LIST_INSERT_HEAD(&po->po_pmcs, pmc, pm_next);
 
-	PROC_LOCK(p);
+	lwkt_gettoken(&p->p_token);
 	p->p_flag |= P_HWPMC;
-	PROC_UNLOCK(p);
+	lwkt_reltoken(&p->p_token);
 
 	if (po->po_flags & PMC_PO_OWNS_LOGFILE)
 		pmclog_process_pmcallocate(pmc);
@@ -2517,7 +2494,7 @@ pmc_can_allocate_row(int ri, enum pmc_mode mode)
 {
 	enum pmc_disp	disp;
 
-	sx_assert(&pmc_sx, SX_XLOCKED);
+	KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE);
 
 	PMCDBG(PMC,ALR,1, "can-allocate-row ri=%d mode=%d", ri, mode);
 
@@ -2707,13 +2684,11 @@ pmc_start(struct pmc *pm)
 
 	pm->pm_state = PMC_STATE_RUNNING;
 
-	critical_enter();
 	if ((error = pcd->pcd_write_pmc(cpu, adjri,
 		 PMC_IS_SAMPLING_MODE(mode) ?
 		 pm->pm_sc.pm_reloadcount :
 		 pm->pm_sc.pm_initial)) == 0)
 		error = pcd->pcd_start_pmc(cpu, adjri);
-	critical_exit();
 
 	pmc_restore_cpu_binding(&pb);
 
@@ -2774,10 +2749,8 @@ pmc_stop(struct pmc *pm)
 	ri = PMC_TO_ROWINDEX(pm);
 	pcd = pmc_ri_to_classdep(md, ri, &adjri);
 
-	critical_enter();
 	if ((error = pcd->pcd_stop_pmc(cpu, adjri)) == 0)
 		error = pcd->pcd_read_pmc(cpu, adjri, &pm->pm_sc.pm_initial);
-	critical_exit();
 
 	pmc_restore_cpu_binding(&pb);
 
@@ -2811,22 +2784,21 @@ static const char *pmc_op_to_name[] = {
  */
 
 #define	PMC_GET_SX_XLOCK(...) do {		\
-	sx_xlock(&pmc_sx);			\
+	lockmgr(&pmc_lock, LK_EXCLUSIVE);	\
 	if (pmc_hook == NULL) {			\
-		sx_xunlock(&pmc_sx);		\
+		lockmgr(&pmc_lock, LK_RELEASE);	\
 		return __VA_ARGS__;		\
 	}					\
 } while (0)
 
 #define	PMC_DOWNGRADE_SX() do {			\
-	sx_downgrade(&pmc_sx);			\
-	is_sx_downgraded = 1;			\
+	lockmgr(&pmc_lock, LK_DOWNGRADE);	\
 } while (0)
 
 static int
 pmc_syscall_handler(struct thread *td, void *syscall_args)
 {
-	int error, is_sx_downgraded, is_sx_locked, op;
+	int error, is_sx_locked, op;
 	struct pmc_syscall_args *c;
 	void *arg;
 
@@ -2834,7 +2806,6 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
 
 	DROP_GIANT();
 
-	is_sx_downgraded = 0;
 	is_sx_locked = 1;
 
 	c = (struct pmc_syscall_args *) syscall_args;
@@ -2865,7 +2836,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
 		struct pmc_owner *po;
 		struct pmc_op_configurelog cl;
 
-		sx_assert(&pmc_sx, SX_XLOCKED);
+		KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE);
 
 		if ((error = copyin(arg, &cl, sizeof(cl))) != 0)
 			break;
@@ -2885,7 +2856,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
 		 * de-configure it.
 		 */
 		if (cl.pm_logfd >= 0) {
-			sx_xunlock(&pmc_sx);
+			lockmgr(&pmc_lock, LK_RELEASE);
 			is_sx_locked = 0;
 			error = pmclog_configure_log(md, po, cl.pm_logfd);
 		} else if (po->po_flags & PMC_PO_OWNS_LOGFILE) {
@@ -2915,7 +2886,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
 	{
 		struct pmc_owner *po;
 
-		sx_assert(&pmc_sx, SX_XLOCKED);
+		KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE);
 
 		if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
 			error = EINVAL;
@@ -3098,7 +3069,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
 		struct pmc_op_pmcadmin pma;
 		struct pmc_binding pb;
 
-		sx_assert(&pmc_sx, SX_XLOCKED);
+		KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE);
 
 		KASSERT(td == curthread,
 		    ("[pmc,%d] td != curthread", __LINE__));
@@ -3434,7 +3405,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
 		struct proc *p;
 		struct pmc_op_pmcattach a;
 
-		sx_assert(&pmc_sx, SX_XLOCKED);
+		KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE);
 
 		if ((error = copyin(arg, &a, sizeof(a))) != 0)
 			break;
@@ -3474,7 +3445,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
 		 */
 		if (p->p_flag & P_WEXIT) {
 			error = ESRCH;
-			PROC_UNLOCK(p);	/* pfind() returns a locked process */
+			PRELE(p);	/* pfind() returns a locked process */
 			break;
 		}
 
@@ -3484,7 +3455,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
 		 */
 		error = p_candebug(curthread, p);
 
-		PROC_UNLOCK(p);
+		PRELE(p);	/* pfind() returns a locked process */
 
 		if (error == 0)
 			error = pmc_attach_process(p, pm);
@@ -3527,7 +3498,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
 		if (p->p_flag & P_WEXIT)
 			error = ESRCH;
 
-		PROC_UNLOCK(p);	/* pfind() returns a locked process */
+		PRELE(p);	/* pfind() returns a locked process */
 
 		if (error == 0)
 			error = pmc_detach_process(p, pm);
@@ -3725,7 +3696,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
 			ri = PMC_TO_ROWINDEX(pm);
 			pcd = pmc_ri_to_classdep(md, ri, &adjri);
 
-			mtx_pool_lock_spin(pmc_mtxpool, pm);
+			spin_lock(&pm->pm_lock);
 			cpu = curthread->td_oncpu;
 
 			if (prw.pm_flags & PMC_F_OLDVALUE) {
@@ -3739,7 +3710,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
 			if (prw.pm_flags & PMC_F_NEWVALUE)
 				pm->pm_gv.pm_savedvalue = prw.pm_value;
 
-			mtx_pool_unlock_spin(pmc_mtxpool, pm);
+			spin_unlock(&pm->pm_lock);
 
 		} else { /* System mode PMCs */
 			cpu = PMC_TO_CPU(pm);
@@ -3755,7 +3726,6 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
 			pmc_save_cpu_binding(&pb);
 			pmc_select_cpu(cpu);
 
-			critical_enter();
 			/* save old value */
 			if (prw.pm_flags & PMC_F_OLDVALUE)
 				if ((error = (*pcd->pcd_read_pmc)(cpu, adjri,
@@ -3766,7 +3736,6 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
 				error = (*pcd->pcd_write_pmc)(cpu, adjri,
 				    prw.pm_value);
 		error:
-			critical_exit();
 			pmc_restore_cpu_binding(&pb);
 			if (error)
 				break;
@@ -3833,7 +3802,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
 		struct pmc *pm;
 		struct pmc_op_simple sp;
 
-		sx_assert(&pmc_sx, SX_XLOCKED);
+		KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE);
 
 		if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
 			break;
@@ -3935,12 +3904,8 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
 		break;
 	}
 
-	if (is_sx_locked != 0) {
-		if (is_sx_downgraded)
-			sx_sunlock(&pmc_sx);
-		else
-			sx_xunlock(&pmc_sx);
-	}
+	if (is_sx_locked != 0)
+		lockmgr(&pmc_lock, LK_RELEASE);
 
 	if (error)
 		atomic_add_int(&pmc_stats.pm_syscall_errors, 1);
@@ -4314,9 +4279,9 @@ pmc_process_exit(void *arg __unused, struct proc *p)
 	struct pmc_classdep *pcd;
 	pmc_value_t newvalue, tmp;
 
-	PROC_LOCK(p);
+	lwkt_gettoken(&p->p_token);
 	is_using_hwpmcs = p->p_flag & P_HWPMC;
-	PROC_UNLOCK(p);
+	lwkt_reltoken(&p->p_token);
 
 	/*
 	 * Log a sysexit event to all SS PMC owners.
@@ -4347,8 +4312,6 @@ pmc_process_exit(void *arg __unused, struct proc *p)
 	PMCDBG(PRC,EXT,1, "process-exit proc=%p (%d, %s)", p, p->p_pid,
 	    p->p_comm);
 
-	critical_enter(); /* no preemption */
-
 	cpu = curthread->td_oncpu;
 
 	if ((pp = pmc_find_process_descriptor(p,
@@ -4408,10 +4371,10 @@ pmc_process_exit(void *arg __unused, struct proc *p)
 				tmp = newvalue -
 				    PMC_PCPU_SAVED(cpu,ri);
 
-				mtx_pool_lock_spin(pmc_mtxpool, pm);
+				spin_lock(&pm->pm_lock);
 				pm->pm_gv.pm_savedvalue += tmp;
 				pp->pp_pmcs[ri].pp_pmcval += tmp;
-				mtx_pool_unlock_spin(pmc_mtxpool, pm);
+				spin_unlock(&pm->pm_lock);
 			}
 
 			atomic_subtract_rel_int(&pm->pm_runcount,1);
@@ -4428,8 +4391,6 @@ pmc_process_exit(void *arg __unused, struct proc *p)
 		 */
 		(void) md->pmd_switch_out(pmc_pcpu[cpu], pp);
 
-		critical_exit(); /* ok to be pre-empted now */
-
 		/*
 		 * Unlink this process from the PMCs that are
 		 * targetting it.  This will send a signal to
@@ -4445,10 +4406,7 @@ pmc_process_exit(void *arg __unused, struct proc *p)
 				pmc_unlink_target_process(pm, pp);
 			}
 		kfree(pp, M_PMC);
-
-	} else
-		critical_exit(); /* pp == NULL */
-
+	}
 
 	/*
 	 * If the process owned PMCs, free them up and free up
@@ -4459,7 +4417,7 @@ pmc_process_exit(void *arg __unused, struct proc *p)
 		pmc_destroy_owner_descriptor(po);
 	}
 
-	sx_xunlock(&pmc_sx);
+	lockmgr(&pmc_lock, LK_RELEASE);
 }
 
 /*
@@ -4482,9 +4440,9 @@ pmc_process_fork(void *arg __unused, struct proc *p1, struct proc *newproc,
 
 	(void) flags;		/* unused parameter */
 
-	PROC_LOCK(p1);
+	lwkt_gettoken(&p1->p_token);
 	is_using_hwpmcs = p1->p_flag & P_HWPMC;
-	PROC_UNLOCK(p1);
+	lwkt_reltoken(&p1->p_token);
 
 	/*
 	 * If there are system-wide sampling PMCs active, we need to
@@ -4546,12 +4504,12 @@ pmc_process_fork(void *arg __unused, struct proc *p1, struct proc *newproc,
 	/*
 	 * Now mark the new process as being tracked by this driver.
 	 */
-	PROC_LOCK(newproc);
+	lwkt_gettoken(&newproc->p_token);
 	newproc->p_flag |= P_HWPMC;
-	PROC_UNLOCK(newproc);
+	lwkt_reltoken(&newproc->p_token);
 
  done:
-	sx_xunlock(&pmc_sx);
+	lockmgr(&pmc_lock, LK_RELEASE);
 }
 
 
@@ -4718,16 +4676,11 @@ pmc_initialize(void)
 
 	pmc_processhash = hashinit(pmc_hashsize, M_PMC,
 	    &pmc_processhashmask);
-	mtx_init(&pmc_processhash_mtx, "pmc-process-hash", "pmc-leaf",
-	    MTX_SPIN);
+	spin_init(&pmc_processhash_lock);
 
 	LIST_INIT(&pmc_ss_owners);
 	pmc_ss_count = 0;
 
-	/* allocate a pool of spin mutexes */
-	pmc_mtxpool = mtx_pool_create("pmc-leaf", pmc_mtxpool_size,
-	    MTX_SPIN);
-
 	PMCDBG(MOD,INI,1, "pmc_ownerhash=%p, mask=0x%lx "
 	    "targethash=%p mask=0x%lx", pmc_ownerhash, pmc_ownerhashmask,
 	    pmc_processhash, pmc_processhashmask);
@@ -4784,9 +4737,9 @@ pmc_cleanup(void)
 	pmc_cpumask = 0;
 	pmc_intr = NULL;
 
-	sx_xlock(&pmc_sx);
+	lockmgr(&pmc_lock, LK_EXCLUSIVE);
 	if (pmc_hook == NULL) {	/* being unloaded already */
-		sx_xunlock(&pmc_sx);
+		lockmgr(&pmc_lock, LK_RELEASE);
 		return;
 	}
 
@@ -4810,19 +4763,15 @@ pmc_cleanup(void)
 				    po->po_owner->p_pid,
 				    po->po_owner->p_comm);
 
-				PROC_LOCK(po->po_owner);
+				lwkt_gettoken(&po->po_owner->p_token);
 				psignal(po->po_owner, SIGBUS);
-				PROC_UNLOCK(po->po_owner);
+				lwkt_reltoken(&po->po_owner->p_token);
 
 				pmc_destroy_owner_descriptor(po);
 			}
 		}
 
-	/* reclaim allocated data structures */
-	if (pmc_mtxpool)
-		mtx_pool_destroy(&pmc_mtxpool);
-
-	mtx_destroy(&pmc_processhash_mtx);
+	spin_uninit(&pmc_processhash_lock);
 	if (pmc_processhash) {
 #ifdef	DEBUG
 		struct pmc_process *pp;
@@ -4905,7 +4854,7 @@ pmc_cleanup(void)
 
 	pmclog_shutdown();
 
-	sx_xunlock(&pmc_sx); 	/* we are done */
+	lockmgr(&pmc_lock, LK_RELEASE);	/* we are done */
 }
 
 /*
diff --git a/sys/dev/misc/hwpmc/hwpmc_piv.c b/sys/dev/misc/hwpmc/hwpmc_piv.c
index 6a8c5ae..5f827a1 100644
--- a/sys/dev/misc/hwpmc/hwpmc_piv.c
+++ b/sys/dev/misc/hwpmc/hwpmc_piv.c
@@ -425,7 +425,7 @@ static int p4_system_has_htt;
 struct p4_cpu {
 	struct pmc_hw	pc_p4pmcs[P4_NPMCS];
 	char		pc_escrs[P4_NESCR];
-	struct mtx	pc_mtx;		/* spin lock */
+	struct spinlock	pc_lock;	/* spinlock */
 	uint32_t	pc_intrflag;	/* NMI handler flags */
 	unsigned int	pc_intrlock;	/* NMI handler spin lock */
 	unsigned char	pc_flags[P4_NPMCS]; /* 4 bits each: {cfg,run}count */
@@ -623,7 +623,7 @@ p4_pcpu_init(struct pmc_mdep *md, int cpu)
 	for (n = 0; n < P4_NESCR; n++)
 		*pescr++ = P4_INVALID_PMC_INDEX;
 
-	mtx_init(&p4c->pc_mtx, "p4-pcpu", "pmc-leaf", MTX_SPIN);
+	spin_init(&p4c->pc_lock);
 
 	return (0);
 }
@@ -659,7 +659,7 @@ p4_pcpu_fini(struct pmc_mdep *md, int cpu)
 		wrmsr(P4_CCCR_MSR_FIRST + i,
 		    rdmsr(P4_CCCR_MSR_FIRST + i) & ~P4_CCCR_ENABLE);
 
-	mtx_destroy(&p4c->pc_mtx);
+	spin_uninit(&p4c->pc_lock);
 
 	kfree(p4c, M_PMC);
 
@@ -806,7 +806,7 @@ p4_config_pmc(int cpu, int ri, struct pmc *pm)
 	    (p4_system_has_htt && phw->phw_pmc == pm),
 	    ("[p4,%d] hwpmc not unconfigured before re-config", __LINE__));
 
-	mtx_lock_spin(&pc->pc_mtx);
+	spin_lock(&pc->pc_lock);
 	cfgflags = P4_PCPU_GET_CFGFLAGS(pc,ri);
 
 	KASSERT(cfgflags >= 0 || cfgflags <= 3,
@@ -841,7 +841,7 @@ p4_config_pmc(int cpu, int ri, struct pmc *pm)
 
 	P4_PCPU_SET_CFGFLAGS(pc,ri,cfgflags);
 
-	mtx_unlock_spin(&pc->pc_mtx);
+	spin_unlock(&pc->pc_lock);
 
 	return (0);
 }
@@ -863,9 +863,9 @@ p4_get_config(int cpu, int ri, struct pmc **ppm)
 
 	pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
 
-	mtx_lock_spin(&pc->pc_mtx);
+	spin_lock(&pc->pc_lock);
 	cfgflags = P4_PCPU_GET_CFGFLAGS(pc,ri);
-	mtx_unlock_spin(&pc->pc_mtx);
+	spin_unlock(&pc->pc_lock);
 
 	if (cfgflags & P4_CPU_TO_FLAG(cpu))
 		*ppm = pc->pc_p4pmcs[ri].phw_pmc; /* PMC config'ed on this CPU */
@@ -1207,7 +1207,7 @@ p4_start_pmc(int cpu, int ri)
 	 *
 	 */
 
-	mtx_lock_spin(&pc->pc_mtx);
+	spin_lock(&pc->pc_lock);
 
 	rc = P4_PCPU_GET_RUNCOUNT(pc,ri);
 	KASSERT(rc == 0 || rc == 1,
@@ -1278,7 +1278,7 @@ p4_start_pmc(int cpu, int ri)
 	++rc;
 	P4_PCPU_SET_RUNCOUNT(pc,ri,rc);
 
-	mtx_unlock_spin(&pc->pc_mtx);
+	spin_unlock(&pc->pc_lock);
 
 	PMCDBG(MDP,STA,2,"p4-start cpu=%d rc=%d ri=%d escr=%d "
 	    "escrmsr=0x%x escrvalue=0x%x cccr_config=0x%x v=%jx", cpu, rc,
@@ -1341,7 +1341,7 @@ p4_stop_pmc(int cpu, int ri)
 		escrtbits >>= 2;
 	}
 
-	mtx_lock_spin(&pc->pc_mtx);
+	spin_lock(&pc->pc_lock);
 
 	rc = P4_PCPU_GET_RUNCOUNT(pc,ri);
 
@@ -1380,7 +1380,7 @@ p4_stop_pmc(int cpu, int ri)
 		wrmsr(pd->pm_cccr_msr, cccrvalue);
 	}
 
-	mtx_unlock_spin(&pc->pc_mtx);
+	spin_unlock(&pc->pc_lock);
 
 	PMCDBG(MDP,STO,2, "p4-stop cpu=%d rc=%d ri=%d escrmsr=0x%x "
 	    "escrval=0x%x cccrval=0x%x v=%jx", cpu, rc, ri, escrmsr,
diff --git a/sys/kern/kern_pmc.c b/sys/kern/kern_pmc.c
index 8ce0423..40df170 100644
--- a/sys/kern/kern_pmc.c
+++ b/sys/kern/kern_pmc.c
@@ -68,18 +68,25 @@ volatile int pmc_ss_count;
  * the presence of this kernel module.
  *
  * This requires us to protect 'pmc_hook' with a
- * shared (sx) lock -- thus making the process of calling into PMC(4)
+ * lockmgr lock -- thus making the process of calling into PMC(4)
  * somewhat more expensive than a simple 'if' check and indirect call.
  */
-struct sx pmc_sx;
+struct lock pmc_lock;
 
 static void
-pmc_init_sx(void)
+pmc_init_lock(void *arg)
 {
-	sx_init_flags(&pmc_sx, "pmc-sx", SX_NOWITNESS);
+	lockinit(&pmc_lock, "pmc-lock", 0, 0);
 }
 
-SYSINIT(pmcsx, SI_SUB_LOCK, SI_ORDER_MIDDLE, pmc_init_sx, NULL);
+static void
+pmc_uninit_lock(void *arg)
+{
+	lockuninit(&pmc_lock);
+}
+
+SYSINIT(pmclock, SI_SUB_DRIVERS, SI_ORDER_FIRST, pmc_init_lock, 0);
+SYSUNINIT(pmclock, SI_SUB_DRIVERS, SI_ORDER_ANY, pmc_uninit_lock, 0);
 
 /*
  * Helper functions.
diff --git a/sys/sys/pmc.h b/sys/sys/pmc.h
index c67d925..e1b985f 100644
--- a/sys/sys/pmc.h
+++ b/sys/sys/pmc.h
@@ -567,7 +567,6 @@ struct pmc_op_getmsr {
 #include <machine/frame.h>
 
 #define	PMC_HASH_SIZE				16
-#define	PMC_MTXPOOL_SIZE			32
 #define	PMC_LOG_BUFFER_SIZE			4
 #define	PMC_NLOGBUFFERS				16
 #define	PMC_NSAMPLES				32
@@ -578,9 +577,9 @@ struct pmc_op_getmsr {
 /*
  * Locking keys
  *
- * (b) - pmc_bufferlist_mtx (spin lock)
- * (k) - pmc_kthread_mtx (sleep lock)
- * (o) - po->po_mtx (spin lock)
+ * (b) - pmc_bufferlist_lock (spinlock)
+ * (k) - pmc_kthread_lock (sleep lock)
+ * (o) - po->po_lock (spinlock)
  */
 
 /*
@@ -685,6 +684,7 @@ struct pmc {
 	struct pmc_owner *pm_owner;	/* owner thread state */
 	int		pm_runcount;	/* #cpus currently on */
 	enum pmc_state	pm_state;	/* current PMC state */
+	struct spinlock	pm_lock;
 
 	/*
 	 * The PMC ID field encodes the row-index for the PMC, its
@@ -753,7 +753,7 @@ struct pmc_owner  {
 	LIST_ENTRY(pmc_owner)	po_ssnext;	/* list of SS PMC owners */
 	LIST_HEAD(, pmc)	po_pmcs;	/* owned PMC list */
 	TAILQ_HEAD(, pmclog_buffer) po_logbuffers; /* (o) logbuffer list */
-	struct mtx		po_mtx;		/* spin lock for (o) */
+	struct spinlock		po_lock;	/* spin lock for (o) */
 	struct proc		*po_owner;	/* owner proc */
 	uint32_t		po_flags;	/* (k) flags PMC_PO_* */
 	struct proc		*po_kthread;	/* (k) helper kthread */
diff --git a/sys/sys/pmckern.h b/sys/sys/pmckern.h
index f5bbc97..52aad73 100644
--- a/sys/sys/pmckern.h
+++ b/sys/sys/pmckern.h
@@ -71,8 +71,8 @@ struct pmckern_map_out {
 extern int (*pmc_hook)(struct thread *_td, int _function, void *_arg);
 extern int (*pmc_intr)(int _cpu, struct trapframe *_frame);
 
-/* SX lock protecting the hook */
-extern struct sx pmc_sx;
+/* catch-all lock */
+extern struct lock pmc_lock;
 
 /* Per-cpu flags indicating availability of sampling data */
 extern volatile cpumask_t pmc_cpumask;
@@ -86,19 +86,19 @@ extern const int pmc_kernel_version;
 /* Hook invocation; for use within the kernel */
 #define	PMC_CALL_HOOK(t, cmd, arg)		\
 do {						\
-	sx_slock(&pmc_sx);			\
+	lockmgr(&pmc_lock, LK_SHARED);		\
 	if (pmc_hook != NULL)			\
 		(pmc_hook)((t), (cmd), (arg));	\
-	sx_sunlock(&pmc_sx);			\
+	lockmgr(&pmc_lock, LK_RELEASE);		\
 } while (0)
 
 /* Hook invocation that needs an exclusive lock */
 #define	PMC_CALL_HOOK_X(t, cmd, arg)		\
 do {						\
-	sx_xlock(&pmc_sx);			\
+	lockmgr(&pmc_lock, LK_EXCLUSIVE);	\
 	if (pmc_hook != NULL)			\
 		(pmc_hook)((t), (cmd), (arg));	\
-	sx_xunlock(&pmc_sx);			\
+	lockmgr(&pmc_lock, LK_RELEASE);		\
 } while (0)
 
 /*
-- 
1.7.0.5

