From 91572e06dffc954a666937e3078fc15e76b08e8a Mon Sep 17 00:00:00 2001
From: Sascha Wildner <saw@online.de>
Date: Tue, 2 Sep 2014 19:46:10 +0200
Subject: [PATCH 2/3] vmmemctl VMware Player 6.0.3 build-1895310

---
 sys/conf/files                                     |    2 +
 sys/dev/virtual/vmware/shared/backdoor_def.h       |   27 +-
 sys/dev/virtual/vmware/shared/circList.h           |    2 +-
 sys/dev/virtual/vmware/shared/includeCheck.h       |   13 +-
 sys/dev/virtual/vmware/shared/vm_assert.h          |  210 +--
 sys/dev/virtual/vmware/shared/vm_atomic.h          |  415 ++++--
 sys/dev/virtual/vmware/shared/vm_basic_asm.h       |  342 ++++-
 sys/dev/virtual/vmware/shared/vm_basic_asm_x86.h   |   18 +
 .../virtual/vmware/shared/vm_basic_asm_x86_64.h    |   21 +-
 sys/dev/virtual/vmware/shared/vm_basic_defs.h      |   85 +-
 sys/dev/virtual/vmware/shared/vm_basic_math.h      |   18 +
 sys/dev/virtual/vmware/shared/vm_basic_types.h     |   79 +-
 sys/dev/virtual/vmware/shared/vm_device_version.h  |   39 +-
 sys/dev/virtual/vmware/shared/x86cpuid.h           | 1369 ++++++++++++--------
 sys/dev/virtual/vmware/shared/x86cpuid_asm.h       |    4 +-
 sys/gnu/dev/virtual/vmware/vmmemctl/Makefile       |    2 +-
 .../dev/virtual/vmware/vmmemctl/backdoorGcc64.c    |    2 +-
 .../dev/virtual/vmware/vmmemctl/backdoor_balloon.c |  362 ++++++
 .../dev/virtual/vmware/vmmemctl/backdoor_balloon.h |   14 +-
 sys/gnu/dev/virtual/vmware/vmmemctl/balloonInt.h   |   75 ++
 sys/gnu/dev/virtual/vmware/vmmemctl/balloon_def.h  |  186 ++-
 sys/gnu/dev/virtual/vmware/vmmemctl/kernelStubs.h  |   89 ++
 .../dev/virtual/vmware/vmmemctl/kernelStubsBSD.c   |  259 ++++
 sys/gnu/dev/virtual/vmware/vmmemctl/os.c           |  119 +-
 sys/gnu/dev/virtual/vmware/vmmemctl/os.h           |    9 +-
 sys/gnu/dev/virtual/vmware/vmmemctl/vmballoon.c    |  893 ++++++-------
 sys/gnu/dev/virtual/vmware/vmmemctl/vmballoon.h    |   75 +-
 27 files changed, 3487 insertions(+), 1242 deletions(-)
 create mode 100644 sys/gnu/dev/virtual/vmware/vmmemctl/backdoor_balloon.c
 create mode 100644 sys/gnu/dev/virtual/vmware/vmmemctl/balloonInt.h
 create mode 100644 sys/gnu/dev/virtual/vmware/vmmemctl/kernelStubs.h
 create mode 100644 sys/gnu/dev/virtual/vmware/vmmemctl/kernelStubsBSD.c

diff --git a/sys/conf/files b/sys/conf/files
index 432c176..7c72581 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -877,6 +877,8 @@ dev/virtual/vmware/vmblock/stubs.c	optional vmblock
 dev/virtual/vmware/vmblock/subr.c	optional vmblock
 dev/virtual/vmware/vmblock/vnops.c	optional vmblock
 dev/virtual/vmware/vmblock/vfsops.c	optional vmblock
+gnu/dev/virtual/vmware/vmmemctl/backdoor_balloon.c optional vmmemctl
+gnu/dev/virtual/vmware/vmmemctl/kernelStubsBSD.c optional vmmemctl
 gnu/dev/virtual/vmware/vmmemctl/os.c	optional vmmemctl
 gnu/dev/virtual/vmware/vmmemctl/vmballoon.c optional vmmemctl
 gnu/vfs/ext2fs/ext2_alloc.c		optional ext2fs
diff --git a/sys/dev/virtual/vmware/shared/backdoor_def.h b/sys/dev/virtual/vmware/shared/backdoor_def.h
index 807d2f2..bf7d480 100644
--- a/sys/dev/virtual/vmware/shared/backdoor_def.h
+++ b/sys/dev/virtual/vmware/shared/backdoor_def.h
@@ -139,6 +139,9 @@
 #  define BDOOR_CMD_FAS_GET_TABLE_DATA        1
 #  define BDOOR_CMD_FAS_GET_PLATFORM_NAME     2
 #  define BDOOR_CMD_FAS_GET_PCIE_OSC_MASK     3
+#  define BDOOR_CMD_FAS_GET_APIC_ROUTING      4
+#  define BDOOR_CMD_FAS_GET_TABLE_SKIP        5
+#  define BDOOR_CMD_FAS_GET_SLEEP_ENABLES     6
 #define   BDOOR_CMD_SENDPSHAREHINTS          66
 #define   BDOOR_CMD_ENABLE_USB_MOUSE         67
 #define   BDOOR_CMD_GET_VCPU_INFO            68
@@ -151,7 +154,19 @@
 #define   BDOOR_CMD_FIRMWARE_ERROR           71 /* CPL 0 only. */
 #  define BDOOR_CMD_FE_INSUFFICIENT_MEM       0
 #  define BDOOR_CMD_FE_EXCEPTION              1
-#define   BDOOR_CMD_MAX                      72
+#define   BDOOR_CMD_VMK_INFO                 72
+#define   BDOOR_CMD_EFI_BOOT_CONFIG          73 /* CPL 0 only. */
+#  define BDOOR_CMD_EBC_LEGACYBOOT_ENABLED    0
+#  define BDOOR_CMD_EBC_GET_ORDER             1
+#  define BDOOR_CMD_EBC_SHELL_ACTIVE          2
+#define   BDOOR_CMD_GET_HW_MODEL             74 /* CPL 0 only. */
+#define   BDOOR_CMD_GET_SVGA_CAPABILITIES    75 /* CPL 0 only. */
+#define	  BDOOR_CMD_GET_FORCE_X2APIC         76 /* CPL 0 only  */
+#define   BDOOR_CMD_SET_PCI_HOLE             77 /* CPL 0 only  */
+#define   BDOOR_CMD_GET_PCI_HOLE             78 /* CPL 0 only  */
+#define   BDOOR_CMD_GET_PCI_BAR              79 /* CPL 0 only  */
+#define   BDOOR_CMD_SHOULD_GENERATE_SYSTEMID 80 /* CPL 0 only  */
+#define   BDOOR_CMD_MAX                      81
 
 
 /*
@@ -174,8 +189,9 @@
 /* Task applied to backdoor pshare hints */
 #define BDOOR_PSHARE_HINTS_CMD_SHARE   0
 #define BDOOR_PSHARE_HINTS_CMD_DROP    1
+#define BDOOR_PSHARE_HINTS_CMD_MAX     2
 
-#define BDOOR_PSHARE_HINTS_CMD(ecx)   (((ecx) >> 24) & 0x1)
+#define BDOOR_PSHARE_HINTS_CMD(ecx)   (((ecx) >> 24) & 0xff)
 
 /* Nesting control operations */
 
@@ -184,6 +200,11 @@
 #define NESTING_CONTROL_QUERY             2
 #define NESTING_CONTROL_MAX               2
 
+/* EFI Boot Order options, nibble-sized. */
+#define EFI_BOOT_ORDER_TYPE_EFI           0x0
+#define EFI_BOOT_ORDER_TYPE_LEGACY        0x1
+#define EFI_BOOT_ORDER_TYPE_NONE          0xf
+
 /* High-bandwidth backdoor port. --hpreg */
 
 #define BDOORHB_PORT 0x5659
@@ -206,6 +227,8 @@
 #define IS_BDOOR_PMC(index)  (((index) | 3) == 0x10003)
 #define BDOOR_CMD(ecx)       ((ecx) & 0xffff)
 
+/* Sub commands for BDOOR_CMD_VMK_INFO */
+#define BDOOR_CMD_VMK_INFO_ENTRY   1
 
 #ifdef VMM
 /*
diff --git a/sys/dev/virtual/vmware/shared/circList.h b/sys/dev/virtual/vmware/shared/circList.h
index d6e134b..420f775 100644
--- a/sys/dev/virtual/vmware/shared/circList.h
+++ b/sys/dev/virtual/vmware/shared/circList.h
@@ -82,7 +82,7 @@ typedef struct ListItem {
  * @member: the name of the list struct within the struct.
  */
 #define LIST_CONTAINER(ptr, type, member) \
-   ((type *)((char *)(ptr) - offsetof(type, member)))
+   VMW_CONTAINER_OF(ptr, type, member)
 
 /*
  * delete item from the list
diff --git a/sys/dev/virtual/vmware/shared/includeCheck.h b/sys/dev/virtual/vmware/shared/includeCheck.h
index 6b2ee65..f4cbc9f 100644
--- a/sys/dev/virtual/vmware/shared/includeCheck.h
+++ b/sys/dev/virtual/vmware/shared/includeCheck.h
@@ -44,6 +44,7 @@
  *	INCLUDE_ALLOW_VMK_MODULE
  *      INCLUDE_ALLOW_VMKDRIVERS
  *      INCLUDE_ALLOW_VMIROM
+ *      INCLUDE_ALLOW_MKS
  *
  * Then include this file.
  *
@@ -84,13 +85,14 @@
 #undef INCLUDE_ALLOW_VMCORE
 
 #if defined VMX86_VMX && !defined VMCORE && \
-    !(defined INCLUDE_ALLOW_VMX || defined INCLUDE_ALLOW_USERLEVEL)
+    !defined INCLUDE_ALLOW_VMX && !defined INCLUDE_ALLOW_USERLEVEL && \
+    !defined INCLUDE_ALLOW_MKS
 #error "The surrounding include file is not allowed in the VMX."
 #endif
 #undef INCLUDE_ALLOW_VMX
 
 #if defined USERLEVEL && !defined VMX86_VMX && !defined VMCORE && \
-    !defined INCLUDE_ALLOW_USERLEVEL
+    !defined INCLUDE_ALLOW_USERLEVEL && !defined INCLUDE_ALLOW_MKS
 #error "The surrounding include file is not allowed at userlevel."
 #endif
 #undef INCLUDE_ALLOW_USERLEVEL
@@ -127,3 +129,10 @@
 #error "The surrounding include file is not allowed in vmirom."
 #endif
 #undef INCLUDE_ALLOW_VMIROM
+
+#if defined INCLUDE_ALLOW_MKS && \
+    !(defined LOCALMKS  || defined REMOTEMKS || \
+      defined SERVERMKS || defined CLIENTMKS)
+#error "The surrounding include file is not allowed outside of the MKS."
+#endif
+#undef INCLUDE_ALLOW_MKS
diff --git a/sys/dev/virtual/vmware/shared/vm_assert.h b/sys/dev/virtual/vmware/shared/vm_assert.h
index 6b88c7a..905e40b 100644
--- a/sys/dev/virtual/vmware/shared/vm_assert.h
+++ b/sys/dev/virtual/vmware/shared/vm_assert.h
@@ -56,38 +56,104 @@
 #include "vm_basic_types.h"
 #include "vm_basic_defs.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
 
 /*
- * XXX old file code
+ * Some bits of vmcore are used in VMKernel code and cannot have
+ * the VMKERNEL define due to other header dependencies.
  */
-
-#ifdef FILECODEINT
-#error "Don't define FILECODEINT.  It is obsolete."
+#if defined(VMKERNEL) && !defined(VMKPANIC)
+#define VMKPANIC 1
 #endif
-#ifdef FILECODE
-#error "Don't define FILECODE.  It is obsolete."
+
+/*
+ * Internal macros, functions, and strings
+ *
+ * The monitor wants to save space at call sites, so it has specialized
+ * functions for each situation.  User level wants to save on implementation
+ * so it uses generic functions.
+ */
+
+#if !defined VMM || defined MONITOR_APP // {
+
+#if defined VMKPANIC
+
+// vmkernel Panic() function does not want a trailing newline.
+#define _ASSERT_PANIC(name) \
+           Panic(_##name##Fmt, __FILE__, __LINE__)
+#define _ASSERT_PANIC_BUG(bug, name) \
+           Panic(_##name##Fmt " bugNr=%d", __FILE__, __LINE__, bug)
+#define _ASSERT_PANIC_NORETURN(name) \
+           Panic_NoReturn(_##name##Fmt, __FILE__, __LINE__)
+
+#else /* !VMKPANIC */
+#define _ASSERT_PANIC(name) \
+           Panic(_##name##Fmt "\n", __FILE__, __LINE__)
+#define _ASSERT_PANIC_BUG(bug, name) \
+           Panic(_##name##Fmt " bugNr=%d\n", __FILE__, __LINE__, bug)
+#endif /* VMKPANIC */
+
+#ifdef VMX86_DEVEL
+#   define _ASSERT_WARNING(name) \
+           Warning(_##name##Fmt "\n", __FILE__, __LINE__)
+#else
+#   define _ASSERT_WARNING(name) \
+           Log(_##name##Fmt "\n", __FILE__, __LINE__)
 #endif
+#define AssertUnexpectedFmt _AssertUnexpectedFmt
+#endif // }
+
+
+// these don't have newline so a bug can be tacked on
+#define _AssertPanicFmt            "PANIC %s:%d"
+#define _AssertAssertFmt           "ASSERT %s:%d"
+#define _AssertNotImplementedFmt   "NOT_IMPLEMENTED %s:%d"
+#define _AssertNotReachedFmt       "NOT_REACHED %s:%d"
+#define _AssertMemAllocFmt         "MEM_ALLOC %s:%d"
+#define _AssertNotTestedFmt        "NOT_TESTED %s:%d"
+#define _AssertUnexpectedFmt       "UNEXPECTED %s:%d bugNr=%d\n"
 
 
 /*
  * Panic and log functions
  */
 
-EXTERN void Log(const char *fmt, ...) PRINTF_DECL(1, 2);
-EXTERN void Warning(const char *fmt, ...) PRINTF_DECL(1, 2);
-EXTERN NORETURN void Panic(const char *fmt, ...) PRINTF_DECL(1, 2);
+void Log(const char *fmt, ...) PRINTF_DECL(1, 2);
+void Warning(const char *fmt, ...) PRINTF_DECL(1, 2);
+#if defined VMKPANIC
+void Panic_SaveRegs(void);
+
+#ifdef VMX86_DEBUG
+void Panic_NoSave(const char *fmt, ...) PRINTF_DECL(1, 2);
+#else
+NORETURN void Panic_NoSave(const char *fmt, ...) PRINTF_DECL(1, 2);
+#endif
 
-EXTERN void LogThrottled(uint32 *count, const char *fmt, ...)
+NORETURN void Panic_NoSaveNoReturn(const char *fmt, ...)
+   PRINTF_DECL(1, 2);
+
+#define Panic(fmt...) do { \
+   Panic_SaveRegs(); \
+   Panic_NoSave(fmt); \
+} while(0)
+
+#define Panic_NoReturn(fmt...) do { \
+   Panic_SaveRegs(); \
+   Panic_NoSaveNoReturn(fmt); \
+} while(0)
+
+
+#else
+NORETURN void Panic(const char *fmt, ...) PRINTF_DECL(1, 2);
+#endif
+
+void LogThrottled(uint32 *count, const char *fmt, ...)
             PRINTF_DECL(2, 3);
-EXTERN void WarningThrottled(uint32 *count, const char *fmt, ...)
+void WarningThrottled(uint32 *count, const char *fmt, ...)
             PRINTF_DECL(2, 3);
 
-/* DB family:  messages which are parsed by logfile database system */
-#define WarningDB Warning
-#define LogDB Log
-#define WarningThrottledDB WarningThrottled
-#define LogThrottledDB LogThrottled
-
 
 /*
  * Stress testing: redefine ASSERT_IFNOT() to taste
@@ -138,29 +204,33 @@ EXTERN void WarningThrottled(uint32 *count, const char *fmt, ...)
 #define PANIC()        _ASSERT_PANIC(AssertPanic)
 #define PANIC_BUG(bug) _ASSERT_PANIC_BUG(bug, AssertPanic)
 
+#ifdef VMKPANIC
+#define ASSERT_OR_IN_PANIC(cond) ASSERT((cond) || Panic_IsSystemInPanic())
+#endif
+
 #define ASSERT_NOT_IMPLEMENTED(cond) \
            ASSERT_IFNOT(cond, NOT_IMPLEMENTED())
 #define ASSERT_NOT_IMPLEMENTED_BUG(bug, cond) \
            ASSERT_IFNOT(cond, NOT_IMPLEMENTED_BUG(bug))
 
+#if defined VMKPANIC && defined VMX86_DEBUG
+#define NOT_IMPLEMENTED()        _ASSERT_PANIC_NORETURN(AssertNotImplemented)
+#else
 #define NOT_IMPLEMENTED()        _ASSERT_PANIC(AssertNotImplemented)
+#endif
 #define NOT_IMPLEMENTED_BUG(bug) _ASSERT_PANIC_BUG(bug, AssertNotImplemented)
 
+#if defined VMKPANIC && defined VMX86_DEBUG
+#define NOT_REACHED()            _ASSERT_PANIC_NORETURN(AssertNotReached)
+#else
 #define NOT_REACHED()            _ASSERT_PANIC(AssertNotReached)
+#endif
 #define NOT_REACHED_BUG(bug)     _ASSERT_PANIC_BUG(bug, AssertNotReached)
 
 #define ASSERT_MEM_ALLOC(cond) \
            ASSERT_IFNOT(cond, _ASSERT_PANIC(AssertMemAlloc))
 
 #ifdef VMX86_DEVEL
-   #define ASSERT_LENGTH(real, expected) \
-              ASSERT_IFNOT((real) == (expected), \
-                 Panic(AssertLengthFmt, __FILE__, __LINE__, real, expected))
-#else
-   #define ASSERT_LENGTH(real, expected) ASSERT((real) == (expected))
-#endif
-
-#ifdef VMX86_DEVEL
    #define ASSERT_DEVEL(cond) ASSERT(cond)
 #else
    #define ASSERT_DEVEL(cond) ((void) 0)
@@ -169,8 +239,10 @@ EXTERN void WarningThrottled(uint32 *count, const char *fmt, ...)
 #define ASSERT_NO_INTERRUPTS()  ASSERT(!INTERRUPTS_ENABLED())
 #define ASSERT_HAS_INTERRUPTS() ASSERT(INTERRUPTS_ENABLED())
 
+#define NOT_TESTED() _ASSERT_WARNING(AssertNotTested)
+#define ASSERT_NOT_TESTED(cond) (UNLIKELY(!(cond)) ? NOT_TESTED() : (void)0)
 #define ASSERT_LOG_UNEXPECTED(bug, cond) \
-           (UNLIKELY(!(cond)) ? LOG_UNEXPECTED(bug) : 0)
+           (UNLIKELY(!(cond)) ? LOG_UNEXPECTED(bug) : (void)0)
 #ifdef VMX86_DEVEL
    #define LOG_UNEXPECTED(bug) \
               Warning(AssertUnexpectedFmt, __FILE__, __LINE__, bug)
@@ -179,21 +251,7 @@ EXTERN void WarningThrottled(uint32 *count, const char *fmt, ...)
               Log(AssertUnexpectedFmt, __FILE__, __LINE__, bug)
 #endif
 
-#define ASSERT_NOT_TESTED(cond) (UNLIKELY(!(cond)) ? NOT_TESTED() : 0)
-#ifdef VMX86_DEVEL
-   #define NOT_TESTED() Warning(AssertNotTestedFmt, __FILE__, __LINE__)
-#else
-   #define NOT_TESTED() Log(AssertNotTestedFmt, __FILE__, __LINE__)
-#endif
-
-#define NOT_TESTED_ONCE()                                               \
-   do {                                                                 \
-      static Bool alreadyPrinted = FALSE;                               \
-      if (UNLIKELY(!alreadyPrinted)) {                                  \
-	 alreadyPrinted = TRUE;                                         \
-	 NOT_TESTED();                                                  \
-      }                                                                 \
-   } while (0)
+#define NOT_TESTED_ONCE() DO_ONCE(NOT_TESTED())
 
 #define NOT_TESTED_1024()                                               \
    do {                                                                 \
@@ -202,14 +260,16 @@ EXTERN void WarningThrottled(uint32 *count, const char *fmt, ...)
       count = (count + 1) & 1023;                                       \
    } while (0)
 
-#define LOG_ONCE(_s)                                                    \
-   do {                                                                 \
-      static Bool logged = FALSE;                                       \
-      if (!logged) {                                                    \
-	 Log _s;                                                        \
-         logged = TRUE;                                                 \
-      }                                                                 \
-   } while (0)
+#define LOG_ONCE(_s) DO_ONCE(Log _s)
+
+#ifdef VMX86_DEVEL
+   #define DEPRECATED(_fix) DO_ONCE(                                     \
+                               Warning("%s:%d: %s is DEPRECATED. %s\n",  \
+                                       __FILE__, __LINE__, __FUNCTION__, \
+                                       _fix))
+#else
+   #define DEPRECATED(_fix) do {} while (0)
+#endif
 
 
 /*
@@ -278,12 +338,17 @@ EXTERN void WarningThrottled(uint32 *count, const char *fmt, ...)
  * The implementation uses both enum and typedef because the typedef alone is
  * insufficient; gcc allows arrays to be declared with non-constant expressions
  * (even in typedefs, where it makes no sense).
+ *
+ * NOTE: if GCC ever changes so that it ignores unused types altogether, this
+ * assert might not fire!  We explicitly mark it as unused because GCC 4.8+
+ * uses -Wunused-local-typedefs as part of -Wall, which means the typedef will
+ * generate a warning.
  */
 
 #define ASSERT_ON_COMPILE(e) \
    do { \
       enum { AssertOnCompileMisused = ((e) ? 1 : -1) }; \
-      typedef char AssertOnCompileFailed[AssertOnCompileMisused]; \
+      UNUSED_TYPE(typedef char AssertOnCompileFailed[AssertOnCompileMisused]); \
    } while (0)
 
 
@@ -308,47 +373,8 @@ EXTERN void WarningThrottled(uint32 *count, const char *fmt, ...)
       assertions \
    }
 
-
-/*
- * Internal macros, functions, and strings
- *
- * The monitor wants to save space at call sites, so it has specialized
- * functions for each situation.  User level wants to save on implementation
- * so it uses generic functions.
- */
-
-#if !defined VMM || defined MONITOR_APP // {
-
-#if defined VMKERNEL
-// vmkernel Panic() function does not want a trailing newline.
-#define _ASSERT_PANIC(name) \
-           Panic(_##name##Fmt, __FILE__, __LINE__)
-#define _ASSERT_PANIC_BUG(bug, name) \
-           Panic(_##name##Fmt " bugNr=%d", __FILE__, __LINE__, bug)
-
-#else /* !VMKERNEL */
-#define _ASSERT_PANIC(name) \
-           Panic(_##name##Fmt "\n", __FILE__, __LINE__)
-#define _ASSERT_PANIC_BUG(bug, name) \
-           Panic(_##name##Fmt " bugNr=%d\n", __FILE__, __LINE__, bug)
-#endif /* VMKERNEL */
-
-#define AssertLengthFmt     _AssertLengthFmt
-#define AssertUnexpectedFmt _AssertUnexpectedFmt
-#define AssertNotTestedFmt  _AssertNotTestedFmt
-
-#endif // }
-
-// these don't have newline so a bug can be tacked on
-#define _AssertPanicFmt            "PANIC %s:%d"
-#define _AssertAssertFmt           "ASSERT %s:%d"
-#define _AssertNotImplementedFmt   "NOT_IMPLEMENTED %s:%d"
-#define _AssertNotReachedFmt       "NOT_REACHED %s:%d"
-#define _AssertMemAllocFmt         "MEM_ALLOC %s:%d"
-
-// these are complete formats with newline
-#define _AssertLengthFmt           "LENGTH %s:%d r=%#x e=%#x\n"
-#define _AssertUnexpectedFmt       "UNEXPECTED %s:%d bugNr=%d\n"
-#define _AssertNotTestedFmt        "NOT_TESTED %s:%d\n"
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
 
 #endif /* ifndef _VM_ASSERT_H_ */
diff --git a/sys/dev/virtual/vmware/shared/vm_atomic.h b/sys/dev/virtual/vmware/shared/vm_atomic.h
index 7cdddc9..4a55a7a 100644
--- a/sys/dev/virtual/vmware/shared/vm_atomic.h
+++ b/sys/dev/virtual/vmware/shared/vm_atomic.h
@@ -102,9 +102,10 @@ long  _InterlockedCompareExchange(long volatile*, long, long);
 long  _InterlockedExchangeAdd(long volatile*, long);
 long  _InterlockedDecrement(long volatile*);
 long  _InterlockedIncrement(long volatile*);
+void  _ReadWriteBarrier(void);
 #pragma intrinsic(_InterlockedExchange, _InterlockedCompareExchange)
 #pragma intrinsic(_InterlockedExchangeAdd, _InterlockedDecrement)
-#pragma intrinsic(_InterlockedIncrement)
+#pragma intrinsic(_InterlockedIncrement, _ReadWriteBarrier)
 
 #if defined(VM_X86_64)
 long     _InterlockedAnd(long volatile*, long);
@@ -133,10 +134,21 @@ __int64  _InterlockedCompareExchange64(__int64 volatile*, __int64, __int64);
 #endif
 #endif /* _MSC_VER */
 
-#ifdef __arm__
+#if defined(__arm__) && !defined(FAKE_ATOMIC)
+/*
+ * LDREX without STREX or CLREX may cause problems in environments where the
+ * context switch may not clear the reference monitor - according ARM manual
+ * the reference monitor should be cleared after a context switch, but some
+ * may not like Linux kernel's non-preemptive context switch path. So use of
+ * ARM routines in kernel code may not be safe.
+ */
 #   if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) ||  \
        defined(__ARM_ARCH_7R__)|| defined(__ARM_ARCH_7M__)
 #      define VM_ARM_V7
+#      ifdef __KERNEL__
+#         warning LDREX/STREX may not be safe in linux kernel, since it      \
+          does not issue CLREX on context switch (as of 2011-09-29).
+#      endif
 #   else
 #     error Only ARMv7 extends the synchronization primitives ldrex/strex.   \
             For the lower ARM version, please implement the atomic functions \
@@ -172,7 +184,7 @@ Atomic_VolatileToAtomic64(volatile uint64 *var)
  *      Determine whether an lfence intruction is executed after
  *	every locked instruction.
  *
- *	Certain AMD processes have a bug (see bug 107024) that
+ *	Certain AMD processors have a bug (see bug 107024) that
  *	requires an lfence after every locked instruction.
  *
  *	The global variable AtomicUseFence controls whether lfence
@@ -369,7 +381,11 @@ static INLINE uint32
 Atomic_ReadWrite(Atomic_uint32 *var, // IN
                  uint32 val)         // IN
 {
-#ifdef __GNUC__
+#ifdef FAKE_ATOMIC
+   uint32 retval = var->value;
+   var->value = val;
+   return retval;
+#elif defined(__GNUC__)
 #ifdef VM_ARM_V7
    register volatile uint32 retVal;
    register volatile uint32 res;
@@ -383,7 +399,7 @@ Atomic_ReadWrite(Atomic_uint32 *var, // IN
       "bne 1b"
       : [retVal] "=&r" (retVal), [res] "=&r" (res)
       : [var] "r" (&var->value), [val] "r" (val)
-      : "memory", "cc"
+      : "cc"
    );
 
    dmb();
@@ -442,7 +458,14 @@ Atomic_ReadIfEqualWrite(Atomic_uint32 *var, // IN
                         uint32 oldVal,      // IN
                         uint32 newVal)      // IN
 {
-#ifdef __GNUC__
+#ifdef FAKE_ATOMIC
+   uint32 readVal = var->value;
+
+   if (oldVal == readVal) {
+     var->value = newVal;
+   }
+   return oldVal;
+#elif defined(__GNUC__)
 #ifdef VM_ARM_V7
    register uint32 retVal;
    register uint32 res;
@@ -451,14 +474,14 @@ Atomic_ReadIfEqualWrite(Atomic_uint32 *var, // IN
 
    __asm__ __volatile__(
    "1: ldrex %[retVal], [%[var]] \n\t"
-      "mov %[res], #1 \n\t"
+      "mov %[res], #0 \n\t"
       "teq %[retVal], %[oldVal] \n\t"
       "strexeq %[res], %[newVal], [%[var]] \n\t"
       "teq %[res], #0 \n\t"
       "bne 1b"
       : [retVal] "=&r" (retVal), [res] "=&r" (res)
       : [var] "r" (&var->value), [oldVal] "r" (oldVal), [newVal] "r" (newVal)
-      : "memory", "cc"
+      : "cc"
    );
 
    dmb();
@@ -503,7 +526,7 @@ Atomic_ReadIfEqualWrite(Atomic_uint32 *var, // IN
 #define Atomic_ReadIfEqualWrite32 Atomic_ReadIfEqualWrite
 
 
-#if defined(__x86_64__)
+#if defined(__x86_64__) || defined(VM_ARM_V7)
 /*
  *-----------------------------------------------------------------------------
  *
@@ -526,6 +549,29 @@ Atomic_ReadIfEqualWrite64(Atomic_uint64 *var, // IN
                           uint64 newVal)      // IN
 {
 #if defined(__GNUC__)
+#ifdef VM_ARM_V7
+   register uint64 retVal;
+   register uint32 res;
+
+   dmb();
+
+   __asm__ __volatile__(
+   "1: ldrexd %[retVal], %H[retVal], [%[var]] \n\t"
+      "mov %[res], #0 \n\t"
+      "teq %[retVal], %[oldVal] \n\t"
+      "teqeq %H[retVal], %H[oldVal] \n\t"
+      "strexdeq %[res], %[newVal], %H[newVal], [%[var]] \n\t"
+      "teq %[res], #0 \n\t"
+      "bne 1b"
+      : [retVal] "=&r" (retVal), [res] "=&r" (res)
+      : [var] "r" (&var->value), [oldVal] "r" (oldVal), [newVal] "r" (newVal)
+      : "cc"
+   );
+
+   dmb();
+
+   return retVal;
+#else // VM_ARM_V7 (assume x86*)
    uint64 val;
 
    /* Checked against the AMD manual and GCC --hpreg */
@@ -539,6 +585,7 @@ Atomic_ReadIfEqualWrite64(Atomic_uint64 *var, // IN
    );
    AtomicEpilogue();
    return val;
+#endif //VM_ARM_V7
 #elif defined _MSC_VER
    return _InterlockedCompareExchange64((__int64 *)&var->value,
 					(__int64)newVal,
@@ -570,7 +617,9 @@ static INLINE void
 Atomic_And(Atomic_uint32 *var, // IN
            uint32 val)         // IN
 {
-#ifdef __GNUC__
+#ifdef FAKE_ATOMIC
+   var->value &= val;
+#elif defined(__GNUC__)
 #ifdef VM_ARM_V7
    register volatile uint32 res;
    register volatile uint32 tmp;
@@ -579,13 +628,13 @@ Atomic_And(Atomic_uint32 *var, // IN
 
    __asm__ __volatile__(
    "1: ldrex %[tmp], [%[var]] \n\t"
-      "and %[tmp], %[val] \n\t"
+      "and %[tmp], %[tmp], %[val] \n\t"
       "strex %[res], %[tmp], [%[var]] \n\t"
       "teq %[res], #0 \n\t"
       "bne 1b"
       : [res] "=&r" (res), [tmp] "=&r" (tmp)
       : [var] "r" (&var->value), [val] "r" (val)
-      : "memory", "cc"
+      : "cc"
    );
 
    dmb();
@@ -634,7 +683,9 @@ static INLINE void
 Atomic_Or(Atomic_uint32 *var, // IN
           uint32 val)         // IN
 {
-#ifdef __GNUC__
+#ifdef FAKE_ATOMIC
+   var->value |= val;
+#elif defined(__GNUC__)
 #ifdef VM_ARM_V7
    register volatile uint32 res;
    register volatile uint32 tmp;
@@ -643,13 +694,13 @@ Atomic_Or(Atomic_uint32 *var, // IN
 
    __asm__ __volatile__(
    "1: ldrex %[tmp], [%[var]] \n\t"
-      "orr %[tmp], %[val] \n\t"
+      "orr %[tmp], %[tmp], %[val] \n\t"
       "strex %[res], %[tmp], [%[var]] \n\t"
       "teq %[res], #0 \n\t"
       "bne 1b"
       : [res] "=&r" (res), [tmp] "=&r" (tmp)
       : [var] "r" (&var->value), [val] "r" (val)
-      : "memory", "cc"
+      : "cc"
    );
 
    dmb();
@@ -698,7 +749,9 @@ static INLINE void
 Atomic_Xor(Atomic_uint32 *var, // IN
            uint32 val)         // IN
 {
-#ifdef __GNUC__
+#ifdef FAKE_ATOMIC
+   var->value ^= val;
+#elif defined(__GNUC__)
 #ifdef VM_ARM_V7
    register volatile uint32 res;
    register volatile uint32 tmp;
@@ -707,13 +760,13 @@ Atomic_Xor(Atomic_uint32 *var, // IN
 
    __asm__ __volatile__(
    "1: ldrex %[tmp], [%[var]] \n\t"
-      "eor %[tmp], %[val] \n\t"
+      "eor %[tmp], %[tmp], %[val] \n\t"
       "strex %[res], %[tmp], [%[var]] \n\t"
       "teq %[res], #0 \n\t"
       "bne 1b"
       : [res] "=&r" (res), [tmp] "=&r" (tmp)
       : [var] "r" (&var->value), [val] "r" (val)
-      : "memory", "cc"
+      : "cc"
    );
 
    dmb();
@@ -801,7 +854,9 @@ static INLINE void
 Atomic_Add(Atomic_uint32 *var, // IN
            uint32 val)         // IN
 {
-#ifdef __GNUC__
+#ifdef FAKE_ATOMIC
+   var->value += val;
+#elif defined(__GNUC__)
 #ifdef VM_ARM_V7
    register volatile uint32 res;
    register volatile uint32 tmp;
@@ -810,13 +865,13 @@ Atomic_Add(Atomic_uint32 *var, // IN
 
    __asm__ __volatile__(
    "1: ldrex %[tmp], [%[var]] \n\t"
-      "add %[tmp], %[val] \n\t"
+      "add %[tmp], %[tmp], %[val] \n\t"
       "strex %[res], %[tmp], [%[var]] \n\t"
       "teq %[res], #0 \n\t"
       "bne 1b"
       : [res] "=&r" (res), [tmp] "=&r" (tmp)
       : [var] "r" (&var->value), [val] "r" (val)
-      : "memory", "cc"
+      : "cc"
    );
 
    dmb();
@@ -904,7 +959,9 @@ static INLINE void
 Atomic_Sub(Atomic_uint32 *var, // IN
            uint32 val)         // IN
 {
-#ifdef __GNUC__
+#ifdef FAKE_ATOMIC
+   var->value -= val;
+#elif defined(__GNUC__)
 #ifdef VM_ARM_V7
    register volatile uint32 res;
    register volatile uint32 tmp;
@@ -913,13 +970,13 @@ Atomic_Sub(Atomic_uint32 *var, // IN
 
    __asm__ __volatile__(
       "1: ldrex %[tmp], [%[var]] \n\t"
-      "sub %[tmp], %[val] \n\t"
+      "sub %[tmp], %[tmp], %[val] \n\t"
       "strex %[res], %[tmp], [%[var]] \n\t"
       "teq %[res], #0 \n\t"
       "bne 1b"
       : [res] "=&r" (res), [tmp] "=&r" (tmp)
       : [var] "r" (&var->value), [val] "r" (val)
-      : "memory", "cc"
+      : "cc"
    );
 
    dmb();
@@ -1007,7 +1064,7 @@ static INLINE void
 Atomic_Inc(Atomic_uint32 *var) // IN
 {
 #ifdef __GNUC__
-#ifdef VM_ARM_V7
+#if defined(VM_ARM_V7) || defined(FAKE_ATOMIC)
    Atomic_Add(var, 1);
 #else // VM_ARM_V7
    /* Checked against the Intel manual and GCC --walken */
@@ -1053,7 +1110,7 @@ static INLINE void
 Atomic_Dec(Atomic_uint32 *var) // IN
 {
 #ifdef __GNUC__
-#ifdef VM_ARM_V7
+#if defined(VM_ARM_V7) || defined(FAKE_ATOMIC)
    Atomic_Sub(var, 1);
 #else // VM_ARM_V7
    /* Checked against the Intel manual and GCC --walken */
@@ -1235,22 +1292,27 @@ static INLINE uint32
 Atomic_FetchAndAddUnfenced(Atomic_uint32 *var, // IN
                            uint32 val)         // IN
 {
-#ifdef __GNUC__
+#ifdef FAKE_ATOMIC
+   uint32 res = var->value;
+   var->value = res + val;
+   return res;
+#elif defined(__GNUC__)
 #ifdef VM_ARM_V7
    register volatile uint32 res;
    register volatile uint32 retVal;
+   register volatile uint32 tmp;
 
    dmb();
 
    __asm__ __volatile__(
       "1: ldrex %[retVal], [%[var]] \n\t"
-      "add %[val], %[retVal] \n\t"
-      "strex %[res], %[val], [%[var]] \n\t"
+      "add %[tmp], %[val], %[retVal] \n\t"
+      "strex %[res], %[tmp], [%[var]] \n\t"
       "teq %[res], #0 \n\t"
       "bne 1b"
-      : [res] "=&r" (res), [retVal] "=&r" (retVal)
+      : [tmp] "=&r" (tmp), [res] "=&r" (res), [retVal] "=&r" (retVal)
       : [var] "r" (&var->value), [val] "r" (val)
-      : "memory", "cc"
+      : "cc"
    );
 
    dmb();
@@ -1314,7 +1376,7 @@ static INLINE uint32
 Atomic_FetchAndAdd(Atomic_uint32 *var, // IN
                    uint32 val)         // IN
 {
-#if defined(__GNUC__) && !defined(VM_ARM_V7)
+#if defined(__GNUC__) && !defined(VM_ARM_V7) && !defined(FAKE_ATOMIC)
    val = Atomic_FetchAndAddUnfenced(var, val);
    AtomicEpilogue();
    return val;
@@ -1362,6 +1424,30 @@ Atomic_ReadAdd64(Atomic_uint64 *var, // IN
 #error No compiler defined for Atomic_ReadAdd64
 #endif
 }
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Atomic_ReadSub64 --
+ *
+ *      Atomic read (returned), sub a value, write.
+ *
+ * Results:
+ *      The value of the variable before the operation.
+ *
+ * Side effects:
+ *      None
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE uint64
+Atomic_ReadSub64(Atomic_uint64 *var, // IN
+                 uint64 val)         // IN
+{
+   // Do an sub by an add and a overflow
+   return Atomic_ReadAdd64(var, -val);
+}
 #endif
 
 
@@ -1537,9 +1623,20 @@ Atomic_CMPXCHG64(Atomic_uint64 *var,   // IN/OUT
                  uint64 const *oldVal, // IN
                  uint64 const *newVal) // IN
 {
-#ifdef __GNUC__
-   Bool equal;
+#ifdef FAKE_ATOMIC
+   uint64 readVal = var->value;
+
+   if (*oldVal == readVal) {
+     var->value = *newVal;
+   }
+   return (*oldVal == readVal);
+#elif defined(__GNUC__)
+#if defined(VM_ARM_V7)
+   return (Atomic_ReadIfEqualWrite64(var, *oldVal, *newVal) == *oldVal);
+
+#else // VM_ARM_V7
 
+   Bool equal;
    /* Checked against the Intel manual and GCC --walken */
 #if defined(__x86_64__)
    uint64 dummy;
@@ -1553,7 +1650,7 @@ Atomic_CMPXCHG64(Atomic_uint64 *var,   // IN/OUT
         "2" (*oldVal)
       : "cc"
    );
-#elif !defined(VM_ARM_V7) /* 32-bit version for non-ARM */
+#else /* 32-bit version for non-ARM */
    int dummy1, dummy2;
 #   if defined __PIC__
    /*
@@ -1572,8 +1669,14 @@ Atomic_CMPXCHG64(Atomic_uint64 *var,   // IN/OUT
     *    %ebx. (This happens when compiling the Fusion UI with gcc 4.2.1, Apple
     *    build 5577.)
     *
+    * 3) Using register classes even for other values is problematic, as gcc
+    *    can decide e.g. %ecx == %edi == 0 (as compile-time constants) and
+    *    ends up using one register for two things. Which breaks xchg's ability
+    *    to temporarily put the PIC pointer somewhere else. PR772455
+    *
     * For that reason alone, the __asm__ statement should keep the regions
-    * where it temporarily modifies %ebx as small as possible.
+    * where it temporarily modifies %ebx as small as possible, and should
+    * prefer specific register assignments.
     */
 #      if __GNUC__ < 3 // Part of #188541 - for RHL 6.2 etc.
    __asm__ __volatile__(
@@ -1602,20 +1705,22 @@ Atomic_CMPXCHG64(Atomic_uint64 *var,   // IN/OUT
       "lock; cmpxchg8b (%3)" "\n\t"
       "xchgl %%ebx, %6"      "\n\t"
       "sete %0"
-      :	"=qm,qm" (equal),
-	"=a,a" (dummy1),
-	"=d,d" (dummy2)
+      :	"=qm" (equal),
+	"=a" (dummy1),
+	"=d" (dummy2)
       : /*
          * See the "Rules for __asm__ statements in __PIC__ code" above: %3
          * must use a register class which does not contain %ebx.
          * "a"/"c"/"d" are already used, so we are left with either "S" or "D".
+         *
+         * Note that this assembly uses ALL GP registers (with %esp reserved for
+         * stack, %ebp reserved for frame, %ebx reserved for PIC).
          */
-        "S,D" (var),
-        "1,1" (((S_uint64 const *)oldVal)->lowValue),
-        "2,2" (((S_uint64 const *)oldVal)->highValue),
-        // %6 cannot use "m": 'newVal' is read-only.
-        "r,r" (((S_uint64 const *)newVal)->lowValue),
-        "c,c" (((S_uint64 const *)newVal)->highValue)
+        "S" (var),
+        "1" (((S_uint64 const *)oldVal)->lowValue),
+        "2" (((S_uint64 const *)oldVal)->highValue),
+        "D" (((S_uint64 const *)newVal)->lowValue),
+        "c" (((S_uint64 const *)newVal)->highValue)
       : "cc", "memory"
    );
 #      endif
@@ -1634,27 +1739,7 @@ Atomic_CMPXCHG64(Atomic_uint64 *var,   // IN/OUT
       : "cc"
    );
 #   endif
-#elif defined(VM_ARM_V7)
-   volatile uint64 tmp;
-
-   dmb();
-
-   __asm__ __volatile__(
-      "ldrexd %[tmp], %H[tmp], [%[var]] \n\t"
-      "mov %[equal], #1 \n\t"
-      "teq %[tmp], %[oldVal] \n\t"
-      "teqeq %H[tmp], %H[oldVal] \n\t"
-      "strexdeq  %[equal], %[newVal], %H[newVal], [%[var]]"
-      : [equal] "=&r" (equal), [tmp] "=&r" (tmp)
-      : [var] "r" (&var->value), [oldVal] "r" (*oldVal), [newVal] "r" (*newVal)
-      : "memory", "cc"
-   );
-
-   dmb();
-
-   return !equal;
 #endif
-#ifndef VM_ARM_V7
    AtomicEpilogue();
    return equal;
 #endif // VM_ARM_V7
@@ -1708,27 +1793,18 @@ Atomic_CMPXCHG32(Atomic_uint32 *var,   // IN/OUT
                  uint32 oldVal, // IN
                  uint32 newVal) // IN
 {
-#ifdef __GNUC__
-   Bool equal;
-#ifdef VM_ARM_V7
-   volatile uint64 tmp;
-
-   dmb();
-
-   __asm__ __volatile__(
-      "ldrex %[tmp], [%[var]] \n\t"
-      "mov %[equal], #1 \n\t"
-      "teq %[tmp], %[oldVal] \n\t"
-      "strexeq  %[equal], %[newVal], [%[var]]"
-      : [equal] "=&r" (equal), [tmp] "=&r" (tmp)
-      : [var] "r" (&var->value), [oldVal] "r" (oldVal), [newVal] "r" (newVal)
-      : "memory", "cc"
-   );
-
-   dmb();
+#ifdef FAKE_ATOMIC
+   uint32 readVal = var->value;
 
-   return !equal;
+   if (oldVal == readVal) {
+     var->value = newVal;
+   }
+   return (oldVal == readVal);
+#elif defined(__GNUC__)
+#ifdef VM_ARM_V7
+   return (Atomic_ReadIfEqualWrite(var, oldVal, newVal) == oldVal);
 #else // VM_ARM_V7
+   Bool equal;
    uint32 dummy;
 
    __asm__ __volatile__(
@@ -1769,7 +1845,9 @@ Atomic_CMPXCHG32(Atomic_uint32 *var,   // IN/OUT
 static INLINE uint64
 Atomic_Read64(Atomic_uint64 const *var) // IN
 {
-#if defined(__GNUC__) && defined(__x86_64__)
+#ifdef FAKE_ATOMIC
+   return var->value;
+#elif defined(__GNUC__) && defined(__x86_64__)
    uint64 value;
 
 #ifdef VMM
@@ -2228,15 +2306,162 @@ Atomic_And64(Atomic_uint64 *var, // IN
 
 
 /*
+ *-----------------------------------------------------------------------------
+ *
+ * Atomic_SetBit64 --
+ *
+ *      Atomic read, set bit N, and write.
+ *      Be careful: if bit is in a register (not in an immediate), then it 
+ *      can specify a bit offset above 63 or even a negative offset.
+ *
+ * Results:
+ *      None
+ *
+ * Side effects:
+ *      None
+ *
+ *-----------------------------------------------------------------------------
+ */
+static INLINE void
+Atomic_SetBit64(Atomic_uint64 *var, // IN/OUT
+                uint64 bit)         // IN
+{
+#if defined(__x86_64__)
+#if defined(__GNUC__)
+   __asm__ __volatile__(
+      "lock; bts %1, %0"
+      : "+m" (var->value)
+      : "ri" (bit)
+      : "cc"
+   );
+   AtomicEpilogue();
+#elif defined _MSC_VER
+   uint64 oldVal;
+   uint64 newVal;
+   do {
+      oldVal = var->value;
+      newVal = oldVal | (CONST64U(1) << bit);
+   } while (!Atomic_CMPXCHG64(var, &oldVal, &newVal));
+#else
+#error No compiler defined for Atomic_SetBit64
+#endif
+#else // __x86_64__
+   uint64 oldVal;
+   uint64 newVal;
+   do {
+      oldVal = var->value;
+      newVal = oldVal | (CONST64U(1) << bit);
+   } while (!Atomic_CMPXCHG64(var, &oldVal, &newVal));
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Atomic_ClearBit64 --
+ *
+ *      Atomic read, clear bit N, and write.
+ *      Be careful: if bit is in a register (not in an immediate), then it 
+ *      can specify a bit offset above 63 or even a negative offset.
+ *
+ * Results:
+ *      None
+ *
+ * Side effects:
+ *      None
+ *
+ *-----------------------------------------------------------------------------
+ */
+static INLINE void
+Atomic_ClearBit64(Atomic_uint64 *var, // IN/OUT
+                  uint64 bit)         // IN
+{
+#if defined(__x86_64__)
+#if defined(__GNUC__)
+   __asm__ __volatile__(
+      "lock; btr %1, %0"
+      : "+m" (var->value)
+      : "ri" (bit)
+      : "cc"
+   );
+   AtomicEpilogue();
+#elif defined _MSC_VER
+   uint64 oldVal;
+   uint64 newVal;
+   do {
+      oldVal = var->value;
+      newVal = oldVal & ~(CONST64U(1) << bit);
+   } while (!Atomic_CMPXCHG64(var, &oldVal, &newVal));
+#else
+#error No compiler defined for Atomic_ClearBit64
+#endif
+#else // __x86_64__
+   uint64 oldVal;
+   uint64 newVal;
+   do {
+      oldVal = var->value;
+      newVal = oldVal & ~(CONST64U(1) << bit);
+   } while (!Atomic_CMPXCHG64(var, &oldVal, &newVal));
+#endif
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Atomic_TestBit64 --
+ *
+ *      Read a bit.
+ *      Be careful: if bit is in a register (not in an immediate), then it 
+ *      can specify a bit offset above 63 or even a negative offset.
+ *
+ * Results:
+ *      TRUE if the tested bit was set; else FALSE.
+ *
+ * Side effects:
+ *      None
+ *
+ *-----------------------------------------------------------------------------
+ */
+static INLINE Bool
+Atomic_TestBit64(Atomic_uint64 *var, // IN
+                 uint64 bit)         // IN
+{
+#if defined(__x86_64__)
+#if defined(__GNUC__)
+   Bool out = FALSE;
+   __asm__ __volatile__(
+      "bt %2, %1; setc %0"
+      : "=rm"(out)
+      : "m" (var->value),
+        "rJ" (bit)
+      : "cc"
+   );
+   return out;
+#elif defined _MSC_VER
+   return (var->value & (CONST64U(1) << bit)) != 0;
+#else
+#error No compiler defined for Atomic_TestBit64
+#endif
+#else // __x86_64__
+   return (var->value & (CONST64U(1) << bit)) != 0;
+#endif
+}
+
+/*
  * Template code for the Atomic_<name> type and its operators.
  *
- * The cast argument is an intermedia type cast to make some
+ * The cast argument is an intermediate type cast to make some
  * compilers stop complaining about casting uint32 <-> void *,
  * even though we only do it in the 32-bit case so they are always
  * the same size.  So for val of type uint32, instead of
  * (void *)val, we have (void *)(uintptr_t)val.
  * The specific problem case is the Windows ddk compiler
  * (as used by the SVGA driver).  -- edward
+ *
+ * NOTE: See the comment in vm_assert.h for why we need UNUSED_TYPE in
+ * AtomicAssertOnCompile(), and why we need to be very careful doing so.
  */
 
 #define MAKE_ATOMIC_TYPE(name, size, in, out, cast)                           \
@@ -2250,7 +2475,7 @@ Atomic_And64(Atomic_uint64 *var, // IN
                                       && 8 * sizeof (out) == size             \
                                       && 8 * sizeof (cast) == size            \
                                          ? 1 : -1 };                          \
-      typedef char AssertOnCompileFailed[AssertOnCompileMisused];             \
+      UNUSED_TYPE(typedef char AssertOnCompileFailed[AssertOnCompileMisused]);\
    }                                                                          \
                                                                               \
                                                                               \
@@ -2418,6 +2643,16 @@ MAKE_ATOMIC_TYPE(Ptr, 32, void const *, void *, uintptr_t)
 MAKE_ATOMIC_TYPE(Int, 32, int, int, int)
 
 
+/* Prevent the compiler from re-ordering memory references. */
+#ifdef __GNUC__
+#define ATOMIC_COMPILER_BARRIER()   __asm__ __volatile__ ("": : :"memory")
+#elif defined(_MSC_VER)
+#define ATOMIC_COMPILER_BARRIER()   _ReadWriteBarrier()
+#else
+#error No compiler defined for ATOMIC_COMPILER_BARRIER
+#endif
+
+
 /*
  *-----------------------------------------------------------------------------
  *
@@ -2442,7 +2677,13 @@ static INLINE void
 Atomic_MFence(void)
 {
    Atomic_uint32 fence;
+   ATOMIC_COMPILER_BARRIER();
    Atomic_Xor(&fence, 0x1);
+   ATOMIC_COMPILER_BARRIER();
 }
 
+#ifdef ATOMIC_COMPILER_BARRIER
+#undef ATOMIC_COMPILER_BARRIER
+#endif
+
 #endif // ifndef _ATOMIC_H_
diff --git a/sys/dev/virtual/vmware/shared/vm_basic_asm.h b/sys/dev/virtual/vmware/shared/vm_basic_asm.h
index 3886f07..1e287bb 100644
--- a/sys/dev/virtual/vmware/shared/vm_basic_asm.h
+++ b/sys/dev/virtual/vmware/shared/vm_basic_asm.h
@@ -109,6 +109,9 @@ void _mm_mfence(void);
 void _mm_lfence(void);
 #pragma intrinsic(_mm_mfence, _mm_lfence)
 
+unsigned int __getcallerseflags(void);
+#pragma intrinsic(__getcallerseflags)
+
 #ifdef VM_X86_64
 /*
  * intrinsic functions only supported by x86-64 windows as of 2k3sp1
@@ -128,9 +131,11 @@ unsigned char  _BitScanForward(unsigned long *, unsigned long);
 unsigned char  _BitScanReverse(unsigned long *, unsigned long);
 #pragma intrinsic(_BitScanForward, _BitScanReverse)
 
+unsigned char  _bittest(const long *, long);
 unsigned char  _bittestandset(long *, long);
 unsigned char  _bittestandreset(long *, long);
-#pragma intrinsic(_bittestandset, _bittestandreset)
+unsigned char  _bittestandcomplement(long *, long);
+#pragma intrinsic(_bittest, _bittestandset, _bittestandreset, _bittestandcomplement)
 #ifdef VM_X86_64
 unsigned char  _bittestandset64(__int64 *, __int64);
 unsigned char  _bittestandreset64(__int64 *, __int64);
@@ -197,6 +202,14 @@ __GCC_IN(l, uint32, IN32)
 #define GET_CURRENT_EIP(_eip) \
       __asm__ __volatile("call 0\n\tpopl %0" : "=r" (_eip): );
 
+static INLINE unsigned int
+GetCallerEFlags(void)
+{
+   unsigned long flags;
+   asm volatile("pushf; pop %0" : "=r"(flags));
+   return flags;
+}
+
 #endif // x86*
 
 #elif defined(_MSC_VER) // } {
@@ -243,6 +256,12 @@ OUT32(uint16 port, uint32 value)
 } while (0)
 #endif // VM_X86_64
 
+static INLINE unsigned int
+GetCallerEFlags(void)
+{
+   return __getcallerseflags();
+}
+
 #else // } {
 #error
 #endif // }
@@ -355,36 +374,115 @@ mssb64_0(const uint64 value)
 #define USE_ARCH_X86_CUSTOM
 #endif
 
+/* **********************************************************
+ *  GCC's intrinsics for the lssb and mssb family produce sub-optimal code,
+ *  so we use inline assembly to improve matters.  However, GCC cannot
+ *  propagate constants through inline assembly, so we help GCC out by
+ *  allowing it to use its intrinsics for compile-time constant values.
+ *  Some day, GCC will make better code and these can collapse to intrinsics.
+ *
+ *  For example, in Decoder_AddressSize, inlined into VVT_GetVTInstrInfo:
+ *  __builtin_ffs(a) compiles to:
+ *  mov   $0xffffffff, %esi
+ *  bsf   %eax, %eax
+ *  cmovz %esi, %eax
+ *  sub   $0x1, %eax
+ *  and   $0x7, %eax
+ *
+ *  While the code below compiles to:
+ *  bsf   %eax, %eax
+ *  sub   $0x1, %eax
+ *
+ *  Ideally, GCC should have recognized non-zero input in the first case.
+ *  Other instances of the intrinsic produce code like
+ *  sub $1, %eax; add $1, %eax; clts
+ * **********************************************************
+ */
+
+#if __GNUC__ < 4
+#define FEWER_BUILTINS
+#endif
+
 static INLINE int
 lssb32_0(uint32 value)
 {
 #if defined(USE_ARCH_X86_CUSTOM)
+   if (!__builtin_constant_p(value)) {
    if (UNLIKELY(value == 0)) {
       return -1;
    } else {
       int pos;
-      __asm__ __volatile__("bsfl %1, %0\n" : "=r" (pos) : "rm" (value) : "cc");
+         __asm__ ("bsfl %1, %0\n" : "=r" (pos) : "rm" (value) : "cc");
       return pos;
    }
-#else
-   return __builtin_ffs(value) - 1;
+   }
 #endif
+   return __builtin_ffs(value) - 1;
 }
 
+#ifndef FEWER_BUILTINS
 static INLINE int
 mssb32_0(uint32 value)
 {
+   /*
+    * We must keep the UNLIKELY(...) outside the #if defined ...
+    * because __builtin_clz(0) is undefined according to gcc's
+    * documentation.
+    */
    if (UNLIKELY(value == 0)) {
       return -1;
    } else {
       int pos;
+#if defined(USE_ARCH_X86_CUSTOM)
+      if (!__builtin_constant_p(value)) {
+         __asm__ ("bsrl %1, %0\n" : "=r" (pos) : "rm" (value) : "cc");
+         return pos;
+      }
+#endif
+      pos = 32 - __builtin_clz(value) - 1;
+      return pos;
+   }
+}
 
+static INLINE int
+lssb64_0(const uint64 value)
+{
 #if defined(USE_ARCH_X86_CUSTOM)
-      __asm__ __volatile__("bsrl %1, %0\n" : "=r" (pos) : "rm" (value) : "cc");
+   if (!__builtin_constant_p(value)) {
+      if (UNLIKELY(value == 0)) {
+         return -1;
+      } else {
+         intptr_t pos;
+   #if defined(VM_X86_64)
+         __asm__ ("bsf %1, %0\n" : "=r" (pos) : "rm" (value) : "cc");
 #else
-      pos = 32 - __builtin_clz(value) - 1;
+         /* The coding was chosen to minimize conditionals and operations */
+         pos = lssb32_0((uint32) value);
+         if (pos == -1) {
+            pos = lssb32_0((uint32) (value >> 32));
+            if (pos != -1) {
+               return pos + 32;
+            }
+         }
+   #endif
+         return pos;
+      }
+   }
 #endif
+   return __builtin_ffsll(value) - 1;
+}
+#endif /* !FEWER_BUILTINS */
 
+#if defined(FEWER_BUILTINS)
+/* GCC 3.3.x does not like __bulitin_clz or __builtin_ffsll. */
+static INLINE int
+mssb32_0(uint32 value)
+{
+   if (UNLIKELY(value == 0)) {
+      return -1;
+   } else {
+      int pos;
+      __asm__ __volatile__("bsrl %1, %0\n" : "=r" (pos) : "rm" (value) : "cc");
       return pos;
    }
 }
@@ -392,7 +490,6 @@ mssb32_0(uint32 value)
 static INLINE int
 lssb64_0(const uint64 value)
 {
-#if defined(USE_ARCH_X86_CUSTOM)
    if (UNLIKELY(value == 0)) {
       return -1;
    } else {
@@ -409,14 +506,12 @@ lssb64_0(const uint64 value)
             return pos + 32;
          }
       }
-#endif
-
+   #endif /* VM_X86_64 */
       return pos;
    }
-#else
-   return __builtin_ffsll(value) - 1;
-#endif
 }
+#endif /* FEWER_BUILTINS */
+
 
 static INLINE int
 mssb64_0(const uint64 value)
@@ -428,7 +523,7 @@ mssb64_0(const uint64 value)
 
 #if defined(USE_ARCH_X86_CUSTOM)
 #if defined(VM_X86_64)
-      __asm__ __volatile__("bsr %1, %0\n" : "=r" (pos) : "rm" (value) : "cc");
+      __asm__ ("bsr %1, %0\n" : "=r" (pos) : "rm" (value) : "cc");
 #else
       /* The coding was chosen to minimize conditionals and operations */
       if (value > 0xFFFFFFFFULL) {
@@ -525,8 +620,8 @@ uint16set(void *dst, uint16 val, size_t count)
         );
    return dst;
 #else
-   int dummy0;
-   int dummy1;
+   size_t dummy0;
+   void *dummy1;
 
    __asm__ __volatile__("\t"
                         "cld"            "\n\t"
@@ -555,8 +650,8 @@ uint32set(void *dst, uint32 val, size_t count)
         );
    return dst;
 #else
-   int dummy0;
-   int dummy1;
+   size_t dummy0;
+   void *dummy1;
 
    __asm__ __volatile__("\t"
                         "cld"            "\n\t"
@@ -662,7 +757,8 @@ Bswap16(uint16 v)
 static INLINE uint32
 Bswap32(uint32 v) // IN
 {
-#if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) || defined(__arm__) // {
+#if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) || \
+   (defined(__arm__) && !defined(__ANDROID__))  // {
 #ifdef __arm__
     __asm__("rev %0, %0" : "+r"(v));
     return v;
@@ -720,6 +816,47 @@ Bswap64(uint64 v) // IN
 
 
 /*
+ *----------------------------------------------------------------------
+ *
+ * COMPILER_FORCED_LOAD_AND_MEM_BARRIER --
+ *
+ *        This macro prevents the compiler from re-ordering memory references
+ *        across the barrier. In addition it emits a forced load from the given
+ *        memory reference. The memory reference has to be either 1, 2, 4 or 8
+ *        bytes wide.
+ *        The forced load of a memory reference can be used exploit details of a
+ *        given CPUs memory model. For example x86 CPUs won't reorder stores to
+ *        a memory location x with loads from a memory location x.
+ *        NOTE: It does not generate any fencing instruction, so the CPU is free
+ *              to reorder instructions according to its memory model.
+ *
+ * Results:
+ *        None
+ *
+ * Side Effects:
+ *        None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+#ifdef VM_X86_64
+#ifdef __GNUC__
+
+#define COMPILER_FORCED_LOAD_AND_MEM_BARRIER(_memory_reference)               \
+   do {                                                                       \
+      typeof(_memory_reference) _dummy;                                       \
+                                                                              \
+      asm volatile("mov %1, %0\n\t"                                           \
+                   : "=r" (_dummy) /* Let compiler choose reg for _dummy */   \
+                   : "m" (_memory_reference)                                  \
+                   : "memory");                                               \
+   } while(0)
+
+#endif /* __GNUC__ */
+#endif /* VM_X86_64 */
+
+
+/*
  * PAUSE is a P4 instruction that improves spinlock power+performance;
  * on non-P4 IA32 systems, the encoding is interpreted as a REPZ-NOP.
  * Use volatile to avoid NOP removal.
@@ -866,18 +1003,22 @@ RDTSC_BARRIER(void)
  *-----------------------------------------------------------------------------
  */
 
+#ifdef __arm__
+#define DEBUGBREAK() __asm__("bkpt")
+#else
 #ifdef _MSC_VER
 #define DEBUGBREAK()   __debugbreak()
 #else
 #define DEBUGBREAK()   __asm__ (" int $3 ")
 #endif
+#endif // __arm__
 #endif // defined(__i386__) || defined(__x86_64__) || defined(__arm__)
 
 
 /*
  *-----------------------------------------------------------------------------
  *
- * {Clear,Set}Bit{32,64} --
+ * {Clear,Set,Test}Bit{32,64} --
  *
  *    Sets or clears a specified single bit in the provided variable.
  *    The index input value specifies which bit to modify and is 0-based.
@@ -891,13 +1032,13 @@ RDTSC_BARRIER(void)
  */
 
 static INLINE void
-SetBit32(uint32 *var, unsigned index)
+SetBit32(uint32 *var, uint32 index)
 {
 #ifdef __GNUC__
    __asm__ (
       "bts %1, %0"
       : "+mr" (*var)
-      : "ri" (index)
+      : "rI" (index)
       : "cc"
    );
 #elif defined(_MSC_VER)
@@ -906,13 +1047,13 @@ SetBit32(uint32 *var, unsigned index)
 }
 
 static INLINE void
-ClearBit32(uint32 *var, unsigned index)
+ClearBit32(uint32 *var, uint32 index)
 {
 #ifdef __GNUC__
    __asm__ (
       "btr %1, %0"
       : "+mr" (*var)
-      : "ri" (index)
+      : "rI" (index)
       : "cc"
    );
 #elif defined(_MSC_VER)
@@ -922,13 +1063,13 @@ ClearBit32(uint32 *var, unsigned index)
 
 #if defined(VM_X86_64)
 static INLINE void
-SetBit64(uint64 *var, unsigned index)
+SetBit64(uint64 *var, uint64 index)
 {
 #ifdef __GNUC__
    __asm__ (
       "bts %1, %0"
       : "+mr" (*var)
-      : "ri" (index)
+      : "rJ" (index)
       : "cc"
    );
 #elif defined _MSC_VER
@@ -937,13 +1078,13 @@ SetBit64(uint64 *var, unsigned index)
 }
 
 static INLINE void
-ClearBit64(uint64 *var, unsigned index)
+ClearBit64(uint64 *var, uint64 index)
 {
 #ifdef __GNUC__
    __asm__ (
-      "btr %1, %0"
+      "btrq %1, %0"
       : "+mr" (*var)
-      : "ri" (index)
+      : "rJ" (index)
       : "cc"
    );
 #elif defined _MSC_VER
@@ -952,6 +1093,136 @@ ClearBit64(uint64 *var, unsigned index)
 }
 #endif /* VM_X86_64 */
 
+static INLINE Bool
+TestBit32(const uint32 *var, uint32 index)
+{
+#ifdef __GNUC__
+   Bool bit;
+   __asm__ (
+      "bt %[index], %[var] \n"
+      "setc %[bit]"
+      : [bit] "=qQm" (bit)
+      : [index] "rI" (index), [var] "r" (*var)
+      : "cc"
+   );
+   return bit;
+#else
+   return (*var & (1 << index)) != 0;
+#endif
+}
+
+static INLINE Bool
+TestBit64(const uint64 *var, uint64 index)
+{
+#if defined __GNUC__ && defined VM_X86_64
+   Bool bit;
+   __asm__ (
+      "bt %[index], %[var] \n"
+      "setc %[bit]"
+      : [bit] "=qQm" (bit)
+      : [index] "rJ" (index), [var] "r" (*var)
+      : "cc"
+   );
+   return bit;
+#else
+   return (*var & (CONST64U(1) << index)) != 0;
+#endif
+}
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * {Clear,Set,Complement,Test}BitVector --
+ *
+ *    Sets, clears, complements, or tests a specified single bit in the
+ *    provided array.  The index input value specifies which bit to modify
+ *    and is 0-based.  Bit number can be +-2Gb (+-128MB) relative from 'var'
+ *    variable.
+ *
+ *    All functions return value of the bit before modification was performed.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static INLINE Bool
+SetBitVector(void *var, int32 index)
+{
+#ifdef __GNUC__
+   Bool bit;
+   __asm__ (
+      "bts %2, %1;"
+      "setc %0"
+      : "=qQm" (bit), "+m" (*(volatile uint32 *)var)
+      : "rI" (index)
+      : "memory", "cc"
+   );
+   return bit;
+#elif defined(_MSC_VER)
+   return _bittestandset((long *)var, index) != 0;
+#else
+#error No compiler defined for SetBitVector
+#endif
+}
+
+static INLINE Bool
+ClearBitVector(void *var, int32 index)
+{
+#ifdef __GNUC__
+   Bool bit;
+   __asm__ (
+      "btr %2, %1;"
+      "setc %0"
+      : "=qQm" (bit), "+m" (*(volatile uint32 *)var)
+      : "rI" (index)
+      : "cc"
+   );
+   return bit;
+#elif defined(_MSC_VER)
+   return _bittestandreset((long *)var, index) != 0;
+#else
+#error No compiler defined for ClearBitVector
+#endif
+}
+
+static INLINE Bool
+ComplementBitVector(void *var, int32 index)
+{
+#ifdef __GNUC__
+   Bool bit;
+   __asm__ (
+      "btc %2, %1;"
+      "setc %0"
+      : "=qQm" (bit), "+m" (*(volatile uint32 *)var)
+      : "rI" (index)
+      : "cc"
+   );
+   return bit;
+#elif defined(_MSC_VER)
+   return _bittestandcomplement((long *)var, index) != 0;
+#else
+#error No compiler defined for ComplementBitVector
+#endif
+}
+
+static INLINE Bool
+TestBitVector(const void *var, int32 index)
+{
+#ifdef __GNUC__
+   Bool bit;
+   __asm__ (
+      "bt %2, %1;"
+      "setc %0"
+      : "=qQm" (bit)
+      : "m" (*(const uint32 *)var), "rI" (index)
+      : "cc"
+   );
+   return bit;
+#elif defined _MSC_VER
+   return _bittest((long *)var, index) != 0;
+#else
+#error No compiler defined for TestBitVector
+#endif
+}
 
 /*
  *-----------------------------------------------------------------------------
@@ -1021,7 +1292,21 @@ RoundUpPow2C32(uint32 value)
 static INLINE uint32
 RoundUpPow2Asm32(uint32 value)
 {
+#ifdef __arm__
+   uint32 out = 1;
+   // Note: None Thumb only!
+   //       The value of the argument "value"
+   //       will be affected!
+   __asm__("sub %[in], %[in], #1;"         // r1 = value - 1 . if value == 0 then r1 = 0xFFFFFFFF
+           "clz %[in], %[in];"             // r1 = log2(value - 1) if value != 1
+                                           // if value == 0 then r1 = 0
+                                           // if value == 1 then r1 = 32
+           "mov %[out], %[out], ror %[in]" // out = 2^(32 - r1)
+                                           // if out == 2^32 then out = 1 as it is right rotate
+       : [in]"+r"(value),[out]"+r"(out));
+#else
    uint32 out = 2;
+
    __asm__("lea -1(%[in]), %%ecx;"      // ecx = value - 1.  Preserve original.
            "bsr %%ecx, %%ecx;"          // ecx = log2(value - 1) if value != 1
                                         // if value == 0, then ecx = 31
@@ -1032,6 +1317,7 @@ RoundUpPow2Asm32(uint32 value)
                                         // zf is always unmodified
            "cmovz %[in], %[out]"        // if value == 1 (zf == 1), write 1 to out.
        : [out]"+r"(out) : [in]"r"(value) : "%ecx", "cc");
+#endif
    return out;
 }
 #endif // __GNUC__
diff --git a/sys/dev/virtual/vmware/shared/vm_basic_asm_x86.h b/sys/dev/virtual/vmware/shared/vm_basic_asm_x86.h
index e4849ab..28cc0b5 100644
--- a/sys/dev/virtual/vmware/shared/vm_basic_asm_x86.h
+++ b/sys/dev/virtual/vmware/shared/vm_basic_asm_x86.h
@@ -56,6 +56,24 @@
 #error "x86-64 not supported"
 #endif
 
+/*
+ * XTEST
+ *     Return TRUE if processor is in transaction region.
+ *
+ */
+#if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
+static INLINE Bool
+xtest(void)
+{
+   uint8 al;
+   __asm__ __volatile__(".byte 0x0f, 0x01, 0xd6    # xtest \n"
+                        "setnz %%al\n"
+                        : "=a"(al) : : "cc");
+   return al;
+}
+
+#endif /* __GNUC__ */
+
 
 /*
  * FXSAVE/FXRSTOR
diff --git a/sys/dev/virtual/vmware/shared/vm_basic_asm_x86_64.h b/sys/dev/virtual/vmware/shared/vm_basic_asm_x86_64.h
index b3e377e..7063896 100644
--- a/sys/dev/virtual/vmware/shared/vm_basic_asm_x86_64.h
+++ b/sys/dev/virtual/vmware/shared/vm_basic_asm_x86_64.h
@@ -285,6 +285,23 @@ XRSTOR_AMD_ES0(const void *load, uint64 mask)
 
 #endif /* __GNUC__ */
 
+/*
+ * XTEST
+ *     Return TRUE if processor is in transaction region.
+ *
+ */
+#if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
+static INLINE Bool
+xtest(void)
+{
+   uint8 al;
+   __asm__ __volatile__(".byte 0x0f, 0x01, 0xd6    # xtest \n"
+                        "setnz %%al\n"
+                        : "=a"(al) : : "cc"); 
+   return al;
+}
+
+#endif /* __GNUC__ */
 
 /*
  *-----------------------------------------------------------------------------
@@ -320,7 +337,7 @@ Mul64x6464(uint64 multiplicand,
    uint64 result, dummy;
 
    __asm__("mulq    %3      \n\t"
-           "shrdq   %1, %0  \n\t"
+           "shrdq   %b4, %1, %0  \n\t"
            : "=a" (result),
              "=d" (dummy)
            : "0"  (multiplier),
@@ -378,7 +395,7 @@ Muls64x64s64(int64 multiplicand, int64 multiplier, uint32 shift)
    int64 result, dummy;
 
    __asm__("imulq   %3      \n\t"
-       "shrdq   %1, %0  \n\t"
+       "shrdq   %b4, %1, %0  \n\t"
        : "=a" (result),
          "=d" (dummy)
        : "0"  (multiplier),
diff --git a/sys/dev/virtual/vmware/shared/vm_basic_defs.h b/sys/dev/virtual/vmware/shared/vm_basic_defs.h
index 8bab751..33d6bec 100644
--- a/sys/dev/virtual/vmware/shared/vm_basic_defs.h
+++ b/sys/dev/virtual/vmware/shared/vm_basic_defs.h
@@ -83,6 +83,9 @@
 #endif
 #endif // __APPLE__
 
+#define VMW_CONTAINER_OF(ptr, type, member) \
+   ((type *)((char *)(ptr) - offsetof(type, member)))
+
 #ifndef ARRAYSIZE
 #define ARRAYSIZE(a) (sizeof (a) / sizeof *(a))
 #endif
@@ -112,6 +115,9 @@ Max(int a, int b)
 }
 #endif
 
+#define VMW_CLAMP(x, min, max) \
+   ((x) < (min) ? (min) : ((x) > (max) ? (max) : (x)))
+
 #define ROUNDUP(x,y)		(((x) + (y) - 1) / (y) * (y))
 #define ROUNDDOWN(x,y)		((x) / (y) * (y))
 #define ROUNDUPBITS(x, bits)	(((uintptr_t) (x) + MASK(bits)) & ~MASK(bits))
@@ -130,6 +136,14 @@ Max(int a, int b)
 
 #define MASK(n)		((1 << (n)) - 1)	    /* make an n-bit mask */
 #define MASK64(n)	((CONST64U(1) << (n)) - 1)  /* make an n-bit mask */
+/*
+ * MASKRANGE64 makes a bit vector starting at bit lo and ending at bit hi.  No
+ * checking for lo < hi is done.
+ */
+#define MASKRANGE64(hi, lo)      (MASK64((hi) - (lo) + 1) << (lo))
+
+/* SIGNEXT64 sign extends a n-bit value to 64-bits. */
+#define SIGNEXT64(val, n)       (((int64)(val) << (64 - (n))) >> (64 - (n)))
 
 #define DWORD_ALIGN(x)          ((((x) + 3) >> 2) << 2)
 #define QWORD_ALIGN(x)          ((((x) + 7) >> 3) << 3)
@@ -169,8 +183,10 @@ Max(int a, int b)
  * Wide versions of string constants.
  */
 
+#ifndef WSTR
 #define WSTR_(X)     L ## X
 #define WSTR(X)      WSTR_(X)
+#endif
 
 
 /*
@@ -232,6 +248,14 @@ Max(int a, int b)
 #define PAGES_2_MBYTES(_npages) ((_npages) >> (20 - PAGE_SHIFT))
 #endif
 
+#ifndef GBYTES_2_PAGES
+#define GBYTES_2_PAGES(_nbytes) ((_nbytes) << (30 - PAGE_SHIFT))
+#endif
+
+#ifndef PAGES_2_GBYTES
+#define PAGES_2_GBYTES(_npages) ((_npages) >> (30 - PAGE_SHIFT))
+#endif
+
 #ifndef BYTES_2_MBYTES
 #define BYTES_2_MBYTES(_nbytes) ((_nbytes) >> 20)
 #endif
@@ -381,6 +405,8 @@ GetCallerFrameAddr(void)
 /* We do not have YIELD() as we do not need it yet... */
 #elif defined(_WIN32)
 #      define YIELD()		Sleep(0)
+#elif defined(VMKERNEL)
+/* We don't have a YIELD macro in the vmkernel */
 #else
 #      include <sched.h>        // For sched_yield.  Don't ask.  --Jeremy.
 #      define YIELD()		sched_yield()
@@ -409,10 +435,12 @@ sleep(unsigned int sec)
    Sleep(sec * 1000);
 }
 
-static INLINE void
+static INLINE int
 usleep(unsigned long usec)
 {
    Sleep(CEILING(usec, 1000));
+
+   return 0;
 }
 
 typedef int pid_t;
@@ -510,6 +538,9 @@ typedef int pid_t;
 #if __GLIBC_PREREQ(2, 5) && !defined GLIBC_VERSION_25
 #define GLIBC_VERSION_25
 #endif
+#if __GLIBC_PREREQ(2, 12) && !defined GLIBC_VERSION_212
+#define GLIBC_VERSION_212
+#endif
 #endif
 
 /*
@@ -549,6 +580,22 @@ typedef int pid_t;
 #define LOG_ONLY(x)
 #endif
 
+#ifdef VMX86_BETA
+#define vmx86_beta     1
+#define BETA_ONLY(x)   x
+#else
+#define vmx86_beta     0
+#define BETA_ONLY(x)
+#endif
+
+#ifdef VMX86_RELEASE
+#define vmx86_release   1
+#define RELEASE_ONLY(x) x
+#else
+#define vmx86_release   0
+#define RELEASE_ONLY(x) 
+#endif
+
 #ifdef VMX86_SERVER
 #define vmx86_server 1
 #define SERVER_ONLY(x) x
@@ -559,14 +606,6 @@ typedef int pid_t;
 #define HOSTED_ONLY(x) x
 #endif
 
-#ifdef VMX86_WGS
-#define vmx86_wgs 1
-#define WGS_ONLY(x) x
-#else
-#define vmx86_wgs 0
-#define WGS_ONLY(x)
-#endif
-
 #ifdef VMKERNEL
 #define vmkernel 1
 #define VMKERNEL_ONLY(x) x
@@ -578,14 +617,18 @@ typedef int pid_t;
 #ifdef _WIN32
 #define WIN32_ONLY(x) x
 #define POSIX_ONLY(x)
+#define vmx86_win32 1
 #else
 #define WIN32_ONLY(x)
 #define POSIX_ONLY(x) x
+#define vmx86_win32 0
 #endif
 
 #ifdef __linux__
+#define vmx86_linux 1
 #define LINUX_ONLY(x) x
 #else
+#define vmx86_linux 0
 #define LINUX_ONLY(x)
 #endif
 
@@ -635,7 +678,11 @@ typedef int pid_t;
 #ifdef _WIN32
 #ifndef USES_OLD_WINDDK
 #if defined(VMX86_LOG)
+#ifdef _WIN64
+#define WinDrvPrint(arg, ...) DbgPrintEx(DPFLTR_IHVDRIVER_ID, (ULONG)~0, arg, __VA_ARGS__)
+#else
 #define WinDrvPrint(arg, ...) DbgPrint(arg, __VA_ARGS__)
+#endif
 #define WinDrvEngPrint(arg, ...) EngDbgPrint(arg, __VA_ARGS__)
 #else
 #define WinDrvPrint(arg, ...)
@@ -657,4 +704,24 @@ typedef int pid_t;
 #define NONCLIENTMETRICSINFO_V1_SIZE CCSIZEOF_STRUCT(NONCLIENTMETRICS, \
                                                      lfMessageFont)
 
+/* This is not intended to be thread-safe. */
+#define DO_ONCE(code)                                                   \
+   do {                                                                 \
+      static Bool _doOnceDone = FALSE;                                  \
+      if (UNLIKELY(!_doOnceDone)) {                                     \
+         _doOnceDone = TRUE;                                            \
+         code;                                                          \
+      }                                                                 \
+   } while (0)
+
+/*
+ * Bug 827422 and 838523.
+ */
+
+#if defined __GNUC__ && __GNUC__ >= 4
+#define VISIBILITY_HIDDEN __attribute__((visibility("hidden")))
+#else
+#define VISIBILITY_HIDDEN /* nothing */
+#endif
+
 #endif // ifndef _VM_BASIC_DEFS_H_
diff --git a/sys/dev/virtual/vmware/shared/vm_basic_math.h b/sys/dev/virtual/vmware/shared/vm_basic_math.h
index 78d8d7f..0f12c22 100644
--- a/sys/dev/virtual/vmware/shared/vm_basic_math.h
+++ b/sys/dev/virtual/vmware/shared/vm_basic_math.h
@@ -75,4 +75,22 @@ ExponentialAvg(uint32 avg, uint32 value, uint32 gainNumer, uint32 gainDenom)
    return (term1 + term2) / gainDenom;
 }
 
+static INLINE Bool
+IsPowerOfTwo(uint32 x)
+{
+   /* Does not check for zero. Callers depend on this. */
+   return !(x & (x - 1));
+}
+
+static INLINE uint32
+GetPowerOfTwo(uint32 x)
+{
+   /* Returns next-greatest power-of-two. */
+   uint32 power2 = 1;
+   while (x > power2) {
+      power2 = power2 << 1;
+   }
+   return power2;
+}
+
 #endif // ifndef _VM_BASIC_MATH_H_
diff --git a/sys/dev/virtual/vmware/shared/vm_basic_types.h b/sys/dev/virtual/vmware/shared/vm_basic_types.h
index 06d9c83..8a24801 100644
--- a/sys/dev/virtual/vmware/shared/vm_basic_types.h
+++ b/sys/dev/virtual/vmware/shared/vm_basic_types.h
@@ -417,7 +417,6 @@ typedef uint8 *TCA;  /* Pointer into TC (usually). */
  */
 typedef uint8 Percent;
 #define AsPercent(v)	((Percent)(v))
-#define CHOOSE_PERCENT  AsPercent(101)
 
 
 typedef uintptr_t VA;
@@ -436,6 +435,7 @@ typedef uint64    BA;
 typedef uint32    BPN;
 typedef uint32    PageNum;
 typedef unsigned  MemHandle;
+typedef unsigned int  IoHandle;
 typedef int32     World_ID;
 
 /* !! do not alter the definition of INVALID_WORLD_ID without ensuring
@@ -540,23 +540,26 @@ typedef void * UserVA;
  * for any region other than buserror.
  */
 #define PHYSMEM_MAX_PPN   ((PPN)0xffffffff)
-#define MAX_PPN           ((PPN)0x1fffffff) /* Maximal observable PPN value. */
+#define MAX_PPN           ((PPN)0x3fffffff) /* Maximal observable PPN value. */
 #define INVALID_PPN       ((PPN)0xffffffff)
-#define APIC_DISABLED_PPN ((PPN)0xfffffffe)
+#define APIC_INVALID_PPN  ((PPN)0xfffffffe)
 
-#define INVALID_BPN       ((BPN)0x1fffffff)
+#define INVALID_BPN       ((BPN)0x3fffffff)
 
 #define RESERVED_MPN      ((MPN) 0)
 #define INVALID_MPN       ((MPN)-1)
 #define MEMREF_MPN        ((MPN)-2)
 #define RELEASED_MPN      ((MPN)-3)
-#define MAX_MPN           ((MPN)0x7fffffff)  /* 43 bits of address space. */
+
+/* 0xfffffffc to account for special MPNs defined above. */
+#define MAX_MPN           ((MPN)0xfffffffc)  /* 44 bits of address space. */
 
 #define INVALID_LPN       ((LPN)-1)
 #define INVALID_VPN       ((VPN)-1)
 #define INVALID_LPN64     ((LPN64)-1)
 #define INVALID_PAGENUM   ((PageNum)-1)
 
+#define INVALID_MPN64     ((MPN64)-1)
 
 /*
  * Format modifier for printing VA, LA, and VPN.
@@ -665,11 +668,24 @@ typedef void * UserVA;
  * that gcc will not do inlining.
  */
 
-#if defined(__GNUC__) && defined(VMM)
+#if defined(__GNUC__) && (defined(VMM) || defined (VMKERNEL) || defined (VMKBOOT))
 #define ABSOLUTELY_NOINLINE __attribute__((__noinline__))
 #endif
 
 /*
+ * Used when a function has no effects except the return value and the
+ * return value depends only on the parameters and/or global variables
+ * Such a function can be subject to common subexpression elimination
+ * and loop optimization just as an arithmetic operator would be. 
+ */
+
+#if defined(__GNUC__) && (defined(VMM) || defined (VMKERNEL))
+#define SIDE_EFFECT_FREE __attribute__((__pure__))
+#else
+#define SIDE_EFFECT_FREE
+#endif
+
+/*
  * Attributes placed on function declarations to tell the compiler
  * that the function never returns.
  */
@@ -683,6 +699,21 @@ typedef void * UserVA;
 #endif
 
 /*
+ * Static profiling hints for functions.
+ *    A function can be either hot, cold, or neither.
+ *    It is an error to specify both hot and cold for the same function.
+ *    Note that there is no annotation for "neither."
+ */
+
+#if defined __GNUC__ && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))
+#define HOT __attribute__((hot))
+#define COLD __attribute__((cold))
+#else
+#define HOT
+#define COLD
+#endif
+
+/*
  * Branch prediction hints:
  *     LIKELY(exp)   - Expression exp is likely TRUE.
  *     UNLIKELY(exp) - Expression exp is likely FALSE.
@@ -745,28 +776,23 @@ typedef void * UserVA;
 # endif
 #endif
 
+#ifndef UNUSED_TYPE
+// XXX _Pragma would better but doesn't always work right now.
+#  define UNUSED_TYPE(_parm) UNUSED_PARAM(_parm)
+#endif
+
+#ifndef UNUSED_VARIABLE
+// XXX is there a better way?
+#  define UNUSED_VARIABLE(_var) (void)_var
+#endif
+
 /*
- * REGPARM defaults to REGPARM3; i.e., a request that gcc
- * put the first three arguments in registers.  (It is fine
- * if the function has fewer than three arguments.)  Gcc only.
- * Syntactically, put REGPARM where you'd put INLINE or NORETURN.
- *
- * Note that 64-bit code already puts the first six arguments in
- * registers, so these attributes are only useful for 32-bit code.
+ * gcc can warn us if we're ignoring returns
  */
-
 #if defined(__GNUC__)
-# define REGPARM0 __attribute__((regparm(0)))
-# define REGPARM1 __attribute__((regparm(1)))
-# define REGPARM2 __attribute__((regparm(2)))
-# define REGPARM3 __attribute__((regparm(3)))
-# define REGPARM REGPARM3
+# define MUST_CHECK_RETURN __attribute__((warn_unused_result))
 #else
-# define REGPARM0
-# define REGPARM1
-# define REGPARM2
-# define REGPARM3
-# define REGPARM
+# define MUST_CHECK_RETURN
 #endif
 
 /*
@@ -815,7 +841,7 @@ typedef void * UserVA;
 #   endif
 
 #else
-#   ifndef _SIZE_T
+#   if !defined(_SIZE_T) && !defined(_SIZE_T_DEFINED)
 #      ifdef VM_I386
 #         define _SIZE_T
 #         ifdef VM_X86_64
@@ -831,7 +857,8 @@ typedef void * UserVA;
 
 #   if !defined(FROBOS) && !defined(_SSIZE_T) && !defined(_SSIZE_T_) && \
        !defined(ssize_t) && !defined(__ssize_t_defined) && \
-       !defined(_SSIZE_T_DECLARED) && !defined(_SSIZE_T_DEFINED_)
+       !defined(_SSIZE_T_DECLARED) && !defined(_SSIZE_T_DEFINED) && \
+       !defined(_SSIZE_T_DEFINED_)
 #      ifdef VM_I386
 #         define _SSIZE_T
 #         define __ssize_t_defined
diff --git a/sys/dev/virtual/vmware/shared/vm_device_version.h b/sys/dev/virtual/vmware/shared/vm_device_version.h
index 4398bf7..6a66422 100644
--- a/sys/dev/virtual/vmware/shared/vm_device_version.h
+++ b/sys/dev/virtual/vmware/shared/vm_device_version.h
@@ -32,7 +32,8 @@
 #define VM_DEVICE_VERSION_H
 
 #define INCLUDE_ALLOW_USERLEVEL
-
+#define INCLUDE_ALLOW_DISTRIBUTE
+#define INCLUDE_ALLOW_VMKDRIVERS
 #define INCLUDE_ALLOW_MODULE
 #define INCLUDE_ALLOW_VMKERNEL
 #define INCLUDE_ALLOW_VMCORE
@@ -67,6 +68,7 @@
 #define PCI_VENDOR_ID_VMWARE                    0x15AD
 #define PCI_DEVICE_ID_VMWARE_SVGA2              0x0405
 #define PCI_DEVICE_ID_VMWARE_SVGA               0x0710
+#define PCI_DEVICE_ID_VMWARE_VGA                0x0711
 #define PCI_DEVICE_ID_VMWARE_NET                0x0720
 #define PCI_DEVICE_ID_VMWARE_SCSI               0x0730
 #define PCI_DEVICE_ID_VMWARE_VMCI               0x0740
@@ -75,7 +77,8 @@
 #define PCI_DEVICE_ID_VMWARE_82546EB            0x0760 /* dual port   */
 #define PCI_DEVICE_ID_VMWARE_EHCI               0x0770
 #define PCI_DEVICE_ID_VMWARE_UHCI               0x0774
-#define PCI_DEVICE_ID_VMWARE_XHCI               0x0778
+#define PCI_DEVICE_ID_VMWARE_XHCI_0096          0x0778
+#define PCI_DEVICE_ID_VMWARE_XHCI_0100          0x0779
 #define PCI_DEVICE_ID_VMWARE_1394               0x0780
 #define PCI_DEVICE_ID_VMWARE_BRIDGE             0x0790
 #define PCI_DEVICE_ID_VMWARE_ROOTPORT           0x07A0
@@ -83,6 +86,7 @@
 #define PCI_DEVICE_ID_VMWARE_VMXWIFI            0x07B8
 #define PCI_DEVICE_ID_VMWARE_PVSCSI             0x07C0
 #define PCI_DEVICE_ID_VMWARE_82574              0x07D0
+#define PCI_DEVICE_ID_VMWARE_AHCI               0x07E0
 #define PCI_DEVICE_ID_VMWARE_HDAUDIO_CODEC      0x1975
 #define PCI_DEVICE_ID_VMWARE_HDAUDIO_CONTROLLER 0x1977
 
@@ -97,6 +101,8 @@
 #define PCI_DEVICE_VMI_INTERFACE        0x00
 #define PCI_DEVICE_VMI_REVISION         0x01
 
+#define PCI_DEVICE_ID_VMWARE_DUMMY      0x0809
+
 /* From linux/pci_ids.h:
  *   AMD Lance Ethernet controller
  *   BusLogic SCSI controller
@@ -117,6 +123,7 @@
  *    Intel 82545EM (e1000, server adapter, single port)
  *    Intel 82546EB (e1000, server adapter, dual port)
  *    Intel HECI (as embedded in ich9m)
+ *    Intel XHCI (as embedded in PANTHERPOINT)
  */
 #define PCI_VENDOR_ID_INTEL             0x8086
 #define PCI_DEVICE_ID_INTEL_82439TX     0x7100
@@ -132,6 +139,7 @@
 #define PCI_DEVICE_ID_INTEL_82574       0x10d3
 #define PCI_DEVICE_ID_INTEL_82574_APPLE 0x10f6
 #define PCI_DEVICE_ID_INTEL_HECI        0x2a74
+#define PCI_DEVICE_ID_INTEL_PANTHERPOINT_XHCI 0x1e31
 
 #define E1000E_PCI_DEVICE_ID_CONFIG_STR "e1000e.pci.deviceID"
 #define E1000E_PCI_SUB_VENDOR_ID_CONFIG_STR "e1000e.pci.subVendorID"
@@ -161,6 +169,11 @@
 #define PCI_REVISION_NEC_UPD720200      0x03
 #define PCI_FIRMWARE_NEC_UPD720200      0x3015
 
+#define SATA_ID_SERIAL_STR "00000000000000000001"  /* Must be 20 Bytes */
+#define SATA_ID_FIRMWARE_STR  "00000001"    /* Must be 8 Bytes */
+
+#define AHCI_ATA_MODEL_STR PRODUCT_GENERIC_NAME " Virtual SATA Hard Drive"
+#define AHCI_ATAPI_MODEL_STR PRODUCT_GENERIC_NAME " Virtual SATA CDRW Drive"
 
 /************* Strings for IDE Identity Fields **************************/
 #define VIDE_ID_SERIAL_STR	"00000000000000000001"	/* Must be 20 Bytes */
@@ -189,6 +202,13 @@
 #define SCSI_MAX_CONTROLLERS	 4	  // Need more than 1 for MSCS clustering
 #define	SCSI_MAX_DEVICES	 16	  // BT-958 emulates only 16
 #define PVSCSI_MAX_DEVICES       255      // 255 (including the controller)
+
+/************* SATA implementation limits ********************************/
+#define SATA_MAX_CONTROLLERS   4
+#define SATA_MAX_DEVICES       30
+#define AHCI_MIN_PORTS         1
+#define AHCI_MAX_PORTS SATA_MAX_DEVICES
+
 /*
  * VSCSI_BV_INTS is the number of uint32's needed for a bit vector
  * to cover all scsi devices per target.
@@ -196,7 +216,15 @@
 #define VSCSI_BV_INTS            CEILING(PVSCSI_MAX_DEVICES, 8 * sizeof (uint32))
 #define SCSI_IDE_CHANNEL         SCSI_MAX_CONTROLLERS
 #define SCSI_IDE_HOSTED_CHANNEL  (SCSI_MAX_CONTROLLERS + 1)
-#define SCSI_MAX_CHANNELS        (SCSI_MAX_CONTROLLERS + 2)
+#define SCSI_SATA_CHANNEL_FIRST  (SCSI_IDE_HOSTED_CHANNEL + 1)
+#define SCSI_MAX_CHANNELS        (SCSI_SATA_CHANNEL_FIRST + SATA_MAX_CONTROLLERS)
+
+/************* SCSI-SATA channel IDs********************************/
+#define SATA_ID_TO_SCSI_ID(sataId)    \
+   (SCSI_SATA_CHANNEL_FIRST + (sataId))
+
+#define SCSI_ID_TO_SATA_ID(scsiId)    \
+   ((scsiId) - SCSI_SATA_CHANNEL_FIRST)
 
 /************* Strings for the VESA BIOS Identity Fields *****************/
 #define VBE_OEM_STRING COMPANY_NAME " SVGA"
@@ -213,7 +241,10 @@
 #define MAX_FLOPPY_DRIVES      2
 
 /************* PCI Passthrough implementation limits ********************/
-#define MAX_PCI_PASSTHRU_DEVICES 6
+#define MAX_PCI_PASSTHRU_DEVICES 16
+
+/************* Test device implementation limits ********************/
+#define MAX_PCI_TEST_DEVICES 16
 
 /************* USB implementation limits ********************************/
 #define MAX_USB_DEVICES_PER_HOST_CONTROLLER 127
diff --git a/sys/dev/virtual/vmware/shared/x86cpuid.h b/sys/dev/virtual/vmware/shared/x86cpuid.h
index 6f534e8..2658f84 100644
--- a/sys/dev/virtual/vmware/shared/x86cpuid.h
+++ b/sys/dev/virtual/vmware/shared/x86cpuid.h
@@ -1,30 +1,18 @@
 /*********************************************************
- * Copyright (C) 1998-2008 VMware, Inc. All rights reserved.
+ * Copyright (C) 1998-2012 VMware, Inc. All rights reserved.
  *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
  *
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. Neither the name of VMware Inc. nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission of VMware Inc.
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
  *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  *
  *********************************************************/
 
@@ -47,6 +35,7 @@
 #include "vm_basic_types.h"
 #include "community_source.h"
 #include "x86vendor.h"
+#include "vm_assert.h"
 
 /*
  * The linux kernel's ptrace.h stupidly defines the bare
@@ -101,7 +90,7 @@ CPUIDQuery;
 /*
  * CPUID levels the monitor caches and ones that are not cached, but
  * have fields defined below (short name and actual value).
- *
+ * 
  * The first parameter defines whether the level has its default masks
  * generated from the values in this file.  Any level which is marked
  * as FALSE here *must* have all monitor support types set to NA.  A
@@ -113,8 +102,15 @@ CPUIDQuery;
    CPUIDLEVEL(TRUE,  1,  1)                     \
    CPUIDLEVEL(FALSE, 5,  5)                     \
    CPUIDLEVEL(TRUE,  7,  7)                     \
+   CPUIDLEVEL(FALSE, A,  0xA)                   \
    CPUIDLEVEL(TRUE,  D,  0xD)                   \
    CPUIDLEVEL(FALSE,400, 0x40000000)            \
+   CPUIDLEVEL(FALSE,401, 0x40000001)            \
+   CPUIDLEVEL(FALSE,402, 0x40000002)            \
+   CPUIDLEVEL(FALSE,403, 0x40000003)            \
+   CPUIDLEVEL(FALSE,404, 0x40000004)            \
+   CPUIDLEVEL(FALSE,405, 0x40000005)            \
+   CPUIDLEVEL(FALSE,406, 0x40000006)            \
    CPUIDLEVEL(FALSE,410, 0x40000010)            \
    CPUIDLEVEL(FALSE, 80, 0x80000000)            \
    CPUIDLEVEL(TRUE,  81, 0x80000001)            \
@@ -125,7 +121,6 @@ CPUIDQuery;
 #define CPUID_UNCACHED_LEVELS                   \
    CPUIDLEVEL(FALSE, 4, 4)                      \
    CPUIDLEVEL(FALSE, 6, 6)                      \
-   CPUIDLEVEL(FALSE, A, 0xA)                    \
    CPUIDLEVEL(FALSE, B, 0xB)                    \
    CPUIDLEVEL(FALSE, 85, 0x80000005)            \
    CPUIDLEVEL(FALSE, 86, 0x80000006)            \
@@ -148,8 +143,20 @@ typedef enum {
    CPUID_NUM_CACHED_LEVELS
 } CpuidCachedLevel;
 
+/* Enum to translate between shorthand name and actual CPUID level value. */
+enum {
+#define CPUIDLEVEL(t, s, v) CPUID_LEVEL_VAL_##s = v,
+   CPUID_ALL_LEVELS
+#undef CPUIDLEVEL
+};
+
 
-/* SVM CPUID feature leaf */
+/* Named feature leaves */
+#define CPUID_FEATURE_INFORMATION  0x01
+#define CPUID_PROCESSOR_TOPOLOGY   4
+#define CPUID_MWAIT_FEATURES       5
+#define CPUID_XSAVE_FEATURES       0xd
+#define CPUID_HYPERVISOR_LEVEL_0   0x40000000
 #define CPUID_SVM_FEATURES         0x8000000a
 
 
@@ -186,7 +193,7 @@ typedef enum {
 #define CPUID_VIA_VENDOR_STRING_FIXED   "CentaurHauls"
 
 /*
- * FIELDDEF can be defined to process the CPUID information provided
+ * FIELD can be defined to process the CPUID information provided
  * in the following CPUID_FIELD_DATA macro.  The first parameter is
  * the CPUID level of the feature (must be defined in
  * CPUID_ALL_LEVELS, above.  The second parameter is the CPUID result
@@ -205,12 +212,9 @@ typedef enum {
  * being used by usermode code (TRUE), or just CPL0 kernel code
  * (FALSE).
  *
- * FIELDDEFA takes a ninth parameter: the name used when generating
- * accessor functions in lib/public/cpuidInfoFuncs.h.
- *
- * FLAGDEF/FLAGDEFA is defined identically to fields, but their
- * accessors are more appropriate for 1-bit flags, and compile-time
- * asserts enforce that the size is 1 bit wide.
+ * FLAG is defined identically to FIELD, but its accessors are more
+ * appropriate for 1-bit flags, and compile-time asserts enforce that
+ * the size is 1 bit wide.
  */
 
 
@@ -262,377 +266,407 @@ typedef enum {
    CPUID_NUM_FIELD_SUPPORTEDS
 } CpuidFieldSupported;
 
-
-/*    LEVEL, REG, VENDOR, POS, SIZE, NAME,       MON SUPP, CPL3, [FUNC] */
-#define CPUID_FIELD_DATA_LEVEL_0                                               \
-FIELDDEF(  0, EAX, COMMON,  0, 32, NUMLEVELS,           ANY, FALSE)     \
-FIELDDEF(  0, EBX, COMMON,  0, 32, VENDOR1,             YES, TRUE)      \
-FIELDDEF(  0, ECX, COMMON,  0, 32, VENDOR3,             YES, TRUE)      \
-FIELDDEF(  0, EDX, COMMON,  0, 32, VENDOR2,             YES, TRUE)
-
-/*    LEVEL, REG, VENDOR, POS, SIZE, NAME,       MON SUPP, CPL3, [FUNC] */
-#define CPUID_FIELD_DATA_LEVEL_1                                               \
-FIELDDEFA( 1, EAX, COMMON,  0,  4, STEPPING,            ANY, FALSE, STEPPING)  \
-FIELDDEFA( 1, EAX, COMMON,  4,  4, MODEL,               ANY, FALSE, MODEL)     \
-FIELDDEFA( 1, EAX, COMMON,  8,  4, FAMILY,              YES, FALSE, FAMILY)    \
-FIELDDEF(  1, EAX, COMMON, 12,  2, TYPE,                ANY, FALSE)            \
-FIELDDEFA( 1, EAX, COMMON, 16,  4, EXTMODEL,            ANY, FALSE, EXT_MODEL) \
-FIELDDEFA( 1, EAX, COMMON, 20,  8, EXTFAMILY,           YES, FALSE, EXT_FAMILY) \
-FIELDDEF(  1, EBX, COMMON,  0,  8, BRAND_ID,            ANY, FALSE)            \
-FIELDDEF(  1, EBX, COMMON,  8,  8, CLFL_SIZE,           ANY, FALSE)            \
-FIELDDEFA( 1, EBX, COMMON, 16,  8, LCPU_COUNT,          ANY, FALSE, LCPU_COUNT) \
-FIELDDEFA( 1, EBX, COMMON, 24,  8, APICID,              ANY, FALSE, APICID)    \
-FLAGDEFA(  1, ECX, COMMON, 0,   1, SSE3,                YES, TRUE,  SSE3)      \
-FLAGDEFA(  1, ECX, COMMON, 1,   1, PCLMULQDQ,           YES, TRUE, PCLMULQDQ)  \
-FLAGDEF(   1, ECX, INTEL,  2,   1, NDA2,                NO,  FALSE)            \
-FLAGDEFA(  1, ECX, COMMON, 3,   1, MWAIT,               ANY, FALSE, MWAIT)     \
-FLAGDEFA(  1, ECX, INTEL,  4,   1, DSCPL,               NO,  FALSE, DSCPL)     \
-FLAGDEFA(  1, ECX, INTEL,  5,   1, VMX,                 YES, FALSE, VMX)       \
-FLAGDEF(   1, ECX, VIA,    5,   1, VMX,                 YES, FALSE)            \
-FLAGDEF(   1, ECX, INTEL,  6,   1, SMX,                 NO,  FALSE)            \
-FLAGDEF(   1, ECX, INTEL,  7,   1, EST,                 NO,  FALSE)            \
-FLAGDEF(   1, ECX, INTEL,  8,   1, TM2,                 NO,  FALSE)            \
-FLAGDEFA(  1, ECX, COMMON, 9,   1, SSSE3,               YES, TRUE,  SSSE3)     \
-FLAGDEF(   1, ECX, INTEL,  10,  1, HTCACHE,             NO,  FALSE)            \
-FLAGDEF(   1, ECX, INTEL,  11,  1, NDA11,               NO,  FALSE)            \
-FLAGDEFA(  1, ECX, COMMON, 12,  1, FMA,                 YES, TRUE,  FMA)       \
-FLAGDEFA(  1, ECX, COMMON, 13,  1, CMPX16,              YES, TRUE,  CMPX16)    \
-FLAGDEF(   1, ECX, INTEL,  14,  1, xPPR,                NO,  FALSE)            \
-FLAGDEF(   1, ECX, INTEL,  15,  1, PERF_MSR,            NO,  FALSE)            \
-FLAGDEF(   1, ECX, INTEL,  17,  1, PCID,                NO,  FALSE)            \
-FLAGDEF(   1, ECX, INTEL,  18,  1, DCA,                 NO,  FALSE)            \
-FLAGDEFA(  1, ECX, COMMON, 19,  1, SSE41,               YES, TRUE,  SSE41)     \
-FLAGDEFA(  1, ECX, COMMON, 20,  1, SSE42,               YES, TRUE,  SSE42)     \
-FLAGDEF(   1, ECX, INTEL,  21,  1, X2APIC,              NO,  FALSE)            \
-FLAGDEFA(  1, ECX, INTEL,  22,  1, MOVBE,               YES, TRUE,  MOVBE)     \
-FLAGDEFA(  1, ECX, COMMON, 23,  1, POPCNT,              YES, TRUE,  POPCNT)    \
-FLAGDEFA(  1, ECX, COMMON, 25,  1, AES,                 YES, TRUE, AES)        \
-FLAGDEFA(  1, ECX, COMMON, 26,  1, XSAVE,               YES, FALSE, XSAVE)     \
-FLAGDEFA(  1, ECX, COMMON, 27,  1, OSXSAVE,             ANY, FALSE, OSXSAVE)   \
-FLAGDEFA(  1, ECX, COMMON, 28,  1, AVX,                 YES, FALSE, AVX)       \
-FLAGDEFA(  1, ECX, COMMON, 29,  1, F16,                 YES, TRUE,  F16)       \
-FLAGDEFA(  1, ECX, COMMON, 30,  1, RDRAND,              YES, TRUE,  RDRAND)    \
-FLAGDEFA(  1, ECX, COMMON, 31,  1, HYPERVISOR,          ANY, FALSE, HYPERVISOR)\
-FLAGDEFA(  1, EDX, COMMON, 0,   1, FPU,                 YES, TRUE, FPU)        \
-FLAGDEFA(  1, EDX, COMMON, 1,   1, VME,                 YES, FALSE, VME)       \
-FLAGDEF(   1, EDX, COMMON, 2,   1, DBGE,                YES, FALSE)            \
-FLAGDEF(   1, EDX, COMMON, 3,   1, PGSZE,               YES, FALSE)            \
-FLAGDEFA(  1, EDX, COMMON, 4,   1, TSC,                 YES, TRUE, TSC)        \
-FLAGDEF(   1, EDX, COMMON, 5,   1, MSR,                 YES, FALSE)            \
-FLAGDEFA(  1, EDX, COMMON, 6,   1, PAE,                 YES, FALSE, PAE)       \
-FLAGDEF(   1, EDX, COMMON, 7,   1, MCK,                 YES, FALSE)            \
-FLAGDEF(   1, EDX, COMMON, 8,   1, CPMX,                YES, TRUE)             \
-FLAGDEFA(  1, EDX, COMMON, 9,   1, APIC,                ANY, FALSE, APIC)      \
-FLAGDEFA(  1, EDX, COMMON, 11,  1, SEP,                 YES, TRUE,  SEP)       \
-FLAGDEFA(  1, EDX, COMMON, 12,  1, MTRR,                YES, FALSE, MTRR)      \
-FLAGDEFA(  1, EDX, COMMON, 13,  1, PGE,                 YES, FALSE, PGE)       \
-FLAGDEFA(  1, EDX, COMMON, 14,  1, MCA,                 YES, FALSE, MCA)       \
-FLAGDEFA(  1, EDX, COMMON, 15,  1, CMOV,                YES, TRUE,  CMOV)      \
-FLAGDEFA(  1, EDX, COMMON, 16,  1, PAT,                 YES, FALSE, PAT)       \
-FLAGDEF(   1, EDX, COMMON, 17,  1, 36PG,                YES, FALSE)            \
-FLAGDEF(   1, EDX, INTEL,  18,  1, PSN,                 YES, FALSE)            \
-FLAGDEFA(  1, EDX, COMMON, 19,  1, CLFL,                YES, TRUE,  CLFL)      \
-FLAGDEF(   1, EDX, INTEL,  21,  1, DTES,                YES, FALSE)            \
-FLAGDEF(   1, EDX, INTEL,  22,  1, ACPI,                YES, FALSE)            \
-FLAGDEFA(  1, EDX, COMMON, 23,  1, MMX,                 YES, TRUE,  MMX)       \
-FLAGDEFA(  1, EDX, COMMON, 24,  1, FXSAVE,              YES, TRUE,  FXSAVE)    \
-FLAGDEFA(  1, EDX, COMMON, 25,  1, SSE,                 YES, TRUE,  SSE)       \
-FLAGDEFA(  1, EDX, COMMON, 26,  1, SSE2,                YES, TRUE,  SSE2)      \
-FLAGDEF(   1, EDX, INTEL,  27,  1, SS,                  YES, FALSE)            \
-FLAGDEFA(  1, EDX, COMMON, 28,  1, HT,                  ANY, FALSE, HT)        \
-FLAGDEF(   1, EDX, INTEL,  29,  1, TM,                  NO,  FALSE)            \
-FLAGDEF(   1, EDX, INTEL,  30,  1, IA64,                NO,  FALSE)            \
-FLAGDEF(   1, EDX, INTEL,  31,  1, PBE,                 NO,  FALSE)
-
-/*    LEVEL, REG, VENDOR, POS, SIZE, NAME,       MON SUPP, CPL3, [FUNC] */
-#define CPUID_FIELD_DATA_LEVEL_4                                               \
-FIELDDEF(  4, EAX, INTEL,   0,  5, CACHE_TYPE,          NA,  FALSE)            \
-FIELDDEF(  4, EAX, INTEL,   5,  3, CACHE_LEVEL,         NA,  FALSE)            \
-FLAGDEF(   4, EAX, INTEL,   8,  1, CACHE_SELF_INIT,     NA,  FALSE)            \
-FLAGDEF(   4, EAX, INTEL,   9,  1, CACHE_FULLY_ASSOC,   NA,  FALSE)            \
-FIELDDEF(  4, EAX, INTEL,  14, 12, CACHE_NUMHT_SHARING, NA,  FALSE)            \
-FIELDDEFA( 4, EAX, INTEL,  26,  6, CORE_COUNT,          NA,  FALSE, INTEL_CORE_COUNT)  \
-FIELDDEF(  4, EBX, INTEL,   0, 12, CACHE_LINE,          NA,  FALSE)            \
-FIELDDEF(  4, EBX, INTEL,  12, 10, CACHE_PART,          NA,  FALSE)            \
-FIELDDEF(  4, EBX, INTEL,  22, 10, CACHE_WAYS,          NA,  FALSE)            \
-FIELDDEF(  4, ECX, INTEL,   0, 32, CACHE_SETS,          NA,  FALSE)            \
-FLAGDEF(   4, EDX, INTEL,   0,  1, CACHE_WBINVD_NOT_GUARANTEED, NA,  FALSE)    \
-FLAGDEF(   4, EDX, INTEL,   1,  1, CACHE_IS_INCLUSIVE,  NA,  FALSE)
-
-/*     LEVEL, REG, VENDOR, POS, SIZE, NAME,       MON SUPP, CPL3, [FUNC] */
-#define CPUID_FIELD_DATA_LEVEL_5                                           \
-FIELDDEFA( 5, EAX, COMMON,  0, 16, MWAIT_MIN_SIZE,      NA,  FALSE, MWAIT_MIN_SIZE) \
-FIELDDEFA( 5, EBX, COMMON,  0, 16, MWAIT_MAX_SIZE,      NA,  FALSE, MWAIT_MAX_SIZE) \
-FLAGDEF(   5, ECX, COMMON,  0,  1, MWAIT_EXTENSIONS,    NA,  FALSE) \
-FLAGDEFA(  5, ECX, COMMON,  1,  1, MWAIT_INTR_BREAK,    NA,  FALSE, MWAIT_INTR_BREAK) \
-FIELDDEF(  5, EDX, INTEL,   0,  4, MWAIT_C0_SUBSTATE,   NA,  FALSE) \
-FIELDDEF(  5, EDX, INTEL,   4,  4, MWAIT_C1_SUBSTATE,   NA,  FALSE) \
-FIELDDEF(  5, EDX, INTEL,   8,  4, MWAIT_C2_SUBSTATE,   NA,  FALSE) \
-FIELDDEF(  5, EDX, INTEL,  12,  4, MWAIT_C3_SUBSTATE,   NA,  FALSE) \
-FIELDDEF(  5, EDX, INTEL,  16,  4, MWAIT_C4_SUBSTATE,   NA,  FALSE)
-
-/*    LEVEL, REG, VENDOR, POS, SIZE, NAME,       MON SUPP, CPL3, [FUNC] */
-#define CPUID_FIELD_DATA_LEVEL_6                                               \
-FLAGDEF(   6, EAX, INTEL,   0,  1, THERMAL_SENSOR,      NA,  FALSE)     \
-FLAGDEF(   6, EAX, INTEL,   1,  1, TURBO_MODE,          NA,  FALSE)     \
-FLAGDEF(   6, EAX, INTEL,   2,  1, APIC_INVARIANT,      NA,  FALSE)     \
-FIELDDEF(  6, EBX, INTEL,   0,  4, NUM_INTR_THRESHOLDS, NA,  FALSE)     \
-FLAGDEF(   6, ECX, INTEL,   0,  1, HW_COORD_FEEDBACK,   NA,  FALSE)	\
-FLAGDEF(   6, ECX, INTEL,   3,  1, ENERGY_PERF_BIAS,    NA,  FALSE)
-
-/*    LEVEL, REG, VENDOR, POS, SIZE, NAME,       MON SUPP, CPL3, [FUNC] */
-#define CPUID_FIELD_DATA_LEVEL_7                                               \
-FLAGDEFA(  7, EBX, INTEL,   0,  1, FSGSBASE,            YES, FALSE, FSGSBASE)  \
-FLAGDEF(   7, EBX, INTEL,   7,  1, SMEP,                NO,  FALSE)            \
-FLAGDEF(   7, EBX, INTEL,   9,  1, ENFSTRG,             YES, FALSE)
-
-/*    LEVEL, REG, VENDOR, POS, SIZE, NAME,       MON SUPP, CPL3, [FUNC] */
-#define CPUID_FIELD_DATA_LEVEL_A                                               \
-FIELDDEFA( A, EAX, INTEL,   0,  8, PMC_VERSION,         NA,  FALSE, PMC_VERSION) \
-FIELDDEFA( A, EAX, INTEL,   8,  8, NUM_PMCS,            NA,  FALSE, NUM_PMCS)  \
-FIELDDEF(  A, EAX, INTEL,  16,  8, PMC_BIT_WIDTH,       NA,  FALSE)            \
-FIELDDEFA( A, EAX, INTEL,  24,  8, PMC_EBX_LENGTH,      NA,  FALSE, PMC_EBX_LENGTH) \
-FLAGDEF(   A, EBX, INTEL,   0,  1, PMC_CORE_CYCLE,      NA,  FALSE)            \
-FLAGDEF(   A, EBX, INTEL,   1,  1, PMC_INSTR_RETIRED,   NA,  FALSE)            \
-FLAGDEF(   A, EBX, INTEL,   2,  1, PMC_REF_CYCLES,      NA,  FALSE)            \
-FLAGDEF(   A, EBX, INTEL,   3,  1, PMC_LAST_LVL_CREF,   NA,  FALSE)            \
-FLAGDEF(   A, EBX, INTEL,   4,  1, PMC_LAST_LVL_CMISS,  NA,  FALSE)            \
-FLAGDEF(   A, EBX, INTEL,   5,  1, PMC_BR_INST_RETIRED, NA,  FALSE)            \
-FLAGDEF(   A, EBX, INTEL,   6,  1, PMC_BR_MISS_RETIRED, NA,  FALSE)            \
-FIELDDEFA( A, EDX, INTEL,   0,  5, PMC_FIXED_NUM,       NA,  FALSE, NUM_FIXED_PMCS) \
-FIELDDEF(  A, EDX, INTEL,   5,  8, PMC_FIXED_SIZE,      NA,  FALSE)
-
-/*    LEVEL, REG, VENDOR, POS, SIZE, NAME,       MON SUPP, CPL3, [FUNC] */
-#define CPUID_FIELD_DATA_LEVEL_B                                               \
-FIELDDEF(  B, EAX, INTEL,   0,  5, MASK_WIDTH,          NA,  FALSE)            \
-FIELDDEF(  B, EBX, INTEL,   0, 16, CPUS_SHARING_LEVEL,  NA,  FALSE)            \
-FIELDDEF(  B, ECX, INTEL,   0,  8, LEVEL_NUMBER,        NA,  FALSE)            \
-FIELDDEF(  B, ECX, INTEL,   8,  8, LEVEL_TYPE,          NA,  FALSE)            \
-FIELDDEF(  B, EDX, INTEL,   0, 32, X2APIC_ID,           NA,  FALSE)
-
-/*    LEVEL, REG, VENDOR, POS, SIZE, NAME,       MON SUPP, CPL3, [FUNC] */
-#define CPUID_FIELD_DATA_LEVEL_D                                               \
-FLAGDEFA(  D, EAX, COMMON,  0,  1, XCR0_MASTER_LEGACY_FP, YES, FALSE, XCR0_MASTER_LEGACY_FP) \
-FLAGDEFA(  D, EAX, COMMON,  1,  1, XCR0_MASTER_SSE,     YES, FALSE, XCR0_MASTER_SSE) \
-FLAGDEFA(  D, EAX, COMMON,  2,  1, XCR0_MASTER_YMM_H,   YES, FALSE, XCR0_MASTER_YMM_H) \
-FIELDDEF(  D, EAX, COMMON,  3, 29, XCR0_MASTER_LOWER,   NO,  FALSE)            \
-FIELDDEF(  D, EBX, COMMON,  0, 32, XSAVE_ENABLED_SIZE,  ANY, FALSE)            \
-FIELDDEFA( D, ECX, COMMON,  0, 32, XSAVE_MAX_SIZE,      YES, FALSE, XSAVE_MAX_SIZE) \
-FIELDDEF(  D, EDX, COMMON,  0, 32, XCR0_MASTER_UPPER,   NO,  FALSE)
-
-/*    LEVEL, REG, VENDOR, POS, SIZE, NAME,       MON SUPP, CPL3, [FUNC] */
-#define CPUID_FIELD_DATA_LEVEL_400                                             \
-FIELDDEF(400, EAX, COMMON,  0, 32, NUM_HYP_LEVELS,      NA,  FALSE)            \
-FIELDDEF(400, EBX, COMMON,  0, 32, HYPERVISOR1,         NA,  FALSE)            \
-FIELDDEF(400, ECX, COMMON,  0, 32, HYPERVISOR2,         NA,  FALSE)            \
-FIELDDEF(400, EDX, COMMON,  0, 32, HYPERVISOR3,         NA,  FALSE)
-
-/*    LEVEL, REG, VENDOR, POS, SIZE, NAME,       MON SUPP, CPL3, [FUNC] */
-#define CPUID_FIELD_DATA_LEVEL_410                                             \
-FIELDDEF(410, EAX, COMMON,  0, 32, TSC_HZ,              NA,  FALSE)            \
-FIELDDEF(410, EBX, COMMON,  0, 32, ACPIBUS_HZ,          NA,  FALSE)
-
-/*    LEVEL, REG, VENDOR, POS, SIZE, NAME,       MON SUPP, CPL3, [FUNC] */
-#define CPUID_FIELD_DATA_LEVEL_80                                              \
-FIELDDEF( 80, EAX, COMMON,  0, 32, NUM_EXT_LEVELS,      NA,  FALSE)            \
-FIELDDEF( 80, EBX, AMD,     0, 32, AMD_VENDOR1,         NA,  FALSE)            \
-FIELDDEF( 80, ECX, AMD,     0, 32, AMD_VENDOR3,         NA,  FALSE)            \
-FIELDDEF( 80, EDX, AMD,     0, 32, AMD_VENDOR2,         NA,  FALSE)
-
-/*    LEVEL, REG, VENDOR, POS, SIZE, NAME,       MON SUPP, CPL3, [FUNC] */
-#define CPUID_FIELD_DATA_LEVEL_81                                              \
-FIELDDEF( 81, EAX, INTEL,   0, 32, UNKNOWN81EAX,        ANY, FALSE)            \
-FIELDDEF( 81, EAX, AMD,     0,  4, STEPPING,            ANY, FALSE)            \
-FIELDDEF( 81, EAX, AMD,     4,  4, MODEL,               ANY, FALSE)            \
-FIELDDEF( 81, EAX, AMD,     8,  4, FAMILY,              ANY, FALSE)            \
-FIELDDEF( 81, EAX, AMD,    12,  2, TYPE,                ANY, FALSE)            \
-FIELDDEF( 81, EAX, AMD,    16,  4, EXTMODEL,            ANY, FALSE)            \
-FIELDDEF( 81, EAX, AMD,    20,  8, EXTFAMILY,           ANY, FALSE)            \
-FIELDDEF( 81, EBX, INTEL,   0, 32, UNKNOWN81EBX,        ANY, FALSE)            \
-FIELDDEF( 81, EBX, AMD,     0, 16, BRAND_ID,            ANY, FALSE)            \
-FIELDDEF( 81, EBX, AMD,    16, 16, UNDEF,               ANY, FALSE)            \
-FLAGDEFA( 81, ECX, COMMON,  0,  1, LAHF,                YES, TRUE,  LAHF64)    \
-FLAGDEFA( 81, ECX, AMD,     1,  1, CMPLEGACY,           NO,  FALSE, CMPLEGACY) \
-FLAGDEFA( 81, ECX, AMD,     2,  1, SVM,                 YES, FALSE, SVM)       \
-FLAGDEFA( 81, ECX, AMD,     3,  1, EXTAPICSPC,          YES, FALSE, EXTAPICSPC) \
-FLAGDEFA( 81, ECX, AMD,     4,  1, CR8AVAIL,            NO,  FALSE, CR8AVAIL)  \
-FLAGDEFA( 81, ECX, AMD,     5,  1, ABM,                 YES, TRUE,  ABM)       \
-FLAGDEFA( 81, ECX, AMD,     6,  1, SSE4A,               YES, TRUE,  SSE4A)     \
-FLAGDEF(  81, ECX, AMD,     7,  1, MISALIGNED_SSE,      YES, TRUE)             \
-FLAGDEFA( 81, ECX, AMD,     8,  1, 3DNPREFETCH,         YES, TRUE, 3DNPREFETCH) \
-FLAGDEFA( 81, ECX, AMD,     9,  1, OSVW,                ANY, FALSE, OSVW)      \
-FLAGDEF(  81, ECX, AMD,    10,  1, IBS,                 NO,  FALSE)            \
-FLAGDEFA( 81, ECX, AMD,    11,  1, XOP,                 YES, TRUE,  XOP)       \
-FLAGDEF(  81, ECX, AMD,    12,  1, SKINIT,              NO,  FALSE)            \
-FLAGDEF(  81, ECX, AMD,    13,  1, WATCHDOG,            NO,  FALSE)            \
-FLAGDEF(  81, ECX, AMD,    15,  1, LWP,                 NO,  FALSE)            \
-FLAGDEFA( 81, ECX, AMD,    16,  1, FMA4,                YES, TRUE,  FMA4)      \
-FLAGDEF(  81, ECX, AMD,    19,  1, NODEID,              NO,  FALSE)            \
-FLAGDEF(  81, ECX, AMD,    22,  1, TOPOLOGY,            NO,  FALSE)            \
-FLAGDEF(  81, ECX, AMD,    23,  1, PERFCTR_EXT_NB,      NO,  FALSE)            \
-FLAGDEF(  81, ECX, AMD,    24,  1, PREFCTR_EXT_CORE,    NO,  FALSE)            \
-FLAGDEF(  81, EDX, AMD,     0,  1, FPU,                 YES, TRUE)             \
-FLAGDEF(  81, EDX, AMD,     1,  1, VME,                 YES, FALSE)            \
-FLAGDEF(  81, EDX, AMD,     2,  1, DBGE,                YES, FALSE)            \
-FLAGDEF(  81, EDX, AMD,     3,  1, PGSZE,               YES, FALSE)            \
-FLAGDEF(  81, EDX, AMD,     4,  1, TSC,                 YES, TRUE)             \
-FLAGDEF(  81, EDX, AMD,     5,  1, MSR,                 YES, FALSE)            \
-FLAGDEF(  81, EDX, AMD,     6,  1, PAE,                 YES, FALSE)            \
-FLAGDEF(  81, EDX, AMD,     7,  1, MCK,                 YES, FALSE)            \
-FLAGDEF(  81, EDX, AMD,     8,  1, CPMX,                YES, TRUE)             \
-FLAGDEF(  81, EDX, AMD,     9,  1, APIC,                ANY, FALSE)            \
-FLAGDEFA( 81, EDX, COMMON, 11,  1, SYSC,                ANY, TRUE, SYSC)       \
-FLAGDEF(  81, EDX, AMD,    12,  1, MTRR,                YES, FALSE)            \
-FLAGDEF(  81, EDX, AMD,    13,  1, PGE,                 YES, FALSE)            \
-FLAGDEF(  81, EDX, AMD,    14,  1, MCA,                 YES, FALSE)            \
-FLAGDEF(  81, EDX, AMD,    15,  1, CMOV,                YES, TRUE)             \
-FLAGDEF(  81, EDX, AMD,    16,  1, PAT,                 YES, FALSE)            \
-FLAGDEF(  81, EDX, AMD,    17,  1, 36PG,                YES, FALSE)            \
-FLAGDEFA( 81, EDX, COMMON, 20,  1, NX,                  YES, FALSE, NX)        \
-FLAGDEFA( 81, EDX, AMD,    22,  1, MMXEXT,              YES, TRUE,  MMXEXT)    \
-FLAGDEF(  81, EDX, AMD,    23,  1, MMX,                 YES, TRUE)             \
-FLAGDEF(  81, EDX, AMD,    24,  1, FXSAVE,              YES, TRUE)             \
-FLAGDEFA( 81, EDX, AMD,    25,  1, FFXSR,               YES, FALSE, FFXSR)     \
-FLAGDEF(  81, EDX, COMMON, 26,  1, PDPE1GB,             NO,  FALSE)            \
-FLAGDEFA( 81, EDX, COMMON, 27,  1, RDTSCP,              YES, TRUE,  RDTSCP)    \
-FLAGDEFA( 81, EDX, COMMON, 29,  1, LM,                  YES, FALSE, LM) \
-FLAGDEFA( 81, EDX, AMD,    30,  1, 3DNOWPLUS,           YES, TRUE,  3DNOWPLUS) \
-FLAGDEFA( 81, EDX, AMD,    31,  1, 3DNOW,               YES, TRUE,  3DNOW)
-
-/*    LEVEL, REG, VENDOR, POS, SIZE, NAME,       MON SUPP, CPL3, [FUNC] */
-#define CPUID_FIELD_DATA_LEVEL_8x                                              \
-FIELDDEF( 85, EAX, AMD,     0,  8, ITLB_ENTRIES_2M4M_PGS, NA, FALSE)           \
-FIELDDEF( 85, EAX, AMD,     8,  8, ITLB_ASSOC_2M4M_PGS, NA,  FALSE)            \
-FIELDDEF( 85, EAX, AMD,    16,  8, DTLB_ENTRIES_2M4M_PGS, NA, FALSE)           \
-FIELDDEF( 85, EAX, AMD,    24,  8, DTLB_ASSOC_2M4M_PGS, NA,  FALSE)            \
-FIELDDEF( 85, EBX, AMD,     0,  8, ITLB_ENTRIES_4K_PGS, NA,  FALSE)            \
-FIELDDEF( 85, EBX, AMD,     8,  8, ITLB_ASSOC_4K_PGS,   NA,  FALSE)            \
-FIELDDEF( 85, EBX, AMD,    16,  8, DTLB_ENTRIES_4K_PGS, NA,  FALSE)            \
-FIELDDEF( 85, EBX, AMD,    24,  8, DTLB_ASSOC_4K_PGS,   NA,  FALSE)            \
-FIELDDEF( 85, ECX, AMD,     0,  8, L1_DCACHE_LINE_SIZE, NA,  FALSE)            \
-FIELDDEF( 85, ECX, AMD,     8,  8, L1_DCACHE_LINES_PER_TAG, NA, FALSE)         \
-FIELDDEF( 85, ECX, AMD,    16,  8, L1_DCACHE_ASSOC,     NA, FALSE)             \
-FIELDDEF( 85, ECX, AMD,    24,  8, L1_DCACHE_SIZE,      NA, FALSE)             \
-FIELDDEF( 85, EDX, AMD,     0,  8, L1_ICACHE_LINE_SIZE, NA,  FALSE)            \
-FIELDDEF( 85, EDX, AMD,     8,  8, L1_ICACHE_LINES_PER_TAG, NA, FALSE)         \
-FIELDDEF( 85, EDX, AMD,    16,  8, L1_ICACHE_ASSOC,     NA, FALSE)             \
-FIELDDEF( 85, EDX, AMD,    24,  8, L1_ICACHE_SIZE,      NA, FALSE)             \
-FIELDDEF( 86, EAX, AMD,     0, 12, L2_ITLB_ENTRIES_2M4M_PGS, NA, FALSE)        \
-FIELDDEF( 86, EAX, AMD,    12,  4, L2_ITLB_ASSOC_2M4M_PGS, NA, FALSE)          \
-FIELDDEF( 86, EAX, AMD,    16, 12, L2_DTLB_ENTRIES_2M4M_PGS, NA, FALSE)        \
-FIELDDEF( 86, EAX, AMD,    28,  4, L2_DTLB_ASSOC_2M4M_PGS, NA, FALSE)          \
-FIELDDEF( 86, EBX, AMD,     0, 12, L2_ITLB_ENTRIES_4K_PGS, NA, FALSE)          \
-FIELDDEF( 86, EBX, AMD,    12,  4, L2_ITLB_ASSOC_4K_PGS, NA, FALSE)            \
-FIELDDEF( 86, EBX, AMD,    16, 12, L2_DTLB_ENTRIES_4K_PGS, NA, FALSE)          \
-FIELDDEF( 86, EBX, AMD,    28,  4, L2_DTLB_ASSOC_4K_PGS,NA,  FALSE)            \
-FIELDDEF( 86, ECX, AMD,     0,  8, L2CACHE_LINE,        NA,  FALSE)            \
-FIELDDEF( 86, ECX, AMD,     8,  4, L2CACHE_LINE_PER_TAG,NA,  FALSE)            \
-FIELDDEF( 86, ECX, AMD,    12,  4, L2CACHE_WAYS,        NA,  FALSE)            \
-FIELDDEF( 86, ECX, AMD,    16, 16, L2CACHE_SIZE,        NA,  FALSE)            \
-FIELDDEF( 86, EDX, AMD,     0,  8, L3CACHE_LINE,        NA,  FALSE)            \
-FIELDDEF( 86, EDX, AMD,     8,  4, L3CACHE_LINE_PER_TAG,NA,  FALSE)            \
-FIELDDEF( 86, EDX, AMD,    12,  4, L3CACHE_WAYS,        NA,  FALSE)            \
-FIELDDEF( 86, EDX, AMD,    18, 14, L3CACHE_SIZE,        NA,  FALSE)            \
-FLAGDEF(  87, EDX, AMD,     0,  1, TS,                  NA,  FALSE)            \
-FLAGDEF(  87, EDX, AMD,     1,  1, FID,                 NA,  FALSE)            \
-FLAGDEF(  87, EDX, AMD,     2,  1, VID,                 NA,  FALSE)            \
-FLAGDEF(  87, EDX, AMD,     3,  1, TTP,                 NA,  FALSE)            \
-FLAGDEF(  87, EDX, AMD,     4,  1, TM,                  NA,  FALSE)            \
-FLAGDEF(  87, EDX, AMD,     5,  1, STC,                 NA,  FALSE)            \
-FLAGDEF(  87, EDX, AMD,     6,  1, 100MHZSTEPS,         NA,  FALSE)            \
-FLAGDEF(  87, EDX, AMD,     7,  1, HWPSTATE,            NA,  FALSE)            \
-FLAGDEF(  87, EDX, COMMON,  8,  1, TSC_INVARIANT,       NA,  FALSE)            \
-FLAGDEF(  87, EDX, COMMON,  9,  1, CORE_PERF_BOOST,     NA,  FALSE)            \
-FIELDDEFA(88, EAX, COMMON,  0,  8, PHYSBITS,            NA,  FALSE, PHYS_BITS) \
-FIELDDEFA(88, EAX, COMMON,  8,  8, VIRTBITS,            NA,  FALSE, VIRT_BITS) \
-FIELDDEF( 88, EAX, COMMON, 16,  8, GUEST_PHYS_ADDR_SZ,  NA,  FALSE)            \
-FIELDDEFA(88, ECX, AMD,     0,  8, CORE_COUNT,          NA,  FALSE, AMD_CORE_COUNT) \
-FIELDDEFA(88, ECX, AMD,    12,  4, APICID_COREID_SIZE,  NA,  FALSE, AMD_APICID_COREID_SIZE) \
-FIELDDEFA(8A, EAX, AMD,     0,  8, SVM_REVISION,        YES, FALSE, SVM_REVISION) \
-FLAGDEF(  8A, EAX, AMD,     8,  1, SVM_HYPERVISOR,      NO,  FALSE)            \
-FIELDDEF( 8A, EAX, AMD,     9, 23, SVMEAX_RSVD,         NO,  FALSE)            \
-FIELDDEFA(8A, EBX, AMD,     0, 32, SVM_N_ASIDS,         YES, FALSE, SVM_NUM_ASIDS) \
-FIELDDEF( 8A, ECX, AMD,     0, 32, SVMECX_RSVD,         NO,  FALSE)            \
-FLAGDEFA( 8A, EDX, AMD,     0,  1, SVM_NP,              YES, FALSE, NPT)       \
-FLAGDEF(  8A, EDX, AMD,     1,  1, SVM_LBR,             NO,  FALSE)            \
-FLAGDEFA( 8A, EDX, AMD,     2,  1, SVM_LOCK,            YES, FALSE, SVM_LOCK)  \
-FLAGDEFA( 8A, EDX, AMD,     3,  1, SVM_NRIP,            YES, FALSE, SVM_NRIP)  \
-FLAGDEF(  8A, EDX, AMD,     4,  1, SVM_TSC_RATE_MSR,    NO,  FALSE)            \
-FLAGDEF(  8A, EDX, AMD,     5,  1, SVM_VMCB_CLEAN,      YES, FALSE)            \
-FLAGDEF(  8A, EDX, AMD,     6,  1, SVM_FLUSH_BY_ASID,   YES, FALSE)            \
-FLAGDEFA( 8A, EDX, AMD,     7,  1, SVM_DECODE_ASSISTS,  YES, FALSE, SVM_DECODE_ASSISTS) \
-FIELDDEF( 8A, EDX, AMD,     8,  2, SVMEDX_RSVD0,        NO,  FALSE)            \
-FLAGDEFA( 8A, EDX, AMD,    10,  1, SVM_PAUSE_FILTER,    NO,  FALSE, PAUSE_FILTER) \
-FLAGDEF(  8A, EDX, AMD,    11,  1, SVMEDX_RSVD1,        NO,  FALSE)            \
-FLAGDEF(  8A, EDX, AMD,    12,  1, SVM_PAUSE_THRESHOLD, NO,  FALSE)            \
-FIELDDEF( 8A, EDX, AMD,    13, 19, SVMEDX_RSVD2,        NO,  FALSE)
-
-/*    LEVEL, REG, VENDOR, POS, SIZE, NAME,       MON SUPP, CPL3, [FUNC] */
-#define CPUID_FIELD_DATA_LEVEL_81x                                             \
-FIELDDEF(819, EAX, AMD,     0, 12, L1_ITLB_ENTRIES_1G_PGS, NA, FALSE)          \
-FIELDDEF(819, EAX, AMD,    12,  4, L1_ITLB_ASSOC_1G_PGS, NA, FALSE)            \
-FIELDDEF(819, EAX, AMD,    16, 12, L1_DTLB_ENTRIES_1G_PGS, NA, FALSE)          \
-FIELDDEF(819, EAX, AMD,    28,  4, L1_DTLB_ASSOC_1G_PGS, NA, FALSE)            \
-FIELDDEF(819, EBX, AMD,     0, 12, L2_ITLB_ENTRIES_1G_PGS, NA, FALSE)          \
-FIELDDEF(819, EBX, AMD,    12,  4, L2_ITLB_ASSOC_1G_PGS, NA, FALSE)            \
-FIELDDEF(819, EBX, AMD,    16, 12, L2_DTLB_ENTRIES_1G_PGS, NA, FALSE)          \
-FIELDDEF(819, EBX, AMD,    28,  4, L2_DTLB_ASSOC_1G_PGS, NA, FALSE)            \
-FLAGDEF( 81A, EAX, AMD,     0,  1, FP128,               NA,  FALSE)            \
-FLAGDEF( 81A, EAX, AMD,     1,  1, MOVU,                NA,  FALSE)            \
-FLAGDEF( 81B, EAX, AMD,     0,  1, IBS_FFV,             NA,  FALSE)            \
-FLAGDEF( 81B, EAX, AMD,     1,  1, IBS_FETCHSAM,        NA,  FALSE)            \
-FLAGDEF( 81B, EAX, AMD,     2,  1, IBS_OPSAM,           NA,  FALSE)            \
-FLAGDEF( 81B, EAX, AMD,     3,  1, RW_OPCOUNT,          NA,  FALSE)            \
-FLAGDEF( 81B, EAX, AMD,     4,  1, OPCOUNT,             NA,  FALSE)            \
-FLAGDEF( 81B, EAX, AMD,     5,  1, BRANCH_TARGET_ADDR,  NA,  FALSE)            \
-FLAGDEF( 81B, EAX, AMD,     6,  1, OPCOUNT_EXT,         NA,  FALSE)            \
-FLAGDEF( 81B, EAX, AMD,     7,  1, RIP_INVALID_CHECK,   NA,  FALSE)            \
-FLAGDEF( 81C, EAX, AMD,     0,  1, LWP_AVAIL,           NA,  FALSE)            \
-FLAGDEF( 81C, EAX, AMD,     1,  1, LWP_VAL_AVAIL,       NA,  FALSE)            \
-FLAGDEF( 81C, EAX, AMD,     2,  1, LWP_IRE_AVAIL,       NA,  FALSE)            \
-FLAGDEF( 81C, EAX, AMD,     3,  1, LWP_BRE_AVAIL,       NA,  FALSE)            \
-FLAGDEF( 81C, EAX, AMD,     4,  1, LWP_DME_AVAIL,       NA,  FALSE)            \
-FLAGDEF( 81C, EAX, AMD,     5,  1, LWP_CNH_AVAIL,       NA,  FALSE)            \
-FLAGDEF( 81C, EAX, AMD,     6,  1, LWP_RNH_AVAIL,       NA,  FALSE)            \
-FLAGDEF( 81C, EAX, AMD,    31,  1, LWP_INT_AVAIL,       NA,  FALSE)            \
-FIELDDEF(81C, EBX, AMD,     0,  8, LWP_CB_SIZE,         NA,  FALSE)            \
-FIELDDEF(81C, EBX, AMD,     8,  8, LWP_EVENT_SIZE,      NA,  FALSE)            \
-FIELDDEF(81C, EBX, AMD,    16,  8, LWP_MAX_EVENTS,      NA,  FALSE)            \
-FIELDDEF(81C, EBX, AMD,    24,  8, LWP_EVENT_OFFSET,    NA,  FALSE)            \
-FIELDDEF(81C, ECX, AMD,     0,  4, LWP_LATENCY_MAX,     NA,  FALSE)            \
-FLAGDEF( 81C, ECX, AMD,     5,  1, LWP_DATA_ADDR_VALID, NA,  FALSE)            \
-FIELDDEF(81C, ECX, AMD,     6,  3, LWP_LATENCY_ROUND,   NA,  FALSE)            \
-FIELDDEF(81C, ECX, AMD,     9,  7, LWP_VERSION,         NA,  FALSE)            \
-FIELDDEF(81C, ECX, AMD,    16,  8, LWP_MIN_BUF_SIZE,    NA,  FALSE)            \
-FLAGDEF( 81C, ECX, AMD,    28,  1, LWP_BRANCH_PRED,     NA,  FALSE)            \
-FLAGDEF( 81C, ECX, AMD,    29,  1, LWP_IP_FILTERING,    NA,  FALSE)            \
-FLAGDEF( 81C, ECX, AMD,    30,  1, LWP_CACHE_LEVEL,     NA,  FALSE)            \
-FLAGDEF( 81C, ECX, AMD,    31,  1, LWP_CACHE_LATENCY,   NA,  FALSE)            \
-FLAGDEF( 81C, EDX, AMD,     0,  1, LWP_SUPPORTED,       NA,  FALSE)            \
-FLAGDEF( 81C, EDX, AMD,     1,  1, LWP_VAL_SUPPORTED,   NA,  FALSE)            \
-FLAGDEF( 81C, EDX, AMD,     2,  1, LWP_IRE_SUPPORTED,   NA,  FALSE)            \
-FLAGDEF( 81C, EDX, AMD,     3,  1, LWP_BRE_SUPPORTED,   NA,  FALSE)            \
-FLAGDEF( 81C, EDX, AMD,     4,  1, LWP_DME_SUPPORTED,   NA,  FALSE)            \
-FLAGDEF( 81C, EDX, AMD,     5,  1, LWP_CNH_SUPPORTED,   NA,  FALSE)            \
-FLAGDEF( 81C, EDX, AMD,     6,  1, LWP_RNH_SUPPORTED,   NA,  FALSE)            \
-FLAGDEF( 81C, EDX, AMD,    31,  1, LWP_INT_SUPPORTED,   NA,  FALSE)            \
-FIELDDEFA( 81D,EAX, AMD,     0,  5, CACHE_TYPE,          NA,  FALSE, AMD_CACHE_TYPE) \
-FIELDDEFA( 81D,EAX, AMD,     5,  3, CACHE_LEVEL,         NA,  FALSE, AMD_CACHE_LEVEL) \
-FLAGDEF( 81D, EAX, AMD,     8,  1, CACHE_SELF_INIT,     NA,  FALSE)            \
-FLAGDEF( 81D, EAX, AMD,     9,  1, CACHE_FULLY_ASSOC,   NA,  FALSE)            \
-FIELDDEFA(81D, EAX, AMD,    14, 12, NUM_SHARING_CACHE,   NA,  FALSE, AMD_NUM_SHARING_CACHE)            \
-FIELDDEF(81D, EBX, AMD,     0, 12, CACHE_LINE_SIZE,     NA,  FALSE)            \
-FIELDDEF(81D, EBX, AMD,    12, 10, CACHE_PHYS_PARTITIONS, NA, FALSE)           \
-FIELDDEFA(81D,EBX, AMD,   22, 10, CACHE_WAYS,          NA,  FALSE, AMD_CACHE_WAYS) \
-FIELDDEF(81D, ECX, AMD,     0, 32, CACHE_NUM_SETS,      NA,  FALSE)            \
-FLAGDEF( 81D, EDX, AMD,     0,  1, CACHE_WBINVD,        NA,  FALSE)            \
-FLAGDEF( 81D, EDX, AMD,     1,  1, CACHE_INCLUSIVE,     NA,  FALSE)            \
-FIELDDEF(81E, EAX, AMD,     0, 32, EXTENDED_APICID,     NA,  FALSE)            \
-FIELDDEF(81E, EBX, AMD,     0,  8, COMPUTE_UNIT_ID,     NA,  FALSE)            \
-FIELDDEFA(81E, EBX, AMD,     8,  2, CORES_PER_COMPUTE_UNIT, NA, FALSE, AMD_CORES_PER_COMPUTE_UNIT)          \
-FIELDDEF(81E, ECX, AMD,     0,  8, NODEID,              NA,  FALSE)            \
-FIELDDEFA( 81E,ECX, AMD,     8,  3, NODES_PER_PKG,       NA,  FALSE, AMD_NODES_PER_PKG)
-
+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,                   MON SUPP, CPL3 */
+#define CPUID_FIELD_DATA_LEVEL_0                                                \
+FIELD(  0,  0, EAX,  0, 32, NUMLEVELS,                              ANY, FALSE) \
+FIELD(  0,  0, EBX,  0, 32, VENDOR1,                                YES, TRUE)  \
+FIELD(  0,  0, ECX,  0, 32, VENDOR3,                                YES, TRUE)  \
+FIELD(  0,  0, EDX,  0, 32, VENDOR2,                                YES, TRUE)
+                                                        
+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,                   MON SUPP, CPL3 */
+#define CPUID_FIELD_DATA_LEVEL_1                                                \
+FIELD(  1,  0, EAX,  0,  4, STEPPING,                               ANY, FALSE) \
+FIELD(  1,  0, EAX,  4,  4, MODEL,                                  ANY, FALSE) \
+FIELD(  1,  0, EAX,  8,  4, FAMILY,                                 YES, FALSE) \
+FIELD(  1,  0, EAX, 12,  2, TYPE,                                   ANY, FALSE) \
+FIELD(  1,  0, EAX, 16,  4, EXTENDED_MODEL,                         ANY, FALSE) \
+FIELD(  1,  0, EAX, 20,  8, EXTENDED_FAMILY,                        YES, FALSE) \
+FIELD(  1,  0, EBX,  0,  8, BRAND_ID,                               ANY, FALSE) \
+FIELD(  1,  0, EBX,  8,  8, CLFL_SIZE,                              ANY, FALSE) \
+FIELD(  1,  0, EBX, 16,  8, LCPU_COUNT,                             ANY, FALSE) \
+FIELD(  1,  0, EBX, 24,  8, APICID,                                 ANY, FALSE) \
+FLAG(   1,  0, ECX,  0,  1, SSE3,                                   YES, TRUE)  \
+FLAG(   1,  0, ECX,  1,  1, PCLMULQDQ,                              YES, TRUE)  \
+FLAG(   1,  0, ECX,  2,  1, DTES64,                                 NO,  FALSE) \
+FLAG(   1,  0, ECX,  3,  1, MWAIT,                                  YES, FALSE) \
+FLAG(   1,  0, ECX,  4,  1, DSCPL,                                  NO,  FALSE) \
+FLAG(   1,  0, ECX,  5,  1, VMX,                                    YES, FALSE) \
+FLAG(   1,  0, ECX,  6,  1, SMX,                                    NO,  FALSE) \
+FLAG(   1,  0, ECX,  7,  1, EIST,                                   NO,  FALSE) \
+FLAG(   1,  0, ECX,  8,  1, TM2,                                    NO,  FALSE) \
+FLAG(   1,  0, ECX,  9,  1, SSSE3,                                  YES, TRUE)  \
+FLAG(   1,  0, ECX, 10,  1, CNXTID,                                 NO,  FALSE) \
+FLAG(   1,  0, ECX, 11,  1, NDA11,                                  NO,  FALSE) \
+FLAG(   1,  0, ECX, 12,  1, FMA,                                    YES, TRUE)  \
+FLAG(   1,  0, ECX, 13,  1, CMPXCHG16B,                             YES, TRUE)  \
+FLAG(   1,  0, ECX, 14,  1, xTPR,                                   NO,  FALSE) \
+FLAG(   1,  0, ECX, 15,  1, PDCM,                                   NO,  FALSE) \
+FLAG(   1,  0, ECX, 17,  1, PCID,                                   YES, FALSE) \
+FLAG(   1,  0, ECX, 18,  1, DCA,                                    NO,  FALSE) \
+FLAG(   1,  0, ECX, 19,  1, SSE41,                                  YES, TRUE)  \
+FLAG(   1,  0, ECX, 20,  1, SSE42,                                  YES, TRUE)  \
+FLAG(   1,  0, ECX, 21,  1, x2APIC,                                 ANY, FALSE) \
+FLAG(   1,  0, ECX, 22,  1, MOVBE,                                  YES, TRUE)  \
+FLAG(   1,  0, ECX, 23,  1, POPCNT,                                 YES, TRUE)  \
+FLAG(   1,  0, ECX, 24,  1, TSC_DEADLINE,                           NO,  FALSE) \
+FLAG(   1,  0, ECX, 25,  1, AES,                                    YES, TRUE)  \
+FLAG(   1,  0, ECX, 26,  1, XSAVE,                                  YES, FALSE) \
+FLAG(   1,  0, ECX, 27,  1, OSXSAVE,                                ANY, FALSE) \
+FLAG(   1,  0, ECX, 28,  1, AVX,                                    YES, FALSE) \
+FLAG(   1,  0, ECX, 29,  1, F16C,                                   YES, TRUE)  \
+FLAG(   1,  0, ECX, 30,  1, RDRAND,                                 YES, TRUE)  \
+FLAG(   1,  0, ECX, 31,  1, HYPERVISOR,                             ANY, TRUE)  \
+FLAG(   1,  0, EDX,  0,  1, FPU,                                    YES, TRUE)  \
+FLAG(   1,  0, EDX,  1,  1, VME,                                    YES, FALSE) \
+FLAG(   1,  0, EDX,  2,  1, DE,                                     YES, FALSE) \
+FLAG(   1,  0, EDX,  3,  1, PSE,                                    YES, FALSE) \
+FLAG(   1,  0, EDX,  4,  1, TSC,                                    YES, TRUE)  \
+FLAG(   1,  0, EDX,  5,  1, MSR,                                    YES, FALSE) \
+FLAG(   1,  0, EDX,  6,  1, PAE,                                    YES, FALSE) \
+FLAG(   1,  0, EDX,  7,  1, MCE,                                    YES, FALSE) \
+FLAG(   1,  0, EDX,  8,  1, CX8,                                    YES, TRUE)  \
+FLAG(   1,  0, EDX,  9,  1, APIC,                                   ANY, FALSE) \
+FLAG(   1,  0, EDX, 11,  1, SEP,                                    YES, TRUE)  \
+FLAG(   1,  0, EDX, 12,  1, MTRR,                                   YES, FALSE) \
+FLAG(   1,  0, EDX, 13,  1, PGE,                                    YES, FALSE) \
+FLAG(   1,  0, EDX, 14,  1, MCA,                                    YES, FALSE) \
+FLAG(   1,  0, EDX, 15,  1, CMOV,                                   YES, TRUE)  \
+FLAG(   1,  0, EDX, 16,  1, PAT,                                    YES, FALSE) \
+FLAG(   1,  0, EDX, 17,  1, PSE36,                                  YES, FALSE) \
+FLAG(   1,  0, EDX, 18,  1, PSN,                                    YES, FALSE) \
+FLAG(   1,  0, EDX, 19,  1, CLFSH,                                  YES, TRUE)  \
+FLAG(   1,  0, EDX, 21,  1, DS,                                     YES, FALSE) \
+FLAG(   1,  0, EDX, 22,  1, ACPI,                                   ANY, FALSE) \
+FLAG(   1,  0, EDX, 23,  1, MMX,                                    YES, TRUE)  \
+FLAG(   1,  0, EDX, 24,  1, FXSR,                                   YES, TRUE)  \
+FLAG(   1,  0, EDX, 25,  1, SSE,                                    YES, TRUE)  \
+FLAG(   1,  0, EDX, 26,  1, SSE2,                                   YES, TRUE)  \
+FLAG(   1,  0, EDX, 27,  1, SS,                                     YES, FALSE) \
+FLAG(   1,  0, EDX, 28,  1, HTT,                                    ANY, FALSE) \
+FLAG(   1,  0, EDX, 29,  1, TM,                                     NO,  FALSE) \
+FLAG(   1,  0, EDX, 30,  1, IA64,                                   NO,  FALSE) \
+FLAG(   1,  0, EDX, 31,  1, PBE,                                    NO,  FALSE)
+
+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,                   MON SUPP, CPL3 */
+#define CPUID_FIELD_DATA_LEVEL_4                                                \
+FIELD(  4,  0, EAX,  0,  5, LEAF4_CACHE_TYPE,                       NA,  FALSE) \
+FIELD(  4,  0, EAX,  5,  3, LEAF4_CACHE_LEVEL,                      NA,  FALSE) \
+FLAG(   4,  0, EAX,  8,  1, LEAF4_CACHE_SELF_INIT,                  NA,  FALSE) \
+FLAG(   4,  0, EAX,  9,  1, LEAF4_CACHE_FULLY_ASSOC,                NA,  FALSE) \
+FIELD(  4,  0, EAX, 14, 12, LEAF4_CACHE_NUMHT_SHARING,              NA,  FALSE) \
+FIELD(  4,  0, EAX, 26,  6, LEAF4_CORE_COUNT,                       NA,  FALSE) \
+FIELD(  4,  0, EBX,  0, 12, LEAF4_CACHE_LINE,                       NA,  FALSE) \
+FIELD(  4,  0, EBX, 12, 10, LEAF4_CACHE_PART,                       NA,  FALSE) \
+FIELD(  4,  0, EBX, 22, 10, LEAF4_CACHE_WAYS,                       NA,  FALSE) \
+FIELD(  4,  0, ECX,  0, 32, LEAF4_CACHE_SETS,                       NA,  FALSE) \
+FLAG(   4,  0, EDX,  0,  1, LEAF4_CACHE_WBINVD_NOT_GUARANTEED,      NA,  FALSE) \
+FLAG(   4,  0, EDX,  1,  1, LEAF4_CACHE_IS_INCLUSIVE,               NA,  FALSE) \
+FLAG(   4,  0, EDX,  2,  1, LEAF4_CACHE_COMPLEX_INDEXING,           NA,  FALSE)
+
+/*     LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,                  MON SUPP, CPL3 */
+#define CPUID_FIELD_DATA_LEVEL_5                                                \
+FIELD(  5,  0, EAX,  0, 16, MWAIT_MIN_SIZE,                         NA,  FALSE) \
+FIELD(  5,  0, EBX,  0, 16, MWAIT_MAX_SIZE,                         NA,  FALSE) \
+FLAG(   5,  0, ECX,  0,  1, MWAIT_EXTENSIONS,                       NA,  FALSE) \
+FLAG(   5,  0, ECX,  1,  1, MWAIT_INTR_BREAK,                       NA,  FALSE) \
+FIELD(  5,  0, EDX,  0,  4, MWAIT_C0_SUBSTATE,                      NA,  FALSE) \
+FIELD(  5,  0, EDX,  4,  4, MWAIT_C1_SUBSTATE,                      NA,  FALSE) \
+FIELD(  5,  0, EDX,  8,  4, MWAIT_C2_SUBSTATE,                      NA,  FALSE) \
+FIELD(  5,  0, EDX, 12,  4, MWAIT_C3_SUBSTATE,                      NA,  FALSE) \
+FIELD(  5,  0, EDX, 16,  4, MWAIT_C4_SUBSTATE,                      NA,  FALSE)
+
+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,                   MON SUPP, CPL3 */
+#define CPUID_FIELD_DATA_LEVEL_6                                                \
+FLAG(   6,  0, EAX,  0,  1, THERMAL_SENSOR,                         NA,  FALSE) \
+FLAG(   6,  0, EAX,  1,  1, TURBO_MODE,                             NA,  FALSE) \
+FLAG(   6,  0, EAX,  2,  1, APIC_INVARIANT,                         NA,  FALSE) \
+FIELD(  6,  0, EBX,  0,  4, NUM_INTR_THRESHOLDS,                    NA,  FALSE) \
+FLAG(   6,  0, ECX,  0,  1, HW_COORD_FEEDBACK,                      NA,  FALSE) \
+FLAG(   6,  0, ECX,  3,  1, ENERGY_PERF_BIAS,                       NA,  FALSE)
+
+
+#define CPUID_7_EBX_13
+
+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,                   MON SUPP, CPL3 */
+#define CPUID_FIELD_DATA_LEVEL_7                                                \
+FLAG(   7,  0, EBX,  0,  1, FSGSBASE,                               YES, FALSE) \
+FLAG(   7,  0, EBX,  3,  1, BMI1,                                   YES, TRUE)  \
+FLAG(   7,  0, EBX,  4,  1, HLE,                                    NO,  TRUE)  \
+FLAG(   7,  0, EBX,  5,  1, AVX2,                                   NO,  TRUE)  \
+FLAG(   7,  0, EBX,  7,  1, SMEP,                                   YES, FALSE) \
+FLAG(   7,  0, EBX,  8,  1, BMI2,                                   NO,  TRUE)  \
+FLAG(   7,  0, EBX,  9,  1, ENFSTRG,                                YES, FALSE) \
+FLAG(   7,  0, EBX, 10,  1, INVPCID,                                NO,  FALSE) \
+FLAG(   7,  0, EBX, 11,  1, RTM,                                    NO,  TRUE)  \
+CPUID_7_EBX_13
+
+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,                   MON SUPP, CPL3 */
+#define CPUID_FIELD_DATA_LEVEL_A                                                \
+FIELD(  A,  0, EAX,  0,  8, PMC_VERSION,                            NA,  FALSE) \
+FIELD(  A,  0, EAX,  8,  8, PMC_NUM_GEN,                            NA,  FALSE) \
+FIELD(  A,  0, EAX, 16,  8, PMC_WIDTH_GEN,                          NA,  FALSE) \
+FIELD(  A,  0, EAX, 24,  8, PMC_EBX_LENGTH,                         NA,  FALSE) \
+FLAG(   A,  0, EBX,  0,  1, PMC_CORE_CYCLES,                        NA,  FALSE) \
+FLAG(   A,  0, EBX,  1,  1, PMC_INSTR_RETIRED,                      NA,  FALSE) \
+FLAG(   A,  0, EBX,  2,  1, PMC_REF_CYCLES,                         NA,  FALSE) \
+FLAG(   A,  0, EBX,  3,  1, PMC_LAST_LVL_CREF,                      NA,  FALSE) \
+FLAG(   A,  0, EBX,  4,  1, PMC_LAST_LVL_CMISS,                     NA,  FALSE) \
+FLAG(   A,  0, EBX,  5,  1, PMC_BR_INST_RETIRED,                    NA,  FALSE) \
+FLAG(   A,  0, EBX,  6,  1, PMC_BR_MISS_RETIRED,                    NA,  FALSE) \
+FIELD(  A,  0, EDX,  0,  5, PMC_NUM_FIXED,                          NA,  FALSE) \
+FIELD(  A,  0, EDX,  5,  8, PMC_WIDTH_FIXED,                        NA,  FALSE)
+
+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,                   MON SUPP, CPL3 */
+#define CPUID_FIELD_DATA_LEVEL_B                                                \
+FIELD(  B,  0, EAX,  0,  5, TOPOLOGY_MASK_WIDTH,                    NA,  FALSE) \
+FIELD(  B,  0, EBX,  0, 16, TOPOLOGY_CPUS_SHARING_LEVEL,            NA,  FALSE) \
+FIELD(  B,  0, ECX,  0,  8, TOPOLOGY_LEVEL_NUMBER,                  NA,  FALSE) \
+FIELD(  B,  0, ECX,  8,  8, TOPOLOGY_LEVEL_TYPE,                    NA,  FALSE) \
+FIELD(  B,  0, EDX,  0, 32, TOPOLOGY_X2APIC_ID,                     NA,  FALSE)
+
+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,                   MON SUPP, CPL3 */
+#define CPUID_FIELD_DATA_LEVEL_D                                                \
+FLAG(   D,  0, EAX,  0,  1, XCR0_MASTER_LEGACY_FP,                  YES, FALSE) \
+FLAG(   D,  0, EAX,  1,  1, XCR0_MASTER_SSE,                        YES, FALSE) \
+FLAG(   D,  0, EAX,  2,  1, XCR0_MASTER_YMM_H,                      YES, FALSE) \
+FIELD(  D,  0, EAX,  3, 29, XCR0_MASTER_LOWER,                      NO,  FALSE) \
+FIELD(  D,  0, EBX,  0, 32, XSAVE_ENABLED_SIZE,                     ANY, FALSE) \
+FIELD(  D,  0, ECX,  0, 32, XSAVE_MAX_SIZE,                         YES, FALSE) \
+FIELD(  D,  0, EDX,  0, 29, XCR0_MASTER_UPPER,                      NO,  FALSE) \
+FLAG(   D,  0, EDX, 30,  1, XCR0_MASTER_LWP,                        NO,  FALSE) \
+FLAG(   D,  0, EDX, 31,  1, XCR0_MASTER_EXTENDED_XSAVE,             NO,  FALSE) \
+FLAG(   D,  1, EAX,  0,  1, XSAVEOPT,                               NO,  FALSE) \
+FIELD(  D,  2, EAX,  0, 32, XSAVE_YMM_SIZE,                         YES, FALSE) \
+FIELD(  D,  2, EBX,  0, 32, XSAVE_YMM_OFFSET,                       YES, FALSE) \
+FIELD(  D,  2, ECX,  0, 32, XSAVE_YMM_RSVD1,                        YES, FALSE) \
+FIELD(  D,  2, EDX,  0, 32, XSAVE_YMM_RSVD2,                        YES, FALSE) \
+FIELD(  D, 62, EAX,  0, 32, XSAVE_LWP_SIZE,                         NO,  FALSE) \
+FIELD(  D, 62, EBX,  0, 32, XSAVE_LWP_OFFSET,                       NO,  FALSE) \
+FIELD(  D, 62, ECX,  0, 32, XSAVE_LWP_RSVD1,                        NO,  FALSE) \
+FIELD(  D, 62, EDX,  0, 32, XSAVE_LWP_RSVD2,                        NO,  FALSE)
+
+
+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,                   MON SUPP, CPL3 */
+#define CPUID_FIELD_DATA_LEVEL_400                                              \
+FIELD(400,  0, EAX,  0, 32, NUM_HYP_LEVELS,                         NA,  FALSE) \
+FIELD(400,  0, EBX,  0, 32, HYPERVISOR1,                            NA,  FALSE) \
+FIELD(400,  0, ECX,  0, 32, HYPERVISOR2,                            NA,  FALSE) \
+FIELD(400,  0, EDX,  0, 32, HYPERVISOR3,                            NA,  FALSE)
+
+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,                   MON SUPP, CPL3 */
+#define CPUID_FIELD_DATA_LEVEL_410                                              \
+FIELD(410,  0, EAX,  0, 32, TSC_HZ,                                 NA,  FALSE) \
+FIELD(410,  0, EBX,  0, 32, ACPIBUS_HZ,                             NA,  FALSE)
+
+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,                   MON SUPP, CPL3 */
+#define CPUID_FIELD_DATA_LEVEL_80                                               \
+FIELD( 80,  0, EAX,  0, 32, NUM_EXT_LEVELS,                         NA,  FALSE) \
+FIELD( 80,  0, EBX,  0, 32, LEAF80_VENDOR1,                         NA,  FALSE) \
+FIELD( 80,  0, ECX,  0, 32, LEAF80_VENDOR3,                         NA,  FALSE) \
+FIELD( 80,  0, EDX,  0, 32, LEAF80_VENDOR2,                         NA,  FALSE)
+
+#define CPUID_81_ECX_17
+
+                                                        
+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,                   MON SUPP, CPL3 */
+#define CPUID_FIELD_DATA_LEVEL_81                                               \
+FIELD( 81,  0, EAX,  0, 32, UNKNOWN81EAX,                           ANY, FALSE) \
+FIELD( 81,  0, EAX,  0,  4, LEAF81_STEPPING,                        ANY, FALSE) \
+FIELD( 81,  0, EAX,  4,  4, LEAF81_MODEL,                           ANY, FALSE) \
+FIELD( 81,  0, EAX,  8,  4, LEAF81_FAMILY,                          ANY, FALSE) \
+FIELD( 81,  0, EAX, 12,  2, LEAF81_TYPE,                            ANY, FALSE) \
+FIELD( 81,  0, EAX, 16,  4, LEAF81_EXTENDED_MODEL,                  ANY, FALSE) \
+FIELD( 81,  0, EAX, 20,  8, LEAF81_EXTENDED_FAMILY,                 ANY, FALSE) \
+FIELD( 81,  0, EBX,  0, 32, UNKNOWN81EBX,                           ANY, FALSE) \
+FIELD( 81,  0, EBX,  0, 16, LEAF81_BRAND_ID,                        ANY, FALSE) \
+FIELD( 81,  0, EBX, 16, 16, UNDEF,                                  ANY, FALSE) \
+FLAG(  81,  0, ECX,  0,  1, LAHF64,                                 YES, TRUE)  \
+FLAG(  81,  0, ECX,  1,  1, CMPLEGACY,                              ANY, FALSE) \
+FLAG(  81,  0, ECX,  2,  1, SVM,                                    YES, FALSE) \
+FLAG(  81,  0, ECX,  3,  1, EXTAPICSPC,                             YES, FALSE) \
+FLAG(  81,  0, ECX,  4,  1, CR8AVAIL,                               YES, FALSE) \
+FLAG(  81,  0, ECX,  5,  1, ABM,                                    YES, TRUE)  \
+FLAG(  81,  0, ECX,  6,  1, SSE4A,                                  YES, TRUE)  \
+FLAG(  81,  0, ECX,  7,  1, MISALIGNED_SSE,                         YES, TRUE)  \
+FLAG(  81,  0, ECX,  8,  1, 3DNPREFETCH,                            YES, TRUE)  \
+FLAG(  81,  0, ECX,  9,  1, OSVW,                                   ANY, FALSE) \
+FLAG(  81,  0, ECX, 10,  1, IBS,                                    NO,  FALSE) \
+FLAG(  81,  0, ECX, 11,  1, XOP,                                    YES, TRUE)  \
+FLAG(  81,  0, ECX, 12,  1, SKINIT,                                 NO,  FALSE) \
+FLAG(  81,  0, ECX, 13,  1, WATCHDOG,                               NO,  FALSE) \
+FLAG(  81,  0, ECX, 15,  1, LWP,                                    NO,  FALSE) \
+FLAG(  81,  0, ECX, 16,  1, FMA4,                                   YES, TRUE)  \
+CPUID_81_ECX_17 \
+FLAG(  81,  0, ECX, 19,  1, NODEID_MSR,                             NO,  FALSE) \
+FLAG(  81,  0, ECX, 21,  1, TBM,                                    YES, TRUE)  \
+FLAG(  81,  0, ECX, 22,  1, TOPOLOGY,                               NO,  FALSE) \
+FLAG(  81,  0, ECX, 23,  1, PERFCORE,                               ANY, TRUE)  \
+FLAG(  81,  0, EDX,  0,  1, LEAF81_FPU,                             YES, TRUE)  \
+FLAG(  81,  0, EDX,  1,  1, LEAF81_VME,                             YES, FALSE) \
+FLAG(  81,  0, EDX,  2,  1, LEAF81_DE,                              YES, FALSE) \
+FLAG(  81,  0, EDX,  3,  1, LEAF81_PSE,                             YES, FALSE) \
+FLAG(  81,  0, EDX,  4,  1, LEAF81_TSC,                             YES, TRUE)  \
+FLAG(  81,  0, EDX,  5,  1, LEAF81_MSR,                             YES, FALSE) \
+FLAG(  81,  0, EDX,  6,  1, LEAF81_PAE,                             YES, FALSE) \
+FLAG(  81,  0, EDX,  7,  1, LEAF81_MCE,                             YES, FALSE) \
+FLAG(  81,  0, EDX,  8,  1, LEAF81_CX8,                             YES, TRUE)  \
+FLAG(  81,  0, EDX,  9,  1, LEAF81_APIC,                            ANY, FALSE) \
+FLAG(  81,  0, EDX, 11,  1, SYSC,                                   ANY, TRUE)  \
+FLAG(  81,  0, EDX, 12,  1, LEAF81_MTRR,                            YES, FALSE) \
+FLAG(  81,  0, EDX, 13,  1, LEAF81_PGE,                             YES, FALSE) \
+FLAG(  81,  0, EDX, 14,  1, LEAF81_MCA,                             YES, FALSE) \
+FLAG(  81,  0, EDX, 15,  1, LEAF81_CMOV,                            YES, TRUE)  \
+FLAG(  81,  0, EDX, 16,  1, LEAF81_PAT,                             YES, FALSE) \
+FLAG(  81,  0, EDX, 17,  1, LEAF81_PSE36,                           YES, FALSE) \
+FLAG(  81,  0, EDX, 20,  1, NX,                                     YES, FALSE) \
+FLAG(  81,  0, EDX, 22,  1, MMXEXT,                                 YES, TRUE)  \
+FLAG(  81,  0, EDX, 23,  1, LEAF81_MMX,                             YES, TRUE)  \
+FLAG(  81,  0, EDX, 24,  1, LEAF81_FXSR,                            YES, TRUE)  \
+FLAG(  81,  0, EDX, 25,  1, FFXSR,                                  YES, FALSE) \
+FLAG(  81,  0, EDX, 26,  1, PDPE1GB,                                YES, FALSE) \
+FLAG(  81,  0, EDX, 27,  1, RDTSCP,                                 YES, TRUE)  \
+FLAG(  81,  0, EDX, 29,  1, LM,                                     YES, FALSE) \
+FLAG(  81,  0, EDX, 30,  1, 3DNOWPLUS,                              YES, TRUE)  \
+FLAG(  81,  0, EDX, 31,  1, 3DNOW,                                  YES, TRUE)
+
+#define CPUID_8A_EDX_11 \
+FLAG(  8A,  0, EDX, 11,  1, SVMEDX_RSVD1,                           NO,  FALSE)
+#define CPUID_8A_EDX_13_31 \
+FIELD( 8A,  0, EDX, 13, 19, SVMEDX_RSVD2,                           NO,  FALSE)
+
+/*    LEVEL, REG, POS, SIZE, NAME,                              MON SUPP, CPL3 */
+#define CPUID_FIELD_DATA_LEVEL_8x                                               \
+FIELD( 85,  0, EAX,  0,  8, ITLB_ENTRIES_2M4M_PGS,                  NA,  FALSE) \
+FIELD( 85,  0, EAX,  8,  8, ITLB_ASSOC_2M4M_PGS,                    NA,  FALSE) \
+FIELD( 85,  0, EAX, 16,  8, DTLB_ENTRIES_2M4M_PGS,                  NA,  FALSE) \
+FIELD( 85,  0, EAX, 24,  8, DTLB_ASSOC_2M4M_PGS,                    NA,  FALSE) \
+FIELD( 85,  0, EBX,  0,  8, ITLB_ENTRIES_4K_PGS,                    NA,  FALSE) \
+FIELD( 85,  0, EBX,  8,  8, ITLB_ASSOC_4K_PGS,                      NA,  FALSE) \
+FIELD( 85,  0, EBX, 16,  8, DTLB_ENTRIES_4K_PGS,                    NA,  FALSE) \
+FIELD( 85,  0, EBX, 24,  8, DTLB_ASSOC_4K_PGS,                      NA,  FALSE) \
+FIELD( 85,  0, ECX,  0,  8, L1_DCACHE_LINE_SIZE,                    NA,  FALSE) \
+FIELD( 85,  0, ECX,  8,  8, L1_DCACHE_LINES_PER_TAG,                NA,  FALSE) \
+FIELD( 85,  0, ECX, 16,  8, L1_DCACHE_ASSOC,                        NA,  FALSE) \
+FIELD( 85,  0, ECX, 24,  8, L1_DCACHE_SIZE,                         NA,  FALSE) \
+FIELD( 85,  0, EDX,  0,  8, L1_ICACHE_LINE_SIZE,                    NA,  FALSE) \
+FIELD( 85,  0, EDX,  8,  8, L1_ICACHE_LINES_PER_TAG,                NA,  FALSE) \
+FIELD( 85,  0, EDX, 16,  8, L1_ICACHE_ASSOC,                        NA,  FALSE) \
+FIELD( 85,  0, EDX, 24,  8, L1_ICACHE_SIZE,                         NA,  FALSE) \
+FIELD( 86,  0, EAX,  0, 12, L2_ITLB_ENTRIES_2M4M_PGS,               NA,  FALSE) \
+FIELD( 86,  0, EAX, 12,  4, L2_ITLB_ASSOC_2M4M_PGS,                 NA,  FALSE) \
+FIELD( 86,  0, EAX, 16, 12, L2_DTLB_ENTRIES_2M4M_PGS,               NA,  FALSE) \
+FIELD( 86,  0, EAX, 28,  4, L2_DTLB_ASSOC_2M4M_PGS,                 NA,  FALSE) \
+FIELD( 86,  0, EBX,  0, 12, L2_ITLB_ENTRIES_4K_PGS,                 NA,  FALSE) \
+FIELD( 86,  0, EBX, 12,  4, L2_ITLB_ASSOC_4K_PGS,                   NA,  FALSE) \
+FIELD( 86,  0, EBX, 16, 12, L2_DTLB_ENTRIES_4K_PGS,                 NA,  FALSE) \
+FIELD( 86,  0, EBX, 28,  4, L2_DTLB_ASSOC_4K_PGS,                   NA,  FALSE) \
+FIELD( 86,  0, ECX,  0,  8, L2CACHE_LINE,                           NA,  FALSE) \
+FIELD( 86,  0, ECX,  8,  4, L2CACHE_LINE_PER_TAG,                   NA,  FALSE) \
+FIELD( 86,  0, ECX, 12,  4, L2CACHE_WAYS,                           NA,  FALSE) \
+FIELD( 86,  0, ECX, 16, 16, L2CACHE_SIZE,                           NA,  FALSE) \
+FIELD( 86,  0, EDX,  0,  8, L3CACHE_LINE,                           NA,  FALSE) \
+FIELD( 86,  0, EDX,  8,  4, L3CACHE_LINE_PER_TAG,                   NA,  FALSE) \
+FIELD( 86,  0, EDX, 12,  4, L3CACHE_WAYS,                           NA,  FALSE) \
+FIELD( 86,  0, EDX, 18, 14, L3CACHE_SIZE,                           NA,  FALSE) \
+FLAG(  87,  0, EDX,  0,  1, TS,                                     NA,  FALSE) \
+FLAG(  87,  0, EDX,  1,  1, FID,                                    NA,  FALSE) \
+FLAG(  87,  0, EDX,  2,  1, VID,                                    NA,  FALSE) \
+FLAG(  87,  0, EDX,  3,  1, TTP,                                    NA,  FALSE) \
+FLAG(  87,  0, EDX,  4,  1, LEAF87_TM,                              NA,  FALSE) \
+FLAG(  87,  0, EDX,  5,  1, STC,                                    NA,  FALSE) \
+FLAG(  87,  0, EDX,  6,  1, 100MHZSTEPS,                            NA,  FALSE) \
+FLAG(  87,  0, EDX,  7,  1, HWPSTATE,                               NA,  FALSE) \
+FLAG(  87,  0, EDX,  8,  1, TSC_INVARIANT,                          NA,  FALSE) \
+FLAG(  87,  0, EDX,  9,  1, CORE_PERF_BOOST,                        NA,  FALSE) \
+FIELD( 88,  0, EAX,  0,  8, PHYS_BITS,                              NA,  FALSE) \
+FIELD( 88,  0, EAX,  8,  8, VIRT_BITS,                              NA,  FALSE) \
+FIELD( 88,  0, EAX, 16,  8, GUEST_PHYS_ADDR_SZ,                     NA,  FALSE) \
+FIELD( 88,  0, ECX,  0,  8, LEAF88_CORE_COUNT,                      NA,  FALSE) \
+FIELD( 88,  0, ECX, 12,  4, APICID_COREID_SIZE,                     NA,  FALSE) \
+FIELD( 8A,  0, EAX,  0,  8, SVM_REVISION,                           YES, FALSE) \
+FLAG(  8A,  0, EAX,  8,  1, SVM_HYPERVISOR,                         NO,  FALSE) \
+FIELD( 8A,  0, EAX,  9, 23, SVMEAX_RSVD,                            NO,  FALSE) \
+FIELD( 8A,  0, EBX,  0, 32, SVM_NUM_ASIDS,                          YES, FALSE) \
+FIELD( 8A,  0, ECX,  0, 32, SVMECX_RSVD,                            NO,  FALSE) \
+FLAG(  8A,  0, EDX,  0,  1, SVM_NPT,                                YES, FALSE) \
+FLAG(  8A,  0, EDX,  1,  1, SVM_LBR,                                NO,  FALSE) \
+FLAG(  8A,  0, EDX,  2,  1, SVM_LOCK,                               ANY, FALSE) \
+FLAG(  8A,  0, EDX,  3,  1, SVM_NRIP,                               YES, FALSE) \
+FLAG(  8A,  0, EDX,  4,  1, SVM_TSC_RATE_MSR,                       NO,  FALSE) \
+FLAG(  8A,  0, EDX,  5,  1, SVM_VMCB_CLEAN,                         YES, FALSE) \
+FLAG(  8A,  0, EDX,  6,  1, SVM_FLUSH_BY_ASID,                      YES, FALSE) \
+FLAG(  8A,  0, EDX,  7,  1, SVM_DECODE_ASSISTS,                     YES, FALSE) \
+FIELD( 8A,  0, EDX,  8,  2, SVMEDX_RSVD0,                           NO,  FALSE) \
+FLAG(  8A,  0, EDX, 10,  1, SVM_PAUSE_FILTER,                       NO,  FALSE) \
+CPUID_8A_EDX_11 \
+FLAG(  8A,  0, EDX, 12,  1, SVM_PAUSE_THRESHOLD,                    NO,  FALSE) \
+CPUID_8A_EDX_13_31
+
+/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,                   MON SUPP, CPL3 */
+#define CPUID_FIELD_DATA_LEVEL_81x                                              \
+FIELD(819,  0, EAX,  0, 12, L1_ITLB_ENTRIES_1G_PGS,                 NA,  FALSE) \
+FIELD(819,  0, EAX, 12,  4, L1_ITLB_ASSOC_1G_PGS,                   NA,  FALSE) \
+FIELD(819,  0, EAX, 16, 12, L1_DTLB_ENTRIES_1G_PGS,                 NA,  FALSE) \
+FIELD(819,  0, EAX, 28,  4, L1_DTLB_ASSOC_1G_PGS,                   NA,  FALSE) \
+FIELD(819,  0, EBX,  0, 12, L2_ITLB_ENTRIES_1G_PGS,                 NA,  FALSE) \
+FIELD(819,  0, EBX, 12,  4, L2_ITLB_ASSOC_1G_PGS,                   NA,  FALSE) \
+FIELD(819,  0, EBX, 16, 12, L2_DTLB_ENTRIES_1G_PGS,                 NA,  FALSE) \
+FIELD(819,  0, EBX, 28,  4, L2_DTLB_ASSOC_1G_PGS,                   NA,  FALSE) \
+FLAG( 81A,  0, EAX,  0,  1, FP128,                                  NA,  FALSE) \
+FLAG( 81A,  0, EAX,  1,  1, MOVU,                                   NA,  FALSE) \
+FLAG( 81B,  0, EAX,  0,  1, IBS_FFV,                                NA,  FALSE) \
+FLAG( 81B,  0, EAX,  1,  1, IBS_FETCHSAM,                           NA,  FALSE) \
+FLAG( 81B,  0, EAX,  2,  1, IBS_OPSAM,                              NA,  FALSE) \
+FLAG( 81B,  0, EAX,  3,  1, RW_OPCOUNT,                             NA,  FALSE) \
+FLAG( 81B,  0, EAX,  4,  1, OPCOUNT,                                NA,  FALSE) \
+FLAG( 81B,  0, EAX,  5,  1, BRANCH_TARGET_ADDR,                     NA,  FALSE) \
+FLAG( 81B,  0, EAX,  6,  1, OPCOUNT_EXT,                            NA,  FALSE) \
+FLAG( 81B,  0, EAX,  7,  1, RIP_INVALID_CHECK,                      NA,  FALSE) \
+FLAG( 81C,  0, EAX,  0,  1, LWP_AVAIL,                              NA,  FALSE) \
+FLAG( 81C,  0, EAX,  1,  1, LWP_VAL_AVAIL,                          NA,  FALSE) \
+FLAG( 81C,  0, EAX,  2,  1, LWP_IRE_AVAIL,                          NA,  FALSE) \
+FLAG( 81C,  0, EAX,  3,  1, LWP_BRE_AVAIL,                          NA,  FALSE) \
+FLAG( 81C,  0, EAX,  4,  1, LWP_DME_AVAIL,                          NA,  FALSE) \
+FLAG( 81C,  0, EAX,  5,  1, LWP_CNH_AVAIL,                          NA,  FALSE) \
+FLAG( 81C,  0, EAX,  6,  1, LWP_RNH_AVAIL,                          NA,  FALSE) \
+FLAG( 81C,  0, EAX, 31,  1, LWP_INT_AVAIL,                          NA,  FALSE) \
+FIELD(81C,  0, EBX,  0,  8, LWP_CB_SIZE,                            NA,  FALSE) \
+FIELD(81C,  0, EBX,  8,  8, LWP_EVENT_SIZE,                         NA,  FALSE) \
+FIELD(81C,  0, EBX, 16,  8, LWP_MAX_EVENTS,                         NA,  FALSE) \
+FIELD(81C,  0, EBX, 24,  8, LWP_EVENT_OFFSET,                       NA,  FALSE) \
+FIELD(81C,  0, ECX,  0,  4, LWP_LATENCY_MAX,                        NA,  FALSE) \
+FLAG( 81C,  0, ECX,  5,  1, LWP_DATA_ADDR_VALID,                    NA,  FALSE) \
+FIELD(81C,  0, ECX,  6,  3, LWP_LATENCY_ROUND,                      NA,  FALSE) \
+FIELD(81C,  0, ECX,  9,  7, LWP_VERSION,                            NA,  FALSE) \
+FIELD(81C,  0, ECX, 16,  8, LWP_MIN_BUF_SIZE,                       NA,  FALSE) \
+FLAG( 81C,  0, ECX, 28,  1, LWP_BRANCH_PRED,                        NA,  FALSE) \
+FLAG( 81C,  0, ECX, 29,  1, LWP_IP_FILTERING,                       NA,  FALSE) \
+FLAG( 81C,  0, ECX, 30,  1, LWP_CACHE_LEVEL,                        NA,  FALSE) \
+FLAG( 81C,  0, ECX, 31,  1, LWP_CACHE_LATENCY,                      NA,  FALSE) \
+FLAG( 81C,  0, EDX,  0,  1, LWP_SUPPORTED,                          NA,  FALSE) \
+FLAG( 81C,  0, EDX,  1,  1, LWP_VAL_SUPPORTED,                      NA,  FALSE) \
+FLAG( 81C,  0, EDX,  2,  1, LWP_IRE_SUPPORTED,                      NA,  FALSE) \
+FLAG( 81C,  0, EDX,  3,  1, LWP_BRE_SUPPORTED,                      NA,  FALSE) \
+FLAG( 81C,  0, EDX,  4,  1, LWP_DME_SUPPORTED,                      NA,  FALSE) \
+FLAG( 81C,  0, EDX,  5,  1, LWP_CNH_SUPPORTED,                      NA,  FALSE) \
+FLAG( 81C,  0, EDX,  6,  1, LWP_RNH_SUPPORTED,                      NA,  FALSE) \
+FLAG( 81C,  0, EDX, 31,  1, LWP_INT_SUPPORTED,                      NA,  FALSE) \
+FIELD(81D,  0, EAX,  0,  5, LEAF81D_CACHE_TYPE,                     NA,  FALSE) \
+FIELD(81D,  0, EAX,  5,  3, LEAF81D_CACHE_LEVEL,                    NA,  FALSE) \
+FLAG( 81D,  0, EAX,  8,  1, LEAF81D_CACHE_SELF_INIT,                NA,  FALSE) \
+FLAG( 81D,  0, EAX,  9,  1, LEAF81D_CACHE_FULLY_ASSOC,              NA,  FALSE) \
+FIELD(81D,  0, EAX, 14, 12, LEAF81D_NUM_SHARING_CACHE,              NA,  FALSE) \
+FIELD(81D,  0, EBX,  0, 12, LEAF81D_CACHE_LINE_SIZE,                NA,  FALSE) \
+FIELD(81D,  0, EBX, 12, 10, LEAF81D_CACHE_PHYS_PARTITIONS,          NA,  FALSE) \
+FIELD(81D,  0, EBX, 22, 10, LEAF81D_CACHE_WAYS,                     NA,  FALSE) \
+FIELD(81D,  0, ECX,  0, 32, LEAF81D_CACHE_NUM_SETS,                 NA,  FALSE) \
+FLAG( 81D,  0, EDX,  0,  1, LEAF81D_CACHE_WBINVD,                   NA,  FALSE) \
+FLAG( 81D,  0, EDX,  1,  1, LEAF81D_CACHE_INCLUSIVE,                NA,  FALSE) \
+FIELD(81E,  0, EAX,  0, 32, EXTENDED_APICID,                        NA,  FALSE) \
+FIELD(81E,  0, EBX,  0,  8, COMPUTE_UNIT_ID,                        NA,  FALSE) \
+FIELD(81E,  0, EBX,  8,  2, CORES_PER_COMPUTE_UNIT,                 NA,  FALSE) \
+FIELD(81E,  0, ECX,  0,  8, NODEID_VAL,                             NA,  FALSE) \
+FIELD(81E,  0, ECX,  8,  3, NODES_PER_PKG,                          NA,  FALSE)
 #define INTEL_CPUID_FIELD_DATA
 
 #define AMD_CPUID_FIELD_DATA
@@ -665,41 +699,33 @@ FIELDDEFA( 81E,ECX, AMD,     8,  3, NODES_PER_PKG,       NA,  FALSE, AMD_NODES_P
  * CPUID_<vendor>_ID<level><reg>_<name>_SHIFT   == offset of field
  *
  * e.g. - CPUID_FEATURE_COMMON_ID1EDX_FPU     = 0x1
- *      - CPUID_COMMON_ID88EAX_VIRTBITS_MASK  = 0xff00
- *      - CPUID_COMMON_ID88EAX_VIRTBITS_SHIFT = 8
+ *      - CPUID_COMMON_ID88EAX_VIRT_BITS_MASK  = 0xff00
+ *      - CPUID_COMMON_ID88EAX_VIRT_BITS_SHIFT = 8
  *
  * Note: The FEATURE/MASK definitions must use some gymnastics to get
  * around a warning when shifting left by 32.
  */
 #define VMW_BIT_MASK(shift)  (((1 << (shift - 1)) << 1) - 1)
 
-#define FIELDDEF(lvl, reg, vend, bitpos, size, name, s, c3)             \
-   CPUID_##vend##_ID##lvl##reg##_##name##_SHIFT = bitpos,               \
-   CPUID_##vend##_ID##lvl##reg##_##name##_MASK  =                       \
-                      VMW_BIT_MASK(size) << bitpos,                     \
-   CPUID_FEATURE_##vend##_ID##lvl##reg##_##name =                       \
-                      CPUID_##vend##_ID##lvl##reg##_##name##_MASK,
+#define FIELD(lvl, ecxIn, reg, bitpos, size, name, s, c3)                    \
+   CPUID_ID##lvl##reg##_##name##_SHIFT = bitpos,                             \
+   CPUID_ID##lvl##reg##_##name##_MASK  = VMW_BIT_MASK(size) << bitpos,       \
+   CPUID_FEATURE_ID##lvl##reg##_##name = CPUID_ID##lvl##reg##_##name##_MASK, \
+   CPUID_INTERNAL_SHIFT_##name         = bitpos,                             \
+   CPUID_INTERNAL_MASK_##name          = VMW_BIT_MASK(size) << bitpos,       \
+   CPUID_INTERNAL_REG_##name           = CPUID_REG_##reg,                    \
+   CPUID_INTERNAL_EAXIN_##name         = CPUID_LEVEL_VAL_##lvl,              \
+   CPUID_INTERNAL_ECXIN_##name         = ecxIn,
 
-/* Before simplifying this take a look at bug 293638... */
-#define FIELDDEFA(lvl, reg, vend, bitpos, size, name, s, c3, f)         \
-   CPUID_##vend##_ID##lvl##reg##_##name##_SHIFT = bitpos,               \
-   CPUID_##vend##_ID##lvl##reg##_##name##_MASK  =                       \
-                      VMW_BIT_MASK(size) << bitpos,                     \
-   CPUID_FEATURE_##vend##_ID##lvl##reg##_##name =                       \
-                      CPUID_##vend##_ID##lvl##reg##_##name##_MASK,
-
-#define FLAGDEFA FIELDDEFA
-#define FLAGDEF FIELDDEF
+#define FLAG FIELD
 
 enum {
    /* Define data for every CPUID field we have */
    CPUID_FIELD_DATA
 };
 #undef VMW_BIT_MASK
-#undef FIELDDEF
-#undef FLAGDEF
-#undef FIELDDEFA
-#undef FLAGDEFA
+#undef FIELD
+#undef FLAG
 
 /* Level D subleaf 1 eax XSAVEOPT */
 #define CPUID_COMMON_IDDsub1EAX_XSAVEOPT 1
@@ -747,59 +773,136 @@ enum {
     NULL)
 
 /*
- * Macro to define GET and SET functions for various common CPUID
- * fields.  To create function for a new field, simply name it (CPUID_
- * and CPUID_SET_ are automatically prepended), and list the field
- * name that it needs to use.
+ * CPUID_MASK --
+ * CPUID_SHIFT --
+ * CPUID_ISSET --
+ * CPUID_GET --
+ * CPUID_SET --
+ * CPUID_CLEAR --
+ * CPUID_SETTO --
+ *
+ * Accessor macros for all CPUID consts/fields/flags.  Level and reg are not
+ * required, but are used to force compile-time asserts which help verify that
+ * the flag is being used on the right CPUID input and result register.
+ *
+ * Note: ASSERT_ON_COMPILE is duplicated rather than factored into its own
+ * macro, because token concatenation does not work as expected if an input is
+ * #defined (e.g. APIC) when macros are nested.  Also, compound statements
+ * within parenthes is a GCC extension, so we must use runtime asserts with
+ * other compilers.
+ */
+
+#if defined(__GNUC__) && !defined(__clang__)
+
+#define CPUID_MASK(eaxIn, reg, flag)                                    \
+   ({                                                                   \
+      ASSERT_ON_COMPILE(eaxIn == CPUID_INTERNAL_EAXIN_##flag &&         \
+              CPUID_REG_##reg == (CpuidReg)CPUID_INTERNAL_REG_##flag);  \
+      CPUID_INTERNAL_MASK_##flag;                                       \
+   })
+
+#define CPUID_SHIFT(eaxIn, reg, flag)                                   \
+   ({                                                                   \
+      ASSERT_ON_COMPILE(eaxIn == CPUID_INTERNAL_EAXIN_##flag &&         \
+              CPUID_REG_##reg == (CpuidReg)CPUID_INTERNAL_REG_##flag);  \
+      CPUID_INTERNAL_SHIFT_##flag;                                      \
+   })
+
+#define CPUID_ISSET(eaxIn, reg, flag, data)                             \
+   ({                                                                   \
+      ASSERT_ON_COMPILE(eaxIn == CPUID_INTERNAL_EAXIN_##flag &&         \
+              CPUID_REG_##reg == (CpuidReg)CPUID_INTERNAL_REG_##flag);  \
+      (((data) & CPUID_INTERNAL_MASK_##flag) != 0);                     \
+   })
+
+#define CPUID_GET(eaxIn, reg, field, data)                              \
+   ({                                                                   \
+      ASSERT_ON_COMPILE(eaxIn == CPUID_INTERNAL_EAXIN_##field &&        \
+              CPUID_REG_##reg == (CpuidReg)CPUID_INTERNAL_REG_##field); \
+      (((uint32)(data) & CPUID_INTERNAL_MASK_##field) >>                \
+       CPUID_INTERNAL_SHIFT_##field);                                   \
+   })
+
+#else
+
+/*
+ * CPUIDCheck --
+ *
+ * Return val after verifying parameters.
  */
 
-#define FIELD_FUNC(name, field)                                 \
-   static INLINE uint32 CPUID_##name(uint32 reg)                \
-   {                                                            \
-      return (reg & field##_MASK) >> field##_SHIFT;             \
-   }                                                            \
-   static INLINE void CPUID_SET_##name(uint32 *reg, uint32 val) \
-   {                                                            \
-      *reg = (*reg & ~field##_MASK) | (val << field##_SHIFT);   \
-   }
+static INLINE uint32
+CPUIDCheck(uint32 eaxIn, uint32 eaxInCheck,
+           CpuidReg reg, CpuidReg regCheck, uint32 val)
+{
+   ASSERT(eaxIn == eaxInCheck && reg == regCheck);
+   return val;
+}
 
-FIELD_FUNC(STEPPING,         CPUID_COMMON_ID1EAX_STEPPING)
-FIELD_FUNC(MODEL,            CPUID_COMMON_ID1EAX_MODEL)
-FIELD_FUNC(FAMILY,           CPUID_COMMON_ID1EAX_FAMILY)
-FIELD_FUNC(TYPE,             CPUID_COMMON_ID1EAX_TYPE)
-FIELD_FUNC(EXTENDED_MODEL,   CPUID_COMMON_ID1EAX_EXTMODEL)
-FIELD_FUNC(EXTENDED_FAMILY,  CPUID_COMMON_ID1EAX_EXTFAMILY)
-FIELD_FUNC(LCPU_COUNT,       CPUID_COMMON_ID1EBX_LCPU_COUNT)
-FIELD_FUNC(APICID,           CPUID_COMMON_ID1EBX_APICID)
-FIELD_FUNC(PA_BITS,          CPUID_COMMON_ID88EAX_PHYSBITS)
-FIELD_FUNC(VIRT_BITS,        CPUID_COMMON_ID88EAX_VIRTBITS)
-FIELD_FUNC(SVM_REVISION,     CPUID_AMD_ID8AEAX_SVM_REVISION)
-FIELD_FUNC(SVM_N_ASIDS,      CPUID_AMD_ID8AEBX_SVM_N_ASIDS)
-FIELD_FUNC(CACHE_TYPE,       CPUID_INTEL_ID4EAX_CACHE_TYPE)
-FIELD_FUNC(INTEL_CORE_COUNT, CPUID_INTEL_ID4EAX_CORE_COUNT)
-FIELD_FUNC(AMD_CORE_COUNT,   CPUID_AMD_ID88ECX_CORE_COUNT)
-FIELD_FUNC(AMD_APICID_COREID_SIZE, CPUID_AMD_ID88ECX_APICID_COREID_SIZE)
-FIELD_FUNC(AMD_EXTAPICSPC,   CPUID_AMD_ID81ECX_EXTAPICSPC)
-FIELD_FUNC(NUM_PMCS,         CPUID_INTEL_IDAEAX_NUM_PMCS)
-FIELD_FUNC(MWAIT_MIN_SIZE,   CPUID_COMMON_ID5EAX_MWAIT_MIN_SIZE)
-FIELD_FUNC(MWAIT_MAX_SIZE,   CPUID_COMMON_ID5EBX_MWAIT_MAX_SIZE)
-FIELD_FUNC(MWAIT_C0_SUBSTATE, CPUID_INTEL_ID5EDX_MWAIT_C0_SUBSTATE)
-FIELD_FUNC(MWAIT_C1_SUBSTATE, CPUID_INTEL_ID5EDX_MWAIT_C1_SUBSTATE)
-FIELD_FUNC(MWAIT_C2_SUBSTATE, CPUID_INTEL_ID5EDX_MWAIT_C2_SUBSTATE)
-FIELD_FUNC(MWAIT_C3_SUBSTATE, CPUID_INTEL_ID5EDX_MWAIT_C3_SUBSTATE)
-FIELD_FUNC(MWAIT_C4_SUBSTATE, CPUID_INTEL_ID5EDX_MWAIT_C4_SUBSTATE)
-FIELD_FUNC(TOPOLOGY_MASK_WIDTH,         CPUID_INTEL_IDBEAX_MASK_WIDTH)
-FIELD_FUNC(TOPOLOGY_CPUS_SHARING_LEVEL, CPUID_INTEL_IDBEBX_CPUS_SHARING_LEVEL)
-FIELD_FUNC(TOPOLOGY_LEVEL_NUMBER,       CPUID_INTEL_IDBECX_LEVEL_NUMBER)
-FIELD_FUNC(TOPOLOGY_LEVEL_TYPE,         CPUID_INTEL_IDBECX_LEVEL_TYPE)
-FIELD_FUNC(TOPOLOGY_X2APIC_ID,          CPUID_INTEL_IDBEDX_X2APIC_ID)
-FIELD_FUNC(AMD_CACHE_TYPE,      CPUID_AMD_ID81DEAX_CACHE_TYPE)
-FIELD_FUNC(AMD_CACHE_LEVEL,     CPUID_AMD_ID81DEAX_CACHE_LEVEL)
-FIELD_FUNC(AMD_CACHE_WAYS,      CPUID_AMD_ID81DEBX_CACHE_WAYS)
-FIELD_FUNC(AMD_NODES_PER_PKG,   CPUID_AMD_ID81EECX_NODES_PER_PKG)
-FIELD_FUNC(AMD_NUM_SHARING_CACHE, CPUID_AMD_ID81DEAX_NUM_SHARING_CACHE)
-FIELD_FUNC(AMD_CORES_PER_COMPUTE_UNIT, CPUID_AMD_ID81EEBX_CORES_PER_COMPUTE_UNIT)
-#undef FIELD_FUNC
+#define CPUID_MASK(eaxIn, reg, flag)                                    \
+   CPUIDCheck((uint32)eaxIn, CPUID_INTERNAL_EAXIN_##flag,               \
+              CPUID_REG_##reg, (CpuidReg)CPUID_INTERNAL_REG_##flag,     \
+              CPUID_INTERNAL_MASK_##flag)
+
+#define CPUID_SHIFT(eaxIn, reg, flag)                                   \
+   CPUIDCheck((uint32)eaxIn, CPUID_INTERNAL_EAXIN_##flag,               \
+              CPUID_REG_##reg, (CpuidReg)CPUID_INTERNAL_REG_##flag,     \
+              CPUID_INTERNAL_SHIFT_##flag)
+
+#define CPUID_ISSET(eaxIn, reg, flag, data)                             \
+   (CPUIDCheck((uint32)eaxIn, CPUID_INTERNAL_EAXIN_##flag,              \
+               CPUID_REG_##reg, (CpuidReg)CPUID_INTERNAL_REG_##flag,    \
+               CPUID_INTERNAL_MASK_##flag & (data)) != 0)
+
+#define CPUID_GET(eaxIn, reg, field, data)                              \
+   CPUIDCheck((uint32)eaxIn, CPUID_INTERNAL_EAXIN_##field,              \
+              CPUID_REG_##reg, (CpuidReg)CPUID_INTERNAL_REG_##field,    \
+              ((uint32)(data) & CPUID_INTERNAL_MASK_##field) >>         \
+              CPUID_INTERNAL_SHIFT_##field)
+
+#endif
+
+
+#define CPUID_SET(eaxIn, reg, flag, dataPtr)                            \
+   do {                                                                 \
+      ASSERT_ON_COMPILE(                                                \
+         (uint32)eaxIn   == (uint32)CPUID_INTERNAL_EAXIN_##flag &&      \
+         CPUID_REG_##reg == (CpuidReg)CPUID_INTERNAL_REG_##flag);       \
+      *(dataPtr) |= CPUID_INTERNAL_MASK_##flag;                         \
+   } while (0)
+
+#define CPUID_CLEAR(eaxIn, reg, flag, dataPtr)                          \
+   do {                                                                 \
+      ASSERT_ON_COMPILE(                                                \
+         (uint32)eaxIn   == (uint32)CPUID_INTERNAL_EAXIN_##flag &&      \
+         CPUID_REG_##reg == (CpuidReg)CPUID_INTERNAL_REG_##flag);       \
+      *(dataPtr) &= ~CPUID_INTERNAL_MASK_##flag;                        \
+   } while (0)
+
+#define CPUID_SETTO(eaxIn, reg, field, dataPtr, val)                    \
+   do {                                                                 \
+      uint32 _v = val;                                                  \
+      uint32 *_d = dataPtr;                                             \
+      ASSERT_ON_COMPILE(                                                \
+         (uint32)eaxIn   == (uint32)CPUID_INTERNAL_EAXIN_##field &&     \
+         CPUID_REG_##reg == (CpuidReg)CPUID_INTERNAL_REG_##field);      \
+      *_d = (*_d & ~CPUID_INTERNAL_MASK_##field) |                      \
+         (_v << CPUID_INTERNAL_SHIFT_##field);                          \
+      ASSERT(_v == (*_d & CPUID_INTERNAL_MASK_##field) >>               \
+             CPUID_INTERNAL_SHIFT_##field);                             \
+   } while (0)
+
+#define CPUID_SETTO_SAFE(eaxIn, reg, field, dataPtr, val)               \
+   do {                                                                 \
+      uint32 _v = val &                                                 \
+         (CPUID_INTERNAL_MASK_##field >> CPUID_INTERNAL_SHIFT_##field); \
+      uint32 *_d = dataPtr;                                             \
+      ASSERT_ON_COMPILE(                                                \
+         (uint32)eaxIn   == (uint32)CPUID_INTERNAL_EAXIN_##field &&     \
+         CPUID_REG_##reg == (CpuidReg)CPUID_INTERNAL_REG_##field);      \
+      *_d = (*_d & ~CPUID_INTERNAL_MASK_##field) |                      \
+         (_v << CPUID_INTERNAL_SHIFT_##field);                          \
+   } while (0)
 
 
 /*
@@ -807,54 +910,66 @@ FIELD_FUNC(AMD_CORES_PER_COMPUTE_UNIT, CPUID_AMD_ID81EEBX_CORES_PER_COMPUTE_UNIT
  * macros/functions for reading cpuid fields.
  */
 
-#define CPUID_FAMILY_EXTENDED 15
+#define CPUID_FAMILY_EXTENDED        15
 
 /* Effective Intel CPU Families */
-#define CPUID_FAMILY_486      4
-#define CPUID_FAMILY_P5       5
-#define CPUID_FAMILY_P6       6
-#define CPUID_FAMILY_P4       15
+#define CPUID_FAMILY_486              4
+#define CPUID_FAMILY_P5               5
+#define CPUID_FAMILY_P6               6
+#define CPUID_FAMILY_P4              15
 
 /* Effective AMD CPU Families */
-#define CPUID_FAMILY_5x86     4
-#define CPUID_FAMILY_K5       5
-#define CPUID_FAMILY_K6       5
-#define CPUID_FAMILY_K7       6
-#define CPUID_FAMILY_K8       15
-#define CPUID_FAMILY_K8L      16
-#define CPUID_FAMILY_K8MOBILE 17
-#define CPUID_FAMILY_LLANO    18
-#define CPUID_FAMILY_BOBCAT   20
-#define CPUID_FAMILY_BULLDOZER 21
+#define CPUID_FAMILY_5x86             4
+#define CPUID_FAMILY_K5               5 
+#define CPUID_FAMILY_K6               5
+#define CPUID_FAMILY_K7               6
+#define CPUID_FAMILY_K8              15
+#define CPUID_FAMILY_K8L             16
+#define CPUID_FAMILY_K8MOBILE        17
+#define CPUID_FAMILY_LLANO           18
+#define CPUID_FAMILY_BOBCAT          20
+#define CPUID_FAMILY_BULLDOZER       21
 
 /* Effective VIA CPU Families */
-#define CPUID_FAMILY_C7       6
+#define CPUID_FAMILY_C7               6
 
 /* Intel model information */
-#define CPUID_MODEL_PPRO       1
-#define CPUID_MODEL_PII_03     3
-#define CPUID_MODEL_PII_05     5
-#define CPUID_MODEL_CELERON_06 6
-#define CPUID_MODEL_PM_09      9
-#define CPUID_MODEL_PM_0D      13
-#define CPUID_MODEL_PM_0E      14    // Yonah / Sossaman
-#define CPUID_MODEL_CORE_0F    15    // Conroe / Merom
-#define CPUID_MODEL_CORE_17    0x17  // Penryn
-#define CPUID_MODEL_NEHALEM_1A 0x1a  // Nehalem / Gainestown
-#define CPUID_MODEL_ATOM_1C    0x1c  // Silverthorne / Diamondville
-#define CPUID_MODEL_CORE_1D    0x1d  // Dunnington
-#define CPUID_MODEL_NEHALEM_1E 0x1e  // Lynnfield
-#define CPUID_MODEL_NEHALEM_1F 0x1f  // Havendale
-#define CPUID_MODEL_NEHALEM_25 0x25  // Westmere / Clarkdale
-#define CPUID_MODEL_NEHALEM_2C 0x2c  // Westmere-EP
-#define CPUID_MODEL_NEHALEM_2E 0x2e  // Nehalem-EX
+#define CPUID_MODEL_PPRO              1
+#define CPUID_MODEL_PII_03            3
+#define CPUID_MODEL_PII_05            5
+#define CPUID_MODEL_CELERON_06        6
+#define CPUID_MODEL_PM_09             9
+#define CPUID_MODEL_PM_0D            13
+#define CPUID_MODEL_PM_0E            14  // Yonah / Sossaman
+#define CPUID_MODEL_CORE_0F          15  // Conroe / Merom
+#define CPUID_MODEL_CORE_17        0x17  // Penryn
+#define CPUID_MODEL_NEHALEM_1A     0x1a  // Nehalem / Gainestown
+#define CPUID_MODEL_ATOM_1C        0x1c  // Silverthorne / Diamondville
+#define CPUID_MODEL_CORE_1D        0x1d  // Dunnington
+#define CPUID_MODEL_NEHALEM_1E     0x1e  // Lynnfield
+#define CPUID_MODEL_NEHALEM_1F     0x1f  // Havendale
+#define CPUID_MODEL_NEHALEM_25     0x25  // Westmere / Clarkdale
+#define CPUID_MODEL_SANDYBRIDGE_2A 0x2a  // Sandybridge (desktop/mobile)
+#define CPUID_MODEL_SANDYBRIDGE_2D 0x2d  // Sandybridge-EP
+#define CPUID_MODEL_NEHALEM_2C     0x2c  // Westmere-EP
+#define CPUID_MODEL_NEHALEM_2E     0x2e  // Nehalem-EX
+#define CPUID_MODEL_NEHALEM_2F     0x2f  // Westmere-EX
+#define CPUID_MODEL_SANDYBRIDGE_3A 0x3a  // Ivy Bridge
+
 
 #define CPUID_MODEL_PIII_07    7
 #define CPUID_MODEL_PIII_08    8
 #define CPUID_MODEL_PIII_0A    10
 
 /* AMD model information */
-#define CPUID_MODEL_BARCELONA_02 0x02 // Barcelona (both Opteron & Phenom kind)
+#define CPUID_MODEL_BARCELONA_02      0x02 // Barcelona (Opteron & Phenom)
+#define CPUID_MODEL_SHANGHAI_04       0x04 // Shanghai RB
+#define CPUID_MODEL_SHANGHAI_05       0x05 // Shanghai BL
+#define CPUID_MODEL_SHANGHAI_06       0x06 // Shanghai DA
+#define CPUID_MODEL_ISTANBUL_MAGNY_08 0x08 // Istanbul (6 core) & Magny-cours (12) HY
+#define CPUID_MODEL_ISTANBUL_MAGNY_09 0x09 // HY - G34 package
+#define CPUID_MODEL_PHAROAH_HOUND_0A  0x0A // Pharoah Hound
+#define CPUID_MODEL_OPTERON_REVF_41   0x41 // family == CPUID_FAMILY_K8
 
 /* VIA model information */
 #define CPUID_MODEL_NANO       15     // Isaiah
@@ -880,7 +995,7 @@ CPUID_IsRawVendor(CPUIDRegs *id0, const char* vendor)
    // hard to get strcmp() in some environments, so do it in the raw
    return (id0->ebx == *(const uint32 *) (vendor + 0) &&
            id0->ecx == *(const uint32 *) (vendor + 4) &&
-           id0->edx == *(const uint32 *) (vendor + 8));
+           id0->edx == *(const uint32 *) (vendor + 8));   
 }
 
 static INLINE Bool
@@ -904,8 +1019,9 @@ CPUID_IsVendorVIA(CPUIDRegs *id0)
 static INLINE uint32
 CPUID_EFFECTIVE_FAMILY(uint32 v) /* %eax from CPUID with %eax=1. */
 {
-   return CPUID_FAMILY(v) +
-      (CPUID_FAMILY(v) == CPUID_FAMILY_EXTENDED ? CPUID_EXTENDED_FAMILY(v) : 0);
+   uint32 f = CPUID_GET(1, EAX, FAMILY, v);
+   return f != CPUID_FAMILY_EXTENDED ? f : f +
+      CPUID_GET(1, EAX, EXTENDED_FAMILY, v);
 }
 
 /* Normally only used when FAMILY==CPUID_FAMILY_EXTENDED, but Intel is
@@ -915,7 +1031,9 @@ CPUID_EFFECTIVE_FAMILY(uint32 v) /* %eax from CPUID with %eax=1. */
 static INLINE uint32
 CPUID_EFFECTIVE_MODEL(uint32 v) /* %eax from CPUID with %eax=1. */
 {
-   return CPUID_MODEL(v) + (CPUID_EXTENDED_MODEL(v) << 4);
+   uint32 m = CPUID_GET(1, EAX, MODEL, v);
+   uint32 em = CPUID_GET(1, EAX, EXTENDED_MODEL, v);
+   return m + (em << 4); 
 }
 
 /*
@@ -985,13 +1103,72 @@ CPUID_UARCH_IS_NEHALEM(uint32 v) // IN: %eax from CPUID with %eax=1.
    uint32 effectiveModel = CPUID_EFFECTIVE_MODEL(v);
 
    return CPUID_FAMILY_IS_P6(v) &&
-          (
-           effectiveModel == CPUID_MODEL_NEHALEM_1A ||
+          (effectiveModel == CPUID_MODEL_NEHALEM_1A ||
            effectiveModel == CPUID_MODEL_NEHALEM_1E ||
            effectiveModel == CPUID_MODEL_NEHALEM_1F ||
            effectiveModel == CPUID_MODEL_NEHALEM_25 ||
            effectiveModel == CPUID_MODEL_NEHALEM_2C ||
-           effectiveModel == CPUID_MODEL_NEHALEM_2E);
+           effectiveModel == CPUID_MODEL_NEHALEM_2E ||
+           effectiveModel == CPUID_MODEL_NEHALEM_2F);
+}
+
+
+static INLINE Bool
+CPUID_UARCH_IS_SANDYBRIDGE(uint32 v) // IN: %eax from CPUID with %eax=1.
+{
+   /* Assumes the CPU manufacturer is Intel. */
+   uint32 effectiveModel = CPUID_EFFECTIVE_MODEL(v);
+
+   return CPUID_FAMILY_IS_P6(v) &&
+          (effectiveModel == CPUID_MODEL_SANDYBRIDGE_2A ||
+           effectiveModel == CPUID_MODEL_SANDYBRIDGE_2D ||
+           effectiveModel == CPUID_MODEL_SANDYBRIDGE_3A);
+}
+
+
+
+static INLINE Bool
+CPUID_MODEL_IS_CENTERTON(uint32 v) // IN: %eax from CPUID with %eax=1.
+{
+   /* Assumes the CPU manufacturer is Intel. */
+   return CPUID_FAMILY_IS_P6(v) && 
+          CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_ATOM_1C;
+}
+
+
+static INLINE Bool
+CPUID_MODEL_IS_WESTMERE(uint32 v) // IN: %eax from CPUID with %eax=1.
+{
+   /* Assumes the CPU manufacturer is Intel. */
+   uint32 effectiveModel = CPUID_EFFECTIVE_MODEL(v);
+
+   return CPUID_FAMILY_IS_P6(v) &&
+          (effectiveModel == CPUID_MODEL_NEHALEM_25 || // Clarkdale
+           effectiveModel == CPUID_MODEL_NEHALEM_2C || // Westmere-EP
+           effectiveModel == CPUID_MODEL_NEHALEM_2F);  // Westmere-EX
+}
+
+
+static INLINE Bool
+CPUID_MODEL_IS_SANDYBRIDGE(uint32 v) // IN: %eax from CPUID with %eax=1.
+{
+   /* Assumes the CPU manufacturer is Intel. */
+   uint32 effectiveModel = CPUID_EFFECTIVE_MODEL(v);
+
+   return CPUID_FAMILY_IS_P6(v) &&
+          (effectiveModel == CPUID_MODEL_SANDYBRIDGE_2A ||
+           effectiveModel == CPUID_MODEL_SANDYBRIDGE_2D);
+}
+
+
+static INLINE Bool
+CPUID_MODEL_IS_IVYBRIDGE(uint32 v) // IN: %eax from CPUID with %eax=1.
+{
+   /* Assumes the CPU manufacturer is Intel. */
+   uint32 effectiveModel = CPUID_EFFECTIVE_MODEL(v);
+
+   return CPUID_FAMILY_IS_P6(v) && (
+       effectiveModel == CPUID_MODEL_SANDYBRIDGE_3A);
 }
 
 
@@ -1017,7 +1194,7 @@ CPUID_FAMILY_IS_K8EXT(uint32 eax)
     * worth a separate function, for syntactic sugar.
     */
    return CPUID_FAMILY_IS_K8(eax) &&
-          CPUID_EXTENDED_MODEL(eax) != 0;
+          CPUID_GET(1, EAX, EXTENDED_MODEL, eax) != 0;
 }
 
 static INLINE Bool
@@ -1028,6 +1205,12 @@ CPUID_FAMILY_IS_K8L(uint32 eax)
 }
 
 static INLINE Bool
+CPUID_FAMILY_IS_LLANO(uint32 eax)
+{
+   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_LLANO;
+}
+
+static INLINE Bool
 CPUID_FAMILY_IS_K8MOBILE(uint32 eax)
 {
    /* Essentially a K8 (not K8L) part, but with mobile features. */
@@ -1057,6 +1240,7 @@ CPUID_FAMILY_IS_BULLDOZER(uint32 eax)
    return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_BULLDOZER;
 }
 
+
 /*
  * AMD Barcelona (of either Opteron or Phenom kind).
  */
@@ -1069,21 +1253,70 @@ CPUID_MODEL_IS_BARCELONA(uint32 v) // IN: %eax from CPUID with %eax=1.
 }
 
 
+static INLINE Bool
+CPUID_MODEL_IS_SHANGHAI(uint32 v) // IN: %eax from CPUID with %eax=1.
+{
+   /* Assumes the CPU manufacturer is AMD. */
+   return CPUID_EFFECTIVE_FAMILY(v) == CPUID_FAMILY_K8L &&
+          (CPUID_MODEL_SHANGHAI_04  <= CPUID_EFFECTIVE_MODEL(v) && 
+           CPUID_EFFECTIVE_MODEL(v) <= CPUID_MODEL_SHANGHAI_06);
+}
+
+
+static INLINE Bool
+CPUID_MODEL_IS_ISTANBUL_MAGNY(uint32 v) // IN: %eax from CPUID with %eax=1.
+{
+   /* Assumes the CPU manufacturer is AMD. */
+   return CPUID_EFFECTIVE_FAMILY(v) == CPUID_FAMILY_K8L &&
+          (CPUID_MODEL_ISTANBUL_MAGNY_08 <= CPUID_EFFECTIVE_MODEL(v) &&
+           CPUID_EFFECTIVE_MODEL(v)      <= CPUID_MODEL_ISTANBUL_MAGNY_09);
+}
+
+
+static INLINE Bool
+CPUID_MODEL_IS_PHAROAH_HOUND(uint32 v) // IN: %eax from CPUID with %eax=1.
+{
+   /* Assumes the CPU manufacturer is AMD. */
+   return CPUID_EFFECTIVE_FAMILY(v) == CPUID_FAMILY_K8L &&
+          CPUID_EFFECTIVE_MODEL(v)  == CPUID_MODEL_PHAROAH_HOUND_0A;
+}
+
+
+static INLINE Bool
+CPUID_MODEL_IS_BULLDOZER(uint32 eax)
+{
+   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_BULLDOZER;
+}
+
+
+
+
 #define CPUID_TYPE_PRIMARY     0
 #define CPUID_TYPE_OVERDRIVE   1
 #define CPUID_TYPE_SECONDARY   2
 
-#define CPUID_INTEL_ID4EAX_CACHE_TYPE_NULL      0
-#define CPUID_INTEL_ID4EAX_CACHE_TYPE_DATA      1
-#define CPUID_INTEL_ID4EAX_CACHE_TYPE_INST      2
-#define CPUID_INTEL_ID4EAX_CACHE_TYPE_UNIF      3
-
-#define CPUID_INTEL_ID4EAX_CACHE_SELF_INIT      0x00000100
-#define CPUID_INTEL_ID4EAX_CACHE_FULLY_ASSOC    0x00000200
+#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_TYPE_NULL      0
+#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_TYPE_DATA      1
+#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_TYPE_INST      2
+#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_TYPE_UNIF      3
+#define CPUID_LEAF4_CACHE_TYPE_NULL      0
+#define CPUID_LEAF4_CACHE_TYPE_DATA      1
+#define CPUID_LEAF4_CACHE_TYPE_INST      2
+#define CPUID_LEAF4_CACHE_TYPE_UNIF      3
+#define CPUID_LEAF4_CACHE_INDEXING_DIRECT  0
+#define CPUID_LEAF4_CACHE_INDEXING_COMPLEX 1
+
+#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_SELF_INIT      0x00000100
+#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_FULLY_ASSOC    0x00000200
+#define CPUID_LEAF4_CACHE_SELF_INIT      0x00000100
+#define CPUID_LEAF4_CACHE_FULLY_ASSOC    0x00000200
 
 #define CPUID_INTEL_IDBECX_LEVEL_TYPE_INVALID   0
 #define CPUID_INTEL_IDBECX_LEVEL_TYPE_SMT       1
 #define CPUID_INTEL_IDBECX_LEVEL_TYPE_CORE      2
+#define CPUID_TOPOLOGY_LEVEL_TYPE_INVALID   0
+#define CPUID_TOPOLOGY_LEVEL_TYPE_SMT       1
+#define CPUID_TOPOLOGY_LEVEL_TYPE_CORE      2
 
 
 /*
@@ -1128,7 +1361,7 @@ CPUID_RequiresFence(CpuidVendor vendor, // IN
 }
 
 
-/*
+/* 
  * The following low-level functions compute the number of
  * cores per cpu.  They should be used cautiously because
  * they do not necessarily work on all types of CPUs.
@@ -1140,7 +1373,7 @@ static INLINE uint32
 CPUID_IntelCoresPerPackage(uint32 v) /* %eax from CPUID with %eax=4 and %ecx=0. */
 {
    // Note: This is not guaranteed to work on older Intel CPUs.
-   return 1 + CPUID_INTEL_CORE_COUNT(v);
+   return 1 + CPUID_GET(4, EAX, LEAF4_CORE_COUNT, v);
 }
 
 
@@ -1148,7 +1381,7 @@ static INLINE uint32
 CPUID_AMDCoresPerPackage(uint32 v) /* %ecx from CPUID with %eax=0x80000008. */
 {
    // Note: This is not guaranteed to work on older AMD CPUs.
-   return 1 + CPUID_AMD_CORE_COUNT(v);
+   return 1 + CPUID_GET(0x80000008, ECX, LEAF88_CORE_COUNT, v);
 }
 
 
@@ -1161,5 +1394,85 @@ CPUID_IsHypervisorLevel(uint32 level)
    return (level & 0xffffff00) == 0x40000000;
 }
 
+/*
+ *----------------------------------------------------------------------
+ *
+ * CPUID_LevelUsesEcx --
+ *
+ *      Returns TRUE for leaves that support input ECX != 0 (subleaves).
+ *
+ *----------------------------------------------------------------------
+ */
+
+static INLINE Bool
+CPUID_LevelUsesEcx(uint32 level) {
+   return level == 4 || level == 7 || level == 0xb || level == 0xd ||
+          level == 0x8000001d;
+}
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * CPUID_IsValid*Subleaf --
+ *
+ *      Functions to determine the last subleaf for the level specified
+ *
+ *----------------------------------------------------------------------
+ */
+
+static INLINE Bool
+CPUID_IsValidBSubleaf(uint32 ebx)  // IN: %ebx = cpuid.b.sublevel.ebx
+{
+   return ebx != 0;
+}
+
+static INLINE Bool
+CPUID_IsValid4Subleaf(uint32 eax)  // IN: %eax = cpuid.4.sublevel.eax
+{
+   return eax != 0;
+}
+
+static INLINE Bool
+CPUID_IsValid7Subleaf(uint32 eax, uint32 subleaf)  // IN: %eax = cpuid.7.0.eax
+{
+   /*
+    * cpuid.7.0.eax is the max ecx (subleaf) index
+    */
+   return subleaf <= eax;
+}
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * CPUID_IsValidDSubleaf --
+ *
+ *    It is the caller's repsonsibility to determine if the processor
+ *    supports XSAVE and therefore has D sub-leaves.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE Bool
+CPUID_IsValidDSubleaf(uint32 subleaf)  // IN: subleaf to check
+{
+   return subleaf <= 63;
+}
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * CPUID_SupportsMsrPlatformInfo --
+ *
+ *    Uses vendor and cpuid.1.0.eax to determine if the processor
+ *    supports MSR_PLATFORM_INFO.
+ *
+ *----------------------------------------------------------------------
+ */
+static INLINE Bool
+CPUID_SupportsMsrPlatformInfo(CpuidVendor vendor, uint32 version)
+{
+   return vendor == CPUID_VENDOR_INTEL &&
+          (CPUID_UARCH_IS_NEHALEM(version) ||
+           CPUID_UARCH_IS_SANDYBRIDGE(version));
+}
 
 #endif
diff --git a/sys/dev/virtual/vmware/shared/x86cpuid_asm.h b/sys/dev/virtual/vmware/shared/x86cpuid_asm.h
index 5aaedae..f49f11f 100644
--- a/sys/dev/virtual/vmware/shared/x86cpuid_asm.h
+++ b/sys/dev/virtual/vmware/shared/x86cpuid_asm.h
@@ -260,7 +260,7 @@ __GET_CPUID(int input, CPUIDRegs *regs)
 #ifdef VM_X86_64
 
 /*
- * No inline assembly in Win64. Implemented in bora/lib/user in
+ * No inline assembly in Win64. Implemented in bora/lib/misc in
  * cpuidMasm64.asm.
  */
 
@@ -404,7 +404,7 @@ __GET_EDX_FROM_CPUID(int input)
 #ifdef VM_X86_64
 
 /*
- * No inline assembly in Win64. Implemented in bora/lib/user in
+ * No inline assembly in Win64. Implemented in bora/lib/misc in
  * cpuidMasm64.asm.
  */
 
diff --git a/sys/gnu/dev/virtual/vmware/vmmemctl/Makefile b/sys/gnu/dev/virtual/vmware/vmmemctl/Makefile
index 3490bdf..369deaa 100644
--- a/sys/gnu/dev/virtual/vmware/vmmemctl/Makefile
+++ b/sys/gnu/dev/virtual/vmware/vmmemctl/Makefile
@@ -1,6 +1,6 @@
 KMOD=	vmmemctl
 
-SRCS=	os.c vmballoon.c
+SRCS=	os.c vmballoon.c backdoor_balloon.c kernelStubsBSD.c
 
 .if $(MACHINE_ARCH) == "x86_64"
 SRCS+=	backdoorGcc64.c
diff --git a/sys/gnu/dev/virtual/vmware/vmmemctl/backdoorGcc64.c b/sys/gnu/dev/virtual/vmware/vmmemctl/backdoorGcc64.c
index 4a7630c..93ce2c6 100644
--- a/sys/gnu/dev/virtual/vmware/vmmemctl/backdoorGcc64.c
+++ b/sys/gnu/dev/virtual/vmware/vmmemctl/backdoorGcc64.c
@@ -125,7 +125,7 @@ Backdoor_InOut(Backdoor_proto *myBp) // IN/OUT
 void
 BackdoorHbIn(Backdoor_proto_hb *myBp) // IN/OUT
 {
-   uint32 dummy;
+   uint64 dummy;
 
    __asm__ __volatile__(
         "pushq %%rbp"           "\n\t"
diff --git a/sys/gnu/dev/virtual/vmware/vmmemctl/backdoor_balloon.c b/sys/gnu/dev/virtual/vmware/vmmemctl/backdoor_balloon.c
new file mode 100644
index 0000000..7036d23
--- /dev/null
+++ b/sys/gnu/dev/virtual/vmware/vmmemctl/backdoor_balloon.c
@@ -0,0 +1,362 @@
+/*********************************************************
+ * Copyright (C) 2012 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+#include "backdoor_balloon.h"
+#include "backdoor.h"
+#include "balloon_def.h"
+#include "os.h"
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * BackdoorCmd --
+ *
+ *      Do the balloon hypercall to the vmkernel.
+ *
+ * Results:
+ *      Hypercall status will be returned and out will be filled
+ *      if it's not NULL, either by bx or cx depending on the command.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static int
+BackdoorCmd(uint16 cmd,     // IN
+            size_t arg1,    // IN
+            uint32 arg2,    // IN
+            uint32 *out,    // OUT
+            int *resetFlag) // OUT
+{
+   Backdoor_proto bp;
+   int status;
+
+   /* prepare backdoor args */
+   bp.in.cx.halfs.low = cmd;
+   bp.in.size = arg1;
+   bp.in.si.word = arg2;
+
+   /* invoke backdoor */
+   bp.in.ax.word = BALLOON_BDOOR_MAGIC;
+   bp.in.dx.halfs.low = BALLOON_BDOOR_PORT;
+   Backdoor_InOut(&bp);
+
+   status = bp.out.ax.word;
+   /* set flag if reset requested */
+   if (status == BALLOON_ERROR_RESET) {
+      *resetFlag = 1;
+   }
+
+   if (out) {
+      if (cmd == BALLOON_BDOOR_CMD_START) {
+         *out = bp.out.cx.word;
+      } else {
+         *out = bp.out.bx.word;
+      }
+   }
+
+   return status;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * Backdoor_MonitorStart --
+ *
+ *      Attempts to contact monitor via backdoor to begin operation.
+ *
+ * Results:
+ *      Returns BALLOON_SUCCESS if successful, otherwise error code.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+int
+Backdoor_MonitorStart(Balloon *b,               // IN
+                      uint32 protoVersion)      // IN
+{
+   uint32 capabilities;
+   int status = BackdoorCmd(BALLOON_BDOOR_CMD_START, protoVersion, 0,
+                            &capabilities, &b->resetFlag);
+   /*
+    * If return code is BALLOON_SUCCESS_WITH_CAPABILITY, then ESX is
+    * sending the common capabilities supported by the monitor and the
+    * guest in cx.
+    */
+   if (status == BALLOON_SUCCESS_WITH_CAPABILITIES) {
+      b->hypervisorCapabilities = capabilities;
+      status = BALLOON_SUCCESS;
+   } else if (status == BALLOON_SUCCESS) {
+      b->hypervisorCapabilities = BALLOON_BASIC_CMDS;
+   }
+
+   /* update stats */
+   STATS_INC(b->stats.start);
+   if (status != BALLOON_SUCCESS) {
+      STATS_INC(b->stats.startFail);
+   }
+
+   return status;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * Backdoor_MonitorGuestType --
+ *
+ *      Attempts to contact monitor and report guest OS identity.
+ *
+ * Results:
+ *      Returns BALLOON_SUCCESS if successful, otherwise error code.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+int
+Backdoor_MonitorGuestType(Balloon *b) // IN
+{
+   int status = BackdoorCmd(BALLOON_BDOOR_CMD_GUEST_ID, b->guestType, 0,
+                            NULL, &b->resetFlag);
+
+   /* update stats */
+   STATS_INC(b->stats.guestType);
+   if (status != BALLOON_SUCCESS) {
+      STATS_INC(b->stats.guestTypeFail);
+   }
+
+   return status;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * Backdoor_MonitorGetTarget --
+ *
+ *      Attempts to contact monitor via backdoor to obtain desired
+ *      balloon size.
+ *
+ *      Predicts the maximum achievable balloon size and sends it
+ *      to vmm => vmkernel via vEbx register.
+ *
+ *      OS_ReservedPageGetLimit() returns either predicted max balloon
+ *      pages or BALLOON_MAX_SIZE_USE_CONFIG. In the later scenario,
+ *      vmkernel uses global config options for determining a guest's max
+ *      balloon size. Note that older vmballoon drivers set vEbx to
+ *      BALLOON_MAX_SIZE_USE_CONFIG, i.e., value 0 (zero). So vmkernel
+ *      will fallback to config-based max balloon size estimation.
+ *
+ * Results:
+ *      If successful, sets "target" to value obtained from monitor,
+ *      and returns BALLOON_SUCCESS. Otherwise returns error code.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+int
+Backdoor_MonitorGetTarget(Balloon *b,     // IN
+                          uint32 *target) // OUT
+{
+   unsigned long limit;
+   uint32 limit32;
+   int status;
+
+   limit = OS_ReservedPageGetLimit();
+
+   /* Ensure limit fits in 32-bits */
+   limit32 = (uint32)limit;
+   if (limit32 != limit) {
+      return BALLOON_FAILURE;
+   }
+
+   status = BackdoorCmd(BALLOON_BDOOR_CMD_TARGET, limit, 0, target,
+                        &b->resetFlag);
+
+   /* update stats */
+   STATS_INC(b->stats.target);
+   if (status != BALLOON_SUCCESS) {
+      STATS_INC(b->stats.targetFail);
+   }
+
+   return status;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * Backdoor_MonitorLockPage --
+ *
+ *      Attempts to contact monitor and add PPN corresponding to
+ *      the page handle to set of "balloon locked" pages.
+ *      If the current protocol support batching, it will balloon all
+ *      PPNs listed in the batch page.
+ *
+ * Results:
+ *      Returns BALLOON_SUCCESS if successful, otherwise error code.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+int
+Backdoor_MonitorLockPage(Balloon *b,    // IN
+                         PPN64 ppn)     // IN
+{
+   int status;
+   uint32 ppn32 = (uint32)ppn;
+
+   /* Ensure PPN fits in 32-bits, i.e. guest memory is limited to 16TB. */
+   if (ppn32 != ppn) {
+      return BALLOON_ERROR_PPN_INVALID;
+   }
+
+   status = BackdoorCmd(BALLOON_BDOOR_CMD_LOCK, ppn32, 0, NULL,
+                        &b->resetFlag);
+
+   /* update stats */
+   STATS_INC(b->stats.lock);
+   if (status != BALLOON_SUCCESS) {
+      STATS_INC(b->stats.lockFail);
+   }
+
+   return status;
+}
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * Backdoor_MonitorUnlockPage --
+ *
+ *      Attempts to contact monitor and remove PPN corresponding to
+ *      the page handle from set of "balloon locked" pages.
+ *      If the current protocol support batching, it will remove all
+ *      the PPNs listed in the batch page.
+ *
+ * Results:
+ *      Returns BALLOON_SUCCESS if successful, otherwise error code.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+int
+Backdoor_MonitorUnlockPage(Balloon *b,  // IN
+                           PPN64 ppn)   // IN
+{
+   int status;
+   uint32 ppn32 = (uint32)ppn;
+
+   /* Ensure PPN fits in 32-bits, i.e. guest memory is limited to 16TB. */
+   if (ppn32 != ppn) {
+      return BALLOON_ERROR_PPN_INVALID;
+   }
+
+   status = BackdoorCmd(BALLOON_BDOOR_CMD_UNLOCK, ppn32, 0, NULL,
+                        &b->resetFlag);
+
+   /* update stats */
+   STATS_INC(b->stats.unlock);
+   if (status != BALLOON_SUCCESS) {
+      STATS_INC(b->stats.unlockFail);
+   }
+
+   return status;
+}
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * Backdoor_MonitorLockPagesBatched --
+ *
+ *      Balloon all PPNs listed in the batch page.
+ *
+ * Results:
+ *      Returns BALLOON_SUCCESS if successful, otherwise error code.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+int
+Backdoor_MonitorLockPagesBatched(Balloon *b,    // IN
+                                 PPN64 ppn,     // IN
+                                 uint32 nPages) // IN
+{
+   int status = BackdoorCmd(BALLOON_BDOOR_CMD_BATCHED_LOCK,
+                            (size_t)ppn, nPages, NULL, &b->resetFlag);
+
+   /* update stats */
+   STATS_INC(b->stats.lock);
+   if (status != BALLOON_SUCCESS) {
+      STATS_INC(b->stats.lockFail);
+   }
+
+   return status;
+}
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * Backdoor_MonitorUnlockPagesBatched --
+ *
+ *      Unballoon all PPNs listed in the batch page.
+ *
+ * Results:
+ *      Returns BALLOON_SUCCESS if successful, otherwise error code.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+int
+Backdoor_MonitorUnlockPagesBatched(Balloon *b,          // IN
+                                   PPN64 ppn,           // IN
+                                   uint32 nPages)       // IN
+{
+   int status = BackdoorCmd(BALLOON_BDOOR_CMD_BATCHED_UNLOCK,
+                            (size_t)ppn, nPages, NULL, &b->resetFlag);
+
+   /* update stats */
+   STATS_INC(b->stats.unlock);
+   if (status != BALLOON_SUCCESS) {
+      STATS_INC(b->stats.unlockFail);
+   }
+
+   return status;
+}
diff --git a/sys/gnu/dev/virtual/vmware/vmmemctl/backdoor_balloon.h b/sys/gnu/dev/virtual/vmware/vmmemctl/backdoor_balloon.h
index 7d06d08..9036f95 100644
--- a/sys/gnu/dev/virtual/vmware/vmmemctl/backdoor_balloon.h
+++ b/sys/gnu/dev/virtual/vmware/vmmemctl/backdoor_balloon.h
@@ -26,14 +26,16 @@
 #ifndef _BACKDOOR_BALLOON_H_
 #define _BACKDOOR_BALLOON_H_
 
+#include "vmballoon.h"
 #include "backdoor.h"
 #include "balloon_def.h"
 
-static INLINE
-void Backdoor_Balloon(Backdoor_proto *myBp) {
-   myBp->in.ax.word = BALLOON_BDOOR_MAGIC;
-   myBp->in.dx.halfs.low = BALLOON_BDOOR_PORT;
-   Backdoor_InOut(myBp);
-}
+int Backdoor_MonitorStart(Balloon *b, uint32 protoVersion);
+int Backdoor_MonitorGuestType(Balloon *b);
+int Backdoor_MonitorGetTarget(Balloon *b, uint32 *target);
+int Backdoor_MonitorLockPage(Balloon *b, PPN64 ppn);
+int Backdoor_MonitorUnlockPage(Balloon *b, PPN64 ppn);
+int Backdoor_MonitorLockPagesBatched(Balloon *b, PPN64 ppn, uint32 nPages);
+int Backdoor_MonitorUnlockPagesBatched(Balloon *b, PPN64 ppn, uint32 nPages);
 
 #endif /* _BACKDOOR_BALLOON_H_ */
diff --git a/sys/gnu/dev/virtual/vmware/vmmemctl/balloonInt.h b/sys/gnu/dev/virtual/vmware/vmmemctl/balloonInt.h
new file mode 100644
index 0000000..3a4225f
--- /dev/null
+++ b/sys/gnu/dev/virtual/vmware/vmmemctl/balloonInt.h
@@ -0,0 +1,75 @@
+/*********************************************************
+ * Copyright (C) 2012 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+#ifndef BALLOONINT_H_
+# define BALLOONINT_H_
+
+/*
+ * Compile-Time Options
+ */
+
+#define BALLOON_NAME                    "vmmemctl"
+#define BALLOON_NAME_VERBOSE            "VMware memory control driver"
+
+#if defined __linux__ || defined __FreeBSD__ || defined _WIN32
+/*
+ * FIXME: Even if the driver support batched commands keep using the
+ * non-batched one until more testing has been done.
+ */
+#define BALLOON_CAPABILITIES    BALLOON_BASIC_CMDS
+#else
+#define BALLOON_CAPABILITIES    BALLOON_BASIC_CMDS
+#endif
+
+#define BALLOON_RATE_ADAPT      1
+
+#define BALLOON_DEBUG           1
+#define BALLOON_DEBUG_VERBOSE   0
+
+#define BALLOON_POLL_PERIOD             1 /* sec */
+#define BALLOON_NOSLEEP_ALLOC_MAX       16384
+
+#define BALLOON_RATE_ALLOC_MIN          512
+#define BALLOON_RATE_ALLOC_MAX          2048
+#define BALLOON_RATE_ALLOC_INC          16
+
+#define BALLOON_RATE_FREE_MIN           512
+#define BALLOON_RATE_FREE_MAX           16384
+#define BALLOON_RATE_FREE_INC           16
+
+/*
+ * Move it to bora/public/balloon_def.h later, if needed. Note that
+ * BALLOON_PAGE_ALLOC_FAILURE is an internal error code used for
+ * distinguishing page allocation failures from monitor-backdoor errors.
+ * We use value 1000 because all monitor-backdoor error codes are < 1000.
+ */
+#define BALLOON_PAGE_ALLOC_FAILURE      1000
+
+#define BALLOON_STATS
+
+#ifdef	BALLOON_STATS
+#define	STATS_INC(stat)	(stat)++
+#define	STATS_DEC(stat)	(stat)--
+#else
+#define	STATS_INC(stat)
+#define	STATS_DEC(stat)
+#endif
+
+#define PPN_2_PA(_ppn)  ((_ppn) << PAGE_SHIFT)
+#define PA_2_PPN(_pa)   ((_pa) >> PAGE_SHIFT)
+
+#endif /* !BALLOONINT_H_ */
diff --git a/sys/gnu/dev/virtual/vmware/vmmemctl/balloon_def.h b/sys/gnu/dev/virtual/vmware/vmmemctl/balloon_def.h
index 20a4348..9e06c11 100644
--- a/sys/gnu/dev/virtual/vmware/vmmemctl/balloon_def.h
+++ b/sys/gnu/dev/virtual/vmware/vmmemctl/balloon_def.h
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2000 VMware, Inc. All rights reserved.
+ * Copyright (C) 2000-2012 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -33,6 +33,8 @@
 #include "includeCheck.h"
 
 #include "vm_basic_types.h"
+#include "vm_basic_defs.h"
+#include "vm_assert.h"
 
 /*
  * constants
@@ -42,12 +44,55 @@
 #define BALLOON_BDOOR_PORT              (0x5670)
 #define BALLOON_BDOOR_MAGIC             (0x456c6d6f)
 
+/*
+ * Backdoor commands availability:
+ *
+ * +====================+======================+
+ * |    CMD             | Capabilities         |
+ * +--------------------+----------------------+
+ * | START              | Always available (*) |
+ * | TARGET             | Always available     |
+ * | LOCK               | BASIC_CMDS           |
+ * | UNLOCK             | BASIC_CMDS           |
+ * | GUEST_ID           | Always available     |
+ * | BATCHED_LOCK       | BATCHED_CMDS         |
+ * | BATCHED_UNLOCK     | BATCHED_CMDS         |
+ * +====================+======================+
+ *
+ * (*) The START command has been slightly modified when more than the
+ * basic commands are available: It returns
+ * BALLOON_SUCCESS_WITH_CAPABILITIES with the available capabilities
+ * stored in %ecx. Previously, a versioned protocol was used, and the
+ * protocol that should be used was also returned in %ecx. Protocol
+ * version 2 was the initial version and the only one shipped. Version 3
+ * was temporary used internally but has caused several issue due to
+ * protocol mismatch between monitor and guest.
+ *
+ */
+
 /* backdoor command numbers */
 #define BALLOON_BDOOR_CMD_START         (0)
 #define BALLOON_BDOOR_CMD_TARGET        (1)
 #define BALLOON_BDOOR_CMD_LOCK          (2)
 #define BALLOON_BDOOR_CMD_UNLOCK        (3)
 #define BALLOON_BDOOR_CMD_GUEST_ID      (4)
+/* The command 5 was shortly used between 1881144 and 1901153. */
+#define BALLOON_BDOOR_CMD_BATCHED_LOCK   (6)
+#define BALLOON_BDOOR_CMD_BATCHED_UNLOCK (7)
+
+/* balloon capabilities */
+typedef enum {
+   /*
+    * Bit 0 is not used and shouldn't be used, due to issue with
+    * protocol v3, to avoid ambiguity between protocol v3 and
+    * capabilities, leave this bit as 0. That way, by masking guest
+    * capabilities with monitor capabilities, bit 0 will always be set
+    * to 0, and buggy v3 tool will automatically switch to unbatched
+    * LOCK and UNLOCK.
+    */
+   BALLOON_BASIC_CMDS           = (1 << 1),
+   BALLOON_BATCHED_CMDS         = (1 << 2)
+} BalloonCapabilities;
 
 /* use config value for max balloon size */
 #define BALLOON_MAX_SIZE_USE_CONFIG     (0)
@@ -80,4 +125,143 @@ typedef enum {
 #define BALLOON_ERROR_RESET             (7)
 #define BALLOON_ERROR_BUSY              (8)
 
+#define BALLOON_SUCCESS_WITH_CAPABILITIES       (0x03000000)
+
+/*
+ * BatchPage.
+ */
+#define BALLOON_BATCH_MAX_PAGES         (PAGE_SIZE / sizeof(PA64))
+
+/*
+ * We are using the fact that for 4k pages, the 12LSB are set to 0, so
+ * we can use them and mask those bit when we need the real PA.
+ *
+ * +=============+==========+========+
+ * |             |          |        |
+ * | Page number | Reserved | Status |
+ * |             |          |        |
+ * +=============+==========+========+
+ * 64  PAGE_SHIFT          6         0
+ *
+ * For now, only 4k pages are supported by the monitor, but by using
+ * reserved bit we can in the future add some flags that will indicate
+ * whether the page is a 2M page or a 1G page.
+ *
+ * The reserved field should be set to 0.
+ *
+ */
+
+
+#define BALLOON_BATCH_STATUS_MASK       MASK64(5)
+#define BALLOON_BATCH_PAGE_MASK         (~MASK64(PAGE_SHIFT))
+
+typedef struct BalloonBatchPage {
+   PA64 pages[BALLOON_BATCH_MAX_PAGES];
+} BalloonBatchPage;
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Balloon_BatchGetPA --
+ *
+ *      Get the page stored in the batch page at idx.
+ *
+ * Results:
+ *      See above.
+ *
+ * Side effects:
+ *      None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+static INLINE PA64
+Balloon_BatchGetPA(BalloonBatchPage *batchPage,         // IN
+                   uint16 idx)                          // IN
+{
+   ASSERT(idx < BALLOON_BATCH_MAX_PAGES);
+   return batchPage->pages[idx] & BALLOON_BATCH_PAGE_MASK;
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Balloon_BatchGetStatus --
+ *
+ *      Get the error code associated with a page.
+ *
+ * Results:
+ *      See above.
+ *
+ * Side effects:
+ *      None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+static INLINE uint8
+Balloon_BatchGetStatus(BalloonBatchPage *batchPage,     // IN
+                       uint16 idx)                      // IN
+{
+   ASSERT(idx < BALLOON_BATCH_MAX_PAGES);
+   return (uint8)(batchPage->pages[idx] & BALLOON_BATCH_STATUS_MASK);
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Balloon_BatchSetPA --
+ *
+ *      Store the page in the batch page at idx.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+static INLINE void
+Balloon_BatchSetPA(BalloonBatchPage *batchPage,            // IN
+                   uint16 idx,                             // IN
+                   PA64 pa)                                // IN
+{
+   ASSERT(idx < BALLOON_BATCH_MAX_PAGES);
+   ASSERT((pa & ~BALLOON_BATCH_PAGE_MASK) == 0);
+   batchPage->pages[idx] = pa;
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Balloon_BatchSetStatus --
+ *
+ *      Set the error code associated with a page.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+static INLINE void
+Balloon_BatchSetStatus(BalloonBatchPage *batchPage,      // IN
+                       uint16 idx,                       // IN
+                       int error)                        // IN
+{
+   PA64 pa = Balloon_BatchGetPA(batchPage, idx);
+   ASSERT(idx < BALLOON_BATCH_MAX_PAGES);
+   ASSERT(error <= BALLOON_ERROR_BUSY && error >= BALLOON_FAILURE);
+   batchPage->pages[idx] = pa | (PPN64)error;
+}
+
+MY_ASSERTS(BALLOON_BATCH_SIZE,
+           ASSERT_ON_COMPILE(sizeof(BalloonBatchPage) == PAGE_SIZE);
+)
+
 #endif  /* _BALLOON_DEF_H */
diff --git a/sys/gnu/dev/virtual/vmware/vmmemctl/kernelStubs.h b/sys/gnu/dev/virtual/vmware/vmmemctl/kernelStubs.h
new file mode 100644
index 0000000..8630c32
--- /dev/null
+++ b/sys/gnu/dev/virtual/vmware/vmmemctl/kernelStubs.h
@@ -0,0 +1,89 @@
+/*********************************************************
+ * Copyright (C) 2006 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * kernelStubs.h
+ *
+ * KernelStubs implements some userspace library functions in terms
+ * of kernel functions to allow library userspace code to be used in a
+ * kernel.
+ */
+
+#ifndef __KERNELSTUBS_H__
+#define __KERNELSTUBS_H__
+
+#include "vm_basic_types.h"
+#ifndef _KERNEL
+#   error "_KERNEL is not defined"
+#endif
+#include <sys/types.h>
+#include <sys/malloc.h>
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <machine/stdarg.h>
+#include <sys/libkern.h>
+
+/*
+ * Function Prototypes
+ */
+
+/* Kernel memory on FreeBSD is tagged for statistics and sanity checking. */
+MALLOC_DECLARE(M_VMWARE_TEMP);
+
+/*
+ * On FreeBSD, the general memory allocator for both userland and the kernel is named
+ * malloc, but the kernel malloc() takes more arguments.  The following alias & macros
+ * work around this, to provide the standard malloc() API for userspace code that is
+ * being used in the kernel.
+ */
+
+#undef malloc
+
+static INLINE void *
+__compat_malloc(unsigned long size, struct malloc_type *type, int flags) {
+   return kmalloc(size, type, flags);
+}
+
+#define malloc(size)         __compat_malloc(size, M_VMWARE_TEMP, M_NOWAIT)
+#define calloc(count, size)  __compat_malloc((count) * (size),       \
+                                                M_VMWARE_TEMP, M_NOWAIT|M_ZERO)
+#define realloc(buf, size)   krealloc(buf, size, M_VMWARE_TEMP, M_NOWAIT)
+#define free(buf)            kfree(buf, M_VMWARE_TEMP)
+#define strchr(s,c)          index(s,c)
+#define strrchr(s,c)         rindex(s,c)
+
+/*
+ * Stub functions we provide.
+ */
+
+void Panic(const char *fmt, ...);
+
+char *Str_Strcpy(char *buf, const char *src, size_t maxSize);
+int Str_Vsnprintf(char *str, size_t size, const char *format,
+                  __va_list arguments);
+char *Str_Vasprintf(size_t *length, const char *format,
+                    __va_list arguments);
+char *Str_Asprintf(size_t *length, const char *Format, ...);
+
+/*
+ * Functions the driver must implement for the stubs.
+ */
+EXTERN void Debug(const char *fmt, ...);
+
+
+#endif /* __KERNELSTUBS_H__ */
diff --git a/sys/gnu/dev/virtual/vmware/vmmemctl/kernelStubsBSD.c b/sys/gnu/dev/virtual/vmware/vmmemctl/kernelStubsBSD.c
new file mode 100644
index 0000000..9d45331
--- /dev/null
+++ b/sys/gnu/dev/virtual/vmware/vmmemctl/kernelStubsBSD.c
@@ -0,0 +1,259 @@
+/*********************************************************
+ * Copyright (C) 2007 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation version 2 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ *********************************************************/
+
+/*
+ * kernelStubsBSD.c --
+ *
+ *      Stub functions for use by miscellaneous VMware code when brought into
+ *      the FreeBSD kernel.
+ */
+
+
+#include <sys/types.h>
+#include <sys/param.h>
+#include <machine/stdarg.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+
+#include "kernelStubs.h"
+
+MALLOC_DEFINE(M_VMWARE_TEMP, "VMwareTemp", "VMware: Temporary Allocations");
+
+void Log (const char *fmt, ...) __attribute__ ((alias ("Debug")));
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Debug --
+ *
+ *      Send a debugging message to the system log and/or console.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+void
+Debug(const char *fmt, ...)
+{
+   __va_list ap;
+
+   __va_start(ap, fmt);
+   kvprintf(fmt, ap);
+   __va_end(ap);
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Panic --
+ *
+ *      Print a panic message & induce a kernel panic.
+ *
+ * Results:
+ *      Does not return.
+ *
+ * Side effects:
+ *      None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+void
+Panic(const char *fmt, ...)
+{
+   __va_list ap;
+
+   __va_start(ap, fmt);
+   kvprintf(fmt, ap);
+   __va_end(ap);
+
+   panic(" ");
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Str_Strcpy --
+ *
+ *      Wrapper around strcpy that panics if the source is too long.
+ *
+ * Results:
+ *      Returns a pointer to buf.
+ *
+ * Side effects:
+ *      None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+char *
+Str_Strcpy(char *buf,           // OUT: destination buffer
+           const char *src,     // IN : source buffer
+           size_t maxSize)      // IN : size of buf
+{
+   size_t srcLen = strlen(src);
+   if (srcLen >= maxSize) {
+      panic("%s:%d Buffer too small %p\n", __FILE__, __LINE__, buf);
+   }
+   return memcpy(buf, src, srcLen + 1);    // Extra byte = terminator
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * Str_Vsnprintf --
+ *
+ *      Compatibility wrapper for kvsnprintf(1).
+ *
+ * Results:
+ *
+ *      int - number of bytes stored in 'str' (not including null
+ *      terminate character), -1 on overflow (insufficient space for
+ *      null terminate is considered overflow)
+ *
+ *      NB: on overflow the buffer WILL be null terminated
+ *
+ * Side effects:
+ *      None
+ *
+ *----------------------------------------------------------------------
+ */
+
+int
+Str_Vsnprintf(char *str,                // OUT: destination buffer
+              size_t size,              // IN : size of str
+              const char *format,       // IN : format for kvsnprintf
+              __va_list arguments)        // IN : variadic args for kvsnprintf
+{
+   int retval;
+
+   retval = kvsnprintf(str, size, format, arguments);
+   if (retval >= size) {
+      retval = -1;
+   }
+   return retval;
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Str_Vasprintf --
+ *
+ *    Allocate and format a string, using the GNU libc way to specify the
+ *    format (i.e. optionally allow the use of positional parameters)
+ *
+ * Results:
+ *    The allocated string on success (if 'length' is not NULL, *length
+ *       is set to the length of the allocated string)
+ *    NULL on failure
+ *
+ * Side effects:
+ *    None
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+char *
+Str_Vasprintf(size_t *length,       // OUT
+              const char *format,   // IN
+              __va_list arguments)    // IN
+{
+   /*
+    * Simple implementation of Str_Vasprintf when userlevel libraries are not
+    * available (e.g. for use in drivers). We just fallback to kvsnprintf,
+    * doubling if we didn't have enough space.
+    */
+   unsigned int bufSize;
+   char *buf;
+   int retval;
+
+   bufSize = strlen(format);
+   buf = NULL;
+
+   do {
+      /*
+       * Initial allocation of strlen(format) * 2. Should this be tunable?
+       * XXX Yes, this could overflow and spin forever when you get near 2GB
+       *     allocations. I don't care. --rrdharan
+       */
+
+      __va_list tmpArgs;
+
+      bufSize *= 2;
+      buf = realloc(buf, bufSize);
+
+      if (!buf) {
+         return NULL;
+      }
+
+      __va_copy(tmpArgs, arguments);
+      retval = Str_Vsnprintf(buf, bufSize, format, tmpArgs);
+      __va_end(tmpArgs);
+   } while (retval == -1);
+
+   if (length) {
+      *length = retval;
+   }
+
+   /*
+    * Try to trim the buffer here to save memory?
+    */
+   return buf;
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * Str_Asprintf --
+ *
+ *    Same as Str_Vasprintf(), but parameters are passed inline --hpreg
+ *
+ * Results:
+ *    Same as Str_Vasprintf()
+ *
+ * Side effects:
+ *    Same as Str_Vasprintf()
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+char *
+Str_Asprintf(size_t *length,       // OUT
+             const char *format,   // IN
+             ...)                  // IN
+{
+   __va_list arguments;
+   char *result;
+
+   __va_start(arguments, format);
+   result = Str_Vasprintf(length, format, arguments);
+   __va_end(arguments);
+
+   return result;
+}
diff --git a/sys/gnu/dev/virtual/vmware/vmmemctl/os.c b/sys/gnu/dev/virtual/vmware/vmmemctl/os.c
index d32f73c..ef05699 100644
--- a/sys/gnu/dev/virtual/vmware/vmmemctl/os.c
+++ b/sys/gnu/dev/virtual/vmware/vmmemctl/os.c
@@ -231,13 +231,13 @@ OS_ReservedPageGetLimit(void)
 /*
  *-----------------------------------------------------------------------------
  *
- * OS_ReservedPageGetPPN --
+ * OS_ReservedPageGetPA --
  *
  *      Convert a page handle (of a physical page previously reserved with
- *      OS_ReservedPageAlloc()) to a ppn.
+ *      OS_ReservedPageAlloc()) to a pa.
  *
  * Results:
- *      The ppn.
+ *      The pa.
  *
  * Side effects:
  *      None.
@@ -245,10 +245,115 @@ OS_ReservedPageGetLimit(void)
  *-----------------------------------------------------------------------------
  */
 
-unsigned long
-OS_ReservedPageGetPPN(PageHandle handle) // IN: A valid page handle
+PA64
+OS_ReservedPageGetPA(PageHandle handle) // IN: A valid page handle
+{
+   return (((vm_page_t)handle)->phys_addr);
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * OS_ReservedPageGetHandle --
+ *
+ *      Convert a pa (of a physical page previously reserved with
+ *      OS_ReservedPageAlloc()) to a page handle.
+ *
+ * Results:
+ *      The page handle.
+ *
+ * Side effects:
+ *      None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+PageHandle
+OS_ReservedPageGetHandle(PA64 pa)     // IN
+{
+   return (PageHandle)PHYS_TO_VM_PAGE(pa);
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * OS_MapPageHandle --
+ *
+ *      Map a page handle into kernel address space, and return the
+ *      mapping to that page handle.
+ *
+ * Results:
+ *      The mapping.
+ *
+ * Side effects:
+ *      None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+Mapping
+OS_MapPageHandle(PageHandle handle)     // IN
+{
+   vm_offset_t res = kmem_alloc_nofault(&kernel_map, PAGE_SIZE, PAGE_SIZE);
+   vm_page_t page = (vm_page_t)handle;
+
+   if (!res) {
+      return MAPPING_INVALID;
+   }
+
+   pmap_qenter(res, &page, 1);
+
+   return (Mapping)res;
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * OS_Mapping2Addr --
+ *
+ *      Return the address of a previously mapped page handle (with
+ *      OS_MapPageHandle).
+ *
+ * Results:
+ *      The mapping address.
+ *
+ * Side effects:
+ *      None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+void *
+OS_Mapping2Addr(Mapping mapping)        // IN
+{
+   return (void *)mapping;
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * OS_UnmapPage --
+ *
+ *      Unmap a previously mapped page handle.
+ *
+ * Results:
+ *      None.
+ *
+ * Side effects:
+ *      None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+void
+OS_UnmapPage(Mapping mapping)           // IN
 {
-   return (((vm_page_t)handle)->phys_addr) >> PAGE_SHIFT;
+   pmap_qremove((vm_offset_t)mapping, 1);
+   kmem_free(&kernel_map, (vm_offset_t)mapping, PAGE_SIZE);
 }
 
 
@@ -417,7 +522,7 @@ os_balloonobject_create(void)
  *      Reserve a physical page for the exclusive use of this driver.
  *
  * Results:
- *      On success: A valid page handle that can be passed to OS_ReservedPageGetPPN()
+ *      On success: A valid page handle that can be passed to OS_ReservedPageGetPA()
  *                  or OS_ReservedPageFree().
  *      On failure: PAGE_HANDLE_INVALID
  *
diff --git a/sys/gnu/dev/virtual/vmware/vmmemctl/os.h b/sys/gnu/dev/virtual/vmware/vmmemctl/os.h
index f27dc35..bbd7f99 100644
--- a/sys/gnu/dev/virtual/vmware/vmmemctl/os.h
+++ b/sys/gnu/dev/virtual/vmware/vmmemctl/os.h
@@ -33,8 +33,10 @@
  */
 
 typedef uintptr_t PageHandle;
+typedef uintptr_t Mapping;
 
 #define PAGE_HANDLE_INVALID 0
+#define MAPPING_INVALID         0
 
 /*
  * Operations
@@ -49,8 +51,13 @@ extern void OS_Free(void *ptr, size_t size);
 extern void OS_Yield(void);
 
 extern unsigned long OS_ReservedPageGetLimit(void);
-extern unsigned long OS_ReservedPageGetPPN(PageHandle handle);
+extern PA64          OS_ReservedPageGetPA(PageHandle handle);
+extern PageHandle    OS_ReservedPageGetHandle(PA64 pa);
 extern PageHandle    OS_ReservedPageAlloc(int canSleep);
 extern void          OS_ReservedPageFree(PageHandle handle);
 
+extern Mapping       OS_MapPageHandle(PageHandle handle);
+extern void          *OS_Mapping2Addr(Mapping mapping);
+extern void          OS_UnmapPage(Mapping mapping);
+
 #endif  /* OS_H */
diff --git a/sys/gnu/dev/virtual/vmware/vmmemctl/vmballoon.c b/sys/gnu/dev/virtual/vmware/vmmemctl/vmballoon.c
index 8a2ab7e..660514c 100644
--- a/sys/gnu/dev/virtual/vmware/vmmemctl/vmballoon.c
+++ b/sys/gnu/dev/virtual/vmware/vmmemctl/vmballoon.c
@@ -35,24 +35,12 @@ extern "C" {
 #endif
 
 /*
- * Compile-Time Options
- */
-
-#define BALLOON_RATE_ADAPT      1
-
-#define BALLOON_DEBUG           1
-#define BALLOON_DEBUG_VERBOSE   0
-
-#define BALLOON_STATS
-
-/*
  * Includes
  */
 
 #include "os.h"
 #include "backdoor.h"
 #include "backdoor_balloon.h"
-#include "dbllnklst.h"
 #include "vmballoon.h"
 
 /*
@@ -63,116 +51,46 @@ extern "C" {
 #define NULL 0
 #endif
 
-#define BALLOON_PROTOCOL_VERSION        2
-
-#define BALLOON_CHUNK_PAGES             1000
-
-#define BALLOON_NOSLEEP_ALLOC_MAX       16384
-
-#define BALLOON_RATE_ALLOC_MIN          512
-#define BALLOON_RATE_ALLOC_MAX          2048
-#define BALLOON_RATE_ALLOC_INC          16
-
-#define BALLOON_RATE_FREE_MIN           512
-#define BALLOON_RATE_FREE_MAX           16384
-#define BALLOON_RATE_FREE_INC           16
-
-#define BALLOON_ERROR_PAGES             16
-
 /*
  * When guest is under memory pressure, use a reduced page allocation
  * rate for next several cycles.
  */
 #define SLOW_PAGE_ALLOCATION_CYCLES     4
 
-/*
- * Move it to bora/public/balloon_def.h later, if needed. Note that
- * BALLOON_PAGE_ALLOC_FAILURE is an internal error code used for
- * distinguishing page allocation failures from monitor-backdoor errors.
- * We use value 1000 because all monitor-backdoor error codes are < 1000.
- */
-#define BALLOON_PAGE_ALLOC_FAILURE      1000
-
 /* Maximum number of page allocations without yielding processor */
 #define BALLOON_ALLOC_YIELD_THRESHOLD   1024
 
-
-/*
- * Types
- */
-
-typedef struct BalloonChunk {
-   PageHandle page[BALLOON_CHUNK_PAGES];
-   uint32 pageCount;
-   DblLnkLst_Links node;
-} BalloonChunk;
-
-typedef struct {
-   PageHandle page[BALLOON_ERROR_PAGES];
-   uint32 pageCount;
-} BalloonErrorPages;
-
-typedef struct {
-   /* sets of reserved physical pages */
-   DblLnkLst_Links chunks;
-   int nChunks;
-
-   /* transient list of non-balloonable pages */
-   BalloonErrorPages errors;
-
-   BalloonGuest guestType;
-
-   /* balloon size */
-   int nPages;
-   int nPagesTarget;
-
-   /* reset flag */
-   int resetFlag;
-
-   /* adjustment rates (pages per second) */
-   int rateAlloc;
-   int rateFree;
-
-   /* slowdown page allocations for next few cycles */
-   int slowPageAllocationCycles;
-
-   /* statistics */
-   BalloonStats stats;
-} Balloon;
-
-/*
- * Globals
- */
-
-static Balloon globalBalloon;
-
 /*
  * Balloon operations
  */
-static int  BalloonPageAlloc(Balloon *b, BalloonPageAllocType allocType);
-static int  BalloonPageFree(Balloon *b, int monitorUnlock);
+static int  BalloonPageFree(Balloon *b);
 static int  BalloonAdjustSize(Balloon *b, uint32 target);
 static void BalloonReset(Balloon *b);
 
-/*
- * Backdoor Operations
- */
-static int BalloonMonitorStart(Balloon *b);
-static int BalloonMonitorGuestType(Balloon *b);
-static int BalloonMonitorGetTarget(Balloon *b, uint32 *nPages);
-static int BalloonMonitorLockPage(Balloon *b, PageHandle handle);
-static int BalloonMonitorUnlockPage(Balloon *b, PageHandle handle);
+static void BalloonAddPage(Balloon *b, uint16 idx, PageHandle page);
+static void BalloonAddPageBatched(Balloon *b, uint16 idx, PageHandle page);
+static int  BalloonLock(Balloon *b, uint16 nPages);
+static int  BalloonLockBatched(Balloon *b, uint16 nPages);
+static int  BalloonUnlock(Balloon *b, uint16 nPages);
+static int  BalloonUnlockBatched(Balloon *b, uint16 nPages);
 
 /*
- * Macros
+ * Globals
  */
 
-#ifdef  BALLOON_STATS
-#define STATS_INC(stat) (stat)++
-#else
-#define STATS_INC(stat)
-#endif
+static Balloon globalBalloon;
 
+static const BalloonOps balloonOps = {
+   .addPage = BalloonAddPage,
+   .lock = BalloonLock,
+   .unlock = BalloonUnlock
+};
+
+static const struct BalloonOps balloonOpsBatched = {
+   .addPage = BalloonAddPageBatched,
+   .lock = BalloonLockBatched,
+   .unlock = BalloonUnlockBatched
+};
 
 /*
  *----------------------------------------------------------------------
@@ -218,7 +136,7 @@ Balloon_GetStats(void)
  * BalloonChunk_Create --
  *
  *      Creates a new BalloonChunk object capable of tracking
- *      BALLOON_CHUNK_PAGES PPNs.
+ *      BALLOON_CHUNK_PAGES PAs.
  *
  *      We do not bother to define two versions (NOSLEEP and CANSLEEP)
  *      of OS_Malloc because Chunk_Create does not require a new page
@@ -300,12 +218,61 @@ Balloon_Deallocate(Balloon *b) // IN
 
    /* free all pages, skipping monitor unlock */
    while (b->nChunks > 0) {
-      (void) BalloonPageFree(b, FALSE);
+      (void) BalloonPageFree(b);
       if (++cnt >= b->rateFree) {
          cnt = 0;
          OS_Yield();
       }
    }
+
+   /* Release the batch page */
+   if (b->batchPageMapping != MAPPING_INVALID) {
+      OS_UnmapPage(b->batchPageMapping);
+      b->batchPageMapping = MAPPING_INVALID;
+      b->batchPage = NULL;
+   }
+
+   if (b->pageHandle != PAGE_HANDLE_INVALID) {
+      OS_ReservedPageFree(b->pageHandle);
+      b->pageHandle = PAGE_HANDLE_INVALID;
+   }
+}
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * BalloonInitBatching --
+ *
+ *      Allocate and map the batch page.
+ *
+ * Results:
+ *      BALLOON_SUCCESS or an error code in case of failure.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static int
+BalloonInitBatching(Balloon *b) // IN
+{
+   b->batchMaxPages = BALLOON_BATCH_MAX_PAGES;
+
+   b->pageHandle = OS_ReservedPageAlloc(BALLOON_PAGE_ALLOC_NOSLEEP);
+   if (b->pageHandle == PAGE_HANDLE_INVALID) {
+      return BALLOON_PAGE_ALLOC_FAILURE;
+   }
+
+   b->batchPageMapping = OS_MapPageHandle(b->pageHandle);
+   if (b->batchPageMapping == MAPPING_INVALID) {
+      OS_ReservedPageFree(b->pageHandle);
+      b->pageHandle = PAGE_HANDLE_INVALID;
+      return BALLOON_PAGE_ALLOC_FAILURE;
+   }
+   b->batchPage = OS_Mapping2Addr(b->batchPageMapping);
+
+   return BALLOON_SUCCESS;
 }
 
 
@@ -329,20 +296,42 @@ Balloon_Deallocate(Balloon *b) // IN
 static void
 BalloonReset(Balloon *b) // IN
 {
-   uint32 status;
+   int status;
 
    /* free all pages, skipping monitor unlock */
    Balloon_Deallocate(b);
 
-   /* send start command */
-   status = BalloonMonitorStart(b);
-   if (status == BALLOON_SUCCESS) {
+   status = Backdoor_MonitorStart(b, BALLOON_CAPABILITIES);
+   if (status != BALLOON_SUCCESS) {
+      return;
+   }
+
+   if ((b->hypervisorCapabilities & BALLOON_BATCHED_CMDS) != 0) {
+      status = BalloonInitBatching(b);
+      if (status != BALLOON_SUCCESS) {
+         /*
+          * We failed to initialize the batching in the guest, inform
+          * the monitor about that by sending a null capability.
+          *
+          * The guest will retry to init itself in one second.
+          */
+         Backdoor_MonitorStart(b, 0);
+         return;
+      }
+   }
+
+   if ((b->hypervisorCapabilities & BALLOON_BATCHED_CMDS) != 0) {
+      b->balloonOps = &balloonOpsBatched;
+   } else if ((b->hypervisorCapabilities & BALLOON_BASIC_CMDS) != 0) {
+      b->balloonOps = &balloonOps;
+      b->batchMaxPages = 1;
+   }
+
       /* clear flag */
-      b->resetFlag = 0;
+   b->resetFlag = FALSE;
 
       /* report guest type */
-      (void) BalloonMonitorGuestType(b);
-   }
+   (void) Backdoor_MonitorGuestType(b);
 }
 
 
@@ -380,7 +369,7 @@ Balloon_QueryAndExecute(void)
    }
 
    /* contact monitor via backdoor */
-   status = BalloonMonitorGetTarget(b, &target);
+   status = Backdoor_MonitorGetTarget(b, &target);
 
    /* decrement slowPageAllocationCycles counter */
    if (b->slowPageAllocationCycles > 0) {
@@ -506,12 +495,14 @@ BalloonGetChunk(Balloon *b)         // IN
 /*
  *----------------------------------------------------------------------
  *
- * BalloonPageStore --
+ * BalloonGetChunkOrFallback --
  *
- *      Add "page" to the given "chunk".
+ *      Attempt to find a "chunk" with a free slot to store locked page.
+ *      If the allocation fails, use the previously allocated
+ *      fallbackChunk.
  *
  * Results:
- *      None.
+ *      A valid chunk.
  *
  * Side effects:
  *      None.
@@ -519,23 +510,30 @@ BalloonGetChunk(Balloon *b)         // IN
  *----------------------------------------------------------------------
  */
 
-static void
-BalloonPageStore(BalloonChunk *chunk, PageHandle page)
+static BalloonChunk *
+BalloonGetChunkOrFallback(Balloon *b)   // IN
 {
-   chunk->page[chunk->pageCount++] = page;
+   BalloonChunk *chunk = BalloonGetChunk(b);
+   if (chunk == NULL) {
+      ASSERT(b->fallbackChunk != NULL);
+      chunk = b->fallbackChunk;
+      b->fallbackChunk = NULL;
+      DblLnkLst_LinkFirst(&b->chunks, &chunk->node);
+      b->nChunks++;
 }
 
+   return chunk;
+}
 
 /*
  *----------------------------------------------------------------------
  *
- * BalloonPageAlloc --
+ * BalloonPageStore --
  *
- *      Attempts to allocate a physical page, inflating balloon "b".
- *      Informs monitor of PPN for allocated page via backdoor.
+ *      Add "page" to the given "chunk".
  *
  * Results:
- *      Returns BALLOON_SUCCESS if successful, otherwise error code.
+ *      None.
  *
  * Side effects:
  *      None.
@@ -543,70 +541,10 @@ BalloonPageStore(BalloonChunk *chunk, PageHandle page)
  *----------------------------------------------------------------------
  */
 
-static int
-BalloonPageAlloc(Balloon *b,                     // IN
-                 BalloonPageAllocType allocType) // IN
+static void
+BalloonPageStore(BalloonChunk *chunk, PageHandle page)
 {
-   PageHandle page;
-   BalloonChunk *chunk = NULL;
-   Bool locked;
-   int status;
-
-   /* allocate page, fail if unable */
-   do {
-      STATS_INC(b->stats.primAlloc[allocType]);
-
-      /*
-       * Attempts to allocate and reserve a physical page.
-       *
-       * If canSleep == 1, i.e., BALLOON_PAGE_ALLOC_CANSLEEP:
-       *      The allocation can wait (sleep) for page writeout (swap) by the guest.
-       * otherwise canSleep == 0, i.e., BALLOON_PAGE_ALLOC_NOSLEEP:
-       *      If allocation of a page requires disk writeout, then just fail. DON'T sleep.
-       *
-       * Returns the physical address of the allocated page, or 0 if error.
-       */
-      page = OS_ReservedPageAlloc(allocType);
-      if (page == PAGE_HANDLE_INVALID) {
-         STATS_INC(b->stats.primAllocFail[allocType]);
-         return BALLOON_PAGE_ALLOC_FAILURE;
-      }
-
-      /* Get the chunk to store allocated page. */
-      if (!chunk) {
-         chunk = BalloonGetChunk(b);
-         if (!chunk) {
-            OS_ReservedPageFree(page);
-            return BALLOON_PAGE_ALLOC_FAILURE;
-         }
-      }
-
-      /* inform monitor via backdoor */
-      status = BalloonMonitorLockPage(b, page);
-      locked = status == BALLOON_SUCCESS;
-      if (!locked) {
-         if (status == BALLOON_ERROR_RESET ||
-             status == BALLOON_ERROR_PPN_NOTNEEDED) {
-            OS_ReservedPageFree(page);
-            return status;
-         }
-
-         /* place on list of non-balloonable pages, retry allocation */
-         status = BalloonErrorPageStore(b, page);
-         if (status != BALLOON_SUCCESS) {
-            OS_ReservedPageFree(page);
-            return status;
-         }
-      }
-   } while (!locked);
-
-   /* track allocated page */
-   BalloonPageStore(chunk, page);
-
-   /* update balloon size */
-   b->nPages++;
-
-   return status;
+   chunk->page[chunk->pageCount++] = page;
 }
 
 
@@ -616,8 +554,7 @@ BalloonPageAlloc(Balloon *b,                     // IN
  * BalloonPageFree --
  *
  *      Attempts to deallocate a physical page, deflating balloon "b".
- *      Informs monitor of PPN for deallocated page via backdoor if
- *      "monitorUnlock" is specified.
+ *      Never informs monitor.
  *
  * Results:
  *      Returns BALLOON_SUCCESS if successful, otherwise error code.
@@ -629,47 +566,21 @@ BalloonPageAlloc(Balloon *b,                     // IN
  */
 
 static int
-BalloonPageFree(Balloon *b,        // IN
-                int monitorUnlock) // IN
+BalloonPageFree(Balloon *b)     // IN
 {
-   DblLnkLst_Links *node, *next;
-   BalloonChunk *chunk = NULL;
+   BalloonChunk *chunk;
    PageHandle page;
-   int status;
 
-   DblLnkLst_ForEachSafe(node, next, &b->chunks) {
-      chunk = DblLnkLst_Container(node, BalloonChunk, node);
-      if (chunk->pageCount > 0) {
-         break;
-      }
-
-      /* destroy empty chunk */
-      DblLnkLst_Unlink1(node);
-      BalloonChunk_Destroy(chunk);
-      chunk = NULL;
-
-      /* update stats */
-      b->nChunks--;
-   }
-
-   if (!chunk) {
-      /* We could not find a single non-empty chunk. */
+   if (!DblLnkLst_IsLinked(&b->chunks)) {
+      /* The chunk list is empty, the balloon cannot be deflated */
       return BALLOON_FAILURE;
    }
 
+   chunk = DblLnkLst_Container(b->chunks.next, BalloonChunk, node);
+
    /* deallocate last page */
    page = chunk->page[--chunk->pageCount];
 
-   /* inform monitor via backdoor */
-   if (monitorUnlock) {
-      status = BalloonMonitorUnlockPage(b, page);
-      if (status != BALLOON_SUCCESS) {
-         /* reset next pointer, fail */
-         chunk->pageCount++;
-         return status;
-      }
-   }
-
    /* deallocate page */
    OS_ReservedPageFree(page);
    STATS_INC(b->stats.primFree);
@@ -690,7 +601,6 @@ BalloonPageFree(Balloon *b,        // IN
    return BALLOON_SUCCESS;
 }
 
-
 /*
  *----------------------------------------------------------------------
  *
@@ -711,7 +621,7 @@ static int
 BalloonInflate(Balloon *b,    // IN
                uint32 target) // IN
 {
-   unsigned int goal;
+   uint32 goal, nPages;
    unsigned int rate;
    unsigned int i;
    unsigned int allocations = 0;
@@ -741,17 +651,14 @@ BalloonInflate(Balloon *b,    // IN
    rate = b->slowPageAllocationCycles ?
                 b->rateAlloc : BALLOON_NOSLEEP_ALLOC_MAX;
 
+   nPages = 0;
    for (i = 0; i < goal; i++) {
+      PageHandle        handle;
 
-      status = BalloonPageAlloc(b, allocType);
-      if (status != BALLOON_SUCCESS) {
-         if (status != BALLOON_PAGE_ALLOC_FAILURE) {
-            /*
-             * Not a page allocation failure, stop this cycle.
-             * Maybe we'll get new target from the host soon.
-             */
-            break;
-         }
+      STATS_INC(b->stats.primAlloc[allocType]);
+      handle = OS_ReservedPageAlloc(allocType);
+      if (handle == PAGE_HANDLE_INVALID) {
+         STATS_INC(b->stats.primAllocFail[allocType]);
 
          if (allocType == BALLOON_PAGE_ALLOC_CANSLEEP) {
             /*
@@ -771,25 +678,41 @@ BalloonInflate(Balloon *b,    // IN
           */
          b->slowPageAllocationCycles = SLOW_PAGE_ALLOCATION_CYCLES;
 
-         if (i >= b->rateAlloc)
+         if (allocations >= b->rateAlloc) {
             break;
+         }
 
          allocType = BALLOON_PAGE_ALLOC_CANSLEEP;
          /* Lower rate for sleeping allocations. */
          rate = b->rateAlloc;
+         continue;
       }
 
-      if (++allocations > BALLOON_ALLOC_YIELD_THRESHOLD) {
+      allocations++;
+      b->balloonOps->addPage(b, nPages++, handle);
+      if (nPages == b->batchMaxPages) {
+         status = b->balloonOps->lock(b, nPages);
+         nPages = 0;
+
+         if (status != BALLOON_SUCCESS) {
+            break;
+         }
+      }
+
+      if (allocations % BALLOON_ALLOC_YIELD_THRESHOLD == 0) {
          OS_Yield();
-         allocations = 0;
       }
 
-      if (i >= rate) {
+      if (allocations >= rate) {
          /* We allocated enough pages, let's take a break. */
          break;
       }
    }
 
+   if (nPages > 0) {
+      status = b->balloonOps->lock(b, nPages);
+   }
+
    /*
     * We reached our goal without failures so try increasing
     * allocation rate.
@@ -810,92 +733,186 @@ BalloonInflate(Balloon *b,    // IN
 /*
  *----------------------------------------------------------------------
  *
- * BalloonDeflate --
+ * BalloonLockBatched --
  *
- *      Frees physical pages to deflate balloon.
+ *      Lock all the batched page, previously stored by
+ *      BalloonAddPageBatched.
  *
  * Results:
- *      Returns BALLOON_SUCCESS if successful, otherwise error code.
+ *      BALLOON_SUCCESS or an error code.
  *
  * Side effects:
  *      None.
  *
  *----------------------------------------------------------------------
  */
-
 static int
-BalloonDeflate(Balloon *b,    // IN
-               uint32 target) // IN
+BalloonLockBatched(Balloon *b,       // IN
+                   uint16 nPages)    // IN
 {
-   int status, i;
-   uint32 nFree = b->nPages - target;
+   int          status;
+   uint32       i;
+   uint32       nLockedPages;
+   PageHandle   handle;
+   PPN64        batchPagePPN;
+   BalloonChunk *chunk = NULL;
 
-   /* limit deallocation rate */
-   nFree = MIN(nFree, b->rateFree);
+   batchPagePPN = PA_2_PPN(OS_ReservedPageGetPA(b->pageHandle));
+
+   /*
+    * Make sure that we will always have an available chunk before doing
+    * the LOCK_BATCHED call.
+    */
+   ASSERT(b->batchMaxPages < BALLOON_CHUNK_PAGES);
+   b->fallbackChunk = BalloonChunk_Create();
+   if (b->fallbackChunk == NULL) {
+      status = BALLOON_PAGE_ALLOC_FAILURE;
+   } else {
+      status = Backdoor_MonitorLockPagesBatched(b, batchPagePPN, nPages);
+   }
 
-   /* free pages to reach target */
-   for (i = 0; i < nFree; i++) {
-      status = BalloonPageFree(b, TRUE);
       if (status != BALLOON_SUCCESS) {
-         if (BALLOON_RATE_ADAPT) {
-            /* quickly decrease rate if error */
-            b->rateFree = MAX(b->rateFree / 2, BALLOON_RATE_FREE_MIN);
+      for (i = 0; i < nPages; i++) {
+         PA64 pa = Balloon_BatchGetPA(b->batchPage, i);
+         handle = OS_ReservedPageGetHandle(pa);
+
+         OS_ReservedPageFree(handle);
          }
-         return status;
+
+      goto out;
       }
+
+   nLockedPages = 0;
+   for (i = 0; i < nPages; i++) {
+      PA64              pa;
+      int               error;
+
+      pa = Balloon_BatchGetPA(b->batchPage, i);
+      handle = OS_ReservedPageGetHandle(pa);
+      error = Balloon_BatchGetStatus(b->batchPage, i);
+      if (error != BALLOON_SUCCESS) {
+         switch (error) {
+         case BALLOON_ERROR_PPN_PINNED:
+         case BALLOON_ERROR_PPN_INVALID:
+            if (BalloonErrorPageStore(b, handle) == BALLOON_SUCCESS) {
+               break;
+            }
+            // Fallthrough.
+         case BALLOON_ERROR_RESET:
+         case BALLOON_ERROR_PPN_NOTNEEDED:
+            OS_ReservedPageFree(handle);
+            break;
+         default:
+            /*
+             * If we fall here, there is definitively a bug in the
+             * driver that needs to be fixed, I'm not sure if
+             * PINNED and INVALID PPN can be seen as a bug in the
+             * driver.
+             */
+            ASSERT(FALSE);
+         }
+         continue;
    }
 
-   if (BALLOON_RATE_ADAPT) {
-      /* slowly increase rate if no errors */
-      b->rateFree = MIN(b->rateFree + BALLOON_RATE_FREE_INC,
-                        BALLOON_RATE_FREE_MAX);
+      if (chunk == NULL) {
+         chunk = BalloonGetChunkOrFallback(b);
    }
 
-   return BALLOON_SUCCESS;
+      BalloonPageStore(chunk, handle);
+      if (chunk->pageCount == BALLOON_CHUNK_PAGES) {
+         chunk = NULL;
+      }
+      nLockedPages++;
+   }
+
+   b->nPages += nLockedPages;
+
+out:
+   if (b->fallbackChunk != NULL) {
+      BalloonChunk_Destroy(b->fallbackChunk);
+      b->fallbackChunk = NULL;
 }
 
+   return status;
+}
 
 /*
  *----------------------------------------------------------------------
  *
- * BalloonAdjustSize --
+ * BalloonUnlockBatched --
  *
- *      Attempts to allocate or deallocate physical pages in order
- *      to reach desired "target" size for balloon "b".
+ *      Unlock all the batched page, previously stored by
+ *      BalloonAddPageBatched.
  *
  * Results:
- *      Returns BALLOON_SUCCESS if successful, otherwise error code.
+ *      BALLOON_SUCCESS or an error code.
  *
  * Side effects:
  *      None.
  *
  *----------------------------------------------------------------------
  */
-
 static int
-BalloonAdjustSize(Balloon *b,    // IN
-                  uint32 target) // IN
+BalloonUnlockBatched(Balloon *b,     // IN
+                     uint16 nPages)  // IN
 {
-   if (b->nPages < target) {
-      return BalloonInflate(b, target);
-   } else if (b->nPages > target) {
-      return BalloonDeflate(b, target);
-   } else {
-      /* already at target */
-      return BALLOON_SUCCESS;
+   uint32 i;
+   int status = BALLOON_SUCCESS;
+   uint32 nUnlockedPages;
+   PPN64 batchPagePPN;
+   BalloonChunk *chunk = NULL;
+
+   batchPagePPN = PA_2_PPN(OS_ReservedPageGetPA(b->pageHandle));
+   status = Backdoor_MonitorUnlockPagesBatched(b, batchPagePPN, nPages);
+
+   if (status != BALLOON_SUCCESS) {
+      for (i = 0; i < nPages; i++) {
+         PA64 pa = Balloon_BatchGetPA(b->batchPage, i);
+         PageHandle handle = OS_ReservedPageGetHandle(pa);
+
+         chunk = BalloonGetChunkOrFallback(b);
+         BalloonPageStore(chunk, handle);
+      }
+      goto out;
    }
+
+   nUnlockedPages = 0;
+   for (i = 0; i < nPages; i++) {
+      int status = Balloon_BatchGetStatus(b->batchPage, i);
+      PA64 pa = Balloon_BatchGetPA(b->batchPage, i);
+      PageHandle handle = OS_ReservedPageGetHandle(pa);
+
+      if (status != BALLOON_SUCCESS) {
+         chunk = BalloonGetChunkOrFallback(b);
+         BalloonPageStore(chunk, handle);
+         continue;
+   }
+
+      OS_ReservedPageFree(handle);
+      STATS_INC(b->stats.primFree);
+
+      nUnlockedPages++;
 }
 
+   b->nPages -= nUnlockedPages;
+
+out:
+   if (b->fallbackChunk != NULL) {
+      BalloonChunk_Destroy(b->fallbackChunk);
+      b->fallbackChunk = NULL;
+   }
+   return status;
+}
 
 /*
  *----------------------------------------------------------------------
  *
- * BalloonMonitorStart --
+ * BalloonAddPageBatched --
  *
- *      Attempts to contact monitor via backdoor to begin operation.
+ *      Add a page to the batch page, that will be ballooned later.
  *
  * Results:
- *      Returns BALLOON_SUCCESS if successful, otherwise error code.
+ *      Nothing.
  *
  * Side effects:
  *      None.
@@ -903,42 +920,25 @@ BalloonAdjustSize(Balloon *b,    // IN
  *----------------------------------------------------------------------
  */
 
-static int
-BalloonMonitorStart(Balloon *b) // IN
+static void
+BalloonAddPageBatched(Balloon *b,            // IN
+                      uint16 idx,            // IN
+                      PageHandle page)       // IN
 {
-   uint32 status, target;
-   Backdoor_proto bp;
-
-   /* prepare backdoor args */
-   bp.in.cx.halfs.low = BALLOON_BDOOR_CMD_START;
-   bp.in.size = BALLOON_PROTOCOL_VERSION;
-
-   /* invoke backdoor */
-   Backdoor_Balloon(&bp);
-
-   /* parse return values */
-   status = bp.out.ax.word;
-   target = bp.out.bx.word;
-
-   /* update stats */
-   STATS_INC(b->stats.start);
-   if (status != BALLOON_SUCCESS) {
-      STATS_INC(b->stats.startFail);
-   }
-
-   return status;
+   PA64 pa = OS_ReservedPageGetPA(page);
+   Balloon_BatchSetPA(b->batchPage, idx, pa);
 }
 
-
 /*
  *----------------------------------------------------------------------
  *
- * BalloonMonitorGuestType --
+ * BalloonLock --
  *
- *      Attempts to contact monitor and report guest OS identity.
+ *      Lock a page, previously stored with a call to BalloonAddPage,
+ *      by notifying the monitor.
  *
  * Results:
- *      Returns BALLOON_SUCCESS if successful, otherwise error code.
+ *      BALLOON_SUCCESS or an error code.
  *
  * Side effects:
  *      None.
@@ -947,58 +947,65 @@ BalloonMonitorStart(Balloon *b) // IN
  */
 
 static int
-BalloonMonitorGuestType(Balloon *b) // IN
+BalloonLock(Balloon *b,       // IN
+            uint16 nPages)    // IN
 {
-   uint32 status, target;
-   Backdoor_proto bp;
-
-   /* prepare backdoor args */
-   bp.in.cx.halfs.low = BALLOON_BDOOR_CMD_GUEST_ID;
-   bp.in.size = b->guestType;
+   PPN pagePPN;
+   BalloonChunk *chunk;
+   int status;
 
-   /* invoke backdoor */
-   Backdoor_Balloon(&bp);
+   /* Get the chunk to store allocated page. */
+   chunk = BalloonGetChunk(b);
+   if (chunk == NULL) {
+      OS_ReservedPageFree(b->pageHandle);
+      status = BALLOON_PAGE_ALLOC_FAILURE;
+      goto out;
+   }
 
-   /* parse return values */
-   status = bp.out.ax.word;
-   target = bp.out.bx.word;
+   /* inform monitor via backdoor */
+   pagePPN = PA_2_PPN(OS_ReservedPageGetPA(b->pageHandle));
+   status = Backdoor_MonitorLockPage(b, pagePPN);
+   if (status != BALLOON_SUCCESS) {
+      int old_status = status;
 
-   /* set flag if reset requested */
-   if (status == BALLOON_ERROR_RESET) {
-      b->resetFlag = 1;
+      if (status == BALLOON_ERROR_RESET ||
+          status == BALLOON_ERROR_PPN_NOTNEEDED) {
+         OS_ReservedPageFree(b->pageHandle);
+         goto out;
    }
 
-   /* update stats */
-   STATS_INC(b->stats.guestType);
+      /* place on list of non-balloonable pages, retry allocation */
+      status = BalloonErrorPageStore(b, b->pageHandle);
    if (status != BALLOON_SUCCESS) {
-      STATS_INC(b->stats.guestTypeFail);
+         OS_ReservedPageFree(b->pageHandle);
+         goto out;
    }
 
-   return status;
+      status = old_status;
+      goto out;
 }
 
+   /* track allocated page */
+   BalloonPageStore(chunk, b->pageHandle);
+
+   /* update balloon size */
+   b->nPages++;
+
+out:
+   b->pageHandle = PAGE_HANDLE_INVALID;
+   return status;
+}
 
 /*
  *----------------------------------------------------------------------
  *
- * BalloonMonitorGetTarget --
- *
- *      Attempts to contact monitor via backdoor to obtain desired
- *      balloon size.
- *
- *      Predicts the maximum achievable balloon size and sends it
- *      to vmm => vmkernel via vEbx register.
+ * BalloonUnlock --
  *
- *      OS_ReservedPageGetLimit() returns either predicted max balloon
- *      pages or BALLOON_MAX_SIZE_USE_CONFIG. In the later scenario,
- *      vmkernel uses global config options for determining a guest's max
- *      balloon size. Note that older vmballoon drivers set vEbx to
- *      BALLOON_MAX_SIZE_USE_CONFIG, i.e., value 0 (zero). So vmkernel
- *      will fallback to config-based max balloon size estimation.
+ *      Unlock a page, previously stored with a call to
+ *      BalloonAddPage, by notifying the monitor.
  *
  * Results:
- *      If successful, sets "target" to value obtained from monitor,
- *      and returns BALLOON_SUCCESS. Otherwise returns error code.
+ *      BALLOON_SUCCESS or an error code.
  *
  * Side effects:
  *      None.
@@ -1007,55 +1014,65 @@ BalloonMonitorGuestType(Balloon *b) // IN
  */
 
 static int
-BalloonMonitorGetTarget(Balloon *b,     // IN
-                         uint32 *target) // OUT
+BalloonUnlock(Balloon *b,     // IN
+              uint16 nPages)  // IN
 {
-   Backdoor_proto bp;
-   unsigned long limit;
-   uint32 limit32;
-   uint32 status;
+   PPN pagePPN = PA_2_PPN(OS_ReservedPageGetPA(b->pageHandle));
+   int status = Backdoor_MonitorUnlockPage(b, pagePPN);
 
-   limit = OS_ReservedPageGetLimit();
-
-   /* Ensure limit fits in 32-bits */
-   limit32 = (uint32)limit;
-   if (limit32 != limit) {
-      return BALLOON_FAILURE;
+   if (status != BALLOON_SUCCESS) {
+      BalloonChunk *chunk = BalloonGetChunkOrFallback(b);
+      BalloonPageStore(chunk, b->pageHandle);
+      goto out;
    }
 
-   /* prepare backdoor args */
-   bp.in.cx.halfs.low = BALLOON_BDOOR_CMD_TARGET;
-   bp.in.size = limit;
-
-   /* invoke backdoor */
-   Backdoor_Balloon(&bp);
+   OS_ReservedPageFree(b->pageHandle);
+   STATS_INC(b->stats.primFree);
 
-   /* parse return values */
-   status  = bp.out.ax.word;
-   *target = bp.out.bx.word;
+   /* update balloon size */
+   b->nPages--;
 
-   /* set flag if reset requested */
-   if (status == BALLOON_ERROR_RESET) {
-      b->resetFlag = 1;
+out:
+   b->pageHandle = PAGE_HANDLE_INVALID;
+   if (b->fallbackChunk != NULL) {
+      BalloonChunk_Destroy(b->fallbackChunk);
+      b->fallbackChunk = NULL;
    }
+   return status;
+}
 
-   /* update stats */
-   STATS_INC(b->stats.target);
-   if (status != BALLOON_SUCCESS) {
-      STATS_INC(b->stats.targetFail);
-   }
+/*
+ *----------------------------------------------------------------------
+ *
+ * BalloonAddPage --
+ *
+ *      Add a page to be ballooned later.
+ *
+ * Results:
+ *      Nothing.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
 
-   return status;
+static void
+BalloonAddPage(Balloon *b,            // IN
+               uint16 idx,            // IN
+               PageHandle page)       // IN
+{
+   ASSERT(b->pageHandle == PAGE_HANDLE_INVALID);
+   b->pageHandle = page;
 }
 
 
 /*
  *----------------------------------------------------------------------
  *
- * BalloonMonitorLockPage --
+ * BalloonDeflate --
  *
- *      Attempts to contact monitor and add PPN corresponding to
- *      the page handle to set of "balloon locked" pages.
+ *      Frees physical pages to deflate balloon.
  *
  * Results:
  *      Returns BALLOON_SUCCESS if successful, otherwise error code.
@@ -1067,44 +1084,71 @@ BalloonMonitorGetTarget(Balloon *b,     // IN
  */
 
 static int
-BalloonMonitorLockPage(Balloon *b,        // IN
-                        PageHandle handle) // IN
+BalloonDeflate(Balloon *b,      // IN
+               uint32 target)   // IN
 {
-   unsigned long ppn;
-   uint32 ppn32;
-   uint32 status, target;
-   Backdoor_proto bp;
+   int                  status = BALLOON_SUCCESS;
+   uint32               goal, nPages;
+   BalloonChunk         *chunk = NULL;
 
-   ppn = OS_ReservedPageGetPPN(handle);
+   goal = b->nPages - target;
 
-   /* Ensure PPN fits in 32-bits, i.e. guest memory is limited to 16TB. */
-   ppn32 = (uint32)ppn;
-   if (ppn32 != ppn) {
-      return BALLOON_ERROR_PPN_INVALID;
-   }
+   /* limit deallocation rate */
+   goal = MIN(goal, b->rateFree);
 
-   /* prepare backdoor args */
-   bp.in.cx.halfs.low = BALLOON_BDOOR_CMD_LOCK;
-   bp.in.size = ppn32;
+   nPages = 0;
+   for ( ; goal > 0; goal--) {
+      PageHandle lockedHandle;
 
-   /* invoke backdoor */
-   Backdoor_Balloon(&bp);
+      if (chunk == NULL) {
+         if (!DblLnkLst_IsLinked(&b->chunks)) {
+            /* The chunk list is empty, the balloon cannot be deflated */
+            status = BALLOON_FAILURE;
+            goto out;
+         }
 
-   /* parse return values */
-   status = bp.out.ax.word;
-   target = bp.out.bx.word;
+         chunk = DblLnkLst_Container(b->chunks.next, BalloonChunk, node);
+      }
+
+      lockedHandle = chunk->page[--chunk->pageCount];
+      if (!chunk->pageCount) {
+         DblLnkLst_Unlink1(&chunk->node);
+         /*
+          * Do not free the chunk, we may need it if the UNLOCK cmd fails
+          */
+         b->fallbackChunk = chunk;
 
-   /* set flag if reset requested */
-   if (status == BALLOON_ERROR_RESET) {
-      b->resetFlag = 1;
+         b->nChunks--;
+         chunk = NULL;
    }
 
-   /* update stats */
-   STATS_INC(b->stats.lock);
+      b->balloonOps->addPage(b, nPages++, lockedHandle);
+      if (nPages == b->batchMaxPages) {
+         status = b->balloonOps->unlock(b, nPages);
+         nPages = 0;
+
    if (status != BALLOON_SUCCESS) {
-      STATS_INC(b->stats.lockFail);
+            if (BALLOON_RATE_ADAPT) {
+               /* quickly decrease rate if error */
+               b->rateFree = MAX(b->rateFree / 2, BALLOON_RATE_FREE_MIN);
+            }
+            goto out;
+         }
+
+      }
    }
 
+   if (nPages) {
+      status = b->balloonOps->unlock(b, nPages);
+   }
+
+   if (status == BALLOON_SUCCESS && BALLOON_RATE_ADAPT) {
+      /* slowly increase rate if no errors */
+      b->rateFree = MIN(b->rateFree + BALLOON_RATE_FREE_INC,
+                        BALLOON_RATE_FREE_MAX);
+   }
+
+out:
    return status;
 }
 
@@ -1112,10 +1156,10 @@ BalloonMonitorLockPage(Balloon *b,        // IN
 /*
  *----------------------------------------------------------------------
  *
- * BalloonMonitorUnlockPage --
+ * BalloonAdjustSize --
  *
- *      Attempts to contact monitor and remove PPN corresponding to
- *      the page handle from set of "balloon locked" pages.
+ *      Attempts to allocate or deallocate physical pages in order
+ *      to reach desired "target" size for balloon "b".
  *
  * Results:
  *      Returns BALLOON_SUCCESS if successful, otherwise error code.
@@ -1127,54 +1171,25 @@ BalloonMonitorLockPage(Balloon *b,        // IN
  */
 
 static int
-BalloonMonitorUnlockPage(Balloon *b,        // IN
-                          PageHandle handle) // IN
+BalloonAdjustSize(Balloon *b,    // IN
+                  uint32 target) // IN
 {
-   unsigned long ppn;
-   uint32 ppn32;
-   uint32 status, target;
-   Backdoor_proto bp;
-
-   ppn = OS_ReservedPageGetPPN(handle);
-
-   /* Ensure PPN fits in 32-bits, i.e. guest memory is limited to 16TB. */
-   ppn32 = (uint32)ppn;
-   if (ppn32 != ppn) {
-      return BALLOON_ERROR_PPN_INVALID;
-   }
-
-   /* prepare backdoor args */
-   bp.in.cx.halfs.low = BALLOON_BDOOR_CMD_UNLOCK;
-   bp.in.size = ppn32;
-
-   /* invoke backdoor */
-   Backdoor_Balloon(&bp);
-
-   /* parse return values */
-   status = bp.out.ax.word;
-   target = bp.out.bx.word;
-
-   /* set flag if reset requested */
-   if (status == BALLOON_ERROR_RESET) {
-      b->resetFlag = 1;
-   }
-
-   /* update stats */
-   STATS_INC(b->stats.unlock);
-   if (status != BALLOON_SUCCESS) {
-      STATS_INC(b->stats.unlockFail);
+   if (b->nPages < target) {
+      return BalloonInflate(b, target);
+   } else if (b->nPages > target) {
+      return BalloonDeflate(b, target);
+   } else {
+      /* already at target */
+      return BALLOON_SUCCESS;
    }
-
-   return status;
 }
 
-
 /*
  *----------------------------------------------------------------------
  *
  * Balloon_Init --
  *
- *      Initializes state oif the balloon.
+ *      Initializes state of the balloon.
  *
  * Results:
  *      Returns TRUE on success (always).
@@ -1201,6 +1216,12 @@ Balloon_Init(BalloonGuest guestType)
    /* initialize reset flag */
    b->resetFlag = TRUE;
 
+   b->hypervisorCapabilities = 0;
+
+   b->pageHandle = PAGE_HANDLE_INVALID;
+   b->batchPageMapping = MAPPING_INVALID;
+   b->batchPage = NULL;
+
    return TRUE;
 }
 
@@ -1231,7 +1252,7 @@ Balloon_Cleanup(void)
     * Reset connection before deallocating memory to avoid potential for
     * additional spurious resets from guest touching deallocated pages.
     */
-   BalloonMonitorStart(b);
+   Backdoor_MonitorStart(b, BALLOON_CAPABILITIES);
    Balloon_Deallocate(b);
 }
 
diff --git a/sys/gnu/dev/virtual/vmware/vmmemctl/vmballoon.h b/sys/gnu/dev/virtual/vmware/vmmemctl/vmballoon.h
index 5b7ab37..f814b30 100644
--- a/sys/gnu/dev/virtual/vmware/vmmemctl/vmballoon.h
+++ b/sys/gnu/dev/virtual/vmware/vmmemctl/vmballoon.h
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2000 VMware, Inc. All rights reserved.
+ * Copyright (C) 2000-2012 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -23,12 +23,10 @@
 #ifndef	VMBALLOON_H
 #define	VMBALLOON_H
 
+#include "balloonInt.h"
 #include "vm_basic_types.h"
-
-#define BALLOON_NAME                    "vmmemctl"
-#define BALLOON_NAME_VERBOSE            "VMware memory control driver"
-
-#define BALLOON_POLL_PERIOD             1 /* sec */
+#include "dbllnklst.h"
+#include "os.h"
 
 /*
  * Page allocation flags
@@ -76,6 +74,71 @@ typedef struct {
    uint32 guestTypeFail;
 } BalloonStats;
 
+#define BALLOON_ERROR_PAGES             16
+
+typedef struct {
+   PageHandle page[BALLOON_ERROR_PAGES];
+   uint32 pageCount;
+} BalloonErrorPages;
+
+#define BALLOON_CHUNK_PAGES             1000
+
+typedef struct BalloonChunk {
+   PageHandle page[BALLOON_CHUNK_PAGES];
+   uint32 pageCount;
+   DblLnkLst_Links node;
+} BalloonChunk;
+
+struct BalloonOps;
+
+typedef struct {
+   /* sets of reserved physical pages */
+   DblLnkLst_Links chunks;
+   int nChunks;
+
+   /* transient list of non-balloonable pages */
+   BalloonErrorPages errors;
+
+   BalloonGuest guestType;
+
+   /* balloon size */
+   int nPages;
+   int nPagesTarget;
+
+   /* reset flag */
+   int resetFlag;
+
+   /* adjustment rates (pages per second) */
+   int rateAlloc;
+   int rateFree;
+
+   /* slowdown page allocations for next few cycles */
+   int slowPageAllocationCycles;
+
+   /* statistics */
+   BalloonStats stats;
+
+   /* hypervisor exposed capabilities */
+   BalloonCapabilities hypervisorCapabilities;
+
+   /* balloon operations, tied to the capabilities */
+   const struct BalloonOps *balloonOps;
+
+   /* Either the batch page handle, or the page to lock on v2 */
+   PageHandle pageHandle;
+   Mapping batchPageMapping;
+   BalloonBatchPage *batchPage;
+   uint16 batchMaxPages;
+
+   BalloonChunk *fallbackChunk;
+} Balloon;
+
+typedef struct BalloonOps {
+   void (*addPage)(Balloon *b, uint16 idx, PageHandle page);
+   int (*lock)(Balloon *b, uint16 nPages);
+   int (*unlock)(Balloon *b, uint16 nPages);
+} BalloonOps;
+
 /*
  * Operations
  */
-- 
2.0.1

