From 0b7381572b131c74051832dc251604e7f77b5a54 Mon Sep 17 00:00:00 2001
From: Tomohiro Kusumi <tkusumi@netbsd.org>
Date: Sat, 26 Dec 2020 01:36:00 +0900
Subject: [PATCH] sys/vfs/hammer2: Add initial multi-volumes support for
 HAMMER2

This commit adds initial multi-volumes support for HAMMER2. Maximum
supported volumes is 64. The feature and implementation is similar to
multi-volumes support in HAMMER1.

1. ondisk changes
=================
This commit bumps volume header version from 1 to 2, and adds four new
volume header fields using reserved fields in version 1. Other ondisk
structures are unchanged.
* "volu_id" - volume id from 0 to 63, where 0 represents root volume.
* "nvolumes" - number of volumes. All volumes have same the same value.
* "total_size" - sum of "volu_size" in volumes. All volumes have the
  same value.
* "volu_loff[HAMMER2_MAX_VOLUMES]" - A 512 bytes table which contains
  start offset of max 64 volumes within "total_size". All volumes have
  the same value.

Version 1 volume header has 0 for above fields, so HAMMER2 internally
treats "nvolumes" as 1, and "total_size" as "volu_size" to be able to
handle version 1 and 2 transparently.

All volumes have 4 headers, but only root volume ones are relevant.
Non-root volume headers have their own unique "volu_id" and "volu_size",
but other fields are unimportant and never used. Non-root volume headers
have sroot blockset[i] whose type is HAMMER2_BREF_TYPE_INVALID. Non-root
volume headers don't have boot/aux area, so freemap area start from
offset 0. Non-root volume headers are readonly and never updated after
creation. This means non-root volumes are just extra storage to extend
fs size and internally make up a single virtual volume whose size is
"total_size".

It currently doesn't automatically upgrade an existing version 1 fs to
version 2. Only newly created fs becomes version 2 for now.

2. volumes layout
=================
Basically similar to HAMMER1. A first block device argument provided for
newfs_hammer2(8) becomes the root volume, and if specified remaining
devices extend "total_size" as non-root volumes. All volumes except for
the last one have 1GiB (freemap level1) aligned "volu_size".

This means each volume's start offset within "total_size" is also 1GiB
(freemap level1) aligned. The start offsets of volumes are stored in
volu_loff[HAMMER2_MAX_VOLUMES]. Each volu_loff[n] (0 <= n < nvolumes)
represents start offset of volume n within "total_size". Unused volumes
have -1 for volu_loff[n].
e.g. If a fs consists of 1 volume, volu_loff[0] has 0 and rests have -1.
e.g. If a fs consists of 3 volumes, x GiB root volume, y GiB volume,
  and z GiB volume, volu_loff[0] has 0, volu_loff[1] has x, volu_loff[2]
  has x+y, and rests have -1.

Low level I/O function in HAMMER2 uses this linear offsets table to
determine a device vnode to use and relative offset within the device
vnode, for a given blockref's "data_off". This is different from HAMMER1
where logical offset had embedded volume id bits (i.e. there were holes
in logical address space). HAMMER2 needs this table to support multi-
volumes without changing current logical offset mechanism.

Unless all volumes are specified and mountable, mount_hammer2(8) fails
like it failed in HAMMER1. This also applies to other userspace commands
which require volumes specification, except for fstyp(8).

3. userspace commands
=====================
Basically same as or similar to HAMMER1.
* newfs_hammer2(8) takes a list of block device paths as argv[].
* mount_hammer2(8) takes block device paths or names in "a:b:c:..."
  format.
* hammer2(8) takes block device paths or names in "a:b:c:..." format for
  directives which require volumes specification. This commit also adds
  "volume-list" directive and an ioctl command HAMMER2IOC_VOLUME_LIST,
  which are similar to the one in HAMMER1.
* fsck_hammer2(8) takes device paths or names in "a:b:c:..." format.
* fstyp(8) takes device paths in "path1:path2:path3:..." format.

4. limitations
==============
* hammer2(8) "info" directive ignores multi-volumes block devices.
* hammer2(8) "growfs" directive doesn't support multi-volumes fs.
* fstyp(8) is unable to find PFS label via -l option if the PFS inode or
  its parent indirect blocks are located beyond root volume.
* hammer2(8) doesn't support "volume-add" and "volume-del" directives
  which existed in HAMMER1, and there is currently no plan to support.
---
 sbin/fsck_hammer2/Makefile         |   2 +-
 sbin/fsck_hammer2/destroy.c        | 104 ++---
 sbin/fsck_hammer2/reconstruct.c    | 110 +++--
 sbin/fsck_hammer2/test.c           | 178 +++-----
 sbin/hammer2/Makefile              |   4 +-
 sbin/hammer2/cmd_debug.c           | 108 +++--
 sbin/hammer2/cmd_info.c            |  22 +-
 sbin/hammer2/cmd_volume.c          | 106 +++++
 sbin/hammer2/hammer2.8             |  11 +-
 sbin/hammer2/hammer2.h             |   1 +
 sbin/hammer2/hammer2_subs.h        |  43 ++
 sbin/hammer2/main.c                |  12 +
 sbin/hammer2/ondisk.c              | 654 ++++++++++++++++++++++++++
 sbin/hammer2/subs.c                |  44 ++
 sbin/mount_hammer2/mount_hammer2.c |   6 +-
 sbin/newfs_hammer2/Makefile        |   2 +-
 sbin/newfs_hammer2/newfs_hammer2.8 |   4 +-
 sbin/newfs_hammer2/newfs_hammer2.c | 322 +++++++------
 sys/conf/files                     |   1 +
 sys/vfs/hammer2/Makefile           |   2 +-
 sys/vfs/hammer2/hammer2.h          |  51 ++-
 sys/vfs/hammer2/hammer2_bulkfree.c |  28 +-
 sys/vfs/hammer2/hammer2_disk.h     |  36 +-
 sys/vfs/hammer2/hammer2_flush.c    |  18 +-
 sys/vfs/hammer2/hammer2_freemap.c  |  12 +-
 sys/vfs/hammer2/hammer2_io.c       |  27 +-
 sys/vfs/hammer2/hammer2_ioctl.c    |  53 +++
 sys/vfs/hammer2/hammer2_ioctl.h    |  22 +
 sys/vfs/hammer2/hammer2_ondisk.c   | 707 +++++++++++++++++++++++++++++
 sys/vfs/hammer2/hammer2_subr.c     |   2 +
 sys/vfs/hammer2/hammer2_vfsops.c   | 346 ++++++--------
 usr.sbin/fstyp/fstyp.c             |   2 +
 usr.sbin/fstyp/fstyp.h             |   2 +
 usr.sbin/fstyp/hammer2.c           | 206 ++++++++-
 34 files changed, 2579 insertions(+), 669 deletions(-)
 create mode 100644 sbin/hammer2/cmd_volume.c
 create mode 100644 sbin/hammer2/ondisk.c
 create mode 100644 sys/vfs/hammer2/hammer2_ondisk.c

diff --git a/sbin/fsck_hammer2/Makefile b/sbin/fsck_hammer2/Makefile
index 03a240ecc7..823908d5cf 100644
--- a/sbin/fsck_hammer2/Makefile
+++ b/sbin/fsck_hammer2/Makefile
@@ -14,6 +14,6 @@ LDFLAGS+=${PRIVATELIB_LDFLAGS}
 .PATH: ${.CURDIR}/../../sys/libkern ${.CURDIR}/../../sys/vfs/hammer2/xxhash
 SRCS+= icrc32.c xxhash.c
 .PATH: ${.CURDIR}/../hammer2
-SRCS+= subs.c
+SRCS+= subs.c ondisk.c
 
 .include <bsd.prog.mk>
diff --git a/sbin/fsck_hammer2/destroy.c b/sbin/fsck_hammer2/destroy.c
index 1d1d0ef2fb..c1e7df38b9 100644
--- a/sbin/fsck_hammer2/destroy.c
+++ b/sbin/fsck_hammer2/destroy.c
@@ -34,8 +34,7 @@
  * SUCH DAMAGE.
  */
 
-// # gcc -Wall -g -I../../sys -I../hammer2
-// ../../sys/libkern/icrc32.c ../hammer2/subs.c ./destroy.c -o destroy
+// # gcc -Wall -g -I../../sys -I../hammer2 ../../sys/libkern/icrc32.c ../hammer2/subs.c ../hammer2/ondisk.c ./destroy.c -o destroy
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -53,13 +52,13 @@
 
 #include "hammer2_subs.h"
 
-static int modify_blockref(int, const hammer2_volume_data_t *, int,
+static int modify_blockref(const hammer2_volume_data_t *, int,
     hammer2_blockref_t *, hammer2_blockref_t *);
-static int modify_inode(int, const hammer2_blockref_t *,
+static int modify_inode(const hammer2_blockref_t *,
+    hammer2_media_data_t *, size_t);
+static int modify_dirent_embedded(int, hammer2_blockref_t *);
+static int modify_dirent(int, hammer2_blockref_t *, const hammer2_blockref_t *,
     hammer2_media_data_t *, size_t);
-static int modify_dirent_embedded(int, int, hammer2_blockref_t *);
-static int modify_dirent(int, int, hammer2_blockref_t *,
-    const hammer2_blockref_t *, hammer2_media_data_t *, size_t);
 
 static hammer2_tid_t src_inode = 0;
 static hammer2_tid_t dst_inode = 0;
@@ -68,7 +67,7 @@ static const char *dst_dirent = NULL;
 static bool ForceOpt = false;
 
 static int
-destroy_blockref(int fd, uint8_t type)
+destroy_blockref(uint8_t type)
 {
 	bool failed = false;
 	int i;
@@ -76,23 +75,25 @@ destroy_blockref(int fd, uint8_t type)
 	for (i = 0; i < HAMMER2_NUM_VOLHDRS; ++i) {
 		hammer2_volume_data_t voldata;
 		hammer2_blockref_t broot;
+		hammer2_off_t off;
 		ssize_t ret;
 
 		memset(&broot, 0, sizeof(broot));
 		broot.type = type;
 		broot.data_off = (i * HAMMER2_ZONE_BYTES64) | HAMMER2_PBUFRADIX;
-		if (lseek(fd, broot.data_off & ~HAMMER2_OFF_MASK_RADIX,
-		    SEEK_SET) == -1) {
+		off = broot.data_off & ~HAMMER2_OFF_MASK_RADIX;
+		if (lseek(hammer2_get_root_volume_fd(),
+		    off - hammer2_get_root_volume_offset(), SEEK_SET) == -1) {
 			perror("lseek");
 			return -1;
 		}
 
-		ret = read(fd, &voldata, HAMMER2_PBUFSIZE);
+		ret = read(hammer2_get_root_volume_fd(), &voldata,
+		    HAMMER2_PBUFSIZE);
 		if (ret == HAMMER2_PBUFSIZE) {
 			fprintf(stdout, "zone.%d %016jx\n",
 			    i, (uintmax_t)broot.data_off);
-			if (modify_blockref(fd, &voldata, -1, &broot, NULL)
-			    == -1)
+			if (modify_blockref(&voldata, -1, &broot, NULL) == -1)
 				failed = true;
 		} else if (ret == -1) {
 			perror("read");
@@ -107,12 +108,13 @@ destroy_blockref(int fd, uint8_t type)
 }
 
 static int
-read_media(int fd, const hammer2_blockref_t *bref, hammer2_media_data_t *media,
+read_media(const hammer2_blockref_t *bref, hammer2_media_data_t *media,
     size_t *media_bytes)
 {
 	hammer2_off_t io_off, io_base;
 	size_t bytes, io_bytes, boff;
 	ssize_t ret;
+	int fd;
 
 	bytes = (bref->data_off & HAMMER2_OFF_MASK_RADIX);
 	if (bytes)
@@ -135,7 +137,9 @@ read_media(int fd, const hammer2_blockref_t *bref, hammer2_media_data_t *media,
 		fprintf(stderr, "Bad I/O bytes\n");
 		return -1;
 	}
-	if (lseek(fd, io_base, SEEK_SET) == -1) {
+	fd = hammer2_get_volume_fd(io_off);
+	if (lseek(fd, io_base - hammer2_get_volume_offset(io_base), SEEK_SET)
+	    == -1) {
 		perror("lseek");
 		return -1;
 	}
@@ -154,13 +158,14 @@ read_media(int fd, const hammer2_blockref_t *bref, hammer2_media_data_t *media,
 }
 
 static int
-write_media(int fd, const hammer2_blockref_t *bref,
-    const hammer2_media_data_t *media, size_t media_bytes)
+write_media(const hammer2_blockref_t *bref, const hammer2_media_data_t *media,
+    size_t media_bytes)
 {
 	hammer2_off_t io_off, io_base;
 	char buf[HAMMER2_PBUFSIZE];
 	size_t bytes, io_bytes, boff;
 	ssize_t ret;
+	int fd;
 
 	bytes = (bref->data_off & HAMMER2_OFF_MASK_RADIX);
 	if (bytes)
@@ -180,7 +185,9 @@ write_media(int fd, const hammer2_blockref_t *bref,
 		fprintf(stderr, "Bad I/O bytes\n");
 		return -1;
 	}
-	if (lseek(fd, io_base, SEEK_SET) == -1) {
+	fd = hammer2_get_volume_fd(io_off);
+	if (lseek(fd, io_base - hammer2_get_volume_offset(io_base), SEEK_SET)
+	    == -1) {
 		perror("lseek");
 		return -1;
 	}
@@ -190,7 +197,8 @@ write_media(int fd, const hammer2_blockref_t *bref,
 	}
 
 	memcpy(buf + boff, media, media_bytes);
-	if (lseek(fd, io_base, SEEK_SET) == -1) {
+	if (lseek(fd, io_base - hammer2_get_volume_offset(io_base), SEEK_SET)
+	    == -1) {
 		perror("lseek");
 		return -1;
 	}
@@ -211,7 +219,7 @@ write_media(int fd, const hammer2_blockref_t *bref,
 }
 
 static int
-modify_blockref(int fd, const hammer2_volume_data_t *voldata, int bi,
+modify_blockref(const hammer2_volume_data_t *voldata, int bi,
     hammer2_blockref_t *bref, hammer2_blockref_t *prev_bref)
 {
 	hammer2_media_data_t media;
@@ -219,7 +227,7 @@ modify_blockref(int fd, const hammer2_volume_data_t *voldata, int bi,
 	int i, bcount, namlen;
 	size_t bytes;
 
-	if (read_media(fd, bref, &media, &bytes) == -1)
+	if (read_media(bref, &media, &bytes) == -1)
 		return -1;
 
 	switch (bref->type) {
@@ -232,7 +240,7 @@ modify_blockref(int fd, const hammer2_volume_data_t *voldata, int bi,
 			bcount = 0;
 		}
 		if (src_inode && media.ipdata.meta.inum == src_inode)
-			if (modify_inode(fd, bref, &media, bytes) == -1)
+			if (modify_inode(bref, &media, bytes) == -1)
 				return -1;
 		break;
 	case HAMMER2_BREF_TYPE_INDIRECT:
@@ -246,12 +254,11 @@ modify_blockref(int fd, const hammer2_volume_data_t *voldata, int bi,
 		if (src_dirent && namlen == strlen(src_dirent)) {
 			if (namlen <= sizeof(bref->check.buf) &&
 			    !memcmp(bref->check.buf, src_dirent, namlen)) {
-				if (modify_dirent_embedded(fd, bi, prev_bref)
-				    == -1)
+				if (modify_dirent_embedded(bi, prev_bref) == -1)
 					return -1;
 			} else if (!memcmp(media.buf, src_dirent, namlen)) {
-				if (modify_dirent(fd, bi, prev_bref, bref,
-				    &media, bytes) == -1)
+				if (modify_dirent(bi, prev_bref, bref, &media,
+				    bytes) == -1)
 					return -1;
 			}
 		}
@@ -268,21 +275,20 @@ modify_blockref(int fd, const hammer2_volume_data_t *voldata, int bi,
 
 	for (i = 0; i < bcount; ++i)
 		if (bscan[i].type != HAMMER2_BREF_TYPE_EMPTY)
-			if (modify_blockref(fd, voldata, i, &bscan[i], bref)
-			    == -1)
+			if (modify_blockref(voldata, i, &bscan[i], bref) == -1)
 				return -1;
 	return 0;
 }
 
 static int
-modify_inode(int fd, const hammer2_blockref_t *bref,
+modify_inode(const hammer2_blockref_t *bref,
     hammer2_media_data_t *media, size_t media_bytes)
 {
 	assert(src_inode == media->ipdata.meta.inum);
 
 	if (ForceOpt) {
 		media->ipdata.meta.inum = dst_inode;
-		if (write_media(fd, bref, media, media_bytes) == -1)
+		if (write_media(bref, media, media_bytes) == -1)
 			return -1;
 	}
 
@@ -293,13 +299,13 @@ modify_inode(int fd, const hammer2_blockref_t *bref,
 }
 
 static int
-modify_dirent_embedded(int fd, int bi, hammer2_blockref_t *prev_bref)
+modify_dirent_embedded(int bi, hammer2_blockref_t *prev_bref)
 {
 	hammer2_media_data_t bscan_media;
 	hammer2_blockref_t *bscan;
 	size_t bytes;
 
-	if (read_media(fd, prev_bref, &bscan_media, &bytes) == -1)
+	if (read_media(prev_bref, &bscan_media, &bytes) == -1)
 		return -1;
 	assert(bytes);
 
@@ -329,7 +335,7 @@ modify_dirent_embedded(int fd, int bi, hammer2_blockref_t *prev_bref)
 		bscan->embed.dirent.namlen = strlen(dst_dirent);
 		bscan->key = dirhash((const unsigned char*)dst_dirent,
 		    strlen(dst_dirent));
-		if (write_media(fd, prev_bref, &bscan_media, bytes) == -1)
+		if (write_media(prev_bref, &bscan_media, bytes) == -1)
 			return -1;
 	}
 
@@ -342,7 +348,7 @@ modify_dirent_embedded(int fd, int bi, hammer2_blockref_t *prev_bref)
 }
 
 static int
-modify_dirent(int fd, int bi, hammer2_blockref_t *prev_bref,
+modify_dirent(int bi, hammer2_blockref_t *prev_bref,
     const hammer2_blockref_t *bref, hammer2_media_data_t *media,
     size_t media_bytes)
 {
@@ -351,7 +357,7 @@ modify_dirent(int fd, int bi, hammer2_blockref_t *prev_bref,
 	size_t bytes;
 
 	assert(!memcmp(src_dirent, media->buf, strlen(src_dirent)));
-	if (read_media(fd, prev_bref, &bscan_media, &bytes) == -1)
+	if (read_media(prev_bref, &bscan_media, &bytes) == -1)
 		return -1;
 	assert(bytes);
 
@@ -389,12 +395,12 @@ modify_dirent(int fd, int bi, hammer2_blockref_t *prev_bref,
 		bscan->embed.dirent.namlen = strlen(dst_dirent);
 		bscan->key = dirhash((const unsigned char*)dst_dirent,
 		    strlen(dst_dirent));
-		if (write_media(fd, bref, media, media_bytes) == -1)
+		if (write_media(bref, media, media_bytes) == -1)
 			return -1;
-		if (write_media(fd, prev_bref, &bscan_media, bytes) == -1) {
+		if (write_media(prev_bref, &bscan_media, bytes) == -1) {
 			memset(media->buf, 0, sizeof(media->buf));
 			memcpy(media->buf, src_dirent, strlen(src_dirent));
-			if (write_media(fd, bref, media, media_bytes) == -1)
+			if (write_media(bref, media, media_bytes) == -1)
 				return -1;
 			return -1;
 		}
@@ -472,8 +478,7 @@ init_args(int argc, char **argv, const char **devpathp)
 int
 main(int argc, char **argv)
 {
-	struct stat st;
-	int ch, fd;
+	int ch;
 	const char *binpath = argv[0];
 	const char *devpath;
 
@@ -497,25 +502,12 @@ main(int argc, char **argv)
 	if (init_args(argc, argv, &devpath) == -1)
 		exit(1);
 
-	fd = open(devpath, O_RDWR);
-	if (fd == -1) {
-		perror("open");
-		exit(1);
-	}
-
-	if (fstat(fd, &st) == -1) {
-		perror("fstat");
-		exit(1);
-	}
-	if (!S_ISCHR(st.st_mode) && !S_ISREG(st.st_mode)) {
-		fprintf(stderr, "Unsupported file type\n");
-		exit(1);
-	}
+	hammer2_init_volumes(devpath, 0);
 
-	if (destroy_blockref(fd, HAMMER2_BREF_TYPE_VOLUME) == -1)
+	if (destroy_blockref(HAMMER2_BREF_TYPE_VOLUME) == -1)
 		exit(1);
 
-	close(fd);
+	hammer2_cleanup_volumes();
 
 	return 0;
 }
diff --git a/sbin/fsck_hammer2/reconstruct.c b/sbin/fsck_hammer2/reconstruct.c
index 5a9f595f15..cb9185f161 100644
--- a/sbin/fsck_hammer2/reconstruct.c
+++ b/sbin/fsck_hammer2/reconstruct.c
@@ -34,8 +34,7 @@
  * SUCH DAMAGE.
  */
 
-// # gcc -Wall -g -I../../sys -I../hammer2 ../../sys/vfs/hammer2/xxhash/xxhash.c
-// ../../sys/libkern/icrc32.c ../hammer2/subs.c ./reconstruct.c -o reconstruct
+// # gcc -Wall -g -I../../sys -I../hammer2 ../../sys/vfs/hammer2/xxhash/xxhash.c ../../sys/libkern/icrc32.c ../hammer2/subs.c ../hammer2/ondisk.c ./reconstruct.c -o reconstruct
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -54,17 +53,17 @@
 
 #include "hammer2_subs.h"
 
-static int modify_volume_header(int, hammer2_volume_data_t *,
+static int modify_volume_header(hammer2_volume_data_t *,
     const hammer2_blockref_t *);
-static int modify_blockref(int, const hammer2_volume_data_t *, int,
+static int modify_blockref(const hammer2_volume_data_t *, int,
     hammer2_blockref_t *, hammer2_blockref_t *, int);
-static int modify_check(int, int, hammer2_blockref_t *,
-    const hammer2_blockref_t *, hammer2_media_data_t *, size_t, int);
+static int modify_check(int, hammer2_blockref_t *, const hammer2_blockref_t *,
+    hammer2_media_data_t *, size_t, int);
 
 static bool ForceOpt = false;
 
 static int
-reconstruct_volume_header(int fd)
+reconstruct_volume_header(void)
 {
 	bool failed = false;
 	int i;
@@ -72,21 +71,24 @@ reconstruct_volume_header(int fd)
 	for (i = 0; i < HAMMER2_NUM_VOLHDRS; ++i) {
 		hammer2_volume_data_t voldata;
 		hammer2_blockref_t broot;
+		hammer2_off_t off;
 		ssize_t ret;
 
 		memset(&broot, 0, sizeof(broot));
 		broot.data_off = (i * HAMMER2_ZONE_BYTES64) | HAMMER2_PBUFRADIX;
-		if (lseek(fd, broot.data_off & ~HAMMER2_OFF_MASK_RADIX,
-		    SEEK_SET) == -1) {
+		off = broot.data_off & ~HAMMER2_OFF_MASK_RADIX;
+		if (lseek(hammer2_get_root_volume_fd(),
+		    off - hammer2_get_root_volume_offset(), SEEK_SET) == -1) {
 			perror("lseek");
 			return -1;
 		}
 
-		ret = read(fd, &voldata, HAMMER2_PBUFSIZE);
+		ret = read(hammer2_get_root_volume_fd(), &voldata,
+		    HAMMER2_PBUFSIZE);
 		if (ret == HAMMER2_PBUFSIZE) {
 			fprintf(stdout, "zone.%d %016jx\n",
 			    i, (uintmax_t)broot.data_off);
-			if (modify_volume_header(fd, &voldata, &broot) == -1)
+			if (modify_volume_header(&voldata, &broot) == -1)
 				failed = true;
 		} else if (ret == -1) {
 			perror("read");
@@ -101,7 +103,7 @@ reconstruct_volume_header(int fd)
 }
 
 static int
-reconstruct_blockref(int fd, uint8_t type)
+reconstruct_blockref(uint8_t type)
 {
 	bool failed = false;
 	int i;
@@ -109,23 +111,26 @@ reconstruct_blockref(int fd, uint8_t type)
 	for (i = 0; i < HAMMER2_NUM_VOLHDRS; ++i) {
 		hammer2_volume_data_t voldata;
 		hammer2_blockref_t broot;
+		hammer2_off_t off;
 		ssize_t ret;
 
 		memset(&broot, 0, sizeof(broot));
 		broot.type = type;
 		broot.data_off = (i * HAMMER2_ZONE_BYTES64) | HAMMER2_PBUFRADIX;
-		if (lseek(fd, broot.data_off & ~HAMMER2_OFF_MASK_RADIX,
-		    SEEK_SET) == -1) {
+		off = broot.data_off & ~HAMMER2_OFF_MASK_RADIX;
+		if (lseek(hammer2_get_root_volume_fd(),
+		    off - hammer2_get_root_volume_offset(), SEEK_SET) == -1) {
 			perror("lseek");
 			return -1;
 		}
 
-		ret = read(fd, &voldata, HAMMER2_PBUFSIZE);
+		ret = read(hammer2_get_root_volume_fd(), &voldata,
+		    HAMMER2_PBUFSIZE);
 		if (ret == HAMMER2_PBUFSIZE) {
 			fprintf(stdout, "zone.%d %016jx\n",
 			    i, (uintmax_t)broot.data_off);
-			if (modify_blockref(fd, &voldata, -1, &broot, NULL, -1)
-			    == -1)
+			if (modify_blockref(&voldata, -1, &broot, NULL, -1) ==
+			    -1)
 				failed = true;
 		} else if (ret == -1) {
 			perror("read");
@@ -140,7 +145,7 @@ reconstruct_blockref(int fd, uint8_t type)
 }
 
 static int
-modify_volume_header(int fd, hammer2_volume_data_t *voldata,
+modify_volume_header(hammer2_volume_data_t *voldata,
     const hammer2_blockref_t *bref)
 {
 	hammer2_crc32_t crc0, crc1;
@@ -195,8 +200,10 @@ modify_volume_header(int fd, hammer2_volume_data_t *voldata,
 
 	if (found && ForceOpt) {
 		ssize_t ret;
-		if (lseek(fd, bref->data_off & ~HAMMER2_OFF_MASK_RADIX,
-		    SEEK_SET) == -1) {
+		int fd = hammer2_get_root_volume_fd();
+		hammer2_off_t off = bref->data_off & ~HAMMER2_OFF_MASK_RADIX;
+		if (lseek(fd, off - hammer2_get_root_volume_offset(), SEEK_SET)
+		    == -1) {
 			perror("lseek");
 			return -1;
 		}
@@ -218,12 +225,13 @@ modify_volume_header(int fd, hammer2_volume_data_t *voldata,
 }
 
 static int
-read_media(int fd, const hammer2_blockref_t *bref, hammer2_media_data_t *media,
+read_media(const hammer2_blockref_t *bref, hammer2_media_data_t *media,
     size_t *media_bytes)
 {
 	hammer2_off_t io_off, io_base;
 	size_t bytes, io_bytes, boff;
 	ssize_t ret;
+	int fd;
 
 	bytes = (bref->data_off & HAMMER2_OFF_MASK_RADIX);
 	if (bytes)
@@ -246,7 +254,9 @@ read_media(int fd, const hammer2_blockref_t *bref, hammer2_media_data_t *media,
 		fprintf(stderr, "Bad I/O bytes\n");
 		return -1;
 	}
-	if (lseek(fd, io_base, SEEK_SET) == -1) {
+	fd = hammer2_get_volume_fd(io_off);
+	if (lseek(fd, io_base - hammer2_get_volume_offset(io_base), SEEK_SET)
+	    == -1) {
 		perror("lseek");
 		return -1;
 	}
@@ -265,13 +275,14 @@ read_media(int fd, const hammer2_blockref_t *bref, hammer2_media_data_t *media,
 }
 
 static int
-write_media(int fd, const hammer2_blockref_t *bref,
-    const hammer2_media_data_t *media, size_t media_bytes)
+write_media(const hammer2_blockref_t *bref, const hammer2_media_data_t *media,
+    size_t media_bytes)
 {
 	hammer2_off_t io_off, io_base;
 	char buf[HAMMER2_PBUFSIZE];
 	size_t bytes, io_bytes, boff;
 	ssize_t ret;
+	int fd;
 
 	bytes = (bref->data_off & HAMMER2_OFF_MASK_RADIX);
 	if (bytes)
@@ -291,7 +302,9 @@ write_media(int fd, const hammer2_blockref_t *bref,
 		fprintf(stderr, "Bad I/O bytes\n");
 		return -1;
 	}
-	if (lseek(fd, io_base, SEEK_SET) == -1) {
+	fd = hammer2_get_volume_fd(io_off);
+	if (lseek(fd, io_base - hammer2_get_volume_offset(io_base), SEEK_SET)
+	    == -1) {
 		perror("lseek");
 		return -1;
 	}
@@ -301,7 +314,8 @@ write_media(int fd, const hammer2_blockref_t *bref,
 	}
 
 	memcpy(buf + boff, media, media_bytes);
-	if (lseek(fd, io_base, SEEK_SET) == -1) {
+	if (lseek(fd, io_base - hammer2_get_volume_offset(io_base), SEEK_SET)
+	    == -1) {
 		perror("lseek");
 		return -1;
 	}
@@ -322,7 +336,7 @@ write_media(int fd, const hammer2_blockref_t *bref,
 }
 
 static int
-modify_blockref(int fd, const hammer2_volume_data_t *voldata, int bi,
+modify_blockref(const hammer2_volume_data_t *voldata, int bi,
     hammer2_blockref_t *bref, hammer2_blockref_t *prev_bref, int depth)
 {
 	hammer2_media_data_t media;
@@ -330,7 +344,7 @@ modify_blockref(int fd, const hammer2_volume_data_t *voldata, int bi,
 	int i, bcount;
 	size_t bytes;
 
-	if (read_media(fd, bref, &media, &bytes) == -1)
+	if (read_media(bref, &media, &bytes) == -1)
 		return -1;
 
 	if (!bytes)
@@ -370,21 +384,21 @@ modify_blockref(int fd, const hammer2_volume_data_t *voldata, int bi,
 
 	for (i = 0; i < bcount; ++i)
 		if (bscan[i].type != HAMMER2_BREF_TYPE_EMPTY)
-			if (modify_blockref(fd, voldata, i, &bscan[i], bref,
+			if (modify_blockref(voldata, i, &bscan[i], bref,
 			    depth + 1) == -1)
 				return -1;
 
 	if (ForceOpt)
-		if (read_media(fd, bref, &media, &bytes) == -1)
+		if (read_media(bref, &media, &bytes) == -1)
 			return -1;
-	if (modify_check(fd, bi, prev_bref, bref, &media, bytes, depth) == -1)
+	if (modify_check(bi, prev_bref, bref, &media, bytes, depth) == -1)
 		return -1;
 
 	return 0;
 }
 
 static int
-modify_check(int fd, int bi, hammer2_blockref_t *prev_bref,
+modify_check(int bi, hammer2_blockref_t *prev_bref,
     const hammer2_blockref_t *bref, hammer2_media_data_t *media,
     size_t media_bytes, int depth)
 {
@@ -404,7 +418,7 @@ modify_check(int fd, int bi, hammer2_blockref_t *prev_bref,
 
 	if (!prev_bref)
 		return 0;
-	if (read_media(fd, prev_bref, &bscan_media, &bytes) == -1)
+	if (read_media(prev_bref, &bscan_media, &bytes) == -1)
 		return -1;
 	assert(bytes);
 
@@ -484,8 +498,7 @@ modify_check(int fd, int bi, hammer2_blockref_t *prev_bref,
 
 	if (found) {
 		if (ForceOpt) {
-			if (write_media(fd, prev_bref, &bscan_media, bytes)
-			    == -1)
+			if (write_media(prev_bref, &bscan_media, bytes) == -1)
 				return -1;
 		}
 		/* If !ForceOpt, only first bad blockref is printed. */
@@ -502,8 +515,7 @@ modify_check(int fd, int bi, hammer2_blockref_t *prev_bref,
 int
 main(int argc, char **argv)
 {
-	struct stat st;
-	int ch, fd;
+	int ch;
 	const char *binpath = argv[0];
 	const char *devpath;
 
@@ -524,34 +536,20 @@ main(int argc, char **argv)
 		exit(1);
 	}
 	devpath = argv[0];
-
-	fd = open(devpath, O_RDWR);
-	if (fd == -1) {
-		perror("open");
-		exit(1);
-	}
-
-	if (fstat(fd, &st) == -1) {
-		perror("fstat");
-		exit(1);
-	}
-	if (!S_ISCHR(st.st_mode) && !S_ISREG(st.st_mode)) {
-		fprintf(stderr, "Unsupported file type\n");
-		exit(1);
-	}
+	hammer2_init_volumes(devpath, 0);
 
 	printf("freemap\n");
-	if (reconstruct_blockref(fd, HAMMER2_BREF_TYPE_FREEMAP) == -1)
+	if (reconstruct_blockref(HAMMER2_BREF_TYPE_FREEMAP) == -1)
 		exit(1);
 	printf("volume\n");
-	if (reconstruct_blockref(fd, HAMMER2_BREF_TYPE_VOLUME) == -1)
+	if (reconstruct_blockref(HAMMER2_BREF_TYPE_VOLUME) == -1)
 		exit(1);
 
 	printf("volume header\n");
-	if (reconstruct_volume_header(fd) == -1)
+	if (reconstruct_volume_header() == -1)
 		exit(1);
 
-	close(fd);
+	hammer2_cleanup_volumes();
 
 	return 0;
 }
diff --git a/sbin/fsck_hammer2/test.c b/sbin/fsck_hammer2/test.c
index cdbfb1ab62..35fe451032 100644
--- a/sbin/fsck_hammer2/test.c
+++ b/sbin/fsck_hammer2/test.c
@@ -125,27 +125,26 @@ typedef struct {
 	long count;
 } delta_stats_t;
 
-static void print_blockref_entry(int, struct blockref_tree *);
+static void print_blockref_entry(struct blockref_tree *);
 static void init_blockref_stats(blockref_stats_t *, uint8_t);
 static void cleanup_blockref_stats(blockref_stats_t *);
 static void init_delta_root(struct blockref_tree *);
 static void cleanup_delta_root(struct blockref_tree *);
 static void print_blockref_stats(const blockref_stats_t *, bool);
 static int verify_volume_header(const hammer2_volume_data_t *);
-static int read_media(int, const hammer2_blockref_t *, hammer2_media_data_t *,
+static int read_media(const hammer2_blockref_t *, hammer2_media_data_t *,
     size_t *);
-static int verify_blockref(int, const hammer2_volume_data_t *,
+static int verify_blockref(const hammer2_volume_data_t *,
     const hammer2_blockref_t *, bool, blockref_stats_t *,
     struct blockref_tree *, delta_stats_t *, int, int);
 static void print_pfs(const hammer2_inode_data_t *);
 static char *get_inode_filename(const hammer2_inode_data_t *);
-static int init_pfs_blockref(int, const hammer2_volume_data_t *,
+static int init_pfs_blockref(const hammer2_volume_data_t *,
     const hammer2_blockref_t *, struct blockref_list *);
 static void cleanup_pfs_blockref(struct blockref_list *);
 static void print_media(FILE *, int, const hammer2_blockref_t *,
     const hammer2_media_data_t *, size_t);
 
-static hammer2_off_t volume_size;
 static int best_zone = -1;
 
 #define TAB 8
@@ -189,20 +188,24 @@ tprintf_zone(int tab, int i, const hammer2_blockref_t *bref)
 }
 
 static int
-init_root_blockref(int fd, int i, uint8_t type, hammer2_blockref_t *bref)
+init_root_blockref(int i, uint8_t type, hammer2_blockref_t *bref)
 {
+	hammer2_off_t off;
+
 	assert(type == HAMMER2_BREF_TYPE_EMPTY ||
 		type == HAMMER2_BREF_TYPE_VOLUME ||
 		type == HAMMER2_BREF_TYPE_FREEMAP);
 	memset(bref, 0, sizeof(*bref));
 	bref->type = type;
 	bref->data_off = (i * HAMMER2_ZONE_BYTES64) | HAMMER2_PBUFRADIX;
+	off = bref->data_off & ~HAMMER2_OFF_MASK_RADIX;
 
-	return lseek(fd, bref->data_off & ~HAMMER2_OFF_MASK_RADIX, SEEK_SET);
+	return lseek(hammer2_get_root_volume_fd(),
+	    off - hammer2_get_root_volume_offset(), SEEK_SET);
 }
 
 static int
-find_best_zone(int fd)
+find_best_zone(void)
 {
 	hammer2_blockref_t best;
 	int i, best_i = -1;
@@ -214,10 +217,12 @@ find_best_zone(int fd)
 		hammer2_blockref_t broot;
 		ssize_t ret;
 
-		if (i * HAMMER2_ZONE_BYTES64 >= volume_size)
+		if (i * HAMMER2_ZONE_BYTES64 >=
+		    hammer2_get_root_volume_size())
 			break;
-		init_root_blockref(fd, i, HAMMER2_BREF_TYPE_EMPTY, &broot);
-		ret = read(fd, &voldata, HAMMER2_PBUFSIZE);
+		init_root_blockref(i, HAMMER2_BREF_TYPE_EMPTY, &broot);
+		ret = read(hammer2_get_root_volume_fd(), &voldata,
+		    HAMMER2_PBUFSIZE);
 		if (ret == HAMMER2_PBUFSIZE) {
 			if ((voldata.magic != HAMMER2_VOLUME_ID_HBO) &&
 			    (voldata.magic != HAMMER2_VOLUME_ID_ABO))
@@ -240,7 +245,7 @@ find_best_zone(int fd)
 }
 
 static int
-test_volume_header(int fd)
+test_volume_header(void)
 {
 	bool failed = false;
 	int i;
@@ -252,12 +257,14 @@ test_volume_header(int fd)
 
 		if (ScanBest && i != best_zone)
 			continue;
-		if (i * HAMMER2_ZONE_BYTES64 >= volume_size) {
+		if (i * HAMMER2_ZONE_BYTES64 >=
+		    hammer2_get_root_volume_size()) {
 			tfprintf(stderr, 0, "zone.%d exceeds volume size\n", i);
 			break;
 		}
-		init_root_blockref(fd, i, HAMMER2_BREF_TYPE_EMPTY, &broot);
-		ret = read(fd, &voldata, HAMMER2_PBUFSIZE);
+		init_root_blockref(i, HAMMER2_BREF_TYPE_EMPTY, &broot);
+		ret = read(hammer2_get_root_volume_fd(), &voldata,
+		    HAMMER2_PBUFSIZE);
 		if (ret == HAMMER2_PBUFSIZE) {
 			tprintf_zone(0, i, &broot);
 			if (verify_volume_header(&voldata) == -1)
@@ -275,7 +282,7 @@ test_volume_header(int fd)
 }
 
 static int
-test_blockref(int fd, uint8_t type)
+test_blockref(uint8_t type)
 {
 	struct blockref_tree droot;
 	bool failed = false;
@@ -289,23 +296,25 @@ test_blockref(int fd, uint8_t type)
 
 		if (ScanBest && i != best_zone)
 			continue;
-		if (i * HAMMER2_ZONE_BYTES64 >= volume_size) {
+		if (i * HAMMER2_ZONE_BYTES64 >=
+		    hammer2_get_root_volume_size()) {
 			tfprintf(stderr, 0, "zone.%d exceeds volume size\n", i);
 			break;
 		}
-		init_root_blockref(fd, i, type, &broot);
-		ret = read(fd, &voldata, HAMMER2_PBUFSIZE);
+		init_root_blockref(i, type, &broot);
+		ret = read(hammer2_get_root_volume_fd(), &voldata,
+		    HAMMER2_PBUFSIZE);
 		if (ret == HAMMER2_PBUFSIZE) {
 			blockref_stats_t bstats;
 			init_blockref_stats(&bstats, type);
 			delta_stats_t ds;
 			memset(&ds, 0, sizeof(ds));
 			tprintf_zone(0, i, &broot);
-			if (verify_blockref(fd, &voldata, &broot, false,
-			    &bstats, &droot, &ds, 0, 0) == -1)
+			if (verify_blockref(&voldata, &broot, false, &bstats,
+			    &droot, &ds, 0, 0) == -1)
 				failed = true;
 			print_blockref_stats(&bstats, true);
-			print_blockref_entry(fd, &bstats.root);
+			print_blockref_entry(&bstats.root);
 			cleanup_blockref_stats(&bstats);
 		} else if (ret == -1) {
 			perror("read");
@@ -323,7 +332,7 @@ test_blockref(int fd, uint8_t type)
 }
 
 static int
-test_pfs_blockref(int fd)
+test_pfs_blockref(void)
 {
 	struct blockref_tree droot;
 	uint8_t type = HAMMER2_BREF_TYPE_VOLUME;
@@ -338,12 +347,14 @@ test_pfs_blockref(int fd)
 
 		if (ScanBest && i != best_zone)
 			continue;
-		if (i * HAMMER2_ZONE_BYTES64 >= volume_size) {
+		if (i * HAMMER2_ZONE_BYTES64 >=
+		    hammer2_get_root_volume_size()) {
 			tfprintf(stderr, 0, "zone.%d exceeds volume size\n", i);
 			break;
 		}
-		init_root_blockref(fd, i, type, &broot);
-		ret = read(fd, &voldata, HAMMER2_PBUFSIZE);
+		init_root_blockref(i, type, &broot);
+		ret = read(hammer2_get_root_volume_fd(), &voldata,
+		    HAMMER2_PBUFSIZE);
 		if (ret == HAMMER2_PBUFSIZE) {
 			struct blockref_list blist;
 			struct blockref_msg *p;
@@ -351,8 +362,7 @@ test_pfs_blockref(int fd)
 
 			tprintf_zone(0, i, &broot);
 			TAILQ_INIT(&blist);
-			if (init_pfs_blockref(fd, &voldata, &broot, &blist) ==
-			    -1) {
+			if (init_pfs_blockref(&voldata, &broot, &blist) == -1) {
 				tfprintf(stderr, 1, "Failed to read PFS "
 				    "blockref\n");
 				failed = true;
@@ -390,11 +400,11 @@ test_pfs_blockref(int fd)
 				init_blockref_stats(&bstats, type);
 				delta_stats_t ds;
 				memset(&ds, 0, sizeof(ds));
-				if (verify_blockref(fd, &voldata, &p->bref,
-				    false, &bstats, &droot, &ds, 0, 0) == -1)
+				if (verify_blockref(&voldata, &p->bref, false,
+				    &bstats, &droot, &ds, 0, 0) == -1)
 					failed = true;
 				print_blockref_stats(&bstats, true);
-				print_blockref_entry(fd, &bstats.root);
+				print_blockref_entry(&bstats.root);
 				cleanup_blockref_stats(&bstats);
 			}
 			cleanup_pfs_blockref(&blist);
@@ -538,17 +548,17 @@ print_blockref_debug(FILE *fp, int depth, int index,
 }
 
 static void
-print_blockref_msg(int fd, const struct blockref_list *head)
+print_blockref_msg(const struct blockref_list *head)
 {
 	struct blockref_msg *m;
 
 	TAILQ_FOREACH(m, head, entry) {
 		hammer2_blockref_t *bref = &m->bref;
 		print_blockref(stderr, bref, m->msg);
-		if (fd != -1 && VerboseOpt > 0) {
+		if (VerboseOpt > 0) {
 			hammer2_media_data_t media;
 			size_t bytes;
-			if (!read_media(fd, bref, &media, &bytes))
+			if (!read_media(bref, &media, &bytes))
 				print_media(stderr, 2, bref, &media, bytes);
 			else
 				tfprintf(stderr, 2, "Failed to read media\n");
@@ -557,12 +567,12 @@ print_blockref_msg(int fd, const struct blockref_list *head)
 }
 
 static void
-print_blockref_entry(int fd, struct blockref_tree *root)
+print_blockref_entry(struct blockref_tree *root)
 {
 	struct blockref_entry *e;
 
 	RB_FOREACH(e, blockref_tree, root)
-		print_blockref_msg(fd, &e->head);
+		print_blockref_msg(&e->head);
 }
 
 static void
@@ -684,11 +694,12 @@ verify_volume_header(const hammer2_volume_data_t *voldata)
 }
 
 static int
-read_media(int fd, const hammer2_blockref_t *bref, hammer2_media_data_t *media,
+read_media(const hammer2_blockref_t *bref, hammer2_media_data_t *media,
     size_t *media_bytes)
 {
 	hammer2_off_t io_off, io_base;
 	size_t bytes, io_bytes, boff;
+	int fd;
 
 	bytes = (bref->data_off & HAMMER2_OFF_MASK_RADIX);
 	if (bytes)
@@ -709,7 +720,9 @@ read_media(int fd, const hammer2_blockref_t *bref, hammer2_media_data_t *media,
 
 	if (io_bytes > sizeof(*media))
 		return -1;
-	if (lseek(fd, io_base, SEEK_SET) == -1)
+	fd = hammer2_get_volume_fd(io_off);
+	if (lseek(fd, io_base - hammer2_get_volume_offset(io_base), SEEK_SET)
+	    == -1)
 		return -2;
 	if (read(fd, media, io_bytes) != (ssize_t)io_bytes)
 		return -2;
@@ -764,7 +777,7 @@ accumulate_delta_stats(delta_stats_t *dst, const delta_stats_t *src)
 }
 
 static int
-verify_blockref(int fd, const hammer2_volume_data_t *voldata,
+verify_blockref(const hammer2_volume_data_t *voldata,
     const hammer2_blockref_t *bref, bool norecurse, blockref_stats_t *bstats,
     struct blockref_tree *droot, delta_stats_t *dstats, int depth, int index)
 {
@@ -862,7 +875,7 @@ verify_blockref(int fd, const hammer2_volume_data_t *voldata,
 		break;
 	}
 
-	switch (read_media(fd, bref, &media, &bytes)) {
+	switch (read_media(bref, &media, &bytes)) {
 	case -1:
 		strlcpy(msg, "Bad I/O bytes", sizeof(msg));
 		add_blockref_entry(&bstats->root, bref, msg, strlen(msg) + 1);
@@ -985,8 +998,8 @@ verify_blockref(int fd, const hammer2_volume_data_t *voldata,
 	for (i = 0; norecurse == false && i < bcount; ++i) {
 		delta_stats_t ds;
 		memset(&ds, 0, sizeof(ds));
-		if (verify_blockref(fd, voldata, &bscan[i], failed, bstats,
-		    droot, &ds, depth + 1, i) == -1)
+		if (verify_blockref(voldata, &bscan[i], failed, bstats, droot,
+		    &ds, depth + 1, i) == -1)
 			return -1;
 		if (!failed)
 			accumulate_delta_stats(dstats, &ds);
@@ -1070,7 +1083,7 @@ __add_pfs_blockref(const hammer2_blockref_t *bref, struct blockref_list *blist,
 }
 
 static int
-init_pfs_blockref(int fd, const hammer2_volume_data_t *voldata,
+init_pfs_blockref(const hammer2_volume_data_t *voldata,
     const hammer2_blockref_t *bref, struct blockref_list *blist)
 {
 	hammer2_media_data_t media;
@@ -1079,7 +1092,7 @@ init_pfs_blockref(int fd, const hammer2_volume_data_t *voldata,
 	int i, bcount;
 	size_t bytes;
 
-	if (read_media(fd, bref, &media, &bytes))
+	if (read_media(bref, &media, &bytes))
 		return -1;
 	if (!bytes)
 		return 0;
@@ -1114,7 +1127,7 @@ init_pfs_blockref(int fd, const hammer2_volume_data_t *voldata,
 	}
 
 	for (i = 0; i < bcount; ++i)
-		if (init_pfs_blockref(fd, voldata, &bscan[i], blist) == -1)
+		if (init_pfs_blockref(voldata, &bscan[i], blist) == -1)
 			return -1;
 	return 0;
 }
@@ -1283,111 +1296,52 @@ print_media(FILE *fp, int tab, const hammer2_blockref_t *bref,
 		free(str);
 }
 
-static hammer2_off_t
-check_volume(int fd)
-{
-	struct partinfo pinfo;
-	hammer2_off_t size;
-
-	if (ioctl(fd, DIOCGPART, &pinfo) < 0) {
-		struct stat st;
-		if (fstat(fd, &st) < 0) {
-			perror("fstat");
-			return -1;
-		}
-		if (!S_ISREG(st.st_mode)) {
-			fprintf(stderr, "Unsupported file type\n");
-			return -1;
-		}
-		size = st.st_size;
-	} else {
-		if (pinfo.reserved_blocks) {
-			fprintf(stderr, "HAMMER2 cannot be placed in a "
-			    "partition which overlaps the disklabel or MBR\n");
-			return -1;
-		}
-		if (pinfo.media_blksize > HAMMER2_PBUFSIZE ||
-		    HAMMER2_PBUFSIZE % pinfo.media_blksize) {
-			fprintf(stderr, "A media sector size of %d is not "
-			    "supported\n", pinfo.media_blksize);
-			return -1;
-		}
-		size = pinfo.media_size;
-	}
-	return size;
-}
-
 int
 test_hammer2(const char *devpath)
 {
-	struct stat st;
 	bool failed = false;
-	int fd;
-
-	fd = open(devpath, O_RDONLY);
-	if (fd == -1) {
-		perror("open");
-		return -1;
-	}
 
-	if (fstat(fd, &st) == -1) {
-		perror("fstat");
-		failed = true;
-		goto end;
-	}
-	if (!S_ISCHR(st.st_mode) && !S_ISREG(st.st_mode)) {
-		fprintf(stderr, "Unsupported file type\n");
-		failed = true;
-		goto end;
-	}
-
-	volume_size = check_volume(fd);
-	if (volume_size == (hammer2_off_t)-1) {
-		fprintf(stderr, "Failed to find volume size\n");
-		failed = true;
-		goto end;
-	}
-	volume_size &= ~HAMMER2_VOLUME_ALIGNMASK64;
+	hammer2_init_volumes(devpath, 1);
 
-	best_zone = find_best_zone(fd);
+	best_zone = find_best_zone();
 	if (best_zone == -1)
 		fprintf(stderr, "Failed to find best zone\n");
 
 	if (PrintPFS) {
-		if (test_pfs_blockref(fd) == -1)
+		if (test_pfs_blockref() == -1)
 			failed = true;
 		goto end; /* print PFS info and exit */
 	}
 
 	printf("volume header\n");
-	if (test_volume_header(fd) == -1) {
+	if (test_volume_header() == -1) {
 		failed = true;
 		if (!ForceOpt)
 			goto end;
 	}
 
 	printf("freemap\n");
-	if (test_blockref(fd, HAMMER2_BREF_TYPE_FREEMAP) == -1) {
+	if (test_blockref(HAMMER2_BREF_TYPE_FREEMAP) == -1) {
 		failed = true;
 		if (!ForceOpt)
 			goto end;
 	}
 	printf("volume\n");
 	if (!ScanPFS) {
-		if (test_blockref(fd, HAMMER2_BREF_TYPE_VOLUME) == -1) {
+		if (test_blockref(HAMMER2_BREF_TYPE_VOLUME) == -1) {
 			failed = true;
 			if (!ForceOpt)
 				goto end;
 		}
 	} else {
-		if (test_pfs_blockref(fd) == -1) {
+		if (test_pfs_blockref() == -1) {
 			failed = true;
 			if (!ForceOpt)
 				goto end;
 		}
 	}
 end:
-	close(fd);
+	hammer2_cleanup_volumes();
 
 	return failed ? -1 : 0;
 }
diff --git a/sbin/hammer2/Makefile b/sbin/hammer2/Makefile
index 1ce25b6895..30b0b71dc5 100644
--- a/sbin/hammer2/Makefile
+++ b/sbin/hammer2/Makefile
@@ -1,10 +1,10 @@
 PROG=	hammer2
-SRCS=	main.c subs.c
+SRCS=	main.c subs.c ondisk.c
 SRCS+=	cmd_remote.c cmd_snapshot.c cmd_pfs.c
 SRCS+=	cmd_service.c cmd_leaf.c cmd_debug.c
 SRCS+=	cmd_rsa.c cmd_stat.c cmd_setcomp.c cmd_setcheck.c
 SRCS+=	cmd_bulkfree.c cmd_cleanup.c cmd_info.c cmd_destroy.c
-SRCS+=	cmd_emergency.c cmd_growfs.c
+SRCS+=	cmd_emergency.c cmd_growfs.c cmd_volume.c
 SRCS+=	print_inode.c
 MAN=	hammer2.8
 
diff --git a/sbin/hammer2/cmd_debug.c b/sbin/hammer2/cmd_debug.c
index 5b80ece840..527b92a9b7 100644
--- a/sbin/hammer2/cmd_debug.c
+++ b/sbin/hammer2/cmd_debug.c
@@ -379,8 +379,8 @@ cmd_debugspan(const char *hostname)
  *				    SHOW				*
  ************************************************************************/
 
-static void show_volhdr(hammer2_volume_data_t *voldata, int fd, int bi);
-static void show_bref(hammer2_volume_data_t *voldata, int fd, int tab,
+static void show_volhdr(hammer2_volume_data_t *voldata, int bi);
+static void show_bref(hammer2_volume_data_t *voldata, int tab,
 			int bi, hammer2_blockref_t *bref, int norecurse);
 static void tabprintf(int tab, const char *ctl, ...);
 
@@ -389,6 +389,23 @@ static hammer2_off_t TotalAccum64[4];
 static hammer2_off_t TotalUnavail;
 static hammer2_off_t TotalFreemap;
 
+static
+hammer2_off_t
+get_next_volume(hammer2_volume_data_t *voldata, hammer2_off_t volu_loff)
+{
+	hammer2_off_t ret = -1;
+	int i;
+
+	for (i = 0; i < HAMMER2_MAX_VOLUMES; ++i) {
+		hammer2_off_t tmp = voldata->volu_loff[i];
+		if (tmp > volu_loff) {
+			ret = tmp;
+			break;
+		}
+	}
+	return ret;
+}
+
 int
 cmd_show(const char *devpath, int which)
 {
@@ -396,6 +413,7 @@ cmd_show(const char *devpath, int which)
 	hammer2_blockref_t best;
 	hammer2_media_data_t media;
 	hammer2_media_data_t best_media;
+	hammer2_off_t off, volu_loff, next_volu_loff = 0;
 	int fd;
 	int i;
 	int best_i;
@@ -418,12 +436,11 @@ cmd_show(const char *devpath, int which)
 			show_depth = -1;
 	}
 
-	fd = open(devpath, O_RDONLY);
-	if (fd < 0) {
-		perror("open");
-		return 1;
-	}
-
+	hammer2_init_volumes(devpath, 1);
+next_volume:
+	volu_loff = next_volu_loff;
+	next_volu_loff = -1;
+	printf("%s\n", hammer2_get_volume_path(volu_loff));
 	/*
 	 * Show the tree using the best volume header.
 	 * -vvv will show the tree for all four volume headers.
@@ -434,7 +451,9 @@ cmd_show(const char *devpath, int which)
 	for (i = 0; i < HAMMER2_NUM_VOLHDRS; ++i) {
 		bzero(&broot, sizeof(broot));
 		broot.data_off = (i * HAMMER2_ZONE_BYTES64) | HAMMER2_PBUFRADIX;
-		lseek(fd, broot.data_off & ~HAMMER2_OFF_MASK_RADIX, SEEK_SET);
+		off = broot.data_off & ~HAMMER2_OFF_MASK_RADIX;
+		fd = hammer2_get_volume_fd(volu_loff);
+		lseek(fd, off, SEEK_SET);
 		if (read(fd, &media, HAMMER2_PBUFSIZE) ==
 		    (ssize_t)HAMMER2_PBUFSIZE) {
 			broot.mirror_tid = media.voldata.mirror_tid;
@@ -450,16 +469,18 @@ cmd_show(const char *devpath, int which)
 				switch(which) {
 				case 0:
 					broot.type = HAMMER2_BREF_TYPE_VOLUME;
-					show_bref(&media.voldata, fd, 0,
-						  i, &broot, 0);
+					show_bref(&media.voldata, 0, i, &broot,
+						  0);
 					break;
 				case 1:
 					broot.type = HAMMER2_BREF_TYPE_FREEMAP;
-					show_bref(&media.voldata, fd, 0,
-						  i, &broot, 0);
+					show_bref(&media.voldata, 0, i, &broot,
+						  0);
 					break;
 				default:
-					show_volhdr(&media.voldata, fd, i);
+					show_volhdr(&media.voldata, i);
+					if (i == 0)
+						next_volu_loff = get_next_volume(&media.voldata, volu_loff);
 					break;
 				}
 				if (i != HAMMER2_NUM_VOLHDRS - 1)
@@ -467,22 +488,31 @@ cmd_show(const char *devpath, int which)
 			}
 		}
 	}
+	if (next_volu_loff != (hammer2_off_t)-1) {
+		printf("---------------------------------------------\n");
+		goto next_volume;
+	}
+
 	if (VerboseOpt < 3) {
 		switch(which) {
 		case 0:
 			best.type = HAMMER2_BREF_TYPE_VOLUME;
-			show_bref(&best_media.voldata, fd, 0, best_i, &best, 0);
+			show_bref(&best_media.voldata, 0, best_i, &best, 0);
 			break;
 		case 1:
 			best.type = HAMMER2_BREF_TYPE_FREEMAP;
-			show_bref(&best_media.voldata, fd, 0, best_i, &best, 0);
+			show_bref(&best_media.voldata, 0, best_i, &best, 0);
 			break;
 		default:
-			show_volhdr(&best_media.voldata, fd, best_i);
+			show_volhdr(&best_media.voldata, best_i);
+			next_volu_loff = get_next_volume(&best_media.voldata, volu_loff);
+			if (next_volu_loff != (hammer2_off_t)-1) {
+				printf("---------------------------------------------\n");
+				goto next_volume;
+			}
 			break;
 		}
 	}
-	close(fd);
 
 	if (which == 1 && VerboseOpt < 3) {
 		printf("Total unallocated storage:   %6.3fGiB (%6.3fGiB in 64KB chunks)\n",
@@ -499,17 +529,19 @@ cmd_show(const char *devpath, int which)
 		printf("Total freemap storage:       %6.3fGiB\n",
 		       (double)TotalFreemap / GIG);
 	}
+	hammer2_cleanup_volumes();
 
 	return 0;
 }
 
 static void
-show_volhdr(hammer2_volume_data_t *voldata, int fd, int bi)
+show_volhdr(hammer2_volume_data_t *voldata, int bi)
 {
 	uint32_t status;
 	uint32_t i;
 	char *str;
 	char *name;
+	char *buf;
 
 	printf("\nVolume header %d {\n", bi);
 	printf("    magic          0x%016jx\n", (intmax_t)voldata->magic);
@@ -531,6 +563,8 @@ show_volhdr(hammer2_volume_data_t *voldata, int fd, int bi)
 	printf("    copyid         %d\n", voldata->copyid);
 	printf("    freemap_vers   %d\n", voldata->freemap_version);
 	printf("    peer_type      %d\n", voldata->peer_type);
+	printf("    volu_id        %d\n", voldata->volu_id);
+	printf("    nvolumes       %d\n", voldata->nvolumes);
 
 	str = NULL;
 	hammer2_uuid_to_str(&voldata->fsid, &str);
@@ -567,6 +601,7 @@ show_volhdr(hammer2_volume_data_t *voldata, int fd, int bi)
 		printf("    reserved00A0/%u 0x%016jx\n",
 		       i, voldata->reserved00A0[0]);
 	}
+	printf("    total_size     0x%016jx\n", voldata->total_size);
 
 	printf("    copyexists    ");
 	for (i = 0; i < nitems(voldata->copyexists); ++i)
@@ -641,29 +676,40 @@ show_volhdr(hammer2_volume_data_t *voldata, int fd, int bi)
 	printf("\n");
 	printf("    sroot_blockset {\n");
 	for (i = 0; i < HAMMER2_SET_COUNT; ++i) {
-		show_bref(voldata, fd, 16, i,
+		show_bref(voldata, 16, i,
 			  &voldata->sroot_blockset.blockref[i], 2);
 	}
 	printf("    }\n");
 
 	printf("    freemap_blockset {\n");
 	for (i = 0; i < HAMMER2_SET_COUNT; ++i) {
-		show_bref(voldata, fd, 16, i,
+		show_bref(voldata, 16, i,
 			  &voldata->freemap_blockset.blockref[i], 2);
 	}
 	printf("    }\n");
 
+	buf = calloc(1, sizeof(voldata->volu_loff));
+	if (bcmp(buf, voldata->volu_loff, sizeof(voldata->volu_loff))) {
+		printf("\n");
+		for (i = 0; i < HAMMER2_MAX_VOLUMES; ++i) {
+			hammer2_off_t loff = voldata->volu_loff[i];
+			if (loff != (hammer2_off_t)-1)
+				printf("    volu_loff[%d]   0x%016jx\n", i, loff);
+		}
+	}
+	free(buf);
+
 	printf("}\n");
 }
 
 static void
-show_bref(hammer2_volume_data_t *voldata, int fd, int tab,
-	  int bi, hammer2_blockref_t *bref, int norecurse)
+show_bref(hammer2_volume_data_t *voldata, int tab, int bi,
+	  hammer2_blockref_t *bref, int norecurse)
 {
 	hammer2_media_data_t media;
 	hammer2_blockref_t *bscan;
 	hammer2_off_t tmp;
-	int i, bcount, namelen, failed, obrace;
+	int i, bcount, namelen, failed, obrace, fd;
 	int type_pad;
 	size_t bytes;
 	const char *type_str;
@@ -703,7 +749,9 @@ show_bref(hammer2_volume_data_t *voldata, int fd, int tab,
 			return;
 		}
 		if (bref->type != HAMMER2_BREF_TYPE_DATA || VerboseOpt >= 1) {
-			lseek(fd, io_base, SEEK_SET);
+			fd = hammer2_get_volume_fd(io_off);
+			lseek(fd, io_base - hammer2_get_volume_offset(io_base),
+			      SEEK_SET);
 			if (read(fd, &media, io_bytes) != (ssize_t)io_bytes) {
 				printf("(media read failed)\n");
 				return;
@@ -755,9 +803,10 @@ show_bref(hammer2_volume_data_t *voldata, int fd, int tab,
 	if (QuietOpt > 0) {
 		tabprintf(tab,
 			  "%s.%-3d %016jx %016jx/%-2d "
-			  "mir=%016jx mod=%016jx leafcnt=%d ",
+			  "vol=%d mir=%016jx mod=%016jx leafcnt=%d ",
 			  type_str, bi, (intmax_t)bref->data_off,
 			  (intmax_t)bref->key, (intmax_t)bref->keybits,
+			  hammer2_get_volume_id(bref->data_off),
 			  (intmax_t)bref->mirror_tid,
 			  (intmax_t)bref->modify_tid,
 			  bref->leaf_count);
@@ -770,7 +819,8 @@ show_bref(hammer2_volume_data_t *voldata, int fd, int tab,
 			printf("\n");
 			tabprintf(tab + 13, "");
 		}
-		printf("mir=%016jx mod=%016jx lfcnt=%d ",
+		printf("vol=%d mir=%016jx mod=%016jx lfcnt=%d ",
+		       hammer2_get_volume_id(bref->data_off),
 		       (intmax_t)bref->mirror_tid, (intmax_t)bref->modify_tid,
 		       bref->leaf_count);
 		if (/*norecurse > 1 && */ (bcount || bref->flags ||
@@ -1079,7 +1129,7 @@ show_bref(hammer2_volume_data_t *voldata, int fd, int tab,
 			hammer2_off_t data_off = bref->key +
 				i * HAMMER2_FREEMAP_LEVEL0_SIZE;
 			if (data_off >= voldata->aux_end &&
-			    data_off < voldata->volu_size) {
+			    data_off < hammer2_get_total_size()) {
 				int j;
 				for (j = 0; j < 4; ++j)
 					CountBlocks(&media.bmdata[i], j,
@@ -1103,7 +1153,7 @@ show_bref(hammer2_volume_data_t *voldata, int fd, int tab,
 	if (show_depth == -1 || ((tab - init_tab) / show_tab) < show_depth) {
 		for (i = 0; norecurse == 0 && i < bcount; ++i) {
 			if (bscan[i].type != HAMMER2_BREF_TYPE_EMPTY) {
-				show_bref(voldata, fd, tab, i, &bscan[i],
+				show_bref(voldata, tab, i, &bscan[i],
 				    failed);
 			}
 		}
diff --git a/sbin/hammer2/cmd_info.c b/sbin/hammer2/cmd_info.c
index 6bb9d48a35..92b1c4ab1a 100644
--- a/sbin/hammer2/cmd_info.c
+++ b/sbin/hammer2/cmd_info.c
@@ -308,6 +308,7 @@ h2disk_check(const char *devpath, cmd_callback callback1)
 	hammer2_blockref_t broot;
 	hammer2_blockref_t best;
 	hammer2_media_data_t media;
+	hammer2_volume_data_t *voldata;
 	struct partinfo partinfo;
 	int fd;
 	int i;
@@ -352,14 +353,19 @@ h2disk_check(const char *devpath, cmd_callback callback1)
 		broot.data_off = (i * HAMMER2_ZONE_BYTES64) |
 				 HAMMER2_PBUFRADIX;
 		lseek(fd, broot.data_off & ~HAMMER2_OFF_MASK_RADIX, SEEK_SET);
-		if (read(fd, &media, HAMMER2_PBUFSIZE) ==
-		    (ssize_t)HAMMER2_PBUFSIZE &&
-		    media.voldata.magic == HAMMER2_VOLUME_ID_HBO) {
-			broot.mirror_tid = media.voldata.mirror_tid;
-			if (best_i < 0 || best.mirror_tid < broot.mirror_tid) {
-				best_i = i;
-				best = broot;
-			}
+		if (read(fd, &media, HAMMER2_PBUFSIZE) !=
+		    (ssize_t)HAMMER2_PBUFSIZE)
+			continue;
+		voldata = &media.voldata;
+		if (voldata->magic != HAMMER2_VOLUME_ID_HBO)
+			continue;
+		/* XXX multiple volumes currently unsupported */
+		if (voldata->nvolumes > 1)
+			break;
+		broot.mirror_tid = voldata->mirror_tid;
+		if (best_i < 0 || best.mirror_tid < broot.mirror_tid) {
+			best_i = i;
+			best = broot;
 		}
 	}
 	if (best_i >= 0)
diff --git a/sbin/hammer2/cmd_volume.c b/sbin/hammer2/cmd_volume.c
new file mode 100644
index 0000000000..cf47c199e4
--- /dev/null
+++ b/sbin/hammer2/cmd_volume.c
@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) 2020 Tomohiro Kusumi <tkusumi@netbsd.org>
+ * Copyright (c) 2020 The DragonFly Project
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The DragonFly Project
+ * by Matthew Dillon <dillon@dragonflybsd.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ * 3. Neither the name of The DragonFly Project nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific, prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "hammer2.h"
+
+int
+cmd_volume_list(int ac, char **av)
+{
+	hammer2_ioc_volume_list_t vollist;
+	hammer2_ioc_volume_t *entry;
+	int fd, i, j, n, w, all = 0, ecode = 0;
+
+	if (ac == 1 && av[0] == NULL) {
+		av = get_hammer2_mounts(&ac);
+		all = 1;
+	}
+	vollist.volumes = calloc(HAMMER2_MAX_VOLUMES, sizeof(*vollist.volumes));
+
+	for (i = 0; i < ac; ++i) {
+		if (i)
+			printf("\n");
+		if (ac > 1 || all)
+			printf("%s\n", av[i]);
+		if ((fd = hammer2_ioctl_handle(av[i])) < 0) {
+			ecode = 1;
+			goto failed;
+		}
+
+		vollist.nvolumes = HAMMER2_MAX_VOLUMES;
+		if (ioctl(fd, HAMMER2IOC_VOLUME_LIST, &vollist) < 0) {
+			perror("ioctl");
+			close(fd);
+			ecode = 1;
+			goto failed;
+		}
+
+		w = 0;
+		for (j = 0; j < vollist.nvolumes; ++j) {
+			entry = &vollist.volumes[j];
+			n = (int)strlen(entry->path);
+			if (n > w)
+				w = n;
+		}
+
+		if (QuietOpt > 0) {
+			for (j = 0; j < vollist.nvolumes; ++j) {
+				entry = &vollist.volumes[j];
+				printf("%s\n", entry->path);
+			}
+		} else {
+			printf("version %d\n", vollist.version);
+			printf("@%s\n", vollist.pfs_name);
+			for (j = 0; j < vollist.nvolumes; ++j) {
+				entry = &vollist.volumes[j];
+				printf("volume%-2d %-*.*s %s",
+				       entry->id, w, w, entry->path,
+				       sizetostr(entry->size));
+				if (VerboseOpt > 0)
+					printf(" 0x%016jx 0x%016jx",
+					       (intmax_t)entry->offset,
+					       (intmax_t)entry->size);
+				printf("\n");
+			}
+		}
+		close(fd);
+	}
+failed:
+	free(vollist.volumes);
+	if (all)
+		put_hammer2_mounts(ac, av);
+
+	return (ecode);
+}
diff --git a/sbin/hammer2/hammer2.8 b/sbin/hammer2/hammer2.8
index 216c689a74..aba9c24b0a 100644
--- a/sbin/hammer2/hammer2.8
+++ b/sbin/hammer2/hammer2.8
@@ -30,7 +30,7 @@
 .\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 .\" SUCH DAMAGE.
 .\"
-.Dd September 27, 2020
+.Dd November 29, 2020
 .Dt HAMMER2 8
 .Os
 .Sh NAME
@@ -416,6 +416,15 @@ No mount is required.
 Dump the volume header for the HAMMER2 filesystem by scanning a
 block device directly.
 No mount is required.
+.\" ==== volume-list ====
+.It Cm volume-list Op path...
+List all volumes associated with all mounted hammer2 storage devices.
+The list may be restricted to a particular filesystem using
+.Fl s Ar mount .
+.Pp
+Note that hammer2 volumes associated with storage devices which have not been
+mounted in any fashion will not be listed.  At least one hammer2 label must
+be mounted for the volumes on that device to be visible.
 .\" ==== setcomp ====
 .It Cm setcomp Ar mode[:level] Ar path...
 Set the compression mode as specified for any newly created elements at or
diff --git a/sbin/hammer2/hammer2.h b/sbin/hammer2/hammer2.h
index 498bd890ee..a885cee7e1 100644
--- a/sbin/hammer2/hammer2.h
+++ b/sbin/hammer2/hammer2.h
@@ -155,6 +155,7 @@ int cmd_emergency_mode(const char *sel_path, int enable,
 			int ac, const char **av);
 int cmd_growfs(const char *sel_path, int ac, const char **av);
 int cmd_show(const char *devpath, int which);
+int cmd_volume_list(int ac, char **av);
 int cmd_rsainit(const char *dir_path);
 int cmd_rsaenc(const char **keys, int nkeys);
 int cmd_rsadec(const char **keys, int nkeys);
diff --git a/sbin/hammer2/hammer2_subs.h b/sbin/hammer2/hammer2_subs.h
index 81cc432422..f2f20116d9 100644
--- a/sbin/hammer2/hammer2_subs.h
+++ b/sbin/hammer2/hammer2_subs.h
@@ -40,6 +40,24 @@
 
 #include <vfs/hammer2/hammer2_disk.h>
 
+typedef struct hammer2_volume {
+	int fd;
+	int id;
+	char *path;
+	hammer2_off_t offset;
+	hammer2_off_t size;
+} hammer2_volume_t;
+
+typedef struct hammer2_ondisk {
+	int version;
+	int nvolumes;
+	hammer2_volume_t volumes[HAMMER2_MAX_VOLUMES];
+	hammer2_off_t total_size;
+	hammer2_off_t free_size;
+	uuid_t fsid;
+	uuid_t fstype;
+} hammer2_ondisk_t;
+
 /*
  * Misc functions
  */
@@ -52,6 +70,7 @@ const char *hammer2_pfssubtype_to_str(uint8_t subtype);
 const char *hammer2_breftype_to_str(uint8_t type);
 const char *sizetostr(hammer2_off_t size);
 const char *counttostr(hammer2_off_t size);
+hammer2_off_t check_volume(int fd);
 hammer2_key_t dirhash(const unsigned char *name, size_t len);
 
 #define hammer2_icrc32(buf, size)	iscsi_crc32((buf), (size))
@@ -62,4 +81,28 @@ uint32_t iscsi_crc32_ext(const void *buf, size_t size, uint32_t ocrc);
 char **get_hammer2_mounts(int *acp);
 void put_hammer2_mounts(int ac, char **av);
 
+void hammer2_init_ondisk(hammer2_ondisk_t *fsp);
+void hammer2_install_volume(hammer2_volume_t *vol, int fd, int id,
+	const char *path, hammer2_off_t offset, hammer2_off_t size);
+void hammer2_uninstall_volume(hammer2_volume_t *vol);
+void hammer2_verify_volumes(hammer2_ondisk_t *fsp,
+	const hammer2_volume_data_t *rootvoldata);
+void hammer2_print_volumes(const hammer2_ondisk_t *fsp);
+void hammer2_init_volumes(const char *blkdevs, int rdonly);
+void hammer2_cleanup_volumes(void);
+
+int hammer2_get_volume_fd(hammer2_off_t offset);
+int hammer2_get_root_volume_fd(void);
+int hammer2_get_volume_id(hammer2_off_t offset);
+int hammer2_get_root_volume_id(void);
+const char *hammer2_get_volume_path(hammer2_off_t offset);
+const char *hammer2_get_root_volume_path(void);
+hammer2_off_t hammer2_get_volume_offset(hammer2_off_t offset);
+hammer2_off_t hammer2_get_root_volume_offset(void);
+hammer2_off_t hammer2_get_volume_size(hammer2_off_t offset);
+hammer2_off_t hammer2_get_root_volume_size(void);
+
+hammer2_off_t hammer2_get_total_size(void);
+hammer2_volume_data_t* hammer2_read_root_volume_header(void);
+
 #endif /* !HAMMER2_HAMMER2_SUBS_H_ */
diff --git a/sbin/hammer2/main.c b/sbin/hammer2/main.c
index fbd62194e5..fd072ee2bd 100644
--- a/sbin/hammer2/main.c
+++ b/sbin/hammer2/main.c
@@ -437,6 +437,16 @@ main(int ac, char **av)
 		} else {
 			cmd_show(av[1], 2);
 		}
+	} else if (strcmp(av[0], "volume-list") == 0) {
+		/*
+		 * List all volumes
+		 */
+		if (ac >= 2) {
+			ecode = cmd_volume_list(ac - 1,
+					     (char **)(void *)&av[1]);
+		} else {
+			ecode = cmd_volume_list(1, &sel_path);
+		}
 	} else if (strcmp(av[0], "setcomp") == 0) {
 		if (ac < 3) {
 			/*
@@ -593,6 +603,8 @@ usage(int code)
 			"Raw hammer2 media dump for freemap\n"
 		"    volhdr <devpath>                  "
 			"Raw hammer2 media dump for the volume header(s)\n"
+		"    volume-list [<path>...]           "
+			"List volumes\n"
 		"    setcomp <comp[:level]> <path>...  "
 			"Set comp algo {none, autozero, lz4, zlib} & level\n"
 		"    setcheck <check> <path>...        "
diff --git a/sbin/hammer2/ondisk.c b/sbin/hammer2/ondisk.c
new file mode 100644
index 0000000000..71b02a78d5
--- /dev/null
+++ b/sbin/hammer2/ondisk.c
@@ -0,0 +1,654 @@
+/*
+ * Copyright (c) 2020 Tomohiro Kusumi <tkusumi@netbsd.org>
+ * Copyright (c) 2020 The DragonFly Project
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The DragonFly Project
+ * by Matthew Dillon <dillon@dragonflybsd.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ * 3. Neither the name of The DragonFly Project nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific, prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fstab.h>
+#include <assert.h>
+#include <err.h>
+
+#include <vfs/hammer2/hammer2_disk.h>
+
+#include "hammer2_subs.h"
+
+static hammer2_ondisk_t fso;
+static int hammer2_volumes_initialized;
+
+static void
+hammer2_init_volume(hammer2_volume_t *vol)
+{
+	vol->fd = -1;
+	vol->id = -1;
+	vol->offset = (hammer2_off_t)-1;
+	vol->size = (hammer2_off_t)-1;
+}
+
+void
+hammer2_init_ondisk(hammer2_ondisk_t *fsp)
+{
+	int i;
+
+	bzero(fsp, sizeof(*fsp));
+	fsp->version = -1;
+	for (i = 0; i < HAMMER2_MAX_VOLUMES; ++i)
+		hammer2_init_volume(&fsp->volumes[i]);
+}
+
+void
+hammer2_install_volume(hammer2_volume_t *vol, int fd, int id, const char *path,
+		       hammer2_off_t offset, hammer2_off_t size)
+{
+	bzero(vol, sizeof(*vol));
+	vol->fd = fd;
+	vol->id = id;
+	vol->path = strdup(path);
+	vol->offset = offset;
+	vol->size = size;
+}
+
+void
+hammer2_uninstall_volume(hammer2_volume_t *vol)
+{
+	fsync(vol->fd);
+	close(vol->fd);
+	free(vol->path);
+	hammer2_init_volume(vol);
+}
+
+static int
+hammer2_read_volume_header(int fd, const char *path,
+			   hammer2_volume_data_t *voldata)
+{
+	hammer2_volume_data_t vd;
+	hammer2_tid_t mirror_tid = -1;
+	hammer2_off_t size = check_volume(fd);
+	hammer2_crc32_t crc0, crc1;
+	const char *p;
+	int i, zone = -1;
+	ssize_t ret;
+
+	for (i = 0; i < HAMMER2_NUM_VOLHDRS; ++i) {
+		if (i * HAMMER2_ZONE_BYTES64 >= size)
+			break;
+		if (lseek(fd, i * HAMMER2_ZONE_BYTES64, SEEK_SET) == -1)
+			break;
+		ret = read(fd, &vd, HAMMER2_PBUFSIZE);
+		if (ret == -1)
+			err(1, "read");
+		else if (ret != HAMMER2_PBUFSIZE)
+			errx(1, "%s #%d: failed to read", path, i);
+
+		p = (const char*)&vd;
+		/* verify volume header magic */
+		if ((vd.magic != HAMMER2_VOLUME_ID_HBO) &&
+		    (vd.magic != HAMMER2_VOLUME_ID_ABO))
+			errx(1, "%s #%d: bad magic", path, i);
+
+		if (vd.magic == HAMMER2_VOLUME_ID_ABO) {
+			/* XXX: Reversed-endianness filesystem */
+			errx(1, "%s #%d: reverse-endian filesystem detected",
+			     path, i);
+		}
+
+		/* verify volume header CRC's */
+		crc0 = vd.icrc_sects[HAMMER2_VOL_ICRC_SECT0];
+		crc1 = hammer2_icrc32(p + HAMMER2_VOLUME_ICRC0_OFF,
+				      HAMMER2_VOLUME_ICRC0_SIZE);
+		if (crc0 != crc1)
+			errx(1, "%s #%d: volume header crc mismatch sect0 %08x/%08x",
+			     path, i, crc0, crc1);
+
+		crc0 = vd.icrc_sects[HAMMER2_VOL_ICRC_SECT1];
+		crc1 = hammer2_icrc32(p + HAMMER2_VOLUME_ICRC1_OFF,
+				      HAMMER2_VOLUME_ICRC1_SIZE);
+		if (crc0 != crc1)
+			errx(1, "%s #%d: volume header crc mismatch sect1 %08x/%08x",
+			     path, i, crc0, crc1);
+
+		crc0 = vd.icrc_volheader;
+		crc1 = hammer2_icrc32(p + HAMMER2_VOLUME_ICRCVH_OFF,
+				      HAMMER2_VOLUME_ICRCVH_SIZE);
+		if (crc0 != crc1)
+			errx(1, "%s #%d: volume header crc mismatch vh %08x/%08x",
+			     path, i, crc0, crc1);
+
+		if (zone == -1 || mirror_tid < vd.mirror_tid) {
+			bcopy(&vd, voldata, sizeof(vd));
+			mirror_tid = vd.mirror_tid;
+			zone = i;
+		}
+	}
+
+	if (zone == -1)
+		errx(1, "%s has no valid volume headers", path);
+	return(zone);
+}
+
+static void
+hammer2_add_volume(const char *path, int rdonly)
+{
+	hammer2_volume_data_t voldata;
+	hammer2_volume_t *vol;
+	struct stat st;
+	int fd, i;
+
+	fd = open(path, rdonly ? O_RDONLY : O_RDWR);
+	if (fd == -1)
+		err(1, "open");
+
+	if (fstat(fd, &st) == -1)
+		err(1, "fstat");
+	if (!S_ISCHR(st.st_mode) && !S_ISREG(st.st_mode))
+		errx(1, "Unsupported file type");
+
+	if (hammer2_read_volume_header(fd, path, &voldata) >= 0) {
+		i = voldata.volu_id;
+		if (i < 0 || i >= HAMMER2_MAX_VOLUMES)
+			errx(1, "%s has bad volume id %d", path, i);
+		vol = &fso.volumes[i];
+		if (vol->id != -1)
+			errx(1, "%s already initialized", path);
+		/* all headers must have the same version, nvolumes and uuid */
+		if (!fso.nvolumes) {
+			fso.version = voldata.version;
+			fso.nvolumes = voldata.nvolumes;
+			fso.fsid = voldata.fsid;
+			fso.fstype = voldata.fstype;
+		} else {
+			if (fso.version != (int)voldata.version)
+				errx(1, "Volume version mismatch %d vs %d",
+				     fso.version, (int)voldata.version);
+			if (fso.nvolumes != voldata.nvolumes)
+				errx(1, "Volume count mismatch %d vs %d",
+				     fso.nvolumes, voldata.nvolumes);
+			if (!uuid_equal(&fso.fsid, &voldata.fsid, NULL))
+				errx(1, "fsid uuid mismatch");
+			if (!uuid_equal(&fso.fstype, &voldata.fstype, NULL))
+				errx(1, "fstype uuid mismatch");
+		}
+		/* all per-volume tests passed */
+		hammer2_install_volume(vol, fd, i, path,
+				       voldata.volu_loff[i], voldata.volu_size);
+		fso.total_size += vol->size;
+	} else {
+		errx(1, "Failed to read volume header");
+	}
+}
+
+static void
+hammer2_verify_volumes_common(const hammer2_ondisk_t *fsp)
+{
+	const hammer2_volume_t *vol;
+	hammer2_off_t size;
+	struct stat *st;
+	const char *path;
+	int i, j, nvolumes = 0;
+
+	if (fsp->version == -1)
+		errx(1, "Bad volume version %d", fsp->version);
+
+	/* check initialized volume count */
+	for (i = 0; i < HAMMER2_MAX_VOLUMES; ++i) {
+		vol = &fsp->volumes[i];
+		if (vol->id != -1)
+			nvolumes++;
+	}
+
+	/* fsp->nvolumes hasn't been verified yet, use nvolumes */
+	st = calloc(nvolumes, sizeof(*st));
+
+	for (i = 0; i < HAMMER2_MAX_VOLUMES; ++i) {
+		vol = &fsp->volumes[i];
+		if (vol->id == -1)
+			continue;
+		path = vol->path;
+		/* check volumes are unique */
+		if (stat(path, &st[i]) != 0)
+			errx(1, "Failed to stat %s", path);
+		if (fstat(vol->fd, &st[i]) != 0)
+			errx(1, "Failed to fstat %d", vol->fd);
+		for (j = 0; j < i; ++j) {
+			if ((st[i].st_ino == st[j].st_ino) &&
+			    (st[i].st_dev == st[j].st_dev))
+				errx(1, "%s specified more than once", path);
+		}
+		/* check volume fields are initialized */
+		if (vol->fd == -1)
+			errx(1, "%s has bad fd %d", path, vol->fd);
+		if (vol->offset == (hammer2_off_t)-1)
+			errx(1, "%s has bad offset 0x%016jx", path,
+			     (intmax_t)vol->offset);
+		if (vol->size == (hammer2_off_t)-1)
+			errx(1, "%s has bad size 0x%016jx", path,
+			     (intmax_t)vol->size);
+		/* check volume size vs block device size */
+		size = check_volume(vol->fd);
+		if (vol->size > size)
+			errx(1, "%s's size 0x%016jx exceeds device size 0x%016jx",
+			     path, (intmax_t)vol->size, size);
+	}
+	free(st);
+}
+
+static void
+hammer2_verify_volumes_1(hammer2_ondisk_t *fsp,
+			 const hammer2_volume_data_t *rootvoldata)
+{
+	const hammer2_volume_t *vol;
+	hammer2_off_t off;
+	const char *path;
+	int i, nvolumes = 0;
+
+	/* check initialized volume count */
+	for (i = 0; i < HAMMER2_MAX_VOLUMES; ++i) {
+		vol = &fsp->volumes[i];
+		if (vol->id != -1)
+			nvolumes++;
+	}
+	if (nvolumes != 1)
+		errx(1, "Only 1 volume supported");
+	if (fsp->nvolumes)
+		errx(1, "Volume count %d must be 0", fsp->nvolumes);
+	fsp->nvolumes = nvolumes; /* adjust with actual count */
+
+	/* check volume header */
+	if (rootvoldata) {
+		if (rootvoldata->volu_id)
+			errx(1, "Volume id %d must be 0", rootvoldata->volu_id);
+		if (rootvoldata->nvolumes)
+			errx(1, "Volume count %d must be 0",
+			     rootvoldata->nvolumes);
+		if (rootvoldata->total_size)
+			errx(1, "Total size 0x%016jx must be 0",
+			     (intmax_t)rootvoldata->total_size);
+		for (i = 0; i < HAMMER2_MAX_VOLUMES; ++i) {
+			off = rootvoldata->volu_loff[i];
+			if (off)
+				errx(1, "Volume offset[%d] 0x%016jx must be 0",
+				     i, (intmax_t)off);
+		}
+	}
+
+	/* check volume */
+	vol = &fsp->volumes[0];
+	path = vol->path;
+	if (vol->id)
+		errx(1, "%s has non zero id %d", path, vol->id);
+	if (vol->offset)
+		errx(1, "%s has non zero offset 0x%016jx", path,
+		     (intmax_t)vol->offset);
+	if (vol->size & HAMMER2_VOLUME_ALIGNMASK64)
+		errx(1, "%s's size is not 0x%016jx aligned", path,
+		     (intmax_t)HAMMER2_VOLUME_ALIGN);
+}
+
+static void
+hammer2_verify_volumes_2(const hammer2_ondisk_t *fsp,
+			 const hammer2_volume_data_t *rootvoldata)
+{
+	const hammer2_volume_t *vol;
+	hammer2_off_t off;
+	const char *path;
+	int i, nvolumes = 0;
+
+	/* check initialized volume count */
+	for (i = 0; i < HAMMER2_MAX_VOLUMES; ++i) {
+		vol = &fsp->volumes[i];
+		if (vol->id != -1)
+			nvolumes++;
+	}
+	if (fsp->nvolumes != nvolumes)
+		errx(1, "Volume count mismatch %d vs %d",
+		     fsp->nvolumes, nvolumes);
+
+	/* check volume header */
+	if (rootvoldata) {
+		if (rootvoldata->volu_id != HAMMER2_ROOT_VOLUME)
+			errx(1, "Volume id %d must be %d",
+			     rootvoldata->volu_id, HAMMER2_ROOT_VOLUME);
+		if (rootvoldata->nvolumes != fso.nvolumes)
+			errx(1, "Volume header requires %d devices, %d specified",
+			     rootvoldata->nvolumes, fso.nvolumes);
+		if (rootvoldata->total_size != fso.total_size)
+			errx(1, "Total size 0x%016jx does not equal sum of "
+			     "volumes 0x%016jx",
+			     rootvoldata->total_size, fso.total_size);
+		for (i = 0; i < nvolumes; ++i) {
+			off = rootvoldata->volu_loff[i];
+			if (off == (hammer2_off_t)-1)
+				errx(1, "Volume offset[%d] 0x%016jx must not be -1",
+				     i, (intmax_t)off);
+		}
+		for (i = nvolumes; i < HAMMER2_MAX_VOLUMES; ++i) {
+			off = rootvoldata->volu_loff[i];
+			if (off != (hammer2_off_t)-1)
+				errx(1, "Volume offset[%d] 0x%016jx must be -1",
+				     i, (intmax_t)off);
+		}
+	}
+
+	/* check volumes */
+	for (i = 0; i < HAMMER2_MAX_VOLUMES; ++i) {
+		vol = &fsp->volumes[i];
+		if (vol->id == -1)
+			continue;
+		path = vol->path;
+		/* check offset */
+		if (vol->offset & HAMMER2_FREEMAP_LEVEL1_MASK)
+			errx(1, "%s's offset 0x%016jx not 0x%016jx aligned",
+			     path, (intmax_t)vol->offset,
+			     HAMMER2_FREEMAP_LEVEL1_SIZE);
+		/* check vs previous volume */
+		if (i) {
+			if (vol->id != (vol-1)->id + 1)
+				errx(1, "%s has inconsistent id %d", path,
+				     vol->id);
+			if (vol->offset != (vol-1)->offset + (vol-1)->size)
+				errx(1, "%s has inconsistent offset 0x%016jx",
+				     path, (intmax_t)vol->offset);
+		} else { /* first */
+			if (vol->offset)
+				errx(1, "%s has non zero offset 0x%016jx", path,
+				     (intmax_t)vol->offset);
+		}
+		/* check size for non-last and last volumes */
+		if (i != fsp->nvolumes - 1) {
+			if (vol->size < HAMMER2_FREEMAP_LEVEL1_SIZE)
+				errx(1, "%s's size must be >= 0x%016jx", path,
+				     (intmax_t)HAMMER2_FREEMAP_LEVEL1_SIZE);
+			if (vol->size & HAMMER2_FREEMAP_LEVEL1_MASK)
+				errx(1, "%s's size is not 0x%016jx aligned",
+				     path,
+				     (intmax_t)HAMMER2_FREEMAP_LEVEL1_SIZE);
+		} else { /* last */
+			if (vol->size & HAMMER2_VOLUME_ALIGNMASK64)
+				errx(1, "%s's size is not 0x%016jx aligned",
+				     path, (intmax_t)HAMMER2_VOLUME_ALIGN);
+		}
+	}
+}
+
+void
+hammer2_verify_volumes(hammer2_ondisk_t *fsp,
+		       const hammer2_volume_data_t *rootvoldata)
+{
+	hammer2_verify_volumes_common(fsp);
+	if (fsp->version >= HAMMER2_VOL_VERSION_MULTI_VOLUMES)
+		hammer2_verify_volumes_2(fsp, rootvoldata);
+	else
+		hammer2_verify_volumes_1(fsp, rootvoldata);
+	assert(fsp->nvolumes > 0);
+}
+
+void
+hammer2_print_volumes(const hammer2_ondisk_t *fsp)
+{
+	const hammer2_volume_t *vol;
+	int i, n, w = 0;
+
+	for (i = 0; i < fsp->nvolumes; ++i) {
+		vol = &fsp->volumes[i];
+		n = (int)strlen(vol->path);
+		if (n > w)
+			w = n;
+	}
+
+	printf("total    %-*.*s 0x%016jx 0x%016jx\n",
+		w, w, "", (intmax_t)0, (intmax_t)fsp->total_size);
+
+	for (i = 0; i < fsp->nvolumes; ++i) {
+		vol = &fsp->volumes[i];
+		printf("volume%-2d %-*.*s 0x%016jx 0x%016jx%s\n",
+		       vol->id, w, w, vol->path, (intmax_t)vol->offset,
+		       (intmax_t)vol->size,
+		       (vol->id == HAMMER2_ROOT_VOLUME ?
+		       " (root volume)" : ""));
+	}
+}
+
+void
+hammer2_init_volumes(const char *blkdevs, int rdonly)
+{
+	hammer2_volume_data_t *rootvoldata;
+	char *p, *devpath;
+
+	if (hammer2_volumes_initialized)
+		errx(1, "Already initialized");
+	if (!blkdevs)
+		errx(1, "NULL blkdevs");
+
+	hammer2_init_ondisk(&fso);
+	p = strdup(blkdevs);
+	while ((devpath = p) != NULL) {
+		if ((p = strchr(p, ':')) != NULL)
+			*p++ = 0;
+		devpath = getdevpath(devpath, 0);
+		if (strchr(devpath, ':'))
+			hammer2_init_volumes(devpath, rdonly);
+		else
+			hammer2_add_volume(devpath, rdonly);
+		free(devpath);
+	}
+	free(p);
+	hammer2_volumes_initialized = 1;
+
+	rootvoldata = hammer2_read_root_volume_header();
+	hammer2_verify_volumes(&fso, rootvoldata);
+	free(rootvoldata);
+}
+
+void
+hammer2_cleanup_volumes(void)
+{
+	hammer2_volume_t *vol;
+	int i;
+
+	for (i = 0; i < HAMMER2_MAX_VOLUMES; ++i) {
+		vol = &fso.volumes[i];
+		if (vol->id == -1)
+			continue;
+		hammer2_uninstall_volume(vol);
+	}
+	hammer2_volumes_initialized = 0;
+}
+
+typedef void (*callback)(const hammer2_volume_t*, void *data);
+
+static int
+hammer2_get_volume_attr(hammer2_off_t offset, callback fn, void *data)
+{
+	hammer2_volume_t *vol;
+	int i;
+
+	assert(hammer2_volumes_initialized == 1);
+	offset &= ~HAMMER2_OFF_MASK_RADIX;
+
+	/* do binary search if users really use this many supported volumes */
+	for (i = 0; i < fso.nvolumes; ++i) {
+		vol = &fso.volumes[i];
+		if ((offset >= vol->offset) &&
+		    (offset < vol->offset + vol->size)) {
+			fn(vol, data);
+			return(0);
+		}
+	}
+
+	return(-1);
+}
+
+/* fd */
+static void
+hammer2_volume_fd_cb(const hammer2_volume_t *vol, void *data)
+{
+	*(int*)data = vol->fd;
+}
+
+int
+hammer2_get_volume_fd(hammer2_off_t offset)
+{
+	int ret = 0;
+
+	if (hammer2_get_volume_attr(offset, hammer2_volume_fd_cb, &ret) < 0)
+		return(-1);
+	return(ret);
+}
+
+int
+hammer2_get_root_volume_fd(void)
+{
+	return(hammer2_get_volume_fd(0));
+}
+
+/* id */
+static void
+hammer2_volume_id_cb(const hammer2_volume_t *vol, void *data)
+{
+	*(int*)data = vol->id;
+}
+
+int
+hammer2_get_volume_id(hammer2_off_t offset)
+{
+	int ret = 0;
+
+	if (hammer2_get_volume_attr(offset, hammer2_volume_id_cb, &ret) < 0)
+		return(-1);
+	return(ret);
+}
+
+int
+hammer2_get_root_volume_id(void)
+{
+	return(hammer2_get_volume_id(0));
+}
+
+/* path */
+static void
+hammer2_volume_path_cb(const hammer2_volume_t *vol, void *data)
+{
+	*(const char**)data = vol->path;
+}
+
+const char *
+hammer2_get_volume_path(hammer2_off_t offset)
+{
+	const char *ret = NULL;
+
+	if (hammer2_get_volume_attr(offset, hammer2_volume_path_cb, &ret) < 0)
+		return(NULL);
+	return(ret);
+}
+
+const char *
+hammer2_get_root_volume_path(void)
+{
+	return(hammer2_get_volume_path(0));
+}
+
+/* offset */
+static void
+hammer2_volume_offset_cb(const hammer2_volume_t *vol, void *data)
+{
+	*(hammer2_off_t*)data = vol->offset;
+}
+
+hammer2_off_t
+hammer2_get_volume_offset(hammer2_off_t offset)
+{
+	hammer2_off_t ret = 0;
+
+	if (hammer2_get_volume_attr(offset, hammer2_volume_offset_cb, &ret) < 0)
+		return(-1);
+	return(ret);
+}
+
+hammer2_off_t
+hammer2_get_root_volume_offset(void)
+{
+	return(hammer2_get_volume_offset(0));
+}
+
+/* size */
+static void
+hammer2_volume_size_cb(const hammer2_volume_t *vol, void *data)
+{
+	*(hammer2_off_t*)data = vol->size;
+}
+
+hammer2_off_t
+hammer2_get_volume_size(hammer2_off_t offset)
+{
+	hammer2_off_t ret = 0;
+
+	if (hammer2_get_volume_attr(offset, hammer2_volume_size_cb, &ret) < 0)
+		return(-1);
+	return(ret);
+}
+
+hammer2_off_t
+hammer2_get_root_volume_size(void)
+{
+	return(hammer2_get_volume_size(0));
+}
+
+/* total size */
+hammer2_off_t
+hammer2_get_total_size(void)
+{
+	return(fso.total_size);
+}
+
+hammer2_volume_data_t*
+hammer2_read_root_volume_header(void)
+{
+	hammer2_volume_data_t *voldata;
+	int fd = hammer2_get_root_volume_fd();
+	const char *path = hammer2_get_root_volume_path();
+
+	if (fd == -1)
+		return(NULL);
+
+	voldata = calloc(1, sizeof(*voldata));
+	if (hammer2_read_volume_header(fd, path, voldata) >= 0)
+		return(voldata);
+	else
+		errx(1, "Failed to read volume header");
+}
diff --git a/sbin/hammer2/subs.c b/sbin/hammer2/subs.c
index dfec9af128..e5ee6a0577 100644
--- a/sbin/hammer2/subs.c
+++ b/sbin/hammer2/subs.c
@@ -38,12 +38,14 @@
 #include <sys/ioctl.h>
 #include <sys/mount.h>
 #include <sys/statvfs.h>
+#include <sys/diskslice.h>
 #include <unistd.h>
 #include <fcntl.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <errno.h>
+#include <err.h>
 #include <uuid.h>
 
 #include <vfs/hammer2/hammer2_disk.h>
@@ -192,6 +194,8 @@ hammer2_breftype_to_str(uint8_t type)
 		return("freemap_node");
 	case HAMMER2_BREF_TYPE_FREEMAP_LEAF:
 		return("freemap_leaf");
+	case HAMMER2_BREF_TYPE_INVALID:
+		return("invalid");
 	case HAMMER2_BREF_TYPE_FREEMAP:
 		return("freemap");
 	case HAMMER2_BREF_TYPE_VOLUME:
@@ -248,6 +252,46 @@ counttostr(hammer2_off_t size)
 	return(buf);
 }
 
+hammer2_off_t
+check_volume(int fd)
+{
+	struct partinfo pinfo;
+	struct stat st;
+	hammer2_off_t size;
+
+	/*
+	 * Get basic information about the volume
+	 */
+	if (ioctl(fd, DIOCGPART, &pinfo) < 0) {
+		/*
+		 * Allow the formatting of regular files as HAMMER2 volumes
+		 */
+		if (fstat(fd, &st) < 0)
+			err(1, "Unable to stat fd %d", fd);
+		if (!S_ISREG(st.st_mode))
+			errx(1, "Unsupported file type for fd %d", fd);
+		size = st.st_size;
+	} else {
+		/*
+		 * When formatting a block device as a HAMMER2 volume the
+		 * sector size must be compatible.  HAMMER2 uses 64K
+		 * filesystem buffers but logical buffers for direct I/O
+		 * can be as small as HAMMER2_LOGSIZE (16KB).
+		 */
+		if (pinfo.reserved_blocks) {
+			errx(1, "HAMMER2 cannot be placed in a partition "
+				"which overlaps the disklabel or MBR");
+		}
+		if (pinfo.media_blksize > HAMMER2_PBUFSIZE ||
+		    HAMMER2_PBUFSIZE % pinfo.media_blksize) {
+			errx(1, "A media sector size of %d is not supported",
+			     pinfo.media_blksize);
+		}
+		size = pinfo.media_size;
+	}
+	return(size);
+}
+
 /*
  * Borrow HAMMER1's directory hash algorithm #1 with a few modifications.
  * The filename is split into fields which are hashed separately and then
diff --git a/sbin/mount_hammer2/mount_hammer2.c b/sbin/mount_hammer2/mount_hammer2.c
index 31d46f8663..6271d81d9c 100644
--- a/sbin/mount_hammer2/mount_hammer2.c
+++ b/sbin/mount_hammer2/mount_hammer2.c
@@ -173,7 +173,7 @@ main(int ac, char *av[])
 	/*
 	 * Try to mount it, prefix if necessary.
 	 */
-	if (devpath[0] != '/' && devpath[0] != '@') {
+	if (!strchr(devpath, ':') && devpath[0] != '/' && devpath[0] != '@') {
 		char *p2;
 		asprintf(&p2, "/dev/%s", devpath);
 		free(devpath);
@@ -190,6 +190,10 @@ main(int ac, char *av[])
 		} else if (errno == ENOENT) {
 			fprintf(stderr, "mount_hammer2: %s not found\n",
 				info.volume);
+		} else if (errno == ENXIO) {
+			fprintf(stderr, "mount_hammer2: incorrect volume "
+				"specification %s\n",
+				info.volume);
 		} else {
 			perror("mount");
 		}
diff --git a/sbin/newfs_hammer2/Makefile b/sbin/newfs_hammer2/Makefile
index 9a87453bbc..db4e0efdf7 100644
--- a/sbin/newfs_hammer2/Makefile
+++ b/sbin/newfs_hammer2/Makefile
@@ -8,6 +8,6 @@ CFLAGS+= -DXXH_NAMESPACE=h2_
 .PATH: ${.CURDIR}/../../sys/libkern ${.CURDIR}/../../sys/vfs/hammer2/xxhash
 SRCS+= icrc32.c xxhash.c
 .PATH: ${.CURDIR}/../hammer2
-SRCS+= subs.c
+SRCS+= subs.c ondisk.c
 
 .include <bsd.prog.mk>
diff --git a/sbin/newfs_hammer2/newfs_hammer2.8 b/sbin/newfs_hammer2/newfs_hammer2.8
index dc78f9cef8..d879f4d0d5 100644
--- a/sbin/newfs_hammer2/newfs_hammer2.8
+++ b/sbin/newfs_hammer2/newfs_hammer2.8
@@ -30,7 +30,7 @@
 .\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 .\" SUCH DAMAGE.
 .\"
-.Dd September 12, 2020
+.Dd November 29, 2020
 .Dt NEWFS_HAMMER2 8
 .Os
 .Sh NAME
@@ -42,7 +42,7 @@
 .Op Fl r Ar auxsize
 .Op Fl V Ar version
 .Op Fl L Ar label ...
-.Ar special
+.Ar special ...
 .Sh DESCRIPTION
 The
 .Nm
diff --git a/sbin/newfs_hammer2/newfs_hammer2.c b/sbin/newfs_hammer2/newfs_hammer2.c
index e67927c81a..6d4be29e0d 100644
--- a/sbin/newfs_hammer2/newfs_hammer2.c
+++ b/sbin/newfs_hammer2/newfs_hammer2.c
@@ -33,9 +33,9 @@
  */
 
 #include <sys/types.h>
+#include <sys/param.h>
 #include <sys/diskslice.h>
 #include <sys/diskmbr.h>
-#include <sys/stat.h>
 #include <sys/time.h>
 #include <sys/sysctl.h>
 #include <sys/ioctl.h>
@@ -58,14 +58,12 @@
 
 #define MAXLABELS	HAMMER2_SET_COUNT
 
-static hammer2_off_t check_volume(const char *path, int *fdp);
 static int64_t getsize(const char *str, int64_t minval, int64_t maxval, int pw);
 static uint64_t nowtime(void);
 static int blkrefary_cmp(const void *b1, const void *b2);
 static void usage(void);
 
-static void format_hammer2(int fd, hammer2_off_t total_space,
-				hammer2_off_t free_space);
+static void format_hammer2(hammer2_ondisk_t *fso, int index);
 static void alloc_direct(hammer2_off_t *basep, hammer2_blockref_t *bref,
 				size_t bytes);
 
@@ -80,16 +78,15 @@ static const char *Label[MAXLABELS];
 static hammer2_off_t BootAreaSize;
 static hammer2_off_t AuxAreaSize;
 static int NLabels;
+static int DebugOpt;
 
 int
 main(int ac, char **av)
 {
 	uint32_t status;
-	hammer2_off_t total_space;
-	hammer2_off_t free_space;
-	hammer2_off_t reserved_space;
+	hammer2_off_t reserved_size;
+	hammer2_ondisk_t fso;
 	int ch;
-	int fd = -1;
 	int i;
 	int defaultlabels = 1;
 	char *vol_fsid = NULL;
@@ -125,7 +122,7 @@ main(int ac, char **av)
 	/*
 	 * Parse arguments
 	 */
-	while ((ch = getopt(ac, av, "L:b:r:V:")) != -1) {
+	while ((ch = getopt(ac, av, "L:b:r:V:d")) != -1) {
 		switch(ch) {
 		case 'L':
 			defaultlabels = 0;
@@ -167,6 +164,9 @@ main(int ac, char **av)
 				     Hammer2Version);
 			}
 			break;
+		case 'd':
+			DebugOpt = 1;
+			break;
 		default:
 			usage();
 			break;
@@ -202,10 +202,11 @@ main(int ac, char **av)
 	ac -= optind;
 	av += optind;
 
-	if (ac != 1 || av[0][0] == 0) {
-		fprintf(stderr, "Exactly one disk device must be specified\n");
-		exit(1);
-	}
+	if (ac == 0)
+		errx(1, "You must specify at least one disk device");
+	if (ac > HAMMER2_MAX_VOLUMES)
+		errx(1, "The maximum number of volumes is %d",
+		     HAMMER2_MAX_VOLUMES);
 
 	/*
 	 * Adjust Label[] and NLabels.
@@ -221,15 +222,38 @@ main(int ac, char **av)
 	}
 
 	/*
-	 * Collect volume information.
+	 * Construct volumes information.
+	 * 1GB alignment (level1 freemap size) for volumes except for the last.
+	 * For the last volume, typically 8MB alignment to avoid edge cases for
+	 * reserved blocks and so raid stripes (if any) operate efficiently.
 	 */
-	total_space = check_volume(av[0], &fd);
+	hammer2_init_ondisk(&fso);
+	fso.version = Hammer2Version;
+	fso.nvolumes = ac;
+	for (i = 0; i < fso.nvolumes; ++i) {
+		hammer2_volume_t *vol = &fso.volumes[i];
+		hammer2_off_t size;
+		int fd = open(av[i], O_RDWR);
+		if (fd < 0)
+			err(1, "Unable to open %s R+W", av[i]);
+		size = check_volume(fd);
+		if (i == fso.nvolumes - 1)
+			size &= ~HAMMER2_VOLUME_ALIGNMASK64;
+		else
+			size &= ~HAMMER2_FREEMAP_LEVEL1_MASK;
+		hammer2_install_volume(vol, fd, i, av[i], fso.total_size, size);
+		fso.total_size += size;
+	}
 
 	/*
-	 * ~typically 8MB alignment to avoid edge cases for reserved blocks
-	 * and so raid stripes (if any) operate efficiently.
+	 * Verify volumes constructed above.
 	 */
-	total_space &= ~HAMMER2_VOLUME_ALIGNMASK64;
+	for (i = 0; i < fso.nvolumes; ++i) {
+		hammer2_volume_t *vol = &fso.volumes[i];
+		printf("Volume %-15s size %s\n", vol->path,
+		       sizetostr(vol->size));
+	}
+	hammer2_verify_volumes(&fso, NULL);
 
 	/*
 	 * Calculate defaults for the boot area size and round to the
@@ -240,7 +264,7 @@ main(int ac, char **av)
 	 */
 	if (BootAreaSize == 0) {
 		BootAreaSize = HAMMER2_BOOT_NOM_BYTES;
-		while (BootAreaSize > total_space / 20)
+		while (BootAreaSize > fso.total_size / 20)
 			BootAreaSize >>= 1;
 		if (BootAreaSize < HAMMER2_BOOT_MIN_BYTES)
 			BootAreaSize = HAMMER2_BOOT_MIN_BYTES;
@@ -259,7 +283,7 @@ main(int ac, char **av)
 	 */
 	if (AuxAreaSize == 0) {
 		AuxAreaSize = HAMMER2_AUX_NOM_BYTES;
-		while (AuxAreaSize > total_space / 20)
+		while (AuxAreaSize > fso.total_size / 20)
 			AuxAreaSize >>= 1;
 		if (AuxAreaSize < HAMMER2_AUX_MIN_BYTES)
 			AuxAreaSize = HAMMER2_AUX_MIN_BYTES;
@@ -290,28 +314,38 @@ main(int ac, char **av)
 	 *     boundaries rather than 2GB boundaries.  Stick with the LEVEL1
 	 *     boundary.
 	 */
-	reserved_space = ((total_space + HAMMER2_FREEMAP_LEVEL1_MASK) /
+	reserved_size = ((fso.total_size + HAMMER2_FREEMAP_LEVEL1_MASK) /
 			  HAMMER2_FREEMAP_LEVEL1_SIZE) * HAMMER2_ZONE_SEG64;
 
-	free_space = total_space - reserved_space - BootAreaSize - AuxAreaSize;
-	if ((int64_t)free_space < 0) {
+	fso.free_size = fso.total_size - reserved_size - BootAreaSize - AuxAreaSize;
+	if ((int64_t)fso.free_size < 0) {
 		fprintf(stderr, "Not enough free space\n");
 		exit(1);
 	}
 
-	format_hammer2(fd, total_space, free_space);
-	fsync(fd);
-	close(fd);
+	/*
+	 * Format HAMMER2 volumes.
+	 */
+	for (i = 0; i < fso.nvolumes; ++i)
+		format_hammer2(&fso, i);
 
 	printf("---------------------------------------------\n");
 	printf("version:          %d\n", Hammer2Version);
 	printf("total-size:       %s (%jd bytes)\n",
-	       sizetostr(total_space),
-	       (intmax_t)total_space);
-	printf("boot-area-size:   %s\n", sizetostr(BootAreaSize));
-	printf("aux-area-size:    %s\n", sizetostr(AuxAreaSize));
-	printf("topo-reserved:    %s\n", sizetostr(reserved_space));
-	printf("free-space:       %s\n", sizetostr(free_space));
+	       sizetostr(fso.total_size),
+	       (intmax_t)fso.total_size);
+	printf("boot-area-size:   %s (%jd bytes)\n",
+	       sizetostr(BootAreaSize),
+	       (intmax_t)BootAreaSize);
+	printf("aux-area-size:    %s (%jd bytes)\n",
+	       sizetostr(AuxAreaSize),
+	       (intmax_t)AuxAreaSize);
+	printf("topo-reserved:    %s (%jd bytes)\n",
+	       sizetostr(reserved_size),
+	       (intmax_t)reserved_size);
+	printf("free-size:        %s (%jd bytes)\n",
+	       sizetostr(fso.free_size),
+	       (intmax_t)fso.free_size);
 	printf("vol-fsid:         %s\n", vol_fsid);
 	printf("sup-clid:         %s\n", sup_clid_name);
 	printf("sup-fsid:         %s\n", sup_fsid_name);
@@ -322,6 +356,10 @@ main(int ac, char **av)
 		printf("    clid %s\n", pfs_clid_name);
 		printf("    fsid %s\n", pfs_fsid_name);
 	}
+	if (DebugOpt) {
+		printf("---------------------------------------------\n");
+		hammer2_print_volumes(&fso);
+	}
 
 	free(vol_fsid);
 	free(sup_clid_name);
@@ -329,6 +367,9 @@ main(int ac, char **av)
 	free(pfs_clid_name);
 	free(pfs_fsid_name);
 
+	for (i = 0; i < fso.nvolumes; ++i)
+		hammer2_uninstall_volume(&fso.volumes[i]);
+
 	return(0);
 }
 
@@ -338,7 +379,7 @@ usage(void)
 {
 	fprintf(stderr,
 		"usage: newfs_hammer2 [-b bootsize] [-r auxsize] "
-		"[-V version] [-L label ...] special\n"
+		"[-V version] [-L label ...] special ...\n"
 	);
 	exit(1);
 }
@@ -411,94 +452,24 @@ nowtime(void)
 	return(xtime);
 }
 
-/*
- * Figure out how big the volume is.
- */
 static
 hammer2_off_t
-check_volume(const char *path, int *fdp)
-{
-	struct partinfo pinfo;
-	struct stat st;
-	hammer2_off_t size;
-
-	/*
-	 * Get basic information about the volume
-	 */
-	*fdp = open(path, O_RDWR);
-	if (*fdp < 0)
-		err(1, "Unable to open %s R+W", path);
-	if (ioctl(*fdp, DIOCGPART, &pinfo) < 0) {
-		/*
-		 * Allow the formatting of regular files as HAMMER2 volumes
-		 */
-		if (fstat(*fdp, &st) < 0)
-			err(1, "Unable to stat %s", path);
-		if (!S_ISREG(st.st_mode))
-			errx(1, "Unsupported file type for %s", path);
-		size = st.st_size;
-	} else {
-		/*
-		 * When formatting a block device as a HAMMER2 volume the
-		 * sector size must be compatible.  HAMMER2 uses 64K
-		 * filesystem buffers but logical buffers for direct I/O
-		 * can be as small as HAMMER2_LOGSIZE (16KB).
-		 */
-		if (pinfo.reserved_blocks) {
-			errx(1, "HAMMER2 cannot be placed in a partition "
-				"which overlaps the disklabel or MBR");
-		}
-		if (pinfo.media_blksize > HAMMER2_PBUFSIZE ||
-		    HAMMER2_PBUFSIZE % pinfo.media_blksize) {
-			errx(1, "A media sector size of %d is not supported",
-			     pinfo.media_blksize);
-		}
-		size = pinfo.media_size;
-	}
-	printf("Volume %-15s size %s\n", path, sizetostr(size));
-	return (size);
-}
-
-/*
- * Create the volume header, the super-root directory inode, and
- * the writable snapshot subdirectory (named via the label) which
- * is to be the initial mount point, or at least the first mount point.
- * newfs_hammer2 doesn't format the freemap bitmaps for these.
- *
- * 0                      4MB
- * [----reserved_area----][boot_area][aux_area]
- * [[vol_hdr][freemap]...]                     [sroot][root][root]...
- *     \                                        ^\     ^     ^
- *      \--------------------------------------/  \---/-----/---...
- *
- * NOTE: The passed total_space is 8MB-aligned to avoid edge cases.
- */
-static
-void
-format_hammer2(int fd, hammer2_off_t total_space, hammer2_off_t free_space)
+format_hammer2_misc(hammer2_volume_t *vol, hammer2_off_t boot_base,
+		    hammer2_off_t aux_base)
 {
 	char *buf = malloc(HAMMER2_PBUFSIZE);
-	hammer2_volume_data_t *voldata;
-	hammer2_inode_data_t *rawip;
-	hammer2_blockref_t sroot_blockref;
-	hammer2_blockref_t root_blockref[MAXLABELS];
-	uint64_t now;
-	hammer2_off_t volu_base = 0;
-	hammer2_off_t boot_base = HAMMER2_ZONE_SEG;
-	hammer2_off_t aux_base = boot_base + BootAreaSize;
 	hammer2_off_t alloc_base = aux_base + AuxAreaSize;
 	hammer2_off_t tmp_base;
 	size_t n;
 	int i;
 
 	/*
-	 * Clear the entire 4MB reserve for the first 2G zone and
-	 * make sure we can write to the last block.
+	 * Clear the entire 4MB reserve for the first 2G zone.
 	 */
 	bzero(buf, HAMMER2_PBUFSIZE);
-	tmp_base = volu_base;
+	tmp_base = 0;
 	for (i = 0; i < HAMMER2_ZONE_BLOCKS_SEG; ++i) {
-		n = pwrite(fd, buf, HAMMER2_PBUFSIZE, tmp_base);
+		n = pwrite(vol->fd, buf, HAMMER2_PBUFSIZE, tmp_base);
 		if (n != HAMMER2_PBUFSIZE) {
 			perror("write");
 			exit(1);
@@ -506,13 +477,6 @@ format_hammer2(int fd, hammer2_off_t total_space, hammer2_off_t free_space)
 		tmp_base += HAMMER2_PBUFSIZE;
 	}
 
-	n = pwrite(fd, buf, HAMMER2_PBUFSIZE,
-		   volu_base + total_space - HAMMER2_PBUFSIZE);
-	if (n != HAMMER2_PBUFSIZE) {
-		perror("write (at-end-of-volume)");
-		exit(1);
-	}
-
 	/*
 	 * Make sure alloc_base won't cross the reserved area at the
 	 * beginning of each 1GB.
@@ -528,13 +492,32 @@ format_hammer2(int fd, hammer2_off_t total_space, hammer2_off_t free_space)
 	 */
 	for (tmp_base = boot_base; tmp_base < alloc_base;
 	     tmp_base += HAMMER2_PBUFSIZE) {
-		n = pwrite(fd, buf, HAMMER2_PBUFSIZE, tmp_base);
+		n = pwrite(vol->fd, buf, HAMMER2_PBUFSIZE, tmp_base);
 		if (n != HAMMER2_PBUFSIZE) {
 			perror("write (boot/aux)");
 			exit(1);
 		}
 	}
 
+	free(buf);
+	return(alloc_base);
+}
+
+static
+hammer2_off_t
+format_hammer2_inode(hammer2_volume_t *vol, hammer2_blockref_t *sroot_blockrefp,
+		     hammer2_off_t alloc_base)
+{
+	char *buf = malloc(HAMMER2_PBUFSIZE);
+	hammer2_inode_data_t *rawip;
+	hammer2_blockref_t sroot_blockref;
+	hammer2_blockref_t root_blockref[MAXLABELS];
+	uint64_t now;
+	size_t n;
+	int i;
+
+	bzero(&sroot_blockref, sizeof(sroot_blockref));
+	bzero(root_blockref, sizeof(root_blockref));
 	now = nowtime();
 	alloc_base &= ~HAMMER2_PBUFMASK64;
 	alloc_direct(&alloc_base, &sroot_blockref, HAMMER2_INODE_BYTES);
@@ -697,42 +680,118 @@ format_hammer2(int fd, hammer2_off_t total_space, hammer2_off_t free_space)
 	 */
 	assert((sroot_blockref.data_off & ~HAMMER2_PBUFMASK64) ==
 		((alloc_base - 1) & ~HAMMER2_PBUFMASK64));
-	n = pwrite(fd, buf, HAMMER2_PBUFSIZE,
+	n = pwrite(vol->fd, buf, HAMMER2_PBUFSIZE,
 		   sroot_blockref.data_off & ~HAMMER2_PBUFMASK64);
 	if (n != HAMMER2_PBUFSIZE) {
 		perror("write");
 		exit(1);
 	}
+	*sroot_blockrefp = sroot_blockref;
+
+	free(buf);
+	return(alloc_base);
+}
+
+/*
+ * Create the volume header, the super-root directory inode, and
+ * the writable snapshot subdirectory (named via the label) which
+ * is to be the initial mount point, or at least the first mount point.
+ * newfs_hammer2 doesn't format the freemap bitmaps for these.
+ *
+ * 0                      4MB
+ * [----reserved_area----][boot_area][aux_area]
+ * [[vol_hdr][freemap]...]                     [sroot][root][root]...
+ *     \                                        ^\     ^     ^
+ *      \--------------------------------------/  \---/-----/---...
+ *
+ * NOTE: The total size is 8MB-aligned to avoid edge cases.
+ */
+static
+void
+format_hammer2(hammer2_ondisk_t *fso, int index)
+{
+	char *buf = malloc(HAMMER2_PBUFSIZE);
+	hammer2_volume_t *vol = &fso->volumes[index];
+	hammer2_volume_data_t *voldata;
+	hammer2_blockset_t sroot_blockset;
+	hammer2_off_t boot_base = HAMMER2_ZONE_SEG;
+	hammer2_off_t aux_base = boot_base + BootAreaSize;
+	hammer2_off_t alloc_base;
+	size_t n;
+	int i;
+
+	/*
+	 * Make sure we can write to the last usable block.
+	 */
+	bzero(buf, HAMMER2_PBUFSIZE);
+	n = pwrite(vol->fd, buf, HAMMER2_PBUFSIZE,
+		   vol->size - HAMMER2_PBUFSIZE);
+	if (n != HAMMER2_PBUFSIZE) {
+		perror("write (at-end-of-volume)");
+		exit(1);
+	}
+
+	/*
+	 * Format misc area and sroot/root inodes for the root volume.
+	 */
+	bzero(&sroot_blockset, sizeof(sroot_blockset));
+	if (vol->id == HAMMER2_ROOT_VOLUME) {
+		alloc_base = format_hammer2_misc(vol, boot_base, aux_base);
+		alloc_base = format_hammer2_inode(vol,
+						  &sroot_blockset.blockref[0],
+						  alloc_base);
+	} else {
+		alloc_base = 0;
+		for (i = 0; i < HAMMER2_SET_COUNT; ++i)
+			sroot_blockset.blockref[i].type = HAMMER2_BREF_TYPE_INVALID;
+	}
 
 	/*
 	 * Format the volume header.
 	 *
-	 * The volume header points to sroot_blockref.  Also be absolutely
-	 * sure that allocator_beg is set.
+	 * The volume header points to sroot_blockset.  Also be absolutely
+	 * sure that allocator_beg is set for the root volume.
 	 */
 	assert(HAMMER2_VOLUME_BYTES <= HAMMER2_PBUFSIZE);
 	bzero(buf, HAMMER2_PBUFSIZE);
 	voldata = (void *)buf;
 
 	voldata->magic = HAMMER2_VOLUME_ID_HBO;
-	voldata->boot_beg = boot_base;
-	voldata->boot_end = boot_base + BootAreaSize;
-	voldata->aux_beg = aux_base;
-	voldata->aux_end = aux_base + AuxAreaSize;
-	voldata->volu_size = total_space;
+	if (vol->id == HAMMER2_ROOT_VOLUME) {
+		voldata->boot_beg = boot_base;
+		voldata->boot_end = boot_base + BootAreaSize;
+		voldata->aux_beg = aux_base;
+		voldata->aux_end = aux_base + AuxAreaSize;
+	}
+	voldata->volu_size = vol->size;
 	voldata->version = Hammer2Version;
 	voldata->flags = 0;
 
+	if (voldata->version >= HAMMER2_VOL_VERSION_MULTI_VOLUMES) {
+		voldata->volu_id = vol->id;
+		voldata->nvolumes = fso->nvolumes;
+		voldata->total_size = fso->total_size;
+		for (i = 0; i < HAMMER2_MAX_VOLUMES; ++i) {
+			if (i < fso->nvolumes)
+				voldata->volu_loff[i] = fso->volumes[i].offset;
+			else
+				voldata->volu_loff[i] = (hammer2_off_t)-1;
+		}
+	}
+
 	voldata->fsid = Hammer2_VolFSID;
 	voldata->fstype = Hammer2_FSType;
 
 	voldata->peer_type = DMSG_PEER_HAMMER2;	/* LNK_CONN identification */
 
-	voldata->allocator_size = free_space;
-	voldata->allocator_free = free_space;
-	voldata->allocator_beg = alloc_base;
+	assert(vol->id == HAMMER2_ROOT_VOLUME || alloc_base == 0);
+	voldata->allocator_size = fso->free_size;
+	if (vol->id == HAMMER2_ROOT_VOLUME) {
+		voldata->allocator_free = fso->free_size;
+		voldata->allocator_beg = alloc_base;
+	}
 
-	voldata->sroot_blockset.blockref[0] = sroot_blockref;
+	voldata->sroot_blockset = sroot_blockset;
 	voldata->mirror_tid = 16;	/* all blockref mirror TIDs set to 16 */
 	voldata->freemap_tid = 16;	/* all blockref mirror TIDs set to 16 */
 	voldata->icrc_sects[HAMMER2_VOL_ICRC_SECT1] =
@@ -755,15 +814,16 @@ format_hammer2(int fd, hammer2_off_t total_space, hammer2_off_t free_space)
 	 * Write the volume header and all alternates.
 	 */
 	for (i = 0; i < HAMMER2_NUM_VOLHDRS; ++i) {
-		if (i * HAMMER2_ZONE_BYTES64 >= total_space)
+		if (i * HAMMER2_ZONE_BYTES64 >= vol->size)
 			break;
-		n = pwrite(fd, buf, HAMMER2_PBUFSIZE,
-			   volu_base + i * HAMMER2_ZONE_BYTES64);
+		n = pwrite(vol->fd, buf, HAMMER2_PBUFSIZE,
+			   i * HAMMER2_ZONE_BYTES64);
 		if (n != HAMMER2_PBUFSIZE) {
 			perror("write");
 			exit(1);
 		}
 	}
+	fsync(vol->fd);
 
 	/*
 	 * Cleanup
diff --git a/sys/conf/files b/sys/conf/files
index 6cbeb09403..4d96eaa4e2 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -2031,6 +2031,7 @@ vfs/hammer2/hammer2_flush.c	optional hammer2
 vfs/hammer2/hammer2_msgops.c	optional hammer2
 vfs/hammer2/hammer2_freemap.c	optional hammer2
 vfs/hammer2/hammer2_strategy.c	optional hammer2
+vfs/hammer2/hammer2_ondisk.c	optional hammer2
 vfs/hammer2/xxhash/xxhash.c	optional hammer2
 vfs/hammer2/zlib/hammer2_zlib_adler32.c	optional hammer2
 vfs/hammer2/zlib/hammer2_zlib_inflate.c	optional hammer2
diff --git a/sys/vfs/hammer2/Makefile b/sys/vfs/hammer2/Makefile
index ab20061ff3..9c5abb19e3 100644
--- a/sys/vfs/hammer2/Makefile
+++ b/sys/vfs/hammer2/Makefile
@@ -15,7 +15,7 @@ KCFLAGS+= -DINVARIANTS
 #KCFLAGS+= -DZ_PREFIX -DXXH_NAMESPACE=h2_
 KMOD=	hammer2
 SRCS=	hammer2_vfsops.c hammer2_vnops.c hammer2_xops.c hammer2_inode.c
-SRCS+=	hammer2_strategy.c
+SRCS+=	hammer2_strategy.c hammer2_ondisk.c
 SRCS+=	hammer2_chain.c hammer2_flush.c hammer2_freemap.c hammer2_cluster.c
 SRCS+=	hammer2_ioctl.c hammer2_msgops.c hammer2_subr.c hammer2_bulkfree.c
 SRCS+=  hammer2_lz4.c hammer2_io.c hammer2_iocom.c hammer2_admin.c
diff --git a/sys/vfs/hammer2/hammer2.h b/sys/vfs/hammer2/hammer2.h
index 657c296579..1e6d0fbbb5 100644
--- a/sys/vfs/hammer2/hammer2.h
+++ b/sys/vfs/hammer2/hammer2.h
@@ -291,7 +291,9 @@ RB_HEAD(hammer2_io_tree, hammer2_io);
 struct hammer2_io {
 	RB_ENTRY(hammer2_io) rbnode;	/* indexed by device offset */
 	struct hammer2_dev *hmp;
+	struct vnode	*devvp;
 	struct buf	*bp;
+	off_t		dbase;		/* offset of devvp within volumes */
 	off_t		pbase;
 	uint64_t	refs;
 	int		psize;
@@ -1101,6 +1103,34 @@ typedef struct hammer2_xop_group hammer2_xop_group_t;
 #define HAMMER2_XOP_FSSYNC		0x00000010
 #define HAMMER2_XOP_IROOT		0x00000020
 
+/*
+ * Device vnode management structure
+ */
+struct hammer2_devvp {
+	TAILQ_ENTRY(hammer2_devvp) entry;
+	struct vnode	*devvp;		/* device vnode */
+	char		*path;		/* device vnode path */
+	int		open;		/* 1 if devvp open */
+};
+
+typedef struct hammer2_devvp hammer2_devvp_t;
+
+TAILQ_HEAD(hammer2_devvp_list, hammer2_devvp);
+
+typedef struct hammer2_devvp_list hammer2_devvp_list_t;
+
+/*
+ * Volume management structure
+ */
+struct hammer2_volume {
+	hammer2_devvp_t *dev;		/* device vnode management */
+	int		id;		/* volume id */
+	hammer2_off_t	offset;		/* offset within volumes */
+	hammer2_off_t	size;		/* volume size */
+};
+
+typedef struct hammer2_volume hammer2_volume_t;
+
 /*
  * Global (per partition) management structure, represents a hard block
  * device.  Typically referenced by hammer2_chain structures when applicable.
@@ -1113,7 +1143,7 @@ typedef struct hammer2_xop_group hammer2_xop_group_t;
  * the set of available clusters.
  */
 struct hammer2_dev {
-	struct vnode	*devvp;		/* device vnode */
+	struct vnode	*devvp;		/* device vnode for root volume */
 	int		ronly;		/* read-only mount */
 	int		mount_count;	/* number of actively mounted PFSs */
 	TAILQ_ENTRY(hammer2_dev) mntentry; /* hammer2_mntlist */
@@ -1138,11 +1168,16 @@ struct hammer2_dev {
 	int		volhdrno;	/* last volhdrno written */
 	uint32_t	hflags;		/* HMNT2 flags applicable to device */
 	hammer2_off_t	free_reserved;	/* nominal free reserved */
+	hammer2_off_t	total_size;	/* total size of volumes */
+	int		nvolumes;	/* total number of volumes */
 	hammer2_thread_t bfthr;		/* bulk-free thread */
 	char		devrepname[64];	/* for kprintf */
 	hammer2_ioc_bulkfree_t bflast;	/* stats for last bulkfree run */
 	hammer2_volume_data_t voldata;
 	hammer2_volume_data_t volsync;	/* synchronized voldata */
+
+	hammer2_devvp_list_t devvpl;	/* list of device vnodes including *devvp */
+	hammer2_volume_t volumes[HAMMER2_MAX_VOLUMES]; /* list of volumes */
 };
 
 typedef struct hammer2_dev hammer2_dev_t;
@@ -1945,6 +1980,20 @@ void hammer2_dedup_record(hammer2_chain_t *chain, hammer2_io_t *dio,
 				const char *data);
 void hammer2_dedup_clear(hammer2_dev_t *hmp);
 
+/*
+ * hammer2_ondisk.c
+ */
+int hammer2_open_devvp(const hammer2_devvp_list_t *devvpl, int ronly);
+int hammer2_close_devvp(const hammer2_devvp_list_t *devvpl, int ronly);
+int hammer2_init_devvp(const char *blkdevs, int rootmount,
+			hammer2_devvp_list_t *devvpl);
+void hammer2_cleanup_devvp(hammer2_devvp_list_t *devvpl);
+int hammer2_init_volumes(struct mount *mp, const hammer2_devvp_list_t *devvpl,
+			hammer2_volume_t *volumes,
+			hammer2_volume_data_t *rootvoldata,
+			struct vnode **rootvoldevvp);
+hammer2_volume_t *hammer2_get_volume(hammer2_dev_t *hmp, hammer2_off_t offset);
+
 /*
  * More complex inlines
  */
diff --git a/sys/vfs/hammer2/hammer2_bulkfree.c b/sys/vfs/hammer2/hammer2_bulkfree.c
index a8353461a1..c171083ef4 100644
--- a/sys/vfs/hammer2/hammer2_bulkfree.c
+++ b/sys/vfs/hammer2/hammer2_bulkfree.c
@@ -479,8 +479,8 @@ hammer2_bulkfree_pass(hammer2_dev_t *hmp, hammer2_chain_t *vchain,
 	 * Cap at the size needed to cover the whole volume to avoid
 	 * making an unnecessarily large allocation.
 	 */
-	if (size > hmp->voldata.volu_size / HAMMER2_FREEMAP_SIZEDIV) {
-		size = (hmp->voldata.volu_size + HAMMER2_FREEMAP_SIZEMASK) /
+	if (size > hmp->total_size / HAMMER2_FREEMAP_SIZEDIV) {
+		size = (hmp->total_size + HAMMER2_FREEMAP_SIZEMASK) /
 			HAMMER2_FREEMAP_SIZEDIV;
 	}
 
@@ -508,8 +508,8 @@ hammer2_bulkfree_pass(hammer2_dev_t *hmp, hammer2_chain_t *vchain,
 	 * 64KB leaf bitmap boundary which represents 2GB of storage.
 	 */
 	cbinfo.sbase = bfi->sbase;
-	if (cbinfo.sbase > hmp->voldata.volu_size)
-		cbinfo.sbase = hmp->voldata.volu_size;
+	if (cbinfo.sbase > hmp->total_size)
+		cbinfo.sbase = hmp->total_size;
 	cbinfo.sbase &= ~HAMMER2_FREEMAP_LEVEL1_MASK;
 	TAILQ_INIT(&cbinfo.list);
 
@@ -520,7 +520,7 @@ hammer2_bulkfree_pass(hammer2_dev_t *hmp, hammer2_chain_t *vchain,
 	 * get through all available storage.
 	 */
 	error = 0;
-	while (cbinfo.sbase < hmp->voldata.volu_size) {
+	while (cbinfo.sbase < hmp->total_size) {
 		/*
 		 * We have enough ram to represent (incr) bytes of storage.
 		 * Each 64KB of ram represents 2GB of storage.
@@ -545,8 +545,8 @@ hammer2_bulkfree_pass(hammer2_dev_t *hmp, hammer2_chain_t *vchain,
 
 		incr = size / HAMMER2_FREEMAP_LEVELN_PSIZE *
 		       HAMMER2_FREEMAP_LEVEL1_SIZE;
-		if (hmp->voldata.volu_size - cbinfo.sbase <= incr) {
-			cbinfo.sstop = hmp->voldata.volu_size;
+		if (hmp->total_size - cbinfo.sbase <= incr) {
+			cbinfo.sstop = hmp->total_size;
 			allmedia = 1;
 		} else {
 			cbinfo.sstop = cbinfo.sbase + incr;
@@ -650,7 +650,7 @@ hammer2_bulkfree_pass(hammer2_dev_t *hmp, hammer2_chain_t *vchain,
 
 	bfi->sstop = cbinfo.sbase;
 
-	incr = bfi->sstop / (hmp->voldata.volu_size / 10000);
+	incr = bfi->sstop / (hmp->total_size / 10000);
 	if (incr > 10000)
 		incr = 10000;
 
@@ -690,7 +690,7 @@ cbinfo_bmap_init(hammer2_bulkfree_info_t *cbinfo, size_t size)
 
 	lokey = (cbinfo->hmp->voldata.allocator_beg + HAMMER2_SEGMASK64) &
 		~HAMMER2_SEGMASK64;
-	hikey = cbinfo->hmp->voldata.volu_size & ~HAMMER2_SEGMASK64;
+	hikey = cbinfo->hmp->total_size & ~HAMMER2_SEGMASK64;
 
 	bzero(bmap, size);
 	while (size) {
@@ -759,7 +759,7 @@ h2_bulkfree_callback(hammer2_bulkfree_info_t *cbinfo, hammer2_blockref_t *bref)
 		return 0;
 	if (data_off < cbinfo->hmp->voldata.allocator_beg)
 		return 0;
-	if (data_off >= cbinfo->hmp->voldata.volu_size)
+	if (data_off >= cbinfo->hmp->total_size)
 		return 0;
 
 	/*
@@ -920,9 +920,9 @@ h2_bulkfree_sync(hammer2_bulkfree_info_t *cbinfo)
 		kprintf("%016jx-",
 			(intmax_t)cbinfo->sbase);
 
-	if (cbinfo->sstop > cbinfo->hmp->voldata.volu_size)
+	if (cbinfo->sstop > cbinfo->hmp->total_size)
 		kprintf("%016jx\n",
-			(intmax_t)cbinfo->hmp->voldata.volu_size);
+			(intmax_t)cbinfo->hmp->total_size);
 	else
 		kprintf("%016jx\n",
 			(intmax_t)cbinfo->sstop);
@@ -943,12 +943,12 @@ h2_bulkfree_sync(hammer2_bulkfree_info_t *cbinfo)
 	while (data_off < cbinfo->sstop) {
 		/*
 		 * The freemap is not used below allocator_beg or beyond
-		 * volu_size.
+		 * total_size.
 		 */
 
 		if (data_off < cbinfo->hmp->voldata.allocator_beg)
 			goto next;
-		if (data_off >= cbinfo->hmp->voldata.volu_size)
+		if (data_off >= cbinfo->hmp->total_size)
 			goto next;
 
 		/*
diff --git a/sys/vfs/hammer2/hammer2_disk.h b/sys/vfs/hammer2/hammer2_disk.h
index cbe30a615e..d2966fa9ff 100644
--- a/sys/vfs/hammer2/hammer2_disk.h
+++ b/sys/vfs/hammer2/hammer2_disk.h
@@ -713,6 +713,7 @@ typedef struct hammer2_blockref hammer2_blockref_t;
 #define HAMMER2_BREF_TYPE_DIRENT	4
 #define HAMMER2_BREF_TYPE_FREEMAP_NODE	5
 #define HAMMER2_BREF_TYPE_FREEMAP_LEAF	6
+#define HAMMER2_BREF_TYPE_INVALID	7
 #define HAMMER2_BREF_TYPE_FREEMAP	254	/* pseudo-type */
 #define HAMMER2_BREF_TYPE_VOLUME	255	/* pseudo-type */
 
@@ -1144,6 +1145,16 @@ typedef struct hammer2_inode_data hammer2_inode_data_t;
 #define HAMMER2_VOLUME_ID_HBO	0x48414d3205172011LLU
 #define HAMMER2_VOLUME_ID_ABO	0x11201705324d4148LLU
 
+/*
+ * If volume version is HAMMER2_VOL_VERSION_MULTI_VOLUMES or above, max
+ * HAMMER2_MAX_VOLUMES volumes are supported. There must be 1 (and only 1)
+ * volume with volume id HAMMER2_ROOT_VOLUME.
+ * Otherwise filesystem only supports 1 volume, and that volume must have
+ * volume id HAMMER2_ROOT_VOLUME(0) which was a reserved field then.
+ */
+#define HAMMER2_MAX_VOLUMES	64
+#define HAMMER2_ROOT_VOLUME	0
+
 struct hammer2_volume_data {
 	/*
 	 * sector #0 - 512 bytes
@@ -1160,8 +1171,10 @@ struct hammer2_volume_data {
 	uint8_t		copyid;			/* 0038 copyid of phys vol */
 	uint8_t		freemap_version;	/* 0039 freemap algorithm */
 	uint8_t		peer_type;		/* 003A HAMMER2_PEER_xxx */
-	uint8_t		reserved003B;		/* 003B */
-	uint32_t	reserved003C;		/* 003C */
+	uint8_t		volu_id;		/* 003B */
+	uint8_t		nvolumes;		/* 003C */
+	uint8_t		reserved003D;		/* 003D */
+	uint16_t	reserved003E;		/* 003E */
 
 	uuid_t		fsid;			/* 0040 */
 	uuid_t		fstype;			/* 0050 */
@@ -1192,7 +1205,9 @@ struct hammer2_volume_data {
 	hammer2_tid_t	reserved0088;		/* 0088 */
 	hammer2_tid_t	freemap_tid;		/* 0090 committed tid (fmap) */
 	hammer2_tid_t	bulkfree_tid;		/* 0098 bulkfree incremental */
-	hammer2_tid_t	reserved00A0[5];	/* 00A0-00C7 */
+	hammer2_tid_t	reserved00A0[4];	/* 00A0-00BF */
+
+	hammer2_off_t	total_size;		/* 00C0 Total volume size, bytes */
 
 	/*
 	 * Copyids are allocated dynamically from the copyexists bitmap.
@@ -1228,14 +1243,19 @@ struct hammer2_volume_data {
 	hammer2_blockset_t sroot_blockset;	/* 0200-03FF Superroot dir */
 
 	/*
-	 * sector #2-7
+	 * sector #2-6
 	 */
 	char	sector2[512];			/* 0400-05FF reserved */
 	char	sector3[512];			/* 0600-07FF reserved */
 	hammer2_blockset_t freemap_blockset;	/* 0800-09FF freemap  */
 	char	sector5[512];			/* 0A00-0BFF reserved */
 	char	sector6[512];			/* 0C00-0DFF reserved */
-	char	sector7[512];			/* 0E00-0FFF reserved */
+
+	/*
+	 * sector #7 - 512 bytes
+	 * Maximum 64 volume offsets within logical offset.
+	 */
+	hammer2_off_t volu_loff[HAMMER2_MAX_VOLUMES];
 
 	/*
 	 * sector #8-71	- 32768 bytes
@@ -1287,9 +1307,11 @@ typedef struct hammer2_volume_data hammer2_volume_data_t;
 #define HAMMER2_VOLUME_ICRC1_SIZE	(512)
 #define HAMMER2_VOLUME_ICRCVH_SIZE	(65536 - 4)
 
+#define HAMMER2_VOL_VERSION_MULTI_VOLUMES	2
+
 #define HAMMER2_VOL_VERSION_MIN		1
-#define HAMMER2_VOL_VERSION_DEFAULT	1
-#define HAMMER2_VOL_VERSION_WIP		2
+#define HAMMER2_VOL_VERSION_DEFAULT	HAMMER2_VOL_VERSION_MULTI_VOLUMES
+#define HAMMER2_VOL_VERSION_WIP		(HAMMER2_VOL_VERSION_MULTI_VOLUMES + 1)
 
 #define HAMMER2_NUM_VOLHDRS		4
 
diff --git a/sys/vfs/hammer2/hammer2_flush.c b/sys/vfs/hammer2/hammer2_flush.c
index 32506eb56d..534caa1172 100644
--- a/sys/vfs/hammer2/hammer2_flush.c
+++ b/sys/vfs/hammer2/hammer2_flush.c
@@ -1323,6 +1323,8 @@ hammer2_xop_inode_flush(hammer2_xop_t *arg, void *scratch __unused, int clindex)
 	hammer2_inode_t *ip;
 	hammer2_dev_t *hmp;
 	hammer2_pfs_t *pmp;
+	hammer2_devvp_t *e;
+	struct vnode *devvp;
 	int flush_error = 0;
 	int fsync_error = 0;
 	int total_error = 0;
@@ -1484,12 +1486,16 @@ hammer2_xop_inode_flush(hammer2_xop_t *arg, void *scratch __unused, int clindex)
 	 *
 	 * XXX this isn't being incremental
 	 */
-	vn_lock(hmp->devvp, LK_EXCLUSIVE | LK_RETRY);
-	fsync_error = VOP_FSYNC(hmp->devvp, MNT_WAIT, 0);
-	vn_unlock(hmp->devvp);
-	if (fsync_error || flush_error) {
-		kprintf("hammer2: sync error fsync=%d h2flush=0x%04x dev=%s\n",
-			fsync_error, flush_error, hmp->devrepname);
+	TAILQ_FOREACH(e, &hmp->devvpl, entry) {
+		devvp = e->devvp;
+		KKASSERT(devvp);
+		vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
+		fsync_error = VOP_FSYNC(devvp, MNT_WAIT, 0);
+		vn_unlock(devvp);
+		if (fsync_error || flush_error) {
+			kprintf("hammer2: sync error fsync=%d h2flush=0x%04x dev=%s\n",
+				fsync_error, flush_error, e->path);
+		}
 	}
 
 	/*
diff --git a/sys/vfs/hammer2/hammer2_freemap.c b/sys/vfs/hammer2/hammer2_freemap.c
index ecc1d11fcb..97e2c0483d 100644
--- a/sys/vfs/hammer2/hammer2_freemap.c
+++ b/sys/vfs/hammer2/hammer2_freemap.c
@@ -272,8 +272,8 @@ hammer2_freemap_alloc(hammer2_chain_t *chain, size_t bytes)
 	 * Make sure bpref is in-bounds.  It's ok if bpref covers a zone's
 	 * reserved area, the try code will iterate past it.
 	 */
-	if (iter.bpref > hmp->voldata.volu_size)
-		iter.bpref = hmp->voldata.volu_size - 1;
+	if (iter.bpref > hmp->total_size)
+		iter.bpref = hmp->total_size - 1;
 
 	/*
 	 * Iterate the freemap looking for free space before and after.
@@ -515,7 +515,7 @@ hammer2_freemap_try_alloc(hammer2_chain_t **parentp,
 		 * reserved segment area for a zone.
 		 */
 		KKASSERT(key >= hmp->voldata.allocator_beg &&
-			 key + bytes <= hmp->voldata.volu_size);
+			 key + bytes <= hmp->total_size);
 		KKASSERT((key & HAMMER2_ZONE_MASK64) >= HAMMER2_ZONE_SEG);
 		bref->data_off = key | radix;
 
@@ -878,8 +878,8 @@ hammer2_freemap_init(hammer2_dev_t *hmp, hammer2_key_t key,
 	 *     allocated.
 	 */
 	hikey = key + HAMMER2_FREEMAP_LEVEL1_SIZE;
-	if (hikey > hmp->voldata.volu_size) {
-		hikey = hmp->voldata.volu_size & ~HAMMER2_SEGMASK64;
+	if (hikey > hmp->total_size) {
+		hikey = hmp->total_size & ~HAMMER2_SEGMASK64;
 	}
 
 	/*
@@ -927,7 +927,7 @@ hammer2_freemap_iterate(hammer2_chain_t **parentp, hammer2_chain_t **chainp,
 
 	iter->bnext &= ~HAMMER2_FREEMAP_LEVEL1_MASK;
 	iter->bnext += HAMMER2_FREEMAP_LEVEL1_SIZE;
-	if (iter->bnext >= hmp->voldata.volu_size) {
+	if (iter->bnext >= hmp->total_size) {
 		iter->bnext = 0;
 		if (++iter->loops >= 2) {
 			if (iter->relaxed == 0)
diff --git a/sys/vfs/hammer2/hammer2_io.c b/sys/vfs/hammer2/hammer2_io.c
index b419f7ddae..124af838da 100644
--- a/sys/vfs/hammer2/hammer2_io.c
+++ b/sys/vfs/hammer2/hammer2_io.c
@@ -132,6 +132,7 @@ hammer2_io_alloc(hammer2_dev_t *hmp, hammer2_key_t data_off, uint8_t btype,
 	hammer2_key_t lbase;
 	hammer2_key_t pbase;
 	hammer2_key_t pmask;
+	hammer2_volume_t *vol;
 	uint64_t refs;
 	int lsize;
 	int psize;
@@ -172,8 +173,12 @@ hammer2_io_alloc(hammer2_dev_t *hmp, hammer2_key_t data_off, uint8_t btype,
 	} else if (createit) {
 		refs = 0;
 		hammer2_spin_unsh(&hmp->io_spin);
+		vol = hammer2_get_volume(hmp, pbase);
 		dio = kmalloc(sizeof(*dio), M_HAMMER2, M_INTWAIT | M_ZERO);
 		dio->hmp = hmp;
+		dio->devvp = vol->dev->devvp;
+		dio->dbase = vol->offset;
+		KKASSERT((dio->dbase & HAMMER2_FREEMAP_LEVEL1_MASK) == 0);
 		dio->pbase = pbase;
 		dio->psize = psize;
 		dio->btype = btype;
@@ -214,6 +219,7 @@ _hammer2_io_getblk(hammer2_dev_t *hmp, int btype, off_t lbase,
 		   int lsize, int op HAMMER2_IO_DEBUG_ARGS)
 {
 	hammer2_io_t *dio;
+	hammer2_off_t dev_pbase;
 	off_t peof;
 	uint64_t orefs;
 	uint64_t nrefs;
@@ -293,13 +299,14 @@ _hammer2_io_getblk(hammer2_dev_t *hmp, int btype, off_t lbase,
 		hce = hammer2_cluster_meta_read;
 
 	error = 0;
+	dev_pbase = dio->pbase - dio->dbase;
 	if (dio->pbase == (lbase & ~HAMMER2_OFF_MASK_RADIX) &&
 	    dio->psize == lsize) {
 		switch(op) {
 		case HAMMER2_DOP_NEW:
 		case HAMMER2_DOP_NEWNZ:
-			dio->bp = getblk(dio->hmp->devvp,
-					 dio->pbase, dio->psize,
+			dio->bp = getblk(dio->devvp,
+					 dev_pbase, dio->psize,
 					 GETBLK_KVABIO, 0);
 			if (op == HAMMER2_DOP_NEW) {
 				bkvasync(dio->bp);
@@ -316,14 +323,15 @@ _hammer2_io_getblk(hammer2_dev_t *hmp, int btype, off_t lbase,
 				 */
 				peof = (dio->pbase + HAMMER2_SEGMASK64) &
 				       ~HAMMER2_SEGMASK64;
-				error = cluster_readx(dio->hmp->devvp,
-						     peof, dio->pbase,
+				peof -= dio->dbase;
+				error = cluster_readx(dio->devvp,
+						     peof, dev_pbase,
 						     dio->psize, bflags,
 						     dio->psize,
 						     HAMMER2_PBUFSIZE*hce,
 						     &dio->bp);
 			} else {
-				error = breadnx(dio->hmp->devvp, dio->pbase,
+				error = breadnx(dio->devvp, dev_pbase,
 						dio->psize, bflags,
 					        NULL, NULL, 0, &dio->bp);
 			}
@@ -335,13 +343,14 @@ _hammer2_io_getblk(hammer2_dev_t *hmp, int btype, off_t lbase,
 			 */
 			peof = (dio->pbase + HAMMER2_SEGMASK64) &
 			       ~HAMMER2_SEGMASK64;
-			error = cluster_readx(dio->hmp->devvp,
-					      peof, dio->pbase, dio->psize,
+			peof -= dio->dbase;
+			error = cluster_readx(dio->devvp,
+					      peof, dev_pbase, dio->psize,
 					      bflags,
 					      dio->psize, HAMMER2_PBUFSIZE*hce,
 					      &dio->bp);
 		} else {
-			error = breadnx(dio->hmp->devvp, dio->pbase,
+			error = breadnx(dio->devvp, dev_pbase,
 				        dio->psize, bflags,
 					NULL, NULL, 0, &dio->bp);
 		}
@@ -522,6 +531,7 @@ _hammer2_io_putblk(hammer2_io_t **diop HAMMER2_IO_DEBUG_ARGS)
 				if ((hce = hammer2_cluster_write) != 0) {
 					peof = (pbase + HAMMER2_SEGMASK64) &
 					       ~HAMMER2_SEGMASK64;
+					peof -= dio->dbase;
 					bp->b_flags |= B_CLUSTEROK;
 					cluster_write(bp, peof, psize, hce);
 				} else {
@@ -659,6 +669,7 @@ hammer2_io_data(hammer2_io_t *dio, off_t lbase)
 	bp = dio->bp;
 	KKASSERT(bp != NULL);
 	bkvasync(bp);
+	lbase -= dio->dbase;
 	off = (lbase & ~HAMMER2_OFF_MASK_RADIX) - bp->b_loffset;
 	KKASSERT(off >= 0 && off < bp->b_bufsize);
 	return(bp->b_data + off);
diff --git a/sys/vfs/hammer2/hammer2_ioctl.c b/sys/vfs/hammer2/hammer2_ioctl.c
index b2e832c5e3..aba71fd1da 100644
--- a/sys/vfs/hammer2/hammer2_ioctl.c
+++ b/sys/vfs/hammer2/hammer2_ioctl.c
@@ -66,6 +66,7 @@ static int hammer2_ioctl_growfs(hammer2_inode_t *ip, void *data,
 //static int hammer2_ioctl_inode_comp_rec_set2(hammer2_inode_t *ip, void *data);
 static int hammer2_ioctl_bulkfree_scan(hammer2_inode_t *ip, void *data);
 static int hammer2_ioctl_destroy(hammer2_inode_t *ip, void *data);
+static int hammer2_ioctl_volume_list(hammer2_inode_t *ip, void *data);
 
 int
 hammer2_ioctl(hammer2_inode_t *ip, u_long com, void *data, int fflag,
@@ -159,6 +160,10 @@ hammer2_ioctl(hammer2_inode_t *ip, u_long com, void *data, int fflag,
 		if (error == 0)
 			error = hammer2_ioctl_growfs(ip, data, cred);
 		break;
+	case HAMMER2IOC_VOLUME_LIST:
+		if (error == 0)
+			error = hammer2_ioctl_volume_list(ip, data);
+		break;
 	default:
 		error = EOPNOTSUPP;
 		break;
@@ -1259,6 +1264,11 @@ hammer2_ioctl_growfs(hammer2_inode_t *ip, void *data, struct ucred *cred)
 
 	hmp = ip->pmp->pfs_hmps[0];
 
+	if (hmp->nvolumes > 1) {
+		kprintf("hammer2: growfs unsupported with multiple volumes\n");
+		return EOPNOTSUPP;
+	}
+
 	/*
 	 * Extract from disklabel
 	 */
@@ -1357,3 +1367,46 @@ hammer2_ioctl_growfs(hammer2_inode_t *ip, void *data, struct ucred *cred)
 
 	return 0;
 }
+
+/*
+ * Get a list of volumes.
+ */
+static int
+hammer2_ioctl_volume_list(hammer2_inode_t *ip, void *data)
+{
+	hammer2_ioc_volume_list_t *vollist = data;
+	hammer2_ioc_volume_t entry;
+	hammer2_volume_t *vol;
+	hammer2_dev_t *hmp;
+	hammer2_pfs_t *pmp;
+	int i, error = 0, cnt = 0;
+
+	pmp = ip->pmp;
+	hmp = pmp->pfs_hmps[0];
+	if (hmp == NULL)
+		return (EINVAL);
+
+	hammer2_voldata_lock(hmp);
+	for (i = 0; i < hmp->nvolumes; ++i) {
+		if (cnt >= vollist->nvolumes)
+			break;
+		vol = &hmp->volumes[i];
+		bzero(&entry, sizeof(entry));
+		/* copy hammer2_volume_t fields */
+		entry.id = vol->id;
+		bcopy(vol->dev->path, entry.path, sizeof(entry.path));
+		entry.offset = vol->offset;
+		entry.size = vol->size;
+		error = copyout(&entry, &vollist->volumes[cnt], sizeof(entry));
+		if (error)
+			goto failed;
+		cnt++;
+	}
+	vollist->nvolumes = cnt;
+	vollist->version = hmp->voldata.version;
+	bcopy(pmp->pfs_names[0], vollist->pfs_name, sizeof(vollist->pfs_name));
+failed:
+	hammer2_voldata_unlock(hmp);
+
+	return error;
+}
diff --git a/sys/vfs/hammer2/hammer2_ioctl.h b/sys/vfs/hammer2/hammer2_ioctl.h
index 4ceb9305bb..20e92bb308 100644
--- a/sys/vfs/hammer2/hammer2_ioctl.h
+++ b/sys/vfs/hammer2/hammer2_ioctl.h
@@ -169,6 +169,27 @@ struct hammer2_ioc_growfs {
 
 typedef struct hammer2_ioc_growfs hammer2_ioc_growfs_t;
 
+/*
+ * Ioctls to manage volumes
+ */
+struct hammer2_ioc_volume {
+	char			path[MAXPATHLEN];
+	int			id;
+	hammer2_off_t		offset;
+	hammer2_off_t		size;
+};
+
+typedef struct hammer2_ioc_volume hammer2_ioc_volume_t;
+
+struct hammer2_ioc_volume_list {
+	hammer2_ioc_volume_t	*volumes;
+	int			nvolumes;
+	int			version;
+	char			pfs_name[HAMMER2_INODE_MAXNAME];
+};
+
+typedef struct hammer2_ioc_volume_list hammer2_ioc_volume_list_t;
+
 /*
  * Ioctl list
  */
@@ -205,5 +226,6 @@ typedef struct hammer2_ioc_growfs hammer2_ioc_growfs_t;
 #define HAMMER2IOC_DESTROY	_IOWR('h', 94, struct hammer2_ioc_destroy)
 #define HAMMER2IOC_EMERG_MODE	_IOWR('h', 95, int)
 #define HAMMER2IOC_GROWFS	_IOWR('h', 96, struct hammer2_ioc_growfs)
+#define HAMMER2IOC_VOLUME_LIST	_IOWR('h', 97, struct hammer2_ioc_volume_list)
 
 #endif /* !_VFS_HAMMER2_IOCTL_H_ */
diff --git a/sys/vfs/hammer2/hammer2_ondisk.c b/sys/vfs/hammer2/hammer2_ondisk.c
new file mode 100644
index 0000000000..096fad2a5c
--- /dev/null
+++ b/sys/vfs/hammer2/hammer2_ondisk.c
@@ -0,0 +1,707 @@
+/*
+ * Copyright (c) 2020 Tomohiro Kusumi <tkusumi@netbsd.org>
+ * Copyright (c) 2020 The DragonFly Project
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The DragonFly Project
+ * by Matthew Dillon <dillon@dragonflybsd.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ * 3. Neither the name of The DragonFly Project nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific, prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/queue.h>
+#include <sys/nlookup.h>
+#include <sys/vnode.h>
+#include <sys/mount.h>
+#include <sys/fcntl.h>
+#include <sys/buf.h>
+#include <sys/uuid.h>
+#include <sys/objcache.h>
+
+#include "hammer2.h"
+
+#define hprintf(X, ...)	kprintf("hammer2_ondisk: " X, ## __VA_ARGS__)
+
+static int
+hammer2_lookup_device(const char *path, int rootmount, struct vnode **devvp)
+{
+	struct vnode *vp = NULL;
+	struct nlookupdata nd;
+	int error = 0;
+
+	KKASSERT(path);
+	KKASSERT(*path != '\0');
+
+	if (rootmount) {
+		error = bdevvp(kgetdiskbyname(path), &vp);
+		if (error)
+			hprintf("cannot find %s %d\n", path, error);
+	} else {
+		error = nlookup_init(&nd, path, UIO_SYSSPACE, NLC_FOLLOW);
+		if (error == 0)
+			error = nlookup(&nd);
+		if (error == 0)
+			error = cache_vref(&nd.nl_nch, nd.nl_cred, &vp);
+		if (error)
+			hprintf("failed to nlookup %s %d\n", path, error);
+		nlookup_done(&nd);
+	}
+
+	if (error == 0) {
+		KKASSERT(vp);
+		if (!vn_isdisk(vp, &error)) {
+			KKASSERT(error);
+			hprintf("%s not a block device %d\n", path, error);
+		}
+	}
+
+	if (error && vp) {
+		vrele(vp);
+		vp = NULL;
+	}
+
+	*devvp = vp;
+	return error;
+}
+
+int
+hammer2_open_devvp(const hammer2_devvp_list_t *devvpl, int ronly)
+{
+	hammer2_devvp_t *e;
+	struct vnode *devvp;
+	const char *path;
+	int count, error;
+
+	TAILQ_FOREACH(e, devvpl, entry) {
+		devvp = e->devvp;
+		path = e->path;
+		KKASSERT(devvp);
+		count = vcount(devvp);
+		if (count > 0) {
+			hprintf("%s already has %d references\n", path, count);
+			return EBUSY;
+		}
+		vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
+		error = vinvalbuf(devvp, V_SAVE, 0, 0);
+		if (error == 0) {
+			KKASSERT(!e->open);
+			error = VOP_OPEN(devvp,
+					 (ronly ? FREAD : FREAD | FWRITE),
+					 FSCRED, NULL);
+			if (error == 0)
+				e->open = 1;
+			else
+				hprintf("failed to open %s %d\n", path, error);
+		}
+		vn_unlock(devvp);
+		if (error)
+			return error;
+		KKASSERT(e->open);
+	}
+
+	return 0;
+}
+
+int
+hammer2_close_devvp(const hammer2_devvp_list_t *devvpl, int ronly)
+{
+	hammer2_devvp_t *e;
+	struct vnode *devvp;
+
+	TAILQ_FOREACH(e, devvpl, entry) {
+		devvp = e->devvp;
+		KKASSERT(devvp);
+		if (e->open) {
+			vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
+			vinvalbuf(devvp, (ronly ? 0 : V_SAVE), 0, 0);
+			VOP_CLOSE(devvp, (ronly ? FREAD : FREAD|FWRITE), NULL);
+			vn_unlock(devvp);
+			e->open = 0;
+		}
+	}
+
+	return 0;
+}
+
+int
+hammer2_init_devvp(const char *blkdevs, int rootmount,
+		   hammer2_devvp_list_t *devvpl)
+{
+	hammer2_devvp_t *e;
+	struct vnode *devvp;
+	const char *p;
+	char *path;
+	int i, error = 0;
+
+	KKASSERT(TAILQ_EMPTY(devvpl));
+	KKASSERT(blkdevs); /* could be empty string */
+	p = blkdevs;
+
+	path = objcache_get(namei_oc, M_WAITOK);
+	while (1) {
+		strcpy(path, "");
+		if (*p != '/') {
+			strcpy(path, "/dev/"); /* relative path */
+		}
+		/* scan beyond "/dev/" */
+		for (i = strlen(path); i < MAXPATHLEN-1; ++i) {
+			if (*p == '\0') {
+				break;
+			} else if (*p == ':') {
+				p++;
+				break;
+			} else {
+				path[i] = *p;
+				p++;
+			}
+		}
+		path[i] = '\0';
+		/* path shorter than "/dev/" means invalid or done */
+		if (strlen(path) <= strlen("/dev/")) {
+			if (strlen(p)) {
+				hprintf("ignore incomplete path %s\n", path);
+				continue;
+			} else {
+				/* end of string */
+				KKASSERT(*p == '\0');
+				break;
+			}
+		}
+		/* lookup path from above */
+		KKASSERT(strncmp(path, "/dev/", 5) == 0);
+		devvp = NULL;
+		error = hammer2_lookup_device(path, rootmount, &devvp);
+		if (error) {
+			KKASSERT(!devvp);
+			hprintf("failed to lookup %s %d\n", path, error);
+			break;
+		}
+		KKASSERT(devvp);
+		e = kmalloc(sizeof(*e), M_HAMMER2, M_WAITOK | M_ZERO);
+		e->devvp = devvp;
+		e->path = kstrdup(path, M_HAMMER2);
+		TAILQ_INSERT_TAIL(devvpl, e, entry);
+	}
+	objcache_put(namei_oc, path);
+
+	return error;
+}
+
+void
+hammer2_cleanup_devvp(hammer2_devvp_list_t *devvpl)
+{
+	hammer2_devvp_t *e;
+
+	while (!TAILQ_EMPTY(devvpl)) {
+		e = TAILQ_FIRST(devvpl);
+		TAILQ_REMOVE(devvpl, e, entry);
+		/* devvp */
+		KKASSERT(e->devvp);
+		e->devvp->v_rdev->si_mountpoint = NULL;
+		vrele(e->devvp);
+		e->devvp = NULL;
+		/* path */
+		KKASSERT(e->path);
+		kfree(e->path, M_HAMMER2);
+		e->path = NULL;
+		kfree(e, M_HAMMER2);
+	}
+}
+
+static int
+hammer2_verify_volumes_common(const hammer2_volume_t *volumes)
+{
+	const hammer2_volume_t *vol;
+	struct partinfo part;
+	const char *path;
+	int i;
+
+	for (i = 0; i < HAMMER2_MAX_VOLUMES; ++i) {
+		vol = &volumes[i];
+		if (vol->id == -1)
+			continue;
+		path = vol->dev->path;
+		/* check volume fields are initialized */
+		if (!vol->dev->devvp) {
+			hprintf("%s has NULL devvp\n", path);
+			return EINVAL;
+		}
+		if (vol->offset == (hammer2_off_t)-1) {
+			hprintf("%s has bad offset 0x%016jx\n", path,
+				(intmax_t)vol->offset);
+			return EINVAL;
+		}
+		if (vol->size == (hammer2_off_t)-1) {
+			hprintf("%s has bad size 0x%016jx\n", path,
+				(intmax_t)vol->size);
+			return EINVAL;
+		}
+		/* check volume size vs block device size */
+		if (VOP_IOCTL(vol->dev->devvp, DIOCGPART, (void*)&part, 0,
+			      curthread->td_ucred , NULL) == 0) {
+			if (vol->size > part.media_size) {
+				hprintf("%s's size 0x%016jx exceeds device size "
+					"0x%016jx\n", path, (intmax_t)vol->size,
+					part.media_size);
+				return EINVAL;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int
+hammer2_verify_volumes_1(const hammer2_volume_t *volumes,
+		         const hammer2_volume_data_t *rootvoldata)
+{
+	const hammer2_volume_t *vol;
+	hammer2_off_t off;
+	const char *path;
+	int i, nvolumes = 0;
+
+	/* check initialized volume count */
+	for (i = 0; i < HAMMER2_MAX_VOLUMES; ++i) {
+		vol = &volumes[i];
+		if (vol->id != -1)
+			nvolumes++;
+	}
+	if (nvolumes != 1) {
+		hprintf("only 1 volume supported\n");
+		return EINVAL;
+	}
+
+	/* check volume header */
+	if (rootvoldata->volu_id) {
+		hprintf("volume id %d must be 0\n", rootvoldata->volu_id);
+		return EINVAL;
+	}
+	if (rootvoldata->nvolumes) {
+		hprintf("volume count %d must be 0\n", rootvoldata->nvolumes);
+		return EINVAL;
+	}
+	if (rootvoldata->total_size) {
+		hprintf("total size 0x%016jx must be 0\n",
+			(intmax_t)rootvoldata->total_size);
+		return EINVAL;
+	}
+	for (i = 0; i < HAMMER2_MAX_VOLUMES; ++i) {
+		off = rootvoldata->volu_loff[i];
+		if (off) {
+			hprintf("volume offset[%d] 0x%016jx must be 0\n", i,
+				(intmax_t)off);
+			return EINVAL;
+		}
+	}
+
+	/* check volume */
+	vol = &volumes[0];
+	path = vol->dev->path;
+	if (vol->id) {
+		hprintf("%s has non zero id %d\n", path, vol->id);
+		return EINVAL;
+	}
+	if (vol->offset) {
+		hprintf("%s has non zero offset 0x%016jx\n", path,
+			(intmax_t)vol->offset);
+		return EINVAL;
+	}
+	if (vol->size & HAMMER2_VOLUME_ALIGNMASK64) {
+		hprintf("%s's size is not 0x%016jx aligned\n", path,
+			(intmax_t)HAMMER2_VOLUME_ALIGN);
+		return EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+hammer2_verify_volumes_2(const hammer2_volume_t *volumes,
+		         const hammer2_volume_data_t *rootvoldata)
+{
+	const hammer2_volume_t *vol;
+	hammer2_off_t off, total_size = 0;
+	const char *path;
+	int i, nvolumes = 0;
+
+	/* check initialized volume count */
+	for (i = 0; i < HAMMER2_MAX_VOLUMES; ++i) {
+		vol = &volumes[i];
+		if (vol->id != -1) {
+			nvolumes++;
+			total_size += vol->size;
+		}
+	}
+
+	/* check volume header */
+	if (rootvoldata->volu_id != HAMMER2_ROOT_VOLUME) {
+		hprintf("volume id %d must be %d\n", rootvoldata->volu_id,
+			HAMMER2_ROOT_VOLUME);
+		return EINVAL;
+	}
+	if (rootvoldata->nvolumes != nvolumes) {
+		hprintf("volume header requires %d devices, %d specified\n",
+			rootvoldata->nvolumes, nvolumes);
+		return EINVAL;
+	}
+	if (rootvoldata->total_size != total_size) {
+		hprintf("total size 0x%016jx does not equal sum of volumes 0x%016jx\n",
+			rootvoldata->total_size, total_size);
+		return EINVAL;
+	}
+	for (i = 0; i < nvolumes; ++i) {
+		off = rootvoldata->volu_loff[i];
+		if (off == (hammer2_off_t)-1) {
+			hprintf("volume offset[%d] 0x%016jx must not be -1\n",
+				i, (intmax_t)off);
+			return EINVAL;
+		}
+	}
+	for (i = nvolumes; i < HAMMER2_MAX_VOLUMES; ++i) {
+		off = rootvoldata->volu_loff[i];
+		if (off != (hammer2_off_t)-1) {
+			hprintf("volume offset[%d] 0x%016jx must be -1\n",
+				i, (intmax_t)off);
+			return EINVAL;
+		}
+	}
+
+	/* check volumes */
+	for (i = 0; i < HAMMER2_MAX_VOLUMES; ++i) {
+		vol = &volumes[i];
+		if (vol->id == -1)
+			continue;
+		path = vol->dev->path;
+		/* check offset */
+		if (vol->offset & HAMMER2_FREEMAP_LEVEL1_MASK) {
+			hprintf("%s's offset 0x%016jx not 0x%016jx aligned\n",
+				path, (intmax_t)vol->offset,
+				HAMMER2_FREEMAP_LEVEL1_SIZE);
+			return EINVAL;
+		}
+		/* check vs previous volume */
+		if (i) {
+			if (vol->id <= (vol-1)->id) {
+				hprintf("%s has inconsistent id %d\n", path,
+					vol->id);
+				return EINVAL;
+			}
+			if (vol->offset != (vol-1)->offset + (vol-1)->size) {
+				hprintf("%s has inconsistent offset 0x%016jx\n",
+					path, (intmax_t)vol->offset);
+				return EINVAL;
+			}
+		} else { /* first */
+			if (vol->offset) {
+				hprintf("%s has non zero offset 0x%016jx\n",
+					path, (intmax_t)vol->offset);
+				return EINVAL;
+			}
+		}
+		/* check size for non-last and last volumes */
+		if (i != rootvoldata->nvolumes - 1) {
+			if (vol->size < HAMMER2_FREEMAP_LEVEL1_SIZE) {
+				hprintf("%s's size must be >= 0x%016jx\n", path,
+					(intmax_t)HAMMER2_FREEMAP_LEVEL1_SIZE);
+				return EINVAL;
+			}
+			if (vol->size & HAMMER2_FREEMAP_LEVEL1_MASK) {
+				hprintf("%s's size is not 0x%016jx aligned\n",
+					path,
+					(intmax_t)HAMMER2_FREEMAP_LEVEL1_SIZE);
+				return EINVAL;
+			}
+		} else { /* last */
+			if (vol->size & HAMMER2_VOLUME_ALIGNMASK64) {
+				hprintf("%s's size is not 0x%016jx aligned\n",
+					path,
+					(intmax_t)HAMMER2_VOLUME_ALIGN);
+				return EINVAL;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int
+hammer2_verify_volumes(const hammer2_volume_t *volumes,
+		       const hammer2_volume_data_t *rootvoldata)
+{
+	int error;
+
+	error = hammer2_verify_volumes_common(volumes);
+	if (error)
+		return error;
+
+	if (rootvoldata->version >= HAMMER2_VOL_VERSION_MULTI_VOLUMES)
+		return hammer2_verify_volumes_2(volumes, rootvoldata);
+	else
+		return hammer2_verify_volumes_1(volumes, rootvoldata);
+}
+
+/*
+ * Returns zone# of returned volume header or < 0 on failure.
+ */
+static int
+hammer2_read_volume_header(struct vnode *devvp, const char *path,
+			   hammer2_volume_data_t *voldata)
+{
+	hammer2_volume_data_t *vd;
+	struct buf *bp = NULL;
+	hammer2_crc32_t crc0, crc1;
+	int zone = -1;
+	int i;
+
+	/*
+	 * There are up to 4 copies of the volume header (syncs iterate
+	 * between them so there is no single master).  We don't trust the
+	 * volu_size field so we don't know precisely how large the filesystem
+	 * is, so depend on the OS to return an error if we go beyond the
+	 * block device's EOF.
+	 */
+	for (i = 0; i < HAMMER2_NUM_VOLHDRS; ++i) {
+		if (bread(devvp, i * HAMMER2_ZONE_BYTES64, HAMMER2_VOLUME_BYTES,
+			  &bp)) {
+			brelse(bp);
+			bp = NULL;
+			continue;
+		}
+
+		vd = (struct hammer2_volume_data *)bp->b_data;
+		/* verify volume header magic */
+		if ((vd->magic != HAMMER2_VOLUME_ID_HBO) &&
+		    (vd->magic != HAMMER2_VOLUME_ID_ABO)) {
+			hprintf("%s #%d: bad magic\n", path, i);
+			brelse(bp);
+			bp = NULL;
+			continue;
+		}
+
+		if (vd->magic == HAMMER2_VOLUME_ID_ABO) {
+			/* XXX: Reversed-endianness filesystem */
+			hprintf("%s #%d: reverse-endian filesystem detected\n",
+				path, i);
+			brelse(bp);
+			bp = NULL;
+			continue;
+		}
+
+		/* verify volume header CRC's */
+		crc0 = vd->icrc_sects[HAMMER2_VOL_ICRC_SECT0];
+		crc1 = hammer2_icrc32(bp->b_data + HAMMER2_VOLUME_ICRC0_OFF,
+				      HAMMER2_VOLUME_ICRC0_SIZE);
+		if (crc0 != crc1) {
+			hprintf("%s #%d: volume header crc mismatch sect0 %08x/%08x\n",
+				path, i, crc0, crc1);
+			brelse(bp);
+			bp = NULL;
+			continue;
+		}
+		crc0 = vd->icrc_sects[HAMMER2_VOL_ICRC_SECT1];
+		crc1 = hammer2_icrc32(bp->b_data + HAMMER2_VOLUME_ICRC1_OFF,
+				      HAMMER2_VOLUME_ICRC1_SIZE);
+		if (crc0 != crc1) {
+			hprintf("%s #%d: volume header crc mismatch sect1 %08x/%08x\n",
+				path, i, crc0, crc1);
+			brelse(bp);
+			bp = NULL;
+			continue;
+		}
+		crc0 = vd->icrc_volheader;
+		crc1 = hammer2_icrc32(bp->b_data + HAMMER2_VOLUME_ICRCVH_OFF,
+				      HAMMER2_VOLUME_ICRCVH_SIZE);
+		if (crc0 != crc1) {
+			hprintf("%s #%d: volume header crc mismatch vh %08x/%08x\n",
+				path, i, crc0, crc1);
+			brelse(bp);
+			bp = NULL;
+			continue;
+		}
+
+		if (zone == -1 || voldata->mirror_tid < vd->mirror_tid) {
+			*voldata = *vd;
+			zone = i;
+		}
+		brelse(bp);
+		bp = NULL;
+	}
+
+	if (zone == -1) {
+		hprintf("%s has no valid volume headers\n", path);
+		return -EINVAL;
+	}
+	return zone;
+}
+
+int
+hammer2_init_volumes(struct mount *mp, const hammer2_devvp_list_t *devvpl,
+		     hammer2_volume_t *volumes,
+		     hammer2_volume_data_t *rootvoldata,
+		     struct vnode **rootvoldevvp)
+{
+	hammer2_devvp_t *e;
+	hammer2_volume_data_t *voldata;
+	hammer2_volume_t *vol;
+	struct vnode *devvp;
+	const char *path;
+	uuid_t fsid, fstype;
+	int i, zone, error = 0, version = -1, nvolumes = 0;
+
+	for (i = 0; i < HAMMER2_MAX_VOLUMES; ++i) {
+		vol = &volumes[i];
+		vol->dev = NULL;
+		vol->id = -1;
+		vol->offset = (hammer2_off_t)-1;
+		vol->size = (hammer2_off_t)-1;
+	}
+
+	voldata = kmalloc(sizeof(*voldata), M_HAMMER2, M_WAITOK | M_ZERO);
+	bzero(&fsid, sizeof(fsid));
+	bzero(&fstype, sizeof(fstype));
+	bzero(rootvoldata, sizeof(*rootvoldata));
+
+	TAILQ_FOREACH(e, devvpl, entry) {
+		devvp = e->devvp;
+		path = e->path;
+		KKASSERT(devvp);
+
+		/* returns negative error or positive zone# */
+		error = hammer2_read_volume_header(devvp, path, voldata);
+		if (error < 0) {
+			hprintf("failed to read %s's volume header\n", path);
+			error = -error;
+			goto done;
+		}
+		zone = error;
+		error = 0; /* reset error */
+
+		if (voldata->volu_id >= HAMMER2_MAX_VOLUMES) {
+			hprintf("%s has bad volume id %d\n", path,
+				voldata->volu_id);
+			error = EINVAL;
+			goto done;
+		}
+		vol = &volumes[voldata->volu_id];
+		if (vol->id != -1) {
+			hprintf("%s already initialized\n", path);
+			error = EINVAL;
+			goto done;
+		}
+		/* all headers must have the same version, nvolumes and uuid */
+		if (version == -1) {
+			version = voldata->version;
+			nvolumes = voldata->nvolumes;
+			fsid = voldata->fsid;
+			fstype = voldata->fstype;
+		} else {
+			if (version != (int)voldata->version) {
+				hprintf("volume version mismatch %d vs %d\n",
+					version, (int)voldata->version);
+				error = ENXIO;
+				goto done;
+			}
+			if (nvolumes != voldata->nvolumes) {
+				hprintf("volume count mismatch %d vs %d\n",
+					nvolumes, voldata->nvolumes);
+				error = ENXIO;
+				goto done;
+			}
+			if (bcmp(&fsid, &voldata->fsid, sizeof(fsid))) {
+				hprintf("fsid uuid mismatch\n");
+				error = ENXIO;
+				goto done;
+			}
+			if (bcmp(&fstype, &voldata->fstype, sizeof(fstype))) {
+				hprintf("fstype uuid mismatch\n");
+				error = ENXIO;
+				goto done;
+			}
+		}
+		if (version < HAMMER2_VOL_VERSION_MIN ||
+		    version > HAMMER2_VOL_VERSION_WIP) {
+			hprintf("bad volume version %d\n", version);
+			error = EINVAL;
+			goto done;
+		}
+		/* all per-volume tests passed */
+		vol->dev = e;
+		vol->id = voldata->volu_id;
+		vol->offset = voldata->volu_loff[vol->id];
+		vol->size = voldata->volu_size;
+		if (vol->id == HAMMER2_ROOT_VOLUME) {
+			bcopy(voldata, rootvoldata, sizeof(*rootvoldata));
+			KKASSERT(*rootvoldevvp == NULL);
+			*rootvoldevvp = e->devvp;
+		}
+		devvp->v_rdev->si_mountpoint = mp;
+		hprintf("\"%s\" zone=%d id=%d offset=0x%016jx size=0x%016jx\n",
+			path, zone, vol->id, (intmax_t)vol->offset,
+			(intmax_t)vol->size);
+	}
+done:
+	if (!error)
+		error = hammer2_verify_volumes(volumes, rootvoldata);
+	kfree(voldata, M_HAMMER2);
+
+	return error;
+}
+
+hammer2_volume_t*
+hammer2_get_volume(hammer2_dev_t *hmp, hammer2_off_t offset)
+{
+	hammer2_volume_t *vol, *ret = NULL;
+	int i;
+
+	offset &= ~HAMMER2_OFF_MASK_RADIX;
+
+	/* locking is unneeded until volume-add support */
+	//hammer2_voldata_lock(hmp);
+	/* do binary search if users really use this many supported volumes */
+	for (i = 0; i < hmp->nvolumes; ++i) {
+		vol = &hmp->volumes[i];
+		if ((offset >= vol->offset) &&
+		    (offset < vol->offset + vol->size)) {
+			ret = vol;
+			break;
+		}
+	}
+	//hammer2_voldata_unlock(hmp);
+
+	if (!ret)
+		panic("no volume for offset 0x%016jx", (intmax_t)offset);
+
+	KKASSERT(ret);
+	KKASSERT(ret->dev);
+	KKASSERT(ret->dev->devvp);
+	KKASSERT(ret->dev->path);
+
+	return ret;
+}
diff --git a/sys/vfs/hammer2/hammer2_subr.c b/sys/vfs/hammer2/hammer2_subr.c
index 482a75abca..ac8e243580 100644
--- a/sys/vfs/hammer2/hammer2_subr.c
+++ b/sys/vfs/hammer2/hammer2_subr.c
@@ -465,6 +465,8 @@ hammer2_bref_type_str(int btype)
 		return("freemap_node");
 	case HAMMER2_BREF_TYPE_FREEMAP_LEAF:
 		return("freemap_leaf");
+	case HAMMER2_BREF_TYPE_INVALID:
+		return("invalid");
 	case HAMMER2_BREF_TYPE_FREEMAP:
 		return("freemap");
 	case HAMMER2_BREF_TYPE_VOLUME:
diff --git a/sys/vfs/hammer2/hammer2_vfsops.c b/sys/vfs/hammer2/hammer2_vfsops.c
index 3fc67c5de5..871273ffa1 100644
--- a/sys/vfs/hammer2/hammer2_vfsops.c
+++ b/sys/vfs/hammer2/hammer2_vfsops.c
@@ -199,7 +199,7 @@ static int hammer2_vfs_uninit(struct vfsconf *vfsp);
 static int hammer2_vfs_mount(struct mount *mp, char *path, caddr_t data,
 				struct ucred *cred);
 static int hammer2_remount(hammer2_dev_t *, struct mount *, char *,
-				struct vnode *, struct ucred *);
+				struct ucred *);
 static int hammer2_recovery(hammer2_dev_t *hmp);
 static int hammer2_vfs_unmount(struct mount *mp, int mntflags);
 static int hammer2_vfs_root(struct mount *mp, struct vnode **vpp);
@@ -214,8 +214,6 @@ static int hammer2_vfs_checkexp(struct mount *mp, struct sockaddr *nam,
 				int *exflagsp, struct ucred **credanonp);
 static int hammer2_vfs_modifying(struct mount *mp);
 
-static int hammer2_install_volume_header(hammer2_dev_t *hmp);
-
 static void hammer2_update_pmps(hammer2_dev_t *hmp);
 
 static void hammer2_mount_helper(struct mount *mp, hammer2_pfs_t *pmp);
@@ -939,24 +937,24 @@ hammer2_vfs_mount(struct mount *mp, char *path, caddr_t data,
 	struct hammer2_mount_info info;
 	hammer2_pfs_t *pmp;
 	hammer2_pfs_t *spmp;
-	hammer2_dev_t *hmp;
+	hammer2_dev_t *hmp, *hmp_tmp;
 	hammer2_dev_t *force_local;
 	hammer2_key_t key_next;
 	hammer2_key_t key_dummy;
 	hammer2_key_t lhc;
-	struct vnode *devvp;
-	struct nlookupdata nd;
 	hammer2_chain_t *parent;
 	hammer2_chain_t *chain;
 	const hammer2_inode_data_t *ripdata;
 	hammer2_blockref_t bref;
+	hammer2_devvp_list_t devvpl;
+	hammer2_devvp_t *e, *e_tmp;
 	struct file *fp;
 	char devstr[MNAMELEN];
 	size_t size;
 	size_t done;
 	char *dev;
 	char *label;
-	int ronly = 1;
+	int ronly = ((mp->mnt_flag & MNT_RDONLY) != 0);
 	int error;
 	int i;
 
@@ -964,7 +962,6 @@ hammer2_vfs_mount(struct mount *mp, char *path, caddr_t data,
 	pmp = NULL;
 	dev = NULL;
 	label = NULL;
-	devvp = NULL;
 	bzero(&info, sizeof(info));
 
 	if (path) {
@@ -994,9 +991,7 @@ hammer2_vfs_mount(struct mount *mp, char *path, caddr_t data,
 			if (cluster->array[i].chain == NULL)
 				continue;
 			hmp = cluster->array[i].chain->hmp;
-			devvp = hmp->devvp;
-			error = hammer2_remount(hmp, mp, path,
-						devvp, cred);
+			error = hammer2_remount(hmp, mp, path, cred);
 			if (error)
 				break;
 		}
@@ -1064,50 +1059,17 @@ hammer2_vfs_mount(struct mount *mp, char *path, caddr_t data,
 	}
 
 	kprintf("hammer2_mount: dev=\"%s\" label=\"%s\" rdonly=%d\n",
-		dev, label, (mp->mnt_flag & MNT_RDONLY));
-
-	/*
-	 * HMP device mount
-	 *
-	 * If a path is specified and dev is not an empty string, lookup the
-	 * name and verify that it referes to a block device.
-	 *
-	 * If a path is specified and dev is an empty string we fall through
-	 * and locate the label in the hmp search.
-	 */
-	if (path && *dev != 0) {
-		error = nlookup_init(&nd, dev, UIO_SYSSPACE, NLC_FOLLOW);
-		if (error == 0)
-			error = nlookup(&nd);
-		if (error == 0)
-			error = cache_vref(&nd.nl_nch, nd.nl_cred, &devvp);
-		nlookup_done(&nd);
-	} else if (path == NULL) {
-		/* root mount */
-		cdev_t cdev = kgetdiskbyname(dev);
-		error = bdevvp(cdev, &devvp);
-		if (error)
-			kprintf("hammer2_mount: cannot find '%s'\n", dev);
-	} else {
-		/*
-		 * We will locate the hmp using the label in the hmp loop.
-		 */
-		error = 0;
-	}
+		dev, label, ronly);
 
 	/*
-	 * Make sure its a block device.  Do not check to see if it is
-	 * already mounted until we determine that its a fresh H2 device.
+	 * Initialize all device vnodes.
 	 */
-	if (error == 0 && devvp) {
-		if (!vn_isdisk(devvp, &error)) {
-			KKASSERT(error);
-			kprintf("hammer2_mount: %s not a block device %d\n",
-				dev, error);
-			vrele(devvp);
-			devvp = NULL;
-			return error;
-		}
+	TAILQ_INIT(&devvpl);
+	error = hammer2_init_devvp(dev, path == NULL, &devvpl);
+	if (error) {
+		kprintf("hammer2: failed to initialize devvp in %s\n", dev);
+		hammer2_cleanup_devvp(&devvpl);
+		return error;
 	}
 
 	/*
@@ -1116,28 +1078,52 @@ hammer2_vfs_mount(struct mount *mp, char *path, caddr_t data,
 	 * hammer2 mounts from the same device.
 	 */
 	lockmgr(&hammer2_mntlk, LK_EXCLUSIVE);
-	if (devvp) {
+	if (!TAILQ_EMPTY(&devvpl)) {
 		/*
 		 * Match the device.  Due to the way devfs works,
 		 * we may not be able to directly match the vnode pointer,
 		 * so also check to see if the underlying device matches.
 		 */
-		TAILQ_FOREACH(hmp, &hammer2_mntlist, mntentry) {
-			if (hmp->devvp == devvp)
-				break;
-			if (devvp->v_rdev &&
-			    hmp->devvp->v_rdev == devvp->v_rdev) {
-				break;
+		TAILQ_FOREACH(hmp_tmp, &hammer2_mntlist, mntentry) {
+			TAILQ_FOREACH(e_tmp, &hmp_tmp->devvpl, entry) {
+				int devvp_found = 0;
+				TAILQ_FOREACH(e, &devvpl, entry) {
+					KKASSERT(e->devvp);
+					if (e_tmp->devvp == e->devvp)
+						devvp_found = 1;
+					if (e_tmp->devvp->v_rdev &&
+					    e_tmp->devvp->v_rdev == e->devvp->v_rdev)
+						devvp_found = 1;
+				}
+				if (!devvp_found)
+					goto next_hmp;
 			}
+			hmp = hmp_tmp;
+			kprintf("hammer2_mount: hmp=%p matched\n", hmp);
+			break;
+next_hmp:
+			continue;
 		}
 
 		/*
 		 * If no match this may be a fresh H2 mount, make sure
 		 * the device is not mounted on anything else.
 		 */
-		if (hmp == NULL)
-			error = vfs_mountedon(devvp);
-	} else if (error == 0) {
+		if (hmp == NULL) {
+			TAILQ_FOREACH(e, &devvpl, entry) {
+				struct vnode *devvp = e->devvp;
+				KKASSERT(devvp);
+				error = vfs_mountedon(devvp);
+				if (error) {
+					kprintf("hammer2_mount: %s mounted %d\n",
+						e->path, error);
+					hammer2_cleanup_devvp(&devvpl);
+					lockmgr(&hammer2_mntlk, LK_RELEASE);
+					return error;
+				}
+			}
+		}
+	} else {
 		/*
 		 * Match the label to a pmp already probed.
 		 */
@@ -1153,9 +1139,10 @@ hammer2_vfs_mount(struct mount *mp, char *path, caddr_t data,
 				break;
 		}
 		if (hmp == NULL) {
-			lockmgr(&hammer2_mntlk, LK_RELEASE);
 			kprintf("hammer2_mount: PFS label \"%s\" not found\n",
 				label);
+			hammer2_cleanup_devvp(&devvpl);
+			lockmgr(&hammer2_mntlk, LK_RELEASE);
 			return ENOENT;
 		}
 	}
@@ -1169,37 +1156,44 @@ hammer2_vfs_mount(struct mount *mp, char *path, caddr_t data,
 		hammer2_xid_t xid;
 		hammer2_xop_head_t xop;
 
-		if (error == 0 && vcount(devvp) > 0) {
-			kprintf("hammer2_mount: Primary device already has references\n");
-			error = EBUSY;
-		}
-
 		/*
 		 * Now open the device
 		 */
+		KKASSERT(!TAILQ_EMPTY(&devvpl));
 		if (error == 0) {
-			ronly = ((mp->mnt_flag & MNT_RDONLY) != 0);
-			vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
-			error = vinvalbuf(devvp, V_SAVE, 0, 0);
-			if (error == 0) {
-				error = VOP_OPEN(devvp,
-					     (ronly ? FREAD : FREAD | FWRITE),
-					     FSCRED, NULL);
+			error = hammer2_open_devvp(&devvpl, ronly);
+			if (error) {
+				hammer2_close_devvp(&devvpl, ronly);
+				hammer2_cleanup_devvp(&devvpl);
+				lockmgr(&hammer2_mntlk, LK_RELEASE);
+				return error;
 			}
-			vn_unlock(devvp);
-		}
-		if (error && devvp) {
-			vrele(devvp);
-			devvp = NULL;
 		}
+
+		/*
+		 * Construct volumes and link with device vnodes.
+		 */
+		hmp = kmalloc(sizeof(*hmp), M_HAMMER2, M_WAITOK | M_ZERO);
+		hmp->devvp = NULL;
+		error = hammer2_init_volumes(mp, &devvpl, hmp->volumes,
+					     &hmp->voldata, &hmp->devvp);
 		if (error) {
+			hammer2_close_devvp(&devvpl, ronly);
+			hammer2_cleanup_devvp(&devvpl);
 			lockmgr(&hammer2_mntlk, LK_RELEASE);
+			kfree(hmp, M_HAMMER2);
 			return error;
 		}
-		hmp = kmalloc(sizeof(*hmp), M_HAMMER2, M_WAITOK | M_ZERO);
+		if (!hmp->devvp) {
+			kprintf("hammer2: failed to initialize root volume\n");
+			hammer2_unmount_helper(mp, NULL, hmp);
+			lockmgr(&hammer2_mntlk, LK_RELEASE);
+			hammer2_vfs_unmount(mp, MNT_FORCE);
+			return EINVAL;
+		}
+
 		ksnprintf(hmp->devrepname, sizeof(hmp->devrepname), "%s", dev);
 		hmp->ronly = ronly;
-		hmp->devvp = devvp;
 		hmp->hflags = info.hflags & HMNT2_DEVFLAGS;
 		kmalloc_create(&hmp->mchain, "HAMMER2-chains");
 		TAILQ_INSERT_TAIL(&hammer2_mntlist, hmp, mntentry);
@@ -1247,16 +1241,36 @@ hammer2_vfs_mount(struct mount *mp, char *path, caddr_t data,
 		hammer2_chain_core_init(&hmp->fchain);
 
 		/*
-		 * Install the volume header and initialize fields from
-		 * voldata.
+		 * Initialize volume header related fields.
 		 */
-		error = hammer2_install_volume_header(hmp);
-		if (error) {
-			hammer2_unmount_helper(mp, NULL, hmp);
-			lockmgr(&hammer2_mntlk, LK_RELEASE);
-			hammer2_vfs_unmount(mp, MNT_FORCE);
-			return error;
+		KKASSERT(hmp->voldata.magic == HAMMER2_VOLUME_ID_HBO ||
+			 hmp->voldata.magic == HAMMER2_VOLUME_ID_ABO);
+		hmp->volhdrno = error;
+		hmp->volsync = hmp->voldata;
+		hmp->free_reserved = hmp->voldata.allocator_size / 20;
+		/*
+		 * Must use hmp instead of volume header for these two
+		 * in order to handle volume versions transparently.
+		 */
+		if (hmp->voldata.version >= HAMMER2_VOL_VERSION_MULTI_VOLUMES) {
+			hmp->nvolumes = hmp->voldata.nvolumes;
+			hmp->total_size = hmp->voldata.total_size;
+		} else {
+			hmp->nvolumes = 1;
+			hmp->total_size = hmp->voldata.volu_size;
 		}
+		KKASSERT(hmp->nvolumes > 0);
+
+		/*
+		 * Move devvpl entries to hmp.
+		 */
+		TAILQ_INIT(&hmp->devvpl);
+		while ((e = TAILQ_FIRST(&devvpl)) != NULL) {
+			TAILQ_REMOVE(&devvpl, e, entry);
+			TAILQ_INSERT_TAIL(&hmp->devvpl, e, entry);
+		}
+		KKASSERT(TAILQ_EMPTY(&devvpl));
+		KKASSERT(!TAILQ_EMPTY(&hmp->devvpl));
 
 		/*
 		 * Really important to get these right or the flush and
@@ -1347,7 +1361,7 @@ hammer2_vfs_mount(struct mount *mp, char *path, caddr_t data,
 		schain = NULL;	/* now invalid */
 		/* leave spmp->iroot with one ref */
 
-		if ((mp->mnt_flag & MNT_RDONLY) == 0) {
+		if (!hmp->ronly) {
 			error = hammer2_recovery(hmp);
 			if (error == 0)
 				error |= hammer2_fixup_pfses(hmp);
@@ -1491,8 +1505,6 @@ hammer2_vfs_mount(struct mount *mp, char *path, caddr_t data,
 	 * Connect up mount pointers.
 	 */
 	hammer2_mount_helper(mp, pmp);
-	hmp->devvp->v_rdev->si_mountpoint = mp;
-
         lockmgr(&hammer2_mntlk, LK_RELEASE);
 
 	/*
@@ -1588,29 +1600,43 @@ hammer2_update_pmps(hammer2_dev_t *hmp)
 static
 int
 hammer2_remount(hammer2_dev_t *hmp, struct mount *mp, char *path __unused,
-		struct vnode *devvp, struct ucred *cred)
+		struct ucred *cred)
 {
-	int error;
+	hammer2_volume_t *vol;
+	struct vnode *devvp;
+	int i, error, result = 0;
+
+	if (!(hmp->ronly && (mp->mnt_kern_flag & MNTK_WANTRDWR)))
+		return 0;
 
-	if (hmp->ronly && (mp->mnt_kern_flag & MNTK_WANTRDWR)) {
+	for (i = 0; i < hmp->nvolumes; ++i) {
+		vol = &hmp->volumes[i];
+		devvp = vol->dev->devvp;
+		KKASSERT(devvp);
 		vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
 		VOP_OPEN(devvp, FREAD | FWRITE, FSCRED, NULL);
 		vn_unlock(devvp);
-		error = hammer2_recovery(hmp);
-		if (error == 0)
-			error |= hammer2_fixup_pfses(hmp);
+		error = 0;
+		if (vol->id == HAMMER2_ROOT_VOLUME) {
+			error = hammer2_recovery(hmp);
+			if (error == 0)
+				error |= hammer2_fixup_pfses(hmp);
+		}
 		vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
 		if (error == 0) {
 			VOP_CLOSE(devvp, FREAD, NULL);
-			hmp->ronly = 0;
 		} else {
 			VOP_CLOSE(devvp, FREAD | FWRITE, NULL);
 		}
 		vn_unlock(devvp);
-	} else {
-		error = 0;
+		result |= error;
 	}
-	return error;
+	if (result == 0) {
+		kprintf("hammer2: enable read/write\n");
+		hmp->ronly = 0;
+	}
+
+	return result;
 }
 
 static
@@ -1717,9 +1743,7 @@ hammer2_unmount_helper(struct mount *mp, hammer2_pfs_t *pmp, hammer2_dev_t *hmp)
 {
 	hammer2_cluster_t *cluster;
 	hammer2_chain_t *rchain;
-	struct vnode *devvp;
 	int dumpcnt;
-	int ronly;
 	int i;
 
 	/*
@@ -1823,22 +1847,11 @@ hammer2_unmount_helper(struct mount *mp, hammer2_pfs_t *pmp, hammer2_dev_t *hmp)
 	/*
 	 * Finish up with the device vnode
 	 */
-	if ((devvp = hmp->devvp) != NULL) {
-		ronly = hmp->ronly;
-		vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
-		kprintf("hammer2_unmount(A): devvp %s rbdirty %p ronly=%d\n",
-			hmp->devrepname, RB_ROOT(&devvp->v_rbdirty_tree),
-			ronly);
-		vinvalbuf(devvp, (ronly ? 0 : V_SAVE), 0, 0);
-		kprintf("hammer2_unmount(B): devvp %s rbdirty %p\n",
-			hmp->devrepname, RB_ROOT(&devvp->v_rbdirty_tree));
-		devvp->v_rdev->si_mountpoint = NULL;
-		hmp->devvp = NULL;
-		VOP_CLOSE(devvp, (ronly ? FREAD : FREAD|FWRITE), NULL);
-		vn_unlock(devvp);
-		vrele(devvp);
-		devvp = NULL;
+	if (!TAILQ_EMPTY(&hmp->devvpl)) {
+		hammer2_close_devvp(&hmp->devvpl, hmp->ronly);
+		hammer2_cleanup_devvp(&hmp->devvpl);
 	}
+	KKASSERT(TAILQ_EMPTY(&hmp->devvpl));
 
 	/*
 	 * Clear vchain/fchain flags that might prevent final cleanup
@@ -2905,99 +2918,6 @@ hammer2_vfs_checkexp(struct mount *mp, struct sockaddr *nam,
 	return error;
 }
 
-/*
- * Support code for hammer2_vfs_mount().  Read, verify, and install the volume
- * header into the HMP
- */
-static
-int
-hammer2_install_volume_header(hammer2_dev_t *hmp)
-{
-	hammer2_volume_data_t *vd;
-	struct buf *bp;
-	hammer2_crc32_t crc0, crc, bcrc0, bcrc;
-	int error_reported;
-	int error;
-	int valid;
-	int i;
-
-	error_reported = 0;
-	error = 0;
-	valid = 0;
-	bp = NULL;
-
-	/*
-	 * There are up to 4 copies of the volume header (syncs iterate
-	 * between them so there is no single master).  We don't trust the
-	 * volu_size field so we don't know precisely how large the filesystem
-	 * is, so depend on the OS to return an error if we go beyond the
-	 * block device's EOF.
-	 */
-	for (i = 0; i < HAMMER2_NUM_VOLHDRS; i++) {
-		error = bread(hmp->devvp, i * HAMMER2_ZONE_BYTES64,
-			      HAMMER2_VOLUME_BYTES, &bp);
-		if (error) {
-			brelse(bp);
-			bp = NULL;
-			continue;
-		}
-
-		vd = (struct hammer2_volume_data *) bp->b_data;
-		if ((vd->magic != HAMMER2_VOLUME_ID_HBO) &&
-		    (vd->magic != HAMMER2_VOLUME_ID_ABO)) {
-			kprintf("hammer2: volume header #%d: bad magic\n", i);
-			brelse(bp);
-			bp = NULL;
-			continue;
-		}
-
-		if (vd->magic == HAMMER2_VOLUME_ID_ABO) {
-			/* XXX: Reversed-endianness filesystem */
-			kprintf("hammer2: volume header #%d: reverse-endian "
-				"filesystem detected\n", i);
-			brelse(bp);
-			bp = NULL;
-			continue;
-		}
-
-		crc = vd->icrc_sects[HAMMER2_VOL_ICRC_SECT0];
-		crc0 = hammer2_icrc32(bp->b_data + HAMMER2_VOLUME_ICRC0_OFF,
-				      HAMMER2_VOLUME_ICRC0_SIZE);
-		bcrc = vd->icrc_sects[HAMMER2_VOL_ICRC_SECT1];
-		bcrc0 = hammer2_icrc32(bp->b_data + HAMMER2_VOLUME_ICRC1_OFF,
-				       HAMMER2_VOLUME_ICRC1_SIZE);
-		if ((crc0 != crc) || (bcrc0 != bcrc)) {
-			kprintf("hammer2: volume header #%d: volume header crc "
-				"mismatch %08x/%08x\n",
-				i, crc0, crc);
-			error_reported = 1;
-			brelse(bp);
-			bp = NULL;
-			continue;
-		}
-		if (valid == 0 || hmp->voldata.mirror_tid < vd->mirror_tid) {
-			valid = 1;
-			hmp->voldata = *vd;
-			hmp->volhdrno = i;
-		}
-		brelse(bp);
-		bp = NULL;
-	}
-	if (valid) {
-		hmp->volsync = hmp->voldata;
-		hmp->free_reserved = hmp->voldata.allocator_size / 20;
-		error = 0;
-		if (error_reported || bootverbose || 1) { /* 1/DEBUG */
-			kprintf("hammer2: using volume header #%d\n",
-				hmp->volhdrno);
-		}
-	} else {
-		error = EINVAL;
-		kprintf("hammer2: no valid volume headers found!\n");
-	}
-	return (error);
-}
-
 /*
  * This handles hysteresis on regular file flushes.  Because the BIOs are
  * routed to a thread it is possible for an excessive number to build up
diff --git a/usr.sbin/fstyp/fstyp.c b/usr.sbin/fstyp/fstyp.c
index de9c624c25..ec24b31c9d 100644
--- a/usr.sbin/fstyp/fstyp.c
+++ b/usr.sbin/fstyp/fstyp.c
@@ -81,6 +81,8 @@ static struct {
 } fsvtypes[] = {
 	{ "hammer", &fsvtyp_hammer, false, NULL }, /* Must be before partial */
 	{ "hammer(partial)", &fsvtyp_hammer_partial, true, NULL },
+	{ "hammer2", &fsvtyp_hammer2, false, NULL }, /* Must be before partial */
+	{ "hammer2(partial)", &fsvtyp_hammer2_partial, true, NULL },
 	{ NULL, NULL, NULL, NULL }
 };
 
diff --git a/usr.sbin/fstyp/fstyp.h b/usr.sbin/fstyp/fstyp.h
index b2bc1eac96..1cb9b64e79 100644
--- a/usr.sbin/fstyp/fstyp.h
+++ b/usr.sbin/fstyp/fstyp.h
@@ -65,5 +65,7 @@ int	fstyp_hammer2(FILE *fp, char *label, size_t size, const char *devpath);
 
 int	fsvtyp_hammer(const char *blkdevs, char *label, size_t size);
 int	fsvtyp_hammer_partial(const char *blkdevs, char *label, size_t size);
+int	fsvtyp_hammer2(const char *blkdevs, char *label, size_t size);
+int	fsvtyp_hammer2_partial(const char *blkdevs, char *label, size_t size);
 
 #endif /* !FSTYP_H */
diff --git a/usr.sbin/fstyp/hammer2.c b/usr.sbin/fstyp/hammer2.c
index 5edc8ba854..9ffbaa35df 100644
--- a/usr.sbin/fstyp/hammer2.c
+++ b/usr.sbin/fstyp/hammer2.c
@@ -31,6 +31,7 @@
 #include <string.h>
 #include <err.h>
 #include <assert.h>
+#include <uuid.h>
 #include <vfs/hammer2/hammer2_disk.h>
 
 #include "fstyp.h"
@@ -54,27 +55,69 @@ get_file_size(FILE *fp)
 	return (siz);
 }
 
+static hammer2_volume_data_t *
+read_voldata(FILE *fp, int i)
+{
+	if (i < 0 || i >= HAMMER2_NUM_VOLHDRS)
+		return (NULL);
+
+	if (i * HAMMER2_ZONE_BYTES64 >= get_file_size(fp))
+		return (NULL);
+
+	return (read_buf(fp, i * HAMMER2_ZONE_BYTES64,
+	    sizeof(hammer2_volume_data_t)));
+}
+
 static int
 test_voldata(FILE *fp)
 {
 	hammer2_volume_data_t *voldata;
-	int i, fail = 0;
+	int i;
+	static int count = 0;
+	static uuid_t fsid, fstype;
 
 	for (i = 0; i < HAMMER2_NUM_VOLHDRS; i++) {
 		if (i * HAMMER2_ZONE_BYTES64 >= get_file_size(fp))
 			break;
-		voldata = read_buf(fp, i * HAMMER2_ZONE_BYTES64,
-		    sizeof(*voldata));
+		voldata = read_voldata(fp, i);
 		if (voldata == NULL) {
 			warnx("hammer2: failed to read volume data");
 			return (1);
 		}
 		if (voldata->magic != HAMMER2_VOLUME_ID_HBO &&
-		    voldata->magic != HAMMER2_VOLUME_ID_ABO)
-			fail++;
+		    voldata->magic != HAMMER2_VOLUME_ID_ABO) {
+			free(voldata);
+			return (1);
+		}
+		if (voldata->volu_id > HAMMER2_MAX_VOLUMES - 1) {
+			free(voldata);
+			return (1);
+		}
+		if (voldata->nvolumes > HAMMER2_MAX_VOLUMES) {
+			free(voldata);
+			return (1);
+		}
+
+		if (count == 0) {
+			count = voldata->nvolumes;
+			memcpy(&fsid, &voldata->fsid, sizeof(fsid));
+			memcpy(&fstype, &voldata->fstype, sizeof(fstype));
+		} else {
+			if (voldata->nvolumes != count) {
+				free(voldata);
+				return (1);
+			}
+			if (!uuid_equal(&fsid, &voldata->fsid, NULL)) {
+				free(voldata);
+				return (1);
+			}
+			if (!uuid_equal(&fstype, &voldata->fstype, NULL)) {
+				free(voldata);
+				return (1);
+			}
+		}
+		free(voldata);
 	}
-	if (fail)
-		return (1);
 
 	return (0);
 }
@@ -84,7 +127,7 @@ read_media(FILE *fp, const hammer2_blockref_t *bref, size_t *media_bytes)
 {
 	hammer2_media_data_t *media;
 	hammer2_off_t io_off, io_base;
-	size_t bytes, io_bytes, boff;
+	size_t bytes, io_bytes, boff, fbytes;
 
 	bytes = (bref->data_off & HAMMER2_OFF_MASK_RADIX);
 	if (bytes)
@@ -109,6 +152,20 @@ read_media(FILE *fp, const hammer2_blockref_t *bref, size_t *media_bytes)
 		return (NULL);
 	}
 
+	/*
+	 * XXX fp is currently always root volume, so read fails if io_base is
+	 * beyond root volume limit. Fail with a message before read_buf() then.
+	 */
+	fbytes = get_file_size(fp);
+	if (fbytes == -1) {
+		warnx("hammer2: failed to get media size");
+		return (NULL);
+	}
+	if (io_base >= fbytes) {
+		warnx("hammer2: XXX read beyond HAMMER2 root volume limit unsupported");
+		return (NULL);
+	}
+
 	if (fseek(fp, io_base, SEEK_SET) == -1) {
 		warnx("hammer2: failed to seek media");
 		return (NULL);
@@ -236,8 +293,7 @@ read_label(FILE *fp, char *label, size_t size, const char *devpath)
 		memset(&broot, 0, sizeof(broot));
 		broot.type = HAMMER2_BREF_TYPE_VOLUME;
 		broot.data_off = (i * HAMMER2_ZONE_BYTES64) | HAMMER2_PBUFRADIX;
-		vols[i] = read_buf(fp, broot.data_off & ~HAMMER2_OFF_MASK_RADIX,
-		    sizeof(*vols[i]));
+		vols[i] = (void*)read_voldata(fp, i);
 		if (vols[i] == NULL) {
 			warnx("hammer2: failed to read volume data");
 			goto fail;
@@ -251,7 +307,7 @@ read_label(FILE *fp, char *label, size_t size, const char *devpath)
 
 	bref = &vols[best_i]->voldata.sroot_blockset.blockref[0];
 	if (bref->type != HAMMER2_BREF_TYPE_INODE) {
-		warnx("hammer2: blockref type is not inode");
+		/* Don't print error as devpath could be non-root volume. */
 		goto fail;
 	}
 
@@ -321,8 +377,132 @@ read_label(FILE *fp, char *label, size_t size, const char *devpath)
 int
 fstyp_hammer2(FILE *fp, char *label, size_t size, const char *devpath)
 {
+	hammer2_volume_data_t *voldata = read_voldata(fp, 0);
+	int error = 1;
+
+	if (voldata->volu_id != HAMMER2_ROOT_VOLUME)
+		goto fail;
+	if (voldata->nvolumes != 0)
+		goto fail;
 	if (test_voldata(fp))
-		return (1);
+		goto fail;
+
+	error = read_label(fp, label, size, devpath);
+fail:
+	free(voldata);
+	return (error);
+}
+
+static int
+__fsvtyp_hammer2(const char *blkdevs, char *label, size_t size, int partial)
+{
+	hammer2_volume_data_t *voldata = NULL;
+	FILE *fp = NULL;
+	char *dup = NULL, *target_label = NULL, *p, *volpath, *rootvolpath;
+	char x[HAMMER2_MAX_VOLUMES];
+	int i, volid, error = 1;
+
+	if (!blkdevs)
+		goto fail;
+
+	memset(x, 0, sizeof(x));
+	p = dup = strdup(blkdevs);
+	if ((p = strchr(p, '@')) != NULL) {
+		*p++ = '\0';
+		target_label = p;
+	}
+	p = dup;
+
+	volpath = NULL;
+	rootvolpath = NULL;
+	volid = -1;
+	while (p) {
+		volpath = p;
+		if ((p = strchr(p, ':')) != NULL)
+			*p++ = '\0';
+		if ((fp = fopen(volpath, "r")) == NULL) {
+			warnx("hammer2: failed to open %s", volpath);
+			goto fail;
+		}
+		if (test_voldata(fp))
+			break;
+		voldata = read_voldata(fp, 0);
+		fclose(fp);
+		if (voldata == NULL) {
+			warnx("hammer2: failed to read volume data");
+			goto fail;
+		}
+		volid = voldata->volu_id;
+		free(voldata);
+		voldata = NULL;
+		assert(volid >= 0);
+		assert(volid < HAMMER2_MAX_VOLUMES);
+		x[volid]++;
+		if (volid == HAMMER2_ROOT_VOLUME)
+			rootvolpath = volpath;
+	}
+
+	/* If no rootvolpath, proceed only if partial mode with volpath. */
+	if (rootvolpath)
+		volpath = rootvolpath;
+	else if (!partial || !volpath)
+		goto fail;
+	if ((fp = fopen(volpath, "r")) == NULL) {
+		warnx("hammer2: failed to open %s", volpath);
+		goto fail;
+	}
+	voldata = read_voldata(fp, 0);
+	if (voldata == NULL) {
+		warnx("hammer2: failed to read volume data");
+		goto fail;
+	}
+
+	if (volid == -1)
+		goto fail;
+	if (partial)
+		goto success;
+
+	for (i = 0; i < HAMMER2_MAX_VOLUMES; i++)
+		if (x[i] > 1)
+			goto fail;
+	for (i = 0; i < HAMMER2_MAX_VOLUMES; i++)
+		if (x[i] == 0)
+			break;
+	if (voldata->nvolumes != i)
+		goto fail;
+	for (; i < HAMMER2_MAX_VOLUMES; i++)
+		if (x[i] != 0)
+			goto fail;
+success:
+	/* Reconstruct @label format path using only root volume. */
+	if (target_label) {
+		int siz = strlen(volpath) + strlen(target_label) + 2;
+		p = calloc(1, siz);
+		snprintf(p, siz, "%s@%s", volpath, target_label);
+		volpath = p;
+	}
+	error = read_label(fp, label, size, volpath);
+	if (target_label)
+		free(p);
+	/* If in partial mode, read label but ignore error. */
+	if (partial)
+		error = 0;
+fail:
+	if (fp)
+		fclose(fp);
+	free(voldata);
+	free(dup);
+	return (error);
+}
 
-	return (read_label(fp, label, size, devpath));
+int
+fsvtyp_hammer2(const char *blkdevs, char *label, size_t size)
+{
+	return (__fsvtyp_hammer2(blkdevs, label, size, 0));
+}
+
+int
+fsvtyp_hammer2_partial(const char *blkdevs, char *label, size_t size)
+{
+	return (__fsvtyp_hammer2(blkdevs, label, size, 1));
 }
-- 
2.28.0

