FAQ Search Today's Posts Mark Forums Read
» Video Reviews

» Linux Archive

Linux-archive is a website aiming to archive linux email lists and to make them easily accessible for linux users/developers.


» Sponsor

» Partners

» Sponsor

Go Back   Linux Archive > Redhat > Device-mapper Development

 
 
LinkBack Thread Tools
 
Old 03-04-2012, 06:18 PM
Mikulas Patocka
 
Default dm: remake of the verity target

Hi

Here I'm posting remake of the dm-verity target originall developed by
Mandeep Singh Baines. It has a compatible target line syntax, so it can be
used as a drop-in replacement.

The major difference is that this driver uses dm-bufio to manage cache of
hash blocks in memory. We talked with Alasdair about using dm-bufio for
caching and I concluded that it is simpler to rewrite the code rather than
transform the original Google code with patches.

Because of dm-bufio, memory consumption is not dependent on device size
(if the system starts running out of memory, dm-bufio discards cached
blocks loaded in memory).

This implementation is smaller, because (unlike the original
implementation), it doesn't create persistent in-memory structures.

This implementation is faster than the original. It uses clustered
prefetch to prefetch several hash blocks at once, greatly improving
performance if the data partition and hash partition are located on the
same disk (the prefetch cluster can be set with prefetch_cluster module
parameter).

Mikulas

---

Disk: Maxtor Atlas 15k2 146GB, 3.8GB partition

The following tests were made:
dd if=/dev/mapper/verity of=/dev/null bs=1M
fsck.ext2 -fn -C 0 /dev/mapper/verity
fio --rw=randread --size=200M --bsrange=1k-128k --filename=/dev/mapper/verity --name=job1 --name=job2 --name=job3 --name=job4

raw partition:
dd: 42s
fsck: 11s
fio: 13s

original google dm-verity implementation:
dd (first time): 571s
dd (next time): 46s
fsck (first time): 39s
fsck (next time): 10s
fio (first time): 26s
fio (next time): 24s

my vm-verity implementation:
dd (first time): 45s
dd (next time): 43s
fsck (first time): 11s
fsck (next time): 11s
fio (first time): 14s
fio (next time): 14s

---

Remake of the google dm-verity patch.

Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>

---
drivers/md/Kconfig | 17 +
drivers/md/Makefile | 1
drivers/md/dm-bufio.c | 97 ++++--
drivers/md/dm-bufio.h | 8
drivers/md/dm-verity.c | 785 +++++++++++++++++++++++++++++++++++++++++++++++++
5 files changed, 887 insertions(+), 21 deletions(-)

Index: linux-3.3-rc5-fast/drivers/md/Kconfig
================================================== =================
--- linux-3.3-rc5-fast.orig/drivers/md/Kconfig 2012-03-03 05:19:33.000000000 +0100
+++ linux-3.3-rc5-fast/drivers/md/Kconfig 2012-03-03 19:39:30.000000000 +0100
@@ -388,4 +388,21 @@ config DM_FLAKEY
---help---
A target that intermittently fails I/O for debugging purposes.

+config DM_VERITY
+ tristate "Verity target support"
+ depends on BLK_DEV_DM
+ select CRYPTO
+ select CRYPTO_HASH
+ select DM_BUFIO
+ ---help---
+ This device-mapper target allows you to create a device that
+ transparently integrity checks the data on it. You'll need to
+ activate the digests you're going to use in the cryptoapi
+ configuration.
+
+ To compile this code as a module, choose M here: the module will
+ be called dm-verity.
+
+ If unsure, say N.
+
endif # MD
Index: linux-3.3-rc5-fast/drivers/md/Makefile
================================================== =================
--- linux-3.3-rc5-fast.orig/drivers/md/Makefile 2012-03-03 05:19:33.000000000 +0100
+++ linux-3.3-rc5-fast/drivers/md/Makefile 2012-03-03 19:39:30.000000000 +0100
@@ -29,6 +29,7 @@ obj-$(CONFIG_MD_FAULTY) += faulty.o
obj-$(CONFIG_BLK_DEV_MD) += md-mod.o
obj-$(CONFIG_BLK_DEV_DM) += dm-mod.o
obj-$(CONFIG_DM_BUFIO) += dm-bufio.o
+obj-$(CONFIG_DM_VERITY) += dm-verity.o
obj-$(CONFIG_DM_CRYPT) += dm-crypt.o
obj-$(CONFIG_DM_DELAY) += dm-delay.o
obj-$(CONFIG_DM_FLAKEY) += dm-flakey.o
Index: linux-3.3-rc5-fast/drivers/md/dm-verity.c
================================================== =================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux-3.3-rc5-fast/drivers/md/dm-verity.c 2012-03-03 05:51:10.000000000 +0100
@@ -0,0 +1,785 @@
+/*
+ * Copyright (C) 2012 Red Hat, Inc.
+ *
+ * Author: Mikulas Patocka <mpatocka@redhat.com>
+ *
+ * Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors
+ *
+ * This file is released under the GPLv2.
+ *
+ * Device mapper target parameters:
+ * <version> 0
+ * <data device>
+ * <hash device>
+ * <hash start> (typically 0)
+ * <block size> (typically 4096)
+ * <algorithm>
+ * <digest>
+ * optional parameters:
+ * <salt> (should have 32 bytes for compatibility with Google code)
+ * <hash block size> (by default it is the same as data block size)
+ *
+ * In the file "/sys/module/dm_verity/parameters/prefetch_cluster" you can set
+ * default prefetch value. Data are read in "prefetch_cluster" chunks from the
+ * hash device. Prefetch cluster greatly improves performance when data and hash
+ * are on the same disk on different partitions.
+ */
+
+#include <linux/module.h>
+#include <linux/device-mapper.h>
+#include <crypto/hash.h>
+#include "dm-bufio.h"
+
+#define DM_MSG_PREFIX "verity"
+
+#define DM_VERITY_IO_VEC_INLINE 16
+#define DM_VERITY_MEMPOOL_SIZE 4
+#define DM_VERITY_PREFETCH_SIZE 262144
+
+#define DM_VERITY_MAX_LEVELS 63
+
+static unsigned prefetch_cluster = DM_VERITY_PREFETCH_SIZE;
+
+module_param_named(prefetch_cluster, prefetch_cluster, uint, S_IRUGO | S_IWUSR);
+
+struct dm_verity {
+ struct dm_dev *data_dev;
+ struct dm_dev *hash_dev;
+ struct dm_target *ti;
+ struct dm_bufio_client *bufio;
+ char *alg_name;
+ struct crypto_shash *tfm;
+ u8 *root_digest;
+ u8 *salt;
+ unsigned salt_size;
+ sector_t data_start;
+ sector_t hash_start;
+ sector_t data_blocks;
+ sector_t hash_blocks;
+ unsigned char data_dev_block_bits;
+ unsigned char hash_dev_block_bits;
+ unsigned char hash_per_block_bits;
+ unsigned char levels;
+ unsigned digest_size;
+ unsigned shash_descsize;
+
+ mempool_t *io_mempool;
+ mempool_t *vec_mempool;
+
+ struct workqueue_struct *verify_wq;
+
+ sector_t hash_level_block[DM_VERITY_MAX_LEVELS];
+};
+
+struct dm_verity_io {
+ struct dm_verity *v;
+ struct bio *bio;
+ bio_end_io_t *orig_bi_end_io;
+ void *orig_bi_private;
+ sector_t block;
+ unsigned n_blocks;
+ struct bio_vec *io_vec;
+ unsigned io_vec_size;
+ struct work_struct work;
+ struct bio_vec io_vec_inline[DM_VERITY_IO_VEC_INLINE];
+ /* u8 hash_desc[crypto_shash_descsize(v->tfm)]; */
+ /* u8 real_digest[v->digest_size]; */
+ /* u8 want_digest[v->digest_size]; */
+};
+
+static struct shash_desc *io_hash_desc(struct dm_verity *v, struct dm_verity_io *io)
+{
+ return (struct shash_desc *)(io + 1);
+}
+
+static u8 *io_real_digest(struct dm_verity *v, struct dm_verity_io *io)
+{
+ return (u8 *)(io + 1) + v->shash_descsize;
+}
+
+static u8 *io_want_digest(struct dm_verity *v, struct dm_verity_io *io)
+{
+ return (u8 *)(io + 1) + v->shash_descsize + v->digest_size;
+}
+
+struct buffer_aux {
+ int hash_verified;
+};
+
+static void dm_bufio_alloc_callback(struct dm_buffer *buf)
+{
+ struct buffer_aux *aux = dm_bufio_get_aux_data(buf);
+ aux->hash_verified = 0;
+}
+
+static sector_t verity_map_sector(struct dm_verity *v, sector_t bi_sector)
+{
+ return v->data_start + dm_target_offset(v->ti, bi_sector);
+}
+
+static sector_t verity_position_at_level(struct dm_verity *v, sector_t block,
+ int level)
+{
+ return block >> (level * v->hash_per_block_bits);
+}
+
+static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level,
+ sector_t *hash_block, unsigned *offset)
+{
+ sector_t position = verity_position_at_level(v, block, level);
+
+ *hash_block = v->hash_level_block[level] + (position >> v->hash_per_block_bits);
+ if (offset)
+ *offset = v->digest_size * (position & ((1 << v->hash_per_block_bits) - 1));
+}
+
+static int verity_verify_level(struct dm_verity_io *io, sector_t block,
+ int level, int skip_unverified)
+{
+ struct dm_verity *v = io->v;
+ struct dm_buffer *buf;
+ struct buffer_aux *aux;
+ u8 *data;
+ int r;
+ sector_t hash_block;
+ unsigned offset;
+
+ verity_hash_at_level(v, block, level, &hash_block, &offset);
+
+ data = dm_bufio_read(v->bufio, hash_block, &buf);
+ if (unlikely(IS_ERR(data)))
+ return PTR_ERR(data);
+
+ aux = dm_bufio_get_aux_data(buf);
+
+ if (!aux->hash_verified) {
+ struct shash_desc *desc;
+ u8 *result;
+
+ if (skip_unverified) {
+ r = 1;
+ goto release_ret_r;
+ }
+
+ desc = io_hash_desc(v, io);
+ desc->tfm = v->tfm;
+ desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
+ r = crypto_shash_init(desc);
+ if (r < 0) {
+ DMERR("crypto_shash_init failed: %d", r);
+ goto release_ret_r;
+ }
+
+ r = crypto_shash_update(desc, data, 1 << v->hash_dev_block_bits);
+ if (r < 0) {
+ DMERR("crypto_shash_update failed: %d", r);
+ goto release_ret_r;
+ }
+
+ r = crypto_shash_update(desc, v->salt, v->salt_size);
+ if (r < 0) {
+ DMERR("crypto_shash_update failed: %d", r);
+ goto release_ret_r;
+ }
+
+ result = io_real_digest(v, io);
+ r = crypto_shash_final(desc, result);
+ if (r < 0) {
+ DMERR("crypto_shash_final failed: %d", r);
+ goto release_ret_r;
+ }
+ if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) {
+ DMERR_LIMIT("metadata block %llu is corrupted",
+ (unsigned long long)hash_block);
+ r = -EIO;
+ goto release_ret_r;
+ } else
+ aux->hash_verified = 1;
+ }
+
+ data += offset;
+
+ memcpy(io_want_digest(v, io), data, v->digest_size);
+
+ dm_bufio_release(buf);
+ return 0;
+
+release_ret_r:
+ dm_bufio_release(buf);
+ return r;
+}
+
+static int verity_verify_io(struct dm_verity_io *io)
+{
+ struct dm_verity *v = io->v;
+ unsigned b;
+ int i;
+ unsigned vector = 0, offset = 0;
+ for (b = 0; b < io->n_blocks; b++) {
+ struct shash_desc *desc;
+ u8 *result;
+ int r;
+ unsigned todo;
+
+ if (likely(v->levels)) {
+ int r = verity_verify_level(io, io->block + b, 0, 1);
+ if (likely(!r))
+ goto test_block_hash;
+ if (r < 0)
+ return r;
+ }
+
+ memcpy(io_want_digest(v, io), v->root_digest, v->digest_size);
+
+ for (i = v->levels - 1; i >= 0; i--) {
+ int r = verity_verify_level(io, io->block + b, i, 0);
+ if (unlikely(r))
+ return r;
+ }
+
+test_block_hash:
+ desc = io_hash_desc(v, io);
+ desc->tfm = v->tfm;
+ desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
+ r = crypto_shash_init(desc);
+ if (r < 0) {
+ DMERR("crypto_shash_init failed: %d", r);
+ return r;
+ }
+
+ todo = 1 << v->data_dev_block_bits;
+ do {
+ struct bio_vec *bv;
+ u8 *page;
+ unsigned len;
+
+ BUG_ON(vector >= io->io_vec_size);
+ bv = &io->io_vec[vector];
+ page = kmap_atomic(bv->bv_page, KM_USER0);
+ len = bv->bv_len - offset;
+ if (likely(len >= todo))
+ len = todo;
+ r = crypto_shash_update(desc,
+ page + bv->bv_offset + offset, len);
+ kunmap_atomic(page, KM_USER0);
+ if (r < 0) {
+ DMERR("crypto_shash_update failed: %d", r);
+ return r;
+ }
+ offset += len;
+ if (likely(offset == bv->bv_len)) {
+ offset = 0;
+ vector++;
+ }
+ todo -= len;
+ } while (todo);
+
+ r = crypto_shash_update(desc, v->salt, v->salt_size);
+ if (r < 0) {
+ DMERR("crypto_shash_update failed: %d", r);
+ return r;
+ }
+
+ result = io_real_digest(v, io);
+ r = crypto_shash_final(desc, result);
+ if (r < 0) {
+ DMERR("crypto_shash_final failed: %d", r);
+ return r;
+ }
+ if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) {
+ DMERR_LIMIT("data block %llu is corrupted",
+ (unsigned long long)(io->block + b));
+ return -EIO;
+ }
+ }
+ BUG_ON(vector != io->io_vec_size);
+ BUG_ON(offset);
+ return 0;
+}
+
+static void verity_finish_io(struct dm_verity_io *io, int error)
+{
+ struct bio *bio = io->bio;
+ struct dm_verity *v = io->v;
+
+ bio->bi_end_io = io->orig_bi_end_io;
+ bio->bi_private = io->orig_bi_private;
+
+ if (io->io_vec != io->io_vec_inline)
+ mempool_free(io->io_vec, v->vec_mempool);
+ mempool_free(io, v->io_mempool);
+
+ bio_endio(bio, error);
+}
+
+static void verity_work(struct work_struct *w)
+{
+ struct dm_verity_io *io = container_of(w, struct dm_verity_io, work);
+
+ verity_finish_io(io, verity_verify_io(io));
+}
+
+static void verity_end_io(struct bio *bio, int error)
+{
+ struct dm_verity_io *io = bio->bi_private;
+ if (error) {
+ verity_finish_io(io, error);
+ return;
+ }
+
+ INIT_WORK(&io->work, verity_work);
+ queue_work(io->v->verify_wq, &io->work);
+}
+
+static void verity_prefetch_io(struct dm_verity *v, struct dm_verity_io *io)
+{
+ int i;
+ for (i = v->levels - 2; i >= 0; i--) {
+ sector_t hash_block_start;
+ sector_t hash_block_end;
+ verity_hash_at_level(v, io->block, i, &hash_block_start, NULL);
+ verity_hash_at_level(v, io->block + io->n_blocks - 1, i, &hash_block_end, NULL);
+ if (!i) {
+ unsigned cluster = prefetch_cluster;
+ /* barrier to stop GCC from re-reading prefetch_cluster again */
+ barrier();
+ cluster >>= v->data_dev_block_bits;
+ if (unlikely(!cluster))
+ goto no_prefetch_cluster;
+ if (unlikely(cluster & (cluster - 1)))
+ cluster = 1 << (fls(cluster) - 1);
+
+ hash_block_start &= ~(sector_t)(cluster - 1);
+ hash_block_end |= cluster - 1;
+ if (unlikely(hash_block_end >= v->hash_blocks))
+ hash_block_end = v->hash_blocks - 1;
+ }
+no_prefetch_cluster:
+ dm_bufio_prefetch(v->bufio, hash_block_start,
+ hash_block_end - hash_block_start + 1);
+ }
+}
+
+static int verity_map(struct dm_target *ti, struct bio *bio,
+ union map_info *map_context)
+{
+ struct dm_verity *v = ti->private;
+ struct dm_verity_io *io;
+
+ if (((unsigned)bio->bi_sector | bio_sectors(bio)) &
+ ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) {
+ DMERR_LIMIT("unaligned io");
+ return -EIO;
+ }
+
+ if ((bio->bi_sector + bio_sectors(bio)) >>
+ (v->data_dev_block_bits - SECTOR_SHIFT) > v->data_blocks) {
+ DMERR_LIMIT("io out of range");
+ return -EIO;
+ }
+
+ if (bio_data_dir(bio) == WRITE)
+ return -EIO;
+
+ io = mempool_alloc(v->io_mempool, GFP_NOIO);
+ io->v = v;
+ io->bio = bio;
+ io->orig_bi_end_io = bio->bi_end_io;
+ io->orig_bi_private = bio->bi_private;
+ io->block = bio->bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT);
+ io->n_blocks = bio->bi_size >> v->data_dev_block_bits;
+
+ bio->bi_end_io = verity_end_io;
+ bio->bi_private = io;
+ bio->bi_bdev = v->data_dev->bdev;
+ bio->bi_sector = verity_map_sector(v, bio->bi_sector);
+
+ io->io_vec_size = bio->bi_vcnt - bio->bi_idx;
+ if (io->io_vec_size < DM_VERITY_IO_VEC_INLINE)
+ io->io_vec = io->io_vec_inline;
+ else
+ io->io_vec = mempool_alloc(v->vec_mempool, GFP_NOIO);
+ memcpy(io->io_vec, bio_iovec(bio),
+ io->io_vec_size * sizeof(struct bio_vec));
+
+ verity_prefetch_io(v, io);
+
+ generic_make_request(bio);
+
+ return DM_MAPIO_SUBMITTED;
+}
+
+static int verity_status(struct dm_target *ti, status_type_t type,
+ char *result, unsigned maxlen)
+{
+ struct dm_verity *v = ti->private;
+ unsigned sz = 0;
+ unsigned x;
+
+ switch (type) {
+ case STATUSTYPE_INFO:
+ result[0] = 0;
+ break;
+ case STATUSTYPE_TABLE:
+ DMEMIT("%u %s %s %llu %u %s ",
+ 0,
+ v->data_dev->name,
+ v->hash_dev->name,
+ (unsigned long long)v->hash_start << (v->hash_dev_block_bits - SECTOR_SHIFT),
+ 1 << v->data_dev_block_bits,
+ v->alg_name
+ );
+ for (x = 0; x < v->digest_size; x++)
+ DMEMIT("%02x", v->root_digest[x]);
+ DMEMIT(" ");
+ if (!v->salt_size)
+ DMEMIT("-");
+ else
+ for (x = 0; x < v->salt_size; x++)
+ DMEMIT("%02x", v->salt[x]);
+ DMEMIT(" %u", 1 << v->hash_dev_block_bits);
+ break;
+ }
+ return 0;
+}
+
+static int verity_ioctl(struct dm_target *ti, unsigned cmd,
+ unsigned long arg)
+{
+ struct dm_verity *v = ti->private;
+ int r = 0;
+
+ if (v->data_start ||
+ ti->len != i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT)
+ r = scsi_verify_blk_ioctl(NULL, cmd);
+
+ return r ? : __blkdev_driver_ioctl(v->data_dev->bdev, v->data_dev->mode,
+ cmd, arg);
+}
+
+static int verity_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
+ struct bio_vec *biovec, int max_size)
+{
+ struct dm_verity *v = ti->private;
+ struct request_queue *q = bdev_get_queue(v->data_dev->bdev);
+
+ if (!q->merge_bvec_fn)
+ return max_size;
+
+ bvm->bi_bdev = v->data_dev->bdev;
+ bvm->bi_sector = verity_map_sector(v, bvm->bi_sector);
+
+ return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
+}
+
+static int verity_iterate_devices(struct dm_target *ti,
+ iterate_devices_callout_fn fn, void *data)
+{
+ struct dm_verity *v = ti->private;
+ return fn(ti, v->data_dev, v->data_start, ti->len, data);
+}
+
+static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits)
+{
+ struct dm_verity *v = ti->private;
+
+ if (limits->logical_block_size < 1 << v->data_dev_block_bits)
+ limits->logical_block_size = 1 << v->data_dev_block_bits;
+ if (limits->physical_block_size < 1 << v->data_dev_block_bits)
+ limits->physical_block_size = 1 << v->data_dev_block_bits;
+ blk_limits_io_min(limits, limits->logical_block_size);
+}
+
+static void verity_dtr(struct dm_target *ti);
+
+static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv)
+{
+ struct dm_verity *v;
+ unsigned num;
+ unsigned long long hs;
+ int r;
+ int i;
+ sector_t hash_position;
+ char dummy;
+
+ v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL);
+ if (!v) {
+ ti->error = "Cannot allocate verity structure";
+ return -ENOMEM;
+ }
+ ti->private = v;
+ v->ti = ti;
+
+ if ((dm_table_get_mode(ti->table) & ~FMODE_READ) != 0) {
+ ti->error = "Device must be readonly";
+ r = -EINVAL;
+ goto bad;
+ }
+
+ if (argc < 7) {
+ ti->error = "Not enough arguments";
+ r = -EINVAL;
+ goto bad;
+ }
+
+ if (sscanf(argv[0], "%d%c", &num, &dummy) != 1 ||
+ num != 0) {
+ ti->error = "Invalid version";
+ r = -EINVAL;
+ goto bad;
+ }
+
+ r = dm_get_device(ti, argv[1], FMODE_READ, &v->data_dev);
+ if (r) {
+ ti->error = "Data device lookup failed";
+ goto bad;
+ }
+
+ r = dm_get_device(ti, argv[2], FMODE_READ, &v->hash_dev);
+ if (r) {
+ ti->error = "Data device lookup failed";
+ goto bad;
+ }
+
+ if (sscanf(argv[3], "%llu%c", &hs, &dummy) != 1 ||
+ hs != (sector_t)hs) {
+ ti->error = "Invalid hash start";
+ r = -EINVAL;
+ goto bad;
+ }
+
+ if (sscanf(argv[4], "%u%c", &num, &dummy) != 1 ||
+ !num || (num & (num - 1)) ||
+ num < bdev_logical_block_size(v->data_dev->bdev) ||
+ num > PAGE_SIZE) {
+ ti->error = "Invalid data device block size";
+ r = -EINVAL;
+ goto bad;
+ }
+ v->data_dev_block_bits = ffs(num) - 1;
+ v->hash_dev_block_bits = ffs(num) - 1;
+
+ v->alg_name = kstrdup(argv[5], GFP_KERNEL);
+ if (!v->alg_name) {
+ ti->error = "Cannot allocate algorithm name";
+ r = -ENOMEM;
+ goto bad;
+ }
+
+ v->tfm = crypto_alloc_shash(v->alg_name, 0, 0);
+ if (IS_ERR(v->tfm)) {
+ ti->error = "Cannot initialize hash function";
+ r = PTR_ERR(v->tfm);
+ v->tfm = NULL;
+ goto bad;
+ }
+ v->digest_size = crypto_shash_digestsize(v->tfm);
+ if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) {
+ ti->error = "Digest size too big";
+ r = -EINVAL;
+ goto bad;
+ }
+ v->shash_descsize =
+ sizeof(struct shash_desc) + crypto_shash_descsize(v->tfm);
+
+ v->root_digest = kmalloc(v->digest_size, GFP_KERNEL);
+ if (!v->root_digest) {
+ ti->error = "Cannot allocate root digest";
+ r = -ENOMEM;
+ goto bad;
+ }
+ if (strlen(argv[6]) != v->digest_size * 2 ||
+ hex2bin(v->root_digest, argv[6], v->digest_size)) {
+ ti->error = "Invalid root digest";
+ r = -EINVAL;
+ goto bad;
+ }
+
+ if (argc > 7 && strcmp(argv[7], "-")) {
+ v->salt_size = strlen(argv[7]) / 2;
+ v->salt = kmalloc(v->salt_size, GFP_KERNEL);
+ if (!v->salt) {
+ ti->error = "Cannot allocate salt";
+ r = -ENOMEM;
+ goto bad;
+ }
+ if (strlen(argv[7]) != v->salt_size * 2 ||
+ hex2bin(v->salt, argv[7], v->salt_size)) {
+ ti->error = "Invalid salt";
+ r = -EINVAL;
+ goto bad;
+ }
+ }
+
+ if (argc > 8) {
+ if (sscanf(argv[8], "%u%c", &num, &dummy) != 1 ||
+ !num || (num & (num - 1)) ||
+ num < bdev_logical_block_size(v->hash_dev->bdev) ||
+ num > INT_MAX) {
+ ti->error = "Invalid hash device block size";
+ r = -EINVAL;
+ goto bad;
+ }
+ v->hash_dev_block_bits = ffs(num) - 1;
+ }
+
+ if (hs & ((1 << (v->hash_dev_block_bits - SECTOR_SHIFT)) - 1)) {
+ ti->error = "Hash start not aligned on block boundary";
+ r = -EINVAL;
+ goto bad;
+ }
+ v->hash_start = hs >> (v->hash_dev_block_bits - SECTOR_SHIFT);
+
+ if (ti->len > i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT) {
+ ti->error = "Data device si too small";
+ r = -EINVAL;
+ goto bad;
+ }
+
+ if (ti->len & ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) {
+ ti->error = "Data device length is not aligned to block size";
+ r = -EINVAL;
+ goto bad;
+ }
+
+ v->data_blocks = ti->len >> (v->data_dev_block_bits - SECTOR_SHIFT);
+
+ v->hash_per_block_bits =
+ fls((1 << v->hash_dev_block_bits) / v->digest_size) - 1;
+
+ v->levels = 0;
+ if (v->data_blocks)
+ while (v->hash_per_block_bits * v->levels < 64 &&
+ (unsigned long long)(v->data_blocks - 1) >>
+ (v->hash_per_block_bits * v->levels))
+ v->levels++;
+
+ if (v->levels > DM_VERITY_MAX_LEVELS) {
+ ti->error = "Too many tree levels";
+ r = -E2BIG;
+ goto bad;
+ }
+
+ hash_position = v->hash_start;
+ for (i = v->levels - 1; i >= 0; i--) {
+ sector_t s;
+ v->hash_level_block[i] = hash_position;
+ s = verity_position_at_level(v, v->data_blocks, i);
+ s = (s >> v->hash_per_block_bits) +
+ !!(s & ((1 << v->hash_per_block_bits) - 1));
+ if (hash_position + s < hash_position) {
+ ti->error = "Hash device offset overflow";
+ r = -E2BIG;
+ goto bad;
+ }
+ hash_position += s;
+ }
+ v->hash_blocks = hash_position;
+
+ v->bufio = dm_bufio_client_create(v->hash_dev->bdev,
+ 1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux),
+ dm_bufio_alloc_callback, NULL);
+ if (IS_ERR(v->bufio)) {
+ ti->error = "Cannot initialize dm-bufio";
+ r = PTR_ERR(v->bufio);
+ v->bufio = NULL;
+ goto bad;
+ }
+
+ if (dm_bufio_get_device_size(v->bufio) < v->hash_blocks) {
+ ti->error = "Hash device is too small";
+ r = -E2BIG;
+ goto bad;
+ }
+
+ v->io_mempool = mempool_create_kmalloc_pool(DM_VERITY_MEMPOOL_SIZE ,
+ sizeof(struct dm_verity_io) + v->shash_descsize + v->digest_size * 2);
+ if (!v->io_mempool) {
+ ti->error = "Cannot allocate io mempool";
+ r = -ENOMEM;
+ goto bad;
+ }
+
+ v->vec_mempool = mempool_create_kmalloc_pool(DM_VERITY_MEMPOOL_SIZE ,
+ BIO_MAX_PAGES * sizeof(struct bio_vec));
+ if (!v->vec_mempool) {
+ ti->error = "Cannot allocate vector mempool";
+ r = -ENOMEM;
+ goto bad;
+ }
+
+ /*v->verify_wq = alloc_workqueue("verityd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 1);*/
+ /* WQ_UNBOUND greatly improves performance when running on ramdisk */
+ v->verify_wq = alloc_workqueue("verityd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, num_online_cpus());
+ if (!v->verify_wq) {
+ ti->error = "Cannot allocate workqueue";
+ r = -ENOMEM;
+ goto bad;
+ }
+
+ return 0;
+
+bad:
+ verity_dtr(ti);
+ return r;
+}
+
+static void verity_dtr(struct dm_target *ti)
+{
+ struct dm_verity *v = ti->private;
+
+ if (v->verify_wq)
+ destroy_workqueue(v->verify_wq);
+ if (v->vec_mempool)
+ mempool_destroy(v->vec_mempool);
+ if (v->io_mempool)
+ mempool_destroy(v->io_mempool);
+ if (v->bufio)
+ dm_bufio_client_destroy(v->bufio);
+ kfree(v->salt);
+ kfree(v->root_digest);
+ if (v->tfm)
+ crypto_free_shash(v->tfm);
+ kfree(v->alg_name);
+ if (v->hash_dev)
+ dm_put_device(ti, v->hash_dev);
+ if (v->data_dev)
+ dm_put_device(ti, v->data_dev);
+ kfree(v);
+}
+
+static struct target_type verity_target = {
+ .name = "verity",
+ .version = {1, 0, 0},
+ .module = THIS_MODULE,
+ .ctr = verity_ctr,
+ .dtr = verity_dtr,
+ .map = verity_map,
+ .status = verity_status,
+ .ioctl = verity_ioctl,
+ .merge = verity_merge,
+ .iterate_devices = verity_iterate_devices,
+ .io_hints = verity_io_hints,
+};
+
+static int __init dm_verity_init(void)
+{
+ int r;
+ r = dm_register_target(&verity_target);
+ if (r < 0)
+ DMERR("register failed %d", r);
+ return r;
+}
+
+static void __exit dm_verity_exit(void)
+{
+ dm_unregister_target(&verity_target);
+}
+
+module_init(dm_verity_init);
+module_exit(dm_verity_exit);
+
+MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>");
+MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking");
+MODULE_LICENSE("GPL");
+
Index: linux-3.3-rc5-fast/drivers/md/dm-bufio.c
================================================== =================
--- linux-3.3-rc5-fast.orig/drivers/md/dm-bufio.c 2012-03-03 05:19:33.000000000 +0100
+++ linux-3.3-rc5-fast/drivers/md/dm-bufio.c 2012-03-03 05:19:35.000000000 +0100
@@ -579,7 +579,7 @@ static void write_endio(struct bio *bio,
struct dm_buffer *b = container_of(bio, struct dm_buffer, bio);

b->write_error = error;
- if (error) {
+ if (unlikely(error)) {
struct dm_bufio_client *c = b->c;
(void)cmpxchg(&c->async_write_error, 0, error);
}
@@ -698,13 +698,20 @@ static void __wait_for_free_buffer(struc
dm_bufio_lock(c);
}

+enum new_flag {
+ NF_FRESH = 0,
+ NF_READ = 1,
+ NF_GET = 2,
+ NF_PREFETCH = 3
+};
+
/*
* Allocate a new buffer. If the allocation is not possible, wait until
* some other thread frees a buffer.
*
* May drop the lock and regain it.
*/
-static struct dm_buffer *__alloc_buffer_wait_no_callback(struct dm_bufio_client *c)
+static struct dm_buffer *__alloc_buffer_wait_no_callback(struct dm_bufio_client *c, enum new_flag nf)
{
struct dm_buffer *b;

@@ -727,6 +734,9 @@ static struct dm_buffer *__alloc_buffer_
return b;
}

+ if (nf == NF_PREFETCH)
+ return NULL;
+
if (!list_empty(&c->reserved_buffers)) {
b = list_entry(c->reserved_buffers.next,
struct dm_buffer, lru_list);
@@ -744,9 +754,12 @@ static struct dm_buffer *__alloc_buffer_
}
}

-static struct dm_buffer *__alloc_buffer_wait(struct dm_bufio_client *c)
+static struct dm_buffer *__alloc_buffer_wait(struct dm_bufio_client *c, enum new_flag nf)
{
- struct dm_buffer *b = __alloc_buffer_wait_no_callback(c);
+ struct dm_buffer *b = __alloc_buffer_wait_no_callback(c, nf);
+
+ if (!b)
+ return NULL;

if (c->alloc_callback)
c->alloc_callback(b);
@@ -866,15 +879,8 @@ static struct dm_buffer *__find(struct d
* Getting a buffer
*--------------------------------------------------------------*/

-enum new_flag {
- NF_FRESH = 0,
- NF_READ = 1,
- NF_GET = 2
-};
-
static struct dm_buffer *__bufio_new(struct dm_bufio_client *c, sector_t block,
- enum new_flag nf, struct dm_buffer **bp,
- int *need_submit)
+ enum new_flag nf, int *need_submit)
{
struct dm_buffer *b, *new_b = NULL;

@@ -882,6 +888,19 @@ static struct dm_buffer *__bufio_new(str

b = __find(c, block);
if (b) {
+found_buffer:
+ if (nf == NF_PREFETCH)
+ return NULL;
+ /*
+ * Note: it is essential that we don't wait for the buffer to be
+ * read if dm_bufio_get function is used. Both dm_bufio_get and
+ * dm_bufio_prefetch can be used in the driver request routine.
+ * If the user called both dm_bufio_prefetch and dm_bufio_get on
+ * the same buffer, it would deadlock if we waited.
+ */
+ if (nf == NF_GET && unlikely(test_bit(B_READING, &b->state)))
+ return NULL;
+
b->hold_count++;
__relink_lru(b, test_bit(B_DIRTY, &b->state) ||
test_bit(B_WRITING, &b->state));
@@ -891,7 +910,9 @@ static struct dm_buffer *__bufio_new(str
if (nf == NF_GET)
return NULL;

- new_b = __alloc_buffer_wait(c);
+ new_b = __alloc_buffer_wait(c, nf);
+ if (!new_b)
+ return NULL;

/*
* We've had a period where the mutex was unlocked, so need to
@@ -900,10 +921,7 @@ static struct dm_buffer *__bufio_new(str
b = __find(c, block);
if (b) {
__free_buffer_wake(new_b);
- b->hold_count++;
- __relink_lru(b, test_bit(B_DIRTY, &b->state) ||
- test_bit(B_WRITING, &b->state));
- return b;
+ goto found_buffer;
}

__check_watermark(c);
@@ -957,7 +975,7 @@ static void *new_read(struct dm_bufio_cl
struct dm_buffer *b;

dm_bufio_lock(c);
- b = __bufio_new(c, block, nf, bp, &need_submit);
+ b = __bufio_new(c, block, nf, &need_submit);
dm_bufio_unlock(c);

if (!b || IS_ERR(b))
@@ -997,6 +1015,8 @@ void *dm_bufio_read(struct dm_bufio_clie
}
EXPORT_SYMBOL_GPL(dm_bufio_read);

+static void dm_bufio_release_unlocked(struct dm_buffer *b);
+
void *dm_bufio_new(struct dm_bufio_client *c, sector_t block,
struct dm_buffer **bp)
{
@@ -1006,13 +1026,35 @@ void *dm_bufio_new(struct dm_bufio_clien
}
EXPORT_SYMBOL_GPL(dm_bufio_new);

-void dm_bufio_release(struct dm_buffer *b)
+void dm_bufio_prefetch(struct dm_bufio_client *c,
+ sector_t block, unsigned n_blocks)
{
- struct dm_bufio_client *c = b->c;
+ struct blk_plug plug;

+ blk_start_plug(&plug);
dm_bufio_lock(c);

- BUG_ON(test_bit(B_READING, &b->state));
+ for (; n_blocks--; block++) {
+ int need_submit;
+ struct dm_buffer *b;
+ b = __bufio_new(c, block, NF_PREFETCH, &need_submit);
+ if (unlikely(b != NULL)) {
+ if (need_submit)
+ submit_io(b, READ, b->block, read_endio);
+ dm_bufio_release_unlocked(b);
+ }
+
+ }
+
+ dm_bufio_unlock(c);
+ blk_finish_plug(&plug);
+}
+EXPORT_SYMBOL(dm_bufio_prefetch);
+
+static void dm_bufio_release_unlocked(struct dm_buffer *b)
+{
+ struct dm_bufio_client *c = b->c;
+
BUG_ON(!b->hold_count);

b->hold_count--;
@@ -1025,12 +1067,23 @@ void dm_bufio_release(struct dm_buffer *
* invalid buffer.
*/
if ((b->read_error || b->write_error) &&
+ !test_bit(B_READING, &b->state) &&
!test_bit(B_WRITING, &b->state) &&
!test_bit(B_DIRTY, &b->state)) {
__unlink_buffer(b);
__free_buffer_wake(b);
}
}
+}
+
+void dm_bufio_release(struct dm_buffer *b)
+{
+ struct dm_bufio_client *c = b->c;
+
+ dm_bufio_lock(c);
+
+ BUG_ON(test_bit(B_READING, &b->state));
+ dm_bufio_release_unlocked(b);

dm_bufio_unlock(c);
}
@@ -1042,6 +1095,8 @@ void dm_bufio_mark_buffer_dirty(struct d

dm_bufio_lock(c);

+ BUG_ON(test_bit(B_READING, &b->state));
+
if (!test_and_set_bit(B_DIRTY, &b->state))
__relink_lru(b, LIST_DIRTY);

Index: linux-3.3-rc5-fast/drivers/md/dm-bufio.h
================================================== =================
--- linux-3.3-rc5-fast.orig/drivers/md/dm-bufio.h 2012-03-03 05:19:33.000000000 +0100
+++ linux-3.3-rc5-fast/drivers/md/dm-bufio.h 2012-03-03 05:19:35.000000000 +0100
@@ -63,6 +63,14 @@ void *dm_bufio_new(struct dm_bufio_clien
struct dm_buffer **bp);

/*
+ * Prefetch the specified blocks to the cache.
+ * The function starts to read the blocks and returns without waiting for
+ * I/O to finish.
+ */
+void dm_bufio_prefetch(struct dm_bufio_client *c,
+ sector_t block, unsigned n_blocks);
+
+/*
* Release a reference obtained with dm_bufio_{read,get,new}. The data
* pointer and dm_buffer pointer is no longer valid after this call.
*/

--
dm-devel mailing list
dm-devel@redhat.com
https://www.redhat.com/mailman/listinfo/dm-devel
 
Old 03-06-2012, 08:59 PM
Mandeep Singh Baines
 
Default dm: remake of the verity target

Mikulas Patocka (mpatocka@redhat.com) wrote:
> Hi
>
> Here I'm posting remake of the dm-verity target originall developed by
> Mandeep Singh Baines. It has a compatible target line syntax, so it can be
> used as a drop-in replacement.
>
> The major difference is that this driver uses dm-bufio to manage cache of
> hash blocks in memory. We talked with Alasdair about using dm-bufio for
> caching and I concluded that it is simpler to rewrite the code rather than
> transform the original Google code with patches.
>
> Because of dm-bufio, memory consumption is not dependent on device size
> (if the system starts running out of memory, dm-bufio discards cached
> blocks loaded in memory).
>
> This implementation is smaller, because (unlike the original
> implementation), it doesn't create persistent in-memory structures.
>
> This implementation is faster than the original. It uses clustered
> prefetch to prefetch several hash blocks at once, greatly improving
> performance if the data partition and hash partition are located on the
> same disk (the prefetch cluster can be set with prefetch_cluster module
> parameter).
>
> Mikulas
>

Hi Mikulus,

This is some nice work. I like that you've been able to abstract a lot
of the hash buffer management with dm-bufio. You got rid of the I/O queue.
I've been meaning to do that for a while. The prefetch is also nice.
We planned to do this but I decided to not do it now in order to get the
base functionality in:

http://crosbug.com/25441

However, there are some things that I don't like. I don't like comments
either but you have none. You also removed our documentation. You are
allocated a complete shash_desc per I/O. We only allocate one per CPU.
We short-circuit the hash at any level. Your implementation can only
shirt circuit at the lowest level.

I'd like to propose that we get the version we sent upstream and then work
together on adding some of your enhancements incrementally. Other than
the changes we've made to cleanup for upstreaming, the version I
submitted is the code we are using in production.

I'm happy to add prefetch now if that is required for merging.

What do you think?

Regards,
Mandeep

> ---
>
> Disk: Maxtor Atlas 15k2 146GB, 3.8GB partition
>
> The following tests were made:
> dd if=/dev/mapper/verity of=/dev/null bs=1M
> fsck.ext2 -fn -C 0 /dev/mapper/verity
> fio --rw=randread --size=200M --bsrange=1k-128k --filename=/dev/mapper/verity --name=job1 --name=job2 --name=job3 --name=job4
>
> raw partition:
> dd: 42s
> fsck: 11s
> fio: 13s
>
> original google dm-verity implementation:
> dd (first time): 571s
> dd (next time): 46s
> fsck (first time): 39s
> fsck (next time): 10s
> fio (first time): 26s
> fio (next time): 24s
>
> my vm-verity implementation:
> dd (first time): 45s
> dd (next time): 43s
> fsck (first time): 11s
> fsck (next time): 11s
> fio (first time): 14s
> fio (next time): 14s
>
> ---
>
> Remake of the google dm-verity patch.
>
> Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
>
> ---
> drivers/md/Kconfig | 17 +
> drivers/md/Makefile | 1
> drivers/md/dm-bufio.c | 97 ++++--
> drivers/md/dm-bufio.h | 8
> drivers/md/dm-verity.c | 785 +++++++++++++++++++++++++++++++++++++++++++++++++
> 5 files changed, 887 insertions(+), 21 deletions(-)
>
> Index: linux-3.3-rc5-fast/drivers/md/Kconfig
> ================================================== =================
> --- linux-3.3-rc5-fast.orig/drivers/md/Kconfig 2012-03-03 05:19:33.000000000 +0100
> +++ linux-3.3-rc5-fast/drivers/md/Kconfig 2012-03-03 19:39:30.000000000 +0100
> @@ -388,4 +388,21 @@ config DM_FLAKEY
> ---help---
> A target that intermittently fails I/O for debugging purposes.
>
> +config DM_VERITY
> + tristate "Verity target support"
> + depends on BLK_DEV_DM
> + select CRYPTO
> + select CRYPTO_HASH
> + select DM_BUFIO
> + ---help---
> + This device-mapper target allows you to create a device that
> + transparently integrity checks the data on it. You'll need to
> + activate the digests you're going to use in the cryptoapi
> + configuration.
> +
> + To compile this code as a module, choose M here: the module will
> + be called dm-verity.
> +
> + If unsure, say N.
> +
> endif # MD
> Index: linux-3.3-rc5-fast/drivers/md/Makefile
> ================================================== =================
> --- linux-3.3-rc5-fast.orig/drivers/md/Makefile 2012-03-03 05:19:33.000000000 +0100
> +++ linux-3.3-rc5-fast/drivers/md/Makefile 2012-03-03 19:39:30.000000000 +0100
> @@ -29,6 +29,7 @@ obj-$(CONFIG_MD_FAULTY) += faulty.o
> obj-$(CONFIG_BLK_DEV_MD) += md-mod.o
> obj-$(CONFIG_BLK_DEV_DM) += dm-mod.o
> obj-$(CONFIG_DM_BUFIO) += dm-bufio.o
> +obj-$(CONFIG_DM_VERITY) += dm-verity.o
> obj-$(CONFIG_DM_CRYPT) += dm-crypt.o
> obj-$(CONFIG_DM_DELAY) += dm-delay.o
> obj-$(CONFIG_DM_FLAKEY) += dm-flakey.o
> Index: linux-3.3-rc5-fast/drivers/md/dm-verity.c
> ================================================== =================
> --- /dev/null 1970-01-01 00:00:00.000000000 +0000
> +++ linux-3.3-rc5-fast/drivers/md/dm-verity.c 2012-03-03 05:51:10.000000000 +0100
> @@ -0,0 +1,785 @@
> +/*
> + * Copyright (C) 2012 Red Hat, Inc.
> + *
> + * Author: Mikulas Patocka <mpatocka@redhat.com>
> + *
> + * Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors
> + *
> + * This file is released under the GPLv2.
> + *
> + * Device mapper target parameters:
> + * <version> 0
> + * <data device>
> + * <hash device>
> + * <hash start> (typically 0)
> + * <block size> (typically 4096)
> + * <algorithm>
> + * <digest>
> + * optional parameters:
> + * <salt> (should have 32 bytes for compatibility with Google code)
> + * <hash block size> (by default it is the same as data block size)
> + *
> + * In the file "/sys/module/dm_verity/parameters/prefetch_cluster" you can set
> + * default prefetch value. Data are read in "prefetch_cluster" chunks from the
> + * hash device. Prefetch cluster greatly improves performance when data and hash
> + * are on the same disk on different partitions.
> + */
> +
> +#include <linux/module.h>
> +#include <linux/device-mapper.h>
> +#include <crypto/hash.h>
> +#include "dm-bufio.h"
> +
> +#define DM_MSG_PREFIX "verity"
> +
> +#define DM_VERITY_IO_VEC_INLINE 16
> +#define DM_VERITY_MEMPOOL_SIZE 4
> +#define DM_VERITY_PREFETCH_SIZE 262144
> +
> +#define DM_VERITY_MAX_LEVELS 63
> +
> +static unsigned prefetch_cluster = DM_VERITY_PREFETCH_SIZE;
> +
> +module_param_named(prefetch_cluster, prefetch_cluster, uint, S_IRUGO | S_IWUSR);
> +
> +struct dm_verity {
> + struct dm_dev *data_dev;
> + struct dm_dev *hash_dev;
> + struct dm_target *ti;
> + struct dm_bufio_client *bufio;
> + char *alg_name;
> + struct crypto_shash *tfm;
> + u8 *root_digest;
> + u8 *salt;
> + unsigned salt_size;
> + sector_t data_start;
> + sector_t hash_start;
> + sector_t data_blocks;
> + sector_t hash_blocks;
> + unsigned char data_dev_block_bits;
> + unsigned char hash_dev_block_bits;
> + unsigned char hash_per_block_bits;
> + unsigned char levels;
> + unsigned digest_size;
> + unsigned shash_descsize;
> +
> + mempool_t *io_mempool;
> + mempool_t *vec_mempool;
> +
> + struct workqueue_struct *verify_wq;
> +
> + sector_t hash_level_block[DM_VERITY_MAX_LEVELS];
> +};
> +
> +struct dm_verity_io {
> + struct dm_verity *v;
> + struct bio *bio;
> + bio_end_io_t *orig_bi_end_io;
> + void *orig_bi_private;
> + sector_t block;
> + unsigned n_blocks;
> + struct bio_vec *io_vec;
> + unsigned io_vec_size;
> + struct work_struct work;
> + struct bio_vec io_vec_inline[DM_VERITY_IO_VEC_INLINE];
> + /* u8 hash_desc[crypto_shash_descsize(v->tfm)]; */
> + /* u8 real_digest[v->digest_size]; */
> + /* u8 want_digest[v->digest_size]; */
> +};
> +
> +static struct shash_desc *io_hash_desc(struct dm_verity *v, struct dm_verity_io *io)
> +{
> + return (struct shash_desc *)(io + 1);
> +}
> +
> +static u8 *io_real_digest(struct dm_verity *v, struct dm_verity_io *io)
> +{
> + return (u8 *)(io + 1) + v->shash_descsize;
> +}
> +
> +static u8 *io_want_digest(struct dm_verity *v, struct dm_verity_io *io)
> +{
> + return (u8 *)(io + 1) + v->shash_descsize + v->digest_size;
> +}
> +
> +struct buffer_aux {
> + int hash_verified;
> +};
> +
> +static void dm_bufio_alloc_callback(struct dm_buffer *buf)
> +{
> + struct buffer_aux *aux = dm_bufio_get_aux_data(buf);
> + aux->hash_verified = 0;
> +}
> +
> +static sector_t verity_map_sector(struct dm_verity *v, sector_t bi_sector)
> +{
> + return v->data_start + dm_target_offset(v->ti, bi_sector);
> +}
> +
> +static sector_t verity_position_at_level(struct dm_verity *v, sector_t block,
> + int level)
> +{
> + return block >> (level * v->hash_per_block_bits);
> +}
> +
> +static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level,
> + sector_t *hash_block, unsigned *offset)
> +{
> + sector_t position = verity_position_at_level(v, block, level);
> +
> + *hash_block = v->hash_level_block[level] + (position >> v->hash_per_block_bits);
> + if (offset)
> + *offset = v->digest_size * (position & ((1 << v->hash_per_block_bits) - 1));
> +}
> +
> +static int verity_verify_level(struct dm_verity_io *io, sector_t block,
> + int level, int skip_unverified)
> +{
> + struct dm_verity *v = io->v;
> + struct dm_buffer *buf;
> + struct buffer_aux *aux;
> + u8 *data;
> + int r;
> + sector_t hash_block;
> + unsigned offset;
> +
> + verity_hash_at_level(v, block, level, &hash_block, &offset);
> +
> + data = dm_bufio_read(v->bufio, hash_block, &buf);
> + if (unlikely(IS_ERR(data)))
> + return PTR_ERR(data);
> +
> + aux = dm_bufio_get_aux_data(buf);
> +
> + if (!aux->hash_verified) {
> + struct shash_desc *desc;
> + u8 *result;
> +
> + if (skip_unverified) {
> + r = 1;
> + goto release_ret_r;
> + }
> +
> + desc = io_hash_desc(v, io);
> + desc->tfm = v->tfm;
> + desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
> + r = crypto_shash_init(desc);
> + if (r < 0) {
> + DMERR("crypto_shash_init failed: %d", r);
> + goto release_ret_r;
> + }
> +
> + r = crypto_shash_update(desc, data, 1 << v->hash_dev_block_bits);
> + if (r < 0) {
> + DMERR("crypto_shash_update failed: %d", r);
> + goto release_ret_r;
> + }
> +
> + r = crypto_shash_update(desc, v->salt, v->salt_size);
> + if (r < 0) {
> + DMERR("crypto_shash_update failed: %d", r);
> + goto release_ret_r;
> + }
> +
> + result = io_real_digest(v, io);
> + r = crypto_shash_final(desc, result);
> + if (r < 0) {
> + DMERR("crypto_shash_final failed: %d", r);
> + goto release_ret_r;
> + }
> + if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) {
> + DMERR_LIMIT("metadata block %llu is corrupted",
> + (unsigned long long)hash_block);
> + r = -EIO;
> + goto release_ret_r;
> + } else
> + aux->hash_verified = 1;
> + }
> +
> + data += offset;
> +
> + memcpy(io_want_digest(v, io), data, v->digest_size);
> +
> + dm_bufio_release(buf);
> + return 0;
> +
> +release_ret_r:
> + dm_bufio_release(buf);
> + return r;
> +}
> +
> +static int verity_verify_io(struct dm_verity_io *io)
> +{
> + struct dm_verity *v = io->v;
> + unsigned b;
> + int i;
> + unsigned vector = 0, offset = 0;
> + for (b = 0; b < io->n_blocks; b++) {
> + struct shash_desc *desc;
> + u8 *result;
> + int r;
> + unsigned todo;
> +
> + if (likely(v->levels)) {
> + int r = verity_verify_level(io, io->block + b, 0, 1);
> + if (likely(!r))
> + goto test_block_hash;
> + if (r < 0)
> + return r;
> + }
> +
> + memcpy(io_want_digest(v, io), v->root_digest, v->digest_size);
> +
> + for (i = v->levels - 1; i >= 0; i--) {
> + int r = verity_verify_level(io, io->block + b, i, 0);
> + if (unlikely(r))
> + return r;
> + }
> +
> +test_block_hash:
> + desc = io_hash_desc(v, io);
> + desc->tfm = v->tfm;
> + desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
> + r = crypto_shash_init(desc);
> + if (r < 0) {
> + DMERR("crypto_shash_init failed: %d", r);
> + return r;
> + }
> +
> + todo = 1 << v->data_dev_block_bits;
> + do {
> + struct bio_vec *bv;
> + u8 *page;
> + unsigned len;
> +
> + BUG_ON(vector >= io->io_vec_size);
> + bv = &io->io_vec[vector];
> + page = kmap_atomic(bv->bv_page, KM_USER0);
> + len = bv->bv_len - offset;
> + if (likely(len >= todo))
> + len = todo;
> + r = crypto_shash_update(desc,
> + page + bv->bv_offset + offset, len);
> + kunmap_atomic(page, KM_USER0);
> + if (r < 0) {
> + DMERR("crypto_shash_update failed: %d", r);
> + return r;
> + }
> + offset += len;
> + if (likely(offset == bv->bv_len)) {
> + offset = 0;
> + vector++;
> + }
> + todo -= len;
> + } while (todo);
> +
> + r = crypto_shash_update(desc, v->salt, v->salt_size);
> + if (r < 0) {
> + DMERR("crypto_shash_update failed: %d", r);
> + return r;
> + }
> +
> + result = io_real_digest(v, io);
> + r = crypto_shash_final(desc, result);
> + if (r < 0) {
> + DMERR("crypto_shash_final failed: %d", r);
> + return r;
> + }
> + if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) {
> + DMERR_LIMIT("data block %llu is corrupted",
> + (unsigned long long)(io->block + b));
> + return -EIO;
> + }
> + }
> + BUG_ON(vector != io->io_vec_size);
> + BUG_ON(offset);
> + return 0;
> +}
> +
> +static void verity_finish_io(struct dm_verity_io *io, int error)
> +{
> + struct bio *bio = io->bio;
> + struct dm_verity *v = io->v;
> +
> + bio->bi_end_io = io->orig_bi_end_io;
> + bio->bi_private = io->orig_bi_private;
> +
> + if (io->io_vec != io->io_vec_inline)
> + mempool_free(io->io_vec, v->vec_mempool);
> + mempool_free(io, v->io_mempool);
> +
> + bio_endio(bio, error);
> +}
> +
> +static void verity_work(struct work_struct *w)
> +{
> + struct dm_verity_io *io = container_of(w, struct dm_verity_io, work);
> +
> + verity_finish_io(io, verity_verify_io(io));
> +}
> +
> +static void verity_end_io(struct bio *bio, int error)
> +{
> + struct dm_verity_io *io = bio->bi_private;
> + if (error) {
> + verity_finish_io(io, error);
> + return;
> + }
> +
> + INIT_WORK(&io->work, verity_work);
> + queue_work(io->v->verify_wq, &io->work);
> +}
> +
> +static void verity_prefetch_io(struct dm_verity *v, struct dm_verity_io *io)
> +{
> + int i;
> + for (i = v->levels - 2; i >= 0; i--) {
> + sector_t hash_block_start;
> + sector_t hash_block_end;
> + verity_hash_at_level(v, io->block, i, &hash_block_start, NULL);
> + verity_hash_at_level(v, io->block + io->n_blocks - 1, i, &hash_block_end, NULL);
> + if (!i) {
> + unsigned cluster = prefetch_cluster;
> + /* barrier to stop GCC from re-reading prefetch_cluster again */
> + barrier();
> + cluster >>= v->data_dev_block_bits;
> + if (unlikely(!cluster))
> + goto no_prefetch_cluster;
> + if (unlikely(cluster & (cluster - 1)))
> + cluster = 1 << (fls(cluster) - 1);
> +
> + hash_block_start &= ~(sector_t)(cluster - 1);
> + hash_block_end |= cluster - 1;
> + if (unlikely(hash_block_end >= v->hash_blocks))
> + hash_block_end = v->hash_blocks - 1;
> + }
> +no_prefetch_cluster:
> + dm_bufio_prefetch(v->bufio, hash_block_start,
> + hash_block_end - hash_block_start + 1);
> + }
> +}
> +
> +static int verity_map(struct dm_target *ti, struct bio *bio,
> + union map_info *map_context)
> +{
> + struct dm_verity *v = ti->private;
> + struct dm_verity_io *io;
> +
> + if (((unsigned)bio->bi_sector | bio_sectors(bio)) &
> + ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) {
> + DMERR_LIMIT("unaligned io");
> + return -EIO;
> + }
> +
> + if ((bio->bi_sector + bio_sectors(bio)) >>
> + (v->data_dev_block_bits - SECTOR_SHIFT) > v->data_blocks) {
> + DMERR_LIMIT("io out of range");
> + return -EIO;
> + }
> +
> + if (bio_data_dir(bio) == WRITE)
> + return -EIO;
> +
> + io = mempool_alloc(v->io_mempool, GFP_NOIO);
> + io->v = v;
> + io->bio = bio;
> + io->orig_bi_end_io = bio->bi_end_io;
> + io->orig_bi_private = bio->bi_private;
> + io->block = bio->bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT);
> + io->n_blocks = bio->bi_size >> v->data_dev_block_bits;
> +
> + bio->bi_end_io = verity_end_io;
> + bio->bi_private = io;
> + bio->bi_bdev = v->data_dev->bdev;
> + bio->bi_sector = verity_map_sector(v, bio->bi_sector);
> +
> + io->io_vec_size = bio->bi_vcnt - bio->bi_idx;
> + if (io->io_vec_size < DM_VERITY_IO_VEC_INLINE)
> + io->io_vec = io->io_vec_inline;
> + else
> + io->io_vec = mempool_alloc(v->vec_mempool, GFP_NOIO);
> + memcpy(io->io_vec, bio_iovec(bio),
> + io->io_vec_size * sizeof(struct bio_vec));
> +
> + verity_prefetch_io(v, io);
> +
> + generic_make_request(bio);
> +
> + return DM_MAPIO_SUBMITTED;
> +}
> +
> +static int verity_status(struct dm_target *ti, status_type_t type,
> + char *result, unsigned maxlen)
> +{
> + struct dm_verity *v = ti->private;
> + unsigned sz = 0;
> + unsigned x;
> +
> + switch (type) {
> + case STATUSTYPE_INFO:
> + result[0] = 0;
> + break;
> + case STATUSTYPE_TABLE:
> + DMEMIT("%u %s %s %llu %u %s ",
> + 0,
> + v->data_dev->name,
> + v->hash_dev->name,
> + (unsigned long long)v->hash_start << (v->hash_dev_block_bits - SECTOR_SHIFT),
> + 1 << v->data_dev_block_bits,
> + v->alg_name
> + );
> + for (x = 0; x < v->digest_size; x++)
> + DMEMIT("%02x", v->root_digest[x]);
> + DMEMIT(" ");
> + if (!v->salt_size)
> + DMEMIT("-");
> + else
> + for (x = 0; x < v->salt_size; x++)
> + DMEMIT("%02x", v->salt[x]);
> + DMEMIT(" %u", 1 << v->hash_dev_block_bits);
> + break;
> + }
> + return 0;
> +}
> +
> +static int verity_ioctl(struct dm_target *ti, unsigned cmd,
> + unsigned long arg)
> +{
> + struct dm_verity *v = ti->private;
> + int r = 0;
> +
> + if (v->data_start ||
> + ti->len != i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT)
> + r = scsi_verify_blk_ioctl(NULL, cmd);
> +
> + return r ? : __blkdev_driver_ioctl(v->data_dev->bdev, v->data_dev->mode,
> + cmd, arg);
> +}
> +
> +static int verity_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
> + struct bio_vec *biovec, int max_size)
> +{
> + struct dm_verity *v = ti->private;
> + struct request_queue *q = bdev_get_queue(v->data_dev->bdev);
> +
> + if (!q->merge_bvec_fn)
> + return max_size;
> +
> + bvm->bi_bdev = v->data_dev->bdev;
> + bvm->bi_sector = verity_map_sector(v, bvm->bi_sector);
> +
> + return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
> +}
> +
> +static int verity_iterate_devices(struct dm_target *ti,
> + iterate_devices_callout_fn fn, void *data)
> +{
> + struct dm_verity *v = ti->private;
> + return fn(ti, v->data_dev, v->data_start, ti->len, data);
> +}
> +
> +static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits)
> +{
> + struct dm_verity *v = ti->private;
> +
> + if (limits->logical_block_size < 1 << v->data_dev_block_bits)
> + limits->logical_block_size = 1 << v->data_dev_block_bits;
> + if (limits->physical_block_size < 1 << v->data_dev_block_bits)
> + limits->physical_block_size = 1 << v->data_dev_block_bits;
> + blk_limits_io_min(limits, limits->logical_block_size);
> +}
> +
> +static void verity_dtr(struct dm_target *ti);
> +
> +static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv)
> +{
> + struct dm_verity *v;
> + unsigned num;
> + unsigned long long hs;
> + int r;
> + int i;
> + sector_t hash_position;
> + char dummy;
> +
> + v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL);
> + if (!v) {
> + ti->error = "Cannot allocate verity structure";
> + return -ENOMEM;
> + }
> + ti->private = v;
> + v->ti = ti;
> +
> + if ((dm_table_get_mode(ti->table) & ~FMODE_READ) != 0) {
> + ti->error = "Device must be readonly";
> + r = -EINVAL;
> + goto bad;
> + }
> +
> + if (argc < 7) {
> + ti->error = "Not enough arguments";
> + r = -EINVAL;
> + goto bad;
> + }
> +
> + if (sscanf(argv[0], "%d%c", &num, &dummy) != 1 ||
> + num != 0) {
> + ti->error = "Invalid version";
> + r = -EINVAL;
> + goto bad;
> + }
> +
> + r = dm_get_device(ti, argv[1], FMODE_READ, &v->data_dev);
> + if (r) {
> + ti->error = "Data device lookup failed";
> + goto bad;
> + }
> +
> + r = dm_get_device(ti, argv[2], FMODE_READ, &v->hash_dev);
> + if (r) {
> + ti->error = "Data device lookup failed";
> + goto bad;
> + }
> +
> + if (sscanf(argv[3], "%llu%c", &hs, &dummy) != 1 ||
> + hs != (sector_t)hs) {
> + ti->error = "Invalid hash start";
> + r = -EINVAL;
> + goto bad;
> + }
> +
> + if (sscanf(argv[4], "%u%c", &num, &dummy) != 1 ||
> + !num || (num & (num - 1)) ||
> + num < bdev_logical_block_size(v->data_dev->bdev) ||
> + num > PAGE_SIZE) {
> + ti->error = "Invalid data device block size";
> + r = -EINVAL;
> + goto bad;
> + }
> + v->data_dev_block_bits = ffs(num) - 1;
> + v->hash_dev_block_bits = ffs(num) - 1;
> +
> + v->alg_name = kstrdup(argv[5], GFP_KERNEL);
> + if (!v->alg_name) {
> + ti->error = "Cannot allocate algorithm name";
> + r = -ENOMEM;
> + goto bad;
> + }
> +
> + v->tfm = crypto_alloc_shash(v->alg_name, 0, 0);
> + if (IS_ERR(v->tfm)) {
> + ti->error = "Cannot initialize hash function";
> + r = PTR_ERR(v->tfm);
> + v->tfm = NULL;
> + goto bad;
> + }
> + v->digest_size = crypto_shash_digestsize(v->tfm);
> + if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) {
> + ti->error = "Digest size too big";
> + r = -EINVAL;
> + goto bad;
> + }
> + v->shash_descsize =
> + sizeof(struct shash_desc) + crypto_shash_descsize(v->tfm);
> +
> + v->root_digest = kmalloc(v->digest_size, GFP_KERNEL);
> + if (!v->root_digest) {
> + ti->error = "Cannot allocate root digest";
> + r = -ENOMEM;
> + goto bad;
> + }
> + if (strlen(argv[6]) != v->digest_size * 2 ||
> + hex2bin(v->root_digest, argv[6], v->digest_size)) {
> + ti->error = "Invalid root digest";
> + r = -EINVAL;
> + goto bad;
> + }
> +
> + if (argc > 7 && strcmp(argv[7], "-")) {
> + v->salt_size = strlen(argv[7]) / 2;
> + v->salt = kmalloc(v->salt_size, GFP_KERNEL);
> + if (!v->salt) {
> + ti->error = "Cannot allocate salt";
> + r = -ENOMEM;
> + goto bad;
> + }
> + if (strlen(argv[7]) != v->salt_size * 2 ||
> + hex2bin(v->salt, argv[7], v->salt_size)) {
> + ti->error = "Invalid salt";
> + r = -EINVAL;
> + goto bad;
> + }
> + }
> +
> + if (argc > 8) {
> + if (sscanf(argv[8], "%u%c", &num, &dummy) != 1 ||
> + !num || (num & (num - 1)) ||
> + num < bdev_logical_block_size(v->hash_dev->bdev) ||
> + num > INT_MAX) {
> + ti->error = "Invalid hash device block size";
> + r = -EINVAL;
> + goto bad;
> + }
> + v->hash_dev_block_bits = ffs(num) - 1;
> + }
> +
> + if (hs & ((1 << (v->hash_dev_block_bits - SECTOR_SHIFT)) - 1)) {
> + ti->error = "Hash start not aligned on block boundary";
> + r = -EINVAL;
> + goto bad;
> + }
> + v->hash_start = hs >> (v->hash_dev_block_bits - SECTOR_SHIFT);
> +
> + if (ti->len > i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT) {
> + ti->error = "Data device si too small";
> + r = -EINVAL;
> + goto bad;
> + }
> +
> + if (ti->len & ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) {
> + ti->error = "Data device length is not aligned to block size";
> + r = -EINVAL;
> + goto bad;
> + }
> +
> + v->data_blocks = ti->len >> (v->data_dev_block_bits - SECTOR_SHIFT);
> +
> + v->hash_per_block_bits =
> + fls((1 << v->hash_dev_block_bits) / v->digest_size) - 1;
> +
> + v->levels = 0;
> + if (v->data_blocks)
> + while (v->hash_per_block_bits * v->levels < 64 &&
> + (unsigned long long)(v->data_blocks - 1) >>
> + (v->hash_per_block_bits * v->levels))
> + v->levels++;
> +
> + if (v->levels > DM_VERITY_MAX_LEVELS) {
> + ti->error = "Too many tree levels";
> + r = -E2BIG;
> + goto bad;
> + }
> +
> + hash_position = v->hash_start;
> + for (i = v->levels - 1; i >= 0; i--) {
> + sector_t s;
> + v->hash_level_block[i] = hash_position;
> + s = verity_position_at_level(v, v->data_blocks, i);
> + s = (s >> v->hash_per_block_bits) +
> + !!(s & ((1 << v->hash_per_block_bits) - 1));
> + if (hash_position + s < hash_position) {
> + ti->error = "Hash device offset overflow";
> + r = -E2BIG;
> + goto bad;
> + }
> + hash_position += s;
> + }
> + v->hash_blocks = hash_position;
> +
> + v->bufio = dm_bufio_client_create(v->hash_dev->bdev,
> + 1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux),
> + dm_bufio_alloc_callback, NULL);
> + if (IS_ERR(v->bufio)) {
> + ti->error = "Cannot initialize dm-bufio";
> + r = PTR_ERR(v->bufio);
> + v->bufio = NULL;
> + goto bad;
> + }
> +
> + if (dm_bufio_get_device_size(v->bufio) < v->hash_blocks) {
> + ti->error = "Hash device is too small";
> + r = -E2BIG;
> + goto bad;
> + }
> +
> + v->io_mempool = mempool_create_kmalloc_pool(DM_VERITY_MEMPOOL_SIZE ,
> + sizeof(struct dm_verity_io) + v->shash_descsize + v->digest_size * 2);
> + if (!v->io_mempool) {
> + ti->error = "Cannot allocate io mempool";
> + r = -ENOMEM;
> + goto bad;
> + }
> +
> + v->vec_mempool = mempool_create_kmalloc_pool(DM_VERITY_MEMPOOL_SIZE ,
> + BIO_MAX_PAGES * sizeof(struct bio_vec));
> + if (!v->vec_mempool) {
> + ti->error = "Cannot allocate vector mempool";
> + r = -ENOMEM;
> + goto bad;
> + }
> +
> + /*v->verify_wq = alloc_workqueue("verityd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 1);*/
> + /* WQ_UNBOUND greatly improves performance when running on ramdisk */
> + v->verify_wq = alloc_workqueue("verityd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, num_online_cpus());
> + if (!v->verify_wq) {
> + ti->error = "Cannot allocate workqueue";
> + r = -ENOMEM;
> + goto bad;
> + }
> +
> + return 0;
> +
> +bad:
> + verity_dtr(ti);
> + return r;
> +}
> +
> +static void verity_dtr(struct dm_target *ti)
> +{
> + struct dm_verity *v = ti->private;
> +
> + if (v->verify_wq)
> + destroy_workqueue(v->verify_wq);
> + if (v->vec_mempool)
> + mempool_destroy(v->vec_mempool);
> + if (v->io_mempool)
> + mempool_destroy(v->io_mempool);
> + if (v->bufio)
> + dm_bufio_client_destroy(v->bufio);
> + kfree(v->salt);
> + kfree(v->root_digest);
> + if (v->tfm)
> + crypto_free_shash(v->tfm);
> + kfree(v->alg_name);
> + if (v->hash_dev)
> + dm_put_device(ti, v->hash_dev);
> + if (v->data_dev)
> + dm_put_device(ti, v->data_dev);
> + kfree(v);
> +}
> +
> +static struct target_type verity_target = {
> + .name = "verity",
> + .version = {1, 0, 0},
> + .module = THIS_MODULE,
> + .ctr = verity_ctr,
> + .dtr = verity_dtr,
> + .map = verity_map,
> + .status = verity_status,
> + .ioctl = verity_ioctl,
> + .merge = verity_merge,
> + .iterate_devices = verity_iterate_devices,
> + .io_hints = verity_io_hints,
> +};
> +
> +static int __init dm_verity_init(void)
> +{
> + int r;
> + r = dm_register_target(&verity_target);
> + if (r < 0)
> + DMERR("register failed %d", r);
> + return r;
> +}
> +
> +static void __exit dm_verity_exit(void)
> +{
> + dm_unregister_target(&verity_target);
> +}
> +
> +module_init(dm_verity_init);
> +module_exit(dm_verity_exit);
> +
> +MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>");
> +MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking");
> +MODULE_LICENSE("GPL");
> +
> Index: linux-3.3-rc5-fast/drivers/md/dm-bufio.c
> ================================================== =================
> --- linux-3.3-rc5-fast.orig/drivers/md/dm-bufio.c 2012-03-03 05:19:33.000000000 +0100
> +++ linux-3.3-rc5-fast/drivers/md/dm-bufio.c 2012-03-03 05:19:35.000000000 +0100
> @@ -579,7 +579,7 @@ static void write_endio(struct bio *bio,
> struct dm_buffer *b = container_of(bio, struct dm_buffer, bio);
>
> b->write_error = error;
> - if (error) {
> + if (unlikely(error)) {
> struct dm_bufio_client *c = b->c;
> (void)cmpxchg(&c->async_write_error, 0, error);
> }
> @@ -698,13 +698,20 @@ static void __wait_for_free_buffer(struc
> dm_bufio_lock(c);
> }
>
> +enum new_flag {
> + NF_FRESH = 0,
> + NF_READ = 1,
> + NF_GET = 2,
> + NF_PREFETCH = 3
> +};
> +
> /*
> * Allocate a new buffer. If the allocation is not possible, wait until
> * some other thread frees a buffer.
> *
> * May drop the lock and regain it.
> */
> -static struct dm_buffer *__alloc_buffer_wait_no_callback(struct dm_bufio_client *c)
> +static struct dm_buffer *__alloc_buffer_wait_no_callback(struct dm_bufio_client *c, enum new_flag nf)
> {
> struct dm_buffer *b;
>
> @@ -727,6 +734,9 @@ static struct dm_buffer *__alloc_buffer_
> return b;
> }
>
> + if (nf == NF_PREFETCH)
> + return NULL;
> +
> if (!list_empty(&c->reserved_buffers)) {
> b = list_entry(c->reserved_buffers.next,
> struct dm_buffer, lru_list);
> @@ -744,9 +754,12 @@ static struct dm_buffer *__alloc_buffer_
> }
> }
>
> -static struct dm_buffer *__alloc_buffer_wait(struct dm_bufio_client *c)
> +static struct dm_buffer *__alloc_buffer_wait(struct dm_bufio_client *c, enum new_flag nf)
> {
> - struct dm_buffer *b = __alloc_buffer_wait_no_callback(c);
> + struct dm_buffer *b = __alloc_buffer_wait_no_callback(c, nf);
> +
> + if (!b)
> + return NULL;
>
> if (c->alloc_callback)
> c->alloc_callback(b);
> @@ -866,15 +879,8 @@ static struct dm_buffer *__find(struct d
> * Getting a buffer
> *--------------------------------------------------------------*/
>
> -enum new_flag {
> - NF_FRESH = 0,
> - NF_READ = 1,
> - NF_GET = 2
> -};
> -
> static struct dm_buffer *__bufio_new(struct dm_bufio_client *c, sector_t block,
> - enum new_flag nf, struct dm_buffer **bp,
> - int *need_submit)
> + enum new_flag nf, int *need_submit)
> {
> struct dm_buffer *b, *new_b = NULL;
>
> @@ -882,6 +888,19 @@ static struct dm_buffer *__bufio_new(str
>
> b = __find(c, block);
> if (b) {
> +found_buffer:
> + if (nf == NF_PREFETCH)
> + return NULL;
> + /*
> + * Note: it is essential that we don't wait for the buffer to be
> + * read if dm_bufio_get function is used. Both dm_bufio_get and
> + * dm_bufio_prefetch can be used in the driver request routine.
> + * If the user called both dm_bufio_prefetch and dm_bufio_get on
> + * the same buffer, it would deadlock if we waited.
> + */
> + if (nf == NF_GET && unlikely(test_bit(B_READING, &b->state)))
> + return NULL;
> +
> b->hold_count++;
> __relink_lru(b, test_bit(B_DIRTY, &b->state) ||
> test_bit(B_WRITING, &b->state));
> @@ -891,7 +910,9 @@ static struct dm_buffer *__bufio_new(str
> if (nf == NF_GET)
> return NULL;
>
> - new_b = __alloc_buffer_wait(c);
> + new_b = __alloc_buffer_wait(c, nf);
> + if (!new_b)
> + return NULL;
>
> /*
> * We've had a period where the mutex was unlocked, so need to
> @@ -900,10 +921,7 @@ static struct dm_buffer *__bufio_new(str
> b = __find(c, block);
> if (b) {
> __free_buffer_wake(new_b);
> - b->hold_count++;
> - __relink_lru(b, test_bit(B_DIRTY, &b->state) ||
> - test_bit(B_WRITING, &b->state));
> - return b;
> + goto found_buffer;
> }
>
> __check_watermark(c);
> @@ -957,7 +975,7 @@ static void *new_read(struct dm_bufio_cl
> struct dm_buffer *b;
>
> dm_bufio_lock(c);
> - b = __bufio_new(c, block, nf, bp, &need_submit);
> + b = __bufio_new(c, block, nf, &need_submit);
> dm_bufio_unlock(c);
>
> if (!b || IS_ERR(b))
> @@ -997,6 +1015,8 @@ void *dm_bufio_read(struct dm_bufio_clie
> }
> EXPORT_SYMBOL_GPL(dm_bufio_read);
>
> +static void dm_bufio_release_unlocked(struct dm_buffer *b);
> +
> void *dm_bufio_new(struct dm_bufio_client *c, sector_t block,
> struct dm_buffer **bp)
> {
> @@ -1006,13 +1026,35 @@ void *dm_bufio_new(struct dm_bufio_clien
> }
> EXPORT_SYMBOL_GPL(dm_bufio_new);
>
> -void dm_bufio_release(struct dm_buffer *b)
> +void dm_bufio_prefetch(struct dm_bufio_client *c,
> + sector_t block, unsigned n_blocks)
> {
> - struct dm_bufio_client *c = b->c;
> + struct blk_plug plug;
>
> + blk_start_plug(&plug);
> dm_bufio_lock(c);
>
> - BUG_ON(test_bit(B_READING, &b->state));
> + for (; n_blocks--; block++) {
> + int need_submit;
> + struct dm_buffer *b;
> + b = __bufio_new(c, block, NF_PREFETCH, &need_submit);
> + if (unlikely(b != NULL)) {
> + if (need_submit)
> + submit_io(b, READ, b->block, read_endio);
> + dm_bufio_release_unlocked(b);
> + }
> +
> + }
> +
> + dm_bufio_unlock(c);
> + blk_finish_plug(&plug);
> +}
> +EXPORT_SYMBOL(dm_bufio_prefetch);
> +
> +static void dm_bufio_release_unlocked(struct dm_buffer *b)
> +{
> + struct dm_bufio_client *c = b->c;
> +
> BUG_ON(!b->hold_count);
>
> b->hold_count--;
> @@ -1025,12 +1067,23 @@ void dm_bufio_release(struct dm_buffer *
> * invalid buffer.
> */
> if ((b->read_error || b->write_error) &&
> + !test_bit(B_READING, &b->state) &&
> !test_bit(B_WRITING, &b->state) &&
> !test_bit(B_DIRTY, &b->state)) {
> __unlink_buffer(b);
> __free_buffer_wake(b);
> }
> }
> +}
> +
> +void dm_bufio_release(struct dm_buffer *b)
> +{
> + struct dm_bufio_client *c = b->c;
> +
> + dm_bufio_lock(c);
> +
> + BUG_ON(test_bit(B_READING, &b->state));
> + dm_bufio_release_unlocked(b);
>
> dm_bufio_unlock(c);
> }
> @@ -1042,6 +1095,8 @@ void dm_bufio_mark_buffer_dirty(struct d
>
> dm_bufio_lock(c);
>
> + BUG_ON(test_bit(B_READING, &b->state));
> +
> if (!test_and_set_bit(B_DIRTY, &b->state))
> __relink_lru(b, LIST_DIRTY);
>
> Index: linux-3.3-rc5-fast/drivers/md/dm-bufio.h
> ================================================== =================
> --- linux-3.3-rc5-fast.orig/drivers/md/dm-bufio.h 2012-03-03 05:19:33.000000000 +0100
> +++ linux-3.3-rc5-fast/drivers/md/dm-bufio.h 2012-03-03 05:19:35.000000000 +0100
> @@ -63,6 +63,14 @@ void *dm_bufio_new(struct dm_bufio_clien
> struct dm_buffer **bp);
>
> /*
> + * Prefetch the specified blocks to the cache.
> + * The function starts to read the blocks and returns without waiting for
> + * I/O to finish.
> + */
> +void dm_bufio_prefetch(struct dm_bufio_client *c,
> + sector_t block, unsigned n_blocks);
> +
> +/*
> * Release a reference obtained with dm_bufio_{read,get,new}. The data
> * pointer and dm_buffer pointer is no longer valid after this call.
> */

--
dm-devel mailing list
dm-devel@redhat.com
https://www.redhat.com/mailman/listinfo/dm-devel
 
Old 03-13-2012, 09:20 PM
Mikulas Patocka
 
Default dm: remake of the verity target

Hi

> Hi Mikulus,
>
> This is some nice work. I like that you've been able to abstract a lot
> of the hash buffer management with dm-bufio. You got rid of the I/O queue.
> I've been meaning to do that for a while. The prefetch is also nice.
> We planned to do this but I decided to not do it now in order to get the
> base functionality in:
>
> http://crosbug.com/25441
>
> However, there are some things that I don't like. I don't like comments
> either but you have none. You also removed our documentation. You are

I added some comments. As for documentation, it's OK to use documentation
from your patch because the on-disk format and the target arguments are
the same (with an enhancement that my code supports different data and
metadata bock size and it has variable-length salt).

> allocated a complete shash_desc per I/O. We only allocate one per CPU.

The hash of 4k block takes 174000 cycles. So trying to optimize
memory latency that is about 250 cycles doesn't make much sense.

I actually observed better performance using verity on ramdisk with
workqueue unbound to specific CPUs. The reason is that the ramdisk bio
completion routine is always run on the same CPU (that one that submitted
the request), so with bound workqueue, everything was executing on one
CPU. With unbound workqueue, I got parallelism.

> We short-circuit the hash at any level. Your implementation can only
> shirt circuit at the lowest level.

It short-circuits hash at all levels. If the function
"verity_verify_level" finds out that "aux->hash_verified" is non-zero, it
doesn't do any hashing, it just copies the hash for the lower level. My
implementation walks the tree from the top to the bottom, but it doesn't
do hash verification if the same block has been verified before.

All this tree-walking from the root to the bottom is 50-times faster than
the actual hashing of the data block (I measured that), so there's not
much point in trying to optimize it. I did a simple optimization (don't
walk the tree if the lowest block is already verified) and I don't need to
do anything complicated given the fact that it can't improve more than by
2%.

> I'd like to propose that we get the version we sent upstream and then work
> together on adding some of your enhancements incrementally.

If you add dm-bufio support, you end up deleting majority of the original
code anyway. That's why I wrote it from scratch and that's why I didn't
attempt to morph your code.

It's simpler to write the code from scratch and it is also less bug-prone.

> Other than
> the changes we've made to cleanup for upstreaming, the version I
> submitted is the code we are using in production.
>
> I'm happy to add prefetch now if that is required for merging.
>
> What do you think?
>
> Regards,
> Mandeep

This is the version with comments added:

Mikulas

----

Remake of the google dm-verity patch.

Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>

---
drivers/md/Kconfig | 17
drivers/md/Makefile | 1
drivers/md/dm-verity.c | 851 +++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 869 insertions(+)

Index: linux-3.3-rc6-fast/drivers/md/Kconfig
================================================== =================
--- linux-3.3-rc6-fast.orig/drivers/md/Kconfig 2012-03-13 21:46:03.000000000 +0100
+++ linux-3.3-rc6-fast/drivers/md/Kconfig 2012-03-13 21:46:05.000000000 +0100
@@ -404,4 +404,21 @@ config DM_VERITY2

If unsure, say N.

+config DM_VERITY
+ tristate "Verity target support"
+ depends on BLK_DEV_DM
+ select CRYPTO
+ select CRYPTO_HASH
+ select DM_BUFIO
+ ---help---
+ This device-mapper target allows you to create a device that
+ transparently integrity checks the data on it. You'll need to
+ activate the digests you're going to use in the cryptoapi
+ configuration.
+
+ To compile this code as a module, choose M here: the module will
+ be called dm-verity.
+
+ If unsure, say N.
+
endif # MD
Index: linux-3.3-rc6-fast/drivers/md/Makefile
================================================== =================
--- linux-3.3-rc6-fast.orig/drivers/md/Makefile 2012-03-13 21:46:03.000000000 +0100
+++ linux-3.3-rc6-fast/drivers/md/Makefile 2012-03-13 21:46:05.000000000 +0100
@@ -29,6 +29,7 @@ obj-$(CONFIG_MD_FAULTY) += faulty.o
obj-$(CONFIG_BLK_DEV_MD) += md-mod.o
obj-$(CONFIG_BLK_DEV_DM) += dm-mod.o
obj-$(CONFIG_DM_BUFIO) += dm-bufio.o
+obj-$(CONFIG_DM_VERITY) += dm-verity.o
obj-$(CONFIG_DM_CRYPT) += dm-crypt.o
obj-$(CONFIG_DM_DELAY) += dm-delay.o
obj-$(CONFIG_DM_FLAKEY) += dm-flakey.o
Index: linux-3.3-rc6-fast/drivers/md/dm-verity.c
================================================== =================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux-3.3-rc6-fast/drivers/md/dm-verity.c 2012-03-13 22:02:05.000000000 +0100
@@ -0,0 +1,851 @@
+/*
+ * Copyright (C) 2012 Red Hat, Inc.
+ *
+ * Author: Mikulas Patocka <mpatocka@redhat.com>
+ *
+ * Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors
+ *
+ * This file is released under the GPLv2.
+ *
+ * Device mapper target parameters:
+ * <version> 0
+ * <data device>
+ * <hash device>
+ * <hash start> (typically 0)
+ * <block size> (typically 4096)
+ * <algorithm>
+ * <digest>
+ * optional parameters:
+ * <salt> (should have 32 bytes for compatibility with Google code)
+ * <hash block size> (by default it is the same as data block size)
+ *
+ * In the file "/sys/module/dm_verity/parameters/prefetch_cluster" you can set
+ * default prefetch value. Data are read in "prefetch_cluster" chunks from the
+ * hash device. Prefetch cluster greatly improves performance when data and hash
+ * are on the same disk on different partitions.
+ */
+
+#include <linux/module.h>
+#include <linux/device-mapper.h>
+#include <crypto/hash.h>
+#include "dm-bufio.h"
+
+#define DM_MSG_PREFIX "verity"
+
+#define DM_VERITY_IO_VEC_INLINE 16
+#define DM_VERITY_MEMPOOL_SIZE 4
+#define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144
+
+#define DM_VERITY_MAX_LEVELS 63
+
+static unsigned prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE;
+
+module_param_named(prefetch_cluster, prefetch_cluster, uint, S_IRUGO | S_IWUSR);
+
+struct dm_verity {
+ struct dm_dev *data_dev;
+ struct dm_dev *hash_dev;
+ struct dm_target *ti;
+ struct dm_bufio_client *bufio;
+ char *alg_name;
+ struct crypto_shash *tfm;
+ u8 *root_digest; /* digest of the root block */
+ u8 *salt; /* salt, its size is salt_size */
+ unsigned salt_size;
+ sector_t data_start; /* data offset in 512-byte sectors */
+ sector_t hash_start; /* hash start in blocks */
+ sector_t data_blocks; /* the number of data blocks */
+ sector_t hash_blocks; /* the number of hash blocks */
+ unsigned char data_dev_block_bits; /* log2(data blocksize) */
+ unsigned char hash_dev_block_bits; /* log2(hash blocksize) */
+ unsigned char hash_per_block_bits; /* log2(hashes in hash block) */
+ unsigned char levels; /* the number of tree levels */
+ unsigned digest_size; /* digest size for the current hash algorithm */
+ unsigned shash_descsize;/* the size of temporary space for crypto */
+
+ mempool_t *io_mempool; /* mempool of struct dm_verity_io */
+ mempool_t *vec_mempool; /* mempool of bio vector */
+
+ struct workqueue_struct *verify_wq;
+
+ /* starting blocks for each tree level. 0 is the lowest level. */
+ sector_t hash_level_block[DM_VERITY_MAX_LEVELS];
+};
+
+struct dm_verity_io {
+ struct dm_verity *v;
+ struct bio *bio;
+
+ /* original values of bio->bi_end_io and bio->bi_private */
+ bio_end_io_t *orig_bi_end_io;
+ void *orig_bi_private;
+
+ sector_t block;
+ unsigned n_blocks;
+
+ /* saved bio vector */
+ struct bio_vec *io_vec;
+ unsigned io_vec_size;
+
+ struct work_struct work;
+
+ /* a space for short vectors; longer vectors are allocated separately */
+ struct bio_vec io_vec_inline[DM_VERITY_IO_VEC_INLINE];
+
+ /* variable-size fields, accessible with functions
+ io_hash_desc, io_real_digest, io_want_digest */
+ /* u8 hash_desc[crypto_shash_descsize(v->tfm)]; */
+ /* u8 real_digest[v->digest_size]; */
+ /* u8 want_digest[v->digest_size]; */
+};
+
+static struct shash_desc *io_hash_desc(struct dm_verity *v, struct dm_verity_io *io)
+{
+ return (struct shash_desc *)(io + 1);
+}
+
+static u8 *io_real_digest(struct dm_verity *v, struct dm_verity_io *io)
+{
+ return (u8 *)(io + 1) + v->shash_descsize;
+}
+
+static u8 *io_want_digest(struct dm_verity *v, struct dm_verity_io *io)
+{
+ return (u8 *)(io + 1) + v->shash_descsize + v->digest_size;
+}
+
+/*
+ * Auxiliary structure appended to each dm-bufio buffer. If the value
+ * hash_verified is nonzero, hash of the block has been verified.
+ *
+ * There is no lock around this value, a race condition can at worst cause
+ * that multiple processes verify the hash of the same buffer simultaneously.
+ * This condition is harmless, so we don't need locking.
+ */
+struct buffer_aux {
+ int hash_verified;
+};
+
+/*
+ * Initialize struct buffer_aux for a freshly created buffer.
+ */
+static void dm_bufio_alloc_callback(struct dm_buffer *buf)
+{
+ struct buffer_aux *aux = dm_bufio_get_aux_data(buf);
+ aux->hash_verified = 0;
+}
+
+/*
+ * Translate input sector number to the sector number on the target device.
+ */
+static sector_t verity_map_sector(struct dm_verity *v, sector_t bi_sector)
+{
+ return v->data_start + dm_target_offset(v->ti, bi_sector);
+}
+
+/*
+ * Return hash position of a specified block at a specified tree level
+ * (0 is the lowest level).
+ * The lowest "hash_per_block_bits"-bits of the result denote hash position
+ * inside a hash block. The remaining bits denode location of the hash block.
+ */
+static sector_t verity_position_at_level(struct dm_verity *v, sector_t block,
+ int level)
+{
+ return block >> (level * v->hash_per_block_bits);
+}
+
+static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level,
+ sector_t *hash_block, unsigned *offset)
+{
+ sector_t position = verity_position_at_level(v, block, level);
+
+ *hash_block = v->hash_level_block[level] + (position >> v->hash_per_block_bits);
+ if (offset)
+ *offset = v->digest_size * (position & ((1 << v->hash_per_block_bits) - 1));
+}
+
+/*
+ * Verify hash of a metadata block pertaining to the specified data block
+ * ("block" argument) at a specified level ("level" argument).
+ *
+ * On successful return, io_want_digest(v, io) contains the hash value for
+ * a lower tree level or for the data block (if we're at the lowest leve).
+ *
+ * If "skip_unverified" is true, unverified buffer is skipped an 1 is returned.
+ * If "skip_unverified" is false, unverified buffer is hashed and verified
+ * against current value of io_want_digest(v, io).
+ */
+static int verity_verify_level(struct dm_verity_io *io, sector_t block,
+ int level, bool skip_unverified)
+{
+ struct dm_verity *v = io->v;
+ struct dm_buffer *buf;
+ struct buffer_aux *aux;
+ u8 *data;
+ int r;
+ sector_t hash_block;
+ unsigned offset;
+
+ verity_hash_at_level(v, block, level, &hash_block, &offset);
+
+ data = dm_bufio_read(v->bufio, hash_block, &buf);
+ if (unlikely(IS_ERR(data)))
+ return PTR_ERR(data);
+
+ aux = dm_bufio_get_aux_data(buf);
+
+ if (!aux->hash_verified) {
+ struct shash_desc *desc;
+ u8 *result;
+
+ if (skip_unverified) {
+ r = 1;
+ goto release_ret_r;
+ }
+
+ desc = io_hash_desc(v, io);
+ desc->tfm = v->tfm;
+ desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
+ r = crypto_shash_init(desc);
+ if (r < 0) {
+ DMERR("crypto_shash_init failed: %d", r);
+ goto release_ret_r;
+ }
+
+ r = crypto_shash_update(desc, data, 1 << v->hash_dev_block_bits);
+ if (r < 0) {
+ DMERR("crypto_shash_update failed: %d", r);
+ goto release_ret_r;
+ }
+
+ r = crypto_shash_update(desc, v->salt, v->salt_size);
+ if (r < 0) {
+ DMERR("crypto_shash_update failed: %d", r);
+ goto release_ret_r;
+ }
+
+ result = io_real_digest(v, io);
+ r = crypto_shash_final(desc, result);
+ if (r < 0) {
+ DMERR("crypto_shash_final failed: %d", r);
+ goto release_ret_r;
+ }
+ if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) {
+ DMERR_LIMIT("metadata block %llu is corrupted",
+ (unsigned long long)hash_block);
+ r = -EIO;
+ goto release_ret_r;
+ } else
+ aux->hash_verified = 1;
+ }
+
+ data += offset;
+
+ memcpy(io_want_digest(v, io), data, v->digest_size);
+
+ dm_bufio_release(buf);
+ return 0;
+
+release_ret_r:
+ dm_bufio_release(buf);
+ return r;
+}
+
+/*
+ * Verify one "dm_verity_io" structure.
+ */
+static int verity_verify_io(struct dm_verity_io *io)
+{
+ struct dm_verity *v = io->v;
+ unsigned b;
+ int i;
+ unsigned vector = 0, offset = 0;
+ for (b = 0; b < io->n_blocks; b++) {
+ struct shash_desc *desc;
+ u8 *result;
+ int r;
+ unsigned todo;
+
+ if (likely(v->levels)) {
+ /*
+ * First, we try to get the requested hash for
+ * the current block. If the hash block itself is
+ * verified, zero is returned. If it isn't, this
+ * function returns 0 and we fall back to whole
+ * chain verification.
+ */
+ int r = verity_verify_level(io, io->block + b, 0, true);
+ if (likely(!r))
+ goto test_block_hash;
+ if (r < 0)
+ return r;
+ }
+
+ memcpy(io_want_digest(v, io), v->root_digest, v->digest_size);
+
+ for (i = v->levels - 1; i >= 0; i--) {
+ int r = verity_verify_level(io, io->block + b, i, false);
+ if (unlikely(r))
+ return r;
+ }
+
+test_block_hash:
+ desc = io_hash_desc(v, io);
+ desc->tfm = v->tfm;
+ desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
+ r = crypto_shash_init(desc);
+ if (r < 0) {
+ DMERR("crypto_shash_init failed: %d", r);
+ return r;
+ }
+
+ todo = 1 << v->data_dev_block_bits;
+ do {
+ struct bio_vec *bv;
+ u8 *page;
+ unsigned len;
+
+ BUG_ON(vector >= io->io_vec_size);
+ bv = &io->io_vec[vector];
+ page = kmap_atomic(bv->bv_page, KM_USER0);
+ len = bv->bv_len - offset;
+ if (likely(len >= todo))
+ len = todo;
+ r = crypto_shash_update(desc,
+ page + bv->bv_offset + offset, len);
+ kunmap_atomic(page, KM_USER0);
+ if (r < 0) {
+ DMERR("crypto_shash_update failed: %d", r);
+ return r;
+ }
+ offset += len;
+ if (likely(offset == bv->bv_len)) {
+ offset = 0;
+ vector++;
+ }
+ todo -= len;
+ } while (todo);
+
+ r = crypto_shash_update(desc, v->salt, v->salt_size);
+ if (r < 0) {
+ DMERR("crypto_shash_update failed: %d", r);
+ return r;
+ }
+
+ result = io_real_digest(v, io);
+ r = crypto_shash_final(desc, result);
+ if (r < 0) {
+ DMERR("crypto_shash_final failed: %d", r);
+ return r;
+ }
+ if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) {
+ DMERR_LIMIT("data block %llu is corrupted",
+ (unsigned long long)(io->block + b));
+ return -EIO;
+ }
+ }
+ BUG_ON(vector != io->io_vec_size);
+ BUG_ON(offset);
+ return 0;
+}
+
+/*
+ * End one "io" structure with a given error.
+ */
+static void verity_finish_io(struct dm_verity_io *io, int error)
+{
+ struct bio *bio = io->bio;
+ struct dm_verity *v = io->v;
+
+ bio->bi_end_io = io->orig_bi_end_io;
+ bio->bi_private = io->orig_bi_private;
+
+ if (io->io_vec != io->io_vec_inline)
+ mempool_free(io->io_vec, v->vec_mempool);
+ mempool_free(io, v->io_mempool);
+
+ bio_endio(bio, error);
+}
+
+static void verity_work(struct work_struct *w)
+{
+ struct dm_verity_io *io = container_of(w, struct dm_verity_io, work);
+
+ verity_finish_io(io, verity_verify_io(io));
+}
+
+static void verity_end_io(struct bio *bio, int error)
+{
+ struct dm_verity_io *io = bio->bi_private;
+ if (error) {
+ verity_finish_io(io, error);
+ return;
+ }
+
+ INIT_WORK(&io->work, verity_work);
+ queue_work(io->v->verify_wq, &io->work);
+}
+
+/*
+ * Prefetch buffers for the specified io.
+ * The root buffer is not prefetched, it is assumed that it will be cached
+ * all the time.
+ */
+static void verity_prefetch_io(struct dm_verity *v, struct dm_verity_io *io)
+{
+ int i;
+ for (i = v->levels - 2; i >= 0; i--) {
+ sector_t hash_block_start;
+ sector_t hash_block_end;
+ verity_hash_at_level(v, io->block, i, &hash_block_start, NULL);
+ verity_hash_at_level(v, io->block + io->n_blocks - 1, i, &hash_block_end, NULL);
+ if (!i) {
+ unsigned cluster = prefetch_cluster;
+ /* barrier to stop GCC from re-reading prefetch_cluster again */
+ barrier();
+ cluster >>= v->data_dev_block_bits;
+ if (unlikely(!cluster))
+ goto no_prefetch_cluster;
+ if (unlikely(cluster & (cluster - 1)))
+ cluster = 1 << (fls(cluster) - 1);
+
+ hash_block_start &= ~(sector_t)(cluster - 1);
+ hash_block_end |= cluster - 1;
+ if (unlikely(hash_block_end >= v->hash_blocks))
+ hash_block_end = v->hash_blocks - 1;
+ }
+no_prefetch_cluster:
+ dm_bufio_prefetch(v->bufio, hash_block_start,
+ hash_block_end - hash_block_start + 1);
+ }
+}
+
+/*
+ * Bio map function. It allocates dm_verity_io structure and bio vector and
+ * fills them. Then it issues prefetches and the I/O.
+ */
+static int verity_map(struct dm_target *ti, struct bio *bio,
+ union map_info *map_context)
+{
+ struct dm_verity *v = ti->private;
+ struct dm_verity_io *io;
+
+ if (((unsigned)bio->bi_sector | bio_sectors(bio)) &
+ ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) {
+ DMERR_LIMIT("unaligned io");
+ return -EIO;
+ }
+
+ if ((bio->bi_sector + bio_sectors(bio)) >>
+ (v->data_dev_block_bits - SECTOR_SHIFT) > v->data_blocks) {
+ DMERR_LIMIT("io out of range");
+ return -EIO;
+ }
+
+ if (bio_data_dir(bio) == WRITE)
+ return -EIO;
+
+ io = mempool_alloc(v->io_mempool, GFP_NOIO);
+ io->v = v;
+ io->bio = bio;
+ io->orig_bi_end_io = bio->bi_end_io;
+ io->orig_bi_private = bio->bi_private;
+ io->block = bio->bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT);
+ io->n_blocks = bio->bi_size >> v->data_dev_block_bits;
+
+ bio->bi_end_io = verity_end_io;
+ bio->bi_private = io;
+ bio->bi_bdev = v->data_dev->bdev;
+ bio->bi_sector = verity_map_sector(v, bio->bi_sector);
+
+ io->io_vec_size = bio->bi_vcnt - bio->bi_idx;
+ if (io->io_vec_size < DM_VERITY_IO_VEC_INLINE)
+ io->io_vec = io->io_vec_inline;
+ else
+ io->io_vec = mempool_alloc(v->vec_mempool, GFP_NOIO);
+ memcpy(io->io_vec, bio_iovec(bio),
+ io->io_vec_size * sizeof(struct bio_vec));
+
+ verity_prefetch_io(v, io);
+
+ generic_make_request(bio);
+
+ return DM_MAPIO_SUBMITTED;
+}
+
+static int verity_status(struct dm_target *ti, status_type_t type,
+ char *result, unsigned maxlen)
+{
+ struct dm_verity *v = ti->private;
+ unsigned sz = 0;
+ unsigned x;
+
+ switch (type) {
+ case STATUSTYPE_INFO:
+ result[0] = 0;
+ break;
+ case STATUSTYPE_TABLE:
+ DMEMIT("%u %s %s %llu %u %s ",
+ 0,
+ v->data_dev->name,
+ v->hash_dev->name,
+ (unsigned long long)v->hash_start << (v->hash_dev_block_bits - SECTOR_SHIFT),
+ 1 << v->data_dev_block_bits,
+ v->alg_name
+ );
+ for (x = 0; x < v->digest_size; x++)
+ DMEMIT("%02x", v->root_digest[x]);
+ DMEMIT(" ");
+ if (!v->salt_size)
+ DMEMIT("-");
+ else
+ for (x = 0; x < v->salt_size; x++)
+ DMEMIT("%02x", v->salt[x]);
+ if (v->data_dev_block_bits != v->hash_dev_block_bits)
+ DMEMIT(" %u", 1 << v->hash_dev_block_bits);
+ break;
+ }
+ return 0;
+}
+
+static int verity_ioctl(struct dm_target *ti, unsigned cmd,
+ unsigned long arg)
+{
+ struct dm_verity *v = ti->private;
+ int r = 0;
+
+ if (v->data_start ||
+ ti->len != i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT)
+ r = scsi_verify_blk_ioctl(NULL, cmd);
+
+ return r ? : __blkdev_driver_ioctl(v->data_dev->bdev, v->data_dev->mode,
+ cmd, arg);
+}
+
+static int verity_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
+ struct bio_vec *biovec, int max_size)
+{
+ struct dm_verity *v = ti->private;
+ struct request_queue *q = bdev_get_queue(v->data_dev->bdev);
+
+ if (!q->merge_bvec_fn)
+ return max_size;
+
+ bvm->bi_bdev = v->data_dev->bdev;
+ bvm->bi_sector = verity_map_sector(v, bvm->bi_sector);
+
+ return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
+}
+
+static int verity_iterate_devices(struct dm_target *ti,
+ iterate_devices_callout_fn fn, void *data)
+{
+ struct dm_verity *v = ti->private;
+ return fn(ti, v->data_dev, v->data_start, ti->len, data);
+}
+
+static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits)
+{
+ struct dm_verity *v = ti->private;
+
+ if (limits->logical_block_size < 1 << v->data_dev_block_bits)
+ limits->logical_block_size = 1 << v->data_dev_block_bits;
+ if (limits->physical_block_size < 1 << v->data_dev_block_bits)
+ limits->physical_block_size = 1 << v->data_dev_block_bits;
+ blk_limits_io_min(limits, limits->logical_block_size);
+}
+
+static void verity_dtr(struct dm_target *ti);
+
+static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv)
+{
+ struct dm_verity *v;
+ unsigned num;
+ unsigned long long hs;
+ int r;
+ int i;
+ sector_t hash_position;
+ char dummy;
+
+ v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL);
+ if (!v) {
+ ti->error = "Cannot allocate verity structure";
+ return -ENOMEM;
+ }
+ ti->private = v;
+ v->ti = ti;
+
+ if ((dm_table_get_mode(ti->table) & ~FMODE_READ) != 0) {
+ ti->error = "Device must be readonly";
+ r = -EINVAL;
+ goto bad;
+ }
+
+ if (argc < 7) {
+ ti->error = "Not enough arguments";
+ r = -EINVAL;
+ goto bad;
+ }
+
+ if (sscanf(argv[0], "%d%c", &num, &dummy) != 1 ||
+ num != 0) {
+ ti->error = "Invalid version";
+ r = -EINVAL;
+ goto bad;
+ }
+
+ r = dm_get_device(ti, argv[1], FMODE_READ, &v->data_dev);
+ if (r) {
+ ti->error = "Data device lookup failed";
+ goto bad;
+ }
+
+ r = dm_get_device(ti, argv[2], FMODE_READ, &v->hash_dev);
+ if (r) {
+ ti->error = "Data device lookup failed";
+ goto bad;
+ }
+
+ if (sscanf(argv[3], "%llu%c", &hs, &dummy) != 1 ||
+ hs != (sector_t)hs) {
+ ti->error = "Invalid hash start";
+ r = -EINVAL;
+ goto bad;
+ }
+
+ if (sscanf(argv[4], "%u%c", &num, &dummy) != 1 ||
+ !num || (num & (num - 1)) ||
+ num < bdev_logical_block_size(v->data_dev->bdev) ||
+ num > PAGE_SIZE) {
+ ti->error = "Invalid data device block size";
+ r = -EINVAL;
+ goto bad;
+ }
+ v->data_dev_block_bits = ffs(num) - 1;
+ v->hash_dev_block_bits = ffs(num) - 1;
+
+ v->alg_name = kstrdup(argv[5], GFP_KERNEL);
+ if (!v->alg_name) {
+ ti->error = "Cannot allocate algorithm name";
+ r = -ENOMEM;
+ goto bad;
+ }
+
+ v->tfm = crypto_alloc_shash(v->alg_name, 0, 0);
+ if (IS_ERR(v->tfm)) {
+ ti->error = "Cannot initialize hash function";
+ r = PTR_ERR(v->tfm);
+ v->tfm = NULL;
+ goto bad;
+ }
+ v->digest_size = crypto_shash_digestsize(v->tfm);
+ if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) {
+ ti->error = "Digest size too big";
+ r = -EINVAL;
+ goto bad;
+ }
+ v->shash_descsize =
+ sizeof(struct shash_desc) + crypto_shash_descsize(v->tfm);
+
+ v->root_digest = kmalloc(v->digest_size, GFP_KERNEL);
+ if (!v->root_digest) {
+ ti->error = "Cannot allocate root digest";
+ r = -ENOMEM;
+ goto bad;
+ }
+ if (strlen(argv[6]) != v->digest_size * 2 ||
+ hex2bin(v->root_digest, argv[6], v->digest_size)) {
+ ti->error = "Invalid root digest";
+ r = -EINVAL;
+ goto bad;
+ }
+
+ if (argc > 7 && strcmp(argv[7], "-")) {
+ v->salt_size = strlen(argv[7]) / 2;
+ v->salt = kmalloc(v->salt_size, GFP_KERNEL);
+ if (!v->salt) {
+ ti->error = "Cannot allocate salt";
+ r = -ENOMEM;
+ goto bad;
+ }
+ if (strlen(argv[7]) != v->salt_size * 2 ||
+ hex2bin(v->salt, argv[7], v->salt_size)) {
+ ti->error = "Invalid salt";
+ r = -EINVAL;
+ goto bad;
+ }
+ }
+
+ if (argc > 8) {
+ if (sscanf(argv[8], "%u%c", &num, &dummy) != 1 ||
+ !num || (num & (num - 1)) ||
+ num < bdev_logical_block_size(v->hash_dev->bdev) ||
+ num > INT_MAX) {
+ ti->error = "Invalid hash device block size";
+ r = -EINVAL;
+ goto bad;
+ }
+ v->hash_dev_block_bits = ffs(num) - 1;
+ }
+
+ if (hs & ((1 << (v->hash_dev_block_bits - SECTOR_SHIFT)) - 1)) {
+ ti->error = "Hash start not aligned on block boundary";
+ r = -EINVAL;
+ goto bad;
+ }
+ v->hash_start = hs >> (v->hash_dev_block_bits - SECTOR_SHIFT);
+
+ if (ti->len > i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT) {
+ ti->error = "Data device si too small";
+ r = -EINVAL;
+ goto bad;
+ }
+
+ if (ti->len & ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) {
+ ti->error = "Data device length is not aligned to block size";
+ r = -EINVAL;
+ goto bad;
+ }
+
+ v->data_blocks = ti->len >> (v->data_dev_block_bits - SECTOR_SHIFT);
+
+ v->hash_per_block_bits =
+ fls((1 << v->hash_dev_block_bits) / v->digest_size) - 1;
+
+ v->levels = 0;
+ if (v->data_blocks)
+ while (v->hash_per_block_bits * v->levels < 64 &&
+ (unsigned long long)(v->data_blocks - 1) >>
+ (v->hash_per_block_bits * v->levels))
+ v->levels++;
+
+ if (v->levels > DM_VERITY_MAX_LEVELS) {
+ ti->error = "Too many tree levels";
+ r = -E2BIG;
+ goto bad;
+ }
+
+ hash_position = v->hash_start;
+ for (i = v->levels - 1; i >= 0; i--) {
+ sector_t s;
+ v->hash_level_block[i] = hash_position;
+ s = verity_position_at_level(v, v->data_blocks, i);
+ s = (s >> v->hash_per_block_bits) +
+ !!(s & ((1 << v->hash_per_block_bits) - 1));
+ if (hash_position + s < hash_position) {
+ ti->error = "Hash device offset overflow";
+ r = -E2BIG;
+ goto bad;
+ }
+ hash_position += s;
+ }
+ v->hash_blocks = hash_position;
+
+ v->bufio = dm_bufio_client_create(v->hash_dev->bdev,
+ 1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux),
+ dm_bufio_alloc_callback, NULL);
+ if (IS_ERR(v->bufio)) {
+ ti->error = "Cannot initialize dm-bufio";
+ r = PTR_ERR(v->bufio);
+ v->bufio = NULL;
+ goto bad;
+ }
+
+ if (dm_bufio_get_device_size(v->bufio) < v->hash_blocks) {
+ ti->error = "Hash device is too small";
+ r = -E2BIG;
+ goto bad;
+ }
+
+ v->io_mempool = mempool_create_kmalloc_pool(DM_VERITY_MEMPOOL_SIZE ,
+ sizeof(struct dm_verity_io) + v->shash_descsize + v->digest_size * 2);
+ if (!v->io_mempool) {
+ ti->error = "Cannot allocate io mempool";
+ r = -ENOMEM;
+ goto bad;
+ }
+
+ v->vec_mempool = mempool_create_kmalloc_pool(DM_VERITY_MEMPOOL_SIZE ,
+ BIO_MAX_PAGES * sizeof(struct bio_vec));
+ if (!v->vec_mempool) {
+ ti->error = "Cannot allocate vector mempool";
+ r = -ENOMEM;
+ goto bad;
+ }
+
+ /*v->verify_wq = alloc_workqueue("verityd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 1);*/
+ /* WQ_UNBOUND greatly improves performance when running on ramdisk */
+ v->verify_wq = alloc_workqueue("verityd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, num_online_cpus());
+ if (!v->verify_wq) {
+ ti->error = "Cannot allocate workqueue";
+ r = -ENOMEM;
+ goto bad;
+ }
+
+ return 0;
+
+bad:
+ verity_dtr(ti);
+ return r;
+}
+
+static void verity_dtr(struct dm_target *ti)
+{
+ struct dm_verity *v = ti->private;
+
+ if (v->verify_wq)
+ destroy_workqueue(v->verify_wq);
+ if (v->vec_mempool)
+ mempool_destroy(v->vec_mempool);
+ if (v->io_mempool)
+ mempool_destroy(v->io_mempool);
+ if (v->bufio)
+ dm_bufio_client_destroy(v->bufio);
+ kfree(v->salt);
+ kfree(v->root_digest);
+ if (v->tfm)
+ crypto_free_shash(v->tfm);
+ kfree(v->alg_name);
+ if (v->hash_dev)
+ dm_put_device(ti, v->hash_dev);
+ if (v->data_dev)
+ dm_put_device(ti, v->data_dev);
+ kfree(v);
+}
+
+static struct target_type verity_target = {
+ .name = "verity",
+ .version = {1, 0, 0},
+ .module = THIS_MODULE,
+ .ctr = verity_ctr,
+ .dtr = verity_dtr,
+ .map = verity_map,
+ .status = verity_status,
+ .ioctl = verity_ioctl,
+ .merge = verity_merge,
+ .iterate_devices = verity_iterate_devices,
+ .io_hints = verity_io_hints,
+};
+
+static int __init dm_verity_init(void)
+{
+ int r;
+ r = dm_register_target(&verity_target);
+ if (r < 0)
+ DMERR("register failed %d", r);
+ return r;
+}
+
+static void __exit dm_verity_exit(void)
+{
+ dm_unregister_target(&verity_target);
+}
+
+module_init(dm_verity_init);
+module_exit(dm_verity_exit);
+
+MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>");
+MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking");
+MODULE_LICENSE("GPL");
+
Index: linux-3.3-rc6-fast/drivers/md/dm-bufio.c
================================================== =================
--- linux-3.3-rc6-fast.orig/drivers/md/dm-bufio.c 2012-03-12 22:43:23.000000000 +0100
+++ linux-3.3-rc6-fast/drivers/md/dm-bufio.c 2012-03-13 15:41:02.000000000 +0100
@@ -579,7 +579,7 @@ static void write_endio(struct bio *bio,
struct dm_buffer *b = container_of(bio, struct dm_buffer, bio);

b->write_error = error;
- if (error) {
+ if (unlikely(error)) {
struct dm_bufio_client *c = b->c;
(void)cmpxchg(&c->async_write_error, 0, error);
}
@@ -698,13 +698,20 @@ static void __wait_for_free_buffer(struc
dm_bufio_lock(c);
}

+enum new_flag {
+ NF_FRESH = 0,
+ NF_READ = 1,
+ NF_GET = 2,
+ NF_PREFETCH = 3
+};
+
/*
* Allocate a new buffer. If the allocation is not possible, wait until
* some other thread frees a buffer.
*
* May drop the lock and regain it.
*/
-static struct dm_buffer *__alloc_buffer_wait_no_callback(struct dm_bufio_client *c)
+static struct dm_buffer *__alloc_buffer_wait_no_callback(struct dm_bufio_client *c, enum new_flag nf)
{
struct dm_buffer *b;

@@ -727,6 +734,9 @@ static struct dm_buffer *__alloc_buffer_
return b;
}

+ if (nf == NF_PREFETCH)
+ return NULL;
+
if (!list_empty(&c->reserved_buffers)) {
b = list_entry(c->reserved_buffers.next,
struct dm_buffer, lru_list);
@@ -744,9 +754,12 @@ static struct dm_buffer *__alloc_buffer_
}
}

-static struct dm_buffer *__alloc_buffer_wait(struct dm_bufio_client *c)
+static struct dm_buffer *__alloc_buffer_wait(struct dm_bufio_client *c, enum new_flag nf)
{
- struct dm_buffer *b = __alloc_buffer_wait_no_callback(c);
+ struct dm_buffer *b = __alloc_buffer_wait_no_callback(c, nf);
+
+ if (!b)
+ return NULL;

if (c->alloc_callback)
c->alloc_callback(b);
@@ -866,15 +879,8 @@ static struct dm_buffer *__find(struct d
* Getting a buffer
*--------------------------------------------------------------*/

-enum new_flag {
- NF_FRESH = 0,
- NF_READ = 1,
- NF_GET = 2
-};
-
static struct dm_buffer *__bufio_new(struct dm_bufio_client *c, sector_t block,
- enum new_flag nf, struct dm_buffer **bp,
- int *need_submit)
+ enum new_flag nf, int *need_submit)
{
struct dm_buffer *b, *new_b = NULL;

@@ -882,6 +888,19 @@ static struct dm_buffer *__bufio_new(str

b = __find(c, block);
if (b) {
+found_buffer:
+ if (nf == NF_PREFETCH)
+ return NULL;
+ /*
+ * Note: it is essential that we don't wait for the buffer to be
+ * read if dm_bufio_get function is used. Both dm_bufio_get and
+ * dm_bufio_prefetch can be used in the driver request routine.
+ * If the user called both dm_bufio_prefetch and dm_bufio_get on
+ * the same buffer, it would deadlock if we waited.
+ */
+ if (nf == NF_GET && unlikely(test_bit(B_READING, &b->state)))
+ return NULL;
+
b->hold_count++;
__relink_lru(b, test_bit(B_DIRTY, &b->state) ||
test_bit(B_WRITING, &b->state));
@@ -891,7 +910,9 @@ static struct dm_buffer *__bufio_new(str
if (nf == NF_GET)
return NULL;

- new_b = __alloc_buffer_wait(c);
+ new_b = __alloc_buffer_wait(c, nf);
+ if (!new_b)
+ return NULL;

/*
* We've had a period where the mutex was unlocked, so need to
@@ -900,10 +921,7 @@ static struct dm_buffer *__bufio_new(str
b = __find(c, block);
if (b) {
__free_buffer_wake(new_b);
- b->hold_count++;
- __relink_lru(b, test_bit(B_DIRTY, &b->state) ||
- test_bit(B_WRITING, &b->state));
- return b;
+ goto found_buffer;
}

__check_watermark(c);
@@ -957,7 +975,7 @@ static void *new_read(struct dm_bufio_cl
struct dm_buffer *b;

dm_bufio_lock(c);
- b = __bufio_new(c, block, nf, bp, &need_submit);
+ b = __bufio_new(c, block, nf, &need_submit);
dm_bufio_unlock(c);

if (!b || IS_ERR(b))
@@ -1006,13 +1024,46 @@ void *dm_bufio_new(struct dm_bufio_clien
}
EXPORT_SYMBOL_GPL(dm_bufio_new);

+void dm_bufio_prefetch(struct dm_bufio_client *c,
+ sector_t block, unsigned n_blocks)
+{
+ struct blk_plug plug;
+
+ blk_start_plug(&plug);
+ dm_bufio_lock(c);
+
+ for (; n_blocks--; block++) {
+ int need_submit;
+ struct dm_buffer *b;
+ b = __bufio_new(c, block, NF_PREFETCH, &need_submit);
+ if (unlikely(b != NULL)) {
+ dm_bufio_unlock(c);
+
+ if (need_submit)
+ submit_io(b, READ, b->block, read_endio);
+ dm_bufio_release(b);
+
+ dm_bufio_cond_resched();
+
+ if (!n_blocks)
+ goto flush_plug;
+ dm_bufio_lock(c);
+ }
+
+ }
+
+ dm_bufio_unlock(c);
+flush_plug:
+ blk_finish_plug(&plug);
+}
+EXPORT_SYMBOL_GPL(dm_bufio_prefetch);
+
void dm_bufio_release(struct dm_buffer *b)
{
struct dm_bufio_client *c = b->c;

dm_bufio_lock(c);

- BUG_ON(test_bit(B_READING, &b->state));
BUG_ON(!b->hold_count);

b->hold_count--;
@@ -1025,6 +1076,7 @@ void dm_bufio_release(struct dm_buffer *
* invalid buffer.
*/
if ((b->read_error || b->write_error) &&
+ !test_bit(B_READING, &b->state) &&
!test_bit(B_WRITING, &b->state) &&
!test_bit(B_DIRTY, &b->state)) {
__unlink_buffer(b);
@@ -1042,6 +1094,8 @@ void dm_bufio_mark_buffer_dirty(struct d

dm_bufio_lock(c);

+ BUG_ON(test_bit(B_READING, &b->state));
+
if (!test_and_set_bit(B_DIRTY, &b->state))
__relink_lru(b, LIST_DIRTY);

Index: linux-3.3-rc6-fast/drivers/md/dm-bufio.h
================================================== =================
--- linux-3.3-rc6-fast.orig/drivers/md/dm-bufio.h 2012-03-12 22:43:23.000000000 +0100
+++ linux-3.3-rc6-fast/drivers/md/dm-bufio.h 2012-03-12 22:43:25.000000000 +0100
@@ -63,6 +63,14 @@ void *dm_bufio_new(struct dm_bufio_clien
struct dm_buffer **bp);

/*
+ * Prefetch the specified blocks to the cache.
+ * The function starts to read the blocks and returns without waiting for
+ * I/O to finish.
+ */
+void dm_bufio_prefetch(struct dm_bufio_client *c,
+ sector_t block, unsigned n_blocks);
+
+/*
* Release a reference obtained with dm_bufio_{read,get,new}. The data
* pointer and dm_buffer pointer is no longer valid after this call.
*/

--
dm-devel mailing list
dm-devel@redhat.com
https://www.redhat.com/mailman/listinfo/dm-devel
 
Old 03-14-2012, 08:13 PM
Will Drewry
 
Default dm: remake of the verity target

Hi Mikulas,

This is a nice rewrite and takes advantage of your dm-bufio layer. I
wish it'd existed (and or we wrote it in 2009 when we started this
work! Some comments below:

On Tue, Mar 13, 2012 at 5:20 PM, Mikulas Patocka <mpatocka@redhat.com> wrote:
> Hi
>
>> Hi Mikulus,
>>
>> This is some nice work. I like that you've been able to abstract a lot
>> of the hash buffer management with dm-bufio. You got rid of the I/O queue.
>> I've been meaning to do that for a while. The prefetch is also nice.
>> We planned to do this but I decided to not do it now in order to get the
>> base functionality in:
>>
>> http://crosbug.com/25441
>>
>> However, there are some things that I don't like. I don't like comments
>> either but you have none. You also removed our documentation. You are
>
> I added some comments. As for documentation, it's OK to use documentation
> from your patch because the on-disk format and the target arguments are
> the same (with an enhancement that my code supports different data and
> metadata bock size and it has variable-length salt).

Sounds good.

>> allocated a complete shash_desc per I/O. We only allocate one per CPU.
>
> The hash of 4k block takes 174000 cycles. So trying to optimize
> memory latency that is about 250 cycles doesn't make much sense.
>
> I actually observed better performance using verity on ramdisk with
> workqueue unbound to specific CPUs. The reason is that the ramdisk bio
> completion routine is always run on the same CPU (that one that submitted
> the request), so with bound workqueue, everything was executing on one
> CPU. With unbound workqueue, I got parallelism.
>
>> We short-circuit the hash at any level. Your implementation can only
>> shirt circuit at the lowest level.
>
> It short-circuits hash at all levels. If the function
> "verity_verify_level" finds out that "aux->hash_verified" is non-zero, it
> doesn't do any hashing, it just copies the hash for the lower level. My
> implementation walks the tree from the top to the bottom, but it doesn't
> do hash verification if the same block has been verified before.
>
> All this tree-walking from the root to the bottom is 50-times faster than
> the actual hashing of the data block (I measured that), so there's not
> much point in trying to optimize it. I did a simple optimization (don't
> walk the tree if the lowest block is already verified) and I don't need to
> do anything complicated given the fact that it can't improve more than by
> 2%.

All we'd done was reverse the walk (we've had it both ways now ,
nothing complicated, but I don't think it's a problem.

>> I'd like to propose that we get the version we sent upstream and then work
>> together on adding some of your enhancements incrementally.
>
> If you add dm-bufio support, you end up deleting majority of the original
> code anyway. That's why I wrote it from scratch and that's why I didn't
> attempt to morph your code.
>
> It's simpler to write the code from scratch and it is also less bug-prone.
>
>> Other than
>> the changes we've made to cleanup for upstreaming, the version I
>> submitted is the code we are using in production.
>>
>> I'm happy to add prefetch now if that is required for merging.
>>
>> What do you think?
>>
>> Regards,
>> Mandeep
>
> This is the version with comments added:
>
> Mikulas
>
> ----
>
> Remake of the google dm-verity patch.
>
> Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
>
> ---
> *drivers/md/Kconfig * * | * 17
> *drivers/md/Makefile * *| * *1
> *drivers/md/dm-verity.c | *851 +++++++++++++++++++++++++++++++++++++++++++++++++
> *3 files changed, 869 insertions(+)
>
> Index: linux-3.3-rc6-fast/drivers/md/Kconfig
> ================================================== =================
> --- linux-3.3-rc6-fast.orig/drivers/md/Kconfig *2012-03-13 21:46:03.000000000 +0100
> +++ linux-3.3-rc6-fast/drivers/md/Kconfig * * * 2012-03-13 21:46:05.000000000 +0100
> @@ -404,4 +404,21 @@ config DM_VERITY2
>
> * * * * * If unsure, say N.
>
> +config DM_VERITY
> + * * * tristate "Verity target support"
> + * * * depends on BLK_DEV_DM
> + * * * select CRYPTO
> + * * * select CRYPTO_HASH
> + * * * select DM_BUFIO
> + * * * ---help---
> + * * * * This device-mapper target allows you to create a device that
> + * * * * transparently integrity checks the data on it. You'll need to
> + * * * * activate the digests you're going to use in the cryptoapi
> + * * * * configuration.
> +
> + * * * * To compile this code as a module, choose M here: the module will
> + * * * * be called dm-verity.
> +
> + * * * * If unsure, say N.
> +
> *endif # MD
> Index: linux-3.3-rc6-fast/drivers/md/Makefile
> ================================================== =================
> --- linux-3.3-rc6-fast.orig/drivers/md/Makefile 2012-03-13 21:46:03.000000000 +0100
> +++ linux-3.3-rc6-fast/drivers/md/Makefile * * *2012-03-13 21:46:05.000000000 +0100
> @@ -29,6 +29,7 @@ obj-$(CONFIG_MD_FAULTY) * * * * * * * += faulty.o
> *obj-$(CONFIG_BLK_DEV_MD) * * * += md-mod.o
> *obj-$(CONFIG_BLK_DEV_DM) * * * += dm-mod.o
> *obj-$(CONFIG_DM_BUFIO) * * * * += dm-bufio.o
> +obj-$(CONFIG_DM_VERITY) * * * * * * * *+= dm-verity.o
> *obj-$(CONFIG_DM_CRYPT) * * * * += dm-crypt.o
> *obj-$(CONFIG_DM_DELAY) * * * * += dm-delay.o
> *obj-$(CONFIG_DM_FLAKEY) * * * * * * * *+= dm-flakey.o
> Index: linux-3.3-rc6-fast/drivers/md/dm-verity.c
> ================================================== =================
> --- /dev/null * 1970-01-01 00:00:00.000000000 +0000
> +++ linux-3.3-rc6-fast/drivers/md/dm-verity.c * 2012-03-13 22:02:05.000000000 +0100
> @@ -0,0 +1,851 @@
> +/*
> + * Copyright (C) 2012 Red Hat, Inc.
> + *
> + * Author: Mikulas Patocka <mpatocka@redhat.com>
> + *
> + * Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors
> + *
> + * This file is released under the GPLv2.
> + *
> + * Device mapper target parameters:
> + * * * <version> * * * 0
> + * * * <data device>
> + * * * <hash device>
> + * * * <hash start> * *(typically 0)
> + * * * <block size> * *(typically 4096)
> + * * * <algorithm>
> + * * * <digest>
> + * * * optional parameters:
> + * * * * * * * <salt> (should have 32 bytes for compatibility with Google code)
> + * * * * * * * <hash block size> (by default it is the same as data block size)
> + *
> + * In the file "/sys/module/dm_verity/parameters/prefetch_cluster" you can set
> + * default prefetch value. Data are read in "prefetch_cluster" chunks from the
> + * hash device. Prefetch cluster greatly improves performance when data and hash
> + * are on the same disk on different partitions.

... on different partitions on devices with poor random access behavior.

> + */
> +
> +#include <linux/module.h>
> +#include <linux/device-mapper.h>
> +#include <crypto/hash.h>
> +#include "dm-bufio.h"
> +
> +#define DM_MSG_PREFIX * * * * * * * * *"verity"
> +
> +#define DM_VERITY_IO_VEC_INLINE * * * * * * * *16
> +#define DM_VERITY_MEMPOOL_SIZE * * * * 4
> +#define DM_VERITY_DEFAULT_PREFETCH_SIZE * * * *262144
> +
> +#define DM_VERITY_MAX_LEVELS * * * * * 63
> +
> +static unsigned prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE;
> +
> +module_param_named(prefetch_cluster, prefetch_cluster, uint, S_IRUGO | S_IWUSR);
> +
> +struct dm_verity {
> + * * * struct dm_dev *data_dev;
> + * * * struct dm_dev *hash_dev;
> + * * * struct dm_target *ti;
> + * * * struct dm_bufio_client *bufio;
> + * * * char *alg_name;
> + * * * struct crypto_shash *tfm;
> + * * * u8 *root_digest; * * * */* digest of the root block */
> + * * * u8 *salt; * * * * * * * /* salt, its size is salt_size */
> + * * * unsigned salt_size;
> + * * * sector_t data_start; * */* data offset in 512-byte sectors */
> + * * * sector_t hash_start; * */* hash start in blocks */
> + * * * sector_t data_blocks; * /* the number of data blocks */
> + * * * sector_t hash_blocks; * /* the number of hash blocks */
> + * * * unsigned char data_dev_block_bits; * * */* log2(data blocksize) */
> + * * * unsigned char hash_dev_block_bits; * * */* log2(hash blocksize) */
> + * * * unsigned char hash_per_block_bits; * * */* log2(hashes in hash block) */
> + * * * unsigned char levels; * /* the number of tree levels */
> + * * * unsigned digest_size; * /* digest size for the current hash algorithm */
> + * * * unsigned shash_descsize;/* the size of temporary space for crypto */
> +
> + * * * mempool_t *io_mempool; */* mempool of struct dm_verity_io */
> + * * * mempool_t *vec_mempool; /* mempool of bio vector */
> +
> + * * * struct workqueue_struct *verify_wq;
> +
> + * * * /* starting blocks for each tree level. 0 is the lowest level. */
> + * * * sector_t hash_level_block[DM_VERITY_MAX_LEVELS];
> +};
> +
> +struct dm_verity_io {
> + * * * struct dm_verity *v;
> + * * * struct bio *bio;
> +
> + * * * /* original values of bio->bi_end_io and bio->bi_private */
> + * * * bio_end_io_t *orig_bi_end_io;
> + * * * void *orig_bi_private;
> +
> + * * * sector_t block;
> + * * * unsigned n_blocks;
> +
> + * * * /* saved bio vector */
> + * * * struct bio_vec *io_vec;
> + * * * unsigned io_vec_size;
> +
> + * * * struct work_struct work;
> +
> + * * * /* a space for short vectors; longer vectors are allocated separately */
> + * * * struct bio_vec io_vec_inline[DM_VERITY_IO_VEC_INLINE];
> +
> + * * * /* variable-size fields, accessible with functions
> + * * * * * * * io_hash_desc, io_real_digest, io_want_digest */
> + * * * /* u8 hash_desc[crypto_shash_descsize(v->tfm)]; */
> + * * * /* u8 real_digest[v->digest_size]; */
> + * * * /* u8 want_digest[v->digest_size]; */
> +};
> +
> +static struct shash_desc *io_hash_desc(struct dm_verity *v, struct dm_verity_io *io)
> +{
> + * * * return (struct shash_desc *)(io + 1);
> +}
> +
> +static u8 *io_real_digest(struct dm_verity *v, struct dm_verity_io *io)
> +{
> + * * * return (u8 *)(io + 1) + v->shash_descsize;
> +}
> +
> +static u8 *io_want_digest(struct dm_verity *v, struct dm_verity_io *io)
> +{
> + * * * return (u8 *)(io + 1) + v->shash_descsize + v->digest_size;
> +}
> +
> +/*
> + * Auxiliary structure appended to each dm-bufio buffer. If the value
> + * hash_verified is nonzero, hash of the block has been verified.
> + *
> + * There is no lock around this value, a race condition can at worst cause
> + * that multiple processes verify the hash of the same buffer simultaneously.
> + * This condition is harmless, so we don't need locking.
> + */

Might be worth clarifying that no consumer will ever write a 0 value
to the hash_verified field after dm_bufio_alloc_callback, as that is
the critical constraint. It's what makes lockless/atomic-less access
acceptable. As you say, the worst case you over verify or you
over-write.

bufio is nice for this use since it encapsulates the need for the
atomic state transitions we were using in our module to stay
lock-free.

> +struct buffer_aux {
> + * * * int hash_verified;
> +};
> +
> +/*
> + * Initialize struct buffer_aux for a freshly created buffer.
> + */
> +static void dm_bufio_alloc_callback(struct dm_buffer *buf)
> +{
> + * * * struct buffer_aux *aux = dm_bufio_get_aux_data(buf);
> + * * * aux->hash_verified = 0;
> +}
> +
> +/*
> + * Translate input sector number to the sector number on the target device.
> + */
> +static sector_t verity_map_sector(struct dm_verity *v, sector_t bi_sector)
> +{
> + * * * return v->data_start + dm_target_offset(v->ti, bi_sector);
> +}
> +
> +/*
> + * Return hash position of a specified block at a specified tree level
> + * (0 is the lowest level).
> + * The lowest "hash_per_block_bits"-bits of the result denote hash position
> + * inside a hash block. The remaining bits denode location of the hash block.

s/denode/denote/

> + */
> +static sector_t verity_position_at_level(struct dm_verity *v, sector_t block,
> + * * * * * * * * * * * * * * * * * * * *int level)
> +{
> + * * * return block >> (level * v->hash_per_block_bits);
> +}
> +
> +static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level,
> + * * * * * * * * * * * * * * * *sector_t *hash_block, unsigned *offset)
> +{
> + * * * sector_t position = verity_position_at_level(v, block, level);
> +
> + * * * *hash_block = v->hash_level_block[level] + (position >> v->hash_per_block_bits);
> + * * * if (offset)
> + * * * * * * * *offset = v->digest_size * (position & ((1 << v->hash_per_block_bits) - 1));
> +}
> +
> +/*
> + * Verify hash of a metadata block pertaining to the specified data block
> + * ("block" argument) at a specified level ("level" argument).
> + *
> + * On successful return, io_want_digest(v, io) contains the hash value for
> + * a lower tree level or for the data block (if we're at the lowest leve).
> + *
> + * If "skip_unverified" is true, unverified buffer is skipped an 1 is returned.
> + * If "skip_unverified" is false, unverified buffer is hashed and verified
> + * against current value of io_want_digest(v, io).
> + */
> +static int verity_verify_level(struct dm_verity_io *io, sector_t block,
> + * * * * * * * * * * * * * * *int level, bool skip_unverified)
> +{
> + * * * struct dm_verity *v = io->v;
> + * * * struct dm_buffer *buf;
> + * * * struct buffer_aux *aux;
> + * * * u8 *data;
> + * * * int r;
> + * * * sector_t hash_block;
> + * * * unsigned offset;
> +
> + * * * verity_hash_at_level(v, block, level, &hash_block, &offset);
> +
> + * * * data = dm_bufio_read(v->bufio, hash_block, &buf);
> + * * * if (unlikely(IS_ERR(data)))
> + * * * * * * * return PTR_ERR(data);
> +
> + * * * aux = dm_bufio_get_aux_data(buf);
> +
> + * * * if (!aux->hash_verified) {
> + * * * * * * * struct shash_desc *desc;
> + * * * * * * * u8 *result;
> +
> + * * * * * * * if (skip_unverified) {
> + * * * * * * * * * * * r = 1;
> + * * * * * * * * * * * goto release_ret_r;
> + * * * * * * * }
> +
> + * * * * * * * desc = io_hash_desc(v, io);
> + * * * * * * * desc->tfm = v->tfm;
> + * * * * * * * desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
> + * * * * * * * r = crypto_shash_init(desc);
> + * * * * * * * if (r < 0) {
> + * * * * * * * * * * * DMERR("crypto_shash_init failed: %d", r);
> + * * * * * * * * * * * goto release_ret_r;
> + * * * * * * * }
> +
> + * * * * * * * r = crypto_shash_update(desc, data, 1 << v->hash_dev_block_bits);
> + * * * * * * * if (r < 0) {
> + * * * * * * * * * * * DMERR("crypto_shash_update failed: %d", r);
> + * * * * * * * * * * * goto release_ret_r;
> + * * * * * * * }
> +
> + * * * * * * * r = crypto_shash_update(desc, v->salt, v->salt_size);
> + * * * * * * * if (r < 0) {
> + * * * * * * * * * * * DMERR("crypto_shash_update failed: %d", r);
> + * * * * * * * * * * * goto release_ret_r;
> + * * * * * * * }
> +
> + * * * * * * * result = io_real_digest(v, io);
> + * * * * * * * r = crypto_shash_final(desc, result);
> + * * * * * * * if (r < 0) {
> + * * * * * * * * * * * DMERR("crypto_shash_final failed: %d", r);
> + * * * * * * * * * * * goto release_ret_r;
> + * * * * * * * }
> + * * * * * * * if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) {
> + * * * * * * * * * * * DMERR_LIMIT("metadata block %llu is corrupted",
> + * * * * * * * * * * * * * * * (unsigned long long)hash_block);
> + * * * * * * * * * * * r = -EIO;
> + * * * * * * * * * * * goto release_ret_r;
> + * * * * * * * } else
> + * * * * * * * * * * * aux->hash_verified = 1;
> + * * * }
> +
> + * * * data += offset;
> +
> + * * * memcpy(io_want_digest(v, io), data, v->digest_size);
> +
> + * * * dm_bufio_release(buf);
> + * * * return 0;
> +
> +release_ret_r:
> + * * * dm_bufio_release(buf);
> + * * * return r;
> +}
> +
> +/*
> + * Verify one "dm_verity_io" structure.
> + */
> +static int verity_verify_io(struct dm_verity_io *io)
> +{
> + * * * struct dm_verity *v = io->v;
> + * * * unsigned b;
> + * * * int i;
> + * * * unsigned vector = 0, offset = 0;
> + * * * for (b = 0; b < io->n_blocks; b++) {
> + * * * * * * * struct shash_desc *desc;
> + * * * * * * * u8 *result;
> + * * * * * * * int r;
> + * * * * * * * unsigned todo;
> +
> + * * * * * * * if (likely(v->levels)) {
> + * * * * * * * * * * * /*
> + * * * * * * * * * * * ** First, we try to get the requested hash for
> + * * * * * * * * * * * ** the current block. If the hash block itself is
> + * * * * * * * * * * * ** verified, zero is returned. If it isn't, this
> + * * * * * * * * * * * ** function returns 0 and we fall back to whole
> + * * * * * * * * * * * ** chain verification.
> + * * * * * * * * * * * **/
> + * * * * * * * * * * * int r = verity_verify_level(io, io->block + b, 0, true);
> + * * * * * * * * * * * if (likely(!r))
> + * * * * * * * * * * * * * * * goto test_block_hash;
> + * * * * * * * * * * * if (r < 0)
> + * * * * * * * * * * * * * * * return r;
> + * * * * * * * }
> +
> + * * * * * * * memcpy(io_want_digest(v, io), v->root_digest, v->digest_size);
> +
> + * * * * * * * for (i = v->levels - 1; i >= 0; i--) {
> + * * * * * * * * * * * int r = verity_verify_level(io, io->block + b, i, false);
> + * * * * * * * * * * * if (unlikely(r))
> + * * * * * * * * * * * * * * * return r;
> + * * * * * * * }
> +
> +test_block_hash:
> + * * * * * * * desc = io_hash_desc(v, io);
> + * * * * * * * desc->tfm = v->tfm;
> + * * * * * * * desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
> + * * * * * * * r = crypto_shash_init(desc);
> + * * * * * * * if (r < 0) {
> + * * * * * * * * * * * DMERR("crypto_shash_init failed: %d", r);
> + * * * * * * * * * * * return r;
> + * * * * * * * }
> +
> + * * * * * * * todo = 1 << v->data_dev_block_bits;
> + * * * * * * * do {
> + * * * * * * * * * * * struct bio_vec *bv;
> + * * * * * * * * * * * u8 *page;
> + * * * * * * * * * * * unsigned len;
> +
> + * * * * * * * * * * * BUG_ON(vector >= io->io_vec_size);
> + * * * * * * * * * * * bv = &io->io_vec[vector];
> + * * * * * * * * * * * page = kmap_atomic(bv->bv_page, KM_USER0);
> + * * * * * * * * * * * len = bv->bv_len - offset;
> + * * * * * * * * * * * if (likely(len >= todo))
> + * * * * * * * * * * * * * * * len = todo;
> + * * * * * * * * * * * r = crypto_shash_update(desc,
> + * * * * * * * * * * * * * * * * * * * page + bv->bv_offset + offset, len);
> + * * * * * * * * * * * kunmap_atomic(page, KM_USER0);
> + * * * * * * * * * * * if (r < 0) {
> + * * * * * * * * * * * * * * * DMERR("crypto_shash_update failed: %d", r);
> + * * * * * * * * * * * * * * * return r;
> + * * * * * * * * * * * }
> + * * * * * * * * * * * offset += len;
> + * * * * * * * * * * * if (likely(offset == bv->bv_len)) {
> + * * * * * * * * * * * * * * * offset = 0;
> + * * * * * * * * * * * * * * * vector++;
> + * * * * * * * * * * * }
> + * * * * * * * * * * * todo -= len;
> + * * * * * * * } while (todo);
> +
> + * * * * * * * r = crypto_shash_update(desc, v->salt, v->salt_size);
> + * * * * * * * if (r < 0) {
> + * * * * * * * * * * * DMERR("crypto_shash_update failed: %d", r);
> + * * * * * * * * * * * return r;
> + * * * * * * * }
> +
> + * * * * * * * result = io_real_digest(v, io);
> + * * * * * * * r = crypto_shash_final(desc, result);
> + * * * * * * * if (r < 0) {
> + * * * * * * * * * * * DMERR("crypto_shash_final failed: %d", r);
> + * * * * * * * * * * * return r;
> + * * * * * * * }
> + * * * * * * * if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) {
> + * * * * * * * * * * * DMERR_LIMIT("data block %llu is corrupted",
> + * * * * * * * * * * * * * * * (unsigned long long)(io->block + b));
> + * * * * * * * * * * * return -EIO;
> + * * * * * * * }
> + * * * }
> + * * * BUG_ON(vector != io->io_vec_size);
> + * * * BUG_ON(offset);
> + * * * return 0;
> +}
> +
> +/*
> + * End one "io" structure with a given error.
> + */
> +static void verity_finish_io(struct dm_verity_io *io, int error)
> +{
> + * * * struct bio *bio = io->bio;
> + * * * struct dm_verity *v = io->v;
> +
> + * * * bio->bi_end_io = io->orig_bi_end_io;
> + * * * bio->bi_private = io->orig_bi_private;
> +
> + * * * if (io->io_vec != io->io_vec_inline)
> + * * * * * * * mempool_free(io->io_vec, v->vec_mempool);
> + * * * mempool_free(io, v->io_mempool);
> +
> + * * * bio_endio(bio, error);
> +}
> +
> +static void verity_work(struct work_struct *w)
> +{
> + * * * struct dm_verity_io *io = container_of(w, struct dm_verity_io, work);
> +
> + * * * verity_finish_io(io, verity_verify_io(io));
> +}
> +
> +static void verity_end_io(struct bio *bio, int error)
> +{
> + * * * struct dm_verity_io *io = bio->bi_private;
> + * * * if (error) {
> + * * * * * * * verity_finish_io(io, error);
> + * * * * * * * return;
> + * * * }
> +
> + * * * INIT_WORK(&io->work, verity_work);
> + * * * queue_work(io->v->verify_wq, &io->work);
> +}
> +
> +/*
> + * Prefetch buffers for the specified io.
> + * The root buffer is not prefetched, it is assumed that it will be cached
> + * all the time.
> + */
> +static void verity_prefetch_io(struct dm_verity *v, struct dm_verity_io *io)
> +{
> + * * * int i;
> + * * * for (i = v->levels - 2; i >= 0; i--) {
> + * * * * * * * sector_t hash_block_start;
> + * * * * * * * sector_t hash_block_end;
> + * * * * * * * verity_hash_at_level(v, io->block, i, &hash_block_start, NULL);
> + * * * * * * * verity_hash_at_level(v, io->block + io->n_blocks - 1, i, &hash_block_end, NULL);
> + * * * * * * * if (!i) {
> + * * * * * * * * * * * unsigned cluster = prefetch_cluster;
> + * * * */* barrier to stop GCC from re-reading prefetch_cluster again */
> + * * * * * * * * * * * barrier();
> + * * * * * * * * * * * cluster >>= v->data_dev_block_bits;

Would:
unsigned cluster = prefetch_cluster >> v->data_dev_block_bits;
not have similar behavior without a barrier? (Yeah yeah I could
compile and see, but I was curious if you already had.)

Since the max iterations here is 61 in a worst-case, I don't think
it's a big deal to barrier() each time, just thought I'd ask.

> + * * * * * * * * * * * if (unlikely(!cluster))
> + * * * * * * * * * * * * * * * goto no_prefetch_cluster;
> + * * * * * * * * * * * if (unlikely(cluster & (cluster - 1)))
> + * * * * * * * * * * * * * * * cluster = 1 << (fls(cluster) - 1);
> +
> + * * * * * * * * * * * hash_block_start &= ~(sector_t)(cluster - 1);
> + * * * * * * * * * * * hash_block_end |= cluster - 1;
> + * * * * * * * * * * * if (unlikely(hash_block_end >= v->hash_blocks))
> + * * * * * * * * * * * * * * * hash_block_end = v->hash_blocks - 1;
> + * * * * * * * }
> +no_prefetch_cluster:
> + * * * * * * * dm_bufio_prefetch(v->bufio, hash_block_start,
> + * * * * * * * * * * * * * * * * * * * hash_block_end - hash_block_start + 1);
> + * * * }
> +}
> +
> +/*
> + * Bio map function. It allocates dm_verity_io structure and bio vector and
> + * fills them. Then it issues prefetches and the I/O.
> + */
> +static int verity_map(struct dm_target *ti, struct bio *bio,
> + * * * * * * * * * * union map_info *map_context)
> +{
> + * * * struct dm_verity *v = ti->private;
> + * * * struct dm_verity_io *io;
> +
> + * * * if (((unsigned)bio->bi_sector | bio_sectors(bio)) &
> + * * * * * ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) {
> + * * * * * * * DMERR_LIMIT("unaligned io");
> + * * * * * * * return -EIO;
> + * * * }
> +
> + * * * if ((bio->bi_sector + bio_sectors(bio)) >>
> + * * * * * (v->data_dev_block_bits - SECTOR_SHIFT) > v->data_blocks) {
> + * * * * * * * DMERR_LIMIT("io out of range");
> + * * * * * * * return -EIO;
> + * * * }
> +
> + * * * if (bio_data_dir(bio) == WRITE)
> + * * * * * * * return -EIO;
> +
> + * * * io = mempool_alloc(v->io_mempool, GFP_NOIO);
> + * * * io->v = v;
> + * * * io->bio = bio;
> + * * * io->orig_bi_end_io = bio->bi_end_io;
> + * * * io->orig_bi_private = bio->bi_private;
> + * * * io->block = bio->bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT);
> + * * * io->n_blocks = bio->bi_size >> v->data_dev_block_bits;
> +
> + * * * bio->bi_end_io = verity_end_io;
> + * * * bio->bi_private = io;
> + * * * bio->bi_bdev = v->data_dev->bdev;
> + * * * bio->bi_sector = verity_map_sector(v, bio->bi_sector);
> +
> + * * * io->io_vec_size = bio->bi_vcnt - bio->bi_idx;
> + * * * if (io->io_vec_size < DM_VERITY_IO_VEC_INLINE)
> + * * * * * * * io->io_vec = io->io_vec_inline;
> + * * * else
> + * * * * * * * io->io_vec = mempool_alloc(v->vec_mempool, GFP_NOIO);
> + * * * memcpy(io->io_vec, bio_iovec(bio),
> + * * * * * * *io->io_vec_size * sizeof(struct bio_vec));
> +
> + * * * verity_prefetch_io(v, io);
> +
> + * * * generic_make_request(bio);
> +
> + * * * return DM_MAPIO_SUBMITTED;
> +}
> +
> +static int verity_status(struct dm_target *ti, status_type_t type,
> + * * * * * * * * * * * *char *result, unsigned maxlen)
> +{
> + * * * struct dm_verity *v = ti->private;
> + * * * unsigned sz = 0;
> + * * * unsigned x;
> +
> + * * * switch (type) {
> + * * * case STATUSTYPE_INFO:
> + * * * * * * * result[0] = 0;
> + * * * * * * * break;
> + * * * case STATUSTYPE_TABLE:
> + * * * * * * * DMEMIT("%u %s %s %llu %u %s ",
> + * * * * * * * * * * * 0,
> + * * * * * * * * * * * v->data_dev->name,
> + * * * * * * * * * * * v->hash_dev->name,

I understand the new approach is to use major:minor instead of the
device name. I don't care which, but I believe agk@ requested that.

> + * * * * * * * * * * * (unsigned long long)v->hash_start << (v->hash_dev_block_bits - SECTOR_SHIFT),
> + * * * * * * * * * * * 1 << v->data_dev_block_bits,
> + * * * * * * * * * * * v->alg_name
> + * * * * * * * * * * * );
> + * * * * * * * for (x = 0; x < v->digest_size; x++)
> + * * * * * * * * * * * DMEMIT("%02x", v->root_digest[x]);
> + * * * * * * * DMEMIT(" ");
> + * * * * * * * if (!v->salt_size)
> + * * * * * * * * * * * DMEMIT("-");
> + * * * * * * * else
> + * * * * * * * * * * * for (x = 0; x < v->salt_size; x++)
> + * * * * * * * * * * * * * * * DMEMIT("%02x", v->salt[x]);
> + * * * * * * * if (v->data_dev_block_bits != v->hash_dev_block_bits)
> + * * * * * * * * * * * DMEMIT(" %u", 1 << v->hash_dev_block_bits);
> + * * * * * * * break;
> + * * * }
> + * * * return 0;
> +}
> +
> +static int verity_ioctl(struct dm_target *ti, unsigned cmd,
> + * * * * * * * * * * * unsigned long arg)
> +{
> + * * * struct dm_verity *v = ti->private;
> + * * * int r = 0;
> +
> + * * * if (v->data_start ||
> + * * * * * ti->len != i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT)
> + * * * * * * * r = scsi_verify_blk_ioctl(NULL, cmd);
> +

Is it worth supporting ioctl at all given these hoops? Nothing stops
a privileged user from directly running the ioctl on the underlying
device/devices, it's just very inconvenient

> + * * * return r ? : __blkdev_driver_ioctl(v->data_dev->bdev, v->data_dev->mode,
> + * * * * * * * * * * * * * * * * * *cmd, arg);
> +}
> +
> +static int verity_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
> + * * * * * * * * * * * struct bio_vec *biovec, int max_size)
> +{
> + * * * struct dm_verity *v = ti->private;
> + * * * struct request_queue *q = bdev_get_queue(v->data_dev->bdev);
> +
> + * * * if (!q->merge_bvec_fn)
> + * * * * * * * return max_size;
> +
> + * * * bvm->bi_bdev = v->data_dev->bdev;
> + * * * bvm->bi_sector = verity_map_sector(v, bvm->bi_sector);
> +
> + * * * return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
> +}
> +
> +static int verity_iterate_devices(struct dm_target *ti,
> + * * * * * * * * * * * * * * * * iterate_devices_callout_fn fn, void *data)
> +{
> + * * * struct dm_verity *v = ti->private;
> + * * * return fn(ti, v->data_dev, v->data_start, ti->len, data);
> +}
> +
> +static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits)
> +{
> + * * * struct dm_verity *v = ti->private;
> +
> + * * * if (limits->logical_block_size < 1 << v->data_dev_block_bits)
> + * * * * * * * limits->logical_block_size = 1 << v->data_dev_block_bits;
> + * * * if (limits->physical_block_size < 1 << v->data_dev_block_bits)
> + * * * * * * * limits->physical_block_size = 1 << v->data_dev_block_bits;
> + * * * blk_limits_io_min(limits, limits->logical_block_size);
> +}
> +
> +static void verity_dtr(struct dm_target *ti);
> +
> +static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv)
> +{
> + * * * struct dm_verity *v;
> + * * * unsigned num;
> + * * * unsigned long long hs;
> + * * * int r;
> + * * * int i;
> + * * * sector_t hash_position;
> + * * * char dummy;
> +
> + * * * v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL);
> + * * * if (!v) {
> + * * * * * * * ti->error = "Cannot allocate verity structure";
> + * * * * * * * return -ENOMEM;
> + * * * }
> + * * * ti->private = v;
> + * * * v->ti = ti;
> +
> + * * * if ((dm_table_get_mode(ti->table) & ~FMODE_READ) != 0) {
> + * * * * * * * ti->error = "Device must be readonly";
> + * * * * * * * r = -EINVAL;
> + * * * * * * * goto bad;
> + * * * }
> +
> + * * * if (argc < 7) {
> + * * * * * * * ti->error = "Not enough arguments";
> + * * * * * * * r = -EINVAL;
> + * * * * * * * goto bad;
> + * * * }
> +
> + * * * if (sscanf(argv[0], "%d%c", &num, &dummy) != 1 ||
> + * * * * * num != 0) {
> + * * * * * * * ti->error = "Invalid version";
> + * * * * * * * r = -EINVAL;
> + * * * * * * * goto bad;
> + * * * }
> +
> + * * * r = dm_get_device(ti, argv[1], FMODE_READ, &v->data_dev);
> + * * * if (r) {
> + * * * * * * * ti->error = "Data device lookup failed";
> + * * * * * * * goto bad;
> + * * * }
> +
> + * * * r = dm_get_device(ti, argv[2], FMODE_READ, &v->hash_dev);
> + * * * if (r) {
> + * * * * * * * ti->error = "Data device lookup failed";
> + * * * * * * * goto bad;
> + * * * }
> +
> + * * * if (sscanf(argv[3], "%llu%c", &hs, &dummy) != 1 ||
> + * * * * * hs != (sector_t)hs) {
> + * * * * * * * ti->error = "Invalid hash start";
> + * * * * * * * r = -EINVAL;
> + * * * * * * * goto bad;
> + * * * }
> +
> + * * * if (sscanf(argv[4], "%u%c", &num, &dummy) != 1 ||
> + * * * * * !num || (num & (num - 1)) ||
> + * * * * * num < bdev_logical_block_size(v->data_dev->bdev) ||
> + * * * * * num > PAGE_SIZE) {
> + * * * * * * * ti->error = "Invalid data device block size";
> + * * * * * * * r = -EINVAL;
> + * * * * * * * goto bad;
> + * * * }
> + * * * v->data_dev_block_bits = ffs(num) - 1;
> + * * * v->hash_dev_block_bits = ffs(num) - 1;
> +
> + * * * v->alg_name = kstrdup(argv[5], GFP_KERNEL);
> + * * * if (!v->alg_name) {
> + * * * * * * * ti->error = "Cannot allocate algorithm name";
> + * * * * * * * r = -ENOMEM;
> + * * * * * * * goto bad;
> + * * * }
> +
> + * * * v->tfm = crypto_alloc_shash(v->alg_name, 0, 0);
> + * * * if (IS_ERR(v->tfm)) {
> + * * * * * * * ti->error = "Cannot initialize hash function";
> + * * * * * * * r = PTR_ERR(v->tfm);
> + * * * * * * * v->tfm = NULL;
> + * * * * * * * goto bad;
> + * * * }
> + * * * v->digest_size = crypto_shash_digestsize(v->tfm);
> + * * * if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) {
> + * * * * * * * ti->error = "Digest size too big";
> + * * * * * * * r = -EINVAL;
> + * * * * * * * goto bad;
> + * * * }
> + * * * v->shash_descsize =
> + * * * * * * * sizeof(struct shash_desc) + crypto_shash_descsize(v->tfm);
> +
> + * * * v->root_digest = kmalloc(v->digest_size, GFP_KERNEL);
> + * * * if (!v->root_digest) {
> + * * * * * * * ti->error = "Cannot allocate root digest";
> + * * * * * * * r = -ENOMEM;
> + * * * * * * * goto bad;
> + * * * }
> + * * * if (strlen(argv[6]) != v->digest_size * 2 ||
> + * * * * * hex2bin(v->root_digest, argv[6], v->digest_size)) {
> + * * * * * * * ti->error = "Invalid root digest";
> + * * * * * * * r = -EINVAL;
> + * * * * * * * goto bad;
> + * * * }
> +
> + * * * if (argc > 7 && strcmp(argv[7], "-")) {
> + * * * * * * * v->salt_size = strlen(argv[7]) / 2;
> + * * * * * * * v->salt = kmalloc(v->salt_size, GFP_KERNEL);
> + * * * * * * * if (!v->salt) {
> + * * * * * * * * * * * ti->error = "Cannot allocate salt";
> + * * * * * * * * * * * r = -ENOMEM;
> + * * * * * * * * * * * goto bad;
> + * * * * * * * }
> + * * * * * * * if (strlen(argv[7]) != v->salt_size * 2 ||
> + * * * * * * * * * hex2bin(v->salt, argv[7], v->salt_size)) {
> + * * * * * * * * * * * ti->error = "Invalid salt";
> + * * * * * * * * * * * r = -EINVAL;
> + * * * * * * * * * * * goto bad;
> + * * * * * * * }
> + * * * }
> +
> + * * * if (argc > 8) {
> + * * * * * * * if (sscanf(argv[8], "%u%c", &num, &dummy) != 1 ||
> + * * * * * * * * * !num || (num & (num - 1)) ||
> + * * * * * * * * * num < bdev_logical_block_size(v->hash_dev->bdev) ||
> + * * * * * * * * * num > INT_MAX) {
> + * * * * * * * * * * * ti->error = "Invalid hash device block size";
> + * * * * * * * * * * * r = -EINVAL;
> + * * * * * * * * * * * goto bad;
> + * * * * * * * }
> + * * * * * * * v->hash_dev_block_bits = ffs(num) - 1;
> + * * * }
> +
> + * * * if (hs & ((1 << (v->hash_dev_block_bits - SECTOR_SHIFT)) - 1)) {
> + * * * * * * * ti->error = "Hash start not aligned on block boundary";
> + * * * * * * * r = -EINVAL;
> + * * * * * * * goto bad;
> + * * * }
> + * * * v->hash_start = hs >> (v->hash_dev_block_bits - SECTOR_SHIFT);
> +
> + * * * if (ti->len > i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT) {
> + * * * * * * * ti->error = "Data device si too small";

s/si/is

Should this also check ti->start + ti->len to ensure it isn't reading
off the end or do you just rely on the requests failing?

> + * * * * * * * r = -EINVAL;
> + * * * * * * * goto bad;
> + * * * }
> +
> + * * * if (ti->len & ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) {
> + * * * * * * * ti->error = "Data device length is not aligned to block size";
> + * * * * * * * r = -EINVAL;
> + * * * * * * * goto bad;
> + * * * }
> +
> + * * * v->data_blocks = ti->len >> (v->data_dev_block_bits - SECTOR_SHIFT);
> +
> + * * * v->hash_per_block_bits =
> + * * * * * * * fls((1 << v->hash_dev_block_bits) / v->digest_size) - 1;
> +
> + * * * v->levels = 0;
> + * * * if (v->data_blocks)
> + * * * * * * * while (v->hash_per_block_bits * v->levels < 64 &&
> + * * * * * * * * * * *(unsigned long long)(v->data_blocks - 1) >>
> + * * * * * * * * * * *(v->hash_per_block_bits * v->levels))
> + * * * * * * * * * * * v->levels++;
> +
> + * * * if (v->levels > DM_VERITY_MAX_LEVELS) {
> + * * * * * * * ti->error = "Too many tree levels";
> + * * * * * * * r = -E2BIG;
> + * * * * * * * goto bad;
> + * * * }
> +
> + * * * hash_position = v->hash_start;
> + * * * for (i = v->levels - 1; i >= 0; i--) {
> + * * * * * * * sector_t s;
> + * * * * * * * v->hash_level_block[i] = hash_position;
> + * * * * * * * s = verity_position_at_level(v, v->data_blocks, i);
> + * * * * * * * s = (s >> v->hash_per_block_bits) +
> + * * * * * * * * * !!(s & ((1 << v->hash_per_block_bits) - 1));
> + * * * * * * * if (hash_position + s < hash_position) {
> + * * * * * * * * * * * ti->error = "Hash device offset overflow";
> + * * * * * * * * * * * r = -E2BIG;
> + * * * * * * * * * * * goto bad;
> + * * * * * * * }
> + * * * * * * * hash_position += s;
> + * * * }
> + * * * v->hash_blocks = hash_position;
> +
> + * * * v->bufio = dm_bufio_client_create(v->hash_dev->bdev,
> + * * * * * * * 1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux),
> + * * * * * * * dm_bufio_alloc_callback, NULL);
> + * * * if (IS_ERR(v->bufio)) {
> + * * * * * * * ti->error = "Cannot initialize dm-bufio";
> + * * * * * * * r = PTR_ERR(v->bufio);
> + * * * * * * * v->bufio = NULL;
> + * * * * * * * goto bad;
> + * * * }
> +
> + * * * if (dm_bufio_get_device_size(v->bufio) < v->hash_blocks) {
> + * * * * * * * ti->error = "Hash device is too small";
> + * * * * * * * r = -E2BIG;
> + * * * * * * * goto bad;
> + * * * }
> +
> + * * * v->io_mempool = mempool_create_kmalloc_pool(DM_VERITY_MEMPOOL_SIZE ,
> + * * * * sizeof(struct dm_verity_io) + v->shash_descsize + v->digest_size * 2);
> + * * * if (!v->io_mempool) {
> + * * * * * * * ti->error = "Cannot allocate io mempool";
> + * * * * * * * r = -ENOMEM;
> + * * * * * * * goto bad;
> + * * * }
> +
> + * * * v->vec_mempool = mempool_create_kmalloc_pool(DM_VERITY_MEMPOOL_SIZE ,
> + * * * * * * * * * * * * * * * * * * * BIO_MAX_PAGES * sizeof(struct bio_vec));
> + * * * if (!v->vec_mempool) {
> + * * * * * * * ti->error = "Cannot allocate vector mempool";
> + * * * * * * * r = -ENOMEM;
> + * * * * * * * goto bad;
> + * * * }
> +
> + * * * /*v->verify_wq = alloc_workqueue("verityd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 1);*/
> + * * * /* WQ_UNBOUND greatly improves performance when running on ramdisk */
> + * * * v->verify_wq = alloc_workqueue("verityd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, num_online_cpus());
> + * * * if (!v->verify_wq) {
> + * * * * * * * ti->error = "Cannot allocate workqueue";
> + * * * * * * * r = -ENOMEM;
> + * * * * * * * goto bad;
> + * * * }
> +
> + * * * return 0;
> +
> +bad:
> + * * * verity_dtr(ti);
> + * * * return r;
> +}
> +
> +static void verity_dtr(struct dm_target *ti)
> +{
> + * * * struct dm_verity *v = ti->private;
> +
> + * * * if (v->verify_wq)
> + * * * * * * * destroy_workqueue(v->verify_wq);
> + * * * if (v->vec_mempool)
> + * * * * * * * mempool_destroy(v->vec_mempool);
> + * * * if (v->io_mempool)
> + * * * * * * * mempool_destroy(v->io_mempool);
> + * * * if (v->bufio)
> + * * * * * * * dm_bufio_client_destroy(v->bufio);
> + * * * kfree(v->salt);
> + * * * kfree(v->root_digest);
> + * * * if (v->tfm)
> + * * * * * * * crypto_free_shash(v->tfm);
> + * * * kfree(v->alg_name);
> + * * * if (v->hash_dev)
> + * * * * * * * dm_put_device(ti, v->hash_dev);
> + * * * if (v->data_dev)
> + * * * * * * * dm_put_device(ti, v->data_dev);
> + * * * kfree(v);
> +}
> +
> +static struct target_type verity_target = {
> + * * * .name * * * * * = "verity",
> + * * * .version * * * *= {1, 0, 0},
> + * * * .module * * * * = THIS_MODULE,
> + * * * .ctr * * * * * *= verity_ctr,
> + * * * .dtr * * * * * *= verity_dtr,
> + * * * .map * * * * * *= verity_map,
> + * * * .status * * * * = verity_status,
> + * * * .ioctl * * * * *= verity_ioctl,
> + * * * .merge * * * * *= verity_merge,
> + * * * .iterate_devices = verity_iterate_devices,
> + * * * .io_hints * * * = verity_io_hints,
> +};
> +
> +static int __init dm_verity_init(void)
> +{
> + * * * int r;
> + * * * r = dm_register_target(&verity_target);
> + * * * if (r < 0)
> + * * * * * * * DMERR("register failed %d", r);
> + * * * return r;
> +}
> +
> +static void __exit dm_verity_exit(void)
> +{
> + * * * dm_unregister_target(&verity_target);
> +}
> +
> +module_init(dm_verity_init);
> +module_exit(dm_verity_exit);
> +
> +MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>");

As per linux/module.h, I'd welcome additional authors as per the
lkml/patch lineage:
MODULE_AUTHOR("Mandeep Baines <msb@chromium.org>");
MODULE_AUTHOR("Will Drewry <wad@chromium.org>");

Regardless, I'll just be happy to see this functionality merge.

> +MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking");
> +MODULE_LICENSE("GPL");
> +
> Index: linux-3.3-rc6-fast/drivers/md/dm-bufio.c

This should be in a separate patch I think.

> ================================================== =================
> --- linux-3.3-rc6-fast.orig/drivers/md/dm-bufio.c * * * 2012-03-12 22:43:23.000000000 +0100
> +++ linux-3.3-rc6-fast/drivers/md/dm-bufio.c * *2012-03-13 15:41:02.000000000 +0100
[snip]
> @@ -882,6 +888,19 @@ static struct dm_buffer *__bufio_new(str
>
> * * * *b = __find(c, block);
> * * * *if (b) {
> +found_buffer:
> + * * * * * * * if (nf == NF_PREFETCH)
> + * * * * * * * * * * * return NULL;
> + * * * * * * * /*
> + * * * * * * * ** Note: it is essential that we don't wait for the buffer to be
> + * * * * * * * ** read if dm_bufio_get function is used. Both dm_bufio_get and
> + * * * * * * * ** dm_bufio_prefetch can be used in the driver request routine.
> + * * * * * * * ** If the user called both dm_bufio_prefetch and dm_bufio_get on
> + * * * * * * * ** the same buffer, it would deadlock if we waited.
> + * * * * * * * **/
> + * * * * * * * if (nf == NF_GET && unlikely(test_bit(B_READING, &b->state)))
> + * * * * * * * * * * * return NULL;
> +
> * * * * * * * *b->hold_count++;

Are these hold_counts safe on architectures with weak memory models?
Should they be atomic_ts? I haven't looked at them in context, but
based on what I see here they make me a bit nervous.

Thanks for jumping in to the fray! None of my comments are blocking,
so I believe the following is appropriate (if not
s/Signed-off/Reviewed-by/).

Signed-off-by: Will Drewry <wad@chromium.org>

cheers!
will

--
dm-devel mailing list
dm-devel@redhat.com
https://www.redhat.com/mailman/listinfo/dm-devel
 
Old 03-14-2012, 08:43 PM
Mandeep Singh Baines
 
Default dm: remake of the verity target

Mikulas Patocka (mpatocka@redhat.com) wrote:
> Hi
>
> > Hi Mikulus,
> >
> > This is some nice work. I like that you've been able to abstract a lot
> > of the hash buffer management with dm-bufio. You got rid of the I/O queue.
> > I've been meaning to do that for a while. The prefetch is also nice.
> > We planned to do this but I decided to not do it now in order to get the
> > base functionality in:
> >
> > http://crosbug.com/25441
> >
> > However, there are some things that I don't like. I don't like comments
> > either but you have none. You also removed our documentation. You are
>
> I added some comments. As for documentation, it's OK to use documentation

Thanks.

> from your patch because the on-disk format and the target arguments are
> the same (with an enhancement that my code supports different data and
> metadata bock size and it has variable-length salt).
>

Would you mind adding the documentation as part 2 of your series.

> > allocated a complete shash_desc per I/O. We only allocate one per CPU.
>
> The hash of 4k block takes 174000 cycles. So trying to optimize
> memory latency that is about 250 cycles doesn't make much sense.
>
> I actually observed better performance using verity on ramdisk with
> workqueue unbound to specific CPUs. The reason is that the ramdisk bio
> completion routine is always run on the same CPU (that one that submitted
> the request), so with bound workqueue, everything was executing on one
> CPU. With unbound workqueue, I got parallelism.
>

I guess it depends on whether you're CPU bound or I/O bound. If you're
CPU-bound and all the schedule is doing a good job of keeping all the
cores busy, then you're just adding extra cache misses. But if you're
not CPU-bound, then you can parallelize the hashing. So I guess it
depends. Anway, arguable which is better without data on real workloads.

At some point, it would be interesting to compare ChromeOS boot performance
with both approaches.


> > We short-circuit the hash at any level. Your implementation can only
> > shirt circuit at the lowest level.
>
> It short-circuits hash at all levels. If the function
> "verity_verify_level" finds out that "aux->hash_verified" is non-zero, it
> doesn't do any hashing, it just copies the hash for the lower level. My
> implementation walks the tree from the top to the bottom, but it doesn't
> do hash verification if the same block has been verified before.
>

I agree. Short-circuiting won't give an extra benefit. For some reason,
I thought you might be re-verifying a node but that's not the case.

> All this tree-walking from the root to the bottom is 50-times faster than
> the actual hashing of the data block (I measured that), so there's not
> much point in trying to optimize it. I did a simple optimization (don't
> walk the tree if the lowest block is already verified) and I don't need to
> do anything complicated given the fact that it can't improve more than by
> 2%.
>
> > I'd like to propose that we get the version we sent upstream and then work
> > together on adding some of your enhancements incrementally.
>
> If you add dm-bufio support, you end up deleting majority of the original
> code anyway. That's why I wrote it from scratch and that's why I didn't
> attempt to morph your code.
>
> It's simpler to write the code from scratch and it is also less bug-prone.
>
> > Other than
> > the changes we've made to cleanup for upstreaming, the version I
> > submitted is the code we are using in production.
> >
> > I'm happy to add prefetch now if that is required for merging.
> >
> > What do you think?
> >
> > Regards,
> > Mandeep
>
> This is the version with comments added:
>
> Mikulas
>
> ----
>
> Remake of the google dm-verity patch.
>
> Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
>

Signed-off-by: Mandeep Singh Baines <msb@chromium.org>

Nice work.

I have a few nits but would be happy to see this merged. It doesn't
look like the version I worked on will ever get merged, maybe you'll
have better luck

> ---
> drivers/md/Kconfig | 17
> drivers/md/Makefile | 1
> drivers/md/dm-verity.c | 851 +++++++++++++++++++++++++++++++++++++++++++++++++
> 3 files changed, 869 insertions(+)
>
> Index: linux-3.3-rc6-fast/drivers/md/Kconfig
> ================================================== =================
> --- linux-3.3-rc6-fast.orig/drivers/md/Kconfig 2012-03-13 21:46:03.000000000 +0100
> +++ linux-3.3-rc6-fast/drivers/md/Kconfig 2012-03-13 21:46:05.000000000 +0100
> @@ -404,4 +404,21 @@ config DM_VERITY2
>
> If unsure, say N.
>
> +config DM_VERITY
> + tristate "Verity target support"
> + depends on BLK_DEV_DM
> + select CRYPTO
> + select CRYPTO_HASH
> + select DM_BUFIO
> + ---help---
> + This device-mapper target allows you to create a device that
> + transparently integrity checks the data on it. You'll need to
> + activate the digests you're going to use in the cryptoapi
> + configuration.
> +
> + To compile this code as a module, choose M here: the module will
> + be called dm-verity.
> +
> + If unsure, say N.
> +
> endif # MD
> Index: linux-3.3-rc6-fast/drivers/md/Makefile
> ================================================== =================
> --- linux-3.3-rc6-fast.orig/drivers/md/Makefile 2012-03-13 21:46:03.000000000 +0100
> +++ linux-3.3-rc6-fast/drivers/md/Makefile 2012-03-13 21:46:05.000000000 +0100
> @@ -29,6 +29,7 @@ obj-$(CONFIG_MD_FAULTY) += faulty.o
> obj-$(CONFIG_BLK_DEV_MD) += md-mod.o
> obj-$(CONFIG_BLK_DEV_DM) += dm-mod.o
> obj-$(CONFIG_DM_BUFIO) += dm-bufio.o
> +obj-$(CONFIG_DM_VERITY) += dm-verity.o
> obj-$(CONFIG_DM_CRYPT) += dm-crypt.o
> obj-$(CONFIG_DM_DELAY) += dm-delay.o
> obj-$(CONFIG_DM_FLAKEY) += dm-flakey.o
> Index: linux-3.3-rc6-fast/drivers/md/dm-verity.c
> ================================================== =================
> --- /dev/null 1970-01-01 00:00:00.000000000 +0000
> +++ linux-3.3-rc6-fast/drivers/md/dm-verity.c 2012-03-13 22:02:05.000000000 +0100
> @@ -0,0 +1,851 @@
> +/*
> + * Copyright (C) 2012 Red Hat, Inc.
> + *
> + * Author: Mikulas Patocka <mpatocka@redhat.com>
> + *
> + * Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors
> + *
> + * This file is released under the GPLv2.
> + *
> + * Device mapper target parameters:
> + * <version> 0
> + * <data device>
> + * <hash device>
> + * <hash start> (typically 0)
> + * <block size> (typically 4096)
> + * <algorithm>
> + * <digest>
> + * optional parameters:
> + * <salt> (should have 32 bytes for compatibility with Google code)
> + * <hash block size> (by default it is the same as data block size)
> + *
> + * In the file "/sys/module/dm_verity/parameters/prefetch_cluster" you can set
> + * default prefetch value. Data are read in "prefetch_cluster" chunks from the
> + * hash device. Prefetch cluster greatly improves performance when data and hash
> + * are on the same disk on different partitions.
> + */
> +
> +#include <linux/module.h>
> +#include <linux/device-mapper.h>
> +#include <crypto/hash.h>
> +#include "dm-bufio.h"
> +
> +#define DM_MSG_PREFIX "verity"
> +
> +#define DM_VERITY_IO_VEC_INLINE 16
> +#define DM_VERITY_MEMPOOL_SIZE 4
> +#define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144
> +
> +#define DM_VERITY_MAX_LEVELS 63
> +
> +static unsigned prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE;
> +
> +module_param_named(prefetch_cluster, prefetch_cluster, uint, S_IRUGO | S_IWUSR);
> +
> +struct dm_verity {
> + struct dm_dev *data_dev;
> + struct dm_dev *hash_dev;
> + struct dm_target *ti;
> + struct dm_bufio_client *bufio;
> + char *alg_name;
> + struct crypto_shash *tfm;
> + u8 *root_digest; /* digest of the root block */
> + u8 *salt; /* salt, its size is salt_size */
> + unsigned salt_size;
> + sector_t data_start; /* data offset in 512-byte sectors */
> + sector_t hash_start; /* hash start in blocks */
> + sector_t data_blocks; /* the number of data blocks */
> + sector_t hash_blocks; /* the number of hash blocks */
> + unsigned char data_dev_block_bits; /* log2(data blocksize) */
> + unsigned char hash_dev_block_bits; /* log2(hash blocksize) */
> + unsigned char hash_per_block_bits; /* log2(hashes in hash block) */
> + unsigned char levels; /* the number of tree levels */
> + unsigned digest_size; /* digest size for the current hash algorithm */
> + unsigned shash_descsize;/* the size of temporary space for crypto */
> +
> + mempool_t *io_mempool; /* mempool of struct dm_verity_io */
> + mempool_t *vec_mempool; /* mempool of bio vector */
> +

Since there are no writes, do we even need mempool? I was thinking of
removing all mempools. I can't think of case where a mempool helps you
for a read-only device. There is no reading under memory pressure.

> + struct workqueue_struct *verify_wq;
> +
> + /* starting blocks for each tree level. 0 is the lowest level. */
> + sector_t hash_level_block[DM_VERITY_MAX_LEVELS];
> +};
> +
> +struct dm_verity_io {
> + struct dm_verity *v;
> + struct bio *bio;
> +
> + /* original values of bio->bi_end_io and bio->bi_private */
> + bio_end_io_t *orig_bi_end_io;
> + void *orig_bi_private;
> +
> + sector_t block;
> + unsigned n_blocks;
> +
> + /* saved bio vector */
> + struct bio_vec *io_vec;
> + unsigned io_vec_size;
> +
> + struct work_struct work;
> +
> + /* a space for short vectors; longer vectors are allocated separately */
> + struct bio_vec io_vec_inline[DM_VERITY_IO_VEC_INLINE];
> +
> + /* variable-size fields, accessible with functions
> + io_hash_desc, io_real_digest, io_want_digest */
> + /* u8 hash_desc[crypto_shash_descsize(v->tfm)]; */
> + /* u8 real_digest[v->digest_size]; */
> + /* u8 want_digest[v->digest_size]; */

Nit. Commented code should be removed.

> +};
> +
> +static struct shash_desc *io_hash_desc(struct dm_verity *v, struct dm_verity_io *io)
> +{
> + return (struct shash_desc *)(io + 1);
> +}
> +
> +static u8 *io_real_digest(struct dm_verity *v, struct dm_verity_io *io)
> +{
> + return (u8 *)(io + 1) + v->shash_descsize;
> +}
> +
> +static u8 *io_want_digest(struct dm_verity *v, struct dm_verity_io *io)
> +{
> + return (u8 *)(io + 1) + v->shash_descsize + v->digest_size;
> +}
> +
> +/*
> + * Auxiliary structure appended to each dm-bufio buffer. If the value
> + * hash_verified is nonzero, hash of the block has been verified.
> + *
> + * There is no lock around this value, a race condition can at worst cause
> + * that multiple processes verify the hash of the same buffer simultaneously.
> + * This condition is harmless, so we don't need locking.
> + */
> +struct buffer_aux {
> + int hash_verified;
> +};
> +
> +/*
> + * Initialize struct buffer_aux for a freshly created buffer.
> + */
> +static void dm_bufio_alloc_callback(struct dm_buffer *buf)
> +{
> + struct buffer_aux *aux = dm_bufio_get_aux_data(buf);
> + aux->hash_verified = 0;
> +}
> +
> +/*
> + * Translate input sector number to the sector number on the target device.
> + */
> +static sector_t verity_map_sector(struct dm_verity *v, sector_t bi_sector)
> +{
> + return v->data_start + dm_target_offset(v->ti, bi_sector);
> +}
> +
> +/*
> + * Return hash position of a specified block at a specified tree level
> + * (0 is the lowest level).
> + * The lowest "hash_per_block_bits"-bits of the result denote hash position
> + * inside a hash block. The remaining bits denode location of the hash block.
> + */
> +static sector_t verity_position_at_level(struct dm_verity *v, sector_t block,
> + int level)
> +{
> + return block >> (level * v->hash_per_block_bits);
> +}
> +
> +static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level,
> + sector_t *hash_block, unsigned *offset)
> +{
> + sector_t position = verity_position_at_level(v, block, level);
> +
> + *hash_block = v->hash_level_block[level] + (position >> v->hash_per_block_bits);
> + if (offset)
> + *offset = v->digest_size * (position & ((1 << v->hash_per_block_bits) - 1));
> +}
> +
> +/*
> + * Verify hash of a metadata block pertaining to the specified data block
> + * ("block" argument) at a specified level ("level" argument).
> + *
> + * On successful return, io_want_digest(v, io) contains the hash value for
> + * a lower tree level or for the data block (if we're at the lowest leve).
> + *
> + * If "skip_unverified" is true, unverified buffer is skipped an 1 is returned.
> + * If "skip_unverified" is false, unverified buffer is hashed and verified
> + * against current value of io_want_digest(v, io).
> + */
> +static int verity_verify_level(struct dm_verity_io *io, sector_t block,
> + int level, bool skip_unverified)
> +{
> + struct dm_verity *v = io->v;
> + struct dm_buffer *buf;
> + struct buffer_aux *aux;
> + u8 *data;
> + int r;
> + sector_t hash_block;
> + unsigned offset;
> +
> + verity_hash_at_level(v, block, level, &hash_block, &offset);
> +
> + data = dm_bufio_read(v->bufio, hash_block, &buf);
> + if (unlikely(IS_ERR(data)))
> + return PTR_ERR(data);
> +
> + aux = dm_bufio_get_aux_data(buf);
> +
> + if (!aux->hash_verified) {
> + struct shash_desc *desc;
> + u8 *result;
> +
> + if (skip_unverified) {
> + r = 1;
> + goto release_ret_r;
> + }
> +
> + desc = io_hash_desc(v, io);
> + desc->tfm = v->tfm;
> + desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
> + r = crypto_shash_init(desc);
> + if (r < 0) {
> + DMERR("crypto_shash_init failed: %d", r);
> + goto release_ret_r;
> + }
> +
> + r = crypto_shash_update(desc, data, 1 << v->hash_dev_block_bits);
> + if (r < 0) {
> + DMERR("crypto_shash_update failed: %d", r);
> + goto release_ret_r;
> + }
> +
> + r = crypto_shash_update(desc, v->salt, v->salt_size);
> + if (r < 0) {
> + DMERR("crypto_shash_update failed: %d", r);
> + goto release_ret_r;
> + }
> +
> + result = io_real_digest(v, io);
> + r = crypto_shash_final(desc, result);
> + if (r < 0) {
> + DMERR("crypto_shash_final failed: %d", r);
> + goto release_ret_r;
> + }
> + if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) {
> + DMERR_LIMIT("metadata block %llu is corrupted",
> + (unsigned long long)hash_block);
> + r = -EIO;
> + goto release_ret_r;
> + } else
> + aux->hash_verified = 1;
> + }
> +
> + data += offset;
> +
> + memcpy(io_want_digest(v, io), data, v->digest_size);
> +
> + dm_bufio_release(buf);
> + return 0;
> +
> +release_ret_r:
> + dm_bufio_release(buf);
> + return r;
> +}
> +
> +/*
> + * Verify one "dm_verity_io" structure.
> + */
> +static int verity_verify_io(struct dm_verity_io *io)
> +{
> + struct dm_verity *v = io->v;
> + unsigned b;
> + int i;
> + unsigned vector = 0, offset = 0;
> + for (b = 0; b < io->n_blocks; b++) {
> + struct shash_desc *desc;
> + u8 *result;
> + int r;
> + unsigned todo;
> +
> + if (likely(v->levels)) {
> + /*
> + * First, we try to get the requested hash for
> + * the current block. If the hash block itself is
> + * verified, zero is returned. If it isn't, this
> + * function returns 0 and we fall back to whole
> + * chain verification.
> + */
> + int r = verity_verify_level(io, io->block + b, 0, true);
> + if (likely(!r))
> + goto test_block_hash;
> + if (r < 0)
> + return r;
> + }
> +
> + memcpy(io_want_digest(v, io), v->root_digest, v->digest_size);
> +
> + for (i = v->levels - 1; i >= 0; i--) {
> + int r = verity_verify_level(io, io->block + b, i, false);
> + if (unlikely(r))
> + return r;
> + }
> +
> +test_block_hash:
> + desc = io_hash_desc(v, io);
> + desc->tfm = v->tfm;
> + desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
> + r = crypto_shash_init(desc);
> + if (r < 0) {
> + DMERR("crypto_shash_init failed: %d", r);
> + return r;
> + }
> +
> + todo = 1 << v->data_dev_block_bits;
> + do {
> + struct bio_vec *bv;
> + u8 *page;
> + unsigned len;
> +
> + BUG_ON(vector >= io->io_vec_size);
> + bv = &io->io_vec[vector];
> + page = kmap_atomic(bv->bv_page, KM_USER0);
> + len = bv->bv_len - offset;
> + if (likely(len >= todo))
> + len = todo;
> + r = crypto_shash_update(desc,
> + page + bv->bv_offset + offset, len);
> + kunmap_atomic(page, KM_USER0);
> + if (r < 0) {
> + DMERR("crypto_shash_update failed: %d", r);
> + return r;
> + }
> + offset += len;
> + if (likely(offset == bv->bv_len)) {
> + offset = 0;
> + vector++;
> + }
> + todo -= len;
> + } while (todo);
> +
> + r = crypto_shash_update(desc, v->salt, v->salt_size);
> + if (r < 0) {
> + DMERR("crypto_shash_update failed: %d", r);
> + return r;
> + }
> +
> + result = io_real_digest(v, io);
> + r = crypto_shash_final(desc, result);
> + if (r < 0) {
> + DMERR("crypto_shash_final failed: %d", r);
> + return r;
> + }
> + if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) {
> + DMERR_LIMIT("data block %llu is corrupted",
> + (unsigned long long)(io->block + b));
> + return -EIO;
> + }
> + }
> + BUG_ON(vector != io->io_vec_size);
> + BUG_ON(offset);
> + return 0;
> +}
> +
> +/*
> + * End one "io" structure with a given error.
> + */
> +static void verity_finish_io(struct dm_verity_io *io, int error)
> +{
> + struct bio *bio = io->bio;
> + struct dm_verity *v = io->v;
> +
> + bio->bi_end_io = io->orig_bi_end_io;
> + bio->bi_private = io->orig_bi_private;
> +
> + if (io->io_vec != io->io_vec_inline)
> + mempool_free(io->io_vec, v->vec_mempool);
> + mempool_free(io, v->io_mempool);
> +
> + bio_endio(bio, error);
> +}
> +
> +static void verity_work(struct work_struct *w)
> +{
> + struct dm_verity_io *io = container_of(w, struct dm_verity_io, work);
> +
> + verity_finish_io(io, verity_verify_io(io));
> +}
> +
> +static void verity_end_io(struct bio *bio, int error)
> +{
> + struct dm_verity_io *io = bio->bi_private;
> + if (error) {
> + verity_finish_io(io, error);
> + return;
> + }
> +
> + INIT_WORK(&io->work, verity_work);
> + queue_work(io->v->verify_wq, &io->work);
> +}
> +
> +/*
> + * Prefetch buffers for the specified io.
> + * The root buffer is not prefetched, it is assumed that it will be cached
> + * all the time.
> + */
> +static void verity_prefetch_io(struct dm_verity *v, struct dm_verity_io *io)
> +{
> + int i;
> + for (i = v->levels - 2; i >= 0; i--) {
> + sector_t hash_block_start;
> + sector_t hash_block_end;
> + verity_hash_at_level(v, io->block, i, &hash_block_start, NULL);
> + verity_hash_at_level(v, io->block + io->n_blocks - 1, i, &hash_block_end, NULL);
> + if (!i) {
> + unsigned cluster = prefetch_cluster;
> + /* barrier to stop GCC from re-reading prefetch_cluster again */
> + barrier();
> + cluster >>= v->data_dev_block_bits;
> + if (unlikely(!cluster))
> + goto no_prefetch_cluster;
> + if (unlikely(cluster & (cluster - 1)))
> + cluster = 1 << (fls(cluster) - 1);
> +
> + hash_block_start &= ~(sector_t)(cluster - 1);
> + hash_block_end |= cluster - 1;
> + if (unlikely(hash_block_end >= v->hash_blocks))
> + hash_block_end = v->hash_blocks - 1;
> + }
> +no_prefetch_cluster:
> + dm_bufio_prefetch(v->bufio, hash_block_start,
> + hash_block_end - hash_block_start + 1);
> + }
> +}
> +
> +/*
> + * Bio map function. It allocates dm_verity_io structure and bio vector and
> + * fills them. Then it issues prefetches and the I/O.
> + */
> +static int verity_map(struct dm_target *ti, struct bio *bio,
> + union map_info *map_context)
> +{
> + struct dm_verity *v = ti->private;
> + struct dm_verity_io *io;
> +
> + if (((unsigned)bio->bi_sector | bio_sectors(bio)) &
> + ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) {
> + DMERR_LIMIT("unaligned io");
> + return -EIO;
> + }
> +
> + if ((bio->bi_sector + bio_sectors(bio)) >>
> + (v->data_dev_block_bits - SECTOR_SHIFT) > v->data_blocks) {
> + DMERR_LIMIT("io out of range");
> + return -EIO;
> + }
> +
> + if (bio_data_dir(bio) == WRITE)
> + return -EIO;
> +
> + io = mempool_alloc(v->io_mempool, GFP_NOIO);
> + io->v = v;
> + io->bio = bio;
> + io->orig_bi_end_io = bio->bi_end_io;
> + io->orig_bi_private = bio->bi_private;
> + io->block = bio->bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT);
> + io->n_blocks = bio->bi_size >> v->data_dev_block_bits;
> +
> + bio->bi_end_io = verity_end_io;
> + bio->bi_private = io;
> + bio->bi_bdev = v->data_dev->bdev;
> + bio->bi_sector = verity_map_sector(v, bio->bi_sector);
> +
> + io->io_vec_size = bio->bi_vcnt - bio->bi_idx;
> + if (io->io_vec_size < DM_VERITY_IO_VEC_INLINE)
> + io->io_vec = io->io_vec_inline;
> + else
> + io->io_vec = mempool_alloc(v->vec_mempool, GFP_NOIO);
> + memcpy(io->io_vec, bio_iovec(bio),
> + io->io_vec_size * sizeof(struct bio_vec));
> +
> + verity_prefetch_io(v, io);
> +
> + generic_make_request(bio);
> +
> + return DM_MAPIO_SUBMITTED;
> +}
> +
> +static int verity_status(struct dm_target *ti, status_type_t type,
> + char *result, unsigned maxlen)
> +{
> + struct dm_verity *v = ti->private;
> + unsigned sz = 0;
> + unsigned x;
> +
> + switch (type) {
> + case STATUSTYPE_INFO:
> + result[0] = 0;
> + break;
> + case STATUSTYPE_TABLE:
> + DMEMIT("%u %s %s %llu %u %s ",
> + 0,
> + v->data_dev->name,
> + v->hash_dev->name,
> + (unsigned long long)v->hash_start << (v->hash_dev_block_bits - SECTOR_SHIFT),
> + 1 << v->data_dev_block_bits,
> + v->alg_name
> + );
> + for (x = 0; x < v->digest_size; x++)
> + DMEMIT("%02x", v->root_digest[x]);
> + DMEMIT(" ");
> + if (!v->salt_size)
> + DMEMIT("-");
> + else
> + for (x = 0; x < v->salt_size; x++)
> + DMEMIT("%02x", v->salt[x]);
> + if (v->data_dev_block_bits != v->hash_dev_block_bits)
> + DMEMIT(" %u", 1 << v->hash_dev_block_bits);
> + break;
> + }
> + return 0;
> +}
> +
> +static int verity_ioctl(struct dm_target *ti, unsigned cmd,
> + unsigned long arg)
> +{
> + struct dm_verity *v = ti->private;
> + int r = 0;
> +
> + if (v->data_start ||
> + ti->len != i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT)
> + r = scsi_verify_blk_ioctl(NULL, cmd);
> +
> + return r ? : __blkdev_driver_ioctl(v->data_dev->bdev, v->data_dev->mode,
> + cmd, arg);
> +}
> +
> +static int verity_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
> + struct bio_vec *biovec, int max_size)
> +{
> + struct dm_verity *v = ti->private;
> + struct request_queue *q = bdev_get_queue(v->data_dev->bdev);
> +
> + if (!q->merge_bvec_fn)
> + return max_size;
> +
> + bvm->bi_bdev = v->data_dev->bdev;
> + bvm->bi_sector = verity_map_sector(v, bvm->bi_sector);
> +
> + return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
> +}
> +
> +static int verity_iterate_devices(struct dm_target *ti,
> + iterate_devices_callout_fn fn, void *data)
> +{
> + struct dm_verity *v = ti->private;
> + return fn(ti, v->data_dev, v->data_start, ti->len, data);
> +}
> +
> +static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits)
> +{
> + struct dm_verity *v = ti->private;
> +
> + if (limits->logical_block_size < 1 << v->data_dev_block_bits)
> + limits->logical_block_size = 1 << v->data_dev_block_bits;
> + if (limits->physical_block_size < 1 << v->data_dev_block_bits)
> + limits->physical_block_size = 1 << v->data_dev_block_bits;
> + blk_limits_io_min(limits, limits->logical_block_size);
> +}
> +
> +static void verity_dtr(struct dm_target *ti);
> +
> +static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv)
> +{
> + struct dm_verity *v;
> + unsigned num;
> + unsigned long long hs;
> + int r;
> + int i;
> + sector_t hash_position;
> + char dummy;
> +
> + v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL);
> + if (!v) {
> + ti->error = "Cannot allocate verity structure";
> + return -ENOMEM;
> + }
> + ti->private = v;
> + v->ti = ti;
> +
> + if ((dm_table_get_mode(ti->table) & ~FMODE_READ) != 0) {
> + ti->error = "Device must be readonly";
> + r = -EINVAL;
> + goto bad;
> + }
> +
> + if (argc < 7) {
> + ti->error = "Not enough arguments";
> + r = -EINVAL;
> + goto bad;
> + }
> +
> + if (sscanf(argv[0], "%d%c", &num, &dummy) != 1 ||
> + num != 0) {
> + ti->error = "Invalid version";
> + r = -EINVAL;
> + goto bad;
> + }
> +
> + r = dm_get_device(ti, argv[1], FMODE_READ, &v->data_dev);
> + if (r) {
> + ti->error = "Data device lookup failed";
> + goto bad;
> + }
> +
> + r = dm_get_device(ti, argv[2], FMODE_READ, &v->hash_dev);
> + if (r) {
> + ti->error = "Data device lookup failed";
> + goto bad;
> + }
> +
> + if (sscanf(argv[3], "%llu%c", &hs, &dummy) != 1 ||
> + hs != (sector_t)hs) {
> + ti->error = "Invalid hash start";
> + r = -EINVAL;
> + goto bad;
> + }
> +
> + if (sscanf(argv[4], "%u%c", &num, &dummy) != 1 ||
> + !num || (num & (num - 1)) ||
> + num < bdev_logical_block_size(v->data_dev->bdev) ||
> + num > PAGE_SIZE) {
> + ti->error = "Invalid data device block size";
> + r = -EINVAL;
> + goto bad;
> + }
> + v->data_dev_block_bits = ffs(num) - 1;
> + v->hash_dev_block_bits = ffs(num) - 1;
> +
> + v->alg_name = kstrdup(argv[5], GFP_KERNEL);
> + if (!v->alg_name) {
> + ti->error = "Cannot allocate algorithm name";
> + r = -ENOMEM;
> + goto bad;
> + }
> +
> + v->tfm = crypto_alloc_shash(v->alg_name, 0, 0);
> + if (IS_ERR(v->tfm)) {
> + ti->error = "Cannot initialize hash function";
> + r = PTR_ERR(v->tfm);
> + v->tfm = NULL;
> + goto bad;
> + }
> + v->digest_size = crypto_shash_digestsize(v->tfm);
> + if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) {
> + ti->error = "Digest size too big";
> + r = -EINVAL;
> + goto bad;
> + }
> + v->shash_descsize =
> + sizeof(struct shash_desc) + crypto_shash_descsize(v->tfm);
> +
> + v->root_digest = kmalloc(v->digest_size, GFP_KERNEL);
> + if (!v->root_digest) {
> + ti->error = "Cannot allocate root digest";
> + r = -ENOMEM;
> + goto bad;
> + }
> + if (strlen(argv[6]) != v->digest_size * 2 ||
> + hex2bin(v->root_digest, argv[6], v->digest_size)) {
> + ti->error = "Invalid root digest";
> + r = -EINVAL;
> + goto bad;
> + }
> +
> + if (argc > 7 && strcmp(argv[7], "-")) {
> + v->salt_size = strlen(argv[7]) / 2;
> + v->salt = kmalloc(v->salt_size, GFP_KERNEL);
> + if (!v->salt) {
> + ti->error = "Cannot allocate salt";
> + r = -ENOMEM;
> + goto bad;
> + }
> + if (strlen(argv[7]) != v->salt_size * 2 ||
> + hex2bin(v->salt, argv[7], v->salt_size)) {
> + ti->error = "Invalid salt";
> + r = -EINVAL;
> + goto bad;
> + }
> + }
> +
> + if (argc > 8) {
> + if (sscanf(argv[8], "%u%c", &num, &dummy) != 1 ||
> + !num || (num & (num - 1)) ||
> + num < bdev_logical_block_size(v->hash_dev->bdev) ||
> + num > INT_MAX) {
> + ti->error = "Invalid hash device block size";
> + r = -EINVAL;
> + goto bad;
> + }
> + v->hash_dev_block_bits = ffs(num) - 1;
> + }
> +
> + if (hs & ((1 << (v->hash_dev_block_bits - SECTOR_SHIFT)) - 1)) {
> + ti->error = "Hash start not aligned on block boundary";
> + r = -EINVAL;
> + goto bad;
> + }
> + v->hash_start = hs >> (v->hash_dev_block_bits - SECTOR_SHIFT);
> +
> + if (ti->len > i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT) {
> + ti->error = "Data device si too small";
> + r = -EINVAL;
> + goto bad;
> + }
> +
> + if (ti->len & ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) {
> + ti->error = "Data device length is not aligned to block size";
> + r = -EINVAL;
> + goto bad;
> + }
> +
> + v->data_blocks = ti->len >> (v->data_dev_block_bits - SECTOR_SHIFT);
> +
> + v->hash_per_block_bits =
> + fls((1 << v->hash_dev_block_bits) / v->digest_size) - 1;
> +
> + v->levels = 0;
> + if (v->data_blocks)
> + while (v->hash_per_block_bits * v->levels < 64 &&
> + (unsigned long long)(v->data_blocks - 1) >>
> + (v->hash_per_block_bits * v->levels))
> + v->levels++;
> +
> + if (v->levels > DM_VERITY_MAX_LEVELS) {
> + ti->error = "Too many tree levels";
> + r = -E2BIG;
> + goto bad;
> + }
> +
> + hash_position = v->hash_start;
> + for (i = v->levels - 1; i >= 0; i--) {
> + sector_t s;
> + v->hash_level_block[i] = hash_position;
> + s = verity_position_at_level(v, v->data_blocks, i);
> + s = (s >> v->hash_per_block_bits) +
> + !!(s & ((1 << v->hash_per_block_bits) - 1));
> + if (hash_position + s < hash_position) {
> + ti->error = "Hash device offset overflow";
> + r = -E2BIG;
> + goto bad;
> + }
> + hash_position += s;
> + }
> + v->hash_blocks = hash_position;
> +
> + v->bufio = dm_bufio_client_create(v->hash_dev->bdev,
> + 1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux),
> + dm_bufio_alloc_callback, NULL);
> + if (IS_ERR(v->bufio)) {
> + ti->error = "Cannot initialize dm-bufio";
> + r = PTR_ERR(v->bufio);
> + v->bufio = NULL;
> + goto bad;
> + }
> +
> + if (dm_bufio_get_device_size(v->bufio) < v->hash_blocks) {
> + ti->error = "Hash device is too small";
> + r = -E2BIG;
> + goto bad;
> + }
> +
> + v->io_mempool = mempool_create_kmalloc_pool(DM_VERITY_MEMPOOL_SIZE ,
> + sizeof(struct dm_verity_io) + v->shash_descsize + v->digest_size * 2);
> + if (!v->io_mempool) {
> + ti->error = "Cannot allocate io mempool";
> + r = -ENOMEM;
> + goto bad;
> + }
> +
> + v->vec_mempool = mempool_create_kmalloc_pool(DM_VERITY_MEMPOOL_SIZE ,
> + BIO_MAX_PAGES * sizeof(struct bio_vec));
> + if (!v->vec_mempool) {
> + ti->error = "Cannot allocate vector mempool";
> + r = -ENOMEM;
> + goto bad;
> + }
> +
> + /*v->verify_wq = alloc_workqueue("verityd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 1);*/
> + /* WQ_UNBOUND greatly improves performance when running on ramdisk */
> + v->verify_wq = alloc_workqueue("verityd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, num_online_cpus());
> + if (!v->verify_wq) {
> + ti->error = "Cannot allocate workqueue";
> + r = -ENOMEM;
> + goto bad;
> + }
> +
> + return 0;
> +
> +bad:
> + verity_dtr(ti);
> + return r;
> +}
> +
> +static void verity_dtr(struct dm_target *ti)
> +{
> + struct dm_verity *v = ti->private;
> +
> + if (v->verify_wq)
> + destroy_workqueue(v->verify_wq);
> + if (v->vec_mempool)
> + mempool_destroy(v->vec_mempool);
> + if (v->io_mempool)
> + mempool_destroy(v->io_mempool);
> + if (v->bufio)
> + dm_bufio_client_destroy(v->bufio);
> + kfree(v->salt);
> + kfree(v->root_digest);
> + if (v->tfm)
> + crypto_free_shash(v->tfm);
> + kfree(v->alg_name);
> + if (v->hash_dev)
> + dm_put_device(ti, v->hash_dev);
> + if (v->data_dev)
> + dm_put_device(ti, v->data_dev);
> + kfree(v);
> +}
> +
> +static struct target_type verity_target = {
> + .name = "verity",
> + .version = {1, 0, 0},
> + .module = THIS_MODULE,
> + .ctr = verity_ctr,
> + .dtr = verity_dtr,
> + .map = verity_map,
> + .status = verity_status,
> + .ioctl = verity_ioctl,
> + .merge = verity_merge,
> + .iterate_devices = verity_iterate_devices,
> + .io_hints = verity_io_hints,
> +};
> +
> +static int __init dm_verity_init(void)
> +{
> + int r;
> + r = dm_register_target(&verity_target);
> + if (r < 0)
> + DMERR("register failed %d", r);
> + return r;
> +}
> +
> +static void __exit dm_verity_exit(void)
> +{
> + dm_unregister_target(&verity_target);
> +}
> +
> +module_init(dm_verity_init);
> +module_exit(dm_verity_exit);
> +
> +MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>");
> +MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking");
> +MODULE_LICENSE("GPL");
> +
> Index: linux-3.3-rc6-fast/drivers/md/dm-bufio.c
> ================================================== =================
> --- linux-3.3-rc6-fast.orig/drivers/md/dm-bufio.c 2012-03-12 22:43:23.000000000 +0100
> +++ linux-3.3-rc6-fast/drivers/md/dm-bufio.c 2012-03-13 15:41:02.000000000 +0100
> @@ -579,7 +579,7 @@ static void write_endio(struct bio *bio,
> struct dm_buffer *b = container_of(bio, struct dm_buffer, bio);
>
> b->write_error = error;
> - if (error) {
> + if (unlikely(error)) {
> struct dm_bufio_client *c = b->c;
> (void)cmpxchg(&c->async_write_error, 0, error);
> }
> @@ -698,13 +698,20 @@ static void __wait_for_free_buffer(struc
> dm_bufio_lock(c);
> }
>
> +enum new_flag {
> + NF_FRESH = 0,
> + NF_READ = 1,
> + NF_GET = 2,
> + NF_PREFETCH = 3
> +};
> +
> /*
> * Allocate a new buffer. If the allocation is not possible, wait until
> * some other thread frees a buffer.
> *
> * May drop the lock and regain it.
> */
> -static struct dm_buffer *__alloc_buffer_wait_no_callback(struct dm_bufio_client *c)
> +static struct dm_buffer *__alloc_buffer_wait_no_callback(struct dm_bufio_client *c, enum new_flag nf)
> {
> struct dm_buffer *b;
>
> @@ -727,6 +734,9 @@ static struct dm_buffer *__alloc_buffer_
> return b;
> }
>
> + if (nf == NF_PREFETCH)
> + return NULL;
> +
> if (!list_empty(&c->reserved_buffers)) {
> b = list_entry(c->reserved_buffers.next,
> struct dm_buffer, lru_list);
> @@ -744,9 +754,12 @@ static struct dm_buffer *__alloc_buffer_
> }
> }
>
> -static struct dm_buffer *__alloc_buffer_wait(struct dm_bufio_client *c)
> +static struct dm_buffer *__alloc_buffer_wait(struct dm_bufio_client *c, enum new_flag nf)
> {
> - struct dm_buffer *b = __alloc_buffer_wait_no_callback(c);
> + struct dm_buffer *b = __alloc_buffer_wait_no_callback(c, nf);
> +
> + if (!b)
> + return NULL;
>
> if (c->alloc_callback)
> c->alloc_callback(b);
> @@ -866,15 +879,8 @@ static struct dm_buffer *__find(struct d
> * Getting a buffer
> *--------------------------------------------------------------*/
>
> -enum new_flag {
> - NF_FRESH = 0,
> - NF_READ = 1,
> - NF_GET = 2
> -};
> -
> static struct dm_buffer *__bufio_new(struct dm_bufio_client *c, sector_t block,
> - enum new_flag nf, struct dm_buffer **bp,
> - int *need_submit)
> + enum new_flag nf, int *need_submit)
> {
> struct dm_buffer *b, *new_b = NULL;
>
> @@ -882,6 +888,19 @@ static struct dm_buffer *__bufio_new(str
>
> b = __find(c, block);
> if (b) {
> +found_buffer:
> + if (nf == NF_PREFETCH)
> + return NULL;
> + /*
> + * Note: it is essential that we don't wait for the buffer to be
> + * read if dm_bufio_get function is used. Both dm_bufio_get and
> + * dm_bufio_prefetch can be used in the driver request routine.
> + * If the user called both dm_bufio_prefetch and dm_bufio_get on
> + * the same buffer, it would deadlock if we waited.
> + */
> + if (nf == NF_GET && unlikely(test_bit(B_READING, &b->state)))
> + return NULL;
> +
> b->hold_count++;
> __relink_lru(b, test_bit(B_DIRTY, &b->state) ||
> test_bit(B_WRITING, &b->state));
> @@ -891,7 +910,9 @@ static struct dm_buffer *__bufio_new(str
> if (nf == NF_GET)
> return NULL;
>
> - new_b = __alloc_buffer_wait(c);
> + new_b = __alloc_buffer_wait(c, nf);
> + if (!new_b)
> + return NULL;
>
> /*
> * We've had a period where the mutex was unlocked, so need to
> @@ -900,10 +921,7 @@ static struct dm_buffer *__bufio_new(str
> b = __find(c, block);
> if (b) {
> __free_buffer_wake(new_b);
> - b->hold_count++;
> - __relink_lru(b, test_bit(B_DIRTY, &b->state) ||
> - test_bit(B_WRITING, &b->state));
> - return b;
> + goto found_buffer;
> }
>
> __check_watermark(c);
> @@ -957,7 +975,7 @@ static void *new_read(struct dm_bufio_cl
> struct dm_buffer *b;
>
> dm_bufio_lock(c);
> - b = __bufio_new(c, block, nf, bp, &need_submit);
> + b = __bufio_new(c, block, nf, &need_submit);
> dm_bufio_unlock(c);
>
> if (!b || IS_ERR(b))
> @@ -1006,13 +1024,46 @@ void *dm_bufio_new(struct dm_bufio_clien
> }
> EXPORT_SYMBOL_GPL(dm_bufio_new);
>
> +void dm_bufio_prefetch(struct dm_bufio_client *c,
> + sector_t block, unsigned n_blocks)
> +{
> + struct blk_plug plug;
> +
> + blk_start_plug(&plug);
> + dm_bufio_lock(c);
> +
> + for (; n_blocks--; block++) {
> + int need_submit;
> + struct dm_buffer *b;
> + b = __bufio_new(c, block, NF_PREFETCH, &need_submit);
> + if (unlikely(b != NULL)) {
> + dm_bufio_unlock(c);
> +
> + if (need_submit)
> + submit_io(b, READ, b->block, read_endio);
> + dm_bufio_release(b);
> +
> + dm_bufio_cond_resched();
> +
> + if (!n_blocks)
> + goto flush_plug;
> + dm_bufio_lock(c);
> + }
> +
> + }
> +
> + dm_bufio_unlock(c);
> +flush_plug:
> + blk_finish_plug(&plug);
> +}
> +EXPORT_SYMBOL_GPL(dm_bufio_prefetch);
> +
> void dm_bufio_release(struct dm_buffer *b)
> {
> struct dm_bufio_client *c = b->c;
>
> dm_bufio_lock(c);
>
> - BUG_ON(test_bit(B_READING, &b->state));
> BUG_ON(!b->hold_count);
>
> b->hold_count--;
> @@ -1025,6 +1076,7 @@ void dm_bufio_release(struct dm_buffer *
> * invalid buffer.
> */
> if ((b->read_error || b->write_error) &&
> + !test_bit(B_READING, &b->state) &&
> !test_bit(B_WRITING, &b->state) &&
> !test_bit(B_DIRTY, &b->state)) {
> __unlink_buffer(b);
> @@ -1042,6 +1094,8 @@ void dm_bufio_mark_buffer_dirty(struct d
>
> dm_bufio_lock(c);
>
> + BUG_ON(test_bit(B_READING, &b->state));
> +
> if (!test_and_set_bit(B_DIRTY, &b->state))
> __relink_lru(b, LIST_DIRTY);
>
> Index: linux-3.3-rc6-fast/drivers/md/dm-bufio.h
> ================================================== =================
> --- linux-3.3-rc6-fast.orig/drivers/md/dm-bufio.h 2012-03-12 22:43:23.000000000 +0100
> +++ linux-3.3-rc6-fast/drivers/md/dm-bufio.h 2012-03-12 22:43:25.000000000 +0100
> @@ -63,6 +63,14 @@ void *dm_bufio_new(struct dm_bufio_clien
> struct dm_buffer **bp);
>
> /*
> + * Prefetch the specified blocks to the cache.
> + * The function starts to read the blocks and returns without waiting for
> + * I/O to finish.
> + */
> +void dm_bufio_prefetch(struct dm_bufio_client *c,
> + sector_t block, unsigned n_blocks);
> +
> +/*
> * Release a reference obtained with dm_bufio_{read,get,new}. The data
> * pointer and dm_buffer pointer is no longer valid after this call.
> */

--
dm-devel mailing list
dm-devel@redhat.com
https://www.redhat.com/mailman/listinfo/dm-devel
 
Old 03-17-2012, 12:16 AM
Mikulas Patocka
 
Default dm: remake of the verity target

Hi Will


On Wed, 14 Mar 2012, Will Drewry wrote:

> Hi Mikulas,
>
> This is a nice rewrite and takes advantage of your dm-bufio layer. I
> wish it'd existed (and or we wrote it in 2009 when we started this
> work! Some comments below:
>
> > ---
> > +static void verity_prefetch_io(struct dm_verity *v, struct dm_verity_io *io)
> > +{
> > + * * * int i;
> > + * * * for (i = v->levels - 2; i >= 0; i--) {
> > + * * * * * * * sector_t hash_block_start;
> > + * * * * * * * sector_t hash_block_end;
> > + * * * * * * * verity_hash_at_level(v, io->block, i, &hash_block_start, NULL);
> > + * * * * * * * verity_hash_at_level(v, io->block + io->n_blocks - 1, i, &hash_block_end, NULL);
> > + * * * * * * * if (!i) {
> > + * * * * * * * * * * * unsigned cluster = prefetch_cluster;
> > + * * * */* barrier to stop GCC from re-reading prefetch_cluster again */
> > + * * * * * * * * * * * barrier();
> > + * * * * * * * * * * * cluster >>= v->data_dev_block_bits;
>
> Would:
> unsigned cluster = prefetch_cluster >> v->data_dev_block_bits;
> not have similar behavior without a barrier? (Yeah yeah I could
> compile and see, but I was curious if you already had.)
>
> Since the max iterations here is 61 in a worst-case, I don't think
> it's a big deal to barrier() each time, just thought I'd ask.
>
> > + * * * * * * * * * * * if (unlikely(!cluster))
> > + * * * * * * * * * * * * * * * goto no_prefetch_cluster;
> > + * * * * * * * * * * * if (unlikely(cluster & (cluster - 1)))
> > + * * * * * * * * * * * * * * * cluster = 1 << (fls(cluster) - 1);
> > +
> > + * * * * * * * * * * * hash_block_start &= ~(sector_t)(cluster - 1);
> > + * * * * * * * * * * * hash_block_end |= cluster - 1;
> > + * * * * * * * * * * * if (unlikely(hash_block_end >= v->hash_blocks))
> > + * * * * * * * * * * * * * * * hash_block_end = v->hash_blocks - 1;
> > + * * * * * * * }
> > +no_prefetch_cluster:
> > + * * * * * * * dm_bufio_prefetch(v->bufio, hash_block_start,
> > + * * * * * * * * * * * * * * * * * * * hash_block_end - hash_block_start + 1);

The problem here is this. If you look at the code, you think that after
the clause "if (unlikely(!cluster)) goto no_prefetch_cluster;", cluster
can't be zero. But this assumption is wrong. The C compiler is allowed to
transform the above code into:

unsigned cluster;
if (!(prefetch_cluster >> v->data_dev_block_bits))
goto no_prefetch_cluster;
cluster = prefetch_cluster >> v->data_dev_block_bits;
if (unlikely(cluster & (cluster - 1)))
cluster = 1 << (fls(cluster) - 1);

I know it's suboptimal, but the C compiler is just allowed to perform this
transformation. Now, if you know that "prefetch_cluster" can change
asynchronously by another thread running simultaneously, the condition "if
(!(prefetch_cluster >> v->data_dev_block_bits))" is useless ---
prefetch_cluster may change just after this condition and we won't catch
the zero value. (if the cluster value is zero in the above code, it ends
up in hash_block_end being ORed with -1 and the prefetch goes wild over
the whole hash device).

That's why I put that "barrier()" there. It would be better to declare
"prefetch_cluster" as volatile, but the module param macros issue warnings
if the variable is volatile.

Or maybe I can change it this way:
"unsigned cluster = *(volatile unsigned *)&prefetch_cluster", it could be
better than the "barrier()".

> > + * * * case STATUSTYPE_TABLE:
> > + * * * * * * * DMEMIT("%u %s %s %llu %u %s ",
> > + * * * * * * * * * * * 0,
> > + * * * * * * * * * * * v->data_dev->name,
> > + * * * * * * * * * * * v->hash_dev->name,
>
> I understand the new approach is to use major:minor instead of the
> device name. I don't care which, but I believe agk@ requested that.

All the device mappers report dm_dev->name in their status routine, so I
do it this way too.

> > +static int verity_ioctl(struct dm_target *ti, unsigned cmd,
> > + * * * * * * * * * * * unsigned long arg)
> > +{
> > + * * * struct dm_verity *v = ti->private;
> > + * * * int r = 0;
> > +
> > + * * * if (v->data_start ||
> > + * * * * * ti->len != i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT)
> > + * * * * * * * r = scsi_verify_blk_ioctl(NULL, cmd);
> > +
>
> Is it worth supporting ioctl at all given these hoops? Nothing stops
> a privileged user from directly running the ioctl on the underlying
> device/devices, it's just very inconvenient

I don't know. The other dm targets attempt to pass-thru ioctls too.

You need ioctl pass-thru if you want to run it over a cd-rom because
the iso9660 filesystem needs to send an ioctl to find its superblock.
Other than that I don't know if other filesystems need ioctls.

> > + * * * if (ti->len > i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT) {
> > + * * * * * * * ti->error = "Data device si too small";
>
> s/si/is
>
> Should this also check ti->start + ti->len to ensure it isn't reading
> off the end or do you just rely on the requests failing?

ti->start is the offset in the target table --- so it shouldn't be checked
here (for example, you can map a verity device having 1024 blocks to a
sector offset 1000000 in the table --- so ti->start == 1000000 and ti->len
== 1024 --- in this case, you have test that the underlying device has at
least 1024 blocks, but you shouldn't test it for 1000000 sectors ---
1000000 is offset in the table, not required device size.

But this reminds me that I had the size test wrong in verity_map ...
fixed.

> > +MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>");
>
> As per linux/module.h, I'd welcome additional authors as per the
> lkml/patch lineage:
> MODULE_AUTHOR("Mandeep Baines <msb@chromium.org>");
> MODULE_AUTHOR("Will Drewry <wad@chromium.org>");

OK, I added you there.

> Regardless, I'll just be happy to see this functionality merge.
>
> > +MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking");
> > +MODULE_LICENSE("GPL");
> > +
> > Index: linux-3.3-rc6-fast/drivers/md/dm-bufio.c
>
> This should be in a separate patch I think.

Yes, it is a separate patch.

> > * * * * * * * *b->hold_count++;
>
> Are these hold_counts safe on architectures with weak memory models?
> Should they be atomic_ts? I haven't looked at them in context, but
> based on what I see here they make me a bit nervous.
>
> Thanks for jumping in to the fray! None of my comments are blocking,
> so I believe the following is appropriate (if not
> s/Signed-off/Reviewed-by/).
>
> Signed-off-by: Will Drewry <wad@chromium.org>
>
> cheers!
> will

hold_count is read or changed only when we hold dm_bufio_client->lock, so
it doesn't have to be atomic.

Mikulas--
dm-devel mailing list
dm-devel@redhat.com
https://www.redhat.com/mailman/listinfo/dm-devel
 
Old 03-17-2012, 02:06 AM
Will Drewry
 
Default dm: remake of the verity target

On Fri, Mar 16, 2012 at 8:16 PM, Mikulas Patocka <mpatocka@redhat.com> wrote:
> Hi Will
>
>
> On Wed, 14 Mar 2012, Will Drewry wrote:
>
>> Hi Mikulas,
>>
>> This is a nice rewrite and takes advantage of your dm-bufio layer. I
>> wish it'd existed (and or we wrote it in 2009 when we started this
>> work! *Some comments below:
>>
>> > ---
>> > +static void verity_prefetch_io(struct dm_verity *v, struct dm_verity_io *io)
>> > +{
>> > + * * * int i;
>> > + * * * for (i = v->levels - 2; i >= 0; i--) {
>> > + * * * * * * * sector_t hash_block_start;
>> > + * * * * * * * sector_t hash_block_end;
>> > + * * * * * * * verity_hash_at_level(v, io->block, i, &hash_block_start, NULL);
>> > + * * * * * * * verity_hash_at_level(v, io->block + io->n_blocks - 1, i, &hash_block_end, NULL);
>> > + * * * * * * * if (!i) {
>> > + * * * * * * * * * * * unsigned cluster = prefetch_cluster;
>> > + * * * */* barrier to stop GCC from re-reading prefetch_cluster again */
>> > + * * * * * * * * * * * barrier();
>> > + * * * * * * * * * * * cluster >>= v->data_dev_block_bits;
>>
>> Would:
>> * unsigned cluster = prefetch_cluster >> v->data_dev_block_bits;
>> not have similar behavior without a barrier? *(Yeah yeah I could
>> compile and see, but I was curious if you already had.)
>>
>> Since the max iterations here is 61 in a worst-case, I don't think
>> it's a big deal to barrier() each time, just thought I'd ask.
>>
>> > + * * * * * * * * * * * if (unlikely(!cluster))
>> > + * * * * * * * * * * * * * * * goto no_prefetch_cluster;
>> > + * * * * * * * * * * * if (unlikely(cluster & (cluster - 1)))
>> > + * * * * * * * * * * * * * * * cluster = 1 << (fls(cluster) - 1);
>> > +
>> > + * * * * * * * * * * * hash_block_start &= ~(sector_t)(cluster - 1);
>> > + * * * * * * * * * * * hash_block_end |= cluster - 1;
>> > + * * * * * * * * * * * if (unlikely(hash_block_end >= v->hash_blocks))
>> > + * * * * * * * * * * * * * * * hash_block_end = v->hash_blocks - 1;
>> > + * * * * * * * }
>> > +no_prefetch_cluster:
>> > + * * * * * * * dm_bufio_prefetch(v->bufio, hash_block_start,
>> > + * * * * * * * * * * * * * * * * * * * hash_block_end - hash_block_start + 1);
>
> The problem here is this. If you look at the code, you think that after
> the clause "if (unlikely(!cluster)) goto no_prefetch_cluster;", cluster
> can't be zero. But this assumption is wrong. The C compiler is allowed to
> transform the above code into:
>
> unsigned cluster;
> if (!(prefetch_cluster >> v->data_dev_block_bits))
> * * * *goto no_prefetch_cluster;
> cluster = prefetch_cluster >> v->data_dev_block_bits;
> if (unlikely(cluster & (cluster - 1)))
> * * * *cluster = 1 << (fls(cluster) - 1);
>
> I know it's suboptimal, but the C compiler is just allowed to perform this
> transformation. Now, if you know that "prefetch_cluster" can change
> asynchronously by another thread running simultaneously, the condition "if
> (!(prefetch_cluster >> v->data_dev_block_bits))" is useless ---
> prefetch_cluster may change just after this condition and we won't catch
> the zero value. (if the cluster value is zero in the above code, it ends
> up in hash_block_end being ORed with -1 and the prefetch goes wild over
> the whole hash device).
>
> That's why I put that "barrier()" there. It would be better to declare
> "prefetch_cluster" as volatile, but the module param macros issue warnings
> if the variable is volatile.
>
> Or maybe I can change it this way:
> "unsigned cluster = *(volatile unsigned *)&prefetch_cluster", it could be
> better than the "barrier()".

I think that'd read a little bit more clearly, and I think the C
standard supports that approach. If it doesn't work in practice, the
barrier isn't the worst.

>> > + * * * case STATUSTYPE_TABLE:
>> > + * * * * * * * DMEMIT("%u %s %s %llu %u %s ",
>> > + * * * * * * * * * * * 0,
>> > + * * * * * * * * * * * v->data_dev->name,
>> > + * * * * * * * * * * * v->hash_dev->name,
>>
>> I understand the new approach is to use major:minor instead of the
>> device name. *I don't care which, but I believe agk@ requested that.
>
> All the device mappers report dm_dev->name in their status routine, so I
> do it this way too.
>
>> > +static int verity_ioctl(struct dm_target *ti, unsigned cmd,
>> > + * * * * * * * * * * * unsigned long arg)
>> > +{
>> > + * * * struct dm_verity *v = ti->private;
>> > + * * * int r = 0;
>> > +
>> > + * * * if (v->data_start ||
>> > + * * * * * ti->len != i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT)
>> > + * * * * * * * r = scsi_verify_blk_ioctl(NULL, cmd);
>> > +
>>
>> Is it worth supporting ioctl at all given these hoops? *Nothing stops
>> a privileged user from directly running the ioctl on the underlying
>> device/devices, it's just very inconvenient
>
> I don't know. The other dm targets attempt to pass-thru ioctls too.
>
> You need ioctl pass-thru if you want to run it over a cd-rom because
> the iso9660 filesystem needs to send an ioctl to find its superblock.
> Other than that I don't know if other filesystems need ioctls.

Makes sense. I just think the passthrough condition is ugly, but at
least it provides some support.

>> > + * * * if (ti->len > i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT) {
>> > + * * * * * * * ti->error = "Data device si too small";
>>
>> s/si/is
>>
>> Should this also check ti->start + ti->len to ensure it isn't reading
>> off the end or do you just rely on the requests failing?
>
> ti->start is the offset in the target table --- so it shouldn't be checked
> here (for example, you can map a verity device having 1024 blocks to a
> sector offset 1000000 in the table --- so ti->start == 1000000 and ti->len
> == 1024 --- in this case, you have test that the underlying device has at
> least 1024 blocks, but you shouldn't test it for 1000000 sectors ---
> 1000000 is offset in the table, not required device size.
>
> But this reminds me that I had the size test wrong in verity_map ...
> fixed.

Well at least some good came of it! I typed ti->start, but I had mean
v->data_start. However, I was misinterpreting the i_size_read as
giving the last sector not the actual size, so my comment was still
pointless.

>> > +MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>");
>>
>> As per linux/module.h, I'd welcome additional authors as per the
>> lkml/patch lineage:
>> MODULE_AUTHOR("Mandeep Baines <msb@chromium.org>");
>> MODULE_AUTHOR("Will Drewry <wad@chromium.org>");
>
> OK, I added you there.

Very much appreciated.

>> Regardless, I'll just be happy to see this functionality merge.
>>
>> > +MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking");
>> > +MODULE_LICENSE("GPL");
>> > +
>> > Index: linux-3.3-rc6-fast/drivers/md/dm-bufio.c
>>
>> This should be in a separate patch I think.
>
> Yes, it is a separate patch.
>
>> > * * * * * * * *b->hold_count++;
>>
>> Are these hold_counts safe on architectures with weak memory models?
>> Should they be atomic_ts? * I haven't looked at them in context, but
>> based on what I see here they make me a bit nervous.
>>
>> Thanks for jumping in to the fray! *None of my comments are blocking,
>> so I believe the following is appropriate (if not
>> s/Signed-off/Reviewed-by/).
>>
>> Signed-off-by: Will Drewry <wad@chromium.org>
>>
>> cheers!
>> will
>
> hold_count is read or changed only when we hold dm_bufio_client->lock, so
> it doesn't have to be atomic.

Ah nice! The little snippet had me worried, but I should've looked at
the full context first.

Thanks!
will

--
dm-devel mailing list
dm-devel@redhat.com
https://www.redhat.com/mailman/listinfo/dm-devel
 
Old 03-20-2012, 02:41 PM
Mandeep Singh Baines
 
Default dm: remake of the verity target

Hi Mikulas,

Can you please resend this patch with a proper commit message.
We'd really like to see this merged. Alasdair, other than that,
what work is remaining for verity to be merged?

Regards,
Mandeep

Mikulas Patocka (mpatocka@redhat.com) wrote:
> Hi
>
> > Hi Mikulus,
> >
> > This is some nice work. I like that you've been able to abstract a lot
> > of the hash buffer management with dm-bufio. You got rid of the I/O queue.
> > I've been meaning to do that for a while. The prefetch is also nice.
> > We planned to do this but I decided to not do it now in order to get the
> > base functionality in:
> >
> > http://crosbug.com/25441
> >
> > However, there are some things that I don't like. I don't like comments
> > either but you have none. You also removed our documentation. You are
>
> I added some comments. As for documentation, it's OK to use documentation
> from your patch because the on-disk format and the target arguments are
> the same (with an enhancement that my code supports different data and
> metadata bock size and it has variable-length salt).
>
> > allocated a complete shash_desc per I/O. We only allocate one per CPU.
>
> The hash of 4k block takes 174000 cycles. So trying to optimize
> memory latency that is about 250 cycles doesn't make much sense.
>
> I actually observed better performance using verity on ramdisk with
> workqueue unbound to specific CPUs. The reason is that the ramdisk bio
> completion routine is always run on the same CPU (that one that submitted
> the request), so with bound workqueue, everything was executing on one
> CPU. With unbound workqueue, I got parallelism.
>
> > We short-circuit the hash at any level. Your implementation can only
> > shirt circuit at the lowest level.
>
> It short-circuits hash at all levels. If the function
> "verity_verify_level" finds out that "aux->hash_verified" is non-zero, it
> doesn't do any hashing, it just copies the hash for the lower level. My
> implementation walks the tree from the top to the bottom, but it doesn't
> do hash verification if the same block has been verified before.
>
> All this tree-walking from the root to the bottom is 50-times faster than
> the actual hashing of the data block (I measured that), so there's not
> much point in trying to optimize it. I did a simple optimization (don't
> walk the tree if the lowest block is already verified) and I don't need to
> do anything complicated given the fact that it can't improve more than by
> 2%.
>
> > I'd like to propose that we get the version we sent upstream and then work
> > together on adding some of your enhancements incrementally.
>
> If you add dm-bufio support, you end up deleting majority of the original
> code anyway. That's why I wrote it from scratch and that's why I didn't
> attempt to morph your code.
>
> It's simpler to write the code from scratch and it is also less bug-prone.
>
> > Other than
> > the changes we've made to cleanup for upstreaming, the version I
> > submitted is the code we are using in production.
> >
> > I'm happy to add prefetch now if that is required for merging.
> >
> > What do you think?
> >
> > Regards,
> > Mandeep
>
> This is the version with comments added:
>
> Mikulas
>
> ----
>
> Remake of the google dm-verity patch.
>
> Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
>
> ---
> drivers/md/Kconfig | 17
> drivers/md/Makefile | 1
> drivers/md/dm-verity.c | 851 +++++++++++++++++++++++++++++++++++++++++++++++++
> 3 files changed, 869 insertions(+)
>
> Index: linux-3.3-rc6-fast/drivers/md/Kconfig
> ================================================== =================
> --- linux-3.3-rc6-fast.orig/drivers/md/Kconfig 2012-03-13 21:46:03.000000000 +0100
> +++ linux-3.3-rc6-fast/drivers/md/Kconfig 2012-03-13 21:46:05.000000000 +0100
> @@ -404,4 +404,21 @@ config DM_VERITY2
>
> If unsure, say N.
>
> +config DM_VERITY
> + tristate "Verity target support"
> + depends on BLK_DEV_DM
> + select CRYPTO
> + select CRYPTO_HASH
> + select DM_BUFIO
> + ---help---
> + This device-mapper target allows you to create a device that
> + transparently integrity checks the data on it. You'll need to
> + activate the digests you're going to use in the cryptoapi
> + configuration.
> +
> + To compile this code as a module, choose M here: the module will
> + be called dm-verity.
> +
> + If unsure, say N.
> +
> endif # MD
> Index: linux-3.3-rc6-fast/drivers/md/Makefile
> ================================================== =================
> --- linux-3.3-rc6-fast.orig/drivers/md/Makefile 2012-03-13 21:46:03.000000000 +0100
> +++ linux-3.3-rc6-fast/drivers/md/Makefile 2012-03-13 21:46:05.000000000 +0100
> @@ -29,6 +29,7 @@ obj-$(CONFIG_MD_FAULTY) += faulty.o
> obj-$(CONFIG_BLK_DEV_MD) += md-mod.o
> obj-$(CONFIG_BLK_DEV_DM) += dm-mod.o
> obj-$(CONFIG_DM_BUFIO) += dm-bufio.o
> +obj-$(CONFIG_DM_VERITY) += dm-verity.o
> obj-$(CONFIG_DM_CRYPT) += dm-crypt.o
> obj-$(CONFIG_DM_DELAY) += dm-delay.o
> obj-$(CONFIG_DM_FLAKEY) += dm-flakey.o
> Index: linux-3.3-rc6-fast/drivers/md/dm-verity.c
> ================================================== =================
> --- /dev/null 1970-01-01 00:00:00.000000000 +0000
> +++ linux-3.3-rc6-fast/drivers/md/dm-verity.c 2012-03-13 22:02:05.000000000 +0100
> @@ -0,0 +1,851 @@
> +/*
> + * Copyright (C) 2012 Red Hat, Inc.
> + *
> + * Author: Mikulas Patocka <mpatocka@redhat.com>
> + *
> + * Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors
> + *
> + * This file is released under the GPLv2.
> + *
> + * Device mapper target parameters:
> + * <version> 0
> + * <data device>
> + * <hash device>
> + * <hash start> (typically 0)
> + * <block size> (typically 4096)
> + * <algorithm>
> + * <digest>
> + * optional parameters:
> + * <salt> (should have 32 bytes for compatibility with Google code)
> + * <hash block size> (by default it is the same as data block size)
> + *
> + * In the file "/sys/module/dm_verity/parameters/prefetch_cluster" you can set
> + * default prefetch value. Data are read in "prefetch_cluster" chunks from the
> + * hash device. Prefetch cluster greatly improves performance when data and hash
> + * are on the same disk on different partitions.
> + */
> +
> +#include <linux/module.h>
> +#include <linux/device-mapper.h>
> +#include <crypto/hash.h>
> +#include "dm-bufio.h"
> +
> +#define DM_MSG_PREFIX "verity"
> +
> +#define DM_VERITY_IO_VEC_INLINE 16
> +#define DM_VERITY_MEMPOOL_SIZE 4
> +#define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144
> +
> +#define DM_VERITY_MAX_LEVELS 63
> +
> +static unsigned prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE;
> +
> +module_param_named(prefetch_cluster, prefetch_cluster, uint, S_IRUGO | S_IWUSR);
> +
> +struct dm_verity {
> + struct dm_dev *data_dev;
> + struct dm_dev *hash_dev;
> + struct dm_target *ti;
> + struct dm_bufio_client *bufio;
> + char *alg_name;
> + struct crypto_shash *tfm;
> + u8 *root_digest; /* digest of the root block */
> + u8 *salt; /* salt, its size is salt_size */
> + unsigned salt_size;
> + sector_t data_start; /* data offset in 512-byte sectors */
> + sector_t hash_start; /* hash start in blocks */
> + sector_t data_blocks; /* the number of data blocks */
> + sector_t hash_blocks; /* the number of hash blocks */
> + unsigned char data_dev_block_bits; /* log2(data blocksize) */
> + unsigned char hash_dev_block_bits; /* log2(hash blocksize) */
> + unsigned char hash_per_block_bits; /* log2(hashes in hash block) */
> + unsigned char levels; /* the number of tree levels */
> + unsigned digest_size; /* digest size for the current hash algorithm */
> + unsigned shash_descsize;/* the size of temporary space for crypto */
> +
> + mempool_t *io_mempool; /* mempool of struct dm_verity_io */
> + mempool_t *vec_mempool; /* mempool of bio vector */
> +
> + struct workqueue_struct *verify_wq;
> +
> + /* starting blocks for each tree level. 0 is the lowest level. */
> + sector_t hash_level_block[DM_VERITY_MAX_LEVELS];
> +};
> +
> +struct dm_verity_io {
> + struct dm_verity *v;
> + struct bio *bio;
> +
> + /* original values of bio->bi_end_io and bio->bi_private */
> + bio_end_io_t *orig_bi_end_io;
> + void *orig_bi_private;
> +
> + sector_t block;
> + unsigned n_blocks;
> +
> + /* saved bio vector */
> + struct bio_vec *io_vec;
> + unsigned io_vec_size;
> +
> + struct work_struct work;
> +
> + /* a space for short vectors; longer vectors are allocated separately */
> + struct bio_vec io_vec_inline[DM_VERITY_IO_VEC_INLINE];
> +
> + /* variable-size fields, accessible with functions
> + io_hash_desc, io_real_digest, io_want_digest */
> + /* u8 hash_desc[crypto_shash_descsize(v->tfm)]; */
> + /* u8 real_digest[v->digest_size]; */
> + /* u8 want_digest[v->digest_size]; */
> +};
> +
> +static struct shash_desc *io_hash_desc(struct dm_verity *v, struct dm_verity_io *io)
> +{
> + return (struct shash_desc *)(io + 1);
> +}
> +
> +static u8 *io_real_digest(struct dm_verity *v, struct dm_verity_io *io)
> +{
> + return (u8 *)(io + 1) + v->shash_descsize;
> +}
> +
> +static u8 *io_want_digest(struct dm_verity *v, struct dm_verity_io *io)
> +{
> + return (u8 *)(io + 1) + v->shash_descsize + v->digest_size;
> +}
> +
> +/*
> + * Auxiliary structure appended to each dm-bufio buffer. If the value
> + * hash_verified is nonzero, hash of the block has been verified.
> + *
> + * There is no lock around this value, a race condition can at worst cause
> + * that multiple processes verify the hash of the same buffer simultaneously.
> + * This condition is harmless, so we don't need locking.
> + */
> +struct buffer_aux {
> + int hash_verified;
> +};
> +
> +/*
> + * Initialize struct buffer_aux for a freshly created buffer.
> + */
> +static void dm_bufio_alloc_callback(struct dm_buffer *buf)
> +{
> + struct buffer_aux *aux = dm_bufio_get_aux_data(buf);
> + aux->hash_verified = 0;
> +}
> +
> +/*
> + * Translate input sector number to the sector number on the target device.
> + */
> +static sector_t verity_map_sector(struct dm_verity *v, sector_t bi_sector)
> +{
> + return v->data_start + dm_target_offset(v->ti, bi_sector);
> +}
> +
> +/*
> + * Return hash position of a specified block at a specified tree level
> + * (0 is the lowest level).
> + * The lowest "hash_per_block_bits"-bits of the result denote hash position
> + * inside a hash block. The remaining bits denode location of the hash block.
> + */
> +static sector_t verity_position_at_level(struct dm_verity *v, sector_t block,
> + int level)
> +{
> + return block >> (level * v->hash_per_block_bits);
> +}
> +
> +static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level,
> + sector_t *hash_block, unsigned *offset)
> +{
> + sector_t position = verity_position_at_level(v, block, level);
> +
> + *hash_block = v->hash_level_block[level] + (position >> v->hash_per_block_bits);
> + if (offset)
> + *offset = v->digest_size * (position & ((1 << v->hash_per_block_bits) - 1));
> +}
> +
> +/*
> + * Verify hash of a metadata block pertaining to the specified data block
> + * ("block" argument) at a specified level ("level" argument).
> + *
> + * On successful return, io_want_digest(v, io) contains the hash value for
> + * a lower tree level or for the data block (if we're at the lowest leve).
> + *
> + * If "skip_unverified" is true, unverified buffer is skipped an 1 is returned.
> + * If "skip_unverified" is false, unverified buffer is hashed and verified
> + * against current value of io_want_digest(v, io).
> + */
> +static int verity_verify_level(struct dm_verity_io *io, sector_t block,
> + int level, bool skip_unverified)
> +{
> + struct dm_verity *v = io->v;
> + struct dm_buffer *buf;
> + struct buffer_aux *aux;
> + u8 *data;
> + int r;
> + sector_t hash_block;
> + unsigned offset;
> +
> + verity_hash_at_level(v, block, level, &hash_block, &offset);
> +
> + data = dm_bufio_read(v->bufio, hash_block, &buf);
> + if (unlikely(IS_ERR(data)))
> + return PTR_ERR(data);
> +
> + aux = dm_bufio_get_aux_data(buf);
> +
> + if (!aux->hash_verified) {
> + struct shash_desc *desc;
> + u8 *result;
> +
> + if (skip_unverified) {
> + r = 1;
> + goto release_ret_r;
> + }
> +
> + desc = io_hash_desc(v, io);
> + desc->tfm = v->tfm;
> + desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
> + r = crypto_shash_init(desc);
> + if (r < 0) {
> + DMERR("crypto_shash_init failed: %d", r);
> + goto release_ret_r;
> + }
> +
> + r = crypto_shash_update(desc, data, 1 << v->hash_dev_block_bits);
> + if (r < 0) {
> + DMERR("crypto_shash_update failed: %d", r);
> + goto release_ret_r;
> + }
> +
> + r = crypto_shash_update(desc, v->salt, v->salt_size);
> + if (r < 0) {
> + DMERR("crypto_shash_update failed: %d", r);
> + goto release_ret_r;
> + }
> +
> + result = io_real_digest(v, io);
> + r = crypto_shash_final(desc, result);
> + if (r < 0) {
> + DMERR("crypto_shash_final failed: %d", r);
> + goto release_ret_r;
> + }
> + if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) {
> + DMERR_LIMIT("metadata block %llu is corrupted",
> + (unsigned long long)hash_block);
> + r = -EIO;
> + goto release_ret_r;
> + } else
> + aux->hash_verified = 1;
> + }
> +
> + data += offset;
> +
> + memcpy(io_want_digest(v, io), data, v->digest_size);
> +
> + dm_bufio_release(buf);
> + return 0;
> +
> +release_ret_r:
> + dm_bufio_release(buf);
> + return r;
> +}
> +
> +/*
> + * Verify one "dm_verity_io" structure.
> + */
> +static int verity_verify_io(struct dm_verity_io *io)
> +{
> + struct dm_verity *v = io->v;
> + unsigned b;
> + int i;
> + unsigned vector = 0, offset = 0;
> + for (b = 0; b < io->n_blocks; b++) {
> + struct shash_desc *desc;
> + u8 *result;
> + int r;
> + unsigned todo;
> +
> + if (likely(v->levels)) {
> + /*
> + * First, we try to get the requested hash for
> + * the current block. If the hash block itself is
> + * verified, zero is returned. If it isn't, this
> + * function returns 0 and we fall back to whole
> + * chain verification.
> + */
> + int r = verity_verify_level(io, io->block + b, 0, true);
> + if (likely(!r))
> + goto test_block_hash;
> + if (r < 0)
> + return r;
> + }
> +
> + memcpy(io_want_digest(v, io), v->root_digest, v->digest_size);
> +
> + for (i = v->levels - 1; i >= 0; i--) {
> + int r = verity_verify_level(io, io->block + b, i, false);
> + if (unlikely(r))
> + return r;
> + }
> +
> +test_block_hash:
> + desc = io_hash_desc(v, io);
> + desc->tfm = v->tfm;
> + desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
> + r = crypto_shash_init(desc);
> + if (r < 0) {
> + DMERR("crypto_shash_init failed: %d", r);
> + return r;
> + }
> +
> + todo = 1 << v->data_dev_block_bits;
> + do {
> + struct bio_vec *bv;
> + u8 *page;
> + unsigned len;
> +
> + BUG_ON(vector >= io->io_vec_size);
> + bv = &io->io_vec[vector];
> + page = kmap_atomic(bv->bv_page, KM_USER0);
> + len = bv->bv_len - offset;
> + if (likely(len >= todo))
> + len = todo;
> + r = crypto_shash_update(desc,
> + page + bv->bv_offset + offset, len);
> + kunmap_atomic(page, KM_USER0);
> + if (r < 0) {
> + DMERR("crypto_shash_update failed: %d", r);
> + return r;
> + }
> + offset += len;
> + if (likely(offset == bv->bv_len)) {
> + offset = 0;
> + vector++;
> + }
> + todo -= len;
> + } while (todo);
> +
> + r = crypto_shash_update(desc, v->salt, v->salt_size);
> + if (r < 0) {
> + DMERR("crypto_shash_update failed: %d", r);
> + return r;
> + }
> +
> + result = io_real_digest(v, io);
> + r = crypto_shash_final(desc, result);
> + if (r < 0) {
> + DMERR("crypto_shash_final failed: %d", r);
> + return r;
> + }
> + if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) {
> + DMERR_LIMIT("data block %llu is corrupted",
> + (unsigned long long)(io->block + b));
> + return -EIO;
> + }
> + }
> + BUG_ON(vector != io->io_vec_size);
> + BUG_ON(offset);
> + return 0;
> +}
> +
> +/*
> + * End one "io" structure with a given error.
> + */
> +static void verity_finish_io(struct dm_verity_io *io, int error)
> +{
> + struct bio *bio = io->bio;
> + struct dm_verity *v = io->v;
> +
> + bio->bi_end_io = io->orig_bi_end_io;
> + bio->bi_private = io->orig_bi_private;
> +
> + if (io->io_vec != io->io_vec_inline)
> + mempool_free(io->io_vec, v->vec_mempool);
> + mempool_free(io, v->io_mempool);
> +
> + bio_endio(bio, error);
> +}
> +
> +static void verity_work(struct work_struct *w)
> +{
> + struct dm_verity_io *io = container_of(w, struct dm_verity_io, work);
> +
> + verity_finish_io(io, verity_verify_io(io));
> +}
> +
> +static void verity_end_io(struct bio *bio, int error)
> +{
> + struct dm_verity_io *io = bio->bi_private;
> + if (error) {
> + verity_finish_io(io, error);
> + return;
> + }
> +
> + INIT_WORK(&io->work, verity_work);
> + queue_work(io->v->verify_wq, &io->work);
> +}
> +
> +/*
> + * Prefetch buffers for the specified io.
> + * The root buffer is not prefetched, it is assumed that it will be cached
> + * all the time.
> + */
> +static void verity_prefetch_io(struct dm_verity *v, struct dm_verity_io *io)
> +{
> + int i;
> + for (i = v->levels - 2; i >= 0; i--) {
> + sector_t hash_block_start;
> + sector_t hash_block_end;
> + verity_hash_at_level(v, io->block, i, &hash_block_start, NULL);
> + verity_hash_at_level(v, io->block + io->n_blocks - 1, i, &hash_block_end, NULL);
> + if (!i) {
> + unsigned cluster = prefetch_cluster;
> + /* barrier to stop GCC from re-reading prefetch_cluster again */
> + barrier();
> + cluster >>= v->data_dev_block_bits;
> + if (unlikely(!cluster))
> + goto no_prefetch_cluster;
> + if (unlikely(cluster & (cluster - 1)))
> + cluster = 1 << (fls(cluster) - 1);
> +
> + hash_block_start &= ~(sector_t)(cluster - 1);
> + hash_block_end |= cluster - 1;
> + if (unlikely(hash_block_end >= v->hash_blocks))
> + hash_block_end = v->hash_blocks - 1;
> + }
> +no_prefetch_cluster:
> + dm_bufio_prefetch(v->bufio, hash_block_start,
> + hash_block_end - hash_block_start + 1);
> + }
> +}
> +
> +/*
> + * Bio map function. It allocates dm_verity_io structure and bio vector and
> + * fills them. Then it issues prefetches and the I/O.
> + */
> +static int verity_map(struct dm_target *ti, struct bio *bio,
> + union map_info *map_context)
> +{
> + struct dm_verity *v = ti->private;
> + struct dm_verity_io *io;
> +
> + if (((unsigned)bio->bi_sector | bio_sectors(bio)) &
> + ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) {
> + DMERR_LIMIT("unaligned io");
> + return -EIO;
> + }
> +
> + if ((bio->bi_sector + bio_sectors(bio)) >>
> + (v->data_dev_block_bits - SECTOR_SHIFT) > v->data_blocks) {
> + DMERR_LIMIT("io out of range");
> + return -EIO;
> + }
> +
> + if (bio_data_dir(bio) == WRITE)
> + return -EIO;
> +
> + io = mempool_alloc(v->io_mempool, GFP_NOIO);
> + io->v = v;
> + io->bio = bio;
> + io->orig_bi_end_io = bio->bi_end_io;
> + io->orig_bi_private = bio->bi_private;
> + io->block = bio->bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT);
> + io->n_blocks = bio->bi_size >> v->data_dev_block_bits;
> +
> + bio->bi_end_io = verity_end_io;
> + bio->bi_private = io;
> + bio->bi_bdev = v->data_dev->bdev;
> + bio->bi_sector = verity_map_sector(v, bio->bi_sector);
> +
> + io->io_vec_size = bio->bi_vcnt - bio->bi_idx;
> + if (io->io_vec_size < DM_VERITY_IO_VEC_INLINE)
> + io->io_vec = io->io_vec_inline;
> + else
> + io->io_vec = mempool_alloc(v->vec_mempool, GFP_NOIO);
> + memcpy(io->io_vec, bio_iovec(bio),
> + io->io_vec_size * sizeof(struct bio_vec));
> +
> + verity_prefetch_io(v, io);
> +
> + generic_make_request(bio);
> +
> + return DM_MAPIO_SUBMITTED;
> +}
> +
> +static int verity_status(struct dm_target *ti, status_type_t type,
> + char *result, unsigned maxlen)
> +{
> + struct dm_verity *v = ti->private;
> + unsigned sz = 0;
> + unsigned x;
> +
> + switch (type) {
> + case STATUSTYPE_INFO:
> + result[0] = 0;
> + break;
> + case STATUSTYPE_TABLE:
> + DMEMIT("%u %s %s %llu %u %s ",
> + 0,
> + v->data_dev->name,
> + v->hash_dev->name,
> + (unsigned long long)v->hash_start << (v->hash_dev_block_bits - SECTOR_SHIFT),
> + 1 << v->data_dev_block_bits,
> + v->alg_name
> + );
> + for (x = 0; x < v->digest_size; x++)
> + DMEMIT("%02x", v->root_digest[x]);
> + DMEMIT(" ");
> + if (!v->salt_size)
> + DMEMIT("-");
> + else
> + for (x = 0; x < v->salt_size; x++)
> + DMEMIT("%02x", v->salt[x]);
> + if (v->data_dev_block_bits != v->hash_dev_block_bits)
> + DMEMIT(" %u", 1 << v->hash_dev_block_bits);
> + break;
> + }
> + return 0;
> +}
> +
> +static int verity_ioctl(struct dm_target *ti, unsigned cmd,
> + unsigned long arg)
> +{
> + struct dm_verity *v = ti->private;
> + int r = 0;
> +
> + if (v->data_start ||
> + ti->len != i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT)
> + r = scsi_verify_blk_ioctl(NULL, cmd);
> +
> + return r ? : __blkdev_driver_ioctl(v->data_dev->bdev, v->data_dev->mode,
> + cmd, arg);
> +}
> +
> +static int verity_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
> + struct bio_vec *biovec, int max_size)
> +{
> + struct dm_verity *v = ti->private;
> + struct request_queue *q = bdev_get_queue(v->data_dev->bdev);
> +
> + if (!q->merge_bvec_fn)
> + return max_size;
> +
> + bvm->bi_bdev = v->data_dev->bdev;
> + bvm->bi_sector = verity_map_sector(v, bvm->bi_sector);
> +
> + return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
> +}
> +
> +static int verity_iterate_devices(struct dm_target *ti,
> + iterate_devices_callout_fn fn, void *data)
> +{
> + struct dm_verity *v = ti->private;
> + return fn(ti, v->data_dev, v->data_start, ti->len, data);
> +}
> +
> +static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits)
> +{
> + struct dm_verity *v = ti->private;
> +
> + if (limits->logical_block_size < 1 << v->data_dev_block_bits)
> + limits->logical_block_size = 1 << v->data_dev_block_bits;
> + if (limits->physical_block_size < 1 << v->data_dev_block_bits)
> + limits->physical_block_size = 1 << v->data_dev_block_bits;
> + blk_limits_io_min(limits, limits->logical_block_size);
> +}
> +
> +static void verity_dtr(struct dm_target *ti);
> +
> +static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv)
> +{
> + struct dm_verity *v;
> + unsigned num;
> + unsigned long long hs;
> + int r;
> + int i;
> + sector_t hash_position;
> + char dummy;
> +
> + v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL);
> + if (!v) {
> + ti->error = "Cannot allocate verity structure";
> + return -ENOMEM;
> + }
> + ti->private = v;
> + v->ti = ti;
> +
> + if ((dm_table_get_mode(ti->table) & ~FMODE_READ) != 0) {
> + ti->error = "Device must be readonly";
> + r = -EINVAL;
> + goto bad;
> + }
> +
> + if (argc < 7) {
> + ti->error = "Not enough arguments";
> + r = -EINVAL;
> + goto bad;
> + }
> +
> + if (sscanf(argv[0], "%d%c", &num, &dummy) != 1 ||
> + num != 0) {
> + ti->error = "Invalid version";
> + r = -EINVAL;
> + goto bad;
> + }
> +
> + r = dm_get_device(ti, argv[1], FMODE_READ, &v->data_dev);
> + if (r) {
> + ti->error = "Data device lookup failed";
> + goto bad;
> + }
> +
> + r = dm_get_device(ti, argv[2], FMODE_READ, &v->hash_dev);
> + if (r) {
> + ti->error = "Data device lookup failed";
> + goto bad;
> + }
> +
> + if (sscanf(argv[3], "%llu%c", &hs, &dummy) != 1 ||
> + hs != (sector_t)hs) {
> + ti->error = "Invalid hash start";
> + r = -EINVAL;
> + goto bad;
> + }
> +
> + if (sscanf(argv[4], "%u%c", &num, &dummy) != 1 ||
> + !num || (num & (num - 1)) ||
> + num < bdev_logical_block_size(v->data_dev->bdev) ||
> + num > PAGE_SIZE) {
> + ti->error = "Invalid data device block size";
> + r = -EINVAL;
> + goto bad;
> + }
> + v->data_dev_block_bits = ffs(num) - 1;
> + v->hash_dev_block_bits = ffs(num) - 1;
> +
> + v->alg_name = kstrdup(argv[5], GFP_KERNEL);
> + if (!v->alg_name) {
> + ti->error = "Cannot allocate algorithm name";
> + r = -ENOMEM;
> + goto bad;
> + }
> +
> + v->tfm = crypto_alloc_shash(v->alg_name, 0, 0);
> + if (IS_ERR(v->tfm)) {
> + ti->error = "Cannot initialize hash function";
> + r = PTR_ERR(v->tfm);
> + v->tfm = NULL;
> + goto bad;
> + }
> + v->digest_size = crypto_shash_digestsize(v->tfm);
> + if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) {
> + ti->error = "Digest size too big";
> + r = -EINVAL;
> + goto bad;
> + }
> + v->shash_descsize =
> + sizeof(struct shash_desc) + crypto_shash_descsize(v->tfm);
> +
> + v->root_digest = kmalloc(v->digest_size, GFP_KERNEL);
> + if (!v->root_digest) {
> + ti->error = "Cannot allocate root digest";
> + r = -ENOMEM;
> + goto bad;
> + }
> + if (strlen(argv[6]) != v->digest_size * 2 ||
> + hex2bin(v->root_digest, argv[6], v->digest_size)) {
> + ti->error = "Invalid root digest";
> + r = -EINVAL;
> + goto bad;
> + }
> +
> + if (argc > 7 && strcmp(argv[7], "-")) {
> + v->salt_size = strlen(argv[7]) / 2;
> + v->salt = kmalloc(v->salt_size, GFP_KERNEL);
> + if (!v->salt) {
> + ti->error = "Cannot allocate salt";
> + r = -ENOMEM;
> + goto bad;
> + }
> + if (strlen(argv[7]) != v->salt_size * 2 ||
> + hex2bin(v->salt, argv[7], v->salt_size)) {
> + ti->error = "Invalid salt";
> + r = -EINVAL;
> + goto bad;
> + }
> + }
> +
> + if (argc > 8) {
> + if (sscanf(argv[8], "%u%c", &num, &dummy) != 1 ||
> + !num || (num & (num - 1)) ||
> + num < bdev_logical_block_size(v->hash_dev->bdev) ||
> + num > INT_MAX) {
> + ti->error = "Invalid hash device block size";
> + r = -EINVAL;
> + goto bad;
> + }
> + v->hash_dev_block_bits = ffs(num) - 1;
> + }
> +
> + if (hs & ((1 << (v->hash_dev_block_bits - SECTOR_SHIFT)) - 1)) {
> + ti->error = "Hash start not aligned on block boundary";
> + r = -EINVAL;
> + goto bad;
> + }
> + v->hash_start = hs >> (v->hash_dev_block_bits - SECTOR_SHIFT);
> +
> + if (ti->len > i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT) {
> + ti->error = "Data device si too small";
> + r = -EINVAL;
> + goto bad;
> + }
> +
> + if (ti->len & ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) {
> + ti->error = "Data device length is not aligned to block size";
> + r = -EINVAL;
> + goto bad;
> + }
> +
> + v->data_blocks = ti->len >> (v->data_dev_block_bits - SECTOR_SHIFT);
> +
> + v->hash_per_block_bits =
> + fls((1 << v->hash_dev_block_bits) / v->digest_size) - 1;
> +
> + v->levels = 0;
> + if (v->data_blocks)
> + while (v->hash_per_block_bits * v->levels < 64 &&
> + (unsigned long long)(v->data_blocks - 1) >>
> + (v->hash_per_block_bits * v->levels))
> + v->levels++;
> +
> + if (v->levels > DM_VERITY_MAX_LEVELS) {
> + ti->error = "Too many tree levels";
> + r = -E2BIG;
> + goto bad;
> + }
> +
> + hash_position = v->hash_start;
> + for (i = v->levels - 1; i >= 0; i--) {
> + sector_t s;
> + v->hash_level_block[i] = hash_position;
> + s = verity_position_at_level(v, v->data_blocks, i);
> + s = (s >> v->hash_per_block_bits) +
> + !!(s & ((1 << v->hash_per_block_bits) - 1));
> + if (hash_position + s < hash_position) {
> + ti->error = "Hash device offset overflow";
> + r = -E2BIG;
> + goto bad;
> + }
> + hash_position += s;
> + }
> + v->hash_blocks = hash_position;
> +
> + v->bufio = dm_bufio_client_create(v->hash_dev->bdev,
> + 1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux),
> + dm_bufio_alloc_callback, NULL);
> + if (IS_ERR(v->bufio)) {
> + ti->error = "Cannot initialize dm-bufio";
> + r = PTR_ERR(v->bufio);
> + v->bufio = NULL;
> + goto bad;
> + }
> +
> + if (dm_bufio_get_device_size(v->bufio) < v->hash_blocks) {
> + ti->error = "Hash device is too small";
> + r = -E2BIG;
> + goto bad;
> + }
> +
> + v->io_mempool = mempool_create_kmalloc_pool(DM_VERITY_MEMPOOL_SIZE ,
> + sizeof(struct dm_verity_io) + v->shash_descsize + v->digest_size * 2);
> + if (!v->io_mempool) {
> + ti->error = "Cannot allocate io mempool";
> + r = -ENOMEM;
> + goto bad;
> + }
> +
> + v->vec_mempool = mempool_create_kmalloc_pool(DM_VERITY_MEMPOOL_SIZE ,
> + BIO_MAX_PAGES * sizeof(struct bio_vec));
> + if (!v->vec_mempool) {
> + ti->error = "Cannot allocate vector mempool";
> + r = -ENOMEM;
> + goto bad;
> + }
> +
> + /*v->verify_wq = alloc_workqueue("verityd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 1);*/
> + /* WQ_UNBOUND greatly improves performance when running on ramdisk */
> + v->verify_wq = alloc_workqueue("verityd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, num_online_cpus());
> + if (!v->verify_wq) {
> + ti->error = "Cannot allocate workqueue";
> + r = -ENOMEM;
> + goto bad;
> + }
> +
> + return 0;
> +
> +bad:
> + verity_dtr(ti);
> + return r;
> +}
> +
> +static void verity_dtr(struct dm_target *ti)
> +{
> + struct dm_verity *v = ti->private;
> +
> + if (v->verify_wq)
> + destroy_workqueue(v->verify_wq);
> + if (v->vec_mempool)
> + mempool_destroy(v->vec_mempool);
> + if (v->io_mempool)
> + mempool_destroy(v->io_mempool);
> + if (v->bufio)
> + dm_bufio_client_destroy(v->bufio);
> + kfree(v->salt);
> + kfree(v->root_digest);
> + if (v->tfm)
> + crypto_free_shash(v->tfm);
> + kfree(v->alg_name);
> + if (v->hash_dev)
> + dm_put_device(ti, v->hash_dev);
> + if (v->data_dev)
> + dm_put_device(ti, v->data_dev);
> + kfree(v);
> +}
> +
> +static struct target_type verity_target = {
> + .name = "verity",
> + .version = {1, 0, 0},
> + .module = THIS_MODULE,
> + .ctr = verity_ctr,
> + .dtr = verity_dtr,
> + .map = verity_map,
> + .status = verity_status,
> + .ioctl = verity_ioctl,
> + .merge = verity_merge,
> + .iterate_devices = verity_iterate_devices,
> + .io_hints = verity_io_hints,
> +};
> +
> +static int __init dm_verity_init(void)
> +{
> + int r;
> + r = dm_register_target(&verity_target);
> + if (r < 0)
> + DMERR("register failed %d", r);
> + return r;
> +}
> +
> +static void __exit dm_verity_exit(void)
> +{
> + dm_unregister_target(&verity_target);
> +}
> +
> +module_init(dm_verity_init);
> +module_exit(dm_verity_exit);
> +
> +MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>");
> +MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking");
> +MODULE_LICENSE("GPL");
> +
> Index: linux-3.3-rc6-fast/drivers/md/dm-bufio.c
> ================================================== =================
> --- linux-3.3-rc6-fast.orig/drivers/md/dm-bufio.c 2012-03-12 22:43:23.000000000 +0100
> +++ linux-3.3-rc6-fast/drivers/md/dm-bufio.c 2012-03-13 15:41:02.000000000 +0100
> @@ -579,7 +579,7 @@ static void write_endio(struct bio *bio,
> struct dm_buffer *b = container_of(bio, struct dm_buffer, bio);
>
> b->write_error = error;
> - if (error) {
> + if (unlikely(error)) {
> struct dm_bufio_client *c = b->c;
> (void)cmpxchg(&c->async_write_error, 0, error);
> }
> @@ -698,13 +698,20 @@ static void __wait_for_free_buffer(struc
> dm_bufio_lock(c);
> }
>
> +enum new_flag {
> + NF_FRESH = 0,
> + NF_READ = 1,
> + NF_GET = 2,
> + NF_PREFETCH = 3
> +};
> +
> /*
> * Allocate a new buffer. If the allocation is not possible, wait until
> * some other thread frees a buffer.
> *
> * May drop the lock and regain it.
> */
> -static struct dm_buffer *__alloc_buffer_wait_no_callback(struct dm_bufio_client *c)
> +static struct dm_buffer *__alloc_buffer_wait_no_callback(struct dm_bufio_client *c, enum new_flag nf)
> {
> struct dm_buffer *b;
>
> @@ -727,6 +734,9 @@ static struct dm_buffer *__alloc_buffer_
> return b;
> }
>
> + if (nf == NF_PREFETCH)
> + return NULL;
> +
> if (!list_empty(&c->reserved_buffers)) {
> b = list_entry(c->reserved_buffers.next,
> struct dm_buffer, lru_list);
> @@ -744,9 +754,12 @@ static struct dm_buffer *__alloc_buffer_
> }
> }
>
> -static struct dm_buffer *__alloc_buffer_wait(struct dm_bufio_client *c)
> +static struct dm_buffer *__alloc_buffer_wait(struct dm_bufio_client *c, enum new_flag nf)
> {
> - struct dm_buffer *b = __alloc_buffer_wait_no_callback(c);
> + struct dm_buffer *b = __alloc_buffer_wait_no_callback(c, nf);
> +
> + if (!b)
> + return NULL;
>
> if (c->alloc_callback)
> c->alloc_callback(b);
> @@ -866,15 +879,8 @@ static struct dm_buffer *__find(struct d
> * Getting a buffer
> *--------------------------------------------------------------*/
>
> -enum new_flag {
> - NF_FRESH = 0,
> - NF_READ = 1,
> - NF_GET = 2
> -};
> -
> static struct dm_buffer *__bufio_new(struct dm_bufio_client *c, sector_t block,
> - enum new_flag nf, struct dm_buffer **bp,
> - int *need_submit)
> + enum new_flag nf, int *need_submit)
> {
> struct dm_buffer *b, *new_b = NULL;
>
> @@ -882,6 +888,19 @@ static struct dm_buffer *__bufio_new(str
>
> b = __find(c, block);
> if (b) {
> +found_buffer:
> + if (nf == NF_PREFETCH)
> + return NULL;
> + /*
> + * Note: it is essential that we don't wait for the buffer to be
> + * read if dm_bufio_get function is used. Both dm_bufio_get and
> + * dm_bufio_prefetch can be used in the driver request routine.
> + * If the user called both dm_bufio_prefetch and dm_bufio_get on
> + * the same buffer, it would deadlock if we waited.
> + */
> + if (nf == NF_GET && unlikely(test_bit(B_READING, &b->state)))
> + return NULL;
> +
> b->hold_count++;
> __relink_lru(b, test_bit(B_DIRTY, &b->state) ||
> test_bit(B_WRITING, &b->state));
> @@ -891,7 +910,9 @@ static struct dm_buffer *__bufio_new(str
> if (nf == NF_GET)
> return NULL;
>
> - new_b = __alloc_buffer_wait(c);
> + new_b = __alloc_buffer_wait(c, nf);
> + if (!new_b)
> + return NULL;
>
> /*
> * We've had a period where the mutex was unlocked, so need to
> @@ -900,10 +921,7 @@ static struct dm_buffer *__bufio_new(str
> b = __find(c, block);
> if (b) {
> __free_buffer_wake(new_b);
> - b->hold_count++;
> - __relink_lru(b, test_bit(B_DIRTY, &b->state) ||
> - test_bit(B_WRITING, &b->state));
> - return b;
> + goto found_buffer;
> }
>
> __check_watermark(c);
> @@ -957,7 +975,7 @@ static void *new_read(struct dm_bufio_cl
> struct dm_buffer *b;
>
> dm_bufio_lock(c);
> - b = __bufio_new(c, block, nf, bp, &need_submit);
> + b = __bufio_new(c, block, nf, &need_submit);
> dm_bufio_unlock(c);
>
> if (!b || IS_ERR(b))
> @@ -1006,13 +1024,46 @@ void *dm_bufio_new(struct dm_bufio_clien
> }
> EXPORT_SYMBOL_GPL(dm_bufio_new);
>
> +void dm_bufio_prefetch(struct dm_bufio_client *c,
> + sector_t block, unsigned n_blocks)
> +{
> + struct blk_plug plug;
> +
> + blk_start_plug(&plug);
> + dm_bufio_lock(c);
> +
> + for (; n_blocks--; block++) {
> + int need_submit;
> + struct dm_buffer *b;
> + b = __bufio_new(c, block, NF_PREFETCH, &need_submit);
> + if (unlikely(b != NULL)) {
> + dm_bufio_unlock(c);
> +
> + if (need_submit)
> + submit_io(b, READ, b->block, read_endio);
> + dm_bufio_release(b);
> +
> + dm_bufio_cond_resched();
> +
> + if (!n_blocks)
> + goto flush_plug;
> + dm_bufio_lock(c);
> + }
> +
> + }
> +
> + dm_bufio_unlock(c);
> +flush_plug:
> + blk_finish_plug(&plug);
> +}
> +EXPORT_SYMBOL_GPL(dm_bufio_prefetch);
> +
> void dm_bufio_release(struct dm_buffer *b)
> {
> struct dm_bufio_client *c = b->c;
>
> dm_bufio_lock(c);
>
> - BUG_ON(test_bit(B_READING, &b->state));
> BUG_ON(!b->hold_count);
>
> b->hold_count--;
> @@ -1025,6 +1076,7 @@ void dm_bufio_release(struct dm_buffer *
> * invalid buffer.
> */
> if ((b->read_error || b->write_error) &&
> + !test_bit(B_READING, &b->state) &&
> !test_bit(B_WRITING, &b->state) &&
> !test_bit(B_DIRTY, &b->state)) {
> __unlink_buffer(b);
> @@ -1042,6 +1094,8 @@ void dm_bufio_mark_buffer_dirty(struct d
>
> dm_bufio_lock(c);
>
> + BUG_ON(test_bit(B_READING, &b->state));
> +
> if (!test_and_set_bit(B_DIRTY, &b->state))
> __relink_lru(b, LIST_DIRTY);
>
> Index: linux-3.3-rc6-fast/drivers/md/dm-bufio.h
> ================================================== =================
> --- linux-3.3-rc6-fast.orig/drivers/md/dm-bufio.h 2012-03-12 22:43:23.000000000 +0100
> +++ linux-3.3-rc6-fast/drivers/md/dm-bufio.h 2012-03-12 22:43:25.000000000 +0100
> @@ -63,6 +63,14 @@ void *dm_bufio_new(struct dm_bufio_clien
> struct dm_buffer **bp);
>
> /*
> + * Prefetch the specified blocks to the cache.
> + * The function starts to read the blocks and returns without waiting for
> + * I/O to finish.
> + */
> +void dm_bufio_prefetch(struct dm_bufio_client *c,
> + sector_t block, unsigned n_blocks);
> +
> +/*
> * Release a reference obtained with dm_bufio_{read,get,new}. The data
> * pointer and dm_buffer pointer is no longer valid after this call.
> */

--
dm-devel mailing list
dm-devel@redhat.com
https://www.redhat.com/mailman/listinfo/dm-devel
 
Old 03-20-2012, 11:54 PM
Mikulas Patocka
 
Default dm: remake of the verity target

On Tue, 20 Mar 2012, Mandeep Singh Baines wrote:

> Hi Mikulas,
>
> Can you please resend this patch with a proper commit message.
> We'd really like to see this merged. Alasdair, other than that,
> what work is remaining for verity to be merged?
>
> Regards,
> Mandeep

Hi

I'm sending this new version of dm-verity to be merged. I've made some
last changes in the format, hopefully no more changes will be needed. This
changes make it incompatible with the original Google code (but the
original code can be trivially changed to support these modifications).

Changes:

* Salt is hashed before the block (it used to be hased after). The reason
is that if random salt is hashed before the block, it makes the process
resilient to hash function collisions - so you can safely use md5, even if
there's a collision attach for it.

* Argument line was simplified, there are no optional arguments.

* There is new argument specifying the size of the data device.

Mikulas

---

Remake of the google dm-verity patch.

Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>

---
drivers/md/Kconfig | 17
drivers/md/Makefile | 1
drivers/md/dm-verity.c | 849 +++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 867 insertions(+)

Index: linux-3.3-fast/drivers/md/Kconfig
================================================== =================
--- linux-3.3-fast.orig/drivers/md/Kconfig 2012-03-19 13:46:54.000000000 +0100
+++ linux-3.3-fast/drivers/md/Kconfig 2012-03-19 13:46:55.000000000 +0100
@@ -404,4 +404,21 @@ config DM_VERITY2

If unsure, say N.

+config DM_VERITY
+ tristate "Verity target support"
+ depends on BLK_DEV_DM
+ select CRYPTO
+ select CRYPTO_HASH
+ select DM_BUFIO
+ ---help---
+ This device-mapper target allows you to create a device that
+ transparently integrity checks the data on it. You'll need to
+ activate the digests you're going to use in the cryptoapi
+ configuration.
+
+ To compile this code as a module, choose M here: the module will
+ be called dm-verity.
+
+ If unsure, say N.
+
endif # MD
Index: linux-3.3-fast/drivers/md/Makefile
================================================== =================
--- linux-3.3-fast.orig/drivers/md/Makefile 2012-03-19 13:46:54.000000000 +0100
+++ linux-3.3-fast/drivers/md/Makefile 2012-03-19 13:46:55.000000000 +0100
@@ -29,6 +29,7 @@ obj-$(CONFIG_MD_FAULTY) += faulty.o
obj-$(CONFIG_BLK_DEV_MD) += md-mod.o
obj-$(CONFIG_BLK_DEV_DM) += dm-mod.o
obj-$(CONFIG_DM_BUFIO) += dm-bufio.o
+obj-$(CONFIG_DM_VERITY) += dm-verity.o
obj-$(CONFIG_DM_CRYPT) += dm-crypt.o
obj-$(CONFIG_DM_DELAY) += dm-delay.o
obj-$(CONFIG_DM_FLAKEY) += dm-flakey.o
Index: linux-3.3-fast/drivers/md/dm-verity.c
================================================== =================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux-3.3-fast/drivers/md/dm-verity.c 2012-03-20 22:03:53.000000000 +0100
@@ -0,0 +1,849 @@
+/*
+ * Copyright (C) 2012 Red Hat, Inc.
+ *
+ * Author: Mikulas Patocka <mpatocka@redhat.com>
+ *
+ * Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors
+ *
+ * This file is released under the GPLv2.
+ *
+ * Device mapper target parameters:
+ * <version> 0
+ * <data device>
+ * <hash device>
+ * <data block size>
+ * <hash block size>
+ * <the number of data blocks>
+ * <hash start block>
+ * <algorithm>
+ * <digest>
+ * <salt> (hex bytes or "-" for no salt)
+ *
+ * In the file "/sys/module/dm_verity/parameters/prefetch_cluster" you can set
+ * default prefetch value. Data are read in "prefetch_cluster" chunks from the
+ * hash device. Prefetch cluster greatly improves performance when data and hash
+ * are on the same disk on different partitions on devices with poor random
+ * access behavior.
+ */
+
+#include <linux/module.h>
+#include <linux/device-mapper.h>
+#include <crypto/hash.h>
+#include "dm-bufio.h"
+
+#define DM_MSG_PREFIX "verity"
+
+#define DM_VERITY_IO_VEC_INLINE 16
+#define DM_VERITY_MEMPOOL_SIZE 4
+#define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144
+
+#define DM_VERITY_MAX_LEVELS 63
+
+static unsigned prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE;
+
+module_param_named(prefetch_cluster, prefetch_cluster, uint, S_IRUGO | S_IWUSR);
+
+struct dm_verity {
+ struct dm_dev *data_dev;
+ struct dm_dev *hash_dev;
+ struct dm_target *ti;
+ struct dm_bufio_client *bufio;
+ char *alg_name;
+ struct crypto_shash *tfm;
+ u8 *root_digest; /* digest of the root block */
+ u8 *salt; /* salt, its size is salt_size */
+ unsigned salt_size;
+ sector_t data_start; /* data offset in 512-byte sectors */
+ sector_t hash_start; /* hash start in blocks */
+ sector_t data_blocks; /* the number of data blocks */
+ sector_t hash_blocks; /* the number of hash blocks */
+ unsigned char data_dev_block_bits; /* log2(data blocksize) */
+ unsigned char hash_dev_block_bits; /* log2(hash blocksize) */
+ unsigned char hash_per_block_bits; /* log2(hashes in hash block) */
+ unsigned char levels; /* the number of tree levels */
+ unsigned digest_size; /* digest size for the current hash algorithm */
+ unsigned shash_descsize;/* the size of temporary space for crypto */
+
+ mempool_t *io_mempool; /* mempool of struct dm_verity_io */
+ mempool_t *vec_mempool; /* mempool of bio vector */
+
+ struct workqueue_struct *verify_wq;
+
+ /* starting blocks for each tree level. 0 is the lowest level. */
+ sector_t hash_level_block[DM_VERITY_MAX_LEVELS];
+};
+
+struct dm_verity_io {
+ struct dm_verity *v;
+ struct bio *bio;
+
+ /* original values of bio->bi_end_io and bio->bi_private */
+ bio_end_io_t *orig_bi_end_io;
+ void *orig_bi_private;
+
+ sector_t block;
+ unsigned n_blocks;
+
+ /* saved bio vector */
+ struct bio_vec *io_vec;
+ unsigned io_vec_size;
+
+ struct work_struct work;
+
+ /* a space for short vectors; longer vectors are allocated separately */
+ struct bio_vec io_vec_inline[DM_VERITY_IO_VEC_INLINE];
+
+ /* variable-size fields, accessible with functions
+ io_hash_desc, io_real_digest, io_want_digest */
+ /* u8 hash_desc[v->shash_descsize]; */
+ /* u8 real_digest[v->digest_size]; */
+ /* u8 want_digest[v->digest_size]; */
+};
+
+static struct shash_desc *io_hash_desc(struct dm_verity *v, struct dm_verity_io *io)
+{
+ return (struct shash_desc *)(io + 1);
+}
+
+static u8 *io_real_digest(struct dm_verity *v, struct dm_verity_io *io)
+{
+ return (u8 *)(io + 1) + v->shash_descsize;
+}
+
+static u8 *io_want_digest(struct dm_verity *v, struct dm_verity_io *io)
+{
+ return (u8 *)(io + 1) + v->shash_descsize + v->digest_size;
+}
+
+/*
+ * Auxiliary structure appended to each dm-bufio buffer. If the value
+ * hash_verified is nonzero, hash of the block has been verified.
+ *
+ * The variable hash_verified is set to 0 when allocating the buffer, then
+ * it can be changed to 1 and it is never reset to 0 again.
+ *
+ * There is no lock around this value, a race condition can at worst cause
+ * that multiple processes verify the hash of the same buffer simultaneously
+ * and write 1 to hash_verified simultaneously.
+ * This condition is harmless, so we don't need locking.
+ */
+struct buffer_aux {
+ int hash_verified;
+};
+
+/*
+ * Initialize struct buffer_aux for a freshly created buffer.
+ */
+static void dm_bufio_alloc_callback(struct dm_buffer *buf)
+{
+ struct buffer_aux *aux = dm_bufio_get_aux_data(buf);
+ aux->hash_verified = 0;
+}
+
+/*
+ * Translate input sector number to the sector number on the target device.
+ */
+static sector_t verity_map_sector(struct dm_verity *v, sector_t bi_sector)
+{
+ return v->data_start + dm_target_offset(v->ti, bi_sector);
+}
+
+/*
+ * Return hash position of a specified block at a specified tree level
+ * (0 is the lowest level).
+ * The lowest "hash_per_block_bits"-bits of the result denote hash position
+ * inside a hash block. The remaining bits denote location of the hash block.
+ */
+static sector_t verity_position_at_level(struct dm_verity *v, sector_t block,
+ int level)
+{
+ return block >> (level * v->hash_per_block_bits);
+}
+
+static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level,
+ sector_t *hash_block, unsigned *offset)
+{
+ sector_t position = verity_position_at_level(v, block, level);
+
+ *hash_block = v->hash_level_block[level] + (position >> v->hash_per_block_bits);
+ if (offset)
+ *offset = (position & ((1 << v->hash_per_block_bits) - 1)) << (v->hash_dev_block_bits - v->hash_per_block_bits);
+}
+
+/*
+ * Verify hash of a metadata block pertaining to the specified data block
+ * ("block" argument) at a specified level ("level" argument).
+ *
+ * On successful return, io_want_digest(v, io) contains the hash value for
+ * a lower tree level or for the data block (if we're at the lowest leve).
+ *
+ * If "skip_unverified" is true, unverified buffer is skipped an 1 is returned.
+ * If "skip_unverified" is false, unverified buffer is hashed and verified
+ * against current value of io_want_digest(v, io).
+ */
+static int verity_verify_level(struct dm_verity_io *io, sector_t block,
+ int level, bool skip_unverified)
+{
+ struct dm_verity *v = io->v;
+ struct dm_buffer *buf;
+ struct buffer_aux *aux;
+ u8 *data;
+ int r;
+ sector_t hash_block;
+ unsigned offset;
+
+ verity_hash_at_level(v, block, level, &hash_block, &offset);
+
+ data = dm_bufio_read(v->bufio, hash_block, &buf);
+ if (unlikely(IS_ERR(data)))
+ return PTR_ERR(data);
+
+ aux = dm_bufio_get_aux_data(buf);
+
+ if (!aux->hash_verified) {
+ struct shash_desc *desc;
+ u8 *result;
+
+ if (skip_unverified) {
+ r = 1;
+ goto release_ret_r;
+ }
+
+ desc = io_hash_desc(v, io);
+ desc->tfm = v->tfm;
+ desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
+ r = crypto_shash_init(desc);
+ if (r < 0) {
+ DMERR("crypto_shash_init failed: %d", r);
+ goto release_ret_r;
+ }
+
+ r = crypto_shash_update(desc, v->salt, v->salt_size);
+ if (r < 0) {
+ DMERR("crypto_shash_update failed: %d", r);
+ goto release_ret_r;
+ }
+
+ r = crypto_shash_update(desc, data, 1 << v->hash_dev_block_bits);
+ if (r < 0) {
+ DMERR("crypto_shash_update failed: %d", r);
+ goto release_ret_r;
+ }
+
+ result = io_real_digest(v, io);
+ r = crypto_shash_final(desc, result);
+ if (r < 0) {
+ DMERR("crypto_shash_final failed: %d", r);
+ goto release_ret_r;
+ }
+ if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) {
+ DMERR_LIMIT("metadata block %llu is corrupted",
+ (unsigned long long)hash_block);
+ r = -EIO;
+ goto release_ret_r;
+ } else
+ aux->hash_verified = 1;
+ }
+
+ data += offset;
+
+ memcpy(io_want_digest(v, io), data, v->digest_size);
+
+ dm_bufio_release(buf);
+ return 0;
+
+release_ret_r:
+ dm_bufio_release(buf);
+ return r;
+}
+
+/*
+ * Verify one "dm_verity_io" structure.
+ */
+static int verity_verify_io(struct dm_verity_io *io)
+{
+ struct dm_verity *v = io->v;
+ unsigned b;
+ int i;
+ unsigned vector = 0, offset = 0;
+ for (b = 0; b < io->n_blocks; b++) {
+ struct shash_desc *desc;
+ u8 *result;
+ int r;
+ unsigned todo;
+
+ if (likely(v->levels)) {
+ /*
+ * First, we try to get the requested hash for
+ * the current block. If the hash block itself is
+ * verified, zero is returned. If it isn't, this
+ * function returns 0 and we fall back to whole
+ * chain verification.
+ */
+ int r = verity_verify_level(io, io->block + b, 0, true);
+ if (likely(!r))
+ goto test_block_hash;
+ if (r < 0)
+ return r;
+ }
+
+ memcpy(io_want_digest(v, io), v->root_digest, v->digest_size);
+
+ for (i = v->levels - 1; i >= 0; i--) {
+ int r = verity_verify_level(io, io->block + b, i, false);
+ if (unlikely(r))
+ return r;
+ }
+
+test_block_hash:
+ desc = io_hash_desc(v, io);
+ desc->tfm = v->tfm;
+ desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
+ r = crypto_shash_init(desc);
+ if (r < 0) {
+ DMERR("crypto_shash_init failed: %d", r);
+ return r;
+ }
+
+ r = crypto_shash_update(desc, v->salt, v->salt_size);
+ if (r < 0) {
+ DMERR("crypto_shash_update failed: %d", r);
+ return r;
+ }
+
+ todo = 1 << v->data_dev_block_bits;
+ do {
+ struct bio_vec *bv;
+ u8 *page;
+ unsigned len;
+
+ BUG_ON(vector >= io->io_vec_size);
+ bv = &io->io_vec[vector];
+ page = kmap_atomic(bv->bv_page, KM_USER0);
+ len = bv->bv_len - offset;
+ if (likely(len >= todo))
+ len = todo;
+ r = crypto_shash_update(desc,
+ page + bv->bv_offset + offset, len);
+ kunmap_atomic(page, KM_USER0);
+ if (r < 0) {
+ DMERR("crypto_shash_update failed: %d", r);
+ return r;
+ }
+ offset += len;
+ if (likely(offset == bv->bv_len)) {
+ offset = 0;
+ vector++;
+ }
+ todo -= len;
+ } while (todo);
+
+ result = io_real_digest(v, io);
+ r = crypto_shash_final(desc, result);
+ if (r < 0) {
+ DMERR("crypto_shash_final failed: %d", r);
+ return r;
+ }
+ if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) {
+ DMERR_LIMIT("data block %llu is corrupted",
+ (unsigned long long)(io->block + b));
+ return -EIO;
+ }
+ }
+ BUG_ON(vector != io->io_vec_size);
+ BUG_ON(offset);
+ return 0;
+}
+
+/*
+ * End one "io" structure with a given error.
+ */
+static void verity_finish_io(struct dm_verity_io *io, int error)
+{
+ struct bio *bio = io->bio;
+ struct dm_verity *v = io->v;
+
+ bio->bi_end_io = io->orig_bi_end_io;
+ bio->bi_private = io->orig_bi_private;
+
+ if (io->io_vec != io->io_vec_inline)
+ mempool_free(io->io_vec, v->vec_mempool);
+ mempool_free(io, v->io_mempool);
+
+ bio_endio(bio, error);
+}
+
+static void verity_work(struct work_struct *w)
+{
+ struct dm_verity_io *io = container_of(w, struct dm_verity_io, work);
+
+ verity_finish_io(io, verity_verify_io(io));
+}
+
+static void verity_end_io(struct bio *bio, int error)
+{
+ struct dm_verity_io *io = bio->bi_private;
+ if (error) {
+ verity_finish_io(io, error);
+ return;
+ }
+
+ INIT_WORK(&io->work, verity_work);
+ queue_work(io->v->verify_wq, &io->work);
+}
+
+/*
+ * Prefetch buffers for the specified io.
+ * The root buffer is not prefetched, it is assumed that it will be cached
+ * all the time.
+ */
+static void verity_prefetch_io(struct dm_verity *v, struct dm_verity_io *io)
+{
+ int i;
+ for (i = v->levels - 2; i >= 0; i--) {
+ sector_t hash_block_start;
+ sector_t hash_block_end;
+ verity_hash_at_level(v, io->block, i, &hash_block_start, NULL);
+ verity_hash_at_level(v, io->block + io->n_blocks - 1, i, &hash_block_end, NULL);
+ if (!i) {
+ unsigned cluster = *(volatile unsigned *)&prefetch_cluster;
+ cluster >>= v->data_dev_block_bits;
+ if (unlikely(!cluster))
+ goto no_prefetch_cluster;
+ if (unlikely(cluster & (cluster - 1)))
+ cluster = 1 << (fls(cluster) - 1);
+
+ hash_block_start &= ~(sector_t)(cluster - 1);
+ hash_block_end |= cluster - 1;
+ if (unlikely(hash_block_end >= v->hash_blocks))
+ hash_block_end = v->hash_blocks - 1;
+ }
+no_prefetch_cluster:
+ dm_bufio_prefetch(v->bufio, hash_block_start,
+ hash_block_end - hash_block_start + 1);
+ }
+}
+
+/*
+ * Bio map function. It allocates dm_verity_io structure and bio vector and
+ * fills them. Then it issues prefetches and the I/O.
+ */
+static int verity_map(struct dm_target *ti, struct bio *bio,
+ union map_info *map_context)
+{
+ struct dm_verity *v = ti->private;
+ struct dm_verity_io *io;
+
+ bio->bi_bdev = v->data_dev->bdev;
+ bio->bi_sector = verity_map_sector(v, bio->bi_sector);
+
+ if (((unsigned)bio->bi_sector | bio_sectors(bio)) &
+ ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) {
+ DMERR_LIMIT("unaligned io");
+ return -EIO;
+ }
+
+ if ((bio->bi_sector + bio_sectors(bio)) >>
+ (v->data_dev_block_bits - SECTOR_SHIFT) > v->data_blocks) {
+ DMERR_LIMIT("io out of range");
+ return -EIO;
+ }
+
+ if (bio_data_dir(bio) == WRITE)
+ return -EIO;
+
+ io = mempool_alloc(v->io_mempool, GFP_NOIO);
+ io->v = v;
+ io->bio = bio;
+ io->orig_bi_end_io = bio->bi_end_io;
+ io->orig_bi_private = bio->bi_private;
+ io->block = bio->bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT);
+ io->n_blocks = bio->bi_size >> v->data_dev_block_bits;
+
+ bio->bi_end_io = verity_end_io;
+ bio->bi_private = io;
+ io->io_vec_size = bio->bi_vcnt - bio->bi_idx;
+ if (io->io_vec_size < DM_VERITY_IO_VEC_INLINE)
+ io->io_vec = io->io_vec_inline;
+ else
+ io->io_vec = mempool_alloc(v->vec_mempool, GFP_NOIO);
+ memcpy(io->io_vec, bio_iovec(bio),
+ io->io_vec_size * sizeof(struct bio_vec));
+
+ verity_prefetch_io(v, io);
+
+ generic_make_request(bio);
+
+ return DM_MAPIO_SUBMITTED;
+}
+
+static int verity_status(struct dm_target *ti, status_type_t type,
+ char *result, unsigned maxlen)
+{
+ struct dm_verity *v = ti->private;
+ unsigned sz = 0;
+ unsigned x;
+
+ switch (type) {
+ case STATUSTYPE_INFO:
+ result[0] = 0;
+ break;
+ case STATUSTYPE_TABLE:
+ DMEMIT("%u %s %s %u %u %llu %llu %s ",
+ 0,
+ v->data_dev->name,
+ v->hash_dev->name,
+ 1 << v->data_dev_block_bits,
+ 1 << v->hash_dev_block_bits,
+ (unsigned long long)v->data_blocks,
+ (unsigned long long)v->hash_start,
+ v->alg_name
+ );
+ for (x = 0; x < v->digest_size; x++)
+ DMEMIT("%02x", v->root_digest[x]);
+ DMEMIT(" ");
+ if (!v->salt_size)
+ DMEMIT("-");
+ else
+ for (x = 0; x < v->salt_size; x++)
+ DMEMIT("%02x", v->salt[x]);
+ break;
+ }
+ return 0;
+}
+
+static int verity_ioctl(struct dm_target *ti, unsigned cmd,
+ unsigned long arg)
+{
+ struct dm_verity *v = ti->private;
+ int r = 0;
+
+ if (v->data_start ||
+ ti->len != i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT)
+ r = scsi_verify_blk_ioctl(NULL, cmd);
+
+ return r ? : __blkdev_driver_ioctl(v->data_dev->bdev, v->data_dev->mode,
+ cmd, arg);
+}
+
+static int verity_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
+ struct bio_vec *biovec, int max_size)
+{
+ struct dm_verity *v = ti->private;
+ struct request_queue *q = bdev_get_queue(v->data_dev->bdev);
+
+ if (!q->merge_bvec_fn)
+ return max_size;
+
+ bvm->bi_bdev = v->data_dev->bdev;
+ bvm->bi_sector = verity_map_sector(v, bvm->bi_sector);
+
+ return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
+}
+
+static int verity_iterate_devices(struct dm_target *ti,
+ iterate_devices_callout_fn fn, void *data)
+{
+ struct dm_verity *v = ti->private;
+ return fn(ti, v->data_dev, v->data_start, ti->len, data);
+}
+
+static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits)
+{
+ struct dm_verity *v = ti->private;
+
+ if (limits->logical_block_size < 1 << v->data_dev_block_bits)
+ limits->logical_block_size = 1 << v->data_dev_block_bits;
+ if (limits->physical_block_size < 1 << v->data_dev_block_bits)
+ limits->physical_block_size = 1 << v->data_dev_block_bits;
+ blk_limits_io_min(limits, limits->logical_block_size);
+}
+
+static void verity_dtr(struct dm_target *ti);
+
+static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv)
+{
+ struct dm_verity *v;
+ unsigned num;
+ unsigned long long num_ll;
+ int r;
+ int i;
+ sector_t hash_position;
+ char dummy;
+
+ v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL);
+ if (!v) {
+ ti->error = "Cannot allocate verity structure";
+ return -ENOMEM;
+ }
+ ti->private = v;
+ v->ti = ti;
+
+ if ((dm_table_get_mode(ti->table) & ~FMODE_READ) != 0) {
+ ti->error = "Device must be readonly";
+ r = -EINVAL;
+ goto bad;
+ }
+
+ if (argc != 10) {
+ ti->error = "Invalid argument count";
+ r = -EINVAL;
+ goto bad;
+ }
+
+ if (sscanf(argv[0], "%d%c", &num, &dummy) != 1 ||
+ num != 0) {
+ ti->error = "Invalid version";
+ r = -EINVAL;
+ goto bad;
+ }
+
+ r = dm_get_device(ti, argv[1], FMODE_READ, &v->data_dev);
+ if (r) {
+ ti->error = "Data device lookup failed";
+ goto bad;
+ }
+
+ r = dm_get_device(ti, argv[2], FMODE_READ, &v->hash_dev);
+ if (r) {
+ ti->error = "Data device lookup failed";
+ goto bad;
+ }
+
+ if (sscanf(argv[3], "%u%c", &num, &dummy) != 1 ||
+ !num || (num & (num - 1)) ||
+ num < bdev_logical_block_size(v->data_dev->bdev) ||
+ num > PAGE_SIZE) {
+ ti->error = "Invalid data device block size";
+ r = -EINVAL;
+ goto bad;
+ }
+ v->data_dev_block_bits = ffs(num) - 1;
+
+ if (sscanf(argv[4], "%u%c", &num, &dummy) != 1 ||
+ !num || (num & (num - 1)) ||
+ num < bdev_logical_block_size(v->hash_dev->bdev) ||
+ num > INT_MAX) {
+ ti->error = "Invalid hash device block size";
+ r = -EINVAL;
+ goto bad;
+ }
+ v->hash_dev_block_bits = ffs(num) - 1;
+
+ if (sscanf(argv[5], "%llu%c", &num_ll, &dummy) != 1 ||
+ num_ll << (v->data_dev_block_bits - SECTOR_SHIFT) !=
+ (sector_t)num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) {
+ ti->error = "Invalid data blocks";
+ r = -EINVAL;
+ goto bad;
+ }
+ v->data_blocks = num_ll;
+
+ if (ti->len > (v->data_blocks << (v->data_dev_block_bits - SECTOR_SHIFT))) {
+ ti->error = "Data device is too small";
+ r = -EINVAL;
+ goto bad;
+ }
+
+ if (sscanf(argv[6], "%llu%c", &num_ll, &dummy) != 1 ||
+ num_ll << (v->hash_dev_block_bits - SECTOR_SHIFT) !=
+ (sector_t)num_ll << (v->hash_dev_block_bits - SECTOR_SHIFT)) {
+ ti->error = "Invalid hash start";
+ r = -EINVAL;
+ goto bad;
+ }
+ v->hash_start = num_ll;
+
+ v->alg_name = kstrdup(argv[7], GFP_KERNEL);
+ if (!v->alg_name) {
+ ti->error = "Cannot allocate algorithm name";
+ r = -ENOMEM;
+ goto bad;
+ }
+
+ v->tfm = crypto_alloc_shash(v->alg_name, 0, 0);
+ if (IS_ERR(v->tfm)) {
+ ti->error = "Cannot initialize hash function";
+ r = PTR_ERR(v->tfm);
+ v->tfm = NULL;
+ goto bad;
+ }
+ v->digest_size = crypto_shash_digestsize(v->tfm);
+ if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) {
+ ti->error = "Digest size too big";
+ r = -EINVAL;
+ goto bad;
+ }
+ v->shash_descsize =
+ sizeof(struct shash_desc) + crypto_shash_descsize(v->tfm);
+
+ v->root_digest = kmalloc(v->digest_size, GFP_KERNEL);
+ if (!v->root_digest) {
+ ti->error = "Cannot allocate root digest";
+ r = -ENOMEM;
+ goto bad;
+ }
+ if (strlen(argv[8]) != v->digest_size * 2 ||
+ hex2bin(v->root_digest, argv[8], v->digest_size)) {
+ ti->error = "Invalid root digest";
+ r = -EINVAL;
+ goto bad;
+ }
+
+ if (strcmp(argv[9], "-")) {
+ v->salt_size = strlen(argv[9]) / 2;
+ v->salt = kmalloc(v->salt_size, GFP_KERNEL);
+ if (!v->salt) {
+ ti->error = "Cannot allocate salt";
+ r = -ENOMEM;
+ goto bad;
+ }
+ if (strlen(argv[9]) != v->salt_size * 2 ||
+ hex2bin(v->salt, argv[9], v->salt_size)) {
+ ti->error = "Invalid salt";
+ r = -EINVAL;
+ goto bad;
+ }
+ }
+
+ v->hash_per_block_bits =
+ fls((1 << v->hash_dev_block_bits) / v->digest_size) - 1;
+
+ v->levels = 0;
+ if (v->data_blocks)
+ while (v->hash_per_block_bits * v->levels < 64 &&
+ (unsigned long long)(v->data_blocks - 1) >>
+ (v->hash_per_block_bits * v->levels))
+ v->levels++;
+
+ if (v->levels > DM_VERITY_MAX_LEVELS) {
+ ti->error = "Too many tree levels";
+ r = -E2BIG;
+ goto bad;
+ }
+
+ hash_position = v->hash_start;
+ for (i = v->levels - 1; i >= 0; i--) {
+ sector_t s;
+ v->hash_level_block[i] = hash_position;
+ s = verity_position_at_level(v, v->data_blocks, i);
+ s = (s >> v->hash_per_block_bits) +
+ !!(s & ((1 << v->hash_per_block_bits) - 1));
+ if (hash_position + s < hash_position) {
+ ti->error = "Hash device offset overflow";
+ r = -E2BIG;
+ goto bad;
+ }
+ hash_position += s;
+ }
+ v->hash_blocks = hash_position;
+
+ v->bufio = dm_bufio_client_create(v->hash_dev->bdev,
+ 1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux),
+ dm_bufio_alloc_callback, NULL);
+ if (IS_ERR(v->bufio)) {
+ ti->error = "Cannot initialize dm-bufio";
+ r = PTR_ERR(v->bufio);
+ v->bufio = NULL;
+ goto bad;
+ }
+
+ if (dm_bufio_get_device_size(v->bufio) < v->hash_blocks) {
+ ti->error = "Hash device is too small";
+ r = -E2BIG;
+ goto bad;
+ }
+
+ v->io_mempool = mempool_create_kmalloc_pool(DM_VERITY_MEMPOOL_SIZE ,
+ sizeof(struct dm_verity_io) + v->shash_descsize + v->digest_size * 2);
+ if (!v->io_mempool) {
+ ti->error = "Cannot allocate io mempool";
+ r = -ENOMEM;
+ goto bad;
+ }
+
+ v->vec_mempool = mempool_create_kmalloc_pool(DM_VERITY_MEMPOOL_SIZE ,
+ BIO_MAX_PAGES * sizeof(struct bio_vec));
+ if (!v->vec_mempool) {
+ ti->error = "Cannot allocate vector mempool";
+ r = -ENOMEM;
+ goto bad;
+ }
+
+ /*v->verify_wq = alloc_workqueue("verityd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 1);*/
+ /* WQ_UNBOUND greatly improves performance when running on ramdisk */
+ v->verify_wq = alloc_workqueue("verityd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, num_online_cpus());
+ if (!v->verify_wq) {
+ ti->error = "Cannot allocate workqueue";
+ r = -ENOMEM;
+ goto bad;
+ }
+
+ return 0;
+
+bad:
+ verity_dtr(ti);
+ return r;
+}
+
+static void verity_dtr(struct dm_target *ti)
+{
+ struct dm_verity *v = ti->private;
+
+ if (v->verify_wq)
+ destroy_workqueue(v->verify_wq);
+ if (v->vec_mempool)
+ mempool_destroy(v->vec_mempool);
+ if (v->io_mempool)
+ mempool_destroy(v->io_mempool);
+ if (v->bufio)
+ dm_bufio_client_destroy(v->bufio);
+ kfree(v->salt);
+ kfree(v->root_digest);
+ if (v->tfm)
+ crypto_free_shash(v->tfm);
+ kfree(v->alg_name);
+ if (v->hash_dev)
+ dm_put_device(ti, v->hash_dev);
+ if (v->data_dev)
+ dm_put_device(ti, v->data_dev);
+ kfree(v);
+}
+
+static struct target_type verity_target = {
+ .name = "verity",
+ .version = {1, 0, 0},
+ .module = THIS_MODULE,
+ .ctr = verity_ctr,
+ .dtr = verity_dtr,
+ .map = verity_map,
+ .status = verity_status,
+ .ioctl = verity_ioctl,
+ .merge = verity_merge,
+ .iterate_devices = verity_iterate_devices,
+ .io_hints = verity_io_hints,
+};
+
+static int __init dm_verity_init(void)
+{
+ int r;
+ r = dm_register_target(&verity_target);
+ if (r < 0)
+ DMERR("register failed %d", r);
+ return r;
+}
+
+static void __exit dm_verity_exit(void)
+{
+ dm_unregister_target(&verity_target);
+}
+
+module_init(dm_verity_init);
+module_exit(dm_verity_exit);
+
+MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>");
+MODULE_AUTHOR("Mandeep Baines <msb@chromium.org>");
+MODULE_AUTHOR("Will Drewry <wad@chromium.org>");
+MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking");
+MODULE_LICENSE("GPL");
+

--
dm-devel mailing list
dm-devel@redhat.com
https://www.redhat.com/mailman/listinfo/dm-devel
 
Old 03-21-2012, 12:10 AM
Mikulas Patocka
 
Default dm: remake of the verity target

Hi

Here I'm sending the new userspace tool:

Changes since the last version:

* Format is changed to be compatible with the current kernel code (i.e.
hash salt before the data)

* If no salt is specified, random salt is generated

* Switch -a is used to activate the device (it executes dmsetup with
the correct parameters)

* There is a superblock that can be used to identify verity hash
partition. Most arguments are stored in the superblock, so they don't have
to be specified when verifying or activating. Superblock stores all the
options except: device names, --hash-start, root block hash.

* The superblock is read or written only by the userspace tool, it is not
read by the kernel code.


Example use:

# create some filesystem on /dev/sdc2, fill it with data and unmount it
...
# read /dev/sdc2 and create appropriate hashes on /dev/sdc3
./veritysetup -c /dev/sdc2 /dev/sdc3
# verify the hashes (supply the real hash as reported by ./veritysetup -c)
./veritysetup -v /dev/sdc2 /dev/sdc3 3f1fe1c1ff7f229f1ff113c2b4b76ce4bcd9f455dd7f990673 b3e71f6a406e38
# activate the kernel driver with device mapper name "v"
./veritysetup -a v /dev/sdc2 /dev/sdc3 3f1fe1c1ff7f229f1ff113c2b4b76ce4bcd9f455dd7f990673 b3e71f6a406e38
# mount the activated filesystem
mount -o ro /dev/mapper/v /mnt/test

---

/* link with -lpopt -lcrypto */

#define _FILE_OFFSET_BITS 64

#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <stdarg.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/ioctl.h>
#include <sys/mount.h>
#include <arpa/inet.h>
#include <popt.h>
#include <openssl/evp.h>
#include <openssl/rand.h>

#define DEFAULT_BLOCK_SIZE 4096
#define DM_VERITY_MAX_LEVELS 63

#define DEFAULT_SALT_SIZE 32
#define MAX_SALT_SIZE 384

#define MODE_VERIFY 0
#define MODE_CREATE 1
#define MODE_ACTIVATE 2

static int mode = -1;
static int use_superblock = 1;

static const char *dm_device;
static const char *data_device;
static const char *hash_device;
static const char *hash_algorithm = NULL;
static const char *root_hash;

static int data_block_size = 0;
static int hash_block_size = 0;
static long long hash_start = 0;
static long long data_blocks = 0;
static const char *salt_string = NULL;

static FILE *data_file;
static FILE *hash_file;

static off_t data_file_blocks;
static off_t hash_file_blocks;
static off_t used_hash_blocks;

static const EVP_MD *evp;

static unsigned char *root_hash_bytes;
static unsigned char *calculated_digest;

static unsigned char *salt_bytes;
static unsigned salt_size;

static unsigned digest_size;
static unsigned char digest_size_bits;
static unsigned char levels;
static unsigned char hash_per_block_bits;

static off_t hash_level_block[DM_VERITY_MAX_LEVELS];
static off_t hash_level_size[DM_VERITY_MAX_LEVELS];

static off_t superblock_position;

static int retval = 0;

struct superblock {
uint8_t signature[8];
uint8_t version;
uint8_t data_block_bits;
uint8_t hash_block_bits;
uint8_t pad1[1];
uint16_t salt_size;
uint8_t pad2[2];
uint32_t data_blocks_hi;
uint32_t data_blocks_lo;
uint8_t algorithm[16];
uint8_t salt[MAX_SALT_SIZE];
uint8_t pad3[88];
};

#define DM_VERITY_SIGNATURE "verity"
#define DM_VERITY_VERSION 0

static void help(poptContext popt_context,
enum poptCallbackReason reason,
struct poptOption *key,
const char *arg,
void *data)
{
poptPrintHelp(popt_context, stdout, 0);
exit(0);
}

static struct poptOption popt_help_options[] = {
{ NULL, 0, POPT_ARG_CALLBACK, help, 0, NULL, NULL },
{ "help", 'h', POPT_ARG_NONE, NULL, 0, "Show help", NULL },
POPT_TABLEEND
};

static struct poptOption popt_options[] = {
{ NULL, '', POPT_ARG_INCLUDE_TABLE, popt_help_options, 0, NULL, NULL },
{ "create", 'c', POPT_ARG_VAL, &mode, MODE_CREATE, "Create hash", NULL },
{ "verify", 'v', POPT_ARG_VAL, &mode, MODE_VERIFY, "Verify integrity", NULL },
{ "activate", 'a', POPT_ARG_VAL, &mode, MODE_ACTIVATE, "Activate the device", NULL },
{ "data-block-size", 0, POPT_ARG_INT, &data_block_size, 0, "Block size on the data device", "bytes" },
{ "hash-block-size", 0, POPT_ARG_INT, &hash_block_size, 0, "Block size on the hash device", "bytes" },
{ "data-blocks", 0, POPT_ARG_LONGLONG, &data_blocks, 0, "The number of blocks in the data file", "blocks" },
{ "hash-start", 0, POPT_ARG_LONGLONG, &hash_start, 0, "Starting block on the hash device", "512-byte sectors" },
{ "salt", 0, POPT_ARG_STRING, &salt_string, 0, "Salt", "hex string" },
{ "algorithm", 0, POPT_ARG_STRING, &hash_algorithm, 0, "Hash algorithm (default sha256)", "string" },
{ "no-superblock", 0, POPT_ARG_VAL, &use_superblock, 0, "Do not create/use superblock" },
POPT_TABLEEND
};

#if defined(__GNUC__) && __GNUC__ >= 2
__attribute__((__format__(__printf__, 1, 2)))
#endif
static void exit_err(const char *msg, ...)
{
va_list args;
va_start(args, msg);
vfprintf(stderr, msg, args);
va_end(args);
fputc('
', stderr);
exit(2);
}

static void stream_err(FILE *f, const char *msg)
{
if (ferror(f)) {
perror(msg);
exit(2);
} else if (feof(f)) {
exit_err("eof on %s", msg);
} else {
exit_err("unknown error on %s", msg);
}
}

static void *xmalloc(size_t s)
{
void *ptr = malloc(!s ? 1 : s);
if (!ptr) exit_err("out of memory");
return ptr;
}

static char *xstrdup(const char *str)
{
return strcpy(xmalloc(strlen(str) + 1), str);
}

static char *xprint(unsigned long long num)
{
size_t s = snprintf(NULL, 0, "%llu", num);
char *p = xmalloc(s + 1);
snprintf(p, s + 1, "%llu", num);
return p;
}

static char *xhexprint(unsigned char *bytes, size_t len)
{
size_t i;
char *p = xmalloc(len * 2 + 1);
p[0] = 0;
for (i = 0; i < len; i++)
snprintf(p + i * 2, 3, "%02x", bytes[i]);
return p;
}

static off_t get_size(FILE *f, const char *name)
{
struct stat st;
int h = fileno(f);
if (h < 0) {
perror("fileno");
exit(2);
}
if (fstat(h, &st)) {
perror("fstat");
exit(2);
}
if (S_ISREG(st.st_mode)) {
return st.st_size;
} else if (S_ISBLK(st.st_mode)) {
unsigned long long size64;
unsigned long sizeul;
if (!ioctl(h, BLKGETSIZE64, &size64)) {
return_size64:
if ((off_t)size64 < 0 || (off_t)size64 != size64) {
size_overflow:
exit_err("%s: device size overflow", name);
}
return size64;
}
if (!ioctl(h, BLKGETSIZE, &sizeul)) {
size64 = (unsigned long long)sizeul * 512;
if (size64 / 512 != sizeul) goto size_overflow;
goto return_size64;
}
perror("BLKGETSIZE");
exit(2);
} else {
exit_err("%s is not a file or a block device", name);
}
return -1; /* never reached, shut up warning */
}

static void block_fseek(FILE *f, off_t block, int block_size)
{
unsigned long long pos = (unsigned long long)block * block_size;
if (pos / block_size != block ||
(off_t)pos < 0 ||
(off_t)pos != pos)
exit_err("seek position overflow");
if (fseeko(f, pos, SEEK_SET)) {
perror("fseek");
exit(2);
}
}

static off_t verity_position_at_level(off_t block, int level)
{
return block >> (level * hash_per_block_bits);
}

static void calculate_positions(void)
{
unsigned long long hash_position;
int i;

digest_size_bits = 0;
while (1 << digest_size_bits < digest_size)
digest_size_bits++;
hash_per_block_bits = 0;
while (((hash_block_size / digest_size) >> hash_per_block_bits) > 1)
hash_per_block_bits++;
if (!hash_per_block_bits)
exit_err("at least two hashes must fit in a hash file block");
levels = 0;

if (data_file_blocks) {
while (hash_per_block_bits * levels < 64 &&
(unsigned long long)(data_file_blocks - 1) >>
(hash_per_block_bits * levels))
levels++;
}

if (levels > DM_VERITY_MAX_LEVELS) exit_err("too many tree levels");

hash_position = hash_start * 512 / hash_block_size;
for (i = levels - 1; i >= 0; i--) {
off_t s;
hash_level_block[i] = hash_position;
s = verity_position_at_level(data_file_blocks, i);
s = (s >> hash_per_block_bits) +
!!(s & ((1 << hash_per_block_bits) - 1));
hash_level_size[i] = s;
if (hash_position + s < hash_position ||
(off_t)(hash_position + s) < 0 ||
(off_t)(hash_position + s) != hash_position + s)
exit_err("hash device offset overflow");
hash_position += s;
}
used_hash_blocks = hash_position;
}

static void create_or_verify_zero(FILE *wr, unsigned char *left_block, unsigned left_bytes)
{
if (left_bytes) {
if (mode != MODE_CREATE) {
unsigned x;
if (fread(left_block, left_bytes, 1, wr) != 1)
stream_err(wr, "read");
for (x = 0; x < left_bytes; x++) if (left_block[x]) {
retval = 1;
fprintf(stderr, "spare area is not zeroed at position %lld
", (long long)ftello(wr) - left_bytes);
}
} else {
if (fwrite(left_block, left_bytes, 1, wr) != 1)
stream_err(wr, "write");
}
}
}

static void create_or_verify_stream(FILE *rd, FILE *wr, int block_size, off_t blocks)
{
unsigned char *left_block = xmalloc(hash_block_size);
unsigned char *data_buffer = xmalloc(block_size);
unsigned char *read_digest = mode != MODE_CREATE ? xmalloc(digest_size) : NULL;
off_t blocks_to_write = (blocks >> hash_per_block_bits) +
!!(blocks & ((1 << hash_per_block_bits) - 1));
EVP_MD_CTX ctx;
EVP_MD_CTX_init(&ctx);
memset(left_block, 0, hash_block_size);
while (blocks_to_write--) {
unsigned x;
unsigned left_bytes = hash_block_size;
for (x = 0; x < 1 << hash_per_block_bits; x++) {
if (!blocks)
break;
blocks--;
if (fread(data_buffer, block_size, 1, rd) != 1)
stream_err(rd, "read");
if (EVP_DigestInit_ex(&ctx, evp, NULL) != 1)
exit_err("EVP_DigestInit_ex failed");
if (EVP_DigestUpdate(&ctx, salt_bytes, salt_size) != 1)
exit_err("EVP_DigestUpdate failed");
if (EVP_DigestUpdate(&ctx, data_buffer, block_size) != 1)
exit_err("EVP_DigestUpdate failed");
if (EVP_DigestFinal_ex(&ctx, calculated_digest, NULL) != 1)
exit_err("EVP_DigestFinal_ex failed");
if (!wr)
break;
if (mode != MODE_CREATE) {
if (fread(read_digest, digest_size, 1, wr) != 1)
stream_err(wr, "read");
if (memcmp(read_digest, calculated_digest, digest_size)) {
retval = 1;
fprintf(stderr, "verification failed at position %lld in %s file
", (long long)ftello(rd) - block_size, rd == data_file ? "data" : "metadata");
}
} else {
if (fwrite(calculated_digest, digest_size, 1, wr) != 1)
stream_err(wr, "write");
}
create_or_verify_zero(wr, left_block, (1 << digest_size_bits) - digest_size);
left_bytes -= 1 << digest_size_bits;
}
if (wr)
create_or_verify_zero(wr, left_block, left_bytes);
}
if (mode == MODE_CREATE && wr) {
if (fflush(wr)) {
perror("fflush");
exit(1);
}
if (ferror(wr)) {
stream_err(wr, "write");
}
}
if (EVP_MD_CTX_cleanup(&ctx) != 1)
exit_err("EVP_MD_CTX_cleanup failed");
free(left_block);
free(data_buffer);
if (mode != MODE_CREATE) free(read_digest);
}

char **make_target_line(void)
{
const int line_elements = 14;
char **line = xmalloc(line_elements * sizeof(char *));
int i = 0;
char *algorithm_copy = xstrdup(hash_algorithm);
/* transform ripemdXXX to rmdXXX */
if (!strncmp(algorithm_copy, "ripemd", 6))
memmove(algorithm_copy + 1, algorithm_copy + 4, strlen(algorithm_copy + 4) + 1);
line[i++] = xstrdup("0");
line[i++] = xprint((unsigned long long)data_file_blocks * data_block_size / 512);
line[i++] = xstrdup("verity");
line[i++] = xstrdup("0");
line[i++] = xstrdup(data_device);
line[i++] = xstrdup(hash_device);
line[i++] = xprint(data_block_size);
line[i++] = xprint(hash_block_size);
line[i++] = xprint(data_file_blocks);
line[i++] = xprint(hash_start * 512 / hash_block_size);
line[i++] = algorithm_copy;
line[i++] = xhexprint(calculated_digest, digest_size);
line[i++] = !salt_size ? xstrdup("-") : xhexprint(salt_bytes, salt_size);
line[i++] = NULL;
if (i > line_elements) exit_err("INTERNAL ERROR: insufficient array size");
return line;
}

void free_target_line(char **line)
{
int i;
for (i = 0; line[i]; i++)
free(line[i]);
free(line);
}

static void create_or_verify(void)
{
int i;
if (mode != MODE_ACTIVATE) for (i = 0; i < levels; i++) {
block_fseek(hash_file, hash_level_block[i], hash_block_size);
if (!i) {
block_fseek(data_file, 0, data_block_size);
create_or_verify_stream(data_file, hash_file, data_block_size, data_file_blocks);
} else {
FILE *hash_file_2 = fopen(hash_device, "r");
if (!hash_file_2) {
perror(hash_device);
exit(2);
}
block_fseek(hash_file_2, hash_level_block[i - 1], hash_block_size);
create_or_verify_stream(hash_file_2, hash_file, hash_block_size, hash_level_size[i - 1]);
fclose(hash_file_2);
}
}

if (levels) {
block_fseek(hash_file, hash_level_block[levels - 1], hash_block_size);
create_or_verify_stream(hash_file, NULL, hash_block_size, 1);
} else {
block_fseek(data_file, 0, data_block_size);
create_or_verify_stream(data_file, NULL, data_block_size, data_file_blocks);
}

if (mode != MODE_CREATE) {
if (memcmp(calculated_digest, root_hash_bytes, digest_size)) {
fprintf(stderr, "verification failed in the root block
");
retval = 1;
}
if (!retval && mode == MODE_VERIFY)
fprintf(stderr, "hash successfully verified
");
} else {
char **target_line;
char *p;
if (fsync(fileno(hash_file))) {
perror("fsync");
exit(1);
}
printf("hash device size: %llu
", (unsigned long long)used_hash_blocks * hash_block_size);
printf("data block size %u, hash block size %u, %u tree levels
", data_block_size, hash_block_size, levels);
if (salt_size) p = xhexprint(salt_bytes, salt_size);
else p = xstrdup("-");
printf("salt: %s
", p);
free(p);
p = xhexprint(calculated_digest, digest_size);
printf("root hash: %s
", p);
free(p);
printf("target line:");
target_line = make_target_line();
for (i = 0; target_line[i]; i++)
printf(" %s", target_line[i]);
free_target_line(target_line);
printf("
");
}
}

static void activate(void)
{
int i;
size_t len = 1;
char *table_arg;
char **target_line = make_target_line();
for (i = 0; target_line[i]; i++) {
if (i) len++;
len += strlen(target_line[i]);
}
table_arg = xmalloc(len);
table_arg[0] = 0;
for (i = 0; target_line[i]; i++) {
if (i) strcat(table_arg, " ");
strcat(table_arg, target_line[i]);
}
free_target_line(target_line);
execlp("dmsetup", "dmsetup", "-r", "create", dm_device, "--table", table_arg, NULL);
perror("dmsetup");
exit(2);
}

static void get_hex(const char *string, unsigned char **result, size_t len, const char *description)
{
size_t rl = strlen(string);
unsigned u;
if (strspn(string, "0123456789ABCDEFabcdef") != rl)
exit_err("invalid %s", description);
if (rl != len * 2)
exit_err("invalid length of %s", description);
*result = xmalloc(len);
memset(*result, 0, len);
for (u = 0; u < rl; u++) {
unsigned char c = (string[u] & 15) + (string[u] > '9' ? 9 : 0);
(*result)[u / 2] |= c << (((u & 1) ^ 1) << 2);
}
}

static struct superblock superblock;

static void load_superblock(void)
{
long long sb_data_blocks;

block_fseek(hash_file, superblock_position, 1);
if (fread(&superblock, sizeof(struct superblock), 1, hash_file) != 1)
stream_err(hash_file, "read");
if (memcmp(superblock.signature, DM_VERITY_SIGNATURE, sizeof(superblock.signature)))
exit_err("superblock not found on the hash device");
if (superblock.version != 0)
exit_err("unknown version");
if (superblock.data_block_bits < 9 || superblock.data_block_bits >= 31)
exit_err("invalid data_block_bits in the superblock");
if (superblock.hash_block_bits < 9 || superblock.hash_block_bits >= 31)
exit_err("invalid data_block_bits in the superblock");
sb_data_blocks = ((unsigned long long)ntohl(superblock.data_blocks_hi) << 31 << 1) | ntohl(superblock.data_blocks_lo);
if (sb_data_blocks < 0 || (off_t)sb_data_blocks < 0 || (off_t)sb_data_blocks != sb_data_blocks)
exit_err("invalid data blocks in the superblock");
if (!memchr(superblock.algorithm, 0, sizeof(superblock.algorithm)))
exit_err("invalid hash algorithm in the superblock");
if (ntohs(superblock.salt_size) > MAX_SALT_SIZE)
exit_err("invalid salt_size in the superblock");

if (!data_block_size) {
data_block_size = 1 << superblock.data_block_bits;
} else {
if (data_block_size != 1 << superblock.data_block_bits)
exit_err("data block size (%d) does not match superblock value (%d)", data_block_size, 1 << superblock.data_block_bits);
}

if (!hash_block_size) {
hash_block_size = 1 << superblock.hash_block_bits;
} else {
if (hash_block_size != 1 << superblock.hash_block_bits)
exit_err("hash block size (%d) does not match superblock value (%d)", hash_block_size, 1 << superblock.hash_block_bits);
}

if (!data_blocks) {
data_blocks = sb_data_blocks;
} else {
if (data_blocks != sb_data_blocks)
exit_err("data blocks (%lld) does not match superblock value (%lld)", data_blocks, sb_data_blocks);
}

if (!hash_algorithm) {
hash_algorithm = (char *)superblock.algorithm;
} else {
if (strcmp(hash_algorithm, (char *)superblock.algorithm))
exit_err("hash algorithm (%s) does not match superblock value (%s)", hash_algorithm, superblock.algorithm);
}

if (!salt_bytes) {
salt_size = ntohs(superblock.salt_size);
salt_bytes = xmalloc(salt_size);
memcpy(salt_bytes, superblock.salt, salt_size);
} else {
if (salt_size != ntohs(superblock.salt_size) ||
memcmp(salt_bytes, superblock.salt, salt_size))
exit_err("salt does not match superblock value");
}
}

static void save_superblock(void)
{
memset(&superblock, 0, sizeof(struct superblock));

memcpy(&superblock.signature, DM_VERITY_SIGNATURE, sizeof(superblock.signature));
superblock.version = 0;
superblock.data_block_bits = ffs(data_block_size) - 1;
superblock.hash_block_bits = ffs(hash_block_size) - 1;
superblock.salt_size = htons(salt_size);
superblock.data_blocks_hi = htonl(data_blocks >> 31 >> 1);
superblock.data_blocks_lo = htonl(data_blocks & 0xFFFFFFFF);
strncpy((char *)superblock.algorithm, hash_algorithm, sizeof superblock.algorithm);
memcpy(superblock.salt, salt_bytes, salt_size);

block_fseek(hash_file, superblock_position, 1);
if (fwrite(&superblock, sizeof(struct superblock), 1, hash_file) != 1)
stream_err(hash_file, "write");
}

int main(int argc, const char **argv)
{
poptContext popt_context;
int r;
const char *s;

if (sizeof(struct superblock) != 512)
exit_err("INTERNAL ERROR: bad superblock size %d", sizeof(struct superblock));

OpenSSL_add_all_digests();

popt_context = poptGetContext("verity", argc, argv, popt_options, 0);

poptSetOtherOptionHelp(popt_context, "[-c | -v | -a] [<device name> if activating] <data device> <hash device> [<root hash> if activating or verifying] [OPTION...]");

if (argc <= 1) {
poptPrintHelp(popt_context, stdout, 0);
exit(1);
}

r = poptGetNextOpt(popt_context);
if (r < -1) exit_err("bad option %s", poptBadOption(popt_context, 0));

if (mode < 0) exit_err("verify, create or activate mode not specified");

if (mode == MODE_ACTIVATE) {
dm_device = poptGetArg(popt_context);
if (!dm_device) exit_err("device name is missing");
if (!*dm_device || strchr(dm_device, '/')) exit_err("invalid device name");
}

data_device = poptGetArg(popt_context);
if (!data_device) exit_err("data device is missing");

hash_device = poptGetArg(popt_context);
if (!hash_device) exit_err("metadata device is missing");

if (mode != MODE_CREATE) {
root_hash = poptGetArg(popt_context);
if (!root_hash) exit_err("root hash not specified");
}

s = poptGetArg(popt_context);
if (s) exit_err("extra argument %s", s);

data_file = fopen(data_device, "r");
if (!data_file) {
perror(data_device);
exit(2);
}

hash_file = fopen(hash_device, mode != MODE_CREATE ? "r" : "r+");
if (!hash_file && errno == ENOENT && mode == MODE_CREATE)
hash_file = fopen(hash_device, "w+");
if (!hash_file) {
perror(hash_device);
exit(2);
}

if (hash_start < 0 ||
(unsigned long long)hash_start * 512 / 512 != hash_start ||
(off_t)(hash_start * 512) < 0 ||
(off_t)(hash_start * 512) != hash_start * 512) exit_err("invalid hash start");

if (salt_string || !use_superblock) {
if (!salt_string || !strcmp(salt_string, "-"))
salt_string = "";
salt_size = strlen(salt_string) / 2;
if (salt_size > MAX_SALT_SIZE)
exit_err("too long salt (max %d bytes)", MAX_SALT_SIZE);
get_hex(salt_string, &salt_bytes, salt_size, "salt");
}

if (use_superblock) {
superblock_position = hash_start * 512;
if (mode != MODE_CREATE)
load_superblock();
}

if (!data_block_size) data_block_size = DEFAULT_BLOCK_SIZE;
if (!hash_block_size) hash_block_size = data_block_size;

if (data_block_size < 512 || (data_block_size & (data_block_size - 1)) || data_block_size >= 1U << 31)
exit_err("invalid data block size");

if (hash_block_size < 512 || (hash_block_size & (hash_block_size - 1)) || hash_block_size >= 1U << 31)
exit_err("invalid hash block size");

if (data_blocks < 0 || (off_t)data_blocks < 0 || (off_t)data_blocks != data_blocks) exit_err("invalid number of data blocks");

data_file_blocks = get_size(data_file, data_device) / data_block_size;
hash_file_blocks = get_size(hash_file, hash_device) / hash_block_size;

if (data_file_blocks < data_blocks) exit_err("data file is too small");
if (data_blocks) {
data_file_blocks = data_blocks;
}

if (use_superblock) {
hash_start = hash_start + (sizeof(struct superblock) + 511) / 512;
hash_start = (hash_start + (hash_block_size / 512 - 1)) & ~(long long)(hash_block_size / 512 - 1);
}

if ((unsigned long long)hash_start * 512 % hash_block_size) exit_err("hash start not aligned on block size");

if (!hash_algorithm)
hash_algorithm = "sha256";
if (strlen(hash_algorithm) >= sizeof(superblock.algorithm) && use_superblock)
exit_err("hash algorithm name is too long");
evp = EVP_get_digestbyname(hash_algorithm);
if (!evp) exit_err("hash algorithm %s not found", hash_algorithm);
digest_size = EVP_MD_size(evp);

if (!salt_bytes) {
salt_size = DEFAULT_SALT_SIZE;
salt_bytes = xmalloc(salt_size);
if (RAND_bytes(salt_bytes, salt_size) != 1)
exit_err("RAND_bytes failed");
}

calculated_digest = xmalloc(digest_size);

if (mode != MODE_CREATE) {
get_hex(root_hash, &root_hash_bytes, digest_size, "root_hash");
}

calculate_positions();

create_or_verify();

if (use_superblock) {
if (mode == MODE_CREATE)
save_superblock();
}

fclose(data_file);
fclose(hash_file);

if (mode == MODE_ACTIVATE && !retval)
activate();

free(salt_bytes);
free(calculated_digest);
if (mode != MODE_CREATE)
free(root_hash_bytes);
poptFreeContext(popt_context);

return retval;
}

--
dm-devel mailing list
dm-devel@redhat.com
https://www.redhat.com/mailman/listinfo/dm-devel
 

Thread Tools




All times are GMT. The time now is 07:08 AM.

VBulletin, Copyright ©2000 - 2014, Jelsoft Enterprises Ltd.
Content Relevant URLs by vBSEO ©2007, Crawlability, Inc.
Copyright 2007 - 2008, www.linux-archive.org