2 * Copyright (C) 2001-2002 Sistina Software (UK) Limited.
3 * Copyright (C) 2006-2008 Red Hat GmbH
5 * This file is released under the GPL.
8 #include "dm-exception-store.h"
11 #include <linux/pagemap.h>
12 #include <linux/vmalloc.h>
13 #include <linux/slab.h>
14 #include <linux/dm-io.h>
16 #define DM_MSG_PREFIX "persistent snapshot"
17 #define DM_CHUNK_SIZE_DEFAULT_SECTORS 32 /* 16KB */
19 /*-----------------------------------------------------------------
20 * Persistent snapshots, by persistent we mean that the snapshot
21 * will survive a reboot.
22 *---------------------------------------------------------------*/
25 * We need to store a record of which parts of the origin have
26 * been copied to the snapshot device. The snapshot code
27 * requires that we copy exception chunks to chunk aligned areas
28 * of the COW store. It makes sense therefore, to store the
29 * metadata in chunk size blocks.
31 * There is no backward or forward compatibility implemented,
32 * snapshots with different disk versions than the kernel will
33 * not be usable. It is expected that "lvcreate" will blank out
34 * the start of a fresh COW device before calling the snapshot
37 * The first chunk of the COW device just contains the header.
38 * After this there is a chunk filled with exception metadata,
39 * followed by as many exception chunks as can fit in the
42 * All on disk structures are in little-endian format. The end
43 * of the exceptions info is indicated by an exception with a
44 * new_chunk of 0, which is invalid since it would point to the
49 * Magic for persistent snapshots: "SnAp" - Feeble isn't it.
51 #define SNAP_MAGIC 0x70416e53
54 * The on-disk version of the metadata.
56 #define SNAPSHOT_DISK_VERSION 1
62 * Is this snapshot valid. There is no way of recovering
63 * an invalid snapshot.
68 * Simple, incrementing version. no backward
77 struct disk_exception
{
82 struct commit_callback
{
83 void (*callback
)(void *, int success
);
88 * The top level structure for a persistent exception store.
91 struct dm_exception_store
*store
;
94 uint32_t exceptions_per_area
;
97 * Now that we have an asynchronous kcopyd there is no
98 * need for large chunk sizes, so it wont hurt to have a
99 * whole chunks worth of metadata in memory at once.
104 * An area of zeros used to clear the next area.
109 * An area used for header. The header can be written
110 * concurrently with metadata (when invalidating the snapshot),
111 * so it needs a separate buffer.
116 * Used to keep track of which metadata area the data in
119 chunk_t current_area
;
122 * The next free chunk for an exception.
127 * The index of next free exception in the current
130 uint32_t current_committed
;
132 atomic_t pending_count
;
133 uint32_t callback_count
;
134 struct commit_callback
*callbacks
;
135 struct dm_io_client
*io_client
;
137 struct workqueue_struct
*metadata_wq
;
140 static unsigned sectors_to_pages(unsigned sectors
)
142 return DIV_ROUND_UP(sectors
, PAGE_SIZE
>> 9);
145 static int alloc_area(struct pstore
*ps
)
150 len
= ps
->store
->chunk_size
<< SECTOR_SHIFT
;
153 * Allocate the chunk_size block of memory that will hold
154 * a single metadata area.
156 ps
->area
= vmalloc(len
);
160 ps
->zero_area
= vmalloc(len
);
163 memset(ps
->zero_area
, 0, len
);
165 ps
->header_area
= vmalloc(len
);
166 if (!ps
->header_area
)
167 goto err_header_area
;
172 vfree(ps
->zero_area
);
181 static void free_area(struct pstore
*ps
)
188 vfree(ps
->zero_area
);
189 ps
->zero_area
= NULL
;
192 vfree(ps
->header_area
);
193 ps
->header_area
= NULL
;
197 struct dm_io_region
*where
;
198 struct dm_io_request
*io_req
;
199 struct work_struct work
;
203 static void do_metadata(struct work_struct
*work
)
205 struct mdata_req
*req
= container_of(work
, struct mdata_req
, work
);
207 req
->result
= dm_io(req
->io_req
, 1, req
->where
, NULL
);
211 * Read or write a chunk aligned and sized block of data from a device.
213 static int chunk_io(struct pstore
*ps
, void *area
, chunk_t chunk
, int rw
,
216 struct dm_io_region where
= {
217 .bdev
= ps
->store
->cow
->bdev
,
218 .sector
= ps
->store
->chunk_size
* chunk
,
219 .count
= ps
->store
->chunk_size
,
221 struct dm_io_request io_req
= {
223 .mem
.type
= DM_IO_VMA
,
225 .client
= ps
->io_client
,
228 struct mdata_req req
;
231 return dm_io(&io_req
, 1, &where
, NULL
);
234 req
.io_req
= &io_req
;
237 * Issue the synchronous I/O from a different thread
238 * to avoid generic_make_request recursion.
240 INIT_WORK(&req
.work
, do_metadata
);
241 queue_work(ps
->metadata_wq
, &req
.work
);
242 flush_workqueue(ps
->metadata_wq
);
248 * Convert a metadata area index to a chunk index.
250 static chunk_t
area_location(struct pstore
*ps
, chunk_t area
)
252 return 1 + ((ps
->exceptions_per_area
+ 1) * area
);
256 * Read or write a metadata area. Remembering to skip the first
257 * chunk which holds the header.
259 static int area_io(struct pstore
*ps
, int rw
)
264 chunk
= area_location(ps
, ps
->current_area
);
266 r
= chunk_io(ps
, ps
->area
, chunk
, rw
, 0);
273 static void zero_memory_area(struct pstore
*ps
)
275 memset(ps
->area
, 0, ps
->store
->chunk_size
<< SECTOR_SHIFT
);
278 static int zero_disk_area(struct pstore
*ps
, chunk_t area
)
280 return chunk_io(ps
, ps
->zero_area
, area_location(ps
, area
), WRITE
, 0);
283 static int read_header(struct pstore
*ps
, int *new_snapshot
)
286 struct disk_header
*dh
;
288 int chunk_size_supplied
= 1;
292 * Use default chunk size (or logical_block_size, if larger)
295 if (!ps
->store
->chunk_size
) {
296 ps
->store
->chunk_size
= max(DM_CHUNK_SIZE_DEFAULT_SECTORS
,
297 bdev_logical_block_size(ps
->store
->cow
->bdev
) >> 9);
298 ps
->store
->chunk_mask
= ps
->store
->chunk_size
- 1;
299 ps
->store
->chunk_shift
= ffs(ps
->store
->chunk_size
) - 1;
300 chunk_size_supplied
= 0;
303 ps
->io_client
= dm_io_client_create(sectors_to_pages(ps
->store
->
305 if (IS_ERR(ps
->io_client
))
306 return PTR_ERR(ps
->io_client
);
312 r
= chunk_io(ps
, ps
->header_area
, 0, READ
, 1);
316 dh
= ps
->header_area
;
318 if (le32_to_cpu(dh
->magic
) == 0) {
323 if (le32_to_cpu(dh
->magic
) != SNAP_MAGIC
) {
324 DMWARN("Invalid or corrupt snapshot");
330 ps
->valid
= le32_to_cpu(dh
->valid
);
331 ps
->version
= le32_to_cpu(dh
->version
);
332 chunk_size
= le32_to_cpu(dh
->chunk_size
);
334 if (ps
->store
->chunk_size
== chunk_size
)
337 if (chunk_size_supplied
)
338 DMWARN("chunk size %u in device metadata overrides "
339 "table chunk size of %u.",
340 chunk_size
, ps
->store
->chunk_size
);
342 /* We had a bogus chunk_size. Fix stuff up. */
345 r
= dm_exception_store_set_chunk_size(ps
->store
, chunk_size
,
348 DMERR("invalid on-disk chunk size %u: %s.",
349 chunk_size
, chunk_err
);
353 r
= dm_io_client_resize(sectors_to_pages(ps
->store
->chunk_size
),
366 static int write_header(struct pstore
*ps
)
368 struct disk_header
*dh
;
370 memset(ps
->header_area
, 0, ps
->store
->chunk_size
<< SECTOR_SHIFT
);
372 dh
= ps
->header_area
;
373 dh
->magic
= cpu_to_le32(SNAP_MAGIC
);
374 dh
->valid
= cpu_to_le32(ps
->valid
);
375 dh
->version
= cpu_to_le32(ps
->version
);
376 dh
->chunk_size
= cpu_to_le32(ps
->store
->chunk_size
);
378 return chunk_io(ps
, ps
->header_area
, 0, WRITE
, 1);
382 * Access functions for the disk exceptions, these do the endian conversions.
384 static struct disk_exception
*get_exception(struct pstore
*ps
, uint32_t index
)
386 BUG_ON(index
>= ps
->exceptions_per_area
);
388 return ((struct disk_exception
*) ps
->area
) + index
;
391 static void read_exception(struct pstore
*ps
,
392 uint32_t index
, struct disk_exception
*result
)
394 struct disk_exception
*e
= get_exception(ps
, index
);
397 result
->old_chunk
= le64_to_cpu(e
->old_chunk
);
398 result
->new_chunk
= le64_to_cpu(e
->new_chunk
);
401 static void write_exception(struct pstore
*ps
,
402 uint32_t index
, struct disk_exception
*de
)
404 struct disk_exception
*e
= get_exception(ps
, index
);
407 e
->old_chunk
= cpu_to_le64(de
->old_chunk
);
408 e
->new_chunk
= cpu_to_le64(de
->new_chunk
);
412 * Registers the exceptions that are present in the current area.
413 * 'full' is filled in to indicate if the area has been
416 static int insert_exceptions(struct pstore
*ps
,
417 int (*callback
)(void *callback_context
,
418 chunk_t old
, chunk_t
new),
419 void *callback_context
,
424 struct disk_exception de
;
426 /* presume the area is full */
429 for (i
= 0; i
< ps
->exceptions_per_area
; i
++) {
430 read_exception(ps
, i
, &de
);
433 * If the new_chunk is pointing at the start of
434 * the COW device, where the first metadata area
435 * is we know that we've hit the end of the
436 * exceptions. Therefore the area is not full.
438 if (de
.new_chunk
== 0LL) {
439 ps
->current_committed
= i
;
445 * Keep track of the start of the free chunks.
447 if (ps
->next_free
<= de
.new_chunk
)
448 ps
->next_free
= de
.new_chunk
+ 1;
451 * Otherwise we add the exception to the snapshot.
453 r
= callback(callback_context
, de
.old_chunk
, de
.new_chunk
);
461 static int read_exceptions(struct pstore
*ps
,
462 int (*callback
)(void *callback_context
, chunk_t old
,
464 void *callback_context
)
469 * Keeping reading chunks and inserting exceptions until
470 * we find a partially full area.
472 for (ps
->current_area
= 0; full
; ps
->current_area
++) {
473 r
= area_io(ps
, READ
);
477 r
= insert_exceptions(ps
, callback
, callback_context
, &full
);
487 static struct pstore
*get_info(struct dm_exception_store
*store
)
489 return (struct pstore
*) store
->context
;
492 static void persistent_usage(struct dm_exception_store
*store
,
493 sector_t
*total_sectors
,
494 sector_t
*sectors_allocated
,
495 sector_t
*metadata_sectors
)
497 struct pstore
*ps
= get_info(store
);
499 *sectors_allocated
= ps
->next_free
* store
->chunk_size
;
500 *total_sectors
= get_dev_size(store
->cow
->bdev
);
503 * First chunk is the fixed header.
504 * Then there are (ps->current_area + 1) metadata chunks, each one
505 * separated from the next by ps->exceptions_per_area data chunks.
507 *metadata_sectors
= (ps
->current_area
+ 2) * store
->chunk_size
;
510 static void persistent_dtr(struct dm_exception_store
*store
)
512 struct pstore
*ps
= get_info(store
);
514 destroy_workqueue(ps
->metadata_wq
);
516 /* Created in read_header */
518 dm_io_client_destroy(ps
->io_client
);
521 /* Allocated in persistent_read_metadata */
523 vfree(ps
->callbacks
);
528 static int persistent_read_metadata(struct dm_exception_store
*store
,
529 int (*callback
)(void *callback_context
,
530 chunk_t old
, chunk_t
new),
531 void *callback_context
)
533 int r
, uninitialized_var(new_snapshot
);
534 struct pstore
*ps
= get_info(store
);
537 * Read the snapshot header.
539 r
= read_header(ps
, &new_snapshot
);
544 * Now we know correct chunk_size, complete the initialisation.
546 ps
->exceptions_per_area
= (ps
->store
->chunk_size
<< SECTOR_SHIFT
) /
547 sizeof(struct disk_exception
);
548 ps
->callbacks
= dm_vcalloc(ps
->exceptions_per_area
,
549 sizeof(*ps
->callbacks
));
554 * Do we need to setup a new snapshot ?
557 r
= write_header(ps
);
559 DMWARN("write_header failed");
563 ps
->current_area
= 0;
564 zero_memory_area(ps
);
565 r
= zero_disk_area(ps
, 0);
567 DMWARN("zero_disk_area(0) failed");
573 if (ps
->version
!= SNAPSHOT_DISK_VERSION
) {
574 DMWARN("unable to handle snapshot disk version %d",
580 * Metadata are valid, but snapshot is invalidated
588 r
= read_exceptions(ps
, callback
, callback_context
);
593 static int persistent_prepare_exception(struct dm_exception_store
*store
,
594 struct dm_exception
*e
)
596 struct pstore
*ps
= get_info(store
);
599 sector_t size
= get_dev_size(store
->cow
->bdev
);
601 /* Is there enough room ? */
602 if (size
< ((ps
->next_free
+ 1) * store
->chunk_size
))
605 e
->new_chunk
= ps
->next_free
;
608 * Move onto the next free pending, making sure to take
609 * into account the location of the metadata chunks.
611 stride
= (ps
->exceptions_per_area
+ 1);
612 next_free
= ++ps
->next_free
;
613 if (sector_div(next_free
, stride
) == 1)
616 atomic_inc(&ps
->pending_count
);
620 static void persistent_commit_exception(struct dm_exception_store
*store
,
621 struct dm_exception
*e
,
622 void (*callback
) (void *, int success
),
623 void *callback_context
)
626 struct pstore
*ps
= get_info(store
);
627 struct disk_exception de
;
628 struct commit_callback
*cb
;
630 de
.old_chunk
= e
->old_chunk
;
631 de
.new_chunk
= e
->new_chunk
;
632 write_exception(ps
, ps
->current_committed
++, &de
);
635 * Add the callback to the back of the array. This code
636 * is the only place where the callback array is
637 * manipulated, and we know that it will never be called
638 * multiple times concurrently.
640 cb
= ps
->callbacks
+ ps
->callback_count
++;
641 cb
->callback
= callback
;
642 cb
->context
= callback_context
;
645 * If there are exceptions in flight and we have not yet
646 * filled this metadata area there's nothing more to do.
648 if (!atomic_dec_and_test(&ps
->pending_count
) &&
649 (ps
->current_committed
!= ps
->exceptions_per_area
))
653 * If we completely filled the current area, then wipe the next one.
655 if ((ps
->current_committed
== ps
->exceptions_per_area
) &&
656 zero_disk_area(ps
, ps
->current_area
+ 1))
660 * Commit exceptions to disk.
662 if (ps
->valid
&& area_io(ps
, WRITE_BARRIER
))
666 * Advance to the next area if this one is full.
668 if (ps
->current_committed
== ps
->exceptions_per_area
) {
669 ps
->current_committed
= 0;
671 zero_memory_area(ps
);
674 for (i
= 0; i
< ps
->callback_count
; i
++) {
675 cb
= ps
->callbacks
+ i
;
676 cb
->callback(cb
->context
, ps
->valid
);
679 ps
->callback_count
= 0;
682 static void persistent_drop_snapshot(struct dm_exception_store
*store
)
684 struct pstore
*ps
= get_info(store
);
687 if (write_header(ps
))
688 DMWARN("write header failed");
691 static int persistent_ctr(struct dm_exception_store
*store
,
692 unsigned argc
, char **argv
)
696 /* allocate the pstore */
697 ps
= kzalloc(sizeof(*ps
), GFP_KERNEL
);
703 ps
->version
= SNAPSHOT_DISK_VERSION
;
705 ps
->zero_area
= NULL
;
706 ps
->header_area
= NULL
;
707 ps
->next_free
= 2; /* skipping the header and first area */
708 ps
->current_committed
= 0;
710 ps
->callback_count
= 0;
711 atomic_set(&ps
->pending_count
, 0);
712 ps
->callbacks
= NULL
;
714 ps
->metadata_wq
= create_singlethread_workqueue("ksnaphd");
715 if (!ps
->metadata_wq
) {
717 DMERR("couldn't start header metadata update thread");
726 static unsigned persistent_status(struct dm_exception_store
*store
,
727 status_type_t status
, char *result
,
733 case STATUSTYPE_INFO
:
735 case STATUSTYPE_TABLE
:
736 DMEMIT(" %s P %llu", store
->cow
->name
,
737 (unsigned long long)store
->chunk_size
);
743 static struct dm_exception_store_type _persistent_type
= {
744 .name
= "persistent",
745 .module
= THIS_MODULE
,
746 .ctr
= persistent_ctr
,
747 .dtr
= persistent_dtr
,
748 .read_metadata
= persistent_read_metadata
,
749 .prepare_exception
= persistent_prepare_exception
,
750 .commit_exception
= persistent_commit_exception
,
751 .drop_snapshot
= persistent_drop_snapshot
,
752 .usage
= persistent_usage
,
753 .status
= persistent_status
,
756 static struct dm_exception_store_type _persistent_compat_type
= {
758 .module
= THIS_MODULE
,
759 .ctr
= persistent_ctr
,
760 .dtr
= persistent_dtr
,
761 .read_metadata
= persistent_read_metadata
,
762 .prepare_exception
= persistent_prepare_exception
,
763 .commit_exception
= persistent_commit_exception
,
764 .drop_snapshot
= persistent_drop_snapshot
,
765 .usage
= persistent_usage
,
766 .status
= persistent_status
,
769 int dm_persistent_snapshot_init(void)
773 r
= dm_exception_store_type_register(&_persistent_type
);
775 DMERR("Unable to register persistent exception store type");
779 r
= dm_exception_store_type_register(&_persistent_compat_type
);
781 DMERR("Unable to register old-style persistent exception "
783 dm_exception_store_type_unregister(&_persistent_type
);
790 void dm_persistent_snapshot_exit(void)
792 dm_exception_store_type_unregister(&_persistent_type
);
793 dm_exception_store_type_unregister(&_persistent_compat_type
);