2 * Copyright (C) 2003 Sistina Software Limited.
3 * Copyright (C) 2005-2008 Red Hat, Inc. All rights reserved.
5 * This file is released under the GPL.
8 #include "dm-bio-record.h"
10 #include <linux/init.h>
11 #include <linux/mempool.h>
12 #include <linux/module.h>
13 #include <linux/pagemap.h>
14 #include <linux/slab.h>
15 #include <linux/workqueue.h>
16 #include <linux/device-mapper.h>
17 #include <linux/dm-io.h>
18 #include <linux/dm-dirty-log.h>
19 #include <linux/dm-kcopyd.h>
20 #include <linux/dm-region-hash.h>
22 #define DM_MSG_PREFIX "raid1"
24 #define MAX_RECOVERY 1 /* Maximum number of regions recovered in parallel. */
26 #define DM_RAID1_HANDLE_ERRORS 0x01
27 #define errors_handled(p) ((p)->features & DM_RAID1_HANDLE_ERRORS)
29 static DECLARE_WAIT_QUEUE_HEAD(_kmirrord_recovery_stopped
);
31 /*-----------------------------------------------------------------
32 * Mirror set structures.
33 *---------------------------------------------------------------*/
42 struct mirror_set
*ms
;
44 unsigned long error_type
;
51 struct list_head list
;
55 spinlock_t lock
; /* protects the lists */
56 struct bio_list reads
;
57 struct bio_list writes
;
58 struct bio_list failures
;
59 struct bio_list holds
; /* bios are waiting until suspend */
61 struct dm_region_hash
*rh
;
62 struct dm_kcopyd_client
*kcopyd_client
;
63 struct dm_io_client
*io_client
;
64 mempool_t
*read_record_pool
;
73 atomic_t default_mirror
; /* Default mirror */
75 struct workqueue_struct
*kmirrord_wq
;
76 struct work_struct kmirrord_work
;
77 struct timer_list timer
;
78 unsigned long timer_pending
;
80 struct work_struct trigger_event
;
83 struct mirror mirror
[0];
86 static void wakeup_mirrord(void *context
)
88 struct mirror_set
*ms
= context
;
90 queue_work(ms
->kmirrord_wq
, &ms
->kmirrord_work
);
93 static void delayed_wake_fn(unsigned long data
)
95 struct mirror_set
*ms
= (struct mirror_set
*) data
;
97 clear_bit(0, &ms
->timer_pending
);
101 static void delayed_wake(struct mirror_set
*ms
)
103 if (test_and_set_bit(0, &ms
->timer_pending
))
106 ms
->timer
.expires
= jiffies
+ HZ
/ 5;
107 ms
->timer
.data
= (unsigned long) ms
;
108 ms
->timer
.function
= delayed_wake_fn
;
109 add_timer(&ms
->timer
);
112 static void wakeup_all_recovery_waiters(void *context
)
114 wake_up_all(&_kmirrord_recovery_stopped
);
117 static void queue_bio(struct mirror_set
*ms
, struct bio
*bio
, int rw
)
123 bl
= (rw
== WRITE
) ? &ms
->writes
: &ms
->reads
;
124 spin_lock_irqsave(&ms
->lock
, flags
);
125 should_wake
= !(bl
->head
);
126 bio_list_add(bl
, bio
);
127 spin_unlock_irqrestore(&ms
->lock
, flags
);
133 static void dispatch_bios(void *context
, struct bio_list
*bio_list
)
135 struct mirror_set
*ms
= context
;
138 while ((bio
= bio_list_pop(bio_list
)))
139 queue_bio(ms
, bio
, WRITE
);
142 #define MIN_READ_RECORDS 20
143 struct dm_raid1_read_record
{
145 struct dm_bio_details details
;
148 static struct kmem_cache
*_dm_raid1_read_record_cache
;
151 * Every mirror should look like this one.
153 #define DEFAULT_MIRROR 0
156 * This is yucky. We squirrel the mirror struct away inside
157 * bi_next for read/write buffers. This is safe since the bh
158 * doesn't get submitted to the lower levels of block layer.
160 static struct mirror
*bio_get_m(struct bio
*bio
)
162 return (struct mirror
*) bio
->bi_next
;
165 static void bio_set_m(struct bio
*bio
, struct mirror
*m
)
167 bio
->bi_next
= (struct bio
*) m
;
170 static struct mirror
*get_default_mirror(struct mirror_set
*ms
)
172 return &ms
->mirror
[atomic_read(&ms
->default_mirror
)];
175 static void set_default_mirror(struct mirror
*m
)
177 struct mirror_set
*ms
= m
->ms
;
178 struct mirror
*m0
= &(ms
->mirror
[0]);
180 atomic_set(&ms
->default_mirror
, m
- m0
);
183 static struct mirror
*get_valid_mirror(struct mirror_set
*ms
)
187 for (m
= ms
->mirror
; m
< ms
->mirror
+ ms
->nr_mirrors
; m
++)
188 if (!atomic_read(&m
->error_count
))
195 * @m: mirror device to fail
196 * @error_type: one of the enum's, DM_RAID1_*_ERROR
198 * If errors are being handled, record the type of
199 * error encountered for this device. If this type
200 * of error has already been recorded, we can return;
201 * otherwise, we must signal userspace by triggering
202 * an event. Additionally, if the device is the
203 * primary device, we must choose a new primary, but
204 * only if the mirror is in-sync.
206 * This function must not block.
208 static void fail_mirror(struct mirror
*m
, enum dm_raid1_error error_type
)
210 struct mirror_set
*ms
= m
->ms
;
216 * error_count is used for nothing more than a
217 * simple way to tell if a device has encountered
220 atomic_inc(&m
->error_count
);
222 if (test_and_set_bit(error_type
, &m
->error_type
))
225 if (!errors_handled(ms
))
228 if (m
!= get_default_mirror(ms
))
233 * Better to issue requests to same failing device
234 * than to risk returning corrupt data.
236 DMERR("Primary mirror (%s) failed while out-of-sync: "
237 "Reads may fail.", m
->dev
->name
);
241 new = get_valid_mirror(ms
);
243 set_default_mirror(new);
245 DMWARN("All sides of mirror have failed.");
248 schedule_work(&ms
->trigger_event
);
251 static int mirror_flush(struct dm_target
*ti
)
253 struct mirror_set
*ms
= ti
->private;
254 unsigned long error_bits
;
257 struct dm_io_region io
[ms
->nr_mirrors
];
259 struct dm_io_request io_req
= {
260 .bi_rw
= WRITE_FLUSH
,
261 .mem
.type
= DM_IO_KMEM
,
262 .mem
.ptr
.addr
= NULL
,
263 .client
= ms
->io_client
,
266 for (i
= 0, m
= ms
->mirror
; i
< ms
->nr_mirrors
; i
++, m
++) {
267 io
[i
].bdev
= m
->dev
->bdev
;
273 dm_io(&io_req
, ms
->nr_mirrors
, io
, &error_bits
);
274 if (unlikely(error_bits
!= 0)) {
275 for (i
= 0; i
< ms
->nr_mirrors
; i
++)
276 if (test_bit(i
, &error_bits
))
277 fail_mirror(ms
->mirror
+ i
,
278 DM_RAID1_FLUSH_ERROR
);
285 /*-----------------------------------------------------------------
288 * When a mirror is first activated we may find that some regions
289 * are in the no-sync state. We have to recover these by
290 * recopying from the default mirror to all the others.
291 *---------------------------------------------------------------*/
292 static void recovery_complete(int read_err
, unsigned long write_err
,
295 struct dm_region
*reg
= context
;
296 struct mirror_set
*ms
= dm_rh_region_context(reg
);
300 /* Read error means the failure of default mirror. */
301 DMERR_LIMIT("Unable to read primary mirror during recovery");
302 fail_mirror(get_default_mirror(ms
), DM_RAID1_SYNC_ERROR
);
306 DMERR_LIMIT("Write error during recovery (error = 0x%lx)",
309 * Bits correspond to devices (excluding default mirror).
310 * The default mirror cannot change during recovery.
312 for (m
= 0; m
< ms
->nr_mirrors
; m
++) {
313 if (&ms
->mirror
[m
] == get_default_mirror(ms
))
315 if (test_bit(bit
, &write_err
))
316 fail_mirror(ms
->mirror
+ m
,
317 DM_RAID1_SYNC_ERROR
);
322 dm_rh_recovery_end(reg
, !(read_err
|| write_err
));
325 static int recover(struct mirror_set
*ms
, struct dm_region
*reg
)
329 struct dm_io_region from
, to
[DM_KCOPYD_MAX_REGIONS
], *dest
;
331 unsigned long flags
= 0;
332 region_t key
= dm_rh_get_region_key(reg
);
333 sector_t region_size
= dm_rh_get_region_size(ms
->rh
);
335 /* fill in the source */
336 m
= get_default_mirror(ms
);
337 from
.bdev
= m
->dev
->bdev
;
338 from
.sector
= m
->offset
+ dm_rh_region_to_sector(ms
->rh
, key
);
339 if (key
== (ms
->nr_regions
- 1)) {
341 * The final region may be smaller than
344 from
.count
= ms
->ti
->len
& (region_size
- 1);
346 from
.count
= region_size
;
348 from
.count
= region_size
;
350 /* fill in the destinations */
351 for (i
= 0, dest
= to
; i
< ms
->nr_mirrors
; i
++) {
352 if (&ms
->mirror
[i
] == get_default_mirror(ms
))
356 dest
->bdev
= m
->dev
->bdev
;
357 dest
->sector
= m
->offset
+ dm_rh_region_to_sector(ms
->rh
, key
);
358 dest
->count
= from
.count
;
363 if (!errors_handled(ms
))
364 set_bit(DM_KCOPYD_IGNORE_ERROR
, &flags
);
366 r
= dm_kcopyd_copy(ms
->kcopyd_client
, &from
, ms
->nr_mirrors
- 1, to
,
367 flags
, recovery_complete
, reg
);
372 static void do_recovery(struct mirror_set
*ms
)
374 struct dm_region
*reg
;
375 struct dm_dirty_log
*log
= dm_rh_dirty_log(ms
->rh
);
379 * Start quiescing some regions.
381 dm_rh_recovery_prepare(ms
->rh
);
384 * Copy any already quiesced regions.
386 while ((reg
= dm_rh_recovery_start(ms
->rh
))) {
387 r
= recover(ms
, reg
);
389 dm_rh_recovery_end(reg
, 0);
393 * Update the in sync flag.
396 (log
->type
->get_sync_count(log
) == ms
->nr_regions
)) {
397 /* the sync is complete */
398 dm_table_event(ms
->ti
->table
);
403 /*-----------------------------------------------------------------
405 *---------------------------------------------------------------*/
406 static struct mirror
*choose_mirror(struct mirror_set
*ms
, sector_t sector
)
408 struct mirror
*m
= get_default_mirror(ms
);
411 if (likely(!atomic_read(&m
->error_count
)))
414 if (m
-- == ms
->mirror
)
416 } while (m
!= get_default_mirror(ms
));
421 static int default_ok(struct mirror
*m
)
423 struct mirror
*default_mirror
= get_default_mirror(m
->ms
);
425 return !atomic_read(&default_mirror
->error_count
);
428 static int mirror_available(struct mirror_set
*ms
, struct bio
*bio
)
430 struct dm_dirty_log
*log
= dm_rh_dirty_log(ms
->rh
);
431 region_t region
= dm_rh_bio_to_region(ms
->rh
, bio
);
433 if (log
->type
->in_sync(log
, region
, 0))
434 return choose_mirror(ms
, bio
->bi_sector
) ? 1 : 0;
440 * remap a buffer to a particular mirror.
442 static sector_t
map_sector(struct mirror
*m
, struct bio
*bio
)
444 if (unlikely(!bio
->bi_size
))
446 return m
->offset
+ dm_target_offset(m
->ms
->ti
, bio
->bi_sector
);
449 static void map_bio(struct mirror
*m
, struct bio
*bio
)
451 bio
->bi_bdev
= m
->dev
->bdev
;
452 bio
->bi_sector
= map_sector(m
, bio
);
455 static void map_region(struct dm_io_region
*io
, struct mirror
*m
,
458 io
->bdev
= m
->dev
->bdev
;
459 io
->sector
= map_sector(m
, bio
);
460 io
->count
= bio
->bi_size
>> 9;
463 static void hold_bio(struct mirror_set
*ms
, struct bio
*bio
)
466 * Lock is required to avoid race condition during suspend
469 spin_lock_irq(&ms
->lock
);
471 if (atomic_read(&ms
->suspend
)) {
472 spin_unlock_irq(&ms
->lock
);
475 * If device is suspended, complete the bio.
477 if (dm_noflush_suspending(ms
->ti
))
478 bio_endio(bio
, DM_ENDIO_REQUEUE
);
480 bio_endio(bio
, -EIO
);
485 * Hold bio until the suspend is complete.
487 bio_list_add(&ms
->holds
, bio
);
488 spin_unlock_irq(&ms
->lock
);
491 /*-----------------------------------------------------------------
493 *---------------------------------------------------------------*/
494 static void read_callback(unsigned long error
, void *context
)
496 struct bio
*bio
= context
;
500 bio_set_m(bio
, NULL
);
502 if (likely(!error
)) {
507 fail_mirror(m
, DM_RAID1_READ_ERROR
);
509 if (likely(default_ok(m
)) || mirror_available(m
->ms
, bio
)) {
510 DMWARN_LIMIT("Read failure on mirror device %s. "
511 "Trying alternative device.",
513 queue_bio(m
->ms
, bio
, bio_rw(bio
));
517 DMERR_LIMIT("Read failure on mirror device %s. Failing I/O.",
519 bio_endio(bio
, -EIO
);
522 /* Asynchronous read. */
523 static void read_async_bio(struct mirror
*m
, struct bio
*bio
)
525 struct dm_io_region io
;
526 struct dm_io_request io_req
= {
528 .mem
.type
= DM_IO_BVEC
,
529 .mem
.ptr
.bvec
= bio
->bi_io_vec
+ bio
->bi_idx
,
530 .notify
.fn
= read_callback
,
531 .notify
.context
= bio
,
532 .client
= m
->ms
->io_client
,
535 map_region(&io
, m
, bio
);
537 BUG_ON(dm_io(&io_req
, 1, &io
, NULL
));
540 static inline int region_in_sync(struct mirror_set
*ms
, region_t region
,
543 int state
= dm_rh_get_state(ms
->rh
, region
, may_block
);
544 return state
== DM_RH_CLEAN
|| state
== DM_RH_DIRTY
;
547 static void do_reads(struct mirror_set
*ms
, struct bio_list
*reads
)
553 while ((bio
= bio_list_pop(reads
))) {
554 region
= dm_rh_bio_to_region(ms
->rh
, bio
);
555 m
= get_default_mirror(ms
);
558 * We can only read balance if the region is in sync.
560 if (likely(region_in_sync(ms
, region
, 1)))
561 m
= choose_mirror(ms
, bio
->bi_sector
);
562 else if (m
&& atomic_read(&m
->error_count
))
566 read_async_bio(m
, bio
);
568 bio_endio(bio
, -EIO
);
572 /*-----------------------------------------------------------------
575 * We do different things with the write io depending on the
576 * state of the region that it's in:
578 * SYNC: increment pending, use kcopyd to write to *all* mirrors
579 * RECOVERING: delay the io until recovery completes
580 * NOSYNC: increment pending, just write to the default mirror
581 *---------------------------------------------------------------*/
584 static void write_callback(unsigned long error
, void *context
)
587 struct bio
*bio
= (struct bio
*) context
;
588 struct mirror_set
*ms
;
592 ms
= bio_get_m(bio
)->ms
;
593 bio_set_m(bio
, NULL
);
596 * NOTE: We don't decrement the pending count here,
597 * instead it is done by the targets endio function.
598 * This way we handle both writes to SYNC and NOSYNC
599 * regions with the same code.
601 if (likely(!error
)) {
606 for (i
= 0; i
< ms
->nr_mirrors
; i
++)
607 if (test_bit(i
, &error
))
608 fail_mirror(ms
->mirror
+ i
, DM_RAID1_WRITE_ERROR
);
611 * Need to raise event. Since raising
612 * events can block, we need to do it in
615 spin_lock_irqsave(&ms
->lock
, flags
);
616 if (!ms
->failures
.head
)
618 bio_list_add(&ms
->failures
, bio
);
619 spin_unlock_irqrestore(&ms
->lock
, flags
);
624 static void do_write(struct mirror_set
*ms
, struct bio
*bio
)
627 struct dm_io_region io
[ms
->nr_mirrors
], *dest
= io
;
629 struct dm_io_request io_req
= {
630 .bi_rw
= WRITE
| (bio
->bi_rw
& WRITE_FLUSH_FUA
),
631 .mem
.type
= DM_IO_BVEC
,
632 .mem
.ptr
.bvec
= bio
->bi_io_vec
+ bio
->bi_idx
,
633 .notify
.fn
= write_callback
,
634 .notify
.context
= bio
,
635 .client
= ms
->io_client
,
638 if (bio
->bi_rw
& REQ_DISCARD
) {
639 io_req
.bi_rw
|= REQ_DISCARD
;
640 io_req
.mem
.type
= DM_IO_KMEM
;
641 io_req
.mem
.ptr
.addr
= NULL
;
644 for (i
= 0, m
= ms
->mirror
; i
< ms
->nr_mirrors
; i
++, m
++)
645 map_region(dest
++, m
, bio
);
648 * Use default mirror because we only need it to retrieve the reference
649 * to the mirror set in write_callback().
651 bio_set_m(bio
, get_default_mirror(ms
));
653 BUG_ON(dm_io(&io_req
, ms
->nr_mirrors
, io
, NULL
));
656 static void do_writes(struct mirror_set
*ms
, struct bio_list
*writes
)
660 struct bio_list sync
, nosync
, recover
, *this_list
= NULL
;
661 struct bio_list requeue
;
662 struct dm_dirty_log
*log
= dm_rh_dirty_log(ms
->rh
);
669 * Classify each write.
671 bio_list_init(&sync
);
672 bio_list_init(&nosync
);
673 bio_list_init(&recover
);
674 bio_list_init(&requeue
);
676 while ((bio
= bio_list_pop(writes
))) {
677 if ((bio
->bi_rw
& REQ_FLUSH
) ||
678 (bio
->bi_rw
& REQ_DISCARD
)) {
679 bio_list_add(&sync
, bio
);
683 region
= dm_rh_bio_to_region(ms
->rh
, bio
);
685 if (log
->type
->is_remote_recovering
&&
686 log
->type
->is_remote_recovering(log
, region
)) {
687 bio_list_add(&requeue
, bio
);
691 state
= dm_rh_get_state(ms
->rh
, region
, 1);
702 case DM_RH_RECOVERING
:
703 this_list
= &recover
;
707 bio_list_add(this_list
, bio
);
711 * Add bios that are delayed due to remote recovery
712 * back on to the write queue
714 if (unlikely(requeue
.head
)) {
715 spin_lock_irq(&ms
->lock
);
716 bio_list_merge(&ms
->writes
, &requeue
);
717 spin_unlock_irq(&ms
->lock
);
722 * Increment the pending counts for any regions that will
723 * be written to (writes to recover regions are going to
726 dm_rh_inc_pending(ms
->rh
, &sync
);
727 dm_rh_inc_pending(ms
->rh
, &nosync
);
730 * If the flush fails on a previous call and succeeds here,
731 * we must not reset the log_failure variable. We need
732 * userspace interaction to do that.
734 ms
->log_failure
= dm_rh_flush(ms
->rh
) ? 1 : ms
->log_failure
;
739 if (unlikely(ms
->log_failure
) && errors_handled(ms
)) {
740 spin_lock_irq(&ms
->lock
);
741 bio_list_merge(&ms
->failures
, &sync
);
742 spin_unlock_irq(&ms
->lock
);
745 while ((bio
= bio_list_pop(&sync
)))
748 while ((bio
= bio_list_pop(&recover
)))
749 dm_rh_delay(ms
->rh
, bio
);
751 while ((bio
= bio_list_pop(&nosync
))) {
752 if (unlikely(ms
->leg_failure
) && errors_handled(ms
)) {
753 spin_lock_irq(&ms
->lock
);
754 bio_list_add(&ms
->failures
, bio
);
755 spin_unlock_irq(&ms
->lock
);
758 map_bio(get_default_mirror(ms
), bio
);
759 generic_make_request(bio
);
764 static void do_failures(struct mirror_set
*ms
, struct bio_list
*failures
)
768 if (likely(!failures
->head
))
772 * If the log has failed, unattempted writes are being
773 * put on the holds list. We can't issue those writes
774 * until a log has been marked, so we must store them.
776 * If a 'noflush' suspend is in progress, we can requeue
777 * the I/O's to the core. This give userspace a chance
778 * to reconfigure the mirror, at which point the core
779 * will reissue the writes. If the 'noflush' flag is
780 * not set, we have no choice but to return errors.
782 * Some writes on the failures list may have been
783 * submitted before the log failure and represent a
784 * failure to write to one of the devices. It is ok
785 * for us to treat them the same and requeue them
788 while ((bio
= bio_list_pop(failures
))) {
789 if (!ms
->log_failure
) {
791 dm_rh_mark_nosync(ms
->rh
, bio
);
795 * If all the legs are dead, fail the I/O.
796 * If we have been told to handle errors, hold the bio
797 * and wait for userspace to deal with the problem.
798 * Otherwise pretend that the I/O succeeded. (This would
799 * be wrong if the failed leg returned after reboot and
800 * got replicated back to the good legs.)
802 if (!get_valid_mirror(ms
))
803 bio_endio(bio
, -EIO
);
804 else if (errors_handled(ms
))
811 static void trigger_event(struct work_struct
*work
)
813 struct mirror_set
*ms
=
814 container_of(work
, struct mirror_set
, trigger_event
);
816 dm_table_event(ms
->ti
->table
);
819 /*-----------------------------------------------------------------
821 *---------------------------------------------------------------*/
822 static void do_mirror(struct work_struct
*work
)
824 struct mirror_set
*ms
= container_of(work
, struct mirror_set
,
826 struct bio_list reads
, writes
, failures
;
829 spin_lock_irqsave(&ms
->lock
, flags
);
832 failures
= ms
->failures
;
833 bio_list_init(&ms
->reads
);
834 bio_list_init(&ms
->writes
);
835 bio_list_init(&ms
->failures
);
836 spin_unlock_irqrestore(&ms
->lock
, flags
);
838 dm_rh_update_states(ms
->rh
, errors_handled(ms
));
840 do_reads(ms
, &reads
);
841 do_writes(ms
, &writes
);
842 do_failures(ms
, &failures
);
845 /*-----------------------------------------------------------------
847 *---------------------------------------------------------------*/
848 static struct mirror_set
*alloc_context(unsigned int nr_mirrors
,
849 uint32_t region_size
,
850 struct dm_target
*ti
,
851 struct dm_dirty_log
*dl
)
854 struct mirror_set
*ms
= NULL
;
856 len
= sizeof(*ms
) + (sizeof(ms
->mirror
[0]) * nr_mirrors
);
858 ms
= kzalloc(len
, GFP_KERNEL
);
860 ti
->error
= "Cannot allocate mirror context";
864 spin_lock_init(&ms
->lock
);
865 bio_list_init(&ms
->reads
);
866 bio_list_init(&ms
->writes
);
867 bio_list_init(&ms
->failures
);
868 bio_list_init(&ms
->holds
);
871 ms
->nr_mirrors
= nr_mirrors
;
872 ms
->nr_regions
= dm_sector_div_up(ti
->len
, region_size
);
876 atomic_set(&ms
->suspend
, 0);
877 atomic_set(&ms
->default_mirror
, DEFAULT_MIRROR
);
879 ms
->read_record_pool
= mempool_create_slab_pool(MIN_READ_RECORDS
,
880 _dm_raid1_read_record_cache
);
882 if (!ms
->read_record_pool
) {
883 ti
->error
= "Error creating mirror read_record_pool";
888 ms
->io_client
= dm_io_client_create();
889 if (IS_ERR(ms
->io_client
)) {
890 ti
->error
= "Error creating dm_io client";
891 mempool_destroy(ms
->read_record_pool
);
896 ms
->rh
= dm_region_hash_create(ms
, dispatch_bios
, wakeup_mirrord
,
897 wakeup_all_recovery_waiters
,
898 ms
->ti
->begin
, MAX_RECOVERY
,
899 dl
, region_size
, ms
->nr_regions
);
900 if (IS_ERR(ms
->rh
)) {
901 ti
->error
= "Error creating dirty region hash";
902 dm_io_client_destroy(ms
->io_client
);
903 mempool_destroy(ms
->read_record_pool
);
911 static void free_context(struct mirror_set
*ms
, struct dm_target
*ti
,
915 dm_put_device(ti
, ms
->mirror
[m
].dev
);
917 dm_io_client_destroy(ms
->io_client
);
918 dm_region_hash_destroy(ms
->rh
);
919 mempool_destroy(ms
->read_record_pool
);
923 static int get_mirror(struct mirror_set
*ms
, struct dm_target
*ti
,
924 unsigned int mirror
, char **argv
)
926 unsigned long long offset
;
928 if (sscanf(argv
[1], "%llu", &offset
) != 1) {
929 ti
->error
= "Invalid offset";
933 if (dm_get_device(ti
, argv
[0], dm_table_get_mode(ti
->table
),
934 &ms
->mirror
[mirror
].dev
)) {
935 ti
->error
= "Device lookup failure";
939 ms
->mirror
[mirror
].ms
= ms
;
940 atomic_set(&(ms
->mirror
[mirror
].error_count
), 0);
941 ms
->mirror
[mirror
].error_type
= 0;
942 ms
->mirror
[mirror
].offset
= offset
;
948 * Create dirty log: log_type #log_params <log_params>
950 static struct dm_dirty_log
*create_dirty_log(struct dm_target
*ti
,
951 unsigned argc
, char **argv
,
954 unsigned param_count
;
955 struct dm_dirty_log
*dl
;
958 ti
->error
= "Insufficient mirror log arguments";
962 if (sscanf(argv
[1], "%u", ¶m_count
) != 1) {
963 ti
->error
= "Invalid mirror log argument count";
967 *args_used
= 2 + param_count
;
969 if (argc
< *args_used
) {
970 ti
->error
= "Insufficient mirror log arguments";
974 dl
= dm_dirty_log_create(argv
[0], ti
, mirror_flush
, param_count
,
977 ti
->error
= "Error creating mirror dirty log";
984 static int parse_features(struct mirror_set
*ms
, unsigned argc
, char **argv
,
987 unsigned num_features
;
988 struct dm_target
*ti
= ms
->ti
;
995 if (sscanf(argv
[0], "%u", &num_features
) != 1) {
996 ti
->error
= "Invalid number of features";
1004 if (num_features
> argc
) {
1005 ti
->error
= "Not enough arguments to support feature count";
1009 if (!strcmp("handle_errors", argv
[0]))
1010 ms
->features
|= DM_RAID1_HANDLE_ERRORS
;
1012 ti
->error
= "Unrecognised feature requested";
1022 * Construct a mirror mapping:
1024 * log_type #log_params <log_params>
1025 * #mirrors [mirror_path offset]{2,}
1026 * [#features <features>]
1028 * log_type is "core" or "disk"
1029 * #log_params is between 1 and 3
1031 * If present, features must be "handle_errors".
1033 static int mirror_ctr(struct dm_target
*ti
, unsigned int argc
, char **argv
)
1036 unsigned int nr_mirrors
, m
, args_used
;
1037 struct mirror_set
*ms
;
1038 struct dm_dirty_log
*dl
;
1040 dl
= create_dirty_log(ti
, argc
, argv
, &args_used
);
1047 if (!argc
|| sscanf(argv
[0], "%u", &nr_mirrors
) != 1 ||
1048 nr_mirrors
< 2 || nr_mirrors
> DM_KCOPYD_MAX_REGIONS
+ 1) {
1049 ti
->error
= "Invalid number of mirrors";
1050 dm_dirty_log_destroy(dl
);
1056 if (argc
< nr_mirrors
* 2) {
1057 ti
->error
= "Too few mirror arguments";
1058 dm_dirty_log_destroy(dl
);
1062 ms
= alloc_context(nr_mirrors
, dl
->type
->get_region_size(dl
), ti
, dl
);
1064 dm_dirty_log_destroy(dl
);
1068 /* Get the mirror parameter sets */
1069 for (m
= 0; m
< nr_mirrors
; m
++) {
1070 r
= get_mirror(ms
, ti
, m
, argv
);
1072 free_context(ms
, ti
, m
);
1080 ti
->split_io
= dm_rh_get_region_size(ms
->rh
);
1081 ti
->num_flush_requests
= 1;
1082 ti
->num_discard_requests
= 1;
1084 ms
->kmirrord_wq
= alloc_workqueue("kmirrord",
1085 WQ_NON_REENTRANT
| WQ_MEM_RECLAIM
, 0);
1086 if (!ms
->kmirrord_wq
) {
1087 DMERR("couldn't start kmirrord");
1089 goto err_free_context
;
1091 INIT_WORK(&ms
->kmirrord_work
, do_mirror
);
1092 init_timer(&ms
->timer
);
1093 ms
->timer_pending
= 0;
1094 INIT_WORK(&ms
->trigger_event
, trigger_event
);
1096 r
= parse_features(ms
, argc
, argv
, &args_used
);
1098 goto err_destroy_wq
;
1104 * Any read-balancing addition depends on the
1105 * DM_RAID1_HANDLE_ERRORS flag being present.
1106 * This is because the decision to balance depends
1107 * on the sync state of a region. If the above
1108 * flag is not present, we ignore errors; and
1109 * the sync state may be inaccurate.
1113 ti
->error
= "Too many mirror arguments";
1115 goto err_destroy_wq
;
1118 r
= dm_kcopyd_client_create(&ms
->kcopyd_client
);
1120 goto err_destroy_wq
;
1126 destroy_workqueue(ms
->kmirrord_wq
);
1128 free_context(ms
, ti
, ms
->nr_mirrors
);
1132 static void mirror_dtr(struct dm_target
*ti
)
1134 struct mirror_set
*ms
= (struct mirror_set
*) ti
->private;
1136 del_timer_sync(&ms
->timer
);
1137 flush_workqueue(ms
->kmirrord_wq
);
1138 flush_work_sync(&ms
->trigger_event
);
1139 dm_kcopyd_client_destroy(ms
->kcopyd_client
);
1140 destroy_workqueue(ms
->kmirrord_wq
);
1141 free_context(ms
, ti
, ms
->nr_mirrors
);
1145 * Mirror mapping function
1147 static int mirror_map(struct dm_target
*ti
, struct bio
*bio
,
1148 union map_info
*map_context
)
1150 int r
, rw
= bio_rw(bio
);
1152 struct mirror_set
*ms
= ti
->private;
1153 struct dm_raid1_read_record
*read_record
= NULL
;
1154 struct dm_dirty_log
*log
= dm_rh_dirty_log(ms
->rh
);
1157 /* Save region for mirror_end_io() handler */
1158 map_context
->ll
= dm_rh_bio_to_region(ms
->rh
, bio
);
1159 queue_bio(ms
, bio
, rw
);
1160 return DM_MAPIO_SUBMITTED
;
1163 r
= log
->type
->in_sync(log
, dm_rh_bio_to_region(ms
->rh
, bio
), 0);
1164 if (r
< 0 && r
!= -EWOULDBLOCK
)
1168 * If region is not in-sync queue the bio.
1170 if (!r
|| (r
== -EWOULDBLOCK
)) {
1172 return -EWOULDBLOCK
;
1174 queue_bio(ms
, bio
, rw
);
1175 return DM_MAPIO_SUBMITTED
;
1179 * The region is in-sync and we can perform reads directly.
1180 * Store enough information so we can retry if it fails.
1182 m
= choose_mirror(ms
, bio
->bi_sector
);
1186 read_record
= mempool_alloc(ms
->read_record_pool
, GFP_NOIO
);
1187 if (likely(read_record
)) {
1188 dm_bio_record(&read_record
->details
, bio
);
1189 map_context
->ptr
= read_record
;
1195 return DM_MAPIO_REMAPPED
;
1198 static int mirror_end_io(struct dm_target
*ti
, struct bio
*bio
,
1199 int error
, union map_info
*map_context
)
1201 int rw
= bio_rw(bio
);
1202 struct mirror_set
*ms
= (struct mirror_set
*) ti
->private;
1203 struct mirror
*m
= NULL
;
1204 struct dm_bio_details
*bd
= NULL
;
1205 struct dm_raid1_read_record
*read_record
= map_context
->ptr
;
1208 * We need to dec pending if this was a write.
1211 if (!(bio
->bi_rw
& REQ_FLUSH
))
1212 dm_rh_dec(ms
->rh
, map_context
->ll
);
1216 if (error
== -EOPNOTSUPP
)
1219 if ((error
== -EWOULDBLOCK
) && (bio
->bi_rw
& REQ_RAHEAD
))
1222 if (unlikely(error
)) {
1225 * There wasn't enough memory to record necessary
1226 * information for a retry or there was no other
1229 DMERR_LIMIT("Mirror read failed.");
1235 DMERR("Mirror read failed from %s. Trying alternative device.",
1238 fail_mirror(m
, DM_RAID1_READ_ERROR
);
1241 * A failed read is requeued for another attempt using an intact
1244 if (default_ok(m
) || mirror_available(ms
, bio
)) {
1245 bd
= &read_record
->details
;
1247 dm_bio_restore(bd
, bio
);
1248 mempool_free(read_record
, ms
->read_record_pool
);
1249 map_context
->ptr
= NULL
;
1250 queue_bio(ms
, bio
, rw
);
1253 DMERR("All replicated volumes dead, failing I/O");
1258 mempool_free(read_record
, ms
->read_record_pool
);
1259 map_context
->ptr
= NULL
;
1265 static void mirror_presuspend(struct dm_target
*ti
)
1267 struct mirror_set
*ms
= (struct mirror_set
*) ti
->private;
1268 struct dm_dirty_log
*log
= dm_rh_dirty_log(ms
->rh
);
1270 struct bio_list holds
;
1273 atomic_set(&ms
->suspend
, 1);
1276 * Process bios in the hold list to start recovery waiting
1277 * for bios in the hold list. After the process, no bio has
1278 * a chance to be added in the hold list because ms->suspend
1281 spin_lock_irq(&ms
->lock
);
1283 bio_list_init(&ms
->holds
);
1284 spin_unlock_irq(&ms
->lock
);
1286 while ((bio
= bio_list_pop(&holds
)))
1290 * We must finish up all the work that we've
1291 * generated (i.e. recovery work).
1293 dm_rh_stop_recovery(ms
->rh
);
1295 wait_event(_kmirrord_recovery_stopped
,
1296 !dm_rh_recovery_in_flight(ms
->rh
));
1298 if (log
->type
->presuspend
&& log
->type
->presuspend(log
))
1299 /* FIXME: need better error handling */
1300 DMWARN("log presuspend failed");
1303 * Now that recovery is complete/stopped and the
1304 * delayed bios are queued, we need to wait for
1305 * the worker thread to complete. This way,
1306 * we know that all of our I/O has been pushed.
1308 flush_workqueue(ms
->kmirrord_wq
);
1311 static void mirror_postsuspend(struct dm_target
*ti
)
1313 struct mirror_set
*ms
= ti
->private;
1314 struct dm_dirty_log
*log
= dm_rh_dirty_log(ms
->rh
);
1316 if (log
->type
->postsuspend
&& log
->type
->postsuspend(log
))
1317 /* FIXME: need better error handling */
1318 DMWARN("log postsuspend failed");
1321 static void mirror_resume(struct dm_target
*ti
)
1323 struct mirror_set
*ms
= ti
->private;
1324 struct dm_dirty_log
*log
= dm_rh_dirty_log(ms
->rh
);
1326 atomic_set(&ms
->suspend
, 0);
1327 if (log
->type
->resume
&& log
->type
->resume(log
))
1328 /* FIXME: need better error handling */
1329 DMWARN("log resume failed");
1330 dm_rh_start_recovery(ms
->rh
);
1334 * device_status_char
1335 * @m: mirror device/leg we want the status of
1337 * We return one character representing the most severe error
1338 * we have encountered.
1339 * A => Alive - No failures
1340 * D => Dead - A write failure occurred leaving mirror out-of-sync
1341 * S => Sync - A sychronization failure occurred, mirror out-of-sync
1342 * R => Read - A read failure occurred, mirror data unaffected
1346 static char device_status_char(struct mirror
*m
)
1348 if (!atomic_read(&(m
->error_count
)))
1351 return (test_bit(DM_RAID1_FLUSH_ERROR
, &(m
->error_type
))) ? 'F' :
1352 (test_bit(DM_RAID1_WRITE_ERROR
, &(m
->error_type
))) ? 'D' :
1353 (test_bit(DM_RAID1_SYNC_ERROR
, &(m
->error_type
))) ? 'S' :
1354 (test_bit(DM_RAID1_READ_ERROR
, &(m
->error_type
))) ? 'R' : 'U';
1358 static int mirror_status(struct dm_target
*ti
, status_type_t type
,
1359 char *result
, unsigned int maxlen
)
1361 unsigned int m
, sz
= 0;
1362 struct mirror_set
*ms
= (struct mirror_set
*) ti
->private;
1363 struct dm_dirty_log
*log
= dm_rh_dirty_log(ms
->rh
);
1364 char buffer
[ms
->nr_mirrors
+ 1];
1367 case STATUSTYPE_INFO
:
1368 DMEMIT("%d ", ms
->nr_mirrors
);
1369 for (m
= 0; m
< ms
->nr_mirrors
; m
++) {
1370 DMEMIT("%s ", ms
->mirror
[m
].dev
->name
);
1371 buffer
[m
] = device_status_char(&(ms
->mirror
[m
]));
1375 DMEMIT("%llu/%llu 1 %s ",
1376 (unsigned long long)log
->type
->get_sync_count(log
),
1377 (unsigned long long)ms
->nr_regions
, buffer
);
1379 sz
+= log
->type
->status(log
, type
, result
+sz
, maxlen
-sz
);
1383 case STATUSTYPE_TABLE
:
1384 sz
= log
->type
->status(log
, type
, result
, maxlen
);
1386 DMEMIT("%d", ms
->nr_mirrors
);
1387 for (m
= 0; m
< ms
->nr_mirrors
; m
++)
1388 DMEMIT(" %s %llu", ms
->mirror
[m
].dev
->name
,
1389 (unsigned long long)ms
->mirror
[m
].offset
);
1391 if (ms
->features
& DM_RAID1_HANDLE_ERRORS
)
1392 DMEMIT(" 1 handle_errors");
1398 static int mirror_iterate_devices(struct dm_target
*ti
,
1399 iterate_devices_callout_fn fn
, void *data
)
1401 struct mirror_set
*ms
= ti
->private;
1405 for (i
= 0; !ret
&& i
< ms
->nr_mirrors
; i
++)
1406 ret
= fn(ti
, ms
->mirror
[i
].dev
,
1407 ms
->mirror
[i
].offset
, ti
->len
, data
);
1412 static struct target_type mirror_target
= {
1414 .version
= {1, 12, 1},
1415 .module
= THIS_MODULE
,
1419 .end_io
= mirror_end_io
,
1420 .presuspend
= mirror_presuspend
,
1421 .postsuspend
= mirror_postsuspend
,
1422 .resume
= mirror_resume
,
1423 .status
= mirror_status
,
1424 .iterate_devices
= mirror_iterate_devices
,
1427 static int __init
dm_mirror_init(void)
1431 _dm_raid1_read_record_cache
= KMEM_CACHE(dm_raid1_read_record
, 0);
1432 if (!_dm_raid1_read_record_cache
) {
1433 DMERR("Can't allocate dm_raid1_read_record cache");
1438 r
= dm_register_target(&mirror_target
);
1440 DMERR("Failed to register mirror target");
1447 kmem_cache_destroy(_dm_raid1_read_record_cache
);
1452 static void __exit
dm_mirror_exit(void)
1454 dm_unregister_target(&mirror_target
);
1455 kmem_cache_destroy(_dm_raid1_read_record_cache
);
1459 module_init(dm_mirror_init
);
1460 module_exit(dm_mirror_exit
);
1462 MODULE_DESCRIPTION(DM_NAME
" mirror target");
1463 MODULE_AUTHOR("Joe Thornber");
1464 MODULE_LICENSE("GPL");