4 This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
6 Copyright (C) 2003-2008, LINBIT Information Technologies GmbH.
7 Copyright (C) 2003-2008, Philipp Reisner <philipp.reisner@linbit.com>.
8 Copyright (C) 2003-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.
10 drbd is free software; you can redistribute it and/or modify
11 it under the terms of the GNU General Public License as published by
12 the Free Software Foundation; either version 2, or (at your option)
15 drbd is distributed in the hope that it will be useful,
16 but WITHOUT ANY WARRANTY; without even the implied warranty of
17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 GNU General Public License for more details.
20 You should have received a copy of the GNU General Public License
21 along with drbd; see the file COPYING. If not, write to
22 the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
26 #include <linux/slab.h>
27 #include <linux/crc32c.h>
28 #include <linux/drbd.h>
29 #include <linux/drbd_limits.h>
30 #include <linux/dynamic_debug.h>
32 #include "drbd_wrappers.h"
35 enum al_transaction_types
{
37 AL_TR_INITIALIZED
= 0xffff
39 /* all fields on disc in big endian */
40 struct __packed al_transaction_on_disk
{
41 /* don't we all like magic */
44 /* to identify the most recent transaction block
45 * in the on disk ring buffer */
48 /* checksum on the full 4k block, with this field set to 0. */
51 /* type of transaction, special transaction types like:
52 * purge-all, set-all-idle, set-all-active, ... to-be-defined
53 * see also enum al_transaction_types */
54 __be16 transaction_type
;
56 /* we currently allow only a few thousand extents,
57 * so 16bit will be enough for the slot number. */
59 /* how many updates in this transaction */
62 /* maximum slot number, "al-extents" in drbd.conf speak.
63 * Having this in each transaction should make reconfiguration
64 * of that parameter easier. */
67 /* slot number the context starts with */
68 __be16 context_start_slot_nr
;
70 /* Some reserved bytes. Expected usage is a 64bit counter of
71 * sectors-written since device creation, and other data generation tag
75 /* --- 36 byte used --- */
77 /* Reserve space for up to AL_UPDATES_PER_TRANSACTION changes
78 * in one transaction, then use the remaining byte in the 4k block for
79 * context information. "Flexible" number of updates per transaction
80 * does not help, as we have to account for the case when all update
81 * slots are used anyways, so it would only complicate code without
84 __be16 update_slot_nr
[AL_UPDATES_PER_TRANSACTION
];
86 /* but the extent number is 32bit, which at an extent size of 4 MiB
87 * allows to cover device sizes of up to 2**54 Byte (16 PiB) */
88 __be32 update_extent_nr
[AL_UPDATES_PER_TRANSACTION
];
90 /* --- 420 bytes used (36 + 64*6) --- */
92 /* 4096 - 420 = 3676 = 919 * 4 */
93 __be32 context
[AL_CONTEXT_PER_TRANSACTION
];
96 struct update_odbm_work
{
101 struct update_al_work
{
103 struct completion event
;
107 static int al_write_transaction(struct drbd_conf
*mdev
);
109 void *drbd_md_get_buffer(struct drbd_conf
*mdev
)
113 wait_event(mdev
->misc_wait
,
114 (r
= atomic_cmpxchg(&mdev
->md_io_in_use
, 0, 1)) == 0 ||
115 mdev
->state
.disk
<= D_FAILED
);
117 return r
? NULL
: page_address(mdev
->md_io_page
);
120 void drbd_md_put_buffer(struct drbd_conf
*mdev
)
122 if (atomic_dec_and_test(&mdev
->md_io_in_use
))
123 wake_up(&mdev
->misc_wait
);
126 static bool md_io_allowed(struct drbd_conf
*mdev
)
128 enum drbd_disk_state ds
= mdev
->state
.disk
;
129 return ds
>= D_NEGOTIATING
|| ds
== D_ATTACHING
;
132 void wait_until_done_or_disk_failure(struct drbd_conf
*mdev
, unsigned int *done
)
134 wait_event(mdev
->misc_wait
, *done
|| !md_io_allowed(mdev
));
137 static int _drbd_md_sync_page_io(struct drbd_conf
*mdev
,
138 struct drbd_backing_dev
*bdev
,
139 struct page
*page
, sector_t sector
,
145 mdev
->md_io
.done
= 0;
146 mdev
->md_io
.error
= -ENODEV
;
148 if ((rw
& WRITE
) && !test_bit(MD_NO_FUA
, &mdev
->flags
))
149 rw
|= REQ_FUA
| REQ_FLUSH
;
152 bio
= bio_alloc_drbd(GFP_NOIO
);
153 bio
->bi_bdev
= bdev
->md_bdev
;
154 bio
->bi_sector
= sector
;
156 if (bio_add_page(bio
, page
, size
, 0) != size
)
158 bio
->bi_private
= &mdev
->md_io
;
159 bio
->bi_end_io
= drbd_md_io_complete
;
162 if (!get_ldev_if_state(mdev
, D_ATTACHING
)) { /* Corresponding put_ldev in drbd_md_io_complete() */
163 dev_err(DEV
, "ASSERT FAILED: get_ldev_if_state() == 1 in _drbd_md_sync_page_io()\n");
168 bio_get(bio
); /* one bio_put() is in the completion handler */
169 atomic_inc(&mdev
->md_io_in_use
); /* drbd_md_put_buffer() is in the completion handler */
170 if (drbd_insert_fault(mdev
, (rw
& WRITE
) ? DRBD_FAULT_MD_WR
: DRBD_FAULT_MD_RD
))
171 bio_endio(bio
, -EIO
);
174 wait_until_done_or_disk_failure(mdev
, &mdev
->md_io
.done
);
175 if (bio_flagged(bio
, BIO_UPTODATE
))
176 err
= mdev
->md_io
.error
;
183 int drbd_md_sync_page_io(struct drbd_conf
*mdev
, struct drbd_backing_dev
*bdev
,
184 sector_t sector
, int rw
)
187 struct page
*iop
= mdev
->md_io_page
;
189 D_ASSERT(atomic_read(&mdev
->md_io_in_use
) == 1);
191 BUG_ON(!bdev
->md_bdev
);
193 dev_dbg(DEV
, "meta_data io: %s [%d]:%s(,%llus,%s)\n",
194 current
->comm
, current
->pid
, __func__
,
195 (unsigned long long)sector
, (rw
& WRITE
) ? "WRITE" : "READ");
197 if (sector
< drbd_md_first_sector(bdev
) ||
198 sector
+ 7 > drbd_md_last_sector(bdev
))
199 dev_alert(DEV
, "%s [%d]:%s(,%llus,%s) out of range md access!\n",
200 current
->comm
, current
->pid
, __func__
,
201 (unsigned long long)sector
, (rw
& WRITE
) ? "WRITE" : "READ");
203 err
= _drbd_md_sync_page_io(mdev
, bdev
, iop
, sector
, rw
, MD_BLOCK_SIZE
);
205 dev_err(DEV
, "drbd_md_sync_page_io(,%llus,%s) failed!\n",
206 (unsigned long long)sector
, (rw
& WRITE
) ? "WRITE" : "READ");
211 static struct lc_element
*_al_get(struct drbd_conf
*mdev
, unsigned int enr
)
213 struct lc_element
*al_ext
;
214 struct lc_element
*tmp
;
217 spin_lock_irq(&mdev
->al_lock
);
218 tmp
= lc_find(mdev
->resync
, enr
/AL_EXT_PER_BM_SECT
);
219 if (unlikely(tmp
!= NULL
)) {
220 struct bm_extent
*bm_ext
= lc_entry(tmp
, struct bm_extent
, lce
);
221 if (test_bit(BME_NO_WRITES
, &bm_ext
->flags
)) {
222 wake
= !test_and_set_bit(BME_PRIORITY
, &bm_ext
->flags
);
223 spin_unlock_irq(&mdev
->al_lock
);
225 wake_up(&mdev
->al_wait
);
229 al_ext
= lc_get(mdev
->act_log
, enr
);
230 spin_unlock_irq(&mdev
->al_lock
);
234 void drbd_al_begin_io(struct drbd_conf
*mdev
, struct drbd_interval
*i
)
236 /* for bios crossing activity log extent boundaries,
237 * we may need to activate two extents in one go */
238 unsigned first
= i
->sector
>> (AL_EXTENT_SHIFT
-9);
239 unsigned last
= (i
->sector
+ (i
->size
>> 9) - 1) >> (AL_EXTENT_SHIFT
-9);
244 D_ASSERT(atomic_read(&mdev
->local_cnt
) > 0);
246 for (enr
= first
; enr
<= last
; enr
++)
247 wait_event(mdev
->al_wait
, _al_get(mdev
, enr
) != NULL
);
249 /* Serialize multiple transactions.
250 * This uses test_and_set_bit, memory barrier is implicit.
252 wait_event(mdev
->al_wait
,
253 mdev
->act_log
->pending_changes
== 0 ||
254 (locked
= lc_try_lock_for_transaction(mdev
->act_log
)));
257 /* drbd_al_write_transaction(mdev,al_ext,enr);
258 * recurses into generic_make_request(), which
259 * disallows recursion, bios being serialized on the
260 * current->bio_tail list now.
261 * we have to delegate updates to the activity log
262 * to the worker thread. */
264 /* Double check: it may have been committed by someone else,
265 * while we have been waiting for the lock. */
266 if (mdev
->act_log
->pending_changes
) {
268 err
= al_write_transaction(mdev
);
271 spin_lock_irq(&mdev
->al_lock
);
274 we need an "lc_cancel" here;
276 lc_committed(mdev
->act_log
);
277 spin_unlock_irq(&mdev
->al_lock
);
279 lc_unlock(mdev
->act_log
);
280 wake_up(&mdev
->al_wait
);
284 void drbd_al_complete_io(struct drbd_conf
*mdev
, struct drbd_interval
*i
)
286 /* for bios crossing activity log extent boundaries,
287 * we may need to activate two extents in one go */
288 unsigned first
= i
->sector
>> (AL_EXTENT_SHIFT
-9);
289 unsigned last
= (i
->sector
+ (i
->size
>> 9) - 1) >> (AL_EXTENT_SHIFT
-9);
291 struct lc_element
*extent
;
295 spin_lock_irqsave(&mdev
->al_lock
, flags
);
297 for (enr
= first
; enr
<= last
; enr
++) {
298 extent
= lc_find(mdev
->act_log
, enr
);
300 dev_err(DEV
, "al_complete_io() called on inactive extent %u\n", enr
);
303 if (lc_put(mdev
->act_log
, extent
) == 0)
306 spin_unlock_irqrestore(&mdev
->al_lock
, flags
);
307 wake_up(&mdev
->al_wait
);
310 #if (PAGE_SHIFT + 3) < (AL_EXTENT_SHIFT - BM_BLOCK_SHIFT)
311 /* Currently BM_BLOCK_SHIFT, BM_EXT_SHIFT and AL_EXTENT_SHIFT
312 * are still coupled, or assume too much about their relation.
313 * Code below will not work if this is violated.
314 * Will be cleaned up with some followup patch.
319 static unsigned int al_extent_to_bm_page(unsigned int al_enr
)
324 /* al extent number to bit */
325 (AL_EXTENT_SHIFT
- BM_BLOCK_SHIFT
));
328 static unsigned int rs_extent_to_bm_page(unsigned int rs_enr
)
333 /* resync extent number to bit */
334 (BM_EXT_SHIFT
- BM_BLOCK_SHIFT
));
338 _al_write_transaction(struct drbd_conf
*mdev
)
340 struct al_transaction_on_disk
*buffer
;
341 struct lc_element
*e
;
348 if (!get_ldev(mdev
)) {
349 dev_err(DEV
, "disk is %s, cannot start al transaction\n",
350 drbd_disk_str(mdev
->state
.disk
));
354 /* The bitmap write may have failed, causing a state change. */
355 if (mdev
->state
.disk
< D_INCONSISTENT
) {
357 "disk is %s, cannot write al transaction\n",
358 drbd_disk_str(mdev
->state
.disk
));
363 buffer
= drbd_md_get_buffer(mdev
); /* protects md_io_buffer, al_tr_cycle, ... */
365 dev_err(DEV
, "disk failed while waiting for md_io buffer\n");
370 memset(buffer
, 0, sizeof(*buffer
));
371 buffer
->magic
= cpu_to_be32(DRBD_AL_MAGIC
);
372 buffer
->tr_number
= cpu_to_be32(mdev
->al_tr_number
);
376 /* Even though no one can start to change this list
377 * once we set the LC_LOCKED -- from drbd_al_begin_io(),
378 * lc_try_lock_for_transaction() --, someone may still
379 * be in the process of changing it. */
380 spin_lock_irq(&mdev
->al_lock
);
381 list_for_each_entry(e
, &mdev
->act_log
->to_be_changed
, list
) {
382 if (i
== AL_UPDATES_PER_TRANSACTION
) {
386 buffer
->update_slot_nr
[i
] = cpu_to_be16(e
->lc_index
);
387 buffer
->update_extent_nr
[i
] = cpu_to_be32(e
->lc_new_number
);
388 if (e
->lc_number
!= LC_FREE
)
389 drbd_bm_mark_for_writeout(mdev
,
390 al_extent_to_bm_page(e
->lc_number
));
393 spin_unlock_irq(&mdev
->al_lock
);
394 BUG_ON(i
> AL_UPDATES_PER_TRANSACTION
);
396 buffer
->n_updates
= cpu_to_be16(i
);
397 for ( ; i
< AL_UPDATES_PER_TRANSACTION
; i
++) {
398 buffer
->update_slot_nr
[i
] = cpu_to_be16(-1);
399 buffer
->update_extent_nr
[i
] = cpu_to_be32(LC_FREE
);
402 buffer
->context_size
= cpu_to_be16(mdev
->act_log
->nr_elements
);
403 buffer
->context_start_slot_nr
= cpu_to_be16(mdev
->al_tr_cycle
);
405 mx
= min_t(int, AL_CONTEXT_PER_TRANSACTION
,
406 mdev
->act_log
->nr_elements
- mdev
->al_tr_cycle
);
407 for (i
= 0; i
< mx
; i
++) {
408 unsigned idx
= mdev
->al_tr_cycle
+ i
;
409 extent_nr
= lc_element_by_index(mdev
->act_log
, idx
)->lc_number
;
410 buffer
->context
[i
] = cpu_to_be32(extent_nr
);
412 for (; i
< AL_CONTEXT_PER_TRANSACTION
; i
++)
413 buffer
->context
[i
] = cpu_to_be32(LC_FREE
);
415 mdev
->al_tr_cycle
+= AL_CONTEXT_PER_TRANSACTION
;
416 if (mdev
->al_tr_cycle
>= mdev
->act_log
->nr_elements
)
417 mdev
->al_tr_cycle
= 0;
419 sector
= mdev
->ldev
->md
.md_offset
420 + mdev
->ldev
->md
.al_offset
421 + mdev
->al_tr_pos
* (MD_BLOCK_SIZE
>>9);
423 crc
= crc32c(0, buffer
, 4096);
424 buffer
->crc32c
= cpu_to_be32(crc
);
426 if (drbd_bm_write_hinted(mdev
))
428 /* drbd_chk_io_error done already */
429 else if (drbd_md_sync_page_io(mdev
, mdev
->ldev
, sector
, WRITE
)) {
431 drbd_chk_io_error(mdev
, 1, true);
433 /* advance ringbuffer position and transaction counter */
434 mdev
->al_tr_pos
= (mdev
->al_tr_pos
+ 1) % (MD_AL_SECTORS
*512/MD_BLOCK_SIZE
);
435 mdev
->al_tr_number
++;
438 drbd_md_put_buffer(mdev
);
445 static int w_al_write_transaction(struct drbd_work
*w
, int unused
)
447 struct update_al_work
*aw
= container_of(w
, struct update_al_work
, w
);
448 struct drbd_conf
*mdev
= w
->mdev
;
451 err
= _al_write_transaction(mdev
);
453 complete(&aw
->event
);
455 return err
!= -EIO
? err
: 0;
458 /* Calls from worker context (see w_restart_disk_io()) need to write the
459 transaction directly. Others came through generic_make_request(),
460 those need to delegate it to the worker. */
461 static int al_write_transaction(struct drbd_conf
*mdev
)
463 struct update_al_work al_work
;
465 if (current
== mdev
->tconn
->worker
.task
)
466 return _al_write_transaction(mdev
);
468 init_completion(&al_work
.event
);
469 al_work
.w
.cb
= w_al_write_transaction
;
470 al_work
.w
.mdev
= mdev
;
471 drbd_queue_work_front(&mdev
->tconn
->data
.work
, &al_work
.w
);
472 wait_for_completion(&al_work
.event
);
477 static int _try_lc_del(struct drbd_conf
*mdev
, struct lc_element
*al_ext
)
481 spin_lock_irq(&mdev
->al_lock
);
482 rv
= (al_ext
->refcnt
== 0);
484 lc_del(mdev
->act_log
, al_ext
);
485 spin_unlock_irq(&mdev
->al_lock
);
491 * drbd_al_shrink() - Removes all active extents form the activity log
492 * @mdev: DRBD device.
494 * Removes all active extents form the activity log, waiting until
495 * the reference count of each entry dropped to 0 first, of course.
497 * You need to lock mdev->act_log with lc_try_lock() / lc_unlock()
499 void drbd_al_shrink(struct drbd_conf
*mdev
)
501 struct lc_element
*al_ext
;
504 D_ASSERT(test_bit(__LC_LOCKED
, &mdev
->act_log
->flags
));
506 for (i
= 0; i
< mdev
->act_log
->nr_elements
; i
++) {
507 al_ext
= lc_element_by_index(mdev
->act_log
, i
);
508 if (al_ext
->lc_number
== LC_FREE
)
510 wait_event(mdev
->al_wait
, _try_lc_del(mdev
, al_ext
));
513 wake_up(&mdev
->al_wait
);
516 static int w_update_odbm(struct drbd_work
*w
, int unused
)
518 struct update_odbm_work
*udw
= container_of(w
, struct update_odbm_work
, w
);
519 struct drbd_conf
*mdev
= w
->mdev
;
520 struct sib_info sib
= { .sib_reason
= SIB_SYNC_PROGRESS
, };
522 if (!get_ldev(mdev
)) {
523 if (__ratelimit(&drbd_ratelimit_state
))
524 dev_warn(DEV
, "Can not update on disk bitmap, local IO disabled.\n");
529 drbd_bm_write_page(mdev
, rs_extent_to_bm_page(udw
->enr
));
534 if (drbd_bm_total_weight(mdev
) <= mdev
->rs_failed
) {
535 switch (mdev
->state
.conn
) {
536 case C_SYNC_SOURCE
: case C_SYNC_TARGET
:
537 case C_PAUSED_SYNC_S
: case C_PAUSED_SYNC_T
:
538 drbd_resync_finished(mdev
);
544 drbd_bcast_event(mdev
, &sib
);
550 /* ATTENTION. The AL's extents are 4MB each, while the extents in the
551 * resync LRU-cache are 16MB each.
552 * The caller of this function has to hold an get_ldev() reference.
554 * TODO will be obsoleted once we have a caching lru of the on disk bitmap
556 static void drbd_try_clear_on_disk_bm(struct drbd_conf
*mdev
, sector_t sector
,
557 int count
, int success
)
559 struct lc_element
*e
;
560 struct update_odbm_work
*udw
;
564 D_ASSERT(atomic_read(&mdev
->local_cnt
));
566 /* I simply assume that a sector/size pair never crosses
567 * a 16 MB extent border. (Currently this is true...) */
568 enr
= BM_SECT_TO_EXT(sector
);
570 e
= lc_get(mdev
->resync
, enr
);
572 struct bm_extent
*ext
= lc_entry(e
, struct bm_extent
, lce
);
573 if (ext
->lce
.lc_number
== enr
) {
575 ext
->rs_left
-= count
;
577 ext
->rs_failed
+= count
;
578 if (ext
->rs_left
< ext
->rs_failed
) {
579 dev_err(DEV
, "BAD! sector=%llus enr=%u rs_left=%d "
580 "rs_failed=%d count=%d\n",
581 (unsigned long long)sector
,
582 ext
->lce
.lc_number
, ext
->rs_left
,
583 ext
->rs_failed
, count
);
586 lc_put(mdev
->resync
, &ext
->lce
);
587 conn_request_state(mdev
->tconn
, NS(conn
, C_DISCONNECTING
), CS_HARD
);
591 /* Normally this element should be in the cache,
592 * since drbd_rs_begin_io() pulled it already in.
594 * But maybe an application write finished, and we set
595 * something outside the resync lru_cache in sync.
597 int rs_left
= drbd_bm_e_weight(mdev
, enr
);
598 if (ext
->flags
!= 0) {
599 dev_warn(DEV
, "changing resync lce: %d[%u;%02lx]"
601 ext
->lce
.lc_number
, ext
->rs_left
,
602 ext
->flags
, enr
, rs_left
);
605 if (ext
->rs_failed
) {
606 dev_warn(DEV
, "Kicking resync_lru element enr=%u "
607 "out with rs_failed=%d\n",
608 ext
->lce
.lc_number
, ext
->rs_failed
);
610 ext
->rs_left
= rs_left
;
611 ext
->rs_failed
= success
? 0 : count
;
612 /* we don't keep a persistent log of the resync lru,
613 * we can commit any change right away. */
614 lc_committed(mdev
->resync
);
616 lc_put(mdev
->resync
, &ext
->lce
);
617 /* no race, we are within the al_lock! */
619 if (ext
->rs_left
== ext
->rs_failed
) {
622 udw
= kmalloc(sizeof(*udw
), GFP_ATOMIC
);
624 udw
->enr
= ext
->lce
.lc_number
;
625 udw
->w
.cb
= w_update_odbm
;
627 drbd_queue_work_front(&mdev
->tconn
->data
.work
, &udw
->w
);
629 dev_warn(DEV
, "Could not kmalloc an udw\n");
633 dev_err(DEV
, "lc_get() failed! locked=%d/%d flags=%lu\n",
635 mdev
->resync
->nr_elements
,
636 mdev
->resync
->flags
);
640 void drbd_advance_rs_marks(struct drbd_conf
*mdev
, unsigned long still_to_go
)
642 unsigned long now
= jiffies
;
643 unsigned long last
= mdev
->rs_mark_time
[mdev
->rs_last_mark
];
644 int next
= (mdev
->rs_last_mark
+ 1) % DRBD_SYNC_MARKS
;
645 if (time_after_eq(now
, last
+ DRBD_SYNC_MARK_STEP
)) {
646 if (mdev
->rs_mark_left
[mdev
->rs_last_mark
] != still_to_go
&&
647 mdev
->state
.conn
!= C_PAUSED_SYNC_T
&&
648 mdev
->state
.conn
!= C_PAUSED_SYNC_S
) {
649 mdev
->rs_mark_time
[next
] = now
;
650 mdev
->rs_mark_left
[next
] = still_to_go
;
651 mdev
->rs_last_mark
= next
;
656 /* clear the bit corresponding to the piece of storage in question:
657 * size byte of data starting from sector. Only clear a bits of the affected
658 * one ore more _aligned_ BM_BLOCK_SIZE blocks.
660 * called by worker on C_SYNC_TARGET and receiver on SyncSource.
663 void __drbd_set_in_sync(struct drbd_conf
*mdev
, sector_t sector
, int size
,
664 const char *file
, const unsigned int line
)
666 /* Is called from worker and receiver context _only_ */
667 unsigned long sbnr
, ebnr
, lbnr
;
668 unsigned long count
= 0;
669 sector_t esector
, nr_sectors
;
673 if (size
<= 0 || !IS_ALIGNED(size
, 512) || size
> DRBD_MAX_BIO_SIZE
) {
674 dev_err(DEV
, "drbd_set_in_sync: sector=%llus size=%d nonsense!\n",
675 (unsigned long long)sector
, size
);
678 nr_sectors
= drbd_get_capacity(mdev
->this_bdev
);
679 esector
= sector
+ (size
>> 9) - 1;
681 if (!expect(sector
< nr_sectors
))
683 if (!expect(esector
< nr_sectors
))
684 esector
= nr_sectors
- 1;
686 lbnr
= BM_SECT_TO_BIT(nr_sectors
-1);
688 /* we clear it (in sync).
689 * round up start sector, round down end sector. we make sure we only
690 * clear full, aligned, BM_BLOCK_SIZE (4K) blocks */
691 if (unlikely(esector
< BM_SECT_PER_BIT
-1))
693 if (unlikely(esector
== (nr_sectors
-1)))
696 ebnr
= BM_SECT_TO_BIT(esector
- (BM_SECT_PER_BIT
-1));
697 sbnr
= BM_SECT_TO_BIT(sector
+ BM_SECT_PER_BIT
-1);
703 * ok, (capacity & 7) != 0 sometimes, but who cares...
704 * we count rs_{total,left} in bits, not sectors.
706 count
= drbd_bm_clear_bits(mdev
, sbnr
, ebnr
);
707 if (count
&& get_ldev(mdev
)) {
708 drbd_advance_rs_marks(mdev
, drbd_bm_total_weight(mdev
));
709 spin_lock_irqsave(&mdev
->al_lock
, flags
);
710 drbd_try_clear_on_disk_bm(mdev
, sector
, count
, true);
711 spin_unlock_irqrestore(&mdev
->al_lock
, flags
);
713 /* just wake_up unconditional now, various lc_chaged(),
714 * lc_put() in drbd_try_clear_on_disk_bm(). */
719 wake_up(&mdev
->al_wait
);
723 * this is intended to set one request worth of data out of sync.
724 * affects at least 1 bit,
725 * and at most 1+DRBD_MAX_BIO_SIZE/BM_BLOCK_SIZE bits.
727 * called by tl_clear and drbd_send_dblock (==drbd_make_request).
728 * so this can be _any_ process.
730 int __drbd_set_out_of_sync(struct drbd_conf
*mdev
, sector_t sector
, int size
,
731 const char *file
, const unsigned int line
)
733 unsigned long sbnr
, ebnr
, lbnr
, flags
;
734 sector_t esector
, nr_sectors
;
735 unsigned int enr
, count
= 0;
736 struct lc_element
*e
;
738 if (size
<= 0 || !IS_ALIGNED(size
, 512) || size
> DRBD_MAX_BIO_SIZE
) {
739 dev_err(DEV
, "sector: %llus, size: %d\n",
740 (unsigned long long)sector
, size
);
745 return 0; /* no disk, no metadata, no bitmap to set bits in */
747 nr_sectors
= drbd_get_capacity(mdev
->this_bdev
);
748 esector
= sector
+ (size
>> 9) - 1;
750 if (!expect(sector
< nr_sectors
))
752 if (!expect(esector
< nr_sectors
))
753 esector
= nr_sectors
- 1;
755 lbnr
= BM_SECT_TO_BIT(nr_sectors
-1);
757 /* we set it out of sync,
758 * we do not need to round anything here */
759 sbnr
= BM_SECT_TO_BIT(sector
);
760 ebnr
= BM_SECT_TO_BIT(esector
);
762 /* ok, (capacity & 7) != 0 sometimes, but who cares...
763 * we count rs_{total,left} in bits, not sectors. */
764 spin_lock_irqsave(&mdev
->al_lock
, flags
);
765 count
= drbd_bm_set_bits(mdev
, sbnr
, ebnr
);
767 enr
= BM_SECT_TO_EXT(sector
);
768 e
= lc_find(mdev
->resync
, enr
);
770 lc_entry(e
, struct bm_extent
, lce
)->rs_left
+= count
;
771 spin_unlock_irqrestore(&mdev
->al_lock
, flags
);
780 struct bm_extent
*_bme_get(struct drbd_conf
*mdev
, unsigned int enr
)
782 struct lc_element
*e
;
783 struct bm_extent
*bm_ext
;
785 unsigned long rs_flags
;
787 spin_lock_irq(&mdev
->al_lock
);
788 if (mdev
->resync_locked
> mdev
->resync
->nr_elements
/2) {
789 spin_unlock_irq(&mdev
->al_lock
);
792 e
= lc_get(mdev
->resync
, enr
);
793 bm_ext
= e
? lc_entry(e
, struct bm_extent
, lce
) : NULL
;
795 if (bm_ext
->lce
.lc_number
!= enr
) {
796 bm_ext
->rs_left
= drbd_bm_e_weight(mdev
, enr
);
797 bm_ext
->rs_failed
= 0;
798 lc_committed(mdev
->resync
);
801 if (bm_ext
->lce
.refcnt
== 1)
802 mdev
->resync_locked
++;
803 set_bit(BME_NO_WRITES
, &bm_ext
->flags
);
805 rs_flags
= mdev
->resync
->flags
;
806 spin_unlock_irq(&mdev
->al_lock
);
808 wake_up(&mdev
->al_wait
);
811 if (rs_flags
& LC_STARVING
)
812 dev_warn(DEV
, "Have to wait for element"
813 " (resync LRU too small?)\n");
814 BUG_ON(rs_flags
& LC_LOCKED
);
820 static int _is_in_al(struct drbd_conf
*mdev
, unsigned int enr
)
824 spin_lock_irq(&mdev
->al_lock
);
825 rv
= lc_is_used(mdev
->act_log
, enr
);
826 spin_unlock_irq(&mdev
->al_lock
);
832 * drbd_rs_begin_io() - Gets an extent in the resync LRU cache and sets it to BME_LOCKED
833 * @mdev: DRBD device.
834 * @sector: The sector number.
836 * This functions sleeps on al_wait. Returns 0 on success, -EINTR if interrupted.
838 int drbd_rs_begin_io(struct drbd_conf
*mdev
, sector_t sector
)
840 unsigned int enr
= BM_SECT_TO_EXT(sector
);
841 struct bm_extent
*bm_ext
;
843 int sa
= 200; /* Step aside 200 times, then grab the extent and let app-IO wait.
844 200 times -> 20 seconds. */
847 sig
= wait_event_interruptible(mdev
->al_wait
,
848 (bm_ext
= _bme_get(mdev
, enr
)));
852 if (test_bit(BME_LOCKED
, &bm_ext
->flags
))
855 for (i
= 0; i
< AL_EXT_PER_BM_SECT
; i
++) {
856 sig
= wait_event_interruptible(mdev
->al_wait
,
857 !_is_in_al(mdev
, enr
* AL_EXT_PER_BM_SECT
+ i
) ||
858 test_bit(BME_PRIORITY
, &bm_ext
->flags
));
860 if (sig
|| (test_bit(BME_PRIORITY
, &bm_ext
->flags
) && sa
)) {
861 spin_lock_irq(&mdev
->al_lock
);
862 if (lc_put(mdev
->resync
, &bm_ext
->lce
) == 0) {
863 bm_ext
->flags
= 0; /* clears BME_NO_WRITES and eventually BME_PRIORITY */
864 mdev
->resync_locked
--;
865 wake_up(&mdev
->al_wait
);
867 spin_unlock_irq(&mdev
->al_lock
);
870 if (schedule_timeout_interruptible(HZ
/10))
873 dev_warn(DEV
,"drbd_rs_begin_io() stepped aside for 20sec."
874 "Resync stalled?\n");
878 set_bit(BME_LOCKED
, &bm_ext
->flags
);
883 * drbd_try_rs_begin_io() - Gets an extent in the resync LRU cache, does not sleep
884 * @mdev: DRBD device.
885 * @sector: The sector number.
887 * Gets an extent in the resync LRU cache, sets it to BME_NO_WRITES, then
888 * tries to set it to BME_LOCKED. Returns 0 upon success, and -EAGAIN
889 * if there is still application IO going on in this area.
891 int drbd_try_rs_begin_io(struct drbd_conf
*mdev
, sector_t sector
)
893 unsigned int enr
= BM_SECT_TO_EXT(sector
);
894 const unsigned int al_enr
= enr
*AL_EXT_PER_BM_SECT
;
895 struct lc_element
*e
;
896 struct bm_extent
*bm_ext
;
899 spin_lock_irq(&mdev
->al_lock
);
900 if (mdev
->resync_wenr
!= LC_FREE
&& mdev
->resync_wenr
!= enr
) {
901 /* in case you have very heavy scattered io, it may
902 * stall the syncer undefined if we give up the ref count
903 * when we try again and requeue.
905 * if we don't give up the refcount, but the next time
906 * we are scheduled this extent has been "synced" by new
907 * application writes, we'd miss the lc_put on the
908 * extent we keep the refcount on.
909 * so we remembered which extent we had to try again, and
910 * if the next requested one is something else, we do
912 * we also have to wake_up
914 e
= lc_find(mdev
->resync
, mdev
->resync_wenr
);
915 bm_ext
= e
? lc_entry(e
, struct bm_extent
, lce
) : NULL
;
917 D_ASSERT(!test_bit(BME_LOCKED
, &bm_ext
->flags
));
918 D_ASSERT(test_bit(BME_NO_WRITES
, &bm_ext
->flags
));
919 clear_bit(BME_NO_WRITES
, &bm_ext
->flags
);
920 mdev
->resync_wenr
= LC_FREE
;
921 if (lc_put(mdev
->resync
, &bm_ext
->lce
) == 0)
922 mdev
->resync_locked
--;
923 wake_up(&mdev
->al_wait
);
925 dev_alert(DEV
, "LOGIC BUG\n");
929 e
= lc_try_get(mdev
->resync
, enr
);
930 bm_ext
= e
? lc_entry(e
, struct bm_extent
, lce
) : NULL
;
932 if (test_bit(BME_LOCKED
, &bm_ext
->flags
))
934 if (!test_and_set_bit(BME_NO_WRITES
, &bm_ext
->flags
)) {
935 mdev
->resync_locked
++;
937 /* we did set the BME_NO_WRITES,
938 * but then could not set BME_LOCKED,
940 * drop the extra reference. */
941 bm_ext
->lce
.refcnt
--;
942 D_ASSERT(bm_ext
->lce
.refcnt
> 0);
946 /* do we rather want to try later? */
947 if (mdev
->resync_locked
> mdev
->resync
->nr_elements
-3)
949 /* Do or do not. There is no try. -- Yoda */
950 e
= lc_get(mdev
->resync
, enr
);
951 bm_ext
= e
? lc_entry(e
, struct bm_extent
, lce
) : NULL
;
953 const unsigned long rs_flags
= mdev
->resync
->flags
;
954 if (rs_flags
& LC_STARVING
)
955 dev_warn(DEV
, "Have to wait for element"
956 " (resync LRU too small?)\n");
957 BUG_ON(rs_flags
& LC_LOCKED
);
960 if (bm_ext
->lce
.lc_number
!= enr
) {
961 bm_ext
->rs_left
= drbd_bm_e_weight(mdev
, enr
);
962 bm_ext
->rs_failed
= 0;
963 lc_committed(mdev
->resync
);
964 wake_up(&mdev
->al_wait
);
965 D_ASSERT(test_bit(BME_LOCKED
, &bm_ext
->flags
) == 0);
967 set_bit(BME_NO_WRITES
, &bm_ext
->flags
);
968 D_ASSERT(bm_ext
->lce
.refcnt
== 1);
969 mdev
->resync_locked
++;
973 for (i
= 0; i
< AL_EXT_PER_BM_SECT
; i
++) {
974 if (lc_is_used(mdev
->act_log
, al_enr
+i
))
977 set_bit(BME_LOCKED
, &bm_ext
->flags
);
979 mdev
->resync_wenr
= LC_FREE
;
980 spin_unlock_irq(&mdev
->al_lock
);
985 mdev
->resync_wenr
= enr
;
986 spin_unlock_irq(&mdev
->al_lock
);
990 void drbd_rs_complete_io(struct drbd_conf
*mdev
, sector_t sector
)
992 unsigned int enr
= BM_SECT_TO_EXT(sector
);
993 struct lc_element
*e
;
994 struct bm_extent
*bm_ext
;
997 spin_lock_irqsave(&mdev
->al_lock
, flags
);
998 e
= lc_find(mdev
->resync
, enr
);
999 bm_ext
= e
? lc_entry(e
, struct bm_extent
, lce
) : NULL
;
1001 spin_unlock_irqrestore(&mdev
->al_lock
, flags
);
1002 if (__ratelimit(&drbd_ratelimit_state
))
1003 dev_err(DEV
, "drbd_rs_complete_io() called, but extent not found\n");
1007 if (bm_ext
->lce
.refcnt
== 0) {
1008 spin_unlock_irqrestore(&mdev
->al_lock
, flags
);
1009 dev_err(DEV
, "drbd_rs_complete_io(,%llu [=%u]) called, "
1010 "but refcnt is 0!?\n",
1011 (unsigned long long)sector
, enr
);
1015 if (lc_put(mdev
->resync
, &bm_ext
->lce
) == 0) {
1016 bm_ext
->flags
= 0; /* clear BME_LOCKED, BME_NO_WRITES and BME_PRIORITY */
1017 mdev
->resync_locked
--;
1018 wake_up(&mdev
->al_wait
);
1021 spin_unlock_irqrestore(&mdev
->al_lock
, flags
);
1025 * drbd_rs_cancel_all() - Removes all extents from the resync LRU (even BME_LOCKED)
1026 * @mdev: DRBD device.
1028 void drbd_rs_cancel_all(struct drbd_conf
*mdev
)
1030 spin_lock_irq(&mdev
->al_lock
);
1032 if (get_ldev_if_state(mdev
, D_FAILED
)) { /* Makes sure ->resync is there. */
1033 lc_reset(mdev
->resync
);
1036 mdev
->resync_locked
= 0;
1037 mdev
->resync_wenr
= LC_FREE
;
1038 spin_unlock_irq(&mdev
->al_lock
);
1039 wake_up(&mdev
->al_wait
);
1043 * drbd_rs_del_all() - Gracefully remove all extents from the resync LRU
1044 * @mdev: DRBD device.
1046 * Returns 0 upon success, -EAGAIN if at least one reference count was
1049 int drbd_rs_del_all(struct drbd_conf
*mdev
)
1051 struct lc_element
*e
;
1052 struct bm_extent
*bm_ext
;
1055 spin_lock_irq(&mdev
->al_lock
);
1057 if (get_ldev_if_state(mdev
, D_FAILED
)) {
1058 /* ok, ->resync is there. */
1059 for (i
= 0; i
< mdev
->resync
->nr_elements
; i
++) {
1060 e
= lc_element_by_index(mdev
->resync
, i
);
1061 bm_ext
= lc_entry(e
, struct bm_extent
, lce
);
1062 if (bm_ext
->lce
.lc_number
== LC_FREE
)
1064 if (bm_ext
->lce
.lc_number
== mdev
->resync_wenr
) {
1065 dev_info(DEV
, "dropping %u in drbd_rs_del_all, apparently"
1066 " got 'synced' by application io\n",
1068 D_ASSERT(!test_bit(BME_LOCKED
, &bm_ext
->flags
));
1069 D_ASSERT(test_bit(BME_NO_WRITES
, &bm_ext
->flags
));
1070 clear_bit(BME_NO_WRITES
, &bm_ext
->flags
);
1071 mdev
->resync_wenr
= LC_FREE
;
1072 lc_put(mdev
->resync
, &bm_ext
->lce
);
1074 if (bm_ext
->lce
.refcnt
!= 0) {
1075 dev_info(DEV
, "Retrying drbd_rs_del_all() later. "
1076 "refcnt=%d\n", bm_ext
->lce
.refcnt
);
1078 spin_unlock_irq(&mdev
->al_lock
);
1081 D_ASSERT(!test_bit(BME_LOCKED
, &bm_ext
->flags
));
1082 D_ASSERT(!test_bit(BME_NO_WRITES
, &bm_ext
->flags
));
1083 lc_del(mdev
->resync
, &bm_ext
->lce
);
1085 D_ASSERT(mdev
->resync
->used
== 0);
1088 spin_unlock_irq(&mdev
->al_lock
);
1094 * drbd_rs_failed_io() - Record information on a failure to resync the specified blocks
1095 * @mdev: DRBD device.
1096 * @sector: The sector number.
1097 * @size: Size of failed IO operation, in byte.
1099 void drbd_rs_failed_io(struct drbd_conf
*mdev
, sector_t sector
, int size
)
1101 /* Is called from worker and receiver context _only_ */
1102 unsigned long sbnr
, ebnr
, lbnr
;
1103 unsigned long count
;
1104 sector_t esector
, nr_sectors
;
1107 if (size
<= 0 || !IS_ALIGNED(size
, 512) || size
> DRBD_MAX_BIO_SIZE
) {
1108 dev_err(DEV
, "drbd_rs_failed_io: sector=%llus size=%d nonsense!\n",
1109 (unsigned long long)sector
, size
);
1112 nr_sectors
= drbd_get_capacity(mdev
->this_bdev
);
1113 esector
= sector
+ (size
>> 9) - 1;
1115 if (!expect(sector
< nr_sectors
))
1117 if (!expect(esector
< nr_sectors
))
1118 esector
= nr_sectors
- 1;
1120 lbnr
= BM_SECT_TO_BIT(nr_sectors
-1);
1123 * round up start sector, round down end sector. we make sure we only
1124 * handle full, aligned, BM_BLOCK_SIZE (4K) blocks */
1125 if (unlikely(esector
< BM_SECT_PER_BIT
-1))
1127 if (unlikely(esector
== (nr_sectors
-1)))
1130 ebnr
= BM_SECT_TO_BIT(esector
- (BM_SECT_PER_BIT
-1));
1131 sbnr
= BM_SECT_TO_BIT(sector
+ BM_SECT_PER_BIT
-1);
1137 * ok, (capacity & 7) != 0 sometimes, but who cares...
1138 * we count rs_{total,left} in bits, not sectors.
1140 spin_lock_irq(&mdev
->al_lock
);
1141 count
= drbd_bm_count_bits(mdev
, sbnr
, ebnr
);
1143 mdev
->rs_failed
+= count
;
1145 if (get_ldev(mdev
)) {
1146 drbd_try_clear_on_disk_bm(mdev
, sector
, count
, false);
1150 /* just wake_up unconditional now, various lc_chaged(),
1151 * lc_put() in drbd_try_clear_on_disk_bm(). */
1154 spin_unlock_irq(&mdev
->al_lock
);
1156 wake_up(&mdev
->al_wait
);