Commit | Line | Data |
---|---|---|
a739ff3f ST |
1 | /* |
2 | * Copyright (C) 2015 Google, Inc. | |
3 | * | |
4 | * Author: Sami Tolvanen <samitolvanen@google.com> | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify it | |
7 | * under the terms of the GNU General Public License as published by the Free | |
8 | * Software Foundation; either version 2 of the License, or (at your option) | |
9 | * any later version. | |
10 | */ | |
11 | ||
12 | #include "dm-verity-fec.h" | |
13 | #include <linux/math64.h> | |
14 | ||
15 | #define DM_MSG_PREFIX "verity-fec" | |
16 | ||
17 | /* | |
18 | * If error correction has been configured, returns true. | |
19 | */ | |
20 | bool verity_fec_is_enabled(struct dm_verity *v) | |
21 | { | |
22 | return v->fec && v->fec->dev; | |
23 | } | |
24 | ||
25 | /* | |
26 | * Return a pointer to dm_verity_fec_io after dm_verity_io and its variable | |
27 | * length fields. | |
28 | */ | |
29 | static inline struct dm_verity_fec_io *fec_io(struct dm_verity_io *io) | |
30 | { | |
31 | return (struct dm_verity_fec_io *) verity_io_digest_end(io->v, io); | |
32 | } | |
33 | ||
34 | /* | |
35 | * Return an interleaved offset for a byte in RS block. | |
36 | */ | |
37 | static inline u64 fec_interleave(struct dm_verity *v, u64 offset) | |
38 | { | |
39 | u32 mod; | |
40 | ||
41 | mod = do_div(offset, v->fec->rsn); | |
42 | return offset + mod * (v->fec->rounds << v->data_dev_block_bits); | |
43 | } | |
44 | ||
45 | /* | |
46 | * Decode an RS block using Reed-Solomon. | |
47 | */ | |
48 | static int fec_decode_rs8(struct dm_verity *v, struct dm_verity_fec_io *fio, | |
49 | u8 *data, u8 *fec, int neras) | |
50 | { | |
51 | int i; | |
52 | uint16_t par[DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN]; | |
53 | ||
54 | for (i = 0; i < v->fec->roots; i++) | |
55 | par[i] = fec[i]; | |
56 | ||
57 | return decode_rs8(fio->rs, data, par, v->fec->rsn, NULL, neras, | |
58 | fio->erasures, 0, NULL); | |
59 | } | |
60 | ||
61 | /* | |
62 | * Read error-correcting codes for the requested RS block. Returns a pointer | |
63 | * to the data block. Caller is responsible for releasing buf. | |
64 | */ | |
65 | static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index, | |
66 | unsigned *offset, struct dm_buffer **buf) | |
67 | { | |
68 | u64 position, block; | |
69 | u8 *res; | |
70 | ||
71 | position = (index + rsb) * v->fec->roots; | |
72 | block = position >> v->data_dev_block_bits; | |
73 | *offset = (unsigned)(position - (block << v->data_dev_block_bits)); | |
74 | ||
75 | res = dm_bufio_read(v->fec->bufio, v->fec->start + block, buf); | |
76 | if (unlikely(IS_ERR(res))) { | |
77 | DMERR("%s: FEC %llu: parity read failed (block %llu): %ld", | |
78 | v->data_dev->name, (unsigned long long)rsb, | |
79 | (unsigned long long)(v->fec->start + block), | |
80 | PTR_ERR(res)); | |
81 | *buf = NULL; | |
82 | } | |
83 | ||
84 | return res; | |
85 | } | |
86 | ||
87 | /* Loop over each preallocated buffer slot. */ | |
88 | #define fec_for_each_prealloc_buffer(__i) \ | |
89 | for (__i = 0; __i < DM_VERITY_FEC_BUF_PREALLOC; __i++) | |
90 | ||
91 | /* Loop over each extra buffer slot. */ | |
92 | #define fec_for_each_extra_buffer(io, __i) \ | |
93 | for (__i = DM_VERITY_FEC_BUF_PREALLOC; __i < DM_VERITY_FEC_BUF_MAX; __i++) | |
94 | ||
95 | /* Loop over each allocated buffer. */ | |
96 | #define fec_for_each_buffer(io, __i) \ | |
97 | for (__i = 0; __i < (io)->nbufs; __i++) | |
98 | ||
99 | /* Loop over each RS block in each allocated buffer. */ | |
100 | #define fec_for_each_buffer_rs_block(io, __i, __j) \ | |
101 | fec_for_each_buffer(io, __i) \ | |
102 | for (__j = 0; __j < 1 << DM_VERITY_FEC_BUF_RS_BITS; __j++) | |
103 | ||
104 | /* | |
105 | * Return a pointer to the current RS block when called inside | |
106 | * fec_for_each_buffer_rs_block. | |
107 | */ | |
108 | static inline u8 *fec_buffer_rs_block(struct dm_verity *v, | |
109 | struct dm_verity_fec_io *fio, | |
110 | unsigned i, unsigned j) | |
111 | { | |
112 | return &fio->bufs[i][j * v->fec->rsn]; | |
113 | } | |
114 | ||
115 | /* | |
116 | * Return an index to the current RS block when called inside | |
117 | * fec_for_each_buffer_rs_block. | |
118 | */ | |
119 | static inline unsigned fec_buffer_rs_index(unsigned i, unsigned j) | |
120 | { | |
121 | return (i << DM_VERITY_FEC_BUF_RS_BITS) + j; | |
122 | } | |
123 | ||
124 | /* | |
125 | * Decode all RS blocks from buffers and copy corrected bytes into fio->output | |
126 | * starting from block_offset. | |
127 | */ | |
128 | static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio, | |
129 | u64 rsb, int byte_index, unsigned block_offset, | |
130 | int neras) | |
131 | { | |
132 | int r, corrected = 0, res; | |
133 | struct dm_buffer *buf; | |
134 | unsigned n, i, offset; | |
135 | u8 *par, *block; | |
136 | ||
137 | par = fec_read_parity(v, rsb, block_offset, &offset, &buf); | |
138 | if (IS_ERR(par)) | |
139 | return PTR_ERR(par); | |
140 | ||
141 | /* | |
142 | * Decode the RS blocks we have in bufs. Each RS block results in | |
143 | * one corrected target byte and consumes fec->roots parity bytes. | |
144 | */ | |
145 | fec_for_each_buffer_rs_block(fio, n, i) { | |
146 | block = fec_buffer_rs_block(v, fio, n, i); | |
147 | res = fec_decode_rs8(v, fio, block, &par[offset], neras); | |
148 | if (res < 0) { | |
149 | dm_bufio_release(buf); | |
150 | ||
151 | r = res; | |
152 | goto error; | |
153 | } | |
154 | ||
155 | corrected += res; | |
156 | fio->output[block_offset] = block[byte_index]; | |
157 | ||
158 | block_offset++; | |
159 | if (block_offset >= 1 << v->data_dev_block_bits) | |
160 | goto done; | |
161 | ||
162 | /* read the next block when we run out of parity bytes */ | |
163 | offset += v->fec->roots; | |
164 | if (offset >= 1 << v->data_dev_block_bits) { | |
165 | dm_bufio_release(buf); | |
166 | ||
167 | par = fec_read_parity(v, rsb, block_offset, &offset, &buf); | |
168 | if (unlikely(IS_ERR(par))) | |
169 | return PTR_ERR(par); | |
170 | } | |
171 | } | |
172 | done: | |
173 | r = corrected; | |
174 | error: | |
175 | if (r < 0 && neras) | |
176 | DMERR_LIMIT("%s: FEC %llu: failed to correct: %d", | |
177 | v->data_dev->name, (unsigned long long)rsb, r); | |
178 | else if (r > 0) | |
179 | DMWARN_LIMIT("%s: FEC %llu: corrected %d errors", | |
180 | v->data_dev->name, (unsigned long long)rsb, r); | |
181 | ||
182 | return r; | |
183 | } | |
184 | ||
185 | /* | |
186 | * Locate data block erasures using verity hashes. | |
187 | */ | |
188 | static int fec_is_erasure(struct dm_verity *v, struct dm_verity_io *io, | |
189 | u8 *want_digest, u8 *data) | |
190 | { | |
191 | if (unlikely(verity_hash(v, verity_io_hash_desc(v, io), | |
192 | data, 1 << v->data_dev_block_bits, | |
193 | verity_io_real_digest(v, io)))) | |
194 | return 0; | |
195 | ||
196 | return memcmp(verity_io_real_digest(v, io), want_digest, | |
197 | v->digest_size) != 0; | |
198 | } | |
199 | ||
200 | /* | |
201 | * Read data blocks that are part of the RS block and deinterleave as much as | |
202 | * fits into buffers. Check for erasure locations if @neras is non-NULL. | |
203 | */ | |
204 | static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, | |
205 | u64 rsb, u64 target, unsigned block_offset, | |
206 | int *neras) | |
207 | { | |
0cc37c2d | 208 | bool is_zero; |
a739ff3f ST |
209 | int i, j, target_index = -1; |
210 | struct dm_buffer *buf; | |
211 | struct dm_bufio_client *bufio; | |
212 | struct dm_verity_fec_io *fio = fec_io(io); | |
213 | u64 block, ileaved; | |
214 | u8 *bbuf, *rs_block; | |
215 | u8 want_digest[v->digest_size]; | |
216 | unsigned n, k; | |
217 | ||
218 | if (neras) | |
219 | *neras = 0; | |
220 | ||
221 | /* | |
222 | * read each of the rsn data blocks that are part of the RS block, and | |
223 | * interleave contents to available bufs | |
224 | */ | |
225 | for (i = 0; i < v->fec->rsn; i++) { | |
226 | ileaved = fec_interleave(v, rsb * v->fec->rsn + i); | |
227 | ||
228 | /* | |
229 | * target is the data block we want to correct, target_index is | |
230 | * the index of this block within the rsn RS blocks | |
231 | */ | |
232 | if (ileaved == target) | |
233 | target_index = i; | |
234 | ||
235 | block = ileaved >> v->data_dev_block_bits; | |
236 | bufio = v->fec->data_bufio; | |
237 | ||
238 | if (block >= v->data_blocks) { | |
239 | block -= v->data_blocks; | |
240 | ||
241 | /* | |
242 | * blocks outside the area were assumed to contain | |
243 | * zeros when encoding data was generated | |
244 | */ | |
245 | if (unlikely(block >= v->fec->hash_blocks)) | |
246 | continue; | |
247 | ||
248 | block += v->hash_start; | |
249 | bufio = v->bufio; | |
250 | } | |
251 | ||
252 | bbuf = dm_bufio_read(bufio, block, &buf); | |
253 | if (unlikely(IS_ERR(bbuf))) { | |
254 | DMWARN_LIMIT("%s: FEC %llu: read failed (%llu): %ld", | |
255 | v->data_dev->name, | |
256 | (unsigned long long)rsb, | |
257 | (unsigned long long)block, PTR_ERR(bbuf)); | |
258 | ||
259 | /* assume the block is corrupted */ | |
260 | if (neras && *neras <= v->fec->roots) | |
261 | fio->erasures[(*neras)++] = i; | |
262 | ||
263 | continue; | |
264 | } | |
265 | ||
266 | /* locate erasures if the block is on the data device */ | |
267 | if (bufio == v->fec->data_bufio && | |
0cc37c2d ST |
268 | verity_hash_for_block(v, io, block, want_digest, |
269 | &is_zero) == 0) { | |
270 | /* skip known zero blocks entirely */ | |
271 | if (is_zero) | |
272 | continue; | |
273 | ||
a739ff3f ST |
274 | /* |
275 | * skip if we have already found the theoretical | |
276 | * maximum number (i.e. fec->roots) of erasures | |
277 | */ | |
278 | if (neras && *neras <= v->fec->roots && | |
279 | fec_is_erasure(v, io, want_digest, bbuf)) | |
280 | fio->erasures[(*neras)++] = i; | |
281 | } | |
282 | ||
283 | /* | |
284 | * deinterleave and copy the bytes that fit into bufs, | |
285 | * starting from block_offset | |
286 | */ | |
287 | fec_for_each_buffer_rs_block(fio, n, j) { | |
288 | k = fec_buffer_rs_index(n, j) + block_offset; | |
289 | ||
290 | if (k >= 1 << v->data_dev_block_bits) | |
291 | goto done; | |
292 | ||
293 | rs_block = fec_buffer_rs_block(v, fio, n, j); | |
294 | rs_block[i] = bbuf[k]; | |
295 | } | |
296 | done: | |
297 | dm_bufio_release(buf); | |
298 | } | |
299 | ||
300 | return target_index; | |
301 | } | |
302 | ||
303 | /* | |
304 | * Allocate RS control structure and FEC buffers from preallocated mempools, | |
305 | * and attempt to allocate as many extra buffers as available. | |
306 | */ | |
307 | static int fec_alloc_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) | |
308 | { | |
309 | unsigned n; | |
310 | ||
311 | if (!fio->rs) { | |
312 | fio->rs = mempool_alloc(v->fec->rs_pool, 0); | |
313 | if (unlikely(!fio->rs)) { | |
314 | DMERR("failed to allocate RS"); | |
315 | return -ENOMEM; | |
316 | } | |
317 | } | |
318 | ||
319 | fec_for_each_prealloc_buffer(n) { | |
320 | if (fio->bufs[n]) | |
321 | continue; | |
322 | ||
323 | fio->bufs[n] = mempool_alloc(v->fec->prealloc_pool, GFP_NOIO); | |
324 | if (unlikely(!fio->bufs[n])) { | |
325 | DMERR("failed to allocate FEC buffer"); | |
326 | return -ENOMEM; | |
327 | } | |
328 | } | |
329 | ||
330 | /* try to allocate the maximum number of buffers */ | |
331 | fec_for_each_extra_buffer(fio, n) { | |
332 | if (fio->bufs[n]) | |
333 | continue; | |
334 | ||
335 | fio->bufs[n] = mempool_alloc(v->fec->extra_pool, GFP_NOIO); | |
336 | /* we can manage with even one buffer if necessary */ | |
337 | if (unlikely(!fio->bufs[n])) | |
338 | break; | |
339 | } | |
340 | fio->nbufs = n; | |
341 | ||
342 | if (!fio->output) { | |
343 | fio->output = mempool_alloc(v->fec->output_pool, GFP_NOIO); | |
344 | ||
345 | if (!fio->output) { | |
346 | DMERR("failed to allocate FEC page"); | |
347 | return -ENOMEM; | |
348 | } | |
349 | } | |
350 | ||
351 | return 0; | |
352 | } | |
353 | ||
354 | /* | |
355 | * Initialize buffers and clear erasures. fec_read_bufs() assumes buffers are | |
356 | * zeroed before deinterleaving. | |
357 | */ | |
358 | static void fec_init_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) | |
359 | { | |
360 | unsigned n; | |
361 | ||
362 | fec_for_each_buffer(fio, n) | |
363 | memset(fio->bufs[n], 0, v->fec->rsn << DM_VERITY_FEC_BUF_RS_BITS); | |
364 | ||
365 | memset(fio->erasures, 0, sizeof(fio->erasures)); | |
366 | } | |
367 | ||
368 | /* | |
369 | * Decode all RS blocks in a single data block and return the target block | |
370 | * (indicated by @offset) in fio->output. If @use_erasures is non-zero, uses | |
371 | * hashes to locate erasures. | |
372 | */ | |
373 | static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io, | |
374 | struct dm_verity_fec_io *fio, u64 rsb, u64 offset, | |
375 | bool use_erasures) | |
376 | { | |
377 | int r, neras = 0; | |
378 | unsigned pos; | |
379 | ||
380 | r = fec_alloc_bufs(v, fio); | |
381 | if (unlikely(r < 0)) | |
382 | return r; | |
383 | ||
384 | for (pos = 0; pos < 1 << v->data_dev_block_bits; ) { | |
385 | fec_init_bufs(v, fio); | |
386 | ||
387 | r = fec_read_bufs(v, io, rsb, offset, pos, | |
388 | use_erasures ? &neras : NULL); | |
389 | if (unlikely(r < 0)) | |
390 | return r; | |
391 | ||
392 | r = fec_decode_bufs(v, fio, rsb, r, pos, neras); | |
393 | if (r < 0) | |
394 | return r; | |
395 | ||
396 | pos += fio->nbufs << DM_VERITY_FEC_BUF_RS_BITS; | |
397 | } | |
398 | ||
399 | /* Always re-validate the corrected block against the expected hash */ | |
400 | r = verity_hash(v, verity_io_hash_desc(v, io), fio->output, | |
401 | 1 << v->data_dev_block_bits, | |
402 | verity_io_real_digest(v, io)); | |
403 | if (unlikely(r < 0)) | |
404 | return r; | |
405 | ||
406 | if (memcmp(verity_io_real_digest(v, io), verity_io_want_digest(v, io), | |
407 | v->digest_size)) { | |
408 | DMERR_LIMIT("%s: FEC %llu: failed to correct (%d erasures)", | |
409 | v->data_dev->name, (unsigned long long)rsb, neras); | |
410 | return -EILSEQ; | |
411 | } | |
412 | ||
413 | return 0; | |
414 | } | |
415 | ||
416 | static int fec_bv_copy(struct dm_verity *v, struct dm_verity_io *io, u8 *data, | |
417 | size_t len) | |
418 | { | |
419 | struct dm_verity_fec_io *fio = fec_io(io); | |
420 | ||
421 | memcpy(data, &fio->output[fio->output_pos], len); | |
422 | fio->output_pos += len; | |
423 | ||
424 | return 0; | |
425 | } | |
426 | ||
427 | /* | |
428 | * Correct errors in a block. Copies corrected block to dest if non-NULL, | |
429 | * otherwise to a bio_vec starting from iter. | |
430 | */ | |
431 | int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io, | |
432 | enum verity_block_type type, sector_t block, u8 *dest, | |
433 | struct bvec_iter *iter) | |
434 | { | |
435 | int r; | |
436 | struct dm_verity_fec_io *fio = fec_io(io); | |
437 | u64 offset, res, rsb; | |
438 | ||
439 | if (!verity_fec_is_enabled(v)) | |
440 | return -EOPNOTSUPP; | |
441 | ||
442 | if (type == DM_VERITY_BLOCK_TYPE_METADATA) | |
443 | block += v->data_blocks; | |
444 | ||
445 | /* | |
446 | * For RS(M, N), the continuous FEC data is divided into blocks of N | |
447 | * bytes. Since block size may not be divisible by N, the last block | |
448 | * is zero padded when decoding. | |
449 | * | |
450 | * Each byte of the block is covered by a different RS(M, N) code, | |
451 | * and each code is interleaved over N blocks to make it less likely | |
452 | * that bursty corruption will leave us in unrecoverable state. | |
453 | */ | |
454 | ||
455 | offset = block << v->data_dev_block_bits; | |
456 | ||
457 | res = offset; | |
458 | div64_u64(res, v->fec->rounds << v->data_dev_block_bits); | |
459 | ||
460 | /* | |
461 | * The base RS block we can feed to the interleaver to find out all | |
462 | * blocks required for decoding. | |
463 | */ | |
464 | rsb = offset - res * (v->fec->rounds << v->data_dev_block_bits); | |
465 | ||
466 | /* | |
467 | * Locating erasures is slow, so attempt to recover the block without | |
468 | * them first. Do a second attempt with erasures if the corruption is | |
469 | * bad enough. | |
470 | */ | |
471 | r = fec_decode_rsb(v, io, fio, rsb, offset, false); | |
472 | if (r < 0) { | |
473 | r = fec_decode_rsb(v, io, fio, rsb, offset, true); | |
474 | if (r < 0) | |
475 | return r; | |
476 | } | |
477 | ||
478 | if (dest) | |
479 | memcpy(dest, fio->output, 1 << v->data_dev_block_bits); | |
480 | else if (iter) { | |
481 | fio->output_pos = 0; | |
482 | r = verity_for_bv_block(v, io, iter, fec_bv_copy); | |
483 | } | |
484 | ||
485 | return r; | |
486 | } | |
487 | ||
488 | /* | |
489 | * Clean up per-bio data. | |
490 | */ | |
491 | void verity_fec_finish_io(struct dm_verity_io *io) | |
492 | { | |
493 | unsigned n; | |
494 | struct dm_verity_fec *f = io->v->fec; | |
495 | struct dm_verity_fec_io *fio = fec_io(io); | |
496 | ||
497 | if (!verity_fec_is_enabled(io->v)) | |
498 | return; | |
499 | ||
500 | mempool_free(fio->rs, f->rs_pool); | |
501 | ||
502 | fec_for_each_prealloc_buffer(n) | |
503 | mempool_free(fio->bufs[n], f->prealloc_pool); | |
504 | ||
505 | fec_for_each_extra_buffer(fio, n) | |
506 | mempool_free(fio->bufs[n], f->extra_pool); | |
507 | ||
508 | mempool_free(fio->output, f->output_pool); | |
509 | } | |
510 | ||
511 | /* | |
512 | * Initialize per-bio data. | |
513 | */ | |
514 | void verity_fec_init_io(struct dm_verity_io *io) | |
515 | { | |
516 | struct dm_verity_fec_io *fio = fec_io(io); | |
517 | ||
518 | if (!verity_fec_is_enabled(io->v)) | |
519 | return; | |
520 | ||
521 | fio->rs = NULL; | |
522 | memset(fio->bufs, 0, sizeof(fio->bufs)); | |
523 | fio->nbufs = 0; | |
524 | fio->output = NULL; | |
525 | } | |
526 | ||
527 | /* | |
528 | * Append feature arguments and values to the status table. | |
529 | */ | |
530 | unsigned verity_fec_status_table(struct dm_verity *v, unsigned sz, | |
531 | char *result, unsigned maxlen) | |
532 | { | |
533 | if (!verity_fec_is_enabled(v)) | |
534 | return sz; | |
535 | ||
536 | DMEMIT(" " DM_VERITY_OPT_FEC_DEV " %s " | |
537 | DM_VERITY_OPT_FEC_BLOCKS " %llu " | |
538 | DM_VERITY_OPT_FEC_START " %llu " | |
539 | DM_VERITY_OPT_FEC_ROOTS " %d", | |
540 | v->fec->dev->name, | |
541 | (unsigned long long)v->fec->blocks, | |
542 | (unsigned long long)v->fec->start, | |
543 | v->fec->roots); | |
544 | ||
545 | return sz; | |
546 | } | |
547 | ||
548 | void verity_fec_dtr(struct dm_verity *v) | |
549 | { | |
550 | struct dm_verity_fec *f = v->fec; | |
551 | ||
552 | if (!verity_fec_is_enabled(v)) | |
553 | goto out; | |
554 | ||
555 | mempool_destroy(f->rs_pool); | |
556 | mempool_destroy(f->prealloc_pool); | |
557 | mempool_destroy(f->extra_pool); | |
558 | kmem_cache_destroy(f->cache); | |
559 | ||
560 | if (f->data_bufio) | |
561 | dm_bufio_client_destroy(f->data_bufio); | |
562 | if (f->bufio) | |
563 | dm_bufio_client_destroy(f->bufio); | |
564 | ||
565 | if (f->dev) | |
566 | dm_put_device(v->ti, f->dev); | |
567 | out: | |
568 | kfree(f); | |
569 | v->fec = NULL; | |
570 | } | |
571 | ||
572 | static void *fec_rs_alloc(gfp_t gfp_mask, void *pool_data) | |
573 | { | |
574 | struct dm_verity *v = (struct dm_verity *)pool_data; | |
575 | ||
576 | return init_rs(8, 0x11d, 0, 1, v->fec->roots); | |
577 | } | |
578 | ||
579 | static void fec_rs_free(void *element, void *pool_data) | |
580 | { | |
581 | struct rs_control *rs = (struct rs_control *)element; | |
582 | ||
583 | if (rs) | |
584 | free_rs(rs); | |
585 | } | |
586 | ||
587 | bool verity_is_fec_opt_arg(const char *arg_name) | |
588 | { | |
589 | return (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_DEV) || | |
590 | !strcasecmp(arg_name, DM_VERITY_OPT_FEC_BLOCKS) || | |
591 | !strcasecmp(arg_name, DM_VERITY_OPT_FEC_START) || | |
592 | !strcasecmp(arg_name, DM_VERITY_OPT_FEC_ROOTS)); | |
593 | } | |
594 | ||
595 | int verity_fec_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, | |
596 | unsigned *argc, const char *arg_name) | |
597 | { | |
598 | int r; | |
599 | struct dm_target *ti = v->ti; | |
600 | const char *arg_value; | |
601 | unsigned long long num_ll; | |
602 | unsigned char num_c; | |
603 | char dummy; | |
604 | ||
605 | if (!*argc) { | |
606 | ti->error = "FEC feature arguments require a value"; | |
607 | return -EINVAL; | |
608 | } | |
609 | ||
610 | arg_value = dm_shift_arg(as); | |
611 | (*argc)--; | |
612 | ||
613 | if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_DEV)) { | |
614 | r = dm_get_device(ti, arg_value, FMODE_READ, &v->fec->dev); | |
615 | if (r) { | |
616 | ti->error = "FEC device lookup failed"; | |
617 | return r; | |
618 | } | |
619 | ||
620 | } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_BLOCKS)) { | |
621 | if (sscanf(arg_value, "%llu%c", &num_ll, &dummy) != 1 || | |
622 | ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) | |
623 | >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { | |
624 | ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS; | |
625 | return -EINVAL; | |
626 | } | |
627 | v->fec->blocks = num_ll; | |
628 | ||
629 | } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_START)) { | |
630 | if (sscanf(arg_value, "%llu%c", &num_ll, &dummy) != 1 || | |
631 | ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) >> | |
632 | (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { | |
633 | ti->error = "Invalid " DM_VERITY_OPT_FEC_START; | |
634 | return -EINVAL; | |
635 | } | |
636 | v->fec->start = num_ll; | |
637 | ||
638 | } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_ROOTS)) { | |
639 | if (sscanf(arg_value, "%hhu%c", &num_c, &dummy) != 1 || !num_c || | |
640 | num_c < (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MAX_RSN) || | |
641 | num_c > (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN)) { | |
642 | ti->error = "Invalid " DM_VERITY_OPT_FEC_ROOTS; | |
643 | return -EINVAL; | |
644 | } | |
645 | v->fec->roots = num_c; | |
646 | ||
647 | } else { | |
648 | ti->error = "Unrecognized verity FEC feature request"; | |
649 | return -EINVAL; | |
650 | } | |
651 | ||
652 | return 0; | |
653 | } | |
654 | ||
655 | /* | |
656 | * Allocate dm_verity_fec for v->fec. Must be called before verity_fec_ctr. | |
657 | */ | |
658 | int verity_fec_ctr_alloc(struct dm_verity *v) | |
659 | { | |
660 | struct dm_verity_fec *f; | |
661 | ||
662 | f = kzalloc(sizeof(struct dm_verity_fec), GFP_KERNEL); | |
663 | if (!f) { | |
664 | v->ti->error = "Cannot allocate FEC structure"; | |
665 | return -ENOMEM; | |
666 | } | |
667 | v->fec = f; | |
668 | ||
669 | return 0; | |
670 | } | |
671 | ||
672 | /* | |
673 | * Validate arguments and preallocate memory. Must be called after arguments | |
674 | * have been parsed using verity_fec_parse_opt_args. | |
675 | */ | |
676 | int verity_fec_ctr(struct dm_verity *v) | |
677 | { | |
678 | struct dm_verity_fec *f = v->fec; | |
679 | struct dm_target *ti = v->ti; | |
680 | u64 hash_blocks; | |
681 | ||
682 | if (!verity_fec_is_enabled(v)) { | |
683 | verity_fec_dtr(v); | |
684 | return 0; | |
685 | } | |
686 | ||
687 | /* | |
688 | * FEC is computed over data blocks, possible metadata, and | |
689 | * hash blocks. In other words, FEC covers total of fec_blocks | |
690 | * blocks consisting of the following: | |
691 | * | |
692 | * data blocks | hash blocks | metadata (optional) | |
693 | * | |
694 | * We allow metadata after hash blocks to support a use case | |
695 | * where all data is stored on the same device and FEC covers | |
696 | * the entire area. | |
697 | * | |
698 | * If metadata is included, we require it to be available on the | |
699 | * hash device after the hash blocks. | |
700 | */ | |
701 | ||
702 | hash_blocks = v->hash_blocks - v->hash_start; | |
703 | ||
704 | /* | |
705 | * Require matching block sizes for data and hash devices for | |
706 | * simplicity. | |
707 | */ | |
708 | if (v->data_dev_block_bits != v->hash_dev_block_bits) { | |
709 | ti->error = "Block sizes must match to use FEC"; | |
710 | return -EINVAL; | |
711 | } | |
712 | ||
713 | if (!f->roots) { | |
714 | ti->error = "Missing " DM_VERITY_OPT_FEC_ROOTS; | |
715 | return -EINVAL; | |
716 | } | |
717 | f->rsn = DM_VERITY_FEC_RSM - f->roots; | |
718 | ||
719 | if (!f->blocks) { | |
720 | ti->error = "Missing " DM_VERITY_OPT_FEC_BLOCKS; | |
721 | return -EINVAL; | |
722 | } | |
723 | ||
724 | f->rounds = f->blocks; | |
725 | if (sector_div(f->rounds, f->rsn)) | |
726 | f->rounds++; | |
727 | ||
728 | /* | |
729 | * Due to optional metadata, f->blocks can be larger than | |
730 | * data_blocks and hash_blocks combined. | |
731 | */ | |
732 | if (f->blocks < v->data_blocks + hash_blocks || !f->rounds) { | |
733 | ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS; | |
734 | return -EINVAL; | |
735 | } | |
736 | ||
737 | /* | |
738 | * Metadata is accessed through the hash device, so we require | |
739 | * it to be large enough. | |
740 | */ | |
741 | f->hash_blocks = f->blocks - v->data_blocks; | |
742 | if (dm_bufio_get_device_size(v->bufio) < f->hash_blocks) { | |
743 | ti->error = "Hash device is too small for " | |
744 | DM_VERITY_OPT_FEC_BLOCKS; | |
745 | return -E2BIG; | |
746 | } | |
747 | ||
748 | f->bufio = dm_bufio_client_create(f->dev->bdev, | |
749 | 1 << v->data_dev_block_bits, | |
750 | 1, 0, NULL, NULL); | |
751 | if (IS_ERR(f->bufio)) { | |
752 | ti->error = "Cannot initialize FEC bufio client"; | |
753 | return PTR_ERR(f->bufio); | |
754 | } | |
755 | ||
756 | if (dm_bufio_get_device_size(f->bufio) < | |
757 | ((f->start + f->rounds * f->roots) >> v->data_dev_block_bits)) { | |
758 | ti->error = "FEC device is too small"; | |
759 | return -E2BIG; | |
760 | } | |
761 | ||
762 | f->data_bufio = dm_bufio_client_create(v->data_dev->bdev, | |
763 | 1 << v->data_dev_block_bits, | |
764 | 1, 0, NULL, NULL); | |
765 | if (IS_ERR(f->data_bufio)) { | |
766 | ti->error = "Cannot initialize FEC data bufio client"; | |
767 | return PTR_ERR(f->data_bufio); | |
768 | } | |
769 | ||
770 | if (dm_bufio_get_device_size(f->data_bufio) < v->data_blocks) { | |
771 | ti->error = "Data device is too small"; | |
772 | return -E2BIG; | |
773 | } | |
774 | ||
775 | /* Preallocate an rs_control structure for each worker thread */ | |
776 | f->rs_pool = mempool_create(num_online_cpus(), fec_rs_alloc, | |
777 | fec_rs_free, (void *) v); | |
778 | if (!f->rs_pool) { | |
779 | ti->error = "Cannot allocate RS pool"; | |
780 | return -ENOMEM; | |
781 | } | |
782 | ||
783 | f->cache = kmem_cache_create("dm_verity_fec_buffers", | |
784 | f->rsn << DM_VERITY_FEC_BUF_RS_BITS, | |
785 | 0, 0, NULL); | |
786 | if (!f->cache) { | |
787 | ti->error = "Cannot create FEC buffer cache"; | |
788 | return -ENOMEM; | |
789 | } | |
790 | ||
791 | /* Preallocate DM_VERITY_FEC_BUF_PREALLOC buffers for each thread */ | |
792 | f->prealloc_pool = mempool_create_slab_pool(num_online_cpus() * | |
793 | DM_VERITY_FEC_BUF_PREALLOC, | |
794 | f->cache); | |
795 | if (!f->prealloc_pool) { | |
796 | ti->error = "Cannot allocate FEC buffer prealloc pool"; | |
797 | return -ENOMEM; | |
798 | } | |
799 | ||
800 | f->extra_pool = mempool_create_slab_pool(0, f->cache); | |
801 | if (!f->extra_pool) { | |
802 | ti->error = "Cannot allocate FEC buffer extra pool"; | |
803 | return -ENOMEM; | |
804 | } | |
805 | ||
806 | /* Preallocate an output buffer for each thread */ | |
807 | f->output_pool = mempool_create_kmalloc_pool(num_online_cpus(), | |
808 | 1 << v->data_dev_block_bits); | |
809 | if (!f->output_pool) { | |
810 | ti->error = "Cannot allocate FEC output pool"; | |
811 | return -ENOMEM; | |
812 | } | |
813 | ||
814 | /* Reserve space for our per-bio data */ | |
30187e1d | 815 | ti->per_io_data_size += sizeof(struct dm_verity_fec_io); |
a739ff3f ST |
816 | |
817 | return 0; | |
818 | } |