2 * Copyright (C) 2012 Google, Inc.
4 * This software is licensed under the terms of the GNU General Public
5 * License version 2, as published by the Free Software Foundation, and
6 * may be copied, distributed, and modified under those terms.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
15 #include <linux/device.h>
16 #include <linux/err.h>
17 #include <linux/errno.h>
18 #include <linux/kernel.h>
19 #include <linux/init.h>
21 #include <linux/list.h>
22 #include <linux/memblock.h>
23 #include <linux/rslib.h>
24 #include <linux/slab.h>
25 #include <linux/vmalloc.h>
26 #include "persistent_ram.h"
28 struct persistent_ram_buffer
{
35 #define PERSISTENT_RAM_SIG (0x43474244) /* DBGC */
37 static __initdata
LIST_HEAD(persistent_ram_list
);
39 static inline size_t buffer_size(struct persistent_ram_zone
*prz
)
41 return atomic_read(&prz
->buffer
->size
);
44 static inline size_t buffer_start(struct persistent_ram_zone
*prz
)
46 return atomic_read(&prz
->buffer
->start
);
49 /* increase and wrap the start pointer, returning the old value */
50 static inline size_t buffer_start_add(struct persistent_ram_zone
*prz
, size_t a
)
56 old
= atomic_read(&prz
->buffer
->start
);
58 while (unlikely(new > prz
->buffer_size
))
59 new -= prz
->buffer_size
;
60 } while (atomic_cmpxchg(&prz
->buffer
->start
, old
, new) != old
);
65 /* increase the size counter until it hits the max size */
66 static inline void buffer_size_add(struct persistent_ram_zone
*prz
, size_t a
)
71 if (atomic_read(&prz
->buffer
->size
) == prz
->buffer_size
)
75 old
= atomic_read(&prz
->buffer
->size
);
77 if (new > prz
->buffer_size
)
78 new = prz
->buffer_size
;
79 } while (atomic_cmpxchg(&prz
->buffer
->size
, old
, new) != old
);
82 /* increase the size counter, retuning an error if it hits the max size */
83 static inline ssize_t
buffer_size_add_clamp(struct persistent_ram_zone
*prz
,
90 old
= atomic_read(&prz
->buffer
->size
);
92 if (new > prz
->buffer_size
)
94 } while (atomic_cmpxchg(&prz
->buffer
->size
, old
, new) != old
);
99 static void notrace
persistent_ram_encode_rs8(struct persistent_ram_zone
*prz
,
100 uint8_t *data
, size_t len
, uint8_t *ecc
)
103 uint16_t par
[prz
->ecc_size
];
105 /* Initialize the parity buffer */
106 memset(par
, 0, sizeof(par
));
107 encode_rs8(prz
->rs_decoder
, data
, len
, par
, 0);
108 for (i
= 0; i
< prz
->ecc_size
; i
++)
112 static int persistent_ram_decode_rs8(struct persistent_ram_zone
*prz
,
113 void *data
, size_t len
, uint8_t *ecc
)
116 uint16_t par
[prz
->ecc_size
];
118 for (i
= 0; i
< prz
->ecc_size
; i
++)
120 return decode_rs8(prz
->rs_decoder
, data
, par
, len
,
121 NULL
, 0, NULL
, 0, NULL
);
124 static void notrace
persistent_ram_update_ecc(struct persistent_ram_zone
*prz
,
125 unsigned int start
, unsigned int count
)
127 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
128 uint8_t *buffer_end
= buffer
->data
+ prz
->buffer_size
;
131 int ecc_block_size
= prz
->ecc_block_size
;
132 int ecc_size
= prz
->ecc_size
;
133 int size
= prz
->ecc_block_size
;
138 block
= buffer
->data
+ (start
& ~(ecc_block_size
- 1));
139 par
= prz
->par_buffer
+ (start
/ ecc_block_size
) * prz
->ecc_size
;
142 if (block
+ ecc_block_size
> buffer_end
)
143 size
= buffer_end
- block
;
144 persistent_ram_encode_rs8(prz
, block
, size
, par
);
145 block
+= ecc_block_size
;
147 } while (block
< buffer
->data
+ start
+ count
);
150 static void persistent_ram_update_header_ecc(struct persistent_ram_zone
*prz
)
152 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
157 persistent_ram_encode_rs8(prz
, (uint8_t *)buffer
, sizeof(*buffer
),
161 static void persistent_ram_ecc_old(struct persistent_ram_zone
*prz
)
163 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
170 block
= buffer
->data
;
171 par
= prz
->par_buffer
;
172 while (block
< buffer
->data
+ buffer_size(prz
)) {
174 int size
= prz
->ecc_block_size
;
175 if (block
+ size
> buffer
->data
+ prz
->buffer_size
)
176 size
= buffer
->data
+ prz
->buffer_size
- block
;
177 numerr
= persistent_ram_decode_rs8(prz
, block
, size
, par
);
179 pr_devel("persistent_ram: error in block %p, %d\n",
181 prz
->corrected_bytes
+= numerr
;
182 } else if (numerr
< 0) {
183 pr_devel("persistent_ram: uncorrectable error in block %p\n",
187 block
+= prz
->ecc_block_size
;
188 par
+= prz
->ecc_size
;
192 static int persistent_ram_init_ecc(struct persistent_ram_zone
*prz
,
196 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
202 prz
->ecc_block_size
= 128;
204 prz
->ecc_symsize
= 8;
205 prz
->ecc_poly
= 0x11d;
207 ecc_blocks
= DIV_ROUND_UP(prz
->buffer_size
, prz
->ecc_block_size
);
208 prz
->buffer_size
-= (ecc_blocks
+ 1) * prz
->ecc_size
;
210 if (prz
->buffer_size
> buffer_size
) {
211 pr_err("persistent_ram: invalid size %zu, non-ecc datasize %zu\n",
212 buffer_size
, prz
->buffer_size
);
216 prz
->par_buffer
= buffer
->data
+ prz
->buffer_size
;
217 prz
->par_header
= prz
->par_buffer
+ ecc_blocks
* prz
->ecc_size
;
220 * first consecutive root is 0
221 * primitive element to generate roots = 1
223 prz
->rs_decoder
= init_rs(prz
->ecc_symsize
, prz
->ecc_poly
, 0, 1,
225 if (prz
->rs_decoder
== NULL
) {
226 pr_info("persistent_ram: init_rs failed\n");
230 prz
->corrected_bytes
= 0;
233 numerr
= persistent_ram_decode_rs8(prz
, buffer
, sizeof(*buffer
),
236 pr_info("persistent_ram: error in header, %d\n", numerr
);
237 prz
->corrected_bytes
+= numerr
;
238 } else if (numerr
< 0) {
239 pr_info("persistent_ram: uncorrectable error in header\n");
246 ssize_t
persistent_ram_ecc_string(struct persistent_ram_zone
*prz
,
247 char *str
, size_t len
)
251 if (prz
->corrected_bytes
|| prz
->bad_blocks
)
252 ret
= snprintf(str
, len
, ""
253 "\n%d Corrected bytes, %d unrecoverable blocks\n",
254 prz
->corrected_bytes
, prz
->bad_blocks
);
256 ret
= snprintf(str
, len
, "\nNo errors detected\n");
261 static void notrace
persistent_ram_update(struct persistent_ram_zone
*prz
,
262 const void *s
, unsigned int start
, unsigned int count
)
264 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
265 memcpy(buffer
->data
+ start
, s
, count
);
266 persistent_ram_update_ecc(prz
, start
, count
);
270 persistent_ram_save_old(struct persistent_ram_zone
*prz
)
272 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
273 size_t size
= buffer_size(prz
);
274 size_t start
= buffer_start(prz
);
277 persistent_ram_ecc_old(prz
);
279 dest
= kmalloc(size
, GFP_KERNEL
);
281 pr_err("persistent_ram: failed to allocate buffer\n");
286 prz
->old_log_size
= size
;
287 memcpy(prz
->old_log
, &buffer
->data
[start
], size
- start
);
288 memcpy(prz
->old_log
+ size
- start
, &buffer
->data
[0], start
);
291 int notrace
persistent_ram_write(struct persistent_ram_zone
*prz
,
292 const void *s
, unsigned int count
)
298 if (unlikely(c
> prz
->buffer_size
)) {
299 s
+= c
- prz
->buffer_size
;
300 c
= prz
->buffer_size
;
303 buffer_size_add_clamp(prz
, c
);
305 start
= buffer_start_add(prz
, c
);
307 rem
= prz
->buffer_size
- start
;
308 if (unlikely(rem
< c
)) {
309 persistent_ram_update(prz
, s
, start
, rem
);
314 persistent_ram_update(prz
, s
, start
, c
);
316 persistent_ram_update_header_ecc(prz
);
321 size_t persistent_ram_old_size(struct persistent_ram_zone
*prz
)
323 return prz
->old_log_size
;
326 void *persistent_ram_old(struct persistent_ram_zone
*prz
)
331 void persistent_ram_free_old(struct persistent_ram_zone
*prz
)
335 prz
->old_log_size
= 0;
338 static int persistent_ram_buffer_map(phys_addr_t start
, phys_addr_t size
,
339 struct persistent_ram_zone
*prz
)
342 phys_addr_t page_start
;
343 unsigned int page_count
;
347 page_start
= start
- offset_in_page(start
);
348 page_count
= DIV_ROUND_UP(size
+ offset_in_page(start
), PAGE_SIZE
);
350 prot
= pgprot_noncached(PAGE_KERNEL
);
352 pages
= kmalloc(sizeof(struct page
*) * page_count
, GFP_KERNEL
);
354 pr_err("%s: Failed to allocate array for %u pages\n", __func__
,
359 for (i
= 0; i
< page_count
; i
++) {
360 phys_addr_t addr
= page_start
+ i
* PAGE_SIZE
;
361 pages
[i
] = pfn_to_page(addr
>> PAGE_SHIFT
);
363 prz
->vaddr
= vmap(pages
, page_count
, VM_MAP
, prot
);
366 pr_err("%s: Failed to map %u pages\n", __func__
, page_count
);
370 prz
->buffer
= prz
->vaddr
+ offset_in_page(start
);
371 prz
->buffer_size
= size
- sizeof(struct persistent_ram_buffer
);
376 static int __init
persistent_ram_buffer_init(const char *name
,
377 struct persistent_ram_zone
*prz
)
380 struct persistent_ram
*ram
;
381 struct persistent_ram_descriptor
*desc
;
384 list_for_each_entry(ram
, &persistent_ram_list
, node
) {
386 for (i
= 0; i
< ram
->num_descs
; i
++) {
387 desc
= &ram
->descs
[i
];
388 if (!strcmp(desc
->name
, name
))
389 return persistent_ram_buffer_map(start
,
399 struct persistent_ram_zone
*__persistent_ram_init(struct device
*dev
, bool ecc
)
401 struct persistent_ram_zone
*prz
;
404 prz
= kzalloc(sizeof(struct persistent_ram_zone
), GFP_KERNEL
);
406 pr_err("persistent_ram: failed to allocate persistent ram zone\n");
407 return ERR_PTR(-ENOMEM
);
410 INIT_LIST_HEAD(&prz
->node
);
412 ret
= persistent_ram_buffer_init(dev_name(dev
), prz
);
414 pr_err("persistent_ram: failed to initialize buffer\n");
419 ret
= persistent_ram_init_ecc(prz
, prz
->buffer_size
);
423 if (prz
->buffer
->sig
== PERSISTENT_RAM_SIG
) {
424 if (buffer_size(prz
) > prz
->buffer_size
||
425 buffer_start(prz
) > buffer_size(prz
))
426 pr_info("persistent_ram: found existing invalid buffer,"
427 " size %zu, start %zu\n",
428 buffer_size(prz
), buffer_start(prz
));
430 pr_info("persistent_ram: found existing buffer,"
431 " size %zu, start %zu\n",
432 buffer_size(prz
), buffer_start(prz
));
433 persistent_ram_save_old(prz
);
436 pr_info("persistent_ram: no valid data in buffer"
437 " (sig = 0x%08x)\n", prz
->buffer
->sig
);
440 prz
->buffer
->sig
= PERSISTENT_RAM_SIG
;
441 atomic_set(&prz
->buffer
->start
, 0);
442 atomic_set(&prz
->buffer
->size
, 0);
447 struct persistent_ram_zone
* __init
448 persistent_ram_init_ringbuffer(struct device
*dev
, bool ecc
)
450 return __persistent_ram_init(dev
, ecc
);
453 int __init
persistent_ram_early_init(struct persistent_ram
*ram
)
457 ret
= memblock_reserve(ram
->start
, ram
->size
);
459 pr_err("Failed to reserve persistent memory from %08lx-%08lx\n",
460 (long)ram
->start
, (long)(ram
->start
+ ram
->size
- 1));
464 list_add_tail(&ram
->node
, &persistent_ram_list
);
466 pr_info("Initialized persistent memory from %08lx-%08lx\n",
467 (long)ram
->start
, (long)(ram
->start
+ ram
->size
- 1));