Commit | Line | Data |
---|---|---|
18aecc2b CM |
1 | /* |
2 | * Copyright 2011 Tilera Corporation. All Rights Reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public License | |
6 | * as published by the Free Software Foundation, version 2. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, but | |
9 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or | |
11 | * NON INFRINGEMENT. See the GNU General Public License for | |
12 | * more details. | |
13 | */ | |
14 | ||
18aecc2b CM |
15 | #include <linux/types.h> |
16 | #include <linux/string.h> | |
17 | #include <linux/module.h> | |
c53c70a9 CM |
18 | #include <arch/chip.h> |
19 | #include "string-endian.h" | |
18aecc2b CM |
20 | |
21 | void *memset(void *s, int c, size_t n) | |
22 | { | |
23 | uint64_t *out64; | |
24 | int n64, to_align64; | |
25 | uint64_t v64; | |
26 | uint8_t *out8 = s; | |
27 | ||
28 | /* Experimentation shows that a trivial tight loop is a win up until | |
29 | * around a size of 20, where writing a word at a time starts to win. | |
30 | */ | |
31 | #define BYTE_CUTOFF 20 | |
32 | ||
33 | #if BYTE_CUTOFF < 7 | |
34 | /* This must be at least at least this big, or some code later | |
35 | * on doesn't work. | |
36 | */ | |
37 | #error "BYTE_CUTOFF is too small" | |
38 | #endif | |
39 | ||
40 | if (n < BYTE_CUTOFF) { | |
41 | /* Strangely, this turns out to be the tightest way to | |
42 | * write this loop. | |
43 | */ | |
44 | if (n != 0) { | |
45 | do { | |
46 | /* Strangely, combining these into one line | |
47 | * performs worse. | |
48 | */ | |
49 | *out8 = c; | |
50 | out8++; | |
51 | } while (--n != 0); | |
52 | } | |
53 | ||
54 | return s; | |
55 | } | |
56 | ||
57 | /* Align 'out8'. We know n >= 7 so this won't write past the end. */ | |
58 | while (((uintptr_t) out8 & 7) != 0) { | |
59 | *out8++ = c; | |
60 | --n; | |
61 | } | |
62 | ||
63 | /* Align 'n'. */ | |
64 | while (n & 7) | |
65 | out8[--n] = c; | |
66 | ||
67 | out64 = (uint64_t *) out8; | |
68 | n64 = n >> 3; | |
69 | ||
70 | /* Tile input byte out to 64 bits. */ | |
c53c70a9 | 71 | v64 = copy_byte(c); |
18aecc2b CM |
72 | |
73 | /* This must be at least 8 or the following loop doesn't work. */ | |
74 | #define CACHE_LINE_SIZE_IN_DOUBLEWORDS (CHIP_L2_LINE_SIZE() / 8) | |
75 | ||
76 | /* Determine how many words we need to emit before the 'out32' | |
77 | * pointer becomes aligned modulo the cache line size. | |
78 | */ | |
79 | to_align64 = (-((uintptr_t)out64 >> 3)) & | |
80 | (CACHE_LINE_SIZE_IN_DOUBLEWORDS - 1); | |
81 | ||
82 | /* Only bother aligning and using wh64 if there is at least | |
83 | * one full cache line to process. This check also prevents | |
84 | * overrunning the end of the buffer with alignment words. | |
85 | */ | |
86 | if (to_align64 <= n64 - CACHE_LINE_SIZE_IN_DOUBLEWORDS) { | |
87 | int lines_left; | |
88 | ||
89 | /* Align out64 mod the cache line size so we can use wh64. */ | |
90 | n64 -= to_align64; | |
91 | for (; to_align64 != 0; to_align64--) { | |
92 | *out64 = v64; | |
93 | out64++; | |
94 | } | |
95 | ||
96 | /* Use unsigned divide to turn this into a right shift. */ | |
97 | lines_left = (unsigned)n64 / CACHE_LINE_SIZE_IN_DOUBLEWORDS; | |
98 | ||
99 | do { | |
100 | /* Only wh64 a few lines at a time, so we don't | |
101 | * exceed the maximum number of victim lines. | |
102 | */ | |
103 | int x = ((lines_left < CHIP_MAX_OUTSTANDING_VICTIMS()) | |
104 | ? lines_left | |
105 | : CHIP_MAX_OUTSTANDING_VICTIMS()); | |
106 | uint64_t *wh = out64; | |
107 | int i = x; | |
108 | int j; | |
109 | ||
110 | lines_left -= x; | |
111 | ||
112 | do { | |
113 | __insn_wh64(wh); | |
114 | wh += CACHE_LINE_SIZE_IN_DOUBLEWORDS; | |
115 | } while (--i); | |
116 | ||
117 | for (j = x * (CACHE_LINE_SIZE_IN_DOUBLEWORDS / 4); | |
118 | j != 0; j--) { | |
119 | *out64++ = v64; | |
120 | *out64++ = v64; | |
121 | *out64++ = v64; | |
122 | *out64++ = v64; | |
123 | } | |
124 | } while (lines_left != 0); | |
125 | ||
126 | /* We processed all full lines above, so only this many | |
127 | * words remain to be processed. | |
128 | */ | |
129 | n64 &= CACHE_LINE_SIZE_IN_DOUBLEWORDS - 1; | |
130 | } | |
131 | ||
132 | /* Now handle any leftover values. */ | |
133 | if (n64 != 0) { | |
134 | do { | |
135 | *out64 = v64; | |
136 | out64++; | |
137 | } while (--n64 != 0); | |
138 | } | |
139 | ||
140 | return s; | |
141 | } | |
142 | EXPORT_SYMBOL(memset); |