2 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of version 2 of the GNU General Public License as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
13 #include <linux/list_sort.h>
14 #include <linux/libnvdimm.h>
15 #include <linux/module.h>
16 #include <linux/mutex.h>
17 #include <linux/ndctl.h>
18 #include <linux/list.h>
19 #include <linux/acpi.h>
20 #include <linux/sort.h>
21 #include <linux/pmem.h>
23 #include <asm/cacheflush.h>
27 * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is
30 #include <asm-generic/io-64-nonatomic-hi-lo.h>
32 static bool force_enable_dimms
;
33 module_param(force_enable_dimms
, bool, S_IRUGO
|S_IWUSR
);
34 MODULE_PARM_DESC(force_enable_dimms
, "Ignore _STA (ACPI DIMM device) status");
36 static u8 nfit_uuid
[NFIT_UUID_MAX
][16];
38 const u8
*to_nfit_uuid(enum nfit_uuids id
)
42 EXPORT_SYMBOL(to_nfit_uuid
);
44 static struct acpi_nfit_desc
*to_acpi_nfit_desc(
45 struct nvdimm_bus_descriptor
*nd_desc
)
47 return container_of(nd_desc
, struct acpi_nfit_desc
, nd_desc
);
50 static struct acpi_device
*to_acpi_dev(struct acpi_nfit_desc
*acpi_desc
)
52 struct nvdimm_bus_descriptor
*nd_desc
= &acpi_desc
->nd_desc
;
55 * If provider == 'ACPI.NFIT' we can assume 'dev' is a struct
58 if (!nd_desc
->provider_name
59 || strcmp(nd_desc
->provider_name
, "ACPI.NFIT") != 0)
62 return to_acpi_device(acpi_desc
->dev
);
65 static int acpi_nfit_ctl(struct nvdimm_bus_descriptor
*nd_desc
,
66 struct nvdimm
*nvdimm
, unsigned int cmd
, void *buf
,
69 struct acpi_nfit_desc
*acpi_desc
= to_acpi_nfit_desc(nd_desc
);
70 const struct nd_cmd_desc
*desc
= NULL
;
71 union acpi_object in_obj
, in_buf
, *out_obj
;
72 struct device
*dev
= acpi_desc
->dev
;
73 const char *cmd_name
, *dimm_name
;
74 unsigned long dsm_mask
;
81 struct nfit_mem
*nfit_mem
= nvdimm_provider_data(nvdimm
);
82 struct acpi_device
*adev
= nfit_mem
->adev
;
86 dimm_name
= nvdimm_name(nvdimm
);
87 cmd_name
= nvdimm_cmd_name(cmd
);
88 dsm_mask
= nfit_mem
->dsm_mask
;
89 desc
= nd_cmd_dimm_desc(cmd
);
90 uuid
= to_nfit_uuid(NFIT_DEV_DIMM
);
91 handle
= adev
->handle
;
93 struct acpi_device
*adev
= to_acpi_dev(acpi_desc
);
95 cmd_name
= nvdimm_bus_cmd_name(cmd
);
96 dsm_mask
= nd_desc
->dsm_mask
;
97 desc
= nd_cmd_bus_desc(cmd
);
98 uuid
= to_nfit_uuid(NFIT_DEV_BUS
);
99 handle
= adev
->handle
;
103 if (!desc
|| (cmd
&& (desc
->out_num
+ desc
->in_num
== 0)))
106 if (!test_bit(cmd
, &dsm_mask
))
109 in_obj
.type
= ACPI_TYPE_PACKAGE
;
110 in_obj
.package
.count
= 1;
111 in_obj
.package
.elements
= &in_buf
;
112 in_buf
.type
= ACPI_TYPE_BUFFER
;
113 in_buf
.buffer
.pointer
= buf
;
114 in_buf
.buffer
.length
= 0;
116 /* libnvdimm has already validated the input envelope */
117 for (i
= 0; i
< desc
->in_num
; i
++)
118 in_buf
.buffer
.length
+= nd_cmd_in_size(nvdimm
, cmd
, desc
,
121 if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG
)) {
122 dev_dbg(dev
, "%s:%s cmd: %s input length: %d\n", __func__
,
123 dimm_name
, cmd_name
, in_buf
.buffer
.length
);
124 print_hex_dump_debug(cmd_name
, DUMP_PREFIX_OFFSET
, 4,
125 4, in_buf
.buffer
.pointer
, min_t(u32
, 128,
126 in_buf
.buffer
.length
), true);
129 out_obj
= acpi_evaluate_dsm(handle
, uuid
, 1, cmd
, &in_obj
);
131 dev_dbg(dev
, "%s:%s _DSM failed cmd: %s\n", __func__
, dimm_name
,
136 if (out_obj
->package
.type
!= ACPI_TYPE_BUFFER
) {
137 dev_dbg(dev
, "%s:%s unexpected output object type cmd: %s type: %d\n",
138 __func__
, dimm_name
, cmd_name
, out_obj
->type
);
143 if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG
)) {
144 dev_dbg(dev
, "%s:%s cmd: %s output length: %d\n", __func__
,
145 dimm_name
, cmd_name
, out_obj
->buffer
.length
);
146 print_hex_dump_debug(cmd_name
, DUMP_PREFIX_OFFSET
, 4,
147 4, out_obj
->buffer
.pointer
, min_t(u32
, 128,
148 out_obj
->buffer
.length
), true);
151 for (i
= 0, offset
= 0; i
< desc
->out_num
; i
++) {
152 u32 out_size
= nd_cmd_out_size(nvdimm
, cmd
, desc
, i
, buf
,
153 (u32
*) out_obj
->buffer
.pointer
);
155 if (offset
+ out_size
> out_obj
->buffer
.length
) {
156 dev_dbg(dev
, "%s:%s output object underflow cmd: %s field: %d\n",
157 __func__
, dimm_name
, cmd_name
, i
);
161 if (in_buf
.buffer
.length
+ offset
+ out_size
> buf_len
) {
162 dev_dbg(dev
, "%s:%s output overrun cmd: %s field: %d\n",
163 __func__
, dimm_name
, cmd_name
, i
);
167 memcpy(buf
+ in_buf
.buffer
.length
+ offset
,
168 out_obj
->buffer
.pointer
+ offset
, out_size
);
171 if (offset
+ in_buf
.buffer
.length
< buf_len
) {
174 * status valid, return the number of bytes left
175 * unfilled in the output buffer
177 rc
= buf_len
- offset
- in_buf
.buffer
.length
;
179 dev_err(dev
, "%s:%s underrun cmd: %s buf_len: %d out_len: %d\n",
180 __func__
, dimm_name
, cmd_name
, buf_len
,
193 static const char *spa_type_name(u16 type
)
195 static const char *to_name
[] = {
196 [NFIT_SPA_VOLATILE
] = "volatile",
197 [NFIT_SPA_PM
] = "pmem",
198 [NFIT_SPA_DCR
] = "dimm-control-region",
199 [NFIT_SPA_BDW
] = "block-data-window",
200 [NFIT_SPA_VDISK
] = "volatile-disk",
201 [NFIT_SPA_VCD
] = "volatile-cd",
202 [NFIT_SPA_PDISK
] = "persistent-disk",
203 [NFIT_SPA_PCD
] = "persistent-cd",
207 if (type
> NFIT_SPA_PCD
)
210 return to_name
[type
];
213 static int nfit_spa_type(struct acpi_nfit_system_address
*spa
)
217 for (i
= 0; i
< NFIT_UUID_MAX
; i
++)
218 if (memcmp(to_nfit_uuid(i
), spa
->range_guid
, 16) == 0)
223 static bool add_spa(struct acpi_nfit_desc
*acpi_desc
,
224 struct acpi_nfit_system_address
*spa
)
226 struct device
*dev
= acpi_desc
->dev
;
227 struct nfit_spa
*nfit_spa
= devm_kzalloc(dev
, sizeof(*nfit_spa
),
232 INIT_LIST_HEAD(&nfit_spa
->list
);
234 list_add_tail(&nfit_spa
->list
, &acpi_desc
->spas
);
235 dev_dbg(dev
, "%s: spa index: %d type: %s\n", __func__
,
237 spa_type_name(nfit_spa_type(spa
)));
241 static bool add_memdev(struct acpi_nfit_desc
*acpi_desc
,
242 struct acpi_nfit_memory_map
*memdev
)
244 struct device
*dev
= acpi_desc
->dev
;
245 struct nfit_memdev
*nfit_memdev
= devm_kzalloc(dev
,
246 sizeof(*nfit_memdev
), GFP_KERNEL
);
250 INIT_LIST_HEAD(&nfit_memdev
->list
);
251 nfit_memdev
->memdev
= memdev
;
252 list_add_tail(&nfit_memdev
->list
, &acpi_desc
->memdevs
);
253 dev_dbg(dev
, "%s: memdev handle: %#x spa: %d dcr: %d\n",
254 __func__
, memdev
->device_handle
, memdev
->range_index
,
255 memdev
->region_index
);
259 static bool add_dcr(struct acpi_nfit_desc
*acpi_desc
,
260 struct acpi_nfit_control_region
*dcr
)
262 struct device
*dev
= acpi_desc
->dev
;
263 struct nfit_dcr
*nfit_dcr
= devm_kzalloc(dev
, sizeof(*nfit_dcr
),
268 INIT_LIST_HEAD(&nfit_dcr
->list
);
270 list_add_tail(&nfit_dcr
->list
, &acpi_desc
->dcrs
);
271 dev_dbg(dev
, "%s: dcr index: %d windows: %d\n", __func__
,
272 dcr
->region_index
, dcr
->windows
);
276 static bool add_bdw(struct acpi_nfit_desc
*acpi_desc
,
277 struct acpi_nfit_data_region
*bdw
)
279 struct device
*dev
= acpi_desc
->dev
;
280 struct nfit_bdw
*nfit_bdw
= devm_kzalloc(dev
, sizeof(*nfit_bdw
),
285 INIT_LIST_HEAD(&nfit_bdw
->list
);
287 list_add_tail(&nfit_bdw
->list
, &acpi_desc
->bdws
);
288 dev_dbg(dev
, "%s: bdw dcr: %d windows: %d\n", __func__
,
289 bdw
->region_index
, bdw
->windows
);
293 static bool add_idt(struct acpi_nfit_desc
*acpi_desc
,
294 struct acpi_nfit_interleave
*idt
)
296 struct device
*dev
= acpi_desc
->dev
;
297 struct nfit_idt
*nfit_idt
= devm_kzalloc(dev
, sizeof(*nfit_idt
),
302 INIT_LIST_HEAD(&nfit_idt
->list
);
304 list_add_tail(&nfit_idt
->list
, &acpi_desc
->idts
);
305 dev_dbg(dev
, "%s: idt index: %d num_lines: %d\n", __func__
,
306 idt
->interleave_index
, idt
->line_count
);
310 static bool add_flush(struct acpi_nfit_desc
*acpi_desc
,
311 struct acpi_nfit_flush_address
*flush
)
313 struct device
*dev
= acpi_desc
->dev
;
314 struct nfit_flush
*nfit_flush
= devm_kzalloc(dev
, sizeof(*nfit_flush
),
319 INIT_LIST_HEAD(&nfit_flush
->list
);
320 nfit_flush
->flush
= flush
;
321 list_add_tail(&nfit_flush
->list
, &acpi_desc
->flushes
);
322 dev_dbg(dev
, "%s: nfit_flush handle: %d hint_count: %d\n", __func__
,
323 flush
->device_handle
, flush
->hint_count
);
327 static void *add_table(struct acpi_nfit_desc
*acpi_desc
, void *table
,
330 struct device
*dev
= acpi_desc
->dev
;
331 struct acpi_nfit_header
*hdr
;
332 void *err
= ERR_PTR(-ENOMEM
);
339 case ACPI_NFIT_TYPE_SYSTEM_ADDRESS
:
340 if (!add_spa(acpi_desc
, table
))
343 case ACPI_NFIT_TYPE_MEMORY_MAP
:
344 if (!add_memdev(acpi_desc
, table
))
347 case ACPI_NFIT_TYPE_CONTROL_REGION
:
348 if (!add_dcr(acpi_desc
, table
))
351 case ACPI_NFIT_TYPE_DATA_REGION
:
352 if (!add_bdw(acpi_desc
, table
))
355 case ACPI_NFIT_TYPE_INTERLEAVE
:
356 if (!add_idt(acpi_desc
, table
))
359 case ACPI_NFIT_TYPE_FLUSH_ADDRESS
:
360 if (!add_flush(acpi_desc
, table
))
363 case ACPI_NFIT_TYPE_SMBIOS
:
364 dev_dbg(dev
, "%s: smbios\n", __func__
);
367 dev_err(dev
, "unknown table '%d' parsing nfit\n", hdr
->type
);
371 return table
+ hdr
->length
;
374 static void nfit_mem_find_spa_bdw(struct acpi_nfit_desc
*acpi_desc
,
375 struct nfit_mem
*nfit_mem
)
377 u32 device_handle
= __to_nfit_memdev(nfit_mem
)->device_handle
;
378 u16 dcr
= nfit_mem
->dcr
->region_index
;
379 struct nfit_spa
*nfit_spa
;
381 list_for_each_entry(nfit_spa
, &acpi_desc
->spas
, list
) {
382 u16 range_index
= nfit_spa
->spa
->range_index
;
383 int type
= nfit_spa_type(nfit_spa
->spa
);
384 struct nfit_memdev
*nfit_memdev
;
386 if (type
!= NFIT_SPA_BDW
)
389 list_for_each_entry(nfit_memdev
, &acpi_desc
->memdevs
, list
) {
390 if (nfit_memdev
->memdev
->range_index
!= range_index
)
392 if (nfit_memdev
->memdev
->device_handle
!= device_handle
)
394 if (nfit_memdev
->memdev
->region_index
!= dcr
)
397 nfit_mem
->spa_bdw
= nfit_spa
->spa
;
402 dev_dbg(acpi_desc
->dev
, "SPA-BDW not found for SPA-DCR %d\n",
403 nfit_mem
->spa_dcr
->range_index
);
404 nfit_mem
->bdw
= NULL
;
407 static int nfit_mem_add(struct acpi_nfit_desc
*acpi_desc
,
408 struct nfit_mem
*nfit_mem
, struct acpi_nfit_system_address
*spa
)
410 u16 dcr
= __to_nfit_memdev(nfit_mem
)->region_index
;
411 struct nfit_memdev
*nfit_memdev
;
412 struct nfit_flush
*nfit_flush
;
413 struct nfit_dcr
*nfit_dcr
;
414 struct nfit_bdw
*nfit_bdw
;
415 struct nfit_idt
*nfit_idt
;
416 u16 idt_idx
, range_index
;
418 list_for_each_entry(nfit_dcr
, &acpi_desc
->dcrs
, list
) {
419 if (nfit_dcr
->dcr
->region_index
!= dcr
)
421 nfit_mem
->dcr
= nfit_dcr
->dcr
;
425 if (!nfit_mem
->dcr
) {
426 dev_dbg(acpi_desc
->dev
, "SPA %d missing:%s%s\n",
427 spa
->range_index
, __to_nfit_memdev(nfit_mem
)
428 ? "" : " MEMDEV", nfit_mem
->dcr
? "" : " DCR");
433 * We've found enough to create an nvdimm, optionally
434 * find an associated BDW
436 list_add(&nfit_mem
->list
, &acpi_desc
->dimms
);
438 list_for_each_entry(nfit_bdw
, &acpi_desc
->bdws
, list
) {
439 if (nfit_bdw
->bdw
->region_index
!= dcr
)
441 nfit_mem
->bdw
= nfit_bdw
->bdw
;
448 nfit_mem_find_spa_bdw(acpi_desc
, nfit_mem
);
450 if (!nfit_mem
->spa_bdw
)
453 range_index
= nfit_mem
->spa_bdw
->range_index
;
454 list_for_each_entry(nfit_memdev
, &acpi_desc
->memdevs
, list
) {
455 if (nfit_memdev
->memdev
->range_index
!= range_index
||
456 nfit_memdev
->memdev
->region_index
!= dcr
)
458 nfit_mem
->memdev_bdw
= nfit_memdev
->memdev
;
459 idt_idx
= nfit_memdev
->memdev
->interleave_index
;
460 list_for_each_entry(nfit_idt
, &acpi_desc
->idts
, list
) {
461 if (nfit_idt
->idt
->interleave_index
!= idt_idx
)
463 nfit_mem
->idt_bdw
= nfit_idt
->idt
;
467 list_for_each_entry(nfit_flush
, &acpi_desc
->flushes
, list
) {
468 if (nfit_flush
->flush
->device_handle
!=
469 nfit_memdev
->memdev
->device_handle
)
471 nfit_mem
->nfit_flush
= nfit_flush
;
480 static int nfit_mem_dcr_init(struct acpi_nfit_desc
*acpi_desc
,
481 struct acpi_nfit_system_address
*spa
)
483 struct nfit_mem
*nfit_mem
, *found
;
484 struct nfit_memdev
*nfit_memdev
;
485 int type
= nfit_spa_type(spa
);
496 list_for_each_entry(nfit_memdev
, &acpi_desc
->memdevs
, list
) {
499 if (nfit_memdev
->memdev
->range_index
!= spa
->range_index
)
502 dcr
= nfit_memdev
->memdev
->region_index
;
503 list_for_each_entry(nfit_mem
, &acpi_desc
->dimms
, list
)
504 if (__to_nfit_memdev(nfit_mem
)->region_index
== dcr
) {
512 nfit_mem
= devm_kzalloc(acpi_desc
->dev
,
513 sizeof(*nfit_mem
), GFP_KERNEL
);
516 INIT_LIST_HEAD(&nfit_mem
->list
);
519 if (type
== NFIT_SPA_DCR
) {
520 struct nfit_idt
*nfit_idt
;
523 /* multiple dimms may share a SPA when interleaved */
524 nfit_mem
->spa_dcr
= spa
;
525 nfit_mem
->memdev_dcr
= nfit_memdev
->memdev
;
526 idt_idx
= nfit_memdev
->memdev
->interleave_index
;
527 list_for_each_entry(nfit_idt
, &acpi_desc
->idts
, list
) {
528 if (nfit_idt
->idt
->interleave_index
!= idt_idx
)
530 nfit_mem
->idt_dcr
= nfit_idt
->idt
;
535 * A single dimm may belong to multiple SPA-PM
536 * ranges, record at least one in addition to
539 nfit_mem
->memdev_pmem
= nfit_memdev
->memdev
;
545 rc
= nfit_mem_add(acpi_desc
, nfit_mem
, spa
);
553 static int nfit_mem_cmp(void *priv
, struct list_head
*_a
, struct list_head
*_b
)
555 struct nfit_mem
*a
= container_of(_a
, typeof(*a
), list
);
556 struct nfit_mem
*b
= container_of(_b
, typeof(*b
), list
);
557 u32 handleA
, handleB
;
559 handleA
= __to_nfit_memdev(a
)->device_handle
;
560 handleB
= __to_nfit_memdev(b
)->device_handle
;
561 if (handleA
< handleB
)
563 else if (handleA
> handleB
)
568 static int nfit_mem_init(struct acpi_nfit_desc
*acpi_desc
)
570 struct nfit_spa
*nfit_spa
;
573 * For each SPA-DCR or SPA-PMEM address range find its
574 * corresponding MEMDEV(s). From each MEMDEV find the
575 * corresponding DCR. Then, if we're operating on a SPA-DCR,
576 * try to find a SPA-BDW and a corresponding BDW that references
577 * the DCR. Throw it all into an nfit_mem object. Note, that
580 list_for_each_entry(nfit_spa
, &acpi_desc
->spas
, list
) {
583 rc
= nfit_mem_dcr_init(acpi_desc
, nfit_spa
->spa
);
588 list_sort(NULL
, &acpi_desc
->dimms
, nfit_mem_cmp
);
593 static ssize_t
revision_show(struct device
*dev
,
594 struct device_attribute
*attr
, char *buf
)
596 struct nvdimm_bus
*nvdimm_bus
= to_nvdimm_bus(dev
);
597 struct nvdimm_bus_descriptor
*nd_desc
= to_nd_desc(nvdimm_bus
);
598 struct acpi_nfit_desc
*acpi_desc
= to_acpi_desc(nd_desc
);
600 return sprintf(buf
, "%d\n", acpi_desc
->nfit
->header
.revision
);
602 static DEVICE_ATTR_RO(revision
);
604 static struct attribute
*acpi_nfit_attributes
[] = {
605 &dev_attr_revision
.attr
,
609 static struct attribute_group acpi_nfit_attribute_group
= {
611 .attrs
= acpi_nfit_attributes
,
614 const struct attribute_group
*acpi_nfit_attribute_groups
[] = {
615 &nvdimm_bus_attribute_group
,
616 &acpi_nfit_attribute_group
,
619 EXPORT_SYMBOL_GPL(acpi_nfit_attribute_groups
);
621 static struct acpi_nfit_memory_map
*to_nfit_memdev(struct device
*dev
)
623 struct nvdimm
*nvdimm
= to_nvdimm(dev
);
624 struct nfit_mem
*nfit_mem
= nvdimm_provider_data(nvdimm
);
626 return __to_nfit_memdev(nfit_mem
);
629 static struct acpi_nfit_control_region
*to_nfit_dcr(struct device
*dev
)
631 struct nvdimm
*nvdimm
= to_nvdimm(dev
);
632 struct nfit_mem
*nfit_mem
= nvdimm_provider_data(nvdimm
);
634 return nfit_mem
->dcr
;
637 static ssize_t
handle_show(struct device
*dev
,
638 struct device_attribute
*attr
, char *buf
)
640 struct acpi_nfit_memory_map
*memdev
= to_nfit_memdev(dev
);
642 return sprintf(buf
, "%#x\n", memdev
->device_handle
);
644 static DEVICE_ATTR_RO(handle
);
646 static ssize_t
phys_id_show(struct device
*dev
,
647 struct device_attribute
*attr
, char *buf
)
649 struct acpi_nfit_memory_map
*memdev
= to_nfit_memdev(dev
);
651 return sprintf(buf
, "%#x\n", memdev
->physical_id
);
653 static DEVICE_ATTR_RO(phys_id
);
655 static ssize_t
vendor_show(struct device
*dev
,
656 struct device_attribute
*attr
, char *buf
)
658 struct acpi_nfit_control_region
*dcr
= to_nfit_dcr(dev
);
660 return sprintf(buf
, "%#x\n", dcr
->vendor_id
);
662 static DEVICE_ATTR_RO(vendor
);
664 static ssize_t
rev_id_show(struct device
*dev
,
665 struct device_attribute
*attr
, char *buf
)
667 struct acpi_nfit_control_region
*dcr
= to_nfit_dcr(dev
);
669 return sprintf(buf
, "%#x\n", dcr
->revision_id
);
671 static DEVICE_ATTR_RO(rev_id
);
673 static ssize_t
device_show(struct device
*dev
,
674 struct device_attribute
*attr
, char *buf
)
676 struct acpi_nfit_control_region
*dcr
= to_nfit_dcr(dev
);
678 return sprintf(buf
, "%#x\n", dcr
->device_id
);
680 static DEVICE_ATTR_RO(device
);
682 static ssize_t
format_show(struct device
*dev
,
683 struct device_attribute
*attr
, char *buf
)
685 struct acpi_nfit_control_region
*dcr
= to_nfit_dcr(dev
);
687 return sprintf(buf
, "%#x\n", dcr
->code
);
689 static DEVICE_ATTR_RO(format
);
691 static ssize_t
serial_show(struct device
*dev
,
692 struct device_attribute
*attr
, char *buf
)
694 struct acpi_nfit_control_region
*dcr
= to_nfit_dcr(dev
);
696 return sprintf(buf
, "%#x\n", dcr
->serial_number
);
698 static DEVICE_ATTR_RO(serial
);
700 static ssize_t
flags_show(struct device
*dev
,
701 struct device_attribute
*attr
, char *buf
)
703 u16 flags
= to_nfit_memdev(dev
)->flags
;
705 return sprintf(buf
, "%s%s%s%s%s\n",
706 flags
& ACPI_NFIT_MEM_SAVE_FAILED
? "save_fail " : "",
707 flags
& ACPI_NFIT_MEM_RESTORE_FAILED
? "restore_fail " : "",
708 flags
& ACPI_NFIT_MEM_FLUSH_FAILED
? "flush_fail " : "",
709 flags
& ACPI_NFIT_MEM_ARMED
? "not_armed " : "",
710 flags
& ACPI_NFIT_MEM_HEALTH_OBSERVED
? "smart_event " : "");
712 static DEVICE_ATTR_RO(flags
);
714 static struct attribute
*acpi_nfit_dimm_attributes
[] = {
715 &dev_attr_handle
.attr
,
716 &dev_attr_phys_id
.attr
,
717 &dev_attr_vendor
.attr
,
718 &dev_attr_device
.attr
,
719 &dev_attr_format
.attr
,
720 &dev_attr_serial
.attr
,
721 &dev_attr_rev_id
.attr
,
722 &dev_attr_flags
.attr
,
726 static umode_t
acpi_nfit_dimm_attr_visible(struct kobject
*kobj
,
727 struct attribute
*a
, int n
)
729 struct device
*dev
= container_of(kobj
, struct device
, kobj
);
731 if (to_nfit_dcr(dev
))
737 static struct attribute_group acpi_nfit_dimm_attribute_group
= {
739 .attrs
= acpi_nfit_dimm_attributes
,
740 .is_visible
= acpi_nfit_dimm_attr_visible
,
743 static const struct attribute_group
*acpi_nfit_dimm_attribute_groups
[] = {
744 &nvdimm_attribute_group
,
745 &nd_device_attribute_group
,
746 &acpi_nfit_dimm_attribute_group
,
750 static struct nvdimm
*acpi_nfit_dimm_by_handle(struct acpi_nfit_desc
*acpi_desc
,
753 struct nfit_mem
*nfit_mem
;
755 list_for_each_entry(nfit_mem
, &acpi_desc
->dimms
, list
)
756 if (__to_nfit_memdev(nfit_mem
)->device_handle
== device_handle
)
757 return nfit_mem
->nvdimm
;
762 static int acpi_nfit_add_dimm(struct acpi_nfit_desc
*acpi_desc
,
763 struct nfit_mem
*nfit_mem
, u32 device_handle
)
765 struct acpi_device
*adev
, *adev_dimm
;
766 struct device
*dev
= acpi_desc
->dev
;
767 const u8
*uuid
= to_nfit_uuid(NFIT_DEV_DIMM
);
770 nfit_mem
->dsm_mask
= acpi_desc
->dimm_dsm_force_en
;
771 adev
= to_acpi_dev(acpi_desc
);
775 adev_dimm
= acpi_find_child_device(adev
, device_handle
, false);
776 nfit_mem
->adev
= adev_dimm
;
778 dev_err(dev
, "no ACPI.NFIT device with _ADR %#x, disabling...\n",
780 return force_enable_dimms
? 0 : -ENODEV
;
783 for (i
= ND_CMD_SMART
; i
<= ND_CMD_VENDOR
; i
++)
784 if (acpi_check_dsm(adev_dimm
->handle
, uuid
, 1, 1ULL << i
))
785 set_bit(i
, &nfit_mem
->dsm_mask
);
790 static int acpi_nfit_register_dimms(struct acpi_nfit_desc
*acpi_desc
)
792 struct nfit_mem
*nfit_mem
;
795 list_for_each_entry(nfit_mem
, &acpi_desc
->dimms
, list
) {
796 struct nvdimm
*nvdimm
;
797 unsigned long flags
= 0;
802 device_handle
= __to_nfit_memdev(nfit_mem
)->device_handle
;
803 nvdimm
= acpi_nfit_dimm_by_handle(acpi_desc
, device_handle
);
806 * If for some reason we find multiple DCRs the
809 dev_err(acpi_desc
->dev
, "duplicate DCR detected: %s\n",
810 nvdimm_name(nvdimm
));
814 if (nfit_mem
->bdw
&& nfit_mem
->memdev_pmem
)
815 flags
|= NDD_ALIASING
;
817 mem_flags
= __to_nfit_memdev(nfit_mem
)->flags
;
818 if (mem_flags
& ACPI_NFIT_MEM_ARMED
)
819 flags
|= NDD_UNARMED
;
821 rc
= acpi_nfit_add_dimm(acpi_desc
, nfit_mem
, device_handle
);
825 nvdimm
= nvdimm_create(acpi_desc
->nvdimm_bus
, nfit_mem
,
826 acpi_nfit_dimm_attribute_groups
,
827 flags
, &nfit_mem
->dsm_mask
);
831 nfit_mem
->nvdimm
= nvdimm
;
834 if ((mem_flags
& ACPI_NFIT_MEM_FAILED_MASK
) == 0)
837 dev_info(acpi_desc
->dev
, "%s flags:%s%s%s%s\n",
839 mem_flags
& ACPI_NFIT_MEM_SAVE_FAILED
? " save_fail" : "",
840 mem_flags
& ACPI_NFIT_MEM_RESTORE_FAILED
? " restore_fail":"",
841 mem_flags
& ACPI_NFIT_MEM_FLUSH_FAILED
? " flush_fail" : "",
842 mem_flags
& ACPI_NFIT_MEM_ARMED
? " not_armed" : "");
846 return nvdimm_bus_check_dimm_count(acpi_desc
->nvdimm_bus
, dimm_count
);
849 static void acpi_nfit_init_dsms(struct acpi_nfit_desc
*acpi_desc
)
851 struct nvdimm_bus_descriptor
*nd_desc
= &acpi_desc
->nd_desc
;
852 const u8
*uuid
= to_nfit_uuid(NFIT_DEV_BUS
);
853 struct acpi_device
*adev
;
856 nd_desc
->dsm_mask
= acpi_desc
->bus_dsm_force_en
;
857 adev
= to_acpi_dev(acpi_desc
);
861 for (i
= ND_CMD_ARS_CAP
; i
<= ND_CMD_ARS_STATUS
; i
++)
862 if (acpi_check_dsm(adev
->handle
, uuid
, 1, 1ULL << i
))
863 set_bit(i
, &nd_desc
->dsm_mask
);
866 static ssize_t
range_index_show(struct device
*dev
,
867 struct device_attribute
*attr
, char *buf
)
869 struct nd_region
*nd_region
= to_nd_region(dev
);
870 struct nfit_spa
*nfit_spa
= nd_region_provider_data(nd_region
);
872 return sprintf(buf
, "%d\n", nfit_spa
->spa
->range_index
);
874 static DEVICE_ATTR_RO(range_index
);
876 static struct attribute
*acpi_nfit_region_attributes
[] = {
877 &dev_attr_range_index
.attr
,
881 static struct attribute_group acpi_nfit_region_attribute_group
= {
883 .attrs
= acpi_nfit_region_attributes
,
886 static const struct attribute_group
*acpi_nfit_region_attribute_groups
[] = {
887 &nd_region_attribute_group
,
888 &nd_mapping_attribute_group
,
889 &nd_device_attribute_group
,
890 &nd_numa_attribute_group
,
891 &acpi_nfit_region_attribute_group
,
895 /* enough info to uniquely specify an interleave set */
896 struct nfit_set_info
{
897 struct nfit_set_info_map
{
904 static size_t sizeof_nfit_set_info(int num_mappings
)
906 return sizeof(struct nfit_set_info
)
907 + num_mappings
* sizeof(struct nfit_set_info_map
);
910 static int cmp_map(const void *m0
, const void *m1
)
912 const struct nfit_set_info_map
*map0
= m0
;
913 const struct nfit_set_info_map
*map1
= m1
;
915 return memcmp(&map0
->region_offset
, &map1
->region_offset
,
919 /* Retrieve the nth entry referencing this spa */
920 static struct acpi_nfit_memory_map
*memdev_from_spa(
921 struct acpi_nfit_desc
*acpi_desc
, u16 range_index
, int n
)
923 struct nfit_memdev
*nfit_memdev
;
925 list_for_each_entry(nfit_memdev
, &acpi_desc
->memdevs
, list
)
926 if (nfit_memdev
->memdev
->range_index
== range_index
)
928 return nfit_memdev
->memdev
;
932 static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc
*acpi_desc
,
933 struct nd_region_desc
*ndr_desc
,
934 struct acpi_nfit_system_address
*spa
)
936 int i
, spa_type
= nfit_spa_type(spa
);
937 struct device
*dev
= acpi_desc
->dev
;
938 struct nd_interleave_set
*nd_set
;
939 u16 nr
= ndr_desc
->num_mappings
;
940 struct nfit_set_info
*info
;
942 if (spa_type
== NFIT_SPA_PM
|| spa_type
== NFIT_SPA_VOLATILE
)
947 nd_set
= devm_kzalloc(dev
, sizeof(*nd_set
), GFP_KERNEL
);
951 info
= devm_kzalloc(dev
, sizeof_nfit_set_info(nr
), GFP_KERNEL
);
954 for (i
= 0; i
< nr
; i
++) {
955 struct nd_mapping
*nd_mapping
= &ndr_desc
->nd_mapping
[i
];
956 struct nfit_set_info_map
*map
= &info
->mapping
[i
];
957 struct nvdimm
*nvdimm
= nd_mapping
->nvdimm
;
958 struct nfit_mem
*nfit_mem
= nvdimm_provider_data(nvdimm
);
959 struct acpi_nfit_memory_map
*memdev
= memdev_from_spa(acpi_desc
,
960 spa
->range_index
, i
);
962 if (!memdev
|| !nfit_mem
->dcr
) {
963 dev_err(dev
, "%s: failed to find DCR\n", __func__
);
967 map
->region_offset
= memdev
->region_offset
;
968 map
->serial_number
= nfit_mem
->dcr
->serial_number
;
971 sort(&info
->mapping
[0], nr
, sizeof(struct nfit_set_info_map
),
973 nd_set
->cookie
= nd_fletcher64(info
, sizeof_nfit_set_info(nr
), 0);
974 ndr_desc
->nd_set
= nd_set
;
975 devm_kfree(dev
, info
);
980 static u64
to_interleave_offset(u64 offset
, struct nfit_blk_mmio
*mmio
)
982 struct acpi_nfit_interleave
*idt
= mmio
->idt
;
983 u32 sub_line_offset
, line_index
, line_offset
;
984 u64 line_no
, table_skip_count
, table_offset
;
986 line_no
= div_u64_rem(offset
, mmio
->line_size
, &sub_line_offset
);
987 table_skip_count
= div_u64_rem(line_no
, mmio
->num_lines
, &line_index
);
988 line_offset
= idt
->line_offset
[line_index
]
990 table_offset
= table_skip_count
* mmio
->table_size
;
992 return mmio
->base_offset
+ line_offset
+ table_offset
+ sub_line_offset
;
995 static void wmb_blk(struct nfit_blk
*nfit_blk
)
998 if (nfit_blk
->nvdimm_flush
) {
1000 * The first wmb() is needed to 'sfence' all previous writes
1001 * such that they are architecturally visible for the platform
1002 * buffer flush. Note that we've already arranged for pmem
1003 * writes to avoid the cache via arch_memcpy_to_pmem(). The
1004 * final wmb() ensures ordering for the NVDIMM flush write.
1007 writeq(1, nfit_blk
->nvdimm_flush
);
1013 static u32
read_blk_stat(struct nfit_blk
*nfit_blk
, unsigned int bw
)
1015 struct nfit_blk_mmio
*mmio
= &nfit_blk
->mmio
[DCR
];
1016 u64 offset
= nfit_blk
->stat_offset
+ mmio
->size
* bw
;
1018 if (mmio
->num_lines
)
1019 offset
= to_interleave_offset(offset
, mmio
);
1021 return readl(mmio
->addr
.base
+ offset
);
1024 static void write_blk_ctl(struct nfit_blk
*nfit_blk
, unsigned int bw
,
1025 resource_size_t dpa
, unsigned int len
, unsigned int write
)
1028 struct nfit_blk_mmio
*mmio
= &nfit_blk
->mmio
[DCR
];
1031 BCW_OFFSET_MASK
= (1ULL << 48)-1,
1033 BCW_LEN_MASK
= (1ULL << 8) - 1,
1037 cmd
= (dpa
>> L1_CACHE_SHIFT
) & BCW_OFFSET_MASK
;
1038 len
= len
>> L1_CACHE_SHIFT
;
1039 cmd
|= ((u64
) len
& BCW_LEN_MASK
) << BCW_LEN_SHIFT
;
1040 cmd
|= ((u64
) write
) << BCW_CMD_SHIFT
;
1042 offset
= nfit_blk
->cmd_offset
+ mmio
->size
* bw
;
1043 if (mmio
->num_lines
)
1044 offset
= to_interleave_offset(offset
, mmio
);
1046 writeq(cmd
, mmio
->addr
.base
+ offset
);
1049 if (nfit_blk
->dimm_flags
& ND_BLK_DCR_LATCH
)
1050 readq(mmio
->addr
.base
+ offset
);
1053 static int acpi_nfit_blk_single_io(struct nfit_blk
*nfit_blk
,
1054 resource_size_t dpa
, void *iobuf
, size_t len
, int rw
,
1057 struct nfit_blk_mmio
*mmio
= &nfit_blk
->mmio
[BDW
];
1058 unsigned int copied
= 0;
1062 base_offset
= nfit_blk
->bdw_offset
+ dpa
% L1_CACHE_BYTES
1063 + lane
* mmio
->size
;
1064 write_blk_ctl(nfit_blk
, lane
, dpa
, len
, rw
);
1069 if (mmio
->num_lines
) {
1072 offset
= to_interleave_offset(base_offset
+ copied
,
1074 div_u64_rem(offset
, mmio
->line_size
, &line_offset
);
1075 c
= min_t(size_t, len
, mmio
->line_size
- line_offset
);
1077 offset
= base_offset
+ nfit_blk
->bdw_offset
;
1082 memcpy_to_pmem(mmio
->addr
.aperture
+ offset
,
1085 if (nfit_blk
->dimm_flags
& ND_BLK_READ_FLUSH
)
1086 mmio_flush_range((void __force
*)
1087 mmio
->addr
.aperture
+ offset
, c
);
1089 memcpy_from_pmem(iobuf
+ copied
,
1090 mmio
->addr
.aperture
+ offset
, c
);
1100 rc
= read_blk_stat(nfit_blk
, lane
) ? -EIO
: 0;
1104 static int acpi_nfit_blk_region_do_io(struct nd_blk_region
*ndbr
,
1105 resource_size_t dpa
, void *iobuf
, u64 len
, int rw
)
1107 struct nfit_blk
*nfit_blk
= nd_blk_region_provider_data(ndbr
);
1108 struct nfit_blk_mmio
*mmio
= &nfit_blk
->mmio
[BDW
];
1109 struct nd_region
*nd_region
= nfit_blk
->nd_region
;
1110 unsigned int lane
, copied
= 0;
1113 lane
= nd_region_acquire_lane(nd_region
);
1115 u64 c
= min(len
, mmio
->size
);
1117 rc
= acpi_nfit_blk_single_io(nfit_blk
, dpa
+ copied
,
1118 iobuf
+ copied
, c
, rw
, lane
);
1125 nd_region_release_lane(nd_region
, lane
);
1130 static void nfit_spa_mapping_release(struct kref
*kref
)
1132 struct nfit_spa_mapping
*spa_map
= to_spa_map(kref
);
1133 struct acpi_nfit_system_address
*spa
= spa_map
->spa
;
1134 struct acpi_nfit_desc
*acpi_desc
= spa_map
->acpi_desc
;
1136 WARN_ON(!mutex_is_locked(&acpi_desc
->spa_map_mutex
));
1137 dev_dbg(acpi_desc
->dev
, "%s: SPA%d\n", __func__
, spa
->range_index
);
1138 if (spa_map
->type
== SPA_MAP_APERTURE
)
1139 memunmap((void __force
*)spa_map
->addr
.aperture
);
1141 iounmap(spa_map
->addr
.base
);
1142 release_mem_region(spa
->address
, spa
->length
);
1143 list_del(&spa_map
->list
);
1147 static struct nfit_spa_mapping
*find_spa_mapping(
1148 struct acpi_nfit_desc
*acpi_desc
,
1149 struct acpi_nfit_system_address
*spa
)
1151 struct nfit_spa_mapping
*spa_map
;
1153 WARN_ON(!mutex_is_locked(&acpi_desc
->spa_map_mutex
));
1154 list_for_each_entry(spa_map
, &acpi_desc
->spa_maps
, list
)
1155 if (spa_map
->spa
== spa
)
1161 static void nfit_spa_unmap(struct acpi_nfit_desc
*acpi_desc
,
1162 struct acpi_nfit_system_address
*spa
)
1164 struct nfit_spa_mapping
*spa_map
;
1166 mutex_lock(&acpi_desc
->spa_map_mutex
);
1167 spa_map
= find_spa_mapping(acpi_desc
, spa
);
1170 kref_put(&spa_map
->kref
, nfit_spa_mapping_release
);
1171 mutex_unlock(&acpi_desc
->spa_map_mutex
);
1174 static void __iomem
*__nfit_spa_map(struct acpi_nfit_desc
*acpi_desc
,
1175 struct acpi_nfit_system_address
*spa
, enum spa_map_type type
)
1177 resource_size_t start
= spa
->address
;
1178 resource_size_t n
= spa
->length
;
1179 struct nfit_spa_mapping
*spa_map
;
1180 struct resource
*res
;
1182 WARN_ON(!mutex_is_locked(&acpi_desc
->spa_map_mutex
));
1184 spa_map
= find_spa_mapping(acpi_desc
, spa
);
1186 kref_get(&spa_map
->kref
);
1187 return spa_map
->addr
.base
;
1190 spa_map
= kzalloc(sizeof(*spa_map
), GFP_KERNEL
);
1194 INIT_LIST_HEAD(&spa_map
->list
);
1196 kref_init(&spa_map
->kref
);
1197 spa_map
->acpi_desc
= acpi_desc
;
1199 res
= request_mem_region(start
, n
, dev_name(acpi_desc
->dev
));
1203 spa_map
->type
= type
;
1204 if (type
== SPA_MAP_APERTURE
)
1205 spa_map
->addr
.aperture
= (void __pmem
*)memremap(start
, n
,
1206 ARCH_MEMREMAP_PMEM
);
1208 spa_map
->addr
.base
= ioremap_nocache(start
, n
);
1211 if (!spa_map
->addr
.base
)
1214 list_add_tail(&spa_map
->list
, &acpi_desc
->spa_maps
);
1215 return spa_map
->addr
.base
;
1218 release_mem_region(start
, n
);
1225 * nfit_spa_map - interleave-aware managed-mappings of acpi_nfit_system_address ranges
1226 * @nvdimm_bus: NFIT-bus that provided the spa table entry
1227 * @nfit_spa: spa table to map
1228 * @type: aperture or control region
1230 * In the case where block-data-window apertures and
1231 * dimm-control-regions are interleaved they will end up sharing a
1232 * single request_mem_region() + ioremap() for the address range. In
1233 * the style of devm nfit_spa_map() mappings are automatically dropped
1234 * when all region devices referencing the same mapping are disabled /
1237 static void __iomem
*nfit_spa_map(struct acpi_nfit_desc
*acpi_desc
,
1238 struct acpi_nfit_system_address
*spa
, enum spa_map_type type
)
1240 void __iomem
*iomem
;
1242 mutex_lock(&acpi_desc
->spa_map_mutex
);
1243 iomem
= __nfit_spa_map(acpi_desc
, spa
, type
);
1244 mutex_unlock(&acpi_desc
->spa_map_mutex
);
1249 static int nfit_blk_init_interleave(struct nfit_blk_mmio
*mmio
,
1250 struct acpi_nfit_interleave
*idt
, u16 interleave_ways
)
1253 mmio
->num_lines
= idt
->line_count
;
1254 mmio
->line_size
= idt
->line_size
;
1255 if (interleave_ways
== 0)
1257 mmio
->table_size
= mmio
->num_lines
* interleave_ways
1264 static int acpi_nfit_blk_get_flags(struct nvdimm_bus_descriptor
*nd_desc
,
1265 struct nvdimm
*nvdimm
, struct nfit_blk
*nfit_blk
)
1267 struct nd_cmd_dimm_flags flags
;
1270 memset(&flags
, 0, sizeof(flags
));
1271 rc
= nd_desc
->ndctl(nd_desc
, nvdimm
, ND_CMD_DIMM_FLAGS
, &flags
,
1274 if (rc
>= 0 && flags
.status
== 0)
1275 nfit_blk
->dimm_flags
= flags
.flags
;
1276 else if (rc
== -ENOTTY
) {
1277 /* fall back to a conservative default */
1278 nfit_blk
->dimm_flags
= ND_BLK_DCR_LATCH
| ND_BLK_READ_FLUSH
;
1286 static int acpi_nfit_blk_region_enable(struct nvdimm_bus
*nvdimm_bus
,
1289 struct nvdimm_bus_descriptor
*nd_desc
= to_nd_desc(nvdimm_bus
);
1290 struct acpi_nfit_desc
*acpi_desc
= to_acpi_desc(nd_desc
);
1291 struct nd_blk_region
*ndbr
= to_nd_blk_region(dev
);
1292 struct nfit_flush
*nfit_flush
;
1293 struct nfit_blk_mmio
*mmio
;
1294 struct nfit_blk
*nfit_blk
;
1295 struct nfit_mem
*nfit_mem
;
1296 struct nvdimm
*nvdimm
;
1299 nvdimm
= nd_blk_region_to_dimm(ndbr
);
1300 nfit_mem
= nvdimm_provider_data(nvdimm
);
1301 if (!nfit_mem
|| !nfit_mem
->dcr
|| !nfit_mem
->bdw
) {
1302 dev_dbg(dev
, "%s: missing%s%s%s\n", __func__
,
1303 nfit_mem
? "" : " nfit_mem",
1304 (nfit_mem
&& nfit_mem
->dcr
) ? "" : " dcr",
1305 (nfit_mem
&& nfit_mem
->bdw
) ? "" : " bdw");
1309 nfit_blk
= devm_kzalloc(dev
, sizeof(*nfit_blk
), GFP_KERNEL
);
1312 nd_blk_region_set_provider_data(ndbr
, nfit_blk
);
1313 nfit_blk
->nd_region
= to_nd_region(dev
);
1315 /* map block aperture memory */
1316 nfit_blk
->bdw_offset
= nfit_mem
->bdw
->offset
;
1317 mmio
= &nfit_blk
->mmio
[BDW
];
1318 mmio
->addr
.base
= nfit_spa_map(acpi_desc
, nfit_mem
->spa_bdw
,
1320 if (!mmio
->addr
.base
) {
1321 dev_dbg(dev
, "%s: %s failed to map bdw\n", __func__
,
1322 nvdimm_name(nvdimm
));
1325 mmio
->size
= nfit_mem
->bdw
->size
;
1326 mmio
->base_offset
= nfit_mem
->memdev_bdw
->region_offset
;
1327 mmio
->idt
= nfit_mem
->idt_bdw
;
1328 mmio
->spa
= nfit_mem
->spa_bdw
;
1329 rc
= nfit_blk_init_interleave(mmio
, nfit_mem
->idt_bdw
,
1330 nfit_mem
->memdev_bdw
->interleave_ways
);
1332 dev_dbg(dev
, "%s: %s failed to init bdw interleave\n",
1333 __func__
, nvdimm_name(nvdimm
));
1337 /* map block control memory */
1338 nfit_blk
->cmd_offset
= nfit_mem
->dcr
->command_offset
;
1339 nfit_blk
->stat_offset
= nfit_mem
->dcr
->status_offset
;
1340 mmio
= &nfit_blk
->mmio
[DCR
];
1341 mmio
->addr
.base
= nfit_spa_map(acpi_desc
, nfit_mem
->spa_dcr
,
1343 if (!mmio
->addr
.base
) {
1344 dev_dbg(dev
, "%s: %s failed to map dcr\n", __func__
,
1345 nvdimm_name(nvdimm
));
1348 mmio
->size
= nfit_mem
->dcr
->window_size
;
1349 mmio
->base_offset
= nfit_mem
->memdev_dcr
->region_offset
;
1350 mmio
->idt
= nfit_mem
->idt_dcr
;
1351 mmio
->spa
= nfit_mem
->spa_dcr
;
1352 rc
= nfit_blk_init_interleave(mmio
, nfit_mem
->idt_dcr
,
1353 nfit_mem
->memdev_dcr
->interleave_ways
);
1355 dev_dbg(dev
, "%s: %s failed to init dcr interleave\n",
1356 __func__
, nvdimm_name(nvdimm
));
1360 rc
= acpi_nfit_blk_get_flags(nd_desc
, nvdimm
, nfit_blk
);
1362 dev_dbg(dev
, "%s: %s failed get DIMM flags\n",
1363 __func__
, nvdimm_name(nvdimm
));
1367 nfit_flush
= nfit_mem
->nfit_flush
;
1368 if (nfit_flush
&& nfit_flush
->flush
->hint_count
!= 0) {
1369 nfit_blk
->nvdimm_flush
= devm_ioremap_nocache(dev
,
1370 nfit_flush
->flush
->hint_address
[0], 8);
1371 if (!nfit_blk
->nvdimm_flush
)
1375 if (!arch_has_wmb_pmem() && !nfit_blk
->nvdimm_flush
)
1376 dev_warn(dev
, "unable to guarantee persistence of writes\n");
1378 if (mmio
->line_size
== 0)
1381 if ((u32
) nfit_blk
->cmd_offset
% mmio
->line_size
1382 + 8 > mmio
->line_size
) {
1383 dev_dbg(dev
, "cmd_offset crosses interleave boundary\n");
1385 } else if ((u32
) nfit_blk
->stat_offset
% mmio
->line_size
1386 + 8 > mmio
->line_size
) {
1387 dev_dbg(dev
, "stat_offset crosses interleave boundary\n");
1394 static void acpi_nfit_blk_region_disable(struct nvdimm_bus
*nvdimm_bus
,
1397 struct nvdimm_bus_descriptor
*nd_desc
= to_nd_desc(nvdimm_bus
);
1398 struct acpi_nfit_desc
*acpi_desc
= to_acpi_desc(nd_desc
);
1399 struct nd_blk_region
*ndbr
= to_nd_blk_region(dev
);
1400 struct nfit_blk
*nfit_blk
= nd_blk_region_provider_data(ndbr
);
1404 return; /* never enabled */
1406 /* auto-free BLK spa mappings */
1407 for (i
= 0; i
< 2; i
++) {
1408 struct nfit_blk_mmio
*mmio
= &nfit_blk
->mmio
[i
];
1410 if (mmio
->addr
.base
)
1411 nfit_spa_unmap(acpi_desc
, mmio
->spa
);
1413 nd_blk_region_set_provider_data(ndbr
, NULL
);
1414 /* devm will free nfit_blk */
1417 static int acpi_nfit_init_mapping(struct acpi_nfit_desc
*acpi_desc
,
1418 struct nd_mapping
*nd_mapping
, struct nd_region_desc
*ndr_desc
,
1419 struct acpi_nfit_memory_map
*memdev
,
1420 struct acpi_nfit_system_address
*spa
)
1422 struct nvdimm
*nvdimm
= acpi_nfit_dimm_by_handle(acpi_desc
,
1423 memdev
->device_handle
);
1424 struct nd_blk_region_desc
*ndbr_desc
;
1425 struct nfit_mem
*nfit_mem
;
1429 dev_err(acpi_desc
->dev
, "spa%d dimm: %#x not found\n",
1430 spa
->range_index
, memdev
->device_handle
);
1434 nd_mapping
->nvdimm
= nvdimm
;
1435 switch (nfit_spa_type(spa
)) {
1437 case NFIT_SPA_VOLATILE
:
1438 nd_mapping
->start
= memdev
->address
;
1439 nd_mapping
->size
= memdev
->region_size
;
1442 nfit_mem
= nvdimm_provider_data(nvdimm
);
1443 if (!nfit_mem
|| !nfit_mem
->bdw
) {
1444 dev_dbg(acpi_desc
->dev
, "spa%d %s missing bdw\n",
1445 spa
->range_index
, nvdimm_name(nvdimm
));
1447 nd_mapping
->size
= nfit_mem
->bdw
->capacity
;
1448 nd_mapping
->start
= nfit_mem
->bdw
->start_address
;
1449 ndr_desc
->num_lanes
= nfit_mem
->bdw
->windows
;
1453 ndr_desc
->nd_mapping
= nd_mapping
;
1454 ndr_desc
->num_mappings
= blk_valid
;
1455 ndbr_desc
= to_blk_region_desc(ndr_desc
);
1456 ndbr_desc
->enable
= acpi_nfit_blk_region_enable
;
1457 ndbr_desc
->disable
= acpi_nfit_blk_region_disable
;
1458 ndbr_desc
->do_io
= acpi_desc
->blk_do_io
;
1459 if (!nvdimm_blk_region_create(acpi_desc
->nvdimm_bus
, ndr_desc
))
1467 static int acpi_nfit_register_region(struct acpi_nfit_desc
*acpi_desc
,
1468 struct nfit_spa
*nfit_spa
)
1470 static struct nd_mapping nd_mappings
[ND_MAX_MAPPINGS
];
1471 struct acpi_nfit_system_address
*spa
= nfit_spa
->spa
;
1472 struct nd_blk_region_desc ndbr_desc
;
1473 struct nd_region_desc
*ndr_desc
;
1474 struct nfit_memdev
*nfit_memdev
;
1475 struct nvdimm_bus
*nvdimm_bus
;
1476 struct resource res
;
1479 if (spa
->range_index
== 0) {
1480 dev_dbg(acpi_desc
->dev
, "%s: detected invalid spa index\n",
1485 memset(&res
, 0, sizeof(res
));
1486 memset(&nd_mappings
, 0, sizeof(nd_mappings
));
1487 memset(&ndbr_desc
, 0, sizeof(ndbr_desc
));
1488 res
.start
= spa
->address
;
1489 res
.end
= res
.start
+ spa
->length
- 1;
1490 ndr_desc
= &ndbr_desc
.ndr_desc
;
1491 ndr_desc
->res
= &res
;
1492 ndr_desc
->provider_data
= nfit_spa
;
1493 ndr_desc
->attr_groups
= acpi_nfit_region_attribute_groups
;
1494 if (spa
->flags
& ACPI_NFIT_PROXIMITY_VALID
)
1495 ndr_desc
->numa_node
= acpi_map_pxm_to_online_node(
1496 spa
->proximity_domain
);
1498 ndr_desc
->numa_node
= NUMA_NO_NODE
;
1500 list_for_each_entry(nfit_memdev
, &acpi_desc
->memdevs
, list
) {
1501 struct acpi_nfit_memory_map
*memdev
= nfit_memdev
->memdev
;
1502 struct nd_mapping
*nd_mapping
;
1504 if (memdev
->range_index
!= spa
->range_index
)
1506 if (count
>= ND_MAX_MAPPINGS
) {
1507 dev_err(acpi_desc
->dev
, "spa%d exceeds max mappings %d\n",
1508 spa
->range_index
, ND_MAX_MAPPINGS
);
1511 nd_mapping
= &nd_mappings
[count
++];
1512 rc
= acpi_nfit_init_mapping(acpi_desc
, nd_mapping
, ndr_desc
,
1518 ndr_desc
->nd_mapping
= nd_mappings
;
1519 ndr_desc
->num_mappings
= count
;
1520 rc
= acpi_nfit_init_interleave_set(acpi_desc
, ndr_desc
, spa
);
1524 nvdimm_bus
= acpi_desc
->nvdimm_bus
;
1525 if (nfit_spa_type(spa
) == NFIT_SPA_PM
) {
1526 if (!nvdimm_pmem_region_create(nvdimm_bus
, ndr_desc
))
1528 } else if (nfit_spa_type(spa
) == NFIT_SPA_VOLATILE
) {
1529 if (!nvdimm_volatile_region_create(nvdimm_bus
, ndr_desc
))
1535 static int acpi_nfit_register_regions(struct acpi_nfit_desc
*acpi_desc
)
1537 struct nfit_spa
*nfit_spa
;
1539 list_for_each_entry(nfit_spa
, &acpi_desc
->spas
, list
) {
1540 int rc
= acpi_nfit_register_region(acpi_desc
, nfit_spa
);
1548 int acpi_nfit_init(struct acpi_nfit_desc
*acpi_desc
, acpi_size sz
)
1550 struct device
*dev
= acpi_desc
->dev
;
1555 INIT_LIST_HEAD(&acpi_desc
->spa_maps
);
1556 INIT_LIST_HEAD(&acpi_desc
->spas
);
1557 INIT_LIST_HEAD(&acpi_desc
->dcrs
);
1558 INIT_LIST_HEAD(&acpi_desc
->bdws
);
1559 INIT_LIST_HEAD(&acpi_desc
->idts
);
1560 INIT_LIST_HEAD(&acpi_desc
->flushes
);
1561 INIT_LIST_HEAD(&acpi_desc
->memdevs
);
1562 INIT_LIST_HEAD(&acpi_desc
->dimms
);
1563 mutex_init(&acpi_desc
->spa_map_mutex
);
1565 data
= (u8
*) acpi_desc
->nfit
;
1567 data
+= sizeof(struct acpi_table_nfit
);
1568 while (!IS_ERR_OR_NULL(data
))
1569 data
= add_table(acpi_desc
, data
, end
);
1572 dev_dbg(dev
, "%s: nfit table parsing error: %ld\n", __func__
,
1574 return PTR_ERR(data
);
1577 if (nfit_mem_init(acpi_desc
) != 0)
1580 acpi_nfit_init_dsms(acpi_desc
);
1582 rc
= acpi_nfit_register_dimms(acpi_desc
);
1586 return acpi_nfit_register_regions(acpi_desc
);
1588 EXPORT_SYMBOL_GPL(acpi_nfit_init
);
1590 static int acpi_nfit_add(struct acpi_device
*adev
)
1592 struct nvdimm_bus_descriptor
*nd_desc
;
1593 struct acpi_nfit_desc
*acpi_desc
;
1594 struct device
*dev
= &adev
->dev
;
1595 struct acpi_table_header
*tbl
;
1596 acpi_status status
= AE_OK
;
1600 status
= acpi_get_table_with_size("NFIT", 0, &tbl
, &sz
);
1601 if (ACPI_FAILURE(status
)) {
1602 dev_err(dev
, "failed to find NFIT\n");
1606 acpi_desc
= devm_kzalloc(dev
, sizeof(*acpi_desc
), GFP_KERNEL
);
1610 dev_set_drvdata(dev
, acpi_desc
);
1611 acpi_desc
->dev
= dev
;
1612 acpi_desc
->nfit
= (struct acpi_table_nfit
*) tbl
;
1613 acpi_desc
->blk_do_io
= acpi_nfit_blk_region_do_io
;
1614 nd_desc
= &acpi_desc
->nd_desc
;
1615 nd_desc
->provider_name
= "ACPI.NFIT";
1616 nd_desc
->ndctl
= acpi_nfit_ctl
;
1617 nd_desc
->attr_groups
= acpi_nfit_attribute_groups
;
1619 acpi_desc
->nvdimm_bus
= nvdimm_bus_register(dev
, nd_desc
);
1620 if (!acpi_desc
->nvdimm_bus
)
1623 rc
= acpi_nfit_init(acpi_desc
, sz
);
1625 nvdimm_bus_unregister(acpi_desc
->nvdimm_bus
);
1631 static int acpi_nfit_remove(struct acpi_device
*adev
)
1633 struct acpi_nfit_desc
*acpi_desc
= dev_get_drvdata(&adev
->dev
);
1635 nvdimm_bus_unregister(acpi_desc
->nvdimm_bus
);
1639 static const struct acpi_device_id acpi_nfit_ids
[] = {
1643 MODULE_DEVICE_TABLE(acpi
, acpi_nfit_ids
);
1645 static struct acpi_driver acpi_nfit_driver
= {
1646 .name
= KBUILD_MODNAME
,
1647 .ids
= acpi_nfit_ids
,
1649 .add
= acpi_nfit_add
,
1650 .remove
= acpi_nfit_remove
,
1654 static __init
int nfit_init(void)
1656 BUILD_BUG_ON(sizeof(struct acpi_table_nfit
) != 40);
1657 BUILD_BUG_ON(sizeof(struct acpi_nfit_system_address
) != 56);
1658 BUILD_BUG_ON(sizeof(struct acpi_nfit_memory_map
) != 48);
1659 BUILD_BUG_ON(sizeof(struct acpi_nfit_interleave
) != 20);
1660 BUILD_BUG_ON(sizeof(struct acpi_nfit_smbios
) != 9);
1661 BUILD_BUG_ON(sizeof(struct acpi_nfit_control_region
) != 80);
1662 BUILD_BUG_ON(sizeof(struct acpi_nfit_data_region
) != 40);
1664 acpi_str_to_uuid(UUID_VOLATILE_MEMORY
, nfit_uuid
[NFIT_SPA_VOLATILE
]);
1665 acpi_str_to_uuid(UUID_PERSISTENT_MEMORY
, nfit_uuid
[NFIT_SPA_PM
]);
1666 acpi_str_to_uuid(UUID_CONTROL_REGION
, nfit_uuid
[NFIT_SPA_DCR
]);
1667 acpi_str_to_uuid(UUID_DATA_REGION
, nfit_uuid
[NFIT_SPA_BDW
]);
1668 acpi_str_to_uuid(UUID_VOLATILE_VIRTUAL_DISK
, nfit_uuid
[NFIT_SPA_VDISK
]);
1669 acpi_str_to_uuid(UUID_VOLATILE_VIRTUAL_CD
, nfit_uuid
[NFIT_SPA_VCD
]);
1670 acpi_str_to_uuid(UUID_PERSISTENT_VIRTUAL_DISK
, nfit_uuid
[NFIT_SPA_PDISK
]);
1671 acpi_str_to_uuid(UUID_PERSISTENT_VIRTUAL_CD
, nfit_uuid
[NFIT_SPA_PCD
]);
1672 acpi_str_to_uuid(UUID_NFIT_BUS
, nfit_uuid
[NFIT_DEV_BUS
]);
1673 acpi_str_to_uuid(UUID_NFIT_DIMM
, nfit_uuid
[NFIT_DEV_DIMM
]);
1675 return acpi_bus_register_driver(&acpi_nfit_driver
);
1678 static __exit
void nfit_exit(void)
1680 acpi_bus_unregister_driver(&acpi_nfit_driver
);
1683 module_init(nfit_init
);
1684 module_exit(nfit_exit
);
1685 MODULE_LICENSE("GPL v2");
1686 MODULE_AUTHOR("Intel Corporation");