2 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of version 2 of the GNU General Public License as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
14 #include <linux/vmalloc.h>
15 #include <linux/uaccess.h>
16 #include <linux/module.h>
17 #include <linux/blkdev.h>
18 #include <linux/fcntl.h>
19 #include <linux/async.h>
20 #include <linux/genhd.h>
21 #include <linux/ndctl.h>
22 #include <linux/sched.h>
23 #include <linux/slab.h>
32 static int nvdimm_bus_major
;
33 static struct class *nd_class
;
35 static int to_nd_device_type(struct device
*dev
)
38 return ND_DEVICE_DIMM
;
39 else if (is_nd_pmem(dev
))
40 return ND_DEVICE_REGION_PMEM
;
41 else if (is_nd_blk(dev
))
42 return ND_DEVICE_REGION_BLK
;
43 else if (is_nd_pmem(dev
->parent
) || is_nd_blk(dev
->parent
))
44 return nd_region_to_nstype(to_nd_region(dev
->parent
));
49 static int nvdimm_bus_uevent(struct device
*dev
, struct kobj_uevent_env
*env
)
52 * Ensure that region devices always have their numa node set as
55 if (is_nd_pmem(dev
) || is_nd_blk(dev
))
56 set_dev_node(dev
, to_nd_region(dev
)->numa_node
);
57 return add_uevent_var(env
, "MODALIAS=" ND_DEVICE_MODALIAS_FMT
,
58 to_nd_device_type(dev
));
61 static int nvdimm_bus_match(struct device
*dev
, struct device_driver
*drv
)
63 struct nd_device_driver
*nd_drv
= to_nd_device_driver(drv
);
65 return test_bit(to_nd_device_type(dev
), &nd_drv
->type
);
68 static struct module
*to_bus_provider(struct device
*dev
)
70 /* pin bus providers while regions are enabled */
71 if (is_nd_pmem(dev
) || is_nd_blk(dev
)) {
72 struct nvdimm_bus
*nvdimm_bus
= walk_to_nvdimm_bus(dev
);
74 return nvdimm_bus
->module
;
79 static void nvdimm_bus_probe_start(struct nvdimm_bus
*nvdimm_bus
)
81 nvdimm_bus_lock(&nvdimm_bus
->dev
);
82 nvdimm_bus
->probe_active
++;
83 nvdimm_bus_unlock(&nvdimm_bus
->dev
);
86 static void nvdimm_bus_probe_end(struct nvdimm_bus
*nvdimm_bus
)
88 nvdimm_bus_lock(&nvdimm_bus
->dev
);
89 if (--nvdimm_bus
->probe_active
== 0)
90 wake_up(&nvdimm_bus
->probe_wait
);
91 nvdimm_bus_unlock(&nvdimm_bus
->dev
);
94 static int nvdimm_bus_probe(struct device
*dev
)
96 struct nd_device_driver
*nd_drv
= to_nd_device_driver(dev
->driver
);
97 struct module
*provider
= to_bus_provider(dev
);
98 struct nvdimm_bus
*nvdimm_bus
= walk_to_nvdimm_bus(dev
);
101 if (!try_module_get(provider
))
104 nvdimm_bus_probe_start(nvdimm_bus
);
105 rc
= nd_drv
->probe(dev
);
107 nd_region_probe_success(nvdimm_bus
, dev
);
109 nd_region_disable(nvdimm_bus
, dev
);
110 nvdimm_bus_probe_end(nvdimm_bus
);
112 dev_dbg(&nvdimm_bus
->dev
, "%s.probe(%s) = %d\n", dev
->driver
->name
,
116 module_put(provider
);
120 static int nvdimm_bus_remove(struct device
*dev
)
122 struct nd_device_driver
*nd_drv
= to_nd_device_driver(dev
->driver
);
123 struct module
*provider
= to_bus_provider(dev
);
124 struct nvdimm_bus
*nvdimm_bus
= walk_to_nvdimm_bus(dev
);
127 rc
= nd_drv
->remove(dev
);
128 nd_region_disable(nvdimm_bus
, dev
);
130 dev_dbg(&nvdimm_bus
->dev
, "%s.remove(%s) = %d\n", dev
->driver
->name
,
132 module_put(provider
);
136 void nd_device_notify(struct device
*dev
, enum nvdimm_event event
)
140 struct nd_device_driver
*nd_drv
;
142 nd_drv
= to_nd_device_driver(dev
->driver
);
144 nd_drv
->notify(dev
, event
);
148 EXPORT_SYMBOL(nd_device_notify
);
150 void nvdimm_region_notify(struct nd_region
*nd_region
, enum nvdimm_event event
)
152 struct nvdimm_bus
*nvdimm_bus
= walk_to_nvdimm_bus(&nd_region
->dev
);
157 /* caller is responsible for holding a reference on the device */
158 nd_device_notify(&nd_region
->dev
, event
);
160 EXPORT_SYMBOL_GPL(nvdimm_region_notify
);
162 static struct bus_type nvdimm_bus_type
= {
164 .uevent
= nvdimm_bus_uevent
,
165 .match
= nvdimm_bus_match
,
166 .probe
= nvdimm_bus_probe
,
167 .remove
= nvdimm_bus_remove
,
170 static ASYNC_DOMAIN_EXCLUSIVE(nd_async_domain
);
172 void nd_synchronize(void)
174 async_synchronize_full_domain(&nd_async_domain
);
176 EXPORT_SYMBOL_GPL(nd_synchronize
);
178 static void nd_async_device_register(void *d
, async_cookie_t cookie
)
180 struct device
*dev
= d
;
182 if (device_add(dev
) != 0) {
183 dev_err(dev
, "%s: failed\n", __func__
);
189 static void nd_async_device_unregister(void *d
, async_cookie_t cookie
)
191 struct device
*dev
= d
;
193 /* flush bus operations before delete */
194 nvdimm_bus_lock(dev
);
195 nvdimm_bus_unlock(dev
);
197 device_unregister(dev
);
201 void __nd_device_register(struct device
*dev
)
203 dev
->bus
= &nvdimm_bus_type
;
205 async_schedule_domain(nd_async_device_register
, dev
,
209 void nd_device_register(struct device
*dev
)
211 device_initialize(dev
);
212 __nd_device_register(dev
);
214 EXPORT_SYMBOL(nd_device_register
);
216 void nd_device_unregister(struct device
*dev
, enum nd_async_mode mode
)
221 async_schedule_domain(nd_async_device_unregister
, dev
,
226 device_unregister(dev
);
230 EXPORT_SYMBOL(nd_device_unregister
);
233 * __nd_driver_register() - register a region or a namespace driver
234 * @nd_drv: driver to register
235 * @owner: automatically set by nd_driver_register() macro
236 * @mod_name: automatically set by nd_driver_register() macro
238 int __nd_driver_register(struct nd_device_driver
*nd_drv
, struct module
*owner
,
239 const char *mod_name
)
241 struct device_driver
*drv
= &nd_drv
->drv
;
244 pr_debug("driver type bitmask not set (%pf)\n",
245 __builtin_return_address(0));
249 if (!nd_drv
->probe
|| !nd_drv
->remove
) {
250 pr_debug("->probe() and ->remove() must be specified\n");
254 drv
->bus
= &nvdimm_bus_type
;
256 drv
->mod_name
= mod_name
;
258 return driver_register(drv
);
260 EXPORT_SYMBOL(__nd_driver_register
);
262 int nvdimm_revalidate_disk(struct gendisk
*disk
)
264 struct device
*dev
= disk
->driverfs_dev
;
265 struct nd_region
*nd_region
= to_nd_region(dev
->parent
);
266 const char *pol
= nd_region
->ro
? "only" : "write";
268 if (nd_region
->ro
== get_disk_ro(disk
))
271 dev_info(dev
, "%s read-%s, marking %s read-%s\n",
272 dev_name(&nd_region
->dev
), pol
, disk
->disk_name
, pol
);
273 set_disk_ro(disk
, nd_region
->ro
);
278 EXPORT_SYMBOL(nvdimm_revalidate_disk
);
280 static ssize_t
modalias_show(struct device
*dev
, struct device_attribute
*attr
,
283 return sprintf(buf
, ND_DEVICE_MODALIAS_FMT
"\n",
284 to_nd_device_type(dev
));
286 static DEVICE_ATTR_RO(modalias
);
288 static ssize_t
devtype_show(struct device
*dev
, struct device_attribute
*attr
,
291 return sprintf(buf
, "%s\n", dev
->type
->name
);
293 static DEVICE_ATTR_RO(devtype
);
295 static struct attribute
*nd_device_attributes
[] = {
296 &dev_attr_modalias
.attr
,
297 &dev_attr_devtype
.attr
,
302 * nd_device_attribute_group - generic attributes for all devices on an nd bus
304 struct attribute_group nd_device_attribute_group
= {
305 .attrs
= nd_device_attributes
,
307 EXPORT_SYMBOL_GPL(nd_device_attribute_group
);
309 static ssize_t
numa_node_show(struct device
*dev
,
310 struct device_attribute
*attr
, char *buf
)
312 return sprintf(buf
, "%d\n", dev_to_node(dev
));
314 static DEVICE_ATTR_RO(numa_node
);
316 static struct attribute
*nd_numa_attributes
[] = {
317 &dev_attr_numa_node
.attr
,
321 static umode_t
nd_numa_attr_visible(struct kobject
*kobj
, struct attribute
*a
,
324 if (!IS_ENABLED(CONFIG_NUMA
))
331 * nd_numa_attribute_group - NUMA attributes for all devices on an nd bus
333 struct attribute_group nd_numa_attribute_group
= {
334 .attrs
= nd_numa_attributes
,
335 .is_visible
= nd_numa_attr_visible
,
337 EXPORT_SYMBOL_GPL(nd_numa_attribute_group
);
339 int nvdimm_bus_create_ndctl(struct nvdimm_bus
*nvdimm_bus
)
341 dev_t devt
= MKDEV(nvdimm_bus_major
, nvdimm_bus
->id
);
344 dev
= device_create(nd_class
, &nvdimm_bus
->dev
, devt
, nvdimm_bus
,
345 "ndctl%d", nvdimm_bus
->id
);
348 dev_dbg(&nvdimm_bus
->dev
, "failed to register ndctl%d: %ld\n",
349 nvdimm_bus
->id
, PTR_ERR(dev
));
355 void nvdimm_bus_destroy_ndctl(struct nvdimm_bus
*nvdimm_bus
)
357 device_destroy(nd_class
, MKDEV(nvdimm_bus_major
, nvdimm_bus
->id
));
360 static const struct nd_cmd_desc __nd_cmd_dimm_descs
[] = {
361 [ND_CMD_IMPLEMENTED
] = { },
364 .out_sizes
= { 4, 8, },
366 [ND_CMD_SMART_THRESHOLD
] = {
368 .out_sizes
= { 4, 8, },
370 [ND_CMD_DIMM_FLAGS
] = {
372 .out_sizes
= { 4, 4 },
374 [ND_CMD_GET_CONFIG_SIZE
] = {
376 .out_sizes
= { 4, 4, 4, },
378 [ND_CMD_GET_CONFIG_DATA
] = {
380 .in_sizes
= { 4, 4, },
382 .out_sizes
= { 4, UINT_MAX
, },
384 [ND_CMD_SET_CONFIG_DATA
] = {
386 .in_sizes
= { 4, 4, UINT_MAX
, },
392 .in_sizes
= { 4, 4, UINT_MAX
, },
394 .out_sizes
= { 4, 4, UINT_MAX
, },
398 const struct nd_cmd_desc
*nd_cmd_dimm_desc(int cmd
)
400 if (cmd
< ARRAY_SIZE(__nd_cmd_dimm_descs
))
401 return &__nd_cmd_dimm_descs
[cmd
];
404 EXPORT_SYMBOL_GPL(nd_cmd_dimm_desc
);
406 static const struct nd_cmd_desc __nd_cmd_bus_descs
[] = {
407 [ND_CMD_IMPLEMENTED
] = { },
410 .in_sizes
= { 8, 8, },
412 .out_sizes
= { 4, 4, 4, 4, },
414 [ND_CMD_ARS_START
] = {
416 .in_sizes
= { 8, 8, 2, 1, 5, },
418 .out_sizes
= { 4, 4, },
420 [ND_CMD_ARS_STATUS
] = {
422 .out_sizes
= { 4, 4, UINT_MAX
, },
426 const struct nd_cmd_desc
*nd_cmd_bus_desc(int cmd
)
428 if (cmd
< ARRAY_SIZE(__nd_cmd_bus_descs
))
429 return &__nd_cmd_bus_descs
[cmd
];
432 EXPORT_SYMBOL_GPL(nd_cmd_bus_desc
);
434 u32
nd_cmd_in_size(struct nvdimm
*nvdimm
, int cmd
,
435 const struct nd_cmd_desc
*desc
, int idx
, void *buf
)
437 if (idx
>= desc
->in_num
)
440 if (desc
->in_sizes
[idx
] < UINT_MAX
)
441 return desc
->in_sizes
[idx
];
443 if (nvdimm
&& cmd
== ND_CMD_SET_CONFIG_DATA
&& idx
== 2) {
444 struct nd_cmd_set_config_hdr
*hdr
= buf
;
446 return hdr
->in_length
;
447 } else if (nvdimm
&& cmd
== ND_CMD_VENDOR
&& idx
== 2) {
448 struct nd_cmd_vendor_hdr
*hdr
= buf
;
450 return hdr
->in_length
;
455 EXPORT_SYMBOL_GPL(nd_cmd_in_size
);
457 u32
nd_cmd_out_size(struct nvdimm
*nvdimm
, int cmd
,
458 const struct nd_cmd_desc
*desc
, int idx
, const u32
*in_field
,
459 const u32
*out_field
)
461 if (idx
>= desc
->out_num
)
464 if (desc
->out_sizes
[idx
] < UINT_MAX
)
465 return desc
->out_sizes
[idx
];
467 if (nvdimm
&& cmd
== ND_CMD_GET_CONFIG_DATA
&& idx
== 1)
469 else if (nvdimm
&& cmd
== ND_CMD_VENDOR
&& idx
== 2)
471 else if (!nvdimm
&& cmd
== ND_CMD_ARS_STATUS
&& idx
== 2)
472 return out_field
[1] - 8;
476 EXPORT_SYMBOL_GPL(nd_cmd_out_size
);
478 void wait_nvdimm_bus_probe_idle(struct device
*dev
)
480 struct nvdimm_bus
*nvdimm_bus
= walk_to_nvdimm_bus(dev
);
483 if (nvdimm_bus
->probe_active
== 0)
485 nvdimm_bus_unlock(&nvdimm_bus
->dev
);
486 wait_event(nvdimm_bus
->probe_wait
,
487 nvdimm_bus
->probe_active
== 0);
488 nvdimm_bus_lock(&nvdimm_bus
->dev
);
492 /* set_config requires an idle interleave set */
493 static int nd_cmd_clear_to_send(struct nvdimm_bus
*nvdimm_bus
,
494 struct nvdimm
*nvdimm
, unsigned int cmd
)
496 struct nvdimm_bus_descriptor
*nd_desc
= nvdimm_bus
->nd_desc
;
498 /* ask the bus provider if it would like to block this request */
499 if (nd_desc
->clear_to_send
) {
500 int rc
= nd_desc
->clear_to_send(nd_desc
, nvdimm
, cmd
);
506 if (!nvdimm
|| cmd
!= ND_CMD_SET_CONFIG_DATA
)
509 /* prevent label manipulation while the kernel owns label updates */
510 wait_nvdimm_bus_probe_idle(&nvdimm_bus
->dev
);
511 if (atomic_read(&nvdimm
->busy
))
516 static int __nd_ioctl(struct nvdimm_bus
*nvdimm_bus
, struct nvdimm
*nvdimm
,
517 int read_only
, unsigned int ioctl_cmd
, unsigned long arg
)
519 struct nvdimm_bus_descriptor
*nd_desc
= nvdimm_bus
->nd_desc
;
520 size_t buf_len
= 0, in_len
= 0, out_len
= 0;
521 static char out_env
[ND_CMD_MAX_ENVELOPE
];
522 static char in_env
[ND_CMD_MAX_ENVELOPE
];
523 const struct nd_cmd_desc
*desc
= NULL
;
524 unsigned int cmd
= _IOC_NR(ioctl_cmd
);
525 void __user
*p
= (void __user
*) arg
;
526 struct device
*dev
= &nvdimm_bus
->dev
;
527 const char *cmd_name
, *dimm_name
;
528 unsigned long dsm_mask
;
533 desc
= nd_cmd_dimm_desc(cmd
);
534 cmd_name
= nvdimm_cmd_name(cmd
);
535 dsm_mask
= nvdimm
->dsm_mask
? *(nvdimm
->dsm_mask
) : 0;
536 dimm_name
= dev_name(&nvdimm
->dev
);
538 desc
= nd_cmd_bus_desc(cmd
);
539 cmd_name
= nvdimm_bus_cmd_name(cmd
);
540 dsm_mask
= nd_desc
->dsm_mask
;
544 if (!desc
|| (desc
->out_num
+ desc
->in_num
== 0) ||
545 !test_bit(cmd
, &dsm_mask
))
548 /* fail write commands (when read-only) */
551 case ND_IOCTL_VENDOR
:
552 case ND_IOCTL_SET_CONFIG_DATA
:
553 case ND_IOCTL_ARS_START
:
554 dev_dbg(&nvdimm_bus
->dev
, "'%s' command while read-only.\n",
555 nvdimm
? nvdimm_cmd_name(cmd
)
556 : nvdimm_bus_cmd_name(cmd
));
562 /* process an input envelope */
563 for (i
= 0; i
< desc
->in_num
; i
++) {
566 in_size
= nd_cmd_in_size(nvdimm
, cmd
, desc
, i
, in_env
);
567 if (in_size
== UINT_MAX
) {
568 dev_err(dev
, "%s:%s unknown input size cmd: %s field: %d\n",
569 __func__
, dimm_name
, cmd_name
, i
);
572 if (in_len
< sizeof(in_env
))
573 copy
= min_t(u32
, sizeof(in_env
) - in_len
, in_size
);
576 if (copy
&& copy_from_user(&in_env
[in_len
], p
+ in_len
, copy
))
581 /* process an output envelope */
582 for (i
= 0; i
< desc
->out_num
; i
++) {
583 u32 out_size
= nd_cmd_out_size(nvdimm
, cmd
, desc
, i
,
584 (u32
*) in_env
, (u32
*) out_env
);
587 if (out_size
== UINT_MAX
) {
588 dev_dbg(dev
, "%s:%s unknown output size cmd: %s field: %d\n",
589 __func__
, dimm_name
, cmd_name
, i
);
592 if (out_len
< sizeof(out_env
))
593 copy
= min_t(u32
, sizeof(out_env
) - out_len
, out_size
);
596 if (copy
&& copy_from_user(&out_env
[out_len
],
597 p
+ in_len
+ out_len
, copy
))
602 buf_len
= out_len
+ in_len
;
603 if (buf_len
> ND_IOCTL_MAX_BUFLEN
) {
604 dev_dbg(dev
, "%s:%s cmd: %s buf_len: %zu > %d\n", __func__
,
605 dimm_name
, cmd_name
, buf_len
,
606 ND_IOCTL_MAX_BUFLEN
);
610 buf
= vmalloc(buf_len
);
614 if (copy_from_user(buf
, p
, buf_len
)) {
619 nvdimm_bus_lock(&nvdimm_bus
->dev
);
620 rc
= nd_cmd_clear_to_send(nvdimm_bus
, nvdimm
, cmd
);
624 rc
= nd_desc
->ndctl(nd_desc
, nvdimm
, cmd
, buf
, buf_len
, NULL
);
627 if (copy_to_user(p
, buf
, buf_len
))
630 nvdimm_bus_unlock(&nvdimm_bus
->dev
);
636 static long nd_ioctl(struct file
*file
, unsigned int cmd
, unsigned long arg
)
638 long id
= (long) file
->private_data
;
639 int rc
= -ENXIO
, read_only
;
640 struct nvdimm_bus
*nvdimm_bus
;
642 read_only
= (O_RDWR
!= (file
->f_flags
& O_ACCMODE
));
643 mutex_lock(&nvdimm_bus_list_mutex
);
644 list_for_each_entry(nvdimm_bus
, &nvdimm_bus_list
, list
) {
645 if (nvdimm_bus
->id
== id
) {
646 rc
= __nd_ioctl(nvdimm_bus
, NULL
, read_only
, cmd
, arg
);
650 mutex_unlock(&nvdimm_bus_list_mutex
);
655 static int match_dimm(struct device
*dev
, void *data
)
657 long id
= (long) data
;
659 if (is_nvdimm(dev
)) {
660 struct nvdimm
*nvdimm
= to_nvdimm(dev
);
662 return nvdimm
->id
== id
;
668 static long nvdimm_ioctl(struct file
*file
, unsigned int cmd
, unsigned long arg
)
670 int rc
= -ENXIO
, read_only
;
671 struct nvdimm_bus
*nvdimm_bus
;
673 read_only
= (O_RDWR
!= (file
->f_flags
& O_ACCMODE
));
674 mutex_lock(&nvdimm_bus_list_mutex
);
675 list_for_each_entry(nvdimm_bus
, &nvdimm_bus_list
, list
) {
676 struct device
*dev
= device_find_child(&nvdimm_bus
->dev
,
677 file
->private_data
, match_dimm
);
678 struct nvdimm
*nvdimm
;
683 nvdimm
= to_nvdimm(dev
);
684 rc
= __nd_ioctl(nvdimm_bus
, nvdimm
, read_only
, cmd
, arg
);
688 mutex_unlock(&nvdimm_bus_list_mutex
);
693 static int nd_open(struct inode
*inode
, struct file
*file
)
695 long minor
= iminor(inode
);
697 file
->private_data
= (void *) minor
;
701 static const struct file_operations nvdimm_bus_fops
= {
702 .owner
= THIS_MODULE
,
704 .unlocked_ioctl
= nd_ioctl
,
705 .compat_ioctl
= nd_ioctl
,
706 .llseek
= noop_llseek
,
709 static const struct file_operations nvdimm_fops
= {
710 .owner
= THIS_MODULE
,
712 .unlocked_ioctl
= nvdimm_ioctl
,
713 .compat_ioctl
= nvdimm_ioctl
,
714 .llseek
= noop_llseek
,
717 int __init
nvdimm_bus_init(void)
721 rc
= bus_register(&nvdimm_bus_type
);
725 rc
= register_chrdev(0, "ndctl", &nvdimm_bus_fops
);
728 nvdimm_bus_major
= rc
;
730 rc
= register_chrdev(0, "dimmctl", &nvdimm_fops
);
732 goto err_dimm_chrdev
;
735 nd_class
= class_create(THIS_MODULE
, "nd");
736 if (IS_ERR(nd_class
)) {
737 rc
= PTR_ERR(nd_class
);
744 unregister_chrdev(nvdimm_major
, "dimmctl");
746 unregister_chrdev(nvdimm_bus_major
, "ndctl");
748 bus_unregister(&nvdimm_bus_type
);
753 void nvdimm_bus_exit(void)
755 class_destroy(nd_class
);
756 unregister_chrdev(nvdimm_bus_major
, "ndctl");
757 unregister_chrdev(nvdimm_major
, "dimmctl");
758 bus_unregister(&nvdimm_bus_type
);