1 /******************************************************************************
4 * Device for accessing (in user-space) pages that have been granted by other
7 * Copyright (c) 2006-2007, D G Murray.
8 * (c) 2009 Gerd Hoffmann <kraxel@redhat.com>
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 #include <linux/module.h>
23 #include <linux/kernel.h>
24 #include <linux/init.h>
25 #include <linux/miscdevice.h>
28 #include <linux/mman.h>
29 #include <linux/mmu_notifier.h>
30 #include <linux/types.h>
31 #include <linux/uaccess.h>
32 #include <linux/sched.h>
33 #include <linux/spinlock.h>
34 #include <linux/slab.h>
35 #include <linux/highmem.h>
38 #include <xen/grant_table.h>
39 #include <xen/balloon.h>
40 #include <xen/gntdev.h>
41 #include <xen/events.h>
42 #include <asm/xen/hypervisor.h>
43 #include <asm/xen/hypercall.h>
44 #include <asm/xen/page.h>
46 MODULE_LICENSE("GPL");
47 MODULE_AUTHOR("Derek G. Murray <Derek.Murray@cl.cam.ac.uk>, "
48 "Gerd Hoffmann <kraxel@redhat.com>");
49 MODULE_DESCRIPTION("User-space granted page access driver");
51 static int limit
= 1024*1024;
52 module_param(limit
, int, 0644);
53 MODULE_PARM_DESC(limit
, "Maximum number of grants that may be mapped by "
56 static atomic_t pages_mapped
= ATOMIC_INIT(0);
58 static int use_ptemod
;
61 struct list_head maps
;
62 /* lock protects maps from concurrent changes */
65 struct mmu_notifier mn
;
70 /* Address relative to the start of the grant_map */
76 struct list_head next
;
77 struct vm_area_struct
*vma
;
82 struct unmap_notify notify
;
83 struct ioctl_gntdev_grant_ref
*grants
;
84 struct gnttab_map_grant_ref
*map_ops
;
85 struct gnttab_unmap_grant_ref
*unmap_ops
;
86 struct gnttab_map_grant_ref
*kmap_ops
;
90 static int unmap_grant_pages(struct grant_map
*map
, int offset
, int pages
);
92 /* ------------------------------------------------------------------ */
94 static void gntdev_print_maps(struct gntdev_priv
*priv
,
95 char *text
, int text_index
)
98 struct grant_map
*map
;
100 pr_debug("%s: maps list (priv %p)\n", __func__
, priv
);
101 list_for_each_entry(map
, &priv
->maps
, next
)
102 pr_debug(" index %2d, count %2d %s\n",
103 map
->index
, map
->count
,
104 map
->index
== text_index
&& text
? text
: "");
108 static void gntdev_free_map(struct grant_map
*map
)
114 free_xenballooned_pages(map
->count
, map
->pages
);
118 kfree(map
->unmap_ops
);
119 kfree(map
->kmap_ops
);
123 static struct grant_map
*gntdev_alloc_map(struct gntdev_priv
*priv
, int count
)
125 struct grant_map
*add
;
128 add
= kzalloc(sizeof(struct grant_map
), GFP_KERNEL
);
132 add
->grants
= kcalloc(count
, sizeof(add
->grants
[0]), GFP_KERNEL
);
133 add
->map_ops
= kcalloc(count
, sizeof(add
->map_ops
[0]), GFP_KERNEL
);
134 add
->unmap_ops
= kcalloc(count
, sizeof(add
->unmap_ops
[0]), GFP_KERNEL
);
135 add
->kmap_ops
= kcalloc(count
, sizeof(add
->kmap_ops
[0]), GFP_KERNEL
);
136 add
->pages
= kcalloc(count
, sizeof(add
->pages
[0]), GFP_KERNEL
);
137 if (NULL
== add
->grants
||
138 NULL
== add
->map_ops
||
139 NULL
== add
->unmap_ops
||
140 NULL
== add
->kmap_ops
||
144 if (alloc_xenballooned_pages(count
, add
->pages
, false /* lowmem */))
147 for (i
= 0; i
< count
; i
++) {
148 add
->map_ops
[i
].handle
= -1;
149 add
->unmap_ops
[i
].handle
= -1;
150 add
->kmap_ops
[i
].handle
= -1;
155 atomic_set(&add
->users
, 1);
160 gntdev_free_map(add
);
164 static void gntdev_add_map(struct gntdev_priv
*priv
, struct grant_map
*add
)
166 struct grant_map
*map
;
168 list_for_each_entry(map
, &priv
->maps
, next
) {
169 if (add
->index
+ add
->count
< map
->index
) {
170 list_add_tail(&add
->next
, &map
->next
);
173 add
->index
= map
->index
+ map
->count
;
175 list_add_tail(&add
->next
, &priv
->maps
);
178 gntdev_print_maps(priv
, "[new]", add
->index
);
181 static struct grant_map
*gntdev_find_map_index(struct gntdev_priv
*priv
,
182 int index
, int count
)
184 struct grant_map
*map
;
186 list_for_each_entry(map
, &priv
->maps
, next
) {
187 if (map
->index
!= index
)
189 if (count
&& map
->count
!= count
)
196 static void gntdev_put_map(struct grant_map
*map
)
201 if (!atomic_dec_and_test(&map
->users
))
204 atomic_sub(map
->count
, &pages_mapped
);
206 if (map
->notify
.flags
& UNMAP_NOTIFY_SEND_EVENT
) {
207 notify_remote_via_evtchn(map
->notify
.event
);
208 evtchn_put(map
->notify
.event
);
211 if (map
->pages
&& !use_ptemod
)
212 unmap_grant_pages(map
, 0, map
->count
);
213 gntdev_free_map(map
);
216 /* ------------------------------------------------------------------ */
218 static int find_grant_ptes(pte_t
*pte
, pgtable_t token
,
219 unsigned long addr
, void *data
)
221 struct grant_map
*map
= data
;
222 unsigned int pgnr
= (addr
- map
->vma
->vm_start
) >> PAGE_SHIFT
;
223 int flags
= map
->flags
| GNTMAP_application_map
| GNTMAP_contains_pte
;
226 BUG_ON(pgnr
>= map
->count
);
227 pte_maddr
= arbitrary_virt_to_machine(pte
).maddr
;
229 gnttab_set_map_op(&map
->map_ops
[pgnr
], pte_maddr
, flags
,
230 map
->grants
[pgnr
].ref
,
231 map
->grants
[pgnr
].domid
);
232 gnttab_set_unmap_op(&map
->unmap_ops
[pgnr
], pte_maddr
, flags
,
237 static int map_grant_pages(struct grant_map
*map
)
242 /* Note: it could already be mapped */
243 if (map
->map_ops
[0].handle
!= -1)
245 for (i
= 0; i
< map
->count
; i
++) {
246 unsigned long addr
= (unsigned long)
247 pfn_to_kaddr(page_to_pfn(map
->pages
[i
]));
248 gnttab_set_map_op(&map
->map_ops
[i
], addr
, map
->flags
,
250 map
->grants
[i
].domid
);
251 gnttab_set_unmap_op(&map
->unmap_ops
[i
], addr
,
252 map
->flags
, -1 /* handle */);
256 * Setup the map_ops corresponding to the pte entries pointing
257 * to the kernel linear addresses of the struct pages.
258 * These ptes are completely different from the user ptes dealt
259 * with find_grant_ptes.
261 for (i
= 0; i
< map
->count
; i
++) {
263 unsigned long address
= (unsigned long)
264 pfn_to_kaddr(page_to_pfn(map
->pages
[i
]));
267 BUG_ON(PageHighMem(map
->pages
[i
]));
269 ptep
= lookup_address(address
, &level
);
270 pte_maddr
= arbitrary_virt_to_machine(ptep
).maddr
;
271 gnttab_set_map_op(&map
->kmap_ops
[i
], pte_maddr
,
276 map
->grants
[i
].domid
);
280 pr_debug("map %d+%d\n", map
->index
, map
->count
);
281 err
= gnttab_map_refs(map
->map_ops
, use_ptemod
? map
->kmap_ops
: NULL
,
282 map
->pages
, map
->count
);
286 for (i
= 0; i
< map
->count
; i
++) {
287 if (map
->map_ops
[i
].status
)
290 BUG_ON(map
->map_ops
[i
].handle
== -1);
291 map
->unmap_ops
[i
].handle
= map
->map_ops
[i
].handle
;
292 pr_debug("map handle=%d\n", map
->map_ops
[i
].handle
);
298 static int __unmap_grant_pages(struct grant_map
*map
, int offset
, int pages
)
302 if (map
->notify
.flags
& UNMAP_NOTIFY_CLEAR_BYTE
) {
303 int pgno
= (map
->notify
.addr
>> PAGE_SHIFT
);
304 if (pgno
>= offset
&& pgno
< offset
+ pages
&& use_ptemod
) {
305 void __user
*tmp
= (void __user
*)
306 map
->vma
->vm_start
+ map
->notify
.addr
;
307 err
= copy_to_user(tmp
, &err
, 1);
310 map
->notify
.flags
&= ~UNMAP_NOTIFY_CLEAR_BYTE
;
311 } else if (pgno
>= offset
&& pgno
< offset
+ pages
) {
312 uint8_t *tmp
= kmap(map
->pages
[pgno
]);
313 tmp
[map
->notify
.addr
& (PAGE_SIZE
-1)] = 0;
314 kunmap(map
->pages
[pgno
]);
315 map
->notify
.flags
&= ~UNMAP_NOTIFY_CLEAR_BYTE
;
319 err
= gnttab_unmap_refs(map
->unmap_ops
+ offset
,
320 use_ptemod
? map
->kmap_ops
+ offset
: NULL
, map
->pages
+ offset
,
325 for (i
= 0; i
< pages
; i
++) {
326 if (map
->unmap_ops
[offset
+i
].status
)
328 pr_debug("unmap handle=%d st=%d\n",
329 map
->unmap_ops
[offset
+i
].handle
,
330 map
->unmap_ops
[offset
+i
].status
);
331 map
->unmap_ops
[offset
+i
].handle
= -1;
336 static int unmap_grant_pages(struct grant_map
*map
, int offset
, int pages
)
340 pr_debug("unmap %d+%d [%d+%d]\n", map
->index
, map
->count
, offset
, pages
);
342 /* It is possible the requested range will have a "hole" where we
343 * already unmapped some of the grants. Only unmap valid ranges.
345 while (pages
&& !err
) {
346 while (pages
&& map
->unmap_ops
[offset
].handle
== -1) {
351 while (range
< pages
) {
352 if (map
->unmap_ops
[offset
+range
].handle
== -1) {
358 err
= __unmap_grant_pages(map
, offset
, range
);
366 /* ------------------------------------------------------------------ */
368 static void gntdev_vma_open(struct vm_area_struct
*vma
)
370 struct grant_map
*map
= vma
->vm_private_data
;
372 pr_debug("gntdev_vma_open %p\n", vma
);
373 atomic_inc(&map
->users
);
376 static void gntdev_vma_close(struct vm_area_struct
*vma
)
378 struct grant_map
*map
= vma
->vm_private_data
;
380 pr_debug("gntdev_vma_close %p\n", vma
);
382 vma
->vm_private_data
= NULL
;
386 static struct vm_operations_struct gntdev_vmops
= {
387 .open
= gntdev_vma_open
,
388 .close
= gntdev_vma_close
,
391 /* ------------------------------------------------------------------ */
393 static void mn_invl_range_start(struct mmu_notifier
*mn
,
394 struct mm_struct
*mm
,
395 unsigned long start
, unsigned long end
)
397 struct gntdev_priv
*priv
= container_of(mn
, struct gntdev_priv
, mn
);
398 struct grant_map
*map
;
399 unsigned long mstart
, mend
;
402 spin_lock(&priv
->lock
);
403 list_for_each_entry(map
, &priv
->maps
, next
) {
406 if (map
->vma
->vm_start
>= end
)
408 if (map
->vma
->vm_end
<= start
)
410 mstart
= max(start
, map
->vma
->vm_start
);
411 mend
= min(end
, map
->vma
->vm_end
);
412 pr_debug("map %d+%d (%lx %lx), range %lx %lx, mrange %lx %lx\n",
413 map
->index
, map
->count
,
414 map
->vma
->vm_start
, map
->vma
->vm_end
,
415 start
, end
, mstart
, mend
);
416 err
= unmap_grant_pages(map
,
417 (mstart
- map
->vma
->vm_start
) >> PAGE_SHIFT
,
418 (mend
- mstart
) >> PAGE_SHIFT
);
421 spin_unlock(&priv
->lock
);
424 static void mn_invl_page(struct mmu_notifier
*mn
,
425 struct mm_struct
*mm
,
426 unsigned long address
)
428 mn_invl_range_start(mn
, mm
, address
, address
+ PAGE_SIZE
);
431 static void mn_release(struct mmu_notifier
*mn
,
432 struct mm_struct
*mm
)
434 struct gntdev_priv
*priv
= container_of(mn
, struct gntdev_priv
, mn
);
435 struct grant_map
*map
;
438 spin_lock(&priv
->lock
);
439 list_for_each_entry(map
, &priv
->maps
, next
) {
442 pr_debug("map %d+%d (%lx %lx)\n",
443 map
->index
, map
->count
,
444 map
->vma
->vm_start
, map
->vma
->vm_end
);
445 err
= unmap_grant_pages(map
, /* offset */ 0, map
->count
);
448 spin_unlock(&priv
->lock
);
451 static struct mmu_notifier_ops gntdev_mmu_ops
= {
452 .release
= mn_release
,
453 .invalidate_page
= mn_invl_page
,
454 .invalidate_range_start
= mn_invl_range_start
,
457 /* ------------------------------------------------------------------ */
459 static int gntdev_open(struct inode
*inode
, struct file
*flip
)
461 struct gntdev_priv
*priv
;
464 priv
= kzalloc(sizeof(*priv
), GFP_KERNEL
);
468 INIT_LIST_HEAD(&priv
->maps
);
469 spin_lock_init(&priv
->lock
);
472 priv
->mm
= get_task_mm(current
);
477 priv
->mn
.ops
= &gntdev_mmu_ops
;
478 ret
= mmu_notifier_register(&priv
->mn
, priv
->mm
);
487 flip
->private_data
= priv
;
488 pr_debug("priv %p\n", priv
);
493 static int gntdev_release(struct inode
*inode
, struct file
*flip
)
495 struct gntdev_priv
*priv
= flip
->private_data
;
496 struct grant_map
*map
;
498 pr_debug("priv %p\n", priv
);
500 while (!list_empty(&priv
->maps
)) {
501 map
= list_entry(priv
->maps
.next
, struct grant_map
, next
);
502 list_del(&map
->next
);
507 mmu_notifier_unregister(&priv
->mn
, priv
->mm
);
512 static long gntdev_ioctl_map_grant_ref(struct gntdev_priv
*priv
,
513 struct ioctl_gntdev_map_grant_ref __user
*u
)
515 struct ioctl_gntdev_map_grant_ref op
;
516 struct grant_map
*map
;
519 if (copy_from_user(&op
, u
, sizeof(op
)) != 0)
521 pr_debug("priv %p, add %d\n", priv
, op
.count
);
522 if (unlikely(op
.count
<= 0))
526 map
= gntdev_alloc_map(priv
, op
.count
);
530 if (unlikely(atomic_add_return(op
.count
, &pages_mapped
) > limit
)) {
531 pr_debug("can't map: over limit\n");
536 if (copy_from_user(map
->grants
, &u
->refs
,
537 sizeof(map
->grants
[0]) * op
.count
) != 0) {
542 spin_lock(&priv
->lock
);
543 gntdev_add_map(priv
, map
);
544 op
.index
= map
->index
<< PAGE_SHIFT
;
545 spin_unlock(&priv
->lock
);
547 if (copy_to_user(u
, &op
, sizeof(op
)) != 0)
553 static long gntdev_ioctl_unmap_grant_ref(struct gntdev_priv
*priv
,
554 struct ioctl_gntdev_unmap_grant_ref __user
*u
)
556 struct ioctl_gntdev_unmap_grant_ref op
;
557 struct grant_map
*map
;
560 if (copy_from_user(&op
, u
, sizeof(op
)) != 0)
562 pr_debug("priv %p, del %d+%d\n", priv
, (int)op
.index
, (int)op
.count
);
564 spin_lock(&priv
->lock
);
565 map
= gntdev_find_map_index(priv
, op
.index
>> PAGE_SHIFT
, op
.count
);
567 list_del(&map
->next
);
570 spin_unlock(&priv
->lock
);
576 static long gntdev_ioctl_get_offset_for_vaddr(struct gntdev_priv
*priv
,
577 struct ioctl_gntdev_get_offset_for_vaddr __user
*u
)
579 struct ioctl_gntdev_get_offset_for_vaddr op
;
580 struct vm_area_struct
*vma
;
581 struct grant_map
*map
;
583 if (copy_from_user(&op
, u
, sizeof(op
)) != 0)
585 pr_debug("priv %p, offset for vaddr %lx\n", priv
, (unsigned long)op
.vaddr
);
587 vma
= find_vma(current
->mm
, op
.vaddr
);
588 if (!vma
|| vma
->vm_ops
!= &gntdev_vmops
)
591 map
= vma
->vm_private_data
;
595 op
.offset
= map
->index
<< PAGE_SHIFT
;
596 op
.count
= map
->count
;
598 if (copy_to_user(u
, &op
, sizeof(op
)) != 0)
603 static long gntdev_ioctl_notify(struct gntdev_priv
*priv
, void __user
*u
)
605 struct ioctl_gntdev_unmap_notify op
;
606 struct grant_map
*map
;
609 unsigned int out_event
;
611 if (copy_from_user(&op
, u
, sizeof(op
)))
614 if (op
.action
& ~(UNMAP_NOTIFY_CLEAR_BYTE
|UNMAP_NOTIFY_SEND_EVENT
))
617 /* We need to grab a reference to the event channel we are going to use
618 * to send the notify before releasing the reference we may already have
619 * (if someone has called this ioctl twice). This is required so that
620 * it is possible to change the clear_byte part of the notification
621 * without disturbing the event channel part, which may now be the last
622 * reference to that event channel.
624 if (op
.action
& UNMAP_NOTIFY_SEND_EVENT
) {
625 if (evtchn_get(op
.event_channel_port
))
629 out_flags
= op
.action
;
630 out_event
= op
.event_channel_port
;
632 spin_lock(&priv
->lock
);
634 list_for_each_entry(map
, &priv
->maps
, next
) {
635 uint64_t begin
= map
->index
<< PAGE_SHIFT
;
636 uint64_t end
= (map
->index
+ map
->count
) << PAGE_SHIFT
;
637 if (op
.index
>= begin
&& op
.index
< end
)
644 if ((op
.action
& UNMAP_NOTIFY_CLEAR_BYTE
) &&
645 (map
->flags
& GNTMAP_readonly
)) {
650 out_flags
= map
->notify
.flags
;
651 out_event
= map
->notify
.event
;
653 map
->notify
.flags
= op
.action
;
654 map
->notify
.addr
= op
.index
- (map
->index
<< PAGE_SHIFT
);
655 map
->notify
.event
= op
.event_channel_port
;
660 spin_unlock(&priv
->lock
);
662 /* Drop the reference to the event channel we did not save in the map */
663 if (out_flags
& UNMAP_NOTIFY_SEND_EVENT
)
664 evtchn_put(out_event
);
669 static long gntdev_ioctl(struct file
*flip
,
670 unsigned int cmd
, unsigned long arg
)
672 struct gntdev_priv
*priv
= flip
->private_data
;
673 void __user
*ptr
= (void __user
*)arg
;
676 case IOCTL_GNTDEV_MAP_GRANT_REF
:
677 return gntdev_ioctl_map_grant_ref(priv
, ptr
);
679 case IOCTL_GNTDEV_UNMAP_GRANT_REF
:
680 return gntdev_ioctl_unmap_grant_ref(priv
, ptr
);
682 case IOCTL_GNTDEV_GET_OFFSET_FOR_VADDR
:
683 return gntdev_ioctl_get_offset_for_vaddr(priv
, ptr
);
685 case IOCTL_GNTDEV_SET_UNMAP_NOTIFY
:
686 return gntdev_ioctl_notify(priv
, ptr
);
689 pr_debug("priv %p, unknown cmd %x\n", priv
, cmd
);
696 static int gntdev_mmap(struct file
*flip
, struct vm_area_struct
*vma
)
698 struct gntdev_priv
*priv
= flip
->private_data
;
699 int index
= vma
->vm_pgoff
;
700 int count
= (vma
->vm_end
- vma
->vm_start
) >> PAGE_SHIFT
;
701 struct grant_map
*map
;
702 int i
, err
= -EINVAL
;
704 if ((vma
->vm_flags
& VM_WRITE
) && !(vma
->vm_flags
& VM_SHARED
))
707 pr_debug("map %d+%d at %lx (pgoff %lx)\n",
708 index
, count
, vma
->vm_start
, vma
->vm_pgoff
);
710 spin_lock(&priv
->lock
);
711 map
= gntdev_find_map_index(priv
, index
, count
);
714 if (use_ptemod
&& map
->vma
)
716 if (use_ptemod
&& priv
->mm
!= vma
->vm_mm
) {
717 printk(KERN_WARNING
"Huh? Other mm?\n");
721 atomic_inc(&map
->users
);
723 vma
->vm_ops
= &gntdev_vmops
;
725 vma
->vm_flags
|= VM_DONTEXPAND
| VM_DONTDUMP
;
728 vma
->vm_flags
|= VM_DONTCOPY
;
730 vma
->vm_private_data
= map
;
736 if ((vma
->vm_flags
& VM_WRITE
) &&
737 (map
->flags
& GNTMAP_readonly
))
740 map
->flags
= GNTMAP_host_map
;
741 if (!(vma
->vm_flags
& VM_WRITE
))
742 map
->flags
|= GNTMAP_readonly
;
745 spin_unlock(&priv
->lock
);
748 err
= apply_to_page_range(vma
->vm_mm
, vma
->vm_start
,
749 vma
->vm_end
- vma
->vm_start
,
750 find_grant_ptes
, map
);
752 printk(KERN_WARNING
"find_grant_ptes() failure.\n");
757 err
= map_grant_pages(map
);
762 for (i
= 0; i
< count
; i
++) {
763 err
= vm_insert_page(vma
, vma
->vm_start
+ i
*PAGE_SIZE
,
773 spin_unlock(&priv
->lock
);
777 spin_unlock(&priv
->lock
);
785 static const struct file_operations gntdev_fops
= {
786 .owner
= THIS_MODULE
,
788 .release
= gntdev_release
,
790 .unlocked_ioctl
= gntdev_ioctl
793 static struct miscdevice gntdev_miscdev
= {
794 .minor
= MISC_DYNAMIC_MINOR
,
795 .name
= "xen/gntdev",
796 .fops
= &gntdev_fops
,
799 /* ------------------------------------------------------------------ */
801 static int __init
gntdev_init(void)
808 use_ptemod
= xen_pv_domain();
810 err
= misc_register(&gntdev_miscdev
);
812 printk(KERN_ERR
"Could not register gntdev device\n");
818 static void __exit
gntdev_exit(void)
820 misc_deregister(&gntdev_miscdev
);
823 module_init(gntdev_init
);
824 module_exit(gntdev_exit
);
826 /* ------------------------------------------------------------------ */