2 * User-space Probes (UProbes)
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 * Copyright (C) IBM Corporation, 2008-2012
22 * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
25 #include <linux/kernel.h>
26 #include <linux/highmem.h>
27 #include <linux/pagemap.h> /* read_mapping_page */
28 #include <linux/slab.h>
29 #include <linux/sched.h>
30 #include <linux/rmap.h> /* anon_vma_prepare */
31 #include <linux/mmu_notifier.h> /* set_pte_at_notify */
32 #include <linux/swap.h> /* try_to_free_swap */
34 #include <linux/uprobes.h>
36 static struct rb_root uprobes_tree
= RB_ROOT
;
38 static DEFINE_SPINLOCK(uprobes_treelock
); /* serialize rbtree access */
40 #define UPROBES_HASH_SZ 13
42 /* serialize (un)register */
43 static struct mutex uprobes_mutex
[UPROBES_HASH_SZ
];
45 #define uprobes_hash(v) (&uprobes_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])
47 /* serialize uprobe->pending_list */
48 static struct mutex uprobes_mmap_mutex
[UPROBES_HASH_SZ
];
49 #define uprobes_mmap_hash(v) (&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])
52 * uprobe_events allows us to skip the uprobe_mmap if there are no uprobe
53 * events active at this time. Probably a fine grained per inode count is
56 static atomic_t uprobe_events
= ATOMIC_INIT(0);
59 * Maintain a temporary per vma info that can be used to search if a vma
60 * has already been handled. This structure is introduced since extending
61 * vm_area_struct wasnt recommended.
64 struct list_head probe_list
;
70 struct rb_node rb_node
; /* node in the rb tree */
72 struct rw_semaphore consumer_rwsem
;
73 struct list_head pending_list
;
74 struct uprobe_consumer
*consumers
;
75 struct inode
*inode
; /* Also hold a ref to inode */
78 struct arch_uprobe arch
;
82 * valid_vma: Verify if the specified vma is an executable vma
83 * Relax restrictions while unregistering: vm_flags might have
84 * changed after breakpoint was inserted.
85 * - is_register: indicates if we are in register context.
86 * - Return 1 if the specified virtual address is in an
89 static bool valid_vma(struct vm_area_struct
*vma
, bool is_register
)
97 if ((vma
->vm_flags
& (VM_READ
|VM_WRITE
|VM_EXEC
|VM_SHARED
)) == (VM_READ
|VM_EXEC
))
103 static loff_t
vma_address(struct vm_area_struct
*vma
, loff_t offset
)
107 vaddr
= vma
->vm_start
+ offset
;
108 vaddr
-= vma
->vm_pgoff
<< PAGE_SHIFT
;
114 * __replace_page - replace page in vma by new page.
115 * based on replace_page in mm/ksm.c
117 * @vma: vma that holds the pte pointing to page
118 * @page: the cowed page we are replacing by kpage
119 * @kpage: the modified page we replace page by
121 * Returns 0 on success, -EFAULT on failure.
123 static int __replace_page(struct vm_area_struct
*vma
, struct page
*page
, struct page
*kpage
)
125 struct mm_struct
*mm
= vma
->vm_mm
;
134 addr
= page_address_in_vma(page
, vma
);
138 pgd
= pgd_offset(mm
, addr
);
139 if (!pgd_present(*pgd
))
142 pud
= pud_offset(pgd
, addr
);
143 if (!pud_present(*pud
))
146 pmd
= pmd_offset(pud
, addr
);
147 if (!pmd_present(*pmd
))
150 ptep
= pte_offset_map_lock(mm
, pmd
, addr
, &ptl
);
155 page_add_new_anon_rmap(kpage
, vma
, addr
);
157 flush_cache_page(vma
, addr
, pte_pfn(*ptep
));
158 ptep_clear_flush(vma
, addr
, ptep
);
159 set_pte_at_notify(mm
, addr
, ptep
, mk_pte(kpage
, vma
->vm_page_prot
));
161 page_remove_rmap(page
);
162 if (!page_mapped(page
))
163 try_to_free_swap(page
);
165 pte_unmap_unlock(ptep
, ptl
);
173 * is_bkpt_insn - check if instruction is breakpoint instruction.
174 * @insn: instruction to be checked.
175 * Default implementation of is_bkpt_insn
176 * Returns true if @insn is a breakpoint instruction.
178 bool __weak
is_bkpt_insn(uprobe_opcode_t
*insn
)
180 return *insn
== UPROBE_BKPT_INSN
;
185 * Expect the breakpoint instruction to be the smallest size instruction for
186 * the architecture. If an arch has variable length instruction and the
187 * breakpoint instruction is not of the smallest length instruction
188 * supported by that architecture then we need to modify read_opcode /
189 * write_opcode accordingly. This would never be a problem for archs that
190 * have fixed length instructions.
194 * write_opcode - write the opcode at a given virtual address.
195 * @auprobe: arch breakpointing information.
196 * @mm: the probed process address space.
197 * @vaddr: the virtual address to store the opcode.
198 * @opcode: opcode to be written at @vaddr.
200 * Called with mm->mmap_sem held (for read and with a reference to
203 * For mm @mm, write the opcode at @vaddr.
204 * Return 0 (success) or a negative errno.
206 static int write_opcode(struct arch_uprobe
*auprobe
, struct mm_struct
*mm
,
207 unsigned long vaddr
, uprobe_opcode_t opcode
)
209 struct page
*old_page
, *new_page
;
210 struct address_space
*mapping
;
211 void *vaddr_old
, *vaddr_new
;
212 struct vm_area_struct
*vma
;
213 struct uprobe
*uprobe
;
217 /* Read the page with vaddr into memory */
218 ret
= get_user_pages(NULL
, mm
, vaddr
, 1, 0, 0, &old_page
, &vma
);
225 * We are interested in text pages only. Our pages of interest
226 * should be mapped for read and execute only. We desist from
227 * adding probes in write mapped pages since the breakpoints
228 * might end up in the file copy.
230 if (!valid_vma(vma
, is_bkpt_insn(&opcode
)))
233 uprobe
= container_of(auprobe
, struct uprobe
, arch
);
234 mapping
= uprobe
->inode
->i_mapping
;
235 if (mapping
!= vma
->vm_file
->f_mapping
)
238 addr
= vma_address(vma
, uprobe
->offset
);
239 if (vaddr
!= (unsigned long)addr
)
243 new_page
= alloc_page_vma(GFP_HIGHUSER_MOVABLE
, vma
, vaddr
);
247 __SetPageUptodate(new_page
);
250 * lock page will serialize against do_wp_page()'s
251 * PageAnon() handling
254 /* copy the page now that we've got it stable */
255 vaddr_old
= kmap_atomic(old_page
);
256 vaddr_new
= kmap_atomic(new_page
);
258 memcpy(vaddr_new
, vaddr_old
, PAGE_SIZE
);
260 /* poke the new insn in, ASSUMES we don't cross page boundary */
262 BUG_ON(vaddr
+ UPROBE_BKPT_INSN_SIZE
> PAGE_SIZE
);
263 memcpy(vaddr_new
+ vaddr
, &opcode
, UPROBE_BKPT_INSN_SIZE
);
265 kunmap_atomic(vaddr_new
);
266 kunmap_atomic(vaddr_old
);
268 ret
= anon_vma_prepare(vma
);
273 ret
= __replace_page(vma
, old_page
, new_page
);
274 unlock_page(new_page
);
277 unlock_page(old_page
);
278 page_cache_release(new_page
);
287 * read_opcode - read the opcode at a given virtual address.
288 * @mm: the probed process address space.
289 * @vaddr: the virtual address to read the opcode.
290 * @opcode: location to store the read opcode.
292 * Called with mm->mmap_sem held (for read and with a reference to
295 * For mm @mm, read the opcode at @vaddr and store it in @opcode.
296 * Return 0 (success) or a negative errno.
298 static int read_opcode(struct mm_struct
*mm
, unsigned long vaddr
, uprobe_opcode_t
*opcode
)
304 ret
= get_user_pages(NULL
, mm
, vaddr
, 1, 0, 0, &page
, NULL
);
309 vaddr_new
= kmap_atomic(page
);
311 memcpy(opcode
, vaddr_new
+ vaddr
, UPROBE_BKPT_INSN_SIZE
);
312 kunmap_atomic(vaddr_new
);
320 static int is_bkpt_at_addr(struct mm_struct
*mm
, unsigned long vaddr
)
322 uprobe_opcode_t opcode
;
325 result
= read_opcode(mm
, vaddr
, &opcode
);
329 if (is_bkpt_insn(&opcode
))
336 * set_bkpt - store breakpoint at a given address.
337 * @auprobe: arch specific probepoint information.
338 * @mm: the probed process address space.
339 * @vaddr: the virtual address to insert the opcode.
341 * For mm @mm, store the breakpoint instruction at @vaddr.
342 * Return 0 (success) or a negative errno.
344 int __weak
set_bkpt(struct arch_uprobe
*auprobe
, struct mm_struct
*mm
, unsigned long vaddr
)
348 result
= is_bkpt_at_addr(mm
, vaddr
);
355 return write_opcode(auprobe
, mm
, vaddr
, UPROBE_BKPT_INSN
);
359 * set_orig_insn - Restore the original instruction.
360 * @mm: the probed process address space.
361 * @auprobe: arch specific probepoint information.
362 * @vaddr: the virtual address to insert the opcode.
363 * @verify: if true, verify existance of breakpoint instruction.
365 * For mm @mm, restore the original opcode (opcode) at @vaddr.
366 * Return 0 (success) or a negative errno.
369 set_orig_insn(struct arch_uprobe
*auprobe
, struct mm_struct
*mm
, unsigned long vaddr
, bool verify
)
374 result
= is_bkpt_at_addr(mm
, vaddr
);
381 return write_opcode(auprobe
, mm
, vaddr
, *(uprobe_opcode_t
*)auprobe
->insn
);
384 static int match_uprobe(struct uprobe
*l
, struct uprobe
*r
)
386 if (l
->inode
< r
->inode
)
389 if (l
->inode
> r
->inode
)
392 if (l
->offset
< r
->offset
)
395 if (l
->offset
> r
->offset
)
401 static struct uprobe
*__find_uprobe(struct inode
*inode
, loff_t offset
)
403 struct uprobe u
= { .inode
= inode
, .offset
= offset
};
404 struct rb_node
*n
= uprobes_tree
.rb_node
;
405 struct uprobe
*uprobe
;
409 uprobe
= rb_entry(n
, struct uprobe
, rb_node
);
410 match
= match_uprobe(&u
, uprobe
);
412 atomic_inc(&uprobe
->ref
);
425 * Find a uprobe corresponding to a given inode:offset
426 * Acquires uprobes_treelock
428 static struct uprobe
*find_uprobe(struct inode
*inode
, loff_t offset
)
430 struct uprobe
*uprobe
;
433 spin_lock_irqsave(&uprobes_treelock
, flags
);
434 uprobe
= __find_uprobe(inode
, offset
);
435 spin_unlock_irqrestore(&uprobes_treelock
, flags
);
440 static struct uprobe
*__insert_uprobe(struct uprobe
*uprobe
)
442 struct rb_node
**p
= &uprobes_tree
.rb_node
;
443 struct rb_node
*parent
= NULL
;
449 u
= rb_entry(parent
, struct uprobe
, rb_node
);
450 match
= match_uprobe(uprobe
, u
);
457 p
= &parent
->rb_left
;
459 p
= &parent
->rb_right
;
464 rb_link_node(&uprobe
->rb_node
, parent
, p
);
465 rb_insert_color(&uprobe
->rb_node
, &uprobes_tree
);
466 /* get access + creation ref */
467 atomic_set(&uprobe
->ref
, 2);
473 * Acquire uprobes_treelock.
474 * Matching uprobe already exists in rbtree;
475 * increment (access refcount) and return the matching uprobe.
477 * No matching uprobe; insert the uprobe in rb_tree;
478 * get a double refcount (access + creation) and return NULL.
480 static struct uprobe
*insert_uprobe(struct uprobe
*uprobe
)
485 spin_lock_irqsave(&uprobes_treelock
, flags
);
486 u
= __insert_uprobe(uprobe
);
487 spin_unlock_irqrestore(&uprobes_treelock
, flags
);
492 static void put_uprobe(struct uprobe
*uprobe
)
494 if (atomic_dec_and_test(&uprobe
->ref
))
498 static struct uprobe
*alloc_uprobe(struct inode
*inode
, loff_t offset
)
500 struct uprobe
*uprobe
, *cur_uprobe
;
502 uprobe
= kzalloc(sizeof(struct uprobe
), GFP_KERNEL
);
506 uprobe
->inode
= igrab(inode
);
507 uprobe
->offset
= offset
;
508 init_rwsem(&uprobe
->consumer_rwsem
);
509 INIT_LIST_HEAD(&uprobe
->pending_list
);
511 /* add to uprobes_tree, sorted on inode:offset */
512 cur_uprobe
= insert_uprobe(uprobe
);
514 /* a uprobe exists for this inode:offset combination */
520 atomic_inc(&uprobe_events
);
526 /* Returns the previous consumer */
527 static struct uprobe_consumer
*
528 consumer_add(struct uprobe
*uprobe
, struct uprobe_consumer
*uc
)
530 down_write(&uprobe
->consumer_rwsem
);
531 uc
->next
= uprobe
->consumers
;
532 uprobe
->consumers
= uc
;
533 up_write(&uprobe
->consumer_rwsem
);
539 * For uprobe @uprobe, delete the consumer @uc.
540 * Return true if the @uc is deleted successfully
543 static bool consumer_del(struct uprobe
*uprobe
, struct uprobe_consumer
*uc
)
545 struct uprobe_consumer
**con
;
548 down_write(&uprobe
->consumer_rwsem
);
549 for (con
= &uprobe
->consumers
; *con
; con
= &(*con
)->next
) {
556 up_write(&uprobe
->consumer_rwsem
);
562 __copy_insn(struct address_space
*mapping
, struct vm_area_struct
*vma
, char *insn
,
563 unsigned long nbytes
, unsigned long offset
)
565 struct file
*filp
= vma
->vm_file
;
574 idx
= (unsigned long)(offset
>> PAGE_CACHE_SHIFT
);
575 off1
= offset
&= ~PAGE_MASK
;
578 * Ensure that the page that has the original instruction is
579 * populated and in page-cache.
581 page
= read_mapping_page(mapping
, idx
, filp
);
583 return PTR_ERR(page
);
585 vaddr
= kmap_atomic(page
);
586 memcpy(insn
, vaddr
+ off1
, nbytes
);
587 kunmap_atomic(vaddr
);
588 page_cache_release(page
);
594 copy_insn(struct uprobe
*uprobe
, struct vm_area_struct
*vma
, unsigned long addr
)
596 struct address_space
*mapping
;
597 unsigned long nbytes
;
601 nbytes
= PAGE_SIZE
- addr
;
602 mapping
= uprobe
->inode
->i_mapping
;
604 /* Instruction at end of binary; copy only available bytes */
605 if (uprobe
->offset
+ MAX_UINSN_BYTES
> uprobe
->inode
->i_size
)
606 bytes
= uprobe
->inode
->i_size
- uprobe
->offset
;
608 bytes
= MAX_UINSN_BYTES
;
610 /* Instruction at the page-boundary; copy bytes in second page */
611 if (nbytes
< bytes
) {
612 if (__copy_insn(mapping
, vma
, uprobe
->arch
.insn
+ nbytes
,
613 bytes
- nbytes
, uprobe
->offset
+ nbytes
))
618 return __copy_insn(mapping
, vma
, uprobe
->arch
.insn
, bytes
, uprobe
->offset
);
622 install_breakpoint(struct uprobe
*uprobe
, struct mm_struct
*mm
,
623 struct vm_area_struct
*vma
, loff_t vaddr
)
629 * If probe is being deleted, unregister thread could be done with
630 * the vma-rmap-walk through. Adding a probe now can be fatal since
631 * nobody will be able to cleanup. Also we could be from fork or
632 * mremap path, where the probe might have already been inserted.
633 * Hence behave as if probe already existed.
635 if (!uprobe
->consumers
)
638 addr
= (unsigned long)vaddr
;
640 if (!(uprobe
->flags
& UPROBE_COPY_INSN
)) {
641 ret
= copy_insn(uprobe
, vma
, addr
);
645 if (is_bkpt_insn((uprobe_opcode_t
*)uprobe
->arch
.insn
))
648 ret
= arch_uprobes_analyze_insn(&uprobe
->arch
, mm
);
652 uprobe
->flags
|= UPROBE_COPY_INSN
;
654 ret
= set_bkpt(&uprobe
->arch
, mm
, addr
);
660 remove_breakpoint(struct uprobe
*uprobe
, struct mm_struct
*mm
, loff_t vaddr
)
662 set_orig_insn(&uprobe
->arch
, mm
, (unsigned long)vaddr
, true);
665 static void delete_uprobe(struct uprobe
*uprobe
)
669 spin_lock_irqsave(&uprobes_treelock
, flags
);
670 rb_erase(&uprobe
->rb_node
, &uprobes_tree
);
671 spin_unlock_irqrestore(&uprobes_treelock
, flags
);
674 atomic_dec(&uprobe_events
);
677 static struct vma_info
*
678 __find_next_vma_info(struct address_space
*mapping
, struct list_head
*head
,
679 struct vma_info
*vi
, loff_t offset
, bool is_register
)
681 struct prio_tree_iter iter
;
682 struct vm_area_struct
*vma
;
683 struct vma_info
*tmpvi
;
688 pgoff
= offset
>> PAGE_SHIFT
;
690 vma_prio_tree_foreach(vma
, &iter
, &mapping
->i_mmap
, pgoff
, pgoff
) {
691 if (!valid_vma(vma
, is_register
))
695 vaddr
= vma_address(vma
, offset
);
697 list_for_each_entry(tmpvi
, head
, probe_list
) {
698 if (tmpvi
->mm
== vma
->vm_mm
&& tmpvi
->vaddr
== vaddr
) {
705 * Another vma needs a probe to be installed. However skip
706 * installing the probe if the vma is about to be unlinked.
708 if (!existing_vma
&& atomic_inc_not_zero(&vma
->vm_mm
->mm_users
)) {
711 list_add(&vi
->probe_list
, head
);
721 * Iterate in the rmap prio tree and find a vma where a probe has not
724 static struct vma_info
*
725 find_next_vma_info(struct address_space
*mapping
, struct list_head
*head
,
726 loff_t offset
, bool is_register
)
728 struct vma_info
*vi
, *retvi
;
730 vi
= kzalloc(sizeof(struct vma_info
), GFP_KERNEL
);
732 return ERR_PTR(-ENOMEM
);
734 mutex_lock(&mapping
->i_mmap_mutex
);
735 retvi
= __find_next_vma_info(mapping
, head
, vi
, offset
, is_register
);
736 mutex_unlock(&mapping
->i_mmap_mutex
);
744 static int register_for_each_vma(struct uprobe
*uprobe
, bool is_register
)
746 struct list_head try_list
;
747 struct vm_area_struct
*vma
;
748 struct address_space
*mapping
;
749 struct vma_info
*vi
, *tmpvi
;
750 struct mm_struct
*mm
;
754 mapping
= uprobe
->inode
->i_mapping
;
755 INIT_LIST_HEAD(&try_list
);
760 vi
= find_next_vma_info(mapping
, &try_list
, uprobe
->offset
, is_register
);
770 down_read(&mm
->mmap_sem
);
771 vma
= find_vma(mm
, (unsigned long)vi
->vaddr
);
772 if (!vma
|| !valid_vma(vma
, is_register
)) {
773 list_del(&vi
->probe_list
);
775 up_read(&mm
->mmap_sem
);
779 vaddr
= vma_address(vma
, uprobe
->offset
);
780 if (vma
->vm_file
->f_mapping
->host
!= uprobe
->inode
||
781 vaddr
!= vi
->vaddr
) {
782 list_del(&vi
->probe_list
);
784 up_read(&mm
->mmap_sem
);
790 ret
= install_breakpoint(uprobe
, mm
, vma
, vi
->vaddr
);
792 remove_breakpoint(uprobe
, mm
, vi
->vaddr
);
794 up_read(&mm
->mmap_sem
);
797 if (ret
&& ret
== -EEXIST
)
804 list_for_each_entry_safe(vi
, tmpvi
, &try_list
, probe_list
) {
805 list_del(&vi
->probe_list
);
812 static int __uprobe_register(struct uprobe
*uprobe
)
814 return register_for_each_vma(uprobe
, true);
817 static void __uprobe_unregister(struct uprobe
*uprobe
)
819 if (!register_for_each_vma(uprobe
, false))
820 delete_uprobe(uprobe
);
822 /* TODO : cant unregister? schedule a worker thread */
826 * uprobe_register - register a probe
827 * @inode: the file in which the probe has to be placed.
828 * @offset: offset from the start of the file.
829 * @uc: information on howto handle the probe..
831 * Apart from the access refcount, uprobe_register() takes a creation
832 * refcount (thro alloc_uprobe) if and only if this @uprobe is getting
833 * inserted into the rbtree (i.e first consumer for a @inode:@offset
834 * tuple). Creation refcount stops uprobe_unregister from freeing the
835 * @uprobe even before the register operation is complete. Creation
836 * refcount is released when the last @uc for the @uprobe
839 * Return errno if it cannot successully install probes
840 * else return 0 (success)
842 int uprobe_register(struct inode
*inode
, loff_t offset
, struct uprobe_consumer
*uc
)
844 struct uprobe
*uprobe
;
847 if (!inode
|| !uc
|| uc
->next
)
850 if (offset
> i_size_read(inode
))
854 mutex_lock(uprobes_hash(inode
));
855 uprobe
= alloc_uprobe(inode
, offset
);
857 if (uprobe
&& !consumer_add(uprobe
, uc
)) {
858 ret
= __uprobe_register(uprobe
);
860 uprobe
->consumers
= NULL
;
861 __uprobe_unregister(uprobe
);
863 uprobe
->flags
|= UPROBE_RUN_HANDLER
;
867 mutex_unlock(uprobes_hash(inode
));
874 * uprobe_unregister - unregister a already registered probe.
875 * @inode: the file in which the probe has to be removed.
876 * @offset: offset from the start of the file.
877 * @uc: identify which probe if multiple probes are colocated.
879 void uprobe_unregister(struct inode
*inode
, loff_t offset
, struct uprobe_consumer
*uc
)
881 struct uprobe
*uprobe
;
886 uprobe
= find_uprobe(inode
, offset
);
890 mutex_lock(uprobes_hash(inode
));
892 if (consumer_del(uprobe
, uc
)) {
893 if (!uprobe
->consumers
) {
894 __uprobe_unregister(uprobe
);
895 uprobe
->flags
&= ~UPROBE_RUN_HANDLER
;
899 mutex_unlock(uprobes_hash(inode
));
905 * Of all the nodes that correspond to the given inode, return the node
906 * with the least offset.
908 static struct rb_node
*find_least_offset_node(struct inode
*inode
)
910 struct uprobe u
= { .inode
= inode
, .offset
= 0};
911 struct rb_node
*n
= uprobes_tree
.rb_node
;
912 struct rb_node
*close_node
= NULL
;
913 struct uprobe
*uprobe
;
917 uprobe
= rb_entry(n
, struct uprobe
, rb_node
);
918 match
= match_uprobe(&u
, uprobe
);
920 if (uprobe
->inode
== inode
)
936 * For a given inode, build a list of probes that need to be inserted.
938 static void build_probe_list(struct inode
*inode
, struct list_head
*head
)
940 struct uprobe
*uprobe
;
944 spin_lock_irqsave(&uprobes_treelock
, flags
);
946 n
= find_least_offset_node(inode
);
948 for (; n
; n
= rb_next(n
)) {
949 uprobe
= rb_entry(n
, struct uprobe
, rb_node
);
950 if (uprobe
->inode
!= inode
)
953 list_add(&uprobe
->pending_list
, head
);
954 atomic_inc(&uprobe
->ref
);
957 spin_unlock_irqrestore(&uprobes_treelock
, flags
);
961 * Called from mmap_region.
962 * called with mm->mmap_sem acquired.
964 * Return -ve no if we fail to insert probes and we cannot
966 * Return 0 otherwise. i.e:
968 * - successful insertion of probes
969 * - (or) no possible probes to be inserted.
970 * - (or) insertion of probes failed but we can bail-out.
972 int uprobe_mmap(struct vm_area_struct
*vma
)
974 struct list_head tmp_list
;
975 struct uprobe
*uprobe
, *u
;
979 if (!atomic_read(&uprobe_events
) || !valid_vma(vma
, true))
982 inode
= vma
->vm_file
->f_mapping
->host
;
986 INIT_LIST_HEAD(&tmp_list
);
987 mutex_lock(uprobes_mmap_hash(inode
));
988 build_probe_list(inode
, &tmp_list
);
992 list_for_each_entry_safe(uprobe
, u
, &tmp_list
, pending_list
) {
995 list_del(&uprobe
->pending_list
);
997 vaddr
= vma_address(vma
, uprobe
->offset
);
998 if (vaddr
>= vma
->vm_start
&& vaddr
< vma
->vm_end
) {
999 ret
= install_breakpoint(uprobe
, vma
->vm_mm
, vma
, vaddr
);
1000 /* Ignore double add: */
1008 mutex_unlock(uprobes_mmap_hash(inode
));
1013 static int __init
init_uprobes(void)
1017 for (i
= 0; i
< UPROBES_HASH_SZ
; i
++) {
1018 mutex_init(&uprobes_mutex
[i
]);
1019 mutex_init(&uprobes_mmap_mutex
[i
]);
1024 static void __exit
exit_uprobes(void)
1028 module_init(init_uprobes
);
1029 module_exit(exit_uprobes
);