2 * Copyright 2014 IBM Corp.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
10 #include <linux/workqueue.h>
11 #include <linux/sched.h>
12 #include <linux/pid.h>
14 #include <linux/moduleparam.h>
16 #undef MODULE_PARAM_PREFIX
17 #define MODULE_PARAM_PREFIX "cxl" "."
18 #include <asm/current.h>
19 #include <asm/copro.h>
25 static bool sste_matches(struct cxl_sste
*sste
, struct copro_slb
*slb
)
27 return ((sste
->vsid_data
== cpu_to_be64(slb
->vsid
)) &&
28 (sste
->esid_data
== cpu_to_be64(slb
->esid
)));
32 * This finds a free SSTE for the given SLB, or returns NULL if it's already in
35 static struct cxl_sste
* find_free_sste(struct cxl_context
*ctx
,
36 struct copro_slb
*slb
)
38 struct cxl_sste
*primary
, *sste
, *ret
= NULL
;
39 unsigned int mask
= (ctx
->sst_size
>> 7) - 1; /* SSTP0[SegTableSize] */
43 if (slb
->vsid
& SLB_VSID_B_1T
)
44 hash
= (slb
->esid
>> SID_SHIFT_1T
) & mask
;
46 hash
= (slb
->esid
>> SID_SHIFT
) & mask
;
48 primary
= ctx
->sstp
+ (hash
<< 3);
50 for (entry
= 0, sste
= primary
; entry
< 8; entry
++, sste
++) {
51 if (!ret
&& !(be64_to_cpu(sste
->esid_data
) & SLB_ESID_V
))
53 if (sste_matches(sste
, slb
))
59 /* Nothing free, select an entry to cast out */
60 ret
= primary
+ ctx
->sst_lru
;
61 ctx
->sst_lru
= (ctx
->sst_lru
+ 1) & 0x7;
66 static void cxl_load_segment(struct cxl_context
*ctx
, struct copro_slb
*slb
)
68 /* mask is the group index, we search primary and secondary here. */
69 struct cxl_sste
*sste
;
72 spin_lock_irqsave(&ctx
->sste_lock
, flags
);
73 sste
= find_free_sste(ctx
, slb
);
77 pr_devel("CXL Populating SST[%li]: %#llx %#llx\n",
78 sste
- ctx
->sstp
, slb
->vsid
, slb
->esid
);
79 trace_cxl_ste_write(ctx
, sste
- ctx
->sstp
, slb
->esid
, slb
->vsid
);
81 sste
->vsid_data
= cpu_to_be64(slb
->vsid
);
82 sste
->esid_data
= cpu_to_be64(slb
->esid
);
84 spin_unlock_irqrestore(&ctx
->sste_lock
, flags
);
87 static int cxl_fault_segment(struct cxl_context
*ctx
, struct mm_struct
*mm
,
90 struct copro_slb slb
= {0,0};
93 if (!(rc
= copro_calculate_slb(mm
, ea
, &slb
))) {
94 cxl_load_segment(ctx
, &slb
);
100 static void cxl_ack_ae(struct cxl_context
*ctx
)
104 cxl_ops
->ack_irq(ctx
, CXL_PSL_TFC_An_AE
, 0);
106 spin_lock_irqsave(&ctx
->lock
, flags
);
107 ctx
->pending_fault
= true;
108 ctx
->fault_addr
= ctx
->dar
;
109 ctx
->fault_dsisr
= ctx
->dsisr
;
110 spin_unlock_irqrestore(&ctx
->lock
, flags
);
112 wake_up_all(&ctx
->wq
);
115 static int cxl_handle_segment_miss(struct cxl_context
*ctx
,
116 struct mm_struct
*mm
, u64 ea
)
120 pr_devel("CXL interrupt: Segment fault pe: %i ea: %#llx\n", ctx
->pe
, ea
);
121 trace_cxl_ste_miss(ctx
, ea
);
123 if ((rc
= cxl_fault_segment(ctx
, mm
, ea
)))
127 mb(); /* Order seg table write to TFC MMIO write */
128 cxl_ops
->ack_irq(ctx
, CXL_PSL_TFC_An_R
, 0);
134 static void cxl_handle_page_fault(struct cxl_context
*ctx
,
135 struct mm_struct
*mm
, u64 dsisr
, u64 dar
)
139 unsigned long access
, flags
, inv_flags
= 0;
141 trace_cxl_pte_miss(ctx
, dsisr
, dar
);
143 if ((result
= copro_handle_mm_fault(mm
, dar
, dsisr
, &flt
))) {
144 pr_devel("copro_handle_mm_fault failed: %#x\n", result
);
145 return cxl_ack_ae(ctx
);
149 * update_mmu_cache() will not have loaded the hash since current->trap
150 * is not a 0x400 or 0x300, so just call hash_page_mm() here.
152 access
= _PAGE_PRESENT
| _PAGE_READ
;
153 if (dsisr
& CXL_PSL_DSISR_An_S
)
154 access
|= _PAGE_WRITE
;
156 access
|= _PAGE_PRIVILEGED
;
157 if ((!ctx
->kernel
) || (REGION_ID(dar
) == USER_REGION_ID
))
158 access
&= ~_PAGE_PRIVILEGED
;
160 if (dsisr
& DSISR_NOHPTE
)
161 inv_flags
|= HPTE_NOHPTE_UPDATE
;
163 local_irq_save(flags
);
164 hash_page_mm(mm
, dar
, access
, 0x300, inv_flags
);
165 local_irq_restore(flags
);
167 pr_devel("Page fault successfully handled for pe: %i!\n", ctx
->pe
);
168 cxl_ops
->ack_irq(ctx
, CXL_PSL_TFC_An_R
, 0);
172 * Returns the mm_struct corresponding to the context ctx via ctx->pid
173 * In case the task has exited we use the task group leader accessible
174 * via ctx->glpid to find the next task in the thread group that has a
175 * valid mm_struct associated with it. If a task with valid mm_struct
176 * is found the ctx->pid is updated to use the task struct for subsequent
177 * translations. In case no valid mm_struct is found in the task group to
178 * service the fault a NULL is returned.
180 static struct mm_struct
*get_mem_context(struct cxl_context
*ctx
)
182 struct task_struct
*task
= NULL
;
183 struct mm_struct
*mm
= NULL
;
184 struct pid
*old_pid
= ctx
->pid
;
186 if (old_pid
== NULL
) {
187 pr_warn("%s: Invalid context for pe=%d\n",
192 task
= get_pid_task(old_pid
, PIDTYPE_PID
);
195 * pid_alive may look racy but this saves us from costly
196 * get_task_mm when the task is a zombie. In worst case
197 * we may think a task is alive, which is about to die
198 * but get_task_mm will return NULL.
200 if (task
!= NULL
&& pid_alive(task
))
201 mm
= get_task_mm(task
);
203 /* release the task struct that was taken earlier */
205 put_task_struct(task
);
207 pr_devel("%s: Context owning pid=%i for pe=%i dead\n",
208 __func__
, pid_nr(old_pid
), ctx
->pe
);
211 * If we couldn't find the mm context then use the group
212 * leader to iterate over the task group and find a task
213 * that gives us mm_struct.
215 if (unlikely(mm
== NULL
&& ctx
->glpid
!= NULL
)) {
218 task
= pid_task(ctx
->glpid
, PIDTYPE_PID
);
221 mm
= get_task_mm(task
);
223 ctx
->pid
= get_task_pid(task
,
227 task
= next_thread(task
);
228 } while (task
&& !thread_group_leader(task
));
231 /* check if we switched pid */
232 if (ctx
->pid
!= old_pid
) {
234 pr_devel("%s:pe=%i switch pid %i->%i\n",
235 __func__
, ctx
->pe
, pid_nr(old_pid
),
238 pr_devel("%s:Cannot find mm for pid=%i\n",
239 __func__
, pid_nr(old_pid
));
241 /* drop the reference to older pid */
251 void cxl_handle_fault(struct work_struct
*fault_work
)
253 struct cxl_context
*ctx
=
254 container_of(fault_work
, struct cxl_context
, fault_work
);
255 u64 dsisr
= ctx
->dsisr
;
257 struct mm_struct
*mm
= NULL
;
259 if (cpu_has_feature(CPU_FTR_HVMODE
)) {
260 if (cxl_p2n_read(ctx
->afu
, CXL_PSL_DSISR_An
) != dsisr
||
261 cxl_p2n_read(ctx
->afu
, CXL_PSL_DAR_An
) != dar
||
262 cxl_p2n_read(ctx
->afu
, CXL_PSL_PEHandle_An
) != ctx
->pe
) {
263 /* Most likely explanation is harmless - a dedicated
264 * process has detached and these were cleared by the
265 * PSL purge, but warn about it just in case
267 dev_notice(&ctx
->afu
->dev
, "cxl_handle_fault: Translation fault regs changed\n");
272 /* Early return if the context is being / has been detached */
273 if (ctx
->status
== CLOSED
) {
278 pr_devel("CXL BOTTOM HALF handling fault for afu pe: %i. "
279 "DSISR: %#llx DAR: %#llx\n", ctx
->pe
, dsisr
, dar
);
283 mm
= get_mem_context(ctx
);
284 /* indicates all the thread in task group have exited */
286 pr_devel("%s: unable to get mm for pe=%d pid=%i\n",
287 __func__
, ctx
->pe
, pid_nr(ctx
->pid
));
291 pr_devel("Handling page fault for pe=%d pid=%i\n",
292 ctx
->pe
, pid_nr(ctx
->pid
));
296 if (dsisr
& CXL_PSL_DSISR_An_DS
)
297 cxl_handle_segment_miss(ctx
, mm
, dar
);
298 else if (dsisr
& CXL_PSL_DSISR_An_DM
)
299 cxl_handle_page_fault(ctx
, mm
, dsisr
, dar
);
301 WARN(1, "cxl_handle_fault has nothing to handle\n");
307 static void cxl_prefault_one(struct cxl_context
*ctx
, u64 ea
)
309 struct mm_struct
*mm
;
311 mm
= get_mem_context(ctx
);
313 pr_devel("cxl_prefault_one unable to get mm %i\n",
318 cxl_fault_segment(ctx
, mm
, ea
);
323 static u64
next_segment(u64 ea
, u64 vsid
)
325 if (vsid
& SLB_VSID_B_1T
)
326 ea
|= (1ULL << 40) - 1;
328 ea
|= (1ULL << 28) - 1;
333 static void cxl_prefault_vma(struct cxl_context
*ctx
)
335 u64 ea
, last_esid
= 0;
336 struct copro_slb slb
;
337 struct vm_area_struct
*vma
;
339 struct mm_struct
*mm
;
341 mm
= get_mem_context(ctx
);
343 pr_devel("cxl_prefault_vm unable to get mm %i\n",
348 down_read(&mm
->mmap_sem
);
349 for (vma
= mm
->mmap
; vma
; vma
= vma
->vm_next
) {
350 for (ea
= vma
->vm_start
; ea
< vma
->vm_end
;
351 ea
= next_segment(ea
, slb
.vsid
)) {
352 rc
= copro_calculate_slb(mm
, ea
, &slb
);
356 if (last_esid
== slb
.esid
)
359 cxl_load_segment(ctx
, &slb
);
360 last_esid
= slb
.esid
;
363 up_read(&mm
->mmap_sem
);
368 void cxl_prefault(struct cxl_context
*ctx
, u64 wed
)
370 switch (ctx
->afu
->prefault_mode
) {
371 case CXL_PREFAULT_WED
:
372 cxl_prefault_one(ctx
, wed
);
374 case CXL_PREFAULT_ALL
:
375 cxl_prefault_vma(ctx
);
This page took 0.038199 seconds and 6 git commands to generate.