2 * S390 low-level entry points.
4 * Copyright IBM Corp. 1999, 2012
5 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
6 * Hartmut Penner (hp@de.ibm.com),
7 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
8 * Heiko Carstens <heiko.carstens@de.ibm.com>
11 #include <linux/init.h>
12 #include <linux/linkage.h>
13 #include <asm/cache.h>
14 #include <asm/errno.h>
15 #include <asm/ptrace.h>
16 #include <asm/thread_info.h>
17 #include <asm/asm-offsets.h>
18 #include <asm/unistd.h>
24 __PT_R1 = __PT_GPRS + 4
25 __PT_R2 = __PT_GPRS + 8
26 __PT_R3 = __PT_GPRS + 12
27 __PT_R4 = __PT_GPRS + 16
28 __PT_R5 = __PT_GPRS + 20
29 __PT_R6 = __PT_GPRS + 24
30 __PT_R7 = __PT_GPRS + 28
31 __PT_R8 = __PT_GPRS + 32
32 __PT_R9 = __PT_GPRS + 36
33 __PT_R10 = __PT_GPRS + 40
34 __PT_R11 = __PT_GPRS + 44
35 __PT_R12 = __PT_GPRS + 48
36 __PT_R13 = __PT_GPRS + 524
37 __PT_R14 = __PT_GPRS + 56
38 __PT_R15 = __PT_GPRS + 60
40 _TIF_WORK_SVC = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
41 _TIF_MCCK_PENDING | _TIF_PER_TRAP )
42 _TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
44 _TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \
45 _TIF_SYSCALL_TRACEPOINT)
46 _TIF_TRANSFER = (_TIF_MCCK_PENDING | _TIF_TLB_WAIT)
48 STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER
49 STACK_SIZE = 1 << STACK_SHIFT
50 STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE
52 #define BASED(name) name-system_call(%r13)
55 #ifdef CONFIG_TRACE_IRQFLAGS
57 l %r1,BASED(.Lhardirqs_on)
58 basr %r14,%r1 # call trace_hardirqs_on_caller
63 #ifdef CONFIG_TRACE_IRQFLAGS
65 l %r1,BASED(.Lhardirqs_off)
66 basr %r14,%r1 # call trace_hardirqs_off_caller
70 .macro LOCKDEP_SYS_EXIT
72 tm __PT_PSW+1(%r11),0x01 # returning to user ?
74 l %r1,BASED(.Llockdep_sys_exit)
75 basr %r14,%r1 # call lockdep_sys_exit
79 .macro CHECK_STACK stacksize,savearea
80 #ifdef CONFIG_CHECK_STACK
81 tml %r15,\stacksize - CONFIG_STACK_GUARD
87 .macro SWITCH_ASYNC savearea,stack,shift
88 tmh %r8,0x0001 # interrupting from user ?
91 sl %r14,BASED(.Lcritical_start)
92 cl %r14,BASED(.Lcritical_length)
94 la %r11,\savearea # inside critical section, do cleanup
95 bras %r14,cleanup_critical
96 tmh %r8,0x0001 # retest problem state after cleanup
98 0: l %r14,\stack # are we already on the target stack?
102 CHECK_STACK 1<<\shift,\savearea
103 ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
105 1: l %r15,\stack # load target stack
106 2: la %r11,STACK_FRAME_OVERHEAD(%r15)
109 .macro ADD64 high,low,timer
116 .macro SUB64 high,low,timer
123 .macro UPDATE_VTIME high,low,enter_timer
124 lm \high,\low,__LC_EXIT_TIMER
125 SUB64 \high,\low,\enter_timer
126 ADD64 \high,\low,__LC_USER_TIMER
127 stm \high,\low,__LC_USER_TIMER
128 lm \high,\low,__LC_LAST_UPDATE_TIMER
129 SUB64 \high,\low,__LC_EXIT_TIMER
130 ADD64 \high,\low,__LC_SYSTEM_TIMER
131 stm \high,\low,__LC_SYSTEM_TIMER
132 mvc __LC_LAST_UPDATE_TIMER(8),\enter_timer
136 st %r8,__LC_RETURN_PSW
137 ni __LC_RETURN_PSW,0xbf
141 .section .kprobes.text, "ax"
144 * Scheduler resume function, called by switch_to
145 * gpr2 = (task_struct *) prev
146 * gpr3 = (task_struct *) next
151 stm %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task
152 st %r15,__THREAD_ksp(%r2) # store kernel stack of prev
153 l %r4,__THREAD_info(%r2) # get thread_info of prev
154 l %r5,__THREAD_info(%r3) # get thread_info of next
156 ahi %r15,STACK_INIT # end of kernel stack of next
157 st %r3,__LC_CURRENT # store task struct of next
158 st %r5,__LC_THREAD_INFO # store thread info of next
159 st %r15,__LC_KERNEL_STACK # store end of kernel stack
160 lctl %c4,%c4,__TASK_pid(%r3) # load pid to control reg. 4
161 mvc __LC_CURRENT_PID(4,%r0),__TASK_pid(%r3) # store pid of next
162 l %r15,__THREAD_ksp(%r3) # load kernel stack of next
163 lhi %r6,_TIF_TRANSFER # transfer TIF bits
164 n %r6,__TI_flags(%r4) # isolate TIF bits
166 o %r6,__TI_flags(%r5) # set TIF bits of next
167 st %r6,__TI_flags(%r5)
168 ni __TI_flags+3(%r4),255-_TIF_TRANSFER # clear TIF bits of prev
169 0: lm %r6,%r15,__SF_GPRS(%r15) # load gprs of next task
174 * SVC interrupt handler routine. System calls are synchronous events and
175 * are executed with interrupts enabled.
179 stpt __LC_SYNC_ENTER_TIMER
181 stm %r8,%r15,__LC_SAVE_AREA_SYNC
182 l %r12,__LC_THREAD_INFO
183 l %r13,__LC_SVC_NEW_PSW+4
185 l %r15,__LC_KERNEL_STACK
186 la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs
188 UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER
189 stm %r0,%r7,__PT_R0(%r11)
190 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_SYNC
191 mvc __PT_PSW(8,%r11),__LC_SVC_OLD_PSW
192 mvc __PT_INT_CODE(4,%r11),__LC_SVC_ILC
194 oi __TI_flags+3(%r12),_TIF_SYSCALL
195 l %r10,__TI_sysc_table(%r12) # 31 bit system call table
196 lh %r8,__PT_INT_CODE+2(%r11)
197 sla %r8,2 # shift and test for svc0
199 # svc 0: system call number in %r1
200 cl %r1,BASED(.Lnr_syscalls)
202 sth %r1,__PT_INT_CODE+2(%r11)
206 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
207 st %r2,__PT_ORIG_GPR2(%r11)
208 st %r7,STACK_FRAME_OVERHEAD(%r15)
209 l %r9,0(%r8,%r10) # get system call addr.
210 tm __TI_flags+2(%r12),_TIF_TRACE >> 8
212 basr %r14,%r9 # call sys_xxxx
213 st %r2,__PT_R2(%r11) # store return value
218 tm __PT_PSW+1(%r11),0x01 # returning to user ?
220 tm __TI_flags+3(%r12),_TIF_WORK_SVC
221 jnz sysc_work # check for work
222 ni __TI_flags+3(%r12),255-_TIF_SYSCALL
224 mvc __LC_RETURN_PSW(8),__PT_PSW(%r11)
226 lm %r0,%r15,__PT_R0(%r11)
231 # One of the work bits is on. Find out which one.
234 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING
236 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED
238 tm __TI_flags+3(%r12),_TIF_PER_TRAP
240 tm __TI_flags+3(%r12),_TIF_SIGPENDING
242 tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME
243 jo sysc_notify_resume
244 j sysc_return # beware of critical section cleanup
247 # _TIF_NEED_RESCHED is set, call schedule
250 l %r1,BASED(.Lschedule)
251 la %r14,BASED(sysc_return)
252 br %r1 # call schedule
255 # _TIF_MCCK_PENDING is set, call handler
258 l %r1,BASED(.Lhandle_mcck)
259 la %r14,BASED(sysc_return)
260 br %r1 # TIF bit will be cleared by handler
263 # _TIF_SIGPENDING is set, call do_signal
266 lr %r2,%r11 # pass pointer to pt_regs
267 l %r1,BASED(.Ldo_signal)
268 basr %r14,%r1 # call do_signal
269 tm __TI_flags+3(%r12),_TIF_SYSCALL
271 lm %r2,%r7,__PT_R2(%r11) # load svc arguments
272 l %r10,__TI_sysc_table(%r12) # 31 bit system call table
273 xr %r8,%r8 # svc 0 returns -ENOSYS
274 clc __PT_INT_CODE+2(2,%r11),BASED(.Lnr_syscalls+2)
275 jnl sysc_nr_ok # invalid svc number -> do svc 0
276 lh %r8,__PT_INT_CODE+2(%r11) # load new svc number
278 j sysc_nr_ok # restart svc
281 # _TIF_NOTIFY_RESUME is set, call do_notify_resume
284 lr %r2,%r11 # pass pointer to pt_regs
285 l %r1,BASED(.Ldo_notify_resume)
286 la %r14,BASED(sysc_return)
287 br %r1 # call do_notify_resume
290 # _TIF_PER_TRAP is set, call do_per_trap
293 ni __TI_flags+3(%r12),255-_TIF_PER_TRAP
294 lr %r2,%r11 # pass pointer to pt_regs
295 l %r1,BASED(.Ldo_per_trap)
296 la %r14,BASED(sysc_return)
297 br %r1 # call do_per_trap
300 # call tracehook_report_syscall_entry/tracehook_report_syscall_exit before
301 # and after the system call
304 l %r1,BASED(.Ltrace_enter)
305 lr %r2,%r11 # pass pointer to pt_regs
308 icm %r0,3,__PT_INT_CODE+2(%r11)
310 basr %r14,%r1 # call do_syscall_trace_enter
311 cl %r2,BASED(.Lnr_syscalls)
317 lm %r3,%r7,__PT_R3(%r11)
318 st %r7,STACK_FRAME_OVERHEAD(%r15)
319 l %r2,__PT_ORIG_GPR2(%r11)
320 basr %r14,%r9 # call sys_xxx
321 st %r2,__PT_R2(%r11) # store return value
323 tm __TI_flags+2(%r12),_TIF_TRACE >> 8
325 l %r1,BASED(.Ltrace_exit)
326 lr %r2,%r11 # pass pointer to pt_regs
327 la %r14,BASED(sysc_return)
328 br %r1 # call do_syscall_trace_exit
331 # a new process exits the kernel with ret_from_fork
334 la %r11,STACK_FRAME_OVERHEAD(%r15)
335 l %r12,__LC_THREAD_INFO
336 l %r13,__LC_SVC_NEW_PSW+4
337 l %r1,BASED(.Lschedule_tail)
338 basr %r14,%r1 # call schedule_tail
340 ssm __LC_SVC_NEW_PSW # reenable interrupts
341 tm __PT_PSW+1(%r11),0x01 # forking a kernel thread ?
343 # it's a kernel thread
344 lm %r9,%r10,__PT_R9(%r11) # load gprs
345 ENTRY(kernel_thread_starter)
351 * Program check handler routine
354 ENTRY(pgm_check_handler)
355 stpt __LC_SYNC_ENTER_TIMER
356 stm %r8,%r15,__LC_SAVE_AREA_SYNC
357 l %r12,__LC_THREAD_INFO
358 l %r13,__LC_SVC_NEW_PSW+4
359 lm %r8,%r9,__LC_PGM_OLD_PSW
360 tmh %r8,0x0001 # test problem state bit
361 jnz 1f # -> fault in user space
362 tmh %r8,0x4000 # PER bit set in old PSW ?
363 jnz 0f # -> enabled, can't be a double fault
364 tm __LC_PGM_ILC+3,0x80 # check for per exception
365 jnz pgm_svcper # -> single stepped svc
366 0: CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC
367 ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
369 1: UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER
370 l %r15,__LC_KERNEL_STACK
371 2: la %r11,STACK_FRAME_OVERHEAD(%r15)
372 stm %r0,%r7,__PT_R0(%r11)
373 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_SYNC
374 stm %r8,%r9,__PT_PSW(%r11)
375 mvc __PT_INT_CODE(4,%r11),__LC_PGM_ILC
376 mvc __PT_INT_PARM_LONG(4,%r11),__LC_TRANS_EXC_CODE
377 tm __LC_PGM_ILC+3,0x80 # check for per exception
379 l %r1,__TI_task(%r12)
380 tmh %r8,0x0001 # kernel per event ?
382 oi __TI_flags+3(%r12),_TIF_PER_TRAP
383 mvc __THREAD_per_address(4,%r1),__LC_PER_ADDRESS
384 mvc __THREAD_per_cause(2,%r1),__LC_PER_CAUSE
385 mvc __THREAD_per_paid(1,%r1),__LC_PER_PAID
387 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
388 l %r1,BASED(.Ljump_table)
390 n %r10,__PT_INT_CODE(%r11)
393 l %r1,0(%r10,%r1) # load address of handler routine
394 lr %r2,%r11 # pass pointer to pt_regs
395 basr %r14,%r1 # branch to interrupt-handler
399 # PER event in supervisor state, must be kprobes
403 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
404 l %r1,BASED(.Ldo_per_trap)
405 lr %r2,%r11 # pass pointer to pt_regs
406 basr %r14,%r1 # call do_per_trap
410 # single stepped system call
413 oi __TI_flags+3(%r12),_TIF_PER_TRAP
414 mvc __LC_RETURN_PSW(4),__LC_SVC_NEW_PSW
415 mvc __LC_RETURN_PSW+4(4),BASED(.Lsysc_per)
416 lpsw __LC_RETURN_PSW # branch to sysc_per and enable irqs
419 * IO interrupt handler routine
422 ENTRY(io_int_handler)
424 stpt __LC_ASYNC_ENTER_TIMER
425 stm %r8,%r15,__LC_SAVE_AREA_ASYNC
426 l %r12,__LC_THREAD_INFO
427 l %r13,__LC_SVC_NEW_PSW+4
428 lm %r8,%r9,__LC_IO_OLD_PSW
429 tmh %r8,0x0001 # interrupting from user ?
431 UPDATE_VTIME %r14,%r15,__LC_ASYNC_ENTER_TIMER
433 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT
434 stm %r0,%r7,__PT_R0(%r11)
435 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_ASYNC
436 stm %r8,%r9,__PT_PSW(%r11)
437 mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID
439 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
441 l %r1,BASED(.Ldo_IRQ)
442 lr %r2,%r11 # pass pointer to pt_regs
444 tm __PT_INT_CODE+8(%r11),0x80 # adapter interrupt ?
446 lhi %r3,THIN_INTERRUPT
448 basr %r14,%r1 # call do_IRQ
449 tm __LC_MACHINE_FLAGS+2,0x10 # MACHINE_FLAG_LPAR
453 mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID
459 tm __TI_flags+3(%r12),_TIF_WORK_INT
460 jnz io_work # there is work to do (signals etc.)
462 mvc __LC_RETURN_PSW(8),__PT_PSW(%r11)
464 lm %r0,%r15,__PT_R0(%r11)
469 # There is work todo, find out in which context we have been interrupted:
470 # 1) if we return to user space we can do all _TIF_WORK_INT work
471 # 2) if we return to kernel code and preemptive scheduling is enabled check
472 # the preemption counter and if it is zero call preempt_schedule_irq
473 # Before any work can be done, a switch to the kernel stack is required.
476 tm __PT_PSW+1(%r11),0x01 # returning to user ?
477 jo io_work_user # yes -> do resched & signal
478 #ifdef CONFIG_PREEMPT
479 # check for preemptive scheduling
480 icm %r0,15,__TI_precount(%r12)
481 jnz io_restore # preemption disabled
482 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED
484 # switch to kernel stack
486 ahi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
487 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
488 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)
489 la %r11,STACK_FRAME_OVERHEAD(%r1)
491 # TRACE_IRQS_ON already done at io_return, call
492 # TRACE_IRQS_OFF to keep things symmetrical
494 l %r1,BASED(.Lpreempt_irq)
495 basr %r14,%r1 # call preempt_schedule_irq
502 # Need to do work before returning to userspace, switch to kernel stack
505 l %r1,__LC_KERNEL_STACK
506 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
507 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)
508 la %r11,STACK_FRAME_OVERHEAD(%r1)
512 # One of the work bits is on. Find out which one.
513 # Checked are: _TIF_SIGPENDING, _TIF_NOTIFY_RESUME, _TIF_NEED_RESCHED
514 # and _TIF_MCCK_PENDING
517 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING
519 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED
521 tm __TI_flags+3(%r12),_TIF_SIGPENDING
523 tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME
525 j io_return # beware of critical section cleanup
528 # _TIF_MCCK_PENDING is set, call handler
531 # TRACE_IRQS_ON already done at io_return
532 l %r1,BASED(.Lhandle_mcck)
533 basr %r14,%r1 # TIF bit will be cleared by handler
538 # _TIF_NEED_RESCHED is set, call schedule
541 # TRACE_IRQS_ON already done at io_return
542 l %r1,BASED(.Lschedule)
543 ssm __LC_SVC_NEW_PSW # reenable interrupts
544 basr %r14,%r1 # call scheduler
545 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
550 # _TIF_SIGPENDING is set, call do_signal
553 # TRACE_IRQS_ON already done at io_return
554 l %r1,BASED(.Ldo_signal)
555 ssm __LC_SVC_NEW_PSW # reenable interrupts
556 lr %r2,%r11 # pass pointer to pt_regs
557 basr %r14,%r1 # call do_signal
558 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
563 # _TIF_SIGPENDING is set, call do_signal
566 # TRACE_IRQS_ON already done at io_return
567 l %r1,BASED(.Ldo_notify_resume)
568 ssm __LC_SVC_NEW_PSW # reenable interrupts
569 lr %r2,%r11 # pass pointer to pt_regs
570 basr %r14,%r1 # call do_notify_resume
571 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
576 * External interrupt handler routine
579 ENTRY(ext_int_handler)
581 stpt __LC_ASYNC_ENTER_TIMER
582 stm %r8,%r15,__LC_SAVE_AREA_ASYNC
583 l %r12,__LC_THREAD_INFO
584 l %r13,__LC_SVC_NEW_PSW+4
585 lm %r8,%r9,__LC_EXT_OLD_PSW
586 tmh %r8,0x0001 # interrupting from user ?
588 UPDATE_VTIME %r14,%r15,__LC_ASYNC_ENTER_TIMER
590 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT
591 stm %r0,%r7,__PT_R0(%r11)
592 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_ASYNC
593 stm %r8,%r9,__PT_PSW(%r11)
594 mvc __PT_INT_CODE(4,%r11),__LC_EXT_CPU_ADDR
595 mvc __PT_INT_PARM(4,%r11),__LC_EXT_PARAMS
597 l %r1,BASED(.Ldo_IRQ)
598 lr %r2,%r11 # pass pointer to pt_regs
599 lhi %r3,EXT_INTERRUPT
600 basr %r14,%r1 # call do_IRQ
604 * Load idle PSW. The second "half" of this function is in cleanup_idle.
607 st %r3,__SF_EMPTY(%r15)
609 la %r1,psw_idle_lpsw+4-.(%r1)
610 st %r1,__SF_EMPTY+4(%r15)
611 oi __SF_EMPTY+4(%r15),0x80
612 stck __CLOCK_IDLE_ENTER(%r2)
613 stpt __TIMER_IDLE_ENTER(%r2)
615 lpsw __SF_EMPTY(%r15)
622 * Machine check handler routines
625 ENTRY(mcck_int_handler)
627 spt __LC_CPU_TIMER_SAVE_AREA # revalidate cpu timer
628 lm %r0,%r15,__LC_GPREGS_SAVE_AREA # revalidate gprs
629 l %r12,__LC_THREAD_INFO
630 l %r13,__LC_SVC_NEW_PSW+4
631 lm %r8,%r9,__LC_MCK_OLD_PSW
632 tm __LC_MCCK_CODE,0x80 # system damage?
633 jo mcck_panic # yes -> rest of mcck code invalid
634 la %r14,__LC_CPU_TIMER_SAVE_AREA
635 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
636 tm __LC_MCCK_CODE+5,0x02 # stored cpu timer value valid?
638 la %r14,__LC_SYNC_ENTER_TIMER
639 clc 0(8,%r14),__LC_ASYNC_ENTER_TIMER
641 la %r14,__LC_ASYNC_ENTER_TIMER
642 0: clc 0(8,%r14),__LC_EXIT_TIMER
644 la %r14,__LC_EXIT_TIMER
645 1: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER
647 la %r14,__LC_LAST_UPDATE_TIMER
649 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
650 3: tm __LC_MCCK_CODE+2,0x09 # mwp + ia of old psw valid?
651 jno mcck_panic # no -> skip cleanup critical
652 tm %r8,0x0001 # interrupting from user ?
654 UPDATE_VTIME %r14,%r15,__LC_MCCK_ENTER_TIMER
656 SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+32,__LC_PANIC_STACK,PAGE_SHIFT
657 stm %r0,%r7,__PT_R0(%r11)
658 mvc __PT_R8(32,%r11),__LC_GPREGS_SAVE_AREA+32
659 stm %r8,%r9,__PT_PSW(%r11)
660 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
661 l %r1,BASED(.Ldo_machine_check)
662 lr %r2,%r11 # pass pointer to pt_regs
663 basr %r14,%r1 # call s390_do_machine_check
664 tm __PT_PSW+1(%r11),0x01 # returning to user ?
666 l %r1,__LC_KERNEL_STACK # switch to kernel stack
667 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
668 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)
669 la %r11,STACK_FRAME_OVERHEAD(%r15)
671 ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off
672 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING
675 l %r1,BASED(.Lhandle_mcck)
676 basr %r14,%r1 # call s390_handle_mcck
679 mvc __LC_RETURN_MCCK_PSW(8),__PT_PSW(%r11) # move return PSW
680 tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
682 lm %r0,%r15,__PT_R0(%r11)
684 lpsw __LC_RETURN_MCCK_PSW
685 0: lm %r0,%r15,__PT_R0(%r11)
686 lpsw __LC_RETURN_MCCK_PSW
689 l %r14,__LC_PANIC_STACK
693 l %r15,__LC_PANIC_STACK
695 0: ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
699 # PSW restart interrupt handler
701 ENTRY(restart_int_handler)
702 st %r15,__LC_SAVE_AREA_RESTART
703 l %r15,__LC_RESTART_STACK
704 ahi %r15,-__PT_SIZE # create pt_regs on stack
705 xc 0(__PT_SIZE,%r15),0(%r15)
706 stm %r0,%r14,__PT_R0(%r15)
707 mvc __PT_R15(4,%r15),__LC_SAVE_AREA_RESTART
708 mvc __PT_PSW(8,%r15),__LC_RST_OLD_PSW # store restart old psw
709 ahi %r15,-STACK_FRAME_OVERHEAD # create stack frame on stack
710 xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
711 l %r1,__LC_RESTART_FN # load fn, parm & source cpu
712 l %r2,__LC_RESTART_DATA
713 l %r3,__LC_RESTART_SOURCE
714 ltr %r3,%r3 # test source cpu address
715 jm 1f # negative -> skip source stop
716 0: sigp %r4,%r3,SIGP_SENSE # sigp sense to source cpu
717 brc 10,0b # wait for status stored
718 1: basr %r14,%r1 # call function
719 stap __SF_EMPTY(%r15) # store cpu address
720 lh %r3,__SF_EMPTY(%r15)
721 2: sigp %r4,%r3,SIGP_STOP # sigp stop to current cpu
725 .section .kprobes.text, "ax"
727 #ifdef CONFIG_CHECK_STACK
729 * The synchronous or the asynchronous stack overflowed. We are dead.
730 * No need to properly save the registers, we are going to panic anyway.
731 * Setup a pt_regs so that show_trace can provide a good call trace.
734 l %r15,__LC_PANIC_STACK # change to panic stack
735 la %r11,STACK_FRAME_OVERHEAD(%r15)
736 stm %r0,%r7,__PT_R0(%r11)
737 stm %r8,%r9,__PT_PSW(%r11)
738 mvc __PT_R8(32,%r11),0(%r14)
740 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
741 lr %r2,%r11 # pass pointer to pt_regs
742 br %r1 # branch to kernel_stack_overflow
743 1: .long kernel_stack_overflow
747 .long system_call + 0x80000000
748 .long sysc_do_svc + 0x80000000
749 .long sysc_tif + 0x80000000
750 .long sysc_restore + 0x80000000
751 .long sysc_done + 0x80000000
752 .long io_tif + 0x80000000
753 .long io_restore + 0x80000000
754 .long io_done + 0x80000000
755 .long psw_idle + 0x80000000
756 .long psw_idle_end + 0x80000000
759 cl %r9,BASED(cleanup_table) # system_call
761 cl %r9,BASED(cleanup_table+4) # sysc_do_svc
762 jl cleanup_system_call
763 cl %r9,BASED(cleanup_table+8) # sysc_tif
765 cl %r9,BASED(cleanup_table+12) # sysc_restore
767 cl %r9,BASED(cleanup_table+16) # sysc_done
768 jl cleanup_sysc_restore
769 cl %r9,BASED(cleanup_table+20) # io_tif
771 cl %r9,BASED(cleanup_table+24) # io_restore
773 cl %r9,BASED(cleanup_table+28) # io_done
774 jl cleanup_io_restore
775 cl %r9,BASED(cleanup_table+32) # psw_idle
777 cl %r9,BASED(cleanup_table+36) # psw_idle_end
782 # check if stpt has been executed
783 cl %r9,BASED(cleanup_system_call_insn)
785 mvc __LC_SYNC_ENTER_TIMER(8),__LC_ASYNC_ENTER_TIMER
786 chi %r11,__LC_SAVE_AREA_ASYNC
788 mvc __LC_SYNC_ENTER_TIMER(8),__LC_MCCK_ENTER_TIMER
789 0: # check if stm has been executed
790 cl %r9,BASED(cleanup_system_call_insn+4)
792 mvc __LC_SAVE_AREA_SYNC(32),0(%r11)
793 0: # set up saved registers r12, and r13
794 st %r12,16(%r11) # r12 thread-info pointer
795 st %r13,20(%r11) # r13 literal-pool pointer
796 # check if the user time calculation has been done
797 cl %r9,BASED(cleanup_system_call_insn+8)
799 l %r10,__LC_EXIT_TIMER
800 l %r15,__LC_EXIT_TIMER+4
801 SUB64 %r10,%r15,__LC_SYNC_ENTER_TIMER
802 ADD64 %r10,%r15,__LC_USER_TIMER
803 st %r10,__LC_USER_TIMER
804 st %r15,__LC_USER_TIMER+4
805 0: # check if the system time calculation has been done
806 cl %r9,BASED(cleanup_system_call_insn+12)
808 l %r10,__LC_LAST_UPDATE_TIMER
809 l %r15,__LC_LAST_UPDATE_TIMER+4
810 SUB64 %r10,%r15,__LC_EXIT_TIMER
811 ADD64 %r10,%r15,__LC_SYSTEM_TIMER
812 st %r10,__LC_SYSTEM_TIMER
813 st %r15,__LC_SYSTEM_TIMER+4
814 0: # update accounting time stamp
815 mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
816 # set up saved register 11
817 l %r15,__LC_KERNEL_STACK
818 la %r9,STACK_FRAME_OVERHEAD(%r15)
819 st %r9,12(%r11) # r11 pt_regs pointer
821 mvc __PT_R8(32,%r9),__LC_SAVE_AREA_SYNC
822 stm %r0,%r7,__PT_R0(%r9)
823 mvc __PT_PSW(8,%r9),__LC_SVC_OLD_PSW
824 mvc __PT_INT_CODE(4,%r9),__LC_SVC_ILC
825 # setup saved register 15
826 st %r15,28(%r11) # r15 stack pointer
827 # set new psw address and exit
828 l %r9,BASED(cleanup_table+4) # sysc_do_svc + 0x80000000
830 cleanup_system_call_insn:
831 .long system_call + 0x80000000
832 .long sysc_stm + 0x80000000
833 .long sysc_vtime + 0x80000000 + 36
834 .long sysc_vtime + 0x80000000 + 76
837 l %r9,BASED(cleanup_table+8) # sysc_tif + 0x80000000
840 cleanup_sysc_restore:
841 cl %r9,BASED(cleanup_sysc_restore_insn)
843 l %r9,12(%r11) # get saved pointer to pt_regs
844 mvc __LC_RETURN_PSW(8),__PT_PSW(%r9)
845 mvc 0(32,%r11),__PT_R8(%r9)
846 lm %r0,%r7,__PT_R0(%r9)
847 0: lm %r8,%r9,__LC_RETURN_PSW
849 cleanup_sysc_restore_insn:
850 .long sysc_done - 4 + 0x80000000
853 l %r9,BASED(cleanup_table+20) # io_tif + 0x80000000
857 cl %r9,BASED(cleanup_io_restore_insn)
859 l %r9,12(%r11) # get saved r11 pointer to pt_regs
860 mvc __LC_RETURN_PSW(8),__PT_PSW(%r9)
861 mvc 0(32,%r11),__PT_R8(%r9)
862 lm %r0,%r7,__PT_R0(%r9)
863 0: lm %r8,%r9,__LC_RETURN_PSW
865 cleanup_io_restore_insn:
866 .long io_done - 4 + 0x80000000
869 # copy interrupt clock & cpu timer
870 mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_INT_CLOCK
871 mvc __TIMER_IDLE_EXIT(8,%r2),__LC_ASYNC_ENTER_TIMER
872 chi %r11,__LC_SAVE_AREA_ASYNC
874 mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_MCCK_CLOCK
875 mvc __TIMER_IDLE_EXIT(8,%r2),__LC_MCCK_ENTER_TIMER
876 0: # check if stck has been executed
877 cl %r9,BASED(cleanup_idle_insn)
879 mvc __CLOCK_IDLE_ENTER(8,%r2),__CLOCK_IDLE_EXIT(%r2)
880 mvc __TIMER_IDLE_ENTER(8,%r2),__TIMER_IDLE_EXIT(%r3)
881 1: # account system time going idle
882 lm %r9,%r10,__LC_STEAL_TIMER
883 ADD64 %r9,%r10,__CLOCK_IDLE_ENTER(%r2)
884 SUB64 %r9,%r10,__LC_LAST_UPDATE_CLOCK
885 stm %r9,%r10,__LC_STEAL_TIMER
886 mvc __LC_LAST_UPDATE_CLOCK(8),__CLOCK_IDLE_EXIT(%r2)
887 lm %r9,%r10,__LC_SYSTEM_TIMER
888 ADD64 %r9,%r10,__LC_LAST_UPDATE_TIMER
889 SUB64 %r9,%r10,__TIMER_IDLE_ENTER(%r2)
890 stm %r9,%r10,__LC_SYSTEM_TIMER
891 mvc __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2)
893 n %r8,BASED(cleanup_idle_wait) # clear irq & wait state bits
894 l %r9,24(%r11) # return from psw_idle
897 .long psw_idle_lpsw + 0x80000000
908 .quad 0x7fffffffffffffff
913 .Ldo_machine_check: .long s390_do_machine_check
914 .Lhandle_mcck: .long s390_handle_mcck
915 .Ldo_IRQ: .long do_IRQ
916 .Ldo_signal: .long do_signal
917 .Ldo_notify_resume: .long do_notify_resume
918 .Ldo_per_trap: .long do_per_trap
919 .Ljump_table: .long pgm_check_table
920 .Lschedule: .long schedule
921 #ifdef CONFIG_PREEMPT
922 .Lpreempt_irq: .long preempt_schedule_irq
924 .Ltrace_enter: .long do_syscall_trace_enter
925 .Ltrace_exit: .long do_syscall_trace_exit
926 .Lschedule_tail: .long schedule_tail
927 .Lsysc_per: .long sysc_per + 0x80000000
928 #ifdef CONFIG_TRACE_IRQFLAGS
929 .Lhardirqs_on: .long trace_hardirqs_on_caller
930 .Lhardirqs_off: .long trace_hardirqs_off_caller
932 #ifdef CONFIG_LOCKDEP
933 .Llockdep_sys_exit: .long lockdep_sys_exit
935 .Lcritical_start: .long __critical_start + 0x80000000
936 .Lcritical_length: .long __critical_end - __critical_start
938 .section .rodata, "a"
939 #define SYSCALL(esa,esame,emu) .long esa
940 .globl sys_call_table
942 #include "syscalls.S"