arch/x86/kernel/fpu/core.c

   1 /*
   2  *  Copyright (C) 1994 Linus Torvalds
   3  *
   4  *  Pentium III FXSR, SSE support
   5  *  General FPU state handling cleanups
   6  *      Gareth Hughes <gareth@valinux.com>, May 2000
   7  */
   8 #include <asm/fpu-internal.h>
   9
  10 static DEFINE_PER_CPU(bool, in_kernel_fpu);
  11
  12 static void kernel_fpu_disable(void)
  13 {
  14         WARN_ON(this_cpu_read(in_kernel_fpu));
  15         this_cpu_write(in_kernel_fpu, true);
  16 }
  17
  18 static void kernel_fpu_enable(void)
  19 {
  20         this_cpu_write(in_kernel_fpu, false);
  21 }
  22
  23 /*
  24  * Were we in an interrupt that interrupted kernel mode?
  25  *
  26  * On others, we can do a kernel_fpu_begin/end() pair *ONLY* if that
  27  * pair does nothing at all: the thread must not have fpu (so
  28  * that we don't try to save the FPU state), and TS must
  29  * be set (so that the clts/stts pair does nothing that is
  30  * visible in the interrupted kernel thread).
  31  *
  32  * Except for the eagerfpu case when we return true; in the likely case
  33  * the thread has FPU but we are not going to set/clear TS.
  34  */
  35 static bool interrupted_kernel_fpu_idle(void)
  36 {
  37         if (this_cpu_read(in_kernel_fpu))
  38                 return false;
  39
  40         if (use_eager_fpu())
  41                 return true;
  42
  43         return !__thread_has_fpu(current) &&
  44                 (read_cr0() & X86_CR0_TS);
  45 }
  46
  47 /*
  48  * Were we in user mode (or vm86 mode) when we were
  49  * interrupted?
  50  *
  51  * Doing kernel_fpu_begin/end() is ok if we are running
  52  * in an interrupt context from user mode - we'll just
  53  * save the FPU state as required.
  54  */
  55 static bool interrupted_user_mode(void)
  56 {
  57         struct pt_regs *regs = get_irq_regs();
  58         return regs && user_mode(regs);
  59 }
  60
  61 /*
  62  * Can we use the FPU in kernel mode with the
  63  * whole "kernel_fpu_begin/end()" sequence?
  64  *
  65  * It's always ok in process context (ie "not interrupt")
  66  * but it is sometimes ok even from an irq.
  67  */
  68 bool irq_fpu_usable(void)
  69 {
  70         return !in_interrupt() ||
  71                 interrupted_user_mode() ||
  72                 interrupted_kernel_fpu_idle();
  73 }
  74 EXPORT_SYMBOL(irq_fpu_usable);
  75
  76 void __kernel_fpu_begin(void)
  77 {
  78         struct task_struct *me = current;
  79
  80         this_cpu_write(in_kernel_fpu, true);
  81
  82         if (__thread_has_fpu(me)) {
  83                 __save_init_fpu(me);
  84         } else {
  85                 this_cpu_write(fpu_owner_task, NULL);
  86                 if (!use_eager_fpu())
  87                         clts();
  88         }
  89 }
  90 EXPORT_SYMBOL(__kernel_fpu_begin);
  91
  92 void __kernel_fpu_end(void)
  93 {
  94         struct task_struct *me = current;
  95
  96         if (__thread_has_fpu(me)) {
  97                 if (WARN_ON(restore_fpu_checking(me)))
  98                         fpu_reset_state(me);
  99         } else if (!use_eager_fpu()) {
 100                 stts();
 101         }
 102
 103         this_cpu_write(in_kernel_fpu, false);
 104 }
 105 EXPORT_SYMBOL(__kernel_fpu_end);
 106
 107 /*
 108  * Save the FPU state (initialize it if necessary):
 109  *
 110  * This only ever gets called for the current task.
 111  */
 112 void fpu__save(struct task_struct *tsk)
 113 {
 114         WARN_ON(tsk != current);
 115
 116         preempt_disable();
 117         if (__thread_has_fpu(tsk)) {
 118                 if (use_eager_fpu()) {
 119                         __save_fpu(tsk);
 120                 } else {
 121                         __save_init_fpu(tsk);
 122                         __thread_fpu_end(tsk);
 123                 }
 124         }
 125         preempt_enable();
 126 }
 127 EXPORT_SYMBOL_GPL(fpu__save);
 128
 129 void fpstate_init(struct fpu *fpu)
 130 {
 131         if (!cpu_has_fpu) {
 132                 finit_soft_fpu(&fpu->state->soft);
 133                 return;
 134         }
 135
 136         memset(fpu->state, 0, xstate_size);
 137
 138         if (cpu_has_fxsr) {
 139                 fx_finit(&fpu->state->fxsave);
 140         } else {
 141                 struct i387_fsave_struct *fp = &fpu->state->fsave;
 142                 fp->cwd = 0xffff037fu;
 143                 fp->swd = 0xffff0000u;
 144                 fp->twd = 0xffffffffu;
 145                 fp->fos = 0xffff0000u;
 146         }
 147 }
 148 EXPORT_SYMBOL_GPL(fpstate_init);
 149
 150 /*
 151  * FPU state allocation:
 152  */
 153 static struct kmem_cache *task_xstate_cachep;
 154
 155 void fpstate_cache_init(void)
 156 {
 157         task_xstate_cachep =
 158                 kmem_cache_create("task_xstate", xstate_size,
 159                                   __alignof__(union thread_xstate),
 160                                   SLAB_PANIC | SLAB_NOTRACK, NULL);
 161         setup_xstate_comp();
 162 }
 163
 164 int fpstate_alloc(struct fpu *fpu)
 165 {
 166         if (fpu->state)
 167                 return 0;
 168
 169         fpu->state = kmem_cache_alloc(task_xstate_cachep, GFP_KERNEL);
 170         if (!fpu->state)
 171                 return -ENOMEM;
 172
 173         /* The CPU requires the FPU state to be aligned to 16 byte boundaries: */
 174         WARN_ON((unsigned long)fpu->state & 15);
 175
 176         return 0;
 177 }
 178 EXPORT_SYMBOL_GPL(fpstate_alloc);
 179
 180 void fpstate_free(struct fpu *fpu)
 181 {
 182         if (fpu->state) {
 183                 kmem_cache_free(task_xstate_cachep, fpu->state);
 184                 fpu->state = NULL;
 185         }
 186 }
 187 EXPORT_SYMBOL_GPL(fpstate_free);
 188
 189 int fpu__copy(struct task_struct *dst, struct task_struct *src)
 190 {
 191         dst->thread.fpu.counter = 0;
 192         dst->thread.fpu.has_fpu = 0;
 193         dst->thread.fpu.state = NULL;
 194
 195         task_disable_lazy_fpu_restore(dst);
 196
 197         if (tsk_used_math(src)) {
 198                 int err = fpstate_alloc(&dst->thread.fpu);
 199
 200                 if (err)
 201                         return err;
 202                 fpu_copy(dst, src);
 203         }
 204         return 0;
 205 }
 206
 207 /*
 208  * Allocate the backing store for the current task's FPU registers
 209  * and initialize the registers themselves as well.
 210  *
 211  * Can fail.
 212  */
 213 int fpstate_alloc_init(struct task_struct *curr)
 214 {
 215         int ret;
 216
 217         if (WARN_ON_ONCE(curr != current))
 218                 return -EINVAL;
 219         if (WARN_ON_ONCE(curr->flags & PF_USED_MATH))
 220                 return -EINVAL;
 221
 222         /*
 223          * Memory allocation at the first usage of the FPU and other state.
 224          */
 225         ret = fpstate_alloc(&curr->thread.fpu);
 226         if (ret)
 227                 return ret;
 228
 229         fpstate_init(&curr->thread.fpu);
 230
 231         /* Safe to do for the current task: */
 232         curr->flags |= PF_USED_MATH;
 233
 234         return 0;
 235 }
 236 EXPORT_SYMBOL_GPL(fpstate_alloc_init);
 237
 238 /*
 239  * The _current_ task is using the FPU for the first time
 240  * so initialize it and set the mxcsr to its default
 241  * value at reset if we support XMM instructions and then
 242  * remember the current task has used the FPU.
 243  */
 244 static int fpu__unlazy_stopped(struct task_struct *child)
 245 {
 246         int ret;
 247
 248         if (WARN_ON_ONCE(child == current))
 249                 return -EINVAL;
 250
 251         if (child->flags & PF_USED_MATH) {
 252                 task_disable_lazy_fpu_restore(child);
 253                 return 0;
 254         }
 255
 256         /*
 257          * Memory allocation at the first usage of the FPU and other state.
 258          */
 259         ret = fpstate_alloc(&child->thread.fpu);
 260         if (ret)
 261                 return ret;
 262
 263         fpstate_init(&child->thread.fpu);
 264
 265         /* Safe to do for stopped child tasks: */
 266         child->flags |= PF_USED_MATH;
 267
 268         return 0;
 269 }
 270
 271 /*
 272  * 'fpu__restore()' saves the current math information in the
 273  * old math state array, and gets the new ones from the current task
 274  *
 275  * Careful.. There are problems with IBM-designed IRQ13 behaviour.
 276  * Don't touch unless you *really* know how it works.
 277  *
 278  * Must be called with kernel preemption disabled (eg with local
 279  * local interrupts as in the case of do_device_not_available).
 280  */
 281 void fpu__restore(void)
 282 {
 283         struct task_struct *tsk = current;
 284
 285         if (!tsk_used_math(tsk)) {
 286                 local_irq_enable();
 287                 /*
 288                  * does a slab alloc which can sleep
 289                  */
 290                 if (fpstate_alloc_init(tsk)) {
 291                         /*
 292                          * ran out of memory!
 293                          */
 294                         do_group_exit(SIGKILL);
 295                         return;
 296                 }
 297                 local_irq_disable();
 298         }
 299
 300         /* Avoid __kernel_fpu_begin() right after __thread_fpu_begin() */
 301         kernel_fpu_disable();
 302         __thread_fpu_begin(tsk);
 303         if (unlikely(restore_fpu_checking(tsk))) {
 304                 fpu_reset_state(tsk);
 305                 force_sig_info(SIGSEGV, SEND_SIG_PRIV, tsk);
 306         } else {
 307                 tsk->thread.fpu.counter++;
 308         }
 309         kernel_fpu_enable();
 310 }
 311 EXPORT_SYMBOL_GPL(fpu__restore);
 312
 313 void fpu__flush_thread(struct task_struct *tsk)
 314 {
 315         if (!use_eager_fpu()) {
 316                 /* FPU state will be reallocated lazily at the first use. */
 317                 drop_fpu(tsk);
 318                 fpstate_free(&tsk->thread.fpu);
 319         } else {
 320                 if (!tsk_used_math(tsk)) {
 321                         /* kthread execs. TODO: cleanup this horror. */
 322                 if (WARN_ON(fpstate_alloc_init(tsk)))
 323                                 force_sig(SIGKILL, tsk);
 324                         user_fpu_begin();
 325                 }
 326                 restore_init_xstate();
 327         }
 328 }
 329
 330 /*
 331  * The xstateregs_active() routine is the same as the fpregs_active() routine,
 332  * as the "regset->n" for the xstate regset will be updated based on the feature
 333  * capabilites supported by the xsave.
 334  */
 335 int fpregs_active(struct task_struct *target, const struct user_regset *regset)
 336 {
 337         return tsk_used_math(target) ? regset->n : 0;
 338 }
 339
 340 int xfpregs_active(struct task_struct *target, const struct user_regset *regset)
 341 {
 342         return (cpu_has_fxsr && tsk_used_math(target)) ? regset->n : 0;
 343 }
 344
 345 int xfpregs_get(struct task_struct *target, const struct user_regset *regset,
 346                 unsigned int pos, unsigned int count,
 347                 void *kbuf, void __user *ubuf)
 348 {
 349         int ret;
 350
 351         if (!cpu_has_fxsr)
 352                 return -ENODEV;
 353
 354         ret = fpu__unlazy_stopped(target);
 355         if (ret)
 356                 return ret;
 357
 358         sanitize_i387_state(target);
 359
 360         return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
 361                                    &target->thread.fpu.state->fxsave, 0, -1);
 362 }
 363
 364 int xfpregs_set(struct task_struct *target, const struct user_regset *regset,
 365                 unsigned int pos, unsigned int count,
 366                 const void *kbuf, const void __user *ubuf)
 367 {
 368         int ret;
 369
 370         if (!cpu_has_fxsr)
 371                 return -ENODEV;
 372
 373         ret = fpu__unlazy_stopped(target);
 374         if (ret)
 375                 return ret;
 376
 377         sanitize_i387_state(target);
 378
 379         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
 380                                  &target->thread.fpu.state->fxsave, 0, -1);
 381
 382         /*
 383          * mxcsr reserved bits must be masked to zero for security reasons.
 384          */
 385         target->thread.fpu.state->fxsave.mxcsr &= mxcsr_feature_mask;
 386
 387         /*
 388          * update the header bits in the xsave header, indicating the
 389          * presence of FP and SSE state.
 390          */
 391         if (cpu_has_xsave)
 392                 target->thread.fpu.state->xsave.xsave_hdr.xstate_bv |= XSTATE_FPSSE;
 393
 394         return ret;
 395 }
 396
 397 int xstateregs_get(struct task_struct *target, const struct user_regset *regset,
 398                 unsigned int pos, unsigned int count,
 399                 void *kbuf, void __user *ubuf)
 400 {
 401         struct xsave_struct *xsave;
 402         int ret;
 403
 404         if (!cpu_has_xsave)
 405                 return -ENODEV;
 406
 407         ret = fpu__unlazy_stopped(target);
 408         if (ret)
 409                 return ret;
 410
 411         xsave = &target->thread.fpu.state->xsave;
 412
 413         /*
 414          * Copy the 48bytes defined by the software first into the xstate
 415          * memory layout in the thread struct, so that we can copy the entire
 416          * xstateregs to the user using one user_regset_copyout().
 417          */
 418         memcpy(&xsave->i387.sw_reserved,
 419                 xstate_fx_sw_bytes, sizeof(xstate_fx_sw_bytes));
 420         /*
 421          * Copy the xstate memory layout.
 422          */
 423         ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, xsave, 0, -1);
 424         return ret;
 425 }
 426
 427 int xstateregs_set(struct task_struct *target, const struct user_regset *regset,
 428                   unsigned int pos, unsigned int count,
 429                   const void *kbuf, const void __user *ubuf)
 430 {
 431         struct xsave_struct *xsave;
 432         int ret;
 433
 434         if (!cpu_has_xsave)
 435                 return -ENODEV;
 436
 437         ret = fpu__unlazy_stopped(target);
 438         if (ret)
 439                 return ret;
 440
 441         xsave = &target->thread.fpu.state->xsave;
 442
 443         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, xsave, 0, -1);
 444         /*
 445          * mxcsr reserved bits must be masked to zero for security reasons.
 446          */
 447         xsave->i387.mxcsr &= mxcsr_feature_mask;
 448         xsave->xsave_hdr.xstate_bv &= pcntxt_mask;
 449         /*
 450          * These bits must be zero.
 451          */
 452         memset(&xsave->xsave_hdr.reserved, 0, 48);
 453         return ret;
 454 }
 455
 456 #if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION
 457
 458 /*
 459  * FPU tag word conversions.
 460  */
 461
 462 static inline unsigned short twd_i387_to_fxsr(unsigned short twd)
 463 {
 464         unsigned int tmp; /* to avoid 16 bit prefixes in the code */
 465
 466         /* Transform each pair of bits into 01 (valid) or 00 (empty) */
 467         tmp = ~twd;
 468         tmp = (tmp | (tmp>>1)) & 0x5555; /* 0V0V0V0V0V0V0V0V */
 469         /* and move the valid bits to the lower byte. */
 470         tmp = (tmp | (tmp >> 1)) & 0x3333; /* 00VV00VV00VV00VV */
 471         tmp = (tmp | (tmp >> 2)) & 0x0f0f; /* 0000VVVV0000VVVV */
 472         tmp = (tmp | (tmp >> 4)) & 0x00ff; /* 00000000VVVVVVVV */
 473
 474         return tmp;
 475 }
 476
 477 #define FPREG_ADDR(f, n)        ((void *)&(f)->st_space + (n) * 16)
 478 #define FP_EXP_TAG_VALID        0
 479 #define FP_EXP_TAG_ZERO         1
 480 #define FP_EXP_TAG_SPECIAL      2
 481 #define FP_EXP_TAG_EMPTY        3
 482
 483 static inline u32 twd_fxsr_to_i387(struct i387_fxsave_struct *fxsave)
 484 {
 485         struct _fpxreg *st;
 486         u32 tos = (fxsave->swd >> 11) & 7;
 487         u32 twd = (unsigned long) fxsave->twd;
 488         u32 tag;
 489         u32 ret = 0xffff0000u;
 490         int i;
 491
 492         for (i = 0; i < 8; i++, twd >>= 1) {
 493                 if (twd & 0x1) {
 494                         st = FPREG_ADDR(fxsave, (i - tos) & 7);
 495
 496                         switch (st->exponent & 0x7fff) {
 497                         case 0x7fff:
 498                                 tag = FP_EXP_TAG_SPECIAL;
 499                                 break;
 500                         case 0x0000:
 501                                 if (!st->significand[0] &&
 502                                     !st->significand[1] &&
 503                                     !st->significand[2] &&
 504                                     !st->significand[3])
 505                                         tag = FP_EXP_TAG_ZERO;
 506                                 else
 507                                         tag = FP_EXP_TAG_SPECIAL;
 508                                 break;
 509                         default:
 510                                 if (st->significand[3] & 0x8000)
 511                                         tag = FP_EXP_TAG_VALID;
 512                                 else
 513                                         tag = FP_EXP_TAG_SPECIAL;
 514                                 break;
 515                         }
 516                 } else {
 517                         tag = FP_EXP_TAG_EMPTY;
 518                 }
 519                 ret |= tag << (2 * i);
 520         }
 521         return ret;
 522 }
 523
 524 /*
 525  * FXSR floating point environment conversions.
 526  */
 527
 528 void
 529 convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk)
 530 {
 531         struct i387_fxsave_struct *fxsave = &tsk->thread.fpu.state->fxsave;
 532         struct _fpreg *to = (struct _fpreg *) &env->st_space[0];
 533         struct _fpxreg *from = (struct _fpxreg *) &fxsave->st_space[0];
 534         int i;
 535
 536         env->cwd = fxsave->cwd | 0xffff0000u;
 537         env->swd = fxsave->swd | 0xffff0000u;
 538         env->twd = twd_fxsr_to_i387(fxsave);
 539
 540 #ifdef CONFIG_X86_64
 541         env->fip = fxsave->rip;
 542         env->foo = fxsave->rdp;
 543         /*
 544          * should be actually ds/cs at fpu exception time, but
 545          * that information is not available in 64bit mode.
 546          */
 547         env->fcs = task_pt_regs(tsk)->cs;
 548         if (tsk == current) {
 549                 savesegment(ds, env->fos);
 550         } else {
 551                 env->fos = tsk->thread.ds;
 552         }
 553         env->fos |= 0xffff0000;
 554 #else
 555         env->fip = fxsave->fip;
 556         env->fcs = (u16) fxsave->fcs | ((u32) fxsave->fop << 16);
 557         env->foo = fxsave->foo;
 558         env->fos = fxsave->fos;
 559 #endif
 560
 561         for (i = 0; i < 8; ++i)
 562                 memcpy(&to[i], &from[i], sizeof(to[0]));
 563 }
 564
 565 void convert_to_fxsr(struct task_struct *tsk,
 566                      const struct user_i387_ia32_struct *env)
 567
 568 {
 569         struct i387_fxsave_struct *fxsave = &tsk->thread.fpu.state->fxsave;
 570         struct _fpreg *from = (struct _fpreg *) &env->st_space[0];
 571         struct _fpxreg *to = (struct _fpxreg *) &fxsave->st_space[0];
 572         int i;
 573
 574         fxsave->cwd = env->cwd;
 575         fxsave->swd = env->swd;
 576         fxsave->twd = twd_i387_to_fxsr(env->twd);
 577         fxsave->fop = (u16) ((u32) env->fcs >> 16);
 578 #ifdef CONFIG_X86_64
 579         fxsave->rip = env->fip;
 580         fxsave->rdp = env->foo;
 581         /* cs and ds ignored */
 582 #else
 583         fxsave->fip = env->fip;
 584         fxsave->fcs = (env->fcs & 0xffff);
 585         fxsave->foo = env->foo;
 586         fxsave->fos = env->fos;
 587 #endif
 588
 589         for (i = 0; i < 8; ++i)
 590                 memcpy(&to[i], &from[i], sizeof(from[0]));
 591 }
 592
 593 int fpregs_get(struct task_struct *target, const struct user_regset *regset,
 594                unsigned int pos, unsigned int count,
 595                void *kbuf, void __user *ubuf)
 596 {
 597         struct user_i387_ia32_struct env;
 598         int ret;
 599
 600         ret = fpu__unlazy_stopped(target);
 601         if (ret)
 602                 return ret;
 603
 604         if (!static_cpu_has(X86_FEATURE_FPU))
 605                 return fpregs_soft_get(target, regset, pos, count, kbuf, ubuf);
 606
 607         if (!cpu_has_fxsr)
 608                 return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
 609                                            &target->thread.fpu.state->fsave, 0,
 610                                            -1);
 611
 612         sanitize_i387_state(target);
 613
 614         if (kbuf && pos == 0 && count == sizeof(env)) {
 615                 convert_from_fxsr(kbuf, target);
 616                 return 0;
 617         }
 618
 619         convert_from_fxsr(&env, target);
 620
 621         return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &env, 0, -1);
 622 }
 623
 624 int fpregs_set(struct task_struct *target, const struct user_regset *regset,
 625                unsigned int pos, unsigned int count,
 626                const void *kbuf, const void __user *ubuf)
 627 {
 628         struct user_i387_ia32_struct env;
 629         int ret;
 630
 631         ret = fpu__unlazy_stopped(target);
 632         if (ret)
 633                 return ret;
 634
 635         sanitize_i387_state(target);
 636
 637         if (!static_cpu_has(X86_FEATURE_FPU))
 638                 return fpregs_soft_set(target, regset, pos, count, kbuf, ubuf);
 639
 640         if (!cpu_has_fxsr)
 641                 return user_regset_copyin(&pos, &count, &kbuf, &ubuf,
 642                                           &target->thread.fpu.state->fsave, 0,
 643                                           -1);
 644
 645         if (pos > 0 || count < sizeof(env))
 646                 convert_from_fxsr(&env, target);
 647
 648         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &env, 0, -1);
 649         if (!ret)
 650                 convert_to_fxsr(target, &env);
 651
 652         /*
 653          * update the header bit in the xsave header, indicating the
 654          * presence of FP.
 655          */
 656         if (cpu_has_xsave)
 657                 target->thread.fpu.state->xsave.xsave_hdr.xstate_bv |= XSTATE_FP;
 658         return ret;
 659 }
 660
 661 /*
 662  * FPU state for core dumps.
 663  * This is only used for a.out dumps now.
 664  * It is declared generically using elf_fpregset_t (which is
 665  * struct user_i387_struct) but is in fact only used for 32-bit
 666  * dumps, so on 64-bit it is really struct user_i387_ia32_struct.
 667  */
 668 int dump_fpu(struct pt_regs *regs, struct user_i387_struct *fpu)
 669 {
 670         struct task_struct *tsk = current;
 671         int fpvalid;
 672
 673         fpvalid = !!used_math();
 674         if (fpvalid)
 675                 fpvalid = !fpregs_get(tsk, NULL,
 676                                       0, sizeof(struct user_i387_ia32_struct),
 677                                       fpu, NULL);
 678
 679         return fpvalid;
 680 }
 681 EXPORT_SYMBOL(dump_fpu);
 682
 683 #endif  /* CONFIG_X86_32 || CONFIG_IA32_EMULATION */