arch/x86/include/asm/paravirt.h

   1 #ifndef _ASM_X86_PARAVIRT_H
   2 #define _ASM_X86_PARAVIRT_H
   3 /* Various instructions on x86 need to be replaced for
   4  * para-virtualization: those hooks are defined here. */
   5
   6 #ifdef CONFIG_PARAVIRT
   7 #include <asm/pgtable_types.h>
   8 #include <asm/asm.h>
   9
  10 #include <asm/paravirt_types.h>
  11
  12 #ifndef __ASSEMBLY__
  13 #include <linux/types.h>
  14 #include <linux/cpumask.h>
  15
  16 static inline int paravirt_enabled(void)
  17 {
  18         return pv_info.paravirt_enabled;
  19 }
  20
  21 static inline void load_sp0(struct tss_struct *tss,
  22                              struct thread_struct *thread)
  23 {
  24         PVOP_VCALL2(pv_cpu_ops.load_sp0, tss, thread);
  25 }
  26
  27 static inline unsigned long get_wallclock(void)
  28 {
  29         return PVOP_CALL0(unsigned long, pv_time_ops.get_wallclock);
  30 }
  31
  32 static inline int set_wallclock(unsigned long nowtime)
  33 {
  34         return PVOP_CALL1(int, pv_time_ops.set_wallclock, nowtime);
  35 }
  36
  37 static inline void (*choose_time_init(void))(void)
  38 {
  39         return pv_time_ops.time_init;
  40 }
  41
  42 /* The paravirtualized CPUID instruction. */
  43 static inline void __cpuid(unsigned int *eax, unsigned int *ebx,
  44                            unsigned int *ecx, unsigned int *edx)
  45 {
  46         PVOP_VCALL4(pv_cpu_ops.cpuid, eax, ebx, ecx, edx);
  47 }
  48
  49 /*
  50  * These special macros can be used to get or set a debugging register
  51  */
  52 static inline unsigned long paravirt_get_debugreg(int reg)
  53 {
  54         return PVOP_CALL1(unsigned long, pv_cpu_ops.get_debugreg, reg);
  55 }
  56 #define get_debugreg(var, reg) var = paravirt_get_debugreg(reg)
  57 static inline void set_debugreg(unsigned long val, int reg)
  58 {
  59         PVOP_VCALL2(pv_cpu_ops.set_debugreg, reg, val);
  60 }
  61
  62 static inline void clts(void)
  63 {
  64         PVOP_VCALL0(pv_cpu_ops.clts);
  65 }
  66
  67 static inline unsigned long read_cr0(void)
  68 {
  69         return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr0);
  70 }
  71
  72 static inline void write_cr0(unsigned long x)
  73 {
  74         PVOP_VCALL1(pv_cpu_ops.write_cr0, x);
  75 }
  76
  77 static inline unsigned long read_cr2(void)
  78 {
  79         return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr2);
  80 }
  81
  82 static inline void write_cr2(unsigned long x)
  83 {
  84         PVOP_VCALL1(pv_mmu_ops.write_cr2, x);
  85 }
  86
  87 static inline unsigned long read_cr3(void)
  88 {
  89         return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr3);
  90 }
  91
  92 static inline void write_cr3(unsigned long x)
  93 {
  94         PVOP_VCALL1(pv_mmu_ops.write_cr3, x);
  95 }
  96
  97 static inline unsigned long read_cr4(void)
  98 {
  99         return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4);
 100 }
 101 static inline unsigned long read_cr4_safe(void)
 102 {
 103         return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4_safe);
 104 }
 105
 106 static inline void write_cr4(unsigned long x)
 107 {
 108         PVOP_VCALL1(pv_cpu_ops.write_cr4, x);
 109 }
 110
 111 #ifdef CONFIG_X86_64
 112 static inline unsigned long read_cr8(void)
 113 {
 114         return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr8);
 115 }
 116
 117 static inline void write_cr8(unsigned long x)
 118 {
 119         PVOP_VCALL1(pv_cpu_ops.write_cr8, x);
 120 }
 121 #endif
 122
 123 static inline void raw_safe_halt(void)
 124 {
 125         PVOP_VCALL0(pv_irq_ops.safe_halt);
 126 }
 127
 128 static inline void halt(void)
 129 {
 130         PVOP_VCALL0(pv_irq_ops.safe_halt);
 131 }
 132
 133 static inline void wbinvd(void)
 134 {
 135         PVOP_VCALL0(pv_cpu_ops.wbinvd);
 136 }
 137
 138 #define get_kernel_rpl()  (pv_info.kernel_rpl)
 139
 140 static inline u64 paravirt_read_msr(unsigned msr, int *err)
 141 {
 142         return PVOP_CALL2(u64, pv_cpu_ops.read_msr, msr, err);
 143 }
 144 static inline u64 paravirt_read_msr_amd(unsigned msr, int *err)
 145 {
 146         return PVOP_CALL2(u64, pv_cpu_ops.read_msr_amd, msr, err);
 147 }
 148 static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high)
 149 {
 150         return PVOP_CALL3(int, pv_cpu_ops.write_msr, msr, low, high);
 151 }
 152
 153 /* These should all do BUG_ON(_err), but our headers are too tangled. */
 154 #define rdmsr(msr, val1, val2)                  \
 155 do {                                            \
 156         int _err;                               \
 157         u64 _l = paravirt_read_msr(msr, &_err); \
 158         val1 = (u32)_l;                         \
 159         val2 = _l >> 32;                        \
 160 } while (0)
 161
 162 #define wrmsr(msr, val1, val2)                  \
 163 do {                                            \
 164         paravirt_write_msr(msr, val1, val2);    \
 165 } while (0)
 166
 167 #define rdmsrl(msr, val)                        \
 168 do {                                            \
 169         int _err;                               \
 170         val = paravirt_read_msr(msr, &_err);    \
 171 } while (0)
 172
 173 #define wrmsrl(msr, val)        wrmsr(msr, (u32)((u64)(val)), ((u64)(val))>>32)
 174 #define wrmsr_safe(msr, a, b)   paravirt_write_msr(msr, a, b)
 175
 176 /* rdmsr with exception handling */
 177 #define rdmsr_safe(msr, a, b)                   \
 178 ({                                              \
 179         int _err;                               \
 180         u64 _l = paravirt_read_msr(msr, &_err); \
 181         (*a) = (u32)_l;                         \
 182         (*b) = _l >> 32;                        \
 183         _err;                                   \
 184 })
 185
 186 static inline int rdmsrl_safe(unsigned msr, unsigned long long *p)
 187 {
 188         int err;
 189
 190         *p = paravirt_read_msr(msr, &err);
 191         return err;
 192 }
 193 static inline int rdmsrl_amd_safe(unsigned msr, unsigned long long *p)
 194 {
 195         int err;
 196
 197         *p = paravirt_read_msr_amd(msr, &err);
 198         return err;
 199 }
 200
 201 static inline u64 paravirt_read_tsc(void)
 202 {
 203         return PVOP_CALL0(u64, pv_cpu_ops.read_tsc);
 204 }
 205
 206 #define rdtscl(low)                             \
 207 do {                                            \
 208         u64 _l = paravirt_read_tsc();           \
 209         low = (int)_l;                          \
 210 } while (0)
 211
 212 #define rdtscll(val) (val = paravirt_read_tsc())
 213
 214 static inline unsigned long long paravirt_sched_clock(void)
 215 {
 216         return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock);
 217 }
 218 #define calibrate_tsc() (pv_time_ops.get_tsc_khz())
 219
 220 static inline unsigned long long paravirt_read_pmc(int counter)
 221 {
 222         return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter);
 223 }
 224
 225 #define rdpmc(counter, low, high)               \
 226 do {                                            \
 227         u64 _l = paravirt_read_pmc(counter);    \
 228         low = (u32)_l;                          \
 229         high = _l >> 32;                        \
 230 } while (0)
 231
 232 static inline unsigned long long paravirt_rdtscp(unsigned int *aux)
 233 {
 234         return PVOP_CALL1(u64, pv_cpu_ops.read_tscp, aux);
 235 }
 236
 237 #define rdtscp(low, high, aux)                          \
 238 do {                                                    \
 239         int __aux;                                      \
 240         unsigned long __val = paravirt_rdtscp(&__aux);  \
 241         (low) = (u32)__val;                             \
 242         (high) = (u32)(__val >> 32);                    \
 243         (aux) = __aux;                                  \
 244 } while (0)
 245
 246 #define rdtscpll(val, aux)                              \
 247 do {                                                    \
 248         unsigned long __aux;                            \
 249         val = paravirt_rdtscp(&__aux);                  \
 250         (aux) = __aux;                                  \
 251 } while (0)
 252
 253 static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries)
 254 {
 255         PVOP_VCALL2(pv_cpu_ops.alloc_ldt, ldt, entries);
 256 }
 257
 258 static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries)
 259 {
 260         PVOP_VCALL2(pv_cpu_ops.free_ldt, ldt, entries);
 261 }
 262
 263 static inline void load_TR_desc(void)
 264 {
 265         PVOP_VCALL0(pv_cpu_ops.load_tr_desc);
 266 }
 267 static inline void load_gdt(const struct desc_ptr *dtr)
 268 {
 269         PVOP_VCALL1(pv_cpu_ops.load_gdt, dtr);
 270 }
 271 static inline void load_idt(const struct desc_ptr *dtr)
 272 {
 273         PVOP_VCALL1(pv_cpu_ops.load_idt, dtr);
 274 }
 275 static inline void set_ldt(const void *addr, unsigned entries)
 276 {
 277         PVOP_VCALL2(pv_cpu_ops.set_ldt, addr, entries);
 278 }
 279 static inline void store_gdt(struct desc_ptr *dtr)
 280 {
 281         PVOP_VCALL1(pv_cpu_ops.store_gdt, dtr);
 282 }
 283 static inline void store_idt(struct desc_ptr *dtr)
 284 {
 285         PVOP_VCALL1(pv_cpu_ops.store_idt, dtr);
 286 }
 287 static inline unsigned long paravirt_store_tr(void)
 288 {
 289         return PVOP_CALL0(unsigned long, pv_cpu_ops.store_tr);
 290 }
 291 #define store_tr(tr)    ((tr) = paravirt_store_tr())
 292 static inline void load_TLS(struct thread_struct *t, unsigned cpu)
 293 {
 294         PVOP_VCALL2(pv_cpu_ops.load_tls, t, cpu);
 295 }
 296
 297 #ifdef CONFIG_X86_64
 298 static inline void load_gs_index(unsigned int gs)
 299 {
 300         PVOP_VCALL1(pv_cpu_ops.load_gs_index, gs);
 301 }
 302 #endif
 303
 304 static inline void write_ldt_entry(struct desc_struct *dt, int entry,
 305                                    const void *desc)
 306 {
 307         PVOP_VCALL3(pv_cpu_ops.write_ldt_entry, dt, entry, desc);
 308 }
 309
 310 static inline void write_gdt_entry(struct desc_struct *dt, int entry,
 311                                    void *desc, int type)
 312 {
 313         PVOP_VCALL4(pv_cpu_ops.write_gdt_entry, dt, entry, desc, type);
 314 }
 315
 316 static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g)
 317 {
 318         PVOP_VCALL3(pv_cpu_ops.write_idt_entry, dt, entry, g);
 319 }
 320 static inline void set_iopl_mask(unsigned mask)
 321 {
 322         PVOP_VCALL1(pv_cpu_ops.set_iopl_mask, mask);
 323 }
 324
 325 /* The paravirtualized I/O functions */
 326 static inline void slow_down_io(void)
 327 {
 328         pv_cpu_ops.io_delay();
 329 #ifdef REALLY_SLOW_IO
 330         pv_cpu_ops.io_delay();
 331         pv_cpu_ops.io_delay();
 332         pv_cpu_ops.io_delay();
 333 #endif
 334 }
 335
 336 #ifdef CONFIG_SMP
 337 static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip,
 338                                     unsigned long start_esp)
 339 {
 340         PVOP_VCALL3(pv_apic_ops.startup_ipi_hook,
 341                     phys_apicid, start_eip, start_esp);
 342 }
 343 #endif
 344
 345 static inline void paravirt_activate_mm(struct mm_struct *prev,
 346                                         struct mm_struct *next)
 347 {
 348         PVOP_VCALL2(pv_mmu_ops.activate_mm, prev, next);
 349 }
 350
 351 static inline void arch_dup_mmap(struct mm_struct *oldmm,
 352                                  struct mm_struct *mm)
 353 {
 354         PVOP_VCALL2(pv_mmu_ops.dup_mmap, oldmm, mm);
 355 }
 356
 357 static inline void arch_exit_mmap(struct mm_struct *mm)
 358 {
 359         PVOP_VCALL1(pv_mmu_ops.exit_mmap, mm);
 360 }
 361
 362 static inline void __flush_tlb(void)
 363 {
 364         PVOP_VCALL0(pv_mmu_ops.flush_tlb_user);
 365 }
 366 static inline void __flush_tlb_global(void)
 367 {
 368         PVOP_VCALL0(pv_mmu_ops.flush_tlb_kernel);
 369 }
 370 static inline void __flush_tlb_single(unsigned long addr)
 371 {
 372         PVOP_VCALL1(pv_mmu_ops.flush_tlb_single, addr);
 373 }
 374
 375 static inline void flush_tlb_others(const struct cpumask *cpumask,
 376                                     struct mm_struct *mm,
 377                                     unsigned long va)
 378 {
 379         PVOP_VCALL3(pv_mmu_ops.flush_tlb_others, cpumask, mm, va);
 380 }
 381
 382 static inline int paravirt_pgd_alloc(struct mm_struct *mm)
 383 {
 384         return PVOP_CALL1(int, pv_mmu_ops.pgd_alloc, mm);
 385 }
 386
 387 static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd)
 388 {
 389         PVOP_VCALL2(pv_mmu_ops.pgd_free, mm, pgd);
 390 }
 391
 392 static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn)
 393 {
 394         PVOP_VCALL2(pv_mmu_ops.alloc_pte, mm, pfn);
 395 }
 396 static inline void paravirt_release_pte(unsigned long pfn)
 397 {
 398         PVOP_VCALL1(pv_mmu_ops.release_pte, pfn);
 399 }
 400
 401 static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn)
 402 {
 403         PVOP_VCALL2(pv_mmu_ops.alloc_pmd, mm, pfn);
 404 }
 405
 406 static inline void paravirt_alloc_pmd_clone(unsigned long pfn, unsigned long clonepfn,
 407                                             unsigned long start, unsigned long count)
 408 {
 409         PVOP_VCALL4(pv_mmu_ops.alloc_pmd_clone, pfn, clonepfn, start, count);
 410 }
 411 static inline void paravirt_release_pmd(unsigned long pfn)
 412 {
 413         PVOP_VCALL1(pv_mmu_ops.release_pmd, pfn);
 414 }
 415
 416 static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn)
 417 {
 418         PVOP_VCALL2(pv_mmu_ops.alloc_pud, mm, pfn);
 419 }
 420 static inline void paravirt_release_pud(unsigned long pfn)
 421 {
 422         PVOP_VCALL1(pv_mmu_ops.release_pud, pfn);
 423 }
 424
 425 #ifdef CONFIG_HIGHPTE
 426 static inline void *kmap_atomic_pte(struct page *page, enum km_type type)
 427 {
 428         unsigned long ret;
 429         ret = PVOP_CALL2(unsigned long, pv_mmu_ops.kmap_atomic_pte, page, type);
 430         return (void *)ret;
 431 }
 432 #endif
 433
 434 static inline void pte_update(struct mm_struct *mm, unsigned long addr,
 435                               pte_t *ptep)
 436 {
 437         PVOP_VCALL3(pv_mmu_ops.pte_update, mm, addr, ptep);
 438 }
 439
 440 static inline void pte_update_defer(struct mm_struct *mm, unsigned long addr,
 441                                     pte_t *ptep)
 442 {
 443         PVOP_VCALL3(pv_mmu_ops.pte_update_defer, mm, addr, ptep);
 444 }
 445
 446 static inline pte_t __pte(pteval_t val)
 447 {
 448         pteval_t ret;
 449
 450         if (sizeof(pteval_t) > sizeof(long))
 451                 ret = PVOP_CALLEE2(pteval_t,
 452                                    pv_mmu_ops.make_pte,
 453                                    val, (u64)val >> 32);
 454         else
 455                 ret = PVOP_CALLEE1(pteval_t,
 456                                    pv_mmu_ops.make_pte,
 457                                    val);
 458
 459         return (pte_t) { .pte = ret };
 460 }
 461
 462 static inline pteval_t pte_val(pte_t pte)
 463 {
 464         pteval_t ret;
 465
 466         if (sizeof(pteval_t) > sizeof(long))
 467                 ret = PVOP_CALLEE2(pteval_t, pv_mmu_ops.pte_val,
 468                                    pte.pte, (u64)pte.pte >> 32);
 469         else
 470                 ret = PVOP_CALLEE1(pteval_t, pv_mmu_ops.pte_val,
 471                                    pte.pte);
 472
 473         return ret;
 474 }
 475
 476 static inline pgd_t __pgd(pgdval_t val)
 477 {
 478         pgdval_t ret;
 479
 480         if (sizeof(pgdval_t) > sizeof(long))
 481                 ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.make_pgd,
 482                                    val, (u64)val >> 32);
 483         else
 484                 ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.make_pgd,
 485                                    val);
 486
 487         return (pgd_t) { ret };
 488 }
 489
 490 static inline pgdval_t pgd_val(pgd_t pgd)
 491 {
 492         pgdval_t ret;
 493
 494         if (sizeof(pgdval_t) > sizeof(long))
 495                 ret =  PVOP_CALLEE2(pgdval_t, pv_mmu_ops.pgd_val,
 496                                     pgd.pgd, (u64)pgd.pgd >> 32);
 497         else
 498                 ret =  PVOP_CALLEE1(pgdval_t, pv_mmu_ops.pgd_val,
 499                                     pgd.pgd);
 500
 501         return ret;
 502 }
 503
 504 #define  __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
 505 static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr,
 506                                            pte_t *ptep)
 507 {
 508         pteval_t ret;
 509
 510         ret = PVOP_CALL3(pteval_t, pv_mmu_ops.ptep_modify_prot_start,
 511                          mm, addr, ptep);
 512
 513         return (pte_t) { .pte = ret };
 514 }
 515
 516 static inline void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
 517                                            pte_t *ptep, pte_t pte)
 518 {
 519         if (sizeof(pteval_t) > sizeof(long))
 520                 /* 5 arg words */
 521                 pv_mmu_ops.ptep_modify_prot_commit(mm, addr, ptep, pte);
 522         else
 523                 PVOP_VCALL4(pv_mmu_ops.ptep_modify_prot_commit,
 524                             mm, addr, ptep, pte.pte);
 525 }
 526
 527 static inline void set_pte(pte_t *ptep, pte_t pte)
 528 {
 529         if (sizeof(pteval_t) > sizeof(long))
 530                 PVOP_VCALL3(pv_mmu_ops.set_pte, ptep,
 531                             pte.pte, (u64)pte.pte >> 32);
 532         else
 533                 PVOP_VCALL2(pv_mmu_ops.set_pte, ptep,
 534                             pte.pte);
 535 }
 536
 537 static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
 538                               pte_t *ptep, pte_t pte)
 539 {
 540         if (sizeof(pteval_t) > sizeof(long))
 541                 /* 5 arg words */
 542                 pv_mmu_ops.set_pte_at(mm, addr, ptep, pte);
 543         else
 544                 PVOP_VCALL4(pv_mmu_ops.set_pte_at, mm, addr, ptep, pte.pte);
 545 }
 546
 547 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
 548 {
 549         pmdval_t val = native_pmd_val(pmd);
 550
 551         if (sizeof(pmdval_t) > sizeof(long))
 552                 PVOP_VCALL3(pv_mmu_ops.set_pmd, pmdp, val, (u64)val >> 32);
 553         else
 554                 PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, val);
 555 }
 556
 557 #if PAGETABLE_LEVELS >= 3
 558 static inline pmd_t __pmd(pmdval_t val)
 559 {
 560         pmdval_t ret;
 561
 562         if (sizeof(pmdval_t) > sizeof(long))
 563                 ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.make_pmd,
 564                                    val, (u64)val >> 32);
 565         else
 566                 ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.make_pmd,
 567                                    val);
 568
 569         return (pmd_t) { ret };
 570 }
 571
 572 static inline pmdval_t pmd_val(pmd_t pmd)
 573 {
 574         pmdval_t ret;
 575
 576         if (sizeof(pmdval_t) > sizeof(long))
 577                 ret =  PVOP_CALLEE2(pmdval_t, pv_mmu_ops.pmd_val,
 578                                     pmd.pmd, (u64)pmd.pmd >> 32);
 579         else
 580                 ret =  PVOP_CALLEE1(pmdval_t, pv_mmu_ops.pmd_val,
 581                                     pmd.pmd);
 582
 583         return ret;
 584 }
 585
 586 static inline void set_pud(pud_t *pudp, pud_t pud)
 587 {
 588         pudval_t val = native_pud_val(pud);
 589
 590         if (sizeof(pudval_t) > sizeof(long))
 591                 PVOP_VCALL3(pv_mmu_ops.set_pud, pudp,
 592                             val, (u64)val >> 32);
 593         else
 594                 PVOP_VCALL2(pv_mmu_ops.set_pud, pudp,
 595                             val);
 596 }
 597 #if PAGETABLE_LEVELS == 4
 598 static inline pud_t __pud(pudval_t val)
 599 {
 600         pudval_t ret;
 601
 602         if (sizeof(pudval_t) > sizeof(long))
 603                 ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.make_pud,
 604                                    val, (u64)val >> 32);
 605         else
 606                 ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.make_pud,
 607                                    val);
 608
 609         return (pud_t) { ret };
 610 }
 611
 612 static inline pudval_t pud_val(pud_t pud)
 613 {
 614         pudval_t ret;
 615
 616         if (sizeof(pudval_t) > sizeof(long))
 617                 ret =  PVOP_CALLEE2(pudval_t, pv_mmu_ops.pud_val,
 618                                     pud.pud, (u64)pud.pud >> 32);
 619         else
 620                 ret =  PVOP_CALLEE1(pudval_t, pv_mmu_ops.pud_val,
 621                                     pud.pud);
 622
 623         return ret;
 624 }
 625
 626 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
 627 {
 628         pgdval_t val = native_pgd_val(pgd);
 629
 630         if (sizeof(pgdval_t) > sizeof(long))
 631                 PVOP_VCALL3(pv_mmu_ops.set_pgd, pgdp,
 632                             val, (u64)val >> 32);
 633         else
 634                 PVOP_VCALL2(pv_mmu_ops.set_pgd, pgdp,
 635                             val);
 636 }
 637
 638 static inline void pgd_clear(pgd_t *pgdp)
 639 {
 640         set_pgd(pgdp, __pgd(0));
 641 }
 642
 643 static inline void pud_clear(pud_t *pudp)
 644 {
 645         set_pud(pudp, __pud(0));
 646 }
 647
 648 #endif  /* PAGETABLE_LEVELS == 4 */
 649
 650 #endif  /* PAGETABLE_LEVELS >= 3 */
 651
 652 #ifdef CONFIG_X86_PAE
 653 /* Special-case pte-setting operations for PAE, which can't update a
 654    64-bit pte atomically */
 655 static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
 656 {
 657         PVOP_VCALL3(pv_mmu_ops.set_pte_atomic, ptep,
 658                     pte.pte, pte.pte >> 32);
 659 }
 660
 661 static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
 662                              pte_t *ptep)
 663 {
 664         PVOP_VCALL3(pv_mmu_ops.pte_clear, mm, addr, ptep);
 665 }
 666
 667 static inline void pmd_clear(pmd_t *pmdp)
 668 {
 669         PVOP_VCALL1(pv_mmu_ops.pmd_clear, pmdp);
 670 }
 671 #else  /* !CONFIG_X86_PAE */
 672 static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
 673 {
 674         set_pte(ptep, pte);
 675 }
 676
 677 static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
 678                              pte_t *ptep)
 679 {
 680         set_pte_at(mm, addr, ptep, __pte(0));
 681 }
 682
 683 static inline void pmd_clear(pmd_t *pmdp)
 684 {
 685         set_pmd(pmdp, __pmd(0));
 686 }
 687 #endif  /* CONFIG_X86_PAE */
 688
 689 #define  __HAVE_ARCH_START_CONTEXT_SWITCH
 690 static inline void arch_start_context_switch(struct task_struct *prev)
 691 {
 692         PVOP_VCALL1(pv_cpu_ops.start_context_switch, prev);
 693 }
 694
 695 static inline void arch_end_context_switch(struct task_struct *next)
 696 {
 697         PVOP_VCALL1(pv_cpu_ops.end_context_switch, next);
 698 }
 699
 700 #define  __HAVE_ARCH_ENTER_LAZY_MMU_MODE
 701 static inline void arch_enter_lazy_mmu_mode(void)
 702 {
 703         PVOP_VCALL0(pv_mmu_ops.lazy_mode.enter);
 704 }
 705
 706 static inline void arch_leave_lazy_mmu_mode(void)
 707 {
 708         PVOP_VCALL0(pv_mmu_ops.lazy_mode.leave);
 709 }
 710
 711 void arch_flush_lazy_mmu_mode(void);
 712
 713 static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx,
 714                                 phys_addr_t phys, pgprot_t flags)
 715 {
 716         pv_mmu_ops.set_fixmap(idx, phys, flags);
 717 }
 718
 719 #if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS)
 720
 721 static inline int __raw_spin_is_locked(struct raw_spinlock *lock)
 722 {
 723         return PVOP_CALL1(int, pv_lock_ops.spin_is_locked, lock);
 724 }
 725
 726 static inline int __raw_spin_is_contended(struct raw_spinlock *lock)
 727 {
 728         return PVOP_CALL1(int, pv_lock_ops.spin_is_contended, lock);
 729 }
 730 #define __raw_spin_is_contended __raw_spin_is_contended
 731
 732 static __always_inline void __raw_spin_lock(struct raw_spinlock *lock)
 733 {
 734         PVOP_VCALL1(pv_lock_ops.spin_lock, lock);
 735 }
 736
 737 static __always_inline void __raw_spin_lock_flags(struct raw_spinlock *lock,
 738                                                   unsigned long flags)
 739 {
 740         PVOP_VCALL2(pv_lock_ops.spin_lock_flags, lock, flags);
 741 }
 742
 743 static __always_inline int __raw_spin_trylock(struct raw_spinlock *lock)
 744 {
 745         return PVOP_CALL1(int, pv_lock_ops.spin_trylock, lock);
 746 }
 747
 748 static __always_inline void __raw_spin_unlock(struct raw_spinlock *lock)
 749 {
 750         PVOP_VCALL1(pv_lock_ops.spin_unlock, lock);
 751 }
 752
 753 #endif
 754
 755 #ifdef CONFIG_X86_32
 756 #define PV_SAVE_REGS "pushl %ecx; pushl %edx;"
 757 #define PV_RESTORE_REGS "popl %edx; popl %ecx;"
 758
 759 /* save and restore all caller-save registers, except return value */
 760 #define PV_SAVE_ALL_CALLER_REGS         "pushl %ecx;"
 761 #define PV_RESTORE_ALL_CALLER_REGS      "popl  %ecx;"
 762
 763 #define PV_FLAGS_ARG "0"
 764 #define PV_EXTRA_CLOBBERS
 765 #define PV_VEXTRA_CLOBBERS
 766 #else
 767 /* save and restore all caller-save registers, except return value */
 768 #define PV_SAVE_ALL_CALLER_REGS                                         \
 769         "push %rcx;"                                                    \
 770         "push %rdx;"                                                    \
 771         "push %rsi;"                                                    \
 772         "push %rdi;"                                                    \
 773         "push %r8;"                                                     \
 774         "push %r9;"                                                     \
 775         "push %r10;"                                                    \
 776         "push %r11;"
 777 #define PV_RESTORE_ALL_CALLER_REGS                                      \
 778         "pop %r11;"                                                     \
 779         "pop %r10;"                                                     \
 780         "pop %r9;"                                                      \
 781         "pop %r8;"                                                      \
 782         "pop %rdi;"                                                     \
 783         "pop %rsi;"                                                     \
 784         "pop %rdx;"                                                     \
 785         "pop %rcx;"
 786
 787 /* We save some registers, but all of them, that's too much. We clobber all
 788  * caller saved registers but the argument parameter */
 789 #define PV_SAVE_REGS "pushq %%rdi;"
 790 #define PV_RESTORE_REGS "popq %%rdi;"
 791 #define PV_EXTRA_CLOBBERS EXTRA_CLOBBERS, "rcx" , "rdx", "rsi"
 792 #define PV_VEXTRA_CLOBBERS EXTRA_CLOBBERS, "rdi", "rcx" , "rdx", "rsi"
 793 #define PV_FLAGS_ARG "D"
 794 #endif
 795
 796 /*
 797  * Generate a thunk around a function which saves all caller-save
 798  * registers except for the return value.  This allows C functions to
 799  * be called from assembler code where fewer than normal registers are
 800  * available.  It may also help code generation around calls from C
 801  * code if the common case doesn't use many registers.
 802  *
 803  * When a callee is wrapped in a thunk, the caller can assume that all
 804  * arg regs and all scratch registers are preserved across the
 805  * call. The return value in rax/eax will not be saved, even for void
 806  * functions.
 807  */
 808 #define PV_CALLEE_SAVE_REGS_THUNK(func)                                 \
 809         extern typeof(func) __raw_callee_save_##func;                   \
 810         static void *__##func##__ __used = func;                        \
 811                                                                         \
 812         asm(".pushsection .text;"                                       \
 813             "__raw_callee_save_" #func ": "                             \
 814             PV_SAVE_ALL_CALLER_REGS                                     \
 815             "call " #func ";"                                           \
 816             PV_RESTORE_ALL_CALLER_REGS                                  \
 817             "ret;"                                                      \
 818             ".popsection")
 819
 820 /* Get a reference to a callee-save function */
 821 #define PV_CALLEE_SAVE(func)                                            \
 822         ((struct paravirt_callee_save) { __raw_callee_save_##func })
 823
 824 /* Promise that "func" already uses the right calling convention */
 825 #define __PV_IS_CALLEE_SAVE(func)                       \
 826         ((struct paravirt_callee_save) { func })
 827
 828 static inline unsigned long __raw_local_save_flags(void)
 829 {
 830         unsigned long f;
 831
 832         asm volatile(paravirt_alt(PARAVIRT_CALL)
 833                      : "=a"(f)
 834                      : paravirt_type(pv_irq_ops.save_fl),
 835                        paravirt_clobber(CLBR_EAX)
 836                      : "memory", "cc");
 837         return f;
 838 }
 839
 840 static inline void raw_local_irq_restore(unsigned long f)
 841 {
 842         asm volatile(paravirt_alt(PARAVIRT_CALL)
 843                      : "=a"(f)
 844                      : PV_FLAGS_ARG(f),
 845                        paravirt_type(pv_irq_ops.restore_fl),
 846                        paravirt_clobber(CLBR_EAX)
 847                      : "memory", "cc");
 848 }
 849
 850 static inline void raw_local_irq_disable(void)
 851 {
 852         asm volatile(paravirt_alt(PARAVIRT_CALL)
 853                      :
 854                      : paravirt_type(pv_irq_ops.irq_disable),
 855                        paravirt_clobber(CLBR_EAX)
 856                      : "memory", "eax", "cc");
 857 }
 858
 859 static inline void raw_local_irq_enable(void)
 860 {
 861         asm volatile(paravirt_alt(PARAVIRT_CALL)
 862                      :
 863                      : paravirt_type(pv_irq_ops.irq_enable),
 864                        paravirt_clobber(CLBR_EAX)
 865                      : "memory", "eax", "cc");
 866 }
 867
 868 static inline unsigned long __raw_local_irq_save(void)
 869 {
 870         unsigned long f;
 871
 872         f = __raw_local_save_flags();
 873         raw_local_irq_disable();
 874         return f;
 875 }
 876
 877
 878 /* Make sure as little as possible of this mess escapes. */
 879 #undef PARAVIRT_CALL
 880 #undef __PVOP_CALL
 881 #undef __PVOP_VCALL
 882 #undef PVOP_VCALL0
 883 #undef PVOP_CALL0
 884 #undef PVOP_VCALL1
 885 #undef PVOP_CALL1
 886 #undef PVOP_VCALL2
 887 #undef PVOP_CALL2
 888 #undef PVOP_VCALL3
 889 #undef PVOP_CALL3
 890 #undef PVOP_VCALL4
 891 #undef PVOP_CALL4
 892
 893 extern void default_banner(void);
 894
 895 #else  /* __ASSEMBLY__ */
 896
 897 #define _PVSITE(ptype, clobbers, ops, word, algn)       \
 898 771:;                                           \
 899         ops;                                    \
 900 772:;                                           \
 901         .pushsection .parainstructions,"a";     \
 902          .align algn;                           \
 903          word 771b;                             \
 904          .byte ptype;                           \
 905          .byte 772b-771b;                       \
 906          .short clobbers;                       \
 907         .popsection
 908
 909
 910 #define COND_PUSH(set, mask, reg)                       \
 911         .if ((~(set)) & mask); push %reg; .endif
 912 #define COND_POP(set, mask, reg)                        \
 913         .if ((~(set)) & mask); pop %reg; .endif
 914
 915 #ifdef CONFIG_X86_64
 916
 917 #define PV_SAVE_REGS(set)                       \
 918         COND_PUSH(set, CLBR_RAX, rax);          \
 919         COND_PUSH(set, CLBR_RCX, rcx);          \
 920         COND_PUSH(set, CLBR_RDX, rdx);          \
 921         COND_PUSH(set, CLBR_RSI, rsi);          \
 922         COND_PUSH(set, CLBR_RDI, rdi);          \
 923         COND_PUSH(set, CLBR_R8, r8);            \
 924         COND_PUSH(set, CLBR_R9, r9);            \
 925         COND_PUSH(set, CLBR_R10, r10);          \
 926         COND_PUSH(set, CLBR_R11, r11)
 927 #define PV_RESTORE_REGS(set)                    \
 928         COND_POP(set, CLBR_R11, r11);           \
 929         COND_POP(set, CLBR_R10, r10);           \
 930         COND_POP(set, CLBR_R9, r9);             \
 931         COND_POP(set, CLBR_R8, r8);             \
 932         COND_POP(set, CLBR_RDI, rdi);           \
 933         COND_POP(set, CLBR_RSI, rsi);           \
 934         COND_POP(set, CLBR_RDX, rdx);           \
 935         COND_POP(set, CLBR_RCX, rcx);           \
 936         COND_POP(set, CLBR_RAX, rax)
 937
 938 #define PARA_PATCH(struct, off)        ((PARAVIRT_PATCH_##struct + (off)) / 8)
 939 #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .quad, 8)
 940 #define PARA_INDIRECT(addr)     *addr(%rip)
 941 #else
 942 #define PV_SAVE_REGS(set)                       \
 943         COND_PUSH(set, CLBR_EAX, eax);          \
 944         COND_PUSH(set, CLBR_EDI, edi);          \
 945         COND_PUSH(set, CLBR_ECX, ecx);          \
 946         COND_PUSH(set, CLBR_EDX, edx)
 947 #define PV_RESTORE_REGS(set)                    \
 948         COND_POP(set, CLBR_EDX, edx);           \
 949         COND_POP(set, CLBR_ECX, ecx);           \
 950         COND_POP(set, CLBR_EDI, edi);           \
 951         COND_POP(set, CLBR_EAX, eax)
 952
 953 #define PARA_PATCH(struct, off)        ((PARAVIRT_PATCH_##struct + (off)) / 4)
 954 #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4)
 955 #define PARA_INDIRECT(addr)     *%cs:addr
 956 #endif
 957
 958 #define INTERRUPT_RETURN                                                \
 959         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE,       \
 960                   jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_iret))
 961
 962 #define DISABLE_INTERRUPTS(clobbers)                                    \
 963         PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \
 964                   PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE);            \
 965                   call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_disable);    \
 966                   PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
 967
 968 #define ENABLE_INTERRUPTS(clobbers)                                     \
 969         PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers,  \
 970                   PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE);            \
 971                   call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_enable);     \
 972                   PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
 973
 974 #define USERGS_SYSRET32                                                 \
 975         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret32),       \
 976                   CLBR_NONE,                                            \
 977                   jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret32))
 978
 979 #ifdef CONFIG_X86_32
 980 #define GET_CR0_INTO_EAX                                \
 981         push %ecx; push %edx;                           \
 982         call PARA_INDIRECT(pv_cpu_ops+PV_CPU_read_cr0); \
 983         pop %edx; pop %ecx
 984
 985 #define ENABLE_INTERRUPTS_SYSEXIT                                       \
 986         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit),    \
 987                   CLBR_NONE,                                            \
 988                   jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_irq_enable_sysexit))
 989
 990
 991 #else   /* !CONFIG_X86_32 */
 992
 993 /*
 994  * If swapgs is used while the userspace stack is still current,
 995  * there's no way to call a pvop.  The PV replacement *must* be
 996  * inlined, or the swapgs instruction must be trapped and emulated.
 997  */
 998 #define SWAPGS_UNSAFE_STACK                                             \
 999         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE,     \
1000                   swapgs)
1001
1002 /*
1003  * Note: swapgs is very special, and in practise is either going to be
1004  * implemented with a single "swapgs" instruction or something very
1005  * special.  Either way, we don't need to save any registers for
1006  * it.
1007  */
1008 #define SWAPGS                                                          \
1009         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE,     \
1010                   call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs)          \
1011                  )
1012
1013 #define GET_CR2_INTO_RCX                                \
1014         call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr2); \
1015         movq %rax, %rcx;                                \
1016         xorq %rax, %rax;
1017
1018 #define PARAVIRT_ADJUST_EXCEPTION_FRAME                                 \
1019         PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_adjust_exception_frame), \
1020                   CLBR_NONE,                                            \
1021                   call PARA_INDIRECT(pv_irq_ops+PV_IRQ_adjust_exception_frame))
1022
1023 #define USERGS_SYSRET64                                                 \
1024         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret64),       \
1025                   CLBR_NONE,                                            \
1026                   jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret64))
1027
1028 #define ENABLE_INTERRUPTS_SYSEXIT32                                     \
1029         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit),    \
1030                   CLBR_NONE,                                            \
1031                   jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_irq_enable_sysexit))
1032 #endif  /* CONFIG_X86_32 */
1033
1034 #endif /* __ASSEMBLY__ */
1035 #else  /* CONFIG_PARAVIRT */
1036 # define default_banner x86_init_noop
1037 #endif /* !CONFIG_PARAVIRT */
1038 #endif /* _ASM_X86_PARAVIRT_H */