arch/sparc/mm/srmmu.c

   1 /*
   2  * srmmu.c:  SRMMU specific routines for memory management.
   3  *
   4  * Copyright (C) 1995 David S. Miller  (davem@caip.rutgers.edu)
   5  * Copyright (C) 1995,2002 Pete Zaitcev (zaitcev@yahoo.com)
   6  * Copyright (C) 1996 Eddie C. Dost    (ecd@skynet.be)
   7  * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
   8  * Copyright (C) 1999,2000 Anton Blanchard (anton@samba.org)
   9  */
  10
  11 #include <linux/kernel.h>
  12 #include <linux/mm.h>
  13 #include <linux/vmalloc.h>
  14 #include <linux/pagemap.h>
  15 #include <linux/init.h>
  16 #include <linux/spinlock.h>
  17 #include <linux/bootmem.h>
  18 #include <linux/fs.h>
  19 #include <linux/seq_file.h>
  20 #include <linux/kdebug.h>
  21 #include <linux/log2.h>
  22 #include <linux/gfp.h>
  23
  24 #include <asm/bitext.h>
  25 #include <asm/page.h>
  26 #include <asm/pgalloc.h>
  27 #include <asm/pgtable.h>
  28 #include <asm/io.h>
  29 #include <asm/vaddrs.h>
  30 #include <asm/traps.h>
  31 #include <asm/smp.h>
  32 #include <asm/mbus.h>
  33 #include <asm/cache.h>
  34 #include <asm/oplib.h>
  35 #include <asm/asi.h>
  36 #include <asm/msi.h>
  37 #include <asm/mmu_context.h>
  38 #include <asm/io-unit.h>
  39 #include <asm/cacheflush.h>
  40 #include <asm/tlbflush.h>
  41
  42 /* Now the cpu specific definitions. */
  43 #include <asm/viking.h>
  44 #include <asm/mxcc.h>
  45 #include <asm/ross.h>
  46 #include <asm/tsunami.h>
  47 #include <asm/swift.h>
  48 #include <asm/turbosparc.h>
  49 #include <asm/leon.h>
  50
  51 #include "srmmu.h"
  52
  53 enum mbus_module srmmu_modtype;
  54 static unsigned int hwbug_bitmask;
  55 int vac_cache_size;
  56 int vac_line_size;
  57
  58 struct ctx_list *ctx_list_pool;
  59 struct ctx_list ctx_free;
  60 struct ctx_list ctx_used;
  61
  62 extern struct resource sparc_iomap;
  63
  64 extern unsigned long last_valid_pfn;
  65
  66 static pgd_t *srmmu_swapper_pg_dir;
  67
  68 const struct sparc32_cachetlb_ops *sparc32_cachetlb_ops;
  69
  70 #ifdef CONFIG_SMP
  71 const struct sparc32_cachetlb_ops *local_ops;
  72
  73 #define FLUSH_BEGIN(mm)
  74 #define FLUSH_END
  75 #else
  76 #define FLUSH_BEGIN(mm) if ((mm)->context != NO_CONTEXT) {
  77 #define FLUSH_END       }
  78 #endif
  79
  80 int flush_page_for_dma_global = 1;
  81
  82 char *srmmu_name;
  83
  84 ctxd_t *srmmu_ctx_table_phys;
  85 static ctxd_t *srmmu_context_table;
  86
  87 int viking_mxcc_present;
  88 static DEFINE_SPINLOCK(srmmu_context_spinlock);
  89
  90 static int is_hypersparc;
  91
  92 static int srmmu_cache_pagetables;
  93
  94 /* these will be initialized in srmmu_nocache_calcsize() */
  95 static unsigned long srmmu_nocache_size;
  96 static unsigned long srmmu_nocache_end;
  97
  98 /* 1 bit <=> 256 bytes of nocache <=> 64 PTEs */
  99 #define SRMMU_NOCACHE_BITMAP_SHIFT (PAGE_SHIFT - 4)
 100
 101 /* The context table is a nocache user with the biggest alignment needs. */
 102 #define SRMMU_NOCACHE_ALIGN_MAX (sizeof(ctxd_t)*SRMMU_MAX_CONTEXTS)
 103
 104 void *srmmu_nocache_pool;
 105 void *srmmu_nocache_bitmap;
 106 static struct bit_map srmmu_nocache_map;
 107
 108 static inline int srmmu_pte_none(pte_t pte)
 109 { return !(pte_val(pte) & 0xFFFFFFF); }
 110
 111 static inline int srmmu_pmd_none(pmd_t pmd)
 112 { return !(pmd_val(pmd) & 0xFFFFFFF); }
 113
 114 static inline pte_t srmmu_pte_wrprotect(pte_t pte)
 115 { return __pte(pte_val(pte) & ~SRMMU_WRITE);}
 116
 117 static inline pte_t srmmu_pte_mkclean(pte_t pte)
 118 { return __pte(pte_val(pte) & ~SRMMU_DIRTY);}
 119
 120 static inline pte_t srmmu_pte_mkold(pte_t pte)
 121 { return __pte(pte_val(pte) & ~SRMMU_REF);}
 122
 123 /* XXX should we hyper_flush_whole_icache here - Anton */
 124 static inline void srmmu_ctxd_set(ctxd_t *ctxp, pgd_t *pgdp)
 125 { set_pte((pte_t *)ctxp, (SRMMU_ET_PTD | (__nocache_pa((unsigned long) pgdp) >> 4))); }
 126
 127 void pmd_set(pmd_t *pmdp, pte_t *ptep)
 128 {
 129         unsigned long ptp;      /* Physical address, shifted right by 4 */
 130         int i;
 131
 132         ptp = __nocache_pa((unsigned long) ptep) >> 4;
 133         for (i = 0; i < PTRS_PER_PTE/SRMMU_REAL_PTRS_PER_PTE; i++) {
 134                 set_pte((pte_t *)&pmdp->pmdv[i], SRMMU_ET_PTD | ptp);
 135                 ptp += (SRMMU_REAL_PTRS_PER_PTE*sizeof(pte_t) >> 4);
 136         }
 137 }
 138
 139 void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, struct page *ptep)
 140 {
 141         unsigned long ptp;      /* Physical address, shifted right by 4 */
 142         int i;
 143
 144         ptp = page_to_pfn(ptep) << (PAGE_SHIFT-4);      /* watch for overflow */
 145         for (i = 0; i < PTRS_PER_PTE/SRMMU_REAL_PTRS_PER_PTE; i++) {
 146                 set_pte((pte_t *)&pmdp->pmdv[i], SRMMU_ET_PTD | ptp);
 147                 ptp += (SRMMU_REAL_PTRS_PER_PTE*sizeof(pte_t) >> 4);
 148         }
 149 }
 150
 151 static inline pte_t srmmu_pte_modify(pte_t pte, pgprot_t newprot)
 152 { return __pte((pte_val(pte) & SRMMU_CHG_MASK) | pgprot_val(newprot)); }
 153
 154 /* to find an entry in a top-level page table... */
 155 static inline pgd_t *srmmu_pgd_offset(struct mm_struct * mm, unsigned long address)
 156 { return mm->pgd + (address >> SRMMU_PGDIR_SHIFT); }
 157
 158 /* Find an entry in the third-level page table.. */
 159 pte_t *pte_offset_kernel(pmd_t * dir, unsigned long address)
 160 {
 161         void *pte;
 162
 163         pte = __nocache_va((dir->pmdv[0] & SRMMU_PTD_PMASK) << 4);
 164         return (pte_t *) pte +
 165             ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1));
 166 }
 167
 168 /*
 169  * size: bytes to allocate in the nocache area.
 170  * align: bytes, number to align at.
 171  * Returns the virtual address of the allocated area.
 172  */
 173 static unsigned long __srmmu_get_nocache(int size, int align)
 174 {
 175         int offset;
 176
 177         if (size < SRMMU_NOCACHE_BITMAP_SHIFT) {
 178                 printk("Size 0x%x too small for nocache request\n", size);
 179                 size = SRMMU_NOCACHE_BITMAP_SHIFT;
 180         }
 181         if (size & (SRMMU_NOCACHE_BITMAP_SHIFT-1)) {
 182                 printk("Size 0x%x unaligned int nocache request\n", size);
 183                 size += SRMMU_NOCACHE_BITMAP_SHIFT-1;
 184         }
 185         BUG_ON(align > SRMMU_NOCACHE_ALIGN_MAX);
 186
 187         offset = bit_map_string_get(&srmmu_nocache_map,
 188                                         size >> SRMMU_NOCACHE_BITMAP_SHIFT,
 189                                         align >> SRMMU_NOCACHE_BITMAP_SHIFT);
 190         if (offset == -1) {
 191                 printk("srmmu: out of nocache %d: %d/%d\n",
 192                     size, (int) srmmu_nocache_size,
 193                     srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT);
 194                 return 0;
 195         }
 196
 197         return (SRMMU_NOCACHE_VADDR + (offset << SRMMU_NOCACHE_BITMAP_SHIFT));
 198 }
 199
 200 unsigned long srmmu_get_nocache(int size, int align)
 201 {
 202         unsigned long tmp;
 203
 204         tmp = __srmmu_get_nocache(size, align);
 205
 206         if (tmp)
 207                 memset((void *)tmp, 0, size);
 208
 209         return tmp;
 210 }
 211
 212 void srmmu_free_nocache(unsigned long vaddr, int size)
 213 {
 214         int offset;
 215
 216         if (vaddr < SRMMU_NOCACHE_VADDR) {
 217                 printk("Vaddr %lx is smaller than nocache base 0x%lx\n",
 218                     vaddr, (unsigned long)SRMMU_NOCACHE_VADDR);
 219                 BUG();
 220         }
 221         if (vaddr+size > srmmu_nocache_end) {
 222                 printk("Vaddr %lx is bigger than nocache end 0x%lx\n",
 223                     vaddr, srmmu_nocache_end);
 224                 BUG();
 225         }
 226         if (!is_power_of_2(size)) {
 227                 printk("Size 0x%x is not a power of 2\n", size);
 228                 BUG();
 229         }
 230         if (size < SRMMU_NOCACHE_BITMAP_SHIFT) {
 231                 printk("Size 0x%x is too small\n", size);
 232                 BUG();
 233         }
 234         if (vaddr & (size-1)) {
 235                 printk("Vaddr %lx is not aligned to size 0x%x\n", vaddr, size);
 236                 BUG();
 237         }
 238
 239         offset = (vaddr - SRMMU_NOCACHE_VADDR) >> SRMMU_NOCACHE_BITMAP_SHIFT;
 240         size = size >> SRMMU_NOCACHE_BITMAP_SHIFT;
 241
 242         bit_map_clear(&srmmu_nocache_map, offset, size);
 243 }
 244
 245 static void srmmu_early_allocate_ptable_skeleton(unsigned long start,
 246                                                  unsigned long end);
 247
 248 extern unsigned long probe_memory(void);        /* in fault.c */
 249
 250 /*
 251  * Reserve nocache dynamically proportionally to the amount of
 252  * system RAM. -- Tomas Szepe <szepe@pinerecords.com>, June 2002
 253  */
 254 static void srmmu_nocache_calcsize(void)
 255 {
 256         unsigned long sysmemavail = probe_memory() / 1024;
 257         int srmmu_nocache_npages;
 258
 259         srmmu_nocache_npages =
 260                 sysmemavail / SRMMU_NOCACHE_ALCRATIO / 1024 * 256;
 261
 262  /* P3 XXX The 4x overuse: corroborated by /proc/meminfo. */
 263         // if (srmmu_nocache_npages < 256) srmmu_nocache_npages = 256;
 264         if (srmmu_nocache_npages < SRMMU_MIN_NOCACHE_PAGES)
 265                 srmmu_nocache_npages = SRMMU_MIN_NOCACHE_PAGES;
 266
 267         /* anything above 1280 blows up */
 268         if (srmmu_nocache_npages > SRMMU_MAX_NOCACHE_PAGES)
 269                 srmmu_nocache_npages = SRMMU_MAX_NOCACHE_PAGES;
 270
 271         srmmu_nocache_size = srmmu_nocache_npages * PAGE_SIZE;
 272         srmmu_nocache_end = SRMMU_NOCACHE_VADDR + srmmu_nocache_size;
 273 }
 274
 275 static void __init srmmu_nocache_init(void)
 276 {
 277         unsigned int bitmap_bits;
 278         pgd_t *pgd;
 279         pmd_t *pmd;
 280         pte_t *pte;
 281         unsigned long paddr, vaddr;
 282         unsigned long pteval;
 283
 284         bitmap_bits = srmmu_nocache_size >> SRMMU_NOCACHE_BITMAP_SHIFT;
 285
 286         srmmu_nocache_pool = __alloc_bootmem(srmmu_nocache_size,
 287                 SRMMU_NOCACHE_ALIGN_MAX, 0UL);
 288         memset(srmmu_nocache_pool, 0, srmmu_nocache_size);
 289
 290         srmmu_nocache_bitmap = __alloc_bootmem(bitmap_bits >> 3, SMP_CACHE_BYTES, 0UL);
 291         bit_map_init(&srmmu_nocache_map, srmmu_nocache_bitmap, bitmap_bits);
 292
 293         srmmu_swapper_pg_dir = (pgd_t *)__srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
 294         memset(__nocache_fix(srmmu_swapper_pg_dir), 0, SRMMU_PGD_TABLE_SIZE);
 295         init_mm.pgd = srmmu_swapper_pg_dir;
 296
 297         srmmu_early_allocate_ptable_skeleton(SRMMU_NOCACHE_VADDR, srmmu_nocache_end);
 298
 299         paddr = __pa((unsigned long)srmmu_nocache_pool);
 300         vaddr = SRMMU_NOCACHE_VADDR;
 301
 302         while (vaddr < srmmu_nocache_end) {
 303                 pgd = pgd_offset_k(vaddr);
 304                 pmd = pmd_offset(__nocache_fix(pgd), vaddr);
 305                 pte = pte_offset_kernel(__nocache_fix(pmd), vaddr);
 306
 307                 pteval = ((paddr >> 4) | SRMMU_ET_PTE | SRMMU_PRIV);
 308
 309                 if (srmmu_cache_pagetables)
 310                         pteval |= SRMMU_CACHE;
 311
 312                 set_pte(__nocache_fix(pte), __pte(pteval));
 313
 314                 vaddr += PAGE_SIZE;
 315                 paddr += PAGE_SIZE;
 316         }
 317
 318         flush_cache_all();
 319         flush_tlb_all();
 320 }
 321
 322 pgd_t *get_pgd_fast(void)
 323 {
 324         pgd_t *pgd = NULL;
 325
 326         pgd = (pgd_t *)__srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
 327         if (pgd) {
 328                 pgd_t *init = pgd_offset_k(0);
 329                 memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
 330                 memcpy(pgd + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,
 331                                                 (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
 332         }
 333
 334         return pgd;
 335 }
 336
 337 /*
 338  * Hardware needs alignment to 256 only, but we align to whole page size
 339  * to reduce fragmentation problems due to the buddy principle.
 340  * XXX Provide actual fragmentation statistics in /proc.
 341  *
 342  * Alignments up to the page size are the same for physical and virtual
 343  * addresses of the nocache area.
 344  */
 345 pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
 346 {
 347         unsigned long pte;
 348         struct page *page;
 349
 350         if ((pte = (unsigned long)pte_alloc_one_kernel(mm, address)) == 0)
 351                 return NULL;
 352         page = pfn_to_page( __nocache_pa(pte) >> PAGE_SHIFT );
 353         pgtable_page_ctor(page);
 354         return page;
 355 }
 356
 357 void pte_free(struct mm_struct *mm, pgtable_t pte)
 358 {
 359         unsigned long p;
 360
 361         pgtable_page_dtor(pte);
 362         p = (unsigned long)page_address(pte);   /* Cached address (for test) */
 363         if (p == 0)
 364                 BUG();
 365         p = page_to_pfn(pte) << PAGE_SHIFT;     /* Physical address */
 366         p = (unsigned long) __nocache_va(p);    /* Nocached virtual */
 367         srmmu_free_nocache(p, PTE_SIZE);
 368 }
 369
 370 /*
 371  */
 372 static inline void alloc_context(struct mm_struct *old_mm, struct mm_struct *mm)
 373 {
 374         struct ctx_list *ctxp;
 375
 376         ctxp = ctx_free.next;
 377         if(ctxp != &ctx_free) {
 378                 remove_from_ctx_list(ctxp);
 379                 add_to_used_ctxlist(ctxp);
 380                 mm->context = ctxp->ctx_number;
 381                 ctxp->ctx_mm = mm;
 382                 return;
 383         }
 384         ctxp = ctx_used.next;
 385         if(ctxp->ctx_mm == old_mm)
 386                 ctxp = ctxp->next;
 387         if(ctxp == &ctx_used)
 388                 panic("out of mmu contexts");
 389         flush_cache_mm(ctxp->ctx_mm);
 390         flush_tlb_mm(ctxp->ctx_mm);
 391         remove_from_ctx_list(ctxp);
 392         add_to_used_ctxlist(ctxp);
 393         ctxp->ctx_mm->context = NO_CONTEXT;
 394         ctxp->ctx_mm = mm;
 395         mm->context = ctxp->ctx_number;
 396 }
 397
 398 static inline void free_context(int context)
 399 {
 400         struct ctx_list *ctx_old;
 401
 402         ctx_old = ctx_list_pool + context;
 403         remove_from_ctx_list(ctx_old);
 404         add_to_free_ctxlist(ctx_old);
 405 }
 406
 407
 408 void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm,
 409                struct task_struct *tsk)
 410 {
 411         if(mm->context == NO_CONTEXT) {
 412                 spin_lock(&srmmu_context_spinlock);
 413                 alloc_context(old_mm, mm);
 414                 spin_unlock(&srmmu_context_spinlock);
 415                 srmmu_ctxd_set(&srmmu_context_table[mm->context], mm->pgd);
 416         }
 417
 418         if (sparc_cpu_model == sparc_leon)
 419                 leon_switch_mm();
 420
 421         if (is_hypersparc)
 422                 hyper_flush_whole_icache();
 423
 424         srmmu_set_context(mm->context);
 425 }
 426
 427 /* Low level IO area allocation on the SRMMU. */
 428 static inline void srmmu_mapioaddr(unsigned long physaddr,
 429     unsigned long virt_addr, int bus_type)
 430 {
 431         pgd_t *pgdp;
 432         pmd_t *pmdp;
 433         pte_t *ptep;
 434         unsigned long tmp;
 435
 436         physaddr &= PAGE_MASK;
 437         pgdp = pgd_offset_k(virt_addr);
 438         pmdp = pmd_offset(pgdp, virt_addr);
 439         ptep = pte_offset_kernel(pmdp, virt_addr);
 440         tmp = (physaddr >> 4) | SRMMU_ET_PTE;
 441
 442         /*
 443          * I need to test whether this is consistent over all
 444          * sun4m's.  The bus_type represents the upper 4 bits of
 445          * 36-bit physical address on the I/O space lines...
 446          */
 447         tmp |= (bus_type << 28);
 448         tmp |= SRMMU_PRIV;
 449         __flush_page_to_ram(virt_addr);
 450         set_pte(ptep, __pte(tmp));
 451 }
 452
 453 void srmmu_mapiorange(unsigned int bus, unsigned long xpa,
 454                       unsigned long xva, unsigned int len)
 455 {
 456         while (len != 0) {
 457                 len -= PAGE_SIZE;
 458                 srmmu_mapioaddr(xpa, xva, bus);
 459                 xva += PAGE_SIZE;
 460                 xpa += PAGE_SIZE;
 461         }
 462         flush_tlb_all();
 463 }
 464
 465 static inline void srmmu_unmapioaddr(unsigned long virt_addr)
 466 {
 467         pgd_t *pgdp;
 468         pmd_t *pmdp;
 469         pte_t *ptep;
 470
 471         pgdp = pgd_offset_k(virt_addr);
 472         pmdp = pmd_offset(pgdp, virt_addr);
 473         ptep = pte_offset_kernel(pmdp, virt_addr);
 474
 475         /* No need to flush uncacheable page. */
 476         __pte_clear(ptep);
 477 }
 478
 479 void srmmu_unmapiorange(unsigned long virt_addr, unsigned int len)
 480 {
 481         while (len != 0) {
 482                 len -= PAGE_SIZE;
 483                 srmmu_unmapioaddr(virt_addr);
 484                 virt_addr += PAGE_SIZE;
 485         }
 486         flush_tlb_all();
 487 }
 488
 489 /*
 490  * On the SRMMU we do not have the problems with limited tlb entries
 491  * for mapping kernel pages, so we just take things from the free page
 492  * pool.  As a side effect we are putting a little too much pressure
 493  * on the gfp() subsystem.  This setup also makes the logic of the
 494  * iommu mapping code a lot easier as we can transparently handle
 495  * mappings on the kernel stack without any special code.
 496  */
 497 struct thread_info *alloc_thread_info_node(struct task_struct *tsk, int node)
 498 {
 499         struct thread_info *ret;
 500
 501         ret = (struct thread_info *)__get_free_pages(GFP_KERNEL,
 502                                                      THREAD_INFO_ORDER);
 503 #ifdef CONFIG_DEBUG_STACK_USAGE
 504         if (ret)
 505                 memset(ret, 0, PAGE_SIZE << THREAD_INFO_ORDER);
 506 #endif /* DEBUG_STACK_USAGE */
 507
 508         return ret;
 509 }
 510
 511 void free_thread_info(struct thread_info *ti)
 512 {
 513         free_pages((unsigned long)ti, THREAD_INFO_ORDER);
 514 }
 515
 516 /* tsunami.S */
 517 extern void tsunami_flush_cache_all(void);
 518 extern void tsunami_flush_cache_mm(struct mm_struct *mm);
 519 extern void tsunami_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
 520 extern void tsunami_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
 521 extern void tsunami_flush_page_to_ram(unsigned long page);
 522 extern void tsunami_flush_page_for_dma(unsigned long page);
 523 extern void tsunami_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr);
 524 extern void tsunami_flush_tlb_all(void);
 525 extern void tsunami_flush_tlb_mm(struct mm_struct *mm);
 526 extern void tsunami_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
 527 extern void tsunami_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
 528 extern void tsunami_setup_blockops(void);
 529
 530 /* swift.S */
 531 extern void swift_flush_cache_all(void);
 532 extern void swift_flush_cache_mm(struct mm_struct *mm);
 533 extern void swift_flush_cache_range(struct vm_area_struct *vma,
 534                                     unsigned long start, unsigned long end);
 535 extern void swift_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
 536 extern void swift_flush_page_to_ram(unsigned long page);
 537 extern void swift_flush_page_for_dma(unsigned long page);
 538 extern void swift_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr);
 539 extern void swift_flush_tlb_all(void);
 540 extern void swift_flush_tlb_mm(struct mm_struct *mm);
 541 extern void swift_flush_tlb_range(struct vm_area_struct *vma,
 542                                   unsigned long start, unsigned long end);
 543 extern void swift_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
 544
 545 #if 0  /* P3: deadwood to debug precise flushes on Swift. */
 546 void swift_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
 547 {
 548         int cctx, ctx1;
 549
 550         page &= PAGE_MASK;
 551         if ((ctx1 = vma->vm_mm->context) != -1) {
 552                 cctx = srmmu_get_context();
 553 /* Is context # ever different from current context? P3 */
 554                 if (cctx != ctx1) {
 555                         printk("flush ctx %02x curr %02x\n", ctx1, cctx);
 556                         srmmu_set_context(ctx1);
 557                         swift_flush_page(page);
 558                         __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : :
 559                                         "r" (page), "i" (ASI_M_FLUSH_PROBE));
 560                         srmmu_set_context(cctx);
 561                 } else {
 562                          /* Rm. prot. bits from virt. c. */
 563                         /* swift_flush_cache_all(); */
 564                         /* swift_flush_cache_page(vma, page); */
 565                         swift_flush_page(page);
 566
 567                         __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : :
 568                                 "r" (page), "i" (ASI_M_FLUSH_PROBE));
 569                         /* same as above: srmmu_flush_tlb_page() */
 570                 }
 571         }
 572 }
 573 #endif
 574
 575 /*
 576  * The following are all MBUS based SRMMU modules, and therefore could
 577  * be found in a multiprocessor configuration.  On the whole, these
 578  * chips seems to be much more touchy about DVMA and page tables
 579  * with respect to cache coherency.
 580  */
 581
 582 /* viking.S */
 583 extern void viking_flush_cache_all(void);
 584 extern void viking_flush_cache_mm(struct mm_struct *mm);
 585 extern void viking_flush_cache_range(struct vm_area_struct *vma, unsigned long start,
 586                                      unsigned long end);
 587 extern void viking_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
 588 extern void viking_flush_page_to_ram(unsigned long page);
 589 extern void viking_flush_page_for_dma(unsigned long page);
 590 extern void viking_flush_sig_insns(struct mm_struct *mm, unsigned long addr);
 591 extern void viking_flush_page(unsigned long page);
 592 extern void viking_mxcc_flush_page(unsigned long page);
 593 extern void viking_flush_tlb_all(void);
 594 extern void viking_flush_tlb_mm(struct mm_struct *mm);
 595 extern void viking_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 596                                    unsigned long end);
 597 extern void viking_flush_tlb_page(struct vm_area_struct *vma,
 598                                   unsigned long page);
 599 extern void sun4dsmp_flush_tlb_all(void);
 600 extern void sun4dsmp_flush_tlb_mm(struct mm_struct *mm);
 601 extern void sun4dsmp_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 602                                    unsigned long end);
 603 extern void sun4dsmp_flush_tlb_page(struct vm_area_struct *vma,
 604                                   unsigned long page);
 605
 606 /* hypersparc.S */
 607 extern void hypersparc_flush_cache_all(void);
 608 extern void hypersparc_flush_cache_mm(struct mm_struct *mm);
 609 extern void hypersparc_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
 610 extern void hypersparc_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
 611 extern void hypersparc_flush_page_to_ram(unsigned long page);
 612 extern void hypersparc_flush_page_for_dma(unsigned long page);
 613 extern void hypersparc_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr);
 614 extern void hypersparc_flush_tlb_all(void);
 615 extern void hypersparc_flush_tlb_mm(struct mm_struct *mm);
 616 extern void hypersparc_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
 617 extern void hypersparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
 618 extern void hypersparc_setup_blockops(void);
 619
 620 /*
 621  * NOTE: All of this startup code assumes the low 16mb (approx.) of
 622  *       kernel mappings are done with one single contiguous chunk of
 623  *       ram.  On small ram machines (classics mainly) we only get
 624  *       around 8mb mapped for us.
 625  */
 626
 627 static void __init early_pgtable_allocfail(char *type)
 628 {
 629         prom_printf("inherit_prom_mappings: Cannot alloc kernel %s.\n", type);
 630         prom_halt();
 631 }
 632
 633 static void __init srmmu_early_allocate_ptable_skeleton(unsigned long start,
 634                                                         unsigned long end)
 635 {
 636         pgd_t *pgdp;
 637         pmd_t *pmdp;
 638         pte_t *ptep;
 639
 640         while(start < end) {
 641                 pgdp = pgd_offset_k(start);
 642                 if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) {
 643                         pmdp = (pmd_t *) __srmmu_get_nocache(
 644                             SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
 645                         if (pmdp == NULL)
 646                                 early_pgtable_allocfail("pmd");
 647                         memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE);
 648                         pgd_set(__nocache_fix(pgdp), pmdp);
 649                 }
 650                 pmdp = pmd_offset(__nocache_fix(pgdp), start);
 651                 if(srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) {
 652                         ptep = (pte_t *)__srmmu_get_nocache(PTE_SIZE, PTE_SIZE);
 653                         if (ptep == NULL)
 654                                 early_pgtable_allocfail("pte");
 655                         memset(__nocache_fix(ptep), 0, PTE_SIZE);
 656                         pmd_set(__nocache_fix(pmdp), ptep);
 657                 }
 658                 if (start > (0xffffffffUL - PMD_SIZE))
 659                         break;
 660                 start = (start + PMD_SIZE) & PMD_MASK;
 661         }
 662 }
 663
 664 static void __init srmmu_allocate_ptable_skeleton(unsigned long start,
 665                                                   unsigned long end)
 666 {
 667         pgd_t *pgdp;
 668         pmd_t *pmdp;
 669         pte_t *ptep;
 670
 671         while(start < end) {
 672                 pgdp = pgd_offset_k(start);
 673                 if (pgd_none(*pgdp)) {
 674                         pmdp = (pmd_t *)__srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
 675                         if (pmdp == NULL)
 676                                 early_pgtable_allocfail("pmd");
 677                         memset(pmdp, 0, SRMMU_PMD_TABLE_SIZE);
 678                         pgd_set(pgdp, pmdp);
 679                 }
 680                 pmdp = pmd_offset(pgdp, start);
 681                 if(srmmu_pmd_none(*pmdp)) {
 682                         ptep = (pte_t *) __srmmu_get_nocache(PTE_SIZE,
 683                                                              PTE_SIZE);
 684                         if (ptep == NULL)
 685                                 early_pgtable_allocfail("pte");
 686                         memset(ptep, 0, PTE_SIZE);
 687                         pmd_set(pmdp, ptep);
 688                 }
 689                 if (start > (0xffffffffUL - PMD_SIZE))
 690                         break;
 691                 start = (start + PMD_SIZE) & PMD_MASK;
 692         }
 693 }
 694
 695 /*
 696  * This is much cleaner than poking around physical address space
 697  * looking at the prom's page table directly which is what most
 698  * other OS's do.  Yuck... this is much better.
 699  */
 700 static void __init srmmu_inherit_prom_mappings(unsigned long start,
 701                                                unsigned long end)
 702 {
 703         pgd_t *pgdp;
 704         pmd_t *pmdp;
 705         pte_t *ptep;
 706         int what = 0; /* 0 = normal-pte, 1 = pmd-level pte, 2 = pgd-level pte */
 707         unsigned long prompte;
 708
 709         while(start <= end) {
 710                 if (start == 0)
 711                         break; /* probably wrap around */
 712                 if(start == 0xfef00000)
 713                         start = KADB_DEBUGGER_BEGVM;
 714                 if(!(prompte = srmmu_hwprobe(start))) {
 715                         start += PAGE_SIZE;
 716                         continue;
 717                 }
 718
 719                 /* A red snapper, see what it really is. */
 720                 what = 0;
 721
 722                 if(!(start & ~(SRMMU_REAL_PMD_MASK))) {
 723                         if(srmmu_hwprobe((start-PAGE_SIZE) + SRMMU_REAL_PMD_SIZE) == prompte)
 724                                 what = 1;
 725                 }
 726
 727                 if(!(start & ~(SRMMU_PGDIR_MASK))) {
 728                         if(srmmu_hwprobe((start-PAGE_SIZE) + SRMMU_PGDIR_SIZE) ==
 729                            prompte)
 730                                 what = 2;
 731                 }
 732
 733                 pgdp = pgd_offset_k(start);
 734                 if(what == 2) {
 735                         *(pgd_t *)__nocache_fix(pgdp) = __pgd(prompte);
 736                         start += SRMMU_PGDIR_SIZE;
 737                         continue;
 738                 }
 739                 if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) {
 740                         pmdp = (pmd_t *)__srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
 741                         if (pmdp == NULL)
 742                                 early_pgtable_allocfail("pmd");
 743                         memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE);
 744                         pgd_set(__nocache_fix(pgdp), pmdp);
 745                 }
 746                 pmdp = pmd_offset(__nocache_fix(pgdp), start);
 747                 if(srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) {
 748                         ptep = (pte_t *) __srmmu_get_nocache(PTE_SIZE,
 749                                                              PTE_SIZE);
 750                         if (ptep == NULL)
 751                                 early_pgtable_allocfail("pte");
 752                         memset(__nocache_fix(ptep), 0, PTE_SIZE);
 753                         pmd_set(__nocache_fix(pmdp), ptep);
 754                 }
 755                 if(what == 1) {
 756                         /*
 757                          * We bend the rule where all 16 PTPs in a pmd_t point
 758                          * inside the same PTE page, and we leak a perfectly
 759                          * good hardware PTE piece. Alternatives seem worse.
 760                          */
 761                         unsigned int x; /* Index of HW PMD in soft cluster */
 762                         x = (start >> PMD_SHIFT) & 15;
 763                         *(unsigned long *)__nocache_fix(&pmdp->pmdv[x]) = prompte;
 764                         start += SRMMU_REAL_PMD_SIZE;
 765                         continue;
 766                 }
 767                 ptep = pte_offset_kernel(__nocache_fix(pmdp), start);
 768                 *(pte_t *)__nocache_fix(ptep) = __pte(prompte);
 769                 start += PAGE_SIZE;
 770         }
 771 }
 772
 773 #define KERNEL_PTE(page_shifted) ((page_shifted)|SRMMU_CACHE|SRMMU_PRIV|SRMMU_VALID)
 774
 775 /* Create a third-level SRMMU 16MB page mapping. */
 776 static void __init do_large_mapping(unsigned long vaddr, unsigned long phys_base)
 777 {
 778         pgd_t *pgdp = pgd_offset_k(vaddr);
 779         unsigned long big_pte;
 780
 781         big_pte = KERNEL_PTE(phys_base >> 4);
 782         *(pgd_t *)__nocache_fix(pgdp) = __pgd(big_pte);
 783 }
 784
 785 /* Map sp_bank entry SP_ENTRY, starting at virtual address VBASE. */
 786 static unsigned long __init map_spbank(unsigned long vbase, int sp_entry)
 787 {
 788         unsigned long pstart = (sp_banks[sp_entry].base_addr & SRMMU_PGDIR_MASK);
 789         unsigned long vstart = (vbase & SRMMU_PGDIR_MASK);
 790         unsigned long vend = SRMMU_PGDIR_ALIGN(vbase + sp_banks[sp_entry].num_bytes);
 791         /* Map "low" memory only */
 792         const unsigned long min_vaddr = PAGE_OFFSET;
 793         const unsigned long max_vaddr = PAGE_OFFSET + SRMMU_MAXMEM;
 794
 795         if (vstart < min_vaddr || vstart >= max_vaddr)
 796                 return vstart;
 797
 798         if (vend > max_vaddr || vend < min_vaddr)
 799                 vend = max_vaddr;
 800
 801         while(vstart < vend) {
 802                 do_large_mapping(vstart, pstart);
 803                 vstart += SRMMU_PGDIR_SIZE; pstart += SRMMU_PGDIR_SIZE;
 804         }
 805         return vstart;
 806 }
 807
 808 static inline void memprobe_error(char *msg)
 809 {
 810         prom_printf(msg);
 811         prom_printf("Halting now...\n");
 812         prom_halt();
 813 }
 814
 815 static inline void map_kernel(void)
 816 {
 817         int i;
 818
 819         if (phys_base > 0) {
 820                 do_large_mapping(PAGE_OFFSET, phys_base);
 821         }
 822
 823         for (i = 0; sp_banks[i].num_bytes != 0; i++) {
 824                 map_spbank((unsigned long)__va(sp_banks[i].base_addr), i);
 825         }
 826 }
 827
 828 /* Paging initialization on the Sparc Reference MMU. */
 829 extern void sparc_context_init(int);
 830
 831 void (*poke_srmmu)(void) __cpuinitdata = NULL;
 832
 833 extern unsigned long bootmem_init(unsigned long *pages_avail);
 834
 835 void __init srmmu_paging_init(void)
 836 {
 837         int i;
 838         phandle cpunode;
 839         char node_str[128];
 840         pgd_t *pgd;
 841         pmd_t *pmd;
 842         pte_t *pte;
 843         unsigned long pages_avail;
 844
 845         sparc_iomap.start = SUN4M_IOBASE_VADDR; /* 16MB of IOSPACE on all sun4m's. */
 846
 847         if (sparc_cpu_model == sun4d)
 848                 num_contexts = 65536; /* We know it is Viking */
 849         else {
 850                 /* Find the number of contexts on the srmmu. */
 851                 cpunode = prom_getchild(prom_root_node);
 852                 num_contexts = 0;
 853                 while(cpunode != 0) {
 854                         prom_getstring(cpunode, "device_type", node_str, sizeof(node_str));
 855                         if(!strcmp(node_str, "cpu")) {
 856                                 num_contexts = prom_getintdefault(cpunode, "mmu-nctx", 0x8);
 857                                 break;
 858                         }
 859                         cpunode = prom_getsibling(cpunode);
 860                 }
 861         }
 862
 863         if(!num_contexts) {
 864                 prom_printf("Something wrong, can't find cpu node in paging_init.\n");
 865                 prom_halt();
 866         }
 867
 868         pages_avail = 0;
 869         last_valid_pfn = bootmem_init(&pages_avail);
 870
 871         srmmu_nocache_calcsize();
 872         srmmu_nocache_init();
 873         srmmu_inherit_prom_mappings(0xfe400000,(LINUX_OPPROM_ENDVM-PAGE_SIZE));
 874         map_kernel();
 875
 876         /* ctx table has to be physically aligned to its size */
 877         srmmu_context_table = (ctxd_t *)__srmmu_get_nocache(num_contexts*sizeof(ctxd_t), num_contexts*sizeof(ctxd_t));
 878         srmmu_ctx_table_phys = (ctxd_t *)__nocache_pa((unsigned long)srmmu_context_table);
 879
 880         for(i = 0; i < num_contexts; i++)
 881                 srmmu_ctxd_set((ctxd_t *)__nocache_fix(&srmmu_context_table[i]), srmmu_swapper_pg_dir);
 882
 883         flush_cache_all();
 884         srmmu_set_ctable_ptr((unsigned long)srmmu_ctx_table_phys);
 885 #ifdef CONFIG_SMP
 886         /* Stop from hanging here... */
 887         local_ops->tlb_all();
 888 #else
 889         flush_tlb_all();
 890 #endif
 891         poke_srmmu();
 892
 893         srmmu_allocate_ptable_skeleton(sparc_iomap.start, IOBASE_END);
 894         srmmu_allocate_ptable_skeleton(DVMA_VADDR, DVMA_END);
 895
 896         srmmu_allocate_ptable_skeleton(
 897                 __fix_to_virt(__end_of_fixed_addresses - 1), FIXADDR_TOP);
 898         srmmu_allocate_ptable_skeleton(PKMAP_BASE, PKMAP_END);
 899
 900         pgd = pgd_offset_k(PKMAP_BASE);
 901         pmd = pmd_offset(pgd, PKMAP_BASE);
 902         pte = pte_offset_kernel(pmd, PKMAP_BASE);
 903         pkmap_page_table = pte;
 904
 905         flush_cache_all();
 906         flush_tlb_all();
 907
 908         sparc_context_init(num_contexts);
 909
 910         kmap_init();
 911
 912         {
 913                 unsigned long zones_size[MAX_NR_ZONES];
 914                 unsigned long zholes_size[MAX_NR_ZONES];
 915                 unsigned long npages;
 916                 int znum;
 917
 918                 for (znum = 0; znum < MAX_NR_ZONES; znum++)
 919                         zones_size[znum] = zholes_size[znum] = 0;
 920
 921                 npages = max_low_pfn - pfn_base;
 922
 923                 zones_size[ZONE_DMA] = npages;
 924                 zholes_size[ZONE_DMA] = npages - pages_avail;
 925
 926                 npages = highend_pfn - max_low_pfn;
 927                 zones_size[ZONE_HIGHMEM] = npages;
 928                 zholes_size[ZONE_HIGHMEM] = npages - calc_highpages();
 929
 930                 free_area_init_node(0, zones_size, pfn_base, zholes_size);
 931         }
 932 }
 933
 934 void mmu_info(struct seq_file *m)
 935 {
 936         seq_printf(m,
 937                    "MMU type\t: %s\n"
 938                    "contexts\t: %d\n"
 939                    "nocache total\t: %ld\n"
 940                    "nocache used\t: %d\n",
 941                    srmmu_name,
 942                    num_contexts,
 943                    srmmu_nocache_size,
 944                    srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT);
 945 }
 946
 947 void destroy_context(struct mm_struct *mm)
 948 {
 949
 950         if(mm->context != NO_CONTEXT) {
 951                 flush_cache_mm(mm);
 952                 srmmu_ctxd_set(&srmmu_context_table[mm->context], srmmu_swapper_pg_dir);
 953                 flush_tlb_mm(mm);
 954                 spin_lock(&srmmu_context_spinlock);
 955                 free_context(mm->context);
 956                 spin_unlock(&srmmu_context_spinlock);
 957                 mm->context = NO_CONTEXT;
 958         }
 959 }
 960
 961 /* Init various srmmu chip types. */
 962 static void __init srmmu_is_bad(void)
 963 {
 964         prom_printf("Could not determine SRMMU chip type.\n");
 965         prom_halt();
 966 }
 967
 968 static void __init init_vac_layout(void)
 969 {
 970         phandle nd;
 971         int cache_lines;
 972         char node_str[128];
 973 #ifdef CONFIG_SMP
 974         int cpu = 0;
 975         unsigned long max_size = 0;
 976         unsigned long min_line_size = 0x10000000;
 977 #endif
 978
 979         nd = prom_getchild(prom_root_node);
 980         while((nd = prom_getsibling(nd)) != 0) {
 981                 prom_getstring(nd, "device_type", node_str, sizeof(node_str));
 982                 if(!strcmp(node_str, "cpu")) {
 983                         vac_line_size = prom_getint(nd, "cache-line-size");
 984                         if (vac_line_size == -1) {
 985                                 prom_printf("can't determine cache-line-size, "
 986                                             "halting.\n");
 987                                 prom_halt();
 988                         }
 989                         cache_lines = prom_getint(nd, "cache-nlines");
 990                         if (cache_lines == -1) {
 991                                 prom_printf("can't determine cache-nlines, halting.\n");
 992                                 prom_halt();
 993                         }
 994
 995                         vac_cache_size = cache_lines * vac_line_size;
 996 #ifdef CONFIG_SMP
 997                         if(vac_cache_size > max_size)
 998                                 max_size = vac_cache_size;
 999                         if(vac_line_size < min_line_size)
1000                                 min_line_size = vac_line_size;
1001                         //FIXME: cpus not contiguous!!
1002                         cpu++;
1003                         if (cpu >= nr_cpu_ids || !cpu_online(cpu))
1004                                 break;
1005 #else
1006                         break;
1007 #endif
1008                 }
1009         }
1010         if(nd == 0) {
1011                 prom_printf("No CPU nodes found, halting.\n");
1012                 prom_halt();
1013         }
1014 #ifdef CONFIG_SMP
1015         vac_cache_size = max_size;
1016         vac_line_size = min_line_size;
1017 #endif
1018         printk("SRMMU: Using VAC size of %d bytes, line size %d bytes.\n",
1019                (int)vac_cache_size, (int)vac_line_size);
1020 }
1021
1022 static void __cpuinit poke_hypersparc(void)
1023 {
1024         volatile unsigned long clear;
1025         unsigned long mreg = srmmu_get_mmureg();
1026
1027         hyper_flush_unconditional_combined();
1028
1029         mreg &= ~(HYPERSPARC_CWENABLE);
1030         mreg |= (HYPERSPARC_CENABLE | HYPERSPARC_WBENABLE);
1031         mreg |= (HYPERSPARC_CMODE);
1032
1033         srmmu_set_mmureg(mreg);
1034
1035 #if 0 /* XXX I think this is bad news... -DaveM */
1036         hyper_clear_all_tags();
1037 #endif
1038
1039         put_ross_icr(HYPERSPARC_ICCR_FTD | HYPERSPARC_ICCR_ICE);
1040         hyper_flush_whole_icache();
1041         clear = srmmu_get_faddr();
1042         clear = srmmu_get_fstatus();
1043 }
1044
1045 static const struct sparc32_cachetlb_ops hypersparc_ops = {
1046         .cache_all      = hypersparc_flush_cache_all,
1047         .cache_mm       = hypersparc_flush_cache_mm,
1048         .cache_page     = hypersparc_flush_cache_page,
1049         .cache_range    = hypersparc_flush_cache_range,
1050         .tlb_all        = hypersparc_flush_tlb_all,
1051         .tlb_mm         = hypersparc_flush_tlb_mm,
1052         .tlb_page       = hypersparc_flush_tlb_page,
1053         .tlb_range      = hypersparc_flush_tlb_range,
1054         .page_to_ram    = hypersparc_flush_page_to_ram,
1055         .sig_insns      = hypersparc_flush_sig_insns,
1056         .page_for_dma   = hypersparc_flush_page_for_dma,
1057 };
1058
1059 static void __init init_hypersparc(void)
1060 {
1061         srmmu_name = "ROSS HyperSparc";
1062         srmmu_modtype = HyperSparc;
1063
1064         init_vac_layout();
1065
1066         is_hypersparc = 1;
1067         sparc32_cachetlb_ops = &hypersparc_ops;
1068
1069         poke_srmmu = poke_hypersparc;
1070
1071         hypersparc_setup_blockops();
1072 }
1073
1074 static void __cpuinit poke_swift(void)
1075 {
1076         unsigned long mreg;
1077
1078         /* Clear any crap from the cache or else... */
1079         swift_flush_cache_all();
1080
1081         /* Enable I & D caches */
1082         mreg = srmmu_get_mmureg();
1083         mreg |= (SWIFT_IE | SWIFT_DE);
1084         /*
1085          * The Swift branch folding logic is completely broken.  At
1086          * trap time, if things are just right, if can mistakenly
1087          * think that a trap is coming from kernel mode when in fact
1088          * it is coming from user mode (it mis-executes the branch in
1089          * the trap code).  So you see things like crashme completely
1090          * hosing your machine which is completely unacceptable.  Turn
1091          * this shit off... nice job Fujitsu.
1092          */
1093         mreg &= ~(SWIFT_BF);
1094         srmmu_set_mmureg(mreg);
1095 }
1096
1097 static const struct sparc32_cachetlb_ops swift_ops = {
1098         .cache_all      = swift_flush_cache_all,
1099         .cache_mm       = swift_flush_cache_mm,
1100         .cache_page     = swift_flush_cache_page,
1101         .cache_range    = swift_flush_cache_range,
1102         .tlb_all        = swift_flush_tlb_all,
1103         .tlb_mm         = swift_flush_tlb_mm,
1104         .tlb_page       = swift_flush_tlb_page,
1105         .tlb_range      = swift_flush_tlb_range,
1106         .page_to_ram    = swift_flush_page_to_ram,
1107         .sig_insns      = swift_flush_sig_insns,
1108         .page_for_dma   = swift_flush_page_for_dma,
1109 };
1110
1111 #define SWIFT_MASKID_ADDR  0x10003018
1112 static void __init init_swift(void)
1113 {
1114         unsigned long swift_rev;
1115
1116         __asm__ __volatile__("lda [%1] %2, %0\n\t"
1117                              "srl %0, 0x18, %0\n\t" :
1118                              "=r" (swift_rev) :
1119                              "r" (SWIFT_MASKID_ADDR), "i" (ASI_M_BYPASS));
1120         srmmu_name = "Fujitsu Swift";
1121         switch(swift_rev) {
1122         case 0x11:
1123         case 0x20:
1124         case 0x23:
1125         case 0x30:
1126                 srmmu_modtype = Swift_lots_o_bugs;
1127                 hwbug_bitmask |= (HWBUG_KERN_ACCBROKEN | HWBUG_KERN_CBITBROKEN);
1128                 /*
1129                  * Gee george, I wonder why Sun is so hush hush about
1130                  * this hardware bug... really braindamage stuff going
1131                  * on here.  However I think we can find a way to avoid
1132                  * all of the workaround overhead under Linux.  Basically,
1133                  * any page fault can cause kernel pages to become user
1134                  * accessible (the mmu gets confused and clears some of
1135                  * the ACC bits in kernel ptes).  Aha, sounds pretty
1136                  * horrible eh?  But wait, after extensive testing it appears
1137                  * that if you use pgd_t level large kernel pte's (like the
1138                  * 4MB pages on the Pentium) the bug does not get tripped
1139                  * at all.  This avoids almost all of the major overhead.
1140                  * Welcome to a world where your vendor tells you to,
1141                  * "apply this kernel patch" instead of "sorry for the
1142                  * broken hardware, send it back and we'll give you
1143                  * properly functioning parts"
1144                  */
1145                 break;
1146         case 0x25:
1147         case 0x31:
1148                 srmmu_modtype = Swift_bad_c;
1149                 hwbug_bitmask |= HWBUG_KERN_CBITBROKEN;
1150                 /*
1151                  * You see Sun allude to this hardware bug but never
1152                  * admit things directly, they'll say things like,
1153                  * "the Swift chip cache problems" or similar.
1154                  */
1155                 break;
1156         default:
1157                 srmmu_modtype = Swift_ok;
1158                 break;
1159         }
1160
1161         sparc32_cachetlb_ops = &swift_ops;
1162         flush_page_for_dma_global = 0;
1163
1164         /*
1165          * Are you now convinced that the Swift is one of the
1166          * biggest VLSI abortions of all time?  Bravo Fujitsu!
1167          * Fujitsu, the !#?!%$'d up processor people.  I bet if
1168          * you examined the microcode of the Swift you'd find
1169          * XXX's all over the place.
1170          */
1171         poke_srmmu = poke_swift;
1172 }
1173
1174 static void turbosparc_flush_cache_all(void)
1175 {
1176         flush_user_windows();
1177         turbosparc_idflash_clear();
1178 }
1179
1180 static void turbosparc_flush_cache_mm(struct mm_struct *mm)
1181 {
1182         FLUSH_BEGIN(mm)
1183         flush_user_windows();
1184         turbosparc_idflash_clear();
1185         FLUSH_END
1186 }
1187
1188 static void turbosparc_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1189 {
1190         FLUSH_BEGIN(vma->vm_mm)
1191         flush_user_windows();
1192         turbosparc_idflash_clear();
1193         FLUSH_END
1194 }
1195
1196 static void turbosparc_flush_cache_page(struct vm_area_struct *vma, unsigned long page)
1197 {
1198         FLUSH_BEGIN(vma->vm_mm)
1199         flush_user_windows();
1200         if (vma->vm_flags & VM_EXEC)
1201                 turbosparc_flush_icache();
1202         turbosparc_flush_dcache();
1203         FLUSH_END
1204 }
1205
1206 /* TurboSparc is copy-back, if we turn it on, but this does not work. */
1207 static void turbosparc_flush_page_to_ram(unsigned long page)
1208 {
1209 #ifdef TURBOSPARC_WRITEBACK
1210         volatile unsigned long clear;
1211
1212         if (srmmu_hwprobe(page))
1213                 turbosparc_flush_page_cache(page);
1214         clear = srmmu_get_fstatus();
1215 #endif
1216 }
1217
1218 static void turbosparc_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr)
1219 {
1220 }
1221
1222 static void turbosparc_flush_page_for_dma(unsigned long page)
1223 {
1224         turbosparc_flush_dcache();
1225 }
1226
1227 static void turbosparc_flush_tlb_all(void)
1228 {
1229         srmmu_flush_whole_tlb();
1230 }
1231
1232 static void turbosparc_flush_tlb_mm(struct mm_struct *mm)
1233 {
1234         FLUSH_BEGIN(mm)
1235         srmmu_flush_whole_tlb();
1236         FLUSH_END
1237 }
1238
1239 static void turbosparc_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1240 {
1241         FLUSH_BEGIN(vma->vm_mm)
1242         srmmu_flush_whole_tlb();
1243         FLUSH_END
1244 }
1245
1246 static void turbosparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
1247 {
1248         FLUSH_BEGIN(vma->vm_mm)
1249         srmmu_flush_whole_tlb();
1250         FLUSH_END
1251 }
1252
1253
1254 static void __cpuinit poke_turbosparc(void)
1255 {
1256         unsigned long mreg = srmmu_get_mmureg();
1257         unsigned long ccreg;
1258
1259         /* Clear any crap from the cache or else... */
1260         turbosparc_flush_cache_all();
1261         mreg &= ~(TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* Temporarily disable I & D caches */
1262         mreg &= ~(TURBOSPARC_PCENABLE);         /* Don't check parity */
1263         srmmu_set_mmureg(mreg);
1264
1265         ccreg = turbosparc_get_ccreg();
1266
1267 #ifdef TURBOSPARC_WRITEBACK
1268         ccreg |= (TURBOSPARC_SNENABLE);         /* Do DVMA snooping in Dcache */
1269         ccreg &= ~(TURBOSPARC_uS2 | TURBOSPARC_WTENABLE);
1270                         /* Write-back D-cache, emulate VLSI
1271                          * abortion number three, not number one */
1272 #else
1273         /* For now let's play safe, optimize later */
1274         ccreg |= (TURBOSPARC_SNENABLE | TURBOSPARC_WTENABLE);
1275                         /* Do DVMA snooping in Dcache, Write-thru D-cache */
1276         ccreg &= ~(TURBOSPARC_uS2);
1277                         /* Emulate VLSI abortion number three, not number one */
1278 #endif
1279
1280         switch (ccreg & 7) {
1281         case 0: /* No SE cache */
1282         case 7: /* Test mode */
1283                 break;
1284         default:
1285                 ccreg |= (TURBOSPARC_SCENABLE);
1286         }
1287         turbosparc_set_ccreg (ccreg);
1288
1289         mreg |= (TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* I & D caches on */
1290         mreg |= (TURBOSPARC_ICSNOOP);           /* Icache snooping on */
1291         srmmu_set_mmureg(mreg);
1292 }
1293
1294 static const struct sparc32_cachetlb_ops turbosparc_ops = {
1295         .cache_all      = turbosparc_flush_cache_all,
1296         .cache_mm       = turbosparc_flush_cache_mm,
1297         .cache_page     = turbosparc_flush_cache_page,
1298         .cache_range    = turbosparc_flush_cache_range,
1299         .tlb_all        = turbosparc_flush_tlb_all,
1300         .tlb_mm         = turbosparc_flush_tlb_mm,
1301         .tlb_page       = turbosparc_flush_tlb_page,
1302         .tlb_range      = turbosparc_flush_tlb_range,
1303         .page_to_ram    = turbosparc_flush_page_to_ram,
1304         .sig_insns      = turbosparc_flush_sig_insns,
1305         .page_for_dma   = turbosparc_flush_page_for_dma,
1306 };
1307
1308 static void __init init_turbosparc(void)
1309 {
1310         srmmu_name = "Fujitsu TurboSparc";
1311         srmmu_modtype = TurboSparc;
1312         sparc32_cachetlb_ops = &turbosparc_ops;
1313         poke_srmmu = poke_turbosparc;
1314 }
1315
1316 static void __cpuinit poke_tsunami(void)
1317 {
1318         unsigned long mreg = srmmu_get_mmureg();
1319
1320         tsunami_flush_icache();
1321         tsunami_flush_dcache();
1322         mreg &= ~TSUNAMI_ITD;
1323         mreg |= (TSUNAMI_IENAB | TSUNAMI_DENAB);
1324         srmmu_set_mmureg(mreg);
1325 }
1326
1327 static const struct sparc32_cachetlb_ops tsunami_ops = {
1328         .cache_all      = tsunami_flush_cache_all,
1329         .cache_mm       = tsunami_flush_cache_mm,
1330         .cache_page     = tsunami_flush_cache_page,
1331         .cache_range    = tsunami_flush_cache_range,
1332         .tlb_all        = tsunami_flush_tlb_all,
1333         .tlb_mm         = tsunami_flush_tlb_mm,
1334         .tlb_page       = tsunami_flush_tlb_page,
1335         .tlb_range      = tsunami_flush_tlb_range,
1336         .page_to_ram    = tsunami_flush_page_to_ram,
1337         .sig_insns      = tsunami_flush_sig_insns,
1338         .page_for_dma   = tsunami_flush_page_for_dma,
1339 };
1340
1341 static void __init init_tsunami(void)
1342 {
1343         /*
1344          * Tsunami's pretty sane, Sun and TI actually got it
1345          * somewhat right this time.  Fujitsu should have
1346          * taken some lessons from them.
1347          */
1348
1349         srmmu_name = "TI Tsunami";
1350         srmmu_modtype = Tsunami;
1351         sparc32_cachetlb_ops = &tsunami_ops;
1352         poke_srmmu = poke_tsunami;
1353
1354         tsunami_setup_blockops();
1355 }
1356
1357 static void __cpuinit poke_viking(void)
1358 {
1359         unsigned long mreg = srmmu_get_mmureg();
1360         static int smp_catch;
1361
1362         if (viking_mxcc_present) {
1363                 unsigned long mxcc_control = mxcc_get_creg();
1364
1365                 mxcc_control |= (MXCC_CTL_ECE | MXCC_CTL_PRE | MXCC_CTL_MCE);
1366                 mxcc_control &= ~(MXCC_CTL_RRC);
1367                 mxcc_set_creg(mxcc_control);
1368
1369                 /*
1370                  * We don't need memory parity checks.
1371                  * XXX This is a mess, have to dig out later. ecd.
1372                 viking_mxcc_turn_off_parity(&mreg, &mxcc_control);
1373                  */
1374
1375                 /* We do cache ptables on MXCC. */
1376                 mreg |= VIKING_TCENABLE;
1377         } else {
1378                 unsigned long bpreg;
1379
1380                 mreg &= ~(VIKING_TCENABLE);
1381                 if(smp_catch++) {
1382                         /* Must disable mixed-cmd mode here for other cpu's. */
1383                         bpreg = viking_get_bpreg();
1384                         bpreg &= ~(VIKING_ACTION_MIX);
1385                         viking_set_bpreg(bpreg);
1386
1387                         /* Just in case PROM does something funny. */
1388                         msi_set_sync();
1389                 }
1390         }
1391
1392         mreg |= VIKING_SPENABLE;
1393         mreg |= (VIKING_ICENABLE | VIKING_DCENABLE);
1394         mreg |= VIKING_SBENABLE;
1395         mreg &= ~(VIKING_ACENABLE);
1396         srmmu_set_mmureg(mreg);
1397 }
1398
1399 static struct sparc32_cachetlb_ops viking_ops = {
1400         .cache_all      = viking_flush_cache_all,
1401         .cache_mm       = viking_flush_cache_mm,
1402         .cache_page     = viking_flush_cache_page,
1403         .cache_range    = viking_flush_cache_range,
1404         .tlb_all        = viking_flush_tlb_all,
1405         .tlb_mm         = viking_flush_tlb_mm,
1406         .tlb_page       = viking_flush_tlb_page,
1407         .tlb_range      = viking_flush_tlb_range,
1408         .page_to_ram    = viking_flush_page_to_ram,
1409         .sig_insns      = viking_flush_sig_insns,
1410         .page_for_dma   = viking_flush_page_for_dma,
1411 };
1412
1413 #ifdef CONFIG_SMP
1414 /* On sun4d the cpu broadcasts local TLB flushes, so we can just
1415  * perform the local TLB flush and all the other cpus will see it.
1416  * But, unfortunately, there is a bug in the sun4d XBUS backplane
1417  * that requires that we add some synchronization to these flushes.
1418  *
1419  * The bug is that the fifo which keeps track of all the pending TLB
1420  * broadcasts in the system is an entry or two too small, so if we
1421  * have too many going at once we'll overflow that fifo and lose a TLB
1422  * flush resulting in corruption.
1423  *
1424  * Our workaround is to take a global spinlock around the TLB flushes,
1425  * which guarentees we won't ever have too many pending.  It's a big
1426  * hammer, but a semaphore like system to make sure we only have N TLB
1427  * flushes going at once will require SMP locking anyways so there's
1428  * no real value in trying any harder than this.
1429  */
1430 static struct sparc32_cachetlb_ops viking_sun4d_smp_ops = {
1431         .cache_all      = viking_flush_cache_all,
1432         .cache_mm       = viking_flush_cache_mm,
1433         .cache_page     = viking_flush_cache_page,
1434         .cache_range    = viking_flush_cache_range,
1435         .tlb_all        = sun4dsmp_flush_tlb_all,
1436         .tlb_mm         = sun4dsmp_flush_tlb_mm,
1437         .tlb_page       = sun4dsmp_flush_tlb_page,
1438         .tlb_range      = sun4dsmp_flush_tlb_range,
1439         .page_to_ram    = viking_flush_page_to_ram,
1440         .sig_insns      = viking_flush_sig_insns,
1441         .page_for_dma   = viking_flush_page_for_dma,
1442 };
1443 #endif
1444
1445 static void __init init_viking(void)
1446 {
1447         unsigned long mreg = srmmu_get_mmureg();
1448
1449         /* Ahhh, the viking.  SRMMU VLSI abortion number two... */
1450         if(mreg & VIKING_MMODE) {
1451                 srmmu_name = "TI Viking";
1452                 viking_mxcc_present = 0;
1453                 msi_set_sync();
1454
1455                 /*
1456                  * We need this to make sure old viking takes no hits
1457                  * on it's cache for dma snoops to workaround the
1458                  * "load from non-cacheable memory" interrupt bug.
1459                  * This is only necessary because of the new way in
1460                  * which we use the IOMMU.
1461                  */
1462                 viking_ops.page_for_dma = viking_flush_page;
1463 #ifdef CONFIG_SMP
1464                 viking_sun4d_smp_ops.page_for_dma = viking_flush_page;
1465 #endif
1466                 flush_page_for_dma_global = 0;
1467         } else {
1468                 srmmu_name = "TI Viking/MXCC";
1469                 viking_mxcc_present = 1;
1470                 srmmu_cache_pagetables = 1;
1471         }
1472
1473         sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *)
1474                 &viking_ops;
1475 #ifdef CONFIG_SMP
1476         if (sparc_cpu_model == sun4d)
1477                 sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *)
1478                         &viking_sun4d_smp_ops;
1479 #endif
1480
1481         poke_srmmu = poke_viking;
1482 }
1483
1484 /* Probe for the srmmu chip version. */
1485 static void __init get_srmmu_type(void)
1486 {
1487         unsigned long mreg, psr;
1488         unsigned long mod_typ, mod_rev, psr_typ, psr_vers;
1489
1490         srmmu_modtype = SRMMU_INVAL_MOD;
1491         hwbug_bitmask = 0;
1492
1493         mreg = srmmu_get_mmureg(); psr = get_psr();
1494         mod_typ = (mreg & 0xf0000000) >> 28;
1495         mod_rev = (mreg & 0x0f000000) >> 24;
1496         psr_typ = (psr >> 28) & 0xf;
1497         psr_vers = (psr >> 24) & 0xf;
1498
1499         /* First, check for sparc-leon. */
1500         if (sparc_cpu_model == sparc_leon) {
1501                 init_leon();
1502                 return;
1503         }
1504
1505         /* Second, check for HyperSparc or Cypress. */
1506         if(mod_typ == 1) {
1507                 switch(mod_rev) {
1508                 case 7:
1509                         /* UP or MP Hypersparc */
1510                         init_hypersparc();
1511                         break;
1512                 case 0:
1513                 case 2:
1514                 case 10:
1515                 case 11:
1516                 case 12:
1517                 case 13:
1518                 case 14:
1519                 case 15:
1520                 default:
1521                         prom_printf("Sparc-Linux Cypress support does not longer exit.\n");
1522                         prom_halt();
1523                         break;
1524                 }
1525                 return;
1526         }
1527
1528         /*
1529          * Now Fujitsu TurboSparc. It might happen that it is
1530          * in Swift emulation mode, so we will check later...
1531          */
1532         if (psr_typ == 0 && psr_vers == 5) {
1533                 init_turbosparc();
1534                 return;
1535         }
1536
1537         /* Next check for Fujitsu Swift. */
1538         if(psr_typ == 0 && psr_vers == 4) {
1539                 phandle cpunode;
1540                 char node_str[128];
1541
1542                 /* Look if it is not a TurboSparc emulating Swift... */
1543                 cpunode = prom_getchild(prom_root_node);
1544                 while((cpunode = prom_getsibling(cpunode)) != 0) {
1545                         prom_getstring(cpunode, "device_type", node_str, sizeof(node_str));
1546                         if(!strcmp(node_str, "cpu")) {
1547                                 if (!prom_getintdefault(cpunode, "psr-implementation", 1) &&
1548                                     prom_getintdefault(cpunode, "psr-version", 1) == 5) {
1549                                         init_turbosparc();
1550                                         return;
1551                                 }
1552                                 break;
1553                         }
1554                 }
1555
1556                 init_swift();
1557                 return;
1558         }
1559
1560         /* Now the Viking family of srmmu. */
1561         if(psr_typ == 4 &&
1562            ((psr_vers == 0) ||
1563             ((psr_vers == 1) && (mod_typ == 0) && (mod_rev == 0)))) {
1564                 init_viking();
1565                 return;
1566         }
1567
1568         /* Finally the Tsunami. */
1569         if(psr_typ == 4 && psr_vers == 1 && (mod_typ || mod_rev)) {
1570                 init_tsunami();
1571                 return;
1572         }
1573
1574         /* Oh well */
1575         srmmu_is_bad();
1576 }
1577
1578 #ifdef CONFIG_SMP
1579 /* Local cross-calls. */
1580 static void smp_flush_page_for_dma(unsigned long page)
1581 {
1582         xc1((smpfunc_t) local_ops->page_for_dma, page);
1583         local_ops->page_for_dma(page);
1584 }
1585
1586 static void smp_flush_cache_all(void)
1587 {
1588         xc0((smpfunc_t) local_ops->cache_all);
1589         local_ops->cache_all();
1590 }
1591
1592 static void smp_flush_tlb_all(void)
1593 {
1594         xc0((smpfunc_t) local_ops->tlb_all);
1595         local_ops->tlb_all();
1596 }
1597
1598 static void smp_flush_cache_mm(struct mm_struct *mm)
1599 {
1600         if (mm->context != NO_CONTEXT) {
1601                 cpumask_t cpu_mask;
1602                 cpumask_copy(&cpu_mask, mm_cpumask(mm));
1603                 cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
1604                 if (!cpumask_empty(&cpu_mask))
1605                         xc1((smpfunc_t) local_ops->cache_mm, (unsigned long) mm);
1606                 local_ops->cache_mm(mm);
1607         }
1608 }
1609
1610 static void smp_flush_tlb_mm(struct mm_struct *mm)
1611 {
1612         if (mm->context != NO_CONTEXT) {
1613                 cpumask_t cpu_mask;
1614                 cpumask_copy(&cpu_mask, mm_cpumask(mm));
1615                 cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
1616                 if (!cpumask_empty(&cpu_mask)) {
1617                         xc1((smpfunc_t) local_ops->tlb_mm, (unsigned long) mm);
1618                         if (atomic_read(&mm->mm_users) == 1 && current->active_mm == mm)
1619                                 cpumask_copy(mm_cpumask(mm),
1620                                              cpumask_of(smp_processor_id()));
1621                 }
1622                 local_ops->tlb_mm(mm);
1623         }
1624 }
1625
1626 static void smp_flush_cache_range(struct vm_area_struct *vma,
1627                                   unsigned long start,
1628                                   unsigned long end)
1629 {
1630         struct mm_struct *mm = vma->vm_mm;
1631
1632         if (mm->context != NO_CONTEXT) {
1633                 cpumask_t cpu_mask;
1634                 cpumask_copy(&cpu_mask, mm_cpumask(mm));
1635                 cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
1636                 if (!cpumask_empty(&cpu_mask))
1637                         xc3((smpfunc_t) local_ops->cache_range,
1638                             (unsigned long) vma, start, end);
1639                 local_ops->cache_range(vma, start, end);
1640         }
1641 }
1642
1643 static void smp_flush_tlb_range(struct vm_area_struct *vma,
1644                                 unsigned long start,
1645                                 unsigned long end)
1646 {
1647         struct mm_struct *mm = vma->vm_mm;
1648
1649         if (mm->context != NO_CONTEXT) {
1650                 cpumask_t cpu_mask;
1651                 cpumask_copy(&cpu_mask, mm_cpumask(mm));
1652                 cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
1653                 if (!cpumask_empty(&cpu_mask))
1654                         xc3((smpfunc_t) local_ops->tlb_range,
1655                             (unsigned long) vma, start, end);
1656                 local_ops->tlb_range(vma, start, end);
1657         }
1658 }
1659
1660 static void smp_flush_cache_page(struct vm_area_struct *vma, unsigned long page)
1661 {
1662         struct mm_struct *mm = vma->vm_mm;
1663
1664         if (mm->context != NO_CONTEXT) {
1665                 cpumask_t cpu_mask;
1666                 cpumask_copy(&cpu_mask, mm_cpumask(mm));
1667                 cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
1668                 if (!cpumask_empty(&cpu_mask))
1669                         xc2((smpfunc_t) local_ops->cache_page,
1670                             (unsigned long) vma, page);
1671                 local_ops->cache_page(vma, page);
1672         }
1673 }
1674
1675 static void smp_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
1676 {
1677         struct mm_struct *mm = vma->vm_mm;
1678
1679         if (mm->context != NO_CONTEXT) {
1680                 cpumask_t cpu_mask;
1681                 cpumask_copy(&cpu_mask, mm_cpumask(mm));
1682                 cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
1683                 if (!cpumask_empty(&cpu_mask))
1684                         xc2((smpfunc_t) local_ops->tlb_page,
1685                             (unsigned long) vma, page);
1686                 local_ops->tlb_page(vma, page);
1687         }
1688 }
1689
1690 static void smp_flush_page_to_ram(unsigned long page)
1691 {
1692         /* Current theory is that those who call this are the one's
1693          * who have just dirtied their cache with the pages contents
1694          * in kernel space, therefore we only run this on local cpu.
1695          *
1696          * XXX This experiment failed, research further... -DaveM
1697          */
1698 #if 1
1699         xc1((smpfunc_t) local_ops->page_to_ram, page);
1700 #endif
1701         local_ops->page_to_ram(page);
1702 }
1703
1704 static void smp_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr)
1705 {
1706         cpumask_t cpu_mask;
1707         cpumask_copy(&cpu_mask, mm_cpumask(mm));
1708         cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
1709         if (!cpumask_empty(&cpu_mask))
1710                 xc2((smpfunc_t) local_ops->sig_insns,
1711                     (unsigned long) mm, insn_addr);
1712         local_ops->sig_insns(mm, insn_addr);
1713 }
1714
1715 static struct sparc32_cachetlb_ops smp_cachetlb_ops = {
1716         .cache_all      = smp_flush_cache_all,
1717         .cache_mm       = smp_flush_cache_mm,
1718         .cache_page     = smp_flush_cache_page,
1719         .cache_range    = smp_flush_cache_range,
1720         .tlb_all        = smp_flush_tlb_all,
1721         .tlb_mm         = smp_flush_tlb_mm,
1722         .tlb_page       = smp_flush_tlb_page,
1723         .tlb_range      = smp_flush_tlb_range,
1724         .page_to_ram    = smp_flush_page_to_ram,
1725         .sig_insns      = smp_flush_sig_insns,
1726         .page_for_dma   = smp_flush_page_for_dma,
1727 };
1728 #endif
1729
1730 /* Load up routines and constants for sun4m and sun4d mmu */
1731 void __init load_mmu(void)
1732 {
1733         extern void ld_mmu_iommu(void);
1734         extern void ld_mmu_iounit(void);
1735
1736         /* Functions */
1737         get_srmmu_type();
1738
1739 #ifdef CONFIG_SMP
1740         /* El switcheroo... */
1741         local_ops = sparc32_cachetlb_ops;
1742
1743         if (sparc_cpu_model == sun4d || sparc_cpu_model == sparc_leon) {
1744                 smp_cachetlb_ops.tlb_all = local_ops->tlb_all;
1745                 smp_cachetlb_ops.tlb_mm = local_ops->tlb_mm;
1746                 smp_cachetlb_ops.tlb_range = local_ops->tlb_range;
1747                 smp_cachetlb_ops.tlb_page = local_ops->tlb_page;
1748         }
1749
1750         if (poke_srmmu == poke_viking) {
1751                 /* Avoid unnecessary cross calls. */
1752                 smp_cachetlb_ops.cache_all = local_ops->cache_all;
1753                 smp_cachetlb_ops.cache_mm = local_ops->cache_mm;
1754                 smp_cachetlb_ops.cache_range = local_ops->cache_range;
1755                 smp_cachetlb_ops.cache_page = local_ops->cache_page;
1756
1757                 smp_cachetlb_ops.page_to_ram = local_ops->page_to_ram;
1758                 smp_cachetlb_ops.sig_insns = local_ops->sig_insns;
1759                 smp_cachetlb_ops.page_for_dma = local_ops->page_for_dma;
1760         }
1761
1762         /* It really is const after this point. */
1763         sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *)
1764                 &smp_cachetlb_ops;
1765 #endif
1766
1767         if (sparc_cpu_model == sun4d)
1768                 ld_mmu_iounit();
1769         else
1770                 ld_mmu_iommu();
1771 #ifdef CONFIG_SMP
1772         if (sparc_cpu_model == sun4d)
1773                 sun4d_init_smp();
1774         else if (sparc_cpu_model == sparc_leon)
1775                 leon_init_smp();
1776         else
1777                 sun4m_init_smp();
1778 #endif
1779 }