psb-kernel-source-4.41.1/psb_mmu.c

   1 /**************************************************************************
   2  * Copyright (c) 2007, Intel Corporation.
   3  * All Rights Reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Intel funded Tungsten Graphics (http://www.tungstengraphics.com) to
  19  * develop this driver.
  20  *
  21  **************************************************************************/
  22 #include "drmP.h"
  23 #include "psb_drv.h"
  24 #include "psb_reg.h"
  25
  26 /*
  27  * Code for the SGX MMU:
  28  */
  29
  30 /*
  31  * clflush on one processor only:
  32  * clflush should apparently flush the cache line on all processors in an
  33  * SMP system.
  34  */
  35
  36 /*
  37  * kmap atomic:
  38  * The usage of the slots must be completely encapsulated within a spinlock, and
  39  * no other functions that may be using the locks for other purposed may be
  40  * called from within the locked region.
  41  * Since the slots are per processor, this will guarantee that we are the only
  42  * user.
  43  */
  44
  45 /*
  46  * TODO: Inserting ptes from an interrupt handler:
  47  * This may be desirable for some SGX functionality where the GPU can fault in
  48  * needed pages. For that, we need to make an atomic insert_pages function, that
  49  * may fail.
  50  * If it fails, the caller need to insert the page using a workqueue function,
  51  * but on average it should be fast.
  52  */
  53
  54 struct psb_mmu_driver {
  55         /* protects driver- and pd structures. Always take in read mode
  56          * before taking the page table spinlock.
  57          */
  58         struct rw_semaphore sem;
  59
  60         /* protects page tables, directory tables and pt tables.
  61          * and pt structures.
  62          */
  63         spinlock_t lock;
  64
  65         atomic_t needs_tlbflush;
  66         atomic_t *msvdx_mmu_invaldc;
  67         uint8_t __iomem *register_map;
  68         struct psb_mmu_pd *default_pd;
  69         uint32_t bif_ctrl;
  70         int has_clflush;
  71         int clflush_add;
  72         unsigned long clflush_mask;
  73 };
  74
  75 struct psb_mmu_pd;
  76
  77 struct psb_mmu_pt {
  78         struct psb_mmu_pd *pd;
  79         uint32_t index;
  80         uint32_t count;
  81         struct page *p;
  82         uint32_t *v;
  83 };
  84
  85 struct psb_mmu_pd {
  86         struct psb_mmu_driver *driver;
  87         int hw_context;
  88         struct psb_mmu_pt **tables;
  89         struct page *p;
  90         struct page *dummy_pt;
  91         struct page *dummy_page;
  92         uint32_t pd_mask;
  93         uint32_t invalid_pde;
  94         uint32_t invalid_pte;
  95 };
  96
  97 static inline uint32_t psb_mmu_pt_index(uint32_t offset)
  98 {
  99         return (offset >> PSB_PTE_SHIFT) & 0x3FF;
 100 }
 101 static inline uint32_t psb_mmu_pd_index(uint32_t offset)
 102 {
 103         return (offset >> PSB_PDE_SHIFT);
 104 }
 105
 106 #if defined(CONFIG_X86)
 107 static inline void psb_clflush(void *addr)
 108 {
 109         __asm__ __volatile__("clflush (%0)\n"::"r"(addr):"memory");
 110 }
 111
 112 static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)
 113 {
 114         if (!driver->has_clflush)
 115                 return;
 116
 117         mb();
 118         psb_clflush(addr);
 119         mb();
 120 }
 121 #else
 122
 123 static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)
 124 {;
 125 }
 126
 127 #endif
 128
 129 static inline void psb_iowrite32(const struct psb_mmu_driver *d,
 130                                  uint32_t val, uint32_t offset)
 131 {
 132         iowrite32(val, d->register_map + offset);
 133 }
 134
 135 static inline uint32_t psb_ioread32(const struct psb_mmu_driver *d,
 136                                     uint32_t offset)
 137 {
 138         return ioread32(d->register_map + offset);
 139 }
 140
 141 static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver, int force)
 142 {
 143         if (atomic_read(&driver->needs_tlbflush) || force) {
 144                 uint32_t val = psb_ioread32(driver, PSB_CR_BIF_CTRL);
 145                 psb_iowrite32(driver, val | _PSB_CB_CTRL_INVALDC,
 146                               PSB_CR_BIF_CTRL);
 147                 wmb();
 148                 psb_iowrite32(driver, val & ~_PSB_CB_CTRL_INVALDC,
 149                               PSB_CR_BIF_CTRL);
 150                 (void)psb_ioread32(driver, PSB_CR_BIF_CTRL);
 151                 if (driver->msvdx_mmu_invaldc)
 152                         atomic_set(driver->msvdx_mmu_invaldc, 1);
 153         }
 154         atomic_set(&driver->needs_tlbflush, 0);
 155 }
 156
 157 static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force)
 158 {
 159         down_write(&driver->sem);
 160         psb_mmu_flush_pd_locked(driver, force);
 161         up_write(&driver->sem);
 162 }
 163
 164 void psb_mmu_flush(struct psb_mmu_driver *driver)
 165 {
 166         uint32_t val;
 167
 168         down_write(&driver->sem);
 169         val = psb_ioread32(driver, PSB_CR_BIF_CTRL);
 170         if (atomic_read(&driver->needs_tlbflush))
 171                 psb_iowrite32(driver, val | _PSB_CB_CTRL_INVALDC,
 172                               PSB_CR_BIF_CTRL);
 173         else
 174                 psb_iowrite32(driver, val | _PSB_CB_CTRL_FLUSH,
 175                               PSB_CR_BIF_CTRL);
 176         wmb();
 177         psb_iowrite32(driver,
 178                       val & ~(_PSB_CB_CTRL_FLUSH | _PSB_CB_CTRL_INVALDC),
 179                       PSB_CR_BIF_CTRL);
 180         (void)psb_ioread32(driver, PSB_CR_BIF_CTRL);
 181         atomic_set(&driver->needs_tlbflush, 0);
 182         if (driver->msvdx_mmu_invaldc)
 183                 atomic_set(driver->msvdx_mmu_invaldc, 1);
 184         up_write(&driver->sem);
 185 }
 186
 187 void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context)
 188 {
 189         uint32_t offset = (hw_context == 0) ? PSB_CR_BIF_DIR_LIST_BASE0 :
 190             PSB_CR_BIF_DIR_LIST_BASE1 + hw_context * 4;
 191
 192         drm_ttm_cache_flush();
 193         down_write(&pd->driver->sem);
 194         psb_iowrite32(pd->driver, (page_to_pfn(pd->p) << PAGE_SHIFT), offset);
 195         wmb();
 196         psb_mmu_flush_pd_locked(pd->driver, 1);
 197         pd->hw_context = hw_context;
 198         up_write(&pd->driver->sem);
 199
 200 }
 201
 202 static inline unsigned long psb_pd_addr_end(unsigned long addr,
 203                                             unsigned long end)
 204 {
 205
 206         addr = (addr + PSB_PDE_MASK + 1) & ~PSB_PDE_MASK;
 207         return (addr < end) ? addr : end;
 208 }
 209
 210 static inline uint32_t psb_mmu_mask_pte(uint32_t pfn, int type)
 211 {
 212         uint32_t mask = PSB_PTE_VALID;
 213
 214         if (type & PSB_MMU_CACHED_MEMORY)
 215                 mask |= PSB_PTE_CACHED;
 216         if (type & PSB_MMU_RO_MEMORY)
 217                 mask |= PSB_PTE_RO;
 218         if (type & PSB_MMU_WO_MEMORY)
 219                 mask |= PSB_PTE_WO;
 220
 221         return (pfn << PAGE_SHIFT) | mask;
 222 }
 223
 224 struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver,
 225                                     int trap_pagefaults, int invalid_type)
 226 {
 227         struct psb_mmu_pd *pd = kmalloc(sizeof(*pd), GFP_KERNEL);
 228         uint32_t *v;
 229         int i;
 230
 231         if (!pd)
 232                 return NULL;
 233
 234         pd->p = alloc_page(GFP_DMA32);
 235         if (!pd->p)
 236                 goto out_err1;
 237         pd->dummy_pt = alloc_page(GFP_DMA32);
 238         if (!pd->dummy_pt)
 239                 goto out_err2;
 240         pd->dummy_page = alloc_page(GFP_DMA32);
 241         if (!pd->dummy_page)
 242                 goto out_err3;
 243
 244         if (!trap_pagefaults) {
 245                 pd->invalid_pde = psb_mmu_mask_pte(page_to_pfn(pd->dummy_pt),
 246                                                    invalid_type);
 247                 pd->invalid_pte = psb_mmu_mask_pte(page_to_pfn(pd->dummy_page),
 248                                                    invalid_type);
 249         } else {
 250                 pd->invalid_pde = 0;
 251                 pd->invalid_pte = 0;
 252         }
 253
 254         v = kmap(pd->dummy_pt);
 255         for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) {
 256                 v[i] = pd->invalid_pte;
 257         }
 258         kunmap(pd->dummy_pt);
 259
 260         v = kmap(pd->p);
 261         for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) {
 262                 v[i] = pd->invalid_pde;
 263         }
 264         kunmap(pd->p);
 265
 266         clear_page(kmap(pd->dummy_page));
 267         kunmap(pd->dummy_page);
 268
 269         pd->tables = vmalloc_user(sizeof(struct psb_mmu_pt *) * 1024);
 270         if (!pd->tables)
 271                 goto out_err4;
 272
 273         pd->hw_context = -1;
 274         pd->pd_mask = PSB_PTE_VALID;
 275         pd->driver = driver;
 276
 277         return pd;
 278
 279       out_err4:
 280         __free_page(pd->dummy_page);
 281       out_err3:
 282         __free_page(pd->dummy_pt);
 283       out_err2:
 284         __free_page(pd->p);
 285       out_err1:
 286         kfree(pd);
 287         return NULL;
 288 }
 289
 290 void psb_mmu_free_pt(struct psb_mmu_pt *pt)
 291 {
 292         __free_page(pt->p);
 293         kfree(pt);
 294 }
 295
 296 void psb_mmu_free_pagedir(struct psb_mmu_pd *pd)
 297 {
 298         struct psb_mmu_driver *driver = pd->driver;
 299         struct psb_mmu_pt *pt;
 300         int i;
 301
 302         down_write(&driver->sem);
 303         if (pd->hw_context != -1) {
 304                 psb_iowrite32(driver, 0,
 305                               PSB_CR_BIF_DIR_LIST_BASE0 + pd->hw_context * 4);
 306                 psb_mmu_flush_pd_locked(driver, 1);
 307         }
 308
 309         /* Should take the spinlock here, but we don't need to do that
 310            since we have the semaphore in write mode. */
 311
 312         for (i = 0; i < 1024; ++i) {
 313                 pt = pd->tables[i];
 314                 if (pt)
 315                         psb_mmu_free_pt(pt);
 316         }
 317
 318         vfree(pd->tables);
 319         __free_page(pd->dummy_page);
 320         __free_page(pd->dummy_pt);
 321         __free_page(pd->p);
 322         kfree(pd);
 323         up_write(&driver->sem);
 324 }
 325
 326 static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd)
 327 {
 328         struct psb_mmu_pt *pt = kmalloc(sizeof(*pt), GFP_KERNEL);
 329         void *v;
 330         uint32_t clflush_add = pd->driver->clflush_add >> PAGE_SHIFT;
 331         uint32_t clflush_count = PAGE_SIZE / clflush_add;
 332         spinlock_t *lock = &pd->driver->lock;
 333         uint8_t *clf;
 334         uint32_t *ptes;
 335         int i;
 336
 337         if (!pt)
 338                 return NULL;
 339
 340         pt->p = alloc_page(GFP_DMA32);
 341         if (!pt->p) {
 342                 kfree(pt);
 343                 return NULL;
 344         }
 345
 346         spin_lock(lock);
 347
 348         v = kmap_atomic(pt->p, KM_USER0);
 349         clf = (uint8_t *) v;
 350         ptes = (uint32_t *) v;
 351         for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) {
 352                 *ptes++ = pd->invalid_pte;
 353         }
 354
 355 #if defined(CONFIG_X86)
 356         if (pd->driver->has_clflush && pd->hw_context != -1) {
 357                 mb();
 358                 for (i = 0; i < clflush_count; ++i) {
 359                         psb_clflush(clf);
 360                         clf += clflush_add;
 361                 }
 362                 mb();
 363         }
 364 #endif
 365         kunmap_atomic(v, KM_USER0);
 366         spin_unlock(lock);
 367
 368         pt->count = 0;
 369         pt->pd = pd;
 370         pt->index = 0;
 371
 372         return pt;
 373 }
 374
 375 struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd,
 376                                              unsigned long addr)
 377 {
 378         uint32_t index = psb_mmu_pd_index(addr);
 379         struct psb_mmu_pt *pt;
 380         volatile uint32_t *v;
 381         spinlock_t *lock = &pd->driver->lock;
 382
 383         spin_lock(lock);
 384         pt = pd->tables[index];
 385         while (!pt) {
 386                 spin_unlock(lock);
 387                 pt = psb_mmu_alloc_pt(pd);
 388                 if (!pt)
 389                         return NULL;
 390                 spin_lock(lock);
 391
 392                 if (pd->tables[index]) {
 393                         spin_unlock(lock);
 394                         psb_mmu_free_pt(pt);
 395                         spin_lock(lock);
 396                         pt = pd->tables[index];
 397                         continue;
 398                 }
 399
 400                 v = kmap_atomic(pd->p, KM_USER0);
 401                 pd->tables[index] = pt;
 402                 v[index] = (page_to_pfn(pt->p) << 12) | pd->pd_mask;
 403                 pt->index = index;
 404                 kunmap_atomic((void *)v, KM_USER0);
 405
 406                 if (pd->hw_context != -1) {
 407                         psb_mmu_clflush(pd->driver, (void *)&v[index]);
 408                         atomic_set(&pd->driver->needs_tlbflush, 1);
 409                 }
 410         }
 411         pt->v = kmap_atomic(pt->p, KM_USER0);
 412         return pt;
 413 }
 414
 415 static struct psb_mmu_pt *psb_mmu_pt_map_lock(struct psb_mmu_pd *pd,
 416                                               unsigned long addr)
 417 {
 418         uint32_t index = psb_mmu_pd_index(addr);
 419         struct psb_mmu_pt *pt;
 420         spinlock_t *lock = &pd->driver->lock;
 421
 422         spin_lock(lock);
 423         pt = pd->tables[index];
 424         if (!pt) {
 425                 spin_unlock(lock);
 426                 return NULL;
 427         }
 428         pt->v = kmap_atomic(pt->p, KM_USER0);
 429         return pt;
 430 }
 431
 432 static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt)
 433 {
 434         struct psb_mmu_pd *pd = pt->pd;
 435         volatile uint32_t *v;
 436
 437         kunmap_atomic(pt->v, KM_USER0);
 438         if (pt->count == 0) {
 439                 v = kmap_atomic(pd->p, KM_USER0);
 440                 v[pt->index] = pd->invalid_pde;
 441                 pd->tables[pt->index] = NULL;
 442
 443                 if (pd->hw_context != -1) {
 444                         psb_mmu_clflush(pd->driver, (void *)&v[pt->index]);
 445                         atomic_set(&pd->driver->needs_tlbflush, 1);
 446                 }
 447                 kunmap_atomic(pt->v, KM_USER0);
 448                 spin_unlock(&pd->driver->lock);
 449                 psb_mmu_free_pt(pt);
 450                 return;
 451         }
 452         spin_unlock(&pd->driver->lock);
 453 }
 454
 455 static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt, unsigned long addr,
 456                                    uint32_t pte)
 457 {
 458         pt->v[psb_mmu_pt_index(addr)] = pte;
 459 }
 460
 461 static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt *pt,
 462                                           unsigned long addr)
 463 {
 464         pt->v[psb_mmu_pt_index(addr)] = pt->pd->invalid_pte;
 465 }
 466
 467 #if 0
 468 static uint32_t psb_mmu_check_pte_locked(struct psb_mmu_pd *pd,
 469                                          uint32_t mmu_offset)
 470 {
 471         uint32_t *v;
 472         uint32_t pfn;
 473
 474         v = kmap_atomic(pd->p, KM_USER0);
 475         if (!v) {
 476                 printk(KERN_INFO "Could not kmap pde page.\n");
 477                 return 0;
 478         }
 479         pfn = v[psb_mmu_pd_index(mmu_offset)];
 480         //      printk(KERN_INFO "pde is 0x%08x\n",pfn);
 481         kunmap_atomic(v, KM_USER0);
 482         if (((pfn & 0x0F) != PSB_PTE_VALID)) {
 483                 printk(KERN_INFO "Strange pde at 0x%08x: 0x%08x.\n",
 484                        mmu_offset, pfn);
 485         }
 486         v = ioremap(pfn & 0xFFFFF000, 4096);
 487         if (!v) {
 488                 printk(KERN_INFO "Could not kmap pte page.\n");
 489                 return 0;
 490         }
 491         pfn = v[psb_mmu_pt_index(mmu_offset)];
 492         // printk(KERN_INFO "pte is 0x%08x\n",pfn);
 493         iounmap(v);
 494         if (((pfn & 0x0F) != PSB_PTE_VALID)) {
 495                 printk(KERN_INFO "Strange pte at 0x%08x: 0x%08x.\n",
 496                        mmu_offset, pfn);
 497         }
 498         return pfn >> PAGE_SHIFT;
 499 }
 500
 501 static void psb_mmu_check_mirrored_gtt(struct psb_mmu_pd *pd,
 502                                        uint32_t mmu_offset, uint32_t gtt_pages)
 503 {
 504         uint32_t start;
 505         uint32_t next;
 506
 507         printk(KERN_INFO "Checking mirrored gtt 0x%08x %d\n",
 508                mmu_offset, gtt_pages);
 509         down_read(&pd->driver->sem);
 510         start = psb_mmu_check_pte_locked(pd, mmu_offset);
 511         mmu_offset += PAGE_SIZE;
 512         gtt_pages -= 1;
 513         while (gtt_pages--) {
 514                 next = psb_mmu_check_pte_locked(pd, mmu_offset);
 515                 if (next != start + 1) {
 516                         printk(KERN_INFO "Ptes out of order: 0x%08x, 0x%08x.\n",
 517                                start, next);
 518                 }
 519                 start = next;
 520                 mmu_offset += PAGE_SIZE;
 521         }
 522         up_read(&pd->driver->sem);
 523 }
 524
 525 #endif
 526
 527 void psb_mmu_mirror_gtt(struct psb_mmu_pd *pd,
 528                         uint32_t mmu_offset, uint32_t gtt_start,
 529                         uint32_t gtt_pages)
 530 {
 531         uint32_t *v;
 532         uint32_t start = psb_mmu_pd_index(mmu_offset);
 533         struct psb_mmu_driver *driver = pd->driver;
 534
 535         down_read(&driver->sem);
 536         spin_lock(&driver->lock);
 537
 538         v = kmap_atomic(pd->p, KM_USER0);
 539         v += start;
 540
 541         while (gtt_pages--) {
 542                 *v++ = gtt_start | pd->pd_mask;
 543                 gtt_start += PAGE_SIZE;
 544         }
 545
 546         drm_ttm_cache_flush();
 547         kunmap_atomic(v, KM_USER0);
 548         spin_unlock(&driver->lock);
 549
 550         if (pd->hw_context != -1)
 551                 atomic_set(&pd->driver->needs_tlbflush, 1);
 552
 553         up_read(&pd->driver->sem);
 554         psb_mmu_flush_pd(pd->driver, 0);
 555 }
 556
 557 struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver)
 558 {
 559         struct psb_mmu_pd *pd;
 560
 561         down_read(&driver->sem);
 562         pd = driver->default_pd;
 563         up_read(&driver->sem);
 564
 565         return pd;
 566 }
 567
 568 /* Returns the physical address of the PD shared by sgx/msvdx */
 569 uint32_t psb_get_default_pd_addr(struct psb_mmu_driver * driver)
 570 {
 571         struct psb_mmu_pd *pd;
 572
 573         pd = psb_mmu_get_default_pd(driver);
 574         return ((page_to_pfn(pd->p) << PAGE_SHIFT));
 575 }
 576
 577 void psb_mmu_driver_takedown(struct psb_mmu_driver *driver)
 578 {
 579         psb_iowrite32(driver, driver->bif_ctrl, PSB_CR_BIF_CTRL);
 580         psb_mmu_free_pagedir(driver->default_pd);
 581         kfree(driver);
 582 }
 583
 584 struct psb_mmu_driver *psb_mmu_driver_init(uint8_t __iomem * registers,
 585                                            int trap_pagefaults,
 586                                            int invalid_type,
 587                                            atomic_t *msvdx_mmu_invaldc)
 588 {
 589         struct psb_mmu_driver *driver;
 590
 591         driver = (struct psb_mmu_driver *)kmalloc(sizeof(*driver), GFP_KERNEL);
 592
 593         if (!driver)
 594                 return NULL;
 595
 596         driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults,
 597                                               invalid_type);
 598         if (!driver->default_pd)
 599                 goto out_err1;
 600
 601         spin_lock_init(&driver->lock);
 602         init_rwsem(&driver->sem);
 603         down_write(&driver->sem);
 604         driver->register_map = registers;
 605         atomic_set(&driver->needs_tlbflush, 1);
 606         driver->msvdx_mmu_invaldc = msvdx_mmu_invaldc;
 607
 608         driver->bif_ctrl = psb_ioread32(driver, PSB_CR_BIF_CTRL);
 609         psb_iowrite32(driver, driver->bif_ctrl | _PSB_CB_CTRL_CLEAR_FAULT,
 610                       PSB_CR_BIF_CTRL);
 611         psb_iowrite32(driver, driver->bif_ctrl & ~_PSB_CB_CTRL_CLEAR_FAULT,
 612                       PSB_CR_BIF_CTRL);
 613
 614         driver->has_clflush = 0;
 615
 616 #if defined(CONFIG_X86)
 617         if (boot_cpu_has(X86_FEATURE_CLFLSH)) {
 618                 uint32_t tfms, misc, cap0, cap4, clflush_size;
 619
 620                 /*
 621                  * clflush size is determined at kernel setup for x86_64 but not for
 622                  * i386. We have to do it here.
 623                  */
 624
 625                 cpuid(0x00000001, &tfms, &misc, &cap0, &cap4);
 626                 clflush_size = ((misc >> 8) & 0xff) * 8;
 627                 driver->has_clflush = 1;
 628                 driver->clflush_add =
 629                     PAGE_SIZE * clflush_size / sizeof(uint32_t);
 630                 driver->clflush_mask = driver->clflush_add - 1;
 631                 driver->clflush_mask = ~driver->clflush_mask;
 632         }
 633 #endif
 634
 635         up_write(&driver->sem);
 636         return driver;
 637
 638       out_err1:
 639         kfree(driver);
 640         return NULL;
 641 }
 642
 643 #if defined(CONFIG_X86)
 644 static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address,
 645                                uint32_t num_pages, uint32_t desired_tile_stride,
 646                                uint32_t hw_tile_stride)
 647 {
 648         struct psb_mmu_pt *pt;
 649         uint32_t rows = 1;
 650         uint32_t i;
 651         unsigned long addr;
 652         unsigned long end;
 653         unsigned long next;
 654         unsigned long add;
 655         unsigned long row_add;
 656         unsigned long clflush_add = pd->driver->clflush_add;
 657         unsigned long clflush_mask = pd->driver->clflush_mask;
 658
 659         if (!pd->driver->has_clflush) {
 660                 drm_ttm_cache_flush();
 661                 return;
 662         }
 663
 664         if (hw_tile_stride)
 665                 rows = num_pages / desired_tile_stride;
 666         else
 667                 desired_tile_stride = num_pages;
 668
 669         add = desired_tile_stride << PAGE_SHIFT;
 670         row_add = hw_tile_stride << PAGE_SHIFT;
 671         mb();
 672         for (i = 0; i < rows; ++i) {
 673
 674                 addr = address;
 675                 end = addr + add;
 676
 677                 do {
 678                         next = psb_pd_addr_end(addr, end);
 679                         pt = psb_mmu_pt_map_lock(pd, addr);
 680                         if (!pt)
 681                                 continue;
 682                         do {
 683                                 psb_clflush(&pt->v[psb_mmu_pt_index(addr)]);
 684                         } while (addr += clflush_add,
 685                                  (addr & clflush_mask) < next);
 686
 687                         psb_mmu_pt_unmap_unlock(pt);
 688                 } while (addr = next, next != end);
 689                 address += row_add;
 690         }
 691         mb();
 692 }
 693 #else
 694 static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address,
 695                                uint32_t num_pages, uint32_t desired_tile_stride,
 696                                uint32_t hw_tile_stride)
 697 {
 698         drm_ttm_cache_flush();
 699 }
 700 #endif
 701
 702 void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd,
 703                                  unsigned long address, uint32_t num_pages)
 704 {
 705         struct psb_mmu_pt *pt;
 706         unsigned long addr;
 707         unsigned long end;
 708         unsigned long next;
 709         unsigned long f_address = address;
 710
 711         down_read(&pd->driver->sem);
 712
 713         addr = address;
 714         end = addr + (num_pages << PAGE_SHIFT);
 715
 716         do {
 717                 next = psb_pd_addr_end(addr, end);
 718                 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
 719                 if (!pt)
 720                         goto out;
 721                 do {
 722                         psb_mmu_invalidate_pte(pt, addr);
 723                         --pt->count;
 724                 } while (addr += PAGE_SIZE, addr < next);
 725                 psb_mmu_pt_unmap_unlock(pt);
 726
 727         } while (addr = next, next != end);
 728
 729       out:
 730         if (pd->hw_context != -1)
 731                 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);
 732
 733         up_read(&pd->driver->sem);
 734
 735         if (pd->hw_context != -1)
 736                 psb_mmu_flush(pd->driver);
 737
 738         return;
 739 }
 740
 741 void psb_mmu_remove_pages(struct psb_mmu_pd *pd, unsigned long address,
 742                           uint32_t num_pages, uint32_t desired_tile_stride,
 743                           uint32_t hw_tile_stride)
 744 {
 745         struct psb_mmu_pt *pt;
 746         uint32_t rows = 1;
 747         uint32_t i;
 748         unsigned long addr;
 749         unsigned long end;
 750         unsigned long next;
 751         unsigned long add;
 752         unsigned long row_add;
 753         unsigned long f_address = address;
 754
 755         if (hw_tile_stride)
 756                 rows = num_pages / desired_tile_stride;
 757         else
 758                 desired_tile_stride = num_pages;
 759
 760         add = desired_tile_stride << PAGE_SHIFT;
 761         row_add = hw_tile_stride << PAGE_SHIFT;
 762
 763         down_read(&pd->driver->sem);
 764
 765         /* Make sure we only need to flush this processor's cache */
 766
 767         for (i = 0; i < rows; ++i) {
 768
 769                 addr = address;
 770                 end = addr + add;
 771
 772                 do {
 773                         next = psb_pd_addr_end(addr, end);
 774                         pt = psb_mmu_pt_map_lock(pd, addr);
 775                         if (!pt)
 776                                 continue;
 777                         do {
 778                                 psb_mmu_invalidate_pte(pt, addr);
 779                                 --pt->count;
 780
 781                         } while (addr += PAGE_SIZE, addr < next);
 782                         psb_mmu_pt_unmap_unlock(pt);
 783
 784                 } while (addr = next, next != end);
 785                 address += row_add;
 786         }
 787         if (pd->hw_context != -1)
 788                 psb_mmu_flush_ptes(pd, f_address, num_pages,
 789                                    desired_tile_stride, hw_tile_stride);
 790
 791         up_read(&pd->driver->sem);
 792
 793         if (pd->hw_context != -1)
 794                 psb_mmu_flush(pd->driver);
 795 }
 796
 797 int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn,
 798                                 unsigned long address, uint32_t num_pages,
 799                                 int type)
 800 {
 801         struct psb_mmu_pt *pt;
 802         uint32_t pte;
 803         unsigned long addr;
 804         unsigned long end;
 805         unsigned long next;
 806         unsigned long f_address = address;
 807         int ret = -ENOMEM;
 808
 809         down_read(&pd->driver->sem);
 810
 811         addr = address;
 812         end = addr + (num_pages << PAGE_SHIFT);
 813
 814         do {
 815                 next = psb_pd_addr_end(addr, end);
 816                 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
 817                 if (!pt) {
 818                         ret = -ENOMEM;
 819                         goto out;
 820                 }
 821                 do {
 822                         pte = psb_mmu_mask_pte(start_pfn++, type);
 823                         psb_mmu_set_pte(pt, addr, pte);
 824                         pt->count++;
 825                 } while (addr += PAGE_SIZE, addr < next);
 826                 psb_mmu_pt_unmap_unlock(pt);
 827
 828         } while (addr = next, next != end);
 829         ret = 0;
 830
 831       out:
 832         if (pd->hw_context != -1)
 833                 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);
 834
 835         up_read(&pd->driver->sem);
 836
 837         if (pd->hw_context != -1)
 838                 psb_mmu_flush(pd->driver);
 839
 840         return 0;
 841 }
 842
 843 int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages,
 844                          unsigned long address, uint32_t num_pages,
 845                          uint32_t desired_tile_stride, uint32_t hw_tile_stride,
 846                          int type)
 847 {
 848         struct psb_mmu_pt *pt;
 849         uint32_t rows = 1;
 850         uint32_t i;
 851         uint32_t pte;
 852         unsigned long addr;
 853         unsigned long end;
 854         unsigned long next;
 855         unsigned long add;
 856         unsigned long row_add;
 857         unsigned long f_address = address;
 858         int ret = -ENOMEM;
 859
 860         if (hw_tile_stride) {
 861                 if (num_pages % desired_tile_stride != 0)
 862                         return -EINVAL;
 863                 rows = num_pages / desired_tile_stride;
 864         } else {
 865                 desired_tile_stride = num_pages;
 866         }
 867
 868         add = desired_tile_stride << PAGE_SHIFT;
 869         row_add = hw_tile_stride << PAGE_SHIFT;
 870
 871         down_read(&pd->driver->sem);
 872
 873         for (i = 0; i < rows; ++i) {
 874
 875                 addr = address;
 876                 end = addr + add;
 877
 878                 do {
 879                         next = psb_pd_addr_end(addr, end);
 880                         pt = psb_mmu_pt_alloc_map_lock(pd, addr);
 881                         if (!pt)
 882                                 goto out;
 883                         do {
 884                                 pte = psb_mmu_mask_pte(page_to_pfn(*pages++),
 885                                                        type);
 886                                 psb_mmu_set_pte(pt, addr, pte);
 887                                 pt->count++;
 888                         } while (addr += PAGE_SIZE, addr < next);
 889                         psb_mmu_pt_unmap_unlock(pt);
 890
 891                 } while (addr = next, next != end);
 892
 893                 address += row_add;
 894         }
 895         ret = 0;
 896       out:
 897         if (pd->hw_context != -1)
 898                 psb_mmu_flush_ptes(pd, f_address, num_pages,
 899                                    desired_tile_stride, hw_tile_stride);
 900
 901         up_read(&pd->driver->sem);
 902
 903         if (pd->hw_context != -1)
 904                 psb_mmu_flush(pd->driver);
 905
 906         return 0;
 907 }
 908
 909 void psb_mmu_enable_requestor(struct psb_mmu_driver *driver, uint32_t mask)
 910 {
 911         mask &= _PSB_MMU_ER_MASK;
 912         psb_iowrite32(driver, psb_ioread32(driver, PSB_CR_BIF_CTRL) & ~mask,
 913                       PSB_CR_BIF_CTRL);
 914         (void)psb_ioread32(driver, PSB_CR_BIF_CTRL);
 915 }
 916
 917 void psb_mmu_disable_requestor(struct psb_mmu_driver *driver, uint32_t mask)
 918 {
 919         mask &= _PSB_MMU_ER_MASK;
 920         psb_iowrite32(driver, psb_ioread32(driver, PSB_CR_BIF_CTRL) | mask,
 921                       PSB_CR_BIF_CTRL);
 922         (void)psb_ioread32(driver, PSB_CR_BIF_CTRL);
 923 }
 924
 925 int psb_mmu_virtual_to_pfn(struct psb_mmu_pd *pd, uint32_t virtual,
 926                            unsigned long *pfn)
 927 {
 928         int ret;
 929         struct psb_mmu_pt *pt;
 930         uint32_t tmp;
 931         spinlock_t *lock = &pd->driver->lock;
 932
 933         down_read(&pd->driver->sem);
 934         pt = psb_mmu_pt_map_lock(pd, virtual);
 935         if (!pt) {
 936                 uint32_t *v;
 937
 938                 spin_lock(lock);
 939                 v = kmap_atomic(pd->p, KM_USER0);
 940                 tmp = v[psb_mmu_pd_index(virtual)];
 941                 kunmap_atomic(v, KM_USER0);
 942                 spin_unlock(lock);
 943
 944                 if (tmp != pd->invalid_pde || !(tmp & PSB_PTE_VALID) ||
 945                     !(pd->invalid_pte & PSB_PTE_VALID)) {
 946                         ret = -EINVAL;
 947                         goto out;
 948                 }
 949                 ret = 0;
 950                 *pfn = pd->invalid_pte >> PAGE_SHIFT;
 951                 goto out;
 952         }
 953         tmp = pt->v[psb_mmu_pt_index(virtual)];
 954         if (!(tmp & PSB_PTE_VALID)) {
 955                 ret = -EINVAL;
 956         } else {
 957                 ret = 0;
 958                 *pfn = tmp >> PAGE_SHIFT;
 959         }
 960         psb_mmu_pt_unmap_unlock(pt);
 961       out:
 962         up_read(&pd->driver->sem);
 963         return ret;
 964 }
 965
 966 void psb_mmu_test(struct psb_mmu_driver *driver, uint32_t offset)
 967 {
 968         struct page *p;
 969         unsigned long pfn;
 970         int ret = 0;
 971         struct psb_mmu_pd *pd;
 972         uint32_t *v;
 973         uint32_t *vmmu;
 974
 975         pd = driver->default_pd;
 976         if (!pd) {
 977                 printk(KERN_WARNING "Could not get default pd\n");
 978         }
 979
 980         p = alloc_page(GFP_DMA32);
 981
 982         if (!p) {
 983                 printk(KERN_WARNING "Failed allocating page\n");
 984                 return;
 985         }
 986
 987         v = kmap(p);
 988         memset(v, 0x67, PAGE_SIZE);
 989
 990         pfn = (offset >> PAGE_SHIFT);
 991
 992         ret = psb_mmu_insert_pages(pd, &p, pfn << PAGE_SHIFT, 1, 0, 0, 0);
 993         if (ret) {
 994                 printk(KERN_WARNING "Failed inserting mmu page\n");
 995                 goto out_err1;
 996         }
 997
 998         /* Ioremap the page through the GART aperture */
 999
1000         vmmu = ioremap(pfn << PAGE_SHIFT, PAGE_SIZE);
1001         if (!vmmu) {
1002                 printk(KERN_WARNING "Failed ioremapping page\n");
1003                 goto out_err2;
1004         }
1005
1006         /* Read from the page with mmu disabled. */
1007         printk(KERN_INFO "Page first dword is 0x%08x\n", ioread32(vmmu));
1008
1009         /* Enable the mmu for host accesses and read again. */
1010         psb_mmu_enable_requestor(driver, _PSB_MMU_ER_HOST);
1011
1012         printk(KERN_INFO "MMU Page first dword is (0x67676767) 0x%08x\n",
1013                ioread32(vmmu));
1014         *v = 0x15243705;
1015         printk(KERN_INFO "MMU Page new dword is (0x15243705) 0x%08x\n",
1016                ioread32(vmmu));
1017         iowrite32(0x16243355, vmmu);
1018         (void)ioread32(vmmu);
1019         printk(KERN_INFO "Page new dword is (0x16243355) 0x%08x\n", *v);
1020
1021         printk(KERN_INFO "Int stat is 0x%08x\n",
1022                psb_ioread32(driver, PSB_CR_BIF_INT_STAT));
1023         printk(KERN_INFO "Fault is 0x%08x\n",
1024                psb_ioread32(driver, PSB_CR_BIF_FAULT));
1025
1026         /* Disable MMU for host accesses and clear page fault register */
1027         psb_mmu_disable_requestor(driver, _PSB_MMU_ER_HOST);
1028         iounmap(vmmu);
1029       out_err2:
1030         psb_mmu_remove_pages(pd, pfn << PAGE_SHIFT, 1, 0, 0);
1031       out_err1:
1032         kunmap(p);
1033         __free_page(p);
1034 }