51 	/* protects driver- and pd structures. Always take in read mode
84 	struct psb_mmu_driver *driver;  member
110 static inline void psb_mmu_clflush(struct psb_mmu_driver *driver,  in psb_mmu_clflush()  argument
113 	if (!driver->has_clflush)  in psb_mmu_clflush()
121 static void psb_page_clflush(struct psb_mmu_driver *driver, struct page* page)  in psb_page_clflush()  argument
123 	uint32_t clflush_add = driver->clflush_add >> PAGE_SHIFT;  in psb_page_clflush()
138 static void psb_pages_clflush(struct psb_mmu_driver *driver,  in psb_pages_clflush()  argument
143 	if (!driver->has_clflush)  in psb_pages_clflush()
147 		psb_page_clflush(driver, *page++);  in psb_pages_clflush()
150 static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver,  in psb_mmu_flush_pd_locked()  argument
153 	atomic_set(&driver->needs_tlbflush, 0);  in psb_mmu_flush_pd_locked()
156 static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force)  in psb_mmu_flush_pd()  argument
158 	down_write(&driver->sem);  in psb_mmu_flush_pd()
159 	psb_mmu_flush_pd_locked(driver, force);  in psb_mmu_flush_pd()
160 	up_write(&driver->sem);  in psb_mmu_flush_pd()
163 void psb_mmu_flush(struct psb_mmu_driver *driver, int rc_prot)  in psb_mmu_flush()  argument
166 		down_write(&driver->sem);  in psb_mmu_flush()
168 		up_write(&driver->sem);  in psb_mmu_flush()
174 	psb_pages_clflush(pd->driver, &pd->p, 1);  in psb_mmu_set_pd_context()
175 	down_write(&pd->driver->sem);  in psb_mmu_set_pd_context()
177 	psb_mmu_flush_pd_locked(pd->driver, 1);  in psb_mmu_set_pd_context()
179 	up_write(&pd->driver->sem);  in psb_mmu_set_pd_context()
205 struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver,  in psb_mmu_alloc_pd()  argument
258 	pd->driver = driver;  in psb_mmu_alloc_pd()
281 	struct psb_mmu_driver *driver = pd->driver;  in psb_mmu_free_pagedir()  local
285 	down_write(&driver->sem);  in psb_mmu_free_pagedir()
287 		psb_mmu_flush_pd_locked(driver, 1);  in psb_mmu_free_pagedir()
303 	up_write(&driver->sem);  in psb_mmu_free_pagedir()
310 	uint32_t clflush_add = pd->driver->clflush_add >> PAGE_SHIFT;  in psb_mmu_alloc_pt()
312 	spinlock_t *lock = &pd->driver->lock;  in psb_mmu_alloc_pt()
335 	if (pd->driver->has_clflush && pd->hw_context != -1) {  in psb_mmu_alloc_pt()
360 	spinlock_t *lock = &pd->driver->lock;  in psb_mmu_pt_alloc_map_lock()
386 			psb_mmu_clflush(pd->driver, (void *) &v[index]);  in psb_mmu_pt_alloc_map_lock()
387 			atomic_set(&pd->driver->needs_tlbflush, 1);  in psb_mmu_pt_alloc_map_lock()
399 	spinlock_t *lock = &pd->driver->lock;  in psb_mmu_pt_map_lock()
423 			psb_mmu_clflush(pd->driver,  in psb_mmu_pt_unmap_unlock()
425 			atomic_set(&pd->driver->needs_tlbflush, 1);  in psb_mmu_pt_unmap_unlock()
428 		spin_unlock(&pd->driver->lock);  in psb_mmu_pt_unmap_unlock()
432 	spin_unlock(&pd->driver->lock);  in psb_mmu_pt_unmap_unlock()
454 	struct psb_mmu_driver *driver = pd->driver;  in psb_mmu_mirror_gtt()  local
457 	down_read(&driver->sem);  in psb_mmu_mirror_gtt()
458 	spin_lock(&driver->lock);  in psb_mmu_mirror_gtt()
469 	psb_pages_clflush(pd->driver, &pd->p, num_pages);  in psb_mmu_mirror_gtt()
471 	spin_unlock(&driver->lock);  in psb_mmu_mirror_gtt()
474 		atomic_set(&pd->driver->needs_tlbflush, 1);  in psb_mmu_mirror_gtt()
476 	up_read(&pd->driver->sem);  in psb_mmu_mirror_gtt()
477 	psb_mmu_flush_pd(pd->driver, 0);  in psb_mmu_mirror_gtt()
480 struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver)  in psb_mmu_get_default_pd()  argument
484 	/* down_read(&driver->sem); */  in psb_mmu_get_default_pd()
485 	pd = driver->default_pd;  in psb_mmu_get_default_pd()
486 	/* up_read(&driver->sem); */  in psb_mmu_get_default_pd()
492 uint32_t psb_get_default_pd_addr(struct psb_mmu_driver *driver)  in psb_get_default_pd_addr()  argument
496 	pd = psb_mmu_get_default_pd(driver);  in psb_get_default_pd_addr()
500 void psb_mmu_driver_takedown(struct psb_mmu_driver *driver)  in psb_mmu_driver_takedown()  argument
502 	psb_mmu_free_pagedir(driver->default_pd);  in psb_mmu_driver_takedown()
503 	kfree(driver);  in psb_mmu_driver_takedown()
511 	struct psb_mmu_driver *driver;  in psb_mmu_driver_init()  local
513 	driver = kmalloc(sizeof(*driver), GFP_KERNEL);  in psb_mmu_driver_init()
515 	if (!driver)  in psb_mmu_driver_init()
517 	driver->dev_priv = dev_priv;  in psb_mmu_driver_init()
519 	driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults,  in psb_mmu_driver_init()
521 	if (!driver->default_pd)  in psb_mmu_driver_init()
524 	spin_lock_init(&driver->lock);  in psb_mmu_driver_init()
525 	init_rwsem(&driver->sem);  in psb_mmu_driver_init()
526 	down_write(&driver->sem);  in psb_mmu_driver_init()
527 	driver->register_map = registers;  in psb_mmu_driver_init()
528 	atomic_set(&driver->needs_tlbflush, 1);  in psb_mmu_driver_init()
530 	driver->has_clflush = 0;  in psb_mmu_driver_init()
542 		driver->has_clflush = 1;  in psb_mmu_driver_init()
543 		driver->clflush_add =  in psb_mmu_driver_init()
545 		driver->clflush_mask = driver->clflush_add - 1;  in psb_mmu_driver_init()
546 		driver->clflush_mask = ~driver->clflush_mask;  in psb_mmu_driver_init()
549 	up_write(&driver->sem);  in psb_mmu_driver_init()
550 	return driver;  in psb_mmu_driver_init()
553 	kfree(driver);  in psb_mmu_driver_init()
570 	unsigned long clflush_add = pd->driver->clflush_add;  in psb_mmu_flush_ptes()
571 	unsigned long clflush_mask = pd->driver->clflush_mask;  in psb_mmu_flush_ptes()
573 	if (!pd->driver->has_clflush) {  in psb_mmu_flush_ptes()
575 		psb_pages_clflush(pd->driver, &pd->p, num_pages);  in psb_mmu_flush_ptes()
620 	down_read(&pd->driver->sem);  in psb_mmu_remove_pfn_sequence()
642 	up_read(&pd->driver->sem);  in psb_mmu_remove_pfn_sequence()
645 		psb_mmu_flush(pd->driver, 0);  in psb_mmu_remove_pfn_sequence()
672 	/* down_read(&pd->driver->sem); */  in psb_mmu_remove_pages()
700 	/* up_read(&pd->driver->sem); */  in psb_mmu_remove_pages()
703 		psb_mmu_flush(pd->driver, 0);  in psb_mmu_remove_pages()
718 	down_read(&pd->driver->sem);  in psb_mmu_insert_pfn_sequence()
743 	up_read(&pd->driver->sem);  in psb_mmu_insert_pfn_sequence()
746 		psb_mmu_flush(pd->driver, 1);  in psb_mmu_insert_pfn_sequence()
779 	down_read(&pd->driver->sem);  in psb_mmu_insert_pages()
811 	up_read(&pd->driver->sem);  in psb_mmu_insert_pages()
814 		psb_mmu_flush(pd->driver, 1);  in psb_mmu_insert_pages()
825 	spinlock_t *lock = &pd->driver->lock;  in psb_mmu_virtual_to_pfn()
827 	down_read(&pd->driver->sem);  in psb_mmu_virtual_to_pfn()
856 	up_read(&pd->driver->sem);  in psb_mmu_virtual_to_pfn()