1 files changed, 410 insertions, 0 deletions
diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_alloc_carveout.c b/drivers/gpu/arm/midgard/mali_kbase_mem_alloc_carveout.c
new file mode 100755
index 000000000000..8fa93b914302
--- /dev/null
+++ b/drivers/gpu/arm/midgard/mali_kbase_mem_alloc_carveout.c
@@ -0,0 +1,410 @@
+/*
+ *
+ * (C) COPYRIGHT ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA  02110-1301, USA.
+ *
+ */
+
+
+
+/**
+ * @file mali_kbase_mem.c
+ * Base kernel memory APIs
+ */
+#include <mali_kbase.h>
+#include <linux/dma-mapping.h>
+#include <linux/highmem.h>
+#include <linux/mempool.h>
+#include <linux/mm.h>
+#include <linux/atomic.h>
+#include <linux/debugfs.h>
+#include <linux/memblock.h>
+#include <linux/seq_file.h>
+#include <linux/version.h>
+
+
+/* This code does not support having multiple kbase devices, or rmmod/insmod */
+
+static unsigned long kbase_carveout_start_pfn = ~0UL;
+static unsigned long kbase_carveout_end_pfn;
+static LIST_HEAD(kbase_carveout_free_list);
+static DEFINE_MUTEX(kbase_carveout_free_list_lock);
+static unsigned int kbase_carveout_pages;
+static atomic_t kbase_carveout_used_pages;
+static atomic_t kbase_carveout_system_pages;
+
+static struct page *kbase_carveout_get_page(struct kbase_mem_allocator *allocator)
+{
+	struct page *p = NULL;
+	gfp_t gfp;
+
+	mutex_lock(&kbase_carveout_free_list_lock);
+	if (!list_empty(&kbase_carveout_free_list)) {
+		p = list_first_entry(&kbase_carveout_free_list, struct page, lru);
+		list_del(&p->lru);
+		atomic_inc(&kbase_carveout_used_pages);
+	}
+	mutex_unlock(&kbase_carveout_free_list_lock);
+
+	if (!p) {
+		dma_addr_t dma_addr;
+#if defined(CONFIG_ARM) && !defined(CONFIG_HAVE_DMA_ATTRS) && LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0)
+		/* DMA cache sync fails for HIGHMEM before 3.5 on ARM */
+		gfp = GFP_USER;
+#else
+		gfp = GFP_HIGHUSER;
+#endif
+
+		if (current->flags & PF_KTHREAD) {
+			/* Don't trigger OOM killer from kernel threads, e.g.
+			 * when growing memory on GPU page fault */
+			gfp |= __GFP_NORETRY;
+		}
+
+		p = alloc_page(gfp);
+		if (!p)
+			goto out;
+
+		dma_addr = dma_map_page(allocator->kbdev->dev, p, 0, PAGE_SIZE,
+				DMA_BIDIRECTIONAL);
+		if (dma_mapping_error(allocator->kbdev->dev, dma_addr)) {
+			__free_page(p);
+			p = NULL;
+			goto out;
+		}
+
+		kbase_set_dma_addr(p, dma_addr);
+		BUG_ON(dma_addr != PFN_PHYS(page_to_pfn(p)));
+		atomic_inc(&kbase_carveout_system_pages);
+	}
+out:
+	return p;
+}
+
+static void kbase_carveout_put_page(struct page *p,
+				    struct kbase_mem_allocator *allocator)
+{
+	if (page_to_pfn(p) >= kbase_carveout_start_pfn &&
+			page_to_pfn(p) <= kbase_carveout_end_pfn) {
+		mutex_lock(&kbase_carveout_free_list_lock);
+		list_add(&p->lru, &kbase_carveout_free_list);
+		atomic_dec(&kbase_carveout_used_pages);
+		mutex_unlock(&kbase_carveout_free_list_lock);
+	} else {
+		dma_unmap_page(allocator->kbdev->dev, kbase_dma_addr(p),
+				PAGE_SIZE,
+				DMA_BIDIRECTIONAL);
+		ClearPagePrivate(p);
+		__free_page(p);
+		atomic_dec(&kbase_carveout_system_pages);
+	}
+}
+
+static int kbase_carveout_seq_show(struct seq_file *s, void *data)
+{
+	seq_printf(s, "carveout pages: %u\n", kbase_carveout_pages);
+	seq_printf(s, "used carveout pages: %u\n",
+			atomic_read(&kbase_carveout_used_pages));
+	seq_printf(s, "used system pages: %u\n",
+			atomic_read(&kbase_carveout_system_pages));
+	return 0;
+}
+
+static int kbasep_carveout_debugfs_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, kbase_carveout_seq_show, NULL);
+}
+
+static const struct file_operations kbase_carveout_debugfs_fops = {
+	.open           = kbasep_carveout_debugfs_open,
+	.read           = seq_read,
+	.llseek         = seq_lseek,
+	.release        = seq_release_private,
+};
+
+static int kbase_carveout_init(struct device *dev)
+{
+	unsigned long pfn;
+	static int once;
+
+	mutex_lock(&kbase_carveout_free_list_lock);
+	BUG_ON(once);
+	once = 1;
+
+	for (pfn = kbase_carveout_start_pfn; pfn <= kbase_carveout_end_pfn; pfn++) {
+		struct page *p = pfn_to_page(pfn);
+		dma_addr_t dma_addr;
+
+		dma_addr = dma_map_page(dev, p, 0, PAGE_SIZE,
+				DMA_BIDIRECTIONAL);
+		if (dma_mapping_error(dev, dma_addr))
+			goto out_rollback;
+
+		kbase_set_dma_addr(p, dma_addr);
+		BUG_ON(dma_addr != PFN_PHYS(page_to_pfn(p)));
+
+		list_add_tail(&p->lru, &kbase_carveout_free_list);
+	}
+
+	mutex_unlock(&kbase_carveout_free_list_lock);
+
+	debugfs_create_file("kbase_carveout", S_IRUGO, NULL, NULL,
+		    &kbase_carveout_debugfs_fops);
+
+	return 0;
+
+out_rollback:
+	while (!list_empty(&kbase_carveout_free_list)) {
+		struct page *p;
+
+		p = list_first_entry(&kbase_carveout_free_list, struct page, lru);
+		dma_unmap_page(dev, kbase_dma_addr(p),
+				PAGE_SIZE,
+				DMA_BIDIRECTIONAL);
+		ClearPagePrivate(p);
+		list_del(&p->lru);
+	}
+
+	mutex_unlock(&kbase_carveout_free_list_lock);
+	return -ENOMEM;
+}
+
+int __init kbase_carveout_mem_reserve(phys_addr_t size)
+{
+	phys_addr_t mem;
+
+#if defined(CONFIG_ARM) && LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0)
+	/* DMA cache sync fails for HIGHMEM before 3.5 on ARM */
+	mem = memblock_alloc_base(size, PAGE_SIZE, MEMBLOCK_ALLOC_ACCESSIBLE);
+#else
+	mem = memblock_alloc_base(size, PAGE_SIZE, MEMBLOCK_ALLOC_ANYWHERE);
+#endif
+	if (mem == 0) {
+		pr_warn("%s: Failed to allocate %d for kbase carveout\n",
+				__func__, size);
+		return -ENOMEM;
+	}
+
+	kbase_carveout_start_pfn = PFN_DOWN(mem);
+	kbase_carveout_end_pfn = PFN_DOWN(mem + size - 1);
+	kbase_carveout_pages = kbase_carveout_end_pfn - kbase_carveout_start_pfn + 1;
+
+	return 0;
+}
+
+int kbase_mem_lowlevel_init(struct kbase_device *kbdev)
+{
+	return kbase_carveout_init(kbdev->dev);
+}
+
+void kbase_mem_lowlevel_term(struct kbase_device *kbdev)
+{
+}
+
+STATIC int kbase_mem_allocator_shrink(struct shrinker *s, struct shrink_control *sc)
+{
+	struct kbase_mem_allocator *allocator;
+	int i;
+	int freed;
+
+	allocator = container_of(s, struct kbase_mem_allocator, free_list_reclaimer);
+
+	if (sc->nr_to_scan == 0)
+		return atomic_read(&allocator->free_list_size);
+
+	might_sleep();
+
+	mutex_lock(&allocator->free_list_lock);
+	i = MIN(atomic_read(&allocator->free_list_size), sc->nr_to_scan);
+	freed = i;
+
+	atomic_sub(i, &allocator->free_list_size);
+
+	while (i--) {
+		struct page *p;
+
+		BUG_ON(list_empty(&allocator->free_list_head));
+		p = list_first_entry(&allocator->free_list_head, struct page, lru);
+		list_del(&p->lru);
+		kbase_carveout_put_page(p, allocator);
+	}
+	mutex_unlock(&allocator->free_list_lock);
+	return atomic_read(&allocator->free_list_size);
+}
+
+mali_error kbase_mem_allocator_init(struct kbase_mem_allocator * const allocator,
+		unsigned int max_size, struct kbase_device *kbdev)
+{
+	KBASE_DEBUG_ASSERT(NULL != allocator);
+	KBASE_DEBUG_ASSERT(kbdev);
+
+	INIT_LIST_HEAD(&allocator->free_list_head);
+
+	allocator->kbdev = kbdev;
+
+	mutex_init(&allocator->free_list_lock);
+
+	atomic_set(&allocator->free_list_size, 0);
+
+	allocator->free_list_max_size = max_size;
+	allocator->free_list_reclaimer.shrink = kbase_mem_allocator_shrink;
+	allocator->free_list_reclaimer.seeks = DEFAULT_SEEKS;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0) /* Kernel versions prior to 3.1 : struct shrinker does not define batch */
+	allocator->free_list_reclaimer.batch = 0;
+#endif
+
+	register_shrinker(&allocator->free_list_reclaimer);
+
+	return MALI_ERROR_NONE;
+}
+
+void kbase_mem_allocator_term(struct kbase_mem_allocator *allocator)
+{
+	KBASE_DEBUG_ASSERT(NULL != allocator);
+
+	unregister_shrinker(&allocator->free_list_reclaimer);
+
+	while (!list_empty(&allocator->free_list_head)) {
+		struct page *p;
+
+		p = list_first_entry(&allocator->free_list_head, struct page,
+				lru);
+		list_del(&p->lru);
+
+		kbase_carveout_put_page(p, allocator);
+	}
+	mutex_destroy(&allocator->free_list_lock);
+}
+
+
+mali_error kbase_mem_allocator_alloc(struct kbase_mem_allocator *allocator, size_t nr_pages, phys_addr_t *pages)
+{
+	struct page *p;
+	void *mp;
+	int i;
+	int num_from_free_list;
+	struct list_head from_free_list = LIST_HEAD_INIT(from_free_list);
+
+	might_sleep();
+
+	KBASE_DEBUG_ASSERT(NULL != allocator);
+
+	/* take from the free list first */
+	mutex_lock(&allocator->free_list_lock);
+	num_from_free_list = MIN(nr_pages, atomic_read(&allocator->free_list_size));
+	atomic_sub(num_from_free_list, &allocator->free_list_size);
+	for (i = 0; i < num_from_free_list; i++) {
+		BUG_ON(list_empty(&allocator->free_list_head));
+		p = list_first_entry(&allocator->free_list_head, struct page, lru);
+		list_move(&p->lru, &from_free_list);
+	}
+	mutex_unlock(&allocator->free_list_lock);
+	i = 0;
+
+	/* Allocate as many pages from the pool of already allocated pages. */
+	list_for_each_entry(p, &from_free_list, lru)
+	{
+		pages[i] = PFN_PHYS(page_to_pfn(p));
+		i++;
+	}
+
+	if (i == nr_pages)
+		return MALI_ERROR_NONE;
+
+	/* If not all pages were sourced from the pool, request new ones. */
+	for (; i < nr_pages; i++) {
+		p = kbase_carveout_get_page(allocator);
+		if (NULL == p)
+			goto err_out_roll_back;
+
+		mp = kmap(p);
+		if (NULL == mp) {
+			kbase_carveout_put_page(p, allocator);
+			goto err_out_roll_back;
+		}
+		memset(mp, 0x00, PAGE_SIZE); /* instead of __GFP_ZERO, so we can
+						do cache maintenance */
+		dma_sync_single_for_device(allocator->kbdev->dev,
+					   kbase_dma_addr(p),
+					   PAGE_SIZE,
+					   DMA_BIDIRECTIONAL);
+		kunmap(p);
+		pages[i] = PFN_PHYS(page_to_pfn(p));
+	}
+
+	return MALI_ERROR_NONE;
+
+err_out_roll_back:
+	while (i--) {
+		struct page *p;
+
+		p = pfn_to_page(PFN_DOWN(pages[i]));
+		pages[i] = (phys_addr_t)0;
+		kbase_carveout_put_page(p, allocator);
+	}
+
+	return MALI_ERROR_OUT_OF_MEMORY;
+}
+
+void kbase_mem_allocator_free(struct kbase_mem_allocator *allocator, u32 nr_pages, phys_addr_t *pages, mali_bool sync_back)
+{
+	int i = 0;
+	int page_count = 0;
+	int tofree;
+
+	LIST_HEAD(new_free_list_items);
+
+	KBASE_DEBUG_ASSERT(NULL != allocator);
+
+	might_sleep();
+
+	/* Starting by just freeing the overspill.
+	* As we do this outside of the lock we might spill too many pages
+	* or get too many on the free list, but the max_size is just a ballpark so it is ok
+	* providing that tofree doesn't exceed nr_pages
+	*/
+	tofree = MAX((int)allocator->free_list_max_size - atomic_read(&allocator->free_list_size), 0);
+	tofree = nr_pages - MIN(tofree, nr_pages);
+	for (; i < tofree; i++) {
+		if (likely(0 != pages[i])) {
+			struct page *p;
+
+			p = pfn_to_page(PFN_DOWN(pages[i]));
+			pages[i] = (phys_addr_t)0;
+			kbase_carveout_put_page(p, allocator);
+		}
+	}
+
+	for (; i < nr_pages; i++) {
+		if (likely(0 != pages[i])) {
+			struct page *p;
+
+			p = pfn_to_page(PFN_DOWN(pages[i]));
+			pages[i] = (phys_addr_t)0;
+			/* Sync back the memory to ensure that future cache
+			 * invalidations don't trample on memory.
+			 */
+			if (sync_back)
+				dma_sync_single_for_cpu(allocator->kbdev->dev,
+						kbase_dma_addr(p),
+						PAGE_SIZE,
+						DMA_BIDIRECTIONAL);
+			list_add(&p->lru, &new_free_list_items);
+			page_count++;
+		}
+	}
+	mutex_lock(&allocator->free_list_lock);
+	list_splice(&new_free_list_items, &allocator->free_list_head);
+	atomic_add(page_count, &allocator->free_list_size);
+	mutex_unlock(&allocator->free_list_lock);
+}
+KBASE_EXPORT_TEST_API(kbase_mem_allocator_free)
+