diff options
author | Mark Brown <broonie@linaro.org> | 2013-11-06 11:51:38 +0000 |
---|---|---|
committer | Mark Brown <broonie@linaro.org> | 2013-11-06 11:51:38 +0000 |
commit | 76e8de404a34bfff318da0b31e752db2642511f3 (patch) | |
tree | f486e22499fc5ce7548b76f299ecb91f52baf8ed /drivers/gpu | |
parent | e9b3bd248b349433215cf01064a866b632fb3f4b (diff) | |
parent | 354d86f4daecdc3553830e553f6fc17add736234 (diff) |
Merge branch 'linaro-android-3.10-lsk' of git://git.linaro.org/people/jstultz/android into lsk-v3.10-jstultzv3.10/topic/android-fixes
Diffstat (limited to 'drivers/gpu')
-rw-r--r-- | drivers/gpu/ion/Makefile | 5 | ||||
-rw-r--r-- | drivers/gpu/ion/compat_ion.c | 172 | ||||
-rw-r--r-- | drivers/gpu/ion/compat_ion.h | 30 | ||||
-rw-r--r-- | drivers/gpu/ion/ion.c | 260 | ||||
-rw-r--r-- | drivers/gpu/ion/ion_carveout_heap.c | 7 | ||||
-rw-r--r-- | drivers/gpu/ion/ion_chunk_heap.c | 15 | ||||
-rw-r--r-- | drivers/gpu/ion/ion_cma_heap.c | 205 | ||||
-rw-r--r-- | drivers/gpu/ion/ion_heap.c | 27 | ||||
-rw-r--r-- | drivers/gpu/ion/ion_priv.h | 26 | ||||
-rw-r--r-- | drivers/gpu/ion/ion_system_heap.c | 25 |
10 files changed, 637 insertions, 135 deletions
diff --git a/drivers/gpu/ion/Makefile b/drivers/gpu/ion/Makefile index 306fff970de..9c956659124 100644 --- a/drivers/gpu/ion/Makefile +++ b/drivers/gpu/ion/Makefile @@ -1,3 +1,6 @@ obj-$(CONFIG_ION) += ion.o ion_heap.o ion_page_pool.o ion_system_heap.o \ - ion_carveout_heap.o ion_chunk_heap.o + ion_carveout_heap.o ion_chunk_heap.o ion_cma_heap.o +ifdef CONFIG_COMPAT +obj-$(CONFIG_ION) += compat_ion.o +endif obj-$(CONFIG_ION_TEGRA) += tegra/ diff --git a/drivers/gpu/ion/compat_ion.c b/drivers/gpu/ion/compat_ion.c new file mode 100644 index 00000000000..e0d2839952a --- /dev/null +++ b/drivers/gpu/ion/compat_ion.c @@ -0,0 +1,172 @@ +/* + * drivers/gpu/ion/compat_ion.c + * + * Copyright (C) 2013 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include <linux/ion.h> +#include <linux/compat.h> +#include <linux/fs.h> +#include <linux/uaccess.h> + +#include "compat_ion.h" + +/* See include/linux/ion.h for the definition of these structs */ +struct compat_ion_allocation_data { + compat_size_t len; + compat_size_t align; + compat_uint_t heap_id_mask; + compat_uint_t flags; + compat_int_t handle; +}; + +struct compat_ion_custom_data { + compat_uint_t cmd; + compat_ulong_t arg; +}; + +static int compat_get_ion_allocation_data( + struct compat_ion_allocation_data __user *data32, + struct ion_allocation_data __user *data) +{ + compat_size_t s; + compat_uint_t u; + compat_int_t i; + int err; + + err = get_user(s, &data32->len); + err |= put_user(s, &data->len); + err |= get_user(s, &data32->align); + err |= put_user(s, &data->align); + err |= get_user(u, &data32->heap_id_mask); + err |= put_user(u, &data->heap_id_mask); + err |= get_user(u, &data32->flags); + err |= put_user(u, &data->flags); + err |= get_user(i, &data32->handle); + err |= put_user(i, &data->handle); + + return err; +} + +static int compat_put_ion_allocation_data( + struct compat_ion_allocation_data __user *data32, + struct ion_allocation_data __user *data) +{ + compat_size_t s; + compat_uint_t u; + compat_int_t i; + int err; + + err = get_user(s, &data->len); + err |= put_user(s, &data32->len); + err |= get_user(s, &data->align); + err |= put_user(s, &data32->align); + err |= get_user(u, &data->heap_id_mask); + err |= put_user(u, &data32->heap_id_mask); + err |= get_user(u, &data->flags); + err |= put_user(u, &data32->flags); + err |= get_user(i, &data->handle); + err |= put_user(i, &data32->handle); + + return err; +} + +static int compat_get_ion_custom_data( + struct compat_ion_custom_data __user *data32, + struct ion_custom_data __user *data) +{ + compat_uint_t cmd; + compat_ulong_t arg; + int err; + + err = get_user(cmd, &data32->cmd); + err |= put_user(cmd, &data->cmd); + err |= get_user(arg, &data32->arg); + err |= put_user(arg, &data->arg); + + return err; +}; + +long compat_ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + long ret; + + if (!filp->f_op || !filp->f_op->unlocked_ioctl) + return -ENOTTY; + + switch (cmd) { + case ION_IOC_ALLOC: + { + struct compat_ion_allocation_data __user *data32; + struct ion_allocation_data __user *data; + int err; + + data32 = compat_ptr(arg); + data = compat_alloc_user_space(sizeof(*data)); + if (data == NULL) + return -EFAULT; + + err = compat_get_ion_allocation_data(data32, data); + if (err) + return err; + + ret = filp->f_op->unlocked_ioctl(filp, cmd, + (unsigned long)data); + err = compat_put_ion_allocation_data(data32, data); + return ret ? ret : err; + } + case ION_IOC_FREE: + { + struct compat_ion_allocation_data __user *data32; + struct ion_allocation_data __user *data; + int err; + + data32 = compat_ptr(arg); + data = compat_alloc_user_space(sizeof(*data)); + if (data == NULL) + return -EFAULT; + + err = compat_get_ion_allocation_data(data32, data); + if (err) + return err; + + return filp->f_op->unlocked_ioctl(filp, cmd, + (unsigned long)data); + } + case ION_IOC_CUSTOM: { + struct compat_ion_custom_data __user *data32; + struct ion_custom_data __user *data; + int err; + + data32 = compat_ptr(arg); + data = compat_alloc_user_space(sizeof(*data)); + if (data == NULL) + return -EFAULT; + + err = compat_get_ion_custom_data(data32, data); + if (err) + return err; + + return filp->f_op->unlocked_ioctl(filp, cmd, + (unsigned long)data); + } + case ION_IOC_SHARE: + case ION_IOC_MAP: + case ION_IOC_IMPORT: + case ION_IOC_SYNC: + return filp->f_op->unlocked_ioctl(filp, cmd, + (unsigned long)compat_ptr(arg)); + default: + return -ENOIOCTLCMD; + } +} diff --git a/drivers/gpu/ion/compat_ion.h b/drivers/gpu/ion/compat_ion.h new file mode 100644 index 00000000000..3a9c8c08c24 --- /dev/null +++ b/drivers/gpu/ion/compat_ion.h @@ -0,0 +1,30 @@ +/* + + * drivers/gpu/ion/compat_ion.h + * + * Copyright (C) 2013 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef _LINUX_COMPAT_ION_H +#define _LINUX_COMPAT_ION_H + +#if IS_ENABLED(CONFIG_COMPAT) + +long compat_ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); + +#else + +#define compat_ion_ioctl NULL + +#endif /* CONFIG_COMPAT */ +#endif /* _LINUX_COMPAT_ION_H */ diff --git a/drivers/gpu/ion/ion.c b/drivers/gpu/ion/ion.c index 6c93365c3db..e4ffc9d5b94 100644 --- a/drivers/gpu/ion/ion.c +++ b/drivers/gpu/ion/ion.c @@ -32,10 +32,13 @@ #include <linux/slab.h> #include <linux/seq_file.h> #include <linux/uaccess.h> +#include <linux/vmalloc.h> #include <linux/debugfs.h> #include <linux/dma-buf.h> +#include <linux/idr.h> #include "ion_priv.h" +#include "compat_ion.h" /** * struct ion_device - the metadata of the ion device node @@ -63,6 +66,7 @@ struct ion_device { * @node: node in the tree of all clients * @dev: backpointer to ion device * @handles: an rb tree of all the handles in this client + * @idr: an idr space for allocating handle ids * @lock: lock protecting the tree of handles * @name: used for debugging * @task: used for debugging @@ -75,6 +79,7 @@ struct ion_client { struct rb_node node; struct ion_device *dev; struct rb_root handles; + struct idr idr; struct mutex lock; const char *name; struct task_struct *task; @@ -89,7 +94,7 @@ struct ion_client { * @buffer: pointer to the buffer * @node: node in the client's handle rbtree * @kmap_cnt: count of times this client has mapped to kernel - * @dmap_cnt: count of times this client has mapped for dma + * @id: client-unique id allocated by client->idr * * Modifications to node, map_cnt or mapping should be protected by the * lock in the client. Other fields are never changed after initialization. @@ -100,17 +105,38 @@ struct ion_handle { struct ion_buffer *buffer; struct rb_node node; unsigned int kmap_cnt; + int id; }; bool ion_buffer_fault_user_mappings(struct ion_buffer *buffer) { - return ((buffer->flags & ION_FLAG_CACHED) && - !(buffer->flags & ION_FLAG_CACHED_NEEDS_SYNC)); + return ((buffer->flags & ION_FLAG_CACHED) && + !(buffer->flags & ION_FLAG_CACHED_NEEDS_SYNC)); } bool ion_buffer_cached(struct ion_buffer *buffer) { - return !!(buffer->flags & ION_FLAG_CACHED); + return !!(buffer->flags & ION_FLAG_CACHED); +} + +static inline struct page *ion_buffer_page(struct page *page) +{ + return (struct page *)((unsigned long)page & ~(1UL)); +} + +static inline bool ion_buffer_page_is_dirty(struct page *page) +{ + return !!((unsigned long)page & 1UL); +} + +static inline void ion_buffer_page_dirty(struct page **page) +{ + *page = (struct page *)((unsigned long)(*page) | 1UL); +} + +static inline void ion_buffer_page_clean(struct page **page) +{ + *page = (struct page *)((unsigned long)(*page) & ~(1UL)); } /* this function should only be called while dev->lock is held */ @@ -139,8 +165,6 @@ static void ion_buffer_add(struct ion_device *dev, rb_insert_color(&buffer->node, &dev->buffers); } -static int ion_buffer_alloc_dirty(struct ion_buffer *buffer); - /* this function should only be called while dev->lock is held */ static struct ion_buffer *ion_buffer_create(struct ion_heap *heap, struct ion_device *dev, @@ -178,24 +202,32 @@ static struct ion_buffer *ion_buffer_create(struct ion_heap *heap, buffer->size = len; table = heap->ops->map_dma(heap, buffer); - if (IS_ERR_OR_NULL(table)) { + if (WARN_ONCE(table == NULL, "heap->ops->map_dma should return ERR_PTR on error")) + table = ERR_PTR(-EINVAL); + if (IS_ERR(table)) { heap->ops->free(buffer); kfree(buffer); return ERR_PTR(PTR_ERR(table)); } buffer->sg_table = table; if (ion_buffer_fault_user_mappings(buffer)) { - for_each_sg(buffer->sg_table->sgl, sg, buffer->sg_table->nents, - i) { - if (sg_dma_len(sg) == PAGE_SIZE) - continue; - pr_err("%s: cached mappings that will be faulted in " - "must have pagewise sg_lists\n", __func__); - ret = -EINVAL; - goto err; + int num_pages = PAGE_ALIGN(buffer->size) / PAGE_SIZE; + struct scatterlist *sg; + int i, j, k = 0; + + buffer->pages = vmalloc(sizeof(struct page *) * num_pages); + if (!buffer->pages) { + ret = -ENOMEM; + goto err1; + } + + for_each_sg(table->sgl, sg, table->nents, i) { + struct page *page = sg_page(sg); + + for (j = 0; j < sg_dma_len(sg) / PAGE_SIZE; j++) + buffer->pages[k++] = page++; } - ret = ion_buffer_alloc_dirty(buffer); if (ret) goto err; } @@ -222,6 +254,9 @@ static struct ion_buffer *ion_buffer_create(struct ion_heap *heap, err: heap->ops->unmap_dma(heap, buffer); heap->ops->free(buffer); +err1: + if (buffer->pages) + vfree(buffer->pages); err2: kfree(buffer); return ERR_PTR(ret); @@ -233,8 +268,8 @@ void ion_buffer_destroy(struct ion_buffer *buffer) buffer->heap->ops->unmap_kernel(buffer->heap, buffer); buffer->heap->ops->unmap_dma(buffer->heap, buffer); buffer->heap->ops->free(buffer); - if (buffer->flags & ION_FLAG_CACHED) - kfree(buffer->dirty); + if (buffer->pages) + vfree(buffer->pages); kfree(buffer); } @@ -326,6 +361,7 @@ static void ion_handle_destroy(struct kref *kref) ion_handle_kmap_put(handle); mutex_unlock(&buffer->lock); + idr_remove(&client->idr, handle->id); if (!RB_EMPTY_NODE(&handle->node)) rb_erase(&handle->node, &client->handles); @@ -353,47 +389,56 @@ static int ion_handle_put(struct ion_handle *handle) static struct ion_handle *ion_handle_lookup(struct ion_client *client, struct ion_buffer *buffer) { - struct rb_node *n; - - for (n = rb_first(&client->handles); n; n = rb_next(n)) { - struct ion_handle *handle = rb_entry(n, struct ion_handle, - node); - if (handle->buffer == buffer) - return handle; - } - return NULL; -} - -static bool ion_handle_validate(struct ion_client *client, struct ion_handle *handle) -{ struct rb_node *n = client->handles.rb_node; while (n) { - struct ion_handle *handle_node = rb_entry(n, struct ion_handle, - node); - if (handle < handle_node) + struct ion_handle *entry = rb_entry(n, struct ion_handle, node); + if (buffer < entry->buffer) n = n->rb_left; - else if (handle > handle_node) + else if (buffer > entry->buffer) n = n->rb_right; else - return true; + return entry; } - return false; + return ERR_PTR(-EINVAL); +} + +static struct ion_handle *ion_uhandle_get(struct ion_client *client, int id) +{ + return idr_find(&client->idr, id); } -static void ion_handle_add(struct ion_client *client, struct ion_handle *handle) +static bool ion_handle_validate(struct ion_client *client, struct ion_handle *handle) { + return (ion_uhandle_get(client, handle->id) == handle); +} + +static int ion_handle_add(struct ion_client *client, struct ion_handle *handle) +{ + int rc; struct rb_node **p = &client->handles.rb_node; struct rb_node *parent = NULL; struct ion_handle *entry; + do { + int id; + rc = idr_pre_get(&client->idr, GFP_KERNEL); + if (!rc) + return -ENOMEM; + rc = idr_get_new_above(&client->idr, handle, 1, &id); + handle->id = id; + } while (rc == -EAGAIN); + + if (rc < 0) + return rc; + while (*p) { parent = *p; entry = rb_entry(parent, struct ion_handle, node); - if (handle < entry) + if (handle->buffer < entry->buffer) p = &(*p)->rb_left; - else if (handle > entry) + else if (handle->buffer > entry->buffer) p = &(*p)->rb_right; else WARN(1, "%s: buffer already found.", __func__); @@ -401,6 +446,8 @@ static void ion_handle_add(struct ion_client *client, struct ion_handle *handle) rb_link_node(&handle->node, parent, p); rb_insert_color(&handle->node, &client->handles); + + return 0; } struct ion_handle *ion_alloc(struct ion_client *client, size_t len, @@ -411,6 +458,7 @@ struct ion_handle *ion_alloc(struct ion_client *client, size_t len, struct ion_device *dev = client->dev; struct ion_buffer *buffer = NULL; struct ion_heap *heap; + int ret; pr_debug("%s: len %d align %d heap_id_mask %u flags %x\n", __func__, len, align, heap_id_mask, flags); @@ -431,7 +479,7 @@ struct ion_handle *ion_alloc(struct ion_client *client, size_t len, if (!((1 << heap->id) & heap_id_mask)) continue; buffer = ion_buffer_create(heap, dev, len, align, flags); - if (!IS_ERR_OR_NULL(buffer)) + if (!IS_ERR(buffer)) break; } up_read(&dev->lock); @@ -450,12 +498,16 @@ struct ion_handle *ion_alloc(struct ion_client *client, size_t len, */ ion_buffer_put(buffer); - if (!IS_ERR(handle)) { - mutex_lock(&client->lock); - ion_handle_add(client, handle); - mutex_unlock(&client->lock); - } + if (IS_ERR(handle)) + return handle; + mutex_lock(&client->lock); + ret = ion_handle_add(client, handle); + if (ret) { + ion_handle_put(handle); + handle = ERR_PTR(ret); + } + mutex_unlock(&client->lock); return handle; } @@ -515,7 +567,9 @@ static void *ion_buffer_kmap_get(struct ion_buffer *buffer) return buffer->vaddr; } vaddr = buffer->heap->ops->map_kernel(buffer->heap, buffer); - if (IS_ERR_OR_NULL(vaddr)) + if (WARN_ONCE(vaddr == NULL, "heap->ops->map_kernel should return ERR_PTR on error")) + return ERR_PTR(-EINVAL); + if (IS_ERR(vaddr)) return vaddr; buffer->vaddr = vaddr; buffer->kmap_cnt++; @@ -532,7 +586,7 @@ static void *ion_handle_kmap_get(struct ion_handle *handle) return buffer->vaddr; } vaddr = ion_buffer_kmap_get(buffer); - if (IS_ERR_OR_NULL(vaddr)) + if (IS_ERR(vaddr)) return vaddr; handle->kmap_cnt++; return vaddr; @@ -673,6 +727,7 @@ struct ion_client *ion_client_create(struct ion_device *dev, client->dev = dev; client->handles = RB_ROOT; + idr_init(&client->idr); mutex_init(&client->lock); client->name = name; client->task = task; @@ -713,6 +768,10 @@ void ion_client_destroy(struct ion_client *client) node); ion_handle_destroy(&handle->ref); } + + idr_remove_all(&client->idr); + idr_destroy(&client->idr); + down_write(&dev->lock); if (client->task) put_task_struct(client->task); @@ -764,17 +823,6 @@ static void ion_unmap_dma_buf(struct dma_buf_attachment *attachment, { } -static int ion_buffer_alloc_dirty(struct ion_buffer *buffer) -{ - unsigned long pages = buffer->sg_table->nents; - unsigned long length = (pages + BITS_PER_LONG - 1)/BITS_PER_LONG; - - buffer->dirty = kzalloc(length * sizeof(unsigned long), GFP_KERNEL); - if (!buffer->dirty) - return -ENOMEM; - return 0; -} - struct ion_vma_list { struct list_head list; struct vm_area_struct *vma; @@ -784,9 +832,9 @@ static void ion_buffer_sync_for_device(struct ion_buffer *buffer, struct device *dev, enum dma_data_direction dir) { - struct scatterlist *sg; - int i; struct ion_vma_list *vma_list; + int pages = PAGE_ALIGN(buffer->size) / PAGE_SIZE; + int i; pr_debug("%s: syncing for device %s\n", __func__, dev ? dev_name(dev) : "null"); @@ -795,11 +843,12 @@ static void ion_buffer_sync_for_device(struct ion_buffer *buffer, return; mutex_lock(&buffer->lock); - for_each_sg(buffer->sg_table->sgl, sg, buffer->sg_table->nents, i) { - if (!test_bit(i, buffer->dirty)) - continue; - dma_sync_sg_for_device(dev, sg, 1, dir); - clear_bit(i, buffer->dirty); + for (i = 0; i < pages; i++) { + struct page *page = buffer->pages[i]; + + if (ion_buffer_page_is_dirty(page)) + __dma_page_cpu_to_dev(page, 0, PAGE_SIZE, dir); + ion_buffer_page_clean(buffer->pages + i); } list_for_each_entry(vma_list, &buffer->vmas, list) { struct vm_area_struct *vma = vma_list->vma; @@ -813,21 +862,18 @@ static void ion_buffer_sync_for_device(struct ion_buffer *buffer, int ion_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) { struct ion_buffer *buffer = vma->vm_private_data; - struct scatterlist *sg; - int i; + int ret; mutex_lock(&buffer->lock); - set_bit(vmf->pgoff, buffer->dirty); + ion_buffer_page_dirty(buffer->pages + vmf->pgoff); - for_each_sg(buffer->sg_table->sgl, sg, buffer->sg_table->nents, i) { - if (i != vmf->pgoff) - continue; - dma_sync_sg_for_cpu(NULL, sg, 1, DMA_BIDIRECTIONAL); - vm_insert_page(vma, (unsigned long)vmf->virtual_address, - sg_page(sg)); - break; - } + BUG_ON(!buffer->pages || !buffer->pages[vmf->pgoff]); + ret = vm_insert_page(vma, (unsigned long)vmf->virtual_address, + ion_buffer_page(buffer->pages[vmf->pgoff])); mutex_unlock(&buffer->lock); + if (ret) + return VM_FAULT_ERROR; + return VM_FAULT_NOPAGE; } @@ -939,8 +985,6 @@ static int ion_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, size_t start, mutex_unlock(&buffer->lock); if (IS_ERR(vaddr)) return PTR_ERR(vaddr); - if (!vaddr) - return -ENOMEM; return 0; } @@ -1017,9 +1061,10 @@ struct ion_handle *ion_import_dma_buf(struct ion_client *client, int fd) struct dma_buf *dmabuf; struct ion_buffer *buffer; struct ion_handle *handle; + int ret; dmabuf = dma_buf_get(fd); - if (IS_ERR_OR_NULL(dmabuf)) + if (IS_ERR(dmabuf)) return ERR_PTR(PTR_ERR(dmabuf)); /* if this memory came from ion */ @@ -1034,14 +1079,18 @@ struct ion_handle *ion_import_dma_buf(struct ion_client *client, int fd) mutex_lock(&client->lock); /* if a handle exists for this buffer just take a reference to it */ handle = ion_handle_lookup(client, buffer); - if (!IS_ERR_OR_NULL(handle)) { + if (!IS_ERR(handle)) { ion_handle_get(handle); goto end; } handle = ion_handle_create(client, buffer); - if (IS_ERR_OR_NULL(handle)) + if (IS_ERR(handle)) goto end; - ion_handle_add(client, handle); + ret = ion_handle_add(client, handle); + if (ret) { + ion_handle_put(handle); + handle = ERR_PTR(ret); + } end: mutex_unlock(&client->lock); dma_buf_put(dmabuf); @@ -1055,7 +1104,7 @@ static int ion_sync_for_device(struct ion_client *client, int fd) struct ion_buffer *buffer; dmabuf = dma_buf_get(fd); - if (IS_ERR_OR_NULL(dmabuf)) + if (IS_ERR(dmabuf)) return PTR_ERR(dmabuf); /* if this memory came from ion */ @@ -1081,17 +1130,20 @@ static long ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) case ION_IOC_ALLOC: { struct ion_allocation_data data; + struct ion_handle *handle; if (copy_from_user(&data, (void __user *)arg, sizeof(data))) return -EFAULT; - data.handle = ion_alloc(client, data.len, data.align, + handle = ion_alloc(client, data.len, data.align, data.heap_id_mask, data.flags); - if (IS_ERR(data.handle)) - return PTR_ERR(data.handle); + if (IS_ERR(handle)) + return PTR_ERR(handle); + + data.handle = handle->id; if (copy_to_user((void __user *)arg, &data, sizeof(data))) { - ion_free(client, data.handle); + ion_free(client, handle); return -EFAULT; } break; @@ -1099,27 +1151,29 @@ static long ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) case ION_IOC_FREE: { struct ion_handle_data data; - bool valid; + struct ion_handle *handle; if (copy_from_user(&data, (void __user *)arg, sizeof(struct ion_handle_data))) return -EFAULT; mutex_lock(&client->lock); - valid = ion_handle_validate(client, data.handle); + handle = ion_uhandle_get(client, data.handle); mutex_unlock(&client->lock); - if (!valid) + if (!handle) return -EINVAL; - ion_free(client, data.handle); + ion_free(client, handle); break; } case ION_IOC_SHARE: case ION_IOC_MAP: { struct ion_fd_data data; + struct ion_handle *handle; if (copy_from_user(&data, (void __user *)arg, sizeof(data))) return -EFAULT; - data.fd = ion_share_dma_buf_fd(client, data.handle); + handle = ion_uhandle_get(client, data.handle); + data.fd = ion_share_dma_buf_fd(client, handle); if (copy_to_user((void __user *)arg, &data, sizeof(data))) return -EFAULT; if (data.fd < 0) @@ -1129,15 +1183,17 @@ static long ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) case ION_IOC_IMPORT: { struct ion_fd_data data; + struct ion_handle *handle; int ret = 0; if (copy_from_user(&data, (void __user *)arg, sizeof(struct ion_fd_data))) return -EFAULT; - data.handle = ion_import_dma_buf(client, data.fd); - if (IS_ERR(data.handle)) { - ret = PTR_ERR(data.handle); - data.handle = NULL; - } + handle = ion_import_dma_buf(client, data.fd); + if (IS_ERR(handle)) + ret = PTR_ERR(handle); + else + data.handle = handle->id; + if (copy_to_user((void __user *)arg, &data, sizeof(struct ion_fd_data))) return -EFAULT; @@ -1189,7 +1245,7 @@ static int ion_open(struct inode *inode, struct file *file) pr_debug("%s: %d\n", __func__, __LINE__); client = ion_client_create(dev, "user"); - if (IS_ERR_OR_NULL(client)) + if (IS_ERR(client)) return PTR_ERR(client); file->private_data = client; @@ -1201,6 +1257,7 @@ static const struct file_operations ion_fops = { .open = ion_open, .release = ion_release, .unlocked_ioctl = ion_ioctl, + .compat_ioctl = compat_ion_ioctl, }; static size_t ion_debug_heap_total(struct ion_client *client, @@ -1271,6 +1328,9 @@ static int ion_debug_heap_show(struct seq_file *s, void *unused) seq_printf(s, "%16.s %16u\n", "total orphaned", total_orphaned_size); seq_printf(s, "%16.s %16u\n", "total ", total_size); + if (heap->flags & ION_HEAP_FLAG_DEFER_FREE) + seq_printf(s, "%16.s %16u\n", "deferred free", + heap->free_list_size); seq_printf(s, "----------------------------------------------------\n"); if (heap->debug_show) @@ -1382,7 +1442,7 @@ struct ion_device *ion_device_create(long (*custom_ioctl) } idev->debug_root = debugfs_create_dir("ion", NULL); - if (IS_ERR_OR_NULL(idev->debug_root)) + if (!idev->debug_root) pr_err("ion: failed to create debug files.\n"); idev->custom_ioctl = custom_ioctl; diff --git a/drivers/gpu/ion/ion_carveout_heap.c b/drivers/gpu/ion/ion_carveout_heap.c index ce8d311968f..86f35545eaf 100644 --- a/drivers/gpu/ion/ion_carveout_heap.c +++ b/drivers/gpu/ion/ion_carveout_heap.c @@ -112,13 +112,18 @@ void ion_carveout_heap_unmap_dma(struct ion_heap *heap, void *ion_carveout_heap_map_kernel(struct ion_heap *heap, struct ion_buffer *buffer) { + void *ret; int mtype = MT_MEMORY_NONCACHED; if (buffer->flags & ION_FLAG_CACHED) mtype = MT_MEMORY; - return __arm_ioremap(buffer->priv_phys, buffer->size, + ret = __arm_ioremap(buffer->priv_phys, buffer->size, mtype); + if (ret == NULL) + return ERR_PTR(-ENOMEM); + + return ret; } void ion_carveout_heap_unmap_kernel(struct ion_heap *heap, diff --git a/drivers/gpu/ion/ion_chunk_heap.c b/drivers/gpu/ion/ion_chunk_heap.c index 8c8f5c3c553..cd01aad6ac2 100644 --- a/drivers/gpu/ion/ion_chunk_heap.c +++ b/drivers/gpu/ion/ion_chunk_heap.c @@ -47,15 +47,15 @@ static int ion_chunk_heap_allocate(struct ion_heap *heap, struct scatterlist *sg; int ret, i; unsigned long num_chunks; + unsigned long allocated_size; if (ion_buffer_fault_user_mappings(buffer)) return -ENOMEM; - num_chunks = ALIGN(size, chunk_heap->chunk_size) / - chunk_heap->chunk_size; - buffer->size = num_chunks * chunk_heap->chunk_size; + allocated_size = ALIGN(size, chunk_heap->chunk_size); + num_chunks = allocated_size / chunk_heap->chunk_size; - if (buffer->size > chunk_heap->size - chunk_heap->allocated) + if (allocated_size > chunk_heap->size - chunk_heap->allocated) return -ENOMEM; table = kzalloc(sizeof(struct sg_table), GFP_KERNEL); @@ -78,7 +78,7 @@ static int ion_chunk_heap_allocate(struct ion_heap *heap, } buffer->priv_virt = table; - chunk_heap->allocated += buffer->size; + chunk_heap->allocated += allocated_size; return 0; err: sg = table->sgl; @@ -100,6 +100,9 @@ static void ion_chunk_heap_free(struct ion_buffer *buffer) struct sg_table *table = buffer->priv_virt; struct scatterlist *sg; int i; + unsigned long allocated_size; + + allocated_size = ALIGN(buffer->size, chunk_heap->chunk_size); ion_heap_buffer_zero(buffer); @@ -111,7 +114,7 @@ static void ion_chunk_heap_free(struct ion_buffer *buffer) gen_pool_free(chunk_heap->pool, page_to_phys(sg_page(sg)), sg_dma_len(sg)); } - chunk_heap->allocated -= buffer->size; + chunk_heap->allocated -= allocated_size; sg_free_table(table); kfree(table); } diff --git a/drivers/gpu/ion/ion_cma_heap.c b/drivers/gpu/ion/ion_cma_heap.c new file mode 100644 index 00000000000..1eaa8c11e04 --- /dev/null +++ b/drivers/gpu/ion/ion_cma_heap.c @@ -0,0 +1,205 @@ +/* + * drivers/gpu/ion/ion_cma_heap.c + * + * Copyright (C) Linaro 2012 + * Author: <benjamin.gaignard@linaro.org> for ST-Ericsson. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include <linux/device.h> +#include <linux/ion.h> +#include <linux/slab.h> +#include <linux/errno.h> +#include <linux/err.h> +#include <linux/dma-mapping.h> + +/* for ion_heap_ops structure */ +#include "ion_priv.h" + +#define ION_CMA_ALLOCATE_FAILED -1 + +struct ion_cma_heap { + struct ion_heap heap; + struct device *dev; +}; + +#define to_cma_heap(x) container_of(x, struct ion_cma_heap, heap) + +struct ion_cma_buffer_info { + void *cpu_addr; + dma_addr_t handle; + struct sg_table *table; +}; + +/* + * Create scatter-list for the already allocated DMA buffer. + * This function could be replaced by dma_common_get_sgtable + * as soon as it will avalaible. + */ +int ion_cma_get_sgtable(struct device *dev, struct sg_table *sgt, + void *cpu_addr, dma_addr_t handle, size_t size) +{ + struct page *page = virt_to_page(cpu_addr); + int ret; + + ret = sg_alloc_table(sgt, 1, GFP_KERNEL); + if (unlikely(ret)) + return ret; + + sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); + return 0; +} + +/* ION CMA heap operations functions */ +static int ion_cma_allocate(struct ion_heap *heap, struct ion_buffer *buffer, + unsigned long len, unsigned long align, + unsigned long flags) +{ + struct ion_cma_heap *cma_heap = to_cma_heap(heap); + struct device *dev = cma_heap->dev; + struct ion_cma_buffer_info *info; + + dev_dbg(dev, "Request buffer allocation len %ld\n", len); + + info = kzalloc(sizeof(struct ion_cma_buffer_info), GFP_KERNEL); + if (!info) { + dev_err(dev, "Can't allocate buffer info\n"); + return ION_CMA_ALLOCATE_FAILED; + } + + info->cpu_addr = dma_alloc_coherent(dev, len, &(info->handle), 0); + + if (!info->cpu_addr) { + dev_err(dev, "Fail to allocate buffer\n"); + goto err; + } + + info->table = kmalloc(sizeof(struct sg_table), GFP_KERNEL); + if (!info->table) { + dev_err(dev, "Fail to allocate sg table\n"); + goto free_mem; + } + + if (ion_cma_get_sgtable + (dev, info->table, info->cpu_addr, info->handle, len)) + goto free_table; + /* keep this for memory release */ + buffer->priv_virt = info; + dev_dbg(dev, "Allocate buffer %p\n", buffer); + return 0; + +free_table: + kfree(info->table); +free_mem: + dma_free_coherent(dev, len, info->cpu_addr, info->handle); +err: + kfree(info); + return ION_CMA_ALLOCATE_FAILED; +} + +static void ion_cma_free(struct ion_buffer *buffer) +{ + struct ion_cma_heap *cma_heap = to_cma_heap(buffer->heap); + struct device *dev = cma_heap->dev; + struct ion_cma_buffer_info *info = buffer->priv_virt; + + dev_dbg(dev, "Release buffer %p\n", buffer); + /* release memory */ + dma_free_coherent(dev, buffer->size, info->cpu_addr, info->handle); + /* release sg table */ + sg_free_table(info->table); + kfree(info->table); + kfree(info); +} + +/* return physical address in addr */ +static int ion_cma_phys(struct ion_heap *heap, struct ion_buffer *buffer, + ion_phys_addr_t *addr, size_t *len) +{ + struct ion_cma_heap *cma_heap = to_cma_heap(buffer->heap); + struct device *dev = cma_heap->dev; + struct ion_cma_buffer_info *info = buffer->priv_virt; + + dev_dbg(dev, "Return buffer %p physical address 0x%x\n", buffer, + info->handle); + + *addr = info->handle; + *len = buffer->size; + + return 0; +} + +struct sg_table *ion_cma_heap_map_dma(struct ion_heap *heap, + struct ion_buffer *buffer) +{ + struct ion_cma_buffer_info *info = buffer->priv_virt; + + return info->table; +} + +void ion_cma_heap_unmap_dma(struct ion_heap *heap, + struct ion_buffer *buffer) +{ + return; +} + +static int ion_cma_mmap(struct ion_heap *mapper, struct ion_buffer *buffer, + struct vm_area_struct *vma) +{ + struct ion_cma_heap *cma_heap = to_cma_heap(buffer->heap); + struct device *dev = cma_heap->dev; + struct ion_cma_buffer_info *info = buffer->priv_virt; + + return dma_mmap_coherent(dev, vma, info->cpu_addr, info->handle, + buffer->size); +} + +void *ion_cma_map_kernel(struct ion_heap *heap, struct ion_buffer *buffer) +{ + struct ion_cma_buffer_info *info = buffer->priv_virt; + /* kernel memory mapping has been done at allocation time */ + return info->cpu_addr; +} + +static struct ion_heap_ops ion_cma_ops = { + .allocate = ion_cma_allocate, + .free = ion_cma_free, + .map_dma = ion_cma_heap_map_dma, + .unmap_dma = ion_cma_heap_unmap_dma, + .phys = ion_cma_phys, + .map_user = ion_cma_mmap, + .map_kernel = ion_cma_map_kernel, +}; + +struct ion_heap *ion_cma_heap_create(struct ion_platform_heap *data) +{ + struct ion_cma_heap *cma_heap; + + cma_heap = kzalloc(sizeof(struct ion_cma_heap), GFP_KERNEL); + + if (!cma_heap) + return ERR_PTR(-ENOMEM); + + cma_heap->heap.ops = &ion_cma_ops; + /* get device from private heaps data, later it will be + * used to make the link with reserved CMA memory */ + cma_heap->dev = data->priv; + cma_heap->heap.type = ION_HEAP_TYPE_DMA; + return &cma_heap->heap; +} + +void ion_cma_heap_destroy(struct ion_heap *heap) +{ + struct ion_cma_heap *cma_heap = to_cma_heap(heap); + + kfree(cma_heap); +} diff --git a/drivers/gpu/ion/ion_heap.c b/drivers/gpu/ion/ion_heap.c index 4a16aa27384..786302de7ed 100644 --- a/drivers/gpu/ion/ion_heap.c +++ b/drivers/gpu/ion/ion_heap.c @@ -56,6 +56,9 @@ void *ion_heap_map_kernel(struct ion_heap *heap, vaddr = vmap(pages, npages, VM_MAP, pgprot); vfree(pages); + if (vaddr == NULL) + return ERR_PTR(-ENOMEM); + return vaddr; } @@ -134,8 +137,22 @@ end: return ret; } -void ion_heap_free_page(struct ion_buffer *buffer, struct page *page, - unsigned int order) +struct page *ion_heap_alloc_pages(struct ion_buffer *buffer, gfp_t gfp_flags, + unsigned int order) +{ + struct page *page = alloc_pages(gfp_flags, order); + + if (!page) + return page; + + if (ion_buffer_fault_user_mappings(buffer)) + split_page(page, order); + + return page; +} + +void ion_heap_free_pages(struct ion_buffer *buffer, struct page *page, + unsigned int order) { int i; @@ -254,6 +271,9 @@ struct ion_heap *ion_heap_create(struct ion_platform_heap *heap_data) case ION_HEAP_TYPE_CHUNK: heap = ion_chunk_heap_create(heap_data); break; + case ION_HEAP_TYPE_DMA: + heap = ion_cma_heap_create(heap_data); + break; default: pr_err("%s: Invalid heap type %d\n", __func__, heap_data->type); @@ -290,6 +310,9 @@ void ion_heap_destroy(struct ion_heap *heap) case ION_HEAP_TYPE_CHUNK: ion_chunk_heap_destroy(heap); break; + case ION_HEAP_TYPE_DMA: + ion_cma_heap_destroy(heap); + break; default: pr_err("%s: Invalid heap type %d\n", __func__, heap->type); diff --git a/drivers/gpu/ion/ion_priv.h b/drivers/gpu/ion/ion_priv.h index f9bccbc131e..32461e94673 100644 --- a/drivers/gpu/ion/ion_priv.h +++ b/drivers/gpu/ion/ion_priv.h @@ -45,9 +45,8 @@ struct ion_buffer *ion_handle_buffer(struct ion_handle *handle); * @vaddr: the kenrel mapping if kmap_cnt is not zero * @dmap_cnt: number of times the buffer is mapped for dma * @sg_table: the sg table for the buffer if dmap_cnt is not zero - * @dirty: bitmask representing which pages of this buffer have - * been dirtied by the cpu and need cache maintenance - * before dma + * @pages: flat array of pages in the buffer -- used by fault + * handler and only valid for buffers that are faulted in * @vmas: list of vma's mapping this buffer * @handle_count: count of handles referencing this buffer * @task_comm: taskcomm of last client to reference this buffer in a @@ -74,7 +73,7 @@ struct ion_buffer { void *vaddr; int dmap_cnt; struct sg_table *sg_table; - unsigned long *dirty; + struct page **pages; struct list_head vmas; /* used to track orphaned buffers */ int handle_count; @@ -94,6 +93,9 @@ void ion_buffer_destroy(struct ion_buffer *buffer); * @map_kernel map memory to the kernel * @unmap_kernel unmap memory to the kernel * @map_user map memory to userspace + * + * allocate, phys, and map_user return 0 on success, -errno on error. + * map_dma and map_kernel return pointer on success, ERR_PTR on error. */ struct ion_heap_ops { int (*allocate) (struct ion_heap *heap, @@ -213,6 +215,19 @@ int ion_heap_map_user(struct ion_heap *, struct ion_buffer *, int ion_heap_buffer_zero(struct ion_buffer *buffer); /** + * ion_heap_alloc_pages - allocate pages from alloc_pages + * @buffer: the buffer to allocate for, used to extract the flags + * @gfp_flags: the gfp_t for the allocation + * @order: the order of the allocatoin + * + * This funciton allocations from alloc pages and also does any other + * necessary operations based on the buffer->flags. For buffers which + * will be faulted in the pages are split using split_page + */ +struct page *ion_heap_alloc_pages(struct ion_buffer *buffer, gfp_t gfp_flags, + unsigned int order); + +/** * ion_heap_init_deferred_free -- initialize deferred free functionality * @heap: the heap * @@ -269,6 +284,9 @@ void ion_carveout_heap_destroy(struct ion_heap *); struct ion_heap *ion_chunk_heap_create(struct ion_platform_heap *); void ion_chunk_heap_destroy(struct ion_heap *); +struct ion_heap *ion_cma_heap_create(struct ion_platform_heap *); +void ion_cma_heap_destroy(struct ion_heap *); + /** * kernel api to allocate/free from carveout -- used when carveout is * used to back an architecture specific custom heap diff --git a/drivers/gpu/ion/ion_system_heap.c b/drivers/gpu/ion/ion_system_heap.c index e101db5da5b..5fe81a76f2f 100644 --- a/drivers/gpu/ion/ion_system_heap.c +++ b/drivers/gpu/ion/ion_system_heap.c @@ -64,7 +64,6 @@ static struct page *alloc_buffer_page(struct ion_system_heap *heap, unsigned long order) { bool cached = ion_buffer_cached(buffer); - bool split_pages = ion_buffer_fault_user_mappings(buffer); struct ion_page_pool *pool = heap->pools[order_to_index(order)]; struct page *page; @@ -75,7 +74,7 @@ static struct page *alloc_buffer_page(struct ion_system_heap *heap, if (order > 4) gfp_flags = high_order_gfp_flags; - page = alloc_pages(gfp_flags, order); + page = ion_heap_alloc_pages(buffer, gfp_flags, order); if (!page) return 0; arm_dma_ops.sync_single_for_device(NULL, @@ -85,8 +84,6 @@ static struct page *alloc_buffer_page(struct ion_system_heap *heap, if (!page) return 0; - if (split_pages) - split_page(page, order); return page; } @@ -153,7 +150,6 @@ static int ion_system_heap_allocate(struct ion_heap *heap, int i = 0; long size_remaining = PAGE_ALIGN(size); unsigned int max_order = orders[0]; - bool split_pages = ion_buffer_fault_user_mappings(buffer); INIT_LIST_HEAD(&pages); while (size_remaining > 0) { @@ -170,28 +166,15 @@ static int ion_system_heap_allocate(struct ion_heap *heap, if (!table) goto err; - if (split_pages) - ret = sg_alloc_table(table, PAGE_ALIGN(size) / PAGE_SIZE, - GFP_KERNEL); - else - ret = sg_alloc_table(table, i, GFP_KERNEL); - + ret = sg_alloc_table(table, i, GFP_KERNEL); if (ret) goto err1; sg = table->sgl; list_for_each_entry_safe(info, tmp_info, &pages, list) { struct page *page = info->page; - if (split_pages) { - for (i = 0; i < (1 << info->order); i++) { - sg_set_page(sg, page + i, PAGE_SIZE, 0); - sg = sg_next(sg); - } - } else { - sg_set_page(sg, page, (1 << info->order) * PAGE_SIZE, - 0); - sg = sg_next(sg); - } + sg_set_page(sg, page, (1 << info->order) * PAGE_SIZE, 0); + sg = sg_next(sg); list_del(&info->list); kfree(info); } |