8f477880f040fe5985617acedd617bce7c988a7c
[android-sdk/kernel-video.git] / drivers / gpu / ion / ion.c
1 /*
3  * drivers/gpu/ion/ion.c
4  *
5  * Copyright (C) 2011 Google, Inc.
6  *
7  * This software is licensed under the terms of the GNU General Public
8  * License version 2, as published by the Free Software Foundation, and
9  * may be copied, distributed, and modified under those terms.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  */
18 #include <linux/device.h>
19 #include <linux/file.h>
20 #include <linux/fs.h>
21 #include <linux/anon_inodes.h>
22 #include <linux/ion.h>
23 #include <linux/list.h>
24 #include <linux/memblock.h>
25 #include <linux/miscdevice.h>
26 #include <linux/export.h>
27 #include <linux/mm.h>
28 #include <linux/mm_types.h>
29 #include <linux/rbtree.h>
30 #include <linux/sched.h>
31 #include <linux/slab.h>
32 #include <linux/seq_file.h>
33 #include <linux/uaccess.h>
34 #include <linux/debugfs.h>
35 #include <linux/dma-buf.h>
37 #include "ion_priv.h"
39 bool ion_buffer_fault_user_mappings(struct ion_buffer *buffer)
40 {
41         return ((buffer->flags & ION_FLAG_CACHED) &&
42                 !(buffer->flags & ION_FLAG_CACHED_NEEDS_SYNC));
43 }
45 bool ion_buffer_cached(struct ion_buffer *buffer)
46 {
47         return !!(buffer->flags & ION_FLAG_CACHED);
48 }
50 /* this function should only be called while dev->lock is held */
51 static void ion_buffer_add(struct ion_device *dev,
52                            struct ion_buffer *buffer)
53 {
54         struct rb_node **p = &dev->buffers.rb_node;
55         struct rb_node *parent = NULL;
56         struct ion_buffer *entry;
58         while (*p) {
59                 parent = *p;
60                 entry = rb_entry(parent, struct ion_buffer, node);
62                 if (buffer < entry) {
63                         p = &(*p)->rb_left;
64                 } else if (buffer > entry) {
65                         p = &(*p)->rb_right;
66                 } else {
67                         pr_err("%s: buffer already found.", __func__);
68                         BUG();
69                 }
70         }
72         rb_link_node(&buffer->node, parent, p);
73         rb_insert_color(&buffer->node, &dev->buffers);
74 }
76 static int ion_buffer_alloc_dirty(struct ion_buffer *buffer);
78 /* this function should only be called while dev->lock is held */
79 static struct ion_buffer *ion_buffer_create(struct ion_heap *heap,
80                                      struct ion_device *dev,
81                                      unsigned long len,
82                                      unsigned long align,
83                                      unsigned long flags)
84 {
85         struct ion_buffer *buffer;
86         struct sg_table *table;
87         struct scatterlist *sg;
88         int i, ret;
90         buffer = kzalloc(sizeof(struct ion_buffer), GFP_KERNEL);
91         if (!buffer)
92                 return ERR_PTR(-ENOMEM);
94         buffer->heap = heap;
95         buffer->flags = flags;
96         kref_init(&buffer->ref);
98         ret = heap->ops->allocate(heap, buffer, len, align, flags);
99         if (ret) {
100                 kfree(buffer);
101                 return ERR_PTR(ret);
102         }
104         buffer->dev = dev;
105         buffer->size = len;
107         table = heap->ops->map_dma(heap, buffer);
108         if (IS_ERR_OR_NULL(table)) {
109                 heap->ops->free(buffer);
110                 kfree(buffer);
111                 return ERR_PTR(PTR_ERR(table));
112         }
113         buffer->sg_table = table;
114         if (ion_buffer_fault_user_mappings(buffer)) {
115                 for_each_sg(buffer->sg_table->sgl, sg, buffer->sg_table->nents,
116                             i) {
117                         if (sg_dma_len(sg) == PAGE_SIZE)
118                                 continue;
119                         pr_err("%s: cached mappings that will be faulted in "
120                                "must have pagewise sg_lists\n", __func__);
121                         ret = -EINVAL;
122                         goto err;
123                 }
125                 ret = ion_buffer_alloc_dirty(buffer);
126                 if (ret)
127                         goto err;
128         }
130         buffer->dev = dev;
131         buffer->size = len;
132         INIT_LIST_HEAD(&buffer->vmas);
133         mutex_init(&buffer->lock);
134         /* this will set up dma addresses for the sglist -- it is not
135            technically correct as per the dma api -- a specific
136            device isn't really taking ownership here.  However, in practice on
137            our systems the only dma_address space is physical addresses.
138            Additionally, we can't afford the overhead of invalidating every
139            allocation via dma_map_sg. The implicit contract here is that
140            memory comming from the heaps is ready for dma, ie if it has a
141            cached mapping that mapping has been invalidated */
142         for_each_sg(buffer->sg_table->sgl, sg, buffer->sg_table->nents, i)
143                 sg_dma_address(sg) = sg_phys(sg);
144         mutex_lock(&dev->buffer_lock);
145         ion_buffer_add(dev, buffer);
146         mutex_unlock(&dev->buffer_lock);
147         return buffer;
149 err:
150         heap->ops->unmap_dma(heap, buffer);
151         heap->ops->free(buffer);
152         kfree(buffer);
153         return ERR_PTR(ret);
156 static void ion_buffer_destroy(struct kref *kref)
158         struct ion_buffer *buffer = container_of(kref, struct ion_buffer, ref);
159         struct ion_device *dev = buffer->dev;
161         if (WARN_ON(buffer->kmap_cnt > 0))
162                 buffer->heap->ops->unmap_kernel(buffer->heap, buffer);
163         buffer->heap->ops->unmap_dma(buffer->heap, buffer);
164         buffer->heap->ops->free(buffer);
165         mutex_lock(&dev->buffer_lock);
166         rb_erase(&buffer->node, &dev->buffers);
167         mutex_unlock(&dev->buffer_lock);
168         if (buffer->flags & ION_FLAG_CACHED)
169                 kfree(buffer->dirty);
170         kfree(buffer);
173 static void ion_buffer_get(struct ion_buffer *buffer)
175         kref_get(&buffer->ref);
178 static int ion_buffer_put(struct ion_buffer *buffer)
180         return kref_put(&buffer->ref, ion_buffer_destroy);
183 static void ion_buffer_add_to_handle(struct ion_buffer *buffer)
185         mutex_lock(&buffer->lock);
186         buffer->handle_count++;
187         mutex_unlock(&buffer->lock);
190 static void ion_buffer_remove_from_handle(struct ion_buffer *buffer)
192         /*
193          * when a buffer is removed from a handle, if it is not in
194          * any other handles, copy the taskcomm and the pid of the
195          * process it's being removed from into the buffer.  At this
196          * point there will be no way to track what processes this buffer is
197          * being used by, it only exists as a dma_buf file descriptor.
198          * The taskcomm and pid can provide a debug hint as to where this fd
199          * is in the system
200          */
201         mutex_lock(&buffer->lock);
202         buffer->handle_count--;
203         BUG_ON(buffer->handle_count < 0);
204         if (!buffer->handle_count) {
205                 struct task_struct *task;
207                 task = current->group_leader;
208                 get_task_comm(buffer->task_comm, task);
209                 buffer->pid = task_pid_nr(task);
210         }
211         mutex_unlock(&buffer->lock);
214 static struct ion_handle *ion_handle_create(struct ion_client *client,
215                                      struct ion_buffer *buffer)
217         struct ion_handle *handle;
219         handle = kzalloc(sizeof(struct ion_handle), GFP_KERNEL);
220         if (!handle)
221                 return ERR_PTR(-ENOMEM);
222         kref_init(&handle->ref);
223         RB_CLEAR_NODE(&handle->node);
224         handle->client = client;
225         ion_buffer_get(buffer);
226         ion_buffer_add_to_handle(buffer);
227         handle->buffer = buffer;
229         return handle;
232 static void ion_handle_kmap_put(struct ion_handle *);
234 static void ion_handle_destroy(struct kref *kref)
236         struct ion_handle *handle = container_of(kref, struct ion_handle, ref);
237         struct ion_client *client = handle->client;
238         struct ion_buffer *buffer = handle->buffer;
240         mutex_lock(&buffer->lock);
241         while (handle->kmap_cnt)
242                 ion_handle_kmap_put(handle);
243         mutex_unlock(&buffer->lock);
245         if (!RB_EMPTY_NODE(&handle->node))
246                 rb_erase(&handle->node, &client->handles);
248         ion_buffer_remove_from_handle(buffer);
249         ion_buffer_put(buffer);
251         kfree(handle);
254 struct ion_buffer *ion_handle_buffer(struct ion_handle *handle)
256         return handle->buffer;
259 static void ion_handle_get(struct ion_handle *handle)
261         kref_get(&handle->ref);
264 static int ion_handle_put(struct ion_handle *handle)
266         return kref_put(&handle->ref, ion_handle_destroy);
269 static struct ion_handle *ion_handle_lookup(struct ion_client *client,
270                                             struct ion_buffer *buffer)
272         struct rb_node *n;
274         for (n = rb_first(&client->handles); n; n = rb_next(n)) {
275                 struct ion_handle *handle = rb_entry(n, struct ion_handle,
276                                                      node);
277                 if (handle->buffer == buffer)
278                         return handle;
279         }
280         return NULL;
283 static bool ion_handle_validate(struct ion_client *client, struct ion_handle *handle)
285         struct rb_node *n = client->handles.rb_node;
287         while (n) {
288                 struct ion_handle *handle_node = rb_entry(n, struct ion_handle,
289                                                           node);
290                 if (handle < handle_node)
291                         n = n->rb_left;
292                 else if (handle > handle_node)
293                         n = n->rb_right;
294                 else
295                         return true;
296         }
297         return false;
300 static void ion_handle_add(struct ion_client *client, struct ion_handle *handle)
302         struct rb_node **p = &client->handles.rb_node;
303         struct rb_node *parent = NULL;
304         struct ion_handle *entry;
306         while (*p) {
307                 parent = *p;
308                 entry = rb_entry(parent, struct ion_handle, node);
310                 if (handle < entry)
311                         p = &(*p)->rb_left;
312                 else if (handle > entry)
313                         p = &(*p)->rb_right;
314                 else
315                         WARN(1, "%s: buffer already found.", __func__);
316         }
318         rb_link_node(&handle->node, parent, p);
319         rb_insert_color(&handle->node, &client->handles);
322 struct ion_handle *ion_alloc(struct ion_client *client, size_t len,
323                              size_t align, unsigned int heap_id_mask,
324                              unsigned int flags)
326         struct ion_handle *handle;
327         struct ion_device *dev = client->dev;
328         struct ion_buffer *buffer = NULL;
329         struct ion_heap *heap;
331         pr_debug("%s: len %d align %d heap_id_mask %u flags %x\n", __func__,
332                  len, align, heap_id_mask, flags);
333         /*
334          * traverse the list of heaps available in this system in priority
335          * order.  If the heap type is supported by the client, and matches the
336          * request of the caller allocate from it.  Repeat until allocate has
337          * succeeded or all heaps have been tried
338          */
339         if (WARN_ON(!len))
340                 return ERR_PTR(-EINVAL);
342         len = PAGE_ALIGN(len);
344         down_read(&dev->lock);
345         plist_for_each_entry(heap, &dev->heaps, node) {
346                 /* if the caller didn't specify this heap id */
347                 if (!((1 << heap->id) & heap_id_mask))
348                         continue;
349                 buffer = ion_buffer_create(heap, dev, len, align, flags);
350                 if (!IS_ERR_OR_NULL(buffer))
351                         break;
352         }
353         up_read(&dev->lock);
355         if (buffer == NULL)
356                 return ERR_PTR(-ENODEV);
358         if (IS_ERR(buffer))
359                 return ERR_PTR(PTR_ERR(buffer));
361         handle = ion_handle_create(client, buffer);
363         /*
364          * ion_buffer_create will create a buffer with a ref_cnt of 1,
365          * and ion_handle_create will take a second reference, drop one here
366          */
367         ion_buffer_put(buffer);
369         if (!IS_ERR(handle)) {
370                 mutex_lock(&client->lock);
371                 ion_handle_add(client, handle);
372                 mutex_unlock(&client->lock);
373         }
376         return handle;
378 EXPORT_SYMBOL(ion_alloc);
380 void ion_free(struct ion_client *client, struct ion_handle *handle)
382         bool valid_handle;
384         BUG_ON(client != handle->client);
386         mutex_lock(&client->lock);
387         valid_handle = ion_handle_validate(client, handle);
389         if (!valid_handle) {
390                 WARN(1, "%s: invalid handle passed to free.\n", __func__);
391                 mutex_unlock(&client->lock);
392                 return;
393         }
394         ion_handle_put(handle);
395         mutex_unlock(&client->lock);
397 EXPORT_SYMBOL(ion_free);
399 int ion_phys(struct ion_client *client, struct ion_handle *handle,
400              ion_phys_addr_t *addr, size_t *len)
402         struct ion_buffer *buffer;
403         int ret;
405         mutex_lock(&client->lock);
406         if (!ion_handle_validate(client, handle)) {
407                 mutex_unlock(&client->lock);
408                 return -EINVAL;
409         }
411         buffer = handle->buffer;
413         if (!buffer->heap->ops->phys) {
414                 pr_err("%s: ion_phys is not implemented by this heap.\n",
415                        __func__);
416                 mutex_unlock(&client->lock);
417                 return -ENODEV;
418         }
419         mutex_unlock(&client->lock);
420         ret = buffer->heap->ops->phys(buffer->heap, buffer, addr, len);
421         return ret;
423 EXPORT_SYMBOL(ion_phys);
425 static void *ion_buffer_kmap_get(struct ion_buffer *buffer)
427         void *vaddr;
429         if (buffer->kmap_cnt) {
430                 buffer->kmap_cnt++;
431                 return buffer->vaddr;
432         }
433         vaddr = buffer->heap->ops->map_kernel(buffer->heap, buffer);
434         if (IS_ERR_OR_NULL(vaddr))
435                 return vaddr;
436         buffer->vaddr = vaddr;
437         buffer->kmap_cnt++;
438         return vaddr;
441 static void *ion_handle_kmap_get(struct ion_handle *handle)
443         struct ion_buffer *buffer = handle->buffer;
444         void *vaddr;
446         if (handle->kmap_cnt) {
447                 handle->kmap_cnt++;
448                 return buffer->vaddr;
449         }
450         vaddr = ion_buffer_kmap_get(buffer);
451         if (IS_ERR_OR_NULL(vaddr))
452                 return vaddr;
453         handle->kmap_cnt++;
454         return vaddr;
457 static void ion_buffer_kmap_put(struct ion_buffer *buffer)
459         buffer->kmap_cnt--;
460         if (!buffer->kmap_cnt) {
461                 buffer->heap->ops->unmap_kernel(buffer->heap, buffer);
462                 buffer->vaddr = NULL;
463         }
466 static void ion_handle_kmap_put(struct ion_handle *handle)
468         struct ion_buffer *buffer = handle->buffer;
470         handle->kmap_cnt--;
471         if (!handle->kmap_cnt)
472                 ion_buffer_kmap_put(buffer);
475 void *ion_map_kernel(struct ion_client *client, struct ion_handle *handle)
477         struct ion_buffer *buffer;
478         void *vaddr;
480         mutex_lock(&client->lock);
481         if (!ion_handle_validate(client, handle)) {
482                 pr_err("%s: invalid handle passed to map_kernel.\n",
483                        __func__);
484                 mutex_unlock(&client->lock);
485                 return ERR_PTR(-EINVAL);
486         }
488         buffer = handle->buffer;
490         if (!handle->buffer->heap->ops->map_kernel) {
491                 pr_err("%s: map_kernel is not implemented by this heap.\n",
492                        __func__);
493                 mutex_unlock(&client->lock);
494                 return ERR_PTR(-ENODEV);
495         }
497         mutex_lock(&buffer->lock);
498         vaddr = ion_handle_kmap_get(handle);
499         mutex_unlock(&buffer->lock);
500         mutex_unlock(&client->lock);
501         return vaddr;
503 EXPORT_SYMBOL(ion_map_kernel);
505 void ion_unmap_kernel(struct ion_client *client, struct ion_handle *handle)
507         struct ion_buffer *buffer;
509         mutex_lock(&client->lock);
510         buffer = handle->buffer;
511         mutex_lock(&buffer->lock);
512         ion_handle_kmap_put(handle);
513         mutex_unlock(&buffer->lock);
514         mutex_unlock(&client->lock);
516 EXPORT_SYMBOL(ion_unmap_kernel);
518 static int ion_debug_client_show(struct seq_file *s, void *unused)
520         struct ion_client *client = s->private;
521         struct rb_node *n;
522         size_t sizes[ION_NUM_HEAP_IDS] = {0};
523         const char *names[ION_NUM_HEAP_IDS] = {0};
524         int i;
526         mutex_lock(&client->lock);
527         for (n = rb_first(&client->handles); n; n = rb_next(n)) {
528                 struct ion_handle *handle = rb_entry(n, struct ion_handle,
529                                                      node);
530                 unsigned int id = handle->buffer->heap->id;
532                 if (!names[id])
533                         names[id] = handle->buffer->heap->name;
534                 sizes[id] += handle->buffer->size;
535         }
536         mutex_unlock(&client->lock);
538         seq_printf(s, "%16.16s: %16.16s\n", "heap_name", "size_in_bytes");
539         for (i = 0; i < ION_NUM_HEAP_IDS; i++) {
540                 if (!names[i])
541                         continue;
542                 seq_printf(s, "%16.16s: %16u\n", names[i], sizes[i]);
543         }
544         return 0;
547 static int ion_debug_client_open(struct inode *inode, struct file *file)
549         return single_open(file, ion_debug_client_show, inode->i_private);
552 static const struct file_operations debug_client_fops = {
553         .open = ion_debug_client_open,
554         .read = seq_read,
555         .llseek = seq_lseek,
556         .release = single_release,
557 };
559 struct ion_client *ion_client_create(struct ion_device *dev,
560                                      const char *name)
562         struct ion_client *client;
563         struct task_struct *task;
564         struct rb_node **p;
565         struct rb_node *parent = NULL;
566         struct ion_client *entry;
567         char debug_name[64];
568         pid_t pid;
570         get_task_struct(current->group_leader);
571         task_lock(current->group_leader);
572         pid = task_pid_nr(current->group_leader);
573         /* don't bother to store task struct for kernel threads,
574            they can't be killed anyway */
575         if (current->group_leader->flags & PF_KTHREAD) {
576                 put_task_struct(current->group_leader);
577                 task = NULL;
578         } else {
579                 task = current->group_leader;
580         }
581         task_unlock(current->group_leader);
583         client = kzalloc(sizeof(struct ion_client), GFP_KERNEL);
584         if (!client) {
585                 if (task)
586                         put_task_struct(current->group_leader);
587                 return ERR_PTR(-ENOMEM);
588         }
590         client->dev = dev;
591         client->handles = RB_ROOT;
592         mutex_init(&client->lock);
593         client->name = name;
594         client->task = task;
595         client->pid = pid;
597         down_write(&dev->lock);
598         p = &dev->clients.rb_node;
599         while (*p) {
600                 parent = *p;
601                 entry = rb_entry(parent, struct ion_client, node);
603                 if (client < entry)
604                         p = &(*p)->rb_left;
605                 else if (client > entry)
606                         p = &(*p)->rb_right;
607         }
608         rb_link_node(&client->node, parent, p);
609         rb_insert_color(&client->node, &dev->clients);
611         snprintf(debug_name, 64, "%u", client->pid);
612         client->debug_root = debugfs_create_file(debug_name, 0664,
613                                                  dev->debug_root, client,
614                                                  &debug_client_fops);
615         up_write(&dev->lock);
617         return client;
619 EXPORT_SYMBOL(ion_client_create);
621 void ion_client_destroy(struct ion_client *client)
623         struct ion_device *dev = client->dev;
624         struct rb_node *n;
626         pr_debug("%s: %d\n", __func__, __LINE__);
627         while ((n = rb_first(&client->handles))) {
628                 struct ion_handle *handle = rb_entry(n, struct ion_handle,
629                                                      node);
630                 ion_handle_destroy(&handle->ref);
631         }
632         down_write(&dev->lock);
633         if (client->task)
634                 put_task_struct(client->task);
635         rb_erase(&client->node, &dev->clients);
636         debugfs_remove_recursive(client->debug_root);
637         up_write(&dev->lock);
639         kfree(client);
641 EXPORT_SYMBOL(ion_client_destroy);
643 struct sg_table *ion_sg_table(struct ion_client *client,
644                               struct ion_handle *handle)
646         struct ion_buffer *buffer;
647         struct sg_table *table;
649         mutex_lock(&client->lock);
650         if (!ion_handle_validate(client, handle)) {
651                 pr_err("%s: invalid handle passed to map_dma.\n",
652                        __func__);
653                 mutex_unlock(&client->lock);
654                 return ERR_PTR(-EINVAL);
655         }
656         buffer = handle->buffer;
657         table = buffer->sg_table;
658         mutex_unlock(&client->lock);
659         return table;
661 EXPORT_SYMBOL(ion_sg_table);
663 static void ion_buffer_sync_for_device(struct ion_buffer *buffer,
664                                        struct device *dev,
665                                        enum dma_data_direction direction);
667 static struct sg_table *ion_map_dma_buf(struct dma_buf_attachment *attachment,
668                                         enum dma_data_direction direction)
670         struct dma_buf *dmabuf = attachment->dmabuf;
671         struct ion_buffer *buffer = dmabuf->priv;
673         ion_buffer_sync_for_device(buffer, attachment->dev, direction);
674         return buffer->sg_table;
677 static void ion_unmap_dma_buf(struct dma_buf_attachment *attachment,
678                               struct sg_table *table,
679                               enum dma_data_direction direction)
683 static int ion_buffer_alloc_dirty(struct ion_buffer *buffer)
685         unsigned long pages = buffer->sg_table->nents;
686         unsigned long length = (pages + BITS_PER_LONG - 1)/BITS_PER_LONG;
688         buffer->dirty = kzalloc(length * sizeof(unsigned long), GFP_KERNEL);
689         if (!buffer->dirty)
690                 return -ENOMEM;
691         return 0;
694 struct ion_vma_list {
695         struct list_head list;
696         struct vm_area_struct *vma;
697 };
699 static void ion_buffer_sync_for_device(struct ion_buffer *buffer,
700                                        struct device *dev,
701                                        enum dma_data_direction dir)
703         struct scatterlist *sg;
704         int i;
705         struct ion_vma_list *vma_list;
707         pr_debug("%s: syncing for device %s\n", __func__,
708                  dev ? dev_name(dev) : "null");
710         if (!ion_buffer_fault_user_mappings(buffer))
711                 return;
713         mutex_lock(&buffer->lock);
714         for_each_sg(buffer->sg_table->sgl, sg, buffer->sg_table->nents, i) {
715                 if (!test_bit(i, buffer->dirty))
716                         continue;
717                 dma_sync_sg_for_device(dev, sg, 1, dir);
718                 clear_bit(i, buffer->dirty);
719         }
720         list_for_each_entry(vma_list, &buffer->vmas, list) {
721                 struct vm_area_struct *vma = vma_list->vma;
723                 zap_page_range(vma, vma->vm_start, vma->vm_end - vma->vm_start,
724                                NULL);
725         }
726         mutex_unlock(&buffer->lock);
729 int ion_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
731         struct ion_buffer *buffer = vma->vm_private_data;
732         struct scatterlist *sg;
733         int i;
735         mutex_lock(&buffer->lock);
736         set_bit(vmf->pgoff, buffer->dirty);
738         for_each_sg(buffer->sg_table->sgl, sg, buffer->sg_table->nents, i) {
739                 if (i != vmf->pgoff)
740                         continue;
741                 dma_sync_sg_for_cpu(NULL, sg, 1, DMA_BIDIRECTIONAL);
742                 vm_insert_page(vma, (unsigned long)vmf->virtual_address,
743                                sg_page(sg));
744                 break;
745         }
746         mutex_unlock(&buffer->lock);
747         return VM_FAULT_NOPAGE;
750 static void ion_vm_open(struct vm_area_struct *vma)
752         struct ion_buffer *buffer = vma->vm_private_data;
753         struct ion_vma_list *vma_list;
755         vma_list = kmalloc(sizeof(struct ion_vma_list), GFP_KERNEL);
756         if (!vma_list)
757                 return;
758         vma_list->vma = vma;
759         mutex_lock(&buffer->lock);
760         list_add(&vma_list->list, &buffer->vmas);
761         mutex_unlock(&buffer->lock);
762         pr_debug("%s: adding %p\n", __func__, vma);
765 static void ion_vm_close(struct vm_area_struct *vma)
767         struct ion_buffer *buffer = vma->vm_private_data;
768         struct ion_vma_list *vma_list, *tmp;
770         pr_debug("%s\n", __func__);
771         mutex_lock(&buffer->lock);
772         list_for_each_entry_safe(vma_list, tmp, &buffer->vmas, list) {
773                 if (vma_list->vma != vma)
774                         continue;
775                 list_del(&vma_list->list);
776                 kfree(vma_list);
777                 pr_debug("%s: deleting %p\n", __func__, vma);
778                 break;
779         }
780         mutex_unlock(&buffer->lock);
783 struct vm_operations_struct ion_vma_ops = {
784         .open = ion_vm_open,
785         .close = ion_vm_close,
786         .fault = ion_vm_fault,
787 };
789 static int ion_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
791         struct ion_buffer *buffer = dmabuf->priv;
792         int ret = 0;
794         if (!buffer->heap->ops->map_user) {
795                 pr_err("%s: this heap does not define a method for mapping "
796                        "to userspace\n", __func__);
797                 return -EINVAL;
798         }
800         if (ion_buffer_fault_user_mappings(buffer)) {
801                 vma->vm_private_data = buffer;
802                 vma->vm_ops = &ion_vma_ops;
803                 ion_vm_open(vma);
804                 return 0;
805         }
807         if (!(buffer->flags & ION_FLAG_CACHED))
808                 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
810         mutex_lock(&buffer->lock);
811         /* now map it to userspace */
812         ret = buffer->heap->ops->map_user(buffer->heap, buffer, vma);
813         mutex_unlock(&buffer->lock);
815         if (ret)
816                 pr_err("%s: failure mapping buffer to userspace\n",
817                        __func__);
819         return ret;
822 static void ion_dma_buf_release(struct dma_buf *dmabuf)
824         struct ion_buffer *buffer = dmabuf->priv;
825         ion_buffer_put(buffer);
828 static void *ion_dma_buf_kmap(struct dma_buf *dmabuf, unsigned long offset)
830         struct ion_buffer *buffer = dmabuf->priv;
831         return buffer->vaddr + offset * PAGE_SIZE;
834 static void ion_dma_buf_kunmap(struct dma_buf *dmabuf, unsigned long offset,
835                                void *ptr)
837         return;
840 static int ion_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, size_t start,
841                                         size_t len,
842                                         enum dma_data_direction direction)
844         struct ion_buffer *buffer = dmabuf->priv;
845         void *vaddr;
847         if (!buffer->heap->ops->map_kernel) {
848                 pr_err("%s: map kernel is not implemented by this heap.\n",
849                        __func__);
850                 return -ENODEV;
851         }
853         mutex_lock(&buffer->lock);
854         vaddr = ion_buffer_kmap_get(buffer);
855         mutex_unlock(&buffer->lock);
856         if (IS_ERR(vaddr))
857                 return PTR_ERR(vaddr);
858         if (!vaddr)
859                 return -ENOMEM;
860         return 0;
863 static void ion_dma_buf_end_cpu_access(struct dma_buf *dmabuf, size_t start,
864                                        size_t len,
865                                        enum dma_data_direction direction)
867         struct ion_buffer *buffer = dmabuf->priv;
869         mutex_lock(&buffer->lock);
870         ion_buffer_kmap_put(buffer);
871         mutex_unlock(&buffer->lock);
874 struct dma_buf_ops dma_buf_ops = {
875         .map_dma_buf = ion_map_dma_buf,
876         .unmap_dma_buf = ion_unmap_dma_buf,
877         .mmap = ion_mmap,
878         .release = ion_dma_buf_release,
879         .begin_cpu_access = ion_dma_buf_begin_cpu_access,
880         .end_cpu_access = ion_dma_buf_end_cpu_access,
881         .kmap_atomic = ion_dma_buf_kmap,
882         .kunmap_atomic = ion_dma_buf_kunmap,
883         .kmap = ion_dma_buf_kmap,
884         .kunmap = ion_dma_buf_kunmap,
885 };
887 int ion_share_dma_buf(struct ion_client *client, struct ion_handle *handle)
889         struct ion_buffer *buffer;
890         struct dma_buf *dmabuf;
891         bool valid_handle;
892         int fd;
894         mutex_lock(&client->lock);
895         valid_handle = ion_handle_validate(client, handle);
896         mutex_unlock(&client->lock);
897         if (!valid_handle) {
898                 WARN(1, "%s: invalid handle passed to share.\n", __func__);
899                 return -EINVAL;
900         }
902         buffer = handle->buffer;
903         ion_buffer_get(buffer);
904         dmabuf = dma_buf_export(buffer, &dma_buf_ops, buffer->size, O_RDWR);
905         if (IS_ERR(dmabuf)) {
906                 ion_buffer_put(buffer);
907                 return PTR_ERR(dmabuf);
908         }
909         fd = dma_buf_fd(dmabuf, O_CLOEXEC);
910         if (fd < 0)
911                 dma_buf_put(dmabuf);
913         return fd;
915 EXPORT_SYMBOL(ion_share_dma_buf);
917 struct ion_handle *ion_import_dma_buf(struct ion_client *client, int fd)
919         struct dma_buf *dmabuf;
920         struct ion_buffer *buffer;
921         struct ion_handle *handle;
923         dmabuf = dma_buf_get(fd);
924         if (IS_ERR_OR_NULL(dmabuf))
925                 return ERR_PTR(PTR_ERR(dmabuf));
926         /* if this memory came from ion */
928         if (dmabuf->ops != &dma_buf_ops) {
929                 pr_err("%s: can not import dmabuf from another exporter\n",
930                        __func__);
931                 dma_buf_put(dmabuf);
932                 return ERR_PTR(-EINVAL);
933         }
934         buffer = dmabuf->priv;
936         mutex_lock(&client->lock);
937         /* if a handle exists for this buffer just take a reference to it */
938         handle = ion_handle_lookup(client, buffer);
939         if (!IS_ERR_OR_NULL(handle)) {
940                 ion_handle_get(handle);
941                 goto end;
942         }
943         handle = ion_handle_create(client, buffer);
944         if (IS_ERR_OR_NULL(handle))
945                 goto end;
946         ion_handle_add(client, handle);
947 end:
948         mutex_unlock(&client->lock);
949         dma_buf_put(dmabuf);
950         return handle;
952 EXPORT_SYMBOL(ion_import_dma_buf);
954 static int ion_sync_for_device(struct ion_client *client, int fd)
956         struct dma_buf *dmabuf;
957         struct ion_buffer *buffer;
959         dmabuf = dma_buf_get(fd);
960         if (IS_ERR_OR_NULL(dmabuf))
961                 return PTR_ERR(dmabuf);
963         /* if this memory came from ion */
964         if (dmabuf->ops != &dma_buf_ops) {
965                 pr_err("%s: can not sync dmabuf from another exporter\n",
966                        __func__);
967                 dma_buf_put(dmabuf);
968                 return -EINVAL;
969         }
970         buffer = dmabuf->priv;
972         dma_sync_sg_for_device(NULL, buffer->sg_table->sgl,
973                                buffer->sg_table->nents, DMA_BIDIRECTIONAL);
974         dma_buf_put(dmabuf);
975         return 0;
978 static long ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
980         struct ion_client *client = filp->private_data;
982         switch (cmd) {
983         case ION_IOC_ALLOC:
984         {
985                 struct ion_allocation_data data;
987                 if (copy_from_user(&data, (void __user *)arg, sizeof(data)))
988                         return -EFAULT;
989                 data.handle = ion_alloc(client, data.len, data.align,
990                                              data.heap_id_mask, data.flags);
992                 if (IS_ERR(data.handle))
993                         return PTR_ERR(data.handle);
995                 if (copy_to_user((void __user *)arg, &data, sizeof(data))) {
996                         ion_free(client, data.handle);
997                         return -EFAULT;
998                 }
999                 break;
1000         }
1001         case ION_IOC_FREE:
1002         {
1003                 struct ion_handle_data data;
1004                 bool valid;
1006                 if (copy_from_user(&data, (void __user *)arg,
1007                                    sizeof(struct ion_handle_data)))
1008                         return -EFAULT;
1009                 mutex_lock(&client->lock);
1010                 valid = ion_handle_validate(client, data.handle);
1011                 mutex_unlock(&client->lock);
1012                 if (!valid)
1013                         return -EINVAL;
1014                 ion_free(client, data.handle);
1015                 break;
1016         }
1017         case ION_IOC_SHARE:
1018         case ION_IOC_MAP:
1019         {
1020                 struct ion_fd_data data;
1022                 if (copy_from_user(&data, (void __user *)arg, sizeof(data)))
1023                         return -EFAULT;
1024                 data.fd = ion_share_dma_buf(client, data.handle);
1025                 if (copy_to_user((void __user *)arg, &data, sizeof(data)))
1026                         return -EFAULT;
1027                 if (data.fd < 0)
1028                         return data.fd;
1029                 break;
1030         }
1031         case ION_IOC_IMPORT:
1032         {
1033                 struct ion_fd_data data;
1034                 int ret = 0;
1035                 if (copy_from_user(&data, (void __user *)arg,
1036                                    sizeof(struct ion_fd_data)))
1037                         return -EFAULT;
1038                 data.handle = ion_import_dma_buf(client, data.fd);
1039                 if (IS_ERR(data.handle)) {
1040                         ret = PTR_ERR(data.handle);
1041                         data.handle = NULL;
1042                 }
1043                 if (copy_to_user((void __user *)arg, &data,
1044                                  sizeof(struct ion_fd_data)))
1045                         return -EFAULT;
1046                 if (ret < 0)
1047                         return ret;
1048                 break;
1049         }
1050         case ION_IOC_SYNC:
1051         {
1052                 struct ion_fd_data data;
1053                 if (copy_from_user(&data, (void __user *)arg,
1054                                    sizeof(struct ion_fd_data)))
1055                         return -EFAULT;
1056                 ion_sync_for_device(client, data.fd);
1057                 break;
1058         }
1059         case ION_IOC_CUSTOM:
1060         {
1061                 struct ion_device *dev = client->dev;
1062                 struct ion_custom_data data;
1064                 if (!dev->custom_ioctl)
1065                         return -ENOTTY;
1066                 if (copy_from_user(&data, (void __user *)arg,
1067                                 sizeof(struct ion_custom_data)))
1068                         return -EFAULT;
1069                 return dev->custom_ioctl(client, data.cmd, data.arg);
1070         }
1071         default:
1072                 return -ENOTTY;
1073         }
1074         return 0;
1077 static int ion_release(struct inode *inode, struct file *file)
1079         struct ion_client *client = file->private_data;
1081         pr_debug("%s: %d\n", __func__, __LINE__);
1082         ion_client_destroy(client);
1083         return 0;
1086 static int ion_open(struct inode *inode, struct file *file)
1088         struct miscdevice *miscdev = file->private_data;
1089         struct ion_device *dev = container_of(miscdev, struct ion_device, dev);
1090         struct ion_client *client;
1092         pr_debug("%s: %d\n", __func__, __LINE__);
1093         client = ion_client_create(dev, "user");
1094         if (IS_ERR_OR_NULL(client))
1095                 return PTR_ERR(client);
1096         file->private_data = client;
1098         return 0;
1101 static const struct file_operations ion_fops = {
1102         .owner          = THIS_MODULE,
1103         .open           = ion_open,
1104         .release        = ion_release,
1105         .unlocked_ioctl = ion_ioctl,
1106 };
1108 static size_t ion_debug_heap_total(struct ion_client *client,
1109                                    unsigned int id)
1111         size_t size = 0;
1112         struct rb_node *n;
1114         mutex_lock(&client->lock);
1115         for (n = rb_first(&client->handles); n; n = rb_next(n)) {
1116                 struct ion_handle *handle = rb_entry(n,
1117                                                      struct ion_handle,
1118                                                      node);
1119                 if (handle->buffer->heap->id == id)
1120                         size += handle->buffer->size;
1121         }
1122         mutex_unlock(&client->lock);
1123         return size;
1126 static int ion_debug_heap_show(struct seq_file *s, void *unused)
1128         struct ion_heap *heap = s->private;
1129         struct ion_device *dev = heap->dev;
1130         struct rb_node *n;
1131         size_t total_size = 0;
1132         size_t total_orphaned_size = 0;
1134         seq_printf(s, "%16.s %16.s %16.s\n", "client", "pid", "size");
1135         seq_printf(s, "----------------------------------------------------\n");
1137         for (n = rb_first(&dev->clients); n; n = rb_next(n)) {
1138                 struct ion_client *client = rb_entry(n, struct ion_client,
1139                                                      node);
1140                 size_t size = ion_debug_heap_total(client, heap->id);
1141                 if (!size)
1142                         continue;
1143                 if (client->task) {
1144                         char task_comm[TASK_COMM_LEN];
1146                         get_task_comm(task_comm, client->task);
1147                         seq_printf(s, "%16.s %16u %16u\n", task_comm,
1148                                    client->pid, size);
1149                 } else {
1150                         seq_printf(s, "%16.s %16u %16u\n", client->name,
1151                                    client->pid, size);
1152                 }
1153         }
1154         seq_printf(s, "----------------------------------------------------\n");
1155         seq_printf(s, "orphaned allocations (info is from last known client):"
1156                    "\n");
1157         mutex_lock(&dev->buffer_lock);
1158         for (n = rb_first(&dev->buffers); n; n = rb_next(n)) {
1159                 struct ion_buffer *buffer = rb_entry(n, struct ion_buffer,
1160                                                      node);
1161                 if (buffer->heap->id != heap->id)
1162                         continue;
1163                 total_size += buffer->size;
1164                 if (!buffer->handle_count) {
1165                         seq_printf(s, "%16.s %16u %16u %d %d\n", buffer->task_comm,
1166                                    buffer->pid, buffer->size, buffer->kmap_cnt,
1167                                    atomic_read(&buffer->ref.refcount));
1168                         total_orphaned_size += buffer->size;
1169                 }
1170         }
1171         mutex_unlock(&dev->buffer_lock);
1172         seq_printf(s, "----------------------------------------------------\n");
1173         seq_printf(s, "%16.s %16u\n", "total orphaned",
1174                    total_orphaned_size);
1175         seq_printf(s, "%16.s %16u\n", "total ", total_size);
1176         seq_printf(s, "----------------------------------------------------\n");
1178         if (heap->debug_show)
1179                 heap->debug_show(heap, s, unused);
1181         return 0;
1184 static int ion_debug_heap_open(struct inode *inode, struct file *file)
1186         return single_open(file, ion_debug_heap_show, inode->i_private);
1189 static const struct file_operations debug_heap_fops = {
1190         .open = ion_debug_heap_open,
1191         .read = seq_read,
1192         .llseek = seq_lseek,
1193         .release = single_release,
1194 };
1196 void ion_device_add_heap(struct ion_device *dev, struct ion_heap *heap)
1198         if (!heap->ops->allocate || !heap->ops->free || !heap->ops->map_dma ||
1199             !heap->ops->unmap_dma)
1200                 pr_err("%s: can not add heap with invalid ops struct.\n",
1201                        __func__);
1203         heap->dev = dev;
1204         down_write(&dev->lock);
1205         /* use negative heap->id to reverse the priority -- when traversing
1206            the list later attempt higher id numbers first */
1207         plist_node_init(&heap->node, -heap->id);
1208         plist_add(&heap->node, &dev->heaps);
1209         debugfs_create_file(heap->name, 0664, dev->debug_root, heap,
1210                             &debug_heap_fops);
1211         up_write(&dev->lock);
1214 struct ion_device *ion_device_create(long (*custom_ioctl)
1215                                      (struct ion_client *client,
1216                                       unsigned int cmd,
1217                                       unsigned long arg))
1219         struct ion_device *idev;
1220         int ret;
1222         idev = kzalloc(sizeof(struct ion_device), GFP_KERNEL);
1223         if (!idev)
1224                 return ERR_PTR(-ENOMEM);
1226         idev->dev.minor = MISC_DYNAMIC_MINOR;
1227         idev->dev.name = "ion";
1228         idev->dev.fops = &ion_fops;
1229         idev->dev.parent = NULL;
1230         ret = misc_register(&idev->dev);
1231         if (ret) {
1232                 pr_err("ion: failed to register misc device.\n");
1233                 return ERR_PTR(ret);
1234         }
1236         idev->debug_root = debugfs_create_dir("ion", NULL);
1237         if (IS_ERR_OR_NULL(idev->debug_root))
1238                 pr_err("ion: failed to create debug files.\n");
1240         idev->custom_ioctl = custom_ioctl;
1241         idev->buffers = RB_ROOT;
1242         mutex_init(&idev->buffer_lock);
1243         init_rwsem(&idev->lock);
1244         plist_head_init(&idev->heaps);
1245         idev->clients = RB_ROOT;
1246         return idev;
1249 void ion_device_destroy(struct ion_device *dev)
1251         misc_deregister(&dev->dev);
1252         /* XXX need to free the heaps and clients ? */
1253         kfree(dev);
1256 void __init ion_reserve(struct ion_platform_data *data)
1258         int i;
1260         for (i = 0; i < data->nr; i++) {
1261                 if (data->heaps[i].size == 0)
1262                         continue;
1264                 if (data->heaps[i].base == 0) {
1265                         phys_addr_t paddr;
1266                         paddr = memblock_alloc_base(data->heaps[i].size,
1267                                                     data->heaps[i].align,
1268                                                     MEMBLOCK_ALLOC_ANYWHERE);
1269                         if (!paddr) {
1270                                 pr_err("%s: error allocating memblock for "
1271                                        "heap %d\n",
1272                                         __func__, i);
1273                                 continue;
1274                         }
1275                         data->heaps[i].base = paddr;
1276                 } else {
1277                         int ret = memblock_reserve(data->heaps[i].base,
1278                                                data->heaps[i].size);
1279                         if (ret)
1280                                 pr_err("memblock reserve of %x@%lx failed\n",
1281                                        data->heaps[i].size,
1282                                        data->heaps[i].base);
1283                 }
1284                 pr_info("%s: %s reserved base %lx size %d\n", __func__,
1285                         data->heaps[i].name,
1286                         data->heaps[i].base,
1287                         data->heaps[i].size);
1288         }