1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * TI Keystone DSP remoteproc driver
4 *
5 * Copyright (C) 2015-2019 Texas Instruments Incorporated - http://www.ti.com/
6 */
8 #include <linux/module.h>
9 #include <linux/uaccess.h>
10 #include <linux/slab.h>
11 #include <linux/io.h>
12 #include <linux/interrupt.h>
13 #include <linux/spinlock.h>
14 #include <linux/platform_device.h>
15 #include <linux/pm_runtime.h>
16 #include <linux/workqueue.h>
17 #include <linux/of_address.h>
18 #include <linux/of_reserved_mem.h>
19 #include <linux/of_gpio.h>
20 #include <linux/regmap.h>
21 #include <linux/mfd/syscon.h>
22 #include <linux/remoteproc.h>
23 #include <linux/miscdevice.h>
24 #include <linux/uio_driver.h>
25 #include <linux/reset.h>
27 #include <uapi/linux/keystone_remoteproc.h>
29 #include "remoteproc_internal.h"
31 #define DRIVER_UIO_VERSION "0.1"
33 #define KEYSTONE_RPROC_MAX_RSC_TABLE SZ_1K
34 #define KEYSTONE_RPROC_LOCAL_ADDRESS_MASK (SZ_16M - 1)
36 /*
37 * XXX: evaluate if this param needs to be enhanced so that the switch between
38 * userspace and remoteproc core loaders can be controlled per device.
39 */
40 static bool use_rproc_core_loader;
41 module_param(use_rproc_core_loader, bool, 0444);
43 /**
44 * struct keystone_rproc_mem - internal memory structure
45 * @cpu_addr: MPU virtual address of the memory region
46 * @bus_addr: Bus address used to access the memory region
47 * @dev_addr: Device address of the memory region from DSP view
48 * @size: Size of the memory region
49 * @kobj: kobject for the sysfs directory file
50 */
51 struct keystone_rproc_mem {
52 void __iomem *cpu_addr;
53 phys_addr_t bus_addr;
54 u32 dev_addr;
55 size_t size;
56 struct kobject kobj;
57 };
59 /**
60 * struct keystone_rproc - keystone remote processor driver structure
61 * @dev: cached device pointer
62 * @rproc: remoteproc device handle
63 * @mem: internal memory regions data
64 * @num_mems: number of internal memory regions
65 * @dev_ctrl: device control regmap handle
66 * @reset: reset control handle
67 * @boot_offset: boot register offset in @dev_ctrl regmap
68 * @irq_ring: irq entry for vring
69 * @irq_fault: irq entry for exception
70 * @kick_gpio: gpio used for virtio kicks
71 * @workqueue: workqueue for processing virtio interrupts
72 * @misc: misc device structure used to expose fops to user-space
73 * @uio: uio device information
74 * @mlock: lock to protect resources in fops
75 * @lock: lock to protect shared resources within UIO interrupt handlers
76 * @flags: flags to keep track of UIO interrupt occurrence
77 * @rsc_table: resource table pointer copied from userspace
78 * @rsc_table_size: size of resource table
79 * @loaded_rsc_table: kernel pointer of loaded resource table
80 * @boot_addr: remote processor boot address used with userspace loader
81 * @open_count: fops open reference counter
82 * @use_userspace_loader: flag to denote if driver is configured for userspace
83 * loader
84 */
85 struct keystone_rproc {
86 struct device *dev;
87 struct rproc *rproc;
88 struct keystone_rproc_mem *mem;
89 int num_mems;
90 struct regmap *dev_ctrl;
91 struct reset_control *reset;
92 u32 boot_offset;
93 int irq_ring;
94 int irq_fault;
95 int kick_gpio;
96 struct work_struct workqueue;
97 struct miscdevice misc;
98 struct uio_info uio;
99 struct mutex mlock; /* fops lock */
100 spinlock_t lock; /* uio handler lock */
101 unsigned long flags;
102 struct resource_table *rsc_table;
103 int rsc_table_size;
104 void *loaded_rsc_table;
105 u32 boot_addr;
106 int open_count;
107 unsigned int use_userspace_loader : 1;
108 };
110 struct mem_sysfs_entry {
111 struct attribute attr;
112 ssize_t (*show)(struct keystone_rproc_mem *mem, char *buf);
113 ssize_t (*store)(struct keystone_rproc_mem *mem, const char *buf,
114 size_t len);
115 };
117 static ssize_t mem_addr_show(struct keystone_rproc_mem *mem, char *buf)
118 {
119 return sprintf(buf, "%pa\n", &mem->bus_addr);
120 }
122 static ssize_t mem_size_show(struct keystone_rproc_mem *mem, char *buf)
123 {
124 return sprintf(buf, "0x%016zx\n", mem->size);
125 }
127 static struct mem_sysfs_entry addr_attribute =
128 __ATTR(addr, 0444, mem_addr_show, NULL);
129 static struct mem_sysfs_entry size_attribute =
130 __ATTR(size, 0444, mem_size_show, NULL);
132 static struct attribute *attrs[] = {
133 &addr_attribute.attr,
134 &size_attribute.attr,
135 NULL, /* sentinel */
136 };
138 #define to_dsp_mem(m) container_of(m, struct keystone_rproc_mem, kobj)
140 static ssize_t mem_type_show(struct kobject *kobj, struct attribute *attr,
141 char *buf)
142 {
143 struct keystone_rproc_mem *mem = to_dsp_mem(kobj);
144 struct mem_sysfs_entry *entry;
146 entry = container_of(attr, struct mem_sysfs_entry, attr);
147 if (!entry->show)
148 return -EIO;
150 return entry->show(mem, buf);
151 }
153 static const struct sysfs_ops mem_sysfs_ops = {
154 .show = mem_type_show,
155 };
157 static struct kobj_type mem_attr_type = {
158 .sysfs_ops = &mem_sysfs_ops,
159 .default_attrs = attrs,
160 };
162 static int keystone_rproc_mem_add_attrs(struct keystone_rproc *ksproc)
163 {
164 int i, ret;
165 struct keystone_rproc_mem *mem;
166 struct kobject *kobj_parent = &ksproc->misc.this_device->kobj;
168 for (i = 0; i < ksproc->num_mems; i++) {
169 mem = &ksproc->mem[i];
170 kobject_init(&mem->kobj, &mem_attr_type);
171 ret = kobject_add(&mem->kobj, kobj_parent, "memory%d", i);
172 if (ret)
173 goto err_kobj;
174 ret = kobject_uevent(&mem->kobj, KOBJ_ADD);
175 if (ret)
176 goto err_kobj;
177 }
179 return 0;
181 err_kobj:
182 for (; i >= 0; i--) {
183 mem = &ksproc->mem[i];
184 kobject_put(&mem->kobj);
185 }
186 return ret;
187 }
189 static void keystone_rproc_mem_del_attrs(struct keystone_rproc *ksproc)
190 {
191 int i;
192 struct keystone_rproc_mem *mem;
194 for (i = 0; i < ksproc->num_mems; i++) {
195 mem = &ksproc->mem[i];
196 kobject_put(&mem->kobj);
197 }
198 }
200 static void *keystone_rproc_da_to_va(struct rproc *rproc, u64 da, int len,
201 u32 flags);
203 /* uio handler dealing with userspace controlled exception interrupt */
204 static irqreturn_t keystone_rproc_uio_handler(int irq, struct uio_info *uio)
205 {
206 struct keystone_rproc *ksproc = uio->priv;
208 spin_lock(&ksproc->lock);
209 if (!__test_and_set_bit(0, &ksproc->flags))
210 disable_irq_nosync(irq);
211 spin_unlock(&ksproc->lock);
213 return IRQ_HANDLED;
214 }
216 /* uio driver interrupt control dealing with exception interrupt */
217 static int keystone_rproc_uio_irqcontrol(struct uio_info *uio, s32 irq_on)
218 {
219 struct keystone_rproc *ksproc = uio->priv;
220 unsigned long flags;
222 spin_lock_irqsave(&ksproc->lock, flags);
223 if (irq_on) {
224 if (__test_and_clear_bit(0, &ksproc->flags))
225 enable_irq(uio->irq);
226 } else {
227 if (!__test_and_set_bit(0, &ksproc->flags))
228 disable_irq(uio->irq);
229 }
230 spin_unlock_irqrestore(&ksproc->lock, flags);
232 return 0;
233 }
235 /* Reset previously set rsc table variables */
236 static void keystone_rproc_reset_rsc_table(struct keystone_rproc *ksproc)
237 {
238 kfree(ksproc->rsc_table);
239 ksproc->rsc_table = NULL;
240 ksproc->loaded_rsc_table = NULL;
241 ksproc->rsc_table_size = 0;
242 }
244 /*
245 * Create/delete the virtio devices in kernel once the user-space loading is
246 * complete, configure the remoteproc states appropriately, and boot or reset
247 * the remote processor. The resource table should have been published through
248 * KEYSTONE_RPROC_IOC_SET_RSC_TABLE & KEYSTONE_RPROC_IOC_SET_LOADED_RSC_TABLE
249 * ioctls before invoking this. The boot address is passed through the
250 * KEYSTONE_RPROC_IOC_SET_STATE ioctl when setting the KEYSTONE_RPROC_RUNNING
251 * state.
252 *
253 * NOTE:
254 * The ioctls KEYSTONE_RPROC_IOC_DSP_RESET and KEYSTONE_RPROC_IOC_DSP_BOOT
255 * are restricted to support the booting or resetting the DSP devices only
256 * for firmware images without any resource table.
257 */
258 static int keystone_rproc_set_state(struct keystone_rproc *ksproc,
259 void __user *argp)
260 {
261 struct rproc *rproc = ksproc->rproc;
262 struct keystone_rproc_set_state_params set_state_params;
263 int ret = 0;
265 if (copy_from_user(&set_state_params, argp, sizeof(set_state_params)))
266 return -EFAULT;
268 switch (set_state_params.state) {
269 case KEYSTONE_RPROC_RUNNING:
270 if (!ksproc->rsc_table || !ksproc->loaded_rsc_table)
271 return -EINVAL;
273 /*
274 * store boot address for .get_boot_addr() rproc fw ops
275 * XXX: validate the boot address so it is not set to a
276 * random address
277 */
278 ksproc->boot_addr = set_state_params.boot_addr;
280 /*
281 * invoke rproc_boot to trigger the boot, the resource table
282 * is parsed during the process and is agnostic of the presence
283 * or absence of virtio devices
284 */
285 ret = rproc_boot(rproc);
286 break;
288 case KEYSTONE_RPROC_OFFLINE:
289 if (rproc->state != RPROC_RUNNING)
290 return -EINVAL;
292 /* invoke rproc_shutdown to match rproc_boot */
293 rproc_shutdown(rproc);
295 mutex_lock(&ksproc->mlock);
296 keystone_rproc_reset_rsc_table(ksproc);
297 mutex_unlock(&ksproc->mlock);
299 break;
301 default:
302 ret = -ENOTSUPP;
303 }
305 return ret;
306 }
308 /* Copy the resource table from userspace into kernel */
309 static int keystone_rproc_set_rsc_table(struct keystone_rproc *ksproc,
310 void __user *data)
311 {
312 unsigned long len = 0;
313 void *rsc_table = NULL;
315 if (!data)
316 return -EFAULT;
318 if (copy_from_user(&len, data, sizeof(len)))
319 return -EFAULT;
321 if (len >= KEYSTONE_RPROC_MAX_RSC_TABLE)
322 return -EOVERFLOW;
324 data += sizeof(len);
326 rsc_table = kzalloc(len, GFP_KERNEL);
327 if (!rsc_table)
328 return -ENOMEM;
330 if (copy_from_user(rsc_table, data, len))
331 goto error_return;
333 mutex_lock(&ksproc->mlock);
335 kfree(ksproc->rsc_table);
337 ksproc->rsc_table = rsc_table;
338 ksproc->rsc_table_size = len;
339 ksproc->loaded_rsc_table = NULL;
341 mutex_unlock(&ksproc->mlock);
343 return 0;
345 error_return:
346 kfree(rsc_table);
347 return -EFAULT;
348 }
350 /*
351 * Store the equivalent kernel virtual address of the loaded resource table in
352 * device memory. Userspace published the device address of the loaded resource
353 * table.
354 */
355 static int keystone_rproc_set_loaded_rsc_table(struct keystone_rproc *ksproc,
356 unsigned int dma_addr)
357 {
358 struct rproc *rproc = ksproc->rproc;
359 void *ptr;
361 if (!ksproc->rsc_table_size || !ksproc->rsc_table)
362 return -EINVAL;
364 ptr = keystone_rproc_da_to_va(rproc, dma_addr, ksproc->rsc_table_size,
365 RPROC_FLAGS_NONE);
366 if (!ptr)
367 return -EINVAL;
369 ksproc->loaded_rsc_table = ptr;
371 return 0;
372 }
374 /* Put the DSP processor into reset */
375 static void keystone_rproc_dsp_reset(struct keystone_rproc *ksproc)
376 {
377 reset_control_assert(ksproc->reset);
378 }
380 /* Configure the boot address and boot the DSP processor */
381 static int keystone_rproc_dsp_boot(struct keystone_rproc *ksproc,
382 uint32_t boot_addr)
383 {
384 int ret;
386 if (boot_addr & (SZ_1K - 1)) {
387 dev_err(ksproc->dev, "invalid boot address 0x%x, must be aligned on a 1KB boundary\n",
388 boot_addr);
389 return -EINVAL;
390 }
392 ret = regmap_write(ksproc->dev_ctrl, ksproc->boot_offset, boot_addr);
393 if (ret) {
394 dev_err(ksproc->dev, "regmap_write of boot address failed, status = %d\n",
395 ret);
396 return ret;
397 }
399 reset_control_deassert(ksproc->reset);
401 return 0;
402 }
404 static long
405 keystone_rproc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
406 {
407 struct miscdevice *misc = filp->private_data;
408 struct keystone_rproc *ksproc =
409 container_of(misc, struct keystone_rproc, misc);
410 void __user *argp = (void __user *)arg;
411 int ret = 0;
413 dev_dbg(ksproc->dev, "%s: cmd 0x%.8x (%d), arg 0x%lx\n",
414 __func__, cmd, _IOC_NR(cmd), arg);
416 if (_IOC_TYPE(cmd) != KEYSTONE_RPROC_IOC_MAGIC)
417 return -ENOTTY;
419 if (_IOC_NR(cmd) >= KEYSTONE_RPROC_IOC_MAXNR)
420 return -ENOTTY;
422 switch (cmd) {
423 case KEYSTONE_RPROC_IOC_SET_STATE:
424 ret = keystone_rproc_set_state(ksproc, argp);
425 break;
427 case KEYSTONE_RPROC_IOC_SET_RSC_TABLE:
428 ret = keystone_rproc_set_rsc_table(ksproc, argp);
429 break;
431 case KEYSTONE_RPROC_IOC_SET_LOADED_RSC_TABLE:
432 ret = keystone_rproc_set_loaded_rsc_table(ksproc, arg);
433 break;
435 case KEYSTONE_RPROC_IOC_DSP_RESET:
436 if (ksproc->rsc_table) {
437 ret = -EINVAL;
438 break;
439 }
441 keystone_rproc_dsp_reset(ksproc);
442 break;
444 case KEYSTONE_RPROC_IOC_DSP_BOOT:
445 if (ksproc->rsc_table) {
446 ret = -EINVAL;
447 break;
448 }
450 ret = keystone_rproc_dsp_boot(ksproc, arg);
451 break;
453 default:
454 ret = -ENOTTY;
455 break;
456 }
458 if (ret) {
459 dev_err(ksproc->dev, "error in ioctl call: cmd 0x%.8x (%d), ret %d\n",
460 cmd, _IOC_NR(cmd), ret);
461 }
463 return ret;
464 }
466 /*
467 * Map DSP memories into userspace for supporting Userspace loading.
468 *
469 * This is a custom mmap function following semantics based on the UIO
470 * mmap implementation. The vm_pgoff passed in the vma structure is a
471 * combination of the memory region index and the actual page offset in
472 * that region. This checks if user request is in valid range before
473 * providing mmap access.
474 *
475 * XXX: Evaluate this approach, as the internal memories can be mapped in
476 * whole into userspace as they are not super-large, or switch to using
477 * direct addresses to look more like a traditional implementation.
478 */
479 static int keystone_rproc_mmap(struct file *file, struct vm_area_struct *vma)
480 {
481 struct miscdevice *misc = file->private_data;
482 struct keystone_rproc *ksproc =
483 container_of(misc, struct keystone_rproc, misc);
484 size_t size = vma->vm_end - vma->vm_start;
485 size_t req_offset;
486 u32 idx;
488 idx = vma->vm_pgoff & KEYSTONE_RPROC_UIO_MAP_INDEX_MASK;
490 if (idx >= ksproc->num_mems) {
491 dev_err(ksproc->dev, "invalid mmap region index %d\n", idx);
492 return -EINVAL;
493 }
495 req_offset = (vma->vm_pgoff - idx) << PAGE_SHIFT;
496 if (req_offset + size < req_offset) {
497 dev_err(ksproc->dev, "invalid request - overflow, mmap offset = 0x%zx size 0x%zx region %d\n",
498 req_offset, size, idx);
499 return -EINVAL;
500 }
502 if ((req_offset + size) > ksproc->mem[idx].size) {
503 dev_err(ksproc->dev, "invalid request - out of range, mmap offset 0x%zx size 0x%zx region %d\n",
504 req_offset, size, idx);
505 return -EINVAL;
506 }
508 vma->vm_page_prot =
509 phys_mem_access_prot(file,
510 (ksproc->mem[idx].bus_addr >> PAGE_SHIFT) +
511 (vma->vm_pgoff - idx), size,
512 vma->vm_page_prot);
514 if (remap_pfn_range(vma, vma->vm_start,
515 (ksproc->mem[idx].bus_addr >> PAGE_SHIFT) +
516 (vma->vm_pgoff - idx), size, vma->vm_page_prot))
517 return -EAGAIN;
519 return 0;
520 }
522 static int keystone_rproc_open(struct inode *inode, struct file *file)
523 {
524 struct miscdevice *misc = file->private_data;
525 struct keystone_rproc *ksproc =
526 container_of(misc, struct keystone_rproc, misc);
528 mutex_lock(&ksproc->mlock);
529 ksproc->open_count++;
530 mutex_unlock(&ksproc->mlock);
532 return 0;
533 }
535 static int keystone_rproc_release(struct inode *inode, struct file *filp)
536 {
537 struct miscdevice *misc = filp->private_data;
538 struct keystone_rproc *ksproc =
539 container_of(misc, struct keystone_rproc, misc);
540 struct rproc *rproc = ksproc->rproc;
542 mutex_lock(&ksproc->mlock);
544 if ((WARN_ON(ksproc->open_count == 0)))
545 goto end;
547 if (--ksproc->open_count > 0)
548 goto end;
550 if (rproc->state != RPROC_OFFLINE) {
551 rproc_shutdown(rproc);
552 WARN_ON(rproc->state != RPROC_OFFLINE);
553 }
555 keystone_rproc_reset_rsc_table(ksproc);
557 end:
558 mutex_unlock(&ksproc->mlock);
559 return 0;
560 }
562 /*
563 * File operations exposed through a miscdevice for supporting
564 * the userspace loader/boot mechanism.
565 */
566 static const struct file_operations keystone_rproc_fops = {
567 .owner = THIS_MODULE,
568 .unlocked_ioctl = keystone_rproc_ioctl,
569 .mmap = keystone_rproc_mmap,
570 .open = keystone_rproc_open,
571 .release = keystone_rproc_release,
572 };
574 /*
575 * Used only with userspace loader/boot mechanism, the parsing of the firmware
576 * is done in userspace, and a copy of the resource table is added for the
577 * kernel-level access through an ioctl. Create the remoteproc cached table
578 * using this resource table and configure the table pointer and table size
579 * accordingly to allow the remoteproc core to process the resource table for
580 * creating the vrings and traces.
581 */
582 static int keystone_rproc_load_rsc_table(struct rproc *rproc,
583 const struct firmware *fw)
584 {
585 struct keystone_rproc *ksproc = rproc->priv;
587 rproc->cached_table = kmemdup(ksproc->rsc_table, ksproc->rsc_table_size,
588 GFP_KERNEL);
589 if (!rproc->cached_table)
590 return -ENOMEM;
592 rproc->table_ptr = rproc->cached_table;
593 rproc->table_sz = ksproc->rsc_table_size;
595 return 0;
596 }
598 /*
599 * Used only with userspace loader/boot mechanism, the device address of the
600 * loaded resource table is published to the kernel-level through an ioctl
601 * at which point the equivalent kernel virtual pointer is stored in a local
602 * variable in the keystone_rproc device structure. Return this kernel pointer
603 * to the remoteproc core for runtime publishing/modification of the resource
604 * table entries.
605 *
606 * NOTE: Only loaded resource tables in the DSP internal memories is supported
607 * at present.
608 */
609 static struct resource_table *
610 keystone_rproc_find_loaded_rsc_table(struct rproc *rproc,
611 const struct firmware *fw)
612 {
613 struct keystone_rproc *ksproc = rproc->priv;
615 return ksproc->loaded_rsc_table;
616 }
618 /*
619 * Used only with userspace loader/boot mechanism, the boot address
620 * is published to the kernel-level through an ioctl call and is
621 * stored in a local variable in the keystone_rproc device structure.
622 * Return this address to the remoteproc core through the .get_boot_addr()
623 * remoteproc firmware ops
624 */
625 static u32 keystone_rproc_get_boot_addr(struct rproc *rproc,
626 const struct firmware *fw)
627 {
628 struct keystone_rproc *ksproc = rproc->priv;
630 return ksproc->boot_addr;
631 }
633 /*
634 * Process the remoteproc exceptions
635 *
636 * The exception reporting on Keystone DSP remote processors is very simple
637 * compared to the equivalent processors on the OMAP family, it is notified
638 * through a software-designed specific interrupt source in the IPC interrupt
639 * generation register.
640 *
641 * This function just invokes the rproc_report_crash to report the exception
642 * to the remoteproc driver core, to trigger a recovery. This is the case
643 * only when using in-kernel remoteproc core loader/boot mechanism, and is
644 * handled through an UIO interrupt otherwise.
645 */
646 static irqreturn_t keystone_rproc_exception_interrupt(int irq, void *dev_id)
647 {
648 struct keystone_rproc *ksproc = dev_id;
650 rproc_report_crash(ksproc->rproc, RPROC_FATAL_ERROR);
652 return IRQ_HANDLED;
653 }
655 /*
656 * Main virtqueue message workqueue function
657 *
658 * This function is executed upon scheduling of the keystone remoteproc
659 * driver's workqueue. The workqueue is scheduled by the vring ISR handler.
660 *
661 * There is no payload message indicating the virtqueue index as is the
662 * case with mailbox-based implementations on OMAP family. As such, this
663 * handler processes both the Tx and Rx virtqueue indices on every invocation.
664 * The rproc_vq_interrupt function can detect if there are new unprocessed
665 * messages or not (returns IRQ_NONE vs IRQ_HANDLED), but there is no need
666 * to check for these return values. The index 0 triggering will process all
667 * pending Rx buffers, and the index 1 triggering will process all newly
668 * available Tx buffers and will wakeup any potentially blocked senders.
669 *
670 * NOTE:
671 * 1. A payload could be added by using some of the source bits in the
672 * IPC interrupt generation registers, but this would need additional
673 * changes to the overall IPC stack, and currently there are no benefits
674 * of adapting that approach.
675 * 2. The current logic is based on an inherent design assumption of supporting
676 * only 2 vrings, but this can be changed if needed.
677 */
678 static void handle_event(struct work_struct *work)
679 {
680 struct keystone_rproc *ksproc =
681 container_of(work, struct keystone_rproc, workqueue);
683 rproc_vq_interrupt(ksproc->rproc, 0);
684 rproc_vq_interrupt(ksproc->rproc, 1);
685 }
687 /*
688 * Interrupt handler for processing vring kicks from remote processor
689 */
690 static irqreturn_t keystone_rproc_vring_interrupt(int irq, void *dev_id)
691 {
692 struct keystone_rproc *ksproc = dev_id;
694 schedule_work(&ksproc->workqueue);
696 return IRQ_HANDLED;
697 }
699 /*
700 * Power up the DSP remote processor.
701 *
702 * This function will be invoked only after the firmware for this rproc
703 * was loaded, parsed successfully, and all of its resource requirements
704 * were met. The function skips releasing the processor from reset and
705 * registering for the exception interrupt if using the userspace controlled
706 * load/boot mechanism. The processor will be started through an ioctl when
707 * controlled from userspace, but the virtio interrupt still is handled at
708 * the kernel layer.
709 */
710 static int keystone_rproc_start(struct rproc *rproc)
711 {
712 struct keystone_rproc *ksproc = rproc->priv;
713 int ret;
715 INIT_WORK(&ksproc->workqueue, handle_event);
717 ret = request_irq(ksproc->irq_ring, keystone_rproc_vring_interrupt, 0,
718 dev_name(ksproc->dev), ksproc);
719 if (ret) {
720 dev_err(ksproc->dev, "failed to enable vring interrupt, ret = %d\n",
721 ret);
722 goto out;
723 }
725 if (!ksproc->use_userspace_loader) {
726 ret = request_irq(ksproc->irq_fault,
727 keystone_rproc_exception_interrupt, 0,
728 dev_name(ksproc->dev), ksproc);
729 if (ret) {
730 dev_err(ksproc->dev, "failed to enable exception interrupt, ret = %d\n",
731 ret);
732 goto free_vring_irq;
733 }
734 }
736 ret = keystone_rproc_dsp_boot(ksproc, rproc->bootaddr);
737 if (ret)
738 goto free_exc_irq;
740 return 0;
742 free_exc_irq:
743 free_irq(ksproc->irq_fault, ksproc);
744 free_vring_irq:
745 free_irq(ksproc->irq_ring, ksproc);
746 flush_work(&ksproc->workqueue);
747 out:
748 return ret;
749 }
751 /*
752 * Stop the DSP remote processor.
753 *
754 * This function puts the DSP processor into reset, and finishes processing
755 * of any pending messages. The reset procedure is completed only if using
756 * kernel-mode remoteproc loading/booting mechanism, it is handled outside
757 * if using userspace load/boot mechanism either through an ioctl, or when
758 * the handle to the device is closed without triggering a reset.
759 */
760 static int keystone_rproc_stop(struct rproc *rproc)
761 {
762 struct keystone_rproc *ksproc = rproc->priv;
764 if (!ksproc->use_userspace_loader) {
765 keystone_rproc_dsp_reset(ksproc);
766 free_irq(ksproc->irq_fault, ksproc);
767 }
769 free_irq(ksproc->irq_ring, ksproc);
770 flush_work(&ksproc->workqueue);
771 return 0;
772 }
774 /*
775 * Kick the remote processor to notify about pending unprocessed messages.
776 * The vqid usage is not used and is inconsequential, as the kick is performed
777 * through a simulated GPIO (an bit in an IPC interrupt-triggering register),
778 * the remote processor is expected to process both its Tx and Rx virtqueues.
779 */
780 static void keystone_rproc_kick(struct rproc *rproc, int vqid)
781 {
782 struct keystone_rproc *ksproc = rproc->priv;
784 if (WARN_ON(ksproc->kick_gpio < 0))
785 return;
787 gpio_set_value(ksproc->kick_gpio, 1);
788 }
790 /*
791 * Custom function to translate a DSP device address (internal RAMs only) to a
792 * kernel virtual address. The DSPs can access their RAMs at either an internal
793 * address visible only from a DSP, or at the SoC-level bus address. Both these
794 * addresses need to be looked through for translation. The translated addresses
795 * can be used either by the remoteproc core for loading (when using kernel
796 * remoteproc loader), or by any rpmsg bus drivers.
797 */
798 static void *keystone_rproc_da_to_va(struct rproc *rproc, u64 da, int len,
799 u32 flags)
800 {
801 struct keystone_rproc *ksproc = rproc->priv;
802 void __iomem *va = NULL;
803 phys_addr_t bus_addr;
804 u32 dev_addr, offset;
805 size_t size;
806 int i;
808 if (len <= 0)
809 return NULL;
811 for (i = 0; i < ksproc->num_mems; i++) {
812 bus_addr = ksproc->mem[i].bus_addr;
813 dev_addr = ksproc->mem[i].dev_addr;
814 size = ksproc->mem[i].size;
816 if (da < KEYSTONE_RPROC_LOCAL_ADDRESS_MASK) {
817 /* handle DSP-view addresses */
818 if ((da >= dev_addr) &&
819 ((da + len) <= (dev_addr + size))) {
820 offset = da - dev_addr;
821 va = ksproc->mem[i].cpu_addr + offset;
822 break;
823 }
824 } else {
825 /* handle SoC-view addresses */
826 if ((da >= bus_addr) &&
827 (da + len) <= (bus_addr + size)) {
828 offset = da - bus_addr;
829 va = ksproc->mem[i].cpu_addr + offset;
830 break;
831 }
832 }
833 }
835 return (__force void *)va;
836 }
838 static const struct rproc_ops keystone_rproc_ops = {
839 .start = keystone_rproc_start,
840 .stop = keystone_rproc_stop,
841 .kick = keystone_rproc_kick,
842 .da_to_va = keystone_rproc_da_to_va,
843 };
845 static int keystone_rproc_of_get_memories(struct platform_device *pdev,
846 struct keystone_rproc *ksproc)
847 {
848 static const char * const mem_names[] = {"l2sram", "l1pram", "l1dram"};
849 struct device *dev = &pdev->dev;
850 struct resource *res;
851 int num_mems = 0;
852 int i;
854 num_mems = ARRAY_SIZE(mem_names);
855 ksproc->mem = devm_kcalloc(ksproc->dev, num_mems,
856 sizeof(*ksproc->mem), GFP_KERNEL);
857 if (!ksproc->mem)
858 return -ENOMEM;
860 for (i = 0; i < num_mems; i++) {
861 res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
862 mem_names[i]);
863 ksproc->mem[i].cpu_addr = devm_ioremap_resource(dev, res);
864 if (IS_ERR(ksproc->mem[i].cpu_addr)) {
865 dev_err(dev, "failed to parse and map %s memory\n",
866 mem_names[i]);
867 return PTR_ERR(ksproc->mem[i].cpu_addr);
868 }
869 ksproc->mem[i].bus_addr = res->start;
870 ksproc->mem[i].dev_addr =
871 res->start & KEYSTONE_RPROC_LOCAL_ADDRESS_MASK;
872 ksproc->mem[i].size = resource_size(res);
874 /* zero out memories to start in a pristine state */
875 memset((__force void *)ksproc->mem[i].cpu_addr, 0,
876 ksproc->mem[i].size);
877 }
878 ksproc->num_mems = num_mems;
880 return 0;
881 }
883 static int keystone_rproc_of_get_dev_syscon(struct platform_device *pdev,
884 struct keystone_rproc *ksproc)
885 {
886 struct device_node *np = pdev->dev.of_node;
887 struct device *dev = &pdev->dev;
888 int ret;
890 if (!of_property_read_bool(np, "ti,syscon-dev")) {
891 dev_err(dev, "ti,syscon-dev property is absent\n");
892 return -EINVAL;
893 }
895 ksproc->dev_ctrl =
896 syscon_regmap_lookup_by_phandle(np, "ti,syscon-dev");
897 if (IS_ERR(ksproc->dev_ctrl)) {
898 ret = PTR_ERR(ksproc->dev_ctrl);
899 return ret;
900 }
902 if (of_property_read_u32_index(np, "ti,syscon-dev", 1,
903 &ksproc->boot_offset)) {
904 dev_err(dev, "couldn't read the boot register offset\n");
905 return -EINVAL;
906 }
908 return 0;
909 }
911 static int keystone_rproc_probe(struct platform_device *pdev)
912 {
913 struct device *dev = &pdev->dev;
914 struct device_node *np = dev->of_node;
915 struct keystone_rproc *ksproc;
916 struct miscdevice *misc;
917 struct uio_info *uio;
918 struct rproc *rproc;
919 int dsp_id;
920 char *uio_name = NULL;
921 char *fw_name = NULL;
922 char *template = "keystone-dsp%d-fw";
923 int name_len = 0;
924 int ret = 0;
926 if (!np) {
927 dev_err(dev, "only DT-based devices are supported\n");
928 return -ENODEV;
929 }
931 dsp_id = of_alias_get_id(np, "rproc");
932 if (dsp_id < 0) {
933 dev_warn(dev, "device does not have an alias id\n");
934 return dsp_id;
935 }
937 /* construct a name for uio devices - assuming a single digit alias */
938 name_len = strlen("dsp%d");
939 uio_name = devm_kzalloc(dev, name_len, GFP_KERNEL);
940 if (!uio_name)
941 return -ENOMEM;
942 snprintf(uio_name, name_len, "dsp%d", dsp_id);
944 /* construct a custom default fw name - subject to change in future */
945 name_len = strlen(template); /* assuming a single digit alias */
946 fw_name = devm_kzalloc(dev, name_len, GFP_KERNEL);
947 if (!fw_name)
948 return -ENOMEM;
949 snprintf(fw_name, name_len, template, dsp_id);
951 rproc = rproc_alloc(dev, dev_name(dev), &keystone_rproc_ops, fw_name,
952 sizeof(*ksproc));
953 if (!rproc)
954 return -ENOMEM;
956 rproc->has_iommu = false;
958 ksproc = rproc->priv;
959 ksproc->rproc = rproc;
960 ksproc->dev = dev;
961 ksproc->use_userspace_loader = !use_rproc_core_loader;
963 /*
964 * customize the remoteproc core config flags and ELF fw ops for
965 * userspace loader/boot mechanism
966 */
967 if (ksproc->use_userspace_loader) {
968 rproc->recovery_disabled = true;
969 rproc->auto_boot = false;
970 rproc->skip_firmware_request = 1;
971 rproc->skip_load = 1;
972 rproc->deny_sysfs_ops = true;
974 rproc->ops->parse_fw = keystone_rproc_load_rsc_table;
975 rproc->ops->find_loaded_rsc_table =
976 keystone_rproc_find_loaded_rsc_table;
977 rproc->ops->get_boot_addr = keystone_rproc_get_boot_addr;
978 rproc->ops->sanity_check = NULL;
979 rproc->ops->load = NULL;
980 }
982 mutex_init(&ksproc->mlock);
983 spin_lock_init(&ksproc->lock);
985 ret = keystone_rproc_of_get_dev_syscon(pdev, ksproc);
986 if (ret)
987 goto free_rproc;
989 ksproc->reset = devm_reset_control_get_exclusive(dev, NULL);
990 if (IS_ERR(ksproc->reset)) {
991 ret = PTR_ERR(ksproc->reset);
992 goto free_rproc;
993 }
995 /* enable clock for accessing DSP internal memories */
996 pm_runtime_enable(dev);
997 ret = pm_runtime_get_sync(dev);
998 if (ret < 0) {
999 dev_err(dev, "failed to enable clock, status = %d\n", ret);
1000 pm_runtime_put_noidle(dev);
1001 goto disable_rpm;
1002 }
1004 ret = keystone_rproc_of_get_memories(pdev, ksproc);
1005 if (ret)
1006 goto disable_clk;
1008 ksproc->irq_ring = platform_get_irq_byname(pdev, "vring");
1009 if (ksproc->irq_ring < 0) {
1010 ret = ksproc->irq_ring;
1011 dev_err(dev, "failed to get vring interrupt, status = %d\n",
1012 ret);
1013 goto disable_clk;
1014 }
1016 ksproc->irq_fault = platform_get_irq_byname(pdev, "exception");
1017 if (ksproc->irq_fault < 0) {
1018 ret = ksproc->irq_fault;
1019 dev_err(dev, "failed to get exception interrupt, status = %d\n",
1020 ret);
1021 goto disable_clk;
1022 }
1024 ksproc->kick_gpio = of_get_named_gpio_flags(np, "kick-gpios", 0, NULL);
1025 if (ksproc->kick_gpio < 0) {
1026 ret = ksproc->kick_gpio;
1027 dev_err(dev, "failed to get gpio for virtio kicks, status = %d\n",
1028 ret);
1029 goto disable_clk;
1030 }
1032 if (of_reserved_mem_device_init(dev))
1033 dev_warn(dev, "device does not have specific CMA pool\n");
1035 /* ensure the DSP is in reset before loading firmware */
1036 ret = reset_control_status(ksproc->reset);
1037 if (ret < 0) {
1038 dev_err(dev, "failed to get reset status, status = %d\n", ret);
1039 goto release_mem;
1040 } else if (ret == 0) {
1041 WARN(1, "device is not in reset\n");
1042 keystone_rproc_dsp_reset(ksproc);
1043 }
1045 ret = rproc_add(rproc);
1046 if (ret) {
1047 dev_err(dev, "failed to add register device with remoteproc core, status = %d\n",
1048 ret);
1049 goto release_mem;
1050 }
1052 platform_set_drvdata(pdev, ksproc);
1054 if (ksproc->use_userspace_loader) {
1055 uio = &ksproc->uio;
1056 uio->name = uio_name;
1057 uio->version = DRIVER_UIO_VERSION;
1058 uio->irq = ksproc->irq_fault;
1059 uio->priv = ksproc;
1060 uio->handler = keystone_rproc_uio_handler;
1061 uio->irqcontrol = keystone_rproc_uio_irqcontrol;
1062 ret = uio_register_device(dev, uio);
1063 if (ret) {
1064 dev_err(dev, "failed to register uio device, status = %d\n",
1065 ret);
1066 goto del_rproc;
1067 }
1068 dev_dbg(dev, "registered uio device %s\n", uio->name);
1070 misc = &ksproc->misc;
1071 misc->minor = MISC_DYNAMIC_MINOR;
1072 misc->name = uio->name;
1073 misc->fops = &keystone_rproc_fops;
1074 misc->parent = dev;
1075 ret = misc_register(misc);
1076 if (ret) {
1077 dev_err(dev, "failed to register misc device, status = %d\n",
1078 ret);
1079 goto unregister_uio;
1080 }
1082 ret = keystone_rproc_mem_add_attrs(ksproc);
1083 if (ret) {
1084 dev_err(ksproc->dev, "error creating sysfs files (%d)\n",
1085 ret);
1086 goto unregister_misc;
1087 }
1089 dev_dbg(dev, "registered misc device %s\n", misc->name);
1090 }
1092 return 0;
1094 unregister_misc:
1095 misc_deregister(misc);
1096 unregister_uio:
1097 uio_unregister_device(uio);
1098 del_rproc:
1099 rproc_del(rproc);
1100 release_mem:
1101 of_reserved_mem_device_release(dev);
1102 disable_clk:
1103 pm_runtime_put_sync(dev);
1104 disable_rpm:
1105 pm_runtime_disable(dev);
1106 free_rproc:
1107 rproc_free(rproc);
1108 return ret;
1109 }
1111 static int keystone_rproc_remove(struct platform_device *pdev)
1112 {
1113 struct keystone_rproc *ksproc = platform_get_drvdata(pdev);
1115 if (ksproc->use_userspace_loader) {
1116 keystone_rproc_mem_del_attrs(ksproc);
1117 misc_deregister(&ksproc->misc);
1118 uio_unregister_device(&ksproc->uio);
1119 }
1120 rproc_del(ksproc->rproc);
1121 pm_runtime_put_sync(&pdev->dev);
1122 pm_runtime_disable(&pdev->dev);
1123 rproc_free(ksproc->rproc);
1124 of_reserved_mem_device_release(&pdev->dev);
1126 return 0;
1127 }
1129 static const struct of_device_id keystone_rproc_of_match[] = {
1130 { .compatible = "ti,k2hk-dsp", },
1131 { .compatible = "ti,k2l-dsp", },
1132 { .compatible = "ti,k2e-dsp", },
1133 { .compatible = "ti,k2g-dsp", },
1134 { /* sentinel */ },
1135 };
1136 MODULE_DEVICE_TABLE(of, keystone_rproc_of_match);
1138 static struct platform_driver keystone_rproc_driver = {
1139 .probe = keystone_rproc_probe,
1140 .remove = keystone_rproc_remove,
1141 .driver = {
1142 .name = "keystone-rproc",
1143 .of_match_table = keystone_rproc_of_match,
1144 },
1145 };
1147 static int __init keystone_rproc_init(void)
1148 {
1149 keystone_rproc_driver.driver.suppress_bind_attrs =
1150 !use_rproc_core_loader;
1152 return platform_driver_register(&keystone_rproc_driver);
1153 }
1154 module_init(keystone_rproc_init);
1156 static void __exit keystone_rproc_exit(void)
1157 {
1158 platform_driver_unregister(&keystone_rproc_driver);
1159 }
1160 module_exit(keystone_rproc_exit);
1162 MODULE_AUTHOR("Suman Anna <s-anna@ti.com>");
1163 MODULE_AUTHOR("Sam Nelson <sam.nelson@ti.com>");
1164 MODULE_LICENSE("GPL v2");
1165 MODULE_DESCRIPTION("TI Keystone DSP Remoteproc driver");