1 /*
2 * Copyright (c) 2011-2014, Texas Instruments Incorporated
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * * Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 *
12 * * Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * * Neither the name of Texas Instruments Incorporated nor the names of
17 * its contributors may be used to endorse or promote products derived
18 * from this software without specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
22 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
24 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
25 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
26 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
27 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
28 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
29 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
30 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 */
32 /** ============================================================================
33 * @file VirtQueue.c
34 *
35 * @brief Virtio Queue implementation for BIOS
36 *
37 * Differences between BIOS version and Linux kernel (include/linux/virtio.h):
38 * - Renamed module from virtio.h to VirtQueue_Object.h to match the API prefixes;
39 * - BIOS (XDC) types and CamelCasing used;
40 * - virtio_device concept removed (i.e, assumes no containing device);
41 * - simplified scatterlist from Linux version;
42 * - VirtQueue_Objects are created statically here, so just added a VirtQueue_Object_init()
43 * fxn to take the place of the Virtio vring_new_virtqueue() API;
44 * - The notify function is implicit in the implementation, and not provided
45 * by the client, as it is in Linux virtio.
46 *
47 * All VirtQueue operations can be called in any context.
48 *
49 * The virtio header should be included in an application as follows:
50 * @code
51 * #include <ti/ipc/family/vayu/VirtQueue.h>
52 * @endcode
53 *
54 */
56 /* this define must precede inclusion of any xdc header file */
57 #define Registry_CURDESC ti_ipc_family_vayu__Desc
58 #define MODULE_NAME "ti.ipc.family.vayu.VirtQueue"
60 #include <xdc/std.h>
61 #include <xdc/runtime/System.h>
62 #include <xdc/runtime/Assert.h>
63 #include <xdc/runtime/Error.h>
64 #include <xdc/runtime/Memory.h>
65 #include <xdc/runtime/Registry.h>
66 #include <xdc/runtime/Log.h>
67 #include <xdc/runtime/Diags.h>
69 #include <ti/sysbios/hal/Hwi.h>
70 #include <ti/sysbios/knl/Clock.h>
71 #include <ti/sysbios/gates/GateHwi.h>
72 #include <ti/sysbios/hal/Cache.h>
74 #include <ti/ipc/MultiProc.h>
76 #include <ti/ipc/rpmsg/virtio_ring.h>
77 #ifndef DSP
78 #include <ti/pm/IpcPower.h>
79 #endif
80 #include <string.h>
82 #include <ti/ipc/remoteproc/Resource.h>
83 #include <ti/ipc/remoteproc/rsc_types.h>
84 #include <ti/ipc/rpmsg/_VirtQueue.h>
86 #include <ti/sdo/ipc/notifyDrivers/IInterrupt.h>
87 #include "InterruptProxy.h"
88 #include "VirtQueue.h"
91 /*
92 * The following three VIRTIO_* defines must match those in
93 * <Linux_kernel>/include/uapi/linux/virtio_config.h
94 */
95 #define VIRTIO_CONFIG_S_ACKNOWLEDGE 1
96 #define VIRTIO_CONFIG_S_DRIVER 2
97 #define VIRTIO_CONFIG_S_DRIVER_OK 4
99 #define VRING_BUFS_PRIMED (VIRTIO_CONFIG_S_ACKNOWLEDGE | \
100 VIRTIO_CONFIG_S_DRIVER | VIRTIO_CONFIG_S_DRIVER_OK)
102 /* Used for defining the size of the virtqueue registry */
103 #define NUM_QUEUES 2
105 /* Predefined device addresses */
106 #ifdef DSP
107 #define IPC_MEM_VRING0 0xA0000000
108 #define IPC_MEM_VRING1 0xA0004000
109 #else
110 #define IPC_MEM_VRING0 0x60000000
111 #define IPC_MEM_VRING1 0x60004000
112 #endif
114 /*
115 * Size of the virtqueues (expressed in number of buffers supported,
116 * and must be power of two)
117 */
118 #define VQ_SIZE 256
120 /*
121 * enum - Predefined Mailbox Messages
122 *
123 * @RP_MSG_MBOX_READY: informs the M3's that we're up and running. will be
124 * followed by another mailbox message that carries the A9's virtual address
125 * of the shared buffer. This would allow the A9's drivers to send virtual
126 * addresses of the buffers.
127 *
128 * @RP_MSG_MBOX_STATE_CHANGE: informs the receiver that there is an inbound
129 * message waiting in its own receive-side vring. please note that currently
130 * this message is optional: alternatively, one can explicitly send the index
131 * of the triggered virtqueue itself. the preferred approach will be decided
132 * as we progress and experiment with those design ideas.
133 *
134 * @RP_MSG_MBOX_CRASH: this message indicates that the BIOS side is unhappy
135 *
136 * @RP_MBOX_ECHO_REQUEST: this message requests the remote processor to reply
137 * with RP_MBOX_ECHO_REPLY
138 *
139 * @RP_MBOX_ECHO_REPLY: this is a reply that is sent when RP_MBOX_ECHO_REQUEST
140 * is received.
141 *
142 * @RP_MBOX_ABORT_REQUEST: tells the M3 to crash on demand
143 *
144 * @RP_MBOX_BOOTINIT_DONE: this message indicates the BIOS side has reached a
145 * certain state during the boot process. This message is used to inform the
146 * host that the basic BIOS initialization is done, and lets the host use this
147 * notification to perform certain actions.
148 */
149 enum {
150 RP_MSG_MBOX_READY = (Int)0xFFFFFF00,
151 RP_MSG_MBOX_STATE_CHANGE = (Int)0xFFFFFF01,
152 RP_MSG_MBOX_CRASH = (Int)0xFFFFFF02,
153 RP_MBOX_ECHO_REQUEST = (Int)0xFFFFFF03,
154 RP_MBOX_ECHO_REPLY = (Int)0xFFFFFF04,
155 RP_MBOX_ABORT_REQUEST = (Int)0xFFFFFF05,
156 RP_MSG_FLUSH_CACHE = (Int)0xFFFFFF06,
157 RP_MSG_BOOTINIT_DONE = (Int)0xFFFFFF07,
158 RP_MSG_HIBERNATION = (Int)0xFFFFFF10,
159 RP_MSG_HIBERNATION_FORCE = (Int)0xFFFFFF11,
160 RP_MSG_HIBERNATION_ACK = (Int)0xFFFFFF12,
161 RP_MSG_HIBERNATION_CANCEL = (Int)0xFFFFFF13
162 };
164 #define DIV_ROUND_UP(n,d) (((n) + (d) - 1) / (d))
165 #define RP_MSG_NUM_BUFS (VQ_SIZE) /* must be power of two */
166 #define RP_MSG_BUF_SIZE (512)
167 #define RP_MSG_BUFS_SPACE (RP_MSG_NUM_BUFS * RP_MSG_BUF_SIZE * 2)
169 #define PAGE_SIZE (4096)
170 /*
171 * The alignment to use between consumer and producer parts of vring.
172 * Note: this is part of the "wire" protocol. If you change this, you need
173 * to update your BIOS image as well
174 */
175 #define RP_MSG_VRING_ALIGN (4096)
177 /* With 256 buffers, our vring will occupy 3 pages */
178 #define RP_MSG_RING_SIZE ((DIV_ROUND_UP(vring_size(RP_MSG_NUM_BUFS, \
179 RP_MSG_VRING_ALIGN), PAGE_SIZE)) * PAGE_SIZE)
181 /* The total IPC space needed to communicate with a remote processor */
182 #define RPMSG_IPC_MEM (RP_MSG_BUFS_SPACE + 2 * RP_MSG_RING_SIZE)
184 typedef struct VirtQueue_Object {
185 /* Id for this VirtQueue_Object */
186 UInt16 id;
188 /* The function to call when buffers are consumed (can be NULL) */
189 VirtQueue_callback callback;
191 /* Shared state */
192 struct vring vring;
194 /* Number of free buffers */
195 UInt16 num_free;
197 /* Last available index; updated by VirtQueue_getAvailBuf */
198 UInt16 last_avail_idx;
200 /* Last available index; updated by VirtQueue_addUsedBuf */
201 UInt16 last_used_idx;
203 /* Will eventually be used to kick remote processor */
204 UInt16 procId;
206 /* Gate to protect from multiple threads */
207 GateHwi_Handle gateH;
208 } VirtQueue_Object;
210 /* module diags mask */
211 Registry_Desc Registry_CURDESC;
213 static struct VirtQueue_Object *queueRegistry[NUM_QUEUES] = {NULL};
215 static UInt16 hostProcId;
217 #define DSPEVENTID 5
218 IInterrupt_IntInfo intInfo;
220 /*!
221 * ======== _VirtQueue_init ========
222 *
223 * This function adds the VirtQueue "module" to the Registry so that
224 * DIAGS will work with this non-XDC module.
225 * Since VirtQueue_init is not called by XDC-VirtQueue module clients, this
226 * function is called in the first VirtQueue fxn called: VirtQueue_create.
227 */
228 static Void _VirtQueue_init()
229 {
230 static int initialized = 0;
232 if (!initialized) {
233 Registry_Result result;
235 /* register with xdc.runtime to get a diags mask */
236 result = Registry_addModule(&Registry_CURDESC, MODULE_NAME);
237 Assert_isTrue(result == Registry_SUCCESS, (Assert_Id)NULL);
239 initialized = 1;
240 }
241 }
243 static inline Void * mapPAtoVA(UInt pa)
244 {
245 return (Void *)((pa & 0x000fffffU) | IPC_MEM_VRING0);
246 }
248 static inline UInt mapVAtoPA(Void * va)
249 {
250 return ((UInt)va & 0x000fffffU) | 0x9cf00000U;
251 }
253 /*!
254 * ======== VirtQueue_kick ========
255 */
256 Void VirtQueue_kick(VirtQueue_Handle vq)
257 {
258 /* For now, simply interrupt remote processor */
259 if (vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT) {
260 Log_print0(Diags_USER1,
261 "VirtQueue_kick: no kick because of VRING_AVAIL_F_NO_INTERRUPT\n");
262 return;
263 }
265 Log_print2(Diags_USER1,
266 "VirtQueue_kick: Sending interrupt to proc %d with payload 0x%x\n",
267 (IArg)vq->procId, (IArg)vq->id);
268 InterruptProxy_intSend(vq->procId, NULL, vq->id);
269 }
271 /*!
272 * ======== VirtQueue_addUsedBuf ========
273 */
274 Int VirtQueue_addUsedBuf(VirtQueue_Handle vq, Int16 head, Int len)
275 {
276 struct vring_used_elem *used;
277 IArg key;
279 key = GateHwi_enter(vq->gateH);
280 if ((head > vq->vring.num) || (head < 0)) {
281 GateHwi_leave(vq->gateH, key);
282 Error_raise(NULL, Error_E_generic, 0, 0);
283 }
285 /*
286 * The virtqueue contains a ring of used buffers. Get a pointer to the
287 * next entry in that used ring.
288 */
289 used = &vq->vring.used->ring[vq->vring.used->idx % vq->vring.num];
290 used->id = head;
291 used->len = len;
293 vq->vring.used->idx++;
294 GateHwi_leave(vq->gateH, key);
296 return (0);
297 }
299 /*!
300 * ======== VirtQueue_addAvailBuf ========
301 */
302 Int VirtQueue_addAvailBuf(VirtQueue_Object *vq, Void *buf)
303 {
304 UInt16 avail;
305 IArg key;
307 if (vq->num_free == 0) {
308 /* There's no more space */
309 Error_raise(NULL, Error_E_generic, 0, 0);
310 }
312 vq->num_free--;
314 key = GateHwi_enter(vq->gateH);
315 avail = vq->vring.avail->idx++ % vq->vring.num;
317 vq->vring.desc[avail].addr = mapVAtoPA(buf);
318 vq->vring.desc[avail].len = RP_MSG_BUF_SIZE;
319 GateHwi_leave(vq->gateH, key);
321 return (vq->num_free);
322 }
324 /*!
325 * ======== VirtQueue_getUsedBuf ========
326 */
327 Void *VirtQueue_getUsedBuf(VirtQueue_Object *vq)
328 {
329 UInt16 head;
330 Void *buf;
331 IArg key;
333 key = GateHwi_enter(vq->gateH);
334 /* There's nothing available? */
335 if (vq->last_used_idx == vq->vring.used->idx) {
336 buf = NULL;
337 }
338 else {
339 head = vq->vring.used->ring[vq->last_used_idx % vq->vring.num].id;
340 vq->last_used_idx++;
342 buf = mapPAtoVA(vq->vring.desc[head].addr);
343 }
344 GateHwi_leave(vq->gateH, key);
346 return (buf);
347 }
349 /*!
350 * ======== VirtQueue_getAvailBuf ========
351 */
352 Int16 VirtQueue_getAvailBuf(VirtQueue_Handle vq, Void **buf, Int *len)
353 {
354 Int16 head;
355 IArg key;
357 key = GateHwi_enter(vq->gateH);
358 Log_print6(Diags_USER1, "getAvailBuf vq: 0x%x %d %d %d 0x%x 0x%x\n",
359 (IArg)vq, vq->last_avail_idx, vq->vring.avail->idx, vq->vring.num,
360 (IArg)&vq->vring.avail, (IArg)vq->vring.avail);
362 /* Clear flag here to avoid race condition with remote processor.
363 * This is a negative flag, clearing it means that we want to
364 * receive an interrupt when a buffer has been added to the pool.
365 */
366 vq->vring.used->flags &= ~VRING_USED_F_NO_NOTIFY;
368 /* There's nothing available? */
369 if (vq->last_avail_idx == vq->vring.avail->idx) {
370 head = (-1);
371 }
372 else {
373 /* No need to be kicked about added buffers anymore */
374 vq->vring.used->flags |= VRING_USED_F_NO_NOTIFY;
376 /*
377 * Grab the next descriptor number they're advertising, and increment
378 * the index we've seen.
379 */
380 head = vq->vring.avail->ring[vq->last_avail_idx++ % vq->vring.num];
382 *buf = mapPAtoVA(vq->vring.desc[head].addr);
383 *len = vq->vring.desc[head].len;
384 }
385 GateHwi_leave(vq->gateH, key);
387 return (head);
388 }
390 /*!
391 * ======== VirtQueue_disableCallback ========
392 */
393 Void VirtQueue_disableCallback(VirtQueue_Object *vq)
394 {
395 /* TODO */
396 Log_print0(Diags_USER1, "VirtQueue_disableCallback called.");
397 }
399 /*!
400 * ======== VirtQueue_enableCallback ========
401 */
402 Bool VirtQueue_enableCallback(VirtQueue_Object *vq)
403 {
404 Log_print0(Diags_USER1, "VirtQueue_enableCallback called.");
406 /* TODO */
407 return (FALSE);
408 }
410 /*!
411 * ======== VirtQueue_isr ========
412 * Note 'arg' is ignored: it is the Hwi argument, not the mailbox argument.
413 */
414 Void VirtQueue_isr(UArg msg)
415 {
416 VirtQueue_Object *vq;
418 msg = InterruptProxy_intClear(hostProcId, &intInfo);
420 Log_print1(Diags_USER1, "VirtQueue_isr received msg = 0x%x\n", msg);
422 switch(msg) {
423 case (UInt)RP_MSG_MBOX_READY:
424 return;
426 case (UInt)RP_MBOX_ECHO_REQUEST:
427 InterruptProxy_intSend(hostProcId, NULL, (UInt)(RP_MBOX_ECHO_REPLY));
428 return;
430 case (UInt)RP_MBOX_ABORT_REQUEST:
431 {
432 /* Suppress Coverity Error: FORWARD_NULL: */
433 /* coverity[assign_zero] */
434 Fxn f = (Fxn)0x0;
435 Log_print0(Diags_USER1, "Crash on demand ...\n");
436 /* coverity[var_deref_op] */
437 f();
438 }
439 return;
441 case (UInt)RP_MSG_FLUSH_CACHE:
442 Cache_wbAll();
443 return;
445 #ifndef DSP
446 case (UInt)RP_MSG_HIBERNATION:
447 if (IpcPower_canHibernate() == FALSE) {
448 InterruptProxy_intSend(hostProcId, NULL,
449 (UInt)RP_MSG_HIBERNATION_CANCEL);
450 return;
451 }
453 /* Fall through */
454 case (UInt)RP_MSG_HIBERNATION_FORCE:
455 /* Ack request */
456 InterruptProxy_intSend(hostProcId, NULL,
457 (UInt)RP_MSG_HIBERNATION_ACK);
458 IpcPower_suspend();
459 return;
460 #endif
461 default:
462 /*
463 * If the message isn't one of the above, it's either part of the
464 * 2-message synchronization sequence or it a virtqueue message
465 */
466 break;
467 }
469 /* Don't let unknown messages to pass as a virtqueue index */
470 if (msg >= NUM_QUEUES) {
471 /* Adding print here deliberately, we should never see this */
472 System_printf("VirtQueue_isr: Invalid mailbox message 0x%x "
473 "received\n", msg);
474 return;
475 }
477 vq = queueRegistry[msg];
478 if (vq) {
479 vq->callback(vq);
480 }
481 }
484 /*!
485 * ======== VirtQueue_create ========
486 */
487 VirtQueue_Handle VirtQueue_create(UInt16 remoteProcId, VirtQueue_Params *params,
488 Error_Block *eb)
489 {
490 VirtQueue_Object *vq;
491 Void *vringAddr;
493 /* Perform initialization we can't do in Instance_init (being non-XDC): */
494 _VirtQueue_init();
496 vq = Memory_alloc(NULL, sizeof(VirtQueue_Object), 0, eb);
497 if (NULL == vq) {
498 return (NULL);
499 }
501 /* Create the thread protection gate */
502 vq->gateH = GateHwi_create(NULL, eb);
503 if (Error_check(eb)) {
504 Log_error0("VirtQueue_create: could not create gate object");
505 Memory_free(NULL, vq, sizeof(VirtQueue_Object));
506 return (NULL);
507 }
509 vq->callback = params->callback;
510 vq->id = params->vqId;
511 vq->procId = remoteProcId;
512 vq->last_avail_idx = 0;
514 switch (vq->id) {
515 /* IPC transport vrings */
516 case ID_SELF_TO_A9:
517 /* IPU/DSP -> A9 */
518 vringAddr = (struct vring *) IPC_MEM_VRING0;
519 break;
520 case ID_A9_TO_SELF:
521 /* A9 -> IPU/DSP */
522 vringAddr = (struct vring *) IPC_MEM_VRING1;
523 break;
524 default:
525 GateHwi_delete(&vq->gateH);
526 Memory_free(NULL, vq, sizeof(VirtQueue_Object));
527 return (NULL);
528 }
530 Log_print3(Diags_USER1,
531 "vring: %d 0x%x (0x%x)\n", vq->id, (IArg)vringAddr,
532 RP_MSG_RING_SIZE);
534 /* See coverity related comment in vring_init() */
535 /* coverity[overrun-call] */
536 vring_init(&(vq->vring), RP_MSG_NUM_BUFS, vringAddr, RP_MSG_VRING_ALIGN);
538 /*
539 * Don't trigger a mailbox message every time MPU makes another buffer
540 * available
541 */
542 if (vq->procId == hostProcId) {
543 vq->vring.used->flags &= ~VRING_USED_F_NO_NOTIFY;
544 }
546 queueRegistry[vq->id] = vq;
548 return (vq);
549 }
551 /*!
552 * ======== VirtQueue_startup ========
553 */
554 Void VirtQueue_startup()
555 {
556 hostProcId = MultiProc_getId("HOST");
558 #ifdef DSP
559 intInfo.intVectorId = DSPEVENTID;
560 #else
561 /* Initilize the IpcPower module */
562 IpcPower_init();
563 #endif
565 /*
566 * Wait for HLOS (Virtio device) to indicate that priming of host's receive
567 * buffers is complete, indicating that host is ready to send.
568 *
569 * Though this is a Linux Virtio configuration status, it must be
570 * implemented by each non-Linux HLOS as well.
571 */
572 Log_print1(Diags_USER1, "VirtQueue_startup: VDEV status: 0x%x\n",
573 Resource_getVdevStatus(VIRTIO_ID_RPMSG));
574 Log_print0(Diags_USER1, "VirtQueue_startup: Polling VDEV status...\n");
575 while (Resource_getVdevStatus(VIRTIO_ID_RPMSG) != VRING_BUFS_PRIMED);
576 Log_print1(Diags_USER1, "VirtQueue_startup: VDEV status: 0x%x\n",
577 Resource_getVdevStatus(VIRTIO_ID_RPMSG));
579 InterruptProxy_intRegister(hostProcId, &intInfo, (Fxn)VirtQueue_isr, NULL);
580 Log_print0(Diags_USER1, "Passed VirtQueue_startup\n");
581 }
583 /*!
584 * ======== VirtQueue_postCrashToMailbox ========
585 */
586 Void VirtQueue_postCrashToMailbox(Void)
587 {
588 InterruptProxy_intSend(0, NULL, (UInt)RP_MSG_MBOX_CRASH);
589 }
591 #define CACHE_WB_TICK_PERIOD 5
593 /*!
594 * ======== VirtQueue_cacheWb ========
595 *
596 * Used for flushing SysMin trace buffer.
597 */
598 Void VirtQueue_cacheWb()
599 {
600 static UInt32 oldticks = 0;
601 UInt32 newticks;
603 newticks = Clock_getTicks();
604 if (newticks - oldticks < (UInt32)CACHE_WB_TICK_PERIOD) {
605 /* Don't keep flushing cache */
606 return;
607 }
609 oldticks = newticks;
611 /* Flush the cache */
612 Cache_wbAll();
613 }