1 /*
2 * Copyright (c) 2011-2013, Texas Instruments Incorporated
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * * Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 *
12 * * Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * * Neither the name of Texas Instruments Incorporated nor the names of
17 * its contributors may be used to endorse or promote products derived
18 * from this software without specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
22 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
24 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
25 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
26 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
27 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
28 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
29 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
30 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 */
32 /** ============================================================================
33 * @file VirtQueue.c
34 *
35 * @brief Virtio Queue implementation for BIOS
36 *
37 * Differences between BIOS version and Linux kernel (include/linux/virtio.h):
38 * - Renamed module from virtio.h to VirtQueue_Object.h to match the API prefixes;
39 * - BIOS (XDC) types and CamelCasing used;
40 * - virtio_device concept removed (i.e, assumes no containing device);
41 * - simplified scatterlist from Linux version;
42 * - VirtQueue_Objects are created statically here, so just added a VirtQueue_Object_init()
43 * fxn to take the place of the Virtio vring_new_virtqueue() API;
44 * - The notify function is implicit in the implementation, and not provided
45 * by the client, as it is in Linux virtio.
46 *
47 * All VirtQueue operations can be called in any context.
48 *
49 * The virtio header should be included in an application as follows:
50 * @code
51 * #include <ti/ipc/family/vayu/VirtQueue.h>
52 * @endcode
53 *
54 */
56 #include <xdc/std.h>
57 #include <xdc/runtime/System.h>
58 #include <xdc/runtime/Error.h>
59 #include <xdc/runtime/Memory.h>
60 #include <xdc/runtime/Log.h>
61 #include <xdc/runtime/Diags.h>
63 #include <ti/sysbios/hal/Hwi.h>
64 #include <ti/sysbios/knl/Clock.h>
65 #include <ti/sysbios/gates/GateAll.h>
66 #include <ti/sysbios/BIOS.h>
67 #include <ti/sysbios/hal/Cache.h>
69 #include <ti/ipc/MultiProc.h>
71 #include <ti/ipc/rpmsg/virtio_ring.h>
72 #ifndef DSP
73 #include <ti/pm/IpcPower.h>
74 #endif
75 #include <string.h>
77 #include <ti/ipc/rpmsg/_VirtQueue.h>
79 #include <ti/sdo/ipc/notifyDrivers/IInterrupt.h>
80 #include "InterruptProxy.h"
81 #include "VirtQueue.h"
84 /* Used for defining the size of the virtqueue registry */
85 #define NUM_QUEUES 4
87 /* Predefined device addresses */
88 #ifdef DSP
89 #define IPC_MEM_VRING0 0xA0000000
90 #define IPC_MEM_VRING1 0xA0004000
91 #define IPC_MEM_VRING2 0xA0008000
92 #define IPC_MEM_VRING3 0xA000c000
93 #else
94 #define IPC_MEM_VRING0 0x60000000
95 #define IPC_MEM_VRING1 0x60004000
96 #define IPC_MEM_VRING2 0x60008000
97 #define IPC_MEM_VRING3 0x6000c000
98 #endif
100 /*
101 * Sizes of the virtqueues (expressed in number of buffers supported,
102 * and must be power of two)
103 */
104 #define VQ0_SIZE 256
105 #define VQ1_SIZE 256
106 #define VQ2_SIZE 256
107 #define VQ3_SIZE 256
109 /*
110 * enum - Predefined Mailbox Messages
111 *
112 * @RP_MSG_MBOX_READY: informs the M3's that we're up and running. will be
113 * followed by another mailbox message that carries the A9's virtual address
114 * of the shared buffer. This would allow the A9's drivers to send virtual
115 * addresses of the buffers.
116 *
117 * @RP_MSG_MBOX_STATE_CHANGE: informs the receiver that there is an inbound
118 * message waiting in its own receive-side vring. please note that currently
119 * this message is optional: alternatively, one can explicitly send the index
120 * of the triggered virtqueue itself. the preferred approach will be decided
121 * as we progress and experiment with those design ideas.
122 *
123 * @RP_MSG_MBOX_CRASH: this message indicates that the BIOS side is unhappy
124 *
125 * @RP_MBOX_ECHO_REQUEST: this message requests the remote processor to reply
126 * with RP_MBOX_ECHO_REPLY
127 *
128 * @RP_MBOX_ECHO_REPLY: this is a reply that is sent when RP_MBOX_ECHO_REQUEST
129 * is received.
130 *
131 * @RP_MBOX_ABORT_REQUEST: tells the M3 to crash on demand
132 *
133 * @RP_MBOX_BOOTINIT_DONE: this message indicates the BIOS side has reached a
134 * certain state during the boot process. This message is used to inform the
135 * host that the basic BIOS initialization is done, and lets the host use this
136 * notification to perform certain actions.
137 */
138 enum {
139 RP_MSG_MBOX_READY = (Int)0xFFFFFF00,
140 RP_MSG_MBOX_STATE_CHANGE = (Int)0xFFFFFF01,
141 RP_MSG_MBOX_CRASH = (Int)0xFFFFFF02,
142 RP_MBOX_ECHO_REQUEST = (Int)0xFFFFFF03,
143 RP_MBOX_ECHO_REPLY = (Int)0xFFFFFF04,
144 RP_MBOX_ABORT_REQUEST = (Int)0xFFFFFF05,
145 RP_MSG_FLUSH_CACHE = (Int)0xFFFFFF06,
146 RP_MSG_BOOTINIT_DONE = (Int)0xFFFFFF07,
147 RP_MSG_HIBERNATION = (Int)0xFFFFFF10,
148 RP_MSG_HIBERNATION_FORCE = (Int)0xFFFFFF11,
149 RP_MSG_HIBERNATION_ACK = (Int)0xFFFFFF12,
150 RP_MSG_HIBERNATION_CANCEL = (Int)0xFFFFFF13
151 };
153 #define DIV_ROUND_UP(n,d) (((n) + (d) - 1) / (d))
154 #define RP_MSG_NUM_BUFS (VQ0_SIZE) /* must be power of two */
155 #define RP_MSG_BUF_SIZE (512)
156 #define RP_MSG_BUFS_SPACE (RP_MSG_NUM_BUFS * RP_MSG_BUF_SIZE * 2)
158 #define PAGE_SIZE (4096)
159 /*
160 * The alignment to use between consumer and producer parts of vring.
161 * Note: this is part of the "wire" protocol. If you change this, you need
162 * to update your BIOS image as well
163 */
164 #define RP_MSG_VRING_ALIGN (4096)
166 /* With 256 buffers, our vring will occupy 3 pages */
167 #define RP_MSG_RING_SIZE ((DIV_ROUND_UP(vring_size(RP_MSG_NUM_BUFS, \
168 RP_MSG_VRING_ALIGN), PAGE_SIZE)) * PAGE_SIZE)
170 /* The total IPC space needed to communicate with a remote processor */
171 #define RPMSG_IPC_MEM (RP_MSG_BUFS_SPACE + 2 * RP_MSG_RING_SIZE)
173 #define ID_SYSM3_TO_A9 ID_SELF_TO_A9
174 #define ID_A9_TO_SYSM3 ID_A9_TO_SELF
175 #define ID_DSP_TO_A9 ID_SELF_TO_A9
176 #define ID_A9_TO_DSP ID_A9_TO_SELF
177 #define ID_APPM3_TO_A9 2
178 #define ID_A9_TO_APPM3 3
180 typedef struct VirtQueue_Object {
181 /* Id for this VirtQueue_Object */
182 UInt16 id;
184 /* The function to call when buffers are consumed (can be NULL) */
185 VirtQueue_callback callback;
187 /* Shared state */
188 struct vring vring;
190 /* Number of free buffers */
191 UInt16 num_free;
193 /* Last available index; updated by VirtQueue_getAvailBuf */
194 UInt16 last_avail_idx;
196 /* Last available index; updated by VirtQueue_addUsedBuf */
197 UInt16 last_used_idx;
199 /* Will eventually be used to kick remote processor */
200 UInt16 procId;
202 /* Gate to protect from multiple threads */
203 GateAll_Handle gateH;
204 } VirtQueue_Object;
206 static struct VirtQueue_Object *queueRegistry[NUM_QUEUES] = {NULL};
208 static UInt16 hostProcId;
209 #ifndef SMP
210 static UInt16 dsp1ProcId;
211 static UInt16 sysm3ProcId;
212 static UInt16 appm3ProcId;
213 #endif
215 #define DSPEVENTID 5
216 IInterrupt_IntInfo intInfo;
218 #if defined(M3_ONLY) && !defined(SMP)
219 extern Void OffloadM3_init();
220 extern Int OffloadM3_processSysM3Tasks(UArg msg);
221 #endif
223 static inline Void * mapPAtoVA(UInt pa)
224 {
225 return (Void *)((pa & 0x000fffffU) | IPC_MEM_VRING0);
226 }
228 static inline UInt mapVAtoPA(Void * va)
229 {
230 return ((UInt)va & 0x000fffffU) | 0x9cf00000U;
231 }
233 /*!
234 * ======== VirtQueue_kick ========
235 */
236 Void VirtQueue_kick(VirtQueue_Handle vq)
237 {
238 /* For now, simply interrupt remote processor */
239 if (vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT) {
240 Log_print0(Diags_USER1,
241 "VirtQueue_kick: no kick because of VRING_AVAIL_F_NO_INTERRUPT\n");
242 return;
243 }
245 Log_print2(Diags_USER1,
246 "VirtQueue_kick: Sending interrupt to proc %d with payload 0x%x\n",
247 (IArg)vq->procId, (IArg)vq->id);
248 InterruptProxy_intSend(vq->procId, NULL, vq->id);
249 }
251 /*!
252 * ======== VirtQueue_addUsedBuf ========
253 */
254 Int VirtQueue_addUsedBuf(VirtQueue_Handle vq, Int16 head, Int len)
255 {
256 struct vring_used_elem *used;
257 IArg key;
259 key = GateAll_enter(vq->gateH);
260 if ((head > vq->vring.num) || (head < 0)) {
261 GateAll_leave(vq->gateH, key);
262 Error_raise(NULL, Error_E_generic, 0, 0);
263 }
265 /*
266 * The virtqueue contains a ring of used buffers. Get a pointer to the
267 * next entry in that used ring.
268 */
269 used = &vq->vring.used->ring[vq->vring.used->idx % vq->vring.num];
270 used->id = head;
271 used->len = len;
273 vq->vring.used->idx++;
274 GateAll_leave(vq->gateH, key);
276 return (0);
277 }
279 /*!
280 * ======== VirtQueue_addAvailBuf ========
281 */
282 Int VirtQueue_addAvailBuf(VirtQueue_Object *vq, Void *buf)
283 {
284 UInt16 avail;
285 IArg key;
287 if (vq->num_free == 0) {
288 /* There's no more space */
289 Error_raise(NULL, Error_E_generic, 0, 0);
290 }
292 vq->num_free--;
294 key = GateAll_enter(vq->gateH);
295 avail = vq->vring.avail->idx++ % vq->vring.num;
297 vq->vring.desc[avail].addr = mapVAtoPA(buf);
298 vq->vring.desc[avail].len = RP_MSG_BUF_SIZE;
299 GateAll_leave(vq->gateH, key);
301 return (vq->num_free);
302 }
304 /*!
305 * ======== VirtQueue_getUsedBuf ========
306 */
307 Void *VirtQueue_getUsedBuf(VirtQueue_Object *vq)
308 {
309 UInt16 head;
310 Void *buf;
311 IArg key;
313 key = GateAll_enter(vq->gateH);
314 /* There's nothing available? */
315 if (vq->last_used_idx == vq->vring.used->idx) {
316 buf = NULL;
317 }
318 else {
319 head = vq->vring.used->ring[vq->last_used_idx % vq->vring.num].id;
320 vq->last_used_idx++;
322 buf = mapPAtoVA(vq->vring.desc[head].addr);
323 }
324 GateAll_leave(vq->gateH, key);
326 return (buf);
327 }
329 /*!
330 * ======== VirtQueue_getAvailBuf ========
331 */
332 Int16 VirtQueue_getAvailBuf(VirtQueue_Handle vq, Void **buf, Int *len)
333 {
334 Int16 head;
335 IArg key;
337 key = GateAll_enter(vq->gateH);
338 Log_print6(Diags_USER1, "getAvailBuf vq: 0x%x %d %d %d 0x%x 0x%x\n",
339 (IArg)vq, vq->last_avail_idx, vq->vring.avail->idx, vq->vring.num,
340 (IArg)&vq->vring.avail, (IArg)vq->vring.avail);
342 /* There's nothing available? */
343 if (vq->last_avail_idx == vq->vring.avail->idx) {
344 /* We need to know about added buffers */
345 vq->vring.used->flags &= ~VRING_USED_F_NO_NOTIFY;
346 head = (-1);
347 }
348 else {
349 /*
350 * Grab the next descriptor number they're advertising, and increment
351 * the index we've seen.
352 */
353 head = vq->vring.avail->ring[vq->last_avail_idx++ % vq->vring.num];
355 *buf = mapPAtoVA(vq->vring.desc[head].addr);
356 *len = vq->vring.desc[head].len;
357 }
358 GateAll_leave(vq->gateH, key);
360 return (head);
361 }
363 /*!
364 * ======== VirtQueue_disableCallback ========
365 */
366 Void VirtQueue_disableCallback(VirtQueue_Object *vq)
367 {
368 //TODO
369 Log_print0(Diags_USER1, "VirtQueue_disableCallback called.");
370 }
372 /*!
373 * ======== VirtQueue_enableCallback ========
374 */
375 Bool VirtQueue_enableCallback(VirtQueue_Object *vq)
376 {
377 Log_print0(Diags_USER1, "VirtQueue_enableCallback called.");
379 //TODO
380 return (FALSE);
381 }
383 /*!
384 * ======== VirtQueue_isr ========
385 * Note 'arg' is ignored: it is the Hwi argument, not the mailbox argument.
386 */
387 Void VirtQueue_isr(UArg msg)
388 {
389 VirtQueue_Object *vq;
391 msg = InterruptProxy_intClear(hostProcId, &intInfo);
393 Log_print1(Diags_USER1, "VirtQueue_isr received msg = 0x%x\n", msg);
395 #ifndef SMP
396 if (MultiProc_self() == sysm3ProcId || MultiProc_self() == dsp1ProcId) {
397 #endif
398 switch(msg) {
399 case (UInt)RP_MSG_MBOX_READY:
400 return;
402 case (UInt)RP_MBOX_ECHO_REQUEST:
403 InterruptProxy_intSend(hostProcId, NULL,
404 (UInt)(RP_MBOX_ECHO_REPLY));
405 return;
407 case (UInt)RP_MBOX_ABORT_REQUEST:
408 {
409 /* Suppress Coverity Error: FORWARD_NULL: */
410 // coverity[assign_zero]
411 Fxn f = (Fxn)0x0;
412 Log_print0(Diags_USER1, "Crash on demand ...\n");
413 // coverity[var_deref_op]
414 f();
415 }
416 return;
418 case (UInt)RP_MSG_FLUSH_CACHE:
419 Cache_wbAll();
420 return;
422 #ifndef DSP
423 case (UInt)RP_MSG_HIBERNATION:
424 if (IpcPower_canHibernate() == FALSE) {
425 InterruptProxy_intSend(hostProcId, NULL,
426 (UInt)RP_MSG_HIBERNATION_CANCEL);
427 return;
428 }
430 /* Fall through */
431 case (UInt)RP_MSG_HIBERNATION_FORCE:
432 #ifndef SMP
433 /* Core0 should notify Core1 */
434 if (MultiProc_self() == sysm3ProcId) {
435 InterruptProxy_intSend(appm3ProcId, NULL,
436 (UInt)(RP_MSG_HIBERNATION));
437 }
438 #endif
439 /* Ack request */
440 InterruptProxy_intSend(hostProcId, NULL,
441 (UInt)RP_MSG_HIBERNATION_ACK);
442 IpcPower_suspend();
443 return;
444 #endif
445 default:
446 #if defined(M3_ONLY) && !defined(SMP)
447 /* Check and process any Inter-M3 Offload messages */
448 if (OffloadM3_processSysM3Tasks(msg))
449 return;
450 #endif
452 /*
453 * If the message isn't one of the above, it's either part of the
454 * 2-message synchronization sequence or it a virtqueue message
455 */
456 break;
457 }
458 #ifndef SMP
459 }
460 else if (msg & 0xFFFF0000) {
461 #ifndef DSP
462 if (msg == (UInt)RP_MSG_HIBERNATION) {
463 IpcPower_suspend();
464 }
465 #endif
466 return;
467 }
469 if (MultiProc_self() == sysm3ProcId && (msg == ID_A9_TO_APPM3 || msg == ID_APPM3_TO_A9)) {
470 InterruptProxy_intSend(appm3ProcId, NULL, (UInt)msg);
471 }
472 else {
473 #endif
474 /* Don't let unknown messages to pass as a virtqueue index */
475 if (msg >= NUM_QUEUES) {
476 /* Adding print here deliberately, we should never see this */
477 System_printf("VirtQueue_isr: Invalid mailbox message 0x%x "
478 "received\n", msg);
479 return;
480 }
482 vq = queueRegistry[msg];
483 if (vq) {
484 vq->callback(vq);
485 }
486 #ifndef SMP
487 }
488 #endif
489 }
492 /*!
493 * ======== VirtQueue_create ========
494 */
495 VirtQueue_Handle VirtQueue_create(UInt16 remoteProcId, VirtQueue_Params *params,
496 Error_Block *eb)
497 {
498 VirtQueue_Object *vq;
499 Void *vringAddr;
501 vq = Memory_alloc(NULL, sizeof(VirtQueue_Object), 0, eb);
502 if (NULL == vq) {
503 return (NULL);
504 }
506 /* Create the thread protection gate */
507 vq->gateH = GateAll_create(NULL, eb);
508 if (Error_check(eb)) {
509 Log_error0("VirtQueue_create: could not create gate object");
510 Memory_free(NULL, vq, sizeof(VirtQueue_Object));
511 return (NULL);
512 }
514 vq->callback = params->callback;
515 vq->id = params->vqId;
516 vq->procId = remoteProcId;
517 vq->last_avail_idx = 0;
519 #ifndef SMP
520 if (MultiProc_self() == appm3ProcId) {
521 /* vqindices that belong to AppM3 should be big so they don't
522 * collide with SysM3's virtqueues */
523 vq->id += 2;
524 }
525 #endif
527 switch (vq->id) {
528 /* IPC transport vrings */
529 case ID_SELF_TO_A9:
530 /* IPU/DSP -> A9 */
531 vringAddr = (struct vring *) IPC_MEM_VRING0;
532 break;
533 case ID_A9_TO_SELF:
534 /* A9 -> IPU/DSP */
535 vringAddr = (struct vring *) IPC_MEM_VRING1;
536 break;
537 #ifndef SMP
538 case ID_APPM3_TO_A9:
539 /* APPM3 -> A9 */
540 vringAddr = (struct vring *) IPC_MEM_VRING2;
541 break;
542 case ID_A9_TO_APPM3:
543 /* A9 -> APPM3 */
544 vringAddr = (struct vring *) IPC_MEM_VRING3;
545 break;
546 #endif
547 default:
548 GateAll_delete(&vq->gateH);
549 Memory_free(NULL, vq, sizeof(VirtQueue_Object));
550 return (NULL);
551 }
553 Log_print3(Diags_USER1,
554 "vring: %d 0x%x (0x%x)\n", vq->id, (IArg)vringAddr,
555 RP_MSG_RING_SIZE);
557 /* See coverity related comment in vring_init() */
558 // coverity[overrun-call]
559 vring_init(&(vq->vring), RP_MSG_NUM_BUFS, vringAddr, RP_MSG_VRING_ALIGN);
561 /*
562 * Don't trigger a mailbox message every time MPU makes another buffer
563 * available
564 */
565 if (vq->procId == hostProcId) {
566 vq->vring.used->flags |= VRING_USED_F_NO_NOTIFY;
567 }
569 queueRegistry[vq->id] = vq;
571 return (vq);
572 }
574 /*!
575 * ======== VirtQueue_startup ========
576 */
577 Void VirtQueue_startup()
578 {
579 hostProcId = MultiProc_getId("HOST");
580 #ifndef SMP
581 dsp1ProcId = MultiProc_getId("DSP1");
582 sysm3ProcId = MultiProc_getId("CORE0");
583 appm3ProcId = MultiProc_getId("CORE1");
584 #endif
586 #ifdef DSP
587 intInfo.intVectorId = DSPEVENTID;
588 #else
589 /* Initilize the IpcPower module */
590 IpcPower_init();
591 #endif
593 InterruptProxy_intRegister(hostProcId, &intInfo, (Fxn)VirtQueue_isr, NULL);
594 }
596 /*!
597 * ======== VirtQueue_postCrashToMailbox ========
598 */
599 Void VirtQueue_postCrashToMailbox(Void)
600 {
601 InterruptProxy_intSend(0, NULL, (UInt)RP_MSG_MBOX_CRASH);
602 }
604 #define CACHE_WB_TICK_PERIOD 5
606 /*!
607 * ======== VirtQueue_cacheWb ========
608 *
609 * Used for flushing SysMin trace buffer.
610 */
611 Void VirtQueue_cacheWb()
612 {
613 static UInt32 oldticks = 0;
614 UInt32 newticks;
616 newticks = Clock_getTicks();
617 if (newticks - oldticks < (UInt32)CACHE_WB_TICK_PERIOD) {
618 /* Don't keep flushing cache */
619 return;
620 }
622 oldticks = newticks;
624 /* Flush the cache */
625 Cache_wbAll();
626 }