drivers/soc/ti/knav_qmss_acc.c

   1 /*
   2  * Keystone accumulator queue manager
   3  *
   4  * Copyright (C) 2014 Texas Instruments Incorporated - http://www.ti.com
   5  * Author:      Sandeep Nair <sandeep_n@ti.com>
   6  *              Cyril Chemparathy <cyril@ti.com>
   7  *              Santosh Shilimkar <santosh.shilimkar@ti.com>
   8  *
   9  * This program is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU General Public License
  11  * version 2 as published by the Free Software Foundation.
  12  *
  13  * This program is distributed in the hope that it will be useful, but
  14  * WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * General Public License for more details.
  17  */
  18
  19 #include <linux/dma-mapping.h>
  20 #include <linux/io.h>
  21 #include <linux/interrupt.h>
  22 #include <linux/module.h>
  23 #include <linux/of_address.h>
  24 #include <linux/soc/ti/knav_qmss.h>
  25
  26 #include "knav_qmss.h"
  27
  28 #define knav_range_offset_to_inst(kdev, range, q)       \
  29         (range->queue_base_inst + (q << kdev->inst_shift))
  30
  31 static void __knav_acc_notify(struct knav_range_info *range,
  32                                 struct knav_acc_channel *acc)
  33 {
  34         struct knav_device *kdev = range->kdev;
  35         struct knav_queue_inst *inst;
  36         int range_base, queue;
  37
  38         range_base = kdev->base_id + range->queue_base;
  39
  40         if (range->flags & RANGE_MULTI_QUEUE) {
  41                 for (queue = 0; queue < range->num_queues; queue++) {
  42                         inst = knav_range_offset_to_inst(kdev, range,
  43                                                                 queue);
  44                         if (inst->notify_needed) {
  45                                 inst->notify_needed = 0;
  46                                 dev_dbg(kdev->dev, "acc-irq: notifying %d\n",
  47                                         range_base + queue);
  48                                 knav_queue_notify(inst);
  49                         }
  50                 }
  51         } else {
  52                 queue = acc->channel - range->acc_info.start_channel;
  53                 inst = knav_range_offset_to_inst(kdev, range, queue);
  54                 dev_dbg(kdev->dev, "acc-irq: notifying %d\n",
  55                         range_base + queue);
  56                 knav_queue_notify(inst);
  57         }
  58 }
  59
  60 static int knav_acc_set_notify(struct knav_range_info *range,
  61                                 struct knav_queue_inst *kq,
  62                                 bool enabled)
  63 {
  64         struct knav_pdsp_info *pdsp = range->acc_info.pdsp;
  65         struct knav_device *kdev = range->kdev;
  66         u32 mask, offset;
  67
  68         /*
  69          * when enabling, we need to re-trigger an interrupt if we
  70          * have descriptors pending
  71          */
  72         if (!enabled || atomic_read(&kq->desc_count) <= 0)
  73                 return 0;
  74
  75         kq->notify_needed = 1;
  76         atomic_inc(&kq->acc->retrigger_count);
  77         mask = BIT(kq->acc->channel % 32);
  78         offset = ACC_INTD_OFFSET_STATUS(kq->acc->channel);
  79         dev_dbg(kdev->dev, "setup-notify: re-triggering irq for %s\n",
  80                 kq->acc->name);
  81         writel_relaxed(mask, pdsp->intd + offset);
  82         return 0;
  83 }
  84
  85 static irqreturn_t knav_acc_int_handler(int irq, void *_instdata)
  86 {
  87         struct knav_acc_channel *acc;
  88         struct knav_queue_inst *kq = NULL;
  89         struct knav_range_info *range;
  90         struct knav_pdsp_info *pdsp;
  91         struct knav_acc_info *info;
  92         struct knav_device *kdev;
  93
  94         u32 *list, *list_cpu, val, idx, notifies;
  95         int range_base, channel, queue = 0;
  96         dma_addr_t list_dma;
  97
  98         range = _instdata;
  99         info  = &range->acc_info;
 100         kdev  = range->kdev;
 101         pdsp  = range->acc_info.pdsp;
 102         acc   = range->acc;
 103
 104         range_base = kdev->base_id + range->queue_base;
 105         if ((range->flags & RANGE_MULTI_QUEUE) == 0) {
 106                 for (queue = 0; queue < range->num_irqs; queue++)
 107                         if (range->irqs[queue].irq == irq)
 108                                 break;
 109                 kq = knav_range_offset_to_inst(kdev, range, queue);
 110                 acc += queue;
 111         }
 112
 113         channel = acc->channel;
 114         list_dma = acc->list_dma[acc->list_index];
 115         list_cpu = acc->list_cpu[acc->list_index];
 116         dev_dbg(kdev->dev, "acc-irq: channel %d, list %d, virt %p, dma %pad\n",
 117                 channel, acc->list_index, list_cpu, &list_dma);
 118         if (atomic_read(&acc->retrigger_count)) {
 119                 atomic_dec(&acc->retrigger_count);
 120                 __knav_acc_notify(range, acc);
 121                 writel_relaxed(1, pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
 122                 /* ack the interrupt */
 123                 writel_relaxed(ACC_CHANNEL_INT_BASE + channel,
 124                                pdsp->intd + ACC_INTD_OFFSET_EOI);
 125
 126                 return IRQ_HANDLED;
 127         }
 128
 129         notifies = readl_relaxed(pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
 130         WARN_ON(!notifies);
 131         dma_sync_single_for_cpu(kdev->dev, list_dma, info->list_size,
 132                                 DMA_FROM_DEVICE);
 133
 134         for (list = list_cpu; list < list_cpu + (info->list_size / sizeof(u32));
 135              list += ACC_LIST_ENTRY_WORDS) {
 136                 if (ACC_LIST_ENTRY_WORDS == 1) {
 137                         dev_dbg(kdev->dev,
 138                                 "acc-irq: list %d, entry @%p, %08x\n",
 139                                 acc->list_index, list, list[0]);
 140                 } else if (ACC_LIST_ENTRY_WORDS == 2) {
 141                         dev_dbg(kdev->dev,
 142                                 "acc-irq: list %d, entry @%p, %08x %08x\n",
 143                                 acc->list_index, list, list[0], list[1]);
 144                 } else if (ACC_LIST_ENTRY_WORDS == 4) {
 145                         dev_dbg(kdev->dev,
 146                                 "acc-irq: list %d, entry @%p, %08x %08x %08x %08x\n",
 147                                 acc->list_index, list, list[0], list[1],
 148                                 list[2], list[3]);
 149                 }
 150
 151                 val = list[ACC_LIST_ENTRY_DESC_IDX];
 152                 if (!val)
 153                         break;
 154
 155                 if (range->flags & RANGE_MULTI_QUEUE) {
 156                         queue = list[ACC_LIST_ENTRY_QUEUE_IDX] >> 16;
 157                         if (queue < range_base ||
 158                             queue >= range_base + range->num_queues) {
 159                                 dev_err(kdev->dev,
 160                                         "bad queue %d, expecting %d-%d\n",
 161                                         queue, range_base,
 162                                         range_base + range->num_queues);
 163                                 break;
 164                         }
 165                         queue -= range_base;
 166                         kq = knav_range_offset_to_inst(kdev, range,
 167                                                                 queue);
 168                 }
 169
 170                 if (atomic_inc_return(&kq->desc_count) >= ACC_DESCS_MAX) {
 171                         atomic_dec(&kq->desc_count);
 172                         dev_err(kdev->dev,
 173                                 "acc-irq: queue %d full, entry dropped\n",
 174                                 queue + range_base);
 175                         continue;
 176                 }
 177
 178                 idx = atomic_inc_return(&kq->desc_tail) & ACC_DESCS_MASK;
 179                 kq->descs[idx] = val;
 180                 kq->notify_needed = 1;
 181                 dev_dbg(kdev->dev, "acc-irq: enqueue %08x at %d, queue %d\n",
 182                         val, idx, queue + range_base);
 183         }
 184
 185         __knav_acc_notify(range, acc);
 186         memset(list_cpu, 0, info->list_size);
 187         dma_sync_single_for_device(kdev->dev, list_dma, info->list_size,
 188                                    DMA_TO_DEVICE);
 189
 190         /* flip to the other list */
 191         acc->list_index ^= 1;
 192
 193         /* reset the interrupt counter */
 194         writel_relaxed(1, pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
 195
 196         /* ack the interrupt */
 197         writel_relaxed(ACC_CHANNEL_INT_BASE + channel,
 198                        pdsp->intd + ACC_INTD_OFFSET_EOI);
 199
 200         return IRQ_HANDLED;
 201 }
 202
 203 static int knav_range_setup_acc_irq(struct knav_range_info *range,
 204                                 int queue, bool enabled)
 205 {
 206         struct knav_device *kdev = range->kdev;
 207         struct knav_acc_channel *acc;
 208         unsigned long cpu_map;
 209         int ret = 0, irq;
 210         u32 old, new;
 211
 212         if (range->flags & RANGE_MULTI_QUEUE) {
 213                 acc = range->acc;
 214                 irq = range->irqs[0].irq;
 215                 cpu_map = range->irqs[0].cpu_map;
 216         } else {
 217                 acc = range->acc + queue;
 218                 irq = range->irqs[queue].irq;
 219                 cpu_map = range->irqs[queue].cpu_map;
 220         }
 221
 222         old = acc->open_mask;
 223         if (enabled)
 224                 new = old | BIT(queue);
 225         else
 226                 new = old & ~BIT(queue);
 227         acc->open_mask = new;
 228
 229         dev_dbg(kdev->dev,
 230                 "setup-acc-irq: open mask old %08x, new %08x, channel %s\n",
 231                 old, new, acc->name);
 232
 233         if (likely(new == old))
 234                 return 0;
 235
 236         if (new && !old) {
 237                 dev_dbg(kdev->dev,
 238                         "setup-acc-irq: requesting %s for channel %s\n",
 239                         acc->name, acc->name);
 240                 ret = request_irq(irq, knav_acc_int_handler, 0, acc->name,
 241                                   range);
 242                 if (!ret && cpu_map) {
 243                         ret = irq_set_affinity_hint(irq, to_cpumask(&cpu_map));
 244                         if (ret) {
 245                                 dev_warn(range->kdev->dev,
 246                                          "Failed to set IRQ affinity\n");
 247                                 return ret;
 248                         }
 249                 }
 250         }
 251
 252         if (old && !new) {
 253                 dev_dbg(kdev->dev, "setup-acc-irq: freeing %s for channel %s\n",
 254                         acc->name, acc->name);
 255                 ret = irq_set_affinity_hint(irq, NULL);
 256                 if (ret)
 257                         dev_warn(range->kdev->dev,
 258                                  "Failed to set IRQ affinity\n");
 259                 free_irq(irq, range);
 260         }
 261
 262         return ret;
 263 }
 264
 265 static const char *knav_acc_result_str(enum knav_acc_result result)
 266 {
 267         static const char * const result_str[] = {
 268                 [ACC_RET_IDLE]                  = "idle",
 269                 [ACC_RET_SUCCESS]               = "success",
 270                 [ACC_RET_INVALID_COMMAND]       = "invalid command",
 271                 [ACC_RET_INVALID_CHANNEL]       = "invalid channel",
 272                 [ACC_RET_INACTIVE_CHANNEL]      = "inactive channel",
 273                 [ACC_RET_ACTIVE_CHANNEL]        = "active channel",
 274                 [ACC_RET_INVALID_QUEUE]         = "invalid queue",
 275                 [ACC_RET_INVALID_RET]           = "invalid return code",
 276         };
 277
 278         if (result >= ARRAY_SIZE(result_str))
 279                 return result_str[ACC_RET_INVALID_RET];
 280         else
 281                 return result_str[result];
 282 }
 283
 284 static enum knav_acc_result
 285 knav_acc_write(struct knav_device *kdev, struct knav_pdsp_info *pdsp,
 286                 struct knav_reg_acc_command *cmd)
 287 {
 288         u32 result;
 289
 290         dev_dbg(kdev->dev, "acc command %08x %08x %08x %08x %08x\n",
 291                 cmd->command, cmd->queue_mask, cmd->list_dma,
 292                 cmd->queue_num, cmd->timer_config);
 293
 294         writel_relaxed(cmd->timer_config, &pdsp->acc_command->timer_config);
 295         writel_relaxed(cmd->queue_num, &pdsp->acc_command->queue_num);
 296         writel_relaxed(cmd->list_dma, &pdsp->acc_command->list_dma);
 297         writel_relaxed(cmd->queue_mask, &pdsp->acc_command->queue_mask);
 298         writel_relaxed(cmd->command, &pdsp->acc_command->command);
 299
 300         /* wait for the command to clear */
 301         do {
 302                 result = readl_relaxed(&pdsp->acc_command->command);
 303         } while ((result >> 8) & 0xff);
 304
 305         return (result >> 24) & 0xff;
 306 }
 307
 308 static void knav_acc_setup_cmd(struct knav_device *kdev,
 309                                 struct knav_range_info *range,
 310                                 struct knav_reg_acc_command *cmd,
 311                                 int queue)
 312 {
 313         struct knav_acc_info *info = &range->acc_info;
 314         struct knav_acc_channel *acc;
 315         int queue_base;
 316         u32 queue_mask;
 317
 318         if (range->flags & RANGE_MULTI_QUEUE) {
 319                 acc = range->acc;
 320                 queue_base = range->queue_base;
 321                 queue_mask = BIT(range->num_queues) - 1;
 322         } else {
 323                 acc = range->acc + queue;
 324                 queue_base = range->queue_base + queue;
 325                 queue_mask = 0;
 326         }
 327
 328         memset(cmd, 0, sizeof(*cmd));
 329         cmd->command    = acc->channel;
 330         cmd->queue_mask = queue_mask;
 331         cmd->list_dma   = (u32)acc->list_dma[0];
 332         cmd->queue_num  = info->list_entries << 16;
 333         cmd->queue_num |= queue_base;
 334
 335         cmd->timer_config = ACC_LIST_ENTRY_TYPE << 18;
 336         if (range->flags & RANGE_MULTI_QUEUE)
 337                 cmd->timer_config |= ACC_CFG_MULTI_QUEUE;
 338         cmd->timer_config |= info->pacing_mode << 16;
 339         cmd->timer_config |= info->timer_count;
 340 }
 341
 342 static void knav_acc_stop(struct knav_device *kdev,
 343                                 struct knav_range_info *range,
 344                                 int queue)
 345 {
 346         struct knav_reg_acc_command cmd;
 347         struct knav_acc_channel *acc;
 348         enum knav_acc_result result;
 349
 350         acc = range->acc + queue;
 351
 352         knav_acc_setup_cmd(kdev, range, &cmd, queue);
 353         cmd.command |= ACC_CMD_DISABLE_CHANNEL << 8;
 354         result = knav_acc_write(kdev, range->acc_info.pdsp, &cmd);
 355
 356         dev_dbg(kdev->dev, "stopped acc channel %s, result %s\n",
 357                 acc->name, knav_acc_result_str(result));
 358 }
 359
 360 static enum knav_acc_result knav_acc_start(struct knav_device *kdev,
 361                                                 struct knav_range_info *range,
 362                                                 int queue)
 363 {
 364         struct knav_reg_acc_command cmd;
 365         struct knav_acc_channel *acc;
 366         enum knav_acc_result result;
 367
 368         acc = range->acc + queue;
 369
 370         knav_acc_setup_cmd(kdev, range, &cmd, queue);
 371         cmd.command |= ACC_CMD_ENABLE_CHANNEL << 8;
 372         result = knav_acc_write(kdev, range->acc_info.pdsp, &cmd);
 373
 374         dev_dbg(kdev->dev, "started acc channel %s, result %s\n",
 375                 acc->name, knav_acc_result_str(result));
 376
 377         return result;
 378 }
 379
 380 static int knav_acc_init_range(struct knav_range_info *range)
 381 {
 382         struct knav_device *kdev = range->kdev;
 383         struct knav_acc_channel *acc;
 384         enum knav_acc_result result;
 385         int queue;
 386
 387         for (queue = 0; queue < range->num_queues; queue++) {
 388                 acc = range->acc + queue;
 389
 390                 knav_acc_stop(kdev, range, queue);
 391                 acc->list_index = 0;
 392                 result = knav_acc_start(kdev, range, queue);
 393
 394                 if (result != ACC_RET_SUCCESS)
 395                         return -EIO;
 396
 397                 if (range->flags & RANGE_MULTI_QUEUE)
 398                         return 0;
 399         }
 400         return 0;
 401 }
 402
 403 static int knav_acc_init_queue(struct knav_range_info *range,
 404                                 struct knav_queue_inst *kq)
 405 {
 406         unsigned id = kq->id - range->queue_base;
 407
 408         kq->descs = devm_kcalloc(range->kdev->dev,
 409                                  ACC_DESCS_MAX, sizeof(u32), GFP_KERNEL);
 410         if (!kq->descs)
 411                 return -ENOMEM;
 412
 413         kq->acc = range->acc;
 414         if ((range->flags & RANGE_MULTI_QUEUE) == 0)
 415                 kq->acc += id;
 416         return 0;
 417 }
 418
 419 static int knav_acc_open_queue(struct knav_range_info *range,
 420                                 struct knav_queue_inst *inst, unsigned flags)
 421 {
 422         unsigned id = inst->id - range->queue_base;
 423
 424         return knav_range_setup_acc_irq(range, id, true);
 425 }
 426
 427 static int knav_acc_close_queue(struct knav_range_info *range,
 428                                         struct knav_queue_inst *inst)
 429 {
 430         unsigned id = inst->id - range->queue_base;
 431
 432         return knav_range_setup_acc_irq(range, id, false);
 433 }
 434
 435 static int knav_acc_free_range(struct knav_range_info *range)
 436 {
 437         struct knav_device *kdev = range->kdev;
 438         struct knav_acc_channel *acc;
 439         struct knav_acc_info *info;
 440         int channel, channels;
 441
 442         info = &range->acc_info;
 443
 444         if (range->flags & RANGE_MULTI_QUEUE)
 445                 channels = 1;
 446         else
 447                 channels = range->num_queues;
 448
 449         for (channel = 0; channel < channels; channel++) {
 450                 acc = range->acc + channel;
 451                 if (!acc->list_cpu[0])
 452                         continue;
 453                 dma_unmap_single(kdev->dev, acc->list_dma[0],
 454                                  info->mem_size, DMA_BIDIRECTIONAL);
 455                 free_pages_exact(acc->list_cpu[0], info->mem_size);
 456         }
 457         devm_kfree(range->kdev->dev, range->acc);
 458         return 0;
 459 }
 460
 461 struct knav_range_ops knav_acc_range_ops = {
 462         .set_notify     = knav_acc_set_notify,
 463         .init_queue     = knav_acc_init_queue,
 464         .open_queue     = knav_acc_open_queue,
 465         .close_queue    = knav_acc_close_queue,
 466         .init_range     = knav_acc_init_range,
 467         .free_range     = knav_acc_free_range,
 468 };
 469
 470 /**
 471  * knav_init_acc_range: Initialise accumulator ranges
 472  *
 473  * @kdev:               qmss device
 474  * @node:               device node
 475  * @range:              qmms range information
 476  *
 477  * Return 0 on success or error
 478  */
 479 int knav_init_acc_range(struct knav_device *kdev,
 480                         struct device_node *node,
 481                         struct knav_range_info *range)
 482 {
 483         struct knav_acc_channel *acc;
 484         struct knav_pdsp_info *pdsp;
 485         struct knav_acc_info *info;
 486         int ret, channel, channels;
 487         int list_size, mem_size;
 488         dma_addr_t list_dma;
 489         void *list_mem;
 490         u32 config[5];
 491
 492         range->flags |= RANGE_HAS_ACCUMULATOR;
 493         info = &range->acc_info;
 494
 495         ret = of_property_read_u32_array(node, "accumulator", config, 5);
 496         if (ret)
 497                 return ret;
 498
 499         info->pdsp_id           = config[0];
 500         info->start_channel     = config[1];
 501         info->list_entries      = config[2];
 502         info->pacing_mode       = config[3];
 503         info->timer_count       = config[4] / ACC_DEFAULT_PERIOD;
 504
 505         if (info->start_channel > ACC_MAX_CHANNEL) {
 506                 dev_err(kdev->dev, "channel %d invalid for range %s\n",
 507                         info->start_channel, range->name);
 508                 return -EINVAL;
 509         }
 510
 511         if (info->pacing_mode > 3) {
 512                 dev_err(kdev->dev, "pacing mode %d invalid for range %s\n",
 513                         info->pacing_mode, range->name);
 514                 return -EINVAL;
 515         }
 516
 517         pdsp = knav_find_pdsp(kdev, info->pdsp_id);
 518         if (!pdsp) {
 519                 dev_err(kdev->dev, "pdsp id %d not found for range %s\n",
 520                         info->pdsp_id, range->name);
 521                 return -EINVAL;
 522         }
 523
 524         if (!pdsp->started) {
 525                 dev_err(kdev->dev, "pdsp id %d not started for range %s\n",
 526                         info->pdsp_id, range->name);
 527                 return -ENODEV;
 528         }
 529
 530         info->pdsp = pdsp;
 531         channels = range->num_queues;
 532         if (of_get_property(node, "multi-queue", NULL)) {
 533                 range->flags |= RANGE_MULTI_QUEUE;
 534                 channels = 1;
 535                 if (range->queue_base & (32 - 1)) {
 536                         dev_err(kdev->dev,
 537                                 "misaligned multi-queue accumulator range %s\n",
 538                                 range->name);
 539                         return -EINVAL;
 540                 }
 541                 if (range->num_queues > 32) {
 542                         dev_err(kdev->dev,
 543                                 "too many queues in accumulator range %s\n",
 544                                 range->name);
 545                         return -EINVAL;
 546                 }
 547         }
 548
 549         /* figure out list size */
 550         list_size  = info->list_entries;
 551         list_size *= ACC_LIST_ENTRY_WORDS * sizeof(u32);
 552         info->list_size = list_size;
 553         mem_size   = PAGE_ALIGN(list_size * 2);
 554         info->mem_size  = mem_size;
 555         range->acc = devm_kcalloc(kdev->dev, channels, sizeof(*range->acc),
 556                                   GFP_KERNEL);
 557         if (!range->acc)
 558                 return -ENOMEM;
 559
 560         for (channel = 0; channel < channels; channel++) {
 561                 acc = range->acc + channel;
 562                 acc->channel = info->start_channel + channel;
 563
 564                 /* allocate memory for the two lists */
 565                 list_mem = alloc_pages_exact(mem_size, GFP_KERNEL | GFP_DMA);
 566                 if (!list_mem)
 567                         return -ENOMEM;
 568
 569                 list_dma = dma_map_single(kdev->dev, list_mem, mem_size,
 570                                           DMA_BIDIRECTIONAL);
 571                 if (dma_mapping_error(kdev->dev, list_dma)) {
 572                         free_pages_exact(list_mem, mem_size);
 573                         return -ENOMEM;
 574                 }
 575
 576                 memset(list_mem, 0, mem_size);
 577                 dma_sync_single_for_device(kdev->dev, list_dma, mem_size,
 578                                            DMA_TO_DEVICE);
 579                 scnprintf(acc->name, sizeof(acc->name), "hwqueue-acc-%d",
 580                           acc->channel);
 581                 acc->list_cpu[0] = list_mem;
 582                 acc->list_cpu[1] = list_mem + list_size;
 583                 acc->list_dma[0] = list_dma;
 584                 acc->list_dma[1] = list_dma + list_size;
 585                 dev_dbg(kdev->dev, "%s: channel %d, dma %pad, virt %8p\n",
 586                         acc->name, acc->channel, &list_dma, list_mem);
 587         }
 588
 589         range->ops = &knav_acc_range_ops;
 590         return 0;
 591 }
 592 EXPORT_SYMBOL_GPL(knav_init_acc_range);