libdrm/nouveau/nouveau_pushbuf.c

   1 /*
   2  * Copyright 2007 Nouveau Project
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
  18  * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  19  * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  20  * SOFTWARE.
  21  */
  22
  23 #include <stdio.h>
  24 #include <stdlib.h>
  25 #include <errno.h>
  26 #include <assert.h>
  27
  28 #include "nouveau_private.h"
  29
  30 #define PB_BUFMGR_DWORDS   (4096 / 2)
  31 #define PB_MIN_USER_DWORDS  2048
  32
  33 static uint32_t
  34 nouveau_pushbuf_calc_reloc(struct drm_nouveau_gem_pushbuf_bo *pbbo,
  35                            struct drm_nouveau_gem_pushbuf_reloc *r)
  36 {
  37         uint32_t push = 0;
  38
  39         if (r->flags & NOUVEAU_GEM_RELOC_LOW)
  40                 push = (pbbo->presumed_offset + r->data);
  41         else
  42         if (r->flags & NOUVEAU_GEM_RELOC_HIGH)
  43                 push = (pbbo->presumed_offset + r->data) >> 32;
  44         else
  45                 push = r->data;
  46
  47         if (r->flags & NOUVEAU_GEM_RELOC_OR) {
  48                 if (pbbo->presumed_domain & NOUVEAU_GEM_DOMAIN_VRAM)
  49                         push |= r->vor;
  50                 else
  51                         push |= r->tor;
  52         }
  53
  54         return push;
  55 }
  56
  57 int
  58 nouveau_pushbuf_emit_reloc(struct nouveau_channel *chan, void *ptr,
  59                            struct nouveau_bo *bo, uint32_t data, uint32_t data2,
  60                            uint32_t flags, uint32_t vor, uint32_t tor)
  61 {
  62         struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(chan->pushbuf);
  63         struct nouveau_bo_priv *nvbo = nouveau_bo(bo);
  64         struct drm_nouveau_gem_pushbuf_reloc *r;
  65         struct drm_nouveau_gem_pushbuf_bo *pbbo;
  66         uint32_t domains = 0;
  67
  68         if (nvpb->nr_relocs >= NOUVEAU_GEM_MAX_RELOCS) {
  69                 fprintf(stderr, "too many relocs!!\n");
  70                 return -ENOMEM;
  71         }
  72
  73         if (nvbo->user && (flags & NOUVEAU_BO_WR)) {
  74                 fprintf(stderr, "write to user buffer!!\n");
  75                 return -EINVAL;
  76         }
  77
  78         pbbo = nouveau_bo_emit_buffer(chan, bo);
  79         if (!pbbo) {
  80                 fprintf(stderr, "buffer emit fail :(\n");
  81                 return -ENOMEM;
  82         }
  83
  84         nvbo->pending_refcnt++;
  85
  86         if (flags & NOUVEAU_BO_VRAM)
  87                 domains |= NOUVEAU_GEM_DOMAIN_VRAM;
  88         if (flags & NOUVEAU_BO_GART)
  89                 domains |= NOUVEAU_GEM_DOMAIN_GART;
  90
  91         if (!(pbbo->valid_domains & domains)) {
  92                 fprintf(stderr, "no valid domains remain!\n");
  93                 return -EINVAL;
  94         }
  95         pbbo->valid_domains &= domains;
  96
  97         assert(flags & NOUVEAU_BO_RDWR);
  98         if (flags & NOUVEAU_BO_RD) {
  99                 pbbo->read_domains |= domains;
 100         }
 101         if (flags & NOUVEAU_BO_WR) {
 102                 pbbo->write_domains |= domains;
 103                 nvbo->write_marker = 1;
 104         }
 105
 106         r = nvpb->relocs + nvpb->nr_relocs++;
 107         r->bo_index = pbbo - nvpb->buffers;
 108         r->reloc_index = (uint32_t *)ptr - nvpb->pushbuf;
 109         r->flags = 0;
 110         if (flags & NOUVEAU_BO_LOW)
 111                 r->flags |= NOUVEAU_GEM_RELOC_LOW;
 112         if (flags & NOUVEAU_BO_HIGH)
 113                 r->flags |= NOUVEAU_GEM_RELOC_HIGH;
 114         if (flags & NOUVEAU_BO_OR)
 115                 r->flags |= NOUVEAU_GEM_RELOC_OR;
 116         r->data = data;
 117         r->vor = vor;
 118         r->tor = tor;
 119
 120         *(uint32_t *)ptr = (flags & NOUVEAU_BO_DUMMY) ? 0 :
 121                 nouveau_pushbuf_calc_reloc(pbbo, r);
 122         return 0;
 123 }
 124
 125 static int
 126 nouveau_pushbuf_space_call(struct nouveau_channel *chan, unsigned min)
 127 {
 128         struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
 129         struct nouveau_pushbuf_priv *nvpb = &nvchan->pb;
 130         struct nouveau_bo *bo;
 131         int ret;
 132
 133         if (min < PB_MIN_USER_DWORDS)
 134                 min = PB_MIN_USER_DWORDS;
 135
 136         nvpb->current_offset = nvpb->base.cur - nvpb->pushbuf;
 137         if (nvpb->current_offset + min + 2 <= nvpb->size)
 138                 return 0;
 139
 140         nvpb->current++;
 141         if (nvpb->current == CALPB_BUFFERS)
 142                 nvpb->current = 0;
 143         bo = nvpb->buffer[nvpb->current];
 144
 145         ret = nouveau_bo_map(bo, NOUVEAU_BO_WR);
 146         if (ret)
 147                 return ret;
 148
 149         nvpb->size = (bo->size - 8) / 4;
 150         nvpb->pushbuf = bo->map;
 151         nvpb->current_offset = 0;
 152
 153         nvpb->base.channel = chan;
 154         nvpb->base.remaining = nvpb->size;
 155         nvpb->base.cur = nvpb->pushbuf;
 156
 157         nouveau_bo_unmap(bo);
 158         return 0;
 159 }
 160
 161 static int
 162 nouveau_pushbuf_space(struct nouveau_channel *chan, unsigned min)
 163 {
 164         struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
 165         struct nouveau_pushbuf_priv *nvpb = &nvchan->pb;
 166
 167         if (nvpb->use_cal)
 168                 return nouveau_pushbuf_space_call(chan, min);
 169
 170         if (nvpb->pushbuf) {
 171                 free(nvpb->pushbuf);
 172                 nvpb->pushbuf = NULL;
 173         }
 174
 175         nvpb->size = min < PB_MIN_USER_DWORDS ? PB_MIN_USER_DWORDS : min;
 176         nvpb->pushbuf = malloc(sizeof(uint32_t) * nvpb->size);
 177
 178         nvpb->base.channel = chan;
 179         nvpb->base.remaining = nvpb->size;
 180         nvpb->base.cur = nvpb->pushbuf;
 181
 182         return 0;
 183 }
 184
 185 static void
 186 nouveau_pushbuf_fini_call(struct nouveau_channel *chan)
 187 {
 188         struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
 189         struct nouveau_pushbuf_priv *nvpb = &nvchan->pb;
 190         int i;
 191
 192         for (i = 0; i < CALPB_BUFFERS; i++)
 193                 nouveau_bo_ref(NULL, &nvpb->buffer[i]);
 194         nvpb->use_cal = 0;
 195         nvpb->pushbuf = NULL;
 196 }
 197
 198 static void
 199 nouveau_pushbuf_init_call(struct nouveau_channel *chan)
 200 {
 201         struct drm_nouveau_gem_pushbuf_call req;
 202         struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
 203         struct nouveau_pushbuf_priv *nvpb = &nvchan->pb;
 204         struct nouveau_device *dev = chan->device;
 205         int i, ret;
 206
 207         req.channel = chan->id;
 208         req.handle = 0;
 209         ret = drmCommandWriteRead(nouveau_device(dev)->fd,
 210                                   DRM_NOUVEAU_GEM_PUSHBUF_CALL2,
 211                                   &req, sizeof(req));
 212         if (ret) {
 213                 ret = drmCommandWriteRead(nouveau_device(dev)->fd,
 214                                           DRM_NOUVEAU_GEM_PUSHBUF_CALL2,
 215                                           &req, sizeof(req));
 216                 if (ret)
 217                         return;
 218
 219                 nvpb->no_aper_update = 1;
 220         }
 221
 222         for (i = 0; i < CALPB_BUFFERS; i++) {
 223                 ret = nouveau_bo_new(dev, NOUVEAU_BO_GART | NOUVEAU_BO_MAP,
 224                                      0, CALPB_BUFSZ, &nvpb->buffer[i]);
 225                 if (ret) {
 226                         nouveau_pushbuf_fini_call(chan);
 227                         return;
 228                 }
 229         }
 230
 231         nvpb->use_cal = 1;
 232         nvpb->cal_suffix0 = req.suffix0;
 233         nvpb->cal_suffix1 = req.suffix1;
 234 }
 235
 236 int
 237 nouveau_pushbuf_init(struct nouveau_channel *chan)
 238 {
 239         struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
 240         struct nouveau_pushbuf_priv *nvpb = &nvchan->pb;
 241         int ret;
 242
 243         nouveau_pushbuf_init_call(chan);
 244
 245         ret = nouveau_pushbuf_space(chan, 0);
 246         if (ret) {
 247                 if (nvpb->use_cal) {
 248                         nouveau_pushbuf_fini_call(chan);
 249                         ret = nouveau_pushbuf_space(chan, 0);
 250                 }
 251
 252                 if (ret)
 253                         return ret;
 254         }
 255
 256         nvpb->buffers = calloc(NOUVEAU_GEM_MAX_BUFFERS,
 257                                sizeof(struct drm_nouveau_gem_pushbuf_bo));
 258         nvpb->relocs = calloc(NOUVEAU_GEM_MAX_RELOCS,
 259                               sizeof(struct drm_nouveau_gem_pushbuf_reloc));
 260
 261         chan->pushbuf = &nvpb->base;
 262         return 0;
 263 }
 264
 265 int
 266 nouveau_pushbuf_flush(struct nouveau_channel *chan, unsigned min)
 267 {
 268         struct nouveau_device_priv *nvdev = nouveau_device(chan->device);
 269         struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
 270         struct nouveau_pushbuf_priv *nvpb = &nvchan->pb;
 271         unsigned i;
 272         int ret;
 273
 274         if (nvpb->base.remaining == nvpb->size)
 275                 return 0;
 276
 277         if (nvpb->use_cal) {
 278                 struct drm_nouveau_gem_pushbuf_call req;
 279
 280                 *(nvpb->base.cur++) = nvpb->cal_suffix0;
 281                 *(nvpb->base.cur++) = nvpb->cal_suffix1;
 282                 if (nvpb->base.remaining > 2) /* space() will fixup if not */
 283                         nvpb->base.remaining -= 2;
 284
 285 restart_cal:
 286                 req.channel = chan->id;
 287                 req.handle = nvpb->buffer[nvpb->current]->handle;
 288                 req.offset = nvpb->current_offset * 4;
 289                 req.nr_buffers = nvpb->nr_buffers;
 290                 req.buffers = (uint64_t)(unsigned long)nvpb->buffers;
 291                 req.nr_relocs = nvpb->nr_relocs;
 292                 req.relocs = (uint64_t)(unsigned long)nvpb->relocs;
 293                 req.nr_dwords = (nvpb->base.cur - nvpb->pushbuf) -
 294                                 nvpb->current_offset;
 295                 req.suffix0 = nvpb->cal_suffix0;
 296                 req.suffix1 = nvpb->cal_suffix1;
 297                 ret = drmCommandWriteRead(nvdev->fd, nvpb->no_aper_update ?
 298                                           DRM_NOUVEAU_GEM_PUSHBUF_CALL :
 299                                           DRM_NOUVEAU_GEM_PUSHBUF_CALL2,
 300                                           &req, sizeof(req));
 301                 if (ret == -EAGAIN)
 302                         goto restart_cal;
 303                 nvpb->cal_suffix0 = req.suffix0;
 304                 nvpb->cal_suffix1 = req.suffix1;
 305                 if (!nvpb->no_aper_update) {
 306                         nvdev->base.vm_vram_size = req.vram_available;
 307                         nvdev->base.vm_gart_size = req.gart_available;
 308                 }
 309         } else {
 310                 struct drm_nouveau_gem_pushbuf req;
 311
 312 restart_push:
 313                 req.channel = chan->id;
 314                 req.nr_dwords = nvpb->size - nvpb->base.remaining;
 315                 req.dwords = (uint64_t)(unsigned long)nvpb->pushbuf;
 316                 req.nr_buffers = nvpb->nr_buffers;
 317                 req.buffers = (uint64_t)(unsigned long)nvpb->buffers;
 318                 req.nr_relocs = nvpb->nr_relocs;
 319                 req.relocs = (uint64_t)(unsigned long)nvpb->relocs;
 320                 ret = drmCommandWrite(nvdev->fd, DRM_NOUVEAU_GEM_PUSHBUF,
 321                                       &req, sizeof(req));
 322                 if (ret == -EAGAIN)
 323                         goto restart_push;
 324         }
 325
 326
 327         /* Update presumed offset/domain for any buffers that moved.
 328          * Dereference all buffers on validate list
 329          */
 330         for (i = 0; i < nvpb->nr_relocs; i++) {
 331                 struct drm_nouveau_gem_pushbuf_reloc *r = &nvpb->relocs[i];
 332                 struct drm_nouveau_gem_pushbuf_bo *pbbo =
 333                         &nvpb->buffers[r->bo_index];
 334                 struct nouveau_bo *bo = (void *)(unsigned long)pbbo->user_priv;
 335                 struct nouveau_bo_priv *nvbo = nouveau_bo(bo);
 336
 337                 if (--nvbo->pending_refcnt)
 338                         continue;
 339
 340                 if (pbbo->presumed_ok == 0) {
 341                         nvbo->domain = pbbo->presumed_domain;
 342                         nvbo->offset = pbbo->presumed_offset;
 343                 }
 344
 345                 nvbo->pending = NULL;
 346                 nouveau_bo_ref(NULL, &bo);
 347         }
 348
 349         nvpb->nr_buffers = 0;
 350         nvpb->nr_relocs = 0;
 351
 352         /* Allocate space for next push buffer */
 353         assert(!nouveau_pushbuf_space(chan, min));
 354
 355         if (chan->flush_notify)
 356                 chan->flush_notify(chan);
 357
 358         nvpb->marker = 0;
 359         return ret;
 360 }
 361
 362 int
 363 nouveau_pushbuf_marker_emit(struct nouveau_channel *chan,
 364                             unsigned wait_dwords, unsigned wait_relocs)
 365 {
 366         struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(chan->pushbuf);
 367
 368         if (AVAIL_RING(chan) < wait_dwords)
 369                 return nouveau_pushbuf_flush(chan, wait_dwords);
 370
 371         if (nvpb->nr_relocs + wait_relocs >= NOUVEAU_GEM_MAX_RELOCS)
 372                 return nouveau_pushbuf_flush(chan, wait_dwords);
 373
 374         nvpb->marker = nvpb->base.cur - nvpb->pushbuf;
 375         nvpb->marker_relocs = nvpb->nr_relocs;
 376         return 0;
 377 }
 378
 379 void
 380 nouveau_pushbuf_marker_undo(struct nouveau_channel *chan)
 381 {
 382         struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(chan->pushbuf);
 383         unsigned i;
 384
 385         if (!nvpb->marker)
 386                 return;
 387
 388         /* undo any relocs/buffers added to the list since last marker */
 389         for (i = nvpb->marker_relocs; i < nvpb->nr_relocs; i++) {
 390                 struct drm_nouveau_gem_pushbuf_reloc *r = &nvpb->relocs[i];
 391                 struct drm_nouveau_gem_pushbuf_bo *pbbo =
 392                         &nvpb->buffers[r->bo_index];
 393                 struct nouveau_bo *bo = (void *)(unsigned long)pbbo->user_priv;
 394                 struct nouveau_bo_priv *nvbo = nouveau_bo(bo);
 395
 396                 if (--nvbo->pending_refcnt)
 397                         continue;
 398
 399                 nvbo->pending = NULL;
 400                 nouveau_bo_ref(NULL, &bo);
 401                 nvpb->nr_buffers--;
 402         }
 403         nvpb->nr_relocs = nvpb->marker_relocs;
 404
 405         /* reset pushbuf back to last marker */
 406         nvpb->base.cur = nvpb->pushbuf + nvpb->marker;
 407         nvpb->base.remaining = nvpb->size - nvpb->marker;
 408         nvpb->marker = 0;
 409 }
 410
 411