"Fossies" - the Fresh Open Source Software Archive

Member "mesa-20.1.8/src/gallium/drivers/freedreno/freedreno_resource.c" (16 Sep 2020, 40302 Bytes) of package /linux/misc/mesa-20.1.8.tar.xz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "freedreno_resource.c" see the Fossies "Dox" file reference documentation and the last Fossies "Diffs" side-by-side code changes report: 20.2.0-rc3_vs_20.2.0-rc4.

    1 /*
    2  * Copyright (C) 2012 Rob Clark <robclark@freedesktop.org>
    3  *
    4  * Permission is hereby granted, free of charge, to any person obtaining a
    5  * copy of this software and associated documentation files (the "Software"),
    6  * to deal in the Software without restriction, including without limitation
    7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
    8  * and/or sell copies of the Software, and to permit persons to whom the
    9  * Software is furnished to do so, subject to the following conditions:
   10  *
   11  * The above copyright notice and this permission notice (including the next
   12  * paragraph) shall be included in all copies or substantial portions of the
   13  * Software.
   14  *
   15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
   18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
   21  * SOFTWARE.
   22  *
   23  * Authors:
   24  *    Rob Clark <robclark@freedesktop.org>
   25  */
   26 
   27 #include "util/format/u_format.h"
   28 #include "util/format/u_format_rgtc.h"
   29 #include "util/format/u_format_zs.h"
   30 #include "util/u_inlines.h"
   31 #include "util/u_transfer.h"
   32 #include "util/u_string.h"
   33 #include "util/u_surface.h"
   34 #include "util/set.h"
   35 #include "util/u_drm.h"
   36 
   37 #include "freedreno_resource.h"
   38 #include "freedreno_batch_cache.h"
   39 #include "freedreno_blitter.h"
   40 #include "freedreno_fence.h"
   41 #include "freedreno_screen.h"
   42 #include "freedreno_surface.h"
   43 #include "freedreno_context.h"
   44 #include "freedreno_query_hw.h"
   45 #include "freedreno_util.h"
   46 
   47 #include "drm-uapi/drm_fourcc.h"
   48 #include <errno.h>
   49 
   50 /* XXX this should go away, needed for 'struct winsys_handle' */
   51 #include "state_tracker/drm_driver.h"
   52 
   53 /* A private modifier for now, so we have a way to request tiled but not
   54  * compressed.  It would perhaps be good to get real modifiers for the
   55  * tiled formats, but would probably need to do some work to figure out
   56  * the layout(s) of the tiled modes, and whether they are the same
   57  * across generations.
   58  */
   59 #define FD_FORMAT_MOD_QCOM_TILED    fourcc_mod_code(QCOM, 0xffffffff)
   60 
   61 /**
   62  * Go through the entire state and see if the resource is bound
   63  * anywhere. If it is, mark the relevant state as dirty. This is
   64  * called on realloc_bo to ensure the necessary state is re-
   65  * emitted so the GPU looks at the new backing bo.
   66  */
   67 static void
   68 rebind_resource_in_ctx(struct fd_context *ctx, struct fd_resource *rsc)
   69 {
   70     struct pipe_resource *prsc = &rsc->base;
   71 
   72     if (ctx->rebind_resource)
   73         ctx->rebind_resource(ctx, rsc);
   74 
   75     /* VBOs */
   76     if (rsc->dirty & FD_DIRTY_VTXBUF) {
   77         struct fd_vertexbuf_stateobj *vb = &ctx->vtx.vertexbuf;
   78         for (unsigned i = 0; i < vb->count && !(ctx->dirty & FD_DIRTY_VTXBUF); i++) {
   79             if (vb->vb[i].buffer.resource == prsc)
   80                 ctx->dirty |= FD_DIRTY_VTXBUF;
   81         }
   82     }
   83 
   84     const enum fd_dirty_3d_state per_stage_dirty =
   85             FD_DIRTY_CONST | FD_DIRTY_TEX | FD_DIRTY_IMAGE | FD_DIRTY_SSBO;
   86 
   87     if (!(rsc->dirty & per_stage_dirty))
   88         return;
   89 
   90     /* per-shader-stage resources: */
   91     for (unsigned stage = 0; stage < PIPE_SHADER_TYPES; stage++) {
   92         /* Constbufs.. note that constbuf[0] is normal uniforms emitted in
   93          * cmdstream rather than by pointer..
   94          */
   95         if ((rsc->dirty & FD_DIRTY_CONST) &&
   96                 !(ctx->dirty_shader[stage] & FD_DIRTY_CONST)) {
   97             struct fd_constbuf_stateobj *cb = &ctx->constbuf[stage];
   98             const unsigned num_ubos = util_last_bit(cb->enabled_mask);
   99             for (unsigned i = 1; i < num_ubos; i++) {
  100                 if (cb->cb[i].buffer == prsc) {
  101                     ctx->dirty_shader[stage] |= FD_DIRTY_SHADER_CONST;
  102                     ctx->dirty |= FD_DIRTY_CONST;
  103                     break;
  104                 }
  105             }
  106         }
  107 
  108         /* Textures */
  109         if ((rsc->dirty & FD_DIRTY_TEX) &&
  110                 !(ctx->dirty_shader[stage] & FD_DIRTY_TEX)) {
  111             struct fd_texture_stateobj *tex = &ctx->tex[stage];
  112             for (unsigned i = 0; i < tex->num_textures; i++) {
  113                 if (tex->textures[i] && (tex->textures[i]->texture == prsc)) {
  114                     ctx->dirty_shader[stage] |= FD_DIRTY_SHADER_TEX;
  115                     ctx->dirty |= FD_DIRTY_TEX;
  116                     break;
  117                 }
  118             }
  119         }
  120 
  121         /* Images */
  122         if ((rsc->dirty & FD_DIRTY_IMAGE) &&
  123                 !(ctx->dirty_shader[stage] & FD_DIRTY_IMAGE)) {
  124             struct fd_shaderimg_stateobj *si = &ctx->shaderimg[stage];
  125             const unsigned num_images = util_last_bit(si->enabled_mask);
  126             for (unsigned i = 0; i < num_images; i++) {
  127                 if (si->si[i].resource == prsc) {
  128                     ctx->dirty_shader[stage] |= FD_DIRTY_SHADER_IMAGE;
  129                     ctx->dirty |= FD_DIRTY_IMAGE;
  130                     break;
  131                 }
  132             }
  133         }
  134 
  135         /* SSBOs */
  136         if ((rsc->dirty & FD_DIRTY_SSBO) &&
  137                 !(ctx->dirty_shader[stage] & FD_DIRTY_SSBO)) {
  138             struct fd_shaderbuf_stateobj *sb = &ctx->shaderbuf[stage];
  139             const unsigned num_ssbos = util_last_bit(sb->enabled_mask);
  140             for (unsigned i = 0; i < num_ssbos; i++) {
  141                 if (sb->sb[i].buffer == prsc) {
  142                     ctx->dirty_shader[stage] |= FD_DIRTY_SHADER_SSBO;
  143                     ctx->dirty |= FD_DIRTY_SSBO;
  144                     break;
  145                 }
  146             }
  147         }
  148     }
  149 }
  150 
  151 static void
  152 rebind_resource(struct fd_resource *rsc)
  153 {
  154     struct fd_screen *screen = fd_screen(rsc->base.screen);
  155 
  156     mtx_lock(&screen->lock);
  157     fd_resource_lock(rsc);
  158 
  159     if (rsc->dirty)
  160         list_for_each_entry (struct fd_context, ctx, &screen->context_list, node)
  161             rebind_resource_in_ctx(ctx, rsc);
  162 
  163     fd_resource_unlock(rsc);
  164     mtx_unlock(&screen->lock);
  165 }
  166 
  167 static void
  168 realloc_bo(struct fd_resource *rsc, uint32_t size)
  169 {
  170     struct pipe_resource *prsc = &rsc->base;
  171     struct fd_screen *screen = fd_screen(rsc->base.screen);
  172     uint32_t flags = DRM_FREEDRENO_GEM_CACHE_WCOMBINE |
  173             DRM_FREEDRENO_GEM_TYPE_KMEM |
  174             COND(prsc->bind & PIPE_BIND_SCANOUT, DRM_FREEDRENO_GEM_SCANOUT);
  175             /* TODO other flags? */
  176 
  177     /* if we start using things other than write-combine,
  178      * be sure to check for PIPE_RESOURCE_FLAG_MAP_COHERENT
  179      */
  180 
  181     if (rsc->bo)
  182         fd_bo_del(rsc->bo);
  183 
  184     rsc->bo = fd_bo_new(screen->dev, size, flags, "%ux%ux%u@%u:%x",
  185             prsc->width0, prsc->height0, prsc->depth0, rsc->layout.cpp, prsc->bind);
  186 
  187     /* Zero out the UBWC area on allocation.  This fixes intermittent failures
  188      * with UBWC, which I suspect are due to the HW having a hard time
  189      * interpreting arbitrary values populating the flags buffer when the BO
  190      * was recycled through the bo cache (instead of fresh allocations from
  191      * the kernel, which are zeroed).  sleep(1) in this spot didn't work
  192      * around the issue, but any memset value seems to.
  193      */
  194     if (rsc->layout.ubwc) {
  195         void *buf = fd_bo_map(rsc->bo);
  196         memset(buf, 0, rsc->layout.slices[0].offset);
  197     }
  198 
  199     rsc->seqno = p_atomic_inc_return(&screen->rsc_seqno);
  200     util_range_set_empty(&rsc->valid_buffer_range);
  201     fd_bc_invalidate_resource(rsc, true);
  202 }
  203 
  204 static void
  205 do_blit(struct fd_context *ctx, const struct pipe_blit_info *blit, bool fallback)
  206 {
  207     struct pipe_context *pctx = &ctx->base;
  208 
  209     /* TODO size threshold too?? */
  210     if (fallback || !fd_blit(pctx, blit)) {
  211         /* do blit on cpu: */
  212         util_resource_copy_region(pctx,
  213                 blit->dst.resource, blit->dst.level, blit->dst.box.x,
  214                 blit->dst.box.y, blit->dst.box.z,
  215                 blit->src.resource, blit->src.level, &blit->src.box);
  216     }
  217 }
  218 
  219 static void
  220 flush_resource(struct fd_context *ctx, struct fd_resource *rsc, unsigned usage);
  221 
  222 /**
  223  * @rsc: the resource to shadow
  224  * @level: the level to discard (if box != NULL, otherwise ignored)
  225  * @box: the box to discard (or NULL if none)
  226  * @modifier: the modifier for the new buffer state
  227  */
  228 static bool
  229 fd_try_shadow_resource(struct fd_context *ctx, struct fd_resource *rsc,
  230         unsigned level, const struct pipe_box *box, uint64_t modifier)
  231 {
  232     struct pipe_context *pctx = &ctx->base;
  233     struct pipe_resource *prsc = &rsc->base;
  234     bool fallback = false;
  235 
  236     if (prsc->next)
  237         return false;
  238 
  239     /* If you have a sequence where there is a single rsc associated
  240      * with the current render target, and then you end up shadowing
  241      * that same rsc on the 3d pipe (u_blitter), because of how we
  242      * swap the new shadow and rsc before the back-blit, you could end
  243      * up confusing things into thinking that u_blitter's framebuffer
  244      * state is the same as the current framebuffer state, which has
  245      * the result of blitting to rsc rather than shadow.
  246      *
  247      * Normally we wouldn't want to unconditionally trigger a flush,
  248      * since that defeats the purpose of shadowing, but this is a
  249      * case where we'd have to flush anyways.
  250      */
  251     if (rsc->write_batch == ctx->batch)
  252         flush_resource(ctx, rsc, 0);
  253 
  254     /* TODO: somehow munge dimensions and format to copy unsupported
  255      * render target format to something that is supported?
  256      */
  257     if (!pctx->screen->is_format_supported(pctx->screen,
  258             prsc->format, prsc->target, prsc->nr_samples,
  259             prsc->nr_storage_samples,
  260             PIPE_BIND_RENDER_TARGET))
  261         fallback = true;
  262 
  263     /* do shadowing back-blits on the cpu for buffers: */
  264     if (prsc->target == PIPE_BUFFER)
  265         fallback = true;
  266 
  267     bool discard_whole_level = box && util_texrange_covers_whole_level(prsc, level,
  268         box->x, box->y, box->z, box->width, box->height, box->depth);
  269 
  270     /* TODO need to be more clever about current level */
  271     if ((prsc->target >= PIPE_TEXTURE_2D) && box && !discard_whole_level)
  272         return false;
  273 
  274     struct pipe_resource *pshadow =
  275         pctx->screen->resource_create_with_modifiers(pctx->screen,
  276                 prsc, &modifier, 1);
  277 
  278     if (!pshadow)
  279         return false;
  280 
  281     assert(!ctx->in_shadow);
  282     ctx->in_shadow = true;
  283 
  284     /* get rid of any references that batch-cache might have to us (which
  285      * should empty/destroy rsc->batches hashset)
  286      */
  287     fd_bc_invalidate_resource(rsc, false);
  288     rebind_resource(rsc);
  289 
  290     mtx_lock(&ctx->screen->lock);
  291 
  292     /* Swap the backing bo's, so shadow becomes the old buffer,
  293      * blit from shadow to new buffer.  From here on out, we
  294      * cannot fail.
  295      *
  296      * Note that we need to do it in this order, otherwise if
  297      * we go down cpu blit path, the recursive transfer_map()
  298      * sees the wrong status..
  299      */
  300     struct fd_resource *shadow = fd_resource(pshadow);
  301 
  302     DBG("shadow: %p (%d) -> %p (%d)\n", rsc, rsc->base.reference.count,
  303             shadow, shadow->base.reference.count);
  304 
  305     /* TODO valid_buffer_range?? */
  306     swap(rsc->bo,        shadow->bo);
  307     swap(rsc->write_batch,   shadow->write_batch);
  308     swap(rsc->layout, shadow->layout);
  309     rsc->seqno = p_atomic_inc_return(&ctx->screen->rsc_seqno);
  310 
  311     /* at this point, the newly created shadow buffer is not referenced
  312      * by any batches, but the existing rsc (probably) is.  We need to
  313      * transfer those references over:
  314      */
  315     debug_assert(shadow->batch_mask == 0);
  316     struct fd_batch *batch;
  317     foreach_batch(batch, &ctx->screen->batch_cache, rsc->batch_mask) {
  318         struct set_entry *entry = _mesa_set_search(batch->resources, rsc);
  319         _mesa_set_remove(batch->resources, entry);
  320         _mesa_set_add(batch->resources, shadow);
  321     }
  322     swap(rsc->batch_mask, shadow->batch_mask);
  323 
  324     mtx_unlock(&ctx->screen->lock);
  325 
  326     struct pipe_blit_info blit = {};
  327     blit.dst.resource = prsc;
  328     blit.dst.format   = prsc->format;
  329     blit.src.resource = pshadow;
  330     blit.src.format   = pshadow->format;
  331     blit.mask = util_format_get_mask(prsc->format);
  332     blit.filter = PIPE_TEX_FILTER_NEAREST;
  333 
  334 #define set_box(field, val) do {     \
  335         blit.dst.field = (val);      \
  336         blit.src.field = (val);      \
  337     } while (0)
  338 
  339     /* blit the other levels in their entirety: */
  340     for (unsigned l = 0; l <= prsc->last_level; l++) {
  341         if (box && l == level)
  342             continue;
  343 
  344         /* just blit whole level: */
  345         set_box(level, l);
  346         set_box(box.width,  u_minify(prsc->width0, l));
  347         set_box(box.height, u_minify(prsc->height0, l));
  348         set_box(box.depth,  u_minify(prsc->depth0, l));
  349 
  350         for (int i = 0; i < prsc->array_size; i++) {
  351             set_box(box.z, i);
  352             do_blit(ctx, &blit, fallback);
  353         }
  354     }
  355 
  356     /* deal w/ current level specially, since we might need to split
  357      * it up into a couple blits:
  358      */
  359     if (box && !discard_whole_level) {
  360         set_box(level, level);
  361 
  362         switch (prsc->target) {
  363         case PIPE_BUFFER:
  364         case PIPE_TEXTURE_1D:
  365             set_box(box.y, 0);
  366             set_box(box.z, 0);
  367             set_box(box.height, 1);
  368             set_box(box.depth, 1);
  369 
  370             if (box->x > 0) {
  371                 set_box(box.x, 0);
  372                 set_box(box.width, box->x);
  373 
  374                 do_blit(ctx, &blit, fallback);
  375             }
  376             if ((box->x + box->width) < u_minify(prsc->width0, level)) {
  377                 set_box(box.x, box->x + box->width);
  378                 set_box(box.width, u_minify(prsc->width0, level) - (box->x + box->width));
  379 
  380                 do_blit(ctx, &blit, fallback);
  381             }
  382             break;
  383         case PIPE_TEXTURE_2D:
  384             /* TODO */
  385         default:
  386             unreachable("TODO");
  387         }
  388     }
  389 
  390     ctx->in_shadow = false;
  391 
  392     pipe_resource_reference(&pshadow, NULL);
  393 
  394     return true;
  395 }
  396 
  397 /**
  398  * Uncompress an UBWC compressed buffer "in place".  This works basically
  399  * like resource shadowing, creating a new resource, and doing an uncompress
  400  * blit, and swapping the state between shadow and original resource so it
  401  * appears to the state tracker as if nothing changed.
  402  */
  403 void
  404 fd_resource_uncompress(struct fd_context *ctx, struct fd_resource *rsc)
  405 {
  406     bool success =
  407         fd_try_shadow_resource(ctx, rsc, 0, NULL, FD_FORMAT_MOD_QCOM_TILED);
  408 
  409     /* shadow should not fail in any cases where we need to uncompress: */
  410     debug_assert(success);
  411 }
  412 
  413 static struct fd_resource *
  414 fd_alloc_staging(struct fd_context *ctx, struct fd_resource *rsc,
  415         unsigned level, const struct pipe_box *box)
  416 {
  417     struct pipe_context *pctx = &ctx->base;
  418     struct pipe_resource tmpl = rsc->base;
  419 
  420     tmpl.width0  = box->width;
  421     tmpl.height0 = box->height;
  422     /* for array textures, box->depth is the array_size, otherwise
  423      * for 3d textures, it is the depth:
  424      */
  425     if (tmpl.array_size > 1) {
  426         if (tmpl.target == PIPE_TEXTURE_CUBE)
  427             tmpl.target = PIPE_TEXTURE_2D_ARRAY;
  428         tmpl.array_size = box->depth;
  429         tmpl.depth0 = 1;
  430     } else {
  431         tmpl.array_size = 1;
  432         tmpl.depth0 = box->depth;
  433     }
  434     tmpl.last_level = 0;
  435     tmpl.bind |= PIPE_BIND_LINEAR;
  436 
  437     struct pipe_resource *pstaging =
  438         pctx->screen->resource_create(pctx->screen, &tmpl);
  439     if (!pstaging)
  440         return NULL;
  441 
  442     return fd_resource(pstaging);
  443 }
  444 
  445 static void
  446 fd_blit_from_staging(struct fd_context *ctx, struct fd_transfer *trans)
  447 {
  448     struct pipe_resource *dst = trans->base.resource;
  449     struct pipe_blit_info blit = {};
  450 
  451     blit.dst.resource = dst;
  452     blit.dst.format   = dst->format;
  453     blit.dst.level    = trans->base.level;
  454     blit.dst.box      = trans->base.box;
  455     blit.src.resource = trans->staging_prsc;
  456     blit.src.format   = trans->staging_prsc->format;
  457     blit.src.level    = 0;
  458     blit.src.box      = trans->staging_box;
  459     blit.mask = util_format_get_mask(trans->staging_prsc->format);
  460     blit.filter = PIPE_TEX_FILTER_NEAREST;
  461 
  462     do_blit(ctx, &blit, false);
  463 }
  464 
  465 static void
  466 fd_blit_to_staging(struct fd_context *ctx, struct fd_transfer *trans)
  467 {
  468     struct pipe_resource *src = trans->base.resource;
  469     struct pipe_blit_info blit = {};
  470 
  471     blit.src.resource = src;
  472     blit.src.format   = src->format;
  473     blit.src.level    = trans->base.level;
  474     blit.src.box      = trans->base.box;
  475     blit.dst.resource = trans->staging_prsc;
  476     blit.dst.format   = trans->staging_prsc->format;
  477     blit.dst.level    = 0;
  478     blit.dst.box      = trans->staging_box;
  479     blit.mask = util_format_get_mask(trans->staging_prsc->format);
  480     blit.filter = PIPE_TEX_FILTER_NEAREST;
  481 
  482     do_blit(ctx, &blit, false);
  483 }
  484 
  485 static void fd_resource_transfer_flush_region(struct pipe_context *pctx,
  486         struct pipe_transfer *ptrans,
  487         const struct pipe_box *box)
  488 {
  489     struct fd_resource *rsc = fd_resource(ptrans->resource);
  490 
  491     if (ptrans->resource->target == PIPE_BUFFER)
  492         util_range_add(&rsc->base, &rsc->valid_buffer_range,
  493                        ptrans->box.x + box->x,
  494                        ptrans->box.x + box->x + box->width);
  495 }
  496 
  497 static void
  498 flush_resource(struct fd_context *ctx, struct fd_resource *rsc, unsigned usage)
  499 {
  500     struct fd_batch *write_batch = NULL;
  501 
  502     mtx_lock(&ctx->screen->lock);
  503     fd_batch_reference_locked(&write_batch, rsc->write_batch);
  504     mtx_unlock(&ctx->screen->lock);
  505 
  506     if (usage & PIPE_TRANSFER_WRITE) {
  507         struct fd_batch *batch, *batches[32] = {};
  508         uint32_t batch_mask;
  509 
  510         /* This is a bit awkward, probably a fd_batch_flush_locked()
  511          * would make things simpler.. but we need to hold the lock
  512          * to iterate the batches which reference this resource.  So
  513          * we must first grab references under a lock, then flush.
  514          */
  515         mtx_lock(&ctx->screen->lock);
  516         batch_mask = rsc->batch_mask;
  517         foreach_batch(batch, &ctx->screen->batch_cache, batch_mask)
  518             fd_batch_reference_locked(&batches[batch->idx], batch);
  519         mtx_unlock(&ctx->screen->lock);
  520 
  521         foreach_batch(batch, &ctx->screen->batch_cache, batch_mask)
  522             fd_batch_flush(batch);
  523 
  524         foreach_batch(batch, &ctx->screen->batch_cache, batch_mask) {
  525             fd_batch_reference(&batches[batch->idx], NULL);
  526         }
  527         assert(rsc->batch_mask == 0);
  528     } else if (write_batch) {
  529         fd_batch_flush(write_batch);
  530     }
  531 
  532     fd_batch_reference(&write_batch, NULL);
  533 
  534     assert(!rsc->write_batch);
  535 }
  536 
  537 static void
  538 fd_flush_resource(struct pipe_context *pctx, struct pipe_resource *prsc)
  539 {
  540     flush_resource(fd_context(pctx), fd_resource(prsc), PIPE_TRANSFER_READ);
  541 }
  542 
  543 static void
  544 fd_resource_transfer_unmap(struct pipe_context *pctx,
  545         struct pipe_transfer *ptrans)
  546 {
  547     struct fd_context *ctx = fd_context(pctx);
  548     struct fd_resource *rsc = fd_resource(ptrans->resource);
  549     struct fd_transfer *trans = fd_transfer(ptrans);
  550 
  551     if (trans->staging_prsc) {
  552         if (ptrans->usage & PIPE_TRANSFER_WRITE)
  553             fd_blit_from_staging(ctx, trans);
  554         pipe_resource_reference(&trans->staging_prsc, NULL);
  555     }
  556 
  557     if (!(ptrans->usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
  558         fd_bo_cpu_fini(rsc->bo);
  559     }
  560 
  561     util_range_add(&rsc->base, &rsc->valid_buffer_range,
  562                    ptrans->box.x,
  563                    ptrans->box.x + ptrans->box.width);
  564 
  565     pipe_resource_reference(&ptrans->resource, NULL);
  566     slab_free(&ctx->transfer_pool, ptrans);
  567 }
  568 
  569 static void *
  570 fd_resource_transfer_map(struct pipe_context *pctx,
  571         struct pipe_resource *prsc,
  572         unsigned level, unsigned usage,
  573         const struct pipe_box *box,
  574         struct pipe_transfer **pptrans)
  575 {
  576     struct fd_context *ctx = fd_context(pctx);
  577     struct fd_resource *rsc = fd_resource(prsc);
  578     struct fdl_slice *slice = fd_resource_slice(rsc, level);
  579     struct fd_transfer *trans;
  580     struct pipe_transfer *ptrans;
  581     enum pipe_format format = prsc->format;
  582     uint32_t op = 0;
  583     uint32_t offset;
  584     char *buf;
  585     int ret = 0;
  586 
  587     DBG("prsc=%p, level=%u, usage=%x, box=%dx%d+%d,%d", prsc, level, usage,
  588         box->width, box->height, box->x, box->y);
  589 
  590     ptrans = slab_alloc(&ctx->transfer_pool);
  591     if (!ptrans)
  592         return NULL;
  593 
  594     /* slab_alloc_st() doesn't zero: */
  595     trans = fd_transfer(ptrans);
  596     memset(trans, 0, sizeof(*trans));
  597 
  598     pipe_resource_reference(&ptrans->resource, prsc);
  599     ptrans->level = level;
  600     ptrans->usage = usage;
  601     ptrans->box = *box;
  602     ptrans->stride = slice->pitch;
  603     ptrans->layer_stride = fd_resource_layer_stride(rsc, level);
  604 
  605     /* we always need a staging texture for tiled buffers:
  606      *
  607      * TODO we might sometimes want to *also* shadow the resource to avoid
  608      * splitting a batch.. for ex, mid-frame texture uploads to a tiled
  609      * texture.
  610      */
  611     if (rsc->layout.tile_mode) {
  612         struct fd_resource *staging_rsc;
  613 
  614         staging_rsc = fd_alloc_staging(ctx, rsc, level, box);
  615         if (staging_rsc) {
  616             struct fdl_slice *staging_slice =
  617                 fd_resource_slice(staging_rsc, 0);
  618             // TODO for PIPE_TRANSFER_READ, need to do untiling blit..
  619             trans->staging_prsc = &staging_rsc->base;
  620             trans->base.stride = staging_slice->pitch;
  621             trans->base.layer_stride = fd_resource_layer_stride(staging_rsc, 0);
  622             trans->staging_box = *box;
  623             trans->staging_box.x = 0;
  624             trans->staging_box.y = 0;
  625             trans->staging_box.z = 0;
  626 
  627             if (usage & PIPE_TRANSFER_READ) {
  628                 fd_blit_to_staging(ctx, trans);
  629 
  630                 fd_bo_cpu_prep(staging_rsc->bo, ctx->pipe,
  631                         DRM_FREEDRENO_PREP_READ);
  632             }
  633 
  634             buf = fd_bo_map(staging_rsc->bo);
  635             offset = 0;
  636 
  637             *pptrans = ptrans;
  638 
  639             ctx->stats.staging_uploads++;
  640 
  641             return buf;
  642         }
  643     }
  644 
  645     if (ctx->in_shadow && !(usage & PIPE_TRANSFER_READ))
  646         usage |= PIPE_TRANSFER_UNSYNCHRONIZED;
  647 
  648     if (usage & PIPE_TRANSFER_READ)
  649         op |= DRM_FREEDRENO_PREP_READ;
  650 
  651     if (usage & PIPE_TRANSFER_WRITE)
  652         op |= DRM_FREEDRENO_PREP_WRITE;
  653 
  654     bool needs_flush = pending(rsc, !!(usage & PIPE_TRANSFER_WRITE));
  655 
  656     if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE) {
  657         if (needs_flush || fd_resource_busy(rsc, op)) {
  658             rebind_resource(rsc);
  659             realloc_bo(rsc, fd_bo_size(rsc->bo));
  660         }
  661     } else if ((usage & PIPE_TRANSFER_WRITE) &&
  662                prsc->target == PIPE_BUFFER &&
  663                !util_ranges_intersect(&rsc->valid_buffer_range,
  664                                       box->x, box->x + box->width)) {
  665         /* We are trying to write to a previously uninitialized range. No need
  666          * to wait.
  667          */
  668     } else if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
  669         struct fd_batch *write_batch = NULL;
  670 
  671         /* hold a reference, so it doesn't disappear under us: */
  672         fd_context_lock(ctx);
  673         fd_batch_reference_locked(&write_batch, rsc->write_batch);
  674         fd_context_unlock(ctx);
  675 
  676         if ((usage & PIPE_TRANSFER_WRITE) && write_batch &&
  677                 write_batch->back_blit) {
  678             /* if only thing pending is a back-blit, we can discard it: */
  679             fd_batch_reset(write_batch);
  680         }
  681 
  682         /* If the GPU is writing to the resource, or if it is reading from the
  683          * resource and we're trying to write to it, flush the renders.
  684          */
  685         bool busy = needs_flush || fd_resource_busy(rsc, op);
  686 
  687         /* if we need to flush/stall, see if we can make a shadow buffer
  688          * to avoid this:
  689          *
  690          * TODO we could go down this path !reorder && !busy_for_read
  691          * ie. we only *don't* want to go down this path if the blit
  692          * will trigger a flush!
  693          */
  694         if (ctx->screen->reorder && busy && !(usage & PIPE_TRANSFER_READ) &&
  695                 (usage & PIPE_TRANSFER_DISCARD_RANGE)) {
  696             /* try shadowing only if it avoids a flush, otherwise staging would
  697              * be better:
  698              */
  699             if (needs_flush && fd_try_shadow_resource(ctx, rsc, level,
  700                             box, DRM_FORMAT_MOD_LINEAR)) {
  701                 needs_flush = busy = false;
  702                 ctx->stats.shadow_uploads++;
  703             } else {
  704                 struct fd_resource *staging_rsc;
  705 
  706                 if (needs_flush) {
  707                     flush_resource(ctx, rsc, usage);
  708                     needs_flush = false;
  709                 }
  710 
  711                 /* in this case, we don't need to shadow the whole resource,
  712                  * since any draw that references the previous contents has
  713                  * already had rendering flushed for all tiles.  So we can
  714                  * use a staging buffer to do the upload.
  715                  */
  716                 staging_rsc = fd_alloc_staging(ctx, rsc, level, box);
  717                 if (staging_rsc) {
  718                     struct fdl_slice *staging_slice =
  719                         fd_resource_slice(staging_rsc, 0);
  720                     trans->staging_prsc = &staging_rsc->base;
  721                     trans->base.stride = staging_slice->pitch;
  722                     trans->base.layer_stride =
  723                         fd_resource_layer_stride(staging_rsc, 0);
  724                     trans->staging_box = *box;
  725                     trans->staging_box.x = 0;
  726                     trans->staging_box.y = 0;
  727                     trans->staging_box.z = 0;
  728                     buf = fd_bo_map(staging_rsc->bo);
  729                     offset = 0;
  730 
  731                     *pptrans = ptrans;
  732 
  733                     fd_batch_reference(&write_batch, NULL);
  734 
  735                     ctx->stats.staging_uploads++;
  736 
  737                     return buf;
  738                 }
  739             }
  740         }
  741 
  742         if (needs_flush) {
  743             flush_resource(ctx, rsc, usage);
  744             needs_flush = false;
  745         }
  746 
  747         fd_batch_reference(&write_batch, NULL);
  748 
  749         /* The GPU keeps track of how the various bo's are being used, and
  750          * will wait if necessary for the proper operation to have
  751          * completed.
  752          */
  753         if (busy) {
  754             ret = fd_bo_cpu_prep(rsc->bo, ctx->pipe, op);
  755             if (ret)
  756                 goto fail;
  757         }
  758     }
  759 
  760     buf = fd_bo_map(rsc->bo);
  761     offset =
  762         box->y / util_format_get_blockheight(format) * ptrans->stride +
  763         box->x / util_format_get_blockwidth(format) * rsc->layout.cpp +
  764         fd_resource_offset(rsc, level, box->z);
  765 
  766     if (usage & PIPE_TRANSFER_WRITE)
  767         rsc->valid = true;
  768 
  769     *pptrans = ptrans;
  770 
  771     return buf + offset;
  772 
  773 fail:
  774     fd_resource_transfer_unmap(pctx, ptrans);
  775     return NULL;
  776 }
  777 
  778 static void
  779 fd_resource_destroy(struct pipe_screen *pscreen,
  780         struct pipe_resource *prsc)
  781 {
  782     struct fd_resource *rsc = fd_resource(prsc);
  783     fd_bc_invalidate_resource(rsc, true);
  784     if (rsc->bo)
  785         fd_bo_del(rsc->bo);
  786     if (rsc->scanout)
  787         renderonly_scanout_destroy(rsc->scanout, fd_screen(pscreen)->ro);
  788 
  789     util_range_destroy(&rsc->valid_buffer_range);
  790     simple_mtx_destroy(&rsc->lock);
  791     FREE(rsc);
  792 }
  793 
  794 static uint64_t
  795 fd_resource_modifier(struct fd_resource *rsc)
  796 {
  797     if (!rsc->layout.tile_mode)
  798         return DRM_FORMAT_MOD_LINEAR;
  799 
  800     if (rsc->layout.ubwc_layer_size)
  801         return DRM_FORMAT_MOD_QCOM_COMPRESSED;
  802 
  803     /* TODO invent a modifier for tiled but not UBWC buffers: */
  804     return DRM_FORMAT_MOD_INVALID;
  805 }
  806 
  807 static bool
  808 fd_resource_get_handle(struct pipe_screen *pscreen,
  809         struct pipe_context *pctx,
  810         struct pipe_resource *prsc,
  811         struct winsys_handle *handle,
  812         unsigned usage)
  813 {
  814     struct fd_resource *rsc = fd_resource(prsc);
  815 
  816     handle->modifier = fd_resource_modifier(rsc);
  817 
  818     return fd_screen_bo_get_handle(pscreen, rsc->bo, rsc->scanout,
  819             fd_resource_slice(rsc, 0)->pitch, handle);
  820 }
  821 
  822 static uint32_t
  823 setup_slices(struct fd_resource *rsc, uint32_t alignment, enum pipe_format format)
  824 {
  825     struct pipe_resource *prsc = &rsc->base;
  826     struct fd_screen *screen = fd_screen(prsc->screen);
  827     enum util_format_layout layout = util_format_description(format)->layout;
  828     uint32_t pitchalign = screen->gmem_alignw;
  829     uint32_t level, size = 0;
  830     uint32_t width = prsc->width0;
  831     uint32_t height = prsc->height0;
  832     uint32_t depth = prsc->depth0;
  833     /* in layer_first layout, the level (slice) contains just one
  834      * layer (since in fact the layer contains the slices)
  835      */
  836     uint32_t layers_in_level = rsc->layout.layer_first ? 1 : prsc->array_size;
  837 
  838     for (level = 0; level <= prsc->last_level; level++) {
  839         struct fdl_slice *slice = fd_resource_slice(rsc, level);
  840         uint32_t blocks;
  841 
  842         if (layout == UTIL_FORMAT_LAYOUT_ASTC)
  843             width = util_align_npot(width, pitchalign * util_format_get_blockwidth(format));
  844         else
  845             width = align(width, pitchalign);
  846         slice->pitch = util_format_get_nblocksx(format, width) * rsc->layout.cpp;
  847         slice->offset = size;
  848         blocks = util_format_get_nblocks(format, width, height);
  849         /* 1d array and 2d array textures must all have the same layer size
  850          * for each miplevel on a3xx. 3d textures can have different layer
  851          * sizes for high levels, but the hw auto-sizer is buggy (or at least
  852          * different than what this code does), so as soon as the layer size
  853          * range gets into range, we stop reducing it.
  854          */
  855         if (prsc->target == PIPE_TEXTURE_3D && (
  856                     level == 1 ||
  857                     (level > 1 && fd_resource_slice(rsc, level - 1)->size0 > 0xf000)))
  858             slice->size0 = align(blocks * rsc->layout.cpp, alignment);
  859         else if (level == 0 || rsc->layout.layer_first || alignment == 1)
  860             slice->size0 = align(blocks * rsc->layout.cpp, alignment);
  861         else
  862             slice->size0 = fd_resource_slice(rsc, level - 1)->size0;
  863 
  864         size += slice->size0 * depth * layers_in_level;
  865 
  866         width = u_minify(width, 1);
  867         height = u_minify(height, 1);
  868         depth = u_minify(depth, 1);
  869     }
  870 
  871     return size;
  872 }
  873 
  874 static uint32_t
  875 slice_alignment(enum pipe_texture_target target)
  876 {
  877     /* on a3xx, 2d array and 3d textures seem to want their
  878      * layers aligned to page boundaries:
  879      */
  880     switch (target) {
  881     case PIPE_TEXTURE_3D:
  882     case PIPE_TEXTURE_1D_ARRAY:
  883     case PIPE_TEXTURE_2D_ARRAY:
  884         return 4096;
  885     default:
  886         return 1;
  887     }
  888 }
  889 
  890 /* cross generation texture layout to plug in to screen->setup_slices()..
  891  * replace with generation specific one as-needed.
  892  *
  893  * TODO for a4xx probably can extract out the a4xx specific logic int
  894  * a small fd4_setup_slices() wrapper that sets up layer_first, and then
  895  * calls this.
  896  */
  897 uint32_t
  898 fd_setup_slices(struct fd_resource *rsc)
  899 {
  900     uint32_t alignment;
  901 
  902     alignment = slice_alignment(rsc->base.target);
  903 
  904     struct fd_screen *screen = fd_screen(rsc->base.screen);
  905     if (is_a4xx(screen)) {
  906         switch (rsc->base.target) {
  907         case PIPE_TEXTURE_3D:
  908             rsc->layout.layer_first = false;
  909             break;
  910         default:
  911             rsc->layout.layer_first = true;
  912             alignment = 1;
  913             break;
  914         }
  915     }
  916 
  917     return setup_slices(rsc, alignment, rsc->base.format);
  918 }
  919 
  920 /* special case to resize query buf after allocated.. */
  921 void
  922 fd_resource_resize(struct pipe_resource *prsc, uint32_t sz)
  923 {
  924     struct fd_resource *rsc = fd_resource(prsc);
  925 
  926     debug_assert(prsc->width0 == 0);
  927     debug_assert(prsc->target == PIPE_BUFFER);
  928     debug_assert(prsc->bind == PIPE_BIND_QUERY_BUFFER);
  929 
  930     prsc->width0 = sz;
  931     realloc_bo(rsc, fd_screen(prsc->screen)->setup_slices(rsc));
  932 }
  933 
  934 static void
  935 fd_resource_layout_init(struct pipe_resource *prsc)
  936 {
  937     struct fd_resource *rsc = fd_resource(prsc);
  938     struct fdl_layout *layout = &rsc->layout;
  939 
  940     layout->width0 = prsc->width0;
  941     layout->height0 = prsc->height0;
  942     layout->depth0 = prsc->depth0;
  943 
  944     layout->cpp = util_format_get_blocksize(prsc->format);
  945     layout->cpp *= fd_resource_nr_samples(prsc);
  946     layout->cpp_shift = ffs(layout->cpp) - 1;
  947 }
  948 
  949 /**
  950  * Create a new texture object, using the given template info.
  951  */
  952 static struct pipe_resource *
  953 fd_resource_create_with_modifiers(struct pipe_screen *pscreen,
  954         const struct pipe_resource *tmpl,
  955         const uint64_t *modifiers, int count)
  956 {
  957     struct fd_screen *screen = fd_screen(pscreen);
  958     struct fd_resource *rsc;
  959     struct pipe_resource *prsc;
  960     enum pipe_format format = tmpl->format;
  961     uint32_t size;
  962 
  963     /* when using kmsro, scanout buffers are allocated on the display device
  964      * create_with_modifiers() doesn't give us usage flags, so we have to
  965      * assume that all calls with modifiers are scanout-possible
  966      */
  967     if (screen->ro &&
  968         ((tmpl->bind & PIPE_BIND_SCANOUT) ||
  969          !(count == 1 && modifiers[0] == DRM_FORMAT_MOD_INVALID))) {
  970         struct pipe_resource scanout_templat = *tmpl;
  971         struct renderonly_scanout *scanout;
  972         struct winsys_handle handle;
  973 
  974         /* apply freedreno alignment requirement */
  975         scanout_templat.width0 = align(tmpl->width0, screen->gmem_alignw);
  976 
  977         scanout = renderonly_scanout_for_resource(&scanout_templat,
  978                                                   screen->ro, &handle);
  979         if (!scanout)
  980             return NULL;
  981 
  982         renderonly_scanout_destroy(scanout, screen->ro);
  983 
  984         assert(handle.type == WINSYS_HANDLE_TYPE_FD);
  985         rsc = fd_resource(pscreen->resource_from_handle(pscreen, tmpl,
  986                                                         &handle,
  987                                                         PIPE_HANDLE_USAGE_FRAMEBUFFER_WRITE));
  988         close(handle.handle);
  989         if (!rsc)
  990             return NULL;
  991 
  992         return &rsc->base;
  993     }
  994 
  995     rsc = CALLOC_STRUCT(fd_resource);
  996     prsc = &rsc->base;
  997 
  998     DBG("%p: target=%d, format=%s, %ux%ux%u, array_size=%u, last_level=%u, "
  999             "nr_samples=%u, usage=%u, bind=%x, flags=%x", prsc,
 1000             tmpl->target, util_format_name(format),
 1001             tmpl->width0, tmpl->height0, tmpl->depth0,
 1002             tmpl->array_size, tmpl->last_level, tmpl->nr_samples,
 1003             tmpl->usage, tmpl->bind, tmpl->flags);
 1004 
 1005     if (!rsc)
 1006         return NULL;
 1007 
 1008     *prsc = *tmpl;
 1009     fd_resource_layout_init(prsc);
 1010 
 1011 #define LINEAR \
 1012     (PIPE_BIND_SCANOUT | \
 1013      PIPE_BIND_LINEAR  | \
 1014      PIPE_BIND_DISPLAY_TARGET)
 1015 
 1016     bool linear = drm_find_modifier(DRM_FORMAT_MOD_LINEAR, modifiers, count);
 1017     if (tmpl->bind & LINEAR)
 1018         linear = true;
 1019 
 1020     if (fd_mesa_debug & FD_DBG_NOTILE)
 1021         linear = true;
 1022 
 1023     /* Normally, for non-shared buffers, allow buffer compression if
 1024      * not shared, otherwise only allow if QCOM_COMPRESSED modifier
 1025      * is requested:
 1026      *
 1027      * TODO we should probably also limit tiled in a similar way,
 1028      * except we don't have a format modifier for tiled.  (We probably
 1029      * should.)
 1030      */
 1031     bool allow_ubwc = drm_find_modifier(DRM_FORMAT_MOD_INVALID, modifiers, count);
 1032     if (tmpl->bind & PIPE_BIND_SHARED)
 1033         allow_ubwc = drm_find_modifier(DRM_FORMAT_MOD_QCOM_COMPRESSED, modifiers, count);
 1034 
 1035     allow_ubwc &= !(fd_mesa_debug & FD_DBG_NOUBWC);
 1036 
 1037     pipe_reference_init(&prsc->reference, 1);
 1038 
 1039     prsc->screen = pscreen;
 1040 
 1041     if (screen->tile_mode &&
 1042             (tmpl->target != PIPE_BUFFER) &&
 1043             !linear) {
 1044         rsc->layout.tile_mode = screen->tile_mode(prsc);
 1045     }
 1046 
 1047     util_range_init(&rsc->valid_buffer_range);
 1048 
 1049     simple_mtx_init(&rsc->lock, mtx_plain);
 1050 
 1051     rsc->internal_format = format;
 1052 
 1053     rsc->layout.ubwc = rsc->layout.tile_mode && is_a6xx(screen) && allow_ubwc;
 1054 
 1055     if (prsc->target == PIPE_BUFFER) {
 1056         assert(prsc->format == PIPE_FORMAT_R8_UNORM);
 1057         size = prsc->width0;
 1058         fdl_layout_buffer(&rsc->layout, size);
 1059     } else {
 1060         size = screen->setup_slices(rsc);
 1061     }
 1062 
 1063     /* special case for hw-query buffer, which we need to allocate before we
 1064      * know the size:
 1065      */
 1066     if (size == 0) {
 1067         /* note, semi-intention == instead of & */
 1068         debug_assert(prsc->bind == PIPE_BIND_QUERY_BUFFER);
 1069         return prsc;
 1070     }
 1071 
 1072     /* Set the layer size if the (non-a6xx) backend hasn't done so. */
 1073     if (rsc->layout.layer_first && !rsc->layout.layer_size) {
 1074         rsc->layout.layer_size = align(size, 4096);
 1075         size = rsc->layout.layer_size * prsc->array_size;
 1076     }
 1077 
 1078     if (fd_mesa_debug & FD_DBG_LAYOUT)
 1079         fdl_dump_layout(&rsc->layout);
 1080 
 1081     realloc_bo(rsc, size);
 1082     if (!rsc->bo)
 1083         goto fail;
 1084 
 1085     return prsc;
 1086 fail:
 1087     fd_resource_destroy(pscreen, prsc);
 1088     return NULL;
 1089 }
 1090 
 1091 static struct pipe_resource *
 1092 fd_resource_create(struct pipe_screen *pscreen,
 1093         const struct pipe_resource *tmpl)
 1094 {
 1095     const uint64_t mod = DRM_FORMAT_MOD_INVALID;
 1096     return fd_resource_create_with_modifiers(pscreen, tmpl, &mod, 1);
 1097 }
 1098 
 1099 /**
 1100  * Create a texture from a winsys_handle. The handle is often created in
 1101  * another process by first creating a pipe texture and then calling
 1102  * resource_get_handle.
 1103  */
 1104 static struct pipe_resource *
 1105 fd_resource_from_handle(struct pipe_screen *pscreen,
 1106         const struct pipe_resource *tmpl,
 1107         struct winsys_handle *handle, unsigned usage)
 1108 {
 1109     struct fd_screen *screen = fd_screen(pscreen);
 1110     struct fd_resource *rsc = CALLOC_STRUCT(fd_resource);
 1111     struct fdl_slice *slice = fd_resource_slice(rsc, 0);
 1112     struct pipe_resource *prsc = &rsc->base;
 1113 
 1114     DBG("target=%d, format=%s, %ux%ux%u, array_size=%u, last_level=%u, "
 1115             "nr_samples=%u, usage=%u, bind=%x, flags=%x",
 1116             tmpl->target, util_format_name(tmpl->format),
 1117             tmpl->width0, tmpl->height0, tmpl->depth0,
 1118             tmpl->array_size, tmpl->last_level, tmpl->nr_samples,
 1119             tmpl->usage, tmpl->bind, tmpl->flags);
 1120 
 1121     if (!rsc)
 1122         return NULL;
 1123 
 1124     *prsc = *tmpl;
 1125     fd_resource_layout_init(prsc);
 1126 
 1127     pipe_reference_init(&prsc->reference, 1);
 1128 
 1129     prsc->screen = pscreen;
 1130 
 1131     util_range_init(&rsc->valid_buffer_range);
 1132 
 1133     simple_mtx_init(&rsc->lock, mtx_plain);
 1134 
 1135     rsc->bo = fd_screen_bo_from_handle(pscreen, handle);
 1136     if (!rsc->bo)
 1137         goto fail;
 1138 
 1139     rsc->internal_format = tmpl->format;
 1140     slice->pitch = handle->stride;
 1141     slice->offset = handle->offset;
 1142     slice->size0 = handle->stride * prsc->height0;
 1143 
 1144     uint32_t pitchalign = fd_screen(pscreen)->gmem_alignw * rsc->layout.cpp;
 1145 
 1146     if ((slice->pitch < align(prsc->width0 * rsc->layout.cpp, pitchalign)) ||
 1147             (slice->pitch & (pitchalign - 1)))
 1148         goto fail;
 1149 
 1150     assert(rsc->layout.cpp);
 1151 
 1152     if (screen->layout_resource_for_modifier(rsc, handle->modifier) < 0)
 1153         goto fail;
 1154 
 1155     if (screen->ro) {
 1156         rsc->scanout =
 1157             renderonly_create_gpu_import_for_resource(prsc, screen->ro, NULL);
 1158         /* failure is expected in some cases.. */
 1159     }
 1160 
 1161     rsc->valid = true;
 1162 
 1163     return prsc;
 1164 
 1165 fail:
 1166     fd_resource_destroy(pscreen, prsc);
 1167     return NULL;
 1168 }
 1169 
 1170 bool
 1171 fd_render_condition_check(struct pipe_context *pctx)
 1172 {
 1173     struct fd_context *ctx = fd_context(pctx);
 1174 
 1175     if (!ctx->cond_query)
 1176         return true;
 1177 
 1178     union pipe_query_result res = { 0 };
 1179     bool wait =
 1180         ctx->cond_mode != PIPE_RENDER_COND_NO_WAIT &&
 1181         ctx->cond_mode != PIPE_RENDER_COND_BY_REGION_NO_WAIT;
 1182 
 1183     if (pctx->get_query_result(pctx, ctx->cond_query, wait, &res))
 1184             return (bool)res.u64 != ctx->cond_cond;
 1185 
 1186     return true;
 1187 }
 1188 
 1189 static void
 1190 fd_invalidate_resource(struct pipe_context *pctx, struct pipe_resource *prsc)
 1191 {
 1192     struct fd_context *ctx = fd_context(pctx);
 1193     struct fd_resource *rsc = fd_resource(prsc);
 1194 
 1195     /*
 1196      * TODO I guess we could track that the resource is invalidated and
 1197      * use that as a hint to realloc rather than stall in _transfer_map(),
 1198      * even in the non-DISCARD_WHOLE_RESOURCE case?
 1199      *
 1200      * Note: we set dirty bits to trigger invalidate logic fd_draw_vbo
 1201      */
 1202 
 1203     if (rsc->write_batch) {
 1204         struct fd_batch *batch = rsc->write_batch;
 1205         struct pipe_framebuffer_state *pfb = &batch->framebuffer;
 1206 
 1207         if (pfb->zsbuf && pfb->zsbuf->texture == prsc) {
 1208             batch->resolve &= ~(FD_BUFFER_DEPTH | FD_BUFFER_STENCIL);
 1209             ctx->dirty |= FD_DIRTY_ZSA;
 1210         }
 1211 
 1212         for (unsigned i = 0; i < pfb->nr_cbufs; i++) {
 1213             if (pfb->cbufs[i] && pfb->cbufs[i]->texture == prsc) {
 1214                 batch->resolve &= ~(PIPE_CLEAR_COLOR0 << i);
 1215                 ctx->dirty |= FD_DIRTY_FRAMEBUFFER;
 1216             }
 1217         }
 1218     }
 1219 
 1220     rsc->valid = false;
 1221 }
 1222 
 1223 static enum pipe_format
 1224 fd_resource_get_internal_format(struct pipe_resource *prsc)
 1225 {
 1226     return fd_resource(prsc)->internal_format;
 1227 }
 1228 
 1229 static void
 1230 fd_resource_set_stencil(struct pipe_resource *prsc,
 1231         struct pipe_resource *stencil)
 1232 {
 1233     fd_resource(prsc)->stencil = fd_resource(stencil);
 1234 }
 1235 
 1236 static struct pipe_resource *
 1237 fd_resource_get_stencil(struct pipe_resource *prsc)
 1238 {
 1239     struct fd_resource *rsc = fd_resource(prsc);
 1240     if (rsc->stencil)
 1241         return &rsc->stencil->base;
 1242     return NULL;
 1243 }
 1244 
 1245 static const struct u_transfer_vtbl transfer_vtbl = {
 1246         .resource_create          = fd_resource_create,
 1247         .resource_destroy         = fd_resource_destroy,
 1248         .transfer_map             = fd_resource_transfer_map,
 1249         .transfer_flush_region    = fd_resource_transfer_flush_region,
 1250         .transfer_unmap           = fd_resource_transfer_unmap,
 1251         .get_internal_format      = fd_resource_get_internal_format,
 1252         .set_stencil              = fd_resource_set_stencil,
 1253         .get_stencil              = fd_resource_get_stencil,
 1254 };
 1255 
 1256 static const uint64_t supported_modifiers[] = {
 1257     DRM_FORMAT_MOD_LINEAR,
 1258 };
 1259 
 1260 static int
 1261 fd_layout_resource_for_modifier(struct fd_resource *rsc, uint64_t modifier)
 1262 {
 1263     switch (modifier) {
 1264     case DRM_FORMAT_MOD_LINEAR:
 1265         /* The dri gallium frontend will pass DRM_FORMAT_MOD_INVALID to us
 1266          * when it's called through any of the non-modifier BO create entry
 1267          * points.  Other drivers will determine tiling from the kernel or
 1268          * other legacy backchannels, but for freedreno it just means
 1269          * LINEAR. */
 1270     case DRM_FORMAT_MOD_INVALID:
 1271         return 0;
 1272     default:
 1273         return -1;
 1274     }
 1275 }
 1276 
 1277 void
 1278 fd_resource_screen_init(struct pipe_screen *pscreen)
 1279 {
 1280     struct fd_screen *screen = fd_screen(pscreen);
 1281     bool fake_rgtc = screen->gpu_id < 400;
 1282 
 1283     pscreen->resource_create = u_transfer_helper_resource_create;
 1284     /* NOTE: u_transfer_helper does not yet support the _with_modifiers()
 1285      * variant:
 1286      */
 1287     pscreen->resource_create_with_modifiers = fd_resource_create_with_modifiers;
 1288     pscreen->resource_from_handle = fd_resource_from_handle;
 1289     pscreen->resource_get_handle = fd_resource_get_handle;
 1290     pscreen->resource_destroy = u_transfer_helper_resource_destroy;
 1291 
 1292     pscreen->transfer_helper = u_transfer_helper_create(&transfer_vtbl,
 1293             true, false, fake_rgtc, true);
 1294 
 1295     if (!screen->setup_slices)
 1296         screen->setup_slices = fd_setup_slices;
 1297     if (!screen->layout_resource_for_modifier)
 1298         screen->layout_resource_for_modifier = fd_layout_resource_for_modifier;
 1299     if (!screen->supported_modifiers) {
 1300         screen->supported_modifiers = supported_modifiers;
 1301         screen->num_supported_modifiers = ARRAY_SIZE(supported_modifiers);
 1302     }
 1303 }
 1304 
 1305 static void
 1306 fd_get_sample_position(struct pipe_context *context,
 1307                          unsigned sample_count, unsigned sample_index,
 1308                          float *pos_out)
 1309 {
 1310     /* The following is copied from nouveau/nv50 except for position
 1311      * values, which are taken from blob driver */
 1312     static const uint8_t pos1[1][2] = { { 0x8, 0x8 } };
 1313     static const uint8_t pos2[2][2] = {
 1314         { 0xc, 0xc }, { 0x4, 0x4 } };
 1315     static const uint8_t pos4[4][2] = {
 1316         { 0x6, 0x2 }, { 0xe, 0x6 },
 1317         { 0x2, 0xa }, { 0xa, 0xe } };
 1318     /* TODO needs to be verified on supported hw */
 1319     static const uint8_t pos8[8][2] = {
 1320         { 0x9, 0x5 }, { 0x7, 0xb },
 1321         { 0xd, 0x9 }, { 0x5, 0x3 },
 1322         { 0x3, 0xd }, { 0x1, 0x7 },
 1323         { 0xb, 0xf }, { 0xf, 0x1 } };
 1324 
 1325     const uint8_t (*ptr)[2];
 1326 
 1327     switch (sample_count) {
 1328     case 1:
 1329         ptr = pos1;
 1330         break;
 1331     case 2:
 1332         ptr = pos2;
 1333         break;
 1334     case 4:
 1335         ptr = pos4;
 1336         break;
 1337     case 8:
 1338         ptr = pos8;
 1339         break;
 1340     default:
 1341         assert(0);
 1342         return;
 1343     }
 1344 
 1345     pos_out[0] = ptr[sample_index][0] / 16.0f;
 1346     pos_out[1] = ptr[sample_index][1] / 16.0f;
 1347 }
 1348 
 1349 static void
 1350 fd_blit_pipe(struct pipe_context *pctx, const struct pipe_blit_info *blit_info)
 1351 {
 1352     /* wrap fd_blit to return void */
 1353     fd_blit(pctx, blit_info);
 1354 }
 1355 
 1356 void
 1357 fd_resource_context_init(struct pipe_context *pctx)
 1358 {
 1359     pctx->transfer_map = u_transfer_helper_transfer_map;
 1360     pctx->transfer_flush_region = u_transfer_helper_transfer_flush_region;
 1361     pctx->transfer_unmap = u_transfer_helper_transfer_unmap;
 1362     pctx->buffer_subdata = u_default_buffer_subdata;
 1363     pctx->texture_subdata = u_default_texture_subdata;
 1364     pctx->create_surface = fd_create_surface;
 1365     pctx->surface_destroy = fd_surface_destroy;
 1366     pctx->resource_copy_region = fd_resource_copy_region;
 1367     pctx->blit = fd_blit_pipe;
 1368     pctx->flush_resource = fd_flush_resource;
 1369     pctx->invalidate_resource = fd_invalidate_resource;
 1370     pctx->get_sample_position = fd_get_sample_position;
 1371 }