"Fossies" - the Fresh Open Source Software Archive

Member "mesa-20.1.8/src/gallium/drivers/radeonsi/si_blit.c" (16 Sep 2020, 48007 Bytes) of package /linux/misc/mesa-20.1.8.tar.xz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "si_blit.c" see the Fossies "Dox" file reference documentation and the last Fossies "Diffs" side-by-side code changes report: 20.1.5_vs_20.2.0-rc1.

    1 /*
    2  * Copyright 2010 Jerome Glisse <glisse@freedesktop.org>
    3  * Copyright 2015 Advanced Micro Devices, Inc.
    4  * All Rights Reserved.
    5  *
    6  * Permission is hereby granted, free of charge, to any person obtaining a
    7  * copy of this software and associated documentation files (the "Software"),
    8  * to deal in the Software without restriction, including without limitation
    9  * on the rights to use, copy, modify, merge, publish, distribute, sub
   10  * license, and/or sell copies of the Software, and to permit persons to whom
   11  * the Software is furnished to do so, subject to the following conditions:
   12  *
   13  * The above copyright notice and this permission notice (including the next
   14  * paragraph) shall be included in all copies or substantial portions of the
   15  * Software.
   16  *
   17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   19  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
   20  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
   21  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
   22  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
   23  * USE OR OTHER DEALINGS IN THE SOFTWARE.
   24  */
   25 
   26 #include "si_compute.h"
   27 #include "si_pipe.h"
   28 #include "util/format/u_format.h"
   29 #include "util/u_log.h"
   30 #include "util/u_surface.h"
   31 
   32 enum
   33 {
   34    SI_COPY =
   35       SI_SAVE_FRAMEBUFFER | SI_SAVE_TEXTURES | SI_SAVE_FRAGMENT_STATE | SI_DISABLE_RENDER_COND,
   36 
   37    SI_BLIT = SI_SAVE_FRAMEBUFFER | SI_SAVE_TEXTURES | SI_SAVE_FRAGMENT_STATE,
   38 
   39    SI_DECOMPRESS = SI_SAVE_FRAMEBUFFER | SI_SAVE_FRAGMENT_STATE | SI_DISABLE_RENDER_COND,
   40 
   41    SI_COLOR_RESOLVE = SI_SAVE_FRAMEBUFFER | SI_SAVE_FRAGMENT_STATE
   42 };
   43 
   44 void si_blitter_begin(struct si_context *sctx, enum si_blitter_op op)
   45 {
   46    util_blitter_save_vertex_shader(sctx->blitter, sctx->vs_shader.cso);
   47    util_blitter_save_tessctrl_shader(sctx->blitter, sctx->tcs_shader.cso);
   48    util_blitter_save_tesseval_shader(sctx->blitter, sctx->tes_shader.cso);
   49    util_blitter_save_geometry_shader(sctx->blitter, sctx->gs_shader.cso);
   50    util_blitter_save_so_targets(sctx->blitter, sctx->streamout.num_targets,
   51                                 (struct pipe_stream_output_target **)sctx->streamout.targets);
   52    util_blitter_save_rasterizer(sctx->blitter, sctx->queued.named.rasterizer);
   53 
   54    if (op & SI_SAVE_FRAGMENT_STATE) {
   55       util_blitter_save_blend(sctx->blitter, sctx->queued.named.blend);
   56       util_blitter_save_depth_stencil_alpha(sctx->blitter, sctx->queued.named.dsa);
   57       util_blitter_save_stencil_ref(sctx->blitter, &sctx->stencil_ref.state);
   58       util_blitter_save_fragment_shader(sctx->blitter, sctx->ps_shader.cso);
   59       util_blitter_save_sample_mask(sctx->blitter, sctx->sample_mask);
   60       util_blitter_save_scissor(sctx->blitter, &sctx->scissors[0]);
   61       util_blitter_save_window_rectangles(sctx->blitter, sctx->window_rectangles_include,
   62                                           sctx->num_window_rectangles, sctx->window_rectangles);
   63    }
   64 
   65    if (op & SI_SAVE_FRAMEBUFFER)
   66       util_blitter_save_framebuffer(sctx->blitter, &sctx->framebuffer.state);
   67 
   68    if (op & SI_SAVE_TEXTURES) {
   69       util_blitter_save_fragment_sampler_states(
   70          sctx->blitter, 2, (void **)sctx->samplers[PIPE_SHADER_FRAGMENT].sampler_states);
   71 
   72       util_blitter_save_fragment_sampler_views(sctx->blitter, 2,
   73                                                sctx->samplers[PIPE_SHADER_FRAGMENT].views);
   74    }
   75 
   76    if (op & SI_DISABLE_RENDER_COND)
   77       sctx->render_cond_force_off = true;
   78 
   79    if (sctx->screen->dpbb_allowed) {
   80       sctx->dpbb_force_off = true;
   81       si_mark_atom_dirty(sctx, &sctx->atoms.s.dpbb_state);
   82    }
   83 }
   84 
   85 void si_blitter_end(struct si_context *sctx)
   86 {
   87    if (sctx->screen->dpbb_allowed) {
   88       sctx->dpbb_force_off = false;
   89       si_mark_atom_dirty(sctx, &sctx->atoms.s.dpbb_state);
   90    }
   91 
   92    sctx->render_cond_force_off = false;
   93 
   94    /* Restore shader pointers because the VS blit shader changed all
   95     * non-global VS user SGPRs. */
   96    sctx->shader_pointers_dirty |= SI_DESCS_SHADER_MASK(VERTEX);
   97    sctx->vertex_buffer_pointer_dirty = sctx->vb_descriptors_buffer != NULL;
   98    sctx->vertex_buffer_user_sgprs_dirty = sctx->num_vertex_elements > 0;
   99    si_mark_atom_dirty(sctx, &sctx->atoms.s.shader_pointers);
  100 }
  101 
  102 static unsigned u_max_sample(struct pipe_resource *r)
  103 {
  104    return r->nr_samples ? r->nr_samples - 1 : 0;
  105 }
  106 
  107 static unsigned si_blit_dbcb_copy(struct si_context *sctx, struct si_texture *src,
  108                                   struct si_texture *dst, unsigned planes, unsigned level_mask,
  109                                   unsigned first_layer, unsigned last_layer, unsigned first_sample,
  110                                   unsigned last_sample)
  111 {
  112    struct pipe_surface surf_tmpl = {{0}};
  113    unsigned layer, sample, checked_last_layer, max_layer;
  114    unsigned fully_copied_levels = 0;
  115 
  116    if (planes & PIPE_MASK_Z)
  117       sctx->dbcb_depth_copy_enabled = true;
  118    if (planes & PIPE_MASK_S)
  119       sctx->dbcb_stencil_copy_enabled = true;
  120    si_mark_atom_dirty(sctx, &sctx->atoms.s.db_render_state);
  121 
  122    assert(sctx->dbcb_depth_copy_enabled || sctx->dbcb_stencil_copy_enabled);
  123 
  124    sctx->decompression_enabled = true;
  125 
  126    while (level_mask) {
  127       unsigned level = u_bit_scan(&level_mask);
  128 
  129       /* The smaller the mipmap level, the less layers there are
  130        * as far as 3D textures are concerned. */
  131       max_layer = util_max_layer(&src->buffer.b.b, level);
  132       checked_last_layer = MIN2(last_layer, max_layer);
  133 
  134       surf_tmpl.u.tex.level = level;
  135 
  136       for (layer = first_layer; layer <= checked_last_layer; layer++) {
  137          struct pipe_surface *zsurf, *cbsurf;
  138 
  139          surf_tmpl.format = src->buffer.b.b.format;
  140          surf_tmpl.u.tex.first_layer = layer;
  141          surf_tmpl.u.tex.last_layer = layer;
  142 
  143          zsurf = sctx->b.create_surface(&sctx->b, &src->buffer.b.b, &surf_tmpl);
  144 
  145          surf_tmpl.format = dst->buffer.b.b.format;
  146          cbsurf = sctx->b.create_surface(&sctx->b, &dst->buffer.b.b, &surf_tmpl);
  147 
  148          for (sample = first_sample; sample <= last_sample; sample++) {
  149             if (sample != sctx->dbcb_copy_sample) {
  150                sctx->dbcb_copy_sample = sample;
  151                si_mark_atom_dirty(sctx, &sctx->atoms.s.db_render_state);
  152             }
  153 
  154             si_blitter_begin(sctx, SI_DECOMPRESS);
  155             util_blitter_custom_depth_stencil(sctx->blitter, zsurf, cbsurf, 1 << sample,
  156                                               sctx->custom_dsa_flush, 1.0f);
  157             si_blitter_end(sctx);
  158          }
  159 
  160          pipe_surface_reference(&zsurf, NULL);
  161          pipe_surface_reference(&cbsurf, NULL);
  162       }
  163 
  164       if (first_layer == 0 && last_layer >= max_layer && first_sample == 0 &&
  165           last_sample >= u_max_sample(&src->buffer.b.b))
  166          fully_copied_levels |= 1u << level;
  167    }
  168 
  169    sctx->decompression_enabled = false;
  170    sctx->dbcb_depth_copy_enabled = false;
  171    sctx->dbcb_stencil_copy_enabled = false;
  172    si_mark_atom_dirty(sctx, &sctx->atoms.s.db_render_state);
  173 
  174    return fully_copied_levels;
  175 }
  176 
  177 /* Helper function for si_blit_decompress_zs_in_place.
  178  */
  179 static void si_blit_decompress_zs_planes_in_place(struct si_context *sctx,
  180                                                   struct si_texture *texture, unsigned planes,
  181                                                   unsigned level_mask, unsigned first_layer,
  182                                                   unsigned last_layer)
  183 {
  184    struct pipe_surface *zsurf, surf_tmpl = {{0}};
  185    unsigned layer, max_layer, checked_last_layer;
  186    unsigned fully_decompressed_mask = 0;
  187 
  188    if (!level_mask)
  189       return;
  190 
  191    if (planes & PIPE_MASK_S)
  192       sctx->db_flush_stencil_inplace = true;
  193    if (planes & PIPE_MASK_Z)
  194       sctx->db_flush_depth_inplace = true;
  195    si_mark_atom_dirty(sctx, &sctx->atoms.s.db_render_state);
  196 
  197    surf_tmpl.format = texture->buffer.b.b.format;
  198 
  199    sctx->decompression_enabled = true;
  200 
  201    while (level_mask) {
  202       unsigned level = u_bit_scan(&level_mask);
  203 
  204       surf_tmpl.u.tex.level = level;
  205 
  206       /* The smaller the mipmap level, the less layers there are
  207        * as far as 3D textures are concerned. */
  208       max_layer = util_max_layer(&texture->buffer.b.b, level);
  209       checked_last_layer = MIN2(last_layer, max_layer);
  210 
  211       for (layer = first_layer; layer <= checked_last_layer; layer++) {
  212          surf_tmpl.u.tex.first_layer = layer;
  213          surf_tmpl.u.tex.last_layer = layer;
  214 
  215          zsurf = sctx->b.create_surface(&sctx->b, &texture->buffer.b.b, &surf_tmpl);
  216 
  217          si_blitter_begin(sctx, SI_DECOMPRESS);
  218          util_blitter_custom_depth_stencil(sctx->blitter, zsurf, NULL, ~0, sctx->custom_dsa_flush,
  219                                            1.0f);
  220          si_blitter_end(sctx);
  221 
  222          pipe_surface_reference(&zsurf, NULL);
  223       }
  224 
  225       /* The texture will always be dirty if some layers aren't flushed.
  226        * I don't think this case occurs often though. */
  227       if (first_layer == 0 && last_layer >= max_layer) {
  228          fully_decompressed_mask |= 1u << level;
  229       }
  230    }
  231 
  232    if (planes & PIPE_MASK_Z)
  233       texture->dirty_level_mask &= ~fully_decompressed_mask;
  234    if (planes & PIPE_MASK_S)
  235       texture->stencil_dirty_level_mask &= ~fully_decompressed_mask;
  236 
  237    sctx->decompression_enabled = false;
  238    sctx->db_flush_depth_inplace = false;
  239    sctx->db_flush_stencil_inplace = false;
  240    si_mark_atom_dirty(sctx, &sctx->atoms.s.db_render_state);
  241 }
  242 
  243 /* Helper function of si_flush_depth_texture: decompress the given levels
  244  * of Z and/or S planes in place.
  245  */
  246 static void si_blit_decompress_zs_in_place(struct si_context *sctx, struct si_texture *texture,
  247                                            unsigned levels_z, unsigned levels_s,
  248                                            unsigned first_layer, unsigned last_layer)
  249 {
  250    unsigned both = levels_z & levels_s;
  251 
  252    /* First, do combined Z & S decompresses for levels that need it. */
  253    if (both) {
  254       si_blit_decompress_zs_planes_in_place(sctx, texture, PIPE_MASK_Z | PIPE_MASK_S, both,
  255                                             first_layer, last_layer);
  256       levels_z &= ~both;
  257       levels_s &= ~both;
  258    }
  259 
  260    /* Now do separate Z and S decompresses. */
  261    if (levels_z) {
  262       si_blit_decompress_zs_planes_in_place(sctx, texture, PIPE_MASK_Z, levels_z, first_layer,
  263                                             last_layer);
  264    }
  265 
  266    if (levels_s) {
  267       si_blit_decompress_zs_planes_in_place(sctx, texture, PIPE_MASK_S, levels_s, first_layer,
  268                                             last_layer);
  269    }
  270 }
  271 
  272 static void si_decompress_depth(struct si_context *sctx, struct si_texture *tex,
  273                                 unsigned required_planes, unsigned first_level, unsigned last_level,
  274                                 unsigned first_layer, unsigned last_layer)
  275 {
  276    unsigned inplace_planes = 0;
  277    unsigned copy_planes = 0;
  278    unsigned level_mask = u_bit_consecutive(first_level, last_level - first_level + 1);
  279    unsigned levels_z = 0;
  280    unsigned levels_s = 0;
  281 
  282    if (required_planes & PIPE_MASK_Z) {
  283       levels_z = level_mask & tex->dirty_level_mask;
  284 
  285       if (levels_z) {
  286          if (si_can_sample_zs(tex, false))
  287             inplace_planes |= PIPE_MASK_Z;
  288          else
  289             copy_planes |= PIPE_MASK_Z;
  290       }
  291    }
  292    if (required_planes & PIPE_MASK_S) {
  293       levels_s = level_mask & tex->stencil_dirty_level_mask;
  294 
  295       if (levels_s) {
  296          if (si_can_sample_zs(tex, true))
  297             inplace_planes |= PIPE_MASK_S;
  298          else
  299             copy_planes |= PIPE_MASK_S;
  300       }
  301    }
  302 
  303    if (unlikely(sctx->log))
  304       u_log_printf(sctx->log,
  305                    "\n------------------------------------------------\n"
  306                    "Decompress Depth (levels %u - %u, levels Z: 0x%x S: 0x%x)\n\n",
  307                    first_level, last_level, levels_z, levels_s);
  308 
  309    /* We may have to allocate the flushed texture here when called from
  310     * si_decompress_subresource.
  311     */
  312    if (copy_planes &&
  313        (tex->flushed_depth_texture || si_init_flushed_depth_texture(&sctx->b, &tex->buffer.b.b))) {
  314       struct si_texture *dst = tex->flushed_depth_texture;
  315       unsigned fully_copied_levels;
  316       unsigned levels = 0;
  317 
  318       assert(tex->flushed_depth_texture);
  319 
  320       if (util_format_is_depth_and_stencil(dst->buffer.b.b.format))
  321          copy_planes = PIPE_MASK_Z | PIPE_MASK_S;
  322 
  323       if (copy_planes & PIPE_MASK_Z) {
  324          levels |= levels_z;
  325          levels_z = 0;
  326       }
  327       if (copy_planes & PIPE_MASK_S) {
  328          levels |= levels_s;
  329          levels_s = 0;
  330       }
  331 
  332       fully_copied_levels = si_blit_dbcb_copy(sctx, tex, dst, copy_planes, levels, first_layer,
  333                                               last_layer, 0, u_max_sample(&tex->buffer.b.b));
  334 
  335       if (copy_planes & PIPE_MASK_Z)
  336          tex->dirty_level_mask &= ~fully_copied_levels;
  337       if (copy_planes & PIPE_MASK_S)
  338          tex->stencil_dirty_level_mask &= ~fully_copied_levels;
  339    }
  340 
  341    if (inplace_planes) {
  342       bool has_htile = si_htile_enabled(tex, first_level, inplace_planes);
  343       bool tc_compat_htile = vi_tc_compat_htile_enabled(tex, first_level, inplace_planes);
  344 
  345       /* Don't decompress if there is no HTILE or when HTILE is
  346        * TC-compatible. */
  347       if (has_htile && !tc_compat_htile) {
  348          si_blit_decompress_zs_in_place(sctx, tex, levels_z, levels_s, first_layer, last_layer);
  349       } else {
  350          /* This is only a cache flush.
  351           *
  352           * Only clear the mask that we are flushing, because
  353           * si_make_DB_shader_coherent() treats different levels
  354           * and depth and stencil differently.
  355           */
  356          if (inplace_planes & PIPE_MASK_Z)
  357             tex->dirty_level_mask &= ~levels_z;
  358          if (inplace_planes & PIPE_MASK_S)
  359             tex->stencil_dirty_level_mask &= ~levels_s;
  360       }
  361 
  362       /* Only in-place decompression needs to flush DB caches, or
  363        * when we don't decompress but TC-compatible planes are dirty.
  364        */
  365       si_make_DB_shader_coherent(sctx, tex->buffer.b.b.nr_samples, inplace_planes & PIPE_MASK_S,
  366                                  tc_compat_htile);
  367    }
  368    /* set_framebuffer_state takes care of coherency for single-sample.
  369     * The DB->CB copy uses CB for the final writes.
  370     */
  371    if (copy_planes && tex->buffer.b.b.nr_samples > 1)
  372       si_make_CB_shader_coherent(sctx, tex->buffer.b.b.nr_samples, false, true /* no DCC */);
  373 }
  374 
  375 static void si_decompress_sampler_depth_textures(struct si_context *sctx,
  376                                                  struct si_samplers *textures)
  377 {
  378    unsigned i;
  379    unsigned mask = textures->needs_depth_decompress_mask;
  380 
  381    while (mask) {
  382       struct pipe_sampler_view *view;
  383       struct si_sampler_view *sview;
  384       struct si_texture *tex;
  385 
  386       i = u_bit_scan(&mask);
  387 
  388       view = textures->views[i];
  389       assert(view);
  390       sview = (struct si_sampler_view *)view;
  391 
  392       tex = (struct si_texture *)view->texture;
  393       assert(tex->db_compatible);
  394 
  395       si_decompress_depth(sctx, tex, sview->is_stencil_sampler ? PIPE_MASK_S : PIPE_MASK_Z,
  396                           view->u.tex.first_level, view->u.tex.last_level, 0,
  397                           util_max_layer(&tex->buffer.b.b, view->u.tex.first_level));
  398    }
  399 }
  400 
  401 static void si_blit_decompress_color(struct si_context *sctx, struct si_texture *tex,
  402                                      unsigned first_level, unsigned last_level,
  403                                      unsigned first_layer, unsigned last_layer,
  404                                      bool need_dcc_decompress, bool need_fmask_expand)
  405 {
  406    void *custom_blend;
  407    unsigned layer, checked_last_layer, max_layer;
  408    unsigned level_mask = u_bit_consecutive(first_level, last_level - first_level + 1);
  409 
  410    if (!need_dcc_decompress)
  411       level_mask &= tex->dirty_level_mask;
  412    if (!level_mask)
  413       goto expand_fmask;
  414 
  415    if (unlikely(sctx->log))
  416       u_log_printf(sctx->log,
  417                    "\n------------------------------------------------\n"
  418                    "Decompress Color (levels %u - %u, mask 0x%x)\n\n",
  419                    first_level, last_level, level_mask);
  420 
  421    if (need_dcc_decompress) {
  422       custom_blend = sctx->custom_blend_dcc_decompress;
  423 
  424       assert(tex->surface.dcc_offset);
  425 
  426       /* disable levels without DCC */
  427       for (int i = first_level; i <= last_level; i++) {
  428          if (!vi_dcc_enabled(tex, i))
  429             level_mask &= ~(1 << i);
  430       }
  431    } else if (tex->surface.fmask_size) {
  432       custom_blend = sctx->custom_blend_fmask_decompress;
  433    } else {
  434       custom_blend = sctx->custom_blend_eliminate_fastclear;
  435    }
  436 
  437    sctx->decompression_enabled = true;
  438 
  439    while (level_mask) {
  440       unsigned level = u_bit_scan(&level_mask);
  441 
  442       /* The smaller the mipmap level, the less layers there are
  443        * as far as 3D textures are concerned. */
  444       max_layer = util_max_layer(&tex->buffer.b.b, level);
  445       checked_last_layer = MIN2(last_layer, max_layer);
  446 
  447       for (layer = first_layer; layer <= checked_last_layer; layer++) {
  448          struct pipe_surface *cbsurf, surf_tmpl;
  449 
  450          surf_tmpl.format = tex->buffer.b.b.format;
  451          surf_tmpl.u.tex.level = level;
  452          surf_tmpl.u.tex.first_layer = layer;
  453          surf_tmpl.u.tex.last_layer = layer;
  454          cbsurf = sctx->b.create_surface(&sctx->b, &tex->buffer.b.b, &surf_tmpl);
  455 
  456          /* Required before and after FMASK and DCC_DECOMPRESS. */
  457          if (custom_blend == sctx->custom_blend_fmask_decompress ||
  458              custom_blend == sctx->custom_blend_dcc_decompress)
  459             sctx->flags |= SI_CONTEXT_FLUSH_AND_INV_CB;
  460 
  461          si_blitter_begin(sctx, SI_DECOMPRESS);
  462          util_blitter_custom_color(sctx->blitter, cbsurf, custom_blend);
  463          si_blitter_end(sctx);
  464 
  465          if (custom_blend == sctx->custom_blend_fmask_decompress ||
  466              custom_blend == sctx->custom_blend_dcc_decompress)
  467             sctx->flags |= SI_CONTEXT_FLUSH_AND_INV_CB;
  468 
  469          pipe_surface_reference(&cbsurf, NULL);
  470       }
  471 
  472       /* The texture will always be dirty if some layers aren't flushed.
  473        * I don't think this case occurs often though. */
  474       if (first_layer == 0 && last_layer >= max_layer) {
  475          tex->dirty_level_mask &= ~(1 << level);
  476       }
  477    }
  478 
  479    sctx->decompression_enabled = false;
  480    si_make_CB_shader_coherent(sctx, tex->buffer.b.b.nr_samples, vi_dcc_enabled(tex, first_level),
  481                               tex->surface.u.gfx9.dcc.pipe_aligned);
  482 
  483 expand_fmask:
  484    if (need_fmask_expand && tex->surface.fmask_offset && !tex->fmask_is_identity) {
  485       si_compute_expand_fmask(&sctx->b, &tex->buffer.b.b);
  486       tex->fmask_is_identity = true;
  487    }
  488 }
  489 
  490 static void si_decompress_color_texture(struct si_context *sctx, struct si_texture *tex,
  491                                         unsigned first_level, unsigned last_level,
  492                                         bool need_fmask_expand)
  493 {
  494    /* CMASK or DCC can be discarded and we can still end up here. */
  495    if (!tex->cmask_buffer && !tex->surface.fmask_size && !tex->surface.dcc_offset)
  496       return;
  497 
  498    si_blit_decompress_color(sctx, tex, first_level, last_level, 0,
  499                             util_max_layer(&tex->buffer.b.b, first_level), false,
  500                             need_fmask_expand);
  501 }
  502 
  503 static void si_decompress_sampler_color_textures(struct si_context *sctx,
  504                                                  struct si_samplers *textures)
  505 {
  506    unsigned i;
  507    unsigned mask = textures->needs_color_decompress_mask;
  508 
  509    while (mask) {
  510       struct pipe_sampler_view *view;
  511       struct si_texture *tex;
  512 
  513       i = u_bit_scan(&mask);
  514 
  515       view = textures->views[i];
  516       assert(view);
  517 
  518       tex = (struct si_texture *)view->texture;
  519 
  520       si_decompress_color_texture(sctx, tex, view->u.tex.first_level, view->u.tex.last_level,
  521                                   false);
  522    }
  523 }
  524 
  525 static void si_decompress_image_color_textures(struct si_context *sctx, struct si_images *images)
  526 {
  527    unsigned i;
  528    unsigned mask = images->needs_color_decompress_mask;
  529 
  530    while (mask) {
  531       const struct pipe_image_view *view;
  532       struct si_texture *tex;
  533 
  534       i = u_bit_scan(&mask);
  535 
  536       view = &images->views[i];
  537       assert(view->resource->target != PIPE_BUFFER);
  538 
  539       tex = (struct si_texture *)view->resource;
  540 
  541       si_decompress_color_texture(sctx, tex, view->u.tex.level, view->u.tex.level,
  542                                   view->access & PIPE_IMAGE_ACCESS_WRITE);
  543    }
  544 }
  545 
  546 static void si_check_render_feedback_texture(struct si_context *sctx, struct si_texture *tex,
  547                                              unsigned first_level, unsigned last_level,
  548                                              unsigned first_layer, unsigned last_layer)
  549 {
  550    bool render_feedback = false;
  551 
  552    if (!tex->surface.dcc_offset)
  553       return;
  554 
  555    for (unsigned j = 0; j < sctx->framebuffer.state.nr_cbufs; ++j) {
  556       struct si_surface *surf;
  557 
  558       if (!sctx->framebuffer.state.cbufs[j])
  559          continue;
  560 
  561       surf = (struct si_surface *)sctx->framebuffer.state.cbufs[j];
  562 
  563       if (tex == (struct si_texture *)surf->base.texture && surf->base.u.tex.level >= first_level &&
  564           surf->base.u.tex.level <= last_level && surf->base.u.tex.first_layer <= last_layer &&
  565           surf->base.u.tex.last_layer >= first_layer) {
  566          render_feedback = true;
  567          break;
  568       }
  569    }
  570 
  571    if (render_feedback)
  572       si_texture_disable_dcc(sctx, tex);
  573 }
  574 
  575 static void si_check_render_feedback_textures(struct si_context *sctx, struct si_samplers *textures)
  576 {
  577    uint32_t mask = textures->enabled_mask;
  578 
  579    while (mask) {
  580       const struct pipe_sampler_view *view;
  581       struct si_texture *tex;
  582 
  583       unsigned i = u_bit_scan(&mask);
  584 
  585       view = textures->views[i];
  586       if (view->texture->target == PIPE_BUFFER)
  587          continue;
  588 
  589       tex = (struct si_texture *)view->texture;
  590 
  591       si_check_render_feedback_texture(sctx, tex, view->u.tex.first_level, view->u.tex.last_level,
  592                                        view->u.tex.first_layer, view->u.tex.last_layer);
  593    }
  594 }
  595 
  596 static void si_check_render_feedback_images(struct si_context *sctx, struct si_images *images)
  597 {
  598    uint32_t mask = images->enabled_mask;
  599 
  600    while (mask) {
  601       const struct pipe_image_view *view;
  602       struct si_texture *tex;
  603 
  604       unsigned i = u_bit_scan(&mask);
  605 
  606       view = &images->views[i];
  607       if (view->resource->target == PIPE_BUFFER)
  608          continue;
  609 
  610       tex = (struct si_texture *)view->resource;
  611 
  612       si_check_render_feedback_texture(sctx, tex, view->u.tex.level, view->u.tex.level,
  613                                        view->u.tex.first_layer, view->u.tex.last_layer);
  614    }
  615 }
  616 
  617 static void si_check_render_feedback_resident_textures(struct si_context *sctx)
  618 {
  619    util_dynarray_foreach (&sctx->resident_tex_handles, struct si_texture_handle *, tex_handle) {
  620       struct pipe_sampler_view *view;
  621       struct si_texture *tex;
  622 
  623       view = (*tex_handle)->view;
  624       if (view->texture->target == PIPE_BUFFER)
  625          continue;
  626 
  627       tex = (struct si_texture *)view->texture;
  628 
  629       si_check_render_feedback_texture(sctx, tex, view->u.tex.first_level, view->u.tex.last_level,
  630                                        view->u.tex.first_layer, view->u.tex.last_layer);
  631    }
  632 }
  633 
  634 static void si_check_render_feedback_resident_images(struct si_context *sctx)
  635 {
  636    util_dynarray_foreach (&sctx->resident_img_handles, struct si_image_handle *, img_handle) {
  637       struct pipe_image_view *view;
  638       struct si_texture *tex;
  639 
  640       view = &(*img_handle)->view;
  641       if (view->resource->target == PIPE_BUFFER)
  642          continue;
  643 
  644       tex = (struct si_texture *)view->resource;
  645 
  646       si_check_render_feedback_texture(sctx, tex, view->u.tex.level, view->u.tex.level,
  647                                        view->u.tex.first_layer, view->u.tex.last_layer);
  648    }
  649 }
  650 
  651 static void si_check_render_feedback(struct si_context *sctx)
  652 {
  653    if (!sctx->need_check_render_feedback)
  654       return;
  655 
  656    /* There is no render feedback if color writes are disabled.
  657     * (e.g. a pixel shader with image stores)
  658     */
  659    if (!si_get_total_colormask(sctx))
  660       return;
  661 
  662    for (int i = 0; i < SI_NUM_SHADERS; ++i) {
  663       si_check_render_feedback_images(sctx, &sctx->images[i]);
  664       si_check_render_feedback_textures(sctx, &sctx->samplers[i]);
  665    }
  666 
  667    si_check_render_feedback_resident_images(sctx);
  668    si_check_render_feedback_resident_textures(sctx);
  669 
  670    sctx->need_check_render_feedback = false;
  671 }
  672 
  673 static void si_decompress_resident_textures(struct si_context *sctx)
  674 {
  675    util_dynarray_foreach (&sctx->resident_tex_needs_color_decompress, struct si_texture_handle *,
  676                           tex_handle) {
  677       struct pipe_sampler_view *view = (*tex_handle)->view;
  678       struct si_texture *tex = (struct si_texture *)view->texture;
  679 
  680       si_decompress_color_texture(sctx, tex, view->u.tex.first_level, view->u.tex.last_level,
  681                                   false);
  682    }
  683 
  684    util_dynarray_foreach (&sctx->resident_tex_needs_depth_decompress, struct si_texture_handle *,
  685                           tex_handle) {
  686       struct pipe_sampler_view *view = (*tex_handle)->view;
  687       struct si_sampler_view *sview = (struct si_sampler_view *)view;
  688       struct si_texture *tex = (struct si_texture *)view->texture;
  689 
  690       si_decompress_depth(sctx, tex, sview->is_stencil_sampler ? PIPE_MASK_S : PIPE_MASK_Z,
  691                           view->u.tex.first_level, view->u.tex.last_level, 0,
  692                           util_max_layer(&tex->buffer.b.b, view->u.tex.first_level));
  693    }
  694 }
  695 
  696 static void si_decompress_resident_images(struct si_context *sctx)
  697 {
  698    util_dynarray_foreach (&sctx->resident_img_needs_color_decompress, struct si_image_handle *,
  699                           img_handle) {
  700       struct pipe_image_view *view = &(*img_handle)->view;
  701       struct si_texture *tex = (struct si_texture *)view->resource;
  702 
  703       si_decompress_color_texture(sctx, tex, view->u.tex.level, view->u.tex.level,
  704                                   view->access & PIPE_IMAGE_ACCESS_WRITE);
  705    }
  706 }
  707 
  708 void si_decompress_textures(struct si_context *sctx, unsigned shader_mask)
  709 {
  710    unsigned compressed_colortex_counter, mask;
  711 
  712    if (sctx->blitter->running)
  713       return;
  714 
  715    /* Update the compressed_colortex_mask if necessary. */
  716    compressed_colortex_counter = p_atomic_read(&sctx->screen->compressed_colortex_counter);
  717    if (compressed_colortex_counter != sctx->last_compressed_colortex_counter) {
  718       sctx->last_compressed_colortex_counter = compressed_colortex_counter;
  719       si_update_needs_color_decompress_masks(sctx);
  720    }
  721 
  722    /* Decompress color & depth textures if needed. */
  723    mask = sctx->shader_needs_decompress_mask & shader_mask;
  724    while (mask) {
  725       unsigned i = u_bit_scan(&mask);
  726 
  727       if (sctx->samplers[i].needs_depth_decompress_mask) {
  728          si_decompress_sampler_depth_textures(sctx, &sctx->samplers[i]);
  729       }
  730       if (sctx->samplers[i].needs_color_decompress_mask) {
  731          si_decompress_sampler_color_textures(sctx, &sctx->samplers[i]);
  732       }
  733       if (sctx->images[i].needs_color_decompress_mask) {
  734          si_decompress_image_color_textures(sctx, &sctx->images[i]);
  735       }
  736    }
  737 
  738    if (shader_mask & u_bit_consecutive(0, SI_NUM_GRAPHICS_SHADERS)) {
  739       if (sctx->uses_bindless_samplers)
  740          si_decompress_resident_textures(sctx);
  741       if (sctx->uses_bindless_images)
  742          si_decompress_resident_images(sctx);
  743 
  744       if (sctx->ps_uses_fbfetch) {
  745          struct pipe_surface *cb0 = sctx->framebuffer.state.cbufs[0];
  746          si_decompress_color_texture(sctx, (struct si_texture *)cb0->texture,
  747                                      cb0->u.tex.first_layer, cb0->u.tex.last_layer, false);
  748       }
  749 
  750       si_check_render_feedback(sctx);
  751    } else if (shader_mask & (1 << PIPE_SHADER_COMPUTE)) {
  752       if (sctx->cs_shader_state.program->sel.info.uses_bindless_samplers)
  753          si_decompress_resident_textures(sctx);
  754       if (sctx->cs_shader_state.program->sel.info.uses_bindless_images)
  755          si_decompress_resident_images(sctx);
  756    }
  757 }
  758 
  759 /* Helper for decompressing a portion of a color or depth resource before
  760  * blitting if any decompression is needed.
  761  * The driver doesn't decompress resources automatically while u_blitter is
  762  * rendering. */
  763 void si_decompress_subresource(struct pipe_context *ctx, struct pipe_resource *tex, unsigned planes,
  764                                unsigned level, unsigned first_layer, unsigned last_layer)
  765 {
  766    struct si_context *sctx = (struct si_context *)ctx;
  767    struct si_texture *stex = (struct si_texture *)tex;
  768 
  769    if (stex->db_compatible) {
  770       planes &= PIPE_MASK_Z | PIPE_MASK_S;
  771 
  772       if (!stex->surface.has_stencil)
  773          planes &= ~PIPE_MASK_S;
  774 
  775       /* If we've rendered into the framebuffer and it's a blitting
  776        * source, make sure the decompression pass is invoked
  777        * by dirtying the framebuffer.
  778        */
  779       if (sctx->framebuffer.state.zsbuf && sctx->framebuffer.state.zsbuf->u.tex.level == level &&
  780           sctx->framebuffer.state.zsbuf->texture == tex)
  781          si_update_fb_dirtiness_after_rendering(sctx);
  782 
  783       si_decompress_depth(sctx, stex, planes, level, level, first_layer, last_layer);
  784    } else if (stex->surface.fmask_size || stex->cmask_buffer || stex->surface.dcc_offset) {
  785       /* If we've rendered into the framebuffer and it's a blitting
  786        * source, make sure the decompression pass is invoked
  787        * by dirtying the framebuffer.
  788        */
  789       for (unsigned i = 0; i < sctx->framebuffer.state.nr_cbufs; i++) {
  790          if (sctx->framebuffer.state.cbufs[i] &&
  791              sctx->framebuffer.state.cbufs[i]->u.tex.level == level &&
  792              sctx->framebuffer.state.cbufs[i]->texture == tex) {
  793             si_update_fb_dirtiness_after_rendering(sctx);
  794             break;
  795          }
  796       }
  797 
  798       si_blit_decompress_color(sctx, stex, level, level, first_layer, last_layer, false, false);
  799    }
  800 }
  801 
  802 struct texture_orig_info {
  803    unsigned format;
  804    unsigned width0;
  805    unsigned height0;
  806    unsigned npix_x;
  807    unsigned npix_y;
  808    unsigned npix0_x;
  809    unsigned npix0_y;
  810 };
  811 
  812 void si_resource_copy_region(struct pipe_context *ctx, struct pipe_resource *dst,
  813                              unsigned dst_level, unsigned dstx, unsigned dsty, unsigned dstz,
  814                              struct pipe_resource *src, unsigned src_level,
  815                              const struct pipe_box *src_box)
  816 {
  817    struct si_context *sctx = (struct si_context *)ctx;
  818    struct si_texture *ssrc = (struct si_texture *)src;
  819    struct si_texture *sdst = (struct si_texture *)dst;
  820    struct pipe_surface *dst_view, dst_templ;
  821    struct pipe_sampler_view src_templ, *src_view;
  822    unsigned dst_width, dst_height, src_width0, src_height0;
  823    unsigned dst_width0, dst_height0, src_force_level = 0;
  824    struct pipe_box sbox, dstbox;
  825 
  826    /* Handle buffers first. */
  827    if (dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) {
  828       si_copy_buffer(sctx, dst, src, dstx, src_box->x, src_box->width);
  829       return;
  830    }
  831 
  832    if (!util_format_is_compressed(src->format) && !util_format_is_compressed(dst->format) &&
  833        !util_format_is_depth_or_stencil(src->format) && src->nr_samples <= 1 &&
  834        !sdst->surface.dcc_offset &&
  835        !(dst->target != src->target &&
  836          (src->target == PIPE_TEXTURE_1D_ARRAY || dst->target == PIPE_TEXTURE_1D_ARRAY))) {
  837       si_compute_copy_image(sctx, dst, dst_level, src, src_level, dstx, dsty, dstz, src_box);
  838       return;
  839    }
  840 
  841    assert(u_max_sample(dst) == u_max_sample(src));
  842 
  843    /* The driver doesn't decompress resources automatically while
  844     * u_blitter is rendering. */
  845    si_decompress_subresource(ctx, src, PIPE_MASK_RGBAZS, src_level, src_box->z,
  846                              src_box->z + src_box->depth - 1);
  847 
  848    dst_width = u_minify(dst->width0, dst_level);
  849    dst_height = u_minify(dst->height0, dst_level);
  850    dst_width0 = dst->width0;
  851    dst_height0 = dst->height0;
  852    src_width0 = src->width0;
  853    src_height0 = src->height0;
  854 
  855    util_blitter_default_dst_texture(&dst_templ, dst, dst_level, dstz);
  856    util_blitter_default_src_texture(sctx->blitter, &src_templ, src, src_level);
  857 
  858    if (util_format_is_compressed(src->format) || util_format_is_compressed(dst->format)) {
  859       unsigned blocksize = ssrc->surface.bpe;
  860 
  861       if (blocksize == 8)
  862          src_templ.format = PIPE_FORMAT_R16G16B16A16_UINT; /* 64-bit block */
  863       else
  864          src_templ.format = PIPE_FORMAT_R32G32B32A32_UINT; /* 128-bit block */
  865       dst_templ.format = src_templ.format;
  866 
  867       dst_width = util_format_get_nblocksx(dst->format, dst_width);
  868       dst_height = util_format_get_nblocksy(dst->format, dst_height);
  869       dst_width0 = util_format_get_nblocksx(dst->format, dst_width0);
  870       dst_height0 = util_format_get_nblocksy(dst->format, dst_height0);
  871       src_width0 = util_format_get_nblocksx(src->format, src_width0);
  872       src_height0 = util_format_get_nblocksy(src->format, src_height0);
  873 
  874       dstx = util_format_get_nblocksx(dst->format, dstx);
  875       dsty = util_format_get_nblocksy(dst->format, dsty);
  876 
  877       sbox.x = util_format_get_nblocksx(src->format, src_box->x);
  878       sbox.y = util_format_get_nblocksy(src->format, src_box->y);
  879       sbox.z = src_box->z;
  880       sbox.width = util_format_get_nblocksx(src->format, src_box->width);
  881       sbox.height = util_format_get_nblocksy(src->format, src_box->height);
  882       sbox.depth = src_box->depth;
  883       src_box = &sbox;
  884 
  885       src_force_level = src_level;
  886    } else if (!util_blitter_is_copy_supported(sctx->blitter, dst, src)) {
  887       if (util_format_is_subsampled_422(src->format)) {
  888          src_templ.format = PIPE_FORMAT_R8G8B8A8_UINT;
  889          dst_templ.format = PIPE_FORMAT_R8G8B8A8_UINT;
  890 
  891          dst_width = util_format_get_nblocksx(dst->format, dst_width);
  892          dst_width0 = util_format_get_nblocksx(dst->format, dst_width0);
  893          src_width0 = util_format_get_nblocksx(src->format, src_width0);
  894 
  895          dstx = util_format_get_nblocksx(dst->format, dstx);
  896 
  897          sbox = *src_box;
  898          sbox.x = util_format_get_nblocksx(src->format, src_box->x);
  899          sbox.width = util_format_get_nblocksx(src->format, src_box->width);
  900          src_box = &sbox;
  901       } else {
  902          unsigned blocksize = ssrc->surface.bpe;
  903 
  904          switch (blocksize) {
  905          case 1:
  906             dst_templ.format = PIPE_FORMAT_R8_UNORM;
  907             src_templ.format = PIPE_FORMAT_R8_UNORM;
  908             break;
  909          case 2:
  910             dst_templ.format = PIPE_FORMAT_R8G8_UNORM;
  911             src_templ.format = PIPE_FORMAT_R8G8_UNORM;
  912             break;
  913          case 4:
  914             dst_templ.format = PIPE_FORMAT_R8G8B8A8_UNORM;
  915             src_templ.format = PIPE_FORMAT_R8G8B8A8_UNORM;
  916             break;
  917          case 8:
  918             dst_templ.format = PIPE_FORMAT_R16G16B16A16_UINT;
  919             src_templ.format = PIPE_FORMAT_R16G16B16A16_UINT;
  920             break;
  921          case 16:
  922             dst_templ.format = PIPE_FORMAT_R32G32B32A32_UINT;
  923             src_templ.format = PIPE_FORMAT_R32G32B32A32_UINT;
  924             break;
  925          default:
  926             fprintf(stderr, "Unhandled format %s with blocksize %u\n",
  927                     util_format_short_name(src->format), blocksize);
  928             assert(0);
  929          }
  930       }
  931    }
  932 
  933    /* SNORM8 blitting has precision issues on some chips. Use the SINT
  934     * equivalent instead, which doesn't force DCC decompression.
  935     * Note that some chips avoid this issue by using SDMA.
  936     */
  937    if (util_format_is_snorm8(dst_templ.format)) {
  938       dst_templ.format = src_templ.format = util_format_snorm8_to_sint8(dst_templ.format);
  939    }
  940 
  941    vi_disable_dcc_if_incompatible_format(sctx, dst, dst_level, dst_templ.format);
  942    vi_disable_dcc_if_incompatible_format(sctx, src, src_level, src_templ.format);
  943 
  944    /* Initialize the surface. */
  945    dst_view = si_create_surface_custom(ctx, dst, &dst_templ, dst_width0, dst_height0, dst_width,
  946                                        dst_height);
  947 
  948    /* Initialize the sampler view. */
  949    src_view =
  950       si_create_sampler_view_custom(ctx, src, &src_templ, src_width0, src_height0, src_force_level);
  951 
  952    u_box_3d(dstx, dsty, dstz, abs(src_box->width), abs(src_box->height), abs(src_box->depth),
  953             &dstbox);
  954 
  955    /* Copy. */
  956    si_blitter_begin(sctx, SI_COPY);
  957    util_blitter_blit_generic(sctx->blitter, dst_view, &dstbox, src_view, src_box, src_width0,
  958                              src_height0, PIPE_MASK_RGBAZS, PIPE_TEX_FILTER_NEAREST, NULL, false);
  959    si_blitter_end(sctx);
  960 
  961    pipe_surface_reference(&dst_view, NULL);
  962    pipe_sampler_view_reference(&src_view, NULL);
  963 }
  964 
  965 static void si_do_CB_resolve(struct si_context *sctx, const struct pipe_blit_info *info,
  966                              struct pipe_resource *dst, unsigned dst_level, unsigned dst_z,
  967                              enum pipe_format format)
  968 {
  969    /* Required before and after CB_RESOLVE. */
  970    sctx->flags |= SI_CONTEXT_FLUSH_AND_INV_CB;
  971 
  972    si_blitter_begin(
  973       sctx, SI_COLOR_RESOLVE | (info->render_condition_enable ? 0 : SI_DISABLE_RENDER_COND));
  974    util_blitter_custom_resolve_color(sctx->blitter, dst, dst_level, dst_z, info->src.resource,
  975                                      info->src.box.z, ~0, sctx->custom_blend_resolve, format);
  976    si_blitter_end(sctx);
  977 
  978    /* Flush caches for possible texturing. */
  979    si_make_CB_shader_coherent(sctx, 1, false, true /* no DCC */);
  980 }
  981 
  982 static bool do_hardware_msaa_resolve(struct pipe_context *ctx, const struct pipe_blit_info *info)
  983 {
  984    struct si_context *sctx = (struct si_context *)ctx;
  985    struct si_texture *src = (struct si_texture *)info->src.resource;
  986    struct si_texture *dst = (struct si_texture *)info->dst.resource;
  987    ASSERTED struct si_texture *stmp;
  988    unsigned dst_width = u_minify(info->dst.resource->width0, info->dst.level);
  989    unsigned dst_height = u_minify(info->dst.resource->height0, info->dst.level);
  990    enum pipe_format format = info->src.format;
  991    struct pipe_resource *tmp, templ;
  992    struct pipe_blit_info blit;
  993 
  994    /* Check basic requirements for hw resolve. */
  995    if (!(info->src.resource->nr_samples > 1 && info->dst.resource->nr_samples <= 1 &&
  996          !util_format_is_pure_integer(format) && !util_format_is_depth_or_stencil(format) &&
  997          util_max_layer(info->src.resource, 0) == 0))
  998       return false;
  999 
 1000    /* Hardware MSAA resolve doesn't work if SPI format = NORM16_ABGR and
 1001     * the format is R16G16. Use R16A16, which does work.
 1002     */
 1003    if (format == PIPE_FORMAT_R16G16_UNORM)
 1004       format = PIPE_FORMAT_R16A16_UNORM;
 1005    if (format == PIPE_FORMAT_R16G16_SNORM)
 1006       format = PIPE_FORMAT_R16A16_SNORM;
 1007 
 1008    /* Check the remaining requirements for hw resolve. */
 1009    if (util_max_layer(info->dst.resource, info->dst.level) == 0 && !info->scissor_enable &&
 1010        (info->mask & PIPE_MASK_RGBA) == PIPE_MASK_RGBA &&
 1011        util_is_format_compatible(util_format_description(info->src.format),
 1012                                  util_format_description(info->dst.format)) &&
 1013        dst_width == info->src.resource->width0 && dst_height == info->src.resource->height0 &&
 1014        info->dst.box.x == 0 && info->dst.box.y == 0 && info->dst.box.width == dst_width &&
 1015        info->dst.box.height == dst_height && info->dst.box.depth == 1 && info->src.box.x == 0 &&
 1016        info->src.box.y == 0 && info->src.box.width == dst_width &&
 1017        info->src.box.height == dst_height && info->src.box.depth == 1 && !dst->surface.is_linear &&
 1018        (!dst->cmask_buffer || !dst->dirty_level_mask)) { /* dst cannot be fast-cleared */
 1019       /* Check the last constraint. */
 1020       if (src->surface.micro_tile_mode != dst->surface.micro_tile_mode) {
 1021          /* The next fast clear will switch to this mode to
 1022           * get direct hw resolve next time if the mode is
 1023           * different now.
 1024           *
 1025           * TODO-GFX10: This does not work in GFX10 because MSAA
 1026           * is restricted to 64KB_R_X and 64KB_Z_X swizzle modes.
 1027           * In some cases we could change the swizzle of the
 1028           * destination texture instead, but the more general
 1029           * solution is to implement compute shader resolve.
 1030           */
 1031          src->last_msaa_resolve_target_micro_mode = dst->surface.micro_tile_mode;
 1032          goto resolve_to_temp;
 1033       }
 1034 
 1035       /* Resolving into a surface with DCC is unsupported. Since
 1036        * it's being overwritten anyway, clear it to uncompressed.
 1037        * This is still the fastest codepath even with this clear.
 1038        */
 1039       if (vi_dcc_enabled(dst, info->dst.level)) {
 1040          if (!vi_dcc_clear_level(sctx, dst, info->dst.level, DCC_UNCOMPRESSED))
 1041             goto resolve_to_temp;
 1042 
 1043          dst->dirty_level_mask &= ~(1 << info->dst.level);
 1044       }
 1045 
 1046       /* Resolve directly from src to dst. */
 1047       si_do_CB_resolve(sctx, info, info->dst.resource, info->dst.level, info->dst.box.z, format);
 1048       return true;
 1049    }
 1050 
 1051 resolve_to_temp:
 1052    /* Shader-based resolve is VERY SLOW. Instead, resolve into
 1053     * a temporary texture and blit.
 1054     */
 1055    memset(&templ, 0, sizeof(templ));
 1056    templ.target = PIPE_TEXTURE_2D;
 1057    templ.format = info->src.resource->format;
 1058    templ.width0 = info->src.resource->width0;
 1059    templ.height0 = info->src.resource->height0;
 1060    templ.depth0 = 1;
 1061    templ.array_size = 1;
 1062    templ.usage = PIPE_USAGE_DEFAULT;
 1063    templ.flags = SI_RESOURCE_FLAG_FORCE_MSAA_TILING | SI_RESOURCE_FLAG_FORCE_MICRO_TILE_MODE |
 1064                  SI_RESOURCE_FLAG_MICRO_TILE_MODE_SET(src->surface.micro_tile_mode) |
 1065                  SI_RESOURCE_FLAG_DISABLE_DCC;
 1066 
 1067    /* The src and dst microtile modes must be the same. */
 1068    if (sctx->chip_class <= GFX8 && src->surface.micro_tile_mode == RADEON_MICRO_MODE_DISPLAY)
 1069       templ.bind = PIPE_BIND_SCANOUT;
 1070    else
 1071       templ.bind = 0;
 1072 
 1073    tmp = ctx->screen->resource_create(ctx->screen, &templ);
 1074    if (!tmp)
 1075       return false;
 1076    stmp = (struct si_texture *)tmp;
 1077 
 1078    assert(!stmp->surface.is_linear);
 1079    assert(src->surface.micro_tile_mode == stmp->surface.micro_tile_mode);
 1080 
 1081    /* resolve */
 1082    si_do_CB_resolve(sctx, info, tmp, 0, 0, format);
 1083 
 1084    /* blit */
 1085    blit = *info;
 1086    blit.src.resource = tmp;
 1087    blit.src.box.z = 0;
 1088 
 1089    si_blitter_begin(sctx, SI_BLIT | (info->render_condition_enable ? 0 : SI_DISABLE_RENDER_COND));
 1090    util_blitter_blit(sctx->blitter, &blit);
 1091    si_blitter_end(sctx);
 1092 
 1093    pipe_resource_reference(&tmp, NULL);
 1094    return true;
 1095 }
 1096 
 1097 static void si_blit(struct pipe_context *ctx, const struct pipe_blit_info *info)
 1098 {
 1099    struct si_context *sctx = (struct si_context *)ctx;
 1100    struct si_texture *dst = (struct si_texture *)info->dst.resource;
 1101 
 1102    if (do_hardware_msaa_resolve(ctx, info)) {
 1103       return;
 1104    }
 1105 
 1106    /* Using SDMA for copying to a linear texture in GTT is much faster.
 1107     * This improves DRI PRIME performance.
 1108     *
 1109     * resource_copy_region can't do this yet, because dma_copy calls it
 1110     * on failure (recursion).
 1111     */
 1112    if (dst->surface.is_linear && util_can_blit_via_copy_region(info, false)) {
 1113       sctx->dma_copy(ctx, info->dst.resource, info->dst.level, info->dst.box.x, info->dst.box.y,
 1114                      info->dst.box.z, info->src.resource, info->src.level, &info->src.box);
 1115       return;
 1116    }
 1117 
 1118    assert(util_blitter_is_blit_supported(sctx->blitter, info));
 1119 
 1120    /* The driver doesn't decompress resources automatically while
 1121     * u_blitter is rendering. */
 1122    vi_disable_dcc_if_incompatible_format(sctx, info->src.resource, info->src.level,
 1123                                          info->src.format);
 1124    vi_disable_dcc_if_incompatible_format(sctx, info->dst.resource, info->dst.level,
 1125                                          info->dst.format);
 1126    si_decompress_subresource(ctx, info->src.resource, PIPE_MASK_RGBAZS, info->src.level,
 1127                              info->src.box.z, info->src.box.z + info->src.box.depth - 1);
 1128 
 1129    if (sctx->screen->debug_flags & DBG(FORCE_SDMA) && util_try_blit_via_copy_region(ctx, info))
 1130       return;
 1131 
 1132    si_blitter_begin(sctx, SI_BLIT | (info->render_condition_enable ? 0 : SI_DISABLE_RENDER_COND));
 1133    util_blitter_blit(sctx->blitter, info);
 1134    si_blitter_end(sctx);
 1135 }
 1136 
 1137 static bool si_generate_mipmap(struct pipe_context *ctx, struct pipe_resource *tex,
 1138                                enum pipe_format format, unsigned base_level, unsigned last_level,
 1139                                unsigned first_layer, unsigned last_layer)
 1140 {
 1141    struct si_context *sctx = (struct si_context *)ctx;
 1142    struct si_texture *stex = (struct si_texture *)tex;
 1143 
 1144    if (!util_blitter_is_copy_supported(sctx->blitter, tex, tex))
 1145       return false;
 1146 
 1147    /* The driver doesn't decompress resources automatically while
 1148     * u_blitter is rendering. */
 1149    vi_disable_dcc_if_incompatible_format(sctx, tex, base_level, format);
 1150    si_decompress_subresource(ctx, tex, PIPE_MASK_RGBAZS, base_level, first_layer, last_layer);
 1151 
 1152    /* Clear dirty_level_mask for the levels that will be overwritten. */
 1153    assert(base_level < last_level);
 1154    stex->dirty_level_mask &= ~u_bit_consecutive(base_level + 1, last_level - base_level);
 1155 
 1156    sctx->generate_mipmap_for_depth = stex->is_depth;
 1157 
 1158    si_blitter_begin(sctx, SI_BLIT | SI_DISABLE_RENDER_COND);
 1159    util_blitter_generate_mipmap(sctx->blitter, tex, format, base_level, last_level, first_layer,
 1160                                 last_layer);
 1161    si_blitter_end(sctx);
 1162 
 1163    sctx->generate_mipmap_for_depth = false;
 1164    return true;
 1165 }
 1166 
 1167 static void si_flush_resource(struct pipe_context *ctx, struct pipe_resource *res)
 1168 {
 1169    struct si_context *sctx = (struct si_context *)ctx;
 1170    struct si_texture *tex = (struct si_texture *)res;
 1171 
 1172    assert(res->target != PIPE_BUFFER);
 1173    assert(!tex->dcc_separate_buffer || tex->dcc_gather_statistics);
 1174 
 1175    /* st/dri calls flush twice per frame (not a bug), this prevents double
 1176     * decompression. */
 1177    if (tex->dcc_separate_buffer && !tex->separate_dcc_dirty)
 1178       return;
 1179 
 1180    if (!tex->is_depth && (tex->cmask_buffer || tex->surface.dcc_offset)) {
 1181       si_blit_decompress_color(sctx, tex, 0, res->last_level, 0, util_max_layer(res, 0),
 1182                                tex->dcc_separate_buffer != NULL, false);
 1183 
 1184       if (tex->surface.display_dcc_offset && tex->displayable_dcc_dirty) {
 1185          si_retile_dcc(sctx, tex);
 1186          tex->displayable_dcc_dirty = false;
 1187       }
 1188    }
 1189 
 1190    /* Always do the analysis even if DCC is disabled at the moment. */
 1191    if (tex->dcc_gather_statistics) {
 1192       bool separate_dcc_dirty = tex->separate_dcc_dirty;
 1193 
 1194       /* If the color buffer hasn't been unbound and fast clear hasn't
 1195        * been used, separate_dcc_dirty is false, but there may have been
 1196        * new rendering. Check if the color buffer is bound and assume
 1197        * it's dirty.
 1198        *
 1199        * Note that DRI2 never unbinds window colorbuffers, which means
 1200        * the DCC pipeline statistics query would never be re-set and would
 1201        * keep adding new results until all free memory is exhausted if we
 1202        * didn't do this.
 1203        */
 1204       if (!separate_dcc_dirty) {
 1205          for (unsigned i = 0; i < sctx->framebuffer.state.nr_cbufs; i++) {
 1206             if (sctx->framebuffer.state.cbufs[i] &&
 1207                 sctx->framebuffer.state.cbufs[i]->texture == res) {
 1208                separate_dcc_dirty = true;
 1209                break;
 1210             }
 1211          }
 1212       }
 1213 
 1214       if (separate_dcc_dirty) {
 1215          tex->separate_dcc_dirty = false;
 1216          vi_separate_dcc_process_and_reset_stats(ctx, tex);
 1217       }
 1218    }
 1219 }
 1220 
 1221 void si_decompress_dcc(struct si_context *sctx, struct si_texture *tex)
 1222 {
 1223    /* If graphics is disabled, we can't decompress DCC, but it shouldn't
 1224     * be compressed either. The caller should simply discard it.
 1225     */
 1226    if (!tex->surface.dcc_offset || !sctx->has_graphics)
 1227       return;
 1228 
 1229    si_blit_decompress_color(sctx, tex, 0, tex->buffer.b.b.last_level, 0,
 1230                             util_max_layer(&tex->buffer.b.b, 0), true, false);
 1231 }
 1232 
 1233 void si_init_blit_functions(struct si_context *sctx)
 1234 {
 1235    sctx->b.resource_copy_region = si_resource_copy_region;
 1236 
 1237    if (sctx->has_graphics) {
 1238       sctx->b.blit = si_blit;
 1239       sctx->b.flush_resource = si_flush_resource;
 1240       sctx->b.generate_mipmap = si_generate_mipmap;
 1241    }
 1242 }