"Fossies" - the Fresh Open Source Software Archive

Member "mesa-20.1.8/src/gallium/drivers/iris/iris_blit.c" (16 Sep 2020, 30796 Bytes) of package /linux/misc/mesa-20.1.8.tar.xz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "iris_blit.c" see the Fossies "Dox" file reference documentation and the last Fossies "Diffs" side-by-side code changes report: 20.1.5_vs_20.2.0-rc1.

    1 /*
    2  * Copyright © 2017 Intel Corporation
    3  *
    4  * Permission is hereby granted, free of charge, to any person obtaining a
    5  * copy of this software and associated documentation files (the "Software"),
    6  * to deal in the Software without restriction, including without limitation
    7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
    8  * and/or sell copies of the Software, and to permit persons to whom the
    9  * Software is furnished to do so, subject to the following conditions:
   10  *
   11  * The above copyright notice and this permission notice shall be included
   12  * in all copies or substantial portions of the Software.
   13  *
   14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
   15  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
   17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
   19  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
   20  * DEALINGS IN THE SOFTWARE.
   21  */
   22 
   23 #include <stdio.h>
   24 #include "pipe/p_defines.h"
   25 #include "pipe/p_state.h"
   26 #include "pipe/p_context.h"
   27 #include "pipe/p_screen.h"
   28 #include "util/format/u_format.h"
   29 #include "util/u_inlines.h"
   30 #include "util/ralloc.h"
   31 #include "intel/blorp/blorp.h"
   32 #include "iris_context.h"
   33 #include "iris_resource.h"
   34 #include "iris_screen.h"
   35 
   36 /**
   37  * Helper function for handling mirror image blits.
   38  *
   39  * If coord0 > coord1, swap them and return "true" (mirrored).
   40  */
   41 static bool
   42 apply_mirror(float *coord0, float *coord1)
   43 {
   44    if (*coord0 > *coord1) {
   45       float tmp = *coord0;
   46       *coord0 = *coord1;
   47       *coord1 = tmp;
   48       return true;
   49    }
   50    return false;
   51 }
   52 
   53 /**
   54  * Compute the number of pixels to clip for each side of a rect
   55  *
   56  * \param x0 The rect's left coordinate
   57  * \param y0 The rect's bottom coordinate
   58  * \param x1 The rect's right coordinate
   59  * \param y1 The rect's top coordinate
   60  * \param min_x The clipping region's left coordinate
   61  * \param min_y The clipping region's bottom coordinate
   62  * \param max_x The clipping region's right coordinate
   63  * \param max_y The clipping region's top coordinate
   64  * \param clipped_x0 The number of pixels to clip from the left side
   65  * \param clipped_y0 The number of pixels to clip from the bottom side
   66  * \param clipped_x1 The number of pixels to clip from the right side
   67  * \param clipped_y1 The number of pixels to clip from the top side
   68  *
   69  * \return false if we clip everything away, true otherwise
   70  */
   71 static inline bool
   72 compute_pixels_clipped(float x0, float y0, float x1, float y1,
   73                        float min_x, float min_y, float max_x, float max_y,
   74                        float *clipped_x0, float *clipped_y0,
   75                        float *clipped_x1, float *clipped_y1)
   76 {
   77    /* If we are going to clip everything away, stop. */
   78    if (!(min_x <= max_x &&
   79          min_y <= max_y &&
   80          x0 <= max_x &&
   81          y0 <= max_y &&
   82          min_x <= x1 &&
   83          min_y <= y1 &&
   84          x0 <= x1 &&
   85          y0 <= y1)) {
   86       return false;
   87    }
   88 
   89    if (x0 < min_x)
   90       *clipped_x0 = min_x - x0;
   91    else
   92       *clipped_x0 = 0;
   93    if (max_x < x1)
   94       *clipped_x1 = x1 - max_x;
   95    else
   96       *clipped_x1 = 0;
   97 
   98    if (y0 < min_y)
   99       *clipped_y0 = min_y - y0;
  100    else
  101       *clipped_y0 = 0;
  102    if (max_y < y1)
  103       *clipped_y1 = y1 - max_y;
  104    else
  105       *clipped_y1 = 0;
  106 
  107    return true;
  108 }
  109 
  110 /**
  111  * Clips a coordinate (left, right, top or bottom) for the src or dst rect
  112  * (whichever requires the largest clip) and adjusts the coordinate
  113  * for the other rect accordingly.
  114  *
  115  * \param mirror true if mirroring is required
  116  * \param src the source rect coordinate (for example src_x0)
  117  * \param dst0 the dst rect coordinate (for example dst_x0)
  118  * \param dst1 the opposite dst rect coordinate (for example dst_x1)
  119  * \param clipped_dst0 number of pixels to clip from the dst coordinate
  120  * \param clipped_dst1 number of pixels to clip from the opposite dst coordinate
  121  * \param scale the src vs dst scale involved for that coordinate
  122  * \param is_left_or_bottom true if we are clipping the left or bottom sides
  123  *        of the rect.
  124  */
  125 static void
  126 clip_coordinates(bool mirror,
  127                  float *src, float *dst0, float *dst1,
  128                  float clipped_dst0,
  129                  float clipped_dst1,
  130                  float scale,
  131                  bool is_left_or_bottom)
  132 {
  133    /* When clipping we need to add or subtract pixels from the original
  134     * coordinates depending on whether we are acting on the left/bottom
  135     * or right/top sides of the rect respectively. We assume we have to
  136     * add them in the code below, and multiply by -1 when we should
  137     * subtract.
  138     */
  139    int mult = is_left_or_bottom ? 1 : -1;
  140 
  141    if (!mirror) {
  142       *dst0 += clipped_dst0 * mult;
  143       *src += clipped_dst0 * scale * mult;
  144    } else {
  145       *dst1 -= clipped_dst1 * mult;
  146       *src += clipped_dst1 * scale * mult;
  147    }
  148 }
  149 
  150 /**
  151  * Apply a scissor rectangle to blit coordinates.
  152  *
  153  * Returns true if the blit was entirely scissored away.
  154  */
  155 static bool
  156 apply_blit_scissor(const struct pipe_scissor_state *scissor,
  157                    float *src_x0, float *src_y0,
  158                    float *src_x1, float *src_y1,
  159                    float *dst_x0, float *dst_y0,
  160                    float *dst_x1, float *dst_y1,
  161                    bool mirror_x, bool mirror_y)
  162 {
  163    float clip_dst_x0, clip_dst_x1, clip_dst_y0, clip_dst_y1;
  164 
  165    /* Compute number of pixels to scissor away. */
  166    if (!compute_pixels_clipped(*dst_x0, *dst_y0, *dst_x1, *dst_y1,
  167                                scissor->minx, scissor->miny,
  168                                scissor->maxx, scissor->maxy,
  169                                &clip_dst_x0, &clip_dst_y0,
  170                                &clip_dst_x1, &clip_dst_y1))
  171       return true;
  172 
  173    // XXX: comments assume source clipping, which we don't do
  174 
  175    /* When clipping any of the two rects we need to adjust the coordinates
  176     * in the other rect considering the scaling factor involved.  To obtain
  177     * the best precision we want to make sure that we only clip once per
  178     * side to avoid accumulating errors due to the scaling adjustment.
  179     *
  180     * For example, if src_x0 and dst_x0 need both to be clipped we want to
  181     * avoid the situation where we clip src_x0 first, then adjust dst_x0
  182     * accordingly but then we realize that the resulting dst_x0 still needs
  183     * to be clipped, so we clip dst_x0 and adjust src_x0 again.  Because we are
  184     * applying scaling factors to adjust the coordinates in each clipping
  185     * pass we lose some precision and that can affect the results of the
  186     * blorp blit operation slightly.  What we want to do here is detect the
  187     * rect that we should clip first for each side so that when we adjust
  188     * the other rect we ensure the resulting coordinate does not need to be
  189     * clipped again.
  190     *
  191     * The code below implements this by comparing the number of pixels that
  192     * we need to clip for each side of both rects considering the scales
  193     * involved.  For example, clip_src_x0 represents the number of pixels
  194     * to be clipped for the src rect's left side, so if clip_src_x0 = 5,
  195     * clip_dst_x0 = 4 and scale_x = 2 it means that we are clipping more
  196     * from the dst rect so we should clip dst_x0 only and adjust src_x0.
  197     * This is because clipping 4 pixels in the dst is equivalent to
  198     * clipping 4 * 2 = 8 > 5 in the src.
  199     */
  200 
  201    if (*src_x0 == *src_x1 || *src_y0 == *src_y1
  202        || *dst_x0 == *dst_x1 || *dst_y0 == *dst_y1)
  203       return true;
  204 
  205    float scale_x = (float) (*src_x1 - *src_x0) / (*dst_x1 - *dst_x0);
  206    float scale_y = (float) (*src_y1 - *src_y0) / (*dst_y1 - *dst_y0);
  207 
  208    /* Clip left side */
  209    clip_coordinates(mirror_x, src_x0, dst_x0, dst_x1,
  210                     clip_dst_x0, clip_dst_x1, scale_x, true);
  211 
  212    /* Clip right side */
  213    clip_coordinates(mirror_x, src_x1, dst_x1, dst_x0,
  214                     clip_dst_x1, clip_dst_x0, scale_x, false);
  215 
  216    /* Clip bottom side */
  217    clip_coordinates(mirror_y, src_y0, dst_y0, dst_y1,
  218                     clip_dst_y0, clip_dst_y1, scale_y, true);
  219 
  220    /* Clip top side */
  221    clip_coordinates(mirror_y, src_y1, dst_y1, dst_y0,
  222                     clip_dst_y1, clip_dst_y0, scale_y, false);
  223 
  224    /* Check for invalid bounds
  225     * Can't blit for 0-dimensions
  226     */
  227    return *src_x0 == *src_x1 || *src_y0 == *src_y1
  228       || *dst_x0 == *dst_x1 || *dst_y0 == *dst_y1;
  229 }
  230 
  231 void
  232 iris_blorp_surf_for_resource(struct isl_device *isl_dev,
  233                              struct blorp_surf *surf,
  234                              struct pipe_resource *p_res,
  235                              enum isl_aux_usage aux_usage,
  236                              unsigned level,
  237                              bool is_render_target)
  238 {
  239    struct iris_resource *res = (void *) p_res;
  240 
  241    assert(!iris_resource_unfinished_aux_import(res));
  242 
  243    if (isl_aux_usage_has_hiz(aux_usage) &&
  244        !iris_resource_level_has_hiz(res, level))
  245       aux_usage = ISL_AUX_USAGE_NONE;
  246 
  247    *surf = (struct blorp_surf) {
  248       .surf = &res->surf,
  249       .addr = (struct blorp_address) {
  250          .buffer = res->bo,
  251          .offset = res->offset,
  252          .reloc_flags = is_render_target ? EXEC_OBJECT_WRITE : 0,
  253          .mocs = iris_mocs(res->bo, isl_dev),
  254       },
  255       .aux_usage = aux_usage,
  256    };
  257 
  258    if (aux_usage != ISL_AUX_USAGE_NONE) {
  259       surf->aux_surf = &res->aux.surf;
  260       surf->aux_addr = (struct blorp_address) {
  261          .buffer = res->aux.bo,
  262          .offset = res->aux.offset,
  263          .reloc_flags = is_render_target ? EXEC_OBJECT_WRITE : 0,
  264          .mocs = iris_mocs(res->bo, isl_dev),
  265       };
  266       surf->clear_color =
  267          iris_resource_get_clear_color(res, NULL, NULL);
  268       surf->clear_color_addr = (struct blorp_address) {
  269          .buffer = res->aux.clear_color_bo,
  270          .offset = res->aux.clear_color_offset,
  271          .reloc_flags = 0,
  272          .mocs = iris_mocs(res->aux.clear_color_bo, isl_dev),
  273       };
  274    }
  275 }
  276 
  277 static bool
  278 is_astc(enum isl_format format)
  279 {
  280    return format != ISL_FORMAT_UNSUPPORTED &&
  281           isl_format_get_layout(format)->txc == ISL_TXC_ASTC;
  282 }
  283 
  284 static void
  285 tex_cache_flush_hack(struct iris_batch *batch,
  286                      enum isl_format view_format,
  287                      enum isl_format surf_format)
  288 {
  289    const struct gen_device_info *devinfo = &batch->screen->devinfo;
  290 
  291    /* The WaSamplerCacheFlushBetweenRedescribedSurfaceReads workaround says:
  292     *
  293     *    "Currently Sampler assumes that a surface would not have two
  294     *     different format associate with it.  It will not properly cache
  295     *     the different views in the MT cache, causing a data corruption."
  296     *
  297     * We may need to handle this for texture views in general someday, but
  298     * for now we handle it here, as it hurts copies and blits particularly
  299     * badly because they ofter reinterpret formats.
  300     *
  301     * If the BO hasn't been referenced yet this batch, we assume that the
  302     * texture cache doesn't contain any relevant data nor need flushing.
  303     *
  304     * Icelake (Gen11+) claims to fix this issue, but seems to still have
  305     * issues with ASTC formats.
  306     */
  307    bool need_flush = devinfo->gen >= 11 ?
  308                      is_astc(surf_format) != is_astc(view_format) :
  309                      view_format != surf_format;
  310    if (!need_flush)
  311       return;
  312 
  313    const char *reason =
  314       "workaround: WaSamplerCacheFlushBetweenRedescribedSurfaceReads";
  315 
  316    iris_emit_pipe_control_flush(batch, reason, PIPE_CONTROL_CS_STALL);
  317    iris_emit_pipe_control_flush(batch, reason,
  318                                 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
  319 }
  320 
  321 static enum isl_aux_usage
  322 iris_resource_blorp_write_aux_usage(struct iris_context *ice,
  323                                     struct iris_resource *res,
  324                                     enum isl_format render_format)
  325 {
  326    if (res->surf.usage & (ISL_SURF_USAGE_DEPTH_BIT |
  327                           ISL_SURF_USAGE_STENCIL_BIT)) {
  328       assert(render_format == res->surf.format);
  329       return res->aux.usage;
  330    } else {
  331       return iris_resource_render_aux_usage(ice, res, render_format,
  332                                             false, false);
  333    }
  334 }
  335 
  336 /**
  337  * The pipe->blit() driver hook.
  338  *
  339  * This performs a blit between two surfaces, which copies data but may
  340  * also perform format conversion, scaling, flipping, and so on.
  341  */
  342 static void
  343 iris_blit(struct pipe_context *ctx, const struct pipe_blit_info *info)
  344 {
  345    struct iris_context *ice = (void *) ctx;
  346    struct iris_screen *screen = (struct iris_screen *)ctx->screen;
  347    const struct gen_device_info *devinfo = &screen->devinfo;
  348    struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER];
  349    enum blorp_batch_flags blorp_flags = 0;
  350    struct iris_resource *src_res = (void *) info->src.resource;
  351    struct iris_resource *dst_res = (void *) info->dst.resource;
  352 
  353    /* We don't support color masking. */
  354    assert((info->mask & PIPE_MASK_RGBA) == PIPE_MASK_RGBA ||
  355           (info->mask & PIPE_MASK_RGBA) == 0);
  356 
  357    if (info->render_condition_enable) {
  358       if (ice->state.predicate == IRIS_PREDICATE_STATE_DONT_RENDER)
  359          return;
  360 
  361       if (ice->state.predicate == IRIS_PREDICATE_STATE_USE_BIT)
  362          blorp_flags |= BLORP_BATCH_PREDICATE_ENABLE;
  363    }
  364 
  365    if (iris_resource_unfinished_aux_import(src_res))
  366       iris_resource_finish_aux_import(ctx->screen, src_res);
  367    if (iris_resource_unfinished_aux_import(dst_res))
  368       iris_resource_finish_aux_import(ctx->screen, dst_res);
  369 
  370    struct iris_format_info src_fmt =
  371       iris_format_for_usage(devinfo, info->src.format,
  372                             ISL_SURF_USAGE_TEXTURE_BIT);
  373    enum isl_aux_usage src_aux_usage =
  374       iris_resource_texture_aux_usage(ice, src_res, src_fmt.fmt);
  375 
  376    if (iris_resource_level_has_hiz(src_res, info->src.level))
  377       assert(src_res->surf.format == src_fmt.fmt);
  378 
  379    bool src_clear_supported = isl_aux_usage_has_fast_clears(src_aux_usage) &&
  380                               src_res->surf.format == src_fmt.fmt;
  381 
  382    iris_resource_prepare_access(ice, batch, src_res, info->src.level, 1,
  383                                 info->src.box.z, info->src.box.depth,
  384                                 src_aux_usage, src_clear_supported);
  385 
  386    struct iris_format_info dst_fmt =
  387       iris_format_for_usage(devinfo, info->dst.format,
  388                             ISL_SURF_USAGE_RENDER_TARGET_BIT);
  389    enum isl_aux_usage dst_aux_usage =
  390       iris_resource_blorp_write_aux_usage(ice, dst_res, dst_fmt.fmt);
  391    bool dst_clear_supported = isl_aux_usage_has_fast_clears(dst_aux_usage);
  392 
  393    struct blorp_surf src_surf, dst_surf;
  394    iris_blorp_surf_for_resource(&screen->isl_dev,  &src_surf,
  395                                 info->src.resource, src_aux_usage,
  396                                 info->src.level, false);
  397    iris_blorp_surf_for_resource(&screen->isl_dev, &dst_surf,
  398                                 info->dst.resource, dst_aux_usage,
  399                                 info->dst.level, true);
  400 
  401    iris_resource_prepare_access(ice, batch, dst_res, info->dst.level, 1,
  402                                 info->dst.box.z, info->dst.box.depth,
  403                                 dst_aux_usage, dst_clear_supported);
  404 
  405    float src_x0 = info->src.box.x;
  406    float src_x1 = info->src.box.x + info->src.box.width;
  407    float src_y0 = info->src.box.y;
  408    float src_y1 = info->src.box.y + info->src.box.height;
  409    float dst_x0 = info->dst.box.x;
  410    float dst_x1 = info->dst.box.x + info->dst.box.width;
  411    float dst_y0 = info->dst.box.y;
  412    float dst_y1 = info->dst.box.y + info->dst.box.height;
  413    bool mirror_x = apply_mirror(&src_x0, &src_x1);
  414    bool mirror_y = apply_mirror(&src_y0, &src_y1);
  415    enum blorp_filter filter;
  416 
  417    if (info->scissor_enable) {
  418       bool noop = apply_blit_scissor(&info->scissor,
  419                                      &src_x0, &src_y0, &src_x1, &src_y1,
  420                                      &dst_x0, &dst_y0, &dst_x1, &dst_y1,
  421                                      mirror_x, mirror_y);
  422       if (noop)
  423          return;
  424    }
  425 
  426    if (abs(info->dst.box.width) == abs(info->src.box.width) &&
  427        abs(info->dst.box.height) == abs(info->src.box.height)) {
  428       if (src_surf.surf->samples > 1 && dst_surf.surf->samples <= 1) {
  429          /* The OpenGL ES 3.2 specification, section 16.2.1, says:
  430           *
  431           *    "If the read framebuffer is multisampled (its effective
  432           *     value of SAMPLE_BUFFERS is one) and the draw framebuffer
  433           *     is not (its value of SAMPLE_BUFFERS is zero), the samples
  434           *     corresponding to each pixel location in the source are
  435           *     converted to a single sample before being written to the
  436           *     destination.  The filter parameter is ignored.  If the
  437           *     source formats are integer types or stencil values, a
  438           *     single sample’s value is selected for each pixel.  If the
  439           *     source formats are floating-point or normalized types,
  440           *     the sample values for each pixel are resolved in an
  441           *     implementation-dependent manner.  If the source formats
  442           *     are depth values, sample values are resolved in an
  443           *     implementation-dependent manner where the result will be
  444           *     between the minimum and maximum depth values in the pixel."
  445           *
  446           * When selecting a single sample, we always choose sample 0.
  447           */
  448          if (util_format_is_depth_or_stencil(info->src.format) ||
  449              util_format_is_pure_integer(info->src.format)) {
  450             filter = BLORP_FILTER_SAMPLE_0;
  451          } else {
  452             filter = BLORP_FILTER_AVERAGE;
  453          }
  454       } else {
  455          /* The OpenGL 4.6 specification, section 18.3.1, says:
  456           *
  457           *    "If the source and destination dimensions are identical,
  458           *     no filtering is applied."
  459           *
  460           * Using BLORP_FILTER_NONE will also handle the upsample case by
  461           * replicating the one value in the source to all values in the
  462           * destination.
  463           */
  464          filter = BLORP_FILTER_NONE;
  465       }
  466    } else if (info->filter == PIPE_TEX_FILTER_LINEAR) {
  467       filter = BLORP_FILTER_BILINEAR;
  468    } else {
  469       filter = BLORP_FILTER_NEAREST;
  470    }
  471 
  472    if (iris_batch_references(batch, src_res->bo))
  473       tex_cache_flush_hack(batch, src_fmt.fmt, src_res->surf.format);
  474 
  475    if (dst_res->base.target == PIPE_BUFFER)
  476       util_range_add(&dst_res->base, &dst_res->valid_buffer_range, dst_x0, dst_x1);
  477 
  478    struct blorp_batch blorp_batch;
  479    blorp_batch_init(&ice->blorp, &blorp_batch, batch, blorp_flags);
  480 
  481    unsigned main_mask;
  482    if (util_format_is_depth_or_stencil(info->dst.format))
  483       main_mask = PIPE_MASK_Z;
  484    else
  485       main_mask = PIPE_MASK_RGBA;
  486 
  487    if (info->mask & main_mask) {
  488       for (int slice = 0; slice < info->dst.box.depth; slice++) {
  489          iris_batch_maybe_flush(batch, 1500);
  490 
  491          blorp_blit(&blorp_batch,
  492                     &src_surf, info->src.level, info->src.box.z + slice,
  493                     src_fmt.fmt, src_fmt.swizzle,
  494                     &dst_surf, info->dst.level, info->dst.box.z + slice,
  495                     dst_fmt.fmt, dst_fmt.swizzle,
  496                     src_x0, src_y0, src_x1, src_y1,
  497                     dst_x0, dst_y0, dst_x1, dst_y1,
  498                     filter, mirror_x, mirror_y);
  499       }
  500    }
  501 
  502    struct iris_resource *stc_dst = NULL;
  503    enum isl_aux_usage stc_src_aux_usage, stc_dst_aux_usage;
  504    if ((info->mask & PIPE_MASK_S) &&
  505        util_format_has_stencil(util_format_description(info->dst.format)) &&
  506        util_format_has_stencil(util_format_description(info->src.format))) {
  507       struct iris_resource *src_res, *junk;
  508       struct blorp_surf src_surf, dst_surf;
  509       iris_get_depth_stencil_resources(info->src.resource, &junk, &src_res);
  510       iris_get_depth_stencil_resources(info->dst.resource, &junk, &stc_dst);
  511 
  512       struct iris_format_info src_fmt =
  513          iris_format_for_usage(devinfo, src_res->base.format,
  514                                ISL_SURF_USAGE_TEXTURE_BIT);
  515       stc_src_aux_usage =
  516          iris_resource_texture_aux_usage(ice, src_res, src_fmt.fmt);
  517 
  518       struct iris_format_info dst_fmt =
  519          iris_format_for_usage(devinfo, stc_dst->base.format,
  520                                ISL_SURF_USAGE_RENDER_TARGET_BIT);
  521       stc_dst_aux_usage =
  522          iris_resource_blorp_write_aux_usage(ice, stc_dst, dst_fmt.fmt);
  523 
  524       iris_resource_prepare_access(ice, batch, src_res, info->src.level, 1,
  525                                    info->src.box.z, info->src.box.depth,
  526                                    stc_src_aux_usage, false);
  527       iris_resource_prepare_access(ice, batch, stc_dst, info->dst.level, 1,
  528                                    info->dst.box.z, info->dst.box.depth,
  529                                    stc_dst_aux_usage, false);
  530       iris_blorp_surf_for_resource(&screen->isl_dev, &src_surf,
  531                                    &src_res->base, stc_src_aux_usage,
  532                                    info->src.level, false);
  533       iris_blorp_surf_for_resource(&screen->isl_dev, &dst_surf,
  534                                    &stc_dst->base, stc_dst_aux_usage,
  535                                    info->dst.level, true);
  536 
  537       for (int slice = 0; slice < info->dst.box.depth; slice++) {
  538          iris_batch_maybe_flush(batch, 1500);
  539 
  540          blorp_blit(&blorp_batch,
  541                     &src_surf, info->src.level, info->src.box.z + slice,
  542                     ISL_FORMAT_R8_UINT, ISL_SWIZZLE_IDENTITY,
  543                     &dst_surf, info->dst.level, info->dst.box.z + slice,
  544                     ISL_FORMAT_R8_UINT, ISL_SWIZZLE_IDENTITY,
  545                     src_x0, src_y0, src_x1, src_y1,
  546                     dst_x0, dst_y0, dst_x1, dst_y1,
  547                     filter, mirror_x, mirror_y);
  548       }
  549    }
  550 
  551    blorp_batch_finish(&blorp_batch);
  552 
  553    tex_cache_flush_hack(batch, src_fmt.fmt, src_res->surf.format);
  554 
  555    if (info->mask & main_mask) {
  556       iris_resource_finish_write(ice, dst_res, info->dst.level, info->dst.box.z,
  557                                  info->dst.box.depth, dst_aux_usage);
  558    }
  559 
  560    if (stc_dst) {
  561       iris_resource_finish_write(ice, stc_dst, info->dst.level, info->dst.box.z,
  562                                  info->dst.box.depth, stc_dst_aux_usage);
  563    }
  564 
  565    iris_flush_and_dirty_for_history(ice, batch, (struct iris_resource *)
  566                                     info->dst.resource,
  567                                     PIPE_CONTROL_RENDER_TARGET_FLUSH,
  568                                     "cache history: post-blit");
  569 }
  570 
  571 static void
  572 get_copy_region_aux_settings(struct iris_context *ice,
  573                              struct iris_resource *res,
  574                              enum isl_aux_usage *out_aux_usage,
  575                              bool *out_clear_supported,
  576                              bool is_render_target)
  577 {
  578    struct iris_screen *screen = (void *) ice->ctx.screen;
  579    const struct gen_device_info *devinfo = &screen->devinfo;
  580 
  581    switch (res->aux.usage) {
  582    case ISL_AUX_USAGE_HIZ:
  583    case ISL_AUX_USAGE_HIZ_CCS:
  584    case ISL_AUX_USAGE_HIZ_CCS_WT:
  585       if (is_render_target) {
  586          *out_aux_usage = res->aux.usage;
  587       } else {
  588          *out_aux_usage = iris_resource_texture_aux_usage(ice, res,
  589                                                           res->surf.format);
  590       }
  591       *out_clear_supported = (*out_aux_usage != ISL_AUX_USAGE_NONE);
  592       break;
  593    case ISL_AUX_USAGE_MCS:
  594    case ISL_AUX_USAGE_MCS_CCS:
  595    case ISL_AUX_USAGE_CCS_E:
  596       *out_aux_usage = res->aux.usage;
  597       /* Prior to Gen9, fast-clear only supported 0/1 clear colors.  Since
  598        * we're going to re-interpret the format as an integer format possibly
  599        * with a different number of components, we can't handle clear colors
  600        * until Gen9.
  601        */
  602       *out_clear_supported = devinfo->gen >= 9;
  603       break;
  604    case ISL_AUX_USAGE_STC_CCS:
  605       *out_aux_usage = res->aux.usage;
  606       *out_clear_supported = false;
  607       break;
  608    default:
  609       *out_aux_usage = ISL_AUX_USAGE_NONE;
  610       *out_clear_supported = false;
  611       break;
  612    }
  613 }
  614 
  615 /**
  616  * Perform a GPU-based raw memory copy between compatible view classes.
  617  *
  618  * Does not perform any flushing - the new data may still be left in the
  619  * render cache, and old data may remain in other caches.
  620  *
  621  * Wraps blorp_copy() and blorp_buffer_copy().
  622  */
  623 void
  624 iris_copy_region(struct blorp_context *blorp,
  625                  struct iris_batch *batch,
  626                  struct pipe_resource *dst,
  627                  unsigned dst_level,
  628                  unsigned dstx, unsigned dsty, unsigned dstz,
  629                  struct pipe_resource *src,
  630                  unsigned src_level,
  631                  const struct pipe_box *src_box)
  632 {
  633    struct blorp_batch blorp_batch;
  634    struct iris_context *ice = blorp->driver_ctx;
  635    struct iris_screen *screen = (void *) ice->ctx.screen;
  636    struct iris_resource *src_res = (void *) src;
  637    struct iris_resource *dst_res = (void *) dst;
  638 
  639    enum isl_aux_usage src_aux_usage, dst_aux_usage;
  640    bool src_clear_supported, dst_clear_supported;
  641    get_copy_region_aux_settings(ice, src_res, &src_aux_usage,
  642                                 &src_clear_supported, false);
  643    get_copy_region_aux_settings(ice, dst_res, &dst_aux_usage,
  644                                 &dst_clear_supported, true);
  645 
  646    if (iris_batch_references(batch, src_res->bo))
  647       tex_cache_flush_hack(batch, ISL_FORMAT_UNSUPPORTED, src_res->surf.format);
  648 
  649    if (dst->target == PIPE_BUFFER)
  650       util_range_add(&dst_res->base, &dst_res->valid_buffer_range, dstx, dstx + src_box->width);
  651 
  652    if (dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) {
  653       struct blorp_address src_addr = {
  654          .buffer = iris_resource_bo(src), .offset = src_box->x,
  655       };
  656       struct blorp_address dst_addr = {
  657          .buffer = iris_resource_bo(dst), .offset = dstx,
  658          .reloc_flags = EXEC_OBJECT_WRITE,
  659       };
  660 
  661       iris_batch_maybe_flush(batch, 1500);
  662 
  663       blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0);
  664       blorp_buffer_copy(&blorp_batch, src_addr, dst_addr, src_box->width);
  665       blorp_batch_finish(&blorp_batch);
  666    } else {
  667       // XXX: what about one surface being a buffer and not the other?
  668 
  669       struct blorp_surf src_surf, dst_surf;
  670       iris_blorp_surf_for_resource(&screen->isl_dev, &src_surf,
  671                                    src, src_aux_usage, src_level, false);
  672       iris_blorp_surf_for_resource(&screen->isl_dev, &dst_surf,
  673                                    dst, dst_aux_usage, dst_level, true);
  674 
  675       iris_resource_prepare_access(ice, batch, src_res, src_level, 1,
  676                                    src_box->z, src_box->depth,
  677                                    src_aux_usage, src_clear_supported);
  678       iris_resource_prepare_access(ice, batch, dst_res, dst_level, 1,
  679                                    dstz, src_box->depth,
  680                                    dst_aux_usage, dst_clear_supported);
  681 
  682       blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0);
  683 
  684       for (int slice = 0; slice < src_box->depth; slice++) {
  685          iris_batch_maybe_flush(batch, 1500);
  686 
  687          blorp_copy(&blorp_batch, &src_surf, src_level, src_box->z + slice,
  688                     &dst_surf, dst_level, dstz + slice,
  689                     src_box->x, src_box->y, dstx, dsty,
  690                     src_box->width, src_box->height);
  691       }
  692       blorp_batch_finish(&blorp_batch);
  693 
  694       iris_resource_finish_write(ice, dst_res, dst_level, dstz,
  695                                  src_box->depth, dst_aux_usage);
  696    }
  697 
  698    tex_cache_flush_hack(batch, ISL_FORMAT_UNSUPPORTED, src_res->surf.format);
  699 }
  700 
  701 static struct iris_batch *
  702 get_preferred_batch(struct iris_context *ice, struct iris_bo *bo)
  703 {
  704    /* If the compute batch is already using this buffer, we'd prefer to
  705     * continue queueing in the compute batch.
  706     */
  707    if (iris_batch_references(&ice->batches[IRIS_BATCH_COMPUTE], bo))
  708       return &ice->batches[IRIS_BATCH_COMPUTE];
  709 
  710    /* Otherwise default to the render batch. */
  711    return &ice->batches[IRIS_BATCH_RENDER];
  712 }
  713 
  714 
  715 /**
  716  * The pipe->resource_copy_region() driver hook.
  717  *
  718  * This implements ARB_copy_image semantics - a raw memory copy between
  719  * compatible view classes.
  720  */
  721 static void
  722 iris_resource_copy_region(struct pipe_context *ctx,
  723                           struct pipe_resource *p_dst,
  724                           unsigned dst_level,
  725                           unsigned dstx, unsigned dsty, unsigned dstz,
  726                           struct pipe_resource *p_src,
  727                           unsigned src_level,
  728                           const struct pipe_box *src_box)
  729 {
  730    struct iris_context *ice = (void *) ctx;
  731    struct iris_screen *screen = (void *) ctx->screen;
  732    struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER];
  733    struct iris_resource *src = (void *) p_src;
  734    struct iris_resource *dst = (void *) p_dst;
  735 
  736    if (iris_resource_unfinished_aux_import(src))
  737       iris_resource_finish_aux_import(ctx->screen, src);
  738    if (iris_resource_unfinished_aux_import(dst))
  739       iris_resource_finish_aux_import(ctx->screen, dst);
  740 
  741    /* Use MI_COPY_MEM_MEM for tiny (<= 16 byte, % 4) buffer copies. */
  742    if (p_src->target == PIPE_BUFFER && p_dst->target == PIPE_BUFFER &&
  743        (src_box->width % 4 == 0) && src_box->width <= 16) {
  744       struct iris_bo *dst_bo = iris_resource_bo(p_dst);
  745       batch = get_preferred_batch(ice, dst_bo);
  746       iris_batch_maybe_flush(batch, 24 + 5 * (src_box->width / 4));
  747       iris_emit_pipe_control_flush(batch,
  748                                    "stall for MI_COPY_MEM_MEM copy_region",
  749                                    PIPE_CONTROL_CS_STALL);
  750       screen->vtbl.copy_mem_mem(batch, dst_bo, dstx, iris_resource_bo(p_src),
  751                                 src_box->x, src_box->width);
  752       return;
  753    }
  754 
  755    iris_copy_region(&ice->blorp, batch, p_dst, dst_level, dstx, dsty, dstz,
  756                     p_src, src_level, src_box);
  757 
  758    if (util_format_is_depth_and_stencil(p_dst->format) &&
  759        util_format_has_stencil(util_format_description(p_src->format))) {
  760       struct iris_resource *junk, *s_src_res, *s_dst_res;
  761       iris_get_depth_stencil_resources(p_src, &junk, &s_src_res);
  762       iris_get_depth_stencil_resources(p_dst, &junk, &s_dst_res);
  763 
  764       iris_copy_region(&ice->blorp, batch, &s_dst_res->base, dst_level, dstx,
  765                        dsty, dstz, &s_src_res->base, src_level, src_box);
  766    }
  767 
  768    iris_flush_and_dirty_for_history(ice, batch, dst,
  769                                     PIPE_CONTROL_RENDER_TARGET_FLUSH,
  770                                     "cache history: post copy_region");
  771 }
  772 
  773 void
  774 iris_init_blit_functions(struct pipe_context *ctx)
  775 {
  776    ctx->blit = iris_blit;
  777    ctx->resource_copy_region = iris_resource_copy_region;
  778 }