"Fossies" - the Fresh Open Source Software Archive

Member "mesa-20.1.8/src/gallium/drivers/iris/iris_resource.c" (16 Sep 2020, 72349 Bytes) of package /linux/misc/mesa-20.1.8.tar.xz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "iris_resource.c" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 20.1.7_vs_20.1.8.

    1 /*
    2  * Copyright © 2017 Intel Corporation
    3  *
    4  * Permission is hereby granted, free of charge, to any person obtaining a
    5  * copy of this software and associated documentation files (the "Software"),
    6  * to deal in the Software without restriction, including without limitation
    7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
    8  * and/or sell copies of the Software, and to permit persons to whom the
    9  * Software is furnished to do so, subject to the following conditions:
   10  *
   11  * The above copyright notice and this permission notice shall be included
   12  * in all copies or substantial portions of the Software.
   13  *
   14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
   15  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
   17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
   19  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
   20  * DEALINGS IN THE SOFTWARE.
   21  */
   22 
   23 /**
   24  * @file iris_resource.c
   25  *
   26  * Resources are images, buffers, and other objects used by the GPU.
   27  *
   28  * XXX: explain resources
   29  */
   30 
   31 #include <stdio.h>
   32 #include <errno.h>
   33 #include "pipe/p_defines.h"
   34 #include "pipe/p_state.h"
   35 #include "pipe/p_context.h"
   36 #include "pipe/p_screen.h"
   37 #include "util/os_memory.h"
   38 #include "util/u_cpu_detect.h"
   39 #include "util/u_inlines.h"
   40 #include "util/format/u_format.h"
   41 #include "util/u_threaded_context.h"
   42 #include "util/u_transfer.h"
   43 #include "util/u_transfer_helper.h"
   44 #include "util/u_upload_mgr.h"
   45 #include "util/ralloc.h"
   46 #include "iris_batch.h"
   47 #include "iris_context.h"
   48 #include "iris_resource.h"
   49 #include "iris_screen.h"
   50 #include "intel/common/gen_aux_map.h"
   51 #include "intel/dev/gen_debug.h"
   52 #include "isl/isl.h"
   53 #include "drm-uapi/drm_fourcc.h"
   54 #include "drm-uapi/i915_drm.h"
   55 
   56 enum modifier_priority {
   57    MODIFIER_PRIORITY_INVALID = 0,
   58    MODIFIER_PRIORITY_LINEAR,
   59    MODIFIER_PRIORITY_X,
   60    MODIFIER_PRIORITY_Y,
   61    MODIFIER_PRIORITY_Y_CCS,
   62 };
   63 
   64 static const uint64_t priority_to_modifier[] = {
   65    [MODIFIER_PRIORITY_INVALID] = DRM_FORMAT_MOD_INVALID,
   66    [MODIFIER_PRIORITY_LINEAR] = DRM_FORMAT_MOD_LINEAR,
   67    [MODIFIER_PRIORITY_X] = I915_FORMAT_MOD_X_TILED,
   68    [MODIFIER_PRIORITY_Y] = I915_FORMAT_MOD_Y_TILED,
   69    [MODIFIER_PRIORITY_Y_CCS] = I915_FORMAT_MOD_Y_TILED_CCS,
   70 };
   71 
   72 static bool
   73 modifier_is_supported(const struct gen_device_info *devinfo,
   74                       enum pipe_format pfmt, uint64_t modifier)
   75 {
   76    /* XXX: do something real */
   77    switch (modifier) {
   78    case I915_FORMAT_MOD_Y_TILED_CCS: {
   79       if (unlikely(INTEL_DEBUG & DEBUG_NO_RBC))
   80          return false;
   81 
   82       enum isl_format rt_format =
   83          iris_format_for_usage(devinfo, pfmt,
   84                                ISL_SURF_USAGE_RENDER_TARGET_BIT).fmt;
   85 
   86       enum isl_format linear_format = isl_format_srgb_to_linear(rt_format);
   87 
   88       if (linear_format == ISL_FORMAT_UNSUPPORTED ||
   89           !isl_format_supports_ccs_e(devinfo, linear_format))
   90          return false;
   91 
   92       return devinfo->gen >= 9 && devinfo->gen <= 11;
   93    }
   94    case I915_FORMAT_MOD_Y_TILED:
   95    case I915_FORMAT_MOD_X_TILED:
   96    case DRM_FORMAT_MOD_LINEAR:
   97       return true;
   98    case DRM_FORMAT_MOD_INVALID:
   99    default:
  100       return false;
  101    }
  102 }
  103 
  104 static uint64_t
  105 select_best_modifier(struct gen_device_info *devinfo, enum pipe_format pfmt,
  106                      const uint64_t *modifiers,
  107                      int count)
  108 {
  109    enum modifier_priority prio = MODIFIER_PRIORITY_INVALID;
  110 
  111    for (int i = 0; i < count; i++) {
  112       if (!modifier_is_supported(devinfo, pfmt, modifiers[i]))
  113          continue;
  114 
  115       switch (modifiers[i]) {
  116       case I915_FORMAT_MOD_Y_TILED_CCS:
  117          prio = MAX2(prio, MODIFIER_PRIORITY_Y_CCS);
  118          break;
  119       case I915_FORMAT_MOD_Y_TILED:
  120          prio = MAX2(prio, MODIFIER_PRIORITY_Y);
  121          break;
  122       case I915_FORMAT_MOD_X_TILED:
  123          prio = MAX2(prio, MODIFIER_PRIORITY_X);
  124          break;
  125       case DRM_FORMAT_MOD_LINEAR:
  126          prio = MAX2(prio, MODIFIER_PRIORITY_LINEAR);
  127          break;
  128       case DRM_FORMAT_MOD_INVALID:
  129       default:
  130          break;
  131       }
  132    }
  133 
  134    return priority_to_modifier[prio];
  135 }
  136 
  137 enum isl_surf_dim
  138 target_to_isl_surf_dim(enum pipe_texture_target target)
  139 {
  140    switch (target) {
  141    case PIPE_BUFFER:
  142    case PIPE_TEXTURE_1D:
  143    case PIPE_TEXTURE_1D_ARRAY:
  144       return ISL_SURF_DIM_1D;
  145    case PIPE_TEXTURE_2D:
  146    case PIPE_TEXTURE_CUBE:
  147    case PIPE_TEXTURE_RECT:
  148    case PIPE_TEXTURE_2D_ARRAY:
  149    case PIPE_TEXTURE_CUBE_ARRAY:
  150       return ISL_SURF_DIM_2D;
  151    case PIPE_TEXTURE_3D:
  152       return ISL_SURF_DIM_3D;
  153    case PIPE_MAX_TEXTURE_TYPES:
  154       break;
  155    }
  156    unreachable("invalid texture type");
  157 }
  158 
  159 static void
  160 iris_query_dmabuf_modifiers(struct pipe_screen *pscreen,
  161                             enum pipe_format pfmt,
  162                             int max,
  163                             uint64_t *modifiers,
  164                             unsigned int *external_only,
  165                             int *count)
  166 {
  167    struct iris_screen *screen = (void *) pscreen;
  168    const struct gen_device_info *devinfo = &screen->devinfo;
  169 
  170    uint64_t all_modifiers[] = {
  171       DRM_FORMAT_MOD_LINEAR,
  172       I915_FORMAT_MOD_X_TILED,
  173       I915_FORMAT_MOD_Y_TILED,
  174       I915_FORMAT_MOD_Y_TILED_CCS,
  175    };
  176 
  177    int supported_mods = 0;
  178 
  179    for (int i = 0; i < ARRAY_SIZE(all_modifiers); i++) {
  180       if (!modifier_is_supported(devinfo, pfmt, all_modifiers[i]))
  181          continue;
  182 
  183       if (supported_mods < max) {
  184          if (modifiers)
  185             modifiers[supported_mods] = all_modifiers[i];
  186 
  187          if (external_only)
  188             external_only[supported_mods] = util_format_is_yuv(pfmt);
  189       }
  190 
  191       supported_mods++;
  192    }
  193 
  194    *count = supported_mods;
  195 }
  196 
  197 static isl_surf_usage_flags_t
  198 pipe_bind_to_isl_usage(unsigned bindings)
  199 {
  200    isl_surf_usage_flags_t usage = 0;
  201 
  202    if (bindings & PIPE_BIND_RENDER_TARGET)
  203       usage |= ISL_SURF_USAGE_RENDER_TARGET_BIT;
  204 
  205    if (bindings & PIPE_BIND_SAMPLER_VIEW)
  206       usage |= ISL_SURF_USAGE_TEXTURE_BIT;
  207 
  208    if (bindings & (PIPE_BIND_SHADER_IMAGE | PIPE_BIND_SHADER_BUFFER))
  209       usage |= ISL_SURF_USAGE_STORAGE_BIT;
  210 
  211    if (bindings & PIPE_BIND_DISPLAY_TARGET)
  212       usage |= ISL_SURF_USAGE_DISPLAY_BIT;
  213 
  214    return usage;
  215 }
  216 
  217 enum isl_format
  218 iris_image_view_get_format(struct iris_context *ice,
  219                            const struct pipe_image_view *img)
  220 {
  221    struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
  222    const struct gen_device_info *devinfo = &screen->devinfo;
  223 
  224    isl_surf_usage_flags_t usage = ISL_SURF_USAGE_STORAGE_BIT;
  225    enum isl_format isl_fmt =
  226       iris_format_for_usage(devinfo, img->format, usage).fmt;
  227 
  228    if (img->shader_access & PIPE_IMAGE_ACCESS_READ) {
  229       /* On Gen8, try to use typed surfaces reads (which support a
  230        * limited number of formats), and if not possible, fall back
  231        * to untyped reads.
  232        */
  233       if (devinfo->gen == 8 &&
  234           !isl_has_matching_typed_storage_image_format(devinfo, isl_fmt))
  235          return ISL_FORMAT_RAW;
  236       else
  237          return isl_lower_storage_image_format(devinfo, isl_fmt);
  238    }
  239 
  240    return isl_fmt;
  241 }
  242 
  243 struct pipe_resource *
  244 iris_resource_get_separate_stencil(struct pipe_resource *p_res)
  245 {
  246    /* For packed depth-stencil, we treat depth as the primary resource
  247     * and store S8 as the "second plane" resource.
  248     */
  249    if (p_res->next && p_res->next->format == PIPE_FORMAT_S8_UINT)
  250       return p_res->next;
  251 
  252    return NULL;
  253 
  254 }
  255 
  256 static void
  257 iris_resource_set_separate_stencil(struct pipe_resource *p_res,
  258                                    struct pipe_resource *stencil)
  259 {
  260    assert(util_format_has_depth(util_format_description(p_res->format)));
  261    pipe_resource_reference(&p_res->next, stencil);
  262 }
  263 
  264 void
  265 iris_get_depth_stencil_resources(struct pipe_resource *res,
  266                                  struct iris_resource **out_z,
  267                                  struct iris_resource **out_s)
  268 {
  269    if (!res) {
  270       *out_z = NULL;
  271       *out_s = NULL;
  272       return;
  273    }
  274 
  275    if (res->format != PIPE_FORMAT_S8_UINT) {
  276       *out_z = (void *) res;
  277       *out_s = (void *) iris_resource_get_separate_stencil(res);
  278    } else {
  279       *out_z = NULL;
  280       *out_s = (void *) res;
  281    }
  282 }
  283 
  284 enum isl_dim_layout
  285 iris_get_isl_dim_layout(const struct gen_device_info *devinfo,
  286                         enum isl_tiling tiling,
  287                         enum pipe_texture_target target)
  288 {
  289    switch (target) {
  290    case PIPE_TEXTURE_1D:
  291    case PIPE_TEXTURE_1D_ARRAY:
  292       return (devinfo->gen >= 9 && tiling == ISL_TILING_LINEAR ?
  293               ISL_DIM_LAYOUT_GEN9_1D : ISL_DIM_LAYOUT_GEN4_2D);
  294 
  295    case PIPE_TEXTURE_2D:
  296    case PIPE_TEXTURE_2D_ARRAY:
  297    case PIPE_TEXTURE_RECT:
  298    case PIPE_TEXTURE_CUBE:
  299    case PIPE_TEXTURE_CUBE_ARRAY:
  300       return ISL_DIM_LAYOUT_GEN4_2D;
  301 
  302    case PIPE_TEXTURE_3D:
  303       return (devinfo->gen >= 9 ?
  304               ISL_DIM_LAYOUT_GEN4_2D : ISL_DIM_LAYOUT_GEN4_3D);
  305 
  306    case PIPE_MAX_TEXTURE_TYPES:
  307    case PIPE_BUFFER:
  308       break;
  309    }
  310    unreachable("invalid texture type");
  311 }
  312 
  313 void
  314 iris_resource_disable_aux(struct iris_resource *res)
  315 {
  316    iris_bo_unreference(res->aux.bo);
  317    iris_bo_unreference(res->aux.clear_color_bo);
  318    free(res->aux.state);
  319 
  320    res->aux.usage = ISL_AUX_USAGE_NONE;
  321    res->aux.possible_usages = 1 << ISL_AUX_USAGE_NONE;
  322    res->aux.sampler_usages = 1 << ISL_AUX_USAGE_NONE;
  323    res->aux.has_hiz = 0;
  324    res->aux.surf.size_B = 0;
  325    res->aux.bo = NULL;
  326    res->aux.extra_aux.surf.size_B = 0;
  327    res->aux.clear_color_bo = NULL;
  328    res->aux.state = NULL;
  329 }
  330 
  331 static void
  332 iris_resource_destroy(struct pipe_screen *screen,
  333                       struct pipe_resource *resource)
  334 {
  335    struct iris_resource *res = (struct iris_resource *)resource;
  336 
  337    if (resource->target == PIPE_BUFFER)
  338       util_range_destroy(&res->valid_buffer_range);
  339 
  340    iris_resource_disable_aux(res);
  341 
  342    iris_bo_unreference(res->bo);
  343    iris_pscreen_unref(res->base.screen);
  344 
  345    free(res);
  346 }
  347 
  348 static struct iris_resource *
  349 iris_alloc_resource(struct pipe_screen *pscreen,
  350                     const struct pipe_resource *templ)
  351 {
  352    struct iris_resource *res = calloc(1, sizeof(struct iris_resource));
  353    if (!res)
  354       return NULL;
  355 
  356    res->base = *templ;
  357    res->base.screen = iris_pscreen_ref(pscreen);
  358    pipe_reference_init(&res->base.reference, 1);
  359 
  360    res->aux.possible_usages = 1 << ISL_AUX_USAGE_NONE;
  361    res->aux.sampler_usages = 1 << ISL_AUX_USAGE_NONE;
  362 
  363    if (templ->target == PIPE_BUFFER)
  364       util_range_init(&res->valid_buffer_range);
  365 
  366    return res;
  367 }
  368 
  369 unsigned
  370 iris_get_num_logical_layers(const struct iris_resource *res, unsigned level)
  371 {
  372    if (res->surf.dim == ISL_SURF_DIM_3D)
  373       return minify(res->surf.logical_level0_px.depth, level);
  374    else
  375       return res->surf.logical_level0_px.array_len;
  376 }
  377 
  378 static enum isl_aux_state **
  379 create_aux_state_map(struct iris_resource *res, enum isl_aux_state initial)
  380 {
  381    assert(res->aux.state == NULL);
  382 
  383    uint32_t total_slices = 0;
  384    for (uint32_t level = 0; level < res->surf.levels; level++)
  385       total_slices += iris_get_num_logical_layers(res, level);
  386 
  387    const size_t per_level_array_size =
  388       res->surf.levels * sizeof(enum isl_aux_state *);
  389 
  390    /* We're going to allocate a single chunk of data for both the per-level
  391     * reference array and the arrays of aux_state.  This makes cleanup
  392     * significantly easier.
  393     */
  394    const size_t total_size =
  395       per_level_array_size + total_slices * sizeof(enum isl_aux_state);
  396 
  397    void *data = malloc(total_size);
  398    if (!data)
  399       return NULL;
  400 
  401    enum isl_aux_state **per_level_arr = data;
  402    enum isl_aux_state *s = data + per_level_array_size;
  403    for (uint32_t level = 0; level < res->surf.levels; level++) {
  404       per_level_arr[level] = s;
  405       const unsigned level_layers = iris_get_num_logical_layers(res, level);
  406       for (uint32_t a = 0; a < level_layers; a++)
  407          *(s++) = initial;
  408    }
  409    assert((void *)s == data + total_size);
  410 
  411    return per_level_arr;
  412 }
  413 
  414 static unsigned
  415 iris_get_aux_clear_color_state_size(struct iris_screen *screen)
  416 {
  417    const struct gen_device_info *devinfo = &screen->devinfo;
  418    return devinfo->gen >= 10 ? screen->isl_dev.ss.clear_color_state_size : 0;
  419 }
  420 
  421 static void
  422 map_aux_addresses(struct iris_screen *screen, struct iris_resource *res)
  423 {
  424    const struct gen_device_info *devinfo = &screen->devinfo;
  425    if (devinfo->gen >= 12 && isl_aux_usage_has_ccs(res->aux.usage)) {
  426       void *aux_map_ctx = iris_bufmgr_get_aux_map_context(screen->bufmgr);
  427       assert(aux_map_ctx);
  428       const unsigned aux_offset = res->aux.extra_aux.surf.size_B > 0 ?
  429          res->aux.extra_aux.offset : res->aux.offset;
  430       gen_aux_map_add_image(aux_map_ctx, &res->surf, res->bo->gtt_offset,
  431                             res->aux.bo->gtt_offset + aux_offset);
  432       res->bo->aux_map_address = res->aux.bo->gtt_offset;
  433    }
  434 }
  435 
  436 static bool
  437 want_ccs_e_for_format(const struct gen_device_info *devinfo,
  438                       enum isl_format format)
  439 {
  440    if (!isl_format_supports_ccs_e(devinfo, format))
  441       return false;
  442 
  443    const struct isl_format_layout *fmtl = isl_format_get_layout(format);
  444 
  445    /* CCS_E seems to significantly hurt performance with 32-bit floating
  446     * point formats.  For example, Paraview's "Wavelet Volume" case uses
  447     * both R32_FLOAT and R32G32B32A32_FLOAT, and enabling CCS_E for those
  448     * formats causes a 62% FPS drop.
  449     *
  450     * However, many benchmarks seem to use 16-bit float with no issues.
  451     */
  452    if (fmtl->channels.r.bits == 32 && fmtl->channels.r.type == ISL_SFLOAT)
  453       return false;
  454 
  455    return true;
  456 }
  457 
  458 /**
  459  * Configure aux for the resource, but don't allocate it. For images which
  460  * might be shared with modifiers, we must allocate the image and aux data in
  461  * a single bo.
  462  *
  463  * Returns false on unexpected error (e.g. allocation failed, or invalid
  464  * configuration result).
  465  */
  466 static bool
  467 iris_resource_configure_aux(struct iris_screen *screen,
  468                             struct iris_resource *res, bool imported,
  469                             uint64_t *aux_size_B,
  470                             uint32_t *alloc_flags)
  471 {
  472    const struct gen_device_info *devinfo = &screen->devinfo;
  473 
  474    /* Try to create the auxiliary surfaces allowed by the modifier or by
  475     * the user if no modifier is specified.
  476     */
  477    assert(!res->mod_info || res->mod_info->aux_usage == ISL_AUX_USAGE_NONE ||
  478                             res->mod_info->aux_usage == ISL_AUX_USAGE_CCS_E);
  479 
  480    const bool has_mcs = !res->mod_info &&
  481       isl_surf_get_mcs_surf(&screen->isl_dev, &res->surf, &res->aux.surf);
  482 
  483    const bool has_hiz = !res->mod_info && !(INTEL_DEBUG & DEBUG_NO_HIZ) &&
  484       isl_surf_get_hiz_surf(&screen->isl_dev, &res->surf, &res->aux.surf);
  485 
  486    const bool has_ccs =
  487       ((!res->mod_info && !(INTEL_DEBUG & DEBUG_NO_RBC)) ||
  488        (res->mod_info && res->mod_info->aux_usage != ISL_AUX_USAGE_NONE)) &&
  489       isl_surf_get_ccs_surf(&screen->isl_dev, &res->surf, &res->aux.surf,
  490                             &res->aux.extra_aux.surf, 0);
  491 
  492    /* Having both HIZ and MCS is impossible. */
  493    assert(!has_mcs || !has_hiz);
  494 
  495    /* Ensure aux surface creation for MCS_CCS and HIZ_CCS is correct. */
  496    if (has_ccs && (has_mcs || has_hiz)) {
  497       assert(res->aux.extra_aux.surf.size_B > 0 &&
  498              res->aux.extra_aux.surf.usage & ISL_SURF_USAGE_CCS_BIT);
  499       assert(res->aux.surf.size_B > 0 &&
  500              res->aux.surf.usage &
  501              (ISL_SURF_USAGE_HIZ_BIT | ISL_SURF_USAGE_MCS_BIT));
  502    }
  503 
  504    if (res->mod_info && has_ccs) {
  505       /* Only allow a CCS modifier if the aux was created successfully. */
  506       res->aux.possible_usages |= 1 << res->mod_info->aux_usage;
  507    } else if (has_mcs) {
  508       res->aux.possible_usages |=
  509          1 << (has_ccs ? ISL_AUX_USAGE_MCS_CCS : ISL_AUX_USAGE_MCS);
  510    } else if (has_hiz) {
  511       if (!has_ccs) {
  512          res->aux.possible_usages |= 1 << ISL_AUX_USAGE_HIZ;
  513       } else if (res->surf.samples == 1 &&
  514                  (res->surf.usage & ISL_SURF_USAGE_TEXTURE_BIT)) {
  515          /* If this resource is single-sampled and will be used as a texture,
  516           * put the HiZ surface in write-through mode so that we can sample
  517           * from it.
  518           */
  519          res->aux.possible_usages |= 1 << ISL_AUX_USAGE_HIZ_CCS_WT;
  520       } else {
  521          res->aux.possible_usages |= 1 << ISL_AUX_USAGE_HIZ_CCS;
  522       }
  523    } else if (has_ccs && isl_surf_usage_is_stencil(res->surf.usage)) {
  524       res->aux.possible_usages |= 1 << ISL_AUX_USAGE_STC_CCS;
  525    } else if (has_ccs) {
  526       if (want_ccs_e_for_format(devinfo, res->surf.format))
  527          res->aux.possible_usages |= 1 << ISL_AUX_USAGE_CCS_E;
  528 
  529       if (isl_format_supports_ccs_d(devinfo, res->surf.format))
  530          res->aux.possible_usages |= 1 << ISL_AUX_USAGE_CCS_D;
  531    }
  532 
  533    res->aux.usage = util_last_bit(res->aux.possible_usages) - 1;
  534 
  535    res->aux.sampler_usages = res->aux.possible_usages;
  536 
  537    /* We don't always support sampling with hiz. But when we do, it must be
  538     * single sampled.
  539     */
  540    if (!devinfo->has_sample_with_hiz || res->surf.samples > 1)
  541       res->aux.sampler_usages &= ~(1 << ISL_AUX_USAGE_HIZ);
  542 
  543    /* ISL_AUX_USAGE_HIZ_CCS doesn't support sampling at all */
  544    res->aux.sampler_usages &= ~(1 << ISL_AUX_USAGE_HIZ_CCS);
  545 
  546    enum isl_aux_state initial_state;
  547    *aux_size_B = 0;
  548    *alloc_flags = 0;
  549    assert(!res->aux.bo);
  550 
  551    switch (res->aux.usage) {
  552    case ISL_AUX_USAGE_NONE:
  553       /* Having no aux buffer is only okay if there's no modifier with aux. */
  554       return !res->mod_info || res->mod_info->aux_usage == ISL_AUX_USAGE_NONE;
  555    case ISL_AUX_USAGE_HIZ:
  556    case ISL_AUX_USAGE_HIZ_CCS:
  557    case ISL_AUX_USAGE_HIZ_CCS_WT:
  558       initial_state = ISL_AUX_STATE_AUX_INVALID;
  559       break;
  560    case ISL_AUX_USAGE_MCS:
  561    case ISL_AUX_USAGE_MCS_CCS:
  562       /* The Ivybridge PRM, Vol 2 Part 1 p326 says:
  563        *
  564        *    "When MCS buffer is enabled and bound to MSRT, it is required
  565        *     that it is cleared prior to any rendering."
  566        *
  567        * Since we only use the MCS buffer for rendering, we just clear it
  568        * immediately on allocation.  The clear value for MCS buffers is all
  569        * 1's, so we simply memset it to 0xff.
  570        */
  571       initial_state = ISL_AUX_STATE_CLEAR;
  572       break;
  573    case ISL_AUX_USAGE_CCS_D:
  574    case ISL_AUX_USAGE_CCS_E:
  575    case ISL_AUX_USAGE_STC_CCS:
  576       /* When CCS_E is used, we need to ensure that the CCS starts off in
  577        * a valid state.  From the Sky Lake PRM, "MCS Buffer for Render
  578        * Target(s)":
  579        *
  580        *    "If Software wants to enable Color Compression without Fast
  581        *     clear, Software needs to initialize MCS with zeros."
  582        *
  583        * A CCS value of 0 indicates that the corresponding block is in the
  584        * pass-through state which is what we want.
  585        *
  586        * For CCS_D, do the same thing.  On Gen9+, this avoids having any
  587        * undefined bits in the aux buffer.
  588        */
  589       if (imported) {
  590          assert(res->aux.usage != ISL_AUX_USAGE_STC_CCS);
  591          initial_state =
  592             isl_drm_modifier_get_default_aux_state(res->mod_info->modifier);
  593       } else {
  594          initial_state = ISL_AUX_STATE_PASS_THROUGH;
  595       }
  596       *alloc_flags |= BO_ALLOC_ZEROED;
  597       break;
  598    case ISL_AUX_USAGE_MC:
  599       unreachable("Unsupported aux mode");
  600    }
  601 
  602    /* Create the aux_state for the auxiliary buffer. */
  603    res->aux.state = create_aux_state_map(res, initial_state);
  604    if (!res->aux.state)
  605       return false;
  606 
  607    /* Increase the aux offset if the main and aux surfaces will share a BO. */
  608    res->aux.offset =
  609       !res->mod_info || res->mod_info->aux_usage == res->aux.usage ?
  610       ALIGN(res->surf.size_B, res->aux.surf.alignment_B) : 0;
  611    uint64_t size = res->aux.surf.size_B;
  612 
  613    /* Allocate space in the buffer for storing the CCS. */
  614    if (res->aux.extra_aux.surf.size_B > 0) {
  615       const uint64_t padded_aux_size =
  616          ALIGN(size, res->aux.extra_aux.surf.alignment_B);
  617       res->aux.extra_aux.offset = res->aux.offset + padded_aux_size;
  618       size = padded_aux_size + res->aux.extra_aux.surf.size_B;
  619    }
  620 
  621    /* Allocate space in the buffer for storing the clear color. On modern
  622     * platforms (gen > 9), we can read it directly from such buffer.
  623     *
  624     * On gen <= 9, we are going to store the clear color on the buffer
  625     * anyways, and copy it back to the surface state during state emission.
  626     *
  627     * Also add some padding to make sure the fast clear color state buffer
  628     * starts at a 4K alignment. We believe that 256B might be enough, but due
  629     * to lack of testing we will leave this as 4K for now.
  630     */
  631    size = ALIGN(size, 4096);
  632    res->aux.clear_color_offset = res->aux.offset + size;
  633    size += iris_get_aux_clear_color_state_size(screen);
  634    *aux_size_B = size;
  635 
  636    if (isl_aux_usage_has_hiz(res->aux.usage)) {
  637       for (unsigned level = 0; level < res->surf.levels; ++level) {
  638          uint32_t width = u_minify(res->surf.phys_level0_sa.width, level);
  639          uint32_t height = u_minify(res->surf.phys_level0_sa.height, level);
  640 
  641          /* Disable HiZ for LOD > 0 unless the width/height are 8x4 aligned.
  642           * For LOD == 0, we can grow the dimensions to make it work.
  643           */
  644          if (level == 0 || ((width & 7) == 0 && (height & 3) == 0))
  645             res->aux.has_hiz |= 1 << level;
  646       }
  647    }
  648 
  649    return true;
  650 }
  651 
  652 /**
  653  * Initialize the aux buffer contents.
  654  *
  655  * Returns false on unexpected error (e.g. mapping a BO failed).
  656  */
  657 static bool
  658 iris_resource_init_aux_buf(struct iris_resource *res, uint32_t alloc_flags,
  659                            unsigned clear_color_state_size)
  660 {
  661    if (!(alloc_flags & BO_ALLOC_ZEROED)) {
  662       void *map = iris_bo_map(NULL, res->aux.bo, MAP_WRITE | MAP_RAW);
  663 
  664       if (!map)
  665          return false;
  666 
  667       if (iris_resource_get_aux_state(res, 0, 0) != ISL_AUX_STATE_AUX_INVALID) {
  668          uint8_t memset_value = isl_aux_usage_has_mcs(res->aux.usage) ? 0xFF : 0;
  669          memset((char*)map + res->aux.offset, memset_value,
  670                 res->aux.surf.size_B);
  671       }
  672 
  673       memset((char*)map + res->aux.extra_aux.offset,
  674              0, res->aux.extra_aux.surf.size_B);
  675 
  676       /* Zero the indirect clear color to match ::fast_clear_color. */
  677       memset((char *)map + res->aux.clear_color_offset, 0,
  678              clear_color_state_size);
  679 
  680       iris_bo_unmap(res->aux.bo);
  681    }
  682 
  683    if (clear_color_state_size > 0) {
  684       res->aux.clear_color_bo = res->aux.bo;
  685       iris_bo_reference(res->aux.clear_color_bo);
  686    }
  687 
  688    return true;
  689 }
  690 
  691 /**
  692  * Allocate the initial aux surface for a resource based on aux.usage
  693  *
  694  * Returns false on unexpected error (e.g. allocation failed, or invalid
  695  * configuration result).
  696  */
  697 static bool
  698 iris_resource_alloc_separate_aux(struct iris_screen *screen,
  699                                  struct iris_resource *res)
  700 {
  701    uint32_t alloc_flags;
  702    uint64_t size;
  703    if (!iris_resource_configure_aux(screen, res, false, &size, &alloc_flags))
  704       return false;
  705 
  706    if (size == 0)
  707       return true;
  708 
  709    /* Allocate the auxiliary buffer.  ISL has stricter set of alignment rules
  710     * the drm allocator.  Therefore, one can pass the ISL dimensions in terms
  711     * of bytes instead of trying to recalculate based on different format
  712     * block sizes.
  713     */
  714    res->aux.bo = iris_bo_alloc_tiled(screen->bufmgr, "aux buffer", size, 4096,
  715                                      IRIS_MEMZONE_OTHER,
  716                                      isl_tiling_to_i915_tiling(res->aux.surf.tiling),
  717                                      res->aux.surf.row_pitch_B, alloc_flags);
  718    if (!res->aux.bo) {
  719       return false;
  720    }
  721 
  722    if (!iris_resource_init_aux_buf(res, alloc_flags,
  723                                    iris_get_aux_clear_color_state_size(screen)))
  724       return false;
  725 
  726    map_aux_addresses(screen, res);
  727 
  728    return true;
  729 }
  730 
  731 void
  732 iris_resource_finish_aux_import(struct pipe_screen *pscreen,
  733                                 struct iris_resource *res)
  734 {
  735    struct iris_screen *screen = (struct iris_screen *)pscreen;
  736    assert(iris_resource_unfinished_aux_import(res));
  737    assert(!res->mod_info->supports_clear_color);
  738 
  739    struct iris_resource *aux_res = (void *) res->base.next;
  740    assert(aux_res->aux.surf.row_pitch_B && aux_res->aux.offset &&
  741           aux_res->aux.bo);
  742 
  743    assert(res->bo == aux_res->aux.bo);
  744    iris_bo_reference(aux_res->aux.bo);
  745    res->aux.bo = aux_res->aux.bo;
  746 
  747    res->aux.offset = aux_res->aux.offset;
  748 
  749    assert(res->bo->size >= (res->aux.offset + res->aux.surf.size_B));
  750    assert(res->aux.clear_color_bo == NULL);
  751    res->aux.clear_color_offset = 0;
  752 
  753    assert(aux_res->aux.surf.row_pitch_B == res->aux.surf.row_pitch_B);
  754 
  755    unsigned clear_color_state_size =
  756       iris_get_aux_clear_color_state_size(screen);
  757 
  758    if (clear_color_state_size > 0) {
  759       res->aux.clear_color_bo =
  760          iris_bo_alloc(screen->bufmgr, "clear color buffer",
  761                        clear_color_state_size, IRIS_MEMZONE_OTHER);
  762       res->aux.clear_color_offset = 0;
  763    }
  764 
  765    iris_resource_destroy(&screen->base, res->base.next);
  766    res->base.next = NULL;
  767 }
  768 
  769 static struct pipe_resource *
  770 iris_resource_create_for_buffer(struct pipe_screen *pscreen,
  771                                 const struct pipe_resource *templ)
  772 {
  773    struct iris_screen *screen = (struct iris_screen *)pscreen;
  774    struct iris_resource *res = iris_alloc_resource(pscreen, templ);
  775 
  776    assert(templ->target == PIPE_BUFFER);
  777    assert(templ->height0 <= 1);
  778    assert(templ->depth0 <= 1);
  779    assert(templ->format == PIPE_FORMAT_NONE ||
  780           util_format_get_blocksize(templ->format) == 1);
  781 
  782    res->internal_format = templ->format;
  783    res->surf.tiling = ISL_TILING_LINEAR;
  784 
  785    enum iris_memory_zone memzone = IRIS_MEMZONE_OTHER;
  786    const char *name = templ->target == PIPE_BUFFER ? "buffer" : "miptree";
  787    if (templ->flags & IRIS_RESOURCE_FLAG_SHADER_MEMZONE) {
  788       memzone = IRIS_MEMZONE_SHADER;
  789       name = "shader kernels";
  790    } else if (templ->flags & IRIS_RESOURCE_FLAG_SURFACE_MEMZONE) {
  791       memzone = IRIS_MEMZONE_SURFACE;
  792       name = "surface state";
  793    } else if (templ->flags & IRIS_RESOURCE_FLAG_DYNAMIC_MEMZONE) {
  794       memzone = IRIS_MEMZONE_DYNAMIC;
  795       name = "dynamic state";
  796    }
  797 
  798    res->bo = iris_bo_alloc(screen->bufmgr, name, templ->width0, memzone);
  799    if (!res->bo) {
  800       iris_resource_destroy(pscreen, &res->base);
  801       return NULL;
  802    }
  803 
  804    if (templ->bind & PIPE_BIND_SHARED)
  805       iris_bo_make_external(res->bo);
  806 
  807    return &res->base;
  808 }
  809 
  810 static struct pipe_resource *
  811 iris_resource_create_with_modifiers(struct pipe_screen *pscreen,
  812                                     const struct pipe_resource *templ,
  813                                     const uint64_t *modifiers,
  814                                     int modifiers_count)
  815 {
  816    struct iris_screen *screen = (struct iris_screen *)pscreen;
  817    struct gen_device_info *devinfo = &screen->devinfo;
  818    struct iris_resource *res = iris_alloc_resource(pscreen, templ);
  819 
  820    if (!res)
  821       return NULL;
  822 
  823    const struct util_format_description *format_desc =
  824       util_format_description(templ->format);
  825    const bool has_depth = util_format_has_depth(format_desc);
  826    uint64_t modifier =
  827       select_best_modifier(devinfo, templ->format, modifiers, modifiers_count);
  828 
  829    isl_tiling_flags_t tiling_flags = ISL_TILING_ANY_MASK;
  830 
  831    if (modifier != DRM_FORMAT_MOD_INVALID) {
  832       res->mod_info = isl_drm_modifier_get_info(modifier);
  833 
  834       tiling_flags = 1 << res->mod_info->tiling;
  835    } else {
  836       if (modifiers_count > 0) {
  837          fprintf(stderr, "Unsupported modifier, resource creation failed.\n");
  838          goto fail;
  839       }
  840 
  841       /* Use linear for staging buffers */
  842       if (templ->usage == PIPE_USAGE_STAGING ||
  843           templ->bind & (PIPE_BIND_LINEAR | PIPE_BIND_CURSOR) )
  844          tiling_flags = ISL_TILING_LINEAR_BIT;
  845       else if (templ->bind & PIPE_BIND_SCANOUT)
  846          tiling_flags = ISL_TILING_X_BIT;
  847    }
  848 
  849    isl_surf_usage_flags_t usage = pipe_bind_to_isl_usage(templ->bind);
  850 
  851    if (templ->target == PIPE_TEXTURE_CUBE ||
  852        templ->target == PIPE_TEXTURE_CUBE_ARRAY)
  853       usage |= ISL_SURF_USAGE_CUBE_BIT;
  854 
  855    if (templ->usage != PIPE_USAGE_STAGING) {
  856       if (templ->format == PIPE_FORMAT_S8_UINT)
  857          usage |= ISL_SURF_USAGE_STENCIL_BIT;
  858       else if (has_depth)
  859          usage |= ISL_SURF_USAGE_DEPTH_BIT;
  860    }
  861 
  862    enum pipe_format pfmt = templ->format;
  863    res->internal_format = pfmt;
  864 
  865    /* Should be handled by u_transfer_helper */
  866    assert(!util_format_is_depth_and_stencil(pfmt));
  867 
  868    struct iris_format_info fmt = iris_format_for_usage(devinfo, pfmt, usage);
  869    assert(fmt.fmt != ISL_FORMAT_UNSUPPORTED);
  870 
  871    UNUSED const bool isl_surf_created_successfully =
  872       isl_surf_init(&screen->isl_dev, &res->surf,
  873                     .dim = target_to_isl_surf_dim(templ->target),
  874                     .format = fmt.fmt,
  875                     .width = templ->width0,
  876                     .height = templ->height0,
  877                     .depth = templ->depth0,
  878                     .levels = templ->last_level + 1,
  879                     .array_len = templ->array_size,
  880                     .samples = MAX2(templ->nr_samples, 1),
  881                     .min_alignment_B = 0,
  882                     .row_pitch_B = 0,
  883                     .usage = usage,
  884                     .tiling_flags = tiling_flags);
  885    assert(isl_surf_created_successfully);
  886 
  887    const char *name = "miptree";
  888    enum iris_memory_zone memzone = IRIS_MEMZONE_OTHER;
  889 
  890    unsigned int flags = 0;
  891    if (templ->usage == PIPE_USAGE_STAGING)
  892       flags |= BO_ALLOC_COHERENT;
  893 
  894    /* These are for u_upload_mgr buffers only */
  895    assert(!(templ->flags & (IRIS_RESOURCE_FLAG_SHADER_MEMZONE |
  896                             IRIS_RESOURCE_FLAG_SURFACE_MEMZONE |
  897                             IRIS_RESOURCE_FLAG_DYNAMIC_MEMZONE)));
  898 
  899    uint32_t aux_preferred_alloc_flags;
  900    uint64_t aux_size = 0;
  901    if (!iris_resource_configure_aux(screen, res, false, &aux_size,
  902                                     &aux_preferred_alloc_flags)) {
  903       goto fail;
  904    }
  905 
  906    /* Modifiers require the aux data to be in the same buffer as the main
  907     * surface, but we combine them even when a modifiers is not being used.
  908     */
  909    const uint64_t bo_size =
  910       MAX2(res->surf.size_B, res->aux.offset + aux_size);
  911    uint32_t alignment = MAX2(4096, res->surf.alignment_B);
  912    res->bo = iris_bo_alloc_tiled(screen->bufmgr, name, bo_size, alignment,
  913                                  memzone,
  914                                  isl_tiling_to_i915_tiling(res->surf.tiling),
  915                                  res->surf.row_pitch_B, flags);
  916 
  917    if (!res->bo)
  918       goto fail;
  919 
  920    if (aux_size > 0) {
  921       res->aux.bo = res->bo;
  922       iris_bo_reference(res->aux.bo);
  923       unsigned clear_color_state_size =
  924          iris_get_aux_clear_color_state_size(screen);
  925       if (!iris_resource_init_aux_buf(res, flags, clear_color_state_size))
  926          goto fail;
  927       map_aux_addresses(screen, res);
  928    }
  929 
  930    if (templ->bind & PIPE_BIND_SHARED)
  931       iris_bo_make_external(res->bo);
  932 
  933    return &res->base;
  934 
  935 fail:
  936    fprintf(stderr, "XXX: resource creation failed\n");
  937    iris_resource_destroy(pscreen, &res->base);
  938    return NULL;
  939 
  940 }
  941 
  942 static struct pipe_resource *
  943 iris_resource_create(struct pipe_screen *pscreen,
  944                      const struct pipe_resource *templ)
  945 {
  946    if (templ->target == PIPE_BUFFER)
  947       return iris_resource_create_for_buffer(pscreen, templ);
  948    else
  949       return iris_resource_create_with_modifiers(pscreen, templ, NULL, 0);
  950 }
  951 
  952 static uint64_t
  953 tiling_to_modifier(uint32_t tiling)
  954 {
  955    static const uint64_t map[] = {
  956       [I915_TILING_NONE]   = DRM_FORMAT_MOD_LINEAR,
  957       [I915_TILING_X]      = I915_FORMAT_MOD_X_TILED,
  958       [I915_TILING_Y]      = I915_FORMAT_MOD_Y_TILED,
  959    };
  960 
  961    assert(tiling < ARRAY_SIZE(map));
  962 
  963    return map[tiling];
  964 }
  965 
  966 static struct pipe_resource *
  967 iris_resource_from_user_memory(struct pipe_screen *pscreen,
  968                                const struct pipe_resource *templ,
  969                                void *user_memory)
  970 {
  971    struct iris_screen *screen = (struct iris_screen *)pscreen;
  972    struct iris_bufmgr *bufmgr = screen->bufmgr;
  973    struct iris_resource *res = iris_alloc_resource(pscreen, templ);
  974    if (!res)
  975       return NULL;
  976 
  977    assert(templ->target == PIPE_BUFFER);
  978 
  979    res->internal_format = templ->format;
  980    res->bo = iris_bo_create_userptr(bufmgr, "user",
  981                                     user_memory, templ->width0,
  982                                     IRIS_MEMZONE_OTHER);
  983    if (!res->bo) {
  984       iris_resource_destroy(pscreen, &res->base);
  985       return NULL;
  986    }
  987 
  988    util_range_add(&res->base, &res->valid_buffer_range, 0, templ->width0);
  989 
  990    return &res->base;
  991 }
  992 
  993 static struct pipe_resource *
  994 iris_resource_from_handle(struct pipe_screen *pscreen,
  995                           const struct pipe_resource *templ,
  996                           struct winsys_handle *whandle,
  997                           unsigned usage)
  998 {
  999    struct iris_screen *screen = (struct iris_screen *)pscreen;
 1000    struct gen_device_info *devinfo = &screen->devinfo;
 1001    struct iris_bufmgr *bufmgr = screen->bufmgr;
 1002    struct iris_resource *res = iris_alloc_resource(pscreen, templ);
 1003    const struct isl_drm_modifier_info *mod_inf =
 1004        isl_drm_modifier_get_info(whandle->modifier);
 1005    uint32_t tiling;
 1006 
 1007    if (!res)
 1008       return NULL;
 1009 
 1010    switch (whandle->type) {
 1011    case WINSYS_HANDLE_TYPE_FD:
 1012       if (mod_inf)
 1013          tiling = isl_tiling_to_i915_tiling(mod_inf->tiling);
 1014       else
 1015          tiling = I915_TILING_LAST + 1;
 1016       res->bo = iris_bo_import_dmabuf(bufmgr, whandle->handle,
 1017                                       tiling, whandle->stride);
 1018       break;
 1019    case WINSYS_HANDLE_TYPE_SHARED:
 1020       res->bo = iris_bo_gem_create_from_name(bufmgr, "winsys image",
 1021                                              whandle->handle);
 1022       break;
 1023    default:
 1024       unreachable("invalid winsys handle type");
 1025    }
 1026    if (!res->bo)
 1027       goto fail;
 1028 
 1029    res->offset = whandle->offset;
 1030 
 1031    if (mod_inf == NULL) {
 1032       mod_inf =
 1033          isl_drm_modifier_get_info(tiling_to_modifier(res->bo->tiling_mode));
 1034    }
 1035    assert(mod_inf);
 1036 
 1037    res->external_format = whandle->format;
 1038    res->mod_info = mod_inf;
 1039 
 1040    isl_surf_usage_flags_t isl_usage = pipe_bind_to_isl_usage(templ->bind);
 1041 
 1042    const struct iris_format_info fmt =
 1043       iris_format_for_usage(devinfo, templ->format, isl_usage);
 1044    res->internal_format = templ->format;
 1045 
 1046    if (templ->target == PIPE_BUFFER) {
 1047       res->surf.tiling = ISL_TILING_LINEAR;
 1048    } else {
 1049       /* Create a surface for each plane specified by the external format. */
 1050       if (whandle->plane < util_format_get_num_planes(whandle->format)) {
 1051          UNUSED const bool isl_surf_created_successfully =
 1052             isl_surf_init(&screen->isl_dev, &res->surf,
 1053                           .dim = target_to_isl_surf_dim(templ->target),
 1054                           .format = fmt.fmt,
 1055                           .width = templ->width0,
 1056                           .height = templ->height0,
 1057                           .depth = templ->depth0,
 1058                           .levels = templ->last_level + 1,
 1059                           .array_len = templ->array_size,
 1060                           .samples = MAX2(templ->nr_samples, 1),
 1061                           .min_alignment_B = 0,
 1062                           .row_pitch_B = whandle->stride,
 1063                           .usage = isl_usage,
 1064                           .tiling_flags = 1 << res->mod_info->tiling);
 1065          assert(isl_surf_created_successfully);
 1066          assert(res->bo->tiling_mode ==
 1067                 isl_tiling_to_i915_tiling(res->surf.tiling));
 1068 
 1069          // XXX: create_ccs_buf_for_image?
 1070          if (whandle->modifier == DRM_FORMAT_MOD_INVALID) {
 1071             if (!iris_resource_alloc_separate_aux(screen, res))
 1072                goto fail;
 1073          } else {
 1074             if (res->mod_info->aux_usage != ISL_AUX_USAGE_NONE) {
 1075                uint32_t alloc_flags;
 1076                uint64_t size;
 1077                bool ok = iris_resource_configure_aux(screen, res, true, &size,
 1078                                                      &alloc_flags);
 1079                assert(ok);
 1080                /* The gallium dri layer will create a separate plane resource
 1081                 * for the aux image. iris_resource_finish_aux_import will
 1082                 * merge the separate aux parameters back into a single
 1083                 * iris_resource.
 1084                 */
 1085             }
 1086          }
 1087       } else {
 1088          /* Save modifier import information to reconstruct later. After
 1089           * import, this will be available under a second image accessible
 1090           * from the main image with res->base.next. See
 1091           * iris_resource_finish_aux_import.
 1092           */
 1093          res->aux.surf.row_pitch_B = whandle->stride;
 1094          res->aux.offset = whandle->offset;
 1095          res->aux.bo = res->bo;
 1096          res->bo = NULL;
 1097       }
 1098    }
 1099 
 1100    return &res->base;
 1101 
 1102 fail:
 1103    iris_resource_destroy(pscreen, &res->base);
 1104    return NULL;
 1105 }
 1106 
 1107 static void
 1108 iris_flush_resource(struct pipe_context *ctx, struct pipe_resource *resource)
 1109 {
 1110    struct iris_context *ice = (struct iris_context *)ctx;
 1111    struct iris_batch *render_batch = &ice->batches[IRIS_BATCH_RENDER];
 1112    struct iris_resource *res = (void *) resource;
 1113    const struct isl_drm_modifier_info *mod = res->mod_info;
 1114 
 1115    iris_resource_prepare_access(ice, render_batch, res,
 1116                                 0, INTEL_REMAINING_LEVELS,
 1117                                 0, INTEL_REMAINING_LAYERS,
 1118                                 mod ? mod->aux_usage : ISL_AUX_USAGE_NONE,
 1119                                 mod ? mod->supports_clear_color : false);
 1120 }
 1121 
 1122 static void
 1123 iris_resource_disable_aux_on_first_query(struct pipe_resource *resource,
 1124                                          unsigned usage)
 1125 {
 1126    struct iris_resource *res = (struct iris_resource *)resource;
 1127    bool mod_with_aux =
 1128       res->mod_info && res->mod_info->aux_usage != ISL_AUX_USAGE_NONE;
 1129 
 1130    /* Disable aux usage if explicit flush not set and this is the first time
 1131     * we are dealing with this resource and the resource was not created with
 1132     * a modifier with aux.
 1133     */
 1134    if (!mod_with_aux &&
 1135       (!(usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH) && res->aux.usage != 0) &&
 1136        p_atomic_read(&resource->reference.count) == 1) {
 1137          iris_resource_disable_aux(res);
 1138    }
 1139 }
 1140 
 1141 static bool
 1142 iris_resource_get_param(struct pipe_screen *pscreen,
 1143                         struct pipe_context *context,
 1144                         struct pipe_resource *resource,
 1145                         unsigned plane,
 1146                         unsigned layer,
 1147                         enum pipe_resource_param param,
 1148                         unsigned handle_usage,
 1149                         uint64_t *value)
 1150 {
 1151    struct iris_screen *screen = (struct iris_screen *)pscreen;
 1152    struct iris_resource *res = (struct iris_resource *)resource;
 1153    bool mod_with_aux =
 1154       res->mod_info && res->mod_info->aux_usage != ISL_AUX_USAGE_NONE;
 1155    bool wants_aux = mod_with_aux && plane > 0;
 1156    bool result;
 1157    unsigned handle;
 1158 
 1159    if (iris_resource_unfinished_aux_import(res))
 1160       iris_resource_finish_aux_import(pscreen, res);
 1161 
 1162    struct iris_bo *bo = wants_aux ? res->aux.bo : res->bo;
 1163 
 1164    iris_resource_disable_aux_on_first_query(resource, handle_usage);
 1165 
 1166    switch (param) {
 1167    case PIPE_RESOURCE_PARAM_NPLANES:
 1168       if (mod_with_aux) {
 1169          *value = 2;
 1170       } else {
 1171          unsigned count = 0;
 1172          for (struct pipe_resource *cur = resource; cur; cur = cur->next)
 1173             count++;
 1174          *value = count;
 1175       }
 1176       return true;
 1177    case PIPE_RESOURCE_PARAM_STRIDE:
 1178       *value = wants_aux ? res->aux.surf.row_pitch_B : res->surf.row_pitch_B;
 1179       return true;
 1180    case PIPE_RESOURCE_PARAM_OFFSET:
 1181       *value = wants_aux ? res->aux.offset : 0;
 1182       return true;
 1183    case PIPE_RESOURCE_PARAM_MODIFIER:
 1184       *value = res->mod_info ? res->mod_info->modifier :
 1185                tiling_to_modifier(res->bo->tiling_mode);
 1186       return true;
 1187    case PIPE_RESOURCE_PARAM_HANDLE_TYPE_SHARED:
 1188       result = iris_bo_flink(bo, &handle) == 0;
 1189       if (result)
 1190          *value = handle;
 1191       return result;
 1192    case PIPE_RESOURCE_PARAM_HANDLE_TYPE_KMS: {
 1193       /* Because we share the same drm file across multiple iris_screen, when
 1194        * we export a GEM handle we must make sure it is valid in the DRM file
 1195        * descriptor the caller is using (this is the FD given at screen
 1196        * creation).
 1197        */
 1198       uint32_t handle;
 1199       if (iris_bo_export_gem_handle_for_device(bo, screen->winsys_fd, &handle))
 1200          return false;
 1201       *value = handle;
 1202       return true;
 1203    }
 1204 
 1205    case PIPE_RESOURCE_PARAM_HANDLE_TYPE_FD:
 1206       result = iris_bo_export_dmabuf(bo, (int *) &handle) == 0;
 1207       if (result)
 1208          *value = handle;
 1209       return result;
 1210    default:
 1211       return false;
 1212    }
 1213 }
 1214 
 1215 static bool
 1216 iris_resource_get_handle(struct pipe_screen *pscreen,
 1217                          struct pipe_context *ctx,
 1218                          struct pipe_resource *resource,
 1219                          struct winsys_handle *whandle,
 1220                          unsigned usage)
 1221 {
 1222    struct iris_screen *screen = (struct iris_screen *) pscreen;
 1223    struct iris_resource *res = (struct iris_resource *)resource;
 1224    bool mod_with_aux =
 1225       res->mod_info && res->mod_info->aux_usage != ISL_AUX_USAGE_NONE;
 1226 
 1227    iris_resource_disable_aux_on_first_query(resource, usage);
 1228 
 1229    struct iris_bo *bo;
 1230    if (mod_with_aux && whandle->plane > 0) {
 1231       assert(res->aux.bo);
 1232       bo = res->aux.bo;
 1233       whandle->stride = res->aux.surf.row_pitch_B;
 1234       whandle->offset = res->aux.offset;
 1235    } else {
 1236       /* If this is a buffer, stride should be 0 - no need to special case */
 1237       whandle->stride = res->surf.row_pitch_B;
 1238       bo = res->bo;
 1239    }
 1240 
 1241    whandle->format = res->external_format;
 1242    whandle->modifier =
 1243       res->mod_info ? res->mod_info->modifier
 1244                     : tiling_to_modifier(res->bo->tiling_mode);
 1245 
 1246 #ifndef NDEBUG
 1247    enum isl_aux_usage allowed_usage =
 1248       usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH ? res->aux.usage :
 1249       res->mod_info ? res->mod_info->aux_usage : ISL_AUX_USAGE_NONE;
 1250 
 1251    if (res->aux.usage != allowed_usage) {
 1252       enum isl_aux_state aux_state = iris_resource_get_aux_state(res, 0, 0);
 1253       assert(aux_state == ISL_AUX_STATE_RESOLVED ||
 1254              aux_state == ISL_AUX_STATE_PASS_THROUGH);
 1255    }
 1256 #endif
 1257 
 1258    switch (whandle->type) {
 1259    case WINSYS_HANDLE_TYPE_SHARED:
 1260       return iris_bo_flink(bo, &whandle->handle) == 0;
 1261    case WINSYS_HANDLE_TYPE_KMS: {
 1262       /* Because we share the same drm file across multiple iris_screen, when
 1263        * we export a GEM handle we must make sure it is valid in the DRM file
 1264        * descriptor the caller is using (this is the FD given at screen
 1265        * creation).
 1266        */
 1267       uint32_t handle;
 1268       if (iris_bo_export_gem_handle_for_device(bo, screen->winsys_fd, &handle))
 1269          return false;
 1270       whandle->handle = handle;
 1271       return true;
 1272    }
 1273    case WINSYS_HANDLE_TYPE_FD:
 1274       return iris_bo_export_dmabuf(bo, (int *) &whandle->handle) == 0;
 1275    }
 1276 
 1277    return false;
 1278 }
 1279 
 1280 static bool
 1281 resource_is_busy(struct iris_context *ice,
 1282                  struct iris_resource *res)
 1283 {
 1284    bool busy = iris_bo_busy(res->bo);
 1285 
 1286    for (int i = 0; i < IRIS_BATCH_COUNT; i++)
 1287       busy |= iris_batch_references(&ice->batches[i], res->bo);
 1288 
 1289    return busy;
 1290 }
 1291 
 1292 static void
 1293 iris_invalidate_resource(struct pipe_context *ctx,
 1294                          struct pipe_resource *resource)
 1295 {
 1296    struct iris_screen *screen = (void *) ctx->screen;
 1297    struct iris_context *ice = (void *) ctx;
 1298    struct iris_resource *res = (void *) resource;
 1299 
 1300    if (resource->target != PIPE_BUFFER)
 1301       return;
 1302 
 1303    /* If it's already invalidated, don't bother doing anything. */
 1304    if (res->valid_buffer_range.start > res->valid_buffer_range.end)
 1305       return;
 1306 
 1307    if (!resource_is_busy(ice, res)) {
 1308       /* The resource is idle, so just mark that it contains no data and
 1309        * keep using the same underlying buffer object.
 1310        */
 1311       util_range_set_empty(&res->valid_buffer_range);
 1312       return;
 1313    }
 1314 
 1315    /* Otherwise, try and replace the backing storage with a new BO. */
 1316 
 1317    /* We can't reallocate memory we didn't allocate in the first place. */
 1318    if (res->bo->userptr)
 1319       return;
 1320 
 1321    // XXX: We should support this.
 1322    if (res->bind_history & PIPE_BIND_STREAM_OUTPUT)
 1323       return;
 1324 
 1325    struct iris_bo *old_bo = res->bo;
 1326    struct iris_bo *new_bo =
 1327       iris_bo_alloc(screen->bufmgr, res->bo->name, resource->width0,
 1328                     iris_memzone_for_address(old_bo->gtt_offset));
 1329    if (!new_bo)
 1330       return;
 1331 
 1332    /* Swap out the backing storage */
 1333    res->bo = new_bo;
 1334 
 1335    /* Rebind the buffer, replacing any state referring to the old BO's
 1336     * address, and marking state dirty so it's reemitted.
 1337     */
 1338    screen->vtbl.rebind_buffer(ice, res);
 1339 
 1340    util_range_set_empty(&res->valid_buffer_range);
 1341 
 1342    iris_bo_unreference(old_bo);
 1343 }
 1344 
 1345 static void
 1346 iris_flush_staging_region(struct pipe_transfer *xfer,
 1347                           const struct pipe_box *flush_box)
 1348 {
 1349    if (!(xfer->usage & PIPE_TRANSFER_WRITE))
 1350       return;
 1351 
 1352    struct iris_transfer *map = (void *) xfer;
 1353 
 1354    struct pipe_box src_box = *flush_box;
 1355 
 1356    /* Account for extra alignment padding in staging buffer */
 1357    if (xfer->resource->target == PIPE_BUFFER)
 1358       src_box.x += xfer->box.x % IRIS_MAP_BUFFER_ALIGNMENT;
 1359 
 1360    struct pipe_box dst_box = (struct pipe_box) {
 1361       .x = xfer->box.x + flush_box->x,
 1362       .y = xfer->box.y + flush_box->y,
 1363       .z = xfer->box.z + flush_box->z,
 1364       .width = flush_box->width,
 1365       .height = flush_box->height,
 1366       .depth = flush_box->depth,
 1367    };
 1368 
 1369    iris_copy_region(map->blorp, map->batch, xfer->resource, xfer->level,
 1370                     dst_box.x, dst_box.y, dst_box.z, map->staging, 0,
 1371                     &src_box);
 1372 }
 1373 
 1374 static void
 1375 iris_unmap_copy_region(struct iris_transfer *map)
 1376 {
 1377    iris_resource_destroy(map->staging->screen, map->staging);
 1378 
 1379    map->ptr = NULL;
 1380 }
 1381 
 1382 static void
 1383 iris_map_copy_region(struct iris_transfer *map)
 1384 {
 1385    struct pipe_screen *pscreen = &map->batch->screen->base;
 1386    struct pipe_transfer *xfer = &map->base;
 1387    struct pipe_box *box = &xfer->box;
 1388    struct iris_resource *res = (void *) xfer->resource;
 1389 
 1390    unsigned extra = xfer->resource->target == PIPE_BUFFER ?
 1391                     box->x % IRIS_MAP_BUFFER_ALIGNMENT : 0;
 1392 
 1393    struct pipe_resource templ = (struct pipe_resource) {
 1394       .usage = PIPE_USAGE_STAGING,
 1395       .width0 = box->width + extra,
 1396       .height0 = box->height,
 1397       .depth0 = 1,
 1398       .nr_samples = xfer->resource->nr_samples,
 1399       .nr_storage_samples = xfer->resource->nr_storage_samples,
 1400       .array_size = box->depth,
 1401       .format = res->internal_format,
 1402    };
 1403 
 1404    if (xfer->resource->target == PIPE_BUFFER)
 1405       templ.target = PIPE_BUFFER;
 1406    else if (templ.array_size > 1)
 1407       templ.target = PIPE_TEXTURE_2D_ARRAY;
 1408    else
 1409       templ.target = PIPE_TEXTURE_2D;
 1410 
 1411    map->staging = iris_resource_create(pscreen, &templ);
 1412    assert(map->staging);
 1413 
 1414    if (templ.target != PIPE_BUFFER) {
 1415       struct isl_surf *surf = &((struct iris_resource *) map->staging)->surf;
 1416       xfer->stride = isl_surf_get_row_pitch_B(surf);
 1417       xfer->layer_stride = isl_surf_get_array_pitch(surf);
 1418    }
 1419 
 1420    if (!(xfer->usage & PIPE_TRANSFER_DISCARD_RANGE)) {
 1421       iris_copy_region(map->blorp, map->batch, map->staging, 0, extra, 0, 0,
 1422                        xfer->resource, xfer->level, box);
 1423       /* Ensure writes to the staging BO land before we map it below. */
 1424       iris_emit_pipe_control_flush(map->batch,
 1425                                    "transfer read: flush before mapping",
 1426                                    PIPE_CONTROL_RENDER_TARGET_FLUSH |
 1427                                    PIPE_CONTROL_CS_STALL);
 1428    }
 1429 
 1430    struct iris_bo *staging_bo = iris_resource_bo(map->staging);
 1431 
 1432    if (iris_batch_references(map->batch, staging_bo))
 1433       iris_batch_flush(map->batch);
 1434 
 1435    map->ptr =
 1436       iris_bo_map(map->dbg, staging_bo, xfer->usage & MAP_FLAGS) + extra;
 1437 
 1438    map->unmap = iris_unmap_copy_region;
 1439 }
 1440 
 1441 static void
 1442 get_image_offset_el(const struct isl_surf *surf, unsigned level, unsigned z,
 1443                     unsigned *out_x0_el, unsigned *out_y0_el)
 1444 {
 1445    if (surf->dim == ISL_SURF_DIM_3D) {
 1446       isl_surf_get_image_offset_el(surf, level, 0, z, out_x0_el, out_y0_el);
 1447    } else {
 1448       isl_surf_get_image_offset_el(surf, level, z, 0, out_x0_el, out_y0_el);
 1449    }
 1450 }
 1451 
 1452 /**
 1453  * This function computes the tile_w (in bytes) and tile_h (in rows) of
 1454  * different tiling patterns.
 1455  */
 1456 static void
 1457 iris_resource_get_tile_dims(enum isl_tiling tiling, uint32_t cpp,
 1458                             uint32_t *tile_w, uint32_t *tile_h)
 1459 {
 1460    switch (tiling) {
 1461    case ISL_TILING_X:
 1462       *tile_w = 512;
 1463       *tile_h = 8;
 1464       break;
 1465    case ISL_TILING_Y0:
 1466       *tile_w = 128;
 1467       *tile_h = 32;
 1468       break;
 1469    case ISL_TILING_LINEAR:
 1470       *tile_w = cpp;
 1471       *tile_h = 1;
 1472       break;
 1473    default:
 1474       unreachable("not reached");
 1475    }
 1476 
 1477 }
 1478 
 1479 /**
 1480  * This function computes masks that may be used to select the bits of the X
 1481  * and Y coordinates that indicate the offset within a tile.  If the BO is
 1482  * untiled, the masks are set to 0.
 1483  */
 1484 static void
 1485 iris_resource_get_tile_masks(enum isl_tiling tiling, uint32_t cpp,
 1486                              uint32_t *mask_x, uint32_t *mask_y)
 1487 {
 1488    uint32_t tile_w_bytes, tile_h;
 1489 
 1490    iris_resource_get_tile_dims(tiling, cpp, &tile_w_bytes, &tile_h);
 1491 
 1492    *mask_x = tile_w_bytes / cpp - 1;
 1493    *mask_y = tile_h - 1;
 1494 }
 1495 
 1496 /**
 1497  * Compute the offset (in bytes) from the start of the BO to the given x
 1498  * and y coordinate.  For tiled BOs, caller must ensure that x and y are
 1499  * multiples of the tile size.
 1500  */
 1501 static uint32_t
 1502 iris_resource_get_aligned_offset(const struct iris_resource *res,
 1503                                  uint32_t x, uint32_t y)
 1504 {
 1505    const struct isl_format_layout *fmtl = isl_format_get_layout(res->surf.format);
 1506    unsigned cpp = fmtl->bpb / 8;
 1507    uint32_t pitch = res->surf.row_pitch_B;
 1508 
 1509    switch (res->surf.tiling) {
 1510    default:
 1511       unreachable("not reached");
 1512    case ISL_TILING_LINEAR:
 1513       return y * pitch + x * cpp;
 1514    case ISL_TILING_X:
 1515       assert((x % (512 / cpp)) == 0);
 1516       assert((y % 8) == 0);
 1517       return y * pitch + x / (512 / cpp) * 4096;
 1518    case ISL_TILING_Y0:
 1519       assert((x % (128 / cpp)) == 0);
 1520       assert((y % 32) == 0);
 1521       return y * pitch + x / (128 / cpp) * 4096;
 1522    }
 1523 }
 1524 
 1525 /**
 1526  * Rendering with tiled buffers requires that the base address of the buffer
 1527  * be aligned to a page boundary.  For renderbuffers, and sometimes with
 1528  * textures, we may want the surface to point at a texture image level that
 1529  * isn't at a page boundary.
 1530  *
 1531  * This function returns an appropriately-aligned base offset
 1532  * according to the tiling restrictions, plus any required x/y offset
 1533  * from there.
 1534  */
 1535 uint32_t
 1536 iris_resource_get_tile_offsets(const struct iris_resource *res,
 1537                                uint32_t level, uint32_t z,
 1538                                uint32_t *tile_x, uint32_t *tile_y)
 1539 {
 1540    uint32_t x, y;
 1541    uint32_t mask_x, mask_y;
 1542 
 1543    const struct isl_format_layout *fmtl = isl_format_get_layout(res->surf.format);
 1544    const unsigned cpp = fmtl->bpb / 8;
 1545 
 1546    iris_resource_get_tile_masks(res->surf.tiling, cpp, &mask_x, &mask_y);
 1547    get_image_offset_el(&res->surf, level, z, &x, &y);
 1548 
 1549    *tile_x = x & mask_x;
 1550    *tile_y = y & mask_y;
 1551 
 1552    return iris_resource_get_aligned_offset(res, x & ~mask_x, y & ~mask_y);
 1553 }
 1554 
 1555 /**
 1556  * Get pointer offset into stencil buffer.
 1557  *
 1558  * The stencil buffer is W tiled. Since the GTT is incapable of W fencing, we
 1559  * must decode the tile's layout in software.
 1560  *
 1561  * See
 1562  *   - PRM, 2011 Sandy Bridge, Volume 1, Part 2, Section 4.5.2.1 W-Major Tile
 1563  *     Format.
 1564  *   - PRM, 2011 Sandy Bridge, Volume 1, Part 2, Section 4.5.3 Tiling Algorithm
 1565  *
 1566  * Even though the returned offset is always positive, the return type is
 1567  * signed due to
 1568  *    commit e8b1c6d6f55f5be3bef25084fdd8b6127517e137
 1569  *    mesa: Fix return type of  _mesa_get_format_bytes() (#37351)
 1570  */
 1571 static intptr_t
 1572 s8_offset(uint32_t stride, uint32_t x, uint32_t y)
 1573 {
 1574    uint32_t tile_size = 4096;
 1575    uint32_t tile_width = 64;
 1576    uint32_t tile_height = 64;
 1577    uint32_t row_size = 64 * stride / 2; /* Two rows are interleaved. */
 1578 
 1579    uint32_t tile_x = x / tile_width;
 1580    uint32_t tile_y = y / tile_height;
 1581 
 1582    /* The byte's address relative to the tile's base addres. */
 1583    uint32_t byte_x = x % tile_width;
 1584    uint32_t byte_y = y % tile_height;
 1585 
 1586    uintptr_t u = tile_y * row_size
 1587                + tile_x * tile_size
 1588                + 512 * (byte_x / 8)
 1589                +  64 * (byte_y / 8)
 1590                +  32 * ((byte_y / 4) % 2)
 1591                +  16 * ((byte_x / 4) % 2)
 1592                +   8 * ((byte_y / 2) % 2)
 1593                +   4 * ((byte_x / 2) % 2)
 1594                +   2 * (byte_y % 2)
 1595                +   1 * (byte_x % 2);
 1596 
 1597    return u;
 1598 }
 1599 
 1600 static void
 1601 iris_unmap_s8(struct iris_transfer *map)
 1602 {
 1603    struct pipe_transfer *xfer = &map->base;
 1604    const struct pipe_box *box = &xfer->box;
 1605    struct iris_resource *res = (struct iris_resource *) xfer->resource;
 1606    struct isl_surf *surf = &res->surf;
 1607 
 1608    if (xfer->usage & PIPE_TRANSFER_WRITE) {
 1609       uint8_t *untiled_s8_map = map->ptr;
 1610       uint8_t *tiled_s8_map =
 1611          iris_bo_map(map->dbg, res->bo, (xfer->usage | MAP_RAW) & MAP_FLAGS);
 1612 
 1613       for (int s = 0; s < box->depth; s++) {
 1614          unsigned x0_el, y0_el;
 1615          get_image_offset_el(surf, xfer->level, box->z + s, &x0_el, &y0_el);
 1616 
 1617          for (uint32_t y = 0; y < box->height; y++) {
 1618             for (uint32_t x = 0; x < box->width; x++) {
 1619                ptrdiff_t offset = s8_offset(surf->row_pitch_B,
 1620                                             x0_el + box->x + x,
 1621                                             y0_el + box->y + y);
 1622                tiled_s8_map[offset] =
 1623                   untiled_s8_map[s * xfer->layer_stride + y * xfer->stride + x];
 1624             }
 1625          }
 1626       }
 1627    }
 1628 
 1629    free(map->buffer);
 1630 }
 1631 
 1632 static void
 1633 iris_map_s8(struct iris_transfer *map)
 1634 {
 1635    struct pipe_transfer *xfer = &map->base;
 1636    const struct pipe_box *box = &xfer->box;
 1637    struct iris_resource *res = (struct iris_resource *) xfer->resource;
 1638    struct isl_surf *surf = &res->surf;
 1639 
 1640    xfer->stride = surf->row_pitch_B;
 1641    xfer->layer_stride = xfer->stride * box->height;
 1642 
 1643    /* The tiling and detiling functions require that the linear buffer has
 1644     * a 16-byte alignment (that is, its `x0` is 16-byte aligned).  Here we
 1645     * over-allocate the linear buffer to get the proper alignment.
 1646     */
 1647    map->buffer = map->ptr = malloc(xfer->layer_stride * box->depth);
 1648    assert(map->buffer);
 1649 
 1650    /* One of either READ_BIT or WRITE_BIT or both is set.  READ_BIT implies no
 1651     * INVALIDATE_RANGE_BIT.  WRITE_BIT needs the original values read in unless
 1652     * invalidate is set, since we'll be writing the whole rectangle from our
 1653     * temporary buffer back out.
 1654     */
 1655    if (!(xfer->usage & PIPE_TRANSFER_DISCARD_RANGE)) {
 1656       uint8_t *untiled_s8_map = map->ptr;
 1657       uint8_t *tiled_s8_map =
 1658          iris_bo_map(map->dbg, res->bo, (xfer->usage | MAP_RAW) & MAP_FLAGS);
 1659 
 1660       for (int s = 0; s < box->depth; s++) {
 1661          unsigned x0_el, y0_el;
 1662          get_image_offset_el(surf, xfer->level, box->z + s, &x0_el, &y0_el);
 1663 
 1664          for (uint32_t y = 0; y < box->height; y++) {
 1665             for (uint32_t x = 0; x < box->width; x++) {
 1666                ptrdiff_t offset = s8_offset(surf->row_pitch_B,
 1667                                             x0_el + box->x + x,
 1668                                             y0_el + box->y + y);
 1669                untiled_s8_map[s * xfer->layer_stride + y * xfer->stride + x] =
 1670                   tiled_s8_map[offset];
 1671             }
 1672          }
 1673       }
 1674    }
 1675 
 1676    map->unmap = iris_unmap_s8;
 1677 }
 1678 
 1679 /* Compute extent parameters for use with tiled_memcpy functions.
 1680  * xs are in units of bytes and ys are in units of strides.
 1681  */
 1682 static inline void
 1683 tile_extents(const struct isl_surf *surf,
 1684              const struct pipe_box *box,
 1685              unsigned level, int z,
 1686              unsigned *x1_B, unsigned *x2_B,
 1687              unsigned *y1_el, unsigned *y2_el)
 1688 {
 1689    const struct isl_format_layout *fmtl = isl_format_get_layout(surf->format);
 1690    const unsigned cpp = fmtl->bpb / 8;
 1691 
 1692    assert(box->x % fmtl->bw == 0);
 1693    assert(box->y % fmtl->bh == 0);
 1694 
 1695    unsigned x0_el, y0_el;
 1696    get_image_offset_el(surf, level, box->z + z, &x0_el, &y0_el);
 1697 
 1698    *x1_B = (box->x / fmtl->bw + x0_el) * cpp;
 1699    *y1_el = box->y / fmtl->bh + y0_el;
 1700    *x2_B = (DIV_ROUND_UP(box->x + box->width, fmtl->bw) + x0_el) * cpp;
 1701    *y2_el = DIV_ROUND_UP(box->y + box->height, fmtl->bh) + y0_el;
 1702 }
 1703 
 1704 static void
 1705 iris_unmap_tiled_memcpy(struct iris_transfer *map)
 1706 {
 1707    struct pipe_transfer *xfer = &map->base;
 1708    const struct pipe_box *box = &xfer->box;
 1709    struct iris_resource *res = (struct iris_resource *) xfer->resource;
 1710    struct isl_surf *surf = &res->surf;
 1711 
 1712    const bool has_swizzling = false;
 1713 
 1714    if (xfer->usage & PIPE_TRANSFER_WRITE) {
 1715       char *dst =
 1716          iris_bo_map(map->dbg, res->bo, (xfer->usage | MAP_RAW) & MAP_FLAGS);
 1717 
 1718       for (int s = 0; s < box->depth; s++) {
 1719          unsigned x1, x2, y1, y2;
 1720          tile_extents(surf, box, xfer->level, s, &x1, &x2, &y1, &y2);
 1721 
 1722          void *ptr = map->ptr + s * xfer->layer_stride;
 1723 
 1724          isl_memcpy_linear_to_tiled(x1, x2, y1, y2, dst, ptr,
 1725                                     surf->row_pitch_B, xfer->stride,
 1726                                     has_swizzling, surf->tiling, ISL_MEMCPY);
 1727       }
 1728    }
 1729    os_free_aligned(map->buffer);
 1730    map->buffer = map->ptr = NULL;
 1731 }
 1732 
 1733 static void
 1734 iris_map_tiled_memcpy(struct iris_transfer *map)
 1735 {
 1736    struct pipe_transfer *xfer = &map->base;
 1737    const struct pipe_box *box = &xfer->box;
 1738    struct iris_resource *res = (struct iris_resource *) xfer->resource;
 1739    struct isl_surf *surf = &res->surf;
 1740 
 1741    xfer->stride = ALIGN(surf->row_pitch_B, 16);
 1742    xfer->layer_stride = xfer->stride * box->height;
 1743 
 1744    unsigned x1, x2, y1, y2;
 1745    tile_extents(surf, box, xfer->level, 0, &x1, &x2, &y1, &y2);
 1746 
 1747    /* The tiling and detiling functions require that the linear buffer has
 1748     * a 16-byte alignment (that is, its `x0` is 16-byte aligned).  Here we
 1749     * over-allocate the linear buffer to get the proper alignment.
 1750     */
 1751    map->buffer =
 1752       os_malloc_aligned(xfer->layer_stride * box->depth, 16);
 1753    assert(map->buffer);
 1754    map->ptr = (char *)map->buffer + (x1 & 0xf);
 1755 
 1756    const bool has_swizzling = false;
 1757 
 1758    if (!(xfer->usage & PIPE_TRANSFER_DISCARD_RANGE)) {
 1759       char *src =
 1760          iris_bo_map(map->dbg, res->bo, (xfer->usage | MAP_RAW) & MAP_FLAGS);
 1761 
 1762       for (int s = 0; s < box->depth; s++) {
 1763          unsigned x1, x2, y1, y2;
 1764          tile_extents(surf, box, xfer->level, s, &x1, &x2, &y1, &y2);
 1765 
 1766          /* Use 's' rather than 'box->z' to rebase the first slice to 0. */
 1767          void *ptr = map->ptr + s * xfer->layer_stride;
 1768 
 1769          isl_memcpy_tiled_to_linear(x1, x2, y1, y2, ptr, src, xfer->stride,
 1770                                     surf->row_pitch_B, has_swizzling,
 1771                                     surf->tiling, ISL_MEMCPY_STREAMING_LOAD);
 1772       }
 1773    }
 1774 
 1775    map->unmap = iris_unmap_tiled_memcpy;
 1776 }
 1777 
 1778 static void
 1779 iris_map_direct(struct iris_transfer *map)
 1780 {
 1781    struct pipe_transfer *xfer = &map->base;
 1782    struct pipe_box *box = &xfer->box;
 1783    struct iris_resource *res = (struct iris_resource *) xfer->resource;
 1784 
 1785    void *ptr = iris_bo_map(map->dbg, res->bo, xfer->usage & MAP_FLAGS);
 1786 
 1787    if (res->base.target == PIPE_BUFFER) {
 1788       xfer->stride = 0;
 1789       xfer->layer_stride = 0;
 1790 
 1791       map->ptr = ptr + box->x;
 1792    } else {
 1793       struct isl_surf *surf = &res->surf;
 1794       const struct isl_format_layout *fmtl =
 1795          isl_format_get_layout(surf->format);
 1796       const unsigned cpp = fmtl->bpb / 8;
 1797       unsigned x0_el, y0_el;
 1798 
 1799       get_image_offset_el(surf, xfer->level, box->z, &x0_el, &y0_el);
 1800 
 1801       xfer->stride = isl_surf_get_row_pitch_B(surf);
 1802       xfer->layer_stride = isl_surf_get_array_pitch(surf);
 1803 
 1804       map->ptr = ptr + (y0_el + box->y) * xfer->stride + (x0_el + box->x) * cpp;
 1805    }
 1806 }
 1807 
 1808 static bool
 1809 can_promote_to_async(const struct iris_resource *res,
 1810                      const struct pipe_box *box,
 1811                      enum pipe_transfer_usage usage)
 1812 {
 1813    /* If we're writing to a section of the buffer that hasn't even been
 1814     * initialized with useful data, then we can safely promote this write
 1815     * to be unsynchronized.  This helps the common pattern of appending data.
 1816     */
 1817    return res->base.target == PIPE_BUFFER && (usage & PIPE_TRANSFER_WRITE) &&
 1818           !(usage & TC_TRANSFER_MAP_NO_INFER_UNSYNCHRONIZED) &&
 1819           !util_ranges_intersect(&res->valid_buffer_range, box->x,
 1820                                  box->x + box->width);
 1821 }
 1822 
 1823 static void *
 1824 iris_transfer_map(struct pipe_context *ctx,
 1825                   struct pipe_resource *resource,
 1826                   unsigned level,
 1827                   enum pipe_transfer_usage usage,
 1828                   const struct pipe_box *box,
 1829                   struct pipe_transfer **ptransfer)
 1830 {
 1831    struct iris_context *ice = (struct iris_context *)ctx;
 1832    struct iris_resource *res = (struct iris_resource *)resource;
 1833    struct isl_surf *surf = &res->surf;
 1834 
 1835    if (iris_resource_unfinished_aux_import(res))
 1836       iris_resource_finish_aux_import(ctx->screen, res);
 1837 
 1838    if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE) {
 1839       /* Replace the backing storage with a fresh buffer for non-async maps */
 1840       if (!(usage & (PIPE_TRANSFER_UNSYNCHRONIZED |
 1841                      TC_TRANSFER_MAP_NO_INVALIDATE)))
 1842          iris_invalidate_resource(ctx, resource);
 1843 
 1844       /* If we can discard the whole resource, we can discard the range. */
 1845       usage |= PIPE_TRANSFER_DISCARD_RANGE;
 1846    }
 1847 
 1848    if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED) &&
 1849        can_promote_to_async(res, box, usage)) {
 1850       usage |= PIPE_TRANSFER_UNSYNCHRONIZED;
 1851    }
 1852 
 1853    bool need_resolve = false;
 1854    bool need_color_resolve = false;
 1855 
 1856    if (resource->target != PIPE_BUFFER) {
 1857       bool need_hiz_resolve = iris_resource_level_has_hiz(res, level);
 1858       bool need_stencil_resolve = res->aux.usage == ISL_AUX_USAGE_STC_CCS;
 1859 
 1860       need_color_resolve =
 1861          (res->aux.usage == ISL_AUX_USAGE_CCS_D ||
 1862           res->aux.usage == ISL_AUX_USAGE_CCS_E) &&
 1863          iris_has_color_unresolved(res, level, 1, box->z, box->depth);
 1864 
 1865       need_resolve = need_color_resolve ||
 1866                      need_hiz_resolve ||
 1867                      need_stencil_resolve;
 1868    }
 1869 
 1870    bool map_would_stall = false;
 1871 
 1872    if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
 1873       map_would_stall = need_resolve || resource_is_busy(ice, res);
 1874 
 1875       if (map_would_stall && (usage & PIPE_TRANSFER_DONTBLOCK) &&
 1876                              (usage & PIPE_TRANSFER_MAP_DIRECTLY))
 1877          return NULL;
 1878    }
 1879 
 1880    if (surf->tiling != ISL_TILING_LINEAR &&
 1881        (usage & PIPE_TRANSFER_MAP_DIRECTLY))
 1882       return NULL;
 1883 
 1884    struct iris_transfer *map = slab_alloc(&ice->transfer_pool);
 1885    struct pipe_transfer *xfer = &map->base;
 1886 
 1887    if (!map)
 1888       return NULL;
 1889 
 1890    memset(map, 0, sizeof(*map));
 1891    map->dbg = &ice->dbg;
 1892 
 1893    pipe_resource_reference(&xfer->resource, resource);
 1894    xfer->level = level;
 1895    xfer->usage = usage;
 1896    xfer->box = *box;
 1897    *ptransfer = xfer;
 1898 
 1899    map->dest_had_defined_contents =
 1900       util_ranges_intersect(&res->valid_buffer_range, box->x,
 1901                             box->x + box->width);
 1902 
 1903    if (usage & PIPE_TRANSFER_WRITE)
 1904       util_range_add(&res->base, &res->valid_buffer_range, box->x, box->x + box->width);
 1905 
 1906    /* Avoid using GPU copies for persistent/coherent buffers, as the idea
 1907     * there is to access them simultaneously on the CPU & GPU.  This also
 1908     * avoids trying to use GPU copies for our u_upload_mgr buffers which
 1909     * contain state we're constructing for a GPU draw call, which would
 1910     * kill us with infinite stack recursion.
 1911     */
 1912    bool no_gpu = usage & (PIPE_TRANSFER_PERSISTENT |
 1913                           PIPE_TRANSFER_COHERENT |
 1914                           PIPE_TRANSFER_MAP_DIRECTLY);
 1915 
 1916    /* GPU copies are not useful for buffer reads.  Instead of stalling to
 1917     * read from the original buffer, we'd simply copy it to a temporary...
 1918     * then stall (a bit longer) to read from that buffer.
 1919     *
 1920     * Images are less clear-cut.  Color resolves are destructive, removing
 1921     * the underlying compression, so we'd rather blit the data to a linear
 1922     * temporary and map that, to avoid the resolve.  (It might be better to
 1923     * a tiled temporary and use the tiled_memcpy paths...)
 1924     */
 1925    if (!(usage & PIPE_TRANSFER_DISCARD_RANGE) && !need_color_resolve)
 1926       no_gpu = true;
 1927 
 1928    const struct isl_format_layout *fmtl = isl_format_get_layout(surf->format);
 1929    if (fmtl->txc == ISL_TXC_ASTC)
 1930       no_gpu = true;
 1931 
 1932    if ((map_would_stall || res->aux.usage == ISL_AUX_USAGE_CCS_E) && !no_gpu) {
 1933       /* If we need a synchronous mapping and the resource is busy, or needs
 1934        * resolving, we copy to/from a linear temporary buffer using the GPU.
 1935        */
 1936       map->batch = &ice->batches[IRIS_BATCH_RENDER];
 1937       map->blorp = &ice->blorp;
 1938       iris_map_copy_region(map);
 1939    } else {
 1940       /* Otherwise we're free to map on the CPU. */
 1941 
 1942       if (need_resolve) {
 1943          iris_resource_access_raw(ice, &ice->batches[IRIS_BATCH_RENDER], res,
 1944                                   level, box->z, box->depth,
 1945                                   usage & PIPE_TRANSFER_WRITE);
 1946       }
 1947 
 1948       if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
 1949          for (int i = 0; i < IRIS_BATCH_COUNT; i++) {
 1950             if (iris_batch_references(&ice->batches[i], res->bo))
 1951                iris_batch_flush(&ice->batches[i]);
 1952          }
 1953       }
 1954 
 1955       if (surf->tiling == ISL_TILING_W) {
 1956          /* TODO: Teach iris_map_tiled_memcpy about W-tiling... */
 1957          iris_map_s8(map);
 1958       } else if (surf->tiling != ISL_TILING_LINEAR) {
 1959          iris_map_tiled_memcpy(map);
 1960       } else {
 1961          iris_map_direct(map);
 1962       }
 1963    }
 1964 
 1965    return map->ptr;
 1966 }
 1967 
 1968 static void
 1969 iris_transfer_flush_region(struct pipe_context *ctx,
 1970                            struct pipe_transfer *xfer,
 1971                            const struct pipe_box *box)
 1972 {
 1973    struct iris_context *ice = (struct iris_context *)ctx;
 1974    struct iris_resource *res = (struct iris_resource *) xfer->resource;
 1975    struct iris_transfer *map = (void *) xfer;
 1976 
 1977    if (map->staging)
 1978       iris_flush_staging_region(xfer, box);
 1979 
 1980    uint32_t history_flush = 0;
 1981 
 1982    if (res->base.target == PIPE_BUFFER) {
 1983       if (map->staging)
 1984          history_flush |= PIPE_CONTROL_RENDER_TARGET_FLUSH;
 1985 
 1986       if (map->dest_had_defined_contents)
 1987          history_flush |= iris_flush_bits_for_history(res);
 1988 
 1989       util_range_add(&res->base, &res->valid_buffer_range, box->x, box->x + box->width);
 1990    }
 1991 
 1992    if (history_flush & ~PIPE_CONTROL_CS_STALL) {
 1993       for (int i = 0; i < IRIS_BATCH_COUNT; i++) {
 1994          struct iris_batch *batch = &ice->batches[i];
 1995          if (batch->contains_draw || batch->cache.render->entries) {
 1996             iris_batch_maybe_flush(batch, 24);
 1997             iris_emit_pipe_control_flush(batch,
 1998                                          "cache history: transfer flush",
 1999                                          history_flush);
 2000          }
 2001       }
 2002    }
 2003 
 2004    /* Make sure we flag constants dirty even if there's no need to emit
 2005     * any PIPE_CONTROLs to a batch.
 2006     */
 2007    iris_dirty_for_history(ice, res);
 2008 }
 2009 
 2010 static void
 2011 iris_transfer_unmap(struct pipe_context *ctx, struct pipe_transfer *xfer)
 2012 {
 2013    struct iris_context *ice = (struct iris_context *)ctx;
 2014    struct iris_transfer *map = (void *) xfer;
 2015 
 2016    if (!(xfer->usage & (PIPE_TRANSFER_FLUSH_EXPLICIT |
 2017                         PIPE_TRANSFER_COHERENT))) {
 2018       struct pipe_box flush_box = {
 2019          .x = 0, .y = 0, .z = 0,
 2020          .width  = xfer->box.width,
 2021          .height = xfer->box.height,
 2022          .depth  = xfer->box.depth,
 2023       };
 2024       iris_transfer_flush_region(ctx, xfer, &flush_box);
 2025    }
 2026 
 2027    if (map->unmap)
 2028       map->unmap(map);
 2029 
 2030    pipe_resource_reference(&xfer->resource, NULL);
 2031    slab_free(&ice->transfer_pool, map);
 2032 }
 2033 
 2034 /**
 2035  * Mark state dirty that needs to be re-emitted when a resource is written.
 2036  */
 2037 void
 2038 iris_dirty_for_history(struct iris_context *ice,
 2039                        struct iris_resource *res)
 2040 {
 2041    uint64_t dirty = 0ull;
 2042 
 2043    if (res->bind_history & PIPE_BIND_CONSTANT_BUFFER) {
 2044       dirty |= ((uint64_t)res->bind_stages) << IRIS_SHIFT_FOR_DIRTY_CONSTANTS;
 2045    }
 2046 
 2047    ice->state.dirty |= dirty;
 2048 }
 2049 
 2050 /**
 2051  * Produce a set of PIPE_CONTROL bits which ensure data written to a
 2052  * resource becomes visible, and any stale read cache data is invalidated.
 2053  */
 2054 uint32_t
 2055 iris_flush_bits_for_history(struct iris_resource *res)
 2056 {
 2057    uint32_t flush = PIPE_CONTROL_CS_STALL;
 2058 
 2059    if (res->bind_history & PIPE_BIND_CONSTANT_BUFFER) {
 2060       flush |= PIPE_CONTROL_CONST_CACHE_INVALIDATE |
 2061                PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
 2062    }
 2063 
 2064    if (res->bind_history & PIPE_BIND_SAMPLER_VIEW)
 2065       flush |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
 2066 
 2067    if (res->bind_history & (PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_INDEX_BUFFER))
 2068       flush |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
 2069 
 2070    if (res->bind_history & (PIPE_BIND_SHADER_BUFFER | PIPE_BIND_SHADER_IMAGE))
 2071       flush |= PIPE_CONTROL_DATA_CACHE_FLUSH;
 2072 
 2073    return flush;
 2074 }
 2075 
 2076 void
 2077 iris_flush_and_dirty_for_history(struct iris_context *ice,
 2078                                  struct iris_batch *batch,
 2079                                  struct iris_resource *res,
 2080                                  uint32_t extra_flags,
 2081                                  const char *reason)
 2082 {
 2083    if (res->base.target != PIPE_BUFFER)
 2084       return;
 2085 
 2086    uint32_t flush = iris_flush_bits_for_history(res) | extra_flags;
 2087 
 2088    iris_emit_pipe_control_flush(batch, reason, flush);
 2089 
 2090    iris_dirty_for_history(ice, res);
 2091 }
 2092 
 2093 bool
 2094 iris_resource_set_clear_color(struct iris_context *ice,
 2095                               struct iris_resource *res,
 2096                               union isl_color_value color)
 2097 {
 2098    if (memcmp(&res->aux.clear_color, &color, sizeof(color)) != 0) {
 2099       res->aux.clear_color = color;
 2100       return true;
 2101    }
 2102 
 2103    return false;
 2104 }
 2105 
 2106 union isl_color_value
 2107 iris_resource_get_clear_color(const struct iris_resource *res,
 2108                               struct iris_bo **clear_color_bo,
 2109                               uint64_t *clear_color_offset)
 2110 {
 2111    assert(res->aux.bo);
 2112 
 2113    if (clear_color_bo)
 2114       *clear_color_bo = res->aux.clear_color_bo;
 2115    if (clear_color_offset)
 2116       *clear_color_offset = res->aux.clear_color_offset;
 2117    return res->aux.clear_color;
 2118 }
 2119 
 2120 static enum pipe_format
 2121 iris_resource_get_internal_format(struct pipe_resource *p_res)
 2122 {
 2123    struct iris_resource *res = (void *) p_res;
 2124    return res->internal_format;
 2125 }
 2126 
 2127 static const struct u_transfer_vtbl transfer_vtbl = {
 2128    .resource_create       = iris_resource_create,
 2129    .resource_destroy      = iris_resource_destroy,
 2130    .transfer_map          = iris_transfer_map,
 2131    .transfer_unmap        = iris_transfer_unmap,
 2132    .transfer_flush_region = iris_transfer_flush_region,
 2133    .get_internal_format   = iris_resource_get_internal_format,
 2134    .set_stencil           = iris_resource_set_separate_stencil,
 2135    .get_stencil           = iris_resource_get_separate_stencil,
 2136 };
 2137 
 2138 void
 2139 iris_init_screen_resource_functions(struct pipe_screen *pscreen)
 2140 {
 2141    pscreen->query_dmabuf_modifiers = iris_query_dmabuf_modifiers;
 2142    pscreen->resource_create_with_modifiers =
 2143       iris_resource_create_with_modifiers;
 2144    pscreen->resource_create = u_transfer_helper_resource_create;
 2145    pscreen->resource_from_user_memory = iris_resource_from_user_memory;
 2146    pscreen->resource_from_handle = iris_resource_from_handle;
 2147    pscreen->resource_get_handle = iris_resource_get_handle;
 2148    pscreen->resource_get_param = iris_resource_get_param;
 2149    pscreen->resource_destroy = u_transfer_helper_resource_destroy;
 2150    pscreen->transfer_helper =
 2151       u_transfer_helper_create(&transfer_vtbl, true, true, false, true);
 2152 }
 2153 
 2154 void
 2155 iris_init_resource_functions(struct pipe_context *ctx)
 2156 {
 2157    ctx->flush_resource = iris_flush_resource;
 2158    ctx->invalidate_resource = iris_invalidate_resource;
 2159    ctx->transfer_map = u_transfer_helper_transfer_map;
 2160    ctx->transfer_flush_region = u_transfer_helper_transfer_flush_region;
 2161    ctx->transfer_unmap = u_transfer_helper_transfer_unmap;
 2162    ctx->buffer_subdata = u_default_buffer_subdata;
 2163    ctx->texture_subdata = u_default_texture_subdata;
 2164 }