"Fossies" - the Fresh Open Source Software Archive

Member "mesa-20.1.8/src/gallium/drivers/radeonsi/si_cp_dma.c" (16 Sep 2020, 23608 Bytes) of package /linux/misc/mesa-20.1.8.tar.xz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "si_cp_dma.c" see the Fossies "Dox" file reference documentation and the last Fossies "Diffs" side-by-side code changes report: 20.1.5_vs_20.2.0-rc1.

    1 /*
    2  * Copyright 2013 Advanced Micro Devices, Inc.
    3  * All Rights Reserved.
    4  *
    5  * Permission is hereby granted, free of charge, to any person obtaining a
    6  * copy of this software and associated documentation files (the "Software"),
    7  * to deal in the Software without restriction, including without limitation
    8  * on the rights to use, copy, modify, merge, publish, distribute, sub
    9  * license, and/or sell copies of the Software, and to permit persons to whom
   10  * the Software is furnished to do so, subject to the following conditions:
   11  *
   12  * The above copyright notice and this permission notice (including the next
   13  * paragraph) shall be included in all copies or substantial portions of the
   14  * Software.
   15  *
   16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   18  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
   19  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
   20  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
   21  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
   22  * USE OR OTHER DEALINGS IN THE SOFTWARE.
   23  */
   24 
   25 #include "si_pipe.h"
   26 #include "sid.h"
   27 
   28 /* Set this if you want the ME to wait until CP DMA is done.
   29  * It should be set on the last CP DMA packet. */
   30 #define CP_DMA_SYNC (1 << 0)
   31 
   32 /* Set this if the source data was used as a destination in a previous CP DMA
   33  * packet. It's for preventing a read-after-write (RAW) hazard between two
   34  * CP DMA packets. */
   35 #define CP_DMA_RAW_WAIT    (1 << 1)
   36 #define CP_DMA_DST_IS_GDS  (1 << 2)
   37 #define CP_DMA_CLEAR       (1 << 3)
   38 #define CP_DMA_PFP_SYNC_ME (1 << 4)
   39 #define CP_DMA_SRC_IS_GDS  (1 << 5)
   40 
   41 /* The max number of bytes that can be copied per packet. */
   42 static inline unsigned cp_dma_max_byte_count(struct si_context *sctx)
   43 {
   44    unsigned max =
   45       sctx->chip_class >= GFX9 ? S_414_BYTE_COUNT_GFX9(~0u) : S_414_BYTE_COUNT_GFX6(~0u);
   46 
   47    /* make it aligned for optimal performance */
   48    return max & ~(SI_CPDMA_ALIGNMENT - 1);
   49 }
   50 
   51 /* Emit a CP DMA packet to do a copy from one buffer to another, or to clear
   52  * a buffer. The size must fit in bits [20:0]. If CP_DMA_CLEAR is set, src_va is a 32-bit
   53  * clear value.
   54  */
   55 static void si_emit_cp_dma(struct si_context *sctx, struct radeon_cmdbuf *cs, uint64_t dst_va,
   56                            uint64_t src_va, unsigned size, unsigned flags,
   57                            enum si_cache_policy cache_policy)
   58 {
   59    uint32_t header = 0, command = 0;
   60 
   61    assert(size <= cp_dma_max_byte_count(sctx));
   62    assert(sctx->chip_class != GFX6 || cache_policy == L2_BYPASS);
   63 
   64    if (sctx->chip_class >= GFX9)
   65       command |= S_414_BYTE_COUNT_GFX9(size);
   66    else
   67       command |= S_414_BYTE_COUNT_GFX6(size);
   68 
   69    /* Sync flags. */
   70    if (flags & CP_DMA_SYNC)
   71       header |= S_411_CP_SYNC(1);
   72    else {
   73       if (sctx->chip_class >= GFX9)
   74          command |= S_414_DISABLE_WR_CONFIRM_GFX9(1);
   75       else
   76          command |= S_414_DISABLE_WR_CONFIRM_GFX6(1);
   77    }
   78 
   79    if (flags & CP_DMA_RAW_WAIT)
   80       command |= S_414_RAW_WAIT(1);
   81 
   82    /* Src and dst flags. */
   83    if (sctx->chip_class >= GFX9 && !(flags & CP_DMA_CLEAR) && src_va == dst_va) {
   84       header |= S_411_DST_SEL(V_411_NOWHERE); /* prefetch only */
   85    } else if (flags & CP_DMA_DST_IS_GDS) {
   86       header |= S_411_DST_SEL(V_411_GDS);
   87       /* GDS increments the address, not CP. */
   88       command |= S_414_DAS(V_414_REGISTER) | S_414_DAIC(V_414_NO_INCREMENT);
   89    } else if (sctx->chip_class >= GFX7 && cache_policy != L2_BYPASS) {
   90       header |=
   91          S_411_DST_SEL(V_411_DST_ADDR_TC_L2) | S_500_DST_CACHE_POLICY(cache_policy == L2_STREAM);
   92    }
   93 
   94    if (flags & CP_DMA_CLEAR) {
   95       header |= S_411_SRC_SEL(V_411_DATA);
   96    } else if (flags & CP_DMA_SRC_IS_GDS) {
   97       header |= S_411_SRC_SEL(V_411_GDS);
   98       /* Both of these are required for GDS. It does increment the address. */
   99       command |= S_414_SAS(V_414_REGISTER) | S_414_SAIC(V_414_NO_INCREMENT);
  100    } else if (sctx->chip_class >= GFX7 && cache_policy != L2_BYPASS) {
  101       header |=
  102          S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2) | S_500_SRC_CACHE_POLICY(cache_policy == L2_STREAM);
  103    }
  104 
  105    if (sctx->chip_class >= GFX7) {
  106       radeon_emit(cs, PKT3(PKT3_DMA_DATA, 5, 0));
  107       radeon_emit(cs, header);
  108       radeon_emit(cs, src_va);       /* SRC_ADDR_LO [31:0] */
  109       radeon_emit(cs, src_va >> 32); /* SRC_ADDR_HI [31:0] */
  110       radeon_emit(cs, dst_va);       /* DST_ADDR_LO [31:0] */
  111       radeon_emit(cs, dst_va >> 32); /* DST_ADDR_HI [31:0] */
  112       radeon_emit(cs, command);
  113    } else {
  114       header |= S_411_SRC_ADDR_HI(src_va >> 32);
  115 
  116       radeon_emit(cs, PKT3(PKT3_CP_DMA, 4, 0));
  117       radeon_emit(cs, src_va);                  /* SRC_ADDR_LO [31:0] */
  118       radeon_emit(cs, header);                  /* SRC_ADDR_HI [15:0] + flags. */
  119       radeon_emit(cs, dst_va);                  /* DST_ADDR_LO [31:0] */
  120       radeon_emit(cs, (dst_va >> 32) & 0xffff); /* DST_ADDR_HI [15:0] */
  121       radeon_emit(cs, command);
  122    }
  123 
  124    /* CP DMA is executed in ME, but index buffers are read by PFP.
  125     * This ensures that ME (CP DMA) is idle before PFP starts fetching
  126     * indices. If we wanted to execute CP DMA in PFP, this packet
  127     * should precede it.
  128     */
  129    if (sctx->has_graphics && flags & CP_DMA_PFP_SYNC_ME) {
  130       radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0));
  131       radeon_emit(cs, 0);
  132    }
  133 }
  134 
  135 void si_cp_dma_wait_for_idle(struct si_context *sctx)
  136 {
  137    /* Issue a dummy DMA that copies zero bytes.
  138     *
  139     * The DMA engine will see that there's no work to do and skip this
  140     * DMA request, however, the CP will see the sync flag and still wait
  141     * for all DMAs to complete.
  142     */
  143    si_emit_cp_dma(sctx, sctx->gfx_cs, 0, 0, 0, CP_DMA_SYNC, L2_BYPASS);
  144 }
  145 
  146 static void si_cp_dma_prepare(struct si_context *sctx, struct pipe_resource *dst,
  147                               struct pipe_resource *src, unsigned byte_count,
  148                               uint64_t remaining_size, unsigned user_flags, enum si_coherency coher,
  149                               bool *is_first, unsigned *packet_flags)
  150 {
  151    /* Fast exit for a CPDMA prefetch. */
  152    if ((user_flags & SI_CPDMA_SKIP_ALL) == SI_CPDMA_SKIP_ALL) {
  153       *is_first = false;
  154       return;
  155    }
  156 
  157    if (!(user_flags & SI_CPDMA_SKIP_BO_LIST_UPDATE)) {
  158       /* Count memory usage in so that need_cs_space can take it into account. */
  159       if (dst)
  160          si_context_add_resource_size(sctx, dst);
  161       if (src)
  162          si_context_add_resource_size(sctx, src);
  163    }
  164 
  165    if (!(user_flags & SI_CPDMA_SKIP_CHECK_CS_SPACE))
  166       si_need_gfx_cs_space(sctx);
  167 
  168    /* This must be done after need_cs_space. */
  169    if (!(user_flags & SI_CPDMA_SKIP_BO_LIST_UPDATE)) {
  170       if (dst)
  171          radeon_add_to_buffer_list(sctx, sctx->gfx_cs, si_resource(dst), RADEON_USAGE_WRITE,
  172                                    RADEON_PRIO_CP_DMA);
  173       if (src)
  174          radeon_add_to_buffer_list(sctx, sctx->gfx_cs, si_resource(src), RADEON_USAGE_READ,
  175                                    RADEON_PRIO_CP_DMA);
  176    }
  177 
  178    /* Flush the caches for the first copy only.
  179     * Also wait for the previous CP DMA operations.
  180     */
  181    if (!(user_flags & SI_CPDMA_SKIP_GFX_SYNC) && sctx->flags)
  182       sctx->emit_cache_flush(sctx);
  183 
  184    if (!(user_flags & SI_CPDMA_SKIP_SYNC_BEFORE) && *is_first && !(*packet_flags & CP_DMA_CLEAR))
  185       *packet_flags |= CP_DMA_RAW_WAIT;
  186 
  187    *is_first = false;
  188 
  189    /* Do the synchronization after the last dma, so that all data
  190     * is written to memory.
  191     */
  192    if (!(user_flags & SI_CPDMA_SKIP_SYNC_AFTER) && byte_count == remaining_size) {
  193       *packet_flags |= CP_DMA_SYNC;
  194 
  195       if (coher == SI_COHERENCY_SHADER)
  196          *packet_flags |= CP_DMA_PFP_SYNC_ME;
  197    }
  198 }
  199 
  200 void si_cp_dma_clear_buffer(struct si_context *sctx, struct radeon_cmdbuf *cs,
  201                             struct pipe_resource *dst, uint64_t offset, uint64_t size,
  202                             unsigned value, unsigned user_flags, enum si_coherency coher,
  203                             enum si_cache_policy cache_policy)
  204 {
  205    struct si_resource *sdst = si_resource(dst);
  206    uint64_t va = (sdst ? sdst->gpu_address : 0) + offset;
  207    bool is_first = true;
  208 
  209    assert(size && size % 4 == 0);
  210 
  211    /* Mark the buffer range of destination as valid (initialized),
  212     * so that transfer_map knows it should wait for the GPU when mapping
  213     * that range. */
  214    if (sdst)
  215       util_range_add(dst, &sdst->valid_buffer_range, offset, offset + size);
  216 
  217    /* Flush the caches. */
  218    if (sdst && !(user_flags & SI_CPDMA_SKIP_GFX_SYNC)) {
  219       sctx->flags |= SI_CONTEXT_PS_PARTIAL_FLUSH | SI_CONTEXT_CS_PARTIAL_FLUSH |
  220                      si_get_flush_flags(sctx, coher, cache_policy);
  221    }
  222 
  223    while (size) {
  224       unsigned byte_count = MIN2(size, cp_dma_max_byte_count(sctx));
  225       unsigned dma_flags = CP_DMA_CLEAR | (sdst ? 0 : CP_DMA_DST_IS_GDS);
  226 
  227       si_cp_dma_prepare(sctx, dst, NULL, byte_count, size, user_flags, coher, &is_first,
  228                         &dma_flags);
  229 
  230       /* Emit the clear packet. */
  231       si_emit_cp_dma(sctx, cs, va, value, byte_count, dma_flags, cache_policy);
  232 
  233       size -= byte_count;
  234       va += byte_count;
  235    }
  236 
  237    if (sdst && cache_policy != L2_BYPASS)
  238       sdst->TC_L2_dirty = true;
  239 
  240    /* If it's not a framebuffer fast clear... */
  241    if (coher == SI_COHERENCY_SHADER) {
  242       sctx->num_cp_dma_calls++;
  243       si_prim_discard_signal_next_compute_ib_start(sctx);
  244    }
  245 }
  246 
  247 /**
  248  * Realign the CP DMA engine. This must be done after a copy with an unaligned
  249  * size.
  250  *
  251  * \param size  Remaining size to the CP DMA alignment.
  252  */
  253 static void si_cp_dma_realign_engine(struct si_context *sctx, unsigned size, unsigned user_flags,
  254                                      enum si_coherency coher, enum si_cache_policy cache_policy,
  255                                      bool *is_first)
  256 {
  257    uint64_t va;
  258    unsigned dma_flags = 0;
  259    unsigned scratch_size = SI_CPDMA_ALIGNMENT * 2;
  260 
  261    assert(size < SI_CPDMA_ALIGNMENT);
  262 
  263    /* Use the scratch buffer as the dummy buffer. The 3D engine should be
  264     * idle at this point.
  265     */
  266    if (!sctx->scratch_buffer || sctx->scratch_buffer->b.b.width0 < scratch_size) {
  267       si_resource_reference(&sctx->scratch_buffer, NULL);
  268       sctx->scratch_buffer = si_aligned_buffer_create(&sctx->screen->b, SI_RESOURCE_FLAG_UNMAPPABLE,
  269                                                       PIPE_USAGE_DEFAULT, scratch_size, 256);
  270       if (!sctx->scratch_buffer)
  271          return;
  272 
  273       si_mark_atom_dirty(sctx, &sctx->atoms.s.scratch_state);
  274    }
  275 
  276    si_cp_dma_prepare(sctx, &sctx->scratch_buffer->b.b, &sctx->scratch_buffer->b.b, size, size,
  277                      user_flags, coher, is_first, &dma_flags);
  278 
  279    va = sctx->scratch_buffer->gpu_address;
  280    si_emit_cp_dma(sctx, sctx->gfx_cs, va, va + SI_CPDMA_ALIGNMENT, size, dma_flags, cache_policy);
  281 }
  282 
  283 /**
  284  * Do memcpy between buffers using CP DMA.
  285  * If src or dst is NULL, it means read or write GDS, respectively.
  286  *
  287  * \param user_flags    bitmask of SI_CPDMA_*
  288  */
  289 void si_cp_dma_copy_buffer(struct si_context *sctx, struct pipe_resource *dst,
  290                            struct pipe_resource *src, uint64_t dst_offset, uint64_t src_offset,
  291                            unsigned size, unsigned user_flags, enum si_coherency coher,
  292                            enum si_cache_policy cache_policy)
  293 {
  294    uint64_t main_dst_offset, main_src_offset;
  295    unsigned skipped_size = 0;
  296    unsigned realign_size = 0;
  297    unsigned gds_flags = (dst ? 0 : CP_DMA_DST_IS_GDS) | (src ? 0 : CP_DMA_SRC_IS_GDS);
  298    bool is_first = true;
  299 
  300    assert(size);
  301 
  302    if (dst) {
  303       /* Skip this for the L2 prefetch. */
  304       if (dst != src || dst_offset != src_offset) {
  305          /* Mark the buffer range of destination as valid (initialized),
  306           * so that transfer_map knows it should wait for the GPU when mapping
  307           * that range. */
  308          util_range_add(dst, &si_resource(dst)->valid_buffer_range, dst_offset, dst_offset + size);
  309       }
  310 
  311       dst_offset += si_resource(dst)->gpu_address;
  312    }
  313    if (src)
  314       src_offset += si_resource(src)->gpu_address;
  315 
  316    /* The workarounds aren't needed on Fiji and beyond. */
  317    if (sctx->family <= CHIP_CARRIZO || sctx->family == CHIP_STONEY) {
  318       /* If the size is not aligned, we must add a dummy copy at the end
  319        * just to align the internal counter. Otherwise, the DMA engine
  320        * would slow down by an order of magnitude for following copies.
  321        */
  322       if (size % SI_CPDMA_ALIGNMENT)
  323          realign_size = SI_CPDMA_ALIGNMENT - (size % SI_CPDMA_ALIGNMENT);
  324 
  325       /* If the copy begins unaligned, we must start copying from the next
  326        * aligned block and the skipped part should be copied after everything
  327        * else has been copied. Only the src alignment matters, not dst.
  328        *
  329        * GDS doesn't need the source address to be aligned.
  330        */
  331       if (src && src_offset % SI_CPDMA_ALIGNMENT) {
  332          skipped_size = SI_CPDMA_ALIGNMENT - (src_offset % SI_CPDMA_ALIGNMENT);
  333          /* The main part will be skipped if the size is too small. */
  334          skipped_size = MIN2(skipped_size, size);
  335          size -= skipped_size;
  336       }
  337    }
  338 
  339    /* Flush the caches. */
  340    if ((dst || src) && !(user_flags & SI_CPDMA_SKIP_GFX_SYNC)) {
  341       sctx->flags |= SI_CONTEXT_PS_PARTIAL_FLUSH | SI_CONTEXT_CS_PARTIAL_FLUSH |
  342                      si_get_flush_flags(sctx, coher, cache_policy);
  343    }
  344 
  345    /* This is the main part doing the copying. Src is always aligned. */
  346    main_dst_offset = dst_offset + skipped_size;
  347    main_src_offset = src_offset + skipped_size;
  348 
  349    while (size) {
  350       unsigned byte_count = MIN2(size, cp_dma_max_byte_count(sctx));
  351       unsigned dma_flags = gds_flags;
  352 
  353       si_cp_dma_prepare(sctx, dst, src, byte_count, size + skipped_size + realign_size, user_flags,
  354                         coher, &is_first, &dma_flags);
  355 
  356       si_emit_cp_dma(sctx, sctx->gfx_cs, main_dst_offset, main_src_offset, byte_count, dma_flags,
  357                      cache_policy);
  358 
  359       size -= byte_count;
  360       main_src_offset += byte_count;
  361       main_dst_offset += byte_count;
  362    }
  363 
  364    /* Copy the part we skipped because src wasn't aligned. */
  365    if (skipped_size) {
  366       unsigned dma_flags = gds_flags;
  367 
  368       si_cp_dma_prepare(sctx, dst, src, skipped_size, skipped_size + realign_size, user_flags,
  369                         coher, &is_first, &dma_flags);
  370 
  371       si_emit_cp_dma(sctx, sctx->gfx_cs, dst_offset, src_offset, skipped_size, dma_flags,
  372                      cache_policy);
  373    }
  374 
  375    /* Finally, realign the engine if the size wasn't aligned. */
  376    if (realign_size) {
  377       si_cp_dma_realign_engine(sctx, realign_size, user_flags, coher, cache_policy, &is_first);
  378    }
  379 
  380    if (dst && cache_policy != L2_BYPASS)
  381       si_resource(dst)->TC_L2_dirty = true;
  382 
  383    /* If it's not a prefetch or GDS copy... */
  384    if (dst && src && (dst != src || dst_offset != src_offset)) {
  385       sctx->num_cp_dma_calls++;
  386       si_prim_discard_signal_next_compute_ib_start(sctx);
  387    }
  388 }
  389 
  390 void cik_prefetch_TC_L2_async(struct si_context *sctx, struct pipe_resource *buf, uint64_t offset,
  391                               unsigned size)
  392 {
  393    assert(sctx->chip_class >= GFX7);
  394 
  395    si_cp_dma_copy_buffer(sctx, buf, buf, offset, offset, size, SI_CPDMA_SKIP_ALL,
  396                          SI_COHERENCY_SHADER, L2_LRU);
  397 }
  398 
  399 static void cik_prefetch_shader_async(struct si_context *sctx, struct si_pm4_state *state)
  400 {
  401    struct pipe_resource *bo = &state->bo[0]->b.b;
  402    assert(state->nbo == 1);
  403 
  404    cik_prefetch_TC_L2_async(sctx, bo, 0, bo->width0);
  405 }
  406 
  407 static void cik_prefetch_VBO_descriptors(struct si_context *sctx)
  408 {
  409    if (!sctx->vertex_elements || !sctx->vertex_elements->vb_desc_list_alloc_size)
  410       return;
  411 
  412    cik_prefetch_TC_L2_async(sctx, &sctx->vb_descriptors_buffer->b.b, sctx->vb_descriptors_offset,
  413                             sctx->vertex_elements->vb_desc_list_alloc_size);
  414 }
  415 
  416 /**
  417  * Prefetch shaders and VBO descriptors.
  418  *
  419  * \param vertex_stage_only  Whether only the the API VS and VBO descriptors
  420  *                           should be prefetched.
  421  */
  422 void cik_emit_prefetch_L2(struct si_context *sctx, bool vertex_stage_only)
  423 {
  424    unsigned mask = sctx->prefetch_L2_mask;
  425    assert(mask);
  426 
  427    /* Prefetch shaders and VBO descriptors to TC L2. */
  428    if (sctx->chip_class >= GFX9) {
  429       /* Choose the right spot for the VBO prefetch. */
  430       if (sctx->queued.named.hs) {
  431          if (mask & SI_PREFETCH_HS)
  432             cik_prefetch_shader_async(sctx, sctx->queued.named.hs);
  433          if (mask & SI_PREFETCH_VBO_DESCRIPTORS)
  434             cik_prefetch_VBO_descriptors(sctx);
  435          if (vertex_stage_only) {
  436             sctx->prefetch_L2_mask &= ~(SI_PREFETCH_HS | SI_PREFETCH_VBO_DESCRIPTORS);
  437             return;
  438          }
  439 
  440          if (mask & SI_PREFETCH_GS)
  441             cik_prefetch_shader_async(sctx, sctx->queued.named.gs);
  442          if (mask & SI_PREFETCH_VS)
  443             cik_prefetch_shader_async(sctx, sctx->queued.named.vs);
  444       } else if (sctx->queued.named.gs) {
  445          if (mask & SI_PREFETCH_GS)
  446             cik_prefetch_shader_async(sctx, sctx->queued.named.gs);
  447          if (mask & SI_PREFETCH_VBO_DESCRIPTORS)
  448             cik_prefetch_VBO_descriptors(sctx);
  449          if (vertex_stage_only) {
  450             sctx->prefetch_L2_mask &= ~(SI_PREFETCH_GS | SI_PREFETCH_VBO_DESCRIPTORS);
  451             return;
  452          }
  453 
  454          if (mask & SI_PREFETCH_VS)
  455             cik_prefetch_shader_async(sctx, sctx->queued.named.vs);
  456       } else {
  457          if (mask & SI_PREFETCH_VS)
  458             cik_prefetch_shader_async(sctx, sctx->queued.named.vs);
  459          if (mask & SI_PREFETCH_VBO_DESCRIPTORS)
  460             cik_prefetch_VBO_descriptors(sctx);
  461          if (vertex_stage_only) {
  462             sctx->prefetch_L2_mask &= ~(SI_PREFETCH_VS | SI_PREFETCH_VBO_DESCRIPTORS);
  463             return;
  464          }
  465       }
  466    } else {
  467       /* GFX6-GFX8 */
  468       /* Choose the right spot for the VBO prefetch. */
  469       if (sctx->tes_shader.cso) {
  470          if (mask & SI_PREFETCH_LS)
  471             cik_prefetch_shader_async(sctx, sctx->queued.named.ls);
  472          if (mask & SI_PREFETCH_VBO_DESCRIPTORS)
  473             cik_prefetch_VBO_descriptors(sctx);
  474          if (vertex_stage_only) {
  475             sctx->prefetch_L2_mask &= ~(SI_PREFETCH_LS | SI_PREFETCH_VBO_DESCRIPTORS);
  476             return;
  477          }
  478 
  479          if (mask & SI_PREFETCH_HS)
  480             cik_prefetch_shader_async(sctx, sctx->queued.named.hs);
  481          if (mask & SI_PREFETCH_ES)
  482             cik_prefetch_shader_async(sctx, sctx->queued.named.es);
  483          if (mask & SI_PREFETCH_GS)
  484             cik_prefetch_shader_async(sctx, sctx->queued.named.gs);
  485          if (mask & SI_PREFETCH_VS)
  486             cik_prefetch_shader_async(sctx, sctx->queued.named.vs);
  487       } else if (sctx->gs_shader.cso) {
  488          if (mask & SI_PREFETCH_ES)
  489             cik_prefetch_shader_async(sctx, sctx->queued.named.es);
  490          if (mask & SI_PREFETCH_VBO_DESCRIPTORS)
  491             cik_prefetch_VBO_descriptors(sctx);
  492          if (vertex_stage_only) {
  493             sctx->prefetch_L2_mask &= ~(SI_PREFETCH_ES | SI_PREFETCH_VBO_DESCRIPTORS);
  494             return;
  495          }
  496 
  497          if (mask & SI_PREFETCH_GS)
  498             cik_prefetch_shader_async(sctx, sctx->queued.named.gs);
  499          if (mask & SI_PREFETCH_VS)
  500             cik_prefetch_shader_async(sctx, sctx->queued.named.vs);
  501       } else {
  502          if (mask & SI_PREFETCH_VS)
  503             cik_prefetch_shader_async(sctx, sctx->queued.named.vs);
  504          if (mask & SI_PREFETCH_VBO_DESCRIPTORS)
  505             cik_prefetch_VBO_descriptors(sctx);
  506          if (vertex_stage_only) {
  507             sctx->prefetch_L2_mask &= ~(SI_PREFETCH_VS | SI_PREFETCH_VBO_DESCRIPTORS);
  508             return;
  509          }
  510       }
  511    }
  512 
  513    if (mask & SI_PREFETCH_PS)
  514       cik_prefetch_shader_async(sctx, sctx->queued.named.ps);
  515 
  516    sctx->prefetch_L2_mask = 0;
  517 }
  518 
  519 void si_test_gds(struct si_context *sctx)
  520 {
  521    struct pipe_context *ctx = &sctx->b;
  522    struct pipe_resource *src, *dst;
  523    unsigned r[4] = {};
  524    unsigned offset = debug_get_num_option("OFFSET", 16);
  525 
  526    src = pipe_buffer_create(ctx->screen, 0, PIPE_USAGE_DEFAULT, 16);
  527    dst = pipe_buffer_create(ctx->screen, 0, PIPE_USAGE_DEFAULT, 16);
  528    si_cp_dma_clear_buffer(sctx, sctx->gfx_cs, src, 0, 4, 0xabcdef01, 0, SI_COHERENCY_SHADER,
  529                           L2_BYPASS);
  530    si_cp_dma_clear_buffer(sctx, sctx->gfx_cs, src, 4, 4, 0x23456789, 0, SI_COHERENCY_SHADER,
  531                           L2_BYPASS);
  532    si_cp_dma_clear_buffer(sctx, sctx->gfx_cs, src, 8, 4, 0x87654321, 0, SI_COHERENCY_SHADER,
  533                           L2_BYPASS);
  534    si_cp_dma_clear_buffer(sctx, sctx->gfx_cs, src, 12, 4, 0xfedcba98, 0, SI_COHERENCY_SHADER,
  535                           L2_BYPASS);
  536    si_cp_dma_clear_buffer(sctx, sctx->gfx_cs, dst, 0, 16, 0xdeadbeef, 0, SI_COHERENCY_SHADER,
  537                           L2_BYPASS);
  538 
  539    si_cp_dma_copy_buffer(sctx, NULL, src, offset, 0, 16, 0, SI_COHERENCY_NONE, L2_BYPASS);
  540    si_cp_dma_copy_buffer(sctx, dst, NULL, 0, offset, 16, 0, SI_COHERENCY_NONE, L2_BYPASS);
  541 
  542    pipe_buffer_read(ctx, dst, 0, sizeof(r), r);
  543    printf("GDS copy  = %08x %08x %08x %08x -> %s\n", r[0], r[1], r[2], r[3],
  544           r[0] == 0xabcdef01 && r[1] == 0x23456789 && r[2] == 0x87654321 && r[3] == 0xfedcba98
  545              ? "pass"
  546              : "fail");
  547 
  548    si_cp_dma_clear_buffer(sctx, sctx->gfx_cs, NULL, offset, 16, 0xc1ea4146, 0, SI_COHERENCY_NONE,
  549                           L2_BYPASS);
  550    si_cp_dma_copy_buffer(sctx, dst, NULL, 0, offset, 16, 0, SI_COHERENCY_NONE, L2_BYPASS);
  551 
  552    pipe_buffer_read(ctx, dst, 0, sizeof(r), r);
  553    printf("GDS clear = %08x %08x %08x %08x -> %s\n", r[0], r[1], r[2], r[3],
  554           r[0] == 0xc1ea4146 && r[1] == 0xc1ea4146 && r[2] == 0xc1ea4146 && r[3] == 0xc1ea4146
  555              ? "pass"
  556              : "fail");
  557 
  558    pipe_resource_reference(&src, NULL);
  559    pipe_resource_reference(&dst, NULL);
  560    exit(0);
  561 }
  562 
  563 void si_cp_write_data(struct si_context *sctx, struct si_resource *buf, unsigned offset,
  564                       unsigned size, unsigned dst_sel, unsigned engine, const void *data)
  565 {
  566    struct radeon_cmdbuf *cs = sctx->gfx_cs;
  567 
  568    assert(offset % 4 == 0);
  569    assert(size % 4 == 0);
  570 
  571    if (sctx->chip_class == GFX6 && dst_sel == V_370_MEM)
  572       dst_sel = V_370_MEM_GRBM;
  573 
  574    radeon_add_to_buffer_list(sctx, cs, buf, RADEON_USAGE_WRITE, RADEON_PRIO_CP_DMA);
  575    uint64_t va = buf->gpu_address + offset;
  576 
  577    radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 2 + size / 4, 0));
  578    radeon_emit(cs, S_370_DST_SEL(dst_sel) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(engine));
  579    radeon_emit(cs, va);
  580    radeon_emit(cs, va >> 32);
  581    radeon_emit_array(cs, (const uint32_t *)data, size / 4);
  582 }
  583 
  584 void si_cp_copy_data(struct si_context *sctx, struct radeon_cmdbuf *cs, unsigned dst_sel,
  585                      struct si_resource *dst, unsigned dst_offset, unsigned src_sel,
  586                      struct si_resource *src, unsigned src_offset)
  587 {
  588    /* cs can point to the compute IB, which has the buffer list in gfx_cs. */
  589    if (dst) {
  590       radeon_add_to_buffer_list(sctx, sctx->gfx_cs, dst, RADEON_USAGE_WRITE, RADEON_PRIO_CP_DMA);
  591    }
  592    if (src) {
  593       radeon_add_to_buffer_list(sctx, sctx->gfx_cs, src, RADEON_USAGE_READ, RADEON_PRIO_CP_DMA);
  594    }
  595 
  596    uint64_t dst_va = (dst ? dst->gpu_address : 0ull) + dst_offset;
  597    uint64_t src_va = (src ? src->gpu_address : 0ull) + src_offset;
  598 
  599    radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
  600    radeon_emit(cs, COPY_DATA_SRC_SEL(src_sel) | COPY_DATA_DST_SEL(dst_sel) | COPY_DATA_WR_CONFIRM);
  601    radeon_emit(cs, src_va);
  602    radeon_emit(cs, src_va >> 32);
  603    radeon_emit(cs, dst_va);
  604    radeon_emit(cs, dst_va >> 32);
  605 }