"Fossies" - the Fresh Open Source Software Archive

Member "stress-ng-0.13.05/stress-io-uring.c" (11 Oct 2021, 21727 Bytes) of package /linux/privat/stress-ng-0.13.05.tar.xz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "stress-io-uring.c" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 0.13.04_vs_0.13.05.

    1 /*
    2  * Copyright (C) 2013-2021 Canonical, Ltd.
    3  *
    4  * This program is free software; you can redistribute it and/or
    5  * modify it under the terms of the GNU General Public License
    6  * as published by the Free Software Foundation; either version 2
    7  * of the License, or (at your option) any later version.
    8  *
    9  * This program is distributed in the hope that it will be useful,
   10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   12  * GNU General Public License for more details.
   13  *
   14  * You should have received a copy of the GNU General Public License
   15  * along with this program; if not, write to the Free Software
   16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
   17  *
   18  * This code is a complete clean re-write of the stress tool by
   19  * Colin Ian King <colin.king@canonical.com> and attempts to be
   20  * backwardly compatible with the stress tool by Amos Waterland
   21  * <apw@rossby.metr.ou.edu> but has more stress tests and more
   22  * functionality.
   23  *
   24  */
   25 #include "stress-ng.h"
   26 #include "io-uring.h"
   27 
   28 static const stress_help_t help[] = {
   29     { NULL, "io-uring N",       "start N workers that issue io-uring I/O requests" },
   30     { NULL, "io-uring-ops N",   "stop after N bogo io-uring I/O requests" },
   31     { NULL, NULL,       NULL }
   32 };
   33 
   34 #if defined(HAVE_LINUX_IO_URING_H) &&   \
   35     defined(__NR_io_uring_enter) && \
   36     defined(__NR_io_uring_setup) && \
   37     defined(IORING_OFF_SQ_RING) &&  \
   38     defined(IORING_OFF_CQ_RING) &&  \
   39     defined(IORING_OFF_SQES) &&     \
   40     defined(HAVE_POSIX_MEMALIGN) && \
   41     (defined(HAVE_IORING_OP_WRITEV) ||  \
   42      defined(HAVE_IORING_OP_READV) ||   \
   43      defined(HAVE_IORING_OP_WRITE) ||   \
   44      defined(HAVE_IORING_OP_READ) ||    \
   45      defined(HAVE_IORING_OP_FSYNC) ||   \
   46      defined(HAVE_IORING_OP_NOP) || \
   47      defined(HAVE_IORING_OP_FALLOCATE) || \
   48      defined(HAVE_IORING_OP_FADVISE) || \
   49      defined(HAVE_IORING_OP_CLOSE) ||   \
   50      defined(HAVE_IORING_OP_MADVISE) || \
   51      defined(HAVE_IORING_OP_STATX) ||   \
   52      defined(HAVE_IORING_OP_SYNC_FILE_RANGE))
   53 
   54 
   55 
   56 /*
   57  *  io uring file info
   58  */
   59 typedef struct {
   60     int fd;         /* file descriptor */
   61     struct iovec *iovecs;   /* iovecs array 1 per block to submit */
   62     size_t iovecs_sz;   /* size of iovecs allocation */
   63     off_t file_size;    /* size of the file (bytes) */
   64     uint32_t blocks;    /* number of blocks to action */
   65     size_t block_size;  /* per block size */
   66 } stress_io_uring_file_t;
   67 
   68 /*
   69  * io uring submission queue info
   70  */
   71 typedef struct {
   72     unsigned *head;
   73     unsigned *tail;
   74     unsigned *ring_mask;
   75     unsigned *ring_entries;
   76     unsigned *flags;
   77     unsigned *array;
   78 } stress_uring_io_sq_ring_t;
   79 
   80 /*
   81  * io uring completion queue info
   82  */
   83 typedef struct {
   84     unsigned *head;
   85     unsigned *tail;
   86     unsigned *ring_mask;
   87     unsigned *ring_entries;
   88     struct io_uring_cqe *cqes;
   89 } stress_uring_io_cq_ring_t;
   90 
   91 /*
   92  *  io uring submission info
   93  */
   94 typedef struct {
   95     stress_uring_io_sq_ring_t sq_ring;
   96     stress_uring_io_cq_ring_t cq_ring;
   97     struct io_uring_sqe *sqes_mmap;
   98     void *sq_mmap;
   99     void *cq_mmap;
  100     int io_uring_fd;
  101     size_t sq_size;
  102     size_t cq_size;
  103     size_t sqes_size;
  104 } stress_io_uring_submit_t;
  105 
  106 typedef void (*stress_io_uring_setup)(stress_io_uring_file_t *io_uring_file, struct io_uring_sqe *sqe);
  107 
  108 /*
  109  *  opcode to human readable name lookup
  110  */
  111 typedef struct {
  112     const uint8_t opcode;           /* opcode */
  113     const char *name;           /* stringified opcode name */
  114     const stress_io_uring_setup setup_func; /* setup function */
  115 } stress_io_uring_setup_info_t;
  116 
  117 static const char *stress_io_uring_opcode_name(const uint8_t opcode);
  118 
  119 /*
  120  *  shim_io_uring_setup
  121  *  wrapper for io_uring_setup()
  122  */
  123 static int shim_io_uring_setup(unsigned entries, struct io_uring_params *p)
  124 {
  125     return (int)syscall(__NR_io_uring_setup, entries, p);
  126 }
  127 
  128 /*
  129  *  shim_io_uring_enter
  130  *  wrapper for io_uring_enter()
  131  */
  132 static int shim_io_uring_enter(
  133     int fd,
  134     unsigned int to_submit,
  135     unsigned int min_complete,
  136     unsigned int flags)
  137 {
  138     return (int)syscall(__NR_io_uring_enter, fd, to_submit,
  139         min_complete, flags, NULL, 0);
  140 }
  141 
  142 /*
  143  *  stress_io_uring_unmap_iovecs()
  144  *  free uring file iovecs
  145  */
  146 static void stress_io_uring_unmap_iovecs(stress_io_uring_file_t *io_uring_file)
  147 {
  148     if (io_uring_file->iovecs) {
  149         size_t i;
  150 
  151         for (i = 0; i < io_uring_file->blocks; i++) {
  152             if (io_uring_file->iovecs[i].iov_base) {
  153                 (void)munmap((void *)io_uring_file->iovecs[i].iov_base, io_uring_file->block_size);
  154                 io_uring_file->iovecs[i].iov_base = NULL;
  155             }
  156         }
  157         (void)munmap((void *)io_uring_file->iovecs, io_uring_file->iovecs_sz);
  158     }
  159     io_uring_file->iovecs = NULL;
  160 }
  161 
  162 /*
  163  *  Avoid GCCism of void * pointer arithmetic by casting to
  164  *  uint8_t *, doing the offset and then casting back to void *
  165  */
  166 #define VOID_ADDR_OFFSET(addr, offset)  \
  167     ((void *)(((uint8_t *)addr) + offset))
  168 
  169 /*
  170  *  stress_setup_io_uring()
  171  *  setup the io uring
  172  */
  173 static int stress_setup_io_uring(
  174     const stress_args_t *args,
  175     stress_io_uring_submit_t *submit)
  176 {
  177     stress_uring_io_sq_ring_t *sring = &submit->sq_ring;
  178     stress_uring_io_cq_ring_t *cring = &submit->cq_ring;
  179     struct io_uring_params p;
  180 
  181     (void)memset(&p, 0, sizeof(p));
  182     submit->io_uring_fd = shim_io_uring_setup(1, &p);
  183     if (submit->io_uring_fd < 0) {
  184         if (errno == ENOSYS) {
  185             pr_inf_skip("%s: io-uring not supported by the kernel, skipping stressor\n",
  186                 args->name);
  187             return EXIT_NOT_IMPLEMENTED;
  188         }
  189         pr_err("%s: io_uring_setup failed, errno=%d (%s)\n",
  190             args->name, errno, strerror(errno));
  191         return EXIT_FAILURE;
  192     }
  193     submit->sq_size = p.sq_off.array + p.sq_entries * sizeof(unsigned);
  194     submit->cq_size = p.cq_off.cqes + p.cq_entries * sizeof(struct io_uring_cqe);
  195     if (p.features & IORING_FEAT_SINGLE_MMAP) {
  196         if (submit->cq_size > submit->sq_size)
  197             submit->sq_size = submit->cq_size;
  198         submit->cq_size = submit->sq_size;
  199     }
  200 
  201     submit->sq_mmap = mmap(NULL, submit->sq_size, PROT_READ | PROT_WRITE,
  202         MAP_SHARED | MAP_POPULATE,
  203         submit->io_uring_fd, IORING_OFF_SQ_RING);
  204     if (submit->sq_mmap == MAP_FAILED) {
  205         pr_inf("%s: could not mmap submission queue buffer, errno=%d (%s)\n",
  206             args->name, errno, strerror(errno));
  207         return EXIT_NO_RESOURCE;
  208     }
  209 
  210     if (p.features & IORING_FEAT_SINGLE_MMAP) {
  211         submit->cq_mmap = submit->sq_mmap;
  212     } else {
  213         submit->cq_mmap = mmap(NULL, submit->cq_size, PROT_READ | PROT_WRITE,
  214                 MAP_SHARED | MAP_POPULATE,
  215                 submit->io_uring_fd, IORING_OFF_CQ_RING);
  216         if (submit->cq_mmap == MAP_FAILED) {
  217             pr_inf("%s: could not mmap completion queue buffer, errno=%d (%s)\n",
  218                 args->name, errno, strerror(errno));
  219             (void)munmap(submit->sq_mmap, submit->cq_size);
  220             return EXIT_NO_RESOURCE;
  221         }
  222     }
  223 
  224     sring->head = VOID_ADDR_OFFSET(submit->sq_mmap, p.sq_off.head);
  225     sring->tail = VOID_ADDR_OFFSET(submit->sq_mmap, p.sq_off.tail);
  226     sring->ring_mask = VOID_ADDR_OFFSET(submit->sq_mmap, p.sq_off.ring_mask);
  227     sring->ring_entries = VOID_ADDR_OFFSET(submit->sq_mmap, p.sq_off.ring_entries);
  228     sring->flags = VOID_ADDR_OFFSET(submit->sq_mmap, p.sq_off.flags);
  229     sring->array = VOID_ADDR_OFFSET(submit->sq_mmap, p.sq_off.array);
  230 
  231     submit->sqes_size = p.sq_entries * sizeof(struct io_uring_sqe);
  232     submit->sqes_mmap = mmap(NULL, submit->sqes_size,
  233             PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
  234             submit->io_uring_fd, IORING_OFF_SQES);
  235     if (submit->sqes_mmap == MAP_FAILED) {
  236         pr_inf("%s: count not mmap submission queue buffer, errno=%d (%s)\n",
  237             args->name, errno, strerror(errno));
  238         if (submit->cq_mmap != submit->sq_mmap)
  239             (void)munmap(submit->cq_mmap, submit->cq_size);
  240         (void)munmap(submit->sq_mmap, submit->sq_size);
  241         return EXIT_NO_RESOURCE;
  242     }
  243 
  244     cring->head = VOID_ADDR_OFFSET(submit->cq_mmap, p.cq_off.head);
  245     cring->tail = VOID_ADDR_OFFSET(submit->cq_mmap, p.cq_off.tail);
  246     cring->ring_mask = VOID_ADDR_OFFSET(submit->cq_mmap, p.cq_off.ring_mask);
  247     cring->ring_entries = VOID_ADDR_OFFSET(submit->cq_mmap, p.cq_off.ring_entries);
  248     cring->cqes = VOID_ADDR_OFFSET(submit->cq_mmap, p.cq_off.cqes);
  249 
  250     return EXIT_SUCCESS;
  251 }
  252 
  253 /*
  254  *  stress_close_io_uring()
  255  *  close and cleanup behind us
  256  */
  257 static void stress_close_io_uring(stress_io_uring_submit_t *submit)
  258 {
  259     if (submit->io_uring_fd >= 0) {
  260         (void)close(submit->io_uring_fd);
  261         submit->io_uring_fd = -1;
  262     }
  263 
  264     if (submit->sqes_mmap) {
  265         (void)munmap(submit->sqes_mmap, submit->sqes_size);
  266         submit->sqes_mmap = NULL;
  267     }
  268 
  269     if (submit->cq_mmap && (submit->cq_mmap != submit->sq_mmap)) {
  270         (void)munmap(submit->cq_mmap, submit->cq_size);
  271         submit->cq_mmap = NULL;
  272     }
  273 
  274     if (submit->sq_mmap) {
  275         (void)munmap(submit->sq_mmap, submit->sq_size);
  276         submit->sq_mmap = NULL;
  277     }
  278 }
  279 
  280 /*
  281  *  stress_io_uring_complete()
  282  *  handle pending I/Os to complete
  283  */
  284 static inline int stress_io_uring_complete(
  285     const stress_args_t *args,
  286     stress_io_uring_submit_t *submit,
  287     const uint8_t opcode,
  288     bool *supported)
  289 {
  290     stress_uring_io_cq_ring_t *cring = &submit->cq_ring;
  291     struct io_uring_cqe *cqe;
  292     unsigned head = *cring->head;
  293     int ret = EXIT_SUCCESS;
  294 
  295     for (;;) {
  296         shim_mb();
  297 
  298         /* Empty? */
  299         if (head == *cring->tail)
  300             break;
  301 
  302         cqe = &cring->cqes[head & *submit->cq_ring.ring_mask];
  303         if ((cqe->res < 0) && (opcode != IORING_OP_FALLOCATE)) {
  304             const int err = abs(cqe->res);
  305 
  306             /* Silently ignore EOPNOTSUPP completion errors */
  307             if (errno == EOPNOTSUPP) {
  308                 *supported = false;
  309             } else  {
  310                 pr_err("%s: completion opcode=%d (%s), error=%d (%s)\n",
  311                     args->name, opcode,
  312                     stress_io_uring_opcode_name(opcode),
  313                     err, strerror(err));
  314                 ret = EXIT_FAILURE;
  315             }
  316         }
  317         head++;
  318     }
  319 
  320     *cring->head = head;
  321     shim_mb();
  322     if (ret == EXIT_SUCCESS)
  323         inc_counter(args);
  324 
  325     return ret;
  326 }
  327 
  328 /*
  329  *  stress_io_uring_submit()
  330  *  submit an io-uring opcode
  331  */
  332 static int stress_io_uring_submit(
  333     const stress_args_t *args,
  334     stress_io_uring_setup setup_func,
  335     stress_io_uring_file_t *io_uring_file,
  336     stress_io_uring_submit_t *submit,
  337     bool *supported)
  338 {
  339     stress_uring_io_sq_ring_t *sring = &submit->sq_ring;
  340     unsigned index = 0, tail = 0, next_tail = 0;
  341     struct io_uring_sqe *sqe;
  342     int ret;
  343     uint8_t opcode;
  344 
  345     next_tail = tail = *sring->tail;
  346     next_tail++;
  347     shim_mb();
  348     index = tail & *submit->sq_ring.ring_mask;
  349     sqe = &submit->sqes_mmap[index];
  350     (void)memset(sqe, 0, sizeof(*sqe));
  351 
  352     setup_func(io_uring_file, sqe);
  353     opcode = sqe->opcode;
  354 
  355     sring->array[index] = index;
  356     tail = next_tail;
  357     if (*sring->tail != tail) {
  358         *sring->tail = tail;
  359         shim_mb();
  360     }
  361 
  362     ret = shim_io_uring_enter(submit->io_uring_fd, 1,
  363         1, IORING_ENTER_GETEVENTS);
  364     if (ret < 0) {
  365         /* Silently ignore ENOSPC failures */
  366         if (errno == ENOSPC)    
  367             return EXIT_SUCCESS;
  368         pr_fail("%s: io_uring_enter failed, opcode=%d (%s), errno=%d (%s)\n",
  369             args->name, opcode,
  370             stress_io_uring_opcode_name(opcode),
  371             errno, strerror(errno));
  372         if (errno == EOPNOTSUPP)
  373             *supported = false;
  374         return EXIT_FAILURE;
  375     }
  376 
  377     return stress_io_uring_complete(args, submit, opcode, supported);
  378 }
  379 
  380 #if defined(HAVE_IORING_OP_READV)
  381 /*
  382  *  stress_io_uring_readv_setup()
  383  *  setup readv submit over io_uring
  384  */
  385 static void stress_io_uring_readv_setup(
  386     stress_io_uring_file_t *io_uring_file,
  387     struct io_uring_sqe *sqe)
  388 {
  389     sqe->fd = io_uring_file->fd;
  390     sqe->flags = 0;
  391     sqe->opcode = IORING_OP_READV;
  392     sqe->addr = (uintptr_t)io_uring_file->iovecs;
  393     sqe->len = io_uring_file->blocks;
  394     sqe->off = (uint64_t)stress_mwc8() * io_uring_file->blocks;
  395     sqe->user_data = (uintptr_t)io_uring_file;
  396 }
  397 #endif
  398 
  399 #if defined(HAVE_IORING_OP_WRITEV)
  400 /*
  401  *  stress_io_uring_writev_setup()
  402  *  setup writev submit over io_uring
  403  */
  404 static void stress_io_uring_writev_setup(
  405     stress_io_uring_file_t *io_uring_file,
  406     struct io_uring_sqe *sqe)
  407 {
  408     sqe->fd = io_uring_file->fd;
  409     sqe->flags = 0;
  410     sqe->opcode = IORING_OP_WRITEV;
  411     sqe->addr = (uintptr_t)io_uring_file->iovecs;
  412     sqe->len = io_uring_file->blocks;
  413     sqe->off = (uint64_t)stress_mwc8() * io_uring_file->blocks;
  414     sqe->user_data = (uintptr_t)io_uring_file;
  415 }
  416 #endif
  417 
  418 #if defined(HAVE_IORING_OP_READ)
  419 /*
  420  *  stress_io_uring_read_setup()
  421  *  setup read submit over io_uring
  422  */
  423 static void stress_io_uring_read_setup(
  424     stress_io_uring_file_t *io_uring_file,
  425     struct io_uring_sqe *sqe)
  426 {
  427     sqe->fd = io_uring_file->fd;
  428     sqe->flags = 0;
  429     sqe->opcode = IORING_OP_READ;
  430     sqe->addr = (uintptr_t)io_uring_file->iovecs[0].iov_base;
  431     sqe->len = io_uring_file->iovecs[0].iov_len;
  432     sqe->off = (uint64_t)stress_mwc8() * io_uring_file->blocks;
  433     sqe->user_data = (uintptr_t)io_uring_file;
  434 }
  435 #endif
  436 
  437 #if defined(HAVE_IORING_OP_WRITE)
  438 /*
  439  *  stress_io_uring_write_setup()
  440  *  setup write submit over io_uring
  441  */
  442 static void stress_io_uring_write_setup(
  443     stress_io_uring_file_t *io_uring_file,
  444     struct io_uring_sqe *sqe)
  445 {
  446     sqe->fd = io_uring_file->fd;
  447     sqe->flags = 0;
  448     sqe->opcode = IORING_OP_WRITE;
  449     sqe->addr = (uintptr_t)io_uring_file->iovecs[0].iov_base;
  450     sqe->len = io_uring_file->iovecs[0].iov_len;
  451     sqe->off = (uint64_t)stress_mwc8() * io_uring_file->blocks;
  452     sqe->user_data = (uintptr_t)io_uring_file;
  453 }
  454 #endif
  455 
  456 #if defined(HAVE_IORING_OP_FSYNC)
  457 /*
  458  *  stress_io_uring_fsync_setup()
  459  *  setup fsync submit over io_uring
  460  */
  461 static void stress_io_uring_fsync_setup(
  462     stress_io_uring_file_t *io_uring_file,
  463     struct io_uring_sqe *sqe)
  464 {
  465     sqe->fd = io_uring_file->fd;
  466     sqe->opcode = IORING_OP_FSYNC;
  467     sqe->len = 512;
  468     sqe->off = 0;
  469     sqe->user_data = (uintptr_t)io_uring_file;
  470     sqe->ioprio = 0;
  471     sqe->buf_index = 0;
  472     sqe->rw_flags = 0;
  473 }
  474 #endif
  475 
  476 #if defined(HAVE_IORING_OP_NOP)
  477 /*
  478  *  stress_io_uring_nop_setup()
  479  *  setup nop submit over io_uring
  480  */
  481 static void stress_io_uring_nop_setup(
  482     stress_io_uring_file_t *io_uring_file,
  483     struct io_uring_sqe *sqe)
  484 {
  485     (void)io_uring_file;
  486 
  487     sqe->opcode = IORING_OP_NOP;
  488 }
  489 #endif
  490 
  491 #if defined(HAVE_IORING_OP_FALLOCATE)
  492 /*
  493  *  stress_io_uring_fallocate_setup()
  494  *  setup fallocate submit over io_uring
  495  */
  496 static void stress_io_uring_fallocate_setup(
  497     stress_io_uring_file_t *io_uring_file,
  498     struct io_uring_sqe *sqe)
  499 {
  500     sqe->fd = io_uring_file->fd;
  501     sqe->opcode = IORING_OP_FALLOCATE;
  502     sqe->off = 0;           /* offset */
  503     sqe->addr = stress_mwc16(); /* length */
  504     sqe->len = 0;           /* mode */
  505     sqe->ioprio = 0;
  506     sqe->buf_index = 0;
  507     sqe->rw_flags = 0;
  508 }
  509 #endif
  510 
  511 #if defined(HAVE_IORING_OP_FADVISE)
  512 /*
  513  *  stress_io_uring_fadvise_setup ()
  514  *  setup fadvise submit over io_uring
  515  */
  516 static void stress_io_uring_fadvise_setup(
  517     stress_io_uring_file_t *io_uring_file,
  518     struct io_uring_sqe *sqe)
  519 {
  520     sqe->fd = io_uring_file->fd;
  521     sqe->opcode = IORING_OP_FADVISE;
  522     sqe->off = 0;           /* offset */
  523     sqe->len = stress_mwc16();  /* length */
  524 #if defined(POSIX_FADV_NORMAL)
  525     sqe->fadvise_advice = POSIX_FADV_NORMAL;
  526 #else
  527     sqe->fadvise_advice = 0;
  528 #endif
  529     sqe->ioprio = 0;
  530     sqe->buf_index = 0;
  531     sqe->addr = 0;
  532 }
  533 #endif
  534 
  535 #if defined(HAVE_IORING_OP_CLOSE)
  536 /*
  537  *  stress_io_uring_close_setup ()
  538  *  setup close submit over io_uring
  539  */
  540 static void stress_io_uring_close_setup(
  541     stress_io_uring_file_t *io_uring_file,
  542     struct io_uring_sqe *sqe)
  543 {
  544     (void)io_uring_file;
  545 
  546     /* don't worry about bad fd if dup fails */
  547     sqe->fd = dup(fileno(stdin));
  548     sqe->opcode = IORING_OP_CLOSE;
  549     sqe->ioprio = 0;
  550     sqe->off = 0;
  551     sqe->addr = 0;
  552     sqe->len = 0;
  553     sqe->rw_flags = 0;
  554     sqe->buf_index = 0;
  555 }
  556 #endif
  557 
  558 #if defined(HAVE_IORING_OP_MADVISE)
  559 /*
  560  *  stress_io_uring_madvise_setup ()
  561  *  setup madvise submit over io_uring
  562  */
  563 static void stress_io_uring_madvise_setup(
  564     stress_io_uring_file_t *io_uring_file,
  565     struct io_uring_sqe *sqe)
  566 {
  567     sqe->fd = io_uring_file->fd;
  568     sqe->opcode = IORING_OP_MADVISE;
  569     sqe->addr = (uintptr_t)io_uring_file->iovecs[0].iov_base;
  570     sqe->len = 4096;
  571 #if defined(MADV_NORMAL)
  572     sqe->fadvise_advice = MADV_NORMAL;
  573 #else
  574     sqe->fadvise_advice = 0;
  575 #endif
  576     sqe->ioprio = 0;
  577     sqe->buf_index = 0;
  578     sqe->off = 0;
  579 }
  580 #endif
  581 
  582 #if defined(HAVE_IORING_OP_STATX)
  583 /*
  584  *  stress_io_uring_statx_setup()
  585  *  setup statx submit over io_uring
  586  */
  587 static void stress_io_uring_statx_setup(
  588     stress_io_uring_file_t *io_uring_file,
  589     struct io_uring_sqe *sqe)
  590 {
  591     static const char *pathname = "";
  592     struct shim_statx statxbuf;
  593 
  594     sqe->opcode = IORING_OP_STATX;
  595     sqe->fd = io_uring_file->fd;
  596     sqe->addr = (uintptr_t)pathname;
  597     sqe->addr2 = (uintptr_t)&statxbuf;
  598     sqe->statx_flags = AT_EMPTY_PATH;
  599     sqe->ioprio = 0;
  600     sqe->buf_index = 0;
  601     sqe->flags = 0;
  602 }
  603 #endif
  604 
  605 #if defined(HAVE_IORING_OP_SYNC_FILE_RANGE)
  606 /*
  607  *  stress_io_uring_sync_file_range_setup()
  608  *  setup sync_file_range submit over io_uring
  609  */
  610 static void stress_io_uring_sync_file_range_setup(
  611     stress_io_uring_file_t *io_uring_file,
  612     struct io_uring_sqe *sqe)
  613 {
  614     sqe->opcode = IORING_OP_SYNC_FILE_RANGE;
  615     sqe->fd = io_uring_file->fd;
  616     sqe->off = stress_mwc16() & ~511UL;
  617     sqe->len = stress_mwc32() & ~511UL;
  618     sqe->flags = 0;
  619     sqe->addr = 0;
  620     sqe->ioprio = 0;
  621     sqe->buf_index = 0;
  622 }
  623 #endif
  624 
  625 static const stress_io_uring_setup_info_t stress_io_uring_setups[] = {
  626 #if defined(HAVE_IORING_OP_READV)
  627     { IORING_OP_READV,  "IORING_OP_READV",  stress_io_uring_readv_setup },
  628 #endif
  629 #if defined(HAVE_IORING_OP_WRITEV)
  630     { IORING_OP_WRITEV, "IORING_OP_WRITEV", stress_io_uring_writev_setup },
  631 #endif
  632 #if defined(HAVE_IORING_OP_READ)
  633     { IORING_OP_READ,   "IORING_OP_READ",   stress_io_uring_read_setup },
  634 #endif
  635 #if defined(HAVE_IORING_OP_WRITE)
  636     { IORING_OP_WRITE,  "IORING_OP_WRITE",  stress_io_uring_write_setup },
  637 #endif
  638 #if defined(HAVE_IORING_OP_FSYNC)
  639     { IORING_OP_FSYNC,  "IORING_OP_FSYNC",  stress_io_uring_fsync_setup },
  640 #endif
  641 #if defined(HAVE_IORING_OP_NOP)
  642     { IORING_OP_NOP,    "IORING_OP_NOP",    stress_io_uring_nop_setup },
  643 #endif
  644 #if defined(HAVE_IORING_OP_FALLOCATE)
  645     { IORING_OP_FALLOCATE,  "IORING_OP_FALLOCATE",  stress_io_uring_fallocate_setup },
  646 #endif
  647 #if defined(HAVE_IORING_OP_FADVISE)
  648     { IORING_OP_FADVISE,    "IORING_OP_FADVISE",    stress_io_uring_fadvise_setup },
  649 #endif
  650 #if defined(HAVE_IORING_OP_CLOSE)
  651     { IORING_OP_CLOSE,  "IORING_OP_CLOSE",  stress_io_uring_close_setup },
  652 #endif
  653 #if defined(HAVE_IORING_OP_MADVISE)
  654     { IORING_OP_MADVISE,    "IORING_OP_MADVISE",    stress_io_uring_madvise_setup },
  655 #endif
  656 #if defined(HAVE_IORING_OP_STATX)
  657     { IORING_OP_STATX,  "IORING_OP_STATX",  stress_io_uring_statx_setup },
  658 #endif
  659 #if defined(HAVE_IORING_OP_SYNC_FILE_RANGE)
  660     { IORING_OP_SYNC_FILE_RANGE, "IORING_OP_SYNC_FILE_RANGE", stress_io_uring_sync_file_range_setup },
  661 #endif
  662 };
  663 
  664 /*
  665  *  stress_io_uring_opcode_name()
  666  *  lookup opcode -> name
  667  */
  668 static const char *stress_io_uring_opcode_name(const uint8_t opcode)
  669 {
  670     size_t i;
  671 
  672     for (i = 0; i < SIZEOF_ARRAY(stress_io_uring_setups); i++) {
  673         if (stress_io_uring_setups[i].opcode == opcode)
  674             return stress_io_uring_setups[i].name;
  675     }
  676     return "unknown";
  677 }
  678 
  679 
  680 /*
  681  *  stress_io_uring
  682  *  stress asynchronous I/O
  683  */
  684 static int stress_io_uring(const stress_args_t *args)
  685 {
  686     int ret, rc;
  687     char filename[PATH_MAX];
  688     stress_io_uring_file_t io_uring_file;
  689     size_t i, j;
  690     const size_t blocks = 1024;
  691     const size_t block_size = 512;
  692     off_t file_size = (off_t)blocks * block_size;
  693     stress_io_uring_submit_t submit;
  694     const pid_t self = getpid();
  695     bool supported[SIZEOF_ARRAY(stress_io_uring_setups)];
  696 
  697     (void)memset(&submit, 0, sizeof(submit));
  698     (void)memset(&io_uring_file, 0, sizeof(io_uring_file));
  699 
  700     io_uring_file.file_size = file_size;
  701     io_uring_file.blocks = blocks;
  702     io_uring_file.block_size = block_size;
  703     io_uring_file.iovecs_sz = blocks * sizeof(*io_uring_file.iovecs);
  704     io_uring_file.iovecs =
  705         mmap(NULL, io_uring_file.iovecs_sz,
  706             PROT_READ | PROT_WRITE,
  707             MAP_SHARED | MAP_POPULATE | MAP_ANONYMOUS, -1, 0);
  708     if (io_uring_file.iovecs == MAP_FAILED) {
  709         io_uring_file.iovecs = NULL;
  710         pr_inf("%s: cannot allocate iovecs\n", args->name);
  711         return EXIT_NO_RESOURCE;
  712     }
  713 
  714     for (i = 0; (i < blocks) && (file_size > 0); i++) {
  715         const size_t iov_length = (file_size > (off_t)block_size) ? (size_t)block_size : (size_t)file_size;
  716 
  717         io_uring_file.iovecs[i].iov_len = iov_length;
  718         io_uring_file.iovecs[i].iov_base =
  719             mmap(NULL, block_size, PROT_READ | PROT_WRITE,
  720                 MAP_SHARED | MAP_POPULATE | MAP_ANONYMOUS, -1, 0);
  721         if (io_uring_file.iovecs[i].iov_base == MAP_FAILED) {
  722             io_uring_file.iovecs[i].iov_base = NULL;
  723             pr_inf("%s: cannot allocate iovec iov_base\n", args->name);
  724             stress_io_uring_unmap_iovecs(&io_uring_file);
  725             return EXIT_NO_RESOURCE;
  726         }
  727         file_size -= iov_length;
  728     }
  729 
  730     ret = stress_temp_dir_mk_args(args);
  731     if (ret < 0) {
  732         stress_io_uring_unmap_iovecs(&io_uring_file);
  733         return exit_status(-ret);
  734     }
  735 
  736     (void)stress_temp_filename_args(args,
  737         filename, sizeof(filename), stress_mwc32());
  738 
  739     rc = stress_setup_io_uring(args, &submit);
  740     if (rc != EXIT_SUCCESS)
  741         goto clean;
  742 
  743     if ((io_uring_file.fd = open(filename, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR)) < 0) {
  744         rc = exit_status(errno);
  745         pr_fail("%s: open on %s failed, errno=%d (%s)\n",
  746             args->name, filename, errno, strerror(errno));
  747         goto clean;
  748     }
  749     (void)unlink(filename);
  750 
  751     stress_set_proc_state(args->name, STRESS_STATE_RUN);
  752 
  753     /*
  754      *  Assume all opcodes are supported
  755      */
  756     for (j = 0; j < SIZEOF_ARRAY(stress_io_uring_setups); j++) {
  757         supported[i] = true;
  758     }
  759 
  760     rc = EXIT_SUCCESS;
  761     i = 0;
  762     do {
  763         for (j = 0; j < SIZEOF_ARRAY(stress_io_uring_setups); j++) {
  764             if (supported[j]) {
  765                 rc = stress_io_uring_submit(args,
  766                     stress_io_uring_setups[j].setup_func,
  767                     &io_uring_file, &submit, &supported[j]);
  768                 if ((rc != EXIT_SUCCESS) || !keep_stressing(args))
  769                     break;
  770             }
  771         }
  772 
  773         if (i++ > 1024) {
  774             i = 0;
  775             (void)stress_read_fdinfo(self, submit.io_uring_fd);
  776         }
  777     } while (keep_stressing(args));
  778 
  779     stress_set_proc_state(args->name, STRESS_STATE_DEINIT);
  780     (void)close(io_uring_file.fd);
  781 clean:
  782     stress_set_proc_state(args->name, STRESS_STATE_DEINIT);
  783     stress_close_io_uring(&submit);
  784     stress_io_uring_unmap_iovecs(&io_uring_file);
  785     (void)stress_temp_dir_rm_args(args);
  786     return rc;
  787 }
  788 
  789 stressor_info_t stress_io_uring_info = {
  790     .stressor = stress_io_uring,
  791     .class = CLASS_IO | CLASS_OS,
  792     .help = help
  793 };
  794 #else
  795 stressor_info_t stress_io_uring_info = {
  796     .stressor = stress_not_implemented,
  797     .class = CLASS_IO | CLASS_OS,
  798     .help = help
  799 };
  800 #endif