"Fossies" - the Fresh Open Source Software Archive

Member "sg3_utils-1.45r831/testing/sg_tst_async.cpp" (9 Sep 2019, 80150 Bytes) of package /linux/misc/sg3_utils-1.45r831.tar.xz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. See also the latest Fossies "Diffs" side-by-side code changes report for "sg_tst_async.cpp": 1.45r830_vs_1.45r831.

    1 /*
    2  * Copyright (c) 2014-2019 Douglas Gilbert.
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer.
   10  * 2. Redistributions in binary form must reproduce the above copyright
   11  *    notice, this list of conditions and the following disclaimer in the
   12  *    documentation and/or other materials provided with the distribution.
   13  *
   14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   24  * SUCH DAMAGE.
   25  *
   26  * SPDX-License-Identifier: BSD-2-Clause
   27  */
   28 
   29 #include <iostream>
   30 #include <vector>
   31 #include <map>
   32 #include <list>
   33 #include <system_error>
   34 #include <thread>
   35 #include <mutex>
   36 #include <chrono>
   37 #include <atomic>
   38 #include <random>
   39 
   40 #include <unistd.h>
   41 #include <fcntl.h>
   42 #include <stdio.h>
   43 #include <stdlib.h>
   44 #include <stdarg.h>
   45 #include <string.h>
   46 #include <poll.h>
   47 #include <errno.h>
   48 #include <ctype.h>
   49 #include <time.h>
   50 #include <limits.h>
   51 #include <getopt.h>
   52 #define __STDC_FORMAT_MACROS 1
   53 #include <inttypes.h>
   54 #include <sys/ioctl.h>
   55 #include <sys/types.h>
   56 #include <sys/stat.h>
   57 #include <sys/mman.h>
   58 #include <sys/resource.h>       /* getrusage */
   59 
   60 
   61 #ifdef HAVE_CONFIG_H
   62 #include "config.h"
   63 #endif
   64 
   65 #ifndef HAVE_LINUX_SG_V4_HDR
   66 
   67 /* Kernel uapi header contain __user decorations on user space pointers
   68  * to indicate they are unsafe in the kernel space. However glibc takes
   69  * all those __user decorations out from headers in /usr/include/linux .
   70  * So to stop compile errors when directly importing include/uapi/scsi/sg.h
   71  * undef __user before doing that include. */
   72 #define __user
   73 
   74 /* Want to block the original sg.h header from also being included. That
   75  * causes lots of multiple definition errors. This will only work if this
   76  * header is included _before_ the original sg.h header.  */
   77 #define _SCSI_GENERIC_H         /* original kernel header guard */
   78 #define _SCSI_SG_H              /* glibc header guard */
   79 
   80 #include "uapi_sg.h"    /* local copy of include/uapi/scsi/sg.h */
   81 
   82 #else
   83 #define __user
   84 #endif  /* end of: ifndef HAVE_LINUX_SG_V4_HDR */
   85 
   86 #include "sg_lib.h"
   87 #include "sg_io_linux.h"
   88 #include "sg_unaligned.h"
   89 #include "sg_pt.h"
   90 #include "sg_cmds.h"
   91 
   92 static const char * version_str = "1.39 20190902";
   93 static const char * util_name = "sg_tst_async";
   94 
   95 /* This is a test program for checking the async usage of the Linux sg
   96  * driver. Each thread opens 1 file descriptor to the next sg device (1
   97  * or more can be given on the command line) and then starts up to
   98  * num_per_thread commands or more while checking with the poll command (or
   99  * ioctl(SG_GET_NUM_WAITING) ) for the completion of those commands. Each
  100  * command has a unique "pack_id" which is a sequence starting at 1.
  101  * Either TEST UNIT UNIT, READ(16) or WRITE(16) commands are issued.
  102  *
  103  * This is C++ code with some things from C++11 (e.g. threads) and was
  104  * only just able to compile (when some things were reverted) with gcc/g++
  105  * version 4.7.3 found in Ubuntu 13.04 . C++11 "feature complete" support
  106  * was not available until g++ version 4.8.1 . It should build okay on
  107  * recent distributions.
  108  *
  109  * The build uses various object files from the <sg3_utils>/lib directory
  110  * which is assumed to be a sibling of this examples directory. Those
  111  * object files in the lib directory can be built with:
  112  *   cd <sg3_utils_package_root> ; ./configure ; cd lib; make
  113  *   cd ../examples
  114  * Then to build sg_tst_async concatenate the next 3 lines:
  115  *   g++ -Wall -std=c++11 -pthread -I ../include ../lib/sg_lib.o
  116  *     ../lib/sg_lib_data.o ../lib/sg_io_linux.o -o sg_tst_async
  117  *     sg_tst_async.cpp
  118  * or use the C++ Makefile in that directory:
  119  *   make -f Makefile.cplus sg_tst_async
  120  *
  121  * Currently this utility is Linux only and uses the sg driver. The bsg
  122  * driver is known to be broken (it doesn't match responses to the
  123  * correct file descriptor that requested them) so this utility won't
  124  * be extended to bsg until that is fixed.
  125  *
  126  * BEWARE: >>> This utility will modify a logical block (default LBA 1000)
  127  * on the given device when the '-W' option is given.
  128  *
  129  */
  130 
  131 using namespace std;
  132 using namespace std::chrono;
  133 
  134 #define DEF_NUM_PER_THREAD 1000
  135 #define DEF_NUM_THREADS 4
  136 #define DEF_WAIT_MS 10          /* 0: yield or no wait */
  137 #define DEF_NANOSEC_WAIT 25000  /* 25 microsecs */
  138 #define DEF_TIMEOUT_MS 20000    /* 20 seconds */
  139 #define DEF_LB_SZ 512
  140 #define DEF_BLOCKING 0
  141 #define DEF_DIRECT false        /* true: direct_io */
  142 #define DEF_MMAP_IO false       /* true: mmap-ed IO with sg */
  143 #define DEF_NO_XFER 0
  144 #define DEF_LBA 1000
  145 
  146 #define MAX_Q_PER_FD 16383      /* sg driver per file descriptor limit */
  147 #define MAX_CONSEC_NOMEMS 4     /* was 16 */
  148 #define URANDOM_DEV "/dev/urandom"
  149 
  150 #ifndef SG_FLAG_Q_AT_TAIL
  151 #define SG_FLAG_Q_AT_TAIL 0x10
  152 #endif
  153 #ifndef SG_FLAG_Q_AT_HEAD
  154 #define SG_FLAG_Q_AT_HEAD 0x20
  155 #endif
  156 
  157 
  158 #define DEF_PT_TIMEOUT 60       /* 60 seconds */
  159 
  160 #define EBUFF_SZ 256
  161 
  162 static mutex console_mutex;
  163 static mutex rand_lba_mutex;
  164 static atomic<int> async_starts(0);
  165 static atomic<int> sync_starts(0);
  166 static atomic<int> async_finishes(0);
  167 static atomic<int> start_ebusy_count(0);
  168 static atomic<int> start_e2big_count(0);
  169 static atomic<int> start_eagain_count(0);
  170 static atomic<int> fin_eagain_count(0);
  171 static atomic<int> fin_ebusy_count(0);
  172 static atomic<int> start_edom_count(0);
  173 static atomic<int> enomem_count(0);
  174 static atomic<int> uniq_pack_id(1);
  175 // static atomic<int> generic_errs(0);
  176 
  177 static int page_size = 4096;   /* rough guess, will ask sysconf() */
  178 
  179 enum command2execute {SCSI_TUR, SCSI_READ16, SCSI_WRITE16};
  180 /* Linux Block layer queue disciplines: */
  181 enum blkLQDiscipline {BLQ_DEFAULT, BLQ_AT_HEAD, BLQ_AT_TAIL};
  182 /* Queue disciplines of this utility. When both completions and
  183  * queuing a new command are both possible: */
  184 enum myQDiscipline {MYQD_LOW,   /* favour completions over new cmds */
  185                     MYQD_MEDIUM,
  186                     MYQD_HIGH}; /* favour new cmds over completions */
  187 
  188 struct opts_t {
  189     vector<const char *> dev_names;
  190     vector<int> blk_szs;
  191     bool block;
  192     bool cmd_time;
  193     bool direct;
  194     bool excl;
  195     bool generic_sync;
  196     bool masync;
  197     bool mmap_io;
  198     bool no_waitq;
  199     bool no_xfer;
  200     bool pack_id_force;
  201     bool sg_vn_ge_40000;
  202     bool sg_vn_ge_40030;
  203     bool submit;
  204     bool verbose_given;
  205     bool v3;
  206     bool v3_given;
  207     bool v4;
  208     bool v4_given;
  209     bool version_given;
  210     int maxq_per_thread;
  211     int num_per_thread;
  212     uint64_t lba;
  213     unsigned int hi_lba;        /* last one, inclusive range */
  214     vector<unsigned int> hi_lbas; /* only used when hi_lba=-1 */
  215     int lb_sz;
  216     int num_lbs;
  217     int ovn;            /* override number for submission */
  218     int stats;
  219     int verbose;
  220     int wait_ms;
  221     command2execute c2e;
  222     blkLQDiscipline blqd;       /* --qat= 0|1 -> at_head|at_tail */
  223     myQDiscipline myqd;         /* --qfav= value (def: 2 --> MYQD_HIGH) */
  224 };
  225 
  226 static struct opts_t a_opts;    /* Expect zero fill on simple types */
  227 
  228 static int pr_rusage(int id);
  229 
  230 #if 0
  231 class Rand_uint {
  232 public:
  233     Rand_uint(unsigned int lo, unsigned int hi) : p{lo, hi} {}
  234     unsigned int operator()() const { return r(); }
  235 private:
  236     uniform_int_distribution<unsigned int>::param_type p;
  237     auto r = bind(uniform_int_distribution<unsigned int>{p},
  238                   default_random_engine());
  239     /* compiler thinks auto should be a static, bs again? */
  240 };
  241 #endif
  242 
  243 #if 0
  244 class Rand_uint {
  245 public:
  246     Rand_uint(unsigned int lo, unsigned int hi, unsigned int my_seed)
  247         : r(bind(uniform_int_distribution<unsigned int>{lo, hi},
  248                  default_random_engine())) { r.seed(myseed); }
  249     unsigned int operator()() const { return r(); }
  250 private:
  251     function<unsigned int()> r;
  252 };
  253 #endif
  254 
  255 /* Use this class to wrap C++11 <random> features to produce uniform random
  256  * unsigned ints in the range [lo, hi] (inclusive) given a_seed */
  257 class Rand_uint {
  258 public:
  259     Rand_uint(unsigned int lo, unsigned int hi, unsigned int a_seed)
  260         : uid(lo, hi), dre(a_seed) { }
  261     /* uid ctor takes inclusive range when integral type */
  262 
  263     unsigned int get() { return uid(dre); }
  264 
  265 private:
  266     uniform_int_distribution<unsigned int> uid;
  267     default_random_engine dre;
  268 };
  269 
  270 static struct option long_options[] = {
  271         {"v3", no_argument, 0, '3'},
  272         {"v4", no_argument, 0, '4'},
  273         {"more-async", no_argument, 0, 'a'},
  274         {"more_async", no_argument, 0, 'a'},
  275         {"masync", no_argument, 0, 'a'},
  276         {"cmd-time", no_argument, 0, 'c'},
  277         {"cmd_time", no_argument, 0, 'c'},
  278         {"direct", no_argument, 0, 'd'},
  279         {"excl", no_argument, 0, 'e'},
  280         {"force", no_argument, 0, 'f'},
  281         {"generic-sync", no_argument, 0, 'g'},
  282         {"generic_sync", no_argument, 0, 'g'},
  283         {"help", no_argument, 0, 'h'},
  284         {"lba", required_argument, 0, 'l'},
  285         {"lbsz", required_argument, 0, 'L'},
  286         {"maxqpt", required_argument, 0, 'M'},
  287         {"mmap-io", no_argument, 0, 'm'},
  288         {"mmap_io", no_argument, 0, 'm'},
  289         {"numpt", required_argument, 0, 'n'},
  290         {"num-pt", required_argument, 0, 'n'},
  291         {"num_pt", required_argument, 0, 'n'},
  292         {"nowaitq", no_argument, 0, 'z'},
  293         {"no_waitq", no_argument, 0, 'z'},
  294         {"no-waitq", no_argument, 0, 'z'},
  295         {"noxfer", no_argument, 0, 'N'},
  296         {"override", required_argument, 0, 'O'},
  297         {"pack-id", no_argument, 0, 'p'},
  298         {"pack_id", no_argument, 0, 'p'},
  299         {"qat", required_argument, 0, 'q'},
  300         {"qfav", required_argument, 0, 'Q'},
  301         {"read", no_argument, 0, 'R'},
  302         {"stats", no_argument, 0, 'S'},
  303         {"submit", no_argument, 0, 'u'},
  304         {"szlb", required_argument, 0, 's'},
  305         {"tnum", required_argument, 0, 't'},
  306         {"tur", no_argument, 0, 'T'},
  307         {"verbose", no_argument, 0, 'v'},
  308         {"version", no_argument, 0, 'V'},
  309         {"wait", required_argument, 0, 'w'},
  310         {"write", no_argument, 0, 'W'},
  311         {0, 0, 0, 0},
  312 };
  313 
  314 
  315 static void
  316 usage(void)
  317 {
  318     printf("Usage: %s [--cmd-time] [--direct] [--excl] [--force]\n"
  319            "                    [--generic-sync] [--help] [--lba=LBA+] "
  320            "[--lbsz=LBSZ]\n"
  321            "                    [--masync] [--maxqpt=QPT] [--mmap-io] "
  322            "[--no-waitq]\n"
  323            "                    [--noxfer] [--numpt=NPT] [--override=OVN] "
  324            "[--pack-id]\n"
  325            "                    [--qat=AT] [-qfav=FAV] [--read] [--stats] "
  326            "[--submit]\n"
  327            "                    [--szlb=LB[,NLBS]] [--tnum=NT] [--tur] "
  328            "[--v3] [--v4]\n"
  329            "                    [--verbose] [--version] [--wait=MS] "
  330            "[--write]\n"
  331            "                    <sg_disk_device>*\n",
  332            util_name);
  333     printf("  where\n");
  334     printf("    --cmd-time|-c    calculate per command average time (ns)\n");
  335     printf("    --direct|-d     do direct_io (def: indirect)\n");
  336     printf("    --excl|-e       do wait_exclusive calls\n");
  337     printf("    --force|-f      force: any sg device (def: only scsi_debug "
  338            "owned)\n");
  339     printf("                    WARNING: <lba> written to if '-W' given\n");
  340     printf("    --generic-sync|-g    use generic synchronous SG_IO ioctl "
  341            "instead\n");
  342     printf("                       of Linux sg driver assuming /dev/sg* "
  343            "(def)\n");
  344     printf("    --help|-h       print this usage message then exit\n");
  345     printf("    --lba=LBA|-l LBA    logical block to access (def: %u)\n",
  346            DEF_LBA);
  347     printf("    --lba=LBA,HI_LBA|-l LBA,HI_LBA    logical block range "
  348            "(inclusive)\n"
  349            "                          if hi_lba=-1 assume last block on "
  350            "device\n");
  351     printf("    --lbsz=LBSZ|-L LBSZ    logical block size in bytes (def: "
  352            "512)\n"
  353            "                           should be power of 2 (0 --> 512)\n");
  354     printf("    --masync|-a     set 'more async' flag on devices\n");
  355     printf("    --maxqpt=QPT|-M QPT    maximum commands queued per thread "
  356            "(def:%d)\n", MAX_Q_PER_FD);
  357     printf("    --mmap-io|-m    mmap-ed IO (1 cmd outstanding per thread)\n");
  358     printf("    --no-waitq|-z    set SGV4_FLAG_NO_WAITQ, bypass poll() if "
  359            "requested\n");
  360     printf("    --noxfer|-N          no data xfer (def: xfer on READ and "
  361            "WRITE)\n");
  362     printf("    --numpt=NPT|-n NPT    number of commands per thread "
  363            "(def: %d)\n", DEF_NUM_PER_THREAD);
  364     printf("    --override OVN|-O OVN    override FAV=2 when OVN queue "
  365            "depth\n"
  366            "                             reached (def: 0 -> no override)\n");
  367     printf("    --pack-id|-p    set FORCE_PACK_ID, pack-id input to "
  368            "read/finish\n");
  369     printf("    --qat=AT|-q AT       AT=0: q_at_head; AT=1: q_at_tail (def: "
  370            "(drv): head)\n");
  371     printf("    --qfav=FAV|-Q FAV    FAV=0: favour completions (smaller q),\n"
  372            "                         FAV=1: medium,\n"
  373            "                         FAV=2: favour submissions (larger q, "
  374            "default)\n");
  375     printf("    --read|-R       do READs (def: TUR)\n");
  376     printf("    --stats|-S      show more statistics on completion\n");
  377     printf("    --submit|-u     use SG_IOSUBMIT+SG_IORECEIVE instead of "
  378            "write+read\n");
  379     printf("    --szlb=LB[,NLBS]|    LB is logical block size (def: 512)\n");
  380     printf("         -s LB[,NLBS]    NLBS is number of logical blocks (def: "
  381            "1)\n");
  382     printf("    --tnum=NT|-t NT    number of threads (def: %d)\n",
  383            DEF_NUM_THREADS);
  384     printf("    --tur|-T        do TEST UNIT READYs (default is TURs)\n");
  385     printf("    --v3|-3         use sg v3 interface (def: v3 if driver < "
  386            "3.9)\n");
  387     printf("    --v4|-4         use sg v4 interface (def if v4 driver). Sets "
  388            "--submit\n");
  389     printf("    --verbose|-v    increase verbosity\n");
  390     printf("    --version|-V    print version number then exit\n");
  391     printf("    --wait=MS|-w MS    >0: poll(<wait_ms>); =0: poll(0); (def: "
  392            "%d)\n", DEF_WAIT_MS);
  393     printf("    --write|-W      do WRITEs (def: TUR)\n\n");
  394     printf("Multiple threads send READ(16), WRITE(16) or TEST UNIT READY "
  395            "(TUR) SCSI\ncommands. There can be 1 or more <sg_disk_device>s "
  396            "and each thread takes\nthe next in a round robin fashion. "
  397            "Each thread queues up to NT commands.\nOne block is transferred "
  398            "by each READ and WRITE; zeros are written. If a\nlogical block "
  399            "range is given, a uniform distribution generates a pseudo\n"
  400            "random sequence of LBAs. Set environment variable\n"
  401            "SG3_UTILS_LINUX_NANO to get command timings in nanoseconds\n");
  402 }
  403 
  404 #ifdef __GNUC__
  405 static int pr2serr_lk(const char * fmt, ...)
  406         __attribute__ ((format (printf, 1, 2)));
  407 static void pr_errno_lk(int e_no, const char * fmt, ...)
  408         __attribute__ ((format (printf, 2, 3)));
  409 #else
  410 static int pr2serr_lk(const char * fmt, ...);
  411 static void pr_errno_lk(int e_no, const char * fmt, ...);
  412 #endif
  413 
  414 
  415 static int
  416 pr2serr_lk(const char * fmt, ...)
  417 {
  418     int n;
  419     va_list args;
  420     lock_guard<mutex> lg(console_mutex);
  421 
  422     va_start(args, fmt);
  423     n = vfprintf(stderr, fmt, args);
  424     va_end(args);
  425     return n;
  426 }
  427 
  428 static void
  429 pr_errno_lk(int e_no, const char * fmt, ...)
  430 {
  431     char b[160];
  432     va_list args;
  433     lock_guard<mutex> lg(console_mutex);
  434 
  435     va_start(args, fmt);
  436     vsnprintf(b, sizeof(b), fmt, args);
  437     fprintf(stderr, "%s: %s\n", b, strerror(e_no));
  438     va_end(args);
  439 }
  440 
  441 static unsigned int
  442 get_urandom_uint(void)
  443 {
  444     unsigned int res = 0;
  445     int n;
  446     uint8_t b[sizeof(unsigned int)];
  447     lock_guard<mutex> lg(rand_lba_mutex);
  448 
  449     int fd = open(URANDOM_DEV, O_RDONLY);
  450     if (fd >= 0) {
  451         n = read(fd, b, sizeof(unsigned int));
  452         if (sizeof(unsigned int) == n)
  453             memcpy(&res, b, sizeof(unsigned int));
  454         close(fd);
  455     }
  456     return res;
  457 }
  458 
  459 #define TUR_CMD_LEN 6
  460 #define READ16_CMD_LEN 16
  461 #define READ16_REPLY_LEN 4096
  462 #define WRITE16_REPLY_LEN 4096
  463 #define WRITE16_CMD_LEN 16
  464 
  465 /* Returns 0 if command injected okay, return -1 for error and 2 for
  466  * not done due to queue data size limit struck. */
  467 static int
  468 start_sg3_cmd(int sg_fd, command2execute cmd2exe, int pack_id, uint64_t lba,
  469               uint8_t * lbp, int xfer_bytes, int flags, bool submit,
  470               unsigned int & enomem, unsigned int & eagains,
  471               unsigned int & ebusy, unsigned int & e2big, unsigned int & edom)
  472 {
  473     struct sg_io_hdr pt;
  474     struct sg_io_v4 p4t;
  475     uint8_t turCmdBlk[TUR_CMD_LEN] = {0, 0, 0, 0, 0, 0};
  476     uint8_t r16CmdBlk[READ16_CMD_LEN] =
  477                 {0x88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0};
  478     uint8_t w16CmdBlk[WRITE16_CMD_LEN] =
  479                 {0x8a, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0};
  480     uint8_t sense_buffer[64];
  481     const char * np = NULL;
  482     struct sg_io_hdr * ptp;
  483 
  484     if (submit) {       /* nest a v3 interface inside a store for v4 */
  485         memset(&p4t, 0, sizeof(p4t));
  486         ptp = (struct sg_io_hdr *)&p4t; /* p4t is larger than pt */
  487     } else {
  488         ptp = &pt;
  489         memset(ptp, 0, sizeof(*ptp));
  490     }
  491     switch (cmd2exe) {
  492     case SCSI_TUR:
  493         np = "TEST UNIT READY";
  494         ptp->cmdp = turCmdBlk;
  495         ptp->cmd_len = sizeof(turCmdBlk);
  496         ptp->dxfer_direction = SG_DXFER_NONE;
  497         break;
  498     case SCSI_READ16:
  499         np = "READ(16)";
  500         if (lba > 0xffffffff)
  501             sg_put_unaligned_be32(lba >> 32, &r16CmdBlk[2]);
  502         sg_put_unaligned_be32(lba & 0xffffffff, &r16CmdBlk[6]);
  503         ptp->cmdp = r16CmdBlk;
  504         ptp->cmd_len = sizeof(r16CmdBlk);
  505         ptp->dxfer_direction = SG_DXFER_FROM_DEV;
  506         ptp->dxferp = lbp;
  507         ptp->dxfer_len = xfer_bytes;
  508         break;
  509     case SCSI_WRITE16:
  510         np = "WRITE(16)";
  511         if (lba > 0xffffffff)
  512             sg_put_unaligned_be32(lba >> 32, &w16CmdBlk[2]);
  513         sg_put_unaligned_be32(lba & 0xffffffff, &w16CmdBlk[6]);
  514         ptp->cmdp = w16CmdBlk;
  515         ptp->cmd_len = sizeof(w16CmdBlk);
  516         ptp->dxfer_direction = SG_DXFER_TO_DEV;
  517         ptp->dxferp = lbp;
  518         ptp->dxfer_len = xfer_bytes;
  519         break;
  520     }
  521     ptp->interface_id = 'S';
  522     ptp->mx_sb_len = sizeof(sense_buffer);
  523     ptp->sbp = sense_buffer;      /* ignored .... */
  524     ptp->timeout = DEF_TIMEOUT_MS;
  525     ptp->pack_id = pack_id;
  526     ptp->flags = flags;
  527 
  528     for (int k = 0;
  529          (submit ? ioctl(sg_fd, SG_IOSUBMIT_V3, ptp) :
  530                    write(sg_fd, ptp, sizeof(*ptp)) < 0);
  531          ++k) {
  532         if ((ENOMEM == errno) && (k < MAX_CONSEC_NOMEMS)) {
  533             ++enomem;
  534             this_thread::yield();
  535             continue;
  536         } else if (EAGAIN == errno) {
  537             ++eagains;
  538             this_thread::yield();
  539             continue;
  540         } else if (EBUSY == errno) {
  541             ++ebusy;
  542             this_thread::yield();
  543             continue;
  544         } else if (E2BIG == errno) {
  545             ++e2big;
  546             return 2;
  547         } else if (EDOM == errno)
  548             ++edom;
  549         else if (ENOMEM == errno)
  550             pr_rusage(-1);
  551         pr_errno_lk(errno, "%s: %s, pack_id=%d", __func__, np, pack_id);
  552         return -1;
  553     }
  554     return 0;
  555 }
  556 
  557 static int
  558 finish_sg3_cmd(int sg_fd, command2execute cmd2exe, int & pack_id,
  559                bool receive, int wait_ms, unsigned int & enomem,
  560                unsigned int & eagains, unsigned int & ebusys,
  561                unsigned int & nanosecs)
  562 {
  563     bool ok;
  564     int res, k;
  565     uint8_t sense_buffer[64];
  566     const char * np = NULL;
  567     struct sg_io_hdr pt;
  568     struct sg_io_hdr * ptp;
  569     struct sg_io_v4 p4t;
  570 
  571     if (receive) {      /* nest a v3 interface inside a store for v4 */
  572         memset(&p4t, 0, sizeof(p4t));
  573         ptp = (struct sg_io_hdr *)&p4t; /* p4t is larger than pt */
  574     } else {
  575         ptp = &pt;
  576         memset(ptp, 0, sizeof(*ptp));
  577     }
  578     switch (cmd2exe) {
  579     case SCSI_TUR:
  580         np = "TEST UNIT READY";
  581         ptp->dxfer_direction = SG_DXFER_NONE;
  582         break;
  583     case SCSI_READ16:
  584         np = "READ(16)";
  585         ptp->dxfer_direction = SG_DXFER_FROM_DEV;
  586         break;
  587     case SCSI_WRITE16:
  588         np = "WRITE(16)";
  589         ptp->dxfer_direction = SG_DXFER_TO_DEV;
  590         break;
  591     }
  592     ptp->interface_id = 'S';
  593     ptp->mx_sb_len = sizeof(sense_buffer);
  594     ptp->sbp = sense_buffer;
  595     ptp->timeout = DEF_TIMEOUT_MS;
  596     /* if SG_SET_FORCE_PACK_ID, then need to set ptp->dxfer_direction */
  597     ptp->pack_id = pack_id;
  598 
  599     k = 0;
  600     while ((((res = receive ? ioctl(sg_fd, SG_IORECEIVE_V3, ptp) :
  601                               read(sg_fd, ptp, sizeof(*ptp)))) < 0) &&
  602            ((EAGAIN == errno) || (EBUSY == errno) || (ENOMEM == errno))) {
  603         if (ENOMEM == errno)
  604             ++enomem;
  605         else if (EAGAIN == errno)
  606             ++eagains;
  607         else
  608             ++ebusys;
  609         ++k;
  610         if (k > 10000) {
  611             pr2serr_lk("%s: sg_fd=%d: after %d EAGAINs, unable to find "
  612                        "pack_id=%d\n", __func__, sg_fd, k, pack_id);
  613             return -1;      /* crash out */
  614         }
  615         if (wait_ms > 0)
  616             this_thread::sleep_for(milliseconds{wait_ms});
  617         else if (0 == wait_ms)
  618             this_thread::yield();
  619         else if (-2 == wait_ms)
  620             sleep(0);                   // process yield ??
  621     }
  622     if (res < 0) {
  623         if (ENOMEM == errno)
  624             pr_rusage(-1);
  625         pr_errno_lk(errno, "%s: %s", __func__, np);
  626         return -1;
  627     }
  628     /* now for the error processing */
  629     pack_id = ptp->pack_id;
  630     ok = false;
  631     switch (sg_err_category3(ptp)) {
  632     case SG_LIB_CAT_CLEAN:
  633         ok = true;
  634         break;
  635     case SG_LIB_CAT_RECOVERED:
  636         pr2serr_lk("%s: Recovered error on %s, continuing\n", __func__, np);
  637         ok = true;
  638         break;
  639     default: /* won't bother decoding other categories */
  640         {
  641             lock_guard<mutex> lg(console_mutex);
  642             sg_chk_n_print3(np, ptp, 1);
  643         }
  644         break;
  645     }
  646     if (ok)
  647         nanosecs = ptp->duration;
  648     return ok ? 0 : -1;
  649 }
  650 
  651 /* Returns 0 if command injected okay, return -1 for error and 2 for
  652  * not done due to queue data size limit struck. */
  653 static int
  654 start_sg4_cmd(int sg_fd, command2execute cmd2exe, int pack_id, uint64_t lba,
  655               uint8_t * lbp, int xfer_bytes, int flags, bool submit,
  656               unsigned int & enomem, unsigned int & eagains,
  657               unsigned int & ebusy, unsigned int & e2big, unsigned int & edom)
  658 {
  659     struct sg_io_v4 p4t;
  660     uint8_t turCmdBlk[TUR_CMD_LEN] = {0, 0, 0, 0, 0, 0};
  661     uint8_t r16CmdBlk[READ16_CMD_LEN] =
  662                 {0x88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0};
  663     uint8_t w16CmdBlk[WRITE16_CMD_LEN] =
  664                 {0x8a, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0};
  665     uint8_t sense_buffer[64];
  666     const char * np = NULL;
  667     struct sg_io_v4 * ptp;
  668 
  669     if (! submit) {
  670         pr2serr_lk("%s: logic error, submit must be true, isn't\n", __func__);
  671         return -1;
  672     }
  673     ptp = &p4t;
  674     memset(ptp, 0, sizeof(*ptp));
  675     switch (cmd2exe) {
  676     case SCSI_TUR:
  677         np = "TEST UNIT READY";
  678         ptp->request = (uint64_t)turCmdBlk;
  679         ptp->request_len = sizeof(turCmdBlk);
  680         break;
  681     case SCSI_READ16:
  682         np = "READ(16)";
  683         if (lba > 0xffffffff)
  684             sg_put_unaligned_be32(lba >> 32, &r16CmdBlk[2]);
  685         sg_put_unaligned_be32(lba & 0xffffffff, &r16CmdBlk[6]);
  686         ptp->request = (uint64_t)r16CmdBlk;
  687         ptp->request_len = sizeof(r16CmdBlk);
  688         ptp->din_xferp = (uint64_t)lbp;
  689         ptp->din_xfer_len = xfer_bytes;
  690         break;
  691     case SCSI_WRITE16:
  692         np = "WRITE(16)";
  693         if (lba > 0xffffffff)
  694             sg_put_unaligned_be32(lba >> 32, &w16CmdBlk[2]);
  695         sg_put_unaligned_be32(lba & 0xffffffff, &w16CmdBlk[6]);
  696         ptp->request = (uint64_t)w16CmdBlk;
  697         ptp->request_len = sizeof(w16CmdBlk);
  698         ptp->dout_xferp = (uint64_t)lbp;
  699         ptp->dout_xfer_len = xfer_bytes;
  700         break;
  701     }
  702     ptp->guard = 'Q';
  703     ptp->max_response_len = sizeof(sense_buffer);
  704     ptp->response = (uint64_t)sense_buffer;      /* ignored .... */
  705     ptp->timeout = DEF_TIMEOUT_MS;
  706     ptp->request_extra = pack_id;
  707     ptp->flags = flags;
  708 
  709     for (int k = 0; ioctl(sg_fd, SG_IOSUBMIT, ptp) < 0; ++k) {
  710         if ((ENOMEM == errno) && (k < MAX_CONSEC_NOMEMS)) {
  711             ++enomem;
  712             this_thread::yield();
  713             continue;
  714         } else if (EAGAIN == errno) {
  715             ++eagains;
  716             this_thread::yield();
  717             continue;
  718         } else if (EBUSY == errno) {
  719             ++ebusy;
  720             this_thread::yield();
  721             continue;
  722         } else if (E2BIG == errno) {
  723             ++e2big;
  724             return 2;
  725         } else if (EDOM == errno)
  726             ++edom;
  727         else if (ENOMEM == errno)
  728             pr_rusage(-1);
  729         pr_errno_lk(errno, "%s: %s, pack_id=%d", __func__, np, pack_id);
  730         return -1;
  731     }
  732     return 0;
  733 }
  734 
  735 static int
  736 finish_sg4_cmd(int sg_fd, command2execute cmd2exe, int & pack_id,
  737                bool receive, int wait_ms, unsigned int & enomem,
  738                unsigned int & eagains, unsigned int & ebusys,
  739                unsigned int & nanosecs)
  740 {
  741     bool ok;
  742     int res, k;
  743     uint8_t sense_buffer[64];
  744     const char * np = NULL;
  745     struct sg_io_v4 * ptp;
  746     struct sg_io_v4 p4t;
  747 
  748     if (! receive) {
  749         pr2serr_lk("%s: logic error, receive must be true, isn't\n",
  750                    __func__);
  751         return -1;
  752     }
  753     ptp = &p4t;
  754     memset(ptp, 0, sizeof(*ptp));
  755     switch (cmd2exe) {
  756     case SCSI_TUR:
  757         np = "TEST UNIT READY";
  758         break;
  759     case SCSI_READ16:
  760         np = "READ(16)";
  761         break;
  762     case SCSI_WRITE16:
  763         np = "WRITE(16)";
  764         break;
  765     }
  766     ptp->guard = 'Q';
  767     ptp->max_response_len = sizeof(sense_buffer);
  768     ptp->response = (uint64_t)sense_buffer;
  769     ptp->timeout = DEF_TIMEOUT_MS;
  770     /* if SG_SET_FORCE_PACK_ID, then need to set ptp->dxfer_direction */
  771     ptp->request_extra = pack_id;
  772 
  773     k = 0;
  774     while ((((res = ioctl(sg_fd, SG_IORECEIVE, ptp))) < 0) &&
  775            ((EAGAIN == errno) || (EBUSY == errno))) {
  776         if (EAGAIN == errno)
  777             ++eagains;
  778         else
  779             ++ebusys;
  780         ++k;
  781         if (k > 10000) {
  782             pr2serr_lk("%s: sg_fd=%d: after %d EAGAINs, unable to find "
  783                        "pack_id=%d\n", __func__, sg_fd, k, pack_id);
  784             return -1;      /* crash out */
  785         }
  786         if (wait_ms > 0)
  787             this_thread::sleep_for(milliseconds{wait_ms});
  788         else if (0 == wait_ms)
  789             this_thread::yield();
  790         else if (-2 == wait_ms)
  791             sleep(0);                   // process yield ??
  792     }
  793     if (res < 0) {
  794         if (ENOMEM == errno) {
  795             ++enomem;
  796             pr_rusage(-1);
  797         }
  798         pr_errno_lk(errno, "%s: %s", __func__, np);
  799         return -1;
  800     }
  801     /* now for the error processing */
  802     pack_id = ptp->request_extra;
  803     ok = false;
  804     res = sg_err_category_new(ptp->device_status, ptp->transport_status,
  805                               ptp->driver_status,
  806                               (const uint8_t *)ptp->response,
  807                               ptp->response_len);
  808     switch (res) {
  809     case SG_LIB_CAT_CLEAN:
  810         ok = true;
  811         break;
  812     case SG_LIB_CAT_RECOVERED:
  813         pr2serr_lk("%s: Recovered error on %s, continuing\n", __func__, np);
  814         ok = true;
  815         break;
  816     default: /* won't bother decoding other categories */
  817         {
  818             lock_guard<mutex> lg(console_mutex);
  819 
  820             sg_linux_sense_print(np, ptp->device_status,
  821                                  ptp->transport_status,
  822                                  ptp->driver_status,
  823                                  (const uint8_t *)ptp->response,
  824                                  ptp->response_len, true);
  825         }
  826         break;
  827     }
  828     if (ok)
  829         nanosecs = ptp->duration;
  830     return ok ? 0 : -1;
  831 }
  832 
  833 static int
  834 num_submitted(int sg_fd)
  835 {
  836     uint32_t num_subm_wait = 0;
  837     struct sg_extended_info sei;
  838     struct sg_extended_info *seip = &sei;
  839     const char * err = NULL;
  840 
  841     memset(seip, 0, sizeof(*seip));
  842     seip->sei_wr_mask |= SG_SEIM_READ_VAL;
  843     seip->sei_rd_mask |= SG_SEIM_READ_VAL;
  844     seip->read_value = SG_SEIRV_SUBMITTED;
  845     if (ioctl(sg_fd, SG_SET_GET_EXTENDED, seip) < 0)
  846         err = "ioctl(SG_SET_GET_EXTENDED) failed\n";
  847     else
  848         num_subm_wait = seip->read_value;
  849     if (err)
  850         pr2serr_lk("%s: %s, errno=%d\n", __func__, err, errno);
  851     return err ? -1 : (int)num_subm_wait;
  852 }
  853 
  854 static int
  855 pr_rusage(int id)
  856 {
  857     int res;
  858     struct rusage ru;
  859 
  860     res = getrusage(RUSAGE_SELF /* RUSAGE_THREAD */, &ru);
  861     if (res < 0) {
  862         pr2serr_lk("%d->id: %s: getrusage() failed, errno=%d\n", id,
  863                    __func__, errno);
  864         return res;
  865     }
  866     pr2serr_lk("%d->id: maxrss=%ldKB  nvcsw=%ld nivcsw=%ld  majflt=%ld\n", id,
  867                ru.ru_maxrss, ru.ru_nvcsw, ru.ru_nivcsw, ru.ru_majflt);
  868     return 0;
  869 }
  870 
  871 static void
  872 work_sync_thread(int id, const char * dev_name, unsigned int /* hi_lba */,
  873                  struct opts_t * op)
  874 {
  875     bool is_rw = (SCSI_TUR != op->c2e);
  876     int k, sg_fd, err, rs, n, sense_cat, ret;
  877     int vb = op->verbose;
  878     int num_errs = 0;
  879     int thr_sync_starts = 0;
  880     struct sg_pt_base * ptp = NULL;
  881     uint8_t cdb[6];
  882     uint8_t sense_b[32];
  883     char b[120];
  884 
  885     if (is_rw) {
  886         pr2serr_lk("id=%d: only support TUR here for now\n", id);
  887         goto err_out;
  888     }
  889     if (op->no_waitq)
  890         pr2serr_lk("id=%d: ignoring --no-waitq option\n", id);
  891     if (op->verbose)
  892         pr2serr_lk("id=%d: using libsgutils generic sync passthrough\n", id);
  893 
  894     if ((sg_fd = sg_cmds_open_device(dev_name, false /* ro */, vb)) < 0) {
  895         pr2serr_lk("id=%d: error opening file: %s: %s\n", id, dev_name,
  896                    safe_strerror(-sg_fd));
  897         if (ENOMEM == -sg_fd)
  898             pr_rusage(id);
  899         goto err_out;
  900     }
  901     if (vb > 2)
  902         pr2serr_lk(">>>> id=%d: open(%s) --> fd=%d\n", id, dev_name, sg_fd);
  903 
  904     ptp = construct_scsi_pt_obj_with_fd(sg_fd, vb);
  905     err = 0;
  906     if ((NULL == ptp) || ((err = get_scsi_pt_os_err(ptp)))) {
  907         ret = sg_convert_errno(err ? err : ENOMEM);
  908         sg_exit2str(ret, true, sizeof(b), b);
  909         pr2serr_lk("id=%d: construct_scsi_pt_obj_with_fd: %s\n", id, b);
  910         goto err_out;
  911     }
  912     for (k = 0; k < op->num_per_thread; ++k) {
  913         /* Might get Unit Attention on first invocation */
  914         memset(cdb, 0, sizeof(cdb));    /* TUR's cdb is 6 zeros */
  915         set_scsi_pt_cdb(ptp, cdb, sizeof(cdb));
  916         set_scsi_pt_sense(ptp, sense_b, sizeof(sense_b));
  917         set_scsi_pt_packet_id(ptp, uniq_pack_id.fetch_add(1));
  918         ++thr_sync_starts;
  919         rs = do_scsi_pt(ptp, -1, DEF_PT_TIMEOUT, vb);
  920         n = sg_cmds_process_resp(ptp, "Test unit ready", rs,
  921                                  (0 == k), vb, &sense_cat);
  922         if (-1 == n) {
  923             ret = sg_convert_errno(get_scsi_pt_os_err(ptp));
  924             sg_exit2str(ret, true, sizeof(b), b);
  925             pr2serr_lk("id=%d: do_scsi_pt: %s\n", id, b);
  926             goto err_out;
  927         } else if (-2 == n) {
  928             switch (sense_cat) {
  929             case SG_LIB_CAT_RECOVERED:
  930             case SG_LIB_CAT_NO_SENSE:
  931                 break;
  932             case SG_LIB_CAT_NOT_READY:
  933                 ++num_errs;
  934                 if (1 ==  op->num_per_thread) {
  935                     pr2serr_lk("id=%d: device not ready\n", id);
  936                 }
  937                 break;
  938             case SG_LIB_CAT_UNIT_ATTENTION:
  939                 ++num_errs;
  940                 if (vb)
  941                     pr2serr_lk("Ignoring Unit attention (sense key)\n");
  942                 break;
  943             default:
  944                 ++num_errs;
  945                 if (1 == op->num_per_thread) {
  946                     sg_get_category_sense_str(sense_cat, sizeof(b), b, vb);
  947                     pr2serr_lk("%s\n", b);
  948                     goto err_out;
  949                 }
  950                 break;
  951             }
  952         }
  953         clear_scsi_pt_obj(ptp);
  954     }
  955 err_out:
  956     if (ptp)
  957         destruct_scsi_pt_obj(ptp);
  958     if (num_errs > 0)
  959         pr2serr_lk("id=%d: number of errors: %d\n", id, num_errs);
  960     sync_starts += thr_sync_starts;
  961 }
  962 
  963 static void
  964 work_thread(int id, struct opts_t * op)
  965 {
  966     bool is_rw = (SCSI_TUR != op->c2e);
  967     bool need_finish, repeat;
  968     bool once = false;
  969     bool once1000 = false;
  970     bool once_2000 = false;
  971     bool once_4000 = false;
  972     bool once5000 = false;
  973     bool once_6000 = false;
  974     bool once_7000 = false;
  975     bool once10_000 = false;
  976     bool once20_000 = false;
  977     int open_flags = O_RDWR;
  978     int thr_async_starts = 0;
  979     int thr_async_finishes = 0;
  980     int thr_ovn_force_read = 0;
  981     int vb = op->verbose;
  982     int k, n, res, sg_fd, num_outstanding, do_inc, npt, pack_id, sg_flags;
  983     int num_waiting_read, num_to_read, sz, ern, encore_pack_id, ask, j, m, o;
  984     int prev_pack_id, blk_sz;
  985     unsigned int thr_enomem_count = 0;
  986     unsigned int thr_start_eagain_count = 0;
  987     unsigned int thr_start_ebusy_count = 0;
  988     unsigned int thr_start_e2big_count = 0;
  989     unsigned int thr_fin_eagain_count = 0;
  990     unsigned int thr_fin_ebusy_count = 0;
  991     unsigned int thr_start_edom_count = 0;
  992     unsigned int seed = 0;
  993     int needed_sz = op->lb_sz * op->num_lbs;
  994     unsigned int nanosecs;
  995     unsigned int hi_lba;
  996     uint64_t lba;
  997     uint64_t sum_nanosecs = 0;
  998     uint8_t * lbp;
  999     uint8_t * free_lbp = NULL;
 1000     uint8_t * wrkMmap = NULL;
 1001     const char * dev_name;
 1002     const char * err = NULL;
 1003     Rand_uint * ruip = NULL;
 1004     char ebuff[EBUFF_SZ];
 1005     struct pollfd  pfd[1];
 1006     list<pair<uint8_t *, uint8_t *> > free_lst;   /* of aligned lb buffers */
 1007     map<int, pair<uint8_t *, uint8_t *> > pi2buff;/* pack_id -> lb buffer */
 1008     map<int, uint64_t> pi_2_lba;            /* pack_id -> LBA */
 1009     pair<uint8_t *, uint8_t *> encore_lbps;
 1010 
 1011     /* device name and hi_lba may depend on id */
 1012     n = op->dev_names.size();
 1013     dev_name = op->dev_names[id % n];
 1014     if (op->blk_szs.size() >= (unsigned)n)
 1015         blk_sz = op->blk_szs[id % n];
 1016     else
 1017         blk_sz = DEF_LB_SZ;
 1018     if ((UINT_MAX == op->hi_lba) && (n == (int)op->hi_lbas.size()))
 1019         hi_lba = op->hi_lbas[id % n];
 1020     else
 1021         hi_lba = op->hi_lba;
 1022 
 1023     if (vb) {
 1024         if ((vb > 1) && hi_lba)
 1025             pr2serr_lk("Enter work_t_id=%d using %s\n"
 1026                        "    LBA range: 0x%x to 0x%x (inclusive)\n",
 1027                        id, dev_name, (unsigned int)op->lba, hi_lba);
 1028         else
 1029             pr2serr_lk("Enter work_t_id=%d using %s\n", id, dev_name);
 1030     }
 1031     if (op->generic_sync) {
 1032         work_sync_thread(id, dev_name, hi_lba, op);
 1033         return;
 1034     }
 1035     if (! op->block)
 1036         open_flags |= O_NONBLOCK;
 1037 
 1038     sg_fd = open(dev_name, open_flags);
 1039     if (sg_fd < 0) {
 1040         pr_errno_lk(errno, "%s: id=%d, error opening file: %s", __func__, id,
 1041                     dev_name);
 1042         if (ENOMEM == -sg_fd)
 1043             pr_rusage(id);
 1044         return;
 1045     }
 1046     if (vb > 2)
 1047         pr2serr_lk(">>>> id=%d: open(%s) --> fd=%d\n", id, dev_name, sg_fd);
 1048     if (op->pack_id_force) {
 1049         k = 1;
 1050         if (ioctl(sg_fd, SG_SET_FORCE_PACK_ID, &k) < 0)
 1051             pr2serr_lk("ioctl(SG_SET_FORCE_PACK_ID) failed, errno=%d %s\n",
 1052                        errno, strerror(errno));
 1053     }
 1054     if (op->sg_vn_ge_40000) {
 1055         if (ioctl(sg_fd, SG_GET_RESERVED_SIZE, &k) >= 0) {
 1056             if (needed_sz > k)
 1057                 ioctl(sg_fd, SG_SET_RESERVED_SIZE, &needed_sz);
 1058         }
 1059         if (op->sg_vn_ge_40030 && (op->cmd_time || op->masync)) {
 1060             struct sg_extended_info sei;
 1061             struct sg_extended_info * seip;
 1062 
 1063             seip = &sei;
 1064             memset(seip, 0, sizeof(*seip));
 1065             seip->sei_wr_mask |= SG_SEIM_CTL_FLAGS;
 1066             seip->sei_rd_mask |= SG_SEIM_CTL_FLAGS;
 1067             if (op->cmd_time) {
 1068                 seip->ctl_flags_wr_mask |= SG_CTL_FLAGM_TIME_IN_NS;
 1069                 seip->ctl_flags_rd_mask |= SG_CTL_FLAGM_TIME_IN_NS;
 1070                 seip->ctl_flags |= SG_CTL_FLAGM_TIME_IN_NS;
 1071             }
 1072             if (op->masync) {
 1073                 seip->ctl_flags_wr_mask |= SG_CTL_FLAGM_MORE_ASYNC;
 1074                 seip->ctl_flags |= SG_CTL_FLAGM_MORE_ASYNC;
 1075             }
 1076             if (op->excl) {
 1077                 seip->ctl_flags_wr_mask |= SG_CTL_FLAGM_EXCL_WAITQ;
 1078                 seip->ctl_flags |= SG_CTL_FLAGM_EXCL_WAITQ;
 1079             }
 1080             if (ioctl(sg_fd, SG_SET_GET_EXTENDED, seip) < 0) {
 1081                 pr2serr_lk("ioctl(EXTENDED(TIME_IN_NS)) failed, errno=%d %s\n",
 1082                            errno, strerror(errno));
 1083             }
 1084             if (op->cmd_time &&
 1085                 (! (SG_CTL_FLAGM_TIME_IN_NS & seip->ctl_flags))) {
 1086                 memset(seip, 0, sizeof(*seip));
 1087                 seip->sei_rd_mask |= SG_SEIM_CTL_FLAGS;
 1088                 seip->sei_wr_mask |= SG_SEIM_CTL_FLAGS;
 1089                 seip->ctl_flags_wr_mask |= SG_CTL_FLAGM_TIME_IN_NS;
 1090                 seip->ctl_flags |= SG_CTL_FLAGM_TIME_IN_NS;
 1091                 if (ioctl(sg_fd, SG_SET_GET_EXTENDED, seip) < 0)
 1092                     pr2serr_lk("ioctl(EXTENDED(TIME_IN_NS)) failed, "
 1093                                "errno=%d %s\n", errno, strerror(errno));
 1094                 else if (vb > 1)
 1095                     pr2serr_lk("t_id: %d: set TIME_IN_NS flag\n", id);
 1096             }
 1097         }
 1098     }
 1099     if (is_rw && op->mmap_io) {
 1100 
 1101         if (ioctl(sg_fd, SG_GET_RESERVED_SIZE, &sz) < 0) {
 1102             pr2serr_lk("t_id=%d: ioctl(SG_GET_RESERVED_SIZE) errno=%d\n",
 1103                        id, errno);
 1104             return;
 1105         }
 1106         if (sz < needed_sz) {
 1107             sz = needed_sz;
 1108             if (ioctl(sg_fd, SG_SET_RESERVED_SIZE, &sz) < 0) {
 1109                 pr2serr_lk("t_id=%d: ioctl(SG_SET_RESERVED_SIZE) errno=%d\n",
 1110                            id, errno);
 1111                 return;
 1112             }
 1113             if (ioctl(sg_fd, SG_GET_RESERVED_SIZE, &sz) < 0) {
 1114                 pr2serr_lk("t_id=%d: ioctl(SG_GET_RESERVED_SIZE) errno=%d\n",
 1115                            id, errno);
 1116                 return;
 1117             }
 1118             if (sz < needed_sz) {
 1119                 pr2serr_lk("t_id=%d: unable to grow reserve buffer to %d "
 1120                            "bytes\n", id, needed_sz);
 1121                 return;
 1122             }
 1123         }
 1124         wrkMmap = (uint8_t *)mmap(NULL, needed_sz, PROT_READ | PROT_WRITE,
 1125                                   MAP_SHARED, sg_fd, 0);
 1126         if (MAP_FAILED == wrkMmap) {
 1127             ern = errno;
 1128             pr2serr_lk("t_id=%d: mmap() failed, errno=%d\n", id, ern);
 1129             return;
 1130         }
 1131     }
 1132     pfd[0].fd = sg_fd;
 1133     pfd[0].events = POLLIN;
 1134     if (is_rw && hi_lba) {
 1135         seed = get_urandom_uint();
 1136         if (vb > 1)
 1137             pr2serr_lk("  id=%d, /dev/urandom seed=0x%x\n", id, seed);
 1138         ruip = new Rand_uint((unsigned int)op->lba, hi_lba, seed);
 1139     }
 1140 
 1141     sg_flags = 0;
 1142     if (BLQ_AT_TAIL == op->blqd)
 1143         sg_flags |= SG_FLAG_Q_AT_TAIL;
 1144     else if (BLQ_AT_HEAD == op->blqd)
 1145         sg_flags |= SG_FLAG_Q_AT_HEAD;
 1146     if (op->direct)
 1147         sg_flags |= SG_FLAG_DIRECT_IO;
 1148     if (op->mmap_io)
 1149         sg_flags |= SG_FLAG_MMAP_IO;
 1150     if (op->no_xfer)
 1151         sg_flags |= SG_FLAG_NO_DXFER;
 1152     if (op->no_waitq)
 1153         sg_flags |= SGV4_FLAG_NO_WAITQ;
 1154     if (vb > 1)
 1155         pr2serr_lk("  id=%d, sg_flags=0x%x, %s cmds\n", id, sg_flags,
 1156                    ((SCSI_TUR == op->c2e) ? "TUR":
 1157                     ((SCSI_READ16 == op->c2e) ? "READ" : "WRITE")));
 1158 
 1159     npt = op->num_per_thread;
 1160     need_finish = false;
 1161     lba = 0;
 1162     pack_id = 0;
 1163     prev_pack_id = 0;
 1164     encore_pack_id = 0;
 1165     do_inc = 0;
 1166     /* main loop, continues until num_per_thread exhausted and there are
 1167      * no more outstanding responses */
 1168     for (k = 0, m = 0, o=0, num_outstanding = 0; (k < npt) || num_outstanding;
 1169          k = do_inc ? k + 1 : k, ++o) {
 1170         if (do_inc)
 1171             m = 0;
 1172         else {
 1173             ++m;
 1174             if (m > 100) {
 1175                 if (vb)
 1176                     pr2serr_lk("%d->id: no main loop inc =%d times\n", id, m);
 1177                 m = 0;
 1178             }
 1179         }
 1180         if (vb && (! once1000) && (num_outstanding >= 1000)) {
 1181             int num_waiting;
 1182             int num_subm = (op->sg_vn_ge_40030) ? num_submitted(sg_fd) :
 1183                                                   pi2buff.size();
 1184 
 1185             once1000 = true;
 1186             if (ioctl(sg_fd, SG_GET_NUM_WAITING, &num_waiting) < 0) {
 1187                 err = "ioctl(SG_GET_NUM_WAITING) failed";
 1188                 break;
 1189             }
 1190             pr2serr_lk("%d->id: once 1000: k=%d, submitted=%d waiting=%d; "
 1191                        "pi2buff.sz=%u\n", id, k, num_subm, num_waiting,
 1192                        (uint32_t)pi2buff.size());
 1193             pr_rusage(id);
 1194         }
 1195         if (vb && ! once5000 && num_outstanding >= 5000) {
 1196             int num_waiting;
 1197             int num_subm = (op->sg_vn_ge_40030) ? num_submitted(sg_fd) :
 1198                                                   pi2buff.size();
 1199 
 1200             once5000 = true;
 1201             if (ioctl(sg_fd, SG_GET_NUM_WAITING, &num_waiting) < 0) {
 1202                 err = "ioctl(SG_GET_NUM_WAITING) failed";
 1203                 break;
 1204             }
 1205             pr2serr_lk("%d->id: once 5000: k=%d, submitted=%d waiting=%d\n",
 1206                        id, k, num_subm, num_waiting);
 1207             pr_rusage(id);
 1208         }
 1209         if (vb && ! once_7000 && num_outstanding >= 7000) {
 1210             int num_waiting;
 1211             int num_subm = (op->sg_vn_ge_40030) ? num_submitted(sg_fd) :
 1212                                                   pi2buff.size();
 1213 
 1214             once_7000 = true;
 1215             if (ioctl(sg_fd, SG_GET_NUM_WAITING, &num_waiting) < 0) {
 1216                 err = "ioctl(SG_GET_NUM_WAITING) failed";
 1217                 break;
 1218             }
 1219             pr2serr_lk("%d->id: once 7000: k=%d, submitted=%d waiting=%d\n",
 1220                        id, k, num_subm, num_waiting);
 1221             pr_rusage(id);
 1222         }
 1223         if (vb && ! once10_000 && num_outstanding >= 10000) {
 1224             int num_waiting;
 1225             int num_subm = (op->sg_vn_ge_40030) ? num_submitted(sg_fd) :
 1226                                                   pi2buff.size();
 1227 
 1228             once10_000 = true;
 1229             if (ioctl(sg_fd, SG_GET_NUM_WAITING, &num_waiting) < 0) {
 1230                 err = "ioctl(SG_GET_NUM_WAITING) failed";
 1231                 break;
 1232             }
 1233             pr2serr_lk("%d->id: once 10^4: k=%d, submitted=%d waiting=%d\n",
 1234                        id, k, num_subm, num_waiting);
 1235             pr_rusage(id);
 1236         }
 1237         if (vb && ! once20_000 && num_outstanding >= 20000) {
 1238             int num_waiting;
 1239             int num_subm = (op->sg_vn_ge_40030) ? num_submitted(sg_fd) :
 1240                                                   pi2buff.size();
 1241 
 1242             once20_000 = true;
 1243             if (ioctl(sg_fd, SG_GET_NUM_WAITING, &num_waiting) < 0) {
 1244                 err = "ioctl(SG_GET_NUM_WAITING) failed";
 1245                 break;
 1246             }
 1247             pr2serr_lk("%d->id: once 20000: k=%d, submitted=%d waiting=%d\n",
 1248                        id, k, num_subm, num_waiting);
 1249             pr_rusage(id);
 1250         }
 1251         do_inc = 0;
 1252         if ((num_outstanding < op->maxq_per_thread) && (k < npt)) {
 1253             do_inc = 1;
 1254             if (need_finish) {
 1255                 pack_id = encore_pack_id;
 1256                 need_finish = false;
 1257                 repeat = true;
 1258             } else {
 1259                 prev_pack_id = pack_id;
 1260                 pack_id = uniq_pack_id.fetch_add(1);
 1261                 repeat = false;
 1262             }
 1263             if (is_rw) {    /* get new lb buffer or one from free list */
 1264                 if (free_lst.empty()) {
 1265                     lbp = sg_memalign(op->lb_sz * op->num_lbs, 0, &free_lbp,
 1266                                       false);
 1267                     if (NULL == lbp) {
 1268                         err = "out of memory";
 1269                         break;
 1270                     }
 1271                 } else if (! repeat) {
 1272                     lbp = free_lst.back().first;
 1273                     free_lbp = free_lst.back().second;
 1274                     free_lst.pop_back();
 1275                 } else {
 1276                     lbp = encore_lbps.first;
 1277                     free_lbp = encore_lbps.second;
 1278                     if (vb && !once && free_lst.size() > 1000) {
 1279                         once = true;
 1280                         pr2serr_lk("%d->id: free_lst.size() over 1000\n", id);
 1281                     }
 1282                     if (vb && !once_2000 && free_lst.size() > 2000) {
 1283                         once_2000 = true;
 1284                         pr2serr_lk("%d->id: free_lst.size() over 2000\n", id);
 1285                     }
 1286                     if (vb && !once_6000 && free_lst.size() > 6000) {
 1287                         once_2000 = true;
 1288                         pr2serr_lk("%d->id: free_lst.size() over 6000\n", id);
 1289                     }
 1290                 }
 1291             } else
 1292                 lbp = NULL;
 1293             if (is_rw) {
 1294                 if (ruip) {
 1295                     if (! repeat) {
 1296                         lba = ruip->get();  /* fetch a random LBA */
 1297                         if (vb > 3)
 1298                             pr2serr_lk("  id=%d: start IO at lba=0x%" PRIx64
 1299                                        "\n", id, lba);
 1300                     }
 1301                 } else
 1302                     lba = op->lba;
 1303             } else
 1304                 lba = 0;
 1305             if (vb > 4)
 1306                 pr2serr_lk("t_id=%d: starting pack_id=%d\n", id, pack_id);
 1307             res = (op->v4) ?
 1308                 start_sg4_cmd(sg_fd, op->c2e, pack_id, lba, lbp,
 1309                               blk_sz * op->num_lbs, sg_flags, op->submit,
 1310                               thr_enomem_count, thr_start_eagain_count,
 1311                               thr_start_ebusy_count, thr_start_e2big_count,
 1312                               thr_start_edom_count)  :
 1313                 start_sg3_cmd(sg_fd, op->c2e, pack_id, lba, lbp,
 1314                               blk_sz * op->num_lbs, sg_flags, op->submit,
 1315                               thr_enomem_count, thr_start_eagain_count,
 1316                               thr_start_ebusy_count, thr_start_e2big_count,
 1317                               thr_start_edom_count);
 1318             if (res) {
 1319                 if (res > 1) { /* here if E2BIG, start not done, try finish */
 1320                     do_inc = 0;
 1321                     need_finish = true;
 1322                     encore_pack_id = pack_id;
 1323                     pack_id = prev_pack_id;
 1324                     encore_lbps = make_pair(lbp, free_lbp);
 1325                     if (vb > 2)
 1326                         pr2serr_lk("t_id=%d: E2BIG hit, prev_pack_id=%d, "
 1327                                    "encore_pack_id=%d\n", id, prev_pack_id,
 1328                                    encore_pack_id);
 1329                 } else {
 1330                     err = "start_sg3_cmd()";
 1331                     break;
 1332                 }
 1333             } else {    /* no error */
 1334                 ++thr_async_starts;
 1335                 ++num_outstanding;
 1336                 pi2buff[pack_id] = make_pair(lbp, free_lbp);
 1337                 if (ruip)
 1338                     pi_2_lba[pack_id] = lba;
 1339             }
 1340             if (vb && !once && (pi2buff.size() > 1000)) {
 1341                 once = true;
 1342                 pr2serr_lk("%d->id: pi2buff.size() over 1000 (b)\n", id);
 1343             }
 1344             if (vb && !once_2000 && free_lst.size() > 2000) {
 1345                 once_2000 = true;
 1346                 pr2serr_lk("%d->id: free_lst.size() over 2000 (b)\n", id);
 1347             }
 1348             if (vb && !once_6000 && free_lst.size() > 6000) {
 1349                 once_2000 = true;
 1350                 pr2serr_lk("%d->id: free_lst.size() over 6000 (b)\n", id);
 1351             }
 1352         }
 1353         num_to_read = 0;
 1354         if (need_finish) {
 1355             num_waiting_read = 0;
 1356             if (ioctl(sg_fd, SG_GET_NUM_WAITING, &num_waiting_read) < 0) {
 1357                 err = "ioctl(SG_GET_NUM_WAITING) failed";
 1358                 break;
 1359             } else if (vb > 3)
 1360                 pr2serr_lk("t_id=%d: num_waiting_read=%d\n", id,
 1361                            num_waiting_read);
 1362             if (num_waiting_read > 0)
 1363                 num_to_read = num_waiting_read;
 1364             else {
 1365                 struct timespec tspec = {0, 100000 /* 100 usecs */};
 1366 
 1367                 nanosleep(&tspec, NULL);
 1368                 if (vb > 3)
 1369                     pr2serr_lk("t_id=%d: E2BIG, 100 usecs sleep\n", id);
 1370                 // err = "strange, E2BIG but nothing to read";
 1371                 // break;
 1372             }
 1373         } else if ((num_outstanding >= op->maxq_per_thread) || (k >= npt)) {
 1374             /* full queue or finished injecting */
 1375             num_waiting_read = 0;
 1376             if (ioctl(sg_fd, SG_GET_NUM_WAITING, &num_waiting_read) < 0) {
 1377                 err = "ioctl(SG_GET_NUM_WAITING) failed";
 1378                 break;
 1379             }
 1380             if (1 == num_waiting_read)
 1381                 num_to_read = num_waiting_read;
 1382             else if (num_waiting_read > 0) {
 1383                 if (k >= npt)
 1384                     num_to_read = num_waiting_read;
 1385                 else {
 1386                     switch (op->myqd) {
 1387                     case MYQD_LOW:
 1388                         num_to_read = num_waiting_read;
 1389                         break;
 1390                     case MYQD_MEDIUM:
 1391                         num_to_read = num_waiting_read / 2;
 1392                         break;
 1393                     case MYQD_HIGH:
 1394                     default:
 1395                         if (op->ovn > 0) {
 1396                             if (op->sg_vn_ge_40030) {
 1397                                 int num_subm = num_submitted(sg_fd);
 1398 
 1399                                 if (num_subm > op->ovn) {
 1400                                     num_to_read = num_waiting_read > 0 ?
 1401                                                     num_waiting_read : 1;
 1402                                     break;
 1403                                 }
 1404                             } else {
 1405                                 if (num_waiting_read > (op->ovn / 2)) {
 1406                                     num_to_read = num_waiting_read / 2;
 1407                                     break;
 1408                                 }
 1409                             }
 1410                         }
 1411                         num_to_read = 1;
 1412                         break;
 1413                     }
 1414                 }
 1415             } else {    /* nothing waiting to be read */
 1416                 if (op->sg_vn_ge_40030) {
 1417                     int val = num_submitted(sg_fd);
 1418 
 1419                     if (0 == val) {
 1420                         err = "nothing submitted now ??";
 1421                         break;
 1422                     } else if (val < 0) {
 1423                         err = "num_submitted failed";
 1424                         break;
 1425                     }
 1426                 }
 1427                 n = (op->wait_ms > 0) ? op->wait_ms : 0;
 1428                 if ((n > 0) && (! op->no_waitq)) {
 1429                     for (j = 0; (j < 1000000) &&
 1430                          (0 == (res = poll(pfd, 1, n)));
 1431                          ++j)
 1432                         ;
 1433                     if (j >= 1000000) {
 1434                         err = "poll() looped 1 million times";
 1435                         break;
 1436                     }
 1437                     if (res < 0) {
 1438                         err = "poll(wait_ms) failed";
 1439                         break;
 1440                     }
 1441                 } else {
 1442                     struct timespec ts;
 1443 
 1444                     ts.tv_sec = 0;
 1445                     ts.tv_nsec = DEF_NANOSEC_WAIT;
 1446                     if (nanosleep(&ts, NULL) < 0) {
 1447                         err = "nanosleep() failed";
 1448                         break;
 1449                     }
 1450                 }
 1451             }
 1452         } else {        /* not full, not finished injecting */
 1453             if (MYQD_HIGH == op->myqd) {
 1454                 num_to_read = 0;
 1455                 if (op->ovn) {
 1456                     if (op->sg_vn_ge_40030) {
 1457                         int num_subm = num_submitted(sg_fd);
 1458 
 1459                         if (num_subm > op->ovn)
 1460                             num_to_read = num_waiting_read > 0 ?
 1461                                             num_waiting_read : 1;
 1462                     } else {
 1463                         num_waiting_read = 0;
 1464                         if (ioctl(sg_fd, SG_GET_NUM_WAITING,
 1465                                   &num_waiting_read) < 0) {
 1466                             err = "ioctl(SG_GET_NUM_WAITING) failed";
 1467                             break;
 1468                         }
 1469                         if (num_waiting_read > (op->ovn / 2))
 1470                             num_to_read = num_waiting_read / 2;
 1471                     }
 1472                 }
 1473             } else {
 1474                 num_waiting_read = 0;
 1475                 if (ioctl(sg_fd, SG_GET_NUM_WAITING, &num_waiting_read) < 0) {
 1476                     err = "ioctl(SG_GET_NUM_WAITING) failed";
 1477                     break;
 1478                 }
 1479                 if (num_waiting_read > 0)
 1480                     num_to_read = num_waiting_read /
 1481                                   ((MYQD_LOW == op->myqd) ? 1 : 2);
 1482                 else
 1483                     num_to_read = 0;
 1484             }
 1485         }
 1486 
 1487         if (vb && !once_4000 && (num_to_read > 4000)) {
 1488             once_4000 = true;
 1489             pr2serr_lk("%d->id: num_to_read=%d\n", id, num_to_read);
 1490         }
 1491         while (num_to_read > 0) {
 1492             --num_to_read;
 1493             if (op->pack_id_force) {
 1494                 j = pi2buff.size();
 1495                 if (j > 0)
 1496                     pack_id = pi2buff.begin()->first;
 1497                 else
 1498                     pack_id = -1;
 1499             } else
 1500                 pack_id = -1;
 1501             ask = pack_id;
 1502             res = (op->v4) ?
 1503                     finish_sg4_cmd(sg_fd, op->c2e, pack_id, op->submit,
 1504                                    op->wait_ms, thr_enomem_count,
 1505                                    thr_fin_eagain_count, thr_fin_ebusy_count,
 1506                                    nanosecs)           :
 1507                     finish_sg3_cmd(sg_fd, op->c2e, pack_id, op->submit,
 1508                                    op->wait_ms, thr_enomem_count,
 1509                                    thr_fin_eagain_count, thr_fin_ebusy_count,
 1510                                    nanosecs);
 1511             if (res) {
 1512                 err = "finish_sg3_cmd()";
 1513                 if (ruip && (pack_id > 0)) {
 1514                     auto q = pi_2_lba.find(pack_id);
 1515 
 1516                     if (q != pi_2_lba.end()) {
 1517                         snprintf(ebuff, sizeof(ebuff), "%s: lba=0x%" PRIx64 ,
 1518                                  err, q->second);
 1519                         err = ebuff;
 1520                     }
 1521                 }
 1522                 break;
 1523             }
 1524             if (op->cmd_time && op->sg_vn_ge_40030)
 1525                 sum_nanosecs += nanosecs;
 1526             ++thr_async_finishes;
 1527             --num_outstanding;
 1528             if (vb > 4)
 1529                 pr2serr_lk("t_id=%d: finishing pack_id ask=%d, got=%d, "
 1530                            "outstanding=%d\n", id, ask, pack_id,
 1531                            num_outstanding);
 1532             auto p = pi2buff.find(pack_id);
 1533 
 1534             if (p == pi2buff.end()) {
 1535                 snprintf(ebuff, sizeof(ebuff), "pack_id=%d from "
 1536                          "finish_sg3_cmd() not found\n", pack_id);
 1537                 if (! err)
 1538                     err = ebuff;
 1539             } else {
 1540                 lbp = p->second.first;
 1541                 free_lbp = p->second.second;
 1542                 pi2buff.erase(p);
 1543                 if (lbp)
 1544                     free_lst.push_front(make_pair(lbp, free_lbp));
 1545             }
 1546             if (ruip && (pack_id > 0)) {
 1547                 auto q = pi_2_lba.find(pack_id);
 1548 
 1549                 if (q != pi_2_lba.end()) {
 1550                     if (vb > 3)
 1551                         pr2serr_lk("    id=%d: finish IO at lba=0x%" PRIx64
 1552                                    "\n", id, q->second);
 1553                     pi_2_lba.erase(q);
 1554                 }
 1555             }
 1556             if (err)
 1557                 break;
 1558         }       /* end of while loop counting down num_to_read */
 1559         if (err)
 1560             break;
 1561     }           /* end of for loop over npt (number per thread) */
 1562     if (vb)
 1563         pr2serr_lk("%d->id: leaving main thread loop; k=%d, o=%d\n", id, k,
 1564                    o);
 1565     close(sg_fd);       // sg driver will handle any commands "in flight"
 1566     if (ruip)
 1567         delete ruip;
 1568 
 1569     if (err || (k < npt)) {
 1570         if (k < npt)
 1571             pr2serr_lk("t_id=%d FAILed at iteration %d%s%s\n", id, k,
 1572                        (err ? ", Reason: " : ""), (err ? err : ""));
 1573         else
 1574             pr2serr_lk("t_id=%d FAILed on last%s%s\n", id,
 1575                        (err ? ", Reason: " : ""), (err ? err : ""));
 1576     }
 1577     n = pi2buff.size();
 1578     if (n > 0)
 1579         pr2serr_lk("t_id=%d Still %d elements in pi2buff map on "
 1580                    "exit\n", id, n);
 1581     for (k = 0; ! free_lst.empty(); ++k) {
 1582         lbp = free_lst.back().first;
 1583         free_lbp = free_lst.back().second;
 1584         free_lst.back().second = NULL;
 1585         free_lst.pop_back();
 1586         if (vb > 6)
 1587             pr2serr_lk("t_id=%d freeing %p (free_ %p)\n", id, lbp, free_lbp);
 1588         if (free_lbp) {
 1589             free(free_lbp);
 1590             free_lbp = NULL;
 1591         }
 1592     }
 1593     if ((vb > 2) && (k > 0))
 1594         pr2serr_lk("%d->id: Maximum number of READ/WRITEs queued: %d\n",
 1595                    id, k);
 1596     if (vb && (thr_ovn_force_read > 0))
 1597         pr2serr_lk("%d->id: Number of ovn (override number) forced reads: "
 1598                    "%d\n", id, thr_ovn_force_read);
 1599     async_starts += thr_async_starts;
 1600     async_finishes += thr_async_finishes;
 1601     start_eagain_count += thr_start_eagain_count;
 1602     start_ebusy_count += thr_start_ebusy_count;
 1603     start_e2big_count += thr_start_e2big_count;
 1604     fin_eagain_count += thr_fin_eagain_count;
 1605     fin_ebusy_count += thr_fin_ebusy_count;
 1606     enomem_count += thr_enomem_count;
 1607     start_edom_count += thr_start_edom_count;
 1608     if (op->cmd_time && op->sg_vn_ge_40030 && (npt > 0)) {
 1609         pr2serr_lk("t_id=%d average nanosecs per cmd: %" PRId64
 1610                    "\n", id, sum_nanosecs / npt);
 1611     }
 1612 }
 1613 
 1614 #define INQ_REPLY_LEN 96
 1615 #define INQ_CMD_LEN 6
 1616 
 1617 /* Send INQUIRY and fetches response. If okay puts PRODUCT ID field
 1618  * in b (up to m_blen bytes). Does not use O_EXCL flag. Returns 0 on success,
 1619  * else -1 . */
 1620 static int
 1621 do_inquiry_prod_id(const char * dev_name, int block, int & sg_ver_num,
 1622                    char * b, int b_mlen)
 1623 {
 1624     int sg_fd, ok, ret;
 1625     struct sg_io_hdr pt;
 1626     uint8_t inqCmdBlk [INQ_CMD_LEN] =
 1627                                 {0x12, 0, 0, 0, INQ_REPLY_LEN, 0};
 1628     uint8_t inqBuff[INQ_REPLY_LEN];
 1629     uint8_t sense_buffer[64];
 1630     int open_flags = O_RDWR;    /* O_EXCL | O_RDONLY fails with EPERM */
 1631 
 1632     if (! block)
 1633         open_flags |= O_NONBLOCK;
 1634     sg_fd = open(dev_name, open_flags);
 1635     if (sg_fd < 0) {
 1636         pr_errno_lk(errno, "%s: error opening file: %s", __func__, dev_name);
 1637         return -1;
 1638     }
 1639     if (ioctl(sg_fd, SG_GET_VERSION_NUM, &sg_ver_num) < 0)
 1640         sg_ver_num = 0;
 1641     /* Prepare INQUIRY command */
 1642     memset(&pt, 0, sizeof(pt));
 1643     pt.interface_id = 'S';
 1644     pt.cmd_len = sizeof(inqCmdBlk);
 1645     /* pt.iovec_count = 0; */  /* memset takes care of this */
 1646     pt.mx_sb_len = sizeof(sense_buffer);
 1647     pt.dxfer_direction = SG_DXFER_FROM_DEV;
 1648     pt.dxfer_len = INQ_REPLY_LEN;
 1649     pt.dxferp = inqBuff;
 1650     pt.cmdp = inqCmdBlk;
 1651     pt.sbp = sense_buffer;
 1652     pt.timeout = 20000;     /* 20000 millisecs == 20 seconds */
 1653     /* pt.flags = 0; */     /* take defaults: indirect IO, etc */
 1654     /* pt.pack_id = 0; */
 1655     /* pt.usr_ptr = NULL; */
 1656 
 1657     if (ioctl(sg_fd, SG_IO, &pt) < 0) {
 1658         pr_errno_lk(errno, "%s: Inquiry SG_IO ioctl error", __func__);
 1659         close(sg_fd);
 1660         return -1;
 1661     }
 1662 
 1663     /* now for the error processing */
 1664     ok = 0;
 1665     switch (sg_err_category3(&pt)) {
 1666     case SG_LIB_CAT_CLEAN:
 1667         ok = 1;
 1668         break;
 1669     case SG_LIB_CAT_RECOVERED:
 1670         pr2serr_lk("Recovered error on INQUIRY, continuing\n");
 1671         ok = 1;
 1672         break;
 1673     default: /* won't bother decoding other categories */
 1674         {
 1675             lock_guard<mutex> lg(console_mutex);
 1676             sg_chk_n_print3("INQUIRY command error", &pt, 1);
 1677         }
 1678         break;
 1679     }
 1680     if (ok) {
 1681         /* Good, so fetch Product ID from response, copy to 'b' */
 1682         if (b_mlen > 0) {
 1683             if (b_mlen > 16) {
 1684                 memcpy(b, inqBuff + 16, 16);
 1685                 b[16] = '\0';
 1686             } else {
 1687                 memcpy(b, inqBuff + 16, b_mlen - 1);
 1688                 b[b_mlen - 1] = '\0';
 1689             }
 1690         }
 1691         ret = 0;
 1692     } else
 1693         ret = -1;
 1694 
 1695     close(sg_fd);
 1696     return ret;
 1697 }
 1698 
 1699 /* Only allow ranges up to 2**32-1 upper limit, so READ CAPACITY(10)
 1700  * sufficient. Return of 0 -> success, -1 -> failure, 2 -> try again */
 1701 static int
 1702 do_read_capacity(const char * dev_name, int block, unsigned int * last_lba,
 1703                  unsigned int * blk_sz)
 1704 {
 1705     int res, sg_fd;
 1706     uint8_t rcCmdBlk [10] = {0x25, 0, 0, 0, 0, 0, 0, 0, 0, 0};
 1707     uint8_t rcBuff[64];
 1708     uint8_t sense_b[64];
 1709     sg_io_hdr_t io_hdr;
 1710     int open_flags = O_RDWR;    /* O_EXCL | O_RDONLY fails with EPERM */
 1711 
 1712     if (! block)
 1713         open_flags |= O_NONBLOCK;
 1714     sg_fd = open(dev_name, open_flags);
 1715     if (sg_fd < 0) {
 1716         pr_errno_lk(errno, "%s: error opening file: %s", __func__, dev_name);
 1717         return -1;
 1718     }
 1719     /* Prepare READ CAPACITY(10) command */
 1720     memset(&io_hdr, 0, sizeof(sg_io_hdr_t));
 1721     io_hdr.interface_id = 'S';
 1722     io_hdr.cmd_len = sizeof(rcCmdBlk);
 1723     io_hdr.mx_sb_len = sizeof(sense_b);
 1724     io_hdr.dxfer_direction = SG_DXFER_FROM_DEV;
 1725     io_hdr.dxfer_len = sizeof(rcBuff);
 1726     io_hdr.dxferp = rcBuff;
 1727     io_hdr.cmdp = rcCmdBlk;
 1728     io_hdr.sbp = sense_b;
 1729     io_hdr.timeout = 20000;     /* 20000 millisecs == 20 seconds */;
 1730 
 1731     if (ioctl(sg_fd, SG_IO, &io_hdr) < 0) {
 1732         pr_errno_lk(errno, "%s (SG_IO) error", __func__);
 1733         close(sg_fd);
 1734         return -1;
 1735     }
 1736     res = sg_err_category3(&io_hdr);
 1737     if (SG_LIB_CAT_UNIT_ATTENTION == res) {
 1738         lock_guard<mutex> lg(console_mutex);
 1739         sg_chk_n_print3("read capacity", &io_hdr, 1);
 1740         close(sg_fd);
 1741         return 2; /* probably have another go ... */
 1742     } else if (SG_LIB_CAT_CLEAN != res) {
 1743         lock_guard<mutex> lg(console_mutex);
 1744         sg_chk_n_print3("read capacity", &io_hdr, 1);
 1745         close(sg_fd);
 1746         return -1;
 1747     }
 1748     *last_lba = sg_get_unaligned_be32(&rcBuff[0]);
 1749     *blk_sz = sg_get_unaligned_be32(&rcBuff[4]);
 1750     close(sg_fd);
 1751     return 0;
 1752 }
 1753 
 1754 
 1755 int
 1756 main(int argc, char * argv[])
 1757 {
 1758     bool maxq_per_thread_given = false;
 1759     int k, n, c, res;
 1760     int force = 0;
 1761     int64_t ll;
 1762     int num_threads = DEF_NUM_THREADS;
 1763     char b[128];
 1764     struct timespec start_tm, end_tm;
 1765     struct opts_t * op;
 1766     const char * cp;
 1767     const char * dev_name;
 1768 
 1769     op = &a_opts;
 1770 #if 0
 1771     memset(op, 0, sizeof(*op));         // C++ doesn't like this
 1772 #endif
 1773     op->direct = DEF_DIRECT;
 1774     op->lba = DEF_LBA;
 1775     op->hi_lba = 0;
 1776     op->lb_sz = DEF_LB_SZ;
 1777     op->maxq_per_thread = MAX_Q_PER_FD;
 1778     op->mmap_io = DEF_MMAP_IO;
 1779     op->num_per_thread = DEF_NUM_PER_THREAD;
 1780     op->num_lbs = 1;
 1781     op->no_xfer = !! DEF_NO_XFER;
 1782     op->verbose = 0;
 1783     op->wait_ms = DEF_WAIT_MS;
 1784     op->c2e = SCSI_TUR;
 1785     op->blqd = BLQ_DEFAULT;
 1786     op->block = !! DEF_BLOCKING;
 1787     op->myqd = MYQD_HIGH;
 1788     page_size = sysconf(_SC_PAGESIZE);
 1789 
 1790     while (1) {
 1791         int option_index = 0;
 1792 
 1793         c = getopt_long(argc, argv,
 1794                         "34acdefghl:L:mM:n:NO:pq:Q:Rs:St:TuvVw:Wz",
 1795                         long_options, &option_index);
 1796         if (c == -1)
 1797             break;
 1798 
 1799         switch (c) {
 1800         case '3':
 1801             op->v3 = true;
 1802             op->v3_given = true;
 1803             op->v4 = false;     /* if '-4 -3' take latter */
 1804             op->v4_given = false;
 1805             break;
 1806         case '4':
 1807             op->v4 = true;
 1808             op->v4_given = true;
 1809             op->v3 = false;
 1810             op->v3_given = false;
 1811             break;
 1812         case 'a':
 1813             op->masync = true;
 1814             break;
 1815         case 'c':
 1816             op->cmd_time = true;
 1817             break;
 1818         case 'd':
 1819             op->direct = true;
 1820             break;
 1821         case 'e':
 1822             op->excl = true;
 1823             break;
 1824         case 'f':
 1825             force = true;
 1826             break;
 1827         case 'g':
 1828             op->generic_sync = true;
 1829             break;
 1830         case 'h':
 1831         case '?':
 1832             usage();
 1833             return 0;
 1834         case 'l':
 1835             if (isdigit(*optarg)) {
 1836                 ll = sg_get_llnum(optarg);
 1837                 if (-1 == ll) {
 1838                     pr2serr_lk("could not decode lba\n");
 1839                     return 1;
 1840                 } else
 1841                     op->lba = (uint64_t)ll;
 1842                 cp = strchr(optarg, ',');
 1843                 if (cp) {
 1844                     if (0 == strcmp("-1", cp + 1))
 1845                         op->hi_lba = UINT_MAX;
 1846                     else {
 1847                         ll = sg_get_llnum(cp + 1);
 1848                         if ((-1 == ll) || (ll > UINT_MAX)) {
 1849                             pr2serr_lk("could not decode hi_lba, or > "
 1850                                        "UINT_MAX\n");
 1851                             return 1;
 1852                         } else
 1853                             op->hi_lba = (unsigned int)ll;
 1854                     }
 1855                 }
 1856             } else {
 1857                 pr2serr_lk("--lba= expects a number\n");
 1858                 return 1;
 1859             }
 1860             break;
 1861         case 'L':
 1862             op->lb_sz = sg_get_num(optarg);
 1863             if (op->lb_sz < 0) {
 1864                 pr2serr_lk("--lbsz= expects power of 2\n");
 1865                 return 1;
 1866             }
 1867             if (0 == op->lb_sz)
 1868                 op->lb_sz = DEF_LB_SZ;
 1869             break;
 1870         case 'm':
 1871             op->mmap_io = true;
 1872             break;
 1873         case 'M':
 1874             if (isdigit(*optarg)) {
 1875                 n = atoi(optarg);
 1876                 if ((n < 1) || (n > MAX_Q_PER_FD)) {
 1877                     pr2serr_lk("-M expects a value from 1 to %d\n",
 1878                                MAX_Q_PER_FD);
 1879                     return 1;
 1880                 }
 1881                 maxq_per_thread_given = true;
 1882                 op->maxq_per_thread = n;
 1883             } else {
 1884                 pr2serr_lk("--maxqpt= expects a number\n");
 1885                 return 1;
 1886             }
 1887             break;
 1888         case 'n':
 1889             if (isdigit(*optarg))
 1890                 op->num_per_thread = sg_get_num(optarg);
 1891             else {
 1892                 pr2serr_lk("--numpt= expects a number\n");
 1893                 return 1;
 1894             }
 1895             break;
 1896         case 'N':
 1897             op->no_xfer = true;
 1898             break;
 1899         case 'O':
 1900             if (isdigit(*optarg))
 1901                 op->ovn = sg_get_num(optarg);
 1902             else {
 1903                 pr2serr_lk("--override= expects a number\n");
 1904                 return 1;
 1905             }
 1906             if (op->ovn < 0) {
 1907                 pr2serr_lk("--override= bad number\n");
 1908                 return 1;
 1909             }
 1910             break;
 1911         case 'p':
 1912             op->pack_id_force = true;
 1913             break;
 1914         case 'q':
 1915             if (isdigit(*optarg)) {
 1916                 n = atoi(optarg);
 1917                 if (0 == n)
 1918                     op->blqd = BLQ_AT_HEAD;
 1919                 else if (1 == n)
 1920                     op->blqd = BLQ_AT_TAIL;
 1921             } else {
 1922                 pr2serr_lk("--qat= expects a number: 0 or 1\n");
 1923                 return 1;
 1924             }
 1925             break;
 1926         case 'Q':
 1927             if (isdigit(*optarg)) {
 1928                 n = atoi(optarg);
 1929                 if (0 == n)
 1930                     op->myqd = MYQD_LOW;
 1931                 else if (1 == n)
 1932                     op->myqd = MYQD_MEDIUM;
 1933                 else if (2 == n)
 1934                     op->myqd = MYQD_HIGH;
 1935             } else {
 1936                 pr2serr_lk("--qfav= expects a number: 0, 1 or 2\n");
 1937                 return 1;
 1938             }
 1939             break;
 1940         case 'R':
 1941             op->c2e = SCSI_READ16;
 1942             break;
 1943         case 's':
 1944             if (isdigit(*optarg)) {
 1945                 op->lb_sz = atoi(optarg);
 1946                 if (op->lb_sz < 256) {
 1947                     cerr << "Strange lb_sz, using 256" << endl;
 1948                     op->lb_sz = 256;
 1949                 }
 1950             } else {
 1951                 pr2serr_lk("--szlb= expects a number\n");
 1952                 return 1;
 1953             }
 1954             if ((cp = strchr(optarg, ','))) {
 1955                 n = sg_get_num(cp + 1);
 1956                 if (n < 1) {
 1957                     pr2serr_lk("could not decode 2nd part of "
 1958                                "--szlb=LBS,NLBS\n");
 1959                     return 1;
 1960                 }
 1961                 op->num_lbs = n;
 1962             }
 1963             break;
 1964         case 'S':
 1965             ++op->stats;
 1966             break;
 1967         case 't':
 1968             if (isdigit(*optarg))
 1969                 num_threads = atoi(optarg);
 1970             else {
 1971                 pr2serr_lk("--tnum= expects a number\n");
 1972                 return 1;
 1973             }
 1974             break;
 1975         case 'T':
 1976             op->c2e = SCSI_TUR;
 1977             break;
 1978         case 'u':
 1979             op->submit = true;
 1980             break;
 1981         case 'v':
 1982             op->verbose_given = true;
 1983             ++op->verbose;
 1984             break;
 1985         case 'V':
 1986             op->version_given = true;
 1987             break;
 1988         case 'w':
 1989             if ((isdigit(*optarg) || ('-' == *optarg))) {
 1990                 if ('-' == *optarg)
 1991                     op->wait_ms = - atoi(optarg + 1);
 1992                 else
 1993                     op->wait_ms = atoi(optarg);
 1994             } else {
 1995                 pr2serr_lk("--wait= expects a number\n");
 1996                 return 1;
 1997             }
 1998             break;
 1999         case 'W':
 2000             op->c2e = SCSI_WRITE16;
 2001             break;
 2002         case 'z':
 2003             op->no_waitq = true;
 2004             break;
 2005         default:
 2006             pr2serr_lk("unrecognised option code 0x%x ??\n", c);
 2007             usage();
 2008             return 1;
 2009         }
 2010     }
 2011     if (optind < argc) {
 2012         if (optind < argc) {
 2013             for (; optind < argc; ++optind)
 2014                 op->dev_names.push_back(argv[optind]);
 2015         }
 2016     }
 2017 #ifdef DEBUG
 2018     pr2serr_lk("In DEBUG mode, ");
 2019     if (op->verbose_given && op->version_given) {
 2020         pr2serr_lk("but override: '-vV' given, zero verbose and continue\n");
 2021         op->verbose_given = false;
 2022         op->version_given = false;
 2023         op->verbose = 0;
 2024     } else if (! op->verbose_given) {
 2025         pr2serr_lk("set '-vv'\n");
 2026         op->verbose = 2;
 2027     } else
 2028         pr2serr_lk("keep verbose=%d\n", op->verbose);
 2029 #else
 2030     if (op->verbose_given && op->version_given)
 2031         pr2serr_lk("Not in DEBUG mode, so '-vV' has no special action\n");
 2032 #endif
 2033     if (op->version_given) {
 2034         pr2serr_lk("version: %s\n", version_str);
 2035         return 0;
 2036     }
 2037     if (op->mmap_io) {
 2038         if (maxq_per_thread_given && (op->maxq_per_thread > 1)) {
 2039             pr2serr_lk("With mmap_io selected, QPT cannot exceed 1\n");
 2040             return 1;
 2041         } else if (op->direct) {
 2042             pr2serr_lk("direct IO and mmap-ed IO cannot both be selected\n");
 2043             return 1;
 2044         } else if (op->generic_sync) {
 2045             pr2serr_lk("--generic-sync and and mmap-ed IO are compatible\n");
 2046             return 1;
 2047         } else
 2048             op->maxq_per_thread = 1;
 2049     }
 2050     if (! op->cmd_time && getenv("SG3_UTILS_LINUX_NANO")) {
 2051         op->cmd_time = true;
 2052         if (op->verbose)
 2053             fprintf(stderr, "setting nanosecond timing due to environment "
 2054                     "variable: SG3_UTILS_LINUX_NANO\n");
 2055     }
 2056     if (0 == op->dev_names.size()) {
 2057         fprintf(stderr, "No sg_disk_device-s given\n\n");
 2058         usage();
 2059         return 1;
 2060     }
 2061     if (op->hi_lba && (op->lba > op->hi_lba)) {
 2062         cerr << "lba,hi_lba range is illegal" << endl;
 2063         return 1;
 2064     }
 2065     if (op->v4) {
 2066         if (! op->submit) {
 2067             op->submit = true;
 2068             if (op->verbose > 1)
 2069                 cerr << "when --v4 is given, --submit will be set" << endl;
 2070         }
 2071     }
 2072 
 2073     try {
 2074         int sg_ver_num;
 2075         unsigned int last_lba;
 2076         unsigned int blk_sz;
 2077         struct stat a_stat;
 2078 
 2079         for (k = 0; k < (int)op->dev_names.size(); ++k) {
 2080             dev_name = op->dev_names[k];
 2081             if (stat(dev_name, &a_stat) < 0) {
 2082                 snprintf(b, sizeof(b), "could not stat() %s", dev_name);
 2083                 perror(b);
 2084                 return 1;
 2085             }
 2086             if (! S_ISCHR(a_stat.st_mode)) {
 2087                 pr2serr_lk("%s should be a sg device which is a char "
 2088                            "device. %s\n", dev_name, dev_name);
 2089                 pr2serr_lk("is not a char device and damage could be done "
 2090                            "if it is a BLOCK\ndevice, exiting ...\n");
 2091                 return 1;
 2092             }
 2093             res = do_inquiry_prod_id(dev_name, op->block, sg_ver_num,
 2094                                      b, sizeof(b));
 2095             if (! force) {
 2096                 if (res) {
 2097                     pr2serr_lk("INQUIRY failed on %s\n", dev_name);
 2098                     return 1;
 2099                 }
 2100                 // For safety, since <lba> written to, only permit scsi_debug
 2101                 // devices. Bypass this with '-f' option.
 2102                 if (0 != memcmp("scsi_debug", b, 10)) {
 2103                     pr2serr_lk("Since this utility may write to LBAs, "
 2104                                "only devices with the\n"
 2105                                "product ID 'scsi_debug' accepted. Use '-f' "
 2106                                "to override.\n");
 2107                     return 2;
 2108                 }
 2109             }
 2110             if (sg_ver_num < 30000) {
 2111                 pr2serr_lk("%s either not sg device or too old\n", dev_name);
 2112                 return 2;
 2113             } else if (sg_ver_num >= 40030) {
 2114                 op->sg_vn_ge_40030 = true;
 2115                 op->sg_vn_ge_40000 = true;
 2116                 if (! (op->v3_given || op->v4_given)) {
 2117                     op->v4 = true;
 2118                     op->v3 = false;
 2119                     op->submit = true;
 2120                 }
 2121             } else if (sg_ver_num >= 40000) {
 2122                 op->sg_vn_ge_40030 = false;
 2123                 op->sg_vn_ge_40000 = true;
 2124                 if (! (op->v3_given || op->v4_given)) {
 2125                     op->v4 = true;
 2126                     op->v3 = false;
 2127                     op->submit = true;
 2128                 }
 2129             } else {
 2130                 if (! (op->v3_given || op->v4_given)) {
 2131                     op->v4 = false;
 2132                     op->v3 = true;
 2133                     op->submit = false;
 2134                 }
 2135             }
 2136 
 2137             if ((SCSI_WRITE16 == op->c2e) || (SCSI_READ16 == op->c2e)) {
 2138                 res = do_read_capacity(dev_name, op->block, &last_lba,
 2139                                        &blk_sz);
 2140                 if (2 == res)
 2141                     res = do_read_capacity(dev_name, op->block, &last_lba,
 2142                                            &blk_sz);
 2143                 if (res) {
 2144                     pr2serr_lk("READ CAPACITY(10) failed on %s\n", dev_name);
 2145                     return 1;
 2146                 }
 2147                 if (blk_sz != (unsigned int)op->lb_sz) {
 2148                     pr2serr_lk(">>> Logical block size (%d) of %s\n"
 2149                                "    differs from command line option (or "
 2150                                "default)\n", blk_sz, dev_name);
 2151                    pr2serr_lk("... continue anyway\n");
 2152                 }
 2153                 op->blk_szs.push_back(blk_sz);
 2154                 if (UINT_MAX == op->hi_lba)
 2155                     op->hi_lbas.push_back(last_lba);
 2156             }
 2157         }
 2158 
 2159         start_tm.tv_sec = 0;
 2160         start_tm.tv_nsec = 0;
 2161         if (clock_gettime(CLOCK_MONOTONIC, &start_tm) < 0)
 2162             perror("clock_gettime failed");
 2163 
 2164         vector<thread *> vt;
 2165 
 2166         /* start multi-threaded section */
 2167         for (k = 0; k < num_threads; ++k) {
 2168             thread * tp = new thread {work_thread, k, op};
 2169             vt.push_back(tp);
 2170         }
 2171 
 2172         // g++ 4.7.3 didn't like range-for loop here
 2173         for (k = 0; k < (int)vt.size(); ++k)
 2174             vt[k]->join();
 2175         /* end multi-threaded section, just this main thread left */
 2176 
 2177         for (k = 0; k < (int)vt.size(); ++k)
 2178             delete vt[k];
 2179 
 2180         n = uniq_pack_id.load() - 1;
 2181         if (((n > 0) || op->generic_sync) &&
 2182             (0 == clock_gettime(CLOCK_MONOTONIC, &end_tm))) {
 2183             struct timespec res_tm;
 2184             double a, b;
 2185 
 2186             if (op->generic_sync)
 2187                 n = op->num_per_thread * num_threads;
 2188             res_tm.tv_sec = end_tm.tv_sec - start_tm.tv_sec;
 2189             res_tm.tv_nsec = end_tm.tv_nsec - start_tm.tv_nsec;
 2190             if (res_tm.tv_nsec < 0) {
 2191                 --res_tm.tv_sec;
 2192                 res_tm.tv_nsec += 1000000000;
 2193             }
 2194             a = res_tm.tv_sec;
 2195             a += (0.000001 * (res_tm.tv_nsec / 1000));
 2196             b = (double)n;
 2197             if (a > 0.000001) {
 2198                 printf("Time to complete %d commands was %d.%06d seconds\n",
 2199                        n, (int)res_tm.tv_sec, (int)(res_tm.tv_nsec / 1000));
 2200                 printf("Implies %.0f IOPS\n", (b / a));
 2201             }
 2202         }
 2203 
 2204         if (op->verbose || op->stats) {
 2205             cout << "Number of sync_starts: " << sync_starts.load() << endl;
 2206             cout << "Number of async_starts: " << async_starts.load() << endl;
 2207             cout << "Number of async_finishes: " << async_finishes.load() <<
 2208                     endl;
 2209             cout << "Last pack_id: " << n << endl;
 2210         }
 2211         n = start_ebusy_count.load();
 2212         if (op->verbose || op->stats || (n > 0))
 2213             cout << "Number of start EBUSYs: " << n << endl;
 2214         n = fin_ebusy_count.load();
 2215         if (op->verbose || op->stats || (n > 0))
 2216             cout << "Number of finish EBUSYs: " << n << endl;
 2217         n = start_eagain_count.load();
 2218         if (op->verbose || op->stats || (n > 0))
 2219             cout << "Number of start EAGAINs: " << n << endl;
 2220         n = fin_eagain_count.load();
 2221         if (op->verbose || op->stats || (n > 0))
 2222             cout << "Number of finish EAGAINs: " << n << endl;
 2223         n = start_e2big_count.load();
 2224         if (op->verbose || op->stats || (n > 0))
 2225             cout << "Number of E2BIGs: " << n << endl;
 2226         n = start_edom_count.load();
 2227         if (op->verbose || op->stats || (n > 0))
 2228             cout << "Number of EDOMs: " << n << endl;
 2229         n = enomem_count.load();
 2230         if (op->verbose || op->stats || (n > 0))
 2231             cout << "Number of ENOMEMs: " << n << endl;
 2232     }
 2233     catch(system_error& e)  {
 2234         cerr << "got a system_error exception: " << e.what() << '\n';
 2235         auto ec = e.code();
 2236         cerr << "category: " << ec.category().name() << '\n';
 2237         cerr << "value: " << ec.value() << '\n';
 2238         cerr << "message: " << ec.message() << '\n';
 2239         cerr << "\nNote: if g++ may need '-pthread' or similar in "
 2240                 "compile/link line" << '\n';
 2241     }
 2242     catch(...) {
 2243         cerr << "got another exception: " << '\n';
 2244     }
 2245     return 0;
 2246 }