"Fossies" - the Fresh Open Source Software Archive

Member "parallel-20201122/src/env_parallel.ksh" (22 Nov 2020, 13726 Bytes) of package /linux/misc/parallel-20201122.tar.bz2:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Bash source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. See also the latest Fossies "Diffs" side-by-side code changes report for "env_parallel.ksh": 20201022_vs_20201122.

    1 #!/usr/bin/env ksh
    2 
    3 # This file must be sourced in ksh:
    4 #
    5 #   source `which env_parallel.ksh`
    6 #
    7 # after which 'env_parallel' works
    8 #
    9 #
   10 # Copyright (C) 2016-2020 Ole Tange, http://ole.tange.dk and Free
   11 # Software Foundation, Inc.
   12 #
   13 # This program is free software; you can redistribute it and/or modify
   14 # it under the terms of the GNU General Public License as published by
   15 # the Free Software Foundation; either version 3 of the License, or
   16 # (at your option) any later version.
   17 #
   18 # This program is distributed in the hope that it will be useful, but
   19 # WITHOUT ANY WARRANTY; without even the implied warranty of
   20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   21 # General Public License for more details.
   22 #
   23 # You should have received a copy of the GNU General Public License
   24 # along with this program; if not, see <http://www.gnu.org/licenses/>
   25 # or write to the Free Software Foundation, Inc., 51 Franklin St,
   26 # Fifth Floor, Boston, MA 02110-1301 USA
   27 
   28 env_parallel() {
   29     # env_parallel.ksh
   30 
   31     _names_of_ALIASES() {
   32     alias | perl -pe 's/=.*//'
   33     }
   34     _bodies_of_ALIASES() {
   35     alias "$@" | perl -pe 's/^/alias /;
   36                      sub warning { print STDERR "env_parallel: Warning: @_\n"; }
   37                      if(/^alias (\S+)=\$.*\\n/) {
   38                  warning("Alias \"$1\" contains newline.");
   39                  warning("Make sure the command has at least one newline after \"$1\".");
   40                  warning("See BUGS in \"man env_parallel\".");
   41                      }'
   42 
   43     }
   44     _names_of_maybe_FUNCTIONS() {
   45     true not used
   46     }
   47     _names_of_FUNCTIONS() {
   48     typeset +p -f | perl -pe 's/\(\).*//'
   49     }
   50     _bodies_of_FUNCTIONS() {
   51     functions "$@"
   52     }
   53     _names_of_VARIABLES() {
   54     typeset +p | perl -pe 's/^typeset .. //'
   55     }
   56     _bodies_of_VARIABLES() {
   57     typeset -p "$@"
   58     }
   59     _ignore_HARDCODED() {
   60     # These names cannot be detected
   61     echo '(_|TIMEOUT|IFS)'
   62     }
   63     _ignore_READONLY() {
   64     readonly | perl -e '@r = map {
   65                 chomp;
   66                 # sh on UnixWare: readonly TIMEOUT
   67             # ash: readonly var='val'
   68             # ksh: var='val'
   69                 s/^(readonly )?([^= ]*)(=.*|)$/$2/ or
   70             # bash: declare -ar BASH_VERSINFO=([0]="4" [1]="4")
   71                   s/^\S+\s+\S+\s+(\S[^=]*)(=.*|$)/$1/;
   72                 $_ } <>;
   73             $vars = join "|",map { quotemeta $_ } @r;
   74             print $vars ? "($vars)" : "(,,nO,,VaRs,,)";
   75             '
   76     }
   77     _remove_bad_NAMES() {
   78     # Do not transfer vars and funcs from env_parallel
   79     _ignore_RO="`_ignore_READONLY`"
   80     _ignore_HARD="`_ignore_HARDCODED`"
   81     # Macos-grep does not like long patterns
   82     # Old Solaris grep does not support -E
   83     # Perl Version of:
   84     # grep -Ev '^(...)$' |
   85     perl -ne '/^(
   86              PARALLEL_ENV|
   87              PARALLEL_TMP|
   88              _alias_NAMES|
   89              _bodies_of_ALIASES|
   90              _bodies_of_FUNCTIONS|
   91              _bodies_of_VARIABLES|
   92              _error_PAR|
   93              _function_NAMES|
   94              _get_ignored_VARS|
   95              _grep_REGEXP|
   96              _ignore_HARD|
   97              _ignore_HARDCODED|
   98              _ignore_READONLY|
   99              _ignore_RO|
  100              _ignore_UNDERSCORE|
  101              _list_alias_BODIES|
  102              _list_function_BODIES|
  103              _list_variable_VALUES|
  104              _make_grep_REGEXP|
  105              _names_of_ALIASES|
  106              _names_of_FUNCTIONS|
  107              _names_of_VARIABLES|
  108              _names_of_maybe_FUNCTIONS|
  109              _parallel_exit_CODE|
  110              _prefix_PARALLEL_ENV|
  111              _prefix_PARALLEL_ENV|
  112              _remove_bad_NAMES|
  113              _remove_readonly|
  114              _variable_NAMES|
  115              _warning_PAR|
  116              _which_PAR)$/x and next;
  117         # Filter names matching --env
  118         /^'"$_grep_REGEXP"'$/ or next;
  119             /^'"$_ignore_UNDERSCORE"'$/ and next;
  120         # Remove readonly variables
  121             /^'"$_ignore_RO"'$/ and next;
  122             /^'"$_ignore_HARD"'$/ and next;
  123             print;'
  124     }
  125     _get_ignored_VARS() {
  126         perl -e '
  127             for(@ARGV){
  128                 $next_is_env and push @envvar, split/,/, $_;
  129                 $next_is_env=/^--env$/;
  130             }
  131             if(grep { /^_$/ } @envvar) {
  132                 if(not open(IN, "<", "$ENV{HOME}/.parallel/ignored_vars")) {
  133                     print STDERR "parallel: Error: ",
  134                     "Run \"parallel --record-env\" in a clean environment first.\n";
  135                 } else {
  136                     chomp(@ignored_vars = <IN>);
  137                 }
  138             }
  139             if($ENV{PARALLEL_IGNORED_NAMES}) {
  140                 push @ignored_vars, split/\s+/, $ENV{PARALLEL_IGNORED_NAMES};
  141                 chomp @ignored_vars;
  142             }
  143             $vars = join "|",map { quotemeta $_ } @ignored_vars;
  144         print $vars ? "($vars)" : "(,,nO,,VaRs,,)";
  145             ' -- "$@"
  146     }
  147 
  148     # Get the --env variables if set
  149     # --env _ should be ignored
  150     # and convert  a b c  to (a|b|c)
  151     # If --env not set: Match everything (.*)
  152     _make_grep_REGEXP() {
  153         perl -e '
  154             for(@ARGV){
  155                 /^_$/ and $next_is_env = 0;
  156                 $next_is_env and push @envvar, split/,/, $_;
  157                 $next_is_env = /^--env$/;
  158             }
  159             $vars = join "|",map { quotemeta $_ } @envvar;
  160             print $vars ? "($vars)" : "(.*)";
  161             ' -- "$@"
  162     }
  163     _which_PAR() {
  164     # type returns:
  165     #   ll is an alias for ls -l (in ash)
  166     #   bash is a tracked alias for /bin/bash
  167     #   true is a shell builtin (in bash)
  168     #   myfunc is a function (in bash)
  169     #   myfunc is a shell function (in zsh)
  170     #   which is /usr/bin/which (in sh, bash)
  171     #   which is hashed (/usr/bin/which)
  172     #   gi is aliased to `grep -i' (in bash)
  173     #   aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde'
  174     # Return 0 if found, 1 otherwise
  175     LANG=C type "$@" |
  176         perl -pe '$exit += (s/ is an alias for .*// ||
  177                             s/ is aliased to .*// ||
  178                                 s/ is a function// ||
  179                                 s/ is a shell function// ||
  180                                 s/ is a shell builtin// ||
  181                                 s/.* is hashed .(\S+).$/$1/ ||
  182                                 s/.* is (a tracked alias for )?//);
  183                       END { exit not $exit }'
  184     }
  185     _warning_PAR() {
  186     echo "env_parallel: Warning: $@" >&2
  187     }
  188     _error_PAR() {
  189     echo "env_parallel: Error: $@" >&2
  190     }
  191 
  192     if _which_PAR parallel >/dev/null; then
  193     true parallel found in path
  194     else
  195     _error_PAR 'parallel must be in $PATH.'
  196     return 255
  197     fi
  198 
  199     # Grep regexp for vars given by --env
  200     _grep_REGEXP="`_make_grep_REGEXP \"$@\"`"
  201     unset _make_grep_REGEXP
  202 
  203     # Deal with --env _
  204     _ignore_UNDERSCORE="`_get_ignored_VARS \"$@\"`"
  205     unset _get_ignored_VARS
  206 
  207     # --record-env
  208     if perl -e 'exit grep { /^--record-env$/ } @ARGV' -- "$@"; then
  209     true skip
  210     else
  211     (_names_of_ALIASES;
  212      _names_of_FUNCTIONS;
  213      _names_of_VARIABLES) |
  214         cat > $HOME/.parallel/ignored_vars
  215     return 0
  216     fi
  217 
  218     # --session
  219     if perl -e 'exit grep { /^--session$/ } @ARGV' -- "$@"; then
  220     true skip
  221     else
  222     # Insert ::: between each level of session
  223     # so you can pop off the last ::: at --end-session
  224     PARALLEL_IGNORED_NAMES="`echo \"$PARALLEL_IGNORED_NAMES\";
  225           echo :::;
  226           (_names_of_ALIASES;
  227        _names_of_FUNCTIONS;
  228        _names_of_VARIABLES) | perl -ne '
  229         BEGIN{
  230           map { $ignored_vars{$_}++ }
  231                 split/\s+/, $ENV{PARALLEL_IGNORED_NAMES};
  232         }
  233         chomp;
  234         for(split/\s+/) {
  235           if(not $ignored_vars{$_}) {
  236             print $_,\"\\n\";
  237           }
  238             }
  239         '`"
  240     export PARALLEL_IGNORED_NAMES
  241     return 0
  242     fi
  243     if perl -e 'exit grep { /^--end.?session$/ } @ARGV' -- "$@"; then
  244     true skip
  245     else
  246     # Pop off last ::: from PARALLEL_IGNORED_NAMES
  247     PARALLEL_IGNORED_NAMES="`perl -e '
  248           $ENV{PARALLEL_IGNORED_NAMES} =~ s/(.*):::.*?$/$1/s;
  249       print $ENV{PARALLEL_IGNORED_NAMES}
  250         '`"
  251     return 0
  252     fi
  253     # Grep alias names
  254     _alias_NAMES="`_names_of_ALIASES | _remove_bad_NAMES | xargs echo`"
  255     _list_alias_BODIES="_bodies_of_ALIASES $_alias_NAMES"
  256     if [ "$_alias_NAMES" = "" ] ; then
  257     # no aliases selected
  258     _list_alias_BODIES="true"
  259     fi
  260     unset _alias_NAMES
  261 
  262     # Grep function names
  263     _function_NAMES="`_names_of_FUNCTIONS | _remove_bad_NAMES | xargs echo`"
  264     _list_function_BODIES="_bodies_of_FUNCTIONS $_function_NAMES"
  265     if [ "$_function_NAMES" = "" ] ; then
  266     # no functions selected
  267     _list_function_BODIES="true"
  268     fi
  269     unset _function_NAMES
  270 
  271     # Grep variable names
  272     _variable_NAMES="`_names_of_VARIABLES | _remove_bad_NAMES | xargs echo`"
  273     _list_variable_VALUES="_bodies_of_VARIABLES $_variable_NAMES"
  274     if [ "$_variable_NAMES" = "" ] ; then
  275     # no variables selected
  276     _list_variable_VALUES="true"
  277     fi
  278     unset _variable_NAMES
  279 
  280     PARALLEL_ENV="`
  281         $_list_alias_BODIES;
  282         $_list_function_BODIES;
  283         $_list_variable_VALUES;
  284     `"
  285     export PARALLEL_ENV
  286     unset _list_alias_BODIES _list_variable_VALUES _list_function_BODIES
  287     unset _bodies_of_ALIASES _bodies_of_VARIABLES _bodies_of_FUNCTIONS
  288     unset _names_of_ALIASES _names_of_VARIABLES _names_of_FUNCTIONS
  289     unset _ignore_HARDCODED _ignore_READONLY _ignore_UNDERSCORE
  290     unset _remove_bad_NAMES _grep_REGEXP
  291     unset _prefix_PARALLEL_ENV
  292     # Test if environment is too big
  293     if `_which_PAR true` >/dev/null 2>/dev/null ; then
  294     parallel "$@"
  295     _parallel_exit_CODE=$?
  296     # Clean up variables/functions
  297     unset PARALLEL_ENV
  298     unset _which_PAR _which_TRUE
  299     unset _warning_PAR _error_PAR
  300     # Unset _parallel_exit_CODE before return
  301     eval "unset _parallel_exit_CODE; return $_parallel_exit_CODE"
  302     else
  303     unset PARALLEL_ENV;
  304     _error_PAR "Your environment is too big."
  305     _error_PAR "You can try 3 different approaches:"
  306     _error_PAR "1. Run 'env_parallel --session' before you set"
  307     _error_PAR "   variables or define functions."
  308     _error_PAR "2. Use --env and only mention the names to copy."
  309     _error_PAR "3. Try running this in a clean environment once:"
  310     _error_PAR "     env_parallel --record-env"
  311     _error_PAR "   And then use '--env _'"
  312     _error_PAR "For details see: man env_parallel"
  313     return 255
  314     fi
  315 }
  316 
  317 parset() {
  318     _parset_PARALLEL_PRG=parallel
  319     _parset_main "$@"
  320 }
  321 
  322 env_parset() {
  323     _parset_PARALLEL_PRG=env_parallel
  324     _parset_main "$@"
  325 }
  326 
  327 _parset_main() {
  328     # If $1 contains ',' or space:
  329     #   Split on , to get the destination variable names
  330     # If $1 is a single destination variable name:
  331     #   Treat it as the name of an array
  332     #
  333     #   # Create array named myvar
  334     #   parset myvar echo ::: {1..10}
  335     #   echo ${myvar[5]}
  336     #
  337     #   # Put output into $var_a $var_b $var_c
  338     #   varnames=(var_a var_b var_c)
  339     #   parset "${varnames[*]}" echo ::: {1..3}
  340     #   echo $var_c
  341     #
  342     #   # Put output into $var_a4 $var_b4 $var_c4
  343     #   parset "var_a4 var_b4 var_c4" echo ::: {1..3}
  344     #   echo $var_c4
  345 
  346     _make_TEMP() {
  347     # mktemp does not exist on some OS
  348     perl -e 'use File::Temp qw(tempfile);
  349                  $ENV{"TMPDIR"} ||= "/tmp";
  350                  print((tempfile(DIR=>$ENV{"TMPDIR"}, TEMPLATE => "parXXXXX"))[1])'
  351     }
  352 
  353     _parset_NAME="$1"
  354     if [ "$_parset_NAME" = "" ] ; then
  355     echo parset: Error: No destination variable given. >&2
  356     echo parset: Error: Try: >&2
  357     echo parset: Error: ' ' parset myarray echo ::: foo bar >&2
  358     return 255
  359     fi
  360     if [ "$_parset_NAME" = "--help" ] ; then
  361     echo parset: Error: Usage: >&2
  362     echo parset: Error: ' ' parset varname GNU Parallel options and command >&2
  363     echo
  364     parallel --help
  365     return 255
  366     fi
  367     if [ "$_parset_NAME" = "--version" ] ; then
  368     echo "parset 20201122 (GNU parallel `parallel --minversion 1`)"
  369     echo "Copyright (C) 2007-2020 Ole Tange, http://ole.tange.dk and Free Software"
  370     echo "Foundation, Inc."
  371     echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
  372     echo "This is free software: you are free to change and redistribute it."
  373     echo "GNU parallel comes with no warranty."
  374     echo
  375     echo "Web site: https://www.gnu.org/software/parallel"
  376     echo
  377     echo "When using programs that use GNU Parallel to process data for publication"
  378     echo "please cite as described in 'parallel --citation'."
  379     echo
  380     return 255
  381     fi
  382     shift
  383     echo "$_parset_NAME" |
  384     perl -ne 'chomp;for (split /[, ]/) {
  385             # Allow: var_32 var[3]
  386         if(not /^[a-zA-Z_][a-zA-Z_0-9]*(\[\d+\])?$/) {
  387                 print STDERR "parset: Error: $_ is an invalid variable name.\n";
  388                 print STDERR "parset: Error: Variable names must be letter followed by letters or digits.\n";
  389         print STDERR "parset: Error: Usage:\n";
  390         print STDERR "parset: Error:   parset varname GNU Parallel options and command\n";
  391                 $exitval = 255;
  392             }
  393         }
  394         exit $exitval;
  395         ' || return 255
  396     _exit_FILE=`_make_TEMP`
  397     if perl -e 'exit not grep /,| /, @ARGV' "$_parset_NAME" ; then
  398     # $_parset_NAME contains , or space
  399     # Split on , or space to get the names
  400     eval "$(
  401         # Compute results into files
  402         ($_parset_PARALLEL_PRG --files -k "$@"; echo $? > "$_exit_FILE") |
  403         # var1=`cat tmpfile1; rm tmpfile1`
  404         # var2=`cat tmpfile2; rm tmpfile2`
  405         parallel -q echo {2}='`cat {1}; rm {1}`' :::: - :::+ $(
  406             echo "$_parset_NAME" | perl -pe 's/,/ /g'
  407              )
  408         );
  409     "
  410     else
  411     # $_parset_NAME does not contain , or space
  412     # => $_parset_NAME is the name of the array to put data into
  413     # Supported in: bash zsh ksh mksh
  414     # Arrays do not work in: sh ash dash
  415     eval "$_parset_NAME=( $(
  416         # Compute results into files. Save exit value
  417         ($_parset_PARALLEL_PRG --files -k "$@"; echo $? > "$_exit_FILE") |
  418                 perl -pe 'chop;$_="\"\`cat $_; rm $_\`\" "'
  419             ) )"
  420     fi
  421     unset _parset_NAME _parset_PARALLEL_PRG _parallel_exit_CODE
  422     # Unset _exit_FILE before return
  423     eval "unset _exit_FILE; return \`cat $_exit_FILE; rm $_exit_FILE\`"
  424 }