"Fossies" - the Fresh Open Source Software Archive

Member "julia-1.1.1/share/julia/base/process.jl" (16 May 2019, 32171 Bytes) of package /linux/misc/julia-1.1.1-linux-i686.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Julia source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file.

    1 # This file is a part of Julia. License is MIT: https://julialang.org/license
    2 
    3 abstract type AbstractCmd end
    4 
    5 # libuv process option flags
    6 const UV_PROCESS_WINDOWS_VERBATIM_ARGUMENTS = UInt8(1 << 2)
    7 const UV_PROCESS_DETACHED = UInt8(1 << 3)
    8 const UV_PROCESS_WINDOWS_HIDE = UInt8(1 << 4)
    9 
   10 struct Cmd <: AbstractCmd
   11     exec::Vector{String}
   12     ignorestatus::Bool
   13     flags::UInt32 # libuv process flags
   14     env::Union{Array{String},Nothing}
   15     dir::String
   16     Cmd(exec::Vector{String}) =
   17         new(exec, false, 0x00, nothing, "")
   18     Cmd(cmd::Cmd, ignorestatus, flags, env, dir) =
   19         new(cmd.exec, ignorestatus, flags, env,
   20             dir === cmd.dir ? dir : cstr(dir))
   21     function Cmd(cmd::Cmd; ignorestatus::Bool=cmd.ignorestatus, env=cmd.env, dir::AbstractString=cmd.dir,
   22                  detach::Bool = 0 != cmd.flags & UV_PROCESS_DETACHED,
   23                  windows_verbatim::Bool = 0 != cmd.flags & UV_PROCESS_WINDOWS_VERBATIM_ARGUMENTS,
   24                  windows_hide::Bool = 0 != cmd.flags & UV_PROCESS_WINDOWS_HIDE)
   25         flags = detach * UV_PROCESS_DETACHED |
   26                 windows_verbatim * UV_PROCESS_WINDOWS_VERBATIM_ARGUMENTS |
   27                 windows_hide * UV_PROCESS_WINDOWS_HIDE
   28         new(cmd.exec, ignorestatus, flags, byteenv(env),
   29             dir === cmd.dir ? dir : cstr(dir))
   30     end
   31 end
   32 
   33 """
   34     Cmd(cmd::Cmd; ignorestatus, detach, windows_verbatim, windows_hide, env, dir)
   35 
   36 Construct a new `Cmd` object, representing an external program and arguments, from `cmd`,
   37 while changing the settings of the optional keyword arguments:
   38 
   39 * `ignorestatus::Bool`: If `true` (defaults to `false`), then the `Cmd` will not throw an
   40   error if the return code is nonzero.
   41 * `detach::Bool`: If `true` (defaults to `false`), then the `Cmd` will be run in a new
   42   process group, allowing it to outlive the `julia` process and not have Ctrl-C passed to
   43   it.
   44 * `windows_verbatim::Bool`: If `true` (defaults to `false`), then on Windows the `Cmd` will
   45   send a command-line string to the process with no quoting or escaping of arguments, even
   46   arguments containing spaces. (On Windows, arguments are sent to a program as a single
   47   "command-line" string, and programs are responsible for parsing it into arguments. By
   48   default, empty arguments and arguments with spaces or tabs are quoted with double quotes
   49   `"` in the command line, and `\\` or `"` are preceded by backslashes.
   50   `windows_verbatim=true` is useful for launching programs that parse their command line in
   51   nonstandard ways.) Has no effect on non-Windows systems.
   52 * `windows_hide::Bool`: If `true` (defaults to `false`), then on Windows no new console
   53   window is displayed when the `Cmd` is executed. This has no effect if a console is
   54   already open or on non-Windows systems.
   55 * `env`: Set environment variables to use when running the `Cmd`. `env` is either a
   56   dictionary mapping strings to strings, an array of strings of the form `"var=val"`, an
   57   array or tuple of `"var"=>val` pairs, or `nothing`. In order to modify (rather than
   58   replace) the existing environment, create `env` by `copy(ENV)` and then set
   59   `env["var"]=val` as desired.
   60 * `dir::AbstractString`: Specify a working directory for the command (instead
   61   of the current directory).
   62 
   63 For any keywords that are not specified, the current settings from `cmd` are used. Normally,
   64 to create a `Cmd` object in the first place, one uses backticks, e.g.
   65 
   66     Cmd(`echo "Hello world"`, ignorestatus=true, detach=false)
   67 """
   68 Cmd
   69 
   70 hash(x::Cmd, h::UInt) = hash(x.exec, hash(x.env, hash(x.ignorestatus, hash(x.dir, hash(x.flags, h)))))
   71 ==(x::Cmd, y::Cmd) = x.exec == y.exec && x.env == y.env && x.ignorestatus == y.ignorestatus &&
   72                      x.dir == y.dir && isequal(x.flags, y.flags)
   73 
   74 struct OrCmds <: AbstractCmd
   75     a::AbstractCmd
   76     b::AbstractCmd
   77     OrCmds(a::AbstractCmd, b::AbstractCmd) = new(a, b)
   78 end
   79 
   80 struct ErrOrCmds <: AbstractCmd
   81     a::AbstractCmd
   82     b::AbstractCmd
   83     ErrOrCmds(a::AbstractCmd, b::AbstractCmd) = new(a, b)
   84 end
   85 
   86 struct AndCmds <: AbstractCmd
   87     a::AbstractCmd
   88     b::AbstractCmd
   89     AndCmds(a::AbstractCmd, b::AbstractCmd) = new(a, b)
   90 end
   91 
   92 hash(x::AndCmds, h::UInt) = hash(x.a, hash(x.b, h))
   93 ==(x::AndCmds, y::AndCmds) = x.a == y.a && x.b == y.b
   94 
   95 shell_escape(cmd::Cmd; special::AbstractString="") =
   96     shell_escape(cmd.exec..., special=special)
   97 shell_escape_posixly(cmd::Cmd) =
   98     shell_escape_posixly(cmd.exec...)
   99 
  100 function show(io::IO, cmd::Cmd)
  101     print_env = cmd.env !== nothing
  102     print_dir = !isempty(cmd.dir)
  103     (print_env || print_dir) && print(io, "setenv(")
  104     print(io, '`')
  105     join(io, map(cmd.exec) do arg
  106         replace(sprint(context=io) do io
  107             with_output_color(:underline, io) do io
  108                 print_shell_word(io, arg, shell_special)
  109             end
  110         end, '`' => "\\`")
  111     end, ' ')
  112     print(io, '`')
  113     print_env && (print(io, ","); show(io, cmd.env))
  114     print_dir && (print(io, "; dir="); show(io, cmd.dir))
  115     (print_dir || print_env) && print(io, ")")
  116 end
  117 
  118 function show(io::IO, cmds::Union{OrCmds,ErrOrCmds})
  119     print(io, "pipeline(")
  120     show(io, cmds.a)
  121     print(io, ", ")
  122     print(io, isa(cmds, ErrOrCmds) ? "stderr=" : "stdout=")
  123     show(io, cmds.b)
  124     print(io, ")")
  125 end
  126 
  127 function show(io::IO, cmds::AndCmds)
  128     show(io, cmds.a)
  129     print(io, " & ")
  130     show(io, cmds.b)
  131 end
  132 
  133 const STDIN_NO  = 0
  134 const STDOUT_NO = 1
  135 const STDERR_NO = 2
  136 
  137 struct FileRedirect
  138     filename::String
  139     append::Bool
  140     FileRedirect(filename::AbstractString, append::Bool) = FileRedirect(convert(String, filename), append)
  141     function FileRedirect(filename::String, append::Bool)
  142         if lowercase(filename) == (@static Sys.iswindows() ? "nul" : "/dev/null")
  143             @warn "For portability use devnull instead of a file redirect" maxlog=1
  144         end
  145         return new(filename, append)
  146     end
  147 end
  148 
  149 # setup_stdio ≈ cconvert
  150 # rawhandle ≈ unsafe_convert
  151 rawhandle(::DevNull) = C_NULL
  152 rawhandle(x::OS_HANDLE) = x
  153 if OS_HANDLE !== RawFD
  154     rawhandle(x::RawFD) = Libc._get_osfhandle(x)
  155 end
  156 
  157 const Redirectable = Union{IO, FileRedirect, RawFD, OS_HANDLE}
  158 const StdIOSet = NTuple{3, Redirectable}
  159 
  160 struct CmdRedirect <: AbstractCmd
  161     cmd::AbstractCmd
  162     handle::Redirectable
  163     stream_no::Int
  164     readable::Bool
  165 end
  166 CmdRedirect(cmd, handle, stream_no) = CmdRedirect(cmd, handle, stream_no, stream_no == STDIN_NO)
  167 
  168 function show(io::IO, cr::CmdRedirect)
  169     print(io, "pipeline(")
  170     show(io, cr.cmd)
  171     print(io, ", ")
  172     if cr.stream_no == STDOUT_NO
  173         print(io, "stdout")
  174     elseif cr.stream_no == STDERR_NO
  175         print(io, "stderr")
  176     elseif cr.stream_no == STDIN_NO
  177         print(io, "stdin")
  178     else
  179         print(io, cr.stream_no)
  180     end
  181     print(io, cr.readable ? "<" : ">")
  182     show(io, cr.handle)
  183     print(io, ")")
  184 end
  185 
  186 """
  187     ignorestatus(command)
  188 
  189 Mark a command object so that running it will not throw an error if the result code is non-zero.
  190 """
  191 ignorestatus(cmd::Cmd) = Cmd(cmd, ignorestatus=true)
  192 ignorestatus(cmd::Union{OrCmds,AndCmds}) =
  193     typeof(cmd)(ignorestatus(cmd.a), ignorestatus(cmd.b))
  194 
  195 """
  196     detach(command)
  197 
  198 Mark a command object so that it will be run in a new process group, allowing it to outlive the julia process, and not have Ctrl-C interrupts passed to it.
  199 """
  200 detach(cmd::Cmd) = Cmd(cmd; detach=true)
  201 
  202 # like String(s), but throw an error if s contains NUL, since
  203 # libuv requires NUL-terminated strings
  204 function cstr(s)
  205     if Base.containsnul(s)
  206         throw(ArgumentError("strings containing NUL cannot be passed to spawned processes"))
  207     end
  208     return String(s)
  209 end
  210 
  211 # convert various env representations into an array of "key=val" strings
  212 byteenv(env::AbstractArray{<:AbstractString}) =
  213     String[cstr(x) for x in env]
  214 byteenv(env::AbstractDict) =
  215     String[cstr(string(k)*"="*string(v)) for (k,v) in env]
  216 byteenv(env::Nothing) = nothing
  217 byteenv(env::Union{AbstractVector{Pair{T}}, Tuple{Vararg{Pair{T}}}}) where {T<:AbstractString} =
  218     String[cstr(k*"="*string(v)) for (k,v) in env]
  219 
  220 """
  221     setenv(command::Cmd, env; dir="")
  222 
  223 Set environment variables to use when running the given `command`. `env` is either a
  224 dictionary mapping strings to strings, an array of strings of the form `"var=val"`, or zero
  225 or more `"var"=>val` pair arguments. In order to modify (rather than replace) the existing
  226 environment, create `env` by `copy(ENV)` and then setting `env["var"]=val` as desired, or
  227 use `withenv`.
  228 
  229 The `dir` keyword argument can be used to specify a working directory for the command.
  230 """
  231 setenv(cmd::Cmd, env; dir="") = Cmd(cmd; env=byteenv(env), dir=dir)
  232 setenv(cmd::Cmd, env::Pair{<:AbstractString}...; dir="") =
  233     setenv(cmd, env; dir=dir)
  234 setenv(cmd::Cmd; dir="") = Cmd(cmd; dir=dir)
  235 
  236 (&)(left::AbstractCmd, right::AbstractCmd) = AndCmds(left, right)
  237 redir_out(src::AbstractCmd, dest::AbstractCmd) = OrCmds(src, dest)
  238 redir_err(src::AbstractCmd, dest::AbstractCmd) = ErrOrCmds(src, dest)
  239 
  240 # Stream Redirects
  241 redir_out(dest::Redirectable, src::AbstractCmd) = CmdRedirect(src, dest, STDIN_NO)
  242 redir_out(src::AbstractCmd, dest::Redirectable) = CmdRedirect(src, dest, STDOUT_NO)
  243 redir_err(src::AbstractCmd, dest::Redirectable) = CmdRedirect(src, dest, STDERR_NO)
  244 
  245 # File redirects
  246 redir_out(src::AbstractCmd, dest::AbstractString) = CmdRedirect(src, FileRedirect(dest, false), STDOUT_NO)
  247 redir_out(src::AbstractString, dest::AbstractCmd) = CmdRedirect(dest, FileRedirect(src, false), STDIN_NO)
  248 redir_err(src::AbstractCmd, dest::AbstractString) = CmdRedirect(src, FileRedirect(dest, false), STDERR_NO)
  249 redir_out_append(src::AbstractCmd, dest::AbstractString) = CmdRedirect(src, FileRedirect(dest, true), STDOUT_NO)
  250 redir_err_append(src::AbstractCmd, dest::AbstractString) = CmdRedirect(src, FileRedirect(dest, true), STDERR_NO)
  251 
  252 """
  253     pipeline(command; stdin, stdout, stderr, append=false)
  254 
  255 Redirect I/O to or from the given `command`. Keyword arguments specify which of the
  256 command's streams should be redirected. `append` controls whether file output appends to the
  257 file. This is a more general version of the 2-argument `pipeline` function.
  258 `pipeline(from, to)` is equivalent to `pipeline(from, stdout=to)` when `from` is a command,
  259 and to `pipeline(to, stdin=from)` when `from` is another kind of data source.
  260 
  261 **Examples**:
  262 
  263 ```julia
  264 run(pipeline(`dothings`, stdout="out.txt", stderr="errs.txt"))
  265 run(pipeline(`update`, stdout="log.txt", append=true))
  266 ```
  267 """
  268 function pipeline(cmd::AbstractCmd; stdin=nothing, stdout=nothing, stderr=nothing, append::Bool=false)
  269     if append && stdout === nothing && stderr === nothing
  270         error("append set to true, but no output redirections specified")
  271     end
  272     if stdin !== nothing
  273         cmd = redir_out(stdin, cmd)
  274     end
  275     if stdout !== nothing
  276         cmd = append ? redir_out_append(cmd, stdout) : redir_out(cmd, stdout)
  277     end
  278     if stderr !== nothing
  279         cmd = append ? redir_err_append(cmd, stderr) : redir_err(cmd, stderr)
  280     end
  281     return cmd
  282 end
  283 
  284 pipeline(cmd::AbstractCmd, dest) = pipeline(cmd, stdout=dest)
  285 pipeline(src::Union{Redirectable,AbstractString}, cmd::AbstractCmd) = pipeline(cmd, stdin=src)
  286 
  287 """
  288     pipeline(from, to, ...)
  289 
  290 Create a pipeline from a data source to a destination. The source and destination can be
  291 commands, I/O streams, strings, or results of other `pipeline` calls. At least one argument
  292 must be a command. Strings refer to filenames. When called with more than two arguments,
  293 they are chained together from left to right. For example, `pipeline(a,b,c)` is equivalent to
  294 `pipeline(pipeline(a,b),c)`. This provides a more concise way to specify multi-stage
  295 pipelines.
  296 
  297 **Examples**:
  298 
  299 ```julia
  300 run(pipeline(`ls`, `grep xyz`))
  301 run(pipeline(`ls`, "out.txt"))
  302 run(pipeline("out.txt", `grep xyz`))
  303 ```
  304 """
  305 pipeline(a, b, c, d...) = pipeline(pipeline(a, b), c, d...)
  306 
  307 mutable struct Process <: AbstractPipe
  308     cmd::Cmd
  309     handle::Ptr{Cvoid}
  310     in::IO
  311     out::IO
  312     err::IO
  313     exitcode::Int64
  314     termsignal::Int32
  315     exitnotify::Condition
  316     closenotify::Condition
  317     function Process(cmd::Cmd, handle::Ptr{Cvoid})
  318         this = new(cmd, handle, devnull, devnull, devnull,
  319                    typemin(fieldtype(Process, :exitcode)),
  320                    typemin(fieldtype(Process, :termsignal)),
  321                    Condition(), Condition())
  322         finalizer(uvfinalize, this)
  323         return this
  324     end
  325 end
  326 pipe_reader(p::Process) = p.out
  327 pipe_writer(p::Process) = p.in
  328 
  329 # Represents a whole pipeline of any number of related processes
  330 # so the entire pipeline can be treated as one entity
  331 mutable struct ProcessChain <: AbstractPipe
  332     processes::Vector{Process}
  333     in::IO
  334     out::IO
  335     err::IO
  336     function ProcessChain()
  337         return new(Process[], devnull, devnull, devnull)
  338     end
  339 end
  340 pipe_reader(p::ProcessChain) = p.out
  341 pipe_writer(p::ProcessChain) = p.in
  342 
  343 # release ownership of the libuv handle
  344 function uvfinalize(proc::Process)
  345     if proc.handle != C_NULL
  346         disassociate_julia_struct(proc.handle)
  347         ccall(:jl_close_uv, Cvoid, (Ptr{Cvoid},), proc.handle)
  348         proc.handle = C_NULL
  349     end
  350     nothing
  351 end
  352 
  353 # called when the process dies
  354 function uv_return_spawn(p::Ptr{Cvoid}, exit_status::Int64, termsignal::Int32)
  355     data = ccall(:jl_uv_process_data, Ptr{Cvoid}, (Ptr{Cvoid},), p)
  356     data == C_NULL && return
  357     proc = unsafe_pointer_to_objref(data)::Process
  358     proc.exitcode = exit_status
  359     proc.termsignal = termsignal
  360     ccall(:jl_close_uv, Cvoid, (Ptr{Cvoid},), proc.handle)
  361     proc.handle = C_NULL
  362     notify(proc.exitnotify)
  363     nothing
  364 end
  365 
  366 # called when the libuv handle is destroyed
  367 function _uv_hook_close(proc::Process)
  368     proc.handle = C_NULL
  369     notify(proc.closenotify)
  370     nothing
  371 end
  372 
  373 const SpawnIOs = Vector{Any} # convenience name for readability
  374 
  375 # handle marshalling of `Cmd` arguments from Julia to C
  376 @noinline function _spawn_primitive(file, cmd::Cmd, stdio::SpawnIOs)
  377     loop = eventloop()
  378     iohandles = Tuple{Cint, UInt}[ # assuming little-endian layout
  379         let h = rawhandle(io)
  380             h === C_NULL     ? (0x00, UInt(0)) :
  381             h isa OS_HANDLE  ? (0x02, UInt(cconvert(@static(Sys.iswindows() ? Ptr{Cvoid} : Cint), h))) :
  382             h isa Ptr{Cvoid} ? (0x04, UInt(h)) :
  383             error("invalid spawn handle $h from $io")
  384         end
  385         for io in stdio]
  386     handle = Libc.malloc(_sizeof_uv_process)
  387     disassociate_julia_struct(handle) # ensure that data field is set to C_NULL
  388     error = ccall(:jl_spawn, Int32,
  389               (Cstring, Ptr{Cstring}, Ptr{Cvoid}, Ptr{Cvoid},
  390                Ptr{Tuple{Cint, UInt}}, Int,
  391                UInt32, Ptr{Cstring}, Cstring, Ptr{Cvoid}),
  392         file, cmd.exec, loop, handle,
  393         iohandles, length(iohandles),
  394         cmd.flags,
  395         cmd.env === nothing ? C_NULL : cmd.env,
  396         isempty(cmd.dir) ? C_NULL : cmd.dir,
  397         uv_jl_return_spawn::Ptr{Cvoid})
  398     if error != 0
  399         ccall(:jl_forceclose_uv, Cvoid, (Ptr{Cvoid},), handle) # will call free on handle eventually
  400         throw(_UVError("could not spawn " * repr(cmd), error))
  401     end
  402     pp = Process(cmd, handle)
  403     associate_julia_struct(handle, pp)
  404     return pp
  405 end
  406 
  407 _spawn(cmds::AbstractCmd) = _spawn(cmds, Any[])
  408 
  409 # optimization: we can spawn `Cmd` directly without allocating the ProcessChain
  410 function _spawn(cmd::Cmd, stdios::SpawnIOs)
  411     isempty(cmd.exec) && throw(ArgumentError("cannot spawn empty command"))
  412     pp = setup_stdios(stdios) do stdios
  413         return _spawn_primitive(cmd.exec[1], cmd, stdios)
  414     end
  415     return pp
  416 end
  417 
  418 # assume that having a ProcessChain means that the stdio are setup
  419 function _spawn(cmds::AbstractCmd, stdios::SpawnIOs)
  420     pp = setup_stdios(stdios) do stdios
  421         return _spawn(cmds, stdios, ProcessChain())
  422     end
  423     return pp
  424 end
  425 
  426 # helper function for making a copy of a SpawnIOs, with replacement
  427 function _stdio_copy(stdios::SpawnIOs, fd::Int, @nospecialize replace)
  428     nio = max(fd, length(stdios))
  429     new = SpawnIOs(undef, nio)
  430     copyto!(fill!(new, devnull), stdios)
  431     new[fd] = replace
  432     return new
  433 end
  434 
  435 function _spawn(redirect::CmdRedirect, stdios::SpawnIOs, args...)
  436     fdnum = redirect.stream_no + 1
  437     io, close_io = setup_stdio(redirect.handle, redirect.readable)
  438     try
  439         stdios = _stdio_copy(stdios, fdnum, io)
  440         return _spawn(redirect.cmd, stdios, args...)
  441     finally
  442         close_io && close_stdio(io)
  443     end
  444 end
  445 
  446 function _spawn(cmds::OrCmds, stdios::SpawnIOs, chain::ProcessChain)
  447     in_pipe, out_pipe = link_pipe(false, false)
  448     try
  449         stdios_left = _stdio_copy(stdios, 2, out_pipe)
  450         _spawn(cmds.a, stdios_left, chain)
  451         stdios_right = _stdio_copy(stdios, 1, in_pipe)
  452         _spawn(cmds.b, stdios_right, chain)
  453     finally
  454         close_pipe_sync(out_pipe)
  455         close_pipe_sync(in_pipe)
  456     end
  457     return chain
  458 end
  459 
  460 function _spawn(cmds::ErrOrCmds, stdios::SpawnIOs, chain::ProcessChain)
  461     in_pipe, out_pipe = link_pipe(false, false)
  462     try
  463         stdios_left = _stdio_copy(stdios, 3, out_pipe)
  464         _spawn(cmds.a, stdios_left, chain)
  465         stdios_right = _stdio_copy(stdios, 1, in_pipe)
  466         _spawn(cmds.b, stdios_right, chain)
  467     finally
  468         close_pipe_sync(out_pipe)
  469         close_pipe_sync(in_pipe)
  470     end
  471     return chain
  472 end
  473 
  474 function _spawn(cmds::AndCmds, stdios::SpawnIOs, chain::ProcessChain)
  475     _spawn(cmds.a, stdios, chain)
  476     _spawn(cmds.b, stdios, chain)
  477     return chain
  478 end
  479 
  480 function _spawn(cmd::Cmd, stdios::SpawnIOs, chain::ProcessChain)
  481     isempty(cmd.exec) && throw(ArgumentError("cannot spawn empty command"))
  482     pp = _spawn_primitive(cmd.exec[1], cmd, stdios)
  483     push!(chain.processes, pp)
  484     return chain
  485 end
  486 
  487 
  488 # open the child end of each element of `stdios`, and initialize the parent end
  489 function setup_stdios(f, stdios::SpawnIOs)
  490     nstdio = length(stdios)
  491     open_io = Vector{Any}(undef, nstdio)
  492     close_io = falses(nstdio)
  493     try
  494         for i in 1:nstdio
  495             open_io[i], close_io[i] = setup_stdio(stdios[i], i == 1)
  496         end
  497         pp = f(open_io)
  498         return pp
  499     finally
  500         for i in 1:nstdio
  501             close_io[i] && close_stdio(open_io[i])
  502         end
  503     end
  504 end
  505 
  506 function setup_stdio(stdio::PipeEndpoint, child_readable::Bool)
  507     if stdio.status == StatusInit
  508         # if the PipeEndpoint isn't open, set it to the parent end
  509         # and pass the other end to the child
  510         rd, wr = link_pipe(!child_readable, child_readable)
  511         try
  512             open_pipe!(stdio, child_readable ? wr : rd)
  513         catch ex
  514             close_pipe_sync(rd)
  515             close_pipe_sync(wr)
  516             rethrow(ex)
  517         end
  518         child = child_readable ? rd : wr
  519         return (child, true)
  520     end
  521     # if it's already open, assume that it's already the child end
  522     # (since we can't do anything else)
  523     return (stdio, false)
  524 end
  525 
  526 function setup_stdio(stdio::Pipe, child_readable::Bool)
  527     if stdio.in.status == StatusInit && stdio.out.status == StatusInit
  528         link_pipe!(stdio)
  529     end
  530     io = child_readable ? stdio.out : stdio.in
  531     return (io, false)
  532 end
  533 
  534 setup_stdio(stdio::AbstractPipe, readable::Bool) =
  535     setup_stdio(readable ? pipe_reader(stdio) : pipe_writer(stdio), readable)
  536 
  537 function setup_stdio(stdio::IOStream, child_readable::Bool)
  538     io = RawFD(fd(stdio))
  539     return (io, false)
  540 end
  541 
  542 function setup_stdio(stdio::FileRedirect, child_readable::Bool)
  543     if child_readable
  544         attr = JL_O_RDONLY
  545         perm = zero(S_IRUSR)
  546     else
  547         attr = JL_O_WRONLY | JL_O_CREAT
  548         attr |= stdio.append ? JL_O_APPEND : JL_O_TRUNC
  549         perm = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH
  550     end
  551     io = Filesystem.open(stdio.filename, attr, perm)
  552     return (io, true)
  553 end
  554 
  555 # incrementally move data between an IOBuffer and a system Pipe
  556 # TODO: probably more efficient (when valid) to use `stdio` directly as the
  557 #       PipeEndpoint buffer field in some cases
  558 function setup_stdio(stdio::Union{IOBuffer, BufferStream}, child_readable::Bool)
  559     parent = PipeEndpoint()
  560     rd, wr = link_pipe(!child_readable, child_readable)
  561     try
  562         open_pipe!(parent, child_readable ? wr : rd)
  563     catch ex
  564         close_pipe_sync(rd)
  565         close_pipe_sync(wr)
  566         rethrow(ex)
  567     end
  568     child = child_readable ? rd : wr
  569     try
  570         let in = (child_readable ? parent : stdio),
  571             out = (child_readable ? stdio : parent)
  572             @async try
  573                 write(in, out)
  574             catch ex
  575                 @warn "Process error" exception=(ex, catch_backtrace())
  576             finally
  577                 close(parent)
  578             end
  579         end
  580     catch ex
  581         close_pipe_sync(child)
  582         rethrow(ex)
  583     end
  584     return (child, true)
  585 end
  586 
  587 function setup_stdio(io, child_readable::Bool)
  588     # if there is no specialization,
  589     # assume that rawhandle is defined for it
  590     return (io, false)
  591 end
  592 
  593 close_stdio(stdio::OS_HANDLE) = close_pipe_sync(stdio)
  594 close_stdio(stdio) = close(stdio)
  595 
  596 # INTERNAL
  597 # pad out stdio to have at least three elements,
  598 # passing either `devnull` or the corresponding `stdio`
  599 # A Redirectable can be any of:
  600 #   - A system IO handle, to be passed to the child
  601 #   - An uninitialized pipe, to be created
  602 #   - devnull (to pass /dev/null for 0-2, or to leave undefined for fd > 2)
  603 #   - An Filesystem.File or IOStream object to redirect the output to
  604 #   - A FileRedirect, containing a string specifying a filename to be opened for the child
  605 
  606 spawn_opts_swallow(stdios::StdIOSet) = Any[stdios...]
  607 spawn_opts_inherit(stdios::StdIOSet) = Any[stdios...]
  608 spawn_opts_swallow(in::Redirectable=devnull, out::Redirectable=devnull, err::Redirectable=devnull) =
  609     Any[in, out, err]
  610 # pass original descriptors to child processes by default, because we might
  611 # have already exhausted and closed the libuv object for our standard streams.
  612 # ref issue #8529
  613 spawn_opts_inherit(in::Redirectable=RawFD(0), out::Redirectable=RawFD(1), err::Redirectable=RawFD(2)) =
  614     Any[in, out, err]
  615 
  616 function eachline(cmd::AbstractCmd; keep::Bool=false)
  617     out = PipeEndpoint()
  618     processes = _spawn(cmd, Any[devnull, out, stderr])
  619     # if the user consumes all the data, also check process exit status for success
  620     ondone = () -> (success(processes) || pipeline_error(processes); nothing)
  621     return EachLine(out, keep=keep, ondone=ondone)::EachLine
  622 end
  623 
  624 function open(cmds::AbstractCmd, mode::AbstractString, other::Redirectable=devnull)
  625     if mode == "r+" || mode == "w+"
  626         return open(cmds, other, read = true, write = true)
  627     elseif mode == "r"
  628         return open(cmds, other)
  629     elseif mode == "w"
  630         return open(cmds, other, write = true)
  631     else
  632         throw(ArgumentError("mode must be \"r\" or \"w\", not \"$mode\""))
  633     end
  634 end
  635 
  636 # return a Process object to read-to/write-from the pipeline
  637 """
  638     open(command, other=devnull; write::Bool = false, read::Bool = !write)
  639 
  640 Start running `command` asynchronously, and return a `process::IO` object.  If `read` is
  641 true, then reads from the process come from the process's standard output and `other` optionally
  642 specifies the process's standard input stream.  If `write` is true, then writes go to
  643 the process's standard input and `other` optionally specifies the process's standard output
  644 stream.
  645 The process's standard error stream is connected to the current global `stderr`.
  646 """
  647 function open(cmds::AbstractCmd, other::Redirectable=devnull; write::Bool=false, read::Bool=!write)
  648     if read && write
  649         other === devnull || throw(ArgumentError("no stream can be specified for `other` in read-write mode"))
  650         in = PipeEndpoint()
  651         out = PipeEndpoint()
  652         processes = _spawn(cmds, Any[in, out, stderr])
  653         processes.in = in
  654         processes.out = out
  655     elseif read
  656         out = PipeEndpoint()
  657         processes = _spawn(cmds, Any[other, out, stderr])
  658         processes.out = out
  659     elseif write
  660         in = PipeEndpoint()
  661         processes = _spawn(cmds, Any[in, other, stderr])
  662         processes.in = in
  663     else
  664         other === devnull || throw(ArgumentError("no stream can be specified for `other` in no-access mode"))
  665         processes = _spawn(cmds, Any[devnull, devnull, stderr])
  666     end
  667     return processes
  668 end
  669 
  670 """
  671     open(f::Function, command, mode::AbstractString="r", stdio=devnull)
  672 
  673 Similar to `open(command, mode, stdio)`, but calls `f(stream)` on the resulting process
  674 stream, then closes the input stream and waits for the process to complete.
  675 Returns the value returned by `f`.
  676 """
  677 function open(f::Function, cmds::AbstractCmd, args...)
  678     P = open(cmds, args...)
  679     ret = try
  680         f(P)
  681     catch
  682         kill(P)
  683         rethrow()
  684     finally
  685         close(P.in)
  686     end
  687     success(P) || pipeline_error(P)
  688     return ret
  689 end
  690 
  691 """
  692     read(command::Cmd)
  693 
  694 Run `command` and return the resulting output as an array of bytes.
  695 """
  696 function read(cmd::AbstractCmd)
  697     procs = open(cmd, "r", devnull)
  698     bytes = read(procs.out)
  699     success(procs) || pipeline_error(procs)
  700     return bytes
  701 end
  702 
  703 """
  704     read(command::Cmd, String)
  705 
  706 Run `command` and return the resulting output as a `String`.
  707 """
  708 read(cmd::AbstractCmd, ::Type{String}) = String(read(cmd))
  709 
  710 """
  711     run(command, args...; wait::Bool = true)
  712 
  713 Run a command object, constructed with backticks. Throws an error if anything goes wrong,
  714 including the process exiting with a non-zero status (when `wait` is true).
  715 
  716 If `wait` is false, the process runs asynchronously. You can later wait for it and check
  717 its exit status by calling `success` on the returned process object.
  718 
  719 When `wait` is false, the process' I/O streams are directed to `devnull`.
  720 When `wait` is true, I/O streams are shared with the parent process.
  721 Use [`pipeline`](@ref) to control I/O redirection.
  722 """
  723 function run(cmds::AbstractCmd, args...; wait::Bool = true)
  724     if wait
  725         ps = _spawn(cmds, spawn_opts_inherit(args...))
  726         success(ps) || pipeline_error(ps)
  727     else
  728         stdios = spawn_opts_swallow(args...)
  729         ps = _spawn(cmds, stdios)
  730         # for each stdio input argument, guess whether the user
  731         # passed a `stdio` placeholder object as input, and thus
  732         # might be able to use the return AbstractProcess as an IO object
  733         # (this really only applies to PipeEndpoint, Pipe, TCPSocket, or an AbstractPipe wrapping one of those)
  734         if length(stdios) > 0
  735             in = stdios[1]
  736             isa(in, IO) && (ps.in = in)
  737             if length(stdios) > 1
  738                 out = stdios[2]
  739                 isa(out, IO) && (ps.out = out)
  740                 if length(stdios) > 2
  741                     err = stdios[3]
  742                     isa(err, IO) && (ps.err = err)
  743                 end
  744             end
  745         end
  746     end
  747     return ps
  748 end
  749 
  750 # some common signal numbers that are usually available on all platforms
  751 # and might be useful as arguments to `kill` or testing against `Process.termsignal`
  752 const SIGHUP  = 1
  753 const SIGINT  = 2
  754 const SIGQUIT = 3 # !windows
  755 const SIGKILL = 9
  756 const SIGPIPE = 13 # !windows
  757 const SIGTERM = 15
  758 
  759 function test_success(proc::Process)
  760     @assert process_exited(proc)
  761     if proc.exitcode < 0
  762         #TODO: this codepath is not currently tested
  763         throw(_UVError("could not start process " * repr(proc.cmd), proc.exitcode))
  764     end
  765     return proc.exitcode == 0 && (proc.termsignal == 0 || proc.termsignal == SIGPIPE)
  766 end
  767 
  768 function success(x::Process)
  769     wait(x)
  770     return test_success(x)
  771 end
  772 success(procs::Vector{Process}) = mapreduce(success, &, procs)
  773 success(procs::ProcessChain) = success(procs.processes)
  774 
  775 """
  776     success(command)
  777 
  778 Run a command object, constructed with backticks, and tell whether it was successful (exited
  779 with a code of 0). An exception is raised if the process cannot be started.
  780 """
  781 success(cmd::AbstractCmd) = success(_spawn(cmd))
  782 
  783 function pipeline_error(proc::Process)
  784     if !proc.cmd.ignorestatus
  785         error("failed process: ", proc, " [", proc.exitcode, "]")
  786     end
  787     nothing
  788 end
  789 
  790 function pipeline_error(procs::ProcessChain)
  791     failed = Process[]
  792     for p = procs.processes
  793         if !test_success(p) && !p.cmd.ignorestatus
  794             push!(failed, p)
  795         end
  796     end
  797     isempty(failed) && return nothing
  798     length(failed) == 1 && pipeline_error(failed[1])
  799     msg = "failed processes:"
  800     for proc in failed
  801         msg = string(msg, "\n  ", proc, " [", proc.exitcode, "]")
  802     end
  803     error(msg)
  804 end
  805 
  806 """
  807     kill(p::Process, signum=SIGTERM)
  808 
  809 Send a signal to a process. The default is to terminate the process.
  810 Returns successfully if the process has already exited, but throws an
  811 error if killing the process failed for other reasons (e.g. insufficient
  812 permissions).
  813 """
  814 function kill(p::Process, signum::Integer)
  815     if process_running(p)
  816         @assert p.handle != C_NULL
  817         err = ccall(:uv_process_kill, Int32, (Ptr{Cvoid}, Int32), p.handle, signum)
  818         if err != 0 && err != UV_ESRCH
  819             throw(_UVError("kill", err))
  820         end
  821     end
  822 end
  823 kill(ps::Vector{Process}) = foreach(kill, ps)
  824 kill(ps::ProcessChain) = foreach(kill, ps.processes)
  825 kill(p::Process) = kill(p, SIGTERM)
  826 
  827 """
  828     getpid(process) -> Int32
  829 
  830 Get the child process ID, if it still exists.
  831 
  832 !!! compat "Julia 1.1"
  833     This function requires at least Julia 1.1.
  834 """
  835 function Libc.getpid(p::Process)
  836     ppid = Int32(0)
  837     if p.handle != C_NULL
  838         ppid = ccall(:jl_uv_process_pid, Int32, (Ptr{Cvoid},), p.handle)
  839     end
  840     ppid <= 0 && throw(_UVError("getpid", UV_ESRCH))
  841     return ppid
  842 end
  843 
  844 function _contains_newline(bufptr::Ptr{Cvoid}, len::Int32)
  845     return (ccall(:memchr, Ptr{Cvoid}, (Ptr{Cvoid},Int32,Csize_t), bufptr, '\n', len) != C_NULL)
  846 end
  847 
  848 ## process status ##
  849 
  850 """
  851     process_running(p::Process)
  852 
  853 Determine whether a process is currently running.
  854 """
  855 process_running(s::Process) = s.handle != C_NULL
  856 process_running(s::Vector{Process}) = any(process_running, s)
  857 process_running(s::ProcessChain) = process_running(s.processes)
  858 
  859 """
  860     process_exited(p::Process)
  861 
  862 Determine whether a process has exited.
  863 """
  864 process_exited(s::Process) = !process_running(s)
  865 process_exited(s::Vector{Process}) = all(process_exited, s)
  866 process_exited(s::ProcessChain) = process_exited(s.processes)
  867 
  868 process_signaled(s::Process) = (s.termsignal > 0)
  869 
  870 function process_status(s::Process)
  871     return process_running(s) ? "ProcessRunning" :
  872            process_signaled(s) ? "ProcessSignaled(" * string(s.termsignal) * ")" :
  873            process_exited(s) ? "ProcessExited(" * string(s.exitcode) * ")" :
  874            error("process status error")
  875 end
  876 
  877 ## implementation of `cmd` syntax ##
  878 
  879 arg_gen() = String[]
  880 arg_gen(x::AbstractString) = String[cstr(x)]
  881 arg_gen(cmd::Cmd) = cmd.exec
  882 
  883 function arg_gen(head)
  884     if isiterable(typeof(head))
  885         vals = String[]
  886         for x in head
  887             push!(vals, cstr(string(x)))
  888         end
  889         return vals
  890     else
  891         return String[cstr(string(head))]
  892     end
  893 end
  894 
  895 function arg_gen(head, tail...)
  896     head = arg_gen(head)
  897     tail = arg_gen(tail...)
  898     vals = String[]
  899     for h = head, t = tail
  900         push!(vals, cstr(string(h,t)))
  901     end
  902     return vals
  903 end
  904 
  905 function cmd_gen(parsed)
  906     args = String[]
  907     for arg in parsed
  908         append!(args, arg_gen(arg...))
  909     end
  910     return Cmd(args)
  911 end
  912 
  913 """
  914     @cmd str
  915 
  916 Similar to `cmd`, generate a `Cmd` from the `str` string which represents the shell command(s) to be executed.
  917 The [`Cmd`](@ref) object can be run as a process and can outlive the spawning julia process (see `Cmd` for more).
  918 
  919 # Examples
  920 ```jldoctest
  921 julia> cm = @cmd " echo 1 "
  922 `echo 1`
  923 
  924 julia> run(cm)
  925 1
  926 Process(`echo 1`, ProcessExited(0))
  927 ```
  928 """
  929 macro cmd(str)
  930     return :(cmd_gen($(esc(shell_parse(str, special=shell_special)[1]))))
  931 end
  932 
  933 wait(x::Process)      = if !process_exited(x); stream_wait(x, x.exitnotify); end
  934 wait(x::ProcessChain) = for p in x.processes; wait(p); end
  935 
  936 show(io::IO, p::Process) = print(io, "Process(", p.cmd, ", ", process_status(p), ")")
  937 
  938 # allow the elements of the Cmd to be accessed as an array or iterator
  939 for f in (:length, :firstindex, :lastindex, :keys, :first, :last, :iterate)
  940     @eval $f(cmd::Cmd) = $f(cmd.exec)
  941 end
  942 eltype(::Type{Cmd}) = eltype(fieldtype(Cmd, :exec))
  943 for f in (:iterate, :getindex)
  944     @eval $f(cmd::Cmd, i) = $f(cmd.exec, i)
  945 end