2 * Copyright 2013 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
18 * Subprocess library, modeled after Python's subprocess module
19 * (http://docs.python.org/2/library/subprocess.html)
21 * This library defines one class (Subprocess) which represents a child
22 * process. Subprocess has two constructors: one that takes a vector<string>
23 * and executes the given executable without using the shell, and one
24 * that takes a string and executes the given command using the shell.
25 * Subprocess allows you to redirect the child's standard input, standard
26 * output, and standard error to/from child descriptors in the parent,
27 * or to create communication pipes between the child and the parent.
29 * The simplest example is a thread-safe version of the system() library
31 * Subprocess(cmd).wait();
32 * which executes the command using the default shell and waits for it
33 * to complete, returning the exit status.
35 * A thread-safe version of popen() (type="r", to read from the child):
36 * Subprocess proc(cmd, Subprocess::pipeStdout());
37 * // read from proc.stdout()
40 * A thread-safe version of popen() (type="w", to write from the child):
41 * Subprocess proc(cmd, Subprocess::pipeStdin());
42 * // write to proc.stdin()
45 * If you want to redirect both stdin and stdout to pipes, you can, but
46 * note that you're subject to a variety of deadlocks. You'll want to use
47 * nonblocking I/O; look at the implementation of communicate() for an example.
49 * communicate() is a way to communicate to a child via its standard input,
50 * standard output, and standard error. It buffers everything in memory,
51 * so it's not great for large amounts of data (or long-running processes),
52 * but it insulates you from the deadlocks mentioned above.
54 #ifndef FOLLY_SUBPROCESS_H_
55 #define FOLLY_SUBPROCESS_H_
57 #include <sys/types.h>
65 #include <boost/container/flat_map.hpp>
66 #include <boost/operators.hpp>
67 #include <boost/noncopyable.hpp>
69 #include "folly/io/IOBufQueue.h"
70 #include "folly/MapUtil.h"
71 #include "folly/Portability.h"
72 #include "folly/Range.h"
77 * Class to wrap a process return code.
80 class ProcessReturnCode {
81 friend class Subprocess;
91 * Process state. One of:
92 * NOT_STARTED: process hasn't been started successfully
93 * RUNNING: process is currently running
94 * EXITED: process exited (successfully or not)
95 * KILLED: process was killed by a signal.
100 * Helper wrappers around state().
102 bool notStarted() const { return state() == NOT_STARTED; }
103 bool running() const { return state() == RUNNING; }
104 bool exited() const { return state() == EXITED; }
105 bool killed() const { return state() == KILLED; }
108 * Exit status. Only valid if state() == EXITED; throws otherwise.
110 int exitStatus() const;
113 * Signal that caused the process's termination. Only valid if
114 * state() == KILLED; throws otherwise.
116 int killSignal() const;
119 * Was a core file generated? Only valid if state() == KILLED; throws
122 bool coreDumped() const;
125 * String representation; one of
128 * "exited with status <status>"
129 * "killed by signal <signal>"
130 * "killed by signal <signal> (core dumped)"
132 std::string str() const;
135 * Helper function to enforce a precondition based on this.
136 * Throws std::logic_error if in an unexpected state.
138 void enforce(State state) const;
140 explicit ProcessReturnCode(int rv) : rawStatus_(rv) { }
141 static constexpr int RV_NOT_STARTED = -2;
142 static constexpr int RV_RUNNING = -1;
148 * Exception thrown by *Checked methods of Subprocess.
150 class CalledProcessError : public std::exception {
152 explicit CalledProcessError(ProcessReturnCode rc);
153 ~CalledProcessError() throw() { }
154 const char* what() const throw() FOLLY_OVERRIDE { return what_.c_str(); }
155 ProcessReturnCode returnCode() const { return returnCode_; }
157 ProcessReturnCode returnCode_;
164 class Subprocess : private boost::noncopyable {
166 static const int CLOSE = -1;
167 static const int PIPE = -2;
168 static const int PIPE_IN = -3;
169 static const int PIPE_OUT = -4;
172 * Class representing various options: file descriptor behavior, and
173 * whether to use $PATH for searching for the executable,
175 * By default, we don't use $PATH, file descriptors are closed if
176 * the close-on-exec flag is set (fcntl FD_CLOEXEC) and inherited
179 class Options : private boost::orable<Options> {
180 friend class Subprocess;
183 : closeOtherFds_(false),
185 parentDeathSignal_(0) {
189 * Change action for file descriptor fd.
191 * "action" may be another file descriptor number (dup2()ed before the
192 * child execs), or one of CLOSE, PIPE_IN, and PIPE_OUT.
194 * CLOSE: close the file descriptor in the child
195 * PIPE_IN: open a pipe *from* the child
196 * PIPE_OUT: open a pipe *to* the child
198 * PIPE is a shortcut; same as PIPE_IN for stdin (fd 0), same as
199 * PIPE_OUT for stdout (fd 1) or stderr (fd 2), and an error for
200 * other file descriptors.
202 Options& fd(int fd, int action);
205 * Shortcut to change the action for standard input.
207 Options& stdin(int action) { return fd(0, action); }
210 * Shortcut to change the action for standard output.
212 Options& stdout(int action) { return fd(1, action); }
215 * Shortcut to change the action for standard error.
216 * Note that stderr(1) will redirect the standard error to the same
217 * file descriptor as standard output; the equivalent of bash's "2>&1"
219 Options& stderr(int action) { return fd(2, action); }
222 * Close all other fds (other than standard input, output, error,
223 * and file descriptors explicitly specified with fd()).
225 * This is potentially slow; it's generally a better idea to
226 * set the close-on-exec flag on all file descriptors that shouldn't
227 * be inherited by the child.
229 * Even with this option set, standard input, output, and error are
230 * not closed; use stdin(CLOSE), stdout(CLOSE), stderr(CLOSE) if you
233 Options& closeOtherFds() { closeOtherFds_ = true; return *this; }
236 * Use the search path ($PATH) when searching for the executable.
238 Options& usePath() { usePath_ = true; return *this; }
241 * Child will receive a signal when the parent exits.
243 Options& parentDeathSignal(int sig) {
244 parentDeathSignal_ = sig;
249 * Helpful way to combine Options.
251 Options& operator|=(const Options& other);
254 typedef boost::container::flat_map<int, int> FdMap;
258 int parentDeathSignal_;
261 static Options pipeStdin() { return Options().stdin(PIPE); }
262 static Options pipeStdout() { return Options().stdout(PIPE); }
263 static Options pipeStderr() { return Options().stderr(PIPE); }
266 * Create a subprocess from the given arguments. argv[0] must be listed.
267 * If not-null, executable must be the actual executable
268 * being used (otherwise it's the same as argv[0]).
270 * If env is not-null, it must contain name=value strings to be used
271 * as the child's environment; otherwise, we inherit the environment
272 * from the parent. env must be null if options.usePath is set.
275 const std::vector<std::string>& argv,
276 const Options& options = Options(),
277 const char* executable = nullptr,
278 const std::vector<std::string>* env = nullptr);
282 * Create a subprocess run as a shell command (as shell -c 'command')
284 * The shell to use is taken from the environment variable $SHELL,
285 * or /bin/sh if $SHELL is unset.
288 const std::string& cmd,
289 const Options& options = Options(),
290 const std::vector<std::string>* env = nullptr);
293 * Append all data, close the stdin (to-child) fd, and read all data,
294 * except that this is done in a safe manner to prevent deadlocking.
296 * If writeStdin() is given in flags, the process must have been opened with
299 * If readStdout() is given in flags, the first returned value will be the
300 * value read from the child's stdout; the child must have been opened with
303 * If readStderr() is given in flags, the second returned value will be the
304 * value read from the child's stderr; the child must have been opened with
307 * Note that communicate() returns when all pipes to/from the child are
308 * closed; the child might stay alive after that, so you must still wait().
310 * communicateIOBuf uses IOBufQueue for buffering (which has the advantage
311 * that it won't try to allocate all data at once). communicate
312 * uses strings for simplicity.
314 class CommunicateFlags : private boost::orable<CommunicateFlags> {
315 friend class Subprocess;
318 : writeStdin_(false), readStdout_(false), readStderr_(false) { }
319 CommunicateFlags& writeStdin() { writeStdin_ = true; return *this; }
320 CommunicateFlags& readStdout() { readStdout_ = true; return *this; }
321 CommunicateFlags& readStderr() { readStderr_ = true; return *this; }
323 CommunicateFlags& operator|=(const CommunicateFlags& other);
330 static CommunicateFlags writeStdin() {
331 return CommunicateFlags().writeStdin();
333 static CommunicateFlags readStdout() {
334 return CommunicateFlags().readStdout();
336 static CommunicateFlags readStderr() {
337 return CommunicateFlags().readStderr();
340 std::pair<IOBufQueue, IOBufQueue> communicateIOBuf(
341 const CommunicateFlags& flags = readStdout(),
342 IOBufQueue data = IOBufQueue());
344 std::pair<std::string, std::string> communicate(
345 const CommunicateFlags& flags = readStdout(),
346 StringPiece data = StringPiece());
349 * Communicate with the child until all pipes to/from the child are closed.
351 * readCallback(pfd, cfd) will be called whenever there's data available
352 * on any pipe *from* the child (PIPE_OUT). pfd is the file descriptor
353 * in the parent (that you use to read from); cfd is the file descriptor
354 * in the child (used for identifying the stream; 1 = child's standard
355 * output, 2 = child's standard error, etc)
357 * writeCallback(pfd, cfd) will be called whenever a pipe *to* the child is
358 * writable (PIPE_IN). pfd is the file descriptor in the parent (that you
359 * use to write to); cfd is the file descriptor in the child (used for
360 * identifying the stream; 0 = child's standard input, etc)
362 * The read and write callbacks must read from / write to pfd and return
363 * false during normal operation or true at end-of-file;
364 * communicate() will then close the pipe. Note that pfd is
365 * nonblocking, so be prepared for read() / write() to return -1 and
366 * set errno to EAGAIN (in which case you should return false).
368 * NOTE that you MUST consume all data passed to readCallback (or return
369 * true, which will close the pipe, possibly sending SIGPIPE to the child or
370 * making its writes fail with EPIPE), and you MUST write to a writable pipe
371 * (or return true, which will close the pipe). To do otherwise is an
372 * error. You must do this even for pipes you are not interested in.
374 * Note that communicate() returns when all pipes to/from the child are
375 * closed; the child might stay alive after that, so you must still wait().
377 * Most users won't need to use this; the simpler version of communicate
378 * (which buffers data in memory) will probably work fine.
380 typedef std::function<bool(int, int)> FdCallback;
381 void communicate(FdCallback readCallback, FdCallback writeCallback);
384 * Return the child's pid, or -1 if the child wasn't successfully spawned
385 * or has already been wait()ed upon.
390 * Return the child's status (as per wait()) if the process has already
391 * been waited on, -1 if the process is still running, or -2 if the process
392 * hasn't been successfully started. NOTE that this does not poll, but
393 * returns the status stored in the Subprocess object.
395 ProcessReturnCode returnCode() const { return returnCode_; }
398 * Poll the child's status and return it, return -1 if the process
399 * is still running. NOTE that it is illegal to call poll again after
400 * poll indicated that the process has terminated, or to call poll on a
401 * process that hasn't been successfully started (the constructor threw an
404 ProcessReturnCode poll();
407 * Poll the child's status. If the process is still running, return false.
408 * Otherwise, return true if the process exited with status 0 (success),
409 * or throw CalledProcessError if the process exited with a non-zero status.
414 * Wait for the process to terminate and return its status.
415 * Similarly to poll, it is illegal to call wait after the process
416 * has already been reaped or if the process has not successfully started.
418 ProcessReturnCode wait();
421 * Wait for the process to terminate, throw if unsuccessful.
426 * Set all pipes from / to child non-blocking. communicate() does
429 void setAllNonBlocking();
432 * Get parent file descriptor corresponding to the given file descriptor
433 * in the child. Throws if childFd isn't a pipe (PIPE_IN / PIPE_OUT).
434 * Do not close() the return file descriptor; use closeParentFd, below.
436 int parentFd(int childFd) const {
437 return pipes_[findByChildFd(childFd)].parentFd;
439 int stdin() const { return parentFd(0); }
440 int stdout() const { return parentFd(1); }
441 int stderr() const { return parentFd(2); }
444 * Close the parent file descriptor given a file descriptor in the child.
446 void closeParentFd(int childFd);
449 * Send a signal to the child. Shortcuts for the commonly used Unix
452 void sendSignal(int signal);
453 void terminate() { sendSignal(SIGTERM); }
454 void kill() { sendSignal(SIGKILL); }
457 static const int RV_RUNNING = ProcessReturnCode::RV_RUNNING;
458 static const int RV_NOT_STARTED = ProcessReturnCode::RV_NOT_STARTED;
461 std::unique_ptr<const char*[]> argv,
462 const char* executable,
463 const Options& options,
464 const std::vector<std::string>* env);
466 // Action to run in child.
467 // Note that this runs after vfork(), so tread lightly.
468 void runChild(const char* executable, char** argv, char** env,
469 const Options& options) const;
472 * Close all file descriptors.
476 // return index in pipes_
477 int findByChildFd(int childFd) const;
480 ProcessReturnCode returnCode_;
482 // The number of pipes between parent and child is assumed to be small,
483 // so we're happy with a vector here, even if it means linear erase.
485 struct PipeInfo : private boost::totally_ordered<PipeInfo> {
488 int direction; // one of PIPE_IN / PIPE_OUT
489 bool operator<(const PipeInfo& other) const {
490 return childFd < other.childFd;
492 bool operator==(const PipeInfo& other) const {
493 return childFd == other.childFd;
496 std::vector<PipeInfo> pipes_;
499 inline Subprocess::Options& Subprocess::Options::operator|=(
500 const Subprocess::Options& other) {
501 if (this == &other) return *this;
503 for (auto& p : other.fdActions_) {
504 fdActions_[p.first] = p.second;
506 closeOtherFds_ |= other.closeOtherFds_;
507 usePath_ |= other.usePath_;
511 inline Subprocess::CommunicateFlags& Subprocess::CommunicateFlags::operator|=(
512 const Subprocess::CommunicateFlags& other) {
513 if (this == &other) return *this;
514 writeStdin_ |= other.writeStdin_;
515 readStdout_ |= other.readStdout_;
516 readStderr_ |= other.readStderr_;
522 #endif /* FOLLY_SUBPROCESS_H_ */