2 * Copyright 2014 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
18 * Subprocess library, modeled after Python's subprocess module
19 * (http://docs.python.org/2/library/subprocess.html)
21 * This library defines one class (Subprocess) which represents a child
22 * process. Subprocess has two constructors: one that takes a vector<string>
23 * and executes the given executable without using the shell, and one
24 * that takes a string and executes the given command using the shell.
25 * Subprocess allows you to redirect the child's standard input, standard
26 * output, and standard error to/from child descriptors in the parent,
27 * or to create communication pipes between the child and the parent.
29 * The simplest example is a thread-safe version of the system() library
31 * Subprocess(cmd).wait();
32 * which executes the command using the default shell and waits for it
33 * to complete, returning the exit status.
35 * A thread-safe version of popen() (type="r", to read from the child):
36 * Subprocess proc(cmd, Subprocess::pipeStdout());
37 * // read from proc.stdout()
40 * A thread-safe version of popen() (type="w", to write to the child):
41 * Subprocess proc(cmd, Subprocess::pipeStdin());
42 * // write to proc.stdin()
45 * If you want to redirect both stdin and stdout to pipes, you can, but
46 * note that you're subject to a variety of deadlocks. You'll want to use
47 * nonblocking I/O; look at the implementation of communicate() for an example.
49 * communicate() is a way to communicate to a child via its standard input,
50 * standard output, and standard error. It buffers everything in memory,
51 * so it's not great for large amounts of data (or long-running processes),
52 * but it insulates you from the deadlocks mentioned above.
54 #ifndef FOLLY_SUBPROCESS_H_
55 #define FOLLY_SUBPROCESS_H_
57 #include <sys/types.h>
69 #include <boost/container/flat_map.hpp>
70 #include <boost/operators.hpp>
71 #include <boost/noncopyable.hpp>
73 #include "folly/io/IOBufQueue.h"
74 #include "folly/MapUtil.h"
75 #include "folly/Portability.h"
76 #include "folly/Range.h"
81 * Class to wrap a process return code.
84 class ProcessReturnCode {
85 friend class Subprocess;
95 * Process state. One of:
96 * NOT_STARTED: process hasn't been started successfully
97 * RUNNING: process is currently running
98 * EXITED: process exited (successfully or not)
99 * KILLED: process was killed by a signal.
104 * Helper wrappers around state().
106 bool notStarted() const { return state() == NOT_STARTED; }
107 bool running() const { return state() == RUNNING; }
108 bool exited() const { return state() == EXITED; }
109 bool killed() const { return state() == KILLED; }
112 * Exit status. Only valid if state() == EXITED; throws otherwise.
114 int exitStatus() const;
117 * Signal that caused the process's termination. Only valid if
118 * state() == KILLED; throws otherwise.
120 int killSignal() const;
123 * Was a core file generated? Only valid if state() == KILLED; throws
126 bool coreDumped() const;
129 * String representation; one of
132 * "exited with status <status>"
133 * "killed by signal <signal>"
134 * "killed by signal <signal> (core dumped)"
136 std::string str() const;
139 * Helper function to enforce a precondition based on this.
140 * Throws std::logic_error if in an unexpected state.
142 void enforce(State state) const;
144 explicit ProcessReturnCode(int rv) : rawStatus_(rv) { }
145 static constexpr int RV_NOT_STARTED = -2;
146 static constexpr int RV_RUNNING = -1;
152 * Base exception thrown by the Subprocess methods.
154 class SubprocessError : public std::exception {};
157 * Exception thrown by *Checked methods of Subprocess.
159 class CalledProcessError : public SubprocessError {
161 explicit CalledProcessError(ProcessReturnCode rc);
162 ~CalledProcessError() throw() { }
163 const char* what() const throw() FOLLY_OVERRIDE { return what_.c_str(); }
164 ProcessReturnCode returnCode() const { return returnCode_; }
166 ProcessReturnCode returnCode_;
171 * Exception thrown if the subprocess cannot be started.
173 class SubprocessSpawnError : public SubprocessError {
175 SubprocessSpawnError(const char* executable, int errCode, int errnoValue);
176 ~SubprocessSpawnError() throw() {}
177 const char* what() const throw() FOLLY_OVERRIDE { return what_.c_str(); }
178 int errnoValue() const { return errnoValue_; }
188 class Subprocess : private boost::noncopyable {
190 static const int CLOSE = -1;
191 static const int PIPE = -2;
192 static const int PIPE_IN = -3;
193 static const int PIPE_OUT = -4;
196 * Class representing various options: file descriptor behavior, and
197 * whether to use $PATH for searching for the executable,
199 * By default, we don't use $PATH, file descriptors are closed if
200 * the close-on-exec flag is set (fcntl FD_CLOEXEC) and inherited
203 class Options : private boost::orable<Options> {
204 friend class Subprocess;
207 : closeOtherFds_(false),
212 * Change action for file descriptor fd.
214 * "action" may be another file descriptor number (dup2()ed before the
215 * child execs), or one of CLOSE, PIPE_IN, and PIPE_OUT.
217 * CLOSE: close the file descriptor in the child
218 * PIPE_IN: open a pipe *from* the child
219 * PIPE_OUT: open a pipe *to* the child
221 * PIPE is a shortcut; same as PIPE_IN for stdin (fd 0), same as
222 * PIPE_OUT for stdout (fd 1) or stderr (fd 2), and an error for
223 * other file descriptors.
225 Options& fd(int fd, int action);
228 * Shortcut to change the action for standard input.
230 Options& stdin(int action) { return fd(STDIN_FILENO, action); }
233 * Shortcut to change the action for standard output.
235 Options& stdout(int action) { return fd(STDOUT_FILENO, action); }
238 * Shortcut to change the action for standard error.
239 * Note that stderr(1) will redirect the standard error to the same
240 * file descriptor as standard output; the equivalent of bash's "2>&1"
242 Options& stderr(int action) { return fd(STDERR_FILENO, action); }
244 Options& pipeStdin() { return fd(STDIN_FILENO, PIPE_IN); }
245 Options& pipeStdout() { return fd(STDOUT_FILENO, PIPE_OUT); }
246 Options& pipeStderr() { return fd(STDERR_FILENO, PIPE_OUT); }
249 * Close all other fds (other than standard input, output, error,
250 * and file descriptors explicitly specified with fd()).
252 * This is potentially slow; it's generally a better idea to
253 * set the close-on-exec flag on all file descriptors that shouldn't
254 * be inherited by the child.
256 * Even with this option set, standard input, output, and error are
257 * not closed; use stdin(CLOSE), stdout(CLOSE), stderr(CLOSE) if you
260 Options& closeOtherFds() { closeOtherFds_ = true; return *this; }
263 * Use the search path ($PATH) when searching for the executable.
265 Options& usePath() { usePath_ = true; return *this; }
268 * Change the child's working directory, after the vfork.
270 Options& chdir(const std::string& dir) { childDir_ = dir; return *this; }
274 * Child will receive a signal when the parent exits.
276 Options& parentDeathSignal(int sig) {
277 parentDeathSignal_ = sig;
283 * Helpful way to combine Options.
285 Options& operator|=(const Options& other);
288 typedef boost::container::flat_map<int, int> FdMap;
292 std::string childDir_; // "" keeps the parent's working directory
294 int parentDeathSignal_{0};
298 static Options pipeStdin() { return Options().stdin(PIPE); }
299 static Options pipeStdout() { return Options().stdout(PIPE); }
300 static Options pipeStderr() { return Options().stderr(PIPE); }
303 * Create a subprocess from the given arguments. argv[0] must be listed.
304 * If not-null, executable must be the actual executable
305 * being used (otherwise it's the same as argv[0]).
307 * If env is not-null, it must contain name=value strings to be used
308 * as the child's environment; otherwise, we inherit the environment
309 * from the parent. env must be null if options.usePath is set.
312 const std::vector<std::string>& argv,
313 const Options& options = Options(),
314 const char* executable = nullptr,
315 const std::vector<std::string>* env = nullptr);
319 * Create a subprocess run as a shell command (as shell -c 'command')
321 * The shell to use is taken from the environment variable $SHELL,
322 * or /bin/sh if $SHELL is unset.
325 const std::string& cmd,
326 const Options& options = Options(),
327 const std::vector<std::string>* env = nullptr);
330 * Communicate with the child until all pipes to/from the child are closed.
332 * The input buffer is written to the process' stdin pipe, and data is read
333 * from the stdout and stderr pipes. Non-blocking I/O is performed on all
334 * pipes simultaneously to avoid deadlocks.
336 * The stdin pipe will be closed after the full input buffer has been written.
337 * An error will be thrown if a non-empty input buffer is supplied but stdin
338 * was not configured as a pipe.
340 * Returns a pair of buffers containing the data read from stdout and stderr.
341 * If stdout or stderr is not a pipe, an empty IOBuf queue will be returned
342 * for the respective buffer.
344 * Note that communicate() and communicateIOBuf() both return when all
345 * pipes to/from the child are closed; the child might stay alive after
346 * that, so you must still wait().
348 * communicateIOBuf() uses IOBufQueue for buffering (which has the
349 * advantage that it won't try to allocate all data at once), but it does
350 * store the subprocess's entire output in memory before returning.
352 * communicate() uses strings for simplicity.
354 std::pair<IOBufQueue, IOBufQueue> communicateIOBuf(
355 IOBufQueue input = IOBufQueue());
357 std::pair<std::string, std::string> communicate(
358 StringPiece input = StringPiece());
361 * Communicate with the child until all pipes to/from the child are closed.
363 * readCallback(pfd, cfd) will be called whenever there's data available
364 * on any pipe *from* the child (PIPE_OUT). pfd is the file descriptor
365 * in the parent (that you use to read from); cfd is the file descriptor
366 * in the child (used for identifying the stream; 1 = child's standard
367 * output, 2 = child's standard error, etc)
369 * writeCallback(pfd, cfd) will be called whenever a pipe *to* the child is
370 * writable (PIPE_IN). pfd is the file descriptor in the parent (that you
371 * use to write to); cfd is the file descriptor in the child (used for
372 * identifying the stream; 0 = child's standard input, etc)
374 * The read and write callbacks must read from / write to pfd and return
375 * false during normal operation or true at end-of-file;
376 * communicate() will then close the pipe. Note that pfd is
377 * nonblocking, so be prepared for read() / write() to return -1 and
378 * set errno to EAGAIN (in which case you should return false).
380 * NOTE that you MUST consume all data passed to readCallback (or return
381 * true, which will close the pipe, possibly sending SIGPIPE to the child or
382 * making its writes fail with EPIPE), and you MUST write to a writable pipe
383 * (or return true, which will close the pipe). To do otherwise is an
384 * error. You must do this even for pipes you are not interested in.
386 * Note that communicate() returns when all pipes to/from the child are
387 * closed; the child might stay alive after that, so you must still wait().
389 * Most users won't need to use this; the simpler version of communicate
390 * (which buffers data in memory) will probably work fine.
392 typedef std::function<bool(int, int)> FdCallback;
393 void communicate(FdCallback readCallback, FdCallback writeCallback);
396 * Enable notifications (callbacks) for one pipe to/from child. By default,
397 * all are enabled. Useful for "chatty" communication -- you want to disable
398 * write callbacks until you receive the expected message.
400 void enableNotifications(int childFd, bool enabled);
403 * Are notifications for one pipe to/from child enabled?
405 bool notificationsEnabled(int childFd) const;
408 * Return the child's pid, or -1 if the child wasn't successfully spawned
409 * or has already been wait()ed upon.
414 * Return the child's status (as per wait()) if the process has already
415 * been waited on, -1 if the process is still running, or -2 if the process
416 * hasn't been successfully started. NOTE that this does not poll, but
417 * returns the status stored in the Subprocess object.
419 ProcessReturnCode returnCode() const { return returnCode_; }
422 * Poll the child's status and return it, return -1 if the process
423 * is still running. NOTE that it is illegal to call poll again after
424 * poll indicated that the process has terminated, or to call poll on a
425 * process that hasn't been successfully started (the constructor threw an
428 ProcessReturnCode poll();
431 * Poll the child's status. If the process is still running, return false.
432 * Otherwise, return true if the process exited with status 0 (success),
433 * or throw CalledProcessError if the process exited with a non-zero status.
438 * Wait for the process to terminate and return its status.
439 * Similarly to poll, it is illegal to call wait after the process
440 * has already been reaped or if the process has not successfully started.
442 ProcessReturnCode wait();
445 * Wait for the process to terminate, throw if unsuccessful.
450 * Set all pipes from / to child non-blocking. communicate() does
453 void setAllNonBlocking();
456 * Get parent file descriptor corresponding to the given file descriptor
457 * in the child. Throws if childFd isn't a pipe (PIPE_IN / PIPE_OUT).
458 * Do not close() the return file descriptor; use closeParentFd, below.
460 int parentFd(int childFd) const {
461 return pipes_[findByChildFd(childFd)].parentFd;
463 int stdin() const { return parentFd(0); }
464 int stdout() const { return parentFd(1); }
465 int stderr() const { return parentFd(2); }
468 * Close the parent file descriptor given a file descriptor in the child.
470 void closeParentFd(int childFd);
473 * Send a signal to the child. Shortcuts for the commonly used Unix
476 void sendSignal(int signal);
477 void terminate() { sendSignal(SIGTERM); }
478 void kill() { sendSignal(SIGKILL); }
481 static const int RV_RUNNING = ProcessReturnCode::RV_RUNNING;
482 static const int RV_NOT_STARTED = ProcessReturnCode::RV_NOT_STARTED;
484 // spawn() sets up a pipe to read errors from the child,
485 // then calls spawnInternal() to do the bulk of the work. Once
486 // spawnInternal() returns it reads the error pipe to see if the child
487 // encountered any errors.
489 std::unique_ptr<const char*[]> argv,
490 const char* executable,
491 const Options& options,
492 const std::vector<std::string>* env);
494 std::unique_ptr<const char*[]> argv,
495 const char* executable,
497 const std::vector<std::string>* env,
500 // Actions to run in child.
501 // Note that this runs after vfork(), so tread lightly.
502 // Returns 0 on success, or an errno value on failure.
503 int prepareChild(const Options& options, const sigset_t* sigmask) const;
504 int runChild(const char* executable, char** argv, char** env,
505 const Options& options) const;
508 * Read from the error pipe, and throw SubprocessSpawnError if the child
509 * failed before calling exec().
511 void readChildErrorPipe(int pfd, const char* executable);
514 * Close all file descriptors.
518 // return index in pipes_
519 int findByChildFd(int childFd) const;
522 ProcessReturnCode returnCode_;
524 // The number of pipes between parent and child is assumed to be small,
525 // so we're happy with a vector here, even if it means linear erase.
527 struct PipeInfo : private boost::totally_ordered<PipeInfo> {
530 int direction = PIPE_IN; // one of PIPE_IN / PIPE_OUT
533 bool operator<(const PipeInfo& other) const {
534 return childFd < other.childFd;
536 bool operator==(const PipeInfo& other) const {
537 return childFd == other.childFd;
540 std::vector<PipeInfo> pipes_;
543 inline Subprocess::Options& Subprocess::Options::operator|=(
544 const Subprocess::Options& other) {
545 if (this == &other) return *this;
547 for (auto& p : other.fdActions_) {
548 fdActions_[p.first] = p.second;
550 closeOtherFds_ |= other.closeOtherFds_;
551 usePath_ |= other.usePath_;
557 #endif /* FOLLY_SUBPROCESS_H_ */