2 * Copyright 2017 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include <folly/Subprocess.h>
19 #include <sys/types.h>
21 #include <boost/container/flat_set.hpp>
22 #include <glog/logging.h>
24 #include <folly/Exception.h>
25 #include <folly/FileUtil.h>
26 #include <folly/Format.h>
27 #include <folly/String.h>
28 #include <folly/experimental/TestUtil.h>
29 #include <folly/experimental/io/FsUtil.h>
30 #include <folly/gen/Base.h>
31 #include <folly/gen/File.h>
32 #include <folly/gen/String.h>
33 #include <folly/portability/GTest.h>
34 #include <folly/portability/Unistd.h>
36 FOLLY_GCC_DISABLE_WARNING("-Wdeprecated-declarations")
38 using namespace folly;
40 TEST(SimpleSubprocessTest, ExitsSuccessfully) {
41 Subprocess proc(std::vector<std::string>{ "/bin/true" });
42 EXPECT_EQ(0, proc.wait().exitStatus());
45 TEST(SimpleSubprocessTest, ExitsSuccessfullyChecked) {
46 Subprocess proc(std::vector<std::string>{ "/bin/true" });
50 TEST(SimpleSubprocessTest, CloneFlagsWithVfork) {
52 std::vector<std::string>{"/bin/true"},
53 Subprocess::Options().useCloneWithFlags(SIGCHLD | CLONE_VFORK));
54 EXPECT_EQ(0, proc.wait().exitStatus());
57 TEST(SimpleSubprocessTest, CloneFlagsWithFork) {
59 std::vector<std::string>{"/bin/true"},
60 Subprocess::Options().useCloneWithFlags(SIGCHLD));
61 EXPECT_EQ(0, proc.wait().exitStatus());
64 TEST(SimpleSubprocessTest, CloneFlagsSubprocessCtorExitsAfterExec) {
66 std::vector<std::string>{"/bin/sleep", "3600"},
67 Subprocess::Options().useCloneWithFlags(SIGCHLD));
68 checkUnixError(::kill(proc.pid(), SIGKILL), "kill");
69 auto retCode = proc.wait();
70 EXPECT_TRUE(retCode.killed());
73 TEST(SimpleSubprocessTest, ExitsWithError) {
74 Subprocess proc(std::vector<std::string>{ "/bin/false" });
75 EXPECT_EQ(1, proc.wait().exitStatus());
78 TEST(SimpleSubprocessTest, ExitsWithErrorChecked) {
79 Subprocess proc(std::vector<std::string>{ "/bin/false" });
80 EXPECT_THROW(proc.waitChecked(), CalledProcessError);
83 TEST(SimpleSubprocessTest, DefaultConstructibleProcessReturnCode) {
84 ProcessReturnCode retcode;
85 EXPECT_TRUE(retcode.notStarted());
88 TEST(SimpleSubprocessTest, MoveSubprocess) {
89 Subprocess old_proc(std::vector<std::string>{ "/bin/true" });
90 EXPECT_TRUE(old_proc.returnCode().running());
91 auto new_proc = std::move(old_proc);
92 EXPECT_TRUE(old_proc.returnCode().notStarted());
93 EXPECT_TRUE(new_proc.returnCode().running());
94 EXPECT_EQ(0, new_proc.wait().exitStatus());
95 // Now old_proc is destroyed, but we don't crash.
98 TEST(SimpleSubprocessTest, DefaultConstructor) {
100 EXPECT_TRUE(proc.returnCode().notStarted());
103 auto p1 = Subprocess(std::vector<std::string>{"/bin/true"});
104 proc = std::move(p1);
107 EXPECT_TRUE(proc.returnCode().running());
108 EXPECT_EQ(0, proc.wait().exitStatus());
111 #define EXPECT_SPAWN_ERROR(err, errMsg, cmd, ...) \
114 Subprocess proc(std::vector<std::string>{ (cmd), ## __VA_ARGS__ }); \
115 ADD_FAILURE() << "expected an error when running " << (cmd); \
116 } catch (const SubprocessSpawnError& ex) { \
117 EXPECT_EQ((err), ex.errnoValue()); \
118 if (StringPiece(ex.what()).find(errMsg) == StringPiece::npos) { \
119 ADD_FAILURE() << "failed to find \"" << (errMsg) << \
120 "\" in exception: \"" << ex.what() << "\""; \
125 TEST(SimpleSubprocessTest, ExecFails) {
126 EXPECT_SPAWN_ERROR(ENOENT, "failed to execute /no/such/file:",
128 EXPECT_SPAWN_ERROR(EACCES, "failed to execute /etc/passwd:",
130 EXPECT_SPAWN_ERROR(ENOTDIR, "failed to execute /etc/passwd/not/a/file:",
131 "/etc/passwd/not/a/file");
134 TEST(SimpleSubprocessTest, ShellExitsSuccesssfully) {
135 Subprocess proc("true");
136 EXPECT_EQ(0, proc.wait().exitStatus());
139 TEST(SimpleSubprocessTest, ShellExitsWithError) {
140 Subprocess proc("false");
141 EXPECT_EQ(1, proc.wait().exitStatus());
144 TEST(SimpleSubprocessTest, ChangeChildDirectorySuccessfully) {
145 // The filesystem root normally lacks a 'true' binary
146 EXPECT_EQ(0, chdir("/"));
147 EXPECT_SPAWN_ERROR(ENOENT, "failed to execute ./true", "./true");
148 // The child can fix that by moving to /bin before exec().
149 Subprocess proc("./true", Subprocess::Options().chdir("/bin"));
150 EXPECT_EQ(0, proc.wait().exitStatus());
153 TEST(SimpleSubprocessTest, ChangeChildDirectoryWithError) {
156 std::vector<std::string>{"/bin/true"},
157 Subprocess::Options().chdir("/usually/this/is/not/a/valid/directory/")
159 ADD_FAILURE() << "expected to fail when changing the child's directory";
160 } catch (const SubprocessSpawnError& ex) {
161 EXPECT_EQ(ENOENT, ex.errnoValue());
162 const std::string expectedError =
163 "error preparing to execute /bin/true: No such file or directory";
164 if (StringPiece(ex.what()).find(expectedError) == StringPiece::npos) {
165 ADD_FAILURE() << "failed to find \"" << expectedError <<
166 "\" in exception: \"" << ex.what() << "\"";
172 boost::container::flat_set<int> getOpenFds() {
174 auto dirname = to<std::string>("/proc/", pid, "/fd");
176 boost::container::flat_set<int> fds;
177 for (fs::directory_iterator it(dirname);
178 it != fs::directory_iterator();
180 int fd = to<int>(it->path().filename().native());
186 template<class Runnable>
187 void checkFdLeak(const Runnable& r) {
188 // Get the currently open fds. Check that they are the same both before and
189 // after calling the specified function. We read the open fds from /proc.
190 // (If we wanted to work even on systems that don't have /proc, we could
191 // perhaps create and immediately close a socket both before and after
192 // running the function, and make sure we got the same fd number both times.)
193 auto fdsBefore = getOpenFds();
195 auto fdsAfter = getOpenFds();
196 EXPECT_EQ(fdsAfter.size(), fdsBefore.size());
200 // Make sure Subprocess doesn't leak any file descriptors
201 TEST(SimpleSubprocessTest, FdLeakTest) {
204 Subprocess proc("true");
205 EXPECT_EQ(0, proc.wait().exitStatus());
207 // Normal execution with pipes
210 "echo foo; echo bar >&2",
211 Subprocess::Options().pipeStdout().pipeStderr());
212 auto p = proc.communicate();
213 EXPECT_EQ("foo\n", p.first);
214 EXPECT_EQ("bar\n", p.second);
218 // Test where the exec call fails()
220 EXPECT_SPAWN_ERROR(ENOENT, "failed to execute", "/no/such/file");
222 // Test where the exec call fails() with pipes
226 std::vector<std::string>({"/no/such/file"}),
227 Subprocess::Options().pipeStdout().stderrFd(Subprocess::PIPE));
228 ADD_FAILURE() << "expected an error when running /no/such/file";
229 } catch (const SubprocessSpawnError& ex) {
230 EXPECT_EQ(ENOENT, ex.errnoValue());
235 TEST(ParentDeathSubprocessTest, ParentDeathSignal) {
236 // Find out where we are.
237 const auto basename = "subprocess_test_parent_death_helper";
238 auto helper = fs::executable_path();
239 helper.remove_filename() /= basename;
240 if (!fs::exists(helper)) {
241 helper = helper.parent_path().parent_path() / basename / basename;
244 fs::path tempFile(fs::temp_directory_path() / fs::unique_path());
246 std::vector<std::string> args {helper.string(), tempFile.string()};
247 Subprocess proc(args);
248 // The helper gets killed by its child, see details in
249 // SubprocessTestParentDeathHelper.cpp
250 ASSERT_EQ(SIGKILL, proc.wait().killSignal());
252 // Now wait for the file to be created, see details in
253 // SubprocessTestParentDeathHelper.cpp
254 while (!fs::exists(tempFile)) {
255 usleep(20000); // 20ms
258 fs::remove(tempFile);
261 TEST(PopenSubprocessTest, PopenRead) {
262 Subprocess proc("ls /", Subprocess::Options().pipeStdout());
264 gen::byLine(File(proc.stdoutFd())) |
265 [&] (StringPiece line) {
266 if (line == "etc" || line == "bin" || line == "usr") {
274 // DANGER: This class runs after fork in a child processes. Be fast, the
275 // parent thread is waiting, but remember that other parent threads are
276 // running and may mutate your state. Avoid mutating any data belonging to
277 // the parent. Avoid interacting with non-POD data that originated in the
278 // parent. Avoid any libraries that may internally reference non-POD data.
279 // Especially beware parent mutexes -- for example, glog's LOG() uses one.
280 struct WriteFileAfterFork
281 : public Subprocess::DangerousPostForkPreExecCallback {
282 explicit WriteFileAfterFork(std::string filename)
283 : filename_(std::move(filename)) {}
284 ~WriteFileAfterFork() override {}
285 int operator()() override {
286 return writeFile(std::string("ok"), filename_.c_str()) ? 0 : errno;
288 const std::string filename_;
291 TEST(AfterForkCallbackSubprocessTest, TestAfterForkCallbackSuccess) {
292 test::ChangeToTempDir td;
293 // Trigger a file write from the child.
294 WriteFileAfterFork write_cob("good_file");
296 std::vector<std::string>{"/bin/echo"},
297 Subprocess::Options().dangerousPostForkPreExecCallback(&write_cob)
299 // The file gets written immediately.
301 EXPECT_TRUE(readFile(write_cob.filename_.c_str(), s));
306 TEST(AfterForkCallbackSubprocessTest, TestAfterForkCallbackError) {
307 test::ChangeToTempDir td;
308 // The child will try to write to a file, whose directory does not exist.
309 WriteFileAfterFork write_cob("bad/file");
312 std::vector<std::string>{"/bin/echo"},
313 Subprocess::Options().dangerousPostForkPreExecCallback(&write_cob)
317 EXPECT_FALSE(fs::exists(write_cob.filename_));
320 TEST(CommunicateSubprocessTest, SimpleRead) {
322 std::vector<std::string>{"/bin/echo", "-n", "foo", "bar"},
323 Subprocess::Options().pipeStdout());
324 auto p = proc.communicate();
325 EXPECT_EQ("foo bar", p.first);
329 TEST(CommunicateSubprocessTest, BigWrite) {
330 const int numLines = 1 << 20;
331 std::string line("hello\n");
333 data.reserve(numLines * line.size());
334 for (int i = 0; i < numLines; ++i) {
338 Subprocess proc("wc -l", Subprocess::Options().pipeStdin().pipeStdout());
339 auto p = proc.communicate(data);
340 EXPECT_EQ(folly::format("{}\n", numLines).str(), p.first);
344 TEST(CommunicateSubprocessTest, Duplex) {
345 // Take 10MB of data and pass them through a filter.
346 // One line, as tr is line-buffered
347 const int bytes = 10 << 20;
348 std::string line(bytes, 'x');
350 Subprocess proc("tr a-z A-Z", Subprocess::Options().pipeStdin().pipeStdout());
351 auto p = proc.communicate(line);
352 EXPECT_EQ(bytes, p.first.size());
353 EXPECT_EQ(std::string::npos, p.first.find_first_not_of('X'));
357 TEST(CommunicateSubprocessTest, ProcessGroupLeader) {
358 const auto testIsLeader = "test $(cut -d ' ' -f 5 /proc/$$/stat) = $$";
359 Subprocess nonLeader(testIsLeader);
360 EXPECT_THROW(nonLeader.waitChecked(), CalledProcessError);
361 Subprocess leader(testIsLeader, Subprocess::Options().processGroupLeader());
362 leader.waitChecked();
365 TEST(CommunicateSubprocessTest, Duplex2) {
367 // Pipe 200,000 lines through sed
368 const size_t numCopies = 100000;
369 auto iobuf = IOBuf::copyBuffer("this is a test\nanother line\n");
371 for (size_t n = 0; n < numCopies; ++n) {
372 input.append(iobuf->clone());
375 std::vector<std::string> cmd({
377 "-e", "s/a test/a successful test/",
378 "-e", "/^another line/w/dev/stderr",
381 Subprocess::Options().pipeStdin().pipeStdout().pipeStderr().usePath();
382 Subprocess proc(cmd, options);
383 auto out = proc.communicateIOBuf(std::move(input));
386 // Convert stdout and stderr to strings so we can call split() on them.
388 if (out.first.front()) {
389 stdoutStr = out.first.move()->moveToFbString();
392 if (out.second.front()) {
393 stderrStr = out.second.move()->moveToFbString();
396 // stdout should be a copy of stdin, with "a test" replaced by
397 // "a successful test"
398 std::vector<StringPiece> stdoutLines;
399 split('\n', stdoutStr, stdoutLines);
400 EXPECT_EQ(numCopies * 2 + 1, stdoutLines.size());
401 // Strip off the trailing empty line
402 if (!stdoutLines.empty()) {
403 EXPECT_EQ("", stdoutLines.back());
404 stdoutLines.pop_back();
407 for (const auto& line : stdoutLines) {
408 if ((linenum & 1) == 0) {
409 EXPECT_EQ("this is a successful test", line);
411 EXPECT_EQ("another line", line);
416 // stderr should only contain the lines containing "another line"
417 std::vector<StringPiece> stderrLines;
418 split('\n', stderrStr, stderrLines);
419 EXPECT_EQ(numCopies + 1, stderrLines.size());
420 // Strip off the trailing empty line
421 if (!stderrLines.empty()) {
422 EXPECT_EQ("", stderrLines.back());
423 stderrLines.pop_back();
425 for (const auto& line : stderrLines) {
426 EXPECT_EQ("another line", line);
433 bool readToString(int fd, std::string& buf, size_t maxSize) {
435 char* dest = &buf.front();
436 size_t remaining = maxSize;
440 n = ::read(fd, dest, remaining);
442 if (errno == EINTR) {
445 if (errno == EAGAIN) {
448 PCHECK("read failed");
456 buf.resize(dest - buf.data());
462 TEST(CommunicateSubprocessTest, Chatty) {
464 const int lineCount = 1000;
470 Subprocess::Options().pipeStdin().pipeStdout().pipeStderr().usePath();
471 std::vector<std::string> cmd {
475 "s/a test/a successful test/",
478 Subprocess proc(cmd, options);
480 auto writeCallback = [&] (int pfd, int cfd) -> bool {
481 EXPECT_EQ(0, cfd); // child stdin
482 EXPECT_EQ(rcount, wcount); // chatty, one read for every write
484 auto msg = folly::to<std::string>("a test ", wcount, "\n");
486 // Not entirely kosher, we should handle partial writes, but this is
487 // fine for writes <= PIPE_BUF
488 EXPECT_EQ(msg.size(), writeFull(pfd, msg.data(), msg.size()));
491 proc.enableNotifications(0, false);
493 return (wcount == lineCount);
496 bool eofSeen = false;
498 auto readCallback = [&] (int pfd, int cfd) -> bool {
503 EXPECT_TRUE(readToString(pfd, lineBuf, 1));
504 EXPECT_EQ(0, lineBuf.size());
508 EXPECT_FALSE(eofSeen);
510 std::string expected;
512 if (rcount < lineCount) {
513 expected = folly::to<std::string>("a successful test ", rcount++, "\n");
516 EXPECT_EQ(wcount, rcount);
518 // Not entirely kosher, we should handle partial reads, but this is
519 // fine for reads <= PIPE_BUF
520 bool atEof = readToString(pfd, lineBuf, expected.size() + 1);
522 // EOF only expected after we finished reading
523 EXPECT_EQ(lineCount, rcount);
527 EXPECT_EQ(expected, lineBuf);
529 if (wcount != lineCount) { // still more to write...
530 proc.enableNotifications(0, true);
536 proc.communicate(readCallback, writeCallback);
538 EXPECT_EQ(lineCount, wcount);
539 EXPECT_EQ(lineCount, rcount);
540 EXPECT_TRUE(eofSeen);
542 EXPECT_EQ(0, proc.wait().exitStatus());
546 TEST(CommunicateSubprocessTest, TakeOwnershipOfPipes) {
547 std::vector<Subprocess::ChildPipe> pipes;
550 "echo $'oh\\nmy\\ncat' | wc -l &", Subprocess::Options().pipeStdout());
551 pipes = proc.takeOwnershipOfPipes();
554 EXPECT_EQ(1, pipes.size());
555 EXPECT_EQ(1, pipes[0].childFd);
558 EXPECT_EQ(2, readFull(pipes[0].pipe.fd(), buf, 10));
560 EXPECT_EQ("3\n", std::string(buf));