2 * Copyright 2014 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include "folly/Subprocess.h"
20 #include <sys/types.h>
24 #include <boost/container/flat_set.hpp>
25 #include <glog/logging.h>
26 #include <gtest/gtest.h>
28 #include "folly/Exception.h"
29 #include "folly/Format.h"
30 #include "folly/FileUtil.h"
31 #include "folly/String.h"
32 #include "folly/gen/Base.h"
33 #include "folly/gen/File.h"
34 #include "folly/gen/String.h"
35 #include "folly/experimental/io/FsUtil.h"
36 #include "folly/Memory.h"
38 using namespace folly;
40 TEST(SimpleSubprocessTest, ExitsSuccessfully) {
41 Subprocess proc(std::vector<std::string>{ "/bin/true" });
42 EXPECT_EQ(0, proc.wait().exitStatus());
45 TEST(SimpleSubprocessTest, ExitsSuccessfullyChecked) {
46 Subprocess proc(std::vector<std::string>{ "/bin/true" });
50 TEST(SimpleSubprocessTest, ExitsWithError) {
51 Subprocess proc(std::vector<std::string>{ "/bin/false" });
52 EXPECT_EQ(1, proc.wait().exitStatus());
55 TEST(SimpleSubprocessTest, ExitsWithErrorChecked) {
56 Subprocess proc(std::vector<std::string>{ "/bin/false" });
57 EXPECT_THROW(proc.waitChecked(), CalledProcessError);
60 #define EXPECT_SPAWN_ERROR(err, errMsg, cmd, ...) \
63 Subprocess proc(std::vector<std::string>{ (cmd), ## __VA_ARGS__ }); \
64 ADD_FAILURE() << "expected an error when running " << (cmd); \
65 } catch (const SubprocessSpawnError& ex) { \
66 EXPECT_EQ((err), ex.errnoValue()); \
67 if (StringPiece(ex.what()).find(errMsg) == StringPiece::npos) { \
68 ADD_FAILURE() << "failed to find \"" << (errMsg) << \
69 "\" in exception: \"" << ex.what() << "\""; \
74 TEST(SimpleSubprocessTest, ExecFails) {
75 EXPECT_SPAWN_ERROR(ENOENT, "failed to execute /no/such/file:",
77 EXPECT_SPAWN_ERROR(EACCES, "failed to execute /etc/passwd:",
79 EXPECT_SPAWN_ERROR(ENOTDIR, "failed to execute /etc/passwd/not/a/file:",
80 "/etc/passwd/not/a/file");
83 TEST(SimpleSubprocessTest, ShellExitsSuccesssfully) {
84 Subprocess proc("true");
85 EXPECT_EQ(0, proc.wait().exitStatus());
88 TEST(SimpleSubprocessTest, ShellExitsWithError) {
89 Subprocess proc("false");
90 EXPECT_EQ(1, proc.wait().exitStatus());
93 TEST(SimpleSubprocessTest, ChangeChildDirectorySuccessfully) {
94 // The filesystem root normally lacks a 'true' binary
95 EXPECT_EQ(0, chdir("/"));
96 EXPECT_SPAWN_ERROR(ENOENT, "failed to execute ./true", "./true");
97 // The child can fix that by moving to /bin before exec().
98 Subprocess proc("./true", Subprocess::Options().chdir("/bin"));
99 EXPECT_EQ(0, proc.wait().exitStatus());
102 TEST(SimpleSubprocessTest, ChangeChildDirectoryWithError) {
105 std::vector<std::string>{"/bin/true"},
106 Subprocess::Options().chdir("/usually/this/is/not/a/valid/directory/")
108 ADD_FAILURE() << "expected to fail when changing the child's directory";
109 } catch (const SubprocessSpawnError& ex) {
110 EXPECT_EQ(ENOENT, ex.errnoValue());
111 const std::string expectedError =
112 "error preparing to execute /bin/true: No such file or directory";
113 if (StringPiece(ex.what()).find(expectedError) == StringPiece::npos) {
114 ADD_FAILURE() << "failed to find \"" << expectedError <<
115 "\" in exception: \"" << ex.what() << "\"";
121 boost::container::flat_set<int> getOpenFds() {
123 auto dirname = to<std::string>("/proc/", pid, "/fd");
125 boost::container::flat_set<int> fds;
126 for (fs::directory_iterator it(dirname);
127 it != fs::directory_iterator();
129 int fd = to<int>(it->path().filename().native());
135 template<class Runnable>
136 void checkFdLeak(const Runnable& r) {
137 // Get the currently open fds. Check that they are the same both before and
138 // after calling the specified function. We read the open fds from /proc.
139 // (If we wanted to work even on systems that don't have /proc, we could
140 // perhaps create and immediately close a socket both before and after
141 // running the function, and make sure we got the same fd number both times.)
142 auto fdsBefore = getOpenFds();
144 auto fdsAfter = getOpenFds();
145 EXPECT_EQ(fdsAfter.size(), fdsBefore.size());
149 // Make sure Subprocess doesn't leak any file descriptors
150 TEST(SimpleSubprocessTest, FdLeakTest) {
153 Subprocess proc("true");
154 EXPECT_EQ(0, proc.wait().exitStatus());
156 // Normal execution with pipes
158 Subprocess proc("echo foo; echo bar >&2",
159 Subprocess::pipeStdout() | Subprocess::pipeStderr());
160 auto p = proc.communicate();
161 EXPECT_EQ("foo\n", p.first);
162 EXPECT_EQ("bar\n", p.second);
166 // Test where the exec call fails()
168 EXPECT_SPAWN_ERROR(ENOENT, "failed to execute", "/no/such/file");
170 // Test where the exec call fails() with pipes
173 Subprocess proc(std::vector<std::string>({"/no/such/file"}),
174 Subprocess::pipeStdout().stderr(Subprocess::PIPE));
175 ADD_FAILURE() << "expected an error when running /no/such/file";
176 } catch (const SubprocessSpawnError& ex) {
177 EXPECT_EQ(ENOENT, ex.errnoValue());
182 TEST(ParentDeathSubprocessTest, ParentDeathSignal) {
183 // Find out where we are.
184 static constexpr size_t pathLength = 2048;
185 char buf[pathLength + 1];
186 int r = readlink("/proc/self/exe", buf, pathLength);
190 fs::path helper(buf);
191 helper.remove_filename();
192 helper /= "subprocess_test_parent_death_helper";
194 fs::path tempFile(fs::temp_directory_path() / fs::unique_path());
196 std::vector<std::string> args {helper.string(), tempFile.string()};
197 Subprocess proc(args);
198 // The helper gets killed by its child, see details in
199 // SubprocessTestParentDeathHelper.cpp
200 ASSERT_EQ(SIGKILL, proc.wait().killSignal());
202 // Now wait for the file to be created, see details in
203 // SubprocessTestParentDeathHelper.cpp
204 while (!fs::exists(tempFile)) {
205 usleep(20000); // 20ms
208 fs::remove(tempFile);
211 TEST(PopenSubprocessTest, PopenRead) {
212 Subprocess proc("ls /", Subprocess::pipeStdout());
214 gen::byLine(File(proc.stdout())) |
215 [&] (StringPiece line) {
216 if (line == "etc" || line == "bin" || line == "usr") {
224 TEST(CommunicateSubprocessTest, SimpleRead) {
225 Subprocess proc(std::vector<std::string>{ "/bin/echo", "-n", "foo", "bar"},
226 Subprocess::pipeStdout());
227 auto p = proc.communicate();
228 EXPECT_EQ("foo bar", p.first);
232 TEST(CommunicateSubprocessTest, BigWrite) {
233 const int numLines = 1 << 20;
234 std::string line("hello\n");
236 data.reserve(numLines * line.size());
237 for (int i = 0; i < numLines; ++i) {
241 Subprocess proc("wc -l", Subprocess::pipeStdin() | Subprocess::pipeStdout());
242 auto p = proc.communicate(data);
243 EXPECT_EQ(folly::format("{}\n", numLines).str(), p.first);
247 TEST(CommunicateSubprocessTest, Duplex) {
248 // Take 10MB of data and pass them through a filter.
249 // One line, as tr is line-buffered
250 const int bytes = 10 << 20;
251 std::string line(bytes, 'x');
253 Subprocess proc("tr a-z A-Z",
254 Subprocess::pipeStdin() | Subprocess::pipeStdout());
255 auto p = proc.communicate(line);
256 EXPECT_EQ(bytes, p.first.size());
257 EXPECT_EQ(std::string::npos, p.first.find_first_not_of('X'));
261 TEST(CommunicateSubprocessTest, Duplex2) {
263 // Pipe 200,000 lines through sed
264 const size_t numCopies = 100000;
265 auto iobuf = IOBuf::copyBuffer("this is a test\nanother line\n");
267 for (int n = 0; n < numCopies; ++n) {
268 input.append(iobuf->clone());
271 std::vector<std::string> cmd({
273 "-e", "s/a test/a successful test/",
274 "-e", "/^another line/w/dev/stderr",
276 auto options = Subprocess::pipeStdin().pipeStdout().pipeStderr().usePath();
277 Subprocess proc(cmd, options);
278 auto out = proc.communicateIOBuf(std::move(input));
281 // Convert stdout and stderr to strings so we can call split() on them.
283 if (out.first.front()) {
284 stdoutStr = out.first.move()->moveToFbString();
287 if (out.second.front()) {
288 stderrStr = out.second.move()->moveToFbString();
291 // stdout should be a copy of stdin, with "a test" replaced by
292 // "a successful test"
293 std::vector<StringPiece> stdoutLines;
294 split('\n', stdoutStr, stdoutLines);
295 EXPECT_EQ(numCopies * 2 + 1, stdoutLines.size());
296 // Strip off the trailing empty line
297 if (!stdoutLines.empty()) {
298 EXPECT_EQ("", stdoutLines.back());
299 stdoutLines.pop_back();
302 for (const auto& line : stdoutLines) {
303 if ((linenum & 1) == 0) {
304 EXPECT_EQ("this is a successful test", line);
306 EXPECT_EQ("another line", line);
311 // stderr should only contain the lines containing "another line"
312 std::vector<StringPiece> stderrLines;
313 split('\n', stderrStr, stderrLines);
314 EXPECT_EQ(numCopies + 1, stderrLines.size());
315 // Strip off the trailing empty line
316 if (!stderrLines.empty()) {
317 EXPECT_EQ("", stderrLines.back());
318 stderrLines.pop_back();
320 for (const auto& line : stderrLines) {
321 EXPECT_EQ("another line", line);
328 bool readToString(int fd, std::string& buf, size_t maxSize) {
329 size_t bytesRead = 0;
332 char* dest = &buf.front();
333 size_t remaining = maxSize;
337 n = ::read(fd, dest, remaining);
339 if (errno == EINTR) {
342 if (errno == EAGAIN) {
345 PCHECK("read failed");
353 buf.resize(dest - buf.data());
359 TEST(CommunicateSubprocessTest, Chatty) {
361 const int lineCount = 1000;
366 auto options = Subprocess::pipeStdin().pipeStdout().pipeStderr().usePath();
367 std::vector<std::string> cmd {
371 "s/a test/a successful test/",
374 Subprocess proc(cmd, options);
376 auto writeCallback = [&] (int pfd, int cfd) -> bool {
377 EXPECT_EQ(0, cfd); // child stdin
378 EXPECT_EQ(rcount, wcount); // chatty, one read for every write
380 auto msg = folly::to<std::string>("a test ", wcount, "\n");
382 // Not entirely kosher, we should handle partial writes, but this is
383 // fine for writes <= PIPE_BUF
384 EXPECT_EQ(msg.size(), writeFull(pfd, msg.data(), msg.size()));
387 proc.enableNotifications(0, false);
389 return (wcount == lineCount);
392 bool eofSeen = false;
394 auto readCallback = [&] (int pfd, int cfd) -> bool {
399 EXPECT_TRUE(readToString(pfd, lineBuf, 1));
400 EXPECT_EQ(0, lineBuf.size());
404 EXPECT_FALSE(eofSeen);
406 std::string expected;
408 if (rcount < lineCount) {
409 expected = folly::to<std::string>("a successful test ", rcount++, "\n");
412 EXPECT_EQ(wcount, rcount);
414 // Not entirely kosher, we should handle partial reads, but this is
415 // fine for reads <= PIPE_BUF
416 bool atEof = readToString(pfd, lineBuf, expected.size() + 1);
418 // EOF only expected after we finished reading
419 EXPECT_EQ(lineCount, rcount);
423 EXPECT_EQ(expected, lineBuf);
425 if (wcount != lineCount) { // still more to write...
426 proc.enableNotifications(0, true);
432 proc.communicate(readCallback, writeCallback);
434 EXPECT_EQ(lineCount, wcount);
435 EXPECT_EQ(lineCount, rcount);
436 EXPECT_TRUE(eofSeen);
438 EXPECT_EQ(0, proc.wait().exitStatus());
442 TEST(ConcurrentSubprocessTest, construction) {
443 std::vector<std::unique_ptr<Subprocess>> ps(100);
444 auto action = [](std::unique_ptr<Subprocess>& p) {
445 p = make_unique<Subprocess>("read", Subprocess::pipeStdout());
447 std::vector<std::thread> threads;
449 threads.emplace_back(action, ref(p));
451 for (auto& t : threads) {