allow to mock stderr output of test step commands

src/lib.rs

@@ -25,7 +25,7 @@ use crate::recorder::{hole_recorder::run_against_tests, Recorder};
 use crate::test_checker::executable_mock;
 use crate::test_spec::yaml::write_yaml;
 use crate::test_spec::Tests;
-use crate::tracer::stdio_redirecting::CaptureStderr;
+use crate::tracer::stdio_redirecting::Capture;
 use crate::tracer::Tracer;
 use std::collections::HashMap;
 use std::path::Path;
@@ -107,6 +107,7 @@ mod run_main {
             &context,
             executable_mock::Config {
                 stdout: b"foo".to_vec(),
+                stderr: vec![],
                 exitcode: 0,
             },
         )?;
@@ -140,7 +141,10 @@ fn print_recorded_test(context: &Context, program: &Path) -> R<ExitCode> {
         program,
         vec![],
         HashMap::new(),
-        CaptureStderr::NoCapture,
+        Capture {
+            stdout: false,
+            stderr: false,
+        },
         Recorder::empty(),
     )?;
     write_yaml(&mut *context.stdout(), &Tests::new(vec![test]).serialize()?)?;

src/recorder/mod.rs

@@ -65,6 +65,7 @@ impl SyscallMock for Recorder {
             self.test.steps.push_back(Step {
                 command_matcher: CommandMatcher::ExactMatch(command),
                 stdout: vec![],
+                stderr: vec![],
                 exitcode,
             });
         }

src/recorder/result.rs

@@ -5,7 +5,7 @@ use crate::test_checker::{
     TestChecker,
 };
 use crate::test_spec::{yaml::write_yaml, Test, Tests};
-use crate::tracer::stdio_redirecting::CaptureStderr;
+use crate::tracer::stdio_redirecting::Capture;
 use crate::tracer::Tracer;
 use crate::{ExitCode, R};
 use std::fs::OpenOptions;
@@ -128,10 +128,9 @@ fn run_against_test(
                 program,
                 test.arguments.clone(),
                 test.env.clone(),
-                if test.stderr.is_some() {
-                    CaptureStderr::Capture
-                } else {
-                    CaptureStderr::NoCapture
+                Capture {
+                    stdout: test.stdout.is_some(),
+                    stderr: test.stderr.is_some(),
                 },
                 $syscall_mock,
             )

src/test_checker/executable_mock.rs

@@ -9,6 +9,7 @@ use std::path::Path;
 #[derive(Debug, Serialize, Deserialize)]
 pub struct Config {
     pub stdout: Vec<u8>,
+    pub stderr: Vec<u8>,
     pub exitcode: i32,
 }
 
@@ -35,6 +36,7 @@ pub fn create_mock_executable(context: &Context, config: Config) -> R<Vec<u8>> {
 pub fn run(context: &Context, executable_mock_path: &Path) -> R<ExitCode> {
     let config: Config = deserialize(&skip_hashbang_line(fs::read(executable_mock_path)?))?;
     context.stdout().write_all(&config.stdout)?;
+    context.stderr().write_all(&config.stderr)?;
     Ok(ExitCode(config.exitcode))
 }
 
@@ -60,6 +62,7 @@ mod create_mock_executable {
             &Context::new_mock(),
             Config {
                 stdout: b"foo".to_vec(),
+                stderr: vec![],
                 exitcode: 0,
             },
         )?)?;
@@ -68,12 +71,28 @@ mod create_mock_executable {
         Ok(())
     }
 
+    #[test]
+    fn renders_an_executable_that_outputs_the_given_stderr() -> R<()> {
+        let mock_executable = TempFile::write_temp_script(&create_mock_executable(
+            &Context::new_mock(),
+            Config {
+                stdout: vec![],
+                stderr: b"foo".to_vec(),
+                exitcode: 0,
+            },
+        )?)?;
+        let output = Command::new(mock_executable.path()).output()?;
+        assert_eq!(output.stderr, b"foo");
+        Ok(())
+    }
+
     #[test]
     fn renders_an_executable_that_exits_with_the_given_exitcode() -> R<()> {
         let mock_executable = TempFile::write_temp_script(&create_mock_executable(
             &Context::new_mock(),
             Config {
                 stdout: b"foo".to_vec(),
+                stderr: vec![],
                 exitcode: 42,
             },
         )?)?;
@@ -92,6 +111,7 @@ mod create_mock_executable {
                 &context,
                 Config {
                     stdout: vec![],
+                    stderr: vec![],
                     exitcode: 42,
                 },
             ),

src/test_checker/mod.rs

@@ -4,7 +4,7 @@ pub mod executable_mock;
 use crate::context::Context;
 use crate::test_spec;
 use crate::test_spec::Test;
-use crate::tracer::stdio_redirecting::Redirector;
+use crate::tracer::stdio_redirecting::{Redirect, Redirector};
 use crate::tracer::{tracee_memory, SyscallMock};
 use crate::utils::short_temp_files::ShortTempFile;
 use crate::R;
@@ -39,6 +39,7 @@ impl TestChecker {
     fn allow_failing_scripts_to_continue() -> executable_mock::Config {
         executable_mock::Config {
             stdout: vec![],
+            stderr: vec![],
             exitcode: 0,
         }
     }
@@ -54,6 +55,7 @@ impl TestChecker {
                 }
                 executable_mock::Config {
                     stdout: next_test_step.stdout,
+                    stderr: next_test_step.stderr,
                     exitcode: next_test_step.exitcode,
                 }
             }
@@ -70,6 +72,28 @@ impl TestChecker {
         Ok(path)
     }
 
+    fn check_expected_output_stream(&mut self, redirect: &Redirect, expected: Vec<u8>) -> R<()> {
+        match redirect.captured()? {
+            None => panic!(
+                "scriptkeeper bug: {} expected, but not captured",
+                redirect.stream_type
+            ),
+            Some(captured) => {
+                if captured != expected {
+                    self.register_error(format!(
+                        "  expected output to {}: {:?}\
+                         \n  received output to {}: {:?}\n",
+                        redirect.stream_type,
+                        String::from_utf8_lossy(&expected).as_ref(),
+                        redirect.stream_type,
+                        String::from_utf8_lossy(&captured).as_ref(),
+                    ));
+                }
+            }
+        }
+        Ok(())
+    }
+
     fn register_step_error(&mut self, expected: &str, received: &str) {
         self.register_error(format!(
             "  expected: {}\n  received: {}\n",
@@ -163,20 +187,11 @@ impl SyscallMock for TestChecker {
                 &format!("<exitcode {}>", exitcode),
             );
         }
+        if let Some(expected_stdout) = &self.test.stdout {
+            self.check_expected_output_stream(&redirector.stdout, expected_stdout.clone())?;
+        }
         if let Some(expected_stderr) = &self.test.stderr {
-            match redirector.stderr.captured()? {
-                None => panic!("scriptkeeper bug: stderr expected, but not captured"),
-                Some(captured_stderr) => {
-                    if &captured_stderr != expected_stderr {
-                        self.register_error(format!(
-                            "  expected output to stderr: {:?}\
-                             \n  received output to stderr: {:?}\n",
-                            String::from_utf8_lossy(&expected_stderr).as_ref(),
-                            String::from_utf8_lossy(&captured_stderr).as_ref(),
-                        ));
-                    }
-                }
-            }
+            self.check_expected_output_stream(&redirector.stderr, expected_stderr.clone())?;
         }
         Ok(self.result)
     }

src/test_spec/mod.rs

@@ -24,6 +24,7 @@ use yaml_rust::{yaml::Hash, Yaml, YamlLoader};
 pub struct Step {
     pub command_matcher: CommandMatcher,
     pub stdout: Vec<u8>,
+    pub stderr: Vec<u8>,
     pub exitcode: i32,
 }
 
@@ -32,6 +33,7 @@ impl Step {
         Step {
             command_matcher,
             stdout: vec![],
+            stderr: vec![],
             exitcode: 0,
         }
     }
@@ -49,7 +51,14 @@ impl Step {
 
     fn add_stdout(&mut self, object: &Hash) -> R<()> {
         if let Ok(stdout) = object.expect_field("stdout") {
-            self.stdout = stdout.expect_str()?.as_bytes().to_vec();
+            self.stdout = stdout.expect_bytes()?;
+        }
+        Ok(())
+    }
+
+    fn add_stderr(&mut self, object: &Hash) -> R<()> {
+        if let Ok(stderr) = object.expect_field("stderr") {
+            self.stderr = stderr.expect_bytes()?;
         }
         Ok(())
     }
@@ -58,7 +67,10 @@ impl Step {
         match yaml {
             Yaml::String(string) => Step::from_string(string),
             Yaml::Hash(object) => {
-                check_keys(&["command", "stdout", "exitcode", "regex"], object)?;
+                check_keys(
+                    &["command", "stdout", "stderr", "exitcode", "regex"],
+                    object,
+                )?;
                 let mut step = match (object.expect_field("command"), object.expect_field("regex"))
                 {
                     (Ok(command_field), Err(_)) => Step::from_string(command_field.expect_str()?)?,
@@ -68,6 +80,7 @@ impl Step {
                     _ => Err("please provide either a 'command' or 'regex' field but not both")?,
                 };
                 step.add_stdout(object)?;
+                step.add_stderr(object)?;
                 step.add_exitcode(object)?;
                 Ok(step)
             }
@@ -169,6 +182,15 @@ mod parse_step {
         Ok(())
     }
 
+    #[test]
+    fn allows_to_specify_stderr() -> R<()> {
+        assert_eq!(
+            test_parse_step(r#"{command: "foo", stderr: "bar"}"#)?.stderr,
+            b"bar".to_vec(),
+        );
+        Ok(())
+    }
+
     mod exitcode {
         use super::*;
 
@@ -196,6 +218,7 @@ pub struct Test {
     pub arguments: Vec<String>,
     pub env: HashMap<String, String>,
     pub cwd: Option<PathBuf>,
+    pub stdout: Option<Vec<u8>>,
     pub stderr: Option<Vec<u8>>,
     pub exitcode: Option<i32>,
     pub mocked_files: Vec<PathBuf>,
@@ -214,9 +237,10 @@ impl Test {
             arguments: vec![],
             env: HashMap::new(),
             cwd: None,
+            stdout: None,
+            stderr: None,
             exitcode: None,
             mocked_files: vec![],
-            stderr: None,
         }
     }
 
@@ -281,9 +305,16 @@ impl Test {
         Ok(())
     }
 
+    fn add_stdout(&mut self, object: &Hash) -> R<()> {
+        if let Ok(stdout) = object.expect_field("stdout") {
+            self.stdout = Some(stdout.expect_bytes()?);
+        }
+        Ok(())
+    }
+
     fn add_stderr(&mut self, object: &Hash) -> R<()> {
         if let Ok(stderr) = object.expect_field("stderr") {
-            self.stderr = Some(stderr.expect_str()?.as_bytes().to_vec());
+            self.stderr = Some(stderr.expect_bytes()?);
         }
         Ok(())
     }
@@ -312,6 +343,7 @@ impl Test {
                 "arguments",
                 "env",
                 "exitcode",
+                "stdout",
                 "stderr",
                 "cwd",
             ],
@@ -321,6 +353,7 @@ impl Test {
         test.add_arguments(&object)?;
         test.add_env(&object)?;
         test.add_cwd(&object)?;
+        test.add_stdout(&object)?;
         test.add_stderr(&object)?;
         test.add_exitcode(&object)?;
         test.add_mocked_files(&object)?;
@@ -575,7 +608,7 @@ mod load {
                      unexpected field 'foo', \
                      possible values: \
                      'steps', 'mockedFiles', 'arguments', 'env', \
-                     'exitcode', 'stderr', 'cwd'",
+                     'exitcode', 'stdout', 'stderr', 'cwd'",
                     path_to_string(&tempfile.path())?
                 )
             );
@@ -598,7 +631,7 @@ mod load {
                 format!(
                     "error in {}.test.yaml: \
                      unexpected field 'foo', \
-                     possible values: 'command', 'stdout', 'exitcode', 'regex'",
+                     possible values: 'command', 'stdout', 'stderr', 'exitcode', 'regex'",
                     path_to_string(&tempfile.path())?
                 )
             );
@@ -995,6 +1028,41 @@ mod load {
         Ok(())
     }
 
+    mod expected_stdout {
+        use super::*;
+        use pretty_assertions::assert_eq;
+
+        #[test]
+        fn allows_to_specify_the_expected_stdout() -> R<()> {
+            assert_eq!(
+                test_parse_one(
+                    r"
+                        |- steps: []
+                        |  stdout: foo
+                    "
+                )?
+                .stdout
+                .map(|s| String::from_utf8(s).unwrap()),
+                Some("foo".to_string())
+            );
+            Ok(())
+        }
+
+        #[test]
+        fn none_is_the_default() -> R<()> {
+            assert_eq!(
+                test_parse_one(
+                    r"
+                        |- steps: []
+                    "
+                )?
+                .stdout,
+                None
+            );
+            Ok(())
+        }
+    }
+
     mod expected_stderr {
         use super::*;
         use pretty_assertions::assert_eq;
@@ -1213,6 +1281,7 @@ mod serialize {
         let test = Test::new(vec![Step {
             command_matcher: CommandMatcher::ExactMatch(Command::new("cp")?),
             stdout: vec![],
+            stderr: vec![],
             exitcode: 42,
         }]);
         roundtrip(Tests::new(vec![test]))