From 6ffa3bb895638a406404cb0efaa8e3710c48d61f Mon Sep 17 00:00:00 2001 From: Marek Kubica Date: Wed, 26 Jan 2022 16:41:47 +0100 Subject: [PATCH 1/6] Test case to see whether non-deterministic mode is applied correctly --- .../mdx-dune-gen/misc/non-deterministic/dune | 52 +++++++++++++++++++ .../dune-mdx-nondeterministic | 18 +++++++ .../dune-mdx-nondeterministic.expected | 18 +++++++ 3 files changed, 88 insertions(+) create mode 100644 test/bin/mdx-dune-gen/misc/non-deterministic/dune create mode 100644 test/bin/mdx-dune-gen/misc/non-deterministic/dune-mdx-nondeterministic create mode 100644 test/bin/mdx-dune-gen/misc/non-deterministic/dune-mdx-nondeterministic.expected diff --git a/test/bin/mdx-dune-gen/misc/non-deterministic/dune b/test/bin/mdx-dune-gen/misc/non-deterministic/dune new file mode 100644 index 000000000..a2c725e81 --- /dev/null +++ b/test/bin/mdx-dune-gen/misc/non-deterministic/dune @@ -0,0 +1,52 @@ +(rule + (target dune_gen.ml) + (action + (with-stdout-to + %{target} + (run ocaml-mdx dune-gen)))) + +(executable + (name dune_gen) + (modules dune_gen) + (modes byte) + (libraries mdx.test)) + +(rule + (target dune-mdx-nondeterministic.deterministic) + (deps + dune_gen.exe + (:input dune-mdx-nondeterministic)) + (action + (with-stdout-to + %{target} + (run ./dune_gen.exe %{input})))) + +(rule + (target dune-mdx-nondeterministic.nondeterministic) + (deps + dune_gen.exe + (:input dune-mdx-nondeterministic)) + (action + (setenv + MDX_RUN_NON_DETERMINISTIC + 1 + (with-stdout-to + %{target} + (run ./dune_gen.exe %{input}))))) + +(rule + (alias runtest) + (action + (diff %{dep:dune-mdx-nondeterministic.expected} + %{dep:dune-mdx-nondeterministic.deterministic}))) + +;; make sure the non-deterministic is different from the deterministic + +(rule + (alias runtest) + (action + (with-accepted-exit-codes + 1 + (ignore-stdout + (run diff %{dep:dune-mdx-nondeterministic.expected} + %{dep:dune-mdx-nondeterministic.nondeterministic}))))) diff --git a/test/bin/mdx-dune-gen/misc/non-deterministic/dune-mdx-nondeterministic b/test/bin/mdx-dune-gen/misc/non-deterministic/dune-mdx-nondeterministic new file mode 100644 index 000000000..76dcef695 --- /dev/null +++ b/test/bin/mdx-dune-gen/misc/non-deterministic/dune-mdx-nondeterministic @@ -0,0 +1,18 @@ +This test checks whether the non-deterministic mode works with the `dune` `mdx` +stanza. + +Deterministic stanzas should get run and corrected, as for 1 plus one is not 3: + +```ocaml +# 1 + 1;; +- : int = 42 +``` + +Non-deterministic ones should not be updated, since whatever `Random` outputs +should be random: + + +```ocaml +# Random.int 1000;; +- : int = 42 +``` diff --git a/test/bin/mdx-dune-gen/misc/non-deterministic/dune-mdx-nondeterministic.expected b/test/bin/mdx-dune-gen/misc/non-deterministic/dune-mdx-nondeterministic.expected new file mode 100644 index 000000000..37c997232 --- /dev/null +++ b/test/bin/mdx-dune-gen/misc/non-deterministic/dune-mdx-nondeterministic.expected @@ -0,0 +1,18 @@ +This test checks whether the non-deterministic mode works with the `dune` `mdx` +stanza. + +Deterministic stanzas should get run and corrected, as for 1 plus one is not 3: + +```ocaml +# 1 + 1;; +- : int = 2 +``` + +Non-deterministic ones should not be updated, since whatever `Random` outputs +should be random: + + +```ocaml +# Random.int 1000;; +- : int = 42 +``` From cb283d1434e9c8a2e3d739f56aad7e11e9d08911 Mon Sep 17 00:00:00 2001 From: Marek Kubica Date: Thu, 27 Jan 2022 15:03:21 +0100 Subject: [PATCH 2/6] Run non-deterministic tests depending on `MDX_RUN_NON_DETERMINISTIC` --- bin/dune_gen.ml | 7 ++++++- test/bin/mdx-dune-gen/misc/basic/dune.gen.expected | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/bin/dune_gen.ml b/bin/dune_gen.ml index 37a38668f..eac4249da 100644 --- a/bin/dune_gen.ml +++ b/bin/dune_gen.ml @@ -32,8 +32,13 @@ let run (`Setup ()) (`Prelude prelude) (`Directories dirs) = line " ]"; line " in"; line " let predicates = Predicate.[ byte; toploop ] in"; + line " let non_deterministic ="; + line " match Sys.getenv_opt \"MDX_RUN_NON_DETERMINISTIC\" with"; + line " | Some _ -> true"; + line " | None -> false"; + line " in"; line " run_exn ~packages ~predicates ~prelude_str:[]"; - line " ~non_deterministic:false"; + line " ~non_deterministic"; line " ~silent_eval:false ~record_backtrace:false"; line " ~syntax:None ~silent:false"; line " ~verbose_findlib:false ~section:None"; diff --git a/test/bin/mdx-dune-gen/misc/basic/dune.gen.expected b/test/bin/mdx-dune-gen/misc/basic/dune.gen.expected index fe37309a6..90973e682 100644 --- a/test/bin/mdx-dune-gen/misc/basic/dune.gen.expected +++ b/test/bin/mdx-dune-gen/misc/basic/dune.gen.expected @@ -9,8 +9,13 @@ let run_exn_defaults = ] in let predicates = Predicate.[ byte; toploop ] in + let non_deterministic = + match Sys.getenv_opt "MDX_RUN_NON_DETERMINISTIC" with + | Some _ -> true + | None -> false + in run_exn ~packages ~predicates ~prelude_str:[] - ~non_deterministic:false + ~non_deterministic ~silent_eval:false ~record_backtrace:false ~syntax:None ~silent:false ~verbose_findlib:false ~section:None From 3c8a9db0a18a55acf528154d161bf875d2c7e95c Mon Sep 17 00:00:00 2001 From: Marek Kubica Date: Thu, 27 Jan 2022 15:07:24 +0100 Subject: [PATCH 3/6] Add changelog entry --- CHANGES.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index c4d2d94f3..5b69c03ab 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,9 @@ - Add support for adding language tags and metadata labels in `mli` files. (#339, #357, @julow, @Leonidas-from-XIV) +- Add support for running non-deterministic tests in `dune` MDX 0.2 stanza by + setting the `MDX_RUN_NON_DETERMINISTIC` environment variable. (#365, + #366, @Leonidas-from-XIV) #### Changed From a83ced28618dffa49eada051b015f049c89c2f6f Mon Sep 17 00:00:00 2001 From: Marek Kubica Date: Thu, 27 Jan 2022 16:41:27 +0100 Subject: [PATCH 4/6] Simplify test build Thanks to @emillon for pointing out all the shortcut that dune allows to avoid declaring things over and over. --- .../mdx-dune-gen/misc/non-deterministic/dune | 40 +++++++------------ 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/test/bin/mdx-dune-gen/misc/non-deterministic/dune b/test/bin/mdx-dune-gen/misc/non-deterministic/dune index a2c725e81..723fddd88 100644 --- a/test/bin/mdx-dune-gen/misc/non-deterministic/dune +++ b/test/bin/mdx-dune-gen/misc/non-deterministic/dune @@ -1,9 +1,7 @@ (rule - (target dune_gen.ml) - (action - (with-stdout-to - %{target} - (run ocaml-mdx dune-gen)))) + (with-stdout-to + dune_gen.ml + (run ocaml-mdx dune-gen))) (executable (name dune_gen) @@ -12,33 +10,23 @@ (libraries mdx.test)) (rule - (target dune-mdx-nondeterministic.deterministic) - (deps - dune_gen.exe - (:input dune-mdx-nondeterministic)) - (action - (with-stdout-to - %{target} - (run ./dune_gen.exe %{input})))) + (with-stdout-to + dune-mdx-nondeterministic.deterministic + (run ./dune_gen.exe %{dep:dune-mdx-nondeterministic}))) (rule - (target dune-mdx-nondeterministic.nondeterministic) - (deps - dune_gen.exe - (:input dune-mdx-nondeterministic)) - (action - (setenv - MDX_RUN_NON_DETERMINISTIC - 1 - (with-stdout-to - %{target} - (run ./dune_gen.exe %{input}))))) + (setenv + MDX_RUN_NON_DETERMINISTIC + 1 + (with-stdout-to + dune-mdx-nondeterministic.nondeterministic + (run ./dune_gen.exe %{dep:dune-mdx-nondeterministic})))) (rule (alias runtest) (action - (diff %{dep:dune-mdx-nondeterministic.expected} - %{dep:dune-mdx-nondeterministic.deterministic}))) + (diff dune-mdx-nondeterministic.expected + dune-mdx-nondeterministic.deterministic))) ;; make sure the non-deterministic is different from the deterministic From 45e2468b0047a00941067fd2bb27e24a84e7cba4 Mon Sep 17 00:00:00 2001 From: Marek Kubica Date: Thu, 27 Jan 2022 17:23:13 +0100 Subject: [PATCH 5/6] Replace shelling out to `diff` by helper program This should be more portable since we don't depend on `diff` being installed or have an exit code of 1 when the files differ. --- .../misc/non-deterministic/different.ml | 38 +++++++++++++++++++ .../mdx-dune-gen/misc/non-deterministic/dune | 11 +++--- 2 files changed, 44 insertions(+), 5 deletions(-) create mode 100644 test/bin/mdx-dune-gen/misc/non-deterministic/different.ml diff --git a/test/bin/mdx-dune-gen/misc/non-deterministic/different.ml b/test/bin/mdx-dune-gen/misc/non-deterministic/different.ml new file mode 100644 index 000000000..fb17a94e6 --- /dev/null +++ b/test/bin/mdx-dune-gen/misc/non-deterministic/different.ml @@ -0,0 +1,38 @@ +(* small helper to determine whether two files differ *) + +let main () = + let first = Sys.argv.(1) |> open_in in + let second = Sys.argv.(2) |> open_in in + let rec loop () = + match input_line first with + | first_line -> ( + match input_line second with + | second_line -> + match String.equal first_line second_line with + | true -> loop () + | false -> + (* we found a difference between the lines *) + exit 0 + | exception End_of_file -> + (* the second file ended before the first *) + exit 0 + ) + | exception End_of_file -> + (* the first file ended first *) + match input_line second with + | _ -> + (* the second file continues: a difference *) + exit 1 + | exception End_of_file -> + (* the second file ended too *) + () + in + loop (); + close_in first; + close_in second; + (* we didn't find a difference, exit with a failure code *) + prerr_endline "The files appear to be identical"; + exit 1 + +let () = + main () diff --git a/test/bin/mdx-dune-gen/misc/non-deterministic/dune b/test/bin/mdx-dune-gen/misc/non-deterministic/dune index 723fddd88..c697b7358 100644 --- a/test/bin/mdx-dune-gen/misc/non-deterministic/dune +++ b/test/bin/mdx-dune-gen/misc/non-deterministic/dune @@ -30,11 +30,12 @@ ;; make sure the non-deterministic is different from the deterministic +(executable + (name different) + (modules different)) + (rule (alias runtest) (action - (with-accepted-exit-codes - 1 - (ignore-stdout - (run diff %{dep:dune-mdx-nondeterministic.expected} - %{dep:dune-mdx-nondeterministic.nondeterministic}))))) + (run ./different.exe %{dep:dune-mdx-nondeterministic.expected} + %{dep:dune-mdx-nondeterministic.nondeterministic}))) From 122e46b534ea62200739d3ee598a27f61821bd8c Mon Sep 17 00:00:00 2001 From: Marek Kubica Date: Thu, 27 Jan 2022 17:29:52 +0100 Subject: [PATCH 6/6] Make the helper a bit simpler to read and close handles properly --- .../misc/non-deterministic/different.ml | 60 ++++++++++--------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/test/bin/mdx-dune-gen/misc/non-deterministic/different.ml b/test/bin/mdx-dune-gen/misc/non-deterministic/different.ml index fb17a94e6..ca6b2ed00 100644 --- a/test/bin/mdx-dune-gen/misc/non-deterministic/different.ml +++ b/test/bin/mdx-dune-gen/misc/non-deterministic/different.ml @@ -1,38 +1,42 @@ (* small helper to determine whether two files differ *) +type comparison = Same | Different + +let rec compare first second = + match input_line first with + | first_line -> ( + match input_line second with + | second_line -> + match String.equal first_line second_line with + | true -> compare first second + | false -> + (* we found a difference between the lines *) + Different + | exception End_of_file -> + (* the second file ended before the first *) + Different) + | exception End_of_file -> + (* the first file ended first *) + match input_line second with + | _ -> + (* the second file continues: a difference *) + Different + | exception End_of_file -> + (* the second file ended too *) + Same + let main () = let first = Sys.argv.(1) |> open_in in let second = Sys.argv.(2) |> open_in in - let rec loop () = - match input_line first with - | first_line -> ( - match input_line second with - | second_line -> - match String.equal first_line second_line with - | true -> loop () - | false -> - (* we found a difference between the lines *) - exit 0 - | exception End_of_file -> - (* the second file ended before the first *) - exit 0 - ) - | exception End_of_file -> - (* the first file ended first *) - match input_line second with - | _ -> - (* the second file continues: a difference *) - exit 1 - | exception End_of_file -> - (* the second file ended too *) - () - in - loop (); + let comparison = compare first second in close_in first; close_in second; - (* we didn't find a difference, exit with a failure code *) - prerr_endline "The files appear to be identical"; - exit 1 + match comparison with + | Same -> + prerr_endline "The files appear to be identical"; + (* we didn't find a difference, exit with a failure code *) + exit 1 + | Different -> () let () = main ()