Over time, running dune's tests has become slow. That is not unexpected, but it also diminishes developer productivity. I find myself pushing things to github and then checking up on the status in CI rather than running things locally. I think we should put in some effort in making the test suite faster. To start, here's a collection of the slowest 30 commands our tests execute:
[
{
"test": "_build/default/test/blackbox-tests/test-cases/melange/enabled-if/eif-melange-emit-name-collision.t",
"command": [
"dune build"
],
"real": 8.0899999999999999,
"user": 2.4500000000000002,
"system": 2.9460000000000002
},
{
"test": "_build/default/test/blackbox-tests/test-cases/melange/enabled-if/eif-melange-emit-name-collision-same-folder.t",
"command": [
"dune build"
],
"real": 7.9240000000000004,
"user": 2.4380000000000002,
"system": 2.9460000000000002
},
{
"test": "_build/default/test/blackbox-tests/test-cases/foreign-stubs/github3766.t/run.t",
"command": [
"./test.exe |",
"sed -e 's,^ls:.*foo/\\*\\.a.*No such file or directory.*$,ls: .../foo/*.a: No such file or directory,g' |",
"sed -e 's,^\\[[1-2]\\]$,[1/2],g'"
],
"real": 5.5019999999999998,
"user": 3.4670000000000001,
"system": 1.4319999999999999
},
{
"test": "_build/default/test/blackbox-tests/test-cases/melange/emit-with-runtime-deps-edge-cases.t",
"command": [
"dune build @mel"
],
"real": 3.4409999999999998,
"user": 1.5740000000000001,
"system": 1.052
},
{
"test": "_build/default/test/blackbox-tests/test-cases/melange/promote-in-source-public-lib.t",
"command": [
"dune build @dist"
],
"real": 3.4399999999999999,
"user": 1.1419999999999999,
"system": 1.405
},
{
"test": "_build/default/test/blackbox-tests/test-cases/melange/emit-select.t",
"command": [
"dune build @mel"
],
"real": 3.4260000000000002,
"user": 1.165,
"system": 1.611
},
{
"test": "_build/default/test/blackbox-tests/test-cases/stanzas/toplevel-stanza/preprocessors.t/run.t",
"command": [
"dune exec ./tt.exe -- -init init.ml | sed -E 's/OCaml version .*$/Ocaml version REDACTED/g'"
],
"real": 3.2229999999999999,
"user": 2.8849999999999998,
"system": 0.28299999999999997
},
{
"test": "_build/default/test/blackbox-tests/test-cases/mdx-stanza/new-deps-field.t/run.t",
"command": [
"dune runtest"
],
"real": 3.1859999999999999,
"user": 2.782,
"system": 0.214
},
{
"test": "_build/default/test/blackbox-tests/test-cases/mdx-stanza/shared-libraries.t",
"command": [
"dune runtest"
],
"real": 3.1190000000000002,
"user": 2.8300000000000001,
"system": 0.17100000000000001
},
{
"test": "_build/default/test/blackbox-tests/test-cases/odoc/odoc-new-sherlodoc.t/run.t",
"command": [
"dune build @doc-new"
],
"real": 3.0779999999999998,
"user": 1.7669999999999999,
"system": 0.95499999999999996
},
{
"test": "_build/default/test/blackbox-tests/test-cases/mdx-stanza/linked-libraries.t/run.t",
"command": [
"dune runtest"
],
"real": 3.0739999999999998,
"user": 2.7970000000000002,
"system": 0.16800000000000001
},
{
"test": "_build/default/test/blackbox-tests/test-cases/toplevel-plugin/toplevel-plugin-fail.t",
"command": [
"dune build @all 2>&1 | dune_cmd sanitize"
],
"real": 3.0089999999999999,
"user": 2.7210000000000001,
"system": 0.224
},
{
"test": "_build/default/test/blackbox-tests/test-cases/odoc/new/odoc-unique-mlds/diff-scope.t/run.t",
"command": [
"dune build @doc-new"
],
"real": 3.0089999999999999,
"user": 1.7250000000000001,
"system": 0.93899999999999995
},
{
"test": "_build/default/test/blackbox-tests/test-cases/mdx-stanza/env-variables.t",
"command": [
"dune runtest --auto-promote"
],
"real": 2.9969999999999999,
"user": 2.8109999999999999,
"system": 0.14599999999999999
},
{
"test": "_build/default/test/blackbox-tests/test-cases/mdx-stanza/locks.t",
"command": [
"dune build"
],
"real": 2.992,
"user": 2.7330000000000001,
"system": 0.13400000000000001
},
{
"test": "_build/default/test/blackbox-tests/test-cases/jsoo/jsoo-config-effects.t/run.t",
"command": [
"dune build bin/bin1.bc.js bin/bin2.bc.js bin/bin3.bc.js"
],
"real": 2.9750000000000001,
"user": 2.7080000000000002,
"system": 0.216
},
{
"test": "_build/default/test/blackbox-tests/test-cases/mdx-stanza/mld-files.t",
"command": [
"dune runtest"
],
"real": 2.9340000000000002,
"user": 2.6989999999999998,
"system": 0.153
},
{
"test": "_build/default/test/blackbox-tests/test-cases/mdx-stanza/paths.t",
"command": [
"dune runtest"
],
"real": 2.9300000000000002,
"user": 2.645,
"system": 0.16
},
{
"test": "_build/default/test/blackbox-tests/test-cases/toplevel-plugin/toplevel-plugin.t",
"command": [
"dune build @all 2>&1 | dune_cmd sanitize"
],
"real": 2.923,
"user": 2.6549999999999998,
"system": 0.222
},
{
"test": "_build/default/test/blackbox-tests/test-cases/mdx-stanza/github7077.t",
"command": [
"dune runtest"
],
"real": 2.8900000000000001,
"user": 2.7170000000000001,
"system": 0.14499999999999999
},
{
"test": "_build/default/test/blackbox-tests/test-cases/toplevel-plugin/toplevel-plugin-fail2.t",
"command": [
"dune build @all 2>&1 | dune_cmd sanitize"
],
"real": 2.883,
"user": 2.6419999999999999,
"system": 0.19700000000000001
},
{
"test": "_build/default/test/blackbox-tests/test-cases/mdx-stanza/implicit-transitive-deps-false.t",
"command": [
"dune build"
],
"real": 2.8109999999999999,
"user": 2.661,
"system": 0.14099999999999999
},
{
"test": "_build/default/test/blackbox-tests/test-cases/jsoo/jsoo-config.t/run.t",
"command": [
"dune build bin/bin1.bc.js bin/bin2.bc.js bin/bin3.bc.js"
],
"real": 2.7549999999999999,
"user": 2.504,
"system": 0.21199999999999999
},
{
"test": "_build/default/test/blackbox-tests/test-cases/odoc/new/odoc-simple.t/run.t",
"command": [
"dune build @doc-new"
],
"real": 2.6659999999999999,
"user": 1.595,
"system": 0.97999999999999998
},
{
"test": "_build/default/test/blackbox-tests/test-cases/mdx-stanza/paths.t",
"command": [
"dune runtest"
],
"real": 2.6230000000000002,
"user": 2.4140000000000001,
"system": 0.14099999999999999
},
{
"test": "_build/default/test/blackbox-tests/test-cases/odoc/new/multi-lib.t/run.t",
"command": [
"dune build @doc-new"
],
"real": 2.5299999999999998,
"user": 1.4930000000000001,
"system": 0.93600000000000005
},
{
"test": "_build/default/test/blackbox-tests/test-cases/odoc/new/transitive-compile-deps.t/run.t",
"command": [
"dune build @doc-new"
],
"real": 2.5009999999999999,
"user": 1.5269999999999999,
"system": 0.876
},
{
"test": "_build/default/test/blackbox-tests/test-cases/odoc/new/odoc-wrapped-lib.t/run.t",
"command": [
"dune build @doc-new"
],
"real": 2.4860000000000002,
"user": 1.5109999999999999,
"system": 0.86799999999999999
},
{
"test": "_build/default/test/blackbox-tests/test-cases/melange/copy-files-include-subdirs.t",
"command": [
"dune build @mel"
],
"real": 2.4449999999999998,
"user": 1.3180000000000001,
"system": 0.75900000000000001
},
{
"test": "_build/default/test/blackbox-tests/test-cases/odoc/new/github1645.t",
"command": [
"dune build @doc-new"
],
"real": 2.4220000000000002,
"user": 1.456,
"system": 0.85799999999999998
}
]I don't expect any miracles, but we should at least review these to see if this is all necessary.
Over time, running dune's tests has become slow. That is not unexpected, but it also diminishes developer productivity. I find myself pushing things to github and then checking up on the status in CI rather than running things locally. I think we should put in some effort in making the test suite faster. To start, here's a collection of the slowest 30 commands our tests execute:
[ { "test": "_build/default/test/blackbox-tests/test-cases/melange/enabled-if/eif-melange-emit-name-collision.t", "command": [ "dune build" ], "real": 8.0899999999999999, "user": 2.4500000000000002, "system": 2.9460000000000002 }, { "test": "_build/default/test/blackbox-tests/test-cases/melange/enabled-if/eif-melange-emit-name-collision-same-folder.t", "command": [ "dune build" ], "real": 7.9240000000000004, "user": 2.4380000000000002, "system": 2.9460000000000002 }, { "test": "_build/default/test/blackbox-tests/test-cases/foreign-stubs/github3766.t/run.t", "command": [ "./test.exe |", "sed -e 's,^ls:.*foo/\\*\\.a.*No such file or directory.*$,ls: .../foo/*.a: No such file or directory,g' |", "sed -e 's,^\\[[1-2]\\]$,[1/2],g'" ], "real": 5.5019999999999998, "user": 3.4670000000000001, "system": 1.4319999999999999 }, { "test": "_build/default/test/blackbox-tests/test-cases/melange/emit-with-runtime-deps-edge-cases.t", "command": [ "dune build @mel" ], "real": 3.4409999999999998, "user": 1.5740000000000001, "system": 1.052 }, { "test": "_build/default/test/blackbox-tests/test-cases/melange/promote-in-source-public-lib.t", "command": [ "dune build @dist" ], "real": 3.4399999999999999, "user": 1.1419999999999999, "system": 1.405 }, { "test": "_build/default/test/blackbox-tests/test-cases/melange/emit-select.t", "command": [ "dune build @mel" ], "real": 3.4260000000000002, "user": 1.165, "system": 1.611 }, { "test": "_build/default/test/blackbox-tests/test-cases/stanzas/toplevel-stanza/preprocessors.t/run.t", "command": [ "dune exec ./tt.exe -- -init init.ml | sed -E 's/OCaml version .*$/Ocaml version REDACTED/g'" ], "real": 3.2229999999999999, "user": 2.8849999999999998, "system": 0.28299999999999997 }, { "test": "_build/default/test/blackbox-tests/test-cases/mdx-stanza/new-deps-field.t/run.t", "command": [ "dune runtest" ], "real": 3.1859999999999999, "user": 2.782, "system": 0.214 }, { "test": "_build/default/test/blackbox-tests/test-cases/mdx-stanza/shared-libraries.t", "command": [ "dune runtest" ], "real": 3.1190000000000002, "user": 2.8300000000000001, "system": 0.17100000000000001 }, { "test": "_build/default/test/blackbox-tests/test-cases/odoc/odoc-new-sherlodoc.t/run.t", "command": [ "dune build @doc-new" ], "real": 3.0779999999999998, "user": 1.7669999999999999, "system": 0.95499999999999996 }, { "test": "_build/default/test/blackbox-tests/test-cases/mdx-stanza/linked-libraries.t/run.t", "command": [ "dune runtest" ], "real": 3.0739999999999998, "user": 2.7970000000000002, "system": 0.16800000000000001 }, { "test": "_build/default/test/blackbox-tests/test-cases/toplevel-plugin/toplevel-plugin-fail.t", "command": [ "dune build @all 2>&1 | dune_cmd sanitize" ], "real": 3.0089999999999999, "user": 2.7210000000000001, "system": 0.224 }, { "test": "_build/default/test/blackbox-tests/test-cases/odoc/new/odoc-unique-mlds/diff-scope.t/run.t", "command": [ "dune build @doc-new" ], "real": 3.0089999999999999, "user": 1.7250000000000001, "system": 0.93899999999999995 }, { "test": "_build/default/test/blackbox-tests/test-cases/mdx-stanza/env-variables.t", "command": [ "dune runtest --auto-promote" ], "real": 2.9969999999999999, "user": 2.8109999999999999, "system": 0.14599999999999999 }, { "test": "_build/default/test/blackbox-tests/test-cases/mdx-stanza/locks.t", "command": [ "dune build" ], "real": 2.992, "user": 2.7330000000000001, "system": 0.13400000000000001 }, { "test": "_build/default/test/blackbox-tests/test-cases/jsoo/jsoo-config-effects.t/run.t", "command": [ "dune build bin/bin1.bc.js bin/bin2.bc.js bin/bin3.bc.js" ], "real": 2.9750000000000001, "user": 2.7080000000000002, "system": 0.216 }, { "test": "_build/default/test/blackbox-tests/test-cases/mdx-stanza/mld-files.t", "command": [ "dune runtest" ], "real": 2.9340000000000002, "user": 2.6989999999999998, "system": 0.153 }, { "test": "_build/default/test/blackbox-tests/test-cases/mdx-stanza/paths.t", "command": [ "dune runtest" ], "real": 2.9300000000000002, "user": 2.645, "system": 0.16 }, { "test": "_build/default/test/blackbox-tests/test-cases/toplevel-plugin/toplevel-plugin.t", "command": [ "dune build @all 2>&1 | dune_cmd sanitize" ], "real": 2.923, "user": 2.6549999999999998, "system": 0.222 }, { "test": "_build/default/test/blackbox-tests/test-cases/mdx-stanza/github7077.t", "command": [ "dune runtest" ], "real": 2.8900000000000001, "user": 2.7170000000000001, "system": 0.14499999999999999 }, { "test": "_build/default/test/blackbox-tests/test-cases/toplevel-plugin/toplevel-plugin-fail2.t", "command": [ "dune build @all 2>&1 | dune_cmd sanitize" ], "real": 2.883, "user": 2.6419999999999999, "system": 0.19700000000000001 }, { "test": "_build/default/test/blackbox-tests/test-cases/mdx-stanza/implicit-transitive-deps-false.t", "command": [ "dune build" ], "real": 2.8109999999999999, "user": 2.661, "system": 0.14099999999999999 }, { "test": "_build/default/test/blackbox-tests/test-cases/jsoo/jsoo-config.t/run.t", "command": [ "dune build bin/bin1.bc.js bin/bin2.bc.js bin/bin3.bc.js" ], "real": 2.7549999999999999, "user": 2.504, "system": 0.21199999999999999 }, { "test": "_build/default/test/blackbox-tests/test-cases/odoc/new/odoc-simple.t/run.t", "command": [ "dune build @doc-new" ], "real": 2.6659999999999999, "user": 1.595, "system": 0.97999999999999998 }, { "test": "_build/default/test/blackbox-tests/test-cases/mdx-stanza/paths.t", "command": [ "dune runtest" ], "real": 2.6230000000000002, "user": 2.4140000000000001, "system": 0.14099999999999999 }, { "test": "_build/default/test/blackbox-tests/test-cases/odoc/new/multi-lib.t/run.t", "command": [ "dune build @doc-new" ], "real": 2.5299999999999998, "user": 1.4930000000000001, "system": 0.93600000000000005 }, { "test": "_build/default/test/blackbox-tests/test-cases/odoc/new/transitive-compile-deps.t/run.t", "command": [ "dune build @doc-new" ], "real": 2.5009999999999999, "user": 1.5269999999999999, "system": 0.876 }, { "test": "_build/default/test/blackbox-tests/test-cases/odoc/new/odoc-wrapped-lib.t/run.t", "command": [ "dune build @doc-new" ], "real": 2.4860000000000002, "user": 1.5109999999999999, "system": 0.86799999999999999 }, { "test": "_build/default/test/blackbox-tests/test-cases/melange/copy-files-include-subdirs.t", "command": [ "dune build @mel" ], "real": 2.4449999999999998, "user": 1.3180000000000001, "system": 0.75900000000000001 }, { "test": "_build/default/test/blackbox-tests/test-cases/odoc/new/github1645.t", "command": [ "dune build @doc-new" ], "real": 2.4220000000000002, "user": 1.456, "system": 0.85799999999999998 } ]I don't expect any miracles, but we should at least review these to see if this is all necessary.
@anmonteiro I see that melange tests are particularly slow. Do you mind looking these over?