[coverage] Automatic merger for LLVM profile data

CMakeLists.txt

@@ -58,6 +58,10 @@ option(SWIFT_INCLUDE_DOCS
     "Create targets for building docs."
     TRUE)
 
+option(SWIFT_ANALYZE_CODE_COVERAGE
+    "Build Swift with code coverage instrumenting enabled."
+    "NONE")
+
 set(SWIFT_VERSION "3.0" CACHE STRING
     "The user-visible version of the Swift compiler")
 set(SWIFT_VENDOR "" CACHE STRING

cmake/modules/AddSwift.cmake

@@ -64,7 +64,7 @@ function(_add_variant_c_compile_link_flags
         "-m${SWIFT_SDK_${sdk}_VERSION_MIN_NAME}-version-min=${SWIFT_SDK_${sdk}_DEPLOYMENT_VERSION}")
 
     if(analyze_code_coverage)
-      list(APPEND result "-fprofile-instr-generate=swift-%p.profraw"
+      list(APPEND result "-fprofile-instr-generate"
                          "-fcoverage-mapping")
     endif()
   endif()
@@ -111,7 +111,7 @@ function(_add_variant_c_compile_flags
   endif()
 
   if(analyze_code_coverage)
-    list(APPEND result "-fprofile-instr-generate=swift-%p.profraw"
+    list(APPEND result "-fprofile-instr-generate"
                        "-fcoverage-mapping")
   endif()
 

test/CMakeLists.txt

@@ -117,6 +117,7 @@ if(PYTHONINTERP_FOUND)
 
   set(TEST_MODES optimize_none optimize optimize_unchecked)
 
+
   foreach(SDK ${SWIFT_SDKS})
     foreach(ARCH ${SWIFT_SDK_${SDK}_ARCHITECTURES})
       foreach(TEST_MODE ${TEST_MODES})
@@ -196,30 +197,50 @@ if(PYTHONINTERP_FOUND)
             "${CMAKE_CURRENT_SOURCE_DIR}/../validation-test/lit.site.cfg.in"
             "${validation_test_bin_dir}/lit.site.cfg"
             "validation-test${VARIANT_SUFFIX}.lit.site.cfg")
+        set(profdata_merge_worker
+            "${CMAKE_CURRENT_SOURCE_DIR}/../utils/profdata_merge_worker.py")
+
+        if(SWIFT_ANALYZE_CODE_COVERAGE STREQUAL "MERGED")
+            set(command_profdata_merge_start
+                COMMAND ${PYTHON_EXECUTABLE} ${profdata_merge_worker} start -o ${swift_test_results_dir})
+            set(command_profdata_merge_stop
+                COMMAND ${PYTHON_EXECUTABLE} ${profdata_merge_worker} stop)
+        else()
+            set(command_profdata_merge_start)
+            set(command_profdata_merge_stop)
+        endif()
 
         add_custom_target("check-swift${test_mode_target_suffix}${VARIANT_SUFFIX}"
             ${command_upload_stdlib}
             ${command_clean_test_results_dir}
+            ${command_profdata_merge_start}
             COMMAND ${lit_command} "${test_bin_dir}"
+            ${command_profdata_merge_stop}
             DEPENDS ${test_dependencies}
             COMMENT "Running Swift tests for ${VARIANT_TRIPLE}"
             ${cmake_3_2_USES_TERMINAL})
 
         add_custom_target("check-swift-validation${test_mode_target_suffix}${VARIANT_SUFFIX}"
             ${command_upload_stdlib}
             ${command_clean_test_results_dir}
+            ${command_profdata_merge_start}
             COMMAND ${lit_command} "${validation_test_bin_dir}"
+            ${command_profdata_merge_stop}
             DEPENDS ${test_dependencies} ${validation_test_dependencies}
             COMMENT "Running Swift validation tests for ${VARIANT_TRIPLE}"
             ${cmake_3_2_USES_TERMINAL})
 
         add_custom_target("check-swift-all${test_mode_target_suffix}${VARIANT_SUFFIX}"
             ${command_upload_stdlib}
             ${command_clean_test_results_dir}
+            ${command_profdata_merge_start}
             COMMAND ${lit_command} "${validation_test_bin_dir}" "${test_bin_dir}"
+            ${command_profdata_merge_stop}
             DEPENDS ${test_dependencies} ${validation_test_dependencies}
             COMMENT "Running all Swift tests for ${VARIANT_TRIPLE}"
             ${cmake_3_2_USES_TERMINAL})
+
+
       endforeach()
     endforeach()
   endforeach()
@@ -245,6 +266,7 @@ if(PYTHONINTERP_FOUND)
 
     add_custom_target(check-swift-all${test_mode_target_suffix}
         DEPENDS "check-swift-all${test_mode_target_suffix}${SWIFT_PRIMARY_VARIANT_SUFFIX}")
+
   endforeach()
 
 endif()

test/lit.cfg

@@ -24,7 +24,10 @@ import re
 import subprocess
 import sys
 import tempfile
+import socket
+import glob
 
+import lit
 import lit.formats
 import lit.util
 
@@ -129,6 +132,39 @@ if config.test_exec_root is None:
 
 ###
 
+class SwiftTest(lit.formats.ShTest):
+    def __init__(self, coverage_mode=None, execute_external=True):
+        lit.formats.ShTest.__init__(self, execute_external)
+        self.coverage_mode = None if coverage_mode == "NONE" else coverage_mode
+
+    def profdir_for_test(self, test):
+        _, tmp_base = lit.TestRunner.getTempPaths(test)
+        return tmp_base + ".profdir"
+
+    def before_test(self, test, litConfig):
+        if self.coverage_mode:
+            profdir = self.profdir_for_test(test)
+            if not os.path.exists(profdir):
+                os.makedirs(profdir)
+
+            test.config.environment["LLVM_PROFILE_FILE"] = os.path.join(profdir, "swift-%p.profraw")
+
+    def after_test(self, test, litConfig, result):
+        if self.coverage_mode == "MERGED":
+            profdir = self.profdir_for_test(test)
+            files = glob.glob(os.path.join(profdir, "*.profraw"))
+            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            sock.connect(('localhost', 12400))
+            sock.send("\n".join(files))
+            sock.close()
+        return result
+
+
+    def execute(self, test, litConfig):
+        self.before_test(test, litConfig)
+        result = super(SwiftTest, self).execute(test, litConfig)
+        return self.after_test(test, litConfig, result)
+
 # name: The name of this test suite.
 config.name = 'Swift'
 
@@ -138,7 +174,7 @@ if platform.system() == 'Darwin':
     config.environment['TOOLCHAINS'] = 'default'
 
 # testFormat: The test format to use to interpret tests.
-config.test_format = lit.formats.ShTest(execute_external=True)
+config.test_format = SwiftTest(coverage_mode=config.coverage_mode)
 
 # suffixes: A list of file extensions to treat as test files.
 config.suffixes = ['.swift', '.ll', '.sil', '.gyb', '.m']

test/lit.site.cfg.in

@@ -18,6 +18,8 @@ config.variant_sdk = "@VARIANT_SDK@"
 config.swiftlib_dir = "@LIT_SWIFTLIB_DIR@"
 config.darwin_xcrun_toolchain = "@SWIFT_DARWIN_XCRUN_TOOLCHAIN@"
 
+config.coverage_mode = "@SWIFT_ANALYZE_CODE_COVERAGE@"
+
 if "@SWIFT_ASAN_BUILD@" == "TRUE":
     config.available_features.add("asan")
 else:
@@ -42,11 +44,6 @@ if "@SWIFT_OPTIMIZED@" == "TRUE":
 if "@SWIFT_HAVE_WORKING_STD_REGEX@" == "FALSE":
     config.available_features.add('broken_std_regex')
 
-if "@SWIFT_ANALYZE_CODE_COVERAGE@" == "TRUE":
-    lit_config.useValgrind = True
-    lit_config.valgrindArgs = [os.path.join(config.swift_src_root,
-                                            "utils/use_profdir.py")]
-
 # Let the main config do the real work.
 if config.test_exec_root is None:
     config.test_exec_root = os.path.dirname(os.path.realpath(__file__))

tools/SourceKit/CMakeLists.txt

@@ -239,9 +239,9 @@ macro(add_sourcekit_executable name)
         PROPERTIES
         LINK_FLAGS "-Wl,-exported_symbol,_main")
 
-      if(SWIFT_ANALYZE_CODE_COVERAGE)
-        set_property(TARGET "${name}" APPEND_STRING PROPERTY
-            LINK_FLAGS " -fprofile-instr-generate=swift-%p.profraw -fcoverage-mapping")
+      if(NOT SWIFT_ANALYZE_CODE_COVERAGE STREQUAL "NONE")
+          set_property(TARGET "${name}" APPEND_STRING PROPERTY
+            LINK_FLAGS " -fprofile-instr-generate -fcoverage-mapping")
       endif()
     endif()
   endif()

tools/SourceKit/tools/complete-test/CMakeLists.txt

@@ -15,9 +15,9 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
     PROPERTIES
     LINK_FLAGS "-Wl,-rpath -Wl,@executable_path/../lib")
 
-  if(SWIFT_ANALYZE_CODE_COVERAGE)
+  if(NOT SWIFT_ANALYZE_CODE_COVERAGE STREQUAL "NONE")
     set_property(TARGET complete-test APPEND_STRING PROPERTY
-        LINK_FLAGS " -fprofile-instr-generate=swift-%p.profraw -fcoverage-mapping")
+        LINK_FLAGS " -fprofile-instr-generate -fcoverage-mapping")
   endif()
 endif()
 

tools/SourceKit/tools/sourcekitd-repl/CMakeLists.txt

@@ -9,9 +9,9 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
     PROPERTIES
     LINK_FLAGS "-Wl,-rpath -Wl,@executable_path/../lib")
 
-  if(SWIFT_ANALYZE_CODE_COVERAGE)
+if(NOT SWIFT_ANALYZE_CODE_COVERAGE STREQUAL "NONE")
     set_property(TARGET sourcekitd-repl APPEND_STRING PROPERTY
-        LINK_FLAGS " -fprofile-instr-generate=swift-%p.profraw -fcoverage-mapping")
+        LINK_FLAGS " -fprofile-instr-generate -fcoverage-mapping")
   endif()
 endif()
 

tools/SourceKit/tools/sourcekitd-test/CMakeLists.txt

@@ -23,9 +23,9 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
     PROPERTIES
     LINK_FLAGS "-Wl,-rpath -Wl,@executable_path/../lib")
 
-  if(SWIFT_ANALYZE_CODE_COVERAGE)
+if(NOT SWIFT_ANALYZE_CODE_COVERAGE STREQUAL "NONE")
     set_property(TARGET sourcekitd-test APPEND_STRING PROPERTY
-        LINK_FLAGS " -fprofile-instr-generate=swift-%p.profraw -fcoverage-mapping")
+        LINK_FLAGS " -fprofile-instr-generate -fcoverage-mapping")
   endif()
 endif()
 

unittests/CMakeLists.txt

@@ -27,9 +27,9 @@ function(add_swift_unittest test_dirname)
     set_property(TARGET "${test_dirname}" APPEND_STRING PROPERTY
       LINK_FLAGS " -Xlinker -rpath -Xlinker ${SWIFT_LIBRARY_OUTPUT_INTDIR}/swift/macosx")
 
-    if(SWIFT_ANALYZE_CODE_COVERAGE)
+    if(NOT SWIFT_ANALYZE_CODE_COVERAGE STREQUAL "NONE")
       set_property(TARGET "${test_dirname}" APPEND_STRING PROPERTY
-          LINK_FLAGS " -fprofile-instr-generate=swift-%p.profraw -fcoverage-mapping")
+          LINK_FLAGS " -fprofile-instr-generate -fcoverage-mapping")
     endif()
   endif()
 endfunction()

utils/build-script

@@ -512,11 +512,12 @@ also build for Apple watchos, but disallow tests that require an watchOS device"
         action="store_true")
 
     parser.add_argument("--swift-analyze-code-coverage",
-        help="enable code coverage analysis in Swift",
-        action="store_const",
-        const=True,
-        dest="swift_analyze_code_coverage",
-        default=False)
+        help="""enable code coverage analysis in Swift (set to `merged` to merge
+and remove intermediary profdata files)""",
+        const="not-merged",
+        choices=["none", "not-merged", "merged"],
+        nargs='?',
+        dest="swift_analyze_code_coverage")
 
     parser.add_argument("--build-subdir",
         help="""

utils/build-script-impl

@@ -71,7 +71,7 @@ KNOWN_SETTINGS=(
     llvm-enable-assertions      "1"              "enable assertions in LLVM and Clang"
     swift-build-type            "Debug"          "the CMake build variant for Swift"
     swift-enable-assertions     "1"              "enable assertions in Swift"
-    swift-analyze-code-coverage "0"              "enable code coverage analysis in Swift"
+    swift-analyze-code-coverage "not-merged"     "Code coverage analysis mode for Swift (none, not-merged, merged). Defaults to none if the argument is not present, and not-merged if the argument is present without a modifier."
     swift-stdlib-build-type     "Debug"          "the CMake build variant for Swift"
     swift-stdlib-enable-assertions "1"           "enable assertions in Swift"
     lldb-build-type             "Debug"          "the CMake build variant for LLDB"
@@ -1600,7 +1600,7 @@ for deployment_target in "${HOST_TARGET}" "${CROSS_TOOLS_DEPLOYMENT_TARGETS[@]}"
                     -DCMAKE_CXX_FLAGS="$(swift_c_flags ${deployment_target})"
                     -DCMAKE_BUILD_TYPE:STRING="${SWIFT_BUILD_TYPE}"
                     -DLLVM_ENABLE_ASSERTIONS:BOOL=$(true_false "${SWIFT_ENABLE_ASSERTIONS}")
-                    -DSWIFT_ANALYZE_CODE_COVERAGE:BOOL=$(true_false "${SWIFT_ANALYZE_CODE_COVERAGE}")
+                    -DSWIFT_ANALYZE_CODE_COVERAGE:STRING=$(toupper "${SWIFT_ANALYZE_CODE_COVERAGE}")
                     -DSWIFT_STDLIB_BUILD_TYPE:STRING="${SWIFT_STDLIB_BUILD_TYPE}"
                     -DSWIFT_STDLIB_ASSERTIONS:BOOL=$(true_false "${SWIFT_STDLIB_ENABLE_ASSERTIONS}")
                     -DSWIFT_NATIVE_LLVM_TOOLS_PATH:STRING="${native_llvm_tools_path}"