diff --git a/.gitmodules b/.gitmodules index 4e1f47d42a2..7c2a1e5cbc5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -206,7 +206,7 @@ url = https://github.com/ClickHouse-Extras/nlp-data.git [submodule "contrib/azure"] path = contrib/azure - url = https://github.com/ClickHouse-Extras/azure-sdk-for-cpp.git + url = https://github.com/timeplus-io/azure-sdk-for-cpp.git [submodule "contrib/wyhash"] path = contrib/wyhash url = https://github.com/wangyi-fudan/wyhash.git diff --git a/CMakeLists.txt b/CMakeLists.txt index df3fdc6b28e..8b7d24419fd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -104,7 +104,6 @@ if (ENABLE_FUZZING) set (ENABLE_CLICKHOUSE_ODBC_BRIDGE OFF) set (ENABLE_LIBRARIES 0) set (ENABLE_SSL 1) - set (USE_UNWIND ON) set (ENABLE_EMBEDDED_COMPILER 0) set (ENABLE_EXAMPLES 0) set (ENABLE_UTILS 0) @@ -187,7 +186,7 @@ if (OS_DARWIN) # from a _specific_ library, which is what we need. set(WHOLE_ARCHIVE -force_load) # The `-noall_load` flag is the default and now obsolete. - set(NO_WHOLE_ARCHIVE "-undefined,error") # Effectively, a no-op. Here to avoid empty "-Wl, " sequence to be generated in the command line. + set(NO_WHOLE_ARCHIVE "-undefined,dynamic_lookup") # Effectively, a no-op. Here to avoid empty "-Wl, " sequence to be generated in the command line. else () set(WHOLE_ARCHIVE --whole-archive) set(NO_WHOLE_ARCHIVE --no-whole-archive) @@ -405,7 +404,7 @@ endif () # Make this extra-checks for correct library dependencies. if (OS_LINUX AND NOT SANITIZE) - target_link_options(global-group INTERFACE "-Wl,--no-undefined") + target_link_options(global-group INTERFACE "LINKER:--no-undefined") endif () ###################################### diff --git a/build.sh b/build.sh index 301415af501..03d8a6a5b50 100755 --- a/build.sh +++ b/build.sh @@ -23,7 +23,6 @@ cmake .. \ -DENABLE_PROTON_INSTALL=ON \ -DENABLE_PROTON_METASTORE=ON \ -DUSE_STATIC_LIBRARIES=ON \ - -DUSE_UNWIND=ON \ -DUSE_DEBUG_HELPERS=ON \ -DENABLE_LIBRARIES=OFF \ -DENABLE_KAFKA=ON \ @@ -32,7 +31,6 @@ cmake .. \ -DENABLE_YAML_CPP=ON \ -DENABLE_SIMDJSON=ON \ -DENABLE_ROCKSDB=ON \ - -DENABLE_REPLXX=ON \ -DENABLE_JEMALLOC=${enable_jemalloc} \ -DENABLE_SSL=ON \ -DENABLE_BZIP2=ON \ diff --git a/cmake/darwin/default_libs.cmake b/cmake/darwin/default_libs.cmake index 2808687779a..a7a98d6bbce 100644 --- a/cmake/darwin/default_libs.cmake +++ b/cmake/darwin/default_libs.cmake @@ -15,6 +15,7 @@ set(CMAKE_OSX_DEPLOYMENT_TARGET 10.15) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) +include (cmake/unwind.cmake) include (cmake/cxx.cmake) target_link_libraries(global-group INTERFACE diff --git a/cmake/tools.cmake b/cmake/tools.cmake index d34cc1f65f2..aa85640ec81 100644 --- a/cmake/tools.cmake +++ b/cmake/tools.cmake @@ -49,18 +49,20 @@ endif () if (NOT LINKER_NAME) if (COMPILER_CLANG) - if (OS_LINUX) - find_program (LLD_PATH NAMES "ld.lld-${COMPILER_VERSION_MAJOR}" "lld-${COMPILER_VERSION_MAJOR}" "ld.lld" "lld") + if (OS_LINUX AND NOT ARCH_S390X) + find_program (LLD_PATH NAMES "ld.lld-${COMPILER_VERSION_MAJOR}" "ld.lld") + elseif (OS_DARWIN) + find_program (LLD_PATH NAMES "ld") endif () endif () - if (OS_LINUX) - if (LLD_PATH) + if (LLD_PATH) + if (OS_LINUX OR OS_DARWIN) if (COMPILER_CLANG) # Clang driver simply allows full linker path. set (LINKER_NAME ${LLD_PATH}) endif () endif () - endif () + endif() endif() if (LINKER_NAME) @@ -82,9 +84,11 @@ endif () if (LINKER_NAME) message(STATUS "Using linker: ${LINKER_NAME}") -else() +elseif (NOT ARCH_S390X AND NOT OS_FREEBSD) + message (FATAL_ERROR "The only supported linker is LLVM's LLD, but we cannot find it.") +else () message(STATUS "Using linker: ") -endif() +endif () # Archiver diff --git a/cmake/unwind.cmake b/cmake/unwind.cmake index c9f5f30a5d6..84e4f01b752 100644 --- a/cmake/unwind.cmake +++ b/cmake/unwind.cmake @@ -1,13 +1 @@ -option (USE_UNWIND "Enable libunwind (better stacktraces)" ${ENABLE_LIBRARIES}) - -if (USE_UNWIND) - add_subdirectory(contrib/libunwind-cmake) - set (UNWIND_LIBRARIES unwind) - set (EXCEPTION_HANDLING_LIBRARY ${UNWIND_LIBRARIES}) - - message (STATUS "Using libunwind: ${UNWIND_LIBRARIES}") -else () - set (EXCEPTION_HANDLING_LIBRARY gcc_eh) -endif () - -message (STATUS "Using exception handler: ${EXCEPTION_HANDLING_LIBRARY}") +add_subdirectory(contrib/libunwind-cmake) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index bb713913f31..7f16105d64f 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -124,9 +124,13 @@ add_contrib (simdjson-cmake simdjson) add_contrib (rapidjson-cmake rapidjson) add_contrib (fastops-cmake fastops) add_contrib (liburing-cmake liburing) -add_contrib (curl-cmake curl) -add_contrib (azure-cmake azure) -add_contrib (sentry-native-cmake sentry-native) # requires: curl + +if (NOT OS_DARWIN) + add_contrib (curl-cmake curl) + add_contrib (azure-cmake azure) + add_contrib (sentry-native-cmake sentry-native) # requires: curl +endif() + add_contrib (fmtlib-cmake fmtlib) add_contrib (krb5-cmake krb5) add_contrib (cyrus-sasl-cmake cyrus-sasl) # for krb5 diff --git a/contrib/NuRaft b/contrib/NuRaft index 545b8c810a9..b2f42d080ce 160000 --- a/contrib/NuRaft +++ b/contrib/NuRaft @@ -1 +1 @@ -Subproject commit 545b8c810a956b2efdc116e86be219af7e83d68a +Subproject commit b2f42d080ce4f0d5a5958e83ade2b74096e0d534 diff --git a/contrib/azure b/contrib/azure index e4fcdfc81e3..e7fcd1ceb3b 160000 --- a/contrib/azure +++ b/contrib/azure @@ -1 +1 @@ -Subproject commit e4fcdfc81e337e589ce231a452dcc280fcbb3f99 +Subproject commit e7fcd1ceb3b36a44dc53c43d847b1ce26b097e10 diff --git a/contrib/curl b/contrib/curl index 462196e6b4a..d755a5f7c00 160000 --- a/contrib/curl +++ b/contrib/curl @@ -1 +1 @@ -Subproject commit 462196e6b4a47f924293a0e26b8e9c23d37ac26f +Subproject commit d755a5f7c009dd63a61b2c745180d8ba937cbfeb diff --git a/contrib/curl-cmake/CMakeLists.txt b/contrib/curl-cmake/CMakeLists.txt index 295f9663269..e24d5bf9d1a 100644 --- a/contrib/curl-cmake/CMakeLists.txt +++ b/contrib/curl-cmake/CMakeLists.txt @@ -8,119 +8,123 @@ endif() set (LIBRARY_DIR "${proton_SOURCE_DIR}/contrib/curl") set (SRCS - "${LIBRARY_DIR}/lib/file.c" - "${LIBRARY_DIR}/lib/timeval.c" + "${LIBRARY_DIR}/lib/altsvc.c" + "${LIBRARY_DIR}/lib/amigaos.c" + "${LIBRARY_DIR}/lib/asyn-thread.c" "${LIBRARY_DIR}/lib/base64.c" - "${LIBRARY_DIR}/lib/hostip.c" - "${LIBRARY_DIR}/lib/progress.c" - "${LIBRARY_DIR}/lib/formdata.c" + "${LIBRARY_DIR}/lib/bufq.c" + "${LIBRARY_DIR}/lib/bufref.c" + "${LIBRARY_DIR}/lib/cf-h1-proxy.c" + "${LIBRARY_DIR}/lib/cf-haproxy.c" + "${LIBRARY_DIR}/lib/cf-https-connect.c" + "${LIBRARY_DIR}/lib/cf-socket.c" + "${LIBRARY_DIR}/lib/cfilters.c" + "${LIBRARY_DIR}/lib/conncache.c" + "${LIBRARY_DIR}/lib/connect.c" + "${LIBRARY_DIR}/lib/content_encoding.c" "${LIBRARY_DIR}/lib/cookie.c" - "${LIBRARY_DIR}/lib/http.c" - "${LIBRARY_DIR}/lib/sendf.c" - "${LIBRARY_DIR}/lib/url.c" + "${LIBRARY_DIR}/lib/curl_addrinfo.c" + "${LIBRARY_DIR}/lib/curl_des.c" + "${LIBRARY_DIR}/lib/curl_endian.c" + "${LIBRARY_DIR}/lib/curl_fnmatch.c" + "${LIBRARY_DIR}/lib/curl_get_line.c" + "${LIBRARY_DIR}/lib/curl_gethostname.c" + "${LIBRARY_DIR}/lib/curl_gssapi.c" + "${LIBRARY_DIR}/lib/curl_memrchr.c" + "${LIBRARY_DIR}/lib/curl_multibyte.c" + "${LIBRARY_DIR}/lib/curl_ntlm_core.c" + "${LIBRARY_DIR}/lib/curl_ntlm_wb.c" + "${LIBRARY_DIR}/lib/curl_path.c" + "${LIBRARY_DIR}/lib/curl_range.c" + "${LIBRARY_DIR}/lib/curl_rtmp.c" + "${LIBRARY_DIR}/lib/curl_sasl.c" + "${LIBRARY_DIR}/lib/curl_sspi.c" + "${LIBRARY_DIR}/lib/curl_threads.c" + "${LIBRARY_DIR}/lib/curl_trc.c" "${LIBRARY_DIR}/lib/dict.c" - "${LIBRARY_DIR}/lib/if2ip.c" - "${LIBRARY_DIR}/lib/speedcheck.c" - "${LIBRARY_DIR}/lib/ldap.c" - "${LIBRARY_DIR}/lib/version.c" - "${LIBRARY_DIR}/lib/getenv.c" - "${LIBRARY_DIR}/lib/escape.c" - "${LIBRARY_DIR}/lib/mprintf.c" - "${LIBRARY_DIR}/lib/telnet.c" - "${LIBRARY_DIR}/lib/netrc.c" - "${LIBRARY_DIR}/lib/getinfo.c" - "${LIBRARY_DIR}/lib/transfer.c" - "${LIBRARY_DIR}/lib/strcase.c" + "${LIBRARY_DIR}/lib/doh.c" + "${LIBRARY_DIR}/lib/dynbuf.c" + "${LIBRARY_DIR}/lib/dynhds.c" "${LIBRARY_DIR}/lib/easy.c" - "${LIBRARY_DIR}/lib/curl_fnmatch.c" + "${LIBRARY_DIR}/lib/escape.c" + "${LIBRARY_DIR}/lib/file.c" "${LIBRARY_DIR}/lib/fileinfo.c" - "${LIBRARY_DIR}/lib/wildcard.c" - "${LIBRARY_DIR}/lib/krb5.c" - "${LIBRARY_DIR}/lib/memdebug.c" - "${LIBRARY_DIR}/lib/http_chunks.c" - "${LIBRARY_DIR}/lib/strtok.c" - "${LIBRARY_DIR}/lib/connect.c" - "${LIBRARY_DIR}/lib/llist.c" + "${LIBRARY_DIR}/lib/fopen.c" + "${LIBRARY_DIR}/lib/formdata.c" + "${LIBRARY_DIR}/lib/getenv.c" + "${LIBRARY_DIR}/lib/getinfo.c" + "${LIBRARY_DIR}/lib/gopher.c" "${LIBRARY_DIR}/lib/hash.c" - "${LIBRARY_DIR}/lib/multi.c" - "${LIBRARY_DIR}/lib/content_encoding.c" - "${LIBRARY_DIR}/lib/share.c" - "${LIBRARY_DIR}/lib/http_digest.c" - "${LIBRARY_DIR}/lib/md4.c" - "${LIBRARY_DIR}/lib/md5.c" - "${LIBRARY_DIR}/lib/http_negotiate.c" - "${LIBRARY_DIR}/lib/inet_pton.c" - "${LIBRARY_DIR}/lib/strtoofft.c" - "${LIBRARY_DIR}/lib/strerror.c" - "${LIBRARY_DIR}/lib/amigaos.c" + "${LIBRARY_DIR}/lib/headers.c" + "${LIBRARY_DIR}/lib/hmac.c" "${LIBRARY_DIR}/lib/hostasyn.c" + "${LIBRARY_DIR}/lib/hostip.c" "${LIBRARY_DIR}/lib/hostip4.c" "${LIBRARY_DIR}/lib/hostip6.c" "${LIBRARY_DIR}/lib/hostsyn.c" + "${LIBRARY_DIR}/lib/hsts.c" + "${LIBRARY_DIR}/lib/http.c" + "${LIBRARY_DIR}/lib/http1.c" + "${LIBRARY_DIR}/lib/http2.c" + "${LIBRARY_DIR}/lib/http_aws_sigv4.c" + "${LIBRARY_DIR}/lib/http_chunks.c" + "${LIBRARY_DIR}/lib/http_digest.c" + "${LIBRARY_DIR}/lib/http_negotiate.c" + "${LIBRARY_DIR}/lib/http_ntlm.c" + "${LIBRARY_DIR}/lib/http_proxy.c" + "${LIBRARY_DIR}/lib/idn.c" + "${LIBRARY_DIR}/lib/if2ip.c" + "${LIBRARY_DIR}/lib/imap.c" "${LIBRARY_DIR}/lib/inet_ntop.c" + "${LIBRARY_DIR}/lib/inet_pton.c" + "${LIBRARY_DIR}/lib/krb5.c" + "${LIBRARY_DIR}/lib/ldap.c" + "${LIBRARY_DIR}/lib/llist.c" + "${LIBRARY_DIR}/lib/md4.c" + "${LIBRARY_DIR}/lib/md5.c" + "${LIBRARY_DIR}/lib/memdebug.c" + "${LIBRARY_DIR}/lib/mime.c" + "${LIBRARY_DIR}/lib/mprintf.c" + "${LIBRARY_DIR}/lib/mqtt.c" + "${LIBRARY_DIR}/lib/multi.c" + "${LIBRARY_DIR}/lib/netrc.c" + "${LIBRARY_DIR}/lib/nonblock.c" + "${LIBRARY_DIR}/lib/noproxy.c" + "${LIBRARY_DIR}/lib/openldap.c" "${LIBRARY_DIR}/lib/parsedate.c" + "${LIBRARY_DIR}/lib/pingpong.c" + "${LIBRARY_DIR}/lib/pop3.c" + "${LIBRARY_DIR}/lib/progress.c" + "${LIBRARY_DIR}/lib/psl.c" + "${LIBRARY_DIR}/lib/rand.c" + "${LIBRARY_DIR}/lib/rename.c" + "${LIBRARY_DIR}/lib/rtsp.c" "${LIBRARY_DIR}/lib/select.c" - "${LIBRARY_DIR}/lib/splay.c" - "${LIBRARY_DIR}/lib/strdup.c" + "${LIBRARY_DIR}/lib/sendf.c" + "${LIBRARY_DIR}/lib/setopt.c" + "${LIBRARY_DIR}/lib/sha256.c" + "${LIBRARY_DIR}/lib/share.c" + "${LIBRARY_DIR}/lib/slist.c" + "${LIBRARY_DIR}/lib/smb.c" + "${LIBRARY_DIR}/lib/smtp.c" + "${LIBRARY_DIR}/lib/socketpair.c" "${LIBRARY_DIR}/lib/socks.c" - "${LIBRARY_DIR}/lib/curl_addrinfo.c" "${LIBRARY_DIR}/lib/socks_gssapi.c" "${LIBRARY_DIR}/lib/socks_sspi.c" - "${LIBRARY_DIR}/lib/curl_sspi.c" - "${LIBRARY_DIR}/lib/slist.c" - "${LIBRARY_DIR}/lib/nonblock.c" - "${LIBRARY_DIR}/lib/curl_memrchr.c" - "${LIBRARY_DIR}/lib/imap.c" - "${LIBRARY_DIR}/lib/pop3.c" - "${LIBRARY_DIR}/lib/smtp.c" - "${LIBRARY_DIR}/lib/pingpong.c" - "${LIBRARY_DIR}/lib/rtsp.c" - "${LIBRARY_DIR}/lib/curl_threads.c" - "${LIBRARY_DIR}/lib/warnless.c" - "${LIBRARY_DIR}/lib/hmac.c" - "${LIBRARY_DIR}/lib/curl_rtmp.c" - "${LIBRARY_DIR}/lib/openldap.c" - "${LIBRARY_DIR}/lib/curl_gethostname.c" - "${LIBRARY_DIR}/lib/gopher.c" - "${LIBRARY_DIR}/lib/idn_win32.c" - "${LIBRARY_DIR}/lib/http_proxy.c" - "${LIBRARY_DIR}/lib/asyn-thread.c" - "${LIBRARY_DIR}/lib/curl_gssapi.c" - "${LIBRARY_DIR}/lib/http_ntlm.c" - "${LIBRARY_DIR}/lib/curl_ntlm_wb.c" - "${LIBRARY_DIR}/lib/curl_ntlm_core.c" - "${LIBRARY_DIR}/lib/curl_sasl.c" - "${LIBRARY_DIR}/lib/rand.c" - "${LIBRARY_DIR}/lib/curl_multibyte.c" - "${LIBRARY_DIR}/lib/conncache.c" - "${LIBRARY_DIR}/lib/dotdot.c" - "${LIBRARY_DIR}/lib/http2.c" - "${LIBRARY_DIR}/lib/smb.c" - "${LIBRARY_DIR}/lib/curl_endian.c" - "${LIBRARY_DIR}/lib/curl_des.c" + "${LIBRARY_DIR}/lib/speedcheck.c" + "${LIBRARY_DIR}/lib/splay.c" + "${LIBRARY_DIR}/lib/strcase.c" + "${LIBRARY_DIR}/lib/strdup.c" + "${LIBRARY_DIR}/lib/strerror.c" + "${LIBRARY_DIR}/lib/strtok.c" + "${LIBRARY_DIR}/lib/strtoofft.c" "${LIBRARY_DIR}/lib/system_win32.c" - "${LIBRARY_DIR}/lib/mime.c" - "${LIBRARY_DIR}/lib/sha256.c" - "${LIBRARY_DIR}/lib/setopt.c" - "${LIBRARY_DIR}/lib/curl_path.c" - "${LIBRARY_DIR}/lib/curl_ctype.c" - "${LIBRARY_DIR}/lib/curl_range.c" - "${LIBRARY_DIR}/lib/psl.c" - "${LIBRARY_DIR}/lib/doh.c" - "${LIBRARY_DIR}/lib/urlapi.c" - "${LIBRARY_DIR}/lib/curl_get_line.c" - "${LIBRARY_DIR}/lib/altsvc.c" - "${LIBRARY_DIR}/lib/socketpair.c" - "${LIBRARY_DIR}/lib/bufref.c" - "${LIBRARY_DIR}/lib/dynbuf.c" - "${LIBRARY_DIR}/lib/hsts.c" - "${LIBRARY_DIR}/lib/http_aws_sigv4.c" - "${LIBRARY_DIR}/lib/mqtt.c" - "${LIBRARY_DIR}/lib/rename.c" - "${LIBRARY_DIR}/lib/h2h3.c" - "${LIBRARY_DIR}/lib/headers.c" + "${LIBRARY_DIR}/lib/telnet.c" "${LIBRARY_DIR}/lib/timediff.c" - "${LIBRARY_DIR}/lib/vauth/vauth.c" + "${LIBRARY_DIR}/lib/timeval.c" + "${LIBRARY_DIR}/lib/transfer.c" + "${LIBRARY_DIR}/lib/url.c" + "${LIBRARY_DIR}/lib/urlapi.c" "${LIBRARY_DIR}/lib/vauth/cleartext.c" "${LIBRARY_DIR}/lib/vauth/cram.c" "${LIBRARY_DIR}/lib/vauth/digest.c" @@ -132,25 +136,24 @@ set (SRCS "${LIBRARY_DIR}/lib/vauth/oauth2.c" "${LIBRARY_DIR}/lib/vauth/spnego_gssapi.c" "${LIBRARY_DIR}/lib/vauth/spnego_sspi.c" - "${LIBRARY_DIR}/lib/vtls/openssl.c" + "${LIBRARY_DIR}/lib/vauth/vauth.c" + "${LIBRARY_DIR}/lib/version.c" + "${LIBRARY_DIR}/lib/vquic/vquic.c" + "${LIBRARY_DIR}/lib/vssh/libssh.c" + "${LIBRARY_DIR}/lib/vssh/libssh2.c" + "${LIBRARY_DIR}/lib/vtls/bearssl.c" "${LIBRARY_DIR}/lib/vtls/gtls.c" - "${LIBRARY_DIR}/lib/vtls/vtls.c" - "${LIBRARY_DIR}/lib/vtls/nss.c" - "${LIBRARY_DIR}/lib/vtls/wolfssl.c" + "${LIBRARY_DIR}/lib/vtls/hostcheck.c" + "${LIBRARY_DIR}/lib/vtls/keylog.c" + "${LIBRARY_DIR}/lib/vtls/mbedtls.c" + "${LIBRARY_DIR}/lib/vtls/openssl.c" "${LIBRARY_DIR}/lib/vtls/schannel.c" "${LIBRARY_DIR}/lib/vtls/schannel_verify.c" "${LIBRARY_DIR}/lib/vtls/sectransp.c" - "${LIBRARY_DIR}/lib/vtls/gskit.c" - "${LIBRARY_DIR}/lib/vtls/mbedtls.c" - "${LIBRARY_DIR}/lib/vtls/bearssl.c" - "${LIBRARY_DIR}/lib/vtls/keylog.c" + "${LIBRARY_DIR}/lib/vtls/vtls.c" + "${LIBRARY_DIR}/lib/vtls/wolfssl.c" "${LIBRARY_DIR}/lib/vtls/x509asn1.c" - "${LIBRARY_DIR}/lib/vtls/hostcheck.c" - "${LIBRARY_DIR}/lib/vquic/ngtcp2.c" - "${LIBRARY_DIR}/lib/vquic/quiche.c" - "${LIBRARY_DIR}/lib/vquic/msh3.c" - "${LIBRARY_DIR}/lib/vssh/libssh2.c" - "${LIBRARY_DIR}/lib/vssh/libssh.c" + "${LIBRARY_DIR}/lib/warnless.c" ) add_library (_curl ${SRCS}) diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index 71d7e00150c..e00542cb057 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -170,16 +170,13 @@ endif () target_compile_definitions(_jemalloc PRIVATE -DJEMALLOC_PROF=1) -if (USE_UNWIND) - # jemalloc provides support for two different libunwind flavors: the original HP libunwind and the one coming with gcc / g++ / libstdc++. - # The latter is identified by `JEMALLOC_PROF_LIBGCC` and uses `_Unwind_Backtrace` method instead of `unw_backtrace`. - # At the time ClickHouse uses LLVM libunwind which follows libgcc's way of backtracing. - - # ClickHouse has to provide `unw_backtrace` method by the means of [commit 8e2b31e](https://github.com/ClickHouse/libunwind/commit/8e2b31e766dd502f6df74909e04a7dbdf5182eb1). - - target_compile_definitions (_jemalloc PRIVATE -DJEMALLOC_PROF_LIBGCC=1) - target_link_libraries (_jemalloc PRIVATE unwind) -endif () +# jemalloc provides support for two different libunwind flavors: the original HP libunwind and the one coming with gcc / g++ / libstdc++. +# The latter is identified by `JEMALLOC_PROF_LIBGCC` and uses `_Unwind_Backtrace` method instead of `unw_backtrace`. +# At the time ClickHouse uses LLVM libunwind which follows libgcc's way of backtracing. +# +# ClickHouse has to provide `unw_backtrace` method by the means of [commit 8e2b31e](https://github.com/ClickHouse/libunwind/commit/8e2b31e766dd502f6df74909e04a7dbdf5182eb1). +target_compile_definitions (_jemalloc PRIVATE -DJEMALLOC_PROF_LIBGCC=1) +target_link_libraries (_jemalloc PRIVATE unwind) # for RTLD_NEXT target_compile_options(_jemalloc PRIVATE -D_GNU_SOURCE) diff --git a/contrib/libcxx-cmake/CMakeLists.txt b/contrib/libcxx-cmake/CMakeLists.txt index 10943593ae0..02f4f1c7e14 100644 --- a/contrib/libcxx-cmake/CMakeLists.txt +++ b/contrib/libcxx-cmake/CMakeLists.txt @@ -61,9 +61,7 @@ target_include_directories(cxx SYSTEM BEFORE PUBLIC $<$:$ target_compile_definitions(cxx PRIVATE -D_LIBCPP_BUILDING_LIBRARY -DLIBCXX_BUILDING_LIBCXXABI) # Enable capturing stack traces for all exceptions. -if (USE_UNWIND) - target_compile_definitions(cxx PUBLIC -DSTD_EXCEPTION_HAS_STACK_TRACE=1) -endif () +target_compile_definitions(cxx PUBLIC -DSTD_EXCEPTION_HAS_STACK_TRACE=1) if (USE_MUSL) target_compile_definitions(cxx PUBLIC -D_LIBCPP_HAS_MUSL_LIBC=1) diff --git a/contrib/libcxxabi-cmake/CMakeLists.txt b/contrib/libcxxabi-cmake/CMakeLists.txt index d3cb8ae039f..edae5681e32 100644 --- a/contrib/libcxxabi-cmake/CMakeLists.txt +++ b/contrib/libcxxabi-cmake/CMakeLists.txt @@ -35,12 +35,10 @@ target_include_directories(cxxabi SYSTEM BEFORE ) target_compile_definitions(cxxabi PRIVATE -D_LIBCPP_BUILDING_LIBRARY) target_compile_options(cxxabi PRIVATE -nostdinc++ -fno-sanitize=undefined -Wno-macro-redefined) # If we don't disable UBSan, infinite recursion happens in dynamic_cast. -target_link_libraries(cxxabi PUBLIC ${EXCEPTION_HANDLING_LIBRARY}) +target_link_libraries(cxxabi PUBLIC unwind) # Enable capturing stack traces for all exceptions. -if (USE_UNWIND) - target_compile_definitions(cxxabi PUBLIC -DSTD_EXCEPTION_HAS_STACK_TRACE=1) -endif () +target_compile_definitions(cxxabi PUBLIC -DSTD_EXCEPTION_HAS_STACK_TRACE=1) install( TARGETS cxxabi diff --git a/contrib/libunwind b/contrib/libunwind index e48aa13f67d..30cc1d3fd36 160000 --- a/contrib/libunwind +++ b/contrib/libunwind @@ -1 +1 @@ -Subproject commit e48aa13f67dc722511b5af33a32ba9b7748176b5 +Subproject commit 30cc1d3fd3655a5cfa0ab112fe320fb9fc0a8344 diff --git a/docker/packager/packager b/docker/packager/packager index bc4812d2a37..bb61f409451 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -202,7 +202,6 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ cmake_flags.append('-DENABLE_PROTON_INSTALL=ON') cmake_flags.append('-DENABLE_PROTON_METASTORE=ON') cmake_flags.append('-DUSE_STATIC_LIBRARIES=ON') - cmake_flags.append('-DUSE_UNWIND=ON') cmake_flags.append('-DUSE_DEBUG_HELPERS=ON') cmake_flags.append('-DENABLE_LIBRARIES=OFF') cmake_flags.append('-DENABLE_KAFKA=ON') @@ -212,7 +211,6 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ cmake_flags.append('-DENABLE_YAML_CPP=ON') cmake_flags.append('-DENABLE_SIMDJSON=ON') cmake_flags.append('-DENABLE_ROCKSDB=ON') - cmake_flags.append('-DENABLE_REPLXX=ON') cmake_flags.append('-DENABLE_SSL=ON') cmake_flags.append('-DENABLE_BZIP2=ON') cmake_flags.append('-DENABLE_PROTOBUF=ON') diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index ff11b68af6b..1b5c40191b9 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -196,10 +196,8 @@ function run_cmake "-DENABLE_UTILS=0" "-DENABLE_EMBEDDED_COMPILER=0" "-DENABLE_THINLTO=0" - "-DUSE_UNWIND=1" "-DENABLE_NURAFT=1" "-DENABLE_JEMALLOC=1" - "-DENABLE_REPLXX=1" "-DENABLE_LIBURING=1" ) diff --git a/docs/_includes/cmake_in_clickhouse_header.md b/docs/_includes/cmake_in_clickhouse_header.md index de4b1ef7af5..c2253942b5a 100644 --- a/docs/_includes/cmake_in_clickhouse_header.md +++ b/docs/_includes/cmake_in_clickhouse_header.md @@ -13,7 +13,6 @@ cmake .. \ -DENABLE_CLICKHOUSE_SERVER=ON \ -DENABLE_CLICKHOUSE_CLIENT=ON \ -DENABLE_LIBRARIES=OFF \ - -DUSE_UNWIND=ON \ -DENABLE_UTILS=OFF \ -DENABLE_TESTS=OFF ``` diff --git a/docs/en/development/build-cross-riscv.md b/docs/en/development/build-cross-riscv.md index f37bd6ce5df..175dc01ed34 100644 --- a/docs/en/development/build-cross-riscv.md +++ b/docs/en/development/build-cross-riscv.md @@ -23,7 +23,7 @@ sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ``` bash cd ClickHouse mkdir build-riscv64 -CC=clang-14 CXX=clang++-14 cmake . -Bbuild-riscv64 -G Ninja -DCMAKE_TOOLCHAIN_FILE=cmake/linux/toolchain-riscv64.cmake -DGLIBC_COMPATIBILITY=OFF -DENABLE_LDAP=OFF -DOPENSSL_NO_ASM=ON -DENABLE_JEMALLOC=ON -DENABLE_PARQUET=OFF -DENABLE_ORC=OFF -DUSE_UNWIND=OFF -DENABLE_GRPC=OFF -DENABLE_HDFS=OFF -DENABLE_MYSQL=OFF +CC=clang-14 CXX=clang++-14 cmake . -Bbuild-riscv64 -G Ninja -DCMAKE_TOOLCHAIN_FILE=cmake/linux/toolchain-riscv64.cmake -DGLIBC_COMPATIBILITY=OFF -DENABLE_LDAP=OFF -DOPENSSL_NO_ASM=ON -DENABLE_JEMALLOC=ON -DENABLE_PARQUET=OFF -DENABLE_ORC=OFF -DENABLE_GRPC=OFF -DENABLE_HDFS=OFF -DENABLE_MYSQL=OFF ninja -C build-riscv64 ``` diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 69ee725b063..34cf5ad5c34 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1508,7 +1508,7 @@ int Server::main(const std::vector & /*args*/) /// Init trace collector only after trace_log system table was created /// Disable it if we collect test coverage information, because it will work extremely slow. -#if USE_UNWIND && !WITH_COVERAGE +#if !WITH_COVERAGE /// Profilers cannot work reliably with any other libunwind or without PHDR cache. if (hasPHDRCache()) { @@ -1531,10 +1531,6 @@ int Server::main(const std::vector & /*args*/) /// Describe multiple reasons when query profiler cannot work. -#if !USE_UNWIND - LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they cannot work without bundled unwind (stack unwinding) library."); -#endif - #if WITH_COVERAGE LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they work extremely slow with test coverage."); #endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8be90641147..ab443e37be1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -306,7 +306,7 @@ else() list (APPEND all_modules dbms) # force all split libs to be linked if (OS_DARWIN) - set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-undefined,error") + set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-undefined,dynamic_lookup") else() set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-as-needed") endif() @@ -371,8 +371,6 @@ target_link_libraries(clickhouse_common_io PUBLIC ch_contrib::re2_st) target_link_libraries(clickhouse_common_io PUBLIC ch_contrib::re2) target_link_libraries(clickhouse_common_io - PRIVATE - ${EXECINFO_LIBRARIES} PUBLIC boost::program_options boost::system diff --git a/src/Common/QueryProfiler.cpp b/src/Common/QueryProfiler.cpp index 4b51a40d23d..e5265a051a1 100644 --- a/src/Common/QueryProfiler.cpp +++ b/src/Common/QueryProfiler.cpp @@ -97,13 +97,13 @@ QueryProfilerBase::QueryProfilerBase(const UInt64 thread_id, const UNUSED(pause_signal); throw Exception("QueryProfiler disabled because they cannot work under sanitizers", ErrorCodes::NOT_IMPLEMENTED); -#elif !USE_UNWIND +#elif defined(OS_DARWIN) UNUSED(thread_id); UNUSED(clock_type); UNUSED(period); UNUSED(pause_signal); - throw Exception("QueryProfiler cannot work with stock libunwind", ErrorCodes::NOT_IMPLEMENTED); + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "QueryProfiler cannot work on OSX"); #else /// Sanity check. if (!hasPHDRCache()) @@ -184,7 +184,7 @@ QueryProfilerBase::~QueryProfilerBase() template void QueryProfilerBase::tryCleanup() { -#if USE_UNWIND +#ifndef OS_DARWIN if (timer_id.has_value()) { if (timer_delete(*timer_id)) diff --git a/src/Common/QueryProfiler.h b/src/Common/QueryProfiler.h index 479defad7d7..5474a790ac3 100644 --- a/src/Common/QueryProfiler.h +++ b/src/Common/QueryProfiler.h @@ -39,7 +39,7 @@ class QueryProfilerBase Poco::Logger * log; -#if USE_UNWIND +#ifndef OS_DARWIN /// Timer id from timer_create(2) std::optional timer_id; #endif diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index 8e7b2605981..aa265365294 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -14,9 +14,7 @@ #include "config.h" -#if USE_UNWIND -# include -#endif +#include std::string signalToErrorMessage(int sig, const siginfo_t & info, [[maybe_unused]] const ucontext_t & context) { @@ -300,10 +298,8 @@ StackTrace::StackTrace(NoCapture) void StackTrace::tryCapture() { size = 0; -#if USE_UNWIND size = unw_backtrace(frame_pointers.data(), capacity); __msan_unpoison(frame_pointers.data(), size * sizeof(frame_pointers[0])); -#endif } size_t StackTrace::getSize() const diff --git a/src/Common/config.h.in b/src/Common/config.h.in index f30fcfc1ea5..4420d976e16 100644 --- a/src/Common/config.h.in +++ b/src/Common/config.h.in @@ -9,7 +9,6 @@ #cmakedefine01 USE_AWS_S3 #cmakedefine01 USE_AZURE_BLOB_STORAGE #cmakedefine01 USE_BROTLI -#cmakedefine01 USE_UNWIND #cmakedefine01 USE_CASSANDRA #cmakedefine01 USE_SENTRY #cmakedefine01 USE_GRPC diff --git a/src/Daemon/CMakeLists.txt b/src/Daemon/CMakeLists.txt index f02fd69aa79..90b5f32c1d8 100644 --- a/src/Daemon/CMakeLists.txt +++ b/src/Daemon/CMakeLists.txt @@ -11,7 +11,7 @@ if (OS_DARWIN AND NOT USE_STATIC_LIBRARIES) target_link_libraries (daemon PUBLIC -Wl,-undefined,dynamic_lookup) endif() -target_link_libraries (daemon PUBLIC loggers common PRIVATE clickhouse_common_io clickhouse_common_config ${EXECINFO_LIBRARIES}) +target_link_libraries (daemon PUBLIC loggers common PRIVATE clickhouse_common_io clickhouse_common_config) if (TARGET ch_contrib::sentry) target_link_libraries (daemon PRIVATE ch_contrib::sentry) diff --git a/src/Storages/System/StorageSystemBuildOptions.cpp.in b/src/Storages/System/StorageSystemBuildOptions.cpp.in index 77883debad2..cd4e95d3e12 100644 --- a/src/Storages/System/StorageSystemBuildOptions.cpp.in +++ b/src/Storages/System/StorageSystemBuildOptions.cpp.in @@ -29,7 +29,6 @@ const char * auto_config_build[] "USE_EMBEDDED_COMPILER", "@USE_EMBEDDED_COMPILER@", "USE_GLIBC_COMPATIBILITY", "@GLIBC_COMPATIBILITY@", "USE_JEMALLOC", "@ENABLE_JEMALLOC@", - "USE_UNWIND", "@USE_UNWIND@", "USE_ICU", "@USE_ICU@", "USE_H3", "@USE_H3@", "USE_MYSQL", "@USE_MYSQL@", diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 7eddd232149..4744e830962 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -15,12 +15,10 @@ add_subdirectory (report) # Not used in package if (ENABLE_UTILS) add_subdirectory (compressor) - add_subdirectory (iotest) add_subdirectory (corrector_utf8) add_subdirectory (wikistat-loader) add_subdirectory (check-marks) add_subdirectory (checksum-for-compressed-block) - add_subdirectory (db-generator) add_subdirectory (wal-dump) # memcpy_jart.S contains position dependent code diff --git a/utils/db-generator/CMakeLists.txt b/utils/db-generator/CMakeLists.txt deleted file mode 100644 index 00f05ccb5ff..00000000000 --- a/utils/db-generator/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -proton_add_executable (query_db_generator query_db_generator.cpp) -target_link_libraries(query_db_generator PRIVATE clickhouse_parsers boost::program_options) \ No newline at end of file diff --git a/utils/db-generator/README.md b/utils/db-generator/README.md deleted file mode 100644 index 5596aac66e4..00000000000 --- a/utils/db-generator/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# Clickhouse query analysis - -Here we will consider only `SELECT` queries, i.e. those queries that get data from the table. -The built-in Clickhouse parser accepts a string as input, which is a query. Among 14 main clauses of `SELECT` statement: `WITH`, `SELECT`, `TABLES`, `PREWHERE`, `WHERE`, `GROUP_BY`, `HAVING`, `ORDER_BY`, `LIMIT_BY_OFFSET`, `LIMIT_BY_LENGTH`, `LIMIT_BY`, `LIMIT_OFFSET`, `LIMIT_LENGTH`, `SETTINGS`, we will analyze the `SELECT`, `TABLES`, `WHERE`, `GROUP_BY`, `HAVING`, `ORDER_BY` clauses because the most of data is there. We need this data to analyze the structure and to identify values. The parser issues a tree structure after parsing a query, where each node is a specific query execution operation, a function over values, a constant, a designation, etc. Nodes also have subtrees where their arguments or suboperations are located. We will try to reveal the data we need by avoiding this tree. - -## Scheme analysis - -It is necessary to determine possible tables by a query. Having a query string, you can understand which parts of it represent the names of the tables, so you can determine their number in our database. -In the Clickhouse parser, `TABLES` (Figure 1) is a query subtree responsible for tables where we get data. It contains the main table where the columns come from, as well as the `JOIN` operations that are performed in the query. Avoiding all nodes in the subtree, we use the names of the tables and databases where they are located, as well as their alias, i.e. the shortened names chosen by the query author. We may need these names to determine the ownership of the column in the future. -Thus, we get a set of databases for the query, as well as tables and their aliases, with the help of them a query is made. - -Then we need to define the set of columns that are in the query and the tables they can refer to. The set of columns in each table is already known during the query execution. Therefore, the program automatically links the column and table at runtime. However, in our case, it is impossible to unambiguously interpret the belonging of a column to a specific table, for example, in the following query `SELECT column1, column2, column3 FROM table1 JOIN table2 on table1.column2 = table2.column3`. In this case, we can say which table `column2` and `column3` belong to. However, `column1` can belong to either the first or the second table. We will refer undefined columns to the main table, on which a query is made, for unambiguous interpretation of such cases. For example, in this case, it will be `table1`. -All columns in the tree are in `IDENTIFIER` type nodes, which are in the `SELECT`, `TABLES`, `WHERE`, `GROUP_BY`, `HAVING`, `ORDER_BY` subtrees. We form a set of all tables recursively avoiding the subtrees, then we split the column into constituents such as the table (if it is explicitly specified with a dot) and the name. Then, since the table can be an alias, we replace the alias with the original table name. We now have a list of all the columns and tables they belong to. We define the main query table for non-table columns. - -## Column analysis - -Then we need to exactly define data types for columns that have a value in the query. An example is the boolean `WHERE` clause where we test boolean expressions in its attributes. If the query specifies `column > 5`, then we can conclude that this column contains a numeric value, or if the `LIKE` expression is applied to the attribute, then the attribute has a string type. -In this part, you need to learn how to extract such expressions from a query and match data types for columns, where it is possible. At the same time, it is clear that it is not always possible to make an unambiguous decision about the type of a particular attribute from the available values. For example, `column > 5` can mean many numeric types such as `UINT8`, `UINT32`, `INT32`, `INT64`, etc. It is necessary to determine the interpretation of certain values since searching through all possible values ​​can be quite large and long. -It can take a long time to iterate over all possible values, so we use `INT64` and `FLOAT64` types for numeric values, `STRING` for strings, `DATE` and `DATETIME` for dates, and `ARRAY`. -We can determine column values ​​using boolean, arithmetic and other functions on the column values ​​that are specified in the query. Such functions are in the `SELECT` and `WHERE` subtrees. The function parameter can be a constant, a column or another function (Figure 2). Thus, the following parameters can help to understand the type of the column: -- The types of arguments that a function can take, for example, the `TOSTARTOFMINUTE` function (truncate time up to a multiple of 5 minutes down) can only accept `DATETIME`, so if the argument of this function is a column, then this column has `DATETIME` type. -- The types of the remaining arguments in this function. For example, the `EQUALS` function means equality of its argument types, so if a constant and a column are present in this function, then we can define the type of the column as the type of the constant. - -Thus, we define the possible argument types, the return type, the parameter for each function, and the function arguments of the identical type. The recursive function handler will determine the possible types of columns used in these functions by the values of the arguments, and then return the possible types of the function's result. -Now, for each column, we have many possible types of values. We will choose one specific type from this set to interpret the query unambiguously. - -## Column values definition - -At this stage, we already have a certain structure of the database tables, we need to fill this table with values. We should understand which columns depend on each other when executing the function (for example, the join is done according to two columns, which means that they must have the same values). We also need to understand what values ​​the columns must have to fulfill various conditions during execution. -We search for all comparison operations in our query to achieve the goal. If the arguments of the operation are two columns, then we consider them linked. If the arguments are the column and the value, then we assign that value to the possible column value and add the value with some noise. A random number is a noise for a numeric type, it is a random number of days for a date, etc. In this case, a handler for this operation is required for each comparison operation, which generates at least two values, one of them is the operation condition, and the other is not. For example, a value greater than 5 and less than or equal to 5 must be assigned for the operation `column1 > 5`, `column1`, for the operation `column2 LIKE some% string` the same is true. The satisfying and not satisfying expression must be assigned to `column2`. -Now we have many associated columns and many values. We know that the connectivity of columns is symmetric, but we need to add transitivity for a complete definition, because if `column1 = column2` and `column2 = column3`, then `column1 = column3`, but this does not follow from the construction. Accordingly, we need to extend the connectivity across all columns. We combine multiple values for each column with the values associated with it. If we have columns with no values, then we generate random values. - -## Generation - -We have a complete view of the database schema as well as many values ​​for each table now. We will generate data by cartesian product of the value set of each column for a specific table. Thus, we get a set for each table, consisting of sets of values for each column. We start generating queries that create this table and fill it with data. We generate the `CREATE QUERY` that creates this table based on the structure of the table and the types of its columns, and then we generate the `INSERT QUERY` over the set of values, which fills the table with data. diff --git a/utils/db-generator/query_db_generator.cpp b/utils/db-generator/query_db_generator.cpp deleted file mode 100644 index 6cf2307bf84..00000000000 --- a/utils/db-generator/query_db_generator.cpp +++ /dev/null @@ -1,1354 +0,0 @@ -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - - -namespace po = boost::program_options; - -using ColumnType = uint32_t; -using TableAndColumn = std::pair; -pcg64 rng; - -std::string randomString(size_t length) -{ - auto randchar = []() -> char - { - const char charset[] = "0123456789" "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"; - const size_t max_index = (sizeof(charset) - 1); - return charset[rng() % max_index]; - }; - std::string str(length, 0); - std::generate_n(str.begin(), length, randchar); - return str; -} -std::string randomInteger(unsigned int min = 0, unsigned int max = 4294967295) -{ - int r = rng() % (max - min) + min; - return std::to_string(r); -} - -std::string randomFloat(unsigned int min = 0, unsigned int max = 4294967295) -{ - float r = static_cast(rng() % max) / (static_cast(rng() % 100)) + min; - return std::to_string(r); -} - -std::string randomDate() -{ - int32_t year = rng() % 136 + 1970; - int32_t month = rng() % 12 + 1; - int32_t day = rng() % 12 + 1; - char answer[13]; - sprintf(answer, "'%04u-%02u-%02u'", year, month, day); - return std::string(answer); -} - -std::string randomDatetime() -{ - int32_t year = rng() % 136 + 1970; - int32_t month = rng() % 12 + 1; - int32_t day = rng() % 12 + 1; - int32_t hours = rng() % 24; - int32_t minutes = rng() % 60; - int32_t seconds = rng() % 60; - char answer[22]; - sprintf( - answer, - "'%04u-%02u-%02u %02u:%02u:%02u'", - year, - month, - day, - hours, - minutes, - seconds); - return std::string(answer); -} -TableAndColumn get_table_a_column(const std::string & c) -{ - auto point_place = c.rfind('.'); - std::string db{}; - std::string column{}; - if (point_place != std::string::npos) - { - db = c.substr(0, point_place); - column = c.substr(point_place + 1); - } - else - { - column = c; - } - return { db, column }; -} - - -enum Type : ColumnType -{ - i = 1, - // int - f = 2, - // float - s = 4, - // string - d = 8, - // date - dt = 16, - // datetime - b = 32, - // bool - all = 63, - a = 64, - // array - t = 128, - // tuple -}; - - -std::map type_definition = -{ - {Type::i, "Int64"}, {Type::f, "Float64"}, {Type::s, "String"}, {Type::d, "Date"}, {Type::dt, "DateTime"}, {Type::b, "UInt8"} -}; - -ColumnType time_type(std::string value) -{ - if (value.length() == 12) - { - for (size_t i : {5, 8}) - { - if (value[i] != '-') - return Type::s; - } - for (size_t i : {1, 2, 3, 4, 6, 7, 9, 10}) - { - if (!isdigit(value[i])) - return Type::s; - } - return Type::d; - } - - if (value.length() == 21) - { - for (size_t i : {5, 8}) - { - if (value[i] != '-') - return Type::s; - } - for (size_t i : {14, 17}) - { - if (value[i] != '-') - return Type::s; - } - if (value[11] != '-') - return Type::s; - return Type::dt; - } - return Type::s; -} -// Casting inner clickhouse parser type to our type -ColumnType type_cast(int t) -{ - switch (t) - { - case 1: - case 2: - case 4: - case 5: - case 19: - case 20: - case 21: - return Type::i; - - case 3: - return Type::f; - - case 16: - return Type::s; - - case 17: - return Type::a | Type::all; - - case 18: - return Type::t | Type::all; - } - return Type::all; -} - - -class FuncRet -{ -public: - FuncRet() = default; - - FuncRet(ColumnType t, std::string v) - : value(v) - , type(t) {} - - FuncRet(ColumnType t, std::string v, bool is_a) - : value(v) - , type(t) - , is_array(is_a) {} - - std::string value{}; - ColumnType type = Type::all; - bool is_array = false; -}; - - -std::map func_to_return_type = { - {"divide", FuncRet(Type::f, "")}, {"e", FuncRet(Type::f, "e()")}, {"pi", FuncRet(Type::f, "pi()")}, {"exp", FuncRet(Type::f, "")}, - {"log", FuncRet(Type::f,"")}, {"exp2", FuncRet(Type::f, "")}, {"log2", FuncRet(Type::f, "")}, {"exp10", FuncRet(Type::f, "")}, - {"log10", FuncRet(Type::f, "")}, {"sqrt", FuncRet(Type::f, "")}, {"cbrt", FuncRet(Type::f, "")}, {"erf", FuncRet(Type::f, "")}, - {"erfc", FuncRet(Type::f, "")}, {"lgamma", FuncRet(Type::f, "")}, {"tgamma", FuncRet(Type::f, "")}, {"sin", FuncRet(Type::f, "")}, - {"cos", FuncRet(Type::f, "")}, {"tan", FuncRet(Type::f, "")}, {"asin", FuncRet(Type::f, "")}, {"acos", FuncRet(Type::f, "")}, - {"atan", FuncRet(Type::f, "")}, {"pow", FuncRet(Type::f, "")}, {"splitbystring", FuncRet(Type::s | Type::a,"")}, - {"splitbychar", FuncRet(Type::s | Type::a, "")}, {"alphatokens", FuncRet(Type::s | Type::a, "")}, {"toyear", FuncRet(Type::i, "")}, - {"tomonth", FuncRet(Type::i, "")}, {"todayofmonth", FuncRet(Type::i, "")}, {"tohour", FuncRet(Type::dt, "")}, {"tominute", FuncRet(Type::dt, "")}, - {"toseconds", FuncRet(Type::dt, "")}, {"tounixtimestamp", FuncRet(Type::i, "")}, {"tostartofyear", FuncRet(Type::dt | Type::d, "")}, - {"tostartofquater",FuncRet(Type::dt | Type::d, "")}, {"tostartofmonth", FuncRet(Type::dt | Type::d, "")}, {"tomonday", FuncRet(Type::dt | Type::d, "")}, - {"tostartoffiveminutes", FuncRet(Type::dt, "")}, {"tostartoftenminutes", FuncRet(Type::dt, "")}, {"tostartoffifteenminutes", FuncRet(Type::dt, "")}, - {"tostartofinterval", FuncRet(Type::dt, "")}, {"totime", FuncRet(Type::dt, "")}, {"torelativemonthnum", FuncRet(Type::i, "")}, - {"torelativeweeknum", FuncRet(Type::i, "")}, {"torelativedaynum", FuncRet(Type::i, "")}, {"torelativehournum", FuncRet(Type::i, "")}, - {"torelativeminutenum", FuncRet(Type::i, "")}, {"torelativesecondsnum", FuncRet(Type::i, "")}, {"datediff", FuncRet(Type::d | Type::dt, "")}, - {"formatdatetime", FuncRet(Type::s, "")}, {"now", FuncRet(Type::dt | Type::d, "now()")}, {"today", FuncRet(Type::d | Type::dt, "today()")}, - {"yesterday", FuncRet(Type::d | Type::dt, "yesterday()")} -}; - -std::set func_args_same_types = { - "equals", "notequals", "less", "greater", "lessorequals", "greaterorequals", "multiply" -}; - -std::map func_to_param_type = { - {"tostartofminute", Type::dt}, {"plus", Type::i | Type::f | Type::d | Type::dt}, {"multiply", Type::i | Type::f}, - {"minus", Type::i | Type::f | Type::d | Type::dt}, {"negate", Type::i | Type::f}, {"divide", Type::i | Type::f}, - {"abs", Type::i | Type::f}, {"gcd", Type::i | Type::f}, {"lcm", Type::i | Type::f}, {"bitnot", Type::i}, {"bitshiftleft", Type::i}, - {"bitshiftright", Type::i}, {"bittest", Type::i}, {"exp", Type::i | Type::f}, {"log", Type::i | Type::f}, - {"exp2", Type::i | Type::f}, {"log2", Type::i | Type::f}, {"exp10", Type::i | Type::f}, {"log10", Type::i | Type::f}, - {"sqrt", Type::i | Type::f}, {"cbrt", Type::i | Type::f}, {"erf", Type::i | Type::f}, {"erfc", Type::i | Type::f}, - {"lgamma", Type::i | Type::f}, {"tgamma", Type::i | Type::f}, {"sin", Type::i | Type::f}, {"cos", Type::i | Type::f}, - {"tan", Type::i | Type::f}, {"asin", Type::i | Type::f}, {"acos", Type::i | Type::f}, {"atan", Type::i | Type::f}, - {"pow", Type::i | Type::f}, {"arrayjoin", Type::all | Type::a}, {"substring", Type::s}, {"splitbystring", Type::s}, {"splitbychar", Type::s}, - {"alphatokens", Type::s}, {"toyear", Type::d | Type::dt}, {"tomonth", Type::d | Type::dt}, {"todayofmonth", Type::d | Type::dt}, {"tohour", Type::dt}, - {"tominute", Type::dt}, {"tosecond", Type::dt}, {"touixtimestamp", Type::dt}, {"tostartofyear", Type::d | Type::dt}, - {"tostartofquarter", Type::d | Type::dt}, {"tostartofmonth", Type::d | Type::dt}, {"tomonday", Type::d | Type::dt}, - {"tostartoffiveminute", Type::dt}, {"tostartoftenminutes", Type::dt}, {"tostartoffifteenminutes", Type::d | Type::dt}, - {"tostartofinterval", Type::d | Type::dt}, {"totime", Type::d | Type::dt}, {"torelativehonthnum", Type::d | Type::dt}, - {"torelativeweeknum", Type::d | Type::dt}, {"torelativedaynum", Type::d | Type::dt}, {"torelativehournum", Type::d | Type::dt}, - {"torelativeminutenum", Type::d | Type::dt}, {"torelativesecondnum", Type::d | Type::dt}, {"datediff", Type::d | Type::dt}, - {"formatdatetime", Type::dt} -}; - - -class Column -{ -public: - TableAndColumn name; - std::set equals; - std::set values; - ColumnType type = Type::all; - bool is_array = false; - - Column() = default; - - explicit Column(const std::string & column_name) - { - name = std::make_pair("", column_name); - type = Type::all; - } - - void merge(Column other) - { - if (name.second.empty()) - name = other.name; - equals.insert(other.equals.begin(), other.equals.end()); - values.insert(other.values.begin(), other.values.end()); - type &= other.type; - is_array |= other.is_array; - } - - void printType() const - { - if (type & Type::i) - std::cout << "I"; - if (type & Type::f) - std::cout << "F"; - if (type & Type::s) - std::cout << "S"; - if (type & Type::d) - std::cout << "D"; - if (type & Type::dt) - std::cout << "DT"; - if (is_array) - std::cout << "ARR"; - std::cout << "\n"; - } - - void print() - { - std::cout << name.first << "." << name.second << "\n"; - std::cout << "type: "; - printType(); - std::cout << "values:"; - for (const auto & val : values) - std::cout << " " << val; - std::cout << "\n"; - std::cout << "equal:"; - for (const auto & col : equals) - std::cout << " " << col.first << "." << col.second; - std::cout << "\n"; - } - - std::string generateOneValue() const - { - if (type & Type::i) - return randomInteger(); - - if (type & Type::f) - return randomFloat(); - - if (type & Type::d) - return randomDate(); - - if (type & Type::dt) - return randomDatetime(); - - if (type & Type::s) - return "'" + randomString(rng() % 40) + "'"; - - if (type & Type::b) - return "0"; - - return ""; - } - - bool generateValues(int amount = 0) - { - if (values.size() > 2 && amount == 0) - return false; - while (values.empty() or amount > 0) - { - amount -= 1; - if (is_array) - { - std::string v = "["; - for (unsigned int i = 0; i < static_cast(rng()) % 10 + 1; ++i) - { - if (i != 0) - v += ", "; - v += generateOneValue(); - } - v += "]"; - values.insert(v); - } - else - { - values.insert(generateOneValue()); - } - } - return true; - } - - void unifyType() - { - if (type & Type::i) - type = Type::i; - else if (type & Type::f) - type = Type::f; - else if (type & Type::d) - type = Type::d; - else if (type & Type::dt) - type = Type::dt; - else if (type & Type::s) - type = Type::s; - else if (type & Type::b) - type = Type::b; - else - throw std::runtime_error("Error in determination column type " + name.first + '.' + name.second); - } -}; - - -std::set> -decartMul( - std::set> & prev, - std::set & mul) -{ - std::set> result; - for (const auto & v : prev) - { - for (const auto & m : mul) - { - std::vector tmp = v; - tmp.push_back(m); - result.insert(tmp); - } - } - return result; -} - - -class Table -{ -public: - Table() = default; - - explicit Table(std::string table_name) - : name(table_name) {} - - std::string name; - std::set columns; - std::map column_description; - - bool columnExists(const std::string & column_name) const - { - return columns.contains(column_name); // || columns_maybe.contains(column_name); - } - - void addColumn(const std::string & column_name) - { - columns.insert(column_name); - } - - void setDescription(Column other) - { - column_description[other.name.second].merge(other); - } - - void print() - { - std::cout << "Table\n"; - std::cout << name << "\n"; - std::cout << "Columns:\n\n"; - for (const auto & column : columns) - { - std::cout << column << "\n"; - if (column_description.contains(column)) - column_description[column].print(); - std::cout << "\n"; - } - std::cout << "\n"; - } - - void merge(Table other) - { - name = other.name; - columns.insert(other.columns.begin(), other.columns.end()); - for (const auto & desc : other.column_description) - column_description[desc.first].merge(desc.second); - } - - std::string createQuery() - { - std::string create; - std::string db, _; - std::tie(db, _) = get_table_a_column(name); - create = "CREATE DATABASE IF NOT EXISTS " + db + ";\n\n"; - create += "CREATE TABLE IF NOT EXISTS " + name + " (\n"; - for (auto column = columns.begin(); column != columns.end(); ++column) - { - if (column != columns.begin()) - create += ", \n"; - create += *column + " "; - create += column_description[*column].is_array ? "Array(" : ""; - create += type_definition[column_description[*column].type]; - create += column_description[*column].is_array ? ")" : ""; - } - create += "\n) ENGINE = Log;\n\n"; - return create; - } - - std::string insertQuery() - { - std::string insert = "INSERT INTO " + name + "\n"; - insert += "("; - std::set> values = {std::vector(0)}; - for (auto column = columns.begin(); column != columns.end(); ++column) - { - if (column != columns.begin()) - insert += ", "; - insert += *column; - values = decartMul(values, column_description[*column].values); - } - insert += ") VALUES \n"; - for (auto val_set_iter = values.begin(); val_set_iter != values.end(); - ++val_set_iter) - { - if (val_set_iter != values.begin()) - insert += ",\n"; - auto val_set = *val_set_iter; - insert += "("; - for (auto val = val_set.begin(); val != val_set.end(); ++val) - { - if (val != val_set.begin()) - insert += ", "; - insert += *val; - } - insert += ")"; - } - insert += ";\n\n"; - return insert; - } -}; - - -class TableList -{ -public: - std::string main_table; - std::map aliases; - std::unordered_map tables; - std::set nested; - - bool tableExists(const std::string & table_name) const - { - return tables.contains(table_name); - } - - void addColumn(std::string full_column) - { - std::string table, column; - std::tie(table, column) = get_table_a_column(full_column); - if (!table.empty()) - { - if (tables.contains(table)) - { - tables[table].addColumn(column); - return; - } - if (aliases.contains(table)) - { - tables[aliases[table]].addColumn(column); - return; - } - nested.insert(table); - } - tables[main_table].addColumn(full_column); - } - - void addTable(std::string table_name) - { - if (tables.contains(table_name)) - return; - - tables[table_name] = Table(table_name); - if (main_table.empty()) - main_table = table_name; - } - - void addDescription(const Column & description) - { - std::string table = description.name.first; - if (tables.contains(table)) - tables[table].setDescription(description); - } - - TableAndColumn getTable(std::string full_column) const - { - std::string table, column; - std::tie(table, column) = get_table_a_column(full_column); - if (!table.empty()) - { - if (tables.contains(table)) - return std::make_pair(table, column); - - if (aliases.contains(table)) - { - table = aliases.find(table)->second; - return std::make_pair(table, column); - } - } - return std::make_pair(main_table, full_column); - } - - void print() - { - for (auto & table : tables) - { - table.second.print(); - std::cout << "\n"; - } - } - - void merge(TableList other) - { - for (const auto & table : other.tables) - tables[table.first].merge(table.second); - nested.insert(other.nested.begin(), other.nested.end()); - if (main_table.empty()) - main_table = other.main_table; - } -}; - -std::string getAlias(DB::ASTPtr ch) -{ - auto x = std::dynamic_pointer_cast(ch); - if (x) - return x->alias; - - for (const auto & child : (*ch).children) - { - auto alias = getAlias(child); - if (!alias.empty()) - return alias; - } - return ""; -} - -using FuncHandler = std::function &)>; -std::map handlers = {}; - -FuncRet arrayJoinFunc(DB::ASTPtr ch, std::map & columns) -{ - auto x = std::dynamic_pointer_cast(ch); - if (x) - { - std::set indents = {}; - for (auto & arg : x->arguments->children) - { - auto ident = std::dynamic_pointer_cast(arg); - if (ident) - indents.insert(ident->name()); - } - for (const auto & indent : indents) - { - auto c = Column(indent); - c.type = Type::all; - c.is_array = true; - if (columns.contains(indent)) - columns[indent].merge(c); - else - columns[indent] = c; - } - FuncRet r(Type::all, ""); - return r; - } - return FuncRet(); -} - -FuncRet inFunc(DB::ASTPtr ch, std::map & columns) -{ - auto x = std::dynamic_pointer_cast(ch); - if (x) - { - std::set indents{}; - std::set values{}; - ColumnType type_value = Type::all; - - for (auto & arg : x->arguments->children) - { - auto ident = std::dynamic_pointer_cast(arg); - if (ident) - { - indents.insert(ident->name()); - } - auto literal = std::dynamic_pointer_cast(arg); - if (literal) - { - ColumnType type = type_cast(literal->value.getType()); - - auto routine = [&](const auto & arr_values) - { - for (auto & val : arr_values) - { - type = type_cast(val.getType()); - if (type == Type::s || type == Type::d || type == Type::dt) - type = time_type(applyVisitor(DB::FieldVisitorToString(), val)); - type_value &= type; - values.insert(applyVisitor(DB::FieldVisitorToString(), val)); - } - }; - - if (type & Type::a) - { - auto arr_values = literal->value.get(); - routine(arr_values); - } - - if (type & Type::a) - { - auto arr_values = literal->value.get(); - routine(arr_values); - } - } - auto subfunc = std::dynamic_pointer_cast(arg); - if (subfunc) - { - FuncHandler f; - auto arg_func_name = std::dynamic_pointer_cast(arg)->name; - if (handlers.contains(arg_func_name)) - f = handlers[arg_func_name]; - else - f = handlers[""]; - FuncRet ret = f(arg, columns); - if (!ret.value.empty()) - { - values.insert(ret.value); - } - type_value &= ret.type; - } - } - for (const auto & indent : indents) - { - auto c = Column(indent); - c.type = type_value; - c.values.insert(values.begin(), values.end()); - c.generateValues(1); - if (columns.contains(indent)) - columns[indent].merge(c); - else - columns[indent] = c; - } - FuncRet r(Type::b | Type::i, ""); - return r; - } - return FuncRet(); -} - -FuncRet arrayFunc(DB::ASTPtr ch, std::map & columns) -{ - auto x = std::dynamic_pointer_cast(ch); - if (x) - { - std::set indents = {}; - std::string value = "["; - ColumnType type_value = Type::i | Type::f | Type::d | Type::dt | Type::s; - bool no_indent = true; - for (const auto & arg : x->arguments->children) - { - auto ident = std::dynamic_pointer_cast(arg); - if (ident) - { - no_indent = false; - indents.insert(ident->name()); - } - auto literal = std::dynamic_pointer_cast(arg); - if (literal) - { - ColumnType type = type_cast(literal->value.getType()); - if (type == Type::s || type == Type::d || type == Type::dt) - type = time_type(value); - type_value &= type; - - if (value != "[") - value += ", "; - value += applyVisitor(DB::FieldVisitorToString(), literal->value); - } - } - for (const auto & indent : indents) - { - auto c = Column(indent); - c.type = type_value; - if (columns.contains(indent)) - columns[indent].merge(c); - else - columns[indent] = c; - } - value += ']'; - FuncRet r(type_value, ""); - r.is_array = true; - if (no_indent) - r.value = value; - return r; - } - return FuncRet(); -} -FuncRet arithmeticFunc(DB::ASTPtr ch, std::map & columns) -{ - auto x = std::dynamic_pointer_cast(ch); - if (x) - { - std::set indents = {}; - std::set values = {}; - ColumnType type_value = Type::i | Type::f | Type::d | Type::dt; - ColumnType args_types = 0; - bool no_indent = true; - for (auto & arg : x->arguments->children) - { - ColumnType type = 0; - auto ident = std::dynamic_pointer_cast(arg); - if (ident) - { - no_indent = false; - indents.insert(ident->name()); - } - auto literal = std::dynamic_pointer_cast(arg); - if (literal) - type = type_cast(literal->value.getType()); - auto subfunc = std::dynamic_pointer_cast(arg); - if (subfunc) - { - FuncHandler f; - auto arg_func_name = std::dynamic_pointer_cast(arg)->name; - if (handlers.contains(arg_func_name)) - f = handlers[arg_func_name]; - else - f = handlers[""]; - FuncRet ret = f(arg, columns); - type = ret.type; - } - args_types |= type; - } - if (args_types & (Type::d | Type::dt)) - type_value -= Type::f; - if (args_types & Type::f) - type_value -= Type::d | Type::dt; - for (const auto & indent : indents) - { - auto c = Column(indent); - c.type = type_value; - if (columns.contains(indent)) - columns[indent].merge(c); - else - columns[indent] = c; - } - ColumnType ret_type = 0; - if (args_types & Type::dt) - ret_type = Type::dt; - else if (args_types & Type::d) - ret_type = Type::d | Type::dt; - else if (args_types & Type::f) - ret_type = Type::f; - else - ret_type = Type::d | Type::f | Type::dt | Type::i; - FuncRet r(ret_type, ""); - if (no_indent) - { - DB::WriteBufferFromOwnString buf; - formatAST(*ch, buf); - r.value = buf.str(); - } - return r; - } - return FuncRet(); -} -FuncRet likeFunc(DB::ASTPtr ch, std::map & columns) -{ - auto x = std::dynamic_pointer_cast(ch); - if (x) - { - std::set indents = {}; - std::set values = {}; - ColumnType type_value = Type::s; - for (auto & arg : x->arguments->children) - { - auto ident = std::dynamic_pointer_cast(arg); - if (ident) - indents.insert(ident->name()); - auto literal = std::dynamic_pointer_cast(arg); - if (literal) - { - std::string value = applyVisitor(DB::FieldVisitorToString(), literal->value); - std::string example{}; - for (size_t i = 0; i != value.size(); ++i) - { - if (value[i] == '%') - example += randomString(rng() % 10); - else if (value[i] == '_') - example += randomString(1); - else - example += value[i]; - } - values.insert(example); - } - } - for (const auto & indent : indents) - { - auto c = Column(indent); - c.type = type_value; - c.values.insert(values.begin(), values.end()); - if (columns.contains(indent)) - columns[indent].merge(c); - else - columns[indent] = c; - } - FuncRet r(Type::b, ""); - return r; - } - return FuncRet(); -} - -FuncRet simpleFunc(DB::ASTPtr ch, std::map & columns) -{ - auto x = std::dynamic_pointer_cast(ch); - if (x) - { - std::set indents = {}; - std::set values = {}; - ColumnType type_value = Type::all; - bool is_array = false; - bool no_indent = true; - if (func_to_param_type.contains(boost::algorithm::to_lower_copy(x->name))) - { - type_value &= func_to_param_type[boost::algorithm::to_lower_copy(x->name)]; - is_array = func_to_param_type[boost::algorithm::to_lower_copy(x->name)] & Type::a; - } - for (const auto & arg : x->arguments->children) - { - ColumnType type = Type::all; - std::string value; - auto ident = std::dynamic_pointer_cast(arg); - if (ident) - { - no_indent = false; - indents.insert(ident->name()); - } - auto literal = std::dynamic_pointer_cast(arg); - if (literal) - { - value = applyVisitor(DB::FieldVisitorToString(), literal->value); - type = type_cast(literal->value.getType()); - is_array |= type & Type::a; - } - auto subfunc = std::dynamic_pointer_cast(arg); - if (subfunc) - { - FuncHandler f; - auto arg_func_name = std::dynamic_pointer_cast(arg)->name; - if (handlers.contains(arg_func_name)) - f = handlers[arg_func_name]; - else - f = handlers[""]; - FuncRet ret = f(arg, columns); - is_array |= ret.is_array; - type = ret.type; - value = ret.value; - if (value.empty()) - no_indent = false; - } - if (!value.empty()) - { - if (type == Type::i) - { - values.insert(value); - values.insert(value + " + " + randomInteger(1, 10)); - values.insert(value + " - " + randomInteger(1, 10)); - } - if (type == Type::f) - { - values.insert(value); - values.insert(value + " + " + randomFloat(1, 10)); - values.insert(value + " - " + randomFloat(1, 10)); - } - if (type & Type::s || type & Type::d || type & Type::dt) - { - if (type == Type::s) - type = time_type(value); - if (type == Type::s) - values.insert(value); - if (type & Type::d) - { - values.insert(value); - values.insert("toDate(" + value + ") + " + randomInteger(1, 10)); - values.insert("toDate(" + value + ") - " + randomInteger(1, 10)); - } - else if (type & Type::dt) - { - values.insert(value); - values.insert( - "toDateTime(" + value + ") + " + randomInteger(1, 10000)); - values.insert( - "toDateTime(" + value + ") - " + randomInteger(1, 10000)); - } - } - } - if (func_args_same_types.contains(boost::algorithm::to_lower_copy(x->name))) - type_value &= type; - } - for (const auto & indent : indents) - { - auto c = Column(indent); - c.type = type_value; - c.is_array = is_array; - if (func_args_same_types.contains( - boost::algorithm::to_lower_copy(x->name))) - c.values = values; - for (const auto & ind : indents) - if (ind != indent) - c.equals.insert(std::make_pair("", ind)); - - if (columns.contains(indent)) - columns[indent].merge(c); - else - columns[indent] = c; - } - if (func_to_return_type.contains(boost::algorithm::to_lower_copy(x->name))) - { - if (no_indent) - { - DB::WriteBufferFromOwnString buf; - formatAST(*ch, buf); - auto r = func_to_return_type[boost::algorithm::to_lower_copy(x->name)]; - r.value = buf.str(); - return r; - } - return func_to_return_type[boost::algorithm::to_lower_copy(x->name)]; - } - else if (func_to_param_type.contains( - boost::algorithm::to_lower_copy(x->name))) - { - if (no_indent) - { - DB::WriteBufferFromOwnString buf; - formatAST(*ch, buf); - return FuncRet( - func_to_param_type[boost::algorithm::to_lower_copy(x->name)], - buf.str()); - } - return FuncRet( - func_to_param_type[boost::algorithm::to_lower_copy(x->name)], - ""); - } - } - return FuncRet(); -} - -void processFunc(DB::ASTPtr ch, std::map & columns) -{ - auto x = std::dynamic_pointer_cast(ch); - if (x) - { - FuncHandler f; - auto arg_func_name = x->name; - if (handlers.contains(arg_func_name)) - f = handlers[arg_func_name]; - else - f = handlers[""]; - f(ch, columns); - } - else - { - for (const auto & child : (*ch).children) - processFunc(child, columns); - } -} - - -std::set getIndent(DB::ASTPtr ch) -{ - if (!ch) - return {}; - - std::set ret = {}; - auto x = std::dynamic_pointer_cast(ch); - if (x) - ret.insert(x->name()); - for (const auto & child : (*ch).children) - { - auto child_ind = getIndent(child); - ret.insert(child_ind.begin(), child_ind.end()); - } - return ret; -} - - -std::set getSelectIndent( - DB::ASTPtr asp, - std::set & column_alias) -{ - std::set ret = {}; - for (auto & ch : asp->children) - { - auto alias = getAlias(ch); - auto columns = getIndent(ch); - if (alias.empty()) - column_alias.insert(alias); - ret.insert(columns.begin(), columns.end()); - } - return ret; -} - - -std::set -connectedEqualityFind( - const Column & now, - std::map & columns_descriptions, - std::set & visited) -{ - std::set result; - for (const auto & column : now.equals) - if (!visited.contains(column)) - { - visited.insert(column); - auto sub_r = connectedEqualityFind( - columns_descriptions[column.first + "." + column.second], - columns_descriptions, - visited); - result.insert(sub_r.begin(), sub_r.end()); - } - result.insert(now.name); - return result; -} - - -std::map -unificateColumns( - std::map columns_descriptions, - const TableList & all_tables) -{ - for (auto & column : columns_descriptions) - { - std::set changed_equals; - for (const auto & eq : column.second.equals) - { - std::string t, c; - std::tie(t, c) = all_tables.getTable(eq.second); - changed_equals.insert(std::make_pair(t, c)); - } - column.second.equals = changed_equals; - } - std::map result; - for (auto & column : columns_descriptions) - { - std::string t, c; - std::tie(t, c) = all_tables.getTable(column.first); - column.second.name = std::make_pair(t, c); - result[t + "." + c].merge(column.second); - } - std::set visited; - for (auto & column : result) - if (!visited.contains(column.second.name)) - { - auto equal = connectedEqualityFind( - result[column.second.name.first + "." + column.second.name.second], - result, - visited); - for (const auto & c : equal) - result[c.first + "." + c.second].equals = equal; - } - for (auto & column : result) - for (const auto & e : column.second.equals) - column.second.merge(result[e.first + "." + e.second]); - - for (auto & column : result) - { - column.second.unifyType(); - if (column.second.generateValues()) - for (const auto & e : column.second.equals) - result[e.first + "." + e.second].merge(column.second); - - } - return result; -} - -std::vector getSelect(DB::ASTPtr vertex) -{ - auto z = std::dynamic_pointer_cast(vertex); - std::vector result; - if (z) - { - result.push_back(vertex); - return result; - } - - for (const auto & child : (*vertex).children) - { - auto v = getSelect(child); - result.insert(result.end(), v.begin(), v.end()); - } - return result; -} - - -void parseSelectQuery(DB::ASTPtr ast, TableList & all_tables) -{ - if (!ast) - throw std::runtime_error("Bad ASTPtr in parseSelectQuery" + StackTrace().toString()); - - auto select_ast = std::dynamic_pointer_cast(ast); - if (!select_ast) - { - std::cerr << "not select query"; - return; - } - std::set columns = {}; - - auto x = select_ast->tables(); - if (!x) - throw std::runtime_error("There is no tables in query. Nothing to generate."); - - for (auto & child : x->children) - { - auto ch = std::dynamic_pointer_cast(child); - auto table_expression_ast = std::dynamic_pointer_cast(ch->table_expression); - if (table_expression_ast && table_expression_ast->database_and_table_name) - { - auto table_name = *(getIndent(table_expression_ast->database_and_table_name).begin()); - all_tables.addTable(table_name); - auto alias = getAlias(ch); - if (!alias.empty()) - all_tables.aliases[alias] = table_name; - } - if (table_expression_ast && table_expression_ast->subquery) - { - for (const auto & select : getSelect(table_expression_ast->subquery)) - { - TableList local; - parseSelectQuery(select, local); - all_tables.merge(local); - } - } - - if (ch->table_join) - { - auto jch = std::dynamic_pointer_cast(ch->table_join); - if (jch->using_expression_list) - { - auto join_columns = getIndent(jch->using_expression_list); - columns.insert(join_columns.begin(), join_columns.end()); - } - else if (jch->on_expression) - { - auto join_columns = getIndent(jch->on_expression); - columns.insert(join_columns.begin(), join_columns.end()); - } - } - } - - std::set column_aliases; - auto select_columns = getSelectIndent(select_ast->select(), column_aliases); - columns.insert(select_columns.begin(), select_columns.end()); - - auto where_columns = getIndent(select_ast->where()); - columns.insert(where_columns.begin(), where_columns.end()); - - auto groupby_columns = getIndent(select_ast->groupBy()); - columns.insert(groupby_columns.begin(), groupby_columns.end()); - - auto orderby_columns = getIndent(select_ast->orderBy()); - columns.insert(orderby_columns.begin(), orderby_columns.end()); - - auto having_columns = getIndent(select_ast->having()); - columns.insert(having_columns.begin(), having_columns.end()); - - std::map columns_descriptions; - processFunc(ast, columns_descriptions); - - for (const auto & column : columns) - if (!column_aliases.contains(column)) - { - if (!columns_descriptions.contains(column)) - columns_descriptions[column] = Column(column); - all_tables.addColumn(column); - } - - columns_descriptions = unificateColumns(columns_descriptions, all_tables); - for (auto & column : columns_descriptions) - all_tables.addDescription(column.second); -} - - -TableList getTablesFromSelect(std::vector queries) -{ - TableList result; - for (std::string & query : queries) - { - DB::ParserQueryWithOutput parser(query.data() + query.size()); - DB::ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, 0); - for (auto & select : getSelect(ast)) - { - TableList local; - parseSelectQuery(select, local); - result.merge(local); - } - } - return result; -} - -int main(int argc, const char *argv[]) -{ - try - { - po::options_description desc("Allowed options"); - desc.add_options() - ("help,h", "Display greeting and allowed options.") - ("input,i", po::value(), "Input filename.") - ("output,o", po::value(), "Output filename."); - - po::variables_map vm; - po::store(po::parse_command_line(argc, argv, desc), vm); - po::notify(vm); - - if (vm.contains("help") || vm.contains("h")) - { - std::cout << "Hello! It is datasets generator for ClickHouse's queries." << std::endl; - std::cout << "Put some query as an input and it will produce queries for table creating and filling." << std::endl; - std::cout << "After that your query could be executed on this tables." << std::endl; - std::cout << desc << std::endl; - return 1; - } - if (vm.contains("input")) - if (!freopen(vm["input"].as().c_str(), "r", stdin)) - std::cout << "Error while input." << std::endl; - if (vm.contains("output")) - if (!freopen(vm["output"].as().c_str(), "w", stdout)) - std::cout << "Error while output." << std::endl; - if (vm.empty()) - std::cout << "Copy your queries (with semicolons) here, press Enter and Ctrl+D." << std::endl; - } - catch (...) - { - std::cerr << "Got error while parse command line arguments: " << DB::getCurrentExceptionMessage(true) << std::endl; - throw; - } - - handlers["plus"] = arithmeticFunc; - handlers["minus"] = arithmeticFunc; - handlers["like"] = likeFunc; - handlers["array"] = arrayFunc; - handlers["in"] = inFunc; - handlers[""] = simpleFunc; - - std::vector queries; - std::string in; - std::string query{}; - while (getline(std::cin, in)) - { - /// Skip comments - if (in.find("--") != std::string::npos) - continue; - - query += in + " "; - - if (in.find(';') != std::string::npos) - { - queries.push_back(query); - query = ""; - } - } - - try - { - auto result = getTablesFromSelect(queries); - - for (auto & table : result.tables) - { - std::cout << table.second.createQuery(); - std::cout << table.second.insertQuery(); - } - - for (auto & q: queries) - std::cout << q << std::endl; - } - catch (std::string & e) - { - std::cerr << "Exception: " << e << std::endl; - } -} diff --git a/utils/iotest/CMakeLists.txt b/utils/iotest/CMakeLists.txt deleted file mode 100644 index dc163ac81c4..00000000000 --- a/utils/iotest/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ - -proton_add_executable (iotest iotest.cpp ${SRCS}) -target_link_libraries (iotest PRIVATE clickhouse_common_io) - -proton_add_executable (iotest_nonblock iotest_nonblock.cpp ${SRCS}) -target_link_libraries (iotest_nonblock PRIVATE clickhouse_common_io) - -proton_add_executable (iotest_aio iotest_aio.cpp ${SRCS}) -target_link_libraries (iotest_aio PRIVATE clickhouse_common_io) diff --git a/utils/iotest/iotest.cpp b/utils/iotest/iotest.cpp deleted file mode 100644 index f23ee6929fa..00000000000 --- a/utils/iotest/iotest.cpp +++ /dev/null @@ -1,198 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include -#include -#include -#include - - -namespace DB -{ - namespace ErrorCodes - { - extern const int CANNOT_OPEN_FILE; - extern const int CANNOT_CLOSE_FILE; - extern const int CANNOT_READ_FROM_FILE_DESCRIPTOR; - extern const int CANNOT_WRITE_TO_FILE_DESCRIPTOR; - } -} - - -enum Mode -{ - MODE_NONE = 0, - MODE_READ = 1, - MODE_WRITE = 2, - MODE_ALIGNED = 4, - MODE_DIRECT = 8, - MODE_SYNC = 16, -}; - - -void thread(int fd, int mode, size_t min_offset, size_t max_offset, size_t block_size, size_t count) -{ - using namespace DB; - - Memory<> direct_buf(block_size, ::getPageSize()); - std::vector simple_buf(block_size); - - char * buf; - if ((mode & MODE_DIRECT)) - buf = direct_buf.data(); - else - buf = &simple_buf[0]; - - pcg64 rng(randomSeed()); - - for (size_t i = 0; i < count; ++i) - { - uint64_t rand_result1 = rng(); - uint64_t rand_result2 = rng(); - uint64_t rand_result3 = rng(); - - size_t rand_result = rand_result1 ^ (rand_result2 << 22) ^ (rand_result3 << 43); - size_t offset; - if ((mode & MODE_DIRECT) || (mode & MODE_ALIGNED)) - offset = min_offset + rand_result % ((max_offset - min_offset) / block_size) * block_size; - else - offset = min_offset + rand_result % (max_offset - min_offset - block_size + 1); - - if (mode & MODE_READ) - { - if (static_cast(block_size) != pread(fd, buf, block_size, offset)) - throwFromErrno("Cannot read", ErrorCodes::CANNOT_READ_FROM_FILE_DESCRIPTOR); - } - else - { - if (static_cast(block_size) != pwrite(fd, buf, block_size, offset)) - throwFromErrno("Cannot write", ErrorCodes::CANNOT_WRITE_TO_FILE_DESCRIPTOR); - } - } -} - - -int mainImpl(int argc, char ** argv) -{ - using namespace DB; - - const char * file_name = nullptr; - int mode = MODE_NONE; - UInt64 min_offset = 0; - UInt64 max_offset = 0; - UInt64 block_size = 0; - UInt64 threads = 0; - UInt64 count = 0; - - if (argc != 8) - { - std::cerr << "Usage: " << argv[0] << " file_name (r|w)[a][d][s] min_offset max_offset block_size threads count" << std::endl << - "a - aligned, d - direct, s - sync" << std::endl; - return 1; - } - - file_name = argv[1]; - min_offset = parse(argv[3]); - max_offset = parse(argv[4]); - block_size = parse(argv[5]); - threads = parse(argv[6]); - count = parse(argv[7]); - - for (int i = 0; argv[2][i]; ++i) - { - char c = argv[2][i]; - switch (c) - { - case 'r': - mode |= MODE_READ; - break; - case 'w': - mode |= MODE_WRITE; - break; - case 'a': - mode |= MODE_ALIGNED; - break; - case 'd': - mode |= MODE_DIRECT; - break; - case 's': - mode |= MODE_SYNC; - break; - default: - throw Poco::Exception("Invalid mode"); - } - } - - ThreadPool pool(threads); - - #ifndef OS_DARWIN - int fd = open(file_name, ((mode & MODE_READ) ? O_RDONLY : O_WRONLY) | ((mode & MODE_DIRECT) ? O_DIRECT : 0) | ((mode & MODE_SYNC) ? O_SYNC : 0)); - #else - int fd = open(file_name, ((mode & MODE_READ) ? O_RDONLY : O_WRONLY) | ((mode & MODE_SYNC) ? O_SYNC : 0)); - #endif - if (-1 == fd) - throwFromErrno("Cannot open file", ErrorCodes::CANNOT_OPEN_FILE); - #ifdef OS_DARWIN - if (mode & MODE_DIRECT) - if (fcntl(fd, F_NOCACHE, 1) == -1) - throwFromErrno("Cannot open file", ErrorCodes::CANNOT_CLOSE_FILE); - #endif - Stopwatch watch; - - for (size_t i = 0; i < threads; ++i) - pool.scheduleOrThrowOnError([=]{ thread(fd, mode, min_offset, max_offset, block_size, count); }); - pool.wait(); - - #if defined(OS_DARWIN) - fsync(fd); - #else - fdatasync(fd); - #endif - - watch.stop(); - - if (0 != close(fd)) - throwFromErrno("Cannot close file", ErrorCodes::CANNOT_CLOSE_FILE); - - std::cout << std::fixed << std::setprecision(2) - << "Done " << count << " * " << threads << " ops"; - if (mode & MODE_ALIGNED) - std::cout << " (aligned)"; - if (mode & MODE_DIRECT) - std::cout << " (direct)"; - if (mode & MODE_SYNC) - std::cout << " (sync)"; - std::cout << " in " << watch.elapsedSeconds() << " sec." - << ", " << count * threads / watch.elapsedSeconds() << " ops/sec." - << ", " << count * threads * block_size / watch.elapsedSeconds() / 1000000 << " MB/sec." - << std::endl; - - return 0; -} - - -int main(int argc, char ** argv) -{ - try - { - return mainImpl(argc, argv); - } - catch (const Poco::Exception & e) - { - std::cerr << e.what() << ", " << e.message() << std::endl; - return 1; - } -} diff --git a/utils/iotest/iotest_aio.cpp b/utils/iotest/iotest_aio.cpp deleted file mode 100644 index 4db76b1eb2d..00000000000 --- a/utils/iotest/iotest_aio.cpp +++ /dev/null @@ -1,203 +0,0 @@ -#if !defined(OS_LINUX) -int main(int, char **) { return 0; } -#else - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ - namespace ErrorCodes - { - extern const int CANNOT_OPEN_FILE; - extern const int CANNOT_CLOSE_FILE; - extern const int CANNOT_IO_SUBMIT; - extern const int CANNOT_IO_GETEVENTS; - } -} - - -enum Mode -{ - MODE_READ = 1, - MODE_WRITE = 2, -}; - - -void thread(int fd, int mode, size_t min_offset, size_t max_offset, size_t block_size, size_t buffers_count, size_t count) -{ - using namespace DB; - - AIOContext ctx; - - std::vector> buffers(buffers_count); - for (size_t i = 0; i < buffers_count; ++i) - buffers[i] = Memory<>(block_size, ::getPageSize()); - - pcg64_fast rng(randomSeed()); - - size_t in_progress = 0; - size_t blocks_sent = 0; - std::vector buffer_used(buffers_count, false); - std::vector iocbs(buffers_count); - std::vector query_cbs; - std::vector events(buffers_count); - - while (blocks_sent < count || in_progress > 0) - { - /// Prepare queries. - query_cbs.clear(); - for (size_t i = 0; i < buffers_count; ++i) - { - if (blocks_sent >= count || in_progress >= buffers_count) - break; - - if (buffer_used[i]) - continue; - - buffer_used[i] = true; - ++blocks_sent; - ++in_progress; - - char * buf = buffers[i].data(); - - uint64_t rand_result1 = rng(); - uint64_t rand_result2 = rng(); - uint64_t rand_result3 = rng(); - - size_t rand_result = rand_result1 ^ (rand_result2 << 22) ^ (rand_result3 << 43); - size_t offset = min_offset + rand_result % ((max_offset - min_offset) / block_size) * block_size; - - iocb & cb = iocbs[i]; - memset(&cb, 0, sizeof(cb)); - cb.aio_buf = reinterpret_cast(buf); - cb.aio_fildes = fd; - cb.aio_nbytes = block_size; - cb.aio_offset = offset; - cb.aio_data = static_cast(i); - - if (mode == MODE_READ) - { - cb.aio_lio_opcode = IOCB_CMD_PREAD; - } - else - { - cb.aio_lio_opcode = IOCB_CMD_PWRITE; - } - - query_cbs.push_back(&cb); - } - - /// Send queries. - if (io_submit(ctx.ctx, query_cbs.size(), &query_cbs[0]) < 0) - throwFromErrno("io_submit failed", ErrorCodes::CANNOT_IO_SUBMIT); - - /// Receive answers. If we have something else to send, then receive at least one answer (after that send them), otherwise wait all answers. - memset(&events[0], 0, buffers_count * sizeof(events[0])); - int evs = io_getevents(ctx.ctx, (blocks_sent < count ? 1 : in_progress), buffers_count, &events[0], nullptr); - if (evs < 0) - throwFromErrno("io_getevents failed", ErrorCodes::CANNOT_IO_GETEVENTS); - - for (int i = 0; i < evs; ++i) - { - int b = static_cast(events[i].data); - if (events[i].res != static_cast(block_size)) - throw Poco::Exception("read/write error"); - --in_progress; - buffer_used[b] = false; - } - } -} - - -int mainImpl(int argc, char ** argv) -{ - using namespace DB; - - const char * file_name = nullptr; - int mode = MODE_READ; - UInt64 min_offset = 0; - UInt64 max_offset = 0; - UInt64 block_size = 0; - UInt64 buffers_count = 0; - UInt64 threads_count = 0; - UInt64 count = 0; - - if (argc != 9) - { - std::cerr << "Usage: " << argv[0] << " file_name r|w min_offset max_offset block_size threads buffers count" << std::endl; - return 1; - } - - file_name = argv[1]; - if (argv[2][0] == 'w') - mode = MODE_WRITE; - min_offset = parse(argv[3]); - max_offset = parse(argv[4]); - block_size = parse(argv[5]); - threads_count = parse(argv[6]); - buffers_count = parse(argv[7]); - count = parse(argv[8]); - - int fd = open(file_name, ((mode == MODE_READ) ? O_RDONLY : O_WRONLY) | O_DIRECT); - if (-1 == fd) - throwFromErrno("Cannot open file", ErrorCodes::CANNOT_OPEN_FILE); - - ThreadPool pool(threads_count); - - Stopwatch watch; - - for (size_t i = 0; i < threads_count; ++i) - pool.scheduleOrThrowOnError([=]{ thread(fd, mode, min_offset, max_offset, block_size, buffers_count, count); }); - pool.wait(); - - watch.stop(); - - if (0 != close(fd)) - throwFromErrno("Cannot close file", ErrorCodes::CANNOT_CLOSE_FILE); - - std::cout << std::fixed << std::setprecision(2) - << "Done " << count << " * " << threads_count << " ops"; - std::cout << " in " << watch.elapsedSeconds() << " sec." - << ", " << count * threads_count / watch.elapsedSeconds() << " ops/sec." - << ", " << count * threads_count * block_size / watch.elapsedSeconds() / 1000000 << " MB/sec." - << std::endl; - - return 0; -} - - -int main(int argc, char ** argv) -{ - try - { - return mainImpl(argc, argv); - } - catch (const Poco::Exception & e) - { - std::cerr << e.what() << ", " << e.message() << std::endl; - return 1; - } -} -#endif diff --git a/utils/iotest/iotest_nonblock.cpp b/utils/iotest/iotest_nonblock.cpp deleted file mode 100644 index f78d0e1e7b0..00000000000 --- a/utils/iotest/iotest_nonblock.cpp +++ /dev/null @@ -1,177 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#if defined (OS_LINUX) -# include -#endif - - -namespace DB -{ - namespace ErrorCodes - { - extern const int CANNOT_OPEN_FILE; - extern const int CANNOT_CLOSE_FILE; - extern const int CANNOT_READ_FROM_FILE_DESCRIPTOR; - extern const int CANNOT_WRITE_TO_FILE_DESCRIPTOR; - extern const int CANNOT_FSYNC; - extern const int SYSTEM_ERROR; - } -} - - -enum Mode -{ - MODE_READ, - MODE_WRITE, -}; - - -int mainImpl(int argc, char ** argv) -{ - using namespace DB; - - const char * file_name = nullptr; - Mode mode = MODE_READ; - UInt64 min_offset = 0; - UInt64 max_offset = 0; - UInt64 block_size = 0; - UInt64 descriptors = 0; - UInt64 count = 0; - - if (argc != 8) - { - std::cerr << "Usage: " << argv[0] << " file_name r|w min_offset max_offset block_size descriptors count" << std::endl; - return 1; - } - - file_name = argv[1]; - min_offset = parse(argv[3]); - max_offset = parse(argv[4]); - block_size = parse(argv[5]); - descriptors = parse(argv[6]); - count = parse(argv[7]); - - if (!strcmp(argv[2], "r")) - mode = MODE_READ; - else if (!strcmp(argv[2], "w")) - mode = MODE_WRITE; - else - throw Poco::Exception("Invalid mode"); - - std::vector fds(descriptors); - for (size_t i = 0; i < descriptors; ++i) - { - fds[i] = open(file_name, O_SYNC | ((mode == MODE_READ) ? O_RDONLY : O_WRONLY)); - if (-1 == fds[i]) - throwFromErrno("Cannot open file", ErrorCodes::CANNOT_OPEN_FILE); - } - - std::vector buf(block_size); - - pcg64 rng(randomSeed()); - - Stopwatch watch; - - std::vector polls(descriptors); - - for (size_t i = 0; i < descriptors; ++i) - { - polls[i].fd = fds[i]; - polls[i].events = (mode == MODE_READ) ? POLLIN : POLLOUT; - polls[i].revents = 0; - } - - size_t ops = 0; - while (ops < count) - { - if (poll(polls.data(), static_cast(descriptors), -1) <= 0) - throwFromErrno("poll failed", ErrorCodes::SYSTEM_ERROR); - for (size_t i = 0; i < descriptors; ++i) - { - if (!polls[i].revents) - continue; - - if (polls[i].revents != polls[i].events) - throw Poco::Exception("revents indicates error"); - polls[i].revents = 0; - ++ops; - - uint64_t rand_result1 = rng(); - uint64_t rand_result2 = rng(); - uint64_t rand_result3 = rng(); - - size_t rand_result = rand_result1 ^ (rand_result2 << 22) ^ (rand_result3 << 43); - size_t offset; - offset = min_offset + rand_result % ((max_offset - min_offset) / block_size) * block_size; - - if (mode == MODE_READ) - { - if (static_cast(block_size) != pread(fds[i], &buf[0], block_size, offset)) - throwFromErrno("Cannot read", ErrorCodes::CANNOT_READ_FROM_FILE_DESCRIPTOR); - } - else - { - if (static_cast(block_size) != pwrite(fds[i], &buf[0], block_size, offset)) - throwFromErrno("Cannot write", ErrorCodes::CANNOT_WRITE_TO_FILE_DESCRIPTOR); - } - } - } - - for (size_t i = 0; i < descriptors; ++i) - { -#if defined(OS_DARWIN) - if (fsync(fds[i])) - throwFromErrno("Cannot fsync", ErrorCodes::CANNOT_FSYNC); -#else - if (fdatasync(fds[i])) - throwFromErrno("Cannot fdatasync", ErrorCodes::CANNOT_FSYNC); -#endif - } - - watch.stop(); - - for (size_t i = 0; i < descriptors; ++i) - { - if (0 != close(fds[i])) - throwFromErrno("Cannot close file", ErrorCodes::CANNOT_CLOSE_FILE); - } - - std::cout << std::fixed << std::setprecision(2) - << "Done " << count << " ops" << " in " << watch.elapsedSeconds() << " sec." - << ", " << count / watch.elapsedSeconds() << " ops/sec." - << ", " << count * block_size / watch.elapsedSeconds() / 1000000 << " MB/sec." - << std::endl; - - return 0; -} - - -int main(int argc, char ** argv) -{ - try - { - return mainImpl(argc, argv); - } - catch (const Poco::Exception & e) - { - std::cerr << e.what() << ", " << e.message() << std::endl; - return 1; - } -}