From a92d3b9a5ff2030dc8649b6aca619c4340f58ff4 Mon Sep 17 00:00:00 2001 From: usernamedt Date: Mon, 13 Feb 2023 15:00:31 +0800 Subject: [PATCH 01/22] Movable DataBase Locales for Cloudberry We inherited this issue from PostgreSQL. PostgreSQL uses glibc to sort strings. In version glibc=2.28, collations broke down badly (in general, there are no guarantees when updating glibc). Changing collations breaks indexes. Similarly, a cluster with different collations also behaves unpredictably. What and when something has changed in glibc can be found on https://github.com/ardentperf/glibc-unicode-sorting Also there is special postgresql-wiki https://wiki.postgresql.org/wiki/Locale_data_changes And you tube video https://www.youtube.com/watch?v=0E6O-V8Jato In short, the issue can be seen through the use of bash: ( echo "1-1"; echo "11" ) | LC_COLLATE=en_US.UTF-8 sort gives the different results in ubunru 18.04 and 22.04. There is no way to solve the problem other than by not changing the symbol order. We freeze symbol order and use it instead of glibc. Here the solution https://github.com/postgredients/mdb-locales. In this PR I have added PostgreSQL patch that replaces all glibc locale-related calls with a calls to an external libary. It activates using new configure parameter --with-mdblocales, which is off by default. Using custom locales needs libmdblocales1 package and mdb-locales package with symbol table. Build needs libmdblocales-dev package with headers. --- configure | 97 ++++++- configure.ac | 17 ++ .../src/cpp/storage/oper/pax_oper.cc | 5 +- gpcontrib/orafce/others.c | 9 +- .../src/unittest/gpos/string/CWStringTest.cpp | 7 +- src/backend/utils/adt/Makefile | 3 +- src/backend/utils/adt/mdb.c | 37 +++ src/backend/utils/adt/pg_locale.c | 63 ++-- src/backend/utils/mb/mbutils.c | 3 +- src/bin/initdb/initdb.c | 14 +- src/bin/pg_upgrade/check.c | 9 +- src/common/exec.c | 4 +- src/include/catalog/pg_proc.dat | 4 +- src/include/common/mdb_locale.h | 41 +++ src/include/pg_config.h.in | 6 + src/interfaces/ecpg/ecpglib/connect.c | 3 +- src/interfaces/ecpg/ecpglib/descriptor.c | 8 +- src/interfaces/ecpg/ecpglib/execute.c | 7 +- src/interfaces/libpq/Makefile | 2 +- src/pl/plperl/plperl.c | 19 +- src/port/chklocale.c | 10 +- src/test/locale/test-ctype.c | 4 +- src/test/regress/input/misc.source | 5 + src/test/regress/output/misc.source | 7 + src/test/regress/sql/misc.sql | 271 ++++++++++++++++++ 25 files changed, 582 insertions(+), 73 deletions(-) create mode 100644 src/backend/utils/adt/mdb.c create mode 100644 src/include/common/mdb_locale.h create mode 100644 src/test/regress/sql/misc.sql diff --git a/configure b/configure index 44d3fae95b6..8778ab19c41 100755 --- a/configure +++ b/configure @@ -698,6 +698,7 @@ BISON MKDIR_P LN_S TAR +USE_MDBLOCALES install_bin INSTALL_DATA INSTALL_SCRIPT @@ -943,6 +944,7 @@ with_rt with_libcurl with_apr_config with_gnu_ld +with_mdblocales with_ssl with_openssl enable_openssl_redirect @@ -1690,6 +1692,7 @@ Optional Packages: --without-libcurl do not use libcurl --with-apr-config=PATH path to apr-1-config utility --with-gnu-ld assume the C compiler uses GNU ld [default=no] + --without-mdblocales build without MDB locales --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -2906,7 +2909,6 @@ PG_PACKAGE_VERSION=14.4 - ac_aux_dir= for ac_dir in config "$srcdir"/config; do if test -f "$ac_dir/install-sh"; then @@ -12124,6 +12126,38 @@ case $INSTALL in esac +# +# MDB locales +# + + + + +# Check whether --with-mdblocales was given. +if test "${with_mdblocales+set}" = set; then : + withval=$with_mdblocales; + case $withval in + yes) + +$as_echo "#define USE_MDBLOCALES 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-mdblocales option" "$LINENO" 5 + ;; + esac + +else + with_mdblocales=no + +fi + + + + if test -z "$TAR"; then for ac_prog in tar do @@ -12760,6 +12794,56 @@ $as_echo "${python_libspec} ${python_additional_libs}" >&6; } +fi + +if test "$with_mdblocales" = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for mdb_setlocale in -lmdblocales" >&5 +$as_echo_n "checking for mdb_setlocale in -lmdblocales... " >&6; } +if ${ac_cv_lib_mdblocales_mdb_setlocale+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lmdblocales $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char mdb_setlocale (); +int +main () +{ +return mdb_setlocale (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_mdblocales_mdb_setlocale=yes +else + ac_cv_lib_mdblocales_mdb_setlocale=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_mdblocales_mdb_setlocale" >&5 +$as_echo "$ac_cv_lib_mdblocales_mdb_setlocale" >&6; } +if test "x$ac_cv_lib_mdblocales_mdb_setlocale" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBMDBLOCALES 1 +_ACEOF + + LIBS="-lmdblocales $LIBS" + +else + as_fn_error $? "mdblocales library not found" "$LINENO" 5 +fi + fi if test x"$cross_compiling" = x"yes" && test -z "$with_system_tzdata"; then @@ -16981,6 +17065,17 @@ fi done +fi + +if test "$with_mdblocales" = yes; then + ac_fn_c_check_header_mongrel "$LINENO" "mdblocales.h" "ac_cv_header_mdblocales_h" "$ac_includes_default" +if test "x$ac_cv_header_mdblocales_h" = xyes; then : + +else + as_fn_error $? "mdblocales header not found." "$LINENO" 5 +fi + + fi if test "$with_gssapi" = yes ; then diff --git a/configure.ac b/configure.ac index 0d0529fc35f..727d8960a4c 100644 --- a/configure.ac +++ b/configure.ac @@ -1448,6 +1448,14 @@ case $INSTALL in esac AC_SUBST(install_bin) +# +# MDB locales +# + +PGAC_ARG_BOOL(with, mdblocales, yes, [build without MDB locales], + [AC_DEFINE([USE_MDBLOCALES], 1, [Define to 1 to build with MDB locales. (--with-mdblocales)])]) +AC_SUBST(USE_MDBLOCALES) + PGAC_PATH_PROGS(TAR, tar) AC_PROG_LN_S AC_PROG_MKDIR_P @@ -1606,6 +1614,11 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_mdblocales" = yes; then + AC_CHECK_LIB(mdblocales, mdb_setlocale, [], + [AC_MSG_ERROR([mdblocales library not found])]) +fi + if test "$enable_external_fts" = yes; then AC_CHECK_LIB(jansson, jansson_version_str, [], [AC_MSG_ERROR([jansson library not found or version is too old, version must >= 2.13])]) @@ -1985,6 +1998,10 @@ if test "$with_lz4" = yes; then AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])]) fi +if test "$with_mdblocales" = yes; then + AC_CHECK_HEADER(mdblocales.h, [], [AC_MSG_ERROR([mdblocales header not found.])]) +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/contrib/pax_storage/src/cpp/storage/oper/pax_oper.cc b/contrib/pax_storage/src/cpp/storage/oper/pax_oper.cc index 44d4e49d7f8..d08c7a445b9 100644 --- a/contrib/pax_storage/src/cpp/storage/oper/pax_oper.cc +++ b/contrib/pax_storage/src/cpp/storage/oper/pax_oper.cc @@ -25,6 +25,7 @@ *------------------------------------------------------------------------- */ +#include "common/mdb_locale.h" #include "storage/oper/pax_oper.h" #include "comm/cbdb_wrappers.h" @@ -588,9 +589,9 @@ static inline bool LocaleIsC(Oid collation) { return (bool)result; } - localeptr = setlocale(LC_COLLATE, NULL); + localeptr = SETLOCALE(LC_COLLATE, NULL); CBDB_CHECK(localeptr, cbdb::CException::ExType::kExTypeCError, - fmt("Invalid locale, fail to `setlocale`, errno: %d", errno)); + fmt("Invalid locale, fail to `SETLOCALE`, errno: %d", errno)); if (strcmp(localeptr, "C") == 0 || // cut line strcmp(localeptr, "POSIX") == 0) { diff --git a/gpcontrib/orafce/others.c b/gpcontrib/orafce/others.c index 2fb612efe19..5bf8b650e4c 100644 --- a/gpcontrib/orafce/others.c +++ b/gpcontrib/orafce/others.c @@ -45,6 +45,7 @@ #include "utils/uuid.h" #include "orafce.h" #include "builtins.h" +#include "common/mdb_locale.h" /* * Source code for nlssort is taken from postgresql-nls-string @@ -322,7 +323,7 @@ _nls_run_strxfrm(text *string, text *locale) */ if (!lc_collate_cache) { - if ((lc_collate_cache = setlocale(LC_COLLATE, NULL))) + if ((lc_collate_cache = SETLOCALE(LC_COLLATE, NULL))) /* Make a copy of the locale name string. */ #ifdef _MSC_VER lc_collate_cache = _strdup(lc_collate_cache); @@ -364,7 +365,7 @@ _nls_run_strxfrm(text *string, text *locale) * If setlocale failed, we know the default stayed the same, * co we can safely elog. */ - if (!setlocale(LC_COLLATE, locale_str)) + if (!SETLOCALE(LC_COLLATE, locale_str)) elog(ERROR, "failed to set the requested LC_COLLATE value [%s]", locale_str); changed_locale = true; @@ -409,7 +410,7 @@ _nls_run_strxfrm(text *string, text *locale) /* * Set original locale */ - if (!setlocale(LC_COLLATE, lc_collate_cache)) + if (!SETLOCALE(LC_COLLATE, lc_collate_cache)) elog(FATAL, "failed to set back the default LC_COLLATE value [%s]", lc_collate_cache); } @@ -422,7 +423,7 @@ _nls_run_strxfrm(text *string, text *locale) /* * Set original locale */ - if (!setlocale(LC_COLLATE, lc_collate_cache)) + if (!SETLOCALE(LC_COLLATE, lc_collate_cache)) elog(FATAL, "failed to set back the default LC_COLLATE value [%s]", lc_collate_cache); pfree(locale_str); } diff --git a/src/backend/gporca/libgpos/server/src/unittest/gpos/string/CWStringTest.cpp b/src/backend/gporca/libgpos/server/src/unittest/gpos/string/CWStringTest.cpp index 60bccf59341..bb086954403 100644 --- a/src/backend/gporca/libgpos/server/src/unittest/gpos/string/CWStringTest.cpp +++ b/src/backend/gporca/libgpos/server/src/unittest/gpos/string/CWStringTest.cpp @@ -12,6 +12,7 @@ #include "unittest/gpos/string/CWStringTest.h" #include +#include "common/mdb_locale.h" #include "gpos/base.h" #include "gpos/error/CAutoTrace.h" @@ -177,18 +178,18 @@ CWStringTest::EresUnittest_AppendFormatInvalidLocale() CWStringDynamic *expected = GPOS_NEW(mp) CWStringDynamic(mp, GPOS_WSZ_LIT("UNKNOWN")); - CHAR *oldLocale = setlocale(LC_CTYPE, nullptr); + CHAR *oldLocale = SETLOCALE(LC_CTYPE, nullptr); CWStringDynamic *pstr1 = GPOS_NEW(mp) CWStringDynamic(mp); GPOS_RESULT eres = GPOS_OK; - setlocale(LC_CTYPE, "C"); + SETLOCALE(LC_CTYPE, "C"); pstr1->AppendFormat(GPOS_WSZ_LIT("%s"), (CHAR *) "ÃË", 123); pstr1->Equals(expected); // cleanup - setlocale(LC_CTYPE, oldLocale); + SETLOCALE(LC_CTYPE, oldLocale); GPOS_DELETE(pstr1); GPOS_DELETE(expected); diff --git a/src/backend/utils/adt/Makefile b/src/backend/utils/adt/Makefile index bd5479c546b..58dd15a6f8b 100644 --- a/src/backend/utils/adt/Makefile +++ b/src/backend/utils/adt/Makefile @@ -117,7 +117,8 @@ OBJS = \ windowfuncs.o \ xid.o \ xid8funcs.o \ - xml.o + xml.o \ + mdb.o jsonpath_scan.c: FLEXFLAGS = -CF -p -p jsonpath_scan.c: FLEX_NO_BACKUP=yes diff --git a/src/backend/utils/adt/mdb.c b/src/backend/utils/adt/mdb.c new file mode 100644 index 00000000000..e5c695de1b6 --- /dev/null +++ b/src/backend/utils/adt/mdb.c @@ -0,0 +1,37 @@ +/*------------------------------------------------------------------------- + * + * mdb.c + * mdb routines + * + * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/utils/adt/mdb.c + * + *------------------------------------------------------------------------- + */ + + +#include "postgres.h" +#include "fmgr.h" +#include "utils/fmgrprotos.h" + +/* + * mdb_admin_enabled + * Check that mdb locale patch is enabled + */ +Datum +mdb_locale_enabled(PG_FUNCTION_ARGS) +{ + bool res; + +#if USE_MDBLOCALES + res = true; +#else + res = false; +#endif + + PG_RETURN_BOOL(res); +} diff --git a/src/backend/utils/adt/pg_locale.c b/src/backend/utils/adt/pg_locale.c index 11392891538..a9acb875eee 100644 --- a/src/backend/utils/adt/pg_locale.c +++ b/src/backend/utils/adt/pg_locale.c @@ -66,6 +66,7 @@ #include "utils/memutils.h" #include "utils/pg_locale.h" #include "utils/syscache.h" +#include "common/mdb_locale.h" #ifdef USE_ICU #include @@ -147,7 +148,7 @@ pg_perm_setlocale(int category, const char *locale) const char *envvar; #ifndef WIN32 - result = setlocale(category, locale); + result = SETLOCALE(category, locale); #else /* @@ -165,7 +166,7 @@ pg_perm_setlocale(int category, const char *locale) } else #endif - result = setlocale(category, locale); + result = SETLOCALE(category, locale); #endif /* WIN32 */ if (result == NULL) @@ -252,7 +253,7 @@ check_locale(int category, const char *locale, char **canonname) if (canonname) *canonname = NULL; /* in case of failure */ - save = setlocale(category, NULL); + save = SETLOCALE(category, NULL); if (!save) return false; /* won't happen, we hope */ @@ -260,14 +261,14 @@ check_locale(int category, const char *locale, char **canonname) save = pstrdup(save); /* set the locale with setlocale, to see if it accepts it. */ - res = setlocale(category, locale); + res = SETLOCALE(category, locale); /* save canonical name if requested. */ if (res && canonname) *canonname = pstrdup(res); /* restore old value. */ - if (!setlocale(category, save)) + if (!SETLOCALE(category, save)) elog(WARNING, "failed to restore old locale \"%s\"", save); pfree(save); @@ -501,12 +502,12 @@ PGLC_localeconv(void) memset(&worklconv, 0, sizeof(worklconv)); /* Save prevailing values of monetary and numeric locales */ - save_lc_monetary = setlocale(LC_MONETARY, NULL); + save_lc_monetary = SETLOCALE(LC_MONETARY, NULL); if (!save_lc_monetary) elog(ERROR, "setlocale(NULL) failed"); save_lc_monetary = pstrdup(save_lc_monetary); - save_lc_numeric = setlocale(LC_NUMERIC, NULL); + save_lc_numeric = SETLOCALE(LC_NUMERIC, NULL); if (!save_lc_numeric) elog(ERROR, "setlocale(NULL) failed"); save_lc_numeric = pstrdup(save_lc_numeric); @@ -528,7 +529,7 @@ PGLC_localeconv(void) */ /* Save prevailing value of ctype locale */ - save_lc_ctype = setlocale(LC_CTYPE, NULL); + save_lc_ctype = SETLOCALE(LC_CTYPE, NULL); if (!save_lc_ctype) elog(ERROR, "setlocale(NULL) failed"); save_lc_ctype = pstrdup(save_lc_ctype); @@ -536,11 +537,11 @@ PGLC_localeconv(void) /* Here begins the critical section where we must not throw error */ /* use numeric to set the ctype */ - setlocale(LC_CTYPE, locale_numeric); + SETLOCALE(LC_CTYPE, locale_numeric); #endif /* Get formatting information for numeric */ - setlocale(LC_NUMERIC, locale_numeric); + SETLOCALE(LC_NUMERIC, locale_numeric); extlconv = localeconv(); /* Must copy data now in case setlocale() overwrites it */ @@ -550,11 +551,11 @@ PGLC_localeconv(void) #ifdef WIN32 /* use monetary to set the ctype */ - setlocale(LC_CTYPE, locale_monetary); + SETLOCALE(LC_CTYPE, locale_monetary); #endif /* Get formatting information for monetary */ - setlocale(LC_MONETARY, locale_monetary); + SETLOCALE(LC_MONETARY, locale_monetary); extlconv = localeconv(); /* Must copy data now in case setlocale() overwrites it */ @@ -584,12 +585,12 @@ PGLC_localeconv(void) * should fail. */ #ifdef WIN32 - if (!setlocale(LC_CTYPE, save_lc_ctype)) + if (!SETLOCALE(LC_CTYPE, save_lc_ctype)) elog(FATAL, "failed to restore LC_CTYPE to \"%s\"", save_lc_ctype); #endif - if (!setlocale(LC_MONETARY, save_lc_monetary)) + if (!SETLOCALE(LC_MONETARY, save_lc_monetary)) elog(FATAL, "failed to restore LC_MONETARY to \"%s\"", save_lc_monetary); - if (!setlocale(LC_NUMERIC, save_lc_numeric)) + if (!SETLOCALE(LC_NUMERIC, save_lc_numeric)) elog(FATAL, "failed to restore LC_NUMERIC to \"%s\"", save_lc_numeric); /* @@ -773,7 +774,7 @@ cache_locale_time(void) */ /* Save prevailing value of time locale */ - save_lc_time = setlocale(LC_TIME, NULL); + save_lc_time = SETLOCALE(LC_TIME, NULL); if (!save_lc_time) elog(ERROR, "setlocale(NULL) failed"); save_lc_time = pstrdup(save_lc_time); @@ -788,16 +789,16 @@ cache_locale_time(void) */ /* Save prevailing value of ctype locale */ - save_lc_ctype = setlocale(LC_CTYPE, NULL); + save_lc_ctype = SETLOCALE(LC_CTYPE, NULL); if (!save_lc_ctype) elog(ERROR, "setlocale(NULL) failed"); save_lc_ctype = pstrdup(save_lc_ctype); /* use lc_time to set the ctype */ - setlocale(LC_CTYPE, locale_time); + SETLOCALE(LC_CTYPE, locale_time); #endif - setlocale(LC_TIME, locale_time); + SETLOCALE(LC_TIME, locale_time); /* We use times close to current time as data for strftime(). */ timenow = time(NULL); @@ -846,10 +847,10 @@ cache_locale_time(void) * failure to do so is fatal. */ #ifdef WIN32 - if (!setlocale(LC_CTYPE, save_lc_ctype)) + if (!SETLOCALE(LC_CTYPE, save_lc_ctype)) elog(FATAL, "failed to restore LC_CTYPE to \"%s\"", save_lc_ctype); #endif - if (!setlocale(LC_TIME, save_lc_time)) + if (!SETLOCALE(LC_TIME, save_lc_time)) elog(FATAL, "failed to restore LC_TIME to \"%s\"", save_lc_time); /* @@ -1225,7 +1226,7 @@ check_strxfrm_bug(void) ereport(ERROR, (errcode(ERRCODE_SYSTEM_ERROR), errmsg_internal("strxfrm(), in locale \"%s\", writes past the specified array length", - setlocale(LC_COLLATE, NULL)), + SETLOCALE(LC_COLLATE, NULL)), errhint("Apply system library package updates."))); } @@ -1339,7 +1340,7 @@ lc_collate_is_c(Oid collation) if (result >= 0) return (bool) result; - localeptr = setlocale(LC_COLLATE, NULL); + localeptr = SETLOCALE(LC_COLLATE, NULL); if (!localeptr) elog(ERROR, "invalid LC_COLLATE setting"); @@ -1389,7 +1390,7 @@ lc_ctype_is_c(Oid collation) if (result >= 0) return (bool) result; - localeptr = setlocale(LC_CTYPE, NULL); + localeptr = SETLOCALE(LC_CTYPE, NULL); if (!localeptr) elog(ERROR, "invalid LC_CTYPE setting"); @@ -1518,8 +1519,10 @@ pg_newlocale_from_collation(Oid collid) /* Normal case where they're the same */ errno = 0; #ifndef WIN32 - loc = newlocale(LC_COLLATE_MASK | LC_CTYPE_MASK, collcollate, + + loc = NEWLOCALE(LC_COLLATE_MASK | LC_CTYPE_MASK, collcollate, NULL); + #else loc = _create_locale(LC_ALL, collcollate); #endif @@ -1533,11 +1536,11 @@ pg_newlocale_from_collation(Oid collid) locale_t loc1; errno = 0; - loc1 = newlocale(LC_COLLATE_MASK, collcollate, NULL); + loc1 = NEWLOCALE(LC_COLLATE_MASK, collcollate, NULL); if (!loc1) report_newlocale_failure(collcollate); errno = 0; - loc = newlocale(LC_CTYPE_MASK, collctype, loc1); + loc = NEWLOCALE(LC_CTYPE_MASK, collctype, loc1); if (!loc) report_newlocale_failure(collctype); #else @@ -1680,12 +1683,16 @@ get_collation_actual_version(char collprovider, const char *collcollate) { #if defined(__GLIBC__) /* Use the glibc version because we don't have anything better. */ +#ifdef USE_MDBLOCALES + collversion = pstrdup(mdb_localesversion()); +#else collversion = pstrdup(gnu_get_libc_version()); +#endif #elif defined(LC_VERSION_MASK) locale_t loc; /* Look up FreeBSD collation version. */ - loc = newlocale(LC_COLLATE, collcollate, NULL); + loc = NEWLOCALE(LC_COLLATE, collcollate, NULL); if (loc) { collversion = diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 29287088ecf..952d1474870 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -40,6 +40,7 @@ #include "utils/builtins.h" #include "utils/memutils.h" #include "utils/syscache.h" +#include "common/mdb_locale.h" /* * We maintain a simple linked list caching the fmgr lookup info for the @@ -1308,7 +1309,7 @@ pg_bind_textdomain_codeset(const char *domainname) int new_msgenc; #ifndef WIN32 - const char *ctype = setlocale(LC_CTYPE, NULL); + const char *ctype = SETLOCALE(LC_CTYPE, NULL); if (pg_strcasecmp(ctype, "C") == 0 || pg_strcasecmp(ctype, "POSIX") == 0) #endif diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 4ed9869a2c9..708cf77ffdf 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -75,6 +75,7 @@ #include "getopt_long.h" #include "mb/pg_wchar.h" #include "miscadmin.h" +#include "common/mdb_locale.h" #include "catalog/catalog.h" @@ -2274,12 +2275,13 @@ locale_date_order(const char *locale) result = DATEORDER_MDY; /* default */ - save = setlocale(LC_TIME, NULL); + save = SETLOCALE(LC_TIME, NULL); + if (!save) return result; save = pg_strdup(save); - setlocale(LC_TIME, locale); + SETLOCALE(LC_TIME, locale); memset(&testtime, 0, sizeof(testtime)); testtime.tm_mday = 22; @@ -2288,7 +2290,7 @@ locale_date_order(const char *locale) res = my_strftime(buf, sizeof(buf), "%x", &testtime); - setlocale(LC_TIME, save); + SETLOCALE(LC_TIME, save); free(save); if (res == 0) @@ -2332,7 +2334,7 @@ check_locale_name(int category, const char *locale, char **canonname) if (canonname) *canonname = NULL; /* in case of failure */ - save = setlocale(category, NULL); + save = SETLOCALE(category, NULL); if (!save) { pg_log_error("setlocale() failed"); @@ -2347,14 +2349,14 @@ check_locale_name(int category, const char *locale, char **canonname) locale = ""; /* set the locale with setlocale, to see if it accepts it. */ - res = setlocale(category, locale); + res = SETLOCALE(category, locale); /* save canonical name if requested. */ if (res && canonname) *canonname = pg_strdup(res); /* restore old value. */ - if (!setlocale(category, save)) + if (!SETLOCALE(category, save)) { pg_log_error("failed to restore old locale \"%s\"", save); exit(1); diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index 1cf10dace16..450c047956a 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -16,6 +16,8 @@ #include "mb/pg_wchar.h" #include "pg_upgrade.h" #include "greenplum/pg_upgrade_greenplum.h" +#include "common/mdb_locale.h" + static void check_new_cluster_is_empty(void); static void check_databases_are_compatible(void); @@ -1627,7 +1629,8 @@ get_canonical_locale_name(int category, const char *locale) char *res; /* get the current setting, so we can restore it. */ - save = setlocale(category, NULL); + + save = SETLOCALE(category, NULL); if (!save) pg_fatal("failed to get the current locale\n"); @@ -1635,7 +1638,7 @@ get_canonical_locale_name(int category, const char *locale) save = (char *) pg_strdup(save); /* set the locale with setlocale, to see if it accepts it. */ - res = setlocale(category, locale); + res = SETLOCALE(category, locale); if (!res) pg_fatal("failed to get system locale name for \"%s\"\n", locale); @@ -1643,7 +1646,7 @@ get_canonical_locale_name(int category, const char *locale) res = pg_strdup(res); /* restore old value. */ - if (!setlocale(category, save)) + if (!SETLOCALE(category, save)) pg_fatal("failed to restore old locale \"%s\"\n", save); pg_free(save); diff --git a/src/common/exec.c b/src/common/exec.c index 7dd2f8c4942..5159b616a39 100644 --- a/src/common/exec.c +++ b/src/common/exec.c @@ -24,6 +24,8 @@ #include #include #include +#include "common/mdb_locale.h" + /* Inhibit mingw CRT's auto-globbing of command line arguments */ #if defined(WIN32) && !defined(_MSC_VER) @@ -443,7 +445,7 @@ set_pglocale_pgservice(const char *argv0, const char *app) /* don't set LC_ALL in the backend */ if (strcmp(app, PG_TEXTDOMAIN("postgres")) != 0) { - setlocale(LC_ALL, ""); + SETLOCALE(LC_ALL, ""); /* * One could make a case for reproducing here PostmasterMain()'s test diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 2a0d5adef40..f292f05bc28 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11754,7 +11754,9 @@ # # GPDB ADDITIONS START HERE # - +{ oid => '16383', descr => 'contains', + proname => 'mdb_locale_enabled', prorettype => 'bool', + proargtypes => '', prosrc => 'mdb_locale_enabled' }, { oid => '7178', descr => 'for use by pg_upgrade', proname => 'binary_upgrade_set_preassigned_oids', provolatile => 'v', proparallel => 'u', prorettype => 'void', proargtypes => '_oid', diff --git a/src/include/common/mdb_locale.h b/src/include/common/mdb_locale.h new file mode 100644 index 00000000000..91d8656c2c2 --- /dev/null +++ b/src/include/common/mdb_locale.h @@ -0,0 +1,41 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * mdb_locale.h + * Generic headers for custom MDB-locales patch. + * + * IDENTIFICATION + * src/include/common/mdb_locale.h + * + *------------------------------------------------------------------------- + */ + +#ifndef PG_MDB_LOCALE_H +#define PG_MDB_LOCALE_H + +#ifdef USE_MDBLOCALES +#include +#define SETLOCALE(category, locale) mdb_setlocale(category, locale) +#define NEWLOCALE(category, locale, base) mdb_newlocale(category, locale, base) +#else +#define SETLOCALE(category, locale) setlocale(category, locale) +#define NEWLOCALE(category, locale, base) newlocale(category, locale, base) +#endif + +#endif /* PG_MDB_LOCALE_H */ diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index a5393e85076..a388c40bc84 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -389,6 +389,9 @@ /* Define to 1 if you have the `m' library (-lm). */ #undef HAVE_LIBM +/* Define to 1 if you have the `mdblocales' library (-lmdblocales). */ +#undef HAVE_LIBMDBLOCALES + /* Define to 1 if you have the `numa' library (-lnuma). */ #undef HAVE_LIBNUMA @@ -1038,6 +1041,9 @@ /* Define to 1 to build with LZ4 support. (--with-lz4) */ #undef USE_LZ4 +/* Define to 1 to build with MDB locales. (--with-mdblocales) */ +#undef USE_MDBLOCALES + /* Define to 1 to build with Mapreduce capabilities (--enable-mapreduce) */ #undef USE_MAPREDUCE diff --git a/src/interfaces/ecpg/ecpglib/connect.c b/src/interfaces/ecpg/ecpglib/connect.c index 056940cb252..f4d2da9173a 100644 --- a/src/interfaces/ecpg/ecpglib/connect.c +++ b/src/interfaces/ecpg/ecpglib/connect.c @@ -9,6 +9,7 @@ #include "ecpglib_extern.h" #include "ecpgtype.h" #include "sqlca.h" +#include "common/mdb_locale.h" #ifdef HAVE_USELOCALE locale_t ecpg_clocale = (locale_t) 0; @@ -517,7 +518,7 @@ ECPGconnect(int lineno, int c, const char *name, const char *user, const char *p #ifdef HAVE_USELOCALE if (!ecpg_clocale) { - ecpg_clocale = newlocale(LC_NUMERIC_MASK, "C", (locale_t) 0); + ecpg_clocale = NEWLOCALE(LC_NUMERIC_MASK, "C", (locale_t) 0); if (!ecpg_clocale) { #ifdef ENABLE_THREAD_SAFETY diff --git a/src/interfaces/ecpg/ecpglib/descriptor.c b/src/interfaces/ecpg/ecpglib/descriptor.c index f1898dec6a6..2238febbbdd 100644 --- a/src/interfaces/ecpg/ecpglib/descriptor.c +++ b/src/interfaces/ecpg/ecpglib/descriptor.c @@ -15,6 +15,8 @@ #include "sql3types.h" #include "sqlca.h" #include "sqlda.h" +#include "common/mdb_locale.h" + static void descriptor_free(struct descriptor *desc); @@ -500,8 +502,8 @@ ECPGget_desc(int lineno, const char *desc_name, int index,...) #ifdef HAVE__CONFIGTHREADLOCALE stmt.oldthreadlocale = _configthreadlocale(_ENABLE_PER_THREAD_LOCALE); #endif - stmt.oldlocale = ecpg_strdup(setlocale(LC_NUMERIC, NULL), lineno); - setlocale(LC_NUMERIC, "C"); + stmt.oldlocale = ecpg_strdup(SETLOCALE(LC_NUMERIC, NULL), lineno); + SETLOCALE(LC_NUMERIC, "C"); #endif /* desperate try to guess something sensible */ @@ -514,7 +516,7 @@ ECPGget_desc(int lineno, const char *desc_name, int index,...) #else if (stmt.oldlocale) { - setlocale(LC_NUMERIC, stmt.oldlocale); + SETLOCALE(LC_NUMERIC, stmt.oldlocale); ecpg_free(stmt.oldlocale); } #ifdef HAVE__CONFIGTHREADLOCALE diff --git a/src/interfaces/ecpg/ecpglib/execute.c b/src/interfaces/ecpg/ecpglib/execute.c index e8e8fb2b2c3..eafdd8e421a 100644 --- a/src/interfaces/ecpg/ecpglib/execute.c +++ b/src/interfaces/ecpg/ecpglib/execute.c @@ -31,6 +31,7 @@ #include "sqlca.h" #include "sqlda-compat.h" #include "sqlda-native.h" +#include "common/mdb_locale.h" /* * This function returns a newly malloced string that has ' and \ @@ -2002,13 +2003,13 @@ ecpg_do_prologue(int lineno, const int compat, const int force_indicator, #ifdef HAVE__CONFIGTHREADLOCALE stmt->oldthreadlocale = _configthreadlocale(_ENABLE_PER_THREAD_LOCALE); #endif - stmt->oldlocale = ecpg_strdup(setlocale(LC_NUMERIC, NULL), lineno); + stmt->oldlocale = ecpg_strdup(SETLOCALE(LC_NUMERIC, NULL), lineno); if (stmt->oldlocale == NULL) { ecpg_do_epilogue(stmt); return false; } - setlocale(LC_NUMERIC, "C"); + SETLOCALE(LC_NUMERIC, "C"); #endif /* @@ -2222,7 +2223,7 @@ ecpg_do_epilogue(struct statement *stmt) uselocale(stmt->oldlocale); #else if (stmt->oldlocale) - setlocale(LC_NUMERIC, stmt->oldlocale); + SETLOCALE(LC_NUMERIC, stmt->oldlocale); #ifdef HAVE__CONFIGTHREADLOCALE /* diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile index 43682574b23..ed3df424ae4 100644 --- a/src/interfaces/libpq/Makefile +++ b/src/interfaces/libpq/Makefile @@ -83,7 +83,7 @@ endif # that are built correctly for use in a shlib. SHLIB_LINK_INTERNAL = -lpgcommon_shlib -lpgport_shlib ifneq ($(PORTNAME), win32) -SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi_krb5 -lgss -lgssapi -lssl -lsocket -lnsl -lresolv -lintl -lm, $(LIBS)) $(LDAP_LIBS_FE) $(PTHREAD_LIBS) +SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi_krb5 -lgss -lgssapi -lssl -lsocket -lnsl -lresolv -lintl -lm -lmdblocales, $(LIBS)) $(LDAP_LIBS_FE) $(PTHREAD_LIBS) else SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi32 -lssl -lsocket -lnsl -lresolv -lintl -lm $(PTHREAD_LIBS), $(LIBS)) $(LDAP_LIBS_FE) endif diff --git a/src/pl/plperl/plperl.c b/src/pl/plperl/plperl.c index 48591e48429..3aff8e95450 100644 --- a/src/pl/plperl/plperl.c +++ b/src/pl/plperl/plperl.c @@ -38,6 +38,7 @@ #include "utils/rel.h" #include "utils/syscache.h" #include "utils/typcache.h" +#include "common/mdb_locale.h" /* define our text domain for translations */ #undef TEXTDOMAIN @@ -743,15 +744,15 @@ plperl_init_interp(void) *save_numeric, *save_time; - loc = setlocale(LC_COLLATE, NULL); + loc = SETLOCALE(LC_COLLATE, NULL); save_collate = loc ? pstrdup(loc) : NULL; - loc = setlocale(LC_CTYPE, NULL); + loc = SETLOCALE(LC_CTYPE, NULL); save_ctype = loc ? pstrdup(loc) : NULL; - loc = setlocale(LC_MONETARY, NULL); + loc = SETLOCALE(LC_MONETARY, NULL); save_monetary = loc ? pstrdup(loc) : NULL; - loc = setlocale(LC_NUMERIC, NULL); + loc = SETLOCALE(LC_NUMERIC, NULL); save_numeric = loc ? pstrdup(loc) : NULL; - loc = setlocale(LC_TIME, NULL); + loc = SETLOCALE(LC_TIME, NULL); save_time = loc ? pstrdup(loc) : NULL; #define PLPERL_RESTORE_LOCALE(name, saved) \ @@ -4167,7 +4168,7 @@ static char * setlocale_perl(int category, char *locale) { dTHX; - char *RETVAL = setlocale(category, locale); + char *RETVAL = SETLOCALE(category, locale); if (RETVAL) { @@ -4182,7 +4183,7 @@ setlocale_perl(int category, char *locale) #ifdef LC_ALL if (category == LC_ALL) - newctype = setlocale(LC_CTYPE, NULL); + newctype = SETLOCALE(LC_CTYPE, NULL); else #endif newctype = RETVAL; @@ -4200,7 +4201,7 @@ setlocale_perl(int category, char *locale) #ifdef LC_ALL if (category == LC_ALL) - newcoll = setlocale(LC_COLLATE, NULL); + newcoll = SETLOCALE(LC_COLLATE, NULL); else #endif newcoll = RETVAL; @@ -4219,7 +4220,7 @@ setlocale_perl(int category, char *locale) #ifdef LC_ALL if (category == LC_ALL) - newnum = setlocale(LC_NUMERIC, NULL); + newnum = SETLOCALE(LC_NUMERIC, NULL); else #endif newnum = RETVAL; diff --git a/src/port/chklocale.c b/src/port/chklocale.c index 3d47d37eae4..2dae78e74e9 100644 --- a/src/port/chklocale.c +++ b/src/port/chklocale.c @@ -18,6 +18,8 @@ #else #include "postgres_fe.h" #endif +#include "common/mdb_locale.h" + #ifdef HAVE_LANGINFO_H #include @@ -343,7 +345,7 @@ pg_get_encoding_from_locale(const char *ctype, bool write_message) pg_strcasecmp(ctype, "POSIX") == 0) return PG_SQL_ASCII; - save = setlocale(LC_CTYPE, NULL); + save = SETLOCALE(LC_CTYPE, NULL); if (!save) return -1; /* setlocale() broken? */ /* must copy result, or it might change after setlocale */ @@ -351,7 +353,7 @@ pg_get_encoding_from_locale(const char *ctype, bool write_message) if (!save) return -1; /* out of memory; unlikely */ - name = setlocale(LC_CTYPE, ctype); + name = SETLOCALE(LC_CTYPE, ctype); if (!name) { free(save); @@ -366,13 +368,13 @@ pg_get_encoding_from_locale(const char *ctype, bool write_message) sys = win32_langinfo(name); #endif - setlocale(LC_CTYPE, save); + SETLOCALE(LC_CTYPE, save); free(save); } else { /* much easier... */ - ctype = setlocale(LC_CTYPE, NULL); + ctype = SETLOCALE(LC_CTYPE, NULL); if (!ctype) return -1; /* setlocale() broken? */ diff --git a/src/test/locale/test-ctype.c b/src/test/locale/test-ctype.c index a3f896c5ecb..10c2b49cb92 100644 --- a/src/test/locale/test-ctype.c +++ b/src/test/locale/test-ctype.c @@ -23,6 +23,8 @@ the author shall be liable for any damage, etc. #include #include #include +#include "common/mdb_locale.h" + char *flag(int b); void describe_char(int c); @@ -62,7 +64,7 @@ main() short c; char *cur_locale; - cur_locale = setlocale(LC_ALL, ""); + cur_locale = SETLOCALE(LC_ALL, ""); if (cur_locale) fprintf(stderr, "Successfully set locale to \"%s\"\n", cur_locale); else diff --git a/src/test/regress/input/misc.source b/src/test/regress/input/misc.source index 331499a2aba..2abe2c82eb8 100644 --- a/src/test/regress/input/misc.source +++ b/src/test/regress/input/misc.source @@ -264,3 +264,8 @@ SELECT *, (equipment(CAST((h.*) AS hobbies_r))).name FROM hobbies_r h; -- -- rewrite rules -- + + +--- mdb-related + +SELECT mdb_locale_enabled(); diff --git a/src/test/regress/output/misc.source b/src/test/regress/output/misc.source index 18bcc227f0a..a0c63418446 100644 --- a/src/test/regress/output/misc.source +++ b/src/test/regress/output/misc.source @@ -609,3 +609,10 @@ CONTEXT: SQL function "equipment" during startup -- -- rewrite rules -- +--- mdb-related +SELECT mdb_locale_enabled(); + mdb_locale_enabled +-------------------- + t +(1 row) + diff --git a/src/test/regress/sql/misc.sql b/src/test/regress/sql/misc.sql new file mode 100644 index 00000000000..5c42672c4f7 --- /dev/null +++ b/src/test/regress/sql/misc.sql @@ -0,0 +1,271 @@ +-- +-- MISC +-- + +-- +-- BTREE +-- +--UPDATE onek +-- SET unique1 = onek.unique1 + 1; + +--UPDATE onek +-- SET unique1 = onek.unique1 - 1; + +-- +-- BTREE partial +-- +-- UPDATE onek2 +-- SET unique1 = onek2.unique1 + 1; + +--UPDATE onek2 +-- SET unique1 = onek2.unique1 - 1; + +-- +-- BTREE shutting out non-functional updates +-- +-- the following two tests seem to take a long time on some +-- systems. This non-func update stuff needs to be examined +-- more closely. - jolly (2/22/96) +-- +/* GPDB TODO: This test is disabled for now, because when running with ORCA, + you get an error: + ERROR: multiple updates to a row by the same query is not allowed +UPDATE tmp + SET stringu1 = reverse_name(onek.stringu1) + FROM onek + WHERE onek.stringu1 = 'JBAAAA' and + onek.stringu1 = tmp.stringu1; + +UPDATE tmp + SET stringu1 = reverse_name(onek2.stringu1) + FROM onek2 + WHERE onek2.stringu1 = 'JCAAAA' and + onek2.stringu1 = tmp.stringu1; +*/ + +DROP TABLE tmp; + +--UPDATE person* +-- SET age = age + 1; + +--UPDATE person* +-- SET age = age + 3 +-- WHERE name = 'linda'; + +-- +-- copy +-- +COPY onek TO '/home/xifos/git/cloudberry-gpdb/src/test/regress/results/onek.data'; + +DELETE FROM onek; + +COPY onek FROM '/home/xifos/git/cloudberry-gpdb/src/test/regress/results/onek.data'; + +SELECT unique1 FROM onek WHERE unique1 < 2 ORDER BY unique1; + +DELETE FROM onek2; + +COPY onek2 FROM '/home/xifos/git/cloudberry-gpdb/src/test/regress/results/onek.data'; + +SELECT unique1 FROM onek2 WHERE unique1 < 2 ORDER BY unique1; + +COPY BINARY stud_emp TO '/home/xifos/git/cloudberry-gpdb/src/test/regress/results/stud_emp.data'; + +DELETE FROM stud_emp; + +COPY BINARY stud_emp FROM '/home/xifos/git/cloudberry-gpdb/src/test/regress/results/stud_emp.data'; + +SELECT * FROM stud_emp; + +-- COPY aggtest FROM stdin; +-- 56 7.8 +-- 100 99.097 +-- 0 0.09561 +-- 42 324.78 +-- . +-- COPY aggtest TO stdout; + + +-- +-- inheritance stress test +-- +SELECT * FROM a_star*; + +SELECT * + FROM b_star* x + WHERE x.b = text 'bumble' or x.a < 3; + +SELECT class, a + FROM c_star* x + WHERE x.c ~ text 'hi'; + +SELECT class, b, c + FROM d_star* x + WHERE x.a < 100; + +SELECT class, c FROM e_star* x WHERE x.c NOTNULL; + +SELECT * FROM f_star* x WHERE x.c ISNULL; + +-- grouping and aggregation on inherited sets have been busted in the past... + +SELECT sum(a) FROM a_star*; + +SELECT class, sum(a) FROM a_star* GROUP BY class ORDER BY class; + + +ALTER TABLE f_star RENAME COLUMN f TO ff; + +ALTER TABLE e_star* RENAME COLUMN e TO ee; + +ALTER TABLE d_star* RENAME COLUMN d TO dd; + +ALTER TABLE c_star* RENAME COLUMN c TO cc; + +ALTER TABLE b_star* RENAME COLUMN b TO bb; + +ALTER TABLE a_star* RENAME COLUMN a TO aa; + +SELECT class, aa + FROM a_star* x + WHERE aa ISNULL; + +-- As of Postgres 7.1, ALTER implicitly recurses, +-- so this should be same as ALTER a_star* + +ALTER TABLE a_star RENAME COLUMN aa TO foo; + +SELECT class, foo + FROM a_star* x + WHERE x.foo >= 2; + +ALTER TABLE a_star RENAME COLUMN foo TO aa; + +SELECT * + from a_star* + WHERE aa < 1000; + +ALTER TABLE f_star ADD COLUMN f int4; + +UPDATE f_star SET f = 10; + +ALTER TABLE e_star* ADD COLUMN e int4; + +--UPDATE e_star* SET e = 42; + +SELECT * FROM e_star*; + +ALTER TABLE a_star* ADD COLUMN a text; + +-- That ALTER TABLE should have added TOAST tables. +SELECT relname, reltoastrelid <> 0 AS has_toast_table + FROM pg_class + WHERE oid::regclass IN ('a_star', 'c_star') + ORDER BY 1; + +--UPDATE b_star* +-- SET a = text 'gazpacho' +-- WHERE aa > 4; + +SELECT class, aa, a FROM a_star*; + + +-- +-- versions +-- + +-- +-- postquel functions +-- +-- +-- mike does post_hacking, +-- joe and sally play basketball, and +-- everyone else does nothing. +-- +SELECT p.name, name(p.hobbies) FROM ONLY person p; + +-- +-- as above, but jeff also does post_hacking. +-- +SELECT p.name, name(p.hobbies) FROM person* p; + +-- +-- the next two queries demonstrate how functions generate bogus duplicates. +-- this is a "feature" .. +-- +SELECT DISTINCT hobbies_r.name, name(hobbies_r.equipment) FROM hobbies_r + ORDER BY 1,2; + +SELECT hobbies_r.name, (hobbies_r.equipment).name FROM hobbies_r; + +-- +-- mike needs advil and peet's coffee, +-- joe and sally need hightops, and +-- everyone else is fine. +-- +SELECT p.name, name(p.hobbies), name(equipment(p.hobbies)) FROM ONLY person p; + +-- +-- as above, but jeff needs advil and peet's coffee as well. +-- +SELECT p.name, name(p.hobbies), name(equipment(p.hobbies)) FROM person* p; + +-- +-- just like the last two, but make sure that the target list fixup and +-- unflattening is being done correctly. +-- +SELECT name(equipment(p.hobbies)), p.name, name(p.hobbies) FROM ONLY person p; + +SELECT (p.hobbies).equipment.name, p.name, name(p.hobbies) FROM person* p; + +SELECT (p.hobbies).equipment.name, name(p.hobbies), p.name FROM ONLY person p; + +SELECT name(equipment(p.hobbies)), name(p.hobbies), p.name FROM person* p; + +SELECT name(equipment(hobby_construct(text 'skywalking', text 'mer'))); + +SELECT name(equipment(hobby_construct_named(text 'skywalking', text 'mer'))); + +SELECT name(equipment_named(hobby_construct_named(text 'skywalking', text 'mer'))); + +SELECT name(equipment_named_ambiguous_1a(hobby_construct_named(text 'skywalking', text 'mer'))); + +SELECT name(equipment_named_ambiguous_1b(hobby_construct_named(text 'skywalking', text 'mer'))); + +SELECT name(equipment_named_ambiguous_1c(hobby_construct_named(text 'skywalking', text 'mer'))); + +SELECT name(equipment_named_ambiguous_2a(text 'skywalking')); + +SELECT name(equipment_named_ambiguous_2b(text 'skywalking')); + +SELECT hobbies_by_name('basketball'); + +SELECT name, overpaid(emp.*) FROM emp; + +-- +-- Try a few cases with SQL-spec row constructor expressions +-- +SELECT * FROM equipment(ROW('skywalking', 'mer')); + +SELECT name(equipment(ROW('skywalking', 'mer'))); + +SELECT *, name(equipment(h.*)) FROM hobbies_r h; + +SELECT *, (equipment(CAST((h.*) AS hobbies_r))).name FROM hobbies_r h; + +-- +-- functional joins +-- + +-- +-- instance rules +-- + +-- +-- rewrite rules +-- + + +--- mdb-related + +SELECT mdb_locale_enabled(); From 75f734fcba3a3d8947ffe792a9b46bd66f48cb52 Mon Sep 17 00:00:00 2001 From: reshke Date: Fri, 19 Sep 2025 18:13:31 +0500 Subject: [PATCH 02/22] Extend multixact SLRU (#3) --- src/include/access/multixact.h | 4 ++-- src/include/access/subtrans.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/include/access/multixact.h b/src/include/access/multixact.h index 4bbb035eaea..f053a30b009 100644 --- a/src/include/access/multixact.h +++ b/src/include/access/multixact.h @@ -30,8 +30,8 @@ #define MaxMultiXactOffset ((MultiXactOffset) 0xFFFFFFFF) /* Number of SLRU buffers to use for multixact */ -#define NUM_MULTIXACTOFFSET_BUFFERS 8 -#define NUM_MULTIXACTMEMBER_BUFFERS 16 +#define NUM_MULTIXACTOFFSET_BUFFERS 32 +#define NUM_MULTIXACTMEMBER_BUFFERS 64 /* * Possible multixact lock modes ("status"). The first four modes are for diff --git a/src/include/access/subtrans.h b/src/include/access/subtrans.h index 9a54dc0fb3b..73503a26dcc 100644 --- a/src/include/access/subtrans.h +++ b/src/include/access/subtrans.h @@ -12,7 +12,7 @@ #define SUBTRANS_H /* Number of SLRU buffers to use for subtrans */ -#define NUM_SUBTRANS_BUFFERS 32 +#define NUM_SUBTRANS_BUFFERS 64 typedef struct SubTransData { From acdcacda1b20ba9d9b1aef7e239164bdfe3d1116 Mon Sep 17 00:00:00 2001 From: reshke Date: Fri, 19 Sep 2025 19:04:10 +0500 Subject: [PATCH 03/22] Delete src/test/regress/sql/misc.sql --- src/test/regress/sql/misc.sql | 271 ---------------------------------- 1 file changed, 271 deletions(-) delete mode 100644 src/test/regress/sql/misc.sql diff --git a/src/test/regress/sql/misc.sql b/src/test/regress/sql/misc.sql deleted file mode 100644 index 5c42672c4f7..00000000000 --- a/src/test/regress/sql/misc.sql +++ /dev/null @@ -1,271 +0,0 @@ --- --- MISC --- - --- --- BTREE --- ---UPDATE onek --- SET unique1 = onek.unique1 + 1; - ---UPDATE onek --- SET unique1 = onek.unique1 - 1; - --- --- BTREE partial --- --- UPDATE onek2 --- SET unique1 = onek2.unique1 + 1; - ---UPDATE onek2 --- SET unique1 = onek2.unique1 - 1; - --- --- BTREE shutting out non-functional updates --- --- the following two tests seem to take a long time on some --- systems. This non-func update stuff needs to be examined --- more closely. - jolly (2/22/96) --- -/* GPDB TODO: This test is disabled for now, because when running with ORCA, - you get an error: - ERROR: multiple updates to a row by the same query is not allowed -UPDATE tmp - SET stringu1 = reverse_name(onek.stringu1) - FROM onek - WHERE onek.stringu1 = 'JBAAAA' and - onek.stringu1 = tmp.stringu1; - -UPDATE tmp - SET stringu1 = reverse_name(onek2.stringu1) - FROM onek2 - WHERE onek2.stringu1 = 'JCAAAA' and - onek2.stringu1 = tmp.stringu1; -*/ - -DROP TABLE tmp; - ---UPDATE person* --- SET age = age + 1; - ---UPDATE person* --- SET age = age + 3 --- WHERE name = 'linda'; - --- --- copy --- -COPY onek TO '/home/xifos/git/cloudberry-gpdb/src/test/regress/results/onek.data'; - -DELETE FROM onek; - -COPY onek FROM '/home/xifos/git/cloudberry-gpdb/src/test/regress/results/onek.data'; - -SELECT unique1 FROM onek WHERE unique1 < 2 ORDER BY unique1; - -DELETE FROM onek2; - -COPY onek2 FROM '/home/xifos/git/cloudberry-gpdb/src/test/regress/results/onek.data'; - -SELECT unique1 FROM onek2 WHERE unique1 < 2 ORDER BY unique1; - -COPY BINARY stud_emp TO '/home/xifos/git/cloudberry-gpdb/src/test/regress/results/stud_emp.data'; - -DELETE FROM stud_emp; - -COPY BINARY stud_emp FROM '/home/xifos/git/cloudberry-gpdb/src/test/regress/results/stud_emp.data'; - -SELECT * FROM stud_emp; - --- COPY aggtest FROM stdin; --- 56 7.8 --- 100 99.097 --- 0 0.09561 --- 42 324.78 --- . --- COPY aggtest TO stdout; - - --- --- inheritance stress test --- -SELECT * FROM a_star*; - -SELECT * - FROM b_star* x - WHERE x.b = text 'bumble' or x.a < 3; - -SELECT class, a - FROM c_star* x - WHERE x.c ~ text 'hi'; - -SELECT class, b, c - FROM d_star* x - WHERE x.a < 100; - -SELECT class, c FROM e_star* x WHERE x.c NOTNULL; - -SELECT * FROM f_star* x WHERE x.c ISNULL; - --- grouping and aggregation on inherited sets have been busted in the past... - -SELECT sum(a) FROM a_star*; - -SELECT class, sum(a) FROM a_star* GROUP BY class ORDER BY class; - - -ALTER TABLE f_star RENAME COLUMN f TO ff; - -ALTER TABLE e_star* RENAME COLUMN e TO ee; - -ALTER TABLE d_star* RENAME COLUMN d TO dd; - -ALTER TABLE c_star* RENAME COLUMN c TO cc; - -ALTER TABLE b_star* RENAME COLUMN b TO bb; - -ALTER TABLE a_star* RENAME COLUMN a TO aa; - -SELECT class, aa - FROM a_star* x - WHERE aa ISNULL; - --- As of Postgres 7.1, ALTER implicitly recurses, --- so this should be same as ALTER a_star* - -ALTER TABLE a_star RENAME COLUMN aa TO foo; - -SELECT class, foo - FROM a_star* x - WHERE x.foo >= 2; - -ALTER TABLE a_star RENAME COLUMN foo TO aa; - -SELECT * - from a_star* - WHERE aa < 1000; - -ALTER TABLE f_star ADD COLUMN f int4; - -UPDATE f_star SET f = 10; - -ALTER TABLE e_star* ADD COLUMN e int4; - ---UPDATE e_star* SET e = 42; - -SELECT * FROM e_star*; - -ALTER TABLE a_star* ADD COLUMN a text; - --- That ALTER TABLE should have added TOAST tables. -SELECT relname, reltoastrelid <> 0 AS has_toast_table - FROM pg_class - WHERE oid::regclass IN ('a_star', 'c_star') - ORDER BY 1; - ---UPDATE b_star* --- SET a = text 'gazpacho' --- WHERE aa > 4; - -SELECT class, aa, a FROM a_star*; - - --- --- versions --- - --- --- postquel functions --- --- --- mike does post_hacking, --- joe and sally play basketball, and --- everyone else does nothing. --- -SELECT p.name, name(p.hobbies) FROM ONLY person p; - --- --- as above, but jeff also does post_hacking. --- -SELECT p.name, name(p.hobbies) FROM person* p; - --- --- the next two queries demonstrate how functions generate bogus duplicates. --- this is a "feature" .. --- -SELECT DISTINCT hobbies_r.name, name(hobbies_r.equipment) FROM hobbies_r - ORDER BY 1,2; - -SELECT hobbies_r.name, (hobbies_r.equipment).name FROM hobbies_r; - --- --- mike needs advil and peet's coffee, --- joe and sally need hightops, and --- everyone else is fine. --- -SELECT p.name, name(p.hobbies), name(equipment(p.hobbies)) FROM ONLY person p; - --- --- as above, but jeff needs advil and peet's coffee as well. --- -SELECT p.name, name(p.hobbies), name(equipment(p.hobbies)) FROM person* p; - --- --- just like the last two, but make sure that the target list fixup and --- unflattening is being done correctly. --- -SELECT name(equipment(p.hobbies)), p.name, name(p.hobbies) FROM ONLY person p; - -SELECT (p.hobbies).equipment.name, p.name, name(p.hobbies) FROM person* p; - -SELECT (p.hobbies).equipment.name, name(p.hobbies), p.name FROM ONLY person p; - -SELECT name(equipment(p.hobbies)), name(p.hobbies), p.name FROM person* p; - -SELECT name(equipment(hobby_construct(text 'skywalking', text 'mer'))); - -SELECT name(equipment(hobby_construct_named(text 'skywalking', text 'mer'))); - -SELECT name(equipment_named(hobby_construct_named(text 'skywalking', text 'mer'))); - -SELECT name(equipment_named_ambiguous_1a(hobby_construct_named(text 'skywalking', text 'mer'))); - -SELECT name(equipment_named_ambiguous_1b(hobby_construct_named(text 'skywalking', text 'mer'))); - -SELECT name(equipment_named_ambiguous_1c(hobby_construct_named(text 'skywalking', text 'mer'))); - -SELECT name(equipment_named_ambiguous_2a(text 'skywalking')); - -SELECT name(equipment_named_ambiguous_2b(text 'skywalking')); - -SELECT hobbies_by_name('basketball'); - -SELECT name, overpaid(emp.*) FROM emp; - --- --- Try a few cases with SQL-spec row constructor expressions --- -SELECT * FROM equipment(ROW('skywalking', 'mer')); - -SELECT name(equipment(ROW('skywalking', 'mer'))); - -SELECT *, name(equipment(h.*)) FROM hobbies_r h; - -SELECT *, (equipment(CAST((h.*) AS hobbies_r))).name FROM hobbies_r h; - --- --- functional joins --- - --- --- instance rules --- - --- --- rewrite rules --- - - ---- mdb-related - -SELECT mdb_locale_enabled(); From 956a8ecced5249ea49b2a61bb6ce6eeff2d91f05 Mon Sep 17 00:00:00 2001 From: reshke Date: Fri, 19 Sep 2025 21:47:24 +0500 Subject: [PATCH 04/22] MDB admin patch & tests (#4) * MDB admin patch & tests This patch introcudes new pseudo-pre-defined role "mdb_admin". Introduces 2 new function: extern bool mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId); extern void check_mdb_admin_is_member_of_role(Oid member, Oid role); To check mdb admin belongship and role-to-role ownership transfer correctness. Our mdb_admin ACL model is the following: * Any roles user or/and roles can be granted with mdb_admin * mdb_admin memeber can tranfser ownershup of relations, namespaces and functions to other roles, if target role in neither: superuser, pg_read_server_files, pg_write_server_files nor pg_execute_server_program. This patch allows mdb admin to tranfers ownership on non-superuser objects * f --- .../regress/expected/create_function_3.out | 4 +- .../expected/create_function_3_optimizer.out | 4 +- src/backend/catalog/namespace.c | 20 +++- src/backend/commands/alter.c | 8 +- src/backend/commands/functioncmds.c | 20 +++- src/backend/commands/schemacmds.c | 13 +- src/backend/commands/tablecmds.c | 12 +- src/backend/storage/ipc/signalfuncs.c | 28 ++++- src/backend/utils/activity/backend_status.c | 16 +++ src/backend/utils/adt/acl.c | 112 ++++++++++++++++++ src/backend/utils/misc/guc.c | 2 +- src/include/utils/acl.h | 7 ++ src/include/utils/backend_status.h | 3 + src/include/utils/guc_tables.h | 2 + src/test/Makefile | 3 + src/test/mdb_admin/.gitignore | 2 + src/test/mdb_admin/Makefile | 23 ++++ src/test/mdb_admin/t/signals.pl | 74 ++++++++++++ .../regress/expected/create_function_3.out | 4 +- .../expected/create_function_3_optimizer.out | 4 +- src/test/regress/expected/mdb_admin.out | 81 +++++++++++++ src/test/regress/parallel_schedule | 4 + src/test/regress/sql/mdb_admin.sql | 77 ++++++++++++ .../expected/create_function_3.out | 4 +- 24 files changed, 494 insertions(+), 33 deletions(-) create mode 100644 src/test/mdb_admin/.gitignore create mode 100644 src/test/mdb_admin/Makefile create mode 100644 src/test/mdb_admin/t/signals.pl create mode 100644 src/test/regress/expected/mdb_admin.out create mode 100644 src/test/regress/sql/mdb_admin.sql diff --git a/contrib/pax_storage/src/test/regress/expected/create_function_3.out b/contrib/pax_storage/src/test/regress/expected/create_function_3.out index 8380df1591f..7842a3c1c82 100644 --- a/contrib/pax_storage/src/test/regress/expected/create_function_3.out +++ b/contrib/pax_storage/src/test/regress/expected/create_function_3.out @@ -166,10 +166,10 @@ SET SESSION AUTHORIZATION regress_unpriv_user; SET search_path TO temp_func_test, public; ALTER FUNCTION functest_E_1(int) NOT LEAKPROOF; ALTER FUNCTION functest_E_2(int) LEAKPROOF; -ERROR: only superuser can define a leakproof function +ERROR: only superuser or mdb_admin can define a leakproof function CREATE FUNCTION functest_E_3(int) RETURNS bool LANGUAGE 'sql' LEAKPROOF AS 'SELECT $1 < 200'; -- fail -ERROR: only superuser can define a leakproof function +ERROR: only superuser or mdb_admin can define a leakproof function RESET SESSION AUTHORIZATION; -- -- CALLED ON NULL INPUT | RETURNS NULL ON NULL INPUT | STRICT diff --git a/contrib/pax_storage/src/test/regress/expected/create_function_3_optimizer.out b/contrib/pax_storage/src/test/regress/expected/create_function_3_optimizer.out index 3ae669d518a..3256709e1aa 100644 --- a/contrib/pax_storage/src/test/regress/expected/create_function_3_optimizer.out +++ b/contrib/pax_storage/src/test/regress/expected/create_function_3_optimizer.out @@ -166,10 +166,10 @@ SET SESSION AUTHORIZATION regress_unpriv_user; SET search_path TO temp_func_test, public; ALTER FUNCTION functest_E_1(int) NOT LEAKPROOF; ALTER FUNCTION functest_E_2(int) LEAKPROOF; -ERROR: only superuser can define a leakproof function +ERROR: only superuser or mdb_admin can define a leakproof function CREATE FUNCTION functest_E_3(int) RETURNS bool LANGUAGE 'sql' LEAKPROOF AS 'SELECT $1 < 200'; -- fail -ERROR: only superuser can define a leakproof function +ERROR: only superuser or mdb_admin can define a leakproof function RESET SESSION AUTHORIZATION; -- -- CALLED ON NULL INPUT | RETURNS NULL ON NULL INPUT | STRICT diff --git a/src/backend/catalog/namespace.c b/src/backend/catalog/namespace.c index f367b00a675..be09847022b 100644 --- a/src/backend/catalog/namespace.c +++ b/src/backend/catalog/namespace.c @@ -2971,7 +2971,6 @@ LookupExplicitNamespace(const char *nspname, bool missing_ok) { Oid namespaceId; AclResult aclresult; - /* check for pg_temp alias */ if (strcmp(nspname, "pg_temp") == 0) { @@ -2989,7 +2988,24 @@ LookupExplicitNamespace(const char *nspname, bool missing_ok) if (missing_ok && !OidIsValid(namespaceId)) return InvalidOid; - aclresult = pg_namespace_aclcheck(namespaceId, GetUserId(), ACL_USAGE); + HeapTuple tuple; + Oid ownerId; + + tuple = SearchSysCache1(NAMESPACEOID, ObjectIdGetDatum(namespaceId)); + if (!HeapTupleIsValid(tuple)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_SCHEMA), + errmsg("schema with OID %u does not exist", namespaceId))); + + ownerId = ((Form_pg_namespace) GETSTRUCT(tuple))->nspowner; + + ReleaseSysCache(tuple); + + if (!mdb_admin_allow_bypass_owner_checks(GetUserId(), ownerId)) { + aclresult = pg_namespace_aclcheck(namespaceId, GetUserId(), ACL_USAGE); + } else { + aclresult = ACLCHECK_OK; + } if (aclresult != ACLCHECK_OK) aclcheck_error(aclresult, OBJECT_SCHEMA, nspname); diff --git a/src/backend/commands/alter.c b/src/backend/commands/alter.c index f5dfd6ff126..6f370a2c9aa 100644 --- a/src/backend/commands/alter.c +++ b/src/backend/commands/alter.c @@ -1085,7 +1085,8 @@ AlterObjectOwner_internal(Relation rel, Oid objectId, Oid new_ownerId) if (!superuser()) { /* must be owner */ - if (!has_privs_of_role(GetUserId(), old_ownerId)) + if (!has_privs_of_role(GetUserId(), old_ownerId) + && !mdb_admin_allow_bypass_owner_checks(GetUserId(), old_ownerId)) { char *objname; char namebuf[NAMEDATALEN]; @@ -1105,14 +1106,13 @@ AlterObjectOwner_internal(Relation rel, Oid objectId, Oid new_ownerId) aclcheck_error(ACLCHECK_NOT_OWNER, get_object_type(classId, objectId), objname); } - /* Must be able to become new owner */ - check_is_member_of_role(GetUserId(), new_ownerId); + + check_mdb_admin_is_member_of_role(GetUserId(), new_ownerId); /* New owner must have CREATE privilege on namespace */ if (OidIsValid(namespaceId)) { AclResult aclresult; - aclresult = pg_namespace_aclcheck(namespaceId, new_ownerId, ACL_CREATE); if (aclresult != ACLCHECK_OK) diff --git a/src/backend/commands/functioncmds.c b/src/backend/commands/functioncmds.c index b99b2419fcc..8a570fa6965 100644 --- a/src/backend/commands/functioncmds.c +++ b/src/backend/commands/functioncmds.c @@ -1525,9 +1525,13 @@ CreateFunction(ParseState *pstate, CreateFunctionStmt *stmt) * by security barrier views or row-level security policies. */ if (isLeakProof && !superuser()) - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("only superuser can define a leakproof function"))); + { + Oid role = get_role_oid("mdb_admin", true); + if (!is_member_of_role(GetUserId(), role)) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("only superuser or mdb_admin can define a leakproof function"))); + } if (transformDefElem) { @@ -1852,9 +1856,13 @@ AlterFunction(ParseState *pstate, AlterFunctionStmt *stmt) { procForm->proleakproof = intVal(leakproof_item->arg); if (procForm->proleakproof && !superuser()) - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("only superuser can define a leakproof function"))); + { + Oid role = get_role_oid("mdb_admin", true); + if (!is_member_of_role(GetUserId(), role)) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("only superuser or mdb_admin can define a leakproof function"))); + } } if (cost_item) { diff --git a/src/backend/commands/schemacmds.c b/src/backend/commands/schemacmds.c index 96757eaa814..03f96bb6499 100644 --- a/src/backend/commands/schemacmds.c +++ b/src/backend/commands/schemacmds.c @@ -598,12 +598,12 @@ AlterSchemaOwner_internal(HeapTuple tup, Relation rel, Oid newOwnerId) AclResult aclresult; /* Otherwise, must be owner of the existing object */ - if (!pg_namespace_ownercheck(nspForm->oid, GetUserId())) + if (!mdb_admin_allow_bypass_owner_checks(GetUserId(), nspForm->nspowner) + && !pg_namespace_ownercheck(nspForm->oid, GetUserId())) aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_SCHEMA, NameStr(nspForm->nspname)); - /* Must be able to become new owner */ - check_is_member_of_role(GetUserId(), newOwnerId); + check_mdb_admin_is_member_of_role(GetUserId(), newOwnerId); /* * must have create-schema rights @@ -614,8 +614,13 @@ AlterSchemaOwner_internal(HeapTuple tup, Relation rel, Oid newOwnerId) * schemas. Because superusers will always have this right, we need * no special case for them. */ - aclresult = pg_database_aclcheck(MyDatabaseId, GetUserId(), + if (mdb_admin_allow_bypass_owner_checks(GetUserId(), nspForm->nspowner)) { + aclresult = ACLCHECK_OK; + } else { + aclresult = pg_database_aclcheck(MyDatabaseId, GetUserId(), ACL_CREATE); + } + if (aclresult != ACLCHECK_OK) aclcheck_error(aclresult, OBJECT_DATABASE, get_database_name(MyDatabaseId)); diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index e731e37142c..301c3548e5e 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -15708,13 +15708,14 @@ ATExecChangeOwner(Oid relationOid, Oid newOwnerId, bool recursing, LOCKMODE lock AclResult aclresult; /* Otherwise, must be owner of the existing object */ - if (!pg_class_ownercheck(relationOid, GetUserId())) + if (!mdb_admin_allow_bypass_owner_checks(GetUserId(), tuple_class->relowner) + && !pg_class_ownercheck(relationOid, GetUserId())) aclcheck_error(ACLCHECK_NOT_OWNER, get_relkind_objtype(get_rel_relkind(relationOid)), RelationGetRelationName(target_rel)); - /* Must be able to become new owner */ - check_is_member_of_role(GetUserId(), newOwnerId); + check_mdb_admin_is_member_of_role(GetUserId(), newOwnerId); + /* New owner must have CREATE privilege on namespace */ aclresult = pg_namespace_aclcheck(namespaceOid, newOwnerId, ACL_CREATE); @@ -20795,7 +20796,7 @@ RangeVarCallbackForAlterRelation(const RangeVar *rv, Oid relid, Oid oldrelid, Form_pg_class classform; AclResult aclresult; char relkind; - + tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); if (!HeapTupleIsValid(tuple)) return; /* concurrently dropped */ @@ -20803,7 +20804,8 @@ RangeVarCallbackForAlterRelation(const RangeVar *rv, Oid relid, Oid oldrelid, relkind = classform->relkind; /* Must own relation. */ - if (!pg_class_ownercheck(relid, GetUserId())) + if (!mdb_admin_allow_bypass_owner_checks(GetUserId(), classform->relowner) + && !pg_class_ownercheck(relid, GetUserId())) aclcheck_error(ACLCHECK_NOT_OWNER, get_relkind_objtype(get_rel_relkind(relid)), rv->relname); /* No system table modifications unless explicitly allowed. */ diff --git a/src/backend/storage/ipc/signalfuncs.c b/src/backend/storage/ipc/signalfuncs.c index 0d5ccaa201d..753b94752d3 100644 --- a/src/backend/storage/ipc/signalfuncs.c +++ b/src/backend/storage/ipc/signalfuncs.c @@ -52,6 +52,7 @@ static int pg_signal_backend(int pid, int sig, char *msg) { PGPROC *proc = BackendPidGetProc(pid); + LocalPgBackendStatus *local_beentry; /* * BackendPidGetProc returns NULL if the pid isn't valid; but by the time @@ -72,9 +73,34 @@ pg_signal_backend(int pid, int sig, char *msg) return SIGNAL_BACKEND_ERROR; } + local_beentry = pgstat_fetch_stat_local_beentry_by_pid(pid); + /* Only allow superusers to signal superuser-owned backends. */ if (superuser_arg(proc->roleId) && !superuser()) - return SIGNAL_BACKEND_NOSUPERUSER; + { + Oid role; + char * appname; + + if (local_beentry == NULL) { + return SIGNAL_BACKEND_NOSUPERUSER; + } + + role = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); + appname = local_beentry->backendStatus.st_appname; + + // only allow mdb_admin to kill su queries + if (!is_member_of_role(GetUserId(), role)) { + return SIGNAL_BACKEND_NOSUPERUSER; + } + + if (local_beentry->backendStatus.st_backendType == B_AUTOVAC_WORKER) { + // ok + } else if (appname != NULL && strcmp(appname, "MDB") == 0) { + // ok + } else { + return SIGNAL_BACKEND_NOSUPERUSER; + } + } /* Users can signal backends they have role membership in. */ if (!has_privs_of_role(GetUserId(), proc->roleId) && diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c index 9a0918bceff..217483c1c61 100644 --- a/src/backend/utils/activity/backend_status.c +++ b/src/backend/utils/activity/backend_status.c @@ -1102,6 +1102,22 @@ pgstat_fetch_stat_local_beentry(int beid) return &localBackendStatusTable[beid - 1]; } +/* -- mdb admin patch -- */ +LocalPgBackendStatus * +pgstat_fetch_stat_local_beentry_by_pid(int pid) +{ + pgstat_read_current_status(); + + for (int i = 1; i <= localNumBackends; ++i) { + if (localBackendStatusTable[i - 1].backendStatus.st_procpid == pid) { + return &localBackendStatusTable[i - 1]; + } + } + + return NULL; +} + +/* -- mdb admin patch end -- */ /* ---------- * pgstat_fetch_stat_numbackends() - diff --git a/src/backend/utils/adt/acl.c b/src/backend/utils/adt/acl.c index 714a536e93d..fc566a575f4 100644 --- a/src/backend/utils/adt/acl.c +++ b/src/backend/utils/adt/acl.c @@ -5012,6 +5012,60 @@ has_privs_of_role(Oid member, Oid role) } +// -- non-upstream patch begin +/* + * Is userId allowed to bypass ownership check + * and tranfer onwership to ownerId role? + */ +bool +mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId) +{ + Oid mdb_admin_roleoid; + /* + * Never allow nobody to grant objects to + * superusers. + * This can result in various CVE. + * For paranoic reasons, check this even before + * membership of mdb_admin role. + */ + if (superuser_arg(ownerId)) { + return false; + } + + mdb_admin_roleoid = get_role_oid("mdb_admin", true /* superuser suggested to be mdb_admin*/); + /* Is userId actually member of mdb admin? */ + if (!is_member_of_role(userId, mdb_admin_roleoid)) { + /* if no, disallow. */ + return false; + } + + /* + * Now, we need to check if ownerId + * is some dangerous role to trasfer membership to. + * + * For now, we check that ownerId does not have + * priviledge to execute server program or/and + * read/write server files. + */ + + if (has_privs_of_role(ownerId, ROLE_PG_READ_SERVER_FILES)) { + return false; + } + + if (has_privs_of_role(ownerId, ROLE_PG_WRITE_SERVER_FILES)) { + return false; + } + + if (has_privs_of_role(ownerId, ROLE_PG_EXECUTE_SERVER_PROGRAM)) { + return false; + } + + /* All checks passed, hope will not be hacked here (again) */ + return true; +} + +// -- non-upstream patch end + /* * Is member a member of role (directly or indirectly)? * @@ -5051,6 +5105,64 @@ check_is_member_of_role(Oid member, Oid role) GetUserNameFromId(role, false)))); } +// -- mdb admin patch +/* + * check_mdb_admin_is_member_of_role + * is_member_of_role with a standard permission-violation error if not in usual case + * Is case `member` in mdb_admin we check that role is neither of superuser, pg_read/write + * server files nor pg_execute_server_program + */ +void +check_mdb_admin_is_member_of_role(Oid member, Oid role) +{ + Oid mdb_admin_roleoid; + /* fast path - if we are superuser, its ok */ + if (superuser_arg(member)) { + return; + } + + mdb_admin_roleoid = get_role_oid("mdb_admin", true /* superuser suggested to be mdb_admin*/); + /* Is userId actually member of mdb admin? */ + if (is_member_of_role(member, mdb_admin_roleoid)) { + /* role is mdb admin */ + if (superuser_arg(role)) { + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("cannot transfer ownership to superuser \"%s\"", + GetUserNameFromId(role, false)))); + } + + if (has_privs_of_role(role, ROLE_PG_READ_SERVER_FILES)) { + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("cannot transfer ownership to pg_read_server_files role in Cloud"))); + } + + if (has_privs_of_role(role, ROLE_PG_WRITE_SERVER_FILES)) { + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("cannot transfer ownership to pg_write_server_files role in Cloud"))); + } + + if (has_privs_of_role(role, ROLE_PG_EXECUTE_SERVER_PROGRAM)) { + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("cannot transfer ownership to pg_execute_server_program role in Cloud"))); + } + } else { + /* if no, check membership transfer in usual way. */ + + if (!is_member_of_role(member, role)) { + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be member of role \"%s\"", + GetUserNameFromId(role, false)))); + } + } +} + +// -- mdb admin patch + /* * Is member a member of role, not considering superuserness? * diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index d8cdf027df5..f73f99851b7 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -4918,7 +4918,7 @@ static struct config_enum ConfigureNamesEnum[] = { {"session_replication_role", PGC_SUSET, CLIENT_CONN_STATEMENT, gettext_noop("Sets the session's behavior for triggers and rewrite rules."), - NULL + NULL, 0, 0, 0, 0, 0, 0, 0, 0, NULL, NULL, NULL, 0, true, }, &SessionReplicationRole, SESSION_REPLICATION_ROLE_ORIGIN, session_replication_role_options, diff --git a/src/include/utils/acl.h b/src/include/utils/acl.h index 223175099bd..271ac942f6f 100644 --- a/src/include/utils/acl.h +++ b/src/include/utils/acl.h @@ -210,6 +210,13 @@ extern bool has_privs_of_role(Oid member, Oid role); extern bool is_member_of_role(Oid member, Oid role); extern bool is_member_of_role_nosuper(Oid member, Oid role); extern bool is_admin_of_role(Oid member, Oid role); + +// -- non-upstream patch begin +extern bool mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId); + +extern void check_mdb_admin_is_member_of_role(Oid member, Oid role); +// -- non-upstream patch end + extern void check_is_member_of_role(Oid member, Oid role); extern Oid get_role_oid(const char *rolename, bool missing_ok); extern Oid get_role_oid_or_public(const char *rolename); diff --git a/src/include/utils/backend_status.h b/src/include/utils/backend_status.h index 139b7355d13..139646d4a40 100644 --- a/src/include/utils/backend_status.h +++ b/src/include/utils/backend_status.h @@ -319,6 +319,9 @@ extern uint64 pgstat_get_my_query_id(void); extern int pgstat_fetch_stat_numbackends(void); extern PgBackendStatus *pgstat_fetch_stat_beentry(int beid); extern LocalPgBackendStatus *pgstat_fetch_stat_local_beentry(int beid); +/* -- mdb admin patch -- */ +extern LocalPgBackendStatus *pgstat_fetch_stat_local_beentry_by_pid(int pid); +/* -- mdb admin patch end -- */ extern char *pgstat_clip_activity(const char *raw_activity); diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h index 17d2a166b09..08584e4db54 100644 --- a/src/include/utils/guc_tables.h +++ b/src/include/utils/guc_tables.h @@ -204,6 +204,8 @@ struct config_generic char *sourcefile; /* file current setting is from (NULL if not * set in config file) */ int sourceline; /* line in source file */ + + bool mdb_admin_allowed; /* is mdb admin allowed to change this, makes sence only for superuser/not superuser ctx */ }; /* bit values in status field */ diff --git a/src/test/Makefile b/src/test/Makefile index d84edb282df..150c4e97b73 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -18,6 +18,9 @@ SUBDIRS = perl regress isolation modules authentication recovery SUBDIRS += fsync walrep heap_checksum isolation2 fdw singlenode_regress singlenode_isolation2 +# MDB addon +SUBDIRS += mdb_admin + # Test suites that are not safe by default but can be run if selected # by the user via the whitespace-separated list in variable # PG_TEST_EXTRA: diff --git a/src/test/mdb_admin/.gitignore b/src/test/mdb_admin/.gitignore new file mode 100644 index 00000000000..871e943d50e --- /dev/null +++ b/src/test/mdb_admin/.gitignore @@ -0,0 +1,2 @@ +# Generated by test suite +/tmp_check/ diff --git a/src/test/mdb_admin/Makefile b/src/test/mdb_admin/Makefile new file mode 100644 index 00000000000..e4e82367da9 --- /dev/null +++ b/src/test/mdb_admin/Makefile @@ -0,0 +1,23 @@ +#------------------------------------------------------------------------- +# +# Makefile for src/test/mdb_admin +# +# Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group +# Portions Copyright (c) 1994, Regents of the University of California +# +# src/test/mdb_admin/Makefile +# +#------------------------------------------------------------------------- + +subdir = src/test/mdb_admin +top_builddir = ../../.. +include $(top_builddir)/src/Makefile.global + +check: + $(prove_check) + +installcheck: + $(prove_installcheck) + +clean distclean maintainer-clean: + rm -rf tmp_check diff --git a/src/test/mdb_admin/t/signals.pl b/src/test/mdb_admin/t/signals.pl new file mode 100644 index 00000000000..a11db27a527 --- /dev/null +++ b/src/test/mdb_admin/t/signals.pl @@ -0,0 +1,74 @@ + +# Copyright (c) 2024-2024, MDB, Mother Russia + +# Minimal test testing streaming replication +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Initialize primary node +my $node_primary = PostgreSQL::Test::Cluster->new('primary'); +$node_primary->init(); +$node_primary->start; + +# Create some content on primary and check its presence in standby nodes +$node_primary->safe_psql('postgres', + " + CREATE DATABASE regress; + CREATE ROLE mdb_admin; + CREATE ROLE mdb_reg_lh_1; + CREATE ROLE mdb_reg_lh_2; + GRANT pg_signal_backend TO mdb_admin; + GRANT pg_signal_backend TO mdb_reg_lh_1; + GRANT mdb_admin TO mdb_reg_lh_2; +"); + +# Create some content on primary and check its presence in standby nodes +$node_primary->safe_psql('regress', + " + CREATE TABLE tab_int(i int); + INSERT INTO tab_int SELECT * FROm generate_series(1, 1000000); + ALTER SYSTEM SET autovacuum_vacuum_cost_limit TO 1; + ALTER SYSTEM SET autovacuum_vacuum_cost_delay TO 100; + ALTER SYSTEM SET autovacuum_naptime TO 1; +"); + +$node_primary->restart; + +sleep 1; + +my $res_pid = $node_primary->safe_psql('regress', + " + SELECT pid FROM pg_stat_activity WHERE backend_type = 'autovacuum worker' and datname = 'regress';; +"); + + +print "pid is $res_pid\n"; + +ok(1); + + +my ($res_reg_lh_1, $stdout_reg_lh_1, $stderr_reg_lh_1) = $node_primary->psql('regress', + " + SET ROLE mdb_reg_lh_1; + SELECT pg_terminate_backend($res_pid); +"); + +# print ($res_reg_lh_1, $stdout_reg_lh_1, $stderr_reg_lh_1, "\n"); + +ok($res_reg_lh_1 != 0, "should fail for non-mdb_admin"); +like($stderr_reg_lh_1, qr/ERROR: must be a superuser to terminate superuser process/, "matches"); + +my ($res_reg_lh_2, $stdout_reg_lh_2, $stderr_reg_lh_2) = $node_primary->psql('regress', + " + SET ROLE mdb_reg_lh_2; + SELECT pg_terminate_backend($res_pid); +"); + +ok($res_reg_lh_2 == 0, "should success for mdb_admin"); + +# print ($res_reg_lh_2, $stdout_reg_lh_2, $stderr_reg_lh_2, "\n"); + +done_testing(); \ No newline at end of file diff --git a/src/test/regress/expected/create_function_3.out b/src/test/regress/expected/create_function_3.out index 8380df1591f..7842a3c1c82 100644 --- a/src/test/regress/expected/create_function_3.out +++ b/src/test/regress/expected/create_function_3.out @@ -166,10 +166,10 @@ SET SESSION AUTHORIZATION regress_unpriv_user; SET search_path TO temp_func_test, public; ALTER FUNCTION functest_E_1(int) NOT LEAKPROOF; ALTER FUNCTION functest_E_2(int) LEAKPROOF; -ERROR: only superuser can define a leakproof function +ERROR: only superuser or mdb_admin can define a leakproof function CREATE FUNCTION functest_E_3(int) RETURNS bool LANGUAGE 'sql' LEAKPROOF AS 'SELECT $1 < 200'; -- fail -ERROR: only superuser can define a leakproof function +ERROR: only superuser or mdb_admin can define a leakproof function RESET SESSION AUTHORIZATION; -- -- CALLED ON NULL INPUT | RETURNS NULL ON NULL INPUT | STRICT diff --git a/src/test/regress/expected/create_function_3_optimizer.out b/src/test/regress/expected/create_function_3_optimizer.out index 3ae669d518a..3256709e1aa 100644 --- a/src/test/regress/expected/create_function_3_optimizer.out +++ b/src/test/regress/expected/create_function_3_optimizer.out @@ -166,10 +166,10 @@ SET SESSION AUTHORIZATION regress_unpriv_user; SET search_path TO temp_func_test, public; ALTER FUNCTION functest_E_1(int) NOT LEAKPROOF; ALTER FUNCTION functest_E_2(int) LEAKPROOF; -ERROR: only superuser can define a leakproof function +ERROR: only superuser or mdb_admin can define a leakproof function CREATE FUNCTION functest_E_3(int) RETURNS bool LANGUAGE 'sql' LEAKPROOF AS 'SELECT $1 < 200'; -- fail -ERROR: only superuser can define a leakproof function +ERROR: only superuser or mdb_admin can define a leakproof function RESET SESSION AUTHORIZATION; -- -- CALLED ON NULL INPUT | RETURNS NULL ON NULL INPUT | STRICT diff --git a/src/test/regress/expected/mdb_admin.out b/src/test/regress/expected/mdb_admin.out new file mode 100644 index 00000000000..5fc2dab10cb --- /dev/null +++ b/src/test/regress/expected/mdb_admin.out @@ -0,0 +1,81 @@ +CREATE ROLE regress_mdb_admin_user1; +CREATE ROLE regress_mdb_admin_user2; +CREATE ROLE regress_mdb_admin_user3; +CREATE ROLE mdb_admin; +CREATE ROLE regress_superuser WITH SUPERUSER; +GRANT mdb_admin TO regress_mdb_admin_user1; +GRANT CREATE ON DATABASE regression TO regress_mdb_admin_user2; +GRANT CREATE ON DATABASE regression TO regress_mdb_admin_user3; +-- mdb admin trasfers ownership to another role +SET ROLE regress_mdb_admin_user2; +CREATE FUNCTION regress_mdb_admin_add(integer, integer) RETURNS integer + AS 'SELECT $1 + $2;' + LANGUAGE SQL + IMMUTABLE + RETURNS NULL ON NULL INPUT; +CREATE SCHEMA regress_mdb_admin_schema; +GRANT CREATE ON SCHEMA regress_mdb_admin_schema TO regress_mdb_admin_user3; +CREATE TABLE regress_mdb_admin_schema.regress_mdb_admin_table(); +CREATE TABLE regress_mdb_admin_table(); +CREATE VIEW regress_mdb_admin_view as SELECT 1; +SET ROLE regress_mdb_admin_user1; +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO regress_mdb_admin_user3; +ALTER VIEW regress_mdb_admin_view OWNER TO regress_mdb_admin_user3; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO regress_mdb_admin_user3; +ALTER TABLE regress_mdb_admin_table OWNER TO regress_mdb_admin_user3; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO regress_mdb_admin_user3; +-- mdb admin fails to transfer ownership to superusers and system roles +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO regress_superuser; +ERROR: cannot transfer ownership to superuser "regress_superuser" +ALTER VIEW regress_mdb_admin_view OWNER TO regress_superuser; +ERROR: cannot transfer ownership to superuser "regress_superuser" +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO regress_superuser; +ERROR: cannot transfer ownership to superuser "regress_superuser" +ALTER TABLE regress_mdb_admin_table OWNER TO regress_superuser; +ERROR: cannot transfer ownership to superuser "regress_superuser" +ALTER SCHEMA regress_mdb_admin_schema OWNER TO regress_superuser; +ERROR: cannot transfer ownership to superuser "regress_superuser" +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_execute_server_program; +ERROR: cannot transfer ownership to pg_execute_server_program role in Cloud +ALTER VIEW regress_mdb_admin_view OWNER TO pg_execute_server_program; +ERROR: cannot transfer ownership to pg_execute_server_program role in Cloud +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_execute_server_program; +ERROR: cannot transfer ownership to pg_execute_server_program role in Cloud +ALTER TABLE regress_mdb_admin_table OWNER TO pg_execute_server_program; +ERROR: cannot transfer ownership to pg_execute_server_program role in Cloud +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_execute_server_program; +ERROR: cannot transfer ownership to pg_execute_server_program role in Cloud +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_write_server_files; +ERROR: cannot transfer ownership to pg_write_server_files role in Cloud +ALTER VIEW regress_mdb_admin_view OWNER TO pg_write_server_files; +ERROR: cannot transfer ownership to pg_write_server_files role in Cloud +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_write_server_files; +ERROR: cannot transfer ownership to pg_write_server_files role in Cloud +ALTER TABLE regress_mdb_admin_table OWNER TO pg_write_server_files; +ERROR: cannot transfer ownership to pg_write_server_files role in Cloud +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_write_server_files; +ERROR: cannot transfer ownership to pg_write_server_files role in Cloud +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_read_server_files; +ERROR: cannot transfer ownership to pg_read_server_files role in Cloud +ALTER VIEW regress_mdb_admin_view OWNER TO pg_read_server_files; +ERROR: cannot transfer ownership to pg_read_server_files role in Cloud +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_server_files; +ERROR: cannot transfer ownership to pg_read_server_files role in Cloud +ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_server_files; +ERROR: cannot transfer ownership to pg_read_server_files role in Cloud +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_server_files; +ERROR: cannot transfer ownership to pg_read_server_files role in Cloud +-- end tests +RESET SESSION AUTHORIZATION; +-- +REVOKE CREATE ON DATABASE regression FROM regress_mdb_admin_user2; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_admin_user3; +DROP VIEW regress_mdb_admin_view; +DROP FUNCTION regress_mdb_admin_add; +DROP TABLE regress_mdb_admin_schema.regress_mdb_admin_table; +DROP TABLE regress_mdb_admin_table; +DROP SCHEMA regress_mdb_admin_schema; +DROP ROLE regress_mdb_admin_user1; +DROP ROLE regress_mdb_admin_user2; +DROP ROLE regress_mdb_admin_user3; +DROP ROLE mdb_admin; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index e2df0208627..6ebdd67731e 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -5,6 +5,10 @@ # this limits the number of connections needed to run the tests. # ---------- +# mdb admin simple checks + +test: mdb_admin + # run tablespace by itself, and first, because it forces a checkpoint; # we'd prefer not to have checkpoints later in the tests because that # interferes with crash-recovery testing. diff --git a/src/test/regress/sql/mdb_admin.sql b/src/test/regress/sql/mdb_admin.sql new file mode 100644 index 00000000000..8552bbdd48a --- /dev/null +++ b/src/test/regress/sql/mdb_admin.sql @@ -0,0 +1,77 @@ +CREATE ROLE regress_mdb_admin_user1; +CREATE ROLE regress_mdb_admin_user2; +CREATE ROLE regress_mdb_admin_user3; +CREATE ROLE mdb_admin; + +CREATE ROLE regress_superuser WITH SUPERUSER; + +GRANT mdb_admin TO regress_mdb_admin_user1; +GRANT CREATE ON DATABASE regression TO regress_mdb_admin_user2; +GRANT CREATE ON DATABASE regression TO regress_mdb_admin_user3; + +-- mdb admin trasfers ownership to another role + +SET ROLE regress_mdb_admin_user2; +CREATE FUNCTION regress_mdb_admin_add(integer, integer) RETURNS integer + AS 'SELECT $1 + $2;' + LANGUAGE SQL + IMMUTABLE + RETURNS NULL ON NULL INPUT; + +CREATE SCHEMA regress_mdb_admin_schema; +GRANT CREATE ON SCHEMA regress_mdb_admin_schema TO regress_mdb_admin_user3; +CREATE TABLE regress_mdb_admin_schema.regress_mdb_admin_table(); +CREATE TABLE regress_mdb_admin_table(); +CREATE VIEW regress_mdb_admin_view as SELECT 1; +SET ROLE regress_mdb_admin_user1; + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO regress_mdb_admin_user3; +ALTER VIEW regress_mdb_admin_view OWNER TO regress_mdb_admin_user3; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO regress_mdb_admin_user3; +ALTER TABLE regress_mdb_admin_table OWNER TO regress_mdb_admin_user3; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO regress_mdb_admin_user3; + + +-- mdb admin fails to transfer ownership to superusers and system roles + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO regress_superuser; +ALTER VIEW regress_mdb_admin_view OWNER TO regress_superuser; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO regress_superuser; +ALTER TABLE regress_mdb_admin_table OWNER TO regress_superuser; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO regress_superuser; + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_execute_server_program; +ALTER VIEW regress_mdb_admin_view OWNER TO pg_execute_server_program; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_execute_server_program; +ALTER TABLE regress_mdb_admin_table OWNER TO pg_execute_server_program; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_execute_server_program; + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_write_server_files; +ALTER VIEW regress_mdb_admin_view OWNER TO pg_write_server_files; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_write_server_files; +ALTER TABLE regress_mdb_admin_table OWNER TO pg_write_server_files; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_write_server_files; + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_read_server_files; +ALTER VIEW regress_mdb_admin_view OWNER TO pg_read_server_files; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_server_files; +ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_server_files; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_server_files; + + +-- end tests + +RESET SESSION AUTHORIZATION; +-- +REVOKE CREATE ON DATABASE regression FROM regress_mdb_admin_user2; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_admin_user3; + +DROP VIEW regress_mdb_admin_view; +DROP FUNCTION regress_mdb_admin_add; +DROP TABLE regress_mdb_admin_schema.regress_mdb_admin_table; +DROP TABLE regress_mdb_admin_table; +DROP SCHEMA regress_mdb_admin_schema; +DROP ROLE regress_mdb_admin_user1; +DROP ROLE regress_mdb_admin_user2; +DROP ROLE regress_mdb_admin_user3; +DROP ROLE mdb_admin; diff --git a/src/test/singlenode_regress/expected/create_function_3.out b/src/test/singlenode_regress/expected/create_function_3.out index 3a4fd451471..6423fdb7965 100644 --- a/src/test/singlenode_regress/expected/create_function_3.out +++ b/src/test/singlenode_regress/expected/create_function_3.out @@ -166,10 +166,10 @@ SET SESSION AUTHORIZATION regress_unpriv_user; SET search_path TO temp_func_test, public; ALTER FUNCTION functest_E_1(int) NOT LEAKPROOF; ALTER FUNCTION functest_E_2(int) LEAKPROOF; -ERROR: only superuser can define a leakproof function +ERROR: only superuser or mdb_admin can define a leakproof function CREATE FUNCTION functest_E_3(int) RETURNS bool LANGUAGE 'sql' LEAKPROOF AS 'SELECT $1 < 200'; -- fail -ERROR: only superuser can define a leakproof function +ERROR: only superuser or mdb_admin can define a leakproof function RESET SESSION AUTHORIZATION; -- -- CALLED ON NULL INPUT | RETURNS NULL ON NULL INPUT | STRICT From ea9e36fb451b7a9c0f1d894153ba790d13c493e6 Mon Sep 17 00:00:00 2001 From: reshke Date: Tue, 30 Sep 2025 14:43:07 +0500 Subject: [PATCH 05/22] Role mdb_superuser: feature and regress testsing (#5) This commit introduces new mdb internal role mdb_superuser. Role is capaple of: GRANT/REVOKE any set of priviledges to/from any object in database. Has power of pg_database_owner in any database, including: DROP any object in database (except system catalog and stuff) Role is NOT capaple of: Create database, role, extension or alter other roles with such priviledges. Transfer ownership to /pass has_priv of roles: PG_READ_ALL_DATA PG_WRITE_ALL_DATA PG_EXECUTE_SERVER_PROGRAM PG_READ_SERVER_FILES PG_WRITE_SERVER_FILES PG_DATABASE_OWNER Fix configure.ac USE_MDBLOCALES option handling Apply autoreconf stuff Set missing ok parameter ito true while acquiring mdb_superuser oid In regress tests, nobody creates mdb_superuser role, so missing ok is fine Allow mdb_superuser to have power of pg_database_owner Allow mdb_superuser to alter objects and grant ACl to objects, owner by pg_database_owner. Also, when acl check, allow mdb_supersuer use pg_database_owner role power to pass check --- src/backend/commands/functioncmds.c | 4 +- src/backend/utils/adt/acl.c | 126 ++++++++++++----- src/backend/utils/misc/guc.c | 12 +- src/include/utils/acl.h | 1 + src/test/regress/expected/mdb_admin.out | 55 +++++--- src/test/regress/expected/mdb_superuser.out | 115 ++++++++++++++++ src/test/regress/expected/test_setup.out | 5 + src/test/regress/parallel_schedule | 8 +- src/test/regress/sql/mdb_admin.sql | 16 ++- src/test/regress/sql/mdb_superuser.sql | 144 ++++++++++++++++++++ src/test/regress/sql/test_setup.sql | 6 + 11 files changed, 431 insertions(+), 61 deletions(-) create mode 100644 src/test/regress/expected/mdb_superuser.out create mode 100644 src/test/regress/expected/test_setup.out create mode 100644 src/test/regress/sql/mdb_superuser.sql create mode 100644 src/test/regress/sql/test_setup.sql diff --git a/src/backend/commands/functioncmds.c b/src/backend/commands/functioncmds.c index 8a570fa6965..1ab3b36dd59 100644 --- a/src/backend/commands/functioncmds.c +++ b/src/backend/commands/functioncmds.c @@ -1526,7 +1526,7 @@ CreateFunction(ParseState *pstate, CreateFunctionStmt *stmt) */ if (isLeakProof && !superuser()) { - Oid role = get_role_oid("mdb_admin", true); + Oid role = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); if (!is_member_of_role(GetUserId(), role)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), @@ -1857,7 +1857,7 @@ AlterFunction(ParseState *pstate, AlterFunctionStmt *stmt) procForm->proleakproof = intVal(leakproof_item->arg); if (procForm->proleakproof && !superuser()) { - Oid role = get_role_oid("mdb_admin", true); + Oid role = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); if (!is_member_of_role(GetUserId(), role)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), diff --git a/src/backend/utils/adt/acl.c b/src/backend/utils/adt/acl.c index fc566a575f4..e3463f636ae 100644 --- a/src/backend/utils/adt/acl.c +++ b/src/backend/utils/adt/acl.c @@ -116,6 +116,7 @@ static AclResult pg_role_aclcheck(Oid role_oid, Oid roleid, AclMode mode); static void RoleMembershipCacheCallback(Datum arg, int cacheid, uint32 hashvalue); +static bool has_privs_of_unwanted_system_role(Oid role); /* * getid @@ -4991,9 +4992,65 @@ roles_is_member_of(Oid roleid, enum RoleRecurseType type, * set; for such roles, membership implies the ability to do SET ROLE, but * the privileges are not available until you've done so. */ + +/* +* This is basically original postgresql privs-check function +*/ + +// -- mdb_superuser patch + +bool +has_privs_of_role_strict(Oid member, Oid role) +{ + /* Fast path for simple case */ + if (member == role) + return true; + + /* Superusers have every privilege, so are part of every role */ + if (superuser_arg(member)) + return true; + + /* + * Find all the roles that member has the privileges of, including + * multi-level recursion, then see if target role is any one of them. + */ + return list_member_oid(roles_is_member_of(member, ROLERECURSE_PRIVS, + InvalidOid, NULL), + role); +} + +/* +* Check that role is either one of "dangerous" system role +* or has "strict" (not through mdb_admin or mdb_superuser) +* privs of this role +*/ + +static bool +has_privs_of_unwanted_system_role(Oid role) { + if (has_privs_of_role_strict(role, ROLE_PG_READ_SERVER_FILES)) { + return true; + } + if (has_privs_of_role_strict(role, ROLE_PG_WRITE_SERVER_FILES)) { + return true; + } + if (has_privs_of_role_strict(role, ROLE_PG_EXECUTE_SERVER_PROGRAM)) { + return true; + } + if (has_privs_of_role_strict(role, ROLE_PG_READ_ALL_DATA)) { + return true; + } + if (has_privs_of_role_strict(role, ROLE_PG_WRITE_ALL_DATA)) { + return true; + } + + return false; +} + bool has_privs_of_role(Oid member, Oid role) { + Oid mdb_superuser_roleoid; + /* Fast path for simple case */ if (member == role) return true; @@ -5002,6 +5059,23 @@ has_privs_of_role(Oid member, Oid role) if (superuser_arg(member)) return true; + mdb_superuser_roleoid = get_role_oid("mdb_superuser", true /*if nodoby created mdb_superuser role in this database*/); + + if (is_member_of_role(member, mdb_superuser_roleoid)) { + /* if target role is superuser, disallow */ + if (!superuser_arg(role)) { + /* we want mdb_roles_admin to bypass + * has_priv_of_roles test + * if target role is neither superuser nor + * some dangerous system role + */ + if (!has_privs_of_unwanted_system_role(role)) { + return true; + } + } + } + + /* * Find all the roles that member has the privileges of, including * multi-level recursion, then see if target role is any one of them. @@ -5011,6 +5085,7 @@ has_privs_of_role(Oid member, Oid role) role); } +// -- mdb_superuser patch // -- non-upstream patch begin /* @@ -5032,7 +5107,7 @@ mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId) return false; } - mdb_admin_roleoid = get_role_oid("mdb_admin", true /* superuser suggested to be mdb_admin*/); + mdb_admin_roleoid = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); /* Is userId actually member of mdb admin? */ if (!is_member_of_role(userId, mdb_admin_roleoid)) { /* if no, disallow. */ @@ -5045,23 +5120,11 @@ mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId) * * For now, we check that ownerId does not have * priviledge to execute server program or/and - * read/write server files. + * read/write server files, or/and pg read/write all data */ - if (has_privs_of_role(ownerId, ROLE_PG_READ_SERVER_FILES)) { - return false; - } - - if (has_privs_of_role(ownerId, ROLE_PG_WRITE_SERVER_FILES)) { - return false; - } - - if (has_privs_of_role(ownerId, ROLE_PG_EXECUTE_SERVER_PROGRAM)) { - return false; - } - /* All checks passed, hope will not be hacked here (again) */ - return true; + return !has_privs_of_unwanted_system_role(ownerId); } // -- non-upstream patch end @@ -5110,7 +5173,7 @@ check_is_member_of_role(Oid member, Oid role) * check_mdb_admin_is_member_of_role * is_member_of_role with a standard permission-violation error if not in usual case * Is case `member` in mdb_admin we check that role is neither of superuser, pg_read/write - * server files nor pg_execute_server_program + * server files nor pg_execute_server_program or pg_read/write all data */ void check_mdb_admin_is_member_of_role(Oid member, Oid role) @@ -5121,9 +5184,10 @@ check_mdb_admin_is_member_of_role(Oid member, Oid role) return; } - mdb_admin_roleoid = get_role_oid("mdb_admin", true /* superuser suggested to be mdb_admin*/); + mdb_admin_roleoid = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); /* Is userId actually member of mdb admin? */ if (is_member_of_role(member, mdb_admin_roleoid)) { + /* role is mdb admin */ if (superuser_arg(role)) { ereport(ERROR, @@ -5132,22 +5196,10 @@ check_mdb_admin_is_member_of_role(Oid member, Oid role) GetUserNameFromId(role, false)))); } - if (has_privs_of_role(role, ROLE_PG_READ_SERVER_FILES)) { + if (has_privs_of_unwanted_system_role(role)) { ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("cannot transfer ownership to pg_read_server_files role in Cloud"))); - } - - if (has_privs_of_role(role, ROLE_PG_WRITE_SERVER_FILES)) { - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("cannot transfer ownership to pg_write_server_files role in Cloud"))); - } - - if (has_privs_of_role(role, ROLE_PG_EXECUTE_SERVER_PROGRAM)) { - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("cannot transfer ownership to pg_execute_server_program role in Cloud"))); + errmsg("forbidden to transfer ownership to this system role in Cloud"))); } } else { /* if no, check membership transfer in usual way. */ @@ -5287,6 +5339,7 @@ select_best_grantor(Oid roleId, AclMode privileges, List *roles_list; int nrights; ListCell *l; + Oid mdb_superuser_roleoid; /* * The object owner is always treated as having all grant options, so if @@ -5301,6 +5354,16 @@ select_best_grantor(Oid roleId, AclMode privileges, return; } + mdb_superuser_roleoid = get_role_oid("mdb_superuser", true /*if nodoby created mdb_superuser role in this database*/); + + if (is_member_of_role(GetUserId(), mdb_superuser_roleoid) + && has_privs_of_role(GetUserId(), ownerId)) { + *grantorId = mdb_superuser_roleoid; + AclMode mdb_superuser_allowed_privs = needed_goptions; + *grantOptions = mdb_superuser_allowed_privs; + return; + } + /* * Otherwise we have to do a careful search to see if roleId has the * privileges of any suitable role. Note: we can hang onto the result of @@ -5309,7 +5372,6 @@ select_best_grantor(Oid roleId, AclMode privileges, */ roles_list = roles_is_member_of(roleId, ROLERECURSE_PRIVS, InvalidOid, NULL); - /* initialize candidate result as default */ *grantorId = roleId; *grantOptions = ACL_NO_RIGHTS; diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index f73f99851b7..30f417b13fa 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -7615,6 +7615,7 @@ set_config_option(const char *name, const char *value, void *newextra = NULL; bool prohibitValueChange = false; bool makeDefault; + Oid role; if (elevel == 0) { @@ -7772,10 +7773,13 @@ set_config_option(const char *name, const char *value, case PGC_SUSET: if (context == PGC_USERSET || context == PGC_BACKEND) { - ereport(elevel, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("permission denied to set parameter \"%s\"", - name))); + role = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); + if (!(record->mdb_admin_allowed && is_member_of_role(GetUserId(), role))) { + ereport(elevel, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("permission denied to set parameter \"%s\"", + name))); + } return 0; } break; diff --git a/src/include/utils/acl.h b/src/include/utils/acl.h index 271ac942f6f..49068f04b2f 100644 --- a/src/include/utils/acl.h +++ b/src/include/utils/acl.h @@ -207,6 +207,7 @@ extern AclMode aclmask(const Acl *acl, Oid roleid, Oid ownerId, extern int aclmembers(const Acl *acl, Oid **roleids); extern bool has_privs_of_role(Oid member, Oid role); +extern bool has_privs_of_role_strict(Oid member, Oid role); extern bool is_member_of_role(Oid member, Oid role); extern bool is_member_of_role_nosuper(Oid member, Oid role); extern bool is_admin_of_role(Oid member, Oid role); diff --git a/src/test/regress/expected/mdb_admin.out b/src/test/regress/expected/mdb_admin.out index 5fc2dab10cb..e4dfc436802 100644 --- a/src/test/regress/expected/mdb_admin.out +++ b/src/test/regress/expected/mdb_admin.out @@ -1,7 +1,6 @@ CREATE ROLE regress_mdb_admin_user1; CREATE ROLE regress_mdb_admin_user2; CREATE ROLE regress_mdb_admin_user3; -CREATE ROLE mdb_admin; CREATE ROLE regress_superuser WITH SUPERUSER; GRANT mdb_admin TO regress_mdb_admin_user1; GRANT CREATE ON DATABASE regression TO regress_mdb_admin_user2; @@ -24,7 +23,7 @@ ALTER VIEW regress_mdb_admin_view OWNER TO regress_mdb_admin_user3; ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO regress_mdb_admin_user3; ALTER TABLE regress_mdb_admin_table OWNER TO regress_mdb_admin_user3; ALTER SCHEMA regress_mdb_admin_schema OWNER TO regress_mdb_admin_user3; --- mdb admin fails to transfer ownership to superusers and system roles +-- mdb admin fails to transfer ownership to superusers and particular system roles ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO regress_superuser; ERROR: cannot transfer ownership to superuser "regress_superuser" ALTER VIEW regress_mdb_admin_view OWNER TO regress_superuser; @@ -36,35 +35,55 @@ ERROR: cannot transfer ownership to superuser "regress_superuser" ALTER SCHEMA regress_mdb_admin_schema OWNER TO regress_superuser; ERROR: cannot transfer ownership to superuser "regress_superuser" ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_execute_server_program; -ERROR: cannot transfer ownership to pg_execute_server_program role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER VIEW regress_mdb_admin_view OWNER TO pg_execute_server_program; -ERROR: cannot transfer ownership to pg_execute_server_program role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_execute_server_program; -ERROR: cannot transfer ownership to pg_execute_server_program role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER TABLE regress_mdb_admin_table OWNER TO pg_execute_server_program; -ERROR: cannot transfer ownership to pg_execute_server_program role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_execute_server_program; -ERROR: cannot transfer ownership to pg_execute_server_program role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_write_server_files; -ERROR: cannot transfer ownership to pg_write_server_files role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER VIEW regress_mdb_admin_view OWNER TO pg_write_server_files; -ERROR: cannot transfer ownership to pg_write_server_files role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_write_server_files; -ERROR: cannot transfer ownership to pg_write_server_files role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER TABLE regress_mdb_admin_table OWNER TO pg_write_server_files; -ERROR: cannot transfer ownership to pg_write_server_files role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_write_server_files; -ERROR: cannot transfer ownership to pg_write_server_files role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_read_server_files; -ERROR: cannot transfer ownership to pg_read_server_files role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER VIEW regress_mdb_admin_view OWNER TO pg_read_server_files; -ERROR: cannot transfer ownership to pg_read_server_files role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_server_files; -ERROR: cannot transfer ownership to pg_read_server_files role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_server_files; -ERROR: cannot transfer ownership to pg_read_server_files role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_server_files; -ERROR: cannot transfer ownership to pg_read_server_files role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_write_all_data; +ERROR: forbidden to transfer ownership to this system role in Cloud +ALTER VIEW regress_mdb_admin_view OWNER TO pg_write_all_data; +ERROR: forbidden to transfer ownership to this system role in Cloud +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_write_all_data; +ERROR: forbidden to transfer ownership to this system role in Cloud +ALTER TABLE regress_mdb_admin_table OWNER TO pg_write_all_data; +ERROR: forbidden to transfer ownership to this system role in Cloud +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_write_all_data; +ERROR: forbidden to transfer ownership to this system role in Cloud +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_read_all_data; +ERROR: forbidden to transfer ownership to this system role in Cloud +ALTER VIEW regress_mdb_admin_view OWNER TO pg_read_all_data; +ERROR: forbidden to transfer ownership to this system role in Cloud +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_all_data; +ERROR: forbidden to transfer ownership to this system role in Cloud +ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_all_data; +ERROR: forbidden to transfer ownership to this system role in Cloud +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_all_data; +ERROR: forbidden to transfer ownership to this system role in Cloud -- end tests RESET SESSION AUTHORIZATION; -- @@ -78,4 +97,4 @@ DROP SCHEMA regress_mdb_admin_schema; DROP ROLE regress_mdb_admin_user1; DROP ROLE regress_mdb_admin_user2; DROP ROLE regress_mdb_admin_user3; -DROP ROLE mdb_admin; +DROP ROLE regress_superuser; diff --git a/src/test/regress/expected/mdb_superuser.out b/src/test/regress/expected/mdb_superuser.out new file mode 100644 index 00000000000..21bafb1011b --- /dev/null +++ b/src/test/regress/expected/mdb_superuser.out @@ -0,0 +1,115 @@ +CREATE ROLE regress_mdb_superuser_user1; +CREATE ROLE regress_mdb_superuser_user2; +CREATE ROLE regress_mdb_superuser_user3; +GRANT mdb_admin TO mdb_superuser; +CREATE ROLE regress_superuser WITH SUPERUSER; +GRANT mdb_superuser TO regress_mdb_superuser_user1; +GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user2; +GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user3; +SET ROLE regress_mdb_superuser_user2; +CREATE FUNCTION regress_mdb_superuser_add(integer, integer) RETURNS integer + AS 'SELECT $1 + $2;' + LANGUAGE SQL + IMMUTABLE + RETURNS NULL ON NULL INPUT; +CREATE SCHEMA regress_mdb_superuser_schema; +CREATE TABLE regress_mdb_superuser_schema.regress_mdb_superuser_table(); +CREATE TABLE regress_mdb_superuser_table(); +CREATE VIEW regress_mdb_superuser_view as SELECT 1; +SET ROLE regress_mdb_superuser_user3; +INSERT INTO regress_mdb_superuser_table SELECT * FROM regress_mdb_superuser_table; +ERROR: permission denied for table regress_mdb_superuser_table +SET ROLE regress_mdb_superuser_user1; +-- mdb_superuser can grant to other role +GRANT USAGE, CREATE ON SCHEMA regress_mdb_superuser_schema TO regress_mdb_superuser_user3; +GRANT ALL PRIVILEGES ON TABLE regress_mdb_superuser_table TO regress_mdb_superuser_user3; +REVOKE ALL PRIVILEGES ON TABLE regress_mdb_superuser_table FROM regress_mdb_superuser_user3; +GRANT INSERT, SELECT ON TABLE regress_mdb_superuser_table TO regress_mdb_superuser_user3; +-- grant works +SET ROLE regress_mdb_superuser_user3; +INSERT INTO regress_mdb_superuser_table SELECT * FROM regress_mdb_superuser_table; +SET ROLE mdb_superuser; +-- mdb_superuser drop object of other role +DROP TABLE regress_mdb_superuser_table; +-- mdb admin fails to transfer ownership to superusers and system roles +RESET SESSION AUTHORIZATION; +CREATE TABLE regress_superuser_table(); +SET ROLE pg_read_server_files; +CREATE TABLE regress_pgrsf_table(); +SET ROLE pg_write_server_files; +CREATE TABLE regress_pgwsf_table(); +SET ROLE pg_execute_server_program; +CREATE TABLE regress_pgxsp_table(); +SET ROLE pg_read_all_data; +CREATE TABLE regress_pgrad_table(); +SET ROLE pg_write_all_data; +CREATE TABLE regress_pgrwd_table(); +SET ROLE mdb_superuser; +-- cannot read all data (fail) +SELECT * FROM pg_authid; +ERROR: permission denied for table pg_authid +-- can not drop superuser objects, because does not has_privs_of pg_database_owner +DROP TABLE regress_superuser_table; +ERROR: must be owner of table regress_superuser_table +DROP TABLE regress_pgrsf_table; +ERROR: must be owner of table regress_pgrsf_table +DROP TABLE regress_pgwsf_table; +ERROR: must be owner of table regress_pgwsf_table +DROP TABLE regress_pgxsp_table; +ERROR: must be owner of table regress_pgxsp_table +DROP TABLE regress_pgrad_table; +ERROR: must be owner of table regress_pgrad_table +DROP TABLE regress_pgrwd_table; +ERROR: must be owner of table regress_pgrwd_table +-- does allowed to creare database, role or extension +-- or grant such priviledge +CREATE DATABASE regress_db_fail; +ERROR: permission denied to create database +CREATE ROLE regress_role_fail; +ERROR: permission denied to create role +ALTER ROLE mdb_superuser WITH CREATEROLE; +ERROR: permission denied +ALTER ROLE mdb_superuser WITH CREATEDB; +ERROR: permission denied +ALTER ROLE regress_mdb_superuser_user2 WITH CREATEROLE; +ERROR: permission denied +ALTER ROLE regress_mdb_superuser_user2 WITH CREATEDB; +ERROR: permission denied +-- mdb_superuser more powerfull than pg_database_owner +RESET SESSION AUTHORIZATION; +CREATE DATABASE regress_check_owner OWNER regress_mdb_superuser_user2; +\c regress_check_owner; +SET ROLE regress_mdb_superuser_user2; +CREATE SCHEMA regtest; +CREATE TABLE regtest.regtest(); +-- this should fail +SET ROLE regress_mdb_superuser_user3; +GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user3; +ERROR: permission denied for schema regtest +ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user3; +ERROR: permission denied for schema regtest +SET ROLE regress_mdb_superuser_user1; +GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user1; +ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user1; +\c regression +DROP DATABASE regress_check_owner; +-- end tests +RESET SESSION AUTHORIZATION; +-- +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user2; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user3; +DROP VIEW regress_mdb_superuser_view; +DROP FUNCTION regress_mdb_superuser_add; +DROP TABLE regress_mdb_superuser_schema.regress_mdb_superuser_table; +DROP TABLE regress_mdb_superuser_table; +ERROR: table "regress_mdb_superuser_table" does not exist +DROP SCHEMA regress_mdb_superuser_schema; +DROP ROLE regress_mdb_superuser_user1; +DROP ROLE regress_mdb_superuser_user2; +DROP ROLE regress_mdb_superuser_user3; +DROP TABLE regress_superuser_table; +DROP TABLE regress_pgrsf_table; +DROP TABLE regress_pgwsf_table; +DROP TABLE regress_pgxsp_table; +DROP TABLE regress_pgrad_table; +DROP TABLE regress_pgrwd_table; diff --git a/src/test/regress/expected/test_setup.out b/src/test/regress/expected/test_setup.out new file mode 100644 index 00000000000..c1cb724ef37 --- /dev/null +++ b/src/test/regress/expected/test_setup.out @@ -0,0 +1,5 @@ +-- +-- TEST_SETUP --- prepare environment expected by regression test scripts +-- +CREATE ROLE mdb_admin; +CREATE ROLE mdb_superuser; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 6ebdd67731e..b2ed818f677 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -6,13 +6,17 @@ # ---------- # mdb admin simple checks - -test: mdb_admin +test: test_setup # run tablespace by itself, and first, because it forces a checkpoint; # we'd prefer not to have checkpoints later in the tests because that # interferes with crash-recovery testing. test: tablespace + +test: mdb_admin + +test: mdb_superuser + # ---------- # The first group of parallel tests # ---------- diff --git a/src/test/regress/sql/mdb_admin.sql b/src/test/regress/sql/mdb_admin.sql index 8552bbdd48a..b6b048e5692 100644 --- a/src/test/regress/sql/mdb_admin.sql +++ b/src/test/regress/sql/mdb_admin.sql @@ -1,7 +1,6 @@ CREATE ROLE regress_mdb_admin_user1; CREATE ROLE regress_mdb_admin_user2; CREATE ROLE regress_mdb_admin_user3; -CREATE ROLE mdb_admin; CREATE ROLE regress_superuser WITH SUPERUSER; @@ -32,7 +31,7 @@ ALTER TABLE regress_mdb_admin_table OWNER TO regress_mdb_admin_user3; ALTER SCHEMA regress_mdb_admin_schema OWNER TO regress_mdb_admin_user3; --- mdb admin fails to transfer ownership to superusers and system roles +-- mdb admin fails to transfer ownership to superusers and particular system roles ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO regress_superuser; ALTER VIEW regress_mdb_admin_view OWNER TO regress_superuser; @@ -58,6 +57,17 @@ ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_se ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_server_files; ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_server_files; +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_write_all_data; +ALTER VIEW regress_mdb_admin_view OWNER TO pg_write_all_data; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_write_all_data; +ALTER TABLE regress_mdb_admin_table OWNER TO pg_write_all_data; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_write_all_data; + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_read_all_data; +ALTER VIEW regress_mdb_admin_view OWNER TO pg_read_all_data; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_all_data; +ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_all_data; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_all_data; -- end tests @@ -74,4 +84,4 @@ DROP SCHEMA regress_mdb_admin_schema; DROP ROLE regress_mdb_admin_user1; DROP ROLE regress_mdb_admin_user2; DROP ROLE regress_mdb_admin_user3; -DROP ROLE mdb_admin; +DROP ROLE regress_superuser; diff --git a/src/test/regress/sql/mdb_superuser.sql b/src/test/regress/sql/mdb_superuser.sql new file mode 100644 index 00000000000..f96338f3aec --- /dev/null +++ b/src/test/regress/sql/mdb_superuser.sql @@ -0,0 +1,144 @@ +CREATE ROLE regress_mdb_superuser_user1; +CREATE ROLE regress_mdb_superuser_user2; +CREATE ROLE regress_mdb_superuser_user3; + +GRANT mdb_admin TO mdb_superuser; + +CREATE ROLE regress_superuser WITH SUPERUSER; + +GRANT mdb_superuser TO regress_mdb_superuser_user1; + +GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user2; +GRANT CREATE ON DATABASE regression TO regress_mdb_superuser_user3; + + +SET ROLE regress_mdb_superuser_user2; + +CREATE FUNCTION regress_mdb_superuser_add(integer, integer) RETURNS integer + AS 'SELECT $1 + $2;' + LANGUAGE SQL + IMMUTABLE + RETURNS NULL ON NULL INPUT; + +CREATE SCHEMA regress_mdb_superuser_schema; +CREATE TABLE regress_mdb_superuser_schema.regress_mdb_superuser_table(); +CREATE TABLE regress_mdb_superuser_table(); +CREATE VIEW regress_mdb_superuser_view as SELECT 1; + +SET ROLE regress_mdb_superuser_user3; +INSERT INTO regress_mdb_superuser_table SELECT * FROM regress_mdb_superuser_table; + +SET ROLE regress_mdb_superuser_user1; + +-- mdb_superuser can grant to other role +GRANT USAGE, CREATE ON SCHEMA regress_mdb_superuser_schema TO regress_mdb_superuser_user3; +GRANT ALL PRIVILEGES ON TABLE regress_mdb_superuser_table TO regress_mdb_superuser_user3; +REVOKE ALL PRIVILEGES ON TABLE regress_mdb_superuser_table FROM regress_mdb_superuser_user3; + +GRANT INSERT, SELECT ON TABLE regress_mdb_superuser_table TO regress_mdb_superuser_user3; + +-- grant works +SET ROLE regress_mdb_superuser_user3; +INSERT INTO regress_mdb_superuser_table SELECT * FROM regress_mdb_superuser_table; + +SET ROLE mdb_superuser; + +-- mdb_superuser drop object of other role +DROP TABLE regress_mdb_superuser_table; +-- mdb admin fails to transfer ownership to superusers and system roles + +RESET SESSION AUTHORIZATION; + +CREATE TABLE regress_superuser_table(); + +SET ROLE pg_read_server_files; + +CREATE TABLE regress_pgrsf_table(); + +SET ROLE pg_write_server_files; + +CREATE TABLE regress_pgwsf_table(); + +SET ROLE pg_execute_server_program; + +CREATE TABLE regress_pgxsp_table(); + +SET ROLE pg_read_all_data; + +CREATE TABLE regress_pgrad_table(); + +SET ROLE pg_write_all_data; + +CREATE TABLE regress_pgrwd_table(); + +SET ROLE mdb_superuser; + +-- cannot read all data (fail) +SELECT * FROM pg_authid; + +-- can not drop superuser objects, because does not has_privs_of pg_database_owner +DROP TABLE regress_superuser_table; +DROP TABLE regress_pgrsf_table; +DROP TABLE regress_pgwsf_table; +DROP TABLE regress_pgxsp_table; +DROP TABLE regress_pgrad_table; +DROP TABLE regress_pgrwd_table; + + +-- does allowed to creare database, role or extension +-- or grant such priviledge + +CREATE DATABASE regress_db_fail; +CREATE ROLE regress_role_fail; + +ALTER ROLE mdb_superuser WITH CREATEROLE; +ALTER ROLE mdb_superuser WITH CREATEDB; + +ALTER ROLE regress_mdb_superuser_user2 WITH CREATEROLE; +ALTER ROLE regress_mdb_superuser_user2 WITH CREATEDB; + +-- mdb_superuser more powerfull than pg_database_owner + +RESET SESSION AUTHORIZATION; +CREATE DATABASE regress_check_owner OWNER regress_mdb_superuser_user2; + +\c regress_check_owner; + +SET ROLE regress_mdb_superuser_user2; +CREATE SCHEMA regtest; +CREATE TABLE regtest.regtest(); + +-- this should fail + +SET ROLE regress_mdb_superuser_user3; +GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user3; +ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user3; + +SET ROLE regress_mdb_superuser_user1; +GRANT ALL ON TABLE regtest.regtest TO regress_mdb_superuser_user1; +ALTER TABLE regtest.regtest OWNER TO regress_mdb_superuser_user1; + +\c regression +DROP DATABASE regress_check_owner; + +-- end tests + +RESET SESSION AUTHORIZATION; +-- +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user2; +REVOKE CREATE ON DATABASE regression FROM regress_mdb_superuser_user3; + +DROP VIEW regress_mdb_superuser_view; +DROP FUNCTION regress_mdb_superuser_add; +DROP TABLE regress_mdb_superuser_schema.regress_mdb_superuser_table; +DROP TABLE regress_mdb_superuser_table; +DROP SCHEMA regress_mdb_superuser_schema; +DROP ROLE regress_mdb_superuser_user1; +DROP ROLE regress_mdb_superuser_user2; +DROP ROLE regress_mdb_superuser_user3; +DROP TABLE regress_superuser_table; +DROP TABLE regress_pgrsf_table; +DROP TABLE regress_pgwsf_table; +DROP TABLE regress_pgxsp_table; +DROP TABLE regress_pgrad_table; +DROP TABLE regress_pgrwd_table; diff --git a/src/test/regress/sql/test_setup.sql b/src/test/regress/sql/test_setup.sql new file mode 100644 index 00000000000..7ec5ccc7471 --- /dev/null +++ b/src/test/regress/sql/test_setup.sql @@ -0,0 +1,6 @@ +-- +-- TEST_SETUP --- prepare environment expected by regression test scripts +-- + +CREATE ROLE mdb_admin; +CREATE ROLE mdb_superuser; From 789969ad28da5a15b30d2ee2406f4ff5011f69ce Mon Sep 17 00:00:00 2001 From: usernamedt Date: Mon, 13 Feb 2023 15:00:31 +0800 Subject: [PATCH 06/22] Movable DataBase Locales for Cloudberry We inherited this issue from PostgreSQL. PostgreSQL uses glibc to sort strings. In version glibc=2.28, collations broke down badly (in general, there are no guarantees when updating glibc). Changing collations breaks indexes. Similarly, a cluster with different collations also behaves unpredictably. What and when something has changed in glibc can be found on https://github.com/ardentperf/glibc-unicode-sorting Also there is special postgresql-wiki https://wiki.postgresql.org/wiki/Locale_data_changes And you tube video https://www.youtube.com/watch?v=0E6O-V8Jato In short, the issue can be seen through the use of bash: ( echo "1-1"; echo "11" ) | LC_COLLATE=en_US.UTF-8 sort gives the different results in ubunru 18.04 and 22.04. There is no way to solve the problem other than by not changing the symbol order. We freeze symbol order and use it instead of glibc. Here the solution https://github.com/postgredients/mdb-locales. In this PR I have added PostgreSQL patch that replaces all glibc locale-related calls with a calls to an external libary. It activates using new configure parameter --with-mdblocales, which is off by default. Using custom locales needs libmdblocales1 package and mdb-locales package with symbol table. Build needs libmdblocales-dev package with headers. --- .../scripts/configure-cloudberry.sh | 12 + src/test/regress/output/misc.source | 2 +- src/test/regress/sql/misc.sql | 271 ++++++++++++++++++ 3 files changed, 284 insertions(+), 1 deletion(-) create mode 100644 src/test/regress/sql/misc.sql diff --git a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh index 58d64922ac7..04c1efebbf2 100755 --- a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh +++ b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh @@ -62,6 +62,12 @@ # --enable-cassert # --enable-debug-extensions # +# ENABLE_MDBLOCALES - Enable custom locales (true/false, defaults to +# false) +# +# When true, add option: +# --with-mdblocales +# # Prerequisites: # - System dependencies must be installed: # * xerces-c development files @@ -132,6 +138,11 @@ if [ "${ENABLE_DEBUG:-false}" = "true" ]; then --enable-debug-extensions" fi +CONFIGURE_MDBLOCALES_OPTS="--without-mdblocales" +if [ "${ENABLE_MDBLOCALES:-false}" = "true" ]; then + CONFIGURE_MDBLOCALES_OPTS="--with-mdblocales" +fi + # Configure build log_section "Configure" execute_cmd ./configure --prefix=${BUILD_DESTINATION} \ @@ -158,6 +169,7 @@ execute_cmd ./configure --prefix=${BUILD_DESTINATION} \ --with-ssl=openssl \ --with-openssl \ --with-uuid=e2fs \ + ${CONFIGURE_MDBLOCALES_OPTS} \ --with-includes=/usr/local/xerces-c/include \ --with-libraries=${BUILD_DESTINATION}/lib || exit 4 log_section_end "Configure" diff --git a/src/test/regress/output/misc.source b/src/test/regress/output/misc.source index a0c63418446..f2f7c0dee32 100644 --- a/src/test/regress/output/misc.source +++ b/src/test/regress/output/misc.source @@ -613,6 +613,6 @@ CONTEXT: SQL function "equipment" during startup SELECT mdb_locale_enabled(); mdb_locale_enabled -------------------- - t + f (1 row) diff --git a/src/test/regress/sql/misc.sql b/src/test/regress/sql/misc.sql new file mode 100644 index 00000000000..2abe2c82eb8 --- /dev/null +++ b/src/test/regress/sql/misc.sql @@ -0,0 +1,271 @@ +-- +-- MISC +-- + +-- +-- BTREE +-- +--UPDATE onek +-- SET unique1 = onek.unique1 + 1; + +--UPDATE onek +-- SET unique1 = onek.unique1 - 1; + +-- +-- BTREE partial +-- +-- UPDATE onek2 +-- SET unique1 = onek2.unique1 + 1; + +--UPDATE onek2 +-- SET unique1 = onek2.unique1 - 1; + +-- +-- BTREE shutting out non-functional updates +-- +-- the following two tests seem to take a long time on some +-- systems. This non-func update stuff needs to be examined +-- more closely. - jolly (2/22/96) +-- +/* GPDB TODO: This test is disabled for now, because when running with ORCA, + you get an error: + ERROR: multiple updates to a row by the same query is not allowed +UPDATE tmp + SET stringu1 = reverse_name(onek.stringu1) + FROM onek + WHERE onek.stringu1 = 'JBAAAA' and + onek.stringu1 = tmp.stringu1; + +UPDATE tmp + SET stringu1 = reverse_name(onek2.stringu1) + FROM onek2 + WHERE onek2.stringu1 = 'JCAAAA' and + onek2.stringu1 = tmp.stringu1; +*/ + +DROP TABLE tmp; + +--UPDATE person* +-- SET age = age + 1; + +--UPDATE person* +-- SET age = age + 3 +-- WHERE name = 'linda'; + +-- +-- copy +-- +COPY onek TO '@abs_builddir@/results/onek.data'; + +DELETE FROM onek; + +COPY onek FROM '@abs_builddir@/results/onek.data'; + +SELECT unique1 FROM onek WHERE unique1 < 2 ORDER BY unique1; + +DELETE FROM onek2; + +COPY onek2 FROM '@abs_builddir@/results/onek.data'; + +SELECT unique1 FROM onek2 WHERE unique1 < 2 ORDER BY unique1; + +COPY BINARY stud_emp TO '@abs_builddir@/results/stud_emp.data'; + +DELETE FROM stud_emp; + +COPY BINARY stud_emp FROM '@abs_builddir@/results/stud_emp.data'; + +SELECT * FROM stud_emp; + +-- COPY aggtest FROM stdin; +-- 56 7.8 +-- 100 99.097 +-- 0 0.09561 +-- 42 324.78 +-- . +-- COPY aggtest TO stdout; + + +-- +-- inheritance stress test +-- +SELECT * FROM a_star*; + +SELECT * + FROM b_star* x + WHERE x.b = text 'bumble' or x.a < 3; + +SELECT class, a + FROM c_star* x + WHERE x.c ~ text 'hi'; + +SELECT class, b, c + FROM d_star* x + WHERE x.a < 100; + +SELECT class, c FROM e_star* x WHERE x.c NOTNULL; + +SELECT * FROM f_star* x WHERE x.c ISNULL; + +-- grouping and aggregation on inherited sets have been busted in the past... + +SELECT sum(a) FROM a_star*; + +SELECT class, sum(a) FROM a_star* GROUP BY class ORDER BY class; + + +ALTER TABLE f_star RENAME COLUMN f TO ff; + +ALTER TABLE e_star* RENAME COLUMN e TO ee; + +ALTER TABLE d_star* RENAME COLUMN d TO dd; + +ALTER TABLE c_star* RENAME COLUMN c TO cc; + +ALTER TABLE b_star* RENAME COLUMN b TO bb; + +ALTER TABLE a_star* RENAME COLUMN a TO aa; + +SELECT class, aa + FROM a_star* x + WHERE aa ISNULL; + +-- As of Postgres 7.1, ALTER implicitly recurses, +-- so this should be same as ALTER a_star* + +ALTER TABLE a_star RENAME COLUMN aa TO foo; + +SELECT class, foo + FROM a_star* x + WHERE x.foo >= 2; + +ALTER TABLE a_star RENAME COLUMN foo TO aa; + +SELECT * + from a_star* + WHERE aa < 1000; + +ALTER TABLE f_star ADD COLUMN f int4; + +UPDATE f_star SET f = 10; + +ALTER TABLE e_star* ADD COLUMN e int4; + +--UPDATE e_star* SET e = 42; + +SELECT * FROM e_star*; + +ALTER TABLE a_star* ADD COLUMN a text; + +-- That ALTER TABLE should have added TOAST tables. +SELECT relname, reltoastrelid <> 0 AS has_toast_table + FROM pg_class + WHERE oid::regclass IN ('a_star', 'c_star') + ORDER BY 1; + +--UPDATE b_star* +-- SET a = text 'gazpacho' +-- WHERE aa > 4; + +SELECT class, aa, a FROM a_star*; + + +-- +-- versions +-- + +-- +-- postquel functions +-- +-- +-- mike does post_hacking, +-- joe and sally play basketball, and +-- everyone else does nothing. +-- +SELECT p.name, name(p.hobbies) FROM ONLY person p; + +-- +-- as above, but jeff also does post_hacking. +-- +SELECT p.name, name(p.hobbies) FROM person* p; + +-- +-- the next two queries demonstrate how functions generate bogus duplicates. +-- this is a "feature" .. +-- +SELECT DISTINCT hobbies_r.name, name(hobbies_r.equipment) FROM hobbies_r + ORDER BY 1,2; + +SELECT hobbies_r.name, (hobbies_r.equipment).name FROM hobbies_r; + +-- +-- mike needs advil and peet's coffee, +-- joe and sally need hightops, and +-- everyone else is fine. +-- +SELECT p.name, name(p.hobbies), name(equipment(p.hobbies)) FROM ONLY person p; + +-- +-- as above, but jeff needs advil and peet's coffee as well. +-- +SELECT p.name, name(p.hobbies), name(equipment(p.hobbies)) FROM person* p; + +-- +-- just like the last two, but make sure that the target list fixup and +-- unflattening is being done correctly. +-- +SELECT name(equipment(p.hobbies)), p.name, name(p.hobbies) FROM ONLY person p; + +SELECT (p.hobbies).equipment.name, p.name, name(p.hobbies) FROM person* p; + +SELECT (p.hobbies).equipment.name, name(p.hobbies), p.name FROM ONLY person p; + +SELECT name(equipment(p.hobbies)), name(p.hobbies), p.name FROM person* p; + +SELECT name(equipment(hobby_construct(text 'skywalking', text 'mer'))); + +SELECT name(equipment(hobby_construct_named(text 'skywalking', text 'mer'))); + +SELECT name(equipment_named(hobby_construct_named(text 'skywalking', text 'mer'))); + +SELECT name(equipment_named_ambiguous_1a(hobby_construct_named(text 'skywalking', text 'mer'))); + +SELECT name(equipment_named_ambiguous_1b(hobby_construct_named(text 'skywalking', text 'mer'))); + +SELECT name(equipment_named_ambiguous_1c(hobby_construct_named(text 'skywalking', text 'mer'))); + +SELECT name(equipment_named_ambiguous_2a(text 'skywalking')); + +SELECT name(equipment_named_ambiguous_2b(text 'skywalking')); + +SELECT hobbies_by_name('basketball'); + +SELECT name, overpaid(emp.*) FROM emp; + +-- +-- Try a few cases with SQL-spec row constructor expressions +-- +SELECT * FROM equipment(ROW('skywalking', 'mer')); + +SELECT name(equipment(ROW('skywalking', 'mer'))); + +SELECT *, name(equipment(h.*)) FROM hobbies_r h; + +SELECT *, (equipment(CAST((h.*) AS hobbies_r))).name FROM hobbies_r h; + +-- +-- functional joins +-- + +-- +-- instance rules +-- + +-- +-- rewrite rules +-- + + +--- mdb-related + +SELECT mdb_locale_enabled(); From a971dd58b7ccfcf8e8302762940e14afc8002a79 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Thu, 15 Jan 2026 11:22:24 +0800 Subject: [PATCH 07/22] Add branch protection for REL_2_STABLE in .asf.yaml Add branch protection rules for the REL_2_STABLE release branch to ensure all modifications must go through pull requests. Configuration added: * Require at least 2 approving reviews before merging * Require conversation threads to be resolved before merging This protects the release branch from direct pushes and enforces code review workflow for all changes. --- .asf.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.asf.yaml b/.asf.yaml index eb828d3bb41..01188659355 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -122,6 +122,18 @@ github: # Require conversation threads to be resolved required_conversation_resolution: true + # Branch protection for REL_2_STABLE release branch + REL_2_STABLE: + # Pull request review requirements + required_pull_request_reviews: + # Require new reviews when new commits are pushed + dismiss_stale_reviews: false + # Require at least 2 approving reviews + required_approving_review_count: 2 + + # Require conversation threads to be resolved + required_conversation_resolution: true + # Branch cleanup settings # Don't automatically delete branches after merging del_branch_on_merge: true From 57517eaa102eb761daecf732ed0d4041353c7291 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Wed, 14 Jan 2026 18:02:03 +0800 Subject: [PATCH 08/22] CI: Add REL_2_STABLE support to GitHub workflows * Extend workflow triggers to include `REL_2_STABLE` branch * Modified 4 workflow files: apache-rat-audit.yml build-cloudberry.yml build-dbg-cloudberry.yml build-deb-cloudberry.yml * Aligns test/build workflows between main and REL_2_STABLE Please note maintainers must sync all the necessary commits from main to REL_2_STABLE to keep workflows running successfully. This commit just add the REL_2_STABLE as the target branch. --- .github/workflows/apache-rat-audit.yml | 4 ++-- .github/workflows/build-cloudberry.yml | 4 ++-- .github/workflows/build-dbg-cloudberry.yml | 4 ++-- .github/workflows/build-deb-cloudberry.yml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/apache-rat-audit.yml b/.github/workflows/apache-rat-audit.yml index ced9402d17e..4826fc89228 100644 --- a/.github/workflows/apache-rat-audit.yml +++ b/.github/workflows/apache-rat-audit.yml @@ -32,9 +32,9 @@ name: Apache Rat License Check on: push: - branches: [main] + branches: [main, REL_2_STABLE] pull_request: - branches: [main] + branches: [main, REL_2_STABLE] types: [opened, synchronize, reopened, edited] workflow_dispatch: diff --git a/.github/workflows/build-cloudberry.yml b/.github/workflows/build-cloudberry.yml index dcad89f30cb..9d44d06bbdc 100644 --- a/.github/workflows/build-cloudberry.yml +++ b/.github/workflows/build-cloudberry.yml @@ -102,9 +102,9 @@ name: Apache Cloudberry Build on: push: - branches: [main] + branches: [main, REL_2_STABLE] pull_request: - branches: [main] + branches: [main, REL_2_STABLE] types: [opened, synchronize, reopened, edited] workflow_dispatch: inputs: diff --git a/.github/workflows/build-dbg-cloudberry.yml b/.github/workflows/build-dbg-cloudberry.yml index 4bc23a5d677..967fc259f0b 100644 --- a/.github/workflows/build-dbg-cloudberry.yml +++ b/.github/workflows/build-dbg-cloudberry.yml @@ -102,9 +102,9 @@ name: Apache Cloudberry Build Debug on: push: - branches: [main] + branches: [main, REL_2_STABLE] pull_request: - branches: [main] + branches: [main, REL_2_STABLE] types: [opened, synchronize, reopened, edited] workflow_dispatch: inputs: diff --git a/.github/workflows/build-deb-cloudberry.yml b/.github/workflows/build-deb-cloudberry.yml index 68dc9daeba1..be28fff9e77 100644 --- a/.github/workflows/build-deb-cloudberry.yml +++ b/.github/workflows/build-deb-cloudberry.yml @@ -74,9 +74,9 @@ name: Apache Cloudberry Debian Build on: push: - branches: [main] + branches: [main, REL_2_STABLE] pull_request: - branches: [main] + branches: [main, REL_2_STABLE] types: [opened, synchronize, reopened, edited] workflow_dispatch: # Manual trigger inputs: From 770364780f48800c23c79dae3825295b3e7eed0b Mon Sep 17 00:00:00 2001 From: "Jianghua.yjh" Date: Wed, 21 Jan 2026 23:27:38 +0800 Subject: [PATCH 09/22] ORCA: Fix memory leak in CWindowOids by adding destructor (#1533) * ORCA: Fix memory leak in CWindowOids by adding destructor CWindowOids class was leaking three CMDIdGPDB objects (m_MDIdRowNumber, m_MDIdRank, m_MDDenseRank) that were allocated in the constructor but never released. Fixes ORCA unit test failures: - gporca_test_CXformTest - gporca_test_CConstExprEvaluatorDefaultTest --- .../gporca/libgpopt/include/gpopt/base/CWindowOids.h | 2 ++ src/backend/gporca/libgpopt/src/base/CWindowOids.cpp | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/src/backend/gporca/libgpopt/include/gpopt/base/CWindowOids.h b/src/backend/gporca/libgpopt/include/gpopt/base/CWindowOids.h index ba120eac015..6d20b4948c2 100644 --- a/src/backend/gporca/libgpopt/include/gpopt/base/CWindowOids.h +++ b/src/backend/gporca/libgpopt/include/gpopt/base/CWindowOids.h @@ -60,6 +60,8 @@ class CWindowOids : public CRefCount CWindowOids(CMemoryPool *mp, OID row_number_oid, OID rank_oid, OID dense_rank_oid); + ~CWindowOids() override; + // accessor of oid value of "row_number" function OID OidRowNumber() const; IMDId *MDIdRowNumber() const; diff --git a/src/backend/gporca/libgpopt/src/base/CWindowOids.cpp b/src/backend/gporca/libgpopt/src/base/CWindowOids.cpp index da744a018d9..e2f326d3aaf 100644 --- a/src/backend/gporca/libgpopt/src/base/CWindowOids.cpp +++ b/src/backend/gporca/libgpopt/src/base/CWindowOids.cpp @@ -18,6 +18,13 @@ CWindowOids::CWindowOids(CMemoryPool *mp, OID row_number_oid, OID rank_oid, m_MDDenseRank = GPOS_NEW(mp) CMDIdGPDB(IMDId::EmdidGeneral, m_oidDenseRank); } +CWindowOids::~CWindowOids() +{ + CRefCount::SafeRelease(m_MDIdRowNumber); + CRefCount::SafeRelease(m_MDIdRank); + CRefCount::SafeRelease(m_MDDenseRank); +} + OID CWindowOids::OidRowNumber() const { From d5b806c346d66ad8684ee8e89eb8459d9c7ecfb6 Mon Sep 17 00:00:00 2001 From: reshke Date: Thu, 22 Jan 2026 10:53:55 +0500 Subject: [PATCH 10/22] Do not initialize programm query string when unneeded. (#1536) Also, while on it, beautify code to conform PG-style coding. Per coverity report 544476 --- src/backend/commands/copy.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index efbf6a0ba39..4ccd3798067 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -1420,20 +1420,17 @@ close_program_pipes(ProgramPipes *program_pipes, bool ifThrow) { int ret = 0; StringInfoData sinfo; - initStringInfo(&sinfo); /* just return if pipes not created, like when relation does not exist */ if (!program_pipes) - { return; - } + + initStringInfo(&sinfo); ret = pclose_with_stderr(program_pipes->pid, program_pipes->pipes, &sinfo); if (ret == 0 || !ifThrow) - { return; - } if (ret == -1) { From be2a74f52edd225a2fd6469d1248a5b2bf6a58b5 Mon Sep 17 00:00:00 2001 From: Hao Wu Date: Wed, 9 Oct 2024 03:20:41 +0000 Subject: [PATCH 11/22] Add answer file for high version perl Perl with higher version(e.g. 5.38.0) will produce different error message, compared with v5.34.0 We should handle these different versions. --- .../src/api/python3/setup-debug.py | 4 +- contrib/pax_storage/src/api/python3/setup.py | 4 +- .../src/cpp/catalog/manifest_api.h | 4 +- .../test/regress/expected/create_index.out | 1 - .../expected/create_index_optimizer.out | 1 - .../src/test/regress/sql/create_index.sql | 1 - gpcontrib/gpmapreduce/output/mapred_1.source | 798 ++++++++++++++++++ src/backend/cdb/cdbpath.c | 1 - src/test/regress/expected/create_index.out | 1 - .../expected/create_index_optimizer.out | 1 - src/test/regress/sql/create_index.sql | 1 - 11 files changed, 804 insertions(+), 13 deletions(-) create mode 100644 gpcontrib/gpmapreduce/output/mapred_1.source diff --git a/contrib/pax_storage/src/api/python3/setup-debug.py b/contrib/pax_storage/src/api/python3/setup-debug.py index 13eadaa357d..da02e2ab237 100644 --- a/contrib/pax_storage/src/api/python3/setup-debug.py +++ b/contrib/pax_storage/src/api/python3/setup-debug.py @@ -51,7 +51,7 @@ def abs_path(file_name): version = '1.0', description = 'PAXPY is the PYTHON3 API of PAX', author = 'jiaqizho', - author_email = 'jiaqizho@hashdata.cn', + author_email = 'jiaqizho@apache.org', url = '-', ext_modules = [paxpy_module] -) \ No newline at end of file +) diff --git a/contrib/pax_storage/src/api/python3/setup.py b/contrib/pax_storage/src/api/python3/setup.py index a60cc779fd9..b739cacc465 100644 --- a/contrib/pax_storage/src/api/python3/setup.py +++ b/contrib/pax_storage/src/api/python3/setup.py @@ -48,7 +48,7 @@ def abs_path(file_name): version = '1.0', description = 'PAXPY is the PYTHON3 API of PAX', author = 'jiaqizho', - author_email = 'jiaqizho@hashdata.cn', + author_email = 'jiaqizho@apache.org', url = '-', ext_modules = [paxpy_module] -) \ No newline at end of file +) diff --git a/contrib/pax_storage/src/cpp/catalog/manifest_api.h b/contrib/pax_storage/src/cpp/catalog/manifest_api.h index f9e8f7c9eb1..3978caf9097 100644 --- a/contrib/pax_storage/src/cpp/catalog/manifest_api.h +++ b/contrib/pax_storage/src/cpp/catalog/manifest_api.h @@ -28,7 +28,7 @@ #pragma once /* - * abstract interface of hashdata manifest + * abstract interface of manifest */ #ifdef __cplusplus @@ -42,7 +42,7 @@ extern "C" { * memory management. */ -/* hashdata manifest support following data types for fields */ +/* manifest support following data types for fields */ typedef enum MetaFieldType { Meta_Field_Type_Int = 1, diff --git a/contrib/pax_storage/src/test/regress/expected/create_index.out b/contrib/pax_storage/src/test/regress/expected/create_index.out index ac4958d56dc..963ce9a6b39 100644 --- a/contrib/pax_storage/src/test/regress/expected/create_index.out +++ b/contrib/pax_storage/src/test/regress/expected/create_index.out @@ -2690,7 +2690,6 @@ COMMIT; -- we keep the `CONCURRENTLY` to make the following commands fail, -- so these commands will not cause deadlock with test create_view, -- like `drop schema xxx cascade;`. --- See more details at https://code.hashdata.xyz/cloudberry/cbdb/-/issues/54 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index diff --git a/contrib/pax_storage/src/test/regress/expected/create_index_optimizer.out b/contrib/pax_storage/src/test/regress/expected/create_index_optimizer.out index 8d0a41352a8..81728a31260 100644 --- a/contrib/pax_storage/src/test/regress/expected/create_index_optimizer.out +++ b/contrib/pax_storage/src/test/regress/expected/create_index_optimizer.out @@ -2717,7 +2717,6 @@ COMMIT; -- we keep the `CONCURRENTLY` to make the following commands fail, -- so these commands will not cause deadlock with test create_view, -- like `drop schema xxx cascade;`. --- See more details at https://code.hashdata.xyz/cloudberry/cbdb/-/issues/54 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index diff --git a/contrib/pax_storage/src/test/regress/sql/create_index.sql b/contrib/pax_storage/src/test/regress/sql/create_index.sql index a0e3fc5ccdc..2cb3cc77fd0 100644 --- a/contrib/pax_storage/src/test/regress/sql/create_index.sql +++ b/contrib/pax_storage/src/test/regress/sql/create_index.sql @@ -1111,7 +1111,6 @@ COMMIT; -- we keep the `CONCURRENTLY` to make the following commands fail, -- so these commands will not cause deadlock with test create_view, -- like `drop schema xxx cascade;`. --- See more details at https://code.hashdata.xyz/cloudberry/cbdb/-/issues/54 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -- These are the toast table and index of pg_authid. diff --git a/gpcontrib/gpmapreduce/output/mapred_1.source b/gpcontrib/gpmapreduce/output/mapred_1.source new file mode 100644 index 00000000000..05c29807feb --- /dev/null +++ b/gpcontrib/gpmapreduce/output/mapred_1.source @@ -0,0 +1,798 @@ +-- +-- map/reduce tests +-- +-- Some notes: the PATH variable doesn't necessarily work correctly in +-- the regression context, so use the "gpwhich" token to get the full +-- path from your environment (see gpstringsubs for more details). And +-- to make this work correctly with gpsourcify, you need to add your +-- gpwhich token to gptokencheck. +-- start_matchsubs +-- m|mapreduce_\d+_run| +-- s|mapreduce_\d+_run|mapreduce_PID_run| +-- end_matchsubs +-- +-- This test makes use of plperlu +-- +-- start_ignore +create language plperlu; +create language plpython3u; +-- Take a look at the installed languages, plperl and plpython should be in the list. +-- + pg_pltemplate contains the list of languages that can be simply installed +-- + pl_language contains the list of languages that are actually installed +select * from pg_pltemplate; + tmplname | tmpltrusted | tmplhandler | tmplvalidator | tmpllibrary | tmplacl +-----------+-------------+-----------------------+-------------------+------------------+--------- + plpgsql | t | plpgsql_call_handler | plpgsql_validator | $libdir/plpgsql | + pltcl | t | pltcl_call_handler | | $libdir/pltcl | + pltclu | f | pltclu_call_handler | | $libdir/pltcl | + plperl | t | plperl_call_handler | plperl_validator | $libdir/plperl | + plperlu | f | plperl_call_handler | plperl_validator | $libdir/plperl | + plpython3u | f | plpython_call_handler | | $libdir/plpython | + plr | f | plr_call_handler | | $libdir/plr | +(7 rows) + +select lanname, lanispl, lanpltrusted from pg_language; + lanname | lanispl | lanpltrusted +-----------+---------+-------------- + internal | f | f + c | f | f + sql | f | t + plpgsql | t | t + plperlu | t | f + plpython3u | t | f +(6 rows) + +-- Check environment variables that should have been set by greenplum_path.sh +-- +-- 1) We need to check these on all segments and on the master. +-- 2) We do this via external table rather than perl/python in case it is part +-- of the cause of a mis-installed plperl/plpython. +-- 3) It is normal for the master to have a slightly different enviornment from +-- the segments (but perhaps not desirable?) +-- +CREATE EXTERNAL WEB TABLE env_segment(var text, value text) +EXECUTE 'env | grep "^[^=]*=[^=]*$"' format 'text' (delimiter '='); +CREATE EXTERNAL WEB TABLE env_master(var text, value text) +EXECUTE 'env | grep "^[^=]*=[^=]*$"' ON COORDINATOR format 'text' (delimiter '='); +CREATE VIEW env AS + SELECT gp_execution_segment(), * FROM env_segment + UNION ALL + SELECT gp_execution_segment(), * FROM env_master; +SELECT * FROM env WHERE var in ( + 'GPHOME', + 'DYLD_LIBRARY_PATH', + 'LD_LIBRARY_PATH', + 'PATH' +) ORDER BY var, gp_execution_segment; + gp_execution_segment | var | value +----------------------+-------------------+---------------------------------------------------------------------------------------------------------------------------------------- + -1 | DYLD_LIBRARY_PATH | @gphome@/lib:@gphome@/ext/python/lib:$DYLD_LIBRARY_PATH + 0 | DYLD_LIBRARY_PATH | @gphome@/lib:@gphome@/ext/python/lib: + 1 | DYLD_LIBRARY_PATH | @gphome@/lib:@gphome@/ext/python/lib: + -1 | GPHOME | @gphome@ + 0 | GPHOME | @gphome@ + 1 | GPHOME | @gphome@ + -1 | LD_LIBRARY_PATH | @gphome@/lib:@gphome@/ext/python/lib: + -1 | PATH | @gphome@/bin:/usr/gnu/bin:/usr/local/bin:/bin:/usr/bin:/usr/sbin:/sbin:/usr/texbin:/usr/X11/bin:/sw/bin:/opt/local/bin:/opt/local/sbin + 0 | PATH | @gphome@/ext/python/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin + 1 | PATH | @gphome@/ext/python/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin +(10 rows) + +-- end_ignore +-- start_ignore +-- +-- Some checks to verify what versions of perl/python we have. +-- If everything has been configured correctly this should be constant +-- across all our installations. +-- +-- All of these checks should return a single row because it should produce +-- a constant across all segments and the master. +-- +-- The current expected version is 2.6.2 +-- +CREATE OR REPLACE FUNCTION python_version() returns text as $$ +import sys +return sys.version_info +$$ language plpython3u NO SQL; +SELECT python_version() FROM env GROUP BY python_version; + python_version +----------------------- + (2, 6, 2, 'final', 0) +(1 row) + +-- +-- Same check for perl version +-- +-- Expected version is perl 5.008xxx +-- +CREATE OR REPLACE FUNCTION perl_version() returns text as $$ +return "Perl $]" +$$ language plperlu NO SQL; +SELECT perl_version() FROM env GROUP BY perl_version; + perl_version +--------------- + Perl 5.008005 +(1 row) + +-- +-- The following two checks need to be put into big ignore blocks +-- because paths can be of differing lengths +-- +CREATE OR REPLACE FUNCTION python_path() returns text as $$ +import sys +return sys.path[0] +$$ language plpython3u NO SQL; +SELECT python_path() FROM env GROUP BY python_path; + python_path +--------------------------------------------- + @gphome@/lib/python +(1 row) + +CREATE OR REPLACE FUNCTION perl_path() returns text as $$ +return join(':', @INC) +$$ language plperlu NO SQL; +SELECT perl_path() FROM env GROUP BY perl_path; + perl_path +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + /System/Library/Perl/5.8.8/darwin-thread-multi-2level:/System/Library/Perl/5.8.8:/Library/Perl/5.8.8/darwin-thread-multi-2level:/Library/Perl/5.8.8:/Library/Perl:/Network/Library/Perl/5.8.8/darwin-thread-multi-2level:/Network/Library/Perl/5.8.8:/Network/Library/Perl:/System/Library/Perl/Extras/5.8.8/darwin-thread-multi-2level:/System/Library/Perl/Extras/5.8.8:/Library/Perl/5.8.6:/Library/Perl/5.8.1:. +(1 row) + +-- end_ignore +-- +-- Create a harness to run shell commands and output stderr and stdout +-- +CREATE OR REPLACE FUNCTION execute(cmd text) returns text as $$ +import subprocess +p = subprocess.Popen(cmd, shell=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) +r = p.communicate() +header = "---------------------\n" +return header + r[0].decode() +$$ LANGUAGE plpython3u NO SQL; +-- +-- Create a harness to run mapreduce jobs on the correct host/port +-- +CREATE OR REPLACE FUNCTION mapreduce(file text) returns setof text as $$ +import subprocess + +rv = plpy.execute("select hostname, port, user as user, " + + " current_database() as db from gp_segment_configuration where content=-1") + +cmd = ['@gpwhich_gpmapreduce@', + '--host=%s' % rv[0]['hostname'], + '--port=%s' % rv[0]['port'], + '--file=%s' % file, + rv[0]['db'], rv[0]['user'] ] +p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) +r = p.communicate() +header = "---------------------" +err = '\nSTDERR> '.join(('STDERR> ' + r[1].decode()).split('\n')).replace('\t', ' ') +out = 'STDOUT>\n' + r[0].decode().replace('\t', ', ') +return [header, err, out] +$$ LANGUAGE plpython3u READS SQL DATA; +CREATE OR REPLACE FUNCTION mapreduce(file text, keys text) returns setof text as $$ +import subprocess + +rv = plpy.execute("select hostname, port, user as user, " + + " current_database() as db from gp_segment_configuration where content=-1") + +cmd = ['@gpwhich_gpmapreduce@', + '--host=%s' % rv[0]['hostname'], + '--port=%s' % rv[0]['port'], + '--file=%s' % file, + rv[0]['db'], rv[0]['user'] ] +for key in keys.split(';'): + cmd.append('--key=%s' % key) +p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) +r = p.communicate() +header = "---------------------" +err = '\nSTDERR> '.join(('STDERR> ' + r[1].decode()).split('\n')).replace('\t', ' ') +out = 'STDOUT>\n' + r[0].decode().replace('\t', ', ') +return [header, err, out] +$$ LANGUAGE plpython3u READS SQL DATA; +-- +-- CHECK 1) make sure plperlu is really installed: +-- +-- Note: if not, you might need to 'export PG_LANG=true' and reconfigure/rebuild +-- +SELECT lanname, lanispl, lanpltrusted FROM pg_language WHERE lanname = 'plperlu'; + lanname | lanispl | lanpltrusted +---------+---------+-------------- + plperlu | t | f +(1 row) + +-- +-- Since many of these tests will end up having variable length output depending +-- on local paths we want to disable the "----" lines from psql so that our diffs +-- look reasonable. +-- +-- The default is "aligned" if that ever changes then we need to change how we restore +-- it at the end of the test +\pset format +\pset format unaligned +-- +-- Check 2) Find gpmapreduce and print the help file +-- +-- start_ignore +select execute( '@gpwhich_gpmapreduce@ --help' ); +execute +--------------------- +@gpwhich_gpmapreduce@ - Cloudberry Map/Reduce Driver 1.00b2 + +Usage: + @gpwhich_gpmapreduce@ [options] -f file.yml [dbname [username]] + +General options: + -? | --help show this help, then exit + -V | --version show version information, then exit + -v | --verbose verbose output + -x | --explain do not run jobs, but produce explain plans + -X | --explain-analyze run jobs and produce explain-analyze plans + -k | --key = sets a yaml variable + +Connection options: + -h | --host database server host or socket directory + -p | --port database server port + -U | --username database user name + -W | --password prompt for password + +Debug options: + -D | --debug enable some debugging output + -P | --print print-only mode, do not run jobs + +(1 row) +-- end_ignore +-- +-- TEST 1) complain about missing file +-- +SELECT mapreduce('nosuchfile') ORDER BY 1; +mapreduce +--------------------- +STDERR> Error: Could not open file 'nosuchfile' +STDERR> +STDOUT> + +(3 rows) +-- +-- TEST 2) Tests reading and sorting an input file; +-- +SELECT mapreduce('@abs_srcdir@/yml/sort.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_73685_run_1 +STDERR> +STDOUT> +value +------------------------- +alex, 30, (1.352,8.2) +belinda, 38, (8.9,1.7) +bertha, 88, (2.75,9.4) +carina, 58, (4.27,8.8) +carmen, 78, (3.8,8.2) +chris, 78, (9.78,2) +denise, 24, (3.78,87.90) +diane, 18, (5.912,5.3) +edna, 18, (1.53,3.5) +esther, 98, (5.36,7.6) +fanny, 08, (1.2,0.9) +gina, 18, (9.82,7.5) +jane, 58, (1.34,0.44) +jean, 28, (8.561,7.3) +jenifer, 38, (6.6,23.3) +joan, 18, (9.4,47.04) +joe, 20, (5.5,2.5) +juanita, 58, (4.57,35.8) +julie, 68, (3.6,7.2) +karen, 48, (8.73,0.0) +koko, 88, (1.7,5.5) +leah, 68, (0.6,3.37) +lita, 25, (1.3,8.7) +liza, 38, (9.76,6.90) +louise, 98, (5.0,8.7) +martie, 88, (8.358,.93) +mary, 08, (3.7,39.20) +melissa, 28, (3.089,087.23) +mike, 40, (3.1,6.2) +nan, 28, (6.35,0.43) +pamela, 48, (8.21,9.3) +pat, 18, (1.19,0.6) +paula, 68, (0.5,0.5) +rean, 48, (8.5,5.0) +sally, 34, (3.8,45.8) +sandra, 19, (9.345,09.6) +sandy, 38, (3.8,0.2) +sarah, 88, (8.4,2.3) +sharon, 78, (9.237,8.8) +sue, 50, (8.34,7.375) +sumi, 38, (1.15,0.6) +susan, 78, (6.579,3) +teresa, 38, (7.7,1.8) +trisha, 88, (1.29,2.2) +trudy, 88, (6.01,0.5) +velma, 68, (8.8,8.9) +vera, 78, (9.73,6.4) +wendy, 78, (2.62,03.3) +zena, 98, (0.35,0) +zola, 58, (2.56,4.3) +(50 rows) + + +(3 rows) +-- +-- TEST 3) Tests a basic map function and parameter passing +-- +SELECT mapreduce('@abs_srcdir@/yml/grep.yml', 'key=an') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_73691_run_1 +STDERR> +STDOUT> +key|value +---+---------------------- +an |diane, 18, (5.912,5.3) +an |fanny, 08, (1.2,0.9) +an |jane, 58, (1.34,0.44) +an |jean, 28, (8.561,7.3) +an |joan, 18, (9.4,47.04) +an |juanita, 58, (4.57,35.8) +an |nan, 28, (6.35,0.43) +an |rean, 48, (8.5,5.0) +an |sandra, 19, (9.345,09.6) +an |sandy, 38, (3.8,0.2) +an |susan, 78, (6.579,3) +(11 rows) + + +(3 rows) +-- +-- Test 4) Tests producing multiple columns +-- +SELECT mapreduce('@abs_srcdir@/yml/grep2.yml', 'key=an') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_73697_run_1 +STDERR> +STDOUT> +name |age|location +-------+---+------------ +diane | 18|(5.912,5.3) +fanny | 8|(1.2,0.9) +jane | 58|(1.34,0.44) +jean | 28|(8.561,7.3) +joan | 18|(9.4,47.04) +juanita| 58|(4.57,35.8) +nan | 28|(6.35,0.43) +rean | 48|(8.5,5.0) +sandra | 19|(9.345,09.6) +sandy | 38|(3.8,0.2) +susan | 78|(6.579,3) +(11 rows) + + +(3 rows) +-- +-- Test 5) Tests a basic reduce function and the builtin counterpart +-- +SELECT mapreduce('@abs_srcdir@/yml/agebracket.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_73703_run_1 +STDERR> +STDOUT> +key |value +---------------+----- +0 => age < 10 | 2 +10 => age < 20 | 6 +20 => age < 30 | 6 +30 => age < 40 | 8 +40 => age < 50 | 4 +50 => age < 60 | 5 +60 => age < 70 | 4 +70 => age < 80 | 6 +80 => age < 90 | 6 +90 => age < 100| 3 +(10 rows) + + +(3 rows) +SELECT mapreduce('@abs_srcdir@/yml/agebracket_builtin.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_73703_run_1 +STDERR> +STDOUT> +key |value +---------------+----- +0 => age < 10 | 2 +10 => age < 20 | 6 +20 => age < 30 | 6 +30 => age < 40 | 8 +40 => age < 50 | 4 +50 => age < 60 | 5 +60 => age < 70 | 4 +70 => age < 80 | 6 +80 => age < 90 | 6 +90 => age < 100| 3 +(10 rows) + + +(3 rows) +-- +-- Test 6) File Output tests +-- +SELECT execute('rm @abs_builddir@/results/fileout_*.out') ORDER BY 1; +execute +--------------------- + +(1 row) +SELECT mapreduce('@abs_srcdir@/yml/fileout.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_73712_run_1 +STDERR> mapreduce_73712_run_2 +STDERR> mapreduce_73712_run_3 +STDERR> +STDOUT> + +(3 rows) +SELECT execute('cat @abs_builddir@/results/fileout_none.out') ORDER BY 1; +execute +--------------------- +row 1:data 1 +row 2:data 2 +row 3:data 3 + +(1 row) +SELECT execute('cat @abs_builddir@/results/fileout_replace.out') ORDER BY 1; +execute +--------------------- +row 1|data 1 +row 2|data 2 +row 3|data 3 + +(1 row) +SELECT execute('cat @abs_builddir@/results/fileout_append.out') ORDER BY 1; +execute +--------------------- +row 1,data 1 +row 2,data 2 +row 3,data 3 + +(1 row) +SELECT mapreduce('@abs_srcdir@/yml/fileout.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_73721_run_1 +STDERR> mapreduce_73721_run_2 +STDERR> mapreduce_73721_run_3 +STDERR> Error: OUTPUT 'out_3': file '@abs_builddir@/results/fileout_none.out' already exists, at line 27 +STDERR> Error: Object creation Failure +STDERR> +STDOUT> + +(3 rows) +SELECT execute('cat @abs_builddir@/results/fileout_none.out') ORDER BY 1; +execute +--------------------- +row 1:data 1 +row 2:data 2 +row 3:data 3 + +(1 row) +SELECT execute('cat @abs_builddir@/results/fileout_replace.out') ORDER BY 1; +execute +--------------------- +row 1|data 1 +row 2|data 2 +row 3|data 3 + +(1 row) +SELECT execute('cat @abs_builddir@/results/fileout_append.out') ORDER BY 1; +execute +--------------------- +row 1,data 1 +row 2,data 2 +row 3,data 3 +row 1,data 1 +row 2,data 2 +row 3,data 3 + +(1 row) +-- +-- Test 7) Syntax error lineno reporting +-- +SELECT mapreduce('@abs_srcdir@/yml/perlerror.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> ERROR: syntax error at line 18, near "[]" +STDERR> Execution of aborted due to compilation errors. +STDERR> CONTEXT: compilation of PL/Perl function "mapreduce_1977_grep_map" +STDERR> Error: Object creation Failure +STDERR> ERROR: syntax error at line 28, near "[]" +STDERR> Execution of aborted due to compilation errors. +STDERR> CONTEXT: compilation of PL/Perl function "mapreduce_1992_grep_map" +STDERR> Error: Object creation Failure +STDERR> ERROR: syntax error at line 37, near "[]" +STDERR> Execution of aborted due to compilation errors. +STDERR> CONTEXT: compilation of PL/Perl function "mapreduce_2000_grep_map" +STDERR> Error: Object creation Failure +STDERR> ERROR: syntax error at line 45, near "[]" +STDERR> Execution of aborted due to compilation errors. +STDERR> CONTEXT: compilation of PL/Perl function "mapreduce_2011_grep_map" +STDERR> Error: Object creation Failure +STDERR> ERROR: syntax error at line 53, near "[]" +STDERR> Execution of aborted due to compilation errors. +STDERR> CONTEXT: compilation of PL/Perl function "mapreduce_2020_grep_map" +STDERR> Error: Object creation Failure +STDERR> ERROR: syntax error at line 67, near "}GABLECK" +STDERR> Execution of aborted due to compilation errors. +STDERR> CONTEXT: compilation of PL/Perl function "mapreduce_52431_mymap" +STDERR> Error: Object creation Failure +STDERR> WARNING: unset parameter - myMap(key => NULL) +STDERR> mapreduce_PID_run_1 +STDERR> ERROR: }GABLECK!{ at line 85. +STDERR> CONTEXT: PL/Perl function "mapreduce_52435_mymap" +STDERR> Error: Execution Failure +STDERR> +STDOUT> + +(3 rows) +SELECT mapreduce('@abs_srcdir@/yml/yamlerror.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> Error: OUTPUT 'output_table_empty': Invalid TABLE, at line 8 +STDERR> Error: OUTPUT 'output_table_empty': Invalid TABLE, at line 8 +STDERR> Error: OUTPUT 'output_table_string': Invalid TABLE, at line 11 +STDERR> Error: OUTPUT 'output_table_string': Invalid TABLE, at line 11 +STDERR> Error: OUTPUT 'output_table_number': Invalid TABLE, at line 14 +STDERR> Error: OUTPUT 'output_table_number': Invalid TABLE, at line 14 +STDERR> Error: OUTPUT 'output_table_list': Invalid TABLE, at line 17 +STDERR> Error: OUTPUT 'output_table_list': Invalid TABLE, at line 17 +STDERR> Error: OUTPUT 'output_table_mapping': Invalid TABLE, at line 20 +STDERR> Error: OUTPUT 'output_table_mapping': Invalid TABLE, at line 20 +STDERR> Error: OUTPUT 'output_table_mapping': Missing FILE or TABLE, at line 21 +STDERR> Error: INPUT 'input_columns_mapping': Invalid COLUMNS, at line 33 +STDERR> Error: Unrecognized VERSION, at line 62 +STDERR> Error: DOCUMENT: Missing VERSION, at line 98 +STDERR> Error: INPUT must contain a YAML MAPPING, at line 138 +STDERR> Error: INPUT: Missing NAME, at line 138 +STDERR> Error: INPUT: Missing FILE, GPFDIST, TABLE, QUERY, or EXEC, at line 138 +STDERR> Error: INPUT 'mystart': Duplicate QUERY for INPUT, at line 144 +STDERR> Error: parse failure +STDERR> +STDOUT> + +(3 rows) +-- Test 8) C functions using LIBRARY +SELECT mapreduce('@abs_srcdir@/yml/c.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_PID_run_1 +STDERR> +STDOUT> +key|value +---+----- + 0| 136 + 1| 401 + 3| 98 + 4| 1162 + 5| 257 + 6| 2073 + 7| 2138 + 8| 1082 + 9| 2241 + 10|11556 + 11| 8236 + 12| 473 + 13| 477 + 14| 0 + 15| 2287 + 16| 2772 + 17| 2359 + 18| 536 + 19|25936 + 20| 910 +(20 rows) + + +(3 rows) +-- Test 9) PageRank +SELECT mapreduce('@abs_srcdir@/yml/pagerank-init.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_PID_run_1 +STDERR> DONE +STDERR> +STDOUT> + +(3 rows) +SELECT count(*) FROM pagerank_source; +count +20 +(1 row) +SELECT mapreduce('@abs_srcdir@/yml/pagerank-iter.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_PID_run_1 +STDERR> DONE +STDERR> +STDOUT> + +(3 rows) +SELECT mapreduce('@abs_srcdir@/yml/pagerank-final.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_PID_run_1 +STDERR> DONE +STDERR> +STDOUT> + +(3 rows) +CREATE OR REPLACE FUNCTION parse_pg_rank_kv(s text) returns text as $$ +i = s.index(",") +rank = s[:i] +neighbors = list(map(int, s[i+1:].strip().strip("()").split(","))) +neighbors.sort() +ns = "(" + ",".join(map(str, neighbors)) + ")" +return ", ".join([rank, ns]) +$$ LANGUAGE plpython3u NO SQL; +-- The result of the following test case is related to the underlying hash algorithm +-- because the intermediate result of computing is saved as table in gpdb and hash-distributed. +-- If you change the underlying hash algorithm, please remember to verify the answer and change +-- here also. +SELECT parse_pg_rank_kv(ps.value), parse_pg_rank_kv(pn.value) FROM pagerank_source ps JOIN pagerank_next pn USING (key) WHERE ps.key = '17'; +parse_pg_rank_kv|parse_pg_rank_kv +0.85, (1,258,259,260,261,262,263,264,265,266)|0.265867988394584, (1,258,259,260,261,262,263,264,265,266) +(1 row) +-- Test setup +CREATE TABLE simple (m int,n int) distributed randomly; +INSERT INTO simple VALUES (1,10), (2,20), (2,21), (2,22), (3,30), (4,40), (5,50), (5,50), (10,100), (2,21); +CREATE FUNCTION tran (state int, arg2 int) returns int language C AS '@abs_srcdir@/regress.so', 'tran'; +CREATE FUNCTION tran (state int) returns int language C AS '@abs_srcdir@/regress.so', 'tran'; +CREATE FUNCTION final (state int) returns int language C AS '@abs_srcdir@/regress.so', 'final'; +CREATE FUNCTION cons (state int, value int, out output int) returns int language C AS '@abs_srcdir@/regress.so', 'cons'; +CREATE FUNCTION cons_wrongtype (state int, value text) returns int language C AS '@abs_srcdir@/regress.so', 'cons'; +CREATE OR REPLACE FUNCTION retcomposite(int) + RETURNS SETOF __retcomposite + AS '@abs_srcdir@/regress.so', 'retcomposite' + LANGUAGE C IMMUTABLE STRICT; +NOTICE: type "__retcomposite" is not yet defined +DETAIL: Creating a shell type definition. +CREATE FUNCTION gpmapred_scube_accum8(numeric,numeric) RETURNS numeric +AS 'SELECT $1 + $2 * $2 * $2' +LANGUAGE SQL +IMMUTABLE +RETURNS NULL ON NULL INPUT; +CREATE AGGREGATE gpmapred_scube(numeric) ( + SFUNC = gpmapred_scube_accum8, + STYPE = numeric, + INITCOND = 0 ); +-- Test 10) Aggregate Function +SELECT mapreduce('@abs_srcdir@/yml/aggFunction.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_PID_run_1 +STDERR> +STDOUT> + m| value +--+------- + 1| 1000 + 2| 37170 + 3| 27000 + 4| 64000 + 5| 250000 +10|1000000 +(6 rows) + + +(3 rows) +-- Test 11) Built-in Functions +SELECT mapreduce('@abs_srcdir@/yml/builtinfunction.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_PID_run_1 +STDERR> mapreduce_PID_run_2 +STDERR> mapreduce_PID_run_3 +STDERR> mapreduce_PID_run_4 +STDERR> mapreduce_PID_run_5 +STDERR> +STDOUT> + m|value +--+----- + 1| 10 + 2| 20 + 3| 30 + 4| 40 + 5| 50 +10| 100 +(6 rows) + + m|value +--+----- + 1| 10 + 2| 22 + 3| 30 + 4| 40 + 5| 50 +10| 100 +(6 rows) + + m| value +--+-------------------- + 1| 10.0000000000000000 + 2| 21.0000000000000000 + 3| 30.0000000000000000 + 4| 40.0000000000000000 + 5| 50.0000000000000000 +10|100.0000000000000000 +(6 rows) + + m|value +--+----- + 1| 1 + 2| 4 + 3| 1 + 4| 1 + 5| 2 +10| 1 +(6 rows) + + m|value +--+----- + 1| 10 + 2| 20 + 2| 21 + 2| 21 + 2| 22 + 3| 30 + 4| 40 + 5| 50 + 5| 50 +10| 100 +(10 rows) + + +(3 rows) +-- Test 12) Ambigious function +SELECT mapreduce('@abs_srcdir@/yml/ambiguousfunction.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> Error: MAP 'tran': Ambiguous function, supply a function prototype for disambiguation, at line 11 +STDERR> Error: Object creation Failure +STDERR> +STDOUT> + +(3 rows) +-- Test 13) Parameter override +SELECT mapreduce('@abs_srcdir@/yml/paramoverride.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_PID_run_1 +STDERR> +STDOUT> + m|retval +--+------ + 1| 20 + 2| 168 + 3| 60 + 4| 80 + 5| 200 +10| 200 +(6 rows) + + +(3 rows) +-- Test 14) Incorrect parameter type +SELECT mapreduce('@abs_srcdir@/yml/paramdifftype.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> ERROR: function cons_wrongtype(integer, integer) does not exist +STDERR> Error: Object creation Failure +STDERR> +STDOUT> + +(3 rows) +-- +-- Cleanup) Restore normal formatting options +-- +\pset format aligned diff --git a/src/backend/cdb/cdbpath.c b/src/backend/cdb/cdbpath.c index 89d15437661..9e3697a3b03 100644 --- a/src/backend/cdb/cdbpath.c +++ b/src/backend/cdb/cdbpath.c @@ -2964,7 +2964,6 @@ can_elide_explicit_motion(PlannerInfo *root, Index rti, Path *subpath, * There was once an idea reseting path's parallel_works to avoid * Motion if inner and outer's parallel_workers doesn't match. * But there are a lot of issues we don't have a clear answer. - * See https://code.hashdata.xyz/cloudberry/cbdb-postgres-merge/-/issues/43. * * We couldn't expect the parallel_workers of outer or inner path. * Partial path may generate locus(parallel_workers=0) if needed, ex: diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index f795078e761..0bc2aaa9f6e 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2683,7 +2683,6 @@ COMMIT; -- we keep the `CONCURRENTLY` to make the following commands fail, -- so these commands will not cause deadlock with test create_view, -- like `drop schema xxx cascade;`. --- See more details at https://code.hashdata.xyz/cloudberry/cbdb/-/issues/54 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index diff --git a/src/test/regress/expected/create_index_optimizer.out b/src/test/regress/expected/create_index_optimizer.out index f661e53f487..65f5f92b8bd 100644 --- a/src/test/regress/expected/create_index_optimizer.out +++ b/src/test/regress/expected/create_index_optimizer.out @@ -2723,7 +2723,6 @@ COMMIT; -- we keep the `CONCURRENTLY` to make the following commands fail, -- so these commands will not cause deadlock with test create_view, -- like `drop schema xxx cascade;`. --- See more details at https://code.hashdata.xyz/cloudberry/cbdb/-/issues/54 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index diff --git a/src/test/regress/sql/create_index.sql b/src/test/regress/sql/create_index.sql index 1b598d13b27..ac5526ea9b1 100644 --- a/src/test/regress/sql/create_index.sql +++ b/src/test/regress/sql/create_index.sql @@ -1109,7 +1109,6 @@ COMMIT; -- we keep the `CONCURRENTLY` to make the following commands fail, -- so these commands will not cause deadlock with test create_view, -- like `drop schema xxx cascade;`. --- See more details at https://code.hashdata.xyz/cloudberry/cbdb/-/issues/54 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -- These are the toast table and index of pg_authid. From b245e4d758a8a1d0c3fb0269b088d51a6ba5e728 Mon Sep 17 00:00:00 2001 From: Andrey Sokolov Date: Thu, 22 Jan 2026 17:11:56 +0300 Subject: [PATCH 12/22] Set join_collapse_limit default value to 13 (#1525) The default value of join_collapse_limit was 20. When this value is set and the query contains about 20 joins (see added test), Postgres query optimizer cannot build a plan during hours and consumes a lot of memory, because the planner checks a lot of possible ways to join the tables. When join_collapse_limit is 8, the query plan is built in reasonable time. --- src/backend/utils/misc/guc.c | 2 +- src/backend/utils/misc/postgresql.conf.sample | 2 +- src/test/regress/expected/bfv_joins.out | 42 +++++++++++++++++++ .../regress/expected/bfv_joins_optimizer.out | 42 +++++++++++++++++++ src/test/regress/sql/bfv_joins.sql | 39 +++++++++++++++++ 5 files changed, 125 insertions(+), 2 deletions(-) diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index fb09180ebe9..154d6e39737 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -2290,7 +2290,7 @@ static struct config_int ConfigureNamesInt[] = GUC_EXPLAIN }, &join_collapse_limit, - 20, 1, INT_MAX, + 13, 1, INT_MAX, NULL, NULL, NULL }, { diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 1f504832eac..4192dfb2748 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -434,7 +434,7 @@ max_prepared_transactions = 250 # can be 0 or more #cursor_tuple_fraction = 0.1 # range 0.0-1.0 #from_collapse_limit = 20 -#join_collapse_limit = 20 # 1 disables collapsing of explicit +#join_collapse_limit = 13 # 1 disables collapsing of explicit # JOIN clauses #force_parallel_mode = off #jit = on # allow JIT compilation diff --git a/src/test/regress/expected/bfv_joins.out b/src/test/regress/expected/bfv_joins.out index 3eab4b55fc1..593c0797b96 100644 --- a/src/test/regress/expected/bfv_joins.out +++ b/src/test/regress/expected/bfv_joins.out @@ -4189,6 +4189,48 @@ INSERT INTO ext_stats_tbl VALUES('tC', true); ANALYZE ext_stats_tbl; explain SELECT 1 FROM ext_stats_tbl t11 FULL JOIN ext_stats_tbl t12 ON t12.c2; ERROR: FULL JOIN is only supported with merge-joinable or hash-joinable join conditions +-- Check that Postgres Planner can build a plan with 20 joins in reasonable time +do $$ +begin + for i in 1..20 loop + execute 'create table tj' || i || '(id int)'; + end loop; +end +$$; +set optimizer to off; +select trunc(extract(epoch from now())) unix_time1 \gset +select * +from tj1 + join tj2 on tj1.id = tj2.id + join tj3 on tj2.id = tj3.id + join tj4 on tj3.id = tj4.id + join tj5 on tj4.id = tj5.id + join tj6 on tj5.id = tj6.id + join tj7 on tj6.id = tj7.id + join tj8 on tj7.id = tj8.id + join tj9 on tj8.id = tj9.id + join tj10 on tj9.id = tj10.id + join tj11 on tj10.id = tj11.id + join tj12 on tj11.id = tj12.id + join tj13 on tj12.id = tj13.id + join tj14 on tj13.id = tj14.id + join tj15 on tj14.id = tj15.id + join tj16 on tj15.id = tj16.id + join tj17 on tj16.id = tj17.id + join tj18 on tj17.id = tj18.id + join tj19 on tj18.id = tj19.id + join tj20 on tj19.id = tj20.id; + id | id | id | id | id | id | id | id | id | id | id | id | id | id | id | id | id | id | id | id +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+---- +(0 rows) + +select (trunc(extract(epoch from now())) - :unix_time1) < 100 is_ok; + is_ok +------- + t +(1 row) + +reset optimizer; -- Clean up. None of the objects we create are very interesting to keep around. reset search_path; set client_min_messages='warning'; diff --git a/src/test/regress/expected/bfv_joins_optimizer.out b/src/test/regress/expected/bfv_joins_optimizer.out index 6ae5ea6d30c..c5585098961 100644 --- a/src/test/regress/expected/bfv_joins_optimizer.out +++ b/src/test/regress/expected/bfv_joins_optimizer.out @@ -4206,6 +4206,48 @@ INSERT INTO ext_stats_tbl VALUES('tC', true); ANALYZE ext_stats_tbl; explain SELECT 1 FROM ext_stats_tbl t11 FULL JOIN ext_stats_tbl t12 ON t12.c2; ERROR: FULL JOIN is only supported with merge-joinable or hash-joinable join conditions +-- Check that Postgres Planner can build a plan with 20 joins in reasonable time +do $$ +begin + for i in 1..20 loop + execute 'create table tj' || i || '(id int)'; + end loop; +end +$$; +set optimizer to off; +select trunc(extract(epoch from now())) unix_time1 \gset +select * +from tj1 + join tj2 on tj1.id = tj2.id + join tj3 on tj2.id = tj3.id + join tj4 on tj3.id = tj4.id + join tj5 on tj4.id = tj5.id + join tj6 on tj5.id = tj6.id + join tj7 on tj6.id = tj7.id + join tj8 on tj7.id = tj8.id + join tj9 on tj8.id = tj9.id + join tj10 on tj9.id = tj10.id + join tj11 on tj10.id = tj11.id + join tj12 on tj11.id = tj12.id + join tj13 on tj12.id = tj13.id + join tj14 on tj13.id = tj14.id + join tj15 on tj14.id = tj15.id + join tj16 on tj15.id = tj16.id + join tj17 on tj16.id = tj17.id + join tj18 on tj17.id = tj18.id + join tj19 on tj18.id = tj19.id + join tj20 on tj19.id = tj20.id; + id | id | id | id | id | id | id | id | id | id | id | id | id | id | id | id | id | id | id | id +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+---- +(0 rows) + +select (trunc(extract(epoch from now())) - :unix_time1) < 100 is_ok; + is_ok +------- + t +(1 row) + +reset optimizer; -- Clean up. None of the objects we create are very interesting to keep around. reset search_path; set client_min_messages='warning'; diff --git a/src/test/regress/sql/bfv_joins.sql b/src/test/regress/sql/bfv_joins.sql index edc39f58a7d..59208087808 100644 --- a/src/test/regress/sql/bfv_joins.sql +++ b/src/test/regress/sql/bfv_joins.sql @@ -604,6 +604,45 @@ ANALYZE ext_stats_tbl; explain SELECT 1 FROM ext_stats_tbl t11 FULL JOIN ext_stats_tbl t12 ON t12.c2; +-- Check that Postgres Planner can build a plan with 20 joins in reasonable time +do $$ +begin + for i in 1..20 loop + execute 'create table tj' || i || '(id int)'; + end loop; +end +$$; + +set optimizer to off; + +select trunc(extract(epoch from now())) unix_time1 \gset + +select * +from tj1 + join tj2 on tj1.id = tj2.id + join tj3 on tj2.id = tj3.id + join tj4 on tj3.id = tj4.id + join tj5 on tj4.id = tj5.id + join tj6 on tj5.id = tj6.id + join tj7 on tj6.id = tj7.id + join tj8 on tj7.id = tj8.id + join tj9 on tj8.id = tj9.id + join tj10 on tj9.id = tj10.id + join tj11 on tj10.id = tj11.id + join tj12 on tj11.id = tj12.id + join tj13 on tj12.id = tj13.id + join tj14 on tj13.id = tj14.id + join tj15 on tj14.id = tj15.id + join tj16 on tj15.id = tj16.id + join tj17 on tj16.id = tj17.id + join tj18 on tj17.id = tj18.id + join tj19 on tj18.id = tj19.id + join tj20 on tj19.id = tj20.id; + +select (trunc(extract(epoch from now())) - :unix_time1) < 100 is_ok; + +reset optimizer; + -- Clean up. None of the objects we create are very interesting to keep around. reset search_path; set client_min_messages='warning'; From 103da7ad86e846cd241ac89e5085a55b9fa37d7e Mon Sep 17 00:00:00 2001 From: Jianghua Yang Date: Sat, 24 Jan 2026 04:44:25 +0800 Subject: [PATCH 13/22] ORCA: Optimize CDatumSortedSet by checking IsSorted before sorting Check IsSorted before Sort to reduce O(n log n) to O(n-1) comparisons for pre-sorted IN lists, improving ORCA optimization time. --- src/backend/gporca/libgpopt/src/base/CDatumSortedSet.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/backend/gporca/libgpopt/src/base/CDatumSortedSet.cpp b/src/backend/gporca/libgpopt/src/base/CDatumSortedSet.cpp index 7b141e767bc..0fe4481953d 100644 --- a/src/backend/gporca/libgpopt/src/base/CDatumSortedSet.cpp +++ b/src/backend/gporca/libgpopt/src/base/CDatumSortedSet.cpp @@ -42,7 +42,13 @@ CDatumSortedSet::CDatumSortedSet(CMemoryPool *mp, CExpression *pexprArray, { return; } - aprngdatum->Sort(&CUtils::IDatumCmp); + // Only sort if not already sorted - IsSorted is O(n-1) comparisons, + // while Sort is O(n log n), so this optimization helps when data + // is already sorted (common case for IN lists in queries) + if (!aprngdatum->IsSorted(&CUtils::IDatumCmp)) + { + aprngdatum->Sort(&CUtils::IDatumCmp); + } // de-duplicate const ULONG ulRangeArrayArity = aprngdatum->Size(); From 017b06b4f8698c47fa9f681103322aa85f167648 Mon Sep 17 00:00:00 2001 From: Andrey Sokolov Date: Tue, 30 Dec 2025 16:02:28 +0300 Subject: [PATCH 14/22] Fix and run gp_exttable_fdw tests Replace "format" with "format_type", because the format option is not supported --- .github/workflows/build-cloudberry.yml | 3 ++- gpcontrib/gp_exttable_fdw/input/gp_exttable_fdw.source | 2 +- gpcontrib/gp_exttable_fdw/output/gp_exttable_fdw.source | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-cloudberry.yml b/.github/workflows/build-cloudberry.yml index 9d44d06bbdc..65990f38a59 100644 --- a/.github/workflows/build-cloudberry.yml +++ b/.github/workflows/build-cloudberry.yml @@ -312,7 +312,8 @@ jobs: "gpcontrib/pxf_fdw:installcheck", "gpcontrib/zstd:installcheck", "gpcontrib/gp_sparse_vector:installcheck", - "gpcontrib/gp_toolkit:installcheck"] + "gpcontrib/gp_toolkit:installcheck", + "gpcontrib/gp_exttable_fdw:installcheck"] }, {"test":"ic-fixme", "make_configs":["src/test/regress:installcheck-fixme"], diff --git a/gpcontrib/gp_exttable_fdw/input/gp_exttable_fdw.source b/gpcontrib/gp_exttable_fdw/input/gp_exttable_fdw.source index 033dddd5215..41012e73c81 100644 --- a/gpcontrib/gp_exttable_fdw/input/gp_exttable_fdw.source +++ b/gpcontrib/gp_exttable_fdw/input/gp_exttable_fdw.source @@ -72,7 +72,7 @@ SELECT * FROM tableless_ext_fdw; -- When using CREATE FOREIGN TABLE syntax, '|' and '\' need to be escaped as '|' --> '\|' and '|' --> '\\'. CREATE FOREIGN TABLE ext_special_uri(a int, b int) SERVER gp_exttable_server -OPTIONS (format 'csv', delimiter ',', +OPTIONS (format_type 'c', delimiter ',', location_uris 'file://@hostname@@abs_srcdir@/data/spe\\cial1\||file://@hostname@@abs_srcdir@/data/\|special2\\'); \a SELECT urilocation FROM pg_exttable WHERE reloid = 'public.ext_special_uri'::regclass; diff --git a/gpcontrib/gp_exttable_fdw/output/gp_exttable_fdw.source b/gpcontrib/gp_exttable_fdw/output/gp_exttable_fdw.source index 337d21a99e8..a3191eb0853 100644 --- a/gpcontrib/gp_exttable_fdw/output/gp_exttable_fdw.source +++ b/gpcontrib/gp_exttable_fdw/output/gp_exttable_fdw.source @@ -76,7 +76,7 @@ SELECT * FROM tableless_ext_fdw; -- When using CREATE FOREIGN TABLE syntax, '|' and '\' need to be escaped as '|' --> '\|' and '|' --> '\\'. CREATE FOREIGN TABLE ext_special_uri(a int, b int) SERVER gp_exttable_server -OPTIONS (format 'csv', delimiter ',', +OPTIONS (format_type 'c', delimiter ',', location_uris 'file://@hostname@@abs_srcdir@/data/spe\\cial1\||file://@hostname@@abs_srcdir@/data/\|special2\\'); \a SELECT urilocation FROM pg_exttable WHERE reloid = 'public.ext_special_uri'::regclass; @@ -85,7 +85,7 @@ urilocation (1 row) SELECT ftoptions FROM pg_foreign_table WHERE ftrelid='public.ext_special_uri'::regclass; ftoptions -{format=csv,"delimiter=,","location_uris=file://@hostname@@abs_srcdir@/data/spe\\\\cial1\\||file://@hostname@@abs_srcdir@/data/\\|special2\\\\"} +{format_type=c,"delimiter=,","location_uris=file://@hostname@@abs_srcdir@/data/spe\\\\cial1\\||file://@hostname@@abs_srcdir@/data/\\|special2\\\\"} (1 row) \a SELECT * FROM ext_special_uri ORDER BY a; From 8887fb50cd34a466d66492738bc733bc6ad982ff Mon Sep 17 00:00:00 2001 From: reshke Date: Thu, 29 Jan 2026 00:44:27 +0500 Subject: [PATCH 15/22] Cherry-pick pg14.5 commit: Fix incorrect permissions-checking code for extended statistics. (#1550) Done more clean cherry-pick of CVE fix postgres/postgres@afe38fb Original commit message follows: ===== * Fix incorrect permissions-checking code for extended statistics. Commit a4d75c86b improved the extended-stats logic to allow extended stats to be collected on expressions not just bare Vars. To apply such stats, we first verify that the user has permissions to read all columns used in the stats. (If not, the query will likely fail at runtime, but the planner ought not do so.) That had to get extended to check permissions of columns appearing within such expressions, but the code for that was completely wrong: it applied pull_varattnos to the wrong pointer, leading to "unrecognized node type" failures. Furthermore, although you couldn't get to this because of that bug, it failed to account for the attnum offset applied by pull_varattnos. This escaped recognition so far because the code in question is not reached when the user has whole-table SELECT privilege (which is the common case), and because only subexpressions not specially handled by statext_is_compatible_clause_internal() are at risk. I think a large part of the reason for this bug is under-documentation of what statext_is_compatible_clause() is doing and what its arguments are, so do some work on the comments to try to improve that. Per bug #17570 from Alexander Kozhemyakin. Patch by Richard Guo; comments and other cosmetic improvements by me. (Thanks also to Japin Li for diagnosis.) Back-patch to v14 where the bug came in. Discussion: https://postgr.es/m/17570-f2f2e0f4bccf0965@postgresql.org --------- Co-authored-by: Tom Lane --- src/backend/statistics/extended_stats.c | 124 ++++++++++++------ src/test/regress/expected/stats_ext.out | 4 + .../regress/expected/stats_ext_optimizer.out | 4 + src/test/regress/sql/stats_ext.sql | 4 + 4 files changed, 98 insertions(+), 38 deletions(-) diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c index d3561b779ab..aff0b0db05b 100644 --- a/src/backend/statistics/extended_stats.c +++ b/src/backend/statistics/extended_stats.c @@ -1318,10 +1318,38 @@ choose_best_statistics(List *stats, char requiredkind, * statext_is_compatible_clause_internal * Determines if the clause is compatible with MCV lists. * - * Does the heavy lifting of actually inspecting the clauses for - * statext_is_compatible_clause. It needs to be split like this because - * of recursion. The attnums bitmap is an input/output parameter collecting - * attribute numbers from all compatible clauses (recursively). + * To be compatible, the given clause must be a combination of supported + * clauses built from Vars or sub-expressions (where a sub-expression is + * something that exactly matches an expression found in statistics objects). + * This function recursively examines the clause and extracts any + * sub-expressions that will need to be matched against statistics. + * + * Currently, we only support the following types of clauses: + * + * (a) OpExprs of the form (Var/Expr op Const), or (Const op Var/Expr), where + * the op is one of ("=", "<", ">", ">=", "<=") + * + * (b) (Var/Expr IS [NOT] NULL) + * + * (c) combinations using AND/OR/NOT + * + * (d) ScalarArrayOpExprs of the form (Var/Expr op ANY (array)) or (Var/Expr + * op ALL (array)) + * + * In the future, the range of supported clauses may be expanded to more + * complex cases, for example (Var op Var). + * + * Arguments: + * clause: (sub)clause to be inspected (bare clause, not a RestrictInfo) + * relid: rel that all Vars in clause must belong to + * *attnums: input/output parameter collecting attribute numbers of all + * mentioned Vars. Note that we do not offset the attribute numbers, + * so we can't cope with system columns. + * *exprs: input/output parameter collecting primitive subclauses within + * the clause tree + * + * Returns false if there is something we definitively can't handle. + * On true return, we can proceed to match the *exprs against statistics. */ static bool statext_is_compatible_clause_internal(PlannerInfo *root, Node *clause, @@ -1345,10 +1373,14 @@ statext_is_compatible_clause_internal(PlannerInfo *root, Node *clause, if (var->varlevelsup > 0) return false; - /* Also skip system attributes (we don't allow stats on those). */ + /* + * Also reject system attributes and whole-row Vars (we don't allow + * stats on those). + */ if (!AttrNumberIsForUserDefinedAttr(var->varattno)) return false; + /* OK, record the attnum for later permissions checks. */ *attnums = bms_add_member(*attnums, var->varattno); return true; @@ -1503,7 +1535,7 @@ statext_is_compatible_clause_internal(PlannerInfo *root, Node *clause, foreach(lc, expr->args) { /* - * Had we found incompatible clause in the arguments, treat the + * If we find an incompatible clause in the arguments, treat the * whole clause as incompatible. */ if (!statext_is_compatible_clause_internal(root, @@ -1542,27 +1574,28 @@ statext_is_compatible_clause_internal(PlannerInfo *root, Node *clause, * statext_is_compatible_clause * Determines if the clause is compatible with MCV lists. * - * Currently, we only support the following types of clauses: + * See statext_is_compatible_clause_internal, above, for the basic rules. + * This layer deals with RestrictInfo superstructure and applies permissions + * checks to verify that it's okay to examine all mentioned Vars. * - * (a) OpExprs of the form (Var/Expr op Const), or (Const op Var/Expr), where - * the op is one of ("=", "<", ">", ">=", "<=") + * Arguments: + * clause: clause to be inspected (in RestrictInfo form) + * relid: rel that all Vars in clause must belong to + * *attnums: input/output parameter collecting attribute numbers of all + * mentioned Vars. Note that we do not offset the attribute numbers, + * so we can't cope with system columns. + * *exprs: input/output parameter collecting primitive subclauses within + * the clause tree * - * (b) (Var/Expr IS [NOT] NULL) - * - * (c) combinations using AND/OR/NOT - * - * (d) ScalarArrayOpExprs of the form (Var/Expr op ANY (array)) or (Var/Expr - * op ALL (array)) - * - * In the future, the range of supported clauses may be expanded to more - * complex cases, for example (Var op Var). + * Returns false if there is something we definitively can't handle. + * On true return, we can proceed to match the *exprs against statistics. */ static bool statext_is_compatible_clause(PlannerInfo *root, Node *clause, Index relid, Bitmapset **attnums, List **exprs) { RangeTblEntry *rte = root->simple_rte_array[relid]; - RestrictInfo *rinfo = (RestrictInfo *) clause; + RestrictInfo *rinfo; int clause_relid; Oid userid; @@ -1591,8 +1624,9 @@ statext_is_compatible_clause(PlannerInfo *root, Node *clause, Index relid, } /* Otherwise it must be a RestrictInfo. */ - if (!IsA(rinfo, RestrictInfo)) + if (!IsA(clause, RestrictInfo)) return false; + rinfo = (RestrictInfo *) clause; /* Pseudoconstants are not really interesting here. */ if (rinfo->pseudoconstant) @@ -1614,34 +1648,48 @@ statext_is_compatible_clause(PlannerInfo *root, Node *clause, Index relid, */ userid = rte->checkAsUser ? rte->checkAsUser : GetUserId(); + /* Table-level SELECT privilege is sufficient for all columns */ if (pg_class_aclcheck(rte->relid, userid, ACL_SELECT) != ACLCHECK_OK) { Bitmapset *clause_attnums = NULL; + int attnum = -1; - /* Don't have table privilege, must check individual columns */ - if (*exprs != NIL) + /* + * We have to check per-column privileges. *attnums has the attnums + * for individual Vars we saw, but there may also be Vars within + * subexpressions in *exprs. We can use pull_varattnos() to extract + * those, but there's an impedance mismatch: attnums returned by + * pull_varattnos() are offset by FirstLowInvalidHeapAttributeNumber, + * while attnums within *attnums aren't. Convert *attnums to the + * offset style so we can combine the results. + */ + while ((attnum = bms_next_member(*attnums, attnum)) >= 0) { - pull_varattnos((Node *) exprs, relid, &clause_attnums); - clause_attnums = bms_add_members(clause_attnums, *attnums); + clause_attnums = + bms_add_member(clause_attnums, + attnum - FirstLowInvalidHeapAttributeNumber); } - else - clause_attnums = *attnums; - if (bms_is_member(InvalidAttrNumber, clause_attnums)) - { - /* Have a whole-row reference, must have access to all columns */ - if (pg_attribute_aclcheck_all(rte->relid, userid, ACL_SELECT, - ACLMASK_ALL) != ACLCHECK_OK) - return false; - } - else + /* Now merge attnums from *exprs into clause_attnums */ + if (*exprs != NIL) + pull_varattnos((Node *) *exprs, relid, &clause_attnums); + + attnum = -1; + while ((attnum = bms_next_member(clause_attnums, attnum)) >= 0) { - /* Check the columns referenced by the clause */ - int attnum = -1; + /* Undo the offset */ + AttrNumber attno = attnum + FirstLowInvalidHeapAttributeNumber; - while ((attnum = bms_next_member(clause_attnums, attnum)) >= 0) + if (attno == InvalidAttrNumber) + { + /* Whole-row reference, so must have access to all columns */ + if (pg_attribute_aclcheck_all(rte->relid, userid, ACL_SELECT, + ACLMASK_ALL) != ACLCHECK_OK) + return false; + } + else { - if (pg_attribute_aclcheck(rte->relid, attnum, userid, + if (pg_attribute_aclcheck(rte->relid, attno, userid, ACL_SELECT) != ACLCHECK_OK) return false; } diff --git a/src/test/regress/expected/stats_ext.out b/src/test/regress/expected/stats_ext.out index 3fc90553026..b752abfc4c6 100644 --- a/src/test/regress/expected/stats_ext.out +++ b/src/test/regress/expected/stats_ext.out @@ -3196,6 +3196,10 @@ GRANT USAGE ON SCHEMA tststats TO regress_stats_user1; SET SESSION AUTHORIZATION regress_stats_user1; SELECT * FROM tststats.priv_test_tbl; -- Permission denied ERROR: permission denied for table priv_test_tbl +-- Check individual columns if we don't have table privilege +SELECT * FROM tststats.priv_test_tbl + WHERE a = 1 and tststats.priv_test_tbl.* > (1, 1) is not null; +ERROR: permission denied for table priv_test_tbl -- Attempt to gain access using a leaky operator CREATE FUNCTION op_leak(int, int) RETURNS bool AS 'BEGIN RAISE NOTICE ''op_leak => %, %'', $1, $2; RETURN $1 < $2; END' diff --git a/src/test/regress/expected/stats_ext_optimizer.out b/src/test/regress/expected/stats_ext_optimizer.out index d19caa775d1..9f1b78af0b3 100644 --- a/src/test/regress/expected/stats_ext_optimizer.out +++ b/src/test/regress/expected/stats_ext_optimizer.out @@ -3231,6 +3231,10 @@ GRANT USAGE ON SCHEMA tststats TO regress_stats_user1; SET SESSION AUTHORIZATION regress_stats_user1; SELECT * FROM tststats.priv_test_tbl; -- Permission denied ERROR: permission denied for table priv_test_tbl +-- Check individual columns if we don't have table privilege +SELECT * FROM tststats.priv_test_tbl + WHERE a = 1 and tststats.priv_test_tbl.* > (1, 1) is not null; +ERROR: permission denied for table priv_test_tbl -- Attempt to gain access using a leaky operator CREATE FUNCTION op_leak(int, int) RETURNS bool AS 'BEGIN RAISE NOTICE ''op_leak => %, %'', $1, $2; RETURN $1 < $2; END' diff --git a/src/test/regress/sql/stats_ext.sql b/src/test/regress/sql/stats_ext.sql index 91edd3a5bba..6840818118d 100644 --- a/src/test/regress/sql/stats_ext.sql +++ b/src/test/regress/sql/stats_ext.sql @@ -1615,6 +1615,10 @@ GRANT USAGE ON SCHEMA tststats TO regress_stats_user1; SET SESSION AUTHORIZATION regress_stats_user1; SELECT * FROM tststats.priv_test_tbl; -- Permission denied +-- Check individual columns if we don't have table privilege +SELECT * FROM tststats.priv_test_tbl + WHERE a = 1 and tststats.priv_test_tbl.* > (1, 1) is not null; + -- Attempt to gain access using a leaky operator CREATE FUNCTION op_leak(int, int) RETURNS bool AS 'BEGIN RAISE NOTICE ''op_leak => %, %'', $1, $2; RETURN $1 < $2; END' From eb0f1cc50db70d2720a14fd1a657600e22708bb4 Mon Sep 17 00:00:00 2001 From: Nikolay Antonov Date: Thu, 29 Jan 2026 18:25:04 +0500 Subject: [PATCH 16/22] Devops: Remove pxf_fdw from default build (#1549) --- .github/workflows/build-cloudberry.yml | 1 - .github/workflows/build-deb-cloudberry.yml | 1 - .../build/automation/cloudberry/scripts/configure-cloudberry.sh | 2 +- devops/sandbox/Dockerfile.RELEASE.rockylinux9 | 2 +- 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-cloudberry.yml b/.github/workflows/build-cloudberry.yml index 65990f38a59..f4104e9ced1 100644 --- a/.github/workflows/build-cloudberry.yml +++ b/.github/workflows/build-cloudberry.yml @@ -309,7 +309,6 @@ jobs: }, {"test":"ic-gpcontrib", "make_configs":["gpcontrib/orafce:installcheck", - "gpcontrib/pxf_fdw:installcheck", "gpcontrib/zstd:installcheck", "gpcontrib/gp_sparse_vector:installcheck", "gpcontrib/gp_toolkit:installcheck", diff --git a/.github/workflows/build-deb-cloudberry.yml b/.github/workflows/build-deb-cloudberry.yml index be28fff9e77..53b3c54e038 100644 --- a/.github/workflows/build-deb-cloudberry.yml +++ b/.github/workflows/build-deb-cloudberry.yml @@ -248,7 +248,6 @@ jobs: }, {"test":"ic-deb-gpcontrib", "make_configs":["gpcontrib/orafce:installcheck", - "gpcontrib/pxf_fdw:installcheck", "gpcontrib/zstd:installcheck", "gpcontrib/gp_sparse_vector:installcheck", "gpcontrib/gp_toolkit:installcheck"] diff --git a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh index bc046695032..54086736a5f 100755 --- a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh +++ b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh @@ -148,7 +148,7 @@ execute_cmd ./configure --prefix=${BUILD_DESTINATION} \ --enable-orafce \ --enable-orca \ --enable-pax \ - --enable-pxf \ + --disable-pxf \ --enable-tap-tests \ ${CONFIGURE_DEBUG_OPTS} \ --with-gssapi \ diff --git a/devops/sandbox/Dockerfile.RELEASE.rockylinux9 b/devops/sandbox/Dockerfile.RELEASE.rockylinux9 index f9f422f57f6..ac394c6cb60 100644 --- a/devops/sandbox/Dockerfile.RELEASE.rockylinux9 +++ b/devops/sandbox/Dockerfile.RELEASE.rockylinux9 @@ -152,7 +152,7 @@ RUN cd /home/gpadmin/cloudberry && \ --enable-orafce \ --enable-orca \ --enable-pax \ - --enable-pxf \ + --disable-pxf \ --enable-tap-tests \ --with-gssapi \ --with-ldap \ From 6cab8f74e211414c1642cca9b6c22a40ed8751ff Mon Sep 17 00:00:00 2001 From: Hao Wu Date: Fri, 30 Jan 2026 03:44:18 +0000 Subject: [PATCH 17/22] Fix typo name for support_io_uring --- contrib/pax_storage/src/cpp/comm/fast_io.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/pax_storage/src/cpp/comm/fast_io.cc b/contrib/pax_storage/src/cpp/comm/fast_io.cc index 7ed96d7a377..c2dcd235541 100644 --- a/contrib/pax_storage/src/cpp/comm/fast_io.cc +++ b/contrib/pax_storage/src/cpp/comm/fast_io.cc @@ -26,7 +26,7 @@ */ #include "fast_io.h" - +#include #include // for pread // uring_likely may not be defined in older liburing versions @@ -42,7 +42,7 @@ namespace pax { bool IOUringFastIO::available() { - static char support_io_uring = 0; + static int8_t support_io_uring = 0; if (support_io_uring == 1) return true; if (support_io_uring == -1) return false; @@ -142,4 +142,4 @@ std::pair SyncFastIO::read(int fd, std::vector &request, st return {retcode, success_read}; } -} // namespace pax \ No newline at end of file +} // namespace pax From 9b98b82839db7843932d4b6239a9f23edea311e0 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Wed, 31 Dec 2025 17:41:24 +0800 Subject: [PATCH 18/22] CI: Add Rocky8 workflow with test matrix support This commit introduces a new GitHub Actions workflow for building and testing Apache Cloudberry on Rocky Linux 8, enabling automated builds, RPM packaging, and regresssion testing alongside the existing Rocky 9 and Ubuntu 22.04 pipelines. Triggers: - Push to main branch - Pull requests modifying this workflow file - Scheduled: Every Monday at 02:00 UTC - Manual workflow dispatch with optional test selection --- .github/workflows/build-cloudberry-rocky8.yml | 1910 +++++++++++++++++ 1 file changed, 1910 insertions(+) create mode 100644 .github/workflows/build-cloudberry-rocky8.yml diff --git a/.github/workflows/build-cloudberry-rocky8.yml b/.github/workflows/build-cloudberry-rocky8.yml new file mode 100644 index 00000000000..5028af1315e --- /dev/null +++ b/.github/workflows/build-cloudberry-rocky8.yml @@ -0,0 +1,1910 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# GitHub Actions Workflow: Apache Cloudberry Build Pipeline (Rocky 8) +# -------------------------------------------------------------------- +# Description: +# +# This workflow builds, tests, and packages Apache Cloudberry on +# Rocky Linux 8. It ensures artifact integrity, performs installation +# tests, validates key operations, and provides detailed test reports, +# including handling for ignored test cases. +# +# Workflow Overview: +# 1. **Check Skip**: +# - Dynamically determines if the workflow should run based on CI skip flags. +# - Evaluates the following fields for skip flags: +# - **Pull Request Events**: PR title and PR body. +# - **Push Events**: Commit message of the head commit. +# - Supports the following skip patterns (case-insensitive): +# - `[skip ci]` +# - `[ci skip]` +# - `[no ci]` +# - **Example Usage**: +# - Add `[skip ci]` to a commit message, PR title, or body to skip the workflow. +# +# 2. **Build Job**: +# - Configures and builds Apache Cloudberry. +# - Supports debug build configuration via ENABLE_DEBUG flag. +# - Runs unit tests and verifies build artifacts. +# - Creates RPM packages (regular or debug), source tarballs, and logs. +# - **Key Artifacts**: RPM package, source tarball, build logs. +# +# 3. **RPM Install Test Job**: +# - Verifies RPM integrity and installs Cloudberry. +# - Validates successful installation. +# - **Key Artifacts**: Installation logs, verification results. +# +# 4. **Test Job (Matrix)**: +# - Executes a test matrix to validate different scenarios. +# - Creates a demo cluster and runs installcheck tests. +# - Parses and reports test results, including failed and ignored tests. +# - Detects and analyzes any core dumps generated during tests. +# - **Key Features**: +# - Regression diffs are displayed if found, aiding quick debugging. +# - Both failed and ignored test names are logged and reported. +# - Core dumps are analyzed using GDB for stack traces. +# - **Key Artifacts**: Test logs, regression files, test summaries, core analyses. +# +# 5. **Report Job**: +# - Aggregates job results into a final report. +# - Sends failure notifications if any step fails. +# +# Execution Environment: +# - **Runs On**: ubuntu-22.04 with Rocky Linux 8 containers. +# - **Resource Requirements**: +# - Disk: Minimum 20GB free space. +# - Memory: Minimum 8GB RAM. +# - CPU: Recommended 4+ cores. +# +# Triggers: +# - Push to `main` branch. +# - Pull request that modifies this workflow file. +# - Scheduled: Every Monday at 02:00 UTC. +# - Manual workflow dispatch. +# +# Container Images: +# - **Build**: `apache/incubator-cloudberry:cbdb-build-rocky8-latest` +# - **Test**: `apache/incubator-cloudberry:cbdb-test-rocky8-latest` +# +# Artifacts: +# - RPM Package (retention: ${{ env.LOG_RETENTION_DAYS }} days). +# - Source Tarball (retention: ${{ env.LOG_RETENTION_DAYS }} days). +# - Logs and Test Results (retention: ${{ env.LOG_RETENTION_DAYS }} days). +# - Regression Diffs (retention: ${{ env.LOG_RETENTION_DAYS }} days). +# - Core Dump Analyses (retention: ${{ env.LOG_RETENTION_DAYS }} days). +# +# Notes: +# - Supports concurrent job execution. +# - Includes robust skip logic for pull requests and pushes. +# - Handles ignored test cases, ensuring results are comprehensive. +# - Provides detailed logs and error handling for failed and ignored tests. +# - Analyzes core dumps generated during test execution. +# - Supports debug builds with preserved symbols. +# -------------------------------------------------------------------- + +name: Apache Cloudberry Build (Rocky 8) + +on: + push: + branches: [main, REL_2_STABLE] + pull_request: + paths: + - '.github/workflows/build-cloudberry-rocky8.yml' + # We can enable the PR test when needed + # branches: [main, REL_2_STABLE] + # types: [opened, synchronize, reopened, edited] + schedule: + # Run every Monday at 02:00 UTC + - cron: '0 2 * * 1' + workflow_dispatch: + inputs: + test_selection: + description: 'Select tests to run (comma-separated). Examples: ic-good-opt-off,ic-contrib' + required: false + default: 'all' + type: string + reuse_artifacts_from_run_id: + description: 'Reuse build artifacts from a previous run ID (leave empty to build fresh)' + required: false + default: '' + type: string + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +# Note: Step details, logs, and artifacts require users to be logged into GitHub +# even for public repositories. This is a GitHub security feature and cannot +# be overridden by permissions. + +permissions: + # READ permissions allow viewing repository contents + contents: read # Required for checking out code and reading repository files + + # READ permissions for packages (Container registry, etc) + packages: read # Allows reading from GitHub package registry + + # WRITE permissions for actions includes read access to: + # - Workflow runs + # - Artifacts (requires GitHub login) + # - Logs (requires GitHub login) + actions: write + + # READ permissions for checks API: + # - Step details visibility (requires GitHub login) + # - Check run status and details + checks: read + + # READ permissions for pull request metadata: + # - PR status + # - Associated checks + # - Review states + pull-requests: read + +env: + LOG_RETENTION_DAYS: 7 + ENABLE_DEBUG: false + +jobs: + + ## ====================================================================== + ## Job: check-skip + ## ====================================================================== + + check-skip: + runs-on: ubuntu-22.04 + outputs: + should_skip: ${{ steps.skip-check.outputs.should_skip }} + steps: + - id: skip-check + shell: bash + env: + EVENT_NAME: ${{ github.event_name }} + PR_TITLE: ${{ github.event.pull_request.title || '' }} + PR_BODY: ${{ github.event.pull_request.body || '' }} + run: | + # Default to not skipping + echo "should_skip=false" >> "$GITHUB_OUTPUT" + + # Apply skip logic only for pull_request events + if [[ "$EVENT_NAME" == "pull_request" ]]; then + # Combine PR title and body for skip check + MESSAGE="${PR_TITLE}\n${PR_BODY}" + + # Escape special characters using printf %s + ESCAPED_MESSAGE=$(printf "%s" "$MESSAGE") + + echo "Checking PR title and body (escaped): $ESCAPED_MESSAGE" + + # Check for skip patterns + if echo -e "$ESCAPED_MESSAGE" | grep -qEi '\[skip[ -]ci\]|\[ci[ -]skip\]|\[no[ -]ci\]'; then + echo "should_skip=true" >> "$GITHUB_OUTPUT" + fi + else + echo "Skip logic is not applied for $EVENT_NAME events." + fi + + - name: Report Skip Status + if: steps.skip-check.outputs.should_skip == 'true' + run: | + echo "CI Skip flag detected in PR - skipping all checks." + exit 0 + + ## ====================================================================== + ## Job: prepare-test-matrix + ## ====================================================================== + + prepare-test-matrix: + runs-on: ubuntu-22.04 + needs: [check-skip] + if: needs.check-skip.outputs.should_skip != 'true' + outputs: + test-matrix: ${{ steps.set-matrix.outputs.matrix }} + + steps: + - id: set-matrix + run: | + echo "=== Matrix Preparation Diagnostics ===" + echo "Event type: ${{ github.event_name }}" + echo "Test selection input: '${{ github.event.inputs.test_selection }}'" + + # Define defaults + DEFAULT_NUM_PRIMARY_MIRROR_PAIRS=3 + DEFAULT_ENABLE_CGROUPS=false + DEFAULT_ENABLE_CORE_CHECK=true + DEFAULT_PG_SETTINGS_OPTIMIZER="" + + # Define base test configurations + ALL_TESTS='{ + "include": [ + {"test":"ic-good-opt-off", + "make_configs":["src/test/regress:installcheck-good"], + "pg_settings":{"optimizer":"off"} + }, + {"test":"ic-good-opt-on", + "make_configs":["src/test/regress:installcheck-good"], + "pg_settings":{"optimizer":"on"} + }, + {"test":"pax-ic-good-opt-off", + "make_configs":[ + "contrib/pax_storage/:pax-test", + "contrib/pax_storage/:regress_test" + ], + "pg_settings":{ + "optimizer":"off", + "default_table_access_method":"pax" + } + }, + {"test":"pax-ic-good-opt-on", + "make_configs":[ + "contrib/pax_storage/:pax-test", + "contrib/pax_storage/:regress_test" + ], + "pg_settings":{ + "optimizer":"on", + "default_table_access_method":"pax" + } + }, + {"test":"pax-ic-isolation2-opt-off", + "make_configs":["contrib/pax_storage/:isolation2_test"], + "pg_settings":{ + "optimizer":"off", + "default_table_access_method":"pax" + }, + "enable_core_check":false + }, + {"test":"pax-ic-isolation2-opt-on", + "make_configs":["contrib/pax_storage/:isolation2_test"], + "pg_settings":{ + "optimizer":"on", + "default_table_access_method":"pax" + }, + "enable_core_check":false + }, + {"test":"ic-expandshrink", + "make_configs":["src/test/isolation2:installcheck-expandshrink"] + }, + {"test":"ic-singlenode", + "make_configs":["src/test/isolation:installcheck-singlenode", + "src/test/singlenode_regress:installcheck-singlenode", + "src/test/singlenode_isolation2:installcheck-singlenode"], + "num_primary_mirror_pairs":0 + }, + {"test":"ic-resgroup-v2", + "make_configs":["src/test/isolation2:installcheck-resgroup-v2"], + "enable_cgroups":true + }, + {"test":"ic-contrib", + "make_configs":["contrib/auto_explain:installcheck", + "contrib/amcheck:installcheck", + "contrib/citext:installcheck", + "contrib/btree_gin:installcheck", + "contrib/btree_gist:installcheck", + "contrib/dblink:installcheck", + "contrib/dict_int:installcheck", + "contrib/dict_xsyn:installcheck", + "contrib/extprotocol:installcheck", + "contrib/file_fdw:installcheck", + "contrib/formatter_fixedwidth:installcheck", + "contrib/hstore:installcheck", + "contrib/indexscan:installcheck", + "contrib/pg_trgm:installcheck", + "contrib/indexscan:installcheck", + "contrib/pgcrypto:installcheck", + "contrib/pgstattuple:installcheck", + "contrib/tablefunc:installcheck", + "contrib/passwordcheck:installcheck", + "contrib/pg_buffercache:installcheck", + "contrib/sslinfo:installcheck"] + }, + {"test":"ic-gpcontrib", + "make_configs":["gpcontrib/orafce:installcheck", + "gpcontrib/pxf_fdw:installcheck", + "gpcontrib/zstd:installcheck", + "gpcontrib/gp_sparse_vector:installcheck", + "gpcontrib/gp_toolkit:installcheck"] + }, + {"test":"ic-fixme", + "make_configs":["src/test/regress:installcheck-fixme"], + "enable_core_check":false + }, + {"test":"ic-isolation2", + "make_configs":["src/test/isolation2:installcheck-isolation2"] + }, + {"test":"ic-isolation2-hot-standby", + "make_configs":["src/test/isolation2:installcheck-hot-standby"] + }, + {"test":"ic-isolation2-crash", + "make_configs":["src/test/isolation2:installcheck-isolation2-crash"], + "enable_core_check":false + }, + {"test":"ic-parallel-retrieve-cursor", + "make_configs":["src/test/isolation2:installcheck-parallel-retrieve-cursor"] + }, + {"test":"ic-cbdb-parallel", + "make_configs":["src/test/regress:installcheck-cbdb-parallel"] + } + ] + }' + + # Function to apply defaults + apply_defaults() { + echo "$1" | jq --arg npm "$DEFAULT_NUM_PRIMARY_MIRROR_PAIRS" \ + --argjson ec "$DEFAULT_ENABLE_CGROUPS" \ + --argjson ecc "$DEFAULT_ENABLE_CORE_CHECK" \ + --arg opt "$DEFAULT_PG_SETTINGS_OPTIMIZER" \ + 'def get_defaults: + { + num_primary_mirror_pairs: ($npm|tonumber), + enable_cgroups: $ec, + enable_core_check: $ecc, + pg_settings: { + optimizer: $opt + } + }; + get_defaults * .' + } + + # Extract all valid test names from ALL_TESTS + VALID_TESTS=$(echo "$ALL_TESTS" | jq -r '.include[].test') + + # Parse input test selection + IFS=',' read -ra SELECTED_TESTS <<< "${{ github.event.inputs.test_selection }}" + + # Default to all tests if selection is empty or 'all' + if [[ "${SELECTED_TESTS[*]}" == "all" || -z "${SELECTED_TESTS[*]}" ]]; then + mapfile -t SELECTED_TESTS <<< "$VALID_TESTS" + fi + + # Validate and filter selected tests + INVALID_TESTS=() + FILTERED_TESTS=() + for TEST in "${SELECTED_TESTS[@]}"; do + TEST=$(echo "$TEST" | tr -d '[:space:]') # Trim whitespace + if echo "$VALID_TESTS" | grep -qw "$TEST"; then + FILTERED_TESTS+=("$TEST") + else + INVALID_TESTS+=("$TEST") + fi + done + + # Handle invalid tests + if [[ ${#INVALID_TESTS[@]} -gt 0 ]]; then + echo "::error::Invalid test(s) selected: ${INVALID_TESTS[*]}" + echo "Valid tests are: $(echo "$VALID_TESTS" | tr '\n' ', ')" + exit 1 + fi + + # Build result JSON with defaults applied + RESULT='{"include":[' + FIRST=true + for TEST in "${FILTERED_TESTS[@]}"; do + CONFIG=$(jq -c --arg test "$TEST" '.include[] | select(.test == $test)' <<< "$ALL_TESTS") + FILTERED_WITH_DEFAULTS=$(apply_defaults "$CONFIG") + if [[ "$FIRST" == true ]]; then + FIRST=false + else + RESULT="${RESULT}," + fi + RESULT="${RESULT}${FILTERED_WITH_DEFAULTS}" + done + RESULT="${RESULT}]}" + + # Output the matrix for GitHub Actions + echo "Final matrix configuration:" + echo "$RESULT" | jq . + + # Fix: Use block redirection + { + echo "matrix<> "$GITHUB_OUTPUT" + + echo "=== Matrix Preparation Complete ===" + + ## ====================================================================== + ## Job: build + ## ====================================================================== + + build: + name: Build Apache Cloudberry RPM (Rocky 8) + env: + JOB_TYPE: build + needs: [check-skip] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + if: github.event.inputs.reuse_artifacts_from_run_id == '' + outputs: + build_timestamp: ${{ steps.set_timestamp.outputs.timestamp }} + + container: + image: apache/incubator-cloudberry:cbdb-build-rocky8-latest + options: >- + --user root + -h cdw + -v /usr/share:/host_usr_share + -v /usr/local:/host_usr_local + -v /opt:/host_opt + + steps: + - name: Free Disk Space + if: needs.check-skip.outputs.should_skip != 'true' + run: | + echo "=== Disk space before cleanup ===" + df -h / + + # Remove pre-installed tools from host to free disk space + rm -rf /host_opt/hostedtoolcache || true # GitHub Actions tool cache + rm -rf /host_usr_local/lib/android || true # Android SDK + rm -rf /host_usr_share/dotnet || true # .NET SDK + rm -rf /host_opt/ghc || true # Haskell GHC + rm -rf /host_usr_local/.ghcup || true # Haskell GHCup + rm -rf /host_usr_share/swift || true # Swift + rm -rf /host_usr_local/share/powershell || true # PowerShell + rm -rf /host_usr_local/share/chromium || true # Chromium + rm -rf /host_usr_share/miniconda || true # Miniconda + rm -rf /host_opt/az || true # Azure CLI + rm -rf /host_usr_share/sbt || true # Scala Build Tool + + echo "=== Disk space after cleanup ===" + df -h / + + - name: Skip Check + if: needs.check-skip.outputs.should_skip == 'true' + run: | + echo "Build skipped via CI skip flag" >> "$GITHUB_STEP_SUMMARY" + exit 0 + + - name: Set build timestamp + if: needs.check-skip.outputs.should_skip != 'true' + id: set_timestamp # Add an ID to reference this step + run: | + timestamp=$(date +'%Y%m%d_%H%M%S') + echo "timestamp=$timestamp" | tee -a "$GITHUB_OUTPUT" # Use GITHUB_OUTPUT for job outputs + echo "BUILD_TIMESTAMP=$timestamp" | tee -a "$GITHUB_ENV" # Also set as environment variable + + - name: Checkout Apache Cloudberry + if: needs.check-skip.outputs.should_skip != 'true' + uses: actions/checkout@v4 + with: + fetch-depth: 1 + submodules: true + + - name: Cloudberry Environment Initialization + if: needs.check-skip.outputs.should_skip != 'true' + env: + LOGS_DIR: build-logs + run: | + set -eo pipefail + if ! su - gpadmin -c "/tmp/init_system.sh"; then + echo "::error::Container initialization failed" + exit 1 + fi + + mkdir -p "${LOGS_DIR}/details" + chown -R gpadmin:gpadmin . + chmod -R 755 . + chmod 777 "${LOGS_DIR}" + + df -kh / + rm -rf /__t/* + df -kh / + + df -h | tee -a "${LOGS_DIR}/details/disk-usage.log" + free -h | tee -a "${LOGS_DIR}/details/memory-usage.log" + + { + echo "=== Environment Information ===" + uname -a + df -h + free -h + env + } | tee -a "${LOGS_DIR}/details/environment.log" + + echo "SRC_DIR=${GITHUB_WORKSPACE}" | tee -a "$GITHUB_ENV" + + - name: Generate Build Job Summary Start + if: needs.check-skip.outputs.should_skip != 'true' + run: | + { + echo "# Build Job Summary" + echo "## Environment" + echo "- Start Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" + echo "- ENABLE_DEBUG: ${{ env.ENABLE_DEBUG }}" + echo "- OS Version: $(cat /etc/redhat-release)" + echo "- GCC Version: $(gcc --version | head -n1)" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Run Apache Cloudberry configure script + if: needs.check-skip.outputs.should_skip != 'true' + env: + SRC_DIR: ${{ github.workspace }} + run: | + set -eo pipefail + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then + echo "::error::Configure script failed" + exit 1 + fi + + - name: Run Apache Cloudberry build script + if: needs.check-skip.outputs.should_skip != 'true' + env: + SRC_DIR: ${{ github.workspace }} + run: | + set -eo pipefail + + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/build-cloudberry.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/build-cloudberry.sh"; then + echo "::error::Build script failed" + exit 1 + fi + + - name: Verify build artifacts + if: needs.check-skip.outputs.should_skip != 'true' + run: | + set -eo pipefail + + echo "Verifying build artifacts..." + { + echo "=== Build Artifacts Verification ===" + echo "Timestamp: $(date -u)" + + if [ ! -d "/usr/local/cloudberry-db" ]; then + echo "::error::Build artifacts directory not found" + exit 1 + fi + + # Verify critical binaries + critical_binaries=( + "/usr/local/cloudberry-db/bin/postgres" + "/usr/local/cloudberry-db/bin/psql" + ) + + echo "Checking critical binaries..." + for binary in "${critical_binaries[@]}"; do + if [ ! -f "$binary" ]; then + echo "::error::Critical binary missing: $binary" + exit 1 + fi + if [ ! -x "$binary" ]; then + echo "::error::Binary not executable: $binary" + exit 1 + fi + echo "Binary verified: $binary" + ls -l "$binary" + done + + # Test binary execution + echo "Testing binary execution..." + if ! /usr/local/cloudberry-db/bin/postgres --version; then + echo "::error::postgres binary verification failed" + exit 1 + fi + if ! /usr/local/cloudberry-db/bin/psql --version; then + echo "::error::psql binary verification failed" + exit 1 + fi + + echo "All build artifacts verified successfully" + } 2>&1 | tee -a build-logs/details/build-verification.log + + - name: Create Source tarball, create RPM and verify artifacts + if: needs.check-skip.outputs.should_skip != 'true' + env: + CBDB_VERSION: 99.0.0 + BUILD_NUMBER: 1 + SRC_DIR: ${{ github.workspace }} + run: | + set -eo pipefail + + { + echo "=== Artifact Creation Log ===" + echo "Timestamp: $(date -u)" + + # Create source tarball + echo "Creating source tarball..." + tar czf "${SRC_DIR}"/../apache-cloudberry-incubating-src.tgz -C "${SRC_DIR}"/.. ./cloudberry + mv "${SRC_DIR}"/../apache-cloudberry-incubating-src.tgz "${SRC_DIR}" + + # Verify tarball contents + echo "Verifying source tarball contents..." + if ! tar tzf "${SRC_DIR}"/apache-cloudberry-incubating-src.tgz > /dev/null; then + echo "::error::Source tarball verification failed" + exit 1 + fi + + # Create RPM + echo "Creating RPM package..." + rpmdev-setuptree + ln -s "${SRC_DIR}"/devops/build/packaging/rpm/apache-cloudberry-db-incubating.spec "${HOME}"/rpmbuild/SPECS/apache-cloudberry-db-incubating.spec + cp "${SRC_DIR}"/LICENSE /usr/local/cloudberry-db + + DEBUG_RPMBUILD_OPT="" + DEBUG_IDENTIFIER="" + if [ "${{ env.ENABLE_DEBUG }}" = "true" ]; then + DEBUG_RPMBUILD_OPT="--with-debug" + DEBUG_IDENTIFIER=".debug" + fi + + "${SRC_DIR}"/devops/build/packaging/rpm/build-rpm.sh --version "${CBDB_VERSION}" --release "${BUILD_NUMBER}" "${DEBUG_RPMBUILD_OPT}" + + # Get OS version and move RPM + os_version=$(grep -oP '(?<=^VERSION_ID=")[0-9]' /etc/os-release) + RPM_FILE="${HOME}"/rpmbuild/RPMS/x86_64/apache-cloudberry-db-incubating-"${CBDB_VERSION}"-"${BUILD_NUMBER}""${DEBUG_IDENTIFIER}".el"${os_version}".x86_64.rpm + cp "${RPM_FILE}" "${SRC_DIR}" + RPM_DEBUG="${HOME}"/rpmbuild/RPMS/x86_64/apache-cloudberry-db-incubating-debuginfo-"${CBDB_VERSION}"-"${BUILD_NUMBER}""${DEBUG_IDENTIFIER}".el"${os_version}".x86_64.rpm + cp "${RPM_DEBUG}" "${SRC_DIR}" + + # Get package information + echo "Package Information:" + rpm -qip "${RPM_FILE}" + + # Verify critical files in RPM + echo "Verifying critical files in RPM..." + for binary in "bin/postgres" "bin/psql"; do + if ! rpm -qlp "${RPM_FILE}" | grep -q "${binary}$"; then + echo "::error::Critical binary '${binary}' not found in RPM" + exit 1 + fi + done + + # Record checksums + echo "Calculating checksums..." + sha256sum "${RPM_FILE}" | tee -a build-logs/details/checksums.log + sha256sum "${SRC_DIR}"/apache-cloudberry-incubating-src.tgz | tee -a build-logs/details/checksums.log + + echo "Artifacts created and verified successfully" + + } 2>&1 | tee -a build-logs/details/artifact-creation.log + + - name: Run Apache Cloudberry unittest script + if: needs.check-skip.outputs.should_skip != 'true' + env: + SRC_DIR: ${{ github.workspace }} + run: | + set -eo pipefail + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/unittest-cloudberry.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/unittest-cloudberry.sh"; then + echo "::error::Unittest script failed" + exit 1 + fi + + - name: Generate Build Job Summary End + if: always() + run: | + { + echo "## Build Results" + echo "- End Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload build logs + if: needs.check-skip.outputs.should_skip != 'true' + uses: actions/upload-artifact@v4 + with: + name: build-logs-rocky8-${{ env.BUILD_TIMESTAMP }} + path: | + build-logs/ + retention-days: ${{ env.LOG_RETENTION_DAYS }} + + - name: Upload Cloudberry RPM build artifacts + if: needs.check-skip.outputs.should_skip != 'true' + uses: actions/upload-artifact@v4 + with: + name: apache-cloudberry-db-incubating-rpm-build-artifacts-rocky8 + retention-days: ${{ env.LOG_RETENTION_DAYS }} + if-no-files-found: error + path: | + *.rpm + + - name: Upload Cloudberry source build artifacts + if: needs.check-skip.outputs.should_skip != 'true' + uses: actions/upload-artifact@v4 + with: + name: apache-cloudberry-db-incubating-source-build-artifacts-rocky8 + retention-days: ${{ env.LOG_RETENTION_DAYS }} + if-no-files-found: error + path: | + apache-cloudberry-incubating-src.tgz + + ## ====================================================================== + ## Job: rpm-install-test + ## ====================================================================== + + rpm-install-test: + name: RPM Install Test Apache Cloudberry (Rocky 8) + needs: [check-skip, build] + if: | + !cancelled() && + (needs.build.result == 'success' || needs.build.result == 'skipped') && + github.event.inputs.reuse_artifacts_from_run_id == '' + runs-on: ubuntu-22.04 + timeout-minutes: 120 + + container: + image: apache/incubator-cloudberry:cbdb-test-rocky8-latest + options: >- + --user root + -h cdw + -v /usr/share:/host_usr_share + -v /usr/local:/host_usr_local + -v /opt:/host_opt + + steps: + - name: Free Disk Space + if: needs.check-skip.outputs.should_skip != 'true' + run: | + echo "=== Disk space before cleanup ===" + df -h / + + # Remove pre-installed tools from host to free disk space + rm -rf /host_opt/hostedtoolcache || true # GitHub Actions tool cache + rm -rf /host_usr_local/lib/android || true # Android SDK + rm -rf /host_usr_share/dotnet || true # .NET SDK + rm -rf /host_opt/ghc || true # Haskell GHC + rm -rf /host_usr_local/.ghcup || true # Haskell GHCup + rm -rf /host_usr_share/swift || true # Swift + rm -rf /host_usr_local/share/powershell || true # PowerShell + rm -rf /host_usr_local/share/chromium || true # Chromium + rm -rf /host_usr_share/miniconda || true # Miniconda + rm -rf /host_opt/az || true # Azure CLI + rm -rf /host_usr_share/sbt || true # Scala Build Tool + + echo "=== Disk space after cleanup ===" + df -h / + + - name: Skip Check + if: needs.check-skip.outputs.should_skip == 'true' + run: | + echo "RPM install test skipped via CI skip flag" >> "$GITHUB_STEP_SUMMARY" + exit 0 + + - name: Download Cloudberry RPM build artifacts + if: needs.check-skip.outputs.should_skip != 'true' + uses: actions/download-artifact@v4 + with: + name: apache-cloudberry-db-incubating-rpm-build-artifacts-rocky8 + path: ${{ github.workspace }}/rpm_build_artifacts + merge-multiple: false + run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Cloudberry Environment Initialization + if: needs.check-skip.outputs.should_skip != 'true' + env: + LOGS_DIR: install-logs + run: | + set -eo pipefail + if ! su - gpadmin -c "/tmp/init_system.sh"; then + echo "::error::Container initialization failed" + exit 1 + fi + + mkdir -p "${LOGS_DIR}/details" + chown -R gpadmin:gpadmin . + chmod -R 755 . + chmod 777 "${LOGS_DIR}" + + df -kh / + rm -rf /__t/* + df -kh / + + df -h | tee -a "${LOGS_DIR}/details/disk-usage.log" + free -h | tee -a "${LOGS_DIR}/details/memory-usage.log" + + { + echo "=== Environment Information ===" + uname -a + df -h + free -h + env + } | tee -a "${LOGS_DIR}/details/environment.log" + + echo "SRC_DIR=${GITHUB_WORKSPACE}" | tee -a "$GITHUB_ENV" + + - name: Verify RPM artifacts + if: needs.check-skip.outputs.should_skip != 'true' + id: verify-artifacts + run: | + set -eo pipefail + + RPM_FILE=$(ls "${GITHUB_WORKSPACE}"/rpm_build_artifacts/apache-cloudberry-db-incubating-[0-9]*.rpm | grep -v "debuginfo") + if [ ! -f "${RPM_FILE}" ]; then + echo "::error::RPM file not found" + exit 1 + fi + + echo "rpm_file=${RPM_FILE}" >> "$GITHUB_OUTPUT" + + echo "Verifying RPM artifacts..." + { + echo "=== RPM Verification Summary ===" + echo "Timestamp: $(date -u)" + echo "RPM File: ${RPM_FILE}" + + # Get RPM metadata and verify contents + echo "Package Information:" + rpm -qip "${RPM_FILE}" + + # Get key RPM attributes for verification + RPM_VERSION=$(rpm -qp --queryformat "%{VERSION}" "${RPM_FILE}") + RPM_RELEASE=$(rpm -qp --queryformat "%{RELEASE}" "${RPM_FILE}") + echo "version=${RPM_VERSION}" >> "$GITHUB_OUTPUT" + echo "release=${RPM_RELEASE}" >> "$GITHUB_OUTPUT" + + # Verify expected binaries are in the RPM + echo "Verifying critical files in RPM..." + for binary in "bin/postgres" "bin/psql"; do + if ! rpm -qlp "${RPM_FILE}" | grep -q "${binary}$"; then + echo "::error::Critical binary '${binary}' not found in RPM" + exit 1 + fi + done + + echo "RPM Details:" + echo "- Version: ${RPM_VERSION}" + echo "- Release: ${RPM_RELEASE}" + + # Calculate and store checksum + echo "Checksum:" + sha256sum "${RPM_FILE}" + + } 2>&1 | tee -a install-logs/details/rpm-verification.log + + - name: Install Cloudberry RPM + if: success() && needs.check-skip.outputs.should_skip != 'true' + env: + RPM_FILE: ${{ steps.verify-artifacts.outputs.rpm_file }} + RPM_VERSION: ${{ steps.verify-artifacts.outputs.version }} + RPM_RELEASE: ${{ steps.verify-artifacts.outputs.release }} + run: | + set -eo pipefail + + if [ -z "${RPM_FILE}" ]; then + echo "::error::RPM_FILE environment variable is not set" + exit 1 + fi + + { + echo "=== RPM Installation Log ===" + echo "Timestamp: $(date -u)" + echo "RPM File: ${RPM_FILE}" + echo "Version: ${RPM_VERSION}" + echo "Release: ${RPM_RELEASE}" + + # Refresh repository metadata to avoid mirror issues + echo "Refreshing repository metadata..." + dnf clean all + dnf makecache --refresh || dnf makecache + + # Clean install location + rm -rf /usr/local/cloudberry-db + + # Install RPM with retry logic for mirror issues + # Use --releasever=8 to pin to stable Rocky Linux 8 repos (not bleeding-edge 8.10) + echo "Starting installation..." + if ! time dnf install -y --setopt=retries=10 --releasever=8 "${RPM_FILE}"; then + echo "::error::RPM installation failed" + exit 1 + fi + + echo "Installation completed successfully" + rpm -qi apache-cloudberry-db-incubating + echo "Installed files:" + rpm -ql apache-cloudberry-db-incubating + } 2>&1 | tee -a install-logs/details/rpm-installation.log + + - name: Upload install logs + if: needs.check-skip.outputs.should_skip != 'true' + uses: actions/upload-artifact@v4 + with: + name: install-logs-rocky8-${{ needs.build.outputs.build_timestamp }} + path: | + install-logs/ + retention-days: ${{ env.LOG_RETENTION_DAYS }} + + - name: Generate Install Test Job Summary End + if: always() + shell: bash {0} + run: | + { + echo "# Installed Package Summary" + echo "\`\`\`" + + rpm -qi apache-cloudberry-db-incubating + echo "\`\`\`" + } >> "$GITHUB_STEP_SUMMARY" || true + + ## ====================================================================== + ## Job: test + ## ====================================================================== + + test: + name: ${{ matrix.test }} (Rocky 8) + needs: [check-skip, build, prepare-test-matrix] + if: | + !cancelled() && + (needs.build.result == 'success' || needs.build.result == 'skipped') + runs-on: ubuntu-22.04 + timeout-minutes: 120 + # actionlint-allow matrix[*].pg_settings + strategy: + fail-fast: false # Continue with other tests if one fails + matrix: ${{ fromJson(needs.prepare-test-matrix.outputs.test-matrix) }} + + container: + image: apache/incubator-cloudberry:cbdb-build-rocky8-latest + options: >- + --privileged + --user root + --hostname cdw + --shm-size=2gb + --ulimit core=-1 + --cgroupns=host + -v /sys/fs/cgroup:/sys/fs/cgroup:rw + -v /usr/share:/host_usr_share + -v /usr/local:/host_usr_local + -v /opt:/host_opt + + steps: + - name: Free Disk Space + if: needs.check-skip.outputs.should_skip != 'true' + run: | + echo "=== Disk space before cleanup ===" + df -h / + + # Remove pre-installed tools from host to free disk space + rm -rf /host_opt/hostedtoolcache || true # GitHub Actions tool cache + rm -rf /host_usr_local/lib/android || true # Android SDK + rm -rf /host_usr_share/dotnet || true # .NET SDK + rm -rf /host_opt/ghc || true # Haskell GHC + rm -rf /host_usr_local/.ghcup || true # Haskell GHCup + rm -rf /host_usr_share/swift || true # Swift + rm -rf /host_usr_local/share/powershell || true # PowerShell + rm -rf /host_usr_local/share/chromium || true # Chromium + rm -rf /host_usr_share/miniconda || true # Miniconda + rm -rf /host_opt/az || true # Azure CLI + rm -rf /host_usr_share/sbt || true # Scala Build Tool + + echo "=== Disk space after cleanup ===" + df -h / + + - name: Skip Check + if: needs.check-skip.outputs.should_skip == 'true' + run: | + echo "Test ${{ matrix.test }} skipped via CI skip flag" >> "$GITHUB_STEP_SUMMARY" + exit 0 + + - name: Use timestamp from previous job + if: needs.check-skip.outputs.should_skip != 'true' + run: | + echo "Timestamp from output: ${{ needs.build.outputs.build_timestamp }}" + + - name: Cloudberry Environment Initialization + env: + LOGS_DIR: build-logs + run: | + set -eo pipefail + if ! su - gpadmin -c "/tmp/init_system.sh"; then + echo "::error::Container initialization failed" + exit 1 + fi + + mkdir -p "${LOGS_DIR}/details" + chown -R gpadmin:gpadmin . + chmod -R 755 . + chmod 777 "${LOGS_DIR}" + + df -kh / + rm -rf /__t/* + df -kh / + + df -h | tee -a "${LOGS_DIR}/details/disk-usage.log" + free -h | tee -a "${LOGS_DIR}/details/memory-usage.log" + + { + echo "=== Environment Information ===" + uname -a + df -h + free -h + env + } | tee -a "${LOGS_DIR}/details/environment.log" + + echo "SRC_DIR=${GITHUB_WORKSPACE}" | tee -a "$GITHUB_ENV" + + - name: Setup cgroups + if: needs.check-skip.outputs.should_skip != 'true' + shell: bash + run: | + set -uxo pipefail + + if [ "${{ matrix.enable_cgroups }}" = "true" ]; then + + echo "Current mounts:" + mount | grep cgroup + + CGROUP_BASEDIR=/sys/fs/cgroup + + # 1. Basic setup with permissions + sudo chmod -R 777 ${CGROUP_BASEDIR}/ + sudo mkdir -p ${CGROUP_BASEDIR}/gpdb + sudo chmod -R 777 ${CGROUP_BASEDIR}/gpdb + sudo chown -R gpadmin:gpadmin ${CGROUP_BASEDIR}/gpdb + + # 2. Enable controllers + sudo bash -c "echo '+cpu +cpuset +memory +io' > ${CGROUP_BASEDIR}/cgroup.subtree_control" || true + sudo bash -c "echo '+cpu +cpuset +memory +io' > ${CGROUP_BASEDIR}/gpdb/cgroup.subtree_control" || true + + # 3. CPU settings + sudo bash -c "echo 'max 100000' > ${CGROUP_BASEDIR}/gpdb/cpu.max" || true + sudo bash -c "echo '100' > ${CGROUP_BASEDIR}/gpdb/cpu.weight" || true + sudo bash -c "echo '0' > ${CGROUP_BASEDIR}/gpdb/cpu.weight.nice" || true + sudo bash -c "echo 0-$(( $(nproc) - 1 )) > ${CGROUP_BASEDIR}/gpdb/cpuset.cpus" || true + sudo bash -c "echo '0' > ${CGROUP_BASEDIR}/gpdb/cpuset.mems" || true + + # 4. Memory settings + sudo bash -c "echo 'max' > ${CGROUP_BASEDIR}/gpdb/memory.max" || true + sudo bash -c "echo '0' > ${CGROUP_BASEDIR}/gpdb/memory.min" || true + sudo bash -c "echo 'max' > ${CGROUP_BASEDIR}/gpdb/memory.high" || true + + # 5. IO settings + echo "Available block devices:" + lsblk + + sudo bash -c " + if [ -f \${CGROUP_BASEDIR}/gpdb/io.stat ]; then + echo 'Detected IO devices:' + cat \${CGROUP_BASEDIR}/gpdb/io.stat + fi + echo '' > \${CGROUP_BASEDIR}/gpdb/io.max || true + " + + # 6. Fix permissions again after all writes + sudo chmod -R 777 ${CGROUP_BASEDIR}/gpdb + sudo chown -R gpadmin:gpadmin ${CGROUP_BASEDIR}/gpdb + + # 7. Check required files + echo "Checking required files:" + required_files=( + "cgroup.procs" + "cpu.max" + "cpu.pressure" + "cpu.weight" + "cpu.weight.nice" + "cpu.stat" + "cpuset.cpus" + "cpuset.mems" + "cpuset.cpus.effective" + "cpuset.mems.effective" + "memory.current" + "io.max" + ) + + for file in "${required_files[@]}"; do + if [ -f "${CGROUP_BASEDIR}/gpdb/$file" ]; then + echo "✓ $file exists" + ls -l "${CGROUP_BASEDIR}/gpdb/$file" + else + echo "✗ $file missing" + fi + done + + # 8. Test subdirectory creation + echo "Testing subdirectory creation..." + sudo -u gpadmin bash -c " + TEST_DIR=\${CGROUP_BASEDIR}/gpdb/test6448 + if mkdir -p \$TEST_DIR; then + echo 'Created test directory' + sudo chmod -R 777 \$TEST_DIR + if echo \$\$ > \$TEST_DIR/cgroup.procs; then + echo 'Successfully wrote to cgroup.procs' + cat \$TEST_DIR/cgroup.procs + # Move processes back to parent before cleanup + echo \$\$ > \${CGROUP_BASEDIR}/gpdb/cgroup.procs + else + echo 'Failed to write to cgroup.procs' + ls -la \$TEST_DIR/cgroup.procs + fi + ls -la \$TEST_DIR/ + rmdir \$TEST_DIR || { + echo 'Moving all processes to parent before cleanup' + cat \$TEST_DIR/cgroup.procs | while read pid; do + echo \$pid > \${CGROUP_BASEDIR}/gpdb/cgroup.procs 2>/dev/null || true + done + rmdir \$TEST_DIR + } + else + echo 'Failed to create test directory' + fi + " + + # 9. Verify setup as gpadmin user + echo "Testing cgroup access as gpadmin..." + sudo -u gpadmin bash -c " + echo 'Checking mounts...' + mount | grep cgroup + + echo 'Checking /proc/self/mounts...' + cat /proc/self/mounts | grep cgroup + + if ! grep -q cgroup2 /proc/self/mounts; then + echo 'ERROR: cgroup2 mount NOT visible to gpadmin' + exit 1 + fi + echo 'SUCCESS: cgroup2 mount visible to gpadmin' + + if ! [ -w ${CGROUP_BASEDIR}/gpdb ]; then + echo 'ERROR: gpadmin cannot write to gpdb cgroup' + exit 1 + fi + echo 'SUCCESS: gpadmin can write to gpdb cgroup' + + echo 'Verifying key files content:' + echo 'cpu.max:' + cat ${CGROUP_BASEDIR}/gpdb/cpu.max || echo 'Failed to read cpu.max' + echo 'cpuset.cpus:' + cat ${CGROUP_BASEDIR}/gpdb/cpuset.cpus || echo 'Failed to read cpuset.cpus' + echo 'cgroup.subtree_control:' + cat ${CGROUP_BASEDIR}/gpdb/cgroup.subtree_control || echo 'Failed to read cgroup.subtree_control' + " + + # 10. Show final state + echo "Final cgroup state:" + ls -la ${CGROUP_BASEDIR}/gpdb/ + echo "Cgroup setup completed successfully" + else + echo "Cgroup setup skipped" + fi + + - name: "Generate Test Job Summary Start: ${{ matrix.test }}" + if: always() + run: | + { + echo "# Test Job Summary: ${{ matrix.test }} (Rocky 8)" + echo "## Environment" + echo "- Start Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" + + if [[ "${{ needs.check-skip.outputs.should_skip }}" == "true" ]]; then + echo "## Skip Status" + echo "✓ Test execution skipped via CI skip flag" + else + echo "- OS Version: $(cat /etc/redhat-release)" + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Download Cloudberry RPM build artifacts + if: needs.check-skip.outputs.should_skip != 'true' + uses: actions/download-artifact@v4 + with: + name: apache-cloudberry-db-incubating-rpm-build-artifacts-rocky8 + path: ${{ github.workspace }}/rpm_build_artifacts + merge-multiple: false + run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Download Cloudberry Source build artifacts + if: needs.check-skip.outputs.should_skip != 'true' + uses: actions/download-artifact@v4 + with: + name: apache-cloudberry-db-incubating-source-build-artifacts-rocky8 + path: ${{ github.workspace }}/source_build_artifacts + merge-multiple: false + run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Verify downloaded artifacts + if: needs.check-skip.outputs.should_skip != 'true' + id: verify-artifacts + run: | + set -eo pipefail + + SRC_TARBALL_FILE=$(ls "${GITHUB_WORKSPACE}"/source_build_artifacts/apache-cloudberry-incubating-src.tgz) + if [ ! -f "${SRC_TARBALL_FILE}" ]; then + echo "::error::SRC TARBALL file not found" + exit 1 + fi + + echo "src_tarball_file=${SRC_TARBALL_FILE}" >> "$GITHUB_OUTPUT" + + echo "Verifying SRC TARBALL artifacts..." + { + echo "=== SRC TARBALL Verification Summary ===" + echo "Timestamp: $(date -u)" + echo "SRC TARBALL File: ${SRC_TARBALL_FILE}" + + # Calculate and store checksum + echo "Checksum:" + sha256sum "${SRC_TARBALL_FILE}" + + } 2>&1 | tee -a build-logs/details/src-tarball-verification.log + + RPM_FILE=$(ls "${GITHUB_WORKSPACE}"/rpm_build_artifacts/apache-cloudberry-db-incubating-[0-9]*.rpm | grep -v "debuginfo") + if [ ! -f "${RPM_FILE}" ]; then + echo "::error::RPM file not found" + exit 1 + fi + + echo "rpm_file=${RPM_FILE}" >> "$GITHUB_OUTPUT" + + echo "Verifying RPM artifacts..." + { + echo "=== RPM Verification Summary ===" + echo "Timestamp: $(date -u)" + echo "RPM File: ${RPM_FILE}" + + # Get RPM metadata and verify contents + echo "Package Information:" + rpm -qip "${RPM_FILE}" + + # Get key RPM attributes for verification + RPM_VERSION=$(rpm -qp --queryformat "%{VERSION}" "${RPM_FILE}") + RPM_RELEASE=$(rpm -qp --queryformat "%{RELEASE}" "${RPM_FILE}") + echo "version=${RPM_VERSION}" >> "$GITHUB_OUTPUT" + echo "release=${RPM_RELEASE}" >> "$GITHUB_OUTPUT" + + # Verify expected binaries are in the RPM + echo "Verifying critical files in RPM..." + for binary in "bin/postgres" "bin/psql"; do + if ! rpm -qlp "${RPM_FILE}" | grep -q "${binary}$"; then + echo "::error::Critical binary '${binary}' not found in RPM" + exit 1 + fi + done + + echo "RPM Details:" + echo "- Version: ${RPM_VERSION}" + echo "- Release: ${RPM_RELEASE}" + + # Calculate and store checksum + echo "Checksum:" + sha256sum "${RPM_FILE}" + + } 2>&1 | tee -a build-logs/details/rpm-verification.log + + - name: Install Cloudberry RPM + if: success() && needs.check-skip.outputs.should_skip != 'true' + env: + RPM_FILE: ${{ steps.verify-artifacts.outputs.rpm_file }} + RPM_VERSION: ${{ steps.verify-artifacts.outputs.version }} + RPM_RELEASE: ${{ steps.verify-artifacts.outputs.release }} + run: | + set -eo pipefail + + if [ -z "${RPM_FILE}" ]; then + echo "::error::RPM_FILE environment variable is not set" + exit 1 + fi + + { + echo "=== RPM Installation Log ===" + echo "Timestamp: $(date -u)" + echo "RPM File: ${RPM_FILE}" + echo "Version: ${RPM_VERSION}" + echo "Release: ${RPM_RELEASE}" + + # Refresh repository metadata to avoid mirror issues + echo "Refreshing repository metadata..." + dnf clean all + dnf makecache --refresh || dnf makecache + + # Clean install location + rm -rf /usr/local/cloudberry-db + + # Install RPM with retry logic for mirror issues + # Use --releasever=8 to pin to stable Rocky Linux 8 repos (not bleeding-edge 8.10) + echo "Starting installation..." + if ! time dnf install -y --setopt=retries=10 --releasever=8 "${RPM_FILE}"; then + echo "::error::RPM installation failed" + exit 1 + fi + + echo "Installation completed successfully" + rpm -qi apache-cloudberry-db-incubating + } 2>&1 | tee -a build-logs/details/rpm-installation.log + + # Clean up downloaded RPM artifacts to free disk space + echo "=== Disk space before RPM cleanup ===" + echo "Human readable:" + df -kh / + echo "Exact KB:" + df -k / + echo "RPM artifacts size:" + du -sh "${GITHUB_WORKSPACE}"/rpm_build_artifacts || true + echo "Cleaning up RPM artifacts to free disk space..." + rm -rf "${GITHUB_WORKSPACE}"/rpm_build_artifacts + echo "=== Disk space after RPM cleanup ===" + echo "Human readable:" + df -kh / + echo "Exact KB:" + df -k / + + - name: Extract source tarball + if: success() && needs.check-skip.outputs.should_skip != 'true' + env: + SRC_TARBALL_FILE: ${{ steps.verify-artifacts.outputs.src_tarball_file }} + SRC_DIR: ${{ github.workspace }} + run: | + set -eo pipefail + + { + echo "=== Source Extraction Log ===" + echo "Timestamp: $(date -u)" + + echo "Starting extraction..." + if ! time tar zxf "${SRC_TARBALL_FILE}" -C "${SRC_DIR}"/.. ; then + echo "::error::Source extraction failed" + exit 1 + fi + + echo "Extraction completed successfully" + echo "Extracted contents:" + ls -la "${SRC_DIR}/../cloudberry" + echo "Directory size:" + du -sh "${SRC_DIR}/../cloudberry" + } 2>&1 | tee -a build-logs/details/source-extraction.log + + # Clean up source tarball to free disk space + echo "=== Disk space before source tarball cleanup ===" + echo "Human readable:" + df -kh / + echo "Exact KB:" + df -k / + echo "Source tarball artifacts size:" + du -sh "${GITHUB_WORKSPACE}"/source_build_artifacts || true + echo "Cleaning up source tarball to free disk space..." + rm -rf "${GITHUB_WORKSPACE}"/source_build_artifacts + echo "=== Disk space after source tarball cleanup ===" + echo "Human readable:" + df -kh / + echo "Exact KB:" + df -k / + + - name: Create Apache Cloudberry demo cluster + if: success() && needs.check-skip.outputs.should_skip != 'true' + env: + SRC_DIR: ${{ github.workspace }} + run: | + set -eo pipefail + + { + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && NUM_PRIMARY_MIRROR_PAIRS='${{ matrix.num_primary_mirror_pairs }}' SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh"; then + echo "::error::Demo cluster creation failed" + exit 1 + fi + + } 2>&1 | tee -a build-logs/details/create-cloudberry-demo-cluster.log + + - name: "Run Tests: ${{ matrix.test }}" + if: success() && needs.check-skip.outputs.should_skip != 'true' + env: + SRC_DIR: ${{ github.workspace }} + shell: bash {0} + run: | + set -o pipefail + + # Initialize test status + overall_status=0 + + # Create logs directory structure + mkdir -p build-logs/details + + # Core file config + mkdir -p "/tmp/cloudberry-cores" + chmod 1777 "/tmp/cloudberry-cores" + sysctl -w kernel.core_pattern="/tmp/cloudberry-cores/core-%e-%s-%u-%g-%p-%t" + sysctl kernel.core_pattern + su - gpadmin -c "ulimit -c" + + # WARNING: PostgreSQL Settings + # When adding new pg_settings key/value pairs: + # 1. Add a new check below for the setting + # 2. Follow the same pattern as optimizer + # 3. Update matrix entries to include the new setting + + # Set PostgreSQL options if defined + PG_OPTS="" + if [[ "${{ matrix.pg_settings.optimizer != '' }}" == "true" ]]; then + PG_OPTS="$PG_OPTS -c optimizer=${{ matrix.pg_settings.optimizer }}" + fi + + if [[ "${{ matrix.pg_settings.default_table_access_method != '' }}" == "true" ]]; then + PG_OPTS="$PG_OPTS -c default_table_access_method=${{ matrix.pg_settings.default_table_access_method }}" + fi + + # Read configs into array + IFS=' ' read -r -a configs <<< "${{ join(matrix.make_configs, ' ') }}" + + echo "=== Starting test execution for ${{ matrix.test }} ===" + echo "Number of configurations to execute: ${#configs[@]}" + echo "" + + # Execute each config separately + for ((i=0; i<${#configs[@]}; i++)); do + config="${configs[$i]}" + IFS=':' read -r dir target <<< "$config" + + echo "=== Executing configuration $((i+1))/${#configs[@]} ===" + echo "Make command: make -C $dir $target" + echo "Environment:" + echo "- PGOPTIONS: ${PG_OPTS}" + + # Create unique log file for this configuration + config_log="build-logs/details/make-${{ matrix.test }}-config$i.log" + + # Clean up any existing core files + echo "Cleaning up existing core files..." + rm -f /tmp/cloudberry-cores/core-* + + # Execute test script with proper environment setup + if ! time su - gpadmin -c "cd ${SRC_DIR} && \ + MAKE_NAME='${{ matrix.test }}-config$i' \ + MAKE_TARGET='$target' \ + MAKE_DIRECTORY='-C $dir' \ + PGOPTIONS='${PG_OPTS}' \ + SRC_DIR='${SRC_DIR}' \ + ${SRC_DIR}/devops/build/automation/cloudberry/scripts/test-cloudberry.sh" \ + 2>&1 | tee "$config_log"; then + echo "::warning::Test execution failed for configuration $((i+1)): make -C $dir $target" + overall_status=1 + fi + + # Check for results directory + results_dir="${dir}/results" + + if [[ -d "$results_dir" ]]; then + echo "-----------------------------------------" | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log + echo "Found results directory: $results_dir" | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log + echo "Contents of results directory:" | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log + + find "$results_dir" -type f -ls >> "$log_file" 2>&1 | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log + echo "-----------------------------------------" | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log + else + echo "-----------------------------------------" + echo "Results directory $results_dir does not exit" + echo "-----------------------------------------" + fi + + # Analyze any core files generated by this test configuration + echo "Analyzing core files for configuration ${{ matrix.test }}-config$i..." + test_id="${{ matrix.test }}-config$i" + + # List the cores directory + echo "-----------------------------------------" + echo "Cores directory: /tmp/cloudberry-cores" + echo "Contents of cores directory:" + ls -Rl "/tmp/cloudberry-cores" + echo "-----------------------------------------" + + "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/analyze_core_dumps.sh "$test_id" + core_analysis_rc=$? + case "$core_analysis_rc" in + 0) echo "No core dumps found for this configuration" ;; + 1) echo "Core dumps were found and analyzed successfully" ;; + 2) echo "::warning::Issues encountered during core dump analysis" ;; + *) echo "::error::Unexpected return code from core dump analysis: $core_analysis_rc" ;; + esac + + echo "Log file: $config_log" + echo "=== End configuration $((i+1)) execution ===" + echo "" + done + + echo "=== Test execution completed ===" + echo "Log files:" + ls -l build-logs/details/ + + # Store number of configurations for parsing step + echo "NUM_CONFIGS=${#configs[@]}" >> "$GITHUB_ENV" + + # Report overall status + if [ $overall_status -eq 0 ]; then + echo "All test executions completed successfully" + else + echo "::warning::Some test executions failed, check individual logs for details" + fi + + exit $overall_status + + - name: "Parse Test Results: ${{ matrix.test }}" + id: test-results + if: always() && needs.check-skip.outputs.should_skip != 'true' + env: + SRC_DIR: ${{ github.workspace }} + shell: bash {0} + run: | + set -o pipefail + + overall_status=0 + + # Get configs array to create context for results + IFS=' ' read -r -a configs <<< "${{ join(matrix.make_configs, ' ') }}" + + echo "=== Starting results parsing for ${{ matrix.test }} ===" + echo "Number of configurations to parse: ${#configs[@]}" + echo "" + + # Parse each configuration's results independently + for ((i=0; i "test_results.$i.txt" + overall_status=1 + continue + fi + + # Parse this configuration's results + + MAKE_NAME="${{ matrix.test }}-config$i" \ + "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/parse-test-results.sh "$config_log" + status_code=$? + + { + echo "SUITE_NAME=${{ matrix.test }}" + echo "DIR=${dir}" + echo "TARGET=${target}" + } >> test_results.txt + + # Process return code + case $status_code in + 0) # All tests passed + echo "All tests passed successfully" + if [ -f test_results.txt ]; then + (echo "MAKE_COMMAND=\"make -C $dir $target\""; cat test_results.txt) | tee "test_results.${{ matrix.test }}.$i.txt" + rm test_results.txt + fi + ;; + 1) # Tests failed but parsed successfully + echo "Test failures detected but properly parsed" + if [ -f test_results.txt ]; then + (echo "MAKE_COMMAND=\"make -C $dir $target\""; cat test_results.txt) | tee "test_results.${{ matrix.test }}.$i.txt" + rm test_results.txt + fi + overall_status=1 + ;; + 2) # Parse error or missing file + echo "::warning::Could not parse test results properly for configuration $((i+1))" + { + echo "MAKE_COMMAND=\"make -C $dir $target\"" + echo "STATUS=parse_error" + echo "TOTAL_TESTS=0" + echo "FAILED_TESTS=0" + echo "PASSED_TESTS=0" + echo "IGNORED_TESTS=0" + } | tee "test_results.${{ matrix.test }}.$i.txt" + overall_status=1 + ;; + *) # Unexpected error + echo "::warning::Unexpected error during test results parsing for configuration $((i+1))" + { + echo "MAKE_COMMAND=\"make -C $dir $target\"" + echo "STATUS=unknown_error" + echo "TOTAL_TESTS=0" + echo "FAILED_TESTS=0" + echo "PASSED_TESTS=0" + echo "IGNORED_TESTS=0" + } | tee "test_results.${{ matrix.test }}.$i.txt" + overall_status=1 + ;; + esac + + echo "Results stored in test_results.$i.txt" + echo "=== End parsing for configuration $((i+1)) ===" + echo "" + done + + # Report status of results files + echo "=== Results file status ===" + echo "Generated results files:" + for ((i=0; i> "$GITHUB_STEP_SUMMARY" || true + + - name: Upload test logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-logs-${{ matrix.test }}-rocky8-${{ needs.build.outputs.build_timestamp }} + path: | + build-logs/ + retention-days: ${{ env.LOG_RETENTION_DAYS }} + + - name: Upload Test Metadata + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-metadata-${{ matrix.test }}-rocky8 + path: | + test_results*.txt + retention-days: ${{ env.LOG_RETENTION_DAYS }} + + - name: Upload test results files + uses: actions/upload-artifact@v4 + with: + name: results-${{ matrix.test }}-rocky8-${{ needs.build.outputs.build_timestamp }} + path: | + **/regression.out + **/regression.diffs + **/results/ + retention-days: ${{ env.LOG_RETENTION_DAYS }} + + - name: Upload test regression logs + if: failure() || cancelled() + uses: actions/upload-artifact@v4 + with: + name: regression-logs-${{ matrix.test }}-rocky8-${{ needs.build.outputs.build_timestamp }} + path: | + **/regression.out + **/regression.diffs + **/results/ + gpAux/gpdemo/datadirs/standby/log/ + gpAux/gpdemo/datadirs/qddir/demoDataDir-1/log/ + gpAux/gpdemo/datadirs/dbfast1/demoDataDir0/log/ + gpAux/gpdemo/datadirs/dbfast2/demoDataDir1/log/ + gpAux/gpdemo/datadirs/dbfast3/demoDataDir2/log/ + gpAux/gpdemo/datadirs/dbfast_mirror1/demoDataDir0/log/ + gpAux/gpdemo/datadirs/dbfast_mirror2/demoDataDir1/log/ + gpAux/gpdemo/datadirs/dbfast_mirror3/demoDataDir2/log/ + retention-days: ${{ env.LOG_RETENTION_DAYS }} + + ## ====================================================================== + ## Job: report + ## ====================================================================== + + report: + name: Generate Apache Cloudberry Build Report (Rocky 8) + needs: [check-skip, build, prepare-test-matrix, rpm-install-test, test] + if: always() + runs-on: ubuntu-22.04 + steps: + - name: Generate Final Report + run: | + { + echo "# Apache Cloudberry Build Pipeline Report (Rocky 8)" + + if [[ "${{ needs.check-skip.outputs.should_skip }}" == "true" ]]; then + echo "## CI Skip Status" + echo "✅ CI checks skipped via skip flag" + echo "- Completion Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" + else + echo "## Job Status" + echo "- Build Job: ${{ needs.build.result }}" + echo "- Test Job: ${{ needs.test.result }}" + echo "- Completion Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" + + if [[ "${{ needs.build.result }}" == "success" && "${{ needs.test.result }}" == "success" ]]; then + echo "✅ Pipeline completed successfully" + else + echo "⚠️ Pipeline completed with failures" + + if [[ "${{ needs.build.result }}" != "success" ]]; then + echo "### Build Job Failure" + echo "Check build logs for details" + fi + + if [[ "${{ needs.test.result }}" != "success" ]]; then + echo "### Test Job Failure" + echo "Check test logs and regression files for details" + fi + fi + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Notify on failure + if: | + needs.check-skip.outputs.should_skip != 'true' && + (needs.build.result != 'success' || needs.test.result != 'success') + run: | + echo "::error::Build/Test pipeline failed! Check job summaries and logs for details" + echo "Timestamp: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" + echo "Build Result: ${{ needs.build.result }}" + echo "Test Result: ${{ needs.test.result }}" From 8d8a3f9684d194fc841c2b4e44550331da0a8266 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Tue, 20 Jan 2026 14:12:32 +0800 Subject: [PATCH 19/22] Fix: make motion_socket test Python 3.6 compatible The `capture_output` argument for `subprocess.run` is not available in Python 3.6, which is the default on Rocky Linux 8. This commit replaces it with `stdout=subprocess.PIPE` and `stderr= subprocess.PIPE` to ensure backward compatibility. This enables `ic-good-opt-*` tests to pass on Rocky 8 environments. See: https://github.com/apache/cloudberry/issues/1538 --- src/test/regress/expected/motion_socket.out | 2 +- src/test/regress/sql/motion_socket.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/regress/expected/motion_socket.out b/src/test/regress/expected/motion_socket.out index 62a70660f59..baca679ecf3 100644 --- a/src/test/regress/expected/motion_socket.out +++ b/src/test/regress/expected/motion_socket.out @@ -48,7 +48,7 @@ for pid in pids_to_check: # We check count of those connections which have not been established. # Use the regex for example: "TCP :\d+ .*" (without '->') lsof_ret = subprocess.run(["lsof", "-i", "-nP", "-a", "-p", str(pid)], - capture_output=True, check=True).stdout + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True).stdout plpy.info( f'Checking postgres backend {pid}, ' \ f'lsof output:\n{os.linesep.join(map(str, lsof_ret.splitlines()))}') diff --git a/src/test/regress/sql/motion_socket.sql b/src/test/regress/sql/motion_socket.sql index 6d1a08973e0..be5cd21b1cd 100644 --- a/src/test/regress/sql/motion_socket.sql +++ b/src/test/regress/sql/motion_socket.sql @@ -50,7 +50,7 @@ for pid in pids_to_check: # We check count of those connections which have not been established. # Use the regex for example: "TCP :\d+ .*" (without '->') lsof_ret = subprocess.run(["lsof", "-i", "-nP", "-a", "-p", str(pid)], - capture_output=True, check=True).stdout + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True).stdout plpy.info( f'Checking postgres backend {pid}, ' \ f'lsof output:\n{os.linesep.join(map(str, lsof_ret.splitlines()))}') From 29019ad6eb12e27647b2f17a03f4fbbc46f01b5d Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Tue, 20 Jan 2026 14:59:57 +0800 Subject: [PATCH 20/22] Fix: pgcrypto regression test failures on Rocky 8 This commit fixes two issues preventing the `ic-contrib` regression tests from passing on Rocky Linux 8 environments (FIPS enabled). 1. Fixed regex matching for "Some PX error": The previous `init_file` rule missed a space in the error message pattern (`ERROR: Cannot use...`), causing FIPS error masking to fail. Added the missing space to correctly match the output. 2. Masked ephemeral line numbers: Different compilation environments (Rocky 8 vs 9) produce different line number references in error messages (e.g., pgcrypto.c:213 vs 215). Added `matchsubs` rules to mask these line numbers with `(pgcrypto.c:XXX)` and updated `expected/fips_2.out` to match, ensuring consistent test results across platforms. Changes: * Modified src/test/regress/init_file * Updated contrib/pgcrypto/expected/fips_2.out See: https://github.com/apache/cloudberry/issues/1539 --- contrib/pgcrypto/expected/fips_2.out | 12 ++++++------ src/test/regress/init_file | 10 ++++++++-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/contrib/pgcrypto/expected/fips_2.out b/contrib/pgcrypto/expected/fips_2.out index 51957b898da..19ba00f8bf8 100644 --- a/contrib/pgcrypto/expected/fips_2.out +++ b/contrib/pgcrypto/expected/fips_2.out @@ -55,7 +55,7 @@ SELECT 'Test gen_salt : EXPECTED FAIL FIPS' as comment; (1 row) UPDATE fipstest SET salt = gen_salt('md5'); -ERROR: requested functionality not allowed in FIPS mode (pgcrypto.c:213) +ERROR: requested functionality not allowed in FIPS mode (pgcrypto.c:XXX) SELECT 'Test crypt : EXPECTED FAIL FIPS' as comment; comment --------------------------------- @@ -63,9 +63,9 @@ SELECT 'Test crypt : EXPECTED FAIL FIPS' as comment; (1 row) UPDATE fipstest SET res = crypt(data, salt); -ERROR: requested functionality not allowed in FIPS mode (pgcrypto.c:266) +ERROR: requested functionality not allowed in FIPS mode (pgcrypto.c:XXX) SELECT res = crypt(data, res) AS "worked" FROM fipstest; -ERROR: requested functionality not allowed in FIPS mode (pgcrypto.c:266) +ERROR: requested functionality not allowed in FIPS mode (pgcrypto.c:XXX) SELECT 'Test pgp : EXPECTED PASS' as comment; comment -------------------------- @@ -73,7 +73,7 @@ SELECT 'Test pgp : EXPECTED PASS' as comment; (1 row) select pgp_sym_decrypt(pgp_sym_encrypt('santa clause', 'mypass', 'cipher-algo=aes256'), 'mypass'); -ERROR: requested functionality not allowed in FIPS mode (openssl.c:772) +ERROR: requested functionality not allowed in FIPS mode (openssl.c:XXX) SELECT 'Test pgp : EXPECTED FAIL FIPS' as comment; comment ------------------------------- @@ -89,7 +89,7 @@ SELECT 'Test raw encrypt : EXPECTED PASS' as comment; (1 row) SELECT encrypt('santa claus', 'mypass', 'aes') as raw_aes; -ERROR: requested functionality not allowed in FIPS mode (openssl.c:772) +ERROR: requested functionality not allowed in FIPS mode (openssl.c:XXX) SELECT 'Test raw encrypt : EXPECTED FAIL FIPS' as comment; comment --------------------------------------- @@ -97,5 +97,5 @@ SELECT 'Test raw encrypt : EXPECTED FAIL FIPS' as comment; (1 row) SELECT encrypt('santa claus', 'mypass', 'bf') as raw_blowfish; -ERROR: requested functionality not allowed in FIPS mode (openssl.c:772) +ERROR: requested functionality not allowed in FIPS mode (openssl.c:XXX) DROP TABLE fipstest; diff --git a/src/test/regress/init_file b/src/test/regress/init_file index 728f6d8ea42..3220ef7bfa6 100644 --- a/src/test/regress/init_file +++ b/src/test/regress/init_file @@ -134,8 +134,14 @@ s/ERROR: FIPS enabled OpenSSL is required for strict FIPS mode .*/ERROR: FIPS # Mask out OpenSSL behavior change in different version m/ERROR: Cannot use "md5": No such hash algorithm/ s/ERROR: Cannot use "md5": No such hash algorithm/ERROR: Cannot use "md5": / -m/ERROR: Cannot use "md5": Some PX error \(not specified\)/ -s/ERROR: Cannot use "md5": Some PX error \(not specified\)/ERROR: Cannot use "md5": / +m/ERROR: Cannot use "md5": Some PX error \(not specified\)/ +s/ERROR: Cannot use "md5": Some PX error \(not specified\)/ERROR: Cannot use "md5": / + +# Mask out FIPS error line numbers +m/ERROR: requested functionality not allowed in FIPS mode \(pgcrypto.c:\d+\)/ +s/ERROR: requested functionality not allowed in FIPS mode \(pgcrypto.c:\d+\)/ERROR: requested functionality not allowed in FIPS mode (pgcrypto.c:XXX)/ +m/ERROR: requested functionality not allowed in FIPS mode \(openssl.c:\d+\)/ +s/ERROR: requested functionality not allowed in FIPS mode \(openssl.c:\d+\)/ERROR: requested functionality not allowed in FIPS mode (openssl.c:XXX)/ # Mask out gp_execution_segment() m/One-Time Filter: \(gp_execution_segment\(\) = \d+/ From eeff10abc76c06e15c18ab628b7fede24ee6e91b Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Sat, 31 Jan 2026 20:05:21 +0800 Subject: [PATCH 21/22] CI: remove pxf_fdw from CI workflow Similar to PR #1549, we will keep the pxf_fdw in `apache/cloudberry-pxf` as the latest version, so remove pxf_fdw test from this CI workflow file. --- .github/workflows/build-cloudberry-rocky8.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-cloudberry-rocky8.yml b/.github/workflows/build-cloudberry-rocky8.yml index 5028af1315e..2abf88060e3 100644 --- a/.github/workflows/build-cloudberry-rocky8.yml +++ b/.github/workflows/build-cloudberry-rocky8.yml @@ -316,7 +316,6 @@ jobs: }, {"test":"ic-gpcontrib", "make_configs":["gpcontrib/orafce:installcheck", - "gpcontrib/pxf_fdw:installcheck", "gpcontrib/zstd:installcheck", "gpcontrib/gp_sparse_vector:installcheck", "gpcontrib/gp_toolkit:installcheck"] From 2cc5674ea6e3a5cc85ff9101daf08b8dee5d533e Mon Sep 17 00:00:00 2001 From: reshke Date: Wed, 4 Feb 2026 08:03:25 +0500 Subject: [PATCH 22/22] Cherry-pick of CVE fix: Fix privilege checks in pg_stats_ext and pg_stats_ext_exprs. (#1551) This is https://github.com/postgres/postgres/commit/c3425383ba67ae6ecaddc8896025a91faadb430a commit, applied to Cloudberry. There was no issues in apply, only changes are to gporca expected output original commit message follows === The catalog view pg_stats_ext fails to consider privileges for expression statistics. The catalog view pg_stats_ext_exprs fails to consider privileges and row-level security policies. To fix, restrict the data in these views to table owners or roles that inherit privileges of the table owner. It may be possible to apply less restrictive privilege checks in some cases, but that is left as a future exercise. Furthermore, for pg_stats_ext_exprs, do not return data for tables with row-level security enabled, as is already done for pg_stats_ext. On the back-branches, a fix-CVE-2024-4317.sql script is provided that will install into the "share" directory. This file can be used to apply the fix to existing clusters. Bumps catversion on 'master' branch only. Reported-by: Lukas Fittl Reviewed-by: Noah Misch, Tomas Vondra, Tom Lane Security: CVE-2024-4317 Backpatch-through: 14 --- doc/src/sgml/catalogs.sgml | 7 +- src/backend/catalog/Makefile | 3 +- src/backend/catalog/fix-CVE-2024-4317.sql | 115 ++++++++++++++++++ src/backend/catalog/system_views.sql | 11 +- src/test/regress/expected/rules.out | 8 +- src/test/regress/expected/stats_ext.out | 43 +++++++ .../regress/expected/stats_ext_optimizer.out | 43 +++++++ src/test/regress/sql/stats_ext.sql | 27 ++++ 8 files changed, 240 insertions(+), 17 deletions(-) create mode 100644 src/backend/catalog/fix-CVE-2024-4317.sql diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 27b1f16b6a8..533856b12d7 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -7480,8 +7480,7 @@ SCRAM-SHA-256$<iteration count>:&l is a publicly readable view on pg_statistic_ext_data (after joining with pg_statistic_ext) that only exposes - information about those tables and columns that are readable by the - current user. + information about tables the current user owns. @@ -12925,7 +12924,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx and pg_statistic_ext_data catalogs. This view allows access only to rows of pg_statistic_ext and pg_statistic_ext_data - that correspond to tables the user has permission to read, and therefore + that correspond to tables the user owns, and therefore it is safe to allow public read access to this view. @@ -13125,7 +13124,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx and pg_statistic_ext_data catalogs. This view allows access only to rows of pg_statistic_ext and pg_statistic_ext_data - that correspond to tables the user has permission to read, and therefore + that correspond to tables the user owns, and therefore it is safe to allow public read access to this view. diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile index f1a0251e1cb..7010ee132ef 100644 --- a/src/backend/catalog/Makefile +++ b/src/backend/catalog/Makefile @@ -205,6 +205,7 @@ endif $(INSTALL_DATA) $(srcdir)/information_schema.sql '$(DESTDIR)$(datadir)/information_schema.sql' $(INSTALL_DATA) $(call vpathsearch,cdb_schema.sql) '$(DESTDIR)$(datadir)/cdb_init.d/cdb_schema.sql' $(INSTALL_DATA) $(srcdir)/sql_features.txt '$(DESTDIR)$(datadir)/sql_features.txt' + $(INSTALL_DATA) $(srcdir)/fix-CVE-2024-4317.sql '$(DESTDIR)$(datadir)/fix-CVE-2024-4317.sql' installdirs: $(MKDIR_P) '$(DESTDIR)$(datadir)' @@ -212,7 +213,7 @@ installdirs: .PHONY: uninstall-data uninstall-data: - rm -f $(addprefix '$(DESTDIR)$(datadir)'/, postgres.bki system_constraints.sql system_functions.sql system_views.sql system_views_gp_summary.sql information_schema.sql cdb_init.d/cdb_schema.sql cdb_init.d/gp_toolkit.sql sql_features.txt) + rm -f $(addprefix '$(DESTDIR)$(datadir)'/, postgres.bki system_constraints.sql system_functions.sql system_views.sql system_views_gp_summary.sql information_schema.sql cdb_init.d/cdb_schema.sql cdb_init.d/gp_toolkit.sql sql_features.txt fix-CVE-2024-4317.sql) ifeq ($(USE_INTERNAL_FTS_FOUND), false) rm -f $(addprefix '$(DESTDIR)$(datadir)'/, external_fts.sql) endif diff --git a/src/backend/catalog/fix-CVE-2024-4317.sql b/src/backend/catalog/fix-CVE-2024-4317.sql new file mode 100644 index 00000000000..9e78c44c410 --- /dev/null +++ b/src/backend/catalog/fix-CVE-2024-4317.sql @@ -0,0 +1,115 @@ +/* + * fix-CVE-2024-4317.sql + * + * Copyright (c) 2024, PostgreSQL Global Development Group + * + * src/backend/catalog/fix-CVE-2024-4317.sql + * + * This file should be run in every database in the cluster to address + * CVE-2024-4317. + */ + +SET search_path = pg_catalog; + +CREATE OR REPLACE VIEW pg_stats_ext WITH (security_barrier) AS + SELECT cn.nspname AS schemaname, + c.relname AS tablename, + sn.nspname AS statistics_schemaname, + s.stxname AS statistics_name, + pg_get_userbyid(s.stxowner) AS statistics_owner, + ( SELECT array_agg(a.attname ORDER BY a.attnum) + FROM unnest(s.stxkeys) k + JOIN pg_attribute a + ON (a.attrelid = s.stxrelid AND a.attnum = k) + ) AS attnames, + pg_get_statisticsobjdef_expressions(s.oid) as exprs, + s.stxkind AS kinds, + sd.stxdndistinct AS n_distinct, + sd.stxddependencies AS dependencies, + m.most_common_vals, + m.most_common_val_nulls, + m.most_common_freqs, + m.most_common_base_freqs + FROM pg_statistic_ext s JOIN pg_class c ON (c.oid = s.stxrelid) + JOIN pg_statistic_ext_data sd ON (s.oid = sd.stxoid) + LEFT JOIN pg_namespace cn ON (cn.oid = c.relnamespace) + LEFT JOIN pg_namespace sn ON (sn.oid = s.stxnamespace) + LEFT JOIN LATERAL + ( SELECT array_agg(values) AS most_common_vals, + array_agg(nulls) AS most_common_val_nulls, + array_agg(frequency) AS most_common_freqs, + array_agg(base_frequency) AS most_common_base_freqs + FROM pg_mcv_list_items(sd.stxdmcv) + ) m ON sd.stxdmcv IS NOT NULL + WHERE pg_has_role(c.relowner, 'USAGE') + AND (c.relrowsecurity = false OR NOT row_security_active(c.oid)); + +CREATE OR REPLACE VIEW pg_stats_ext_exprs WITH (security_barrier) AS + SELECT cn.nspname AS schemaname, + c.relname AS tablename, + sn.nspname AS statistics_schemaname, + s.stxname AS statistics_name, + pg_get_userbyid(s.stxowner) AS statistics_owner, + stat.expr, + (stat.a).stanullfrac AS null_frac, + (stat.a).stawidth AS avg_width, + (stat.a).stadistinct AS n_distinct, + (CASE + WHEN (stat.a).stakind1 = 1 THEN (stat.a).stavalues1 + WHEN (stat.a).stakind2 = 1 THEN (stat.a).stavalues2 + WHEN (stat.a).stakind3 = 1 THEN (stat.a).stavalues3 + WHEN (stat.a).stakind4 = 1 THEN (stat.a).stavalues4 + WHEN (stat.a).stakind5 = 1 THEN (stat.a).stavalues5 + END) AS most_common_vals, + (CASE + WHEN (stat.a).stakind1 = 1 THEN (stat.a).stanumbers1 + WHEN (stat.a).stakind2 = 1 THEN (stat.a).stanumbers2 + WHEN (stat.a).stakind3 = 1 THEN (stat.a).stanumbers3 + WHEN (stat.a).stakind4 = 1 THEN (stat.a).stanumbers4 + WHEN (stat.a).stakind5 = 1 THEN (stat.a).stanumbers5 + END) AS most_common_freqs, + (CASE + WHEN (stat.a).stakind1 = 2 THEN (stat.a).stavalues1 + WHEN (stat.a).stakind2 = 2 THEN (stat.a).stavalues2 + WHEN (stat.a).stakind3 = 2 THEN (stat.a).stavalues3 + WHEN (stat.a).stakind4 = 2 THEN (stat.a).stavalues4 + WHEN (stat.a).stakind5 = 2 THEN (stat.a).stavalues5 + END) AS histogram_bounds, + (CASE + WHEN (stat.a).stakind1 = 3 THEN (stat.a).stanumbers1[1] + WHEN (stat.a).stakind2 = 3 THEN (stat.a).stanumbers2[1] + WHEN (stat.a).stakind3 = 3 THEN (stat.a).stanumbers3[1] + WHEN (stat.a).stakind4 = 3 THEN (stat.a).stanumbers4[1] + WHEN (stat.a).stakind5 = 3 THEN (stat.a).stanumbers5[1] + END) correlation, + (CASE + WHEN (stat.a).stakind1 = 4 THEN (stat.a).stavalues1 + WHEN (stat.a).stakind2 = 4 THEN (stat.a).stavalues2 + WHEN (stat.a).stakind3 = 4 THEN (stat.a).stavalues3 + WHEN (stat.a).stakind4 = 4 THEN (stat.a).stavalues4 + WHEN (stat.a).stakind5 = 4 THEN (stat.a).stavalues5 + END) AS most_common_elems, + (CASE + WHEN (stat.a).stakind1 = 4 THEN (stat.a).stanumbers1 + WHEN (stat.a).stakind2 = 4 THEN (stat.a).stanumbers2 + WHEN (stat.a).stakind3 = 4 THEN (stat.a).stanumbers3 + WHEN (stat.a).stakind4 = 4 THEN (stat.a).stanumbers4 + WHEN (stat.a).stakind5 = 4 THEN (stat.a).stanumbers5 + END) AS most_common_elem_freqs, + (CASE + WHEN (stat.a).stakind1 = 5 THEN (stat.a).stanumbers1 + WHEN (stat.a).stakind2 = 5 THEN (stat.a).stanumbers2 + WHEN (stat.a).stakind3 = 5 THEN (stat.a).stanumbers3 + WHEN (stat.a).stakind4 = 5 THEN (stat.a).stanumbers4 + WHEN (stat.a).stakind5 = 5 THEN (stat.a).stanumbers5 + END) AS elem_count_histogram + FROM pg_statistic_ext s JOIN pg_class c ON (c.oid = s.stxrelid) + LEFT JOIN pg_statistic_ext_data sd ON (s.oid = sd.stxoid) + LEFT JOIN pg_namespace cn ON (cn.oid = c.relnamespace) + LEFT JOIN pg_namespace sn ON (sn.oid = s.stxnamespace) + JOIN LATERAL ( + SELECT unnest(pg_get_statisticsobjdef_expressions(s.oid)) AS expr, + unnest(sd.stxdexpr)::pg_statistic AS a + ) stat ON (stat.expr IS NOT NULL) + WHERE pg_has_role(c.relowner, 'USAGE') + AND (c.relrowsecurity = false OR NOT row_security_active(c.oid)); diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 23a68ab22c6..ffd5a8a1207 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -310,12 +310,7 @@ CREATE VIEW pg_stats_ext WITH (security_barrier) AS array_agg(base_frequency) AS most_common_base_freqs FROM pg_mcv_list_items(sd.stxdmcv) ) m ON sd.stxdmcv IS NOT NULL - WHERE NOT EXISTS - ( SELECT 1 - FROM unnest(stxkeys) k - JOIN pg_attribute a - ON (a.attrelid = s.stxrelid AND a.attnum = k) - WHERE NOT has_column_privilege(c.oid, a.attnum, 'select') ) + WHERE pg_has_role(c.relowner, 'USAGE') AND (c.relrowsecurity = false OR NOT row_security_active(c.oid)); CREATE VIEW pg_stats_ext_exprs WITH (security_barrier) AS @@ -384,7 +379,9 @@ CREATE VIEW pg_stats_ext_exprs WITH (security_barrier) AS JOIN LATERAL ( SELECT unnest(pg_get_statisticsobjdef_expressions(s.oid)) AS expr, unnest(sd.stxdexpr)::pg_statistic AS a - ) stat ON (stat.expr IS NOT NULL); + ) stat ON (stat.expr IS NOT NULL) + WHERE pg_has_role(c.relowner, 'USAGE') + AND (c.relrowsecurity = false OR NOT row_security_active(c.oid)); -- unprivileged users may read pg_statistic_ext but not pg_statistic_ext_data REVOKE ALL ON pg_statistic_ext_data FROM public; diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index cf2f24a5a67..16a471245a9 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2439,10 +2439,7 @@ pg_stats_ext| SELECT cn.nspname AS schemaname, array_agg(pg_mcv_list_items.frequency) AS most_common_freqs, array_agg(pg_mcv_list_items.base_frequency) AS most_common_base_freqs FROM pg_mcv_list_items(sd.stxdmcv) pg_mcv_list_items(index, "values", nulls, frequency, base_frequency)) m ON ((sd.stxdmcv IS NOT NULL))) - WHERE ((NOT (EXISTS ( SELECT 1 - FROM (unnest(s.stxkeys) k(k) - JOIN pg_attribute a ON (((a.attrelid = s.stxrelid) AND (a.attnum = k.k)))) - WHERE (NOT has_column_privilege(c.oid, a.attnum, 'select'::text))))) AND ((c.relrowsecurity = false) OR (NOT row_security_active(c.oid)))); + WHERE (pg_has_role(c.relowner, 'USAGE'::text) AND ((c.relrowsecurity = false) OR (NOT row_security_active(c.oid)))); pg_stats_ext_exprs| SELECT cn.nspname AS schemaname, c.relname AS tablename, sn.nspname AS statistics_schemaname, @@ -2514,7 +2511,8 @@ pg_stats_ext_exprs| SELECT cn.nspname AS schemaname, LEFT JOIN pg_namespace cn ON ((cn.oid = c.relnamespace))) LEFT JOIN pg_namespace sn ON ((sn.oid = s.stxnamespace))) JOIN LATERAL ( SELECT unnest(pg_get_statisticsobjdef_expressions(s.oid)) AS expr, - unnest(sd.stxdexpr) AS a) stat ON ((stat.expr IS NOT NULL))); + unnest(sd.stxdexpr) AS a) stat ON ((stat.expr IS NOT NULL))) + WHERE (pg_has_role(c.relowner, 'USAGE'::text) AND ((c.relrowsecurity = false) OR (NOT row_security_active(c.oid)))); pg_tables| SELECT n.nspname AS schemaname, c.relname AS tablename, pg_get_userbyid(c.relowner) AS tableowner, diff --git a/src/test/regress/expected/stats_ext.out b/src/test/regress/expected/stats_ext.out index b752abfc4c6..06aff4d5bd0 100644 --- a/src/test/regress/expected/stats_ext.out +++ b/src/test/regress/expected/stats_ext.out @@ -3235,10 +3235,53 @@ SELECT * FROM tststats.priv_test_tbl WHERE a <<< 0 AND b <<< 0; -- Should not le (0 rows) DELETE FROM tststats.priv_test_tbl WHERE a <<< 0 AND b <<< 0; -- Should not leak +-- privilege checks for pg_stats_ext and pg_stats_ext_exprs +RESET SESSION AUTHORIZATION; +CREATE TABLE stats_ext_tbl (id INT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, col TEXT); +INSERT INTO stats_ext_tbl (col) VALUES ('secret'), ('secret'), ('very secret'); +CREATE STATISTICS s_col ON id, col FROM stats_ext_tbl; +CREATE STATISTICS s_expr ON mod(id, 2), lower(col) FROM stats_ext_tbl; +ANALYZE stats_ext_tbl; +-- unprivileged role should not have access +SET SESSION AUTHORIZATION regress_stats_user1; +SELECT statistics_name, most_common_vals FROM pg_stats_ext x + WHERE tablename = 'stats_ext_tbl' ORDER BY ROW(x.*); + statistics_name | most_common_vals +-----------------+------------------ +(0 rows) + +SELECT statistics_name, most_common_vals FROM pg_stats_ext_exprs x + WHERE tablename = 'stats_ext_tbl' ORDER BY ROW(x.*); + statistics_name | most_common_vals +-----------------+------------------ +(0 rows) + +-- give unprivileged role ownership of table +RESET SESSION AUTHORIZATION; +ALTER TABLE stats_ext_tbl OWNER TO regress_stats_user1; +-- unprivileged role should now have access +SET SESSION AUTHORIZATION regress_stats_user1; +SELECT statistics_name, most_common_vals FROM pg_stats_ext x + WHERE tablename = 'stats_ext_tbl' ORDER BY ROW(x.*); + statistics_name | most_common_vals +-----------------+------------------------------------------- + s_col | {{1,secret},{2,secret},{3,"very secret"}} + s_expr | {{0,secret},{1,secret},{1,"very secret"}} +(2 rows) + +SELECT statistics_name, most_common_vals FROM pg_stats_ext_exprs x + WHERE tablename = 'stats_ext_tbl' ORDER BY ROW(x.*); + statistics_name | most_common_vals +-----------------+------------------ + s_expr | {secret} + s_expr | {1} +(2 rows) + -- Tidy up DROP OPERATOR <<< (int, int); DROP FUNCTION op_leak(int, int); RESET SESSION AUTHORIZATION; +DROP TABLE stats_ext_tbl; DROP SCHEMA tststats CASCADE; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to table tststats.priv_test_tbl diff --git a/src/test/regress/expected/stats_ext_optimizer.out b/src/test/regress/expected/stats_ext_optimizer.out index 9f1b78af0b3..dafbf0a28b4 100644 --- a/src/test/regress/expected/stats_ext_optimizer.out +++ b/src/test/regress/expected/stats_ext_optimizer.out @@ -3270,10 +3270,53 @@ SELECT * FROM tststats.priv_test_tbl WHERE a <<< 0 AND b <<< 0; -- Should not le (0 rows) DELETE FROM tststats.priv_test_tbl WHERE a <<< 0 AND b <<< 0; -- Should not leak +-- privilege checks for pg_stats_ext and pg_stats_ext_exprs +RESET SESSION AUTHORIZATION; +CREATE TABLE stats_ext_tbl (id INT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, col TEXT); +INSERT INTO stats_ext_tbl (col) VALUES ('secret'), ('secret'), ('very secret'); +CREATE STATISTICS s_col ON id, col FROM stats_ext_tbl; +CREATE STATISTICS s_expr ON mod(id, 2), lower(col) FROM stats_ext_tbl; +ANALYZE stats_ext_tbl; +-- unprivileged role should not have access +SET SESSION AUTHORIZATION regress_stats_user1; +SELECT statistics_name, most_common_vals FROM pg_stats_ext x + WHERE tablename = 'stats_ext_tbl' ORDER BY ROW(x.*); + statistics_name | most_common_vals +-----------------+------------------ +(0 rows) + +SELECT statistics_name, most_common_vals FROM pg_stats_ext_exprs x + WHERE tablename = 'stats_ext_tbl' ORDER BY ROW(x.*); + statistics_name | most_common_vals +-----------------+------------------ +(0 rows) + +-- give unprivileged role ownership of table +RESET SESSION AUTHORIZATION; +ALTER TABLE stats_ext_tbl OWNER TO regress_stats_user1; +-- unprivileged role should now have access +SET SESSION AUTHORIZATION regress_stats_user1; +SELECT statistics_name, most_common_vals FROM pg_stats_ext x + WHERE tablename = 'stats_ext_tbl' ORDER BY ROW(x.*); + statistics_name | most_common_vals +-----------------+------------------------------------------- + s_col | {{1,secret},{2,secret},{3,"very secret"}} + s_expr | {{0,secret},{1,secret},{1,"very secret"}} +(2 rows) + +SELECT statistics_name, most_common_vals FROM pg_stats_ext_exprs x + WHERE tablename = 'stats_ext_tbl' ORDER BY ROW(x.*); + statistics_name | most_common_vals +-----------------+------------------ + s_expr | {secret} + s_expr | {1} +(2 rows) + -- Tidy up DROP OPERATOR <<< (int, int); DROP FUNCTION op_leak(int, int); RESET SESSION AUTHORIZATION; +DROP TABLE stats_ext_tbl; DROP SCHEMA tststats CASCADE; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to table tststats.priv_test_tbl diff --git a/src/test/regress/sql/stats_ext.sql b/src/test/regress/sql/stats_ext.sql index 6840818118d..744bb00c161 100644 --- a/src/test/regress/sql/stats_ext.sql +++ b/src/test/regress/sql/stats_ext.sql @@ -1649,10 +1649,37 @@ SET SESSION AUTHORIZATION regress_stats_user1; SELECT * FROM tststats.priv_test_tbl WHERE a <<< 0 AND b <<< 0; -- Should not leak DELETE FROM tststats.priv_test_tbl WHERE a <<< 0 AND b <<< 0; -- Should not leak +-- privilege checks for pg_stats_ext and pg_stats_ext_exprs +RESET SESSION AUTHORIZATION; +CREATE TABLE stats_ext_tbl (id INT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, col TEXT); +INSERT INTO stats_ext_tbl (col) VALUES ('secret'), ('secret'), ('very secret'); +CREATE STATISTICS s_col ON id, col FROM stats_ext_tbl; +CREATE STATISTICS s_expr ON mod(id, 2), lower(col) FROM stats_ext_tbl; +ANALYZE stats_ext_tbl; + +-- unprivileged role should not have access +SET SESSION AUTHORIZATION regress_stats_user1; +SELECT statistics_name, most_common_vals FROM pg_stats_ext x + WHERE tablename = 'stats_ext_tbl' ORDER BY ROW(x.*); +SELECT statistics_name, most_common_vals FROM pg_stats_ext_exprs x + WHERE tablename = 'stats_ext_tbl' ORDER BY ROW(x.*); + +-- give unprivileged role ownership of table +RESET SESSION AUTHORIZATION; +ALTER TABLE stats_ext_tbl OWNER TO regress_stats_user1; + +-- unprivileged role should now have access +SET SESSION AUTHORIZATION regress_stats_user1; +SELECT statistics_name, most_common_vals FROM pg_stats_ext x + WHERE tablename = 'stats_ext_tbl' ORDER BY ROW(x.*); +SELECT statistics_name, most_common_vals FROM pg_stats_ext_exprs x + WHERE tablename = 'stats_ext_tbl' ORDER BY ROW(x.*); + -- Tidy up DROP OPERATOR <<< (int, int); DROP FUNCTION op_leak(int, int); RESET SESSION AUTHORIZATION; +DROP TABLE stats_ext_tbl; DROP SCHEMA tststats CASCADE; DROP USER regress_stats_user1;