Send rows in binary mode for ANALYZE (#15159)

Send results of select pg_catalog.gp_acquire_sample_rows query in binary mode.
That allows to avoid overflow for max double.

For example, if run the following prior to this fix:

set extra_float_digits to 0;
create table t (a double precision);
insert into t values (1.7976931348623157e+308);
analyze t;
the following message will be printed:
ERROR: value out of range: overflow

For text mode (default) when analyze for table is performed the
master calls gp_acquire_sample_rows() helper function on each
segment. That eventually calls float8out function on segment to
converts float8 number to a string with snprintf:

snprintf(ascii, MAXDOUBLEWIDTH + 1, "%.*g", ndig, num);
When ndig is 15 the maximum float8 value 1.7976931348623157e+308 is
rounded to "1.79769313486232e+308" that has no representation.

And on master acquire_sample_rows_dispatcher function
process gp_acquire_sample_rows result and eventually float8in
function is called to convert string to float8 with strtold:
val = strtold(num, &endptr);
This is where overflow for "1.79769313486232e+308" happens but works
fine for "1.7976931348623157e+308".

Transferring in binary mode allows to avoid conversion from double to
string on segments and then back to double on master. And this will
much faster than before.

Using CdbDispatchPlan instead of CdbDispatchCommand allows
to receive data in binary mode in MemTuple, and this is much faster than before.
And use tuplestore to store received tuples to avoid use too many memory.

Co-authored-by: zxuejing <[email protected]>

Author: zxuejing

src/backend/commands/analyze.c

@@ -215,6 +215,15 @@ gp_acquire_sample_rows(PG_FUNCTION_ARGS)
 
 		ctx->index = 0;
 		ctx->summary_sent = false;
+		/*
+		 * we only get sample data from segindex 0 for replicated table
+		 */
+		if (Gp_role == GP_ROLE_EXECUTE && GpPolicyIsReplicated(onerel->rd_cdbpolicy)
+									   && GpIdentity.segindex > 0)
+		{
+			ctx->index = ctx->num_sample_rows;
+			ctx->summary_sent = true;
+		}
 
 		MemoryContextSwitchTo(oldcontext);
 	}

src/backend/commands/analyzefuncs.c

@@ -1086,26 +1086,57 @@ SELECT correlation FROM pg_stats WHERE tablename ='analyze_table';
 (2 rows)
 
 -- test6: randomly table
+-- we use weighted mean algorithm to calculate correlations.
+-- the formula for calculating the weighted mean is:
+-- sum(correlationOnSeg[i] * (totalRowsOnSeg[i] / totalRows))
+-- i is from 0 to N. N is the number of segments.
+-- however, for randomly table the data in each segment may diff each time.
+-- it will affect the value of correlation.
+-- So ignore the results
 drop table analyze_table;
 create table analyze_table(tc1 int,tc2 int) distributed randomly;
 insert into analyze_table select i,i from generate_series(1,100) i;
 analyze analyze_table;
+-- start_ignore
 SELECT correlation FROM pg_stats WHERE tablename ='analyze_table';
  correlation 
 -------------
            1
            1
 (2 rows)
 
+-- end_ignore
 alter table analyze_table drop column tc1;
 analyze analyze_table;
+-- start_ignore
 SELECT correlation FROM pg_stats WHERE tablename ='analyze_table';
  correlation 
 -------------
            1
 (1 row)
 
--- test7: inherit table
+-- end_ignore
+-- test7: replicated table
+drop table analyze_table;
+create table analyze_table(tc1 int,tc2 int) distributed replicated;
+insert into analyze_table select i,i from generate_series(1,100) i;
+analyze analyze_table;
+SELECT correlation FROM pg_stats WHERE tablename ='analyze_table';
+ correlation 
+-------------
+           1
+           1
+(2 rows)
+
+analyze analyze_table;
+SELECT correlation FROM pg_stats WHERE tablename ='analyze_table';
+ correlation 
+-------------
+           1
+           1
+(2 rows)
+
+-- test8: inherit table
 drop table analyze_parent cascade;
 ERROR:  table "analyze_parent" does not exist
 create table analyze_parent (tc1 int,tc2 int);
@@ -1128,7 +1159,7 @@ SELECT correlation,attname,inherited FROM pg_stats WHERE tablename ='analyze_chi
 -------------+---------+-----------
 (0 rows)
 
--- test8: partition table test
+-- test9: partition table test
 CREATE TABLE partition_table (
     tc1 int,
     tc2 int
@@ -1179,3 +1210,23 @@ SELECT correlation,attname,inherited FROM pg_stats WHERE tablename ='partition_t
            1 | tc2     | f
 (2 rows)
 
+--
+-- Test analyze for table with maximum float8 value 1.7976931348623157e+308
+-- There should be no "ERROR:  value out of range: overflow"
+--
+set extra_float_digits to 0;
+create table test_max_float8(a double precision);
+insert into test_max_float8 values(1.7976931348623157e+308);
+analyze test_max_float8;
+drop table test_max_float8;
+reset extra_float_digits;
+-- test analyze when table has large column
+create table ttt_large_column(tc1 int,tc2 char(1500),tc3 char(1500));
+insert into ttt_large_column select i,repeat('wwweereeer',150),repeat('ssddbbbbbb',150) from generate_series(1,5) i;
+analyze ttt_large_column;
+drop table ttt_large_column;
+--test analyze replicated table
+create table analyze_replicated(tc1 int,tc2 int) distributed replicated;
+insert into analyze_replicated select i, i from generate_series(1,1000) i;
+analyze analyze_replicated;
+drop table analyze_replicated;

src/test/regress/expected/analyze.out

@@ -19,6 +19,7 @@ insert into dd_singlecol_1 values(null, null);
 INFO:  (slice 0) Dispatch command to SINGLE content
 INFO:  Distributed transaction command 'Distributed Commit (one-phase)' to SINGLE content
 analyze dd_singlecol_1;
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
 -- ctas tests
 create table dd_ctas_1 as select * from dd_singlecol_1 where a=1 distributed by (a);
 INFO:  (slice 0) Dispatch command to ALL contents: 0 1 2
@@ -317,7 +318,9 @@ insert into dd_singlecol_idx2 values(null, null);
 INFO:  (slice 0) Dispatch command to SINGLE content
 INFO:  Distributed transaction command 'Distributed Commit (one-phase)' to SINGLE content
 analyze dd_singlecol_idx;
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
 analyze dd_singlecol_idx2;
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
 -- disjunction with index scans
 select * from dd_singlecol_idx where (a=1 or a=2) and b<2;
 INFO:  (slice 1) Dispatch command to PARTIAL contents: 1 0
@@ -366,6 +369,7 @@ insert into dd_singlecol_bitmap_idx values(null, null);
 INFO:  (slice 0) Dispatch command to SINGLE content
 INFO:  Distributed transaction command 'Distributed Commit (one-phase)' to SINGLE content
 analyze dd_singlecol_bitmap_idx;
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
 -- disjunction with bitmap index scans
 select * from dd_singlecol_bitmap_idx where (a=1 or a=2) and b<2;
 INFO:  (slice 1) Dispatch command to PARTIAL contents: 1 0
@@ -442,6 +446,12 @@ insert into dd_singlecol_part_bitmap_idx values(null, null);
 INFO:  (slice 0) Dispatch command to SINGLE content
 INFO:  Distributed transaction command 'Distributed Commit (one-phase)' to SINGLE content
 analyze dd_singlecol_part_bitmap_idx;
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
 -- bitmap indexes on partitioned tables
 select * from dd_singlecol_part_bitmap_idx where a=1 and b=0;
 INFO:  (slice 1) Dispatch command to SINGLE content
@@ -487,6 +497,7 @@ insert into dd_multicol_idx values(null, null);
 INFO:  (slice 0) Dispatch command to SINGLE content
 INFO:  Distributed transaction command 'Distributed Commit (one-phase)' to SINGLE content
 analyze dd_multicol_idx;
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
 select count(*) from dd_multicol_idx;
 INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
  count 
@@ -610,7 +621,19 @@ insert into dd_singlecol_part_idx2 values(null, null);
 INFO:  (slice 0) Dispatch command to SINGLE content
 INFO:  Distributed transaction command 'Distributed Commit (one-phase)' to SINGLE content
 analyze dd_singlecol_part_idx;
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
 analyze dd_singlecol_part_idx2;
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
 -- indexes on partitioned tables
 select * from dd_singlecol_part_idx where a=1 and b>0;
 INFO:  (slice 1) Dispatch command to SINGLE content

src/test/regress/expected/bfv_dd.out

@@ -29,6 +29,7 @@ insert into dd_multicol_1 values(null, 1);
 INFO:  (slice 0) Dispatch command to SINGLE content
 INFO:  Distributed transaction command 'Distributed Commit (one-phase)' to SINGLE content
 analyze dd_multicol_1;
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
 insert into dd_multicol_2 select g, g%2 from generate_series(1, 100) g;
 INFO:  (slice 0) Dispatch command to ALL contents: 0 1 2
 INFO:  (slice 1) Dispatch command to SINGLE content

src/test/regress/expected/bfv_dd_multicolumn.out

@@ -31,6 +31,7 @@ INFO:  (slice 0) Dispatch command to ALL contents: 0 1 2
 INFO:  Distributed transaction command 'Distributed Prepare' to ALL contents: 0 1 2
 INFO:  Distributed transaction command 'Distributed Commit Prepared' to ALL contents: 0 1 2
 analyze dd_multicol_1;
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
 insert into dd_multicol_2 select g, g%2 from generate_series(1, 100) g;
 INFO:  (slice 0) Dispatch command to ALL contents: 0 1 2
 INFO:  Distributed transaction command 'Distributed Prepare' to ALL contents: 0 1 2

src/test/regress/expected/bfv_dd_multicolumn_optimizer.out

@@ -18,6 +18,7 @@ insert into dd_singlecol_1 values(null, null);
 INFO:  (slice 0) Dispatch command to SINGLE content
 INFO:  Distributed transaction command 'Distributed Commit (one-phase)' to SINGLE content
 analyze dd_singlecol_1;
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
 -- ctas tests
 create table dd_ctas_1 as select * from dd_singlecol_1 where a=1 distributed by (a);
 INFO:  (slice 0) Dispatch command to ALL contents: 0 1 2
@@ -315,7 +316,9 @@ insert into dd_singlecol_idx2 values(null, null);
 INFO:  (slice 0) Dispatch command to SINGLE content
 INFO:  Distributed transaction command 'Distributed Commit (one-phase)' to SINGLE content
 analyze dd_singlecol_idx;
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
 analyze dd_singlecol_idx2;
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
 -- disjunction with index scans
 select * from dd_singlecol_idx where (a=1 or a=2) and b<2;
 INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
@@ -363,6 +366,7 @@ insert into dd_singlecol_bitmap_idx values(null, null);
 INFO:  (slice 0) Dispatch command to SINGLE content
 INFO:  Distributed transaction command 'Distributed Commit (one-phase)' to SINGLE content
 analyze dd_singlecol_bitmap_idx;
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
 -- disjunction with bitmap index scans
 select * from dd_singlecol_bitmap_idx where (a=1 or a=2) and b<2;
 INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
@@ -438,6 +442,12 @@ insert into dd_singlecol_part_bitmap_idx values(null, null);
 INFO:  (slice 0) Dispatch command to SINGLE content
 INFO:  Distributed transaction command 'Distributed Commit (one-phase)' to SINGLE content
 analyze dd_singlecol_part_bitmap_idx;
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
 -- bitmap indexes on partitioned tables
 select * from dd_singlecol_part_bitmap_idx where a=1 and b=0;
 INFO:  (slice 1) Dispatch command to SINGLE content
@@ -483,6 +493,7 @@ INFO:  (slice 0) Dispatch command to ALL contents: 0 1 2
 INFO:  Distributed transaction command 'Distributed Prepare' to ALL contents: 0 1 2
 INFO:  Distributed transaction command 'Distributed Commit Prepared' to ALL contents: 0 1 2
 analyze dd_multicol_idx;
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
 select count(*) from dd_multicol_idx;
 INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
  count 
@@ -604,7 +615,19 @@ insert into dd_singlecol_part_idx2 values(null, null);
 INFO:  (slice 0) Dispatch command to SINGLE content
 INFO:  Distributed transaction command 'Distributed Commit (one-phase)' to SINGLE content
 analyze dd_singlecol_part_idx;
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
 analyze dd_singlecol_part_idx2;
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
+INFO:  (slice 1) Dispatch command to ALL contents: 0 1 2
 -- indexes on partitioned tables
 select * from dd_singlecol_part_idx where a=1 and b>0;
 INFO:  (slice 1) Dispatch command to SINGLE content

src/test/regress/expected/bfv_dd_optimizer.out

@@ -551,16 +551,36 @@ analyze analyze_table;
 SELECT correlation FROM pg_stats WHERE tablename ='analyze_table';
 
 -- test6: randomly table
+-- we use weighted mean algorithm to calculate correlations.
+-- the formula for calculating the weighted mean is:
+-- sum(correlationOnSeg[i] * (totalRowsOnSeg[i] / totalRows))
+-- i is from 0 to N. N is the number of segments.
+-- however, for randomly table the data in each segment may diff each time.
+-- it will affect the value of correlation.
+-- So ignore the results
 drop table analyze_table;
 create table analyze_table(tc1 int,tc2 int) distributed randomly;
 insert into analyze_table select i,i from generate_series(1,100) i;
 analyze analyze_table;
+-- start_ignore
 SELECT correlation FROM pg_stats WHERE tablename ='analyze_table';
+-- end_ignore
 alter table analyze_table drop column tc1;
 analyze analyze_table;
+-- start_ignore
 SELECT correlation FROM pg_stats WHERE tablename ='analyze_table';
+-- end_ignore
 
--- test7: inherit table
+-- test7: replicated table
+drop table analyze_table;
+create table analyze_table(tc1 int,tc2 int) distributed replicated;
+insert into analyze_table select i,i from generate_series(1,100) i;
+analyze analyze_table;
+SELECT correlation FROM pg_stats WHERE tablename ='analyze_table';
+analyze analyze_table;
+SELECT correlation FROM pg_stats WHERE tablename ='analyze_table';
+
+-- test8: inherit table
 drop table analyze_parent cascade;
 create table analyze_parent (tc1 int,tc2 int);
 create table analyze_child(tc3 int,tc4 int)inherits (analyze_parent);
@@ -571,7 +591,7 @@ analyze analyze_parent;
 SELECT correlation,attname,inherited FROM pg_stats WHERE tablename ='analyze_parent';
 SELECT correlation,attname,inherited FROM pg_stats WHERE tablename ='analyze_child';
 
--- test8: partition table test
+-- test9: partition table test
 CREATE TABLE partition_table (
     tc1 int,
     tc2 int
@@ -589,3 +609,25 @@ SELECT correlation,attname,inherited FROM pg_stats WHERE tablename ='partition_t
 SELECT correlation,attname,inherited FROM pg_stats WHERE tablename ='partition_table_1_prt_3';
 SELECT correlation,attname,inherited FROM pg_stats WHERE tablename ='partition_table_1_prt_4';
 SELECT correlation,attname,inherited FROM pg_stats WHERE tablename ='partition_table_1_prt_5';
+--
+-- Test analyze for table with maximum float8 value 1.7976931348623157e+308
+-- There should be no "ERROR:  value out of range: overflow"
+--
+set extra_float_digits to 0;
+create table test_max_float8(a double precision);
+insert into test_max_float8 values(1.7976931348623157e+308);
+analyze test_max_float8;
+drop table test_max_float8;
+reset extra_float_digits;
+
+-- test analyze when table has large column
+create table ttt_large_column(tc1 int,tc2 char(1500),tc3 char(1500));
+insert into ttt_large_column select i,repeat('wwweereeer',150),repeat('ssddbbbbbb',150) from generate_series(1,5) i;
+analyze ttt_large_column;
+drop table ttt_large_column;
+
+--test analyze replicated table
+create table analyze_replicated(tc1 int,tc2 int) distributed replicated;
+insert into analyze_replicated select i, i from generate_series(1,1000) i;
+analyze analyze_replicated;
+drop table analyze_replicated;