Skip to content

Commit

Permalink
Add warning for poor compression ratio
Browse files Browse the repository at this point in the history 
Adds functionality to emit a warning to the user when the compression
ratio is "bad".

Currently, "bad" is defined to be when size after compression is higher
than after compression.
  • Loading branch information
@kpan2034
kpan2034 committed Mar 4, 2025
1 parent 4ebfedf commit 076e1b0
Show file tree
Hide file tree
Showing 7 changed files with 94 additions and 0 deletions.
11 changes: 11 additions & 0 deletions src/guc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,7 @@ TSDLLEXPORT bool ts_guc_default_hypercore_use_access_method = false;
bool ts_guc_enable_chunk_skipping = false;
TSDLLEXPORT bool ts_guc_enable_segmentwise_recompression = true;
TSDLLEXPORT bool ts_guc_enable_bool_compression = false;
TSDLLEXPORT bool ts_guc_enable_compression_ratio_warnings = true;

/* Enable of disable columnar scans for columnar-oriented storage engines. If
* disabled, regular sequence scans will be used instead. */
Expand Down Expand Up @@ -761,6 +762,16 @@ _guc_init(void)
NULL,
NULL);

DefineCustomBoolVariable(MAKE_EXTOPTION("enable_compression_ratio_warnings"),
"Enable warnings for poor compression ratio",
"Enable warnings for poor compression ratio",
&ts_guc_enable_compression_ratio_warnings,
true,
PGC_USERSET,
0,
NULL,
NULL,
NULL);
/*
* Define the limit on number of invalidation-based refreshes we allow per
* refresh call. If this limit is exceeded, fall back to a single refresh that
Expand Down
1 change: 1 addition & 0 deletions src/guc.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ extern TSDLLEXPORT bool ts_guc_enable_merge_on_cagg_refresh;
extern bool ts_guc_enable_chunk_skipping;
extern TSDLLEXPORT bool ts_guc_enable_segmentwise_recompression;
extern TSDLLEXPORT bool ts_guc_enable_bool_compression;
extern TSDLLEXPORT bool ts_guc_enable_compression_ratio_warnings;

#ifdef USE_TELEMETRY
typedef enum TelemetryLevel
Expand Down
5 changes: 5 additions & 0 deletions tsl/src/compression/api.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -544,6 +544,11 @@ compress_chunk_impl(Oid hypertable_relid, Oid chunk_relid)
*/
ts_chunk_constraints_create(cxt.compress_ht, compress_ht_chunk);
ts_trigger_create_all_on_chunk(compress_ht_chunk);

/* Detect and emit warning if poor compression ratio is found */
float compression_ratio = ((float)before_size.total_size / after_size.total_size);
float POOR_COMPRESSION_THRESHOLD = 1.0;
elog(ts_guc_enable_compression_ratio_warnings && compression_ratio < POOR_COMPRESSION_THRESHOLD ? WARNING : DEBUG1, "size before compression: %ld bytes, size after compression: %ld bytes", before_size.total_size, after_size.total_size);
}
else
{
Expand Down
16 changes: 16 additions & 0 deletions tsl/test/isolation/expected/compression_ddl_iso.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ step UnlockChunk: ROLLBACK;
step I1: <... completed>
step Ic: COMMIT;
step C1: <... completed>
C: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand Down Expand Up @@ -75,6 +76,7 @@ step C1:
step I1: BEGIN; INSERT INTO ts_device_table VALUES (1, 1, 100, 100); <waiting ...>
step UnlockChunk: ROLLBACK;
step C1: <... completed>
C: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand Down Expand Up @@ -145,6 +147,7 @@ step UnlockChunk: ROLLBACK;
step A1: <... completed>
step A2: COMMIT;
step C1: <... completed>
C: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand Down Expand Up @@ -191,6 +194,7 @@ f

step UnlockChunk: ROLLBACK;
step C1: <... completed>
C: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand Down Expand Up @@ -231,6 +235,7 @@ count

step UnlockChunk: ROLLBACK;
step C1: <... completed>
C: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand Down Expand Up @@ -276,6 +281,7 @@ count

step UnlockChunk: ROLLBACK;
step C1: <... completed>
C: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand All @@ -300,6 +306,7 @@ step C1:
WHERE ch.hypertable_id = ht.id AND ht.table_name like 'ts_device_table'
ORDER BY ch.id LIMIT 1;

C: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand Down Expand Up @@ -342,6 +349,7 @@ step C1:
WHERE ch.hypertable_id = ht.id AND ht.table_name like 'ts_device_table'
ORDER BY ch.id LIMIT 1;

C: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand Down Expand Up @@ -384,6 +392,7 @@ step C1:
WHERE ch.hypertable_id = ht.id AND ht.table_name like 'ts_device_table'
ORDER BY ch.id LIMIT 1;

C: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand Down Expand Up @@ -436,6 +445,7 @@ step C1:
WHERE ch.hypertable_id = ht.id AND ht.table_name like 'ts_device_table'
ORDER BY ch.id LIMIT 1;

C: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand Down Expand Up @@ -485,6 +495,9 @@ step CA1:
FROM show_chunks('ts_device_table') AS ch
ORDER BY ch::text;

CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand Down Expand Up @@ -593,6 +606,9 @@ step CA1:
FROM show_chunks('ts_device_table') AS ch
ORDER BY ch::text;

CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand Down
42 changes: 42 additions & 0 deletions tsl/test/isolation/expected/compression_dml_iso.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ step CA1:
FROM show_chunks('ts_device_table') AS ch
ORDER BY ch::text;

CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand Down Expand Up @@ -69,6 +72,9 @@ step CA1:
FROM show_chunks('ts_device_table') AS ch
ORDER BY ch::text;

CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand Down Expand Up @@ -116,6 +122,9 @@ step CA1:
FROM show_chunks('ts_device_table') AS ch
ORDER BY ch::text;

CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand Down Expand Up @@ -164,6 +173,9 @@ step CA1:
FROM show_chunks('ts_device_table') AS ch
ORDER BY ch::text;

CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand Down Expand Up @@ -224,6 +236,9 @@ step CA1:
FROM show_chunks('ts_device_table') AS ch
ORDER BY ch::text;

CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand Down Expand Up @@ -274,6 +289,9 @@ step CA1:
FROM show_chunks('ts_device_table') AS ch
ORDER BY ch::text;

CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand Down Expand Up @@ -321,6 +339,9 @@ step CA1:
FROM show_chunks('ts_device_table') AS ch
ORDER BY ch::text;

CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand Down Expand Up @@ -369,6 +390,9 @@ step CA1:
FROM show_chunks('ts_device_table') AS ch
ORDER BY ch::text;

CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand Down Expand Up @@ -418,6 +442,9 @@ step CA1:
FROM show_chunks('ts_device_table') AS ch
ORDER BY ch::text;

CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand Down Expand Up @@ -466,6 +493,9 @@ step CA1:
FROM show_chunks('ts_device_table') AS ch
ORDER BY ch::text;

CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand Down Expand Up @@ -515,6 +545,9 @@ step CA1:
FROM show_chunks('ts_device_table') AS ch
ORDER BY ch::text;

CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand Down Expand Up @@ -563,6 +596,9 @@ step CA1:
FROM show_chunks('ts_device_table') AS ch
ORDER BY ch::text;

CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand Down Expand Up @@ -612,6 +648,9 @@ step CA1:
FROM show_chunks('ts_device_table') AS ch
ORDER BY ch::text;

CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand Down Expand Up @@ -660,6 +699,9 @@ step CA1:
FROM show_chunks('ts_device_table') AS ch
ORDER BY ch::text;

CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
CompressAll: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
compress
--------
t
Expand Down
12 changes: 12 additions & 0 deletions tsl/test/isolation/expected/deadlock_drop_chunks_compress.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ step s2_commit: COMMIT;
step s1_commit: COMMIT;

starting permutation: s2_compress_chunk_1 s1_drop s1_commit s2_compress_chunk_2 s2_commit
s2: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
step s2_compress_chunk_1:
SELECT count(compress_chunk(chunk))
FROM (SELECT chunk FROM chunks_to_compress ORDER BY 1 ASC LIMIT 1) AS chunk;
Expand All @@ -112,6 +113,7 @@ step s1_drop:
step s1_drop: <... completed>
ERROR: some chunks could not be read since they are being concurrently updated
step s1_commit: COMMIT;
s2: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
step s2_compress_chunk_2:
SELECT count(compress_chunk(chunk))
FROM (SELECT chunk FROM chunks_to_compress ORDER BY 1 DESC LIMIT 1) AS chunk;
Expand All @@ -124,6 +126,7 @@ count
step s2_commit: COMMIT;

starting permutation: s2_compress_chunk_1 s1_drop s2_compress_chunk_2 s1_commit s2_commit
s2: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
step s2_compress_chunk_1:
SELECT count(compress_chunk(chunk))
FROM (SELECT chunk FROM chunks_to_compress ORDER BY 1 ASC LIMIT 1) AS chunk;
Expand All @@ -137,6 +140,7 @@ step s1_drop:
SELECT count (*)
FROM drop_chunks('conditions', older_than => '2018-12-03 00:00'::timestamptz);
<waiting ...>
s2: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
step s2_compress_chunk_2:
SELECT count(compress_chunk(chunk))
FROM (SELECT chunk FROM chunks_to_compress ORDER BY 1 DESC LIMIT 1) AS chunk;
Expand All @@ -152,6 +156,7 @@ step s1_commit: COMMIT;
step s2_commit: COMMIT;

starting permutation: s2_compress_chunk_1 s1_drop s2_compress_chunk_2 s2_commit s1_commit
s2: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
step s2_compress_chunk_1:
SELECT count(compress_chunk(chunk))
FROM (SELECT chunk FROM chunks_to_compress ORDER BY 1 ASC LIMIT 1) AS chunk;
Expand All @@ -165,6 +170,7 @@ step s1_drop:
SELECT count (*)
FROM drop_chunks('conditions', older_than => '2018-12-03 00:00'::timestamptz);
<waiting ...>
s2: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
step s2_compress_chunk_2:
SELECT count(compress_chunk(chunk))
FROM (SELECT chunk FROM chunks_to_compress ORDER BY 1 DESC LIMIT 1) AS chunk;
Expand All @@ -184,6 +190,7 @@ count
step s1_commit: COMMIT;

starting permutation: s2_compress_chunk_1 s2_compress_chunk_2 s1_drop s1_commit s2_commit
s2: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
step s2_compress_chunk_1:
SELECT count(compress_chunk(chunk))
FROM (SELECT chunk FROM chunks_to_compress ORDER BY 1 ASC LIMIT 1) AS chunk;
Expand All @@ -193,6 +200,7 @@ count
1
(1 row)

s2: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
step s2_compress_chunk_2:
SELECT count(compress_chunk(chunk))
FROM (SELECT chunk FROM chunks_to_compress ORDER BY 1 DESC LIMIT 1) AS chunk;
Expand All @@ -212,6 +220,7 @@ step s1_commit: COMMIT;
step s2_commit: COMMIT;

starting permutation: s2_compress_chunk_1 s2_compress_chunk_2 s1_drop s2_commit s1_commit
s2: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
step s2_compress_chunk_1:
SELECT count(compress_chunk(chunk))
FROM (SELECT chunk FROM chunks_to_compress ORDER BY 1 ASC LIMIT 1) AS chunk;
Expand All @@ -221,6 +230,7 @@ count
1
(1 row)

s2: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
step s2_compress_chunk_2:
SELECT count(compress_chunk(chunk))
FROM (SELECT chunk FROM chunks_to_compress ORDER BY 1 DESC LIMIT 1) AS chunk;
Expand All @@ -244,6 +254,7 @@ count
step s1_commit: COMMIT;

starting permutation: s2_compress_chunk_1 s2_compress_chunk_2 s2_commit s1_drop s1_commit
s2: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
step s2_compress_chunk_1:
SELECT count(compress_chunk(chunk))
FROM (SELECT chunk FROM chunks_to_compress ORDER BY 1 ASC LIMIT 1) AS chunk;
Expand All @@ -253,6 +264,7 @@ count
1
(1 row)

s2: WARNING: size before compression: 24576 bytes, size after compression: 40960 bytes
step s2_compress_chunk_2:
SELECT count(compress_chunk(chunk))
FROM (SELECT chunk FROM chunks_to_compress ORDER BY 1 DESC LIMIT 1) AS chunk;
Expand Down
Loading

0 comments on commit 076e1b0

Please sign in to comment.