From ebbb9a3aa949e07c9737e7adb52dac28b561c073 Mon Sep 17 00:00:00 2001 From: Thomas Hatzopoulos Date: Mon, 29 Aug 2022 14:36:12 -0500 Subject: [PATCH] Release automation phase 1 changes --- Cargo.lock | 3 + tests/update/original_update_tests.md | 41 ++ tools/update-tester/Cargo.toml | 3 + tools/update-tester/src/main.rs | 105 ++++- tools/update-tester/src/parser.rs | 465 ++++++++++++++++++++++ tools/update-tester/src/testrunner.rs | 541 +++++++++++++++++++++----- 6 files changed, 1042 insertions(+), 116 deletions(-) create mode 100644 tests/update/original_update_tests.md create mode 100644 tools/update-tester/src/parser.rs diff --git a/Cargo.lock b/Cargo.lock index fef92cd0..ea58e8ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2590,13 +2590,16 @@ checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" name = "update-tester" version = "0.3.0" dependencies = [ + "bytecount", "clap", "colored", "control_file_reader", "postgres", "postgres_connection_configuration", + "pulldown-cmark", "semver 1.0.13", "toml_edit", + "walkdir", "xshell", ] diff --git a/tests/update/original_update_tests.md b/tests/update/original_update_tests.md new file mode 100644 index 00000000..2405f5f1 --- /dev/null +++ b/tests/update/original_update_tests.md @@ -0,0 +1,41 @@ +# Original Update Tests + + + +```sql,creation,min-toolkit-version=1.4.0 +SET TIME ZONE 'UTC'; +CREATE TABLE test_data(ts timestamptz, val DOUBLE PRECISION); + INSERT INTO test_data + SELECT '2020-01-01 00:00:00+00'::timestamptz + i * '1 hour'::interval, + 100 + i % 100 + FROM generate_series(0, 10000) i; + +CREATE MATERIALIZED VIEW regression_view AS + SELECT + counter_agg(ts, val) AS countagg, + hyperloglog(1024, val) AS hll, + time_weight('locf', ts, val) AS twa, + uddsketch(100, 0.001, val) as udd, + tdigest(100, val) as tdig, + stats_agg(val) as stats + FROM test_data; +``` + + + +```sql,validation,min-toolkit-version=1.4.0 +SELECT + num_resets(countagg), + distinct_count(hll), + average(twa), + approx_percentile(0.1, udd), + approx_percentile(0.1, tdig), + kurtosis(stats) +FROM regression_view; +``` + +```output + num_resets | distinct_count | average | approx_percentile | approx_percentile | kurtosis +------------+----------------+---------+--------------------+--------------------+-------------------- + 100 | 100 | 149.5 | 108.96220333142547 | 109.50489521100047 | 1.7995661075080858 +``` diff --git a/tools/update-tester/Cargo.toml b/tools/update-tester/Cargo.toml index 6709acca..bc234a85 100644 --- a/tools/update-tester/Cargo.toml +++ b/tools/update-tester/Cargo.toml @@ -15,3 +15,6 @@ postgres = "0.19.1" semver = "1.0.9" toml_edit = "0.14.3" xshell = "0.1.14" +pulldown-cmark = "0.8.0" +walkdir = "2.3.2" +bytecount = "0.6.3" diff --git a/tools/update-tester/src/main.rs b/tools/update-tester/src/main.rs index 9a97a0de..b70c6b8e 100644 --- a/tools/update-tester/src/main.rs +++ b/tools/update-tester/src/main.rs @@ -32,6 +32,7 @@ macro_rules! path { } mod installer; +mod parser; mod testrunner; fn main() { @@ -149,6 +150,8 @@ fn main() { .long("database") .takes_value(true) ) + + .arg(Arg::new("ROOT_DIR").takes_value(true).default_value(".")) ) // Mutates help, removing the short flag (-h) so that it can be used by HOST .mut_arg("help", |_h| { @@ -188,6 +191,21 @@ fn main() { .value_of("CARGO_PGX_OLD") .expect("missing cargo_pgx_old"); + let mut num_errors = 0; + let stdout = io::stdout(); + let mut out = stdout.lock(); + let on_error = |test: parser::Test, error: testrunner::TestError| { + num_errors += 1; + let _ = writeln!( + &mut out, + "{} {}\n", + test.location.bold().blue(), + test.header.bold().dimmed() + ); + let _ = writeln!(&mut out, "{}", error.annotate_position(&test.text)); + let _ = writeln!(&mut out, "{}\n", error); + }; + let res = try_main( root_dir, cache_dir, @@ -196,11 +214,16 @@ fn main() { cargo_pgx, cargo_pgx_old, reinstall, + on_error, ); if let Err(err) = res { eprintln!("{}", err); process::exit(1); } + if num_errors > 0 { + process::exit(1) + } + let _ = writeln!(&mut out, "{}\n", "Tests Passed".bold().green()); } Some(("create-test-objects", create_test_object_matches)) => { let connection_config = ConnectionConfig { @@ -211,11 +234,35 @@ fn main() { database: create_test_object_matches.value_of("DB"), }; - let res = try_create_objects(&connection_config); + let mut num_errors = 0; + let stdout = io::stdout(); + let mut out = stdout.lock(); + let on_error = |test: parser::Test, error: testrunner::TestError| { + num_errors += 1; + let _ = writeln!( + &mut out, + "{} {}\n", + test.location.bold().blue(), + test.header.bold().dimmed() + ); + let _ = writeln!(&mut out, "{}", error.annotate_position(&test.text)); + let _ = writeln!(&mut out, "{}\n", error); + }; + + let res = try_create_objects(&connection_config, on_error); if let Err(err) = res { eprintln!("{}", err); process::exit(1); } + if num_errors > 0 { + let _ = writeln!(&mut out, "{}\n", "Object Creation Failed".bold().red()); + process::exit(1) + } + let _ = writeln!( + &mut out, + "{}\n", + "Objects Created Successfully".bold().green() + ); } Some(("validate-test-objects", validate_test_object_matches)) => { let connection_config = ConnectionConfig { @@ -225,18 +272,46 @@ fn main() { password: validate_test_object_matches.value_of("PASSWORD"), database: validate_test_object_matches.value_of("DB"), }; + let mut num_errors = 0; + let stdout = io::stdout(); + let mut out = stdout.lock(); + let on_error = |test: parser::Test, error: testrunner::TestError| { + num_errors += 1; + let _ = writeln!( + &mut out, + "{} {}\n", + test.location.bold().blue(), + test.header.bold().dimmed() + ); + let _ = writeln!(&mut out, "{}", error.annotate_position(&test.text)); + let _ = writeln!(&mut out, "{}\n", error); + }; - let res = try_validate_objects(&connection_config); + let root_dir = validate_test_object_matches + .value_of("ROOT_DIR") + .expect("missing path to root of the toolkit repo"); + let res = try_validate_objects(&connection_config, root_dir, on_error); if let Err(err) = res { eprintln!("{}", err); process::exit(1); } + if num_errors > 0 { + let _ = writeln!(&mut out, "{}\n", "Validation Failed".bold().red()); + process::exit(1) + } + + let _ = writeln!( + &mut out, + "{}\n", + "Validations Completed Successfully".bold().green() + ); } _ => unreachable!(), // if all subcommands are defined, anything else is unreachable } } -fn try_main( +#[allow(clippy::too_many_arguments)] +fn try_main( root_dir: &str, cache_dir: Option<&str>, db_conn: &ConnectionConfig<'_>, @@ -244,6 +319,7 @@ fn try_main( cargo_pgx: &str, cargo_pgx_old: &str, reinstall: HashSet<&str>, + on_error: OnErr, ) -> xshell::Result<()> { let (current_version, old_versions) = get_version_info(root_dir)?; if old_versions.is_empty() { @@ -263,16 +339,25 @@ fn try_main( &reinstall, )?; - testrunner::run_update_tests(db_conn, current_version, old_versions) + testrunner::run_update_tests(db_conn, current_version, old_versions, on_error) } -fn try_create_objects(db_conn: &ConnectionConfig<'_>) -> xshell::Result<()> { - testrunner::create_test_objects_for_package_testing(db_conn); - Ok(()) +fn try_create_objects( + db_conn: &ConnectionConfig<'_>, + on_error: OnErr, +) -> xshell::Result<()> { + testrunner::create_test_objects_for_package_testing(db_conn, on_error) } -fn try_validate_objects(db_conn: &ConnectionConfig<'_>) -> xshell::Result<()> { - testrunner::update_to_and_validate_new_toolkit_version(db_conn); - Ok(()) +fn try_validate_objects( + _conn: &ConnectionConfig<'_>, + root_dir: &str, + on_error: OnErr, +) -> xshell::Result<()> { + let (current_version, old_versions) = get_version_info(root_dir)?; + if old_versions.is_empty() { + panic!("no old versions to upgrade from") + } + testrunner::update_to_and_validate_new_toolkit_version(current_version, _conn, on_error) } fn get_version_info(root_dir: &str) -> xshell::Result<(String, Vec)> { diff --git a/tools/update-tester/src/parser.rs b/tools/update-tester/src/parser.rs new file mode 100644 index 00000000..bb785f40 --- /dev/null +++ b/tools/update-tester/src/parser.rs @@ -0,0 +1,465 @@ +use std::{collections::HashMap, ffi::OsStr, fs}; + +use pulldown_cmark::{ + CodeBlockKind::Fenced, + CowStr, Event, Parser, + Tag::{CodeBlock, Heading}, +}; +use semver::Version; + +#[derive(Debug, PartialEq, Eq)] +#[must_use] +pub struct TestFile { + name: String, + stateless: bool, + pub tests: Vec, +} + +#[derive(Debug, PartialEq, Eq, Clone)] +#[must_use] +pub struct Test { + pub location: String, + pub header: String, + pub text: String, + pub output: Vec>, + transactional: bool, + ignore_output: bool, + pub precision_limits: HashMap, + pub creation: bool, + pub validation: bool, + pub min_toolkit_version: Option, +} + +pub fn extract_tests(root: &str) -> Vec { + // TODO handle when root is a file + let mut all_tests = vec![]; + let walker = walkdir::WalkDir::new(root) + .follow_links(true) + .sort_by(|a, b| a.path().cmp(b.path())); + for entry in walker { + let entry = entry.unwrap(); + if !entry.file_type().is_file() { + continue; + } + + if entry.path().extension() != Some(OsStr::new("md")) { + continue; + } + + let contents = fs::read_to_string(entry.path()).unwrap(); + + let tests = extract_tests_from_string(&*contents, &*entry.path().to_string_lossy()); + if !tests.tests.is_empty() { + all_tests.push(tests) + } + } + all_tests +} +// parsers the grammar `(heading* (test output?)*)*` +pub fn extract_tests_from_string(s: &str, file_stem: &str) -> TestFile { + let mut parser = Parser::new(s).into_offset_iter().peekable(); + let mut heading_stack: Vec = vec![]; + let mut tests = vec![]; + + let mut last_test_seen_at = 0; + let mut lines_seen = 0; + + let mut stateless = true; + + // consume the parser until an tag is reached, performing an action on each text + macro_rules! consume_text_until { + ($parser: ident yields $end: pat => $action: expr) => { + for (event, _) in &mut parser { + match event { + Event::Text(text) => $action(text), + $end => break, + _ => (), + } + } + }; + } + + 'block_hunt: while let Some((event, span)) = parser.next() { + match event { + // we found a heading, add it to the stack + Event::Start(Heading(level)) => { + heading_stack.truncate(level as usize - 1); + let mut header = "`".to_string(); + consume_text_until!(parser yields Event::End(Heading(..)) => + |text: CowStr| header.push_str(&*text) + ); + header.truncate(header.trim_end().len()); + header.push('`'); + heading_stack.push(header); + } + + // we found a code block, if it's a test add the test + Event::Start(CodeBlock(Fenced(ref info))) => { + let code_block_info = parse_code_block_info(info); + + // non-test code block, consume it and continue looking + if let BlockKind::Other = code_block_info.kind { + for (event, _) in &mut parser { + if let Event::End(CodeBlock(Fenced(..))) = event { + break; + } + } + continue 'block_hunt; + } + + let current_line = { + let offset = span.start; + lines_seen += bytecount::count(&s.as_bytes()[last_test_seen_at..offset], b'\n'); + last_test_seen_at = offset; + lines_seen + 1 + }; + + if let BlockKind::Output = code_block_info.kind { + panic!( + "found output with no test test.\n{}:{} {:?}", + file_stem, current_line, heading_stack + ) + } + + assert!(matches!(code_block_info.kind, BlockKind::Sql)); + + stateless &= code_block_info.transactional; + let mut test = Test { + location: format!("{}:{}", file_stem, current_line), + header: if heading_stack.is_empty() { + "".to_string() + } else { + heading_stack.join("::") + }, + text: String::new(), + output: Vec::new(), + transactional: code_block_info.transactional, + ignore_output: code_block_info.ignore_output, + precision_limits: code_block_info.precision_limits, + min_toolkit_version: code_block_info.min_toolkit_version, + creation: code_block_info.creation, + validation: code_block_info.validation, + }; + + // consume the lines of the test + consume_text_until!(parser yields Event::End(CodeBlock(Fenced(..))) => + |text: CowStr| test.text.push_str(&*text) + ); + + // search to see if we have output + loop { + match parser.peek() { + // we found a code block, is it output? + Some((Event::Start(CodeBlock(Fenced(info))), _)) => { + let code_block_info = parse_code_block_info(info); + match code_block_info.kind { + // non-output, continue at the top + BlockKind::Sql | BlockKind::Other => { + tests.push(test); + continue 'block_hunt; + } + + // output, consume it + BlockKind::Output => { + if !test.precision_limits.is_empty() + && !code_block_info.precision_limits.is_empty() + { + panic!( + "cannot have precision limits on both test and output.\n{}:{} {:?}", + file_stem, current_line, heading_stack + ) + } + test.precision_limits = code_block_info.precision_limits; + let _ = parser.next(); + break; + } + } + } + + // test must be over, continue at the top + Some((Event::Start(CodeBlock(..)), _)) + | Some((Event::Start(Heading(..)), _)) => { + tests.push(test); + continue 'block_hunt; + } + + // EOF, we're done + None => { + tests.push(test); + break 'block_hunt; + } + + // for now we allow text between the test and it's output + // TODO should/can we forbid this? + _ => { + let _ = parser.next(); + } + }; + } + + // consume the output + consume_text_until!(parser yields Event::End(CodeBlock(Fenced(..))) => + |text: CowStr| { + let rows = text.split('\n').skip(2).filter(|s| !s.is_empty()).map(|s| + s.split('|').map(|s| s.trim().to_string()).collect::>() + ); + test.output.extend(rows); + } + ); + + tests.push(test); + } + + _ => (), + } + } + TestFile { + name: file_stem.to_string(), + stateless, + tests, + } +} + +struct CodeBlockInfo { + kind: BlockKind, + transactional: bool, + ignore_output: bool, + precision_limits: HashMap, + min_toolkit_version: Option, + creation: bool, + validation: bool, +} + +#[derive(Clone, Copy)] +enum BlockKind { + Sql, + Output, + Other, +} + +fn parse_code_block_info(info: &str) -> CodeBlockInfo { + let tokens = info.split(','); + + let mut info = CodeBlockInfo { + kind: BlockKind::Other, + transactional: true, + ignore_output: false, + precision_limits: HashMap::new(), + min_toolkit_version: None, + creation: false, + validation: false, + }; + + for token in tokens { + match token.trim() { + "ignore" => { + if let BlockKind::Sql = info.kind { + info.kind = BlockKind::Other; + } + } + "non-transactional" => info.transactional = false, + "ignore-output" => info.ignore_output = true, + m if m.starts_with("min-toolkit-version") => { + // TODO Can we assume that version is greater than 1.10.1 since current tests don't have a min version? This means we ccan skip edge cases of 1.4/1.10.0-dev/etc. + info.min_toolkit_version = + Some(Version::parse(token.trim_start_matches("min-toolkit-version=")).unwrap()) + } // not great, shouldn't assume they typed in a valid version. fix later + "creation" => info.creation = true, + "validation" => info.validation = true, + "output" => info.kind = BlockKind::Output, + s if s.to_ascii_lowercase() == "sql" => info.kind = BlockKind::Sql, + p if p.starts_with("precision") => { + // syntax `precision(col: bytes)` + let precision_err = + || -> ! { panic!("invalid syntax for `precision(col: bytes)` found `{}`", p) }; + let arg = &p["precision".len()..]; + if arg.as_bytes().first() != Some(&b'(') || arg.as_bytes().last() != Some(&b')') { + precision_err() + } + let arg = &arg[1..arg.len() - 1]; + let args: Vec<_> = arg.split(':').collect(); + if args.len() != 2 { + precision_err() + } + let column = args[0].trim().parse().unwrap_or_else(|_| precision_err()); + let length = args[1].trim().parse().unwrap_or_else(|_| precision_err()); + let old = info.precision_limits.insert(column, length); + if old.is_some() { + panic!("duplicate precision for column {}", column) + } + } + _ => {} + } + } + + info +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use semver::{BuildMetadata, Prerelease, Version}; + + #[test] + fn extract() { + use super::{Test, TestFile}; + + let file = r##" +# Test Parsing +```SQL,creation +select * from foo +``` +```output +``` + +```SQL,creation +select * from multiline +``` +```output + ?column? +---------- + value +``` + +## ignored +```SQL,ignore,creation +select * from foo +``` + +## non-transactional,creation +```SQL,non-transactional,creation +select * from bar +``` +```output, precision(1: 3) + a | b +---+--- + 1 | 2 +``` + +## no output +```SQL,ignore-output,creation +select * from baz +``` + +## end by header +```SQL,creation +select * from quz +``` + +## end by file +```SQL,creation +select * from qat +``` + +## has a min-toolkit-version +```SQL,creation,min-toolkit-version=1.10.1 +select * from qat +``` +"##; + + let tests = super::extract_tests_from_string(file, "/test/file.md"); + let expected = TestFile { + name: "/test/file.md".to_string(), + stateless: false, + tests: vec![ + Test { + location: "/test/file.md:3".to_string(), + header: "`Test Parsing`".to_string(), + text: "select * from foo\n".to_string(), + output: vec![], + transactional: true, + ignore_output: false, + precision_limits: HashMap::new(), + creation: true, + min_toolkit_version: None, + validation: false, + }, + Test { + location: "/test/file.md:9".to_string(), + header: "`Test Parsing`".to_string(), + text: "select * from multiline\n".to_string(), + output: vec![vec!["value".to_string()]], + transactional: true, + ignore_output: false, + precision_limits: HashMap::new(), + creation: true, + min_toolkit_version: None, + validation: false, + }, + Test { + location: "/test/file.md:24".to_string(), + header: "`Test Parsing`::`non-transactional,creation`".to_string(), + text: "select * from bar\n".to_string(), + output: vec![vec!["1".to_string(), "2".to_string()]], + transactional: false, + ignore_output: false, + precision_limits: [(1, 3)].iter().cloned().collect(), + creation: true, + min_toolkit_version: None, + validation: false, + }, + Test { + location: "/test/file.md:34".to_string(), + header: "`Test Parsing`::`no output`".to_string(), + text: "select * from baz\n".to_string(), + output: vec![], + transactional: true, + ignore_output: true, + precision_limits: HashMap::new(), + creation: true, + min_toolkit_version: None, + validation: false, + }, + Test { + location: "/test/file.md:39".to_string(), + header: "`Test Parsing`::`end by header`".to_string(), + text: "select * from quz\n".to_string(), + output: vec![], + transactional: true, + ignore_output: false, + precision_limits: HashMap::new(), + creation: true, + min_toolkit_version: None, + validation: false, + }, + Test { + location: "/test/file.md:44".to_string(), + header: "`Test Parsing`::`end by file`".to_string(), + text: "select * from qat\n".to_string(), + output: vec![], + transactional: true, + ignore_output: false, + precision_limits: HashMap::new(), + creation: true, + min_toolkit_version: None, + validation: false, + }, + Test { + location: "/test/file.md:49".to_string(), + header: "`Test Parsing`::`has a min-toolkit-version`".to_string(), + text: "select * from qat\n".to_string(), + output: vec![], + transactional: true, + ignore_output: false, + precision_limits: HashMap::new(), + creation: true, + min_toolkit_version: Some(Version { + major: 1, + minor: 10, + patch: 1, + pre: Prerelease::EMPTY, + build: BuildMetadata::EMPTY, + }), + validation: false, + }, + ], + }; + assert!( + tests == expected, + "left: {:#?}\n right: {:#?}", + tests, + expected + ); + } +} diff --git a/tools/update-tester/src/testrunner.rs b/tools/update-tester/src/testrunner.rs index c51c78a6..e60bdc99 100644 --- a/tools/update-tester/src/testrunner.rs +++ b/tools/update-tester/src/testrunner.rs @@ -1,64 +1,91 @@ use colored::Colorize; +use semver::{BuildMetadata, Prerelease, Version}; +use crate::{defer, parser, Deferred}; use postgres::{Client, NoTls, SimpleQueryMessage}; - use postgres_connection_configuration::ConnectionConfig; - -use crate::{defer, Deferred}; - mod stabilization; -pub fn run_update_tests( +use crate::parser::Test; +use postgres::error::DbError; + +use std::{borrow::Cow, error::Error, fmt}; +pub fn run_update_tests( root_config: &ConnectionConfig, - current_version: String, - old_versions: Vec, + current_toolkit_version: String, + old_toolkit_versions: Vec, + mut on_error: OnErr, ) -> Result<(), xshell::Error> { - for old_version in old_versions { + for old_toolkit_version in old_toolkit_versions { eprintln!( " {} {} -> {}", "Testing".bold().cyan(), - old_version, - current_version + old_toolkit_version, + current_toolkit_version ); - let test_db_name = format!("tsdb_toolkit_test_{}--{}", old_version, current_version); + let test_db_name = format!( + "tsdb_toolkit_test_{}--{}", + old_toolkit_version, current_toolkit_version + ); let test_config = root_config.with_db(&test_db_name); with_temporary_db(&test_db_name, root_config, || { let mut test_client = connect_to(&test_config); - test_client.install_toolkit_at_version(&old_version); + test_client.install_toolkit_at_version(&old_toolkit_version); let installed_version = test_client.get_installed_extension_version(); assert_eq!( - installed_version, old_version, + installed_version, old_toolkit_version, "installed unexpected version" ); - let validation_values = test_client.create_test_objects(); + test_client.set_timezone_utc(); + + let errors = + test_client.create_test_objects_from_file(test_config, installed_version.clone()); - test_client.update_to_current_version(); + for (test, error) in errors { + match error { + Ok(..) => continue, + Err(error) => on_error(test, error), + } + } + + test_client.update_to_current_toolkit_version(); let new_version = test_client.get_installed_extension_version(); assert_eq!( - new_version, current_version, + new_version, current_toolkit_version, "updated to unexpected version" ); - test_client.validate_test_objects(validation_values); + let errors = + test_client.validate_test_objects_from_files(test_config, installed_version); + + for (test, error) in errors { + match error { + Ok(..) => continue, + Err(error) => on_error(test, error), + } + } - test_client.check_no_references_to_the_old_binary_leaked(¤t_version); + test_client.check_no_references_to_the_old_binary_leaked(¤t_toolkit_version); test_client.validate_stable_objects_exist(); }); eprintln!( "{} {} -> {}", "Finished".bold().green(), - old_version, - current_version + old_toolkit_version, + current_toolkit_version ); } Ok(()) } -pub fn create_test_objects_for_package_testing(root_config: &ConnectionConfig) { +pub fn create_test_objects_for_package_testing( + root_config: &ConnectionConfig, + mut on_error: OnErr, +) -> Result<(), xshell::Error> { eprintln!(" {}", "Creating test objects".bold().cyan()); let test_db_name = "tsdb_toolkit_test"; @@ -82,9 +109,20 @@ pub fn create_test_objects_for_package_testing(root_config: &ConnectionConfig) { .simple_query(create) .unwrap_or_else(|e| panic!("could not install extension due to {}", e,)); + let current_toolkit_version = test_client.get_installed_extension_version(); + + test_client.set_timezone_utc(); // create test objects - let _results = test_client.create_test_objects(); - eprintln!("{}", "Finished creating objects".bold().green()); + let errors = test_client.create_test_objects_from_file(test_config, current_toolkit_version); + + for (test, error) in errors { + match error { + Ok(..) => continue, + Err(error) => on_error(test, error), + } + } + eprintln!("{}", "Finished Object Creation".bold().green()); + Ok(()) } fn connect_to(config: &ConnectionConfig<'_>) -> TestClient { @@ -98,26 +136,35 @@ fn connect_to(config: &ConnectionConfig<'_>) -> TestClient { TestClient(client) } -pub fn update_to_and_validate_new_toolkit_version(root_config: &ConnectionConfig) { +pub fn update_to_and_validate_new_toolkit_version( + current_toolkit_version: String, + root_config: &ConnectionConfig, + mut on_error: OnErr, +) -> Result<(), xshell::Error> { // update extension to new version let test_db_name = "tsdb_toolkit_test"; let test_config = root_config.with_db(test_db_name); let mut test_client = connect_to(&test_config); - test_client.update_to_current_version(); + test_client.set_timezone_utc(); + // get the currently installed version before updating + let old_toolkit_version = test_client.get_installed_extension_version(); + + test_client.update_to_current_toolkit_version(); // run validation tests - let expected_results: QueryValues = vec![vec![ - Some("100".to_string()), - Some("108".to_string()), - Some("149.5".to_string()), - Some("108.96220333142547".to_string()), - Some("109.50489521100047".to_string()), - Some("1.7995661075080858".to_string()), - ]]; - test_client.validate_test_objects(expected_results); - - eprintln!("{}", "Finished validating objects".bold().green()); + let errors = test_client.validate_test_objects_from_files(test_config, old_toolkit_version); + + for (test, error) in errors { + match error { + Ok(..) => continue, + Err(error) => on_error(test, error), + } + } + + test_client.check_no_references_to_the_old_binary_leaked(¤t_toolkit_version); + + test_client.validate_stable_objects_exist(); // This close needs to happen before trying to drop the DB or else panics with `There is 1 other session using the database.` test_client @@ -126,12 +173,13 @@ pub fn update_to_and_validate_new_toolkit_version(root_config: &ConnectionConfig .unwrap_or_else(|e| panic!("Could not close connection to postgres DB due to {}", e)); // if the validation passes, drop the db let mut client = connect_to(root_config).0; - eprintln!("{}", "Tests pass, dropping database.".bold().green()); + eprintln!("{}", "Dropping database.".bold().green()); let drop = format!(r#"DROP DATABASE IF EXISTS "{}""#, test_db_name); client .simple_query(&drop) .unwrap_or_else(|e| panic!("could not drop db {} due to {}", test_db_name, e)); + Ok(()) } //---------------// @@ -179,99 +227,143 @@ struct TestClient(Client); type QueryValues = Vec>>; impl TestClient { - fn install_toolkit_at_version(&mut self, old_version: &str) { + fn install_toolkit_at_version(&mut self, old_toolkit_version: &str) { let create = format!( r#"CREATE EXTENSION timescaledb_toolkit VERSION "{}""#, - old_version + old_toolkit_version ); self.simple_query(&create).unwrap_or_else(|e| { panic!( "could not install extension at version {} due to {}", - old_version, e, + old_toolkit_version, e, ) }); } - #[must_use] - fn create_test_objects(&mut self) -> QueryValues { - let create_data_table = "\ - CREATE TABLE test_data(ts timestamptz, val DOUBLE PRECISION);\ - INSERT INTO test_data \ - SELECT '2020-01-01 00:00:00+00'::timestamptz + i * '1 hour'::interval, \ - 100 + i % 100\ - FROM generate_series(0, 10000) i;\ - "; - self.simple_query(create_data_table) - .unwrap_or_else(|e| panic!("could create the data table due to {}", e)); - - // TODO JOSH - I want to have additional stuff for newer versions, - // but it's not ready yet - let create_test_view = "\ - CREATE MATERIALIZED VIEW regression_view AS \ - SELECT \ - counter_agg(ts, val) AS countagg, \ - hyperloglog(1024, val) AS hll, \ - time_weight('locf', ts, val) AS twa, \ - uddsketch(100, 0.001, val) as udd, \ - tdigest(100, val) as tdig, \ - stats_agg(val) as stats \ - FROM test_data;\ - "; - self.simple_query(create_test_view) - .unwrap_or_else(|e| panic!("could create the regression view due to {}", e)); - - let query_test_view = "\ - SET TIME ZONE 'UTC'; \ - SELECT \ - num_resets(countagg), \ - distinct_count(hll), \ - average(twa), \ - approx_percentile(0.1, udd), \ - approx_percentile(0.1, tdig), \ - kurtosis(stats) \ - FROM regression_view;\ - "; - let view_output = self - .simple_query(query_test_view) - .unwrap_or_else(|e| panic!("could query the regression view due to {}", e)); - get_values(view_output) + fn set_timezone_utc(&mut self) { + self.simple_query("SET TIME ZONE 'UTC';") + .unwrap_or_else(|e| panic!("could not set time zone to UTC due to {}", e)); + } + + fn create_test_objects_from_file( + &mut self, + root_config: ConnectionConfig<'_>, + current_toolkit_version: String, + ) -> Vec<(Test, Result<(), TestError>)> { + let all_tests = parser::extract_tests("tests/update"); + // Hack to match previous versions of toolkit that don't conform to Semver. + let current_toolkit_semver = match current_toolkit_version.as_str() { + "1.4" => Version { + major: 1, + minor: 4, + patch: 0, + pre: Prerelease::EMPTY, + build: BuildMetadata::EMPTY, + }, + "1.5" => Version { + major: 1, + minor: 5, + patch: 0, + pre: Prerelease::EMPTY, + build: BuildMetadata::EMPTY, + }, + "1.10.0-dev" => Version { + major: 1, + minor: 10, + patch: 0, + pre: Prerelease::EMPTY, + build: BuildMetadata::EMPTY, + }, + x => Version::parse(x).unwrap(), + }; + + let errors: Vec<_> = all_tests + .into_iter() + .flat_map(|tests| { + let mut client = connect_to(&root_config).0; + tests + .tests + .into_iter() + .filter(|x| x.creation) + .filter(|x| match &x.min_toolkit_version { + Some(version) => version <= ¤t_toolkit_semver, + None => true, + }) + .map(move |test| { + let output = run_test(&mut client, &test); + (test.clone(), output) + }) + }) + .collect(); + errors } - fn update_to_current_version(&mut self) { + fn update_to_current_toolkit_version(&mut self) { let update = "ALTER EXTENSION timescaledb_toolkit UPDATE"; self.simple_query(update) .unwrap_or_else(|e| panic!("could not update extension due to {}", e)); } - fn validate_test_objects(&mut self, validation_values: QueryValues) { - let query_test_view = "\ - SET TIME ZONE 'UTC'; \ - SELECT \ - num_resets(countagg), \ - distinct_count(hll), \ - average(twa), \ - approx_percentile(0.1, udd), \ - approx_percentile(0.1, tdig), \ - kurtosis(stats) \ - FROM regression_view;\ - "; - let view_output = self - .simple_query(query_test_view) - .unwrap_or_else(|e| panic!("could query the regression view due to {}", e)); - let new_values = get_values(view_output); - assert_eq!( - new_values, validation_values, - "values returned by the view changed on update", - ); + fn validate_test_objects_from_files( + &mut self, + root_config: ConnectionConfig<'_>, + old_toolkit_version: String, + ) -> Vec<(Test, Result<(), TestError>)> { + let all_tests = parser::extract_tests("tests/update"); + + let old_toolkit_semver = match old_toolkit_version.as_str() { + "1.4" => Version { + major: 1, + minor: 4, + patch: 0, + pre: Prerelease::EMPTY, + build: BuildMetadata::EMPTY, + }, + "1.5" => Version { + major: 1, + minor: 5, + patch: 0, + pre: Prerelease::EMPTY, + build: BuildMetadata::EMPTY, + }, + "1.10.0-dev" => Version { + major: 1, + minor: 10, + patch: 0, + pre: Prerelease::EMPTY, + build: BuildMetadata::EMPTY, + }, + x => Version::parse(x).unwrap(), + }; + let errors: Vec<_> = all_tests + .into_iter() + .flat_map(|tests| { + let mut client = connect_to(&root_config).0; + tests + .tests + .into_iter() + .filter(|x| x.validation) + .filter(|x| match &x.min_toolkit_version { + Some(min_version) => min_version <= &old_toolkit_semver, + None => true, + }) + .map(move |test| { + let output = run_test(&mut client, &test); + // ensure that the DB is dropped after the client + (test.clone(), output) + }) + }) + .collect(); + errors } - fn check_no_references_to_the_old_binary_leaked(&mut self, current_version: &str) { + fn check_no_references_to_the_old_binary_leaked(&mut self, current_toolkit_version: &str) { let query_get_leaked_objects = format!( "SELECT pg_proc.proname \ FROM pg_catalog.pg_proc \ WHERE pg_proc.probin LIKE '$libdir/timescaledb_toolkit%' \ AND pg_proc.probin <> '$libdir/timescaledb_toolkit-{}';", - current_version, + current_toolkit_version, ); let leaks = self .simple_query(&query_get_leaked_objects) @@ -362,3 +454,240 @@ fn get_values(query_results: Vec) -> QueryValues { }) .collect() } + +// Functions below this line are originally from sql-doctester/src/runner.rs + +pub fn validate_output(output: Vec, test: &Test) -> Result<(), TestError> { + use SimpleQueryMessage::*; + + let mut rows = Vec::with_capacity(test.output.len()); + for r in output { + match r { + Row(r) => { + let mut row: Vec = Vec::with_capacity(r.len()); + for i in 0..r.len() { + row.push(r.get(i).unwrap_or("").to_string()) + } + rows.push(row); + } + CommandComplete(_) => { + break; + } + _ => unreachable!(), + } + } + let output_error = |header: &str| { + format!( + "{}\n{expected}\n{}{}\n\n{received}\n{}{}\n\n{delta}\n{}", + header, + stringify_table(&test.output), + format!("({} rows)", test.output.len()).dimmed(), + stringify_table(&rows), + format!("({} rows)", rows.len()).dimmed(), + stringify_delta(&test.output, &rows), + expected = "Expected".bold().blue(), + received = "Received".bold().blue(), + delta = "Delta".bold().blue(), + ) + }; + + if test.output.len() != rows.len() { + return Err(TestError::OutputError(output_error( + "output has a different number of rows than expected.", + ))); + } + + fn clamp_len<'s>(mut col: &'s str, idx: usize, test: &Test) -> &'s str { + let max_len = test.precision_limits.get(&idx); + if let Some(&max_len) = max_len { + if col.len() > max_len { + col = &col[..max_len] + } + } + col + } + + let all_eq = test.output.iter().zip(rows.iter()).all(|(out, row)| { + out.len() == row.len() + && out + .iter() + .zip(row.iter()) + .enumerate() + .all(|(i, (o, r))| clamp_len(o, i, test) == clamp_len(r, i, test)) + }); + if !all_eq { + return Err(TestError::OutputError(output_error( + "output has a different values than expected.", + ))); + } + Ok(()) +} +fn stringify_table(table: &[Vec]) -> String { + use std::{cmp::max, fmt::Write}; + if table.is_empty() { + return "---".to_string(); + } + let mut width = vec![0; table[0].len()]; + for row in table { + // Ensure that we have width for every column + // TODO this shouldn't be needed, but sometimes is? + if width.len() < row.len() { + width.extend((0..row.len() - width.len()).map(|_| 0)); + } + for (i, value) in row.iter().enumerate() { + width[i] = max(width[i], value.len()) + } + } + let mut output = String::with_capacity(width.iter().sum::() + width.len() * 3); + for row in table { + for (i, value) in row.iter().enumerate() { + if i != 0 { + output.push_str(" | ") + } + let _ = write!(&mut output, "{:>width$}", value, width = width[i]); + } + output.push('\n') + } + + output +} + +#[allow(clippy::needless_range_loop)] +fn stringify_delta(left: &[Vec], right: &[Vec]) -> String { + use std::{cmp::max, fmt::Write}; + + static EMPTY_ROW: Vec = vec![]; + static EMPTY_VAL: String = String::new(); + + let mut width = vec![ + 0; + max( + left.get(0).map(Vec::len).unwrap_or(0), + right.get(0).map(Vec::len).unwrap_or(0) + ) + ]; + let num_rows = max(left.len(), right.len()); + for i in 0..num_rows { + let left = left.get(i).unwrap_or(&EMPTY_ROW); + let right = right.get(i).unwrap_or(&EMPTY_ROW); + let cols = max(left.len(), right.len()); + for j in 0..cols { + let left = left.get(j).unwrap_or(&EMPTY_VAL); + let right = right.get(j).unwrap_or(&EMPTY_VAL); + if left == right { + width[j] = max(width[j], left.len()) + } else { + width[j] = max(width[j], left.len() + right.len() + 2) + } + } + } + let mut output = String::with_capacity(width.iter().sum::() + width.len() * 3); + for i in 0..num_rows { + let left = left.get(i).unwrap_or(&EMPTY_ROW); + let right = right.get(i).unwrap_or(&EMPTY_ROW); + let cols = max(left.len(), right.len()); + for j in 0..cols { + let left = left.get(j).unwrap_or(&EMPTY_VAL); + let right = right.get(j).unwrap_or(&EMPTY_VAL); + if j != 0 { + let _ = write!(&mut output, " | "); + } + let (value, padding) = if left == right { + (left.to_string(), width[j] - left.len()) + } else { + let padding = width[j] - (left.len() + right.len() + 2); + let value = format!( + "{}{}{}{}", + "-".magenta(), + left.magenta(), + "+".yellow(), + right.yellow() + ); + (value, padding) + }; + // trick to ensure correct padding, the color characters are counted + // if done the normal way. + let _ = write!(&mut output, "{:>padding$}{}", "", value, padding = padding); + } + let _ = writeln!(&mut output); + } + output +} + +#[derive(Debug)] +pub enum TestError { + PgError(postgres::Error), + OutputError(String), +} + +impl fmt::Display for TestError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + TestError::PgError(error) => { + match error.source().and_then(|e| e.downcast_ref::()) { + Some(e) => { + use postgres::error::ErrorPosition::*; + let pos = match e.position() { + Some(Original(pos)) => format!("At character {}", pos), + Some(Internal { position, query }) => { + format!("In internal query `{}` at {}", query, position) + } + None => String::new(), + }; + write!( + f, + "{}\n{}\n{}\n{}", + "Postgres Error:".bold().red(), + e, + e.detail().unwrap_or(""), + pos, + ) + } + None => write!(f, "{}", error), + } + } + TestError::OutputError(err) => write!(f, "{} {}", "Error:".bold().red(), err), + } + } +} + +impl From for TestError { + fn from(error: postgres::Error) -> Self { + TestError::PgError(error) + } +} + +impl TestError { + pub fn annotate_position<'s>(&self, sql: &'s str) -> Cow<'s, str> { + match self.location() { + None => sql.into(), + Some(pos) => format!( + "{}{}{}", + &sql[..pos as usize], + "~>".bright_red(), + &sql[pos as usize..], + ) + .into(), + } + } + + fn location(&self) -> Option { + use postgres::error::ErrorPosition::*; + match self { + TestError::OutputError(..) => None, + TestError::PgError(e) => match e + .source() + .and_then(|e| e.downcast_ref::().and_then(DbError::position)) + { + None => None, + Some(Internal { .. }) => None, + Some(Original(pos)) => Some(pos.saturating_sub(1)), + }, + } + } +} + +fn run_test(client: &mut Client, test: &Test) -> Result<(), TestError> { + let output = client.simple_query(&test.text)?; + validate_output(output, test) +}