Merge #522

522: Move update-tester tests to markdown files r=thatzopoulos a=thatzopoulos

Changes to the update-tester:

- Hardcoded tests are now read in from markdown files, similar to sql-doctester
- Update-tester will now set all timezones to UTC before running sql scripts
- Ability to set a min_toolkit_version that tests can run on

Co-authored-by: Thomas Hatzopoulos <thomas@timescale.com>

tests/update/original_update_tests.md

@@ -0,0 +1,41 @@
+# Original Update Tests
+
+
+
+```sql,creation,min-toolkit-version=1.4.0
+SET TIME ZONE 'UTC';
+CREATE TABLE test_data(ts timestamptz, val DOUBLE PRECISION);
+    INSERT INTO test_data
+        SELECT '2020-01-01 00:00:00+00'::timestamptz + i * '1 hour'::interval,
+        100 + i % 100
+    FROM generate_series(0, 10000) i;
+	
+CREATE MATERIALIZED VIEW regression_view AS
+    SELECT
+        counter_agg(ts, val) AS countagg,
+        hyperloglog(1024, val) AS hll,
+        time_weight('locf', ts, val) AS twa,
+        uddsketch(100, 0.001, val) as udd,
+        tdigest(100, val) as tdig,
+        stats_agg(val) as stats
+    FROM test_data;
+```
+
+
+
+```sql,validation,min-toolkit-version=1.4.0
+SELECT
+    num_resets(countagg),
+    distinct_count(hll),
+    average(twa),
+    approx_percentile(0.1, udd),
+    approx_percentile(0.1, tdig),
+    kurtosis(stats)
+FROM regression_view;
+```
+
+```output
+ num_resets | distinct_count | average | approx_percentile  | approx_percentile  |      kurtosis
+------------+----------------+---------+--------------------+--------------------+--------------------
+        100 |            100 |   149.5 | 108.96220333142547 | 109.50489521100047 | 1.7995661075080858
+```

tools/update-tester/Cargo.toml

@@ -15,3 +15,6 @@ postgres = "0.19.1"
 semver = "1.0.9"
 toml_edit = "0.14.3"
 xshell = "0.1.14"
+pulldown-cmark = "0.8.0"
+walkdir = "2.3.2"
+bytecount = "0.6.3"

tools/update-tester/src/main.rs

@@ -32,6 +32,7 @@ macro_rules! path {
 }
 
 mod installer;
+mod parser;
 mod testrunner;
 
 fn main() {
@@ -149,6 +150,8 @@ fn main() {
                     .long("database")
                     .takes_value(true)
             )
+
+            .arg(Arg::new("ROOT_DIR").takes_value(true).default_value("."))
 	)
 // Mutates help, removing the short flag (-h) so that it can be used by HOST
 	.mut_arg("help", |_h| {
@@ -188,6 +191,21 @@ fn main() {
                 .value_of("CARGO_PGX_OLD")
                 .expect("missing cargo_pgx_old");
 
+            let mut num_errors = 0;
+            let stdout = io::stdout();
+            let mut out = stdout.lock();
+            let on_error = |test: parser::Test, error: testrunner::TestError| {
+                num_errors += 1;
+                let _ = writeln!(
+                    &mut out,
+                    "{} {}\n",
+                    test.location.bold().blue(),
+                    test.header.bold().dimmed()
+                );
+                let _ = writeln!(&mut out, "{}", error.annotate_position(&test.text));
+                let _ = writeln!(&mut out, "{}\n", error);
+            };
+
             let res = try_main(
                 root_dir,
                 cache_dir,
@@ -196,11 +214,16 @@ fn main() {
                 cargo_pgx,
                 cargo_pgx_old,
                 reinstall,
+                on_error,
             );
             if let Err(err) = res {
                 eprintln!("{}", err);
                 process::exit(1);
             }
+            if num_errors > 0 {
+                process::exit(1)
+            }
+            let _ = writeln!(&mut out, "{}\n", "Tests Passed".bold().green());
         }
         Some(("create-test-objects", create_test_object_matches)) => {
             let connection_config = ConnectionConfig {
@@ -211,11 +234,35 @@ fn main() {
                 database: create_test_object_matches.value_of("DB"),
             };
 
-            let res = try_create_objects(&connection_config);
+            let mut num_errors = 0;
+            let stdout = io::stdout();
+            let mut out = stdout.lock();
+            let on_error = |test: parser::Test, error: testrunner::TestError| {
+                num_errors += 1;
+                let _ = writeln!(
+                    &mut out,
+                    "{} {}\n",
+                    test.location.bold().blue(),
+                    test.header.bold().dimmed()
+                );
+                let _ = writeln!(&mut out, "{}", error.annotate_position(&test.text));
+                let _ = writeln!(&mut out, "{}\n", error);
+            };
+
+            let res = try_create_objects(&connection_config, on_error);
             if let Err(err) = res {
                 eprintln!("{}", err);
                 process::exit(1);
             }
+            if num_errors > 0 {
+                let _ = writeln!(&mut out, "{}\n", "Object Creation Failed".bold().red());
+                process::exit(1)
+            }
+            let _ = writeln!(
+                &mut out,
+                "{}\n",
+                "Objects Created Successfully".bold().green()
+            );
         }
         Some(("validate-test-objects", validate_test_object_matches)) => {
             let connection_config = ConnectionConfig {
@@ -225,25 +272,54 @@ fn main() {
                 password: validate_test_object_matches.value_of("PASSWORD"),
                 database: validate_test_object_matches.value_of("DB"),
             };
+            let mut num_errors = 0;
+            let stdout = io::stdout();
+            let mut out = stdout.lock();
+            let on_error = |test: parser::Test, error: testrunner::TestError| {
+                num_errors += 1;
+                let _ = writeln!(
+                    &mut out,
+                    "{} {}\n",
+                    test.location.bold().blue(),
+                    test.header.bold().dimmed()
+                );
+                let _ = writeln!(&mut out, "{}", error.annotate_position(&test.text));
+                let _ = writeln!(&mut out, "{}\n", error);
+            };
 
-            let res = try_validate_objects(&connection_config);
+            let root_dir = validate_test_object_matches
+                .value_of("ROOT_DIR")
+                .expect("missing path to root of the toolkit repo");
+            let res = try_validate_objects(&connection_config, root_dir, on_error);
             if let Err(err) = res {
                 eprintln!("{}", err);
                 process::exit(1);
             }
+            if num_errors > 0 {
+                let _ = writeln!(&mut out, "{}\n", "Validation Failed".bold().red());
+                process::exit(1)
+            }
+
+            let _ = writeln!(
+                &mut out,
+                "{}\n",
+                "Validations Completed Successfully".bold().green()
+            );
         }
         _ => unreachable!(), // if all subcommands are defined, anything else is unreachable
     }
 }
 
-fn try_main(
+#[allow(clippy::too_many_arguments)]
+fn try_main<OnErr: FnMut(parser::Test, testrunner::TestError)>(
     root_dir: &str,
     cache_dir: Option<&str>,
     db_conn: &ConnectionConfig<'_>,
     pg_config: &str,
     cargo_pgx: &str,
     cargo_pgx_old: &str,
     reinstall: HashSet<&str>,
+    on_error: OnErr,
 ) -> xshell::Result<()> {
     let (current_version, old_versions) = get_version_info(root_dir)?;
     if old_versions.is_empty() {
@@ -263,16 +339,25 @@ fn try_main(
         &reinstall,
     )?;
 
-    testrunner::run_update_tests(db_conn, current_version, old_versions)
+    testrunner::run_update_tests(db_conn, current_version, old_versions, on_error)
 }
-fn try_create_objects(db_conn: &ConnectionConfig<'_>) -> xshell::Result<()> {
-    testrunner::create_test_objects_for_package_testing(db_conn);
-    Ok(())
+fn try_create_objects<OnErr: FnMut(parser::Test, testrunner::TestError)>(
+    db_conn: &ConnectionConfig<'_>,
+    on_error: OnErr,
+) -> xshell::Result<()> {
+    testrunner::create_test_objects_for_package_testing(db_conn, on_error)
 }
 
-fn try_validate_objects(db_conn: &ConnectionConfig<'_>) -> xshell::Result<()> {
-    testrunner::update_to_and_validate_new_toolkit_version(db_conn);
-    Ok(())
+fn try_validate_objects<OnErr: FnMut(parser::Test, testrunner::TestError)>(
+    _conn: &ConnectionConfig<'_>,
+    root_dir: &str,
+    on_error: OnErr,
+) -> xshell::Result<()> {
+    let (current_version, old_versions) = get_version_info(root_dir)?;
+    if old_versions.is_empty() {
+        panic!("no old versions to upgrade from")
+    }
+    testrunner::update_to_and_validate_new_toolkit_version(current_version, _conn, on_error)
 }
 
 fn get_version_info(root_dir: &str) -> xshell::Result<(String, Vec<String>)> {