[wip] gather data on how frequently the compiler uses library feature…

…s that aren't in beta

libs_features.txt

@@ -0,0 +1,66 @@
+dispatch_from_dyn
+cell_leak
+pointer_byte_offsets
+alloc_error_handler
+vec_into_raw_parts
+nonzero_ops
+result_option_inspect
+iter_is_partitioned
+hasher_prefixfree_extras
+exact_size_is_empty
+slice_internals
+proc_macro_diagnostic
+strict_provenance
+iter_order_by
+try_reserve_kind
+const_default_impls
+slice_ptr_get
+maybe_uninit_uninit_array
+allocator_api
+control_flow_enum
+hash_drain_filter
+drain_filter
+is_terminal
+unwrap_infallible
+assert_matches
+trusted_step
+const_option
+const_btree_len
+core_intrinsics
+int_roundings
+iterator_try_reduce
+proc_macro_quote
+alloc_layout_extra
+internal_output_capture
+unsize
+is_some_and
+generator_trait
+thread_id_value
+hash_raw_entry
+map_try_insert
+test
+array_windows
+trusted_len
+slice_partition_dedup
+discriminant_kind
+iter_from_generator
+maybe_uninit_slice
+never_type
+iter_intersperse
+proc_macro_span
+once_cell
+thread_spawn_unchecked
+map_many_mut
+const_trait_impl
+proc_macro_internals
+step_trait
+fmt_helpers_for_derive
+slice_as_chunks
+new_uninit
+option_get_or_insert_default
+variant_count
+is_sorted
+get_mut_unchecked
+const_black_box
+coerce_unsized
+extend_one

src/tools/tidy/src/features.rs

@@ -22,7 +22,7 @@ use regex::Regex;
 mod tests;
 
 mod version;
-use version::Version;
+pub use version::Version;
 
 const FEATURE_GROUP_START_PREFIX: &str = "// feature-group-start";
 const FEATURE_GROUP_END_PREFIX: &str = "// feature-group-end";

src/tools/tidy/src/main.rs

@@ -4,16 +4,21 @@
 //! etc. This is run by default on `./x.py test` and as part of the auto
 //! builders. The tidy checks can be executed with `./x.py test tidy`.
 
+use regex::Regex;
+use tidy::features::Version;
+use tidy::features::{collect_lang_features, Status};
+use tidy::walk::walk;
 use tidy::*;
 
-use std::collections::VecDeque;
+use std::collections::{HashSet, VecDeque};
 use std::env;
+use std::ffi::OsStr;
 use std::num::NonZeroUsize;
 use std::path::PathBuf;
 use std::process;
 use std::str::FromStr;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::thread::{scope, ScopedJoinHandle};
+// use std::sync::atomic::{AtomicBool, Ordering};
+// use std::thread::{scope, ScopedJoinHandle};
 
 fn main() {
     let root_path: PathBuf = env::args_os().nth(1).expect("need path to root of repo").into();
@@ -33,85 +38,143 @@ fn main() {
     let verbose = args.iter().any(|s| *s == "--verbose");
     let bless = args.iter().any(|s| *s == "--bless");
 
-    let bad = std::sync::Arc::new(AtomicBool::new(false));
+    // let bad = std::sync::Arc::new(AtomicBool::new(false));
+    let mut bad = false;
+    let current_version: Version = include_str!("../../../version").trim_end().parse().unwrap();
 
-    scope(|s| {
-        let mut handles: VecDeque<ScopedJoinHandle<'_, ()>> =
-            VecDeque::with_capacity(concurrency.get());
+    let all_lang_features = collect_lang_features(&compiler_path, &mut bad);
+    assert!(!all_lang_features.is_empty());
 
-        macro_rules! check {
-            ($p:ident $(, $args:expr)* ) => {
-                while handles.len() >= concurrency.get() {
-                    handles.pop_front().unwrap().join().unwrap();
-                }
-
-                let handle = s.spawn(|| {
-                    let mut flag = false;
-                    $p::check($($args),* , &mut flag);
-                    if (flag) {
-                        bad.store(true, Ordering::Relaxed);
-                    }
-                });
-                handles.push_back(handle);
-            }
-        }
-
-        check!(target_specific_tests, &src_path);
+    let all_lib_features = features::collect_lib_features(&library_path);
+    assert!(!all_lib_features.is_empty());
 
-        // Checks that are done on the cargo workspace.
-        check!(deps, &root_path, &cargo);
-        check!(extdeps, &root_path);
-
-        // Checks over tests.
-        check!(debug_artifacts, &src_path);
-        check!(ui_tests, &src_path);
-        check!(mir_opt_tests, &src_path, bless);
-
-        // Checks that only make sense for the compiler.
-        check!(errors, &compiler_path);
-        check!(error_codes_check, &[&src_path, &compiler_path]);
-
-        // Checks that only make sense for the std libs.
-        check!(pal, &library_path);
-        check!(primitive_docs, &library_path);
-
-        // Checks that need to be done for both the compiler and std libraries.
-        check!(unit_tests, &src_path);
-        check!(unit_tests, &compiler_path);
-        check!(unit_tests, &library_path);
-
-        if bins::check_filesystem_support(&[&root_path], &output_directory) {
-            check!(bins, &root_path);
-        }
+    let mut num_features = 0;
+    let mut unstable_libs_features = HashSet::new();
 
-        check!(style, &src_path);
-        check!(style, &compiler_path);
-        check!(style, &library_path);
-
-        check!(edition, &src_path);
-        check!(edition, &compiler_path);
-        check!(edition, &library_path);
-
-        check!(alphabetical, &src_path);
-        check!(alphabetical, &compiler_path);
-        check!(alphabetical, &library_path);
-
-        let collected = {
-            while handles.len() >= concurrency.get() {
-                handles.pop_front().unwrap().join().unwrap();
+    let mut check_feature = |name: &str| {
+        num_features += 1;
+        let lib_feature = match all_lib_features.get(name) {
+            Some(feat) => feat,
+            None => {
+                assert!(all_lang_features.contains_key(name), "unknown feature {name}");
+                return;
             }
-            let mut flag = false;
-            let r = features::check(&src_path, &compiler_path, &library_path, &mut flag, verbose);
-            if flag {
-                bad.store(true, Ordering::Relaxed);
-            }
-            r
         };
-        check!(unstable_book, &src_path, collected);
-    });
-
-    if bad.load(Ordering::Relaxed) {
-        eprintln!("some tidy checks failed");
-        process::exit(1);
+        match lib_feature.level {
+            Status::Removed => panic!("using removed feature"),
+            Status::Stable => {} // totally fine
+            Status::Unstable => {
+                // let stable_date = lib_feature.since.unwrap_or_else(|| panic!("missing `since` for {name}"));
+                // if stable_date == current_version {
+                // num_just_added_features += 1;
+                unstable_libs_features.insert(name.to_owned());
+                // }
+            }
+        }
+    };
+
+    let feature_regex = Regex::new(r"^\s*#!\[.*feature\(([a-zA-Z][^)]*)\)]").unwrap();
+    walk(
+        &compiler_path,
+        &mut |path| path.is_file() && path.extension() != Some(OsStr::new("rs")),
+        &mut |entry, contents| {
+            for line in contents.lines() {
+                if let Some(captures) = feature_regex.captures(line) {
+                    let features = captures.get(1).unwrap();
+                    for feature in features.as_str().split(",") {
+                        check_feature(feature.trim());
+                    }
+                }
+            }
+        },
+    );
+
+    assert!(num_features > 0);
+    println!(
+        "found {} library features used in the compiler",
+        unstable_libs_features.len()
+    );
+    for feature in unstable_libs_features {
+        println!("{feature}");
     }
+
+    // scope(|s| {
+    //     let mut handles: VecDeque<ScopedJoinHandle<'_, ()>> =
+    //         VecDeque::with_capacity(concurrency.get());
+
+    //     macro_rules! check {
+    //         ($p:ident $(, $args:expr)* ) => {
+    //             while handles.len() >= concurrency.get() {
+    //                 handles.pop_front().unwrap().join().unwrap();
+    //             }
+
+    //             let handle = s.spawn(|| {
+    //                 let mut flag = false;
+    //                 $p::check($($args),* , &mut flag);
+    //                 if (flag) {
+    //                     bad.store(true, Ordering::Relaxed);
+    //                 }
+    //             });
+    //             handles.push_back(handle);
+    //         }
+    //     }
+
+    // check!(target_specific_tests, &src_path);
+
+    // // Checks that are done on the cargo workspace.
+    // check!(deps, &root_path, &cargo);
+    // check!(extdeps, &root_path);
+
+    // // Checks over tests.
+    // check!(debug_artifacts, &src_path);
+    // check!(ui_tests, &src_path);
+    // check!(mir_opt_tests, &src_path, bless);
+
+    // // Checks that only make sense for the compiler.
+    // check!(errors, &compiler_path);
+    // check!(error_codes_check, &[&src_path, &compiler_path]);
+
+    // // Checks that only make sense for the std libs.
+    // check!(pal, &library_path);
+    // check!(primitive_docs, &library_path);
+
+    // // Checks that need to be done for both the compiler and std libraries.
+    // check!(unit_tests, &src_path);
+    // check!(unit_tests, &compiler_path);
+    // check!(unit_tests, &library_path);
+
+    // if bins::check_filesystem_support(&[&root_path], &output_directory) {
+    //     check!(bins, &root_path);
+    // }
+
+    // check!(style, &src_path);
+    // check!(style, &compiler_path);
+    // check!(style, &library_path);
+
+    // check!(edition, &src_path);
+    // check!(edition, &compiler_path);
+    // check!(edition, &library_path);
+
+    // check!(alphabetical, &src_path);
+    // check!(alphabetical, &compiler_path);
+    // check!(alphabetical, &library_path);
+
+    // let collected = {
+    //     while handles.len() >= concurrency.get() {
+    //         handles.pop_front().unwrap().join().unwrap();
+    //     }
+    //     let mut flag = false;
+    //     let r = features::check(&src_path, &compiler_path, &library_path, &mut flag, verbose);
+    //     if flag {
+    //         bad.store(true, Ordering::Relaxed);
+    //     }
+    //     r
+    // };
+    // check!(unstable_book, &src_path, collected);
+    // });
+
+    // if bad.load(Ordering::Relaxed) {
+    //     eprintln!("some tidy checks failed");
+    //     process::exit(1);
+    // }
 }

unstable_feature_versions.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+while read -r feature; do
+    echo "Looking for $feature (commit is $commit)"
+    # Commit where this feature was first introduced.
+    commit=$(git log --pretty=%H --reverse --pickaxe-regex -S 'feature\s*=\s*"'$feature \
+        -- library/ src/lib{core,alloc,std,test,proc_macro,unwind,stdarch,unwind,rtstartup,portable-simd,panic_unwind,panic_abort,backtrace} \
+        | tee /dev/tty | head -n1 | tee /dev/tty)
+    echo "$feature introduced in $commit"
+    # src/version was first introduced in 1.48. Before that we have to parse `channel.rs`.
+    # As a hack, just pretend that any feature introduced earlier was introduced in 1.47; we don't
+    # actually care about the exact version, just whether it was used in the compiler on beta or
+    # nightly.
+    if ! version=$(git show $commit:src/version 2>/dev/null); then
+        if ! [ -e src/version ]; then
+            # We are *currently* in 1.47 or earlier; no version is reliable.
+            # TODO: parse `CFG_RELEASE_NUM` instead.
+            echo "error: detecting versions not supported for compiler versions earlier than 1.48"
+            exit 1
+        fi
+        version="1.47.0"
+    fi
+    echo "$feature introduced in $version"
+done < libs_features.txt