diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 000000000..2102087c8 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,3 @@ +[alias] +cross-dev = ["run", "--bin", "cross-dev", "--features", "dev", "--"] +build-docker-image = ["cross-dev", "build-docker-image"] \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a7f7c055..097ef9e92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,7 +54,7 @@ jobs: uses: actions-rs/cargo@v1 with: command: clippy - args: --locked -- -D warnings + args: --locked -- -D warnings --all-features test: runs-on: ${{ matrix.os }} @@ -74,7 +74,7 @@ jobs: uses: actions-rs/cargo@v1 with: command: test - args: --locked --all-targets --workspace + args: --locked --all-targets --workspace --all-features timeout-minutes: 5 generate-matrix: @@ -206,6 +206,11 @@ jobs: - name: Set up Docker Buildx if: runner.os == 'Linux' uses: docker/setup-buildx-action@v1 + + - name: Install cross + if: matrix.deploy + run: cargo install --path . --force --features=dev + - name: Docker Meta if: runner.os == 'Linux' id: docker-meta @@ -221,7 +226,7 @@ jobs: id: build-docker-image if: runner.os == 'Linux' timeout-minutes: 60 - run: ./build-docker-image.sh "${TARGET}" + run: cross-dev build-docker-image "${TARGET}" env: TARGET: ${{ matrix.target }} LABELS: ${{ steps.docker-meta.outputs.labels }} @@ -247,14 +252,6 @@ jobs: RUN: ${{ matrix.run }} RUNNERS: ${{ matrix.runners }} shell: bash - - - name: Install cross - if: matrix.deploy - run: cargo install --path . --force - - - name: Build cross-dev - run: cargo build --features=dev --bin cross-dev - - uses: ./.github/actions/cargo-install-upload-artifacts if: matrix.deploy with: @@ -287,7 +284,7 @@ jobs: github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || startsWith(github.ref, 'refs/tags/v') ) - run: ./build-docker-image.sh --push "${TARGET}" + run: cross-dev build-docker-image --push "${TARGET}" env: TARGET: ${{ matrix.target }} LABELS: ${{ steps.docker-meta2.outputs.labels }} diff --git a/Cargo.toml b/Cargo.toml index 75a7b0070..0288bafca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,11 +11,11 @@ edition = "2021" [features] default = [] -dev = ["serde_yaml"] +dev = ["serde_yaml", "walkdir"] [dependencies] atty = "0.2" -clap = { version = "3.1.18", features = ["derive"] } +clap = { version = "3.1.18", features = ["derive", "env"] } color-eyre = "0.6" eyre = "0.6" home = "0.5" @@ -28,6 +28,7 @@ serde_json = "1" serde_yaml = { version = "0.8", optional = true } serde_ignored = "0.1.2" shell-words = "1.1.0" +walkdir = { version = "2", optional = true } [target.'cfg(not(windows))'.dependencies] nix = { version = "0.24", default-features = false, features = ["user"] } diff --git a/build-docker-image.sh b/build-docker-image.sh deleted file mode 100755 index 8b372a3bf..000000000 --- a/build-docker-image.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env bash - -set -x -set -euo pipefail - -version="$(cargo metadata --format-version 1 --no-deps | jq --raw-output '.packages[0].version')" -targets=() -push=false - -for arg in "${@}"; do - if [[ "${arg}" == --push ]]; then - push=true - else - targets+=("${arg}") - fi -done - -pushd docker - -run() { - local push="${1}" - local build_args=() - - if "${push}"; then - build_args+=(--push) - else - build_args+=(--load) - fi - - local dockerfile="Dockerfile.${2}" - local image_name="ghcr.io/cross-rs/${2}" - - local tags=() - - case "${GITHUB_REF_TYPE-}:${GITHUB_REF_NAME-}" in - tag:v*) - local tag_version="${GITHUB_REF_NAME##v}" - - if [[ "${tag_version}" == "${version}" ]]; then - echo "Git tag does not match package version." >&2 - exit 1 - fi - - tags+=("${image_name}:${tag_version}") - - # Tag stable versions as latest. - if ! [[ "${tag_version}" =~ -.* ]]; then - tags+=("${image_name}:latest") - fi - ;; - branch:*) - # Tag active branch as edge. - tags+=("${image_name}:${GITHUB_REF_NAME}") - if ! [[ "${GITHUB_REF_NAME-}" =~ staging ]] && ! [[ "${GITHUB_REF_NAME-}" =~ trying ]]; then - tags+=("${image_name}:edge") - fi - ;; - *) - if "${push}"; then - echo "Refusing to push without tag or branch." >&2 - exit 1 - fi - - # Local development. - tags+=("${image_name}:local") - ;; - esac - - build_args+=( - --pull - --cache-from "type=registry,ref=${image_name}:main" - ) - - if "${push}"; then - build_args+=(--cache-to 'type=inline') - fi - - for tag in "${tags[@]}"; do - build_args+=(--tag "${tag}") - done - - if [ -n "${LABELS:-}" ]; then - local labels - mapfile -t labels -d '' <<< "${LABELS}" - for label in "${labels[@]}"; do - build_args+=(--label "${label}") - done - fi - - docker buildx build "${build_args[@]}" -f "${dockerfile}" --progress plain . - docker inspect "${tags[0]}" | jq -C .[0].Config.Labels - if [[ -n "${GITHUB_ACTIONS-}" ]]; then - echo "::set-output name=image::${tags[0]}" - fi -} - -if [[ "${#targets[@]}" -eq 0 ]]; then - for dockerfile in Dockerfile.*; do - target="${dockerfile##Dockerfile.}" - run "${push}" "${target}" - done -else - for target in "${targets[@]}"; do - run "${push}" "${target}" - done -fi diff --git a/cross-dev/build_docker_image.rs b/cross-dev/build_docker_image.rs new file mode 100644 index 000000000..edfa1c5ae --- /dev/null +++ b/cross-dev/build_docker_image.rs @@ -0,0 +1,158 @@ +use std::{path::Path, process::Command}; + +use clap::Args; +use cross::CommandExt; + +#[derive(Args, Debug)] +pub struct BuildDockerImage { + #[clap(long, hide = true, env = "GITHUB_REF_TYPE")] + ref_type: Option, + #[clap(long, hide = true, env = "GITHUB_REF_NAME")] + ref_name: Option, + /// Newline separated labels + #[clap(long, env = "LABELS")] + labels: Option, + /// Provide verbose diagnostic output. + #[clap(short, long)] + verbose: bool, + #[clap(long)] + force: bool, + #[clap(short, long)] + push: bool, + /// Container engine (such as docker or podman). + #[clap(long)] + pub engine: Option, + + /// Targets to build for + #[clap()] + targets: Vec, +} + +pub fn build_docker_image( + BuildDockerImage { + mut targets, + verbose, + force, + push, + ref_type, + ref_name, + labels, + .. + }: BuildDockerImage, + engine: &Path, +) -> cross::Result<()> { + let metadata = cross::cargo_metadata_with_args( + Some(Path::new(env!("CARGO_MANIFEST_DIR"))), + None, + verbose, + )? + .ok_or_else(|| eyre::eyre!("could not find cross workspace and its current version"))?; + let version = metadata + .packages + .get(0) + .expect("one package expected in workspace") + .version + .clone(); + if targets.is_empty() { + targets = walkdir::WalkDir::new(metadata.workspace_root.join("docker")) + .max_depth(1) + .contents_first(true) + .into_iter() + .filter_map(|e| e.ok().filter(|f| f.file_type().is_file())) + .filter_map(|f| { + (&f.file_name()) + .to_string_lossy() + .strip_prefix("Dockerfile.") + .map(ToOwned::to_owned) + }) + .collect(); + } + for target in targets { + let mut docker_build = Command::new(engine); + docker_build.args(&["buildx", "build"]); + docker_build.current_dir(metadata.workspace_root.join("docker")); + + if push { + docker_build.arg("--push"); + } else { + docker_build.arg("--load"); + } + + let dockerfile = format!("Dockerfile.{target}"); + let image_name = format!("ghcr.io/cross-rs/{target}"); + let mut tags = vec![]; + + match (ref_type.as_deref(), ref_name.as_deref()) { + (Some(ref_type), Some(ref_name)) if ref_type == "tag" && ref_name.starts_with('v') => { + let tag_version = ref_name + .strip_prefix('v') + .expect("tag name should start with v"); + if version != tag_version { + eyre::bail!("git tag does not match package version.") + } + tags.push(format!("{image_name}:{version}")); + // Check for unstable releases, tag stable releases as `latest` + if version.contains('-') { + // TODO: Don't tag if version is older than currently released version. + tags.push(format!("{image_name}:latest")) + } + } + (Some(ref_type), Some(ref_name)) if ref_type == "branch" => { + tags.push(format!("{image_name}:{ref_name}")); + + if ["staging", "trying"] + .iter() + .any(|branch| branch == &ref_name) + { + tags.push(format!("{image_name}:edge")); + } + } + _ => { + if push { + eyre::bail!("Refusing to push without tag or branch. {ref_type:?}:{ref_name:?}") + } + tags.push(format!("{image_name}:local")) + } + } + + docker_build.arg("--pull"); + docker_build.args(&[ + "--cache-from", + &format!("type=registry,ref={image_name}:main"), + ]); + + if push { + docker_build.args(&["--cache-to", "type=inline"]); + } + + for tag in &tags { + docker_build.args(&["--tag", tag]); + } + + for label in labels + .as_deref() + .unwrap_or("") + .split('\n') + .filter(|s| !s.is_empty()) + { + docker_build.args(&["--label", label]); + } + + docker_build.args(&["-f", &dockerfile]); + docker_build.args(&["--progress", "plain"]); + docker_build.arg("."); + + if (force || push && std::env::var("GITHUB_ACTIONS").is_ok()) || !push { + docker_build.run(verbose)?; + } else { + docker_build.print_verbose(true); + } + if std::env::var("GITHUB_ACTIONS").is_ok() { + println!("::set-output name=image::{}", &tags[0]) + } + } + if !(force || push && std::env::var("GITHUB_ACTIONS").is_ok()) { + println!("refusing to push, use --force to override"); + } + Ok(()) +} diff --git a/cross-dev/main.rs b/cross-dev/main.rs index b289cb8b0..92332fe30 100644 --- a/cross-dev/main.rs +++ b/cross-dev/main.rs @@ -1,17 +1,14 @@ #![deny(missing_debug_implementations, rust_2018_idioms)] -use std::collections::BTreeMap; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; +pub mod build_docker_image; +pub mod target_info; + +use std::path::PathBuf; use clap::{Parser, Subcommand}; -use cross::CommandExt; -use serde::Deserialize; -// Store raw text data in the binary so we don't need a data directory -// when extracting all targets, or running our target info script. -const TARGET_INFO_SCRIPT: &str = include_str!("target_info.sh"); -const WORKFLOW: &str = include_str!("../.github/workflows/ci.yml"); +use self::build_docker_image::BuildDockerImage; +use self::target_info::TargetInfo; #[derive(Parser, Debug)] #[clap(version, about, long_about = None)] @@ -23,65 +20,25 @@ struct Cli { #[derive(Subcommand, Debug)] enum Commands { /// Extract and print info for targets. - TargetInfo { - /// If not provided, get info for all targets. - targets: Vec, - /// Provide verbose diagnostic output. - #[clap(short, long)] - verbose: bool, - /// Image registry. - #[clap(long, default_value_t = String::from("ghcr.io"))] - registry: String, - /// Image repository. - #[clap(long, default_value_t = String::from("cross-rs"))] - repository: String, - /// Image tag. - #[clap(long, default_value_t = String::from("main"))] - tag: String, - /// Container engine (such as docker or podman). - #[clap(long)] - engine: Option, - }, -} - -#[derive(Debug, PartialEq, Deserialize)] -struct Workflow { - jobs: Jobs, -} - -#[derive(Debug, PartialEq, Deserialize)] -struct Jobs { - #[serde(rename = "generate-matrix")] - generate_matrix: GenerateMatrix, -} - -#[derive(Debug, PartialEq, Deserialize)] -struct GenerateMatrix { - steps: Vec, + TargetInfo(TargetInfo), + BuildDockerImage(BuildDockerImage), } -#[derive(Debug, PartialEq, Deserialize)] -struct Steps { - env: Env, -} - -#[derive(Debug, PartialEq, Deserialize)] -struct Env { - matrix: String, -} - -#[derive(Debug, PartialEq, Deserialize)] -struct Matrix { - target: String, - #[serde(default)] - run: i64, -} - -impl Matrix { - fn has_test(&self, target: &str) -> bool { - // bare-metal targets don't have unittests right now - self.run != 0 && !target.contains("-none-") +pub fn main() -> cross::Result<()> { + cross::install_panic_hook()?; + let cli = Cli::parse(); + match cli.command { + Commands::TargetInfo(args) => { + let engine = get_container_engine(args.engine.as_deref())?; + target_info::target_info(args, &engine)?; + } + Commands::BuildDockerImage(args) => { + let engine = get_container_engine(args.engine.as_deref())?; + build_docker_image::build_docker_image(args, &engine)?; + } } + + Ok(()) } fn get_container_engine(engine: Option<&str>) -> Result { @@ -91,126 +48,3 @@ fn get_container_engine(engine: Option<&str>) -> Result { cross::get_container_engine() } } - -fn target_has_image(target: &str) -> bool { - let imageless = ["-msvc", "-darwin", "-apple-ios"]; - !imageless.iter().any(|t| target.ends_with(t)) -} - -fn format_image(registry: &str, repository: &str, target: &str, tag: &str) -> String { - let mut output = format!("{target}:{tag}"); - if !repository.is_empty() { - output = format!("{repository}/{output}"); - } - if !registry.is_empty() { - output = format!("{registry}/{output}"); - } - - output -} - -fn pull_image(engine: &Path, image: &str, verbose: bool) -> cross::Result<()> { - let mut command = Command::new(engine); - command.arg("pull"); - command.arg(image); - if !verbose { - // capture output to avoid polluting table - command.stdout(Stdio::null()); - command.stderr(Stdio::null()); - } - command.run(verbose) -} - -fn image_info( - engine: &Path, - target: &str, - image: &str, - tag: &str, - verbose: bool, - has_test: bool, -) -> cross::Result<()> { - if !tag.starts_with("local") { - pull_image(engine, image, verbose)?; - } - - let mut command = Command::new(engine); - command.arg("run"); - command.arg("-it"); - command.arg("--rm"); - command.args(&["-e", &format!("TARGET={target}")]); - if has_test { - command.args(&["-e", "HAS_TEST=1"]); - } - command.arg(image); - command.args(&["bash", "-c", TARGET_INFO_SCRIPT]); - - if !verbose { - // capture stderr to avoid polluting table - command.stderr(Stdio::null()); - } - command.run(verbose) -} - -fn target_info( - mut targets: Vec, - engine: &Path, - verbose: bool, - registry: &str, - repository: &str, - tag: &str, -) -> cross::Result<()> { - let workflow: Workflow = serde_yaml::from_str(WORKFLOW)?; - let matrix = &workflow.jobs.generate_matrix.steps[0].env.matrix; - let matrix: Vec = serde_yaml::from_str(matrix)?; - let test_map: BTreeMap<&str, bool> = matrix - .iter() - .map(|i| (i.target.as_ref(), i.has_test(&i.target))) - .collect(); - - if targets.is_empty() { - targets = matrix - .iter() - .map(|t| t.target.clone()) - .filter(|t| target_has_image(t)) - .collect(); - } - - for target in targets { - let target = target.as_ref(); - let image = format_image(registry, repository, target, tag); - let has_test = test_map - .get(&target) - .cloned() - .ok_or_else(|| eyre::eyre!("invalid target name {}", target))?; - image_info(engine, target, &image, tag, verbose, has_test)?; - } - - Ok(()) -} - -pub fn main() -> cross::Result<()> { - cross::install_panic_hook()?; - let cli = Cli::parse(); - match &cli.command { - Commands::TargetInfo { - targets, - verbose, - registry, - repository, - tag, - engine, - } => { - let engine = get_container_engine(engine.as_deref())?; - target_info( - targets.to_vec(), - &engine, - *verbose, - registry, - repository, - tag, - )?; - } - } - - Ok(()) -} diff --git a/cross-dev/target_info.rs b/cross-dev/target_info.rs new file mode 100644 index 000000000..c44522cf6 --- /dev/null +++ b/cross-dev/target_info.rs @@ -0,0 +1,174 @@ +use std::{ + collections::BTreeMap, + path::Path, + process::{Command, Stdio}, +}; + +use clap::Args; +use cross::CommandExt; +use serde::Deserialize; + +// Store raw text data in the binary so we don't need a data directory +// when extracting all targets, or running our target info script. +const TARGET_INFO_SCRIPT: &str = include_str!("target_info.sh"); +const WORKFLOW: &str = include_str!("../.github/workflows/ci.yml"); + +#[derive(Args, Debug)] +pub struct TargetInfo { + /// If not provided, get info for all targets. + targets: Vec, + /// Provide verbose diagnostic output. + #[clap(short, long)] + verbose: bool, + /// Image registry. + #[clap(long, default_value_t = String::from("ghcr.io"))] + registry: String, + /// Image repository. + #[clap(long, default_value_t = String::from("cross-rs"))] + repository: String, + /// Image tag. + #[clap(long, default_value_t = String::from("main"))] + tag: String, + /// Container engine (such as docker or podman). + #[clap(long)] + pub engine: Option, +} + +#[derive(Debug, PartialEq, Eq, Deserialize)] +struct Workflow { + jobs: Jobs, +} + +#[derive(Debug, PartialEq, Eq, Deserialize)] +struct Jobs { + #[serde(rename = "generate-matrix")] + generate_matrix: GenerateMatrix, +} + +#[derive(Debug, PartialEq, Eq, Deserialize)] +struct GenerateMatrix { + steps: Vec, +} + +#[derive(Debug, PartialEq, Eq, Deserialize)] +struct Steps { + env: Env, +} + +#[derive(Debug, PartialEq, Eq, Deserialize)] +struct Env { + matrix: String, +} + +#[derive(Debug, PartialEq, Eq, Deserialize)] +struct Matrix { + target: String, + #[serde(default)] + run: i64, +} + +impl Matrix { + fn has_test(&self, target: &str) -> bool { + // bare-metal targets don't have unittests right now + self.run != 0 && !target.contains("-none-") + } +} + +fn target_has_image(target: &str) -> bool { + let imageless = ["-msvc", "-darwin", "-apple-ios"]; + !imageless.iter().any(|t| target.ends_with(t)) +} + +fn format_image(registry: &str, repository: &str, target: &str, tag: &str) -> String { + let mut output = format!("{target}:{tag}"); + if !repository.is_empty() { + output = format!("{repository}/{output}"); + } + if !registry.is_empty() { + output = format!("{registry}/{output}"); + } + + output +} + +fn pull_image(engine: &Path, image: &str, verbose: bool) -> cross::Result<()> { + let mut command = Command::new(engine); + command.arg("pull"); + command.arg(image); + if !verbose { + // capture output to avoid polluting table + command.stdout(Stdio::null()); + command.stderr(Stdio::null()); + } + command.run(verbose) +} + +fn image_info( + engine: &Path, + target: &str, + image: &str, + tag: &str, + verbose: bool, + has_test: bool, +) -> cross::Result<()> { + if !tag.starts_with("local") { + pull_image(engine, image, verbose)?; + } + + let mut command = Command::new(engine); + command.arg("run"); + command.arg("-it"); + command.arg("--rm"); + command.args(&["-e", &format!("TARGET={target}")]); + if has_test { + command.args(&["-e", "HAS_TEST=1"]); + } + command.arg(image); + command.args(&["bash", "-c", TARGET_INFO_SCRIPT]); + + if !verbose { + // capture stderr to avoid polluting table + command.stderr(Stdio::null()); + } + command.run(verbose) +} + +pub fn target_info( + TargetInfo { + mut targets, + verbose, + registry, + repository, + tag, + .. + }: TargetInfo, + engine: &Path, +) -> cross::Result<()> { + let workflow: Workflow = serde_yaml::from_str(WORKFLOW)?; + let matrix = &workflow.jobs.generate_matrix.steps[0].env.matrix; + let matrix: Vec = serde_yaml::from_str(matrix)?; + let test_map: BTreeMap<&str, bool> = matrix + .iter() + .map(|i| (i.target.as_ref(), i.has_test(&i.target))) + .collect(); + + if targets.is_empty() { + targets = matrix + .iter() + .map(|t| t.target.clone()) + .filter(|t| target_has_image(t)) + .collect(); + } + + for target in targets { + let target = target.as_ref(); + let image = format_image(®istry, &repository, target, &tag); + let has_test = test_map + .get(&target) + .cloned() + .ok_or_else(|| eyre::eyre!("invalid target name {}", target))?; + image_info(engine, target, &image, &tag, verbose, has_test)?; + } + + Ok(()) +} diff --git a/src/cargo.rs b/src/cargo.rs index 544dabc31..4aea83f33 100644 --- a/src/cargo.rs +++ b/src/cargo.rs @@ -76,9 +76,10 @@ impl CargoMetadata { #[derive(Debug, Deserialize)] pub struct Package { - id: String, - manifest_path: PathBuf, - source: Option, + pub id: String, + pub manifest_path: PathBuf, + pub source: Option, + pub version: String, } impl Package { diff --git a/src/lib.rs b/src/lib.rs index f4642c6ff..b80f7e3cf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -39,7 +39,7 @@ use config::Config; use rustc_version::Channel; use serde::Deserialize; -use self::cargo::{CargoMetadata, Subcommand}; +pub use self::cargo::{cargo_metadata_with_args, CargoMetadata, Subcommand}; use self::cross_toml::CrossToml; use self::errors::Context; use self::rustc::{TargetList, VersionMetaExt}; @@ -295,7 +295,7 @@ pub fn run() -> Result { let host_version_meta = rustc_version::version_meta().wrap_err("couldn't fetch the `rustc` version")?; let cwd = std::env::current_dir()?; - if let Some(metadata) = cargo::cargo_metadata_with_args(None, Some(&args), verbose)? { + if let Some(metadata) = cargo_metadata_with_args(None, Some(&args), verbose)? { let host = host_version_meta.host(); let toml = toml(&metadata)?; let config = Config::new(toml); diff --git a/src/tests.rs b/src/tests.rs index 42f0aadba..c6330c838 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -14,7 +14,7 @@ static WORKSPACE: OnceCell = OnceCell::new(); pub fn get_cargo_workspace() -> &'static Path { let manifest_dir = env!("CARGO_MANIFEST_DIR"); WORKSPACE.get_or_init(|| { - crate::cargo::cargo_metadata_with_args(Some(manifest_dir.as_ref()), None, true) + crate::cargo_metadata_with_args(Some(manifest_dir.as_ref()), None, true) .unwrap() .unwrap() .workspace_root