WIP: [offline] implement one-file-per-query

.github/workflows/sqlx.yml

@@ -170,16 +170,47 @@ jobs:
             target
           key: ${{ runner.os }}-sqlite-${{ matrix.runtime }}-${{ matrix.tls }}-${{ hashFiles('**/Cargo.lock') }}
 
+      # Create data dir for offline mode
+      - run: mkdir .sqlx
+
+      - uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: >
+            --no-default-features
+            --features any,macros,migrate,sqlite,all-types,offline,runtime-${{ matrix.runtime }}-${{ matrix.tls }}
+            --
+            --test-threads=1
+        env:
+          DATABASE_URL: sqlite://tests/sqlite/sqlite.db
+
+      # Remove test artifacts
+      - run: cargo clean -p sqlx
+
+      # Build the macros-test in offline mode (omit DATABASE_URL)
+      - uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: >
+            --no-default-features
+            --test sqlite-macros
+            --features any,macros,migrate,sqlite,all-types,offline,runtime-${{ matrix.runtime }}-${{ matrix.tls }}
+        env:
+          SQLX_OFFLINE: true
+
+      # Test macros in offline mode (still needs DATABASE_URL to run)
       - uses: actions-rs/cargo@v1
         with:
           command: test
           args: >
             --no-default-features
-            --features any,macros,migrate,sqlite,all-types,runtime-${{ matrix.runtime }}-${{ matrix.tls }}
+            --test sqlite-macros
+            --features any,macros,migrate,sqlite,all-types,offline,runtime-${{ matrix.runtime }}-${{ matrix.tls }}
             --
             --test-threads=1
         env:
           DATABASE_URL: sqlite://tests/sqlite/sqlite.db
+          SQLX_OFFLINE: true
 
   postgres:
     name: Postgres
@@ -207,6 +238,9 @@ jobs:
             target
           key: ${{ runner.os }}-postgres-${{ matrix.runtime }}-${{ matrix.tls }}-${{ hashFiles('**/Cargo.lock') }}
 
+      # Create data dir for offline mode
+      - run: mkdir .sqlx
+
       - uses: actions-rs/cargo@v1
         env:
           # FIXME: needed to disable `ltree` tests in Postgres 9.6
@@ -226,7 +260,7 @@ jobs:
           command: test
           args: >
             --no-default-features
-            --features any,postgres,macros,all-types,runtime-${{ matrix.runtime }}-${{ matrix.tls }}
+            --features any,postgres,macros,all-types,offline,runtime-${{ matrix.runtime }}-${{ matrix.tls }}
         env:
           DATABASE_URL: postgres://postgres:password@localhost:5432/sqlx
           # FIXME: needed to disable `ltree` tests in Postgres 9.6
@@ -238,13 +272,45 @@ jobs:
           command: test
           args: >
             --no-default-features
-            --features any,postgres,macros,migrate,all-types,runtime-${{ matrix.runtime }}-${{ matrix.tls }}
+            --features any,postgres,macros,migrate,all-types,offline,runtime-${{ matrix.runtime }}-${{ matrix.tls }}
         env:
           DATABASE_URL: postgres://postgres:password@localhost:5432/sqlx?sslmode=verify-ca&sslrootcert=.%2Ftests%2Fcerts%2Fca.crt
           # FIXME: needed to disable `ltree` tests in Postgres 9.6
           # but `PgLTree` should just fall back to text format
           RUSTFLAGS: --cfg postgres_${{ matrix.postgres }}
 
+      # Remove test artifacts
+      - run: cargo clean -p sqlx
+
+      # Build the macros-test in offline mode (omit DATABASE_URL)
+      - uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: >
+            --no-default-features
+            --test postgres-macros
+            --features any,postgres,macros,migrate,all-types,offline,runtime-${{ matrix.runtime }}-${{ matrix.tls }}
+        env:
+          SQLX_OFFLINE: true
+          # FIXME: needed to disable `ltree` tests in Postgres 9.6
+          # but `PgLTree` should just fall back to text format
+          RUSTFLAGS: --cfg postgres_${{ matrix.postgres }}
+
+      # Test macros in offline mode (still needs DATABASE_URL to run)
+      - uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: >
+            --no-default-features
+            --test postgres-macros
+            --features any,postgres,macros,migrate,all-types,offline,runtime-${{ matrix.runtime }}-${{ matrix.tls }}
+        env:
+          DATABASE_URL: postgres://postgres:password@localhost:5432/sqlx
+          SQLX_OFFLINE: true
+          # FIXME: needed to disable `ltree` tests in Postgres 9.6
+          # but `PgLTree` should just fall back to text format
+          RUSTFLAGS: --cfg postgres_${{ matrix.postgres }}
+
   mysql:
     name: MySQL
     runs-on: ubuntu-20.04
@@ -271,6 +337,9 @@ jobs:
             target
           key: ${{ runner.os }}-mysql-${{ matrix.runtime }}-${{ matrix.tls }}-${{ hashFiles('**/Cargo.lock') }}
 
+      # Create data dir for offline mode
+      - run: mkdir .sqlx
+
       - uses: actions-rs/cargo@v1
         with:
           command: build
@@ -285,10 +354,36 @@ jobs:
           command: test
           args: >
             --no-default-features
-            --features any,mysql,macros,migrate,all-types,runtime-${{ matrix.runtime }}-${{ matrix.tls }}
+            --features any,mysql,macros,migrate,all-types,offline,runtime-${{ matrix.runtime }}-${{ matrix.tls }}
         env:
           DATABASE_URL: mysql://root:password@localhost:3306/sqlx
 
+      # Remove test artifacts
+      - run: cargo clean -p sqlx
+
+      # Build the macros-test in offline mode (omit DATABASE_URL)
+      - uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: >
+            --no-default-features
+            --test mysql-macros
+            --features any,mysql,macros,migrate,all-types,offline,runtime-${{ matrix.runtime }}-${{ matrix.tls }}
+        env:
+          SQLX_OFFLINE: true
+
+      # Test macros in offline mode (still needs DATABASE_URL to run)
+      - uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: >
+            --no-default-features
+            --test mysql-macros
+            --features any,mysql,macros,migrate,all-types,offline,runtime-${{ matrix.runtime }}-${{ matrix.tls }}
+        env:
+          DATABASE_URL: mysql://root:password@localhost:3306/sqlx
+          SQLX_OFFLINE: true
+
   mariadb:
     name: MariaDB
     runs-on: ubuntu-20.04
@@ -316,6 +411,9 @@ jobs:
             target
           key: ${{ runner.os }}-mysql-${{ matrix.runtime }}-${{ matrix.tls }}-${{ hashFiles('**/Cargo.lock') }}
 
+      # Create data dir for offline mode
+      - run: mkdir .sqlx
+
       - uses: actions-rs/cargo@v1
         with:
           command: build
@@ -330,9 +428,35 @@ jobs:
           command: test
           args: >
             --no-default-features
-            --features any,mysql,macros,migrate,all-types,runtime-${{ matrix.runtime }}-${{ matrix.tls }}
+            --features any,mysql,macros,migrate,all-types,offline,runtime-${{ matrix.runtime }}-${{ matrix.tls }}
+        env:
+          DATABASE_URL: mysql://root:password@localhost:3306/sqlx
+
+      # Remove test artifacts
+      - run: cargo clean -p sqlx
+
+      # Build the macros-test in offline mode (omit DATABASE_URL)
+      - uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: >
+            --no-default-features
+            --test mysql-macros
+            --features any,mysql,macros,migrate,all-types,offline,runtime-${{ matrix.runtime }}-${{ matrix.tls }}
+        env:
+          SQLX_OFFLINE: true
+
+      # Test macros in offline mode (still needs DATABASE_URL to run)
+      - uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: >
+            --no-default-features
+            --test mysql-macros
+            --features any,mysql,macros,migrate,all-types,offline,runtime-${{ matrix.runtime }}-${{ matrix.tls }}
         env:
           DATABASE_URL: mysql://root:password@localhost:3306/sqlx
+          SQLX_OFFLINE: true
 
   mssql:
     name: MSSQL
@@ -360,6 +484,9 @@ jobs:
             target
           key: ${{ runner.os }}-mssql-${{ matrix.runtime }}-${{ matrix.tls }}-${{ hashFiles('**/Cargo.lock') }}
 
+      # Create data dir for offline mode
+      - run: mkdir .sqlx
+
       - uses: actions-rs/cargo@v1
         with:
           command: build
@@ -374,6 +501,32 @@ jobs:
           command: test
           args: >
             --no-default-features
-            --features any,mssql,macros,migrate,all-types,runtime-${{ matrix.runtime }}-${{ matrix.tls }}
+            --features any,mssql,macros,migrate,all-types,offline,runtime-${{ matrix.runtime }}-${{ matrix.tls }}
+        env:
+          DATABASE_URL: mssql://sa:Password123!@localhost/sqlx
+
+      # Remove test artifacts
+      - run: cargo clean -p sqlx
+
+      # Build the macros-test in offline mode (omit DATABASE_URL)
+      - uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: >
+            --no-default-features
+            --test mssql-macros
+            --features any,mssql,macros,migrate,all-types,offline,runtime-${{ matrix.runtime }}-${{ matrix.tls }}
+        env:
+          SQLX_OFFLINE: true
+
+      # Test macros in offline mode (still needs DATABASE_URL to run)
+      - uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: >
+            --no-default-features
+            --test mssql-macros
+            --features any,mssql,macros,migrate,all-types,offline,runtime-${{ matrix.runtime }}-${{ matrix.tls }}
         env:
           DATABASE_URL: mssql://sa:Password123!@localhost/sqlx
+          SQLX_OFFLINE: true

sqlx-cli/src/cargo.rs

@@ -0,0 +1,47 @@
+use anyhow::Context;
+use serde::Deserialize;
+use std::env;
+use std::ffi::{OsStr, OsString};
+use std::path::PathBuf;
+use std::process::Command;
+use std::str;
+
+#[derive(Deserialize)]
+pub struct CargoMetadata {
+    pub target_directory: PathBuf,
+    pub workspace_root: PathBuf,
+}
+
+/// Path to the `cargo` executable
+pub fn cargo_path() -> anyhow::Result<OsString> {
+    env::var_os("CARGO").context("Failed to obtain value of `CARGO`")
+}
+
+pub fn manifest_dir(cargo: &OsStr) -> anyhow::Result<PathBuf> {
+    let stdout = Command::new(&cargo)
+        .args(&["locate-project", "--message-format=plain"])
+        .output()
+        .context("could not locate manifest dir")?
+        .stdout;
+
+    let mut manifest_path: PathBuf = str::from_utf8(&stdout)
+        .context("output of `cargo locate-project` was not valid UTF-8")?
+        // get rid of the trailing newline
+        .trim()
+        .into();
+
+    manifest_path.pop();
+
+    Ok(manifest_path)
+}
+
+pub fn metadata(cargo: &OsStr) -> anyhow::Result<CargoMetadata> {
+    let output = Command::new(&cargo)
+        .args(&["metadata", "--format-version=1"])
+        .output()
+        .context("Could not fetch metadata")?;
+
+    serde_json::from_slice(&output.stdout)
+        .context("Invalid `cargo metadata` output")
+        .map_err(Into::into)
+}

sqlx-cli/src/lib.rs

@@ -2,6 +2,12 @@ use anyhow::Result;
 
 use crate::opt::{Command, DatabaseCommand, MigrateCommand};
 
+use anyhow::{anyhow, Context};
+use dotenv::dotenv;
+use prepare::PrepareCtx;
+use std::env;
+
+mod cargo;
 mod database;
 // mod migration;
 // mod migrator;
@@ -74,18 +80,36 @@ pub async fn run(opt: Opt) -> Result<()> {
         },
 
         Command::Prepare {
-            check: false,
-            merged,
-            args,
+            check,
+            workspace,
             database_url,
-        } => prepare::run(&database_url, merged, args)?,
-
-        Command::Prepare {
-            check: true,
-            merged,
             args,
-            database_url,
-        } => prepare::check(&database_url, merged, args)?,
+        } => {
+            let cargo_path = cargo::cargo_path()?;
+            println!("cargo path: {:?}", cargo_path);
+
+            let manifest_dir = cargo::manifest_dir(&cargo_path)?;
+            let metadata = cargo::metadata(&cargo_path)
+                .context("`prepare` subcommand may only be invoked as `cargo sqlx prepare`")?;
+
+            let ctx = PrepareCtx {
+                workspace,
+                cargo: cargo_path,
+                cargo_args: args,
+                manifest_dir,
+                target_dir: metadata.target_directory,
+                workspace_root: metadata.workspace_root,
+                database_url,
+            };
+
+            println!("{:?}", ctx);
+
+            if check {
+                prepare::check(&ctx)?
+            } else {
+                prepare::run(&ctx)?
+            }
+        }
     };
 
     Ok(())

sqlx-cli/src/opt.rs

@@ -29,16 +29,21 @@ pub enum Command {
         #[clap(long)]
         check: bool,
 
-        /// Generate a single top-level `sqlx-data.json` file when using a cargo workspace.
+        /// Do a clean build of all crates in the workspace.
+        ///
+        /// This option is intended for workspaces where multiple crates use SQLx; if there is only
+        /// one, it is better to run `cargo sqlx prepare` without this option inside of that crate.
         #[clap(long)]
-        merged: bool,
+        workspace: bool,
 
         /// Arguments to be passed to `cargo rustc ...`.
         #[clap(last = true)]
         args: Vec<String>,
 
-        #[clap(flatten)]
-        database_url: DatabaseUrl,
+        // `DatabaseUrl` doesn't allow it to be optional
+        /// Location of the DB, by default will be read from the DATABASE_URL env var
+        #[clap(long, short = 'D', env)]
+        database_url: Option<String>,
     },
 
     #[clap(alias = "mig")]