Merge pull request #1099 from dbt-labs/move-cli-to-dbt-metricflow

Move MetricFlow CLI package to dbt-metricflow

.changes/0.0.0.md

@@ -222,4 +222,4 @@
 
 ### Special thanks
 
-Special thanks to @zzsza for the quick fix for our BigQuery token parsing bug!
+Special thanks to @zzsza for the quick fix for our BigQuery token parsing bug!

.changes/0.200.0.md

@@ -88,4 +88,4 @@
 - [@courtneyholcomb](https://github.com/courtneyholcomb)
 - [@nhandel](https://github.com/nhandel)
 - [@plypaul](https://github.com/plypaul)
-- [@tlento](https://github.com/tlento)
+- [@tlento](https://github.com/tlento)

.changes/README.md

@@ -50,4 +50,4 @@ changie merge
 - Changie generates markdown files in the `.changes` directory that are parsed together with the `changie merge` command.  Every time `changie merge` is run, it regenerates the entire file.  For this reason, any changes made directly to `CHANGELOG.md` will be overwritten on the next run of `changie merge`.
 - If changes need to be made to the `CHANGELOG.md`, make the changes to the relevant `<version>.md` file located in the `/.changes` directory.  You will then run `changie merge` to regenerate the `CHANGELOG.MD`.
 - Do not run `changie batch` again on released versions.  Our final release workflow deletes all of the yaml files associated with individual changes.  If for some reason modifications to the `CHANGELOG.md` are required after we've generated the final release `CHANGELOG.md`, the modifications need to be done manually to the `<version>.md` file in the `/.changes` directory.
-- changie can modify, create and delete files depending on the command you run.  This is expected.  Be sure to commit everything that has been modified and deleted.
+- changie can modify, create and delete files depending on the command you run.  This is expected.  Be sure to commit everything that has been modified and deleted.

.changes/unreleased/Breaking Changes-20240322-102334.yaml

@@ -0,0 +1,6 @@
+kind: Breaking Changes
+body: Move MetricFlow CLI to dbt-metricflow package
+time: 2024-03-22T10:23:34.027621-07:00
+custom:
+  Author: tlento
+  Issue: "1090"

.changie.yaml

@@ -82,4 +82,4 @@ footerFormat: |
   {{- range $k,$v := $contributorDict }}
   - [@{{$k}}](https://github.com/{{$k}}){{if $v}} ({{ join ", " $v }}){{end}}
   {{- end }}
-  {{- end }}
+  {{- end }}

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -62,7 +62,7 @@ body:
           - **OS**: Ubuntu 20.04
           - **Python**: 3.9.12 (`python3 --version`)
           - **dbt**: 1.1.1 (`dbt --version`)
-          - **metricflow**: 0.14.1 
+          - **metricflow**: 0.14.1
       value: |
         - OS:
         - Python:
@@ -92,4 +92,4 @@ body:
 
         Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in.
     validations:
-      required: false
+      required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -8,4 +8,4 @@ contact_links:
     about: Are you using dbt Cloud? Contact our support team for help!
   - name: Participate in Discussions
     url: https://github.com/dbt-labs/metricflow/discussions
-    about: Do you have a Big Idea for metricflow? Read open discussions, or start a new one
+    about: Do you have a Big Idea for metricflow? Read open discussions, or start a new one

.github/ISSUE_TEMPLATE/feature-request.yml

@@ -56,4 +56,4 @@ body:
       description: |
         Links? References? Anything that will give us more context about the feature you are suggesting!
     validations:
-      required: false
+      required: false

.github/ISSUE_TEMPLATE/regression-report.yml

@@ -90,4 +90,4 @@ body:
 
         Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in.
     validations:
-      required: false
+      required: false

.github/actions/run-mf-tests/action.yaml

@@ -42,4 +42,4 @@ runs:
     env:
       MF_SQL_ENGINE_URL: ${{ inputs.mf_sql_engine_url }}
       MF_SQL_ENGINE_PASSWORD: ${{ inputs.mf_sql_engine_password }}
-      METRICFLOW_CLIENT_EMAIL: ci-tester@gmail.com
+      METRICFLOW_CLIENT_EMAIL: ci-tester@gmail.com

.github/actions/setup-python-env/action.yaml

@@ -42,4 +42,4 @@ runs:
 
   - name: Install Hatch
     shell: bash
-    run: pip3 install hatch
+    run: pip3 install hatch

.github/atlas.yaml

@@ -1,4 +1,4 @@
 github_team: semantic-layer
 issue_project: SL
 issue_system: linear
-slack_channel: semantic-layer-alerts
+slack_channel: semantic-layer-alerts

.github/workflows/changelog-existence.yml

@@ -37,4 +37,4 @@ jobs:
     with:
       changelog_comment: 'Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/metricflow/blob/main/CONTRIBUTING.md).'
       skip_label: 'Skip Changelog'
-    secrets: inherit
+    secrets: inherit

.pre-commit-config.yaml

@@ -1,4 +1,3 @@
-files: ^(metricflow|tests)/
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v3.2.0

Makefile

@@ -89,4 +89,4 @@ populate-persistent-source-schemas:
 # Re-generate snapshots for the default SQL engine.
 .PHONY: test-snap
 test-snap:
-	make test ADDITIONAL_PYTEST_OPTIONS=--overwrite-snapshots
+	make test ADDITIONAL_PYTEST_OPTIONS=--overwrite-snapshots

attribution.md

@@ -5614,5 +5614,3 @@ copied and put under another distribution licence
 [FOSSA]: # (Do not touch the comments below)
 
 [FOSSA]: # (==depsig=7d31d2b0248f982b2a93b839e445f202bde83b1ff7a6bf5e75bfa17a23c56962==)
-
-

dbt-metricflow/LICENSE

@@ -87,4 +87,4 @@ under an Open Source License, as stated in this License.
 For more information on the use of the Business Source License for MariaDB
 products, please visit the MariaDB Business Source License FAQ. For more
 information on the use of the Business Source License generally, please visit
-the Adopting and Developing Business Source License FAQ.
+the Adopting and Developing Business Source License FAQ.

dbt-metricflow/README.md

@@ -15,4 +15,4 @@ This repo encapsulates the dbt-core, MetricFlow, and supported dbt-adapters pack
 ## Repo use cases
 - dbt-core and MetricFlow both depend on dbt-semantic-interfaces, which includes the schemas and interfaces for objects related to the semantic layer. Bundled versioning is necessary to ensure that the dbt-core and MetricFlow versions have compatible dbt-semantic-interfaces dependencies.
 - Bundling the installation makes it much easier on end users, as they no longer need to install `dbt-core` + `metricflow` + `dbt-adapter` and reconcile versions - instead they can simply install `dbt-metricflow[adapter]`.
-- Because this encapsulates both dbt-core and MetricFlow, this repo can be used to build logic that should be shared across the packages. For example, the CLI from MetricFlow can live in this repo, as it uses logic from all of the bundled packages. This can streamline dependency requirements in MetricFlow.
+- Because this encapsulates both dbt-core and MetricFlow, this repo can be used to build logic that should be shared across the packages. For example, the CLI from MetricFlow can live in this repo, as it uses logic from all of the bundled packages. This can streamline dependency requirements in MetricFlow.

metricflow/cli/cli_context.py → dbt-metricflow/dbt_metricflow/cli/cli_context.py

@@ -7,8 +7,8 @@
 
 from dbt_semantic_interfaces.protocols.semantic_manifest import SemanticManifest
 
-from metricflow.cli.dbt_connectors.adapter_backed_client import AdapterBackedSqlClient
-from metricflow.cli.dbt_connectors.dbt_config_accessor import dbtArtifacts, dbtProjectMetadata
+from dbt_metricflow.cli.dbt_connectors.adapter_backed_client import AdapterBackedSqlClient
+from dbt_metricflow.cli.dbt_connectors.dbt_config_accessor import dbtArtifacts, dbtProjectMetadata
 from metricflow.engine.metricflow_engine import MetricFlowEngine
 from metricflow.model.semantic_manifest_lookup import SemanticManifestLookup
 from metricflow.protocols.sql_client import SqlClient

metricflow/cli/main.py → dbt-metricflow/dbt_metricflow/cli/main.py

@@ -14,23 +14,21 @@
 
 import click
 import jinja2
-import pandas as pd
 from dbt_semantic_interfaces.protocols.semantic_manifest import SemanticManifest
 from dbt_semantic_interfaces.validations.semantic_manifest_validator import SemanticManifestValidator
 from dbt_semantic_interfaces.validations.validator_helpers import SemanticManifestValidationResults
 from halo import Halo
-from packaging.version import parse
 from update_checker import UpdateChecker
 
-import metricflow.cli.custom_click_types as click_custom
-from metricflow.cli import PACKAGE_NAME
-from metricflow.cli.cli_context import CLIContext
-from metricflow.cli.constants import DEFAULT_RESULT_DECIMAL_PLACES, MAX_LIST_OBJECT_ELEMENTS
-from metricflow.cli.dbt_connectors.dbt_config_accessor import dbtArtifacts
-from metricflow.cli.tutorial import (
+import dbt_metricflow.cli.custom_click_types as click_custom
+from dbt_metricflow.cli import PACKAGE_NAME
+from dbt_metricflow.cli.cli_context import CLIContext
+from dbt_metricflow.cli.constants import DEFAULT_RESULT_DECIMAL_PLACES, MAX_LIST_OBJECT_ELEMENTS
+from dbt_metricflow.cli.dbt_connectors.dbt_config_accessor import dbtArtifacts
+from dbt_metricflow.cli.tutorial import (
     dbtMetricFlowTutorialHelper,
 )
-from metricflow.cli.utils import (
+from dbt_metricflow.cli.utils import (
     dbt_project_file_exists,
     error_if_not_in_dbt_project,
     exception_handler,
@@ -343,11 +341,7 @@ def query(
             df.to_csv(csv, index=False)  # type: ignore
             click.echo(f"🖨 Successfully written query output to {csv.name}")
         else:
-            # NOTE: remove `to_string` if no pandas dependency is < 1.1.0
-            if parse(pd.__version__) >= parse("1.1.0"):
-                click.echo(df.to_markdown(index=False, floatfmt=f".{decimals}f"))
-            else:
-                click.echo(df.to_string(index=False, float_format=lambda x: format(x, f".{decimals}f")))
+            click.echo(df.to_markdown(index=False, floatfmt=f".{decimals}f"))
 
         if display_plans:
             temp_path = tempfile.mkdtemp()

metricflow/cli/utils.py → dbt-metricflow/dbt_metricflow/cli/utils.py

@@ -10,8 +10,8 @@
 import click
 from dateutil.parser import parse
 
-import metricflow.cli.custom_click_types as click_custom
-from metricflow.cli.cli_context import CLIContext
+import dbt_metricflow.cli.custom_click_types as click_custom
+from dbt_metricflow.cli.cli_context import CLIContext
 
 logger = logging.getLogger(__name__)
 

dbt-metricflow/pyproject.toml

@@ -24,13 +24,26 @@ classifiers = [
   "Programming Language :: Python :: Implementation :: PyPy",
 ]
 dependencies = [
+  # cli dependencies
+  "Jinja2>=3.1.3",
+  "click>=7.1.2",
+  "halo>=0.0.31, <0.1.0",
+  "update-checker>=0.18.0, <0.19.0",
+
+  # Internal dependencies
   "dbt-core>=1.7.4, <1.8.0",
-  "metricflow>=0.205.0, <0.206.0"
+  "metricflow>=0.205.0, <0.206.0",
+
+  # dsi version should be fixed by MetricFlow/dbt-core, not set here
+  "dbt-semantic-interfaces",
 ]
 
 [project.urls]
 "Source Code" = "https://github.com/dbt-labs/metricflow/tree/main/dbt-metricflow"
 
+[project.scripts]
+mf = 'dbt_metricflow.cli.main:cli'
+
 [project.optional-dependencies]
 bigquery = [
   "dbt-bigquery>=1.7.0, <1.8.0"

pyproject.toml

@@ -27,11 +27,8 @@ classifiers = [
 dependencies = [
   "Jinja2>=3.1.3",
   "PyYAML>=6.0, <7.0.0",
-  "click>=7.1.2",
-  "dbt-core>=1.7.0, <1.8.0",
   "dbt-semantic-interfaces>=0.4.2, <0.5.0",
   "graphviz>=0.18.2, <0.21",
-  "halo>=0.0.31, <0.1.0",
   "more-itertools>=8.10.0, <10.2.0",
   "pandas>=1.5.0, <1.6.0",
   "pydantic>=1.10.0, <1.11.0",
@@ -40,28 +37,36 @@ dependencies = [
   "ruamel.yaml>=0.17.21, <0.18.0",
   "tabulate>=0.8.9",
   "typing_extensions>=4.4, <5.0",
-  "update-checker>=0.18.0, <0.19.0",
 ]
 
 [project.urls]
 Documentation = "https://transform.co/metricflow"
 "Source Code" = "https://github.com/transform-data/metricflow"
 
-[project.scripts]
-mf = 'metricflow.cli.main:cli'
-
 [project.optional-dependencies]
 dev-packages = [
+  # Developer tools
   "mypy>=1.7.0, <1.8.0",
   "pre-commit>=3.2.2, <3.3.0",
-  # Bug with mypy: https://github.com/pallets/click/issues/2558#issuecomment-1656546003
-  "click>=8.1.6",
   "pytest-mock>=3.7.0, <3.8.0",
   "pytest-xdist>=3.2.1, <3.3.0",
   "pytest>=7.1.1, < 7.2.0",
   "types-PyYAML",
   "types-python-dateutil",
   "types-tabulate",
+
+  # Test and CLI development dependencies.
+
+  # This should generally match what's in dbt-metricflow/pyproject.toml, but may
+  # diverge during upgrade phases
+  "dbt-core>=1.7.4, <1.8.0",
+
+  # These need to match the ones in dbt-metricflow/pyproject.toml
+  "halo>=0.0.31, <0.1.0",
+  "update-checker>=0.18.0, <0.19.0",
+
+  # Bug with mypy: https://github.com/pallets/click/issues/2558#issuecomment-1656546003
+  "click>=8.1.6",
 ]
 
 # These are currently separate for ease of removal, but due to the way Python
@@ -117,8 +122,21 @@ exclude = [
   "/tests",
 ]
 
+
+# Environment setup
+# Due to our current wonky package organization, we need to do an editable install
+# of dbt-metricflow in every test running environment in order for the tests to work.
+# In theory we can install the adapter extra for dbt-metricflow and rely on that, but
+# for ease of managing versioning through upgrade phases we retain the explicit
+# dependencies on the specific adapter versions defined here.
+
 [tool.hatch.envs.dev-env]
 description = "Environment for development. Includes a DuckDB-backed client."
+
+pre-install-commands = [
+  "pip install -e ./dbt-metricflow"
+]
+
 features = [
   "dev-packages",
   "sql-client-packages",
@@ -136,6 +154,11 @@ MF_TEST_ADAPTER_TYPE="postgres"
 
 [tool.hatch.envs.postgres-env]
 description = "Dev environment for working with Postgres adapter"
+
+pre-install-commands = [
+  "pip install -e ./dbt-metricflow"
+]
+
 features = [
   "dev-packages",
   "dbt-postgres",
@@ -150,6 +173,11 @@ MF_TEST_ADAPTER_TYPE="bigquery"
 
 [tool.hatch.envs.bigquery-env]
 description = "Dev environment for working with the BigQuery adapter"
+
+pre-install-commands = [
+  "pip install -e ./dbt-metricflow"
+]
+
 features = [
   "dev-packages",
   "dbt-bigquery",
@@ -161,6 +189,11 @@ MF_TEST_ADAPTER_TYPE="databricks"
 
 [tool.hatch.envs.databricks-env]
 description = "Dev environment for working with the Databricks adapter"
+
+pre-install-commands = [
+  "pip install -e ./dbt-metricflow"
+]
+
 features = [
   "dev-packages",
   "dbt-databricks",
@@ -172,6 +205,11 @@ MF_TEST_ADAPTER_TYPE="redshift"
 
 [tool.hatch.envs.redshift-env]
 description = "Dev environment for working with the Redshift adapter"
+
+pre-install-commands = [
+  "pip install -e ./dbt-metricflow"
+]
+
 features = [
   "dev-packages",
   "dbt-redshift",
@@ -183,6 +221,11 @@ MF_TEST_ADAPTER_TYPE="snowflake"
 
 [tool.hatch.envs.snowflake-env]
 description = "Dev environment for working with Snowflake adapter"
+
+pre-install-commands = [
+  "pip install -e ./dbt-metricflow"
+]
+
 features = [
   "dev-packages",
   "dbt-snowflake",
@@ -196,6 +239,11 @@ DBT_ENV_SECRET_CATALOG="memory"
 
 [tool.hatch.envs.trino-env]
 description = "Dev environment for working with the Trino adapter"
+
+pre-install-commands = [
+  "pip install -e ./dbt-metricflow"
+]
+
 features = [
   "dev-packages",
   "dbt-trino",

ruff.toml

@@ -81,3 +81,4 @@ convention = "google"
 
 [lint.isort]
 required-imports = ["from __future__ import annotations"]
+known-first-party = ["dbt_metricflow", "metricflow"]

tests/cli/test_cli.py

@@ -15,8 +15,8 @@
 from dbt_semantic_interfaces.parsing.objects import YamlConfigFile
 from dbt_semantic_interfaces.test_utils import base_semantic_manifest_file
 
-from metricflow.cli.cli_context import CLIContext
-from metricflow.cli.main import (
+from dbt_metricflow.cli.cli_context import CLIContext
+from dbt_metricflow.cli.main import (
     dimension_values,
     dimensions,
     entities,

tests/cli/test_custom_click_types.py

@@ -4,7 +4,7 @@
 import pytest
 from click.testing import CliRunner
 
-from metricflow.cli.custom_click_types import MutuallyExclusiveOption, SequenceParamType
+from dbt_metricflow.cli.custom_click_types import MutuallyExclusiveOption, SequenceParamType
 
 
 def test_check_min_length() -> None: