From 2abaa9719f83ed106be57ae87772e23ba54f3c5b Mon Sep 17 00:00:00 2001 From: Marina Samuel Date: Fri, 28 Jul 2017 06:36:19 -0500 Subject: [PATCH] Schema Viewer Drawer (#3291) * Process extra column metadata for a few sql-based data sources. * Add Table and Column metadata tables. * Periodically update table and column schema tables in a celery task. * Fetching schema returns data from table and column metadata tables. * Add tests for backend changes. * Front-end shows extra table metadata and uses new schema response. * Delete datasource schema data when deleting a data source. * Process and store data source schema when a data source is first created or after a migration. * Tables should have a unique name per datasource. * Addressing review comments. * Update migration file for mixins. * Appease PEP8 * Upgrade migration file for rebase. * Cascade delete. * Adding org_id * Remove redundant column and table prefixes. * Non-existing tables and columns should be filtered out on the server side not client side. * Fetching table samples should be optional and should happen in a separate task per table. * Allow users to force a schema refresh. * Use updated_at to help prune old schema metadata periodically. * Using settings.SCHEMAS_REFRESH_QUEUE * fix for #2426 test * more stable test_interactive_new * Closes #927, #928: Schema refresh improvements. * Closes #934, #935: Remove type from schema browser and don't show empty example column in schema drawer (#936) * Speed up schema fetch requests with fewer postgres queries. * Add column metadata to Athena glue processing. * Fix bug assuming 'metadata' exists for every table. * Closes #939: Persisted, existing table metadata should be updated. * Sample processing should be rate-limited. * Add cli command for refreshing data samples. * Schema refreshes should not overwrite column 'example' field. * refresh_samples() should filter tables_to_sample on the datasource's id being sampled * Correctly wrap long text in schema drawer. Co-authored-by: Alison Schema Improvements Part 2: Add data source config options. Adding BigQuery schema drawer with data types and samples. --- client/app/assets/less/ant.less | 1 + client/app/assets/less/inc/base.less | 4 + client/app/assets/less/inc/popover.less | 4 +- .../app/assets/less/inc/schema-browser.less | 14 +- client/app/assets/less/redash/query.less | 14 + .../dynamic-form/dynamicFormHelper.js | 8 + client/app/components/keywordBuilder.js | 8 +- client/app/components/proptypes.js | 22 +- client/app/components/queries/SchemaData.jsx | 143 ++++++++ .../components/queries/schema-browser.html | 17 +- .../app/components/queries/schema-browser.js | 20 ++ .../app/pages/data-sources/EditDataSource.jsx | 25 +- .../schema-table-components/EditableTable.jsx | 91 +++++ .../schema-table-components/QueryListItem.jsx | 49 +++ .../QuerySearchDialog.jsx | 119 +++++++ .../SampleQueryList.jsx | 95 ++++++ .../schema-table-components/SchemaTable.jsx | 270 +++++++++++++++ .../TableVisibilityCheckbox.jsx | 29 ++ .../schema-table-components/schema-table.css | 24 ++ client/app/pages/queries/query.html | 3 + client/app/services/data-source.js | 13 + migrations/versions/118aa16f565b_.py | 32 ++ migrations/versions/280daa582976_.py | 55 +++ migrations/versions/6adb92e75691_.py | 25 ++ migrations/versions/ba150362b02e_.py | 24 ++ migrations/versions/cf135a57332e_.py | 26 ++ redash/cli/data_sources.py | 24 +- redash/handlers/chrome_logger.py | 7 +- redash/handlers/data_sources.py | 54 ++- redash/models/__init__.py | 106 ++++-- redash/query_runner/__init__.py | 20 ++ redash/query_runner/athena.py | 31 +- redash/query_runner/big_query.py | 117 ++++++- redash/query_runner/mysql.py | 18 +- redash/query_runner/pg.py | 36 +- redash/query_runner/presto.py | 18 +- redash/serializers/__init__.py | 63 ++++ redash/settings/__init__.py | 12 +- redash/tasks/__init__.py | 2 +- redash/tasks/queries.py | 288 +++++++++++++++- redash/worker.py | 6 +- tests/factories.py | 17 + tests/handlers/test_data_sources.py | 34 ++ tests/models/test_data_sources.py | 45 --- tests/query_runner/test_athena.py | 24 +- tests/query_runner/test_bigquery.py | 76 +++++ tests/query_runner/test_get_schema_format.py | 77 +++++ tests/tasks/test_queries.py | 28 +- tests/tasks/test_refresh_schemas.py | 320 +++++++++++++++++- tests/test_cli.py | 15 +- tests/test_models.py | 5 +- 51 files changed, 2449 insertions(+), 129 deletions(-) create mode 100644 client/app/components/queries/SchemaData.jsx create mode 100644 client/app/pages/data-sources/schema-table-components/EditableTable.jsx create mode 100644 client/app/pages/data-sources/schema-table-components/QueryListItem.jsx create mode 100644 client/app/pages/data-sources/schema-table-components/QuerySearchDialog.jsx create mode 100644 client/app/pages/data-sources/schema-table-components/SampleQueryList.jsx create mode 100644 client/app/pages/data-sources/schema-table-components/SchemaTable.jsx create mode 100644 client/app/pages/data-sources/schema-table-components/TableVisibilityCheckbox.jsx create mode 100644 client/app/pages/data-sources/schema-table-components/schema-table.css create mode 100644 migrations/versions/118aa16f565b_.py create mode 100644 migrations/versions/280daa582976_.py create mode 100644 migrations/versions/6adb92e75691_.py create mode 100644 migrations/versions/ba150362b02e_.py create mode 100644 migrations/versions/cf135a57332e_.py create mode 100644 tests/query_runner/test_bigquery.py create mode 100644 tests/query_runner/test_get_schema_format.py diff --git a/client/app/assets/less/ant.less b/client/app/assets/less/ant.less index d1509a48b5..6238066681 100644 --- a/client/app/assets/less/ant.less +++ b/client/app/assets/less/ant.less @@ -14,6 +14,7 @@ @import '~antd/lib/radio/style/index'; @import '~antd/lib/time-picker/style/index'; @import '~antd/lib/pagination/style/index'; +@import '~antd/lib/drawer/style/index'; @import '~antd/lib/table/style/index'; @import '~antd/lib/popover/style/index'; @import '~antd/lib/icon/style/index'; diff --git a/client/app/assets/less/inc/base.less b/client/app/assets/less/inc/base.less index d5f05424ef..55faf52d6e 100755 --- a/client/app/assets/less/inc/base.less +++ b/client/app/assets/less/inc/base.less @@ -124,6 +124,10 @@ strong { transition: height 0s, width 0s !important; } +.admin-schema-editor { + padding: 50px 0; +} + // Ace Editor .ace_editor { border: 1px solid fade(@redash-gray, 15%) !important; diff --git a/client/app/assets/less/inc/popover.less b/client/app/assets/less/inc/popover.less index 5fcad7089b..c687a089a2 100755 --- a/client/app/assets/less/inc/popover.less +++ b/client/app/assets/less/inc/popover.less @@ -1,5 +1,7 @@ .popover { box-shadow: fade(@redash-gray, 25%) 0px 0px 15px 0px; + color: #000000; + z-index: 1000000001; // So that it can popover a dropdown menu } .popover-title { @@ -19,4 +21,4 @@ p { margin-bottom: 0; } -} \ No newline at end of file +} diff --git a/client/app/assets/less/inc/schema-browser.less b/client/app/assets/less/inc/schema-browser.less index 0034391086..d547a78790 100644 --- a/client/app/assets/less/inc/schema-browser.less +++ b/client/app/assets/less/inc/schema-browser.less @@ -7,14 +7,14 @@ div.table-name { border-radius: @redash-radius; position: relative; - .copy-to-editor { + .copy-to-editor, .info { display: none; } &:hover { background: fade(@redash-gray, 10%); - .copy-to-editor { + .copy-to-editor, .info { display: flex; } } @@ -36,7 +36,7 @@ div.table-name { background: transparent; } - .copy-to-editor { + .copy-to-editor, .info { color: fade(@redash-gray, 90%); cursor: pointer; position: absolute; @@ -49,6 +49,10 @@ div.table-name { justify-content: center; } + .info { + right: 20px + } + .table-open { padding: 0 22px 0 26px; overflow: hidden; @@ -56,14 +60,14 @@ div.table-name { white-space: nowrap; position: relative; - .copy-to-editor { + .copy-to-editor, .info { display: none; } &:hover { background: fade(@redash-gray, 10%); - .copy-to-editor { + .copy-to-editor, .info { display: flex; } } diff --git a/client/app/assets/less/redash/query.less b/client/app/assets/less/redash/query.less index 83ca054291..fc057bb490 100644 --- a/client/app/assets/less/redash/query.less +++ b/client/app/assets/less/redash/query.less @@ -697,3 +697,17 @@ nav .rg-bottom { } } } + +.ui-select-choices-row .info { + display: none; +} + +.ui-select-choices-row { + &:hover { + .info { + cursor: pointer; + width: 20px; + display: inline; + } + } +} diff --git a/client/app/components/dynamic-form/dynamicFormHelper.js b/client/app/components/dynamic-form/dynamicFormHelper.js index c1d17997d3..bafc523759 100644 --- a/client/app/components/dynamic-form/dynamicFormHelper.js +++ b/client/app/components/dynamic-form/dynamicFormHelper.js @@ -75,6 +75,13 @@ function getFields(type = {}, target = { options: {} }) { placeholder: `My ${type.name}`, autoFocus: isNewTarget, }, + { + name: 'description', + title: 'Description', + type: 'text', + required: false, + initialValue: target.description, + }, ...orderedInputs(configurationSchema.properties, configurationSchema.order, target.options), ]; @@ -83,6 +90,7 @@ function getFields(type = {}, target = { options: {} }) { function updateTargetWithValues(target, values) { target.name = values.name; + target.description = values.description; Object.keys(values).forEach((key) => { if (key !== 'name') { target.options[key] = values[key]; diff --git a/client/app/components/keywordBuilder.js b/client/app/components/keywordBuilder.js index 1b7287186e..fe9bc5b02e 100644 --- a/client/app/components/keywordBuilder.js +++ b/client/app/components/keywordBuilder.js @@ -4,9 +4,9 @@ function buildTableColumnKeywords(table) { const keywords = []; table.columns.forEach((column) => { keywords.push({ - caption: column, - name: `${table.name}.${column}`, - value: `${table.name}.${column}`, + caption: column.name, + name: `${table.name}.${column.name}`, + value: `${table.name}.${column.name}`, score: 100, meta: 'Column', className: 'completion', @@ -29,7 +29,7 @@ function buildKeywordsFromSchema(schema) { }); tableColumnKeywords[table.name] = buildTableColumnKeywords(table); table.columns.forEach((c) => { - columnKeywords[c] = 'Column'; + columnKeywords[c.name] = 'Column'; }); }); diff --git a/client/app/components/proptypes.js b/client/app/components/proptypes.js index a1240cf029..91b1afbc32 100644 --- a/client/app/components/proptypes.js +++ b/client/app/components/proptypes.js @@ -11,8 +11,16 @@ export const DataSource = PropTypes.shape({ type_name: PropTypes.string, }); +export const DataSourceMetadata = PropTypes.shape({ + key: PropTypes.number, + name: PropTypes.string, + type: PropTypes.string, + example: PropTypes.string, + description: PropTypes.string, +}); + export const Table = PropTypes.shape({ - columns: PropTypes.arrayOf(PropTypes.string).isRequired, + columns: PropTypes.arrayOf(PropTypes.object).isRequired, }); export const Schema = PropTypes.arrayOf(Table); @@ -31,6 +39,18 @@ export const RefreshScheduleDefault = { until: null, }; +export const TableMetadata = PropTypes.shape({ + key: PropTypes.number.isRequired, + name: PropTypes.string.isRequired, + description: PropTypes.string, + visible: PropTypes.bool.isRequired, +}); + +export const Query = PropTypes.shape({ + id: PropTypes.number, + name: PropTypes.string, +}); + export const Field = PropTypes.shape({ name: PropTypes.string.isRequired, title: PropTypes.string, diff --git a/client/app/components/queries/SchemaData.jsx b/client/app/components/queries/SchemaData.jsx new file mode 100644 index 0000000000..8d23ce58cd --- /dev/null +++ b/client/app/components/queries/SchemaData.jsx @@ -0,0 +1,143 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import { react2angular } from 'react2angular'; +import Drawer from 'antd/lib/drawer'; +import Table from 'antd/lib/table'; + +import { DataSourceMetadata, Query } from '@/components/proptypes'; + +function textWrapRenderer(text) { + return ( +
+ {text} +
+ ); +} + +class SchemaData extends React.PureComponent { + static propTypes = { + show: PropTypes.bool.isRequired, + onClose: PropTypes.func.isRequired, + tableName: PropTypes.string, + tableDescription: PropTypes.string, + sampleQueries: PropTypes.arrayOf(Query), + tableMetadata: PropTypes.arrayOf(DataSourceMetadata), + }; + + static defaultProps = { + tableName: '', + tableDescription: '', + tableMetadata: [], + sampleQueries: [], + }; + + render() { + const tableDataColumns = [{ + title: 'Metadata', + dataIndex: 'metadata', + width: 400, + key: 'metadata', + }, { + title: 'Value', + dataIndex: 'value', + width: 400, + key: 'value', + render: (text) => { + if (typeof text === 'string') { + return text; + } + return ( + + ); + }, + }]; + + const columnDataColumns = [{ + title: 'Column Name', + dataIndex: 'name', + width: 400, + key: 'name', + render: textWrapRenderer, + }, { + title: 'Column Type', + dataIndex: 'type', + width: 400, + key: 'type', + render: textWrapRenderer, + }]; + + const hasDescription = + this.props.tableMetadata.some(columnMetadata => columnMetadata.description); + + const hasExample = + this.props.tableMetadata.some(columnMetadata => columnMetadata.example); + + if (hasDescription) { + columnDataColumns.push({ + title: 'Description', + dataIndex: 'description', + width: 400, + key: 'description', + render: textWrapRenderer, + }); + } + + if (hasExample) { + columnDataColumns.push({ + title: 'Example', + dataIndex: 'example', + width: 400, + key: 'example', + render: textWrapRenderer, + }); + } + const tableData = [{ + metadata: 'Table Description', + value: this.props.tableDescription || 'N/A', + }, { + metadata: 'Sample Usage', + value: this.props.sampleQueries.length > 0 ? this.props.sampleQueries : 'N/A', + }]; + + return ( + +

{this.props.tableName}

+
+
Table Data
+ +
+
Column Data
+
+ + ); + } +} + +export default function init(ngModule) { + ngModule.component('schemaData', react2angular(SchemaData, null, [])); +} + +init.init = true; diff --git a/client/app/components/queries/schema-browser.html b/client/app/components/queries/schema-browser.html index fe7e26669e..804264e352 100644 --- a/client/app/components/queries/schema-browser.html +++ b/client/app/components/queries/schema-browser.html @@ -19,22 +19,33 @@
-
+
{{table.name}} ({{table.size}}) +
-
{{column}} +
+ {{column.name}} + ng-click="$ctrl.itemSelected($event, [column.name])">
+
diff --git a/client/app/components/queries/schema-browser.js b/client/app/components/queries/schema-browser.js index ded89d09bc..14916fb384 100644 --- a/client/app/components/queries/schema-browser.js +++ b/client/app/components/queries/schema-browser.js @@ -11,6 +11,19 @@ function SchemaBrowserCtrl($rootScope, $scope) { $scope.$broadcast('vsRepeatTrigger'); }; + $scope.showSchemaInfo = false; + $scope.openSchemaInfo = ($event, table) => { + $scope.tableName = table.name; + $scope.tableDescription = table.description; + $scope.tableMetadata = table.columns; + $scope.sampleQueries = Object.values(table.sample_queries); + $scope.showSchemaInfo = true; + $event.stopPropagation(); + }; + $scope.closeSchemaInfo = () => { + $scope.$apply(() => { $scope.showSchemaInfo = false; }); + }; + this.getSize = (table) => { let size = 22; @@ -34,6 +47,13 @@ function SchemaBrowserCtrl($rootScope, $scope) { } }; + this.itemExists = (item) => { + if ('visible' in item) { + return item.visible; + } + return false; + }; + this.itemSelected = ($event, hierarchy) => { $rootScope.$broadcast('query-editor.command', 'paste', hierarchy.join('.')); $event.preventDefault(); diff --git a/client/app/pages/data-sources/EditDataSource.jsx b/client/app/pages/data-sources/EditDataSource.jsx index 4db6045387..4ea39e51e3 100644 --- a/client/app/pages/data-sources/EditDataSource.jsx +++ b/client/app/pages/data-sources/EditDataSource.jsx @@ -10,6 +10,7 @@ import notification from '@/services/notification'; import PromiseRejectionError from '@/lib/promise-rejection-error'; import LoadingState from '@/components/items-list/components/LoadingState'; import DynamicForm from '@/components/dynamic-form/DynamicForm'; +import SchemaTable from '@/pages/data-sources/schema-table-components/SchemaTable'; import helper from '@/components/dynamic-form/dynamicFormHelper'; import { HelpTrigger, TYPES as HELP_TRIGGER_TYPES } from '@/components/HelpTrigger'; @@ -26,13 +27,23 @@ class EditDataSource extends React.Component { dataSource: null, type: null, loading: true, + schema: null, }; componentDidMount() { DataSource.get({ id: $route.current.params.dataSourceId }).$promise.then((dataSource) => { const { type } = dataSource; this.setState({ dataSource }); - DataSource.types(types => this.setState({ type: find(types, { type }), loading: false })); + + const typesPromise = DataSource.types().$promise; + const schemaPromise = DataSource.schema({ id: $route.current.params.dataSourceId }).$promise; + + typesPromise.then(types => this.setState({ type: find(types, { type }) })); + schemaPromise.then(data => this.setState({ schema: data.schema })); + + Promise.all([typesPromise, schemaPromise]).then(() => { + this.setState({ loading: false }); + }); }).catch((error) => { // ANGULAR_REMOVE_ME This code is related to Angular's HTTP services if (error.status && error.data) { @@ -78,6 +89,12 @@ class EditDataSource extends React.Component { }); }; + updateSchema = (schema, tableId, columnId) => { + const { dataSource } = this.state; + const data = { tableId, columnId, schema }; + DataSource.updateSchema({ id: dataSource.id }, data); + }; + testConnection = (callback) => { const { dataSource } = this.state; DataSource.test({ id: dataSource.id }, (httpResponse) => { @@ -124,6 +141,12 @@ class EditDataSource extends React.Component {
+
+ +
); } diff --git a/client/app/pages/data-sources/schema-table-components/EditableTable.jsx b/client/app/pages/data-sources/schema-table-components/EditableTable.jsx new file mode 100644 index 0000000000..93ee20f02b --- /dev/null +++ b/client/app/pages/data-sources/schema-table-components/EditableTable.jsx @@ -0,0 +1,91 @@ +import React from 'react'; +import Form from 'antd/lib/form'; +import Input from 'antd/lib/input'; +import PropTypes from 'prop-types'; +import { TableMetadata } from '@/components/proptypes'; +import TableVisibilityCheckbox from './TableVisibilityCheckbox'; +import SampleQueryList from './SampleQueryList'; + +import './schema-table.css'; + +const FormItem = Form.Item; +const { TextArea } = Input; +export const EditableContext = React.createContext(); + +// eslint-disable-next-line react/prop-types +const EditableRow = ({ form, index, ...props }) => ( + + + +); + +export const EditableFormRow = Form.create()(EditableRow); + +export class EditableCell extends React.Component { + static propTypes = { + dataIndex: PropTypes.string, + input_type: PropTypes.string, + editing: PropTypes.bool, + record: TableMetadata, + }; + + static defaultProps = { + dataIndex: undefined, + input_type: undefined, + editing: false, + record: {}, + }; + + constructor(props) { + super(props); + this.state = { + visible: this.props.record ? this.props.record.visible : false, + }; + } + + onChange = () => { + this.setState(prevState => ({ visible: !prevState.visible })); + } + + getInput = () => { + if (this.props.input_type === 'visible') { + return ( + + ); + } else if (this.props.input_type === 'sample_queries') { + return ; + } + return