From ebf55bf20dead58b2ceb93d0aad9cbafe2c7e028 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Wed, 16 Mar 2016 20:25:41 -0700 Subject: [PATCH] Adding docstrings ! --- panoramix/models.py | 51 +++++++++++++++++++++++++++++++++++++++------ panoramix/utils.py | 26 +++++++++++++---------- panoramix/views.py | 28 ++++++++++++++++--------- 3 files changed, 78 insertions(+), 27 deletions(-) diff --git a/panoramix/models.py b/panoramix/models.py index 52a5c81c2b58b..7cf78042f2f27 100644 --- a/panoramix/models.py +++ b/panoramix/models.py @@ -36,6 +36,10 @@ class AuditMixinNullable(AuditMixin): + + """Altering the AuditMixin to use nullable fields, allows creating + objects programmatically outside of CRUD""" + created_on = Column(DateTime, default=datetime.now, nullable=True) changed_on = Column( DateTime, default=datetime.now, @@ -142,6 +146,7 @@ def json_data(self): @property def slice_url(self): + """Defines the url to access the slice""" try: slice_params = json.loads(self.params) except Exception as e: @@ -175,7 +180,7 @@ def slice_link(self): class Dashboard(Model, AuditMixinNullable): - """A dash to slash""" + """The dashboard object!""" __tablename__ = 'dashboards' id = Column(Integer, primary_key=True) @@ -240,6 +245,9 @@ def dttm_cols(self): class Database(Model, AuditMixinNullable): + + """An ORM object that stores Database related information""" + __tablename__ = 'dbs' id = Column(Integer, primary_key=True) database_name = Column(String(250), unique=True) @@ -256,15 +264,14 @@ def safe_sqlalchemy_uri(self): return self.sqlalchemy_uri def grains(self): + """Defines time granularity database-specific expressions. - """Defines time granularity database-specific expressions. The idea - here is to make it easy for users to change the time grain form a - datetime (maybe the source grain is arbitrary timestamps, daily + The idea here is to make it easy for users to change the time grain + form a datetime (maybe the source grain is arbitrary timestamps, daily or 5 minutes increments) to another, "truncated" datetime. Since each database has slightly different but similar datetime functions, this allows a mapping between database engines and actual functions. """ - Grain = namedtuple('Grain', 'name function') DB_TIME_GRAINS = { 'presto': ( @@ -314,6 +321,9 @@ def sql_link(self): class SqlaTable(Model, Queryable, AuditMixinNullable): + + """An ORM object for SqlAlchemy table references""" + type = "table" __tablename__ = 'tables' @@ -554,6 +564,7 @@ def query( df=df, duration=datetime.now() - qry_start_dttm, query=sql) def fetch_metadata(self): + """Fetches the metadata for the table and merges it in""" table = self.database.get_table(self.table_name) try: table = self.database.get_table(self.table_name) @@ -653,6 +664,9 @@ def fetch_metadata(self): class SqlMetric(Model, AuditMixinNullable): + + """ORM object for metrics, each table can have multiple metrics""" + __tablename__ = 'sql_metrics' id = Column(Integer, primary_key=True) metric_name = Column(String(512)) @@ -666,6 +680,9 @@ class SqlMetric(Model, AuditMixinNullable): class TableColumn(Model, AuditMixinNullable): + + """ORM object for table columns, each table can have multiple columns""" + __tablename__ = 'table_columns' id = Column(Integer, primary_key=True) table_id = Column(Integer, ForeignKey('tables.id')) @@ -693,6 +710,9 @@ def isnum(self): class DruidCluster(Model, AuditMixinNullable): + + """ORM object referencing the Druid clusters""" + __tablename__ = 'clusters' id = Column(Integer, primary_key=True) cluster_name = Column(String(250), unique=True) @@ -726,6 +746,9 @@ def refresh_datasources(self): class DruidDatasource(Model, AuditMixinNullable, Queryable): + + """ORM object referencing Druid datasources (tables)""" + type = "druid" baselink = "datasourcemodelview" @@ -793,6 +816,7 @@ def get_metric_obj(self, metric_name): ][0] def latest_metadata(self): + """Returns segment metadata from the latest segment""" client = self.cluster.get_pydruid_client() results = client.time_boundary(datasource=self.datasource_name) if not results: @@ -813,6 +837,8 @@ def generate_metrics(self): @classmethod def sync_to_db(cls, name, cluster): + """Fetches the metadata for that datasource and merges it into + the Panoramix database""" print("Syncing Druid datasource [{}]".format(name)) session = get_session() datasource = session.query(cls).filter_by(datasource_name=name).first() @@ -855,8 +881,11 @@ def query( timeseries_limit=None, row_limit=None, inner_from_dttm=None, inner_to_dttm=None, - extras=None, + extras=None, # noqa select=None): + """Runs a query against Druid and returns a dataframe. + This query interface is common to SqlAlchemy and Druid""" + # TODO refactor into using a TBD Query object qry_start_dttm = datetime.now() inner_from_dttm = inner_from_dttm or from_dttm @@ -996,6 +1025,9 @@ def query( class Log(Model): + + """ORM object used to log Panoramix actions to the database""" + __tablename__ = 'logs' id = Column(Integer, primary_key=True) @@ -1033,6 +1065,9 @@ def wrapper(*args, **kwargs): class DruidMetric(Model): + + """ORM object referencing Druid metrics for a datasource""" + __tablename__ = 'metrics' id = Column(Integer, primary_key=True) metric_name = Column(String(512)) @@ -1055,6 +1090,9 @@ def json_obj(self): class DruidColumn(Model): + + """ORM model for storing Druid datasource column metadata""" + __tablename__ = 'columns' id = Column(Integer, primary_key=True) datasource_name = Column( @@ -1080,6 +1118,7 @@ def isnum(self): return self.type in ('LONG', 'DOUBLE', 'FLOAT') def generate_metrics(self): + """Generate metrics based on the column metadata""" M = DruidMetric metrics = [] metrics.append(DruidMetric( diff --git a/panoramix/utils.py b/panoramix/utils.py index d244092643c5e..f642be29c2e2f 100644 --- a/panoramix/utils.py +++ b/panoramix/utils.py @@ -1,3 +1,5 @@ +"""Utility functions used across Panoramix""" + from datetime import datetime import hashlib import functools @@ -12,10 +14,12 @@ class memoized(object): + """Decorator that caches a function's return value each time it is called. If called later with the same arguments, the cached value is returned, and not re-evaluated. """ + def __init__(self, func): self.func = func self.cache = {} @@ -47,8 +51,7 @@ def list_minus(l, minus): def parse_human_datetime(s): """ - Use the parsedatetime lib to return ``datetime.datetime`` from human - generated strings + Returns ``datetime.datetime`` from human readable strings >>> from datetime import date, timedelta >>> from dateutil.relativedelta import relativedelta @@ -92,8 +95,7 @@ def merge_perm(sm, permission_name, view_menu_name): def parse_human_timedelta(s): """ - Use the parsedatetime lib to return ``datetime.datetime`` from human - generated strings + Returns ``datetime.datetime`` from natural language time deltas >>> parse_human_datetime("now") <= datetime.now() True @@ -107,7 +109,9 @@ def parse_human_timedelta(s): class JSONEncodedDict(TypeDecorator): + """Represents an immutable structure as a json-encoded string.""" + impl = TEXT def process_bind_param(self, value, dialect): if value is not None: @@ -122,6 +126,9 @@ def process_result_value(self, value, dialect): class ColorFactory(object): + + """Used to generated arrays of colors server side""" + BNB_COLORS = [ #rausch hackb kazan babu lima beach barol '#ff5a5f', '#7b0051', '#007A87', '#00d1c1', '#8ce071', '#ffb400', '#b4a76c', @@ -134,7 +141,8 @@ def __init__(self, hash_based=False): self.hash_based = hash_based def get(self, s): - """ + """Gets a color from a string and memoize the association + >>> cf = ColorFactory() >>> cf.get('item_1') '#ff5a5f' @@ -155,9 +163,7 @@ def get(self, s): def init(panoramix): - """ - Inits the Panoramix application with security roles and such - """ + """Inits the Panoramix application with security roles and such""" db = panoramix.db models = panoramix.models sm = panoramix.appbuilder.sm @@ -204,9 +210,7 @@ def init(panoramix): def datetime_f(dttm): - """ - Formats datetime to take less room is recent - """ + """Formats datetime to take less room when it is recent""" if dttm: dttm = dttm.isoformat() now_iso = datetime.now().isoformat() diff --git a/panoramix/views.py b/panoramix/views.py index 9518bb8c8a748..97c2739bfe20f 100644 --- a/panoramix/views.py +++ b/panoramix/views.py @@ -1,3 +1,5 @@ +"""Flask web views for Panoramix""" + from datetime import datetime import json import logging @@ -44,7 +46,7 @@ class PanoramixModelView(ModelView): page_size = 500 -class TableColumnInlineView(CompactCRUDMixin, PanoramixModelView): +class TableColumnInlineView(CompactCRUDMixin, PanoramixModelView): # noqa datamodel = SQLAInterface(models.TableColumn) can_delete = False edit_columns = [ @@ -72,7 +74,8 @@ class TableColumnInlineView(CompactCRUDMixin, PanoramixModelView): appbuilder.add_separator("Sources") -class DruidColumnInlineView(CompactCRUDMixin, PanoramixModelView): + +class DruidColumnInlineView(CompactCRUDMixin, PanoramixModelView): # noqa datamodel = SQLAInterface(models.DruidColumn) edit_columns = [ 'column_name', 'description', 'datasource', 'groupby', @@ -89,7 +92,7 @@ def post_update(self, col): appbuilder.add_view_no_menu(DruidColumnInlineView) -class SqlMetricInlineView(CompactCRUDMixin, PanoramixModelView): +class SqlMetricInlineView(CompactCRUDMixin, PanoramixModelView): # noqa datamodel = SQLAInterface(models.SqlMetric) list_columns = ['metric_name', 'verbose_name', 'metric_type'] edit_columns = [ @@ -100,7 +103,7 @@ class SqlMetricInlineView(CompactCRUDMixin, PanoramixModelView): appbuilder.add_view_no_menu(SqlMetricInlineView) -class DruidMetricInlineView(CompactCRUDMixin, PanoramixModelView): +class DruidMetricInlineView(CompactCRUDMixin, PanoramixModelView): # noqa datamodel = SQLAInterface(models.DruidMetric) list_columns = ['metric_name', 'verbose_name', 'metric_type'] edit_columns = [ @@ -115,7 +118,7 @@ class DruidMetricInlineView(CompactCRUDMixin, PanoramixModelView): appbuilder.add_view_no_menu(DruidMetricInlineView) -class DatabaseView(PanoramixModelView, DeleteMixin): +class DatabaseView(PanoramixModelView, DeleteMixin): # noqa datamodel = SQLAInterface(models.Database) list_columns = ['database_name', 'sql_link', 'created_by_', 'changed_on'] order_columns = utils.list_minus(list_columns, ['created_by_']) @@ -149,7 +152,7 @@ def pre_update(self, db): category_icon='fa-database',) -class TableModelView(PanoramixModelView, DeleteMixin): +class TableModelView(PanoramixModelView, DeleteMixin): # noqa datamodel = SQLAInterface(models.SqlaTable) list_columns = [ 'table_link', 'database', 'sql_link', 'is_featured', @@ -191,7 +194,7 @@ def post_update(self, table): appbuilder.add_separator("Sources") -class DruidClusterModelView(PanoramixModelView, DeleteMixin): +class DruidClusterModelView(PanoramixModelView, DeleteMixin): # noqa datamodel = SQLAInterface(models.DruidCluster) add_columns = [ 'cluster_name', @@ -209,7 +212,7 @@ class DruidClusterModelView(PanoramixModelView, DeleteMixin): category_icon='fa-database',) -class SliceModelView(PanoramixModelView, DeleteMixin): +class SliceModelView(PanoramixModelView, DeleteMixin): # noqa datamodel = SQLAInterface(models.Slice) can_add = False list_columns = [ @@ -237,7 +240,7 @@ class SliceModelView(PanoramixModelView, DeleteMixin): category_icon='',) -class DashboardModelView(PanoramixModelView, DeleteMixin): +class DashboardModelView(PanoramixModelView, DeleteMixin): # noqa datamodel = SQLAInterface(models.Dashboard) list_columns = ['dashboard_link', 'created_by_', 'changed_on'] order_columns = utils.list_minus(list_columns, ['created_by_']) @@ -289,7 +292,7 @@ class LogModelView(PanoramixModelView): icon="fa-list-ol") -class DruidDatasourceModelView(PanoramixModelView, DeleteMixin): +class DruidDatasourceModelView(PanoramixModelView, DeleteMixin): # noqa datamodel = SQLAInterface(models.DruidDatasource) list_columns = [ 'datasource_link', 'cluster', 'owner', @@ -362,6 +365,7 @@ def shortner(self): class Panoramix(BaseView): + """The base views for Panoramix!""" @has_access @expose("/explore///") @@ -502,6 +506,7 @@ def explore(self, datasource_type, datasource_id): @has_access @expose("/checkbox////", methods=['GET']) def checkbox(self, model_view, id_, attr, value): + """endpoint for checking/unchecking any boolean in a sqla model""" model = None if model_view == 'TableColumnInlineView': model = models.TableColumn @@ -518,6 +523,7 @@ def checkbox(self, model_view, id_, attr, value): @has_access @expose("/save_dash//", methods=['GET', 'POST']) def save_dash(self, dashboard_id): + """Save a dashboard's metadata""" data = json.loads(request.form.get('data')) positions = data['positions'] slice_ids = [int(d['slice_id']) for d in positions] @@ -540,6 +546,7 @@ def save_dash(self, dashboard_id): @has_access @expose("/testconn", methods=["POST", "GET"]) def testconn(self): + """Tests a sqla connection""" try: uri = request.form.get('uri') engine = create_engine(uri) @@ -554,6 +561,7 @@ def testconn(self): @has_access @expose("/dashboard//") def dashboard(self, dashboard_id): + """Server side rendering for a dashboard""" session = db.session() qry = session.query(models.Dashboard) if dashboard_id.isdigit():