From d8ecbe22d4bfe17c1e1a9d21f36d578e1678238e Mon Sep 17 00:00:00 2001 From: radheyakale Date: Wed, 22 Jun 2022 17:11:21 +0530 Subject: [PATCH 1/4] =?UTF-8?q?GRAMEX-207=20=E2=81=83=20Logviewer=20should?= =?UTF-8?q?=20log=20to=20a=20central=20database=20(#553)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gramex/apps/logviewer/gramex.yaml | 6 + gramex/apps/logviewer/logviewer.py | 135 ++++++++---------- gramex/config.py | 30 ++-- gramex/data.py | 7 +- testlib/test_config/__init__.py | 4 + testlib/test_config/config.template.base.yaml | 8 ++ 6 files changed, 102 insertions(+), 88 deletions(-) diff --git a/gramex/apps/logviewer/gramex.yaml b/gramex/apps/logviewer/gramex.yaml index 4c86f321b..9f03939af 100644 --- a/gramex/apps/logviewer/gramex.yaml +++ b/gramex/apps/logviewer/gramex.yaml @@ -1,4 +1,5 @@ # Configurable variables +# LOGVIEWER_DB # LOGVIEWER_PATH_UI # LOGVIEWER_PATH_RENDER # $LOGVIEWER_FORMHANDLER_KWARGS @@ -9,6 +10,9 @@ # $LOGVIEWER_SCHEDULER_KWARGS variables: + LOGVIEWER_DB: + default: + url: sqlite:///$GRAMEXDATA/logs/logviewer.db LOGVIEWER_SCHEDULER_PORT: default: '' LOGVIEWER_PATH_UI: @@ -149,6 +153,7 @@ schedule: apps/logviewer-$* if '--listen.port=' + LOGVIEWER_SCHEDULER_PORT in ''.join(sys.argv[1:]) or not LOGVIEWER_SCHEDULER_PORT: function: logviewer.summarize kwargs: + db: $LOGVIEWER_DB custom_dims: import.merge: $LOGVIEWER_CUSTOM_DIMENSIONS session_threshold: 15 @@ -187,6 +192,7 @@ schedule: op: NOTCONTAINS value: '\.js|\.css|\.ico|\.png|\.jpg|\.jpeg|\.gif|\.otf|\.woff.*|\.eot' as: uri_1 + # TODO: this may not work as logviewer.summarize() does not accept any kwargs! import.merge: $LOGVIEWER_SCHEDULER_KWARGS startup: true # Run at 6pm local time. In India, this is a bit after 0:00 UTC, diff --git a/gramex/apps/logviewer/logviewer.py b/gramex/apps/logviewer/logviewer.py index e3b3b854d..c48837691 100644 --- a/gramex/apps/logviewer/logviewer.py +++ b/gramex/apps/logviewer/logviewer.py @@ -1,7 +1,6 @@ import re import sys import os.path -import sqlite3 from glob import glob # lxml.etree is safe on https://github.com/tiran/defusedxml/tree/main/xmltestdata from lxml.etree import Element # nosec: lxml is fixed @@ -13,6 +12,7 @@ from gramex import conf from gramex.config import app_log from gramex.transforms import build_transform +from typing import List if sys.version_info.major == 3: unicode = str @@ -20,8 +20,10 @@ DB_CONFIG = { 'table': 'agg{}', 'levels': ['M', 'W', 'D'], - 'dimensions': [{'key': 'time', 'freq': '?level'}, - 'user.id', 'ip', 'status', 'uri'], + 'dimensions': [ + {'key': 'time', 'freq': '?level'}, + 'user.id', 'ip', 'status', 'uri' + ], 'metrics': { 'duration': ['count', 'sum'], 'new_session': ['sum'], @@ -29,6 +31,7 @@ } } +# TODO: extra_columns should not be a global. Once instance may use multiple logviewers! extra_columns = [] for key in conf.get('schedule', []): if 'kwargs' in conf.schedule[key] and 'custom_dims' in conf.schedule[key].kwargs: @@ -61,12 +64,6 @@ def pdagg(df, groups, aggfuncs): return dff.reset_index() -def table_exists(table, conn): - '''check if table exists in sqlite db''' - query = "SELECT name FROM sqlite_master WHERE type='table' AND name=?" - return not pd.read_sql(query, conn, params=[table]).empty - - def add_session(df, duration=30, cutoff_buffer=0): '''add new_session based on `duration` threshold add cutoff_buffer in minutes for first and last session requests @@ -103,21 +100,28 @@ def prepare_logs(df, session_threshold=15, cutoff_buffer=0, custom_dims={}): return df -def create_column_if_not_exists(table, freq, conn): - for col in extra_columns: - for row in conn.execute(f'PRAGMA table_info({table(freq)})'): - if row[1] == col: - break - else: - query = f'ALTER TABLE {table(freq)} ADD COLUMN "{col}" TEXT DEFAULT ""' - conn.execute(query) - conn.commit() - - -def summarize(transforms=[], post_transforms=[], run=True, - session_threshold=15, cutoff_buffer=0, custom_dims=None): - '''summarize''' - app_log.info('logviewer: Summarize started') +def summarize( + db: dict, + transforms: List[dict] = [], + post_transforms: List[dict] = [], + session_threshold: float = 15, + cutoff_buffer: float = 0, + custom_dims: dict = None) -> None: + '''Summarizes log files into a database periodically. + + Parameters: + db: SQLAlchemy database configuration. + transforms: List of transforms to be applied on data. + post_transforms: List of post transforms to be applied on data. + session_threshold: Minimum threshold for the session. + cutoff_buffer: In minutes for first and last session requests. + custom_dims: Custom columns to be added to the logviewer. + + This function is called by a scheduler and/or on start of gramex. + It will aggregate and update logs from requests.csv file by comparing the + timestamp of last added logs. It creates the aggregation tables if they don't exist. + ''' + app_log.info('logviewer.summarize started') levels = DB_CONFIG['levels'] table = DB_CONFIG['table'].format # dimensions and metrics to summarize @@ -126,70 +130,52 @@ def summarize(transforms=[], post_transforms=[], run=True, log_file = conf.log.handlers.requests.filename # Handle for multiple instances requests.csv$LISTENPORT log_file = '{0}{1}'.format(*log_file.partition('.csv')) - folder = os.path.dirname(log_file) - conn = sqlite3.connect(os.path.join(folder, 'logviewer.db')) - for freq in levels: - try: - create_column_if_not_exists(table, freq, conn) - except sqlite3.OperationalError: - # Inform when table is created for the first time - app_log.info('logviewer: OperationalError: Table does not exist') - - # drop agg tables from database - if run in ['drop', 'reload']: - droptable = 'DROP TABLE IF EXISTS {}'.format - for freq in levels: - app_log.info('logviewer: Dropping {} table'.format(table(freq))) - conn.execute(droptable(table(freq))) - conn.commit() - conn.execute('VACUUM') - if run == 'drop': - conn.close() - return # all log files sorted by modified time log_files = sorted(glob(log_file + '*'), key=os.path.getmtime) - max_date = None def filesince(filename, date): match = re.search(r'(\d{4}-\d{2}-\d{2})$', filename) backupdate = match.group() if match else '' return backupdate >= date or backupdate == '' - # get this month log files if db is already created - if table_exists(table(levels[-1]), conn): - query = 'SELECT MAX(time) FROM {}'.format(table(levels[-1])) # nosec: table() is safe - max_date = pd.read_sql(query, conn).iloc[0, 0] - app_log.info(f'logviewer: last processed till {max_date}') - this_month = max_date[:8] + '01' - log_files = [f for f in log_files if filesince(f, this_month)] + # get most recent log files if db is already created + try: + log_filter = gramex.data.filter(**db, table=table(levels[-1]), args={}) + max_date = log_filter.sort_values('time', ascending=False)['time'].iloc[0] max_date = pd.to_datetime(max_date) + except Exception: # noqa + max_date = None + else: + app_log.info(f'logviewer.summarize: processing since {max_date}') + this_month = max_date.strftime('%Y-%m-01') + log_files = [f for f in log_files if filesince(f, this_month)] if not log_files: - app_log.info('logviewer: no log files to process') + app_log.info('logviewer.summarize: no log files to process') return # Create dataframe from log files columns = conf.log.handlers.requests['keys'] - # TODO: avoid concat? - app_log.info(f'logviewer: files to process {log_files}') + app_log.info(f'logviewer.summarize: processing {log_files}') data = pd.concat([ pd.read_csv(f, names=columns, encoding='utf-8').fillna('-') for f in log_files ], ignore_index=True) app_log.info( - 'logviewer: prepare_logs {} rows with {} mint session_threshold'.format( + 'logviewer.summarize: prepare_logs {} rows with session_threshold={}'.format( len(data.index), session_threshold)) data = prepare_logs(df=data, session_threshold=session_threshold, cutoff_buffer=cutoff_buffer, custom_dims=custom_dims) - app_log.info('logviewer: processed and returned {} rows'.format(len(data.index))) + app_log.info('logviewer.summarize: processed {} rows'.format(len(data.index))) # apply transforms on raw data - app_log.info('logviewer: applying transforms') + app_log.info('logviewer.summarize: applying transforms') for spec in transforms: apply_transform(data, spec) # applies on copy # levels should go from M > W > D for freq in levels: + app_log.info('logviewer.summarize: aggregating {}'.format(table(freq))) # filter dataframe for max_date.level if max_date: date_from = max_date @@ -199,29 +185,30 @@ def filesince(filename, date): date_from -= pd.offsets.MonthBegin(1) data = data[data.time.ge(date_from)] # delete old records - query = f'DELETE FROM {table(freq)} WHERE time >= ?' # nosec: table() is safe - conn.execute(query, (f'{date_from}',)) - conn.commit() + gramex.data.delete(**db, table=table(freq), args={'time>~': [date_from]}, id=['time']) groups[0]['freq'] = freq # get summary view - app_log.info('logviewer: pdagg for {}'.format(table(freq))) dff = pdagg(data, groups, aggfuncs) # apply post_transforms here - app_log.info('logviewer: applying post_transforms') for spec in post_transforms: apply_transform(dff, spec) # insert new records - try: - dff.to_sql(table(freq), conn, if_exists='append', index=False) - # dff columns should match with table columns - # if not, call summarize run='reload' to - # drop all the tables and rerun the job - except sqlite3.OperationalError: - app_log.info('logviewer: OperationalError: run: reload') - summarize(transforms=transforms, run='reload') - return - conn.close() - app_log.info('logviewer: Summarize completed') + cols = {} + for col in dff.columns: + dt = dff[col].dtype.type + if pd.api.types.is_datetime64_any_dtype(dt): + cols[col] = 'DATETIME' + elif pd.api.types.is_bool_dtype(dt): + cols[col] = 'BOOLEAN' + elif pd.api.types.is_integer_dtype(dt): + cols[col] = 'INTEGER' + elif pd.api.types.is_numeric_dtype(dt): + cols[col] = 'REAL' + else: + cols[col] = 'TEXT' + gramex.data.alter(**db, table=table(freq), columns=cols) + gramex.data.insert(**db, table=table(freq), args=dff.to_dict()) + app_log.info('logviewer.summarize: completed') return diff --git a/gramex/config.py b/gramex/config.py index 6e3560499..785f8563a 100644 --- a/gramex/config.py +++ b/gramex/config.py @@ -227,23 +227,27 @@ def _substitute_variable(val): def _calc_value(val, key): - ''' - Calculate the value to assign to this key. + '''Calculate the value to assign to this key. - If ``val`` is not a dictionary that has a ``function`` key, return it as-is. + If val is a scalar (string, boolean, dict, etc), return as-is. - If it has a function key, call that function (with specified args, kwargs, - etc) and allow the ``key`` parameter as an argument. + If it's a list or a dict, return calculated values of underlying values. - If the function is a generator, the first value is used. + If it's a dict with a `function` key, evaluation the function and return the non-None value. + If the returned value(s) are None, return the calculated 'default' value. ''' - if hasattr(val, 'get') and val.get('function'): - from .transforms import build_transform - function = build_transform(val, vars={'key': None}, filename=f'config:{key}') - for result in function(key): - if result is not None: - return result - return val.get('default') + if isinstance(val, dict): + if val.get('function'): + from .transforms import build_transform + function = build_transform(val, vars={'key': None}, filename=f'config:{key}') + for result in function(key): + if result is not None: + return result + return _calc_value(val.get('default', None), key) + else: + return {k: _calc_value(v, k) for k, v in val.items()} + elif isinstance(val, (list, tuple)): + return [_calc_value(v, key) for v in val] else: return _substitute_variable(val) diff --git a/gramex/data.py b/gramex/data.py index 51fcfa702..e86bf3d7d 100644 --- a/gramex/data.py +++ b/gramex/data.py @@ -14,6 +14,7 @@ from typing import Callable, List, Union from gramex.config import merge, app_log from orderedattrdict import AttrDict +from urllib.parse import urlparse _ENGINE_CACHE = {} _METADATA_CACHE = {} @@ -904,6 +905,11 @@ def alter(url: str, table: str, columns: dict = None, **kwargs: dict) -> sa.engi engine = create_engine(url, **kwargs) if columns is None: return engine + # alter is not required for schema-less databases. For now, hard-code engine names + scheme = urlparse(url).scheme + if scheme in {'mongodb', 'elasticsearch', 'influxdb'}: + app_log.info(f'alter() not required for schema-less DB {engine.driver}') + return engine try: db_table = get_table(engine, table) except sa.exc.NoSuchTableError: @@ -1783,7 +1789,6 @@ def _insert_influxdb(url, rows, meta, args, bucket, **kwargs): def _filter_servicenow(url, controls, args, meta, table=None, columns=None, query=None, **kwargs): import pysnow from gramex.config import locate - from urllib.parse import urlparse urlinfo = urlparse(url) c = pysnow.Client(instance=urlinfo.hostname, user=urlinfo.username, password=urlinfo.password) diff --git a/testlib/test_config/__init__.py b/testlib/test_config/__init__.py index 6079b6d90..842bb17a1 100644 --- a/testlib/test_config/__init__.py +++ b/testlib/test_config/__init__.py @@ -246,6 +246,10 @@ def test_variables(self): eq_(conf['boolean'], True) eq_(conf['object'], {'x': 1}) eq_(conf['list'], [1, 2]) + # Check if substitutions work inside dicts and lists + eq_(conf['object_calc'], {'x': 'base'}) + eq_(conf['list_calc'], ['base']) + eq_(conf['object_default'], {'x': 'base'}) # Check if variables of different types are string substituted eq_(conf['numeric_subst'], '/1') diff --git a/testlib/test_config/config.template.base.yaml b/testlib/test_config/config.template.base.yaml index 494aad773..60bbba704 100644 --- a/testlib/test_config/config.template.base.yaml +++ b/testlib/test_config/config.template.base.yaml @@ -11,13 +11,18 @@ variables: default: function: str args: ['base'] + DEFAULT_OBJECT: + default: + x: $THIS FUNCTION_VAR: function: gramex.config.variables['ROOT'] + gramex.config.variables['THIS'] DERIVED: $THIS/derived NUMERIC: 1 BOOLEAN: true OBJECT: {x: 1} + OBJECT_CALC: {x: $THIS} LIST: [1, 2] + LIST_CALC: [$THIS] CONDITION1: function: condition @@ -93,7 +98,10 @@ path: $YAMLPATH numeric: "$NUMERIC" boolean: '$BOOLEAN' object: $OBJECT +object_calc: $OBJECT_CALC +object_default: $DEFAULT_OBJECT list: $LIST +list_calc: $LIST_CALC numeric_subst: "/${NUMERIC}" boolean_subst: "/${BOOLEAN}" object_subst: "/${OBJECT}" From be05246433945543128a23c0864fda46dbed376c Mon Sep 17 00:00:00 2001 From: S Anand Date: Thu, 23 Jun 2022 07:57:09 +0530 Subject: [PATCH 2/4] STY: use editorconfig-checker instead of eclint. Fix lint errors * REF: If config is deleted, log as DEBUG not INFO * STY: use editorconfig-checker instead of eclint. Fix lint errors --- .editorconfig | 14 +- gramex/apps/ui/setup.js | 4 +- .../ui/theme/themes-guide/blue_voltage.scss | 2 +- .../apps/ui/theme/themes-guide/boldstrap.scss | 2 +- .../theme/themes-guide/bootstrap_purple.scss | 2 +- .../apps/ui/theme/themes-guide/darkster.scss | 2 +- gramex/apps/ui/theme/themes-guide/fresca.scss | 2 +- .../apps/ui/theme/themes-guide/greyson.scss | 2 +- .../ui/theme/themes-guide/hello_kiddie.scss | 2 +- gramex/apps/ui/theme/themes-guide/herbie.scss | 2 +- .../apps/ui/theme/themes-guide/hootstrap.scss | 2 +- gramex/apps/ui/theme/themes-guide/lovey.scss | 2 +- .../apps/ui/theme/themes-guide/monotony.scss | 2 +- .../apps/ui/theme/themes-guide/poypull.scss | 2 +- gramex/apps/ui/theme/themes-guide/signal.scss | 2 +- .../apps/ui/theme/themes-guide/tequila.scss | 2 +- gramex/apps/ui/theme/themes.json | 2 +- gramex/apps/uifactory/assets/data/input.json | 4 +- gramex/config.py | 3 +- package-lock.json | 439 +++++++++++++----- package.json | 1 + task | 11 +- 22 files changed, 349 insertions(+), 157 deletions(-) diff --git a/.editorconfig b/.editorconfig index 067b2ad69..212d1fdf8 100644 --- a/.editorconfig +++ b/.editorconfig @@ -7,7 +7,7 @@ root = true # Apply common styles for most standard code files. # Do not apply to * - that covers binary files as well -[*.{js,html,php,py,css,svg,json,less,yaml,yml,scss,xml,sh,java,bat,R}] +[*.{js,html,php,css,svg,json,less,yaml,yml,scss,xml,sh,java,bat,R}] end_of_line = lf insert_final_newline = true trim_trailing_whitespace = true @@ -15,15 +15,3 @@ charset = utf-8 # Stick to 2-space indenting by default, to conserve space indent_style = space indent_size = 2 - -[*.py] -indent_size = 4 - -[Makefile] -indent_style = tab -indent_size = 4 - -[testlib/test_config/config.empty.yaml] -insert_final_newline = false -[tests/dir/gramex.yaml] -insert_final_newline = false diff --git a/gramex/apps/ui/setup.js b/gramex/apps/ui/setup.js index adba9bd23..6a537a148 100644 --- a/gramex/apps/ui/setup.js +++ b/gramex/apps/ui/setup.js @@ -41,14 +41,14 @@ fs.readdirSync(themes_guide_root).forEach(function (dir) { fs.readFileSync(theme_file, 'utf8') .replace('@import "bootstrap";', '@import "gramexui";') // Themes Guide disables grid classes. But we want to use them, so kill this line - .replace('$enable-grid-classes:false;\n', '')) + .replace('$enable-grid-classes:false;\n', '') + '\n') themes.push(`themes-guide/${dir}`) } }) execSync('rm -rf bootstrap-themes', { cwd: tmp }) // Save list of themes -fs.writeFileSync('theme/themes.json', JSON.stringify({ 'themes': themes })) +fs.writeFileSync('theme/themes.json', JSON.stringify({ 'themes': themes }) + '\n') // Utility functions diff --git a/gramex/apps/ui/theme/themes-guide/blue_voltage.scss b/gramex/apps/ui/theme/themes-guide/blue_voltage.scss index 229821f3b..e256652fc 100644 --- a/gramex/apps/ui/theme/themes-guide/blue_voltage.scss +++ b/gramex/apps/ui/theme/themes-guide/blue_voltage.scss @@ -24,4 +24,4 @@ $btn-border-radius-lg:1.6rem; $btn-border-radius-sm:.8rem; @import "gramexui"; -// Add SASS theme customizations here.. \ No newline at end of file +// Add SASS theme customizations here.. diff --git a/gramex/apps/ui/theme/themes-guide/boldstrap.scss b/gramex/apps/ui/theme/themes-guide/boldstrap.scss index c56f06a43..d875ed755 100644 --- a/gramex/apps/ui/theme/themes-guide/boldstrap.scss +++ b/gramex/apps/ui/theme/themes-guide/boldstrap.scss @@ -18,4 +18,4 @@ $dark:#3c4055; $body-bg:#efefef; @import "gramexui"; -// Add SASS theme customizations here.. \ No newline at end of file +// Add SASS theme customizations here.. diff --git a/gramex/apps/ui/theme/themes-guide/bootstrap_purple.scss b/gramex/apps/ui/theme/themes-guide/bootstrap_purple.scss index b761e9935..6ffad9cbb 100644 --- a/gramex/apps/ui/theme/themes-guide/bootstrap_purple.scss +++ b/gramex/apps/ui/theme/themes-guide/bootstrap_purple.scss @@ -10,4 +10,4 @@ $light:#f8f9fa; $dark:#343434; @import "gramexui"; -// Add SASS theme customizations here.. \ No newline at end of file +// Add SASS theme customizations here.. diff --git a/gramex/apps/ui/theme/themes-guide/darkster.scss b/gramex/apps/ui/theme/themes-guide/darkster.scss index f5ec56255..33aa5deca 100644 --- a/gramex/apps/ui/theme/themes-guide/darkster.scss +++ b/gramex/apps/ui/theme/themes-guide/darkster.scss @@ -73,4 +73,4 @@ $breadcrumb-active-color:$gray-500; @import "gramexui"; // Add SASS theme customizations here.. -.navbar-dark.bg-primary {background-color:#111111 !important;} \ No newline at end of file +.navbar-dark.bg-primary {background-color:#111111 !important;} diff --git a/gramex/apps/ui/theme/themes-guide/fresca.scss b/gramex/apps/ui/theme/themes-guide/fresca.scss index dcb680c4a..e22c4a37d 100644 --- a/gramex/apps/ui/theme/themes-guide/fresca.scss +++ b/gramex/apps/ui/theme/themes-guide/fresca.scss @@ -12,4 +12,4 @@ $light:#FAFAFA; $dark:#4e4e4e; @import "gramexui"; -// Add SASS theme customizations here.. \ No newline at end of file +// Add SASS theme customizations here.. diff --git a/gramex/apps/ui/theme/themes-guide/greyson.scss b/gramex/apps/ui/theme/themes-guide/greyson.scss index 1598eaef0..d9d679e7f 100644 --- a/gramex/apps/ui/theme/themes-guide/greyson.scss +++ b/gramex/apps/ui/theme/themes-guide/greyson.scss @@ -20,4 +20,4 @@ $dark:#1e2b37; $enable-rounded:false; @import "gramexui"; -// Add SASS theme customizations here.. \ No newline at end of file +// Add SASS theme customizations here.. diff --git a/gramex/apps/ui/theme/themes-guide/hello_kiddie.scss b/gramex/apps/ui/theme/themes-guide/hello_kiddie.scss index 897d83c15..65a69ca75 100644 --- a/gramex/apps/ui/theme/themes-guide/hello_kiddie.scss +++ b/gramex/apps/ui/theme/themes-guide/hello_kiddie.scss @@ -13,4 +13,4 @@ $dark:#223322; @import "gramexui"; // Add SASS theme customizations here.. -html {font-size: 0.8rem;}@include media-breakpoint-up(sm) {html {font-size: .9rem;}}@include media-breakpoint-up(md) {html {font-size: 1rem;}}@include media-breakpoint-up(lg) {html {font-size: 1.1rem;}} \ No newline at end of file +html {font-size: 0.8rem;}@include media-breakpoint-up(sm) {html {font-size: .9rem;}}@include media-breakpoint-up(md) {html {font-size: 1rem;}}@include media-breakpoint-up(lg) {html {font-size: 1.1rem;}} diff --git a/gramex/apps/ui/theme/themes-guide/herbie.scss b/gramex/apps/ui/theme/themes-guide/herbie.scss index 1cdf060a7..eb4f6db1d 100644 --- a/gramex/apps/ui/theme/themes-guide/herbie.scss +++ b/gramex/apps/ui/theme/themes-guide/herbie.scss @@ -14,4 +14,4 @@ $light:#F2F2F0; $dark:#072247; @import "gramexui"; -// Add SASS theme customizations here.. \ No newline at end of file +// Add SASS theme customizations here.. diff --git a/gramex/apps/ui/theme/themes-guide/hootstrap.scss b/gramex/apps/ui/theme/themes-guide/hootstrap.scss index 39f857bea..6a5b45887 100644 --- a/gramex/apps/ui/theme/themes-guide/hootstrap.scss +++ b/gramex/apps/ui/theme/themes-guide/hootstrap.scss @@ -14,4 +14,4 @@ $light:#FDFBF7; $dark:#555555; @import "gramexui"; -// Add SASS theme customizations here.. \ No newline at end of file +// Add SASS theme customizations here.. diff --git a/gramex/apps/ui/theme/themes-guide/lovey.scss b/gramex/apps/ui/theme/themes-guide/lovey.scss index 09431f7b4..cd6236871 100644 --- a/gramex/apps/ui/theme/themes-guide/lovey.scss +++ b/gramex/apps/ui/theme/themes-guide/lovey.scss @@ -14,4 +14,4 @@ $light:#eeeeee; $dark:#353535; @import "gramexui"; -// Add SASS theme customizations here.. \ No newline at end of file +// Add SASS theme customizations here.. diff --git a/gramex/apps/ui/theme/themes-guide/monotony.scss b/gramex/apps/ui/theme/themes-guide/monotony.scss index 1a535b0b6..be6e4253e 100644 --- a/gramex/apps/ui/theme/themes-guide/monotony.scss +++ b/gramex/apps/ui/theme/themes-guide/monotony.scss @@ -14,4 +14,4 @@ $light:#eceeec; $dark:#111111; @import "gramexui"; -// Add SASS theme customizations here.. \ No newline at end of file +// Add SASS theme customizations here.. diff --git a/gramex/apps/ui/theme/themes-guide/poypull.scss b/gramex/apps/ui/theme/themes-guide/poypull.scss index 9bfcf1063..9dfb935c7 100644 --- a/gramex/apps/ui/theme/themes-guide/poypull.scss +++ b/gramex/apps/ui/theme/themes-guide/poypull.scss @@ -21,4 +21,4 @@ $display3-weight:600; $navbar-dark-color:#f3f3f3; @import "gramexui"; -// Add SASS theme customizations here.. \ No newline at end of file +// Add SASS theme customizations here.. diff --git a/gramex/apps/ui/theme/themes-guide/signal.scss b/gramex/apps/ui/theme/themes-guide/signal.scss index 5d9ee4d16..f63914441 100644 --- a/gramex/apps/ui/theme/themes-guide/signal.scss +++ b/gramex/apps/ui/theme/themes-guide/signal.scss @@ -20,4 +20,4 @@ $dark:#222222; $gray-300:#e0e0e0; @import "gramexui"; -// Add SASS theme customizations here.. \ No newline at end of file +// Add SASS theme customizations here.. diff --git a/gramex/apps/ui/theme/themes-guide/tequila.scss b/gramex/apps/ui/theme/themes-guide/tequila.scss index f6a7e19cd..2e6eb40dd 100644 --- a/gramex/apps/ui/theme/themes-guide/tequila.scss +++ b/gramex/apps/ui/theme/themes-guide/tequila.scss @@ -12,4 +12,4 @@ $light:#eef0f2; $dark:#000633; @import "gramexui"; -// Add SASS theme customizations here.. \ No newline at end of file +// Add SASS theme customizations here.. diff --git a/gramex/apps/ui/theme/themes.json b/gramex/apps/ui/theme/themes.json index b465b313b..6381a2972 100644 --- a/gramex/apps/ui/theme/themes.json +++ b/gramex/apps/ui/theme/themes.json @@ -1 +1 @@ -{"themes":["default","bootstrap5","bootswatch/cerulean","bootswatch/cosmo","bootswatch/cyborg","bootswatch/darkly","bootswatch/flatly","bootswatch/journal","bootswatch/litera","bootswatch/lumen","bootswatch/lux","bootswatch/materia","bootswatch/minty","bootswatch/pulse","bootswatch/sandstone","bootswatch/simplex","bootswatch/sketchy","bootswatch/slate","bootswatch/solar","bootswatch/spacelab","bootswatch/superhero","bootswatch/united","bootswatch/yeti","themes-guide/blue_voltage","themes-guide/boldstrap","themes-guide/bootstrap_purple","themes-guide/darkster","themes-guide/fresca","themes-guide/greyson","themes-guide/hello_kiddie","themes-guide/herbie","themes-guide/hootstrap","themes-guide/lovey","themes-guide/monotony","themes-guide/poypull","themes-guide/signal","themes-guide/tequila"]} \ No newline at end of file +{"themes":["default","bootstrap5","bootswatch/cerulean","bootswatch/cosmo","bootswatch/cyborg","bootswatch/darkly","bootswatch/flatly","bootswatch/journal","bootswatch/litera","bootswatch/lumen","bootswatch/lux","bootswatch/materia","bootswatch/minty","bootswatch/pulse","bootswatch/sandstone","bootswatch/simplex","bootswatch/sketchy","bootswatch/slate","bootswatch/solar","bootswatch/spacelab","bootswatch/superhero","bootswatch/united","bootswatch/yeti","themes-guide/blue_voltage","themes-guide/boldstrap","themes-guide/bootstrap_purple","themes-guide/darkster","themes-guide/fresca","themes-guide/greyson","themes-guide/hello_kiddie","themes-guide/herbie","themes-guide/hootstrap","themes-guide/lovey","themes-guide/monotony","themes-guide/poypull","themes-guide/signal","themes-guide/tequila"]} diff --git a/gramex/apps/uifactory/assets/data/input.json b/gramex/apps/uifactory/assets/data/input.json index 47ef887a5..77280425b 100644 --- a/gramex/apps/uifactory/assets/data/input.json +++ b/gramex/apps/uifactory/assets/data/input.json @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:73efc0f7bc39eaf036dcfbbdcedd2ffed55d6a28a4e8ecf4c4388cb76bc0f484 -size 1673 +oid sha256:1e4cbe6614bf72a9618009bf15a82532ecb4801cc40e4932ef05044d33d8ffe4 +size 1674 diff --git a/gramex/config.py b/gramex/config.py index 785f8563a..b63ff7de4 100644 --- a/gramex/config.py +++ b/gramex/config.py @@ -608,9 +608,10 @@ def __pos__(self): # ... or if an imported file is deleted / updated for imp in self.__info__.imports: exists = imp.path.exists() + # If the path existed but has now been deleted, log it if not exists and imp.stat is not None: reload = True - app_log.info(f'No config found: {imp.path}') + app_log.debug(f'Config deleted: {imp.path}') break if exists and (imp.path.stat().st_mtime > imp.stat.st_mtime or imp.path.stat().st_size != imp.stat.st_size): diff --git a/package-lock.json b/package-lock.json index 8a83dca5c..f0cba4e90 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,6 +6,7 @@ "": { "license": "ISC", "devDependencies": { + "editorconfig-checker": "^4.0.2", "eslint": "^8.18.0", "eslint-plugin-html": "^6.2.0", "eslint-plugin-template": "^0.6.0" @@ -37,23 +38,6 @@ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "dev": true }, - "node_modules/@eslint/eslintrc/node_modules/debug": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", - "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "dev": true, - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/@eslint/eslintrc/node_modules/js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", @@ -66,12 +50,6 @@ "js-yaml": "bin/js-yaml.js" } }, - "node_modules/@eslint/eslintrc/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, "node_modules/@humanwhocodes/config-array": { "version": "0.9.5", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.9.5.tgz", @@ -86,29 +64,6 @@ "node": ">=10.10.0" } }, - "node_modules/@humanwhocodes/config-array/node_modules/debug": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", - "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "dev": true, - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@humanwhocodes/config-array/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, "node_modules/@humanwhocodes/object-schema": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz", @@ -136,6 +91,18 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "dev": true, + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -177,12 +144,38 @@ "node": ">=6" } }, + "node_modules/chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "dev": true, + "engines": { + "node": ">=10" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "dev": true }, + "node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -265,6 +258,21 @@ "url": "https://github.com/fb55/domutils?sponsor=1" } }, + "node_modules/editorconfig-checker": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/editorconfig-checker/-/editorconfig-checker-4.0.2.tgz", + "integrity": "sha512-tUI7ABIzMB1kfwTUQmX+gaZGCMNuUgGuRHJ+Xu4Tk9T8lV8Vy5w/EaQsSZ7NKrOgLxbekptw6MUgrzHTvhceLw==", + "dev": true, + "dependencies": { + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.6.0", + "tar": "^6.0.0" + }, + "bin": { + "ec": "dist/index.js", + "editorconfig-checker": "dist/index.js" + } + }, "node_modules/entities": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/entities/-/entities-3.0.1.tgz", @@ -550,23 +558,6 @@ "node": ">= 8" } }, - "node_modules/eslint/node_modules/debug": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", - "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "dev": true, - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/eslint/node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -624,12 +615,6 @@ "js-yaml": "bin/js-yaml.js" } }, - "node_modules/eslint/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, "node_modules/eslint/node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -804,6 +789,18 @@ "integrity": "sha512-WIWGi2L3DyTUvUrwRKgGi9TwxQMUEqPOPQBVi71R96jZXJdFskXEmf54BoZaS1kknGODoIGASGEzBUYdyMCBJg==", "dev": true }, + "node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dev": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -870,6 +867,19 @@ "entities": "^3.0.1" } }, + "node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "dev": true, + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/ignore": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.0.tgz", @@ -978,12 +988,75 @@ "node": "*" } }, + "node_modules/minipass": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.3.tgz", + "integrity": "sha512-N0BOsdFAlNRfmwMhjAsLVWOk7Ljmeb39iqFlsV1At+jqRhSUP9yeof8FyJu4imaJiSUp8vQebWD/guZwGQC8iA==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dev": true, + "dependencies": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true, + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, + "node_modules/node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "dev": true, + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -1140,12 +1213,35 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/tar": { + "version": "6.1.11", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.11.tgz", + "integrity": "sha512-an/KZQzQUkZCkuoAA64hM92X0Urb6VpRhAFllDzz44U2mcD5scmT3zBc4VgVpkugF580+DQn8eAFSyoQt0tznA==", + "dev": true, + "dependencies": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^3.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">= 10" + } + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", "dev": true }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -1191,6 +1287,22 @@ "integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==", "dev": true }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", @@ -1205,6 +1317,12 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "dev": true + }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true } }, "dependencies": { @@ -1231,15 +1349,6 @@ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "dev": true }, - "debug": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", - "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, "js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", @@ -1248,12 +1357,6 @@ "requires": { "argparse": "^2.0.1" } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true } } }, @@ -1266,23 +1369,6 @@ "@humanwhocodes/object-schema": "^1.2.1", "debug": "^4.1.1", "minimatch": "^3.0.4" - }, - "dependencies": { - "debug": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", - "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - } } }, "@humanwhocodes/object-schema": { @@ -1304,6 +1390,15 @@ "dev": true, "requires": {} }, + "agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "dev": true, + "requires": { + "debug": "4" + } + }, "ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -1338,12 +1433,27 @@ "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", "dev": true }, + "chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "dev": true + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "dev": true }, + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, "deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -1404,6 +1514,17 @@ "domhandler": "^4.2.0" } }, + "editorconfig-checker": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/editorconfig-checker/-/editorconfig-checker-4.0.2.tgz", + "integrity": "sha512-tUI7ABIzMB1kfwTUQmX+gaZGCMNuUgGuRHJ+Xu4Tk9T8lV8Vy5w/EaQsSZ7NKrOgLxbekptw6MUgrzHTvhceLw==", + "dev": true, + "requires": { + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.6.0", + "tar": "^6.0.0" + } + }, "entities": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/entities/-/entities-3.0.1.tgz", @@ -1510,15 +1631,6 @@ "which": "^2.0.1" } }, - "debug": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", - "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, "escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -1558,12 +1670,6 @@ "argparse": "^2.0.1" } }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, "path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -1820,6 +1926,15 @@ "integrity": "sha512-WIWGi2L3DyTUvUrwRKgGi9TwxQMUEqPOPQBVi71R96jZXJdFskXEmf54BoZaS1kknGODoIGASGEzBUYdyMCBJg==", "dev": true }, + "fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dev": true, + "requires": { + "minipass": "^3.0.0" + } + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -1867,6 +1982,16 @@ "entities": "^3.0.1" } }, + "https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "dev": true, + "requires": { + "agent-base": "6", + "debug": "4" + } + }, "ignore": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.0.tgz", @@ -1954,12 +2079,52 @@ "brace-expansion": "^1.1.7" } }, + "minipass": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.3.tgz", + "integrity": "sha512-N0BOsdFAlNRfmwMhjAsLVWOk7Ljmeb39iqFlsV1At+jqRhSUP9yeof8FyJu4imaJiSUp8vQebWD/guZwGQC8iA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dev": true, + "requires": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + } + }, + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, + "node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "dev": true, + "requires": { + "whatwg-url": "^5.0.0" + } + }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -2063,12 +2228,32 @@ "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", "dev": true }, + "tar": { + "version": "6.1.11", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.11.tgz", + "integrity": "sha512-an/KZQzQUkZCkuoAA64hM92X0Urb6VpRhAFllDzz44U2mcD5scmT3zBc4VgVpkugF580+DQn8eAFSyoQt0tznA==", + "dev": true, + "requires": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^3.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + } + }, "text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", "dev": true }, + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true + }, "type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -2105,6 +2290,22 @@ "integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==", "dev": true }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", @@ -2116,6 +2317,12 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "dev": true + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true } } } diff --git a/package.json b/package.json index 7fa94d7a7..0ce2f2e9e 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ }, "license": "ISC", "devDependencies": { + "editorconfig-checker": "^4.0.2", "eslint": "^8.18.0", "eslint-plugin-html": "^6.2.0", "eslint-plugin-template": "^0.6.0" diff --git a/task b/task index 6552ec14f..cae99327e 100644 --- a/task +++ b/task @@ -19,16 +19,11 @@ lint () { # Python security check with bandit command -v bandit 2>/dev/null 2>&1 || pip install bandit - bandit gramex --aggregate vuln --recursive --exclude '*/node_modules/*' --quiet + bandit gramex --aggregate vuln --recursive --quiet - # Install eslint and dependencies. Then test all apps + # Install linters. Then test all apps npm install - npx eslint --ext js,html gramex/ - - # Avoid eclint in package.json (npm audit issues). Install globally - command -v eclint 2>/dev/null 2>&1 || npm install -g eclint - # Run .editorconfig checks. - find . -type f ! -path '*/.git/*' ! -path '*/node_modules/*' ! -path '*/mkdocs/*' ! -path '*/__pycache__/*' ! -path '*/filemanager/test/tape.js' -print0 | xargs -0 eclint check + npx editorconfig-checker # Avoid htmllint-cli in package.json (npm audit issues). Install globally command -v htmllint 2>/dev/null 2>&1 || npm install -g htmllint-cli From 1906389650a482aad5676dea5132113a72755401 Mon Sep 17 00:00:00 2001 From: S Anand Date: Thu, 23 Jun 2022 07:58:27 +0530 Subject: [PATCH 3/4] Use bandit nosec only for specific security tests (#562) Bandit security tests can be skipped with a # nosec comment. But these are broad and skip all security tests. Use to specifically skip only 1 test for the given line. --- .bandit | 5 +++++ gramex/apps/admin/controlpanel.py | 5 +++-- gramex/apps/admin2/gramexadmin.py | 17 +++++++++-------- gramex/apps/logviewer/logviewer.py | 6 +++--- gramex/apps/ui/__init__.py | 6 ++++-- gramex/cache.py | 26 ++++++++++++++++---------- gramex/config.py | 13 ++++++++----- gramex/data.py | 3 ++- gramex/handlers/capturehandler.py | 8 +++++--- gramex/handlers/drivehandler.py | 3 ++- gramex/handlers/processhandler.py | 3 ++- gramex/handlers/socialhandler.py | 3 ++- gramex/install.py | 28 ++++++++++++++++------------ gramex/license.py | 2 +- gramex/pptgen/commands.py | 15 +++++++-------- gramex/pptgen/utils.py | 6 +++--- gramex/pptgen2/__init__.py | 4 ++-- gramex/pptgen2/commands.py | 4 ++-- gramex/services/__init__.py | 4 ++-- gramex/services/rediscache.py | 9 ++++++--- gramex/services/urlcache.py | 6 ++++-- gramex/transforms/transforms.py | 16 ++++++++-------- task | 2 +- testlib/test_cache_module.py | 4 ++-- testlib/test_pptgen.py | 3 ++- tests/__init__.py | 4 +++- 26 files changed, 120 insertions(+), 85 deletions(-) create mode 100644 .bandit diff --git a/.bandit b/.bandit new file mode 100644 index 000000000..f8db4b86d --- /dev/null +++ b/.bandit @@ -0,0 +1,5 @@ +[bandit] +; Only test the Gramex source folder, not tests or testlib +exclude = */tests/*,*/testlib/*,*/node_modules/* +; B101:assert_used - assertions are used in test cases and are harmless in code +skips = B101 diff --git a/gramex/apps/admin/controlpanel.py b/gramex/apps/admin/controlpanel.py index 37c6696e9..89bba100c 100644 --- a/gramex/apps/admin/controlpanel.py +++ b/gramex/apps/admin/controlpanel.py @@ -117,10 +117,11 @@ def evaluate(handler, code): # Run code and get the result. (Result is None for exec) try: context = contexts.setdefault(handler.session['id'], {}) + # B307:eval B102:exec_used is safe since only admin can run this if mode == 'eval': - result = eval(co, context) # nosec: only admin can run this + result = eval(co, context) # nosec B307 else: - exec(co, context) # nosec: only admin can run this + exec(co, context) # nosec B102 result = None except Exception as e: result = e diff --git a/gramex/apps/admin2/gramexadmin.py b/gramex/apps/admin2/gramexadmin.py index e87386877..450f7edfa 100644 --- a/gramex/apps/admin2/gramexadmin.py +++ b/gramex/apps/admin2/gramexadmin.py @@ -149,10 +149,11 @@ def evaluate(handler, code): try: context = contexts.setdefault(handler.session['id'], {}) context['handler'] = handler + # B307:eval B102:exec_used is safe since only admin can run this if mode == 'eval': - result = eval(co, context) # nosec: only admin can run this + result = eval(co, context) # nosec B307 else: - exec(co, context) # nosec: only admin can run this + exec(co, context) # nosec B102 result = None except Exception as e: result = e @@ -200,12 +201,12 @@ def system_information(handler): from gramex.cache import Subprocess apps = { - # shell=True is safe here since the code is constructed entirely in this function - # We use shell to pick up the commands' paths from the shell. - ('node', 'version'): Subprocess('node --version', shell=True), # nosec - ('npm', 'version'): Subprocess('npm --version', shell=True), # nosec - ('yarn', 'version'): Subprocess('yarn --version', shell=True), # nosec - ('git', 'version'): Subprocess('git --version', shell=True), # nosec + # B602:any_other_function_with_shell_equals_true is safe here since the code is + # constructed entirely in this function. We use shell to pick up the commands' paths. + ('node', 'version'): Subprocess('node --version', shell=True), # nosec 602 + ('npm', 'version'): Subprocess('npm --version', shell=True), # nosec 602 + ('yarn', 'version'): Subprocess('yarn --version', shell=True), # nosec 602 + ('git', 'version'): Subprocess('git --version', shell=True), # nosec 602 } for key, proc in apps.items(): stdout, stderr = yield proc.wait_for_exit() diff --git a/gramex/apps/logviewer/logviewer.py b/gramex/apps/logviewer/logviewer.py index c48837691..57a82b9d1 100644 --- a/gramex/apps/logviewer/logviewer.py +++ b/gramex/apps/logviewer/logviewer.py @@ -2,9 +2,9 @@ import sys import os.path from glob import glob -# lxml.etree is safe on https://github.com/tiran/defusedxml/tree/main/xmltestdata -from lxml.etree import Element # nosec: lxml is fixed -from lxml.html import fromstring, tostring # nosec: lxml is fixed +# B410:import_lxml lxml.etree is safe on https://github.com/tiran/defusedxml/tree/main/xmltestdata +from lxml.etree import Element # nosec B410 +from lxml.html import fromstring, tostring # nosec B410 import numpy as np import pandas as pd import gramex.data diff --git a/gramex/apps/ui/__init__.py b/gramex/apps/ui/__init__.py index 055d4e64c..9ac230e0a 100644 --- a/gramex/apps/ui/__init__.py +++ b/gramex/apps/ui/__init__.py @@ -6,7 +6,8 @@ import gramex import gramex.cache import string -import subprocess # nosec: only for JS compilation +# B404:import_subprocess only for JS compilation +import subprocess # nosec B404 from hashlib import md5 from tornado.gen import coroutine, Return from functools import partial @@ -29,7 +30,8 @@ def join(*args): def get_cache_key(state): cache_key = json.dumps(state, sort_keys=True, ensure_ascii=True).encode('utf-8') - return md5(cache_key).hexdigest()[:5] # nosec: non-cryptographic use + # B303:md5 is safe here - it's not for cryptographic use + return md5(cache_key).hexdigest()[:5] # nosec B303 @coroutine diff --git a/gramex/cache.py b/gramex/cache.py index c95397cf8..fae6dec8f 100644 --- a/gramex/cache.py +++ b/gramex/cache.py @@ -9,7 +9,8 @@ import pandas as pd import re import requests -import subprocess # nosec: only enabled via app developer +# B404:import_subprocess only developers can access this, not users +import subprocess # nosec B404 import sys import tempfile import time @@ -480,7 +481,8 @@ def __init__( # http://stackoverflow.com/a/4896288/100904 kwargs['close_fds'] = 'posix' in sys.builtin_module_names - self.proc = subprocess.Popen(args, **kwargs) # nosec - developer-initiated + # B603:subprocess_without_shell_equals_true: only developers can access this, not users + self.proc = subprocess.Popen(args, **kwargs) # nosec B603 self.thread = {} # Has the running threads self.future = {} # Stores the futures indicating stream close self.loop = _get_current_ioloop() @@ -618,14 +620,16 @@ def daemon(args, restart=1, first_line=None, stream=True, timeout=5, buffer_size # If process was never started, start it if key not in _daemons: - started = _daemons[key] = Subprocess(args, **kwargs) # nosec: developer-initiated + # B404:import_subprocess only developers can access this, not users + started = _daemons[key] = Subprocess(args, **kwargs) # nosec B404 # Ensure that process is running. Restart if required proc = _daemons[key] restart = int(restart) while proc.proc.returncode is not None and restart > 0: restart -= 1 - proc = started = _daemons[key] = Subprocess(args, **kwargs) # nosec: developer-initiated + # B404:import_subprocess only developers can access this, not users + proc = started = _daemons[key] = Subprocess(args, **kwargs) # nosec B404 if proc.proc.returncode is not None: raise RuntimeError(f'Error {proc.proc.returncode} starting {arg_str}') if started: @@ -1089,7 +1093,8 @@ def _markdown(handle, **kwargs): def _yaml(handle, **kwargs): import yaml kwargs.setdefault('Loader', yaml.SafeLoader) - return yaml.load(handle.read(), **kwargs) # nosec: kwargs uses SafeLoader + # B506:yaml_load we load safely using SafeLoader + return yaml.load(handle.read(), **kwargs) # nosec B506 def _template(path, **kwargs): @@ -1285,20 +1290,21 @@ def _table_status(engine, tables): if dialect == 'mysql': # https://dev.mysql.com/doc/refman/8.0/en/information-schema-tables-table.html # Works only on MySQL 5.7 and above - q = ('SELECT update_time FROM information_schema.tables WHERE ' + # nosec + # B608:hardcoded_sql_expressions only used internally + q = ('SELECT update_time FROM information_schema.tables WHERE ' + # nosec B608 _wheres('table_schema', 'table_name', db, tables)) elif dialect == 'snowflake': # https://docs.snowflake.com/en/sql-reference/info-schema/tables.html - q = ('SELECT last_altered FROM information_schema.tables WHERE ' + # nosec + q = ('SELECT last_altered FROM information_schema.tables WHERE ' + # nosec B608 _wheres('table_schema', 'table_name', db, tables)) elif dialect == 'mssql': # https://goo.gl/b4aL9m - q = ('SELECT last_user_update FROM sys.dm_db_index_usage_stats WHERE ' + # nosec + q = ('SELECT last_user_update FROM sys.dm_db_index_usage_stats WHERE ' + # nosec B608 _wheres('database_id', 'object_id', db, tables, fn=['DB_ID', 'OBJECT_ID'])) elif dialect == 'postgresql': # https://www.postgresql.org/docs/9.6/static/monitoring-stats.html - q = ('SELECT n_tup_ins, n_tup_upd, n_tup_del FROM pg_stat_all_tables WHERE ' + # nosec - _wheres('schemaname', 'relname', 'public', tables)) + q = ('SELECT n_tup_ins, n_tup_upd, n_tup_del FROM pg_stat_all_tables ' + # nosec B608 + 'WHERE ' + _wheres('schemaname', 'relname', 'public', tables)) elif dialect == 'sqlite': if not db: raise KeyError(f'gramex.cache.query: does not support memory sqlite "{dialect}"') diff --git a/gramex/config.py b/gramex/config.py index b63ff7de4..8c137231b 100644 --- a/gramex/config.py +++ b/gramex/config.py @@ -257,7 +257,8 @@ def _calc_value(val, key): def random_string(size, chars=_valid_key_chars): '''Return random string of length size using chars (which defaults to alphanumeric)''' - return ''.join(choice(chars) for index in range(size)) # nosec: non-cryptographic use + # B311:random random() is safe since it's for non-cryptographic use + return ''.join(choice(chars) for index in range(size)) # nosec B311 RANDOM_KEY = r'$*' @@ -329,7 +330,8 @@ def _yaml_open(path, default=AttrDict(), **kwargs): app_log.debug(f'Loading config: {path}') with path.open(encoding='utf-8') as handle: try: - result = yaml.load(handle, Loader=ConfigYAMLLoader) # nosec: SafeLoader + # B506:yaml_load we use a safe loader + result = yaml.load(handle, Loader=ConfigYAMLLoader) # nosec B506 except Exception: app_log.exception(f'Config error: {path}') return default @@ -377,8 +379,8 @@ def _yaml_open(path, default=AttrDict(), **kwargs): # Evaluate conditional base, expr = key.split(' if ', 2) try: - # eval() is safe here since `expr` is written by app developer - condition = eval(expr, globals(), frozen_vars) # nosec: developer-initiated + # B307:eval this is safe since `expr` is written by app developer + condition = eval(expr, globals(), frozen_vars) # nosec B307 except Exception: condition = False app_log.exception(f'Failed condition evaluation: {key}') @@ -860,7 +862,8 @@ def setup_secrets(path, max_age_days=1000000, clear=True): from tornado.web import decode_signed_value app_log.info(f'Fetching remote secrets from {secrets_url}') # Load string from the URL -- but ignore comments. file:// URLs are fine too - value = yaml.safe_load(urlopen(secrets_url)) # nosec: allow file:// URLs + # B310:urllib_urlopen secrets can be local files or URLs + value = yaml.safe_load(urlopen(secrets_url)) # nosec B310 value = decode_signed_value(secrets_key, '', value, max_age_days=max_age_days) result.update(loads(value.decode('utf-8'))) # If SECRETS_IMPORT: is set, fetch secrets from those file(s) as well. diff --git a/gramex/data.py b/gramex/data.py index e86bf3d7d..be7433fee 100644 --- a/gramex/data.py +++ b/gramex/data.py @@ -921,7 +921,8 @@ def alter(url: str, table: str, columns: dict = None, **kwargs: dict) -> sa.engi if isinstance(col_type, str): # Use eval() to handle direct types like INTEGER *and* expressions like VARCHAR(3) # eval() is safe here since `col_type` is written by app developer - row['type'] = eval(col_type.upper(), vars(sa.types)) # nosec: frozen input + # B307:eval is safe here since `col_type` is written by app developer + row['type'] = eval(col_type.upper(), vars(sa.types)) # nosec B307 row['type_'] = row.pop('type') if 'default' in row: row['server_default'] = str(row.pop('default')) diff --git a/gramex/handlers/capturehandler.py b/gramex/handlers/capturehandler.py index 0567b94ce..13b804d9e 100644 --- a/gramex/handlers/capturehandler.py +++ b/gramex/handlers/capturehandler.py @@ -9,7 +9,8 @@ import tornado.gen from orderedattrdict import AttrDict from threading import Thread, Lock -from subprocess import Popen, PIPE, STDOUT # nosec: only for JS compilation +# B404:import_subprocess only for JS compilation +from subprocess import Popen, PIPE, STDOUT # nosec B404 from urllib.parse import urlencode, urljoin from tornado.web import HTTPError from tornado.httpclient import AsyncHTTPClient @@ -125,8 +126,9 @@ def _start(self): # Try starting the process again app_log.info(f'Starting {script} via {self.cmd}') self.close() - # self.cmd is taken from the YAML configuration. Safe to run - self.proc = Popen( # nosec: frozen input + # B603:subprocess_without_shell_equals_true is safe since self.cmd is taken from + # the YAML configuration (from developers) + self.proc = Popen( # nosec B603 shlex.split(self.cmd), stdout=PIPE, stderr=STDOUT) self.proc.poll() atexit.register(self.close) diff --git a/gramex/handlers/drivehandler.py b/gramex/handlers/drivehandler.py index 11dff83cf..a0a59344c 100644 --- a/gramex/handlers/drivehandler.py +++ b/gramex/handlers/drivehandler.py @@ -117,8 +117,9 @@ def post(self, *path_args, **path_kwargs): file = os.path.basename(upload.get('filename', '')) ext = os.path.splitext(file)[1] path = slug.filename(file) + # B311:random random() is safe since it's for non-cryptographic use while os.path.exists(os.path.join(self.path, path)): - randomletter = choice(digits + ascii_lowercase) # nosec: non-cryptographic + randomletter = choice(digits + ascii_lowercase) # nosec B311 path = os.path.splitext(path)[0] + randomletter + ext self.args['file'][i] = file self.args['ext'][i] = ext.lower() diff --git a/gramex/handlers/processhandler.py b/gramex/handlers/processhandler.py index eab0ca5ab..b757ef9d9 100644 --- a/gramex/handlers/processhandler.py +++ b/gramex/handlers/processhandler.py @@ -125,7 +125,8 @@ def get(self, *path_args): proc = Subprocess( self.cmdargs, # NOTE: developer should sanitize args if shell=True - shell=self.shell, # nosec: developer-initiated + # B604:any_other_function_with_shell_equals_true + shell=self.shell, # nosec B604 cwd=self.cwd, stream_stdout=self.stream_stdout, stream_stderr=self.stream_stderr, diff --git a/gramex/handlers/socialhandler.py b/gramex/handlers/socialhandler.py index 9f311d78a..9400d8605 100644 --- a/gramex/handlers/socialhandler.py +++ b/gramex/handlers/socialhandler.py @@ -109,7 +109,8 @@ def get_token(self, key, fetch=lambda info, key, val: info.get(key, val)): info = self.session.get(self.user_info, {}) token = self.kwargs.get(key, None) # Get from config session_token = fetch(info, key, None) - if token == 'persist': # nosec: false positive + # B105:hardcoded_password_string: 'persist' is not a password + if token == 'persist': # nosec B105 token = self.read_store().get(key, None) # If persist, use store if token is None and session_token: # Or persist from session self.write_store(info) diff --git a/gramex/install.py b/gramex/install.py index db796470b..0e7030649 100644 --- a/gramex/install.py +++ b/gramex/install.py @@ -14,8 +14,8 @@ import requests from shutilwhich import which from pathlib import Path -# subprocess is safe since it runs developer-initiated commands -from subprocess import Popen, check_output, CalledProcessError # nosec: developer-initiated +# B404:import_subprocess only developers can access this, not users +from subprocess import Popen, check_output, CalledProcessError # nosec B404 from orderedattrdict import AttrDict from orderedattrdict.yamlutils import AttrDictYAMLLoader from zipfile import ZipFile @@ -154,8 +154,8 @@ gramex license accept # Accept Gramex license gramex license reject # Reject Gramex license ''' -# yaml.load is safe since it only reads the string above, not user-created content -usage = yaml.load(usage, Loader=AttrDictYAMLLoader) # nosec: frozen input +# B506:yaml_load yaml.load is safe since it only reads the string above, not user-created content +usage = yaml.load(usage, Loader=AttrDictYAMLLoader) # nosec B506 class TryAgainError(Exception): @@ -320,13 +320,14 @@ def run_command(config): appcmd = shlex.split(appcmd) # If the app is a Cygwin app, TARGET should be a Cygwin path too. target = config.target - cygcheck, cygpath, kwargs = which('cygcheck'), which('cygpath'), {'universal_newlines': True} - if cygcheck is not None and cygpath is not None: + cygwin, cygpath, kwargs = which('cygcheck'), which('cygpath'), {'universal_newlines': True} + if cygwin is not None and cygpath is not None: # subprocess.check_output is safe here since these are developer-initiated - app_path = check_output([cygpath, '-au', which(appcmd[0])], **kwargs).strip() # nosec - is_cygwin_app = check_output([cygcheck, '-f', app_path], **kwargs).strip() # nosec + # B404:import_subprocess check_output is safe here since these are developer-initiated + path = check_output([cygpath, '-au', which(appcmd[0])], **kwargs).strip() # nosec 404 + is_cygwin_app = check_output([cygwin, '-f', path], **kwargs).strip() # nosec 404 if is_cygwin_app: - target = check_output([cygpath, '-au', target], **kwargs).strip() # nosec + target = check_output([cygpath, '-au', target], **kwargs).strip() # nosec 404 # Replace TARGET with the actual target if 'TARGET' in appcmd: appcmd = [target if arg == 'TARGET' else arg for arg in appcmd] @@ -336,7 +337,8 @@ def run_command(config): if not safe_rmtree(config.target): app_log.error(f'Cannot delete target {config.target}. Aborting installation') return - proc = Popen(appcmd, bufsize=-1, **kwargs) # nosec: developer-initiated + # B603:subprocess_without_shell_equals_true is safe since this is developer-initiated + proc = Popen(appcmd, bufsize=-1, **kwargs) # nosec 603 proc.communicate() return proc.returncode @@ -576,7 +578,8 @@ def service(args, kwargs): def _check_output(cmd, default=b'', **kwargs): '''Run cmd and return output. Return default in case the command fails''' try: - return check_output(shlex.split(cmd), **kwargs).strip() # nosec: developer-initiated + # B603:subprocess_without_shell_equals_true is safe since this is developer-initiated + return check_output(shlex.split(cmd), **kwargs).strip() # nosec B603 # OSError is raised if the cmd is not found. # CalledProcessError is raised if the cmd returns an error. except (OSError, CalledProcessError): @@ -587,7 +590,8 @@ def _run_console(cmd, **kwargs): '''Run cmd and pipe output to console. Log and raise error if cmd is not found''' cmd = shlex.split(cmd) try: - proc = Popen(cmd, bufsize=-1, universal_newlines=True, **kwargs) + # B603:subprocess_without_shell_equals_true is safe since this is developer-initiated + proc = Popen(cmd, bufsize=-1, universal_newlines=True, **kwargs) # nosec B603 except OSError: app_log.error(f'Cannot find command: {cmd[0]}') raise diff --git a/gramex/license.py b/gramex/license.py index 632306886..c5fa81505 100644 --- a/gramex/license.py +++ b/gramex/license.py @@ -37,7 +37,7 @@ def accept(force=False): gramex.console(EULA) result = 'y' if force or not sys.stdin else '' while not result: - result = input('Do you accept the license (Y/N): ').strip() # nosec: safe in PY3 + result = input('Do you accept the license (Y/N): ').strip() if result.lower().startswith('y'): store.dump('accepted', time.time()) store.flush() diff --git a/gramex/pptgen/commands.py b/gramex/pptgen/commands.py index 84fc62e79..0d16c3966 100644 --- a/gramex/pptgen/commands.py +++ b/gramex/pptgen/commands.py @@ -10,8 +10,8 @@ import pandas as pd import matplotlib.cm import matplotlib.colors -# lxml.etree is safe on https://github.com/tiran/defusedxml/tree/main/xmltestdata -from lxml.etree import fromstring # nosec: lxml is fixed +# B410:import_lxml lxml.etree is safe on https://github.com/tiran/defusedxml/tree/main/xmltestdata +from lxml.etree import fromstring # nosec B410 from tornado.template import Template from tornado.escape import to_unicode from pptx.chart import data as pptxcd @@ -73,8 +73,8 @@ def text(shape, spec, data): # Updating default css with css from config. default_css.update(style) default_css['color'] = default_css.get('color', '#0000000') - # lxml.etree is safe on https://github.com/tiran/defusedxml/tree/main/xmltestdata - update_text = fromstring('{}'.format(template(spec['text'], data))) # nosec + # B320: lxml.etree is safe on https://github.com/tiran/defusedxml/tree/main/xmltestdata + update_text = fromstring('{}'.format(template(spec['text'], data))) # nosec B320 paragraph.runs[0].text = update_text.text if update_text.text else '' utils.apply_text_css(shape, paragraph.runs[0], paragraph, **default_css) index = 1 @@ -388,12 +388,11 @@ def chart(shape, spec, data): for series_point in {'point', 'series'}: # Replacing point with series to change color in legend fillpoint = color_mapping[chart_name].replace('point', series_point) - chart_css(eval(fillpoint).fill, # nosec: developer-initiaated - point_css, point_css['color']) + # B307:eval this is safe since `expr` is written by app developer + chart_css(eval(fillpoint).fill, point_css, point_css['color']) # nosec B307 # Will apply on outer line of chart shape line(like stroke in html) _stroke = point_css.get('stroke', point_css['color']) - chart_css(eval(fillpoint).line.fill, # nosec: developer-initiaated - point_css, _stroke) + chart_css(eval(fillpoint).line.fill, point_css, _stroke) # nosec B307 # Custom Charts Functions below(Sankey, Treemap, Calendarmap). diff --git a/gramex/pptgen/utils.py b/gramex/pptgen/utils.py index 77dde4c59..45763a003 100644 --- a/gramex/pptgen/utils.py +++ b/gramex/pptgen/utils.py @@ -5,9 +5,9 @@ import platform import numpy as np import pandas as pd -# lxml.etree is safe on https://github.com/tiran/defusedxml/tree/main/xmltestdata -from lxml import objectify # nosec: lxml is fixed -from lxml.builder import ElementMaker # nosec: lxml is fixed +# B410:import_lxml lxml.etree is safe on https://github.com/tiran/defusedxml/tree/main/xmltestdata +from lxml import objectify # nosec B410 +from lxml.builder import ElementMaker # nosec B410 from pptx.util import Inches from pptx.dml.color import RGBColor from pptx.enum.base import EnumValue diff --git a/gramex/pptgen2/__init__.py b/gramex/pptgen2/__init__.py index 216d628f8..09731adc9 100644 --- a/gramex/pptgen2/__init__.py +++ b/gramex/pptgen2/__init__.py @@ -486,5 +486,5 @@ def commandline(args=None): # If --no-open is specified, or the OS doesn't have startfile (e.g. Linux), stop here. # Otherwise, open the output PPTX created if not rules.get('no-open', False) and hasattr(os, 'startfile'): - # os.startfile() is safe since the target is an explicit file we've created - os.startfile(rules['target']) # nosec: developer-initiated + # B606:start_process_with_no_shell is safe -- it's a file we've explicitly created + os.startfile(rules['target']) # nosec B606 diff --git a/gramex/pptgen2/commands.py b/gramex/pptgen2/commands.py index 61c2778ca..5add4e879 100644 --- a/gramex/pptgen2/commands.py +++ b/gramex/pptgen2/commands.py @@ -22,8 +22,8 @@ from gramex import console from gramex.config import app_log, objectpath from gramex.transforms import build_transform -# lxml.etree is safe on https://github.com/tiran/defusedxml/tree/main/xmltestdata -from lxml.html import fragments_fromstring, builder, HtmlElement # nosec: lxml is safe +# B410:import_lxml lxml.etree is safe on https://github.com/tiran/defusedxml/tree/main/xmltestdata +from lxml.html import fragments_fromstring, builder, HtmlElement # nosec B410 from orderedattrdict import AttrDict from pptx.chart import data as pptxchartdata from pptx.dml.color import RGBColor diff --git a/gramex/services/__init__.py b/gramex/services/__init__.py index 8bd6bcd70..01732e231 100644 --- a/gramex/services/__init__.py +++ b/gramex/services/__init__.py @@ -901,8 +901,8 @@ def _get_cache_key(conf, name): 'missing': '~', 'argsep': ', ', # join args using comma } - # exec() is safe here since the code is constructed entirely in this function - exec(method, context) # nosec: frozen input + # B102:exec_used is safe since the code is constructed entirely in this function + exec(method, context) # nosec B102 return context['cache_key'] diff --git a/gramex/services/rediscache.py b/gramex/services/rediscache.py index a207ffbf0..72ba88af3 100644 --- a/gramex/services/rediscache.py +++ b/gramex/services/rediscache.py @@ -1,4 +1,5 @@ -import pickle # nosec: we only pickle Gramex internal objects +# B403:import_public we only pickle Gramex internal objects +import pickle # nosec B403 from redis import StrictRedis @@ -51,7 +52,8 @@ def __init__(self, path=None, maxsize=None, *args, **kwargs): def __getitem__(self, key): key = pickle.dumps(key, pickle.HIGHEST_PROTOCOL) result = self.store.get(key) - return None if result is None else pickle.loads(result) # nosec: frozen input + # B301:pickle key is set by developers and safe to pickle + return None if result is None else pickle.loads(result) # nosec B301 def __setitem__(self, key, value, expire=None): key = pickle.dumps(key, pickle.HIGHEST_PROTOCOL) @@ -67,7 +69,8 @@ def __len__(self): def __iter__(self): for key in self.store.scan_iter(): try: - yield pickle.loads(key) # nosec: key is safe + # B301:pickle key is set by developers and safe to pickle + yield pickle.loads(key) # nosec B301 except pickle.UnpicklingError: # If redis already has keys created by other apps, yield them as-is yield key diff --git a/gramex/services/urlcache.py b/gramex/services/urlcache.py index 0662b87d8..4eeaa70bc 100644 --- a/gramex/services/urlcache.py +++ b/gramex/services/urlcache.py @@ -10,7 +10,8 @@ See gramex.handlers.BaseHandler for examples on how to use these objects. ''' -import pickle # nosec: only pickling internal state +# B403:import_public we only pickle Gramex internal objects +import pickle # nosec B403 from diskcache import Cache as DiskCache from .ttlcache import TTLCache as MemoryCache from .rediscache import RedisCache @@ -61,7 +62,8 @@ def wrap(self, handler): class MemoryCacheFile(CacheFile): def get(self): result = self.store.get(self.key) - return None if result is None else pickle.loads(result) # nosec: only internal state + # B301:pickle key is an internal state string and safe to pickle + return None if result is None else pickle.loads(result) # nosec B301 def wrap(self, handler): self._finish = handler.finish diff --git a/gramex/transforms/transforms.py b/gramex/transforms/transforms.py index dc8df502d..a6379c0f3 100644 --- a/gramex/transforms/transforms.py +++ b/gramex/transforms/transforms.py @@ -306,8 +306,8 @@ def transform(_val): **{key: getattr(gramex.transforms, key) for key in gramex.transforms.__all__} ) code = compile(''.join(body), filename=filename, mode='exec') - # exec() is safe here since the code is written by app developer - exec(code, context) # nosec: developer-initiated + # B102:exec_used is safe since the code is written by app developer + exec(code, context) # nosec B102 # Return the transformed function result = context['transform'] @@ -470,8 +470,8 @@ def condition(*args): pairs = zip(args[0::2], args[1::2]) for cond, val in pairs: if isinstance(cond, str): - # eval() is safe here since `cond` is written by app developer - if eval(Template(cond).substitute(var_defaults)): # nosec: developer-initiated + # B307:eval is safe here since `cond` is written by app developer + if eval(Template(cond).substitute(var_defaults)): # nosec B307 return val elif bool(cond): return val @@ -542,8 +542,8 @@ def assign(field, target, catch_errors=False): body.append('\treturn r') code = compile(''.join(body), filename=f'flattener:{filename}', mode='exec') context = {'AttrDict': AttrDict, 'default': default} - # eval() is safe here since the code is constructed entirely in this function - eval(code, context) # nosec: developer-initiated + # B307:eval is safe here since the code is constructed entirely in this function + eval(code, context) # nosec B307 return context[filename] @@ -804,6 +804,6 @@ def build_log_info(keys: List, *vars: List): 'def fn(handler, %s):\n\treturn {%s}' % (', '.join(vars), ' '.join(vals)), filename='log', mode='exec') context = {'os': os, 'time': time, 'datetime': datetime, 'conf': conf, 'AttrDict': AttrDict} - # exec() is safe here since the code is constructed entirely in this function - exec(code, context) # nosec: developer-initiated + # B102:exec_used is safe here since the code is constructed entirely in this function + exec(code, context) # nosec B102 return context['fn'] diff --git a/task b/task index cae99327e..b370430dc 100644 --- a/task +++ b/task @@ -19,7 +19,7 @@ lint () { # Python security check with bandit command -v bandit 2>/dev/null 2>&1 || pip install bandit - bandit gramex --aggregate vuln --recursive --quiet + bandit . --recursive --quiet # Install linters. Then test all apps npm install diff --git a/testlib/test_cache_module.py b/testlib/test_cache_module.py index ceb5ecc50..08bb42d30 100644 --- a/testlib/test_cache_module.py +++ b/testlib/test_cache_module.py @@ -235,9 +235,9 @@ def check(reload): def test_open_yaml(self): path = os.path.join(cache_folder, 'data.yaml') with io.open(path, encoding='utf-8') as handle: - # yaml.load() is safe to use here since we're loading from a known safe file. + # B506:yaml_load is safe to use here since we're loading from a known safe file. # Specifically, we're testing whether Loader= is passed to gramex.cache.open. - expected = yaml.load(handle, Loader=AttrDictYAMLLoader) # nosec: test case + expected = yaml.load(handle, Loader=AttrDictYAMLLoader) # nosec B506 def check(reload): result, reloaded = gramex.cache.open( diff --git a/testlib/test_pptgen.py b/testlib/test_pptgen.py index 7880a0a2f..4c887dd77 100644 --- a/testlib/test_pptgen.py +++ b/testlib/test_pptgen.py @@ -736,7 +736,8 @@ def test_sankey(self): grpobj = data.groupby(grp) frame = pd.DataFrame({ 'size': grpobj[grp].count(), - 'seq': eval(grp_order)(grpobj), # nosec: test case + # B307:eval is safe here since we've constructed `grp_order` + 'seq': eval(grp_order)(grpobj), # nosec B307 }) frame['width'] = frame['size'] / float(frame['size'].sum()) * width frame = frame.sort_values(by=['seq']) diff --git a/tests/__init__.py b/tests/__init__.py index fef1b1aba..fc0db1922 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -2,7 +2,9 @@ import requests import shutil import unittest -from lxml import etree # nosec: lxml is safe # noqa: F401 - other modules use this +# B410:import_lxml lxml.etree is safe on https://github.com/tiran/defusedxml/tree/main/xmltestdata +# F401: we import here since other modules use this +from lxml import etree # noqa: F401 # nosec B410 from . import server from nose.tools import eq_, ok_ from orderedattrdict import AttrDict From 144a86ac73022894f880f96b6556c7d9e4519121 Mon Sep 17 00:00:00 2001 From: S Anand Date: Wed, 29 Jun 2022 01:07:31 +0800 Subject: [PATCH 4/4] BUG: Invalid computed variables revert to default. Fixes #564 --- gramex/config.py | 9 ++++++--- testlib/test_config/__init__.py | 2 ++ testlib/test_config/config.template.base.yaml | 4 ++++ testlib/test_config/config.template.child.yaml | 1 + testlib/test_config/dir/config.template.subdir.yaml | 1 + 5 files changed, 14 insertions(+), 3 deletions(-) diff --git a/gramex/config.py b/gramex/config.py index 8c137231b..4832bfe7e 100644 --- a/gramex/config.py +++ b/gramex/config.py @@ -240,9 +240,12 @@ def _calc_value(val, key): if val.get('function'): from .transforms import build_transform function = build_transform(val, vars={'key': None}, filename=f'config:{key}') - for result in function(key): - if result is not None: - return result + try: + for result in function(key): + if result is not None: + return result + except Exception: + app_log.exception(f'Error in calculated variable: {key}: {val}') return _calc_value(val.get('default', None), key) else: return {k: _calc_value(v, k) for k, v in val.items()} diff --git a/testlib/test_config/__init__.py b/testlib/test_config/__init__.py index 842bb17a1..3177ae159 100644 --- a/testlib/test_config/__init__.py +++ b/testlib/test_config/__init__.py @@ -224,6 +224,8 @@ def test_variables(self): eq_(conf['%s_FUNCTION' % key], key) # Default functions "underride" values eq_(conf['%s_DEFAULT_FUNCTION' % key], 'base') + # Invalid functions switch to defaults + eq_(conf['%s_INVALID_FUNCTION' % key], 'DEFAULT') # Functions can use variables using gramex.config.variables eq_(conf['%s_FUNCTION_VAR' % key], conf.base_ROOT + key) # Derived variables diff --git a/testlib/test_config/config.template.base.yaml b/testlib/test_config/config.template.base.yaml index 60bbba704..d6273d6a7 100644 --- a/testlib/test_config/config.template.base.yaml +++ b/testlib/test_config/config.template.base.yaml @@ -11,6 +11,9 @@ variables: default: function: str args: ['base'] + INVALID_FUNCTION: + function: nonexistent() + default: DEFAULT DEFAULT_OBJECT: default: x: $THIS @@ -85,6 +88,7 @@ base_THIS: $THIS base_DEFAULT: $BASE_DEFAULT base_FUNCTION: $FUNCTION base_DEFAULT_FUNCTION: $DEFAULT_FUNCTION +base_INVALID_FUNCTION: $INVALID_FUNCTION base_FUNCTION_VAR: $FUNCTION_VAR base_DERIVED: $DERIVED base_YAMLURL_VAR: $YAMLURL_VAR diff --git a/testlib/test_config/config.template.child.yaml b/testlib/test_config/config.template.child.yaml index 0fe0b8217..5cf072110 100644 --- a/testlib/test_config/config.template.child.yaml +++ b/testlib/test_config/config.template.child.yaml @@ -25,6 +25,7 @@ child_THIS: $THIS child_DEFAULT: $CHILD_DEFAULT child_FUNCTION: $FUNCTION child_DEFAULT_FUNCTION: $DEFAULT_FUNCTION +child_INVALID_FUNCTION: $INVALID_FUNCTION child_FUNCTION_VAR: $FUNCTION_VAR child_DERIVED: $DERIVED child_URLROOT: $URLROOT diff --git a/testlib/test_config/dir/config.template.subdir.yaml b/testlib/test_config/dir/config.template.subdir.yaml index e94bcf13d..c9fc67454 100644 --- a/testlib/test_config/dir/config.template.subdir.yaml +++ b/testlib/test_config/dir/config.template.subdir.yaml @@ -25,6 +25,7 @@ subdir_THIS: $THIS subdir_DEFAULT: $SUBDIR_DEFAULT subdir_FUNCTION: $FUNCTION subdir_DEFAULT_FUNCTION: $DEFAULT_FUNCTION +subdir_INVALID_FUNCTION: $INVALID_FUNCTION subdir_FUNCTION_VAR: $FUNCTION_VAR subdir_DERIVED: $DERIVED subdir_YAMLURL_VAR: $YAMLURL_VAR