From 2f4cccaa4106e1ba9bbbcb461561250b3eda58a4 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Mon, 10 Jul 2023 14:30:26 +0200 Subject: [PATCH 1/8] backend modules reffactor --- app/application.js | 4 +- app/config/index.js | 2 +- app/config/services.js | 3 +- app/lib/Utils.js | 258 ------------------ .../AbstractServiceSynchronizer.js | 88 ++---- .../alimonitor-services/BookkeepingService.js | 4 +- app/lib/alimonitor-services/JsonsFetcher.js | 106 ------- .../alimonitor-services/MonalisaService.js | 2 +- .../MonalisaServiceDetails.js | 2 +- .../alimonitor-services/MonalisaServiceMC.js | 2 +- .../MonalisaServiceMCDetails.js | 2 +- .../PassCorrectnessMonitor.js | 58 ++++ .../ServicesDataCommons.js | 50 ++-- app/lib/database/DatabaseService.js | 4 +- app/lib/database/views/flags_view.js | 2 +- app/lib/database/views/runs_views.js | 3 +- .../{other => server}/AuthControlManager.js | 0 app/{ => lib}/server/WebUiServer.js | 8 +- app/{ => lib}/server/index.js | 0 app/lib/{ => utils}/LogsStacker.js | 2 +- app/lib/{ => utils}/ResProvider.js | 8 +- app/lib/utils/http-utils.js | 126 +++++++++ app/lib/utils/index.js | 26 ++ app/lib/utils/obj-utils.js | 110 ++++++++ app/lib/utils/sql-utils.js | 79 ++++++ test/lib/config/configProvider.test.js | 8 +- test/lib/config/index.js | 2 +- test/lib/config/publicConfigProvider.test.js | 4 +- test/lib/resProvider.test.js | 6 +- test/lib/utils.test.js | 59 ++-- 30 files changed, 515 insertions(+), 513 deletions(-) delete mode 100644 app/lib/Utils.js delete mode 100644 app/lib/alimonitor-services/JsonsFetcher.js create mode 100644 app/lib/alimonitor-services/PassCorrectnessMonitor.js rename app/lib/{other => server}/AuthControlManager.js (100%) rename app/{ => lib}/server/WebUiServer.js (86%) rename app/{ => lib}/server/index.js (100%) rename app/lib/{ => utils}/LogsStacker.js (98%) rename app/lib/{ => utils}/ResProvider.js (97%) create mode 100644 app/lib/utils/http-utils.js create mode 100644 app/lib/utils/index.js create mode 100644 app/lib/utils/obj-utils.js create mode 100644 app/lib/utils/sql-utils.js diff --git a/app/application.js b/app/application.js index c8452a213..2071aa3c9 100644 --- a/app/application.js +++ b/app/application.js @@ -18,7 +18,7 @@ const { buildPublicConfig } = require('./lib/config/publicConfigProvider.js'); // IO const readline = require('readline'); -const Utils = require('./lib/Utils.js'); +const Utils = require('./lib/utils'); const { Console } = require('node:console'); // Services @@ -28,7 +28,7 @@ const services = require('./lib/alimonitor-services'); const database = require('./lib/database'); // Server -const { webUiServer } = require('./server/index.js'); +const { webUiServer } = require('./lib/server'); // Extract important const EP = config.public.endpoints; diff --git a/app/config/index.js b/app/config/index.js index bba132878..297080e6b 100644 --- a/app/config/index.js +++ b/app/config/index.js @@ -11,7 +11,7 @@ * granted to it by virtue of its status as an Intergovernmental Organization * or submit itself to any jurisdiction. */ -const ResProvider = require('../lib/ResProvider.js'); +const { ResProvider } = require('../lib/utils'); module.exports = Object.freeze({ // Web-Ui config diff --git a/app/config/services.js b/app/config/services.js index c3c2222e7..8f372a679 100644 --- a/app/config/services.js +++ b/app/config/services.js @@ -12,7 +12,8 @@ * granted to it by virtue of its status as an Intergovernmental Organization * or submit itself to any jurisdiction. */ -const ResProvider = require('../lib/ResProvider.js'); + +const { ResProvider } = require('../lib/utils'); const services = { bookkeeping: { diff --git a/app/lib/Utils.js b/app/lib/Utils.js deleted file mode 100644 index 847fc87ab..000000000 --- a/app/lib/Utils.js +++ /dev/null @@ -1,258 +0,0 @@ -/** - * - * @license - * Copyright 2019-2020 CERN and copyright holders of ALICE O2. - * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. - * All rights not expressly granted are reserved. - * - * This software is distributed under the terms of the GNU General Public - * License v3 (GPL Version 3), copied verbatim in the file "COPYING". - * - * In applying this license CERN does not waive the privileges and immunities - * granted to it by virtue of its status as an Intergovernmental Organization - * or submit itself to any jurisdiction. - */ - -const util = require('util'); -const http = require('http'); -const https = require('https'); -const { Log } = require('@aliceo2/web-ui'); - -const exec = util.promisify(require('child_process').exec); - -const logger = new Log('Utils'); - -const keywords = ['DEFAULT', 'NULL']; -class Utils { - static async exec(cmd) { - if (Array.isArray(cmd)) { - cmd = cmd.join(' '); - } - return await exec(cmd); - } - - static reversePrimitiveObject(obj) { - return Object.fromEntries(Object.entries(obj).map((e) => e.reverse())); - } - - static filterObject(obj, keptFields, suppressUndefined = false) { - if (!keptFields) { - return obj; - } - const res = {}; - for (const [nr, nl] of Object.entries(keptFields)) { - if (!suppressUndefined || res[nl]) { - res[nl] = obj[nr]; - } - } - return res; - } - - static adjusetObjValuesToSql(obj) { - const res = {}; - for (const k in obj) { - if (obj[k]) { - if (typeof obj[k] == 'object') { - if (Array.isArray(obj[k])) { - res[k] = `ARRAY[${obj[k].map((d) => Utils.parseValueToSql(d)).join(',')}]`; - } else { - res[k] = Utils.adjusetObjValuesToSql(obj[k]); - } - } else { - res[k] = Utils.parseValueToSql(obj[k]); - } - } else { - res[k] = null; - } - } - return res; - } - - static parseValueToSql(v) { - if (!v) { - return null; - } - if (typeof v == 'string' && !keywords.includes(v.toUpperCase())) { - if (v.length == 0) { - return null; - } else { - return `'${v}'`; - } - } else { - return v; - } - } - - static parseValuesToSql(values) { - return values.map((v) => Utils.parseValueToSql(v)); - } - - static preserveSQLKeywords(words) { - return words.map((w) => { - if (['end'].includes(w)) { - return `"${w}"`; - } else { - return w; - } - }); - } - - static simpleBuildInsertQuery(targetTable, valuesObj) { - return `INSERT INTO ${targetTable}(${Utils.preserveSQLKeywords(Object.keys(valuesObj)).join(', ')}) - VALUES(${Utils.parseValuesToSql(Object.values(valuesObj)).join(', ')})`; - } - - static switchCase(caseName, cases, defaultCaseValue) { - return Object.prototype.hasOwnProperty.call(cases, caseName) - ? cases[caseName] - : defaultCaseValue - ? defaultCaseValue : Utils.throw('no case, no default case'); - } - - static delay(time) { - return new Promise((resolve) => setTimeout(resolve, time)); - } - - static replaceAll(s, pattern, replace) { - const p = s.split(pattern); - return p.join(replace); - } - - static arrayToChunks(arr, chunkSize) { - const chunks = []; - for (let i = 0; i < arr.length; i += chunkSize) { - chunks.push(arr.slice(i, i + chunkSize)); - } - return chunks; - } - - static extractPeriodYear(name) { - try { - const year = parseInt(name.slice(3, 5), 10); - if (isNaN(year)) { - return 'NULL'; - } - return year > 50 ? year + 1900 : year + 2000; - } catch (e) { - return 'NULL'; - } - } - - static applyOptsToObj(obj, options) { - Object.entries(options).forEach(([k, v]) => { - obj[k] = v; - }); - } - - static nullIfThrows(func, args, errorHandler) { - try { - return func(...args); - } catch (e) { - if (errorHandler) { - errorHandler(e, args); - } - return null; - } - } - - static distinct(arr) { - return arr.filter((value, index, self) => self.indexOf(value) === index); - } - - static checkClientType(endpoint) { - const unspecifiedProtocolMessage = 'unspecified protocol in url'; - - switch (endpoint.protocol) { - case 'http:': - return http; - case 'https:': - return https; - default: - logger.error(unspecifiedProtocolMessage); - throw new Error(unspecifiedProtocolMessage); - } - } - - static makeHttpRequestForJSON(endpoint, opts, logger, onSuccess, onFailure) { - return new Promise((resolve, reject) => { - let rawData = ''; - const req = Utils.checkClientType(endpoint).request(endpoint, opts, async (res) => { - const { statusCode } = res; - const contentType = res.headers['content-type']; - - let error; - let redirect = false; - if (statusCode == 302 || statusCode == 301) { - const mess = `Redirect. Status Code: ${statusCode}; red. to ${res.headers.location}`; - if (opts.allowRedirects) { - redirect = true; - logger.warn(mess); - const nextHop = new URL(endpoint.origin + res.headers.location); - nextHop.searchParams.set('res_path', 'json'); - logger.warn(`from ${endpoint.href} to ${nextHop.href}`); - resolve(await Utils.makeHttpRequestForJSON(nextHop, opts, logger, onSuccess, onFailure)); - } else { - throw new Error(mess); - } - } else if (statusCode !== 200) { - error = new Error(`Request Failed. Status Code: ${statusCode}`); - } else if (!/^application\/json/.test(contentType)) { - error = new Error(`Invalid content-type. Expected application/json but received ${contentType}`); - } - if (error) { - logger.error(error.message); - res.resume(); - return; - } - - res.on('data', (chunk) => { - rawData += chunk; - }); - - req.on('error', (e) => { - logger.error(`ERROR httpGet: ${e}`); - if (onFailure) { - onFailure(endpoint, e); - } - reject(e); - }); - - res.on('end', () => { - try { - if (!redirect) { - const data = JSON.parse(rawData); - if (onSuccess) { - onSuccess(endpoint, data); - } - resolve(data); - } - } catch (e) { - logger.error(`${e.message} for endpoint ${endpoint}`); - if (onFailure) { - onFailure(endpoint, e); - } - reject(e); - } - }); - }); - req.on('error', (err) => { - reject(err); - }); - req.end(); - }); - } - - static throwNotImplemented() { - throw new Error('Not implemented'); - } - - static throwAbstract() { - throw new Error('Abstract, can not be used'); - } - - static throw(mess) { - throw new Error(mess); - } -} - -module.exports = Utils; diff --git a/app/lib/alimonitor-services/AbstractServiceSynchronizer.js b/app/lib/alimonitor-services/AbstractServiceSynchronizer.js index b5392ad9d..76cfb8834 100644 --- a/app/lib/alimonitor-services/AbstractServiceSynchronizer.js +++ b/app/lib/alimonitor-services/AbstractServiceSynchronizer.js @@ -17,65 +17,18 @@ const { Client } = require('pg'); const { SocksProxyAgent } = require('socks-proxy-agent'); const { Log } = require('@aliceo2/web-ui'); const config = require('../config/configProvider.js'); -const ResProvider = require('../ResProvider.js'); -const Utils = require('../Utils.js'); -const JsonsFetcher = require('./JsonsFetcher.js'); +const { ResProvider, makeHttpRequestForJSON, arrayToChunks, applyOptsToObj, throwNotImplemented } = require('../utils'); const Cacher = require('./Cacher.js'); +const PassCorrectnessMonitor = require('./PassCorrectnessMonitor.js'); const defaultServiceSynchronizerOptions = { forceStop: false, - rawCacheUse: true, useCacheJsonInsteadIfPresent: false, omitWhenCached: false, - - batchedRequestes: true, batchSize: 4, }; -const additionalHttpOpts = { - allowRedirects: true, // TODO -}; - -class PassCorrectnessMonitor { - constructor(logger, errorsLoggingDepth) { - this.logger = logger; - this.errorsLoggingDepth = errorsLoggingDepth; - this.correct = 0; - this.incorrect = 0; - this.omitted = 0; - this.errors = []; - } - - handleCorrect() { - this.correct++; - } - - handleIncorrect(e, data) { - this.incorrect++; - e.data = data; - this.errors.push(e); - } - - handleOmitted() { - this.omitted++; - } - - logResults() { - const { correct, incorrect, omitted, errors, errorsLoggingDepth, logger } = this; - const dataSize = incorrect + correct + omitted; - - if (incorrect > 0) { - const logFunc = Utils.switchCase(errorsLoggingDepth, config.errorsLoggingDepths); - errors.forEach((e) => logFunc(logger, e)); - logger.warn(`sync unseccessful for ${incorrect}/${dataSize}`); - } - if (omitted > 0) { - logger.info(`omitted data units ${omitted}/${dataSize}`); - } - } -} - class AbstractServiceSynchronizer { constructor() { this.name = this.constructor.name; @@ -83,10 +36,13 @@ class AbstractServiceSynchronizer { this.opts = this.createHttpOpts(); - this.metaStore = {}; + this.metaStore = { processedCtr: 0 }; + this.errorsLoggingDepth = config.defaultErrorsLogginDepth; - Utils.applyOptsToObj(this, defaultServiceSynchronizerOptions); - Utils.applyOptsToObj(this.opts, additionalHttpOpts); + applyOptsToObj(this, defaultServiceSynchronizerOptions); + applyOptsToObj(this.opts, { + allowRedirects: true, + }); } createHttpOpts() { @@ -117,11 +73,11 @@ class AbstractServiceSynchronizer { ); const { logsStacker } = pfxWrp; - logsStacker.typeLog('info'); + logsStacker.logType('info'); if (!pfxWrp.content) { if (logsStacker.any('error')) { - logsStacker.typeLog('warn'); - logsStacker.typeLog('error'); + logsStacker.logType('warn'); + logsStacker.logType('error'); } } const passphrase = ResProvider.passphraseProvider(); @@ -144,7 +100,7 @@ class AbstractServiceSynchronizer { } setLogginLevel(logginLevel) { - Utils.throwNotImplemented(); + throwNotImplemented(); logginLevel = parseInt(logginLevel, 10); if (!logginLevel || logginLevel < 0 || logginLevel > 3) { throw new Error('Invalid debug level') ; @@ -196,11 +152,7 @@ class AbstractServiceSynchronizer { return f; }); - if (this.batchedRequestes) { - await this.makeBatchedRequest(data); - } else { - await this.makeSequentialRequest(data); - } + await this.makeBatchedRequest(data); this.monitor.logResults(); } catch (fatalError) { @@ -213,21 +165,17 @@ class AbstractServiceSynchronizer { } async makeBatchedRequest(data) { - const rowsChunks = Utils.arrayToChunks(data, this.batchSize); + const rowsChunks = arrayToChunks(data, this.batchSize); + const total = this.metaStore.totalCount || data.length; for (const chunk of rowsChunks) { const promises = chunk.map((dataUnit) => this.dbAction(this.dbClient, dataUnit) .then(() => this.monitor.handleCorrect()) .catch((e) => this.monitor.handleIncorrect(e, { dataUnit: dataUnit }))); await Promise.all(promises); - } - } - async makeSequentialRequest(data) { - for (const dataUnit of data) { - await this.dbAction(this.dbClient, dataUnit) - .then(this.monitor.handleCorrect) - .catch((e) => this.monitor.handleIncorrect(e, { dataUnit: dataUnit })); + this.metaStore['processedCtr'] += chunk.length; + this.logger.info(`progress of ${this.metaStore['processedCtr']} / ${total}`); } } @@ -242,7 +190,7 @@ class AbstractServiceSynchronizer { Cacher.cache(this.name, endpoint, data); } }; - return await JsonsFetcher.makeHttpRequestForJSON(endpoint, this.opts, this.logger, onSucces); + return await makeHttpRequestForJSON(endpoint, this.opts, this.logger, onSucces); } async dbConnect() { diff --git a/app/lib/alimonitor-services/BookkeepingService.js b/app/lib/alimonitor-services/BookkeepingService.js index 5196515fe..0139f9cc9 100644 --- a/app/lib/alimonitor-services/BookkeepingService.js +++ b/app/lib/alimonitor-services/BookkeepingService.js @@ -14,7 +14,7 @@ */ const AbstractServiceSynchronizer = require('./AbstractServiceSynchronizer.js'); -const Utils = require('../Utils.js'); +const Utils = require('../utils'); const ServicesDataCommons = require('./ServicesDataCommons.js'); const EndpintFormatter = require('./ServicesEndpointsFormatter.js'); @@ -114,7 +114,7 @@ class BookkeepingService extends AbstractServiceSynchronizer { async dbAction(dbClient, d) { const { period } = d; - const year = Utils.extractPeriodYear(period); + const year = ServicesDataCommons.extractPeriodYear(period); d = Utils.adjusetObjValuesToSql(d); const period_insert = d.period ? `call insert_period(${d.period}, ${year}, ${d.beam_type});` : ''; diff --git a/app/lib/alimonitor-services/JsonsFetcher.js b/app/lib/alimonitor-services/JsonsFetcher.js deleted file mode 100644 index e5fe33274..000000000 --- a/app/lib/alimonitor-services/JsonsFetcher.js +++ /dev/null @@ -1,106 +0,0 @@ -/** - * - * @license - * Copyright 2019-2020 CERN and copyright holders of ALICE O2. - * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. - * All rights not expressly granted are reserved. - * - * This software is distributed under the terms of the GNU General Public - * License v3 (GPL Version 3), copied verbatim in the file "COPYING". - * - * In applying this license CERN does not waive the privileges and immunities - * granted to it by virtue of its status as an Intergovernmental Organization - * or submit itself to any jurisdiction. - */ -const fs = require('fs'); -const path = require('path'); -const Utils = require('../Utils.js'); - -class JsonsFetcher { - static makeHttpRequestForJSON(endpoint, opts, logger, onSuccess, onFailure) { - return new Promise((resolve, reject) => { - let rawData = ''; - const req = Utils.checkClientType(endpoint).request(endpoint, opts, async (res) => { - const { statusCode } = res; - const contentType = res.headers['content-type']; - - let error; - let redirect = false; - if (statusCode == 302 || statusCode == 301) { - const mess = `Redirect. Status Code: ${statusCode}; red. to ${res.headers.location}`; - if (opts.allowRedirects) { - redirect = true; - logger.warn(mess); - const nextHop = new URL(endpoint.origin + res.headers.location); - nextHop.searchParams.set('res_path', 'json'); - logger.warn(`from ${endpoint.href} to ${nextHop.href}`); - resolve(await Utils.makeHttpRequestForJSON(nextHop)); - } else { - throw new Error(mess); - } - } else if (statusCode !== 200) { - error = new Error(`Request Failed. Status Code: ${statusCode}`); - } else if (!/^application\/json/.test(contentType)) { - error = new Error(`Invalid content-type. Expected application/json but received ${contentType}`); - } - if (error) { - logger.error(error.message); - res.resume(); - return; - } - - res.on('data', (chunk) => { - rawData += chunk; - }); - - req.on('error', (e) => { - logger.error(`ERROR httpGet: ${e}`); - if (onFailure) { - onFailure(endpoint, e); - } - reject(e); - }); - - res.on('end', () => { - try { - if (!redirect) { - /* - * TMP incorrect format handling - * if (/: *,/.test(rawData)) { - * rawData = rawData.replaceAll(/: *,/ig, ':"",'); - * } - */ - - const data = JSON.parse(rawData); - if (onSuccess) { - onSuccess(endpoint, data); - } - resolve(data); - } - } catch (e) { - logger.error(`${e.message} for endpoint: ${endpoint.href}`); - const fp = path.join( - __dirname, - '..', - '..', - '..', - 'database', - 'cache', - 'rawJson', - 'failing-endpoints.txt', - ); - fs.appendFileSync(fp, `${endpoint.href}\n ${e.message}\n`); - if (onFailure) { - onFailure(endpoint, e); - } - reject(e); - } - }); - }); - - req.end(); - }); - } -} - -module.exports = JsonsFetcher; diff --git a/app/lib/alimonitor-services/MonalisaService.js b/app/lib/alimonitor-services/MonalisaService.js index 097532564..712793b4d 100644 --- a/app/lib/alimonitor-services/MonalisaService.js +++ b/app/lib/alimonitor-services/MonalisaService.js @@ -14,7 +14,7 @@ */ const AbstractServiceSynchronizer = require('./AbstractServiceSynchronizer.js'); -const Utils = require('../Utils.js'); +const Utils = require('../utils'); const ServicesDataCommons = require('./ServicesDataCommons.js'); const EndpointsFormatter = require('./ServicesEndpointsFormatter.js'); const MonalisaServiceDetails = require('./MonalisaServiceDetails.js'); diff --git a/app/lib/alimonitor-services/MonalisaServiceDetails.js b/app/lib/alimonitor-services/MonalisaServiceDetails.js index cd5c9191f..b0676f7c7 100644 --- a/app/lib/alimonitor-services/MonalisaServiceDetails.js +++ b/app/lib/alimonitor-services/MonalisaServiceDetails.js @@ -14,7 +14,7 @@ */ const AbstractServiceSynchronizer = require('./AbstractServiceSynchronizer.js'); -const Utils = require('../Utils.js'); +const Utils = require('../utils'); const EndpointsFormatter = require('./ServicesEndpointsFormatter.js'); class MonalisaServiceDetails extends AbstractServiceSynchronizer { diff --git a/app/lib/alimonitor-services/MonalisaServiceMC.js b/app/lib/alimonitor-services/MonalisaServiceMC.js index bd3c231ab..3e0efdc84 100644 --- a/app/lib/alimonitor-services/MonalisaServiceMC.js +++ b/app/lib/alimonitor-services/MonalisaServiceMC.js @@ -14,7 +14,7 @@ */ const AbstractServiceSynchronizer = require('./AbstractServiceSynchronizer.js'); -const Utils = require('../Utils.js'); +const Utils = require('../utils'); const EndpointsFormatter = require('./ServicesEndpointsFormatter.js'); const MonalisaServiceMCDetails = require('./MonalisaServiceMCDetails.js'); const config = require('../config/configProvider.js'); diff --git a/app/lib/alimonitor-services/MonalisaServiceMCDetails.js b/app/lib/alimonitor-services/MonalisaServiceMCDetails.js index 5e25f32bb..c44e0d97e 100644 --- a/app/lib/alimonitor-services/MonalisaServiceMCDetails.js +++ b/app/lib/alimonitor-services/MonalisaServiceMCDetails.js @@ -14,7 +14,7 @@ */ const AbstractServiceSynchronizer = require('./AbstractServiceSynchronizer.js'); -const Utils = require('../Utils.js'); +const Utils = require('../utils'); const EndpointsFormatter = require('./ServicesEndpointsFormatter.js'); class MonalisaServiceMCDetails extends AbstractServiceSynchronizer { diff --git a/app/lib/alimonitor-services/PassCorrectnessMonitor.js b/app/lib/alimonitor-services/PassCorrectnessMonitor.js new file mode 100644 index 000000000..08974ae0f --- /dev/null +++ b/app/lib/alimonitor-services/PassCorrectnessMonitor.js @@ -0,0 +1,58 @@ +/** + * + * @license + * Copyright 2019-2020 CERN and copyright holders of ALICE O2. + * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. + * All rights not expressly granted are reserved. + * + * This software is distributed under the terms of the GNU General Public + * License v3 (GPL Version 3), copied verbatim in the file "COPYING". + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +const { switchCase } = require('../utils'); +const config = require('../config/configProvider'); + +class PassCorrectnessMonitor { + constructor(logger, errorsLoggingDepth) { + this.logger = logger; + this.errorsLoggingDepth = errorsLoggingDepth; + this.correct = 0; + this.incorrect = 0; + this.omitted = 0; + this.errors = []; + } + + handleCorrect() { + this.correct++; + } + + handleIncorrect(e, data) { + this.incorrect++; + e.data = data; + this.errors.push(e); + } + + handleOmitted() { + this.omitted++; + } + + logResults() { + const { correct, incorrect, omitted, errors, errorsLoggingDepth, logger } = this; + const dataSize = incorrect + correct + omitted; + + if (incorrect > 0) { + const logFunc = switchCase(errorsLoggingDepth, config.errorsLoggingDepths); + errors.forEach((e) => logFunc(logger, e)); + logger.warn(`sync unseccessful for ${incorrect}/${dataSize}`); + } + if (omitted > 0) { + logger.info(`omitted data units ${omitted}/${dataSize}`); + } + } +} + +module.exports = PassCorrectnessMonitor; diff --git a/app/lib/alimonitor-services/ServicesDataCommons.js b/app/lib/alimonitor-services/ServicesDataCommons.js index 8ebd7df56..a286f818c 100644 --- a/app/lib/alimonitor-services/ServicesDataCommons.js +++ b/app/lib/alimonitor-services/ServicesDataCommons.js @@ -13,24 +13,42 @@ * or submit itself to any jurisdiction. */ -const Utils = require('../Utils.js'); +const Utils = require('../utils'); const { databasePersistance } = require('../config/configProvider.js'); -class ServicesDataCommons { - /** - * Update objectData.beam_type to valid format if mapping is provided with app config - * if not there is assumption that in other scenerio name is consistant with foramt '-' - * @param {Object} dataObject o - * @returns {Object} dataObject - */ - static mapBeamTypeToCommonFormat(dataObject) { - dataObject.beam_type = Utils.switchCase( - dataObject.beam_type, - databasePersistance.beam_type_mappings, - dataObject.beam_type, - ); - return dataObject; +/** + * Update objectData.beam_type to valid format if mapping is provided with app config + * if not there is assumption that in other scenerio name is consistant with foramt '-' + * @param {Object} dataObject o + * @returns {Object} dataObject + */ +function mapBeamTypeToCommonFormat(dataObject) { + dataObject.beam_type = Utils.switchCase( + dataObject.beam_type, + databasePersistance.beam_type_mappings, + dataObject.beam_type, + ); + return dataObject; +} + +/** + * Extract year from data/simulation pass name + * @param {string} name name of pass, like LHC22a_apass1 + * @returns {Number} year + */ +function extractPeriodYear(name) { + try { + const year = parseInt(name.slice(3, 5), 10); + if (isNaN(year)) { + return 'NULL'; + } + return year > 50 ? year + 1900 : year + 2000; + } catch (e) { + return 'NULL'; } } -module.exports = ServicesDataCommons; +module.exports = { + mapBeamTypeToCommonFormat, + extractPeriodYear, +}; diff --git a/app/lib/database/DatabaseService.js b/app/lib/database/DatabaseService.js index c32f34825..5bdded7c9 100644 --- a/app/lib/database/DatabaseService.js +++ b/app/lib/database/DatabaseService.js @@ -16,7 +16,7 @@ const { Log } = require('@aliceo2/web-ui'); const { Client, Pool } = require('pg'); const QueryBuilder = require('./QueryBuilder.js'); const config = require('./../config/configProvider.js'); -const Utils = require('../Utils.js') +const {distinct} = require('../utils') const DRP = config.public.dataReqParams; const DRF = config.public.dataRespondFields; @@ -138,7 +138,7 @@ class DatabaseService { } data[DRF.rows] = rows; - data[DRF.fields] = Utils.distinct(fields.map(f => f.name)).map(n => { return { name: n } }); + data[DRF.fields] = distinct(fields.map(f => f.name)).map(n => { return { name: n } }); res.json({ data: data }); }; diff --git a/app/lib/database/views/flags_view.js b/app/lib/database/views/flags_view.js index b83ad00d3..28b25a8c8 100644 --- a/app/lib/database/views/flags_view.js +++ b/app/lib/database/views/flags_view.js @@ -34,7 +34,7 @@ const flags_view = (query) => { qcf.id, qcf.time_start, qcf.time_end, - ftd.name, + ftd.name as flag_name, qcf.comment, r.run_number, ds.name, diff --git a/app/lib/database/views/runs_views.js b/app/lib/database/views/runs_views.js index 74a2c15e1..2cb88b469 100644 --- a/app/lib/database/views/runs_views.js +++ b/app/lib/database/views/runs_views.js @@ -37,7 +37,8 @@ const queryForRunsFields = ` const runs_per_period_view = (query) => ` SELECT - ${queryForRunsFields} + ${queryForRunsFields}, + ${run_detectors_field_in_sql_query} FROM runs AS r INNER JOIN periods AS p ON p.id = r.period_id diff --git a/app/lib/other/AuthControlManager.js b/app/lib/server/AuthControlManager.js similarity index 100% rename from app/lib/other/AuthControlManager.js rename to app/lib/server/AuthControlManager.js diff --git a/app/server/WebUiServer.js b/app/lib/server/WebUiServer.js similarity index 86% rename from app/server/WebUiServer.js rename to app/lib/server/WebUiServer.js index 9bd7fd0c4..4e97ce950 100644 --- a/app/server/WebUiServer.js +++ b/app/lib/server/WebUiServer.js @@ -12,9 +12,9 @@ */ const { HttpServer } = require('@aliceo2/web-ui'); -const AuthControlManager = require('../lib/other/AuthControlManager.js'); +const AuthControlManager = require('./AuthControlManager.js'); const path = require('path'); -const config = require('../lib/config/configProvider.js'); +const config = require('../config/configProvider.js'); const EP = config.public.endpoints; @@ -43,8 +43,8 @@ class WebUiServer { defineStaticRoutes() { const { httpServer } = this; - httpServer.addStaticPath(path.join(__dirname, '..', 'public')); - httpServer.addStaticPath(path.join(__dirname, '../..', 'node_modules', 'less/dist'), '/scripts'); + httpServer.addStaticPath(path.join(__dirname, '../..', 'public')); + httpServer.addStaticPath(path.join(__dirname, '../../..', 'node_modules', 'less/dist'), '/scripts'); } buildAuthControl() { diff --git a/app/server/index.js b/app/lib/server/index.js similarity index 100% rename from app/server/index.js rename to app/lib/server/index.js diff --git a/app/lib/LogsStacker.js b/app/lib/utils/LogsStacker.js similarity index 98% rename from app/lib/LogsStacker.js rename to app/lib/utils/LogsStacker.js index 2aaa7ce2a..39a7afdf1 100644 --- a/app/lib/LogsStacker.js +++ b/app/lib/utils/LogsStacker.js @@ -36,7 +36,7 @@ class LogsStacker { this.messages[type].forEach((m) => func(type, m)); } - typeLog(type) { + logType(type) { if (this.messages[type]) { for (const m of this.messages[type]) { this.logger[type](m); diff --git a/app/lib/ResProvider.js b/app/lib/utils/ResProvider.js similarity index 97% rename from app/lib/ResProvider.js rename to app/lib/utils/ResProvider.js index d888c15f0..3fed674a5 100644 --- a/app/lib/ResProvider.js +++ b/app/lib/utils/ResProvider.js @@ -17,13 +17,13 @@ const fs = require('fs'); const { Log } = require('@aliceo2/web-ui'); const path = require('path'); const LogsStacker = require('./LogsStacker.js'); -const Utils = require('./Utils.js'); +const { switchCase, reversePrimitiveObject } = require('./obj-utils.js'); // eslint-disable-next-line prefer-const let logger; const resProviderDefaults = { - defaultSecuredDirPath: path.join(__dirname, '..', '..', 'security'), + defaultSecuredDirPath: path.join(__dirname, '../../..', 'security'), }; class ResProvider { @@ -49,7 +49,7 @@ class ResProvider { } else if (typeof onFailureAction == 'function') { return onFailureAction(res, objDefinition); } else if (typeof onFailureAction == 'string') { - return Utils.switchCase(onFailureAction, { + return switchCase(onFailureAction, { error: ResProvider.onFailureAction_error, warn: ResProvider.onFailureAction_warn, no: () => res, @@ -76,7 +76,7 @@ class ResProvider { } static nulledGetter(res, objDef) { - return Object.entries(res).filter((e) => ! e[1]).map((e) => `${e[0]}{<-${Utils.reversePrimitiveObject(objDef)[e[0]]}}`); + return Object.entries(res).filter((e) => ! e[1]).map((e) => `${e[0]}{<-${reversePrimitiveObject(objDef)[e[0]]}}`); } static nulledMessageGetter(res, objDef) { diff --git a/app/lib/utils/http-utils.js b/app/lib/utils/http-utils.js new file mode 100644 index 000000000..5e6efce1e --- /dev/null +++ b/app/lib/utils/http-utils.js @@ -0,0 +1,126 @@ +/** + * + * @license + * Copyright 2019-2020 CERN and copyright holders of ALICE O2. + * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. + * All rights not expressly granted are reserved. + * + * This software is distributed under the terms of the GNU General Public + * License v3 (GPL Version 3), copied verbatim in the file "COPYING". + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +const http = require('http'); +const https = require('https'); +const { Log } = require('@aliceo2/web-ui'); +const path = require('path'); +const fs = require('fs'); + +const logger = new Log('Utils'); + +function checkClientType(endpoint) { + const unspecifiedProtocolMessage = 'unspecified protocol in url'; + + switch (endpoint.protocol) { + case 'http:': + return http; + case 'https:': + return https; + default: + logger.error(unspecifiedProtocolMessage); + throw new Error(unspecifiedProtocolMessage); + } +} + +function makeHttpRequestForJSON(endpoint, opts, logger, onSuccess, onFailure) { + return new Promise((resolve, reject) => { + let rawData = ''; + const req = checkClientType(endpoint).request(endpoint, opts, async (res) => { + const { statusCode } = res; + const contentType = res.headers['content-type']; + + let error; + let redirect = false; + if (statusCode == 302 || statusCode == 301) { + const mess = `Redirect. Status Code: ${statusCode}; red. to ${res.headers.location}`; + if (opts.allowRedirects) { + redirect = true; + logger.warn(mess); + const nextHop = new URL(endpoint.origin + res.headers.location); + nextHop.searchParams.set('res_path', 'json'); + logger.warn(`from ${endpoint.href} to ${nextHop.href}`); + resolve(await makeHttpRequestForJSON(nextHop)); + } else { + throw new Error(mess); + } + } else if (statusCode !== 200) { + error = new Error(`Request Failed. Status Code: ${statusCode}`); + } else if (!/^application\/json/.test(contentType)) { + error = new Error(`Invalid content-type. Expected application/json but received ${contentType}`); + } + if (error) { + logger.error(error.message); + res.resume(); + return; + } + + res.on('data', (chunk) => { + rawData += chunk; + }); + + req.on('error', (e) => { + logger.error(`ERROR httpGet: ${e}`); + if (onFailure) { + onFailure(endpoint, e); + } + reject(e); + }); + + res.on('end', () => { + try { + if (!redirect) { + /* + * TMP incorrect format handling + * if (/: *,/.test(rawData)) { + * rawData = rawData.replaceAll(/: *,/ig, ':"",'); + * } + */ + + const data = JSON.parse(rawData); + if (onSuccess) { + onSuccess(endpoint, data); + } + resolve(data); + } + } catch (e) { + logger.error(`${e.message} for endpoint: ${endpoint.href}`); + const fp = path.join( + __dirname, + '..', + '..', + '..', + 'database', + 'cache', + 'rawJson', + 'failing-endpoints.txt', + ); + fs.appendFileSync(fp, `${endpoint.href}\n ${e.message}\n`); + if (onFailure) { + onFailure(endpoint, e); + } + reject(e); + } + }); + }); + + req.end(); + }); +} + +module.exports = { + checkClientType, + makeHttpRequestForJSON, +}; diff --git a/app/lib/utils/index.js b/app/lib/utils/index.js new file mode 100644 index 000000000..af25b184d --- /dev/null +++ b/app/lib/utils/index.js @@ -0,0 +1,26 @@ +/** + * @license + * Copyright CERN and copyright holders of ALICE O2. This software is + * distributed under the terms of the GNU General Public License v3 (GPL + * Version 3), copied verbatim in the file "COPYING". + * + * See http://alice-o2.web.cern.ch/license for full licensing information. + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +const httpUtils = require('./http-utils.js'); +const LogsStacker = require('./LogsStacker.js'); +const objUtils = require('./obj-utils.js'); +const ResProvider = require('./ResProvider.js'); +const sqlUtils = require('./sql-utils.js'); + +module.exports = { + ResProvider, + LogsStacker, + ...sqlUtils, + ...httpUtils, + ...objUtils, +}; diff --git a/app/lib/utils/obj-utils.js b/app/lib/utils/obj-utils.js new file mode 100644 index 000000000..ca101b311 --- /dev/null +++ b/app/lib/utils/obj-utils.js @@ -0,0 +1,110 @@ +/** + * + * @license + * Copyright 2019-2020 CERN and copyright holders of ALICE O2. + * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. + * All rights not expressly granted are reserved. + * + * This software is distributed under the terms of the GNU General Public + * License v3 (GPL Version 3), copied verbatim in the file "COPYING". + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +const util = require('util'); + +const execp = util.promisify(require('child_process').exec); + +async function exec(cmd) { + if (Array.isArray(cmd)) { + cmd = cmd.join(' '); + } + return await execp(cmd); +} + +function reversePrimitiveObject(obj) { + return Object.fromEntries(Object.entries(obj).map((e) => e.reverse())); +} + +function filterObject(obj, keptFields, suppressUndefined = false) { + if (!keptFields) { + return obj; + } + const res = {}; + for (const [nr, nl] of Object.entries(keptFields)) { + if (!suppressUndefined || res[nl]) { + res[nl] = obj[nr]; + } + } + return res; +} + +function switchCase(caseName, cases, defaultCaseValue) { + return Object.prototype.hasOwnProperty.call(cases, caseName) + ? cases[caseName] + // eslint-disable-next-line brace-style + : defaultCaseValue ? defaultCaseValue : () => {throw new Error('no case, no default case'); }; +} + +function delay(time) { + return new Promise((resolve) => setTimeout(resolve, time)); +} + +function replaceAll(s, pattern, replace) { + const p = s.split(pattern); + return p.join(replace); +} + +function arrayToChunks(arr, chunkSize) { + const chunks = []; + for (let i = 0; i < arr.length; i += chunkSize) { + chunks.push(arr.slice(i, i + chunkSize)); + } + return chunks; +} + +function applyOptsToObj(obj, options) { + Object.entries(options).forEach(([k, v]) => { + obj[k] = v; + }); +} + +function nullIfThrows(func, args, errorHandler) { + try { + return func(...args); + } catch (e) { + if (errorHandler) { + errorHandler(e, args); + } + return null; + } +} + +function distinct(arr) { + return arr.filter((value, index, self) => self.indexOf(value) === index); +} + +function throwNotImplemented() { + throw new Error('Not implemented'); +} + +function throwAbstract() { + throw new Error('Abstract, can not be used'); +} + +module.exports = { + exec, + reversePrimitiveObject, + filterObject, + switchCase, + delay, + replaceAll, + arrayToChunks, + applyOptsToObj, + nullIfThrows, + distinct, + throwNotImplemented, + throwAbstract, +}; diff --git a/app/lib/utils/sql-utils.js b/app/lib/utils/sql-utils.js new file mode 100644 index 000000000..00d9902c6 --- /dev/null +++ b/app/lib/utils/sql-utils.js @@ -0,0 +1,79 @@ +/** + * + * @license + * Copyright 2019-2020 CERN and copyright holders of ALICE O2. + * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. + * All rights not expressly granted are reserved. + * + * This software is distributed under the terms of the GNU General Public + * License v3 (GPL Version 3), copied verbatim in the file "COPYING". + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +const keywords = ['DEFAULT', 'NULL']; + +function adjusetObjValuesToSql(obj) { + const res = {}; + for (const k in obj) { + const v = obj[k]; + if (v) { + if (typeof v == 'object') { + if (Array.isArray(v)) { + res[k] = `ARRAY[${parseValuesToSql(v).join(',')}]`; + } else { + res[k] = adjusetObjValuesToSql(v); + } + } else { + res[k] = parseValuesToSql(v); + } + } else { + res[k] = null; + } + } + return res; +} + +function parseValuesToSql(v) { + if (!v) { + return null; + } + + if (Array.isArray(v)) { + return v.map((vv) => parseValuesToSql(vv)); + } + + if (typeof v == 'string' && !keywords.includes(v.toUpperCase())) { + if (v.length == 0) { + return null; + } else { + return `'${v}'`; + } + } else { + return v; + } +} + +function preserveSQLKeywords(words) { + return words.map((w) => { + if (['end'].includes(w)) { + return `"${w}"`; + } else { + return w; + } + }); +} + +function simpleBuildInsertQuery(targetTable, valuesObj) { + return `INSERT INTO ${targetTable}(${preserveSQLKeywords(Object.keys(valuesObj)).join(', ')}) + VALUES(${parseValuesToSql(Object.values(valuesObj)).join(', ')})`; +} + +module.exports = { + adjusetObjValuesToSql, + parseValuesToSql, + preserveSQLKeywords, + simpleBuildInsertQuery, +}; diff --git a/test/lib/config/configProvider.test.js b/test/lib/config/configProvider.test.js index e81707f70..d8bb85650 100644 --- a/test/lib/config/configProvider.test.js +++ b/test/lib/config/configProvider.test.js @@ -13,22 +13,24 @@ const assert = require('assert'); const config = require('../../../app/lib/config/configProvider'); - + module.exports = () => { - describe('Config Provider', () => { + describe('Config Provider', () => { it('should return config file', () => { assert(config !== null); }); it('should handle loading error properly', () => { - // todo + // Todo }); it('should provide jwt configuration', () => { + // eslint-disable-next-line no-prototype-builtins assert(config.hasOwnProperty('jwt')); }); it('should provide http server configuration', () => { + // eslint-disable-next-line no-prototype-builtins assert(config.hasOwnProperty('http')); }); }); diff --git a/test/lib/config/index.js b/test/lib/config/index.js index 2764b10df..cfe7a4aa6 100644 --- a/test/lib/config/index.js +++ b/test/lib/config/index.js @@ -13,7 +13,7 @@ const PublicConfigProviderSuite = require('./publicConfigProvider.test'); const ConfigProviderSuite = require('./configProvider.test'); - + module.exports = () => { describe('Config Provider Suite', ConfigProviderSuite); describe('Public Config Provider', PublicConfigProviderSuite); diff --git a/test/lib/config/publicConfigProvider.test.js b/test/lib/config/publicConfigProvider.test.js index 64a00b3bd..e29821c68 100644 --- a/test/lib/config/publicConfigProvider.test.js +++ b/test/lib/config/publicConfigProvider.test.js @@ -13,9 +13,9 @@ const assert = require('assert'); const { buildPublicConfig } = require('../../../app/lib/config/publicConfigProvider'); - + module.exports = () => { - describe('Public Config Provider', () => { + describe('Public Config Provider', () => { describe('Filtering objects', () => { it('should provide public config', () => { assert.doesNotThrow(() => buildPublicConfig()); diff --git a/test/lib/resProvider.test.js b/test/lib/resProvider.test.js index 7b3531a16..bbb6eb599 100644 --- a/test/lib/resProvider.test.js +++ b/test/lib/resProvider.test.js @@ -12,10 +12,10 @@ */ const assert = require('assert'); -const ResProvider = require('../../app/lib/ResProvider'); - +const ResProvider = require('../../app/lib/utils/ResProvider'); + module.exports = () => { - describe('Res Provider', () => { + describe('Res Provider', () => { describe('Sth', () => { it('should not throw error providing a passPhrase', () => { assert.doesNotThrow(() => ResProvider.passphraseProvider()); diff --git a/test/lib/utils.test.js b/test/lib/utils.test.js index 5b3f4cb7e..7c4004ccc 100644 --- a/test/lib/utils.test.js +++ b/test/lib/utils.test.js @@ -12,33 +12,31 @@ */ const assert = require('assert'); -const Utils = require('../../app/lib/Utils'); +const Utils = require('../../app/lib/utils'); +const ServicesDataCommons = require('../../app/lib/alimonitor-services/ServicesDataCommons.js'); -const arrayEquals = (a, b) => { - return Array.isArray(a) && +const arrayEquals = (a, b) => Array.isArray(a) && Array.isArray(b) && a.length === b.length && a.every((val, index) => { if (Array.isArray(val)) { - return arrayEquals(val, b[index]) - } else return val === b[index]; + return arrayEquals(val, b[index]); + } else { + return val === b[index]; + } }); -} module.exports = () => { - describe('Utils', () => { + describe('Utils', () => { describe('Filtering objects', () => { const objectSample = { - field1: "value1", - field2: "value2", - } + field1: 'value1', + field2: 'value2', + }; const keptFields = { - field1: 2 - } - const expectedRes = { - '2': 'value1', - } - + field1: 2, + }; + it('should do nothing when no keptFields provided', () => { assert(Utils.filterObject(objectSample) === objectSample); }); @@ -58,17 +56,16 @@ module.exports = () => { const sampleValues3 = [4, 5, 'DEFAULT']; it('should return the same values when not NaN nor DEFAULT', () => { - Utils.parseValuesToSql(sampleValues1).forEach( (obj, index) => - assert(obj) == sampleValues1[index] - ) + Utils.parseValuesToSql(sampleValues1).forEach((obj, index) => + assert(obj) == sampleValues1[index]); }); it('should parse undefined values as null', () => { - assert((Utils.parseValuesToSql(sampleValues2))[2] === null); + assert(Utils.parseValuesToSql(sampleValues2)[2] === null); }); it('should return wrap DEFAULT in quotes', () => { - assert((Utils.parseValuesToSql(sampleValues3))[2] === "DEFAULT"); + assert(Utils.parseValuesToSql(sampleValues3)[2] === 'DEFAULT'); }); }); @@ -103,10 +100,10 @@ module.exports = () => { const defaultVal = 'default'; const caseNames = ['a', 'b']; const cases = { - a: () => {return 'a';}, - b: () => {return 'b';} + a: () => 'a', + b: () => 'b', }; - const defaultCase = () => {return defaultVal;}; + const defaultCase = () => defaultVal; it('should return correct value for each case', () => { assert(Utils.switchCase(caseNames[0], cases, defaultCase)() === caseNames[0]); @@ -125,8 +122,8 @@ module.exports = () => { await Utils.delay(delayTime); const end = Date.now(); assert(start + delayTime <= end); - }); - }); + }); + }); describe('Array to chunks', () => { it('Should split an array into chunks', async () => { @@ -136,16 +133,16 @@ module.exports = () => { const outcome = Utils.arrayToChunks(array, chunkSize); assert(arrayEquals(expectedOutcome, outcome)); - }); - }); + }); + }); describe('Extracting period year', () => { it('Should extract period year from period name', () => { const periodNameSamples = ['LHC12c', 'LHC23j', 'LHC00q', 'LHC', 'LHC51', null]; const expectedOutcome = [2012, 2023, 2000, 'NULL', 1951, 'NULL']; - const outcome = periodNameSamples.map(periodName => Utils.extractPeriodYear(periodName)); + const outcome = periodNameSamples.map((periodName) => ServicesDataCommons.extractPeriodYear(periodName)); assert(arrayEquals(expectedOutcome, outcome)); - }) - }) + }); + }); }); }; From cc73b352336e18d6d09ee2f4aa9cfc32522b68d4 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Mon, 10 Jul 2023 14:56:49 +0200 Subject: [PATCH 2/8] config refactor --- app/config/databasePersistance.js | 60 ++++++++++++++++--------------- app/lib/utils/ResProvider.js | 20 +++++++++-- 2 files changed, 48 insertions(+), 32 deletions(-) diff --git a/app/config/databasePersistance.js b/app/config/databasePersistance.js index 9239c640a..ac6ae6f1d 100644 --- a/app/config/databasePersistance.js +++ b/app/config/databasePersistance.js @@ -13,7 +13,7 @@ */ /* eslint-disable max-len */ -const detectors = require('./detectors.js'); +const detectors = require('./detectors.js').sort(); const flags = require('./flagsDefinitions.json'); const particle_phys_data = { @@ -39,35 +39,37 @@ const particle_phys_data = { }, }; -module.exports = { - suppressHealthcheckLogs: true, - - detectors: detectors.sort(), - healthcheckQueries: { - detectors: { - description: 'detectors dict insert', - query: detectors.map((d) => `INSERT INTO detectors_subsystems("id", "name") VALUES (DEFAULT, '${d}');`), - }, - particle: { - description: 'particles dict insert', - query: Object.entries(particle_phys_data).map(([name, d]) => `INSERT INTO particle_phys_data("id", "name", "full_name", "A", "Z") - VALUES (DEFAULT, '${name}', '${d.full_name}', ${d.A}, ${d.Z});`), - }, - flags: { - description: 'flags types dict insert', - query: flags.map((f) => `INSERT INTO flags_types_dictionary("id", "name", "method", "bad", "obsolate") - VALUES (${f.id}, '${f.name}', '${f.method}', ${f.bad}::bool, ${f.obsolete}::bool);`), - }, +const healthcheckQueries = { + detectors: { + description: 'detectors dict insert', + query: detectors.map((d) => `INSERT INTO detectors_subsystems("id", "name") VALUES (DEFAULT, '${d}');`), }, - - beam_type_mappings: { - pp: 'p-p', - nn: 'n-n', - XeXe: 'Xe-Xe', - PbPb: 'Pb-Pb', - pPb: 'p-Pb', - Pbp: 'p-Pb', - pA: 'p-A', + particle: { + description: 'particles dict insert', + query: Object.entries(particle_phys_data).map(([name, d]) => `INSERT INTO particle_phys_data("id", "name", "full_name", "A", "Z") + VALUES (DEFAULT, '${name}', '${d.full_name}', ${d.A}, ${d.Z});`), + }, + flags: { + description: 'flags types dict insert', + query: flags.map((f) => `INSERT INTO flags_types_dictionary("id", "name", "method", "bad", "obsolate") + VALUES (${f.id}, '${f.name}', '${f.method}', ${f.bad}::bool, ${f.obsolete}::bool);`), }, +}; + +const suppressHealthcheckLogs = true; +const beam_type_mappings = { + pp: 'p-p', + nn: 'n-n', + XeXe: 'Xe-Xe', + PbPb: 'Pb-Pb', + pPb: 'p-Pb', + Pbp: 'p-Pb', + pA: 'p-A', +}; +module.exports = { + suppressHealthcheckLogs, + detectors, + healthcheckQueries, + beam_type_mappings, }; diff --git a/app/lib/utils/ResProvider.js b/app/lib/utils/ResProvider.js index 3fed674a5..c0577b140 100644 --- a/app/lib/utils/ResProvider.js +++ b/app/lib/utils/ResProvider.js @@ -38,12 +38,27 @@ class ResProvider { * @returns {Object} desired env vars stored in object under names defined by mapping */ static viaEnvVars(objDefinition, failurePredicate, onFailureAction) { + const res = ResProvider.readEnvVarsObj(objDefinition); + ResProvider.validateEnvVars(res, objDefinition, failurePredicate, onFailureAction); + return res; + } + + static readEnvVarsObj(objDefinition) { const res = {}; for (const [envVName, key] of Object.entries(objDefinition)) { - res[key] = process.env[envVName]; + if (typeof key == 'string') { + res[key] = process.env[envVName]; + } else if (Array.isArray(key)) { + const [k, def] = key; + res[k] == process.env[envVName] || def; + } } + return res; + } + + static validateEnvVars(res, objDefinition, failurePredicate, onFailureAction) { if (!failurePredicate && !ResProvider.areDesiredValuesPresent(res, objDefinition) - || failurePredicate && failurePredicate(res, objDefinition)) { + || failurePredicate && failurePredicate(res, objDefinition)) { if (!onFailureAction) { ResProvider.onFailureAction_error(res, objDefinition); } else if (typeof onFailureAction == 'function') { @@ -56,7 +71,6 @@ class ResProvider { })(res, objDefinition); } } - return res; } static envOrDef(name, def, castType = String) { From d895dbd28ce673328a3ae87cbfcf719d871aba12 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Mon, 10 Jul 2023 16:28:24 +0200 Subject: [PATCH 3/8] config refactoring --- app/config/index.js | 2 +- app/config/public.js | 6 ++- app/config/rct-data/beamTypesMappings.js | 25 +++++++++++ app/config/{ => rct-data}/detectors.js | 1 + .../flags.json} | 0 .../healthcheckQueries.js} | 42 ++----------------- app/config/rct-data/index.js | 29 +++++++++++++ app/config/rct-data/physicalParticlesData.js | 38 +++++++++++++++++ app/config/{ => rct-data}/roles.js | 3 +- .../ServicesDataCommons.js | 4 +- app/lib/database/DatabaseService.js | 4 +- app/lib/database/views/runs_views.js | 2 +- app/lib/utils/ResProvider.js | 5 ++- app/lib/utils/rct-data-helpers.js | 20 +++++++++ package.json | 6 +-- 15 files changed, 134 insertions(+), 53 deletions(-) create mode 100644 app/config/rct-data/beamTypesMappings.js rename app/config/{ => rct-data}/detectors.js (99%) rename app/config/{flagsDefinitions.json => rct-data/flags.json} (100%) rename app/config/{databasePersistance.js => rct-data/healthcheckQueries.js} (63%) create mode 100644 app/config/rct-data/index.js create mode 100644 app/config/rct-data/physicalParticlesData.js rename app/config/{ => rct-data}/roles.js (99%) create mode 100644 app/lib/utils/rct-data-helpers.js diff --git a/app/config/index.js b/app/config/index.js index 297080e6b..382211f21 100644 --- a/app/config/index.js +++ b/app/config/index.js @@ -23,7 +23,7 @@ module.exports = Object.freeze({ winston: ResProvider.winston(), database: ResProvider.database(), syncTaskAtStart: ResProvider.envOrDef('RCT_SYNC_TASK_AT_START', false, Boolean), - databasePersistance: require('./databasePersistance.js'), + rctData: require('./rct-data'), public: require('./public.js'), // External services config diff --git a/app/config/public.js b/app/config/public.js index 35e8305a3..ac82fc350 100644 --- a/app/config/public.js +++ b/app/config/public.js @@ -54,9 +54,11 @@ const mcViews = { number_of_events: fromToType, }; +const { roles, flags: flagsTypes } = require('./rct-data'); + module.exports = { // Properties that will be provided to frontend in the public folder - roles: require('./roles.js'), - flagsTypes: require('./flagsDefinitions.json'), + roles, + flagsTypes, endpoints: { login: '/login/', logout: '/logout/', diff --git a/app/config/rct-data/beamTypesMappings.js b/app/config/rct-data/beamTypesMappings.js new file mode 100644 index 000000000..f183d4e42 --- /dev/null +++ b/app/config/rct-data/beamTypesMappings.js @@ -0,0 +1,25 @@ +/** + * @license + * Copyright 2019-2020 CERN and copyright holders of ALICE O2. + * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. + * All rights not expressly granted are reserved. + * + * This software is distributed under the terms of the GNU General Public + * License v3 (GPL Version 3), copied verbatim in the file "COPYING". + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +const beamTypesMappings = { + pp: 'p-p', + nn: 'n-n', + XeXe: 'Xe-Xe', + PbPb: 'Pb-Pb', + pPb: 'p-Pb', + Pbp: 'p-Pb', + pA: 'p-A', +}; + +module.exports = beamTypesMappings; diff --git a/app/config/detectors.js b/app/config/rct-data/detectors.js similarity index 99% rename from app/config/detectors.js rename to app/config/rct-data/detectors.js index e96f97b32..c11ef3265 100644 --- a/app/config/detectors.js +++ b/app/config/rct-data/detectors.js @@ -11,6 +11,7 @@ * granted to it by virtue of its status as an Intergovernmental Organization * or submit itself to any jurisdiction. */ + const detectors = [ 'CPV', 'EMC', diff --git a/app/config/flagsDefinitions.json b/app/config/rct-data/flags.json similarity index 100% rename from app/config/flagsDefinitions.json rename to app/config/rct-data/flags.json diff --git a/app/config/databasePersistance.js b/app/config/rct-data/healthcheckQueries.js similarity index 63% rename from app/config/databasePersistance.js rename to app/config/rct-data/healthcheckQueries.js index ac6ae6f1d..b2c87915c 100644 --- a/app/config/databasePersistance.js +++ b/app/config/rct-data/healthcheckQueries.js @@ -11,33 +11,10 @@ * granted to it by virtue of its status as an Intergovernmental Organization * or submit itself to any jurisdiction. */ -/* eslint-disable max-len */ const detectors = require('./detectors.js').sort(); -const flags = require('./flagsDefinitions.json'); - -const particle_phys_data = { - p: { - full_name: 'proton', - Z: 1, - A: 1, - }, - Pb: { - full_name: 'lead', - Z: 83, - A: 207, - }, - O: { - full_name: 'oxygen', - A: 8, - Z: 16, - }, - Xe: { - full_name: 'xenon', - A: 54, - Z: 131, - }, -}; +const flags = require('./flags.json'); +const physicalParticlesData = require('./physicalParticlesData.js'); const healthcheckQueries = { detectors: { @@ -46,7 +23,7 @@ const healthcheckQueries = { }, particle: { description: 'particles dict insert', - query: Object.entries(particle_phys_data).map(([name, d]) => `INSERT INTO particle_phys_data("id", "name", "full_name", "A", "Z") + query: Object.entries(physicalParticlesData).map(([name, d]) => `INSERT INTO particle_phys_data("id", "name", "full_name", "A", "Z") VALUES (DEFAULT, '${name}', '${d.full_name}', ${d.A}, ${d.Z});`), }, flags: { @@ -56,20 +33,9 @@ const healthcheckQueries = { }, }; -const suppressHealthcheckLogs = true; -const beam_type_mappings = { - pp: 'p-p', - nn: 'n-n', - XeXe: 'Xe-Xe', - PbPb: 'Pb-Pb', - pPb: 'p-Pb', - Pbp: 'p-Pb', - pA: 'p-A', -}; +const suppressHealthcheckLogs = (process.env['RCT_SUPRESS_HEALTHCECK_LOGS']?.toLowerCase() || 'false') == 'true'; module.exports = { suppressHealthcheckLogs, - detectors, healthcheckQueries, - beam_type_mappings, }; diff --git a/app/config/rct-data/index.js b/app/config/rct-data/index.js new file mode 100644 index 000000000..1422dcfb7 --- /dev/null +++ b/app/config/rct-data/index.js @@ -0,0 +1,29 @@ +/** + * @license + * Copyright 2019-2020 CERN and copyright holders of ALICE O2. + * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. + * All rights not expressly granted are reserved. + * + * This software is distributed under the terms of the GNU General Public + * License v3 (GPL Version 3), copied verbatim in the file "COPYING". + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +const detectors = require('./detectors.js').sort(); +const flags = require('./flags.json'); +const roles = require('./roles.js'); +const physicalParticlesData = require('./physicalParticlesData.js'); +const beamTypesMappings = require('./beamTypesMappings.js'); +const healthcheckQueries = require('./healthcheckQueries.js'); + +module.exports = { + detectors, + roles, + flags, + physicalParticlesData, + beamTypesMappings, + ...healthcheckQueries, +}; diff --git a/app/config/rct-data/physicalParticlesData.js b/app/config/rct-data/physicalParticlesData.js new file mode 100644 index 000000000..f779834a8 --- /dev/null +++ b/app/config/rct-data/physicalParticlesData.js @@ -0,0 +1,38 @@ +/** + * @license + * Copyright 2019-2020 CERN and copyright holders of ALICE O2. + * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. + * All rights not expressly granted are reserved. + * + * This software is distributed under the terms of the GNU General Public + * License v3 (GPL Version 3), copied verbatim in the file "COPYING". + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +const physicalParticlesData = { + p: { + full_name: 'proton', + Z: 1, + A: 1, + }, + Pb: { + full_name: 'lead', + Z: 83, + A: 207, + }, + O: { + full_name: 'oxygen', + A: 8, + Z: 16, + }, + Xe: { + full_name: 'xenon', + A: 54, + Z: 131, + }, +}; + +module.exports = physicalParticlesData; diff --git a/app/config/roles.js b/app/config/rct-data/roles.js similarity index 99% rename from app/config/roles.js rename to app/config/rct-data/roles.js index 06d9fa52c..33ba42164 100644 --- a/app/config/roles.js +++ b/app/config/rct-data/roles.js @@ -11,13 +11,12 @@ * granted to it by virtue of its status as an Intergovernmental Organization * or submit itself to any jurisdiction. */ +const detectors = require('./detectors.js'); const meta = { SSO_DET_ROLE: 'det-', }; -const detectors = require('./detectors.js'); - const dict = { Admin: 'admin', Global: 'global', diff --git a/app/lib/alimonitor-services/ServicesDataCommons.js b/app/lib/alimonitor-services/ServicesDataCommons.js index a286f818c..d565bb724 100644 --- a/app/lib/alimonitor-services/ServicesDataCommons.js +++ b/app/lib/alimonitor-services/ServicesDataCommons.js @@ -14,7 +14,7 @@ */ const Utils = require('../utils'); -const { databasePersistance } = require('../config/configProvider.js'); +const { rctData } = require('../config/configProvider.js'); /** * Update objectData.beam_type to valid format if mapping is provided with app config @@ -25,7 +25,7 @@ const { databasePersistance } = require('../config/configProvider.js'); function mapBeamTypeToCommonFormat(dataObject) { dataObject.beam_type = Utils.switchCase( dataObject.beam_type, - databasePersistance.beam_type_mappings, + rctData.beamTypesMappings, dataObject.beam_type, ); return dataObject; diff --git a/app/lib/database/DatabaseService.js b/app/lib/database/DatabaseService.js index 5bdded7c9..fef13f8ea 100644 --- a/app/lib/database/DatabaseService.js +++ b/app/lib/database/DatabaseService.js @@ -216,10 +216,10 @@ class DatabaseService { } async healthcheck() { - for (const [d, def] of Object.entries(config.databasePersistance.healthcheckQueries)) { + for (const [d, def] of Object.entries(config.rctData.healthcheckQueries)) { this.logger.info(`healthcheck for ${def.description}`); for (const q of def.query) { - const logger = config.databasePersistance.suppressHealthcheckLogs ? null : (e) => this.logger.error(e.stack) + const logger = config.rctData.suppressHealthcheckLogs ? null : (e) => this.logger.error(e.stack) await this.pgExec(q, logger, null, logger) } } diff --git a/app/lib/database/views/runs_views.js b/app/lib/database/views/runs_views.js index 2cb88b469..673748cc1 100644 --- a/app/lib/database/views/runs_views.js +++ b/app/lib/database/views/runs_views.js @@ -12,7 +12,7 @@ * or submit itself to any jurisdiction. */ const config = require('../../config/configProvider.js'); - const run_detectors_field_in_sql_query = config.databasePersistance.detectors + const run_detectors_field_in_sql_query = config.rctData.detectors .map(d => `(SELECT get_run_det_data(r.run_number, '${d.toUpperCase()}')) as ${d.toUpperCase()}_detector`) .join(',\n') diff --git a/app/lib/utils/ResProvider.js b/app/lib/utils/ResProvider.js index c0577b140..4a0165265 100644 --- a/app/lib/utils/ResProvider.js +++ b/app/lib/utils/ResProvider.js @@ -39,8 +39,7 @@ class ResProvider { */ static viaEnvVars(objDefinition, failurePredicate, onFailureAction) { const res = ResProvider.readEnvVarsObj(objDefinition); - ResProvider.validateEnvVars(res, objDefinition, failurePredicate, onFailureAction); - return res; + return ResProvider.validateEnvVars(res, objDefinition, failurePredicate, onFailureAction); } static readEnvVarsObj(objDefinition) { @@ -71,6 +70,8 @@ class ResProvider { })(res, objDefinition); } } + + return res; } static envOrDef(name, def, castType = String) { diff --git a/app/lib/utils/rct-data-helpers.js b/app/lib/utils/rct-data-helpers.js new file mode 100644 index 000000000..89ed1653d --- /dev/null +++ b/app/lib/utils/rct-data-helpers.js @@ -0,0 +1,20 @@ +/** + * + * @license + * Copyright 2019-2020 CERN and copyright holders of ALICE O2. + * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. + * All rights not expressly granted are reserved. + * + * This software is distributed under the terms of the GNU General Public + * License v3 (GPL Version 3), copied verbatim in the file "COPYING". + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + + + +module.exports = { + extractPeriodYear, +}; diff --git a/package.json b/package.json index 99bb63fa2..eb1ad7bd7 100644 --- a/package.json +++ b/package.json @@ -27,13 +27,13 @@ "coverage:test": "mocha --exit test/*", "coverage:report": "nyc report --report=html --report=json", "coverage": "nyc npm run coverage:test && npm run coverage:report ", - "test": "mkdir -pv ./reports; npm run static; npm run coverage; if [ \"RUNNING_ENV\" = \"DOCKER\" ]; then chmod -R o+w ./reports; fi;", + "test": "mkdir -pv ./reports; npm run static; npm run coverage; if [ \"$RUNNING_ENV\" = \"DOCKER\" ]; then chmod -R o+w ./reports; fi;", "test:mocha": "env-cmd -f .env.test mocha", "reports:show": "open ./reports/**/*.html", "reports:clean": "rm -rf ./reports; rm -rf .nyc_output", "start:dev": "nodemon app/main.js --watch main.js --watch app --ignore app/public", - "start:dev:local": "(export RCT_DB_HOST=${RCT_DB_HOST:-localhost}; bash -c 'set -o allexport && ls && source ./docker/env_file-dev && set +o allexport && npm run start:dev')", - "deploy:db:local": "./database/setup-db.sh --env ./docker/env_file-dev", + "start:dev:local": "(export RCT_DB_HOST=${RCT_DB_HOST:-localhost}; bash -c 'set -o allexport && ls && source ./docker/dev.env && set +o allexport && npm run start:dev')", + "deploy:db:local": "./database/setup-db.sh --env ./docker/dev.env", "dev": "./rctmake prune,db:export,build,attach,stop --target dev", "node-dev-env": "(export RCT_DB_HOST=$(docker inspect o2-rct_database -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'); bash -c 'set -o allexport && ls && source ./docker/dev.env && set +o allexport && node')", "dev:up": "./rctmake build[up],attach --target dev", From cbd7b3f7aa1152b6cef6b4fb45f15aa116773efc Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 11 Jul 2023 15:54:43 +0200 Subject: [PATCH 4/8] switchCase func extension; filtering adjustment --- app/config/rct-data/healthcheckQueries.js | 2 +- app/lib/database/QueryBuilder.js | 51 +++++++++++++++++++---- app/lib/utils/obj-utils.js | 6 ++- 3 files changed, 49 insertions(+), 10 deletions(-) diff --git a/app/config/rct-data/healthcheckQueries.js b/app/config/rct-data/healthcheckQueries.js index b2c87915c..e68604e52 100644 --- a/app/config/rct-data/healthcheckQueries.js +++ b/app/config/rct-data/healthcheckQueries.js @@ -33,7 +33,7 @@ const healthcheckQueries = { }, }; -const suppressHealthcheckLogs = (process.env['RCT_SUPRESS_HEALTHCECK_LOGS']?.toLowerCase() || 'false') == 'true'; +const suppressHealthcheckLogs = (process.env['RCT_SUPRESS_HEALTHCECK_LOGS']?.toLowerCase() || 'true') == 'true'; module.exports = { suppressHealthcheckLogs, diff --git a/app/lib/database/QueryBuilder.js b/app/lib/database/QueryBuilder.js index 8f5ea9a7d..36f2ca83f 100644 --- a/app/lib/database/QueryBuilder.js +++ b/app/lib/database/QueryBuilder.js @@ -31,6 +31,18 @@ pageToViewName[PN.flags] = 'flags_view' /** * Class responsible for parsing url params, payloads of client request to sql queries */ + +const ops = { + NOTLIKE: 'NOT LIKE', + LIKE: 'LIKE', + IN: 'IN', + NOTIN: 'NOT IN', + FROM: '>=', + TO: '<=', +}; + + + class QueryBuilder { static filteringPart(params) { @@ -41,12 +53,27 @@ class QueryBuilder { from: [], to: [] } + const filtersTypesToSqlOperand = { match: 'LIKE', exclude: 'NOT LIKE', from: '>=', to: '<=' } + + const logicalOperandsPerFilters = { + from: ops.FROM, + to: ops.TO, + exclude: { + string: ops.NOTLIKE, + number: ops.NOTIN, + }, + match: { + string: ops.LIKE, + number: ops.IN, + } + }; + const filtersTypesToSqlValueQuoted = { match: '\'', exclude: '\'', @@ -58,16 +85,20 @@ class QueryBuilder { const filterTypesRegex= new RegExp(filterTypes.map((t) => `(.*-${t})`).join('|')); const filterParams = Object.entries(params).filter(([k, v]) => k.match(filterTypesRegex)); - for (let [filedNameAndFilterType, value] of Object.entries(params)) { + for (let [filedNameAndFilterType, value] of Object.entries(filterParams)) { const [fieldName, filterType] = filedNameAndFilterType.split('-'); - if (Array.isArray(value)) { - value = value[0]; - // TODO - } if (filterType in filtersTypesToParams) { filtersTypesToParams[filterType].push({ fieldName, value }) } } + + // console.log(filtersTypesToParams) + + // Object.entries(filtersTypesToParams).map(([t, pli]) => { + // pli.map(([fN, fv]) => { + // if (t + // }) + // }) // Joining previous to sql clause const sqlWhereClause = Object.keys(filtersTypesToParams) @@ -75,7 +106,7 @@ class QueryBuilder { const qt = filtersTypesToSqlValueQuoted[t]; const operand = filtersTypesToSqlOperand[t]; return filtersTypesToParams[t] - .map(({ queryParam, value }) => `"${queryParam}" ${operand} ${qt}${value}${qt}`) + .map(({ fieldName, value }) => `"${fieldName}" ${operand} ${qt}${value}${qt}`) .join("AND");}) .filter((clause) => clause?.length > 0) .join("AND"); @@ -85,7 +116,9 @@ class QueryBuilder { } static buildSelect(params) { - + // console.log(params) + delete params.aaa; + const dataSubsetQueryPart = (params) => params[DRP.countRecords] === 'true' ? '' : `LIMIT ${params[DRP.rowsOnSite]} OFFSET ${params[DRP.rowsOnSite] * (params[DRP.site] - 1)}`; @@ -106,13 +139,15 @@ class QueryBuilder { const viewName = pageToViewName[params.page] const viewGen = views[viewName] - return `WITH ${viewName} AS ( + const a = `WITH ${viewName} AS ( ${viewGen(params)}) SELECT * FROM ${viewName} ${QueryBuilder.filteringPart(params)} ${orderingPart(params)} ${dataSubsetQueryPart(params)};`; + // console.log(a); + return a; } static buildInsertOrUpdate(params) { diff --git a/app/lib/utils/obj-utils.js b/app/lib/utils/obj-utils.js index ca101b311..5171de072 100644 --- a/app/lib/utils/obj-utils.js +++ b/app/lib/utils/obj-utils.js @@ -42,10 +42,14 @@ function filterObject(obj, keptFields, suppressUndefined = false) { } function switchCase(caseName, cases, defaultCaseValue) { + if (Array.isArray(caseName)) { + return caseName.length > 1 ? switchCase(caseName.slice(1), cases[caseName[0]], defaultCaseValue) : + switchCase(caseName[0], cases, defaultCaseValue); + } return Object.prototype.hasOwnProperty.call(cases, caseName) ? cases[caseName] // eslint-disable-next-line brace-style - : defaultCaseValue ? defaultCaseValue : () => {throw new Error('no case, no default case'); }; + : defaultCaseValue ? defaultCaseValue : (() => {throw new Error('no case, no default case'); })(); } function delay(time) { From 7d6f22a593a28745497e20422c8fe93d9f6ec34d Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 11 Jul 2023 16:16:17 +0200 Subject: [PATCH 5/8] switch case extension --- app/lib/utils/obj-utils.js | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/app/lib/utils/obj-utils.js b/app/lib/utils/obj-utils.js index 5171de072..fbaf54efa 100644 --- a/app/lib/utils/obj-utils.js +++ b/app/lib/utils/obj-utils.js @@ -41,15 +41,24 @@ function filterObject(obj, keptFields, suppressUndefined = false) { return res; } -function switchCase(caseName, cases, defaultCaseValue) { +/** + * Get case (object or function) using cases definition - possibly nested object using caseName(s) - like going down tree decision tree + * @param {string} caseName asdf + * @param {*} cases - cases defintion + * @param {*} opts - lastfound: true || false, default: any [return when there is not proper path defined by caseName]. + * 'lastfound' have precedende before default. + * @returns {Object} case + */ +function switchCase(caseName, cases, opts) { if (Array.isArray(caseName)) { - return caseName.length > 1 ? switchCase(caseName.slice(1), cases[caseName[0]], defaultCaseValue) : - switchCase(caseName[0], cases, defaultCaseValue); + return caseName.length > 1 ? switchCase(caseName.slice(1), cases[caseName[0]], opts) : + switchCase(caseName[0], cases, opts); } return Object.prototype.hasOwnProperty.call(cases, caseName) ? cases[caseName] - // eslint-disable-next-line brace-style - : defaultCaseValue ? defaultCaseValue : (() => {throw new Error('no case, no default case'); })(); + : opts.lastfound ? cases : + // eslint-disable-next-line brace-style + opts.default ? opts.default : (() => { throw new Error('not last found option, no case, no default case'); })(); } function delay(time) { From 89cf9d670d71267c32591364edb1e581d2593a6a Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 11 Jul 2023 16:47:43 +0200 Subject: [PATCH 6/8] adjustmens for filtering --- app/lib/database/QueryBuilder.js | 93 +++++++++++++++++++++---------- app/lib/utils/rct-data-helpers.js | 12 +++- app/lib/utils/sql-utils.js | 18 +++--- test/lib/utils.test.js | 6 +- 4 files changed, 88 insertions(+), 41 deletions(-) diff --git a/app/lib/database/QueryBuilder.js b/app/lib/database/QueryBuilder.js index 36f2ca83f..369ab8187 100644 --- a/app/lib/database/QueryBuilder.js +++ b/app/lib/database/QueryBuilder.js @@ -32,6 +32,9 @@ pageToViewName[PN.flags] = 'flags_view' * Class responsible for parsing url params, payloads of client request to sql queries */ + +const filterTypes = ['match', 'exclude', 'from', 'to']; + const ops = { NOTLIKE: 'NOT LIKE', LIKE: 'LIKE', @@ -39,14 +42,72 @@ const ops = { NOTIN: 'NOT IN', FROM: '>=', TO: '<=', + EQ: '==', + NE: '!=', + AND: 'AND', + OR: 'OR', +}; + +const filtersTypesToSqlOperand = { + match: 'LIKE', + exclude: 'NOT LIKE', + from: '>=', + to: '<=' +} + +const logicalOperandsPerFilters = { + match: { + string: ops.LIKE, + number: ops.IN, + }, + exclude: { + string: ops.NOTLIKE, + number: ops.NOTIN, + }, + from: ops.FROM, + to: ops.TO, }; +const filtersTypesToSqlValueQuoted = { + match: '\'', + exclude: '\'', + from: '', + to: '' +} + +//match take precedens +const controlForNoArrays = { + notarray: { + match: { + string: [ops.LIKE, ops.OR], + number: [ops.EQ ], + }, + exclude: { + string: ops.NOTLIKE, + number: ops.NE, + }, + from: ops.FROM, + to: ops.TO, + }, + + array: { + match: { + string: ops.LIKE, + number: ops.EQ, + }, + exclude: { + string: ops.NOTLIKE, + number: ops.NE, + }, + from: ops.FROM, + to: ops.TO, + }, +} class QueryBuilder { static filteringPart(params) { - const filterTypes = ['match', 'exclude', 'from', 'to']; const filtersTypesToParams = { match: [], exclude: [], @@ -54,39 +115,15 @@ class QueryBuilder { to: [] } - const filtersTypesToSqlOperand = { - match: 'LIKE', - exclude: 'NOT LIKE', - from: '>=', - to: '<=' - } - - const logicalOperandsPerFilters = { - from: ops.FROM, - to: ops.TO, - exclude: { - string: ops.NOTLIKE, - number: ops.NOTIN, - }, - match: { - string: ops.LIKE, - number: ops.IN, - } - }; - - const filtersTypesToSqlValueQuoted = { - match: '\'', - exclude: '\'', - from: '', - to: '' - } - // assert correctness of previous // Mapping search params to categorized { key, value } pairs const filterTypesRegex= new RegExp(filterTypes.map((t) => `(.*-${t})`).join('|')); const filterParams = Object.entries(params).filter(([k, v]) => k.match(filterTypesRegex)); for (let [filedNameAndFilterType, value] of Object.entries(filterParams)) { const [fieldName, filterType] = filedNameAndFilterType.split('-'); + if (! Array.isArray(value) && /.*,.*/.test(value)) { + value = value.split(',').map((s) => s.trim()) + } if (filterType in filtersTypesToParams) { filtersTypesToParams[filterType].push({ fieldName, value }) } diff --git a/app/lib/utils/rct-data-helpers.js b/app/lib/utils/rct-data-helpers.js index 89ed1653d..cd5478944 100644 --- a/app/lib/utils/rct-data-helpers.js +++ b/app/lib/utils/rct-data-helpers.js @@ -13,7 +13,17 @@ * or submit itself to any jurisdiction. */ - +function extractPeriodYear(name) { + try { + const year = parseInt(name.slice(3, 5), 10); + if (isNaN(year)) { + return 'NULL'; + } + return year > 50 ? year + 1900 : year + 2000; + } catch (e) { + return 'NULL'; + } +} module.exports = { extractPeriodYear, diff --git a/app/lib/utils/sql-utils.js b/app/lib/utils/sql-utils.js index 00d9902c6..5aed781f8 100644 --- a/app/lib/utils/sql-utils.js +++ b/app/lib/utils/sql-utils.js @@ -13,7 +13,7 @@ * or submit itself to any jurisdiction. */ -const keywords = ['DEFAULT', 'NULL']; +const sqlkeywords = ['DEFAULT', 'NULL', 'END']; function adjusetObjValuesToSql(obj) { const res = {}; @@ -22,12 +22,12 @@ function adjusetObjValuesToSql(obj) { if (v) { if (typeof v == 'object') { if (Array.isArray(v)) { - res[k] = `ARRAY[${parseValuesToSql(v).join(',')}]`; + res[k] = `ARRAY[${adjustValuesToSql(v).join(',')}]`; } else { res[k] = adjusetObjValuesToSql(v); } } else { - res[k] = parseValuesToSql(v); + res[k] = adjustValuesToSql(v); } } else { res[k] = null; @@ -36,16 +36,16 @@ function adjusetObjValuesToSql(obj) { return res; } -function parseValuesToSql(v) { +function adjustValuesToSql(v) { if (!v) { return null; } if (Array.isArray(v)) { - return v.map((vv) => parseValuesToSql(vv)); + return v.map((vv) => adjustValuesToSql(vv)); } - if (typeof v == 'string' && !keywords.includes(v.toUpperCase())) { + if (typeof v == 'string') { if (v.length == 0) { return null; } else { @@ -58,7 +58,7 @@ function parseValuesToSql(v) { function preserveSQLKeywords(words) { return words.map((w) => { - if (['end'].includes(w)) { + if (sqlkeywords.includes(w)) { return `"${w}"`; } else { return w; @@ -68,12 +68,12 @@ function preserveSQLKeywords(words) { function simpleBuildInsertQuery(targetTable, valuesObj) { return `INSERT INTO ${targetTable}(${preserveSQLKeywords(Object.keys(valuesObj)).join(', ')}) - VALUES(${parseValuesToSql(Object.values(valuesObj)).join(', ')})`; + VALUES(${adjustValuesToSql(Object.values(valuesObj)).join(', ')})`; } module.exports = { adjusetObjValuesToSql, - parseValuesToSql, + adjustValuesToSql, preserveSQLKeywords, simpleBuildInsertQuery, }; diff --git a/test/lib/utils.test.js b/test/lib/utils.test.js index 7c4004ccc..b7036e5a0 100644 --- a/test/lib/utils.test.js +++ b/test/lib/utils.test.js @@ -56,16 +56,16 @@ module.exports = () => { const sampleValues3 = [4, 5, 'DEFAULT']; it('should return the same values when not NaN nor DEFAULT', () => { - Utils.parseValuesToSql(sampleValues1).forEach((obj, index) => + Utils.adjustValuesToSql(sampleValues1).forEach((obj, index) => assert(obj) == sampleValues1[index]); }); it('should parse undefined values as null', () => { - assert(Utils.parseValuesToSql(sampleValues2)[2] === null); + assert(Utils.adjustValuesToSql(sampleValues2)[2] === null); }); it('should return wrap DEFAULT in quotes', () => { - assert(Utils.parseValuesToSql(sampleValues3)[2] === 'DEFAULT'); + assert(Utils.adjustValuesToSql(sampleValues3)[2] === 'DEFAULT'); }); }); From e06c9558972009ad297805494072f99f94eb8e12 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Wed, 12 Jul 2023 11:35:01 +0200 Subject: [PATCH 7/8] drop unnecessary public config options --- app/config/public.js | 78 +++----------------------------- app/lib/database/QueryBuilder.js | 10 ++-- 2 files changed, 12 insertions(+), 76 deletions(-) diff --git a/app/config/public.js b/app/config/public.js index 926baba48..ed7c96b66 100644 --- a/app/config/public.js +++ b/app/config/public.js @@ -12,51 +12,17 @@ * or submit itself to any jurisdiction. */ -const matchExcludeType = 'match-exclude-type'; -const fromToType = 'from-to-type'; - -const runsViewsFilteringTypes = { - name: matchExcludeType, - run_number: fromToType, - time_start: fromToType, - time_end: fromToType, - time_trg_start: fromToType, - time_trg_end: fromToType, - l3_current: fromToType, - dipole_current: fromToType, - energy_per_beam: fromToType, - ir: fromToType, - filling_scheme: fromToType, - triggers_conf: matchExcludeType, - fill_number: fromToType, - run_type: matchExcludeType, - mu: matchExcludeType, - center_of_mass_energy: fromToType, -}; - -const dataPassesViews = { - name: matchExcludeType, - description: matchExcludeType, - pass_type: matchExcludeType, - jira: matchExcludeType, - ml: matchExcludeType, - number_of_events: fromToType, - software_version: matchExcludeType, - size: fromToType, -}; - -const mcViews = { - name: matchExcludeType, - description: matchExcludeType, - jira: matchExcludeType, - ml: matchExcludeType, - pwg: matchExcludeType, - number_of_events: fromToType, -}; +const matchExcludeType = 'matchexclude'; +const fromToType = 'fromto'; const { roles, flags: flagsTypes, detectors } = require('./rct-data'); module.exports = { // Properties that will be provided to frontend in the public folder + filterTypes: { + matchExcludeType, + fromToType, + }, + roles, flagsTypes, endpoints: { @@ -107,36 +73,6 @@ module.exports = { // Properties that will be provided to frontend in the public verification_insert: 'verification_insert', }, - filteringParams: { - types: { - matchExcludeType: matchExcludeType, - fromToType: fromToType, - }, - pages: { - periods: { - name: matchExcludeType, - year: fromToType, - beam: matchExcludeType, - energy: fromToType, - }, - runsPerPeriod: runsViewsFilteringTypes, - mc: mcViews, - anchoredPerMC: dataPassesViews, - anchoragePerDatapass: mcViews, - dataPasses: dataPassesViews, - runsPerDataPass: runsViewsFilteringTypes, - - flags: { - start: fromToType, - end: fromToType, - flag: matchExcludeType, - comment: matchExcludeType, - production_id: fromToType, - name: matchExcludeType, - }, - }, - - }, detectors: detectors, themes: { ehevi: 'ehevi', diff --git a/app/lib/database/QueryBuilder.js b/app/lib/database/QueryBuilder.js index 369ab8187..3e67f3da6 100644 --- a/app/lib/database/QueryBuilder.js +++ b/app/lib/database/QueryBuilder.js @@ -80,14 +80,14 @@ const controlForNoArrays = { notarray: { match: { string: [ops.LIKE, ops.OR], - number: [ops.EQ ], + number: [ops.EQ, ops.OR], }, exclude: { - string: ops.NOTLIKE, - number: ops.NE, + string: [ops.NOTLIKE, ops.AND], + number: [ops.NE, ops.AND], }, - from: ops.FROM, - to: ops.TO, + from: [ops.FROM, ops.AND], + to: [ops.TO, ops.AND], }, array: { From 0532c56367d0414447d9fec45235ec112e7a38e1 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Wed, 12 Jul 2023 12:04:16 +0200 Subject: [PATCH 8/8] tests repaired --- .../ServicesDataCommons.js | 2 +- app/lib/utils/sql-utils.js | 9 ++++--- test/lib/utils.test.js | 24 ++++--------------- 3 files changed, 9 insertions(+), 26 deletions(-) diff --git a/app/lib/alimonitor-services/ServicesDataCommons.js b/app/lib/alimonitor-services/ServicesDataCommons.js index d565bb724..24b273fee 100644 --- a/app/lib/alimonitor-services/ServicesDataCommons.js +++ b/app/lib/alimonitor-services/ServicesDataCommons.js @@ -26,7 +26,7 @@ function mapBeamTypeToCommonFormat(dataObject) { dataObject.beam_type = Utils.switchCase( dataObject.beam_type, rctData.beamTypesMappings, - dataObject.beam_type, + { default: dataObject.beam_type }, ); return dataObject; } diff --git a/app/lib/utils/sql-utils.js b/app/lib/utils/sql-utils.js index 5aed781f8..7b99337a3 100644 --- a/app/lib/utils/sql-utils.js +++ b/app/lib/utils/sql-utils.js @@ -13,7 +13,7 @@ * or submit itself to any jurisdiction. */ -const sqlkeywords = ['DEFAULT', 'NULL', 'END']; +const sqlValueKeywords = ['DEFAULT', 'NULL']; function adjusetObjValuesToSql(obj) { const res = {}; @@ -48,17 +48,16 @@ function adjustValuesToSql(v) { if (typeof v == 'string') { if (v.length == 0) { return null; - } else { + } else if (! sqlValueKeywords.includes(v?.trim().toUpperCase())) { return `'${v}'`; } - } else { - return v; } + return v; } function preserveSQLKeywords(words) { return words.map((w) => { - if (sqlkeywords.includes(w)) { + if (['end'].includes(w)) { return `"${w}"`; } else { return w; diff --git a/test/lib/utils.test.js b/test/lib/utils.test.js index b7036e5a0..598bffd91 100644 --- a/test/lib/utils.test.js +++ b/test/lib/utils.test.js @@ -69,22 +69,6 @@ module.exports = () => { }); }); - describe('Building insert query', () => { - const sampleValues = { - name: 'LHC00', - beam: 'PbPb', - energy: 962, - year: 2000, - }; - - const compareString = `INSERT INTO periods(name, beam, energy, year) - VALUES('LHC00', 'PbPb', 962, 2000)`; - - it('should create insert query correctly', () => { - assert(Utils.simpleBuildInsertQuery('periods', sampleValues) === compareString); - }); - }); - describe('Preserve SQL keywords', () => { const expectedRes = ['"end"']; const basicCase = ['sth else']; @@ -103,15 +87,15 @@ module.exports = () => { a: () => 'a', b: () => 'b', }; - const defaultCase = () => defaultVal; + const opts = { default: () => defaultVal }; it('should return correct value for each case', () => { - assert(Utils.switchCase(caseNames[0], cases, defaultCase)() === caseNames[0]); - assert(Utils.switchCase(caseNames[1], cases, defaultCase)() === caseNames[1]); + assert(Utils.switchCase(caseNames[0], cases, opts)() === caseNames[0]); + assert(Utils.switchCase(caseNames[1], cases, opts)() === caseNames[1]); }); it('should return default value when called with an unknown case', () => { - assert(Utils.switchCase(caseNames[2], cases, defaultCase)() === defaultVal); + assert(Utils.switchCase(caseNames[2], cases, opts)() === defaultVal); }); });