From 046b3c5c95972c227fb64bdf7a78e09fb86b9c04 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 5 Sep 2023 11:22:28 +0200 Subject: [PATCH 01/15] hooks update --- hooks/pre-commit/newlines.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hooks/pre-commit/newlines.sh b/hooks/pre-commit/newlines.sh index 81b1d10cf..e70cd3154 100755 --- a/hooks/pre-commit/newlines.sh +++ b/hooks/pre-commit/newlines.sh @@ -7,12 +7,16 @@ done; FIND_EXCLUDE_PHRASE="$FIND_EXCLUDE_PHRASE -not -path './.git/*'" FIND_EXCLUDE_PHRASE="$FIND_EXCLUDE_PHRASE -not -path '*.png'" FIND_EXCLUDE_PHRASE="$FIND_EXCLUDE_PHRASE -not -path './app/public/styles/images/favicon/favicon.ico'" # it seems to be binary but for some reason `find ... -type f` finds it +FIND_EXCLUDE_PHRASE="$FIND_EXCLUDE_PHRASE -not -path './test/lib/alimonitor-services/cache/*'" + FINDCMD="find . -type f $FIND_EXCLUDE_PHRASE" f() { for p in $(bash -c "$FINDCMD"); do - test $(tail -c1 "$p" | tr $'\n' '_') != '_' && echo "No newline at end of $p"; - test "$(tail -c2 "$p" | tr $'\n' '_')" = '__' && echo "More than one newline at the end of $p"; + if [ $(echo $p | grep "$(git ls-files -md; git diff --cached --name-only)") ]; then + test $(tail -c1 "$p" | tr $'\n' '_') != '_' && echo "No newline at end of $p"; + test "$(tail -c2 "$p" | tr $'\n' '_')" = '__' && echo "More than one newline at the end of $p"; + fi; done; } From 65e7a78edcd162be9cee32588a10e7381dfdabfd Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 5 Sep 2023 11:22:55 +0200 Subject: [PATCH 02/15] add cacher env var into test env --- docker/test.env | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/test.env b/docker/test.env index 21a572a2b..d7ed46618 100644 --- a/docker/test.env +++ b/docker/test.env @@ -78,3 +78,6 @@ RCT_EP_ML_MC_DET_PATH=${RCT_EP_ML_MC_DET_PATH:-/job_events.jsp?timesel=0&res_pat RCT_EP_ML_MC_TAG_PROT=$_DEF_ML_PROT RCT_EP_ML_MC_TAG_HOST=${RCT_EP_ML_MC_TAG_HOST:-$_DEF_ML_HOST} RCT_EP_ML_MC_TAG_PATH=${RCT_EP_ML_MC_TAG_PATH:-/MC/prodDetails.jsp?res_path=json} + +### cache +RAW_JSON_CACHE_PATH=${RAW_JSON_CACHE_PATH:-/opt/RunConditionTable/test/alimonitor-services/cache} From 330244b2b73bbee67ffbf5f950c8aae264f243bd Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 5 Sep 2023 17:01:00 +0200 Subject: [PATCH 03/15] add necessary env test vars --- docker/test.env | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/test.env b/docker/test.env index d7ed46618..18632c970 100644 --- a/docker/test.env +++ b/docker/test.env @@ -80,4 +80,8 @@ RCT_EP_ML_MC_TAG_HOST=${RCT_EP_ML_MC_TAG_HOST:-$_DEF_ML_HOST} RCT_EP_ML_MC_TAG_PATH=${RCT_EP_ML_MC_TAG_PATH:-/MC/prodDetails.jsp?res_path=json} ### cache -RAW_JSON_CACHE_PATH=${RAW_JSON_CACHE_PATH:-/opt/RunConditionTable/test/alimonitor-services/cache} +RCT_DEV_USE_CACHE_INSTEAD=${RCT_DEV_USE_CACHE_INSTEAD:-true} +RAW_JSON_CACHE_PATH=${RAW_JSON_CACHE_PATH:-/opt/RunConditionTable/test/lib/alimonitor-services/cache/} + +### other +RCT_ERR_DEPTH=full From c16e20527ff32fdeea46d508d10b0c24339d9826 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 5 Sep 2023 17:03:29 +0200 Subject: [PATCH 04/15] fix period find or Create bug in BookkeepingService --- app/lib/alimonitor-services/BookkeepingService.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/lib/alimonitor-services/BookkeepingService.js b/app/lib/alimonitor-services/BookkeepingService.js index 648ac3150..2d84fc9fb 100644 --- a/app/lib/alimonitor-services/BookkeepingService.js +++ b/app/lib/alimonitor-services/BookkeepingService.js @@ -135,6 +135,9 @@ class BookkeepingService extends AbstractServiceSynchronizer { .then(async ([beamType, _]) => await PeriodRepository.T.findOrCreate({ where: { name: period.name, + }, + default: { + name: period.name, year: period.year, BeamTypeId: beamType.id, }, @@ -189,7 +192,7 @@ class BookkeepingService extends AbstractServiceSynchronizer { } syncTraversStop(currentState) { - return this.isStopped() || currentState['page'] > this.metaStore['pageCount']; + return this.isStopped() || currentState['page'] >= this.metaStore['pageCount']; } nextState(state) { From b858673a1dd21f90f0458d6f89d0e8043d9dbb91 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 5 Sep 2023 17:04:42 +0200 Subject: [PATCH 05/15] amend Cacher config --- app/lib/alimonitor-services/helpers/Cacher.js | 1 + 1 file changed, 1 insertion(+) diff --git a/app/lib/alimonitor-services/helpers/Cacher.js b/app/lib/alimonitor-services/helpers/Cacher.js index 434351274..6bfa7ce48 100644 --- a/app/lib/alimonitor-services/helpers/Cacher.js +++ b/app/lib/alimonitor-services/helpers/Cacher.js @@ -60,6 +60,7 @@ class Cacher { static serviceCacheDir(synchronizerName) { return path.join( config.services.rawJsonCachePath, + 'rawJson', synchronizerName, ); } From 06124ab8bae0bfca09634b984e9914c03309c7ab Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 5 Sep 2023 17:05:15 +0200 Subject: [PATCH 06/15] init of syncManager test --- .../alimonitor-services/syncManager.test.js | 24 +++++++++++++++++++ test/lib/index.js | 2 ++ 2 files changed, 26 insertions(+) create mode 100644 test/lib/alimonitor-services/syncManager.test.js diff --git a/test/lib/alimonitor-services/syncManager.test.js b/test/lib/alimonitor-services/syncManager.test.js new file mode 100644 index 000000000..2a62af542 --- /dev/null +++ b/test/lib/alimonitor-services/syncManager.test.js @@ -0,0 +1,24 @@ +/** + * @license + * Copyright 2019-2020 CERN and copyright holders of ALICE O2. + * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. + * All rights not expressly granted are reserved. + * + * This software is distributed under the terms of the GNU General Public + * License v3 (GPL Version 3), copied verbatim in the file "COPYING". + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +const { syncManager } = require('../../../app/lib/alimonitor-services/SyncManager.js'); +const assert = require('assert'); + +module.exports = () => describe('SyncManager suite', () => { + describe('BookkeepingService suite', () => { + it('should performe sync with random data withour errors', async () => { + await assert.doesNotReject(syncManager.services.bookkeepingService.setSyncTask()); + }); + }); +}); diff --git a/test/lib/index.js b/test/lib/index.js index 3973c8306..a400ef4f8 100644 --- a/test/lib/index.js +++ b/test/lib/index.js @@ -16,6 +16,7 @@ const ResProviderSuite = require('./utils/resProvider.test'); const ConfigSuite = require('./config'); const ServerSuite = require('./server'); const ServicesSuite = require('./services.test'); +const SyncManagerSuite = require('./alimonitor-services/syncManager.test'); module.exports = () => { describe('Utilities', UtilitiesSuite); @@ -23,4 +24,5 @@ module.exports = () => { describe('Config', ConfigSuite); describe('Server', ServerSuite); ServicesSuite(); + SyncManagerSuite(); }; From 8ee010845568f3e5bdd3e5e5b5324898e36411a8 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 5 Sep 2023 17:25:14 +0200 Subject: [PATCH 07/15] utilize generated in place random Bookkeeping data --- docker/test.env | 2 +- .../alimonitor-services/syncManager.test.js | 5 ++ .../testutil/cache-for-test.js | 85 +++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 test/lib/alimonitor-services/testutil/cache-for-test.js diff --git a/docker/test.env b/docker/test.env index 18632c970..61ab8349d 100644 --- a/docker/test.env +++ b/docker/test.env @@ -81,7 +81,7 @@ RCT_EP_ML_MC_TAG_PATH=${RCT_EP_ML_MC_TAG_PATH:-/MC/prodDetails.jsp?res_path=json ### cache RCT_DEV_USE_CACHE_INSTEAD=${RCT_DEV_USE_CACHE_INSTEAD:-true} -RAW_JSON_CACHE_PATH=${RAW_JSON_CACHE_PATH:-/opt/RunConditionTable/test/lib/alimonitor-services/cache/} +RAW_JSON_CACHE_PATH=${RAW_JSON_CACHE_PATH:-/opt/RunConditionTable/4c3a64a02110a9f3ad7ad66f63f13fb84a2dc99c/} ### other RCT_ERR_DEPTH=full diff --git a/test/lib/alimonitor-services/syncManager.test.js b/test/lib/alimonitor-services/syncManager.test.js index 2a62af542..2928d8b2d 100644 --- a/test/lib/alimonitor-services/syncManager.test.js +++ b/test/lib/alimonitor-services/syncManager.test.js @@ -13,9 +13,14 @@ */ const { syncManager } = require('../../../app/lib/alimonitor-services/SyncManager.js'); +const { generateRandomBookkeepingCachedRawJsons } = require('./testutil/cache-for-test.js'); const assert = require('assert'); module.exports = () => describe('SyncManager suite', () => { + before(() => { + generateRandomBookkeepingCachedRawJsons(); + }); + describe('BookkeepingService suite', () => { it('should performe sync with random data withour errors', async () => { await assert.doesNotReject(syncManager.services.bookkeepingService.setSyncTask()); diff --git a/test/lib/alimonitor-services/testutil/cache-for-test.js b/test/lib/alimonitor-services/testutil/cache-for-test.js new file mode 100644 index 000000000..68c26ce1d --- /dev/null +++ b/test/lib/alimonitor-services/testutil/cache-for-test.js @@ -0,0 +1,85 @@ +const fs = require('fs'); +const path = require('path'); +const { Cacher } = require('../../../../app/lib/alimonitor-services/helpers'); + +const randint = (min = 0, max = 0) => Math.round(Math.random() * (max - min) + min); +const choice = (arr) => arr[Math.floor(Math.random() * arr.length)]; + +const detectors = [ + 'CPV', + 'EMC', + 'FDD', + 'FT0', + 'FV0', + 'ITS', + 'HMP', + 'MCH', + 'MFT', + 'MID', + 'PHS', + 'TOF', + 'TPC', + 'TRD', + 'TST', + 'ZDC', +]; + +const ketpFields = { + runNumber: () => randint(1000000, 9000000), + lhcPeriod: () => `LHC${choice([22, 18])}${choice('abceadbfarebivaavgauvgzxvcm')}`, + timeO2Start: () => randint(100000000, 200000000), + timeO2End: () => randint(100000000, 200000000), + timeTrgStart: () => randint(200000000, 300000000), + timeTrgEnd: () => randint(200000000, 300000000), + definition: () => 'PHYSICS', + lhcBeamEnergy: () => randint(1000, 10000), + detectorsQualities: () => detectors.map((d) => ({ name: d, quality: choice(['good', 'bad', null]) })), + aliceL3Current: () => randint(10000, 20000), + aliceL3Polarity: () => choice(['POSITIVE', 'NEGATIVE']), + aliceDipoleCurrent: () => randint(10000, 20000), + aliceDipolePolarity: () => choice(['POSITIVE', 'NEGATIVE']), + fillNumber: () => randint(5000, 9000), + pdpBeamType: () => choice(['pp', 'PbPb', 'pPb']), +}; + +const genRun = () => Object.fromEntries(Object.entries(ketpFields).map(([k, v]) => [k, v()])); + +const genRunsBatch = (size, files) => { + const filesN = files.length; + const totalCount = size * filesN; + const pData = [...new Array(filesN)] + .map((_, pageIndex) => [ + files[pageIndex], + { + data: [...new Array(size)] + .map(() => genRun()), + meta: { + page: { pageCount: filesN, totalCount }, + }, + }, + ]); + return pData; +}; + +const bkpSourceFiles = [ + 'filter%5Bdefinitions%5D=PHYSICS&page%5Boffset%5D=0&page%5Blimit%5D=100.json', + 'filter%5Bdefinitions%5D=PHYSICS&page%5Boffset%5D=100&page%5Blimit%5D=100.json', +]; + +const generateRandomBookkeepingCachedRawJsons = () => genRunsBatch(100, bkpSourceFiles) + .map(([fN, data]) => { + const cacheDir = Cacher.serviceCacheDir('BookkeepingService'); + if (!fs.existsSync(cacheDir)) { + fs.mkdirSync(cacheDir, { recursive: true }); + } + + const targetPath = path.join(cacheDir, fN); + fs.writeFileSync( + targetPath, + JSON.stringify(data, null, 2), + ); + }); + +module.exports = { + generateRandomBookkeepingCachedRawJsons, +}; From 0ef2b8dea3ed9031b4a448aecdbdad3b79b0ce6e Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 5 Sep 2023 17:48:56 +0200 Subject: [PATCH 08/15] rewtire AbstractServiceSynchronizer to then style --- .../AbstractServiceSynchronizer.js | 49 +++++++++---------- .../alimonitor-services/BookkeepingService.js | 4 +- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/app/lib/alimonitor-services/AbstractServiceSynchronizer.js b/app/lib/alimonitor-services/AbstractServiceSynchronizer.js index 3c68a477c..f88b48697 100644 --- a/app/lib/alimonitor-services/AbstractServiceSynchronizer.js +++ b/app/lib/alimonitor-services/AbstractServiceSynchronizer.js @@ -18,6 +18,7 @@ const { Log } = require('@aliceo2/web-ui'); const config = require('../config/configProvider.js'); const { ResProvider, makeHttpRequestForJSON, arrayToChunks, applyOptsToObj, throwNotImplemented } = require('../utils'); const { Cacher, PassCorrectnessMonitor, ProgressMonitor } = require('./helpers'); +const { rawJsonCachePath } = require('../../config/services.js'); const defaultServiceSynchronizerOptions = { forceStop: false, @@ -128,32 +129,32 @@ class AbstractServiceSynchronizer { endpoint, metaDataHandler = null, ) { - try { - this.monitor = new PassCorrectnessMonitor(this.logger, this.errorsLoggingDepth); - - const rawResponse = await this.getRawResponse(endpoint); - if (metaDataHandler) { - await metaDataHandler(rawResponse); - } - - const data = this.processRawResponse(rawResponse) + this.monitor = new PassCorrectnessMonitor(this.logger, this.errorsLoggingDepth); + return await this.getRawResponse(endpoint) + .then(async (rawResponse) => { + if (metaDataHandler) { + await metaDataHandler(rawResponse); + } + return rawResponse; + }) + .then(async (rawResponse) => this.processRawResponse(rawResponse) .filter((r) => { const f = r && this.isDataUnitValid(r); if (!f) { this.monitor.handleOmitted(); } return f; - }); - - await this.makeBatchedRequest(data, endpoint); - - this.monitor.logResults(); - return true; - } catch (fatalError) { - this.logger.error(fatalError.message + fatalError.stack); - await this.interrtuptSyncTask(); - return false; - } + })) + .then(async (data) => await this.makeBatchedRequest(data, endpoint)) + .then(() => { + this.monitor.logResults(); + return true; // Passed without major errors + }) + .catch(async (fatalError) => { + this.logger.error(fatalError.message + fatalError.stack); + this.interrtuptSyncTask(); + return false; + }); } async makeBatchedRequest(data, endpoint) { @@ -200,10 +201,8 @@ class AbstractServiceSynchronizer { this.progressMonitor = new ProgressMonitor({ logger: this.logger.info.bind(this.logger), percentageStep: 0.25 }); this.forceStop = false; return await this.sync(options) - .then(() => { - if (this.forceStop) { - this.logger.info(`${this.name} forced to stop`); - } + .catch((error) => { + this.logger.error(`critical error ${error.message}}`); }); } @@ -213,7 +212,7 @@ class AbstractServiceSynchronizer { * It will NOT interrupt any sequelize call being executed. * @return {void} */ - async interrtuptSyncTask() { + interrtuptSyncTask() { this.forceStop = true; } diff --git a/app/lib/alimonitor-services/BookkeepingService.js b/app/lib/alimonitor-services/BookkeepingService.js index 2d84fc9fb..5367d74bd 100644 --- a/app/lib/alimonitor-services/BookkeepingService.js +++ b/app/lib/alimonitor-services/BookkeepingService.js @@ -184,7 +184,7 @@ class BookkeepingService extends AbstractServiceSynchronizer { const { page } = rawResponse['meta']; if (!page || !page['pageCount']) { this.logger.error(`No metadata found in Bookkeeping for the requested page: ${JSON.stringify(rawResponse)}`); - await this.interrtuptSyncTask(); + this.interrtuptSyncTask(); return; } this.metaStore['pageCount'] = page['pageCount']; @@ -192,7 +192,7 @@ class BookkeepingService extends AbstractServiceSynchronizer { } syncTraversStop(currentState) { - return this.isStopped() || currentState['page'] >= this.metaStore['pageCount']; + return this.isStopped() || currentState['page'] > this.metaStore['pageCount']; } nextState(state) { From fae49952e43857da4ba9199624d911ca16c3aa18 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 5 Sep 2023 21:22:19 +0200 Subject: [PATCH 09/15] fix bug in makeHttpRequestForJSON --- .../AbstractServiceSynchronizer.js | 8 ++-- app/lib/utils/http-utils.js | 47 +++++-------------- 2 files changed, 16 insertions(+), 39 deletions(-) diff --git a/app/lib/alimonitor-services/AbstractServiceSynchronizer.js b/app/lib/alimonitor-services/AbstractServiceSynchronizer.js index f88b48697..d656ed6cd 100644 --- a/app/lib/alimonitor-services/AbstractServiceSynchronizer.js +++ b/app/lib/alimonitor-services/AbstractServiceSynchronizer.js @@ -18,7 +18,6 @@ const { Log } = require('@aliceo2/web-ui'); const config = require('../config/configProvider.js'); const { ResProvider, makeHttpRequestForJSON, arrayToChunks, applyOptsToObj, throwNotImplemented } = require('../utils'); const { Cacher, PassCorrectnessMonitor, ProgressMonitor } = require('./helpers'); -const { rawJsonCachePath } = require('../../config/services.js'); const defaultServiceSynchronizerOptions = { forceStop: false, @@ -153,7 +152,7 @@ class AbstractServiceSynchronizer { .catch(async (fatalError) => { this.logger.error(fatalError.message + fatalError.stack); this.interrtuptSyncTask(); - return false; + return false; // Major error occurred }); } @@ -201,8 +200,9 @@ class AbstractServiceSynchronizer { this.progressMonitor = new ProgressMonitor({ logger: this.logger.info.bind(this.logger), percentageStep: 0.25 }); this.forceStop = false; return await this.sync(options) - .catch((error) => { - this.logger.error(`critical error ${error.message}}`); + .catch((fatalError) => { + this.logger.error(`${fatalError.message} :: ${fatalError.stack}`); + return false; }); } diff --git a/app/lib/utils/http-utils.js b/app/lib/utils/http-utils.js index 82c0d1e16..387f7a44e 100644 --- a/app/lib/utils/http-utils.js +++ b/app/lib/utils/http-utils.js @@ -38,7 +38,7 @@ function checkClientType(endpoint) { function makeHttpRequestForJSON(url, opts, logger, onSuccess, onFailure) { url = new URL(url); return new Promise((resolve, reject) => { - let rawData = ''; + let rawDataAccumulator = ''; const req = checkClientType(url).request(url, opts, async (res) => { const { statusCode } = res; const contentType = res.headers['content-type']; @@ -46,13 +46,12 @@ function makeHttpRequestForJSON(url, opts, logger, onSuccess, onFailure) { let error; let redirect = false; if (statusCode == 302 || statusCode == 301) { - const mess = `Redirect. Status Code: ${statusCode}; red. to ${res.headers.location}`; + const mess = `Redirect. Status Code: ${statusCode}; red. to ${res.headers.location} from ${url.href}`; if (opts.allowRedirects) { redirect = true; logger.warn(mess); const nextHop = new URL(url.origin + res.headers.location); nextHop.searchParams.set('res_path', 'json'); - logger.warn(`from ${url.href} to ${nextHop.href}`); resolve(await makeHttpRequestForJSON(nextHop)); } else { throw new Error(mess); @@ -63,52 +62,23 @@ function makeHttpRequestForJSON(url, opts, logger, onSuccess, onFailure) { error = new Error(`Invalid content-type. Expected application/json but received ${contentType}`); } if (error) { - logger.error(error.message); - res.resume(); - return; + reject(error); } res.on('data', (chunk) => { - rawData += chunk; - }); - - req.on('error', (e) => { - logger.error(`ERROR httpGet: ${e}`); - if (onFailure) { - onFailure(url, e); - } - reject(e); + rawDataAccumulator += chunk; }); res.on('end', () => { try { if (!redirect) { - /* - * TMP incorrect format handling - * if (/: *,/.test(rawData)) { - * rawData = rawData.replaceAll(/: *,/ig, ':"",'); - * } - */ - - const data = JSON.parse(rawData); + const data = JSON.parse(rawDataAccumulator); if (onSuccess) { onSuccess(url, data); } resolve(data); } } catch (e) { - logger.error(`${e.message} for endpoint: ${url.href}`); - const fp = path.join( - __dirname, - '..', - '..', - '..', - 'database', - 'cache', - 'rawJson', - 'failing-endpoints.txt', - ); - fs.appendFileSync(fp, `${url.href}\n ${e.message}\n`); if (onFailure) { onFailure(url, e); } @@ -117,6 +87,13 @@ function makeHttpRequestForJSON(url, opts, logger, onSuccess, onFailure) { }); }); + req.on('error', (e) => { + if (onFailure) { + onFailure(url, e); + } + reject(e); + }); + req.end(); }); } From d55e17b2f8af3a0979a3b79fb2709bb1f42bbd3c Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 5 Sep 2023 21:23:10 +0200 Subject: [PATCH 10/15] cleanup --- app/lib/utils/http-utils.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/lib/utils/http-utils.js b/app/lib/utils/http-utils.js index 387f7a44e..e57459d2a 100644 --- a/app/lib/utils/http-utils.js +++ b/app/lib/utils/http-utils.js @@ -16,8 +16,6 @@ const http = require('http'); const https = require('https'); const { Log } = require('@aliceo2/web-ui'); -const path = require('path'); -const fs = require('fs'); const logger = new Log('Utils'); From 985345092052a075605217a90d7ff86aefe5d67c Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 5 Sep 2023 21:40:36 +0200 Subject: [PATCH 11/15] restore correct behaviour --- .../alimonitor-services/BookkeepingService.js | 2 +- .../alimonitor-services/syncManager.test.js | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/app/lib/alimonitor-services/BookkeepingService.js b/app/lib/alimonitor-services/BookkeepingService.js index 5367d74bd..60344c53f 100644 --- a/app/lib/alimonitor-services/BookkeepingService.js +++ b/app/lib/alimonitor-services/BookkeepingService.js @@ -192,7 +192,7 @@ class BookkeepingService extends AbstractServiceSynchronizer { } syncTraversStop(currentState) { - return this.isStopped() || currentState['page'] > this.metaStore['pageCount']; + return this.isStopped() || currentState['page'] >= this.metaStore['pageCount']; } nextState(state) { diff --git a/test/lib/alimonitor-services/syncManager.test.js b/test/lib/alimonitor-services/syncManager.test.js index 2928d8b2d..e3744410a 100644 --- a/test/lib/alimonitor-services/syncManager.test.js +++ b/test/lib/alimonitor-services/syncManager.test.js @@ -13,17 +13,24 @@ */ const { syncManager } = require('../../../app/lib/alimonitor-services/SyncManager.js'); +const { databaseManager } = require('../../../app/lib/database/DatabaseManager.js'); const { generateRandomBookkeepingCachedRawJsons } = require('./testutil/cache-for-test.js'); const assert = require('assert'); module.exports = () => describe('SyncManager suite', () => { - before(() => { - generateRandomBookkeepingCachedRawJsons(); - }); - describe('BookkeepingService suite', () => { - it('should performe sync with random data withour errors', async () => { - await assert.doesNotReject(syncManager.services.bookkeepingService.setSyncTask()); + before(() => { + generateRandomBookkeepingCachedRawJsons(); + }); + + after(async () => { + await databaseManager.repositories.RunRepository + .findAll({ raw: true }) + .then((data) => assert(data.length > 0)); + }); + + it('should performe sync with random data withour major errors', async () => { + assert.strictEqual(await syncManager.services.bookkeepingService.setSyncTask(), true); }); }); }); From 8c80b74b4704d994364c7df682fbb01f13f8c22a Mon Sep 17 00:00:00 2001 From: xsalonx <65893715+xsalonx@users.noreply.github.com> Date: Wed, 6 Sep 2023 10:58:16 +0200 Subject: [PATCH 12/15] refactor --- .../AbstractServiceSynchronizer.js | 2 +- .../testutil/cache-for-test.js | 27 +++++-------------- 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/app/lib/alimonitor-services/AbstractServiceSynchronizer.js b/app/lib/alimonitor-services/AbstractServiceSynchronizer.js index d656ed6cd..51b382d46 100644 --- a/app/lib/alimonitor-services/AbstractServiceSynchronizer.js +++ b/app/lib/alimonitor-services/AbstractServiceSynchronizer.js @@ -150,7 +150,7 @@ class AbstractServiceSynchronizer { return true; // Passed without major errors }) .catch(async (fatalError) => { - this.logger.error(fatalError.message + fatalError.stack); + this.logger.error(`${fatalError.message} :: ${fatalError.stack}`); this.interrtuptSyncTask(); return false; // Major error occurred }); diff --git a/test/lib/alimonitor-services/testutil/cache-for-test.js b/test/lib/alimonitor-services/testutil/cache-for-test.js index 68c26ce1d..0263a2595 100644 --- a/test/lib/alimonitor-services/testutil/cache-for-test.js +++ b/test/lib/alimonitor-services/testutil/cache-for-test.js @@ -1,29 +1,11 @@ const fs = require('fs'); const path = require('path'); const { Cacher } = require('../../../../app/lib/alimonitor-services/helpers'); +const { rctData: { detectors } } = require('../../../../app/lib/config/configProvider.js'); const randint = (min = 0, max = 0) => Math.round(Math.random() * (max - min) + min); const choice = (arr) => arr[Math.floor(Math.random() * arr.length)]; -const detectors = [ - 'CPV', - 'EMC', - 'FDD', - 'FT0', - 'FV0', - 'ITS', - 'HMP', - 'MCH', - 'MFT', - 'MID', - 'PHS', - 'TOF', - 'TPC', - 'TRD', - 'TST', - 'ZDC', -]; - const ketpFields = { runNumber: () => randint(1000000, 9000000), lhcPeriod: () => `LHC${choice([22, 18])}${choice('abceadbfarebivaavgauvgzxvcm')}`, @@ -42,7 +24,10 @@ const ketpFields = { pdpBeamType: () => choice(['pp', 'PbPb', 'pPb']), }; -const genRun = () => Object.fromEntries(Object.entries(ketpFields).map(([k, v]) => [k, v()])); +const genSingleRunData = () => Object.fromEntries( + Object.entries(ketpFields) + .map(([runField, fieldDataGenerator]) => [runField, fieldDataGenerator()]), +); const genRunsBatch = (size, files) => { const filesN = files.length; @@ -52,7 +37,7 @@ const genRunsBatch = (size, files) => { files[pageIndex], { data: [...new Array(size)] - .map(() => genRun()), + .map(() => genSingleRunData()), meta: { page: { pageCount: filesN, totalCount }, }, From 9d16f66d618675f768548fa66c3ad5896877449c Mon Sep 17 00:00:00 2001 From: xsalonx <65893715+xsalonx@users.noreply.github.com> Date: Wed, 6 Sep 2023 11:26:49 +0200 Subject: [PATCH 13/15] one more hook --- .../alimonitor-services/syncManager.test.js | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/test/lib/alimonitor-services/syncManager.test.js b/test/lib/alimonitor-services/syncManager.test.js index e3744410a..541445e94 100644 --- a/test/lib/alimonitor-services/syncManager.test.js +++ b/test/lib/alimonitor-services/syncManager.test.js @@ -12,25 +12,40 @@ * or submit itself to any jurisdiction. */ +const { rctData: { detectors } } = require('../../../app/lib/config/configProvider.js'); const { syncManager } = require('../../../app/lib/alimonitor-services/SyncManager.js'); -const { databaseManager } = require('../../../app/lib/database/DatabaseManager.js'); +const { databaseManager: { repositories: { + RunRepository, + RunDetectorsRepository, + DetectorSubsystemRepository, +}, +} } = require('../../../app/lib/database/DatabaseManager.js'); const { generateRandomBookkeepingCachedRawJsons } = require('./testutil/cache-for-test.js'); const assert = require('assert'); module.exports = () => describe('SyncManager suite', () => { + before('should fetch detectors data from DB the same as in config', async () => await DetectorSubsystemRepository + .findAll({ raw: true }) + .then((detectoSubsystemData) => detectoSubsystemData.map(({ name }) => name)) + .then((detectoSubsystemNames) => assert.deepStrictEqual(detectoSubsystemNames.sort(), detectors.sort()))); + describe('BookkeepingService suite', () => { before(() => { generateRandomBookkeepingCachedRawJsons(); }); - after(async () => { - await databaseManager.repositories.RunRepository - .findAll({ raw: true }) - .then((data) => assert(data.length > 0)); - }); - it('should performe sync with random data withour major errors', async () => { assert.strictEqual(await syncManager.services.bookkeepingService.setSyncTask(), true); }); + + it('should fetch some run data directly from DB', async () => + await RunRepository + .findAll({ raw: true }) + .then((data) => assert(data.length > 0))); + + it('should fetch some run_detector data directly from DB', async () => + await RunDetectorsRepository + .findAll({ raw: true }) + .then((data) => assert(data.length > 0))); }); }); From 5b3796dab2bcdf9fc140db9ad600164be222f88b Mon Sep 17 00:00:00 2001 From: xsalonx <65893715+xsalonx@users.noreply.github.com> Date: Wed, 6 Sep 2023 12:54:35 +0200 Subject: [PATCH 14/15] separate tests --- .../alimonitor-services/syncManager.test.js | 42 ++++++++++++------- .../testutil/cache-for-test.js | 5 +++ 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/test/lib/alimonitor-services/syncManager.test.js b/test/lib/alimonitor-services/syncManager.test.js index 541445e94..c7e0021ad 100644 --- a/test/lib/alimonitor-services/syncManager.test.js +++ b/test/lib/alimonitor-services/syncManager.test.js @@ -20,7 +20,7 @@ const { databaseManager: { repositories: { DetectorSubsystemRepository, }, } } = require('../../../app/lib/database/DatabaseManager.js'); -const { generateRandomBookkeepingCachedRawJsons } = require('./testutil/cache-for-test.js'); +const { generateRandomBookkeepingCachedRawJsons, cleanCachedBkpData } = require('./testutil/cache-for-test.js'); const assert = require('assert'); module.exports = () => describe('SyncManager suite', () => { @@ -30,22 +30,34 @@ module.exports = () => describe('SyncManager suite', () => { .then((detectoSubsystemNames) => assert.deepStrictEqual(detectoSubsystemNames.sort(), detectors.sort()))); describe('BookkeepingService suite', () => { - before(() => { - generateRandomBookkeepingCachedRawJsons(); - }); + describe('with artificial cache data', () => { + before(() => { + generateRandomBookkeepingCachedRawJsons(); + }); - it('should performe sync with random data withour major errors', async () => { - assert.strictEqual(await syncManager.services.bookkeepingService.setSyncTask(), true); - }); + after(() => { + cleanCachedBkpData(); + }); + + it('should performe sync with random data withour major errors', async () => { + assert.strictEqual(await syncManager.services.bookkeepingService.setSyncTask(), true); + }); - it('should fetch some run data directly from DB', async () => - await RunRepository - .findAll({ raw: true }) - .then((data) => assert(data.length > 0))); + it('should fetch some run data directly from DB', async () => + await RunRepository + .findAll({ raw: true }) + .then((data) => assert(data.length > 0))); - it('should fetch some run_detector data directly from DB', async () => - await RunDetectorsRepository - .findAll({ raw: true }) - .then((data) => assert(data.length > 0))); + it('should fetch some run_detector data directly from DB', async () => + await RunDetectorsRepository + .findAll({ raw: true }) + .then((data) => assert(data.length > 0))); + }); + + describe('without artificial cache data', () => { + it('should performe sync with major error', async () => { + assert.strictEqual(await syncManager.services.bookkeepingService.setSyncTask(), false); + }); + }); }); }); diff --git a/test/lib/alimonitor-services/testutil/cache-for-test.js b/test/lib/alimonitor-services/testutil/cache-for-test.js index 0263a2595..394256c15 100644 --- a/test/lib/alimonitor-services/testutil/cache-for-test.js +++ b/test/lib/alimonitor-services/testutil/cache-for-test.js @@ -65,6 +65,11 @@ const generateRandomBookkeepingCachedRawJsons = () => genRunsBatch(100, bkpSourc ); }); +const cleanCachedBkpData = () => { + fs.rmSync(Cacher.serviceCacheDir('BookkeepingService'), { recursive: true, force: true }); +}; + module.exports = { generateRandomBookkeepingCachedRawJsons, + cleanCachedBkpData, }; From b0551f3e9f222d1b292328da741dcd268f2469b4 Mon Sep 17 00:00:00 2001 From: xsalonx <65893715+xsalonx@users.noreply.github.com> Date: Wed, 6 Sep 2023 12:59:36 +0200 Subject: [PATCH 15/15] amend test --- .../alimonitor-services/AbstractServiceSynchronizer.js | 3 ++- test/lib/alimonitor-services/syncManager.test.js | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/app/lib/alimonitor-services/AbstractServiceSynchronizer.js b/app/lib/alimonitor-services/AbstractServiceSynchronizer.js index 51b382d46..3b273bc74 100644 --- a/app/lib/alimonitor-services/AbstractServiceSynchronizer.js +++ b/app/lib/alimonitor-services/AbstractServiceSynchronizer.js @@ -23,6 +23,7 @@ const defaultServiceSynchronizerOptions = { forceStop: false, cacheRawResponse: process.env['RCT_DEV_USE_CACHE'] === 'false' ? false : true, useCacheJsonInsteadIfPresent: process.env['RCT_DEV_USE_CACHE_INSTEAD'] === 'true' ? true : false, + forceToUseOnlyCache: process.env['RCT_DEV_FORCE_CACHE_USAGE'] === 'true' ? true : false, batchSize: 4, }; @@ -175,7 +176,7 @@ class AbstractServiceSynchronizer { } async getRawResponse(endpoint) { - if (this.useCacheJsonInsteadIfPresent && Cacher.isCached(this.name, endpoint)) { + if (this.useCacheJsonInsteadIfPresent && Cacher.isCached(this.name, endpoint) || this.forceToUseOnlyCache) { this.logger.info(`using cached json :: ${Cacher.cachedFilePath(this.name, endpoint)}`); return Cacher.getJsonSync(this.name, endpoint); } diff --git a/test/lib/alimonitor-services/syncManager.test.js b/test/lib/alimonitor-services/syncManager.test.js index c7e0021ad..1b88ac953 100644 --- a/test/lib/alimonitor-services/syncManager.test.js +++ b/test/lib/alimonitor-services/syncManager.test.js @@ -55,6 +55,14 @@ module.exports = () => describe('SyncManager suite', () => { }); describe('without artificial cache data', () => { + before(() => { + syncManager.services.bookkeepingService.forceToUseOnlyCache = true; + }); + + after(() => { + syncManager.services.bookkeepingService.forceToUseOnlyCache = false; + }); + it('should performe sync with major error', async () => { assert.strictEqual(await syncManager.services.bookkeepingService.setSyncTask(), false); });