diff --git a/app/lib/alimonitor-services/BookkeepingService.js b/app/lib/alimonitor-services/BookkeepingService.js index 3da32ca10..e21b3d6f1 100644 --- a/app/lib/alimonitor-services/BookkeepingService.js +++ b/app/lib/alimonitor-services/BookkeepingService.js @@ -36,7 +36,7 @@ class BookkeepingService extends AbstractServiceSynchronizer { timeTrgEnd: 'time_trg_end', definition: 'run_type', lhcBeamEnergy: 'energy', - detectors: 'detectors', + detectorsQualities: 'detectors', aliceL3Current: 'l3_current_val', aliceL3Polarity: 'l3_current_polarity', aliceDipoleCurrent: 'dipole_current_val', @@ -75,17 +75,11 @@ class BookkeepingService extends AbstractServiceSynchronizer { dataAdjuster(run) { try { run = Utils.filterObject(run, this.ketpFields); - if (run.detectors) { - if (typeof run.detectors === 'string') { - if (run.detectors.includes(',')) { // TODO may other delimiters - run.detectors = run.detectors.split(/,/).map((d) => d.trim().toUpperCase()); - } else { - run.detectors = run.detectors.split(/ +/).map((d) => d.trim().toUpperCase()); - } - } - } else { - run.detectors = []; - } + const { detectors } = run; + delete run.detectors; + run.detectorNames = detectors.map(({ name }) => name.trim()); + run.detectorQualities = detectors.map(({ quality }) => quality); + this.coilsCurrentsFieldsParsing(run, 'l3_current_val', 'l3_current_polarity', 'l3_current'); this.coilsCurrentsFieldsParsing(run, 'dipole_current_val', 'dipole_current_polarity', 'dipole_current'); ServicesDataCommons.mapBeamTypeToCommonFormat(run); @@ -115,10 +109,10 @@ class BookkeepingService extends AbstractServiceSynchronizer { const { period } = d; const year = ServicesDataCommons.extractPeriodYear(period); d = Utils.adjusetObjValuesToSql(d); - const period_insert = d.period ? `call insert_period(${d.period}, ${year}, ${d.beam_type});` : ''; - const detectorsInSql = `${d.detectors}::varchar[]`; + const detectorInSql = `${d.detectorNames}::varchar[]`; + const detectorQualitiesInSql = `${d.detectorQualities}::varchar[]`; const pgCommand = `${period_insert}; call insert_run ( ${d.run_number}, ${d.period}, @@ -129,7 +123,8 @@ class BookkeepingService extends AbstractServiceSynchronizer { ${d.run_type}, ${d.fill_number}, ${d.energy}, - ${detectorsInSql}, + ${detectorInSql}, + ${detectorQualitiesInSql}, ${d.l3_current}, ${d.dipole_current} );`; diff --git a/app/lib/alimonitor-services/PassCorrectnessMonitor.js b/app/lib/alimonitor-services/PassCorrectnessMonitor.js index 08974ae0f..813e79b64 100644 --- a/app/lib/alimonitor-services/PassCorrectnessMonitor.js +++ b/app/lib/alimonitor-services/PassCorrectnessMonitor.js @@ -46,7 +46,7 @@ class PassCorrectnessMonitor { if (incorrect > 0) { const logFunc = switchCase(errorsLoggingDepth, config.errorsLoggingDepths); - errors.forEach((e) => logFunc(logger, e)); + errors.forEach((e) => logFunc(logger, e.stack)); logger.warn(`sync unseccessful for ${incorrect}/${dataSize}`); } if (omitted > 0) { diff --git a/app/lib/database/adapters/DetectorSubsystemAdapter.js b/app/lib/database/adapters/DetectorSubsystemAdapter.js index 5b98aaddf..a006e1cf1 100644 --- a/app/lib/database/adapters/DetectorSubsystemAdapter.js +++ b/app/lib/database/adapters/DetectorSubsystemAdapter.js @@ -23,11 +23,13 @@ class DetectorSubsystemAdapter { const { id, name, + RunDetectors: { quality }, } = databaseObject; return { id, name, + quality, }; } diff --git a/app/lib/database/models/DetectorSubsystem.js b/app/lib/database/models/DetectorSubsystem.js index 7cb8a6e57..b136f3925 100644 --- a/app/lib/database/models/DetectorSubsystem.js +++ b/app/lib/database/models/DetectorSubsystem.js @@ -23,7 +23,7 @@ module.exports = (sequelize) => { DetectorSubsystem.associate = (models) => { DetectorSubsystem.belongsToMany(models.Run, { - through: 'runs_detectors', + through: models.RunDetectors, foreignKey: 'detector_id', timestamps: false, }) diff --git a/app/lib/database/models/Run.js b/app/lib/database/models/Run.js index 1f6073e0f..372520fc8 100644 --- a/app/lib/database/models/Run.js +++ b/app/lib/database/models/Run.js @@ -97,7 +97,7 @@ module.exports = (sequelize) => { Run.associate = (models) => { Run.belongsTo(models.Period); Run.belongsToMany(models.DetectorSubsystem, { - through: 'runs_detectors', + through: models.RunDetectors, foreignKey: 'run_number', timestamps: false, }); diff --git a/app/lib/database/models/RunDetectors.js b/app/lib/database/models/RunDetectors.js new file mode 100644 index 000000000..e8919ce09 --- /dev/null +++ b/app/lib/database/models/RunDetectors.js @@ -0,0 +1,24 @@ +/** + * @license + * Copyright CERN and copyright holders of ALICE O2. This software is + * distributed under the terms of the GNU General Public License v3 (GPL + * Version 3), copied verbatim in the file "COPYING". + * + * See http://alice-o2.web.cern.ch/license for full licensing information. + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +const Sequelize = require('sequelize'); + +module.exports = (sequelize) => { + const RunDetectors = sequelize.define('RunDetectors', { + quality: { + type: Sequelize.STRING, + }, + }, { timestamps: false, tableName: 'runs_detectors' }); + + return RunDetectors; +}; diff --git a/app/lib/database/models/index.js b/app/lib/database/models/index.js index 5c133322a..8a4d91171 100644 --- a/app/lib/database/models/index.js +++ b/app/lib/database/models/index.js @@ -19,6 +19,7 @@ const Run = require('./Run.js'); const DataPass = require('./DataPass.js'); const DetectorSubsystem = require('./DetectorSubsystem.js'); const SimulationPass = require('./SimulationPass.js'); +const RunDetectors = require('./RunDetectors.js'); /** * @@ -33,6 +34,7 @@ const modelsFactory = (sequelize) => { DataPass, DetectorSubsystem, SimulationPass, + RunDetectors, }; models = Object.entries(models).map(([modelName, model]) => [modelName, model(sequelize)]); // instantiate models models.forEach(([_, modelInstance]) => modelInstance.associate?.(sequelize.models)); // associate models diff --git a/app/lib/database/models/typdefs/SequelizeDetectorSubsystem.js b/app/lib/database/models/typdefs/SequelizeDetectorSubsystem.js index ee8d6ecdb..dc4c098bc 100644 --- a/app/lib/database/models/typdefs/SequelizeDetectorSubsystem.js +++ b/app/lib/database/models/typdefs/SequelizeDetectorSubsystem.js @@ -16,4 +16,5 @@ * * @property {number} id * @property {string} name + * @property {string} quality */ diff --git a/app/lib/domain/entities/DetectorSubsystem.js b/app/lib/domain/entities/DetectorSubsystem.js index 4630d797d..a0a88b803 100644 --- a/app/lib/domain/entities/DetectorSubsystem.js +++ b/app/lib/domain/entities/DetectorSubsystem.js @@ -16,4 +16,5 @@ * * @property {number} id * @property {string} name + * @property {string} quality */ diff --git a/app/lib/utils/sql-utils.js b/app/lib/utils/sql-utils.js index ce65c5bec..14df0926a 100644 --- a/app/lib/utils/sql-utils.js +++ b/app/lib/utils/sql-utils.js @@ -30,7 +30,7 @@ function adjusetObjValuesToSql(obj) { res[k] = adjustValuesToSql(v); } } else { - res[k] = null; + res[k] = 'null'; } } return res; @@ -38,7 +38,7 @@ function adjusetObjValuesToSql(obj) { function adjustValuesToSql(v) { if (!v) { - return null; + return 'null'; } if (Array.isArray(v)) { @@ -47,7 +47,7 @@ function adjustValuesToSql(v) { if (typeof v == 'string') { if (v.length == 0) { - return null; + return 'null'; } else if (! sqlValueKeywords.includes(v?.trim().toUpperCase()) && isNaN(v)) { return `'${v}'`; } diff --git a/database/design.dbm b/database/design.dbm index 56e9d69d6..d7b16d5ac 100644 --- a/database/design.dbm +++ b/database/design.dbm @@ -3,7 +3,7 @@ CAUTION: Do not modify this file unless you know what you are doing. Unexpected results may occur if the code is changed deliberately. --> - + + + diff --git a/database/exported/create-tables.sql b/database/exported/create-tables.sql index 3f7b8d402..f14219aea 100644 --- a/database/exported/create-tables.sql +++ b/database/exported/create-tables.sql @@ -271,6 +271,7 @@ ALTER SEQUENCE public.runs_detectors_id_seq OWNER TO postgres; CREATE TABLE public.runs_detectors ( detector_id integer NOT NULL, run_number integer NOT NULL, + quality varchar, CONSTRAINT runs_detectors_pkey PRIMARY KEY (detector_id,run_number), CONSTRAINT rd_pair_unique UNIQUE (detector_id,run_number) ); diff --git a/database/exported/design.png b/database/exported/design.png index a82046bbd..ea571aa2b 100644 Binary files a/database/exported/design.png and b/database/exported/design.png differ diff --git a/database/stored-sql-functionalities/procedures/insert_detectors_for_runs.sql b/database/stored-sql-functionalities/procedures/insert_detectors_for_runs.sql index 77453b720..d37d57e78 100644 --- a/database/stored-sql-functionalities/procedures/insert_detectors_for_runs.sql +++ b/database/stored-sql-functionalities/procedures/insert_detectors_for_runs.sql @@ -2,16 +2,27 @@ create or replace procedure insert_detectors_for_runs( _run_number bigint, - _detectors varchar[] + _detectors varchar[], + _detectors_qualities varchar[] ) LANGUAGE plpgsql AS $$ -DEClARE d varchar; -DEClARE trg_id int; +DECLARE d varchar; +DECLARE q varchar; + +DECLARE dq varchar[]; +DEClARE trg_id integer; BEGIN - foreach d in array _detectors loop - SELECT id INTO trg_id FROM detectors_subsystems WHERE name = d; + IF array_upper(_detectors, 1) IS NULL OR array_upper(_detectors_qualities, 1) IS NULL THEN + return; + END IF; + for i in 1 .. array_upper(_detectors, 1) loop + d = _detectors[i]; + q = _detectors_qualities[i]; + SELECT id INTO trg_id FROM detectors_subsystems WHERE name = d; + + raise notice 'detector xdxd %', d; -- inserting new detectors IF trg_id IS NULL THEN raise notice 'new detector % !!!', d; @@ -22,7 +33,7 @@ BEGIN -- inserting run x detector relation IF NOT EXISTS (SELECT * FROM runs_detectors WHERE detector_id = trg_id AND run_number = _run_number) THEN SELECT id INTO trg_id FROM detectors_subsystems WHERE name = d; - INSERT INTO runs_detectors(detector_id, run_number) VALUES(trg_id, _run_number); + INSERT INTO runs_detectors(detector_id, run_number, quality) VALUES(trg_id, _run_number, q); END IF; end loop; END; diff --git a/database/stored-sql-functionalities/procedures/insert_mc_details.sql b/database/stored-sql-functionalities/procedures/insert_mc_details.sql index be499784c..6b499154a 100644 --- a/database/stored-sql-functionalities/procedures/insert_mc_details.sql +++ b/database/stored-sql-functionalities/procedures/insert_mc_details.sql @@ -8,7 +8,7 @@ LANGUAGE plpgsql AS $$ DEClARE prod_id int; BEGIN - call insert_run(_run_number, _period, null, null, null, null, null, null, null, ARRAY[]::varchar[], null, null); + call insert_run(_run_number, _period, null, null, null, null, null, null, null, ARRAY[]::varchar[], ARRAY[]::varchar[], null, null); SELECT id FROM simulation_passes INTO prod_id WHERE name = _prod_name; if NOT EXISTS (SELECT * FROM runs WHERE run_number = _run_number) OR prod_id IS NULL THEN RAISE EXCEPTION 'nulls %', now(); @@ -32,7 +32,7 @@ DEClARE prod_id int; DECLARE _run_number integer; BEGIN foreach _run_number in array _run_numbers loop - call insert_run(_run_number, _period, null, null, null, null, null, null, null, ARRAY[]::varchar[], null, null); + call insert_run(_run_number, _period, null, null, null, null, null, null, null, ARRAY[]::varchar[], ARRAY[]::varchar[], null, null); SELECT id FROM simulation_passes INTO prod_id WHERE name = _prod_name; if NOT EXISTS (SELECT * FROM runs WHERE run_number = _run_number) OR prod_id IS NULL THEN RAISE EXCEPTION 'nulls %', now(); diff --git a/database/stored-sql-functionalities/procedures/insert_prod_details.sql b/database/stored-sql-functionalities/procedures/insert_prod_details.sql index 11d40efc9..55758ed65 100644 --- a/database/stored-sql-functionalities/procedures/insert_prod_details.sql +++ b/database/stored-sql-functionalities/procedures/insert_prod_details.sql @@ -8,7 +8,7 @@ LANGUAGE plpgsql AS $$ DEClARE prod_id int; BEGIN - call insert_run(_run_number, _period, null, null, null, null, null, null, null, ARRAY[]::varchar[], null, null); + call insert_run(_run_number, _period, null, null, null, null, null, null, null, ARRAY[]::varchar[], ARRAY[]::varchar[], null, null); SELECT id FROM data_passes INTO prod_id WHERE name = _prod_name; IF NOT EXISTS (SELECT * FROM runs WHERE run_number = _run_number) THEN RAISE EXCEPTION 'no run %', now(); @@ -36,7 +36,7 @@ DEClARE prod_id int; DECLARE _run_number integer; BEGIN foreach _run_number in array _run_numbers loop - call insert_run(_run_number, _period, null, null, null, null, null, null, null, ARRAY[]::varchar[], null, null); + call insert_run(_run_number, _period, null, null, null, null, null, null, null, ARRAY[]::varchar[], ARRAY[]::varchar[], null, null); SELECT id FROM data_passes INTO prod_id WHERE name = _prod_name; IF NOT EXISTS (SELECT * FROM runs WHERE run_number = _run_number) OR prod_id IS NULL THEN RAISE EXCEPTION 'nulls %', now(); diff --git a/database/stored-sql-functionalities/procedures/insert_run.sql b/database/stored-sql-functionalities/procedures/insert_run.sql index 8ff897040..be523ede8 100644 --- a/database/stored-sql-functionalities/procedures/insert_run.sql +++ b/database/stored-sql-functionalities/procedures/insert_run.sql @@ -10,6 +10,7 @@ create or replace procedure insert_run( _fill_number integer, _energy_per_beam float, _detectors varchar[], + _detectors_qualities varchar[], _l3_current float, _dipole_current float) LANGUAGE plpgsql @@ -74,6 +75,6 @@ BEGIN END IF; END IF; - call insert_detectors_for_runs(_run_number, _detectors); + call insert_detectors_for_runs(_run_number, _detectors, _detectors_qualities); END; $$; diff --git a/rctmake b/rctmake index 5086cd149..418bc2a24 100755 --- a/rctmake +++ b/rctmake @@ -294,7 +294,7 @@ for stage in $STAGES; do ;; clean | truncate) docker cp "$PROJECT_DIR/database/utils/delete-data.sql" "$DB_CONTAINER_NAME:$DB_WORKINGDIR/delete-data.sql" \ - && docker exec $DB_CONTAINER_NAME psql -U $RCT_DB_USER -d $RCT_DB_NAME -f "$DB_WORKINGDIR/run/delete-data.sql"; + && docker exec $DB_CONTAINER_NAME psql -U $RCT_DB_USER -d $RCT_DB_NAME -f "$DB_WORKINGDIR/delete-data.sql"; ;; ip) diff --git a/test/lib/utils/utils.test.js b/test/lib/utils/utils.test.js index 30d9cd70b..8b4de6c4f 100644 --- a/test/lib/utils/utils.test.js +++ b/test/lib/utils/utils.test.js @@ -47,7 +47,7 @@ module.exports = () => { }); it('should parse undefined values as null', () => { - assert(Utils.adjustValuesToSql(undefined) === null); + assert(Utils.adjustValuesToSql(undefined) === 'null'); }); it('should return unquoted DEFAULT', () => {