04 December 2024 (DB version 0.0.23): add system metric related tables and columns
CREATE TABLE "ATLAS_PANDA"."JOB_METRICS" (
"PANDAID" NUMBER(11) NOT NULL,
"JEDITASKID" NUMBER(11),
"CREATIONTIME" DATE,
"MODIFICATIONTIME" DATE,
"DATA" CLOB,
CONSTRAINT "PK_JOB_METRICS" PRIMARY KEY ("PANDAID")
)
PARTITION BY RANGE ("MODIFICATIONTIME")
INTERVAL (NUMTOYMINTERVAL(1, 'MONTH')) (
PARTITION "P_BASE" VALUES LESS THAN (TO_DATE('2024-12-01', 'YYYY-MM-DD'))
);
COMMENT ON TABLE "ATLAS_PANDA"."JOB_METRICS" IS 'System metrics per job';
COMMENT ON COLUMN "ATLAS_PANDA"."JOB_METRICS"."PANDAID" IS 'PandaID for the job';
COMMENT ON COLUMN "ATLAS_PANDA"."JOB_METRICS"."JEDITASKID" IS 'JEDI task ID for the job';
COMMENT ON COLUMN "ATLAS_PANDA"."JOB_METRICS"."CREATIONTIME" IS 'Time of data creation';
COMMENT ON COLUMN "ATLAS_PANDA"."JOB_METRICS"."MODIFICATIONTIME" IS 'Time of last update';
COMMENT ON COLUMN "ATLAS_PANDA"."JOB_METRICS"."DATA" IS 'Serialized dictionary of job metrics';
CREATE TABLE "ATLAS_PANDA"."TASK_METRICS" (
"JEDITASKID" NUMBER(11) NOT NULL,
"CREATIONTIME" DATE,
"MODIFICATIONTIME" DATE,
"DATA" CLOB,
CONSTRAINT "PK_TASK_METRICS" PRIMARY KEY ("JEDITASKID")
)
PARTITION BY RANGE ("MODIFICATIONTIME")
INTERVAL (NUMTOYMINTERVAL(1, 'MONTH')) (
PARTITION "P_BASE" VALUES LESS THAN (TO_DATE('2024-12-01', 'YYYY-MM-DD'))
);
COMMENT ON TABLE "ATLAS_PANDA"."TASK_METRICS" IS 'System metrics per task';
COMMENT ON COLUMN "ATLAS_PANDA"."TASK_METRICS"."JEDITASKID" IS 'JEDI task ID for the task';
COMMENT ON COLUMN "ATLAS_PANDA"."TASK_METRICS"."CREATIONTIME" IS 'Time of data creation';
COMMENT ON COLUMN "ATLAS_PANDA"."TASK_METRICS"."MODIFICATIONTIME" IS 'Time of last update';
COMMENT ON COLUMN "ATLAS_PANDA"."TASK_METRICS"."DATA" IS 'Serialized dictionary of task metrics';
ALTER SESSION set DDL_LOCK_TIMEOUT = 30;
ALTER TABLE "ATLAS_PANDA"."JEDI_TASKS" ADD "ACTIVATEDTIME" DATE;
ALTER TABLE "ATLAS_PANDA"."JEDI_TASKS" ADD "QUEUEDTIME" DATE;
COMMENT ON COLUMN "ATLAS_PANDA"."JEDI_TASKS"."ACTIVATEDTIME" IS 'Time of activation processing workload';
COMMENT ON COLUMN "ATLAS_PANDA"."JEDI_TASKS"."QUEUEDTIME" IS 'Start time of queuing period ready to generate jobs';
ALTER TABLE "ATLAS_PANDA"."CONFIG"
ADD "VALUE_JSON" CLOB
CONSTRAINT "VALUE_JSON_IS_JSON" CHECK ("VALUE_JSON" IS JSON);
ALTER TABLE "ATLAS_PANDA"."CONFIG"
DROP CONSTRAINT "CONFIG_VALUE_NN";
UPDATE "ATLAS_PANDA"."PANDADB_VERSION" SET major=0, minor=0, patch=23 where component ='JEDI';
UPDATE "ATLAS_PANDA"."PANDADB_VERSION" SET major=0, minor=0, patch=23 where component ='SERVER';
21 November 2024 (DB version 0.0.22): add ERROR_CLASSIFICATION table
CREATE TABLE "ATLAS_PANDA"."ERROR_CLASSIFICATION" (
"ID" NUMBER GENERATED ALWAYS AS IDENTITY ( START WITH 1000000 INCREMENT BY 1 NOCACHE NOORDER ) NOT NULL,
"ERROR_SOURCE" VARCHAR2(30 BYTE) NOT NULL,
"ERROR_CODE" NUMBER(10, 0) NOT NULL,
"ERROR_DIAG" VARCHAR2(256 BYTE) NOT NULL,
"DESCRIPTION" VARCHAR2(250 BYTE),
"ERROR_CLASS" VARCHAR2(30 BYTE) NOT NULL,
"ACTIVE" CHAR(1 BYTE) NOT NULL,
"REG_DATE" TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "PK_ERROR_CLASSIFICATION" PRIMARY KEY (ID)
);
COMMENT ON TABLE "ATLAS_PANDA"."ERROR_CLASSIFICATION" IS 'Classification of job error codes+messages to system, user or others';
COMMENT ON COLUMN "ATLAS_PANDA"."ERROR_CLASSIFICATION"."ID" IS 'Sequential ID of the request. 1M offset to avoid overlapping IDs with retry module';
COMMENT ON COLUMN "ATLAS_PANDA"."ERROR_CLASSIFICATION"."ERROR_SOURCE" IS 'Source of the error: pilotErrorCode, exeErrorCode, ddmErrorCode...';
COMMENT ON COLUMN "ATLAS_PANDA"."ERROR_CLASSIFICATION"."ERROR_CODE" IS 'Error code number';
COMMENT ON COLUMN "ATLAS_PANDA"."ERROR_CLASSIFICATION"."ERROR_DIAG" IS 'Error message';
COMMENT ON COLUMN "ATLAS_PANDA"."ERROR_CLASSIFICATION"."DESCRIPTION" IS 'Any description or comment on the entry';
COMMENT ON COLUMN "ATLAS_PANDA"."ERROR_CLASSIFICATION"."ERROR_CLASS" IS 'Error class: system, user,...';
COMMENT ON COLUMN "ATLAS_PANDA"."ERROR_CLASSIFICATION"."ACTIVE" IS 'Y or N. Depending on whether the entry is confirmed';
COMMENT ON COLUMN "ATLAS_PANDA"."ERROR_CLASSIFICATION"."REG_DATE" IS 'Registration date, defaults to current timestamp';
UPDATE "ATLAS_PANDA"."PANDADB_VERSION" SET major=0, minor=0, patch=22 where component ='JEDI';
UPDATE "ATLAS_PANDA"."PANDADB_VERSION" SET major=0, minor=0, patch=22 where component ='SERVER';
COMMIT;
18 November 2024 (DB version 0.0.21): add DATA_CAROUSEL tables&sequence. Add new column JEDI_TASKS.framework
CREATE TABLE "ATLAS_PANDA"."DATA_CAROUSEL_REQUESTS" (
"REQUEST_ID" NUMBER NOT NULL,
"DATASET" VARCHAR2(256 BYTE) NOT NULL,
"SOURCE_RSE" VARCHAR2(64 BYTE),
"DESTINATION_RSE" VARCHAR2(64 BYTE),
"DDM_RULE_ID" VARCHAR2(64 BYTE),
"STATUS" VARCHAR2(32 BYTE),
"TOTAL_FILES" NUMBER(9,0),
"STAGED_FILES" NUMBER(9,0),
"DATASET_SIZE" NUMBER(18,0),
"STAGED_SIZE" NUMBER(18,0),
"CREATION_TIME" DATE,
"START_TIME" DATE,
"END_TIME" DATE,
"MODIFICATION_TIME" DATE,
"CHECK_TIME" DATE,
CONSTRAINT "DATA_CAROU_REQ_PK" PRIMARY KEY ("REQUEST_ID") ENABLE
);
COMMENT ON TABLE "ATLAS_PANDA"."DATA_CAROUSEL_REQUESTS" IS 'Table of Data Carousel requests';
COMMENT ON COLUMN "ATLAS_PANDA"."DATA_CAROUSEL_REQUESTS"."REQUEST_ID" IS 'Sequential ID of the request, generated from Oracle sequence object ATLAS_PANDA.JEDI_DATA_CAROUSEL_REQUEST_ID_SEQ when new request is inserted';
COMMENT ON COLUMN "ATLAS_PANDA"."DATA_CAROUSEL_REQUESTS"."DATASET" IS 'Dataset to stage';
COMMENT ON COLUMN "ATLAS_PANDA"."DATA_CAROUSEL_REQUESTS"."SOURCE_RSE" IS 'Source RSE (usually tape) of staging';
COMMENT ON COLUMN "ATLAS_PANDA"."DATA_CAROUSEL_REQUESTS"."DESTINATION_RSE" IS 'Destination RSE (usually DATADISK) of staging';
COMMENT ON COLUMN "ATLAS_PANDA"."DATA_CAROUSEL_REQUESTS"."DDM_RULE_ID" IS 'DDM rule ID of the staging rule';
COMMENT ON COLUMN "ATLAS_PANDA"."DATA_CAROUSEL_REQUESTS"."STATUS" IS 'Status of the request';
COMMENT ON COLUMN "ATLAS_PANDA"."DATA_CAROUSEL_REQUESTS"."TOTAL_FILES" IS 'Number of total files of the dataset';
COMMENT ON COLUMN "ATLAS_PANDA"."DATA_CAROUSEL_REQUESTS"."STAGED_FILES" IS 'Number of files already staged';
COMMENT ON COLUMN "ATLAS_PANDA"."DATA_CAROUSEL_REQUESTS"."DATASET_SIZE" IS 'Size in bytes of the dataset';
COMMENT ON COLUMN "ATLAS_PANDA"."DATA_CAROUSEL_REQUESTS"."STAGED_SIZE" IS 'Size in bytes of files already staged';
COMMENT ON COLUMN "ATLAS_PANDA"."DATA_CAROUSEL_REQUESTS"."CREATION_TIME" IS 'Timestamp when the request is created';
COMMENT ON COLUMN "ATLAS_PANDA"."DATA_CAROUSEL_REQUESTS"."START_TIME" IS 'Timestamp when the request starts staging';
COMMENT ON COLUMN "ATLAS_PANDA"."DATA_CAROUSEL_REQUESTS"."END_TIME" IS 'Timestamp when the request ended';
COMMENT ON COLUMN "ATLAS_PANDA"."DATA_CAROUSEL_REQUESTS"."MODIFICATION_TIME" IS 'Timestamp of the last request update';
COMMENT ON COLUMN "ATLAS_PANDA"."DATA_CAROUSEL_REQUESTS"."CHECK_TIME" IS 'Last time when the request was checked';
CREATE SEQUENCE "ATLAS_PANDA"."JEDI_DATA_CAROUSEL_REQUEST_ID_SEQ" MINVALUE 1 MAXVALUE 9999999999999999999999999999 INCREMENT BY 1 START WITH 1 NOCACHE NOORDER NOCYCLE;
CREATE TABLE "ATLAS_PANDA"."DATA_CAROUSEL_RELATIONS" (
"REQUEST_ID" NUMBER(12,0) NOT NULL,
"TASK_ID" NUMBER(12,0) NOT NULL,
CONSTRAINT "DATA_CAROU_REL_UC" UNIQUE ("REQUEST_ID", "TASK_ID") ENABLE
);
COMMENT ON TABLE "ATLAS_PANDA"."DATA_CAROUSEL_RELATIONS" IS 'Table of mapping between Data Carousel requests and tasks';
COMMENT ON COLUMN "ATLAS_PANDA"."DATA_CAROUSEL_RELATIONS"."REQUEST_ID" IS 'ID of the request';
COMMENT ON COLUMN "ATLAS_PANDA"."DATA_CAROUSEL_RELATIONS"."TASK_ID" IS 'ID of the task';
ALTER TABLE "ATLAS_PANDA"."JEDI_TASKS" ADD "FRAMEWORK" VARCHAR2(50);
COMMENT ON COLUMN "ATLAS_PANDA"."JEDI_TASKS"."FRAMEWORK" IS 'Submission framework that was used to generate the task';
UPDATE "ATLAS_PANDA"."PANDADB_VERSION" SET major=0, minor=0, patch=21 where component ='JEDI';
UPDATE "ATLAS_PANDA"."PANDADB_VERSION" SET major=0, minor=0, patch=21 where component ='SERVER';
16 September 2024 (DB version 0.0.20): add ATLAS_PANDA.JOBS_SHARE_STATS.PRORATED_MEM_AVG and ATLAS_PANDA.JOBSDEFINED_SHARE_STATS.PRORATED_MEM_AVG
ALTER SESSION set DDL_LOCK_TIMEOUT = 30;
ALTER TABLE "ATLAS_PANDA"."JOBS_SHARE_STATS" ADD "PRORATED_MEM_AVG" NUMBER(11,2);
ALTER TABLE "ATLAS_PANDA"."JOBSDEFINED_SHARE_STATS" ADD "PRORATED_MEM_AVG" NUMBER(11,2);
COMMENT ON COLUMN "ATLAS_PANDA"."JOBS_SHARE_STATS"."PRORATED_MEM_AVG" IS 'avg(minRamCount/corecount)';
COMMENT ON COLUMN "ATLAS_PANDA"."JOBSDEFINED_SHARE_STATS"."PRORATED_MEM_AVG" IS 'avg(minRamCount/corecount)';
set define off;
CREATE OR REPLACE PROCEDURE "ATLAS_PANDA"."UPDATE_JOBSACT_STATS_BY_GSHARE"
AS
BEGIN
-- 16th Sept 2024 , ver 1.6
-- 14th Nov 2023 , ver 1.5
-- 27th Nov 2020 , ver 1.4
-- 29th Jan 2018 , ver 1.3
-- to easily identify the session and better view on resource usage by setting a dedicated module for the PanDA jobs
DBMS_APPLICATION_INFO.SET_MODULE( module_name => 'PanDA scheduler job', action_name => 'Aggregates data by global share for the active jobs!');
DBMS_APPLICATION_INFO.SET_CLIENT_INFO ( client_info => sys_context('userenv', 'host') || ' ( ' || sys_context('userenv', 'ip_address') || ' )' );
DELETE from ATLAS_PANDA.JOBS_SHARE_STATS;
INSERT INTO ATLAS_PANDA.JOBS_SHARE_STATS (TS, GSHARE, WORKQUEUE_ID, RESOURCE_TYPE,
COMPUTINGSITE, JOBSTATUS,
MAXPRIORITY, PRORATED_DISKIO_AVG, PRORATED_MEM_AVG, NJOBS, HS, VO)
WITH
sc_slimmed AS (
SELECT /*+ MATERIALIZE */ sc.panda_queue AS pq, sc.data.corepower AS cp
FROM ATLAS_PANDA.schedconfig_json sc
)
SELECT sysdate, gshare, workqueue_id, ja4.resource_type, computingSite, jobStatus,
MAX(currentPriority) AS maxPriority,
AVG(diskIO/NVL(ja4.coreCount, 1)) AS proratedDiskioAvg, AVG(minRamCount/NVL(ja4.coreCount, 1)) AS proratedMemAvg,
COUNT(*) AS num_of_jobs, COUNT(*) * NVL(ja4.coreCount, 1) * sc_s.cp AS HS, VO
FROM ATLAS_PANDA.jobsActive4 ja4, sc_slimmed sc_s
WHERE ja4.computingsite = sc_s.pq
GROUP BY sysdate, gshare, workqueue_id, ja4.resource_type, computingSite, jobStatus, ja4.coreCount, sc_s.cp, VO;
COMMIT;
DBMS_APPLICATION_INFO.SET_MODULE( module_name => null, action_name => null);
DBMS_APPLICATION_INFO.SET_CLIENT_INFO ( client_info => null);
end;
/
--------------------------------------------------------
-- DDL for Procedure UPDATE_JOBSDEF_STATS_BY_GSHARE
--------------------------------------------------------
set define off;
CREATE OR REPLACE PROCEDURE "ATLAS_PANDA"."UPDATE_JOBSDEF_STATS_BY_GSHARE"
AS
BEGIN
-- 16th Sept 2024 , ver 1.2
-- 14th Nov 2023 , ver 1.1
-- 27th Nov 2020 , ver 1.0
-- Based on UPDATE_JOBSACT_STATS_BY_GSHARE
-- to easily identify the session and better view on resource usage by setting a dedicated module for the PanDA jobs
DBMS_APPLICATION_INFO.SET_MODULE( module_name => 'PanDA scheduler job', action_name => 'Aggregates data by global share for the active jobs!');
DBMS_APPLICATION_INFO.SET_CLIENT_INFO ( client_info => sys_context('userenv', 'host') || ' ( ' || sys_context('userenv', 'ip_address') || ' )' );
DELETE from ATLAS_PANDA.JOBSDEFINED_SHARE_STATS;
INSERT INTO ATLAS_PANDA.JOBSDEFINED_SHARE_STATS (TS, GSHARE, WORKQUEUE_ID, RESOURCE_TYPE,
COMPUTINGSITE, JOBSTATUS,
MAXPRIORITY, PRORATED_DISKIO_AVG, PRORATED_MEM_AVG, NJOBS, HS, VO)
WITH
sc_slimmed AS (
SELECT /*+ MATERIALIZE */ sc.panda_queue AS pq, sc.data.corepower AS cp
FROM ATLAS_PANDA.schedconfig_json sc
)
SELECT sysdate, gshare, workqueue_id, ja4.resource_type, computingSite, jobStatus,
MAX(currentPriority) AS maxPriority,
AVG(diskIO/NVL(ja4.coreCount, 1)) AS proratedDiskioAvg, AVG(minRamCount/NVL(ja4.coreCount, 1)) AS proratedDiskioAvg,
COUNT(*) AS num_of_jobs, COUNT(*) * NVL(ja4.coreCount, 1) * sc_s.cp AS HS, VO
FROM ATLAS_PANDA.jobsDefined4 ja4, sc_slimmed sc_s
WHERE ja4.computingsite = sc_s.pq
GROUP BY sysdate, gshare, workqueue_id, ja4.resource_type, computingSite, jobStatus, ja4.coreCount, sc_s.cp, VO;
COMMIT;
DBMS_APPLICATION_INFO.SET_MODULE( module_name => null, action_name => null);
DBMS_APPLICATION_INFO.SET_CLIENT_INFO ( client_info => null);
end;
/
UPDATE "ATLAS_PANDA"."PANDADB_VERSION" SET major=0, minor=0, patch=20 where component ='JEDI';
UPDATE "ATLAS_PANDA"."PANDADB_VERSION" SET major=0, minor=0, patch=20 where component ='SERVER';
COMMIT;
2 September 2024 (DB version 0.0.19): add NUM_OF_CORES column to mv_jobsactive4_stats
ALTER SESSION set DDL_LOCK_TIMEOUT = 30;
ALTER TABLE "ATLAS_PANDA"."MV_JOBSACTIVE4_STATS" ADD "NUM_OF_CORES" NUMBER;
COMMENT ON COLUMN "ATLAS_PANDA"."MV_JOBSACTIVE4_STATS"."NUM_OF_CORES" IS 'Number of cores computed by grouping all set of attributes(columns) listed in that column, ';
set define off;
CREATE OR REPLACE PROCEDURE "ATLAS_PANDA"."UPDATE_JOBSACTIVE_STATS"
AS
BEGIN
-- ver 1.3 , last modified on 2nd September 2024
-- added NUM_OF_CORES columns
-- ver 1.2 , last modified on 2th July 2013
-- added VO and WORKQUEUE_ID columns
-- to easy identify the session and better view on resource usage by setting a dedicated module for the PanDA jobs
DBMS_APPLICATION_INFO.SET_MODULE( module_name => 'PanDA scheduler job', action_name => 'Aggregates data for the active jobs!');
DBMS_APPLICATION_INFO.SET_CLIENT_INFO ( client_info => sys_context('userenv', 'host') || ' ( ' || sys_context('userenv', 'ip_address') || ' )' );
DELETE from mv_jobsactive4_stats;
INSERT INTO mv_jobsactive4_stats
(CUR_DATE,
CLOUD,
COMPUTINGSITE,
COUNTRYGROUP,
WORKINGGROUP,
RELOCATIONFLAG,
JOBSTATUS,
PROCESSINGTYPE,
PRODSOURCELABEL,
CURRENTPRIORITY,
VO,
WORKQUEUE_ID,
NUM_OF_JOBS,
NUM_OF_CORES
)
SELECT
sysdate,
cloud,
computingSite,
countrygroup,
workinggroup,
relocationflag,
jobStatus,
processingType,
prodSourceLabel,
TRUNC(currentPriority, -1) AS currentPriority,
VO,
WORKQUEUE_ID,
COUNT(*) AS num_of_jobs,
SUM(COALESCE(actualcorecount, corecount)) AS num_of_cores
FROM jobsActive4
GROUP BY
sysdate,
cloud,
computingSite,
countrygroup,
workinggroup,
relocationflag,
jobStatus,
processingType,
prodSourceLabel,
TRUNC(currentPriority, -1),
VO,
WORKQUEUE_ID;
commit;
DBMS_APPLICATION_INFO.SET_MODULE( module_name => null, action_name => null);
DBMS_APPLICATION_INFO.SET_CLIENT_INFO ( client_info => null);
end;
/
UPDATE "ATLAS_PANDA"."PANDADB_VERSION" SET major=0, minor=0, patch=19 where component ='JEDI';
UPDATE "ATLAS_PANDA"."PANDADB_VERSION" SET major=0, minor=0, patch=19 where component ='SERVER';
COMMIT;
10 July 2024 (DB version 0.0.18): add MINRAMCOUNT column to Harvester Worker table
ALTER SESSION set DDL_LOCK_TIMEOUT = 30;
ALTER TABLE "ATLAS_PANDA"."HARVESTER_WORKERS" ADD "MINRAMCOUNT" NUMBER(11,0);
COMMENT ON COLUMN "ATLAS_PANDA"."HARVESTER_WORKERS"."MINRAMCOUNT" IS 'Worker memory requirements';
UPDATE "ATLAS_PANDA"."PANDADB_VERSION" SET major=0, minor=0, patch=18 where component ='JEDI';
UPDATE "ATLAS_PANDA"."PANDADB_VERSION" SET major=0, minor=0, patch=18 where component ='SERVER';
COMMIT;
21 May 2024 (DB version 0.0.17): add OUTPUTFILETYPE column to JOBS tables
-- As ATLAS_PANDA
ALTER SESSION set DDL_LOCK_TIMEOUT = 30;
ALTER TABLE "ATLAS_PANDA"."JOBSDEFINED4" ADD "OUTPUTFILETYPE" VARCHAR2(32);
ALTER TABLE "ATLAS_PANDA"."JOBSWAITING4" ADD "OUTPUTFILETYPE" VARCHAR2(32);
ALTER TABLE "ATLAS_PANDA"."JOBSACTIVE4" ADD "OUTPUTFILETYPE" VARCHAR2(32);
ALTER TABLE "ATLAS_PANDA"."JOBSARCHIVED4" ADD "OUTPUTFILETYPE" VARCHAR2(32);
COMMENT ON COLUMN "ATLAS_PANDA"."JOBSDEFINED4"."OUTPUTFILETYPE" IS 'Comma-separated list of output file types for the job';
COMMENT ON COLUMN "ATLAS_PANDA"."JOBSWAITING4"."OUTPUTFILETYPE" IS 'Comma-separated list of output file types for the job';
COMMENT ON COLUMN "ATLAS_PANDA"."JOBSACTIVE4"."OUTPUTFILETYPE" IS 'Comma-separated list of output file types for the job';
COMMENT ON COLUMN "ATLAS_PANDA"."JOBSARCHIVED4"."OUTPUTFILETYPE" IS 'Comma-separated list of output file types for the job';
UPDATE "ATLAS_PANDA"."PANDADB_VERSION" SET major=0, minor=0, patch=17 where component ='JEDI';
UPDATE "ATLAS_PANDA"."PANDADB_VERSION" SET major=0, minor=0, patch=17 where component ='SERVER';
COMMIT;
-- Update BULKCOPY_PANDA_PARTITIONS
-- As ATLAS_PANDAARCH
ALTER TABLE "ATLAS_PANDAARCH"."JOBSARCHIVED" ADD "OUTPUTFILETYPE" VARCHAR2(32);
COMMENT ON COLUMN "ATLAS_PANDAARCH"."JOBSARCHIVED"."OUTPUTFILETYPE" IS 'Comma-separated list of output file types for the job';
28 Aug 2023 (DB version 0.0.16): add REALMODIFICATIONTIME column to JEDI_TASKS table
ALTER SESSION set DDL_LOCK_TIMEOUT = 30;
-- MODIFICATIONTIME TRIGGER
-- New column JEDI_TASKS.REALMODIFICATIONTIME
ALTER TABLE "ATLAS_PANDA"."JEDI_TASKS" ADD ("REALMODIFICATIONTIME" DATE);
COMMENT ON COLUMN "ATLAS_PANDA"."JEDI_TASKS"."REALMODIFICATIONTIME" IS 'Set ALWAYS to last modification time, without any tricks like old timestamps';
CREATE INDEX "ATLAS_PANDA"."JEDI_TASKS_REALMODTIME_IDX" ON "ATLAS_PANDA"."JEDI_TASKS" ("REALMODIFICATIONTIME");
-- Trigger to set JEDI_TASKS.REALMODIFICATIONTIME to current UTC timestamp
CREATE OR REPLACE TRIGGER "ATLAS_PANDA"."UPDATE_REALMODIFICATIONTIME"
BEFORE UPDATE OR INSERT OF MODIFICATIONTIME ON "ATLAS_PANDA"."JEDI_TASKS"
FOR EACH ROW
BEGIN
:NEW."REALMODIFICATIONTIME" := SYS_EXTRACT_UTC(systimestamp);
END;
/
-- SQL_QUEUE TABLE
CREATE TABLE "ATLAS_PANDA"."SQL_QUEUE"
(
"TOPIC" VARCHAR2(50 BYTE),
"PANDAID" NUMBER(11,0),
"EXECUTION_ORDER" NUMBER(5,0),
"JEDITASKID" NUMBER(11,0),
"CREATIONTIME" DATE,
"DATA" VARCHAR2(4000 BYTE),
CONSTRAINT "SQL_QUEUE_PK" PRIMARY KEY ("TOPIC", "PANDAID", "EXECUTION_ORDER")
);
CREATE INDEX "ATLAS_PANDA"."SQL_QUEUE_TOPIC_TASK_IDX" ON "ATLAS_PANDA"."SQL_QUEUE" ("TOPIC", "JEDITASKID");
CREATE INDEX "ATLAS_PANDA"."SQL_QUEUE_TOPIC_CREATIONTIME_IDX" ON "ATLAS_PANDA"."SQL_QUEUE" ("TOPIC", "CREATIONTIME");
COMMENT ON TABLE "ATLAS_PANDA"."SQL_QUEUE" IS 'Queue to send messages between agents';
COMMENT ON COLUMN "ATLAS_PANDA"."SQL_QUEUE"."TOPIC" IS 'Topic of the message';
COMMENT ON COLUMN "ATLAS_PANDA"."SQL_QUEUE"."PANDAID" IS 'Job ID';
COMMENT ON COLUMN "ATLAS_PANDA"."SQL_QUEUE"."EXECUTION_ORDER" IS 'In case multiple SQLs need to be executed together';
COMMENT ON COLUMN "ATLAS_PANDA"."SQL_QUEUE"."JEDITASKID" IS 'JEDI Task ID in case the messages want to be batched';
COMMENT ON COLUMN "ATLAS_PANDA"."SQL_QUEUE"."CREATIONTIME" IS 'Timestamp when the message was created';
COMMENT ON COLUMN "ATLAS_PANDA"."SQL_QUEUE"."DATA" IS 'CLOB in JSON format containing the SQL query and variables';
-- Update versions
UPDATE "ATLAS_PANDA"."PANDADB_VERSION" SET major=0, minor=0, patch=16 where component='JEDI';
UPDATE "ATLAS_PANDA"."PANDADB_VERSION" SET major=0, minor=0, patch=16 where component='SERVER';
COMMIT;
28 Jun 2023 (DB version 0.0.15): add ERROR_DIAG column to JEDI_EVENTS table
-- AS ATLAS_PANDA
ALTER SESSION set DDL_LOCK_TIMEOUT = 30;
ALTER TABLE "ATLAS_PANDA"."JEDI_EVENTS" ADD ("ERROR_DIAG" VARCHAR2(500 BYTE));
DELETE FROM "ATLAS_PANDA"."PANDADB_VERSION" WHERE component='SCHEMA';
UPDATE "ATLAS_PANDA"."PANDADB_VERSION" SET major=0, minor=0, patch=15 where component='JEDI';
UPDATE "ATLAS_PANDA"."PANDADB_VERSION" SET major=0, minor=0, patch=15 where component='SERVER';
COMMIT;
28 Mar 2023 (DB version 0.0.14): add ERROR_DIAG column to job tables
-- AS ATLAS_PANDA
ALTER SESSION set DDL_LOCK_TIMEOUT = 30;
ALTER TABLE "ATLAS_PANDA"."JOBSDEFINED4" ADD ("CPU_ARCHITECTURE_LEVEL" VARCHAR2(20));
ALTER TABLE "ATLAS_PANDA"."JOBSWAITING4" ADD ("CPU_ARCHITECTURE_LEVEL" VARCHAR2(20));
ALTER TABLE "ATLAS_PANDA"."JOBSACTIVE4" ADD ("CPU_ARCHITECTURE_LEVEL" VARCHAR2(20));
ALTER TABLE "ATLAS_PANDA"."JOBSARCHIVED4" ADD ("CPU_ARCHITECTURE_LEVEL" VARCHAR2(20));
COMMENT ON COLUMN "ATLAS_PANDA"."JOBSDEFINED4"."CPU_ARCHITECTURE_LEVEL" IS 'Micro architecture level, e.g. x86-64-v1';
COMMENT ON COLUMN "ATLAS_PANDA"."JOBSWAITING4"."CPU_ARCHITECTURE_LEVEL" IS 'Micro architecture level, e.g. x86-64-v1';
COMMENT ON COLUMN "ATLAS_PANDA"."JOBSACTIVE4"."CPU_ARCHITECTURE_LEVEL" IS 'Micro architecture level, e.g. x86-64-v1';
COMMENT ON COLUMN "ATLAS_PANDA"."JOBSARCHIVED4"."CPU_ARCHITECTURE_LEVEL" IS 'Micro architecture level, e.g. x86-64-v1';
-- ADD THE COLUMNS TO THE BULKCOPY_PANDA_PARTITIONS JOB
UPDATE "ATLAS_PANDA"."PANDADB_VERSION" SET major=0, minor=0, patch=14 where component ='SCHEMA';
UPDATE "ATLAS_PANDA"."PANDADB_VERSION" SET major=0, minor=0, patch=14 where component ='JEDI';
UPDATE "ATLAS_PANDA"."PANDADB_VERSION" SET major=0, minor=0, patch=14 where component ='SERVER';
COMMIT;
-- As ATLAS_PANDAARCH
ALTER TABLE "ATLAS_PANDAARCH"."JOBSARCHIVED" ADD ("CPU_ARCHITECTURE_LEVEL" VARCHAR2(20));
10 Feb 2023 (DB version 0.0.13): add ATLAS_PANDA.CARBON_REGION_EMISSIONS table
-- AS ATLAS_PANDA
CREATE TABLE "ATLAS_PANDA"."CARBON_REGION_EMISSIONS"
(
"REGION" VARCHAR2(16 BYTE),
"TIMESTAMP" DATE,
"VALUE" NUMBER(10,0),
CONSTRAINT "CARBON_REGION_EMISSIONS_PK" PRIMARY KEY ("REGION", "TIMESTAMP") ENABLE
) ORGANIZATION INDEX COMPRESS 1;
COMMENT ON TABLE "ATLAS_PANDA"."CARBON_REGION_EMISSIONS" IS 'Table to store CO2 emission time series for a region or country';
COMMENT ON COLUMN "ATLAS_PANDA"."CARBON_REGION_EMISSIONS"."REGION" IS 'The name of the region. The region is usually a country, but can be more fine grained in large countries';
COMMENT ON COLUMN "ATLAS_PANDA"."CARBON_REGION_EMISSIONS"."TIMESTAMP" IS 'When the value was taken';
COMMENT ON COLUMN "ATLAS_PANDA"."CARBON_REGION_EMISSIONS"."VALUE" IS 'Emissions value in gCO2/kWh';
ALTER SESSION set DDL_LOCK_TIMEOUT = 30;
ALTER TABLE "ATLAS_PANDA"."JOBSDEFINED4" ADD ("GCO2_REGIONAL" NUMBER(10, 2), "GCO2_GLOBAL NUMBER"(10, 2));
ALTER TABLE "ATLAS_PANDA"."JOBSWAITING4" ADD ("GCO2_REGIONAL" NUMBER(10, 2), "GCO2_GLOBAL NUMBER"(10, 2));
ALTER TABLE "ATLAS_PANDA"."JOBSACTIVE4" ADD ("GCO2_REGIONAL" NUMBER(10, 2), "GCO2_GLOBAL NUMBER"(10, 2));
ALTER TABLE "ATLAS_PANDA"."JOBSARCHIVED4" ADD ("GCO2_REGIONAL" NUMBER(10, 2), "GCO2_GLOBAL NUMBER"(10, 2));
-- ALSO ADD THE COLUMNS TO BULKCOPY_PANDA_PARTITIONS
UPDATE "ATLAS_PANDA"."PANDADB_VERSION" SET major=0, minor=0, patch=13 where component ='SCHEMA';
UPDATE "ATLAS_PANDA"."PANDADB_VERSION" SET major=0, minor=0, patch=13 where component ='JEDI';
UPDATE "ATLAS_PANDA"."PANDADB_VERSION" SET major=0, minor=0, patch=13 where component ='SERVER';
COMMIT;
-- As ATLAS_PANDAARCH
ALTER TABLE ATLAS_PANDAARCH.JOBSARCHIVED ADD (GCO2_REGIONAL NUMBER(10, 2), GCO2_GLOBAL NUMBER(10, 2));
16 Sept 2022: add ATLAS_PANDA.TASK_EVALUATION table
CREATE TABLE "ATLAS_PANDA"."TASK_EVALUATION"
(
"JEDITASKID" NUMBER(11) NOT NULL,
"METRIC" VARCHAR2(128),
"VALUE_JSON" CLOB,
"TIMESTAMP" DATE,
CONSTRAINT ensure_json_task_evaluation CHECK (value_json IS JSON),
CONSTRAINT TASK_EVALUATION_PK PRIMARY KEY(jeditaskid, metric)
);
COMMENT ON TABLE "ATLAS_PANDA"."TASK_EVALUATION" IS 'Evaluation values for active user tasks used to improve analysis job brokerage';
COMMENT ON COLUMN "ATLAS_PANDA"."TASK_EVALUATION"."JEDITASKID" IS 'Task id';
COMMENT ON COLUMN "ATLAS_PANDA"."TASK_EVALUATION"."METRIC" IS 'Metric being stored';
COMMENT ON COLUMN "ATLAS_PANDA"."TASK_EVALUATION"."VALUE_JSON" IS 'Value in JSON format';
COMMENT ON COLUMN "ATLAS_PANDA"."TASK_EVALUATION"."TIMESTAMP" IS 'Timestamp when metric was generated';
20 Jul 2022: add JEDI_DATASETS.NFILESMISSNG
ALTER SESSION SET DDL_LOCK_TIMEOUT=600;
ALTER TABLE ATLAS_PANDA.JEDI_DATASETS ADD NFILESMISSING NUMBER(10);
COMMENT ON COLUMN "ATLAS_PANDA"."JEDI_DATASETS"."NFILESMISSING" IS 'The number of missing files for datasets';
commit;
15 Jun 2022: add validation for schedconfig json
ALTER TABLE ATLAS_PANDA.schedconfig_json
ADD CONSTRAINT schedconfig_data_validation check (data is JSON);
24 May 2022: add TASK_ATTEMPTS table and schedule clean up job to keep only last years data
CREATE TABLE "ATLAS_PANDA"."TASK_ATTEMPTS"
(
jeditaskid NUMBER(11),
attemptnr NUMBER(6),
starttime DATE,
endtime DATE,
startstatus VARCHAR2(32),
endstatus VARCHAR2(32),
CONSTRAINT TASK_ATTEMPTS_PK PRIMARY KEY (jeditaskid, attemptnr)
)
PARTITION BY RANGE (starttime) INTERVAL (NUMTODSINTERVAL(1, 'DAY'))
(PARTITION "DATA_BEFORE_20220524" VALUES LESS THAN (TIMESTAMP '2022-05-24 00:00:00') )
;
BEGIN
dbms_scheduler.create_job(
job_name => 'SL_WINDOW_TASK_ATTEMPTS_1YEAR',
job_type => 'PLSQL_BLOCK',
job_action => 'BEGIN ATLAS_PANDA.PANDA_TABLE_SL_WINDOW(''TASK_ATTEMPTS'',''STARTTIME'', 365); END; ',
start_date => SYSTIMESTAMP,
repeat_interval => 'FREQ=WEEKLY; BYDAY=TUE; BYHOUR=11; BYMINUTE=0; BYSECOND=0;',
auto_drop => FALSE,
enabled => TRUE,
comments => 'Sustains 365 days of data sliding window on the TASK_ATTEMPTS table! The table is partitioned daily using the automatic INTERVAL approach');
END;
26 Nov 2021: add floating point precision to MEMORY_LEAK_X2 columns
ALTER TABLE ATLAS_PANDAARCH.JOBSARCHIVED modify memory_leak_x2 NUMBER(14,3);
ALTER TABLE ATLAS_PANDA.JOBSDEFINED4 modify memory_leak_x2 NUMBER(14,3);
ALTER TABLE ATLAS_PANDA.JOBSWAITING4 modify memory_leak_x2 NUMBER(14,3);
ALTER TABLE ATLAS_PANDA.JOBSACTIVE4 modify memory_leak_x2 NUMBER(14,3);
ALTER TABLE ATLAS_PANDA.JOBSARCHIVED4 modify memory_leak_x2 NUMBER(14,3);
ALTER TABLE ATLAS_PANDA.JEDI_TASKS modify memory_leak_x2 NUMBER(14,3);
22 Nov 2021: creation of SECRETS table
CREATE TABLE "ATLAS_PANDA"."SECRETS"
( "OWNER" VARCHAR2(60) NOT NULL ENABLE,
"UPDATED_AT" TIMESTAMP (6) NOT NULL ENABLE,
"DATA" CLOB,
PRIMARY KEY ("OWNER")
);
9 Nov 2021: creation of METRICS table and adding new columns to HARVESTER_WORKERS
ALTER TABLE ATLAS_PANDA.HARVESTER_WORKERS ADD (pilotStartTime DATE,
pilotEndTime DATE,
pilotStatus VARCHAR2(80 BYTE),
pilotStatusSyncTime DATE);
CREATE TABLE "DOMA_PANDA"."METRICS"
(
computingsite VARCHAR2(128),
gshare VARCHAR2(32),
metric VARCHAR2(128),
value_json CLOB,
timestamp DATE,
CONSTRAINT ensure_json_metrics CHECK (value_json IS JSON),
CONSTRAINT METRICS_SITE_GSHARE_METRIC_UQ UNIQUE (computingsite, gshare, metric)
);