From d76d7a845922e9e65f00324c9b7576a19c6ac8f0 Mon Sep 17 00:00:00 2001 From: ViacheslavP Date: Tue, 10 Dec 2024 15:58:38 +0400 Subject: [PATCH 01/10] Fix healthchecks --- oracle/docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oracle/docker-compose.yaml b/oracle/docker-compose.yaml index e9788d6..4eb27ed 100644 --- a/oracle/docker-compose.yaml +++ b/oracle/docker-compose.yaml @@ -16,7 +16,7 @@ services: - ./sakila-schema.sql:/container-entrypoint-initdb.d/2.sql - ./oracle-sakila-insert-data.sql:/container-entrypoint-initdb.d/3.sql healthcheck: - test: "${ORACLE_BASE}/healthcheck.sh" + test: "/opt/oracle/healthcheck.sh" interval: "3s" timeout: "3s" start_period: "3s" @@ -34,7 +34,7 @@ services: volumes: - ./create_user.sql:/container-entrypoint-initdb.d/1.sql healthcheck: - test: "${ORACLE_BASE}/healthcheck.sh" + test: "/opt/oracle/healthcheck.sh" interval: "3s" timeout: "3s" start_period: "3s" From 9af3fa26f246c83316d58b12513cfd8301bbae87 Mon Sep 17 00:00:00 2001 From: ViacheslavP Date: Tue, 10 Dec 2024 22:36:32 +0400 Subject: [PATCH 02/10] Add user permission --- oracle/create_user.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/oracle/create_user.sql b/oracle/create_user.sql index 5d36bd1..d9c57dc 100644 --- a/oracle/create_user.sql +++ b/oracle/create_user.sql @@ -1,2 +1,3 @@ CREATE USER test IDENTIFIED BY test; GRANT ALL PRIVILEGES TO test; +GRANT SELECT ON V_$DATABASE TO test; \ No newline at end of file From 9b3724ed73d9850e30d3c2079ea5f20762386cb9 Mon Sep 17 00:00:00 2001 From: ViacheslavP Date: Tue, 10 Dec 2024 22:46:46 +0400 Subject: [PATCH 03/10] Create triggers after inserts --- oracle/create-triggers.sql | 306 +++++++++++++++++++++++++++++++++++++ oracle/docker-compose.yaml | 7 +- oracle/sakila-schema.sql | 263 ------------------------------- 3 files changed, 310 insertions(+), 266 deletions(-) create mode 100644 oracle/create-triggers.sql diff --git a/oracle/create-triggers.sql b/oracle/create-triggers.sql new file mode 100644 index 0000000..e818942 --- /dev/null +++ b/oracle/create-triggers.sql @@ -0,0 +1,306 @@ +conn test/test +-- +-- TABLE ACTOR +-- +CREATE OR REPLACE TRIGGER actor_before_trigger +BEFORE INSERT ON actor FOR EACH ROW +BEGIN + IF (:NEW.actor_id IS NULL) THEN + SELECT actor_sequence.nextval INTO :NEW.actor_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER actor_before_update +BEFORE UPDATE ON actor FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +-- +-- TABLE COUNTRY +-- +CREATE OR REPLACE TRIGGER country_before_trigger +BEFORE INSERT ON country FOR EACH ROW +BEGIN + IF (:NEW.country_id IS NULL) THEN + SELECT country_sequence.nextval INTO :NEW.country_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER country_before_update +BEFORE UPDATE ON country FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +-- +-- TABLE CITY +-- +CREATE OR REPLACE TRIGGER city_before_trigger +BEFORE INSERT ON city FOR EACH ROW +BEGIN + IF (:NEW.city_id IS NULL) THEN + SELECT city_sequence.nextval INTO :NEW.city_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER city_before_update +BEFORE UPDATE ON city FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +-- +-- TABLE ADDRESS +-- +CREATE OR REPLACE TRIGGER address_before_trigger +BEFORE INSERT ON address FOR EACH ROW +BEGIN + IF (:NEW.address_id IS NULL) THEN + SELECT address_sequence.nextval INTO :NEW.address_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER address_before_update +BEFORE UPDATE ON address FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +-- +-- TABLE LANGUAGE +-- +CREATE OR REPLACE TRIGGER language_before_trigger +BEFORE INSERT ON language FOR EACH ROW +BEGIN + IF (:NEW.language_id IS NULL) THEN + SELECT language_sequence.nextval INTO :NEW.language_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER language_before_update +BEFORE UPDATE ON language FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +-- +-- TABLE CATEGORY +-- +CREATE OR REPLACE TRIGGER category_before_trigger +BEFORE INSERT ON category FOR EACH ROW +BEGIN + IF (:NEW.category_id IS NULL) THEN + SELECT category_sequence.nextval INTO :NEW.category_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER category_before_update +BEFORE UPDATE ON category FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +-- +-- TABLE CUSTOMER +-- +CREATE OR REPLACE TRIGGER customer_before_trigger +BEFORE INSERT ON customer FOR EACH ROW +BEGIN + IF (:NEW.customer_id IS NULL) THEN + SELECT customer_sequence.nextval INTO :NEW.customer_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; + :NEW.create_date:=current_date; +END; +/ +CREATE OR REPLACE TRIGGER customer_before_update +BEFORE UPDATE ON customer FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +-- +-- TABLE FILM +-- +CREATE OR REPLACE TRIGGER film_before_trigger +BEFORE INSERT ON film FOR EACH ROW +BEGIN + IF (:NEW.film_id IS NULL) THEN + SELECT film_sequence.nextval INTO :NEW.film_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER film_before_update +BEFORE UPDATE ON film FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +-- +-- TABLE FILM_ACTOR +-- +CREATE OR REPLACE TRIGGER film_actor_before_trigger +BEFORE INSERT ON film_actor FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER film_actor_before_update +BEFORE UPDATE ON film_actor FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +-- +-- TABLE FILM_CATEGORY +-- +CREATE OR REPLACE TRIGGER film_category_before_trigger +BEFORE INSERT ON film_category FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER film_category_before_update +BEFORE UPDATE ON film_category FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +-- +-- TABLE INVENTORY +-- +CREATE OR REPLACE TRIGGER inventory_before_trigger +BEFORE INSERT ON inventory FOR EACH ROW +BEGIN + IF (:NEW.inventory_id IS NULL) THEN + SELECT inventory_sequence.nextval INTO :NEW.inventory_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ +CREATE OR REPLACE TRIGGER inventory_before_update +BEFORE UPDATE ON inventory FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +-- +-- TABLE STAFF +-- +CREATE OR REPLACE TRIGGER staff_before_trigger +BEFORE INSERT ON staff FOR EACH ROW +BEGIN + IF (:NEW.staff_id IS NULL) THEN + SELECT staff_sequence.nextval INTO :NEW.staff_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER staff_before_update +BEFORE UPDATE ON staff FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +-- +-- TABLE STORE +-- +CREATE OR REPLACE TRIGGER store_before_trigger +BEFORE INSERT ON store FOR EACH ROW +BEGIN + IF (:NEW.store_id IS NULL) THEN + SELECT store_sequence.nextval INTO :NEW.store_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER store_before_update +BEFORE UPDATE ON store FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +-- +-- TABLE PAYMENT +-- +CREATE OR REPLACE TRIGGER payment_before_trigger +BEFORE INSERT ON payment FOR EACH ROW +BEGIN + IF (:NEW.payment_id IS NULL) THEN + SELECT payment_sequence.nextval INTO :NEW.payment_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER payment_before_update +BEFORE UPDATE ON payment FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +-- +-- TABLE RENTAL +-- +CREATE OR REPLACE TRIGGER rental_before_trigger +BEFORE INSERT ON rental FOR EACH ROW +BEGIN + IF (:NEW.rental_id IS NULL) THEN + SELECT rental_sequence.nextval INTO :NEW.rental_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER rental_before_update +BEFORE UPDATE ON rental FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ \ No newline at end of file diff --git a/oracle/docker-compose.yaml b/oracle/docker-compose.yaml index 4eb27ed..63f0888 100644 --- a/oracle/docker-compose.yaml +++ b/oracle/docker-compose.yaml @@ -12,9 +12,10 @@ services: environment: ORACLE_PASSWORD: ${ORACLE_USER:-secret_password} volumes: - - ./create_user.sql:/container-entrypoint-initdb.d/1.sql - - ./sakila-schema.sql:/container-entrypoint-initdb.d/2.sql - - ./oracle-sakila-insert-data.sql:/container-entrypoint-initdb.d/3.sql + - ./create_user.sql:/container-entrypoint-initdb.d/01_create_user.sql + - ./sakila-schema.sql:/container-entrypoint-initdb.d/02_sakila-schema.sql + - ./oracle-sakila-insert-data.sql:/container-entrypoint-initdb.d/03_oracle-sakila-insert-data.sql + - ./create-triggers.sql:/container-entrypoint-initdb.d/04_post-insert.sql healthcheck: test: "/opt/oracle/healthcheck.sh" interval: "3s" diff --git a/oracle/sakila-schema.sql b/oracle/sakila-schema.sql index ebd55fc..5d59ea2 100644 --- a/oracle/sakila-schema.sql +++ b/oracle/sakila-schema.sql @@ -34,27 +34,6 @@ CREATE INDEX idx_actor_last_name ON actor(last_name); CREATE SEQUENCE actor_sequence; / - - -CREATE OR REPLACE TRIGGER actor_before_trigger -BEFORE INSERT ON actor FOR EACH ROW -BEGIN - IF (:NEW.actor_id IS NULL) THEN - SELECT actor_sequence.nextval INTO :NEW.actor_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER actor_before_update -BEFORE UPDATE ON actor FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - - -- -- Table structure for table country -- @@ -71,26 +50,6 @@ CREATE TABLE country ( CREATE SEQUENCE country_sequence; / - -CREATE OR REPLACE TRIGGER country_before_trigger -BEFORE INSERT ON country FOR EACH ROW -BEGIN - IF (:NEW.country_id IS NULL) THEN - SELECT country_sequence.nextval INTO :NEW.country_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER country_before_update -BEFORE UPDATE ON country FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - - -- -- Table structure for table city -- @@ -112,25 +71,6 @@ CREATE INDEX idx_fk_country_id ON city(country_id); CREATE SEQUENCE city_sequence; / -CREATE OR REPLACE TRIGGER city_before_trigger -BEFORE INSERT ON city FOR EACH ROW -BEGIN - IF (:NEW.city_id IS NULL) THEN - SELECT city_sequence.nextval INTO :NEW.city_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER city_before_update -BEFORE UPDATE ON city FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - - -- -- Table structure for table address -- @@ -158,24 +98,6 @@ ALTER TABLE address ADD CONSTRAINT fk_address_city FOREIGN KEY (city_id) REFERE CREATE SEQUENCE address_sequence; / -CREATE OR REPLACE TRIGGER address_before_trigger -BEFORE INSERT ON address FOR EACH ROW -BEGIN - IF (:NEW.address_id IS NULL) THEN - SELECT address_sequence.nextval INTO :NEW.address_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER address_before_update -BEFORE UPDATE ON address FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - -- -- Table structure for table language -- @@ -192,24 +114,6 @@ CREATE TABLE language ( CREATE SEQUENCE language_sequence; / -CREATE OR REPLACE TRIGGER language_before_trigger -BEFORE INSERT ON language FOR EACH ROW -BEGIN - IF (:NEW.language_id IS NULL) THEN - SELECT language_sequence.nextval INTO :NEW.language_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER language_before_update -BEFORE UPDATE ON language FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - -- -- Table structure for table category -- @@ -226,24 +130,6 @@ CREATE TABLE category ( CREATE SEQUENCE category_sequence; / -CREATE OR REPLACE TRIGGER category_before_trigger -BEFORE INSERT ON category FOR EACH ROW -BEGIN - IF (:NEW.category_id IS NULL) THEN - SELECT category_sequence.nextval INTO :NEW.category_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER category_before_update -BEFORE UPDATE ON category FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - -- -- Table structure for table customer -- @@ -273,23 +159,6 @@ CREATE INDEX idx_customer_last_name ON customer(last_name); CREATE SEQUENCE customer_sequence; / -CREATE OR REPLACE TRIGGER customer_before_trigger -BEFORE INSERT ON customer FOR EACH ROW -BEGIN - IF (:NEW.customer_id IS NULL) THEN - SELECT customer_sequence.nextval INTO :NEW.customer_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; - :NEW.create_date:=current_date; -END; -/ -CREATE OR REPLACE TRIGGER customer_before_update -BEFORE UPDATE ON customer FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ -- -- Table structure for table film -- @@ -331,24 +200,6 @@ CREATE INDEX idx_fk_original_language_id ON film(original_language_id); CREATE SEQUENCE film_sequence; / -CREATE OR REPLACE TRIGGER film_before_trigger -BEFORE INSERT ON film FOR EACH ROW -BEGIN - IF (:NEW.film_id IS NULL) THEN - SELECT film_sequence.nextval INTO :NEW.film_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER film_before_update -BEFORE UPDATE ON film FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - -- -- Table structure for table film_actor -- @@ -368,20 +219,6 @@ CREATE INDEX idx_fk_film_actor_film ON film_actor(film_id); CREATE INDEX idx_fk_film_actor_actor ON film_actor(actor_id) ; / -CREATE OR REPLACE TRIGGER film_actor_before_trigger -BEFORE INSERT ON film_actor FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER film_actor_before_update -BEFORE UPDATE ON film_actor FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - -- -- Table structure for table film_category -- @@ -400,19 +237,6 @@ CREATE INDEX idx_fk_film_category_film ON film_category(film_id); CREATE INDEX idx_fk_film_category_category ON film_category(category_id); / -CREATE OR REPLACE TRIGGER film_category_before_trigger -BEFORE INSERT ON film_category FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER film_category_before_update -BEFORE UPDATE ON film_category FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ -- -- Table structure for table film_text -- @@ -448,23 +272,6 @@ CREATE INDEX idx_fk_film_id_store_id ON inventory(store_id,film_id); CREATE SEQUENCE inventory_sequence; / -CREATE OR REPLACE TRIGGER inventory_before_trigger -BEFORE INSERT ON inventory FOR EACH ROW -BEGIN - IF (:NEW.inventory_id IS NULL) THEN - SELECT inventory_sequence.nextval INTO :NEW.inventory_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; -END; -/ -CREATE OR REPLACE TRIGGER inventory_before_update -BEFORE UPDATE ON inventory FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - -- -- Table structure for table staff -- @@ -496,23 +303,6 @@ CREATE INDEX idx_fk_staff_address_id ON staff(address_id); CREATE SEQUENCE staff_sequence; / -CREATE OR REPLACE TRIGGER staff_before_trigger -BEFORE INSERT ON staff FOR EACH ROW -BEGIN - IF (:NEW.staff_id IS NULL) THEN - SELECT staff_sequence.nextval INTO :NEW.staff_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER staff_before_update -BEFORE UPDATE ON staff FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ -- -- Table structure for table store @@ -540,25 +330,6 @@ CREATE SEQUENCE store_sequence; / - -CREATE OR REPLACE TRIGGER store_before_trigger -BEFORE INSERT ON store FOR EACH ROW -BEGIN - IF (:NEW.store_id IS NULL) THEN - SELECT store_sequence.nextval INTO :NEW.store_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER store_before_update -BEFORE UPDATE ON store FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - -- -- Table structure for table payment -- @@ -586,23 +357,6 @@ CREATE INDEX idx_fk_customer_id ON payment(customer_id); CREATE SEQUENCE payment_sequence; / -CREATE OR REPLACE TRIGGER payment_before_trigger -BEFORE INSERT ON payment FOR EACH ROW -BEGIN - IF (:NEW.payment_id IS NULL) THEN - SELECT payment_sequence.nextval INTO :NEW.payment_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER payment_before_update -BEFORE UPDATE ON payment FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ CREATE TABLE rental ( rental_id INT NOT NULL, @@ -632,23 +386,6 @@ CREATE UNIQUE INDEX idx_rental_uq ON rental (rental_date,inventory_id,custome CREATE SEQUENCE rental_sequence; / -CREATE OR REPLACE TRIGGER rental_before_trigger -BEFORE INSERT ON rental FOR EACH ROW -BEGIN - IF (:NEW.rental_id IS NULL) THEN - SELECT rental_sequence.nextval INTO :NEW.rental_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER rental_before_update -BEFORE UPDATE ON rental FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ -- FK CONSTRAINTS ALTER TABLE customer ADD CONSTRAINT fk_customer_store FOREIGN KEY (store_id) REFERENCES store (store_id); From ecdf2409548641a8ca1c61cec22a148fe0b8158c Mon Sep 17 00:00:00 2001 From: ViacheslavP Date: Wed, 11 Dec 2024 12:13:45 +0400 Subject: [PATCH 04/10] Revert "Create triggers after inserts" This reverts commit 9b3724ed73d9850e30d3c2079ea5f20762386cb9. --- oracle/create-triggers.sql | 306 ------------------------------------- oracle/docker-compose.yaml | 7 +- oracle/sakila-schema.sql | 263 +++++++++++++++++++++++++++++++ 3 files changed, 266 insertions(+), 310 deletions(-) delete mode 100644 oracle/create-triggers.sql diff --git a/oracle/create-triggers.sql b/oracle/create-triggers.sql deleted file mode 100644 index e818942..0000000 --- a/oracle/create-triggers.sql +++ /dev/null @@ -1,306 +0,0 @@ -conn test/test --- --- TABLE ACTOR --- -CREATE OR REPLACE TRIGGER actor_before_trigger -BEFORE INSERT ON actor FOR EACH ROW -BEGIN - IF (:NEW.actor_id IS NULL) THEN - SELECT actor_sequence.nextval INTO :NEW.actor_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER actor_before_update -BEFORE UPDATE ON actor FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - --- --- TABLE COUNTRY --- -CREATE OR REPLACE TRIGGER country_before_trigger -BEFORE INSERT ON country FOR EACH ROW -BEGIN - IF (:NEW.country_id IS NULL) THEN - SELECT country_sequence.nextval INTO :NEW.country_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER country_before_update -BEFORE UPDATE ON country FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - --- --- TABLE CITY --- -CREATE OR REPLACE TRIGGER city_before_trigger -BEFORE INSERT ON city FOR EACH ROW -BEGIN - IF (:NEW.city_id IS NULL) THEN - SELECT city_sequence.nextval INTO :NEW.city_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER city_before_update -BEFORE UPDATE ON city FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - --- --- TABLE ADDRESS --- -CREATE OR REPLACE TRIGGER address_before_trigger -BEFORE INSERT ON address FOR EACH ROW -BEGIN - IF (:NEW.address_id IS NULL) THEN - SELECT address_sequence.nextval INTO :NEW.address_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER address_before_update -BEFORE UPDATE ON address FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - --- --- TABLE LANGUAGE --- -CREATE OR REPLACE TRIGGER language_before_trigger -BEFORE INSERT ON language FOR EACH ROW -BEGIN - IF (:NEW.language_id IS NULL) THEN - SELECT language_sequence.nextval INTO :NEW.language_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER language_before_update -BEFORE UPDATE ON language FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - --- --- TABLE CATEGORY --- -CREATE OR REPLACE TRIGGER category_before_trigger -BEFORE INSERT ON category FOR EACH ROW -BEGIN - IF (:NEW.category_id IS NULL) THEN - SELECT category_sequence.nextval INTO :NEW.category_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER category_before_update -BEFORE UPDATE ON category FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - --- --- TABLE CUSTOMER --- -CREATE OR REPLACE TRIGGER customer_before_trigger -BEFORE INSERT ON customer FOR EACH ROW -BEGIN - IF (:NEW.customer_id IS NULL) THEN - SELECT customer_sequence.nextval INTO :NEW.customer_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; - :NEW.create_date:=current_date; -END; -/ -CREATE OR REPLACE TRIGGER customer_before_update -BEFORE UPDATE ON customer FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - --- --- TABLE FILM --- -CREATE OR REPLACE TRIGGER film_before_trigger -BEFORE INSERT ON film FOR EACH ROW -BEGIN - IF (:NEW.film_id IS NULL) THEN - SELECT film_sequence.nextval INTO :NEW.film_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER film_before_update -BEFORE UPDATE ON film FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - --- --- TABLE FILM_ACTOR --- -CREATE OR REPLACE TRIGGER film_actor_before_trigger -BEFORE INSERT ON film_actor FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER film_actor_before_update -BEFORE UPDATE ON film_actor FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - --- --- TABLE FILM_CATEGORY --- -CREATE OR REPLACE TRIGGER film_category_before_trigger -BEFORE INSERT ON film_category FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER film_category_before_update -BEFORE UPDATE ON film_category FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - --- --- TABLE INVENTORY --- -CREATE OR REPLACE TRIGGER inventory_before_trigger -BEFORE INSERT ON inventory FOR EACH ROW -BEGIN - IF (:NEW.inventory_id IS NULL) THEN - SELECT inventory_sequence.nextval INTO :NEW.inventory_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; -END; -/ -CREATE OR REPLACE TRIGGER inventory_before_update -BEFORE UPDATE ON inventory FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - --- --- TABLE STAFF --- -CREATE OR REPLACE TRIGGER staff_before_trigger -BEFORE INSERT ON staff FOR EACH ROW -BEGIN - IF (:NEW.staff_id IS NULL) THEN - SELECT staff_sequence.nextval INTO :NEW.staff_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER staff_before_update -BEFORE UPDATE ON staff FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - --- --- TABLE STORE --- -CREATE OR REPLACE TRIGGER store_before_trigger -BEFORE INSERT ON store FOR EACH ROW -BEGIN - IF (:NEW.store_id IS NULL) THEN - SELECT store_sequence.nextval INTO :NEW.store_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER store_before_update -BEFORE UPDATE ON store FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - --- --- TABLE PAYMENT --- -CREATE OR REPLACE TRIGGER payment_before_trigger -BEFORE INSERT ON payment FOR EACH ROW -BEGIN - IF (:NEW.payment_id IS NULL) THEN - SELECT payment_sequence.nextval INTO :NEW.payment_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER payment_before_update -BEFORE UPDATE ON payment FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ - --- --- TABLE RENTAL --- -CREATE OR REPLACE TRIGGER rental_before_trigger -BEFORE INSERT ON rental FOR EACH ROW -BEGIN - IF (:NEW.rental_id IS NULL) THEN - SELECT rental_sequence.nextval INTO :NEW.rental_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER rental_before_update -BEFORE UPDATE ON rental FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ \ No newline at end of file diff --git a/oracle/docker-compose.yaml b/oracle/docker-compose.yaml index 63f0888..4eb27ed 100644 --- a/oracle/docker-compose.yaml +++ b/oracle/docker-compose.yaml @@ -12,10 +12,9 @@ services: environment: ORACLE_PASSWORD: ${ORACLE_USER:-secret_password} volumes: - - ./create_user.sql:/container-entrypoint-initdb.d/01_create_user.sql - - ./sakila-schema.sql:/container-entrypoint-initdb.d/02_sakila-schema.sql - - ./oracle-sakila-insert-data.sql:/container-entrypoint-initdb.d/03_oracle-sakila-insert-data.sql - - ./create-triggers.sql:/container-entrypoint-initdb.d/04_post-insert.sql + - ./create_user.sql:/container-entrypoint-initdb.d/1.sql + - ./sakila-schema.sql:/container-entrypoint-initdb.d/2.sql + - ./oracle-sakila-insert-data.sql:/container-entrypoint-initdb.d/3.sql healthcheck: test: "/opt/oracle/healthcheck.sh" interval: "3s" diff --git a/oracle/sakila-schema.sql b/oracle/sakila-schema.sql index 5d59ea2..ebd55fc 100644 --- a/oracle/sakila-schema.sql +++ b/oracle/sakila-schema.sql @@ -34,6 +34,27 @@ CREATE INDEX idx_actor_last_name ON actor(last_name); CREATE SEQUENCE actor_sequence; / + + +CREATE OR REPLACE TRIGGER actor_before_trigger +BEFORE INSERT ON actor FOR EACH ROW +BEGIN + IF (:NEW.actor_id IS NULL) THEN + SELECT actor_sequence.nextval INTO :NEW.actor_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER actor_before_update +BEFORE UPDATE ON actor FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + + -- -- Table structure for table country -- @@ -50,6 +71,26 @@ CREATE TABLE country ( CREATE SEQUENCE country_sequence; / + +CREATE OR REPLACE TRIGGER country_before_trigger +BEFORE INSERT ON country FOR EACH ROW +BEGIN + IF (:NEW.country_id IS NULL) THEN + SELECT country_sequence.nextval INTO :NEW.country_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER country_before_update +BEFORE UPDATE ON country FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + + -- -- Table structure for table city -- @@ -71,6 +112,25 @@ CREATE INDEX idx_fk_country_id ON city(country_id); CREATE SEQUENCE city_sequence; / +CREATE OR REPLACE TRIGGER city_before_trigger +BEFORE INSERT ON city FOR EACH ROW +BEGIN + IF (:NEW.city_id IS NULL) THEN + SELECT city_sequence.nextval INTO :NEW.city_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER city_before_update +BEFORE UPDATE ON city FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + + -- -- Table structure for table address -- @@ -98,6 +158,24 @@ ALTER TABLE address ADD CONSTRAINT fk_address_city FOREIGN KEY (city_id) REFERE CREATE SEQUENCE address_sequence; / +CREATE OR REPLACE TRIGGER address_before_trigger +BEFORE INSERT ON address FOR EACH ROW +BEGIN + IF (:NEW.address_id IS NULL) THEN + SELECT address_sequence.nextval INTO :NEW.address_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER address_before_update +BEFORE UPDATE ON address FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + -- -- Table structure for table language -- @@ -114,6 +192,24 @@ CREATE TABLE language ( CREATE SEQUENCE language_sequence; / +CREATE OR REPLACE TRIGGER language_before_trigger +BEFORE INSERT ON language FOR EACH ROW +BEGIN + IF (:NEW.language_id IS NULL) THEN + SELECT language_sequence.nextval INTO :NEW.language_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER language_before_update +BEFORE UPDATE ON language FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + -- -- Table structure for table category -- @@ -130,6 +226,24 @@ CREATE TABLE category ( CREATE SEQUENCE category_sequence; / +CREATE OR REPLACE TRIGGER category_before_trigger +BEFORE INSERT ON category FOR EACH ROW +BEGIN + IF (:NEW.category_id IS NULL) THEN + SELECT category_sequence.nextval INTO :NEW.category_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER category_before_update +BEFORE UPDATE ON category FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + -- -- Table structure for table customer -- @@ -159,6 +273,23 @@ CREATE INDEX idx_customer_last_name ON customer(last_name); CREATE SEQUENCE customer_sequence; / +CREATE OR REPLACE TRIGGER customer_before_trigger +BEFORE INSERT ON customer FOR EACH ROW +BEGIN + IF (:NEW.customer_id IS NULL) THEN + SELECT customer_sequence.nextval INTO :NEW.customer_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; + :NEW.create_date:=current_date; +END; +/ +CREATE OR REPLACE TRIGGER customer_before_update +BEFORE UPDATE ON customer FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ -- -- Table structure for table film -- @@ -200,6 +331,24 @@ CREATE INDEX idx_fk_original_language_id ON film(original_language_id); CREATE SEQUENCE film_sequence; / +CREATE OR REPLACE TRIGGER film_before_trigger +BEFORE INSERT ON film FOR EACH ROW +BEGIN + IF (:NEW.film_id IS NULL) THEN + SELECT film_sequence.nextval INTO :NEW.film_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER film_before_update +BEFORE UPDATE ON film FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + -- -- Table structure for table film_actor -- @@ -219,6 +368,20 @@ CREATE INDEX idx_fk_film_actor_film ON film_actor(film_id); CREATE INDEX idx_fk_film_actor_actor ON film_actor(actor_id) ; / +CREATE OR REPLACE TRIGGER film_actor_before_trigger +BEFORE INSERT ON film_actor FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER film_actor_before_update +BEFORE UPDATE ON film_actor FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + -- -- Table structure for table film_category -- @@ -237,6 +400,19 @@ CREATE INDEX idx_fk_film_category_film ON film_category(film_id); CREATE INDEX idx_fk_film_category_category ON film_category(category_id); / +CREATE OR REPLACE TRIGGER film_category_before_trigger +BEFORE INSERT ON film_category FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER film_category_before_update +BEFORE UPDATE ON film_category FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ -- -- Table structure for table film_text -- @@ -272,6 +448,23 @@ CREATE INDEX idx_fk_film_id_store_id ON inventory(store_id,film_id); CREATE SEQUENCE inventory_sequence; / +CREATE OR REPLACE TRIGGER inventory_before_trigger +BEFORE INSERT ON inventory FOR EACH ROW +BEGIN + IF (:NEW.inventory_id IS NULL) THEN + SELECT inventory_sequence.nextval INTO :NEW.inventory_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ +CREATE OR REPLACE TRIGGER inventory_before_update +BEFORE UPDATE ON inventory FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + -- -- Table structure for table staff -- @@ -303,6 +496,23 @@ CREATE INDEX idx_fk_staff_address_id ON staff(address_id); CREATE SEQUENCE staff_sequence; / +CREATE OR REPLACE TRIGGER staff_before_trigger +BEFORE INSERT ON staff FOR EACH ROW +BEGIN + IF (:NEW.staff_id IS NULL) THEN + SELECT staff_sequence.nextval INTO :NEW.staff_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER staff_before_update +BEFORE UPDATE ON staff FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ -- -- Table structure for table store @@ -330,6 +540,25 @@ CREATE SEQUENCE store_sequence; / + +CREATE OR REPLACE TRIGGER store_before_trigger +BEFORE INSERT ON store FOR EACH ROW +BEGIN + IF (:NEW.store_id IS NULL) THEN + SELECT store_sequence.nextval INTO :NEW.store_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER store_before_update +BEFORE UPDATE ON store FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + -- -- Table structure for table payment -- @@ -357,6 +586,23 @@ CREATE INDEX idx_fk_customer_id ON payment(customer_id); CREATE SEQUENCE payment_sequence; / +CREATE OR REPLACE TRIGGER payment_before_trigger +BEFORE INSERT ON payment FOR EACH ROW +BEGIN + IF (:NEW.payment_id IS NULL) THEN + SELECT payment_sequence.nextval INTO :NEW.payment_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER payment_before_update +BEFORE UPDATE ON payment FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ CREATE TABLE rental ( rental_id INT NOT NULL, @@ -386,6 +632,23 @@ CREATE UNIQUE INDEX idx_rental_uq ON rental (rental_date,inventory_id,custome CREATE SEQUENCE rental_sequence; / +CREATE OR REPLACE TRIGGER rental_before_trigger +BEFORE INSERT ON rental FOR EACH ROW +BEGIN + IF (:NEW.rental_id IS NULL) THEN + SELECT rental_sequence.nextval INTO :NEW.rental_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER rental_before_update +BEFORE UPDATE ON rental FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ -- FK CONSTRAINTS ALTER TABLE customer ADD CONSTRAINT fk_customer_store FOREIGN KEY (store_id) REFERENCES store (store_id); From eeed2456d09e90fcff75029f236547f91eb82516 Mon Sep 17 00:00:00 2001 From: ViacheslavP Date: Wed, 11 Dec 2024 12:42:15 +0400 Subject: [PATCH 05/10] Remove fixes --- oracle/config.tdk.yaml | 109 ----------------------------------------- 1 file changed, 109 deletions(-) diff --git a/oracle/config.tdk.yaml b/oracle/config.tdk.yaml index 66f464f..f714cac 100644 --- a/oracle/config.tdk.yaml +++ b/oracle/config.tdk.yaml @@ -1,115 +1,6 @@ default_config: mode: MASKING -tables: - - table_name_with_schema: "TEST.CITY" - transformations: - - columns: [ "LAST_UPDATE" ] - params: - type: "date_generator" - mean: 2022-03-01T12:00:00Z - std: 7776000000 - min: 2022-01-01T12:00:00Z - max: 2022-07-01T12:00:00Z - - - table_name_with_schema: "TEST.CUSTOMER" - transformations: - - columns: [ "CREATE_DATE" ] - params: - type: "date_generator" - mean: 2022-03-01T12:00:00Z - std: 7776000000 - min: 2022-01-01T12:00:00Z - max: 2022-07-01T12:00:00Z - - columns: [ "LAST_UPDATE" ] - params: - type: "date_generator" - mean: 2022-03-01T12:00:00Z - std: 7776000000 - min: 2022-01-01T12:00:00Z - max: 2022-07-01T12:00:00Z - - - table_name_with_schema: "TEST.FILM" - transformations: - - columns: [ "LAST_UPDATE" ] - params: - type: "date_generator" - mean: 2022-03-01T12:00:00Z - std: 7776000000 - min: 2022-01-01T12:00:00Z - max: 2022-07-01T12:00:00Z - - - table_name_with_schema: "TEST.FILM_ACTOR" - transformations: - - columns: [ "LAST_UPDATE" ] - params: - type: "date_generator" - mean: 2022-03-01T12:00:00Z - std: 7776000000 - min: 2022-01-01T12:00:00Z - max: 2022-07-01T12:00:00Z - - - table_name_with_schema: "TEST.ACTOR" - transformations: - - columns: [ "LAST_UPDATE" ] - params: - type: "date_generator" - mean: 2022-03-01T12:00:00Z - std: 7776000000 - min: 2022-01-01T12:00:00Z - max: 2022-07-01T12:00:00Z - - - table_name_with_schema: "TEST.FILM_CATEGORY" - transformations: - - columns: [ "LAST_UPDATE" ] - params: - type: "date_generator" - mean: 2022-03-01T12:00:00Z - std: 7776000000 - min: 2022-01-01T12:00:00Z - max: 2022-07-01T12:00:00Z - - - table_name_with_schema: "TEST.INVENTORY" - transformations: - - columns: [ "LAST_UPDATE" ] - params: - type: "date_generator" - mean: 2022-03-01T12:00:00Z - std: 7776000000 - min: 2022-01-01T12:00:00Z - max: 2022-07-01T12:00:00Z - - - table_name_with_schema: "TEST.PAYMENT" - transformations: - - columns: [ "LAST_UPDATE" ] - params: - type: "date_generator" - mean: 2022-03-01T12:00:00Z - std: 7776000000 - min: 2022-01-01T12:00:00Z - max: 2022-07-01T12:00:00Z - - - table_name_with_schema: "TEST.RENTAL" - transformations: - - columns: [ "LAST_UPDATE" ] - params: - type: "date_generator" - mean: 2022-03-01T12:00:00Z - std: 7776000000 - min: 2022-01-01T12:00:00Z - max: 2022-07-01T12:00:00Z - - - table_name_with_schema: "TEST.ADDRESS" - transformations: - - columns: [ "LAST_UPDATE" ] - params: - type: "date_generator" - mean: 2022-03-01T12:00:00Z - std: 7776000000 - min: 2022-01-01T12:00:00Z - max: 2022-07-01T12:00:00Z - - cycle_resolution_strategy: DEFER_FOREIGN_KEY table_truncation_mode: TRUNCATE schema_creation_mode: CREATE_IF_NOT_EXISTS From e1fb4d996a98100e2d853550381cf621b8972498 Mon Sep 17 00:00:00 2001 From: ViacheslavP Date: Wed, 11 Dec 2024 14:39:26 +0400 Subject: [PATCH 06/10] Add sodacheck and healthcheck --- oracle/create_flag.sh | 3 ++ oracle/docker-compose.yaml | 40 ++++++++++------ oracle/soda/checks_for_masking.yaml | 47 +++++++++++++++++++ oracle/soda/configuration.yaml | 11 +++++ .../soda/masked_film_rating_distribution.yml | 16 +++++++ 5 files changed, 102 insertions(+), 15 deletions(-) create mode 100644 oracle/create_flag.sh create mode 100644 oracle/soda/checks_for_masking.yaml create mode 100644 oracle/soda/configuration.yaml create mode 100644 oracle/soda/masked_film_rating_distribution.yml diff --git a/oracle/create_flag.sh b/oracle/create_flag.sh new file mode 100644 index 0000000..fb6cdb3 --- /dev/null +++ b/oracle/create_flag.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +touch /opt/oracle/flag \ No newline at end of file diff --git a/oracle/docker-compose.yaml b/oracle/docker-compose.yaml index 4eb27ed..ad0c9ba 100644 --- a/oracle/docker-compose.yaml +++ b/oracle/docker-compose.yaml @@ -2,9 +2,9 @@ version: '3' services: - oracle_source: + input_db: image: gvenzl/oracle-xe - container_name: oracle_source + container_name: input_db networks: - simple-network ports: @@ -15,16 +15,17 @@ services: - ./create_user.sql:/container-entrypoint-initdb.d/1.sql - ./sakila-schema.sql:/container-entrypoint-initdb.d/2.sql - ./oracle-sakila-insert-data.sql:/container-entrypoint-initdb.d/3.sql + - ./create_flag.sh:/container-entrypoint-initdb.d/4.sh healthcheck: - test: "/opt/oracle/healthcheck.sh" - interval: "3s" - timeout: "3s" - start_period: "3s" - retries: 30 + test: ["CMD", "sh", "-c", "if [ -e /opt/oracle/flag ]; then exit 0; else exit 1; fi"] + interval: "6s" + timeout: "6s" + start_period: "6s" + retries: 60 - oracle_target: + output_db: image: gvenzl/oracle-xe - container_name: oracle_target + container_name: output_db networks: - simple-network ports: @@ -42,14 +43,14 @@ services: databases: container_name: databases - image: docker/whalesay + image: synthesizedio/whalesay command: > cowsay "Both databases have been started and are ready for TDK exercises!" depends_on: - oracle_source: + input_db: condition: service_healthy - oracle_target: + output_db: condition: service_healthy @@ -60,10 +61,10 @@ services: networks: - simple-network environment: - SYNTHESIZED_INPUT_URL: jdbc:oracle:thin:@oracle_source:1521:xe + SYNTHESIZED_INPUT_URL: jdbc:oracle:thin:@input_db:1521:xe SYNTHESIZED_INPUT_USERNAME: test SYNTHESIZED_INPUT_PASSWORD: test - SYNTHESIZED_OUTPUT_URL: jdbc:oracle:thin:@oracle_target:1521:xe + SYNTHESIZED_OUTPUT_URL: jdbc:oracle:thin:@output_db:1521:xe SYNTHESIZED_OUTPUT_USERNAME: test SYNTHESIZED_OUTPUT_PASSWORD: test SYNTHESIZED_USERCONFIG_FILE: /app/config.yaml @@ -82,7 +83,9 @@ services: volumes: - ./config.tdk.yaml:/app/config.yaml - ./banner.txt:/app/banner.txt - + depends_on: + databases: + condition: service_completed_successfully ora2pg: image: georgmoser/ora2pg @@ -111,6 +114,13 @@ services: - ./postgres_output_data/COPY_TEST_output.sql:/docker-entrypoint-initdb.d/4.sql ports: [ "5432:5432" ] + check: + container_name: check + image: sodadata/soda-core + networks: + - simple-network + volumes: + - ./soda:/sodacl networks: simple-network: diff --git a/oracle/soda/checks_for_masking.yaml b/oracle/soda/checks_for_masking.yaml new file mode 100644 index 0000000..ae8f357 --- /dev/null +++ b/oracle/soda/checks_for_masking.yaml @@ -0,0 +1,47 @@ +checks for staff: + - row_count same as staff in input_db + +checks for film: + - row_count same as film in input_db + - distribution_difference(rating) = 0.0: + method: swd + distribution reference file: /sodacl/masked_film_rating_distribution.yml + +checks for actor: + - row_count same as actor in input_db + +checks for address: + - row_count same as address in input_db + +checks for category: + - row_count same as category in input_db + +checks for city: + - row_count same as city in input_db + +checks for country: + - row_count same as country in input_db + +checks for customer: + - row_count same as customer in input_db + +checks for film_actor: + - row_count same as film_actor in input_db + +checks for film_category: + - row_count same as film_category in input_db + +checks for inventory: + - row_count same as inventory in input_db + +checks for language: + - row_count same as language in input_db + +checks for payment: + - row_count same as payment in input_db + +checks for rental: + - row_count same as rental in input_db + +checks for store: + - row_count same as store in input_db diff --git a/oracle/soda/configuration.yaml b/oracle/soda/configuration.yaml new file mode 100644 index 0000000..c0e4bc3 --- /dev/null +++ b/oracle/soda/configuration.yaml @@ -0,0 +1,11 @@ +data_source input_db: + type: oracle + username: test + password: test + connectstring: input_db:1521/XE + +data_source output_db: + type: oracle + username: test + password: test + connectstring: output_db:1521/XE diff --git a/oracle/soda/masked_film_rating_distribution.yml b/oracle/soda/masked_film_rating_distribution.yml new file mode 100644 index 0000000..d0f3719 --- /dev/null +++ b/oracle/soda/masked_film_rating_distribution.yml @@ -0,0 +1,16 @@ +dataset: film +column: rating +distribution_type: categorical +distribution_reference: + weights: + - 0.223 + - 0.21 + - 0.195 + - 0.194 + - 0.178 + bins: + - MR-08 + - HB-85 + - E + - MR + - N From 04fb82879bb9d06d10b006f51c03ec142b949694 Mon Sep 17 00:00:00 2001 From: ViacheslavP Date: Wed, 11 Dec 2024 15:19:58 +0400 Subject: [PATCH 07/10] Added workflow --- .github/workflows/test_oracle.yml | 46 +++++++++++++++++++++++ oracle/soda/checks_for_input_db.yaml | 47 ++++++++++++++++++++++++ oracle/soda/film_rating_distribution.yml | 16 ++++++++ 3 files changed, 109 insertions(+) create mode 100644 .github/workflows/test_oracle.yml create mode 100644 oracle/soda/checks_for_input_db.yaml create mode 100644 oracle/soda/film_rating_distribution.yml diff --git a/.github/workflows/test_oracle.yml b/.github/workflows/test_oracle.yml new file mode 100644 index 0000000..fceecaa --- /dev/null +++ b/.github/workflows/test_oracle.yml @@ -0,0 +1,46 @@ +name: test_oracle + +on: + push: + branches: [ 'main' ] + pull_request: + branches: [ '*' ] + +defaults: + run: + working-directory: ./oracle + +jobs: + + masking: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Build docker compose + run: | + docker compose pull + docker compose build + + - name: Run databases + continue-on-error: true + run: | + docker compose run databases + + - name: Run TDK + env: + SYNTHESIZED_KEY: ${{ secrets.SYNTHESIZED_LICENSE_KEY }} + run: | + docker compose run tdk + + - name: Verify input data + run: > + docker compose run check scan -d input_db + -c /sodacl/configuration.yaml + /sodacl/checks_for_input_db.yaml + + - name: Verify output data + run: > + docker compose run check scan -d output_db + -c /sodacl/configuration.yaml + /sodacl/checks_for_masking.yaml diff --git a/oracle/soda/checks_for_input_db.yaml b/oracle/soda/checks_for_input_db.yaml new file mode 100644 index 0000000..42a22b9 --- /dev/null +++ b/oracle/soda/checks_for_input_db.yaml @@ -0,0 +1,47 @@ +checks for actor: + - row_count = 200 + +checks for address: + - row_count = 603 + +checks for category: + - row_count = 16 + +checks for city: + - row_count = 600 + +checks for country: + - row_count = 109 + +checks for customer: + - row_count = 599 + +checks for film: + - row_count = 1000 +# - distribution_difference(rating) < 0.01: +# method: chi_square +# distribution reference file: /sodacl/film_rating_distribution.yml + +checks for film_actor: + - row_count = 5462 + +checks for film_category: + - row_count = 1000 + +checks for inventory: + - row_count = 4581 + +checks for language: + - row_count = 6 + +checks for payment: + - row_count = 16049 + +checks for rental: + - row_count = 16044 + +checks for staff: + - row_count = 2 + +checks for store: + - row_count = 2 diff --git a/oracle/soda/film_rating_distribution.yml b/oracle/soda/film_rating_distribution.yml new file mode 100644 index 0000000..2f1885b --- /dev/null +++ b/oracle/soda/film_rating_distribution.yml @@ -0,0 +1,16 @@ +dataset: film +column: rating +distribution_type: categorical +distribution_reference: + weights: + - 0.223 + - 0.21 + - 0.195 + - 0.194 + - 0.178 + bins: + - PG-13 + - NC-17 + - R + - PG + - G From 5b12243a7c33896f91254f19b620ee491a63d133 Mon Sep 17 00:00:00 2001 From: ViacheslavP Date: Thu, 12 Dec 2024 13:00:19 +0400 Subject: [PATCH 08/10] Renew documaentation --- oracle/README.md | 2 +- oracle/ora2pg.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/oracle/README.md b/oracle/README.md index 562f564..84e73c4 100644 --- a/oracle/README.md +++ b/oracle/README.md @@ -20,7 +20,7 @@ export SYNTHESIZED_KEY=kDpeQB... Spin up Oracle databases: ```shell -docker compose up oracle_source oracle_target -d +docker compose run databases ``` Run TDK: diff --git a/oracle/ora2pg.conf b/oracle/ora2pg.conf index 4d5f993..2d3ce5f 100644 --- a/oracle/ora2pg.conf +++ b/oracle/ora2pg.conf @@ -1,6 +1,6 @@ # PG_VERSION 11 -ORACLE_DSN dbi:Oracle:host=oracle_target;sid=xe;port=1521 +ORACLE_DSN dbi:Oracle:host=input_db;sid=xe;port=1521 ORACLE_USER test ORACLE_PWD test From a724955cd42b052041a54ae97b53abcc2ece89d9 Mon Sep 17 00:00:00 2001 From: ViacheslavP Date: Thu, 12 Dec 2024 13:03:51 +0400 Subject: [PATCH 09/10] lint --- oracle/create_flag.sh | 2 +- oracle/create_user.sql | 2 +- oracle/docker-compose.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/oracle/create_flag.sh b/oracle/create_flag.sh index fb6cdb3..d56a574 100644 --- a/oracle/create_flag.sh +++ b/oracle/create_flag.sh @@ -1,3 +1,3 @@ #!/bin/bash -touch /opt/oracle/flag \ No newline at end of file +touch /opt/oracle/flag diff --git a/oracle/create_user.sql b/oracle/create_user.sql index d9c57dc..28af92c 100644 --- a/oracle/create_user.sql +++ b/oracle/create_user.sql @@ -1,3 +1,3 @@ CREATE USER test IDENTIFIED BY test; GRANT ALL PRIVILEGES TO test; -GRANT SELECT ON V_$DATABASE TO test; \ No newline at end of file +GRANT SELECT ON V_$DATABASE TO test; diff --git a/oracle/docker-compose.yaml b/oracle/docker-compose.yaml index ad0c9ba..0c51dc4 100644 --- a/oracle/docker-compose.yaml +++ b/oracle/docker-compose.yaml @@ -17,7 +17,7 @@ services: - ./oracle-sakila-insert-data.sql:/container-entrypoint-initdb.d/3.sql - ./create_flag.sh:/container-entrypoint-initdb.d/4.sh healthcheck: - test: ["CMD", "sh", "-c", "if [ -e /opt/oracle/flag ]; then exit 0; else exit 1; fi"] + test: ["CMD", "sh", "-c", "if [ -e /opt/oracle/flag ]; then exit 0; else exit 1; fi"] interval: "6s" timeout: "6s" start_period: "6s" From b6f0c1032289508edbf471f0bcc83072ec5cfd8d Mon Sep 17 00:00:00 2001 From: ViacheslavP Date: Fri, 13 Dec 2024 19:07:27 +0400 Subject: [PATCH 10/10] Update readme: deleted known problems part as it is resloved --- oracle/README.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/oracle/README.md b/oracle/README.md index 84e73c4..afaad0e 100644 --- a/oracle/README.md +++ b/oracle/README.md @@ -39,10 +39,3 @@ Run the PostgreSQL database: ```shell docker compose up postgres_masked -d ``` - - -## Knowing problems - -- get rid of `last_update` columns in the cofiguration file -- truncate shcema with recursive FKs -- mask the `film.description` column