fp update9

goitacademy · Jan 29, 2025 · c74c823 · c74c823
1 parent 875caa9
commit c74c823
Show file tree

Hide file tree

Showing 8 changed files with 54 additions and 50 deletions.
diff --git a/encrypted_file/eugene_etl_pipeline.tar_encrypted.tar.gz b/encrypted_file/eugene_etl_pipeline.tar_encrypted.tar.gz
diff --git a/eugene/eugene_etl_pipeline.tar.gz b/eugene/eugene_etl_pipeline.tar.gz
diff --git a/eugene/project_solution.py b/eugene/project_solution.py
diff --git a/multi_hop_datalake_eugene/dags/project_solution.py b/multi_hop_datalake_eugene/dags/project_solution.py
@@ -0,0 +1,54 @@
+from airflow import DAG
+from airflow.utils.dates import days_ago
+from airflow.providers.apache.spark.operators.spark_submit import SparkSubmitOperator
+from datetime import datetime
+import os
+
+# Налаштування за замовчуванням
+default_args = {
+    'owner': 'eugene',
+    'start_date': datetime(2024, 8, 4),
+}
+
+# 🔍 Шлях до скриптів
+SCRIPTS_DIR = "/root/airflow-docker/dags/scripts"
+
+# Створення DAG
+with DAG(
+    dag_id='multi_hop_datalake_eugene',
+    default_args=default_args,
+    schedule_interval=None,
+    catchup=False,
+    tags=["eugene"],
+    description='Pipeline for multi-hop datalake: landing → bronze → silver → gold',
+) as dag:
+
+    # Landing to bronze
+    landing_to_bronze_all = SparkSubmitOperator(
+        application=f"{SCRIPTS_DIR}/landing_to_bronze.py",
+        task_id='landing_to_bronze_all_tables',
+        conn_id='spark-default',
+        verbose=1,
+        dag=dag
+    )
+
+    # Bronze to silver
+    bronze_to_silver_all = SparkSubmitOperator(
+        application=f"{SCRIPTS_DIR}/bronze_to_silver.py",
+        task_id='bronze_to_silver_all_tables',
+        conn_id='spark-default',
+        verbose=1,
+        dag=dag
+    )
+
+    # Silver to gold (агрегація)
+    silver_to_gold_avg_stats = SparkSubmitOperator(
+        application=f"{SCRIPTS_DIR}/silver_to_gold.py",
+        task_id='silver_to_gold_avg_stats',
+        conn_id='spark-default',
+        verbose=1,
+        dag=dag
+    )
+
+    # Послідовність виконання
+    landing_to_bronze_all >> bronze_to_silver_all >> silver_to_gold_avg_stats
diff --git a/multi_hop_datalake_eugene/eugene_etl_pipeline.tar.gz b/multi_hop_datalake_eugene/eugene_etl_pipeline.tar.gz
diff --git a/eugene/bronze_to_silver.py → ...talake_eugene/scripts/bronze_to_silver.py b/eugene/bronze_to_silver.py → ...talake_eugene/scripts/bronze_to_silver.py
diff --git a/eugene/landing_to_bronze.py → ...alake_eugene/scripts/landing_to_bronze.py b/eugene/landing_to_bronze.py → ...alake_eugene/scripts/landing_to_bronze.py
diff --git a/eugene/silver_to_gold.py → ...datalake_eugene/scripts/silver_to_gold.py b/eugene/silver_to_gold.py → ...datalake_eugene/scripts/silver_to_gold.py