From 90575a5739669cea485fa05d2c5cf9888031eaef Mon Sep 17 00:00:00 2001
From: Brem090 <brem090@gmail.com>
Date: Fri, 31 Jan 2025 16:24:50 +0200
Subject: [PATCH] fpu14

---
 .../eugene_etl_pipeline.tar_encrypted.tar.gz    | Bin 467 -> 466 bytes
 eugene/bronze_to_silver.py                      |   8 ++++++--
 eugene/silver_to_gold.py                        |   8 ++++----
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/encrypted_file/eugene_etl_pipeline.tar_encrypted.tar.gz b/encrypted_file/eugene_etl_pipeline.tar_encrypted.tar.gz
index 876108bedb7bbc01e91939702ebb121491fa796c..9e22b0119fcb8acaeb5e84a2aabd9dfeee8b4d9d 100644
GIT binary patch
delta 319
zcmV-F0l@y#1JVPKEDBRI69Z$QxH(V-kvB(w-@D+nllT9_*3!0d`&fVNy?py>{Eyeq
z+83yZ)n53pb6T?R!;Q%cCbI^wOxK$6^uv$C)mmK&zVB^|q<7wLRVej)ut~|ei*a?5
zar)H%nYSPBRQ-L!#pK|OAAG%w<y3zkEIR!nG@|n64vio0?{~&*U;83Sq&{cP&7S&y
zE${uRGh9B;wlg}O#rH7sz|X?_S*IslytMH3b(2TMt{uuJlTKx=U*gwktbBg0&cAgF
zS&xba7jOL6bV<;5@}(0CJ0|?izSH9LBGGTA(O(56!8!l$mgnSGGoDShi>U5T|GPz;
zhi~<pqyVLa&$296{SMvjmcF~hUuJ(5DE||w_uNxLA3yq%BNm*YFlnH}f0LmBCX=86
R5DWrJ006m^@O1zv002M~tla<r

delta 319
zcmV-F0l@y!1JeVLEDK{ZV`CGbxVf2u5s@}We>YvV?m_s+&DmYoJG?L1E)+h^n46G&
zIY~+V_tAWp*R^S3ucwJ^ZaJB0-Zl4ioIb}4Md5$d%~LvyUDQABUYs`l`JX&xRkL$D
zd=^#lDXk9rWLLBFt6a4)$I+hIlU>RsekHLx`oB6ZWFq~aS?KVWx*CT!zs>HgO@4WB
zf5Hi!w~x5)itSyJFr~ftnR5kGNBiE(6P{K7SSTVWc0TJt`^CC)p$R{yu3CITW68WL
zFBmiy_=NuWFQk;S{ln6QwZDp=Nz2~tV|}@Mc6i{6LpIes=eAcx?!R$lPwm9IGY`c#
z@~FPK-QxB9!k+yXx!-eEaxlzT!LO0b7+~3vn<e<|^#swU{V%7VmW&$c@E--EV3Uvm
RGYo(d006SFH?aUH004|er`rGk

diff --git a/eugene/bronze_to_silver.py b/eugene/bronze_to_silver.py
index 5e1e00b..57807cf 100644
--- a/eugene/bronze_to_silver.py
+++ b/eugene/bronze_to_silver.py
@@ -1,5 +1,5 @@
 from pyspark.sql import SparkSession
-from pyspark.sql.functions import udf, col
+from pyspark.sql.functions import udf, col, lit
 from pyspark.sql.types import StringType
 import re, os, logging
 
@@ -36,7 +36,11 @@ def process_table(spark, table_name):
 
     # Видалення дублікатів рядків
     df = df.dropDuplicates()
-    
+
+    # Перевірка та заповнення відсутніх значень у стовпці medal значенням "No Medal"
+    if "medal" in df.columns:
+        df = df.withColumn("medal", col("medal").na.fill("No Medal"))
+
     # Логування попереднього перегляду
     logger.info(f"Data preview after cleaning for {table_name}:")
     df.show(20)  
diff --git a/eugene/silver_to_gold.py b/eugene/silver_to_gold.py
index d1ae878..6759975 100644
--- a/eugene/silver_to_gold.py
+++ b/eugene/silver_to_gold.py
@@ -26,9 +26,9 @@
     bio_df = spark.read.parquet("silver/athlete_bio", schema=bio_schema)
     results_df = spark.read.parquet("silver/athlete_event_results")
 
-    # Фільтрація медалей
-    valid_medals = ["Gold", "Silver", "Bronze"]
-    results_selected = results_df.select("athlete_id", "sport", "medal").filter(col("medal").isin(valid_medals))
+    # Вибір колонок з athlete_event_results та обробка відсутніх значень у колонці medal
+    results_selected = results_df.select("athlete_id", "sport", "medal") \
+        .withColumn("medal", col("medal").na.fill("No Medal"))
 
     # Вибір колонок з athlete_bio
     bio_selected = bio_df.select("athlete_id", "sex", "country_noc", "weight", "height")
@@ -63,4 +63,4 @@
     filtered_stats.repartition("sport", "medal").write.parquet("gold/avg_stats", mode="overwrite")
 
     # Завершення сесії
-    spark.stop()
+    spark.stop()
\ No newline at end of file