Merge pull request #44 from TeoMeWhy/feat/dota

Optimize file readers
TeoMeWhy · Aug 5, 2023 · 9feaff0 · 9feaff0
2 parents d0c5c86 + d56cd8b
commit 9feaff0
Showing 1 changed file with 4 additions and 9 deletions.
diff --git a/src/01.raw/dota/pos_collect.py b/src/01.raw/dota/pos_collect.py
@@ -1,18 +1,13 @@
 # Databricks notebook source
 from pyspark.sql import types
+import pandas as pd
 
 # COMMAND ----------
 
 # DBTITLE 1,Consolida collect
-schema = types.StructType(fields=[
-            types.StructField(name="match_id",
-                              dataType=types.StringType())
-])
-
-df = (spark.read
-           .format("json")
-           .schema(schema) 
-           .load("/mnt/datalake/dota/matches_details"))
+files = [i.name.split(".")[0] for i in dbutils.fs.ls("/mnt/datalake/dota/matches_details")]
+
+df = spark.createDataFrame(pd.DataFrame({"match_id":files}))
 
 (df.write
    .format("delta")