Day 3 - Kabbage DF.ipynb

{"cells":[{"cell_type":"code","source":["from pyspark.sql import Row"],"metadata":{},"outputs":[],"execution_count":1},{"cell_type":"code","source":["row1 = Row(name=\"Awi\", age=10)"],"metadata":{},"outputs":[],"execution_count":2},{"cell_type":"code","source":["row1"],"metadata":{},"outputs":[],"execution_count":3},{"cell_type":"code","source":["row1.name"],"metadata":{},"outputs":[],"execution_count":4},{"cell_type":"code","source":["\nEmployee = Row(\"name\",\"location\")"],"metadata":{},"outputs":[],"execution_count":5},{"cell_type":"code","source":["Employee(\"Awi\",\"Bangalore\")"],"metadata":{},"outputs":[],"execution_count":6},{"cell_type":"code","source":["row1.asDict()"],"metadata":{},"outputs":[],"execution_count":7},{"cell_type":"code","source":["Row(name=\"A\", val=Row(sub='Science',loc='Mumbai'))"],"metadata":{},"outputs":[],"execution_count":8},{"cell_type":"code","source":["# Creation of dataframes\nspark.createDataFrame([('Alice',1,22),('Byom',3,66)],['name','age','salary']).show()"],"metadata":{},"outputs":[],"execution_count":9},{"cell_type":"code","source":["db = [{'name':'Akbar','loc':'delhi'},{ 'name' :'sikandar', 'loc':'lucknow'}]"],"metadata":{},"outputs":[],"execution_count":10},{"cell_type":"code","source":["spark.createDataFrame(db).show()"],"metadata":{},"outputs":[],"execution_count":11},{"cell_type":"code","source":["l = [('Awi',1),('Bwi',2)]"],"metadata":{},"outputs":[],"execution_count":12},{"cell_type":"code","source":["rdd = sc.parallelize(l)"],"metadata":{},"outputs":[],"execution_count":13},{"cell_type":"code","source":["Person = Row(\"name\",\"age\")"],"metadata":{},"outputs":[],"execution_count":14},{"cell_type":"code","source":["p1 = rdd.map( lambda x: Person(*x))"],"metadata":{},"outputs":[],"execution_count":15},{"cell_type":"code","source":["df = spark.createDataFrame(p1)"],"metadata":{},"outputs":[],"execution_count":16},{"cell_type":"code","source":["df.show()"],"metadata":{},"outputs":[],"execution_count":17},{"cell_type":"code","source":["df.select('name').show()"],"metadata":{},"outputs":[],"execution_count":18},{"cell_type":"code","source":["import pandas as pd"],"metadata":{},"outputs":[],"execution_count":19},{"cell_type":"code","source":["pdf = df.toPandas()"],"metadata":{},"outputs":[],"execution_count":20},{"cell_type":"code","source":["spark.createDataFrame(pdf).show()"],"metadata":{},"outputs":[],"execution_count":21},{"cell_type":"code","source":["data = spark.read.csv('/FileStore/tables/vc9z36401490339549155/database.csv',inferSchema=True, header=True)"],"metadata":{},"outputs":[],"execution_count":22},{"cell_type":"code","source":["display(data)"],"metadata":{},"outputs":[],"execution_count":23},{"cell_type":"code","source":["data.printSchema()"],"metadata":{},"outputs":[],"execution_count":24},{"cell_type":"code","source":["data.count()"],"metadata":{},"outputs":[],"execution_count":25},{"cell_type":"code","source":["data.columns"],"metadata":{},"outputs":[],"execution_count":26},{"cell_type":"code","source":["display(data.select('Relationship').distinct())"],"metadata":{},"outputs":[],"execution_count":27},{"cell_type":"code","source":[""],"metadata":{},"outputs":[],"execution_count":28}],"metadata":{"name":"Day 3 - Kabbage DF","notebookId":768895070231145},"nbformat":4,"nbformat_minor":0}