diff --git a/practice_pyspark/PySpark Basics.ipynb b/practice_pyspark/PySpark Basics.ipynb index 57a90ef..470fcee 100644 --- a/practice_pyspark/PySpark Basics.ipynb +++ b/practice_pyspark/PySpark Basics.ipynb @@ -1299,10 +1299,478 @@ "df_renamed_column.show()" ] }, + { + "cell_type": "markdown", + "id": "2e547a3a-2d4b-445c-9ec1-ad9ab7944772", + "metadata": {}, + "source": [ + "## Handling Missing Values" + ] + }, + { + "cell_type": "code", + "execution_count": 89, + "id": "3fcdfeb2-b19e-47c9-868b-5b3231d474bf", + "metadata": {}, + "outputs": [], + "source": [ + "null_df=spark.read.csv('customers-100-na.csv', header='true', inferSchema='true')" + ] + }, + { + "cell_type": "code", + "execution_count": 90, + "id": "8606abd6-f837-492e-8d9f-698a0d4e0fc5", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------+------------------+---------------+----------+---------+--------------------+------------+--------+-------------------+-------------------+--------------------+--------------------+\n", + "|summary| Index| Customer Id|First Name|Last Name| Company| City| Country| Phone 1| Phone 2| Email| Website|\n", + "+-------+------------------+---------------+----------+---------+--------------------+------------+--------+-------------------+-------------------+--------------------+--------------------+\n", + "| count| 100| 100| 99| 98| 99| 99| 100| 100| 99| 100| 99|\n", + "| mean| 50.5| NULL| NULL| NULL| NULL| NULL| NULL|3.977821758076923E9| 4.153267376E9| NULL| NULL|\n", + "| stddev|29.011491975882016| NULL| NULL| NULL| NULL| NULL| NULL|2.897944246069217E9|4.078648325054578E9| NULL| NULL|\n", + "| min| 1|010468dAA11382c| Aimee| Alvarado|Acosta, Petersen ...|Acevedoville| Albania| (041)737-3846| (026)401-7353x2417|aharper@maddox-to...| http://acosta.org/|\n", + "| max| 100|faCEF517ae7D8eB| Yvonne| Zuniga| Winters-Mendoza| West Samuel|Zimbabwe| 981-544-9452| 999-728-1637|zvalencia@phelps.com|https://www.vang....|\n", + "+-------+------------------+---------------+----------+---------+--------------------+------------+--------+-------------------+-------------------+--------------------+--------------------+\n", + "\n" + ] + } + ], + "source": [ + "null_df.describe().show()" + ] + }, + { + "cell_type": "code", + "execution_count": 91, + "id": "1b52b5e9-d032-41f7-8ff0-c0db646df006", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------+------------------+---------------+----------+---------+--------------------+-----------+--------+--------------------+-------------------+--------------------+--------------------+\n", + "|summary| Index| Customer Id|First Name|Last Name| Company| City| Country| Phone 1| Phone 2| Email| Website|\n", + "+-------+------------------+---------------+----------+---------+--------------------+-----------+--------+--------------------+-------------------+--------------------+--------------------+\n", + "| count| 92| 92| 92| 92| 92| 92| 92| 92| 92| 92| 92|\n", + "| mean| 52.18478260869565| NULL| NULL| NULL| NULL| NULL| NULL|3.7840032974166665E9| 4.153267376E9| NULL| NULL|\n", + "| stddev|29.438678805991465| NULL| NULL| NULL| NULL| NULL| NULL| 2.937480944802339E9|4.078648325054578E9| NULL| NULL|\n", + "| min| 1|010468dAA11382c| Aimee| Alvarado|Acosta, Petersen ...| Bensonview| Algeria| (050)136-9025| (026)401-7353x2417|aharper@maddox-to...|http://burgess-fr...|\n", + "| max| 100|faCEF517ae7D8eB| Yvonne| Zuniga| Winters-Mendoza|West Samuel|Zimbabwe| 981-544-9452| 999-728-1637|zvalencia@phelps.com|https://www.vang....|\n", + "+-------+------------------+---------------+----------+---------+--------------------+-----------+--------+--------------------+-------------------+--------------------+--------------------+\n", + "\n" + ] + } + ], + "source": [ + "null_df.na.drop().describe().show()" + ] + }, + { + "cell_type": "code", + "execution_count": 94, + "id": "ad565fbf-c1f0-44da-9dc1-9d80671950b4", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------+------------------+---------------+----------+---------+--------------------+------------+--------+-------------------+-------------------+--------------------+--------------------+\n", + "|summary| Index| Customer Id|First Name|Last Name| Company| City| Country| Phone 1| Phone 2| Email| Website|\n", + "+-------+------------------+---------------+----------+---------+--------------------+------------+--------+-------------------+-------------------+--------------------+--------------------+\n", + "| count| 100| 100| 99| 98| 99| 99| 100| 100| 99| 100| 99|\n", + "| mean| 50.5| NULL| NULL| NULL| NULL| NULL| NULL|3.977821758076923E9| 4.153267376E9| NULL| NULL|\n", + "| stddev|29.011491975882016| NULL| NULL| NULL| NULL| NULL| NULL|2.897944246069217E9|4.078648325054578E9| NULL| NULL|\n", + "| min| 1|010468dAA11382c| Aimee| Alvarado|Acosta, Petersen ...|Acevedoville| Albania| (041)737-3846| (026)401-7353x2417|aharper@maddox-to...| http://acosta.org/|\n", + "| max| 100|faCEF517ae7D8eB| Yvonne| Zuniga| Winters-Mendoza| West Samuel|Zimbabwe| 981-544-9452| 999-728-1637|zvalencia@phelps.com|https://www.vang....|\n", + "+-------+------------------+---------------+----------+---------+--------------------+------------+--------+-------------------+-------------------+--------------------+--------------------+\n", + "\n" + ] + } + ], + "source": [ + "null_df.na.drop(how='all').describe().show()" + ] + }, + { + "cell_type": "code", + "execution_count": 95, + "id": "5e560a1f-894a-445d-b798-1d10258ff311", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------+------------------+---------------+----------+---------+--------------------+-----------+--------+--------------------+-------------------+--------------------+--------------------+\n", + "|summary| Index| Customer Id|First Name|Last Name| Company| City| Country| Phone 1| Phone 2| Email| Website|\n", + "+-------+------------------+---------------+----------+---------+--------------------+-----------+--------+--------------------+-------------------+--------------------+--------------------+\n", + "| count| 92| 92| 92| 92| 92| 92| 92| 92| 92| 92| 92|\n", + "| mean| 52.18478260869565| NULL| NULL| NULL| NULL| NULL| NULL|3.7840032974166665E9| 4.153267376E9| NULL| NULL|\n", + "| stddev|29.438678805991465| NULL| NULL| NULL| NULL| NULL| NULL| 2.937480944802339E9|4.078648325054578E9| NULL| NULL|\n", + "| min| 1|010468dAA11382c| Aimee| Alvarado|Acosta, Petersen ...| Bensonview| Algeria| (050)136-9025| (026)401-7353x2417|aharper@maddox-to...|http://burgess-fr...|\n", + "| max| 100|faCEF517ae7D8eB| Yvonne| Zuniga| Winters-Mendoza|West Samuel|Zimbabwe| 981-544-9452| 999-728-1637|zvalencia@phelps.com|https://www.vang....|\n", + "+-------+------------------+---------------+----------+---------+--------------------+-----------+--------+--------------------+-------------------+--------------------+--------------------+\n", + "\n" + ] + } + ], + "source": [ + "null_df.na.drop(how='any').describe().show()" + ] + }, + { + "cell_type": "code", + "execution_count": 96, + "id": "46c82f84-0132-4e09-a67e-8583d604f094", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------+------------------+---------------+----------+---------+--------------------+------------+--------+-------------------+-------------------+--------------------+--------------------+\n", + "|summary| Index| Customer Id|First Name|Last Name| Company| City| Country| Phone 1| Phone 2| Email| Website|\n", + "+-------+------------------+---------------+----------+---------+--------------------+------------+--------+-------------------+-------------------+--------------------+--------------------+\n", + "| count| 100| 100| 99| 98| 99| 99| 100| 100| 99| 100| 99|\n", + "| mean| 50.5| NULL| NULL| NULL| NULL| NULL| NULL|3.977821758076923E9| 4.153267376E9| NULL| NULL|\n", + "| stddev|29.011491975882016| NULL| NULL| NULL| NULL| NULL| NULL|2.897944246069217E9|4.078648325054578E9| NULL| NULL|\n", + "| min| 1|010468dAA11382c| Aimee| Alvarado|Acosta, Petersen ...|Acevedoville| Albania| (041)737-3846| (026)401-7353x2417|aharper@maddox-to...| http://acosta.org/|\n", + "| max| 100|faCEF517ae7D8eB| Yvonne| Zuniga| Winters-Mendoza| West Samuel|Zimbabwe| 981-544-9452| 999-728-1637|zvalencia@phelps.com|https://www.vang....|\n", + "+-------+------------------+---------------+----------+---------+--------------------+------------+--------+-------------------+-------------------+--------------------+--------------------+\n", + "\n" + ] + } + ], + "source": [ + "null_df.na.drop(how='any',thresh=2).describe().show()" + ] + }, + { + "cell_type": "code", + "execution_count": 97, + "id": "a20918cb-d653-44f5-a08e-acb263f5f579", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------+------------------+---------------+----------+---------+--------------------+------------+--------+--------------------+-------------------+--------------------+--------------------+\n", + "|summary| Index| Customer Id|First Name|Last Name| Company| City| Country| Phone 1| Phone 2| Email| Website|\n", + "+-------+------------------+---------------+----------+---------+--------------------+------------+--------+--------------------+-------------------+--------------------+--------------------+\n", + "| count| 99| 99| 98| 97| 98| 98| 99| 99| 99| 99| 98|\n", + "| mean|50.666666666666664| NULL| NULL| NULL| NULL| NULL| NULL|3.7840032974166665E9| 4.153267376E9| NULL| NULL|\n", + "| stddev| 29.11097696044976| NULL| NULL| NULL| NULL| NULL| NULL| 2.937480944802339E9|4.078648325054578E9| NULL| NULL|\n", + "| min| 1|010468dAA11382c| Aimee| Alvarado|Acosta, Petersen ...|Acevedoville| Albania| (041)737-3846| (026)401-7353x2417|aharper@maddox-to...| http://acosta.org/|\n", + "| max| 100|faCEF517ae7D8eB| Yvonne| Zuniga| Winters-Mendoza| West Samuel|Zimbabwe| 981-544-9452| 999-728-1637|zvalencia@phelps.com|https://www.vang....|\n", + "+-------+------------------+---------------+----------+---------+--------------------+------------+--------+--------------------+-------------------+--------------------+--------------------+\n", + "\n" + ] + } + ], + "source": [ + "null_df.na.drop(how='any',subset='Phone 2').describe().show()" + ] + }, + { + "cell_type": "code", + "execution_count": 99, + "id": "2bcec9ff-f246-4447-9f18-17ab392ea576", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-----+---------------+----------+----------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----------------+--------------------+\n", + "|Index| Customer Id|First Name| Last Name| Company| City| Country| Phone 1| Phone 2| Email|Subscription Date| Website|\n", + "+-----+---------------+----------+----------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----------------+--------------------+\n", + "| 1|DD37Cf93aecA6Dc| Sheryl| Baxter| Rasmussen Group| East Leonard| Chile| 229.077.5154| 397.884.0519x718|zunigavanessa@smi...| 2020-08-24|http://www.stephe...|\n", + "| 2|1Ef7b82A4CAAD10| Preston| Lozano| Vega-Gentry| East Jimmychester| Djibouti| 5153435776| 686-620-1820x944| vmata@colon.com| 2021-04-23|http://www.hobbs....|\n", + "| 3|6F94879bDAfE5a6| Roy| Berry| Murillo-Perry| Isabelborough| Antigua and Barbuda| -1199| (496)978-3969x58947| beckycarr@hogan.com| 2020-03-25|http://www.lawren...|\n", + "| 4|5Cef8BFA16c5e3c| Linda| Olsen|Dominguez, Mcmill...| Bensonview| Dominican Republic|001-808-617-6467x...| -9892|stanleyblackwell@...| 2020-06-02|http://www.good-l...|\n", + "| 5|053d585Ab6b3159| Joanna| Bender|Martin, Lang and ...| West Priscilla|Slovakia (Slovak ...|001-234-203-0635x...|001-199-446-3860x...|colinalvarado@mil...| 2021-04-17|https://goodwin-i...|\n", + "| 6|2d08FB17EE273F4| Aimee| Downs| Steele Group| Chavezborough|Bosnia and Herzeg...| (283)437-3886x88321| 999-728-1637| louis27@gilbert.com| 2020-02-25|http://www.berger...|\n", + "| 7|EA4d384DfDbBf77| Darren| Peck|Lester, Woodard a...| Lake Ana| Pitcairn Islands| (496)452-6181x3291|+1-247-266-0963x4995| tgates@cantrell.com| 2021-08-24| https://www.le.com/|\n", + "| 8|0e04AFde9f225dE| Brett| Mullen|Sanford, Davenpor...| Kimport| Bulgaria|001-583-352-7197x297| 001-333-145-0369| asnow@colon.com| 2021-04-12|https://hammond-r...|\n", + "| 9|C2dE4dEEc489ae0| Sheryl| Meyers| Browning-Simon| Robersonstad| Cyprus| 854-138-4911x5772| +1-448-910-2276x729|mariokhan@ryan-po...| 2020-01-13|https://www.bullo...|\n", + "| 10|8C2811a503C7c5a| Michelle| Gallagher| Beck-Hendrix| Elaineberg| Timor-Leste| 739.218.2516x459|001-054-401-0347x617| mdyer@escobar.net| 2021-11-08| https://arias.com/|\n", + "| 11|216E205d6eBb815| Carl| Schroeder|Oconnell, Meza an...| Shannonville| Guernsey| 637-854-0256x825| 114.336.0784x788| kirksalas@webb.com| 2021-10-20|https://simmons-h...|\n", + "| 12|CEDec94deE6d69B| Jenna| Dodson|Hoffman, Reed and...| East Andrea| Vietnam| (041)737-3846|+1-556-888-3485x4...| mark42@robbins.com| 2020-11-29| Missing|\n", + "| 13|e35426EbDEceaFF| Tracey| Mata| Graham-Francis| South Joannamouth| Togo| 001-949-844-8787| (855)713-8773| alex56@walls.org| 2021-12-02|http://www.beck.com/|\n", + "| 14|A08A8aF8BE9FaD4| Kristine| Cox| Carpenter-Cook| Jodyberg| Sri Lanka| 786-284-3358x62152|+1-315-627-1796x8074|holdenmiranda@cla...| 2021-02-08|https://www.brand...|\n", + "| 15|6fEaA1b7cab7B6C| Faith| Lutz| Carter-Hancock| Burchbury| Singapore| (781)861-7180x8306| 207-185-3665|cassieparrish@ble...| 2022-01-26|http://stevenson....|\n", + "| 16|8cad0b4CBceaeec| Miranda| Beasley| Missing| Desireeshire| Oman| 540.085.3135x185|+1-600-462-6432x2...|vduncan@parks-har...| 2022-04-12| http://acosta.org/|\n", + "| 17|a5DC21AE3a21eaA| Caroline| Foley| Winters-Mendoza| West Adriennestad| Western Sahara| 936.222.4746x9924|001-469-948-6341x359|holtgwendolyn@wat...| 2021-03-10|http://www.benson...|\n", + "| 18|F8Aa9d6DfcBeeF8| Greg| Mata| Valentine LLC| Lake Leslie| Mozambique| (701)087-2415| (195)156-1861x26241|jaredjuarez@carro...| 2022-03-26|http://pitts-cher...|\n", + "| 19|F160f5Db3EfE973| Clifford| Jacobson| Simon LLC| Harmonview|South Georgia and...|001-151-330-3524x...| (748)477-7174|joseph26@jacobson...| 2020-09-24|https://mcconnell...|\n", + "| 20|0F60FF3DdCd7aB0| Joanna| Kirk| Mays-Mccormick| Jamesshire| French Polynesia| (266)131-7001x711| (283)312-5579x11543|tuckerangie@salaz...| 2021-09-24|https://www.camac...|\n", + "| 21|9F9AdB7B8A6f7F2| Maxwell| Frye| Patterson Inc| East Carly| Malta| 423.262.3059| 202-880-0688x7491|fgibson@drake-web...| 2022-01-12|http://www.robert...|\n", + "| 22|FBd0Ded4F02a742| Kiara| Houston|Manning, Hester a...| South Alvin| Netherlands|001-274-040-3582x...|+1-528-175-0973x4684|blanchardbob@wall...| 2020-09-15|https://www.reid-...|\n", + "| 23|2FB0FAA1d429421| Colleen| Howard| Greer and Sons| Brittanyview| Paraguay| 1935085151| (947)115-7711x5488|rsingleton@ryan-c...| 2020-08-19| http://paul.biz/|\n", + "| 24|010468dAA11382c| Janet|Valenzuela| Watts-Donaldson| Veronicamouth|Lao People's Demo...| 354.259.5062x7538| 500.433.2022|stefanie71@spence...| 2020-09-08| https://moreno.biz/|\n", + "| 25|eC1927Ca84E033e| Shane| Wilcox| Tucker LLC| Bryanville| Albania| (429)005-9030x11004| 541-116-4501| mariah88@santos.com| NULL|https://www.ramos...|\n", + "| 26|09D7D7C8Fe09aea| Marcus| Moody| Giles Ltd| Kaitlyntown| Panama| 674-677-8623| 909-277-5485x566|donnamullins@norr...| 2022-05-24|https://www.curry...|\n", + "| 27|aBdfcF2c50b0bfD| Dakota| Poole| Simmons Group| Michealshire| Belarus| (371)987-8576x4720| 071-152-1376| stacey67@fields.org| 2022-02-20|https://sanford-w...|\n", + "| 28|b92EBfdF8a3f0E6| Frederick| Harper|Hinton, Chaney an...| South Marissatown| Switzerland|+1-077-121-1558x0687| 264.742.7149|jacobkhan@bright.biz| 2022-05-26|https://callahan....|\n", + "| 29|3B5dAAFA41AFa22| Stefanie| Missing| Santana-Duran| Acevedoville|Saint Vincent and...| (752)776-3286|+1-472-021-4814x8...| wterrell@clark.com| 2020-07-30| https://meyers.com/|\n", + "| 30|EDA69ca7a6e96a2| Kent| Bradshaw| Sawyer PLC| North Harold| Tanzania| +1-472-143-5037x884| 126.922.6153| qjimenez@boyd.com| 2020-04-26|http://maynard-ho...|\n", + "| 31|64DCcDFaB9DFd4e| Jack| Tate|Acosta, Petersen ...| West Samuel| Zimbabwe| 965-108-4406x20714| 046.906.1442x6784|gfigueroa@boone-z...| 2021-09-15|http://www.hawkin...|\n", + "| 32|679c6c83DD872d6| Tom| Trujillo| Mcgee Group| Cunninghamborough| Denmark| 416-338-3758| (775)890-7209|tapiagreg@beard.info| 2022-01-13|http://www.daniel...|\n", + "| 33|7Ce381e4Afa4ba9| Gabriel| Mejia| Adkins-Salinas| Port Annatown| Liechtenstein| 4077245425| 646.044.0696x66800|coleolson@jenning...| 2021-04-24|https://patel-han...|\n", + "| 34|A09AEc6E3bF70eE| Kaitlyn| Santana| Herrera Group| New Kaitlyn|United States of ...| 6303643286| Missing|georgeross@miles.org| 2021-09-21| http://pham.com/|\n", + "| 35|aA9BAFfBc3710fe| Faith| Moon|Waters, Chase and...| West Marthaburgh| Bahamas|+1-586-217-0359x6317| -2419|willistonya@rando...| 2021-11-03|https://spencer-c...|\n", + "| 36|E11dfb2DB8C9f72| Tammie| Haley|Palmer, Barnes an...| East Teresa| Belize|001-276-734-4113x...| (430)300-8770|harrisisaiah@jenk...| 2022-01-04|http://evans-simo...|\n", + "| 37|889eCf90f68c5Da| Nicholas| Sosa| Jordan Ltd| South Hunter| Uruguay| (661)425-6042| 975-998-1519| fwolfe@dorsey.com| 2021-08-10|https://www.flemi...|\n", + "| 38|7a1Ee69F4fF4B4D| Jordan| Gay| Glover and Sons| South Walter| Solomon Islands| 7208417020| 8035336772|tiffanydavies@har...| 2021-02-24| http://www.lee.org/|\n", + "| 39|dca4f1D0A0fc5c9| Bruce| Esparza| Huerta-Mclean| Poolefurt| Montenegro| 559-529-4424|001-625-000-7132x...|preese@frye-vega.com| 2021-10-22|http://www.farley...|\n", + "| 40|17aD8e2dB3df03D| Missing| Garza| Anderson Ltd| West John| Poland|001-067-713-6440x158| (978)289-8785x5766| ann48@miller.com| 2021-11-01| http://spence.com/|\n", + "| 41|2f79Cd309624Abb| Natalie| Gentry| Monroe PLC| West Darius| Dominican Republic| 830.996.8238| 499.122.5415|tcummings@fitzpat...| 2020-10-10|http://www.dorsey...|\n", + "| 42|6e5ad5a5e2bB5Ca| Bryan| Dunn| Kaufman and Sons| North Jimstad| Burkina Faso| 001-710-802-5565| 078.699.8982x13881|woodwardandres@ph...| 2021-09-08|http://www.butler...|\n", + "| 43|7E441b6B228DBcA| Wayne| Simpson| Perkins-Trevino|East Rebekahborough| Bolivia| (344)156-8632x1869| 463-445-3702x38463|barbarapittman@ho...| 2020-12-13|https://gillespie...|\n", + "| 44|D3fC11A9C235Dc6| Luis| Greer| Cross PLC| North Drew| Bulgaria|001-336-025-6849x701| 684.698.2911x6092|bstuart@williamso...| 2022-05-15|https://fletcher-...|\n", + "| 45|30Dfa48fe5Ede78| Rhonda| Frost|Herrera, Shepherd...| Lake Lindaburgh| Monaco| (127)081-9339|+1-431-028-3337x3492|zkrueger@wolf-cha...| 2021-12-06|http://www.khan.com/|\n", + "| 46|fD780ED8dbEae7B| Joanne| Missing|Price, Sexton and...| Gwendolynview| Palau| (897)726-7952| (467)886-9467x5721| juan80@henson.net| 2020-07-01| http://ochoa.com/|\n", + "| 47|300A40d3ce24bBA| Geoffrey| Guzman| Short-Wiggins| Missing| Uzbekistan| 975.235.8921x269| (983)188-6873|bauercrystal@gay.com| 2020-04-23|https://decker-kl...|\n", + "| 48|283DFCD0Dba40aF| Gloria| Mccall|Brennan, Acosta a...| North Kerriton| Ghana| 445-603-6729|001-395-959-4736x...|bartlettjenna@zun...| 2022-03-11|http://burgess-fr...|\n", + "| 49|F4Fc91fEAEad286| Brady| Cohen| Osborne-Erickson| North Eileenville|United Arab Emirates| 741.849.0139x524|+1-028-691-7497x0894|mccalltyrone@durh...| 2022-03-10|http://hammond-ba...|\n", + "| 50|80F33Fd2AcebF05| Latoya| Mccann|Hobbs, Garrett an...| Port Sergiofort| Belarus| (530)287-4548x29481| 162-234-0249x32790|bobhammond@barry.biz| 2021-12-02|https://www.burto...|\n", + "| 51|Aa20BDe68eAb0e9| Gerald| Hawkins|Phelps, Forbes an...| New Alberttown| Canada|+1-323-239-1456x9...| (092)508-0269|uwarner@steele-ar...| 2021-03-19|https://valenzuel...|\n", + "| 52|e898eEB1B9FE22b| Samuel| Crawford|May, Goodwin and ...| South Jasmine| Algeria| 802-242-7457| 626.116.9535x8578|xpittman@ritter-c...| 2021-03-27|https://guerrero....|\n", + "| 53|faCEF517ae7D8eB| Patricia| Goodwin|Christian, Winter...| Cowanfort| Swaziland| 322.549.7139x70040| (111)741-4173|vaughanchristy@la...| 2021-03-08| http://clark.info/|\n", + "| 54|c09952De6Cda8aA| Stacie| Richard| Byrd Inc| New Deborah| Madagascar|001-622-948-3641x...|001-731-168-2893x...|clinton85@colon-a...| 2020-10-15| https://kim.com/|\n", + "| 55|f3BEf3Be028166f| Robin| West|Nixon, Blackwell ...| Wallstown| Ecuador| 698.303.4267|001-683-837-7651x525|greenemiranda@zim...| 2022-01-13|https://www.mora....|\n", + "| 56|C6F2Fc6a7948a4e| Ralph| Haas| Montes PLC| Lake Ellenchester|Palestinian Terri...| 2239271999|001-962-434-0867x649|goodmancesar@figu...| 2020-05-25| http://may.com/|\n", + "| 57|c8FE57cBBdCDcb2| Phyllis| Maldonado| Costa PLC| Lake Whitney| Saint Barthelemy| 4500370767|001-508-064-6725x017|yhanson@warner-di...| 2021-01-25|http://www.bernar...|\n", + "| 58|B5acdFC982124F2| Danny| Parrish| Novak LLC| East Jaredbury|United Arab Emirates| (669)384-8597x8794| 506.731.5952x571|howelldarren@hous...| 2021-03-17|http://www.parson...|\n", + "| 59|8c7DdF10798bCC3| Kathy| Hill|Moore, Mccoy and ...| Selenabury|South Georgia and...|001-171-716-2175x310| 888.625.0654|ncamacho@boone-si...| 2020-11-15| http://hayden.com/|\n", + "| 60|C681dDd0cc422f7| Kelli| Hardy| Petty Ltd| Huangfort|Sao Tome and Prin...| 020.324.2191x2022| 424-157-8216|kristopher62@oliv...| 2020-12-20|http://www.kidd.com/|\n", + "| 61|a940cE42e035F28| Lynn| Pham|Brennan, Camacho ...| East Pennyshire| Portugal| 846.468.6834x611| 001-248-691-0006|mpham@rios-guzman...| 2020-08-21|https://www.murph...|\n", + "| 62|9Cf5E6AFE0aeBfd| Shelley| Harris|Prince, Malone an...| Port Jasminborough| Togo| 423.098.0315x8373|+1-386-458-8944x1...|zachary96@mitchel...| 2020-12-10|https://www.ryan....|\n", + "| 63|aEcbe5365BbC67D| Eddie| Jimenez| Caldwell Group| West Kristine| Ethiopia|+1-235-657-1073x6306| (026)401-7353x2417|kristiwhitney@ber...| 2022-03-24| http://cherry.com/|\n", + "| 64|FCBdfCEAe20A8Dc| Chloe|Hutchinson| Simon LLC| South Julia| Netherlands| 981-544-9452| +1-288-552-4666x060|leah85@sutton-ter...| 2022-05-15|https://mitchell....|\n", + "| 65|636cBF0835E10ff| Eileen| Lynch|Knight, Abbott an...| Helenborough| Liberia|+1-158-951-4131x5...|001-673-779-6713x680|levigiles@vincent...| 2021-01-02| http://mckay.com/|\n", + "| 66|fF1b6c9E8Fbf1ff| Fernando| Lambert| Church-Banks| Lake Nancy| Lithuania| 497.829.9038| 3863743398|fisherlinda@schae...| 2021-04-23|https://www.vang....|\n", + "| 67|2A13F74EAa7DA6c| Makayla| Cannon| Henderson Inc| Georgeport| New Caledonia|001-215-801-6392x...| 027-609-6460|scottcurtis@hurle...| 2020-01-20|http://www.velazq...|\n", + "| 68|a014Ec1b9FccC1E| Tom| Alvarado| Donaldson-Dougherty| South Sophiaberg| Kiribati| (585)606-2980x2258| 730-797-3594x5614|nicholsonnina@mon...| 2020-08-18|http://odom-masse...|\n", + "| 69|421a109cABDf5fa| Virginia| Dudley| Warren Ltd| Hartbury|French Southern T...| 027.846.3705x14184|+1-439-171-1846x4636|zvalencia@phelps.com| 2021-01-31|http://hunter-esp...|\n", + "| 70|CC68FD1D3Bbbf22| Riley| Good| Wade PLC| Erikaville| Canada| 6977745822| 855-436-7641| alex06@galloway.com| 2020-02-03| http://conway.org/|\n", + "| 71|CBCd2Ac8E3eBDF9|Alexandria| Buck| Keller-Coffey| Nicolasfort| Iran| 078-900-4760x76668| 414-112-8700x68751| lee48@manning.com| 2021-02-20| https://ramsey.org/|\n", + "| 72|Ef859092FbEcC07| Richard| Roth| Conway-Mcbride| New Jasmineshire| Morocco| 581-440-6539| 9857827463|aharper@maddox-to...| 2020-02-23|https://www.brook...|\n", + "| 73|F560f2d3cDFb618| Candice| Keller| Huynh and Sons| East Summerstad| Zimbabwe|001-927-965-8550x...|001-243-038-4271x...|buckleycory@odonn...| 2020-08-22|https://www.lucer...|\n", + "| 74|A3F76Be153Df4a3| Anita| Benson| Parrish Ltd| Skinnerport| Russian Federation| 874.617.5668x69878| (399)820-6418x0071|angie04@oconnell.com| 2020-02-09| http://oconnor.com/|\n", + "| 75|D01Af0AF7cBbFeA| Regina| Stein| Guzman-Brown| Raystad| Solomon Islands|001-469-848-0724x...|001-085-360-4426x...|zrosario@rojas-ha...| 2022-01-15|http://www.johnst...|\n", + "| 76|d40e89dCade7b2F| Debra| Riddle|Chang, Aguirre an...| Colinhaven|United States Vir...|+1-768-182-6014x1...| (303)961-4491|shieldskerry@robl...| 2020-07-11| http://kaiser.info/|\n", + "| 77|BF6a1f9bd1bf8DE| Brittany| Zuniga| Mason-Hester| West Reginald| Kyrgyz Republic| (050)136-9025|001-480-851-2496x...|mchandler@cochran...| 2021-07-24|http://www.boyle....|\n", + "| 78|FfaeFFbbbf280db| Cassidy| Mcmahon|Mcguire, Huynh an...| Lake Sherryborough| Myanmar| 5040771311| 684-682-0021x1326|katrinalane@fitzg...| 2020-10-21| https://hurst.com/|\n", + "| 79|CbAE1d1e9a8dCb1| Laurie|Pennington|Sanchez, Marsh an...|Port Katherineville| Dominica| 007.155.3406x553| +1-809-862-5566x277|cookejill@powell.com| 2020-06-08|http://www.hebert...|\n", + "| 80|A7F85c1DE4dB87f| Alejandro| Blair|Combs, Waller and...| Thomasland| Iceland| (690)068-4641x51468| 555.509.8691x2329|elizabethbarr@ewi...| 2020-09-19|https://mercado-b...|\n", + "| 81|D6CEAfb3BDbaa1A| Leslie| Jennings| Blankenship-Arias| Coreybury| Micronesia| 629.198.6346| 075.256.0829| corey75@wiggins.com| 2021-11-13|https://www.juare...|\n", + "| 82|Ebdb6F6F7c90b69| Kathleen| Mckay|Coffey, Lamb and ...| Lake Janiceton|Saint Vincent and...| (733)910-9968| (691)247-4128x0665|chloelester@higgi...| 2021-09-12|http://www.owens-...|\n", + "| 83|E8E7e8Cfe516ef0| Hunter| Moreno|Fitzpatrick-Lawrence| East Clinton| Isle of Man| (733)833-6754| 001-761-013-7121|isaac26@benton-fi...| 2020-12-28| http://walls.info/|\n", + "| 84|78C06E9b6B3DF20| Chad| Davidson| Garcia-Jimenez| South Joshuashire| Oman| 8275702958| (804)842-4715|justinwalters@jim...| 2021-11-15|http://www.garner...|\n", + "| 85|03A1E62ADdeb31c| Corey| Holt|Mcdonald, Bird an...| New Glenda| Fiji|001-439-242-4986x...| 3162708934|maurice46@morgan.com| 2020-02-18|http://www.watson...|\n", + "| 86|C6763c99d0bd16D| Emma|Cunningham| Stephens Inc| North Jillianview| New Zealand| 128-059-0206x60217| (312)164-4545x2284| walter83@juarez.org| 2022-05-13|http://www.reid.i...|\n", + "| 87|ebe77E5Bf9476CE| Duane| Woods| Montoya-Miller| Lyonsberg| Maldives| (636)544-7783x7288| (203)287-1003x5932| kmercer@wagner.com| 2020-07-21| http://murray.org/|\n", + "| 88|E4Bbcd8AD81fC5f| Alison| Vargas|Vaughn, Watts and...| East Cristinabury| Benin| 365-273-8144| 053-308-7653x6287| vcantu@norton.com| 2020-11-10| http://mason.info/|\n", + "| 89|efeb73245CDf1fF| Vernon| Kane| Carter-Strickland| Thomasfurt| Yemen| 114-854-1159x555| 499-608-4612|hilljesse@barrett...| 2021-04-15|http://www.duffy-...|\n", + "| 90|37Ec4B395641c1E| Lori| Flowers| Decker-Mcknight| North Joeburgh| Namibia| 679.415.1210| 945-842-3659x4581|tyrone77@valenzue...| 2021-01-09|http://www.deleon...|\n", + "| 91|5ef6d3eefdD43bE| Nina| Chavez| Byrd-Campbell| Cassidychester| Bhutan| 053-344-3205| +1-330-920-5422x571|elliserica@frank.com| 2020-03-26|https://www.pugh....|\n", + "| 92|98b3aeDcC3B9FF3| Shane| Foley| Rocha-Hart| South Dannymouth| Hungary| -1692|001-626-114-5844x...| nsteele@sparks.com| 2021-07-06|https://www.holt-...|\n", + "| 93|aAb6AFc7AfD0fF3| Collin| Ayers| Lamb-Peterson| South Lonnie| Anguilla| 404-645-5351x012|001-257-582-8850x...|dudleyemily@gonza...| 2021-06-29|http://www.ruiz.com/|\n", + "| 94|54B5B5Fe9F1B6C5| Sherry| Young|Lee, Lucero and J...| Frankchester| Solomon Islands| 158-687-1764| (438)375-6207x003|alan79@gates-mcla...| 2021-04-04| https://travis.net/|\n", + "| 95|BE91A0bdcA49Bbc| Darrell| Douglas|Newton, Petersen ...| Daisyborough| Mali|001-084-845-9524x...| 001-769-564-6303|grayjean@lowery-g...| 2022-02-17| https://banks.biz/|\n", + "| 96|cb8E23e48d22Eae| Karl| Greer| Carey LLC| East Richard| Guyana| (188)169-1674x58692|001-841-293-3519x614| hhart@jensen.com| 2022-01-30|http://hayes-pere...|\n", + "| 97|CeD220bdAaCfaDf| Lynn| Atkinson|Ware, Burns and O...| New Bradview| Sri Lanka| -3769| 605.413.3198| vkemp@ferrell.com| 2021-07-10|https://novak-all...|\n", + "| 98|28CDbC0dFe4b1Db| Fred| Guerra| Schmitt-Jones| Ortegaland| Solomon Islands|+1-753-067-8419x7170|+1-632-666-7507x9...| swagner@kane.org| 2021-09-18|https://www.ross....|\n", + "| 99|c23d1D9EE8DEB0A| Yvonne| Farmer| Fitzgerald-Harrell| Lake Elijahview| Aruba| (530)311-9786|001-869-452-0943x...|mccarthystephen@h...| 2021-08-11|http://watkins.info/|\n", + "| 100|2354a0E336A91A1| Clarence| Haynes| Le, Nash and Cross| Judymouth| Honduras| (753)813-6941| 783.639.1472|colleen91@faulkne...| 2020-03-11|http://www.hatfie...|\n", + "+-----+---------------+----------+----------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----------------+--------------------+\n", + "\n" + ] + } + ], + "source": [ + "null_df.na.fill('Missing').show(100)" + ] + }, + { + "cell_type": "code", + "execution_count": 100, + "id": "d0ccd454-f85c-44d9-89e1-68e1a498d2fa", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-----+---------------+----------+----------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----------------+--------------------+\n", + "|Index| Customer Id|First Name| Last Name| Company| City| Country| Phone 1| Phone 2| Email|Subscription Date| Website|\n", + "+-----+---------------+----------+----------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----------------+--------------------+\n", + "| 1|DD37Cf93aecA6Dc| Sheryl| Baxter| Rasmussen Group| East Leonard| Chile| 229.077.5154| 397.884.0519x718|zunigavanessa@smi...| 2020-08-24|http://www.stephe...|\n", + "| 2|1Ef7b82A4CAAD10| Preston| Lozano| Vega-Gentry| East Jimmychester| Djibouti| 5153435776| 686-620-1820x944| vmata@colon.com| 2021-04-23|http://www.hobbs....|\n", + "| 3|6F94879bDAfE5a6| Roy| Berry| Murillo-Perry| Isabelborough| Antigua and Barbuda| -1199| (496)978-3969x58947| beckycarr@hogan.com| 2020-03-25|http://www.lawren...|\n", + "| 4|5Cef8BFA16c5e3c| Linda| Olsen|Dominguez, Mcmill...| Bensonview| Dominican Republic|001-808-617-6467x...| -9892|stanleyblackwell@...| 2020-06-02|http://www.good-l...|\n", + "| 5|053d585Ab6b3159| Joanna| Bender|Martin, Lang and ...| West Priscilla|Slovakia (Slovak ...|001-234-203-0635x...|001-199-446-3860x...|colinalvarado@mil...| 2021-04-17|https://goodwin-i...|\n", + "| 6|2d08FB17EE273F4| Aimee| Downs| Steele Group| Chavezborough|Bosnia and Herzeg...| (283)437-3886x88321| 999-728-1637| louis27@gilbert.com| 2020-02-25|http://www.berger...|\n", + "| 7|EA4d384DfDbBf77| Darren| Peck|Lester, Woodard a...| Lake Ana| Pitcairn Islands| (496)452-6181x3291|+1-247-266-0963x4995| tgates@cantrell.com| 2021-08-24| https://www.le.com/|\n", + "| 8|0e04AFde9f225dE| Brett| Mullen|Sanford, Davenpor...| Kimport| Bulgaria|001-583-352-7197x297| 001-333-145-0369| asnow@colon.com| 2021-04-12|https://hammond-r...|\n", + "| 9|C2dE4dEEc489ae0| Sheryl| Meyers| Browning-Simon| Robersonstad| Cyprus| 854-138-4911x5772| +1-448-910-2276x729|mariokhan@ryan-po...| 2020-01-13|https://www.bullo...|\n", + "| 10|8C2811a503C7c5a| Michelle| Gallagher| Beck-Hendrix| Elaineberg| Timor-Leste| 739.218.2516x459|001-054-401-0347x617| mdyer@escobar.net| 2021-11-08| https://arias.com/|\n", + "| 11|216E205d6eBb815| Carl| Schroeder|Oconnell, Meza an...| Shannonville| Guernsey| 637-854-0256x825| 114.336.0784x788| kirksalas@webb.com| 2021-10-20|https://simmons-h...|\n", + "| 12|CEDec94deE6d69B| Jenna| Dodson|Hoffman, Reed and...| East Andrea| Vietnam| (041)737-3846|+1-556-888-3485x4...| mark42@robbins.com| 2020-11-29| NULL|\n", + "| 13|e35426EbDEceaFF| Tracey| Mata| Graham-Francis| South Joannamouth| Togo| 001-949-844-8787| (855)713-8773| alex56@walls.org| 2021-12-02|http://www.beck.com/|\n", + "| 14|A08A8aF8BE9FaD4| Kristine| Cox| Carpenter-Cook| Jodyberg| Sri Lanka| 786-284-3358x62152|+1-315-627-1796x8074|holdenmiranda@cla...| 2021-02-08|https://www.brand...|\n", + "| 15|6fEaA1b7cab7B6C| Faith| Lutz| Carter-Hancock| Burchbury| Singapore| (781)861-7180x8306| 207-185-3665|cassieparrish@ble...| 2022-01-26|http://stevenson....|\n", + "| 16|8cad0b4CBceaeec| Miranda| Beasley| NULL| Desireeshire| Oman| 540.085.3135x185|+1-600-462-6432x2...|vduncan@parks-har...| 2022-04-12| http://acosta.org/|\n", + "| 17|a5DC21AE3a21eaA| Caroline| Foley| Winters-Mendoza| West Adriennestad| Western Sahara| 936.222.4746x9924|001-469-948-6341x359|holtgwendolyn@wat...| 2021-03-10|http://www.benson...|\n", + "| 18|F8Aa9d6DfcBeeF8| Greg| Mata| Valentine LLC| Lake Leslie| Mozambique| (701)087-2415| (195)156-1861x26241|jaredjuarez@carro...| 2022-03-26|http://pitts-cher...|\n", + "| 19|F160f5Db3EfE973| Clifford| Jacobson| Simon LLC| Harmonview|South Georgia and...|001-151-330-3524x...| (748)477-7174|joseph26@jacobson...| 2020-09-24|https://mcconnell...|\n", + "| 20|0F60FF3DdCd7aB0| Joanna| Kirk| Mays-Mccormick| Jamesshire| French Polynesia| (266)131-7001x711| (283)312-5579x11543|tuckerangie@salaz...| 2021-09-24|https://www.camac...|\n", + "| 21|9F9AdB7B8A6f7F2| Maxwell| Frye| Patterson Inc| East Carly| Malta| 423.262.3059| 202-880-0688x7491|fgibson@drake-web...| 2022-01-12|http://www.robert...|\n", + "| 22|FBd0Ded4F02a742| Kiara| Houston|Manning, Hester a...| South Alvin| Netherlands|001-274-040-3582x...|+1-528-175-0973x4684|blanchardbob@wall...| 2020-09-15|https://www.reid-...|\n", + "| 23|2FB0FAA1d429421| Colleen| Howard| Greer and Sons| Brittanyview| Paraguay| 1935085151| (947)115-7711x5488|rsingleton@ryan-c...| 2020-08-19| http://paul.biz/|\n", + "| 24|010468dAA11382c| Janet|Valenzuela| Watts-Donaldson| Veronicamouth|Lao People's Demo...| 354.259.5062x7538| 500.433.2022|stefanie71@spence...| 2020-09-08| https://moreno.biz/|\n", + "| 25|eC1927Ca84E033e| Shane| Wilcox| Tucker LLC| Bryanville| Albania| (429)005-9030x11004| 541-116-4501| mariah88@santos.com| NULL|https://www.ramos...|\n", + "| 26|09D7D7C8Fe09aea| Marcus| Moody| Giles Ltd| Kaitlyntown| Panama| 674-677-8623| 909-277-5485x566|donnamullins@norr...| 2022-05-24|https://www.curry...|\n", + "| 27|aBdfcF2c50b0bfD| Dakota| Poole| Simmons Group| Michealshire| Belarus| (371)987-8576x4720| 071-152-1376| stacey67@fields.org| 2022-02-20|https://sanford-w...|\n", + "| 28|b92EBfdF8a3f0E6| Frederick| Harper|Hinton, Chaney an...| South Marissatown| Switzerland|+1-077-121-1558x0687| 264.742.7149|jacobkhan@bright.biz| 2022-05-26|https://callahan....|\n", + "| 29|3B5dAAFA41AFa22| Stefanie| Missing| Santana-Duran| Acevedoville|Saint Vincent and...| (752)776-3286|+1-472-021-4814x8...| wterrell@clark.com| 2020-07-30| https://meyers.com/|\n", + "| 30|EDA69ca7a6e96a2| Kent| Bradshaw| Sawyer PLC| North Harold| Tanzania| +1-472-143-5037x884| 126.922.6153| qjimenez@boyd.com| 2020-04-26|http://maynard-ho...|\n", + "| 31|64DCcDFaB9DFd4e| Jack| Tate|Acosta, Petersen ...| West Samuel| Zimbabwe| 965-108-4406x20714| 046.906.1442x6784|gfigueroa@boone-z...| 2021-09-15|http://www.hawkin...|\n", + "| 32|679c6c83DD872d6| Tom| Trujillo| Mcgee Group| Cunninghamborough| Denmark| 416-338-3758| (775)890-7209|tapiagreg@beard.info| 2022-01-13|http://www.daniel...|\n", + "| 33|7Ce381e4Afa4ba9| Gabriel| Mejia| Adkins-Salinas| Port Annatown| Liechtenstein| 4077245425| 646.044.0696x66800|coleolson@jenning...| 2021-04-24|https://patel-han...|\n", + "| 34|A09AEc6E3bF70eE| Kaitlyn| Santana| Herrera Group| New Kaitlyn|United States of ...| 6303643286| NULL|georgeross@miles.org| 2021-09-21| http://pham.com/|\n", + "| 35|aA9BAFfBc3710fe| Faith| Moon|Waters, Chase and...| West Marthaburgh| Bahamas|+1-586-217-0359x6317| -2419|willistonya@rando...| 2021-11-03|https://spencer-c...|\n", + "| 36|E11dfb2DB8C9f72| Tammie| Haley|Palmer, Barnes an...| East Teresa| Belize|001-276-734-4113x...| (430)300-8770|harrisisaiah@jenk...| 2022-01-04|http://evans-simo...|\n", + "| 37|889eCf90f68c5Da| Nicholas| Sosa| Jordan Ltd| South Hunter| Uruguay| (661)425-6042| 975-998-1519| fwolfe@dorsey.com| 2021-08-10|https://www.flemi...|\n", + "| 38|7a1Ee69F4fF4B4D| Jordan| Gay| Glover and Sons| South Walter| Solomon Islands| 7208417020| 8035336772|tiffanydavies@har...| 2021-02-24| http://www.lee.org/|\n", + "| 39|dca4f1D0A0fc5c9| Bruce| Esparza| Huerta-Mclean| Poolefurt| Montenegro| 559-529-4424|001-625-000-7132x...|preese@frye-vega.com| 2021-10-22|http://www.farley...|\n", + "| 40|17aD8e2dB3df03D| Missing| Garza| Anderson Ltd| West John| Poland|001-067-713-6440x158| (978)289-8785x5766| ann48@miller.com| 2021-11-01| http://spence.com/|\n", + "| 41|2f79Cd309624Abb| Natalie| Gentry| Monroe PLC| West Darius| Dominican Republic| 830.996.8238| 499.122.5415|tcummings@fitzpat...| 2020-10-10|http://www.dorsey...|\n", + "| 42|6e5ad5a5e2bB5Ca| Bryan| Dunn| Kaufman and Sons| North Jimstad| Burkina Faso| 001-710-802-5565| 078.699.8982x13881|woodwardandres@ph...| 2021-09-08|http://www.butler...|\n", + "| 43|7E441b6B228DBcA| Wayne| Simpson| Perkins-Trevino|East Rebekahborough| Bolivia| (344)156-8632x1869| 463-445-3702x38463|barbarapittman@ho...| 2020-12-13|https://gillespie...|\n", + "| 44|D3fC11A9C235Dc6| Luis| Greer| Cross PLC| North Drew| Bulgaria|001-336-025-6849x701| 684.698.2911x6092|bstuart@williamso...| 2022-05-15|https://fletcher-...|\n", + "| 45|30Dfa48fe5Ede78| Rhonda| Frost|Herrera, Shepherd...| Lake Lindaburgh| Monaco| (127)081-9339|+1-431-028-3337x3492|zkrueger@wolf-cha...| 2021-12-06|http://www.khan.com/|\n", + "| 46|fD780ED8dbEae7B| Joanne| Missing|Price, Sexton and...| Gwendolynview| Palau| (897)726-7952| (467)886-9467x5721| juan80@henson.net| 2020-07-01| http://ochoa.com/|\n", + "| 47|300A40d3ce24bBA| Geoffrey| Guzman| Short-Wiggins| NULL| Uzbekistan| 975.235.8921x269| (983)188-6873|bauercrystal@gay.com| 2020-04-23|https://decker-kl...|\n", + "| 48|283DFCD0Dba40aF| Gloria| Mccall|Brennan, Acosta a...| North Kerriton| Ghana| 445-603-6729|001-395-959-4736x...|bartlettjenna@zun...| 2022-03-11|http://burgess-fr...|\n", + "| 49|F4Fc91fEAEad286| Brady| Cohen| Osborne-Erickson| North Eileenville|United Arab Emirates| 741.849.0139x524|+1-028-691-7497x0894|mccalltyrone@durh...| 2022-03-10|http://hammond-ba...|\n", + "| 50|80F33Fd2AcebF05| Latoya| Mccann|Hobbs, Garrett an...| Port Sergiofort| Belarus| (530)287-4548x29481| 162-234-0249x32790|bobhammond@barry.biz| 2021-12-02|https://www.burto...|\n", + "| 51|Aa20BDe68eAb0e9| Gerald| Hawkins|Phelps, Forbes an...| New Alberttown| Canada|+1-323-239-1456x9...| (092)508-0269|uwarner@steele-ar...| 2021-03-19|https://valenzuel...|\n", + "| 52|e898eEB1B9FE22b| Samuel| Crawford|May, Goodwin and ...| South Jasmine| Algeria| 802-242-7457| 626.116.9535x8578|xpittman@ritter-c...| 2021-03-27|https://guerrero....|\n", + "| 53|faCEF517ae7D8eB| Patricia| Goodwin|Christian, Winter...| Cowanfort| Swaziland| 322.549.7139x70040| (111)741-4173|vaughanchristy@la...| 2021-03-08| http://clark.info/|\n", + "| 54|c09952De6Cda8aA| Stacie| Richard| Byrd Inc| New Deborah| Madagascar|001-622-948-3641x...|001-731-168-2893x...|clinton85@colon-a...| 2020-10-15| https://kim.com/|\n", + "| 55|f3BEf3Be028166f| Robin| West|Nixon, Blackwell ...| Wallstown| Ecuador| 698.303.4267|001-683-837-7651x525|greenemiranda@zim...| 2022-01-13|https://www.mora....|\n", + "| 56|C6F2Fc6a7948a4e| Ralph| Haas| Montes PLC| Lake Ellenchester|Palestinian Terri...| 2239271999|001-962-434-0867x649|goodmancesar@figu...| 2020-05-25| http://may.com/|\n", + "| 57|c8FE57cBBdCDcb2| Phyllis| Maldonado| Costa PLC| Lake Whitney| Saint Barthelemy| 4500370767|001-508-064-6725x017|yhanson@warner-di...| 2021-01-25|http://www.bernar...|\n", + "| 58|B5acdFC982124F2| Danny| Parrish| Novak LLC| East Jaredbury|United Arab Emirates| (669)384-8597x8794| 506.731.5952x571|howelldarren@hous...| 2021-03-17|http://www.parson...|\n", + "| 59|8c7DdF10798bCC3| Kathy| Hill|Moore, Mccoy and ...| Selenabury|South Georgia and...|001-171-716-2175x310| 888.625.0654|ncamacho@boone-si...| 2020-11-15| http://hayden.com/|\n", + "| 60|C681dDd0cc422f7| Kelli| Hardy| Petty Ltd| Huangfort|Sao Tome and Prin...| 020.324.2191x2022| 424-157-8216|kristopher62@oliv...| 2020-12-20|http://www.kidd.com/|\n", + "| 61|a940cE42e035F28| Lynn| Pham|Brennan, Camacho ...| East Pennyshire| Portugal| 846.468.6834x611| 001-248-691-0006|mpham@rios-guzman...| 2020-08-21|https://www.murph...|\n", + "| 62|9Cf5E6AFE0aeBfd| Shelley| Harris|Prince, Malone an...| Port Jasminborough| Togo| 423.098.0315x8373|+1-386-458-8944x1...|zachary96@mitchel...| 2020-12-10|https://www.ryan....|\n", + "| 63|aEcbe5365BbC67D| Eddie| Jimenez| Caldwell Group| West Kristine| Ethiopia|+1-235-657-1073x6306| (026)401-7353x2417|kristiwhitney@ber...| 2022-03-24| http://cherry.com/|\n", + "| 64|FCBdfCEAe20A8Dc| Chloe|Hutchinson| Simon LLC| South Julia| Netherlands| 981-544-9452| +1-288-552-4666x060|leah85@sutton-ter...| 2022-05-15|https://mitchell....|\n", + "| 65|636cBF0835E10ff| Eileen| Lynch|Knight, Abbott an...| Helenborough| Liberia|+1-158-951-4131x5...|001-673-779-6713x680|levigiles@vincent...| 2021-01-02| http://mckay.com/|\n", + "| 66|fF1b6c9E8Fbf1ff| Fernando| Lambert| Church-Banks| Lake Nancy| Lithuania| 497.829.9038| 3863743398|fisherlinda@schae...| 2021-04-23|https://www.vang....|\n", + "| 67|2A13F74EAa7DA6c| Makayla| Cannon| Henderson Inc| Georgeport| New Caledonia|001-215-801-6392x...| 027-609-6460|scottcurtis@hurle...| 2020-01-20|http://www.velazq...|\n", + "| 68|a014Ec1b9FccC1E| Tom| Alvarado| Donaldson-Dougherty| South Sophiaberg| Kiribati| (585)606-2980x2258| 730-797-3594x5614|nicholsonnina@mon...| 2020-08-18|http://odom-masse...|\n", + "| 69|421a109cABDf5fa| Virginia| Dudley| Warren Ltd| Hartbury|French Southern T...| 027.846.3705x14184|+1-439-171-1846x4636|zvalencia@phelps.com| 2021-01-31|http://hunter-esp...|\n", + "| 70|CC68FD1D3Bbbf22| Riley| Good| Wade PLC| Erikaville| Canada| 6977745822| 855-436-7641| alex06@galloway.com| 2020-02-03| http://conway.org/|\n", + "| 71|CBCd2Ac8E3eBDF9|Alexandria| Buck| Keller-Coffey| Nicolasfort| Iran| 078-900-4760x76668| 414-112-8700x68751| lee48@manning.com| 2021-02-20| https://ramsey.org/|\n", + "| 72|Ef859092FbEcC07| Richard| Roth| Conway-Mcbride| New Jasmineshire| Morocco| 581-440-6539| 9857827463|aharper@maddox-to...| 2020-02-23|https://www.brook...|\n", + "| 73|F560f2d3cDFb618| Candice| Keller| Huynh and Sons| East Summerstad| Zimbabwe|001-927-965-8550x...|001-243-038-4271x...|buckleycory@odonn...| 2020-08-22|https://www.lucer...|\n", + "| 74|A3F76Be153Df4a3| Anita| Benson| Parrish Ltd| Skinnerport| Russian Federation| 874.617.5668x69878| (399)820-6418x0071|angie04@oconnell.com| 2020-02-09| http://oconnor.com/|\n", + "| 75|D01Af0AF7cBbFeA| Regina| Stein| Guzman-Brown| Raystad| Solomon Islands|001-469-848-0724x...|001-085-360-4426x...|zrosario@rojas-ha...| 2022-01-15|http://www.johnst...|\n", + "| 76|d40e89dCade7b2F| Debra| Riddle|Chang, Aguirre an...| Colinhaven|United States Vir...|+1-768-182-6014x1...| (303)961-4491|shieldskerry@robl...| 2020-07-11| http://kaiser.info/|\n", + "| 77|BF6a1f9bd1bf8DE| Brittany| Zuniga| Mason-Hester| West Reginald| Kyrgyz Republic| (050)136-9025|001-480-851-2496x...|mchandler@cochran...| 2021-07-24|http://www.boyle....|\n", + "| 78|FfaeFFbbbf280db| Cassidy| Mcmahon|Mcguire, Huynh an...| Lake Sherryborough| Myanmar| 5040771311| 684-682-0021x1326|katrinalane@fitzg...| 2020-10-21| https://hurst.com/|\n", + "| 79|CbAE1d1e9a8dCb1| Laurie|Pennington|Sanchez, Marsh an...|Port Katherineville| Dominica| 007.155.3406x553| +1-809-862-5566x277|cookejill@powell.com| 2020-06-08|http://www.hebert...|\n", + "| 80|A7F85c1DE4dB87f| Alejandro| Blair|Combs, Waller and...| Thomasland| Iceland| (690)068-4641x51468| 555.509.8691x2329|elizabethbarr@ewi...| 2020-09-19|https://mercado-b...|\n", + "| 81|D6CEAfb3BDbaa1A| Leslie| Jennings| Blankenship-Arias| Coreybury| Micronesia| 629.198.6346| 075.256.0829| corey75@wiggins.com| 2021-11-13|https://www.juare...|\n", + "| 82|Ebdb6F6F7c90b69| Kathleen| Mckay|Coffey, Lamb and ...| Lake Janiceton|Saint Vincent and...| (733)910-9968| (691)247-4128x0665|chloelester@higgi...| 2021-09-12|http://www.owens-...|\n", + "| 83|E8E7e8Cfe516ef0| Hunter| Moreno|Fitzpatrick-Lawrence| East Clinton| Isle of Man| (733)833-6754| 001-761-013-7121|isaac26@benton-fi...| 2020-12-28| http://walls.info/|\n", + "| 84|78C06E9b6B3DF20| Chad| Davidson| Garcia-Jimenez| South Joshuashire| Oman| 8275702958| (804)842-4715|justinwalters@jim...| 2021-11-15|http://www.garner...|\n", + "| 85|03A1E62ADdeb31c| Corey| Holt|Mcdonald, Bird an...| New Glenda| Fiji|001-439-242-4986x...| 3162708934|maurice46@morgan.com| 2020-02-18|http://www.watson...|\n", + "| 86|C6763c99d0bd16D| Emma|Cunningham| Stephens Inc| North Jillianview| New Zealand| 128-059-0206x60217| (312)164-4545x2284| walter83@juarez.org| 2022-05-13|http://www.reid.i...|\n", + "| 87|ebe77E5Bf9476CE| Duane| Woods| Montoya-Miller| Lyonsberg| Maldives| (636)544-7783x7288| (203)287-1003x5932| kmercer@wagner.com| 2020-07-21| http://murray.org/|\n", + "| 88|E4Bbcd8AD81fC5f| Alison| Vargas|Vaughn, Watts and...| East Cristinabury| Benin| 365-273-8144| 053-308-7653x6287| vcantu@norton.com| 2020-11-10| http://mason.info/|\n", + "| 89|efeb73245CDf1fF| Vernon| Kane| Carter-Strickland| Thomasfurt| Yemen| 114-854-1159x555| 499-608-4612|hilljesse@barrett...| 2021-04-15|http://www.duffy-...|\n", + "| 90|37Ec4B395641c1E| Lori| Flowers| Decker-Mcknight| North Joeburgh| Namibia| 679.415.1210| 945-842-3659x4581|tyrone77@valenzue...| 2021-01-09|http://www.deleon...|\n", + "| 91|5ef6d3eefdD43bE| Nina| Chavez| Byrd-Campbell| Cassidychester| Bhutan| 053-344-3205| +1-330-920-5422x571|elliserica@frank.com| 2020-03-26|https://www.pugh....|\n", + "| 92|98b3aeDcC3B9FF3| Shane| Foley| Rocha-Hart| South Dannymouth| Hungary| -1692|001-626-114-5844x...| nsteele@sparks.com| 2021-07-06|https://www.holt-...|\n", + "| 93|aAb6AFc7AfD0fF3| Collin| Ayers| Lamb-Peterson| South Lonnie| Anguilla| 404-645-5351x012|001-257-582-8850x...|dudleyemily@gonza...| 2021-06-29|http://www.ruiz.com/|\n", + "| 94|54B5B5Fe9F1B6C5| Sherry| Young|Lee, Lucero and J...| Frankchester| Solomon Islands| 158-687-1764| (438)375-6207x003|alan79@gates-mcla...| 2021-04-04| https://travis.net/|\n", + "| 95|BE91A0bdcA49Bbc| Darrell| Douglas|Newton, Petersen ...| Daisyborough| Mali|001-084-845-9524x...| 001-769-564-6303|grayjean@lowery-g...| 2022-02-17| https://banks.biz/|\n", + "| 96|cb8E23e48d22Eae| Karl| Greer| Carey LLC| East Richard| Guyana| (188)169-1674x58692|001-841-293-3519x614| hhart@jensen.com| 2022-01-30|http://hayes-pere...|\n", + "| 97|CeD220bdAaCfaDf| Lynn| Atkinson|Ware, Burns and O...| New Bradview| Sri Lanka| -3769| 605.413.3198| vkemp@ferrell.com| 2021-07-10|https://novak-all...|\n", + "| 98|28CDbC0dFe4b1Db| Fred| Guerra| Schmitt-Jones| Ortegaland| Solomon Islands|+1-753-067-8419x7170|+1-632-666-7507x9...| swagner@kane.org| 2021-09-18|https://www.ross....|\n", + "| 99|c23d1D9EE8DEB0A| Yvonne| Farmer| Fitzgerald-Harrell| Lake Elijahview| Aruba| (530)311-9786|001-869-452-0943x...|mccarthystephen@h...| 2021-08-11|http://watkins.info/|\n", + "| 100|2354a0E336A91A1| Clarence| Haynes| Le, Nash and Cross| Judymouth| Honduras| (753)813-6941| 783.639.1472|colleen91@faulkne...| 2020-03-11|http://www.hatfie...|\n", + "+-----+---------------+----------+----------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----------------+--------------------+\n", + "\n" + ] + } + ], + "source": [ + "null_df.na.fill('Missing', ['First Name','Last name']).show(100)" + ] + }, + { + "cell_type": "code", + "execution_count": 110, + "id": "d695058f-a842-4875-9d3b-0f5eda24748c", + "metadata": {}, + "outputs": [], + "source": [ + "## Impute\n", + "\n", + "from pyspark.ml.feature import Imputer\n", + "imputer= Imputer(\n", + " inputCols=['First Name', 'Last Name', 'Company','City'],\n", + " outputCols=[\"{}_imputed\".format(c) for c in ['First Name', 'Last Name', 'Company','City']]\n", + ").setStrategy('median')" + ] + }, + { + "cell_type": "code", + "execution_count": 111, + "id": "b7243ced-b34b-483d-9afc-3efc96d8d713", + "metadata": {}, + "outputs": [ + { + "ename": "IllegalArgumentException", + "evalue": "requirement failed: Column First Name must be of type numeric but was actually of type string.", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mIllegalArgumentException\u001b[0m Traceback (most recent call last)", + "Cell \u001b[1;32mIn[111], line 2\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[38;5;66;03m## Add imputations\u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m \u001b[43mimputer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnull_df\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mtransform(null_df)\u001b[38;5;241m.\u001b[39mshow()\n", + "File \u001b[1;32m~\\.virtualenvs\\datascience_practice_python-XzMsjZK6\\Lib\\site-packages\\pyspark\\ml\\base.py:205\u001b[0m, in \u001b[0;36mEstimator.fit\u001b[1;34m(self, dataset, params)\u001b[0m\n\u001b[0;32m 203\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcopy(params)\u001b[38;5;241m.\u001b[39m_fit(dataset)\n\u001b[0;32m 204\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m--> 205\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_fit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdataset\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 206\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m 207\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\n\u001b[0;32m 208\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mParams must be either a param map or a list/tuple of param maps, \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 209\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbut got \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m \u001b[38;5;28mtype\u001b[39m(params)\n\u001b[0;32m 210\u001b[0m )\n", + "File \u001b[1;32m~\\.virtualenvs\\datascience_practice_python-XzMsjZK6\\Lib\\site-packages\\pyspark\\ml\\wrapper.py:381\u001b[0m, in \u001b[0;36mJavaEstimator._fit\u001b[1;34m(self, dataset)\u001b[0m\n\u001b[0;32m 380\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_fit\u001b[39m(\u001b[38;5;28mself\u001b[39m, dataset: DataFrame) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m JM:\n\u001b[1;32m--> 381\u001b[0m java_model \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_fit_java\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdataset\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 382\u001b[0m model \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_create_model(java_model)\n\u001b[0;32m 383\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_copyValues(model)\n", + "File \u001b[1;32m~\\.virtualenvs\\datascience_practice_python-XzMsjZK6\\Lib\\site-packages\\pyspark\\ml\\wrapper.py:378\u001b[0m, in \u001b[0;36mJavaEstimator._fit_java\u001b[1;34m(self, dataset)\u001b[0m\n\u001b[0;32m 375\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_java_obj \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m 377\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_transfer_params_to_java()\n\u001b[1;32m--> 378\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_java_obj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdataset\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_jdf\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[1;32m~\\.virtualenvs\\datascience_practice_python-XzMsjZK6\\Lib\\site-packages\\py4j\\java_gateway.py:1322\u001b[0m, in \u001b[0;36mJavaMember.__call__\u001b[1;34m(self, *args)\u001b[0m\n\u001b[0;32m 1316\u001b[0m command \u001b[38;5;241m=\u001b[39m proto\u001b[38;5;241m.\u001b[39mCALL_COMMAND_NAME \u001b[38;5;241m+\u001b[39m\\\n\u001b[0;32m 1317\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcommand_header \u001b[38;5;241m+\u001b[39m\\\n\u001b[0;32m 1318\u001b[0m args_command \u001b[38;5;241m+\u001b[39m\\\n\u001b[0;32m 1319\u001b[0m proto\u001b[38;5;241m.\u001b[39mEND_COMMAND_PART\n\u001b[0;32m 1321\u001b[0m answer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgateway_client\u001b[38;5;241m.\u001b[39msend_command(command)\n\u001b[1;32m-> 1322\u001b[0m return_value \u001b[38;5;241m=\u001b[39m \u001b[43mget_return_value\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m 1323\u001b[0m \u001b[43m \u001b[49m\u001b[43manswer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgateway_client\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtarget_id\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 1325\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m temp_arg \u001b[38;5;129;01min\u001b[39;00m temp_args:\n\u001b[0;32m 1326\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(temp_arg, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_detach\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n", + "File \u001b[1;32m~\\.virtualenvs\\datascience_practice_python-XzMsjZK6\\Lib\\site-packages\\pyspark\\errors\\exceptions\\captured.py:185\u001b[0m, in \u001b[0;36mcapture_sql_exception..deco\u001b[1;34m(*a, **kw)\u001b[0m\n\u001b[0;32m 181\u001b[0m converted \u001b[38;5;241m=\u001b[39m convert_exception(e\u001b[38;5;241m.\u001b[39mjava_exception)\n\u001b[0;32m 182\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(converted, UnknownException):\n\u001b[0;32m 183\u001b[0m \u001b[38;5;66;03m# Hide where the exception came from that shows a non-Pythonic\u001b[39;00m\n\u001b[0;32m 184\u001b[0m \u001b[38;5;66;03m# JVM exception message.\u001b[39;00m\n\u001b[1;32m--> 185\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m converted \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m 186\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m 187\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m\n", + "\u001b[1;31mIllegalArgumentException\u001b[0m: requirement failed: Column First Name must be of type numeric but was actually of type string." + ] + } + ], + "source": [ + "## Add imputations\n", + "imputer.fit(null_df).transform(null_df).show()\n" + ] + }, { "cell_type": "code", "execution_count": null, - "id": "b1833e21-0732-4198-8987-eee8f3f4ad39", + "id": "a0b616ff-9d58-4dfb-9b1d-b5f90b12ceaa", "metadata": {}, "outputs": [], "source": [] diff --git a/practice_pyspark/customers-100-na.csv b/practice_pyspark/customers-100-na.csv new file mode 100644 index 0000000..119e05e --- /dev/null +++ b/practice_pyspark/customers-100-na.csv @@ -0,0 +1,101 @@ +Index,Customer Id,First Name,Last Name,Company,City,Country,Phone 1,Phone 2,Email,Subscription Date,Website +1,DD37Cf93aecA6Dc,Sheryl,Baxter,Rasmussen Group,East Leonard,Chile,229.077.5154,397.884.0519x718,zunigavanessa@smith.info,2020-08-24,http://www.stephenson.com/ +2,1Ef7b82A4CAAD10,Preston,Lozano,Vega-Gentry,East Jimmychester,Djibouti,5153435776,686-620-1820x944,vmata@colon.com,2021-04-23,http://www.hobbs.com/ +3,6F94879bDAfE5a6,Roy,Berry,Murillo-Perry,Isabelborough,Antigua and Barbuda,-1199,(496)978-3969x58947,beckycarr@hogan.com,2020-03-25,http://www.lawrence.com/ +4,5Cef8BFA16c5e3c,Linda,Olsen,"Dominguez, Mcmillan and Donovan",Bensonview,Dominican Republic,001-808-617-6467x12895,-9892,stanleyblackwell@benson.org,2020-06-02,http://www.good-lyons.com/ +5,053d585Ab6b3159,Joanna,Bender,"Martin, Lang and Andrade",West Priscilla,Slovakia (Slovak Republic),001-234-203-0635x76146,001-199-446-3860x3486,colinalvarado@miles.net,2021-04-17,https://goodwin-ingram.com/ +6,2d08FB17EE273F4,Aimee,Downs,Steele Group,Chavezborough,Bosnia and Herzegovina,(283)437-3886x88321,999-728-1637,louis27@gilbert.com,2020-02-25,http://www.berger.net/ +7,EA4d384DfDbBf77,Darren,Peck,"Lester, Woodard and Mitchell",Lake Ana,Pitcairn Islands,(496)452-6181x3291,+1-247-266-0963x4995,tgates@cantrell.com,2021-08-24,https://www.le.com/ +8,0e04AFde9f225dE,Brett,Mullen,"Sanford, Davenport and Giles",Kimport,Bulgaria,001-583-352-7197x297,001-333-145-0369,asnow@colon.com,2021-04-12,https://hammond-ramsey.com/ +9,C2dE4dEEc489ae0,Sheryl,Meyers,Browning-Simon,Robersonstad,Cyprus,854-138-4911x5772,+1-448-910-2276x729,mariokhan@ryan-pope.org,2020-01-13,https://www.bullock.net/ +10,8C2811a503C7c5a,Michelle,Gallagher,Beck-Hendrix,Elaineberg,Timor-Leste,739.218.2516x459,001-054-401-0347x617,mdyer@escobar.net,2021-11-08,https://arias.com/ +11,216E205d6eBb815,Carl,Schroeder,"Oconnell, Meza and Everett",Shannonville,Guernsey,637-854-0256x825,114.336.0784x788,kirksalas@webb.com,2021-10-20,https://simmons-hurley.com/ +12,CEDec94deE6d69B,Jenna,Dodson,"Hoffman, Reed and Mcclain",East Andrea,Vietnam,(041)737-3846,+1-556-888-3485x42608,mark42@robbins.com,2020-11-29, +13,e35426EbDEceaFF,Tracey,Mata,Graham-Francis,South Joannamouth,Togo,001-949-844-8787,(855)713-8773,alex56@walls.org,2021-12-02,http://www.beck.com/ +14,A08A8aF8BE9FaD4,Kristine,Cox,Carpenter-Cook,Jodyberg,Sri Lanka,786-284-3358x62152,+1-315-627-1796x8074,holdenmiranda@clarke.com,2021-02-08,https://www.brandt.com/ +15,6fEaA1b7cab7B6C,Faith,Lutz,Carter-Hancock,Burchbury,Singapore,(781)861-7180x8306,207-185-3665,cassieparrish@blevins-chapman.net,2022-01-26,http://stevenson.org/ +16,8cad0b4CBceaeec,Miranda,Beasley,,Desireeshire,Oman,540.085.3135x185,+1-600-462-6432x21881,vduncan@parks-hardy.com,2022-04-12,http://acosta.org/ +17,a5DC21AE3a21eaA,Caroline,Foley,Winters-Mendoza,West Adriennestad,Western Sahara,936.222.4746x9924,001-469-948-6341x359,holtgwendolyn@watson-davenport.com,2021-03-10,http://www.benson-roth.com/ +18,F8Aa9d6DfcBeeF8,Greg,Mata,Valentine LLC,Lake Leslie,Mozambique,(701)087-2415,(195)156-1861x26241,jaredjuarez@carroll.org,2022-03-26,http://pitts-cherry.com/ +19,F160f5Db3EfE973,Clifford,Jacobson,Simon LLC,Harmonview,South Georgia and the South Sandwich Islands,001-151-330-3524x0469,(748)477-7174,joseph26@jacobson.com,2020-09-24,https://mcconnell.com/ +20,0F60FF3DdCd7aB0,Joanna,Kirk,Mays-Mccormick,Jamesshire,French Polynesia,(266)131-7001x711,(283)312-5579x11543,tuckerangie@salazar.net,2021-09-24,https://www.camacho.net/ +21,9F9AdB7B8A6f7F2,Maxwell,Frye,Patterson Inc,East Carly,Malta,423.262.3059,202-880-0688x7491,fgibson@drake-webb.com,2022-01-12,http://www.roberts.com/ +22,FBd0Ded4F02a742,Kiara,Houston,"Manning, Hester and Arroyo",South Alvin,Netherlands,001-274-040-3582x10611,+1-528-175-0973x4684,blanchardbob@wallace-shannon.com,2020-09-15,https://www.reid-potts.com/ +23,2FB0FAA1d429421,Colleen,Howard,Greer and Sons,Brittanyview,Paraguay,1935085151,(947)115-7711x5488,rsingleton@ryan-cherry.com,2020-08-19,http://paul.biz/ +24,010468dAA11382c,Janet,Valenzuela,Watts-Donaldson,Veronicamouth,Lao People's Democratic Republic,354.259.5062x7538,500.433.2022,stefanie71@spence.com,2020-09-08,https://moreno.biz/ +25,eC1927Ca84E033e,Shane,Wilcox,Tucker LLC,Bryanville,Albania,(429)005-9030x11004,541-116-4501,mariah88@santos.com,,https://www.ramos.com/ +26,09D7D7C8Fe09aea,Marcus,Moody,Giles Ltd,Kaitlyntown,Panama,674-677-8623,909-277-5485x566,donnamullins@norris-barrett.org,2022-05-24,https://www.curry.com/ +27,aBdfcF2c50b0bfD,Dakota,Poole,Simmons Group,Michealshire,Belarus,(371)987-8576x4720,071-152-1376,stacey67@fields.org,2022-02-20,https://sanford-wilcox.biz/ +28,b92EBfdF8a3f0E6,Frederick,Harper,"Hinton, Chaney and Stokes",South Marissatown,Switzerland,+1-077-121-1558x0687,264.742.7149,jacobkhan@bright.biz,2022-05-26,https://callahan.org/ +29,3B5dAAFA41AFa22,Stefanie,,Santana-Duran,Acevedoville,Saint Vincent and the Grenadines,(752)776-3286,+1-472-021-4814x85074,wterrell@clark.com,2020-07-30,https://meyers.com/ +30,EDA69ca7a6e96a2,Kent,Bradshaw,Sawyer PLC,North Harold,Tanzania,+1-472-143-5037x884,126.922.6153,qjimenez@boyd.com,2020-04-26,http://maynard-ho.com/ +31,64DCcDFaB9DFd4e,Jack,Tate,"Acosta, Petersen and Morrow",West Samuel,Zimbabwe,965-108-4406x20714,046.906.1442x6784,gfigueroa@boone-zavala.com,2021-09-15,http://www.hawkins-ramsey.com/ +32,679c6c83DD872d6,Tom,Trujillo,Mcgee Group,Cunninghamborough,Denmark,416-338-3758,(775)890-7209,tapiagreg@beard.info,2022-01-13,http://www.daniels-klein.com/ +33,7Ce381e4Afa4ba9,Gabriel,Mejia,Adkins-Salinas,Port Annatown,Liechtenstein,4077245425,646.044.0696x66800,coleolson@jennings.net,2021-04-24,https://patel-hanson.info/ +34,A09AEc6E3bF70eE,Kaitlyn,Santana,Herrera Group,New Kaitlyn,United States of America,6303643286,,georgeross@miles.org,2021-09-21,http://pham.com/ +35,aA9BAFfBc3710fe,Faith,Moon,"Waters, Chase and Aguilar",West Marthaburgh,Bahamas,+1-586-217-0359x6317,-2419,willistonya@randolph-baker.com,2021-11-03,https://spencer-charles.info/ +36,E11dfb2DB8C9f72,Tammie,Haley,"Palmer, Barnes and Houston",East Teresa,Belize,001-276-734-4113x6087,(430)300-8770,harrisisaiah@jenkins.com,2022-01-04,http://evans-simon.com/ +37,889eCf90f68c5Da,Nicholas,Sosa,Jordan Ltd,South Hunter,Uruguay,(661)425-6042,975-998-1519,fwolfe@dorsey.com,2021-08-10,https://www.fleming-richards.com/ +38,7a1Ee69F4fF4B4D,Jordan,Gay,Glover and Sons,South Walter,Solomon Islands,7208417020,8035336772,tiffanydavies@harris-mcfarland.org,2021-02-24,http://www.lee.org/ +39,dca4f1D0A0fc5c9,Bruce,Esparza,Huerta-Mclean,Poolefurt,Montenegro,559-529-4424,001-625-000-7132x0367,preese@frye-vega.com,2021-10-22,http://www.farley.org/ +40,17aD8e2dB3df03D,,Garza,Anderson Ltd,West John,Poland,001-067-713-6440x158,(978)289-8785x5766,ann48@miller.com,2021-11-01,http://spence.com/ +41,2f79Cd309624Abb,Natalie,Gentry,Monroe PLC,West Darius,Dominican Republic,830.996.8238,499.122.5415,tcummings@fitzpatrick-ashley.com,2020-10-10,http://www.dorsey.biz/ +42,6e5ad5a5e2bB5Ca,Bryan,Dunn,Kaufman and Sons,North Jimstad,Burkina Faso,001-710-802-5565,078.699.8982x13881,woodwardandres@phelps.com,2021-09-08,http://www.butler.com/ +43,7E441b6B228DBcA,Wayne,Simpson,Perkins-Trevino,East Rebekahborough,Bolivia,(344)156-8632x1869,463-445-3702x38463,barbarapittman@holder.com,2020-12-13,https://gillespie-holder.com/ +44,D3fC11A9C235Dc6,Luis,Greer,Cross PLC,North Drew,Bulgaria,001-336-025-6849x701,684.698.2911x6092,bstuart@williamson-mcclure.com,2022-05-15,https://fletcher-nielsen.com/ +45,30Dfa48fe5Ede78,Rhonda,Frost,"Herrera, Shepherd and Underwood",Lake Lindaburgh,Monaco,(127)081-9339,+1-431-028-3337x3492,zkrueger@wolf-chavez.net,2021-12-06,http://www.khan.com/ +46,fD780ED8dbEae7B,Joanne,,"Price, Sexton and Mcdaniel",Gwendolynview,Palau,(897)726-7952,(467)886-9467x5721,juan80@henson.net,2020-07-01,http://ochoa.com/ +47,300A40d3ce24bBA,Geoffrey,Guzman,Short-Wiggins,,Uzbekistan,975.235.8921x269,(983)188-6873,bauercrystal@gay.com,2020-04-23,https://decker-kline.com/ +48,283DFCD0Dba40aF,Gloria,Mccall,"Brennan, Acosta and Ramos",North Kerriton,Ghana,445-603-6729,001-395-959-4736x4524,bartlettjenna@zuniga-moss.biz,2022-03-11,http://burgess-frank.com/ +49,F4Fc91fEAEad286,Brady,Cohen,Osborne-Erickson,North Eileenville,United Arab Emirates,741.849.0139x524,+1-028-691-7497x0894,mccalltyrone@durham-rose.biz,2022-03-10,http://hammond-barron.com/ +50,80F33Fd2AcebF05,Latoya,Mccann,"Hobbs, Garrett and Sanford",Port Sergiofort,Belarus,(530)287-4548x29481,162-234-0249x32790,bobhammond@barry.biz,2021-12-02,https://www.burton.com/ +51,Aa20BDe68eAb0e9,Gerald,Hawkins,"Phelps, Forbes and Koch",New Alberttown,Canada,+1-323-239-1456x96168,(092)508-0269,uwarner@steele-arias.com,2021-03-19,https://valenzuela.com/ +52,e898eEB1B9FE22b,Samuel,Crawford,"May, Goodwin and Martin",South Jasmine,Algeria,802-242-7457,626.116.9535x8578,xpittman@ritter-carney.net,2021-03-27,https://guerrero.org/ +53,faCEF517ae7D8eB,Patricia,Goodwin,"Christian, Winters and Ellis",Cowanfort,Swaziland,322.549.7139x70040,(111)741-4173,vaughanchristy@lara.biz,2021-03-08,http://clark.info/ +54,c09952De6Cda8aA,Stacie,Richard,Byrd Inc,New Deborah,Madagascar,001-622-948-3641x24810,001-731-168-2893x8891,clinton85@colon-arias.org,2020-10-15,https://kim.com/ +55,f3BEf3Be028166f,Robin,West,"Nixon, Blackwell and Sosa",Wallstown,Ecuador,698.303.4267,001-683-837-7651x525,greenemiranda@zimmerman.com,2022-01-13,https://www.mora.com/ +56,C6F2Fc6a7948a4e,Ralph,Haas,Montes PLC,Lake Ellenchester,Palestinian Territory,2239271999,001-962-434-0867x649,goodmancesar@figueroa.biz,2020-05-25,http://may.com/ +57,c8FE57cBBdCDcb2,Phyllis,Maldonado,Costa PLC,Lake Whitney,Saint Barthelemy,4500370767,001-508-064-6725x017,yhanson@warner-diaz.org,2021-01-25,http://www.bernard.com/ +58,B5acdFC982124F2,Danny,Parrish,Novak LLC,East Jaredbury,United Arab Emirates,(669)384-8597x8794,506.731.5952x571,howelldarren@house-cohen.com,2021-03-17,http://www.parsons-hudson.com/ +59,8c7DdF10798bCC3,Kathy,Hill,"Moore, Mccoy and Glass",Selenabury,South Georgia and the South Sandwich Islands,001-171-716-2175x310,888.625.0654,ncamacho@boone-simmons.org,2020-11-15,http://hayden.com/ +60,C681dDd0cc422f7,Kelli,Hardy,Petty Ltd,Huangfort,Sao Tome and Principe,020.324.2191x2022,424-157-8216,kristopher62@oliver.com,2020-12-20,http://www.kidd.com/ +61,a940cE42e035F28,Lynn,Pham,"Brennan, Camacho and Tapia",East Pennyshire,Portugal,846.468.6834x611,001-248-691-0006,mpham@rios-guzman.com,2020-08-21,https://www.murphy.com/ +62,9Cf5E6AFE0aeBfd,Shelley,Harris,"Prince, Malone and Pugh",Port Jasminborough,Togo,423.098.0315x8373,+1-386-458-8944x15194,zachary96@mitchell-bryant.org,2020-12-10,https://www.ryan.com/ +63,aEcbe5365BbC67D,Eddie,Jimenez,Caldwell Group,West Kristine,Ethiopia,+1-235-657-1073x6306,(026)401-7353x2417,kristiwhitney@bernard.com,2022-03-24,http://cherry.com/ +64,FCBdfCEAe20A8Dc,Chloe,Hutchinson,Simon LLC,South Julia,Netherlands,981-544-9452,+1-288-552-4666x060,leah85@sutton-terrell.com,2022-05-15,https://mitchell.info/ +65,636cBF0835E10ff,Eileen,Lynch,"Knight, Abbott and Hubbard",Helenborough,Liberia,+1-158-951-4131x53578,001-673-779-6713x680,levigiles@vincent.com,2021-01-02,http://mckay.com/ +66,fF1b6c9E8Fbf1ff,Fernando,Lambert,Church-Banks,Lake Nancy,Lithuania,497.829.9038,3863743398,fisherlinda@schaefer.net,2021-04-23,https://www.vang.com/ +67,2A13F74EAa7DA6c,Makayla,Cannon,Henderson Inc,Georgeport,New Caledonia,001-215-801-6392x46009,027-609-6460,scottcurtis@hurley.biz,2020-01-20,http://www.velazquez.net/ +68,a014Ec1b9FccC1E,Tom,Alvarado,Donaldson-Dougherty,South Sophiaberg,Kiribati,(585)606-2980x2258,730-797-3594x5614,nicholsonnina@montgomery.info,2020-08-18,http://odom-massey.com/ +69,421a109cABDf5fa,Virginia,Dudley,Warren Ltd,Hartbury,French Southern Territories,027.846.3705x14184,+1-439-171-1846x4636,zvalencia@phelps.com,2021-01-31,http://hunter-esparza.com/ +70,CC68FD1D3Bbbf22,Riley,Good,Wade PLC,Erikaville,Canada,6977745822,855-436-7641,alex06@galloway.com,2020-02-03,http://conway.org/ +71,CBCd2Ac8E3eBDF9,Alexandria,Buck,Keller-Coffey,Nicolasfort,Iran,078-900-4760x76668,414-112-8700x68751,lee48@manning.com,2021-02-20,https://ramsey.org/ +72,Ef859092FbEcC07,Richard,Roth,Conway-Mcbride,New Jasmineshire,Morocco,581-440-6539,9857827463,aharper@maddox-townsend.org,2020-02-23,https://www.brooks.com/ +73,F560f2d3cDFb618,Candice,Keller,Huynh and Sons,East Summerstad,Zimbabwe,001-927-965-8550x92406,001-243-038-4271x53076,buckleycory@odonnell.net,2020-08-22,https://www.lucero.com/ +74,A3F76Be153Df4a3,Anita,Benson,Parrish Ltd,Skinnerport,Russian Federation,874.617.5668x69878,(399)820-6418x0071,angie04@oconnell.com,2020-02-09,http://oconnor.com/ +75,D01Af0AF7cBbFeA,Regina,Stein,Guzman-Brown,Raystad,Solomon Islands,001-469-848-0724x4407,001-085-360-4426x00357,zrosario@rojas-hardin.net,2022-01-15,http://www.johnston.info/ +76,d40e89dCade7b2F,Debra,Riddle,"Chang, Aguirre and Leblanc",Colinhaven,United States Virgin Islands,+1-768-182-6014x14336,(303)961-4491,shieldskerry@robles.com,2020-07-11,http://kaiser.info/ +77,BF6a1f9bd1bf8DE,Brittany,Zuniga,Mason-Hester,West Reginald,Kyrgyz Republic,(050)136-9025,001-480-851-2496x0157,mchandler@cochran-huerta.org,2021-07-24,http://www.boyle.com/ +78,FfaeFFbbbf280db,Cassidy,Mcmahon,"Mcguire, Huynh and Hopkins",Lake Sherryborough,Myanmar,5040771311,684-682-0021x1326,katrinalane@fitzgerald.com,2020-10-21,https://hurst.com/ +79,CbAE1d1e9a8dCb1,Laurie,Pennington,"Sanchez, Marsh and Hale",Port Katherineville,Dominica,007.155.3406x553,+1-809-862-5566x277,cookejill@powell.com,2020-06-08,http://www.hebert.com/ +80,A7F85c1DE4dB87f,Alejandro,Blair,"Combs, Waller and Durham",Thomasland,Iceland,(690)068-4641x51468,555.509.8691x2329,elizabethbarr@ewing.com,2020-09-19,https://mercado-blevins.com/ +81,D6CEAfb3BDbaa1A,Leslie,Jennings,Blankenship-Arias,Coreybury,Micronesia,629.198.6346,075.256.0829,corey75@wiggins.com,2021-11-13,https://www.juarez.com/ +82,Ebdb6F6F7c90b69,Kathleen,Mckay,"Coffey, Lamb and Johnson",Lake Janiceton,Saint Vincent and the Grenadines,(733)910-9968,(691)247-4128x0665,chloelester@higgins-wilkinson.com,2021-09-12,http://www.owens-mooney.com/ +83,E8E7e8Cfe516ef0,Hunter,Moreno,Fitzpatrick-Lawrence,East Clinton,Isle of Man,(733)833-6754,001-761-013-7121,isaac26@benton-finley.com,2020-12-28,http://walls.info/ +84,78C06E9b6B3DF20,Chad,Davidson,Garcia-Jimenez,South Joshuashire,Oman,8275702958,(804)842-4715,justinwalters@jimenez.com,2021-11-15,http://www.garner-oliver.com/ +85,03A1E62ADdeb31c,Corey,Holt,"Mcdonald, Bird and Ramirez",New Glenda,Fiji,001-439-242-4986x7918,3162708934,maurice46@morgan.com,2020-02-18,http://www.watson.com/ +86,C6763c99d0bd16D,Emma,Cunningham,Stephens Inc,North Jillianview,New Zealand,128-059-0206x60217,(312)164-4545x2284,walter83@juarez.org,2022-05-13,http://www.reid.info/ +87,ebe77E5Bf9476CE,Duane,Woods,Montoya-Miller,Lyonsberg,Maldives,(636)544-7783x7288,(203)287-1003x5932,kmercer@wagner.com,2020-07-21,http://murray.org/ +88,E4Bbcd8AD81fC5f,Alison,Vargas,"Vaughn, Watts and Leach",East Cristinabury,Benin,365-273-8144,053-308-7653x6287,vcantu@norton.com,2020-11-10,http://mason.info/ +89,efeb73245CDf1fF,Vernon,Kane,Carter-Strickland,Thomasfurt,Yemen,114-854-1159x555,499-608-4612,hilljesse@barrett.info,2021-04-15,http://www.duffy-hensley.net/ +90,37Ec4B395641c1E,Lori,Flowers,Decker-Mcknight,North Joeburgh,Namibia,679.415.1210,945-842-3659x4581,tyrone77@valenzuela.info,2021-01-09,http://www.deleon-crosby.com/ +91,5ef6d3eefdD43bE,Nina,Chavez,Byrd-Campbell,Cassidychester,Bhutan,053-344-3205,+1-330-920-5422x571,elliserica@frank.com,2020-03-26,https://www.pugh.com/ +92,98b3aeDcC3B9FF3,Shane,Foley,Rocha-Hart,South Dannymouth,Hungary,-1692,001-626-114-5844x55073,nsteele@sparks.com,2021-07-06,https://www.holt-sparks.com/ +93,aAb6AFc7AfD0fF3,Collin,Ayers,Lamb-Peterson,South Lonnie,Anguilla,404-645-5351x012,001-257-582-8850x8516,dudleyemily@gonzales.biz,2021-06-29,http://www.ruiz.com/ +94,54B5B5Fe9F1B6C5,Sherry,Young,"Lee, Lucero and Johnson",Frankchester,Solomon Islands,158-687-1764,(438)375-6207x003,alan79@gates-mclaughlin.com,2021-04-04,https://travis.net/ +95,BE91A0bdcA49Bbc,Darrell,Douglas,"Newton, Petersen and Mathis",Daisyborough,Mali,001-084-845-9524x1777,001-769-564-6303,grayjean@lowery-good.com,2022-02-17,https://banks.biz/ +96,cb8E23e48d22Eae,Karl,Greer,Carey LLC,East Richard,Guyana,(188)169-1674x58692,001-841-293-3519x614,hhart@jensen.com,2022-01-30,http://hayes-perez.com/ +97,CeD220bdAaCfaDf,Lynn,Atkinson,"Ware, Burns and Oneal",New Bradview,Sri Lanka,-3769,605.413.3198,vkemp@ferrell.com,2021-07-10,https://novak-allison.com/ +98,28CDbC0dFe4b1Db,Fred,Guerra,Schmitt-Jones,Ortegaland,Solomon Islands,+1-753-067-8419x7170,+1-632-666-7507x92121,swagner@kane.org,2021-09-18,https://www.ross.com/ +99,c23d1D9EE8DEB0A,Yvonne,Farmer,Fitzgerald-Harrell,Lake Elijahview,Aruba,(530)311-9786,001-869-452-0943x12424,mccarthystephen@horn-green.biz,2021-08-11,http://watkins.info/ +100,2354a0E336A91A1,Clarence,Haynes,"Le, Nash and Cross",Judymouth,Honduras,(753)813-6941,783.639.1472,colleen91@faulkner.biz,2020-03-11,http://www.hatfield-saunders.net/