Table of Contents

• 1  Description
• 2  Import the modules
• 3  Load the data
• 4  Data Cleaning
• 5  Memory reduction
• 6  Sample data
• 7  Validation

Description

Import the modules

import numpy as np
import pandas as pd
import os,sys,time

# visualization
import matplotlib.pyplot as plt
import seaborn as sns
sns.set()

import tqdm

# settings
SEED = 100
pd.set_option('max_columns',100)

%matplotlib inline
%load_ext watermark
%watermark -iv

tqdm     4.62.0
pandas   1.3.0
numpy    1.19.5
seaborn  0.11.0
json     2.0.9
autopep8 1.5.2

Load the data

!ls ../data

0_20k_.csv.zip               diabetes_project_dataset.csv

%%time

ifile = '../data/diabetes_project_dataset.csv'

df = pd.read_csv(ifile)
print(df.shape)

df.head(2).append(df.tail(2))

(8291, 60207)
CPU times: user 10min 55s, sys: 1min 41s, total: 12min 37s
Wall time: 14min 22s

	SampleID	existing_diabetes	incident_diabetes	diabetes_time	age	male	BMI	HDL	LDL	trig	SBP	DBP	hypertension	fasting	fasting_glucose	fasting_insulin	HbA1c	current_smoker	ex_smoker	exercise	healthy_vegetables	junk_food	total_fiber	mtb_1368087	mtb_1380093	mtb_1812369	mtb_1838668	mtb_1042362	mtb_1091716	mtb_1228672	mtb_1542487	mtb_1272352	mtb_1391826	mtb_1435571	mtb_1521753	mtb_1834574	mtb_1050860	mtb_606773	mtb_638620	mtb_752773	mtb_590255	mtb_709794	mtb_352255	mtb_509192	mtb_1230298	mtb_841524	mtb_1957718	mtb_1937123	mtb_1940724	mtb_18238	...	mtb_2123877	mtb_2123880	mtb_2124220	mtb_2124253	mtb_2124302	mtb_2124379	mtb_2124525	mtb_2124555	mtb_2124558	mtb_2124642	mtb_2124874	mtb_2124876	mtb_2125022	mtb_2125072	mtb_2125261	mtb_2125276	mtb_2125431	mtb_2125593	mtb_2125601	mtb_2125919	mtb_2126385	mtb_2126559	mtb_2126569	mtb_2126600	mtb_2126936	mtb_2126937	mtb_2127038	mtb_2127059	mtb_2127078	mtb_2127092	mtb_2127172	mtb_2127305	mtb_2127364	mtb_2127543	mtb_2127755	mtb_2127787	mtb_2127791	mtb_2127998	mtb_2128301	mtb_2128425	mtb_2129028	mtb_2129035	mtb_2129040	mtb_2129060	mtb_2129124	mtb_2129133	mtb_2129210	mtb_2129554	mtb_2129677	mtb_2129684
0	1	1.0	0.0	-37.51	58	0	49.586780	0.84	2.20	0.66	NaN	NaN	1.0	4.0	NaN	NaN	77.0	0.0	1	1.0	7.0	7.0	NaN	12.363110	14.838897	12.978673	11.074794	16.234650	16.435935	13.047940	12.321112	19.705813	15.746634	8.758477	17.678620	8.794702	21.605308	22.049880	15.308294	31.607882	25.023950	15.201024	15.496855	11.419300	15.977376	12.263096	9.600262	10.619338	13.914962	NaN	...	NaN	NaN	NaN	NaN	24.735746	7.286769	23.328052	NaN	8.276049	11.498900	NaN	9.436027	12.237526	13.302705	21.627131	NaN	10.544953	NaN	7.567015	NaN	2.562513	3.311077	NaN	NaN	30.817060	NaN	8.815905	NaN	NaN	NaN	NaN	4.546776	NaN	8.147375	8.080623	11.174161	NaN	NaN	6.640186	9.989815	16.137078	11.213977	NaN	2.521601	NaN	2.105956	4.148656	NaN	NaN	NaN
1	2	0.0	0.0	14.82	69	0	43.784050	1.60	3.88	1.85	178.0	79.0	0.0	5.0	6.91	19.6	37.0	0.0	0	3.0	15.0	7.0	41.0	12.523948	12.396497	8.915601	5.859528	15.870070	19.927809	11.281649	13.025576	19.707109	13.068911	9.878442	13.175717	6.628580	23.001835	22.663677	16.092091	32.842780	25.215257	15.909793	13.316935	14.594176	12.938364	12.524154	4.902424	6.965313	6.533694	8.051288	...	NaN	NaN	10.181594	NaN	22.780314	7.750491	16.692948	2.044146	6.282495	10.191683	NaN	6.054671	7.968107	9.071785	18.382458	NaN	7.464505	NaN	6.869950	NaN	1.967274	5.873054	NaN	NaN	32.622251	NaN	NaN	NaN	5.564728	6.471889	5.083502	3.906631	NaN	3.784477	5.569190	9.795326	NaN	NaN	NaN	NaN	18.938895	11.136169	5.074787	6.009284	16.353468	4.296854	3.747985	NaN	NaN	NaN
8289	8290	0.0	0.0	14.89	30	0	25.044784	1.63	2.11	0.85	123.0	71.0	0.0	5.0	NaN	NaN	37.0	0.0	0	3.0	10.0	8.0	30.0	13.299996	15.078436	11.924299	9.638035	16.296303	17.859562	10.641213	12.219351	19.919162	13.648144	12.108454	14.119564	9.434672	22.936264	23.055772	13.434320	33.027964	25.023950	16.889846	15.592734	17.760990	12.748022	14.042498	6.699196	8.218026	9.104366	10.402439	...	4.802855	3.065631	NaN	NaN	NaN	5.456070	17.999510	NaN	6.496850	10.340384	NaN	NaN	11.029112	9.818075	22.306311	NaN	7.683844	NaN	5.452548	NaN	NaN	4.244626	NaN	NaN	NaN	8.9151	NaN	NaN	NaN	NaN	NaN	NaN	NaN	6.056959	8.458553	9.746477	10.736888	NaN	NaN	7.994906	18.119646	10.288271	7.622332	NaN	NaN	NaN	NaN	NaN	NaN	NaN
8290	8291	0.0	0.0	14.89	27	0	21.744848	1.20	2.44	1.61	134.0	64.0	0.0	2.0	NaN	NaN	32.0	1.0	0	1.0	11.0	7.0	28.0	11.609244	11.574393	8.033199	7.756296	14.437769	16.814914	10.953024	10.849725	16.654139	12.243418	9.186343	13.496724	5.335492	22.432492	21.959500	14.434513	31.701197	24.095182	14.383838	13.671504	14.788930	14.048033	12.505380	9.062953	9.221019	10.346668	NaN	...	5.237648	2.628545	NaN	NaN	NaN	5.868853	19.409315	4.769605	5.990160	10.317581	NaN	7.153352	9.342941	7.297754	20.717364	NaN	7.320188	NaN	8.168727	NaN	NaN	3.908679	6.411201	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	5.261189	7.023384	8.615390	NaN	NaN	NaN	8.618082	14.948970	6.683449	4.710859	NaN	NaN	NaN	NaN	NaN	NaN	NaN

4 rows × 60207 columns

print(f"""
Number of rows   : {df.shape[0]:,}
Number of columns: {df.shape[1]:,}
""")

Number of rows   : 8,291
Number of columns: 60,207

Data Cleaning

Memory reduction

sys.getsizeof(df) / 1024 / 1024 # MB (original is is bytes)

3993410056

%%time
cols_int = df.select_dtypes(include=['int']).columns.to_list()
dict_int = {i:np.int32 for i in cols_int}

cols_float = df.select_dtypes(include=['float']).columns.to_list()
dict_float = {i:np.float32 for i in cols_float}

dict_dtypes = {**dict_int, **dict_float}

CPU times: user 1.42 s, sys: 3.49 s, total: 4.91 s
Wall time: 4.97 s

%%time
# change dtypes
df = df.astype(dict_dtypes)

CPU times: user 9.06 s, sys: 4.46 s, total: 13.5 s
Wall time: 19.7 s

sys.getsizeof(df) / 1024 / 1024 # MB (original is is bytes)

df.shape

(8291, 60207)

num_cores = 4
F = df.shape[0] // num_cores
print(F)

2072

# %%time
# df.loc[0:F,:].to_csv(f"../data/part_01.csv.gzip",compression='gzip',index=False)

CPU times: user 5min 58s, sys: 34.8 s, total: 6min 33s
Wall time: 6min 42s

# %%time
# df.loc[F:F*2,:].to_csv(f"../data/part_02.csv.gzip",compression='gzip',index=False)

CPU times: user 5min 57s, sys: 35 s, total: 6min 31s
Wall time: 6min 46s

# %%time
# df.loc[F*2:F*3,:].to_csv(f"../data/part_03.csv.gzip",compression='gzip',index=False)

CPU times: user 5min 45s, sys: 31.6 s, total: 6min 17s
Wall time: 6min 23s

# %%time
# df.loc[F*3:,:].to_csv(f"../data/part_04.csv.gzip",compression='gzip',index=False)

CPU times: user 6min 6s, sys: 34.9 s, total: 6min 40s
Wall time: 6min 56s

# parquet format for dask

%%time
df.loc[0:F,:].to_parquet(f"../data/part_01.parquet.gzip",compression='gzip',engine='pyarrow',index=False)

CPU times: user 22.3 s, sys: 864 ms, total: 23.2 s
Wall time: 23.5 s

%%time
df.loc[F:F*2,:].to_parquet(f"../data/part_02.parquet.gzip",compression='gzip',engine='pyarrow',index=False)

CPU times: user 21.9 s, sys: 821 ms, total: 22.8 s
Wall time: 23.1 s

%%time
df.loc[F*2:F*3,:].to_parquet(f"../data/part_03.parquet.gzip",compression='gzip',engine='pyarrow',index=False)

CPU times: user 22.4 s, sys: 851 ms, total: 23.2 s
Wall time: 23.7 s

%%time
df.loc[F*3:,:].to_parquet(f"../data/part_04.parquet.gzip",compression='gzip',engine='pyarrow',index=False)

CPU times: user 24.4 s, sys: 1.09 s, total: 25.5 s
Wall time: 27.1 s

Sample data

# sample data
df.iloc[:100,:].to_csv("../data/first_100.csv",index=False)

# sample data
df.iloc[:100,:].to_csv("../data/first_100.csv.zip",compression='zip',index=False)

# sample data
df.iloc[:100,:].to_csv("../data/first_100.csv.gzip",compression='gzip',index=False)

# sample data
df.iloc[:100,:].to_parquet("../data/first_100.parquet.gzip",compression='gzip',index=False)

Validation

%%time
df1 = pd.read_csv("../data/first_100.csv")
df1.head()

CPU times: user 31.6 s, sys: 253 ms, total: 31.8 s
Wall time: 32.4 s

	SampleID	existing_diabetes	incident_diabetes	diabetes_time	age	male	BMI	HDL	LDL	trig	SBP	DBP	hypertension	fasting	fasting_glucose	fasting_insulin	HbA1c	current_smoker	ex_smoker	exercise	healthy_vegetables	junk_food	total_fiber	mtb_1368087	mtb_1380093	mtb_1812369	mtb_1838668	mtb_1042362	mtb_1091716	mtb_1228672	mtb_1542487	mtb_1272352	mtb_1391826	mtb_1435571	mtb_1521753	mtb_1834574	mtb_1050860	mtb_606773	mtb_638620	mtb_752773	mtb_590255	mtb_709794	mtb_352255	mtb_509192	mtb_1230298	mtb_841524	mtb_1957718	mtb_1937123	mtb_1940724	mtb_18238	...	mtb_2123877	mtb_2123880	mtb_2124220	mtb_2124253	mtb_2124302	mtb_2124379	mtb_2124525	mtb_2124555	mtb_2124558	mtb_2124642	mtb_2124874	mtb_2124876	mtb_2125022	mtb_2125072	mtb_2125261	mtb_2125276	mtb_2125431	mtb_2125593	mtb_2125601	mtb_2125919	mtb_2126385	mtb_2126559	mtb_2126569	mtb_2126600	mtb_2126936	mtb_2126937	mtb_2127038	mtb_2127059	mtb_2127078	mtb_2127092	mtb_2127172	mtb_2127305	mtb_2127364	mtb_2127543	mtb_2127755	mtb_2127787	mtb_2127791	mtb_2127998	mtb_2128301	mtb_2128425	mtb_2129028	mtb_2129035	mtb_2129040	mtb_2129060	mtb_2129124	mtb_2129133	mtb_2129210	mtb_2129554	mtb_2129677	mtb_2129684
0	1	1.0	0.0	-37.51	58	0	49.58678	0.84	2.20	0.66	NaN	NaN	1.0	4.0	NaN	NaN	77.0	0.0	1	1.0	7.0	7.0	NaN	12.363110	14.838897	12.978673	11.074794	16.234650	16.435934	13.047940	12.321112	19.705812	15.746634	8.758476	17.678620	8.794702	21.605309	22.049880	15.308294	31.607882	25.023950	15.201024	15.496856	11.419300	15.977376	12.263097	9.600262	10.619338	13.914962	NaN	...	NaN	NaN	NaN	NaN	24.735746	7.286769	23.328053	NaN	8.276049	11.498900	NaN	9.436027	12.237526	13.302705	21.627130	NaN	10.544953	NaN	7.567015	NaN	2.562513	3.311077	NaN	NaN	30.817060	NaN	8.815906	NaN	NaN	NaN	NaN	4.546776	NaN	8.147375	8.080623	11.174161	NaN	NaN	6.640186	9.989815	16.137077	11.213977	NaN	2.521601	NaN	2.105956	4.148656	NaN	NaN	NaN
1	2	0.0	0.0	14.82	69	0	43.78405	1.60	3.88	1.85	178.0	79.0	0.0	5.0	6.91	19.6	37.0	0.0	0	3.0	15.0	7.0	41.0	12.523948	12.396497	8.915601	5.859528	15.870069	19.927809	11.281649	13.025576	19.707110	13.068912	9.878442	13.175717	6.628580	23.001835	22.663677	16.092090	32.842780	25.215258	15.909793	13.316935	14.594176	12.938364	12.524154	4.902424	6.965313	6.533694	8.051289	...	NaN	NaN	10.181594	NaN	22.780313	7.750492	16.692947	2.044146	6.282495	10.191683	NaN	6.054671	7.968107	9.071785	18.382458	NaN	7.464505	NaN	6.869950	NaN	1.967274	5.873054	NaN	NaN	32.622250	NaN	NaN	NaN	5.564728	6.471889	5.083502	3.906631	NaN	3.784477	5.569190	9.795325	NaN	NaN	NaN	NaN	18.938894	11.136169	5.074787	6.009284	16.353468	4.296854	3.747986	NaN	NaN	NaN
2	3	0.0	0.0	14.82	72	1	23.03596	1.55	2.97	1.12	156.0	75.0	1.0	6.0	NaN	NaN	37.0	0.0	0	3.0	8.0	6.0	32.0	9.891712	10.823469	8.935202	7.129169	13.763427	14.078740	9.379045	9.098672	17.783300	11.628487	7.408712	12.876341	6.722477	23.632578	20.834490	10.957421	30.440447	22.090017	13.495890	16.517088	17.804686	12.506579	15.638863	4.902578	7.093513	7.723044	8.740543	...	NaN	NaN	NaN	7.972753	21.544556	6.155103	16.350948	3.966285	6.546598	10.582582	NaN	5.176280	8.352525	8.525721	17.918280	NaN	9.146388	NaN	6.685533	NaN	NaN	5.081663	NaN	NaN	33.664610	NaN	NaN	NaN	NaN	6.634011	4.048846	NaN	NaN	4.126249	8.761478	9.380795	NaN	NaN	NaN	9.497226	13.882453	7.759390	NaN	2.390744	NaN	3.651907	6.140204	NaN	NaN	NaN
3	4	0.0	1.0	2.20	68	0	39.42166	1.20	2.80	2.33	154.0	80.0	0.0	4.0	8.83	33.4	NaN	0.0	0	1.0	13.0	NaN	35.0	13.192851	13.231675	9.492927	7.637889	20.835545	15.902070	8.607652	12.304895	19.097034	14.864097	8.813502	14.029215	7.987217	28.004500	23.900194	16.622800	30.224667	24.068956	11.018594	14.770869	16.506992	9.386408	13.064268	5.418974	7.051772	7.632268	NaN	...	NaN	NaN	4.923232	7.513059	24.610344	NaN	17.286161	NaN	4.754910	9.092149	NaN	5.779519	6.589979	NaN	18.128048	NaN	7.322501	NaN	NaN	NaN	2.169996	4.640553	9.081938	NaN	32.130253	NaN	NaN	NaN	NaN	NaN	4.654498	5.638649	NaN	4.143766	5.127863	9.120014	NaN	NaN	NaN	8.241730	17.269463	10.619393	NaN	5.325468	13.148665	2.920644	3.976569	NaN	NaN	NaN
4	5	0.0	0.0	14.82	60	0	27.89668	1.70	2.98	1.29	121.0	77.0	0.0	6.0	5.86	8.8	38.0	0.0	0	2.0	9.0	8.0	37.0	11.095172	12.698000	10.401415	8.504086	16.343100	16.567852	12.855010	10.810644	19.652111	13.417355	10.779993	16.110594	5.791803	23.175488	23.143604	12.069803	31.646250	26.044937	12.934877	13.945308	16.996647	13.202727	16.219149	6.628916	8.441991	8.714303	9.213964	...	NaN	NaN	5.698162	NaN	24.675653	5.522934	19.825472	5.890806	7.068063	11.985300	12.334053	NaN	9.571555	13.951574	19.545973	NaN	10.287803	NaN	10.184063	NaN	NaN	6.394354	7.145855	NaN	32.622350	NaN	NaN	NaN	NaN	NaN	4.101276	4.229185	NaN	5.531142	8.809899	10.826915	NaN	NaN	NaN	9.373282	15.158063	7.355921	NaN	4.374935	12.825463	3.178521	5.143971	NaN	NaN	NaN

5 rows × 60207 columns

%%time
df1 = pd.read_csv("../data/first_100.csv.zip",compression='zip')
df1.head()

CPU times: user 32.3 s, sys: 339 ms, total: 32.7 s
Wall time: 33.7 s

	SampleID	existing_diabetes	incident_diabetes	diabetes_time	age	male	BMI	HDL	LDL	trig	SBP	DBP	hypertension	fasting	fasting_glucose	fasting_insulin	HbA1c	current_smoker	ex_smoker	exercise	healthy_vegetables	junk_food	total_fiber	mtb_1368087	mtb_1380093	mtb_1812369	mtb_1838668	mtb_1042362	mtb_1091716	mtb_1228672	mtb_1542487	mtb_1272352	mtb_1391826	mtb_1435571	mtb_1521753	mtb_1834574	mtb_1050860	mtb_606773	mtb_638620	mtb_752773	mtb_590255	mtb_709794	mtb_352255	mtb_509192	mtb_1230298	mtb_841524	mtb_1957718	mtb_1937123	mtb_1940724	mtb_18238	...	mtb_2123877	mtb_2123880	mtb_2124220	mtb_2124253	mtb_2124302	mtb_2124379	mtb_2124525	mtb_2124555	mtb_2124558	mtb_2124642	mtb_2124874	mtb_2124876	mtb_2125022	mtb_2125072	mtb_2125261	mtb_2125276	mtb_2125431	mtb_2125593	mtb_2125601	mtb_2125919	mtb_2126385	mtb_2126559	mtb_2126569	mtb_2126600	mtb_2126936	mtb_2126937	mtb_2127038	mtb_2127059	mtb_2127078	mtb_2127092	mtb_2127172	mtb_2127305	mtb_2127364	mtb_2127543	mtb_2127755	mtb_2127787	mtb_2127791	mtb_2127998	mtb_2128301	mtb_2128425	mtb_2129028	mtb_2129035	mtb_2129040	mtb_2129060	mtb_2129124	mtb_2129133	mtb_2129210	mtb_2129554	mtb_2129677	mtb_2129684
0	1	1.0	0.0	-37.51	58	0	49.58678	0.84	2.20	0.66	NaN	NaN	1.0	4.0	NaN	NaN	77.0	0.0	1	1.0	7.0	7.0	NaN	12.363110	14.838897	12.978673	11.074794	16.234650	16.435934	13.047940	12.321112	19.705812	15.746634	8.758476	17.678620	8.794702	21.605309	22.049880	15.308294	31.607882	25.023950	15.201024	15.496856	11.419300	15.977376	12.263097	9.600262	10.619338	13.914962	NaN	...	NaN	NaN	NaN	NaN	24.735746	7.286769	23.328053	NaN	8.276049	11.498900	NaN	9.436027	12.237526	13.302705	21.627130	NaN	10.544953	NaN	7.567015	NaN	2.562513	3.311077	NaN	NaN	30.817060	NaN	8.815906	NaN	NaN	NaN	NaN	4.546776	NaN	8.147375	8.080623	11.174161	NaN	NaN	6.640186	9.989815	16.137077	11.213977	NaN	2.521601	NaN	2.105956	4.148656	NaN	NaN	NaN
1	2	0.0	0.0	14.82	69	0	43.78405	1.60	3.88	1.85	178.0	79.0	0.0	5.0	6.91	19.6	37.0	0.0	0	3.0	15.0	7.0	41.0	12.523948	12.396497	8.915601	5.859528	15.870069	19.927809	11.281649	13.025576	19.707110	13.068912	9.878442	13.175717	6.628580	23.001835	22.663677	16.092090	32.842780	25.215258	15.909793	13.316935	14.594176	12.938364	12.524154	4.902424	6.965313	6.533694	8.051289	...	NaN	NaN	10.181594	NaN	22.780313	7.750492	16.692947	2.044146	6.282495	10.191683	NaN	6.054671	7.968107	9.071785	18.382458	NaN	7.464505	NaN	6.869950	NaN	1.967274	5.873054	NaN	NaN	32.622250	NaN	NaN	NaN	5.564728	6.471889	5.083502	3.906631	NaN	3.784477	5.569190	9.795325	NaN	NaN	NaN	NaN	18.938894	11.136169	5.074787	6.009284	16.353468	4.296854	3.747986	NaN	NaN	NaN
2	3	0.0	0.0	14.82	72	1	23.03596	1.55	2.97	1.12	156.0	75.0	1.0	6.0	NaN	NaN	37.0	0.0	0	3.0	8.0	6.0	32.0	9.891712	10.823469	8.935202	7.129169	13.763427	14.078740	9.379045	9.098672	17.783300	11.628487	7.408712	12.876341	6.722477	23.632578	20.834490	10.957421	30.440447	22.090017	13.495890	16.517088	17.804686	12.506579	15.638863	4.902578	7.093513	7.723044	8.740543	...	NaN	NaN	NaN	7.972753	21.544556	6.155103	16.350948	3.966285	6.546598	10.582582	NaN	5.176280	8.352525	8.525721	17.918280	NaN	9.146388	NaN	6.685533	NaN	NaN	5.081663	NaN	NaN	33.664610	NaN	NaN	NaN	NaN	6.634011	4.048846	NaN	NaN	4.126249	8.761478	9.380795	NaN	NaN	NaN	9.497226	13.882453	7.759390	NaN	2.390744	NaN	3.651907	6.140204	NaN	NaN	NaN
3	4	0.0	1.0	2.20	68	0	39.42166	1.20	2.80	2.33	154.0	80.0	0.0	4.0	8.83	33.4	NaN	0.0	0	1.0	13.0	NaN	35.0	13.192851	13.231675	9.492927	7.637889	20.835545	15.902070	8.607652	12.304895	19.097034	14.864097	8.813502	14.029215	7.987217	28.004500	23.900194	16.622800	30.224667	24.068956	11.018594	14.770869	16.506992	9.386408	13.064268	5.418974	7.051772	7.632268	NaN	...	NaN	NaN	4.923232	7.513059	24.610344	NaN	17.286161	NaN	4.754910	9.092149	NaN	5.779519	6.589979	NaN	18.128048	NaN	7.322501	NaN	NaN	NaN	2.169996	4.640553	9.081938	NaN	32.130253	NaN	NaN	NaN	NaN	NaN	4.654498	5.638649	NaN	4.143766	5.127863	9.120014	NaN	NaN	NaN	8.241730	17.269463	10.619393	NaN	5.325468	13.148665	2.920644	3.976569	NaN	NaN	NaN
4	5	0.0	0.0	14.82	60	0	27.89668	1.70	2.98	1.29	121.0	77.0	0.0	6.0	5.86	8.8	38.0	0.0	0	2.0	9.0	8.0	37.0	11.095172	12.698000	10.401415	8.504086	16.343100	16.567852	12.855010	10.810644	19.652111	13.417355	10.779993	16.110594	5.791803	23.175488	23.143604	12.069803	31.646250	26.044937	12.934877	13.945308	16.996647	13.202727	16.219149	6.628916	8.441991	8.714303	9.213964	...	NaN	NaN	5.698162	NaN	24.675653	5.522934	19.825472	5.890806	7.068063	11.985300	12.334053	NaN	9.571555	13.951574	19.545973	NaN	10.287803	NaN	10.184063	NaN	NaN	6.394354	7.145855	NaN	32.622350	NaN	NaN	NaN	NaN	NaN	4.101276	4.229185	NaN	5.531142	8.809899	10.826915	NaN	NaN	NaN	9.373282	15.158063	7.355921	NaN	4.374935	12.825463	3.178521	5.143971	NaN	NaN	NaN

5 rows × 60207 columns

%%time
df1 = pd.read_csv("../data/first_100.csv.gzip",compression='gzip')
df1.head()

CPU times: user 31.5 s, sys: 263 ms, total: 31.7 s
Wall time: 32.4 s

	SampleID	existing_diabetes	incident_diabetes	diabetes_time	age	male	BMI	HDL	LDL	trig	SBP	DBP	hypertension	fasting	fasting_glucose	fasting_insulin	HbA1c	current_smoker	ex_smoker	exercise	healthy_vegetables	junk_food	total_fiber	mtb_1368087	mtb_1380093	mtb_1812369	mtb_1838668	mtb_1042362	mtb_1091716	mtb_1228672	mtb_1542487	mtb_1272352	mtb_1391826	mtb_1435571	mtb_1521753	mtb_1834574	mtb_1050860	mtb_606773	mtb_638620	mtb_752773	mtb_590255	mtb_709794	mtb_352255	mtb_509192	mtb_1230298	mtb_841524	mtb_1957718	mtb_1937123	mtb_1940724	mtb_18238	...	mtb_2123877	mtb_2123880	mtb_2124220	mtb_2124253	mtb_2124302	mtb_2124379	mtb_2124525	mtb_2124555	mtb_2124558	mtb_2124642	mtb_2124874	mtb_2124876	mtb_2125022	mtb_2125072	mtb_2125261	mtb_2125276	mtb_2125431	mtb_2125593	mtb_2125601	mtb_2125919	mtb_2126385	mtb_2126559	mtb_2126569	mtb_2126600	mtb_2126936	mtb_2126937	mtb_2127038	mtb_2127059	mtb_2127078	mtb_2127092	mtb_2127172	mtb_2127305	mtb_2127364	mtb_2127543	mtb_2127755	mtb_2127787	mtb_2127791	mtb_2127998	mtb_2128301	mtb_2128425	mtb_2129028	mtb_2129035	mtb_2129040	mtb_2129060	mtb_2129124	mtb_2129133	mtb_2129210	mtb_2129554	mtb_2129677	mtb_2129684
0	1	1.0	0.0	-37.51	58	0	49.58678	0.84	2.20	0.66	NaN	NaN	1.0	4.0	NaN	NaN	77.0	0.0	1	1.0	7.0	7.0	NaN	12.363110	14.838897	12.978673	11.074794	16.234650	16.435934	13.047940	12.321112	19.705812	15.746634	8.758476	17.678620	8.794702	21.605309	22.049880	15.308294	31.607882	25.023950	15.201024	15.496856	11.419300	15.977376	12.263097	9.600262	10.619338	13.914962	NaN	...	NaN	NaN	NaN	NaN	24.735746	7.286769	23.328053	NaN	8.276049	11.498900	NaN	9.436027	12.237526	13.302705	21.627130	NaN	10.544953	NaN	7.567015	NaN	2.562513	3.311077	NaN	NaN	30.817060	NaN	8.815906	NaN	NaN	NaN	NaN	4.546776	NaN	8.147375	8.080623	11.174161	NaN	NaN	6.640186	9.989815	16.137077	11.213977	NaN	2.521601	NaN	2.105956	4.148656	NaN	NaN	NaN
1	2	0.0	0.0	14.82	69	0	43.78405	1.60	3.88	1.85	178.0	79.0	0.0	5.0	6.91	19.6	37.0	0.0	0	3.0	15.0	7.0	41.0	12.523948	12.396497	8.915601	5.859528	15.870069	19.927809	11.281649	13.025576	19.707110	13.068912	9.878442	13.175717	6.628580	23.001835	22.663677	16.092090	32.842780	25.215258	15.909793	13.316935	14.594176	12.938364	12.524154	4.902424	6.965313	6.533694	8.051289	...	NaN	NaN	10.181594	NaN	22.780313	7.750492	16.692947	2.044146	6.282495	10.191683	NaN	6.054671	7.968107	9.071785	18.382458	NaN	7.464505	NaN	6.869950	NaN	1.967274	5.873054	NaN	NaN	32.622250	NaN	NaN	NaN	5.564728	6.471889	5.083502	3.906631	NaN	3.784477	5.569190	9.795325	NaN	NaN	NaN	NaN	18.938894	11.136169	5.074787	6.009284	16.353468	4.296854	3.747986	NaN	NaN	NaN
2	3	0.0	0.0	14.82	72	1	23.03596	1.55	2.97	1.12	156.0	75.0	1.0	6.0	NaN	NaN	37.0	0.0	0	3.0	8.0	6.0	32.0	9.891712	10.823469	8.935202	7.129169	13.763427	14.078740	9.379045	9.098672	17.783300	11.628487	7.408712	12.876341	6.722477	23.632578	20.834490	10.957421	30.440447	22.090017	13.495890	16.517088	17.804686	12.506579	15.638863	4.902578	7.093513	7.723044	8.740543	...	NaN	NaN	NaN	7.972753	21.544556	6.155103	16.350948	3.966285	6.546598	10.582582	NaN	5.176280	8.352525	8.525721	17.918280	NaN	9.146388	NaN	6.685533	NaN	NaN	5.081663	NaN	NaN	33.664610	NaN	NaN	NaN	NaN	6.634011	4.048846	NaN	NaN	4.126249	8.761478	9.380795	NaN	NaN	NaN	9.497226	13.882453	7.759390	NaN	2.390744	NaN	3.651907	6.140204	NaN	NaN	NaN
3	4	0.0	1.0	2.20	68	0	39.42166	1.20	2.80	2.33	154.0	80.0	0.0	4.0	8.83	33.4	NaN	0.0	0	1.0	13.0	NaN	35.0	13.192851	13.231675	9.492927	7.637889	20.835545	15.902070	8.607652	12.304895	19.097034	14.864097	8.813502	14.029215	7.987217	28.004500	23.900194	16.622800	30.224667	24.068956	11.018594	14.770869	16.506992	9.386408	13.064268	5.418974	7.051772	7.632268	NaN	...	NaN	NaN	4.923232	7.513059	24.610344	NaN	17.286161	NaN	4.754910	9.092149	NaN	5.779519	6.589979	NaN	18.128048	NaN	7.322501	NaN	NaN	NaN	2.169996	4.640553	9.081938	NaN	32.130253	NaN	NaN	NaN	NaN	NaN	4.654498	5.638649	NaN	4.143766	5.127863	9.120014	NaN	NaN	NaN	8.241730	17.269463	10.619393	NaN	5.325468	13.148665	2.920644	3.976569	NaN	NaN	NaN
4	5	0.0	0.0	14.82	60	0	27.89668	1.70	2.98	1.29	121.0	77.0	0.0	6.0	5.86	8.8	38.0	0.0	0	2.0	9.0	8.0	37.0	11.095172	12.698000	10.401415	8.504086	16.343100	16.567852	12.855010	10.810644	19.652111	13.417355	10.779993	16.110594	5.791803	23.175488	23.143604	12.069803	31.646250	26.044937	12.934877	13.945308	16.996647	13.202727	16.219149	6.628916	8.441991	8.714303	9.213964	...	NaN	NaN	5.698162	NaN	24.675653	5.522934	19.825472	5.890806	7.068063	11.985300	12.334053	NaN	9.571555	13.951574	19.545973	NaN	10.287803	NaN	10.184063	NaN	NaN	6.394354	7.145855	NaN	32.622350	NaN	NaN	NaN	NaN	NaN	4.101276	4.229185	NaN	5.531142	8.809899	10.826915	NaN	NaN	NaN	9.373282	15.158063	7.355921	NaN	4.374935	12.825463	3.178521	5.143971	NaN	NaN	NaN

5 rows × 60207 columns

%%time
df1 = pd.read_parquet("../data/first_100.parquet.gzip")
df1.head()

CPU times: user 5min 10s, sys: 2.54 s, total: 5min 12s
Wall time: 5min 14s

	SampleID	existing_diabetes	incident_diabetes	diabetes_time	age	male	BMI	HDL	LDL	trig	SBP	DBP	hypertension	fasting	fasting_glucose	fasting_insulin	HbA1c	current_smoker	ex_smoker	exercise	healthy_vegetables	junk_food	total_fiber	mtb_1368087	mtb_1380093	mtb_1812369	mtb_1838668	mtb_1042362	mtb_1091716	mtb_1228672	mtb_1542487	mtb_1272352	mtb_1391826	mtb_1435571	mtb_1521753	mtb_1834574	mtb_1050860	mtb_606773	mtb_638620	mtb_752773	mtb_590255	mtb_709794	mtb_352255	mtb_509192	mtb_1230298	mtb_841524	mtb_1957718	mtb_1937123	mtb_1940724	mtb_18238	...	mtb_2123877	mtb_2123880	mtb_2124220	mtb_2124253	mtb_2124302	mtb_2124379	mtb_2124525	mtb_2124555	mtb_2124558	mtb_2124642	mtb_2124874	mtb_2124876	mtb_2125022	mtb_2125072	mtb_2125261	mtb_2125276	mtb_2125431	mtb_2125593	mtb_2125601	mtb_2125919	mtb_2126385	mtb_2126559	mtb_2126569	mtb_2126600	mtb_2126936	mtb_2126937	mtb_2127038	mtb_2127059	mtb_2127078	mtb_2127092	mtb_2127172	mtb_2127305	mtb_2127364	mtb_2127543	mtb_2127755	mtb_2127787	mtb_2127791	mtb_2127998	mtb_2128301	mtb_2128425	mtb_2129028	mtb_2129035	mtb_2129040	mtb_2129060	mtb_2129124	mtb_2129133	mtb_2129210	mtb_2129554	mtb_2129677	mtb_2129684
0	1	1.0	0.0	-37.509998	58	0	49.586781	0.84	2.20	0.66	NaN	NaN	1.0	4.0	NaN	NaN	77.0	0.0	1	1.0	7.0	7.0	NaN	12.363110	14.838897	12.978673	11.074794	16.234650	16.435934	13.047940	12.321112	19.705812	15.746634	8.758476	17.678619	8.794702	21.605309	22.049879	15.308294	31.607882	25.023951	15.201024	15.496856	11.419300	15.977376	12.263097	9.600262	10.619338	13.914962	NaN	...	NaN	NaN	NaN	NaN	24.735746	7.286769	23.328053	NaN	8.276049	11.498900	NaN	9.436027	12.237526	13.302705	21.627131	NaN	10.544953	NaN	7.567015	NaN	2.562513	3.311077	NaN	NaN	30.817060	NaN	8.815906	NaN	NaN	NaN	NaN	4.546776	NaN	8.147375	8.080623	11.174161	NaN	NaN	6.640186	9.989815	16.137077	11.213977	NaN	2.521601	NaN	2.105956	4.148656	NaN	NaN	NaN
1	2	0.0	0.0	14.820000	69	0	43.784050	1.60	3.88	1.85	178.0	79.0	0.0	5.0	6.91	19.600000	37.0	0.0	0	3.0	15.0	7.0	41.0	12.523948	12.396497	8.915601	5.859528	15.870070	19.927809	11.281649	13.025576	19.707109	13.068912	9.878442	13.175717	6.628580	23.001835	22.663677	16.092091	32.842781	25.215258	15.909793	13.316935	14.594176	12.938364	12.524154	4.902424	6.965313	6.533694	8.051289	...	NaN	NaN	10.181594	NaN	22.780313	7.750492	16.692947	2.044146	6.282495	10.191683	NaN	6.054671	7.968107	9.071785	18.382458	NaN	7.464505	NaN	6.869950	NaN	1.967274	5.873054	NaN	NaN	32.622250	NaN	NaN	NaN	5.564728	6.471889	5.083502	3.906631	NaN	3.784477	5.569190	9.795325	NaN	NaN	NaN	NaN	18.938894	11.136169	5.074787	6.009284	16.353468	4.296854	3.747986	NaN	NaN	NaN
2	3	0.0	0.0	14.820000	72	1	23.035959	1.55	2.97	1.12	156.0	75.0	1.0	6.0	NaN	NaN	37.0	0.0	0	3.0	8.0	6.0	32.0	9.891712	10.823469	8.935202	7.129169	13.763427	14.078740	9.379045	9.098672	17.783300	11.628487	7.408712	12.876341	6.722477	23.632578	20.834490	10.957421	30.440447	22.090017	13.495890	16.517088	17.804686	12.506579	15.638863	4.902578	7.093513	7.723044	8.740543	...	NaN	NaN	NaN	7.972753	21.544556	6.155103	16.350948	3.966285	6.546598	10.582582	NaN	5.176280	8.352525	8.525721	17.918280	NaN	9.146388	NaN	6.685533	NaN	NaN	5.081663	NaN	NaN	33.664612	NaN	NaN	NaN	NaN	6.634011	4.048846	NaN	NaN	4.126249	8.761478	9.380795	NaN	NaN	NaN	9.497226	13.882453	7.759390	NaN	2.390744	NaN	3.651907	6.140204	NaN	NaN	NaN
3	4	0.0	1.0	2.200000	68	0	39.421661	1.20	2.80	2.33	154.0	80.0	0.0	4.0	8.83	33.400002	NaN	0.0	0	1.0	13.0	NaN	35.0	13.192851	13.231675	9.492927	7.637889	20.835545	15.902070	8.607652	12.304895	19.097034	14.864097	8.813502	14.029215	7.987217	28.004499	23.900194	16.622801	30.224667	24.068956	11.018594	14.770869	16.506992	9.386408	13.064268	5.418974	7.051772	7.632268	NaN	...	NaN	NaN	4.923232	7.513059	24.610344	NaN	17.286161	NaN	4.754910	9.092149	NaN	5.779519	6.589979	NaN	18.128048	NaN	7.322501	NaN	NaN	NaN	2.169996	4.640553	9.081938	NaN	32.130253	NaN	NaN	NaN	NaN	NaN	4.654498	5.638649	NaN	4.143766	5.127863	9.120014	NaN	NaN	NaN	8.241730	17.269463	10.619393	NaN	5.325468	13.148665	2.920644	3.976569	NaN	NaN	NaN
4	5	0.0	0.0	14.820000	60	0	27.896681	1.70	2.98	1.29	121.0	77.0	0.0	6.0	5.86	8.800000	38.0	0.0	0	2.0	9.0	8.0	37.0	11.095172	12.698000	10.401415	8.504086	16.343100	16.567852	12.855010	10.810644	19.652111	13.417355	10.779993	16.110594	5.791803	23.175488	23.143604	12.069803	31.646250	26.044937	12.934877	13.945308	16.996647	13.202727	16.219149	6.628916	8.441991	8.714303	9.213964	...	NaN	NaN	5.698162	NaN	24.675653	5.522934	19.825472	5.890806	7.068063	11.985300	12.334053	NaN	9.571555	13.951574	19.545973	NaN	10.287803	NaN	10.184063	NaN	NaN	6.394354	7.145855	NaN	32.622349	NaN	NaN	NaN	NaN	NaN	4.101276	4.229185	NaN	5.531142	8.809899	10.826915	NaN	NaN	NaN	9.373282	15.158063	7.355921	NaN	4.374935	12.825463	3.178521	5.143970	NaN	NaN	NaN

5 rows × 60207 columns