Robert Howell
02/14/2025, 7:46 PMRobert Howell
02/14/2025, 7:58 PMimport daft
def summarize(df: daft.DataFrame):
cols = [] # column :: utf8
typs = [] # type :: utf8
mins = [] # min :: utf8
maxs = [] # max :: utf8
cnts = [] # count :: int64
nuls = [] # nulls :: int64
unqs = [] # approx_distinct :: int64
for field in df.schema():
col = daft.col(field.name)
cols.append(daft.lit(field.name))
typs.append(daft.lit(str(field.dtype)))
mins.append(col.min().cast(daft.DataType.string()))
maxs.append(col.max().cast(daft.DataType.string()))
cnts.append(col.count("valid"))
nuls.append(col.count("null"))
unqs.append(col.approx_count_distinct())
df = df.agg(
[
daft.list_(*cols).alias("column"),
daft.list_(*typs).alias("type"),
daft.list_(*mins).alias("min"),
daft.list_(*maxs).alias("max"),
daft.list_(*cnts).alias("count"),
daft.list_(*nuls).alias("count_nulls"),
daft.list_(*unqs).alias("approx_count_distinct"),
]
)
return df.explode(*df.columns)