# Write with smaller row groups (useful for filtering large files)
pq.write_table(
    table_large,
    'transactions_grouped.parquet',
    compression='ZSTD',
    row_group_size=10000
)

# Use PyArrow's dataset API for efficient filtering
import pyarrow.dataset as ds

dataset = ds.dataset('transactions_grouped.parquet', format='parquet')

# Filter: only transactions from the Downtown store over $150
filtered = dataset.to_table(
    filter=(ds.field('store') == 'Downtown') & (ds.field('amount') > 150)
)

df_filtered = filtered.to_pandas()
print(f"Matching transactions: {len(df_filtered)}")
print(df_filtered.head())