In [13]:
import polars as pl
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import math
In [4]:
lazy_df = pl.scan_parquet('Data/07-30-2021_cutoff.parquet')
In [5]:
lazy_df.collect_schema()
Out[5]:
Schema([('id', Int64),
('datetime', Datetime(time_unit='us', time_zone=None)),
('buy_open', Int64),
('buy_high', Int64),
('buy_low', Int64),
('buy_close', Int64),
('sell_open', Int64),
('sell_high', Int64),
('sell_low', Int64),
('sell_close', Int64),
('daily_avg_supply', Float64),
('daily_avg_demand', Float64)])
In [6]:
df = lazy_df.collect()
In [7]:
numeric_cols = ["buy_open", "buy_high", "buy_low", "buy_close",
"sell_close", "daily_avg_supply",
"daily_avg_demand"]
In [8]:
df.describe()
Out[8]:
<style>
.dataframe > thead > tr,
.dataframe > tbody > tr {
text-align: right;
white-space: pre-wrap;
}
</style>
<small>shape: (9, 13)</small><thead><th>statistic</th><th>id</th><th>datetime</th><th>buy_open</th><th>buy_high</th><th>buy_low</th><th>buy_close</th><th>sell_open</th><th>sell_high</th><th>sell_low</th><th>sell_close</th><th>daily_avg_supply</th><th>daily_avg_demand</th></thead><tbody></tbody><tbody>
| str | f64 | str | f64 | f64 | f64 | f64 | f64 | f64 | f64 | f64 | f64 | f64 |
| "count" | 4.8635739e7 | "48635739" | 4.8635739e7 | 4.8635739e7 | 4.8635739e7 | 4.8635739e7 | 4.8635739e7 | 4.8635739e7 | 4.8635739e7 | 4.8635739e7 | 4.8635739e7 | 4.8635739e7 |
| "null_count" | 0.0 | "0" | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 |
| "mean" | 33825.276431 | "2024-02-07 16:10:40.156803" | 127964.178461 | 128833.592657 | 127061.2305 | 127892.014231 | 274518.903501 | 275803.405189 | 273475.474243 | 274765.785219 | 27591.431237 | 6596.970101 |
| "std" | 27255.997708 | null | 1.6065e6 | 1.6185e6 | 1.5939e6 | 1.6056e6 | 2.9672e6 | 2.9771e6 | 2.9588e6 | 2.9688e6 | 619279.738731 | 136807.294732 |
| "min" | 24.0 | "2021-07-31 00:00:00" | 0.0 | 0.0 | 0.0 | 0.0 | 1.0 | 1.0 | 1.0 | 1.0 | 0.0 | 0.0 |
| "25%" | 12604.0 | "2022-11-06 00:00:00" | 60.0 | 60.0 | 60.0 | 60.0 | 202.0 | 205.0 | 200.0 | 203.0 | 105.0 | 53.0 |
| "50%" | 27494.0 | "2024-02-14 00:00:00" | 307.0 | 308.0 | 306.0 | 307.0 | 1516.0 | 1542.0 | 1500.0 | 1525.0 | 274.0 | 286.0 |
| "75%" | 45960.0 | "2025-05-10 00:00:00" | 4838.0 | 4875.0 | 4804.0 | 4833.0 | 14297.0 | 14487.0 | 14116.0 | 14365.0 | 716.0 | 893.0 |
| "max" | 109815.0 | "2026-07-29 00:00:00" | 9.5001499e7 | 9.5001499e7 | 9.5001499e7 | 9.5001499e7 | 1e8 | 1e8 | 1e8 | 1e8 | 5.9389635e7 | 1.6725496e7 |
In [9]:
# Set up a 4x2 grid (4 rows, 2 columns) to hold all 8 plots
fig, axes = plt.subplots(7, 2, figsize=(14, 16))
colors = ["blue", "green", "purple", "orange", 'black', 'yellow', 'red']
for i, (col, color) in enumerate(zip(numeric_cols, colors)):
# Extract as a zero-copy NumPy array directly from Polars
# This completely bypasses Pandas and keeps memory overhead near zero
raw_data = df[col].drop_nulls().to_numpy()
# --- 1. Standard Scale Histogram ---
# Compute counts and bin edges in NumPy
counts, bins = np.histogram(raw_data, bins=50)
# Plot using a simple bar chart (Matplotlib only renders 50 bars instead of millions of points)
axes[i, 0].bar(bins[:-1], counts, width=np.diff(bins), align="edge", color=color, alpha=0.7, edgecolor="black")
axes[i, 0].set_title(f"{col} Distribution")
# --- 2. Log Scale Histogram ---
# Filter for strictly positive values since log10(0) is undefined
pos_data = raw_data[raw_data > 0]
if len(pos_data) > 0:
# Calculate Log10 values and bin them
log_data = np.log10(pos_data)
log_counts, log_bins = np.histogram(log_data, bins=50)
axes[i, 1].bar(log_bins[:-1], log_counts, width=np.diff(log_bins), align="edge", color=color, alpha=0.7, edgecolor="black")
axes[i, 1].set_title(f"{col} Distribution (Log10 Scale)")
else:
axes[i, 1].set_title(f"{col} Distribution - No Positive Values")
plt.tight_layout()
plt.show()
In [10]:
# 1. Define expressions to safely log-transform columns inside Polars
# This replaces values <= 0 with Null so they are safely ignored in the correlation
log_exprs = [
pl.when(pl.col(col) > 0)
.then(pl.col(col).log())
.otherwise(None)
.alias(col)
for col in numeric_cols
]
# Apply the transformations lazily/natively
df_log = df.select(log_exprs)
# 2. Compute pairwise correlations directly using Polars expressions
# This forces the C/Rust engine to do the heavy lifting across the 100M+ rows
corr_exprs = [
pl.corr(c1, c2).alias(f"{c1}__x__{c2}")
for c1 in numeric_cols for c2 in numeric_cols
]
# Execute and extract the single row of results
corr_flat = df_log.select(corr_exprs).to_dicts()[0]
# 3. Restructure the flat results into a square Pandas DataFrame
# This is completely memory-safe because the DataFrame is only the size of your columns (e.g., 8x8)
corr_df = pd.DataFrame(index=numeric_cols, columns=numeric_cols, dtype=float)
for key, val in corr_flat.items():
c1, c2 = key.split("__x__")
corr_df.loc[c1, c2] = val
# 4. Plot the Correlation Heatmap
plt.figure(figsize=(10, 8))
sns.heatmap(corr_df, annot=True, fmt=".2f", cmap="coolwarm", vmin=-1, vmax=1)
plt.title("Guild Wars 2 Market Features Correlation Matrix (Log Scale)")
plt.tight_layout()
plt.show()
In [11]:
# Pick an item ID to inspect (e.g., item ID 19684 for Glob of Ectoplasm, or any ID in your dataset)
sample_id = df["id"].unique()[0] # Takes the first item ID in your dataset
sample_df = (
df
.filter(pl.col("id") == sample_id)
.sort("datetime")
.to_pandas()
)
plt.figure(figsize=(14, 6))
plt.plot(sample_df["datetime"], sample_df["buy_close"], label="Close Price", color="black", linewidth=1.5)
plt.fill_between(sample_df["datetime"], sample_df["buy_low"], sample_df["buy_high"], color="skyblue", alpha=0.4, label="Daily High-Low Range")
plt.title(f"Daily High-Low Volatility Boundary for Item ID: {sample_id}")
plt.xlabel("Date")
plt.ylabel("Price (in copper)")
plt.legend()
plt.grid(True, linestyle="--", alpha=0.5)
plt.tight_layout()
plt.show()
In [14]:
def plot_grid_scatter_polars(df, x_cols, y_col, sample_size=50000):
"""
Generates a grid of scatterplots optimized for massive Polars DataFrames.
Samples the data natively in Polars to prevent Matplotlib memory freezes.
"""
# 1. Calculate Grid Setup (4 columns wide to match your screenshot)
n_cols = 4
n_rows = math.ceil(len(x_cols) / n_cols)
fig, axes = plt.subplots(n_rows, n_cols, figsize=(16, 4 * n_rows))
axes = axes.flatten()
# 2. Filter and Sample in Polars FIRST
# Drop rows where the target Y variable is null
df_clean = df.drop_nulls(subset=[y_col])
# Randomly sample to prevent rendering crashes, using a seed for reproducibility
if df_clean.height > sample_size:
print(f"Sampling {sample_size} rows from {df_clean.height} total rows for visualization...")
df_sampled = df_clean.sample(n=sample_size, seed=42)
else:
df_sampled = df_clean
# Extract the Y column directly to a zero-copy NumPy array
y_data = df_sampled[y_col].to_numpy()
# 3. Build the Scatterplots
for i, x_col in enumerate(x_cols):
ax = axes[i]
# Extract X column to NumPy (handle any nulls in this specific X column by masking)
x_data = df_sampled[x_col].to_numpy()
# Mask out NaNs so Matplotlib doesn't complain
valid_mask = ~np.isnan(x_data) & ~np.isnan(y_data)
# Plot with low alpha (transparency) to see density in overlapping points
ax.scatter(
x_data[valid_mask],
y_data[valid_mask],
alpha=0.3, # Transparency helps visualize dense clusters
s=15, # Small dot size
color='#0072B2', # Clean blue color
edgecolors='none'
)
ax.set_title(x_col, fontsize=12, fontweight='bold')
ax.grid(True, linestyle="--", alpha=0.3)
# Optional: If you want log scales for highly skewed GW2 data, uncomment below:
# if x_data[valid_mask].max() > 1000:
# ax.set_xscale('log')
# 4. Cleanup Empty Subplots
for j in range(i + 1, len(axes)):
fig.delaxes(axes[j])
plt.tight_layout()
plt.show()
In [17]:
# Assuming your target variable is next day's high or today's close
target_column = "sell_high"
plot_grid_scatter_polars(
df=df,
x_cols=numeric_cols,
y_col=target_column,
sample_size=75000 # Adjust based on how dense you want the plots
)
Sampling 75000 rows from 48635739 total rows for visualization...
In [ ]: