import polars as pl
import pandas as pd
pl.Config.set_tbl_rows(15)
pl.Config(float_precision=2)

import gc
import numpy as np
import matplotlib.pyplot as plt

from datetime import datetime, timedelta
from bs4 import BeautifulSoup
import requests
import math

days = '3'

cutoff_date = datetime(2023, 7, 1)
df = pl.read_parquet(f'Data/all_stats_{cutoff_date.strftime('%m-%d-%Y')}_cutoff.parquet')
df.head()

<style> .dataframe > thead > tr, .dataframe > tbody > tr { text-align: right; white-space: pre-wrap; } </style> <small>shape: (5, 15)</small><thead><th>buy_open</th><th>buy_high</th><th>buy_low</th><th>buy_close</th><th>id</th><th>date</th><th>sell_open</th><th>sell_high</th><th>sell_low</th><th>sell_close</th><th>sell_sma</th><th>buy_sma</th><th>rsi</th><th>supply</th><th>demand</th></thead><tbody></tbody><tbody></tbody>

i64	i64	i64	i64	i64	datetime[μs]	i64	i64	i64	i64	f64	f64	f64	i64	i64
2570	2586	2570	2586	125	2023-07-17 00:00:00	4223	4223	2830	3124	3739.07	2319.57	38.51	1734	1092
57	57	57	57	135	2025-07-19 00:00:00	158	158	155	155	154.21	57.00	52.97	2582	1888
105	105	105	105	135	2023-10-30 00:00:00	166	168	166	168	162.86	105.00	66.59	2560	1866
73	73	73	73	126	2025-05-06 00:00:00	150	150	150	150	157.79	73.00	48.02	252	1384
8390	8390	8390	8390	139	2024-11-24 00:00:00	69988	69988	69988	69988	68875.71	8389.71	63.43	58	83

# May use this later but for now this line has been cut from use
cols_need_from_items = ['id', 'type', 'level', 'rarity', 'vendor_value', 'flags']

item_info = pl.read_csv('Data/CraftingMaterial_Trophy_items.csv')

# Left join columns we need from the all_items table on id
df = df.join(item_info, on='id', how='left')
# Filter so only Crafting Materials and trophies are included
df = df.filter(pl.col('type').is_in(['CraftingMaterial', 'Trophy']))#.drop('type')

df.head()

<style> .dataframe > thead > tr, .dataframe > tbody > tr { text-align: right; white-space: pre-wrap; } </style> <small>shape: (5, 30)</small><thead><th>buy_open</th><th>buy_high</th><th>buy_low</th><th>buy_close</th><th>id</th><th>date</th><th>sell_open</th><th>sell_high</th><th>sell_low</th><th>sell_close</th><th>sell_sma</th><th>buy_sma</th><th>rsi</th><th>supply</th><th>demand</th><th>name</th><th>type</th><th>level</th><th>rarity</th><th>vendor_value</th><th>default_skin</th><th>game_types</th><th>flags</th><th>restrictions</th><th>chat_link</th><th>icon</th><th>details</th><th>description</th><th>upgrades_from</th><th>upgrades_into</th></thead><tbody></tbody><tbody></tbody>

i64	i64	i64	i64	i64	datetime[μs]	i64	i64	i64	i64	f64	f64	f64	i64	i64	str	str	i64	str	i64	str	str	str	str	str	str	f64	str	str	str
209	214	209	214	12142	2025-06-17 00:00:00	265	265	246	246	294.07	234.86	49.92	54740	143556	"Onion"	"CraftingMaterial"	0	"Basic"	1	null	"{Activity,Wvw,Dungeon,Pve}"	"{NoSalvage}"	"{}"	"[&AgFuLwAA]"	"https://render.guildwars2.com/…	NaN	"Ingredient"	null	null
223	223	219	219	12142	2024-09-14 00:00:00	226	227	226	227	230.14	218.93	54.66	131282	187516	"Onion"	"CraftingMaterial"	0	"Basic"	1	null	"{Activity,Wvw,Dungeon,Pve}"	"{NoSalvage}"	"{}"	"[&AgFuLwAA]"	"https://render.guildwars2.com/…	NaN	"Ingredient"	null	null
8	8	8	8	12156	2026-02-10 00:00:00	11	11	11	11	11.00	8.07	49.41	571047	2862776	"Jug of Water"	"CraftingMaterial"	0	"Basic"	1	null	"{Activity,Wvw,Dungeon,Pve}"	"{NoSalvage}"	"{}"	"[&AgF8LwAA]"	"https://render.guildwars2.com/…	NaN	"Ingredient"	null	null
171	174	171	174	12142	2025-01-10 00:00:00	173	181	173	181	186.43	175.79	48.42	200084	171230	"Onion"	"CraftingMaterial"	0	"Basic"	1	null	"{Activity,Wvw,Dungeon,Pve}"	"{NoSalvage}"	"{}"	"[&AgFuLwAA]"	"https://render.guildwars2.com/…	NaN	"Ingredient"	null	null
314	325	314	323	12138	2023-12-19 00:00:00	375	382	375	379	335.57	299.79	59.92	377848	400420	"Stick of Butter"	"CraftingMaterial"	0	"Basic"	1	null	"{Activity,Wvw,Dungeon,Pve}"	"{NoSalvage}"	"{}"	"[&AgFqLwAA]"	"https://render.guildwars2.com/…	NaN	"Ingredient"	null	null

# Let's create one field for price that is the average of open, high, low, and close.
# Then we won't need those columns anymore (mostly because they are not accessible to us at the time of prediction
# in deployment
df = df.with_columns(
    pl.col('buy_high').alias('current_buy_price'),
    pl.col('sell_high').alias('current_sell_price')
).drop(['buy_open', 'buy_high', 'buy_low', 'buy_close', 'sell_open', 'sell_high', 'sell_low', 'sell_close'])
# df = df.with_columns(
#     pl.mean_horizontal('buy_open', 'buy_high', 'buy_low', 'buy_close').alias('daily_buy_avg'),
#     pl.mean_horizontal('sell_open', 'sell_high', 'sell_low', 'sell_close').alias('daily_sell_avg')
# ).drop(['buy_open', 'buy_high', 'buy_low', 'buy_close', 'sell_open', 'sell_high', 'sell_low', 'sell_close'])

# Now we'll add in our supply-demand ratio and the drop those two 
df = df.with_columns(
    (pl.col('supply') / (pl.col('demand')+ 1)).alias('supply_demand_ratio')
).drop(['supply', 'demand'])

# We're also dropping buy and sell sma and rsi because they are not accessible to us in the moment of prediction
df = df.drop(['sell_sma', 'buy_sma'])
df.head()

<style> .dataframe > thead > tr, .dataframe > tbody > tr { text-align: right; white-space: pre-wrap; } </style> <small>shape: (5, 21)</small><thead><th>id</th><th>date</th><th>rsi</th><th>name</th><th>type</th><th>level</th><th>rarity</th><th>vendor_value</th><th>default_skin</th><th>game_types</th><th>flags</th><th>restrictions</th><th>chat_link</th><th>icon</th><th>details</th><th>description</th><th>upgrades_from</th><th>upgrades_into</th><th>current_buy_price</th><th>current_sell_price</th><th>supply_demand_ratio</th></thead><tbody></tbody><tbody></tbody>

i64

datetime[μs]

f64

str

str

i64

str

i64

str

str

str

str

str

str

f64

str

str

str

i64

i64

f64

12142

2025-06-17 00:00:00

49.92

"Onion"

"CraftingMaterial"

0

"Basic"

1

null

"{Activity,Wvw,Dungeon,Pve}"

"{NoSalvage}"

"{}"

"[&AgFuLwAA]"

"https://render.guildwars2.com/…

NaN

"Ingredient"

null

null

214

265

0.38

12142

2024-09-14 00:00:00

54.66

"Onion"

"CraftingMaterial"

0

"Basic"

1

null

"{Activity,Wvw,Dungeon,Pve}"

"{NoSalvage}"

"{}"

"[&AgFuLwAA]"

"https://render.guildwars2.com/…

NaN

"Ingredient"

null

null

223

227

0.70

12156

2026-02-10 00:00:00

49.41

"Jug of Water"

"CraftingMaterial"

0

"Basic"

1

null

"{Activity,Wvw,Dungeon,Pve}"

"{NoSalvage}"

"{}"

"[&AgF8LwAA]"

"https://render.guildwars2.com/…

NaN

"Ingredient"

null

null

8

11

0.20

12142

2025-01-10 00:00:00

48.42

"Onion"

"CraftingMaterial"

0

"Basic"

1

null

"{Activity,Wvw,Dungeon,Pve}"

"{NoSalvage}"

"{}"

"[&AgFuLwAA]"

"https://render.guildwars2.com/…

NaN

"Ingredient"

null

null

174

181

1.17

12138

2023-12-19 00:00:00

59.92

"Stick of Butter"

"CraftingMaterial"

0

"Basic"

1

null

"{Activity,Wvw,Dungeon,Pve}"

"{NoSalvage}"

"{}"

"[&AgFqLwAA]"

"https://render.guildwars2.com/…

NaN

"Ingredient"

null

null

325

382

0.94

# Let's go ahead and finish getting rid of what we don't need
df = df.drop(['rsi', 
              # 'name', # Keeping name in for now for spot checks.
              'default_skin', 
              'chat_link', 
              'icon', 
              'details',
              'description', 
              'upgrades_from', 
              'upgrades_into'])
df.head()

<style> .dataframe > thead > tr, .dataframe > tbody > tr { text-align: right; white-space: pre-wrap; } </style> <small>shape: (5, 13)</small><thead><th>id</th><th>date</th><th>name</th><th>type</th><th>level</th><th>rarity</th><th>vendor_value</th><th>game_types</th><th>flags</th><th>restrictions</th><th>current_buy_price</th><th>current_sell_price</th><th>supply_demand_ratio</th></thead><tbody></tbody><tbody></tbody>

i64

datetime[μs]

str

str

i64

str

i64

str

str

str

i64

i64

f64

12142

2025-06-17 00:00:00

"Onion"

"CraftingMaterial"

0

"Basic"

1

"{Activity,Wvw,Dungeon,Pve}"

"{NoSalvage}"

"{}"

214

265

0.38

12142

2024-09-14 00:00:00

"Onion"

"CraftingMaterial"

0

"Basic"

1

"{Activity,Wvw,Dungeon,Pve}"

"{NoSalvage}"

"{}"

223

227

0.70

12156

2026-02-10 00:00:00

"Jug of Water"

"CraftingMaterial"

0

"Basic"

1

"{Activity,Wvw,Dungeon,Pve}"

"{NoSalvage}"

"{}"

8

11

0.20

12142

2025-01-10 00:00:00

"Onion"

"CraftingMaterial"

0

"Basic"

1

"{Activity,Wvw,Dungeon,Pve}"

"{NoSalvage}"

"{}"

174

181

1.17

12138

2023-12-19 00:00:00

"Stick of Butter"

"CraftingMaterial"

0

"Basic"

1

"{Activity,Wvw,Dungeon,Pve}"

"{NoSalvage}"

"{}"

325

382

0.94

# The main feature we need to build is to keep track of the festivals in Guild Wars 2
# This is likely to be by far both the most important feature and the most prone to failure (since it will have
# to be updated manually)
def get_soup(url):
    # Request the HTML from the page
    page = requests.get(url)
    # Turn it into soup and return the soup
    return BeautifulSoup(page.content, 'html.parser')
# get_current_data()

def make_datetimes(soup):
    dates = [x.get_text().replace('(', '').replace(')', '')
                 for x in soup.find_all('small') if '—' in x.get_text()]
    dates = [(x.split('—')[0].strip(), x.split('—')[1].strip()) for x in dates]
    try:
        return [(int(datetime.strptime(x[0], '%Y-%m-%d').timestamp()), 
                 int(datetime.strptime(x[1], '%Y-%m-%d').timestamp()))
                for x in dates]
    except ValueError:
        new_list = []
        for x in dates:
            try:
                this_tuple = (int(datetime.strptime(x[0], '%Y-%m-%d').timestamp()), 
                              int(datetime.strptime(x[1], '%Y-%m-%d').timestamp()),)
                new_list.append(this_tuple)
            except ValueError:
                continue
        return new_list

def get_festivals():
    lunar_new_year_url = 'https://wiki.guildwars2.com/wiki/Lunar_New_Year'
    super_adv_fest_url = 'https://wiki.guildwars2.com/wiki/Super_Adventure_Festival'
    dragon_bash_fest_url = 'https://wiki.guildwars2.com/wiki/Dragon_Bash'
    fest_four_wind_url = 'https://wiki.guildwars2.com/wiki/Festival_of_the_Four_Winds'
    halloween_url = 'https://wiki.guildwars2.com/wiki/Halloween'
    wintersday_url = 'https://wiki.guildwars2.com/wiki/Wintersday'

    lunar_new_year_soup = get_soup(lunar_new_year_url)
    lny_datetimes = make_datetimes(lunar_new_year_soup)

    super_adv_fest_soup = get_soup(super_adv_fest_url)
    saf_datetimes = make_datetimes(super_adv_fest_soup)

    dragon_bash_fest_soup = get_soup(dragon_bash_fest_url)
    dbf_datetimes = make_datetimes(dragon_bash_fest_soup)

    fest_four_wind_soup = get_soup(fest_four_wind_url)
    ffw_datetimes = make_datetimes(fest_four_wind_soup)

    halloween_soup = get_soup(halloween_url)
    h_datetimes = make_datetimes(halloween_soup)

    wintersday_soup = get_soup(wintersday_url)
    w_datetimes = make_datetimes(wintersday_soup)

    return {'lunar_new_year': lny_datetimes,
                 'super_adventure_festival': saf_datetimes,
                 'dragon_bash': dbf_datetimes,
                 'festival_of_the_four_winds': ffw_datetimes,
                 'halloween': h_datetimes,
                 'wintersday': w_datetimes
                 }
    

datetime_dict = get_festivals()

# We now have a dictionary with festival names as keys and list of tuples as values each tuple has the beginning
# of a festival and end of the festival in datetime objects:
# {'lunar_new_year': [(datetime(beginning date), (datetime(end date)]}
for festival, tuple_list in datetime_dict.items():
    festival_dates = []
    next_week_dates = []
    last_week_dates = []
    for starttime, endtime in tuple_list:
        start_dt = datetime.fromtimestamp(starttime)
        end_dt = datetime.fromtimestamp(endtime)

        festival_dates.append(
            (pl.col('date') >= start_dt) &
            (pl.col('date') <= end_dt)
        )

        next_week_dates.append(
            (pl.col('date') >= (start_dt - timedelta(days=7))) &
            (pl.col('date') <= (end_dt - timedelta(days=1)))
        )

        last_week_dates.append(
            (pl.col('date') >= (start_dt + timedelta(days=1))) &
            (pl.col('date') <= (end_dt + timedelta(days=7)))
        )
        
    df = df.with_columns(
        pl.when(pl.any_horizontal(festival_dates))
        .then(1)
        .otherwise(0)
        .alias(festival)
    )
    df = df.with_columns(
        pl.when(pl.any_horizontal(next_week_dates))
        .then(1)
        .otherwise(0)
        .alias(f'{festival}_next_week')
    )
    df = df.with_columns(
        pl.when(pl.any_horizontal(last_week_dates))
        .then(1)
        .otherwise(0)
        .alias(f'{festival}_last_week')
    )

# THis should probably be moved up and done on just the items list before joining with historical data
# Now I want to separate that flags and game types into their own buckets.
types = ['PvpLobby', 'Activity', 'Wvw', 'Dungeon', 'Pve']
for t in types:
    df = df.with_columns(
        pl.when(pl.col('game_types').str.contains(t))
        .then(1)
        .otherwise(0)
        .alias(t)
    )

# Drop the original column
df = df.drop('game_types') 

# Now we should do the same with flags.
flags = ['NoSell', 'NoSalvage', 'Unique', 'NoMysticForge', 'DeleteWarning', 'NotUpgradeable']
for f in flags:
    df = df.with_columns(
        pl.when(pl.col('flags').str.contains(f))
        .then(1)
        .otherwise(0)
        .alias(f)
    )
df = df.drop('flags')

df['restrictions'].value_counts()

<style> .dataframe > thead > tr, .dataframe > tbody > tr { text-align: right; white-space: pre-wrap; } </style> <small>shape: (1, 2)</small><thead><th>restrictions</th><th>count</th></thead><tbody></tbody><tbody></tbody>

str	u32
"{}"	1785690

# That can be dropped since they're all empty
df = df.drop('restrictions')

# Now we're going to take our dates and make day of week, day of year, day of month, and sin and cosine of
# each so the model can understand the time series
df = df.with_columns(
    day_of_year = pl.col('date').dt.ordinal_day(),
    day_of_month = pl.col('date').dt.day(),
    day_of_week = pl.col('date').dt.weekday()
)

df = df.with_columns(
    sin_day_of_year = (2 * math.pi * pl.col('day_of_year') / 365).sin(),
    cos_day_of_year = (2 * math.pi * pl.col('day_of_year') / 365).cos(),
    
    sin_day_of_week = (2 * math.pi * pl.col('day_of_week') / 7).sin(),
    cos_day_of_week = (2 * math.pi * pl.col('day_of_week') / 7).cos(),
    
    sin_day_of_month = (2 * math.pi * pl.col('day_of_month') / pl.col('date').dt.month_end().dt.day()).sin(),
    cos_day_of_month = (2 * math.pi * pl.col('day_of_month') / pl.col('date').dt.month_end().dt.day()).cos()
)

# Drop what we don't need for date now
df = df.drop(['day_of_year', 'day_of_month', 'day_of_week'])

df = df.sort(["id", "date"])
df = df.with_columns(
    pl.col('current_buy_price').shift(-3).over('id').alias(f'target_{days}d')
)

df.describe()

<style> .dataframe > thead > tr, .dataframe > tbody > tr { text-align: right; white-space: pre-wrap; } </style> <small>shape: (9, 47)</small><thead><th>statistic</th><th>id</th><th>date</th><th>name</th><th>type</th><th>level</th><th>rarity</th><th>vendor_value</th><th>current_buy_price</th><th>current_sell_price</th><th>supply_demand_ratio</th><th>lunar_new_year</th><th>lunar_new_year_next_week</th><th>lunar_new_year_last_week</th><th>super_adventure_festival</th><th>super_adventure_festival_next_week</th><th>super_adventure_festival_last_week</th><th>dragon_bash</th><th>dragon_bash_next_week</th><th>dragon_bash_last_week</th><th>festival_of_the_four_winds</th><th>festival_of_the_four_winds_next_week</th><th>festival_of_the_four_winds_last_week</th><th>halloween</th><th>halloween_next_week</th><th>halloween_last_week</th><th>wintersday</th><th>wintersday_next_week</th><th>wintersday_last_week</th><th>PvpLobby</th><th>Activity</th><th>Wvw</th><th>Dungeon</th><th>Pve</th><th>NoSell</th><th>NoSalvage</th><th>Unique</th><th>NoMysticForge</th><th>DeleteWarning</th><th>NotUpgradeable</th><th>sin_day_of_year</th><th>cos_day_of_year</th><th>sin_day_of_week</th><th>cos_day_of_week</th><th>sin_day_of_month</th><th>cos_day_of_month</th><th>target_3d</th></thead><tbody></tbody><tbody></tbody>

str	f64	str	str	str	f64	str	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64
"count"	1785690.00	"1785690"	"1785690"	"1785690"	1785690.00	"1785690"	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1785690.00	1780818.00
"null_count"	0.00	"0"	"0"	"0"	0.00	"0"	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	4872.00
"mean"	34820.22	"2025-01-16 13:05:56.994103"	null	null	10.97	null	76.15	34690.62	65127.45	107212.38	0.06	0.08	0.08	0.06	0.08	0.08	0.06	0.08	0.08	0.06	0.07	0.07	0.06	0.08	0.08	0.07	0.08	0.08	0.03	0.92	0.99	0.99	1.00	0.09	0.95	0.00	0.00	0.01	0.00	0.01	-0.02	-0.00	-0.00	0.01	-0.00	34709.25
"std"	26867.50	null	null	null	24.45	null	950.76	576215.63	1002059.61	1907790.10	0.24	0.26	0.26	0.24	0.27	0.27	0.24	0.27	0.27	0.23	0.26	0.26	0.24	0.27	0.27	0.25	0.28	0.28	0.17	0.27	0.11	0.10	0.06	0.28	0.21	0.02	0.04	0.08	0.06	0.70	0.72	0.71	0.71	0.71	0.71	576557.07
"min"	12128.00	"2023-07-02 00:00:00"	"Aetherized Metal Scrap"	"CraftingMaterial"	0.00	"Ascended"	0.00	0.00	2.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	-1.00	-1.00	-0.97	-0.90	-1.00	-1.00	0.00
"25%"	13086.00	"2024-04-10 00:00:00"	null	null	0.00	null	5.00	51.00	127.00	0.51	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	1.00	1.00	1.00	1.00	0.00	1.00	0.00	0.00	0.00	0.00	-0.67	-0.76	-0.78	-0.90	-0.72	-0.73	51.00
"50%"	19917.00	"2025-01-19 00:00:00"	null	null	0.00	null	8.00	267.00	600.00	1.27	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	1.00	1.00	1.00	1.00	0.00	1.00	0.00	0.00	0.00	0.00	-0.01	-0.03	-0.00	-0.22	-0.00	-0.05	267.00
"75%"	62943.00	"2025-10-21 00:00:00"	null	null	0.00	null	21.00	2315.00	4127.00	3.54	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	1.00	1.00	1.00	1.00	0.00	1.00	0.00	0.00	0.00	0.00	0.70	0.70	0.78	0.62	0.72	0.69	2316.00
"max"	105004.00	"2026-07-29 00:00:00"	"Zucchini"	"Trophy"	80.00	"Rare"	30000.00	33333333.00	100000000.00	48696219.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	0.97	1.00	1.00	1.00	33333333.00

# We're going to convert the date back into its epoch time
df = df.with_columns(
    pl.col('date').dt.epoch(time_unit='s')
)

# Now we'll drop name.
df = df.drop(['name'])
df = df.drop_nulls()
df.head()

<style> .dataframe > thead > tr, .dataframe > tbody > tr { text-align: right; white-space: pre-wrap; } </style> <small>shape: (5, 45)</small><thead><th>id</th><th>date</th><th>type</th><th>level</th><th>rarity</th><th>vendor_value</th><th>current_buy_price</th><th>current_sell_price</th><th>supply_demand_ratio</th><th>lunar_new_year</th><th>lunar_new_year_next_week</th><th>lunar_new_year_last_week</th><th>super_adventure_festival</th><th>super_adventure_festival_next_week</th><th>super_adventure_festival_last_week</th><th>dragon_bash</th><th>dragon_bash_next_week</th><th>dragon_bash_last_week</th><th>festival_of_the_four_winds</th><th>festival_of_the_four_winds_next_week</th><th>festival_of_the_four_winds_last_week</th><th>halloween</th><th>halloween_next_week</th><th>halloween_last_week</th><th>wintersday</th><th>wintersday_next_week</th><th>wintersday_last_week</th><th>PvpLobby</th><th>Activity</th><th>Wvw</th><th>Dungeon</th><th>Pve</th><th>NoSell</th><th>NoSalvage</th><th>Unique</th><th>NoMysticForge</th><th>DeleteWarning</th><th>NotUpgradeable</th><th>sin_day_of_year</th><th>cos_day_of_year</th><th>sin_day_of_week</th><th>cos_day_of_week</th><th>sin_day_of_month</th><th>cos_day_of_month</th><th>target_3d</th></thead><tbody></tbody><tbody></tbody>

i64	i64	str	i64	str	i64	i64	i64	f64	i32	i32	i32	i32	i32	i32	i32	i32	i32	i32	i32	i32	i32	i32	i32	i32	i32	i32	i32	i32	i32	i32	i32	i32	i32	i32	i32	i32	i32	f64	f64	f64	f64	f64	f64	i64
12128	1688256000	"CraftingMaterial"	80	"Basic"	9	55	59	21.45	0	0	0	0	0	0	0	0	1	0	0	0	0	0	0	0	0	0	0	1	1	1	1	0	1	0	0	0	0	-0.01	-1.00	-0.00	1.00	0.39	0.92	65
12128	1688342400	"CraftingMaterial"	80	"Basic"	9	58	63	22.16	0	0	0	0	0	0	0	0	1	0	0	0	0	0	0	0	0	0	0	1	1	1	1	0	1	0	0	0	0	-0.03	-1.00	0.78	0.62	0.57	0.82	60
12128	1688428800	"CraftingMaterial"	80	"Basic"	9	64	68	20.10	0	0	0	0	0	0	0	0	1	0	0	0	0	0	0	0	0	0	0	1	1	1	1	0	1	0	0	0	0	-0.04	-1.00	0.97	-0.22	0.72	0.69	60
12128	1688515200	"CraftingMaterial"	80	"Basic"	9	65	67	20.68	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	1	1	1	1	0	1	0	0	0	0	-0.06	-1.00	0.43	-0.90	0.85	0.53	60
12128	1688601600	"CraftingMaterial"	80	"Basic"	9	60	70	20.26	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	1	1	1	1	0	1	0	0	0	0	-0.08	-1.00	-0.43	-0.90	0.94	0.35	62

# Based on early testing, I could see a clear split in performance for lower cost items
# We're going to split our dataset into two now and train two separate models
threshold = 10000 # 10,000 copper = 1 gold.
penny_df= df.filter(pl.col('current_buy_price') < threshold)
luxury_df = df.filter(pl.col('current_buy_price') >= threshold)

from sklearn.compose import ColumnTransformer
# from sklearn.feature_selection import mutual_info_regression, SelectKBest
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler, OneHotEncoder, MinMaxScaler, RobustScaler, FunctionTransformer
from tensorflow.keras.layers import Concatenate, Dense, Dropout, Embedding, Flatten, Input, IntegerLookup
from tensorflow.keras.models import Model
from tensorflow.keras.regularizers import l2
from tensorflow.keras.callbacks import EarlyStopping

2026-08-21 20:16:54.339415: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: SSE3 SSE4.1 SSE4.2 AVX AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.

target_cols = [f'target_{days}d']
# We need to separate out our target, which will be buy_high
penny_targets = penny_df.select(target_cols)
luxury_targets = luxury_df.select(target_cols)
penny_df = penny_df.drop(target_cols)
luxury_df = luxury_df.drop(target_cols)
print('Target separated from features.')

Target separated from features.

del df
gc.collect()
print('df deleted and garbage collected.')

df deleted and garbage collected.

# Turn datetime back to seconds (epoch), change NaNs to None and 0
penny_df = penny_df.with_columns(
    pl.col('rarity').fill_null('None'),
    pl.col('level').fill_null(0),
    pl.col('vendor_value').fill_null(0),
).to_pandas()
# Turn datetime back to seconds (epoch), change NaNs to None and 0
luxury_df = luxury_df.with_columns(
    pl.col('rarity').fill_null('None'),
    pl.col('level').fill_null(0),
    pl.col('vendor_value').fill_null(0),
).to_pandas()
print('Dataframes converted to Pandas')

Dataframes converted to Pandas

penny_df.columns == luxury_df.columns

array([ True,  True,  True,  True,  True,  True,  True,  True,  True,
        True,  True,  True,  True,  True,  True,  True,  True,  True,
        True,  True,  True,  True,  True,  True,  True,  True,  True,
        True,  True,  True,  True,  True,  True,  True,  True,  True,
        True,  True,  True,  True,  True,  True,  True,  True])

# set masks for the dates
penny_train_mask = penny_df['date'] <= datetime(2024, 12, 31).timestamp()
penny_val_mask = (penny_df['date'] >= datetime(2025, 1, 1).timestamp()) & (penny_df['date'] <= datetime(2025, 6, 30).timestamp())
penny_test_mask = penny_df['date'] >= datetime(2025, 7, 1).timestamp()

penny_df = penny_df.drop('date', axis=1)

penny_x_train = penny_df[penny_train_mask]
penny_x_val = penny_df[penny_val_mask]
penny_x_test = penny_df[penny_test_mask]

penny_y_train = penny_targets.to_pandas().loc[penny_x_train.index]
penny_y_val = penny_targets.to_pandas().loc[penny_x_val.index]
penny_y_test = penny_targets.to_pandas().loc[penny_x_test.index]

# Do exactly the same as above but for luxury
luxury_train_mask = luxury_df['date'] <= datetime(2024, 12, 31).timestamp()
luxury_val_mask = (luxury_df['date'] >= datetime(2025, 1, 1).timestamp()) & (luxury_df['date'] <= datetime(2025, 6, 30).timestamp())
luxury_test_mask = luxury_df['date'] >= datetime(2025, 7, 1).timestamp()

luxury_df = luxury_df.drop('date', axis=1)

luxury_x_train = luxury_df[luxury_train_mask]
luxury_x_val = luxury_df[luxury_val_mask]
luxury_x_test = luxury_df[luxury_test_mask]

luxury_y_train = luxury_targets.to_pandas().loc[luxury_x_train.index]
luxury_y_val = luxury_targets.to_pandas().loc[luxury_x_val.index]
luxury_y_test = luxury_targets.to_pandas().loc[luxury_x_test.index]

numeric_cols = ['supply_demand_ratio', 'level', 'vendor_value', 'current_sell_price', 'current_buy_price']
cat_cols = ['type', 'rarity']
passthrough_cols = [x for x in penny_df.columns if x not in numeric_cols and x not in cat_cols]
print('Train, validation, and test sets created.')
print('Numeric columns:', numeric_cols)
print('Categorical columns:', cat_cols)
print('Passthrough columns:', passthrough_cols)

Train, validation, and test sets created.
Numeric columns: ['supply_demand_ratio', 'level', 'vendor_value', 'current_sell_price', 'current_buy_price']
Categorical columns: ['type', 'rarity']
Passthrough columns: ['id', 'lunar_new_year', 'lunar_new_year_next_week', 'lunar_new_year_last_week', 'super_adventure_festival', 'super_adventure_festival_next_week', 'super_adventure_festival_last_week', 'dragon_bash', 'dragon_bash_next_week', 'dragon_bash_last_week', 'festival_of_the_four_winds', 'festival_of_the_four_winds_next_week', 'festival_of_the_four_winds_last_week', 'halloween', 'halloween_next_week', 'halloween_last_week', 'wintersday', 'wintersday_next_week', 'wintersday_last_week', 'PvpLobby', 'Activity', 'Wvw', 'Dungeon', 'Pve', 'NoSell', 'NoSalvage', 'Unique', 'NoMysticForge', 'DeleteWarning', 'NotUpgradeable', 'sin_day_of_year', 'cos_day_of_year', 'sin_day_of_week', 'cos_day_of_week', 'sin_day_of_month', 'cos_day_of_month']

# Extract unique item IDs from your Polars dataframe to build the vocabulary
penny_unique_ids = penny_df['id'].unique()
luxury_unique_ids = luxury_df['id'].unique()

rarities = ['Basic', 'Fine', 'Rare', 'Masterwork', 'Exotic', 'Ascended', 'Legendary']
penny_preprocess = ColumnTransformer(transformers=[
    ('num_scaler', RobustScaler(), numeric_cols),
    ('cat_encode', OneHotEncoder(categories='auto', handle_unknown='error', sparse_output=False), cat_cols),
    ('passthrough', 'passthrough', passthrough_cols)
],)

luxury_preprocess = ColumnTransformer(transformers=[
    ('num_scaler', RobustScaler(), numeric_cols),
    ('cat_encode', OneHotEncoder(categories='auto', handle_unknown='error', sparse_output=False), cat_cols),
    ('passthrough', 'passthrough', passthrough_cols)
],)

# Make it output dataframes
penny_preprocess.set_output(transform='pandas')
luxury_preprocess.set_output(transform='pandas')

# Set up y preprocessing
penny_y_preprocess = Pipeline([
    ('log_trans', FunctionTransformer(func=np.log1p, inverse_func=np.expm1, validate=False)),
    ('scaler', MinMaxScaler())
])

luxury_y_preprocess = Pipeline([
    ('log_trans', FunctionTransformer(func=np.log1p, inverse_func=np.expm1, validate=False)),
    ('scaler', MinMaxScaler())
])

print('Pipelines ready')

Pipelines ready

penny_raw_test_buy_high = penny_x_test['current_buy_price'].values
luxury_raw_test_buy_high = luxury_x_test['current_buy_price'].values

penny_x_train = penny_preprocess.fit_transform(penny_x_train)
penny_x_val = penny_preprocess.transform(penny_x_val)
penny_x_test = penny_preprocess.transform(penny_x_test)
print('Penny x_train and x_test preprocessed.')
print(penny_x_train.columns)
penny_y_train = penny_y_preprocess.fit_transform(penny_y_train)
penny_y_val = penny_y_preprocess.transform(penny_y_val)
penny_y_test = penny_y_preprocess.transform(penny_y_test)
print('Penny y_train and y_test preprocessed')

luxury_x_train = luxury_preprocess.fit_transform(luxury_x_train)
luxury_x_val = luxury_preprocess.transform(luxury_x_val)
luxury_x_test = luxury_preprocess.transform(luxury_x_test)
print('luxury x_train and x_test preprocessed.')
print(luxury_x_train.columns)
luxury_y_train = luxury_y_preprocess.fit_transform(luxury_y_train)
luxury_y_val = luxury_y_preprocess.transform(luxury_y_val)
luxury_y_test = luxury_y_preprocess.transform(luxury_y_test)
print('luxury y_train and y_test preprocessed')

Penny x_train and x_test preprocessed.
Index(['num_scaler__supply_demand_ratio', 'num_scaler__level',
       'num_scaler__vendor_value', 'num_scaler__current_sell_price',
       'num_scaler__current_buy_price', 'cat_encode__type_CraftingMaterial',
       'cat_encode__type_Trophy', 'cat_encode__rarity_Ascended',
       'cat_encode__rarity_Basic', 'cat_encode__rarity_Exotic',
       'cat_encode__rarity_Fine', 'cat_encode__rarity_Legendary',
       'cat_encode__rarity_Masterwork', 'cat_encode__rarity_Rare',
       'passthrough__id', 'passthrough__lunar_new_year',
       'passthrough__lunar_new_year_next_week',
       'passthrough__lunar_new_year_last_week',
       'passthrough__super_adventure_festival',
       'passthrough__super_adventure_festival_next_week',
       'passthrough__super_adventure_festival_last_week',
       'passthrough__dragon_bash', 'passthrough__dragon_bash_next_week',
       'passthrough__dragon_bash_last_week',
       'passthrough__festival_of_the_four_winds',
       'passthrough__festival_of_the_four_winds_next_week',
       'passthrough__festival_of_the_four_winds_last_week',
       'passthrough__halloween', 'passthrough__halloween_next_week',
       'passthrough__halloween_last_week', 'passthrough__wintersday',
       'passthrough__wintersday_next_week',
       'passthrough__wintersday_last_week', 'passthrough__PvpLobby',
       'passthrough__Activity', 'passthrough__Wvw', 'passthrough__Dungeon',
       'passthrough__Pve', 'passthrough__NoSell', 'passthrough__NoSalvage',
       'passthrough__Unique', 'passthrough__NoMysticForge',
       'passthrough__DeleteWarning', 'passthrough__NotUpgradeable',
       'passthrough__sin_day_of_year', 'passthrough__cos_day_of_year',
       'passthrough__sin_day_of_week', 'passthrough__cos_day_of_week',
       'passthrough__sin_day_of_month', 'passthrough__cos_day_of_month'],
      dtype='str')
Penny y_train and y_test preprocessed
luxury x_train and x_test preprocessed.
Index(['num_scaler__supply_demand_ratio', 'num_scaler__level',
       'num_scaler__vendor_value', 'num_scaler__current_sell_price',
       'num_scaler__current_buy_price', 'cat_encode__type_CraftingMaterial',
       'cat_encode__type_Trophy', 'cat_encode__rarity_Ascended',
       'cat_encode__rarity_Basic', 'cat_encode__rarity_Exotic',
       'cat_encode__rarity_Fine', 'cat_encode__rarity_Legendary',
       'cat_encode__rarity_Masterwork', 'cat_encode__rarity_Rare',
       'passthrough__id', 'passthrough__lunar_new_year',
       'passthrough__lunar_new_year_next_week',
       'passthrough__lunar_new_year_last_week',
       'passthrough__super_adventure_festival',
       'passthrough__super_adventure_festival_next_week',
       'passthrough__super_adventure_festival_last_week',
       'passthrough__dragon_bash', 'passthrough__dragon_bash_next_week',
       'passthrough__dragon_bash_last_week',
       'passthrough__festival_of_the_four_winds',
       'passthrough__festival_of_the_four_winds_next_week',
       'passthrough__festival_of_the_four_winds_last_week',
       'passthrough__halloween', 'passthrough__halloween_next_week',
       'passthrough__halloween_last_week', 'passthrough__wintersday',
       'passthrough__wintersday_next_week',
       'passthrough__wintersday_last_week', 'passthrough__PvpLobby',
       'passthrough__Activity', 'passthrough__Wvw', 'passthrough__Dungeon',
       'passthrough__Pve', 'passthrough__NoSell', 'passthrough__NoSalvage',
       'passthrough__Unique', 'passthrough__NoMysticForge',
       'passthrough__DeleteWarning', 'passthrough__NotUpgradeable',
       'passthrough__sin_day_of_year', 'passthrough__cos_day_of_year',
       'passthrough__sin_day_of_week', 'passthrough__cos_day_of_week',
       'passthrough__sin_day_of_month', 'passthrough__cos_day_of_month'],
      dtype='str')
luxury y_train and y_test preprocessed

# Extract 'id' for the embedding input
penny_train_id = penny_x_train['passthrough__id']
penny_val_id = penny_x_val['passthrough__id']
penny_test_id = penny_x_test['passthrough__id']

# Extract all other continuous and one-hot features for the dense input
penny_train_features = penny_x_train.drop(columns=['passthrough__id'])
penny_val_features = penny_x_val.drop(columns=['passthrough__id'])
penny_test_features = penny_x_test.drop(columns=['passthrough__id'])

# Do the same as above for luxury
luxury_train_id = luxury_x_train['passthrough__id']
luxury_val_id = luxury_x_val['passthrough__id']
luxury_test_id = luxury_x_test['passthrough__id']

# Extract all other continuous and one-hot features for the dense input
luxury_train_features = luxury_x_train.drop(columns=['passthrough__id'])
luxury_val_features = luxury_x_val.drop(columns=['passthrough__id'])
luxury_test_features = luxury_x_test.drop(columns=['passthrough__id'])

import joblib
joblib.dump(penny_preprocess, f'Models/PennyLuxury/penny_preprocess_{days}d.joblib')
joblib.dump(penny_y_preprocess, f'Models/PennyLuxury/penny_y_preprocess_{days}d.joblib')

joblib.dump(luxury_preprocess, f'Models/PennyLuxury/luxury_preprocess_{days}d.joblib')
joblib.dump(luxury_y_preprocess, f'Models/PennyLuxury/luxury_y_preprocess_{days}d.joblib')

['Models/PennyLuxury/luxury_y_preprocess_3d.joblib']