Compare commits
13
Commits
No files matched your search
Vendored
+4
-1
@@ -1,3 +1,6 @@
|
||||
{
|
||||
"recommendations": ["ms-python.python", "ms-toolsai.jupyter"]
|
||||
"recommendations": [
|
||||
"ms-python.python",
|
||||
"ms-toolsai.jupyter"
|
||||
]
|
||||
}
|
||||
@@ -14,8 +14,10 @@ how to git? do not `push --force`. prefer pull requests. its like github
|
||||
|
||||
we will plan to do this in multiple different ways using scikit-learn and pytorch. tensorflow is ancient.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> phase out tensorflow. dont use it.
|
||||
> [!NOTE]
|
||||
> ~phase out tensorflow. dont use it.~
|
||||
>
|
||||
> its 2 different platforms for 2 different types of people. use whatever ig
|
||||
|
||||
maybe also fit more weather related stats? this dataset is taken from nasa so sources are there
|
||||
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import pandas as pd
|
||||
import io
|
||||
from typing import Union
|
||||
|
||||
|
||||
def cut_headers(stream: Union[io.StringIO, io.FileIO, io.TextIOWrapper]):
|
||||
"""Remove custom header metadata from the CSV file."""
|
||||
headers = []
|
||||
exist_startheader = False
|
||||
while line := stream.readline():
|
||||
if "-BEGIN HEADER-" in line and not exist_startheader:
|
||||
exist_startheader = True
|
||||
headers.append(str(line.strip()))
|
||||
elif "-END HEADER-" in line and exist_startheader:
|
||||
headers.append(str(line.strip()))
|
||||
break
|
||||
elif exist_startheader:
|
||||
headers.append(str(line.strip()))
|
||||
elif not exist_startheader:
|
||||
continue
|
||||
if not exist_startheader:
|
||||
stream.seek(0)
|
||||
return "", stream
|
||||
return "\n".join(headers), stream
|
||||
|
||||
|
||||
def load_weather_data(file_path: str) -> pd.DataFrame:
|
||||
"""Load weather data and preprocess the datetime index."""
|
||||
with open(file_path, "r", encoding="utf-8") as file:
|
||||
_, csv_file = cut_headers(file)
|
||||
data = pd.read_csv(csv_file, na_values=-999)
|
||||
|
||||
# Convert YEAR, MO, DY to datetime index
|
||||
if all(col in data.columns for col in ("YEAR", "MO", "DY")):
|
||||
data["date"] = pd.to_datetime(
|
||||
{k: data[v] for v, k in zip(["YEAR", "MO", "DY"], ["year", "month", "day"])}
|
||||
)
|
||||
data.drop(columns=["YEAR", "MO", "DY"], inplace=True)
|
||||
data.set_index("date", inplace=True)
|
||||
|
||||
data.dropna(inplace=True)
|
||||
data = data.asfreq("D") # Assumes regular daily data
|
||||
return data
|
||||
-14429
File diff suppressed because it is too large.
Load diff
+19
-13
@@ -2,9 +2,11 @@ import urllib.parse
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
|
||||
data_file = "hanoi_weather_data.csv"
|
||||
|
||||
# Define parameters
|
||||
latitude, longitude = 21.04815477090267, 105.80123159165096 # lmao
|
||||
start_date = "2010-01-01"
|
||||
start_date = "1985-01-01"
|
||||
end_date = datetime.now().strftime("%Y-%m-%d") # Today's date
|
||||
|
||||
# Define desired parameters
|
||||
@@ -16,9 +18,12 @@ end_date = datetime.now().strftime("%Y-%m-%d") # Today's date
|
||||
# WS10M MERRA-2 Wind Speed at 10 Meters (m/s)
|
||||
# WD10M MERRA-2 Wind Direction at 10 Meters (Degrees)
|
||||
# more at https://power.larc.nasa.gov/data-access-viewer/
|
||||
weather_params = [
|
||||
"QV2M",
|
||||
]
|
||||
|
||||
params = {
|
||||
"parameters": "QV2M,T2M,PRECTOTCORR",
|
||||
"parameters": ",".join(weather_params),
|
||||
"community": "RE",
|
||||
"longitude": longitude,
|
||||
"latitude": latitude,
|
||||
@@ -36,14 +41,15 @@ base_url = "https://power.larc.nasa.gov/api/temporal/daily/point"
|
||||
url = f"{base_url}?{query_string}"
|
||||
|
||||
# Make the API request using urllib
|
||||
try:
|
||||
with urllib.request.urlopen(url) as response:
|
||||
if response.status == 200:
|
||||
csv_data = response.read().decode("utf-8")
|
||||
with open("hanoi_weather_data.csv", "w", encoding="utf-8") as file:
|
||||
file.write(csv_data)
|
||||
print("Data successfully downloaded and saved to 'hanoi_weather_data.csv'.")
|
||||
else:
|
||||
print(f"Failed to retrieve data: HTTP Status Code {response.status}")
|
||||
except urllib.error.URLError as e:
|
||||
print(f"Failed to retrieve data: {e.reason}")
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
with urllib.request.urlopen(url) as response:
|
||||
if response.status == 200:
|
||||
csv_data = response.read()
|
||||
with open(data_file, "wb") as file:
|
||||
file.write(csv_data)
|
||||
print(f"Data successfully downloaded and saved to {data_file}.")
|
||||
else:
|
||||
print(f"Failed to retrieve data: HTTP Status Code {response.status}")
|
||||
except urllib.error.URLError as e:
|
||||
print(f"Failed to retrieve data: {e.reason}")
|
||||
File diff suppressed because it is too large.
Load diff
@@ -8,7 +8,7 @@ from sklearn.preprocessing import MinMaxScaler
|
||||
import numpy as np
|
||||
from load_data import load_this_data
|
||||
|
||||
a = load_this_data("dataset.csv")
|
||||
a = load_this_data()
|
||||
|
||||
# time series forecasting
|
||||
|
||||
@@ -19,6 +19,7 @@ scaled_data = scaler.fit_transform(a)
|
||||
|
||||
|
||||
def create_sequences(data, seq_length):
|
||||
"see this? return[0] is the big x and return[1] is the big y. and h(x) = y. and h(x) is the model."
|
||||
xs = []
|
||||
ys = []
|
||||
for i in range(len(data) - seq_length):
|
||||
@@ -0,0 +1 @@
|
||||
DATA_FILE_NAME = "hanoi_weather_data.csv"
|
||||
@@ -0,0 +1,28 @@
|
||||
"extracting the load data step from the notebook"
|
||||
|
||||
import io
|
||||
|
||||
# from itertools import islice
|
||||
from typing import Union
|
||||
|
||||
def cut_headers(stream: Union[io.StringIO, io.FileIO, io.TextIOWrapper]):
|
||||
headers = []
|
||||
exist_startheader = False
|
||||
|
||||
while line := stream.readline():
|
||||
if "-BEGIN HEADER-" in line and not exist_startheader:
|
||||
exist_startheader = True
|
||||
headers.append(str(line.strip()))
|
||||
elif "-END HEADER-" in line and exist_startheader:
|
||||
headers.append(str(line.strip()))
|
||||
break
|
||||
elif exist_startheader:
|
||||
# print(f"{line = }")
|
||||
headers.append(str(line.strip()))
|
||||
elif not exist_startheader:
|
||||
continue
|
||||
if not exist_startheader:
|
||||
# If no headers are found, reset the stream and return
|
||||
stream.seek(0)
|
||||
return "", stream
|
||||
return "\n".join(headers), stream
|
||||
+2100
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,163 @@
|
||||
from sklearn.model_selection import train_test_split
|
||||
import torch
|
||||
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
from sklearn.preprocessing import MinMaxScaler
|
||||
|
||||
import numpy as np
|
||||
from load_data import load_this_data
|
||||
|
||||
a = load_this_data()
|
||||
|
||||
# time series forecasting
|
||||
|
||||
|
||||
scaler = MinMaxScaler()
|
||||
print(a.columns)
|
||||
scaled_data = scaler.fit_transform(a)
|
||||
|
||||
|
||||
def create_sequences(data, seq_length):
|
||||
"see this? return[0] is the big x and return[1] is the big y. and h(x) = y. and h(x) is the model."
|
||||
xs = []
|
||||
ys = []
|
||||
for i in range(len(data) - seq_length):
|
||||
x = data[i : (i + seq_length)]
|
||||
y = data[i + seq_length]
|
||||
xs.append(x)
|
||||
ys.append(y)
|
||||
return np.array(xs), np.array(ys)
|
||||
|
||||
|
||||
seq_length = 20
|
||||
|
||||
X, y = create_sequences(scaled_data, seq_length)
|
||||
|
||||
X_train, X_test, y_train, y_test = train_test_split(
|
||||
X, y, test_size=0.2, random_state=42
|
||||
)
|
||||
from torch.utils.data import TensorDataset, DataLoader
|
||||
|
||||
X_train_tensor = torch.tensor(X_train, dtype=torch.float32)
|
||||
y_train_tensor = torch.tensor(y_train, dtype=torch.float32)
|
||||
|
||||
train_dataset = TensorDataset(X_train_tensor, y_train_tensor)
|
||||
train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
|
||||
|
||||
|
||||
import torch.nn as nn
|
||||
|
||||
|
||||
class LSTMModel(nn.Module):
|
||||
def __init__(self, input_size, hidden_size, num_layers, output_size):
|
||||
super(LSTMModel, self).__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.num_layers = num_layers
|
||||
self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)
|
||||
self.fc = nn.Linear(hidden_size, output_size)
|
||||
|
||||
def forward(self, x):
|
||||
h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device)
|
||||
c0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device)
|
||||
|
||||
out, _ = self.lstm(x, (h0, c0))
|
||||
out = self.fc(out[:, -1, :]) # Take the output from the last time step
|
||||
return out
|
||||
|
||||
|
||||
model = LSTMModel(
|
||||
input_size=X_train.shape[2],
|
||||
hidden_size=64,
|
||||
num_layers=2,
|
||||
output_size=y_train.shape[1],
|
||||
)
|
||||
model.to(device)
|
||||
criterion = nn.MSELoss()
|
||||
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
|
||||
|
||||
train_losses = []
|
||||
val_losses = []
|
||||
|
||||
|
||||
for epoch in range(100):
|
||||
model.train()
|
||||
running_train_loss = 0.0
|
||||
|
||||
# inputs, targets
|
||||
for batch_X, batch_y in train_loader:
|
||||
batch_X = batch_X.to(device)
|
||||
batch_y = batch_y.to(device)
|
||||
|
||||
outputs = model(batch_X)
|
||||
loss = criterion(outputs, batch_y)
|
||||
|
||||
optimizer.zero_grad()
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
|
||||
running_train_loss += loss.item() * batch_X.size(0)
|
||||
if (epoch + 1) % 10 == 0:
|
||||
print(f"Epoch [{epoch+1}/100], Loss: {loss.item():.4f}")
|
||||
|
||||
epoch_train_loss = running_train_loss / len(train_loader.dataset)
|
||||
train_losses.append(epoch_train_loss)
|
||||
|
||||
model.eval()
|
||||
|
||||
X_test_tensor = torch.tensor(X_test, dtype=torch.float32).to(device)
|
||||
y_test_tensor = torch.tensor(y_test, dtype=torch.float32).to(device)
|
||||
val_dataset = TensorDataset(X_test_tensor, y_test_tensor)
|
||||
val_loader = DataLoader(val_dataset, batch_size=64, shuffle=False)
|
||||
|
||||
# with torch.no_grad():
|
||||
# running_val_loss = 0.0
|
||||
# predictions = model(X_test_tensor)
|
||||
# test_loss = criterion(predictions, y_test_tensor)
|
||||
# print(f"Test Loss: {test_loss.item():.4f}")
|
||||
|
||||
running_val_loss = 0.0
|
||||
predictions = []
|
||||
with torch.no_grad():
|
||||
for inputs, targets in val_loader:
|
||||
inputs, targets = inputs.to(device), targets.to(device)
|
||||
outputs = model(inputs)
|
||||
loss = criterion(outputs, targets)
|
||||
running_val_loss += loss.item() * inputs.size(0)
|
||||
predictions.append(outputs.cpu().numpy()) # Collect all batches
|
||||
|
||||
epoch_val_loss = running_val_loss / len(val_loader.dataset) # len(val_dataset)
|
||||
val_losses.append(epoch_val_loss)
|
||||
|
||||
predictions = np.concatenate(predictions, axis=0)
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
y_test_cpu = y_test_tensor.cpu().numpy()
|
||||
|
||||
predictions_rescaled = scaler.inverse_transform(predictions)
|
||||
y_test_rescaled = scaler.inverse_transform(y_test_cpu)
|
||||
|
||||
|
||||
fig, (ax1, ax2) = plt.subplots(
|
||||
1, 2, figsize=(14, 5), gridspec_kw={"width_ratios": [1, 3]}
|
||||
)
|
||||
|
||||
# Loss curve on the left
|
||||
ax1.plot(train_losses, label="Train Loss")
|
||||
ax1.plot(val_losses, label="Val Loss")
|
||||
ax1.set_title("Loss Curve")
|
||||
ax1.set_xlabel("Epoch")
|
||||
ax1.set_ylabel("Loss")
|
||||
ax1.legend()
|
||||
ax1.grid(True)
|
||||
|
||||
# Prediction vs Actual on the right
|
||||
ax2.plot(y_test_rescaled, label="Actual")
|
||||
ax2.plot(predictions_rescaled, label="Predicted")
|
||||
ax2.set_title("Actual vs. Predicted")
|
||||
ax2.set_xlabel("Sample")
|
||||
ax2.set_ylabel("Value")
|
||||
ax2.legend()
|
||||
|
||||
plt.tight_layout()
|
||||
plt.show()
|
||||
@@ -0,0 +1,185 @@
|
||||
import sys
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import matplotlib.pyplot as plt
|
||||
import seaborn as sns
|
||||
import io
|
||||
from typing import Union
|
||||
from sklearn.metrics import mean_squared_error, r2_score
|
||||
from sklearn.preprocessing import MinMaxScaler
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.linear_model import LinearRegression
|
||||
from statsmodels.tsa.ar_model import AutoReg
|
||||
|
||||
# ==== LOAD & CLEAN DATA ====
|
||||
DATA_FILE_NAME = "hanoi_weather_data.csv"
|
||||
|
||||
def cut_headers(stream: Union[io.StringIO, io.FileIO, io.TextIOWrapper]):
|
||||
headers = []
|
||||
exist_startheader = False
|
||||
while line := stream.readline():
|
||||
if "-BEGIN HEADER-" in line and not exist_startheader:
|
||||
exist_startheader = True
|
||||
headers.append(str(line.strip()))
|
||||
elif "-END HEADER-" in line and exist_startheader:
|
||||
headers.append(str(line.strip()))
|
||||
break
|
||||
elif exist_startheader:
|
||||
headers.append(str(line.strip()))
|
||||
elif not exist_startheader:
|
||||
continue
|
||||
if not exist_startheader:
|
||||
stream.seek(0)
|
||||
return "", stream
|
||||
return "\n".join(headers), stream
|
||||
|
||||
with open(DATA_FILE_NAME, "r", encoding="utf-8") as file:
|
||||
data_header, csv_file = cut_headers(file)
|
||||
data = pd.read_csv(csv_file, na_values=-999)
|
||||
|
||||
if all(z in data.columns for z in ("YEAR", "MO", "DY")):
|
||||
data["date"] = pd.to_datetime(
|
||||
{k: data[v] for v, k in zip(["YEAR", "MO", "DY"], ["year", "month", "day"])}
|
||||
)
|
||||
data.drop(columns=["YEAR", "MO", "DY"], inplace=True)
|
||||
data.set_index("date", inplace=True)
|
||||
|
||||
data.dropna(inplace=True)
|
||||
|
||||
# ==== VISUALIZE CORRELATION ====
|
||||
plt.figure(figsize=(12, 8))
|
||||
sns.heatmap(data.corr(), annot=True, cmap="coolwarm")
|
||||
plt.title("Feature Correlation Heatmap")
|
||||
plt.show()
|
||||
|
||||
# ==== SCALE DATA ====
|
||||
scaler = MinMaxScaler()
|
||||
scaled_data = scaler.fit_transform(data)
|
||||
|
||||
# ==== CREATE SEQUENCES ====
|
||||
def create_sequences(data, seq_length):
|
||||
xs, ys = [], []
|
||||
for i in range(len(data) - seq_length):
|
||||
x = data[i:i + seq_length].flatten()
|
||||
y = data[i + seq_length]
|
||||
xs.append(x)
|
||||
ys.append(y)
|
||||
return np.array(xs), np.array(ys)
|
||||
|
||||
seq_length = 50
|
||||
X, y = create_sequences(scaled_data, seq_length)
|
||||
|
||||
# ==== SPLIT ====
|
||||
date_index = data.index[seq_length:]
|
||||
X_train, X_test, y_train, y_test, date_train, date_test = train_test_split(
|
||||
X, y, date_index, test_size=0.02, shuffle=False
|
||||
)
|
||||
|
||||
# ==== TRAIN LINEAR REGRESSION ====
|
||||
model = LinearRegression()
|
||||
model.fit(X_train, y_train)
|
||||
predictions = model.predict(X_test)
|
||||
|
||||
# ==== EVALUATION ====
|
||||
mse = mean_squared_error(y_test, predictions)
|
||||
r2 = r2_score(y_test, predictions)
|
||||
|
||||
print(f"Test MSE: {mse:.4f}")
|
||||
print(f"Test R2: {r2:.4f}")
|
||||
print(f"Model Score: {model.score(X_test, y_test):.4f}")
|
||||
|
||||
# ==== INVERSE TRANSFORM ====
|
||||
predictions_rescaled = scaler.inverse_transform(
|
||||
predictions.reshape(-1, data.shape[1])
|
||||
)
|
||||
y_test_rescaled = scaler.inverse_transform(
|
||||
y_test.reshape(-1, data.shape[1])
|
||||
)
|
||||
|
||||
# ==== PLOTS ====
|
||||
|
||||
# Actual vs Predicted with Real Dates
|
||||
plt.figure(figsize=(12, 6))
|
||||
plt.plot(date_test, y_test_rescaled[:, 0], label="Actual")
|
||||
plt.plot(date_test, predictions_rescaled[:, 0], label="Predicted")
|
||||
plt.title("Actual vs Predicted (First Feature)")
|
||||
plt.xlabel("Date")
|
||||
plt.ylabel("Original Scale")
|
||||
plt.xticks(rotation=45)
|
||||
plt.legend()
|
||||
plt.grid(True)
|
||||
plt.tight_layout()
|
||||
plt.show()
|
||||
|
||||
# Residual Plot
|
||||
residuals = y_test[:, 0] - predictions[:, 0]
|
||||
plt.figure(figsize=(10, 4))
|
||||
plt.plot(date_test, residuals, label="Residuals")
|
||||
plt.hlines(0, 0, len(residuals), colors="red")
|
||||
plt.title("Residuals Over Time (First Feature)")
|
||||
plt.xlabel("Date")
|
||||
plt.ylabel("Error")
|
||||
plt.xticks(rotation=45)
|
||||
plt.legend()
|
||||
plt.tight_layout()
|
||||
plt.grid(True)
|
||||
plt.show()
|
||||
|
||||
# Error Distribution
|
||||
errors = np.abs(y_test[:, 0] - predictions[:, 0])
|
||||
plt.hist(errors, bins=50)
|
||||
plt.title("Error Distribution (First Feature)")
|
||||
plt.xlabel("Absolute Error")
|
||||
plt.ylabel("Frequency")
|
||||
plt.grid(True)
|
||||
plt.show()
|
||||
|
||||
# Scatter Plot
|
||||
plt.scatter(y_test[:, 0], predictions[:, 0], alpha=0.5)
|
||||
plt.plot([0, 1], [0, 1], "r--")
|
||||
plt.title("Predicted vs Actual (Normalized)")
|
||||
plt.xlabel("Actual")
|
||||
plt.ylabel("Predicted")
|
||||
plt.grid(True)
|
||||
plt.show()
|
||||
|
||||
# ==== AUTOREGRESSION FORECAST ====
|
||||
target_column = data.columns[0]
|
||||
series = data[target_column]
|
||||
|
||||
train_size = int(len(series) * 0.98)
|
||||
train_series = series[:train_size]
|
||||
test_series = series[train_size:]
|
||||
|
||||
ar_model = AutoReg(train_series, lags=30).fit()
|
||||
ar_preds = ar_model.predict(start=len(train_series), end=len(series)-1)
|
||||
|
||||
# Plot AR Forecast
|
||||
plt.figure(figsize=(12, 5))
|
||||
plt.plot(test_series.index, test_series, label="Actual")
|
||||
plt.plot(test_series.index, ar_preds, label="AR Forecast", linestyle="--")
|
||||
plt.title(f"AutoRegression Forecast - {target_column}")
|
||||
plt.xlabel("Date")
|
||||
plt.ylabel(target_column)
|
||||
plt.xticks(rotation=45)
|
||||
plt.legend()
|
||||
plt.grid(True)
|
||||
plt.tight_layout()
|
||||
plt.show()
|
||||
|
||||
# ==== FUTURE FORECAST ====
|
||||
future_steps = 30
|
||||
future_index = pd.date_range(start=series.index[-1], periods=future_steps+1, freq="D")[1:]
|
||||
future_forecast = ar_model.predict(start=len(series), end=len(series) + future_steps - 1)
|
||||
|
||||
plt.figure(figsize=(12, 5))
|
||||
plt.plot(series[-100:], label="Recent Data")
|
||||
plt.plot(future_index, future_forecast, label="Future Forecast", color="red")
|
||||
plt.title(f"{target_column} - 30 Day Forecast (AutoRegression)")
|
||||
plt.xlabel("Date")
|
||||
plt.ylabel(target_column)
|
||||
plt.xticks(rotation=45)
|
||||
plt.legend()
|
||||
plt.tight_layout()
|
||||
plt.grid(True)
|
||||
plt.show()
|
||||
@@ -0,0 +1,110 @@
|
||||
import sys
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn.metrics import r2_score
|
||||
from sklearn.preprocessing import MinMaxScaler
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.linear_model import LinearRegression
|
||||
from sklearn.metrics import mean_squared_error
|
||||
import matplotlib.pyplot as plt
|
||||
# from global_config import DATA_FILE_NAME
|
||||
DATA_FILE_NAME = "hanoi_weather_data.csv"
|
||||
|
||||
# from load_data import cut_headers
|
||||
|
||||
import io
|
||||
|
||||
# from itertools import islice
|
||||
from typing import Union
|
||||
|
||||
def cut_headers(stream: Union[io.StringIO, io.FileIO, io.TextIOWrapper]):
|
||||
headers = []
|
||||
exist_startheader = False
|
||||
|
||||
while line := stream.readline():
|
||||
if "-BEGIN HEADER-" in line and not exist_startheader:
|
||||
exist_startheader = True
|
||||
headers.append(str(line.strip()))
|
||||
elif "-END HEADER-" in line and exist_startheader:
|
||||
headers.append(str(line.strip()))
|
||||
break
|
||||
elif exist_startheader:
|
||||
# print(f"{line = }")
|
||||
headers.append(str(line.strip()))
|
||||
elif not exist_startheader:
|
||||
continue
|
||||
if not exist_startheader:
|
||||
# If no headers are found, reset the stream and return
|
||||
stream.seek(0)
|
||||
return "", stream
|
||||
return "\n".join(headers), stream
|
||||
|
||||
# 1. Load data
|
||||
|
||||
with open(DATA_FILE_NAME, "r", encoding="utf-8") as file:
|
||||
data_header, csv_file = cut_headers(file)
|
||||
data = pd.read_csv(csv_file, na_values=-999)
|
||||
|
||||
if all(z in data.columns for z in ("YEAR", "MO", "DY")):
|
||||
data["date"] = pd.to_datetime(
|
||||
{k: data[v] for v, k in (zip(["YEAR", "MO", "DY"], ["year", "month", "day"]))}
|
||||
)
|
||||
data.drop(inplace=True, columns=["YEAR", "MO", "DY"])
|
||||
data.set_index(["date"], inplace=True)
|
||||
data.dropna(inplace=True)
|
||||
|
||||
# 2. Normalize data
|
||||
scaler = MinMaxScaler()
|
||||
print(data.columns)
|
||||
scaled_data = scaler.fit_transform(data)
|
||||
|
||||
|
||||
# 3. Create sequences for supervised learning
|
||||
def create_sequences(data, seq_length):
|
||||
xs, ys = [], []
|
||||
for i in range(len(data) - seq_length):
|
||||
x = data[i : i + seq_length].flatten()
|
||||
y = data[i + seq_length]
|
||||
xs.append(x)
|
||||
ys.append(y)
|
||||
return np.array(xs), np.array(ys)
|
||||
|
||||
|
||||
seq_length = 50
|
||||
X, y = create_sequences(scaled_data, seq_length)
|
||||
|
||||
# 4. Split into training and testing
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.02)
|
||||
|
||||
# 5. Train linear regression model
|
||||
model = LinearRegression()
|
||||
model.fit(X_train, y_train)
|
||||
|
||||
# 6. Make predictions
|
||||
predictions = model.predict(X_test)
|
||||
|
||||
# 7. Evaluate performance
|
||||
mse = mean_squared_error(y_test, predictions)
|
||||
r2 = r2_score(y_test, predictions)
|
||||
|
||||
print(f"Test MSE: {mse:.4f}")
|
||||
print(f"Test r2: {r2:.4f}")
|
||||
|
||||
print(f"Score: {model.score(X_test, y_test)}")
|
||||
# 8. Inverse transform predictions
|
||||
predictions_rescaled = scaler.inverse_transform(predictions.reshape(-1, data.shape[1]))
|
||||
y_test_rescaled = scaler.inverse_transform(y_test.reshape(-1, data.shape[1]))
|
||||
|
||||
# 9. Plot results
|
||||
|
||||
# Loss bar
|
||||
|
||||
|
||||
# Prediction vs Actual
|
||||
plt.plot(y_test_rescaled, label="Actual")
|
||||
plt.plot(predictions_rescaled, label="Predicted")
|
||||
plt.title("Actual vs. Predicted")
|
||||
plt.xlabel("Sample")
|
||||
plt.ylabel("Value")
|
||||
plt.legend()
|
||||
plt.show()
|
||||
+14752
File diff suppressed because it is too large.
Load diff
@@ -1,87 +0,0 @@
|
||||
"extracting the load data step from the notebook"
|
||||
|
||||
import io
|
||||
|
||||
# from itertools import islice
|
||||
from typing import Union
|
||||
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def cut_headers(stream: Union[io.StringIO, io.FileIO, io.TextIOWrapper]):
|
||||
headers = []
|
||||
exist_startheader = False
|
||||
|
||||
while line := stream.readline():
|
||||
if "-BEGIN HEADER-" in line and not exist_startheader:
|
||||
exist_startheader = True
|
||||
headers.append(str(line.strip()))
|
||||
elif "-END HEADER-" in line and exist_startheader:
|
||||
headers.append(str(line.strip()))
|
||||
break
|
||||
elif exist_startheader:
|
||||
# print(f"{line = }")
|
||||
headers.append(str(line.strip()))
|
||||
elif not exist_startheader:
|
||||
continue
|
||||
if not exist_startheader:
|
||||
# If no headers are found, reset the stream and return
|
||||
stream.seek(0)
|
||||
return "", stream
|
||||
return "\n".join(headers), stream
|
||||
|
||||
|
||||
# if __name__ == "__main__":
|
||||
# with open("hanoi_qv2m_latest.csv", "r", encoding="utf-8") as file:
|
||||
# header, file = cut_headers(file)
|
||||
# print("Extracted Header:")
|
||||
# print(header)
|
||||
# data = pd.read_csv(file)
|
||||
# print("\nFirst 5 Lines of Data:")
|
||||
# print(data.head())
|
||||
# print(data.isnull().sum())
|
||||
|
||||
|
||||
def load_data(
|
||||
weather_file_name, columns=["Year", "Month", "Day", "Humidity"].copy(), na=-999
|
||||
):
|
||||
with open(weather_file_name, "r", encoding="utf-8") as file:
|
||||
_header, file = cut_headers(file)
|
||||
data = pd.read_csv(file, na_values=na)
|
||||
# print(header)
|
||||
data.columns = columns
|
||||
return data
|
||||
|
||||
|
||||
def date_combine(data: pd.DataFrame):
|
||||
data["Date"] = pd.to_datetime(data[["Year", "Month", "Day"]])
|
||||
data = data.drop(columns=["Year", "Month", "Day"])
|
||||
data = data.set_index("Date")
|
||||
return data
|
||||
|
||||
|
||||
def validate_view(data: pd.DataFrame, verbose=False):
|
||||
if verbose:
|
||||
print(f"Shape with N/A values: {data.shape}")
|
||||
isn = data.isnull().sum().iloc[0]
|
||||
print(f"Count of N/A values: {isn}")
|
||||
data = data.dropna() # Drop the rows with missing values
|
||||
if verbose:
|
||||
isn = data.isnull().sum().iloc[0]
|
||||
print(f"Count after drop: {isn}")
|
||||
print(data.describe())
|
||||
return data
|
||||
|
||||
|
||||
# if __name__ == "__main__":
|
||||
def load_this_data(filename, verbose=False):
|
||||
d = load_data(filename)
|
||||
# if verbose:
|
||||
# print(d.columns)
|
||||
d = date_combine(d)
|
||||
# if verbose:
|
||||
# print(d.columns)
|
||||
d = validate_view(d, verbose)
|
||||
# if verbose:
|
||||
# print(d.columns)
|
||||
return d
|
||||
-1161
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,71 @@
|
||||
# import pandas as pd
|
||||
from data_loader import load_weather_data
|
||||
from preprocessing import scale_data, create_sequences, inverse_scale
|
||||
from model_train import (
|
||||
train_linear_regression_gd,
|
||||
predict_linear_regression,
|
||||
evaluate_model,
|
||||
)
|
||||
from visualize import (
|
||||
plot_original_vs_scaled,
|
||||
plot_predictions,
|
||||
plot_residuals,
|
||||
plot_error_hist,
|
||||
plot_scatter,
|
||||
plot_loss_curve,
|
||||
plot_value_distribution,
|
||||
visualize_sequence_window,
|
||||
)
|
||||
from sklearn.model_selection import train_test_split
|
||||
|
||||
# Step 1: Load data
|
||||
data = load_weather_data("hanoi_weather_data.csv")
|
||||
|
||||
plot_value_distribution(data)
|
||||
|
||||
# Step 2: Preprocess (scale and sequence)
|
||||
scaled_data, scaler = scale_data(data)
|
||||
plot_original_vs_scaled(data, scaled_data)
|
||||
|
||||
seq_length = 50
|
||||
X, y = create_sequences(scaled_data, seq_length)
|
||||
|
||||
visualize_sequence_window(data["QV2M"], seq_length=10, start_idx=1000)
|
||||
|
||||
# Step 3: Train/test split
|
||||
date_index = data.index[seq_length:]
|
||||
X_train, X_test, y_train, y_test = train_test_split(
|
||||
X, y.reshape(-1), test_size=0.02, shuffle=False
|
||||
)
|
||||
date_train, date_test = train_test_split(date_index, test_size=0.02, shuffle=False)
|
||||
|
||||
# Step 4: Train model with gradient descent
|
||||
# import math
|
||||
# lr_schedule = lambda epoch: max(0.005, 0.01 * (1 + math.cos(epoch / 1000 * math.pi)) / 2)
|
||||
|
||||
theta, losses = train_linear_regression_gd(
|
||||
X_train,
|
||||
y_train,
|
||||
# lr_schedule=lr_schedule,
|
||||
n_epochs=10000,
|
||||
)
|
||||
|
||||
predictions = predict_linear_regression(X_test, theta)
|
||||
|
||||
# Step 5: Evaluate
|
||||
mse, r2 = evaluate_model(y_test, predictions)
|
||||
print(f"MSE: {mse:.4f}, R2: {r2:.4f}")
|
||||
|
||||
# Step 6: Rescale predictions
|
||||
y_test_rescaled = inverse_scale(y_test, scaler, data.shape[1])
|
||||
predictions_rescaled = inverse_scale(predictions, scaler, data.shape[1])
|
||||
|
||||
# Step 7: Visualize
|
||||
residuals = y_test - predictions
|
||||
errors = abs(residuals)
|
||||
|
||||
plot_predictions(date_test, y_test_rescaled, predictions_rescaled)
|
||||
plot_residuals(date_test.to_numpy(), residuals)
|
||||
plot_error_hist(errors)
|
||||
plot_scatter(y_test.reshape(-1, 1), predictions.reshape(-1, 1))
|
||||
plot_loss_curve(losses)
|
||||
@@ -0,0 +1,45 @@
|
||||
import numpy as np
|
||||
from sklearn.metrics import mean_squared_error, r2_score
|
||||
|
||||
|
||||
def train_linear_regression_gd(X_train, y_train, lr_schedule= lambda _e: 0.01, n_epochs=1000):
|
||||
"""
|
||||
Train linear regression model using batch gradient descent.
|
||||
|
||||
Returns:
|
||||
theta: learned weights
|
||||
losses: list of MSE loss per epoch
|
||||
"""
|
||||
m, n = X_train.shape
|
||||
X_b = np.hstack([np.ones((m, 1)), X_train])
|
||||
theta = np.zeros(n + 1)
|
||||
losses = []
|
||||
|
||||
for epoch in range(n_epochs):
|
||||
lr = lr_schedule(epoch)
|
||||
predictions = X_b @ theta
|
||||
errors = predictions - y_train
|
||||
gradients = (2 / m) * X_b.T @ errors
|
||||
gradients = np.clip(gradients, -1, 1) # gradient clipping
|
||||
theta -= lr * gradients
|
||||
loss = np.mean(errors ** 2)
|
||||
losses.append(loss)
|
||||
|
||||
if epoch % 1000 == 0:
|
||||
print(f"Epoch {epoch}: Loss = {loss:.6f}, LR: {lr:.6f}")
|
||||
|
||||
print(f"Final Loss: {losses[-1]:.6f}")
|
||||
return theta, losses
|
||||
|
||||
|
||||
def predict_linear_regression(X, theta):
|
||||
"""Predict using learned linear model."""
|
||||
X_b = np.hstack([np.ones((X.shape[0], 1)), X])
|
||||
return X_b @ theta
|
||||
|
||||
|
||||
def evaluate_model(y_true, y_pred):
|
||||
"""Evaluate using sklearn's built-in MSE and R² functions."""
|
||||
mse = mean_squared_error(y_true, y_pred)
|
||||
r2 = r2_score(y_true, y_pred)
|
||||
return mse, r2
|
||||
@@ -0,0 +1,26 @@
|
||||
import numpy as np
|
||||
from sklearn.preprocessing import MinMaxScaler
|
||||
from sklearn.model_selection import train_test_split
|
||||
|
||||
|
||||
def scale_data(data):
|
||||
"""Normalize the data using MinMaxScaler."""
|
||||
scaler = MinMaxScaler()
|
||||
scaled_data = scaler.fit_transform(data)
|
||||
return scaled_data, scaler
|
||||
|
||||
|
||||
def inverse_scale(scaled, scaler, n_features):
|
||||
"""Inverse transform using stored scaler."""
|
||||
return scaler.inverse_transform(scaled.reshape(-1, n_features))
|
||||
|
||||
|
||||
def create_sequences(data, seq_length):
|
||||
"""Transform the time series into overlapping sequences."""
|
||||
xs, ys = [], []
|
||||
for i in range(len(data) - seq_length):
|
||||
x = data[i : i + seq_length].flatten()
|
||||
y = data[i + seq_length]
|
||||
xs.append(x)
|
||||
ys.append(y)
|
||||
return np.array(xs), np.array(ys)
|
||||
@@ -1,304 +0,0 @@
|
||||
"""
|
||||
This script demonstrates how to port a TensorFlow LSTM model to PyTorch.
|
||||
It includes detailed explanations for every step.
|
||||
"""
|
||||
|
||||
# Step 1: Import necessary libraries
|
||||
# ----------------------------------
|
||||
# PyTorch is the core library for building and training neural networks.
|
||||
# Torch.nn contains modules for defining layers and loss functions.
|
||||
# Torch.optim provides optimizers like Adam for training.
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import torch
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.preprocessing import MinMaxScaler
|
||||
from torch import nn, optim
|
||||
from torch.utils.data import DataLoader, TensorDataset
|
||||
|
||||
import sys
|
||||
|
||||
sys.path.append(r"c:\Users\Admin\Documents\lecture ass shit\ml1\codemidterm\ml_midterm")
|
||||
|
||||
from load_data import load_this_data
|
||||
|
||||
# Step 2: Data Preparation
|
||||
# ------------------------
|
||||
# In PyTorch, data must be converted into tensors. We'll simulate the data
|
||||
# preprocessing steps from your TensorFlow code.
|
||||
|
||||
|
||||
# Simulate some data (replace this with your actual data loading logic)
|
||||
np.random.seed(42) # For reproducibility
|
||||
# data = np.sin(np.linspace(0, 100, 1000)) # Example: Sine wave data
|
||||
# data = data.reshape(-1, 1) # Reshape to (samples, features)
|
||||
|
||||
|
||||
# Load the dataset (replace "dataset.csv" with your actual file path)
|
||||
data = load_this_data("dataset.csv")
|
||||
|
||||
|
||||
# Extract the "Humidity" column as the target variable
|
||||
data = data["Humidity"].values.reshape(-1, 1)
|
||||
|
||||
# Normalize the data using MinMaxScaler (same as in TensorFlow)
|
||||
scaler = MinMaxScaler()
|
||||
data_scaled = scaler.fit_transform(data)
|
||||
|
||||
# Define lookback window (sequence length for LSTM)
|
||||
n_lookback = 100 # Number of past timesteps to consider
|
||||
n_features = 1 # Number of features (e.g., 1 for univariate time series)
|
||||
|
||||
|
||||
# Helper function to create sequences
|
||||
def create_sequences(data, n_lookback):
|
||||
"""
|
||||
Create sequences of length n_lookback for LSTM input.
|
||||
|
||||
Args:
|
||||
data: Scaled data as a NumPy array or PyTorch tensor.
|
||||
n_lookback: Number of past timesteps to consider.
|
||||
|
||||
Returns:
|
||||
x: Input sequences of shape (num_samples, n_lookback, n_features).
|
||||
y: Target values of shape (num_samples, 1).
|
||||
"""
|
||||
x, y = [], []
|
||||
for i in range(len(data) - n_lookback):
|
||||
x.append(data[i : i + n_lookback])
|
||||
y.append(data[i + n_lookback])
|
||||
return np.array(x), np.array(y)
|
||||
|
||||
|
||||
# Split into training and testing sets
|
||||
split_index = int(0.8 * len(data_scaled))
|
||||
train_data = data_scaled[:split_index]
|
||||
test_data = data_scaled[split_index:]
|
||||
|
||||
# Create sequences for training and testing
|
||||
train_x, train_y = create_sequences(train_data, n_lookback)
|
||||
test_x, test_y = create_sequences(test_data, n_lookback)
|
||||
|
||||
# train_y = scaler.transform(train_y)
|
||||
# test_y = scaler.transform(test_y)
|
||||
# Convert sequences to PyTorch tensors
|
||||
train_x = torch.tensor(train_x, dtype=torch.float32)
|
||||
train_y = torch.tensor(train_y, dtype=torch.float32)
|
||||
test_x = torch.tensor(test_x, dtype=torch.float32)
|
||||
test_y = torch.tensor(test_y, dtype=torch.float32)
|
||||
|
||||
|
||||
# Create datasets and dataloaders
|
||||
train_dataset = TensorDataset(train_x, train_y)
|
||||
test_dataset = TensorDataset(test_x, test_y)
|
||||
|
||||
train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
|
||||
test_loader = DataLoader(test_dataset, batch_size=32, shuffle=False)
|
||||
|
||||
# Step 3: Define the LSTM Model
|
||||
# -----------------------------
|
||||
# PyTorch models are defined by subclassing `nn.Module`.
|
||||
# The forward method specifies how data flows through the model.
|
||||
|
||||
|
||||
# class LSTMModel(nn.Module):
|
||||
# def __init__(self, n_features, n_lookback, n_neurons):
|
||||
# """
|
||||
# Initialize the LSTM model.
|
||||
|
||||
# Args:
|
||||
# n_features: Number of input features (e.g., 1 for univariate time series).
|
||||
# n_lookback: Number of past timesteps to consider.
|
||||
# n_neurons: Number of neurons in each LSTM layer.
|
||||
# """
|
||||
# super(LSTMModel, self).__init__()
|
||||
# self.lstm1 = nn.LSTM(
|
||||
# n_features,
|
||||
# n_neurons,
|
||||
# batch_first=True,
|
||||
# dropout=0.2,
|
||||
# )
|
||||
# self.lstm2 = nn.LSTM(
|
||||
# n_neurons,
|
||||
# n_neurons,
|
||||
# batch_first=True,
|
||||
# )
|
||||
# self.lstm3 = nn.LSTM(n_neurons, n_neurons, batch_first=True)
|
||||
# self.lstm4 = nn.LSTM(n_neurons, n_neurons, batch_first=True)
|
||||
# self.fc = nn.Linear(n_neurons, 1)
|
||||
# self.dropout = nn.Dropout(0.2)
|
||||
|
||||
# def forward(self, x):
|
||||
# """
|
||||
# Forward pass through the model.
|
||||
|
||||
# Args:
|
||||
# x: Input tensor of shape (batch, seq_len, n_features).
|
||||
|
||||
# Returns:
|
||||
# Output tensor of shape (batch, 1).
|
||||
# """
|
||||
# x, _ = self.lstm1(x)
|
||||
# x, _ = self.lstm2(x)
|
||||
# x, _ = self.lstm3(x)
|
||||
# x, _ = self.lstm4(x)
|
||||
# x = self.dropout(x)
|
||||
# x = self.fc(x[:, -1, :]) # Use the last timestep's output
|
||||
# return x
|
||||
|
||||
print("Input sequences (x):", train_x.shape)
|
||||
print("Target values (y):", train_y.shape)
|
||||
|
||||
print("Sample input:", train_x[0])
|
||||
print("Corresponding target:", train_y[0])
|
||||
|
||||
class LSTMModel(nn.Module):
|
||||
def __init__(self, n_features, n_lookback, n_neurons):
|
||||
super(LSTMModel, self).__init__()
|
||||
self.lstm = nn.LSTM(n_features, n_neurons, batch_first=True, dropout=0.2)
|
||||
self.lstm2 = nn.LSTM(n_neurons, n_neurons, batch_first=True, dropout=0.2)
|
||||
|
||||
self.fc = nn.Linear(n_neurons, 1)
|
||||
|
||||
def forward(self, x):
|
||||
x, _ = self.lstm(x)
|
||||
x, _ = self.lstm2(x)
|
||||
x = self.fc(x[:, -1, :]) # Use the last timestep's output
|
||||
return x
|
||||
|
||||
|
||||
# Instantiate the model
|
||||
n_neurons = 128 # Number of neurons in the LSTM
|
||||
model = LSTMModel(n_features, n_lookback, n_neurons)
|
||||
|
||||
# Step 4: Define Loss Function and Optimizer
|
||||
# ------------------------------------------
|
||||
# Loss function: Mean Squared Error (MSE) for regression tasks.
|
||||
# Optimizer: Adam optimizer for efficient training.
|
||||
criterion = nn.MSELoss()
|
||||
optimizer = optim.Adam(model.parameters(), lr=0.001)
|
||||
|
||||
# Step 5: Train the Model
|
||||
# -----------------------
|
||||
# PyTorch requires an explicit training loop. This gives you full control
|
||||
# over the training process.
|
||||
|
||||
epochs = 50 # Number of training epochs
|
||||
|
||||
# Check for CUDA and set the device
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
print(f"Using device: {device}")
|
||||
|
||||
# Move the model to the device
|
||||
model = LSTMModel(n_features, n_lookback, n_neurons).to(device)
|
||||
|
||||
# scheduler = optim.lr_scheduler.ReduceLROnPlateau(
|
||||
# optimizer, mode="min", patience=5, factor=0.5, min_lr=1e-6
|
||||
# )
|
||||
# Move data to the device during training and evaluation
|
||||
for epoch in range(epochs):
|
||||
model.train() # Set the model to training mode
|
||||
train_loss = 0.0
|
||||
|
||||
for x_batch, y_batch in train_loader:
|
||||
# Move data to the device
|
||||
x_batch = x_batch.to(device)
|
||||
y_batch = y_batch.to(device)
|
||||
# Reshape input to (batch, seq_len, n_features)
|
||||
x_batch = x_batch.view(-1, n_lookback, n_features)
|
||||
# y_batch = y_batch.view(-1, 1)
|
||||
# print("x_batch shape:", x_batch.shape)
|
||||
# print("y_batch shape:", y_batch.shape)
|
||||
# print("x_batch dtype:", x_batch.dtype)
|
||||
# print("y_batch dtype:", y_batch.dtype)
|
||||
# break
|
||||
# Zero the gradients from the previous step
|
||||
optimizer.zero_grad()
|
||||
|
||||
# Forward pass: Compute predictions
|
||||
predictions = model(x_batch)
|
||||
# if epoch == 32:
|
||||
# for name, param in model.named_parameters():
|
||||
# if param.grad is not None:
|
||||
# print(f"{name} gradient mean: {param.grad.mean().item()}")
|
||||
# else:
|
||||
# print(f"{name} has no gradient")
|
||||
# print("Predictions device:", predictions.device)
|
||||
# print("y_batch device:", y_batch.device)
|
||||
# print("Predictions shape:", predictions.shape)
|
||||
# print("y_batch shape:", y_batch.shape)
|
||||
# Compute the loss
|
||||
loss = criterion(predictions, y_batch)
|
||||
|
||||
# Backward pass: Compute gradients
|
||||
loss.backward()
|
||||
|
||||
# Clip gradients to prevent exploding gradients
|
||||
torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
|
||||
|
||||
# Update model parameters
|
||||
optimizer.step()
|
||||
# scheduler.step(train_loss / len(train_loader))
|
||||
# Accumulate the loss for reporting
|
||||
train_loss += loss.item()
|
||||
|
||||
# Print epoch loss
|
||||
print(f"Epoch {epoch+1}/{epochs}, Loss: {train_loss/len(train_loader):.4f}")
|
||||
|
||||
if epoch % 10 == 0: # Print every 10 epochs
|
||||
print(
|
||||
f"Sample prediction: {predictions[0].item()}, Actual: {y_batch[0].item()}"
|
||||
)
|
||||
# Step 6: Evaluate the Model
|
||||
# --------------------------
|
||||
# Switch to evaluation mode and disable gradient computation.
|
||||
model.eval()
|
||||
with torch.no_grad():
|
||||
predictions = []
|
||||
actuals = []
|
||||
|
||||
for x_batch, y_batch in test_loader:
|
||||
# Move data to the device
|
||||
x_batch = x_batch.to(device)
|
||||
y_batch = y_batch.to(device)
|
||||
|
||||
# Reshape input to (batch, seq_len, n_features)
|
||||
# x_batch = x_batch.view(-1, n_lookback, n_features)
|
||||
|
||||
# Make predictions
|
||||
preds = model(x_batch)
|
||||
|
||||
# Move predictions and actuals back to CPU for further processing
|
||||
predictions.append(preds.cpu().numpy())
|
||||
actuals.append(y_batch.cpu().numpy())
|
||||
|
||||
# Convert predictions and actuals to NumPy arrays for comparison
|
||||
predictions = np.concatenate(predictions)
|
||||
actuals = np.concatenate(actuals)
|
||||
|
||||
# Step 7: Post-Processing and Visualization
|
||||
# -----------------------------------------
|
||||
# Inverse transform the predictions and actuals to get original scale
|
||||
predictions = scaler.inverse_transform(predictions)
|
||||
actuals = scaler.inverse_transform(actuals)
|
||||
|
||||
# Plot the results
|
||||
print("Sample scaled train_y:", train_y[:5])
|
||||
print("Sample scaled test_y:", test_y[:5])
|
||||
|
||||
print("Predictions shape:", predictions.shape)
|
||||
print("Actuals shape:", actuals.shape)
|
||||
|
||||
print("Sample inverse-transformed predictions:", predictions[:5])
|
||||
print("Sample inverse-transformed actuals:", actuals[:5])
|
||||
|
||||
plt.figure(figsize=(20, 5))
|
||||
plt.plot(actuals, label="Actual")
|
||||
plt.plot(predictions, label="Predicted")
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
from sklearn.metrics import r2_score, mean_squared_error
|
||||
|
||||
print("R² Score:", r2_score(actuals, predictions))
|
||||
print("MSE:", mean_squared_error(actuals, predictions))
|
||||
@@ -1,124 +0,0 @@
|
||||
"""this uses new tech
|
||||
make by conda new -p path python=3.12; conda activate -p path; conda install all those shi
|
||||
|
||||
conda install pytorch torchvision torchaudio pytorch-cuda -c pytorch -c nvidia
|
||||
"""
|
||||
|
||||
import sys
|
||||
|
||||
sys.path.append(".")
|
||||
from load_data import load_this_data
|
||||
import torch
|
||||
|
||||
# a = load_this_data("dataset.csv")
|
||||
|
||||
x = torch.tensor([[1, 2], [3, 4]])
|
||||
print(x)
|
||||
|
||||
|
||||
print(x.shape) # Output: torch.Size([2, 2])
|
||||
print(x.dtype) # Output: torch.int64
|
||||
print(x.device) # Output: cpu
|
||||
|
||||
y = torch.ones_like(x)
|
||||
print(x + y)
|
||||
print(x * y)
|
||||
|
||||
|
||||
x = torch.randn(3, requires_grad=True)
|
||||
y = x * 2
|
||||
z = y.mean()
|
||||
z.backward()
|
||||
print(x.grad)
|
||||
|
||||
import torch.nn as nn
|
||||
|
||||
class SimpleNet(nn.Module):
|
||||
def __init__(self):
|
||||
super(SimpleNet, self).__init__()
|
||||
self.fc = nn.Linear(2, 1)
|
||||
|
||||
def forward(self, x):
|
||||
return self.fc(x)
|
||||
|
||||
|
||||
model = SimpleNet()
|
||||
sample_input = torch.tensor([[1.0, 2.0]])
|
||||
output = model(sample_input)
|
||||
print(output)
|
||||
|
||||
import torch.optim as optim
|
||||
|
||||
criterion = nn.MSELoss()
|
||||
optimizer = optim.SGD(model.parameters(), lr=0.01)
|
||||
|
||||
for epoch in range(100):
|
||||
optimizer.zero_grad()
|
||||
outputs = model(sample_input)
|
||||
loss = criterion(outputs, torch.tensor([[1.0]]))
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
if (epoch+1) % 10 == 0:
|
||||
print(f'Epoch [{epoch+1}/100], Loss: {loss.item():.4f}')
|
||||
|
||||
model.eval()
|
||||
|
||||
sample_input_2 = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
|
||||
output_2 = model(sample_input_2)
|
||||
print(output_2)
|
||||
|
||||
|
||||
from torchvision import datasets, transforms
|
||||
from torch.utils.data import DataLoader
|
||||
|
||||
# Define a transform to normalize the data
|
||||
transform = transforms.Compose([transforms.ToTensor()])
|
||||
|
||||
# Download and load the training data
|
||||
train_dataset = datasets.MNIST(root='data', train=True, download=True, transform=transform)
|
||||
train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)
|
||||
|
||||
|
||||
# import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
class Net(nn.Module):
|
||||
def __init__(self):
|
||||
super(Net, self).__init__()
|
||||
self.fc1 = nn.Linear(28 * 28, 128)
|
||||
self.fc2 = nn.Linear(128, 10)
|
||||
|
||||
def forward(self, x):
|
||||
x = x.view(-1, 28 * 28) # Flatten the input
|
||||
x = F.relu(self.fc1(x))
|
||||
x = self.fc2(x)
|
||||
return x
|
||||
|
||||
model_2 = Net()
|
||||
criterion_2 = nn.CrossEntropyLoss()
|
||||
optimizer_2 = torch.optim.SGD(model.parameters(), lr=0.01)
|
||||
|
||||
|
||||
for epoch in range(35): # loop over the dataset multiple times
|
||||
for inputs, labels in train_loader:
|
||||
optimizer_2.zero_grad() # zero the parameter gradients
|
||||
outputs = model_2(inputs) # forward
|
||||
loss = criterion_2(outputs, labels) # compute loss
|
||||
loss.backward() # backward
|
||||
optimizer_2.step() # optimize
|
||||
print(f'Epoch {epoch+1}, Loss: {loss.item()}')
|
||||
|
||||
|
||||
test_dataset = datasets.MNIST(root='data', train=False, download=True, transform=transform)
|
||||
test_loader = DataLoader(test_dataset, batch_size=64, shuffle=False)
|
||||
|
||||
|
||||
correct = 0
|
||||
total = 0
|
||||
with torch.no_grad():
|
||||
for inputs, labels in test_loader:
|
||||
outputs = model_2(inputs)
|
||||
_, predicted = torch.max(outputs.data, 1)
|
||||
total += labels.size(0)
|
||||
correct += (predicted == labels).sum().item()
|
||||
print(f'Accuracy: {100 * correct / total}%')
|
||||
+137
@@ -0,0 +1,137 @@
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
|
||||
def plot_predictions(date_test, y_test_rescaled, predictions_rescaled):
|
||||
plt.figure(figsize=(12, 6))
|
||||
plt.plot(date_test, y_test_rescaled[:, 0], label="Actual")
|
||||
plt.plot(date_test, predictions_rescaled[:, 0], label="Predicted")
|
||||
plt.title("Actual vs Predicted (First Feature)")
|
||||
plt.xlabel("Date")
|
||||
plt.ylabel("Value")
|
||||
plt.xticks(rotation=45)
|
||||
plt.legend()
|
||||
plt.tight_layout()
|
||||
plt.grid(True)
|
||||
plt.show()
|
||||
|
||||
|
||||
def plot_residuals(date_test, residuals):
|
||||
plt.figure(figsize=(10, 4))
|
||||
plt.plot(date_test, residuals, label="Residuals")
|
||||
plt.hlines(0, 0, len(residuals), colors="red")
|
||||
plt.xlim(date_test.min(), date_test.max())
|
||||
plt.title("Residuals Over Time")
|
||||
plt.xlabel("Date")
|
||||
plt.ylabel("Error")
|
||||
plt.xticks(rotation=45)
|
||||
plt.grid(True)
|
||||
plt.tight_layout()
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
|
||||
def plot_error_hist(errors):
|
||||
plt.hist(errors, bins=50)
|
||||
plt.title("Error Distribution")
|
||||
plt.xlabel("Absolute Error")
|
||||
plt.ylabel("Frequency")
|
||||
plt.grid(True)
|
||||
plt.show()
|
||||
|
||||
|
||||
def plot_scatter(y_test, predictions):
|
||||
plt.scatter(y_test[:, 0], predictions[:, 0], alpha=0.5)
|
||||
plt.plot([0, 1], [0, 1], "r--")
|
||||
plt.title("Predicted vs Actual (Normalized)")
|
||||
plt.xlabel("Actual")
|
||||
plt.ylabel("Predicted")
|
||||
plt.grid(True)
|
||||
plt.show()
|
||||
|
||||
|
||||
def plot_loss_curve(losses):
|
||||
plt.plot(losses)
|
||||
plt.title("Training Loss Curve")
|
||||
plt.xlabel("Epoch")
|
||||
plt.ylabel("MSE Loss")
|
||||
plt.grid(True)
|
||||
plt.tight_layout()
|
||||
plt.show()
|
||||
|
||||
|
||||
def plot_original_vs_scaled(original_data, scaled_data, feature_names=None):
|
||||
"""
|
||||
Plot original and scaled data, aligned by date.
|
||||
"""
|
||||
if feature_names is None:
|
||||
feature_names = original_data.columns[:2]
|
||||
|
||||
dates = original_data.index # Grab the datetime index
|
||||
|
||||
fig, axs = plt.subplots(2, 1, figsize=(12, 6), sharex=True)
|
||||
|
||||
for col in feature_names:
|
||||
axs[0].plot(dates, original_data[col], label=f"Original {col}")
|
||||
axs[0].set_title("Original Data")
|
||||
axs[0].legend()
|
||||
axs[0].grid(True)
|
||||
|
||||
for i, col in enumerate(feature_names):
|
||||
col_index = original_data.columns.get_loc(col)
|
||||
axs[1].plot(dates, scaled_data[:, col_index], label=f"Scaled {col}")
|
||||
axs[1].set_title("Scaled Data (MinMaxScaler)")
|
||||
axs[1].legend()
|
||||
axs[1].grid(True)
|
||||
|
||||
plt.xlabel("Date")
|
||||
plt.tight_layout()
|
||||
plt.show()
|
||||
|
||||
|
||||
def plot_value_distribution(data):
|
||||
# import matplotlib.pyplot as plt
|
||||
|
||||
plt.figure(figsize=(8, 4))
|
||||
plt.hist(data.iloc[:, 0], bins=40, color="skyblue", edgecolor="black")
|
||||
plt.title("Distribution of Values")
|
||||
plt.xlabel(data.columns[0])
|
||||
plt.ylabel("Frequency")
|
||||
plt.grid(True)
|
||||
plt.tight_layout()
|
||||
plt.show()
|
||||
|
||||
|
||||
def visualize_sequence_window(series, seq_length=5, start_idx=0):
|
||||
"""
|
||||
Visualize how a sequence window is used to create X (input) and y (target).
|
||||
|
||||
Parameters:
|
||||
series (pd.Series or np.ndarray): 1D time series data
|
||||
seq_length (int): Length of input window
|
||||
start_idx (int): Where to start slicing from for demo
|
||||
"""
|
||||
if isinstance(series, np.ndarray):
|
||||
data = series
|
||||
else:
|
||||
data = series.to_numpy()
|
||||
|
||||
x_seq = data[start_idx:start_idx + seq_length]
|
||||
y_val = data[start_idx + seq_length]
|
||||
|
||||
full_seq = np.concatenate([x_seq, [y_val]])
|
||||
|
||||
plt.figure(figsize=(8, 3))
|
||||
plt.plot(range(start_idx, start_idx + seq_length + 1), full_seq, marker='o')
|
||||
plt.axvline(x=start_idx + seq_length - 0.5, linestyle="--", color="gray")
|
||||
|
||||
for i in range(seq_length):
|
||||
plt.text(start_idx + i, x_seq[i], f"x[{i}]", ha="center", va="bottom", fontsize=9)
|
||||
plt.text(start_idx + seq_length, y_val, "y", ha="center", va="bottom", color="red", fontsize=9)
|
||||
|
||||
plt.title(f"Sequence Window (seq_length={seq_length}) → Predict Next")
|
||||
plt.xlabel("Time Step")
|
||||
plt.ylabel("Value")
|
||||
plt.grid(True)
|
||||
plt.tight_layout()
|
||||
plt.show()
|
||||
Reference in new issue
Block a user