wth

main.py

@@ -5,14 +5,21 @@ from sklearn.model_selection import train_test_split  # type: ignore[import]
 
 from data_loader import load_weather_data
 from model_train import (
-    cosine_lr_schedule,
+    # cosine_lr_schedule,
+    cyclical_lr_schedule,
     evaluate_model,
+    # exponential_decay_schedule,
+    # linear_decay_schedule,
+    # polynomial_decay_schedule,
     predict_linear_regression,
     run_autoregressive_prediction,
+    # step_decay_schedule,
     train_linear_regression_gd,
+    # warmup_schedule,
 )
 from preprocessing import create_sequences, inverse_scale, scale_data
 from visualize import (
+    # inspect_residuals,
     plot_error_hist,  # plot_loss_curve,
     plot_feature_importance,
     # plot_interactive_time_series,
@@ -27,6 +34,7 @@ from visualize import (
     visualize_sequence_window,
 )
 
+plt.ioff()
 # Step 1: Load data
 data = load_weather_data("hanoi_weather_data.csv")
 
@@ -50,14 +58,39 @@ X_train, X_test, y_train, y_test = train_test_split(
 )
 date_train, date_test = train_test_split(date_index, test_size=test_size, shuffle=False)
 
+# this is the lr schedulers
+# cos_lr = lambda epoch: cosine_lr_schedule(epoch, max_lr=0.01, min_lr=0.001)
+# step_lr = lambda epoch: step_decay_schedule(
+#     epoch, initial_lr=0.01, drop=0.5, epochs_drop=100
+# )
+# expo_lr = lambda epoch: exponential_decay_schedule(
+#     epoch, initial_lr=0.01, decay_rate=0.96, decay_steps=100
+# )
+# linear_lr = lambda epoch: linear_decay_schedule(
+#     epoch, initial_lr=0.01, min_lr=0.001, total_epochs=1000
+# )
+# cycle_lr = lambda epoch: cyclical_lr_schedule(
+#     epoch, base_lr=0.001, max_lr=0.01, step_size=200
+# )
+# poly_lr = lambda epoch: polynomial_decay_schedule(
+#     epoch, initial_lr=0.01, min_lr=0.001, total_epochs=1000, power=2
+# )
+# warmup_lr = lambda epoch: warmup_schedule(
+#     epoch, warmup_epochs=10, initial_lr=0.001, max_lr=0.01
+# )
+
 # Step 4: Train model with gradient descent
-
-
 theta, losses = train_linear_regression_gd(
     X_train,
     y_train,
-    lr_schedule=lambda epoch: cosine_lr_schedule(epoch, max_lr=0.01, min_lr=0.001),
-    # n_epochs=1000,
+    lr_schedule=lambda epoch: cyclical_lr_schedule(
+    epoch, base_lr=0.0005, max_lr=0.01, step_size=150
+),
+    n_epochs=1000,
+    batch_size=32,
+    patience=80,
+    l2_reg=0.001,  # Enable L2 regularization
+    l1_reg=0.0,  # Disable L1 regularization
 )
 
 predictions = predict_linear_regression(X_test, theta)
@@ -81,6 +114,7 @@ plot_residuals(date_test.to_numpy(), residuals)
 plot_error_hist(errors)
 plot_scatter(y_test.reshape(-1, 1), predictions.reshape(-1, 1))
 plot_residual_scatter(y_test, predictions)
+# == inspect_residuals(predictions, residuals)
 plot_loss_curve_with_annotations(losses)
 plot_feature_importance(theta, feature_names=[f"QV2M_t-{i}" for i in range(seq_length)])
 
@@ -89,3 +123,5 @@ plot_feature_importance(theta, feature_names=[f"QV2M_t-{i}" for i in range(seq_l
 autoregressive_predictions = run_autoregressive_prediction(
     theta, X_test, y_test, scaler, date_test, data.shape[1]
 )
+
+plt.ion()

model_train.py

@@ -1,7 +1,10 @@
 # from itertools import islice
-from matplotlib import pyplot as plt
+import time
+
 import numpy as np
-from sklearn.metrics import mean_squared_error, r2_score  # type: ignore[import]
+from matplotlib import pyplot as plt
+from sklearn.metrics import (mean_squared_error,  # type: ignore[import]
+                             r2_score)
 
 from preprocessing import inverse_scale
 
@@ -13,21 +16,9 @@ def train_linear_regression_gd(
     n_epochs=1000,
     batch_size=32,
     patience=20,
+    l2_reg=0.0,
+    l1_reg=0.0,
 ):
-    """
-    Train linear regression model using batch gradient descent.
-
-    Args:
-        X_train: Training features.
-        y_train: Training labels.
-        lr_schedule: Learning rate schedule function.
-        n_epochs: Number of epochs.
-        batch_size: Size of each batch.
-
-    Returns:
-        theta: Learned weights.
-        losses: List of MSE loss per epoch.
-    """
     m, n = X_train.shape
     X_b = np.hstack([np.ones((m, 1)), X_train])  # Add bias term
     theta = np.zeros(n + 1)
@@ -37,12 +28,16 @@ def train_linear_regression_gd(
     patience_counter = 0
 
     for epoch in range(n_epochs):
+        start_time = time.time()  # Start timing the epoch
+
         # Shuffle the data at the start of each epoch
         indices = np.random.permutation(m)
         X_b_shuffled = X_b[indices]
         y_shuffled = y_train[indices]
 
         epoch_loss = 0.0
+        gradient_norm = 0.0
+
         for i in range(0, m, batch_size):
             X_batch = X_b_shuffled[i : i + batch_size]
             y_batch = y_shuffled[i : i + batch_size]
@@ -50,16 +45,32 @@ def train_linear_regression_gd(
             lr = lr_schedule(epoch)
             predictions = X_batch @ theta
             errors = predictions - y_batch
+
+            # Compute gradients with regularization
             gradients = (2 / batch_size) * X_batch.T @ errors
-            gradients = np.clip(gradients, -1, 1)  # Gradient clipping
+            if l2_reg > 0:
+                gradients[1:] += 2 * l2_reg * theta[1:]
+            if l1_reg > 0:
+                gradients[1:] += l1_reg * np.sign(theta[1:])
+
+            gradient_norm += np.linalg.norm(gradients)  # Track gradient norm
             theta -= lr * gradients
 
             batch_loss = np.mean(errors**2)
+            if l2_reg > 0:
+                batch_loss += l2_reg * np.sum(theta[1:] ** 2)
+            if l1_reg > 0:
+                batch_loss += l1_reg * np.sum(np.abs(theta[1:]))
+
             epoch_loss += batch_loss
 
         epoch_loss /= m // batch_size
         losses.append(epoch_loss)
 
+        # Compute weight norm
+        weight_norm = np.linalg.norm(theta)
+
+        # Check for early stopping
         if epoch_loss < best_loss:
             best_loss = epoch_loss
             patience_counter = 0
@@ -70,8 +81,14 @@ def train_linear_regression_gd(
             print(f"Early stopping at epoch {epoch}")
             break
 
-        if epoch % 100 == 0:
-            print(f"Epoch {epoch}: Loss = {losses[-1]:.6f}, LR: {lr:.6f}")
+        # Log statistics
+        epoch_time = time.time() - start_time
+        if epoch % 100 == 0 or epoch == n_epochs - 1:
+            print(
+                f"Epoch {epoch}: Loss = {epoch_loss:.6f}, LR: {lr:.6f}, "
+                f"Gradient Norm = {gradient_norm:.4f}, Weight Norm = {weight_norm:.4f}, "
+                f"Epoch Time = {epoch_time:.2f}s"
+            )
 
     print(f"Final Loss: {losses[-1]:.6f}")
     return theta, losses
@@ -99,6 +116,109 @@ def cosine_lr_schedule(epoch, max_lr=0.01, min_lr=0.001, total_epochs=1000):
     )
 
 
+def step_decay_schedule(epoch, initial_lr=0.01, drop=0.5, epochs_drop=50):
+    """
+    Step decay learning rate scheduler.
+
+    Args:
+        epoch: Current epoch.
+        initial_lr: Initial learning rate.
+        drop: Factor by which to reduce the learning rate.
+        epochs_drop: Number of epochs after which to reduce the learning rate.
+
+    Returns:
+        Updated learning rate.
+    """
+    return initial_lr * (drop ** (epoch // epochs_drop))
+
+
+def exponential_decay_schedule(epoch, initial_lr=0.01, decay_rate=0.1, decay_steps=100):
+    """
+    Exponential decay learning rate scheduler.
+
+    Args:
+        epoch: Current epoch.
+        initial_lr: Initial learning rate.
+        decay_rate: Rate of decay.
+        decay_steps: Number of steps for decay.
+
+    Returns:
+        Updated learning rate.
+    """
+    return initial_lr * (decay_rate ** (epoch / decay_steps))
+
+
+def linear_decay_schedule(epoch, initial_lr=0.01, min_lr=0.001, total_epochs=1000):
+    """
+    Linear decay learning rate scheduler.
+
+    Args:
+        epoch: Current epoch.
+        initial_lr: Initial learning rate.
+        min_lr: Minimum learning rate.
+        total_epochs: Total number of epochs.
+
+    Returns:
+        Updated learning rate.
+    """
+    return max(min_lr, initial_lr - (initial_lr - min_lr) * (epoch / total_epochs))
+
+
+def cyclical_lr_schedule(epoch, base_lr=0.001, max_lr=0.01, step_size=200):
+    """
+    Cyclical learning rate scheduler.
+
+    Args:
+        epoch: Current epoch.
+        base_lr: Minimum learning rate.
+        max_lr: Maximum learning rate.
+        step_size: Number of epochs for half a cycle.
+
+    Returns:
+        Updated learning rate.
+    """
+    cycle = np.floor(1 + epoch / (2 * step_size))
+    x = np.abs(epoch / step_size - 2 * cycle + 1)
+    return base_lr + (max_lr - base_lr) * max(0, (1 - x))
+
+
+def polynomial_decay_schedule(
+    epoch, initial_lr=0.01, min_lr=0.001, total_epochs=1000, power=2
+):
+    """
+    Polynomial decay learning rate scheduler.
+
+    Args:
+        epoch: Current epoch.
+        initial_lr: Initial learning rate.
+        min_lr: Minimum learning rate.
+        total_epochs: Total number of epochs.
+        power: Polynomial power.
+
+    Returns:
+        Updated learning rate.
+    """
+    return min_lr + (initial_lr - min_lr) * ((1 - epoch / total_epochs) ** power)
+
+
+def warmup_schedule(epoch, warmup_epochs=10, initial_lr=0.01, max_lr=0.1):
+    """
+    Warm-up learning rate scheduler.
+
+    Args:
+        epoch: Current epoch.
+        warmup_epochs: Number of warm-up epochs.
+        initial_lr: Initial learning rate.
+        max_lr: Maximum learning rate after warm-up.
+
+    Returns:
+        Updated learning rate.
+    """
+    if epoch < warmup_epochs:
+        return initial_lr + (max_lr - initial_lr) * (epoch / warmup_epochs)
+    return max_lr
+
+
 def autoregressive_prediction(model, initial_sequence, steps, scaler, n_features):
     """
     Perform autoregressive predictions using the model.

visualize.py

@@ -269,3 +269,15 @@ def plot_feature_importance(theta, feature_names):
 
 #     fig = px.line(data, x=data.index, y=feature_names, title="Interactive Time Series")
 #     fig.show()
+
+
+def inspect_residuals(predictions, residuals):
+    plt.figure(figsize=(10, 6))
+    plt.scatter(predictions, residuals, alpha=0.5)
+    plt.axhline(0, color="red", linestyle="--")
+    plt.title("Residuals vs Predictions")
+    plt.xlabel("Predicted Values")
+    plt.ylabel("Residuals")
+    plt.grid(True)
+    plt.tight_layout()
+    plt.show()