not much

dataset_update.py

@@ -17,8 +17,21 @@ end_date = datetime.now().strftime("%Y-%m-%d")  # Today's date
 # WS10M           MERRA-2 Wind Speed at 10 Meters (m/s)
 # WD10M           MERRA-2 Wind Direction at 10 Meters (Degrees)
 # more at https://power.larc.nasa.gov/data-access-viewer/
-weather_params = ["QV2M"]
+weather_params = [
-
+    "T2M",
+    "T2M_MAX",
+    "T2M_MIN",
+    "T2MDEW",
+    "PRECTOTCORR",
+    "QV2M",
+    "RH2M",
+    "PS",
+    "CLOUD_AMT",
+    "WS10M",
+    "WD10M",
+    "ALLSKY_KT",
+    "ALLSKY_SFC_SW_DWN",
+]
 
 params = {
     "parameters": ",".join(weather_params),

dead/pytorch_port.py

@@ -0,0 +1,163 @@
+from sklearn.model_selection import train_test_split
+import torch
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+from sklearn.preprocessing import MinMaxScaler
+
+import numpy as np
+from load_data import load_this_data
+
+a = load_this_data()
+
+# time series forecasting
+
+
+scaler = MinMaxScaler()
+print(a.columns)
+scaled_data = scaler.fit_transform(a)
+
+
+def create_sequences(data, seq_length):
+    "see this? return[0] is the big x and return[1] is the big y. and h(x) = y. and h(x) is the model."
+    xs = []
+    ys = []
+    for i in range(len(data) - seq_length):
+        x = data[i : (i + seq_length)]
+        y = data[i + seq_length]
+        xs.append(x)
+        ys.append(y)
+    return np.array(xs), np.array(ys)
+
+
+seq_length = 20
+
+X, y = create_sequences(scaled_data, seq_length)
+
+X_train, X_test, y_train, y_test = train_test_split(
+    X, y, test_size=0.2, random_state=42
+)
+from torch.utils.data import TensorDataset, DataLoader
+
+X_train_tensor = torch.tensor(X_train, dtype=torch.float32)
+y_train_tensor = torch.tensor(y_train, dtype=torch.float32)
+
+train_dataset = TensorDataset(X_train_tensor, y_train_tensor)
+train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
+
+
+import torch.nn as nn
+
+
+class LSTMModel(nn.Module):
+    def __init__(self, input_size, hidden_size, num_layers, output_size):
+        super(LSTMModel, self).__init__()
+        self.hidden_size = hidden_size
+        self.num_layers = num_layers
+        self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)
+        self.fc = nn.Linear(hidden_size, output_size)
+
+    def forward(self, x):
+        h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device)
+        c0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device)
+
+        out, _ = self.lstm(x, (h0, c0))
+        out = self.fc(out[:, -1, :])  # Take the output from the last time step
+        return out
+
+
+model = LSTMModel(
+    input_size=X_train.shape[2],
+    hidden_size=64,
+    num_layers=2,
+    output_size=y_train.shape[1],
+)
+model.to(device)
+criterion = nn.MSELoss()
+optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
+
+train_losses = []
+val_losses = []
+
+
+for epoch in range(100):
+    model.train()
+    running_train_loss = 0.0
+
+    # inputs, targets
+    for batch_X, batch_y in train_loader:
+        batch_X = batch_X.to(device)
+        batch_y = batch_y.to(device)
+
+        outputs = model(batch_X)
+        loss = criterion(outputs, batch_y)
+
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()
+
+        running_train_loss += loss.item() * batch_X.size(0)
+    if (epoch + 1) % 10 == 0:
+        print(f"Epoch [{epoch+1}/100], Loss: {loss.item():.4f}")
+
+    epoch_train_loss = running_train_loss / len(train_loader.dataset)
+    train_losses.append(epoch_train_loss)
+
+model.eval()
+
+X_test_tensor = torch.tensor(X_test, dtype=torch.float32).to(device)
+y_test_tensor = torch.tensor(y_test, dtype=torch.float32).to(device)
+val_dataset = TensorDataset(X_test_tensor, y_test_tensor)
+val_loader = DataLoader(val_dataset, batch_size=64, shuffle=False)
+
+# with torch.no_grad():
+#     running_val_loss = 0.0
+#     predictions = model(X_test_tensor)
+#     test_loss = criterion(predictions, y_test_tensor)
+#     print(f"Test Loss: {test_loss.item():.4f}")
+
+running_val_loss = 0.0
+predictions = []
+with torch.no_grad():
+    for inputs, targets in val_loader:
+        inputs, targets = inputs.to(device), targets.to(device)
+        outputs = model(inputs)
+        loss = criterion(outputs, targets)
+        running_val_loss += loss.item() * inputs.size(0)
+        predictions.append(outputs.cpu().numpy())  # Collect all batches
+
+epoch_val_loss = running_val_loss / len(val_loader.dataset)  # len(val_dataset)
+val_losses.append(epoch_val_loss)
+
+predictions = np.concatenate(predictions, axis=0)
+import matplotlib.pyplot as plt
+
+y_test_cpu = y_test_tensor.cpu().numpy()
+
+predictions_rescaled = scaler.inverse_transform(predictions)
+y_test_rescaled = scaler.inverse_transform(y_test_cpu)
+
+
+fig, (ax1, ax2) = plt.subplots(
+    1, 2, figsize=(14, 5), gridspec_kw={"width_ratios": [1, 3]}
+)
+
+# Loss curve on the left
+ax1.plot(train_losses, label="Train Loss")
+ax1.plot(val_losses, label="Val Loss")
+ax1.set_title("Loss Curve")
+ax1.set_xlabel("Epoch")
+ax1.set_ylabel("Loss")
+ax1.legend()
+ax1.grid(True)
+
+# Prediction vs Actual on the right
+ax2.plot(y_test_rescaled, label="Actual")
+ax2.plot(predictions_rescaled, label="Predicted")
+ax2.set_title("Actual vs. Predicted")
+ax2.set_xlabel("Sample")
+ax2.set_ylabel("Value")
+ax2.legend()
+
+plt.tight_layout()
+plt.show()

sklearn_port.py

@@ -1,37 +1,51 @@
 import sys
 import numpy as np
 import pandas as pd
+from sklearn.metrics import r2_score
 from sklearn.preprocessing import MinMaxScaler
 from sklearn.model_selection import train_test_split
 from sklearn.linear_model import LinearRegression
 from sklearn.metrics import mean_squared_error
 import matplotlib.pyplot as plt
+from global_config import DATA_FILE_NAME
 from load_data import cut_headers
 
 # 1. Load data
 
-sys.exit(1)
+with open(DATA_FILE_NAME, "r", encoding="utf-8") as file:
+    data_header, csv_file = cut_headers(file)
+    data = pd.read_csv(csv_file, na_values=-999)
+
+if all(z in data.columns for z in ("YEAR", "MO", "DY")):
+    data["date"] = pd.to_datetime(
+        {k: data[v] for v, k in (zip(["YEAR", "MO", "DY"], ["year", "month", "day"]))}
+    )
+    data.drop(inplace=True, columns=["YEAR", "MO", "DY"])
+    data.set_index(["date"], inplace=True)
+data.dropna(inplace=True)
 
 # 2. Normalize data
 scaler = MinMaxScaler()
-print(a.columns)
+print(data.columns)
-scaled_data = scaler.fit_transform(a)
+scaled_data = scaler.fit_transform(data)
+
 
 # 3. Create sequences for supervised learning
 def create_sequences(data, seq_length):
     xs, ys = [], []
     for i in range(len(data) - seq_length):
-        x = data[i : i + seq_length].flatten()  # Flatten for Linear Regression
+        x = data[i : i + seq_length]
         y = data[i + seq_length]
         xs.append(x)
         ys.append(y)
     return np.array(xs), np.array(ys)
 
-seq_length = 20
+
+seq_length = 50
 X, y = create_sequences(scaled_data, seq_length)
 
 # 4. Split into training and testing
-X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.02)
 
 # 5. Train linear regression model
 model = LinearRegression()
@@ -42,17 +56,21 @@ predictions = model.predict(X_test)
 
 # 7. Evaluate performance
 mse = mean_squared_error(y_test, predictions)
-print(f"Test MSE: {mse:.4f}")
+r2 = r2_score(y_test, predictions)
 
+print(f"Test MSE: {mse:.4f}")
+print(f"Test r2: {r2:.4f}")
+
+print(f"Score: {model.score(X_test, y_test)}")
 # 8. Inverse transform predictions
-predictions_rescaled = scaler.inverse_transform(predictions.reshape(-1, a.shape[1]))
+predictions_rescaled = scaler.inverse_transform(predictions.reshape(-1, data.shape[1]))
-y_test_rescaled = scaler.inverse_transform(y_test.reshape(-1, a.shape[1]))
+y_test_rescaled = scaler.inverse_transform(y_test.reshape(-1, data.shape[1]))
 
 # 9. Plot results
-fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5), gridspec_kw={"width_ratios": [1, 3]})
+fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5), width_ratios=[1, 3])
 
 # Loss bar
-ax1.bar(["Linear Regression"], [mse], color='orange')
+ax1.bar(["Linear Regression"], [mse], color="orange")
 ax1.set_title("Validation MSE")
 ax1.set_ylabel("Loss")
 ax1.grid(True)