real

dataset_update.py

@@ -5,7 +5,7 @@ from global_config import DATA_FILE_NAME
 
 # Define parameters
 latitude, longitude = 21.04815477090267, 105.80123159165096  # lmao
-start_date = "1981-01-01"
+start_date = "2000-01-01"
 end_date = datetime.now().strftime("%Y-%m-%d")  # Today's date
 
 # Define desired parameters
@@ -18,19 +18,9 @@ end_date = datetime.now().strftime("%Y-%m-%d")  # Today's date
 # WD10M           MERRA-2 Wind Direction at 10 Meters (Degrees)
 # more at https://power.larc.nasa.gov/data-access-viewer/
 weather_params = [
-    "T2M",
+
-    "T2M_MAX",
-    "T2M_MIN",
-    "T2MDEW",
-    "PRECTOTCORR",
     "QV2M",
-    "RH2M",
+
-    "PS",
-    "CLOUD_AMT",
-    "WS10M",
-    "WD10M",
-    "ALLSKY_KT",
-    "ALLSKY_SFC_SW_DWN",
 ]
 
 params = {

sk2_port.py

@@ -0,0 +1,185 @@
+import sys
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+import io
+from typing import Union
+from sklearn.metrics import mean_squared_error, r2_score
+from sklearn.preprocessing import MinMaxScaler
+from sklearn.model_selection import train_test_split
+from sklearn.linear_model import LinearRegression
+from statsmodels.tsa.ar_model import AutoReg
+
+# ==== LOAD & CLEAN DATA ====
+DATA_FILE_NAME = "hanoi_weather_data.csv"
+
+def cut_headers(stream: Union[io.StringIO, io.FileIO, io.TextIOWrapper]):
+    headers = []
+    exist_startheader = False
+    while line := stream.readline():
+        if "-BEGIN HEADER-" in line and not exist_startheader:
+            exist_startheader = True
+            headers.append(str(line.strip()))
+        elif "-END HEADER-" in line and exist_startheader:
+            headers.append(str(line.strip()))
+            break
+        elif exist_startheader:
+            headers.append(str(line.strip()))
+        elif not exist_startheader:
+            continue
+    if not exist_startheader:
+        stream.seek(0)
+        return "", stream
+    return "\n".join(headers), stream
+
+with open(DATA_FILE_NAME, "r", encoding="utf-8") as file:
+    data_header, csv_file = cut_headers(file)
+    data = pd.read_csv(csv_file, na_values=-999)
+
+if all(z in data.columns for z in ("YEAR", "MO", "DY")):
+    data["date"] = pd.to_datetime(
+        {k: data[v] for v, k in zip(["YEAR", "MO", "DY"], ["year", "month", "day"])}
+    )
+    data.drop(columns=["YEAR", "MO", "DY"], inplace=True)
+    data.set_index("date", inplace=True)
+
+data.dropna(inplace=True)
+
+# ==== VISUALIZE CORRELATION ====
+plt.figure(figsize=(12, 8))
+sns.heatmap(data.corr(), annot=True, cmap="coolwarm")
+plt.title("Feature Correlation Heatmap")
+plt.show()
+
+# ==== SCALE DATA ====
+scaler = MinMaxScaler()
+scaled_data = scaler.fit_transform(data)
+
+# ==== CREATE SEQUENCES ====
+def create_sequences(data, seq_length):
+    xs, ys = [], []
+    for i in range(len(data) - seq_length):
+        x = data[i:i + seq_length].flatten()
+        y = data[i + seq_length]
+        xs.append(x)
+        ys.append(y)
+    return np.array(xs), np.array(ys)
+
+seq_length = 50
+X, y = create_sequences(scaled_data, seq_length)
+
+# ==== SPLIT ====
+date_index = data.index[seq_length:]
+X_train, X_test, y_train, y_test, date_train, date_test = train_test_split(
+    X, y, date_index, test_size=0.02, shuffle=False
+)
+
+# ==== TRAIN LINEAR REGRESSION ====
+model = LinearRegression()
+model.fit(X_train, y_train)
+predictions = model.predict(X_test)
+
+# ==== EVALUATION ====
+mse = mean_squared_error(y_test, predictions)
+r2 = r2_score(y_test, predictions)
+
+print(f"Test MSE: {mse:.4f}")
+print(f"Test R2: {r2:.4f}")
+print(f"Model Score: {model.score(X_test, y_test):.4f}")
+
+# ==== INVERSE TRANSFORM ====
+predictions_rescaled = scaler.inverse_transform(
+    predictions.reshape(-1, data.shape[1])
+)
+y_test_rescaled = scaler.inverse_transform(
+    y_test.reshape(-1, data.shape[1])
+)
+
+# ==== PLOTS ====
+
+# Actual vs Predicted with Real Dates
+plt.figure(figsize=(12, 6))
+plt.plot(date_test, y_test_rescaled[:, 0], label="Actual")
+plt.plot(date_test, predictions_rescaled[:, 0], label="Predicted")
+plt.title("Actual vs Predicted (First Feature)")
+plt.xlabel("Date")
+plt.ylabel("Original Scale")
+plt.xticks(rotation=45)
+plt.legend()
+plt.grid(True)
+plt.tight_layout()
+plt.show()
+
+# Residual Plot
+residuals = y_test[:, 0] - predictions[:, 0]
+plt.figure(figsize=(10, 4))
+plt.plot(date_test, residuals, label="Residuals")
+plt.hlines(0, 0, len(residuals), colors="red")
+plt.title("Residuals Over Time (First Feature)")
+plt.xlabel("Date")
+plt.ylabel("Error")
+plt.xticks(rotation=45)
+plt.legend()
+plt.tight_layout()
+plt.grid(True)
+plt.show()
+
+# Error Distribution
+errors = np.abs(y_test[:, 0] - predictions[:, 0])
+plt.hist(errors, bins=50)
+plt.title("Error Distribution (First Feature)")
+plt.xlabel("Absolute Error")
+plt.ylabel("Frequency")
+plt.grid(True)
+plt.show()
+
+# Scatter Plot
+plt.scatter(y_test[:, 0], predictions[:, 0], alpha=0.5)
+plt.plot([0, 1], [0, 1], "r--")
+plt.title("Predicted vs Actual (Normalized)")
+plt.xlabel("Actual")
+plt.ylabel("Predicted")
+plt.grid(True)
+plt.show()
+
+# ==== AUTOREGRESSION FORECAST ====
+target_column = data.columns[0]
+series = data[target_column]
+
+train_size = int(len(series) * 0.98)
+train_series = series[:train_size]
+test_series = series[train_size:]
+
+ar_model = AutoReg(train_series, lags=30).fit()
+ar_preds = ar_model.predict(start=len(train_series), end=len(series)-1)
+
+# Plot AR Forecast
+plt.figure(figsize=(12, 5))
+plt.plot(test_series.index, test_series, label="Actual")
+plt.plot(test_series.index, ar_preds, label="AR Forecast", linestyle="--")
+plt.title(f"AutoRegression Forecast - {target_column}")
+plt.xlabel("Date")
+plt.ylabel(target_column)
+plt.xticks(rotation=45)
+plt.legend()
+plt.grid(True)
+plt.tight_layout()
+plt.show()
+
+# ==== FUTURE FORECAST ====
+future_steps = 30
+future_index = pd.date_range(start=series.index[-1], periods=future_steps+1, freq="D")[1:]
+future_forecast = ar_model.predict(start=len(series), end=len(series) + future_steps - 1)
+
+plt.figure(figsize=(12, 5))
+plt.plot(series[-100:], label="Recent Data")
+plt.plot(future_index, future_forecast, label="Future Forecast", color="red")
+plt.title(f"{target_column} - 30 Day Forecast (AutoRegression)")
+plt.xlabel("Date")
+plt.ylabel(target_column)
+plt.xticks(rotation=45)
+plt.legend()
+plt.tight_layout()
+plt.grid(True)
+plt.show()

sklearn_port.py

@@ -7,8 +7,37 @@ from sklearn.model_selection import train_test_split
 from sklearn.linear_model import LinearRegression
 from sklearn.metrics import mean_squared_error
 import matplotlib.pyplot as plt
-from global_config import DATA_FILE_NAME
+# from global_config import DATA_FILE_NAME
-from load_data import cut_headers
+DATA_FILE_NAME = "hanoi_weather_data.csv"
+
+# from load_data import cut_headers
+
+import io
+
+# from itertools import islice
+from typing import Union
+
+def cut_headers(stream: Union[io.StringIO, io.FileIO, io.TextIOWrapper]):
+    headers = []
+    exist_startheader = False
+
+    while line := stream.readline():
+        if "-BEGIN HEADER-" in line and not exist_startheader:
+            exist_startheader = True
+            headers.append(str(line.strip()))
+        elif "-END HEADER-" in line and exist_startheader:
+            headers.append(str(line.strip()))
+            break
+        elif exist_startheader:
+            # print(f"{line = }")
+            headers.append(str(line.strip()))
+        elif not exist_startheader:
+            continue
+    if not exist_startheader:
+        # If no headers are found, reset the stream and return
+        stream.seek(0)
+        return "", stream
+    return "\n".join(headers), stream
 
 # 1. Load data
 
@@ -34,7 +63,7 @@ scaled_data = scaler.fit_transform(data)
 def create_sequences(data, seq_length):
     xs, ys = [], []
     for i in range(len(data) - seq_length):
-        x = data[i : i + seq_length]
+        x = data[i : i + seq_length].flatten()
         y = data[i + seq_length]
         xs.append(x)
         ys.append(y)
@@ -67,21 +96,15 @@ predictions_rescaled = scaler.inverse_transform(predictions.reshape(-1, data.sha
 y_test_rescaled = scaler.inverse_transform(y_test.reshape(-1, data.shape[1]))
 
 # 9. Plot results
-fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5), width_ratios=[1, 3])
 
 # Loss bar
-ax1.bar(["Linear Regression"], [mse], color="orange")
+
-ax1.set_title("Validation MSE")
-ax1.set_ylabel("Loss")
-ax1.grid(True)
 
 # Prediction vs Actual
-ax2.plot(y_test_rescaled, label="Actual")
+plt.plot(y_test_rescaled, label="Actual")
-ax2.plot(predictions_rescaled, label="Predicted")
+plt.plot(predictions_rescaled, label="Predicted")
-ax2.set_title("Actual vs. Predicted")
+plt.title("Actual vs. Predicted")
-ax2.set_xlabel("Sample")
+plt.xlabel("Sample")
-ax2.set_ylabel("Value")
+plt.ylabel("Value")
-ax2.legend()
+plt.legend()
-
-plt.tight_layout()
 plt.show()