oh noes

load_data.py

@@ -5,11 +5,6 @@ import io
 # from itertools import islice
 from typing import Union
 
-import pandas as pd
-
-from global_config import DATA_FILE_NAME
-
-
 def cut_headers(stream: Union[io.StringIO, io.FileIO, io.TextIOWrapper]):
     headers = []
     exist_startheader = False
@@ -30,64 +25,4 @@ def cut_headers(stream: Union[io.StringIO, io.FileIO, io.TextIOWrapper]):
         # If no headers are found, reset the stream and return
         stream.seek(0)
         return "", stream
-    return "\n".join(headers), stream
+    return "\n".join(headers), stream
-
-
-# if __name__ == "__main__":
-#     with open("hanoi_qv2m_latest.csv", "r", encoding="utf-8") as file:
-#         header, file = cut_headers(file)
-#         print("Extracted Header:")
-#         print(header)
-#         data = pd.read_csv(file)
-#         print("\nFirst 5 Lines of Data:")
-#         print(data.head())
-#         print(data.isnull().sum())
-
-
-def load_data(
-    weather_file_name, columns=["Year", "Month", "Day", "Humidity"].copy(), na=-999
-):
-    with open(weather_file_name, "r", encoding="utf-8") as file:
-        _header, file = cut_headers(file)
-        data = pd.read_csv(file, na_values=na)
-    # print(header)
-    data.columns = columns
-    return data
-
-
-def date_combine(data: pd.DataFrame):
-    data["Date"] = pd.to_datetime(data[["Year", "Month", "Day"]])
-    data = data.drop(columns=["Year", "Month", "Day"])
-    data = data.set_index("Date")
-    return data
-
-
-def validate_view(data: pd.DataFrame, verbose=False):
-    if verbose:
-        print(f"Shape with N/A values: {data.shape}")
-        isn = data.isnull().sum().iloc[0]
-        print(f"Count of N/A values: {isn}")
-    data = data.dropna()  # Drop the rows with missing values
-    if verbose:
-        isn = data.isnull().sum().iloc[0]
-        print(f"Count after drop: {isn}")
-        print(data.describe())
-    return data
-
-
-def load_this_data(filename=DATA_FILE_NAME, verbose=False, *a):
-    d = load_data(filename, *a)
-    # if verbose:
-    #     print(d.columns)
-    d = date_combine(d)
-    # if verbose:
-    #     print(d.columns)
-    d = validate_view(d, verbose)
-    if verbose:
-        print(d.head())
-    #     print(d.columns)
-    return d
-
-
-if __name__ == "__main__":
-    load_this_data(verbose=True)

sklearn_port.py

@@ -0,0 +1,69 @@
+import sys
+import numpy as np
+import pandas as pd
+from sklearn.preprocessing import MinMaxScaler
+from sklearn.model_selection import train_test_split
+from sklearn.linear_model import LinearRegression
+from sklearn.metrics import mean_squared_error
+import matplotlib.pyplot as plt
+from load_data import cut_headers
+
+# 1. Load data
+
+sys.exit(1)
+
+# 2. Normalize data
+scaler = MinMaxScaler()
+print(a.columns)
+scaled_data = scaler.fit_transform(a)
+
+# 3. Create sequences for supervised learning
+def create_sequences(data, seq_length):
+    xs, ys = [], []
+    for i in range(len(data) - seq_length):
+        x = data[i : i + seq_length].flatten()  # Flatten for Linear Regression
+        y = data[i + seq_length]
+        xs.append(x)
+        ys.append(y)
+    return np.array(xs), np.array(ys)
+
+seq_length = 20
+X, y = create_sequences(scaled_data, seq_length)
+
+# 4. Split into training and testing
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
+
+# 5. Train linear regression model
+model = LinearRegression()
+model.fit(X_train, y_train)
+
+# 6. Make predictions
+predictions = model.predict(X_test)
+
+# 7. Evaluate performance
+mse = mean_squared_error(y_test, predictions)
+print(f"Test MSE: {mse:.4f}")
+
+# 8. Inverse transform predictions
+predictions_rescaled = scaler.inverse_transform(predictions.reshape(-1, a.shape[1]))
+y_test_rescaled = scaler.inverse_transform(y_test.reshape(-1, a.shape[1]))
+
+# 9. Plot results
+fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5), gridspec_kw={"width_ratios": [1, 3]})
+
+# Loss bar
+ax1.bar(["Linear Regression"], [mse], color='orange')
+ax1.set_title("Validation MSE")
+ax1.set_ylabel("Loss")
+ax1.grid(True)
+
+# Prediction vs Actual
+ax2.plot(y_test_rescaled, label="Actual")
+ax2.plot(predictions_rescaled, label="Predicted")
+ax2.set_title("Actual vs. Predicted")
+ax2.set_xlabel("Sample")
+ax2.set_ylabel("Value")
+ax2.legend()
+
+plt.tight_layout()
+plt.show()