poor man's branch

README.md

@@ -14,6 +14,9 @@ how to git? do not `push --force`. prefer pull requests. its like github
 
 we will plan to do this in multiple different ways using scikit-learn and pytorch. tensorflow is ancient.
 
+> [!IMPORTANT]
+> phase out tensorflow. dont use it.
+
 maybe also fit more weather related stats? this dataset is taken from nasa so sources are there
 
 ## collab

hanoi_qv2m_latest.csv

@@ -1,12 +1,22 @@
+
 -BEGIN HEADER-
+
 NASA/POWER Source Native Resolution Daily Data 
+
 Dates (month/day/year): 01/01/2010 through 05/06/2025 in LST
+
 Location: latitude  21.0305   longitude 105.8458 
+
 elevation from MERRA-2: Average for 0.5 x 0.625 degree lat/lon region = 87.73 meters
+
 The value for missing source data that cannot be computed or is outside of the sources availability range: -999 
+
 parameter(s): 
+
 QV2M     MERRA-2 Specific Humidity at 2 Meters (g/kg) 
+
 -END HEADER-
+
 YEAR,MO,DY,QV2M
 2010,1,1,10.88
 2010,1,2,11.26

load_data.py

@@ -0,0 +1,79 @@
+"extracting the load data step from the notebook"
+
+import io
+
+# from itertools import islice
+from typing import Union
+
+import pandas as pd
+
+
+def cut_headers(stream: Union[io.StringIO, io.FileIO, io.TextIOWrapper]):
+    headers = []
+    exist_startheader = False
+
+    while line := stream.readline():
+        if "-BEGIN HEADER-" in line and not exist_startheader:
+            exist_startheader = True
+            headers.append(str(line.strip()))
+        elif "-END HEADER-" in line and exist_startheader:
+            headers.append(str(line.strip()))
+            break
+        elif exist_startheader:
+            # print(f"{line = }")
+            headers.append(str(line.strip()))
+        elif not exist_startheader:
+            continue
+    if not exist_startheader:
+        # If no headers are found, reset the stream and return
+        stream.seek(0)
+        return "", stream
+    return "\n".join(headers), stream
+
+
+# if __name__ == "__main__":
+#     with open("hanoi_qv2m_latest.csv", "r", encoding="utf-8") as file:
+#         header, file = cut_headers(file)
+#         print("Extracted Header:")
+#         print(header)
+#         data = pd.read_csv(file)
+#         print("\nFirst 5 Lines of Data:")
+#         print(data.head())
+#         print(data.isnull().sum())
+
+
+def load_data(
+    weather_file_name, columns=["Year", "Month", "Day", "Humidity"].copy(), na=-999
+):
+    with open(weather_file_name, "r", encoding="utf-8") as file:
+        _header, file = cut_headers(file)
+        data = pd.read_csv(file, na_values=na)
+    # print(header)
+    data.columns = columns
+    return data
+
+
+def date_combine(data: pd.DataFrame):
+    data["Date"] = pd.to_datetime(data[["Year", "Month", "Day"]])
+    data = data.drop(columns=["Year", "Month", "Day"])
+    data = data.set_index("Date")
+    return data
+
+
+def validate_view(data: pd.DataFrame):
+    print(f"Shape with N/A values: {data.shape}")
+    isn = data.isnull().sum().iloc[0]
+    print(f"Count of N/A values: {isn}")
+    data = data.dropna()  # Drop the rows with missing values
+    isn = data.isnull().sum().iloc[0]
+    print(f"Count after drop: {isn}")
+    print(data.describe())
+    return data
+
+
+# if __name__ == "__main__":
+def load_this_data(filename):
+    d = load_data(filename)
+    d = date_combine(d)
+    d = validate_view(d)
+    return d

main.ipynb

@@ -1139,7 +1139,7 @@
       "provenance": []
     },
     "kernelspec": {
-      "display_name": ".venv",
+      "display_name": "Python 3",
       "language": "python",
       "name": "python3"
     },
@@ -1153,7 +1153,7 @@
       "name": "python",
       "nbconvert_exporter": "python",
       "pygments_lexer": "ipython3",
-      "version": "3.10.9"
+      "version": "3.9.18"
     }
   },
   "nbformat": 4,