Upload folder using huggingface_hub

rag_fns/setup_load.py

@@ -7,6 +7,8 @@ import numpy as np
 from dotenv import load_dotenv
 from openai import OpenAI
 from pyprojroot import here
+import requests
+import io
 
 logger = logging.getLogger(__name__)
 
@@ -38,8 +40,13 @@ def import_data() -> tuple[list[str], np.ndarray, dict[str, Any]]:
         tuple[pd.DataFrame, dict, dict]: A tuple containing the talks dataframe, transcript dictionaries, and full embeddings.
     """
 
-    with open(here() / "data" / "interim" / "embeds_talks_dcr.pkl", "rb") as f:
-        data2load = pickle.load(f)
+    target_file_url = "https://raw.githubusercontent.com/AlanFeder/dcr-multi-frameworks/main/data/interim/embeds_talks_dcr.pkl"
+
+    response = requests.get(target_file_url)
+    response.raise_for_status()  # Ensure we got a successful response
+
+    # Load the .pkl file into a Python object
+    data2load = pickle.load(io.BytesIO(response.content))
 
     talk_ids = data2load["talk_ids"]
     embeds = data2load["embeds"]