minor changes

src/config.py

@@ -1,4 +1,5 @@
 from dataclasses import dataclass
+from pathlib import Path
 
 
 @dataclass
@@ -6,11 +7,9 @@ class Config:
     playlist_url: str = "https://www.youtube.com/playlist?list=PLD80i8An1OEEb1jP0sjEyiLG8ULRXFob_"
 
     # paths
-    root_data_dir: str = "../data"
-    yt_scraped_data_path: str = "../data/yt_data.csv"
-    chromadb_dir: str = "../data/chromadb"
-
-    # artifacts
+    root_data_dir: Path = Path("data")
+    # wandb
+    project_name: str = "gradient_dissent_qabot"
     yt_podcast_data_artifact: str = "gladiator/gradient_dissent_bot/yt_podcast_data:latest"
     summarized_data_artifact: str = "gladiator/gradient_dissent_bot/summary_data:latest"
     summarized_que_data_artifact: str = "gladiator/gradient_dissent_bot/summary_que_data:latest"

src/podcast_data.py

@@ -2,11 +2,11 @@ import time
 from dataclasses import asdict
 
 import pandas as pd
+import wandb
 from langchain.document_loaders import YoutubeLoader
 from pytube import Playlist, YouTube
 from tqdm import tqdm
 
-import wandb
 from config import config
 
 
@@ -35,7 +35,7 @@ def retry_access_yt_object(url, max_retries=5, interval_secs=5):
 
 
 if __name__ == "__main__":
-    run = wandb.init(project="gradient_dissent_bot", job_type="dataset", config=asdict(config))
+    run = wandb.init(project=config.project_name, job_type="dataset", config=asdict(config))
 
     playlist = Playlist(config.playlist_url)
     playlist_video_urls = playlist.video_urls
@@ -62,10 +62,12 @@ if __name__ == "__main__":
 
     print(f"Total podcast episodes scraped: {len(video_data)}")
 
+    # save the scraped data to a csv file
     df = pd.DataFrame(video_data)
-    df.to_csv(config.yt_scraped_data_path, index=False)
+    df.to_csv(config.root_data_dir / "yt_podcast_transcript.csv", index=False)
 
-    artifact = wandb.Artifact("yt_podcast_data", type="dataset")
+    # upload the scraped data to wandb
+    artifact = wandb.Artifact("yt_podcast_transcript", type="dataset")
     artifact.add_file(config.yt_scraped_data_path)
     run.log_artifact(artifact)