{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Twitter Scraping for viral Tweets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As of now (October 2022), there is no way to retrieve the tweets from the Topic \"viral tweets\" from the Twitter API. Those are tweets that Twitter labels as viral based on number of likes, retweets and reaction surrounding the tweet in general. That's why we used online scraper tools to scrape tweets off this Twitter Topic page (namely https://twitter.com/i/topics/1284234742661963776)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH = \"../../../data\"\n",
    "SCRAPED_TWEETS_PATH = f\"{DATA_PATH}/scraped\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Octoparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "OCTOPARSE = f\"{SCRAPED_TWEETS_PATH}/octoparse/\"\n",
    "data_files = []\n",
    "\n",
    "# iterate over files in\n",
    "# that directory\n",
    "for filename in os.listdir(OCTOPARSE):\n",
    "    f = os.path.join(OCTOPARSE, filename)\n",
    "    # checking if it is a file\n",
    "    if os.path.isfile(f):\n",
    "        data_files.append(f)\n",
    "\n",
    "data_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import json\n",
    "\n",
    "links = []\n",
    "\n",
    "for data_file in data_files:\n",
    "    with open(data_file, \"r\", encoding='utf8') as read_file:\n",
    "        data = json.load(read_file)\n",
    "\n",
    "        for field in data:\n",
    "            for elem in field.values():\n",
    "                soup = BeautifulSoup(elem)\n",
    "                #print(soup.prettify())\n",
    "                for link in soup.find_all(\"a\"):\n",
    "                    links.append(link.get(\"href\"))  \n",
    "\n",
    "octoparse_tweet_ids = set([link.split('/')[3] for link in links if 'status' in link])  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(octoparse_tweet_ids)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Other scraper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "SAMPLE_SCRAPER = f\"{SCRAPED_TWEETS_PATH}/sample-scraper/\"\n",
    "data_files = []\n",
    "\n",
    "# iterate over files in\n",
    "# that directory\n",
    "for filename in os.listdir(SAMPLE_SCRAPER):\n",
    "    f = os.path.join(SAMPLE_SCRAPER, filename)\n",
    "    # checking if it is a file\n",
    "    if os.path.isfile(f):\n",
    "        data_files.append(f)\n",
    "\n",
    "data_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "links = []\n",
    "\n",
    "for data_file in data_files:\n",
    "    with open(data_file, \"r\", encoding='utf8') as read_file:\n",
    "        data = json.load(read_file)\n",
    "\n",
    "        for field in data.values():\n",
    "            if type(field) == list:\n",
    "                for elem in field:\n",
    "                    links.append(elem['url'])\n",
    "\n",
    "sample_scraper_tweet_ids = set([link.split('/')[5] for link in links if 'status' in link])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(sample_scraper_tweet_ids)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save all to csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "viral_tweet_ids = list(octoparse_tweet_ids) + list(sample_scraper_tweet_ids)\n",
    "len(viral_tweet_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the ids into a file\n",
    "import pandas as pd\n",
    "\n",
    "viral_tweet_ids_df = pd.DataFrame(viral_tweet_ids, columns=['tweet_id'])\n",
    "# Append csv\n",
    "viral_tweet_ids_df.to_csv(f\"{SCRAPED_TWEETS_PATH}/scraped_tweets_ids.csv\", index=False, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.11 ('ada')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.15"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "71d2f77bccee14ca7852d7b7a1fa8ea4708b81087104d93973081337557f0ee6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}