Spaces:
Sleeping
Sleeping
File size: 7,206 Bytes
c26bbaf |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 |
import os
import re
import time
from typing import List
import requests
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.expected_conditions import \
presence_of_element_located
from selenium.webdriver.support.wait import WebDriverWait
from Powers import *
class SCRAP_DATA:
"""Class to get and handel scrapped data"""
def __init__(self, urls: List[str] or str) -> None:
self.urls = urls
self.path = "./scrapped/"
if not os.path.isdir(self.path):
os.makedirs("./scrapped/")
def get_images(self) -> list:
images = []
if isinstance(self.urls, str):
requested = requests.get(self.urls)
try:
name = self.path + f"img_{time.time()}.jpg"
with open(name, "wb") as f:
f.write(requested.content)
images.append(name)
except Exception as e:
LOGGER.error(e)
requested.close()
else:
for i in self.urls:
if i:
requested = requests.get(i)
else:
continue
try:
name = self.path + f"img_{time.time()}.jpg"
with open(name, "wb") as f:
f.write(requested.content)
images.append(name)
except Exception as e:
LOGGER.error(e)
requested.close()
continue
return images
def get_videos(self) -> list:
videos = []
if isinstance(self.urls, str):
if i:
requested = requests.get(i)
else:
return []
try:
name = self.path + f"vid_{time.time()}.mp4"
with open(name, "wb") as f:
f.write(requested.content)
videos.append(name)
except Exception as e:
LOGGER.error(e)
requested.close()
else:
for i in self.urls:
if i:
requested = requests.get(i)
else:
continue
try:
name = self.path + f"vid_{time.time()}.mp4"
with open(name, "wb") as f:
f.write(requested.content)
videos.append(name)
except Exception as e:
LOGGER.error(e)
requested.close()
continue
return videos
class DRIVER:
"""Class to make selenium driver"""
def __init__(self) -> None:
self.BIN = CHROME_BIN
self.CHROME_DRIVER = CHROME_DRIVER
def initialize_driver(self):
if not self.BIN:
LOGGER.error(
"ChromeBinaryErr: No binary path found! Install Chromium or Google Chrome.")
return (
None,
"ChromeBinaryErr: No binary path found! Install Chromium or Google Chrome.",
)
try:
options = Options()
options.binary_location = self.BIN
options.add_argument("--disable-dev-shm-usage")
options.add_argument("--ignore-certificate-errors")
options.add_argument("--disable-gpu")
options.add_argument("--headless=new")
options.add_argument("--test-type")
options.add_argument("--no-sandbox")
service = Service(self.CHROME_DRIVER)
driver = webdriver.Chrome(options, service)
return driver, None
except Exception as e:
LOGGER.error(f"ChromeDriverErr: {e}")
return None, f"ChromeDriverErr: {e}"
def driver_close(self, driver: webdriver.Chrome):
driver.close()
driver.quit()
class INSTAGRAM(DRIVER):
"""Class to scrap data from instagram"""
def __init__(self, url: str) -> None:
self.url = url
self.article = "article._aa6a"
self.ul_class = "_acay"
self.image_class = "x5yr21d"
self.video_class = "x1lliihq"
self.next_button = "button._afxw"
self.return_dict = {"image": [], "video": []}
super().__init__()
def is_correct_link(self):
return bool((re.compile(r"^https?://(?:www\.)?instagram\.com/")).match(self.url))
def get_all(self):
driver, error = self.initialize_driver()
if not driver:
return error
driver.get(self.url)
wait = WebDriverWait(driver, 30)
if "reel" in self.url:
element = wait.until(
presence_of_element_located((By.TAG_NAME, "video")))
reels = element.get_attribute("src")
self.driver_close(driver)
self.return_dict.get("video").append(reels)
return self.return_dict
elif bool((re.compile(r"^https?://(?:www\.)?instagram\.com/p/")).match(self.url)):
image_links = []
video_links = []
try:
element = wait.until(presence_of_element_located(
(By.CLASS_NAME, self.ul_class)))
while True:
sub_element = element.find_elements(
By.CLASS_NAME, self.image_class)
for i in sub_element:
url = i.get_attribute("src")
image_links.append(url)
sub_element = element.find_elements(
By.CLASS_NAME, self.video_class)
for i in sub_element:
url = i.get_attribute("src")
video_links.append(url)
try:
driver.find_element(
By.CSS_SELECTOR, self.next_button).click()
except: # Failed to either find the element or click on next i.e. no more media left in post
break
except:
element = wait.until(presence_of_element_located(
(By.CSS_SELECTOR, self.article)))
try:
sub_element = element.find_element(By.TAG_NAME, "img")
image_links.append(sub_element.get_attribute("src"))
except:
sub_element = element.find_element(By.TAG_NAME, "video")
video_links.append(sub_element.get_attribute("src"))
self.driver_close(driver)
# To remove duplicates here I am converting into set
if image_links:
image_links = list(set(image_links))
if video_links:
video_links = list(set(video_links))
for i in video_links:
image_links.remove(i)
self.return_dict.get("image").extend(image_links)
self.return_dict.get("video").extend(video_links)
return self.return_dict
else:
return {}
|