Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def wait_for_element_by_css_selector(self, selector, timeout=TIMEOUT):
return WebDriverWait(self.driver, timeout).until(
EC.presence_of_element_located((By.CSS_SELECTOR, selector)),
'Could not find element with selector "{}"'.format(selector)
)
def get_products():
try:
wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '#s-results-list-atf')))
html = browser.page_source
soup = BeautifulSoup(html, 'lxml')
doc = lxml.html.fromstring(html)
date = doc.xpath('//*[@class="s-result-item celwidget "]/div/div[2]/div[1]/span[2]/text()')
content = soup.find_all(attrs={"id": re.compile(r'result_\d+')})
for item, time in zip(content, date):
product = {
'title': item.find(class_='s-access-title').get_text(),
'image': item.find(class_='s-access-image cfMarker').get('src'),
'price': item.find(class_='a-size-base a-color-price s-price a-text-bold').get_text(),
'date': time
}
save_to_mongo(product)
print(product)
except Exception as e:
print(e)
usernameStr = '0xxyyzz00@gmail.com'
passwordStr = 'xyzT!Tan1c'
driver = webdriver.Chrome()
#Initializes a Chrome Browser or creates a new instance of Chrome
driver.get('https://stockx.com/login')
username = driver.find_element_by_id('login[username]')
username.send_keys(usernameStr)
# nextButton = driver.find_element_by_id('next')
# nextButton.click()
password = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.ID, 'login[password]')))
password.send_keys(passwordStr)
signInButton = driver.find_element_by_id('login-button')
signInButton.click()
#Long list of urls from which we want to scrafe info. 35 top selling sneakers from the most popular collection of Adidas, Nike, Air Jordan
long_url_list = ["https://stockx.com/air-jordan-1-retro-black-blue-2017", #1 {### Air Jordan: One ###}
"https://stockx.com/air-jordan-1-retro-mid-new-love-2017",
"https://stockx.com/jordan-1-retro-bred-2016",
"https://stockx.com/jordan-1-retro-yin-yang-black",
"https://stockx.com/air-jordan-1-retro-all-star-2017",
"https://stockx.com/air-jordan-1-retro-top-3",
"https://stockx.com/jordan-1-retro-yin-yang-white",
"https://stockx.com/air-jordan-1-retro-black-toe-2016",
"https://stockx.com/air-jordan-1-retro-high-gold-top-3",
"https://stockx.com/air-jordan-1-retro-high-og-metallic-red-2017",
def _loop_find_m3u8_url(self, video_url, waitsecond=15, max_attempts=5):
stale_attempt = 1
refresh_attempt = 1
while True:
self._driver.get(video_url)
try:
# wait for maximum second before timeout
WebDriverWait(self._driver, waitsecond).until(
EC.presence_of_element_located((By.ID, "content-player")))
return self._driver.find_element_by_id(
'content-player').find_element_by_tag_name(
'video').get_attribute('src')
except selenium.common.exceptions.TimeoutException:
if refresh_attempt >= max_attempts:
print(
'\r\nERROR: Connection timeouted after {} second for {} attempts... \
Possibly internet problem?'.format(
waitsecond, max_attempts))
raise
refresh_attempt += 1
except StaleElementReferenceException:
if stale_attempt >= max_attempts:
print(
'\r\nERROR: Elements are not stable to retrieve after {} attempts... \
Possibly internet problem?'.format(max_attempts))
def load_index(self,keywords='',location=''):
url = 'http://www.linkedin.com/jobs/search/?keywords={}&location={}&sortBy==DD'.format(keywords,location)
self.driver.get(url)
# Wait for page to load dynamically via javascript
try:
myElem = WebDriverWait(self.driver, self.timeout).until(AnyEC(
EC.presence_of_element_located(
(By.CSS_SELECTOR, '.jobs-search-results')),
EC.presence_of_element_located(
(By.CSS_SELECTOR, '.not-found-404'))
))
except TimeoutException as e:
raise Exception(
"""Took too long to load search results. Common problems/solution:
1. Invalid LI_AT value: ensure that yours is correct (they
update frequently)
2. Slow internet: increase the timeout parameter in the Scraper constructor""")
# Make sure we are in the classic view
self.change_view(view='classic')
# Scroll to load lazy-loaded content
self.scroll_to_bottom()
def shot_campaign_var(live_server, organizer, event, logged_in_client, campaign_web, campaign_twitter, orders, clicks,
var):
event.plugins += ',pretix_campaigns'
event.save()
logged_in_client.get(live_server.url + '/control/event/{}/{}/campaigns/{}/'.format(
organizer.slug, event.slug, campaign_twitter.code
))
WebDriverWait(logged_in_client, 10).until(
EC.presence_of_element_located((By.CSS_SELECTOR, "#cbd_chart svg"))
)
time.sleep(.5)
screenshot(logged_in_client, 'website/control/campaigns_detail_var.png')
def sign_off(self):
"""Submit questionnaire and finish.
This uses Selenium to click the submit button on the questionnaire
and return to the original window.
"""
try:
logger.info("Bot player signing off.")
feedback = WebDriverWait(self.driver, 20).until(
EC.presence_of_element_located((By.ID, "submit-questionnaire"))
)
self.complete_questionnaire()
feedback.click()
logger.info("Clicked submit questionnaire button.")
self.driver.switch_to_window(self.driver.window_handles[0])
self.driver.set_window_size(1024, 768)
logger.info("Switched back to initial window.")
return True
except TimeoutException:
logger.error("Error during experiment sign off.")
return False
def _find_deferred_element_by(self, search, by, timeout=5):
elem = WebDriverWait(self, timeout).until(
expected_conditions.presence_of_element_located((by, search)),
'Timeout occurred while waiting for element: %s' % search
)
return elem
def Elem(self,css:str) -> WebElement:
return self._wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,css)))
def get_position(self):
"""
获取验证码位置
:return: 验证码位置元组
"""
img = self.wait.until(EC.presence_of_element_located((By.ID, 'checkcode'))) # 获取图形验证码的节点
time.sleep(2)
location = img.location # 获取图形验证码在网页中的相对位置
size = img.size # 获取图形验证码的大小
top, bottom, left, right = location['y'], location['y'] + size['height'], location['x'], location['x'] + size[
'width'] # 分别获取左上角和右下角的坐标
return [top, bottom, left, right] # 返回坐标