Python爬虫之Selenium库的使用方法

 # 基本使用 from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.wait import WebDriverWait browser = webdriver.Chrome() try: browser.get('https://www.baidu.com') input = browser.find_element_by_id('kw') input.send_keys('Python') input.send_keys(Keys.ENTER) wait = WebDriverWait(browser, 10) wait.until(EC.presence_of_element_located((By.ID, 'content_left'))) print(browser.current_url) print(browser.get_cookies()) print(browser.page_source) finally: browser.close() # 声明浏览器对象 from selenium import webdriver browser = webdriver.Chrome() browser = webdriver.Firefox() browser = webdriver.Edge() browser = webdriver.PhantomJS() browser = webdriver.Safari() # 访问页面 from selenium import webdriver browser = webdriver.Chrome() browser.get('https://www.taobao.com') print(browser.page_source) browser.close() # 查找元素 # 单个元素 from selenium import webdriver browser = webdriver.Chrome() browser.get('https://www.taobao.com') # 下面三个效果是一样的 input_first = browser.find_element_by_id('q') input_second = browser.find_element_by_css_selector('#q') input_third = browser.find_element_by_xpath('//*[@id="q"]') print(input_first) print(input_second) print(input_third) browser.close() from selenium import webdriver from selenium.webdriver.common.by import By browser = webdriver.Chrome() browser.get('https://www.taobao.com') input_first = browser.find_element(By.ID, 'q') print(input_first) browser.close() # 多个元素 from selenium import webdriver browser = webdriver.Chrome() browser.get('https://www.taobao.com') lis = browser.find_elements_by_css_selector('.service-bd li') print(lis) browser.close() from selenium import webdriver from selenium.webdriver.common.by import By browser = webdriver.Chrome() browser.get('https://www.taobao.com') lis = browser.find_elements(By.CSS_SELECTOR, '.service-bd li') print(lis) browser.close() # 元素交互操作 # 对获取的元素调用交互方法 from selenium import webdriver import time browser = webdriver.Chrome() browser.get('https://www.taobao.com') input = browser.find_element_by_id('q') input.send_keys('笔记本电脑') time.sleep(5) input.clear() input.send_keys('iPad') button = browser.find_element_by_class_name('btn-search') # button.click() # 交互动作 # 将动作附加到动作链中串行执行 from selenium import webdriver from selenium.webdriver import ActionChains browser = webdriver.Chrome() url = "http://www.runoob.com/try/try.php?filename=jqueryui-api-droppable" browser.get(url) browser.switch_to.frame('iframeResult') source = browser.find_element_by_css_selector('#draggable') target = browser.find_element_by_css_selector('#droppable') actions = ActionChains(browser) actions.drag_and_drop(source, target) actions.perform() # 执行JavaScript from selenium import webdriver browser = webdriver.Chrome() browser.get("https://www.zhihu.com/explore") browser.execute_script('window.scrollTo(0, document.body.scrollHeight)') browser.execute_script('alert("To Bottom")') # 获取元素信息 # 获取属性 from selenium import webdriver from selenium.webdriver import ActionChains browser = webdriver.Chrome() browser.get("https://www.zhihu.com/explore") logo = browser.find_element_by_id('zh-top-link-logo') print(logo) print(logo.get_attribute('class')) # 获取文本值 from selenium import webdriver browser = webdriver.Chrome() browser.get('https://www.zhihu.com/explore') input = browser.find_element_by_class_name('zu-top-add-question') print(input.text) # 获取ID、位置、标签名、大小 from selenium import webdriver browser = webdriver.Chrome() browser.get('https://www.zhihu.com/explore') input = browser.find_element_by_class_name('zu-top-add-question') print(input.id) print(input.location) print(input.tag_name) print(input.size) # Frame import time from selenium import webdriver from selenium.common.exceptions import NoSuchElementException browser = webdriver.Chrome() browser.get('http://www.runoob.com/try/try.php?filename=jqueryui-api-droppable') browser.switch_to.frame('iframeResult') source = browser.find_element_by_css_selector('#draggable') print(source) try: logo = browser.find_element_by_class_name('logo') except NoSuchElementException: print('NO LOGO') browser.switch_to.parent_frame() logo = browser.find_element_by_class_name('logo') print(logo) print(logo.text) # 等待 # 隐式等待 # 当使用了隐式等待执行测试的时候，如果WebDriver没有在DOM中找到元素，将继续等待，超出设定时间后抛出找不到元素的异常， # 换句话说，当查找元素或元素并没有立即出现的时候，隐式等待将等待一段时间再查找DOM，默认的时间是0 from selenium import webdriver browser = webdriver.Chrome() browser.implicitly_wait(10) browser.get('https://www.zhihu.com/explore') input = browser.find_element_by_class_name('zu-top-add-question') print(input) # 显示等待 from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC browser = webdriver.Chrome() browser.get('https://www.taobao.com/') wait = WebDriverWait(browser, 10) input = wait.until(EC.presence_of_element_located((By.ID, 'q'))) button = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, '.btn-search'))) print(input, button) # 前进后退 import time from selenium import webdriver browser = webdriver.Chrome() browser.get('https://www.baidu.com') browser.get('https://www.taobao.com') browser.get('https://www.python.org') browser.back() time.sleep(5) browser.forward() browser.close() # Cookies from selenium import webdriver browser = webdriver.Chrome() browser.get('https://www.zhihu.com/explore') print(browser.get_cookies()) browser.add_cookie({'name':'name', 'domain':'www.zhihu.com', 'value':'germey'}) print(browser.get_cookies()) browser.delete_all_cookies() print(browser.get_cookies()) # 选项卡管理 import time from selenium import webdriver browser = webdriver.Chrome() browser.get('https://www.baidu.com') browser.execute_script('window.open()') print(browser.window_handles) browser.switch_to_window(browser.window_handles[1]) browser.get('https://www.taobao.com') time.sleep(5) browser.switch_to_window(browser.window_handles[0]) browser.get('https://python.org') # 异常处理 from selenium import webdriver browser = webdriver.Chrome() browser.get('https://www.baidu.com') browser.find_element_by_id('hello') from selenium import webdriver from selenium.common.exceptions import TimeoutException, NoSuchElementException browser = webdriver.Chrome() try: browser.get('https://www.baidu.com') except TimeoutException: print('Time Out') try: browser.find_element_by_id('hello') except NoSuchElementException: print('No Element') finally: browser.close()