对于静态页面可以用request获取当前页面上html的源代码，这里不多说了，代码中也已经注释。方法也比较简单，本人觉得现在的网站多数是采用的动态页面，所以如何爬取js动态加载后的html内容还是比较重要的。以上海快3开奖结果为例，实时的爬取快3的开奖结果，代码如下：

# -*- coding:utf-8 -*- from bs4 import BeautifulSoup import datetime,time from selenium.webdriver import Firefox from selenium.webdriver.firefox.options import Options from selenium.webdriver.common.desired_capabilities import DesiredCapabilities import logging import traceback from retrying import retry now_time = datetime.datetime.now() url='http://fucai.eastday.com/LotteryNew/K3Result.aspx' # user_agent={ # "User-Agent":"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:48.0) Gecko/20100101 Firefox/48.0"} # headers = {"User-Agent": user_agent } #用无图形化Firefox来获取当前页面的html，所以动态静态都一手掌控，稳定性很好，而且不容易被反爬虫拦截 @retry def get_html(url): options = Options() options.add_argument('-headless') # 无头参数 dcap = dict(DesiredCapabilities.FIREFOX) dcap['FIREFOX.page.settings.userAgent'] = ( 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36') browser = Firefox(firefox_options=options,desired_capabilities=dcap) time.sleep(2) browser.get(url) htmlPage=browser.page_source browser.close() return htmlPage # def get_html(url): # htmlPage=requests.get(url).text # return htmlPage #下一步就是根据获取到的html来做分析，取出我们需要的开奖结果的数值 def analyze(htmlPage): soup = BeautifulSoup(htmlPage, "lxml") # print soup.prettify() # tagTable = soup.find('table', classname='san-sheet-alternating-2') # result=soup.find_all(attrs={"class": "td_title02"}) # result = soup.find(text=re.compile('\d\s*,\s*\d\s*,\s*\d')) results = soup.find_all('span')[:3] list=[] for i in results: list.append(i.string) result=list[0]+','+list[1]+','+list[2] now_time = datetime.datetime.now() print now_time,u'开奖结果为:',result #通过while循环来做到实时获取 while 1: htmlPage = get_html(url) analyze(htmlPage) time.sleep(60)