网络爬虫之正则和bs4方式

#coding=utf-8

import urllib

import urllib2

import re

import sys 

reload(sys) 

sys.setdefaultencoding('utf-8')

f = open("D:\Spider.txt","w")

type = sys.getfilesystemencoding()

print type

page=1

for page in range(1,58):

    url='http://yyk.39.net/shenzhen/hospitals/c_p'+str(page)+'/'

    user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'

    headers = { 'User-Agent' : user_agent }

    

    try:

        request=urllib2.Request(url,headers = headers)

        response=urllib2.urlopen(request)

        content = response.read()

        pattern=re.compile('<a href="/sz/zonghe/.*?.html" class="yy-name" title=".*?">(.*?)</a>.*?</div>.*?<p>(.*?)</p>.*?<p class="di">.*?<a href="/sz/zonghe/.*?.html" title=".*?">(.*?)</a>.*?</p>',re.S)

        items=re.findall(pattern,content)

        for item in items:

            f.write(item[0]+","+item[1].strip()+','+item[2]+"\n")

    except urllib2.URLError,e:

        if hasattr(e, "code"):

            print e.code

        if hasattr(e, "reason"):

            print e.reason

f.close()

#coding=utf-8

import urllib

import re

import urllib2

import bs4

from bs4 import BeautifulSoup

import json

import sys 

reload(sys) 

sys.setdefaultencoding('utf-8')

page=1

for page in range(1,2):

    url='http://yyk.39.net/shenzhen/hospitals/c_p'+str(page)+'/'

    user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'

    headers = { 'User-Agent' : user_agent }

    

    try:

        request=urllib2.Request(url,headers = headers)

        response=urllib2.urlopen(request)

        content = response.read()

        soup=BeautifulSoup(content)

        pattern=re.compile('<p class="di"><a href=".*?>(.*?)</a></p>',re.S)

        #items=re.findall(pattern,content)

        #print soup.prettify()

#         for yyname in soup.find_all('a',class_='yy-name'):

#             print yyname

        L= soup.find_all('p',class_='di')

        print type(list(L))

        LL= [re.findall(pattern, str(ss))[0] for ss in L]

        res_data=json.dumps(LL,ensure_ascii=False,encoding="utf-8")

        print res_data

                        

    except urllib2.URLError,e:

        if hasattr(e, "code"):

            print e.code

        if hasattr(e, "reason"):

            print e.reason