【Python】爬虫(Xpath):批量爬取站长免费简历

Geneying

话不多说吧  直接上,判定有点少,不要写0  也不要超过所提示的页数

1. from lxml import etree
   
2. import requests
   
3. import os
   
4. 
   
5. #封装解析下载函数
   
6. def cv_down(cv_href, headers):
   
7.     for href in cv_href:
   
8.         act_response = requests.get(url=href, headers=headers).text
   
9.         act_tree = etree.HTML(act_response)
   
10.         cv_title = act_tree.xpath('//div[@class="ppt_tit clearfix"]/h1/text()')
    
11.         cv_title = cv_title[0].encode('ISO-8859-1').decode('utf-8') + '.rar'
    
12.         dow_url = act_tree.xpath('//div[@class="clearfix mt20 downlist"]/ul/li[1]/a/@href')[0]
    
13.         doc = requests.get(url=dow_url, headers=headers).content
    
14.         cv_path = './免费简历/' + cv_title
    
15.         with open(cv_path, 'wb') as fp:
    
16.             fp.write(doc)
    
17.             print(cv_title, '下载完成！！！')
    
18. 
    
19. #检查文件夹是否存在,并创建文件夹
    
20. if not os.path.exists('./免费简历'):
    
21.     os.mkdir('./免费简历')
    
22. 
    
23. first_page_url = 'http://sc.chinaz.com/jianli/free.html'
    
24. headers = {
    
25.     "User-Agent":'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.36'
    
26. }
    
27. 
    
28. tree = etree.HTML(requests.get(url=first_page_url, headers=headers).text)
    
29. page_limit = tree.xpath('//div[@class="pagination fr clearfix clear"]/a[8]/b/text()')
    
30. 
    
31. 
    
32. print(page_limit[0] + " pages at most")
    
33. page = input("Please enter how many page you want: " )
    
34. 
    
35. if page.isdigit():
    
36.     if int(page) == 1 :
    
37.         cv_href = tree.xpath('//div[@class="sc_warp  mt20"]/div/div/div/a/@href')
    
38.         cv_down(cv_href, headers)
    
39. 
    
40.     else:
    
41.         for i in range(1, int(page)+1):
    
42.             if i == 1:
    
43.                 cv_href = tree.xpath('//div[@class="sc_warp  mt20"]/div/div/div/a/@href')
    
44.                 cv_down(cv_href, headers)
    
45.             else:
    
46.                 other_page_url = 'http://sc.chinaz.com/jianli/free_' + str(i) + '.html' #一页以上的特殊性需要重新制定链接
    
47.                 response = requests.get(url=other_page_url, headers=headers).text
    
48.                 tree = etree.HTML(response)
    
49.                 cv_href = tree.xpath('//div[@class="sc_warp  mt20"]/div/div/div/a/@href')
    
50.                 cv_down(cv_href, headers)
    
51. 
    
52. else:
    
53.     print("You can only enter numbers!!!")

复制代码

283261149

谢谢大牛

Suger

6666

zq2672687519@16

谢谢大佬

笑东风丶

感谢大佬分享，大牛因有你而更精彩。

1720034274

感谢楼主分享

lpllpl

6666666666