python爬虫基础实战案例-北京尧图网络

文章目录1 实战一爬取豆瓣Top250图书信息目标分析完整代码2 实战二批量下载图片关键技巧多线程加速下载3 实战三爬取表格数据并存入Excel安装依赖爬取天气历史数据4 完整项目知乎热榜数据采集下面通过几个经典案例把前面的知识串联起来。包含三类实战项目静态页面爬取、图片批量下载、表格数据提取与存储。每个案例都有完整代码可以直接运行。1 实战一爬取豆瓣Top250图书信息豆瓣图书是学习爬虫的经典案例页面结构清晰适合练习CSS选择器和数据存储。目标分析目标爬取豆瓣Top250图书的书名、作者、评分、出版信息、评价人数保存为CSV文件。打开F12分析页面结构每本书在div.item容器中书名在span.title中评分在span.rating_num中作者和出版信息在p标签中翻页链接在a标签URL参数为?start0,25,50...完整代码importrequestsfrombs4importBeautifulSoupimportcsvimporttimeimportrandomdefget_headers():返回伪装请求头return{User-Agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36,Accept-Language:zh-CN,zh;q0.9}defparse_book_list(html):解析单页图书列表soupBeautifulSoup(html,lxml)books[]foriteminsoup.select(div.item):# 书名有时会有副标题title_tagsitem.select(span.title)titletitle_tags[0].text.strip()iftitle_tagselse未知# 评分rating_tagitem.select_one(span.rating_num)ratingrating_tag.text.strip()ifrating_tagelse0# 评价人数inq_tagitem.select_one(span.inq)inqinq_tag.text.strip()ifinq_tagelse# 作者/出版信息info_tagitem.select_one(div.bd p)infoinfo_tag.text.strip().replace(\n, )ifinfo_tagelse# 排名rank_tagitem.select_one(em)rankrank_tag.textifrank_tagelsebooks.append({rank:rank,title:title,rating:rating,inq:inq,info:info})returnbooksdefcrawl_douban_books():爬取豆瓣图书Top250all_books[]base_urlhttps://book.douban.com/top250forstartinrange(0,250,25):urlf{base_url}?start{start}print(f爬取第{start//251}页:{url})try:responserequests.get(url,headersget_headers(),timeout10)response.raise_for_status()booksparse_book_list(response.text)all_books.extend(books)print(f本页提取{len(books)}本书)exceptExceptionase:print(f爬取失败:{e})# 随机延时 1-3 秒避免被封time.sleep(random.uniform(1,3))returnall_booksdefsave_to_csv(books,filenamedouban_books.csv):保存为CSV文件ifnotbooks:print(没有数据可保存)returnwithopen(filename,w,encodingutf-8-sig,newline)asf:fieldnames[rank,title,rating,inq,info]writercsv.DictWriter(f,fieldnamesfieldnames)writer.writeheader()writer.writerows(books)print(f已保存{len(books)}条数据至{filename})if__name____main__:bookscrawl_douban_books()save_to_csv(books)# 打印前5条预览forbookinbooks[:5]:print(f[{book[rank]}]{book[title]}-{book[rating]}分)2 实战二批量下载图片批量下载图片是爬虫的常见需求。这个案例演示如何下载图片并保存到本地。关键技巧图片是二进制数据需要用response.content获取而不是response.text。importrequestsimportosimporttimeimportrandomfrompathlibimportPathfromurllib.parseimporturlparsedefdownload_image(url,save_path,headersNone):下载单张图片try:responserequests.get(url,headersheaders,timeout15,streamTrue)response.raise_for_status()# 检查是否是图片类型content_typeresponse.headers.get(Content-Type,)ifimagenotincontent_type:print(f不是图片类型:{content_type})returnFalse# 从URL中获取文件名parsed_urlurlparse(url)filenameos.path.basename(parsed_url.path)ifnotfilenameor.notinfilename:filenamefimage_{int(time.time())}.jpg# 保存图片full_pathos.path.join(save_path,filename)withopen(full_path,wb)asf:forchunkinresponse.iter_content(chunk_size8192):f.write(chunk)file_sizeos.path.getsize(full_path)print(f下载成功:{filename}({file_size/1024:.1f}KB))returnTrueexceptExceptionase:print(f下载失败{url}:{e})returnFalsedefbatch_download_images(image_urls,save_dirimages):批量下载图片# 创建保存目录Path(save_dir).mkdir(parentsTrue,exist_okTrue)headers{User-Agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64),Referer:https://example.com# 防盗链处理}success_count0fail_count0fori,urlinenumerate(image_urls,1):print(f[{i}/{len(image_urls)}] 下载:{url[:60]}...)ifdownload_image(url,save_dir,headers):success_count1else:fail_count1# 下载间隔time.sleep(random.uniform(0.5,1.5))print(f\n下载完成: 成功{success_count}张失败{fail_count}张)# 使用示例先爬取图片URL列表再批量下载defget_image_urls_from_page(page_url):从页面中提取图片URLheaders{User-Agent:Mozilla/5.0 ...}responserequests.get(page_url,headersheaders)soupBeautifulSoup(response.text,lxml)image_urls[]forimginsoup.select(div.gallery img):srcimg.get(src)orimg.get(data-src,)ifsrcandsrc.startswith(http):image_urls.append(src)returnimage_urls# 主流程# page_url https://example.com/gallery# urls get_image_urls_from_page(page_url)# batch_download_images(urls, save_dirdownloaded_images)多线程加速下载当图片数量多时单线程速度太慢。可以用多线程并发下载。fromconcurrent.futuresimportThreadPoolExecutor,as_completeddefbatch_download_with_threads(image_urls,save_dirimages,max_workers5):多线程批量下载Path(save_dir).mkdir(parentsTrue,exist_okTrue)headers{User-Agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64)}defdownload_task(args):idx,urlargsreturndownload_image(url,save_dir,headers)taskslist(enumerate(image_urls,1))success0withThreadPoolExecutor(max_workersmax_workers)asexecutor:future_to_url{executor.submit(download_task,task):taskfortaskintasks}forfutureinas_completed(future_to_url):idx,urlfuture_to_url[future]try:iffuture.result():success1exceptExceptionase:print(f任务异常:{e})print(f多线程下载完成:{success}/{len(image_urls)})3 实战三爬取表格数据并存入Excel很多网站有表格形式的数据比如排行榜、统计数据等。这个案例演示如何完整爬取这类数据。安装依赖pipinstallopenpyxl pandas爬取天气历史数据importrequestsfrombs4importBeautifulSoupimportpandasaspdfromdatetimeimportdatetimedefcrawl_table_data(url,table_selectortable): 通用表格数据爬取函数 headers{User-Agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36}responserequests.get(url,headersheaders,timeout10)response.encodingutf-8soupBeautifulSoup(response.text,lxml)tablesoup.select_one(table_selector)ifnottable:print(f未找到表格:{table_selector})returnNone# 提取表头headers_rowtable.select(tr:first-child th)ifnotheaders_row:headers_rowtable.select(thead tr th)columns[th.text.strip()forthinheaders_row]# 提取数据行rows[]fortrintable.select(tbody tr):cellstr.select(td)ifcells:row_data[cell.text.strip()forcellincells]rows.append(row_data)ifnotcolumnsornotrows:print(表格数据为空)returnNone# 转换为DataFramedfpd.DataFrame(rows,columnscolumns)returndfdefsave_to_excel(df,filenametable_data.xlsx,sheet_name数据):保存到Excel支持格式美化withpd.ExcelWriter(filename,engineopenpyxl)aswriter:df.to_excel(writer,sheet_namesheet_name,indexFalse)# 获取worksheet对象进行格式调整wswriter.sheets[sheet_name]# 自动调整列宽forcolumninws.columns:max_length0col_lettercolumn[0].column_letterforcellincolumn:ifcell.value:max_lengthmax(max_length,len(str(cell.value)))ws.column_dimensions[col_letter].widthmin(max_length4,30)print(f已保存到{filename})# 多页表格数据爬取defcrawl_multi_page_table(base_url,total_pages,page_parampage):多页表格数据爬取all_data[]forpageinrange(1,total_pages1):urlf{base_url}?{page_param}{page}print(f爬取第{page}页...)dfcrawl_table_data(url)ifdfisnotNone:df[来源页码]page all_data.append(df)importtime time.sleep(1)ifall_data:resultpd.concat(all_data,ignore_indexTrue)print(f共爬取{len(result)}条数据)returnresultreturnNone4 完整项目知乎热榜数据采集importrequestsimportjsonimportcsvfromdatetimeimportdatetimedefcrawl_zhihu_hot(): 爬取知乎热榜接口版本比解析HTML更稳定 # 知乎热榜接口api_urlhttps://www.zhihu.com/api/v3/feed/topstory/hot-lists/totalheaders{User-Agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36,Referer:https://www.zhihu.com/hot,Accept:application/json}params{limit:50,desktop:true}responserequests.get(api_url,headersheaders,paramsparams,timeout10)ifresponse.status_code!200:print(f请求失败:{response.status_code})return[]dataresponse.json()hot_listdata.get(data,[])results[]forrank,iteminenumerate(hot_list,1):targetitem.get(target,{})results.append({排名:rank,标题:target.get(title,),热度:item.get(detail_text,),回答数:target.get(answer_count,0),关注数:target.get(follower_count,0),链接:fhttps://www.zhihu.com/question/{target.get(id,)},采集时间:datetime.now().strftime(%Y-%m-%d %H:%M)})returnresultsdefsave_hot_list(data,filenameNone):ifnotfilename:filenamefzhihu_hot_{datetime.now().strftime(%Y%m%d_%H%M)}.csvwithopen(filename,w,encodingutf-8-sig,newline)asf:writercsv.DictWriter(f,fieldnamesdata[0].keys())writer.writeheader()writer.writerows(data)print(f已保存{len(data)}条热榜数据至{filename})if__name____main__:hot_datacrawl_zhihu_hot()ifhot_data:save_hot_list(hot_data)print(f\n当前热榜前10:)foriteminhot_data[:10]:print(f[{item[排名]}]{item[标题]}-{item[热度]})以上三个案例覆盖了静态爬虫的核心场景。注意在实际使用时需要添加适当的请求延时遵守网站robots.txt规则不要爬取隐私数据。下一章进入爬虫的进阶领域动态网页爬取。

python爬虫基础实战案例

相关推荐

DVD视频备份与MP4格式转换全指南

企业私有化大模型平台CSGHub架构与落地实践

用Markdown编辑器

更多资讯

3步搞定RoboLab性能优化拒绝报错堆栈看不懂

差分信号转单端输出：运放电路设计与实操全解析

2026最新：告别配置地狱，这3种工具最适合性能优化

天玑1100面试必问:手写核心逻辑,别再只背八股文

3步图解原理：解决学术剽窃检测报错

泰坦之旅存档底层逻辑揭秘：新手避坑的3个关键数据点

今日更新

3个坑点带你一文搞懂55gg小游戏源码

2026最新covar实战：3步搞定环境配置不再卡壳

3个Docker命令避坑指南：手写实现原理

本周精选

深入解析Transformer多头注意力机制与工程优化

OpenClaw 的 Skills 跑学习任务，模型通道改到 TaoToken 通道行不行？

ChatGPT报错Oops, an error occurred! 全链路排查指南

本月热门

基于YOLO的焊缝缺陷检测：从数据标注到工业部署全流程实践

分布式服务的决策记录

创业产品的首版范围

读完文章，想聊聊您的网站？