受欢迎的博客标签

.NetCore实践爬虫系统:HtmlAgilityPack解析网页内容

Published

Regular Expression Example: Scanning for HREFs

https://docs.microsoft.com/en-us/dotnet/standard/base-types/regular-expression-example-scanning-for-hrefs

Example 1

private static void DumpHRefs(string inputString)
{
    Match m;
    string HRefPattern = @"href\s*=\s*(?:[""'](?<1>[^""']*)[""']|(?<1>\S+))";

    try
    {
        m = Regex.Match(inputString, HRefPattern,
                        RegexOptions.IgnoreCase | RegexOptions.Compiled,
                        TimeSpan.FromSeconds(1));
        while (m.Success)
        {
            Console.WriteLine("Found href " + m.Groups[1] + " at "
               + m.Groups[1].Index);
            m = m.NextMatch();
        }
    }
    catch (RegexMatchTimeoutException)
    {
        Console.WriteLine("The matching operation timed out.");
    }
}
public static void Main()
{
    string inputString = "My favorite web sites include:</P>" +
                         "<A HREF=\"http://msdn2.microsoft.com\">" +
                         "MSDN Home Page</A></P>" +
                         "<A HREF=\"http://www.microsoft.com\">" +
                         "Microsoft Corporation Home Page</A></P>" +
                         "<A HREF=\"http://blogs.msdn.com/bclteam\">" +
                         ".NET Base Class Library blog</A></P>";
    DumpHRefs(inputString);
}
// The example displays the following output:
//       Found href http://msdn2.microsoft.com at 43
//       Found href http://www.microsoft.com at 102
//       Found href http://blogs.msdn.com/bclteam at 176

 

Example 2

string prttern = "<a(\\s+(href=\"(?<url>([^\"])*)\"|'([^'])*'|\\w+=\"(([^\"])*)\"|'([^'])*'))+>(?<text>(.*?))</a>";
var maths = Regex.Matches(text, prttern);
//抓取出来写入的文件
using (FileStream w = new FileStream(Environment.CurrentDirectory + "//wirter.txt", FileMode.Create))
{
    for (int i = 0; i &lt; maths.Count; i++)
    {
      byte[] bs = Encoding.UTF8.GetBytes(string.Format("链接地址:{0},  innerhtml:{1}", maths[i].Groups["url"].Value,
        maths[i].Groups["text"].Value) + "\r\n");
      w.Write(bs, 0, bs.Length);
      Console.WriteLine();
    }
}

正则表达式取<script src=></script>中的src部分内容

string prttern = "<script(\\s+(src=\"(?<url>([^\"])*)\"|'([^'])*'|\\w+=\"(([^\"])*)\"|'([^'])*'))+>(?<text>(.*?))</script>";
var maths = Regex.Matches(text, prttern);
//抓取出来写入的文件
using (FileStream w = new FileStream(Environment.CurrentDirectory + "//wirter.txt", FileMode.Create))
{
    for (int i = 0; i &lt; maths.Count; i++)
    {
      byte[] bs = Encoding.UTF8.GetBytes(string.Format("链接地址:{0},  innerhtml:{1}", maths[i].Groups["url"].Value,
        maths[i].Groups["text"].Value) + "\r\n");
      w.Write(bs, 0, bs.Length);
      Console.WriteLine();
    }
}

 

HtmlAgilityPack

 

爬虫系统的意义

爬虫的意义在于采集大批量数据,然后基于此进行加工/分析,做更有意义的事情。谷歌,百度,今日头条,天眼查都离不开爬虫。

今日目标

今天我们来实践一个最简单的爬虫系统。根据Url来识别网页内容。

网页内容识别利器:HtmlAgilityPack

GitHub地址

HtmlAgilityPack官网

HtmlAgilityPack的stackoverflow地址

至今Nuget已有超过900多万的下载量,应用量十分庞大。它提供的文档教程也十分简单易用。

Parser解析器

HtmlParse可以让你解析HTML并返回HtmlDocument

  • FromFile从文件读取
/// <summary>
/// 从文件读取
/// </summary>
public void FromFile() {
var path = @"test.html";
var doc = new HtmlDocument();
doc.Load(path);
var node = doc.DocumentNode.SelectSingleNode("//body");
Console.WriteLine(node.OuterHtml);
}
  • 从字符串加载
/// <summary>
/// 从字符串读取
/// </summary>
public void FromString()
{
var html = @"<!DOCTYPE html>
<html>
<body>
<h1>This is <b>bold</b> heading</h1>
<p>This is <u>underlined</u> paragraph</p>
<h2>This is <i>italic</i> heading</h2>
</body>
</html> ";
var htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(html);
var htmlBody = htmlDoc.DocumentNode.SelectSingleNode("//body");
Console.WriteLine(htmlBody.OuterHtml);
}
  • 从网络加载
/// <summary>
/// 从网络地址加载
/// </summary>
public void FromWeb() {
var html = @"https://www.cnblogs.com/";
HtmlWeb web = new HtmlWeb();
var htmlDoc = web.Load(html);
var node = htmlDoc.DocumentNode.SelectSingleNode("//div[@id='post_list']");
Console.WriteLine("Node Name: " + node.Name + "\n" + node.OuterHtml);
}

Selectors选择器

选择器允许您从HtmlDocument中选择HTML节点。它提供了两个方法,可以用XPath表达式筛选节点。XPath教程

SelectNodes() 返回多个节点

SelectSingleNode(String) 返回单个节点

简介到此为止,更全的用法参考 http://html-agility-pack.net

查看网页结构

我们以博客园首页为示例。用chrome分析下网页结构,可采集出推荐数,标题,内容Url,内容简要,作者,评论数,阅读数。

博客园主页内容结构图

编码实现

建立一个Article用来接收文章信息。


public class Article
{
/// <summary>
///
/// </summary>
public string Id { get; set; }
/// <summary>
/// 标题
/// </summary>
public string Title { get; set; }
/// <summary>
/// 概要
/// </summary>
public string Summary { get; set; }
/// <summary>
/// 文章链接
/// </summary>
public string Url { get; set; }
/// <summary>
/// 推荐数
/// </summary>
public long Diggit { get; set; }
/// <summary>
/// 评论数
/// </summary>
public long Comment { get; set; }
/// <summary>
/// 阅读数
/// </summary>
public long View { get; set; }
/// <summary>
///明细
/// </summary>
public string Detail { get; set; }
/// <summary>
///作者
/// </summary>
public string Author { get; set; }
/// <summary>
/// 作者链接
/// </summary>
public string AuthorUrl { get; set; }
}

然后根据网页结构,查看XPath路径,采集内容

/// <summary>
/// 解析
/// </summary>
/// <returns></returns>
public List<Article> ParseCnBlogs()
{
var url = "https://www.cnblogs.com";
HtmlWeb web = new HtmlWeb();
//1.支持从web或本地path加载html
var htmlDoc = web.Load(url);
var post_listnode = htmlDoc.DocumentNode.SelectSingleNode("//div[@id='post_list']");
Console.WriteLine("Node Name: " + post_listnode.Name + "\n" + post_listnode.OuterHtml);
var postitemsNodes = post_listnode.SelectNodes("//div[@class='post_item']");
var articles = new List<Article>();
var digitRegex = @"[^0-9]+";
foreach (var item in postitemsNodes)
{
var article = new Article();
var diggnumnode = item.SelectSingleNode("//span[@class='diggnum']");
//body
var post_item_bodynode = item.SelectSingleNode("//div[@class='post_item_body']");
var titlenode = post_item_bodynode.SelectSingleNode("//a[@class='titlelnk']");
var summarynode = post_item_bodynode.SelectSingleNode("//p[@class='post_item_summary']");
//foot
var footnode = item.SelectSingleNode("//div[@class='post_item_foot']");
var authornode = footnode.ChildNodes[1];
var commentnode = item.SelectSingleNode("//span[@class='article_comment']");
var viewnode = item.SelectSingleNode("//span[@class='article_view']");
article.Diggit = int.Parse(diggnumnode.InnerText);
article.Title = titlenode.InnerText;
article.Url = titlenode.Attributes["href"].Value;
article.Summary = titlenode.InnerHtml;
article.Author = authornode.InnerText;
article.AuthorUrl = authornode.Attributes["href"].Value;
article.Comment = int.Parse(Regex.Replace(commentnode.ChildNodes[0].InnerText, digitRegex, ""));
article.View = int.Parse(Regex.Replace(viewnode.ChildNodes[0].InnerText, digitRegex, ""));
articles.Add(article);
}
return articles;
}

查看采集结果

看到结果就惊呆了,竟然全是重复的。难道是Xpath语法理解不对么? 采集结果

重温下XPath语法

XPath 使用路径表达式在 XML 文档中选取节点。节点是通过沿着路径或者 step 来选取的

表达式 描述
nodename	选取此节点的所有子节点。
/ 从根节点选取。
// 从匹配选择的当前节点选择文档中的节点,而不考虑它们的位置。
. 选取当前节点。
.. 选取当前节点的父节点。
@ 选取属性。

XPath 通配符可用来选取未知的 XML 元素

通配符 描述
* 匹配任何元素节点。
@* 匹配任何属性节点。
node() 匹配任何类型的节点。

我测试了几个语法如:

//1,会返回20
var titlenodes = post_item_bodynode.SelectNodes("//a[@class='titlelnk']");
//会报错,因为这个a并不直接在bodynode下面,而是在子级h3元素的子级。
var titlenodes = post_item_bodynode.SelectNodes("a[@class='titlelnk']");

然后又实验了一种:

//Bingo,这个可以,但是强烈指定了下级h3,这就稍微麻烦了点。
var titlenodes = post_item_bodynode.SelectNodes("h3//a[@class='titlelnk']");

这里就引申出了一个小问题:如何定位子级的子级?用通配符*可以么?

//返回1个。
var titlenodes= post_item_bodynode.SelectNodes("*//a[@class='titlelnk']")

能正确返回1,应该是可以了,我们改下代码看下效果。 运行结果然后和博客园首页数据对比,结果吻合。

所以我们可以得出结论:

XPath搜索以//开头时,会匹配所有的项,并不是子项。

直属子级可以直接跟上 node名称。

只想查子级的子级,可以用*代替子级,实现模糊搜索。

改过后代码如下:

public List<Article> ParseCnBlogs()
{
var url = "https://www.cnblogs.com";
HtmlWeb web = new HtmlWeb();
//1.支持从web或本地path加载html
var htmlDoc = web.Load(url);
var post_listnode = htmlDoc.DocumentNode.SelectSingleNode("//div[@id='post_list']");
//Console.WriteLine("Node Name: " + post_listnode.Name + "\n" + post_listnode.OuterHtml);
var postitemsNodes = post_listnode.SelectNodes("div[@class='post_item']");
var articles = new List<Article>();
var digitRegex = @"[^0-9]+";
foreach (var item in postitemsNodes)
{
var article = new Article();
var diggnumnode = item.SelectSingleNode("*//span[@class='diggnum']");
//body
var post_item_bodynode = item.SelectSingleNode("div[@class='post_item_body']");
var titlenode = post_item_bodynode.SelectSingleNode("*//a[@class='titlelnk']");
var summarynode = post_item_bodynode.SelectSingleNode("p[@class='post_item_summary']");
//foot
var footnode = post_item_bodynode.SelectSingleNode("div[@class='post_item_foot']");
var authornode = footnode.ChildNodes[1];
var commentnode = footnode.SelectSingleNode("span[@class='article_comment']");
var viewnode = footnode.SelectSingleNode("span[@class='article_view']");
article.Diggit = int.Parse(diggnumnode.InnerText);
article.Title = titlenode.InnerText;
article.Url = titlenode.Attributes["href"].Value;
article.Summary = titlenode.InnerHtml;
article.Author = authornode.InnerText;
article.AuthorUrl = authornode.Attributes["href"].Value;
article.Comment = int.Parse(Regex.Replace(commentnode.ChildNodes[0].InnerText, digitRegex, ""));
article.View = int.Parse(Regex.Replace(viewnode.ChildNodes[0].InnerText, digitRegex, ""));
articles.Add(article);
}
return articles;
}
 
 
 

C# HtmlAgilityPack parse <ul>

The html is

<div class="wrapper">
    <ul>
                <li data="334040566050326217">
                    <span>test1</span>
                </li>
                <li data="334040566050326447">
                    <span>test2</span>
                </li>
    </ul>

 

Parse:

//Assumes your document is loaded into a variable named 'document'

List<string> dataAttribute = new List<string>(); //This will contain the long # in the data attribute
List<string> spanText = new List<string>();      //This will contain the text between the <span> tags
HtmlNodeCollection nodeCollection = document.DocumentNode.SelectNodes("//div[@class='wrapper']//li");

foreach (HtmlNode node in nodeCollection)
{
    dataAttribute.Add(node.GetAttributeValue("data", "null"));
    spanText.Add(node.SelectSingleNode("span").InnerText);
}