本文转载自微信公众号「UP技术控」，可以通过以下二维码关注。转载本文请联系UP技术控公众号。
背景
Selenium是一个用于Web应用程序测试的工具。Selenium测试直接运行在浏览器中，就像真正的用户在操作一样。而对于爬虫来说，使用Selenium操控浏览器来爬取网上的数据那么肯定是爬虫中的杀手武器。这里，我将介绍selenium + 谷歌浏览器的一般使用。
需求
在平常的爬虫开发中，有时候网页是一堆js堆起来的代码，涉及很多异步计算，如果是普通的http 控制台请求，那么得到的源文件是一堆js ，需要自己在去组装数据，很费力;但是采用Selenium+ChromeDriver可以达到所见即所得的完美效果。
实现方式
项目结构：为了方便使用，用的winform程序，附nuget包
以下是form1.cs的代码，这里就只放关键方法代码了。需要安装最新的chrome浏览器+代码中使用的chromedriver是 v2.9.248315
private void crawlingWebFunc()
{
SetText("\r\n开始尝试...");
List<testfold> surls = new List<testfold>();
string path = System.Environment.CurrentDirectory + "\\图片url\\";
DirectoryInfo root = new DirectoryInfo(path);
DirectoryInfo[] dics = root.GetDirectories();
foreach (var itemdic in dics)
{
string txt = "";
StreamReader sr = new StreamReader(itemdic.FullName + "\\data.txt");
while (!sr.EndOfStream)
{
string str = sr.ReadLine();
txt += str;// + "\n";
}
sr.Close();
surls.Add(new testfold() { key = itemdic.FullName, picurl = txt });
}
ChromeDriverService service = ChromeDriverService.CreateDefaultService(System.Environment.CurrentDirectory);
// service.HideCommandPromptWindow = true;
ChromeOptions options = new ChromeOptions();
options.AddArguments("--test-type", "--ignore-certificate-errors");
options.AddArgument("enable-automation");
// options.AddArgument("headless");
// options.AddArguments("--proxy-server=http://user:password@yourProxyServer.com:8080");
using (IWebDriver driver = new OpenQA.Selenium.Chrome.ChromeDriver(service, options, TimeSpan.FromSeconds(120)))
{
driver.Url = "https://www.1688.com/";
Thread.Sleep(200);
try
{
int a = 1;
foreach (var itemsurls in surls)
{
SetText("\r\n第" + a.ToString() + "个");
driver.Navigate().GoToUrl(itemsurls.picurl);
//登录
if (driver.Url.Contains("login.1688.com"))
{
SetText("\r\n需要登录，开始尝试...");
trylogin(driver); //尝试登录完成
//再试试
driver.Navigate().GoToUrl("https://s.1688.com/youyuan/index.htm?tab=imageSearch&imageType=oss&imageAddress=cbuimgsearch/eWXC7XHHPN1607529600000&spm=");
if (driver.Url.Contains("login.1688.com"))
{
//没办法退出
SetText("\r\n退出，换ip重试...");
return;
}
}
//鼠标放上去的内容因为页面自带只能显示一个的原因 没办法做到全部显示 然后在下载 只能是其他方式下载
// var elements = document.getElementsByClassName('hover-container');
// Array.prototype.forEach.call(elements, function(element) {
// element.style.display = "block";
// console.log(element);
// });
// IJavaScriptExecutor js = (IJavaScriptExecutor)driver;
// var sss = js.ExecuteScript(" var elements = document.getElementsByClassName('hover-container'); Array.prototype.forEach.call(elements, function(element) { console.log(element); element.setAttribute(\"class\", \"测试title\"); element.style.display = \"block\"; console.log(element); });");
Thread.Sleep(500);
var responseModel = Write(itemsurls.key, driver.PageSource, Pagetypeenum.列表);
Thread.Sleep(500);
int i = 1;
foreach (var offer in responseModel?.data?.offerList ?? new List<OfferItemModel>())
{
driver.Navigate().GoToUrl(offer.information.detailUrl);
string responseDatadetail = driver.PageSource;
Write(itemsurls.key, driver.PageSource, Pagetypeenum.详情);
SetText("\r\n第" + a.ToString() + "-" + i.ToString() + "个");
Thread.Sleep(500);
i++;
}
}
}
catch (Exception ex)
{
CloseChromeDriver(driver);
throw;
}
}
}
- 1.
- 2.
- 3.
- 4.
- 5.
- 6.
- 7.
- 8.
- 9.
- 10.
- 11.
- 12.
- 13.
- 14.
- 15.
- 16.
- 17.
- 18.
- 19.
- 20.
- 21.
- 22.
- 23.
- 24.
- 25.
- 26.
- 27.
- 28.
- 29.
- 30.
- 31.
- 32.
- 33.
- 34.
- 35.
- 36.
- 37.
- 38.
- 39.
- 40.
- 41.
- 42.
- 43.
- 44.
- 45.
- 46.
- 47.
- 48.
- 49.
- 50.
- 51.
- 52.
- 53.
- 54.
- 55.
- 56.
- 57.
- 58.
- 59.
- 60.
- 61.
- 62.
- 63.
- 64.
- 65.
- 66.
- 67.
- 68.
- 69.
- 70.
- 71.
- 72.
- 73.
- 74.
- 75.
- 76.
- 77.
- 78.
- 79.
- 80.
- 81.
- 82.
- 83.
- 84.
- 85.
- 86.
- 87.
- 88.
- 89.
#region 异常 退出chromedriver
[DllImport("user32.dll", EntryPoint = "FindWindow")]
private extern static IntPtr FindWindow(string lpClassName, string lpWindowName);
[DllImport("user32.dll", EntryPoint = "SendMessage")]
public static extern int SendMessage(IntPtr hWnd, int Msg, int wParam, int lParam);
public const int SW_HIDE = 0;
public const int SW_SHOW = 5;
[DllImport("user32.dll", EntryPoint = "ShowWindow")]
public static extern int ShowWindow(IntPtr hwnd, int nCmdShow);
/// <summary>
/// 获取窗口句柄
/// </summary>
/// <returns></returns>
public IntPtr GetWindowHandle()
{
string name = (Environment.CurrentDirectory + "\\chromedriver.exe");
IntPtr hwd = FindWindow(null, name);
return hwd;
}
/// <summary>
/// 关闭chromedriver窗口
/// </summary>
public void CloseWindow()
{
try
{
IntPtr hwd = GetWindowHandle();
SendMessage(hwd, 0x10, 0, 0);
}
catch { }
}
/// <summary>
/// 退出chromedriver
/// </summary>
/// <param name="driver"></param>
public void CloseChromeDriver(IWebDriver driver)
{
try
{
driver.Quit();
driver.Dispose();
}
catch { }
CloseWindow();
}
#endregion 异常 退出chromedriver
- 1.
- 2.
- 3.
- 4.
- 5.
- 6.
- 7.
- 8.
- 9.
- 10.
- 11.
- 12.
- 13.
- 14.
- 15.
- 16.
- 17.
- 18.
- 19.
- 20.
- 21.
- 22.
- 23.
- 24.
- 25.
- 26.
- 27.
- 28.
- 29.
- 30.
- 31.
- 32.
- 33.
- 34.
- 35.
- 36.
- 37.
- 38.
- 39.
- 40.
- 41.
- 42.
- 43.
- 44.
- 45.
- 46.
- 47.
- 48.
- 49.
- 50.
- 51.
- 52.
- 53.
- 54.
效果
总结
说一下思路:
1.跳转到指定的网页driver.Navigate().GoToUrl
2.确定数据源，从driver.PageSource读取数据
3.对html数据进行解析
