java|HtmlUnit入门 java|框架

HtmlUnit介绍 【java|HtmlUnit入门】htmlunit 是一款开源的java 页面分析工具，读取页面后，可以有效的使用htmlunit分析页面上的内容。项目可以模拟浏览器运行，被誉为java浏览器的开源实现。是一个没有界面的浏览器，运行速度迅速。
入门案例 pom.xml加入依赖

net.sourceforge.htmlunit htmlunit 2.23

获取页面内容

/** * 获取页面内容 */ public class HtmlUnitDemo03 {public static void main(String[] args) { //实例化web客户端 WebClient webClient = new WebClient(BrowserVersion.FIREFOX_45,"124.206.107.125",3128); //使用代理IP try { //解析获取页面 HtmlPage page = webClient.getPage("http://www.java1234.com"); HtmlElement div = page.getHtmlElementById("navMenu"); System.out.println(div.asXml()); System.out.println("---------------"); //获取a标签 DomNodeList dnList = page.getElementsByTagName("a"); for (int i = 0; i < dnList.getLength(); i++) { DomElement element = dnList.get(i); System.out.println(element.asXml()); System.out.println(element.asText()); } System.out.println("------------------"); List itemList = (List) page.getByXPath("//div[@id='navMenu']/ul/li"); System.out.println(itemList.get(0).asXml()); } catch (FailingHttpStatusCodeException e) { e.printStackTrace(); } catch (MalformedURLException e) { e.printStackTrace(); } catch (ElementNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } finally { //释放资源 webClient.close(); } } }

取消css,js支持

/** * 取消css,js支持 */ public class HtmlUnitDemo04 {public static void main(String[] args) { //实例化web客户端 WebClient webClient = new WebClient(BrowserVersion.FIREFOX_45,"124.206.107.125",3128); //使用代理IP //取消css支持 webClient.getOptions().setCssEnabled(false); //取消js支持 webClient.getOptions().setJavaScriptEnabled(false); try { //解析获取页面 HtmlPage page = webClient.getPage("http://www.java1234.com"); HtmlElement div = page.getHtmlElementById("navMenu"); System.out.println(div.asXml()); System.out.println("---------------"); //获取a标签 DomNodeList dnList = page.getElementsByTagName("a"); for (int i = 0; i < dnList.getLength(); i++) { DomElement element = dnList.get(i); System.out.println(element.asXml()); } System.out.println("------------------"); List itemList = (List) page.getByXPath("//div[@id='navMenu']/ul/li"); System.out.println(itemList.get(0).asXml()); } catch (FailingHttpStatusCodeException e) { e.printStackTrace(); } catch (MalformedURLException e) { e.printStackTrace(); } catch (ElementNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } finally { //释放资源 webClient.close(); } } }

模拟表单搜索

/** * 模拟表单搜索 */ public class HtmlUnitDemo05 {public static void main(String[] args) { //实例化web客户端 WebClient webClient = new WebClient(BrowserVersion.FIREFOX_45,"124.206.107.125",3128); //使用代理IP try { //解析获取页面 HtmlPage page = webClient.getPage("http://blog.java1234.com/index.html"); //获取搜索表单 HtmlForm form = page.getFormByName("myform"); //获取查询文本框 HtmlTextInput textInput = form.getInputByName("q"); //获取提交按钮 HtmlSubmitInput submitInput = form.getInputByName("submitButton"); //输入文本框 textInput.setValueAttribute("java"); //模拟提交(点击) HtmlPage resultPage = submitInput.click(); System.out.println(resultPage.asXml()); } catch (FailingHttpStatusCodeException e) { e.printStackTrace(); } catch (MalformedURLException e) { e.printStackTrace(); } catch (ElementNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } finally { //释放资源 webClient.close(); } } }

获取ajax加载的内容

/** * 获取ajax加载的内容 */ public class HtmlUnitDemo06 {public static void main(String[] args) { //实例化web客户端 WebClient webClient = new WebClient(BrowserVersion.FIREFOX_45); //使用代理IP try { //解析获取页面 HtmlPage page = webClient.getPage("https://pan.baidu.com/share/home?uk=305605848#category/type=0"); //等待htmlunit执行js Thread.sleep(10000); System.out.println("网页html:"+page.asXml()); } catch (FailingHttpStatusCodeException e) { e.printStackTrace(); } catch (MalformedURLException e) { e.printStackTrace(); } catch (ElementNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } catch (InterruptedException e) { e.printStackTrace(); } finally { //释放资源 webClient.close(); } } }