java读取(正则表达式分析)网页内容--688IT编程网

http:forphone.taobao：

package com.xiaofeng.picup;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.MalformedURLException;
import java.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import Matcher;
import Pattern;

/**
*

* @author 晓峰2007.1.18 抓取雅虎知识堂的文章标题及内容（测试）手动输入网址抓取，可进一步自动抓取整个知识堂的全部内容
*
*/
public class WebContent
{
/**
* 读取一个网页全部内容
*/
public String getOneHtml(final String htmlurl) throws IOException
{
URL url;
String temp;
final StringBuffer sb = new StringBuffer();
try

{
url = new URL(htmlurl);
final BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream(), "utf-8"));// 读取网页全部内容
while ((temp = in.readLine()) != null)
{
sb.append(temp);
}
in.close();
}
catch (final MalformedURLException me)
{
System.out.println("你输入的URL格式有问题！请仔细输入");
me.getMessage();
throw me;

}
catch (final IOException e)
{
e.printStackTrace();
throw e;
}
String();
}

/**
*
* @param s
* @return 获得网页标题
*/
public String getTitle(final String s)
{

String regex;
String title = "";
final List<String> list = new ArrayList<String>();
regex = "<title>.*?</title>";
final Pattern pa = Patternpile(regex, Pattern.CANON_EQ);
final Matcher ma = pa.matcher(s);
while (ma.find())
{
list.up());
}
for (int i = 0; i < list.size(); i++)
{
title = title + (i);
}
return outTag(title);
}

/**
*
* @param s
* @return 获得链接
*/
public List<String> getLink(final String s)
{
String regex;
final List<String> list = new ArrayList<String>();
regex = "<a[^>]*href=(/"([^/"]*)/"|/'([^/']*)/'|([^//s>]*))[^>]*>(.*?)</a>";
final Pattern pa = Patternpile(regex, Pattern.DOTALL);
final Matcher ma = pa.matcher(s);
while (ma.find())
{
list.up());

}
return list;
}

/**
*
* @param s
* @return 获得脚本代码
*/
public List<String> getScript(final String s)
{
String regex;
final List<String> list = new ArrayList<String>();
regex = "<script.*?</script>";
final Pattern pa = Patternpile(regex, Pattern.DOTALL);
final Matcher ma = pa.matcher(s);

while (ma.find())
{
list.up());
}
return list;
}

/**
*
* @param s
* @return 获得CSS
*/
public List<String> getCSS(final String s)
{
String regex;
final List<String> list = new ArrayList<String>();

regex = "<style.*?</style>";
final Pattern pa = Patternpile(regex, Pattern.DOTALL);
final Matcher ma = pa.matcher(s);
while (ma.find())
{
list.up());
}
return list;
}

/**
*
* @param s
* @return 去掉标记
*/
public String outTag(final String s)

{
placeAll("<.*?>", "");
}

/**
*
* @param s
* @return 获取雅虎知识堂文章标题及内容
*/
public HashMap<String, String> getFromYahoo(final String s)
{
final HashMap<String, String> hm = new HashMap<String, String>();
final StringBuffer sb = new StringBuffer();
String html = "";
System.out.println("/n------------------开始读取网页(" + s + ")--------------------");
try

{
html = getOneHtml(s);
}
catch (final Exception e)
{
e.getMessage();
}
// System.out.println(html);
System.out.println("------------------读取网页(" + s + ")结束--------------------/n");
System.out.println("------------------分析(" + s + ")结果如下--------------------/n");
String title = outTag(getTitle(html));
title = placeAll("_雅虎知识堂", "");
// Pattern pa=Patternpile("<div
// class=/"original/">(.*?)((/r/n)*)(.*?)((/r/n)*)(.*?)</div>",Pattern.DOTALL);
final Pattern pa = Patternpile("<div class=/"original/">(.*?)</p></div>", Pattern.DOTA

java streamLL);
final Matcher ma = pa.matcher(html);
while (ma.find())
{
sb.up());
}
String temp = sb.toString();
temp = placeAll("(<br>)+?", "/n");// 转化换行
temp = placeAll("<p><em>.*?</em></p>", "");// 去图片注释
hm.put("title", title);
hm.put("original", outTag(temp));
return hm;

}

/**

*
* @param args
* 测试一组网页，针对雅虎知识堂
*/
public static void main(final String args[])
{
String url = "";
final List<String> list = new ArrayList<String>();
System.out.print("输入URL，一行一个，输入结束后输入 go 程序开始运行: /n");
/*
* ks.yahoo/question/1307121201133.html
* ks.yahoo/question/1307121101907.html
* ks.yahoo/question/1307121101907_2.html
* ks.yahoo/question/1307121101907_3.html
* ks.yahoo/question/1307121101907_4.html

* ks.yahoo/question/1307121101907_5.html
* ks.yahoo/question/1307121101907_6.html
* ks.yahoo/question/1307121101907_7.html
* ks.yahoo/question/1307121101907_8.html
*/
final BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
try
{
while (!(url = br.readLine()).equals("go"))
{
list.add(url);
}
}
catch (final Exception e)
{

e.getMessage();
}
final WebContent wc = new WebContent();
HashMap<String, String> hm = new HashMap<String, String>();
for (int i = 0; i < list.size(); i++)
{
hm = wc.(i));
System.out.println("标题： " + hm.get("title"));
System.out.println("内容： /n" + hm.get("original"));
}
/*
* String htmlurl[] = {
* "ks.yahoo/question/1307121201133.html",
* "ks.yahoo/question/1307121101907.html",
* "ks.yahoo/question/1307121101907_2.html",

* "ks.yahoo/question/1307121101907_3.html",
* "ks.yahoo/question/1307121101907_4.html",
* "ks.yahoo/question/1307121101907_5.html",
* "ks.yahoo/question/1307121101907_6.html",
* "ks.yahoo/question/1307121101907_7.html",
* "ks.yahoo/question/1307121101907_8.html" }; WebContent
* wc = new WebContent(); HashMap<String, String> hm = new HashMap<String,
* String>(); for (int i = 0; i < htmlurl.length; i++) { hm =
* wc.getFromYahoo(htmlurl[i]); System.out.println("标题： " +
* hm.get("title")); System.out.println("内容： /n" + hm.get("original")); }
*/
/*
* String html=""; String link=""; String sscript=""; String content="";
* System.out.println(htmlurl+" 开始读取网页内容：");
* OneHtml(htmlurl); System.out.println(htmlurl+"

* 读取完毕开始分析……"); placeAll("(<script.*?)((/r/n)*)(.*?)((/r/n)*)(.*?)(</script>)","
* ");//去除脚本 placeAll("(<style.*?)((/r/n)*)(.*?)((/r/n)*)(.*?)(</style>)","
* ");//去掉CSS placeAll("<title>.*?</title>"," ");//除去页面标题
* placeAll("<a[^>]*href=(/"([^/"]*)/"|/'([^/']*)/'|([^//s>]*))[^>]*>(.*?)</a>","
* ");//去掉链接 placeAll("(//s){2,}?"," ");//除去多余空格
* html=wc.outTag(html);//多余标记 System.out.println(html);
*/

/*
* String s[]=html.split(" +"); for(int i=0;i<s.length;i++){
* content=(content.length()>s[i].length())?content:s[i]; }
* System.out.println(content);
*/

// System.out.println(htmlurl+"网页内容结束");
/

*
* System.out.println(htmlurl+"网页脚本开始："); List
* Script(html); for(int i=0;i<script.size();i++){
* System.out.(i)); }
* System.out.println(htmlurl+"网页脚本结束：");
*
* System.out.println(htmlurl+"CSS开始："); List CSS(html);
* for(int i=0;i<css.size();i++){ System.out.(i)); }
* System.out.println(htmlurl+"CSS结束：");
*
* System.out.println(htmlurl+"全部链接内容开始："); List Link(html);
* for(int i=0;i<list.size();i++){ (i).toString(); }
* System.out.println(htmlurl+"全部链接内容结束：");
*
* System.out.println("内容"); System.out.println(wc.outTag(html));

*/
}
}

package com.xiaofeng.picup;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.MalformedURLException;
import java.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import Matcher;
import Pattern;

/** *//**
*

* @author 晓峰2007.1.18 抓取雅虎知识堂的文章标题及内容（测试）手动输入网址抓取，可进一步自动抓取整个知识堂的全部内容
*
*/
public class WebContent ...{
/** *//**
* 读取一个网页全部内容
*/
public String getOneHtml(String htmlurl) throws IOException...{
URL url;
String temp;
StringBuffer sb = new StringBuffer();
try ...{
url = new URL(htmlurl);
BufferedReader in = new BufferedReader(new InputStreamReader(url

.openStream(), "utf-8"));// 读取网页全部内容
while ((temp = in.readLine()) != null) ...{
sb.append(temp);
}
in.close();
}catch(MalformedURLException me)...{
System.out.println("你输入的URL格式有问题！请仔细输入");
me.getMessage();
throw me;
}catch (IOException e) ...{
e.printStackTrace();
throw e;
}
return sb.toString();
}

/** *//**
*
* @param s
* @return 获得网页标题
*/
public String getTitle(String s) ...{
String regex;
String title = "";
List<String> list = new ArrayList<String>();
regex = "<title>.*?</title>";
Pattern pa = Patternpile(regex, Pattern.CANON_EQ);
Matcher ma = pa.matcher(s);
while (ma.find()) ...{
list.up());

}
for (int i = 0; i < list.size(); i++) ...{
title = title + (i);
}
return outTag(title);
}

/** *//**
*
* @param s
* @return 获得链接
*/
public List<String> getLink(String s) ...{
String regex;
List<String> list = new ArrayList<String>();

regex = "<a[^>]*href=("([^"]*)"|'([^']*)'|([^/s>]*))[^>]*>(.*?)</a>";
Pattern pa = Patternpile(regex, Pattern.DOTALL);
Matcher ma = pa.matcher(s);
while (ma.find()) ...{
list.up());
}
return list;
}

/** *//**
*
* @param s
* @return 获得脚本代码
*/
public List<String> getScript(String s) ...{

String regex;
List<String> list = new ArrayList<String>();
regex = "<script.*?</script>";
Pattern pa = Patternpile(regex, Pattern.DOTALL);
Matcher ma = pa.matcher(s);
while (ma.find()) ...{
list.up());
}
return list;
}

/** *//**
*
* @param s
* @return 获得CSS

*/
public List<String> getCSS(String s) ...{
String regex;
List<String> list = new ArrayList<String>();
regex = "<style.*?</style>";
Pattern pa = Patternpile(regex, Pattern.DOTALL);
Matcher ma = pa.matcher(s);
while (ma.find()) ...{
list.up());
}
return list;
}

/** *//**
*

* @param s
* @return 去掉标记
*/
public String outTag(String s) ...{
return s.replaceAll("<.*?>", "");
}

/** *//**
*
* @param s
* @return 获取雅虎知识堂文章标题及内容
*/
public HashMap<String, String> getFromYahoo(String s) ...{
HashMap<String, String> hm = new HashMap<String, String>();
StringBuffer sb = new StringBuffer();

String html="";
System.out.println(" ------------------开始读取网页(" + s
+ ")--------------------");
try...{
html= getOneHtml(s);
}catch(Exception e)...{
e.getMessage();
}
// System.out.println(html);
System.out.println("------------------读取网页(" + s
+ ")结束-------------------- ");
System.out.println("------------------分析(" + s
+ ")结果如下-------------------- ");
String title = outTag(getTitle(html));
title = placeAll("_雅虎知识堂", "");

// Pattern pa=Patternpile("<div
// class="original">(.*?)(( )*)(.*?)(( )*)(.*?)</div>",Pattern.DOTALL);
Pattern pa = Patternpile("<div class="original">(.*?)</p></div>",
Pattern.DOTALL);
Matcher ma = pa.matcher(html);
while (ma.find()) ...{
sb.up());
}
String temp = sb.toString();
temp = placeAll("(<br>)+?", " ");// 转化换行
temp = placeAll("<p><em>.*?</em></p>", "");// 去图片注释
hm.put("title", title);
hm.put("original", outTag(temp));
return hm;

}

/** *//**
*
* @param args
* 测试一组网页，针对雅虎知识堂
*/
public static void main(String args[]) ...{
String url="";
List<String> list = new ArrayList<String>();
System.out.print("输入URL，一行一个，输入结束后输入 go 程序开始运行: ");
/**//*
ks.yahoo/question/1307121201133.html
ks.yahoo/question/1307121101907.html
ks.yahoo/question/1307121101907_2.html

ks.yahoo/question/1307121101907_3.html
ks.yahoo/question/1307121101907_4.html
ks.yahoo/question/1307121101907_5.html
ks.yahoo/question/1307121101907_6.html
ks.yahoo/question/1307121101907_7.html
ks.yahoo/question/1307121101907_8.html
*/
BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
try ...{
while(!(adLine()).equals("go"))...{
list.add(url);
}
} catch (Exception e) ...{
e.getMessage();
}

WebContent wc = new WebContent();
HashMap<String, String> hm = new HashMap<String, String>();
for (int i = 0; i < list.size(); i++) ...{
hm = wc.(i));
System.out.println("标题： " + hm.get("title"));
System.out.println("内容： " + hm.get("original"));
}
/**//*
String htmlurl[] = {
"ks.yahoo/question/1307121201133.html",
"ks.yahoo/question/1307121101907.html",
"ks.yahoo/question/1307121101907_2.html",
"ks.yahoo/question/1307121101907_3.html",
"ks.yahoo/question/1307121101907_4.html",
"ks.yahoo/question/1307121101907_5.html",

"ks.yahoo/question/1307121101907_6.html",
"ks.yahoo/question/1307121101907_7.html",
"ks.yahoo/question/1307121101907_8.html" };
WebContent wc = new WebContent();
HashMap<String, String> hm = new HashMap<String, String>();
for (int i = 0; i < htmlurl.length; i++) {
hm = wc.getFromYahoo(htmlurl[i]);
System.out.println("标题： " + hm.get("title"));
System.out.println("内容： " + hm.get("original"));
}
*/
/**//*
* String html=""; String link=""; String sscript=""; String content="";
* System.out.println(htmlurl+" 开始读取网页内容：");
* OneHtml(htmlurl); System.out.println(htmlurl+"

* 读取完毕开始分析……"); placeAll("(<script.*?)(( )*)(.*?)(( )*)(.*?)(</script>)","
* ");//去除脚本 placeAll("(<style.*?)(( )*)(.*?)(( )*)(.*?)(</style>)","
* ");//去掉CSS placeAll("<title>.*?</title>"," ");//除去页面标题
* placeAll("<a[^>]*href=("([^"]*)"|'([^']*)'|([^/s>]*))[^>]*>(.*?)</a>","
* ");//去掉链接 placeAll("(/s){2,}?"," ");//除去多余空格
* html=wc.outTag(html);//多余标记 System.out.println(html);
*/

/**//*
* String s[]=html.split(" +"); for(int i=0;i<s.length;i++){
* content=(content.length()>s[i].length())?content:s[i]; }
* System.out.println(content);
*/

// System.out.println(htmlurl+"网页内容结束");
/**//*
* System.out.println(htmlurl+"网页脚本开始："); List
* Script(html); for(int i=0;i<script.size();i++){
* System.out.(i)); }
* System.out.println(htmlurl+"网页脚本结束：");
*
* System.out.println(htmlurl+"CSS开始："); List CSS(html);
* for(int i=0;i<css.size();i++){ System.out.(i)); }
* System.out.println(htmlurl+"CSS结束：");
*
* System.out.println(htmlurl+"全部链接内容开始："); List Link(html);
* for(int i=0;i<list.size();i++){ (i).toString(); }
* System.out.println(htmlurl+"全部链接内容结束：");
*

* System.out.println("内容"); System.out.println(wc.outTag(html));
*/
}
}

688IT编程网

java读取(正则表达式分析)网页内容

发表评论

推荐文章

随机森林算法介绍及R语言实现

基于随机森林优化的神经网络算法在冬小麦产量预测中的应用研究_百度文 ...

基于正则化贪心森林算法的情感分析方法研究

随机森林算法和grandientboosting算法

基于随机森林的图像分类算法研究

热门文章

随机森林特征选择原理

自动驾驶系统中的随机森林算法解析

随机森林算法及其在生物信息学中的应用

监督学习中的随机森林算法解析(六)

随机森林算法在数据分析中的应用

机器学习——随机森林,RandomForestClassifier参数含义详解

随机森林的算法

随机森林算法作用

监督学习中的随机森林算法解析(十)

随机森林算法案例

随机森林案例

二分类问题常用的模型

绘制ssd框架训练流程

一种基于信息熵和DTW的多维时间序列相似性度量算法

SVM训练过程范文

如何使用支持向量机进行股票预测与交易分析

二分类交叉熵损失函数binary

tinybert_训练中文文本分类模型_概述说明

基于门控可形变卷积和分层Transformer的图像修复模型及其应用

人工智能开发技术的测试和评估方法

最新文章

基于随机森林的数据分类算法改进

人工智能中的智能识别与分类技术

基于人工智能技术的随机森林算法在医疗数据挖掘中的应用

随机森林回归模型的建模步骤

r语言随机森林预测模型校准曲线

《2024年随机森林算法优化研究》范文

标签列表