java爬蟲軟件源代碼 爬蟲代碼下載
JAVA怎么弄爬蟲
以下是一個使用java實現(xiàn)的簡單爬蟲核心代碼：
創(chuàng)新互聯(lián)公司專注于湘潭縣網(wǎng)站建設(shè)服務(wù)及定制，我們擁有豐富的企業(yè)做網(wǎng)站經(jīng)驗。 熱誠為您提供湘潭縣營銷型網(wǎng)站建設(shè)，湘潭縣網(wǎng)站制作、湘潭縣網(wǎng)頁設(shè)計、湘潭縣網(wǎng)站官網(wǎng)定制、小程序開發(fā)服務(wù)，打造湘潭縣網(wǎng)絡(luò)公司原創(chuàng)品牌,更為您提供湘潭縣網(wǎng)站排名全網(wǎng)營銷落地服務(wù)。
public void crawl() throws Throwable {
while (continueCrawling()) {
CrawlerUrl url = getNextUrl(); //獲取待爬取隊列中的下一個URL
if (url != null) {
printCrawlInfo();
String content = getContent(url); //獲取URL的文本信息
//聚焦爬蟲只爬取與主題內(nèi)容相關(guān)的網(wǎng)頁，這里采用正則匹配簡單處理
if (isContentRelevant(content, this.regexpSearchPattern)) {
saveContent(url, content); //保存網(wǎng)頁至本地
//獲取網(wǎng)頁內(nèi)容中的鏈接，并放入待爬取隊列中
Collection urlStrings = extractUrls(content, url);
addUrlsToUrlQueue(url, urlStrings);
} else {
System.out.println(url + " is not relevant ignoring ...");
}
//延時防止被對方屏蔽
Thread.sleep(this.delayBetweenUrls);
}
}
closeOutputStream();
}
private CrawlerUrl getNextUrl() throws Throwable {
CrawlerUrl nextUrl = null;
while ((nextUrl == null) (!urlQueue.isEmpty())) {
CrawlerUrl crawlerUrl = this.urlQueue.remove();
//doWeHavePermissionToVisit：是否有權(quán)限訪問該URL，友好的爬蟲會根據(jù)網(wǎng)站提供的"Robot.txt"中配置的規(guī)則進(jìn)行爬取
//isUrlAlreadyVisited：URL是否訪問過，大型的搜索引擎往往采用BloomFilter進(jìn)行排重，這里簡單使用HashMap
//isDepthAcceptable：是否達(dá)到指定的深度上限。爬蟲一般采取廣度優(yōu)先的方式。一些網(wǎng)站會構(gòu)建爬蟲陷阱（自動生成一些無效鏈接使爬蟲陷入死循環(huán)），采用深度限制加以避免
if (doWeHavePermissionToVisit(crawlerUrl)
(!isUrlAlreadyVisited(crawlerUrl))
isDepthAcceptable(crawlerUrl)) {
nextUrl = crawlerUrl;
// System.out.println("Next url to be visited is " + nextUrl);
}
}
return nextUrl;
}
private String getContent(CrawlerUrl url) throws Throwable {
//HttpClient4.1的調(diào)用與之前的方式不同
HttpClient client = new DefaultHttpClient();
HttpGet httpGet = new HttpGet(url.getUrlString());
StringBuffer strBuf = new StringBuffer();
HttpResponse response = client.execute(httpGet);
if (HttpStatus.SC_OK == response.getStatusLine().getStatusCode()) {
HttpEntity entity = response.getEntity();
if (entity != null) {
BufferedReader reader = new BufferedReader(
new InputStreamReader(entity.getContent(), "UTF-8"));
String line = null;
if (entity.getContentLength() 0) {
strBuf = new StringBuffer((int) entity.getContentLength());
while ((line = reader.readLine()) != null) {
strBuf.append(line);
}
}
}
if (entity != null) {
nsumeContent();
}
}
//將url標(biāo)記為已訪問
markUrlAsVisited(url);
return strBuf.toString();
}
public static boolean isContentRelevant(String content,
Pattern regexpPattern) {
boolean retValue = false;
if (content != null) {
//是否符合正則表達(dá)式的條件
Matcher m = regexpPattern.matcher(content.toLowerCase());
retValue = m.find();
}
return retValue;
}
public List extractUrls(String text, CrawlerUrl crawlerUrl) {
Map urlMap = new HashMap();
extractHttpUrls(urlMap, text);
extractRelativeUrls(urlMap, text, crawlerUrl);
return new ArrayList(urlMap.keySet());
}
private void extractHttpUrls(Map urlMap, String text) {
Matcher m = (text);
while (m.find()) {
String url = m.group();
String[] terms = url.split("a href=\"");
for (String term : terms) {
// System.out.println("Term = " + term);
if (term.startsWith("http")) {
int index = term.indexOf("\"");
if (index 0) {
term = term.substring(0, index);
}
urlMap.put(term, term);
System.out.println("Hyperlink: " + term);
}
}
}
}
private void extractRelativeUrls(Map urlMap, String text,
CrawlerUrl crawlerUrl) {
Matcher m = relativeRegexp.matcher(text);
URL textURL = crawlerUrl.getURL();
String host = textURL.getHost();
while (m.find()) {
String url = m.group();
String[] terms = url.split("a href=\"");
for (String term : terms) {
if (term.startsWith("/")) {
int index = term.indexOf("\"");
if (index 0) {
term = term.substring(0, index);
}
String s = //" + host + term;
urlMap.put(s, s);
System.out.println("Relative url: " + s);
}
}
}
}
public static void main(String[] args) {
try {
String url = "";
Queue urlQueue = new LinkedList();
String regexp = "java";
urlQueue.add(new CrawlerUrl(url, 0));
NaiveCrawler crawler = new NaiveCrawler(urlQueue, 100, 5, 1000L,
regexp);
// boolean allowCrawl = crawler.areWeAllowedToVisit(url);
// System.out.println("Allowed to crawl: " + url + " " +
// allowCrawl);
crawler.crawl();
} catch (Throwable t) {
System.out.println(t.toString());
t.printStackTrace();
}
}
java 網(wǎng)絡(luò)爬蟲怎么實現(xiàn)
網(wǎng)絡(luò)爬蟲是一個自動提取網(wǎng)頁的程序，它為搜索引擎從萬維網(wǎng)上下載網(wǎng)頁，是搜索引擎的重要組成。
傳統(tǒng)爬蟲從一個或若干初始網(wǎng)頁的URL開始，獲得初始網(wǎng)頁上的URL，在抓取網(wǎng)頁的過程中，不斷從當(dāng)前頁面上抽取新的URL放入隊列，直到滿足系統(tǒng)的一定停止條件。對于垂直搜索來說，聚焦爬蟲，即有針對性地爬取特定主題網(wǎng)頁的爬蟲，更為適合。
以下是一個使用java實現(xiàn)的簡單爬蟲核心代碼：
public void crawl() throws Throwable {
while (continueCrawling()) {
CrawlerUrl url = getNextUrl(); //獲取待爬取隊列中的下一個URL
if (url != null) {
printCrawlInfo();
String content = getContent(url); //獲取URL的文本信息
//聚焦爬蟲只爬取與主題內(nèi)容相關(guān)的網(wǎng)頁，這里采用正則匹配簡單處理
if (isContentRelevant(content, this.regexpSearchPattern)) {
saveContent(url, content); //保存網(wǎng)頁至本地
//獲取網(wǎng)頁內(nèi)容中的鏈接，并放入待爬取隊列中
Collection urlStrings = extractUrls(content, url);
addUrlsToUrlQueue(url, urlStrings);
} else {
System.out.println(url + " is not relevant ignoring ...");
}
//延時防止被對方屏蔽
Thread.sleep(this.delayBetweenUrls);
}
}
closeOutputStream();
}
private CrawlerUrl getNextUrl() throws Throwable {
CrawlerUrl nextUrl = null;
while ((nextUrl == null) (!urlQueue.isEmpty())) {
CrawlerUrl crawlerUrl = this.urlQueue.remove();
//doWeHavePermissionToVisit：是否有權(quán)限訪問該URL，友好的爬蟲會根據(jù)網(wǎng)站提供的"Robot.txt"中配置的規(guī)則進(jìn)行爬取
//isUrlAlreadyVisited：URL是否訪問過，大型的搜索引擎往往采用BloomFilter進(jìn)行排重，這里簡單使用HashMap
//isDepthAcceptable：是否達(dá)到指定的深度上限。爬蟲一般采取廣度優(yōu)先的方式。一些網(wǎng)站會構(gòu)建爬蟲陷阱（自動生成一些無效鏈接使爬蟲陷入死循環(huán)），采用深度限制加以避免
if (doWeHavePermissionToVisit(crawlerUrl)
(!isUrlAlreadyVisited(crawlerUrl))
isDepthAcceptable(crawlerUrl)) {
nextUrl = crawlerUrl;
// System.out.println("Next url to be visited is " + nextUrl);
}
}
return nextUrl;
}
private String getContent(CrawlerUrl url) throws Throwable {
//HttpClient4.1的調(diào)用與之前的方式不同
HttpClient client = new DefaultHttpClient();
HttpGet httpGet = new HttpGet(url.getUrlString());
StringBuffer strBuf = new StringBuffer();
HttpResponse response = client.execute(httpGet);
if (HttpStatus.SC_OK == response.getStatusLine().getStatusCode()) {
HttpEntity entity = response.getEntity();
if (entity != null) {
BufferedReader reader = new BufferedReader(
new InputStreamReader(entity.getContent(), "UTF-8"));
String line = null;
if (entity.getContentLength() 0) {
strBuf = new StringBuffer((int) entity.getContentLength());
while ((line = reader.readLine()) != null) {
strBuf.append(line);
}
}
}
if (entity != null) {
nsumeContent();
}
}
//將url標(biāo)記為已訪問
markUrlAsVisited(url);
return strBuf.toString();
}
public static boolean isContentRelevant(String content,
Pattern regexpPattern) {
boolean retValue = false;
if (content != null) {
//是否符合正則表達(dá)式的條件
Matcher m = regexpPattern.matcher(content.toLowerCase());
retValue = m.find();
}
return retValue;
}
public List extractUrls(String text, CrawlerUrl crawlerUrl) {
Map urlMap = new HashMap();
extractHttpUrls(urlMap, text);
extractRelativeUrls(urlMap, text, crawlerUrl);
return new ArrayList(urlMap.keySet());
}
private void extractHttpUrls(Map urlMap, String text) {
Matcher m = (text);
while (m.find()) {
String url = m.group();
String[] terms = url.split("a href=\"");
for (String term : terms) {
// System.out.println("Term = " + term);
if (term.startsWith("http")) {
int index = term.indexOf("\"");
if (index 0) {
term = term.substring(0, index);
}
urlMap.put(term, term);
System.out.println("Hyperlink: " + term);
}
}
}
}
private void extractRelativeUrls(Map urlMap, String text,
CrawlerUrl crawlerUrl) {
Matcher m = relativeRegexp.matcher(text);
URL textURL = crawlerUrl.getURL();
String host = textURL.getHost();
while (m.find()) {
String url = m.group();
String[] terms = url.split("a href=\"");
for (String term : terms) {
if (term.startsWith("/")) {
int index = term.indexOf("\"");
if (index 0) {
term = term.substring(0, index);
}
String s = //" + host + term;
urlMap.put(s, s);
System.out.println("Relative url: " + s);
}
}
}
}
public static void main(String[] args) {
try {
String url = "";
Queue urlQueue = new LinkedList();
String regexp = "java";
urlQueue.add(new CrawlerUrl(url, 0));
NaiveCrawler crawler = new NaiveCrawler(urlQueue, 100, 5, 1000L,
regexp);
// boolean allowCrawl = crawler.areWeAllowedToVisit(url);
// System.out.println("Allowed to crawl: " + url + " " +
// allowCrawl);
crawler.crawl();
} catch (Throwable t) {
System.out.println(t.toString());
t.printStackTrace();
}
}
用java編寫 網(wǎng)絡(luò)爬蟲求代碼和流程 急
import java.awt.*;
import java.awt.event.*;
import java.io.*;
import java.net.*;
import java.util.*;
import java.util.regex.*;
import javax.swing.*;
import javax.swing.table.*;//一個Web的爬行者(注：爬行在這里的意思與抓取，捕獲相同)
public class SearchCrawler extends JFrame{
//最大URL保存值
private static final String[] MAX_URLS={"50","100","500","1000"};
//緩存robot禁止爬行列表
private HashMap disallowListCache=new HashMap();
//搜索GUI控件
private JTextField startTextField;
private JComboBox maxComboBox;
private JCheckBox limitCheckBox;
private JTextField logTextField;
private JTextField searchTextField;
private JCheckBox caseCheckBox;
private JButton searchButton;
//搜索狀態(tài)GUI控件
private JLabel crawlingLabel2;
private JLabel crawledLabel2;
private JLabel toCrawlLabel2;
private JProgressBar progressBar;
private JLabel matchesLabel2;
//搜索匹配項表格列表
private JTable table;
//標(biāo)記爬行機(jī)器是否正在爬行
private boolean crawling;
//寫日志匹配文件的引用
private PrintWriter logFileWriter;
//網(wǎng)絡(luò)爬行者的構(gòu)造函數(shù)
public SearchCrawler(){
//設(shè)置應(yīng)用程序標(biāo)題欄
setTitle("搜索爬行者");
//設(shè)置窗體大小
setSize(600,600);
//處理窗體關(guān)閉事件
addWindowListener(new WindowAdapter(){
public void windowClosing(WindowEvent e){
actionExit();
}
});
//設(shè)置文件菜單
JMenuBar menuBar=new JMenuBar();
JMenu fileMenu=new JMenu("文件");
fileMenu.setMnemonic(KeyEvent.VK_F);
JMenuItem fileExitMenuItem=new JMenuItem("退出",KeyEvent.VK_X);
fileExitMenuItem.addActionListener(new ActionListener(){
public void actionPerformed(ActionEvent e){
actionExit();
}
});
fileMenu.add(fileExitMenuItem);
menuBar.add(fileMenu);
setJMenuBar(menuBar);
網(wǎng)頁標(biāo)題：java爬蟲軟件源代碼 爬蟲代碼下載
轉(zhuǎn)載源于：http://chinadenli.net/article22/hpejjc.html
成都網(wǎng)站建設(shè)公司_創(chuàng)新互聯(lián)，為您提供品牌網(wǎng)站制作、做網(wǎng)站、靜態(tài)網(wǎng)站、App開發(fā)、微信小程序、外貿(mào)建站
聲明：本網(wǎng)站發(fā)布的內(nèi)容（圖片、視頻和文字）以用戶投稿、用戶轉(zhuǎn)載內(nèi)容為主，如果涉及侵權(quán)請盡快告知，我們將會在第一時間刪除。文章觀點不代表本網(wǎng)站立場，如需處理請聯(lián)系客服。電話：028-86922220；郵箱：631063699@qq.com。內(nèi)容未經(jīng)允許不得轉(zhuǎn)載，或轉(zhuǎn)載時需注明來源： 創(chuàng)新互聯(lián)
- 網(wǎng)站制作和優(yōu)化時都存在哪些誤區(qū)？ 2016-11-09
- 成都網(wǎng)站建設(shè)使用虛擬主機(jī)存在哪些缺點？ 2017-01-11
- 成都企業(yè)網(wǎng)站制作需要多長時間_成都網(wǎng)站建設(shè)創(chuàng)新互聯(lián)科技 2021-11-16
- [成都網(wǎng)站制作]如何挖掘網(wǎng)絡(luò)推廣之軟文寫作 2023-02-02
- 營銷型網(wǎng)站制作需要考慮的3個因素 2022-05-01
- 網(wǎng)站制作中，如何提升網(wǎng)站排名 2021-12-10
- 網(wǎng)站制作哪家好？選擇網(wǎng)絡(luò)公司的四大技巧 2022-08-27
- 提高網(wǎng)站用戶體驗的幾個方法 2016-04-27
- 網(wǎng)站制作中的細(xì)節(jié)不可忽視 2016-10-31
- 五大方法搞定網(wǎng)站制作中頁面加載速度問題! 2015-08-23
- 域名注冊要注意哪些問題？ 2016-06-21
- 成都網(wǎng)站建設(shè)：一個好的H5響應(yīng)式網(wǎng)站有哪些標(biāo)準(zhǔn)？ 2017-01-05