JAVA怎么弄爬蟲(chóng)

　以下是一個(gè)使用java實(shí)現(xiàn)的簡(jiǎn)單爬蟲(chóng)核心代碼：

創(chuàng)新互聯(lián)公司專(zhuān)注于湘潭縣網(wǎng)站建設(shè)服務(wù)及定制，我們擁有豐富的企業(yè)做網(wǎng)站經(jīng)驗(yàn)。 熱誠(chéng)為您提供湘潭縣營(yíng)銷(xiāo)型網(wǎng)站建設(shè)，湘潭縣網(wǎng)站制作、湘潭縣網(wǎng)頁(yè)設(shè)計(jì)、湘潭縣網(wǎng)站官網(wǎng)定制、小程序開(kāi)發(fā)服務(wù)，打造湘潭縣網(wǎng)絡(luò)公司原創(chuàng)品牌,更為您提供湘潭縣網(wǎng)站排名全網(wǎng)營(yíng)銷(xiāo)落地服務(wù)。

public void crawl() throws Throwable {

while (continueCrawling()) {

CrawlerUrl url = getNextUrl(); //獲取待爬取隊(duì)列中的下一個(gè)URL

if (url != null) {

printCrawlInfo();

String content = getContent(url); //獲取URL的文本信息

//聚焦爬蟲(chóng)只爬取與主題內(nèi)容相關(guān)的網(wǎng)頁(yè)，這里采用正則匹配簡(jiǎn)單處理

if (isContentRelevant(content, this.regexpSearchPattern)) {

saveContent(url, content); //保存網(wǎng)頁(yè)至本地

//獲取網(wǎng)頁(yè)內(nèi)容中的鏈接，并放入待爬取隊(duì)列中

Collection urlStrings = extractUrls(content, url);

addUrlsToUrlQueue(url, urlStrings);

} else {

System.out.println(url + " is not relevant ignoring ...");

}

//延時(shí)防止被對(duì)方屏蔽

Thread.sleep(this.delayBetweenUrls);

}

}

closeOutputStream();

}

private CrawlerUrl getNextUrl() throws Throwable {

CrawlerUrl nextUrl = null;

while ((nextUrl == null) (!urlQueue.isEmpty())) {

CrawlerUrl crawlerUrl = this.urlQueue.remove();

//doWeHavePermissionToVisit：是否有權(quán)限訪問(wèn)該URL，友好的爬蟲(chóng)會(huì)根據(jù)網(wǎng)站提供的"Robot.txt"中配置的規(guī)則進(jìn)行爬取

//isUrlAlreadyVisited：URL是否訪問(wèn)過(guò)，大型的搜索引擎往往采用BloomFilter進(jìn)行排重，這里簡(jiǎn)單使用HashMap

//isDepthAcceptable：是否達(dá)到指定的深度上限。爬蟲(chóng)一般采取廣度優(yōu)先的方式。一些網(wǎng)站會(huì)構(gòu)建爬蟲(chóng)陷阱（自動(dòng)生成一些無(wú)效鏈接使爬蟲(chóng)陷入死循環(huán)），采用深度限制加以避免

if (doWeHavePermissionToVisit(crawlerUrl)

(!isUrlAlreadyVisited(crawlerUrl))

isDepthAcceptable(crawlerUrl)) {

nextUrl = crawlerUrl;

// System.out.println("Next url to be visited is " + nextUrl);

}

}

return nextUrl;

}

private String getContent(CrawlerUrl url) throws Throwable {

//HttpClient4.1的調(diào)用與之前的方式不同

HttpClient client = new DefaultHttpClient();

HttpGet httpGet = new HttpGet(url.getUrlString());

StringBuffer strBuf = new StringBuffer();

HttpResponse response = client.execute(httpGet);

if (HttpStatus.SC_OK == response.getStatusLine().getStatusCode()) {

HttpEntity entity = response.getEntity();

if (entity != null) {

BufferedReader reader = new BufferedReader(

new InputStreamReader(entity.getContent(), "UTF-8"));

String line = null;

if (entity.getContentLength() 0) {

strBuf = new StringBuffer((int) entity.getContentLength());

while ((line = reader.readLine()) != null) {

strBuf.append(line);

}

}

}

if (entity != null) {

nsumeContent();

}

}

//將url標(biāo)記為已訪問(wèn)

markUrlAsVisited(url);

return strBuf.toString();

}

public static boolean isContentRelevant(String content,

Pattern regexpPattern) {

boolean retValue = false;

if (content != null) {

//是否符合正則表達(dá)式的條件

Matcher m = regexpPattern.matcher(content.toLowerCase());

retValue = m.find();

}

return retValue;

}

public List extractUrls(String text, CrawlerUrl crawlerUrl) {

Map urlMap = new HashMap();

extractHttpUrls(urlMap, text);

extractRelativeUrls(urlMap, text, crawlerUrl);

return new ArrayList(urlMap.keySet());

}

private void extractHttpUrls(Map urlMap, String text) {

Matcher m = (text);

while (m.find()) {

String url = m.group();

String[] terms = url.split("a href=\"");

for (String term : terms) {

// System.out.println("Term = " + term);

if (term.startsWith("http")) {

int index = term.indexOf("\"");

if (index 0) {

term = term.substring(0, index);

}

urlMap.put(term, term);

System.out.println("Hyperlink: " + term);

}

}

}

}

private void extractRelativeUrls(Map urlMap, String text,

CrawlerUrl crawlerUrl) {

Matcher m = relativeRegexp.matcher(text);

URL textURL = crawlerUrl.getURL();

String host = textURL.getHost();

while (m.find()) {

String url = m.group();

String[] terms = url.split("a href=\"");

for (String term : terms) {

if (term.startsWith("/")) {

int index = term.indexOf("\"");

if (index 0) {

term = term.substring(0, index);

}

String s = //" + host + term;

urlMap.put(s, s);

System.out.println("Relative url: " + s);

}

}

}

}

public static void main(String[] args) {

try {

String url = "";

Queue urlQueue = new LinkedList();

String regexp = "java";

urlQueue.add(new CrawlerUrl(url, 0));

NaiveCrawler crawler = new NaiveCrawler(urlQueue, 100, 5, 1000L,

regexp);

// boolean allowCrawl = crawler.areWeAllowedToVisit(url);

// System.out.println("Allowed to crawl: " + url + " " +

// allowCrawl);

crawler.crawl();

} catch (Throwable t) {

System.out.println(t.toString());

t.printStackTrace();

}

}

java 網(wǎng)絡(luò)爬蟲(chóng)怎么實(shí)現(xiàn)

網(wǎng)絡(luò)爬蟲(chóng)是一個(gè)自動(dòng)提取網(wǎng)頁(yè)的程序，它為搜索引擎從萬(wàn)維網(wǎng)上下載網(wǎng)頁(yè)，是搜索引擎的重要組成。

傳統(tǒng)爬蟲(chóng)從一個(gè)或若干初始網(wǎng)頁(yè)的URL開(kāi)始，獲得初始網(wǎng)頁(yè)上的URL，在抓取網(wǎng)頁(yè)的過(guò)程中，不斷從當(dāng)前頁(yè)面上抽取新的URL放入隊(duì)列，直到滿(mǎn)足系統(tǒng)的一定停止條件。對(duì)于垂直搜索來(lái)說(shuō)，聚焦爬蟲(chóng)，即有針對(duì)性地爬取特定主題網(wǎng)頁(yè)的爬蟲(chóng)，更為適合。

以下是一個(gè)使用java實(shí)現(xiàn)的簡(jiǎn)單爬蟲(chóng)核心代碼：

public void crawl() throws Throwable {

while (continueCrawling()) {

CrawlerUrl url = getNextUrl(); //獲取待爬取隊(duì)列中的下一個(gè)URL

if (url != null) {

printCrawlInfo();

String content = getContent(url); //獲取URL的文本信息

//聚焦爬蟲(chóng)只爬取與主題內(nèi)容相關(guān)的網(wǎng)頁(yè)，這里采用正則匹配簡(jiǎn)單處理

if (isContentRelevant(content, this.regexpSearchPattern)) {

saveContent(url, content); //保存網(wǎng)頁(yè)至本地

//獲取網(wǎng)頁(yè)內(nèi)容中的鏈接，并放入待爬取隊(duì)列中

Collection urlStrings = extractUrls(content, url);

addUrlsToUrlQueue(url, urlStrings);

} else {

System.out.println(url + " is not relevant ignoring ...");

}

//延時(shí)防止被對(duì)方屏蔽

Thread.sleep(this.delayBetweenUrls);

}

}

closeOutputStream();

}

private CrawlerUrl getNextUrl() throws Throwable {

CrawlerUrl nextUrl = null;

while ((nextUrl == null) (!urlQueue.isEmpty())) {

CrawlerUrl crawlerUrl = this.urlQueue.remove();

//doWeHavePermissionToVisit：是否有權(quán)限訪問(wèn)該URL，友好的爬蟲(chóng)會(huì)根據(jù)網(wǎng)站提供的"Robot.txt"中配置的規(guī)則進(jìn)行爬取

//isUrlAlreadyVisited：URL是否訪問(wèn)過(guò)，大型的搜索引擎往往采用BloomFilter進(jìn)行排重，這里簡(jiǎn)單使用HashMap

//isDepthAcceptable：是否達(dá)到指定的深度上限。爬蟲(chóng)一般采取廣度優(yōu)先的方式。一些網(wǎng)站會(huì)構(gòu)建爬蟲(chóng)陷阱（自動(dòng)生成一些無(wú)效鏈接使爬蟲(chóng)陷入死循環(huán)），采用深度限制加以避免

if (doWeHavePermissionToVisit(crawlerUrl)

(!isUrlAlreadyVisited(crawlerUrl))

isDepthAcceptable(crawlerUrl)) {

nextUrl = crawlerUrl;

// System.out.println("Next url to be visited is " + nextUrl);

}

}

return nextUrl;

}

private String getContent(CrawlerUrl url) throws Throwable {

//HttpClient4.1的調(diào)用與之前的方式不同

HttpClient client = new DefaultHttpClient();

HttpGet httpGet = new HttpGet(url.getUrlString());

StringBuffer strBuf = new StringBuffer();

HttpResponse response = client.execute(httpGet);

if (HttpStatus.SC_OK == response.getStatusLine().getStatusCode()) {

HttpEntity entity = response.getEntity();

if (entity != null) {

BufferedReader reader = new BufferedReader(

new InputStreamReader(entity.getContent(), "UTF-8"));

String line = null;

if (entity.getContentLength() 0) {

strBuf = new StringBuffer((int) entity.getContentLength());

while ((line = reader.readLine()) != null) {

strBuf.append(line);

}

}

}

if (entity != null) {

nsumeContent();

}

}

//將url標(biāo)記為已訪問(wèn)

markUrlAsVisited(url);

return strBuf.toString();

}

public static boolean isContentRelevant(String content,

Pattern regexpPattern) {

boolean retValue = false;

if (content != null) {

//是否符合正則表達(dá)式的條件

Matcher m = regexpPattern.matcher(content.toLowerCase());

retValue = m.find();

}

return retValue;

}

public List extractUrls(String text, CrawlerUrl crawlerUrl) {

Map urlMap = new HashMap();

extractHttpUrls(urlMap, text);

extractRelativeUrls(urlMap, text, crawlerUrl);

return new ArrayList(urlMap.keySet());

}

private void extractHttpUrls(Map urlMap, String text) {

Matcher m = (text);

while (m.find()) {

String url = m.group();

String[] terms = url.split("a href=\"");

for (String term : terms) {

// System.out.println("Term = " + term);

if (term.startsWith("http")) {

int index = term.indexOf("\"");

if (index 0) {

term = term.substring(0, index);

}

urlMap.put(term, term);

System.out.println("Hyperlink: " + term);

}

}

}

}

private void extractRelativeUrls(Map urlMap, String text,

CrawlerUrl crawlerUrl) {

Matcher m = relativeRegexp.matcher(text);

URL textURL = crawlerUrl.getURL();

String host = textURL.getHost();

while (m.find()) {

String url = m.group();

String[] terms = url.split("a href=\"");

for (String term : terms) {

if (term.startsWith("/")) {

int index = term.indexOf("\"");

if (index 0) {

term = term.substring(0, index);

}

String s = //" + host + term;

urlMap.put(s, s);

System.out.println("Relative url: " + s);

}

}

}

}

public static void main(String[] args) {

try {

String url = "";

Queue urlQueue = new LinkedList();

String regexp = "java";

urlQueue.add(new CrawlerUrl(url, 0));

NaiveCrawler crawler = new NaiveCrawler(urlQueue, 100, 5, 1000L,

regexp);

// boolean allowCrawl = crawler.areWeAllowedToVisit(url);

// System.out.println("Allowed to crawl: " + url + " " +

// allowCrawl);

crawler.crawl();

} catch (Throwable t) {

System.out.println(t.toString());

t.printStackTrace();

}

}

用java編寫(xiě) 網(wǎng)絡(luò)爬蟲(chóng)求代碼和流程 急

import java.awt.*;

import java.awt.event.*;

import java.io.*;

import java.net.*;

import java.util.*;

import java.util.regex.*;

import javax.swing.*;

import javax.swing.table.*;//一個(gè)Web的爬行者(注：爬行在這里的意思與抓取，捕獲相同)

public class SearchCrawler extends JFrame{

//最大URL保存值

private static final String[] MAX_URLS={"50","100","500","1000"};

//緩存robot禁止爬行列表

private HashMap disallowListCache=new HashMap();

//搜索GUI控件

private JTextField startTextField;

private JComboBox maxComboBox;

private JCheckBox limitCheckBox;

private JTextField logTextField;

private JTextField searchTextField;

private JCheckBox caseCheckBox;

private JButton searchButton;

//搜索狀態(tài)GUI控件

private JLabel crawlingLabel2;

private JLabel crawledLabel2;

private JLabel toCrawlLabel2;

private JProgressBar progressBar;

private JLabel matchesLabel2;

//搜索匹配項(xiàng)表格列表

private JTable table;

//標(biāo)記爬行機(jī)器是否正在爬行

private boolean crawling;

//寫(xiě)日志匹配文件的引用

private PrintWriter logFileWriter;

//網(wǎng)絡(luò)爬行者的構(gòu)造函數(shù)

public SearchCrawler(){

//設(shè)置應(yīng)用程序標(biāo)題欄

setTitle("搜索爬行者");

//設(shè)置窗體大小

setSize(600,600);

//處理窗體關(guān)閉事件

addWindowListener(new WindowAdapter(){

public void windowClosing(WindowEvent e){

actionExit();

}

});

//設(shè)置文件菜單

JMenuBar menuBar=new JMenuBar();

JMenu fileMenu=new JMenu("文件");

fileMenu.setMnemonic(KeyEvent.VK_F);

JMenuItem fileExitMenuItem=new JMenuItem("退出",KeyEvent.VK_X);

fileExitMenuItem.addActionListener(new ActionListener(){

public void actionPerformed(ActionEvent e){

actionExit();

}

});

fileMenu.add(fileExitMenuItem);

menuBar.add(fileMenu);

setJMenuBar(menuBar);

網(wǎng)頁(yè)標(biāo)題：java爬蟲(chóng)軟件源代碼 爬蟲(chóng)代碼下載
轉(zhuǎn)載源于：http://chinadenli.net/article22/hpejjc.html

成都網(wǎng)站建設(shè)公司_創(chuàng)新互聯(lián)，為您提供品牌網(wǎng)站制作、做網(wǎng)站、靜態(tài)網(wǎng)站、App開(kāi)發(fā)、微信小程序、外貿(mào)建站

廣告

聲明：本網(wǎng)站發(fā)布的內(nèi)容（圖片、視頻和文字）以用戶(hù)投稿、用戶(hù)轉(zhuǎn)載內(nèi)容為主，如果涉及侵權(quán)請(qǐng)盡快告知，我們將會(huì)在第一時(shí)間刪除。文章觀點(diǎn)不代表本網(wǎng)站立場(chǎng)，如需處理請(qǐng)聯(lián)系客服。電話(huà)：028-86922220；郵箱：631063699@qq.com。內(nèi)容未經(jīng)允許不得轉(zhuǎn)載，或轉(zhuǎn)載時(shí)需注明來(lái)源： 創(chuàng)新互聯(lián)