Elasticsearch系列(十)----使用webmagic爬取数据导入到ES

webmagic主要有两个文件

一个是对爬取页面进行处理，一个是对页面处理之后的数据进行保存:

CSDNPageProcessor

[html] view plain copy print?

1. package com.fendo.webmagic;  
2.   
3. import java.io.IOException;  
4. import java.net.InetAddress;  
5. import java.net.UnknownHostException;  
6. import java.util.List;  
7.   
8. import com.fasterxml.jackson.core.JsonProcessingException;  
9. import com.fasterxml.jackson.databind.ObjectMapper;  
10.   
11. import org.elasticsearch.action.index.IndexResponse;  
12. import org.elasticsearch.client.transport.TransportClient;  
13. import org.elasticsearch.common.settings.Settings;  
14. import org.elasticsearch.common.transport.InetSocketTransportAddress;  
15. import org.elasticsearch.transport.client.PreBuiltTransportClient;  
16. import org.springframework.beans.factory.annotation.Autowired;  
17. import org.springframework.context.ApplicationContext;  
18. import org.springframework.context.support.FileSystemXmlApplicationContext;  
19.   
20. import com.fendo.common.ClientFactory;  
21. import com.fendo.common.CommonUtils;  
22. import com.fendo.entity.CsdnBlog;  
23.   
24. import io.searchbox.client.JestClient;  
25. import io.searchbox.client.JestResult;  
26. import io.searchbox.indices.CreateIndex;  
27. import us.codecraft.webmagic.Page;  
28. import us.codecraft.webmagic.Site;  
29. import us.codecraft.webmagic.Spider;  
30. import us.codecraft.webmagic.processor.PageProcessor;  
31.   
32. /**  
33.  * CSDN页面爬取  
34.  * @author fendo  
35.  *  
36.  */  
37. //@RunWith(SpringJUnit4ClassRunner.class)  
38. //@WebAppConfiguration  
39. //@ContextConfiguration(locations = {"classpath:applicationContext.xml"})  
40. public class CSDNPageProcessor implements PageProcessor{  
41.   
42.       
43.     @Autowired  
44.     private static JdbcPipeline jdbcPipeline;  
45.       
46.     private static String username="u011781521";  // 设置csdn用户名    
47.       
48.     private static int size = 0;// 共抓取到的文章数量    
49.     
50.     private JestClient jestClient;  
51.           
52.     // 抓取网站的相关配置，包括：编码、抓取间隔、重试次数等        
53.     private Site site = Site.me()  
54.             .setRetryTimes(3)  
55.             .setSleepTime(1000)  
56.             .setUserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.134 Safari/537.36");  
57.   
58.       
59.     @Override  
60.     public Site getSite() {  
61.         return site;  
62.     }  
63.   
64.   
65.       
66.     @Override  
67.     public void process(Page page) {  
68.           
69.         // 列表页    
70.         if (!page.getUrl().regex("http://blog\\.csdn\\.net/" + username + "/article/details/\\d+").match()) {    
71.             // 添加所有文章页    
72.             page.addTargetRequests(page.getHtml().xpath("//div[@id='article_list']").links()// 限定文章列表获取区域    
73.                     .regex("/" + username + "/article/details/\\d+")    
74.                     .replace("/" + username + "/", "http://blog.csdn.net/" + username + "/")// 巧用替换给把相对url转换成绝对url    
75.                     .all());    
76.             // 添加其他列表页    
77.             page.addTargetRequests(page.getHtml().xpath("//div[@id='papelist']").links()// 限定其他列表页获取区域    
78.                     .regex("/" + username + "/article/list/\\d+")    
79.                     .replace("/" + username + "/", "http://blog.csdn.net/" + username + "/")// 巧用替换给把相对url转换成绝对url    
80.                     .all());    
81.             // 文章页    
82.         } else {    
83.             size++;// 文章数量加1    
84.             page.putField("key", Integer.parseInt(page.getUrl().regex("http://blog\\.csdn\\.net/" + username + "/article/details/(\\d+)").get()));  
85.             page.putField("title", CommonUtils.replaceHTML(page.getHtml().xpath("//div[@class= 'article_title']//span[@class='link_title']/a/text()").get()));  
86.             page.putField("content",CommonUtils.replaceHTML(page.getHtml().xpath("//div[@class= 'article_content']/allText()").get()));  
87.             page.putField("dates",page.getHtml().xpath("//div[@class= 'article_r']/span[@class='link_postdate']/text()").get());  
88.             System.out.println("+++++++++++++++date:"+page.getHtml().xpath("//div[@class= 'article_r']/span[@class='link_postdate']/text()").get());  
89.             page.putField("tags",CommonUtils.replaceHTML(listToString(page.getHtml().xpath("//div[@class= 'article_l']/span[@class='link_categories']/a/allText()").all())));  
90.             page.putField("category",CommonUtils.replaceHTML(listToString(page.getHtml().xpath("//div[@class= 'category_r']/label/span/text()").all())));  
91.             page.putField("view", Integer.parseInt(page.getHtml().xpath("//div[@class= 'article_r']/span[@class='link_view']").regex("(\\d+)人阅读").get()));  
92.             page.putField("comments",Integer.parseInt(page.getHtml().xpath("//div[@class= 'article_r']/span[@class='link_comments']").regex("\\((\\d+)\\)").get()));  
93.             page.putField("copyright",page.getHtml().regex("bog_copyright").match() ? 1 : 0);  
94.             page.putField("url", page.getUrl().get());  
95.               
96.               
97.             //创建索引  
98.             ObjectMapper mapper = new ObjectMapper();  
99.   
100.             //创建client  
101.             TransportClient client;  
102.   
103.               
104.             CsdnBlog csdnBlog = new CsdnBlog();  
105.             csdnBlog.setId(size);  
106.             csdnBlog.setTags((String)page.getResultItems().get("tags"));  
107.             csdnBlog.setKeyes((Integer)page.getResultItems().get("key"));  
108.             csdnBlog.setTitles((String)page.getResultItems().get("title"));  
109.             csdnBlog.setDates((String)page.getResultItems().get("dates"));  
110.             csdnBlog.setCategory((String)page.getResultItems().get("category"));  
111.             csdnBlog.setViews((Integer)page.getResultItems().get("view"));  
112.             csdnBlog.setComments((Integer)page.getResultItems().get("comments"));  
113.             csdnBlog.setCopyright((Integer)page.getResultItems().get("copyright"));  
114.             csdnBlog.setContent((String)page.getResultItems().get("content"));  
115.               
116.               
117.             try {  
118.                   
119.                 //设置集群名称  
120.                 Settings settings =  Settings.builder().put("cluster.name", "my-application").build();// 集群名  
121.                 client = new PreBuiltTransportClient(settings)  
122.                         .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("127.0.0.1"), 9300));  
123.                 IndexResponse response =  client.prepareIndex("csdnblog", "article").setSource(mapper.writeValueAsString(csdnBlog)).execute().actionGet();  
124.                 System.out.println(response.toString());  
125.             } catch (Exception e) {  
126.                 e.printStackTrace();  
127.             }  
128.   
129.               
130.             // 把对象输出控制台    
131.             //System.out.println("获取的数据:"+page.toString());   
132.         }    
133.     }  
134.       
135.     // 把list转换为string，用,分割    
136.     public static String listToString(List<String> stringList) {    
137.         if (stringList == null) {    
138.             return null;    
139.         }    
140.         StringBuilder result = new StringBuilder();    
141.         boolean flag = false;    
142.         for (String string : stringList) {    
143.             if (flag) {    
144.                 result.append(",");    
145.             } else {    
146.                 flag = true;    
147.             }    
148.             result.append(string);    
149.         }    
150.         return result.toString();    
151.     }    
152.   
153.       
154.     public static void main(String[] args) {    
155.         long startTime, endTime;    
156.         System.out.println("【爬虫开始】...");    
157.         startTime = System.currentTimeMillis();    
158.           
159.           
160.         //ApplicationContext applicationContext = new ClassPathXmlApplicationContext("applicationContext.xml");  
161.           
162.         ApplicationContext applicationContext = new FileSystemXmlApplicationContext(    
163.                 "classpath:applicationContext.xml");    
164.           
165.         JdbcPipeline jdbcPipeline=(JdbcPipeline)applicationContext.getBean("jdbcPipeline");  
166.           
167.         System.out.println(jdbcPipeline.toString());  
168.           
169.         Spider.create(new CSDNPageProcessor())  
170.         .addUrl("http://blog.csdn.net/u011781521/article/list/1")  
171.         //.addUrl("http://blog.csdn.net/u011781521/article/list/1")  
172.         .addPipeline(jdbcPipeline)  
173.         .thread(5)  
174.         .run();  
175.           
176.         // 从用户博客首页开始抓，开启5个线程，启动爬虫    
177.        // Spider.create(new CsdnBlogPageProcessor()).addUrl("http://blog.csdn.net/" + username).thread(5).run();    
178.           
179.         endTime = System.currentTimeMillis();    
180.         System.out.println("【爬虫结束】共抓取" + size + "篇文章，耗时约" + ((endTime - startTime) / 1000) + "秒，已保存到数据库，请查收！");    
181.     }    
182. }  

package com.fendo.webmagic;

import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.List;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;

import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.FileSystemXmlApplicationContext;

import com.fendo.common.ClientFactory;
import com.fendo.common.CommonUtils;
import com.fendo.entity.CsdnBlog;

import io.searchbox.client.JestClient;
import io.searchbox.client.JestResult;
import io.searchbox.indices.CreateIndex;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

/**
 * CSDN页面爬取
 * @author fendo
 *
 */
//@RunWith(SpringJUnit4ClassRunner.class)
//@WebAppConfiguration
//@ContextConfiguration(locations = {"classpath:applicationContext.xml"})
public class CSDNPageProcessor implements PageProcessor{

	
    @Autowired
    private static JdbcPipeline jdbcPipeline;
	
    private static String username="u011781521";  // 设置csdn用户名  
    
    private static int size = 0;// 共抓取到的文章数量  
  
    private JestClient jestClient;
        
    // 抓取网站的相关配置，包括：编码、抓取间隔、重试次数等      
    private Site site = Site.me()
            .setRetryTimes(3)
            .setSleepTime(1000)
            .setUserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.134 Safari/537.36");

    
	@Override
	public Site getSite() {
		return site;
	}


	
	@Override
	public void process(Page page) {
		
		// 列表页  
        if (!page.getUrl().regex("http://blog\\.csdn\\.net/" + username + "/article/details/\\d+").match()) {  
            // 添加所有文章页  
            page.addTargetRequests(page.getHtml().xpath("//div[@id='article_list']").links()// 限定文章列表获取区域  
                    .regex("/" + username + "/article/details/\\d+")  
                    .replace("/" + username + "/", "http://blog.csdn.net/" + username + "/")// 巧用替换给把相对url转换成绝对url  
                    .all());  
            // 添加其他列表页  
            page.addTargetRequests(page.getHtml().xpath("//div[@id='papelist']").links()// 限定其他列表页获取区域  
                    .regex("/" + username + "/article/list/\\d+")  
                    .replace("/" + username + "/", "http://blog.csdn.net/" + username + "/")// 巧用替换给把相对url转换成绝对url  
                    .all());  
            // 文章页  
        } else {  
            size++;// 文章数量加1  
            page.putField("key", Integer.parseInt(page.getUrl().regex("http://blog\\.csdn\\.net/" + username + "/article/details/(\\d+)").get()));
            page.putField("title", CommonUtils.replaceHTML(page.getHtml().xpath("//div[@class='article_title']//span[@class='link_title']/a/text()").get()));
            page.putField("content",CommonUtils.replaceHTML(page.getHtml().xpath("//div[@class='article_content']/allText()").get()));
            page.putField("dates",page.getHtml().xpath("//div[@class='article_r']/span[@class='link_postdate']/text()").get());
            System.out.println("+++++++++++++++date:"+page.getHtml().xpath("//div[@class='article_r']/span[@class='link_postdate']/text()").get());
            page.putField("tags",CommonUtils.replaceHTML(listToString(page.getHtml().xpath("//div[@class='article_l']/span[@class='link_categories']/a/allText()").all())));
            page.putField("category",CommonUtils.replaceHTML(listToString(page.getHtml().xpath("//div[@class='category_r']/label/span/text()").all())));
            page.putField("view", Integer.parseInt(page.getHtml().xpath("//div[@class='article_r']/span[@class='link_view']").regex("(\\d+)人阅读").get()));
            page.putField("comments",Integer.parseInt(page.getHtml().xpath("//div[@class='article_r']/span[@class='link_comments']").regex("\\((\\d+)\\)").get()));
            page.putField("copyright",page.getHtml().regex("bog_copyright").match() ? 1 : 0);
            page.putField("url", page.getUrl().get());
            
            
            //创建索引
            ObjectMapper mapper = new ObjectMapper();

            //创建client
            TransportClient client;

            
            CsdnBlog csdnBlog = new CsdnBlog();
            csdnBlog.setId(size);
            csdnBlog.setTags((String)page.getResultItems().get("tags"));
            csdnBlog.setKeyes((Integer)page.getResultItems().get("key"));
            csdnBlog.setTitles((String)page.getResultItems().get("title"));
            csdnBlog.setDates((String)page.getResultItems().get("dates"));
            csdnBlog.setCategory((String)page.getResultItems().get("category"));
            csdnBlog.setViews((Integer)page.getResultItems().get("view"));
            csdnBlog.setComments((Integer)page.getResultItems().get("comments"));
            csdnBlog.setCopyright((Integer)page.getResultItems().get("copyright"));
            csdnBlog.setContent((String)page.getResultItems().get("content"));
            
            
    		try {
                
                //设置集群名称
                Settings settings = Settings.builder().put("cluster.name", "my-application").build();// 集群名
				client = new PreBuiltTransportClient(settings)
				        .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("127.0.0.1"), 9300));
				IndexResponse response = client.prepareIndex("csdnblog", "article").setSource(mapper.writeValueAsString(csdnBlog)).execute().actionGet();
				System.out.println(response.toString());
			} catch (Exception e) {
				e.printStackTrace();
			}

            
            // 把对象输出控制台  
            //System.out.println("获取的数据:"+page.toString()); 
        }  
	}
	
    // 把list转换为string，用,分割  
    public static String listToString(List<String> stringList) {  
        if (stringList == null) {  
            return null;  
        }  
        StringBuilder result = new StringBuilder();  
        boolean flag = false;  
        for (String string : stringList) {  
            if (flag) {  
                result.append(",");  
            } else {  
                flag = true;  
            }  
            result.append(string);  
        }  
        return result.toString();  
    }  

    
    public static void main(String[] args) {  
        long startTime, endTime;  
        System.out.println("【爬虫开始】...");  
        startTime = System.currentTimeMillis();  
        
        
        //ApplicationContext applicationContext = new ClassPathXmlApplicationContext("applicationContext.xml");
        
        ApplicationContext applicationContext = new FileSystemXmlApplicationContext(  
                "classpath:applicationContext.xml");  
        
        JdbcPipeline jdbcPipeline=(JdbcPipeline)applicationContext.getBean("jdbcPipeline");
        
        System.out.println(jdbcPipeline.toString());
        
        Spider.create(new CSDNPageProcessor())
        .addUrl("http://blog.csdn.net/u011781521/article/list/1")
        //.addUrl("http://blog.csdn.net/u011781521/article/list/1")
        .addPipeline(jdbcPipeline)
        .thread(5)
        .run();
        
        // 从用户博客首页开始抓，开启5个线程，启动爬虫  
       // Spider.create(new CsdnBlogPageProcessor()).addUrl("http://blog.csdn.net/" + username).thread(5).run();  
        
        endTime = System.currentTimeMillis();  
        System.out.println("【爬虫结束】共抓取" + size + "篇文章，耗时约" + ((endTime - startTime) / 1000) + "秒，已保存到数据库，请查收！");  
    }  
}

注意:

在上面的代码中，不但通过jdbcPipeline保存了数据，还通过TransportClient 往ES中保存了数据!!

JdbcPipeline

[html] view plain copy print?

1. package com.fendo.webmagic;  
2.   
3. import java.util.Map;  
4.   
5. import org.springframework.beans.factory.annotation.Autowired;  
6. import org.springframework.stereotype.Component;  
7. import org.springframework.stereotype.Service;  
8.   
9. import com.fendo.entity.CsdnBlog;  
10. import com.fendo.mapper.CsdnBlogMapper;  
11.   
12. import us.codecraft.webmagic.ResultItems;  
13. import us.codecraft.webmagic.Task;  
14. import us.codecraft.webmagic.pipeline.Pipeline;  
15.   
16. @Component("jdbcPipeline")  
17. public class JdbcPipeline implements Pipeline{  
18.   
19.       
20.     @Autowired  
21.     CsdnBlogMapper csdnBlogMapper;  
22.       
23.     @Override  
24.     public void process(ResultItems resultItems, Task task) {  
25.         Map<String,Object> items =  resultItems.getAll();  
26.         if(resultItems!=null&&resultItems.getAll().size()>0){  
27.             CsdnBlog csdnBlog = new CsdnBlog();  
28.             csdnBlog.setTags((String)items.get("tags"));  
29.             csdnBlog.setKeyes((Integer)items.get("key"));  
30.             csdnBlog.setTitles((String)items.get("title"));  
31.             csdnBlog.setDates((String)items.get("dates"));  
32.             csdnBlog.setCategory((String)items.get("category"));  
33.             csdnBlog.setViews((Integer)items.get("view"));  
34.             csdnBlog.setComments((Integer)items.get("comments"));  
35.             csdnBlog.setCopyright((Integer)items.get("copyright"));  
36.             csdnBlog.setContent((String)items.get("content"));  
37.             System.out.println("-----------------------------------------------------------------------process:"+csdnBlog.toString());  
38.             csdnBlogMapper.insert(csdnBlog);  
39.     }  
40.   }  
41.   
42. }  

package com.fendo.webmagic;

import java.util.Map;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.stereotype.Service;

import com.fendo.entity.CsdnBlog;
import com.fendo.mapper.CsdnBlogMapper;

import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.pipeline.Pipeline;

@Component("jdbcPipeline")
public class JdbcPipeline implements Pipeline{

	
	@Autowired
	CsdnBlogMapper csdnBlogMapper;
	
	@Override
	public void process(ResultItems resultItems, Task task) {
		Map<String,Object> items = resultItems.getAll();
        if(resultItems!=null&&resultItems.getAll().size()>0){
            CsdnBlog csdnBlog = new CsdnBlog();
            csdnBlog.setTags((String)items.get("tags"));
            csdnBlog.setKeyes((Integer)items.get("key"));
            csdnBlog.setTitles((String)items.get("title"));
            csdnBlog.setDates((String)items.get("dates"));
            csdnBlog.setCategory((String)items.get("category"));
            csdnBlog.setViews((Integer)items.get("view"));
            csdnBlog.setComments((Integer)items.get("comments"));
            csdnBlog.setCopyright((Integer)items.get("copyright"));
            csdnBlog.setContent((String)items.get("content"));
            System.out.println("-----------------------------------------------------------------------process:"+csdnBlog.toString());
            csdnBlogMapper.insert(csdnBlog);
	}
  }

}

对应的数据库脚本:

[html] view plain copy print?

1. CREATE TABLE `csdnblog` (  
2.   `id` int(11) unsigned NOT NULL AUTO_INCREMENT,  
3.   `keyes` int(11) unsigned NOT NULL,  
4.   `titles` varchar(255) NOT NULL,  
5.   `content` varchar(10240) NOT NULL,  
6.   `dates` varchar(255) DEFAULT NULL,  
7.   `tags` varchar(255) DEFAULT NULL,  
8.   `category` varchar(255) DEFAULT NULL,  
9.   `views` int(11) unsigned DEFAULT NULL,  
10.   `comments` int(11) unsigned DEFAULT NULL,  
11.   `copyright` int(20) unsigned DEFAULT NULL,  
12.   `url` varchar(255) DEFAULT NULL,  
13.   PRIMARY KEY (`id`)  
14. ) ENGINE=InnoDB AUTO_INCREMENT=3301 DEFAULT  CHARSET=utf8;  

CREATE TABLE `csdnblog` (
  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
  `keyes` int(11) unsigned NOT NULL,
  `titles` varchar(255) NOT NULL,
  `content` varchar(10240) NOT NULL,
  `dates` varchar(255) DEFAULT NULL,
  `tags` varchar(255) DEFAULT NULL,
  `category` varchar(255) DEFAULT NULL,
  `views` int(11) unsigned DEFAULT NULL,
  `comments` int(11) unsigned DEFAULT NULL,
  `copyright` int(20) unsigned DEFAULT NULL,
  `url` varchar(255) DEFAULT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=3301 DEFAULT CHARSET=utf8;

完整项目: download.csdn.net/download/u0…