根据WebMagic写的一个爬取煎蛋网的小爬虫

//处理图片类
   private void processPicture(Page page) {
   //得到所有Gif的li标签

       List<String> gifLists = page.getHtml().xpath("//ol[@class='commentlist']/li[@id]").all();
       for (String gif:gifLists){
           //得到标题
           String title=xpath("//div[@class='author']/strong").selectElement(gif).attr("title");
           logger.info("title:"+title);
           //得到上传者
           String author=xpath("//div[@class='author']/strong").selectElement(gif).text();
           //将标题中的防伪码转换为：上传者名称
           title=title.replace("防伪码",author);
           //图片url
           //如果有org_src属性，则是gif图片
           String url=xpath("//div[@class='text']/p/img").selectElement(gif).attr("src");
           String gifUrl=xpath("//div[@class='text']/p/img").selectElement(gif).attr("org_src");
           if(StringUtils.isNotEmpty(gifUrl)){
               logger.info("Gif图片...替换新链接...");
               url=gifUrl;//如果是gif则用大图链接替换缩略图链接
           }
           logger.info("图片url:" + url);
           //保存图片到本地
           String filePath=downloadDir+ File.separator+author;
           String picType=url.substring(url.length()-3);
           try {
               FileUtil.downloadFile(url,filePath,title, picType);
           } catch (Exception e) {
               e.printStackTrace();
           }
       }
   }

package com.zeusjava.jandan.util;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.log4j.Logger;

import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;

/**
 * Created by LittleXuan on 2015/8/11.
 * 文件操作工具类
 */
public class FileUtil {
    private static Logger logger = Logger.getLogger(FileUtil.class);
    /**
     * 下载文件
     *
     * @param url
     *            文件http地址
     * @param filePath
     *            目标文件路径
     * @param fileName
     *            目标文件
     * @param picType
     *            文件类型
     * @throws java.io.IOException
     */
    public static synchronized void downloadFile(String url,String filePath, String fileName,String picType)
            throws Exception {
        logger.info("----------------------下载文件开始---------------------");
        CloseableHttpClient httpclient = HttpClients.createDefault();
        if (url == null || "".equals(url)) {
            return;
        }
        //目标目录
        File desPathFile = new File(filePath);
        if (!desPathFile.exists()) {
            desPathFile.mkdirs();
        }
        //得到文件绝对路径
        String fullPath =filePath +File.separator+fileName+"."+picType;
        logger.info("文件路径："+filePath);
        logger.info("文件名："+fileName);
        logger.info("源文件url："+url);
        //从元网址下载图片
        HttpGet httpget = new HttpGet(url);
        HttpResponse response = httpclient.execute(httpget);
        HttpEntity entity = response.getEntity();
        InputStream in = entity.getContent();
        //设置下载地址
        File file = new File(fullPath);

        try {
            FileOutputStream fout = new FileOutputStream(file);
            int l = -1;
            byte[] tmp = new byte[1024];
            while ((l = in.read(tmp)) != -1) {
                fout.write(tmp,0,l);
            }
            fout.flush();
            fout.close();
        } finally {

            in.close();
        }
        logger.info("----------------------下载文件结束---------------------");
    }
}

@Override
   public void process(Page page) {
       System.out.println("================================");
       //定义抽取信息，并保存信息
       processPicture(page);
       //得到下一页链接
       String comments=page.getHtml().xpath("//a[@class='previous-comment-page']").toString();
       logger.info("comments:"+comments);
       String link = xpath("a/@href").select(comments);
       logger.info("link:" + link);
       Request request = new Request(link);
       page.addTargetRequest(request);
       System.out.println("================================");

   }

//得到网站配置
    private Site site = Site.me().setDomain("jandan.net").addHeader("Accept",
            "application/x-ms-application, image/jpeg, application/xaml+xml, image/gif, image/pjpeg, application/x-ms-xbap, */*")
            .addHeader("Referer", "http://jandan.net/pic").setSleepTime(10000).setUserAgent("zhaohongxuan")
            .addStartUrl("http://jandan.net/pic");

public class JanDanSpiderTest {
    private static Logger logger = Logger.getLogger(JanDanSpiderTest.class);

    public static void main(String[] args) {
        PropertyConfigurator.configure(ClassLoader.getSystemResourceAsStream("log4j.properties"));
        Spider.create(new PicProcessor()).scheduler(new PriorityScheduler()).run();
    }
}

Spider.create(new PicProcessor()).scheduler(new PriorityScheduler()).thread(10).run();