灵活的Java爬虫框架

<dependency>
    <groupId>us.codecraft</groupId>
    <artifactId>webmagic-core</artifactId>
    <version>1.0.3</version>
</dependency>
<dependency>
    <groupId>us.codecraft</groupId>
    <artifactId>webmagic-extension</artifactId>
    <version>1.0.3</version>
</dependency>

public class GithubRepoPageProcessor implements PageProcessor {
    private Site site = Site.me().setRetryTimes(3).setSleepTime(100);

    @Override
    public void process(Page page) {
        page.addTargetRequests(page.getHtml().links()
            .regex("(https://github\\.com/\\w+/\\w+)").all());
        page.putField("author", page.getUrl()
            .regex("https://github\\.com/(\\w+)/.*").toString());
        page.putField("name", page.getHtml()
            .xpath("//h1[@class='public']/strong/a/text()").toString());
    }

    @Override
    public Site getSite() {
        return site;
    }
}

Spider.create(new GithubRepoPageProcessor())
    .addUrl("https://github.com/code4craft")
    .addPipeline(new ConsolePipeline())
    .thread(5)
    .run();