diff --git a/src/crawler/MatcherGetter.java b/src/crawler/MatcherGetter.java deleted file mode 100644 index ad9cbb8..0000000 --- a/src/crawler/MatcherGetter.java +++ /dev/null @@ -1,11 +0,0 @@ -package crawler; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class MatcherGetter { - public static Matcher getMatcher(String regex, String htmlContent) { - Pattern p = Pattern.compile(regex); - return p.matcher(htmlContent); - } -} diff --git a/src/crawler/Test.java b/src/crawler/Test.java index 864c179..b63d4f7 100644 --- a/src/crawler/Test.java +++ b/src/crawler/Test.java @@ -2,11 +2,15 @@ package crawler; import java.io.IOException; import java.io.PrintWriter; -import java.util.regex.Matcher; public class Test { + private static final String RANK_MARKER = ""; + private static final String TITLE_MARKER = ""; + private static final String DIRECTOR_MARKER = "导演: "; + private static final String ACTOR_MARKER = "   主演: "; + private static final String RATING_MARKER = ""; + public static void main(String[] args) { - String regex = "(\\d+).*?(.*?).*?导演: (.*?)   主演: (.*?)
.*?(\\d{4}) / .*?(.*?)"; StringBuilder htmlContent = new StringBuilder(); htmlContent.append(HtmlFetcher.fetchHtmlContent("https://movie.douban.com/top250")); htmlContent.append(HtmlFetcher.fetchHtmlContent("https://movie.douban.com/top250?start=25")); @@ -18,17 +22,33 @@ public class Test { htmlContent.append(HtmlFetcher.fetchHtmlContent("https://movie.douban.com/top250?start=175")); htmlContent.append(HtmlFetcher.fetchHtmlContent("https://movie.douban.com/top250?start=200")); htmlContent.append(HtmlFetcher.fetchHtmlContent("https://movie.douban.com/top250?start=225")); - Matcher m = MatcherGetter.getMatcher(regex, htmlContent.toString()); StringBuilder movieInfo = new StringBuilder(); movieInfo.append("豆瓣电影 Top 250\n\n"); - while (m.find()) { - movieInfo.append("排名: ").append(m.group(1)).append("\n"); - movieInfo.append("电影名: ").append(m.group(2)).append("\n"); - movieInfo.append("导演: ").append(m.group(3)).append("\n"); - movieInfo.append("主演: ").append(m.group(4)).append("\n"); - movieInfo.append("年份: ").append(m.group(5)).append("\n"); - movieInfo.append("评分: ").append(m.group(6)).append("\n"); - movieInfo.append("\n"); + int searchFrom = 0; + String html = htmlContent.toString(); + while (true) { + int rankStart = html.indexOf(RANK_MARKER, searchFrom); + if (rankStart == -1) { + break; + } + int nextRank = html.indexOf(RANK_MARKER, rankStart + RANK_MARKER.length()); + String block = html.substring(rankStart, nextRank == -1 ? html.length() : nextRank); + String rank = between(block, RANK_MARKER, "
"); + String title = between(block, TITLE_MARKER, ""); + String director = between(block, DIRECTOR_MARKER, ACTOR_MARKER); + String actor = between(block, ACTOR_MARKER, "
"); + String year = firstYearBefore(block, " / "); + String rating = between(block, RATING_MARKER, ""); + if (!rank.isEmpty() && !title.isEmpty() && !rating.isEmpty()) { + movieInfo.append("排名: ").append(rank).append("\n"); + movieInfo.append("电影名: ").append(title).append("\n"); + movieInfo.append("导演: ").append(director).append("\n"); + movieInfo.append("主演: ").append(actor).append("\n"); + movieInfo.append("年份: ").append(year).append("\n"); + movieInfo.append("评分: ").append(rating).append("\n"); + movieInfo.append("\n"); + } + searchFrom = nextRank == -1 ? html.length() : nextRank; } try (PrintWriter out = new PrintWriter("crawler\\movieInfo.txt")) { out.println(movieInfo); @@ -36,4 +56,31 @@ public class Test { e.printStackTrace(); } } + + private static String between(String text, String startMarker, String endMarker) { + int start = text.indexOf(startMarker); + if (start == -1) { + return ""; + } + start += startMarker.length(); + int end = text.indexOf(endMarker, start); + if (end == -1) { + return ""; + } + return text.substring(start, end).trim(); + } + + private static String firstYearBefore(String text, String endMarker) { + int end = text.indexOf(endMarker); + if (end == -1) { + return ""; + } + for (int index = Math.max(0, end - 16); index + 4 <= end; index++) { + String candidate = text.substring(index, index + 4); + if (candidate.chars().allMatch(Character::isDigit)) { + return candidate; + } + } + return ""; + } }