diff --git a/src/crawler/MatcherGetter.java b/src/crawler/MatcherGetter.java
deleted file mode 100644
index ad9cbb8..0000000
--- a/src/crawler/MatcherGetter.java
+++ /dev/null
@@ -1,11 +0,0 @@
-package crawler;
-
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class MatcherGetter {
- public static Matcher getMatcher(String regex, String htmlContent) {
- Pattern p = Pattern.compile(regex);
- return p.matcher(htmlContent);
- }
-}
diff --git a/src/crawler/Test.java b/src/crawler/Test.java
index 864c179..b63d4f7 100644
--- a/src/crawler/Test.java
+++ b/src/crawler/Test.java
@@ -2,11 +2,15 @@ package crawler;
import java.io.IOException;
import java.io.PrintWriter;
-import java.util.regex.Matcher;
public class Test {
+ private static final String RANK_MARKER = "";
+ private static final String TITLE_MARKER = "";
+ private static final String DIRECTOR_MARKER = "导演: ";
+ private static final String ACTOR_MARKER = " 主演: ";
+ private static final String RATING_MARKER = "";
+
public static void main(String[] args) {
- String regex = "(\\d+).*?(.*?).*?导演: (.*?) 主演: (.*?)
.*?(\\d{4}) / .*?(.*?)";
StringBuilder htmlContent = new StringBuilder();
htmlContent.append(HtmlFetcher.fetchHtmlContent("https://movie.douban.com/top250"));
htmlContent.append(HtmlFetcher.fetchHtmlContent("https://movie.douban.com/top250?start=25"));
@@ -18,17 +22,33 @@ public class Test {
htmlContent.append(HtmlFetcher.fetchHtmlContent("https://movie.douban.com/top250?start=175"));
htmlContent.append(HtmlFetcher.fetchHtmlContent("https://movie.douban.com/top250?start=200"));
htmlContent.append(HtmlFetcher.fetchHtmlContent("https://movie.douban.com/top250?start=225"));
- Matcher m = MatcherGetter.getMatcher(regex, htmlContent.toString());
StringBuilder movieInfo = new StringBuilder();
movieInfo.append("豆瓣电影 Top 250\n\n");
- while (m.find()) {
- movieInfo.append("排名: ").append(m.group(1)).append("\n");
- movieInfo.append("电影名: ").append(m.group(2)).append("\n");
- movieInfo.append("导演: ").append(m.group(3)).append("\n");
- movieInfo.append("主演: ").append(m.group(4)).append("\n");
- movieInfo.append("年份: ").append(m.group(5)).append("\n");
- movieInfo.append("评分: ").append(m.group(6)).append("\n");
- movieInfo.append("\n");
+ int searchFrom = 0;
+ String html = htmlContent.toString();
+ while (true) {
+ int rankStart = html.indexOf(RANK_MARKER, searchFrom);
+ if (rankStart == -1) {
+ break;
+ }
+ int nextRank = html.indexOf(RANK_MARKER, rankStart + RANK_MARKER.length());
+ String block = html.substring(rankStart, nextRank == -1 ? html.length() : nextRank);
+ String rank = between(block, RANK_MARKER, "");
+ String title = between(block, TITLE_MARKER, "");
+ String director = between(block, DIRECTOR_MARKER, ACTOR_MARKER);
+ String actor = between(block, ACTOR_MARKER, "
");
+ String year = firstYearBefore(block, " / ");
+ String rating = between(block, RATING_MARKER, "");
+ if (!rank.isEmpty() && !title.isEmpty() && !rating.isEmpty()) {
+ movieInfo.append("排名: ").append(rank).append("\n");
+ movieInfo.append("电影名: ").append(title).append("\n");
+ movieInfo.append("导演: ").append(director).append("\n");
+ movieInfo.append("主演: ").append(actor).append("\n");
+ movieInfo.append("年份: ").append(year).append("\n");
+ movieInfo.append("评分: ").append(rating).append("\n");
+ movieInfo.append("\n");
+ }
+ searchFrom = nextRank == -1 ? html.length() : nextRank;
}
try (PrintWriter out = new PrintWriter("crawler\\movieInfo.txt")) {
out.println(movieInfo);
@@ -36,4 +56,31 @@ public class Test {
e.printStackTrace();
}
}
+
+ private static String between(String text, String startMarker, String endMarker) {
+ int start = text.indexOf(startMarker);
+ if (start == -1) {
+ return "";
+ }
+ start += startMarker.length();
+ int end = text.indexOf(endMarker, start);
+ if (end == -1) {
+ return "";
+ }
+ return text.substring(start, end).trim();
+ }
+
+ private static String firstYearBefore(String text, String endMarker) {
+ int end = text.indexOf(endMarker);
+ if (end == -1) {
+ return "";
+ }
+ for (int index = Math.max(0, end - 16); index + 4 <= end; index++) {
+ String candidate = text.substring(index, index + 4);
+ if (candidate.chars().allMatch(Character::isDigit)) {
+ return candidate;
+ }
+ }
+ return "";
+ }
}