fix: address code scanning alerts
This commit is contained in:
1 parent
71447fc642
commit
9fffb557c3
2 files changed
+57
-21
No files matched your search
@@ -1,11 +0,0 @@
|
|||||||
package crawler;
|
|
||||||
|
|
||||||
import java.util.regex.Matcher;
|
|
||||||
import java.util.regex.Pattern;
|
|
||||||
|
|
||||||
public class MatcherGetter {
|
|
||||||
public static Matcher getMatcher(String regex, String htmlContent) {
|
|
||||||
Pattern p = Pattern.compile(regex);
|
|
||||||
return p.matcher(htmlContent);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+57
-10
@@ -2,11 +2,15 @@ package crawler;
|
|||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.PrintWriter;
|
import java.io.PrintWriter;
|
||||||
import java.util.regex.Matcher;
|
|
||||||
|
|
||||||
public class Test {
|
public class Test {
|
||||||
|
private static final String RANK_MARKER = "<em class=\"\">";
|
||||||
|
private static final String TITLE_MARKER = "<span class=\"title\">";
|
||||||
|
private static final String DIRECTOR_MARKER = "导演: ";
|
||||||
|
private static final String ACTOR_MARKER = " 主演: ";
|
||||||
|
private static final String RATING_MARKER = "<span class=\"rating_num\" property=\"v:average\">";
|
||||||
|
|
||||||
public static void main(String[] args) {
|
public static void main(String[] args) {
|
||||||
String regex = "<em class=\"\">(\\d+)</em>.*?<span class=\"title\">(.*?)</span>.*?导演: (.*?) 主演: (.*?)<br>.*?(\\d{4}) / .*?<span class=\"rating_num\" property=\"v:average\">(.*?)</span>";
|
|
||||||
StringBuilder htmlContent = new StringBuilder();
|
StringBuilder htmlContent = new StringBuilder();
|
||||||
htmlContent.append(HtmlFetcher.fetchHtmlContent("https://movie.douban.com/top250"));
|
htmlContent.append(HtmlFetcher.fetchHtmlContent("https://movie.douban.com/top250"));
|
||||||
htmlContent.append(HtmlFetcher.fetchHtmlContent("https://movie.douban.com/top250?start=25"));
|
htmlContent.append(HtmlFetcher.fetchHtmlContent("https://movie.douban.com/top250?start=25"));
|
||||||
@@ -18,22 +22,65 @@ public class Test {
|
|||||||
htmlContent.append(HtmlFetcher.fetchHtmlContent("https://movie.douban.com/top250?start=175"));
|
htmlContent.append(HtmlFetcher.fetchHtmlContent("https://movie.douban.com/top250?start=175"));
|
||||||
htmlContent.append(HtmlFetcher.fetchHtmlContent("https://movie.douban.com/top250?start=200"));
|
htmlContent.append(HtmlFetcher.fetchHtmlContent("https://movie.douban.com/top250?start=200"));
|
||||||
htmlContent.append(HtmlFetcher.fetchHtmlContent("https://movie.douban.com/top250?start=225"));
|
htmlContent.append(HtmlFetcher.fetchHtmlContent("https://movie.douban.com/top250?start=225"));
|
||||||
Matcher m = MatcherGetter.getMatcher(regex, htmlContent.toString());
|
|
||||||
StringBuilder movieInfo = new StringBuilder();
|
StringBuilder movieInfo = new StringBuilder();
|
||||||
movieInfo.append("豆瓣电影 Top 250\n\n");
|
movieInfo.append("豆瓣电影 Top 250\n\n");
|
||||||
while (m.find()) {
|
int searchFrom = 0;
|
||||||
movieInfo.append("排名: ").append(m.group(1)).append("\n");
|
String html = htmlContent.toString();
|
||||||
movieInfo.append("电影名: ").append(m.group(2)).append("\n");
|
while (true) {
|
||||||
movieInfo.append("导演: ").append(m.group(3)).append("\n");
|
int rankStart = html.indexOf(RANK_MARKER, searchFrom);
|
||||||
movieInfo.append("主演: ").append(m.group(4)).append("\n");
|
if (rankStart == -1) {
|
||||||
movieInfo.append("年份: ").append(m.group(5)).append("\n");
|
break;
|
||||||
movieInfo.append("评分: ").append(m.group(6)).append("\n");
|
}
|
||||||
|
int nextRank = html.indexOf(RANK_MARKER, rankStart + RANK_MARKER.length());
|
||||||
|
String block = html.substring(rankStart, nextRank == -1 ? html.length() : nextRank);
|
||||||
|
String rank = between(block, RANK_MARKER, "</em>");
|
||||||
|
String title = between(block, TITLE_MARKER, "</span>");
|
||||||
|
String director = between(block, DIRECTOR_MARKER, ACTOR_MARKER);
|
||||||
|
String actor = between(block, ACTOR_MARKER, "<br>");
|
||||||
|
String year = firstYearBefore(block, " / ");
|
||||||
|
String rating = between(block, RATING_MARKER, "</span>");
|
||||||
|
if (!rank.isEmpty() && !title.isEmpty() && !rating.isEmpty()) {
|
||||||
|
movieInfo.append("排名: ").append(rank).append("\n");
|
||||||
|
movieInfo.append("电影名: ").append(title).append("\n");
|
||||||
|
movieInfo.append("导演: ").append(director).append("\n");
|
||||||
|
movieInfo.append("主演: ").append(actor).append("\n");
|
||||||
|
movieInfo.append("年份: ").append(year).append("\n");
|
||||||
|
movieInfo.append("评分: ").append(rating).append("\n");
|
||||||
movieInfo.append("\n");
|
movieInfo.append("\n");
|
||||||
}
|
}
|
||||||
|
searchFrom = nextRank == -1 ? html.length() : nextRank;
|
||||||
|
}
|
||||||
try (PrintWriter out = new PrintWriter("crawler\\movieInfo.txt")) {
|
try (PrintWriter out = new PrintWriter("crawler\\movieInfo.txt")) {
|
||||||
out.println(movieInfo);
|
out.println(movieInfo);
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static String between(String text, String startMarker, String endMarker) {
|
||||||
|
int start = text.indexOf(startMarker);
|
||||||
|
if (start == -1) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
start += startMarker.length();
|
||||||
|
int end = text.indexOf(endMarker, start);
|
||||||
|
if (end == -1) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
return text.substring(start, end).trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String firstYearBefore(String text, String endMarker) {
|
||||||
|
int end = text.indexOf(endMarker);
|
||||||
|
if (end == -1) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
for (int index = Math.max(0, end - 16); index + 4 <= end; index++) {
|
||||||
|
String candidate = text.substring(index, index + 4);
|
||||||
|
if (candidate.chars().allMatch(Character::isDigit)) {
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "";
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user