40 lines
1.5 KiB
Java
40 lines
1.5 KiB
Java
package crawler;
|
|
|
|
import java.io.BufferedReader;
|
|
import java.io.IOException;
|
|
import java.io.InputStreamReader;
|
|
import java.net.HttpURLConnection;
|
|
import java.net.MalformedURLException;
|
|
import java.net.URL;
|
|
|
|
public class HtmlFetcher {
|
|
public static String fetchHtmlContent(String urlString) {
|
|
StringBuilder htmlContent = new StringBuilder();
|
|
HttpURLConnection connection = null;
|
|
try {
|
|
@SuppressWarnings("deprecation")
|
|
URL url = new URL(urlString);
|
|
connection = (HttpURLConnection) url.openConnection();
|
|
connection.setRequestMethod("GET");
|
|
connection.setRequestProperty("User-Agent",
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3");
|
|
int code = connection.getResponseCode();
|
|
if (code == 200) {
|
|
BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), "utf-8"));
|
|
String line = null;
|
|
while ((line = reader.readLine()) != null)
|
|
htmlContent.append(line);
|
|
reader.close();
|
|
}
|
|
} catch (MalformedURLException e) {
|
|
e.printStackTrace();
|
|
} catch (IOException e) {
|
|
e.printStackTrace();
|
|
} finally {
|
|
if (connection != null)
|
|
connection.disconnect();
|
|
}
|
|
return htmlContent.toString();
|
|
}
|
|
}
|