Update

src/crawler/HtmlFetcher.java

@@ -0,0 +1,39 @@
+package crawler;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URL;
+
+public class HtmlFetcher {
+    public static String fetchHtmlContent(String urlString) {
+        StringBuilder htmlContent = new StringBuilder();
+        HttpURLConnection connection = null;
+        try {
+            @SuppressWarnings("deprecation")
+            URL url = new URL(urlString);
+            connection = (HttpURLConnection) url.openConnection();
+            connection.setRequestMethod("GET");
+            connection.setRequestProperty("User-Agent",
+                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3");
+            int code = connection.getResponseCode();
+            if (code == 200) {
+                BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), "utf-8"));
+                String line = null;
+                while ((line = reader.readLine()) != null)
+                    htmlContent.append(line);
+                reader.close();
+            }
+        } catch (MalformedURLException e) {
+            e.printStackTrace();
+        } catch (IOException e) {
+            e.printStackTrace();
+        } finally {
+            if (connection != null)
+                connection.disconnect();
+        }
+        return htmlContent.toString();
+    }
+}