a
This commit is contained in:
1 parent
8d25963b92
commit
9a5f8c7a90
12 files changed
+230
-268
No files matched your search
@@ -79,10 +79,10 @@ public class Client {
|
|||||||
update();
|
update();
|
||||||
break;
|
break;
|
||||||
case 6:
|
case 6:
|
||||||
ExportToCSV.to();
|
ExportToCSV.to("output.csv", "select * from book");
|
||||||
break;
|
break;
|
||||||
case 7:
|
case 7:
|
||||||
ExportToXLS.to();
|
ExportToXLS.to("select * from book", "output.xls");
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
break;
|
break;
|
||||||
|
|||||||
+12
-8
@@ -6,14 +6,18 @@ import java.net.Socket;
|
|||||||
import server.thread.HandleThread;
|
import server.thread.HandleThread;
|
||||||
|
|
||||||
public class Server {
|
public class Server {
|
||||||
public static void main(String[] args) throws Exception {
|
private static final int PORT = 9999;
|
||||||
int port = 9999;
|
|
||||||
ServerSocket server = new ServerSocket(port);
|
public static void main(String[] args) {
|
||||||
System.out.println("The server is listening for client connection requests on port " + port + ".");
|
try (ServerSocket server = new ServerSocket(PORT)) {
|
||||||
while (true) {
|
System.out.println("Server listening on port " + PORT);
|
||||||
Socket socket = server.accept();
|
while (true) {
|
||||||
System.out.println("Client " + socket.getInetAddress().getHostAddress() + " successfully connected!");
|
Socket clientSocket = server.accept();
|
||||||
new HandleThread(socket).start();
|
System.out.println("Client connected: " + clientSocket.getInetAddress().getHostAddress());
|
||||||
|
new Thread(new HandleThread(clientSocket)).start();
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.err.println("Server exception: " + e.getMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+25
-27
@@ -10,38 +10,36 @@ import server.vo.User;
|
|||||||
|
|
||||||
public class UserDAO {
|
public class UserDAO {
|
||||||
public static User get(String userName) throws Exception {
|
public static User get(String userName) throws Exception {
|
||||||
User user = null;
|
String sql = "SELECT * FROM user WHERE userName=?";
|
||||||
Connection con = DBConnection.getConnection();
|
try (Connection con = DBConnection.getConnection();
|
||||||
String sql = "select * from user where userName=?";
|
PreparedStatement pst = con.prepareStatement(sql)) {
|
||||||
PreparedStatement pst = con.prepareStatement(sql);
|
pst.setString(1, userName);
|
||||||
pst.setString(1, userName);
|
try (ResultSet rs = pst.executeQuery()) {
|
||||||
ResultSet rs = pst.executeQuery();
|
if (rs.next())
|
||||||
while (rs.next()) {
|
return createUserFromResultSet(rs);
|
||||||
user = new User();
|
}
|
||||||
user.setUserName(rs.getString("userName"));
|
|
||||||
user.setPassword(rs.getString("password"));
|
|
||||||
user.setName(rs.getString("name"));
|
|
||||||
user.setRole(rs.getString("role"));
|
|
||||||
}
|
}
|
||||||
DBConnection.close(con, pst);
|
return null;
|
||||||
return user;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static ArrayList<User> query(User userCondition) throws Exception {
|
public static ArrayList<User> query(User userCondition) throws Exception {
|
||||||
ArrayList<User> list = new ArrayList<User>();
|
ArrayList<User> list = new ArrayList<>();
|
||||||
Connection con = DBConnection.getConnection();
|
String sql = "SELECT * FROM user"; // 示例SQL,实际应根据userCondition调整
|
||||||
String sql = "*******";
|
try (Connection con = DBConnection.getConnection();
|
||||||
PreparedStatement pst = con.prepareStatement(sql);
|
PreparedStatement pst = con.prepareStatement(sql);
|
||||||
ResultSet rs = pst.executeQuery();
|
ResultSet rs = pst.executeQuery()) {
|
||||||
while (rs.next()) {
|
while (rs.next())
|
||||||
User user = new User();
|
list.add(createUserFromResultSet(rs));
|
||||||
user.setUserName(rs.getString("userName"));
|
|
||||||
user.setPassword(rs.getString("password"));
|
|
||||||
user.setName(rs.getString("name"));
|
|
||||||
user.setRole(rs.getString("role"));
|
|
||||||
list.add(user);
|
|
||||||
}
|
}
|
||||||
DBConnection.close(con, pst);
|
|
||||||
return list;
|
return list;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static User createUserFromResultSet(ResultSet rs) throws Exception {
|
||||||
|
User user = new User();
|
||||||
|
user.setUserName(rs.getString("userName"));
|
||||||
|
user.setPassword(rs.getString("password"));
|
||||||
|
user.setName(rs.getString("name"));
|
||||||
|
user.setRole(rs.getString("role"));
|
||||||
|
return user;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -1,9 +1,7 @@
|
|||||||
package server.thread;
|
package server.thread;
|
||||||
|
|
||||||
import java.io.BufferedReader;
|
import java.io.BufferedReader;
|
||||||
import java.io.InputStream;
|
|
||||||
import java.io.InputStreamReader;
|
import java.io.InputStreamReader;
|
||||||
import java.io.OutputStream;
|
|
||||||
import java.io.OutputStreamWriter;
|
import java.io.OutputStreamWriter;
|
||||||
import java.io.PrintWriter;
|
import java.io.PrintWriter;
|
||||||
import java.net.Socket;
|
import java.net.Socket;
|
||||||
@@ -11,98 +9,81 @@ import java.util.HashMap;
|
|||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
import com.google.gson.Gson;
|
import com.google.gson.Gson;
|
||||||
import com.google.gson.JsonElement;
|
|
||||||
import com.google.gson.JsonObject;
|
import com.google.gson.JsonObject;
|
||||||
import com.google.gson.JsonParser;
|
|
||||||
|
|
||||||
import server.dao.UserDAO;
|
import server.dao.UserDAO;
|
||||||
import server.vo.User;
|
import server.vo.User;
|
||||||
|
|
||||||
public class HandleThread extends Thread {
|
public class HandleThread extends Thread {
|
||||||
private Socket socket;
|
private Socket socket;
|
||||||
private BufferedReader buf;
|
|
||||||
private PrintWriter write;
|
|
||||||
|
|
||||||
public HandleThread(Socket socket) {
|
public HandleThread(Socket socket) {
|
||||||
super();
|
|
||||||
this.socket = socket;
|
this.socket = socket;
|
||||||
try {
|
|
||||||
InputStream is = socket.getInputStream();
|
|
||||||
buf = new BufferedReader(new InputStreamReader(is, "utf-8"));
|
|
||||||
OutputStream os = socket.getOutputStream();
|
|
||||||
write = new PrintWriter(new OutputStreamWriter(os, "utf-8"));
|
|
||||||
} catch (Exception e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
while (true) {
|
try (BufferedReader buf = new BufferedReader(new InputStreamReader(socket.getInputStream(), "utf-8"));
|
||||||
try {
|
PrintWriter write = new PrintWriter(new OutputStreamWriter(socket.getOutputStream(), "utf-8"))) {
|
||||||
String receiveStr = buf.readLine();
|
String receiveStr;
|
||||||
|
while ((receiveStr = buf.readLine()) != null) {
|
||||||
System.out.println("Receive the client's request data: " + receiveStr);
|
System.out.println("Receive the client's request data: " + receiveStr);
|
||||||
JsonElement element = JsonParser.parseString(receiveStr);
|
JsonObject obj = new Gson().fromJson(receiveStr, JsonObject.class);
|
||||||
JsonObject obj = element.getAsJsonObject();
|
|
||||||
int code = obj.get("code").getAsInt();
|
int code = obj.get("code").getAsInt();
|
||||||
if (code == 8) {
|
if (code == 8)
|
||||||
close();
|
|
||||||
break;
|
break;
|
||||||
}
|
|
||||||
String data = obj.get("data").getAsString();
|
String data = obj.get("data").getAsString();
|
||||||
String response = "";
|
String response = handleRequest(code, data);
|
||||||
switch (code) {
|
|
||||||
case 0:
|
|
||||||
response = loginHandle(data);
|
|
||||||
break;
|
|
||||||
case 1:
|
|
||||||
break;
|
|
||||||
case 2:
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
System.out.println("Invalid command!");
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
write.println(response);
|
write.println(response);
|
||||||
write.flush();
|
write.flush();
|
||||||
} catch (Exception e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
} finally {
|
||||||
|
System.out.println("Client exited!");
|
||||||
|
closeResources();
|
||||||
}
|
}
|
||||||
System.out.println("Client exited!");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void close() throws Exception {
|
private String handleRequest(int code, String data) throws Exception {
|
||||||
if (write != null)
|
switch (code) {
|
||||||
write.close();
|
case 0:
|
||||||
if (buf != null)
|
return loginHandle(data);
|
||||||
buf.close();
|
case 1:
|
||||||
if (socket != null)
|
// Handle other cases
|
||||||
socket.close();
|
return "";
|
||||||
|
case 2:
|
||||||
|
// Handle other cases
|
||||||
|
return "";
|
||||||
|
default:
|
||||||
|
System.out.println("Invalid command!");
|
||||||
|
return "";
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public String loginHandle(String str) throws Exception {
|
private String loginHandle(String str) throws Exception {
|
||||||
User loginUser = new Gson().fromJson(str, User.class);
|
User loginUser = new Gson().fromJson(str, User.class);
|
||||||
User user = UserDAO.get(loginUser.getUserName());
|
User user = UserDAO.get(loginUser.getUserName());
|
||||||
String responseData = "";
|
Map<String, Object> map = new HashMap<>();
|
||||||
if (user == null) {
|
if (user == null) {
|
||||||
Map<String, Object> map = new HashMap<>();
|
|
||||||
map.put("code", 0);
|
map.put("code", 0);
|
||||||
map.put("data", "Username does not exist!");
|
map.put("data", "Username does not exist!");
|
||||||
responseData = new Gson().toJson(map);
|
} else if (loginUser.getPassword().equals(user.getPassword())) {
|
||||||
|
map.put("code", 1);
|
||||||
|
map.put("data", new Gson().toJson(user));
|
||||||
} else {
|
} else {
|
||||||
if (loginUser.getPassword().equals(user.getPassword())) {
|
map.put("code", 0);
|
||||||
Map<String, Object> map = new HashMap<>();
|
map.put("data", "Wrong password!");
|
||||||
map.put("code", 1);
|
|
||||||
map.put("data", new Gson().toJson(user));
|
|
||||||
responseData = new Gson().toJson(map);
|
|
||||||
} else {
|
|
||||||
Map<String, Object> map = new HashMap<>();
|
|
||||||
map.put("code", 0);
|
|
||||||
map.put("data", "Wrong password!");
|
|
||||||
responseData = new Gson().toJson(map);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return responseData;
|
return new Gson().toJson(map);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
private void closeResources() {
|
||||||
|
try {
|
||||||
|
if (socket != null)
|
||||||
|
socket.close();
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,7 +2,6 @@ package server.tools;
|
|||||||
|
|
||||||
import java.sql.Connection;
|
import java.sql.Connection;
|
||||||
import java.sql.DriverManager;
|
import java.sql.DriverManager;
|
||||||
import java.sql.PreparedStatement;
|
|
||||||
import java.sql.SQLException;
|
import java.sql.SQLException;
|
||||||
|
|
||||||
public class DBConnection {
|
public class DBConnection {
|
||||||
@@ -10,32 +9,32 @@ public class DBConnection {
|
|||||||
private final static String url = "jdbc:mysql://localhost:3306/lda?allowPublicKeyRetrieval=true&useSSL=false";
|
private final static String url = "jdbc:mysql://localhost:3306/lda?allowPublicKeyRetrieval=true&useSSL=false";
|
||||||
private final static String user = "root";
|
private final static String user = "root";
|
||||||
private final static String password = "516849";
|
private final static String password = "516849";
|
||||||
|
|
||||||
static {
|
static {
|
||||||
try {
|
try {
|
||||||
Class.forName(driverName);
|
Class.forName(driverName);
|
||||||
} catch (Exception e) {
|
} catch (ClassNotFoundException e) {
|
||||||
e.printStackTrace();
|
throw new ExceptionInInitializerError(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static Connection getConnection() {
|
public static Connection getConnection() {
|
||||||
Connection connection = null;
|
|
||||||
try {
|
try {
|
||||||
connection = DriverManager.getConnection(url, user, password);
|
return DriverManager.getConnection(url, user, password);
|
||||||
} catch (SQLException e) {
|
} catch (SQLException e) {
|
||||||
e.printStackTrace();
|
throw new RuntimeException("Failed to get database connection", e);
|
||||||
}
|
}
|
||||||
return connection;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void close(Connection con, PreparedStatement pst) {
|
public static void close(AutoCloseable... resources) {
|
||||||
try {
|
for (AutoCloseable resource : resources) {
|
||||||
if (pst != null)
|
if (resource != null) {
|
||||||
pst.close();
|
try {
|
||||||
if (con != null)
|
resource.close();
|
||||||
con.close();
|
} catch (Exception e) {
|
||||||
} catch (SQLException e) {
|
e.printStackTrace();
|
||||||
e.printStackTrace();
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -8,29 +8,36 @@ import java.sql.SQLException;
|
|||||||
import java.sql.Statement;
|
import java.sql.Statement;
|
||||||
|
|
||||||
public class ExportToCSV {
|
public class ExportToCSV {
|
||||||
public static void to() {
|
public static void to(String outputFile, String query) {
|
||||||
String outputFile = "Library Data.csv";
|
|
||||||
try (Connection connection = DBConnection.getConnection();
|
try (Connection connection = DBConnection.getConnection();
|
||||||
Statement statement = connection.createStatement();
|
Statement statement = connection.createStatement();
|
||||||
ResultSet resultSet = statement.executeQuery("SELECT * FROM book");
|
ResultSet resultSet = statement.executeQuery(query);
|
||||||
FileWriter fileWriter = new FileWriter(outputFile)) {
|
FileWriter fileWriter = new FileWriter(outputFile)) {
|
||||||
for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) {
|
writeColumnNames(resultSet, fileWriter);
|
||||||
fileWriter.append(resultSet.getMetaData().getColumnName(i));
|
writeDataRows(resultSet, fileWriter);
|
||||||
if (i < resultSet.getMetaData().getColumnCount())
|
System.out.println("Data exported successfully to " + outputFile);
|
||||||
fileWriter.append(",");
|
|
||||||
}
|
|
||||||
fileWriter.append("\n");
|
|
||||||
while (resultSet.next()) {
|
|
||||||
for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) {
|
|
||||||
fileWriter.append(resultSet.getString(i));
|
|
||||||
if (i < resultSet.getMetaData().getColumnCount())
|
|
||||||
fileWriter.append(",");
|
|
||||||
}
|
|
||||||
fileWriter.append("\n");
|
|
||||||
}
|
|
||||||
System.out.println("Data exported successfully!");
|
|
||||||
} catch (SQLException | IOException e) {
|
} catch (SQLException | IOException e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
private static void writeColumnNames(ResultSet resultSet, FileWriter fileWriter) throws SQLException, IOException {
|
||||||
|
for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) {
|
||||||
|
fileWriter.append(resultSet.getMetaData().getColumnName(i));
|
||||||
|
if (i < resultSet.getMetaData().getColumnCount())
|
||||||
|
fileWriter.append(",");
|
||||||
|
}
|
||||||
|
fileWriter.append("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void writeDataRows(ResultSet resultSet, FileWriter fileWriter) throws SQLException, IOException {
|
||||||
|
while (resultSet.next()) {
|
||||||
|
for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) {
|
||||||
|
fileWriter.append(resultSet.getString(i));
|
||||||
|
if (i < resultSet.getMetaData().getColumnCount())
|
||||||
|
fileWriter.append(",");
|
||||||
|
}
|
||||||
|
fileWriter.append("\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -11,37 +11,38 @@ import jxl.write.WritableSheet;
|
|||||||
import jxl.write.WritableWorkbook;
|
import jxl.write.WritableWorkbook;
|
||||||
|
|
||||||
public class ExportToXLS {
|
public class ExportToXLS {
|
||||||
public static void to() {
|
public static void to(String query, String outputPath) {
|
||||||
|
String[] columnHeaders = { "title", "author", "publisher", "oldprice", "newprice", "href" };
|
||||||
try (Connection connection = DBConnection.getConnection();
|
try (Connection connection = DBConnection.getConnection();
|
||||||
Statement statement = connection.createStatement();
|
Statement statement = connection.createStatement();
|
||||||
ResultSet rs = statement.executeQuery("SELECT * FROM book");) {
|
ResultSet rs = statement.executeQuery(query)) {
|
||||||
WritableWorkbook book = Workbook.createWorkbook(new File("Library Data.xls"));
|
WritableWorkbook workbook = Workbook.createWorkbook(new File(outputPath));
|
||||||
WritableSheet sheet = book.createSheet("sheet1", 0);
|
WritableSheet sheet = workbook.createSheet("Sheet1", 0);
|
||||||
Label label = new Label(0, 0, "title");
|
createColumnHeaders(sheet, columnHeaders);
|
||||||
sheet.addCell(label);
|
writeDataRows(sheet, rs);
|
||||||
label = new Label(1, 0, "author");
|
workbook.write();
|
||||||
sheet.addCell(label);
|
workbook.close();
|
||||||
label = new Label(2, 0, "publisher");
|
System.out.println("Data exported successfully to " + outputPath);
|
||||||
sheet.addCell(label);
|
|
||||||
label = new Label(3, 0, "oldprice");
|
|
||||||
sheet.addCell(label);
|
|
||||||
label = new Label(4, 0, "newprice");
|
|
||||||
sheet.addCell(label);
|
|
||||||
label = new Label(5, 0, "href");
|
|
||||||
sheet.addCell(label);
|
|
||||||
int i = 1;
|
|
||||||
while (rs.next()) {
|
|
||||||
for (int j = 0; j <= 5; j++) {
|
|
||||||
label = new Label(j, i, rs.getString(j + 1));
|
|
||||||
sheet.addCell(label);
|
|
||||||
}
|
|
||||||
i++;
|
|
||||||
}
|
|
||||||
book.write();
|
|
||||||
book.close();
|
|
||||||
System.out.println("Data exported successfully!");
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static void createColumnHeaders(WritableSheet sheet, String[] columnHeaders) throws Exception {
|
||||||
|
for (int i = 0; i < columnHeaders.length; i++) {
|
||||||
|
Label label = new Label(i, 0, columnHeaders[i]);
|
||||||
|
sheet.addCell(label);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void writeDataRows(WritableSheet sheet, ResultSet rs) throws Exception {
|
||||||
|
int rowIndex = 1;
|
||||||
|
while (rs.next()) {
|
||||||
|
for (int columnIndex = 0; columnIndex <= 5; columnIndex++) {
|
||||||
|
Label label = new Label(columnIndex, rowIndex, rs.getString(columnIndex + 1));
|
||||||
|
sheet.addCell(label);
|
||||||
|
}
|
||||||
|
rowIndex++;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -69,6 +69,7 @@ public class Book {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return "Title: " + this.title + ", Author: " + this.author + ", Publisher: " + this.publisher + ", Original Price: " + this.oldprice + ", Discounted Price: " + this.newprice + ", URL: " + this.href;
|
return "Title: " + this.title + ", Author: " + this.author + ", Publisher: " + this.publisher
|
||||||
|
+ ", Original Price: " + this.oldprice + ", Discounted Price: " + this.newprice + ", URL: " + this.href;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2,9 +2,7 @@ package server.tools.crawling;
|
|||||||
|
|
||||||
import java.io.BufferedReader;
|
import java.io.BufferedReader;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
|
||||||
import java.io.InputStreamReader;
|
import java.io.InputStreamReader;
|
||||||
import java.io.UnsupportedEncodingException;
|
|
||||||
import java.net.HttpURLConnection;
|
import java.net.HttpURLConnection;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import java.net.URLEncoder;
|
import java.net.URLEncoder;
|
||||||
@@ -12,43 +10,22 @@ import java.util.regex.Matcher;
|
|||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
public class CrawlerTools {
|
public class CrawlerTools {
|
||||||
public static String get(String urlStr, String charset) {
|
public static String get(String urlStr, String charset) throws IOException {
|
||||||
StringBuffer buf = new StringBuffer();
|
StringBuilder buf = new StringBuilder();
|
||||||
HttpURLConnection con = null;
|
URL url = new URL(urlStr);
|
||||||
InputStream in = null;
|
HttpURLConnection con = (HttpURLConnection) url.openConnection();
|
||||||
BufferedReader read = null;
|
con.setRequestProperty("User-Agent",
|
||||||
try {
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36 Edg/101.0.1210.53");
|
||||||
URL url = new URL(urlStr);
|
if (con.getResponseCode() == 200) {
|
||||||
con = (HttpURLConnection) url.openConnection();
|
try (BufferedReader read = new BufferedReader(new InputStreamReader(con.getInputStream(), charset))) {
|
||||||
con.setRequestProperty("User-Agent",
|
String info;
|
||||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36 Edg/101.0.1210.53");
|
|
||||||
int code = con.getResponseCode();
|
|
||||||
if (code == 200) {
|
|
||||||
in = con.getInputStream();
|
|
||||||
read = new BufferedReader(new InputStreamReader(in, charset));
|
|
||||||
String info = "";
|
|
||||||
while ((info = read.readLine()) != null)
|
while ((info = read.readLine()) != null)
|
||||||
buf.append(info);
|
buf.append(info);
|
||||||
} else
|
}
|
||||||
System.out.println("Error! Response code: " + code + ".");
|
} else {
|
||||||
} catch (Exception e) {
|
System.out.println("Error! Response code: " + con.getResponseCode() + ".");
|
||||||
e.printStackTrace();
|
|
||||||
} finally {
|
|
||||||
if (read != null)
|
|
||||||
try {
|
|
||||||
read.close();
|
|
||||||
} catch (IOException e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
|
||||||
if (in != null)
|
|
||||||
try {
|
|
||||||
in.close();
|
|
||||||
} catch (IOException e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
|
||||||
if (con != null)
|
|
||||||
con.disconnect();
|
|
||||||
}
|
}
|
||||||
|
con.disconnect();
|
||||||
return buf.toString();
|
return buf.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -58,15 +35,12 @@ public class CrawlerTools {
|
|||||||
Matcher mat = pat.matcher(url);
|
Matcher mat = pat.matcher(url);
|
||||||
while (mat.find()) {
|
while (mat.find()) {
|
||||||
String hanzi = mat.group();
|
String hanzi = mat.group();
|
||||||
System.out.println(hanzi);
|
|
||||||
String encodehanzi = "";
|
|
||||||
try {
|
try {
|
||||||
encodehanzi = URLEncoder.encode(hanzi, "utf-8");
|
url = url.replace(hanzi, URLEncoder.encode(hanzi, "utf-8"));
|
||||||
} catch (UnsupportedEncodingException e) {
|
} catch (IOException e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
}
|
}
|
||||||
url = url.replaceAll(hanzi, encodehanzi);
|
|
||||||
}
|
}
|
||||||
return url;
|
return url;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,19 +1,32 @@
|
|||||||
package server.tools.crawling;
|
package server.tools.crawling;
|
||||||
|
|
||||||
|
import java.util.concurrent.ExecutorService;
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
public class Driver {
|
public class Driver {
|
||||||
|
private static final String URL_PREFIX = "http://bang.dangdang.com/books/bestsellers/01.00.00.00.00.00-24hours-0-0-1-";
|
||||||
|
|
||||||
public static void crawl() {
|
public static void crawl() {
|
||||||
|
ExecutorService executor = Executors.newFixedThreadPool(5);
|
||||||
try {
|
try {
|
||||||
String url = "http://bang.dangdang.com/books/bestsellers/01.00.00.00.00.00-24hours-0-0-1-1";
|
for (int i = 1; i <= 5; i++) {
|
||||||
Thread thread1 = new Thread(new NewsThread(url));
|
String url = URL_PREFIX + i;
|
||||||
thread1.start();
|
executor.submit(new NewsThread(url));
|
||||||
for (int i = 2; i <= 5; i++) {
|
if (i > 1)
|
||||||
String url2 = "http://bang.dangdang.com/books/bestsellers/01.00.00.00.00.00-24hours-0-0-1-" + i;
|
TimeUnit.SECONDS.sleep(10);
|
||||||
Thread thread2 = new Thread(new NewsThread(url2));
|
}
|
||||||
thread2.start();
|
} catch (InterruptedException e) {
|
||||||
Thread.sleep(10000);
|
Thread.currentThread().interrupt();
|
||||||
|
System.err.println("Thread was interrupted, Failed to complete operation");
|
||||||
|
} finally {
|
||||||
|
executor.shutdown();
|
||||||
|
try {
|
||||||
|
if (!executor.awaitTermination(800, TimeUnit.MILLISECONDS))
|
||||||
|
executor.shutdownNow();
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
executor.shutdownNow();
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,7 +1,9 @@
|
|||||||
package server.tools.crawling;
|
package server.tools.crawling;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
import java.sql.Connection;
|
import java.sql.Connection;
|
||||||
import java.sql.PreparedStatement;
|
import java.sql.PreparedStatement;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
|
||||||
import org.jsoup.Jsoup;
|
import org.jsoup.Jsoup;
|
||||||
import org.jsoup.nodes.Document;
|
import org.jsoup.nodes.Document;
|
||||||
@@ -14,63 +16,50 @@ public class NewsThread implements Runnable {
|
|||||||
private String urlPath;
|
private String urlPath;
|
||||||
|
|
||||||
public NewsThread(String urlPath) {
|
public NewsThread(String urlPath) {
|
||||||
super();
|
|
||||||
this.urlPath = urlPath;
|
this.urlPath = urlPath;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
String content = CrawlerTools.get(urlPath, "GB2312");
|
try {
|
||||||
Document doc = Jsoup.parse(content);
|
String content = CrawlerTools.get(urlPath, "GB2312");
|
||||||
Elements elements = doc.select(".bang_wrapper .bang_list_box ul li");
|
Document doc = Jsoup.parse(content);
|
||||||
Connection con = DBConnection.getConnection();
|
Elements elements = doc.select(".bang_wrapper .bang_list_box ul li");
|
||||||
PreparedStatement ps = null;
|
try (Connection con = DBConnection.getConnection()) {
|
||||||
String sql = null;
|
for (Element bookElement : elements.subList(0, Math.min(elements.size(), 20))) {
|
||||||
int index = 0;
|
String title = bookElement.select(".name a").text();
|
||||||
for (Element bookelement : elements) {
|
Elements publisherInfoElements = bookElement.select(".publisher_info");
|
||||||
index++;
|
String author = publisherInfoElements.size() > 0 ? publisherInfoElements.get(0).select("a").text()
|
||||||
if (index >= 20)
|
: null;
|
||||||
break;
|
String publisher = publisherInfoElements.size() > 1
|
||||||
String title = bookelement.select(".name a").text();
|
? publisherInfoElements.get(1).select("a").text()
|
||||||
Elements publisherInfoElements = bookelement.select(".publisher_info");
|
: null;
|
||||||
String author = null;
|
String oprice = bookElement.select(".price_r").text();
|
||||||
String publisher = null;
|
String nprice = bookElement.select(".price_n").first().text();
|
||||||
if (!publisherInfoElements.isEmpty()) {
|
String href = bookElement.select(".name a").attr("href");
|
||||||
Element firstPublisherInfoElement = publisherInfoElements.get(0);
|
double oldprice = Double.parseDouble(oprice.substring(1));
|
||||||
Element secondPublisherInfoElement = publisherInfoElements.get(1);
|
double newprice = Double.parseDouble(nprice.substring(1));
|
||||||
author = firstPublisherInfoElement.select("a").text();
|
insertBook(con, title, author, publisher, oldprice, newprice, href);
|
||||||
publisher = secondPublisherInfoElement.select("a").text();
|
}
|
||||||
}
|
} catch (SQLException e) {
|
||||||
String oprice = bookelement.select(".price_r").text();
|
|
||||||
String nprice = bookelement.select(".price_n").first().text();
|
|
||||||
String href = bookelement.select(".name a").attr("href");
|
|
||||||
double oldprice = Double.parseDouble(oprice.substring(1));
|
|
||||||
double newprice = Double.parseDouble(nprice.substring(1));
|
|
||||||
sql = "insert into book values(?,?,?,?,?,?)";
|
|
||||||
try {
|
|
||||||
ps = con.prepareStatement(sql);
|
|
||||||
ps.setString(1, title);
|
|
||||||
ps.setString(2, author);
|
|
||||||
ps.setString(3, publisher);
|
|
||||||
ps.setDouble(4, oldprice);
|
|
||||||
ps.setDouble(5, newprice);
|
|
||||||
ps.setString(6, href);
|
|
||||||
ps.executeUpdate();
|
|
||||||
} catch (Exception e) {
|
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
}
|
}
|
||||||
System.out.println(title);
|
} catch (IOException e) {
|
||||||
System.out.println(author);
|
|
||||||
System.out.println(publisher);
|
|
||||||
System.out.println(oldprice);
|
|
||||||
System.out.println(newprice);
|
|
||||||
System.out.println(href);
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
ps.close();
|
|
||||||
con.close();
|
|
||||||
} catch (Exception e) {
|
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void insertBook(Connection con, String title, String author, String publisher, double oldprice,
|
||||||
|
double newprice, String href) throws SQLException {
|
||||||
|
String sql = "INSERT INTO book VALUES (?, ?, ?, ?, ?, ?)";
|
||||||
|
try (PreparedStatement ps = con.prepareStatement(sql)) {
|
||||||
|
ps.setString(1, title);
|
||||||
|
ps.setString(2, author);
|
||||||
|
ps.setString(3, publisher);
|
||||||
|
ps.setDouble(4, oldprice);
|
||||||
|
ps.setDouble(5, newprice);
|
||||||
|
ps.setString(6, href);
|
||||||
|
ps.executeUpdate();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -45,9 +45,4 @@ public class User {
|
|||||||
public void setRole(String role) {
|
public void setRole(String role) {
|
||||||
this.role = role;
|
this.role = role;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
public String toString() {
|
|
||||||
return userName + "\t" + password + "\t" + name + "\t" + role;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user