76 lines
2.6 KiB
Java
76 lines
2.6 KiB
Java
package de.gurkengewuerz.ripmewrapper;
|
|
|
|
import java.io.File;
|
|
import java.io.FileInputStream;
|
|
import java.nio.file.Files;
|
|
import java.nio.file.Paths;
|
|
import java.sql.Connection;
|
|
import java.sql.DriverManager;
|
|
import java.sql.PreparedStatement;
|
|
import java.sql.ResultSet;
|
|
import java.sql.Statement;
|
|
import java.util.HashSet;
|
|
import java.util.List;
|
|
import java.util.TimerTask;
|
|
import java.util.logging.Level;
|
|
import java.util.logging.Logger;
|
|
import java.util.stream.Collectors;
|
|
|
|
/**
|
|
* Created by gurkengewuerz.de on 09.10.2018.
|
|
*/
|
|
public class ImageCrawler extends TimerTask {
|
|
|
|
@Override
|
|
public void run() {
|
|
Webserver.CRAWLER_IS_RUNNING = true;
|
|
Logger.getLogger(ImageCrawler.class.getName()).log(Level.INFO, ("Run Timer"));
|
|
try {
|
|
Connection connection = DriverManager.getConnection("jdbc:sqlite:reddit_meta.db");
|
|
Statement statement = connection.createStatement();
|
|
statement.setQueryTimeout(30); // set timeout to 30 sec.
|
|
|
|
HashSet<String> findIds = new HashSet<>();
|
|
ResultSet rs = statement.executeQuery("SELECT metalist.id FROM metalist LEFT JOIN filelist ON metalist.id = filelist.id WHERE filelist.path IS NULL");
|
|
while (rs.next()) {
|
|
findIds.add(rs.getString("id"));
|
|
}
|
|
|
|
|
|
List<String> pathList = Files.find(Paths.get(System.getProperty("user.dir")), 100,
|
|
(path, basicFileAttributes) -> {
|
|
File file = path.toFile();
|
|
return !file.isDirectory();
|
|
}).map(String::valueOf)
|
|
.collect(Collectors.toList());
|
|
|
|
PreparedStatement ps = connection.prepareStatement("INSERT INTO filelist VALUES (NULL, ?, ?)");
|
|
HashSet<String> foundIds = new HashSet<>();
|
|
for (String s : pathList) {
|
|
String id = s.substring(s.lastIndexOf(File.separator) + 1).split("-")[0];
|
|
if (!findIds.contains(id)) continue;
|
|
ps.setString(1, id);
|
|
ps.setString(2, s);
|
|
ps.executeUpdate();
|
|
foundIds.add(id);
|
|
}
|
|
ps.close();
|
|
|
|
findIds.removeAll(foundIds);
|
|
|
|
ps = connection.prepareStatement("DELETE FROM metalist WHERE id = ?");
|
|
for (String id : findIds) {
|
|
ps.setString(1, id);
|
|
ps.executeUpdate();
|
|
}
|
|
ps.close();
|
|
|
|
connection.close();
|
|
} catch (Exception e) {
|
|
Logger.getLogger(ImageCrawler.class.getName()).log(Level.SEVERE, null, e);
|
|
} finally {
|
|
Webserver.CRAWLER_IS_RUNNING = false;
|
|
}
|
|
}
|
|
}
|