package download;
i
mport java.io.IOException;
i
mport java.sql.PreparedStatement;
i
mport java.sql.SQLException;
i
mport java.util.ArrayList;
i
mport org.jsoup.Connection.Response;
i
mport org.jsoup.Jsoup;
i
mport org.jsoup.nodes.D
ocument;
i
mport org.jsoup.nodes.Element;
i
mport org.jsoup.select.Elements;
i
mport utils.MySQL;
i
mport java.util.concurrent.Callable;
public class Download implements Callable<Boolean> {
private MaintainL
inks maintainQueue = new MaintainL
inks();
private Db db = new Db();
private boolean full = false;
private static final int expextedL
inks = 150000;
private static String ua = "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36";
public static void main(String[] args) {
String start = "http://www.zol.com.cn/";
D
ocument doc = null;
try {
Respo
nse res = Jsoup.co
nnect(start).respo
nse();
// res.header("");
doc = res.parse();
} catch (IOException e) {
e.printStackTrace();
}
getAllL
inks(doc);
}
@Override
public Boolean call() throws Exception {
while (!maintainQueue.isWaitEmpty()) {
maintainQueue.addWait(cachedWait);
cachedWait.clear();
ArrayList<String> currentQueue = maintainQueue.getOptWait();
for (String url : currentQueue) {
if (!maintainQueue.hasDownloaded(url)) {
download(url);
}
}
System.out.println("线程:" + Thread.currentThread().getId()
+ ";当前队列链接数：" + maintainQueue.waitSize() + "已下载:"
+ maintainQueue.downloadSize());
//
if (!full) {
full = maintainQueue.waitSize() > expextedL
inks;
}
}
db.commit();
db.finalise();
maintainQueue.persistence();
return true;
}
// private HashSet<String> cachedSuccess = new HashSet<String>();
private ArrayList<String> cachedWait = new ArrayList<String>();
public void download(String url) {
WrapDownload result = null;
try {
result = Download.getPage(url);
// db.addBatch(result);
db.addBatch(result);
// 下载队列里面只保存前100w个网页
if (!full) {
cachedWait.addAll(result.getL
inkList());
}
maintainQueue.addSuccess(url);
} catch (IOException e) {
// e.printStackTrace();
maintainQueue.addFailed(url);
System.err.println(url + "下载失败");
}
}
public static WrapDownload getPage(String url) throws IOException {
D
ocument doc = null;
doc = Jsoup.co
nnect(url).userAgent(ua).timeout(10000).get();
return new WrapDownload(getAllL
inks(doc), getText(doc), url);
}
public static ArrayList<String> getAllL
inks(D
ocument doc) {
ArrayList<String> l
inkList = new ArrayList<String>();
String url = "";
int temp = 0;
Elements elements = doc.select("a[href]");
for (Element e : elements) {
String attr = e.attr("href");
if (attr.startsWith(".")) {
url = e.b
aseUri() + e.attr("href").substrin
g(1);
url = url.replace("//", "/").replace("http:/", "http://");
if((temp=url.indexOf("#"))!=-1){
url = url.substrin
g(0,temp);
}
if(url.endsWith("/")){
url = url.substrin
g(0,url.length()-1);
}
l
inkList.add(url);
} else if (attr.startsWith("http")) {
url = e.attr("href");
if((temp=url.indexOf("#"))!=-1){
url = url.substrin
g(0,temp);
}
if(url.endsWith("/")){
url = url.substrin
g(0,url.length()-1);
}
l
inkList.add(url);
}
}
return l
inkList;
}
public static String getText(D
ocument doc) {
return doc.html();
}
public class Db {
public PreparedStatement ps = null;
public int count = 0;
private int fail = 0;
private MySQL mysql;
public Db() {
String sql = "INSERT INTO html(url,html_text) VALUES(?,?)";
mysql = new MySQL();
try {
mysql.conn.setAutoCommit(false);
ps = mysql.conn.prepareStatement(sql);
} catch (SQLException e) {
e.printStackTrace();
}
}
public void addBatch(WrapDownload data) {
try {
ps.setString(1, data.getUrl());
ps.setString(2, data.getHtml());
ps.addBatch();
count++;
if (count % 20 == 0) {
ps.executeBatch();
mysql.conn.commit();
}
} catch (SQLException e) {
e.printStackTrace();
System.out.println(data.getUrl() + "存储失败" + (++fail));
}
}
public void commit() {
try {
ps.executeBatch();
mysql.conn.commit();
} catch (SQLException e) {
e.printStackTrace();
}
}
public void finalise() {
try {
mysql.conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
System.out.println("共失败" + fail);
}
public void insert(WrapDownload data) {
String sql = "INSERT INTO html(url,html_text) VALUES(?,?)";
try {
PreparedStatement ps = mysql.conn.prepareStatement(sql);
ps.setString(1, data.getUrl());
ps.setString(2, data.getHtml());
ps.executeBatch();
} catch (SQLException e) {
System.out.println(data.getUrl());
e.printStackTrace();
}
}
}
