Question

HttpClient+Jsoup制作网络蜘蛛遇到问题，一直提示java.lang.NullPointerException

public class NetSpiderBiz {
private static CopyOnWriteArrayList<String> seeds = null;
private static CopyOnWriteArrayList<String> visited = null;
static {
seeds = new CopyOnWriteArrayList<String>();
visited = new CopyOnWriteArrayList<String>();
}
private CloseableHttpClient httpClient = null;
private HttpGet httpGet = null;
private HttpEntity entity = null;
private HttpResponse response = null;
private InputStream inputStream = null;
private InputStreamReader inputStreamReader = null;
private BufferedReader reader = null;
private Document document = null;
private Elements elements = null;
public void run() {
try {
if (seeds != null && seeds.size() > 0 && !seeds.isEmpty()) {
for (String seed : seeds) {
if (seed != null && seed.trim().length() > 0 && !"".equals(seed)) {
if (!visited.contains(seed)) {
httpClient = HttpClientBuilder.create().build();
httpGet = new HttpGet(seed);
httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:6.0.2) Gecko/20100101 Firefox/6.0.2");
response = httpClient.execute(httpGet);
if (response.getStatusLine().getStatusCode() == 200) {
entity = response.getEntity();
inputStream = entity.getContent();
inputStreamReader = new InputStreamReader(inputStream , "utf-8");
reader = new BufferedReader(inputStreamReader);
String line = null;
String code = "";
while ((line = reader.readLine()) != null) {
code += line;
}
System.out.println(code);

visited.add(seed);
seeds.remove(seed);

document = Jsoup.parse(code);
if (document != null) {

}
elements = document.body().select("a");
/*if (elements != null && elements.size() > 0) {
for (Element e : elements) {
String a = e.attr("abs:href");
if (a != null && a.trim().length() > 0 && !"".equals(a)) {
if (!seeds.contains(a)) {
seeds.add(a);
System.out.println(e.text() + " : " + e.attr("abs:href"));
}
}
}
}*/
}
}
}
}
}
} catch (ClientProtocolException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}/* finally {
try {
reader.close();
inputStreamReader.close();
inputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}*/
//run();
}

public static void main(String[] args) {
NetSpiderBiz n = new NetSpiderBiz();
seeds.add("http://www.sste.com/its/default.htm");
n.run();
}

}

Answer 1

你的 getAllId() 这个方法执行 for (String string : t.getNextUrl(maxSize)) 这个循环的时候报错了.
因为你的 getNextUrl() 方法最终返回的是 null 所以这里遍历肯定不行

Answer 2

设置断点在debug里面找，话说你属性为啥设置那么多null