Question

用java编写网络爬虫，用来爬网络音乐资源，再返回java页面显示该怎么实现

Answer 1

下面是源代码,希望可以帮到你~~
package com.ly.mainprocess;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;

import org.apache.http.Consts;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.StatusLine;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.cookie.Cookie;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;

public class Test1 {
public static void main(String[] args){
Test1 test1 = new Test1();
System.out.println(test1.process("******","******"));
}

@SuppressWarnings("deprecation")
public boolean process(String username,String password) {
boolean ret=false;
DefaultHttpClient httpclient = new DefaultHttpClient();
try {
HttpGet httpget;
HttpResponse response;
HttpEntity entity;

List<Cookie> cookies;

//组建登录的post包
HttpPost httppost = new HttpPost("http://login.hi.mop.com/Login.do"); // 用户登录
List<NameValuePair> nvps = new ArrayList<NameValuePair>();
nvps.add(new BasicNameValuePair("nickname", username));
nvps.add(new BasicNameValuePair("password", password));
nvps.add(new BasicNameValuePair("origURL", "http://hi.mop.com/SysHome.do"));
nvps.add(new BasicNameValuePair("loginregFrom", "index"));
nvps.add(new BasicNameValuePair("ss", "10101"));

httppost.setEntity(new UrlEncodedFormEntity(nvps, Consts.UTF_8));
httppost.addHeader("Referer", "http://hi.mop.com/SysHome.do");
httppost.addHeader("Connection", "keep-alive");
httppost.addHeader("Content-Type", "application/x-www-form-urlencoded");
httppost.addHeader("Accept-Language", "zh-CN,zh;q=0.8");
httppost.addHeader("Origin", "http://hi.mop.com");
httppost.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36");
response = httpclient.execute(httppost);
entity = response.getEntity();
// System.out.println("Login form get: " + response.getStatusLine());
EntityUtils.consume(entity);

// System.out.println("Post logon cookies:");
cookies = httpclient.getCookieStore().getCookies();
if (cookies.isEmpty()) {
// System.out.println("None");
} else {
for (int i = 0; i < cookies.size(); i++) {
// System.out.println("- " + cookies.get(i).toString());
}
}

//进行页面跳转
String url = ""; // 页面跳转
Header locationHeader = response.getFirstHeader("Location");
// System.out.println(locationHeader.getValue());
if (locationHeader != null) {
url = locationHeader.getValue(); // 得到跳转href
HttpGet httpget1 = new HttpGet(url);
response = httpclient.execute(httpget1);
// 登陆成功。。。hoho
}
entity = response.getEntity();
// System.out.println(response.getStatusLine());
if (entity != null) {
// System.out.println("Response content length: " + entity.getContentLength());
}
// 显示结果
BufferedReader reader = new BufferedReader(new InputStreamReader(entity.getContent(), "UTF-8"));
String line = null;
while ((line = reader.readLine()) != null) {
// System.out.println(line);
}

//自动打卡
// 访问网站的子网页。
HttpPost httppost1 = new HttpPost("http://home.hi.mop.com/ajaxGetContinusLoginAward.do"); // 设置个人信息页面
httppost1.addHeader("Content-Type", "text/plain;charset=UTF-8");
httppost1.addHeader("Accept", "text/plain, */*");
httppost1.addHeader("X-Requested-With", "XMLHttpRequest");
httppost1.addHeader("Referer", "http://home.hi.mop.com/Home.do");
response = httpclient.execute(httppost1);
entity = response.getEntity();
// System.out.println(response.getStatusLine());
if(response.getStatusLine().toString().indexOf("HTTP/1.1 200 OK")>=0){
ret = true;
}
if (entity != null) {
// System.out.println("Response content length: " + entity.getContentLength());
}
// 显示结果
reader = new BufferedReader(new InputStreamReader(entity.getContent(), "UTF-8"));
line = null;
while ((line = reader.readLine()) != null) {
System.out.println(line);
}
} catch (Exception e) {

} finally {
httpclient.getConnectionManager().shutdown();
}
return ret;
}
}