Question

用c#将网页抓取数据写入一个excel

用c#将http://zh.weather-forecast.com/locations/Beijing/forecasts/latest#forecast-part-0上days 1-3的中每一天的早中晚的温度 风力 湿度抓下来放入一个excel中 每天下午一点自动更新 请写出代码

justea @ yeah . net

Answer 1

100分让写那么多代码。。。

 用webClient 或者HttpWebRequest访问地址，返回html代码以后，

用HtmlAgilityPack.dll 解析html。或者用正则

HtmlAgilityPack.dll应该好用多了，然后插入excel即可

HtmlAgilityPack.dll的用法：

var doc = new HtmlDocument();
                doc.LoadHtml(queryPageHtml);
                var node = doc.DocumentNode.SelectSingleNode("//input[@name='struts.token.name']");
                var nodeStoken = doc.DocumentNode.SelectSingleNode("//input[@name='struts.token']");
                var strutStokenName = node.GetAttributeValue("value", "");
                var strutStoken = nodeStoken.GetAttributeValue("value", "");

 现成的http请求类，可以稍作修改

webclient获取

WebClient wc = new WebClient();
 wc.Credentials = CredentialCache.DefaultCredentials;
  Stream resStream = wc.OpenRead(PageUrl);
        StreamReader sr = new StreamReader(resStream,System.Text.Encoding.Default;
        string HtmlStr = sr.ReadToEnd();
        resStream.Close();
        wc.Dispose();
 下面的是HttpWebRequest的，代码比较多。用webClient就行了

using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Mime;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Net;
using System.IO;
using System.Drawing;
using System.Windows.Forms;

namespace Hosting
{
    /// <summary>
    /// 模拟网页操作,提交、获取订单页面数据
    /// </summary>
    public class HttpWebRequestExtension
    {
        private static string contentType = "application/x-www-form-urlencoded";
        private static string accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/x-silverlight, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/x-silverlight-2-b1, */*";
        private static string userAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; Zune 4.7; BOIE9;ZHCN)";


        /// <summary>
        /// 提交订单数据
        /// </summary>
        /// <param name="url"></param>
        /// <param name="cookie"></param>
        /// <param name="param"></param>
        /// <returns></returns>
        public static string PostWebContent(string url, CookieContainer cookie, Dictionary<string, string> param)
        {
            //byte[] bs = Encoding.ASCII.GetBytes(param);
            HttpWebRequest httpWebRequest = null;
            ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3;
            ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(RemoteCertificateValidate);
            httpWebRequest = (HttpWebRequest)HttpWebRequest.Create(url);
            httpWebRequest.ProtocolVersion = HttpVersion.Version10;
            httpWebRequest.Credentials = CredentialCache.DefaultCredentials;
            X509Certificate objx509 = new X509Certificate(Application.StartupPath + "\\yz.cer");   //添加证书到请求里       
            httpWebRequest.ClientCertificates.Add(objx509);
            httpWebRequest.CookieContainer = cookie;
            httpWebRequest.ContentType = contentType;
            httpWebRequest.Accept = accept;
            httpWebRequest.UserAgent = userAgent;
            httpWebRequest.Method = "POST";
            Encoding requestEncoding = Encoding.GetEncoding("UTF-8");
            //如果需要POST数据  
            if (!(param == null || param.Count == 0))
            {
                StringBuilder buffer = new StringBuilder();
                int i = 0;
                foreach (string key in param.Keys)
                {
                    if (i > 0)
                    {
                        buffer.AppendFormat("&{0}={1}", key, param[key]);
                    }
                    else
                    {
                        buffer.AppendFormat("{0}={1}", key, param[key]);
                    }
                    i++;
                }
                byte[] data = requestEncoding.GetBytes(buffer.ToString());
                httpWebRequest.ContentLength = data.Length;
                using (Stream stream = httpWebRequest.GetRequestStream())
                {
                    stream.Write(data, 0, data.Length);
                }
            }


            var httpWebResponse = (HttpWebResponse)httpWebRequest.GetResponse();
            Stream responseStream = httpWebResponse.GetResponseStream();
            StreamReader streamReader = new StreamReader(responseStream, Encoding.UTF8);
            string html = streamReader.ReadToEnd();

            streamReader.Close();
            responseStream.Close();

            httpWebRequest.Abort();
            httpWebResponse.Close();

            return html;
        }


        /// <summary>
        /// 获取页面数据
        /// </summary>
        /// <param name="url"></param>
        /// <param name="cookie"></param>
        /// <returns></returns>
        public static string GetWebContent(string url, CookieContainer cookie)
        {
            HttpWebRequest httpWebRequest = null;
            ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3;
            ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(RemoteCertificateValidate);
            httpWebRequest = (HttpWebRequest)HttpWebRequest.Create(url);
            httpWebRequest.ProtocolVersion = HttpVersion.Version10;
            httpWebRequest.Credentials = CredentialCache.DefaultCredentials;
            X509Certificate objx509 = new X509Certificate(Application.StartupPath + "\\yz.cer");   //添加证书到请求里       
            httpWebRequest.ClientCertificates.Add(objx509);
            httpWebRequest.CookieContainer = cookie;
            httpWebRequest.ContentType = contentType;
            httpWebRequest.KeepAlive = true;
            httpWebRequest.Accept = accept;
            httpWebRequest.UserAgent = userAgent;
            httpWebRequest.Method = "GET";
            httpWebRequest.ServicePoint.ConnectionLimit = int.MaxValue;

            var httpWebResponse = (HttpWebResponse)httpWebRequest.GetResponse();
            Stream responseStream = httpWebResponse.GetResponseStream();
            StreamReader streamReader = new StreamReader(responseStream, Encoding.UTF8);
            string html = streamReader.ReadToEnd();

            streamReader.Close();
            responseStream.Close();

            httpWebRequest.Abort();
            httpWebResponse.Close();

            return html;
        }



        /// <summary>
        /// Sets the cert policy.
        /// </summary>
        public static void SetCertificatePolicy()
        {
            ServicePointManager.ServerCertificateValidationCallback
                       += RemoteCertificateValidate;
        }

        /// <summary>
        /// Remotes the certificate validate.
        /// </summary>
        private static bool RemoteCertificateValidate(
           object sender, X509Certificate cert,
            X509Chain chain, SslPolicyErrors error)
        {

            // trust any certificate!!!
            return true;
        }
    }
}

 写入excel的代码：

public static bool SaveDataTableToExcel(System.Data.DataTable excelTable, string filePath)
        {
            Microsoft.Office.Interop.Excel.Application app =
                new Microsoft.Office.Interop.Excel.ApplicationClass();
            try
            {
                app.Visible = false;
                Workbook wBook = app.Workbooks.Add(true);
                Worksheet wSheet = wBook.Worksheets[1] as Worksheet;
                if (excelTable.Rows.Count > 0)
                {
                    int row = 0;
                    row = excelTable.Rows.Count;
                    int col = excelTable.Columns.Count;
                    for (int i = 0; i < row; i++)
                    {
                        for (int j = 0; j < col; j++)
                        {
                            string str = excelTable.Rows[i][j].ToString();
                            wSheet.Cells[i + 2, j + 1] = str;
                        }
                    }
                }

                int size = excelTable.Columns.Count;
                for (int i = 0; i < size; i++)
                {
                    wSheet.Cells[1, 1 + i] = excelTable.Columns[i].ColumnName;
                }
                //设置禁止弹出保存和覆盖的询问提示框 
                app.DisplayAlerts = false;
                app.AlertBeforeOverwriting = false;
                //保存工作簿 
                wBook.Save();
                //保存excel文件 
                app.Save(filePath);
                app.SaveWorkspace(filePath);
                app.Quit();
                app = null;
                return true;
            }
            catch (Exception err)
            {
                MessageBox.Show("导出Excel出错！错误原因：" + err.Message, "提示信息",
                    MessageBoxButtons.OK, MessageBoxIcon.Information);
                return false;
            }
            finally
            {
            }
        }

追问

能不能帮忙写出来整个的代码 已经加了50分 可以再加100

追答

不是分的问题。。自己整理整理我给你的代码，就OK了。。。