c# HttpRequest使用,包括POST的多文件+参数模拟提交

直接对HttpRequest进行的包装,可以直接使用对网页进行爬取或者模拟各种动作提交到服务器。特别适合多个文件+参数的提交。直接贴代码,带cookie操作,可以进行模拟登陆

System.Net.ServicePointManager.DefaultConnectionLimit = int.MaxValue; 多线程 设置

public class JfResquest
    {
        public HttpWebRequest request = null;
        //配置
        public string url = "";
        public string method = "GET";
        public Dictionary<string, string> para = new Dictionary<string, string>(); //参数 成对参数
        public string parastr = ""; //参数 或者raw参数
        public Encoding encoding = Encoding.UTF8;  //字符集
        public CookieContainer cookies = new CookieContainer(); //cookie
        public CookieCollection reCookies = new CookieCollection(); //Request返回的cookie    
        public WebProxy proxy = null;
        public Dictionary<string, string> files = new Dictionary<string, string>();//上传的文件
        public string host = "";
        public string referer = "";
        public string accept = "";
        public string contentType = "";
        public JfResquest(string _url)
        {
            this.url = _url;
        }
        /// <summary>
        /// 初始化
        /// </summary>
        private void _init()
        {

            if (method.Equals("POST"))
            {
                request = (HttpWebRequest)WebRequest.Create(url);
                request.CookieContainer = this.cookies;//加入cookie
                request.Method = "POST";
                //文件上传
                if (files.Count > 0)
                {
                    var memStream = new MemoryStream();
                    string boundary = "---------------------------" + DateTime.Now.Ticks.ToString("x");
                    string Enter = "\r\n";
                    foreach (var item in para)
                    {
                        string pstr = Enter + "--" + boundary + Enter
                      + "Content-Disposition: form-data; name=\"" + item.Key + "\"" + Enter + Enter
                      + item.Value;
                        var StrByte = encoding.GetBytes(pstr);//所有字符串二进制
                        memStream.Write(StrByte, 0, StrByte.Length);
                    }
                    //文件拼接  
                    foreach (var file in files)
                    {
                        byte[] fileContentByte = new byte[1024]; // 文件内容二进制
                        FileStream fs = new FileStream(file.Value, FileMode.Open, FileAccess.Read);
                        fileContentByte = new byte[fs.Length]; // 二进制文件
                        fs.Read(fileContentByte, 0, Convert.ToInt32(fs.Length));
                        fs.Close();

                        string fileContentStr = Enter + "--" + boundary + Enter
                       + $"Content-Disposition: form-data; name=\"{file.Key}\"; filename=\"{Path.GetFileName(file.Value)}\""
                       + Enter + "Content-Type:application/octet-stream" + Enter + Enter;
                        var fileContentStrByte = Encoding.UTF8.GetBytes(fileContentStr);//fileContent一些名称等信息的二进制(不包含文件本身)
                        memStream.Write(fileContentStrByte, 0, fileContentStrByte.Length);
                        memStream.Write(fileContentByte, 0, fileContentByte.Length);

                    }
                    var endBoundary = Encoding.UTF8.GetBytes(Enter + "--" + boundary + "--" + Enter);
                    memStream.Write(endBoundary, 0, endBoundary.Length);
                    memStream.Position = 0;
                    var tempBuffer = new byte[memStream.Length];
                    memStream.Read(tempBuffer, 0, tempBuffer.Length);
                    memStream.Close();

                    request.ContentLength = tempBuffer.Length;
                    request.ContentType = "multipart/form-data;boundary=" + boundary;
                    using (Stream reqStream = request.GetRequestStream())
                    {
                        reqStream.Write(tempBuffer, 0, tempBuffer.Length);
                        reqStream.Close();
                    }
                }
                else
                {
                    request.ContentType = "application/x-www-form-urlencoded";
                    byte[] tempBuffer = encoding.GetBytes(getParaStr());//getParaStr即为发送的数据, 
                    request.ContentLength = tempBuffer.Length;
                    using (Stream reqStream = request.GetRequestStream())
                    {
                        reqStream.Write(tempBuffer, 0, tempBuffer.Length);
                        reqStream.Close();
                    }
                }

            }
            else
            {
                request = (HttpWebRequest)WebRequest.Create(url + "?" + getParaStr());
                request.Method = "GET";
                request.ContentType = "text/html; charset=utf-8";
                request.CookieContainer = this.cookies;//加入cookie
            }
            request.Timeout = 5 * 60 * 1000;
            //设置User-Agent,伪装成Google Chrome浏览器
            request.UserAgent = "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36";
            request.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip, deflate, sdch");//定义gzip压缩页面支持
            request.Headers.Add(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.8");
            request.KeepAlive = true;//启用长连接  
            if (proxy != null)
                request.Proxy = proxy;//设置代理服务器IP,伪装请求地址 
            if (!string.IsNullOrEmpty(host))
                request.Host = host;
            if (!string.IsNullOrEmpty(referer))
                request.Referer = referer;
            if (!string.IsNullOrEmpty(accept))
                request.Accept = accept;
            if (!string.IsNullOrEmpty(contentType))
                request.ContentType = contentType;

        }

        /// <summary>
        /// 获取网页内容
        /// </summary>
        /// <returns>网页内容</returns>
        public string GetHtml()
        {
            _init();
            string html = "";
            using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
            {
                if ((response != null) && (response.StatusCode == HttpStatusCode.OK))
                    html = GetResponseBody(response);
                else if (response.StatusCode == HttpStatusCode.Moved)
                {
                    string newUrl = response.Headers.GetValues("Location")[0];
                    JfResquest x = new JfResquest($"{newUrl}");
                    this.reCookies = x.reCookies;
                    return x.GetHtml();
                }
                else
                    throw new Exception("response StatusCode not ok ");
                this.reCookies = response.Cookies;
                response.Close();
            }
            return html;
        }
        public async Task<string> GetHtmlAsync()
        {
            _init();
            string html = "";
            System.GC.Collect();
            using (var response = request.GetResponseAsync())
            {
                await response;
                if ((response != null) && (((HttpWebResponse)response.Result).StatusCode == HttpStatusCode.OK))
                    html = GetResponseBody(((HttpWebResponse)response.Result));
                else if (((HttpWebResponse)response.Result).StatusCode == HttpStatusCode.Moved)
                {
                    string newUrl = ((HttpWebResponse)response.Result).Headers.GetValues("Location")[0];
                    JfResquest x = new JfResquest($"{newUrl}");
                    var h = x.GetHtmlAsync();
                    await h;
                    return h.Result;
                }
                else
                    throw new Exception("response StatusCode not ok ");

                response.Result.Close();
            }
            return html;
        }

        /// <summary>
        /// 下载文件
        /// </summary>
        /// <param name="path">保存地址</param>
        public void GetFile(string path)
        {
            _init();
            using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
            {
                if ((response != null) && (response.StatusCode == HttpStatusCode.OK))
                {
                    using (Stream responseStream = response.GetResponseStream())
                    {
                        byte[] buffer = new byte[4096];
                        using (FileStream fs = System.IO.File.Create(path))
                        {
                            int bytesRead;
                            do
                            {
                                bytesRead = responseStream.Read(buffer, 0, buffer.Length);
                                fs.Write(buffer, 0, bytesRead);
                            }
                            while (bytesRead > 0);
                            fs.Flush();
                        }
                    }
                }
                else if (response.StatusCode == HttpStatusCode.Moved)
                {
                    string newUrl = response.Headers.GetValues("Location")[0];
                    JfResquest x = new JfResquest($"{newUrl}");
                    x.GetFile(path);
                }
                else
                    throw new Exception("response StatusCode not ok ");
            }

        }

        /// <summary>
        /// 获取数据
        /// </summary>       
        protected string GetResponseBody(HttpWebResponse response)
        {
            string responseBody = string.Empty;
            if (response.ContentEncoding.ToLower().Contains("gzip"))
            {
                using (GZipStream stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress))
                {
                    using (StreamReader reader = new StreamReader(stream, this.encoding))
                    {
                        responseBody = reader.ReadToEnd();
                        responseBody = responseBody.Replace("\0", "");
                    }
                }
            }
            else if (response.ContentEncoding.ToLower().Contains("deflate"))
            {
                using (DeflateStream stream = new DeflateStream(response.GetResponseStream(), CompressionMode.Decompress))
                {
                    using (StreamReader reader = new StreamReader(stream, this.encoding))
                    {
                        responseBody = reader.ReadToEnd();
                        responseBody = responseBody.Replace("\0", "");
                    }
                }
            }
            else
            {
                using (Stream stream = response.GetResponseStream())
                {
                    using (StreamReader reader = new StreamReader(stream, this.encoding))
                    {
                        responseBody = reader.ReadToEnd();
                        responseBody = responseBody.Replace("\0", "");
                    }
                }
            }
            return responseBody;
        }

        /// <summary>
        /// 拼接参数
        /// </summary>  
        protected string getParaStr()
        {
            string p = "";
            foreach (var item in para)
                p += $"{item.Key}={item.Value}&";
            p += parastr;
            return p;
        }

    }

 

 

发表评论