using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;
using Common.Utils;
using Utils;
namespace 爬虫{
public partial class Form1 : Form {
#region 变量 /// <summary> /// 已完成字节数 /// </summary> private long completedCount = 0;
/// <summary> /// 是否完成 /// </summary> private bool isCompleted = true;
/// <summary> /// 数据块队列 /// </summary> private ConcurrentQueue<MemoryStream> msQueue = new ConcurrentQueue<MemoryStream>();
/// <summary> /// 下载开始位置 /// </summary> private long range = 0;
/// <summary> /// 文件大小 /// </summary> private long total = 0;
/// <summary> /// 一段时间内的完成节点数，计算网速用 /// </summary> private long unitCount = 0;
/// <summary> /// 上次计时时间，计算网速用 /// </summary> private DateTime lastTime = DateTime.MinValue;
/// <summary> /// 一段时间内的完成字节数，控制网速用 /// </summary> private long unitCountForLimit = 0;
/// <summary> /// 上次计时时间，控制网速用 /// </summary> private DateTime lastTimeForLimit = DateTime.MinValue;
/// <summary> /// 下载文件sleep时间，控制速度用 /// </summary> private int sleepTime = 1;
#endregion #region Form1 public Form1() {
InitializeComponent();
}
#endregion #region Form1_Load private void Form1_Load(object sender, EventArgs e) {
lblMsg.Text = string.Empty;
lblByteMsg.Text = string.Empty;
lblSpeed.Text = string.Empty;
}
#endregion #region Form1_FormClosing private void Form1_FormClosing(object sender, FormClosingEventArgs e) {
}
#endregion #region btnDownload_Click 下载 private void btnDownload_Click(object sender, EventArgs e) {
isCompleted = false;
btnDownload.Enabled = false;
string url = txtUrl.Text.Trim();
string filePath = CreateFilePath(url);
#region 下载线程 Thread thread = new Thread(new ThreadStart(() => {
int tryTimes = 0;
while (!HttpDownloadFile(url, filePath)) {
Thread.Sleep(10000);
tryTimes++;
LogUtil.Log("请求服务器失败，重新请求" + tryTimes.ToString() + "次");
this.Invoke(new InvokeDelegate(() => {
lblMsg.Text = "请求服务器失败，重新请求" + tryTimes.ToString() + "次";
}
));
HttpDownloadFile(url, filePath);
}
}
));
thread.IsBackground = true;
thread.Start();
#endregion #region 保存文件线程 thread = new Thread(new ThreadStart(() => {
while (!isCompleted) {
MemoryStream ms;
if (msQueue.TryDequeue(out ms)) {
using (FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Write)) {
fs.Seek(completedCount, SeekOrigin.Begin);
fs.Write(ms.ToArray(), 0, (int)ms.Length);
fs.Close();
}
completedCount += ms.Length;
}
if (total != 0 && total == completedCount) {
Thread.Sleep(100);
isCompleted = true;
}
Thread.Sleep(1);
}
}
));
thread.IsBackground = true;
thread.Start();
#endregion #region 计算网速/进度线程 thread = new Thread(new ThreadStart(() => {
while (!isCompleted) {
Thread.Sleep(1000);
if (lastTime != DateTime.MinValue) {
double sec = DateTime.Now.Subtract(lastTime).TotalSeconds;
double speed = unitCount / sec / 1024;
try {
#region 显示速度 if (speed < 1024) {
this.Invoke(new InvokeDelegate(() => {
lblSpeed.Text = string.Format("{
0}
KB/S", speed.ToString("0.00"));
}
));
}
else {
this.Invoke(new InvokeDelegate(() => {
lblSpeed.Text = string.Format("{
0}
MB/S", (speed / 1024).ToString("0.00"));
}
));
}
#endregion #region 显示进度 this.Invoke(new InvokeDelegate(() => {
string strTotal = (total / 1024 / 1024).ToString("0.00") + "MB";
if (total < 1024 * 1024) {
strTotal = (total / 1024).ToString("0.00") + "KB";
}
string completed = (completedCount / 1024 / 1024).ToString("0.00") + "MB";
if (completedCount < 1024 * 1024) {
completed = (completedCount / 1024).ToString("0.00") + "KB";
}
lblMsg.Text = string.Format("进度：{
0}
/{
1}
", completed, strTotal);
lblByteMsg.Text = string.Format("已下载：{
0}
rn总大小：{
1}
", completedCount, total);
if (completedCount == total) {
MessageBox.Show("完成");
}
}
));
#endregion }
catch {
}
lastTime = DateTime.Now;
unitCount = 0;
}
}
}
));
thread.IsBackground = true;
thread.Start();
#endregion #region 限制网速线程 thread = new Thread(new ThreadStart(() => {
while (!isCompleted) {
Thread.Sleep(100);
if (lastTimeForLimit != DateTime.MinValue) {
double sec = DateTime.Now.Subtract(lastTimeForLimit).TotalSeconds;
double speed = unitCountForLimit / sec / 1024;
try {
#region 限速/解除限速 double limitSpeed = 0;
if (double.TryParse(txtSpeed.Text.Trim(), out limitSpeed)) {
if (speed > limitSpeed && sleepTime < 1000) {
sleepTime += 1;
}
if (speed < limitSpeed - 10 && sleepTime >= 2) {
sleepTime -= 1;
}
}
else {
this.Invoke(new InvokeDelegate(() => {
txtSpeed.Text = "100";
}
));
}
#endregion }
catch {
}
lastTimeForLimit = DateTime.Now;
unitCountForLimit = 0;
}
}
}
));
thread.IsBackground = true;
thread.Start();
#endregion }
#endregion #region HttpDownloadFile 下载文件 /// <summary> /// Http下载文件 /// </summary> public bool HttpDownloadFile(string url, string filePath) {
try {
if (!File.Exists(filePath)) {
using (FileStream fs = new FileStream(filePath, FileMode.Create)) {
fs.Close();
}
}
else {
FileInfo fileInfo = new FileInfo(filePath);
range = fileInfo.Length;
}
// 设置参数 HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest;
request.UserAgent = "Mozilla/4.0 (compatible;
MSIE 8.0;
Windows NT 6.0;
Trident/4.0)";
request.Proxy = null;
//发送请求并获取相应回应数据 HttpWebResponse response = request.GetResponse() as HttpWebResponse;
if (response.ContentLength == range) {
this.Invoke(new InvokeDelegate(() => {
lblMsg.Text = "文件已下载";
}
));
return true;
}
// 设置参数 request = WebRequest.Create(url) as HttpWebRequest;
request.UserAgent = "Mozilla/4.0 (compatible;
MSIE 8.0;
Windows NT 6.0;
Trident/4.0)";
request.Proxy = null;
request.AddRange(range);
//发送请求并获取相应回应数据 response = request.GetResponse() as HttpWebResponse;
//直到request.GetResponse()程序才开始向目标网页发送Post请求 Stream responseStream = response.GetResponseStream();
total = range + response.ContentLength;
completedCount = range;
MemoryStream ms = new MemoryStream();
byte[] bArr = new byte[1024];
lastTime = DateTime.Now;
lastTimeForLimit = DateTime.Now;
int size = responseStream.Read(bArr, 0, (int)bArr.Length);
unitCount += size;
unitCountForLimit += size;
ms.Write(bArr, 0, size);
while (!isCompleted) {
size = responseStream.Read(bArr, 0, (int)bArr.Length);
unitCount += size;
unitCountForLimit += size;
ms.Write(bArr, 0, size);
if (ms.Length > 102400) {
msQueue.Enqueue(ms);
ms = new MemoryStream();
}
if (completedCount + ms.Length == total) {
msQueue.Enqueue(ms);
ms = new MemoryStream();
}
Thread.Sleep(sleepTime);
}
responseStream.Close();
return true;
}
catch (Exception ex) {
LogUtil.LogError(ex.Message + "rn" + ex.StackTrace);
return false;
}
}
#endregion #region 根据URL生成文件保存路径 private string CreateFilePath(string url) {
string path = Application.StartupPath + "\download";
if (!Directory.Exists(path)) {
Directory.CreateDirectory(path);
}
string fileName = Path.GetFileName(url);
if (fileName.IndexOf("?") > 0) {
return path + "\" + fileName.Substring(0, fileName.IndexOf("?"));
}
else {
return path + "\" + fileName;
}
}
#endregion }
//end Form1类}