利用C#实现网络爬虫

 private void DispatchWork() { if (_stop) //判断是否中止下载 { return; } for (int i = 0; i <_reqCount; i++) { if (!_reqsBusy[i]) //判断此编号的工作实例是否空闲 { RequestResource(i); //让此工作实例请求资源 } } } 

 private bool[] _reqsBusy = null; //每个元素代表一个工作实例是否正在工作 private int _reqCount = 4; //工作实例的数量 

 private void RequestResource(int index) { int depth; string url = ""; try { lock (_locker) { if (_urlsUnload.Count <= 0) //判断是否还有未下载的URL { _workingSignals.FinishWorking(index); //设置工作实例的状态为Finished return; } _reqsBusy[index] = true; _workingSignals.StartWorking(index); //设置工作状态为Working depth = _urlsUnload.First().Value; //取出第一个未下载的URL url = _urlsUnload.First().Key; _urlsLoaded.Add(url, depth); //把该URL加入到已下载里 _urlsUnload.Remove(url); //把该URL从未下载中移除 } HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url); req.Method = _method; //请求方法 req.Accept = _accept; //接受的内容 req.UserAgent = _userAgent; //用户代理 RequestState rs = new RequestState(req, url, depth, index); //回调方法的参数 var result = req.BeginGetResponse(new AsyncCallback(ReceivedResource), rs); //异步请求 ThreadPool.RegisterWaitForSingleObject(result.AsyncWaitHandle, //注册超时处理方法 TimeoutCallback, rs, _maxTime, true); } catch (WebException we) { MessageBox.Show("RequestResource " + we.Message + url + we.Status); } } 

 class RequestState { private const int BUFFER_SIZE = 131072; //接收数据包的空间大小 private byte[] _data = new byte[BUFFER_SIZE]; //接收数据包的buffer private StringBuilder _sb = new StringBuilder(); //存放所有接收到的字符 public HttpWebRequest Req { get; private set; } //请求 public string Url { get; private set; } //请求的URL public int Depth { get; private set; } //此次请求的相对深度 public int Index { get; private set; } //工作实例的编号 public Stream ResStream { get; set; } //接收数据流 public StringBuilder Html { get { return _sb; } } public byte[] Data { get { return _data; } } public int BufferSize { get { return BUFFER_SIZE; } } public RequestState(HttpWebRequest req, string url, int depth, int index) { Req = req; Url = url; Depth = depth; Index = index; } } 

 private void TimeoutCallback(object state, bool timedOut) { if (timedOut) //判断是否是超时 { RequestState rs = state as RequestState; if (rs != null) { rs.Req.Abort(); //撤销请求 } _reqsBusy[rs.Index] = false; //重置工作状态 DispatchWork(); //分配新任务 } } 

 private void ReceivedResource(IAsyncResult ar) { RequestState rs = (RequestState)ar.AsyncState; //得到请求时传入的参数 HttpWebRequest req = rs.Req; string url = rs.Url; try { HttpWebResponse res = (HttpWebResponse)req.EndGetResponse(ar); //获取响应 if (_stop) //判断是否中止下载 { res.Close(); req.Abort(); return; } if (res != null && res.StatusCode == HttpStatusCode.OK) //判断是否成功获取响应 { Stream resStream = res.GetResponseStream(); //得到资源流 rs.ResStream = resStream; var result = resStream.BeginRead(rs.Data, 0, rs.BufferSize, //异步请求读取数据 new AsyncCallback(ReceivedData), rs); } else //响应失败 { res.Close(); rs.Req.Abort(); _reqsBusy[rs.Index] = false; //重置工作状态 DispatchWork(); //分配新任务 } } catch (WebException we) { MessageBox.Show("ReceivedResource " + we.Message + url + we.Status); } } 

 private void ReceivedData(IAsyncResult ar) { RequestState rs = (RequestState)ar.AsyncState; //获取参数 HttpWebRequest req = rs.Req; Stream resStream = rs.ResStream; string url = rs.Url; int depth = rs.Depth; string html = null; int index = rs.Index; int read = 0; try { read = resStream.EndRead(ar); //获得数据读取结果 if (_stop)//判断是否中止下载 { rs.ResStream.Close(); req.Abort(); return; } if (read > 0) { MemoryStream ms = new MemoryStream(rs.Data, 0, read); //利用获得的数据创建内存流 StreamReader reader = new StreamReader(ms, _encoding); string str = reader.ReadToEnd(); //读取所有字符 rs.Html.Append(str); // 添加到之前的末尾 var result = resStream.BeginRead(rs.Data, 0, rs.BufferSize, //再次异步请求读取数据 new AsyncCallback(ReceivedData), rs); return; } html = rs.Html.ToString(); SaveContents(html, url); //保存到本地 string[] links = GetLinks(html); //获取页面中的链接 AddUrls(links, depth + 1); //过滤链接并添加到未下载集合中 _reqsBusy[index] = false; //重置工作状态 DispatchWork(); //分配新任务 } catch (WebException we) { MessageBox.Show("ReceivedData Web " + we.Message + url + we.Status); } } 

 private void StartDownload() { _checkTimer = new Timer(new TimerCallback(CheckFinish), null, 0, 300); DispatchWork(); } private void CheckFinish(object param) { if (_workingSignals.IsFinished()) //检查是否所有工作实例都为Finished { _checkTimer.Dispose(); //停止定时器 _checkTimer = null; if (DownloadFinish != null && _ui != null) //判断是否注册了完成事件 { _ui.Dispatcher.Invoke(DownloadFinish, _index); //调用事件 } } } 

 public delegate void DownloadFinishHandler(int count); /// 
 /// 全部链接下载分析完毕后触发 /// 
 public event DownloadFinishHandler DownloadFinish = null; 

 private void SaveContents(string html, string url) { if (string.IsNullOrEmpty(html)) //判断html字符串是否有效 { return; } string path = string.Format("{0}\\{1}.txt", _path, _index++); //生成文件名 try { using (StreamWriter fs = new StreamWriter(path)) { fs.Write(html); //写文件 } } catch (IOException ioe) { MessageBox.Show("SaveContents IO" + ioe.Message + " path=" + path); } if (ContentsSaved != null) { _ui.Dispatcher.Invoke(ContentsSaved, path, url); //调用保存文件事件 } } 

 public delegate void ContentsSavedHandler(string path, string url); /// 
 /// 文件被保存到本地后触发 /// 
 public event ContentsSavedHandler ContentsSaved = null; 

 private string[] GetLinks(string html) { const string pattern = @"http://([\w-]+\.)+[\w-]+(/[\w- ./?%&=]*)?"; Regex r = new Regex(pattern, RegexOptions.IgnoreCase); //新建正则模式 MatchCollection m = r.Matches(html); //获得匹配结果 string[] links = new string[m.Count]; for (int i = 0; i 

 //判断链接是否已经下载或者已经处于未下载集合中 private bool UrlExists(string url) { bool result = _urlsUnload.ContainsKey(url); result |= _urlsLoaded.ContainsKey(url); return result; } private bool UrlAvailable(string url) { if (UrlExists(url)) { return false; //已经存在 } if (url.Contains(".jpg-600") || url.Contains(".gif") || url.Contains(".png-600") || url.Contains(".css") || url.Contains(".js")) { return false; //去掉一些图片之类的资源 } return true; } private void AddUrls(string[] urls, int depth) { if (depth >= _maxDepth) { return; //深度过大 } foreach (string url in urls) { string cleanUrl = url.Trim(); //去掉前后空格 cleanUrl = cleanUrl.TrimEnd('/'); //统一去掉最后面的'/' if (UrlAvailable(cleanUrl)) { if (cleanUrl.Contains(_baseUrl)) { _urlsUnload.Add(cleanUrl, depth); //是内链，直接加入未下载集合 } else { // 外链处理 } } } } 

 /// 
 /// 下载根Url /// 
 public string RootUrl { get { return _rootUrl; } set { if (!value.Contains("http://")) { _rootUrl = "http://" + value; } else { _rootUrl = value; } _baseUrl = _rootUrl.Replace("www.", ""); //全站的话去掉www _baseUrl = _baseUrl.Replace("http://", ""); //去掉协议名 _baseUrl = _baseUrl.TrimEnd('/'); //去掉末尾的'/' } }