306 lines
9.2 KiB
C#
306 lines
9.2 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.ComponentModel;
|
|
using System.Data;
|
|
using System.Drawing;
|
|
using System.Globalization;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Net;
|
|
using System.Text;
|
|
using System.Text.RegularExpressions;
|
|
using System.Threading.Tasks;
|
|
using System.Windows.Forms;
|
|
|
|
namespace NewsCrawler
|
|
{
|
|
public partial class Form1 : Form
|
|
{
|
|
class NEWS_ITEM
|
|
{
|
|
public string m_strTitle;
|
|
public DateTime m_Time;
|
|
public string m_strURL;
|
|
public string m_strReference;
|
|
}
|
|
|
|
System.Timers.Timer m_CrawlTimer = new System.Timers.Timer();
|
|
int m_iCrawlInterval = 500;
|
|
Timer m_UITimer = new Timer();
|
|
List<NEWS_ITEM> m_ItemList = new List<NEWS_ITEM>();
|
|
int m_iLastItemCnt = 0;
|
|
|
|
public Form1()
|
|
{
|
|
InitializeComponent();
|
|
|
|
wbView.ScriptErrorsSuppressed = false;
|
|
|
|
m_CrawlTimer.Elapsed+=CrawlTimer_Tick;
|
|
m_CrawlTimer.Interval=1000;
|
|
m_CrawlTimer.Start();
|
|
|
|
m_UITimer.Tick += UITimer_Tick;
|
|
m_UITimer.Interval = m_iCrawlInterval;
|
|
m_UITimer.Start();
|
|
}
|
|
|
|
bool ReadAsiaE()
|
|
{
|
|
bool bHasNew = false;
|
|
|
|
try
|
|
{
|
|
string strServerURL = "http://www.asiae.co.kr";
|
|
WebRequest request = WebRequest.Create("http://www.asiae.co.kr/news/sokbo/sokbo_left.htm");
|
|
request.Credentials=CredentialCache.DefaultCredentials;
|
|
request.Timeout=2000;
|
|
|
|
using(HttpWebResponse response = (HttpWebResponse)request.GetResponse())
|
|
{
|
|
using(Stream dataStream = response.GetResponseStream())
|
|
{
|
|
using(StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("EUC-KR")))
|
|
{
|
|
string responseFromServer = WebUtility.HtmlDecode(reader.ReadToEnd());
|
|
|
|
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
|
|
doc.LoadHtml(responseFromServer);
|
|
string strXPath = "//div[contains(@class, 'txtform')]/ul/li";
|
|
var lists = doc.DocumentNode.SelectNodes(strXPath);
|
|
foreach(var item in lists)
|
|
{
|
|
string strTitle = item.SelectSingleNode(".//a").GetAttributeValue("title", "");
|
|
string strTime = item.SelectSingleNode(".//span").InnerText;
|
|
string strURL = strServerURL+item.SelectSingleNode(".//a").GetAttributeValue("href", "");
|
|
|
|
|
|
if(Regex.IsMatch(strTime, @"\d+/\d+")==true)
|
|
{
|
|
//Console.WriteLine("어제 기사 : " + item.InnerHtml);
|
|
continue;
|
|
}
|
|
|
|
|
|
if(m_ItemList.Any(c => c.m_strURL==strURL)==false)
|
|
{
|
|
NEWS_ITEM news = new NEWS_ITEM();
|
|
news.m_strTitle=strTitle;
|
|
news.m_Time=DateTime.ParseExact(strTime, "HH:mm", CultureInfo.CurrentCulture);
|
|
news.m_strURL=strURL;
|
|
news.m_strReference="아시아경제";
|
|
|
|
m_ItemList.Add(news);
|
|
|
|
bHasNew=true;
|
|
}
|
|
}
|
|
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch(Exception e)
|
|
{
|
|
Console.WriteLine(e.ToString());
|
|
}
|
|
|
|
return bHasNew;
|
|
}
|
|
|
|
bool ReadEtoday()
|
|
{
|
|
bool bHasNew = false;
|
|
|
|
try
|
|
{
|
|
//string strServerURL = "http://www.etoday.co.kr";
|
|
WebRequest request = WebRequest.Create("http://www.etoday.co.kr/news/flash/flash_list.php?tab=1");
|
|
request.Credentials=CredentialCache.DefaultCredentials;
|
|
request.Timeout=2000;
|
|
|
|
using(HttpWebResponse response = (HttpWebResponse)request.GetResponse())
|
|
{
|
|
using(Stream dataStream = response.GetResponseStream())
|
|
{
|
|
using(StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("utf-8")))
|
|
{
|
|
string responseFromServer = WebUtility.HtmlDecode(reader.ReadToEnd());
|
|
|
|
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
|
|
doc.LoadHtml(responseFromServer);
|
|
string strXPath = "//div[contains(@class, 'flash_tab_lst')]/ul/li";
|
|
var lists = doc.DocumentNode.SelectNodes(strXPath);
|
|
foreach(var item in lists)
|
|
{
|
|
string strTitle = item.SelectSingleNode(".//a").InnerText;
|
|
string strTime = item.SelectSingleNode(".//span[contains(@class, 'flash_press')]").InnerText;
|
|
string strURL = item.SelectSingleNode(".//a").GetAttributeValue("href", "");
|
|
strURL="http://www.etoday.co.kr/news/section/newsview.php?idxno="+Regex.Replace(strURL, @"\D", "");
|
|
|
|
if(Regex.IsMatch(strTime, @"\d+/\d+")==true)
|
|
{
|
|
//Console.WriteLine("어제 기사 : " + item.InnerHtml);
|
|
continue;
|
|
}
|
|
|
|
if(m_ItemList.Any(c => c.m_strURL==strURL)==false)
|
|
{
|
|
NEWS_ITEM news = new NEWS_ITEM();
|
|
news.m_strTitle=strTitle;
|
|
news.m_Time=DateTime.ParseExact(strTime, "HH:mm", CultureInfo.CurrentCulture);
|
|
news.m_strURL=strURL;
|
|
news.m_strReference="이투데이";
|
|
|
|
m_ItemList.Add(news);
|
|
|
|
bHasNew=true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch(Exception e)
|
|
{
|
|
Console.WriteLine(e.ToString());
|
|
}
|
|
|
|
return bHasNew;
|
|
}
|
|
|
|
bool ReadDart()
|
|
{
|
|
bool bHasNew = false;
|
|
|
|
try
|
|
{
|
|
string strServerURL = "https://dart.fss.or.kr";
|
|
WebRequest request = WebRequest.Create("https://dart.fss.or.kr/dsac001/mainAll.do");
|
|
request.Credentials=CredentialCache.DefaultCredentials;
|
|
request.Timeout=2000;
|
|
|
|
using(HttpWebResponse response = (HttpWebResponse)request.GetResponse())
|
|
{
|
|
using(Stream dataStream = response.GetResponseStream())
|
|
{
|
|
using(StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("utf-8")))
|
|
{
|
|
string responseFromServer = WebUtility.HtmlDecode(reader.ReadToEnd());
|
|
|
|
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
|
|
doc.LoadHtml(responseFromServer);
|
|
|
|
string strXPath = "//div[@id='listContents']/div[contains(@class, 'table_list')]/table/tr";
|
|
var lists = doc.DocumentNode.SelectNodes(strXPath);
|
|
foreach(var item in lists)
|
|
{
|
|
var rows = item.SelectNodes(".//td");
|
|
|
|
string strTitle = rows[2].InnerText;
|
|
strTitle=strTitle.Trim();
|
|
string strTime = item.SelectSingleNode(".//td[contains(@class, 'cen_txt')]").InnerText;
|
|
strTime=strTime.Trim();
|
|
string strURL = rows[2].SelectSingleNode(".//a").GetAttributeValue("href", "");
|
|
strURL=strURL.Trim();
|
|
strURL =strServerURL+strURL;
|
|
|
|
if(Regex.IsMatch(strTime, @"\d+/\d+")==true)
|
|
{
|
|
//Console.WriteLine("어제 기사 : " + item.InnerHtml);
|
|
continue;
|
|
}
|
|
|
|
if(m_ItemList.Any(c => c.m_strURL==strURL)==false)
|
|
{
|
|
NEWS_ITEM news = new NEWS_ITEM();
|
|
news.m_strTitle=strTitle;
|
|
news.m_Time=DateTime.ParseExact(strTime, "HH:mm", CultureInfo.CurrentCulture);
|
|
news.m_strURL=strURL;
|
|
news.m_strReference="전자공시";
|
|
|
|
m_ItemList.Add(news);
|
|
|
|
bHasNew=true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch(Exception e)
|
|
{
|
|
Console.WriteLine(e.ToString());
|
|
}
|
|
|
|
return bHasNew;
|
|
}
|
|
|
|
private void CrawlTimer_Tick(object sender, EventArgs e)
|
|
{
|
|
m_CrawlTimer.Enabled = false;
|
|
|
|
if(chAutoReload.Checked == true)
|
|
{
|
|
ReadDart();
|
|
ReadAsiaE();
|
|
ReadEtoday();
|
|
}
|
|
|
|
m_CrawlTimer.Interval = m_iCrawlInterval;
|
|
m_CrawlTimer.Enabled = true;
|
|
}
|
|
|
|
private void UITimer_Tick(object sender, EventArgs e)
|
|
{
|
|
if (m_iLastItemCnt == m_ItemList.Count)
|
|
return;
|
|
|
|
m_ItemList.Sort((NEWS_ITEM news1, NEWS_ITEM news2) => news1.m_Time.CompareTo(news2.m_Time));
|
|
|
|
foreach (var item in m_ItemList)
|
|
{
|
|
string strResult = string.Format("[{0}] {1} ({2})", item.m_Time, item.m_strTitle, item.m_strURL);
|
|
lvList.Items.Add(new ListViewItem(new string[] { item.m_Time.ToString("HH:mm:ss"), item.m_strTitle, "", "", item.m_strReference, item.m_strURL }));
|
|
|
|
//Console.WriteLine(strResult);
|
|
//System.IO.File.AppendAllText("output.txt", strResult+"\n", Encoding.GetEncoding("EUC-KR"));
|
|
}
|
|
|
|
if (chAutoSelect.Checked == true)
|
|
{
|
|
lvList.Items[lvList.Items.Count - 1].Selected = true;
|
|
lvList.Select();
|
|
lvList.SelectedItems[0].EnsureVisible();
|
|
}
|
|
|
|
foreach (ColumnHeader col in lvList.Columns)
|
|
col.Width = -2;
|
|
|
|
m_iLastItemCnt = m_ItemList.Count;
|
|
}
|
|
|
|
private void lvList_SelectedIndexChanged(object sender, EventArgs e)
|
|
{
|
|
if (lvList.SelectedItems.Count <= 0)
|
|
return;
|
|
|
|
string strURL = lvList.SelectedItems[0].SubItems[chLink.Index].Text;
|
|
wbView.ScriptErrorsSuppressed = true;
|
|
wbView.Navigate(strURL);
|
|
}
|
|
|
|
private void tbInterval_KeyPress(object sender, KeyPressEventArgs e)
|
|
{
|
|
if(e.KeyChar == Convert.ToChar(Keys.Enter))
|
|
{
|
|
m_iCrawlInterval = (int)(float.Parse(Regex.Replace(tbInterval.Text, @"\D\.", "")) * 1000);
|
|
if (m_iCrawlInterval < 1)
|
|
m_iCrawlInterval = 500;
|
|
|
|
tbInterval.Text = (m_iCrawlInterval / (float)1000).ToString("##0.0") + "초";
|
|
}
|
|
}
|
|
}
|
|
}
|