trade/NewsCrawler
1
0
Fork 0
Code Issues 1 Pull Requests Releases Activity

- 전자공시 추가

Browse Source 
- 타임아웃 적용
This commit is contained in:
mjjo
2016-08-02 11:56:12 +09:00
parent b35ec4e09a
commit 08e6ff57cd
1 changed files with 162 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

135
Form1.cs
View File

@@ -49,17 +49,19 @@ namespace NewsCrawler
{ {
bool bHasNew = false; bool bHasNew = false;
try
{
string strServerURL = "http://www.asiae.co.kr"; string strServerURL = "http://www.asiae.co.kr";
WebRequest request = WebRequest.Create("http://www.asiae.co.kr/news/sokbo/sokbo_left.htm"); WebRequest request = WebRequest.Create("http://www.asiae.co.kr/news/sokbo/sokbo_left.htm");
//request.Method = "POST";
//request.
request.Credentials=CredentialCache.DefaultCredentials; request.Credentials=CredentialCache.DefaultCredentials;
request.Timeout=2000;
HttpWebResponse response = (HttpWebResponse)request.GetResponse(); using(HttpWebResponse response = (HttpWebResponse)request.GetResponse())
Console.WriteLine(response.StatusDescription); {
using(Stream dataStream = response.GetResponseStream())
Stream dataStream = response.GetResponseStream(); {
StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("EUC-KR")); using(StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("EUC-KR")))
{
string responseFromServer = WebUtility.HtmlDecode(reader.ReadToEnd()); string responseFromServer = WebUtility.HtmlDecode(reader.ReadToEnd());
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument(); HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
@@ -73,20 +75,20 @@ namespace NewsCrawler
string strURL = strServerURL+item.SelectSingleNode(".//a").GetAttributeValue("href", ""); string strURL = strServerURL+item.SelectSingleNode(".//a").GetAttributeValue("href", "");
if (Regex.IsMatch(strTime, @"\d+/\d+") == true) if(Regex.IsMatch(strTime, @"\d+/\d+")==true)
{ {
//Console.WriteLine("어제 기사 : " + item.InnerHtml); //Console.WriteLine("어제 기사 : " + item.InnerHtml);
continue; continue;
} }
if (m_ItemList.Any(c => c.m_strURL==strURL)==false) if(m_ItemList.Any(c => c.m_strURL==strURL)==false)
{ {
NEWS_ITEM news = new NEWS_ITEM(); NEWS_ITEM news = new NEWS_ITEM();
news.m_strTitle=strTitle; news.m_strTitle=strTitle;
news.m_Time=DateTime.ParseExact(strTime, "HH:mm", CultureInfo.CurrentCulture); news.m_Time=DateTime.ParseExact(strTime, "HH:mm", CultureInfo.CurrentCulture);
news.m_strURL=strURL; news.m_strURL=strURL;
news.m_strReference = "아시아경제"; news.m_strReference="아시아경제";
m_ItemList.Add(news); m_ItemList.Add(news);
@@ -94,9 +96,14 @@ namespace NewsCrawler
} }
} }
reader.Close(); }
dataStream.Close(); }
response.Close(); }
}
catch(Exception e)
{
Console.WriteLine(e.ToString());
}
return bHasNew; return bHasNew;
} }
@@ -105,15 +112,19 @@ namespace NewsCrawler
{ {
bool bHasNew = false; bool bHasNew = false;
try
{
//string strServerURL = "http://www.etoday.co.kr"; //string strServerURL = "http://www.etoday.co.kr";
WebRequest request = WebRequest.Create("http://www.etoday.co.kr/news/flash/flash_list.php?tab=2"); WebRequest request = WebRequest.Create("http://www.etoday.co.kr/news/flash/flash_list.php?tab=1");
request.Credentials=CredentialCache.DefaultCredentials; request.Credentials=CredentialCache.DefaultCredentials;
request.Timeout=2000;
HttpWebResponse response = (HttpWebResponse)request.GetResponse(); using(HttpWebResponse response = (HttpWebResponse)request.GetResponse())
Console.WriteLine(response.StatusDescription); {
using(Stream dataStream = response.GetResponseStream())
Stream dataStream = response.GetResponseStream(); {
StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("utf-8")); using(StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("utf-8")))
{
string responseFromServer = WebUtility.HtmlDecode(reader.ReadToEnd()); string responseFromServer = WebUtility.HtmlDecode(reader.ReadToEnd());
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument(); HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
@@ -125,31 +136,102 @@ namespace NewsCrawler
string strTitle = item.SelectSingleNode(".//a").InnerText; string strTitle = item.SelectSingleNode(".//a").InnerText;
string strTime = item.SelectSingleNode(".//span[contains(@class, 'flash_press')]").InnerText; string strTime = item.SelectSingleNode(".//span[contains(@class, 'flash_press')]").InnerText;
string strURL = item.SelectSingleNode(".//a").GetAttributeValue("href", ""); string strURL = item.SelectSingleNode(".//a").GetAttributeValue("href", "");
strURL = "http://www.etoday.co.kr/news/section/newsview.php?idxno="+Regex.Replace(strURL, @"\D", ""); strURL="http://www.etoday.co.kr/news/section/newsview.php?idxno="+Regex.Replace(strURL, @"\D", "");
if (Regex.IsMatch(strTime, @"\d+/\d+") == true) if(Regex.IsMatch(strTime, @"\d+/\d+")==true)
{ {
//Console.WriteLine("어제 기사 : " + item.InnerHtml); //Console.WriteLine("어제 기사 : " + item.InnerHtml);
continue; continue;
} }
if (m_ItemList.Any(c => c.m_strURL==strURL)==false) if(m_ItemList.Any(c => c.m_strURL==strURL)==false)
{ {
NEWS_ITEM news = new NEWS_ITEM(); NEWS_ITEM news = new NEWS_ITEM();
news.m_strTitle=strTitle; news.m_strTitle=strTitle;
news.m_Time=DateTime.ParseExact(strTime, "HH:mm", CultureInfo.CurrentCulture); news.m_Time=DateTime.ParseExact(strTime, "HH:mm", CultureInfo.CurrentCulture);
news.m_strURL=strURL; news.m_strURL=strURL;
news.m_strReference = "이투데이"; news.m_strReference="이투데이";
m_ItemList.Add(news); m_ItemList.Add(news);
bHasNew=true; bHasNew=true;
} }
} }
}
}
}
}
catch(Exception e)
{
Console.WriteLine(e.ToString());
}
reader.Close(); return bHasNew;
dataStream.Close(); }
response.Close();
bool ReadDart()
{
bool bHasNew = false;
try
{
string strServerURL = "https://dart.fss.or.kr";
WebRequest request = WebRequest.Create("https://dart.fss.or.kr/dsac001/mainAll.do");
request.Credentials=CredentialCache.DefaultCredentials;
request.Timeout=2000;
using(HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
using(Stream dataStream = response.GetResponseStream())
{
using(StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("utf-8")))
{
string responseFromServer = WebUtility.HtmlDecode(reader.ReadToEnd());
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(responseFromServer);
string strXPath = "//div[@id='listContents']/div[contains(@class, 'table_list')]/table/tr";
var lists = doc.DocumentNode.SelectNodes(strXPath);
foreach(var item in lists)
{
var rows = item.SelectNodes(".//td");
string strTitle = rows[2].InnerText;
strTitle=strTitle.Trim();
string strTime = item.SelectSingleNode(".//td[contains(@class, 'cen_txt')]").InnerText;
strTime=strTime.Trim();
string strURL = rows[2].SelectSingleNode(".//a").GetAttributeValue("href", "");
strURL=strURL.Trim();
strURL =strServerURL+strURL;
if(Regex.IsMatch(strTime, @"\d+/\d+")==true)
{
//Console.WriteLine("어제 기사 : " + item.InnerHtml);
continue;
}
if(m_ItemList.Any(c => c.m_strURL==strURL)==false)
{
NEWS_ITEM news = new NEWS_ITEM();
news.m_strTitle=strTitle;
news.m_Time=DateTime.ParseExact(strTime, "HH:mm", CultureInfo.CurrentCulture);
news.m_strURL=strURL;
news.m_strReference="전자공시";
m_ItemList.Add(news);
bHasNew=true;
}
}
}
}
}
}
catch(Exception e)
{
Console.WriteLine(e.ToString());
}
return bHasNew; return bHasNew;
} }
@@ -160,6 +242,7 @@ namespace NewsCrawler
if(chAutoReload.Checked == true) if(chAutoReload.Checked == true)
{ {
ReadDart();
ReadAsiaE(); ReadAsiaE();
ReadEtoday(); ReadEtoday();
} }