trade/NewsCrawler
1
0
Fork 0
Code Issues 1 Pull Requests Releases Activity

- 전자공시 추가

Browse Source 
- 타임아웃 적용
This commit is contained in:
mjjo
2016-08-02 11:56:12 +09:00
parent b35ec4e09a
commit 08e6ff57cd
1 changed files with 162 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

135
Form1.cs
View File

@@ -49,17 +49,19 @@ namespace NewsCrawler
{
bool bHasNew = false;
try
{
string strServerURL = "http://www.asiae.co.kr";
WebRequest request = WebRequest.Create("http://www.asiae.co.kr/news/sokbo/sokbo_left.htm");
//request.Method = "POST";
//request.
request.Credentials=CredentialCache.DefaultCredentials;
request.Timeout=2000;
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
Console.WriteLine(response.StatusDescription);
Stream dataStream = response.GetResponseStream();
StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("EUC-KR"));
using(HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
using(Stream dataStream = response.GetResponseStream())
{
using(StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("EUC-KR")))
{
string responseFromServer = WebUtility.HtmlDecode(reader.ReadToEnd());
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
@@ -73,20 +75,20 @@ namespace NewsCrawler
string strURL = strServerURL+item.SelectSingleNode(".//a").GetAttributeValue("href", "");
if (Regex.IsMatch(strTime, @"\d+/\d+") == true)
if(Regex.IsMatch(strTime, @"\d+/\d+")==true)
{
//Console.WriteLine("어제 기사 : " + item.InnerHtml);
continue;
}
if (m_ItemList.Any(c => c.m_strURL==strURL)==false)
if(m_ItemList.Any(c => c.m_strURL==strURL)==false)
{
NEWS_ITEM news = new NEWS_ITEM();
news.m_strTitle=strTitle;
news.m_Time=DateTime.ParseExact(strTime, "HH:mm", CultureInfo.CurrentCulture);
news.m_strURL=strURL;
news.m_strReference = "아시아경제";
news.m_strReference="아시아경제";
m_ItemList.Add(news);
@@ -94,9 +96,14 @@ namespace NewsCrawler
}
}
reader.Close();
dataStream.Close();
response.Close();
}
}
}
}
catch(Exception e)
{
Console.WriteLine(e.ToString());
}
return bHasNew;
}
@@ -105,15 +112,19 @@ namespace NewsCrawler
{
bool bHasNew = false;
try
{
//string strServerURL = "http://www.etoday.co.kr";
WebRequest request = WebRequest.Create("http://www.etoday.co.kr/news/flash/flash_list.php?tab=2");
WebRequest request = WebRequest.Create("http://www.etoday.co.kr/news/flash/flash_list.php?tab=1");
request.Credentials=CredentialCache.DefaultCredentials;
request.Timeout=2000;
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
Console.WriteLine(response.StatusDescription);
Stream dataStream = response.GetResponseStream();
StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("utf-8"));
using(HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
using(Stream dataStream = response.GetResponseStream())
{
using(StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("utf-8")))
{
string responseFromServer = WebUtility.HtmlDecode(reader.ReadToEnd());
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
@@ -125,31 +136,102 @@ namespace NewsCrawler
string strTitle = item.SelectSingleNode(".//a").InnerText;
string strTime = item.SelectSingleNode(".//span[contains(@class, 'flash_press')]").InnerText;
string strURL = item.SelectSingleNode(".//a").GetAttributeValue("href", "");
strURL = "http://www.etoday.co.kr/news/section/newsview.php?idxno="+Regex.Replace(strURL, @"\D", "");
strURL="http://www.etoday.co.kr/news/section/newsview.php?idxno="+Regex.Replace(strURL, @"\D", "");
if (Regex.IsMatch(strTime, @"\d+/\d+") == true)
if(Regex.IsMatch(strTime, @"\d+/\d+")==true)
{
//Console.WriteLine("어제 기사 : " + item.InnerHtml);
continue;
}
if (m_ItemList.Any(c => c.m_strURL==strURL)==false)
if(m_ItemList.Any(c => c.m_strURL==strURL)==false)
{
NEWS_ITEM news = new NEWS_ITEM();
news.m_strTitle=strTitle;
news.m_Time=DateTime.ParseExact(strTime, "HH:mm", CultureInfo.CurrentCulture);
news.m_strURL=strURL;
news.m_strReference = "이투데이";
news.m_strReference="이투데이";
m_ItemList.Add(news);
bHasNew=true;
}
}
}
}
}
}
catch(Exception e)
{
Console.WriteLine(e.ToString());
}
reader.Close();
dataStream.Close();
response.Close();
return bHasNew;
}
bool ReadDart()
{
bool bHasNew = false;
try
{
string strServerURL = "https://dart.fss.or.kr";
WebRequest request = WebRequest.Create("https://dart.fss.or.kr/dsac001/mainAll.do");
request.Credentials=CredentialCache.DefaultCredentials;
request.Timeout=2000;
using(HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
using(Stream dataStream = response.GetResponseStream())
{
using(StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("utf-8")))
{
string responseFromServer = WebUtility.HtmlDecode(reader.ReadToEnd());
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(responseFromServer);
string strXPath = "//div[@id='listContents']/div[contains(@class, 'table_list')]/table/tr";
var lists = doc.DocumentNode.SelectNodes(strXPath);
foreach(var item in lists)
{
var rows = item.SelectNodes(".//td");
string strTitle = rows[2].InnerText;
strTitle=strTitle.Trim();
string strTime = item.SelectSingleNode(".//td[contains(@class, 'cen_txt')]").InnerText;
strTime=strTime.Trim();
string strURL = rows[2].SelectSingleNode(".//a").GetAttributeValue("href", "");
strURL=strURL.Trim();
strURL =strServerURL+strURL;
if(Regex.IsMatch(strTime, @"\d+/\d+")==true)
{
//Console.WriteLine("어제 기사 : " + item.InnerHtml);
continue;
}
if(m_ItemList.Any(c => c.m_strURL==strURL)==false)
{
NEWS_ITEM news = new NEWS_ITEM();
news.m_strTitle=strTitle;
news.m_Time=DateTime.ParseExact(strTime, "HH:mm", CultureInfo.CurrentCulture);
news.m_strURL=strURL;
news.m_strReference="전자공시";
m_ItemList.Add(news);
bHasNew=true;
}
}
}
}
}
}
catch(Exception e)
{
Console.WriteLine(e.ToString());
}
return bHasNew;
}
@@ -160,6 +242,7 @@ namespace NewsCrawler
if(chAutoReload.Checked == true)
{
ReadDart();
ReadAsiaE();
ReadEtoday();
}