trade/NewsCrawler
1
0
Fork 0
Code Issues 1 Pull Requests Releases Activity

- 이투데이 rss 추가

Browse Source 
- 요청 시간, 받은 시간 추가
- CYBOS 뉴스 추가
- 중복종목 버그 수정
- 실행시 키워드 수 추가
This commit is contained in:
mjjo
2016-12-27 10:49:19 +09:00
parent a93faf9eb5
commit a93d3a8f45
5 changed files with 237 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

171
NewsForm.cs
View File

@@ -22,8 +22,6 @@ namespace NewsCrawler
TextCondition m_Condition = null;
ConfigForm m_ConfigForm = null;
delegate void InsertListView(string strTitle, DateTime time, string strURL, string strRef, bool bInitial);
System.Timers.Timer m_CrawlTimer = new System.Timers.Timer();
int m_iCrawlInterval = 500;
bool m_bBrowserReload = false;
@@ -39,6 +37,7 @@ namespace NewsCrawler
Util.SetLogView(tbLog);
m_CodeList = new CodeList();
m_Condition = new TextCondition();
MessageBox.Show("Keywords : \n\n" + m_Condition.GetKeywordsCnt());
wbView.ScriptErrorsSuppressed = false;
@@ -46,6 +45,7 @@ namespace NewsCrawler
ReadKIND(true);
ReadDart(true);
ReadEtoday(true);
ReadEToday2(true);
ReadAsiaE(true);
ReadEdaily(true);
ReadMoneyToday(true);
@@ -71,7 +71,7 @@ namespace NewsCrawler
m_CrawlTimer.Start();
m_CybosHelper = new CybosHelper();
m_CybosHelper = new CybosHelper(this);
Test();
}
@@ -81,10 +81,10 @@ namespace NewsCrawler
if(Util.IsDebugging() == false)
return;
InsertItem("[test] 덕산하이메탈, 덕산네오룩스 66만여주 취득14:38", new DateTime(2016, 11, 30, 00, 00, 00), "http://www.asiae.co.kr/news/sokbo/sokbo_view.htm?idxno=2016112914371817318", "asiae", false);
InsertItem("[test] 자연과환경, 12월15일~22일 주주명부폐쇄14:19", new DateTime(2016, 11, 30, 00, 00, 00), "http://www.asiae.co.kr/news/sokbo/sokbo_view.htm?idxno=2016112914193170301", "asiae", false);
InsertItem("[test] 이엠코리아, 한국항공우주산업과 3억원 규모 공급계약14:06", new DateTime(2016, 11, 30, 00, 00, 00), "http://www.asiae.co.kr/news/sokbo/sokbo_view.htm?idxno=2016112914055964082", "asiae", false);
InsertItem("[test] 덕산하이메탈, 덕산네오룩스 66만여주 취득14:38", "", "", new DateTime(2016, 11, 30, 00, 00, 00), DateTime.Now, "http://www.asiae.co.kr/news/sokbo/sokbo_view.htm?idxno=2016112914371817318", "asiae", 0, false);
InsertItem("[test] 자연과환경, 12월15일~22일 주주명부폐쇄14:19", "", "", new DateTime(2016, 11, 30, 00, 00, 00), DateTime.Now, "http://www.asiae.co.kr/news/sokbo/sokbo_view.htm?idxno=2016112914193170301", "asiae", 0, false);
InsertItem("[test] 이엠코리아, 한국항공우주산업과 3억원 규모 공급계약14:06", "", "", new DateTime(2016, 11, 30, 00, 00, 00), DateTime.Now, "http://www.asiae.co.kr/news/sokbo/sokbo_view.htm?idxno=2016112914055964082", "asiae", 0, false);
InsertItem("[test] [특징주] 진원생명과학, 국외 AI 인체감염증...", "", "", new DateTime(2016, 11, 30, 00, 00, 00), DateTime.Now, "http://www.etoday.co.kr/news/sokbo/sokbo_view.htm?idxno=2016112914055964082", "이투데이", 0, false);
}
private void BuyItem(CodeList.CODE_VALUE Code)
@@ -92,8 +92,11 @@ namespace NewsCrawler
if(m_bBuy == false)
return;
m_CybosHelper.Buy(Code, Config.GetBuyPrice());
m_CodeList.AddDuplicatedList(Code.m_strCode, Code.m_strName);
if(m_CodeList.IsBuyableCode(Code) == true)
{
m_CybosHelper.Buy(Code, Config.GetBuyPrice());
m_CodeList.AddDuplicatedList(Code.m_strCode, Code.m_strName);
}
}
public void OnManualItemBuy(CodeList.CODE_VALUE Code)
@@ -101,9 +104,14 @@ namespace NewsCrawler
BuyItem(Code);
}
void ProcessSearchAndBuy(string strTitle, string strRef)
void ProcessSearchAndBuy(string strTitle, string strName, string strCode, string strRef)
{
CodeList.CODE_VALUE Code = m_CodeList.SearchCode(strTitle);
CodeList.CODE_VALUE Code;
if(strCode == "")
Code = m_CodeList.SearchCode(strTitle);
else
Code = m_CodeList.GetCode(strCode);
if(Code != null)
{
TextCondition.RESULT MatchResult = m_Condition.Match(strTitle);
@@ -148,24 +156,21 @@ namespace NewsCrawler
}
}
private void InsertItem(string strTitle, DateTime time, string strURL, string strRef, bool bInitial)
delegate void InsertListView(string strTitle, string strName, string strCode, DateTime time, DateTime ResTime, string strURL, string strRef, double responseT, bool bInitial);
public void InsertItem(string strTitle, string strName, string strCode, DateTime time, DateTime ResTime, string strURL, string strRef, double responseT, bool bInitial)
{
try
{
if(this.InvokeRequired)
{
this.Invoke(new InsertListView(InsertItem), strTitle, time, strURL, strRef, bInitial);
this.Invoke(new InsertListView(InsertItem), strTitle, strName, strCode, time, ResTime, strURL, strRef, responseT, bInitial);
}
else
{
foreach(ListViewItem item in lvList.Items)
{
if(item.SubItems[chLink.Index].Text == strURL)
return;
}
lvList.Items.Add(new ListViewItem(new string[] { time.ToString("HH:mm:ss"), strTitle, strRef, strURL }));
if(lvList.Items.Cast<ListViewItem>().Any(s => s.SubItems[chLink.Index].Text == strURL))
return;
lvList.Items.Add(new ListViewItem(new string[] { time.ToString("HH:mm:ss"), ResTime.ToString("HH:mm:ss"), strRef, strTitle, string.Format("{0} ms", responseT), strURL }));
if(bInitial == false && chAutoSelect.Checked == true)
{
@@ -179,7 +184,7 @@ namespace NewsCrawler
if(bInitial == false)
{
lvList.Sort();
ProcessSearchAndBuy(strTitle, strRef);
ProcessSearchAndBuy(strTitle, strName, strCode, strRef);
}
}
}
@@ -200,8 +205,12 @@ namespace NewsCrawler
request.Credentials=CredentialCache.DefaultCredentials;
request.Timeout=2000;
System.Diagnostics.Stopwatch timer = new System.Diagnostics.Stopwatch();
timer.Start();
using(HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
timer.Stop();
using(Stream dataStream = response.GetResponseStream())
{
using(StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("EUC-KR")))
@@ -228,7 +237,10 @@ namespace NewsCrawler
continue;
}
InsertItem(strTitle, DateTime.ParseExact(strTime, "HH:mm", CultureInfo.CurrentCulture), strURL, "아시아경제", bInitial);
InsertItem(strTitle, "", "",
DateTime.ParseExact(strTime, "HH:mm", CultureInfo.CurrentCulture),
DateTime.Now,
strURL, "아시아경제", timer.Elapsed.TotalMilliseconds, bInitial);
}
}
@@ -254,8 +266,12 @@ namespace NewsCrawler
request.Credentials=CredentialCache.DefaultCredentials;
request.Timeout=2000;
System.Diagnostics.Stopwatch timer = new System.Diagnostics.Stopwatch();
timer.Start();
using(HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
timer.Stop();
using(Stream dataStream = response.GetResponseStream())
{
using(StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("utf-8")))
@@ -279,7 +295,75 @@ namespace NewsCrawler
continue;
}
InsertItem(strTitle, DateTime.ParseExact(strTime, "HH:mm", CultureInfo.CurrentCulture), strURL, "이투데이", bInitial);
InsertItem(strTitle, "", "",
DateTime.ParseExact(strTime, "HH:mm", CultureInfo.CurrentCulture),
DateTime.Now,
strURL, "이투데이", timer.Elapsed.TotalMilliseconds, bInitial);
}
}
}
}
}
catch(Exception e)
{
Console.WriteLine(e.ToString());
}
return bHasNew;
}
bool ReadEToday2(bool bInitial = false)
{
bool bHasNew = false;
try
{
WebRequest request = WebRequest.Create("http://rss.etoday.co.kr/etoday/etoday_news_all.xml");
request.Credentials=CredentialCache.DefaultCredentials;
request.Timeout=2000;
int iCDATALen = "<![CDATA[".Length;
DateTime today = DateTime.Now;
System.Diagnostics.Stopwatch timer = new System.Diagnostics.Stopwatch();
timer.Start();
using(HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
timer.Stop();
using(Stream dataStream = response.GetResponseStream())
{
using(StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("utf-8")))
{
string responseFromServer = WebUtility.HtmlDecode(reader.ReadToEnd());
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(responseFromServer);
string strXPath = "//item";
var lists = doc.DocumentNode.SelectNodes(strXPath);
if(lists == null)
return false;
foreach(var item in lists)
{
string strTitle = item.SelectSingleNode(".//title").InnerText;
strTitle = strTitle.Substring(iCDATALen, strTitle.Length-iCDATALen-3);
string strTime = item.ChildNodes["pubDate"].InnerText;
DateTime time = Convert.ToDateTime(strTime);
strTime = time.ToString("HH:mm:ss");
string strURL = "[rss] " + item.ChildNodes["link"].NextSibling.InnerText;
if(time.Day != today.Day)
{
//Console.WriteLine("어제 기사 : " + item.InnerHtml);
continue;
}
InsertItem(strTitle, "", "",
DateTime.ParseExact(strTime, "HH:mm:ss", CultureInfo.CurrentCulture),
DateTime.Now,
strURL, "이투데이 rss", timer.Elapsed.TotalMilliseconds, bInitial);
}
}
}
@@ -304,8 +388,12 @@ namespace NewsCrawler
request.Credentials=CredentialCache.DefaultCredentials;
request.Timeout=2000;
System.Diagnostics.Stopwatch timer = new System.Diagnostics.Stopwatch();
timer.Start();
using(HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
timer.Stop();
using(Stream dataStream = response.GetResponseStream())
{
using(StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("utf-8")))
@@ -337,7 +425,10 @@ namespace NewsCrawler
continue;
}
InsertItem(strTitle, DateTime.ParseExact(strTime, "HH:mm", CultureInfo.CurrentCulture), strURL, "DART", bInitial);
InsertItem(strTitle, "", "",
DateTime.ParseExact(strTime, "HH:mm", CultureInfo.CurrentCulture),
DateTime.Now,
strURL, "DART", timer.Elapsed.TotalMilliseconds, bInitial);
}
}
}
@@ -363,8 +454,13 @@ namespace NewsCrawler
request.Timeout=2000;
int iCDATALen = "<![CDATA[".Length;
System.Diagnostics.Stopwatch timer = new System.Diagnostics.Stopwatch();
timer.Start();
using(HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
timer.Stop();
using(Stream dataStream = response.GetResponseStream())
{
using(StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("utf-8")))
@@ -395,7 +491,10 @@ namespace NewsCrawler
continue;
}
InsertItem(strTitle, DateTime.ParseExact(strTime, "HH:mm:ss", CultureInfo.CurrentCulture), strURL, "KIND", bInitial);
InsertItem(strTitle, "", "",
DateTime.ParseExact(strTime, "HH:mm:ss", CultureInfo.CurrentCulture),
DateTime.Now,
strURL, "KIND", timer.Elapsed.TotalMilliseconds, bInitial);
}
}
}
@@ -422,8 +521,11 @@ namespace NewsCrawler
DateTime PrevTime = DateTime.Now;
System.Diagnostics.Stopwatch timer = new System.Diagnostics.Stopwatch();
timer.Start();
using(HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
timer.Stop();
using(Stream dataStream = response.GetResponseStream())
{
using(StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("EUC-KR")))
@@ -449,7 +551,7 @@ namespace NewsCrawler
break;
PrevTime = Time;
InsertItem(strTitle, Time, strURL, "이데일리", bInitial);
InsertItem(strTitle, "", "", Time, DateTime.Now, strURL, "이데일리", timer.Elapsed.TotalMilliseconds, bInitial);
}
}
}
@@ -476,8 +578,13 @@ namespace NewsCrawler
DateTime PrevTime = DateTime.Now;
System.Diagnostics.Stopwatch timer = new System.Diagnostics.Stopwatch();
timer.Start();
using(HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
timer.Stop();
using(Stream dataStream = response.GetResponseStream())
{
using(StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("EUC-KR")))
@@ -508,7 +615,7 @@ namespace NewsCrawler
break;
PrevTime = Time;
InsertItem(strTitle, Time, strURL, "머니투데이", bInitial);
InsertItem(strTitle, "", "", Time, DateTime.Now, strURL, "머니투데이", timer.Elapsed.TotalMilliseconds, bInitial);
}
}
}
@@ -535,8 +642,13 @@ namespace NewsCrawler
DateTime Today = DateTime.Now;
System.Diagnostics.Stopwatch timer = new System.Diagnostics.Stopwatch();
timer.Start();
using(HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
timer.Stop();
using(Stream dataStream = response.GetResponseStream())
{
using(StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("UTF-8")))
@@ -555,7 +667,7 @@ namespace NewsCrawler
string strURL = strServerURL+strCode;
if(Time.DayOfYear == Today.DayOfYear)
InsertItem(strTitle, Time, strURL, "파이낸셜뉴스", bInitial);
InsertItem(strTitle, "", "", Time, DateTime.Now, strURL, "파이낸셜뉴스", timer.Elapsed.TotalMilliseconds, bInitial);
}
}
}
@@ -596,7 +708,7 @@ namespace NewsCrawler
{
m_CrawlTimer.Enabled = false;
Console.WriteLine(string.Format("timer thread {0}", System.Threading.Thread.CurrentThread.ManagedThreadId.ToString()));
//Console.WriteLine(string.Format("timer thread {0}", System.Threading.Thread.CurrentThread.ManagedThreadId.ToString()));
if(chAutoReload.Checked == true)
{
@@ -604,6 +716,7 @@ namespace NewsCrawler
ReadDart();
ReadAsiaE();
ReadEtoday();
ReadEToday2();
ReadEdaily();
ReadMoneyToday();
ReadFinacialNews();