Skip to content

Commit

Permalink
remove self newtonsoft.json
Browse files Browse the repository at this point in the history
  • Loading branch information
@zlzforever
zlzforever committed Apr 16, 2016
1 parent 05c1755 commit ea68373
Show file tree
Hide file tree
Showing 205 changed files with 49 additions and 50,613 deletions.
10 changes: 5 additions & 5 deletions src/HtmlAgilityPack/project.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@

"dependencies": {
"NETStandard.Library": "1.0.0-rc3-23829",
"System.Xml.XmlDocument": "4.0.1-rc3-23829",
"System.Xml.ReaderWriter": "4.0.11-rc3-23829",
"System.Xml.XPath": "4.0.1-rc3-23829",
"System.Xml.XPath.XDocument": "4.0.1-rc3-23829",
"System.Diagnostics.StackTrace": "4.0.1-rc3-23829"
"System.Xml.XmlDocument": "4.0.1-rc3-23829",
"System.Xml.ReaderWriter": "4.0.11-rc3-23829",
"System.Xml.XPath": "4.0.1-rc3-23829",
"System.Xml.XPath.XDocument": "4.0.1-rc3-23829",
"System.Diagnostics.StackTrace": "4.0.1-rc3-23829"
},

"frameworks": {
Expand Down
20 changes: 10 additions & 10 deletions src/Java2Dotnet.Spider.Common/project.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,16 @@

"dependencies": {
"NETStandard.Library": "1.0.0-rc3-23829",
"System.Security.Cryptography.Algorithms": "4.0.0-rc3-23829",
"System.Security.Cryptography.Primitives": "4.0.0-rc3-23829",
"System.Threading.Thread": "4.0.0-rc3-23829",
"System.Threading.ThreadPool": "4.0.10-rc3-23829",
"System.Runtime.InteropServices.RuntimeInformation": "4.0.0-rc3-23829",
"System.Diagnostics.Process":"4.1.0-rc3-23829" ,
"System.Dynamic.Runtime":"4.0.11-rc3-23829",
"System.Net.NameResolution":"4.0.0-rc3-23829" ,
"Microsoft.CSharp": "4.0.1-rc3-23829",
"System.Collections.NonGeneric":"4.0.1-rc3-23829"
"System.Security.Cryptography.Algorithms": "4.0.0-rc3-23829",
"System.Security.Cryptography.Primitives": "4.0.0-rc3-23829",
"System.Threading.Thread": "4.0.0-rc3-23829",
"System.Threading.ThreadPool": "4.0.10-rc3-23829",
"System.Runtime.InteropServices.RuntimeInformation": "4.0.0-rc3-23829",
"System.Diagnostics.Process":"4.1.0-rc3-23829" ,
"System.Dynamic.Runtime":"4.0.11-rc3-23829",
"System.Net.NameResolution":"4.0.0-rc3-23829" ,
"Microsoft.CSharp": "4.0.1-rc3-23829",
"System.Collections.NonGeneric":"4.0.1-rc3-23829"
},
"frameworks": {
"dnxcore50": {
Expand Down
5 changes: 4 additions & 1 deletion src/Java2Dotnet.Spider.Core/Request.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,10 @@ namespace Java2Dotnet.Spider.Core
/// Object contains url to crawl.
/// It contains some additional information.
/// </summary>
public class Request : IDisposable, ICloneable
public class Request : IDisposable
#if !NET_CORE
, ICloneable
#endif
{
public const string CycleTriedTimes = "983009ae-baee-467b-92cd-44188da2b021";
public const string StatusCode = "02d71099-b897-49dd-a180-55345fe9abfc";
Expand Down
24 changes: 6 additions & 18 deletions src/Java2Dotnet.Spider.Core/Spider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -341,23 +341,22 @@ public void InitComponent()
#endif
if (Scheduler is QueueDuplicateRemovedScheduler)
{
Parallel.ForEach(StartRequests, new ParallelOptions() { MaxDegreeOfParallelism = 10 }, request =>
Parallel.ForEach(StartRequests, new ParallelOptions() { MaxDegreeOfParallelism = 4 }, request =>
{
Scheduler.Push((Request)request.Clone(), this);
Scheduler.Push(request, this);
});
}
else
{
QueueDuplicateRemovedScheduler scheduler = new QueueDuplicateRemovedScheduler();
Parallel.ForEach(StartRequests, new ParallelOptions() { MaxDegreeOfParallelism = 10 }, request =>
Parallel.ForEach(StartRequests, new ParallelOptions() { MaxDegreeOfParallelism = 4 }, request =>
{
scheduler.Push((Request)request.Clone(), this);
scheduler.Push(request, this);
});
Scheduler.Load(scheduler.ToList(this), this);
ClearStartRequests();
}

ClearStartRequests();

#if NET_CORE
Logger.Info($"Push Request: {StartRequests.Count} to Scheduler success.", true);
#else
Expand Down Expand Up @@ -414,7 +413,7 @@ public void Run()
_runningExit = false;

#if !NET_CORE
// ���뿪�����߳�����
// 开启多线程支持
System.Net.ServicePointManager.DefaultConnectionLimit = int.MaxValue;
#endif
Logger.Info("Spider " + Identity + " InitComponent...");
Expand Down Expand Up @@ -511,7 +510,6 @@ public void Run()
ThreadPool.WaitToExit();
FinishedTime = DateTime.Now;

// Pipeline���п����л�������, ��Ҫ����/���������ܰ�ȫ�˳�/��ͣ
foreach (IPipeline pipeline in Pipelines)
{
SafeDestroy(pipeline);
Expand Down Expand Up @@ -617,7 +615,6 @@ protected void OnError(Request request)
{
lock (this)
{
//д���ļ���, �û������յĽ�������֪���ж��ٸ�Requestû����. �ṩReRun, Spider������������������Request�����ܹ�
FileInfo file = FilePersistentBase.PrepareFile(Path.Combine(DataRootDirectory, "ErrorRequests.txt"));
File.AppendAllText(file.FullName, JsonConvert.SerializeObject(request) + Environment.NewLine, Encoding.UTF8);
}
Expand All @@ -636,7 +633,6 @@ protected Page AddToCycleRetry(Request request, Site site)
dynamic cycleTriedTimesObject = request.GetExtra(Request.CycleTriedTimes);
if (cycleTriedTimesObject == null)
{
// ���Լ��ӵ�Ŀ��Request��(�޷��������߳��ټ��ش�Request), �������̺߳�����TargetRequest�ӵ�Pool��
request.Priority = 0;
page.AddTargetRequest(request.PutExtra(Request.CycleTriedTimes, 1));
}
Expand All @@ -646,7 +642,6 @@ protected Page AddToCycleRetry(Request request, Site site)
cycleTriedTimes++;
if (cycleTriedTimes >= site.CycleRetryTimes)
{
// �����������Դ���, ���ؿ�.
return null;
}
request.Priority = 0;
Expand All @@ -664,19 +659,15 @@ protected void ProcessRequest(Request request)
{
try
{
// ����ҳ��
page = Downloader.Download(request, this);

if (page.IsSkip)
{
return;
}

// ����Page����
CustomizePage?.Invoke(page);

// ����ҳ������
// PageProcess��2�ִ�����1 ���ص�HTML���� 2��ʵ�ֵ�IPageProcessor����
PageProcessor.Process(page);

break;
Expand All @@ -702,8 +693,6 @@ protected void ProcessRequest(Request request)
return;
}

// for cycle retry, �������س���ʱ, ��������Request�ӻ�TargetUrls�����ظ����������Դ�ʱ��targetRequestsֻ�б���
// ������Ҫ���� MissTargetUrls������
if (page.IsNeedCycleRetry)
{
ExtractAndAddRequests(page, true);
Expand All @@ -722,7 +711,6 @@ protected void ProcessRequest(Request request)
ExtractAndAddRequests(page, SpawnUrl);
}

// Pipeline�������������ݱ����ȹ���, �Dz��������κβ�����, ��������,���ݴ�һ���϶�Ҳ��������, ����ֱ�ӹҵ�Spider�ȽϺá�
if (!page.ResultItems.IsSkip)
{
foreach (IPipeline pipeline in Pipelines)
Expand Down
27 changes: 14 additions & 13 deletions src/Java2Dotnet.Spider.Core/project.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,27 +9,28 @@
"NETStandard.Library": "1.0.0-rc3-23829",
"Java2Dotnet.Spider.Common": "0.0.9",
"Java2Dotnet.Spider.JLog": "0.0.9",
"Newtonsoft.Json": "0.0.9",
"HtmlAgilityPack": "0.0.9",
"HtmlAgilityPack.Css": "0.0.9",
"Java2Dotnet.Spider.Redial": "0.0.9",
"System.Security.Cryptography.Algorithms": "4.0.0-rc3-23829",
"System.Security.Cryptography.Primitives": "4.0.0-rc3-23829",
"System.Threading.Thread": "4.0.0-rc3-23829",
"System.Dynamic.Runtime": "4.0.11-rc3-23829",
"System.Collections.NonGeneric": "4.0.1-rc3-23829",
"System.Threading.Tasks.Parallel": "4.0.1-rc3-23829",
"System.Net.Http": "4.0.1-rc3-23829",
"System.Runtime.Extensions": "4.1.0-rc3-23829",
"Microsoft.CSharp": "4.0.1-rc3-23829",
"System.IO.Compression": "4.1.0-rc3-23829",
"System.Net.Requests": "4.0.11-rc3-23829"
"System.Security.Cryptography.Algorithms": "4.0.0-rc3-23829",
"System.Security.Cryptography.Primitives": "4.0.0-rc3-23829",
"System.Threading.Thread": "4.0.0-rc3-23829",
"System.Dynamic.Runtime": "4.0.11-rc3-23829",
"System.Collections.NonGeneric": "4.0.1-rc3-23829",
"System.Threading.Tasks.Parallel": "4.0.1-rc3-23829",
"System.Net.Http": "4.0.1-rc3-23829",
"System.Runtime.Extensions": "4.1.0-rc3-23829",
"Microsoft.CSharp": "4.0.1-rc3-23829",
"System.IO.Compression": "4.1.0-rc3-23829",
"System.Net.Requests": "4.0.11-rc3-23829",
"Newtonsoft.Json": "7.0.1",
"System.Runtime.Serialization.Primitives": "4.1.1-rc3-23902",
"System.Xml.XDocument": "4.0.11-rc3-23829"
},

"frameworks": {
"dnxcore50": {
"dependencies": {

},
"compilationOptions": {
"define": [ "NET_CORE" ]
Expand Down
1 change: 0 additions & 1 deletion src/Java2Dotnet.Spider.Extension/Pipeline/EntityGeneralPipeline.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
using System.Collections.Generic;
using System.Data;
using System.Data.Common;
using System.Data.SqlClient;
using System.Linq;
using System.Reflection;
using System.Reflection.Emit;
Expand Down
9 changes: 6 additions & 3 deletions src/Java2Dotnet.Spider.Extension/project.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,17 @@
"Java2Dotnet.Spider.Common": "0.0.9",
"Java2Dotnet.Spider.Core": "0.0.9",
"Java2Dotnet.Spider.JLog": "0.0.9",
"Newtonsoft.Json": "0.0.9",
"HtmlAgilityPack": "0.0.9",
"HtmlAgilityPack.Css": "0.0.9",
"MySql.Data": "0.0.9",
"Java2Dotnet.Spider.Redial": "0.0.9",
"Java2Dotnet.Spider.Validation": "0.0.9",
"System.Data.Common": "4.0.1-rc3-23829",
"Microsoft.CSharp": "4.0.1-rc3-23829"
"System.Data.Common": "4.0.1-rc3-23829",
"Microsoft.CSharp": "4.0.1-rc3-23829",
"Newtonsoft.Json": "7.0.1",
"System.Dynamic.Runtime":"4.0.11-rc3-23829",
"System.Runtime.Serialization.Primitives": "4.1.1-rc3-23902",
"System.Xml.XDocument": "4.0.11-rc3-23829"
},

"frameworks": {
Expand Down
8 changes: 4 additions & 4 deletions src/Java2Dotnet.Spider.JLog/project.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,15 @@
},

"dependencies": {
"NETStandard.Library": "1.0.0-rc3-23829"
"NETStandard.Library": "1.0.0-rc3-23829",
"System.Net.NameResolution":"4.0.0-rc3-23829",
"System.Net.Http":"4.0.1-rc3-23829",
"System.Threading.Thread": "4.0.0-rc3-23829"
},

"frameworks": {
"dnxcore50": {
"dependencies": {
"System.Net.NameResolution":"4.0.0-rc3-23829",
"System.Net.Http":"4.0.1-rc3-23829",
"System.Threading.Thread": "4.0.0-rc3-23829"
},
"compilationOptions": {
"define": [ "NET_CORE" ]
Expand Down
44 changes: 0 additions & 44 deletions src/Newtonsoft.Json/Bson/BsonBinaryType.cs
View file

This file was deleted.

Loading

0 comments on commit ea68373

Please sign in to comment.