RedisServer

cloupid · Apr 2, 2016 · 63faefc · 63faefc
1 parent 77ea86a
commit 63faefc
Show file tree

Hide file tree

Showing 8 changed files with 305 additions and 255 deletions.
diff --git a/src/Java2Dotnet.Spider.Common/RedisSharp.cs → src/Java2Dotnet.Spider.Common/RedisServer.cs b/src/Java2Dotnet.Spider.Common/RedisSharp.cs → src/Java2Dotnet.Spider.Common/RedisServer.cs
diff --git a/src/Java2Dotnet.Spider.Common/net45/Java2Dotnet.Spider.Common.csproj b/src/Java2Dotnet.Spider.Common/net45/Java2Dotnet.Spider.Common.csproj
@@ -130,8 +130,8 @@
     <Compile Include="..\PriorityBlockingQueue.cs">
       <Link>PriorityBlockingQueue.cs</Link>
     </Compile>
-    <Compile Include="..\RedisSharp.cs">
-      <Link>RedisSharp.cs</Link>
+    <Compile Include="..\RedisServer.cs">
+      <Link>RedisServer.cs</Link>
     </Compile>
     <Compile Include="..\SafeExecutor.cs">
       <Link>SafeExecutor.cs</Link>

diff --git a/src/Java2Dotnet.Spider.Extension/ScriptSpider.cs b/src/Java2Dotnet.Spider.Extension/ScriptSpider.cs
@@ -66,6 +66,8 @@ public ScriptSpider(SpiderContext spiderContext)
 
 		private void InitEnvoriment()
 		{
+			redis = RedisProvider.GetProvider();
+
 			if (_spiderContext.Redialer != null)
 			{
 				//RedialManagerUtils.RedialManager = FileLockerRedialManager.Default;
@@ -115,8 +117,6 @@ private void CheckValidations()
 
 		private void DoValidate()
 		{
-			redis = RedisProvider.GetProvider();
-
 			string key = "locker-validate-" + Name;
 			try
 			{

diff --git a/src/Java2Dotnet.Spider.Test/Example/JdSkuSample.cs b/src/Java2Dotnet.Spider.Test/Example/JdSkuSample.cs
@@ -18,7 +18,7 @@ protected override SpiderContext CreateSpiderContext()
 		{
 			return new SpiderContext
 			{
-				SpiderName = "JD sku/store test " + DateTimeUtils.FirstDayofThisWeek.ToString("yyyy-MM-dd"),
+				SpiderName = "JD sku/store test " + DateTime.Now.ToString("yyyy-MM-dd"),
 				CachedSize = 1,
 				ThreadNum = 1,
 				Site = new Site
@@ -46,7 +46,6 @@ protected override SpiderContext CreateSpiderContext()
 		protected override HashSet<Type> EntiTypes => new HashSet<Type>() { typeof(Product) };
 
 		[Schema("test", "sku", TableSuffix.Today)]
-		[TargetUrl(new[] { @"page=[0-9]+" }, "//*[@id=\"J_bottomPage\"]")]
 		[TypeExtractBy(Expression = "//li[@class='gl-item']/div[contains(@class,'j-sku-item')]", Multi = true)]
 		[Indexes(Index = new[] { "category" }, Unique = new[] { "category,sku", "sku" })]
 		public class Product : ISpiderEntity

diff --git a/src/Java2Dotnet.Spider.Test/RedisRedialManager/RedisRedialManagerTests.cs b/src/Java2Dotnet.Spider.Test/RedisRedialManager/RedisRedialManagerTests.cs
@@ -27,7 +27,7 @@ public void SubPubTest()
 				result = message;
 			});
 
-			bool reslut = redis.Publish("messages", "hello");
+			redis.Publish("messages", "hello");
 
 			Thread.Sleep(2000);
 

diff --git a/src/Java2Dotnet.Spider.Test/RedisSchedulerTest.cs b/src/Java2Dotnet.Spider.Test/RedisSchedulerTest.cs
@@ -13,9 +13,13 @@ public class RedisSchedulerTest
 		[TestMethod]
 		public void RedisTest()
 		{
+
 			RedisScheduler redisScheduler = new RedisScheduler("localhost", "");
 
 			ISpider spider = new TestSpider();
+			RedisSchedulerManager m = new RedisSchedulerManager("localhost");
+			m.RemoveTask(spider.Identity);
+
 			Request request = new Request("http://www.ibm.com/developerworks/cn/java/j-javadev2-22/", 1, null);
 			request.PutExtra("1", "2");
 			redisScheduler.Push(request, spider);
@@ -25,7 +29,7 @@ public void RedisTest()
 			Assert.IsNull(result1);
 			redisScheduler.Dispose();
 
-			RedisSchedulerManager m = new RedisSchedulerManager("localhost");
+
 			m.RemoveTask(spider.Identity);
 		}
 

diff --git a/src/Java2Dotnet.Spider.Test/RedisTest.cs b/src/Java2Dotnet.Spider.Test/RedisTest.cs
@@ -109,6 +109,11 @@ public void RedisBaseTest()
 
 			#endregion
 
+			#region locker
+
+			r.LockTake("locker", DateTime.Now.ToString(), new TimeSpan(0, 0, 0, 30));
+			#endregion
+
 			r.FlushDb();
 			r.Dispose();
 		}
@@ -128,7 +133,7 @@ public void MultiTheadTestForStactExtanceRedis()
 				}
 			});
 			var r = context.GetDatabase(3);
- 
+
 			DateTime start1 = DateTime.Now;
 
 			Parallel.For(0, 100000, new ParallelOptions() { MaxDegreeOfParallelism = 20 }, j =>

diff --git a/src/Java2Dotnet.Spider.Test/net45/app.config b/src/Java2Dotnet.Spider.Test/net45/app.config
@@ -1,11 +1,17 @@
 <?xml version="1.0" encoding="utf-8"?>
 <configuration>
-  <runtime>
-    <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
-      <dependentAssembly>
-        <assemblyIdentity name="HtmlAgilityPack" publicKeyToken="bd319b19eaf3b43a" culture="neutral" />
-        <bindingRedirect oldVersion="0.0.0.0-1.4.9.0" newVersion="1.4.9.0" />
-      </dependentAssembly>
-    </assemblyBinding>
-  </runtime>
-<startup><supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" /></startup></configuration>
+	<runtime>
+		<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
+			<dependentAssembly>
+				<assemblyIdentity name="HtmlAgilityPack" publicKeyToken="bd319b19eaf3b43a" culture="neutral" />
+				<bindingRedirect oldVersion="0.0.0.0-1.4.9.0" newVersion="1.4.9.0" />
+			</dependentAssembly>
+		</assemblyBinding>
+	</runtime>
+	<startup>
+		<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
+	</startup>
+	<appSettings>
+		<add key="redisServer" value="localhost"/>
+	</appSettings>
+</configuration>