Merge pull request #1130 from oliverw/oliverw

- Implement support for Postgres TLS - Don't expose Ergo Wallet Password through API - Don't expose confidential Tls information through API - Fixed a bug that would delay job updates to all miners if the previous update was delayed due to a dead connection - Remove artificial outbound stratum message length and optimize memory usage - Re-introduce Ethereum Job backlog - Build script fixes (@warren-ru)
oliverw · Jan 24, 2022 · 05b7810 · 05b7810
2 parents fb0f2f7 + c7a59d3
commit 05b7810
Show file tree

Hide file tree

Showing 81 changed files with 1,114 additions and 1,197 deletions.
diff --git a/src/.idea/.idea.Miningcore/.idea/dataSources.xml b/src/.idea/.idea.Miningcore/.idea/dataSources.xml
diff --git a/src/.idea/.idea.Miningcore/.idea/inspectionProfiles/Project_Default.xml b/src/.idea/.idea.Miningcore/.idea/inspectionProfiles/Project_Default.xml
diff --git a/src/.idea/.idea.Miningcore/.idea/sqldialects.xml b/src/.idea/.idea.Miningcore/.idea/sqldialects.xml
diff --git a/src/Miningcore/Api/Controllers/AdminApiController.cs b/src/Miningcore/Api/Controllers/AdminApiController.cs
@@ -66,7 +66,7 @@ public async Task<decimal> GetMinerBalanceAsync(string poolId, string address)
         if(string.IsNullOrEmpty(address))
             throw new ApiException("Invalid or missing miner address", HttpStatusCode.NotFound);
 
-        var result = await cf.Run(con=> minerRepo.GetSettings(con, null, pool.Id, address));
+        var result = await cf.Run(con=> minerRepo.GetSettingsAsync(con, null, pool.Id, address));
 
         if(result == null)
             throw new ApiException("No settings found", HttpStatusCode.NotFound);
@@ -98,9 +98,9 @@ public async Task<decimal> GetMinerBalanceAsync(string poolId, string address)
 
         var result = await cf.RunTx(async (con, tx) =>
         {
-            await minerRepo.UpdateSettings(con, tx, mapped);
+            await minerRepo.UpdateSettingsAsync(con, tx, mapped);
 
-            return await minerRepo.GetSettings(con, tx, mapped.PoolId, mapped.Address);
+            return await minerRepo.GetSettingsAsync(con, tx, mapped.PoolId, mapped.Address);
         });
 
         logger.Info(()=> $"Updated settings for pool {pool.Id}, miner {address}");

diff --git a/src/Miningcore/Api/Controllers/ClusterApiController.cs b/src/Miningcore/Api/Controllers/ClusterApiController.cs
@@ -38,11 +38,12 @@ public ClusterApiController(IComponentContext ctx) : base(ctx)
     public async Task<Responses.Block[]> PageBlocksPagedAsync(
         [FromQuery] int page, [FromQuery] int pageSize = 15, [FromQuery] BlockStatus[] state = null)
     {
-        var blockStates = state != null && state.Length > 0 ?
+        var ct = HttpContext.RequestAborted;
+        var blockStates = state is { Length: > 0 } ?
             state :
             new[] { BlockStatus.Confirmed, BlockStatus.Pending, BlockStatus.Orphaned };
 
-        var blocks = (await cf.Run(con => blocksRepo.PageBlocksAsync(con, blockStates, page, pageSize)))
+        var blocks = (await cf.Run(con => blocksRepo.PageBlocksAsync(con, blockStates, page, pageSize, ct)))
             .Select(mapper.Map<Responses.Block>)
             .Where(x => enabledPools.Contains(x.PoolId))
             .ToArray();

diff --git a/src/Miningcore/Api/Controllers/PoolApiController.cs b/src/Miningcore/Api/Controllers/PoolApiController.cs
diff --git a/src/Miningcore/Api/Extensions/MiningPoolExtensions.cs b/src/Miningcore/Api/Extensions/MiningPoolExtensions.cs
@@ -1,7 +1,9 @@
 using AutoMapper;
 using Miningcore.Api.Responses;
 using Miningcore.Blockchain;
+using Miningcore.Blockchain.Ergo.Configuration;
 using Miningcore.Configuration;
+using Miningcore.Extensions;
 using Miningcore.Mining;
 
 namespace Miningcore.Api.Extensions;
@@ -28,9 +30,19 @@ public static PoolInfo ToPoolInfo(this PoolConfig poolConfig, IMapper mapper, Pe
         {
             var extra = poolInfo.PaymentProcessing.Extra;
 
-            //extra.StripValue(nameof(EthereumPoolPaymentProcessingConfigExtra.CoinbasePassword));
+            extra.StripValue(nameof(ErgoPaymentProcessingConfigExtra.WalletPassword));
         }
 
+        if(poolInfo.Ports != null)
+        {
+            foreach(var port in poolInfo.Ports.Keys)
+            {
+                var portInfo = poolInfo.Ports[port];
+
+                portInfo.TlsPfxFile = null;
+                portInfo.TlsPfxPassword = null;
+            }
+        }
         return poolInfo;
     }
 }
diff --git a/src/Miningcore/Api/Responses/GetPoolsResponse.cs b/src/Miningcore/Api/Responses/GetPoolsResponse.cs
@@ -2,7 +2,6 @@
 using Miningcore.Blockchain;
 using Miningcore.Configuration;
 using Miningcore.Mining;
-using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 
 namespace Miningcore.Api.Responses;

diff --git a/src/Miningcore/AutoMapperProfile.cs b/src/Miningcore/AutoMapperProfile.cs
@@ -3,6 +3,7 @@
 using Miningcore.Configuration;
 using Miningcore.Persistence.Model;
 using Miningcore.Persistence.Model.Projections;
+using Newtonsoft.Json.Linq;
 using MinerStats = Miningcore.Persistence.Model.Projections.MinerStats;
 
 namespace Miningcore;
@@ -13,6 +14,9 @@ public class AutoMapperProfile : Profile
 
     public AutoMapperProfile()
     {
+        // Fix for Automapper 11 which chokes on recursive objects such as JToken
+        CreateMap<JToken, JToken>().ConvertUsing(x=> x);
+
         //////////////////////
         // outgoing mappings
 

diff --git a/src/Miningcore/AutofacMetadata.cs b/src/Miningcore/AutofacMetadata.cs
@@ -1,12 +1,10 @@
-using JetBrains.Annotations;
 using Miningcore.Configuration;
 
 namespace Miningcore;
 
 public class CoinFamilyAttribute : Attribute
 {
-    [UsedImplicitly]
-    public CoinFamilyAttribute(IDictionary<string, object> values)
+        public CoinFamilyAttribute(IDictionary<string, object> values)
     {
         if(values.ContainsKey(nameof(SupportedFamilies)))
             SupportedFamilies = (CoinFamily[]) values[nameof(SupportedFamilies)];

diff --git a/src/Miningcore/AutofacModule.cs b/src/Miningcore/AutofacModule.cs
@@ -19,6 +19,7 @@
 using Newtonsoft.Json.Serialization;
 using Module = Autofac.Module;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.IO;
 using Miningcore.Blockchain.Ergo;
 using Miningcore.Nicehash;
 using Miningcore.Pushover;
@@ -51,6 +52,13 @@ protected override void Load(ContainerBuilder builder)
             .AsImplementedInterfaces()
             .SingleInstance();
 
+        builder.RegisterInstance(new RecyclableMemoryStreamManager
+        {
+            MaximumFreeSmallPoolBytes = 0x100000,   // 1 MB
+            MaximumFreeLargePoolBytes = 0x1000000,  // 16 MB
+            ThrowExceptionOnToArray = true
+        });
+
         builder.RegisterType<StandardClock>()
             .AsImplementedInterfaces()
             .SingleInstance();

diff --git a/src/Miningcore/Blockchain/Bitcoin/BitcoinJobManager.cs b/src/Miningcore/Blockchain/Bitcoin/BitcoinJobManager.cs
@@ -46,8 +46,6 @@ protected override object[] GetBlockTemplateParams()
 
     protected async Task<RpcResponse<BlockTemplate>> GetBlockTemplateAsync(CancellationToken ct)
     {
-        logger.LogInvoke();
-
         var result = await rpc.ExecuteAsync<BlockTemplate>(logger,
             BitcoinCommands.GetBlockTemplate, ct, extraPoolConfig?.GBTArgs ?? (object) GetBlockTemplateParams());
 
@@ -56,8 +54,6 @@ protected async Task<RpcResponse<BlockTemplate>> GetBlockTemplateAsync(Cancellat
 
     protected RpcResponse<BlockTemplate> GetBlockTemplateFromJson(string json)
     {
-        logger.LogInvoke();
-
         var result = JsonConvert.DeserializeObject<JsonRpcResponse>(json);
 
         return new RpcResponse<BlockTemplate>(result!.ResultAs<BlockTemplate>());
@@ -81,8 +77,6 @@ protected override void PostChainIdentifyConfigure()
 
     protected override async Task<(bool IsNew, bool Force)> UpdateJob(CancellationToken ct, bool forceUpdate, string via = null, string json = null)
     {
-        logger.LogInvoke();
-
         try
         {
             if(forceUpdate)
@@ -212,8 +206,6 @@ public virtual async ValueTask<Share> SubmitShareAsync(StratumConnection worker,
         Contract.RequiresNonNull(worker, nameof(worker));
         Contract.RequiresNonNull(submission, nameof(submission));
 
-        logger.LogInvoke(new object[] { worker.ConnectionId });
-
         if(submission is not object[] submitParams)
             throw new StratumException(StratumError.Other, "invalid params");
 

diff --git a/src/Miningcore/Blockchain/Bitcoin/BitcoinJobManagerBase.cs b/src/Miningcore/Blockchain/Bitcoin/BitcoinJobManagerBase.cs
@@ -7,7 +7,6 @@
 using Miningcore.Configuration;
 using Miningcore.Contracts;
 using Miningcore.Extensions;
-using Miningcore.JsonRpc;
 using Miningcore.Messaging;
 using Miningcore.Mining;
 using Miningcore.Notifications.Messages;
@@ -222,8 +221,6 @@ protected virtual async Task ShowDaemonSyncProgressAsync(CancellationToken ct)
 
     private async Task UpdateNetworkStatsAsync(CancellationToken ct)
     {
-        logger.LogInvoke();
-
         try
         {
             var results = await rpc.ExecuteBatchAsync(logger, ct,
@@ -340,8 +337,6 @@ protected async Task ShowDaemonSyncProgressLegacyAsync(CancellationToken ct)
 
     private async Task UpdateNetworkStatsLegacyAsync(CancellationToken ct)
     {
-        logger.LogInvoke();
-
         try
         {
             var results = await rpc.ExecuteBatchAsync(logger, ct,

diff --git a/src/Miningcore/Blockchain/Bitcoin/BitcoinPayoutHandler.cs b/src/Miningcore/Blockchain/Bitcoin/BitcoinPayoutHandler.cs
@@ -4,7 +4,6 @@
 using Miningcore.Blockchain.Bitcoin.DaemonResponses;
 using Miningcore.Configuration;
 using Miningcore.Extensions;
-using Miningcore.JsonRpc;
 using Miningcore.Messaging;
 using Miningcore.Mining;
 using Miningcore.Payments;

diff --git a/src/Miningcore/Blockchain/Bitcoin/BitcoinPool.cs b/src/Miningcore/Blockchain/Bitcoin/BitcoinPool.cs
@@ -4,7 +4,7 @@
 using System.Reactive.Threading.Tasks;
 using Autofac;
 using AutoMapper;
-using JetBrains.Annotations;
+using Microsoft.IO;
 using Miningcore.Configuration;
 using Miningcore.Extensions;
 using Miningcore.JsonRpc;
@@ -24,7 +24,6 @@
 namespace Miningcore.Blockchain.Bitcoin;
 
 [CoinFamily(CoinFamily.Bitcoin)]
-[UsedImplicitly]
 public class BitcoinPool : PoolBase
 {
     public BitcoinPool(IComponentContext ctx,
@@ -34,8 +33,9 @@ public BitcoinPool(IComponentContext ctx,
         IMapper mapper,
         IMasterClock clock,
         IMessageBus messageBus,
+        RecyclableMemoryStreamManager rmsm,
         NicehashService nicehashService) :
-        base(ctx, serializerSettings, cf, statsRepo, mapper, clock, messageBus, nicehashService)
+        base(ctx, serializerSettings, cf, statsRepo, mapper, clock, messageBus, rmsm, nicehashService)
     {
     }
 
@@ -321,11 +321,8 @@ protected virtual Task OnNewJobAsync(object jobParams)
 
         logger.Info(() => "Broadcasting job");
 
-        return Guard(()=> Task.WhenAll(ForEachConnection(async connection =>
+        return Guard(Task.WhenAll(TaskForEach(async connection =>
         {
-            if(!connection.IsAlive)
-                return;
-
             var context = connection.ContextAs<BitcoinWorkerContext>();
 
             if(!context.IsSubscribed || !context.IsAuthorized || CloseIfDead(connection, context))
@@ -393,9 +390,9 @@ protected override async Task SetupJobManager(CancellationToken ct)
         }
     }
 
-    protected override async Task InitStatsAsync()
+    protected override async Task InitStatsAsync(CancellationToken ct)
     {
-        await base.InitStatsAsync();
+        await base.InitStatsAsync(ct);
 
         blockchainStats = manager.BlockchainStats;
     }

diff --git a/src/Miningcore/Blockchain/Bitcoin/Configuration/BitcoinPoolConfigExtra.cs b/src/Miningcore/Blockchain/Bitcoin/Configuration/BitcoinPoolConfigExtra.cs
@@ -1,10 +1,8 @@
-using JetBrains.Annotations;
 using Miningcore.Configuration;
 using Newtonsoft.Json.Linq;
 
 namespace Miningcore.Blockchain.Bitcoin.Configuration;
 
-[UsedImplicitly]
 public class BitcoinPoolConfigExtra
 {
     public BitcoinAddressType AddressType { get; set; } = BitcoinAddressType.Legacy;

diff --git a/src/Miningcore/Blockchain/Bitcoin/DaemonResponses/Masternode.cs b/src/Miningcore/Blockchain/Bitcoin/DaemonResponses/Masternode.cs
@@ -1,4 +1,3 @@
-using JetBrains.Annotations;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 
@@ -11,7 +10,6 @@ public class Masternode
     public long Amount { get; set; }
 }
 
-[UsedImplicitly]
 public class SuperBlock
 {
     public string Payee { get; set; }

diff --git a/src/Miningcore/Blockchain/Bitcoin/DaemonResponses/Utxo.cs b/src/Miningcore/Blockchain/Bitcoin/DaemonResponses/Utxo.cs
@@ -1,8 +1,5 @@
-using JetBrains.Annotations;
-
 namespace Miningcore.Blockchain.Bitcoin.DaemonResponses;
 
-[UsedImplicitly]
 public class Utxo
 {
     public string TxId { get; set; }

diff --git a/src/Miningcore/Blockchain/Cryptonote/CryptonoteJobManager.cs b/src/Miningcore/Blockchain/Cryptonote/CryptonoteJobManager.cs
@@ -60,8 +60,6 @@ public CryptonoteJobManager(
 
     protected async Task<bool> UpdateJob(CancellationToken ct, string via = null, string json = null)
     {
-        logger.LogInvoke();
-
         try
         {
             var response = string.IsNullOrEmpty(json) ? await GetBlockTemplateAsync(ct) : GetBlockTemplateFromJson(json);
@@ -144,8 +142,6 @@ protected async Task<bool> UpdateJob(CancellationToken ct, string via = null, st
 
     private async Task<RpcResponse<GetBlockTemplateResponse>> GetBlockTemplateAsync(CancellationToken ct)
     {
-        logger.LogInvoke();
-
         var request = new GetBlockTemplateRequest
         {
             WalletAddress = poolConfig.Address,
@@ -157,8 +153,6 @@ private async Task<RpcResponse<GetBlockTemplateResponse>> GetBlockTemplateAsync(
 
     private RpcResponse<GetBlockTemplateResponse> GetBlockTemplateFromJson(string json)
     {
-        logger.LogInvoke();
-
         var result = JsonConvert.DeserializeObject<JsonRpcResponse>(json);
 
         return new RpcResponse<GetBlockTemplateResponse>(result.ResultAs<GetBlockTemplateResponse>());
@@ -182,8 +176,6 @@ private async Task ShowDaemonSyncProgressAsync(CancellationToken ct)
 
     private async Task UpdateNetworkStatsAsync(CancellationToken ct)
     {
-        logger.LogInvoke();
-
         try
         {
             var response = await rpc.ExecuteAsync(logger, CryptonoteCommands.GetInfo, ct);
@@ -336,7 +328,6 @@ public async ValueTask<Share> SubmitShareAsync(StratumConnection worker,
         Contract.RequiresNonNull(worker, nameof(worker));
         Contract.RequiresNonNull(request, nameof(request));
 
-        logger.LogInvoke(new object[] { worker.ConnectionId });
         var context = worker.ContextAs<CryptonoteWorkerContext>();
 
         var job = currentJob;

diff --git a/src/Miningcore/Blockchain/Cryptonote/CryptonotePayoutHandler.cs b/src/Miningcore/Blockchain/Cryptonote/CryptonotePayoutHandler.cs
@@ -6,7 +6,6 @@
 using Miningcore.Blockchain.Cryptonote.DaemonResponses;
 using Miningcore.Configuration;
 using Miningcore.Extensions;
-using Miningcore.JsonRpc;
 using Miningcore.Messaging;
 using Miningcore.Mining;
 using Miningcore.Native;