LedgerConfiguration.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Waher.Content;
using Waher.Events;
using Waher.IoTGateway;
using Waher.IoTGateway.Setup;
using Waher.Networking.HTTP;
using Waher.Persistence;
using Waher.Persistence.Attributes;
using Waher.Persistence.NeuroLedger;
using Waher.Persistence.NeuroLedger.Registry;
using Waher.Runtime.Language;
using Waher.Security.CallStack;
using Waher.Security.EllipticCurves;
using Waher.Security.SHA3;
 
namespace Waher.Service.NeuroLedger.Setup
{
    public class LedgerConfiguration : SystemConfiguration
    {
        private static LedgerConfiguration instance = null;
        internal static readonly Regex FromSaveUnsaved = new Regex(@"Waher[.]Persistence[.]Files[.]ObjectBTreeFile[.+]<?SaveUnsaved>?\w*[.]\w*",
            RegexOptions.Compiled | RegexOptions.Singleline);
        internal static readonly Regex FromUpdateObject = new Regex(@"Waher[.]Persistence[.]Files[.]ObjectBTreeFile[.+]<?UpdateObject>?\w*[.]\w*",
            RegexOptions.Compiled | RegexOptions.Singleline);
        internal static readonly Regex GatewayStartup = new Regex(@"Waher[.]IoTGateway[.]Gateway([.]Start|[.+]<?Start>?\w*[.]\w*)*",
            RegexOptions.Compiled | RegexOptions.Singleline);
        internal static readonly string LegalComponent = "Waher.Service.IoTBroker.Legal.LegalComponent";
        internal static readonly string PaiwiseComponent = "Waher.Service.IoTBroker.Paiwise.PaiwiseProcessor";
        internal static readonly string NeuroFeaturesComponent = "Waher.Service.IoTBroker.NeuroFeatures.NeuroFeaturesProcessor";
        internal static readonly string EDalerComponent = "Waher.Service.IoTBroker.EDaler.EDalerProcessor";
        internal static readonly string MarketplaceComponent = "Waher.Service.IoTBroker.Marketplace.MarketplaceProcessor";
        internal static readonly string SetContractState = "Waher.Service.IoTBroker.WebServices.SetContractState";
        internal static readonly Regex UnitTests = new Regex(@"Waher[.]Service[.]IoTBroker[.]Test[.](LegalIdentitiesTests|SmartContractsTests|AgentTests|EDalerTests|EDalerFederatedTests|MarketplaceTests|NeuroFeaturesTests|QuickLoginTests)",
            RegexOptions.Compiled | RegexOptions.Singleline);
        private static readonly ICallStackCheck[] approvedSources = Assert.Convert(new object[]
        {
            FromSaveUnsaved,
            FromUpdateObject,
            GatewayStartup,
            typeof(LedgerConfiguration),
            typeof(NeuroLedgerProvider),
            typeof(NeuroLedgerModule),
            LegalComponent,
            PaiwiseComponent,
            NeuroFeaturesComponent,
            EDalerComponent,
            SetContractState,
            MarketplaceComponent    // Hard coded, to make sure other components cannot register new approved sources.
        });
        private static readonly ICallStackCheck[] approvedSourcesTest = Assert.Convert(new object[]
        {
            FromSaveUnsaved,
            FromUpdateObject,
            GatewayStartup,
            typeof(LedgerConfiguration),
            typeof(NeuroLedgerProvider),
            typeof(NeuroLedgerModule),
            LegalComponent,
            PaiwiseComponent,
            NeuroFeaturesComponent,
            EDalerComponent,
            SetContractState,
            MarketplaceComponent,   // Hard coded, to make sure other components cannot register new approved sources.
            "System.Threading.PortableThreadPool+WorkerThread",
            UnitTests
        });
 
        private string dataProtectionAgreementId = string.Empty;
        private byte[] privateKey = null;
        private byte[] salt = null;
        private int collectionTimeSeconds = 5 * 60;
        private int blockSizeThreshold = 1 * 1024 * 1024;     // 1 MB
        private Edwards448 ed448 = null;
        private readonly SHA3_512 sha3_512 = new SHA3_512();
 
        private HttpResource setLedgerProperties = null;
 
        public LedgerConfiguration()
            : base()
        {
        }
 
        [DefaultValueNull]
        public byte[] PrivateKey
        {
            get
            {
                Assert.CallFromSource((DomainConfiguration.Instance?.UseDomainName ?? false) ? approvedSources : approvedSourcesTest);
                return this.privateKey;
            }
 
            set
            {
                if (!(this.privateKey is null))
                    Assert.CallFromSource((DomainConfiguration.Instance?.UseDomainName ?? false) ? approvedSources : approvedSourcesTest);
 
                this.privateKey = value;
                this.ed448 = new Edwards448(this.privateKey);
            }
        }
 
        [DefaultValueNull]
        public byte[] PublicKey
        {
            get
            {
                if (this.CheckKey() && this.ObjectId != Guid.Empty)
                {
                    Task _ = Database.UpdateLazy(this);
                }
 
                return this.ed448.PublicKey;
            }
        }
 
        [DefaultValueNull]
        public byte[] Salt
        {
            get
            {
                Assert.CallFromSource((DomainConfiguration.Instance?.UseDomainName ?? false) ? approvedSources : approvedSourcesTest);
                return this.salt;
            }
 
            set
            {
                if (!(this.salt is null))
                    Assert.CallFromSource((DomainConfiguration.Instance?.UseDomainName ?? false) ? approvedSources : approvedSourcesTest);
 
                this.salt = value;
            }
        }
 
        [DefaultValueStringEmpty]
        public string DataProtectionAgreementId
        {
            get
            {
                Assert.CallFromSource((DomainConfiguration.Instance?.UseDomainName ?? false) ? approvedSources : approvedSourcesTest);
                return this.dataProtectionAgreementId;
            }
 
            set
            {
                if (!string.IsNullOrEmpty(this.dataProtectionAgreementId))
                    Assert.CallFromSource((DomainConfiguration.Instance?.UseDomainName ?? false) ? approvedSources : approvedSourcesTest);
 
                this.dataProtectionAgreementId = value;
            }
        }
 
        public string SignatureAlgorithm
        {
            get
            {
                if (this.CheckKey())
                {
                    Task _ = Database.UpdateLazy(this);
                }
 
                return this.ed448.CurveName;
            }
        }
 
        private bool CheckKey()
        {
            bool Result = false;
 
            if (this.privateKey is null)
            {
                this.privateKey = Gateway.NextBytes(56);
                this.ed448 = new Edwards448(this.privateKey);
                Result = true;
            }
 
            if (this.salt is null)
            {
                this.salt = Gateway.NextBytes(64);
                Result = true;
            }
 
            return Result;
        }
 
        public string LedgerFolder => Path.Combine(Gateway.AppDataFolder, "Ledger");
 
        public int CollectionTimeSeconds
        {
            get => this.collectionTimeSeconds;
            set => this.collectionTimeSeconds = value;
        }
 
        public int BlockSizeThreshold
        {
            get => this.blockSizeThreshold;
            set => this.blockSizeThreshold = value;
        }
 
        public static LedgerConfiguration Instance => instance;
 
        public override string Resource => "/Settings/Ledger.md";
 
        public override int Priority => 350;
 
        public override Task<string> Title(Language Language)
        {
            return Language.GetStringAsync(typeof(LedgerConfiguration), 1, "Neuro-Ledger");
        }
 
        public override void SetStaticInstance(ISystemConfiguration Configuration)
        {
            instance = Configuration as LedgerConfiguration;
        }
 
        public override async Task ConfigureSystem()
        {
            if (this.CheckKey())
                await Database.Update(this);
 
            if (!Ledger.HasProvider || !Ledger.Locked)
            {
                string LedgerFolder = this.LedgerFolder;
 
                NeuroLedgerProvider Provider = new NeuroLedgerProvider(LedgerFolder,
                    TimeSpan.FromSeconds(this.collectionTimeSeconds), this.blockSizeThreshold, this.salt, "Default",
                    XmppConfiguration.Instance.BareJid, this.ed448, this.sha3_512.ComputeVariable, false);
 
                Ledger.Register(Provider);
 
                string[] RepairedCollections = DatabaseConfiguration.RepairedCollections;
                string FileName;
 
                if (!(RepairedCollections is null))
                {
                    foreach (string Collection in RepairedCollections)
                        this.AddRepairNoteFile(Collection);
                }
 
                FileName = Path.Combine(this.RepairFolder, BlockReference.BlockReferencesCollection + ".txt");
                if (File.Exists(FileName))
                {
                    try
                    {
                        Log.Informational("Repairing collection from ledger.", BlockReference.BlockReferencesCollection);
                        await Provider.RepairRegistry();
                        Log.Informational("Repaired collection from ledger.", BlockReference.BlockReferencesCollection);
 
                        try
                        {
                            File.Delete(FileName);
                        }
                        catch (Exception ex)
                        {
                            Log.Error(ex, FileName);
                        }
                    }
                    catch (Exception ex2)
                    {
                        Log.Exception(ex2, BlockReference.BlockReferencesCollection);
                    }
                }
 
                string[] ToRepair = Directory.GetFiles(this.RepairFolder, "*.txt", SearchOption.TopDirectoryOnly);
                foreach (string FileName2 in ToRepair)
                {
                    string CollectionName = Path.GetFileName(FileName2);
                    CollectionName = CollectionName[..^4];
 
                    try
                    {
                        Log.Informational("Repairing collection from ledger.", CollectionName);
 
                        if (CollectionName == BlockReference.BlockReferencesCollection)
                            await Provider.RepairRegistry();
                        else
                            await Provider.RepairCollection(CollectionName);
 
                        Log.Informational("Repaired collection from ledger.", CollectionName);
 
                        try
                        {
                            File.Delete(FileName2);
                        }
                        catch (Exception ex)
                        {
                            Log.Error(ex, FileName2);
                        }
                    }
                    catch (Exception ex2)
                    {
                        Log.Exception(ex2, FileName2);
                    }
                }
            }
            else
                Log.Warning("Ledger settings changed. Restart the system for changes to take effect.");
        }
 
        private string RepairFolder
        {
            get
            {
                string Result = Path.Combine(this.LedgerFolder, "Repair");
                if (!Directory.Exists(Result))
                    Directory.CreateDirectory(Result);
 
                return Result;
            }
        }
 
        private string AddRepairNoteFile(string Collection)
        {
            string FileName = Path.Combine(this.RepairFolder, Collection + ".txt");
 
            try
            {
                if (!File.Exists(FileName))
                    File.WriteAllText(FileName, DateTime.UtcNow.ToString());
            }
            catch (Exception ex)
            {
                Log.Exception(ex);
            }
 
            return FileName;
        }
 
        internal async Task Database_CollectionRepaired(object Sender, CollectionEventArgs e)
        {
            try
            {
                if (!(Ledger.Provider is NeuroLedgerProvider Provider))
                    return;
 
                string FileName = this.AddRepairNoteFile(e.Collection);
 
                lock (this.repairQueue)
                {
                    if (this.repairing)
                    {
                        this.repairQueue.AddLast(FileName);
                        return;
                    }
 
                    this.repairing = true;
                }
 
                while (!string.IsNullOrEmpty(FileName))
                {
                    string CollectionName = Path.GetFileName(FileName);
                    CollectionName = CollectionName[..^4];
 
                    if (CollectionName == BlockReference.BlockReferencesCollection)
                        await Provider.RepairRegistry();
                    else
                        await Provider.RepairCollection(CollectionName);
 
                    try
                    {
                        File.Delete(FileName);
                    }
                    catch (Exception ex)
                    {
                        Log.Exception(ex, FileName);
                    }
 
                    lock (this.repairQueue)
                    {
                        if (this.repairQueue.First is null)
                        {
                            FileName = null;
                            this.repairing = false;
                        }
                        else
                        {
                            FileName = this.repairQueue.First.Value;
                            this.repairQueue.RemoveFirst();
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                Log.Exception(ex);
            }
        }
 
        private readonly LinkedList<string> repairQueue = new LinkedList<string>();
        private bool repairing = false;
 
        public override Task<bool> SimplifiedConfiguration()
        {
            return Task.FromResult(true);
        }
 
        public override Task InitSetup(HttpServer WebServer)
        {
            this.setLedgerProperties = WebServer.Register("/Settings/SetLedgerProperties", null, this.SetLedgerProperties, true, false, true);
 
            return base.InitSetup(WebServer);
        }
 
        public override Task UnregisterSetup(HttpServer WebServer)
        {
            WebServer.Unregister(this.setLedgerProperties);
 
            return base.UnregisterSetup(WebServer);
        }
 
        protected override string ConfigPrivilege => "Admin.NeuroLedger.Settings";
 
        private async Task SetLedgerProperties(HttpRequest Request, HttpResponse Response)
        {
            Gateway.AssertUserAuthenticated(Request, this.ConfigPrivilege);
 
            if (!Request.HasData)
            {
                await Response.SendResponse(new BadRequestException());
                return;
            }
 
            ContentResponse Content = await Request.DecodeDataAsync();
            if (Content.HasError || !(Content.Decoded is IDictionary<string, object> Data))
            {
                await Response.SendResponse(new UnsupportedMediaTypeException("Expected form."));
                return;
            }
 
            if (!Data.TryGetValue("collectionTime", out object Obj) ||
                !(Obj is string CollectionTimeStr) ||
                !int.TryParse(CollectionTimeStr, out int CollectionTime) ||
                CollectionTime <= 0 ||
                CollectionTime > 3600)
            {
                await Response.SendResponse(new BadRequestException("Collection time can be between 1 and 3600 seconds."));
                return;
            }
 
            if (!Data.TryGetValue("blockSizeThreshold", out Obj) ||
                !(Obj is string BlockSizeThresholdStr) ||
                !int.TryParse(BlockSizeThresholdStr, out int BlockSizeThreshold) ||
                BlockSizeThreshold <= 0 ||
                BlockSizeThreshold > 1024 * 1024 * 1024)
            {
                await Response.SendResponse(new BadRequestException("Block Size Threshold can be between 1 and 1073741824 bytes (1 GB)."));
                return;
            }
 
            this.collectionTimeSeconds = CollectionTime;
            this.blockSizeThreshold = BlockSizeThreshold;
 
            await Database.Update(this);
 
            Response.StatusCode = 200;
            Response.StatusMessage = "OK";
        }
 
        public static byte[] Sign(byte[] Data)
        {
            Assert.CallFromSource((DomainConfiguration.Instance?.UseDomainName ?? false) ? approvedSources : approvedSourcesTest);
            return instance?.ed448?.Sign(Data);
        }
 
        public static byte[] Sign(Stream Data)
        {
            Assert.CallFromSource((DomainConfiguration.Instance?.UseDomainName ?? false) ? approvedSources : approvedSourcesTest);
            return instance?.ed448?.Sign(Data);
        }
 
        public static bool Verify(byte[] Data, byte[] Signature)
        {
            return instance?.ed448?.Verify(Data, instance?.PublicKey, Signature) ?? false;
        }
 
        public const string NEURO_LEDGER_COLLECTION = nameof(NEURO_LEDGER_COLLECTION);
 
        public const string NEURO_LEDGER_MAXSIZE = nameof(NEURO_LEDGER_MAXSIZE);
 
        public override Task<bool> EnvironmentConfiguration()
        {
            string Value = Environment.GetEnvironmentVariable(NEURO_LEDGER_COLLECTION);
            if (!string.IsNullOrEmpty(Value))
            {
                if (!int.TryParse(Value, out int i))
                {
                    this.LogEnvironmentVariableInvalidIntegerError(NEURO_LEDGER_COLLECTION, Value);
                    return Task.FromResult(false);
                }
 
                if (i < 1 || i > 3600)
                {
                    this.LogEnvironmentVariableInvalidRangeError(1, 3600, NEURO_LEDGER_COLLECTION, i);
                    return Task.FromResult(false);
                }
 
                this.collectionTimeSeconds = i;
            }
 
            Value = Environment.GetEnvironmentVariable(NEURO_LEDGER_MAXSIZE);
            if (!string.IsNullOrEmpty(Value))
            {
                if (!int.TryParse(Value, out int i))
                {
                    this.LogEnvironmentVariableInvalidIntegerError(NEURO_LEDGER_MAXSIZE, Value);
                    return Task.FromResult(false);
                }
 
                if (i < 1 || i > 1073741824)
                {
                    this.LogEnvironmentVariableInvalidRangeError(1, 1073741824, NEURO_LEDGER_MAXSIZE, i);
                    return Task.FromResult(false);
                }
 
                this.blockSizeThreshold = i;
            }
 
            return Task.FromResult(true);
        }
    }
}