NeuroLedgerModule.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using System.Xml;
using Waher.Content.Xml;
using Waher.Events;
using Waher.IoTGateway;
using Waher.IoTGateway.Setup;
using Waher.Networking.HTTP;
using Waher.Networking.PeerToPeer;
using Waher.Networking.Sniffers;
using Waher.Networking.XMPP;
using Waher.Networking.XMPP.Contracts;
using Waher.Networking.XMPP.Contracts.EventArguments;
using Waher.Networking.XMPP.Events;
using Waher.Networking.XMPP.HTTPX;
using Waher.Networking.XMPP.NeuroLedger;
using Waher.Networking.XMPP.P2P;
using Waher.Persistence;
using Waher.Persistence.NeuroLedger;
using Waher.Persistence.NeuroLedger.Registry;
using Waher.Persistence.Serialization;
using Waher.Runtime.Inventory;
using Waher.Runtime.Settings;
using Waher.Runtime.Threading;
using Waher.Service.NeuroLedger.Setup;
using Waher.Service.NeuroLedger.Web;
 
namespace Waher.Service.NeuroLedger
{
    [Singleton]
    public class NeuroLedgerModule : IModule
    {
        private const string DefaultDataProtectionAgreementTemplate = "25e2a2b5-4184-5cc8-a824-8b9e562c081b@legal.waher.se";
        private static NeuroLedgerModule instance = null;
 
        private NeuroLedgerClient client = null;
        private NeuroLedgerProvider provider = null;
        private HttpResource synchronizationResource;
        private readonly ManualResetEvent p2pInitiationComplete = new ManualResetEvent(false);
        private EndpointSecurity e2eEncryption;
        private XmppServerlessMessaging serverlessMessaging;
        private bool disposeE2eEncryption = false;
        private bool started = false;
        private bool presenceSent = false;
 
        public NeuroLedgerModule()
        {
        }
 
        public async Task Start()
        {
            instance = this;
 
            this.provider = Ledger.Provider as NeuroLedgerProvider;
            if (this.provider is null)
                return;
 
            try
            {
                Database.ObjectInserted += Database_ObjectInserted;
                Database.ObjectUpdated += Database_ObjectUpdated;
                Database.ObjectDeleted += Database_ObjectDeleted;
                Database.CollectionCleared += this.Database_CollectionCleared;
                Database.CollectionRepaired += Database_CollectionRepaired;
 
                RosterConfiguration.OnGetGroupSuggestions += this.RosterConfiguration_OnGetGroupSuggestions;
 
                Gateway.OnAfterBackup += this.AfterBackup;
 
                Gateway.XmppClient.EntityHashFunction = Security.HashFunction.SHA256;
                Gateway.XmppClient.EntityNode = "http://waher.se/NL";
                Gateway.XmppClient.SetTag("ShowE2E", true);
 
                List<ISniffer> P2PSniffers = new List<ISniffer>();
                XmlFileSniffer XmlFileSniffer;
                bool HasSniffers = false;
 
                foreach (ISniffer Sniffer in Gateway.XmppClient.Sniffers)
                {
                    HasSniffers = true;
 
                    XmlFileSniffer = Sniffer as XmlFileSniffer;
                    if (!(XmlFileSniffer is null))
                    {
                        XmlFileSniffer = new XmlFileSniffer(Path.Combine(Gateway.AppDataFolder, "UPnP", "UPnP Log %YEAR%-%MONTH%-%DAY%T%HOUR%.xml"),
                            Path.Combine(Gateway.AppDataFolder, "Transforms", "SnifferXmlToHtml.xslt"), 7, BinaryPresentationMethod.ByteCount);
 
                        P2PSniffers.Add(XmlFileSniffer);
                    }
                }
 
                Gateway.XmppClient.OnStateChanged += this.XmppClient_OnStateChanged;
 
                this.p2pInitiationComplete.Reset();
                this.serverlessMessaging = new XmppServerlessMessaging("Neuro-Ledger", Gateway.XmppClient.FullJID,
                    (ushort)await RuntimeSettings.GetAsync("NL.P2P.LocalPort", 0),
                    (ushort)await RuntimeSettings.GetAsync("NL.P2P.ExternalPort", 0), P2PSniffers.ToArray());
                this.serverlessMessaging.Network.OnStateChange += this.P2PNetwork_OnStateChange;
                this.serverlessMessaging.OnNewXmppClient += this.ServerlessMessaging_OnNewXmppClient;
                this.serverlessMessaging.OnResynch += this.ServerlessMessaging_OnResynch;
 
                using Runtime.Threading.Semaphore Lock = await Semaphores.BeginWrite(
                    ContractsClient.E2eKeySemaphoreName);
                {
                    if (Networking.XMPP.P2P.EndpointSecurity.TryGetEndpointSecurity(
                        Gateway.XmppClient, out EndpointSecurity EndpointSecurity))
                    {
                        this.e2eEncryption = EndpointSecurity;
                    }
                    else
                    {
                        this.e2eEncryption = new EndpointSecurity(Gateway.XmppClient, this.serverlessMessaging, 128);
                        this.disposeE2eEncryption = true;
                    }
                }
 
                this.e2eEncryption.PeerUnavailable += this.E2eEncryption_PeerUnavailable;
 
                Gateway.AvatarClient.E2E = this.e2eEncryption;
 
                if (HasSniffers)
                {
                    ChunkCounter Counter = new ChunkCounter(true, Gateway.XmppClient, false);
                    Gateway.XmppClient.OnReceiveText += Counter.ShortenChunks;
                    Gateway.XmppClient.OnInformation += Counter.RemoveChunkMessage;
                    Gateway.XmppClient.OnDisposed += Counter.Client_OnDisposed;
 
                    Counter = new ChunkCounter(false, Gateway.XmppClient, false);
                    Gateway.XmppClient.OnTransmitText += Counter.ShortenChunks;
                    Gateway.XmppClient.OnDisposed += Counter.Client_OnDisposed;
                }
 
                if (!(Gateway.HttpxProxy is null))
                {
                    if (Gateway.HttpxProxy.ServerlessMessaging is null)
                        Gateway.HttpxProxy.ServerlessMessaging = this.serverlessMessaging;
 
                    if (Gateway.HttpxProxy.DefaultHttpxClient.E2e is null)
                        Gateway.HttpxProxy.DefaultHttpxClient.E2e = this.e2eEncryption;
                }
 
                if (!(Gateway.HttpxServer is null))
                    Gateway.HttpxServer.RequiresE2e = true;
 
                Gateway.ContractsClient.PetitionForIdentityReceived += this.ContractsClient_PetitionForIdentityReceived;
 
                this.client = new NeuroLedgerClient(Gateway.XmppClient, this.e2eEncryption, Gateway.HttpServer, this.provider);
                this.synchronizationResource = Gateway.HttpServer.Register(new SynchronizationResource("/NL/Synchronize", this.client, "User"));
 
                this.p2pInitiationComplete.WaitOne(60000);
 
                this.started = true;
                Log.Informational("Neuro-Ledger successfully started.");
 
                if (!(Gateway.XmppClient is null) && Gateway.XmppClient.State == XmppState.Connected)
                {
                    if (!this.presenceSent)
                        await this.SetPresence();
 
                    await this.CheckContracts();
                }
            }
            catch (Exception ex)
            {
                Log.Exception(ex);
            }
        }
 
        private async Task ContractsClient_PetitionForIdentityReceived(object Sender, LegalIdentityPetitionEventArgs e)
        {
            if (LegalIdentityConfiguration.TryGetMyIdentity(e.RequestedIdentityId, out LegalIdentity Id))
            {
                bool Accept = !(Id.State == IdentityState.Compromised || Id.State == IdentityState.Rejected);
                await Gateway.ContractsClient.PetitionIdentityResponseAsync(e.RequestedIdentityId, e.PetitionId, e.RequestorFullJid, Accept);
 
                List<KeyValuePair<string, object>> Tags = new List<KeyValuePair<string, object>>()
                {
                    new KeyValuePair<string, object>("RequestorFullJid", e.RequestorFullJid),
                    new KeyValuePair<string, object>("Provider", e.RequestorIdentity.Provider)
                };
 
                foreach (Property P in e.RequestorIdentity.Properties)
                    Tags.Add(new KeyValuePair<string, object>(P.Name, P.Value));
 
                Log.Informational("Legal ID petitioned.", e.RequestedIdentityId, e.RequestedIdentityId, "IdPetition", Tags.ToArray());
            }
            else
                await Gateway.ContractsClient.PetitionIdentityResponseAsync(e.Id, e.PetitionId, e.RequestorFullJid, false);
        }
 
        private Task RosterConfiguration_OnGetGroupSuggestions(object Sender, SuggestionEventArgs e)
        {
            if (Ledger.Provider is NeuroLedgerProvider NeuroLedgerProvider)
            {
                foreach (string Collection in NeuroLedgerProvider.Collections)
                    e.AddSuggestion(Collection);
            }
 
            return Task.CompletedTask;
        }
 
        public async Task Stop()
        {
            instance = null;
            this.started = false;
 
            if (!(this.client is null))
            {
                this.client.Dispose();
                this.client = null;
 
                if (this.disposeE2eEncryption)
                    this.e2eEncryption?.Dispose();
 
                this.e2eEncryption = null;
 
                if (!(this.serverlessMessaging is null))
                {
                    await this.serverlessMessaging.DisposeAsync();
                    this.serverlessMessaging = null;
                }
 
                Gateway.OnAfterBackup -= this.AfterBackup;
 
                RosterConfiguration.OnGetGroupSuggestions -= this.RosterConfiguration_OnGetGroupSuggestions;
 
                Database.ObjectInserted -= Database_ObjectInserted;
                Database.ObjectUpdated -= Database_ObjectUpdated;
                Database.ObjectDeleted -= Database_ObjectDeleted;
                Database.CollectionCleared -= this.Database_CollectionCleared;
                Database.CollectionRepaired -= Database_CollectionRepaired;
 
                Gateway.XmppClient.OnStateChanged -= this.XmppClient_OnStateChanged;
 
                Gateway.ContractsClient.PetitionForIdentityReceived -= this.ContractsClient_PetitionForIdentityReceived;
 
                Gateway.HttpServer.Unregister(this.synchronizationResource);
                this.synchronizationResource = null;
            }
        }
 
        private Task CheckContracts()
        {
            return this.CheckContracts(null);
        }
 
        private async Task CheckContracts(object P)
        {
            Contract Dpa;
 
            if (!string.IsNullOrEmpty(LedgerConfiguration.Instance.DataProtectionAgreementId))
            {
                if (Gateway.ContractsClient is null)
                {
                    Gateway.ScheduleEvent(this.CheckContracts, DateTime.Now.AddMinutes(1), null);
                    return;
                }
                else
                    Dpa = await Gateway.ContractsClient.GetContractAsync(LedgerConfiguration.Instance.DataProtectionAgreementId);
            }
            else
            {
                try
                {
                    string Xml = File.ReadAllText(Path.Combine(Gateway.AppDataFolder, "Contracts", "BlockSubscription.xml"));
                    XmlDocument Doc = XML.ParseXml(Xml, true);
 
                    ParsedContract Parsed = await Contract.Parse(Doc.DocumentElement, Gateway.ContractsClient, true);
                    Contract Template = Parsed.Contract;
 
                    string TemplateId = await RuntimeSettings.GetAsync("NL.DPA.TemplateId", DefaultDataProtectionAgreementTemplate);
                    if (string.IsNullOrEmpty(TemplateId))
                    {
                        Template = await Gateway.ContractsClient.CreateContractAsync(Template.ForMachines, Template.ForHumans,
                            Template.Roles, Template.Parts, Template.Parameters, Template.Visibility, Template.PartsMode, Template.Duration,
                            Template.ArchiveRequired, Template.ArchiveOptional, Template.SignAfter, Template.SignBefore, true);
 
                        await RuntimeSettings.SetAsync("NL.DPA.TemplateId", TemplateId = Template.ContractId);
                    }
 
                    Dpa = await Gateway.ContractsClient.CreateContractAsync(TemplateId, Array.Empty<Part>(),
                        new Parameter[] 
                        { 
                            new StringParameter() 
                            { 
                                Name = "Source", 
                                Value = Gateway.XmppClient.BareJID 
                            } 
                        },
                        ContractVisibility.Public, ContractParts.Open, Template.Duration, Template.ArchiveRequired,
                        Template.ArchiveOptional, null, null, false);
                }
                catch (Exception ex)
                {
                    Log.Error("Unable to create Data Protection Smart Contract.",
                        new KeyValuePair<string, object>("Message", ex.Message));
                    return;
                }
 
                LedgerConfiguration.Instance.DataProtectionAgreementId = Dpa.ContractId;
                await Database.Update(LedgerConfiguration.Instance);
            }
 
            bool GeneratorSigned = false;
 
            if (!(Dpa?.ClientSignatures is null))
            {
                foreach (ClientSignature Signature in Dpa.ClientSignatures)
                {
                    if (Signature.Role == "Generator")
                    {
                        GeneratorSigned = true;
                        break;
                    }
                }
            }
 
            if (!GeneratorSigned)
            {
                await Gateway.RequestContractSignature(Dpa, "Generator", "Data Protection Agreement required to be signed by " +
                    "designated parties before they can access blocks generated by the Neuro-Ledger on this machine.");
            }
        }
 
        private static readonly Dictionary<Type, bool> processInLedger = new Dictionary<Type, bool>();
 
        private static async Task<bool> Process(object Object)
        {
            Type T = Object.GetType();
            bool Result;
 
            lock (processInLedger)
            {
                if (processInLedger.TryGetValue(T, out Result))
                    return Result;
            }
 
            if (Ledger.Provider is NeuroLedgerProvider NeuroLedgerProvider)
            {
                ObjectSerializer Serializer = await NeuroLedgerProvider.GetObjectSerializerEx(T);
                Result = Serializer.ArchiveObjects;
            }
            else
                Result = false;
 
            lock (processInLedger)
            {
                processInLedger[T] = Result;
            }
 
            return Result;
        }
 
        private static async void Database_ObjectInserted(object Sender, ObjectEventArgs e)
        {
            try
            {
                if (!await Process(e.Object))
                    return;
 
                GenericObject Obj = await Database.Generalize(e.Object);
                await Ledger.NewEntry(Obj);
            }
            catch (Exception ex)
            {
                Log.Exception(ex);
            }
        }
 
        private static async void Database_ObjectUpdated(object Sender, ObjectEventArgs e)
        {
            try
            {
                if (!await Process(e.Object))
                    return;
 
                GenericObject Obj = await Database.Generalize(e.Object);
                await Ledger.UpdatedEntry(Obj);
            }
            catch (Exception ex)
            {
                Log.Exception(ex);
            }
        }
 
        private static async void Database_ObjectDeleted(object Sender, ObjectEventArgs e)
        {
            try
            {
                if (!await Process(e.Object))
                    return;
 
                GenericObject Obj = await Database.Generalize(e.Object);
                await Ledger.DeletedEntry(Obj);
            }
            catch (Exception ex)
            {
                Log.Exception(ex);
            }
        }
 
        private async Task Database_CollectionCleared(object Sender, CollectionEventArgs e)
        {
            try
            {
                string Collection = e.Collection;
 
                if (Array.IndexOf(this.provider.Collections, Collection) >= 0)
                    await Ledger.ClearedCollection(Collection);
            }
            catch (Exception ex)
            {
                Log.Exception(ex);
            }
        }
 
        private async Task XmppClient_OnStateChanged(object Sender, XmppState NewState)
        {
            try
            {
                if (Gateway.XmppClient is null)
                    return;
 
                switch (NewState)
                {
                    case XmppState.RequestingSession:
                        if (!(this.serverlessMessaging is null))
                            this.serverlessMessaging.FullJid = Gateway.XmppClient.FullJID;
                        break;
 
                    case XmppState.Connected:
                        foreach (RosterItem Item in Gateway.XmppClient.Roster)
                        {
                            if (Item.PendingSubscription == PendingSubscription.Subscribe && Item.BareJid != Gateway.XmppClient.BareJID)
                                await Gateway.XmppClient.RequestPresenceSubscription(Item.BareJid);
                        }
 
                        await this.CheckContracts();
                        break;
                }
            }
            catch (Exception ex)
            {
                Log.Exception(ex);
            }
        }
 
        private Task ServerlessMessaging_OnNewXmppClient(object Sender, PeerConnectionEventArgs e)
        {
            XmppClient Client = e.Client;
            HttpxServer HttpxServer = new HttpxServer(Client, Gateway.HttpServer, 32768);
            HttpxClient HttpxClient = new HttpxClient(Client, this.e2eEncryption, 32768)
            {
                PostResource = Gateway.HttpxProxy.PostResource
            };
            XmlFileSniffer XmlFileSniffer;
            bool Sniffers = false;
 
            HttpxServer.RequiresE2e = true;
 
            Client.SetTag("HttpxServer", HttpxServer);
            Client.SetTag("HttpxClient", HttpxClient);
            Client.SetTag("ShowE2E", false);
            Client.SetTag("ExpectedSender", e.RemoteJid.ToLower());
 
            Client.OnValidateSender += this.PeerClient_OnValidateSender;
            Client.OnDisposed += this.P2PClient_OnDisposed;
 
            this.e2eEncryption.RegisterHandlers(Client);
 
            try
            {
                foreach (ISniffer Sniffer in Gateway.XmppClient.Sniffers)
                {
                    Sniffers = true;
 
                    XmlFileSniffer = Sniffer as XmlFileSniffer;
                    if (!(XmlFileSniffer is null) && Types.TryGetModuleParameter("AppData", out string AppDataFolder))
                    {
                        XmlFileSniffer = new XmlFileSniffer(AppDataFolder + "XMPP" + Path.DirectorySeparatorChar +
                            e.RemoteJid + Path.DirectorySeparatorChar + "XMPP Log %YEAR%-%MONTH%-%DAY%T%HOUR%.xml",
                            AppDataFolder + "Transforms" + Path.DirectorySeparatorChar + "SnifferXmlToHtml.xslt",
                            7, BinaryPresentationMethod.ByteCount);
 
                        Client.Add(XmlFileSniffer);
                    }
                    else
                        Client.Add(Sniffer);
                }
            }
            catch (Exception ex)
            {
                Log.Exception(ex);
            }
 
            if (Sniffers)
            {
                ChunkCounter Counter = new ChunkCounter(true, Client, true);
                Client.OnReceiveText += Counter.ShortenChunks;
                Client.OnInformation += Counter.RemoveChunkMessage;
                Client.OnDisposed += Counter.Client_OnDisposed;
 
                Counter = new ChunkCounter(false, Client, true);
                Client.OnTransmitText += Counter.ShortenChunks;
                Client.OnDisposed += Counter.Client_OnDisposed;
            }
 
            return Task.CompletedTask;
        }
 
        private Task ServerlessMessaging_OnResynch(object Sender, ResynchEventArgs e)
        {
            this.e2eEncryption?.SynchronizeE2e(e.RemoteFullJid, (Sender2, e2) =>
            {
                e.Done(e2.Ok);
                return Task.CompletedTask;
            });
 
            return Task.CompletedTask;
        }
 
        private Task PeerClient_OnValidateSender(object Sender, ValidateSenderEventArgs e)
        {
            if (!e.Client.TryGetTag("ExpectedSender", out object Obj) || !(Obj is string s) || string.Compare(e.From, s, true) != 0)
                e.Reject();
 
            return Task.CompletedTask;
        }
 
        private async Task P2PClient_OnDisposed(object Sender, EventArgs e)
        {
            if (Sender is XmppClient Client)
            {
                this.e2eEncryption?.UnregisterHandlers(Client);
 
                if (Client.TryGetTag("HttpxServer", out object Obj))
                {
                    if (Obj is IDisposableAsync DisposableAsync)
                        await DisposableAsync.DisposeAsync();
                    else if (Obj is IDisposable Disposable)
                        Disposable.Dispose();
                }
 
                if (Client.TryGetTag("HttpxClient", out Obj))
                {
                    if (Obj is IDisposableAsync DisposableAsync)
                        await DisposableAsync.DisposeAsync();
                    else if (Obj is IDisposable Disposable)
                        Disposable.Dispose();
                }
            }
        }
 
        private async Task P2PNetwork_OnStateChange(object _, PeerToPeerNetworkState NewState)
        {
            if (this.serverlessMessaging is null)
                return;
 
            try
            {
                PeerToPeerNetwork P2P = this.serverlessMessaging.Network;
 
                switch (NewState)
                {
                    case PeerToPeerNetworkState.Ready:
                        Log.Informational("Gateway available on Peer-to-Peer network: " + P2P.ExternalEndpoint.ToString());
 
                        P2P.DesiredLocalPort = (ushort)P2P.LocalEndpoint.Port;
                        P2P.DesiredExternalPort = (ushort)P2P.ExternalEndpoint.Port;
 
                        await RuntimeSettings.SetAsync("NL.P2P.LocalPort", P2P.LocalEndpoint.Port);
                        await RuntimeSettings.SetAsync("NL.P2P.ExternalPort", P2P.ExternalEndpoint.Port);
 
                        if (this.started && Gateway.XmppClient.State == XmppState.Connected)
                            await this.SetPresence();
 
                        this.p2pInitiationComplete.Set();
                        break;
 
                    case PeerToPeerNetworkState.Error:
                        P2P.DesiredLocalPort = 0;
                        P2P.DesiredExternalPort = 0;
 
                        this.p2pInitiationComplete.Set();
                        break;
                }
            }
            catch (Exception ex)
            {
                Log.Exception(ex);
            }
        }
 
        private Task E2eEncryption_PeerUnavailable(object Sender, PresenceEventArgs e)
        {
            this.serverlessMessaging?.RemovePeerAddresses(e.From);
            return Task.CompletedTask;
        }
 
        public static XmppServerlessMessaging ServerlessMessaging
        {
            get { return instance?.serverlessMessaging; }
        }
 
        public static EndpointSecurity E2eEncryption
        {
            get { return instance?.e2eEncryption; }
        }
 
        private async Task SetPresence()
        {
            if (!(Gateway.XmppClient is null) && this.started)
            {
                await Gateway.XmppClient.SetPresence(Availability.Online);
                this.presenceSent = true;
            }
        }
 
        private async Task AfterBackup(object Sender, EventArgs e)
        {
            try
            {
                this.e2eEncryption.GenerateNewKey();
                await this.SetPresence();
            }
            catch (Exception ex)
            {
                Log.Exception(ex);
            }
        }
 
        private static Task Database_CollectionRepaired(object Sender, CollectionEventArgs e)
        {
            return LedgerConfiguration.Instance?.Database_CollectionRepaired(Sender, e) ?? Task.CompletedTask;
        }
 
        public static Task ProcessBlockRequest(Networking.HTTP.HeaderFields.HttpFieldAccept Accept, HttpResponse Response, BlockReference Ref)
        {
            return instance.client.ProcessBlockRequest(Accept, Response, Ref);
        }
 
    }
}