NeuroLedgerProvider.cs

using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using Waher.Content;
using Waher.Events;
using Waher.Persistence.Filters;
using Waher.Persistence.NeuroLedger.Registry;
using Waher.Persistence.Serialization;
using Waher.Runtime.Cache;
using Waher.Runtime.Inventory;
using Waher.Runtime.Profiling;
using Waher.Runtime.Queue;
using Waher.Runtime.Settings;
using Waher.Security;
using Waher.Security.CallStack;
 
namespace Waher.Persistence.NeuroLedger
{
    public class NeuroLedgerProvider : ILedgerProvider, ISerializerContext, IDisposable
    {
        private const int BlockPageSize = 10;
 
        private readonly byte[] salt;
        private readonly string id;
        private readonly string folder;
        private readonly string bucketFolder;
        private readonly string blockFolder;
        private readonly string defaultCollectionName;
        private readonly string externalIdentity;
        private readonly int saltLength;
        private readonly int maxBlockSize;
        private readonly bool debug;
        private readonly AsyncQueue<WorkItem> eventQueue = new AsyncQueue<WorkItem>();
        private readonly ISignatureAlgorithm signatureAlgorithm;
        private readonly HashFunctionStream hashFunction;
        private readonly SortedDictionary<string, bool> collections = new SortedDictionary<string, bool>();
        private TaskCompletionSource<bool> completed = new TaskCompletionSource<bool>();
        private SerializerCollection serializers;
        private Cache<string, Bucket> buckets;
        private AesCryptoServiceProvider aes;
        private ILedgerExternalEvents externalEvents;
        private bool disposed = false;
        private bool stopped = false;
 
        public NeuroLedgerProvider(string Folder, TimeSpan CollectionTime, int MaxBlockSize, byte[] Salt,
            string DefaultCollectionName, string ExternalIdentity, ISignatureAlgorithm SignatureAlgorithm,
            HashFunctionStream HashFunction, bool Debug)
        {
            this.id = Guid.NewGuid().ToString().Replace("-", string.Empty);
            this.folder = Path.GetFullPath(Folder);
            this.bucketFolder = Path.Combine(this.folder, "Buckets");
            this.blockFolder = Path.Combine(this.folder, "Blocks");
            this.salt = Salt;
            this.saltLength = this.salt.Length;
            this.defaultCollectionName = DefaultCollectionName;
            this.maxBlockSize = MaxBlockSize;
            this.externalIdentity = ExternalIdentity;
            this.signatureAlgorithm = SignatureAlgorithm;
            this.hashFunction = HashFunction;
            this.debug = Debug;
            this.serializers = new SerializerCollection(this, true);
 
            this.aes = new AesCryptoServiceProvider()
            {
                BlockSize = 128,
                KeySize = 256,
                Mode = CipherMode.CBC,
                Padding = PaddingMode.Zeros
            };
 
            if (!string.IsNullOrEmpty(this.folder) && this.folder[^1] != Path.DirectorySeparatorChar)
                this.folder += Path.DirectorySeparatorChar;
 
            if (!Directory.Exists(this.folder))
                Directory.CreateDirectory(this.folder);
 
            if (!Directory.Exists(this.bucketFolder))
                Directory.CreateDirectory(this.bucketFolder);
 
            if (!Directory.Exists(this.blockFolder))
                Directory.CreateDirectory(this.blockFolder);
 
            this.blockFolder += Path.DirectorySeparatorChar;
 
            this.buckets = new Cache<string, Bucket>(int.MaxValue, CollectionTime, CollectionTime, true);
            this.buckets.Removed += this.Buckets_Removed;
 
            Task _ = this.LoadCollections();
        }
 
        private async Task LoadCollections()
        {
            string s = await RuntimeSettings.GetAsync("NL.Collections", string.Empty);
 
            if (!string.IsNullOrEmpty(s))
            {
                lock (this.collections)
                {
                    foreach (string Collection in s.Split(CommonTypes.CRLF, StringSplitOptions.RemoveEmptyEntries))
                        this.collections[Collection] = true;
                }
            }
        }
 
        private Task Buckets_Removed(object Sender, CacheItemEventArgs<string, Bucket> e)
        {
            if (!this.disposed && !this.stopped)
            {
                // Bucket is set, and Object is null, signifies Bucket is completed and block
                // is to be generated.
 
                this.eventQueue.QueueFirst(new WorkItem()
                {
                    Bucket = e.Value,
                    Type = EntryType.New,
                    Object = null
                });
            }
 
            return Task.CompletedTask;
        }
 
        private class WorkItem
        {
            public Bucket Bucket;
            public EntryType Type;
            public object Object;
        }
 
        public string BucketFolder => this.bucketFolder;
 
        public string BlockFolder => this.blockFolder;
 
        public HashFunctionStream HashFunction => this.hashFunction;
 
        public ISignatureAlgorithm SignatureAlgorithm => this.signatureAlgorithm;
 
        private async Task StorageTask()
        {
            try
            {
                WorkItem Item;
                Type LastType = null;
                ObjectSerializer LastSerializer = null;
                DateTime NextCheck = DateTime.Now.AddMinutes(1);
 
                while (!((Item = await this.eventQueue.Wait()) is null))
                {
                    try
                    {
                        Bucket Bucket = Item.Bucket;
                        if (Bucket is null)
                        {
                            // Write Entry
 
                            EntryType EntryType = Item.Type;
                            object Object = Item.Object;
                            Type T = Object.GetType();
                            ObjectSerializer Serializer;
                            string CollectionName;
                            string BucketName;
                            bool DynamicArchiveTime;
                            int? ArchivingTimeDays;
 
                            if (T == LastType)
                                Serializer = LastSerializer;
                            else
                            {
                                Serializer = await this.GetObjectSerializerEx(Object);
                                LastType = T;
                                LastSerializer = Serializer;
                            }
 
                            if (!Serializer.ArchiveObjects)
                                continue;
 
                            ArchivingTimeDays = Serializer.GetArchivingTimeDays(Object);
                            DynamicArchiveTime = Serializer.ArchiveTimeDynamic;
                            CollectionName = await Serializer.CollectionName(Object);
                            if (string.IsNullOrEmpty(CollectionName))
                                CollectionName = this.defaultCollectionName;
 
                            BucketName = CollectionName;
 
                            if (ArchivingTimeDays.HasValue && ArchivingTimeDays.Value != int.MaxValue)
                                BucketName += "." + ArchivingTimeDays.Value.ToString();
                            else
                                BucketName += ".x";
 
                            if (ArchivingTimeDays.HasValue && ArchivingTimeDays.Value <= 0)
                                continue;
 
                            BinarySerializer Output = new BinarySerializer(CollectionName, Encoding.UTF8);
                            byte[] Binary;
 
                            try
                            {
                                await Serializer.Serialize(Output, false, false, Object, null);
                                Binary = Output.GetSerialization();
                            }
                            catch (Exception ex)
                            {
                                Guid? ObjectId;
 
                                try
                                {
                                    ObjectId = await Serializer.GetObjectId(Object, false, null);
                                }
                                catch (Exception)
                                {
                                    ObjectId = null;
                                }
 
                                Binary = null;
                                Log.Alert("Unable to store object in ledger.",
                                    new KeyValuePair<string, object>("Type", T.FullName),
                                    new KeyValuePair<string, object>("Collection", CollectionName),
                                    new KeyValuePair<string, object>("ObjectId", ObjectId.HasValue ? ObjectId.Value.ToString() : string.Empty),
                                    new KeyValuePair<string, object>("Message", ex.Message),
                                    new KeyValuePair<string, object>("StackTrace", ex.StackTrace),
                                    new KeyValuePair<string, object>("JSON", JSON.Encode(Object, true)));
                            }
 
                            if (!(Binary is null))
                            {
                                if (!this.buckets.TryGetValue(BucketName, out Bucket))
                                {
                                    DateTime Expires;
 
                                    if (ArchivingTimeDays.HasValue && ArchivingTimeDays.Value < int.MaxValue)
                                    {
                                        try
                                        {
                                            Expires = DateTime.UtcNow.AddDays(ArchivingTimeDays.Value);
                                        }
                                        catch (ArgumentOutOfRangeException ex)
                                        {
                                            Expires = DateTime.MaxValue;
 
                                            Log.Error("Invalid archiving time encountered.", T.FullName, "Neuro-Ledger",
                                                new KeyValuePair<string, object>("Type", T.FullName),
                                                new KeyValuePair<string, object>("ArchivingTimeDays", ArchivingTimeDays.Value),
                                                new KeyValuePair<string, object>("Message", ex.Message));
                                        }
                                    }
                                    else
                                        Expires = DateTime.MaxValue;
 
                                    try
                                    {
                                        Bucket = await Bucket.Create(Path.Combine(this.bucketFolder, BucketName + ".bin"), CollectionName, Expires, this);
                                    }
                                    catch (IOException)
                                    {
                                        // File locked. Place item last and try again.
                                        // Note: This can happen when an item event is
                                        // registered at the same time as a bucket
                                        // collection time elapses and a block is generated.
 
                                        this.eventQueue.QueueLast(Item);
                                        await Task.Delay(10);   // Small delay to avoid spam and 100% CPU while file is saved.
 
                                        continue;
                                    }
 
                                    this.buckets.Add(BucketName, Bucket);
 
                                    string[] Collections = null;
 
                                    lock (this.collections)
                                    {
                                        if (!this.collections.ContainsKey(CollectionName))
                                        {
                                            this.collections[CollectionName] = true;
 
                                            Collections = new string[this.collections.Count];
                                            this.collections.Keys.CopyTo(Collections, 0);
                                        }
                                    }
 
                                    if (!(Collections is null))
                                    {
                                        StringBuilder sb = new StringBuilder();
 
                                        foreach (string Collection in Collections)
                                            sb.AppendLine(Collection);
 
                                        await RuntimeSettings.SetAsync("NL.Collections", sb.ToString());
                                    }
                                }
 
                                if (await Bucket.WriteEntry(EntryType, Binary) > this.maxBlockSize)
                                    this.buckets.Remove(BucketName);    // Triggers removal event which will add bucket to queue to make it into a block.
                            }
                        }
                        else if (!Bucket.HasEntries)
                            await Bucket.Delete();
                        else
                        {
                            // Generate new block from contents in bucket
 
                            byte[] Digest = Bucket.Hash(this.hashFunction);
                            byte[] Signature = Bucket.Sign(this.signatureAlgorithm);
                            ulong Bytes = (ulong)Bucket.Length;
 
                            Digest = this.CalcCombinationDigest(Digest, Signature);
 
                            string FileName = this.GetFullFileName(Bucket.Header, Digest);
 
                            using (ICryptoTransform Aes = this.GetAes(FileName, true))
                            {
                                using FileStream fs = File.Create(FileName);
                                using CryptoStream cs = new CryptoStream(fs, Aes, CryptoStreamMode.Write);
 
                                await Bucket.CopyTo(cs, Signature);
 
                                cs.FlushFinalBlock();
                                await Bucket.Delete();
                            }
 
                            string LocalFileName = this.GetLocalFileName(FileName);
 
                            BlockReference Ref = new BlockReference(Bucket.Header, LocalFileName, Digest, Signature, Bytes);
                            await this.AddBlockReference(Ref);
                        }
 
                        DateTime Now = DateTime.Now;
                        if (Now >= NextCheck)
                        {
                            NextCheck = Now.AddHours(1);
                            await this.DeleteExpiredBlocks(Now);
                        }
                    }
                    catch (Exception ex)
                    {
                        Log.Exception(ex);
                    }
                }
            }
            catch (Exception ex)
            {
                Log.Exception(ex);
            }
            finally
            {
                this.completed.TrySetResult(true);
            }
        }
 
        private byte[] CalcCombinationDigest(byte[] ContentDigest, byte[] Signature)
        {
            int c = ContentDigest.Length;
            int d = Signature.Length;
            byte[] Bin = new byte[c + d];
 
            Buffer.BlockCopy(ContentDigest, 0, Bin, 0, c);
            Buffer.BlockCopy(Signature, 0, Bin, c, d);
 
            using MemoryStream ms = new MemoryStream();
 
            ms.Write(ContentDigest, 0, ContentDigest.Length);
            ms.Write(Signature, 0, Signature.Length);
 
            ms.Position = 0;
 
            return this.hashFunction(ms);
        }
 
        private string GetFullFileName(BlockHeader Header, byte[] Digest)
        {
            string FileName = Path.Combine(this.blockFolder,
                Header.Created.Year.ToString("D4"),
                Header.Created.Month.ToString("D2"),
                Header.Created.Day.ToString("D2"));
 
            if (!Directory.Exists(FileName))
                Directory.CreateDirectory(FileName);
 
            return Path.Combine(FileName, Hashes.BinaryToString(Digest)) + ".block";
        }
 
        private string GetLocalFileName(string FullFileName)
        {
            string LocalFileName;
 
            if (FullFileName.StartsWith(this.blockFolder))
                LocalFileName = FullFileName[this.blockFolder.Length..];
            else
                LocalFileName = FullFileName;
 
            return LocalFileName;
        }
 
        public async Task AddBlockFile(Stream File, BlockReference BlockReference)
        {
            File.Position = 0;
            BlockReader Reader = await BlockReader.CreateAsync(File, this);
 
            File.Position = 0;
            byte[] Digest = this.hashFunction(File);
 
            File.Position = 0;
            byte[] Signature = this.signatureAlgorithm.Sign(File, false);
 
            // TODO: Validate signature
 
            Digest = this.CalcCombinationDigest(Digest, Signature);
 
            string FileName = this.GetFullFileName(Reader.Header, Digest);
 
            using (ICryptoTransform Aes = this.GetAes(FileName, true))
            {
                using FileStream fs = System.IO.File.Create(FileName);
                using CryptoStream cs = new CryptoStream(fs, Aes, CryptoStreamMode.Write);
 
                File.Position = 0;
                File.CopyTo(cs);
                cs.FlushFinalBlock();
            }
 
            FileName = this.GetLocalFileName(FileName);
 
            if (BlockReference.FileName != FileName && !string.IsNullOrEmpty(FileName))
            {
                BlockReference.FileName = FileName;
                await Database.Update(BlockReference);
            }
 
            await this.BlockAdded.Raise(this, new BlockReferenceEventArgs(BlockReference));
        }
 
        public event EventHandlerAsync<BlockReferenceEventArgs> BlockAdded = null;
 
        private async Task AddBlockReference(BlockReference Block)
        {
            try
            {
                await Database.Insert(Block);
                await this.BlockAdded.Raise(this, new BlockReferenceEventArgs(Block));
            }
            catch (Exception ex)
            {
                Log.Exception(ex);
            }
        }
 
        public string GetFullFileName(string LocalFileName)
        {
            if (string.IsNullOrEmpty(LocalFileName))
                return string.Empty;
            else if (Path.IsPathRooted(LocalFileName))
                return LocalFileName;
            else
                return Path.Combine(this.blockFolder, LocalFileName);
        }
 
        private async Task DeleteExpiredBlocks(DateTime Now)
        {
            try
            {
                Dictionary<string, bool> Directories = null;
                string FileName;
 
                foreach (BlockReference Ref in await Database.Find<BlockReference>(new FilterFieldLesserOrEqualTo("Expires", Now)))
                {
                    FileName = this.GetFullFileName(Ref.FileName);
 
                    if (File.Exists(FileName))
                    {
                        try
                        {
                            File.Delete(FileName);
 
                            Directories ??= new Dictionary<string, bool>();
 
                            string Folder = Path.GetDirectoryName(FileName);
 
                            Directories[Folder] = true;
                        }
                        catch (Exception ex)
                        {
                            Log.Error("Unable to delete block.",
                                new KeyValuePair<string, object>("FileName", FileName),
                                new KeyValuePair<string, object>("Message", ex.Message));
 
                            continue;
                        }
                    }
 
                    await this.DeleteBlockReference(Ref);
                }
 
                if (!(Directories is null))
                {
                    foreach (string Key in Directories.Keys)
                    {
                        string FolderName = Key;
 
                        while (Directory.GetFiles(FolderName, "*.*", SearchOption.TopDirectoryOnly).Length == 0 &&
                            Directory.GetDirectories(FolderName, "*.*", SearchOption.TopDirectoryOnly).Length == 0)
                        {
                            try
                            {
                                Directory.Delete(FolderName, false);
 
                                int i = FolderName.LastIndexOf(Path.DirectorySeparatorChar);
                                if (i < 0)
                                    break;
                                else
                                    FolderName = FolderName[..i];
                            }
                            catch (Exception ex)
                            {
                                Log.Error("Unable to delete folder.",
                                    new KeyValuePair<string, object>("Folder", FolderName),
                                    new KeyValuePair<string, object>("Message", ex.Message));
                                break;
                            }
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                Log.Exception(ex);
            }
        }
 
        public event EventHandlerAsync<BlockReferenceEventArgs> BlockDeleted = null;
 
        private async Task DeleteBlockReference(BlockReference Block)
        {
            try
            {
                await Database.Delete(Block);
                await this.BlockDeleted.Raise(this, new BlockReferenceEventArgs(Block));
            }
            catch (Exception ex)
            {
                Log.Exception(ex);
            }
        }
 
        #region IDisposable
 
        public void Dispose()
        {
            if (!this.disposed)
            {
                this.disposed = true;
                this.eventQueue.Dispose();
            }
        }
 
        #endregion
 
        #region ISerializerContext
 
        public string Id => this.id;
 
        public string DefaultCollectionName => this.defaultCollectionName;
 
        public int TimeoutMilliseconds
        {
            get
            {
                throw new NotSupportedException("Objects must be embedded. They cannot be referenced and separately stored.");
            }
        }
 
        public bool Debug => this.debug;
 
        public bool NormalizedNames => false;
 
        public string ExternalIdentity => this.externalIdentity;
 
        public Task<ulong> GetFieldCode(string Collection, string FieldName)
        {
            throw new NotSupportedException("Field codes not used.");
        }
 
        public Task<string> GetFieldName(string Collection, ulong FieldCode)
        {
            throw new NotSupportedException("Field codes not used.");
        }
 
        public Task<IObjectSerializer> GetObjectSerializer(Type Type)
        {
            if (this.serializers is null)
                throw new ObjectDisposedException("Service is closing down.");
 
            return this.serializers.GetObjectSerializer(Type);
        }
 
        public Task<IObjectSerializer> GetObjectSerializerNoCreate(Type Type)
        {
            if (this.serializers is null)
                throw new ObjectDisposedException("Service is closing down.");
 
            return this.serializers.GetObjectSerializerNoCreate(Type);
        }
 
        public Task<ObjectSerializer> GetObjectSerializerEx(object Object)
        {
            return this.GetObjectSerializerEx(Object.GetType());
        }
 
        public async Task<ObjectSerializer> GetObjectSerializerEx(Type Type)
        {
            if (!(await this.GetObjectSerializer(Type) is ObjectSerializer Serializer))
                throw new Exception("Objects of type " + Type.FullName + " must be embedded.");
 
            return Serializer;
        }
 
        public Guid CreateGuid()
        {
            return Guid.NewGuid();
        }
 
        public Task<Guid> SaveNewObject(object Value, object State)
        {
            throw new NotSupportedException("Objects must be embedded. They cannot be referenced and separately stored.");
        }
 
        public Task<T> TryLoadObject<T>(Guid ObjectId, EmbeddedObjectSetter EmbeddedSetter)
            where T : class
        {
            throw new NotSupportedException("Objects must be embedded. They cannot be referenced and separately stored.");
        }
 
        public Task<object> TryLoadObject(Type T, Guid ObjectId, EmbeddedObjectSetter EmbeddedSetter)
        {
            throw new NotSupportedException("Objects must be embedded. They cannot be referenced and separately stored.");
        }
 
        public Task<byte[]> Encrypt(byte[] Data, string Property, string Collection, Guid ObjectId,
            int MinLength)
        {
            return Task.FromResult(new byte[MinLength]);
        }
 
        public Task<byte[]> Decrypt(byte[] Data, string Property, string Collection, Guid ObjectId)
        {
            return Task.FromResult(Data);
        }
 
        #endregion
 
        public Task NewEntry(object Object)
        {
            this.Archive(EntryType.New, Object);
            return Task.CompletedTask;
        }
 
        public Task UpdatedEntry(object Object)
        {
            this.Archive(EntryType.Update, Object);
            return Task.CompletedTask;
 
            // TODO: Update strategies
        }
 
        public Task DeletedEntry(object Object)
        {
            this.Archive(EntryType.Delete, Object);
            return Task.CompletedTask;
 
            // TODO: Delete strategies
        }
 
        public Task ClearedCollection(string Collection)
        {
            // TODO: Archive clear event.
            // TODO: Clear strategies
 
            return Task.CompletedTask;
        }
 
        private void Archive(EntryType EntryType, object Object)
        {
            if (this.disposed)
                throw new ObjectDisposedException("The Neuro-Ledger has been disposed and does not accept more entries.");
 
            if (Object is null)
                return;
 
            if (!this.stopped)
            {
                // No assigned bucket and Object not null serializes object into bucket
                // corresponding to object's collection and archiving time.
 
                this.eventQueue.QueueLast(new WorkItem()
                {
                    Bucket = null,
                    Type = EntryType,
                    Object = Object
                });
            }
        }
 
        public async Task<ILedgerEnumerator<T>> GetEnumerator<T>()
        {
            ObjectSerializer Serializer = await this.GetObjectSerializerEx(typeof(T));
            string CollectionName = await Serializer.CollectionName(null);
            PaginatedEnumerator<BlockReference> BlockEnumerator = await this.GetCollectionBlockEnumerator(CollectionName, true);
            return await ObjectEnumerator<T>.Create(BlockEnumerator, this);
        }
 
        public async Task<ILedgerEnumerator<object>> GetEnumerator(string CollectionName)
        {
            PaginatedEnumerator<BlockReference> BlockEnumerator = await this.GetCollectionBlockEnumerator(CollectionName, true);
            return await ObjectEnumerator<object>.Create(BlockEnumerator, this);
        }
 
        public async Task<PaginatedEnumerator<BlockReference>> GetBlockEnumerator(bool Ascending)
        {
            return await Database.Enumerate<BlockReference>(
                BlockPageSize, Ascending ? "Created" : "-Created");
        }
 
        public async Task<PaginatedEnumerator<BlockReference>> GetCollectionBlockEnumerator(string Collection, bool Ascending)
        {
            return await Database.Enumerate<BlockReference>(
                BlockPageSize, new FilterFieldEqualTo("Collection", Collection),
                Ascending ? "Created" : "-Created");
        }
 
        public async Task<PaginatedEnumerator<BlockReference>> GetCreatorBlockEnumerator(string Creator, bool Ascending)
        {
            return await Database.Enumerate<BlockReference>(
                BlockPageSize, new FilterAnd(
                new FilterFieldEqualTo("Creator", Creator),
                new FilterFieldEqualTo("AccessDenied", false)),
                Ascending ? "Created" : "-Created");
        }
 
        public async Task Start()
        {
            if (this.disposed)
                throw new ObjectDisposedException("The Neuro-Ledger has been disposed and does not accept more entries.");
 
            this.stopped = false;
            this.completed = new TaskCompletionSource<bool>();
 
            string[] Files = Directory.GetFiles(this.bucketFolder, "*.bin", SearchOption.TopDirectoryOnly);
            foreach (string FileName in Files)
            {
                try
                {
                    string BucketName = Path.GetFileName(FileName);
                    BucketName = BucketName[..^4];
 
                    int i = BucketName.LastIndexOf('.');
                    if (i < 0)
                    {
                        File.Delete(FileName);
                        continue;
                    }
 
                    string Suffix = BucketName[(i + 1)..];
                    string CollectionName = BucketName[..i];
                    int MaxDays = (int)(DateTime.MaxValue - DateTime.UtcNow).TotalDays;
                    DateTime Expires;
 
                    if (int.TryParse(Suffix, out int NrDays) && NrDays < MaxDays)
                        Expires = DateTime.UtcNow.AddDays(NrDays);
                    else
                        Expires = DateTime.MaxValue;
 
                    Bucket Bucket = await Bucket.Create(FileName, CollectionName, Expires, this);
 
                    if (Bucket.HasEntries)
                        this.buckets.Add(BucketName, Bucket);
                    else
                        await Bucket.Delete();
                }
                catch (Exception ex)
                {
                    Log.Exception(ex);
                }
            }
 
            Task _ = Task.Run(() => this.StorageTask());
        }
 
        public async Task Stop()
        {
            if (this.disposed)
                throw new ObjectDisposedException("The Neuro-Ledger has been disposed and does not accept more entries.");
 
            this.stopped = true;
 
            this.eventQueue.Disposed += async (Sender, e) =>
            {
                this.buckets?.Clear();
                this.buckets?.Dispose();
                this.buckets = null;
 
                if (!(this.serializers is null))
                {
                    await this.serializers.DisposeAsync();
                    this.serializers = null;
                }
 
                this.aes?.Dispose();
                this.aes = null;
            };
 
            await this.eventQueue.Terminate();
            await this.completed.Task;
        }
 
        public Task Flush()
        {
            if (this.disposed)
                throw new ObjectDisposedException("The Neuro-Ledger has been disposed and does not accept more entries.");
 
            this.buckets?.Clear();
            return Task.CompletedTask;
        }
 
        internal ICryptoTransform GetAes(string FileName, bool Encryptor)
        {
            byte[] BinFileName = Encoding.UTF8.GetBytes(FileName);
            int c = BinFileName.Length;
            byte[] Concat = new byte[c + this.saltLength];
 
            Buffer.BlockCopy(BinFileName, 0, Concat, 0, c);
            Buffer.BlockCopy(this.salt, 0, Concat, c, this.saltLength);
 
            byte[] Key = Hashes.ComputeSHA256Hash(Concat);
            byte[] IV = Hashes.ComputeHMACSHA256Hash(Key, BinFileName);
 
            Array.Resize(ref IV, 16);
 
            if (Encryptor)
                return this.aes.CreateEncryptor(Key, IV);
            else
                return this.aes.CreateDecryptor(Key, IV);
        }
 
        public static Task<BlockReference> FindReference(byte[] Digest)
        {
            return Database.FindFirstIgnoreRest<BlockReference>(new FilterFieldEqualTo("Digest", Digest));
        }
 
        public async Task RepairRegistry()
        {
            int NrAdded = 0;
            int NrUpdated = 0;
            int NrDeleted = 0;
 
            Log.Warning("Block Registry collection repaired during start-up. Scanning existing blocks in ledger to make sure block registry is up to date.");
 
            await this.RepairRegistry(
                null,
                (Sender, e) =>
                {
                    NrAdded++;
                    return Task.CompletedTask;
                },
                (Sender, e) =>
                {
                    NrUpdated++;
                    return Task.CompletedTask;
                },
                (Sender, e) =>
                {
                    NrDeleted++;
                    return Task.CompletedTask;
                });
 
            if (NrAdded == 0 && NrUpdated == 0 && NrDeleted == 0)
            {
                Log.Notice("Block Registry OK. Nothing to repair.",
                    new KeyValuePair<string, object>("NrAdded", NrAdded),
                    new KeyValuePair<string, object>("NrUpdated", NrUpdated),
                    new KeyValuePair<string, object>("NrDeleted", NrDeleted));
            }
            else
            {
                Log.Warning("Block Registry repaired, based on blocks available in ledger.",
                    new KeyValuePair<string, object>("NrAdded", NrAdded),
                    new KeyValuePair<string, object>("NrUpdated", NrUpdated),
                    new KeyValuePair<string, object>("NrDeleted", NrDeleted));
            }
        }
 
        public async Task RepairRegistry(
            EventHandlerAsync<BlockReferenceEventArgs> NoChangeCallback,
            EventHandlerAsync<BlockReferenceEventArgs> AddedCallback,
            EventHandlerAsync<BlockReferenceEventArgs> UpdatedCallback,
            EventHandlerAsync<BlockReferenceEventArgs> DeletedCallback)
        {
            // First, scan blocks and recreate missing block reference objects.
 
            IEnumerable<string> Files = Directory.EnumerateFiles(this.blockFolder, "*.block", SearchOption.AllDirectories);
 
            foreach (string BlockFile in Files)
            {
                try
                {
                    BlockReader Reader = await BlockReader.CreateAsync(BlockFile, this);
                    string LocalFileName;
 
                    if (BlockFile.StartsWith(this.blockFolder))
                        LocalFileName = BlockFile[this.blockFolder.Length..];
                    else
                        LocalFileName = BlockFile;
 
                    string s = Path.GetFileName(LocalFileName);
                    s = s[..^6];
                    byte[] Digest = Hashes.StringToBinary(s);
                    if (Digest is null)
                        continue;
 
                    BlockReference Ref = await FindReference(Digest);
                    int i;
 
                    if (Ref is null)
                    {
                        Ref = new BlockReference(Reader.Header, LocalFileName, Digest, Reader.Signature, Reader.Bytes);
                        await Database.Provider.Insert(Ref);
 
                        await AddedCallback.Raise(this, new BlockReferenceEventArgs(Ref));
                    }
                    else if ((i = this.Compare(Ref, Reader, LocalFileName, Digest)) != 0)
                    {
                        Ref.Creator = Reader.Header.Creator;
                        Ref.Created = Reader.Header.Created;
                        Ref.Updated = Reader.Header.Updated;
                        Ref.Expires = Reader.Header.Expires;
                        Ref.Status = Reader.Header.Status;
                        Ref.Link = Reader.Header.Link;
                        Ref.FileName = LocalFileName;
                        Ref.Collection = Reader.CollectionName;
                        Ref.Bytes = Reader.Bytes;
                        Ref.Digest = (byte[])Digest.Clone();
                        Ref.Signature = (byte[])Reader.Signature.Clone();
 
                        await Database.Provider.Update(Ref);
 
                        await UpdatedCallback.Raise(this, new BlockReferenceEventArgs(Ref));
                    }
                    else
                        await NoChangeCallback.Raise(this, new BlockReferenceEventArgs(Ref));
                }
                catch (Exception ex)
                {
                    Log.Error("Unable to process block file.",
                        new KeyValuePair<string, object>("FileName", BlockFile),
                        new KeyValuePair<string, object>("Message", ex.Message));
                }
            }
 
            // Second, scan block reference objects and remove obsolete reference objects.
 
            string LastObjectId = null;
            const int Max = 1000;
            int Count;
 
            do
            {
                IEnumerable<BlockReference> References;
 
                if (LastObjectId is null)
                    References = await Database.Find<BlockReference>(0, Max, "ObjectId");
                else
                    References = await Database.Find<BlockReference>(0, Max, new FilterFieldGreaterThan("ObjectId", LastObjectId), "ObjectId");
 
                Count = 0;
                foreach (BlockReference Ref in References)
                {
                    Count++;
 
                    if (string.IsNullOrEmpty(Ref.FileName))
                    {
                        if (Ref.Creator == this.externalIdentity)
                        {
                            await Database.Provider.Delete(Ref);
 
                            await DeletedCallback.Raise(this, new BlockReferenceEventArgs(Ref));
                        }
                        else
                            await NoChangeCallback.Raise(this, new BlockReferenceEventArgs(Ref));
                    }
                    else
                    {
                        string FileName = this.GetFullFileName(Ref.FileName);
 
                        if (!File.Exists(FileName))
                        {
                            if (Ref.Creator == this.externalIdentity)
                            {
                                await Database.Provider.Delete(Ref);
 
                                await DeletedCallback.Raise(this, new BlockReferenceEventArgs(Ref));
                            }
                            else
                            {
                                Ref.FileName = string.Empty;
                                await Database.Provider.Update(Ref);
                                await UpdatedCallback.Raise(this, new BlockReferenceEventArgs(Ref));
                            }
                        }
                    }
 
                    LastObjectId = Ref.ObjectId;
                }
            }
            while (Count >= Max);
        }
 
        private int Compare(BlockReference Ref, BlockReader Reader, string LocalFileName, byte[] Digest)
        {
            if (Ref.Creator != Reader.Header.Creator)
                return 1;
 
            if (!AreEqual(Ref.Created, Reader.Header.Created))
                return 2;
 
            if (!AreEqual(Ref.Updated, Reader.Header.Updated))
                return 3;
 
            if (!AreEqual(Ref.Expires, Reader.Header.Expires))
                return 4;
 
            if (Ref.Status != Reader.Header.Status)
                return 5;
 
            if (Ref.Link != Reader.Header.Link)
                return 6;
 
            if (Ref.FileName != LocalFileName)
                return 7;
 
            if (Ref.Collection != Reader.CollectionName)
                return 8;
 
            if (Ref.Bytes != Reader.Bytes)
                return 9;
 
            if (Convert.ToBase64String(Ref.Digest) != Convert.ToBase64String(Digest))
                return 10;
 
            if (Convert.ToBase64String(Ref.Signature) != Convert.ToBase64String(Reader.Signature))
                return 11;
 
            return 0;
        }
 
        public static bool AreEqual(DateTime TP1, DateTime TP2)
        {
            return
                TP1.Millisecond == TP2.Millisecond &&
                TP1.Second == TP2.Second &&
                TP1.Minute == TP2.Minute &&
                TP1.Hour == TP2.Hour &&
                TP1.Day == TP2.Day &&
                TP1.Month == TP2.Month &&
                TP1.Year == TP2.Year &&
                TP1.Kind == TP2.Kind;
        }
 
        public async Task RepairCollection(string CollectionName)
        {
            PaginatedEnumerator<BlockReference> Blocks = await this.GetCollectionBlockEnumerator(CollectionName, false);
            CachedStringDictionary Dictionary = null;
            List<ObjectState> ObjectsInBlock = new List<ObjectState>();
            DateTime Start = DateTime.Now;
            int Count = 0;
            bool Cleared = false;
            uint NrAdded = 0;
            uint NrUpdated = 0;
            uint NrDeleted = 0;
            uint NrErrors = 0;
 
            try
            {
                bool Started = true;
 
                await Database.StartBulk();
                try
                {
                    while (await Blocks.MoveNextAsync())
                    {
                        BlockReference Ref = Blocks.Current;
                        if (Ref.Status != BlockStatus.Valid || Ref.AccessDenied)
                            continue;
 
                        if (Dictionary is null)
                        {
                            IPersistentDictionary PersistentDictionary = await Database.GetDictionary(CollectionName);
                            await PersistentDictionary.ClearAsync();
 
                            Dictionary = new CachedStringDictionary(100000, PersistentDictionary);
 
                            Log.Warning(CollectionName + " collection repaired during start-up. Scanning existing blocks in ledger to make sure collection is up to date.");
                        }
 
                        using (BlockEnumerator TempBlockEnumerator = new BlockEnumerator(Ref, this))
                        {
                            using (ObjectEnumerator<GenericObject> e = await ObjectEnumerator<GenericObject>.Create(TempBlockEnumerator, this))
                            {
                                while (await e.MoveNextAsync())
                                    ObjectsInBlock.Add(new ObjectState(e.CurrentEntry.Type, e.Current));
                            }
 
                            ObjectsInBlock.Reverse();
 
                            Exception FirstException = null;
 
                            foreach (ObjectState ObjectState in ObjectsInBlock)
                            {
                                if (ObjectState.Type == EntryType.Clear)
                                {
                                    Cleared = true;
                                    break;
                                }
 
                                try
                                {
                                    string Key = ObjectState.Object.ObjectId.ToString();
 
                                    if (await Dictionary.ContainsKeyAsync(Key))
                                        continue;   // Only latest is of importance.
 
                                    await Dictionary.AddAsync(Key, ObjectState);
                                }
                                catch (Exception ex)
                                {
                                    FirstException ??= ex;
                                    break;
                                }
 
                                if (++Count >= 100)
                                {
                                    await Database.EndBulk();
                                    Started = false;
 
                                    await Database.StartBulk();
                                    Started = true;
                                    Count = 0;
                                }
                            }
 
                            if (!(FirstException is null))
                            {
                                Log.Error("Unable to enumerate objects in block properly when repairing collection:\r\n\r\n" +
                                    FirstException.Message, TempBlockEnumerator.Current.FileName, string.Empty, string.Empty,
                                    EventLevel.Major, string.Empty, string.Empty, Log.CleanStackTrace(FirstException.StackTrace),
                                    new KeyValuePair<string, object>("Collection", CollectionName));
                            }
 
                            ObjectsInBlock.Clear();
                        }
 
                        if (Cleared)
                            break;
                    }
 
                    if (Cleared)
                        await Database.Provider.Clear(CollectionName);
                }
                finally
                {
                    if (Started)
                        await Database.EndBulk();
                }
 
                if (!(Dictionary is null))
                {
                    Tuple<uint, uint, uint, uint> Counts = await this.Process(Dictionary, CollectionName);
 
                    NrAdded = Counts.Item1;
                    NrUpdated = Counts.Item2;
                    NrDeleted = Counts.Item3;
                    NrErrors = Counts.Item4;
                }
            }
            finally
            {
                await Blocks.DisposeAsync();
 
                if (!(Dictionary is null))
                {
                    await Dictionary.ClearAsync();
                    Dictionary.DeleteAndDispose();
                }
            }
 
            TimeSpan Elapsed = DateTime.Now - Start;
 
            if (NrAdded == 0 && NrUpdated == 0 && NrDeleted == 0 && NrErrors == 0)
            {
                Log.Notice("Collection OK. Nothing to repair.",
                    new KeyValuePair<string, object>("CollectionName", CollectionName),
                    new KeyValuePair<string, object>("NrAdded", NrAdded),
                    new KeyValuePair<string, object>("NrUpdated", NrUpdated),
                    new KeyValuePair<string, object>("NrDeleted", NrDeleted),
                    new KeyValuePair<string, object>("NrErrors", NrErrors),
                    new KeyValuePair<string, object>("Time", Elapsed.ToString()));
            }
            else
            {
                Log.Alert("Collection repaired, based on blocks available in ledger.",
                    new KeyValuePair<string, object>("CollectionName", CollectionName),
                    new KeyValuePair<string, object>("NrAdded", NrAdded),
                    new KeyValuePair<string, object>("NrUpdated", NrUpdated),
                    new KeyValuePair<string, object>("NrDeleted", NrDeleted),
                    new KeyValuePair<string, object>("NrErrors", NrErrors),
                    new KeyValuePair<string, object>("Time", Elapsed.ToString()));
            }
        }
 
        public async Task<Tuple<uint, uint, uint, uint>> Process(CachedStringDictionary Records, string CollectionName)
        {
            uint NrAdded = 0;
            uint NrUpdated = 0;
            uint NrDeleted = 0;
            uint NrErrors = 0;
 
            await Database.StartBulk();
            try
            {
                IEnumerator<KeyValuePair<string, object>> e = await Records.GetEnumeratorAsync();
                try
                {
                    int Count = 0;
 
                    if (!(e is IAsyncEnumerator eAsync))
                        eAsync = new PseudoAsyncEnumerator(e);
 
                    while (await eAsync.MoveNextAsync())
                    {
                        if (!(e.Current.Value is ObjectState ObjectState))
                            continue;
 
                        try
                        {
                            switch (ObjectState.Type)
                            {
                                // Note: Use of Database.Provider avoids generating events that result in the creation of new blocks.
 
                                case EntryType.New:
                                case EntryType.Update:
                                    GenericObject Obj2 = await Database.Provider.TryLoadObject<GenericObject>(CollectionName, ObjectState.Object.ObjectId);
                                    if (Obj2 is null)
                                    {
                                        await Database.Provider.Insert(ObjectState.Object);
                                        NrAdded++;
                                    }
                                    else
                                    {
                                        if (Obj2.Equals(ObjectState.Object))
                                            continue;
 
                                        await Database.Provider.Update(ObjectState.Object);
                                        NrUpdated++;
                                    }
                                    break;
 
                                case EntryType.Delete:
                                    Obj2 = await Database.Provider.TryLoadObject<GenericObject>(ObjectState.Object.ObjectId);
                                    if (Obj2 is null)
                                        continue;
                                    else
                                    {
                                        await Database.Provider.Delete(Obj2);
                                        NrDeleted++;
                                    }
                                    break;
 
                                default:
                                    continue;
                            }
 
                            if (++Count >= 100)
                            {
                                await Database.EndBulk();
                                await Database.StartBulk();
                                Count = 0;
                            }
                        }
                        catch (Exception ex)
                        {
                            Log.Exception(ex);
                            NrErrors++;
                        }
                    }
                }
                finally
                {
                    if (e is IDisposableAsync DisposableAsync)
                        await DisposableAsync.DisposeAsync();
                    else if (e is IDisposable Disposable)
                        Disposable.Dispose();
                }
            }
            finally
            {
                await Database.EndBulk();
            }
 
            return new Tuple<uint, uint, uint, uint>(NrAdded, NrUpdated, NrDeleted, NrErrors);
        }
 
        private class PseudoAsyncEnumerator : IAsyncEnumerator
        {
            private readonly IEnumerator e;
 
            public PseudoAsyncEnumerator(IEnumerator e)
            {
                this.e = e;
            }
 
            public object Current => this.e.Current;
            public bool MoveNext() => this.e.MoveNext();
            public Task<bool> MoveNextAsync() => Task.FromResult(this.e.MoveNext());
            public void Reset() => this.e.Reset();
        }
 
        public string[] Collections
        {
            get
            {
                string[] Result;
 
                lock (this.collections)
                {
                    Result = new string[this.collections.Count];
                    this.collections.Keys.CopyTo(Result, 0);
                }
 
                return Result;
            }
        }
 
        public Task<string[]> GetCollections()
        {
            return Task.FromResult<string[]>(this.Collections);
        }
 
        public Task<bool> Export(ILedgerExport Output, LedgerExportRestriction Restriction)
        {
            return this.Export(Output, Restriction, null);
        }
 
        public async Task<bool> Export(ILedgerExport Output, LedgerExportRestriction Restriction, ProfilerThread Thread)
        {
            Thread?.Start();
 
            if (!await Output.StartLedger(this))
                return false;
 
            bool Continue;
 
            try
            {
                string[] Collections = Restriction?.CollectionNames ?? await Ledger.GetCollections();
 
                foreach (string Collection in Collections)
                {
                    Thread?.NewState(Collection);
                    if (!await Output.StartCollection(Collection))
                        return false;
                    try
                    {
                        string[] BlockIds = Restriction?.BlockIds ?? new string[] { null };
 
                        foreach (string BlockId in BlockIds)
                        {
                            string[] Creators = Restriction?.Creators ?? new string[] { null };
 
                            foreach (string Creator in Creators)
                            {
                                List<Filter> Filters = new List<Filter>()
                                {
                                    new FilterFieldEqualTo("Collection", Collection)
                                };
 
                                if (!string.IsNullOrEmpty(BlockId))
                                    Filters.Add(new FilterFieldEqualTo("ObjectId", BlockId));
 
                                if (!string.IsNullOrEmpty(Creator))
                                    Filters.Add(new FilterFieldEqualTo("Creator", Creator));
 
                                if (Restriction.MinCreated.HasValue)
                                {
                                    if (Restriction.MinCreatedIncluded)
                                        Filters.Add(new FilterFieldGreaterOrEqualTo("Created", Restriction.MinCreated.Value));
                                    else
                                        Filters.Add(new FilterFieldGreaterThan("Created", Restriction.MinCreated.Value));
                                }
 
                                if (Restriction.MaxCreated.HasValue)
                                {
                                    if (Restriction.MaxCreatedIncluded)
                                        Filters.Add(new FilterFieldLesserOrEqualTo("Created", Restriction.MaxCreated.Value));
                                    else
                                        Filters.Add(new FilterFieldLesserThan("Created", Restriction.MaxCreated.Value));
                                }
 
                                Filter Filter;
 
                                if (Filters.Count > 1)
                                    Filter = new FilterAnd(Filters.ToArray());
                                else
                                    Filter = Filters[0];
 
                                using PaginatedEnumerator<BlockReference> Blocks =
                                    await Database.Enumerate<BlockReference>(BlockPageSize, Filter, "Created");
 
                                while (await Blocks.MoveNextAsync())
                                {
                                    BlockReference Ref = Blocks.Current;
                                    if (Ref.Status != BlockStatus.Valid || Ref.AccessDenied)
                                        continue;
 
                                    await Output.StartBlock(Ref.ObjectId);
                                    try
                                    {
                                        if (!await Output.BlockMetaData("Bytes", Ref.Bytes))
                                            return false;
 
                                        if (!await Output.BlockMetaData("Created", Ref.Created))
                                            return false;
 
                                        if (!await Output.BlockMetaData("Creator", Ref.Creator))
                                            return false;
 
                                        if (!await Output.BlockMetaData("Digest", Ref.Digest))
                                            return false;
 
                                        if (!await Output.BlockMetaData("Expires", Ref.Expires))
                                            return false;
 
                                        if (!await Output.BlockMetaData("FileName", Ref.FileName))
                                            return false;
 
                                        if (!await Output.BlockMetaData("Signature", Ref.Signature))
                                            return false;
 
                                        using BlockEnumerator TempBlockEnumerator = new BlockEnumerator(Blocks.Current, this);
                                        using ObjectEnumerator<GenericObject> e = await ObjectEnumerator<GenericObject>.Create(TempBlockEnumerator, this);
                                        GenericObject Obj;
 
                                        while (await e.MoveNextAsync())
                                        {
                                            Obj = e.Current;
 
                                            if (!await Output.StartEntry(Obj.ObjectId.ToString(), Obj.TypeName, e.CurrentEntry.Type, e.CurrentEntry.Timestamp))
                                                return false;
                                            try
                                            {
                                                foreach (KeyValuePair<string, object> P in Obj)
                                                {
                                                    if (!await Output.ReportProperty(P.Key, P.Value))
                                                        return false;
                                                }
                                            }
                                            catch (Exception ex)
                                            {
                                                Thread?.Exception(ex);
                                                if (!await this.ReportException(ex, Output))
                                                    return false;
                                            }
                                            finally
                                            {
                                                Continue = await Output.EndEntry();
                                            }
 
                                            if (!Continue)
                                                return false;
                                        }
                                    }
                                    finally
                                    {
                                        Continue = await Output.EndBlock();
                                    }
 
                                    if (!Continue)
                                        return false;
                                }
                            }
                        }
                    }
                    catch (Exception ex)
                    {
                        Thread?.Exception(ex);
                        if (!await this.ReportException(ex, Output))
                            return false;
                    }
                    finally
                    {
                        Continue = await Output.EndCollection();
                    }
 
                    if (!Continue)
                        return false;
                }
            }
            catch (Exception ex)
            {
                Thread?.Exception(ex);
                if (!await this.ReportException(ex, Output))
                    return false;
            }
            finally
            {
                Continue = await Output.EndLedger();
 
                Thread?.Idle();
                Thread?.Stop();
            }
 
            return Continue;
        }
 
        private async Task<bool> ReportException(Exception ex, ILedgerExport Output)
        {
            ex = Log.UnnestException(ex);
 
            if (ex is AggregateException ex2)
            {
                foreach (Exception ex3 in ex2.InnerExceptions)
                {
                    if (!await Output.ReportException(ex3))
                        return false;
                }
 
                return true;
            }
            else
                return await Output.ReportException(ex);
        }
 
        public void Register(ILedgerExternalEvents ExternalEvents)
        {
            if (!(this.externalEvents is null) && this.externalEvents != ExternalEvents)
                throw new Exception("An interface for external events has already been registered.");
 
            this.externalEvents = ExternalEvents;
        }
 
        public void Unregister(ILedgerExternalEvents ExternalEvents)
        {
            if (!(this.externalEvents is null) && this.externalEvents != ExternalEvents)
                throw new Exception("The registered interface for external events differs from the one presented.");
 
            this.externalEvents = null;
        }
 
        public ILedgerExternalEvents ExternalEvents
        {
            get
            {
                neuroLedgerClientType ??= Types.GetType("Waher.Networking.XMPP.NeuroLedger.NeuroLedgerClient");
                Assert.CallFromSource(new ApproveType(neuroLedgerClientType));
                return this.externalEvents;
            }
        }
 
        private static Type neuroLedgerClientType = null;
 
    }
}