FullTextSearchModule.cs

using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using Waher.Events;
using Waher.Persistence.Attributes;
using Waher.Persistence.Filters;
using Waher.Persistence.FullTextSearch.Files;
using Waher.Persistence.FullTextSearch.Keywords;
using Waher.Persistence.FullTextSearch.Orders;
using Waher.Persistence.FullTextSearch.Tokenizers;
using Waher.Persistence.LifeCycle;
using Waher.Persistence.Serialization;
using Waher.Runtime.Cache;
using Waher.Runtime.Collections;
using Waher.Runtime.Inventory;
using Waher.Runtime.Threading;
using Waher.Script.Model;
 
namespace Waher.Persistence.FullTextSearch
{
    [ModuleDependency(typeof(DatabaseModule))]
    public class FullTextSearchModule : IModule
    {
        private static readonly MultiReadSingleWriteObject synchObj = new MultiReadSingleWriteObject(typeof(FullTextSearchModule), false);
        private static Cache<string, QueryRecord> queryCache;
        private static Dictionary<string, bool> stopWords = new Dictionary<string, bool>();
        private static IPersistentDictionary collectionInformation;
        private static Dictionary<string, CollectionInformation> collections;
        private static Dictionary<string, IPersistentDictionary> indices;
        private static Dictionary<Type, TypeInformation> types;
        private static FullTextSearchModule instance = null;
 
        public FullTextSearchModule()
        {
        }
 
        public async Task Start()
        {
            collectionInformation = await Database.GetDictionary("FullTextSearchCollections");
            collections = new Dictionary<string, CollectionInformation>();
            indices = new Dictionary<string, IPersistentDictionary>();
            types = new Dictionary<Type, TypeInformation>();
            queryCache = new Cache<string, QueryRecord>(int.MaxValue, TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1));
 
            instance = this;
 
            Database.ObjectInserted += this.Database_ObjectInserted;
            Database.ObjectUpdated += this.Database_ObjectUpdated;
            Database.ObjectDeleted += this.Database_ObjectDeleted;
            Database.CollectionCleared += this.Database_CollectionCleared;
 
            Types.OnInvalidated += this.Types_OnInvalidated;
        }
 
        public async Task Stop()
        {
            Database.ObjectInserted -= this.Database_ObjectInserted;
            Database.ObjectUpdated -= this.Database_ObjectUpdated;
            Database.ObjectDeleted -= this.Database_ObjectDeleted;
            Database.CollectionCleared -= this.Database_CollectionCleared;
 
            Types.OnInvalidated -= this.Types_OnInvalidated;
 
            // TODO: Wait for current objects to be finished.
 
            await synchObj.BeginWrite();
            try
            {
                queryCache?.Dispose();
                queryCache = null;
 
                if (!(indices is null))
                {
                    foreach (IPersistentDictionary Index in indices.Values)
                        Index.Dispose();
 
                    indices.Clear();
                    indices = null;
                }
 
                collectionInformation?.Dispose();
                collectionInformation = null;
 
                collections?.Clear();
                collections = null;
 
                types?.Clear();
                types = null;
 
            }
            finally
            {
                await synchObj.EndWrite();
                instance = null;
            }
        }
 
        private void Database_ObjectInserted(object Sender, ObjectEventArgs e)
        {
            Task.Run(() => this.ObjectInserted(e));
        }
 
        private async Task ObjectInserted(ObjectEventArgs e)
        {
            try
            {
                Tuple<CollectionInformation, TypeInformation, GenericObject> P = await Prepare(e.Object);
                if (P is null)
                    return;
 
                object ObjectId = await Database.TryGetObjectId(e.Object);
                if (ObjectId is null)
                    return;
 
                CollectionInformation CollectionInfo = P.Item1;
                TypeInformation TypeInfo = P.Item2;
                GenericObject GenObj = P.Item3;
                TokenCount[] Tokens;
                string IndexName;
 
                if (GenObj is null)
                {
                    IndexName = TypeInfo.GetIndexCollection(e.Object);
                    Tokens = await TypeInfo.Tokenize(e.Object, CollectionInfo.Properties);
                }
                else
                {
                    IndexName = CollectionInfo.IndexCollectionName;
                    Tokens = await Tokenize(GenObj, CollectionInfo.Properties);
                }
 
                if (Tokens is null || Tokens.Length == 0)
                    return;
 
                ObjectReference Ref;
 
                await synchObj.BeginWrite();
                try
                {
                    ulong Index = await GetNextIndexNrLocked(IndexName);
 
                    Ref = new ObjectReference()
                    {
                        IndexCollection = IndexName,
                        Collection = CollectionInfo.CollectionName,
                        ObjectInstanceId = ObjectId,
                        Index = Index,
                        Tokens = Tokens,
                        Indexed = DateTime.UtcNow
                    };
 
                    await AddTokensToIndexLocked(Ref);
                    await Database.Insert(Ref);
                }
                finally
                {
                    await synchObj.EndWrite();
                }
 
                queryCache?.Clear();
 
                await Search.RaiseObjectAddedToIndex(this, new ObjectReferenceEventArgs(Ref));
            }
            catch (Exception ex)
            {
                Log.Exception(ex);
            }
        }
 
        private static async Task<IPersistentDictionary> GetIndexLocked(string IndexCollection, bool CreateIfNotFound)
        {
            if (indices.TryGetValue(IndexCollection, out IPersistentDictionary Result))
                return Result;
 
            if (CreateIfNotFound)
            {
                Result = await Database.GetDictionary(IndexCollection);
                indices[IndexCollection] = Result;
            }
 
            return Result;
        }
 
        private static async Task AddTokensToIndexLocked(ObjectReference Ref)
        {
            DateTime TP = DateTime.UtcNow;
            IPersistentDictionary Index = await GetIndexLocked(Ref.IndexCollection, true);
 
            foreach (TokenCount Token in Ref.Tokens)
            {
                KeyValuePair<bool, object> P = await Index.TryGetValueAsync(Token.Token);
                int c;
 
                if (!P.Key || !(P.Value is TokenReferences References))
                {
                    References = new TokenReferences()
                    {
                        LastBlock = 0,
                        ObjectReferences = new ulong[] { Ref.Index },
                        Counts = new uint[] { (uint)Token.DocIndex.Length },
                        Timestamps = new DateTime[] { TP }
                    };
 
                    await Index.AddAsync(Token.Token, References, true);
                }
                else if ((c = References.ObjectReferences.Length) < TokenReferences.MaxReferences)
                {
                    ulong[] NewReferences = new ulong[c + 1];
                    uint[] NewCounts = new uint[c + 1];
                    DateTime[] NewTimestamps = new DateTime[c + 1];
 
                    Array.Copy(References.ObjectReferences, 0, NewReferences, 0, c);
                    Array.Copy(References.Counts, 0, NewCounts, 0, c);
                    Array.Copy(References.Timestamps, 0, NewTimestamps, 0, c);
 
                    NewReferences[c] = Ref.Index;
                    NewCounts[c] = (uint)Token.DocIndex.Length;
                    NewTimestamps[c] = TP;
 
                    References.ObjectReferences = NewReferences;
                    References.Counts = NewCounts;
                    References.Timestamps = NewTimestamps;
 
                    await Index.AddAsync(Token.Token, References, true);
                }
                else
                {
                    References.LastBlock++;
 
                    TokenReferences NewBlock = new TokenReferences()
                    {
                        LastBlock = 0,
                        Counts = References.Counts,
                        ObjectReferences = References.ObjectReferences,
                        Timestamps = References.Timestamps
                    };
 
                    await Index.AddAsync(Token.Token + " " + References.LastBlock.ToString(), NewBlock, true);
 
                    References.ObjectReferences = new ulong[] { Ref.Index };
                    References.Counts = new uint[] { (uint)Token.DocIndex.Length };
                    References.Timestamps = new DateTime[] { TP };
 
                    await Index.AddAsync(Token.Token, References, true);
                }
 
                Token.Block = References.LastBlock + 1;
            }
        }
 
        private static async Task<ulong> GetNextIndexNrLocked(string IndexedCollection)
        {
            if (collectionInformation is null)
                throw new ObjectDisposedException(nameof(FullTextSearchModule));
 
            string Key = " C(" + IndexedCollection + ")";
            KeyValuePair<bool, object> P = await collectionInformation.TryGetValueAsync(Key);
 
            if (!P.Key || !(P.Value is ulong Nr))
                Nr = 0;
 
            Nr++;
 
            await collectionInformation.AddAsync(Key, Nr, true);
 
            return Nr;
        }
 
        private static Task<CollectionInformation> GetCollectionInfoLocked(string CollectionName, bool CreateIfNotExists)
        {
            return GetCollectionInfoLocked(CollectionName, CollectionName, CreateIfNotExists);
        }
 
        private static async Task<CollectionInformation> GetCollectionInfoLocked(
            string IndexCollectionName, string CollectionName, bool CreateIfNotExists)
        {
            if (collections is null)
            {
                if (CreateIfNotExists)
                    throw new NotSupportedException("Service not initialized or shut down.");
                else
                    return null;
            }
 
            if (collections.TryGetValue(CollectionName, out CollectionInformation Result))
                return Result;
 
            KeyValuePair<bool, object> P = await collectionInformation.TryGetValueAsync(CollectionName);
            if (P.Key && P.Value is CollectionInformation Result2)
            {
                collections[CollectionName] = Result2;
                return Result2;
            }
 
            if (!CreateIfNotExists)
                return null;
 
            Result = new CollectionInformation(IndexCollectionName, CollectionName, false);
            collections[CollectionName] = Result;
            await collectionInformation.AddAsync(CollectionName, Result, true);
 
            return Result;
        }
 
        public static async Task<Dictionary<string, string[]>> GetCollectionNames()
        {
            Dictionary<string, ChunkedList<string>> ByIndex = new Dictionary<string, ChunkedList<string>>();
 
            await synchObj.BeginRead();
            try
            {
                object[] Values = await collectionInformation.GetValuesAsync();
                    
                foreach (object Obj in Values)
                {
                    if (Obj is CollectionInformation Info && 
                        Info.IndexForFullTextSearch &&
                        !string.IsNullOrEmpty(Info.IndexCollectionName))
                    {
                        if (!ByIndex.TryGetValue(Info.IndexCollectionName, out ChunkedList<string> Collections))
                        {
                            Collections = new ChunkedList<string>();
                            ByIndex[Info.IndexCollectionName] = Collections;
                        }
 
                        Collections.Add(Info.CollectionName);
                    }
                }
            }
            finally
            {
                await synchObj.EndRead();
            }
 
            Dictionary<string, string[]> Result = new Dictionary<string, string[]>();
 
            foreach (KeyValuePair<string, ChunkedList<string>> Rec in ByIndex)
                Result[Rec.Key] = Rec.Value.ToArray();
 
            return Result;
        }
 
        public static async Task<string[]> GetCollectionNames(string IndexCollectionName)
        {
            await synchObj.BeginRead();
            try
            {
                return await GetCollectionNamesLocked(IndexCollectionName);
            }
            finally
            {
                await synchObj.EndRead();
            }
        }
 
        private static async Task<string[]> GetCollectionNamesLocked(string IndexCollectionName)
        {
            ChunkedList<string> Result = new ChunkedList<string>();
 
            foreach (object Obj in await collectionInformation.GetValuesAsync())
            {
                if (Obj is CollectionInformation Info && Info.IndexForFullTextSearch)
                {
                    if (Info.IndexCollectionName == IndexCollectionName)
                        Result.Add(Info.CollectionName);
                }
            }
 
            return Result.ToArray();
        }
 
        internal static async Task<bool> SetFullTextSearchIndexCollection(string IndexCollection, string CollectionName)
        {
            await synchObj.BeginWrite();
            try
            {
                CollectionInformation Info = await GetCollectionInfoLocked(IndexCollection, CollectionName, false);
                bool Created;
 
                if (Info is null)
                {
                    Created = true;
                    Info = await GetCollectionInfoLocked(IndexCollection, CollectionName, true);
                }
                else
                    Created = false;
 
                if (Info.IndexCollectionName != IndexCollection)
                {
                    Info.IndexCollectionName = IndexCollection;
                    await collectionInformation.AddAsync(Info.CollectionName, Info, true);
 
                    return true;
                }
                else
                    return Created;
            }
            finally
            {
                await synchObj.EndWrite();
            }
        }
 
        internal static async Task<bool> AddFullTextSearch(string CollectionName, params PropertyDefinition[] Properties)
        {
            await synchObj.BeginWrite();
            try
            {
                CollectionInformation Info = await GetCollectionInfoLocked(CollectionName, true);
 
                if (Info.AddIndexableProperties(Properties))
                {
                    await collectionInformation.AddAsync(Info.CollectionName, Info, true);
                    return true;
                }
                else
                    return false;
            }
            finally
            {
                await synchObj.EndWrite();
            }
        }
 
        internal static async Task<bool> RemoveFullTextSearch(string CollectionName, params PropertyDefinition[] Properties)
        {
            await synchObj.BeginWrite();
            try
            {
                CollectionInformation Info = await GetCollectionInfoLocked(CollectionName, true);
 
                if (Info.RemoveIndexableProperties(Properties))
                {
                    await collectionInformation.AddAsync(Info.CollectionName, Info, true);
                    return true;
                }
                else
                    return false;
            }
            finally
            {
                await synchObj.EndWrite();
            }
        }
 
        internal static async Task<Dictionary<string, PropertyDefinition[]>> GetFullTextSearchIndexedProperties()
        {
            Dictionary<string, PropertyDefinition[]> Result = new Dictionary<string, PropertyDefinition[]>();
 
            await synchObj.BeginRead();
            try
            {
                foreach (object Obj in await collectionInformation.GetValuesAsync())
                {
                    if (Obj is CollectionInformation Info && Info.IndexForFullTextSearch)
                        Result[Info.CollectionName] = Info.Properties;
                }
            }
            finally
            {
                await synchObj.EndRead();
            }
 
            return Result;
        }
 
        internal static async Task<PropertyDefinition[]> GetFullTextSearchIndexedProperties(string CollectionName)
        {
            await synchObj.BeginRead();
            try
            {
                CollectionInformation Info = await GetCollectionInfoLocked(CollectionName, false);
 
                if (Info is null || !Info.IndexForFullTextSearch)
                    return Array.Empty<PropertyDefinition>();
                else
                    return (PropertyDefinition[])Info.Properties.Clone();
            }
            finally
            {
                await synchObj.EndRead();
            }
        }
 
        private static async Task<Tuple<CollectionInformation, TypeInformation, GenericObject>> Prepare(object Object)
        {
            Object = await ScriptNode.WaitPossibleTask(Object);
 
            await synchObj.BeginWrite();
            try
            {
                if (Object is GenericObject GenObj)
                    return await PrepareLocked(GenObj);
                else
                    return await PrepareLocked(Object.GetType(), Object);
            }
            finally
            {
                await synchObj.EndWrite();
            }
        }
 
        private static async Task<Tuple<CollectionInformation, TypeInformation, GenericObject>> PrepareLocked(GenericObject GenObj)
        {
            CollectionInformation CollectionInfo = await GetCollectionInfoLocked(GenObj.CollectionName, true);
 
            if (CollectionInfo.IndexForFullTextSearch)
                return new Tuple<CollectionInformation, TypeInformation, GenericObject>(CollectionInfo, null, GenObj);
            else
                return null;
        }
 
        private static async Task<TypeInformation> GetTypeInfoLocked(Type T, object Instance)
        {
            if (types is null)
                throw new Exception("Full text search module not started, or in the process of being stopped.");
 
            if (types.TryGetValue(T, out TypeInformation Result))
                return Result;
 
            TypeInfo TI = T.GetTypeInfo();
            IEnumerable<FullTextSearchAttribute> SearchAttrs = TI.GetCustomAttributes<FullTextSearchAttribute>(true);
            CollectionNameAttribute CollectionAttr = TI.GetCustomAttribute<CollectionNameAttribute>(true);
            ITokenizer CustomTokenizer = Types.FindBest<ITokenizer, Type>(T);
 
            if (CollectionAttr is null)
                Result = new TypeInformation(T, TI, null, null, CustomTokenizer, null);
            else
            {
                string CollectionName = CollectionAttr.Name;
                bool DynamicIndex = false;
                string IndexName;
 
                if (!(SearchAttrs is null))
                {
                    foreach (FullTextSearchAttribute Attribute in SearchAttrs)
                    {
                        if (Attribute.DynamicIndexCollection)
                        {
                            DynamicIndex = true;
                            break;
                        }
                    }
                }
 
                if (DynamicIndex)
                    IndexName = null;
                else
                {
                    IndexName = CollectionName;
                    foreach (FullTextSearchAttribute Attribute in SearchAttrs)
                    {
                        IndexName = Attribute.GetIndexCollection(Instance);
                        break;
                    }
                }
 
                CollectionInformation Info = await GetCollectionInfoLocked(IndexName, CollectionName, true);
 
                Result = new TypeInformation(T, TI, CollectionName, Info, CustomTokenizer, SearchAttrs);
 
                if (Result.HasPropertyDefinitions && Info.AddIndexableProperties(Result.Properties))
                    await collectionInformation.AddAsync(CollectionName, Info, true);
                else if (!(CustomTokenizer is null) && !Info.IndexForFullTextSearch)
                {
                    Info.IndexForFullTextSearch = true;
                    await collectionInformation.AddAsync(CollectionName, Info, true);
                }
            }
 
            types[T] = Result;
 
            return Result;
        }
 
        private static async Task<Tuple<CollectionInformation, TypeInformation, GenericObject>> PrepareLocked(Type T, object Instance)
        {
            TypeInformation TypeInfo = await GetTypeInfoLocked(T, Instance);
            if (!TypeInfo.HasCollection)
                return null;
 
            if (!TypeInfo.CollectionInformation?.IndexForFullTextSearch ?? false)
                return null;
 
            return new Tuple<CollectionInformation, TypeInformation, GenericObject>(TypeInfo.CollectionInformation, TypeInfo, null);
        }
 
        internal static Keyword[] ParseKeywords(string Search, bool TreatKeywordsAsPrefixes)
        {
            return ParseKeywords(Search, TreatKeywordsAsPrefixes, true);
        }
 
        private static Keyword[] ParseKeywords(string Search, bool TreatKeywordsAsPrefixes,
            bool ParseQuotes)
        {
            ChunkedList<Keyword> Result = new ChunkedList<Keyword>();
            StringBuilder sb = new StringBuilder();
            bool First = true;
            bool Required = false;
            bool Prohibited = false;
            string Wildcard = null;
            int Type = 0;
            Keyword Keyword;
            string Token;
 
            foreach (char ch in Search.ToLower().Normalize(NormalizationForm.FormD))
            {
                UnicodeCategory Category = CharUnicodeInfo.GetUnicodeCategory(ch);
                if (Category == UnicodeCategory.NonSpacingMark)
                    continue;
 
                if (char.IsLetterOrDigit(ch))
                {
                    sb.Append(ch);
                    First = false;
                }
                else if (Type == 2)
                {
                    if (ch == '/')
                    {
                        Token = sb.ToString();
                        sb.Clear();
                        First = true;
                        Type = 0;
 
                        Add(new RegexKeyword(Token), Result, ref Required, ref Prohibited);
                    }
                    else
                    {
                        sb.Append(ch);
                        First = false;
                    }
                }
                else if (Type == 3)
                {
                    if (ch == '"')
                    {
                        Token = sb.ToString();
                        sb.Clear();
                        First = true;
                        Type = 0;
 
                        Add(new SequenceOfKeywords(ParseKeywords(Token, false)),
                            Result, ref Required, ref Prohibited);
                    }
                    else
                        sb.Append(ch);
                }
                else if (Type == 4)
                {
                    if (ch == '\'')
                    {
                        Token = sb.ToString();
                        sb.Clear();
                        First = true;
                        Type = 0;
 
                        Add(new SequenceOfKeywords(ParseKeywords(Token, false)),
                            Result, ref Required, ref Prohibited);
                    }
                    else
                        sb.Append(ch);
                }
                else if (Type == 0 && (ch == '*' || ch == '%' || ch == '¤' || ch == '#'))
                {
                    sb.Append(ch);
                    Type = 1;
                    Wildcard = new string(ch, 1);
                }
                else
                {
                    if (!First)
                    {
                        Token = sb.ToString();
                        sb.Clear();
                        First = true;
 
                        if (Type == 1)
                        {
                            Keyword = new WildcardKeyword(Token, Wildcard);
                            Wildcard = null;
                        }
                        else if (TreatKeywordsAsPrefixes)
                            Keyword = new WildcardKeyword(Token);
                        else
                            Keyword = new PlainKeyword(Token);
 
                        Add(Keyword, Result, ref Required, ref Prohibited);
                        Type = 0;
                    }
 
                    if (ch == '+')
                    {
                        Required = true;
                        Prohibited = false;
                    }
                    else if (ch == '-')
                    {
                        Prohibited = true;
                        Required = false;
                    }
                    else if (ch == '/')
                        Type = 2;
                    else if (ch == '"' && ParseQuotes)
                        Type = 3;
                    else if (ch == '\'' && ParseQuotes)
                        Type = 4;
                }
            }
 
            if (!First)
            {
                Token = sb.ToString();
                sb.Clear();
 
                switch (Type)
                {
                    case 0:
                    default:
                        if (TreatKeywordsAsPrefixes)
                            Keyword = new WildcardKeyword(Token);
                        else
                            Keyword = new PlainKeyword(Token);
                        break;
 
                    case 1:
                        Keyword = new WildcardKeyword(Token, Wildcard);
                        break;
 
                    case 2:
                        Keyword = new RegexKeyword(Token);
                        break;
                }
 
                Add(Keyword, Result, ref Required, ref Prohibited);
            }
 
            return Result.ToArray();
        }
 
        private static void Add(Keyword Keyword, ChunkedList<Keyword> Result, ref bool Required, ref bool Prohibited)
        {
            if (Required)
            {
                Keyword = new RequiredKeyword(Keyword);
                Required = false;
            }
 
            if (Prohibited)
            {
                Keyword = new ProhibitedKeyword(Keyword);
                Prohibited = false;
            }
 
            Result.Add(Keyword);
        }
 
        internal static async Task<T[]> FullTextSearch<T>(string IndexCollection,
            int Offset, int MaxCount, FullTextSearchOrder Order,
            PaginationStrategy PaginationStrategy, params Keyword[] Keywords)
            where T : class
        {
            if (MaxCount <= 0 || Keywords is null)
                return Array.Empty<T>();
 
            int NrKeywords = Keywords.Length;
            if (NrKeywords == 0)
                return Array.Empty<T>();
 
            Keywords = (Keyword[])Keywords.Clone();
            Array.Sort(Keywords, orderOfProcessing);
 
            StringBuilder sb = new StringBuilder();
 
            sb.Append(IndexCollection);
            sb.Append(' ');
            sb.Append(Order.ToString());
 
            foreach (Keyword Keyword in Keywords)
            {
                if (!Keyword.Ignore)
                {
                    sb.Append(' ');
                    sb.Append(Keyword.ToString());
                }
            }
 
            string Key = sb.ToString();
            MatchInformation[] FoundReferences;
            SearchProcess Process = null;
 
            if (queryCache.TryGetValue(Key, out QueryRecord QueryRecord))
            {
                FoundReferences = QueryRecord.FoundReferences;
                Process = QueryRecord.Process;
            }
            else
            {
                IPersistentDictionary Index;
 
                await synchObj.BeginRead();
                try
                {
                    Index = await GetIndexLocked(IndexCollection, false);
 
                    if (!(Index is null))
                    {
                        Process = new SearchProcess(Index, IndexCollection);
 
                        foreach (Keyword Keyword in Keywords)
                        {
                            if (Keyword.Ignore)
                                continue;
 
                            if (!await Keyword.Process(Process))
                                return Array.Empty<T>();
                        }
                    }
                }
                finally
                {
                    await synchObj.EndRead();
                }
 
                if (Index is null)
                {
                    await synchObj.BeginWrite();
                    try
                    {
                        Index = await GetIndexLocked(IndexCollection, true);
 
                        Process = new SearchProcess(Index, IndexCollection);
 
                        foreach (Keyword Keyword in Keywords)
                        {
                            if (Keyword.Ignore)
                                continue;
 
                            if (!await Keyword.Process(Process))
                                return Array.Empty<T>();
                        }
                    }
                    finally
                    {
                        await synchObj.EndWrite();
                    }
                }
 
                int c = Process.ReferencesByObject.Count;
 
                FoundReferences = new MatchInformation[c];
                Process.ReferencesByObject.Values.CopyTo(FoundReferences, 0);
 
                switch (Order)
                {
                    case FullTextSearchOrder.Relevance:
                    default:
                        Array.Sort(FoundReferences, relevanceOrder);
                        break;
 
                    case FullTextSearchOrder.Occurrences:
                        Array.Sort(FoundReferences, occurrencesOrder);
                        break;
 
                    case FullTextSearchOrder.Newest:
                        Array.Sort(FoundReferences, newestOrder);
                        break;
 
                    case FullTextSearchOrder.Oldest:
                        Array.Sort(FoundReferences, oldestOrder);
                        break;
                }
 
                queryCache[Key] = new QueryRecord()
                {
                    FoundReferences = FoundReferences,
                    Process = Process
                };
            }
 
            ChunkedList<T> Result = new ChunkedList<T>();
 
            switch (PaginationStrategy)
            {
                case PaginationStrategy.PaginateOverObjectsNullIfIncompatible:
                default:
                    foreach (MatchInformation ObjectReference in FoundReferences)
                    {
                        if (Offset > 0)
                        {
                            Offset--;
                            continue;
                        }
 
                        ulong RefIndex = ObjectReference.ObjectReference;
                        ObjectReference Ref = await Process.TryGetObjectReference(RefIndex, true);
                        if (Ref is null)
                            Result.Add(null);
                        else
                        {
                            T Object = await Database.TryLoadObject<T>(Ref.Collection, Ref.ObjectInstanceId);
                            if (Object is null)
                                Result.Add(null);
                            else
                                Result.Add(Object);
                        }
 
                        MaxCount--;
 
                        if (MaxCount <= 0)
                            break;
                    }
                    break;
 
                case PaginationStrategy.PaginateOverObjectsOnlyCompatible:
                    foreach (MatchInformation ObjectReference in FoundReferences)
                    {
                        if (Offset > 0)
                        {
                            Offset--;
                            continue;
                        }
 
                        ulong RefIndex = ObjectReference.ObjectReference;
                        ObjectReference Ref = await Process.TryGetObjectReference(RefIndex, true);
                        if (Ref is null)
                            continue;
 
                        T Object = await Database.TryLoadObject<T>(Ref.Collection, Ref.ObjectInstanceId);
                        if (Object is null)
                            continue;
 
                        Result.Add(Object);
                        MaxCount--;
 
                        if (MaxCount <= 0)
                            break;
                    }
                    break;
 
                case PaginationStrategy.PaginationOverCompatibleOnly:
                    foreach (MatchInformation ObjectReference in FoundReferences)
                    {
                        ulong RefIndex = ObjectReference.ObjectReference;
                        ObjectReference Ref = await Process.TryGetObjectReference(RefIndex, true);
 
                        if (Ref is null)
                            continue;
 
                        T Object = await Database.TryLoadObject<T>(Ref.Collection, Ref.ObjectInstanceId);
                        if (Object is null)
                            continue;
 
                        if (Offset > 0)
                        {
                            Offset--;
                            continue;
                        }
 
                        Result.Add(Object);
                        MaxCount--;
 
                        if (MaxCount <= 0)
                            break;
                    }
                    break;
            }
 
            return Result.ToArray();
        }
 
        private static readonly OrderOfProcessing orderOfProcessing = new OrderOfProcessing();
        private static readonly RelevanceOrder relevanceOrder = new RelevanceOrder();
        private static readonly OccurrencesOrder occurrencesOrder = new OccurrencesOrder();
        private static readonly NewestOrder newestOrder = new NewestOrder();
        private static readonly OldestOrder oldestOrder = new OldestOrder();
 
        private class QueryRecord
        {
            public MatchInformation[] FoundReferences;
            public SearchProcess Process;
        }
 
        private void Database_ObjectDeleted(object Sender, ObjectEventArgs e)
        {
            Task.Run(() => this.ObjectDeleted(e));
        }
 
        private async Task ObjectDeleted(ObjectEventArgs e)
        {
            try
            {
                Tuple<CollectionInformation, TypeInformation, GenericObject> P = await Prepare(e.Object);
                if (P is null)
                    return;
 
                object ObjectId = await Database.TryGetObjectId(e.Object);
                if (ObjectId is null)
                    return;
 
                ObjectReference Ref = await Database.FindFirstIgnoreRest<ObjectReference>(new FilterAnd(
                    new FilterFieldEqualTo("Collection", P.Item1.CollectionName),
                    new FilterFieldEqualTo("ObjectInstanceId", ObjectId)));
 
                if (Ref is null)
                    return;
 
                await synchObj.BeginWrite();
                try
                {
                    await RemoveTokensFromIndexLocked(Ref);
                }
                finally
                {
                    await synchObj.EndWrite();
                }
 
                queryCache?.Clear();
 
                await Search.RaiseObjectRemovedFromIndex(this, new ObjectReferenceEventArgs(Ref));
            }
            catch (Exception ex)
            {
                Log.Exception(ex);
            }
        }
 
        private static async Task RemoveTokensFromIndexLocked(ObjectReference Ref)
        {
            IPersistentDictionary Index = await GetIndexLocked(Ref.IndexCollection, true);
 
            foreach (TokenCount Token in Ref.Tokens)
            {
                string Suffix = " " + Token.Block.ToString();
                KeyValuePair<bool, object> P = await Index.TryGetValueAsync(Token.Token + Suffix);
 
                if (!P.Key)
                    P = await Index.TryGetValueAsync(Token.Token);
 
                if (!P.Key || !(P.Value is TokenReferences References))
                    continue;
 
                int i = Array.IndexOf(References.ObjectReferences, Ref.Index);
                if (i < 0)
                    continue;
 
                int c = References.ObjectReferences.Length;
                ulong[] NewReferences = new ulong[c - 1];
                uint[] NewCounts = new uint[c - 1];
                DateTime[] NewTimestamps = new DateTime[c - 1];
 
                if (i > 0)
                {
                    Array.Copy(References.ObjectReferences, 0, NewReferences, 0, i);
                    Array.Copy(References.Counts, 0, NewCounts, 0, i);
                    Array.Copy(References.Timestamps, 0, NewTimestamps, 0, i);
                }
 
                if (i < c - 1)
                {
                    Array.Copy(References.ObjectReferences, i + 1, NewReferences, i, c - i - 1);
                    Array.Copy(References.Counts, i + 1, NewCounts, i, c - i - 1);
                    Array.Copy(References.Timestamps, i + 1, NewTimestamps, i, c - i - 1);
                }
 
                References.ObjectReferences = NewReferences;
                References.Counts = NewCounts;
                References.Timestamps = NewTimestamps;
 
                await Index.AddAsync(Token.Token, References, true);
            }
        }
 
        private void Database_ObjectUpdated(object Sender, ObjectEventArgs e)
        {
            Task.Run(() => ProcessObjectUpdate(e.Object));
        }
 
        public static async Task ProcessObjectUpdate(object Object)
        {
            try
            {
                Tuple<CollectionInformation, TypeInformation, GenericObject> P = await Prepare(Object);
                if (P is null)
                    return;
 
                object ObjectId = await Database.TryGetObjectId(Object);
                if (ObjectId is null)
                    return;
 
                CollectionInformation CollectionInfo = P.Item1;
                TypeInformation TypeInfo = P.Item2;
                GenericObject GenObj = P.Item3;
                TokenCount[] Tokens;
 
                if (GenObj is null)
                    Tokens = await TypeInfo.Tokenize(Object, CollectionInfo.Properties);
                else
                    Tokens = await Tokenize(GenObj, CollectionInfo.Properties);
 
                ObjectReference Ref = await Database.FindFirstIgnoreRest<ObjectReference>(new FilterAnd(
                    new FilterFieldEqualTo("Collection", P.Item1.CollectionName),
                    new FilterFieldEqualTo("ObjectInstanceId", ObjectId)));
 
                if (AreSame(Tokens, Ref?.Tokens))
                    return;
 
                bool Added = false;
 
                await synchObj.BeginWrite();
                try
                {
                    if (Ref is null)
                    {
                        if (Tokens.Length == 0)
                            return;
 
                        string IndexName;
 
                        if (GenObj is null)
                            IndexName = TypeInfo.GetIndexCollection(Object);
                        else
                            IndexName = CollectionInfo.IndexCollectionName;
 
                        ulong Index = await GetNextIndexNrLocked(IndexName);
 
                        Ref = new ObjectReference()
                        {
                            IndexCollection = IndexName,
                            Collection = CollectionInfo.CollectionName,
                            ObjectInstanceId = ObjectId,
                            Index = Index,
                            Tokens = Tokens,
                            Indexed = DateTime.UtcNow
                        };
 
                        await AddTokensToIndexLocked(Ref);
                        await Database.Insert(Ref);
 
                        Added = true;
                    }
                    else
                    {
                        await RemoveTokensFromIndexLocked(Ref);
 
                        Ref.Tokens = Tokens;
                        await AddTokensToIndexLocked(Ref);
 
                        await Database.Update(Ref);
                    }
                }
                finally
                {
                    await synchObj.EndWrite();
                }
 
                queryCache.Clear();
 
                if (Added)
                    await Search.RaiseObjectAddedToIndex(instance, new ObjectReferenceEventArgs(Ref));
                else
                    await Search.RaiseObjectUpdatedInIndex(instance, new ObjectReferenceEventArgs(Ref));
            }
            catch (Exception ex)
            {
                Log.Exception(ex);
            }
        }
 
        private static bool AreSame(TokenCount[] Tokens1, TokenCount[] Tokens2)
        {
            int c = Tokens1?.Length ?? 0;
            int d = Tokens2?.Length ?? 0;
 
            if (c != d)
                return false;
 
            int i;
 
            for (i = 0; i < c; i++)
            {
                if (!Tokens1[i].Equals(Tokens2[i]))
                    return false;
            }
 
            return true;
        }
 
        private async Task Database_CollectionCleared(object Sender, CollectionEventArgs e)
        {
            try
            {
                IEnumerable<ObjectReference> ObjectsDeleted;
 
                do
                {
                    ObjectsDeleted = await Database.FindDelete<ObjectReference>(0, 1000,
                        new FilterFieldEqualTo("Collection", e.Collection));
 
                    foreach (ObjectReference Ref in ObjectsDeleted)
                    {
                        await synchObj.BeginWrite();
                        try
                        {
                            await RemoveTokensFromIndexLocked(Ref);
                        }
                        finally
                        {
                            await synchObj.EndWrite();
                        }
 
                        queryCache.Clear();
 
                        await Search.RaiseObjectRemovedFromIndex(this, new ObjectReferenceEventArgs(Ref));
                    }
                }
                while (!IsEmpty(ObjectsDeleted));
            }
            catch (Exception ex)
            {
                Log.Exception(ex);
            }
        }
 
        public static async Task<long> ReindexCollection(string IndexCollectionName)
        {
            IPersistentDictionary Index;
            string[] Collections;
 
            await synchObj.BeginWrite();
            try
            {
                Index = await GetIndexLocked(IndexCollectionName, true);
                await Index.ClearAsync();
 
                Collections = await GetCollectionNamesLocked(IndexCollectionName);
            }
            finally
            {
                await synchObj.EndWrite();
            }
 
            IEnumerable<ObjectReference> ObjectsDeleted;
 
            do
            {
                ObjectsDeleted = await Database.FindDelete<ObjectReference>(0, 1000,
                    new FilterFieldEqualTo("IndexCollection", IndexCollectionName));
 
                foreach (ObjectReference Ref in ObjectsDeleted)
                    await Search.RaiseObjectRemovedFromIndex(instance, new ObjectReferenceEventArgs(Ref));
            }
            while (!IsEmpty(ObjectsDeleted));
 
            ReindexCollectionIteration Iteration = new ReindexCollectionIteration();
 
            await Database.Iterate(Iteration, Collections);
 
            return Iteration.NrObjectsProcessed;
        }
 
        private static bool IsEmpty(IEnumerable<ObjectReference> Objects)
        {
            foreach (ObjectReference _ in Objects)
                return false;
 
            return true;
        }
 
        private class ReindexCollectionIteration : IDatabaseIteration<object>
        {
            public Task StartDatabase() => Task.CompletedTask;
            public Task EndDatabase() => Task.CompletedTask;
            public Task EndCollection() => Task.CompletedTask;
            public Task IncompatibleObject(object ObjectId) => Task.CompletedTask;
 
            public long NrObjectsProcessed = 0;
            public int NrCollectionsProcessed = 0;
 
            public Task StartCollection(string CollectionName)
            {
                this.NrCollectionsProcessed++;
                return Task.CompletedTask;
            }
 
            public async Task ProcessObject(object Object)
            {
                this.NrObjectsProcessed++;
                await instance.ObjectInserted(new ObjectEventArgs(Object));
            }
 
            public Task ReportException(Exception Exception)
            {
                Log.Exception(Exception);
                return Task.CompletedTask;
            }
        }
 
        internal static void RegisterStopWords(params string[] StopWords)
        {
            Dictionary<string, bool> NewList = new Dictionary<string, bool>();
 
            foreach (KeyValuePair<string, bool> P in stopWords)
                NewList[P.Key] = P.Value;
 
            foreach (string StopWord in StopWords)
                NewList[StopWord] = true;
 
            stopWords = NewList;
        }
 
        internal static bool IsStopWord(string StopWord)
        {
            return stopWords.TryGetValue(StopWord, out bool b) && b;
        }
 
        public static async Task<TokenCount[]> Tokenize(IEnumerable<object> Objects)
        {
            TokenizationProcess Process = new TokenizationProcess();
            await Tokenize(Objects, Process);
 
            return Process.ToArray();
        }
 
        public static async Task Tokenize(IEnumerable<object> Objects,
            TokenizationProcess Process)
        {
            foreach (object Object in Objects)
            {
                if (Object is null)
                    continue;
 
                object Object2 = await ScriptNode.WaitPossibleTask(Object);
 
                Type T = Object2.GetType();
                ITokenizer Tokenizer;
                bool Found;
 
                lock (tokenizers)
                {
                    Found = tokenizers.TryGetValue(T, out Tokenizer);
                }
 
                if (!Found)
                {
                    Tokenizer = Types.FindBest<ITokenizer, Type>(T);
 
                    lock (tokenizers)
                    {
                        tokenizers[T] = Tokenizer;
                    }
                }
 
                if (Tokenizer is null)
                {
                    Tuple<CollectionInformation, TypeInformation, GenericObject> P = await Prepare(Object2);
                    if (P is null)
                        continue;
 
                    object ObjectId = await Database.TryGetObjectId(Object2);
                    if (ObjectId is null)
                        return;
 
                    CollectionInformation CollectionInfo = P.Item1;
                    TypeInformation TypeInfo = P.Item2;
                    GenericObject GenObj = P.Item3;
 
                    if (GenObj is null)
                        await TypeInfo.Tokenize(Object2, Process, CollectionInfo.Properties);
                    else
                        await Tokenize(GenObj, Process, CollectionInfo.Properties);
                }
                else
                    await Tokenizer.Tokenize(Object2, Process);
 
                Process.DocumentIndexOffset++;
            }
        }
 
        private static readonly Dictionary<Type, ITokenizer> tokenizers = new Dictionary<Type, ITokenizer>();
 
        private void Types_OnInvalidated(object Sender, EventArgs e)
        {
            lock (tokenizers)
            {
                tokenizers.Clear();
            }
        }
 
        internal static async Task<TokenCount[]> Tokenize(GenericObject Obj, params PropertyDefinition[] Properties)
        {
            ChunkedList<object> Values = await GetValues(Obj, Properties);
 
            if (!Values.HasFirstItem)
                return null;
 
            return await Tokenize(Values);
        }
 
        internal static async Task Tokenize(GenericObject Obj, TokenizationProcess Process, params PropertyDefinition[] Properties)
        {
            ChunkedList<object> Values = await GetValues(Obj, Properties);
 
            if (Values.HasFirstItem)
                await Tokenize(Values, Process);
        }
 
        internal static async Task<ChunkedList<object>> GetValues(GenericObject Obj, params PropertyDefinition[] Properties)
        {
            ChunkedList<object> Values = new ChunkedList<object>();
            object Value;
 
            if (!(Obj is null))
            {
                foreach (PropertyDefinition Property in Properties)
                {
                    Value = await Property.GetValue(Obj);
                    if (!(Value is null))
                        Values.Add(Value);
                }
            }
 
            return Values;
        }
 
        internal static async Task<FolderIndexationStatistics> IndexFolder(string IndexCollection, string Folder, bool Recursive,
            params string[] ExcludeSubfolders)
        {
            if (string.IsNullOrEmpty(IndexCollection))
                throw new ArgumentException("Empty index.", nameof(IndexCollection));
 
            if (string.IsNullOrEmpty(Folder))
                throw new ArgumentException("Empty folder.", nameof(Folder));
 
            Folder = Path.GetFullPath(Folder);
            if (!Directory.Exists(Folder))
                throw new ArgumentException("Folder does not exist.", nameof(Folder));
 
            if (Folder[Folder.Length - 1] != Path.DirectorySeparatorChar)
                Folder += Path.DirectorySeparatorChar;
 
            string[] FileNames = Directory.GetFiles(Folder, "*.*", Recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly);
            Dictionary<CaseInsensitiveString, FileReference> References = new Dictionary<CaseInsensitiveString, FileReference>();
            FolderIndexationStatistics Result = new FolderIndexationStatistics();
            int i, c, d;
 
            if (!(ExcludeSubfolders is null))
            {
                ExcludeSubfolders = (string[])ExcludeSubfolders.Clone();
                c = ExcludeSubfolders.Length;
 
                for (i = 0; i < c; i++)
                {
                    ExcludeSubfolders[i] = Path.GetFullPath(ExcludeSubfolders[i]);
                    d = ExcludeSubfolders[i].Length;
 
                    if (d == 0 || ExcludeSubfolders[i][d - 1] != Path.DirectorySeparatorChar)
                        ExcludeSubfolders[i] += Path.DirectorySeparatorChar;
                }
            }
 
            IEnumerable<FileReference> ReferencesInDB = await Database.Find<FileReference>(
                new FilterAnd(
                    new FilterFieldEqualTo("IndexCollection", IndexCollection),
                    new FilterFieldLikeRegEx("FileName", Database.WildcardToRegex(Folder + "*", "*"))));
 
            foreach (FileReference Reference in ReferencesInDB)
                References[Reference.FileName] = Reference;
 
            foreach (string FileName in FileNames)
            {
                if (!(ExcludeSubfolders is null))
                {
                    bool Exclude = false;
 
                    foreach (string s in ExcludeSubfolders)
                    {
                        if (FileName.StartsWith(s))
                        {
                            Exclude = true;
                            break;
                        }
                    }
 
                    if (Exclude)
                        continue;
                }
 
                if (!FileReferenceTokenizer.HasTokenizer(FileName))
                    continue;
 
                Result.NrFiles++;
 
                DateTime TP = File.GetLastWriteTimeUtc(FileName);
 
                if (References.TryGetValue(FileName, out FileReference Ref))
                {
                    References.Remove(FileName);
 
                    if (Ref.Timestamp == TP)
                        continue;
 
                    Ref.Timestamp = TP;
                    await Database.Update(Ref); // Will trigger retokenization of file.
 
                    Result.NrUpdated++;
                    Result.TotalChanges++;
                }
                else
                {
                    Ref = new FileReference()
                    {
                        FileName = FileName,
                        IndexCollection = IndexCollection,
                        Timestamp = TP
                    };
 
                    await Database.Insert(Ref); // Will trigger tokenization of file.
 
                    Result.NrAdded++;
                    Result.TotalChanges++;
                }
            }
 
            foreach (FileReference Reference in References.Values)
            {
                await Database.Delete(Reference);   // Will trigger removal of tokens.
                Result.NrDeleted++;
                Result.TotalChanges++;
            }
 
            return Result;
        }
 
        internal static async Task<bool> IndexFile(string IndexCollection, string FileName)
        {
            FileName = Path.GetFullPath(FileName);
 
            FileReference ReferenceInDB = await Database.FindFirstIgnoreRest<FileReference>(
                new FilterAnd(
                    new FilterFieldEqualTo("IndexCollection", IndexCollection),
                    new FilterFieldEqualTo("FileName", FileName)));
 
            if (!FileReferenceTokenizer.HasTokenizer(FileName))
                return false;
 
            if (File.Exists(FileName))
            {
                DateTime TP = File.GetLastWriteTimeUtc(FileName);
 
                if (ReferenceInDB is null)
                {
                    ReferenceInDB = new FileReference()
                    {
                        FileName = FileName,
                        IndexCollection = IndexCollection,
                        Timestamp = TP
                    };
 
                    await Database.Insert(ReferenceInDB); // Will trigger tokenization of file.
 
                    return true;
                }
                else
                {
                    if (ReferenceInDB.Timestamp == TP)
                        return false;
 
                    ReferenceInDB.Timestamp = TP;
                    await Database.Update(ReferenceInDB); // Will trigger retokenization of file.
 
                    return true;
                }
            }
            else if (ReferenceInDB is null)
                return false;
            else
            {
                await Database.Delete(ReferenceInDB);   // Will trigger removal of tokens.
                return true;
            }
        }
 
    }
}