HumanReadableTextsTokenizer.cs

using System;
using System.Collections.Generic;
using System.Reflection;
using System.Threading.Tasks;
using Waher.Persistence.FullTextSearch;
using Waher.Persistence.FullTextSearch.Tokenizers;
using Waher.Runtime.Inventory;
using Waher.Service.IoTBroker.Legal.HumanReadable;
 
namespace Waher.Service.IoTBroker.Legal.Contracts.Tokenizers
{
    public class HumanReadableTextsTokenizer : ITokenizer
    {
        public HumanReadableTextsTokenizer()
        {
        }
 
        public Grade Supports(Type Type)
        {
            if (typeInfo.IsAssignableFrom(Type))
                return Grade.Ok;
            else
                return Grade.NotAtAll;
        }
 
        private static readonly TypeInfo typeInfo = typeof(IEnumerable<HumanReadableText>).GetTypeInfo();
 
        public async Task Tokenize(object Value, TokenizationProcess Process)
        {
            if (Value is IEnumerable<HumanReadableText> HumanReadableTexts)
                await Tokenize(HumanReadableTexts, null, Process);
        }
 
        public static async Task Tokenize(IEnumerable<HumanReadableText> HumanReadableTexts,
            Contract Contract, TokenizationProcess Process)
        {
            if (!(HumanReadableTexts is null))
            {
                foreach (HumanReadableText HumanReadableText in HumanReadableTexts)
                {
                    await HumanReadableTextTokenizer.Tokenize(HumanReadableText, Contract, Process);
                    Process.DocumentIndexOffset++;  // Make sure sequences of keywords don't cross element boundaries.
                }
            }
        }
    }
}