HumanReadableTextTokenizer.cs

using System;
using System.Text;
using System.Threading.Tasks;
using Waher.Content.Markdown;
using Waher.Persistence.FullTextSearch;
using Waher.Persistence.FullTextSearch.Tokenizers;
using Waher.Runtime.Inventory;
using Waher.Service.IoTBroker.Legal.HumanReadable;
 
namespace Waher.Service.IoTBroker.Legal.Contracts.Tokenizers
{
    public class HumanReadableTextTokenizer : ITokenizer
    {
        public HumanReadableTextTokenizer()
        {
        }
 
        public Grade Supports(Type Type)
        {
            if (Type == typeof(HumanReadableText))
                return Grade.Ok;
            else
                return Grade.NotAtAll;
        }
 
        public async Task Tokenize(object Value, TokenizationProcess Process)
        {
            if (Value is HumanReadableText HumanReadableText)
                await Tokenize(HumanReadableText, null, Process);
        }
 
        public static async Task Tokenize(HumanReadableText HumanReadableText, 
            Contract Contract, TokenizationProcess Process)
        {
            if (!(HumanReadableText is null))
            {
                MarkdownOutput Markdown = new MarkdownOutput();
                MarkdownOutputState State = new MarkdownOutputState()
                {
                    Language = HumanReadableText.Language,
                    Contract = Contract,
                    Level = 1,
                    Indentation = 0
                };
 
                await HumanReadableText.ToMarkdown(Markdown, State);
 
                MarkdownDocument Doc = await MarkdownDocument.CreateAsync(Markdown.ToString());
                string Text = await Doc.GeneratePlainText();
 
                StringTokenizer.Tokenize(Text, Process);
            }
        }
    }
}