NeuroFeatureTokenizer.cs

using System;
using System.Threading.Tasks;
using Waher.Content.Markdown;
using Waher.Persistence.FullTextSearch;
using Waher.Persistence.FullTextSearch.Tokenizers;
using Waher.Runtime.Inventory;
using Waher.Service.IoTBroker.Legal.Contracts;
 
namespace Waher.Service.IoTBroker.NeuroFeatures.Tokenizers
{
    public class NeuroFeatureTokenizer : ITokenizer
    {
        public NeuroFeatureTokenizer()
        {
        }
 
        public Grade Supports(Type Type)
        {
            if (Type == typeof(Token))
                return Grade.Ok;
            else
                return Grade.NotAtAll;
        }
 
        public async Task Tokenize(object Value, TokenizationProcess Process)
        {
            if (Value is Token Token)
                await Tokenize(Token, Process);
        }
 
        public static async Task Tokenize(Token Token, TokenizationProcess Process)
        {
            if (!(Token is null) &&
                Token.Visibility == ContractVisibility.PublicSearchable)
            {
                StringTokenizer.Tokenize(Token.FriendlyName, Process);
                Process.DocumentIndexOffset++;
                StringTokenizer.Tokenize(Token.Category, Process);
                Process.DocumentIndexOffset++;
 
                if (!string.IsNullOrEmpty(Token.Description))
                {
                    MarkdownDocument Doc = await MarkdownDocument.CreateAsync(Token.Description);
                    string Text = await Doc.GeneratePlainText();
 
                    StringTokenizer.Tokenize(Text, Process);
                }
            }
        }
    }
}