NeuroFeatureTokenizer.cs

using System;
using System.Threading.Tasks;
using Waher.Content.Markdown;
using Waher.Persistence.FullTextSearch;
using Waher.Persistence.FullTextSearch.Tokenizers;
using Waher.Persistence.Serialization;
using Waher.Runtime.Inventory;
 
namespace Waher.Service.IoTBroker.NeuroFeatures.Tokenizers
{
    public class NeuroFeatureTokenizer : ITokenizer, IPropertyEvaluator
    {
        private string definition = string.Empty;
 
        public NeuroFeatureTokenizer()
        {
        }
 
        public Grade Supports(Type Type)
        {
            if (Type == typeof(Token))
                return Grade.Ok;
            else
                return Grade.NotAtAll;
        }
 
        public async Task Tokenize(object Value, TokenizationProcess Process)
        {
            if (Value is Token Token)
                await Tokenize(Token, Process);
        }
 
        public static async Task Tokenize(Token Token, TokenizationProcess Process)
        {
            if (!(Token is null))
            {
                StringTokenizer.Tokenize(Token.FriendlyName, Process);
                Process.DocumentIndexOffset++;
                StringTokenizer.Tokenize(Token.Category, Process);
                Process.DocumentIndexOffset++;
 
                if (!string.IsNullOrEmpty(Token.Description))
                {
                    MarkdownDocument Doc = await MarkdownDocument.CreateAsync(Token.Description);
                    string Text = await Doc.GeneratePlainText();
 
                    StringTokenizer.Tokenize(Text, Process);
                }
            }
        }
 
        public Task Prepare(string Definition)
        {
            this.definition = Definition;
            return Task.CompletedTask;
        }
 
        public Task<object> Evaluate(object Instance)
        {
            if (Instance is Token Token)
            {
                switch (this.definition)
                {
                    case "FriendlyName": return Task.FromResult<object>(Token.FriendlyName);
                    case "Category": return Task.FromResult<object>(Token.Category);
                }
            }
            else if (Instance is GenericObject Obj)
            {
                if (Obj.TryGetFieldValue(this.definition, out object Obj2))
                    return Task.FromResult(Obj2);
            }
 
            return Task.FromResult<object>(null);
        }
    }
}