MarkdownTokenizer.cs

using System;
using System.Diagnostics;
using System.Text;
using System.Threading.Tasks;
using Waher.Content.Markdown;
using Waher.Content.Markdown.Rendering;
using Waher.Persistence.FullTextSearch;
using Waher.Persistence.FullTextSearch.Files;
using Waher.Persistence.FullTextSearch.Tokenizers;
using Waher.Persistence.Serialization;
using Waher.Runtime.Inventory;
using Waher.Runtime.IO;
 
namespace Waher.IoTGateway.Tokenizers
{
    public class MarkdownTokenizer : ITokenizer, IFileTokenizer, IPropertyEvaluator
    {
        private string definition = string.Empty;
 
        public MarkdownTokenizer()
        {
        }
 
        public Grade Supports(Type Type)
        {
            if (Type == typeof(MarkdownDocument))
                return Grade.Ok;
            else
                return Grade.NotAtAll;
        }
 
        public Grade Supports(string Extension)
        {
            if (Extension == "md")
                return Grade.Ok;
            else
                return Grade.NotAtAll;
        }
 
        public async Task Tokenize(object Value, TokenizationProcess Process)
        {
            if (Value is MarkdownDocument Doc)
                await Tokenize(Doc, Process);
        }
 
        public static async Task Tokenize(MarkdownDocument Doc, TokenizationProcess Process)
        {
            StringBuilder sb = new StringBuilder();
 
            Append(sb, Doc.Description);
            Append(sb, Doc.Author);
            Append(sb, Doc.Keywords);
            Append(sb, Doc.Subtitle);
            Append(sb, Doc.Title);
 
            using (TextRenderer Renderer = new TextRenderer(sb))
            {
                await Renderer.RenderDocument(Doc, true);
            }
 
            StringTokenizer.Tokenize(sb.ToString(), Process);
        }
 
        private static void Append(StringBuilder sb, string[] Strings)
        {
            if (!(Strings is null))
            {
                foreach (string s in Strings)
                    sb.AppendLine(s);
            }
        }
 
        public async Task Tokenize(FileReference Reference, TokenizationProcess Process)
        {
            string Text = await Files.ReadAllTextAsync(Reference.FileName);
            MarkdownDocument Doc = await MarkdownDocument.CreateAsync(Text);
 
            await Tokenize(Doc, Process);
        }
 
        #region IPropertyEvaluator
 
        public Task Prepare(string Definition)
        {
            this.definition = Definition;
            return Task.CompletedTask;
        }
 
        public async Task<object> Evaluate(object Instance)
        {
            if (Instance is GenericObject Obj)
            {
                if (Obj.TryGetFieldValue(this.definition, out object Obj2) &&
                    Obj2 is string Markdown)
                {
                    MarkdownDocument Doc = await MarkdownDocument.CreateAsync(Markdown);
 
                    if (MarkdownDocument.HeaderEndPosition(Markdown).HasValue)
                        return Doc;
                    else
                        return await Doc.GeneratePlainText();
                }
            }
 
            return null;
        }
 
        #endregion
    }
}