MarkdownTokenizer.cs

using System;
using System.Text;
using System.Threading.Tasks;
using Waher.Content;
using Waher.Content.Markdown;
using Waher.Persistence.FullTextSearch;
using Waher.Persistence.FullTextSearch.Files;
using Waher.Persistence.FullTextSearch.Tokenizers;
using Waher.Runtime.Inventory;
 
namespace Waher.IoTGateway.Tokenizers
{
    public class MarkdownTokenizer : ITokenizer, IFileTokenizer
    {
        public MarkdownTokenizer()
        {
        }
 
        public Grade Supports(Type Type)
        {
            if (Type == typeof(MarkdownDocument))
                return Grade.Ok;
            else
                return Grade.NotAtAll;
        }
 
        public Grade Supports(string Extension)
        {
            if (Extension == "md")
                return Grade.Ok;
            else
                return Grade.NotAtAll;
        }
 
        public async Task Tokenize(object Value, TokenizationProcess Process)
        {
            if (Value is MarkdownDocument Doc)
                await Tokenize(Doc, Process);
        }
 
        public static async Task Tokenize(MarkdownDocument Doc, TokenizationProcess Process)
        {
            StringBuilder sb = new StringBuilder();
 
            Append(sb, Doc.Description);
            Append(sb, Doc.Author);
            Append(sb, Doc.Keywords);
            Append(sb, Doc.Subtitle);
            Append(sb, Doc.Title);
 
            await Doc.GeneratePlainText(sb);
            StringTokenizer.Tokenize(sb.ToString(), Process);
        }
 
        private static void Append(StringBuilder sb, string[] Strings)
        {
            if (!(Strings is null))
            {
                foreach (string s in Strings)
                    sb.AppendLine(s);
            }
        }
 
        public async Task Tokenize(FileReference Reference, TokenizationProcess Process)
        {
            string Text = await Resources.ReadAllTextAsync(Reference.FileName);
            MarkdownDocument Doc = await MarkdownDocument.CreateAsync(Text);
 
            await Tokenize(Doc, Process);
        }
    }
}