HtmlTokenizer.cs

using System;
using System.Text;
using System.Threading.Tasks;
using Waher.Content.Html;
using Waher.Persistence.FullTextSearch;
using Waher.Persistence.FullTextSearch.Files;
using Waher.Persistence.FullTextSearch.Tokenizers;
using Waher.Runtime.Inventory;
using Waher.Runtime.IO;
 
namespace Waher.IoTGateway.Tokenizers
{
    public class HtmlTokenizer : ITokenizer, IFileTokenizer
    {
        public HtmlTokenizer()
        {
        }
 
        public Grade Supports(Type Type)
        {
            if (Type == typeof(HtmlDocument))
                return Grade.Ok;
            else
                return Grade.NotAtAll;
        }
 
        public Grade Supports(string Extension)
        {
            switch (Extension)
            {
                case "htm":
                case "html":
                case "xhtml":
                    return Grade.Ok;
 
                default:
                    return Grade.NotAtAll;
            }
        }
 
        public async Task Tokenize(object Value, TokenizationProcess Process)
        {
            if (Value is HtmlDocument Doc)
                await Tokenize(Doc, Process);
        }
 
        public static Task Tokenize(HtmlDocument Doc, TokenizationProcess Process)
        {
            StringBuilder sb = new StringBuilder();
 
            GetText(Doc.Root, sb);
 
            StringTokenizer.Tokenize(sb.ToString(), Process);
 
            return Task.CompletedTask;
        }
 
        private static void GetText(HtmlNode N, StringBuilder Text)
        {
            if (N is HtmlElement E)
            {
                if (E.HasAttributes)
                {
                    foreach (HtmlAttribute Attr in E.Attributes)
                    {
                        Text.Append(' ');
                        Text.Append(Attr.Value);
                    }
                }
 
                if (E.HasChildren)
                {
                    foreach (HtmlNode N2 in E.Children)
                        GetText(N2, Text);
                }
            }
            else if (N is HtmlText T)
            {
                Text.Append(' ');
                Text.Append(T.InlineText);
            }
        }
 
        public async Task Tokenize(FileReference Reference, TokenizationProcess Process)
        {
            string Text = await Files.ReadAllTextAsync(Reference.FileName);
            HtmlDocument Doc = new HtmlDocument(Text);
 
            await Tokenize(Doc, Process);
        }
    }
}