TextFileTokenizer.cs

using System.Threading.Tasks;
using Waher.Persistence.FullTextSearch.Tokenizers;
using Waher.Runtime.Inventory;
 
namespace Waher.Persistence.FullTextSearch.Files
{
    public class TextFileTokenizer : IFileTokenizer
    {
        public TextFileTokenizer()
        {
        }
 
        public Grade Supports(string Extension)
        {
            if (Extension == "txt")
                return Grade.Ok;
            else
                return Grade.NotAtAll;
        }
 
        public async Task Tokenize(FileReference Reference, TokenizationProcess Process)
        {
            string Text = await Runtime.IO.Files.ReadAllTextAsync(Reference.FileName);
            StringTokenizer.Tokenize(Text, Process);
        }
    }
}