TokenizationProcess.cs

using System;
using System.Collections.Generic;
using Waher.Runtime.Collections;
 
namespace Waher.Persistence.FullTextSearch.Tokenizers
{
    public class TokenizationProcess
    {
        public TokenizationProcess()
        {
            this.TokenCounts = new Dictionary<string, ChunkedList<uint>>();
        }
 
        public Dictionary<string, ChunkedList<uint>> TokenCounts { get; }
 
        public uint DocumentIndexOffset { get; set; }
 
        public TokenCount[] ToArray()
        {
            int c = this.TokenCounts.Count;
            if (c == 0)
                return Array.Empty<TokenCount>();
 
            int i = 0;
            TokenCount[] Counts = new TokenCount[c];
 
            foreach (KeyValuePair<string, ChunkedList<uint>> P in this.TokenCounts)
                Counts[i++] = new TokenCount(P.Key, P.Value.ToArray());
 
            return Counts;
        }
    }
}