Consolidator.cs

using SkiaSharp;
using System;
using System.Collections.Generic;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Waher.Events;
using Waher.Layout.Layout2D.Functions;
using Waher.Runtime.Collections;
using Waher.Runtime.Threading;
using Waher.Script;
using Waher.Script.Graphs;
 
namespace Waher.Content.Markdown.Consolidation
{
    public class Consolidator : IConsolidator
    {
        private readonly string threadId;
        private readonly SortedDictionary<string, SourceState> sources = new SortedDictionary<string, SourceState>();
        private readonly MultiReadSingleWriteObject synchObject;
        private readonly int maxPaletteSize;
        private Dictionary<string, KeyValuePair<SKColor, int>> legend = null;
        private DocumentType type = DocumentType.Empty;
        private SKColor[] palette = null;
        private object tag = null;
        private int nrTop = 0;
        private int nrBottom = 0;
        private bool filterSources = false;
 
        public Consolidator(string ThreadId, int MaxPaletteSize)
        {
            this.synchObject = new MultiReadSingleWriteObject(this);
            this.threadId = ThreadId;
            this.maxPaletteSize = MaxPaletteSize;
        }
 
        public string ThreadId => this.threadId;
 
        public async Task<string[]> GetSources()
        {
            await this.synchObject.BeginRead();
            try
            {
                string[] Result = new string[this.sources.Count];
                this.sources.Keys.CopyTo(Result, 0);
                return Result;
            }
            finally
            {
                await this.synchObject.EndRead();
            }
        }
 
        public async Task<int> GetNrReportedSources()
        {
            await this.synchObject.BeginRead();
            try
            {
                int Result = 0;
 
                foreach (SourceState State in this.sources.Values)
                {
                    if (!State.IsDefault)
                        Result++;
                }
 
                return Result;
            }
            finally
            {
                await this.synchObject.EndRead();
            }
        }
 
        public bool FilterSources
        {
            get => this.filterSources;
            set => this.filterSources = value;
        }
 
        public object Tag
        {
            get => this.tag;
            set => this.tag = value;
        }
 
        public Task<bool> Add(string Source, MarkdownDocument Markdown)
        {
            return this.Add(Source, Markdown, string.Empty);
        }
 
        public Task<bool> Add(string Source, MarkdownDocument Markdown, string Id)
        {
            return this.Add(Source, Markdown, Id, false, false);
        }
 
        public Task<bool> Add(string Source, string Text)
        {
            return this.Add(Source, Text, string.Empty);
        }
 
        public Task<bool> Add(string Source, string Text, string Id)
        {
            return this.Add(Source, Text, Id, false, false);
        }
 
        public Task<bool> Update(string Source, MarkdownDocument Markdown, string Id)
        {
            return this.Add(Source, Markdown, Id, true, false);
        }
 
        public Task<bool> Update(string Source, string Text, string Id)
        {
            return this.Add(Source, Text, Id, true, false);
        }
 
        private async Task<bool> Add(string Source, string Text, string Id, bool Update, bool IsDefault)
        {
            MarkdownDocument Doc = await MarkdownDocument.CreateAsync(Text);
            return await this.Add(Source, Doc, Id, Update, IsDefault);
        }
 
        public Task<bool> AddDefault(string Source, string Text)
        {
            return this.Add(Source, Text, string.Empty, false, true);
        }
 
        public Task<bool> AddDefault(string Source, MarkdownDocument Markdown)
        {
            return this.Add(Source, Markdown, string.Empty, false, true);
        }
 
        private async Task<bool> Add(string Source, MarkdownDocument Markdown, string Id, bool Update, bool IsDefault)
        {
            DocumentType Type;
            bool Result;
 
            await this.synchObject.BeginWrite();
            try
            {
                if ((Result = !this.sources.TryGetValue(Source, out SourceState State)) || State.IsDefault)
                {
                    if (Result && this.filterSources)
                        return false;
 
                    State = new SourceState(Source, IsDefault);
                    this.sources[Source] = State;
                }
 
                if (Markdown is null)
                    return false;
 
                if (Update)
                    Type = await State.Update(Markdown, Id);
                else
                    Type = await State.Add(Markdown, Id);
 
                if ((int)(this.type & Type) != 0)
                    this.type = (DocumentType)Math.Max((int)this.type, (int)Type);
                else
                    this.type = DocumentType.Complex;
 
                this.nrTop = 0;
                this.nrBottom = 0;
 
                switch (this.type)
                {
                    case DocumentType.SingleCode:
                        int i, c, d, d0 = 0;
                        bool First = true;
                        string[] Rows0 = null;
                        string[] Rows;
 
                        foreach (SourceState Info in this.sources.Values)
                        {
                            Rows = Info.FirstDocument.Rows;
                            d = Rows.Length;
 
                            if (First)
                            {
                                First = false;
                                this.nrTop = this.nrBottom = d;
                                Rows0 = Rows;
                                d0 = d;
                            }
                            else
                            {
                                c = Math.Min(this.nrTop, d);
 
                                for (i = 0; i < c; i++)
                                {
                                    if (Rows[i] != Rows0[i])
                                        break;
                                }
 
                                this.nrTop = i;
 
                                c = Math.Min(this.nrBottom, d);
 
                                for (i = 0; i < c; i++)
                                {
                                    if (Rows[d - i - 1] != Rows0[d0 - i - 1])
                                        break;
                                }
 
                                this.nrBottom = i;
                            }
                        }
 
                        if (this.nrTop < 1 || this.nrBottom <= 1)
                            this.type = DocumentType.Complex;
                        break;
 
                    case DocumentType.SingleXml:
                        if (this.sources.Count >= 2)
                            this.type = DocumentType.Complex;
                        break;
                }
            }
            finally
            {
                await this.synchObject.EndWrite();
            }
 
            if (Update)
                await this.Raise(this.Updated, Source);
            else
                await this.Raise(this.Added, Source);
 
            return Result;
        }
 
        private Task Raise(EventHandlerAsync<SourceEventArgs> Handler, string Source)
        {
            return Handler.Raise(this, new SourceEventArgs(Source));
        }
 
        public event EventHandlerAsync<SourceEventArgs> Added = null;
 
        public event EventHandlerAsync<SourceEventArgs> Updated = null;
 
        [Obsolete("Use GenerateMarkdownAsync() instead.")]
        public string GenerateMarkdown()
        {
            return this.GenerateMarkdownAsync().Result;
        }
 
        public async Task<string> GenerateMarkdownAsync()
        {
            StringBuilder Markdown = new StringBuilder();
 
            await this.synchObject.BeginRead();
            try
            {
                switch (this.type)
                {
                    case DocumentType.Empty:
                        return string.Empty;
 
                    case DocumentType.SingleNumber:
                    case DocumentType.SingleLine:
 
                        Markdown.AppendLine("| Nr | Source | Response |");
 
                        if (this.type == DocumentType.SingleNumber)
                            Markdown.AppendLine("|---:|:-------|-------:|");
                        else
                            Markdown.AppendLine("|---:|:-------|:-------|");
 
                        int Nr = 0;
 
                        foreach (KeyValuePair<string, SourceState> P in this.sources)
                        {
                            Markdown.Append("| ");
                            Markdown.Append((++Nr).ToString());
                            Markdown.Append(" | `");
                            Markdown.Append(P.Key);
                            Markdown.Append("` | ");
                            Markdown.Append((await P.Value.GetFirstText()).Trim());
                            Markdown.AppendLine(" |");
                        }
                        break;
 
                    case DocumentType.SingleParagraph:
 
                        Markdown.AppendLine("| Nr | Source | Response |");
                        Markdown.AppendLine("|---:|:-------|:-------|");
 
                        Nr = 0;
 
                        foreach (KeyValuePair<string, SourceState> P in this.sources)
                        {
                            Markdown.Append("| ");
                            Markdown.Append((++Nr).ToString());
                            Markdown.Append(" | `");
                            Markdown.Append(P.Key);
                            Markdown.Append("` | ");
 
                            foreach (string Row in P.Value.FirstDocument.Rows)
                            {
                                Markdown.Append(Row);
                                Markdown.Append("<br/>");
                            }
 
                            Markdown.AppendLine("|");
                        }
                        break;
 
                    case DocumentType.SingleCode:
                    case DocumentType.SingleXml:
 
                        ChunkedList<string> ConsolidatedRows = new ChunkedList<string>();
                        int j = 0;
                        int d = this.sources.Count;
 
                        foreach (KeyValuePair<string, SourceState> P in this.sources)
                        {
                            string[] Rows = P.Value.FirstDocument.Rows;
                            int i = 0;
                            int c = Rows.Length;
 
                            j++;
                            if (j > 1)
                                i += this.nrTop;
 
                            if (j < d)
                                c -= this.nrBottom;
 
                            for (; i < c; i++)
                                ConsolidatedRows.Add(Rows[i]);
                        }
 
                        if (ConsolidatedRows[0].StartsWith("```dot", StringComparison.OrdinalIgnoreCase))
                            OptimizeDotEdges(ConsolidatedRows);
 
                        foreach (string Row in ConsolidatedRows)
                            Markdown.AppendLine(Row);
                        break;
 
                    case DocumentType.SingleTable:
 
                        ConsolidatedTable Table = null;
 
                        try
                        {
                            foreach (KeyValuePair<string, SourceState> P in this.sources)
                            {
                                foreach (DocumentInformation Doc in P.Value.Documents)
                                {
                                    if (!(Doc?.Table is null))
                                    {
                                        if (Table is null)
                                            Table = await ConsolidatedTable.CreateAsync(P.Key, Doc.Table);
                                        else
                                            await Table.Add(P.Key, Doc.Table);
                                    }
                                }
                            }
 
                            Table?.Export(Markdown);
                        }
                        catch (Exception ex)
                        {
                            Log.Exception(ex);
                            this.GenerateComplexLocked(Markdown);
                        }
                        break;
 
                    case DocumentType.SingleGraph:
                        Graph G = null;
 
                        try
                        {
                            int i;
 
                            this.legend ??= new Dictionary<string, KeyValuePair<SKColor, int>>();
                            this.palette ??= CreatePalette(this.maxPaletteSize);
 
                            foreach (KeyValuePair<string, SourceState> P in this.sources)
                            {
                                foreach (DocumentInformation Doc in P.Value.Documents)
                                {
                                    if (!(Doc?.Graph is null))
                                    {
                                        i = this.legend.Count % this.maxPaletteSize;
                                        if (Doc.Graph.TrySetDefaultColor(this.palette[i]))
                                            this.legend[P.Key] = new KeyValuePair<SKColor, int>(this.palette[i], i);
 
                                        if (G is null)
                                            G = Doc.Graph;
                                        else
                                            G = (Graph)G.AddRightElementWise(Doc.Graph);
                                    }
                                }
                            }
 
                            Markdown.AppendLine("```Graph");
                            G.ToXml(Markdown);
                            Markdown.AppendLine();
                            Markdown.AppendLine("```");
 
                            if (this.legend.Count > 0)
                            {
                                string[] Labels = new string[this.legend.Count];
                                SKColor[] Colors = new SKColor[this.legend.Count];
                                
                                i = 0;
                                Markdown.AppendLine();
 
                                foreach (KeyValuePair<string, KeyValuePair<SKColor, int>> P in this.legend)
                                {
                                    Labels[i] = P.Key;
                                    Colors[i] = P.Value.Key;
                                    i++;
                                }
 
                                string LegandXml = Legend.CreateLayout(Labels, Colors, SKColors.Black, 4, new Variables());
 
                                Markdown.AppendLine("```layout");
                                Markdown.AppendLine(LegandXml);
                                Markdown.AppendLine("```");
                                Markdown.AppendLine();
                            }
 
                            Markdown.AppendLine();
                        }
                        catch (Exception ex)
                        {
                            Log.Exception(ex);
                            this.GenerateComplexLocked(Markdown);
                        }
                        break;
 
                    case DocumentType.Complex:
                    default:
                        this.GenerateComplexLocked(Markdown);
                        break;
                }
            }
            finally
            {
                await this.synchObject.EndRead();
            }
 
            return Markdown.ToString();
        }
 
        private static readonly Regex dotEdge = new Regex("^(?'A'(\"[^\"]*\"|'[^']*'|[^\\s]*))\\s*->\\s*(?'B'(\"[^\"]*\"|'[^']*'|[^\\s]*))\\s*([\\[](?'Note'[^\\]]*)[\\]]\\s*)?;\\s*$", RegexOptions.Singleline | RegexOptions.Compiled);
 
        private static void OptimizeDotEdges(ChunkedList<string> Rows)
        {
            Match[] Matches;
            Match M;
            string Row, Row2, A, B, Note;
            int i, j, c = Rows.Count;
            bool Changed = false;
 
            Matches = new Match[c];
 
            for (i = 0; i < c; i++)
            {
                Row = Rows[i];
                M = Matches[i];
                if (M is null)
                {
                    M = dotEdge.Match(Row);
                    Matches[i] = M;
                }
 
                if (!M.Success || M.Index > 0 || M.Length < Row.Length)
                    continue;
 
                A = M.Groups["A"].Value;
                B = M.Groups["B"].Value;
                Note = M.Groups["Note"].Value;
 
                for (j = i + 1; j < c; j++)
                {
                    Row2 = Rows[j];
                    M = Matches[j];
                    if (M is null)
                    {
                        M = dotEdge.Match(Row2);
                        Matches[j] = M;
                    }
 
                    if (M.Success &&
                        M.Index == 0 &&
                        M.Length == Row.Length &&
                        M.Groups["A"].Value == B &&
                        M.Groups["B"].Value == A &&
                        M.Groups["Note"].Value == Note)
                    {
                        StringBuilder sb = new StringBuilder();
 
                        sb.Append(A);
                        sb.Append(" -> ");
                        sb.Append(B);
                        sb.Append(" [");
                        sb.Append(Note);
 
                        if (!string.IsNullOrEmpty(Note))
                            sb.Append(", ");
 
                        sb.Append("dir=both];");
 
                        Rows[i] = sb.ToString();
                        Matches[i] = null;
                        Rows[j] = string.Empty;
                        Matches[j] = null;
                        Changed = true;
                        break;
                    }
                }
            }
 
            if (Changed)
            {
                for (i = 0; i < c;)
                {
                    if (string.IsNullOrEmpty(Rows[i]))
                    {
                        Rows.RemoveAt(i);
                        c--;
                    }
                    else
                        i++;
                }
            }
        }
 
        public static SKColor[] CreatePalette(int N)
        {
            SKColor[] Result = new SKColor[N];
            double d = 360.0 / Math.Max(N, 12);
            int i;
 
            for (i = 0; i < N; i++)
                Result[i] = SKColor.FromHsl((float)(d * i), 100, 75);
 
            return Result;
        }
 
        private void GenerateComplexLocked(StringBuilder Markdown)
        {
            foreach (KeyValuePair<string, SourceState> P in this.sources)
            {
                Markdown.Append('`');
                Markdown.Append(P.Key);
                Markdown.AppendLine("`");
                Markdown.AppendLine();
 
                DocumentInformation[] Info = P.Value.Documents;
 
                if (Info.Length == 0)
                    Markdown.AppendLine(":\t");
                else
                {
                    bool First = true;
 
                    foreach (DocumentInformation Doc in P.Value.Documents)
                    {
                        if (First)
                            First = false;
                        else
                            Markdown.AppendLine(":\t");
 
                        if (!(Doc?.Rows is null))
                        {
                            foreach (string Row in Doc.Rows)
                            {
                                Markdown.Append(":\t");
                                Markdown.AppendLine(Row);
                            }
                        }
                    }
                }
 
                Markdown.AppendLine();
            }
        }
 
        [Obsolete("Use DisposeAsync() instead.")]
        public void Dispose()
        {
            try
            {
                this.DisposeAsync().Wait();
            }
            catch (Exception ex)
            {
                Log.Exception(ex);
            }
        }
 
        public Task DisposeAsync()
        {
            return this.Disposed.Raise(this, EventArgs.Empty, false);
        }
 
        public event EventHandlerAsync Disposed = null;
    }
}