HttpStatistics.cs

using System;
using System.Collections.Generic;
using System.Net;
using System.Text;
using System.Threading.Tasks;
using Waher.Content.Markdown;
using Waher.Persistence;
using Waher.Persistence.Filters;
using Waher.Runtime.Language;
using Waher.Script;
using Waher.Script.Graphs;
using Waher.Things;
using Waher.Things.Attributes;
using Waher.Things.Queries;
 
namespace Waher.Service.IoTBroker.Sources.Protocols.Commands
{
    public class HttpStatistics : ICommand
    {
        [Page(1, "Time", 150)]
        [Header(2, "From:")]
        [ToolTip(3, "Search for records from this point in time.")]
        public DateTime From = DateTime.MinValue;
 
        [Page(1, "Time", 150)]
        [Header(4, "To:")]
        [ToolTip(5, "Search for records to this point in time.")]
        public DateTime To = DateTime.MaxValue;
 
        public string CommandID => "Statistics";
        public CommandType Type => CommandType.Query;
        public string SortCategory => "Protocols";
        public string SortKey => "Statistics";
 
        public Task<string> GetNameAsync(Language Language)
        {
            return Language.GetStringAsync(typeof(HttpStatistics), 6, "Statistics...");
        }
 
        public Task<bool> CanExecuteAsync(RequestOrigin Caller)
        {
            return XmppServerModule.IsAdmin(Caller.From);
        }
 
        public ICommand Copy()
        {
            return new HttpStatistics()
            {
                From = this.From,
                To = this.To
            };
        }
 
        public Task ExecuteCommandAsync()
        {
            throw new NotSupportedException();
        }
 
        public Task<string> GetConfirmationStringAsync(Language Language)
        {
            return Task.FromResult(string.Empty);
        }
 
        public Task<string> GetFailureStringAsync(Language Language)
        {
            return Task.FromResult(string.Empty);
        }
 
        public Task<string> GetSuccessStringAsync(Language Language)
        {
            return Task.FromResult(string.Empty);
        }
 
        public Task StartQueryExecutionAsync(Query Query, Language Language)
        {
            this.Execute(Query, Language);
            return Task.CompletedTask;
        }
 
        private async void Execute(Query Query, Language Language)
        {
            try
            {
                await Query.Start();
                await Query.SetTitle(await Language.GetStringAsync(typeof(HttpStatistics), 7, "Requests"));
 
                if (this.To < this.From)
                {
                    DateTime Temp = this.To;
                    this.To = this.From;
                    this.From = Temp;
                }
 
                List<Filter> Filters = new List<Filter>();
                IEnumerable<HttpStatistic> Records;
 
                if (this.From > DateTime.MinValue)
                    Filters.Add(new FilterFieldGreaterOrEqualTo("Timestamp", this.From));
 
                if (this.To < DateTime.MaxValue)
                    Filters.Add(new FilterFieldLesserOrEqualTo("Timestamp", this.To));
 
                await Query.SetStatus(await Language.GetStringAsync(typeof(HttpStatistics), 8, "Searching database..."));
 
                switch (Filters.Count)
                {
                    case 0:
                        Records = await Database.Find<HttpStatistic>("Timestamp");
                        break;
 
                    case 1:
                        Records = await Database.Find<HttpStatistic>(Filters[0], "Timestamp");
                        break;
 
                    default:
                        Records = await Database.Find<HttpStatistic>(new FilterAnd(Filters.ToArray()), "Timestamp");
                        break;
                }
 
                string Header = await Language.GetStringAsync(typeof(HttpStatistics), 9, "HTTP Statistics");
                await Query.BeginSection(Header);
 
                int NrRecords = 0;
 
                foreach (HttpStatistic Record in Records)
                    NrRecords++;
 
                const int MaxRecords = 50;
                bool ShowTable = NrRecords <= MaxRecords;
 
                if (ShowTable)
                {
                    await Query.NewTable("Statistics", Header, new Column[]
                    {
                        new Column("Start", await Language.GetStringAsync(typeof(HttpStatistics), 10, "Start"),
                            null, null, null, null, ColumnAlignment.Left, null),
                        new Column("Timestamp", await Language.GetStringAsync(typeof(HttpStatistics), 11, "Timestamp"),
                            null, null, null, null, ColumnAlignment.Left, null),
                        new Column("Requests", await Language.GetStringAsync(typeof(HttpStatistics), 12, "#Requests"),
                            null, null, null, null, ColumnAlignment.Left, null),
                        new Column("Rx", await Language.GetStringAsync(typeof(HttpStatistics), 13, "#Rx"),
                            null, null, null, null, ColumnAlignment.Left, null),
                        new Column("Tx", await Language.GetStringAsync(typeof(HttpStatistics), 14, "#Tx"),
                            null, null, null, null, ColumnAlignment.Left, null)
                    });
                }
                else
                {
                    StringBuilder Markdown = new StringBuilder();
 
                    Markdown.Append(await Language.GetStringAsync(typeof(HttpStatistics), 18, "Number of records:"));
                    Markdown.Append(" **");
                    Markdown.Append(NrRecords.ToString());
                    Markdown.Append("**. (");
                    Markdown.Append((await Language.GetStringAsync(typeof(HttpStatistics), 19, "Table shown if %0% records or less.")).Replace("%0%", MaxRecords.ToString()));
                    Markdown.Append(")");
 
                    await Query.NewObject(new MarkdownContent(Markdown.ToString()));
                }
 
                SortedDictionary<string, double> PerMethod = new SortedDictionary<string, double>();
                SortedDictionary<string, double> PerUserAgent = new SortedDictionary<string, double>();
                SortedDictionary<string, double> PerFrom = new SortedDictionary<string, double>();
                SortedDictionary<string, double> PerResource = new SortedDictionary<string, double>();
                List<DateTime> Timestamps = new List<DateTime>();
                List<double> NrCalls = new List<double>();
                List<double> NrRx = new List<double>();
                List<double> NrTx = new List<double>();
                double MaxNrCalls = 0;
                double MaxNrRx = 0;
                double MaxNrTx = 0;
                double DivisorNrCalls = 1;
                double DivisorNrRx = 1;
                double DivisorNrTx = 1;
                string PrefixNrCalls = string.Empty;
                string PrefixNrRx = string.Empty;
                string PrefixNrTx = string.Empty;
 
                foreach (HttpStatistic Rec in Records)
                {
                    if (ShowTable)
                    {
                        await Query.NewRecords("Statistics", new Record(new object[]
                        {
                            Rec.Start,
                            Rec.Timestamp,
                            Rec.NrCalls,
                            Rec.NrBytesRx,
                            Rec.NrBytesTx
                        }));
                    }
 
                    Timestamps.Add(Rec.Timestamp);
                    NrCalls.Add(Rec.NrCalls);
                    NrRx.Add(Rec.NrBytesRx);
                    NrTx.Add(Rec.NrBytesTx);
 
                    if (Rec.NrCalls > MaxNrCalls)
                        MaxNrCalls = Rec.NrCalls;
 
                    if (Rec.NrBytesRx > MaxNrRx)
                        MaxNrRx = Rec.NrBytesRx;
 
                    if (Rec.NrBytesTx > MaxNrTx)
                        MaxNrTx = Rec.NrBytesTx;
 
                    Accumulate(PerMethod, Rec.CallsPerMethod, null);
                    Accumulate(PerUserAgent, Rec.CallsPerUserAgent, (s) =>
                    {
                        StringBuilder sb = new StringBuilder();
                        bool InParenthesis = false;
                        bool Removed = false;
 
                        foreach (char ch in s)
                        {
                            if (InParenthesis)
                            {
                                if (ch == ')')
                                {
                                    InParenthesis = false;
                                    sb.Append(')');
                                }
                                else if (!Removed)
                                {
                                    sb.Append("...");
                                    Removed = true;
                                }
                            }
                            else
                            {
                                if (ch >= '0' && ch <= '9')
                                    sb.Append('#');
                                else if (ch == '(')
                                {
                                    sb.Append('(');
                                    InParenthesis = true;
                                    Removed = false;
                                }
                                else
                                    sb.Append(ch);
                            }
                        }
 
                        return sb.ToString();
                    });
                    Accumulate(PerFrom, Rec.CallsPerFrom, (s) =>
                    {
                        int i, j;
 
                        if (IPAddress.TryParse(s, out _))
                            s = "IP Address";
                        else if ((i = s.IndexOf('@')) > 0 & (j = s.LastIndexOf('/')) > 0 && j > i)
                            s = s.Substring(0, j);
 
                        return s;
                    });
                    Accumulate(PerResource, Rec.CallsPerResource, null);
                }
 
                ScaleAmounts(MaxNrCalls, 1000, ref DivisorNrCalls, ref PrefixNrCalls);
                ScaleAmounts(MaxNrRx, 1024, ref DivisorNrRx, ref PrefixNrRx);
                ScaleAmounts(MaxNrTx, 1024, ref DivisorNrTx, ref PrefixNrTx);
 
                if (DivisorNrRx < DivisorNrTx)
                {
                    DivisorNrRx = DivisorNrTx;
                    PrefixNrRx = PrefixNrTx;
                }
 
                if (ShowTable)
                    await Query.TableDone("Statistics");
 
                await Query.SetStatus(await Language.GetStringAsync(typeof(HttpStatistics), 20, "Preparing Request graph."));
                await Query.NewObject(await Expression.EvalAsync(
                    "G:=plot2dline(Timestamps,NrCalls/" + DivisorNrCalls.ToString() + ",'Red');" +
                    "G.Title:=Title;" +
                    "G.LabelX:=LabelX;" +
                    "G.LabelY:=LabelY;" +
                    "G",
                    new Variables()
                    {
                        { Graph.GraphWidthVariableName, 1280 },
                        { Graph.GraphHeightVariableName, 500 },
                        { "Timestamps", Timestamps.ToArray() },
                        { "NrCalls", NrCalls.ToArray() },
                        { "Title", await Language.GetStringAsync(typeof(HttpStatistics), 26, "HTTP Requests over time") },
                        { "LabelX", await Language.GetStringAsync(typeof(HttpStatistics), 1, "Time") },
                        { "LabelY", PrefixNrCalls + await Language.GetStringAsync(typeof(HttpStatistics), 27, "Requests/day") }
                    }));
 
                await Query.SetStatus(await Language.GetStringAsync(typeof(HttpStatistics), 21, "Preparing transfer graph."));
                await Query.NewObject(await Expression.EvalAsync(
                    "G:=plot2dline(Timestamps,NrRx/" + DivisorNrTx.ToString() + ",'Blue')+" +
                    "plot2dline(Timestamps,NrTx/" + DivisorNrTx.ToString() + ",'Red');" +
                    "G.Title:=Title;" +
                    "G.LabelX:=LabelX;" +
                    "G.LabelY:=LabelY;" +
                    "G",
                    new Variables()
                    {
                        { Graph.GraphWidthVariableName, 1280 },
                        { Graph.GraphHeightVariableName, 500 },
                        { "Timestamps", Timestamps.ToArray() },
                        { "NrRx", NrRx.ToArray() },
                        { "NrTx", NrTx.ToArray() },
                        { "Title", await Language.GetStringAsync(typeof(HttpStatistics), 28, "HTTP Bytes Transmitted over time") },
                        { "LabelX", await Language.GetStringAsync(typeof(HttpStatistics), 1, "Time") },
                        { "LabelY", PrefixNrTx + await Language.GetStringAsync(typeof(HttpStatistics), 29, "Bytes/day") }
                    }));
                await Query.NewObject(await Expression.EvalAsync(
                    "legend(['NrRx','NrTx'],['Blue','Red'],'White',2)"));
 
                double Height = 100 + 20 * PerMethod.Count;
                double Scale = Height > 8000 ? Scale = 8000 / Height : 1;
                double Max = CalcMax(PerMethod);
                double Divisor = 1;
                string Prefix = string.Empty;
                ScaleAmounts(Max, 1000, ref Divisor, ref Prefix);
 
                await Query.SetStatus(await Language.GetStringAsync(typeof(HttpStatistics), 22, "Preparing method graph."));
                await Query.NewObject(await Expression.EvalAsync(
                    "G:=horizontalbars(" +
                    "[foreach Label in PerMethod.Keys: Left(Label,50)]," +
                    "[foreach Value in PerMethod.Values: Value/Divisor]);" +
                    "G.Title:=Title;" +
                    "G.LabelX:=LabelX;" +
                    "G.LabelY:=LabelY;" +
                    "G",
                    new Variables()
                    {
                        { Graph.GraphWidthVariableName, 1280 },
                        { Graph.GraphHeightVariableName, Height * Scale },
                        { Graph.GraphLabelFontSizeVariableName, 18 * Scale },
                        { "Divisor", Divisor },
                        { "PerMethod", PerMethod },
                        { "Title", await Language.GetStringAsync(typeof(HttpStatistics), 30, "Calls per Method") },
                        { "LabelX", Prefix + await Language.GetStringAsync(typeof(HttpStatistics), 31, "Calls") },
                        { "LabelY", await Language.GetStringAsync(typeof(HttpStatistics), 32, "Method") }
                    }));
 
                Height = 100 + 20 * PerUserAgent.Count;
                Scale = Height > 8000 ? Scale = 8000 / Height : 1;
                Max = CalcMax(PerUserAgent);
                Divisor = 1;
                Prefix = string.Empty;
                ScaleAmounts(Max, 1000, ref Divisor, ref Prefix);
 
                await Query.SetStatus(await Language.GetStringAsync(typeof(HttpStatistics), 23, "Preparing User-Agent graph."));
                await Query.NewObject(await Expression.EvalAsync(
                    "G:=horizontalbars(" +
                    "[foreach Label in PerUserAgent.Keys: Left(Label,50)]," +
                    "[foreach Value in PerUserAgent.Values: Value/Divisor]);" +
                    "G.Title:=Title;" +
                    "G.LabelX:=LabelX;" +
                    "G.LabelY:=LabelY;" +
                    "G",
                    new Variables()
                    {
                        { Graph.GraphWidthVariableName, 1280 },
                        { Graph.GraphHeightVariableName, Height * Scale },
                        { Graph.GraphLabelFontSizeVariableName, 18 * Scale },
                        { "Divisor", Divisor },
                        { "PerUserAgent", PerUserAgent },
                        { "Title", await Language.GetStringAsync(typeof(HttpStatistics), 33, "Calls per User-Agent") },
                        { "LabelX", Prefix + await Language.GetStringAsync(typeof(HttpStatistics), 31, "Calls") },
                        { "LabelY", "User-Agent" }
                    }));
 
                Height = 100 + 20 * PerFrom.Count;
                Scale = Height > 8000 ? Scale = 8000 / Height : 1;
                Max = CalcMax(PerFrom);
                Divisor = 1;
                Prefix = string.Empty;
                ScaleAmounts(Max, 1000, ref Divisor, ref Prefix);
 
                await Query.SetStatus(await Language.GetStringAsync(typeof(HttpStatistics), 24, "Preparing From graph."));
                await Query.NewObject(await Expression.EvalAsync(
                    "G:=horizontalbars(" +
                    "[foreach Label in PerFrom.Keys: Left(Label,50)]," +
                    "[foreach Value in PerFrom.Values: Value/Divisor]);" +
                    "G.Title:=Title;" +
                    "G.LabelX:=LabelX;" +
                    "G.LabelY:=LabelY;" +
                    "G",
                    new Variables()
                    {
                        { Graph.GraphWidthVariableName, 1280 },
                        { Graph.GraphHeightVariableName, Height * Scale },
                        { Graph.GraphLabelFontSizeVariableName, 18 * Scale },
                        { "Divisor", Divisor },
                        { "PerFrom", PerFrom },
                        { "Title", await Language.GetStringAsync(typeof(HttpStatistics), 34, "Calls per From") },
                        { "LabelX", Prefix + await Language.GetStringAsync(typeof(HttpStatistics), 31, "Calls") },
                        { "LabelY", "From" }
                    }));
 
                Height = 100 + 20 * PerResource.Count;
                Scale = Height > 8000 ? Scale = 8000 / Height : 1;
                Max = CalcMax(PerResource);
                Divisor = 1;
                Prefix = string.Empty;
                ScaleAmounts(Max, 1000, ref Divisor, ref Prefix);
 
                await Query.SetStatus(await Language.GetStringAsync(typeof(HttpStatistics), 25, "Preparing resource graph."));
                await Query.NewObject(await Expression.EvalAsync(
                    "G:=horizontalbars(" +
                    "[foreach Label in PerResource.Keys: Left(Label,50)]," +
                    "[foreach Value in PerResource.Values: Value/Divisor]);" +
                    "G.Title:=Title;" +
                    "G.LabelX:=LabelX;" +
                    "G.LabelY:=LabelY;" +
                    "G",
                    new Variables()
                    {
                        { Graph.GraphWidthVariableName, 1280 },
                        { Graph.GraphHeightVariableName, Height * Scale },
                        { Graph.GraphLabelFontSizeVariableName, 18 * Scale },
                        { "Divisor", Divisor },
                        { "PerResource", PerResource },
                        { "Title", await Language.GetStringAsync(typeof(HttpStatistics), 35, "Calls per Resource") },
                        { "LabelX", Prefix + await Language.GetStringAsync(typeof(HttpStatistics), 31, "Calls") },
                        { "LabelY", await Language.GetStringAsync(typeof(HttpStatistics), 36, "Resource") }
                    }));
 
                await Query.SetStatus(string.Empty);
                await Query.EndSection();
            }
            catch (Exception ex)
            {
                await Query.LogMessage(ex);
            }
            finally
            {
                await Query.Done();
            }
        }
 
        private static double CalcMax(SortedDictionary<string, double> Accumulated)
        {
            double Result = 0;
 
            foreach (double d in Accumulated.Values)
            {
                if (d > Result)
                    Result = d;
            }
 
            return Result;
        }
 
        private delegate string ReduceCallback(string s);
 
        private static void Accumulate(SortedDictionary<string, double> Accumulated, PersistedData.Commands.Statistic[] Statistics, ReduceCallback ReduceEndpoint)
        {
            if (!(Statistics is null))
            {
                foreach (PersistedData.Commands.Statistic Statistic in Statistics)
                {
                    string s = Statistic.Name;
 
                    if (!(ReduceEndpoint is null))
                        s = ReduceEndpoint(s);
 
                    if (!Accumulated.TryGetValue(s, out double Count))
                        Count = 0;
 
                    Accumulated[s] = Count + Statistic.Count;
                }
            }
        }
 
        private static void ScaleAmounts(double Value, double Thousand, ref double Divisor, ref string Prefix)
        {
            while (Value > 2000)
            {
                switch (Prefix)
                {
                    case "":
                        Prefix = "k";
                        break;
 
                    case "k":
                        Prefix = "M";
                        break;
 
                    case "M":
                        Prefix = "G";
                        break;
 
                    case "G":
                        Prefix = "T";
                        break;
 
                    case "T":
                        Prefix = "P";
                        break;
 
                    case "P":
                        Prefix = "E";
                        break;
 
                    case "E":
                        Prefix = "Z";
                        break;
 
                    case "Z":
                        Prefix = "Y";
                        break;
 
                    default:
                        return;
                }
 
                Divisor *= Thousand;
                Value /= Thousand;
            }
        }
    }
}