WebGetter.cs

using System;
using System.Collections.Generic;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Net.Security;
using System.Reflection;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Threading.Tasks;
using Waher.Content.Binary;
using Waher.Events;
using Waher.Runtime.Collections;
using Waher.Runtime.Inventory;
using Waher.Runtime.IO;
using Waher.Runtime.Temporary;
using Waher.Security;
 
namespace Waher.Content.Getters
{
    public class WebGetter : IContentGetter, IContentHeader
    {
        private static bool useProxy = true;
        private static bool enforceHttps = false;
 
        public WebGetter()
        {
        }
 
        public string[] UriSchemes => new string[] { "http", "https" };
 
        public bool CanGet(Uri Uri, out Grade Grade)
        {
            switch (Uri.Scheme.ToLower())
            {
                case "http":
                case "https":
                    Grade = Grade.Ok;
                    return true;
 
                default:
                    Grade = Grade.NotAtAll;
                    return false;
            }
        }
 
        public Task<ContentResponse> GetAsync(Uri Uri, X509Certificate Certificate,
            EventHandler<RemoteCertificateEventArgs> RemoteCertificateValidator,
            params KeyValuePair<string, string>[] Headers)
        {
            return this.GetAsync(Uri, Certificate, RemoteCertificateValidator, InternetContent.DefaultTimeout, Headers);
        }
 
        public async Task<ContentResponse> GetAsync(Uri Uri, X509Certificate Certificate,
            EventHandler<RemoteCertificateEventArgs> RemoteCertificateValidator,
            int TimeoutMs, params KeyValuePair<string, string>[] Headers)
        {
            HttpClientHandler Handler = GetClientHandler(Certificate, RemoteCertificateValidator, Uri);
            using (HttpClient HttpClient = new HttpClient(Handler, true)
            {
                Timeout = TimeSpan.FromMilliseconds(TimeoutMs)
            })
            {
                using (HttpRequestMessage Request = new HttpRequestMessage()
                {
                    Method = HttpMethod.Get
                })
                {
                    Request.RequestUri = CheckUri(Uri, Request);
                    PrepareHeaders(Request, Headers, Handler);
 
                    HttpResponseMessage Response = await HttpClient.SendAsync(Request);
 
                    return await ProcessResponse(Response, Uri);
                }
            }
        }
 
        public static bool TrustServer(Uri Uri)
        {
            if (Uri.Host != "localhost")
                return false;
 
            if (Uri.Scheme != "https")
                return false;
 
            if (!Types.TryGetModuleParameter("HTTP", out object Obj) || Obj is null)
                return false;
 
            int Port = Uri.Port;
            if (Port <= 0)
                Port = 443;
 
            Type T = Obj.GetType();
            PropertyInfo PI = T.GetProperty("OpenHttpsPorts");
            if (PI is null || !PI.CanRead || !PI.GetMethod.IsPublic)
                return false;
 
            Obj = PI.GetValue(Obj);
            if (!(Obj is int[] OpenHttpsPorts))
                return false;
 
            foreach (int OpenPort in OpenHttpsPorts)
            {
                if (OpenPort == Port)
                    return true;
            }
 
            return false;
        }
 
        public static HttpClientHandler GetClientHandler()
        {
            return GetClientHandler(null, null, false);
        }
 
        public static HttpClientHandler GetClientHandler(bool TrustServer)
        {
            return GetClientHandler(null, null, TrustServer);
        }
 
        public static HttpClientHandler GetClientHandler(X509Certificate Certificate,
            EventHandler<RemoteCertificateEventArgs> RemoteCertificateValidator)
        {
            return GetClientHandler(Certificate, RemoteCertificateValidator, false);
        }
 
        public static HttpClientHandler GetClientHandler(X509Certificate Certificate,
            EventHandler<RemoteCertificateEventArgs> RemoteCertificateValidator, Uri Uri)
        {
            return GetClientHandler(Certificate, RemoteCertificateValidator, TrustServer(Uri));
        }
 
        public static HttpClientHandler GetClientHandler(X509Certificate Certificate,
            EventHandler<RemoteCertificateEventArgs> RemoteCertificateValidator,
            bool TrustServer)
        {
            RemoteCertificateValidator Validator = new RemoteCertificateValidator(RemoteCertificateValidator, TrustServer);
 
            HttpClientHandler Handler = new HttpClientHandler()
            {
                AllowAutoRedirect = true,
                CheckCertificateRevocationList = true,
                ClientCertificateOptions = ClientCertificateOption.Automatic,
                ServerCertificateCustomValidationCallback = Validator.RemoteCertificateValidationCallback,
                AutomaticDecompression = (DecompressionMethods)(-1),    // All
                UseProxy = useProxy
            };
 
            try
            {
                Handler.SslProtocols = Crypto.TlsOnly;
            }
            catch (PlatformNotSupportedException)
            {
                // Ignore
            }
 
            if (!(Certificate is null))
            {
                Handler.ClientCertificateOptions = ClientCertificateOption.Manual;
                Handler.ClientCertificates.Add(Certificate);
            }
 
            return Handler;
        }
 
        public static bool UseProxy
        {
            get => useProxy;
            set => useProxy = value;
        }
 
        public static bool EnforceHttps
        {
            get => enforceHttps;
            set => enforceHttps = value;
        }
 
        public static Uri CheckUri(Uri Uri, HttpRequestMessage Request)
        {
            EventHandler<HttpUriEventArgs> h = HttpUriEventHandler;
 
            if (h is null && !enforceHttps)
                return Uri;
 
            if (Uri.Scheme.ToLower() != "http")
                return Uri;
 
            if (!(h is null))
            {
                HttpUriEventArgs e = new HttpUriEventArgs(Uri, Request);
                
                h.Raise(null, e);
 
                if (e.Uri != Uri)
                    return e.Uri;
                else if (!enforceHttps)
                    return Uri;
            }
 
            if (Uri.Authority.IndexOf(':') >= 0)
                return Uri;
 
            if (Uri.TryCreate(Uri.OriginalString.Replace("http:", "https:"), UriKind.Absolute, out Uri Uri2))
                return Uri2;
            else
                return Uri;
        }
 
        public static event EventHandler<HttpUriEventArgs> HttpUriEventHandler;
 
        private class RemoteCertificateValidator
        {
            private readonly EventHandler<RemoteCertificateEventArgs> callback;
            private readonly bool trustServer;
 
            public RemoteCertificateValidator(EventHandler<RemoteCertificateEventArgs> Callback)
                : this(Callback, false)
            {
            }
 
            public RemoteCertificateValidator(EventHandler<RemoteCertificateEventArgs> Callback, 
                bool TrustServer)
            {
                this.callback = Callback;
                this.trustServer = TrustServer;
            }
 
            public bool RemoteCertificateValidationCallback(object Sender,
                X509Certificate Certificate, X509Chain Chain, SslPolicyErrors SslPolicyErrors)
            {
                if (!(this.callback is null))
                {
                    RemoteCertificateEventArgs e = new RemoteCertificateEventArgs(Certificate, Chain, SslPolicyErrors);
 
                    this.callback.Raise(Sender, e);
 
                    if (e.IsValid.HasValue)
                        return e.IsValid.Value;
                }
 
                if (this.trustServer)
                    return true;
                else if (SslPolicyErrors == SslPolicyErrors.None)
                    return true;
                else
                {
                    // Check for incomplete revocation check in the chain
 
                    if (SslPolicyErrors.HasFlag(SslPolicyErrors.RemoteCertificateChainErrors) && !(Chain is null))
                    {
                        foreach (X509ChainStatus Status in Chain.ChainStatus)
                        {
                            // Apple-specific error code for incomplete revocation check
 
                            if (Status.Status == X509ChainStatusFlags.RevocationStatusUnknown ||
                                Status.Status == X509ChainStatusFlags.OfflineRevocation)
                            {
                                continue; // Ignore this error
                            }
 
                            if (Status.Status != X509ChainStatusFlags.NoError)
                            {
                                if (Certificate is X509Certificate2 Certificate2)
                                    return Certificate2.Verify();
 
                                Certificate2 = new X509Certificate2(Certificate.GetRawCertData());
 
                                return Certificate2.Verify(); // Check if certificate fails on other errors
                            }
                        }
 
                        return true; // Only revocation check failed, allow
                    }
 
                    return false;
                }
            }
        }
 
        public static async Task<ContentResponse> ProcessResponse(HttpResponseMessage Response, Uri Uri)
        {
            byte[] Bin = await Response.Content.ReadAsByteArrayAsync();
            string ContentType;
            ContentResponse Decoded;
 
            if (Response.Content.Headers.ContentType is null)
            {
                if (Bin.Length == 0)
                    Decoded = new ContentResponse(string.Empty, null, Bin);
                else
                    Decoded = new ContentResponse(BinaryCodec.DefaultContentType, Bin, Bin);
            }
            else
            {
                ContentType = Response.Content.Headers.ContentType.ToString();
                Decoded = await InternetContent.DecodeAsync(ContentType, Bin, Uri);
                if (Decoded.HasError)
                    return Decoded;
            }
 
            if (Decoded.Decoded is IWebServerMetaContent WebServerMetaContent)
                await WebServerMetaContent.DecodeMetaInformation(Response);
 
            if (!Response.IsSuccessStatusCode)
            {
                if (!(Decoded.Decoded is string Message))
                {
                    if (Decoded.Decoded is null ||
                        Decoded.Decoded is Dictionary<string, object>)
                    {
                        Message = Response.ReasonPhrase;
                    }
                    else if (Decoded.Decoded is byte[] Bin2)
                    {
                        if (Bin2.Length == 0)
                            Message = Response.ReasonPhrase;
                        else
                            Message = Strings.GetString(Bin2, Encoding.UTF8);
                    }
                    else
                    {
                        Message = Decoded.ToString();
                        if (Message == Decoded.GetType().FullName)
                            Message = Response.ReasonPhrase;
                    }
                }
 
                Decoded = new ContentResponse(new WebException(Message, Response.StatusCode,
                    Decoded.ContentType, Bin, Decoded.Decoded, Response.Headers));
            }
 
            return Decoded;
        }
 
        public static void PrepareHeaders(HttpRequestMessage Request, KeyValuePair<string, string>[] Headers, HttpClientHandler Handler)
        {
            if (!(Headers is null))
            {
                foreach (KeyValuePair<string, string> Header in Headers)
                {
                    switch (Header.Key)
                    {
                        case "Accept":
                            if (!Request.Headers.Accept.TryParseAdd(Header.Value))
                                throw new InvalidOperationException("Invalid Accept header value: " + Header.Value);
                            break;
 
                        case "Authorization":
                            int i = Header.Value.IndexOf(' ');
                            if (i < 0)
                                Request.Headers.Authorization = new AuthenticationHeaderValue(Header.Value);
                            else
                                Request.Headers.Authorization = new AuthenticationHeaderValue(Header.Value.Substring(0, i), Header.Value.Substring(i + 1).TrimStart());
                            break;
 
                        case "Cookie":
                            foreach (KeyValuePair<string, string> P in CommonTypes.ParseFieldValues(Header.Value))
                                Handler.CookieContainer.Add(Request.RequestUri, new Cookie(P.Key, P.Value));
                            break;
 
                        default:
                            Request.Headers.Add(Header.Key, Header.Value);
                            break;
                    }
                }
            }
        }
 
        public Task<ContentStreamResponse> GetTempStreamAsync(Uri Uri, X509Certificate Certificate,
            EventHandler<RemoteCertificateEventArgs> RemoteCertificateValidator, params KeyValuePair<string, string>[] Headers)
        {
            return this.GetTempStreamAsync(Uri, Certificate, RemoteCertificateValidator, InternetContent.DefaultTimeout, Headers);
        }
 
        public Task<ContentStreamResponse> GetTempStreamAsync(Uri Uri, X509Certificate Certificate,
            EventHandler<RemoteCertificateEventArgs> RemoteCertificateValidator, TemporaryStream Destination, params KeyValuePair<string, string>[] Headers)
        {
            return this.GetTempStreamAsync(Uri, Certificate, RemoteCertificateValidator, InternetContent.DefaultTimeout, Destination, Headers);
        }
 
        public Task<ContentStreamResponse> GetTempStreamAsync(Uri Uri, X509Certificate Certificate,
            EventHandler<RemoteCertificateEventArgs> RemoteCertificateValidator, int TimeoutMs, params KeyValuePair<string, string>[] Headers)
        {
            return this.GetTempStreamAsync(Uri, Certificate, RemoteCertificateValidator, TimeoutMs, null, Headers);
        }
 
        public async Task<ContentStreamResponse> GetTempStreamAsync(Uri Uri, X509Certificate Certificate,
            EventHandler<RemoteCertificateEventArgs> RemoteCertificateValidator, int TimeoutMs, TemporaryStream Destination,
            params KeyValuePair<string, string>[] Headers)
        {
            HttpClientHandler Handler = GetClientHandler(Certificate, RemoteCertificateValidator, Uri);
            using (HttpClient HttpClient = new HttpClient(Handler, true)
            {
                Timeout = TimeSpan.FromMilliseconds(10000)
            })
            {
                using (HttpRequestMessage Request = new HttpRequestMessage()
                {
                    Method = HttpMethod.Get
                })
                {
                    Request.RequestUri = CheckUri(Uri, Request);
                    PrepareHeaders(Request, Headers, Handler);
 
                    HttpResponseMessage Response = await HttpClient.SendAsync(Request, HttpCompletionOption.ResponseHeadersRead);
 
                    if (!Response.IsSuccessStatusCode)
                    {
                        ContentResponse Temp = await ProcessResponse(Response, Uri);
                        return new ContentStreamResponse(Temp.Error);
                    }
 
                    string ContentType = Response.Content.Headers.ContentType.ToString();
                    bool TempStreamCreated = false;
 
                    if (Destination is null)
                    {
                        Destination = new TemporaryStream();
                        TempStreamCreated = true;
                    }
 
                    try
                    {
                        await Response.Content.CopyToAsync(Destination);
                    }
                    catch (Exception ex)
                    {
                        if (TempStreamCreated)
                        {
                            Destination.Dispose();
                            Destination = null;
                        }
 
                        return new ContentStreamResponse(ex);
                    }
 
                    return new ContentStreamResponse(ContentType, Destination);
                }
            }
        }
 
        public bool CanHead(Uri Uri, out Grade Grade)
        {
            return this.CanGet(Uri, out Grade);
        }
 
        public Task<ContentResponse> HeadAsync(Uri Uri, X509Certificate Certificate,
            EventHandler<RemoteCertificateEventArgs> RemoteCertificateValidator,
            params KeyValuePair<string, string>[] Headers)
        {
            return this.HeadAsync(Uri, Certificate, RemoteCertificateValidator, InternetContent.DefaultTimeout, Headers);
        }
 
        public async Task<ContentResponse> HeadAsync(Uri Uri, X509Certificate Certificate,
            EventHandler<RemoteCertificateEventArgs> RemoteCertificateValidator,
            int TimeoutMs, params KeyValuePair<string, string>[] Headers)
        {
            HttpClientHandler Handler = GetClientHandler(Certificate, RemoteCertificateValidator, Uri);
            using (HttpClient HttpClient = new HttpClient(Handler, true)
            {
                Timeout = TimeSpan.FromMilliseconds(TimeoutMs)
            })
            {
                using (HttpRequestMessage Request = new HttpRequestMessage()
                {
                    Method = HttpMethod.Head
                })
                {
                    Request.RequestUri = CheckUri(Uri, Request);
                    PrepareHeaders(Request, Headers, Handler);
 
                    HttpResponseMessage Response = await HttpClient.SendAsync(Request);
                    Dictionary<string, object> Result = new Dictionary<string, object>()
                    {
                        { "Status", Response.StatusCode },
                        { "StatusCode", (int)Response.StatusCode },
                        { "Message", Response.ReasonPhrase },
                        { "IsSuccessStatusCode", Response.IsSuccessStatusCode },
                        { "Version", Response.Version.ToString() }
                    };
 
                    foreach (KeyValuePair<string, IEnumerable<string>> Header in Response.Headers)
                    {
                        string s = null;
                        ChunkedList<string> List = null;
 
                        foreach (string Value in Header.Value)
                        {
                            if (s is null)
                                s = Value;
                            else
                            {
                                if (List is null)
                                    List = new ChunkedList<string>() { s };
 
                                List.Add(Value);
                            }
                        }
 
                        if (List is null)
                            Result[Header.Key] = s;
                        else
                            Result[Header.Key] = List.ToArray();
                    }
 
                    return new ContentResponse(Response.Content?.Headers.ContentType?.ToString() ?? string.Empty, Result, null);
                }
            }
        }
 
    }
}