WebGetter.cs

using System;
using System.Collections.Generic;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Net.Security;
using System.Runtime.ExceptionServices;
using System.Security.Authentication;
using System.Security.Cryptography.X509Certificates;
using System.Threading.Tasks;
using Waher.Runtime.Inventory;
using Waher.Runtime.Temporary;
 
namespace Waher.Content.Getters
{
    public class WebGetter : IContentGetter, IContentHeader
    {
        public WebGetter()
        {
        }
 
        public string[] UriSchemes => new string[] { "http", "https" };
 
        public bool CanGet(Uri Uri, out Grade Grade)
        {
            switch (Uri.Scheme.ToLower())
            {
                case "http":
                case "https":
                    Grade = Grade.Ok;
                    return true;
 
                default:
                    Grade = Grade.NotAtAll;
                    return false;
            }
        }
 
        public Task<object> GetAsync(Uri Uri, X509Certificate Certificate,
            RemoteCertificateEventHandler RemoteCertificateValidator, 
            params KeyValuePair<string, string>[] Headers)
        {
            return this.GetAsync(Uri, Certificate, RemoteCertificateValidator, 60000, Headers);
        }
 
        public async Task<object> GetAsync(Uri Uri, X509Certificate Certificate,
            RemoteCertificateEventHandler RemoteCertificateValidator, 
            int TimeoutMs, params KeyValuePair<string, string>[] Headers)
        {
            HttpClientHandler Handler = GetClientHandler(Certificate, RemoteCertificateValidator);
            using (HttpClient HttpClient = new HttpClient(Handler, true)
            {
                Timeout = TimeSpan.FromMilliseconds(TimeoutMs)
            })
            {
                using (HttpRequestMessage Request = new HttpRequestMessage()
                {
                    RequestUri = Uri,
                    Method = HttpMethod.Get
                })
                {
                    PrepareHeaders(Request, Headers, Handler);
 
                    HttpResponseMessage Response = await HttpClient.SendAsync(Request);
 
                    return await ProcessResponse(Response, Uri);
                }
            }
        }
 
        public static HttpClientHandler GetClientHandler()
        {
            return GetClientHandler(null, null);
        }
 
        public static HttpClientHandler GetClientHandler(X509Certificate Certificate,
            RemoteCertificateEventHandler RemoteCertificateValidator)
        {
            RemoteCertificateValidator Validator = new RemoteCertificateValidator(RemoteCertificateValidator);
 
            HttpClientHandler Handler = new HttpClientHandler()
            {
                AllowAutoRedirect = true,
                CheckCertificateRevocationList = true,
                ClientCertificateOptions = ClientCertificateOption.Automatic,
                ServerCertificateCustomValidationCallback = Validator.RemoteCertificateValidationCallback,
                AutomaticDecompression = (DecompressionMethods)(-1)     // All
            };
 
            try
            {
                Handler.SslProtocols = SslProtocols.Tls | SslProtocols.Tls11 | SslProtocols.Tls12;
            }
            catch (PlatformNotSupportedException)
            {
                // Ignore
            }
 
            if (!(Certificate is null))
            {
                Handler.ClientCertificateOptions = ClientCertificateOption.Manual;
                Handler.ClientCertificates.Add(Certificate);
            }
 
            return Handler;
        }
 
        private class RemoteCertificateValidator
        {
            private readonly RemoteCertificateEventHandler callback;
 
            public RemoteCertificateValidator(RemoteCertificateEventHandler Callback)
            {
                this.callback = Callback;
            }
 
            public bool RemoteCertificateValidationCallback(object Sender,
                X509Certificate Certificate, X509Chain Chain, SslPolicyErrors SslPolicyErrors)
            {
                RemoteCertificateEventArgs e = new RemoteCertificateEventArgs(Certificate, Chain, SslPolicyErrors);
 
                if (!(this.callback is null))
                    this.callback(Sender, e);
 
                if (e.IsValid.HasValue)
                    return e.IsValid.Value;
                else
                    return SslPolicyErrors == SslPolicyErrors.None;
            }
        }
 
        public static async Task<object> ProcessResponse(HttpResponseMessage Response, Uri Uri)
        {
            byte[] Bin = await Response.Content.ReadAsByteArrayAsync();
            string ContentType;
            object Decoded;
 
            if (Bin.Length == 0 || Response.Content.Headers.ContentType is null)
            {
                Decoded = Bin;
                ContentType = string.Empty;
            }
            else
            {
                ContentType = Response.Content.Headers.ContentType.ToString();
                Decoded = await InternetContent.DecodeAsync(ContentType, Bin, Uri);
            }
 
            if (Decoded is IWebServerMetaContent WebServerMetaContent)
                await WebServerMetaContent.DecodeMetaInformation(Response);
 
            if (!Response.IsSuccessStatusCode)
            {
                if (!(Decoded is string Message))
                {
                    if (Decoded is null ||
                        (Decoded is byte[] Bin2 && Bin2.Length == 0) ||
                        Decoded is Dictionary<string, object>)
                    {
                        Message = Response.ReasonPhrase;
                    }
                    else
                        Message = Decoded.ToString();
                }
 
                throw new WebException(Message, Response.StatusCode, ContentType, Bin, Decoded, Response.Headers);
            }
 
            return Decoded;
        }
 
        internal static void PrepareHeaders(HttpRequestMessage Request, KeyValuePair<string, string>[] Headers, HttpClientHandler Handler)
        {
            if (!(Headers is null))
            {
                foreach (KeyValuePair<string, string> Header in Headers)
                {
                    switch (Header.Key)
                    {
                        case "Accept":
                            if (!Request.Headers.Accept.TryParseAdd(Header.Value))
                                throw new InvalidOperationException("Invalid Accept header value: " + Header.Value);
                            break;
 
                        case "Authorization":
                            int i = Header.Value.IndexOf(' ');
                            if (i < 0)
                                Request.Headers.Authorization = new AuthenticationHeaderValue(Header.Value);
                            else
                                Request.Headers.Authorization = new AuthenticationHeaderValue(Header.Value.Substring(0, i), Header.Value.Substring(i + 1).TrimStart());
                            break;
 
                        case "Cookie":
                            foreach (KeyValuePair<string, string> P in CommonTypes.ParseFieldValues(Header.Value))
                                Handler.CookieContainer.Add(Request.RequestUri, new Cookie(P.Key, P.Value));
                            break;
 
                        default:
                            Request.Headers.Add(Header.Key, Header.Value);
                            break;
                    }
                }
            }
        }
 
        public Task<KeyValuePair<string, TemporaryStream>> GetTempStreamAsync(Uri Uri, X509Certificate Certificate,
            RemoteCertificateEventHandler RemoteCertificateValidator, params KeyValuePair<string, string>[] Headers)
        {
            return this.GetTempStreamAsync(Uri, Certificate, RemoteCertificateValidator, 60000, Headers);
        }
 
        public async Task<KeyValuePair<string, TemporaryStream>> GetTempStreamAsync(Uri Uri, X509Certificate Certificate,
            RemoteCertificateEventHandler RemoteCertificateValidator, int TimeoutMs, params KeyValuePair<string, string>[] Headers)
        {
            HttpClientHandler Handler = GetClientHandler(Certificate, RemoteCertificateValidator);
            using (HttpClient HttpClient = new HttpClient(Handler, true)
            {
                Timeout = TimeSpan.FromMilliseconds(10000)
            })
            {
                using (HttpRequestMessage Request = new HttpRequestMessage()
                {
                    RequestUri = Uri,
                    Method = HttpMethod.Get
                })
                {
                    PrepareHeaders(Request, Headers, Handler);
 
                    HttpResponseMessage Response = await HttpClient.SendAsync(Request);
 
                    if (!Response.IsSuccessStatusCode)
                        await ProcessResponse(Response, Uri);
 
                    string ContentType = Response.Content.Headers.ContentType.ToString();
                    TemporaryStream File = new TemporaryStream();
                    try
                    {
                        await Response.Content.CopyToAsync(File);
                    }
                    catch (Exception ex)
                    {
                        File.Dispose();
                        File = null;
 
                        ExceptionDispatchInfo.Capture(ex).Throw();
                    }
 
                    return new KeyValuePair<string, TemporaryStream>(ContentType, File);
                }
            }
        }
 
        public bool CanHead(Uri Uri, out Grade Grade)
        {
            return this.CanGet(Uri, out Grade);
        }
 
        public Task<object> HeadAsync(Uri Uri, X509Certificate Certificate,
            RemoteCertificateEventHandler RemoteCertificateValidator, 
            params KeyValuePair<string, string>[] Headers)
        {
            return this.HeadAsync(Uri, Certificate, RemoteCertificateValidator, 60000, Headers);
        }
 
        public async Task<object> HeadAsync(Uri Uri, X509Certificate Certificate,
            RemoteCertificateEventHandler RemoteCertificateValidator, 
            int TimeoutMs, params KeyValuePair<string, string>[] Headers)
        {
            HttpClientHandler Handler = GetClientHandler(Certificate, RemoteCertificateValidator);
            using (HttpClient HttpClient = new HttpClient(Handler, true)
            {
                Timeout = TimeSpan.FromMilliseconds(TimeoutMs)
            })
            {
                using (HttpRequestMessage Request = new HttpRequestMessage()
                {
                    RequestUri = Uri,
                    Method = HttpMethod.Head
                })
                {
                    PrepareHeaders(Request, Headers, Handler);
 
                    HttpResponseMessage Response = await HttpClient.SendAsync(Request);
                    Dictionary<string, object> Result = new Dictionary<string, object>()
                    {
                        { "Status", Response.StatusCode },
                        { "StatusCode", (int)Response.StatusCode },
                        { "Message", Response.ReasonPhrase },
                        { "IsSuccessStatusCode", Response.IsSuccessStatusCode },
                        { "Version", Response.Version.ToString() }
                    };
 
                    foreach (KeyValuePair<string, IEnumerable<string>> Header in Response.Headers)
                    {
                        string s = null;
                        List<string> List = null;
 
                        foreach (string Value in Header.Value)
                        {
                            if (s is null)
                                s = Value;
                            else
                            {
                                if (List is null)
                                    List = new List<string>() { s };
 
                                List.Add(Value);
                            }
                        }
 
                        if (List is null)
                            Result[Header.Key] = s;
                        else
                            Result[Header.Key] = List.ToArray();
                    }
 
                    return Result;
                }
            }
        }
 
    }
}