HttpFileUploadComponent.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using System.Web;
using Waher.Content;
using Waher.Content.Xml;
using Waher.Events;
using Waher.Networking.HTTP;
using Waher.Persistence;
using Waher.Runtime.Cache;
using Waher.Runtime.IO;
 
namespace Waher.Networking.XMPP.Server.HttpFileUpload
{
    public class HttpFileUploadComponent : Component
    {
        private static readonly char[] invalidCharacters = Path.GetInvalidFileNameChars();
 
        private readonly HttpFileUploadSettings settings;
        private Cache<CaseInsensitiveString, UploadInformation> statusByClient;
        private Cache<CaseInsensitiveString, FileInfo> files;
 
        public const string Namespace = "urn:xmpp:http:upload:0";
 
        public const string BackupNamespace = "http://waher.se/Schema/Backups.xsd";
 
        public const string EncryptedStorageNamespace = "http://waher.se/Schema/EncryptedStorage.xsd";
 
        public const string PubSubNamespace = "http://waher.se/Schema/PubSub.xsd";
 
        public const string InternalTransferNamespace = "http://waher.se/Schema/InternalTransfer.xsd";
 
        public HttpFileUploadComponent(XmppServer Server, CaseInsensitiveString Subdomain, HttpFileUploadSettings Settings)
            : base(Server, Subdomain, "HTTP File Upload")
        {
            this.settings = Settings;
 
            this.DeleteUploadedFiles();
 
            if (!Directory.Exists(Settings.FileFolder))
                Directory.CreateDirectory(Settings.FileFolder);
 
            this.statusByClient = new Cache<CaseInsensitiveString, UploadInformation>(int.MaxValue, TimeSpan.MaxValue, this.settings.FileLifetime, true);
            this.files = new Cache<CaseInsensitiveString, FileInfo>(int.MaxValue, TimeSpan.MaxValue, this.settings.FileLifetime, true);
 
            this.files.Removed += this.Files_Removed;
 
            this.RegisterIqGetHandler("request", Namespace, this.RequestHandler, true);
            this.RegisterIqSetHandler("prepare", BackupNamespace, this.PrepareBackupHandler, true);
            this.RegisterIqSetHandler("prepare", EncryptedStorageNamespace, this.PrepareEncryptedStorageHandler, true);
            this.RegisterIqSetHandler("prepare", PubSubNamespace, this.PreparePubSubStorageHandler, true);
            this.RegisterIqSetHandler("prepare", InternalTransferNamespace, this.PrepareInternalTransferHandler, true);
        }
 
        private void DeleteUploadedFiles()
        {
            if (Directory.Exists(this.settings.FileFolder))
            {
                try
                {
                    Directory.Delete(this.settings.FileFolder, true);
                }
                catch (Exception ex)
                {
                    Log.Exception(ex);
                }
            }
        }
 
        public override void Dispose()
        {
            base.Dispose();
 
            this.UnregisterIqGetHandler("request", Namespace, this.RequestHandler, true);
            this.UnregisterIqSetHandler("prepare", BackupNamespace, this.PrepareBackupHandler, true);
            this.UnregisterIqSetHandler("prepare", EncryptedStorageNamespace, this.PrepareEncryptedStorageHandler, true);
            this.UnregisterIqSetHandler("prepare", PubSubNamespace, this.PreparePubSubStorageHandler, true);
            this.UnregisterIqSetHandler("prepare", InternalTransferNamespace, this.PrepareInternalTransferHandler, true);
 
            this.statusByClient?.Dispose();
            this.statusByClient = null;
 
            this.files?.Dispose();
            this.files = null;
 
            this.DeleteUploadedFiles();
        }
 
        public override bool SupportsAccounts => false;
 
        protected override Task AppendServiceDiscoveryIdentities(StringBuilder Xml, IqEventArgs e, string Node)
        {
            Xml.Append("<identity category='store' type='file' name='HTTP File Upload' />");
 
            return Task.CompletedTask;
        }
 
        protected override Task AppendServiceDiscoveryFeatures(StringBuilder Xml, IqEventArgs e, string Node)
        {
            Xml.Append("<x type='result' xmlns='jabber:x:data'>");
            Xml.Append("<field var='FORM_TYPE' type='hidden'>");
            Xml.Append("<value>");
            Xml.Append(Namespace);
            Xml.Append("</value>");
            Xml.Append("</field>");
            Xml.Append("<field var='max-file-size'>");
            Xml.Append("<value>");
            Xml.Append(this.settings.MaxFileSize.ToString());
            Xml.Append("</value>");
            Xml.Append("</field>");
            Xml.Append("</x>");
 
            return Task.CompletedTask;
        }
 
        private async Task RequestHandler(object Sender, IqEventArgs e)
        {
            string FileName = XML.Attribute(e.Query, "filename");
            string ContentType = XML.Attribute(e.Query, "content-type");
            long Size = XML.Attribute(e.Query, "size", 0L);
 
            if (!await this.IsFileOk(FileName, ContentType, Size, e))
                return;
 
            if (!e.To.IsDomain)
            {
                await e.IqErrorBadRequest(e.To, "Invalid destination address.", "en");
                return;
            }
 
            string Extension = Path.GetExtension(FileName).ToLower();
            if (Extension.StartsWith('.'))
                Extension = Extension[1..];
 
            string BaseContentType = ContentType;
 
            InternetContent.ParseContentType(ref BaseContentType, out _, out _);
 
            if (InternetContent.TryGetContentType(Extension, out string ExtensionType))
            {
                if (InternetContent.Decodes(ExtensionType, out _, out IContentDecoder ExtensionDecoder) &&
                    InternetContent.Decodes(BaseContentType, out _, out IContentDecoder ContentDecoder) &&
                    ExtensionDecoder.GetType() != ContentDecoder.GetType())
                {
                    await e.IqErrorBadRequest(e.To, "File extension does not match Content-Type.", "en");
                    return;
                }
            }
            else if (InternetContent.Decodes(BaseContentType, out _, out IContentDecoder ContentDecoder) &&
                Array.IndexOf(ContentDecoder.FileExtensions, Extension) < 0)
            {
                await e.IqErrorBadRequest(e.To, "File extension does not match Content-Type.", "en");
                return;
            }
 
            CaseInsensitiveString BareJid = e.From.BareJid;
            FilePurpose Purpose;
 
            if (this.statusByClient.TryGetValue(BareJid, out UploadInformation Status) &&
                Status.TryGetSpecialFile(FileName, ContentType, Size, out SpecialFile SpecialFile))
            {
                Purpose = SpecialFile.Purpose;
            }
            else
                Purpose = FilePurpose.Temporary;
 
            if (Purpose == FilePurpose.Temporary || Purpose == FilePurpose.InternalTransfer)
            {
                int i = BareJid.IndexOf('@');
                if (i < 0)
                {
                    await e.IqErrorForbidden(e.To, "Only local clients are allowed to upload content.", "en");
                    return;
                }
 
                CaseInsensitiveString Domain = BareJid.Substring(i + 1);
                if (!this.Server.IsServerDomain(Domain, true))
                {
                    await e.IqErrorForbidden(e.To, "Only local clients are allowed to upload content.", "en");
                    return;
                }
 
                CaseInsensitiveString UserName = BareJid.Substring(0, i);
                IAccount Account = await this.Server.GetAccount(UserName);
                if (Account is null)
                {
                    await e.IqErrorForbidden(e.To, "Only local clients are allowed to upload content.", "en");
                    return;
                }
 
                if (Size > this.settings.MaxFileSize)
                {
                    await e.IqError("modify", "<not-acceptable xmlns='urn:ietf:params:xml:ns:xmpp-stanzas' />" +
                        "<file-too-large xmlns='urn:xmpp:http:upload:0'>" +
                        "<max-file-size>" + this.settings.MaxFileSize.ToString() + "</max-file-size>" +
                        "</file-too-large>", e.To, "File too large. The maximum file size is " + this.settings.MaxFileSize.ToString() +
                        " bytes.", "en");
                    return;
                }
            }
 
            if (!await this.Server.PersistenceLayer.IsPermitted(BareJid, "HTTP.Upload"))
            {
                await e.IqErrorNotAllowed(e.To, "You are not allowed to upload files.", "en");
                return;
            }
 
            if (Status is null)
            {
                Status = new UploadInformation(BareJid);
                this.statusByClient[BareJid] = Status;
            }
 
            switch (Status.CanUploadFile(Size, Purpose, this.settings))
            {
                case CanUploadResult.FileQuotaReached:
                    await e.IqErrorResourceConstraint(e.To, "Quota reached. You can only upload " +
                        this.settings.MaxFilesPerMinute.ToString() + " files per minute.", "en");
                    return;
 
                case CanUploadResult.ByteQuotaReached:
                    await e.IqErrorResourceConstraint(e.To, "Quota reached. You can only upload " +
                        this.settings.MaxBytesPerMinute.ToString() + " bytes per minute.", "en");
                    return;
 
                case CanUploadResult.Permitted:
                    StringBuilder Xml = new StringBuilder();
                    string Id = this.Server.NewId(32);
                    string Resource = "/" + Id + "/" + HttpUtility.UrlEncode(FileName);
                    string FilePath;
                    string PutUrl = this.settings.HttpFolder + Resource;
                    string GetUrl = PutUrl;
                    string Key = this.Server.NewId(32);
 
                    switch (Purpose)
                    {
                        case FilePurpose.Temporary:
                        default:
                            FilePath = Path.Combine(this.settings.FileFolder, BareJid, Id + ".bin");
                            break;
 
                        case FilePurpose.Backup:
                            if (FileName.EndsWith(".key", StringComparison.InvariantCultureIgnoreCase))
                                FilePath = Path.Combine(this.settings.KeyFolder, BareJid, FileName);
                            else
                                FilePath = Path.Combine(this.settings.BackupFolder, BareJid, FileName);
                            break;
 
                        case FilePurpose.Encrypted:
                            DateTime Today = DateTime.Today;
                            string EncryptedResource = Path.Combine(Today.Year.ToString("D4"),
                                Today.Month.ToString("D2"), Today.Day.ToString("D2"), Id + ".bin");
                            FilePath = Path.Combine(this.settings.EncryptedStorageFolder, EncryptedResource);
                            GetUrl = this.settings.EncryptedStorageRoot + "/" + EncryptedResource.Replace(Path.DirectorySeparatorChar, '/');
                            break;
 
                        case FilePurpose.PubSub:
                            Today = DateTime.Today;
                            string PubSubResource = Path.Combine(Today.Year.ToString("D4"),
                                Today.Month.ToString("D2"), Today.Day.ToString("D2"), FileName);
                            FilePath = Path.Combine(this.settings.PubSubStorageFolder, PubSubResource);
                            if (File.Exists(FilePath))
                            {
                                await e.IqErrorConflict(e.To, "File already exists.", "en");
                                return;
                            }
 
                            GetUrl = this.settings.PubSubStorageRoot + "/" + PubSubResource.Replace(Path.DirectorySeparatorChar, '/');
                            break;
 
                        case FilePurpose.InternalTransfer:
                            FilePath = Path.Combine(this.settings.InternalTransferFolder, Id + ".bin");
                            break;
                    }
 
                    this.files.Add(Resource, new FileInfo()
                    {
                        Jid = e.From.Address,
                        Resource = Resource,
                        Key = Key,
                        Url = GetUrl,
                        ContentType = ContentType,
                        FilePath = FilePath,
                        Size = Size,
                        HasBeenPut = false,
                        Purpose = Purpose
                    });
 
                    Xml.Append("<slot xmlns='urn:xmpp:http:upload:0'>");
                    Xml.Append("<put url='");
                    Xml.Append(XML.Encode(PutUrl));
                    Xml.Append("'><header name='X-Key'>");
                    Xml.Append(Key);
                    Xml.Append("</header></put>");
                    Xml.Append("<get url='");
                    Xml.Append(XML.Encode(GetUrl));
                    Xml.Append("'/></slot>");
 
                    await e.IqResult(Xml.ToString(), e.To);
                    break;
            }
        }
 
        private async Task<bool> IsFileOk(string FileName, string ContentType, long Size, IqEventArgs e)
        {
            if (Size < 0)
            {
                await e.IqErrorBadRequest(e.To, "Invalid size.", "en");
                return false;
            }
 
            if (string.IsNullOrEmpty(FileName) ||
                FileName.Contains("\\") ||
                FileName.Contains("/") ||
                FileName.Contains("..") ||
                FileName.IndexOfAny(invalidCharacters) >= 0)
            {
                await e.IqErrorBadRequest(e.To, "Invalid file name.", "en");
                return false;
            }
 
            if (string.IsNullOrEmpty(ContentType))
            {
                await e.IqErrorBadRequest(e.To, "Invalid content type.", "en");
                return false;
            }
 
            return true;
        }
 
        private Task PrepareBackupHandler(object Sender, IqEventArgs e)
        {
            return this.Prepare(e, FilePurpose.Backup);
        }
 
        private Task PrepareEncryptedStorageHandler(object Sender, IqEventArgs e)
        {
            return this.Prepare(e, FilePurpose.Encrypted);
        }
 
        private Task PreparePubSubStorageHandler(object Sender, IqEventArgs e)
        {
            return this.Prepare(e, FilePurpose.PubSub);
        }
 
        private Task PrepareInternalTransferHandler(object Sender, IqEventArgs e)
        {
            return this.Prepare(e, FilePurpose.InternalTransfer);
        }
 
        private async Task Prepare(IqEventArgs e, FilePurpose Purpose)
        {
            CaseInsensitiveString BareJid = e.From.BareJid;
            string FileName = XML.Attribute(e.Query, "filename");
            string ContentType = XML.Attribute(e.Query, "content-type");
            long Size = XML.Attribute(e.Query, "size", 0L);
 
            if (!await this.IsFileOk(FileName, ContentType, Size, e))
                return;
 
            if (!this.statusByClient.TryGetValue(BareJid, out UploadInformation Status))
            {
                Status = new UploadInformation(BareJid);
                this.statusByClient[BareJid] = Status;
            }
 
            Status.AddSpecialFile(FileName, ContentType, Size, Purpose);
 
            await e.IqResult(string.Empty, e.To);
        }
 
        private Task Files_Removed(object Sender, CacheItemEventArgs<CaseInsensitiveString, FileInfo> e)
        {
            try
            {
                if ((e.Value.Purpose == FilePurpose.Temporary ||
                    e.Value.Purpose == FilePurpose.InternalTransfer) &&
                    File.Exists(e.Value.FilePath))
                {
                    File.Delete(e.Value.FilePath);
 
                    string Folder = Path.GetDirectoryName(e.Value.FilePath);
 
                    if (Directory.GetFiles(Folder, "*.*", SearchOption.AllDirectories).Length == 0)
                        Directory.Delete(Folder);
                }
            }
            catch (Exception ex)
            {
                Log.Exception(ex);
            }
 
            return Task.CompletedTask;
        }
 
        public async Task<bool> DataUploaded(string Resource, string Key, Stream Data,
            ContentByteRangeInterval Interval, string ContentType, string RemoteEndPoint)
        {
            if (!this.files.TryGetValue(Resource, out FileInfo FileInfo))
            {
                Log.Error("PUT/PATCH request rejected. Resource slot not found.",
                    new KeyValuePair<string, object>("Resource", Resource),
                    new KeyValuePair<string, object>("RemoteEndPoint", RemoteEndPoint));
 
                return false;
            }
 
            if (Key != FileInfo.Key)
            {
                Log.Error("PUT/PATCH request rejected. Invalid key.",
                    new KeyValuePair<string, object>("Resource", Resource),
                    new KeyValuePair<string, object>("Key", Key),
                    new KeyValuePair<string, object>("RemoteEndPoint", RemoteEndPoint));
 
                return false;
            }
 
            if (ContentType != FileInfo.ContentType)
            {
                Log.Error("PUT/PATCH request rejected. Invalid Content-Type.",
                    new KeyValuePair<string, object>("Resource", Resource),
                    new KeyValuePair<string, object>("Key", Key),
                    new KeyValuePair<string, object>("RemoteEndPoint", RemoteEndPoint));
 
                return false;
            }
 
            if (FileInfo.HasBeenPut)
            {
                Log.Error("PUT/PATCH request rejected. File has already been put.",
                    new KeyValuePair<string, object>("Resource", Resource),
                    new KeyValuePair<string, object>("RemoteEndPoint", RemoteEndPoint));
 
                return false;
            }
 
            if (Interval.Last - Interval.First + 1 != Data.Length)
            {
                Log.Error("PUT/PATCH request rejected. Range error.",
                    new KeyValuePair<string, object>("Resource", Resource),
                    new KeyValuePair<string, object>("Actual", Data.Length),
                    new KeyValuePair<string, object>("Expected", FileInfo.Size),
                    new KeyValuePair<string, object>("RemoteEndPoint", RemoteEndPoint));
 
                return false;
            }
 
            if (Interval.Last >= FileInfo.Size)
            {
                Log.Error("PUT/PATCH request rejected. File size mismatch.",
                    new KeyValuePair<string, object>("Resource", Resource),
                    new KeyValuePair<string, object>("Actual", Data.Length),
                    new KeyValuePair<string, object>("Expected", FileInfo.Size),
                    new KeyValuePair<string, object>("RemoteEndPoint", RemoteEndPoint));
 
                return false;
            }
 
            Data.Position = 0;
 
            string Folder = Path.GetDirectoryName(FileInfo.FilePath);
            if (!Directory.Exists(Folder))
                Directory.CreateDirectory(Folder);
 
            using FileStream Dest = File.Exists(FileInfo.FilePath)
                ? File.OpenWrite(FileInfo.FilePath)
                : File.Create(FileInfo.FilePath);
 
            if (Interval.First > 0)
            {
                if (Dest.Length < Interval.First)
                {
                    Dest.Position = Dest.Length;
 
                    long Rest = Interval.First - Dest.Length;
                    byte[] Buffer = new byte[Math.Min(Rest, 65536)];
 
                    while (Rest > 0)
                    {
                        int c = (int)Math.Min(Rest, 65536);
                        await Dest.WriteAsync(Buffer, 0, c);
                        Rest -= c;
                    }
                }
                else
                    Dest.Position = Interval.First;
            }
 
            await Data.CopyToAsync(Dest);
 
            if (Data.Position >= FileInfo.Size)
            {
                FileInfo.HasBeenPut = true;
 
                if (InternetContent.Decodes(ContentType, out _, out _))
                {
                    string Error = null;
 
                    try
                    {
                        Dest.Position = 0;
                        byte[] Data2 = await Dest.ReadAllAsync();
 
                        ContentResponse Decoded = await InternetContent.DecodeAsync(ContentType,
                            Data2, new Uri(FileInfo.Url));
 
                        if (Decoded.HasError)
                            Error = Decoded.Error.Message;
                    }
                    catch (Exception ex)
                    {
                        Error = ex.Message;
                    }
 
                    if (!string.IsNullOrEmpty(Error))
                    {
                        Log.Error("PUT/PATCH request rejected. Uploaded content could not be " +
                            "decoded properly: " + Error,
                            new KeyValuePair<string, object>("Resource", Resource),
                            new KeyValuePair<string, object>("RemoteEndPoint", RemoteEndPoint));
 
                        Dest.Close();
                        File.Delete(FileInfo.FilePath);
                        this.files.Remove(Resource);
 
                        return false;
                    }
                }
 
                Log.Informational("File uploaded.",
                    new KeyValuePair<string, object>("JID", FileInfo.Jid.Value),
                    new KeyValuePair<string, object>("ContentType", FileInfo.ContentType),
                    new KeyValuePair<string, object>("Path", FileInfo.FilePath),
                    new KeyValuePair<string, object>("Purpose", FileInfo.Purpose),
                    new KeyValuePair<string, object>("Size", Data.Length),
                    new KeyValuePair<string, object>("RemoteEndPoint", RemoteEndPoint));
            }
 
            return true;
        }
 
        public bool TryGetFile(string Resource, out string FilePath, out string ContentType,
            out FilePurpose Purpose)
        {
            if (this.files.TryGetValue(Resource, out FileInfo FileInfo))
            {
                FilePath = FileInfo.FilePath;
                ContentType = FileInfo.ContentType;
                Purpose = FileInfo.Purpose;
                return true;
            }
            else
            {
                FilePath = null;
                ContentType = null;
                Purpose = FilePurpose.Temporary;
                return false;
            }
        }
 
    }
}