FtpServer.cs

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
using System.Net;
using System.Net.NetworkInformation;
using System.Net.Sockets;
using System.Security.Authentication;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Threading.Tasks;
using Waher.Events;
using Waher.Networking.SASL;
using Waher.Networking.Sniffers;
using Waher.Persistence;
using Waher.Runtime.Cache;
using Waher.Runtime.IO;
using Waher.Runtime.Queue;
using Waher.Security;
 
namespace Waher.Networking.FTP.Server
{
    public class FtpServer : IDisposable, ITlsCertificateEndpoint
    {
        public const int DefaultFtpControlPort = 21;
 
        public const int DefaultFtpsControlPort = 990;
 
        public const int DefaultFtpDataPort = 20;
 
        public const int DefaultConnectionBacklog = 10;
 
        public const int DefaultBufferSize = 16384;
 
        public const string FtpReadPrivilegePrefix = "FTP.Read.";
 
        public const string FtpWritePrivilegePrefix = "FTP.Write.";
 
        private LinkedList<TcpListener> listeners = new LinkedList<TcpListener>();
        private Cache<Guid, FtpClientConnection> clientConnections;
        private X509Certificate serverCertificate;
        private Dictionary<int, KeyValuePair<ClientCertificates, bool>> portSpecificMTlsSettings;
        private ClientCertificates clientCertificates = ClientCertificates.NotUsed;
        private bool trustClientCertificates = false;
        private bool clientCertificateSettingsLocked = false;
        private readonly AsyncQueue<DataPortReference> dataPorts = null;
        private readonly CommunicationLayer externalSniffers = new CommunicationLayer(false);
        private readonly IFtpServerPersistenceLayer persistenceLayer;
        private readonly CaseInsensitiveString domain;
        private readonly bool hasDataPorts = false;
        private bool encryptionRequired;
        private bool disposed = false;
 
        #region Constructors
 
        public FtpServer(CaseInsensitiveString Domain, X509Certificate ServerCertificate, bool EncryptionRequired,
            IFtpServerPersistenceLayer PersistenceLayer)
            : this(Domain, DefaultFtpControlPort, new int[] { DefaultFtpDataPort },
                  ServerCertificate, EncryptionRequired, PersistenceLayer)
        {
        }
 
        public FtpServer(CaseInsensitiveString Domain, int ControlPort,
            X509Certificate ServerCertificate, bool EncryptionRequired,
            IFtpServerPersistenceLayer PersistenceLayer)
            : this(Domain, ControlPort, new int[] { DefaultFtpDataPort }, ServerCertificate,
                  EncryptionRequired, PersistenceLayer)
        {
        }
 
        public FtpServer(CaseInsensitiveString Domain, int[] Ports,
            X509Certificate ServerCertificate, bool EncryptionRequired,
            IFtpServerPersistenceLayer PersistenceLayer)
            : this(Domain, GetControlPort(Ports), GetEncryptedControlPort(Ports),
                  GetDataPorts(Ports), ServerCertificate, EncryptionRequired,
                  PersistenceLayer)
        {
        }
 
        private static int GetControlPort(int[] Ports)
        {
            if ((Ports?.Length ?? 0) == 0)
                throw new ArgumentException("No FTP ports defined.", nameof(Ports));
 
            foreach (int Port in Ports)
            {
                if (Port == DefaultFtpControlPort)
                    return Port;
            }
 
            return Ports[0];
        }
 
        private static int? GetEncryptedControlPort(int[] Ports)
        {
            if ((Ports?.Length ?? 0) == 0)
                return null;
 
            foreach (int Port in Ports)
            {
                if (Port == DefaultFtpsControlPort)
                    return Port;
            }
 
            return null;
        }
 
        private static int[] GetDataPorts(int[] Ports)
        {
            List<int> Result = new List<int>();
            int ControlPort = GetControlPort(Ports);
 
            foreach (int Port in Ports)
            {
                if (Port != ControlPort)
                    Result.Add(Port);
            }
 
            return Result.ToArray();
        }
 
        private class DataPortReference
        {
            public int Port;
 
            public DataPortReference(int Port)
            {
                this.Port = Port;
            }
        }
 
        public FtpServer(CaseInsensitiveString Domain, int ControlPort, int[] DataPorts,
            X509Certificate ServerCertificate, bool EncryptionRequired,
            IFtpServerPersistenceLayer PersistenceLayer)
            : this(Domain, ControlPort, null, DataPorts, ServerCertificate,
                  EncryptionRequired, PersistenceLayer)
        {
        }
 
        public FtpServer(CaseInsensitiveString Domain, int ControlPort, int? EncryptedControlPort,
            int[] DataPorts, X509Certificate ServerCertificate, bool EncryptionRequired,
            IFtpServerPersistenceLayer PersistenceLayer)
            : this(Domain, new int[] { ControlPort }, EncryptedControlPort.HasValue ? new int[] { EncryptedControlPort.Value } : Array.Empty<int>(),
                  DataPorts, ServerCertificate, EncryptionRequired, PersistenceLayer)
        {
        }
 
        public FtpServer(CaseInsensitiveString Domain, int[] ControlPorts, int[] EncryptedControlPorts,
            int[] DataPorts, X509Certificate ServerCertificate, bool EncryptionRequired,
            IFtpServerPersistenceLayer PersistenceLayer)
            : this(Domain, ControlPorts, EncryptedControlPorts, DataPorts, ServerCertificate,
                 EncryptionRequired, ClientCertificates.NotUsed, false, null, false, PersistenceLayer)
        {
        }
 
        public FtpServer(CaseInsensitiveString Domain, int[] ControlPorts, int[] EncryptedControlPorts,
            int[] DataPorts, X509Certificate ServerCertificate, bool EncryptionRequired,
            ClientCertificates ClientCertificates, bool TrustClientCertificates,
            Dictionary<int, KeyValuePair<ClientCertificates, bool>> PortSpecificSettings, bool LockSettings,
            IFtpServerPersistenceLayer PersistenceLayer)
        {
            this.persistenceLayer = PersistenceLayer;
            this.serverCertificate = ServerCertificate;
            this.clientCertificates = ClientCertificates;
            this.trustClientCertificates = TrustClientCertificates;
            this.portSpecificMTlsSettings = PortSpecificSettings;
            this.clientCertificateSettingsLocked = LockSettings;
            this.encryptionRequired = EncryptionRequired;
            this.domain = Domain;
 
            if (EncryptionRequired && this.serverCertificate is null)
                throw new ArgumentException("Server Certificate must be provided, if encryption is required.", nameof(ServerCertificate));
 
            this.clientConnections = new Cache<Guid, FtpClientConnection>(int.MaxValue, TimeSpan.MaxValue, TimeSpan.FromMinutes(2), true);
            this.clientConnections.Removed += this.ClientConnections_Removed;
 
            if ((DataPorts?.Length ?? 0) > 0)
            {
                this.hasDataPorts = true;
                this.dataPorts = new AsyncQueue<DataPortReference>();
 
                foreach (int Port in DataPorts)
                    this.dataPorts.Queue(new DataPortReference(Port));
            }
 
            try
            {
                foreach (NetworkInterface Interface in NetworkInterface.GetAllNetworkInterfaces())
                {
                    if (Interface.OperationalStatus != OperationalStatus.Up)
                        continue;
 
                    IPInterfaceProperties Properties = Interface.GetIPProperties();
 
                    foreach (UnicastIPAddressInformation UnicastAddress in Properties.UnicastAddresses)
                    {
                        if ((UnicastAddress.Address.AddressFamily == AddressFamily.InterNetwork && Socket.OSSupportsIPv4) ||
                            (UnicastAddress.Address.AddressFamily == AddressFamily.InterNetworkV6 && Socket.OSSupportsIPv6))
                        {
                            foreach (int Port in ControlPorts)
                            {
                                this.OpenDataListener(this.AcceptTcpClientFtpControlCallback,
                                    UnicastAddress.Address, Port, false);
                            }
 
                            if (!(EncryptedControlPorts is null))
                            {
                                foreach (int Port in EncryptedControlPorts)
                                {
                                    this.GetMTlsSettings(Port, out ClientCertificates ClientCertificates2, out bool TrustCertificates);
 
                                    this.OpenDataListener(this.AcceptTcpClientFtpsControlCallback,
                                        UnicastAddress.Address, Port, false);
                                }
                            }
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                Log.Exception(ex);
            }
        }
 
        public bool HasDataPorts => this.hasDataPorts;
 
        public async Task<int?> GetDataPort(int Timeout)
        {
            if (this.dataPorts is null)
                return 0;
 
            DataPortReference Ref = await this.dataPorts.Wait(Timeout);
 
            return Ref?.Port;
        }
 
        public void ReleaseDataPort(int Port)
        {
            this.dataPorts?.Queue(new DataPortReference(Port));
        }
 
        internal DataPort OpenDataListener(AsyncCallback Callback, IPAddress Address,
            int Port, bool ReleaseAfterUse)
        {
            try
            {
                if (Port == 0)
                    this.externalSniffers.Information("Opening listening port on " + Address.ToString() + ".");
                else
                    this.externalSniffers.Information("Opening listening port " + Port.ToString() + " on " + Address.ToString() + ".");
 
                DataPort Result = new DataPort()
                {
                    Listener = new TcpListener(Address, Port),
                    ReleaseAfterUse = ReleaseAfterUse,
                    Server = this
                };
 
                Result.Listener.Start(DefaultConnectionBacklog);
                Result.Listener.BeginAcceptTcpClient(Callback, Result);
 
                lock (this.listeners)
                {
                    this.listeners.AddLast(Result.Listener);
                }
 
                if (Result.Listener.LocalEndpoint is IPEndPoint LocalEndpoint)
                {
                    this.externalSniffers.Information("Port " + LocalEndpoint.Port.ToString() + " on " + LocalEndpoint.Address.ToString() + " opened.");
                    Result.Port = LocalEndpoint.Port;
                    Result.LocalEndpoint = LocalEndpoint;
                }
                else
                {
                    this.externalSniffers.Information("Port on " + Address.ToString() + " opened.");
                    Result.Port = Port;
                }
 
                return Result;
            }
            catch (Exception ex)
            {
                Log.Exception(ex, Address.ToString() + ":" + Port);
                return null;
            }
        }
 
        internal void Remove(DataPort DataPort)
        {
            lock (this.listeners)
            {
                this.listeners.Remove(DataPort.Listener);
            }
 
            if (DataPort.ReleaseAfterUse)
                this.ReleaseDataPort(DataPort.Port);
        }
 
        public CommunicationLayer ExternalSniffers => this.externalSniffers;
 
        public bool Disposed => this.disposed;
 
        public bool AllowClearDataChannel { get; set; } = true;
 
        public bool AllowSafeDataChannel { get; set; } = true;
 
        public bool AllowConfidentialDataChannel { get; set; } = true;
 
        public bool AllowPrivateDataChannel { get; set; } = true;
 
        private async Task ClientConnections_Removed(object Sender, CacheItemEventArgs<Guid, FtpClientConnection> e)
        {
            try
            {
                await this.ClientConnectionRemoved.Raise(this, new ClientConnectionEventArgs(e.Value));
                await e.Value.DisposeAsync();
            }
            catch (Exception ex)
            {
                Log.Exception(ex);
            }
        }
 
        public int NrClientConnections
        {
            get => this.clientConnections.Count;
        }
 
        public FtpClientConnection[] GetClientConnections()
        {
            List<FtpClientConnection> Connections = new List<FtpClientConnection>();
 
            foreach (Guid Id in this.clientConnections.GetKeys())
            {
                if (this.clientConnections.TryGetValue(Id, out FtpClientConnection Connection))
                    Connections.Add(Connection);
            }
 
            Connections.Sort((c1, c2) =>
            {
                return c1.UserName.CompareTo(c2.UserName);
            });
 
            return Connections.ToArray();
        }
 
        public bool TryGetClientConnection(Guid ID, out FtpClientConnection Connection)
        {
            return this.clientConnections.TryGetValue(ID, out Connection);
        }
 
        public CaseInsensitiveString Domain => this.domain;
 
        public X509Certificate ServerCertificate => this.serverCertificate;
 
        public void UpdateCertificate(X509Certificate ServerCertificate)
        {
            this.serverCertificate = ServerCertificate;
        }
 
        public bool EncryptionRequired
        {
            get => this.encryptionRequired;
            set => this.encryptionRequired = value;
        }
 
        public void ConfigureMutualTls(ClientCertificates ClientCertificates, bool TrustClientCertificates, bool LockSettings)
        {
            this.ConfigureMutualTls(ClientCertificates, TrustClientCertificates, null, LockSettings);
        }
 
        public void ConfigureMutualTls(ClientCertificates ClientCertificates, bool TrustClientCertificates,
            Dictionary<int, KeyValuePair<ClientCertificates, bool>> PortSpecificSettings, bool LockSettings)
        {
            if (this.clientCertificateSettingsLocked)
                throw new InvalidOperationException("Mutual TLS settings locked.");
 
            this.clientCertificates = ClientCertificates;
            this.trustClientCertificates = TrustClientCertificates;
            this.portSpecificMTlsSettings = PortSpecificSettings;
            this.clientCertificateSettingsLocked = LockSettings;
        }
 
        public ClientCertificates ClientCertificates => this.clientCertificates;
 
        public bool TrustClientCertificates => this.trustClientCertificates;
 
        public void GetMTlsSettings(int Port, out ClientCertificates ClientCertificates, out bool TrustClientCertificates)
        {
            if (!(this.portSpecificMTlsSettings is null) && 
                this.portSpecificMTlsSettings.TryGetValue(Port, out KeyValuePair<ClientCertificates, bool> P))
            {
                ClientCertificates = P.Key;
                TrustClientCertificates = P.Value;
            }
            else
            {
                ClientCertificates = this.clientCertificates;
                TrustClientCertificates = this.trustClientCertificates;
            }
        }
 
        internal IFtpServerPersistenceLayer PersistenceLayer => this.persistenceLayer;
 
        public void Dispose()
        {
            this.disposed = true;
 
            if (!(this.clientConnections is null))
            {
                this.clientConnections.Clear();
                this.clientConnections.Dispose();
                this.clientConnections = null;
            }
 
            if (!(this.listeners is null))
            {
                LinkedList<TcpListener> Listeners = this.listeners;
                this.listeners = null;
 
                foreach (TcpListener Listener in Listeners)
                    Listener.Stop();
            }
 
            if (!(this.externalSniffers is null))
            {
                foreach (ISniffer Sniffer in this.externalSniffers)
                    (Sniffer as IDisposable)?.Dispose();
            }
        }
 
        public int[] OpenPorts
        {
            get
            {
                lock (this.listeners)
                {
                    return this.GetOpenPorts(this.listeners);
                }
            }
        }
 
        private int[] GetOpenPorts(LinkedList<TcpListener> Listeners)
        {
            SortedDictionary<int, bool> Open = new SortedDictionary<int, bool>();
 
            if (!(Listeners is null))
            {
                IPEndPoint IPEndPoint;
 
                foreach (TcpListener Listener in Listeners)
                {
                    IPEndPoint = Listener.LocalEndpoint as IPEndPoint;
                    if (!(IPEndPoint is null))
                        Open[IPEndPoint.Port] = true;
                }
            }
 
            int[] Result = new int[Open.Count];
            Open.Keys.CopyTo(Result, 0);
 
            return Result;
        }
 
        #endregion
 
        #region Connections
 
        private void AcceptTcpClientFtpControlCallback(IAsyncResult ar)
        {
            this.AcceptTcpClientControlCallback(ar, false);
        }
 
        private void AcceptTcpClientFtpsControlCallback(IAsyncResult ar)
        {
            this.AcceptTcpClientControlCallback(ar, true);
        }
 
        private async void AcceptTcpClientControlCallback(IAsyncResult ar, bool Tls)
        {
            try
            {
                if (this.disposed || NetworkingModule.Stopping)
                    return;
 
                DataPort Port = (DataPort)ar.AsyncState;
                TcpListener Listener = Port.Listener;
 
                try
                {
                    TcpClient Client = Listener.EndAcceptTcpClient(ar);
                    FtpClientControlConnection ClientConnection;
                    ISniffer[] Sniffers;
 
                    if (this.externalSniffers.HasSniffers)
                        Sniffers = this.externalSniffers.Sniffers;
                    else
                        Sniffers = Array.Empty<ISniffer>();
 
                    BinaryTcpClient BinaryTcpClient = new BinaryTcpClient(Client, false, false, Sniffers);
                    BinaryTcpClient.Bind(true);
 
                    if (Tls)
                    {
                        this.GetMTlsSettings(Port.Port, out ClientCertificates ClientCertificates, out bool TrustCertificates);
 
                        Task _ = this.SwitchToTls(BinaryTcpClient, ClientCertificates, TrustCertificates, true, Sniffers);
                    }
                    else
                    {
                        ClientConnection = new FtpClientControlConnection(BinaryTcpClient, this, this.persistenceLayer,
                            ClientCertificates.NotUsed, this.trustClientCertificates, Sniffers);
 
                        ClientConnection.Information("Control Connection accepted from " + BinaryTcpClient.RemoteEndPoint + ".");
 
                        ClientConnection.OnStateChanged += this.Nop;
 
                        this.clientConnections[ClientConnection.ID] = ClientConnection;
 
                        await this.ClientConnectionAdded.Raise(this, new ClientConnectionEventArgs(ClientConnection));
 
                        BinaryTcpClient.Continue();
                        await ClientConnection.BeginWrite("220 Service ready.\r\n", null, null);
                    }
                }
                finally
                {
                    if (!this.disposed)
                    {
                        if (Tls)
                            Listener.BeginAcceptTcpClient(this.AcceptTcpClientFtpsControlCallback, Port);
                        else
                            Listener.BeginAcceptTcpClient(this.AcceptTcpClientFtpControlCallback, Port);
                    }
                }
            }
            catch (SocketException)
            {
                // Ignore
            }
            catch (ObjectDisposedException)
            {
                // Ignore
            }
            catch (NullReferenceException)
            {
                // Ignore
            }
            catch (Exception ex)
            {
                if (this.listeners is null)
                    return;
 
                Log.Exception(ex);
            }
        }
 
        internal async Task SwitchToTls(BinaryTcpClient Client, ClientCertificates ClientCertificates, 
            bool TrustCertificates, bool Control, ISniffer[] Sniffers)
        {
            string RemoteEndpoint = Client.RemoteEndPoint.RemovePortNumber();
 
            if (Security.LoginMonitor.LoginAuditor.CanStartTls(RemoteEndpoint))
            {
                try
                {
                    if (this.externalSniffers.HasSniffers)
                    {
                        this.externalSniffers.Information("Switching to TLS. (Client Certificates: " + ClientCertificates.ToString() +
                            ", Trust Certificates: " + TrustCertificates.ToString() + ")");
                    }
 
                    await Client.UpgradeToTlsAsServer(this.serverCertificate, Crypto.SecureTls,
                        ClientCertificates, null, TrustCertificates, "ftp");
 
                    if (this.externalSniffers.HasSniffers)
                    {
                        StringBuilder sb = new StringBuilder();
 
                        sb.Append("TLS established");
                        sb.Append(". Cipher Strength: ");
                        sb.Append(Client.CipherStrength.ToString());
                        sb.Append(", Hash Strength: ");
                        sb.Append(Client.HashStrength.ToString());
                        sb.Append(", Key Exchange Strength: ");
                        sb.Append(Client.KeyExchangeStrength.ToString());
 
                        this.externalSniffers.Information(sb.ToString());
 
                        if (!(Client.RemoteCertificate is null))
                        {
                            if (this.externalSniffers.HasSniffers)
                            {
                                sb.Clear();
 
                                sb.Append("Remote Certificate received. Valid: ");
                                sb.Append(Client.RemoteCertificateValid.ToString());
                                sb.Append(", Subject: ");
                                sb.Append(Client.RemoteCertificate.Subject);
                                sb.Append(", Issuer: ");
                                sb.Append(Client.RemoteCertificate.Issuer);
                                sb.Append(", S/N: ");
                                sb.Append(Convert.ToBase64String(Client.RemoteCertificate.GetSerialNumber()));
                                sb.Append(", Hash: ");
                                sb.Append(Convert.ToBase64String(Client.RemoteCertificate.GetCertHash()));
 
                                this.externalSniffers.Information(sb.ToString());
                            }
                        }
                    }
 
                    if (Control)
                    {
                        FtpClientControlConnection ClientConnection = new FtpClientControlConnection(Client, 
                            this, this.persistenceLayer, ClientCertificates, TrustCertificates, Sniffers);
 
                        ClientConnection.Information("Encrypted Control Connection accepted from " + Client.RemoteEndPoint + ".");
                        ClientConnection.OnStateChanged += this.Nop;
 
                        this.clientConnections[ClientConnection.ID] = ClientConnection;
 
                        await this.ClientConnectionAdded.Raise(this, new ClientConnectionEventArgs(ClientConnection));
 
                        Client.Continue();
 
                        await ClientConnection.BeginWrite("220 Service ready.\r\n", null, null);
                    }
                    else
                        Client.Continue();
                }
                catch (AuthenticationException ex)
                {
                    if (this.externalSniffers.HasSniffers)
                        this.externalSniffers.Exception(ex);
 
                    await this.LoginFailure(ex, Client, RemoteEndpoint);
                }
                catch (SocketException ex)
                {
                    if (this.externalSniffers.HasSniffers)
                        this.externalSniffers.Exception(ex);
 
                    await Client.DisposeAsync();
                }
                catch (Win32Exception ex)
                {
                    if (this.externalSniffers.HasSniffers)
                        this.externalSniffers.Exception(ex);
 
                    await this.LoginFailure(ex, Client, RemoteEndpoint);
                }
                catch (IOException ex)
                {
                    if (this.externalSniffers.HasSniffers)
                        this.externalSniffers.Exception(ex);
 
                    await Client.DisposeAsync();
                }
                catch (Exception ex)
                {
                    if (this.externalSniffers.HasSniffers)
                        this.externalSniffers.Exception(ex);
 
                    await Client.DisposeAsync();
                    Log.Exception(ex);
                }
            }
            else
                await Client.DisposeAsync();
        }
 
        internal async Task LoginFailure(Exception ex, BinaryTcpClient Client, string RemoteIpEndpoint)
        {
            Exception ex2 = Log.UnnestException(ex);
            Security.LoginMonitor.LoginAuditor.ReportTlsHackAttempt(RemoteIpEndpoint, "TLS handshake failed: " + ex2.Message, "HTTPS");
 
            await Client.DisposeAsync();
        }
 
        private Task Nop(object Sender, FtpControlConnectionState NewState)
        {
            return Task.CompletedTask;
        }
 
        internal string GetTransformPath()
        {
            foreach (ISniffer Sniffer in this.externalSniffers.Sniffers)
            {
                if (Sniffer is XmlFileSniffer XmlFileSniffer)
                    return XmlFileSniffer.Transform;
            }
 
            return null;
        }
 
        internal void Closed(FtpClientConnection Connection)
        {
            this.clientConnections?.Remove(Connection.ID);
        }
 
        internal void Ping(FtpClientConnection Connection)
        {
            this.clientConnections?.ContainsKey(Connection.ID);
        }
 
        #endregion
 
        #region Accounts
 
        internal Task<IAccount> GetAccount(CaseInsensitiveString UserName)
        {
            return this.persistenceLayer.GetAccount(UserName);
        }
 
        public event EventHandlerAsync<ClientConnectionEventArgs> ClientConnectionAdded = null;
 
        public event EventHandlerAsync<ClientConnectionEventArgs> ClientConnectionRemoved = null;
 
        #endregion
 
    }
}