WaterFlush/Packages/com.tivadar.best.http/Runtime/HTTP/Caching/HTTPCache.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;

using Best.HTTP.Shared;
using Best.HTTP.Shared.Extensions;
using Best.HTTP.Shared.Logger;
using Best.HTTP.Shared.PlatformSupport.FileSystem;
using Best.HTTP.Shared.PlatformSupport.Threading;

using UnityEngine;

using static System.Math;
using static Best.HTTP.Hosts.Connections.HTTP1.Constants;
using static Best.HTTP.Response.HTTPStatusCodes;

namespace Best.HTTP.Caching
{
    internal sealed class HTTPCacheAcquireLockException : Exception
    {
        public HTTPCacheAcquireLockException(string message) : base(message)
        {
        }
    }

    /// <summary>
    /// Types of errors that can occur during cache validation.
    /// </summary>
    public enum ErrorTypeForValidation
    {
        /// <summary>
        /// Indicates that no error has occurred during validation.
        /// </summary>
        None,

        /// <summary>
        /// Indicates a server error has occurred during validation.
        /// </summary>
        ServerError,

        /// <summary>
        /// Indicates a connection error has occurred during validation.
        /// </summary>
        ConnectionError
    }

    /// <summary>
    /// Represents a delegate that can be used to perform actions before caching of an entity begins.
    /// </summary>
    /// <param name="method">The HTTP method used in the request.</param>
    /// <param name="uri">The URI of the HTTP request.</param>
    /// <param name="statusCode">The HTTP status code of the response.</param>
    /// <param name="headers">The HTTP response headers.</param>
    /// <param name="context">An optional logging context for debugging.</param>
    public delegate void OnBeforeBeginCacheDelegate(HTTPMethods method, Uri uri, int statusCode, Dictionary<string, List<string>> headers, LoggingContext context = null);

    /// <summary>
    /// Represents a delegate that can be used to handle cache size change events.
    /// </summary>
    public delegate void OnCacheSizeChangedDelegate();

    /// <summary>
    /// Manages caching of HTTP responses and associated metadata.
    /// </summary>
    /// <remarks>
    /// <para>The `HTTPCache` class provides a powerful caching mechanism for HTTP responses in Unity applications. 
    /// It allows you to store and retrieve HTTP responses efficiently, reducing network requests and improving 
    /// the performance of your application. By utilizing HTTP caching, you can enhance user experience, reduce 
    /// bandwidth usage, and optimize loading times.
    /// </para>
    /// <para>
    /// Key features:
    /// <list type="bullet">
    ///     <item><term>Optimal User Experience</term><description>Users experience faster load times and smoother interactions, enhancing user satisfaction.</description></item>
    ///     <item><term>Efficient Caching</term><description>It enables efficient caching of HTTP responses, reducing the need to fetch data from the network repeatedly.</description></item>
    ///     <item><term>Improved Performance</term><description>Caching helps improve the performance of your Unity application by reducing latency and decreasing loading times.</description></item>
    ///     <item><term>Bandwidth Optimization</term><description>By storing and reusing cached responses, you can minimize bandwidth usage, making your application more data-friendly.</description></item>
    ///     <item><term>Offline Access</term><description>Cached responses allow your application to function even when the device is offline or has limited connectivity.</description></item>
    ///     <item><term>Reduced Server Load</term><description>Fewer network requests mean less load on your server infrastructure, leading to cost savings and improved server performance.</description></item>
    ///     <item><term>Manual Cache Control</term><description>You can also manually control caching by adding, removing, or updating cached responses.</description></item>
    /// </list>
    /// </para>
    /// </remarks>
    [Best.HTTP.Shared.PlatformSupport.IL2CPP.Il2CppEagerStaticClassConstruction]
    public class HTTPCache : IDisposable, IHeartbeat
    {
        /// <summary>
        /// Constants defining folder and file names used in the HTTP cache storage.
        /// </summary>

        private const string RootFolderName = "LocalCache";
        private const string DatabaseFolderName = "Database";
        private const string ContentFolderName = "Content";
        private const string HeaderFileName = "headers.txt";
        private const string ContentFileName = "content.bin";

        /// <summary>
        /// This is the reversed domain the plugin uses for file paths when it have to load content from the local cache.
        /// </summary>
        public const string CacheHostName = "com.Tivadar.Best.HTTP.Local.Cache";

        /// <summary>
        /// Event that is triggered when the size of the cache changes.
        /// </summary>
        public OnCacheSizeChangedDelegate OnCacheSizeChanged;

        /// <summary>
        /// Gets the options that define the behavior of the HTTP cache.
        /// </summary>
        public HTTPCacheOptions Options { get; private set; }

        /// <summary>
        /// Gets the current size of the HTTP cache in bytes.
        /// </summary>
        public long CacheSize { get => this._cacheSize; }
        private long _cacheSize;

        /// <summary>
        /// Called before the plugin calls <see cref="BeginCache(HTTPMethods, Uri, int, Dictionary{string, List{string}}, LoggingContext)"/> to decide whether the content will be cached or not.
        /// </summary>
        public OnBeforeBeginCacheDelegate OnBeforeBeginCache;

        private int _subscribed;

        private bool _isSupported;
        private HTTPCacheDatabase _database;
        private string _baseDirectory;

        /// <summary>
        /// Initializes a new instance of the HTTPCache class with the specified cache options.
        /// </summary>
        /// <param name="options">The HTTP cache options specifying cache size and deletion policy.</param>
        public HTTPCache(HTTPCacheOptions options)
        {
            this.Options = options ?? new HTTPCacheOptions();

            try
            {
                _baseDirectory = Path.Combine(HTTPManager.GetRootSaveFolder(), RootFolderName);

#if UNITY_WEBGL && !UNITY_EDITOR
                this._isSupported = false;
                this._database = null;
#else            

                var dbBaseDir = Path.Combine(_baseDirectory, DatabaseFolderName);

                if (!HTTPManager.IOService.DirectoryExists(dbBaseDir))
                    HTTPManager.IOService.DirectoryCreate(dbBaseDir);

                _database = new HTTPCacheDatabase(dbBaseDir);

                var cacheDir = Path.Combine(_baseDirectory, ContentFolderName);
                if (!HTTPManager.IOService.DirectoryExists(cacheDir))
                    HTTPManager.IOService.DirectoryCreate(cacheDir);

                _isSupported = true;
#endif
            }
            catch (Exception ex)
            {
                if (HTTPManager.Logger.IsDiagnostic)
                    HTTPManager.Logger.Exception(nameof(HTTPCache), "ctr", ex);

                _isSupported = false;
                _database?.Dispose();
            }
        }

        /// <summary>
        /// Calculates a unique hash identifier based on the HTTP method and URI.
        /// </summary>
        /// <param name="method">The HTTP method used in the request.</param>
        /// <param name="uri">The URI of the HTTP request.</param>
        /// <returns>A unique hash identifier for the combination of method and URI.</returns>
        public static Hash128 CalculateHash(HTTPMethods method, Uri uri)
        {
            Hash128 hash = new Hash128();

            hash.Append((byte)method);
            hash.Append(uri.ToString());

            return hash;
        }

        /// <summary>
        /// Generates the directory path based on the given hash where cached content is stored.
        /// </summary>
        /// <param name="hash">A unique hash identifier for the cached content, returned by <see cref="HTTPCache.CalculateHash(HTTPMethods, Uri)"/>.</param>
        /// <returns>The directory path for the cached content associated with the given hash.</returns>
        public string GetHashDirectory(Hash128 hash)
            => Path.Combine(_baseDirectory, ContentFolderName, hash.ToString());

        /// <summary>
        /// Generates the file path for the header cache associated with the given hash.
        /// </summary>
        /// <param name="hash">A unique hash identifier for the cached content, returned by <see cref="HTTPCache.CalculateHash(HTTPMethods, Uri)"/>.</param>
        /// <returns>The file path for the header cache associated with the given hash.</returns>
        public string GetHeaderPathFromHash(Hash128 hash)
            => Path.Combine(_baseDirectory, ContentFolderName, hash.ToString(), "headers.cache");

        /// <summary>
        /// Generates the file path for the content cache associated with the given hash.
        /// </summary>
        /// <param name="hash">A unique hash identifier for the cached content, returned by <see cref="HTTPCache.CalculateHash(HTTPMethods, Uri)"/>.</param>
        /// <returns>The file path for the content cache associated with the given hash.</returns>
        public string GetContentPathFromHash(Hash128 hash)
            => Path.Combine(_baseDirectory, ContentFolderName, hash.ToString(), "content.cache");

        /// <summary>
        /// Checks whether cache files (header and content) associated with the given hash exist.
        /// </summary>
        /// <param name="hash">A unique hash identifier for the cached content.</param>
        /// <returns><c>true</c> if both header and content cache files exist, otherwise <c>false</c>.</returns>
        public bool AreCacheFilesExists(Hash128 hash)
            => HTTPManager.IOService.FileExists(GetHeaderPathFromHash(hash)) &&
               HTTPManager.IOService.FileExists(GetContentPathFromHash(hash));

        /// <summary>
        /// Sets up validation headers on an HTTP request if a locally cached response exists.
        /// </summary>
        /// <param name="request">The <see cref="HTTPRequest"/> to which validation headers will be added.</param>
        public void SetupValidationHeaders(HTTPRequest request)
        {
            var hash = CalculateHash(request.MethodType, request.CurrentUri);

            if (HTTPManager.Logger.IsDiagnostic)
                HTTPManager.Logger.Information(nameof(HTTPCache), $"{nameof(SetupValidationHeaders)}({request}, {hash})", request.Context);

            request.RemoveHeader("If-None-Match");
            request.RemoveHeader("If-Modified-Since");

            if (!_isSupported)
                return;

            if (!hash.isValid)
                return;

            // find&load content for the hash
            var content = _database.FindByHashAndUpdateRequestTime(hash, request.Context);

            if (content == null)
                return;

            if (!AreCacheFilesExists(hash))
            {
                Delete(hash, request.Context);
                return;
            }

            if (!string.IsNullOrEmpty(content.ETag))
                request.SetHeader("If-None-Match", content.ETag);

            if (content.LastModified != DateTime.MinValue)
                request.SetHeader("If-Modified-Since", content.LastModified.ToString("R"));
        }

        /// <summary>
        /// If necessary tries to make enough space in the cache by calling Maintain.
        /// </summary>
        internal bool IsThereEnoughSpaceAfterMaintain(ulong spaceNeeded, LoggingContext context)
        {
            // Run maintenance and see whether we have enough space for the new content.
            if ((ulong)(CacheSize + (long)spaceNeeded) > Options.MaxCacheSize)
                Maintain(contentLength: spaceNeeded, deleteLockedEntries: false, context: context);

            return (ulong)(CacheSize + (long)spaceNeeded) <= Options.MaxCacheSize;
        }

        /// <summary>
        /// Initiates the caching process for an HTTP response, creating an <see cref="HTTPCacheContentWriter"/> if caching is enabled and all predconditions are met.
        /// </summary>
        /// <param name="method">The <see cref="HTTPRequest"/> method used to fetch the response.</param>
        /// <param name="uri">The URI for the response.</param>
        /// <param name="statusCode">The HTTP status code of the response.</param>
        /// <param name="headers">The HTTP headers of the response.</param>
        /// <param name="context">An optional logging context for debugging.</param>
        /// <returns>An <see cref="HTTPCacheContentWriter"/> instance for writing the response content to the cache, or null if caching is not enabled or not possible.</returns>
        public HTTPCacheContentWriter BeginCache(HTTPMethods method, Uri uri, int statusCode, Dictionary<string, List<string>> headers, LoggingContext context)
        {
            if (HTTPManager.Logger.IsDiagnostic)
                HTTPManager.Logger.Information(nameof(HTTPCache), $"{nameof(BeginCache)}({method}, {uri}, {statusCode}, {headers?.Count})", context);

            if (!_isSupported)
                return null;

            // Check if the response is cacheable based on method, URI, and status code.
            // The original IsCachable got split into two:
            //  - first check method, uri and status code before calling OnBeforeBeginCache
            if (!IsCacheble(method, uri, statusCode))
                return null;

            if (headers == null)
                return null;

            // Log caching headers for debugging purposes.
            LogCachingHeaders(headers, context);

            var onBeforeBeginCache = OnBeforeBeginCache;
            if (onBeforeBeginCache != null)
            {
                try
                {
                    HTTPManager.Logger.Information(nameof(HTTPCache), $"{nameof(BeginCache)} - Calling {nameof(OnBeforeBeginCache)}", context);

                    // Invoke the OnBeforeBeginCache callback if provided.
                    onBeforeBeginCache?.Invoke(method, uri, statusCode, headers, context);

                    // Log caching headers after the callback.
                    LogCachingHeaders(headers, context);
                }
                catch (Exception ex)
                {
                    HTTPManager.Logger.Exception(nameof(HTTPCache), nameof(OnBeforeBeginCache), ex, context);
                }
            }

            // Check if there is enough space in the cache for the response content.
            var contentLengthStr = headers.GetFirstHeaderValue("content-length");
            if (ulong.TryParse(contentLengthStr, out var contentLength))
            {
                if (!IsThereEnoughSpaceAfterMaintain(contentLength, context))
                {
                    HTTPManager.Logger.Information(nameof(HTTPCache), $"{nameof(BeginCache)} - Not enough space({contentLength:N0}) in cache({CacheSize:N0}), even after Maintain!", context);
                    return null;
                }
            }

            // Check if the response headers indicate that the response is cacheable.
            //  (second half of the original IsCachable)
            //  - then existence of the required caching headers after OnBeforeBeginCache
            if (!IsCacheble(headers))
                return null;

            // Check if the calculated hash is valid.
            var hash = CalculateHash(method, uri);
            if (!hash.isValid)
                return null;

            // Try to get a lock on the cache entity. Prevents other requests from updating or loading from it.
            if (!_database.TryAcquireWriteLock(hash, headers, context))
            {
                HTTPManager.Logger.Information(nameof(HTTPCache), $"{nameof(BeginCache)} - Couldn't acquire write lock!", context);
                return null;
            }

            // Add or replace the "Date" header in the response if it is missing or invalid.
            // https://www.rfc-editor.org/rfc/rfc9110#section-6.6.1-8
            // A recipient with a clock that receives a response message without a Date header field
            // MUST record the time it was received and append a corresponding Date header field
            // to the message's header section if it is cached or forwarded downstream.

            var date = headers.GetFirstHeaderValue("date");
            if (string.IsNullOrEmpty(date) || !DateTime.TryParse(date, out var _))
            {
                // A recipient with a clock that receives a response with an invalid Date header field value
                // MAY replace that value with the time that response was received.
                headers.RemoveHeader("date");
                headers.AddHeader("date", DateTime.Now.ToString("R"));
            }

            Stream contentStream = null;
            try
            {
                // Create the cache directory if it does not exist.
                var hashDir = GetHashDirectory(hash);
                if (!HTTPManager.IOService.DirectoryExists(hashDir))
                    HTTPManager.IOService.DirectoryCreate(hashDir);

                // Create and write the header cache file.
                using (var headStream = HTTPManager.IOService.CreateFileStream(GetHeaderPathFromHash(hash), FileStreamModes.Create))
                    WriteHeaders(headStream, headers);

                // Create/open the content cache file.
                contentStream = HTTPManager.IOService.CreateFileStream(GetContentPathFromHash(hash), FileStreamModes.Create);
            }
            catch (Exception ex)
            {
                // Handle exceptions that may occur during cache file creation

                HTTPManager.Logger.Exception(nameof(HTTPCache), nameof(BeginCache), ex, context);

                contentStream?.Dispose();
                contentStream = null;

                // Delete the cache entry if an exception occurs.
                Delete(hash, context);
            }

            // Return an HTTPCacheContentWriter for writing response content to the cache.
            return new HTTPCacheContentWriter(this, hash, contentStream, contentLength, context);
        }

        /// <summary>
        /// Finalizes the caching process and takes appropriate actions based on the completion status.
        /// </summary>
        /// <param name="cacheResult">The <see cref="HTTPCacheContentWriter"/> instance representing the caching operation.</param>
        /// <param name="completedWithoutIssue">A boolean indicating whether the caching process completed without issues.</param>
        /// <param name="context">An optional logging context for debugging.</param>
        public void EndCache(HTTPCacheContentWriter cacheResult, bool completedWithoutIssue, LoggingContext context)
        {
            if (HTTPManager.Logger.IsDiagnostic)
                HTTPManager.Logger.Information(nameof(HTTPCache), $"{nameof(EndCache)}({cacheResult}, {completedWithoutIssue})", context);

            if (cacheResult == null || !cacheResult.Hash.isValid || !_isSupported)
                return;

            var hash = cacheResult.Hash;

            cacheResult.Close();

            if (completedWithoutIssue)
            {
                _database.ReleaseWriteLock(hash, cacheResult.ProcessedLength, cacheResult.Context);
                IncrementCacheSize(cacheResult.ProcessedLength);
            }
            else
            {
                Delete(hash, cacheResult.Context);
            }
        }

        /// <summary>
        /// Initiates the process of reading cached content associated with a given hash. Call BeginReadContent to acquire a Stream object that points to the cached resource.
        /// </summary>
        /// <param name="hash">A hash from <see cref="HTTPCache.CalculateHash(HTTPMethods, Uri)"/> identifying the resource.</param>
        /// <param name="context">An optional <see cref="LoggingContext"/></param>
        /// <returns>A stream for reading the cached content, or null if the content could not be read (the resource isn't cached or currently downloading).</returns>
        public Stream BeginReadContent(Hash128 hash, LoggingContext context)
        {
            if (HTTPManager.Logger.IsDiagnostic)
                HTTPManager.Logger.Information(nameof(HTTPCache), $"{nameof(BeginReadContent)}({hash})", context);

            if (!_isSupported)
                return null;

            if (!_database.TryAcquireReadLock(hash, context))
                return null;

            _database.UpdateLastAccessTime(hash, context);

            var contentPath = GetContentPathFromHash(hash);

            return HTTPManager.IOService.CreateFileStream(contentPath, FileStreamModes.OpenRead);
        }

        /// <summary>
        /// Finalizes the process of reading cached content associated with a given hash.
        /// </summary>
        /// <param name="hash">The unique hash identifier for the cached content.</param>
        /// <param name="context">An optional logging context for debugging.</param>
        public void EndReadContent(Hash128 hash, LoggingContext context)
        {
            if (HTTPManager.Logger.IsDiagnostic)
                HTTPManager.Logger.Information(nameof(HTTPCache), $"{nameof(EndReadContent)}({hash})", context);

            if (!_isSupported)
                return;

            _database.ReleaseReadLock(hash, context);
        }

        /// <summary>
        /// Deletes a cached entry identified by the given hash, including its associated header and content files.
        /// </summary>
        /// <param name="hash">The unique hash identifier for the cached entry to be deleted.</param>
        /// <param name="context">An optional logging context for debugging.</param>
        public void Delete(Hash128 hash, LoggingContext context)
        {
            if (HTTPManager.Logger.IsDiagnostic)
                HTTPManager.Logger.Information(nameof(HTTPCache), $"{nameof(Delete)}({hash})", context);

            if (!_isSupported)
                return;

            // Calling this function more than once with the same hash should be fine, the DB is locked and
            //  only one will find the metadata.

            try
            {
                _database.EnterWriteLock(context);

                try
                {
                    var headerPath = GetHeaderPathFromHash(hash);
                    if (HTTPManager.IOService.FileExists(headerPath))
                        HTTPManager.IOService.FileDelete(headerPath);

                    var contentPath = GetContentPathFromHash(hash);
                    if (HTTPManager.IOService.FileExists(contentPath))
                        HTTPManager.IOService.FileDelete(contentPath);

                    var hashDirectory = GetHashDirectory(hash);
                    if (HTTPManager.IOService.DirectoryExists(hashDirectory))
                        HTTPManager.IOService.DirectoryDelete(hashDirectory);
                }
                catch (Exception ex)
                {
                    HTTPManager.Logger.Exception(nameof(HTTPCache), $"{nameof(Delete)}({hash})", ex, context);
                }

                DecrementCacheSize(_database.Delete(hash, context));
            }
            finally
            {
                _database.ExitWriteLock(context);
            }
        }

        /// <summary>
        /// Refreshes the headers of a cached HTTP response with new headers.
        /// </summary>
        /// <param name="hash">A unique hash identifier for the cached response from a <see cref="HTTPCache.CalculateHash(HTTPMethods, Uri)"/> call.</param>
        /// <param name="newHeaders">A dictionary of new headers to replace or merge with existing headers.</param>
        /// <param name="context">Used by the plugin to add an addition logging context for debugging. It can be <c>null</c>.</param>
        /// <returns><c>true</c> if the headers were successfully refreshed; otherwise, <c>false</c>.</returns>
        public bool RefreshHeaders(Hash128 hash, Dictionary<string, List<string>> newHeaders, LoggingContext context)
        {
            // To Refresh stored cache related values from the headers described here:
            // 1.) https://www.rfc-editor.org/rfc/rfc9111.html#name-freshening-stored-responses
            // 2.) https://www.rfc-editor.org/rfc/rfc9111.html#name-updating-stored-header-fiel

            if (HTTPManager.Logger.IsDiagnostic)
                HTTPManager.Logger.Information(nameof(HTTPCache), $"{nameof(RefreshHeaders)}({hash}, {newHeaders?.Count})", context);

            if (!_isSupported)
                return false;

            // Log the new headers for debugging purposes.
            LogCachingHeaders(newHeaders, context);

            // Update the metadata with the new headers.
            if (_database.Update(hash, newHeaders, context))
            {
                // https://www.rfc-editor.org/rfc/rfc9111.html#name-updating-stored-header-fiel
                //  Load stored header, merge them with the received ones and store them.
                try
                {
                    using (var headerStream = HTTPManager.IOService.CreateFileStream(GetHeaderPathFromHash(hash), FileStreamModes.OpenReadWrite))
                    {
                        // Load existing headers.
                        var oldHeaders = LoadHeaders(headerStream);

                        foreach (var kvp in newHeaders)
                        {
                            if (oldHeaders.TryGetValue(kvp.Key, out var value))
                            {
                                // Replace existing header values with new values.
                                value.Clear();
                                value.AddRange(kvp.Value);
                            }
                            else
                            {
                                // Add new headers if they don't already exist.
                                oldHeaders.Add(kvp.Key, value);
                            }
                        }

                        // Seek to the beginning of the header file and write the updated headers.
                        headerStream.Seek(0, SeekOrigin.Begin);
                        headerStream.SetLength(0);
                        WriteHeaders(headerStream, oldHeaders);
                    }

                    // Everything went as expected, return true
                    return true;
                }
                catch (Exception ex)
                {
                    HTTPManager.Logger.Warning(nameof(HTTPCache), $"{nameof(RefreshHeaders)} - Couldn't merge/store headers. Exception: {ex}", context);

                    // Delete the cached response associated with the hash.
                    Delete(hash, context);
                }
            }

            return false;
        }

        private bool IsCacheble(Dictionary<string, List<string>> headers)
        {
            if (!_isSupported)
                return false;

            // Responses with byte ranges not supported.
            var byteRanges = headers.GetHeaderValues("content-range");
            if (byteRanges != null)
                return false;

            //http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2
            bool hasValidMaxAge = false;
            var cacheControls = headers.GetHeaderValues("cache-control");

            if (cacheControls != null)
            {
                // A local function that checks the header value for any indication that prohibits caching.
                // So, it must return TRUE, if it's NOT cachable.
                bool CheckHeader(string headerValue)
                {
                    HeaderParser parser = new HeaderParser(headerValue);
                    if (parser.Values != null && parser.Values.Count > 0)
                    {
                        for (int i = 0; i < parser.Values.Count; ++i)
                        {
                            var value = parser.Values[i];

                            // https://csswizardry.com/2019/03/cache-control-for-civilians/#no-store
                            if (value.Key == "no-store")
                                return true;

                            if (value.Key == "max-age" && value.HasValue)
                            {
                                double maxAge;
                                if (double.TryParse(value.Value, out maxAge))
                                {
                                    // A negative max-age value is a no cache
                                    if (maxAge <= 0)
                                        return true;
                                    hasValidMaxAge = true;
                                }
                            }
                        }
                    }

                    return false;
                }

                if (cacheControls.Exists(CheckHeader))
                    return false;
            }

            // It has an ETag header
            var etag = headers.GetFirstHeaderValue("etag");
            if (!string.IsNullOrEmpty(etag))
                return true;

            // It has an Expires header, and it's in the future
            var expires = headers.GetFirstHeaderValue("expires").ToDateTime(DateTime.FromBinary(0));
            if (expires > DateTime.Now)
                return true;

            // It has a Last-Modified header
            if (headers.GetFirstHeaderValue("last-modified") != null)
                return true;

            return hasValidMaxAge;
        }

        private bool IsCacheble(HTTPMethods method, Uri uri, int statusCode)
        {
            if (!_isSupported)
                return false;

            if (!uri.Scheme.StartsWith("http", StringComparison.OrdinalIgnoreCase))
                return false;

            if (method != HTTPMethods.Get)
                return false;

            // https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/204
            if (statusCode != OK && statusCode != NoContent)
                return false;

            return true;
        }

        /// <summary>
        /// Checks whether the caches resource identified by the hash is can be served from the local store with the given error conditions. 
        /// </summary>
        /// <remarks>This check reflects the very current state, even if it returns <c>true</c>, a request might just executing to get a write lock on it to refresh the content.</remarks>
        /// <param name="hash"><see cref="Hash128"/> hash returned by <see cref="HTTPCache.CalculateHash(HTTPMethods, Uri)"/> identifying a resource.</param>
        /// <param name="errorType">Possible error condition that can occur during validation. Servers can provision that certain stalled resources can be served if revalidation fails.</param>
        /// <param name="context">Used by the plugin to add an addition logging context for debugging. It can be <c>null</c>.</param>
        /// <returns><c>true</c> if the cached response can be served without validating it with the origin server; otherwise, <c>false</c></returns>
        public bool CanServeWithoutValidation(Hash128 hash, ErrorTypeForValidation errorType, LoggingContext context)
        {
            if (HTTPManager.Logger.IsDiagnostic)
                HTTPManager.Logger.Information(nameof(HTTPCache), $"{nameof(CanServeWithoutValidation)}({hash}, {errorType})", context);

            if (!_isSupported || !hash.isValid)
                return false;

            (CacheMetadataContent content, CacheMetadata metadata) = (null, null);
            try
            {
                // Attempt to find the cached content and metadata for the given hash.
                (content, metadata) = _database.FindContentAndMetadataLocked(hash);
                if (content == null)
                    return false;
            }
            catch(Exception ex)
            {
                HTTPManager.Logger.Exception(nameof(HTTPCache), $"{nameof(_database.FindContentAndMetadataLocked)}", ex, context);

                Delete(hash, context);
                
                return false;
            }

            // 
            if (metadata.Lock == LockTypes.Write)
                return false;

            // Check if cache files associated with the hash exist.
            if (!AreCacheFilesExists(hash))
            {
                Delete(hash, context);
                return false;
            }

            if ((content.Flags & CacheFlags.NoCache) != 0)
                return false;

            // Calculate the current age of the cached content, described here:
            // 1.) https://www.rfc-editor.org/rfc/rfc9111.html#name-freshness
            // 2.) https://www.rfc-editor.org/rfc/rfc9111.html#name-calculating-age
            if (content.MaxAge > 0)
            {
                long current_age = content.Age;

                // If there are more than one requests accessing the same resource it's possible that the first one sets the RequestTime
                //  but ResponseTime is the same old value while the second request tries to calculate the resrouce's Age. In this case,
                // we will just use the received Age.
                if (content.ResponseTime > content.RequestTime)
                {
                    var apparent_age = Max(0, (int)(content.ResponseTime - content.Date).TotalSeconds);
                    var response_delay = (int)(content.ResponseTime - content.RequestTime).TotalSeconds;
                    var corrected_age_value = content.Age + response_delay;

                    var corrected_initial_age = Max(apparent_age, corrected_age_value);

                    var resident_time = DateTime.Now - content.ResponseTime;
                    current_age = corrected_initial_age + (int)resident_time.TotalSeconds;
                }

                var maxAge = content.MaxAge;

                switch(errorType)
                {
                    case ErrorTypeForValidation.None:
                        // https://www.rfc-editor.org/rfc/rfc5861.html#section-1
                        // The stale-while-revalidate HTTP Cache-Control extension allows a
                        // cache to immediately return a stale response while it revalidates it
                        // in the background, thereby hiding latency (both in the network and on
                        // the server) from clients.

                        // If it's stalled but there's a value set for StaleWhileRevalidate and it's fresh with its value
                        if (current_age > maxAge && content.StaleWhileRevalidate > 0 && current_age <= (maxAge + content.StaleWhileRevalidate))
                        {
                            maxAge += content.StaleWhileRevalidate;
                            // TODO: send revalidate request
                        }
                        break;

                    case ErrorTypeForValidation.ServerError:
                    case ErrorTypeForValidation.ConnectionError:
                        // Handle stale-if-error caching extension:
                        // https://www.rfc-editor.org/rfc/rfc5861.html#section-4
                        if (content.StaleIfError > 0)
                            maxAge += content.StaleIfError;
                        break;
                }

                return current_age <= maxAge;
            }

            // Check if the content has not expired based on the 'Expires' header.
            return content.Expires > DateTime.Now;
        }

        /// <summary>
        /// Redirects a request to a cached entity.
        /// </summary>
        /// <param name="request">The <see cref="HTTPRequest"/> that will be redirected.</param>
        /// <param name="hash">Hash obtained from <see cref="HTTPCache.CalculateHash(HTTPMethods, Uri)"/>.</param>
        public void Redirect(HTTPRequest request, Hash128 hash)
        {
            if (HTTPManager.Logger.IsDiagnostic)
                HTTPManager.Logger.Information(nameof(HTTPCache), $"{nameof(Redirect)}({request}, {hash})", request.Context);

            if (!_isSupported || request == null || !hash.isValid)
                return;

            // Redirect to the local cache
            request.RedirectSettings.RedirectUri = new Uri($"file://{CacheHostName}/{hash}");
            request.RedirectSettings.IsRedirected = true;
        }

        internal void Maintain(ulong contentLength, bool deleteLockedEntries, LoggingContext context)
        {
            if (!_isSupported)
                return;

            HTTPManager.Logger.Information(nameof(HTTPCache), $"Maintain({contentLength:N0}, {deleteLockedEntries}, {System.Threading.Thread.CurrentThread.ManagedThreadId})", context);

            if (HTTPUpdateDelegator.Instance.IsMainThread())
                ThreadedRunner.RunShortLiving<ulong, bool, DateTime, LoggingContext>(MaintainImplementation, contentLength, deleteLockedEntries, HTTPManager.CurrentFrameDateTime, context);
            else
                MaintainImplementation(contentLength, deleteLockedEntries, HTTPManager.CurrentFrameDateTime, context);
        }

        private void ZeroOutCacheSize()
        {
            //HTTPManager.Logger.Information(nameof(HTTPCache), $"CacheSize - ZeroOutCacheSize()");

            Interlocked.Exchange(ref this._cacheSize, 0);
            if (Interlocked.CompareExchange(ref this._subscribed, 1, 0) == 0)
                HTTPManager.Heartbeats.Subscribe(this);
        }

        private void IncrementCacheSize(ulong withSize)
        {
            //HTTPManager.Logger.Information(nameof(HTTPCache), $"CacheSize - IncrementCacheSize({withSize:N0}) => {Interlocked.Add(ref this._cacheSize, (long)withSize):N0}");

            Interlocked.Add(ref this._cacheSize, (long)withSize);

            if (Interlocked.CompareExchange(ref this._subscribed, 1, 0) == 0)
                HTTPManager.Heartbeats.Subscribe(this);
        }

        private void DecrementCacheSize(ulong withSize)
        {
            //HTTPManager.Logger.Information(nameof(HTTPCache), $"CacheSize - DecrementCacheSize({-(long)withSize:N0}) => {Interlocked.Add(ref this._cacheSize, -(long)withSize):N0}");

            Interlocked.Add(ref this._cacheSize, -(long)withSize);

            if (Interlocked.CompareExchange(ref this._subscribed, 1, 0) == 0)
                HTTPManager.Heartbeats.Subscribe(this);
        }

        private void MaintainImplementation(ulong contentLength, bool deleteLockedEntries, DateTime now, LoggingContext context)
        {
            List<Hash128> markedForDelete = null;

            // lock down the whole database
            _database.EnterWriteLock(null);
            ZeroOutCacheSize();
            try
            {
                var deleteOlderDT = Options.DeleteOlder == TimeSpan.MaxValue ? DateTime.MinValue : now - Options.DeleteOlder;

                // Go through hashes in the DB metadata and compare them to the directory names in the cache folder
                // delete those that aren't in the DB/file system.

                for (int i = 0; i < _database.MetadataService.Metadatas.Count; ++i)
                {
                    var metadata = _database.MetadataService.Metadatas[i];

                    // When Maintain first called on startup, we can search for locked entries.
                    // An entry can remeain write locked if the process is terminated unexpectedly while a download is in progress.
                    // By deleting it here we can prevent serving incomplete content.
                    bool isIncomplete = deleteLockedEntries && metadata.Lock != LockTypes.Unlocked;
                    if (isIncomplete)
                        HTTPManager.Logger.Warning(nameof(HTTPCache), $"Incomplete cache entry({metadata}) found!", context);

                    bool isAnyFileMissing = !AreCacheFilesExists(metadata.Hash) && metadata.Lock == LockTypes.Unlocked;

                    if (isAnyFileMissing || isIncomplete || metadata.LastAccessTime <= deleteOlderDT)
                    {
                        if (markedForDelete == null)
                            markedForDelete = new List<Hash128>();

                        markedForDelete.Add(metadata.Hash);
                        metadata.MarkForDelete();
                    }
                    else
                    {
                        IncrementCacheSize(metadata.ContentLength);
                    }
                }

                var sortedMetadatas = new List<CacheMetadata>(_database.MetadataService.Metadatas);
                sortedMetadatas.Sort((x, y) => x.LastAccessTime.CompareTo(y.LastAccessTime));

                var cacheSize = CacheSize;
                var targetCacheSize = (long)(Options.MaxCacheSize - contentLength);
                for (int i = 0; i < sortedMetadatas.Count && cacheSize > targetCacheSize; ++i)
                {
                    var metadata = sortedMetadatas[i];

                    // already marked for deletion
                    if (metadata.IsDeleted)
                        continue;

                    // is in use
                    if (metadata.Lock != LockTypes.Unlocked)
                        continue;

                    if (markedForDelete == null)
                        markedForDelete = new List<Hash128>();

                    markedForDelete.Add(metadata.Hash);
                    cacheSize -= (long)metadata.ContentLength;
                }
            }
            finally
            {
                _database.ExitWriteLock(null);
            }

            if (markedForDelete != null)
            {
                HTTPManager.Logger.Information(nameof(HTTPCache), $"Maintain - collected {markedForDelete.Count} entries for deletion!", context);

                foreach (Hash128 key in markedForDelete)
                    Delete(key, context);

                markedForDelete.Clear();
            }
            else
                HTTPManager.Logger.Information(nameof(HTTPCache), "Maintain - collected 0 entries for deletion!", context);
        }

        private static void WriteHeaders(Stream headerStream, Dictionary<string, List<string>> headers)
        {
            if (headerStream == null || headers == null)
                return;

            foreach (var kvp in headers)
            {
                // https://www.rfc-editor.org/rfc/rfc9111.html#name-storing-header-and-trailer-
                // TODO: expand the no-save list
                if (kvp.Key.Equals("alt-svc", StringComparison.OrdinalIgnoreCase) ||
                    kvp.Key.Equals("content-encoding", StringComparison.OrdinalIgnoreCase) ||
                    kvp.Key.Equals("transfer-encoding", StringComparison.OrdinalIgnoreCase) ||
                    kvp.Key.Equals("connection", StringComparison.OrdinalIgnoreCase) ||
                    kvp.Key.Equals("proxy-authenticate", StringComparison.OrdinalIgnoreCase) ||
                    kvp.Key.Equals("content-length", StringComparison.OrdinalIgnoreCase))
                    continue;

                if (kvp.Value == null)
                {
                    headerStream.WriteString(kvp.Key);
                    headerStream.WriteString(":");
                    headerStream.WriteString(string.Empty);
                    headerStream.WriteLine();

                    continue;
                }

                foreach (var value in kvp.Value)
                {
                    headerStream.WriteString(kvp.Key);
                    headerStream.WriteString(":");
                    headerStream.WriteString(value);
                    headerStream.WriteLine();
                }
            }

            headerStream.WriteLine();
            headerStream.Flush();
        }

        private static Dictionary<string, List<string>> LoadHeaders(Stream headersStream)
        {
            var result = new Dictionary<string, List<string>>(StringComparer.OrdinalIgnoreCase);

            string headerName = HTTPResponse.ReadTo(headersStream, (byte)':', LF);
            while (headerName != string.Empty)
            {
                string value = HTTPResponse.ReadTo(headersStream, LF);

                result.AddHeader(headerName, value);

                headerName = HTTPResponse.ReadTo(headersStream, (byte)':', LF);
            }

            return result;
        }

        public void Dispose()
        {
            HTTPManager.Logger.Information(nameof(HTTPCache), $"{nameof(Dispose)}");

            ZeroOutCacheSize();

            try
            {
                _database?.Dispose();
                _database = null;
            }
            catch (Exception ex)
            {
                HTTPManager.Logger.Exception(nameof(HTTPCache), $"{nameof(Dispose)}", ex);
            }

            HTTPManager.Logger.Information(nameof(HTTPCache), $"{nameof(Dispose)} - Disposed!");
        }

        private static void LogCachingHeaders(Dictionary<string, List<string>> headers, LoggingContext context)
        {
            if (!HTTPManager.Logger.IsDiagnostic)
                return;

            var etag = headers.GetFirstHeaderValue("etag");
            var expires = headers.GetFirstHeaderValue("expires");
            var lastModified = headers.GetFirstHeaderValue("last-modified");
            var age = headers.GetFirstHeaderValue("age");
            var date = headers.GetFirstHeaderValue("date");
            var cacheControl = headers.GetFirstHeaderValue("cache-control");

            if (etag != null)
                HTTPManager.Logger.Verbose(nameof(HTTPCache), "ETag: " + etag, context);

            if (expires != null)
                HTTPManager.Logger.Verbose(nameof(HTTPCache), "Expires: " + expires, context);

            if (lastModified != null)
                HTTPManager.Logger.Verbose(nameof(HTTPCache), "Last-Modified: " + lastModified, context);

            if (age != null)
                HTTPManager.Logger.Verbose(nameof(HTTPCache), "Age: " + age, context);

            if (date != null)
                HTTPManager.Logger.Verbose(nameof(HTTPCache), "Date: " + date, context);

            if (cacheControl != null)
                HTTPManager.Logger.Verbose(nameof(HTTPCache), "Cache-Control: " + cacheControl, context);
        }

        /// <summary>
        /// Clears the HTTP cache by removing all cached entries and associated metadata.
        /// </summary>
        public void Clear()
        {
            if (!_isSupported)
                return;

            //_database.EnterWriteLock(null);
            try
            {
                var copyOfMetadatas = new List<CacheMetadata>(_database.MetadataService.Metadatas);

                foreach (var metadata in copyOfMetadatas)
                    Delete(metadata.Hash, null);
            }
            finally
            {
                //_database.ExitWriteLock(null);
            }
        }

        void IHeartbeat.OnHeartbeatUpdate(DateTime now, TimeSpan dif)
        {
            try
            {
                this.OnCacheSizeChanged?.Invoke();
            }
            catch(Exception ex)
            {
                HTTPManager.Logger.Exception(nameof(HTTPCache), "OnCacheSizeChanged", ex, null);
            }

            HTTPManager.Heartbeats.Unsubscribe(this);
            Interlocked.Exchange(ref this._subscribed, 0);
        }
    }
}