using System;
using System.Collections.Generic;
using System.IO;
using Best.HTTP.Shared;
using Best.HTTP.Shared.Databases;
using Best.HTTP.Shared.Databases.Indexing;
using Best.HTTP.Shared.Databases.Indexing.Comparers;
using Best.HTTP.Shared.Databases.MetadataIndexFinders;
using Best.HTTP.Shared.Databases.Utils;
using Best.HTTP.Shared.Extensions;
using Best.HTTP.Shared.Logger;
using Best.HTTP.Shared.PlatformSupport.Threading;
using UnityEngine;
namespace Best.HTTP.Caching
{
struct v128View
{
public ulong low;
public ulong high;
}
///
/// Possible lock-states a cache-content can be in.
///
public enum LockTypes : byte
{
///
/// No reads or writes are happening on the cached content.
///
Unlocked,
///
/// There's one writer operating on the cached content. No other writes or reads allowed while this lock is held on the content.
///
Write,
///
/// There's at least one read operation happening on the cached content. No writes allowed while this lock is held on the content.
///
Read
}
///
/// Metadata stored for every cached content. It contains only limited data about the content to help early cache decision making and cache management.
///
internal class CacheMetadata : Metadata
{
///
/// Unique hash of the cached content, generated by .
///
public UnityEngine.Hash128 Hash { get; set; }
///
/// Size of the stored content in bytes.
///
public ulong ContentLength { get; set; }
///
/// When the last time the content is accessed. Also initialized when the initial download completes.
///
public DateTime LastAccessTime { get; set; }
///
/// What kind of lock the content is currently in.
///
public LockTypes Lock { get; set; }
///
/// Number of readers.
///
public int ReadLockCount { get; set; }
public unsafe override void SaveTo(Stream stream)
{
base.SaveTo(stream);
var hash = this.Hash;
v128View view = *(v128View*)&hash;
stream.EncodeUnsignedVariableByteInteger(view.low);
stream.EncodeUnsignedVariableByteInteger(view.high);
stream.EncodeUnsignedVariableByteInteger(ContentLength);
stream.EncodeSignedVariableByteInteger(LastAccessTime.ToBinary() >> CacheMetadataContentParser.PrecisionShift);
// Only Write locks should persist as Reads doesn't alter the cached content
if (this.Lock == LockTypes.Write)
stream.EncodeUnsignedVariableByteInteger((byte)this.Lock);
else
stream.EncodeUnsignedVariableByteInteger((byte)LockTypes.Unlocked);
}
public unsafe override void LoadFrom(Stream stream)
{
base.LoadFrom(stream);
var hash = default(v128View);
hash.low = stream.DecodeUnsignedVariableByteInteger();
hash.high = stream.DecodeUnsignedVariableByteInteger();
this.Hash = *(UnityEngine.Hash128*)&hash;
this.ContentLength = stream.DecodeUnsignedVariableByteInteger();
this.LastAccessTime = DateTime.FromBinary(stream.DecodeSignedVariableByteInteger() << CacheMetadataContentParser.PrecisionShift);
this.Lock = (LockTypes)stream.DecodeUnsignedVariableByteInteger();
}
public override string ToString() => $"[Metadata {Hash}, {ContentLength:N0}, {Lock}, {ReadLockCount}]";
}
///
/// Possible caching flags that a `Cache-Control` header can send.
///
[Flags]
public enum CacheFlags : byte
{
///
/// No special treatment required.
///
None = 0x00,
///
/// Indicates whether the entity must be revalidated with the server or can be serverd directly from the cache without touching the server when the content is considered stale.
///
///
/// More details can be found here:
///
///
///
///
MustRevalidate = 0x01,
///
/// If it's true, the client always have to revalidate the cached content when it's stale.
///
///
/// More details can be found here:
///
///
///
///
NoCache = 0x02
}
///
/// Cached content associated with a .
///
/// This is NOT the cached content received from the server! It's for storing caching values to decide on how the content can be used.
internal sealed class CacheMetadataContent
{
///
/// ETag of the entity.
///
///
/// More details can be found here:
///
///
///
///
public string ETag;
///
/// LastModified date of the entity. Use ToString("r") to convert it to the format defined in RFC 1123.
///
public DateTime LastModified = DateTime.MinValue;
///
/// When the cache will expire.
///
///
/// More details can be found here:
///
///
///
///
public DateTime Expires = DateTime.MinValue;
///
/// The age that came with the response
///
///
/// More details can be found here:
///
///
///
///
public uint Age;
///
/// Maximum how long the entry should served from the cache without revalidation.
///
///
/// More details can be found here:
///
///
///
///
public uint MaxAge;
///
/// The Date that came with the response.
///
///
/// More details can be found here:
///
///
///
///
public DateTime Date = DateTime.MinValue;
///
/// It's a grace period to serve staled content without revalidation.
///
///
/// More details can be found here:
///
///
///
///
public uint StaleWhileRevalidate;
///
/// Allows the client to serve stale content if the server responds with an 5xx error.
///
///
/// More details can be found here:
///
///
///
///
public uint StaleIfError;
///
/// bool values packed into one single flag.
///
public CacheFlags Flags = CacheFlags.None;
///
/// The value of the clock at the time of the request that resulted in the stored response.
///
///
/// More details can be found here:
///
///
///
///
public DateTime RequestTime = DateTime.MinValue;
///
/// The value of the clock at the time the response was received.
///
public DateTime ResponseTime = DateTime.MinValue;
public CacheMetadataContent()
{
}
internal void From(Dictionary> headers)
{
this.ETag = headers.GetFirstHeaderValue("ETag").ToStr(this.ETag ?? string.Empty);
this.Expires = headers.GetFirstHeaderValue("Expires").ToDateTime(this.Expires);
if (this.Expires < DateTime.Now)
this.Expires = DateTime.MinValue;
this.LastModified = headers.GetFirstHeaderValue("Last-Modified").ToDateTime(DateTime.MinValue);
this.Age = headers.GetFirstHeaderValue("Age").ToUInt32(this.Age);
this.Date = headers.GetFirstHeaderValue("Date").ToDateTime(this.Date);
// https://www.rfc-editor.org/rfc/rfc9111.html#section-4.2.1-4
// When there is more than one value present for a given directive
// (e.g., two Expires header field lines or multiple Cache-Control: max-age directives),
// either the first occurrence should be used or the response should be considered stale.
var cacheControl = headers.GetFirstHeaderValue("cache-control");
if (!string.IsNullOrEmpty(cacheControl))
{
HeaderParser parser = new HeaderParser(cacheControl);
if (parser.Values != null)
{
this.Flags = CacheFlags.None;
for (int i = 0; i < parser.Values.Count; ++i)
{
var kvp = parser.Values[i];
switch (kvp.Key.ToLowerInvariant())
{
// https://www.rfc-editor.org/rfc/rfc9111.html#name-max-age-2
case "max-age":
if (kvp.HasValue)
{
// Some cache proxies will return float values
double maxAge;
if (double.TryParse(kvp.Value, out maxAge) && maxAge >= 0)
this.MaxAge = (uint)maxAge;
else
this.MaxAge = 0;
}
else
this.MaxAge = 0;
// https://www.rfc-editor.org/rfc/rfc9111.html#section-5.3-8
// https://www.rfc-editor.org/rfc/rfc9111.html#cache-response-directive.s-maxage
// If a response includes a Cache-Control header field with the max-age directive (Section 5.2.2.1), a recipient MUST ignore the Expires header field.
this.Expires = DateTime.MinValue;
break;
// https://www.rfc-editor.org/rfc/rfc9111.html#name-must-revalidate
case "must-revalidate": this.Flags |= CacheFlags.MustRevalidate; break;
// https://www.rfc-editor.org/rfc/rfc9111.html#name-no-cache-2
case "no-cache": this.Flags |= CacheFlags.NoCache; break;
// https://www.rfc-editor.org/rfc/rfc5861.html#section-3
case "stale-while-revalidate": this.StaleWhileRevalidate = kvp.HasValue ? kvp.Value.ToUInt32(0) : 0; break;
// https://www.rfc-editor.org/rfc/rfc5861.html#section-4
case "stale-if-error": this.StaleIfError = kvp.HasValue ? kvp.Value.ToUInt32(0) : 0; break;
}
}
}
}
}
}
internal sealed class CacheMetadataContentParser : IDiskContentParser
{
public const int PrecisionShift = 24;
public void Encode(Stream stream, CacheMetadataContent content)
{
stream.EncodeUnsignedVariableByteInteger(content.MaxAge);
stream.WriteLengthPrefixedString(content.ETag);
stream.EncodeSignedVariableByteInteger(content.LastModified.ToBinary() >> PrecisionShift);
stream.EncodeSignedVariableByteInteger(content.Expires.ToBinary() >> PrecisionShift);
stream.EncodeUnsignedVariableByteInteger(content.Age);
stream.EncodeSignedVariableByteInteger(content.Date.ToBinary() >> PrecisionShift);
stream.EncodeUnsignedVariableByteInteger((byte)content.Flags);
stream.EncodeUnsignedVariableByteInteger(content.StaleWhileRevalidate);
stream.EncodeUnsignedVariableByteInteger(content.StaleIfError);
stream.EncodeSignedVariableByteInteger(content.RequestTime.ToBinary() >> PrecisionShift);
stream.EncodeSignedVariableByteInteger(content.ResponseTime.ToBinary() >> PrecisionShift);
}
public CacheMetadataContent Parse(Stream stream, int length)
{
CacheMetadataContent content = new CacheMetadataContent();
content.MaxAge = (uint)stream.DecodeUnsignedVariableByteInteger();
content.ETag = stream.ReadLengthPrefixedString();
content.LastModified = DateTime.FromBinary(stream.DecodeSignedVariableByteInteger() << PrecisionShift);
content.Expires = DateTime.FromBinary(stream.DecodeSignedVariableByteInteger() << PrecisionShift);
content.Age = (uint)stream.DecodeUnsignedVariableByteInteger();
content.Date = DateTime.FromBinary(stream.DecodeSignedVariableByteInteger() << PrecisionShift);
content.Flags = (CacheFlags)stream.DecodeUnsignedVariableByteInteger();
content.StaleWhileRevalidate = (uint)stream.DecodeUnsignedVariableByteInteger();
content.StaleIfError = (uint)stream.DecodeUnsignedVariableByteInteger();
content.RequestTime = DateTime.FromBinary(stream.DecodeSignedVariableByteInteger() << PrecisionShift);
content.ResponseTime = DateTime.FromBinary(stream.DecodeSignedVariableByteInteger() << PrecisionShift);
return content;
}
}
internal sealed class CacheMetadataIndexingService : IndexingService
{
private AVLTree index_Hash = new AVLTree(new Hash128Comparer());
public override void Index(CacheMetadata metadata)
{
base.Index(metadata);
this.index_Hash.Add(metadata.Hash, metadata.Index);
}
public override void Remove(CacheMetadata metadata)
{
base.Remove(metadata);
this.index_Hash.Remove(metadata.Hash);
}
public override void Clear()
{
base.Clear();
this.index_Hash.Clear();
}
public override IEnumerable GetOptimizedIndexes() => this.index_Hash.WalkHorizontal();
public bool ContainsHash(UnityEngine.Hash128 hash) => this.index_Hash.ContainsKey(hash);
public List FindByHash(UnityEngine.Hash128 hash) => this.index_Hash.Find(hash);
}
internal sealed class CacheMetadataService : MetadataService
{
public CacheMetadataService(IndexingService indexingService, IEmptyMetadataIndexFinder emptyMetadataIndexFinder)
: base(indexingService, emptyMetadataIndexFinder)
{
}
public override CacheMetadata CreateFrom(Stream stream)
{
return base.CreateFrom(stream);
}
public CacheMetadata Create(UnityEngine.Hash128 hash, CacheMetadataContent value, int filePos, int length)
{
var result = base.CreateDefault(value, filePos, length, (content, metadata) => {
metadata.Hash = hash;
});
return result;
}
}
internal sealed class CacheDatabaseOptions : DatabaseOptions
{
public CacheDatabaseOptions() : base("CacheDatabase")
{
base.UseHashFile = false;
}
}
internal sealed class HTTPCacheDatabase : Database
{
public HTTPCacheDatabase(string directory)
: this(directory, new CacheDatabaseOptions(), new CacheMetadataIndexingService())
{
}
private HTTPCacheDatabase(string directory,
DatabaseOptions options,
CacheMetadataIndexingService indexingService)
: base(directory, options, indexingService, new CacheMetadataContentParser(), new CacheMetadataService(indexingService, new FindDeletedMetadataIndexFinder()))
{
}
public CacheMetadataContent FindByHashAndUpdateRequestTime(UnityEngine.Hash128 hash, LoggingContext context)
{
if (HTTPManager.Logger.IsDiagnostic)
HTTPManager.Logger.Verbose(nameof(HTTPCacheDatabase), $"{nameof(FindByHashAndUpdateRequestTime)}({hash})", context);
if (!hash.isValid)
return default;
using var _ = new WriteLock(this.rwlock);
var (content, metadata) = FindContentAndMetadata(hash);
if (content != null)
{
content.RequestTime = DateTime.Now;
UpdateMetadataAndContent(metadata, content);
}
return content;
}
public bool TryAcquireWriteLock(Hash128 hash, Dictionary> headers, LoggingContext context)
{
if (HTTPManager.Logger.IsDiagnostic)
HTTPManager.Logger.Verbose(nameof(HTTPCacheDatabase), $"{nameof(TryAcquireWriteLock)}({hash}, {headers?.Count})", context);
if (!hash.isValid)
return false;
using var _ = new WriteLock(this.rwlock);
// FindMetadata filters out logically deleted entries, what we need here because we want to load it too.
var metadata = FindMetadata(hash);
CacheMetadataContent content = null;
if (metadata != null)
{
if (HTTPManager.Logger.IsDiagnostic)
HTTPManager.Logger.Verbose(nameof(HTTPCacheDatabase), $"{nameof(TryAcquireWriteLock)} - Metadata found: {metadata}", context);
if (metadata.Lock != LockTypes.Unlocked)
return false;
metadata.Lock = LockTypes.Write;
content = this.FromMetadata(metadata);
content.From(headers);
UpdateMetadataAndContent(metadata, content);
}
else
{
if (HTTPManager.Logger.IsDiagnostic)
HTTPManager.Logger.Verbose(nameof(HTTPCacheDatabase), $"{nameof(TryAcquireWriteLock)} - Creating new DB entry", context);
content = new CacheMetadataContent();
content.RequestTime = DateTime.Now;
content.From(headers);
(int filePos, int length) = this.DiskManager.Append(content);
metadata = this.MetadataService.Create(hash, content, filePos, length);
metadata.Lock = LockTypes.Write;
}
FlagDirty(1);
return true;
}
public bool Update(Hash128 hash, Dictionary> headers, LoggingContext context)
{
if (HTTPManager.Logger.IsDiagnostic)
HTTPManager.Logger.Verbose(nameof(HTTPCacheDatabase), $"{nameof(Update)}({hash}, {headers?.Count})", context);
if (!hash.isValid)
return false;
using var _ = new WriteLock(this.rwlock);
var (content, metadata) = FindContentAndMetadata(hash);
if (content != null)
{
content.From(headers);
content.ResponseTime = DateTime.Now;
UpdateMetadataAndContent(metadata, content);
}
return content != null;
}
public void ReleaseWriteLock(Hash128 hash, ulong length, LoggingContext context)
{
if (HTTPManager.Logger.IsDiagnostic)
HTTPManager.Logger.Verbose(nameof(HTTPCacheDatabase), $"{nameof(ReleaseWriteLock)}({hash}, {length:N0})", context);
if (!hash.isValid)
return;
using var _ = new WriteLock(this.rwlock);
var (content, metadata) = FindContentAndMetadata(hash);
if (content == null)
{
HTTPManager.Logger.Warning(nameof(HTTPCacheDatabase), $"{nameof(ReleaseWriteLock)} - Couldn't find content!", context);
return;
}
if (metadata.Lock != LockTypes.Write)
HTTPManager.Logger.Error(nameof(HTTPCacheDatabase), $"{nameof(ReleaseWriteLock)} - Is NOT Write Locked! {metadata}", context);
metadata.Lock = LockTypes.Unlocked;
if (content != null)
{
metadata.ContentLength = length;
var now = DateTime.Now;
metadata.LastAccessTime = now;
content.ResponseTime = now;
UpdateMetadataAndContent(metadata, content);
}
FlagDirty(1);
}
public bool TryAcquireReadLock(Hash128 hash, LoggingContext context)
{
if (HTTPManager.Logger.IsDiagnostic)
HTTPManager.Logger.Verbose(nameof(HTTPCacheDatabase), $"{nameof(TryAcquireReadLock)}({hash})", context);
if (!hash.isValid)
return false;
using var _ = new WriteLock(this.rwlock);
var metadata = FindMetadata(hash);
if (metadata == null)
return false;
if (metadata.Lock == LockTypes.Write)
return false;
metadata.Lock = LockTypes.Read;
// we are behind a write lock, it's safe to increment it like this
metadata.ReadLockCount++;
if (HTTPManager.Logger.IsDiagnostic)
HTTPManager.Logger.Verbose(nameof(HTTPCacheDatabase), $"{nameof(TryAcquireReadLock)} - {metadata}", context);
return true;
}
public void ReleaseReadLock(Hash128 hash, LoggingContext context)
{
if (HTTPManager.Logger.IsDiagnostic)
HTTPManager.Logger.Verbose(nameof(HTTPCacheDatabase), $"{nameof(ReleaseReadLock)}({hash})", context);
if (!hash.isValid)
return;
using var _ = new WriteLock(this.rwlock);
var metadata = FindMetadata(hash);
if (metadata == null)
{
HTTPManager.Logger.Warning(nameof(HTTPCacheDatabase), $"{nameof(ReleaseReadLock)} - Couldn't find metadata!", context);
return;
}
if (metadata.Lock != LockTypes.Read)
HTTPManager.Logger.Warning(nameof(HTTPCacheDatabase), $"{nameof(ReleaseReadLock)} - Is NOT Locked!", context);
if (metadata.ReadLockCount == 0)
HTTPManager.Logger.Error(nameof(HTTPCacheDatabase), $"{nameof(ReleaseReadLock)} - ReadLockCount already zero!", context);
if (--metadata.ReadLockCount == 0)
metadata.Lock = LockTypes.Unlocked;
if (HTTPManager.Logger.IsDiagnostic)
HTTPManager.Logger.Verbose(nameof(HTTPCacheDatabase), $"{nameof(ReleaseReadLock)} - {metadata}", context);
}
internal ulong Delete(Hash128 hash, LoggingContext context)
{
if (HTTPManager.Logger.IsDiagnostic)
HTTPManager.Logger.Verbose(nameof(HTTPCacheDatabase), $"{nameof(Delete)}({hash})", context);
if (!hash.isValid)
return 0;
using var _ = new WriteLock(this.rwlock);
// Don't use FindMetadata, because it would return null for a logically deleted metadata
// so DeleteMetadata wouldn't be called!
var byHash = this.IndexingService.FindByHash(hash);
if (byHash == null || byHash.Count == 0)
return 0;
var metadata = this.MetadataService.Metadatas[byHash[0]];
if (metadata == null)
return 0;
var contentLength = metadata.ContentLength;
base.DeleteMetadata(metadata);
return contentLength;
}
public void EnterWriteLock(LoggingContext context)
{
if (HTTPManager.Logger.IsDiagnostic)
HTTPManager.Logger.Verbose(nameof(HTTPCacheDatabase), $"{nameof(EnterWriteLock)}()", context);
this.rwlock.EnterWriteLock();
}
public void ExitWriteLock(LoggingContext context)
{
if (HTTPManager.Logger.IsDiagnostic)
HTTPManager.Logger.Verbose(nameof(HTTPCacheDatabase), $"{nameof(ExitWriteLock)}()", context);
this.rwlock.ExitWriteLock();
}
public void UpdateLastAccessTime(Hash128 hash, LoggingContext context)
{
if (HTTPManager.Logger.IsDiagnostic)
HTTPManager.Logger.Verbose(nameof(HTTPCacheDatabase), $"{nameof(UpdateLastAccessTime)}({hash})", context);
if (!hash.isValid)
return;
using var _ = new WriteLock(this.rwlock);
var metadata = FindMetadata(hash);
if (metadata != null)
{
metadata.LastAccessTime = DateTime.Now;
FlagDirty(1);
}
}
private CacheMetadata FindMetadata(Hash128 hash)
{
var byHash = this.IndexingService.FindByHash(hash);
if (byHash == null || byHash.Count == 0)
return null;
var metadata = this.MetadataService.Metadatas[byHash[0]];
if (metadata != null && metadata.IsDeleted)
return null;
return metadata;
}
public (CacheMetadataContent, CacheMetadata) FindContentAndMetadataLocked(Hash128 hash)
{
using var _ = new WriteLock(this.rwlock);
return FindContentAndMetadata(hash);
}
private (CacheMetadataContent, CacheMetadata) FindContentAndMetadata(Hash128 hash)
{
var byHash = this.IndexingService.FindByHash(hash);
if (byHash == null || byHash.Count == 0)
return (null, null);
var metadata = this.MetadataService.Metadatas[byHash[0]];
if (metadata != null && metadata.IsDeleted)
return (null, null);
var content = this.FromMetadataIndex(metadata.Index);
return (content, metadata);
}
private void UpdateMetadataAndContent(Metadata metadata, CacheMetadataContent content)
{
this.DiskManager.SaveChanged(metadata, content);
FlagDirty(1);
}
}
}