Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 93 additions & 13 deletions CollapseLauncher/Classes/CachesManagement/Honkai/Check.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
using CollapseLauncher.Helper;
using CollapseLauncher.Helper.StreamUtility;
using CollapseLauncher.Interfaces;
using Hi3Helper.Data;
using Hi3Helper.EncTool.Parser.AssetMetadata;
using Hi3Helper.Log;
using System;
using System.Buffers;
Expand Down Expand Up @@ -35,8 +37,7 @@ private async Task<List<CacheAsset>> Check(List<CacheAsset> assetIndex, Cancella
Directory.CreateDirectory(GamePath!);
}

// Check for unused files
CheckUnusedAssets(assetIndex, returnAsset);
HashSet<string> exceptionFromUnusedCategory = [];

// Do check in parallelization.
await Parallel.ForEachAsync(assetIndex!, new ParallelOptions
Expand All @@ -45,8 +46,11 @@ private async Task<List<CacheAsset>> Check(List<CacheAsset> assetIndex, Cancella
CancellationToken = token
}, async (asset, localToken) =>
{
await CheckAsset(asset, returnAsset, localToken);
await CheckAsset(asset, returnAsset, exceptionFromUnusedCategory, localToken);
});

// Check for unused files
CheckUnusedAssets(assetIndex, returnAsset, exceptionFromUnusedCategory);
}
catch (AggregateException ex)
{
Expand All @@ -57,10 +61,10 @@ private async Task<List<CacheAsset>> Check(List<CacheAsset> assetIndex, Cancella
return returnAsset;
}

private void CheckUnusedAssets(List<CacheAsset> assetIndex, List<CacheAsset> returnAsset)
private void CheckUnusedAssets(List<CacheAsset> assetIndex, List<CacheAsset> returnAsset, HashSet<string> exceptionFromUnusedCategory)
{
// Directory info and if the directory doesn't exist, return
DirectoryInfo directoryInfo = new DirectoryInfo(GamePath);
DirectoryInfo directoryInfo = new(GamePath);
if (!directoryInfo.Exists)
{
return;
Expand All @@ -82,10 +86,10 @@ private void CheckUnusedAssets(List<CacheAsset> assetIndex, List<CacheAsset> ret
foreach (FileInfo fileInfo in directoryInfo.EnumerateFiles("*", SearchOption.AllDirectories)
.EnumerateNoReadOnly())
{
ReadOnlySpan<char> filePath = fileInfo.FullName;

if (filePath.ContainsAny(unusedSearchValues)
|| assetIndex.Exists(x => x.ConcatPath == fileInfo.FullName))
bool excludedFromList = exceptionFromUnusedCategory.Contains(fileInfo.FullName);
if (excludedFromList ||
fileInfo.FullName.AsSpan().ContainsAny(unusedSearchValues) ||
assetIndex.Exists(x => x.ConcatPath == fileInfo.FullName))
{
continue;
}
Expand All @@ -94,7 +98,7 @@ private void CheckUnusedAssets(List<CacheAsset> assetIndex, List<CacheAsset> ret
ProgressAllCountFound++;

// Add asset to the returnAsset
CacheAsset asset = new CacheAsset
CacheAsset asset = new()
{
BasePath = Path.GetDirectoryName(fileInfo.FullName),
N = Path.GetFileName(fileInfo.FullName),
Expand All @@ -121,13 +125,19 @@ private void CheckUnusedAssets(List<CacheAsset> assetIndex, List<CacheAsset> ret
}
}

private async ValueTask CheckAsset(CacheAsset asset, List<CacheAsset> returnAsset, CancellationToken token)
private async ValueTask CheckAsset(CacheAsset asset, List<CacheAsset> returnAsset, HashSet<string> exceptionFromUnusedCategory, CancellationToken token)
{
// Increment the count and update the status
Interlocked.Add(ref ProgressAllCountCurrent, 1);
Status.ActivityStatus = string.Format(Locale.Current.Lang?._CachesPage?.CachesStatusChecking ?? "", asset!.DataType, asset.N);
Status.ActivityAll = string.Format(Locale.Current.Lang?._CachesPage?.CachesTotalStatusChecking ?? "", ProgressAllCountCurrent, ProgressAllCountTotal);

// Check for patch ifnormation first.
if (!UseFastMethod && await IsPatchExist(asset, returnAsset, exceptionFromUnusedCategory, token))
{
return;
}

// Assign the file info.
FileInfo fileInfo = new FileInfo(asset.ConcatPath).EnsureNoReadOnly(out bool isExist);

Expand Down Expand Up @@ -163,15 +173,85 @@ private async ValueTask CheckAsset(CacheAsset asset, List<CacheAsset> returnAsse
}
}

private async ValueTask<bool> IsPatchExist(
CacheAsset asset,
List<CacheAsset> returnAsset,
HashSet<string> exceptionFromUnusedCategory,
CancellationToken token)
{
if (asset.PatchInfo is not { } patchInfoDict)
return false;

string assetDir = Path.Combine(GetAssetBasePathByType(asset.DataType), asset.DataType.ToString().ToLowerInvariant());
byte[] hashArrayBuffer = ArrayPool<byte>.Shared.Rent(20);

try
{
foreach ((string oldFileName, CachePatchInfo patchInfo) in patchInfoDict)
{
string combinedOldPath = Path.Combine(assetDir, $"{oldFileName}.unity3d");
string combinedPatchPath = Path.Combine(assetDir, patchInfo.PatchName);
ConverterTool.NormalizePathInplaceNoTrim(combinedOldPath);

FileInfo fileInfo = new(combinedOldPath);

if (!fileInfo.Exists)
continue;

// If above passes, then run the CRC check
await using FileStream oldFileStream = await NaivelyOpenFileStreamAsync(fileInfo, FileMode.Open, FileAccess.Read, FileShare.Read);
// Calculate the asset CRC (SHA1)
byte[] hashArray = await GetCryptoHashAsync<HMACSHA1>(oldFileStream, GameSalt, true, true, token);

// If the asset CRC doesn't match, then add the file to asset index.
ReadOnlySpan<byte> hashSpan = GetHashFromName(hashArrayBuffer, oldFileName);
if (!IsArrayMatch(hashSpan, hashArray))
{
continue;
}

// Clear patch info and re-add single patch info.
patchInfoDict.Clear();
patchInfoDict.Add(oldFileName, patchInfo);
exceptionFromUnusedCategory.Add(combinedOldPath);
exceptionFromUnusedCategory.Add(combinedPatchPath);
Comment on lines +216 to +217

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bug: HashSet<string> exceptionFromUnusedCategory is mutated concurrently without locking in Parallel.ForEachAsync, which can corrupt internal state or drop excluded paths.
Severity: MEDIUM

Suggested Fix

Protect exceptionFromUnusedCategory.Add with a lock or replace the HashSet<string> with a thread-safe collection such as ConcurrentBag<string> or ConcurrentDictionary<string, byte>.

Prompt for AI Agent
Review the code at the location below. A potential bug has been identified by an AI
agent. Verify if this is a real issue. If it is, propose a fix; if not, explain why it's
not valid.

Location: CollapseLauncher/Classes/CachesManagement/Honkai/Check.cs#L216-L217

Potential issue: During asset checking, `CheckAsset` is executed concurrently across
multiple threads in `Parallel.ForEachAsync`. Inside `CheckAsset`, `IsPatchExist` calls
`Add` on the shared `exceptionFromUnusedCategory` `HashSet<string>` without
synchronization. Because `HashSet<T>` is not thread-safe for concurrent writes, racing
operations can corrupt internal collection state or drop added paths, potentially
causing required patch or base files to be incorrectly flagged as unused and deleted by
`CheckUnusedAssets`.

Did we get this right? 👍 / 👎 to inform future reviews.


AddGenericCheckAsset(asset, CacheAssetStatus.Patch, returnAsset, hashArray, asset.CRCArray);
return true;
}
}
finally
{
ArrayPool<byte>.Shared.Return(hashArrayBuffer);
}

// Unassign patch info if none is found.
asset.PatchInfo = null;
return false;

static ReadOnlySpan<byte> GetHashFromName(Span<byte> buffer, ReadOnlySpan<char> name)
{
ReadOnlySpan<char> hexString = name.GetSplit(1, '_');
if (hexString.IsEmpty || HexTool.TryHexToBytesUnsafe(hexString, buffer, out _, out int length) != OperationStatus.Done)
return ReadOnlySpan<byte>.Empty;

return buffer[..length];
}
}

private void AddGenericCheckAsset(CacheAsset asset, CacheAssetStatus assetStatus, List<CacheAsset> returnAsset, byte[] localCrc, byte[] remoteCrc)
{
long size = asset.PatchInfo?.Values.FirstOrDefault() is { } patchInfo
? patchInfo.PatchFileSize
: asset.CS;

// Increment the count and total size
lock (this)
{
// Set Indetermined status as false
Status.IsProgressAllIndetermined = false;
ProgressAllCountFound++;
ProgressAllSizeFound += asset!.CS;
ProgressAllSizeFound += size;
}

// Add file into asset index
Expand All @@ -188,7 +268,7 @@ private void AddGenericCheckAsset(CacheAsset asset, CacheAssetStatus assetStatus
Path.GetFileName(asset.N) ?? "",
asset.DataType,
Path.GetDirectoryName(asset.N),
asset.CS,
size,
localCrc,
remoteCrc
))
Expand Down
70 changes: 56 additions & 14 deletions CollapseLauncher/Classes/CachesManagement/Honkai/Fetch.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using CollapseLauncher.RepairManagement;
using Hi3Helper.EncTool;
using Hi3Helper.EncTool.Enc;
using Hi3Helper.EncTool.Parser.AssetMetadata;
using Hi3Helper.EncTool.Parser.KianaDispatch;
using Hi3Helper.Log;
using Hi3Helper.UABT;
Expand All @@ -22,6 +23,7 @@
// ReSharper disable StringLiteralTypo
#pragma warning disable IDE0130

#nullable enable
namespace CollapseLauncher
{
internal partial class HonkaiCache
Expand All @@ -46,6 +48,9 @@ private async Task<List<CacheAsset>> Fetch(CancellationToken token)
// Build _gameRepoURL from loading Dispatcher and Gateway
await BuildGameRepoURL(httpClientNew, token);

// Iterate patch for "Data" type
CachePatchManifest dataTypePatch = await FetchPatchByType(httpClientNew, CacheAssetType.Data, token);

// Iterate type and do fetch
await Parallel.ForEachAsync(
Enum.GetValues<CacheAssetType>(),
Expand Down Expand Up @@ -81,16 +86,19 @@ await Parallel.ForEachAsync(
}
});

// Merge patch info
MergeAssetPatchInfo(dataTypePatch, returnAsset);

// Return asset index
return returnAsset;
}

private async Task BuildGameRepoURL(HttpClient client, CancellationToken token)
{
KianaDispatch dispatch = null;
Exception lastException = null;
KianaDispatch? dispatch = null;
Exception? lastException = null;

PresetConfig presetConfig = GameVersionManager!.GamePreset!;
PresetConfig presetConfig = GameVersionManager.GamePreset;
int[] version = GameVersion.VersionArray;
string key = presetConfig.DispatcherKey ?? "";

Expand Down Expand Up @@ -127,7 +135,7 @@ private async Task BuildGameRepoURL(HttpClient client, CancellationToken token)

// Get gatewayURl and fetch the gateway
GameGateway = await KianaDispatch.GetGameServerAsync(client,
dispatch!,
dispatch,
presetConfig.GameGatewayDefault!,
key,
version,
Expand Down Expand Up @@ -164,6 +172,45 @@ private async Task BuildGameRepoURL(HttpClient client, CancellationToken token)
return returnValue;
}

private async Task<CachePatchManifest> FetchPatchByType(HttpClient client, CacheAssetType type, CancellationToken token)
{
Status.ActivityStatus = string.Format(Locale.Current.Lang?._CachesPage?.CachesStatusFetchingType ?? "", $"{type}Patch");
Status.IsProgressAllIndetermined = true;
Status.IsIncludePerFileIndicator = false;
UpdateStatus();

// Get the asset index properties
string baseUrl = string.Format(BuildAssetBundleURL(GameGateway), type.ToString().ToLowerInvariant());
string fileUrl = string.Format(baseUrl.CombineURLFromString("patch/patchconfig.xmf?t={0}"), DateTime.Now.Ticks);

CDNCacheResult result = await client.TryGetCachedStreamFrom(fileUrl, token: token);
if (!result.IsSuccessStatusCode)
{
Logger.WriteLine($"Cache Patch information file is unavailable due to error: {result.StatusCode} ({(int)result.StatusCode}). URL: {fileUrl}",
LogType.Warning,
true);
return CachePatchManifest.Empty;
}

await using Stream resultStream = result.Stream;
return new CachePatchManifest(resultStream);
}

private static void MergeAssetPatchInfo(CachePatchManifest manifest, List<CacheAsset> assets)
{
Dictionary<string, Dictionary<string, CachePatchInfo>>.AlternateLookup<ReadOnlySpan<char>> lookup = manifest.GetAlternateLookup<ReadOnlySpan<char>>();
foreach (CacheAsset asset in assets.Where(x => x.DataType == CacheAssetType.Data))
{
ReadOnlySpan<char> fileNameNew = Path.GetFileNameWithoutExtension(asset.ConcatNRemote);
if (!lookup.TryGetValue(fileNameNew, out Dictionary<string, CachePatchInfo>? cachePatchInfoDict))
{
continue;
}

asset.PatchInfo = cachePatchInfoDict;
}
}

private IEnumerable<CacheAsset> EnumerateCacheTextAsset(CacheAssetType type, IEnumerable<string> enumerator, string baseUrl)
{
// Set isFirst flag as true if type is Data and
Expand Down Expand Up @@ -199,7 +246,7 @@ private IEnumerable<CacheAsset> EnumerateCacheTextAsset(CacheAssetType type, IEn
continue;
}

CacheAsset content;
CacheAsset? content;
try
{
// Deserialize the line and set the type
Expand Down Expand Up @@ -231,18 +278,13 @@ private IEnumerable<CacheAsset> EnumerateCacheTextAsset(CacheAssetType type, IEn
}
}

private async ValueTask<ValueTuple<int, long>> BuildAssetIndex(CacheAssetType type, string baseURL, Stream stream,
List<CacheAsset> assetIndex, CancellationToken token)
private async ValueTask<ValueTuple<int, long>> BuildAssetIndex(CacheAssetType type, string baseURL, Stream stream, List<CacheAsset> assetIndex, CancellationToken token)
{
int count = 0;
long size = 0;

// Set isFirst flag as true if type is Data and
// also convert type as lowered string.

// Unused as of Aug 4th 2024, bonk @bagusnl if not true
// bool isFirst = type == CacheAssetType.Data;
// bool isNeedReadLuckyNumber = type == CacheAssetType.Data;

// Parse asset index file from UABT
BundleFile bundleFile = new(stream);
Expand Down Expand Up @@ -306,7 +348,7 @@ await Parallel.ForEachAsync(EnumerateCacheTextAsset(type, dataTextAsset.GetStrin
private static byte[] GetAssetIndexSalt(string data)
{
// Get the salt from the string and return as byte[]
byte[] key;
byte[]? key;
if (DataCooker.IsServeV3Data(MetadataHelper.CurrentMasterKey?.Key))
{
DataCooker.GetServeV3DataSize(MetadataHelper.CurrentMasterKey?.Key, out long keyCompSize,
Expand All @@ -325,13 +367,13 @@ private static byte[] GetAssetIndexSalt(string data)
key,
saltBytes,
out _,
out Exception exception))
out Exception? exception))
throw exception;

return saltBytes;
}

private string GetAssetBasePathByType(CacheAssetType type) => Path.Combine(GamePath!, type == CacheAssetType.Data ? "Data" : "Resources");
private string GetAssetBasePathByType(CacheAssetType type) => Path.Combine(GamePath, type == CacheAssetType.Data ? "Data" : "Resources");

private static bool IsValidRegionFile(string input, string lang)
{
Expand Down
Loading
Loading